Add the NetworkQualityEstimator class that is notified

when net stack receives data.

Expose PeakKbps and FastestRTT Statistics through
GetEstimate() API which may be used by clients like Lo-Fi.

PeakKbps and FastestRTT computation is now
based on prefilter bytes which correctly
represent the amount of data that was transferred
over the wire.

BUG=472678, 478162

Review URL: https://codereview.chromium.org/1061583002

Cr-Commit-Position: refs/heads/master@{#331192}

chrome/browser/io_thread.cc

@@ -48,6 +48,7 @@
 #include "content/public/browser/cookie_store_factory.h"
 #include "net/base/host_mapping_rules.h"
 #include "net/base/net_util.h"
+#include "net/base/network_quality_estimator.h"
 #include "net/base/sdch_manager.h"
 #include "net/cert/cert_policy_enforcer.h"
 #include "net/cert/cert_verifier.h"
@@ -275,6 +276,8 @@ ConstructSystemRequestContext(IOThread::Globals* globals,
   context->set_network_delegate(globals->system_network_delegate.get());
   context->set_http_user_agent_settings(
       globals->http_user_agent_settings.get());
+  context->set_network_quality_estimator(
+      globals->network_quality_estimator.get());
   return context;
 }
 
@@ -670,6 +673,9 @@ void IOThread::InitAsync() {
           "466432 IOThread::InitAsync::CreateGlobalHostResolver"));
   globals_->system_network_delegate = chrome_network_delegate.Pass();
   globals_->host_resolver = CreateGlobalHostResolver(net_log_);
+
+  globals_->network_quality_estimator.reset(new net::NetworkQualityEstimator());
+
   // TODO(erikchen): Remove ScopedTracker below once http://crbug.com/466432
   // is fixed.
   tracked_objects::ScopedTracker tracking_profile5(

chrome/browser/io_thread.h

@@ -56,6 +56,7 @@ class HttpServerProperties;
 class HttpTransactionFactory;
 class HttpUserAgentSettings;
 class NetworkDelegate;
+class NetworkQualityEstimator;
 class ProxyConfigService;
 class ProxyService;
 class SSLConfigService;
@@ -161,6 +162,7 @@ class IOThread : public content::BrowserThreadDelegate {
 #endif
     scoped_ptr<net::HostMappingRules> host_mapping_rules;
     scoped_ptr<net::HttpUserAgentSettings> http_user_agent_settings;
+    scoped_ptr<net::NetworkQualityEstimator> network_quality_estimator;
     bool ignore_certificate_errors;
     bool use_stale_while_revalidate;
     uint16 testing_fixed_http_port;

chrome/browser/profiles/profile_impl_io_data.cc

@@ -571,6 +571,8 @@ void ProfileImplIOData::InitializeInternal(
       main_context->network_delegate(),
       ftp_factory_.get());
   main_context->set_job_factory(main_job_factory_.get());
+  main_context->set_network_quality_estimator(
+      io_thread_globals->network_quality_estimator.get());
 
 #if defined(ENABLE_EXTENSIONS)
   InitializeExtensionsRequestContext(profile_params);

net/base/network_quality.h

@@ -0,0 +1,51 @@
+// Copyright 2015 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef NET_BASE_NETWORK_QUALITY_H_
+#define NET_BASE_NETWORK_QUALITY_H_
+
+#include <stdint.h>
+
+#include "base/time/time.h"
+
+namespace net {
+
+// API that is used to report the current network quality as estimated by the
+// NetworkQualityEstimator.
+struct NET_EXPORT_PRIVATE NetworkQuality {
+  NetworkQuality(const base::TimeDelta& fastest_rtt,
+                 double fastest_rtt_confidence,
+                 uint64_t peak_throughput_kbps,
+                 double peak_throughput_kbps_confidence)
+      : fastest_rtt(fastest_rtt),
+        fastest_rtt_confidence(fastest_rtt_confidence),
+        peak_throughput_kbps(peak_throughput_kbps),
+        peak_throughput_kbps_confidence(peak_throughput_kbps_confidence) {}
+
+  ~NetworkQuality() {}
+
+  // The fastest round trip time observed for the current connection.
+  const base::TimeDelta fastest_rtt;
+
+  // Confidence of the |fastest_rtt| estimate. Value lies between 0.0 and 1.0
+  // with 0.0 being no confidence and 1.0 implying that estimates are same as
+  // ground truth.
+  // TODO(tbansal): Define units so values intermediate between 0.0 and 1.0 are
+  // meaningful.
+  const double fastest_rtt_confidence;
+
+  // Peak throughput in Kbps observed for the current connection.
+  const uint64_t peak_throughput_kbps;
+
+  // Confidence of the |peak_throughput_kbps| estimate. Value lies between 0.0
+  // and 1.0 with 0.0 being no confidence and 1.0 implying that estimates are
+  // same as ground truth.
+  // TODO(tbansal): Define units so values intermediate between 0.0 and 1.0 are
+  // meaningful.
+  const double peak_throughput_kbps_confidence;
+};
+
+}  // namespace net
+
+#endif  // NET_BASE_NETWORK_QUALITY_H_

net/base/network_quality_estimator.cc

@@ -0,0 +1,178 @@
+// Copyright 2015 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "net/base/network_quality_estimator.h"
+
+#include <string>
+
+#include "base/metrics/histogram.h"
+#include "net/base/net_util.h"
+#include "net/base/network_quality.h"
+#include "net/url_request/url_request.h"
+#include "url/gurl.h"
+
+namespace net {
+
+NetworkQualityEstimator::NetworkQualityEstimator()
+    : NetworkQualityEstimator(false) {
+}
+
+NetworkQualityEstimator::NetworkQualityEstimator(
+    bool allow_local_host_requests_for_tests)
+    : allow_localhost_requests_(allow_local_host_requests_for_tests),
+      last_connection_change_(base::TimeTicks::Now()),
+      current_connection_type_(NetworkChangeNotifier::GetConnectionType()),
+      bytes_read_since_last_connection_change_(false),
+      peak_kbps_since_last_connection_change_(0) {
+  static_assert(kMinRequestDurationMicroseconds > 0,
+                "Minimum request duration must be > 0");
+  NetworkChangeNotifier::AddConnectionTypeObserver(this);
+}
+
+NetworkQualityEstimator::~NetworkQualityEstimator() {
+  DCHECK(thread_checker_.CalledOnValidThread());
+  NetworkChangeNotifier::RemoveConnectionTypeObserver(this);
+}
+
+void NetworkQualityEstimator::NotifyDataReceived(const URLRequest& request,
+                                                 int64_t prefilter_bytes_read) {
+  DCHECK(thread_checker_.CalledOnValidThread());
+  DCHECK_GT(prefilter_bytes_read, 0);
+
+  if (!request.url().is_valid() ||
+      (!allow_localhost_requests_ && IsLocalhost(request.url().host())) ||
+      !request.url().SchemeIsHTTPOrHTTPS() ||
+      // Verify that response headers are received, so it can be ensured that
+      // response is not cached.
+      request.response_info().response_time.is_null() || request.was_cached())
+    return;
+
+  base::TimeTicks now = base::TimeTicks::Now();
+  base::TimeDelta request_duration = now - request.creation_time();
+  DCHECK_GE(request_duration, base::TimeDelta());
+  if (!bytes_read_since_last_connection_change_)
+    fastest_RTT_since_last_connection_change_ = request_duration;
+
+  bytes_read_since_last_connection_change_ = true;
+  if (request_duration < fastest_RTT_since_last_connection_change_)
+    fastest_RTT_since_last_connection_change_ = request_duration;
+
+  // Ignore tiny transfers which will not produce accurate rates.
+  // Ignore short duration transfers.
+  if (prefilter_bytes_read >= kMinTransferSizeInBytes &&
+      request_duration >=
+          base::TimeDelta::FromMicroseconds(kMinRequestDurationMicroseconds) &&
+      request.creation_time() > last_connection_change_) {
+    uint64_t kbps = static_cast<uint64_t>(prefilter_bytes_read * 8 * 1000 /
+                                          request_duration.InMicroseconds());
+    if (kbps > peak_kbps_since_last_connection_change_)
+      peak_kbps_since_last_connection_change_ = kbps;
+  }
+}
+
+void NetworkQualityEstimator::OnConnectionTypeChanged(
+    NetworkChangeNotifier::ConnectionType type) {
+  DCHECK(thread_checker_.CalledOnValidThread());
+  if (bytes_read_since_last_connection_change_) {
+    switch (current_connection_type_) {
+      case NetworkChangeNotifier::CONNECTION_UNKNOWN:
+        UMA_HISTOGRAM_TIMES("NQE.FastestRTT.Unknown",
+                            fastest_RTT_since_last_connection_change_);
+        break;
+      case NetworkChangeNotifier::CONNECTION_ETHERNET:
+        UMA_HISTOGRAM_TIMES("NQE.FastestRTT.Ethernet",
+                            fastest_RTT_since_last_connection_change_);
+        break;
+      case NetworkChangeNotifier::CONNECTION_WIFI:
+        UMA_HISTOGRAM_TIMES("NQE.FastestRTT.Wifi",
+                            fastest_RTT_since_last_connection_change_);
+        break;
+      case NetworkChangeNotifier::CONNECTION_2G:
+        UMA_HISTOGRAM_TIMES("NQE.FastestRTT.2G",
+                            fastest_RTT_since_last_connection_change_);
+        break;
+      case NetworkChangeNotifier::CONNECTION_3G:
+        UMA_HISTOGRAM_TIMES("NQE.FastestRTT.3G",
+                            fastest_RTT_since_last_connection_change_);
+        break;
+      case NetworkChangeNotifier::CONNECTION_4G:
+        UMA_HISTOGRAM_TIMES("NQE.FastestRTT.4G",
+                            fastest_RTT_since_last_connection_change_);
+        break;
+      case NetworkChangeNotifier::CONNECTION_NONE:
+        UMA_HISTOGRAM_TIMES("NQE.FastestRTT.None",
+                            fastest_RTT_since_last_connection_change_);
+        break;
+      case NetworkChangeNotifier::CONNECTION_BLUETOOTH:
+        UMA_HISTOGRAM_TIMES("NQE.FastestRTT.Bluetooth",
+                            fastest_RTT_since_last_connection_change_);
+        break;
+      default:
+        NOTREACHED();
+        break;
+    }
+  }
+
+  if (peak_kbps_since_last_connection_change_) {
+    switch (current_connection_type_) {
+      case NetworkChangeNotifier::CONNECTION_UNKNOWN:
+        UMA_HISTOGRAM_COUNTS("NQE.PeakKbps.Unknown",
+                             peak_kbps_since_last_connection_change_);
+        break;
+      case NetworkChangeNotifier::CONNECTION_ETHERNET:
+        UMA_HISTOGRAM_COUNTS("NQE.PeakKbps.Ethernet",
+                             peak_kbps_since_last_connection_change_);
+        break;
+      case NetworkChangeNotifier::CONNECTION_WIFI:
+        UMA_HISTOGRAM_COUNTS("NQE.PeakKbps.Wifi",
+                             peak_kbps_since_last_connection_change_);
+        break;
+      case NetworkChangeNotifier::CONNECTION_2G:
+        UMA_HISTOGRAM_COUNTS("NQE.PeakKbps.2G",
+                             peak_kbps_since_last_connection_change_);
+        break;
+      case NetworkChangeNotifier::CONNECTION_3G:
+        UMA_HISTOGRAM_COUNTS("NQE.PeakKbps.3G",
+                             peak_kbps_since_last_connection_change_);
+        break;
+      case NetworkChangeNotifier::CONNECTION_4G:
+        UMA_HISTOGRAM_COUNTS("NQE.PeakKbps.4G",
+                             peak_kbps_since_last_connection_change_);
+        break;
+      case NetworkChangeNotifier::CONNECTION_NONE:
+        UMA_HISTOGRAM_COUNTS("NQE.PeakKbps.None",
+                             peak_kbps_since_last_connection_change_);
+        break;
+      case NetworkChangeNotifier::CONNECTION_BLUETOOTH:
+        UMA_HISTOGRAM_COUNTS("NQE.PeakKbps.Bluetooth",
+                             peak_kbps_since_last_connection_change_);
+        break;
+      default:
+        NOTREACHED();
+        break;
+    }
+  }
+
+  last_connection_change_ = base::TimeTicks::Now();
+  bytes_read_since_last_connection_change_ = false;
+  peak_kbps_since_last_connection_change_ = 0;
+  current_connection_type_ = type;
+}
+
+NetworkQuality NetworkQualityEstimator::GetEstimate() const {
+  DCHECK(thread_checker_.CalledOnValidThread());
+
+  if (!bytes_read_since_last_connection_change_) {
+    return NetworkQuality(fastest_RTT_since_last_connection_change_, 0,
+                          peak_kbps_since_last_connection_change_, 0);
+  }
+  if (!peak_kbps_since_last_connection_change_) {
+    return NetworkQuality(fastest_RTT_since_last_connection_change_, 0.1,
+                          peak_kbps_since_last_connection_change_, 0);
+  }
+  return NetworkQuality(fastest_RTT_since_last_connection_change_, 0.1,
+                        peak_kbps_since_last_connection_change_, 0.1);
+}
+
+}  // namespace net

net/base/network_quality_estimator.h

@@ -0,0 +1,103 @@
+// Copyright 2015 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef NET_BASE_NETWORK_QUALITY_ESTIMATOR_H_
+#define NET_BASE_NETWORK_QUALITY_ESTIMATOR_H_
+
+#include <stdint.h>
+
+#include "base/gtest_prod_util.h"
+#include "base/macros.h"
+#include "base/threading/thread_checker.h"
+#include "base/time/time.h"
+#include "net/base/network_change_notifier.h"
+
+namespace net {
+
+struct NetworkQuality;
+
+// NetworkQualityEstimator provides network quality estimates (quality of the
+// full paths to all origins that have been connected to).
+// The estimates are based on the observed organic traffic.
+// A NetworkQualityEstimator instance is attached to URLRequestContexts and
+// observes the traffic of URLRequests spawned from the URLRequestContexts.
+// A single instance of NQE can be attached to multiple URLRequestContexts,
+// thereby increasing the single NQE instance's accuracy by providing more
+// observed traffic characteristics.
+class NET_EXPORT_PRIVATE NetworkQualityEstimator
+    : public NetworkChangeNotifier::ConnectionTypeObserver {
+ public:
+  // Creates a new NetworkQualityEstimator.
+  NetworkQualityEstimator();
+
+  ~NetworkQualityEstimator() override;
+
+  // Returns an estimate of the current network quality.
+  NetworkQuality GetEstimate() const;
+
+  // Notifies NetworkQualityEstimator that a response has been received.
+  // |prefilter_bytes_read| is the count of the bytes received prior to
+  // applying filters (e.g. decompression, SDCH) from request creation time
+  // until now.
+  void NotifyDataReceived(const URLRequest& request,
+                          int64_t prefilter_bytes_read);
+
+ private:
+  FRIEND_TEST_ALL_PREFIXES(NetworkQualityEstimatorTest,
+                           TestPeakKbpsFastestRTTUpdates);
+  FRIEND_TEST_ALL_PREFIXES(URLRequestTestHTTP, NetworkQualityEstimator);
+
+  // Tiny transfer sizes may give inaccurate throughput results.
+  // Minimum size of the transfer over which the throughput is computed.
+  static const int kMinTransferSizeInBytes = 10000;
+
+  // Minimum duration (in microseconds) of the transfer over which the
+  // throughput is computed.
+  static const int kMinRequestDurationMicroseconds = 1000;
+
+  // Construct a NetworkQualityEstimator instance allowing for test
+  // configuration.
+  // Registers for network type change notifications so estimates can be kept
+  // network specific.
+  // |allow_local_host_requests_for_tests| should only be true when testing
+  // against local HTTP server and allows the requests to local host to be
+  // used for network quality estimation.
+  explicit NetworkQualityEstimator(bool allow_local_host_requests_for_tests);
+
+  // NetworkChangeNotifier::ConnectionTypeObserver implementation.
+  void OnConnectionTypeChanged(
+      NetworkChangeNotifier::ConnectionType type) override;
+
+  // Determines if the requests to local host can be used in estimating the
+  // network quality. Set to true only for tests.
+  const bool allow_localhost_requests_;
+
+  // Time when last connection change was observed.
+  base::TimeTicks last_connection_change_;
+
+  // Last value passed to |OnConnectionTypeChanged|. This indicates the
+  // current connection type.
+  NetworkChangeNotifier::ConnectionType current_connection_type_;
+
+  // Set if any network data has been received since last connectivity change.
+  bool bytes_read_since_last_connection_change_;
+
+  // Fastest round-trip-time (RTT) since last connectivity change. RTT measured
+  // from URLRequest creation until first byte received.
+  base::TimeDelta fastest_RTT_since_last_connection_change_;
+
+  // Rough measurement of downlink peak Kbps witnessed since last connectivity
+  // change. The accuracy is decreased by ignoring these factors:
+  // 1) Multiple URLRequests can occur concurrently.
+  // 2) The transfer time includes at least one RTT while no bytes are read.
+  uint64_t peak_kbps_since_last_connection_change_;
+
+  base::ThreadChecker thread_checker_;
+
+  DISALLOW_COPY_AND_ASSIGN(NetworkQualityEstimator);
+};
+
+}  // namespace net
+
+#endif  // NET_BASE_NETWORK_QUALITY_ESTIMATOR_H_