Implement URLRequestBackoffManager for managing Backoff headers

This CL implemented a URLRequestBackoffManager to manage
Backoff response headers. This new feature is off by default
in this CL.

Design doc:
https://docs.google.com/document/d/1aAxwXK7Vw3VigFd6MmrItbAIgMdKAf-XxXXbhWXdID0/edit?usp=sharing

BUG=486891

Review URL: https://codereview.chromium.org/1153093002

Cr-Commit-Position: refs/heads/master@{#339961}

chrome/app/generated_resources.grd

@@ -9764,6 +9764,15 @@ I don't think this site should be blocked!
         Extension requests to this URL have been temporarily throttled.
       </message>
 
+      <message name="IDS_ERRORPAGES_SUMMARY_TEMPORARY_BACKOFF" desc="Summary in the error page when we temporarily stop sending requests to a server because of a Backoff header received earlier.">
+        The server hosting the webpage might be overloaded or under maintainance.
+        In order to avoid generating too much traffic and making the situation worse,
+        requests to this URL has been temporarily disallowed.
+      </message>
+      <message name="IDS_ERRORPAGES_DETAILS_TEMPORARY_BACKOFF" desc="The error message displayed when we temporarily stop sending requests to a server in order to avoid overloading it.">
+        Requests to this URL have been temporarily throttled.
+      </message>
+
       <message name="IDS_ERRORPAGES_SUMMARY_BLOCKED" desc="Summary in the error page when an extension blocks a request.">
         A third-party extension has blocked access to this webpage.
       </message>

chrome/common/localized_error.cc

@@ -321,6 +321,13 @@ const LocalizedErrorMap net_error_options[] = {
    IDS_ERRORPAGES_DETAILS_SSL_VERSION_OR_CIPHER_MISMATCH,
    SUGGEST_NONE,
   },
+  {net::ERR_TEMPORARY_BACKOFF,
+   IDS_ERRORPAGES_TITLE_ACCESS_DENIED,
+   IDS_ERRORPAGES_HEADING_ACCESS_DENIED,
+   IDS_ERRORPAGES_SUMMARY_TEMPORARY_BACKOFF,
+   IDS_ERRORPAGES_DETAILS_TEMPORARY_BACKOFF,
+   SUGGEST_NONE,
+  },
 };
 
 // Special error page to be used in the case of navigating back to a page

net/base/net_error_list.h

@@ -642,6 +642,10 @@ NET_ERROR(PAC_SCRIPT_TERMINATED, -367)
 // https://tools.ietf.org/id/draft-ietf-httpbis-alt-svc-06.html#host_auth.
 NET_ERROR(ALTERNATIVE_CERT_NOT_VALID_FOR_ORIGIN, -368)
 
+// Request is throttled because of a Backoff header.
+// See: crbug.com/486891.
+NET_ERROR(TEMPORARY_BACKOFF, -369)
+
 // The cache does not have the requested entry.
 NET_ERROR(CACHE_MISS, -400)
 

net/net.gypi

@@ -1245,6 +1245,8 @@
       'url_request/url_request_throttler_entry_interface.h',
       'url_request/url_request_throttler_manager.cc',
       'url_request/url_request_throttler_manager.h',
+      'url_request/url_request_backoff_manager.cc',
+      'url_request/url_request_backoff_manager.h',
       'url_request/view_cache_helper.cc',
       'url_request/view_cache_helper.h',
       'url_request/websocket_handshake_userdata_key.cc',
@@ -1765,6 +1767,7 @@
       'url_request/sdch_dictionary_fetcher_unittest.cc',
       'url_request/url_fetcher_impl_unittest.cc',
       'url_request/url_fetcher_response_writer_unittest.cc',
+      'url_request/url_request_backoff_manager_unittest.cc',
       'url_request/url_request_context_builder_unittest.cc',
       'url_request/url_request_data_job_unittest.cc',
       'url_request/url_request_file_job_unittest.cc',

net/url_request/url_request_backoff_manager.cc

@@ -0,0 +1,144 @@
+// Copyright 2015 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "net/url_request/url_request_backoff_manager.h"
+
+#include "base/logging.h"
+#include "base/strings/string_number_conversions.h"
+#include "base/strings/string_piece.h"
+#include "base/strings/string_util.h"
+#include "net/http/http_response_headers.h"
+
+namespace net {
+
+const uint16 URLRequestBackoffManager::kMinimumBackoffInSeconds = 1;
+const uint16 URLRequestBackoffManager::kMaximumBackoffInSeconds = 50000;
+const uint16 URLRequestBackoffManager::kNewEntriesBetweenCollecting = 200;
+
+URLRequestBackoffManager::URLRequestBackoffManager()
+    : new_entries_since_last_gc_(0) {
+  url_id_replacements_.ClearPassword();
+  url_id_replacements_.ClearUsername();
+  url_id_replacements_.ClearQuery();
+  url_id_replacements_.ClearRef();
+
+  NetworkChangeNotifier::AddIPAddressObserver(this);
+  NetworkChangeNotifier::AddConnectionTypeObserver(this);
+}
+
+URLRequestBackoffManager::~URLRequestBackoffManager() {
+  NetworkChangeNotifier::RemoveIPAddressObserver(this);
+  NetworkChangeNotifier::RemoveConnectionTypeObserver(this);
+  for (UrlEntryMap::iterator it = url_entries_.begin();
+       it != url_entries_.end(); ++it) {
+    delete it->second;
+  }
+  url_entries_.clear();
+}
+
+void URLRequestBackoffManager::UpdateWithResponse(
+    const GURL& url,
+    HttpResponseHeaders* headers,
+    const base::Time& response_time) {
+  CalledOnValidThread();
+  base::TimeDelta result;
+  if (GetBackoffTime(headers, &result)) {
+    new_entries_since_last_gc_++;
+    std::string url_id = GetIdFromUrl(url);
+    url_entries_[url_id] =
+        new Entry(response_time + result, response_time + result * 1.1);
+    GarbageCollectEntriesIfNecessary();
+  }
+}
+
+bool URLRequestBackoffManager::ShouldRejectRequest(
+    const GURL& url,
+    const base::Time& request_time) {
+  CalledOnValidThread();
+  std::string url_id = GetIdFromUrl(url);
+  UrlEntryMap::iterator it = url_entries_.find(url_id);
+  if (it == url_entries_.end())
+    return false;
+  Entry* entry = it->second;
+  if (request_time < entry->throttled_time)
+    return true;
+  // Allow one request between throttled_time and release_time.
+  if (request_time >= entry->throttled_time &&
+      request_time < entry->release_time) {
+    if (entry->used)
+      return true;
+    entry->used = true;
+  }
+  return false;
+}
+
+void URLRequestBackoffManager::OnIPAddressChanged() {
+  OnNetworkChange();
+}
+
+void URLRequestBackoffManager::OnConnectionTypeChanged(
+    NetworkChangeNotifier::ConnectionType type) {
+  OnNetworkChange();
+}
+
+int URLRequestBackoffManager::GetNumberOfEntriesForTests() const {
+  return url_entries_.size();
+}
+
+void URLRequestBackoffManager::GarbageCollectEntriesIfNecessary() {
+  CalledOnValidThread();
+  if (new_entries_since_last_gc_ < kNewEntriesBetweenCollecting)
+    return;
+
+  new_entries_since_last_gc_ = 0;
+  UrlEntryMap::iterator it = url_entries_.begin();
+  while (it != url_entries_.end()) {
+    Entry* entry = it->second;
+    if (entry->IsOutDated()) {
+      url_entries_.erase(it++);
+      delete entry;
+    } else {
+      ++it;
+    }
+  }
+}
+
+bool URLRequestBackoffManager::GetBackoffTime(HttpResponseHeaders* headers,
+                                              base::TimeDelta* result) const {
+  base::StringPiece name("Backoff");
+  std::string value;
+  void* iter = NULL;
+  while (headers->EnumerateHeader(&iter, name, &value)) {
+    int64 seconds;
+    base::StringToInt64(value, &seconds);
+    if (seconds >= kMinimumBackoffInSeconds &&
+        seconds <= kMaximumBackoffInSeconds) {
+      *result = base::TimeDelta::FromSeconds(seconds);
+      return true;
+    }
+  }
+  return false;
+}
+
+std::string URLRequestBackoffManager::GetIdFromUrl(const GURL& url) const {
+  if (!url.is_valid())
+    return url.possibly_invalid_spec();
+
+  GURL id = url.ReplaceComponents(url_id_replacements_);
+  return base::StringToLowerASCII(id.spec()).c_str();
+}
+
+void URLRequestBackoffManager::OnNetworkChange() {
+  CalledOnValidThread();
+
+  new_entries_since_last_gc_ = 0;
+  // Remove all entries.
+  for (UrlEntryMap::iterator it = url_entries_.begin();
+       it != url_entries_.end(); ++it) {
+    delete it->second;
+  }
+  url_entries_.clear();
+}
+
+}  // namespace net

net/url_request/url_request_backoff_manager.h

@@ -0,0 +1,136 @@
+// Copyright 2015 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef NET_URL_REQUEST_URL_REQUEST_BACKOFF_MANAGER_H_
+#define NET_URL_REQUEST_URL_REQUEST_BACKOFF_MANAGER_H_
+
+#include <map>
+#include <string>
+
+#include "base/basictypes.h"
+#include "base/threading/non_thread_safe.h"
+#include "base/time/time.h"
+#include "net/base/net_export.h"
+#include "net/base/network_change_notifier.h"
+#include "url/gurl.h"
+
+namespace net {
+
+class HttpResponseHeaders;
+
+// Class that manages information on Backoff headers. URL requests for HTTPS
+// contents should update their URLs in this manager on each response.
+//
+// Design doc:
+// https://docs.google.com/document/d/1aAxwXK7Vw3VigFd6MmrItbAIgMdKAf-XxXXbhWXdID0/edit?usp=sharing
+//
+// URLRequestBackoffManager maintains a map of URL IDs to
+// URLRequestBackoffManager::Entry. It creates an entry when a request receives
+// a Backoff header, and does garbage collection from time to time in order to
+// clean out outdated entries. URL ID consists of lowercased scheme, host, port
+// and path. A newer request with the same ID will override the old entry.
+//
+// Note that the class does not implement logic to retry a request at random
+// with uniform distribution.
+// TODO(xunjieli): Expose release time so that the caller can retry accordingly.
+class NET_EXPORT URLRequestBackoffManager
+    : NON_EXPORTED_BASE(public base::NonThreadSafe),
+      public NetworkChangeNotifier::IPAddressObserver,
+      public NetworkChangeNotifier::ConnectionTypeObserver {
+ public:
+  // Minimum number of seconds that a Backoff header can specify.
+  static const uint16 kMinimumBackoffInSeconds;
+  // Maximum number of seconds that a Backoff header can specify.
+  static const uint16 kMaximumBackoffInSeconds;
+  // Number of throttled requests that will be made between garbage collection.
+  static const uint16 kNewEntriesBetweenCollecting;
+
+  URLRequestBackoffManager();
+  ~URLRequestBackoffManager() override;
+
+  // Updates internal states with a response.
+  void UpdateWithResponse(const GURL& url,
+                          HttpResponseHeaders* headers,
+                          const base::Time& response_time);
+
+  // Returns whether the request should be rejected because of a Backoff header.
+  bool ShouldRejectRequest(const GURL& url, const base::Time& request_time);
+
+  // IPAddressObserver implementation.
+  void OnIPAddressChanged() override;
+
+  // ConnectionTypeObserver implementation.
+  void OnConnectionTypeChanged(
+      NetworkChangeNotifier::ConnectionType type) override;
+
+  // Used by tests.
+  int GetNumberOfEntriesForTests() const;
+
+ private:
+  // An struct that holds relevant information obtained from a Backoff header.
+  struct Entry {
+    Entry(const base::Time& time1, const base::Time& time2)
+        : throttled_time(time1), release_time(time2), used(false) {}
+    ~Entry() {}
+
+    // Returns whether this entry is outdated.
+    bool IsOutDated() { return base::Time::Now() >= release_time; }
+
+    // Before this time, requests with the same URL ID should be throttled.
+    const base::Time throttled_time;
+
+    // Only one request with the same URL ID should be allowed in
+    // [|throttled_time|, |release_time|).
+    // After this time, all requests with the same URL ID are allowed.
+    const base::Time release_time;
+
+    // Indicates whether a request has been made in
+    // [|throttled_time|, |release_time|).
+    bool used;
+  };
+
+  // From each URL, generate an ID composed of the scheme, host, port and path
+  // that allows unique mapping an entry to it.
+  typedef std::map<std::string, Entry*> UrlEntryMap;
+
+  // Method that ensures the map gets cleaned from time to time. The period at
+  // which garbage collecting happens is adjustable with the
+  // kNewEntriesBetweenCollecting constant.
+  void GarbageCollectEntriesIfNecessary();
+
+  // Return true if there is a well-formed Backoff header key-value pair,
+  // and write the Backoff header value in |result|. Return false if no header
+  // is found or the value is invalid (i.e. less than kMinimumBackoffInSeconds
+  // or greater than kMaximumBackoffInSeconds).
+  bool GetBackoffTime(HttpResponseHeaders* headers,
+                      base::TimeDelta* result) const;
+
+  // Method that transforms a URL into an ID that can be used in the map.
+  // Resulting IDs will be lowercase and consist of the scheme, host, port
+  // and path (without query string, fragment, etc.).
+  // If the URL is invalid, the invalid spec will be returned, without any
+  // transformation.
+  std::string GetIdFromUrl(const GURL& url) const;
+
+  // When switching from online to offline or change IP addresses,
+  // clear all back-off history. This is a precaution in case the change in
+  // online state now allows communicating without errors with servers that
+  // were previously returning Backoff headers.
+  void OnNetworkChange();
+
+  UrlEntryMap url_entries_;
+
+  // Keeps track of how many new entries are created since last garbage
+  // collection.
+  unsigned int new_entries_since_last_gc_;
+
+  // Valid after construction.
+  GURL::Replacements url_id_replacements_;
+
+  DISALLOW_COPY_AND_ASSIGN(URLRequestBackoffManager);
+};
+
+}  // namespace net
+
+#endif  // NET_URL_REQUEST_URL_REQUEST_BACKOFF_MANAGER_H_