Add stale-while-revalidate support to network stack.

This changes adds a load flag that blink will set to request
resources that are stale from the network cache. Upon receiving this
flag and returning a stale resource the network stack will set a
timeout of 1 minute for the resource to be validated. During this
period the stale resource can continue to be returned but after that
if not revalidated the resource is considered invalid.

A large portion of this change is reverting
crrev.com/75221e53b71cc5eb37cbfb957124025259aa5dab

BUG=348877

Change-Id: I889f6e4c9b59b95bc26ce3a8f8f9d9c15da0ed71
Reviewed-on: https://chromium-review.googlesource.com/1085850
Commit-Queue: Dave Tapuska <dtapuska@chromium.org>
Reviewed-by: David Benjamin <davidben@chromium.org>
Reviewed-by: Maks Orlovich <morlovich@chromium.org>
Cr-Commit-Position: refs/heads/master@{#566258}

net/base/load_flags_list.h

@@ -92,3 +92,13 @@ LOAD_FLAG(DISABLE_CONNECTION_MIGRATION, 1 << 16)
 // Indicates that the cache should not check that the request matches the
 // response's vary header.
 LOAD_FLAG(SKIP_VARY_CHECK, 1 << 17)
+
+// The creator of this URLRequest wishes to receive stale responses when allowed
+// by the "Cache-Control: stale-while-revalidate" directive and is able to issue
+// an async revalidation to update the cache. If the callee needs to revalidate
+// the resource |async_revalidation_requested| attribute will be set on the
+// associated HttpResponseInfo. If indicated the callee should revalidate the
+// resource by issuing a new request without this flag set. If the revalidation
+// does not complete in 60 seconds, the cache treat the stale resource as
+// invalid, as it did not specify stale-while-revalidate.
+LOAD_FLAG(SUPPORT_ASYNC_REVALIDATION, 1 << 18)

net/http/http_cache_transaction.cc

@@ -18,16 +18,20 @@
 #include "base/bind_helpers.h"
 #include "base/callback_helpers.h"
 #include "base/compiler_specific.h"
+#include "base/format_macros.h"
 #include "base/location.h"
 #include "base/macros.h"
 #include "base/metrics/histogram_functions.h"
 #include "base/metrics/histogram_macros.h"
 #include "base/single_thread_task_runner.h"
 #include "base/strings/string_number_conversions.h"  // For HexEncode.
+#include "base/strings/string_piece.h"
 #include "base/strings/string_util.h"  // For LowerCaseEqualsASCII.
+#include "base/strings/stringprintf.h"
 #include "base/threading/thread_task_runner_handle.h"
 #include "base/time/clock.h"
 #include "base/trace_event/trace_event.h"
+#include "base/values.h"
 #include "net/base/auth.h"
 #include "net/base/load_flags.h"
 #include "net/base/load_timing_info.h"
@@ -54,6 +58,8 @@ using CacheEntryStatus = HttpResponseInfo::CacheEntryStatus;
 
 namespace {
 
+constexpr TimeDelta kStaleRevalidateTimeout = TimeDelta::FromSeconds(60);
+
 // From http://tools.ietf.org/html/draft-ietf-httpbis-p6-cache-21#section-6
 //      a "non-error response" is one with a 2xx (Successful) or 3xx
 //      (Redirection) status code.
@@ -71,6 +77,13 @@ void RecordNoStoreHeaderHistogram(int load_flags,
   }
 }
 
+enum ExternallyConditionalizedType {
+  EXTERNALLY_CONDITIONALIZED_CACHE_REQUIRES_VALIDATION,
+  EXTERNALLY_CONDITIONALIZED_CACHE_USABLE,
+  EXTERNALLY_CONDITIONALIZED_MISMATCHED_VALIDATORS,
+  EXTERNALLY_CONDITIONALIZED_MAX
+};
+
 }  // namespace
 
 #define CACHE_STATUS_HISTOGRAMS(type)                                        \
@@ -897,6 +910,17 @@ int HttpCache::Transaction::DoLoop(int result) {
       case STATE_COMPLETE_PARTIAL_CACHE_VALIDATION:
         rv = DoCompletePartialCacheValidation(rv);
         break;
+      case STATE_CACHE_UPDATE_STALE_WHILE_REVALIDATE_TIMEOUT:
+        DCHECK_EQ(OK, rv);
+        rv = DoCacheUpdateStaleWhileRevalidateTimeout();
+        break;
+      case STATE_CACHE_UPDATE_STALE_WHILE_REVALIDATE_TIMEOUT_COMPLETE:
+        rv = DoCacheUpdateStaleWhileRevalidateTimeoutComplete(rv);
+        break;
+      case STATE_SETUP_ENTRY_FOR_READ:
+        DCHECK_EQ(OK, rv);
+        rv = DoSetupEntryForRead();
+        break;
       case STATE_SEND_REQUEST:
         DCHECK_EQ(OK, rv);
         rv = DoSendRequest();
@@ -1592,6 +1616,24 @@ int HttpCache::Transaction::DoCompletePartialCacheValidation(int result) {
   return BeginCacheValidation();
 }
 
+int HttpCache::Transaction::DoCacheUpdateStaleWhileRevalidateTimeout() {
+  TRACE_EVENT0(
+      "io", "HttpCacheTransaction::DoCacheUpdateStaleWhileRevalidateTimeout");
+  response_.stale_revalidate_timeout =
+      cache_->clock_->Now() + kStaleRevalidateTimeout;
+  TransitionToState(STATE_CACHE_UPDATE_STALE_WHILE_REVALIDATE_TIMEOUT_COMPLETE);
+  return WriteResponseInfoToEntry(false);
+}
+
+int HttpCache::Transaction::DoCacheUpdateStaleWhileRevalidateTimeoutComplete(
+    int result) {
+  TRACE_EVENT0(
+      "io",
+      "HttpCacheTransaction::DoCacheUpdateStaleWhileRevalidateTimeoutComplete");
+  TransitionToState(STATE_SETUP_ENTRY_FOR_READ);
+  return OnWriteResponseInfoToEntryComplete(result);
+}
+
 int HttpCache::Transaction::DoSendRequest() {
   TRACE_EVENT0("io", "HttpCacheTransaction::DoSendRequest");
   DCHECK(mode_ & WRITE || mode_ == NONE);
@@ -1779,6 +1821,7 @@ int HttpCache::Transaction::DoUpdateCachedResponse() {
   // Update the cached response based on the headers and properties of
   // new_response_.
   response_.headers->Update(*new_response_->headers.get());
+  response_.stale_revalidate_timeout = base::Time();
   response_.response_time = new_response_->response_time;
   response_.request_time = new_response_->request_time;
   response_.network_accessed = new_response_->network_accessed;
@@ -2410,7 +2453,7 @@ int HttpCache::Transaction::BeginCacheRead() {
     return ERR_CACHE_MISS;
   }
 
-  if (RequiresValidation()) {
+  if (RequiresValidation() != VALIDATION_NONE) {
     TransitionToState(STATE_FINISH_HEADERS);
     return ERR_CACHE_MISS;
   }
@@ -2429,13 +2472,27 @@ int HttpCache::Transaction::BeginCacheRead() {
 int HttpCache::Transaction::BeginCacheValidation() {
   DCHECK_EQ(mode_, READ_WRITE);
 
-  bool skip_validation = !RequiresValidation();
+  ValidationType required_validation = RequiresValidation();
+
+  bool skip_validation = (required_validation == VALIDATION_NONE);
+  bool needs_stale_while_revalidate_cache_update = false;
+
+  if ((effective_load_flags_ & LOAD_SUPPORT_ASYNC_REVALIDATION) &&
+      required_validation == VALIDATION_ASYNCHRONOUS) {
+    DCHECK_EQ(request_->method, "GET");
+    skip_validation = true;
+    response_.async_revalidation_requested = true;
+    needs_stale_while_revalidate_cache_update =
+        response_.stale_revalidate_timeout.is_null();
+  }
 
   if (method_ == "HEAD" &&
       (truncated_ || response_.headers->response_code() == 206)) {
     DCHECK(!partial_);
-    if (skip_validation)
-      return SetupEntryForRead();
+    if (skip_validation) {
+      TransitionToState(STATE_SETUP_ENTRY_FOR_READ);
+      return OK;
+    }
 
     // Bail out!
     TransitionToState(STATE_SEND_REQUEST);
@@ -2460,7 +2517,10 @@ int HttpCache::Transaction::BeginCacheValidation() {
 
   if (skip_validation) {
     UpdateCacheEntryStatus(CacheEntryStatus::ENTRY_USED);
-    return SetupEntryForRead();
+    TransitionToState(needs_stale_while_revalidate_cache_update
+                          ? STATE_CACHE_UPDATE_STALE_WHILE_REVALIDATE_TIMEOUT
+                          : STATE_SETUP_ENTRY_FOR_READ);
+    return OK;
   } else {
     // Make the network request conditional, to see if we may reuse our cached
     // response.  If we cannot do so, then we just resort to a normal fetch.
@@ -2596,7 +2656,7 @@ int HttpCache::Transaction::RestartNetworkRequestWithAuth(
   return rv;
 }
 
-bool HttpCache::Transaction::RequiresValidation() {
+ValidationType HttpCache::Transaction::RequiresValidation() {
   // TODO(darin): need to do more work here:
   //  - make sure we have a matching request method
   //  - watch out for cached responses that depend on authentication
@@ -2607,11 +2667,11 @@ bool HttpCache::Transaction::RequiresValidation() {
                                           *response_.headers.get())) {
     vary_mismatch_ = true;
     validation_cause_ = VALIDATION_CAUSE_VARY_MISMATCH;
-    return true;
+    return VALIDATION_SYNCHRONOUS;
   }
 
   if (effective_load_flags_ & LOAD_SKIP_CACHE_VALIDATION)
-    return false;
+    return VALIDATION_NONE;
 
   if (response_.unused_since_prefetch &&
       !(effective_load_flags_ & LOAD_PREFETCH) &&
@@ -2620,21 +2680,23 @@ bool HttpCache::Transaction::RequiresValidation() {
           cache_->clock_->Now()) < TimeDelta::FromMinutes(kPrefetchReuseMins)) {
     // The first use of a resource after prefetch within a short window skips
     // validation.
-    return false;
+    return VALIDATION_NONE;
   }
 
   if (effective_load_flags_ & LOAD_VALIDATE_CACHE) {
     validation_cause_ = VALIDATION_CAUSE_VALIDATE_FLAG;
-    return true;
+    return VALIDATION_SYNCHRONOUS;
   }
 
   if (method_ == "PUT" || method_ == "DELETE")
-    return true;
+    return VALIDATION_SYNCHRONOUS;
 
-  bool validation_required_by_headers = response_.headers->RequiresValidation(
-      response_.request_time, response_.response_time, cache_->clock_->Now());
+  ValidationType validation_required_by_headers =
+      response_.headers->RequiresValidation(response_.request_time,
+                                            response_.response_time,
+                                            cache_->clock_->Now());
 
-  if (validation_required_by_headers) {
+  if (validation_required_by_headers != VALIDATION_NONE) {
     HttpResponseHeaders::FreshnessLifetimes lifetimes =
         response_.headers->GetFreshnessLifetimes(response_.response_time);
     if (lifetimes.freshness == base::TimeDelta()) {
@@ -2648,6 +2710,19 @@ bool HttpCache::Transaction::RequiresValidation() {
     }
   }
 
+  if (validation_required_by_headers == VALIDATION_ASYNCHRONOUS) {
+    // Asynchronous revalidation is only supported for GET methods.
+    if (request_->method != "GET")
+      return VALIDATION_SYNCHRONOUS;
+
+    // If the timeout on the staleness revalidation is set don't hand out
+    // a resource that hasn't been async validated.
+    if (!response_.stale_revalidate_timeout.is_null() &&
+        response_.stale_revalidate_timeout < cache_->clock_->Now()) {
+      return VALIDATION_SYNCHRONOUS;
+    }
+  }
+
   return validation_required_by_headers;
 }
 
@@ -2922,7 +2997,7 @@ void HttpCache::Transaction::FixHeadersForHead() {
   }
 }
 
-int HttpCache::Transaction::SetupEntryForRead() {
+int HttpCache::Transaction::DoSetupEntryForRead() {
   if (network_trans_)
     ResetNetworkTransaction();
   if (partial_) {

net/http/http_cache_transaction.h

@@ -256,6 +256,9 @@ class NET_EXPORT_PRIVATE HttpCache::Transaction : public HttpTransaction {
     STATE_CACHE_QUERY_DATA_COMPLETE,
     STATE_START_PARTIAL_CACHE_VALIDATION,
     STATE_COMPLETE_PARTIAL_CACHE_VALIDATION,
+    STATE_CACHE_UPDATE_STALE_WHILE_REVALIDATE_TIMEOUT,
+    STATE_CACHE_UPDATE_STALE_WHILE_REVALIDATE_TIMEOUT_COMPLETE,
+    STATE_SETUP_ENTRY_FOR_READ,
     STATE_SEND_REQUEST,
     STATE_SEND_REQUEST_COMPLETE,
     STATE_SUCCESSFUL_SEND_REQUEST,
@@ -333,6 +336,9 @@ class NET_EXPORT_PRIVATE HttpCache::Transaction : public HttpTransaction {
   int DoCacheDispatchValidation();
   int DoCacheQueryData();
   int DoCacheQueryDataComplete(int result);
+  int DoCacheUpdateStaleWhileRevalidateTimeout();
+  int DoCacheUpdateStaleWhileRevalidateTimeoutComplete(int result);
+  int DoSetupEntryForRead();
   int DoStartPartialCacheValidation();
   int DoCompletePartialCacheValidation(int result);
   int DoSendRequest();
@@ -407,8 +413,9 @@ class NET_EXPORT_PRIVATE HttpCache::Transaction : public HttpTransaction {
   // Returns network error code.
   int RestartNetworkRequestWithAuth(const AuthCredentials& credentials);
 
-  // Called to determine if we need to validate the cache entry before using it.
-  bool RequiresValidation();
+  // Called to determine if we need to validate the cache entry before using it,
+  // and whether the validation should be synchronous or asynchronous.
+  ValidationType RequiresValidation();
 
   // Called to make the request conditional (to ask the server if the cached
   // copy is valid).  Returns true if able to make the request conditional.
@@ -443,9 +450,6 @@ class NET_EXPORT_PRIVATE HttpCache::Transaction : public HttpTransaction {
   // Fixes the response headers to match expectations for a HEAD request.
   void FixHeadersForHead();
 
-  // Setups the transaction for reading from the cache entry.
-  int SetupEntryForRead();
-
   // Called to write data to the cache entry.  If the write fails, then the
   // cache entry is destroyed.  Future calls to this function will just do
   // nothing without side-effect.  Returns a network error code.