Reland "Sticky and Overlay-popup ads recorded in ads intervention man…

…ager"

This is a reland of 32cf67c

Reason for overlay-popup-ad browsertest failure on Mac 10.15: in the
original patch the popup ad was given a fixed 300x300 size. This is
sufficiently large relative to the window area on regular tryjob bots;
however, the window area on the Mac 10.15 bot is bigger, so the ad will
be skipped as it's too small compared to the window's dimension.

This patch fixed the problem by specifying a size relative to the
window's dimension, i.e. 0.5 * window_width X 0.5 * window_height.

Original change's description:
> Sticky and Overlay-popup ads recorded in ads intervention manager
>
> This change propagates signals for sticky ad and overlay-popup ad
> detection from the renderer to the ads intervention manager through
> the subresource filter component.
>
> As the ads intervention manager is running in dry run mode and
> does not block ads, this allows us to record intervention behavior
> when triggered by one of these two interventions. Understanding
> the intervention behavior is a precursor to the ads intervention
> manager intervening to actively intervening on sites with better
> ads standard violations.
>
> Add base::Feature flags to allow disabling the frequency capping from
> either web_tests and browsertests. Move relevant tests into a virtual
> test suite that has the flag disabled.
>
> Bug: 1099741
> Change-Id: Id0c922adfbb7cd71f641c6e9586eed9c311feb18
> Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2525769
> Commit-Queue: Yao Xiao <yaoxia@chromium.org>
> Reviewed-by: Eric Robinson <ericrobinson@chromium.org>
> Reviewed-by: Daniel Cheng <dcheng@chromium.org>
> Reviewed-by: Alex Moshchuk <alexmos@chromium.org>
> Reviewed-by: Stefan Zager <szager@chromium.org>
> Cr-Commit-Position: refs/heads/master@{#829547}

Bug: 1099741
Change-Id: I6fed9cb2e4ad223a05e150b67811141b72124202
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2553977
Commit-Queue: Yao Xiao <yaoxia@chromium.org>
Reviewed-by: Daniel Cheng <dcheng@chromium.org>
Reviewed-by: Stefan Zager <szager@chromium.org>
Reviewed-by: Avi Drissman <avi@chromium.org>
Reviewed-by: Eric Robinson <ericrobinson@chromium.org>
Cr-Commit-Position: refs/heads/master@{#832086}

chrome/browser/page_load_metrics/observers/ad_metrics/ads_page_load_metrics_observer_browsertest.cc

@@ -1580,39 +1580,49 @@ IN_PROC_BROWSER_TEST_F(AdsPageLoadMetricsObserverResourceBrowserTest,
   waiter->Wait();
   int64_t initial_page_bytes = waiter->current_network_bytes();
 
+  // Make the response large enough so that normal editing to the resource files
+  // won't interfere with the test expectations.
+  const int response_kilobytes = 64;
+  const int response_bytes = response_kilobytes * 1024;
+
   // Ad resource will not finish loading but should be reported to metrics.
   incomplete_resource_response->WaitForRequest();
   incomplete_resource_response->Send(kHttpOkResponseHeader);
-  incomplete_resource_response->Send(std::string(2048, ' '));
+  incomplete_resource_response->Send(std::string(response_bytes, ' '));
 
   // Wait for the resource update to be received for the incomplete response.
-  waiter->AddMinimumNetworkBytesExpectation(2048);
+
+  waiter->AddMinimumNetworkBytesExpectation(response_bytes);
   waiter->Wait();
 
   // Close all tabs instead of navigating as the embedded_test_server will
   // hang waiting for loads to finish when we have an unfinished
   // ControllableHttpResponse.
   browser()->tab_strip_model()->CloseAllTabs();
 
-  int expected_page_kilobytes = (initial_page_bytes + 2048) / 1024;
+  int expected_page_kilobytes = (initial_page_bytes + response_bytes) / 1024;
 
   histogram_tester.ExpectBucketCount(
       "PageLoad.Clients.Ads.Bytes.FullPage.Network", expected_page_kilobytes,
       1);
   histogram_tester.ExpectBucketCount(
-      "PageLoad.Clients.Ads.Bytes.AdFrames.Aggregate.Network", 2, 1);
+      "PageLoad.Clients.Ads.Bytes.AdFrames.Aggregate.Network",
+      response_kilobytes, 1);
   histogram_tester.ExpectBucketCount(
-      "PageLoad.Clients.Ads.Bytes.AdFrames.Aggregate.Total2", 2, 1);
+      "PageLoad.Clients.Ads.Bytes.AdFrames.Aggregate.Total2",
+      response_kilobytes, 1);
   histogram_tester.ExpectBucketCount(
-      "PageLoad.Clients.Ads.Bytes.AdFrames.PerFrame.Network", 2, 1);
+      "PageLoad.Clients.Ads.Bytes.AdFrames.PerFrame.Network",
+      response_kilobytes, 1);
   histogram_tester.ExpectBucketCount(
-      "PageLoad.Clients.Ads.Bytes.AdFrames.PerFrame.Total2", 2, 1);
+      "PageLoad.Clients.Ads.Bytes.AdFrames.PerFrame.Total2", response_kilobytes,
+      1);
   auto entries =
       ukm_recorder.GetEntriesByName(ukm::builders::AdFrameLoad::kEntryName);
   EXPECT_EQ(1u, entries.size());
   ukm_recorder.ExpectEntryMetric(
       entries.front(), ukm::builders::AdFrameLoad::kLoading_NetworkBytesName,
-      ukm::GetExponentialBucketMinForBytes(2048));
+      ukm::GetExponentialBucketMinForBytes(response_bytes));
   ukm_recorder.ExpectEntryMetric(
       entries.front(), ukm::builders::AdFrameLoad::kLoading_CacheBytes2Name, 0);
 }

chrome/browser/page_load_metrics/observers/ad_metrics/large_sticky_ad_intervention_browsertest.cc

@@ -0,0 +1,204 @@
+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include <memory>
+
+#include "base/test/scoped_feature_list.h"
+#include "build/build_config.h"
+#include "chrome/browser/infobars/infobar_service.h"
+#include "chrome/browser/subresource_filter/chrome_subresource_filter_client.h"
+#include "chrome/browser/subresource_filter/subresource_filter_browser_test_harness.h"
+#include "chrome/browser/ui/browser.h"
+#include "chrome/browser/ui/browser_commands.h"
+#include "chrome/common/chrome_features.h"
+#include "chrome/test/base/chrome_test_utils.h"
+#include "components/infobars/core/infobar.h"
+#include "components/infobars/core/infobar_delegate.h"
+#include "components/infobars/core/infobar_manager.h"
+#include "components/page_load_metrics/browser/page_load_metrics_test_waiter.h"
+#include "components/subresource_filter/content/browser/content_subresource_filter_throttle_manager.h"
+#include "components/subresource_filter/core/common/common_features.h"
+#include "components/subresource_filter/core/common/test_ruleset_utils.h"
+#include "components/subresource_filter/core/mojom/subresource_filter.mojom.h"
+#include "components/ukm/test_ukm_recorder.h"
+#include "content/public/test/browser_test.h"
+#include "content/public/test/browser_test_utils.h"
+#include "content/public/test/test_navigation_observer.h"
+#include "testing/gtest/include/gtest/gtest.h"
+#include "third_party/blink/public/common/features.h"
+#include "url/gurl.h"
+
+namespace {
+
+const char kAdsInterventionRecordedHistogram[] =
+    "SubresourceFilter.PageLoad.AdsInterventionTriggered";
+
+}  // namespace
+
+class LargeStickyAdViolationBrowserTest
+    : public subresource_filter::SubresourceFilterBrowserTest {
+ public:
+  LargeStickyAdViolationBrowserTest() = default;
+
+  void SetUp() override {
+    std::vector<base::Feature> enabled = {
+        subresource_filter::kAdTagging,
+        subresource_filter::kAdsInterventionsEnforced,
+        features::kSitePerProcess};
+    std::vector<base::Feature> disabled = {
+        blink::features::kFrequencyCappingForLargeStickyAdDetection};
+
+    feature_list_.InitWithFeatures(enabled, disabled);
+    subresource_filter::SubresourceFilterBrowserTest::SetUp();
+  }
+
+  void SetUpOnMainThread() override {
+    SubresourceFilterBrowserTest::SetUpOnMainThread();
+    SetRulesetWithRules(
+        {subresource_filter::testing::CreateSuffixRule("ad_iframe_writer.js")});
+  }
+
+  // Navigate to |url| and wait until we see the first contentful paint. FCP is
+  // a prerequisite for starting the large sticky ad detection.
+  void NavigateAndWaitForFirstContentfulPaint(const GURL& url) {
+    content::WebContents* web_contents =
+        chrome_test_utils::GetActiveWebContents(this);
+    auto waiter =
+        std::make_unique<page_load_metrics::PageLoadMetricsTestWaiter>(
+            web_contents);
+    waiter->AddPageExpectation(page_load_metrics::PageLoadMetricsTestWaiter::
+                                   TimingField::kFirstContentfulPaint);
+
+    EXPECT_TRUE(content::NavigateToURL(web_contents, url));
+    waiter->Wait();
+    waiter.reset();
+  }
+
+  // Create a large sticky ad and trigger a series of actions and layout updates
+  // for the ad to be detected by the sticky ad detector.
+  void TriggerAndDetectLargeStickyAd() {
+    content::WebContents* web_contents =
+        chrome_test_utils::GetActiveWebContents(this);
+
+    // Create the large-sticky-ad.
+    EXPECT_TRUE(ExecJs(
+        web_contents,
+        "let frame = createStickyAdIframeAtBottomOfViewport(window.innerWidth, "
+        "window.innerHeight * 0.35);"));
+
+    // Force a layout update to capture the initial state. Then scroll further
+    // down.
+    ASSERT_TRUE(
+        EvalJsAfterLifecycleUpdate(web_contents, "", "window.scrollTo(0, 5000)")
+            .error.empty());
+
+    // Force a layout update to capture the final state. At this point the
+    // detector should have detected the large-sticky-ad.
+    ASSERT_TRUE(EvalJsAfterLifecycleUpdate(web_contents, "", "").error.empty());
+  }
+
+ protected:
+  base::test::ScopedFeatureList feature_list_;
+};
+
+IN_PROC_BROWSER_TEST_F(LargeStickyAdViolationBrowserTest,
+                       NoLargeStickyAd_AdInterventionNotTriggered) {
+  base::HistogramTester histogram_tester;
+
+  GURL url = embedded_test_server()->GetURL(
+      "a.com", "/ads_observer/large_scrollable_page_with_adiframe_writer.html");
+
+  NavigateAndWaitForFirstContentfulPaint(url);
+
+  content::WebContents* web_contents =
+      chrome_test_utils::GetActiveWebContents(this);
+
+  // Reload the page. Since we haven't seen any ad violations, expect that the
+  // ad script is loaded and that the subresource filter UI doesn't show up.
+  EXPECT_TRUE(content::NavigateToURL(web_contents, url));
+
+  EXPECT_TRUE(WasParsedScriptElementLoaded(web_contents->GetMainFrame()));
+  histogram_tester.ExpectBucketCount(
+      "SubresourceFilter.Actions2",
+      subresource_filter::SubresourceFilterAction::kUIShown, 0);
+  histogram_tester.ExpectBucketCount(
+      kAdsInterventionRecordedHistogram,
+      subresource_filter::mojom::AdsViolation::kLargeStickyAd, 0);
+}
+
+IN_PROC_BROWSER_TEST_F(LargeStickyAdViolationBrowserTest,
+                       LargeStickyAd_AdInterventionTriggered) {
+  base::HistogramTester histogram_tester;
+
+  GURL url = embedded_test_server()->GetURL(
+      "a.com", "/ads_observer/large_scrollable_page_with_adiframe_writer.html");
+
+  NavigateAndWaitForFirstContentfulPaint(url);
+
+  TriggerAndDetectLargeStickyAd();
+
+  content::WebContents* web_contents =
+      chrome_test_utils::GetActiveWebContents(this);
+
+  // Reload the page. Since we are enforcing ad blocking on ads violations,
+  // expect that the ad script is not loaded and that the subresource filter UI
+  // shows up.
+  EXPECT_TRUE(content::NavigateToURL(web_contents, url));
+
+  EXPECT_FALSE(WasParsedScriptElementLoaded(web_contents->GetMainFrame()));
+  histogram_tester.ExpectBucketCount(
+      "SubresourceFilter.Actions2",
+      subresource_filter::SubresourceFilterAction::kUIShown, 1);
+  histogram_tester.ExpectBucketCount(
+      kAdsInterventionRecordedHistogram,
+      subresource_filter::mojom::AdsViolation::kLargeStickyAd, 1);
+}
+
+class LargeStickyAdViolationBrowserTestWithoutEnforcement
+    : public LargeStickyAdViolationBrowserTest {
+ public:
+  LargeStickyAdViolationBrowserTestWithoutEnforcement() = default;
+
+  void SetUp() override {
+    std::vector<base::Feature> enabled = {subresource_filter::kAdTagging};
+    std::vector<base::Feature> disabled = {
+        subresource_filter::kAdsInterventionsEnforced,
+        blink::features::kFrequencyCappingForLargeStickyAdDetection};
+
+    feature_list_.InitWithFeatures(enabled, disabled);
+    subresource_filter::SubresourceFilterBrowserTest::SetUp();
+  }
+
+ private:
+  base::test::ScopedFeatureList feature_list_;
+};
+
+IN_PROC_BROWSER_TEST_F(LargeStickyAdViolationBrowserTestWithoutEnforcement,
+                       LargeStickyAd_NoAdInterventionTriggered) {
+  base::HistogramTester histogram_tester;
+
+  GURL url = embedded_test_server()->GetURL(
+      "a.com", "/ads_observer/large_scrollable_page_with_adiframe_writer.html");
+
+  NavigateAndWaitForFirstContentfulPaint(url);
+
+  TriggerAndDetectLargeStickyAd();
+
+  content::WebContents* web_contents =
+      chrome_test_utils::GetActiveWebContents(this);
+
+  // Reload the page. Since we are not enforcing ad blocking on ads violations,
+  // expect that the ad script is loaded and that the subresource filter UI
+  // doesn't show up. Expect a histogram recording as the intervention is
+  // running in dry run mode.
+  EXPECT_TRUE(content::NavigateToURL(web_contents, url));
+
+  EXPECT_TRUE(WasParsedScriptElementLoaded(web_contents->GetMainFrame()));
+  histogram_tester.ExpectBucketCount(
+      "SubresourceFilter.Actions2",
+      subresource_filter::SubresourceFilterAction::kUIShown, 0);
+  histogram_tester.ExpectBucketCount(
+      kAdsInterventionRecordedHistogram,
+      subresource_filter::mojom::AdsViolation::kLargeStickyAd, 1);
+}