Let virtual tests also inherit [ Skip ] if no override

This followups crrev.com/c/2173656 which let virtual tests inherit
base tests' expectations except for [ Skip ]. This CL also allows
inheritance of [ Skip ].

In less common cases, we intentionally skip some base tests which is
only meaningful to run with specific flags in virtual tests. In these
cases we need to override the [ Skip ] with [ Pass ], like:

(In NeverFixTests)
# This test run under virtual suites only.
crbug.com/981970 external/wpt/fetch/http-cache/split-cache.tentative.html [ Skip ]
virtual/not-split-http-cache/external/wpt/fetch/http-cache/split-cache.tentative.html [ Pass ]
virtual/split-http-cache/external/wpt/fetch/http-cache/split-cache.tentative.html [ Pass ]

Bug: 1072015
Change-Id: Ie6c5b5472203bd81120dd2c3589a89d9df6fa45b
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2178173
Commit-Queue: Xianzhu Wang <wangxianzhu@chromium.org>
Reviewed-by: Rakib Hasan <rmhasan@google.com>
Reviewed-by: Robert Ma <robertma@chromium.org>
Reviewed-by: Philip Rogers <pdr@chromium.org>
Reviewed-by: Dirk Pranke <dpranke@chromium.org>
Cr-Commit-Position: refs/heads/master@{#766598}

Author: wangxianzhu

docs/testing/web_tests.md

@@ -322,6 +322,13 @@ Consider the following when choosing between them:
   in the suite or `virtual/blocking_repaint/fast/repaint/dir` to run real
   or virtual tests in the suite under a specific directory.
 
+*** note
+We can run a virtual test with additional flags. Both the virtual args and the
+additional flags will be applied. The fallback order of baselines and
+expectations will be: 1) flag-specific virtual, 2) non-flag-specific virtual,
+3) flag-specific base, 4) non-flag-specific base
+***
+
 ## Tracking Test Failures
 
 All bugs, associated with web test failures must have the

third_party/blink/tools/blinkpy/web_tests/lint_test_expectations.py

@@ -159,11 +159,10 @@ def _check_expectations_file_content(content):
     return failures
 
 
-def _check_expectations(host, port, path, test_expectations):
+def _check_directory_glob(host, port, path, expectations):
     failures = []
-    for exp in test_expectations.get_updated_lines(path):
-        if (exp.is_glob or not exp.to_string().strip()
-                or exp.to_string().strip().startswith('#')):
+    for exp in expectations:
+        if not exp.test or exp.is_glob:
             continue
 
         test_name, _ = port.split_webdriver_test_name(exp.test)
@@ -183,6 +182,56 @@ def _check_expectations(host, port, path, test_expectations):
     return failures
 
 
+def _check_redundant_virtual_expectations(host, port, path, expectations):
+    # FlagExpectations are not checked because something like the following in
+    # a flag-specific expectations file looks redundant but is needed to
+    # override the virtual expectations in TestExpectations. For example, in
+    # the main TestExpectations:
+    #   foo/bar/test.html [ Failure ]
+    #   virtual/suite/foo/bar/test.html [ Timeout ]
+    # and in a flag expectation file, we want to override both to [ Crash ]:
+    #   foo/bar/test.html [ Crash ]
+    #   virtual/suite/foo/bar/test.html [ Crash ]
+    if 'FlagExpectations' in path:
+        return []
+
+    failures = []
+    expectations_by_test = {}
+    for exp in expectations:
+        if exp.test:
+            expectations_by_test.setdefault(exp.test, []).append(exp)
+
+    for exp in expectations:
+        if not exp.test:
+            continue
+
+        base_test = port.lookup_virtual_test_base(exp.test)
+        if not base_test:
+            continue
+
+        for base_exp in expectations_by_test.get(base_test, []):
+            if (base_exp.results == exp.results
+                    and base_exp.is_slow_test == exp.is_slow_test
+                    and base_exp.tags.issubset(exp.tags)):
+                error = "{}:{} Expectation '{}' is redundant with '{}' in line {}".format(
+                    host.filesystem.basename(path), exp.lineno, exp.test,
+                    base_test, base_exp.lineno)
+                _log.error(error)
+                failures.append(error)
+
+    return failures
+
+
+def _check_expectations(host, port, path, test_expectations):
+    # Check for original expectation lines (from get_updated_lines) instead of
+    # expectations filtered for the current port (test_expectations).
+    expectations = test_expectations.get_updated_lines(path)
+    failures = _check_directory_glob(host, port, path, expectations)
+    failures.extend(
+        _check_redundant_virtual_expectations(host, port, path, expectations))
+    return failures
+
+
 def check_virtual_test_suites(host, options):
     port = host.port_factory.get(options=options)
     fs = host.filesystem

third_party/blink/tools/blinkpy/web_tests/lint_test_expectations_unittest.py

@@ -245,6 +245,38 @@ def test_lint_globs(self):
         all_logs = ''.join(self.logMessages())
         self.assertIn('directory', all_logs)
 
+    def test_virtual_test_redundant_expectation(self):
+        options = optparse.Values({
+            'additional_expectations': [],
+            'platform': 'test',
+            'debug_rwt_logging': False
+        })
+        host = MockHost()
+
+        port = host.port_factory.get(options.platform, options=options)
+        port.virtual_test_suites = lambda: [
+            VirtualTestSuite(prefix='foo', bases=['test'], args=['--foo'])
+        ]
+        test_expectations = (
+            '# tags: [ mac win ]\n'
+            '# tags: [ debug release ]\n'
+            '# results: [ Failure Pass ]\n'
+            '[ mac ] test/test1.html [ Failure ]\n'
+            '[ mac debug ] virtual/foo/test/test1.html [ Failure ]\n'
+            '[ win ] virtual/foo/test/test1.html [ Failure ]\n'
+            '[ mac release ] virtual/foo/test/test1.html [ Pass ]\n')
+        port.expectations_dict = lambda: {
+            'testexpectations': test_expectations
+        }
+
+        host.port_factory.get = lambda platform, options=None: port
+        host.port_factory.all_port_names = lambda platform=None: [port.name()]
+
+        res = lint_test_expectations.lint(host, options)
+
+        self.assertEquals(len(res), 1)
+        self.assertIn('redundant', res[0])
+
 
 class CheckVirtualSuiteTest(unittest.TestCase):
     def setUp(self):

third_party/blink/tools/blinkpy/web_tests/models/test_expectations.py

@@ -340,18 +340,8 @@ def get_expectations(self, test, fallback_for_test=None):
                                    fallback_for_test))
         base_test = self.port.lookup_virtual_test_base(test)
         if base_test:
-            base_expectations = self.get_expectations(base_test, test)
-            if ResultType.Skip in base_expectations.results:
-                # TODO(crbug.com/1072015#c9): Temporarily remove Skip from the
-                # inherited expectations to avoid unexpected Skip.
-                base_expectations = typ_types.Expectation(
-                    test=base_expectations.test,
-                    results=base_expectations.results - set([ResultType.Skip]),
-                    is_slow_test=base_expectations.is_slow_test,
-                    reason=base_expectations.reason,
-                    trailing_comments=base_expectations.trailing_comments)
             return self._override_or_fallback_expectations(
-                expectations, base_expectations)
+                expectations, self.get_expectations(base_test, test))
         return expectations
 
     def get_flag_expectations(self, test):

third_party/blink/tools/blinkpy/web_tests/models/test_expectations_unittest.py

@@ -127,7 +127,7 @@ def test_virtual_expectations(self):
             'virtual/virtual_failures/failures/expected/crash.html',
             ResultType.Crash)
         self.assert_exp('virtual/virtual_passes/passes/image.html',
-                        ResultType.Pass)
+                        ResultType.Skip)
         # Non existence virtual suite doesn't fallback.
         self.assert_exp('virtual/xyz/failures/expected/crash.html',
                         ResultType.Pass)

third_party/blink/tools/blinkpy/web_tests/port/base.py

@@ -1974,6 +1974,8 @@ def _wpt_test_urls_matching_paths(self, filter_paths, virtual_prefixes=[]):
         return tests
 
     def _lookup_virtual_suite(self, test_name):
+        if not test_name.startswith('virtual/'):
+            return None
         for suite in self.virtual_test_suites():
             if test_name.startswith(suite.full_prefix):
                 return suite

third_party/blink/tools/blinkpy/web_tests/port/test.py

@@ -137,12 +137,12 @@ def __getitem__(self, item):
 #
 TOTAL_TESTS = 171
 TOTAL_WONTFIX = 3
-TOTAL_SKIPS = 23 + TOTAL_WONTFIX
+TOTAL_SKIPS = 26 + TOTAL_WONTFIX
 TOTAL_CRASHES = 78
 
 UNEXPECTED_PASSES = 2
 UNEXPECTED_NON_VIRTUAL_FAILURES = 34
-UNEXPECTED_FAILURES = 68
+UNEXPECTED_FAILURES = 67
 
 
 def unit_test_list():

third_party/blink/web_tests/MSANExpectations

@@ -115,7 +115,6 @@ crbug.com/856601 [ Linux ] external/wpt/IndexedDB/idlharness.https.any.servicewo
 crbug.com/856601 [ Linux ] external/wpt/IndexedDB/interfaces.any.html [ Pass Timeout ]
 crbug.com/856601 [ Linux ] external/wpt/appmanifest/idlharness.window.html [ Skip ]
 crbug.com/856601 [ Linux ] external/wpt/bluetooth/idl/idlharness.tentative.window.html [ Skip ]
-crbug.com/856601 [ Linux ] virtual/web-bluetooth-new-permissions-backend/external/wpt/bluetooth/idl/idlharness.tentative.https.window.html [ Skip ]
 crbug.com/856601 [ Linux ] external/wpt/cookie-store/idlharness.tentative.https.html [ Skip ]
 crbug.com/856601 [ Linux ] external/wpt/css/css-animations/idlharness.html [ Skip ]
 crbug.com/856601 [ Linux ] external/wpt/css/css-font-loading/idlharness.https.html [ Skip ]
@@ -159,7 +158,6 @@ crbug.com/856601 [ Linux ] external/wpt/touch-events/idlharness.window.html [ Sk
 crbug.com/856601 [ Linux ] external/wpt/screen-wake-lock/idlharness.https.window.html [ Skip ]
 crbug.com/856601 [ Linux ] external/wpt/webstorage/idlharness.window.html [ Skip ]
 crbug.com/856601 [ Linux ] virtual/feature-policy-permissions/external/wpt/mediacapture-streams/MediaDevices-IDL-enumerateDevices.html [ Pass Timeout ]
-crbug.com/856601 [ Linux ] virtual/feature-policy-permissions/external/wpt/mediacapture-streams/idlharness.https.window.html [ Skip ]
 crbug.com/856601 [ Linux ] external/wpt/fetch/api/idl.any.sharedworker.html [ Pass Timeout ]
 crbug.com/856601 [ Linux ] external/wpt/fetch/cors-rfc1918/idlharness.tentative.https.any.serviceworker.html [ Skip ]
 crbug.com/856601 [ Linux ] external/wpt/media-capabilities/idlharness.any.html [ Skip ]
@@ -172,7 +170,6 @@ crbug.com/856601 [ Linux ] external/wpt/media-source/idlharness.any.html [ Skip
 crbug.com/856601 [ Linux ] external/wpt/page-visibility/idlharness.window.html [ Skip ]
 crbug.com/856601 [ Linux ] external/wpt/css/filter-effects/interfaces.any.html [ Timeout Pass ]
 crbug.com/856601 [ Linux ] external/wpt/webxr/idlharness.https.window.html [ Skip ]
-crbug.com/856601 [ Linux ] virtual/webrtc-wpt-plan-b/external/wpt/webrtc/idlharness.https.window.html [ Skip ]
 crbug.com/856601 [ Linux ] fast/js/toString-stack-overflow.html [ Pass Timeout ]
 crbug.com/856601 [ Linux ] external/wpt/css/css-masking/idlharness.html [ Skip ]
 crbug.com/856601 [ Linux ] external/wpt/css/css-transitions/idlharness.html [ Skip ]
@@ -226,9 +223,6 @@ crbug.com/856601 [ Linux ] external/wpt/fullscreen/idlharness.window.html [ Skip
 crbug.com/856601 [ Linux ] external/wpt/payment-handler/idlharness.https.any.html [ Skip ]
 crbug.com/856601 [ Linux ] external/wpt/payment-handler/idlharness.https.any.serviceworker.html [ Skip ]
 crbug.com/856601 [ Linux ] external/wpt/payment-request/idlharness.https.window.html [ Skip ]
-crbug.com/856601 [ Linux ] virtual/omt-worker-fetch/external/wpt/resource-timing/idlharness.any.html [ Skip ]
-crbug.com/856601 [ Linux ] virtual/speech-with-unified-autoplay/external/wpt/speech-api/idlharness.window.html [ Skip ]
-crbug.com/856601 [ Linux ] virtual/unified-autoplay/external/wpt/feature-policy/idlharness.window.html [ Skip ]
 
 # Sheriff 2019-02-22
 crbug.com/929122 [ Linux ] external/wpt/html/dom/interfaces.worker.html [ Timeout ]
@@ -273,7 +267,6 @@ crbug.com/856601 [ Linux ] external/wpt/xhr/idlharness.any.worker.html [ Skip ]
 crbug.com/856601 [ Linux ] http/tests/devtools/a11y-axe-core/console/console-a11y-test.js [ Pass Timeout ]
 crbug.com/856601 [ Linux ] http/tests/devtools/a11y-axe-core/network/network-condition-a11y-test.js [ Pass Timeout ]
 crbug.com/856601 [ Linux ] external/wpt/webrtc-identity/idlharness.https.window.html [ Skip ]
-crbug.com/856601 [ Linux ] virtual/omt-worker-fetch/external/wpt/fetch/api/idlharness.any.worker.html [ Skip ]
 
 # Sheriff 2019-06-28
 crbug.com/856601 [ Linux ] external/wpt/battery-status/battery-interface-idlharness.https.window.html [ Skip ]
@@ -287,7 +280,6 @@ crbug.com/856601 [ Linux ] http/tests/devtools/indexeddb/live-update-indexeddb-c
 crbug.com/856601 [ Linux ] http/tests/devtools/indexeddb/live-update-indexeddb-list.js [ Pass Timeout ]
 
 # Sheriff 2019-07-19
-crbug.com/856601 [ Linux ] virtual/omt-worker-fetch/external/wpt/fetch/cors-rfc1918/idlharness.tentative.any.serviceworker.html [ Skip ]
 crbug.com/856601 [ Linux ] external/wpt/WebCryptoAPI/idlharness.https.any.html [ Skip ]
 crbug.com/856601 [ Linux ] external/wpt/web-animations/interfaces/Animation/idlharness.window.html [ Skip ]
 
@@ -304,10 +296,6 @@ crbug.com/856601 [ Linux ] external/wpt/web-nfc/idlharness.https.window.html [ S
 crbug.com/856601 [ Linux ] external/wpt/xhr/idlharness.any.html [ Skip ]
 crbug.com/856601 [ Linux ] external/wpt/xhr/idlharness.any.sharedworker.html [ Skip ]
 crbug.com/856601 [ Linux ] http/tests/devtools/a11y-axe-core/settings/geolocations-a11y-test.js [ Pass Timeout ]
-crbug.com/856601 [ Linux ] virtual/omt-worker-fetch/external/wpt/fetch/cors-rfc1918/idlharness.tentative.any.sharedworker.html [ Skip ]
-crbug.com/856601 [ Linux ] virtual/omt-worker-fetch/external/wpt/xhr/idlharness.any.html [ Skip ]
-crbug.com/856601 [ Linux ] virtual/omt-worker-fetch/external/wpt/xhr/idlharness.any.sharedworker.html [ Skip ]
-crbug.com/856601 [ Linux ] virtual/omt-worker-fetch/external/wpt/xhr/idlharness.any.worker.html [ Skip ]
 crbug.com/856601 [ Linux ] wpt_internal/std-switch/tentative/interface.html [ Pass Timeout ]
 crbug.com/856601 [ Linux ] http/tests/devtools/a11y-axe-core/elements/main-tool-test.js [ Pass Timeout ]
 crbug.com/856601 [ Linux ] webaudio/Analyser/realtimeanalyser-freq-data.html [ Pass Timeout ]
@@ -322,11 +310,6 @@ crbug.com/856601 [ Linux ] external/wpt/css/filter-effects/idlharness.any.worker
 crbug.com/856601 [ Linux ] external/wpt/fetch/api/idlharness.any.worker.html [ Skip ]
 crbug.com/856601 [ Linux ] external/wpt/html/dom/idlharness.worker.html [ Skip ]
 crbug.com/856601 [ Linux ] external/wpt/mediasession/idlharness.window.html [ Skip ]
-crbug.com/856601 [ Linux ] virtual/cascade/external/wpt/css/css-typed-om/idlharness.html [ Skip ]
-crbug.com/856601 [ Linux ] virtual/scalefactor200/external/wpt/css/filter-effects/idlharness.any.html [ Skip ]
-crbug.com/856601 [ Linux ] virtual/scalefactor200/external/wpt/css/filter-effects/idlharness.any.worker.html [ Skip ]
-crbug.com/856601 [ Linux ] virtual/omt-worker-fetch/external/wpt/fetch/api/idlharness.any.serviceworker.html [ Skip ]
-crbug.com/856601 [ Linux ] virtual/omt-worker-fetch/external/wpt/fetch/api/idlharness.any.sharedworker.html [ Skip ]
 crbug.com/856601 [ Linux ] external/wpt/BackgroundSync/idlharness.https.any.worker.html [ Skip ]
 crbug.com/856601 [ Linux ] external/wpt/fetch/api/idlharness.any.serviceworker.html [ Skip ]
 crbug.com/856601 [ Linux ] external/wpt/BackgroundSync/idlharness.https.any.html [ Skip ]
@@ -337,8 +320,6 @@ crbug.com/856601 [ Linux ] external/wpt/fetch/api/idlharness.any.sharedworker.ht
 crbug.com/856601 [ Linux ] external/wpt/generic-sensor/idlharness.https.window.html [ Skip ]
 crbug.com/856601 [ Linux ] external/wpt/html-media-capture/idlharness.window.html [ Skip ]
 crbug.com/856601 [ Linux ] external/wpt/pointerlock/idlharness.window.html [ Skip ]
-crbug.com/856601 [ Linux ] virtual/omt-worker-fetch/external/wpt/fetch/cors-rfc1918/idlharness.tentative.any.worker.html [ Skip ]
-crbug.com/856601 [ Linux ] virtual/omt-worker-fetch/external/wpt/resource-timing/idlharness.any.worker.html [ Skip ]
 
 # Sheriff 2019-09-06
 crbug.com/856601 [ Linux ] external/wpt/idle-detection/idlharness.https.any.html [ Skip ]
@@ -347,7 +328,6 @@ crbug.com/856601 [ Linux ] external/wpt/idle-detection/idlharness.https.any.html
 
 crbug.com/856601 [ Linux ] external/wpt/web-animations/idlharness.window.html [ Skip ]
 crbug.com/856601 [ Linux ] virtual/controls-refresh/color/color-picker-manual-color-change-invalid-values.html [ Pass Timeout ]
-crbug.com/856601 [ Linux ] virtual/omt-worker-fetch/external/wpt/fetch/cors-rfc1918/idlharness.tentative.any.html [ Skip ]
 
 # Note to sheriffs: Timeouts about "idlharness" or "interfaces" are usually crbug.com/856601.
 crbug.com/856601 [ Linux ] external/wpt/secure-contexts/idlharness.any.html [ Skip ]