Support diffing results between content shell and Chrome linux

plus some minor changes:
1. mention product name in CSV head explicitly
2. Add --ignore-missing to ignore legacy tests on content shell
3. Do not handle flakiness if no flaky data found for some products

Change-Id: Ia221e1fac535a03317b12ea1fee66878717da1ba
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3077538
Reviewed-by: Rakib Hasan <rmhasan@google.com>
Commit-Queue: Weizhong Xia <weizhong@google.com>
Cr-Commit-Position: refs/heads/master@{#909938}

Author: WeizhongX

third_party/blink/tools/diff_wpt_results.py

@@ -38,15 +38,32 @@
 from blinkpy.web_tests.port.android import (
     PRODUCTS, PRODUCTS_TO_STEPNAMES)
 
-CSV_HEADING = ('Test name, Test Result, Baseline Result, '
+CSV_HEADING = ('Test name, %s Result, %s Result, '
                'Result Comparison, Test Flaky Results, '
                'Baseline Flaky Results, Unreliable Comparison\n')
 YES = 'Yes'
 NO = 'No'
 _log = logging.getLogger(os.path.basename(__file__))
+
+# Extend this script to compare the results between wptrunner/Chrome
+# and rwt/content_shell on Linux
+PRODUCTS = PRODUCTS + ['chrome_linux', 'content_shell']
+PRODUCTS_TO_STEPNAMES.update({
+    'chrome_linux': 'wpt_tests_suite',
+    'content_shell': 'blink_web_tests'})
+PRODUCTS_TO_BUILDER_NAME = {
+    'android_weblayer': 'android-weblayer-pie-x86-wpt-fyi-rel',
+    'android_webview': 'android-webview-pie-x86-wpt-fyi-rel',
+    'chrome_android': 'android-web-platform-pie-x86-fyi-rel',
+    'chrome_linux': 'linux-wpt-fyi-rel',
+    'content_shell': "Linux Tests"}
+
 STEP_NAME_VARIANTS = {
     'chrome_public_wpt': ['chrome_public_wpt on Ubuntu-16.04 or Ubuntu-18.04'],
-    'weblayer_shell_wpt': ['weblayer_shell_wpt on Ubuntu-16.04 or Ubuntu-18.04']
+    'weblayer_shell_wpt': ['weblayer_shell_wpt on Ubuntu-16.04 or Ubuntu-18.04'],
+    'system_webview_wpt': ['system_webview_wpt on Ubuntu-16.04 or Ubuntu-18.04'],
+    'wpt_tests_suite': ['wpt_tests_suite on Ubuntu-18.04'],
+    'blink_web_tests': ['blink_web_tests on Ubuntu-18.04']
 }
 
 def map_tests_to_results(output_mp, input_mp, path=''):
@@ -60,30 +77,38 @@ def map_tests_to_results(output_mp, input_mp, path=''):
 class WPTResultsDiffer(object):
 
     def __init__(self, args, host, actual_results_map,
-                 baseline_results_map, csv_output):
+                 baseline_results_map, csv_output, ignore_missing=False):
         self._args = args
         self._host = host
         self._actual_results_map = actual_results_map
         self._baseline_results_map = baseline_results_map
         self._csv_output = csv_output
-        self._test_flaky_results = self._get_flaky_test_results(
-            args.product_to_compare)
-        self._baseline_flaky_results = self._get_flaky_test_results(
-            args.baseline_product)
+        self._ignore_missing = ignore_missing
+        self._test_flaky_results = None
+        self._baseline_flaky_results = None
+
+        try:
+            self._test_flaky_results = self._get_flaky_test_results(
+                args.product_to_compare)
+        except:
+            _log.info('Failed to get flaky results for %s' % args.product_to_compare)
+
+        try:
+            self._baseline_flaky_results = self._get_flaky_test_results(
+                args.baseline_product)
+        except:
+            _log.info('Failed to get flaky results for %s' % args.baseline_product)
+
+        self._handle_flaky = self._test_flaky_results is not None \
+            and self._baseline_flaky_results is not None
 
     def _get_flaky_test_results(self, product):
         return self._get_bot_expectations(product).flakes_by_path(
             False, ignore_bot_expected_results=True,
             consider_only_flaky_runs=False)
 
     def _get_bot_expectations(self, product):
-        specifiers = [product]
-        builders = self._host.builders.filter_builders(
-            include_specifiers=specifiers)
-        assert len(builders) == 1, (
-            'Multiple builders match the specifiers %s' % specifiers)
-
-        builder_name = builders[0]
+        builder_name = PRODUCTS_TO_BUILDER_NAME[product]
         bot_expectations_factory = BotTestExpectationsFactory(
             self._host.builders, PRODUCTS_TO_STEPNAMES[product])
 
@@ -96,7 +121,8 @@ def flaky_results(self, test_name, flaky_dict):
     def create_csv(self):
         super_set = (set(self._actual_results_map.keys()) |
                      set(self._baseline_results_map.keys()))
-        file_output = CSV_HEADING
+        file_output = CSV_HEADING % (self._args.product_to_compare,
+                                     self._args.baseline_product)
 
         for test in sorted(super_set):
             if ',' in test:
@@ -112,11 +138,13 @@ def create_csv(self):
             if line[-1] == line[-2]:
                 line.append('SAME RESULTS')
             elif 'MISSING' in (line[-1], line[-2]):
+                if self._ignore_missing:
+                    continue
                 line.append('MISSING RESULTS')
             else:
                 line.append('DIFFERENT RESULTS')
 
-            if line[-1] != 'MISSING RESULTS':
+            if self._handle_flaky and line[-1] != 'MISSING RESULTS':
                 test_flaky_results = self.flaky_results(
                     test, self._test_flaky_results)
 
@@ -169,19 +197,15 @@ def _get_product_test_results(host, product, results_path=None):
     else:
         _log.info(('Retrieving test results for '
                    'product %s using the bb command'), product)
-        specifiers = [product]
-        builders = host.builders.filter_builders(
-            include_specifiers=specifiers)
-        assert len(builders) == 1
-
-        builder_name = builders[0]
+        builder_name = PRODUCTS_TO_BUILDER_NAME[product]
+        # TODO: Note the builder name and number in the CSV file
         latest_build = host.bb_agent.get_latest_finished_build(
             builder_name)
         _log.debug('The latest build for %s is %d',
                    builder_name, latest_build.build_number)
 
         build_results = _get_build_test_results(host, product, latest_build)
-        json_results_obj = tempfile.TemporaryFile()
+        json_results_obj = tempfile.NamedTemporaryFile()
         json_results_obj.write(json.dumps(build_results))
         json_results_obj.seek(0)
 
@@ -197,16 +221,19 @@ def main(args):
                         help='Path to baseline test results JSON file')
     parser.add_argument('--baseline-product', required=True, action='store',
                         choices=PRODUCTS,
-                        help='Name of the baseline WPT product')
+                        help='Name of the baseline product')
     parser.add_argument('--test-results-to-compare', required=False,
                         help='Path to actual test results JSON file')
     parser.add_argument('--product-to-compare', required=True, action='store',
                         choices=PRODUCTS,
-                        help='Name of the WPT product being compared')
+                        help='Name of the product being compared')
     parser.add_argument('--csv-output', required=True,
                         help='Path to CSV output file')
     parser.add_argument('--verbose', '-v', action='count', default=1,
                         help='Verbosity level')
+    parser.add_argument('--ignore-missing', action='store_true',
+                        required=False, default=False,
+                        help='Ignore tests that are not run for one of the product')
     args = parser.parse_args()
 
     if args.verbose >= 3:
@@ -240,14 +267,26 @@ def main(args):
         # names to their results map
         tests_to_actual_results = {}
         tests_to_baseline_results = {}
+        if args.product_to_compare == 'chrome_linux':
+            path = '/external/wpt'
+        else:
+            path = ''
         map_tests_to_results(tests_to_actual_results,
-                             actual_results_json['tests'])
+                             actual_results_json['tests'],
+                             path=path)
+
+        if args.baseline_product == 'chrome_linux':
+            path = '/external/wpt'
+        else:
+            path = ''
         map_tests_to_results(tests_to_baseline_results,
-                             baseline_results_json['tests'])
+                             baseline_results_json['tests'],
+                             path=path)
 
         # Create a CSV file which compares tests results to baseline results
         WPTResultsDiffer(args, host, tests_to_actual_results,
-                         tests_to_baseline_results, csv_output).create_csv()
+                         tests_to_baseline_results, csv_output,
+                         args.ignore_missing).create_csv()
 
     return 0
 

third_party/blink/tools/diff_wpt_results_unittest.py

@@ -86,7 +86,8 @@ def test_name_with_comma_escaped_in_csv(self):
             MockWPTResultsDiffer(actual_mp, actual_mp, csv_out).create_csv()
             csv_out.seek(0)
             content = csv_out.read()
-            self.assertEquals(content, CSV_HEADING +
+            heading = CSV_HEADING % (TEST_PRODUCT, TEST_BASELINE_PRODUCT)
+            self.assertEquals(content, heading +
                               ('"test, name.html",PASS,PASS,'
                                'SAME RESULTS,"{FAIL, TIMEOUT, PASS}",'
                                '"{CRASH, PASS}",Yes\n'))
@@ -97,7 +98,8 @@ def test_create_csv_with_same_result(self):
             MockWPTResultsDiffer(actual_mp, actual_mp, csv_out).create_csv()
             csv_out.seek(0)
             content = csv_out.read()
-            self.assertEquals(content, CSV_HEADING +
+            heading = CSV_HEADING % (TEST_PRODUCT, TEST_BASELINE_PRODUCT)
+            self.assertEquals(content, heading +
                               ('test.html,PASS,PASS,SAME RESULTS,'
                                '"{FAIL, TIMEOUT, PASS}","{CRASH, PASS}",Yes\n'))
 
@@ -109,7 +111,8 @@ def test_create_csv_with_reliable_different_result(self):
             MockWPTResultsDiffer(actual_mp, baseline_mp, csv_out).create_csv()
             csv_out.seek(0)
             content = csv_out.read()
-            self.assertEquals(content, CSV_HEADING +
+            heading = CSV_HEADING % (TEST_PRODUCT, TEST_BASELINE_PRODUCT)
+            self.assertEquals(content, heading +
                               ('test.html,PASS,FAIL,DIFFERENT RESULTS,'
                                '"{FAIL, TIMEOUT, PASS}","{FAIL, CRASH}",No\n'))
 
@@ -121,7 +124,8 @@ def test_create_csv_with_unreliable_different_result(self):
             MockWPTResultsDiffer(actual_mp, baseline_mp, csv_out).create_csv()
             csv_out.seek(0)
             content = csv_out.read()
-            self.assertEquals(content, CSV_HEADING +
+            heading = CSV_HEADING % (TEST_PRODUCT, TEST_BASELINE_PRODUCT)
+            self.assertEquals(content, heading +
                               ('test.html,CRASH,FAIL,DIFFERENT RESULTS,'
                                '"{FAIL, CRASH, TIMEOUT}","{FAIL, CRASH}",Yes\n'))
 
@@ -131,7 +135,8 @@ def test_create_csv_with_missing_result(self):
             MockWPTResultsDiffer(actual_mp, {}, csv_out).create_csv()
             csv_out.seek(0)
             content = csv_out.read()
-            self.assertEquals(content, CSV_HEADING +
+            heading = CSV_HEADING % (TEST_PRODUCT, TEST_BASELINE_PRODUCT)
+            self.assertEquals(content, heading +
                               'test.html,PASS,MISSING,MISSING RESULTS,{},{},No\n')
 
     def test_use_bb_to_get_results(self):
@@ -175,7 +180,8 @@ def process_cmds(cmd_args):
                                  csv_out).create_csv()
             csv_out.seek(0)
             content = csv_out.read()
-            self.assertEquals(content, CSV_HEADING +
+            heading = CSV_HEADING % (TEST_PRODUCT, TEST_BASELINE_PRODUCT)
+            self.assertEquals(content, heading +
                               ('test.html,PASS,FAIL,DIFFERENT RESULTS,'
                                '"{FAIL, TIMEOUT, PASS}","{FAIL, CRASH}",No\n'))