flake_suppressor tool: Generalize queries and tag_utils code.

verified with -
./suppress_flakes.py --project  chrome-unexpected-pass-data --sample-period 2  --bypass-up-to-date-check --no-prompt-for-user-input

Bug: 1358733

Change-Id: Ia4d0eb308b921b1b4abf4dc0567bf025aa1d3791
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3880831
Commit-Queue: Preethi Mohan <preethim@google.com>
Reviewed-by: Brian Sheedy <bsheedy@chromium.org>
Reviewed-by: Erik Staab <estaab@chromium.org>
Cr-Commit-Position: refs/heads/main@{#1046652}

content/test/BUILD.gn

@@ -3134,6 +3134,7 @@ group("gpu_pytype") {
     "//content/test/gpu/validate_tag_consistency.py",
 
     "//content/test/gpu/gold_inexact_matching/",
+    "//content/test/gpu/flake_suppressor/",
     "//testing/flake_suppressor_common/",
     "//content/test/gpu/unexpected_passes/",
     "//testing/unexpected_passes_common/",

content/test/gpu/PRESUBMIT.py

@@ -67,6 +67,17 @@ def CommonChecks(input_api, output_api):
           run_on_python3=True,
           skip_shebang_check=True))
 
+  results.extend(
+      input_api.canned_checks.RunUnitTestsInDirectory(
+          input_api,
+          output_api,
+          input_api.os_path.join(input_api.PresubmitLocalPath(),
+                                 'flake_suppressor'), [r'^.+_unittest\.py$'],
+          env=gpu_env,
+          run_on_python2=False,
+          run_on_python3=True,
+          skip_shebang_check=True))
+
   pylint_extra_paths = [
       input_api.os_path.join(chromium_src_path, *component)
       for component in EXTRA_PATHS_COMPONENTS

content/test/gpu/flake_suppressor/__init__.py

@@ -0,0 +1,7 @@
+# Copyright 2022 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+from gpu_path_util import setup_telemetry_paths  # pylint: disable=unused-import
+from gpu_path_util import setup_testing_paths  # pylint: disable=unused-import
+from gpu_path_util import setup_typ_paths  # pylint: disable=unused-import

content/test/gpu/flake_suppressor/gpu_queries.py

@@ -0,0 +1,145 @@
+# Copyright 2022 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+from flake_suppressor_common import queries as queries_module
+
+SUBMITTED_BUILDS_SUBQUERY = """\
+  submitted_builds AS ({chromium_builds}
+  ),""".format(chromium_builds=queries_module.SUBMITTED_BUILDS_TEMPLATE)
+
+# Gets all failures from the past |sample_period| days from CI bots that did not
+# already have an associated test suppression when the test ran.
+CI_FAILED_TEST_QUERY = """\
+WITH
+  failed_tests AS (
+    SELECT
+      exported.id,
+      test_metadata.name,
+      ARRAY(
+        SELECT value
+        FROM tr.tags
+        WHERE key = "typ_tag") as typ_tags,
+      ARRAY(
+        SELECT value
+        FROM tr.tags
+        WHERE key = "raw_typ_expectation") as typ_expectations
+    FROM
+      `chrome-luci-data.chromium.gpu_ci_test_results` tr
+    WHERE
+      status = "FAIL"
+      AND exported.realm = "chromium:ci"
+      AND partition_time > TIMESTAMP_SUB(CURRENT_TIMESTAMP(),
+                                         INTERVAL @sample_period DAY)
+  )
+SELECT *
+FROM failed_tests ft
+WHERE
+  ARRAY_TO_STRING(ft.typ_expectations, '') = "Pass"
+"""
+
+# Gets all failures from the past |sample_period| days from trybots that did not
+# already have an associated test suppresssion when the test ran, only including
+# data from builds that were used for CL submission.
+TRY_FAILED_TEST_QUERY = """\
+WITH
+  {submitted_builds_subquery}
+  failed_tests AS (
+    SELECT
+      exported.id,
+      test_metadata.name,
+      ARRAY(
+        SELECT value
+        FROM tr.tags
+        WHERE key = "typ_tag") as typ_tags,
+      ARRAY(
+        SELECT value
+        FROM tr.tags
+        WHERE key = "raw_typ_expectation") as typ_expectations
+    FROM
+      `chrome-luci-data.chromium.gpu_try_test_results` tr,
+      submitted_builds sb
+    WHERE
+      status = "FAIL"
+      AND exported.realm = "chromium:try"
+      AND partition_time > TIMESTAMP_SUB(CURRENT_TIMESTAMP(),
+                                         INTERVAL @sample_period DAY)
+      AND exported.id = sb.id
+  )
+SELECT *
+FROM failed_tests ft
+WHERE
+  ARRAY_TO_STRING(ft.typ_expectations, '') = "Pass"
+""".format(submitted_builds_subquery=SUBMITTED_BUILDS_SUBQUERY)
+
+# Gets the count of all results in the past |sample_period| days for distinct
+# test/tag combinations from CI bots.
+CI_RESULT_COUNT_QUERY = """\
+WITH
+  grouped_results AS (
+    SELECT
+      exported.id as id,
+      test_metadata.name as name,
+      ARRAY(
+        SELECT value
+        FROM tr.tags
+        WHERE key = "typ_tag") as typ_tags
+    FROM
+      `chrome-luci-data.chromium.gpu_ci_test_results` tr
+    WHERE
+      exported.realm = "chromium:ci"
+      AND partition_time > TIMESTAMP_SUB(CURRENT_TIMESTAMP(),
+                                         INTERVAL @sample_period DAY)
+  )
+SELECT
+  COUNT(gr.id) as result_count,
+  ANY_VALUE(gr.name) as test_name,
+  ANY_VALUE(gr.typ_tags) as typ_tags
+FROM grouped_results gr
+GROUP BY gr.name, ARRAY_TO_STRING(gr.typ_tags, '')
+"""
+
+# Gets the count of all results in the past |sample_period| days for distinct
+# test/tag combinations from trybots, only including data from builds that were
+# used for CL submission.
+TRY_RESULT_COUNT_QUERY = """\
+WITH
+  {submitted_builds_subquery}
+  grouped_results AS (
+    SELECT
+      exported.id as id,
+      test_metadata.name as name,
+      ARRAY(
+        SELECT value
+        FROM tr.tags
+        WHERE key = "typ_tag") as typ_tags
+    FROM
+      `chrome-luci-data.chromium.gpu_try_test_results` tr,
+      submitted_builds sb
+    WHERE
+      exported.realm = "chromium:try"
+      AND partition_time > TIMESTAMP_SUB(CURRENT_TIMESTAMP(),
+                                         INTERVAL @sample_period DAY)
+      AND exported.id = sb.id
+  )
+SELECT
+  COUNT(gr.id) as result_count,
+  ANY_VALUE(gr.name) as test_name,
+  ANY_VALUE(gr.typ_tags) as typ_tags
+FROM grouped_results gr
+GROUP BY gr.name, ARRAY_TO_STRING(gr.typ_tags, '')
+""".format(submitted_builds_subquery=SUBMITTED_BUILDS_SUBQUERY)
+
+
+class GpuBigQueryQuerier(queries_module.BigQueryQuerier):
+  def GetFlakyOrFailingCiQuery(self) -> str:
+    return CI_FAILED_TEST_QUERY
+
+  def GetFlakyOrFailingTryQuery(self) -> str:
+    return TRY_FAILED_TEST_QUERY
+
+  def GetResultCountCIQuery(self) -> str:
+    return CI_RESULT_COUNT_QUERY
+
+  def GetResultCountTryQuery(self) -> str:
+    return TRY_RESULT_COUNT_QUERY

content/test/gpu/flake_suppressor/gpu_queries_unittest.py

@@ -0,0 +1,89 @@
+#!/usr/bin/env vpython3
+# Copyright 2022 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# pylint: disable=protected-access
+
+import json
+import unittest
+import unittest.mock as mock
+
+from flake_suppressor import gpu_queries as queries
+from flake_suppressor import gpu_tag_utils as tag_utils
+from flake_suppressor_common import unittest_utils as uu
+from flake_suppressor_common import tag_utils as common_tag_utils
+
+
+class GpuQueriesUnittest(unittest.TestCase):
+  def setUp(self) -> None:
+    common_tag_utils.SetTagUtilsImplementation(tag_utils.GpuTagUtils)
+    self._querier_instance = queries.GpuBigQueryQuerier(1, 'project')
+    self._querier_instance._submitted_builds = set(['build-1234', 'build-2345'])
+    self._subprocess_patcher = mock.patch(
+        'flake_suppressor_common.queries.subprocess.run')
+    self._subprocess_mock = self._subprocess_patcher.start()
+    self.addCleanup(self._subprocess_patcher.stop)
+
+  def testIgnoredTags(self) -> None:
+    """Tests that ignored tags are removed and their counts merged."""
+
+    def SideEffect(*_, **kwargs) -> uu.FakeProcess:
+      query = kwargs['input']
+      if 'submitted_builds' in query:
+        # Try results.
+        query_result = [
+            {
+                'typ_tags': ['linux', 'nvidia'],
+                'test_name': 'garbage.suite.garbage.linux',
+                'result_count': '25',
+            },
+            {
+                'typ_tags': ['linux', 'win-laptop'],
+                'test_name': 'garbage.suite.garbage.linux',
+                'result_count': '50',
+            },
+        ]
+      else:
+        # CI results.
+        query_result = [{
+            'typ_tags': ['win', 'win-laptop'],
+            'test_name': 'garbage.suite.garbage.windows',
+            'result_count': '100',
+        }, {
+            'typ_tags': ['win'],
+            'test_name': 'garbage.suite.garbage.windows',
+            'result_count': '50',
+        }, {
+            'typ_tags': ['mac', 'exact'],
+            'test_name': 'garbage.suite.garbage.mac',
+            'result_count': '200',
+        }, {
+            'typ_tags': ['linux'],
+            'test_name': 'garbage.suite.garbage.linux',
+            'result_count': '300',
+        }]
+      return uu.FakeProcess(stdout=json.dumps(query_result))
+
+    self._subprocess_mock.side_effect = SideEffect
+    result_counts = self._querier_instance.GetResultCounts()
+    expected_result_counts = {
+        tuple(['win']): {
+            'windows': 150,
+        },
+        tuple(['mac']): {
+            'mac': 200,
+        },
+        tuple(['linux']): {
+            'linux': 350,
+        },
+        tuple(['linux', 'nvidia']): {
+            'linux': 25,
+        },
+    }
+    self.assertEqual(result_counts, expected_result_counts)
+    self.assertEqual(self._subprocess_mock.call_count, 2)
+
+
+if __name__ == '__main__':
+  unittest.main(verbosity=2)

content/test/gpu/flake_suppressor/gpu_tag_utils.py

@@ -0,0 +1,34 @@
+# Copyright 2022 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+"""Module for tag-related helper functions."""
+
+from typing import Iterable
+
+from flake_suppressor_common import common_typing as ct
+from flake_suppressor_common import tag_utils
+
+from gpu_tests import gpu_integration_test
+
+IGNORED_TAGS_TO_TEMPORARILY_KEEP = set([
+    'webgl-version-1',
+    'webgl-version-2',
+])
+
+
+class GpuTagUtils(tag_utils.BaseTagUtils):
+  def RemoveMostIgnoredTags(self, tags: Iterable[str]) -> ct.TagTupleType:
+    ignored_tags = set(gpu_integration_test.GpuIntegrationTest.IgnoredTags())
+    tags = set(tags)
+    ignored_tags_to_keep = tags & IGNORED_TAGS_TO_TEMPORARILY_KEEP
+    tags -= ignored_tags
+    tags |= ignored_tags_to_keep
+    tags = list(tags)
+    tags.sort()
+    return tuple(tags)
+
+  def RemoveTemporarilyKeptIgnoredTags(self,
+                                       tags: Iterable[str]) -> ct.TagTupleType:
+    tags = list(set(tags) - IGNORED_TAGS_TO_TEMPORARILY_KEEP)
+    tags.sort()
+    return tuple(tags)

testing/flake_suppressor_common/tag_utils_unittest.py → content/test/gpu/flake_suppressor/gpu_tag_utils_unittest.py

@@ -5,13 +5,13 @@
 
 import unittest
 
-from flake_suppressor_common import tag_utils
+from flake_suppressor import gpu_tag_utils as tag_utils
 
 
 class RemoveMostIgnoredTagsUnittest(unittest.TestCase):
   def testBasic(self) -> None:
     tags = ['win', 'win-laptop', 'webgl-version-1']
-    filtered_tags = tag_utils.RemoveMostIgnoredTags(tags)
+    filtered_tags = tag_utils.GpuTagUtils().RemoveMostIgnoredTags(tags)
     self.assertEqual(filtered_tags, ('webgl-version-1', 'win'))
 
 
@@ -21,7 +21,8 @@ def testBasic(self) -> None:
     # by RemoveMostIgnoredTags(), but since this is *only* supposed to remove
     # temporarily kept ignored tags, include it to test that.
     tags = ['win', 'win-laptop', 'webgl-version-1', 'amd']
-    filtered_tags = tag_utils.RemoveTemporarilyKeptIgnoredTags(tags)
+    filtered_tags = tag_utils.GpuTagUtils().RemoveTemporarilyKeptIgnoredTags(
+        tags)
     self.assertEqual(filtered_tags, ('amd', 'win', 'win-laptop'))
 
 

content/test/gpu/run_pytype.py

@@ -34,6 +34,7 @@
 EXTRA_PATHS.append(GPU_DIR)
 
 FILES_AND_DIRECTORIES_TO_CHECK = [
+    'flake_suppressor',
     'gold_inexact_matching',
     'gpu_tests',
     'unexpected_passes',

content/test/gpu/suppress_flakes.py

@@ -25,9 +25,11 @@
 
 # pylint: disable=wrong-import-position
 from flake_suppressor_common import expectations
-from flake_suppressor_common import queries
 from flake_suppressor_common import result_output
 from flake_suppressor_common import results as results_module
+from flake_suppressor_common import tag_utils as common_tag_utils
+from flake_suppressor import gpu_queries
+from flake_suppressor import gpu_tag_utils as tag_utils
 # pylint: enable=wrong-import-position
 
 
@@ -108,9 +110,11 @@ def ParseArgs():
 
 def main():
   args = ParseArgs()
+  common_tag_utils.SetTagUtilsImplementation(tag_utils.GpuTagUtils)
   if not args.bypass_up_to_date_check:
     expectations.AssertCheckoutIsUpToDate()
-  querier_instance = queries.BigQueryQuerier(args.sample_period, args.project)
+  querier_instance = gpu_queries.GpuBigQueryQuerier(args.sample_period,
+                                                    args.project)
   results = querier_instance.GetFlakyOrFailingCiTests()
   results.extend(querier_instance.GetFlakyOrFailingTryTests())
   aggregated_results = results_module.AggregateResults(results)

testing/flake_suppressor_common/expectations.py

@@ -302,7 +302,7 @@ def ModifyFileForResult(suite: str, test: str, typ_tags: ct.TagTupleType,
     # Remove temporarily un-ignored tags, namely webgl-version-x tags, since
     # those were necessary to find the correct file. However, we do not want
     # to actually include them in the file since they are unused/ignored.
-    typ_tags = tag_utils.RemoveTemporarilyKeptIgnoredTags(typ_tags)
+    typ_tags = tag_utils.TagUtils.RemoveTemporarilyKeptIgnoredTags(typ_tags)
     typ_tags = FilterToMostSpecificTypTags(typ_tags, expectation_file)
   bug = '%s ' % bug if bug else bug