[ci/release] Allow for preferring smoke tests when filtering (ray-pro…

…ject#23887)

What: Adds a setting "prefer_smoke_tests" to the Buildkite settings. With this, user can specify to kick off smoke tests, if available.

Why: The filtering interface of the release testing dialog is a bit complicated at the moment - in order to kick off smoke tests, users have to know with which frequency they are configured to run. Instead users should usually just filter the tests they want to run (using frequency ANY) and optionally specify to run smoke tests, if available.

release/ray_release/buildkite/filter.py

@@ -10,6 +10,7 @@ def filter_tests(
     test_collection: List[Test],
     frequency: Frequency,
     test_attr_regex_filters: Optional[Dict[str, str]] = None,
+    prefer_smoke_tests: bool = False,
 ) -> List[Tuple[Test, bool]]:
     if test_attr_regex_filters is None:
         test_attr_regex_filters = {}
@@ -31,7 +32,13 @@ def filter_tests(
             continue
 
         if frequency == Frequency.ANY or frequency == test_frequency:
-            tests_to_run.append((test, False))
+            if prefer_smoke_tests and "smoke_test" in test:
+                # If we prefer smoke tests and a smoke test is available for this test,
+                # then use the smoke test
+                smoke_test = True
+            else:
+                smoke_test = False
+            tests_to_run.append((test, smoke_test))
             continue
 
         elif "smoke_test" in test:

release/ray_release/buildkite/settings.py

@@ -19,6 +19,7 @@ class Frequency(enum.Enum):
 frequency_str_to_enum = {
     "disabled": Frequency.DISABLED,
     "any": Frequency.ANY,
+    "any-smoke": Frequency.ANY,
     "multi": Frequency.MULTI,
     "nightly": Frequency.NIGHTLY,
     "weekly": Frequency.WEEKLY,
@@ -123,6 +124,7 @@ def get_pipeline_settings() -> Dict:
 def get_default_settings() -> Dict:
     settings = {
         "frequency": Frequency.ANY,
+        "prefer_smoke_tests": False,
         "test_attr_regex_filters": None,
         "ray_wheels": None,
         "ray_test_repo": None,
@@ -137,6 +139,13 @@ def update_settings_from_environment(settings: Dict) -> Dict:
     if "RELEASE_FREQUENCY" in os.environ:
         settings["frequency"] = get_frequency(os.environ["RELEASE_FREQUENCY"])
 
+    if "RELEASE_PREFER_SMOKE_TESTS" in os.environ:
+        settings["prefer_smoke_tests"] = bool(
+            int(os.environ["RELEASE_PREFER_SMOKE_TESTS"])
+        )
+    elif os.environ.get("RELEASE_FREQUENCY", "").endswith("-smoke"):
+        settings["prefer_smoke_tests"] = True
+
     if "RAY_TEST_REPO" in os.environ:
         settings["ray_test_repo"] = os.environ["RAY_TEST_REPO"]
         settings["ray_test_branch"] = os.environ.get("RAY_TEST_BRANCH", DEFAULT_BRANCH)
@@ -173,6 +182,8 @@ def update_settings_from_buildkite(settings: Dict):
     release_frequency = get_buildkite_prompt_value("release-frequency")
     if release_frequency:
         settings["frequency"] = get_frequency(release_frequency)
+        if release_frequency.endswith("-smoke"):
+            settings["prefer_smoke_tests"] = True
 
     ray_test_repo_branch = get_buildkite_prompt_value("release-ray-test-repo-branch")
     if ray_test_repo_branch:

release/ray_release/config.py

@@ -10,7 +10,7 @@
 import yaml
 
 from ray_release.anyscale_util import find_cloud_by_name
-from ray_release.exception import ReleaseTestConfigError
+from ray_release.exception import ReleaseTestConfigError, ReleaseTestCLIError
 from ray_release.logger import logger
 from ray_release.util import deep_update
 
@@ -134,11 +134,10 @@ def find_test(test_collection: List[Test], test_name: str) -> Optional[Test]:
 
 def as_smoke_test(test: Test) -> Test:
     if "smoke_test" not in test:
-        logger.warning(
+        raise ReleaseTestCLIError(
             f"Requested smoke test, but test with name {test['name']} does "
             f"not have any smoke test configuration."
         )
-        return test
 
     smoke_test_config = test.pop("smoke_test")
     new_test = deep_update(test, smoke_test_config)

release/ray_release/scripts/build_pipeline.py

@@ -67,13 +67,15 @@ def main(test_collection_file: Optional[str] = None):
         shutil.rmtree(tmpdir, ignore_errors=True)
 
     frequency = settings["frequency"]
+    prefer_smoke_tests = settings["prefer_smoke_tests"]
     test_attr_regex_filters = settings["test_attr_regex_filters"]
     ray_wheels = settings["ray_wheels"]
     priority = settings["priority"]
 
     logger.info(
         f"Found the following buildkite pipeline settings:\n\n"
         f"  frequency =               {settings['frequency']}\n"
+        f"  prefer_smoke_tests =      {settings['prefer_smoke_tests']}\n"
         f"  test_attr_regex_filters = {settings['test_attr_regex_filters']}\n"
         f"  ray_wheels =              {settings['ray_wheels']}\n"
         f"  ray_test_repo =           {settings['ray_test_repo']}\n"
@@ -86,6 +88,7 @@ def main(test_collection_file: Optional[str] = None):
         test_collection,
         frequency=frequency,
         test_attr_regex_filters=test_attr_regex_filters,
+        prefer_smoke_tests=prefer_smoke_tests,
     )
     logger.info(f"Found {len(filtered_tests)} tests to run.")
     if len(filtered_tests) == 0:

release/ray_release/tests/test_buildkite.py

@@ -158,6 +158,23 @@ def testSettingsOverrideEnv(self):
             updated_settings,
             {
                 "frequency": Frequency.NIGHTLY,
+                "prefer_smoke_tests": False,
+                "test_attr_regex_filters": {"name": "name_filter"},
+                "ray_wheels": "custom-wheels",
+                "ray_test_repo": "https://github.com/user/ray.git",
+                "ray_test_branch": "sub/branch",
+                "priority": Priority.MANUAL,
+                "no_concurrency_limit": False,
+            },
+        )
+
+        os.environ["RELEASE_FREQUENCY"] = "any-smoke"
+        update_settings_from_environment(updated_settings)
+        self.assertDictEqual(
+            updated_settings,
+            {
+                "frequency": Frequency.ANY,
+                "prefer_smoke_tests": True,
                 "test_attr_regex_filters": {"name": "name_filter"},
                 "ray_wheels": "custom-wheels",
                 "ray_test_repo": "https://github.com/user/ray.git",
@@ -307,6 +324,24 @@ def testSettingsOverrideBuildkite(self):
                 updated_settings,
                 {
                     "frequency": Frequency.NIGHTLY,
+                    "prefer_smoke_tests": False,
+                    "test_attr_regex_filters": {"name": "name_filter"},
+                    "ray_wheels": "custom-wheels",
+                    "ray_test_repo": "https://github.com/user/ray.git",
+                    "ray_test_branch": "sub/branch",
+                    "priority": Priority.MANUAL,
+                    "no_concurrency_limit": False,
+                },
+            )
+
+            self.buildkite["release-frequency"] = "any-smoke"
+            update_settings_from_buildkite(updated_settings)
+
+            self.assertDictEqual(
+                updated_settings,
+                {
+                    "frequency": Frequency.ANY,
+                    "prefer_smoke_tests": True,
                     "test_attr_regex_filters": {"name": "name_filter"},
                     "ray_wheels": "custom-wheels",
                     "ray_test_repo": "https://github.com/user/ray.git",
@@ -363,6 +398,22 @@ def testFilterTests(self):
             ],
         )
 
+        filtered = self._filter_names_smoke(
+            tests,
+            frequency=Frequency.ANY,
+            prefer_smoke_tests=True,
+        )
+        self.assertSequenceEqual(
+            filtered,
+            [
+                ("test_1", True),
+                ("test_2", True),
+                ("other_1", False),
+                ("other_2", True),
+                ("test_3", False),
+            ],
+        )
+
         filtered = self._filter_names_smoke(tests, frequency=Frequency.NIGHTLY)
         self.assertSequenceEqual(
             filtered,
@@ -374,6 +425,21 @@ def testFilterTests(self):
             ],
         )
 
+        filtered = self._filter_names_smoke(
+            tests,
+            frequency=Frequency.NIGHTLY,
+            prefer_smoke_tests=True,
+        )
+        self.assertSequenceEqual(
+            filtered,
+            [
+                ("test_1", True),
+                ("test_2", True),
+                ("other_2", True),
+                ("test_3", False),
+            ],
+        )
+
         filtered = self._filter_names_smoke(tests, frequency=Frequency.WEEKLY)
         self.assertSequenceEqual(filtered, [("test_2", False), ("other_1", False)])
 

release/ray_release/tests/test_wheels.py

@@ -19,7 +19,9 @@
 
 class WheelsFinderTest(unittest.TestCase):
     def setUp(self) -> None:
-        pass
+        for key in os.environ:
+            if key.startswith("BUILDKITE"):
+                os.environ.pop(key)
 
     def testGetRayVersion(self):
         init_file = os.path.join(
@@ -72,8 +74,6 @@ def testFindRayWheelsBuildkite(self):
 
     @patch("ray_release.wheels.get_ray_version", lambda *a, **kw: "2.0.0.dev0")
     def testFindRayWheelsCommitOnly(self):
-        os.environ.pop("BUILDKITE_BRANCH")
-
         repo = DEFAULT_REPO
         branch = "master"
         commit = "1234" * 10