From 6e37a486323fed2e3dd41b929fdcea74e7eb20b4 Mon Sep 17 00:00:00 2001 From: Kai Fricke Date: Thu, 14 Apr 2022 06:12:27 +0100 Subject: [PATCH] [ci/release] Allow for preferring smoke tests when filtering (#23887) What: Adds a setting "prefer_smoke_tests" to the Buildkite settings. With this, user can specify to kick off smoke tests, if available. Why: The filtering interface of the release testing dialog is a bit complicated at the moment - in order to kick off smoke tests, users have to know with which frequency they are configured to run. Instead users should usually just filter the tests they want to run (using frequency ANY) and optionally specify to run smoke tests, if available. --- release/ray_release/buildkite/filter.py | 9 ++- release/ray_release/buildkite/settings.py | 11 ++++ release/ray_release/config.py | 5 +- release/ray_release/scripts/build_pipeline.py | 3 + release/ray_release/tests/test_buildkite.py | 66 +++++++++++++++++++ release/ray_release/tests/test_wheels.py | 6 +- 6 files changed, 93 insertions(+), 7 deletions(-) diff --git a/release/ray_release/buildkite/filter.py b/release/ray_release/buildkite/filter.py index fb03b9a610ec..499b21e7eb75 100644 --- a/release/ray_release/buildkite/filter.py +++ b/release/ray_release/buildkite/filter.py @@ -10,6 +10,7 @@ def filter_tests( test_collection: List[Test], frequency: Frequency, test_attr_regex_filters: Optional[Dict[str, str]] = None, + prefer_smoke_tests: bool = False, ) -> List[Tuple[Test, bool]]: if test_attr_regex_filters is None: test_attr_regex_filters = {} @@ -31,7 +32,13 @@ def filter_tests( continue if frequency == Frequency.ANY or frequency == test_frequency: - tests_to_run.append((test, False)) + if prefer_smoke_tests and "smoke_test" in test: + # If we prefer smoke tests and a smoke test is available for this test, + # then use the smoke test + smoke_test = True + else: + smoke_test = False + tests_to_run.append((test, smoke_test)) continue elif "smoke_test" in test: diff --git a/release/ray_release/buildkite/settings.py b/release/ray_release/buildkite/settings.py index a3d98654ffdc..aa6f732055fb 100644 --- a/release/ray_release/buildkite/settings.py +++ b/release/ray_release/buildkite/settings.py @@ -19,6 +19,7 @@ class Frequency(enum.Enum): frequency_str_to_enum = { "disabled": Frequency.DISABLED, "any": Frequency.ANY, + "any-smoke": Frequency.ANY, "multi": Frequency.MULTI, "nightly": Frequency.NIGHTLY, "weekly": Frequency.WEEKLY, @@ -123,6 +124,7 @@ def get_pipeline_settings() -> Dict: def get_default_settings() -> Dict: settings = { "frequency": Frequency.ANY, + "prefer_smoke_tests": False, "test_attr_regex_filters": None, "ray_wheels": None, "ray_test_repo": None, @@ -137,6 +139,13 @@ def update_settings_from_environment(settings: Dict) -> Dict: if "RELEASE_FREQUENCY" in os.environ: settings["frequency"] = get_frequency(os.environ["RELEASE_FREQUENCY"]) + if "RELEASE_PREFER_SMOKE_TESTS" in os.environ: + settings["prefer_smoke_tests"] = bool( + int(os.environ["RELEASE_PREFER_SMOKE_TESTS"]) + ) + elif os.environ.get("RELEASE_FREQUENCY", "").endswith("-smoke"): + settings["prefer_smoke_tests"] = True + if "RAY_TEST_REPO" in os.environ: settings["ray_test_repo"] = os.environ["RAY_TEST_REPO"] settings["ray_test_branch"] = os.environ.get("RAY_TEST_BRANCH", DEFAULT_BRANCH) @@ -173,6 +182,8 @@ def update_settings_from_buildkite(settings: Dict): release_frequency = get_buildkite_prompt_value("release-frequency") if release_frequency: settings["frequency"] = get_frequency(release_frequency) + if release_frequency.endswith("-smoke"): + settings["prefer_smoke_tests"] = True ray_test_repo_branch = get_buildkite_prompt_value("release-ray-test-repo-branch") if ray_test_repo_branch: diff --git a/release/ray_release/config.py b/release/ray_release/config.py index 9d1ed45de861..11d3f38a702d 100644 --- a/release/ray_release/config.py +++ b/release/ray_release/config.py @@ -10,7 +10,7 @@ import yaml from ray_release.anyscale_util import find_cloud_by_name -from ray_release.exception import ReleaseTestConfigError +from ray_release.exception import ReleaseTestConfigError, ReleaseTestCLIError from ray_release.logger import logger from ray_release.util import deep_update @@ -134,11 +134,10 @@ def find_test(test_collection: List[Test], test_name: str) -> Optional[Test]: def as_smoke_test(test: Test) -> Test: if "smoke_test" not in test: - logger.warning( + raise ReleaseTestCLIError( f"Requested smoke test, but test with name {test['name']} does " f"not have any smoke test configuration." ) - return test smoke_test_config = test.pop("smoke_test") new_test = deep_update(test, smoke_test_config) diff --git a/release/ray_release/scripts/build_pipeline.py b/release/ray_release/scripts/build_pipeline.py index fdf5e4288a9f..e6316024120d 100644 --- a/release/ray_release/scripts/build_pipeline.py +++ b/release/ray_release/scripts/build_pipeline.py @@ -67,6 +67,7 @@ def main(test_collection_file: Optional[str] = None): shutil.rmtree(tmpdir, ignore_errors=True) frequency = settings["frequency"] + prefer_smoke_tests = settings["prefer_smoke_tests"] test_attr_regex_filters = settings["test_attr_regex_filters"] ray_wheels = settings["ray_wheels"] priority = settings["priority"] @@ -74,6 +75,7 @@ def main(test_collection_file: Optional[str] = None): logger.info( f"Found the following buildkite pipeline settings:\n\n" f" frequency = {settings['frequency']}\n" + f" prefer_smoke_tests = {settings['prefer_smoke_tests']}\n" f" test_attr_regex_filters = {settings['test_attr_regex_filters']}\n" f" ray_wheels = {settings['ray_wheels']}\n" f" ray_test_repo = {settings['ray_test_repo']}\n" @@ -86,6 +88,7 @@ def main(test_collection_file: Optional[str] = None): test_collection, frequency=frequency, test_attr_regex_filters=test_attr_regex_filters, + prefer_smoke_tests=prefer_smoke_tests, ) logger.info(f"Found {len(filtered_tests)} tests to run.") if len(filtered_tests) == 0: diff --git a/release/ray_release/tests/test_buildkite.py b/release/ray_release/tests/test_buildkite.py index ed52b649aab7..419714a9959b 100644 --- a/release/ray_release/tests/test_buildkite.py +++ b/release/ray_release/tests/test_buildkite.py @@ -158,6 +158,23 @@ def testSettingsOverrideEnv(self): updated_settings, { "frequency": Frequency.NIGHTLY, + "prefer_smoke_tests": False, + "test_attr_regex_filters": {"name": "name_filter"}, + "ray_wheels": "custom-wheels", + "ray_test_repo": "https://github.com/user/ray.git", + "ray_test_branch": "sub/branch", + "priority": Priority.MANUAL, + "no_concurrency_limit": False, + }, + ) + + os.environ["RELEASE_FREQUENCY"] = "any-smoke" + update_settings_from_environment(updated_settings) + self.assertDictEqual( + updated_settings, + { + "frequency": Frequency.ANY, + "prefer_smoke_tests": True, "test_attr_regex_filters": {"name": "name_filter"}, "ray_wheels": "custom-wheels", "ray_test_repo": "https://github.com/user/ray.git", @@ -307,6 +324,24 @@ def testSettingsOverrideBuildkite(self): updated_settings, { "frequency": Frequency.NIGHTLY, + "prefer_smoke_tests": False, + "test_attr_regex_filters": {"name": "name_filter"}, + "ray_wheels": "custom-wheels", + "ray_test_repo": "https://github.com/user/ray.git", + "ray_test_branch": "sub/branch", + "priority": Priority.MANUAL, + "no_concurrency_limit": False, + }, + ) + + self.buildkite["release-frequency"] = "any-smoke" + update_settings_from_buildkite(updated_settings) + + self.assertDictEqual( + updated_settings, + { + "frequency": Frequency.ANY, + "prefer_smoke_tests": True, "test_attr_regex_filters": {"name": "name_filter"}, "ray_wheels": "custom-wheels", "ray_test_repo": "https://github.com/user/ray.git", @@ -363,6 +398,22 @@ def testFilterTests(self): ], ) + filtered = self._filter_names_smoke( + tests, + frequency=Frequency.ANY, + prefer_smoke_tests=True, + ) + self.assertSequenceEqual( + filtered, + [ + ("test_1", True), + ("test_2", True), + ("other_1", False), + ("other_2", True), + ("test_3", False), + ], + ) + filtered = self._filter_names_smoke(tests, frequency=Frequency.NIGHTLY) self.assertSequenceEqual( filtered, @@ -374,6 +425,21 @@ def testFilterTests(self): ], ) + filtered = self._filter_names_smoke( + tests, + frequency=Frequency.NIGHTLY, + prefer_smoke_tests=True, + ) + self.assertSequenceEqual( + filtered, + [ + ("test_1", True), + ("test_2", True), + ("other_2", True), + ("test_3", False), + ], + ) + filtered = self._filter_names_smoke(tests, frequency=Frequency.WEEKLY) self.assertSequenceEqual(filtered, [("test_2", False), ("other_1", False)]) diff --git a/release/ray_release/tests/test_wheels.py b/release/ray_release/tests/test_wheels.py index 7bd18c874038..2c06f5728093 100644 --- a/release/ray_release/tests/test_wheels.py +++ b/release/ray_release/tests/test_wheels.py @@ -19,7 +19,9 @@ class WheelsFinderTest(unittest.TestCase): def setUp(self) -> None: - pass + for key in os.environ: + if key.startswith("BUILDKITE"): + os.environ.pop(key) def testGetRayVersion(self): init_file = os.path.join( @@ -72,8 +74,6 @@ def testFindRayWheelsBuildkite(self): @patch("ray_release.wheels.get_ray_version", lambda *a, **kw: "2.0.0.dev0") def testFindRayWheelsCommitOnly(self): - os.environ.pop("BUILDKITE_BRANCH") - repo = DEFAULT_REPO branch = "master" commit = "1234" * 10