diff --git a/release/ray_release/buildkite/filter.py b/release/ray_release/buildkite/filter.py index fb03b9a610ecc..499b21e7eb755 100644 --- a/release/ray_release/buildkite/filter.py +++ b/release/ray_release/buildkite/filter.py @@ -10,6 +10,7 @@ def filter_tests( test_collection: List[Test], frequency: Frequency, test_attr_regex_filters: Optional[Dict[str, str]] = None, + prefer_smoke_tests: bool = False, ) -> List[Tuple[Test, bool]]: if test_attr_regex_filters is None: test_attr_regex_filters = {} @@ -31,7 +32,13 @@ def filter_tests( continue if frequency == Frequency.ANY or frequency == test_frequency: - tests_to_run.append((test, False)) + if prefer_smoke_tests and "smoke_test" in test: + # If we prefer smoke tests and a smoke test is available for this test, + # then use the smoke test + smoke_test = True + else: + smoke_test = False + tests_to_run.append((test, smoke_test)) continue elif "smoke_test" in test: diff --git a/release/ray_release/buildkite/settings.py b/release/ray_release/buildkite/settings.py index a3d98654ffdc0..aa6f732055fbe 100644 --- a/release/ray_release/buildkite/settings.py +++ b/release/ray_release/buildkite/settings.py @@ -19,6 +19,7 @@ class Frequency(enum.Enum): frequency_str_to_enum = { "disabled": Frequency.DISABLED, "any": Frequency.ANY, + "any-smoke": Frequency.ANY, "multi": Frequency.MULTI, "nightly": Frequency.NIGHTLY, "weekly": Frequency.WEEKLY, @@ -123,6 +124,7 @@ def get_pipeline_settings() -> Dict: def get_default_settings() -> Dict: settings = { "frequency": Frequency.ANY, + "prefer_smoke_tests": False, "test_attr_regex_filters": None, "ray_wheels": None, "ray_test_repo": None, @@ -137,6 +139,13 @@ def update_settings_from_environment(settings: Dict) -> Dict: if "RELEASE_FREQUENCY" in os.environ: settings["frequency"] = get_frequency(os.environ["RELEASE_FREQUENCY"]) + if "RELEASE_PREFER_SMOKE_TESTS" in os.environ: + settings["prefer_smoke_tests"] = bool( + int(os.environ["RELEASE_PREFER_SMOKE_TESTS"]) + ) + elif os.environ.get("RELEASE_FREQUENCY", "").endswith("-smoke"): + settings["prefer_smoke_tests"] = True + if "RAY_TEST_REPO" in os.environ: settings["ray_test_repo"] = os.environ["RAY_TEST_REPO"] settings["ray_test_branch"] = os.environ.get("RAY_TEST_BRANCH", DEFAULT_BRANCH) @@ -173,6 +182,8 @@ def update_settings_from_buildkite(settings: Dict): release_frequency = get_buildkite_prompt_value("release-frequency") if release_frequency: settings["frequency"] = get_frequency(release_frequency) + if release_frequency.endswith("-smoke"): + settings["prefer_smoke_tests"] = True ray_test_repo_branch = get_buildkite_prompt_value("release-ray-test-repo-branch") if ray_test_repo_branch: diff --git a/release/ray_release/config.py b/release/ray_release/config.py index 9d1ed45de8611..11d3f38a702d5 100644 --- a/release/ray_release/config.py +++ b/release/ray_release/config.py @@ -10,7 +10,7 @@ import yaml from ray_release.anyscale_util import find_cloud_by_name -from ray_release.exception import ReleaseTestConfigError +from ray_release.exception import ReleaseTestConfigError, ReleaseTestCLIError from ray_release.logger import logger from ray_release.util import deep_update @@ -134,11 +134,10 @@ def find_test(test_collection: List[Test], test_name: str) -> Optional[Test]: def as_smoke_test(test: Test) -> Test: if "smoke_test" not in test: - logger.warning( + raise ReleaseTestCLIError( f"Requested smoke test, but test with name {test['name']} does " f"not have any smoke test configuration." ) - return test smoke_test_config = test.pop("smoke_test") new_test = deep_update(test, smoke_test_config) diff --git a/release/ray_release/scripts/build_pipeline.py b/release/ray_release/scripts/build_pipeline.py index fdf5e4288a9f3..e6316024120d5 100644 --- a/release/ray_release/scripts/build_pipeline.py +++ b/release/ray_release/scripts/build_pipeline.py @@ -67,6 +67,7 @@ def main(test_collection_file: Optional[str] = None): shutil.rmtree(tmpdir, ignore_errors=True) frequency = settings["frequency"] + prefer_smoke_tests = settings["prefer_smoke_tests"] test_attr_regex_filters = settings["test_attr_regex_filters"] ray_wheels = settings["ray_wheels"] priority = settings["priority"] @@ -74,6 +75,7 @@ def main(test_collection_file: Optional[str] = None): logger.info( f"Found the following buildkite pipeline settings:\n\n" f" frequency = {settings['frequency']}\n" + f" prefer_smoke_tests = {settings['prefer_smoke_tests']}\n" f" test_attr_regex_filters = {settings['test_attr_regex_filters']}\n" f" ray_wheels = {settings['ray_wheels']}\n" f" ray_test_repo = {settings['ray_test_repo']}\n" @@ -86,6 +88,7 @@ def main(test_collection_file: Optional[str] = None): test_collection, frequency=frequency, test_attr_regex_filters=test_attr_regex_filters, + prefer_smoke_tests=prefer_smoke_tests, ) logger.info(f"Found {len(filtered_tests)} tests to run.") if len(filtered_tests) == 0: diff --git a/release/ray_release/tests/test_buildkite.py b/release/ray_release/tests/test_buildkite.py index ed52b649aab70..419714a9959b1 100644 --- a/release/ray_release/tests/test_buildkite.py +++ b/release/ray_release/tests/test_buildkite.py @@ -158,6 +158,23 @@ def testSettingsOverrideEnv(self): updated_settings, { "frequency": Frequency.NIGHTLY, + "prefer_smoke_tests": False, + "test_attr_regex_filters": {"name": "name_filter"}, + "ray_wheels": "custom-wheels", + "ray_test_repo": "https://github.com/user/ray.git", + "ray_test_branch": "sub/branch", + "priority": Priority.MANUAL, + "no_concurrency_limit": False, + }, + ) + + os.environ["RELEASE_FREQUENCY"] = "any-smoke" + update_settings_from_environment(updated_settings) + self.assertDictEqual( + updated_settings, + { + "frequency": Frequency.ANY, + "prefer_smoke_tests": True, "test_attr_regex_filters": {"name": "name_filter"}, "ray_wheels": "custom-wheels", "ray_test_repo": "https://github.com/user/ray.git", @@ -307,6 +324,24 @@ def testSettingsOverrideBuildkite(self): updated_settings, { "frequency": Frequency.NIGHTLY, + "prefer_smoke_tests": False, + "test_attr_regex_filters": {"name": "name_filter"}, + "ray_wheels": "custom-wheels", + "ray_test_repo": "https://github.com/user/ray.git", + "ray_test_branch": "sub/branch", + "priority": Priority.MANUAL, + "no_concurrency_limit": False, + }, + ) + + self.buildkite["release-frequency"] = "any-smoke" + update_settings_from_buildkite(updated_settings) + + self.assertDictEqual( + updated_settings, + { + "frequency": Frequency.ANY, + "prefer_smoke_tests": True, "test_attr_regex_filters": {"name": "name_filter"}, "ray_wheels": "custom-wheels", "ray_test_repo": "https://github.com/user/ray.git", @@ -363,6 +398,22 @@ def testFilterTests(self): ], ) + filtered = self._filter_names_smoke( + tests, + frequency=Frequency.ANY, + prefer_smoke_tests=True, + ) + self.assertSequenceEqual( + filtered, + [ + ("test_1", True), + ("test_2", True), + ("other_1", False), + ("other_2", True), + ("test_3", False), + ], + ) + filtered = self._filter_names_smoke(tests, frequency=Frequency.NIGHTLY) self.assertSequenceEqual( filtered, @@ -374,6 +425,21 @@ def testFilterTests(self): ], ) + filtered = self._filter_names_smoke( + tests, + frequency=Frequency.NIGHTLY, + prefer_smoke_tests=True, + ) + self.assertSequenceEqual( + filtered, + [ + ("test_1", True), + ("test_2", True), + ("other_2", True), + ("test_3", False), + ], + ) + filtered = self._filter_names_smoke(tests, frequency=Frequency.WEEKLY) self.assertSequenceEqual(filtered, [("test_2", False), ("other_1", False)]) diff --git a/release/ray_release/tests/test_wheels.py b/release/ray_release/tests/test_wheels.py index 7bd18c8740385..2c06f57280937 100644 --- a/release/ray_release/tests/test_wheels.py +++ b/release/ray_release/tests/test_wheels.py @@ -19,7 +19,9 @@ class WheelsFinderTest(unittest.TestCase): def setUp(self) -> None: - pass + for key in os.environ: + if key.startswith("BUILDKITE"): + os.environ.pop(key) def testGetRayVersion(self): init_file = os.path.join( @@ -72,8 +74,6 @@ def testFindRayWheelsBuildkite(self): @patch("ray_release.wheels.get_ray_version", lambda *a, **kw: "2.0.0.dev0") def testFindRayWheelsCommitOnly(self): - os.environ.pop("BUILDKITE_BRANCH") - repo = DEFAULT_REPO branch = "master" commit = "1234" * 10