From 99806705b7cf120269808d5f4c6726117ba5afd9 Mon Sep 17 00:00:00 2001 From: matthewdeng Date: Tue, 29 Aug 2023 11:35:36 -0700 Subject: [PATCH] [train] enable train + tune tests and examples (#39021) Signed-off-by: Matthew Deng Signed-off-by: e428265 --- .buildkite/pipeline.ml.yml | 31 ++++++++--------------------- python/ray/train/__init__.py | 1 + python/ray/train/tests/test_tune.py | 9 +++++++-- 3 files changed, 16 insertions(+), 25 deletions(-) diff --git a/.buildkite/pipeline.ml.yml b/.buildkite/pipeline.ml.yml index 960e50b8b78ab..685a4f740a468 100644 --- a/.buildkite/pipeline.ml.yml +++ b/.buildkite/pipeline.ml.yml @@ -47,16 +47,14 @@ --test_tag_filters=-gpu_only,-gpu,-minimal,-tune,-doctest python/ray/train/... - -# Currently empty test suite -#- label: ":steam_locomotive: :octopus: Train + Tune tests and examples" -# conditions: ["NO_WHEELS_REQUIRED", "RAY_CI_TRAIN_AFFECTED"] -# instance_size: medium -# commands: -# - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT -# - TRAIN_TESTING=1 TUNE_TESTING=1 ./ci/env/install-dependencies.sh -# - ./ci/env/env_info.sh -# - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=tune,-gpu_only,-ray_air,-gpu,-doctest,-new_storage python/ray/train/... +- label: ":steam_locomotive: :octopus: Train + Tune tests and examples" + conditions: ["NO_WHEELS_REQUIRED", "RAY_CI_TRAIN_AFFECTED"] + instance_size: medium + commands: + - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT + - TRAIN_TESTING=1 TUNE_TESTING=1 ./ci/env/install-dependencies.sh + - ./ci/env/env_info.sh + - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=tune,-gpu_only,-ray_air,-gpu,-doctest python/ray/train/... - label: ":brain: RLlib: Benchmarks (Torch 2.x)" @@ -361,19 +359,6 @@ ##### STORAGE REFACTOR -# TODO(krfricke): Add new test for this suite -# - label: ":steam_locomotive: :octopus: :floppy_disk: New persistence mode: Train + Tune tests and examples" -# conditions: ["NO_WHEELS_REQUIRED", "RAY_CI_TRAIN_AFFECTED"] -# instance_size: medium -# commands: -# - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT -# - TRAIN_TESTING=1 TUNE_TESTING=1 ./ci/env/install-dependencies.sh -# - ./ci/env/env_info.sh -# - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only -# --test_tag_filters=tune,-gpu_only,-ray_air,-gpu,-doctest,-no_new_storage -# --test_env=RAY_AIR_NEW_PERSISTENCE_MODE=1 -# python/ray/train/... - - label: ":octopus: :floppy_disk: New persistence mode: Tune tests and examples (small)" conditions: ["NO_WHEELS_REQUIRED", "RAY_CI_TUNE_AFFECTED"] diff --git a/python/ray/train/__init__.py b/python/ray/train/__init__.py index a20500daff7b9..85d2c85553203 100644 --- a/python/ray/train/__init__.py +++ b/python/ray/train/__init__.py @@ -34,6 +34,7 @@ usage_lib.record_library_usage("train") + __all__ = [ "get_checkpoint", "get_context", diff --git a/python/ray/train/tests/test_tune.py b/python/ray/train/tests/test_tune.py index a5cda8028ba92..089b4f0ae577e 100644 --- a/python/ray/train/tests/test_tune.py +++ b/python/ray/train/tests/test_tune.py @@ -219,7 +219,12 @@ def train_func(): assert len(df[TRAINING_ITERATION]) == 4 -def test_restore_with_new_trainer(ray_start_4_cpus, tmpdir, propagate_logs, caplog): +def test_restore_with_new_trainer( + ray_start_4_cpus, tmpdir, propagate_logs, caplog, monkeypatch +): + + monkeypatch.setenv("RAY_AIR_LOCAL_CACHE_DIR", str(tmpdir)) + def train_func(config): raise RuntimeError("failing!") @@ -227,7 +232,7 @@ def train_func(config): train_func, backend_config=TestConfig(), scaling_config=ScalingConfig(num_workers=1), - run_config=RunConfig(local_dir=str(tmpdir), name="restore_new_trainer"), + run_config=RunConfig(name="restore_new_trainer"), datasets={"train": ray.data.from_items([{"a": i} for i in range(10)])}, ) results = Tuner(trainer).fit()