[py312] Fix or skip tests+examples for Tune dependencies that do not …

…support py312 (ray-project#46761)

* Tests for library integrations will be skipped on py312 if that
library doesn't support py312.
* Tests for library integrations will be skipped altogether if it
imposes a version cap on a more important library.
* Important library integrations that do not support py312 yet will be
added to the issue tracker and treated as a medium high priority that
needs to be solved as soon as the library adds the py312 support.

---------

Signed-off-by: Justin Yu <justinvyu@anyscale.com>

.buildkite/others.rayci.yml

@@ -1,7 +1,7 @@
 group: others
 depends_on:
-  - oss-ci-base_build
   - forge
+  - oss-ci-base_build
 steps:
   #build
   - name: doctestbuild

ci/ci.sh

@@ -109,9 +109,8 @@ compile_pip_dependencies() {
     "${WORKSPACE_DIR}/python/requirements/ml/tune-test-requirements.txt" \
     "${WORKSPACE_DIR}/python/requirements/security-requirements.txt"
 
-  # Remove some pins from upstream dependencies:
-  # ray, xgboost-ray, lightgbm-ray, tune-sklearn
-  sed -i "/^ray==/d;/^xgboost-ray==/d;/^lightgbm-ray==/d;/^tune-sklearn==/d" "${WORKSPACE_DIR}/python/$TARGET"
+  # Remove some pins from upstream dependencies: ray
+  sed -i "/^ray==/d" "${WORKSPACE_DIR}/python/$TARGET"
 
   # Delete local installation
   sed -i "/@ file/d" "${WORKSPACE_DIR}/python/$TARGET"

doc/source/conf.py

@@ -235,7 +235,6 @@ def __init__(self, version: str):
         "https://mvnrepository.com/artifact/*",  # working but somehow not with linkcheck
         # This should be fixed -- is temporal the successor of cadence? Do the examples need to be updated?
         "https://github.com/serverlessworkflow/specification/blob/main/comparisons/comparison-cadence.md",
-        # TODO(richardliaw): The following probably needs to be fixed in the tune_sklearn package
         "https://www.oracle.com/java/technologies/javase-jdk15-downloads.html",  # forbidden for client
         "https://speakerdeck.com/*",  # forbidden for bots
         r"https://huggingface.co/*",  # seems to be flaky

doc/source/tune/examples/BUILD

@@ -22,6 +22,8 @@ py_test_run_all_notebooks(
         "tune-xgboost.ipynb",
         "pbt_transformers.ipynb",  # Transformers uses legacy Tune APIs.
         "horovod_simple.ipynb",  # CI do not have Horovod
+        "tune-aim.ipynb",  # CI does not have `aim`
+        "bohb_example.ipynb",  # CI does not have bohb requirements
    ],
     tags = [
         "exclusive",

python/ray/air/tests/test_air_usage.py

@@ -18,7 +18,6 @@
 from ray.tune.callback import Callback
 from ray.tune.experiment.experiment import Experiment
 from ray.tune.logger import LoggerCallback
-from ray.tune.logger.aim import AimLoggerCallback
 from ray.tune.utils.callback import DEFAULT_CALLBACK_CLASSES
 
 
@@ -113,7 +112,6 @@ class _CustomCallback(Callback):
     wandb.WandbLoggerCallback,
     mlflow.MLflowLoggerCallback,
     comet.CometLoggerCallback,
-    AimLoggerCallback,
     _CustomLoggerCallback,
     _CustomLoggerCallback,
     _CustomCallback,
@@ -152,7 +150,6 @@ def test_tag_setup_mlflow(mock_record, monkeypatch):
                 "WandbLoggerCallback": 1,
                 "MLflowLoggerCallback": 1,
                 "CometLoggerCallback": 1,
-                "AimLoggerCallback": 1,
                 "CustomLoggerCallback": 2,
                 "CustomCallback": 1,
             },

python/ray/air/tests/test_keras_callback.py

@@ -63,7 +63,7 @@ def test_report_with_default_arguments(self, mock_report, model):
         # simulates the end of an epoch, and asserts that a metric and checkpoint are
         # reported.
         callback = ReportCheckpointCallback()
-        callback.model = model
+        callback.set_model(model)
 
         callback.on_epoch_end(0, {"loss": 0})
 

python/ray/air/tests/test_tracebacks.py

@@ -95,7 +95,7 @@ def failing(config):
 
     tuner = Tuner(failing)
     results = tuner.fit()
-    assert len(str(results[0].error).split("\n")) <= 12
+    assert len(str(results[0].error).split("\n")) <= 20
 
 
 def test_traceback_trainer(ray_start_2_cpus):

python/ray/tune/examples/bohb_example.py

@@ -50,6 +50,13 @@ def load_checkpoint(self, checkpoint_dir):
 
 
 if __name__ == "__main__":
+    import sys
+
+    import pytest
+
+    if sys.version_info >= (3, 12):
+        pytest.skip("TuneBOHB is not compatible with Python 3.12")
+
     ray.init(num_cpus=8)
 
     config = {

python/ray/tune/tests/test_convergence.py

@@ -1,4 +1,5 @@
 import math
+import sys
 import unittest
 
 import numpy as np
@@ -79,6 +80,9 @@ def testConvergenceBayesOpt(self):
         assert len(analysis.trials) < 50
         assert math.isclose(analysis.best_config["x"], 0, abs_tol=1e-5)
 
+    @pytest.mark.skipif(
+        sys.version_info >= (3, 12), reason="HEBO doesn't support py312"
+    )
     def testConvergenceHEBO(self):
         from ray.tune.search.hebo import HEBOSearch
 
@@ -137,6 +141,4 @@ def testConvergenceZoopt(self):
 
 
 if __name__ == "__main__":
-    import sys
-
     sys.exit(pytest.main(["-v", __file__]))

python/ray/tune/tests/test_logger.py

@@ -3,13 +3,15 @@
 import json
 import os
 import shutil
+import sys
 import tempfile
 import unittest
 from dataclasses import dataclass
 from pathlib import Path
 from typing import Optional
 
 import numpy as np
+import pytest
 
 import ray
 from ray.air.constants import (
@@ -295,6 +297,7 @@ def testBadTBX(self):
         assert "INFO" in cm.output[0]
 
 
+@pytest.mark.skipif(sys.version_info >= (3, 12), reason="Aim doesn't support py312")
 class AimLoggerSuite(unittest.TestCase):
     """Test Aim integration."""
 

python/ray/tune/tests/test_sample.py

@@ -4,11 +4,13 @@
 change your pytest running directory to ray/python/ray/tune/tests/
 """
 
+import sys
 import unittest
 from collections import defaultdict
 from unittest.mock import patch
 
 import numpy as np
+import pytest
 
 import ray
 import ray.tune.search.sample
@@ -768,6 +770,9 @@ def config_generator():
 
         self._testTuneSampleAPI(config_generator(), ignore=ignore)
 
+    @pytest.mark.skipif(
+        sys.version_info >= (3, 12), reason="HEBO doesn't support py312"
+    )
     def testConvertHEBO(self):
         import torch
         from hebo.design_space.design_space import DesignSpace
@@ -824,6 +829,9 @@ def testConvertHEBO(self):
 
         # Mixed configs are not supported
 
+    @pytest.mark.skipif(
+        sys.version_info >= (3, 12), reason="HEBO doesn't support py312"
+    )
     def testSampleBoundsHEBO(self):
         from ray.tune.search.hebo import HEBOSearch
 
@@ -1864,8 +1872,4 @@ def set_search_properties(self, metric, mode, config):
 
 
 if __name__ == "__main__":
-    import sys
-
-    import pytest
-
     sys.exit(pytest.main(["-v", __file__] + sys.argv[1:]))

python/ray/tune/tests/test_searchers.py

@@ -1,6 +1,7 @@
 import contextlib
 import os
 import shutil
+import sys
 import tempfile
 import unittest
 from copy import deepcopy
@@ -162,6 +163,9 @@ def testBOHB(self):
             )
         self.assertCorrectExperimentOutput(out)
 
+    @pytest.mark.skipif(
+        sys.version_info >= (3, 12), reason="HEBO doesn't support py312"
+    )
     def testHEBO(self):
         if Version(pandas.__version__) >= Version("2.0.0"):
             pytest.skip("HEBO does not support pandas>=2.0.0")
@@ -415,6 +419,9 @@ def dbr_space(trial):
         with self.assertRaises(TypeError):
             dbr_searcher.add_evaluated_point(point, 1.0)
 
+    @pytest.mark.skipif(
+        sys.version_info >= (3, 12), reason="HEBO doesn't support py312"
+    )
     def testHEBO(self):
         if Version(pandas.__version__) >= Version("2.0.0"):
             pytest.skip("HEBO does not support pandas>=2.0.0")
@@ -541,6 +548,9 @@ def testBOHB(self):
 
         assert "not_completed" in searcher.trial_to_params
 
+    @pytest.mark.skipif(
+        sys.version_info >= (3, 12), reason="HEBO doesn't support py312"
+    )
     def testHEBO(self):
         if Version(pandas.__version__) >= Version("2.0.0"):
             pytest.skip("HEBO does not support pandas>=2.0.0")

python/ray/tune/tests/test_trial_scheduler.py

@@ -932,6 +932,9 @@ def result(score, ts):
             [t.status for t in trials], [Trial.PAUSED, Trial.PAUSED, Trial.PAUSED]
         )
 
+    @pytest.mark.skipif(
+        sys.version_info >= (3, 12), reason="BOHB doesn't support py312"
+    )
     def testNonstopBOHB(self):
         from ray.tune.search.bohb import TuneBOHB
 
@@ -2551,6 +2554,9 @@ def testAnonymousMetricEndToEndFIFO(self):
     def testAnonymousMetricEndToEndASHA(self):
         self._testAnonymousMetricEndToEnd(AsyncHyperBandScheduler)
 
+    @pytest.mark.skipif(
+        sys.version_info >= (3, 12), reason="BOHB doesn't support py312"
+    )
     def testAnonymousMetricEndToEndBOHB(self):
         from ray.tune.search.bohb import TuneBOHB
 

python/ray/tune/tests/test_tune_restore_warm_start.py

@@ -1,13 +1,13 @@
 # coding: utf-8
 import os
 import shutil
+import sys
 import tempfile
 import unittest
 
 import numpy as np
 import pandas
 import pytest
-from hebo.design_space.design_space import DesignSpace as HEBODesignSpace
 from hyperopt import hp
 from nevergrad.optimization import optimizerlib
 from packaging.version import Version
@@ -251,11 +251,14 @@ def cost(param):
         return search_alg, cost
 
 
+@pytest.mark.skipif(sys.version_info >= (3, 12), reason="HEBO doesn't support py312")
 class HEBOWarmStartTest(AbstractWarmStartTest, unittest.TestCase):
     def set_basic_conf(self):
         if Version(pandas.__version__) >= Version("2.0.0"):
             pytest.skip("HEBO does not support pandas>=2.0.0")
 
+        from hebo.design_space.design_space import DesignSpace as HEBODesignSpace
+
         space_config = [
             {"name": "width", "type": "num", "lb": 0, "ub": 20},
             {"name": "height", "type": "num", "lb": -100, "ub": 100},
@@ -345,6 +348,4 @@ def get_scheduler(self):
 
 
 if __name__ == "__main__":
-    import sys
-
     sys.exit(pytest.main(["-v", __file__] + sys.argv[1:]))

python/requirements/ml/core-requirements.txt

@@ -1,5 +1,5 @@
 # ML tracking integrations
-comet-ml==3.31.9
+comet-ml==3.44.1
 mlflow==2.9.2
 wandb==0.17.0
 

python/requirements/ml/rllib-requirements.txt

@@ -6,7 +6,7 @@ imageio-ffmpeg==0.4.5
 onnx==1.15.0; sys_platform != 'darwin' or platform_machine != 'arm64'
 onnxruntime==1.18.0; sys_platform != 'darwin' or platform_machine != 'arm64'
 tf2onnx==1.15.1; sys_platform != 'darwin' or platform_machine != 'arm64'
-rich==12.6.0
+rich==13.3.2
 # Msgpack checkpoint stuff.
 msgpack
 msgpack_numpy

python/requirements/ml/tune-requirements.txt

@@ -4,11 +4,9 @@ ax-platform[mysql]==0.3.2
 bayesian-optimization==1.4.3
 
 # BOHB
-ConfigSpace==0.7.1
-hpbandster==0.7.4
+ConfigSpace==0.7.1; python_version < "3.12"
+hpbandster==0.7.4; python_version < "3.12"
 
 hyperopt==0.2.7
 nevergrad==0.4.3.post7
 optuna==3.2.0
-
-tune-sklearn==0.4.6; python_version < "3.12"

python/requirements/test-requirements.txt

@@ -81,7 +81,7 @@ colorama
 myst-parser==1.0.0
 myst-nb==1.1.0
 sphinx==6.2.1
-jupytext==1.13.6
+jupytext>1.13.6
 jinja2==3.1.2
 pytest-docker-tools==3.1.3
 pytest-forked==1.4.0
@@ -97,7 +97,7 @@ tensorboardX
 tensorboard
 tensorboard-data-server==0.7.2
 h11==0.12.0
-markdown-it-py==1.1.0
+markdown-it-py
 attrs==21.4.0
 pytz==2022.7.1
 # Compatibility with spacy 3.5 (model en_core_web_sm)