[ci] Update RayCI to 0.25.0 and switch to test rules files

.buildkite/always.rules.test.txt

@@ -0,0 +1,10 @@
+# Unit tests for always.rules.txt
+#
+# Format: file_path: tag1 tag2 tag3
+#
+# This rules file adds "always lint" to all files via wildcard fallthrough.
+
+# Any file gets always lint tags
+some/random/file.txt: always lint
+python/ray/actor.py: always lint
+doc/source/index.rst: always lint

.buildkite/always.rules.txt

@@ -0,0 +1,20 @@
+# Ray CI conditional testing rules definition files. Applied to all files.
+#
+# Comment content, after '#', will be ignored.
+# Empty lines will be ignored too.
+#
+#   ! tag1 tag2 tag3  # Declares a tag. A tag must be declared first to be used.
+#   # Tags must be declared at the beginning.
+#
+#   dir/  # Directory to match
+#   file  # File to match
+#   dir/*.py  # Pattern to match, using fnmatch, matches dir/a.py dir/dir/b.py or dir/.py
+#   @ tag1 tag2 tag3  # Tags to emit for a rule. A rule without tags is a skipping rule.
+#
+#   ;  # Semicolon to separate rules
+
+! always lint
+
+*
+@ always lint
+;

.buildkite/cicd.rayci.yml

@@ -1,5 +1,13 @@
 group: reef tests
 steps:
+  - label: ":test_tube: test-rules"
+    key: test-rules
+    commands:
+      - curl -sfL "https://raw.githubusercontent.com/ray-project/rayci/stable/run_rayci.sh" | bash -s -- test-rules
+    instance_type: small
+    tags:
+      - tools
+      - always
   - label: ":coral: reef: ci+release tooling tests"
     key: reef-tests
     commands:

.buildkite/macos/always.rules.test.txt

@@ -0,0 +1 @@
+../always.rules.test.txt

.buildkite/macos/always.rules.txt

@@ -0,0 +1 @@
+../always.rules.txt

.buildkite/macos/config.yml

@@ -16,5 +16,3 @@ hook_env_keys:
   - RAYCI_CHECKOUT_DIR
 skip_tags:
   - disabled
-tag_filter_command:
-  - ./ci/ci_tags_from_change.sh

.buildkite/macos/test.rules.test.txt

@@ -0,0 +1 @@
+../test.rules.test.txt

.buildkite/macos/test.rules.txt

@@ -0,0 +1 @@
+../test.rules.txt

.buildkite/test.rules.test.txt

@@ -0,0 +1,80 @@
+# Test cases for Ray CI test rules
+# Based on ci/pipeline/test_conditional_testing.py _TESTS_YAML
+#
+# Format: file: tag1 tag2 tag3
+
+# === CI/Pipeline ===
+
+ci/pipeline/test_conditional_testing.py: tools
+
+# === Ray Libraries ===
+
+python/ray/data/__init__.py: data ml train
+python/ray/air/__init__.py: ml train tune data linux_wheels
+python/ray/llm/llm.py: llm
+python/ray/workflow/workflow.py: workflow
+python/ray/tune/tune.py: ml train tune linux_wheels
+python/ray/train/train.py: ml train linux_wheels
+python/ray/serve/serve.py: serve linux_wheels java
+python/ray/dashboard/dashboard.py: dashboard linux_wheels python
+
+# === Ray Utilities ===
+
+python/ray/util/dask/dask.py: python dask
+python/ray/dag/dag.py: python cgraphs_direct_transport
+python/ray/experimental/gpu_object_manager/gpu_object_manager.py: python cgraphs_direct_transport
+
+# === RLlib ===
+
+rllib/rllib.py: rllib rllib_gpu rllib_directly
+
+# === Python Core ===
+
+python/core.py: ml tune train data python dashboard linux_wheels macos_wheels java
+python/setup.py: ml tune train serve workflow data python dashboard linux_wheels macos_wheels java python_dependencies
+python/requirements/test-requirements.txt: ml tune train serve workflow data python dashboard linux_wheels macos_wheels java python_dependencies
+python/_raylet.pyx: ml tune train data python dashboard linux_wheels macos_wheels java
+
+# === Buildkite Configs ===
+
+.buildkite/ml.rayci.yml: ml train tune
+.buildkite/core.rayci.yml: python core_cpp
+.buildkite/others.rayci.yml: java
+.buildkite/lint.rayci.yml: tools
+.buildkite/macos.rayci.yml: macos_wheels
+.buildkite/base.rayci.yml: docker linux_wheels tools
+
+# === Java/C++ ===
+
+java/ray.java: java
+cpp/ray.cc: cpp
+src/ray.cpp: core_cpp cpp java python linux_wheels macos_wheels dashboard release_tests
+
+# === Docker ===
+
+docker/Dockerfile.ray: docker linux_wheels
+
+# === Documentation ===
+
+doc/code.py: doc
+doc/example.ipynb: doc
+doc/tutorial.rst: doc
+.readthedocs.yaml: doc
+.vale.ini: doc
+.vale/styles/config/vocabularies/Core/accept.txt: doc
+
+# === Release ===
+
+release/requirements.txt: release_tests
+release/requirements_py310.txt: tools
+release/release_tests.yaml: tools
+
+# === CI Scripts ===
+
+ci/lint/lint.sh: tools
+ci/ray_ci/tester.py: tools
+ci/ci.sh: tools
+
+# === Root Files ===
+
+BUILD.bazel: ml tune train data serve core_cpp cpp java python doc linux_wheels macos_wheels dashboard tools release_tests

ci/pipeline/test_rules.txt → .buildkite/test.rules.txt

@@ -68,7 +68,7 @@ ci/docker/min.build.wanda.yaml
 
 rllib/
 python/ray/rllib/
-ray_ci/rllib.tests.yml
+ci/ray_ci/rllib.tests.yml
 .buildkite/rllib.rayci.yml
 @ rllib rllib_gpu rllib_directly
 ;
@@ -232,17 +232,17 @@ ci/ci.sh
 @ tools
 ;
 
+src/ray/core_worker/experimental*.h
+src/ray/core_worker/experimental*.cc
+@ cgraphs_direct_transport
+;
+
 src/
 @ core_cpp cpp
 @ java python linux_wheels macos_wheels
 @ dashboard release_tests
 ;
 
-src/ray/core_worker/experimental*.h
-src/ray/core_worker/experimental*.cc
-@ cgraphs_direct_transport
-;
-
 .github/
 README.rst
 SECURITY.md
@@ -257,3 +257,9 @@ setup_hooks.sh
 .fossa.yml
 # pass
 ;
+
+*
+@ ml tune train data serve
+@ core_cpp cpp java python doc
+@ linux_wheels macos_wheels dashboard tools release_tests
+;

.rayciversion

@@ -1 +1 @@
-0.21.0
+0.25.0

ci/pipeline/BUILD.bazel

@@ -1,28 +1,6 @@
-load("@py_deps_py310//:requirements.bzl", ci_require = "requirement")
-load("@rules_python//python:defs.bzl", "py_library", "py_test")
+load("@rules_python//python:defs.bzl", "py_library")
 
 py_library(
     name = "determine_tests_to_run",
     srcs = ["determine_tests_to_run.py"],
 )
-
-py_test(
-    name = "test_conditional_testing",
-    size = "small",
-    srcs = ["test_conditional_testing.py"],
-    data = [
-        ":determine_tests_to_run.py",
-        ":test_rules.txt",
-    ],
-    exec_compatible_with = ["//bazel:py3"],
-    tags = [
-        "ci_unit",
-        "team:ci",
-    ],
-    deps = [
-        ":determine_tests_to_run",
-        ci_require("bazel-runfiles"),
-        ci_require("pytest"),
-        ci_require("pyyaml"),
-    ],
-)