[ci] use prebuilt ray core binaries and ray dashboard (#56903)

use core binary bits and dashboard builds from previous bits for
testing. this caches c/c++ binary parts much more aggressively, and
speeds up CI speed for python-only changes for about 15 minutes.

Signed-off-by: Lonnie Liu <lonnie@anyscale.com>

Author: aslonnie

.buildkite/core.rayci.yml

@@ -2,6 +2,8 @@ group: core tests
 depends_on:
   - forge
   - oss-ci-base_build
+  - ray-core-build
+  - ray-dashboard-build
 steps:
   # builds
   - name: corebuild

.buildkite/data.rayci.yml

@@ -2,6 +2,8 @@ group: data tests
 depends_on:
   - forge
   - oss-ci-base_ml
+  - ray-core-build
+  - ray-dashboard-build
 steps:
   # builds
   - name: data9build

.buildkite/llm.rayci.yml

@@ -1,6 +1,8 @@
 group: llm tests
 depends_on:
   - forge
+  - ray-core-build
+  - ray-dashboard-build
 steps:
   - name: llmbuild
     wanda: ci/docker/llm.build.wanda.yaml

.buildkite/ml.rayci.yml

@@ -1,4 +1,8 @@
 group: ml tests
+depends_on:
+  - forge
+  - ray-core-build
+  - ray-dashboard-build
 steps:
   # builds
   - name: minbuild-ml

.buildkite/others.rayci.yml

@@ -18,7 +18,10 @@ steps:
         --only-tags doctest
         --except-tags gpu
         --parallelism-per-worker 3
-    depends_on: doctestbuild
+    depends_on:
+      - doctestbuild
+      - ray-core-build
+      - ray-dashboard-build
 
   # java
   - label: ":java: java tests"

.buildkite/rllib.rayci.yml

@@ -1,6 +1,8 @@
 group: rllib tests
 depends_on:
   - forge
+  - ray-core-build
+  - ray-dashboard-build
 steps:
   # builds
   - name: rllibbuild

.buildkite/serve.rayci.yml

@@ -2,6 +2,8 @@ group: serve tests
 depends_on:
   - forge
   - oss-ci-base_build
+  - ray-core-build
+  - ray-dashboard-build
 steps:
   # builds
   - name: servebuild

ci/ray_ci/docker_container.py

@@ -66,6 +66,16 @@ def __init__(
         upload: bool = False,
     ) -> None:
         assert "RAYCI_CHECKOUT_DIR" in os.environ, "RAYCI_CHECKOUT_DIR not set"
+        rayci_checkout_dir = os.environ["RAYCI_CHECKOUT_DIR"]
+
+        super().__init__(
+            "forge" if architecture == "x86_64" else "forge-aarch64",
+            python_version=python_version,
+            volumes=[
+                f"{rayci_checkout_dir}:/rayci",
+                "/var/run/docker.sock:/var/run/docker.sock",
+            ],
+        )
 
         if image_type in [RayType.RAY_ML, RayType.RAY_ML_EXTRA]:
             assert python_version in PYTHON_VERSIONS_RAY_ML
@@ -81,22 +91,12 @@ def __init__(
             assert platform in PLATFORMS_RAY
             assert architecture in ARCHITECTURES_RAY
 
-        rayci_checkout_dir = os.environ["RAYCI_CHECKOUT_DIR"]
-        self.python_version = python_version
         self.platform = platform
         self.image_type = image_type
         self.architecture = architecture
         self.canonical_tag = canonical_tag
         self.upload = upload
 
-        super().__init__(
-            "forge" if architecture == "x86_64" else "forge-aarch64",
-            volumes=[
-                f"{rayci_checkout_dir}:/rayci",
-                "/var/run/docker.sock:/var/run/docker.sock",
-            ],
-        )
-
     def _get_image_version_tags(self, external: bool) -> List[str]:
         """
         Get version tags.

ci/ray_ci/linux_container.py

@@ -1,9 +1,11 @@
 import os
+import platform
 import subprocess
 import sys
 from typing import List, Optional, Tuple
 
-from ci.ray_ci.container import Container
+from ci.ray_ci.configs import DEFAULT_ARCHITECTURE, DEFAULT_PYTHON_VERSION
+from ci.ray_ci.container import Container, get_docker_image
 
 _DOCKER_CAP_ADD = [
     "SYS_PTRACE",
@@ -18,17 +20,29 @@ def __init__(
         docker_tag: str,
         volumes: Optional[List[str]] = None,
         envs: Optional[List[str]] = None,
+        python_version: Optional[str] = None,
         tmp_filesystem: Optional[str] = None,
+        architecture: Optional[str] = None,
         privileged: bool = False,
     ) -> None:
         super().__init__(docker_tag, volumes, envs)
 
         if tmp_filesystem is not None:
             if tmp_filesystem != "tmpfs":
                 raise ValueError("Only tmpfs is supported for tmp filesystem")
+
+        self.python_version = python_version or DEFAULT_PYTHON_VERSION
         self.tmp_filesystem = tmp_filesystem
         self.privileged = privileged
 
+        if architecture is None:
+            architecture = platform.machine()
+            if architecture.lower() == "amd64":
+                architecture = "x86_64"
+            if architecture == "arm64":
+                architecture = "aarch64"
+        self.architecture = architecture
+
     def install_ray(
         self, build_type: Optional[str] = None, mask: Optional[str] = None
     ) -> List[str]:
@@ -50,13 +64,21 @@ def install_ray(
             "--build-arg",
             f"BUILDKITE_CACHE_READONLY={cache_readonly}",
         ]
+
+        if not build_type or build_type == "optimized":
+            python_version = self.python_version
+            core_image_tag = f"ray-core-py{python_version}"
+            if self.architecture != DEFAULT_ARCHITECTURE:
+                core_image_tag += f"-{self.architecture}"
+            ray_core_image = get_docker_image(core_image_tag)
+            build_cmd += ["--build-arg", f"RAY_CORE_IMAGE={ray_core_image}"]
+            ray_dashboard_image = get_docker_image("ray-dashboard")
+            build_cmd += ["--build-arg", f"RAY_DASHBOARD_IMAGE={ray_dashboard_image}"]
+
         if mask:
             build_cmd += ["--build-arg", "RAY_INSTALL_MASK=" + mask]
-        build_cmd += [
-            "-f",
-            "/ray/ci/ray_ci/tests.env.Dockerfile",
-            "/ray",
-        ]
+
+        build_cmd += ["-f", "ci/ray_ci/tests.env.Dockerfile", "/ray"]
         subprocess.check_call(
             build_cmd,
             env=env,

ci/ray_ci/linux_tester_container.py

@@ -16,6 +16,7 @@ def __init__(
         shard_ids: Optional[List[int]] = None,
         skip_ray_installation: bool = False,
         build_type: Optional[str] = None,
+        python_version: Optional[str] = None,
         install_mask: Optional[str] = None,
         tmp_filesystem: Optional[str] = None,
         privileged: bool = False,
@@ -28,6 +29,7 @@ def __init__(
                 f"{os.environ.get('RAYCI_CHECKOUT_DIR')}:/ray-mount",
                 "/var/run/docker.sock:/var/run/docker.sock",
             ],
+            python_version=python_version,
             tmp_filesystem=tmp_filesystem,
             privileged=privileged,
         )