Skip to content

[ci] port Buildkite Ray Images to Wanda #60455

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
andrew-anyscale wants to merge 1 commit into
base: master
Choose a base branch
from
Open

[ci] port Buildkite Ray Images to Wanda #60455

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
268 changes: 219 additions & 49 deletions .buildkite/build.rayci.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -133,91 +133,261 @@ steps:
depends_on: manylinux-x86_64
job_env: manylinux-x86_64

- label: ":tapioca: build: ray py{{matrix}} docker (x86_64)"
key: ray_images
- name: ray-image-cpu-build
label: "wanda: ray py{{matrix}} cpu (x86_64)"
wanda: ci/docker/ray-image-cpu.wanda.yaml
matrix:
- "3.10"
- "3.11"
- "3.12"
- "3.13"
env_file: rayci.env
env:
PYTHON_VERSION: "{{matrix}}"
ARCH_SUFFIX: ""
tags:
- python_dependencies
- docker
- oss
instance_type: medium
commands:
- bazel run //ci/ray_ci:build_in_docker -- docker --python-version {{matrix}}
--platform cu11.7.1-cudnn8 --platform cu11.8.0-cudnn8
--platform cu12.1.1-cudnn8 --platform cu12.3.2-cudnn9
--platform cu12.4.1-cudnn --platform cu12.5.1-cudnn
--platform cu12.6.3-cudnn --platform cu12.8.1-cudnn
--platform cu12.9.1-cudnn
--platform cpu
--image-type ray --upload
depends_on:
- manylinux-x86_64
- forge
- raycudabase
- ray-wheel-build
- raycpubase

- name: ray-image-cuda-build
label: "wanda: ray py{{matrix.python}} cu{{matrix.cuda}} (x86_64)"
wanda: ci/docker/ray-image-cuda.wanda.yaml
matrix:
setup:
python:
- "3.10"
- "3.11"
- "3.12"
- "3.13"
cuda:
- "11.7.1-cudnn8"
- "11.8.0-cudnn8"
- "12.1.1-cudnn8"
- "12.3.2-cudnn9"
- "12.4.1-cudnn"
- "12.5.1-cudnn"
- "12.6.3-cudnn"
- "12.8.1-cudnn"
- "12.9.1-cudnn"
env_file: rayci.env
env:
PYTHON_VERSION: "{{matrix.python}}"
CUDA_VERSION: "{{matrix.cuda}}"
ARCH_SUFFIX: ""
tags:
- python_dependencies
- docker
- oss
depends_on:
- ray-wheel-build
- raycudabase

- label: ":crane: publish: ray py{{matrix}} (x86_64)"
key: ray_images_push
instance_type: small
commands:
- bazel run //.buildkite:copy_files -- --destination docker_login
- bazel run //ci/ray_ci/automation:push_ray_image --
--python-version {{matrix}}
--platform cpu
--platform cu11.7.1-cudnn8
--platform cu11.8.0-cudnn8
--platform cu12.1.1-cudnn8
--platform cu12.3.2-cudnn9
--platform cu12.4.1-cudnn
--platform cu12.5.1-cudnn
--platform cu12.6.3-cudnn
--platform cu12.8.1-cudnn
--platform cu12.9.1-cudnn
--image-type ray
--architecture x86_64
matrix:
- "3.10"
- "3.11"
- "3.12"
- "3.13"
depends_on:
- ray-image-cpu-build
- ray-image-cuda-build
tags:
- python_dependencies
- docker
- oss
- skip-on-premerge

- label: ":tapioca: build: ray-extra py{{matrix}} docker (x86_64)"
key: ray_extra_images
- name: ray-extra-image-cpu-build
label: "wanda: ray-extra py{{matrix}} cpu (x86_64)"
wanda: ci/docker/ray-extra-image-cpu.wanda.yaml
matrix:
- "3.10"
- "3.11"
- "3.12"
- "3.13"
env_file: rayci.env
env:
PYTHON_VERSION: "{{matrix}}"
ARCH_SUFFIX: ""
tags:
- python_dependencies
- docker
- oss
instance_type: medium
commands:
- bazel run //ci/ray_ci:build_in_docker -- docker --python-version {{matrix}}
--platform cu11.7.1-cudnn8 --platform cu11.8.0-cudnn8
--platform cu12.1.1-cudnn8 --platform cu12.3.2-cudnn9
--platform cu12.4.1-cudnn --platform cu12.5.1-cudnn
--platform cu12.6.3-cudnn --platform cu12.8.1-cudnn
--platform cu12.9.1-cudnn
--platform cpu
--image-type ray-extra --upload
depends_on:
- manylinux-x86_64
- forge
- ray-wheel-build
- raycpubaseextra

- name: ray-extra-image-cuda-build
label: "wanda: ray-extra py{{matrix.python}} cu{{matrix.cuda}} (x86_64)"
wanda: ci/docker/ray-extra-image-cuda.wanda.yaml
matrix:
setup:
python:
- "3.10"
- "3.11"
- "3.12"
- "3.13"
cuda:
- "11.7.1-cudnn8"
- "11.8.0-cudnn8"
- "12.1.1-cudnn8"
- "12.3.2-cudnn9"
- "12.4.1-cudnn"
- "12.5.1-cudnn"
- "12.6.3-cudnn"
- "12.8.1-cudnn"
- "12.9.1-cudnn"
env_file: rayci.env
env:
PYTHON_VERSION: "{{matrix.python}}"
CUDA_VERSION: "{{matrix.cuda}}"
ARCH_SUFFIX: ""
tags:
- python_dependencies
- docker
- oss
depends_on:
- ray-wheel-build
- raycudabaseextra

- name: ray-llm-image-cuda-build
label: "wanda: ray-llm py{{matrix.python}} cu{{matrix.cuda}} (x86_64)"
wanda: ci/docker/ray-llm-image-cuda.wanda.yaml
matrix:
setup:
python:
- "3.11"
cuda:
- "12.8.1-cudnn"
env_file: rayci.env
env:
PYTHON_VERSION: "{{matrix.python}}"
CUDA_VERSION: "{{matrix.cuda}}"
ARCH_SUFFIX: ""
tags:
- python_dependencies
- docker
- oss
depends_on:
- ray-wheel-build
- ray-llmbase

- name: ray-llm-extra-image-cuda-build
label: "wanda: ray-llm-extra py{{matrix.python}} cu{{matrix.cuda}} (x86_64)"
wanda: ci/docker/ray-llm-extra-image-cuda.wanda.yaml
matrix:
setup:
python:
- "3.11"
cuda:
- "12.8.1-cudnn"
env_file: rayci.env
env:
PYTHON_VERSION: "{{matrix.python}}"
CUDA_VERSION: "{{matrix.cuda}}"
ARCH_SUFFIX: ""
tags:
- python_dependencies
- docker
- oss
depends_on:
- ray-wheel-build
- ray-llmbaseextra

- label: ":crane: publish: ray-extra py{{matrix}} (x86_64)"
key: ray_extra_images_push
instance_type: small
commands:
- bazel run //.buildkite:copy_files -- --destination docker_login
- bazel run //ci/ray_ci/automation:push_ray_image --
--python-version {{matrix}}
--platform cpu
--platform cu11.7.1-cudnn8
--platform cu11.8.0-cudnn8
--platform cu12.1.1-cudnn8
--platform cu12.3.2-cudnn9
--platform cu12.4.1-cudnn
--platform cu12.5.1-cudnn
--platform cu12.6.3-cudnn
--platform cu12.8.1-cudnn
--platform cu12.9.1-cudnn
--image-type ray-extra
--architecture x86_64
matrix:
- "3.10"
- "3.11"
- "3.12"
- "3.13"

- label: ":tapioca: build: ray-llm py{{matrix}} docker (x86_64)"
depends_on:
- ray-extra-image-cpu-build
- ray-extra-image-cuda-build
tags:
- python_dependencies
- docker
- oss
instance_type: medium
- skip-on-premerge

- label: ":crane: publish: ray-llm py{{matrix}} (x86_64)"
key: ray_llm_images_cuda_push
instance_type: small
commands:
- bazel run //ci/ray_ci:build_in_docker -- docker --python-version {{matrix}}
--platform cu12.8.1-cudnn --image-type ray-llm --upload
depends_on:
- manylinux-x86_64
- forge
- ray-llmbase
- bazel run //.buildkite:copy_files -- --destination docker_login
- bazel run //ci/ray_ci/automation:push_ray_image --
--python-version {{matrix}}
--platform cu12.8.1-cudnn
--image-type ray-llm
--architecture x86_64
matrix:
- "3.11"

- label: ":tapioca: build: ray-llm-extra py{{matrix}} docker (x86_64)"
depends_on:
- ray-llm-image-cuda-build
tags:
- python_dependencies
- docker
- oss
instance_type: medium
- skip-on-premerge

- label: ":crane: publish: ray-llm-extra py{{matrix}} (x86_64)"
key: ray_llm_extra_images_cuda_push
instance_type: small
commands:
- bazel run //ci/ray_ci:build_in_docker -- docker --python-version {{matrix}}
--platform cu12.8.1-cudnn --image-type ray-llm-extra --upload
depends_on:
- manylinux-x86_64
- forge
- ray-llmbaseextra
- bazel run //.buildkite:copy_files -- --destination docker_login
- bazel run //ci/ray_ci/automation:push_ray_image --
--python-version {{matrix}}
--platform cu12.8.1-cudnn
--image-type ray-llm-extra
--architecture x86_64
matrix:
- "3.11"
depends_on:
- ray-llm-extra-image-cuda-build
tags:
- python_dependencies
- docker
- oss
- skip-on-premerge

- label: ":tapioca: smoke test build-docker.sh"
tags:
Expand All @@ -244,6 +414,6 @@ steps:
- bazel run .buildkite:copy_files -- --destination docker_login
- bazel run //ci/ray_ci/automation:generate_index -- --prefix nightly
depends_on:
- ray_images
- ray_images_aarch64
- ray_images_push
- ray_images_push_aarch64
- forge
Loading