[image] comply with constraint file in base image (ray-project#48764)

also uses 1.3 syntax and heredoc for multiline commands

Signed-off-by: Lonnie Liu <lonnie@anyscale.com>

ci/docker/ray-ml.cpu.base.wanda.yaml

@@ -3,7 +3,6 @@ froms: ["cr.ray.io/rayproject/ray-py$PYTHON_VERSION-cpu-base"]
 dockerfile: docker/ray-ml/Dockerfile
 srcs:
   - python/requirements.txt
-  - python/requirements_compiled.txt
   - python/requirements/ml/dl-cpu-requirements.txt
   - python/requirements/ml/dl-gpu-requirements.txt
   - python/requirements/ml/core-requirements.txt

ci/docker/ray-ml.cuda.base.wanda.yaml

@@ -3,7 +3,6 @@ froms: ["cr.ray.io/rayproject/ray-py$PYTHON_VERSION-cu$CUDA_VERSION-base"]
 dockerfile: docker/ray-ml/Dockerfile
 srcs:
   - python/requirements.txt
-  - python/requirements_compiled.txt
   - python/requirements/ml/dl-cpu-requirements.txt
   - python/requirements/ml/dl-gpu-requirements.txt
   - python/requirements/ml/core-requirements.txt

ci/docker/ray.cpu.base.aarch64.wanda.yaml

@@ -1,6 +1,8 @@
 name: "ray-py$PYTHON_VERSION-cpu-base-aarch64"
 froms: ["ubuntu:22.04"]
 dockerfile: docker/base-deps/Dockerfile
+srcs:
+  - python/requirements_compiled.txt
 build_args:
   - PYTHON_VERSION
   - BASE_IMAGE=ubuntu:22.04

ci/docker/ray.cpu.base.wanda.yaml

@@ -1,6 +1,8 @@
 name: "ray-py$PYTHON_VERSION-cpu-base"
 froms: ["ubuntu:22.04"]
 dockerfile: docker/base-deps/Dockerfile
+srcs:
+  - python/requirements_compiled.txt
 build_args:
   - PYTHON_VERSION
   - BASE_IMAGE=ubuntu:22.04

ci/docker/ray.cuda.base.aarch64.wanda.yaml

@@ -1,6 +1,8 @@
 name: "ray-py$PYTHON_VERSION-cu$CUDA_VERSION-base-aarch64"
 froms: ["nvidia/cuda:$CUDA_VERSION-devel-ubuntu22.04"]
 dockerfile: docker/base-deps/Dockerfile
+srcs:
+  - python/requirements_compiled.txt
 build_args:
   - PYTHON_VERSION
   - BASE_IMAGE=nvidia/cuda:$CUDA_VERSION-devel-ubuntu22.04

ci/docker/ray.cuda.base.wanda.yaml

@@ -1,6 +1,8 @@
 name: "ray-py$PYTHON_VERSION-cu$CUDA_VERSION-base"
 froms: ["nvidia/cuda:$CUDA_VERSION-devel-ubuntu22.04"]
 dockerfile: docker/base-deps/Dockerfile
+srcs:
+  - python/requirements_compiled.txt
 build_args:
   - PYTHON_VERSION
   - BASE_IMAGE=nvidia/cuda:$CUDA_VERSION-devel-ubuntu22.04

docker/base-deps/Dockerfile

@@ -1,3 +1,5 @@
+# syntax=docker/dockerfile:1.3-labs
+
 # The base-deps Docker image installs main libraries needed to run Ray
 
 # The GPU options are NVIDIA CUDA developer images.
@@ -19,71 +21,109 @@ ARG HOSTTYPE=${HOSTTYPE:-x86_64}
 ARG RAY_UID=1000
 ARG RAY_GID=100
 
-RUN apt-get update -y \
-    && apt-get install -y sudo tzdata \
-    && useradd -ms /bin/bash -d /home/ray ray --uid $RAY_UID --gid $RAY_GID \
-    && usermod -aG sudo ray \
-    && echo 'ray ALL=NOPASSWD: ALL' >> /etc/sudoers \
-    && rm -rf /var/lib/apt/lists/* \
-    && apt-get clean
+RUN <<EOF
+#!/bin/bash
+
+set -euo pipefail
+
+apt-get update -y
+apt-get upgrade -y
+
+APT_PKGS=(
+    sudo
+    tzdata
+    git
+    libjemalloc-dev
+    wget
+    cmake
+    g++ 
+    zlib1g-dev
+)
+if [[ "$AUTOSCALER" == "autoscaler" ]]; then
+    APT_PKGS+=(
+        tmux
+        screen
+        rsync
+        netbase
+        openssh-client
+        gnupg
+    )
+fi
+
+apt-get install -y "${APT_PKGS[@]}"
+
+useradd -ms /bin/bash -d /home/ray ray --uid $RAY_UID --gid $RAY_GID
+usermod -aG sudo ray
+echo 'ray ALL=NOPASSWD: ALL' >> /etc/sudoers
+
+EOF
 
 USER $RAY_UID
 ENV HOME=/home/ray
 
+COPY python/requirements_compiled.txt /home/ray/requirements_compiled.txt
+
 SHELL ["/bin/bash", "-c"]
-RUN sudo apt-get update -y && sudo apt-get upgrade -y \
-    && sudo apt-get install -y \
-        git \
-        libjemalloc-dev \
-        wget \
-        cmake \
-        g++ \ 
-        zlib1g-dev \
-        $(if [[ "$AUTOSCALER" == "autoscaler" ]]; then echo \
-        tmux \
-        screen \
-        rsync \
-        netbase \
-        openssh-client \
-        gnupg; fi) \
-    && wget --quiet \
-        "https://repo.anaconda.com/miniconda/Miniconda3-py311_24.4.0-0-Linux-${HOSTTYPE}.sh" \
-        -O /tmp/miniconda.sh \
-    && /bin/bash /tmp/miniconda.sh -b -u -p $HOME/anaconda3 \
-    && $HOME/anaconda3/bin/conda init \ 
-    && echo 'export PATH=$HOME/anaconda3/bin:$PATH' >> /home/ray/.bashrc \
-    && rm /tmp/miniconda.sh  \
-    && $HOME/anaconda3/bin/conda install -y libgcc-ng python=$PYTHON_VERSION \
-    && $HOME/anaconda3/bin/conda install -y -c conda-forge libffi=3.4.2 \
-    && $HOME/anaconda3/bin/conda clean -y --all \
-    && $HOME/anaconda3/bin/pip install --no-cache-dir \
-        flatbuffers \
-        cython==0.29.37 \
-        # Necessary for Dataset to work properly.
-        numpy\>=1.20 \
-        psutil \
-        # Required a recent version of setuptools to be compatible with python 3.12+.
-        setuptools==71.1.0 \
-    # To avoid the following error on Jenkins:
-    # AttributeError: 'numpy.ufunc' object has no attribute '__module__'
-    && $HOME/anaconda3/bin/pip uninstall -y dask \ 
-    # We install cmake temporarily to get psutil
-    && sudo apt-get autoremove -y cmake zlib1g-dev \
-        # We keep g++ on GPU images, because uninstalling removes CUDA Devel tooling
-        $(if [[ "$BASE_IMAGE" == "ubuntu:22.04" && "$HOSTTYPE" == "x86_64" ]]; then echo \
-        g++; fi) \
-    && sudo rm -rf /var/lib/apt/lists/* \
-    && sudo apt-get clean \
-    && (if [[ "$AUTOSCALER" == "autoscaler" ]]; \
-        then $HOME/anaconda3/bin/pip --no-cache-dir install \
-        "redis>=3.5.0,<4.0.0" \
-        "six==1.13.0" \
-        "boto3==1.26.76" \
-        "pyOpenSSL==22.1.0" \
-        "cryptography==38.0.1" \
-        "google-api-python-client==1.7.8" \
-        "google-oauth" \
-        ; \
-    fi;)
+
+RUN <<EOF
+#!/bin/bash
+
+set -euo pipefail
+
+# Install miniconda
+wget --quiet \
+    "https://repo.anaconda.com/miniconda/Miniconda3-py311_24.4.0-0-Linux-${HOSTTYPE}.sh" \
+    -O /tmp/miniconda.sh
+
+/bin/bash /tmp/miniconda.sh -b -u -p $HOME/anaconda3
+
+$HOME/anaconda3/bin/conda init
+echo 'export PATH=$HOME/anaconda3/bin:$PATH' >> /home/ray/.bashrc
+rm /tmp/miniconda.sh
+$HOME/anaconda3/bin/conda install -y libgcc-ng python=$PYTHON_VERSION
+$HOME/anaconda3/bin/conda install -y -c conda-forge libffi=3.4.2
+$HOME/anaconda3/bin/conda clean -y --all
+
+PIP_PKGS=(
+    # Required a recent version of setuptools to be compatible with python 3.12+.
+    setuptools==71.1.0
+
+    flatbuffers
+    cython
+    numpy  # Necessary for Dataset to work properly.
+    psutil
+)
+if [[ "$AUTOSCALER" == "autoscaler" ]]; then
+    PIP_PKGS+=(
+        redis
+        six
+        boto3
+        pyopenssl
+        cryptography
+        google-api-python-client
+        google-oauth
+    )
+fi
+
+$HOME/anaconda3/bin/pip install --no-cache-dir \
+    -c $HOME/requirements_compiled.txt \
+    "${PIP_PKGS[@]}"
+
+# To avoid the following error on Jenkins:
+# AttributeError: 'numpy.ufunc' object has no attribute '__module__'
+$HOME/anaconda3/bin/pip uninstall -y dask
+
+# We install cmake temporarily to get psutil
+sudo apt-get autoremove -y cmake zlib1g-dev
+
+# We keep g++ on GPU images, because uninstalling removes CUDA Devel tooling
+if [[ "$BASE_IMAGE" == "ubuntu:22.04" && "$HOSTTYPE" == "x86_64" ]]; then
+    sudo apt-get autoremove -y g++
+fi
+
+sudo rm -rf /var/lib/apt/lists/*
+sudo apt-get clean
+
+EOF
 
 WORKDIR $HOME

docker/ray-ml/Dockerfile

@@ -1,19 +1,13 @@
+# syntax=docker/dockerfile:1.3-labs
+
 ARG BASE_IMAGE
 ARG FULL_BASE_IMAGE=rayproject/ray:nightly"$BASE_IMAGE"
 FROM "$FULL_BASE_IMAGE"
 
-# The python/* paths only exist in civ2, so we put them as non-first arguments. Docker
-# will ignore non-existent paths if they are non-first arguments.
-#
-# TODO(can): simplify this once civ1 is completely deprecated.
-COPY *requirements.txt \
-     python/*requirements.txt \
+COPY python/*requirements.txt \
      python/requirements/ml/*requirements.txt  \
      python/requirements/docker/*requirements.txt ./
-COPY *requirements_compiled.txt \
-     python/*requirements_compiled.txt ./
-COPY *install-ml-docker-requirements.sh \
-     docker/ray-ml/*install-ml-docker-requirements.sh ./
+COPY docker/ray-ml/install-ml-docker-requirements.sh ./
 
 RUN sudo chmod +x install-ml-docker-requirements.sh \
     && ./install-ml-docker-requirements.sh

docker/ray-ml/install-ml-docker-requirements.sh

@@ -52,7 +52,8 @@ pip --no-cache-dir install \
 
 sudo apt-get clean
 
-sudo rm ./*requirements*.txt
+# requirements_compiled.txt will be kept.
+sudo rm ./*requirements.txt requirements_compiled_gpu.txt
 
 # MuJoCo Installation.
 export MUJOCO_GL=osmesa

docker/ray/Dockerfile

@@ -1,3 +1,5 @@
+# syntax=docker/dockerfile:1.3-labs
+
 ARG BASE_IMAGE
 ARG FULL_BASE_IMAGE=rayproject/ray-deps:nightly"$BASE_IMAGE"
 FROM $FULL_BASE_IMAGE
@@ -6,7 +8,6 @@ ARG WHEEL_PATH
 ARG FIND_LINKS_PATH=".whl"
 ARG CONSTRAINTS_FILE="requirements_compiled.txt"
 
-COPY requirements_compiled.txt ./
 COPY $WHEEL_PATH .
 COPY $FIND_LINKS_PATH $FIND_LINKS_PATH