fix py313 env creation failed when building lmdeploy-builder image (InternLM#3739)

Author: lvhan028

builder/manywheel/Dockerfile_2014

@@ -12,34 +12,37 @@ RUN yum install -y \
     devtoolset-${DEVTOOLSET_VERSION}-gcc \
     devtoolset-${DEVTOOLSET_VERSION}-gcc-c++ \
     devtoolset-${DEVTOOLSET_VERSION}-gcc-gfortran \
-    devtoolset-${DEVTOOLSET_VERSION}-binutils
+    devtoolset-${DEVTOOLSET_VERSION}-binutils \
+    wget \
+    rapidjson-devel \
+    glog-devel && \
+    yum clean all
+
 ENV PATH=/opt/rh/devtoolset-${DEVTOOLSET_VERSION}/root/usr/bin:$PATH
 ENV LD_LIBRARY_PATH=/opt/rh/devtoolset-${DEVTOOLSET_VERSION}/root/usr/lib64:/opt/rh/devtoolset-${DEVTOOLSET_VERSION}/root/usr/lib:$LD_LIBRARY_PATH
 
-RUN yum install -y wget rapidjson-devel glog-devel
-
 FROM base as cuda
-ADD manywheel/scripts/install_cuda.sh install_cuda.sh
-RUN bash ./install_cuda.sh ${BASE_CUDA_VERSION} && rm install_cuda.sh
+COPY manywheel/scripts/install_cuda.sh /tmp/install_cuda.sh
+RUN bash /tmp/install_cuda.sh ${BASE_CUDA_VERSION} && rm /tmp/install_cuda.sh
 
 FROM base as conda
-ADD manywheel/scripts/install_conda.sh install_conda.sh
-RUN bash ./install_conda.sh && rm install_conda.sh
-
-RUN /opt/conda/bin/conda create -n py39 python=3.9 -yq && \
-    /opt/conda/envs/py39/bin/pip install pybind11 && \
-    /opt/conda/bin/conda create -n py310 python=3.10 -yq && \
-    /opt/conda/envs/py310/bin/pip install pybind11 && \
-    /opt/conda/bin/conda create -n py311 python=3.11 -yq && \
-    /opt/conda/envs/py311/bin/pip install pybind11 && \
-    /opt/conda/bin/conda create -n py312 python=3.12 -yq && \
-    /opt/conda/envs/py312/bin/pip install pybind11
-    /opt/conda/bin/conda create -n py313 python=3.13 -yq && \
-    /opt/conda/envs/py313/bin/pip install pybind11
+COPY manywheel/scripts/install_conda.sh /tmp/install_conda.sh
+RUN bash /tmp/install_conda.sh && rm /tmp/install_conda.sh
+
+# Accept Anaconda's Terms of Service to avoid `CondaToSNonInteractiveError`
+RUN /opt/conda/bin/conda tos accept --override-channels --channel https://repo.anaconda.com/pkgs/main && \
+    /opt/conda/bin/conda tos accept --override-channels --channel https://repo.anaconda.com/pkgs/r
+
+RUN PY_VERSIONS=(3.9 3.10 3.11 3.12 3.13) && \
+    for pyver in "${PY_VERSIONS[@]}"; do \
+        /opt/conda/bin/conda create -n py${pyver//./} python=${pyver} -yq && \
+        /opt/conda/envs/py${pyver//./}/bin/pip install --no-cache-dir pybind11; \
+    done && \
+    /opt/conda/bin/conda clean -ya
 
 FROM base as cuda_final
-COPY --from=cuda            /usr/local/cuda-${BASE_CUDA_VERSION}  /usr/local/cuda-${BASE_CUDA_VERSION}
+COPY --from=cuda /usr/local/cuda-${BASE_CUDA_VERSION} /usr/local/cuda-${BASE_CUDA_VERSION}
 RUN ln -sf /usr/local/cuda-${BASE_CUDA_VERSION} /usr/local/cuda
 ENV PATH=/usr/local/cuda/bin:$PATH
-COPY --from=conda           /opt/conda                            /opt/conda
+COPY --from=conda /opt/conda /opt/conda
 RUN /opt/conda/bin/conda init bash

builder/manywheel/README.md

@@ -1,22 +1,25 @@
-# Build lmdeploy manylinux wheel
+# LMDeploy Build System
 
-## Prepare docker image
+## Building lmdeploy builder images
 
-To build all docker images you can use the convenient script:
+To build all lmdeploy builder images, such as "lmdeploy-builder:cuda11.8", ""lmdeploy-builder:cuda12.4", execute:
 
 ```bash
-./build_all_docker.sh
-# Build with pushing
-WITH_PUSH=true ./build_all_docker.sh
+./build_all_lmdeploy_builders.sh
+
+# Build and push images (for CI/CD)
+WITH_PUSH=true ./build_all_lmdeploy_builders.sh
 ```
 
-To build a docker image with specific cuda version or manylinux-docker version, you may use:
+For custom builds with specific versions:
 
 ```bash
-MANY_LINUX_VERSION=2014 GPU_ARCH_VERSION=12.4 ./build_docker.sh
+MANY_LINUX_VERSION=2014 GPU_ARCH_VERSION=12.4 ./build_lmdeploy_builder.sh
 ```
 
-## Build lmdeploy wheel
+## Build lmdeploy wheels
+
+Compile all wheel packages:
 
 ```bash
 ./build_all_wheel.sh

builder/manywheel/build_all_docker.sh renamed to builder/manywheel/build_all_lmdeploy_builders.sh

@@ -4,6 +4,6 @@ set -eou pipefail
 
 TOPDIR=$(git rev-parse --show-toplevel)/builder
 
-for cuda_version in 12.4; do
-    MANY_LINUX_VERSION=2014 GPU_ARCH_VERSION="${cuda_version}" "${TOPDIR}/manywheel/build_docker.sh"
+for cuda_version in 11.8 12.4; do
+    MANY_LINUX_VERSION=2014 GPU_ARCH_VERSION="${cuda_version}" "${TOPDIR}/manywheel/build_lmdeploy_builder.sh"
 done

builder/manywheel/build_docker.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+
+set -eou pipefail
+
+TOPDIR=$(git rev-parse --show-toplevel)/builder
+GPU_ARCH_VERSION=${GPU_ARCH_VERSION}
+WITH_PUSH=${WITH_PUSH:-}
+
+TARGET=cuda_final
+DOCKER_TAG=cuda${GPU_ARCH_VERSION}
+
+DOCKER_IMAGE=openmmlab/lmdeploy-builder:${DOCKER_TAG}
+DOCKERFILE_SUFFIX=$([[ -n ${MANY_LINUX_VERSION} ]] && echo "_${MANY_LINUX_VERSION}" || echo "")
+
+# List of all build arguments (format: KEY=VALUE)
+# Empty values will be automatically filtered out later
+BUILD_ARGS=(
+    "BASE_CUDA_VERSION=${GPU_ARCH_VERSION}"
+    "DEVTOOLSET_VERSION=9"
+    "HTTPS_PROXY=${HTTPS_PROXY:-}"
+    "HTTP_PROXY=${HTTP_PROXY:-}"
+    # Add more parameters here if needed
+)
+
+# Base Docker build command arguments
+docker_build_args=(
+    -t "${DOCKER_IMAGE}"
+    --target "${TARGET}"
+    -f "${TOPDIR}/manywheel/Dockerfile${DOCKERFILE_SUFFIX}"
+)
+
+# Process build arguments: filter empty values and format as --build-arg
+for arg in "${BUILD_ARGS[@]}"; do
+    IFS='=' read -r key value <<< "$arg"  # Split KEY=VALUE
+    if [[ -n "$value" ]]; then  # Only add non-empty values
+        docker_build_args+=(--build-arg "$arg")
+    fi
+done
+
+(
+    set -x
+    DOCKER_BUILDKIT=1 docker build "${docker_build_args[@]}" "${TOPDIR}"
+)
+
+if [[ "${WITH_PUSH}" == true ]]; then
+    (
+        set -x
+        docker push "${DOCKER_IMAGE}"
+    )
+fi