From f3114952fff65daadb022a6a80441d3813919bb4 Mon Sep 17 00:00:00 2001 From: Kai Fricke Date: Fri, 22 Sep 2023 14:55:19 -0700 Subject: [PATCH] [wip][dependencies] pip-compile dependencies as a build step (#39610) It's currently complicated to update dependencies, as it requires a (manual) recompilation of the `requirements_compiled.txt` file. This PR adds a separate build job that runs pip compile on all input files to generate an updated `requirements_compiled.txt`, that is than exposed as an artifact in the buildkite job. To update a dependency in the future, the process would be: 1. Open a PR that updates the respective requirements in the input requirements file 2. Wait for the CI job to run through and output the updated compiled requirements file 3. If the compilation was successful, add the contents of the file to the PR Signed-off-by: Kai Fricke --- .buildkite/build.rayci.yml | 12 +++++++++++- .buildkite/hooks/pre-command | 4 ++++ ci/ci.sh | 2 +- 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/.buildkite/build.rayci.yml b/.buildkite/build.rayci.yml index 6da3cb67c885d..135a716aa9b3f 100644 --- a/.buildkite/build.rayci.yml +++ b/.buildkite/build.rayci.yml @@ -47,8 +47,18 @@ steps: commands: - bazel run //ci/ray_ci:build_in_docker -- docker --python-version py38 --platform cu118 --image-type ray-ml - depends_on: + depends_on: - manylinux - forge - ray-mlpy38cu118base job_env: forge + + - label: ":tapioca: build: pip-compile dependencies" + instance_type: small + commands: + - rm ./python/requirements_compiled.txt + - ./ci/ci.sh compile_pip_dependencies + - cp -f ./python/requirements_compiled.txt /artifact-mount/ + depends_on: + - forge + job_env: forge diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 02194c6ba76be..d8f5462c8d700 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -25,5 +25,9 @@ if [ -d "/tmp/artifacts" ]; then find /tmp/artifacts -print || true fi +echo "Creating var/ artifact directory..." +docker run --rm -v /tmp/artifacts:/artifact-mount alpine:latest /bin/sh -c 'chown -R 2000 /artifact-mount/' || true + + RAYCI_CHECKOUT_DIR="$(pwd)" export RAYCI_CHECKOUT_DIR diff --git a/ci/ci.sh b/ci/ci.sh index 7855ae2e19170..0e8cc61380d80 100755 --- a/ci/ci.sh +++ b/ci/ci.sh @@ -113,7 +113,7 @@ compile_pip_dependencies() { pip install --no-cache-dir numpy torch pip-compile --resolver=backtracking -q \ - --pip-args --no-deps --strip-extras --no-annotate --no-header -o \ + --pip-args --no-deps --strip-extras --no-header -o \ "${WORKSPACE_DIR}/python/$TARGET" \ "${WORKSPACE_DIR}/python/requirements.txt" \ "${WORKSPACE_DIR}/python/requirements/lint-requirements.txt" \