[ci] add civ2 ray_ci script using hermetic python (#37993)

Signed-off-by: Lonnie Liu <lonnie@anyscale.com>

BUILD.bazel

@@ -7,17 +7,58 @@
 # If you would like to help with the move in your PR, please use `git mv` so that the history of the file is retained.
 
 load("@rules_proto//proto:defs.bzl", "proto_library")
-load("@rules_python//python:defs.bzl", "py_library")
+load("@rules_python//python:defs.bzl", "py_library", "py_runtime", "py_runtime_pair")
 load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "cc_proto_library", "cc_test")
 load("@com_github_grpc_grpc//bazel:cc_grpc_library.bzl", "cc_grpc_library")
 load("@com_github_grpc_grpc//bazel:cython_library.bzl", "pyx_library")
 load("@com_github_google_flatbuffers//:build_defs.bzl", "flatbuffer_cc_library")
 load("//bazel:ray.bzl", "COPTS", "PYX_COPTS", "PYX_SRCS", "copy_to_workspace")
+load("@python3_9//:defs.bzl", python39 = "interpreter")
 
 package(
     default_visibility = ["//visibility:public"],
 )
 
+# Hermetic python envionment, currently only used for CI infra and scripts.
+
+py_runtime(
+    name = "python3_runtime",
+    interpreter = python39,
+    python_version = "PY3",
+    visibility = ["//visibility:private"],
+)
+
+py_runtime_pair(
+    name = "python_runtime_pair",
+    py2_runtime = None,
+    py3_runtime = ":python3_runtime",
+    visibility = ["//visibility:private"],
+)
+
+constraint_setting(name = "hermetic")
+
+constraint_value(
+    name = "hermetic_python",
+    constraint_setting = ":hermetic",
+    visibility = ["//visibility:public"],
+)
+
+toolchain(
+    name = "python_toolchain",
+    exec_compatible_with = [":hermetic_python"],
+    toolchain = ":python_runtime_pair",
+    toolchain_type = "@bazel_tools//tools/python:toolchain_type",
+)
+
+platform(
+    name = "hermetic_python_platform",
+    constraint_values = [":hermetic_python"],
+    parents = ["@local_config_platform//:host"],
+    visibility = ["//visibility:private"],
+)
+
+# C/C++ toolchain constraint configs.
+
 config_setting(
     name = "msvc-cl",
     flag_values = {"@bazel_tools//tools/cpp:compiler": "msvc-cl"},
@@ -475,10 +516,10 @@ cc_library(
         "//src/ray/common:asio",
         "//src/ray/common:constants",
         "//src/ray/common:event_stats",
+        "//src/ray/common:file_system_monitor",
         "//src/ray/common:grpc_util",
         "//src/ray/common:id",
         "//src/ray/common:memory_monitor",
-        "//src/ray/common:file_system_monitor",
         "//src/ray/common:network",
         "//src/ray/common:ray_config",
         "//src/ray/common:ray_syncer",
@@ -748,11 +789,11 @@ cc_library(
         ":worker_rpc",
         "//src/ray/protobuf:agent_manager_cc_proto",
         "//src/ray/protobuf:common_cc_proto",
-        "//src/ray/util",
         "//src/ray/protobuf:runtime_env_agent_cc_proto",
+        "//src/ray/util",
         "@boost//:asio",
-        "@boost//:system",
         "@boost//:beast",
+        "@boost//:system",
         "@com_github_jupp0r_prometheus_cpp//pull",
         "@com_google_absl//absl/base:core_headers",
         "@com_google_absl//absl/container:flat_hash_set",
@@ -829,7 +870,7 @@ cc_library(
         "//src/ray/util",
         "@boost//:circular_buffer",
         "@boost//:fiber",
-        "@com_google_absl//absl/cleanup:cleanup",
+        "@com_google_absl//absl/cleanup",
         "@com_google_absl//absl/container:btree",
         "@com_google_absl//absl/container:flat_hash_map",
         "@com_google_absl//absl/container:flat_hash_set",

WORKSPACE

@@ -45,7 +45,7 @@ python_register_toolchains(
     register_toolchains = False,
 )
 
-load("@python3_9//:defs.bzl", bk_python = "interpreter")
+load("@python3_9//:defs.bzl", python39 = "interpreter")
 load("@rules_python//python/pip_install:repositories.bzl", "pip_install_dependencies")
 
 pip_install_dependencies()
@@ -54,17 +54,26 @@ load("@rules_python//python:pip.bzl", "pip_parse")
 
 pip_parse(
     name = "py_deps_buildkite",
-    python_interpreter_target = bk_python,
+    python_interpreter_target = python39,
+    requirements_lock = "//release:requirements_buildkite.txt",
+)
+
+pip_parse(
+    name = "py_deps_ray_ci",
+    python_interpreter_target = python39,
     requirements_lock = "//release:requirements_buildkite.txt",
 )
 
 load("@py_deps_buildkite//:requirements.bzl", install_py_deps_buildkite = "install_deps")
+load("@py_deps_ray_ci//:requirements.bzl", install_py_deps_ray_ci = "install_deps")
 
 install_py_deps_buildkite()
 
-register_toolchains("//release:python_toolchain")
+install_py_deps_ray_ci()
+
+register_toolchains("//:python_toolchain")
 
 register_execution_platforms(
     "@local_config_platform//:host",
-    "//release:hermetic_python_platform",
+    "//:hermetic_python_platform",
 )

ci/ray_ci/BUILD.bazel

@@ -0,0 +1,49 @@
+load("@rules_python//python:defs.bzl", "py_binary", "py_library", "py_test")
+load("@rules_python//python:pip.bzl", "compile_pip_requirements")
+load("@py_deps_ray_ci//:requirements.bzl", ci_require = "requirement")
+
+compile_pip_requirements(
+    name = "requirements",
+    requirements_in = "requirements.in",
+    requirements_txt = "requirements.txt",
+    tags = [
+        "team:ci",
+    ],
+    visibility = ["//visibility:private"],
+)
+
+py_library(
+    name = "ray_ci_lib",
+    srcs = glob(
+        ["*.py"],
+        exclude = [
+            "test_*.py",
+            "ray_ci.py",
+        ],
+    ),
+    visibility = ["//visibility:private"],
+    deps = [
+        ci_require("pyyaml"),
+        ci_require("click"),
+    ],
+)
+
+py_binary(
+    name = "ray_ci",
+    srcs = ["ray_ci.py"],
+    deps = [":ray_ci_lib"],
+)
+
+py_test(
+    name = "test_runner",
+    size = "small",
+    srcs = ["test_runner.py"],
+    tags = [
+        "ci_unit",
+        "team:ci",
+    ],
+    deps = [
+        ":ray_ci_lib",
+        ci_require("pytest"),
+    ],
+)

ci/ray_ci/core.tests.yml

@@ -0,0 +1,14 @@
+flaky_tests:
+  - //python/ray/tests:test_runtime_env_working_dir_3
+  - //python/ray/tests:test_placement_group_3
+  - //python/ray/tests:test_memory_pressure
+  - //python/ray/tests:test_placement_group_5
+  - //python/ray/tests:test_runtime_env_2
+  - //python/ray/tests:test_gcs_fault_tolerance
+  - //python/ray/tests:test_gcs_ha_e2e
+  - //python/ray/tests:test_plasma_unlimited
+  - //python/ray/tests:test_scheduling_performance
+  - //python/ray/tests:test_object_manager
+  - //python/ray/tests:test_tensorflow
+  - //python/ray/tests:test_threaded_actor
+  - //python/ray/tests:test_unhandled_error

ci/ray_ci/lint.sh

@@ -0,0 +1,107 @@
+#!/usr/bin/env bash
+
+set -euxo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)/.."
+WORKSPACE_DIR="${ROOT_DIR}/.."
+
+readme() {
+  if python -s -c "import docutils" >/dev/null 2>/dev/null; then
+    (
+      cd "${WORKSPACE_DIR}"/python
+      python setup.py check --restructuredtext --strict --metadata
+    )
+  else
+    echo "Skipping README lint because the docutils package is not installed" 1>&2
+  fi
+}
+
+code_format() {
+  FORMAT_SH_PRINT_DIFF=1 "${ROOT_DIR}"/lint/format.sh --all-scripts
+}
+
+banned_words() {
+  "${ROOT_DIR}"/lint/check-banned-words.sh
+}
+
+annotations() {
+  "${ROOT_DIR}"/lint/check_api_annotations.py
+}
+
+run_semgrep() {
+  semgrep ci --config "${WORKSPACE_DIR}/semgrep.yml"
+}
+
+bazel_buildifier() {
+  LINT_BAZEL_TMP="$(mktemp -d)"
+  curl -sl "https://github.com/bazelbuild/buildtools/releases/download/v6.1.2/buildifier-linux-amd64" \
+    -o "${LINT_BAZEL_TMP}/buildifier"
+  chmod +x "${LINT_BAZEL_TMP}/buildifier"
+  BUILDIFIER="${LINT_BAZEL_TMP}/buildifier" "${ROOT_DIR}/lint/bazel-format.sh"
+
+  rm -rf "${LINT_BAZEL_TMP}"  # Clean up
+}
+
+bazel_pytest() {
+  pip install yq
+  cd "${WORKSPACE_DIR}"
+  for team in "team:core" "team:ml" "team:rllib" "team:serve"; do
+    # this does the following:
+    # - find all py_test rules in bazel that have the specified team tag EXCEPT ones with "no_main" tag and outputs them as xml
+    # - converts the xml to json
+    # - feeds the json into pytest_checker.py
+    bazel query "kind(py_test.*, tests(python/...) intersect attr(tags, \"\b$team\b\", python/...) except attr(tags, \"\bno_main\b\", python/...))" --output xml | xq | python ci/lint/pytest_checker.py
+  done
+}
+
+web() {
+  (
+    cd "${WORKSPACE_DIR}"/python/ray/dashboard/client
+    set +x # suppress set -x since it'll get very noisy here
+
+    if [ -z "${BUILDKITE-}" ]; then
+      . "${HOME}/.nvm/nvm.sh"
+      NODE_VERSION="14"
+      nvm install $NODE_VERSION
+      nvm use --silent $NODE_VERSION
+    fi
+
+    npm ci
+    local filenames
+    # shellcheck disable=SC2207
+    filenames=($(find src -name "*.ts" -or -name "*.tsx"))
+    node_modules/.bin/eslint --max-warnings 0 "${filenames[@]}"
+    node_modules/.bin/prettier --check "${filenames[@]}"
+    node_modules/.bin/prettier --check public/index.html
+  )
+}
+
+copyright() {
+  (
+    "${ROOT_DIR}"/lint/copyright-format.sh -c
+  )
+}
+
+clang() {
+  if command -v clang-format > /dev/null; then
+    "${ROOT_DIR}"/lint/check-git-clang-format-output.sh
+  else
+    { echo "WARNING: Skipping linting C/C++ as clang-format is not installed."; } 2> /dev/null
+  fi
+}
+
+test_script() {
+  # lint test script
+  pushd "${WORKSPACE_DIR}"
+    bazel query 'kind("cc_test", //...)' --output=xml | python "${ROOT_DIR}"/lint/check-bazel-team-owner.py
+    bazel query 'kind("py_test", //...)' --output=xml | python "${ROOT_DIR}"/lint/check-bazel-team-owner.py
+  popd
+}
+
+test_run() {
+  # Make sure tests will be run by CI.
+  python "${ROOT_DIR}"/pipeline/check-test-run.py
+}
+
+LINT=1 "${ROOT_DIR}"/env/install-dependencies.sh
+"$@"

ci/ray_ci/ray_ci.py

@@ -0,0 +1,4 @@
+from ci.ray_ci.runner import main
+
+if __name__ == "__main__":
+    main()

ci/ray_ci/requirements.in

@@ -0,0 +1,3 @@
+click
+pytest
+pyyaml

ci/ray_ci/requirements.txt

@@ -0,0 +1,76 @@
+#
+# This file is autogenerated by pip-compile with python 3.8
+# To update, run:
+#
+#    bazel run //ci/ray_ci:requirements.update
+#
+click==8.1.6 \
+    --hash=sha256:48ee849951919527a045bfe3bf7baa8a959c423134e1a5b98c05c20ba75a1cbd \
+    --hash=sha256:fa244bb30b3b5ee2cae3da8f55c9e5e0c0e86093306301fb418eb9dc40fbded5
+    # via -r ci/ray_ci/requirements.in
+exceptiongroup==1.1.2 \
+    --hash=sha256:12c3e887d6485d16943a309616de20ae5582633e0a2eda17f4e10fd61c1e8af5 \
+    --hash=sha256:e346e69d186172ca7cf029c8c1d16235aa0e04035e5750b4b95039e65204328f
+    # via pytest
+iniconfig==2.0.0 \
+    --hash=sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3 \
+    --hash=sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374
+    # via pytest
+packaging==23.1 \
+    --hash=sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61 \
+    --hash=sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f
+    # via pytest
+pluggy==1.2.0 \
+    --hash=sha256:c2fd55a7d7a3863cba1a013e4e2414658b1d07b6bc57b3919e0c63c9abb99849 \
+    --hash=sha256:d12f0c4b579b15f5e054301bb226ee85eeeba08ffec228092f8defbaa3a4c4b3
+    # via pytest
+pytest==7.4.0 \
+    --hash=sha256:78bf16451a2eb8c7a2ea98e32dc119fd2aa758f1d5d66dbf0a59d69a3969df32 \
+    --hash=sha256:b4bf8c45bd59934ed84001ad51e11b4ee40d40a1229d2c79f9c592b0a3f6bd8a
+    # via -r ci/ray_ci/requirements.in
+pyyaml==6.0.1 \
+    --hash=sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc \
+    --hash=sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741 \
+    --hash=sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206 \
+    --hash=sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27 \
+    --hash=sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595 \
+    --hash=sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62 \
+    --hash=sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98 \
+    --hash=sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696 \
+    --hash=sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d \
+    --hash=sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867 \
+    --hash=sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47 \
+    --hash=sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486 \
+    --hash=sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6 \
+    --hash=sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3 \
+    --hash=sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007 \
+    --hash=sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938 \
+    --hash=sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c \
+    --hash=sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735 \
+    --hash=sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d \
+    --hash=sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba \
+    --hash=sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8 \
+    --hash=sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5 \
+    --hash=sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd \
+    --hash=sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3 \
+    --hash=sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0 \
+    --hash=sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515 \
+    --hash=sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c \
+    --hash=sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c \
+    --hash=sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924 \
+    --hash=sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34 \
+    --hash=sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43 \
+    --hash=sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859 \
+    --hash=sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673 \
+    --hash=sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a \
+    --hash=sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab \
+    --hash=sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa \
+    --hash=sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c \
+    --hash=sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585 \
+    --hash=sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d \
+    --hash=sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f
+    # via -r ci/ray_ci/requirements.in
+tomli==2.0.1 \
+    --hash=sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc \
+    --hash=sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f
+    # via pytest