From 254b1ec370a07bbd7d60c502ce3b38f4c1fef567 Mon Sep 17 00:00:00 2001 From: mehrdadn Date: Wed, 29 Apr 2020 21:19:02 -0700 Subject: [PATCH] Set up testing and wheels for Windows on GitHub Actions (#8131) * Move some Java tests into ci.sh * Move C++ worker tests into ci.sh * Define run() * Prepare to move Python tests into ci.sh * Fix issues in install-dependencies.sh * Reload environment for GitHub Actions * Move wheels to ci.sh and fix related issues * Don't bypass failures in install-ray.sh anymore * Make CI a little quieter * Move linting into ci.sh * Add vitals test right after build * Fix os.uname() unavailability on Windows Co-authored-by: Mehrdad --- .github/workflows/main.yml | 12 + .travis.yml | 65 +---- BUILD.bazel | 2 +- build.sh | 2 +- ci/travis/ci.sh | 266 +++++++++++++++--- ci/travis/install-dependencies.sh | 47 ++-- ci/travis/install-ray.sh | 6 +- ci/travis/test-wheels.sh | 23 +- doc/requirements-doc.txt | 1 - python/ray/dashboard/dashboard.py | 3 +- python/ray/log_monitor.py | 3 +- python/ray/reporter.py | 2 +- python/setup.py | 11 +- src/ray/test/run_core_worker_tests.sh | 0 src/ray/test/run_object_manager_tests.sh | 0 src/ray/test/run_object_manager_valgrind.sh | 0 .../src/test/run_streaming_queue_test.sh | 0 17 files changed, 300 insertions(+), 143 deletions(-) mode change 100644 => 100755 src/ray/test/run_core_worker_tests.sh mode change 100644 => 100755 src/ray/test/run_object_manager_tests.sh mode change 100644 => 100755 src/ray/test/run_object_manager_valgrind.sh mode change 100644 => 100755 streaming/src/test/run_streaming_queue_test.sh diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a89633f222606..3a929f846416e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -18,7 +18,11 @@ jobs: - name: macos os: macos-10.15 env: + GITHUB_PULL_REQUEST: ${{ github.event.number }} PYTHON: 3.6 + TRAVIS_BUILD_DIR: ${{ github.workspace }} + TRAVIS_COMMIT: ${{ github.sha }} + #TRAVIS_PULL_REQUEST is defined in the scripts to account for "false" steps: - name: Pre-checkout system configuration # We add -l to process .bashrc, but need -e -o pipefail for consistency with GitHub Actions's default behavior. @@ -40,9 +44,17 @@ jobs: env: BAZEL_CACHE_CREDENTIAL_B64: ${{ secrets.BAZEL_CACHE_CREDENTIAL_B64 }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + #LINUX_WHEELS: 1 + LINT: 1 + #MAC_WHEELS: 1 + RAY_DEFAULT_BUILD: 1 + WINDOWS_WHEELS: 1 run: | . ./ci/travis/ci.sh init + . ./ci/travis/ci.sh lint . ./ci/travis/ci.sh build + . ./ci/travis/ci.sh test_python + . ./ci/travis/ci.sh test_wheels - name: Run Clang Include-What-You-Use continue-on-error: true if: runner.os == 'Linux' diff --git a/.travis.yml b/.travis.yml index 86934a89599ec..bed3726884f08 100644 --- a/.travis.yml +++ b/.travis.yml @@ -37,7 +37,6 @@ matrix: install: - . ./ci/travis/ci.sh build script: - - . ./ci/travis/ci.sh preload - ./java/test.sh - os: linux @@ -51,7 +50,6 @@ matrix: install: - . ./ci/travis/ci.sh build script: - - . ./ci/travis/ci.sh preload # Streaming cpp test. - if [ $RAY_CI_STREAMING_CPP_AFFECTED == "1" ]; then bazel test --config=ci --test_output=all //streaming:all && bash streaming/src/test/run_streaming_queue_test.sh; fi - if [ $RAY_CI_STREAMING_PYTHON_AFFECTED == "1" ]; then python -m pytest -v --durations=5 --timeout=300 streaming/python/tests/; fi @@ -70,7 +68,6 @@ matrix: install: - . ./ci/travis/ci.sh build script: - - . ./ci/travis/ci.sh preload - ./ci/suppress_output bash src/ray/test/run_core_worker_tests.sh - ./ci/suppress_output bash streaming/src/test/run_streaming_queue_test.sh - ./java/test.sh @@ -88,7 +85,6 @@ matrix: install: - . ./ci/travis/ci.sh build script: - - . ./ci/travis/ci.sh preload - ./ci/keep_alive bazel test --config=ci --test_tag_filters=-jenkins_only python/ray/tests/... - os: linux @@ -100,43 +96,7 @@ matrix: install: - . ./ci/travis/ci.sh build script: - - . ./ci/travis/ci.sh preload - - ./ci/travis/check-git-clang-format-output.sh - # Try generating Sphinx documentation. To do this, we need to install - # Ray first. - - cd doc - # readthedocs has an antiquated build env. - # This is a best effort to reproduce it locally to avoid doc build failures - # and hidden errors. - - pip install -r requirements-rtd.txt - - pip install -r requirements-doc.txt - - pip install yapf==0.23.0 - - sphinx-build -q -W -E -T -b html source _build/html - - cd .. - # Run Python linting, ignore dict vs {} (C408), others are defaults - - flake8 --inline-quotes '"' --no-avoid-escape --exclude=python/ray/core/generated/,streaming/python/generated,doc/source/conf.py,python/ray/cloudpickle/,python/ray/thirdparty_files --ignore=C408,E121,E123,E126,E226,E24,E704,W503,W504,W605 - - ./ci/travis/format.sh --all - # Make sure that the README is formatted properly. - - cd python - - python setup.py check --restructuredtext --strict --metadata - - cd .. - # Run Bazel linter Buildifier. - - wget -q https://dl.google.com/go/go1.14.2.linux-amd64.tar.gz - - tar -xf go1.14.2.linux-amd64.tar.gz - - mkdir $HOME/go_dir - - export GOROOT=`pwd`/go - - export GOPATH="$HOME/go_dir" - - export PATH="$GOROOT/bin:$GOPATH/bin:$PATH" - - go get github.com/bazelbuild/buildtools/buildifier - - ./ci/travis/bazel-format.sh - # Run TypeScript and HTML linting. - - pushd python/ray/dashboard/client - - source "$HOME/.nvm/nvm.sh" - - nvm use node - - node_modules/.bin/eslint --max-warnings 0 $(find src -name "*.ts" -or -name "*.tsx") - - node_modules/.bin/prettier --check $(find src -name "*.ts" -or -name "*.tsx") - - node_modules/.bin/prettier --check public/index.html - - popd + - . ./ci/travis/ci.sh lint # Build Linux wheels. - os: linux @@ -149,9 +109,7 @@ matrix: install: - . ./ci/travis/ci.sh build script: - - . ./ci/travis/ci.sh preload - # Explicitly sleep 60 seconds for logs to go through - - ./ci/travis/test-wheels.sh || { cat /tmp/ray/session_latest/logs/* && sleep 60 && false; } + - . ./ci/travis/ci.sh test_wheels cache: false # Build MacOS wheels. @@ -166,9 +124,7 @@ matrix: install: - . ./ci/travis/ci.sh build script: - - . ./ci/travis/ci.sh preload - # Explicitly sleep 60 seconds for logs to go through - - ./ci/travis/test-wheels.sh || { cat /tmp/ray/session_latest/logs/* && sleep 60 && false; } + - . ./ci/travis/ci.sh test_wheels # RLlib: Learning tests (from rllib/tuned_examples/regression_tests/*.yaml). - os: linux @@ -185,7 +141,6 @@ matrix: install: - . ./ci/travis/ci.sh build script: - - . ./ci/travis/ci.sh preload - travis_wait 120 bazel test --config=ci --test_output=streamed --build_tests_only --test_tag_filters=learning_tests rllib/... # RLlib: Learning tests with tf=1.x (from rllib/tuned_examples/regression_tests/*.yaml). @@ -204,7 +159,6 @@ matrix: install: - . ./ci/travis/ci.sh build script: - - . ./ci/travis/ci.sh preload - travis_wait 120 bazel test --config=ci --test_output=streamed --build_tests_only --test_tag_filters=learning_tests rllib/... # RLlib: Quick Agent train.py runs (compilation & running, no(!) learning). @@ -223,7 +177,6 @@ matrix: install: - . ./ci/travis/ci.sh build script: - - . ./ci/travis/ci.sh preload - travis_wait 60 bazel test --config=ci --build_tests_only --test_tag_filters=quick_train rllib/... # Test everything that does not have any of the "main" labels: # "learning_tests|quick_train|examples|tests_dir". @@ -244,7 +197,6 @@ matrix: install: - . ./ci/travis/ci.sh build script: - - . ./ci/travis/ci.sh preload - ./ci/keep_alive bazel test --config=ci --build_tests_only --test_tag_filters=examples_A,examples_B rllib/... - ./ci/keep_alive bazel test --config=ci --build_tests_only --test_tag_filters=examples_C rllib/... - ./ci/keep_alive bazel test --config=ci --build_tests_only --test_tag_filters=examples_E,examples_L,examples_M,examples_N,examples_P rllib/... @@ -265,7 +217,6 @@ matrix: install: - . ./ci/travis/ci.sh build script: - - . ./ci/travis/ci.sh preload - ./ci/keep_alive bazel test --config=ci --build_tests_only --test_tag_filters=tests_dir_A,tests_dir_B,tests_dir_C,tests_dir_D,tests_dir_E,tests_dir_F,tests_dir_G,tests_dir_H,tests_dir_I rllib/... # RLlib: tests_dir: Everything in rllib/tests/ directory (J-Z). @@ -283,7 +234,6 @@ matrix: install: - . ./ci/travis/ci.sh build script: - - . ./ci/travis/ci.sh preload - ./ci/keep_alive bazel test --config=ci --build_tests_only --test_tag_filters=tests_dir_J,tests_dir_K,tests_dir_L,tests_dir_M,tests_dir_N,tests_dir_O,tests_dir_P,tests_dir_Q,tests_dir_R,tests_dir_S,tests_dir_T,tests_dir_U,tests_dir_V,tests_dir_W,tests_dir_X,tests_dir_Y,tests_dir_Z rllib/... # Cpp worker test @@ -296,11 +246,9 @@ matrix: install: - . ./ci/travis/ci.sh build script: - - . ./ci/travis/ci.sh preload - - bazel test --config=ci //cpp:all --build_tests_only --test_output=streamed + - . ./ci/travis/ci.sh test_cpp script: - - . ./ci/travis/ci.sh preload # raylet integration tests - ./ci/suppress_output bash src/ray/test/run_core_worker_tests.sh - ./ci/suppress_output bash src/ray/test/run_object_manager_tests.sh @@ -312,10 +260,7 @@ script: - if [ $RAY_CI_SERVE_AFFECTED == "1" ]; then ./ci/keep_alive bazel test --config=ci --test_tag_filters=-jenkins_only python/ray/serve/...; fi # ray operator tests - - cd ./deploy/ray-operator/ - - CC=gcc ../../ci/suppress_output go build - - CC=gcc ../../ci/suppress_output go test ./... - - cd ../.. + - (cd deploy/ray-operator && export CC=gcc && suppress_output go build && suppress_output go test ./...) # random python tests TODO(ekl): these should be moved to bazel - if [ $RAY_CI_PYTHON_AFFECTED == "1" ]; then python -m pytest -v --durations=5 --timeout=300 python/ray/experimental/test/async_test.py; fi diff --git a/BUILD.bazel b/BUILD.bazel index 26e60b6ba16c9..dd5a9b866aa6e 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -1530,7 +1530,7 @@ genrule( tmpdir="redis.tmp" && path=$(location @com_github_antirez_redis//:file) && cp -p -L -R -- "$${path%/*}" "$${tmpdir}" && - make -s -C "$${tmpdir}" -j"$$(getconf _NPROCESSORS_ONLN || echo 1)" && + make -s -C "$${tmpdir}" -j"$$(getconf _NPROCESSORS_ONLN || echo 1)" V=0 && mv "$${tmpdir}"/src/redis-server $(location redis-server) && chmod +x $(location redis-server) && mv "$${tmpdir}"/src/redis-cli $(location redis-cli) && diff --git a/build.sh b/build.sh index d2462ef06c8ad..19aa893b6f081 100755 --- a/build.sh +++ b/build.sh @@ -132,7 +132,7 @@ pushd $WORK_DIR git clone https://github.com/suquark/pickle5-backport pushd pickle5-backport git checkout 8ffe41ceba9d5e2ce8a98190f6b3d2f3325e5a72 - CC=gcc "$PYTHON_EXECUTABLE" setup.py bdist_wheel + CC=gcc "$PYTHON_EXECUTABLE" setup.py --quiet bdist_wheel unzip -o dist/*.whl -d "$ROOT_DIR/python/ray/pickle5_files" popd popd diff --git a/ci/travis/ci.sh b/ci/travis/ci.sh index 7065ac47f0f94..2f4914e5637df 100755 --- a/ci/travis/ci.sh +++ b/ci/travis/ci.sh @@ -38,21 +38,210 @@ should_run_job() { return "${skip}" } +# Idempotent environment loading reload_env() { # TODO: We should really just use a new login shell instead of doing this manually. # Otherwise we might source a script that isn't idempotent (e.g. one that blindly prepends to PATH). - { local set_x="${-//[^x]/}"; } 2> /dev/null # save set -x to suppress noise - set +x - local to_add="$HOME/miniconda/bin" old_path=":${PATH}:" - if [ "${old_path}" = "${old_path##*:${to_add}:*}" ]; then - PATH="${to_add}:$PATH" + + if [ -n "${GITHUB_PULL_REQUEST-}" ]; then + case "${GITHUB_PULL_REQUEST}" in + [1-9]*) TRAVIS_PULL_REQUEST="${GITHUB_PULL_REQUEST}";; + *) TRAVIS_PULL_REQUEST=false;; + esac + export TRAVIS_PULL_REQUEST fi - test -z "${set_x}" || set -x # restore set -x + + export PYTHON3_BIN_PATH=python + export GOROOT="${HOME}/go" GOPATH="${HOME}/go_dir" if [ "${OSTYPE}" = msys ]; then - PATH="${PATH// :/:}" # HACK: Work around https://github.com/actions/virtual-environments/issues/635#issue-589318442 + export USE_CLANG_CL=1 + fi + + # NOTE: Modifying PATH invalidates Bazel's cache! Do not add to PATH unnecessarily. + PATH="${HOME}/miniconda/bin":"${PATH}" + if [ "${OSTYPE}" = msys ]; then + PATH="${HOME}/miniconda/bin/Scripts":"${PATH}" + fi + + # Deduplicate PATH + PATH="$(set +x && printf "%s\n" "${PATH}" | tr ":" "\n" | awk '!a[$0]++' | tr "\n" ":")" + PATH="${PATH%:}" # Remove trailing colon +} + +need_wheels() { + local error_code=1 + case "${OSTYPE}" in + linux*) if [ "${LINUX_WHEELS-}" = 1 ]; then error_code=0; fi;; + darwin*) if [ "${MAC_WHEELS-}" = 1 ]; then error_code=0; fi;; + msys*) if [ "${WINDOWS_WHEELS-}" = 1 ]; then error_code=0; fi;; + esac + return "${error_code}" +} + +test_python() { + if [ "${OSTYPE}" = msys ]; then + # Windows -- most tests won't work yet; just do the ones we know work + PYTHONPATH=python python -m pytest --durations=5 --timeout=300 python/ray/tests/test_mini.py + fi +} + +test_cpp() { + bazel test --config=ci //cpp:all --build_tests_only --test_output=streamed +} + +test_wheels() { + local result=0 flush_logs=0 + + if need_wheels; then + "${WORKSPACE_DIR}"/ci/travis/test-wheels.sh || { result=$? && flush_logs=1; } fi - PYTHON3_BIN_PATH=python - export PATH PYTHON3_BIN_PATH + + if [ 0 -ne "${flush_logs}" ]; then + cat -- /tmp/ray/session_latest/logs/* || true + sleep 60 # Explicitly sleep 60 seconds for logs to go through + fi + + return "${result}" +} + +build_sphinx_docs() { + ( + cd "${WORKSPACE_DIR}"/doc + if [ "${OSTYPE}" = msys ]; then + echo "WARNING: Documentation not built on Windows due to currently-unresolved issues" + else + sphinx-build -q -W -E -T -b html source _build/html + fi + ) +} + +install_cython_examples() { + "${ROOT_DIR}"/install-cython-examples.sh +} + +install_go() { + eval "$(curl -sL https://raw.githubusercontent.com/travis-ci/gimme/master/gimme | GIMME_GO_VERSION=stable bash)" +} + +install_ray() { + bazel build -k "//:*" # Do a full build first to ensure everything passes + "${ROOT_DIR}"/install-ray.sh +} + +build_wheels() { + case "${OSTYPE}" in + linux*) + # Mount bazel cache dir to the docker container. + # For the linux wheel build, we use a shared cache between all + # wheels, but not between different travis runs, because that + # caused timeouts in the past. See the "cache: false" line below. + local MOUNT_BAZEL_CACHE=(-v "${HOME}/ray-bazel-cache":/root/ray-bazel-cache -e TRAVIS=true -e TRAVIS_PULL_REQUEST="${TRAVIS_PULL_REQUEST:-false}" -e encrypted_1c30b31fe1ee_key="${encrypted_1c30b31fe1ee_key-}" -e encrypted_1c30b31fe1ee_iv="${encrypted_1c30b31fe1ee_iv-}") + + # This command should be kept in sync with ray/python/README-building-wheels.md, + # except the "${MOUNT_BAZEL_CACHE[@]}" part. + suppress_output docker run --rm -w /ray -v "${PWD}":/ray "${MOUNT_BAZEL_CACHE[@]}" -e TRAVIS_COMMIT="${TRAVIS_COMMIT}" rayproject/arrow_linux_x86_64_base:python-3.8.0 /ray/python/build-wheel-manylinux1.sh + ;; + darwin*) + # This command should be kept in sync with ray/python/README-building-wheels.md. + suppress_output "${WORKSPACE_DIR}"/python/build-wheel-macos.sh + ;; + msys*) + if ! python -c "import ray" 2> /dev/null; then + # This installs e.g. the pickle5 dependencies, which are needed for correct wheel testing + install_ray + fi + ( + cd "${WORKSPACE_DIR}"/python + python setup.py --quiet bdist_wheel + ) + ;; + esac +} + +lint_readme() { + ( + cd "${WORKSPACE_DIR}"/python + python setup.py check --restructuredtext --strict --metadata + ) +} + +lint_python() { + # ignore dict vs {} (C408), others are defaults + command -V python + python -m flake8 --inline-quotes '"' --no-avoid-escape --exclude=python/ray/core/generated/,streaming/python/generated,doc/source/conf.py,python/ray/cloudpickle/,python/ray/thirdparty_files --ignore=C408,E121,E123,E126,E226,E24,E704,W503,W504,W605 + "${ROOT_DIR}"/format.sh --all +} + +lint_bazel() { + # Run buildifier without affecting external environment variables + ( + # TODO: Move installing Go & building buildifier to the dependency installation step? + if [ ! -d "${GOROOT}" ]; then + curl -s -L "https://dl.google.com/go/go1.14.2.linux-amd64.tar.gz" | tar -C "${GOROOT%/*}" -xz + fi + mkdir -p -- "${GOPATH}" + export PATH="${GOPATH}/bin":"${GOROOT}/bin":"${PATH}" + + # Build buildifier + go get github.com/bazelbuild/buildtools/buildifier + + # Now run buildifier + "${ROOT_DIR}"/bazel-format.sh + ) +} + +lint_web() { + ( + cd "${WORKSPACE_DIR}"/python/ray/dashboard/client + set +x # suppress set -x since it'll get very noisy here + . "${HOME}/.nvm/nvm.sh" + install_npm_project + nvm use node + node_modules/.bin/eslint --max-warnings 0 $(find src -name "*.ts" -or -name "*.tsx") + node_modules/.bin/prettier --check $(find src -name "*.ts" -or -name "*.tsx") + node_modules/.bin/prettier --check public/index.html + ) +} + +_lint() { + local platform="" + case "${OSTYPE}" in + linux*) platform=linux;; + esac + + if [ -n "${TRAVIS_BRANCH-}" ]; then + "${ROOT_DIR}"/check-git-clang-format-output.sh + else + # TODO(mehrdadn): Implement this on GitHub Actions + echo "WARNING: Not running clang-format due to TRAVIS_BRANCH not being defined." + fi + + # Run Python linting + lint_python + + # Make sure that the README is formatted properly. + lint_readme + + if [ "${platform}" = linux ]; then + # Run Bazel linter Buildifier. + lint_bazel + + # Run TypeScript and HTML linting. + lint_web + fi +} + +lint() { + # Checkout a clean copy of the repo to avoid seeing changes that have been made to the current one + ( + WORKSPACE_DIR="$(TMPDIR="${WORKSPACE_DIR}/.." mktemp -d)" + ROOT_DIR="${WORKSPACE_DIR}"/ci/travis + git worktree add -q "${WORKSPACE_DIR}" + pushd "${WORKSPACE_DIR}" + . "${ROOT_DIR}"/ci.sh _lint + popd # this is required so we can remove the worktree when we're done + git worktree remove --force "${WORKSPACE_DIR}" + ) } preload() { @@ -85,6 +274,13 @@ EOF1 fi } +configure_system() { + git config --global advice.detachedHead false + git config --global core.askpass "" + git config --global credential.helper "" + git config --global credential.modalprompt false +} + # Initializes the environment for the current job. Performs the following tasks: # - Calls 'exit 0' in this job step and all subsequent steps to quickly exit if provided a list of job names and # none of them has been triggered. @@ -98,53 +294,47 @@ EOF1 init() { preload - if [ "${OSTYPE}" = msys ]; then - export USE_CLANG_CL=1 - fi + configure_system - local wheels="${LINUX_WHEELS-}${MAC_WHEELS-}" - if [ -z "${wheels}" ]; then # NOT building wheels - "${ROOT_DIR}"/install-bazel.sh - fi + "${ROOT_DIR}"/install-bazel.sh . "${ROOT_DIR}"/install-dependencies.sh - reload_env # We just modified our environment; reload it so we can continue + + cat <> ~/.bashrc +. "${BASH_SOURCE:-$0}" # reload environment +EOF } build() { - local wheels="${LINUX_WHEELS-}${MAC_WHEELS-}" - if [ -z "${wheels}" ]; then # NOT building wheels - if [ "${LINT-}" != 1 ]; then # NOT linting - bazel build -k "//:*" # Do a full build first to ensure it passes - "${ROOT_DIR}"/install-ray.sh + if ! need_wheels; then + install_ray + if [ "${LINT-}" = 1 ]; then + # Try generating Sphinx documentation. To do this, we need to install Ray first. + build_sphinx_docs fi fi if [ "${RAY_CYTHON_EXAMPLES-}" = 1 ]; then - "${ROOT_DIR}"/install-cython-examples.sh + install_cython_examples fi if [ "${RAY_DEFAULT_BUILD-}" = 1 ]; then - eval "$(curl -sL https://raw.githubusercontent.com/travis-ci/gimme/master/gimme | GIMME_GO_VERSION=stable bash)" + install_go fi - if [ "${LINUX_WHEELS-}" = 1 ]; then - # Mount bazel cache dir to the docker container. - # For the linux wheel build, we use a shared cache between all - # wheels, but not between different travis runs, because that - # caused timeouts in the past. See the "cache: false" line below. - local MOUNT_BAZEL_CACHE=(-v "${HOME}/ray-bazel-cache":/root/ray-bazel-cache -e TRAVIS=true -e TRAVIS_PULL_REQUEST="${TRAVIS_PULL_REQUEST}" -e encrypted_1c30b31fe1ee_key="${encrypted_1c30b31fe1ee_key-}" -e encrypted_1c30b31fe1ee_iv="${encrypted_1c30b31fe1ee_iv-}") - - # This command should be kept in sync with ray/python/README-building-wheels.md, - # except the "${MOUNT_BAZEL_CACHE[@]}" part. - suppress_output docker run --rm -w /ray -v "${PWD}":/ray "${MOUNT_BAZEL_CACHE[@]}" -e TRAVIS_COMMIT="${TRAVIS_COMMIT}" -ti rayproject/arrow_linux_x86_64_base:python-3.8.0 /ray/python/build-wheel-manylinux1.sh + if need_wheels; then + build_wheels fi +} - if [ "${MAC_WHEELS-}" = 1 ]; then - # This command should be kept in sync with ray/python/README-building-wheels.md. - suppress_output "${WORKSPACE_DIR}"/python/build-wheel-macos.sh +_main() { + if [ -n "${GITHUB_WORKFLOW-}" ]; then + # Necessary for GitHub Actions (which uses separate shells for different commands) + # Unnecessary for Travis (which uses one shell for different commands) + reload_env fi + "$@" } -"$@" +_main "$@" { set -vx; eval "${SHELLOPTS_STACK##*|}"; SHELLOPTS_STACK="${SHELLOPTS_STACK%|*}"; } 2> /dev/null # Pop caller's shell options (quietly) diff --git a/ci/travis/install-dependencies.sh b/ci/travis/install-dependencies.sh index 4f094dd195945..09f196795507f 100755 --- a/ci/travis/install-dependencies.sh +++ b/ci/travis/install-dependencies.sh @@ -22,7 +22,9 @@ install_base() { pkg_install_helper build-essential curl unzip tmux gdb libunwind-dev python3-pip python3-setuptools if [ "${LINUX_WHEELS-}" = 1 ]; then pkg_install_helper docker - sudo usermod -a -G docker travis + if [ -n "${TRAVIS-}" ]; then + sudo usermod -a -G docker travis + fi fi if [ -n "${PYTHON-}" ]; then "${ROOT_DIR}/install-strace.sh" || true @@ -32,6 +34,13 @@ install_base() { } install_miniconda() { + local miniconda_dir="${HOME}/miniconda" + if [ "${OSTYPE}" = msys ]; then + miniconda_dir="${miniconda_dir}/bin" # HACK: Compensate for python.exe being in the installation root on Windows + fi + if [ -d "${miniconda_dir}" ]; then + return 0 # already installed + fi local miniconda_version="3-4.5.4" miniconda_platform="" exe_suffix=".sh" case "${OSTYPE}" in linux*) miniconda_platform=Linux;; @@ -39,25 +48,18 @@ install_miniconda() { msys*) miniconda_platform=Windows; exe_suffix=".exe";; esac local miniconda_url="https://repo.continuum.io/miniconda/Miniconda${miniconda_version}-${miniconda_platform}-${HOSTTYPE}${exe_suffix}" - local miniconda_target="./${miniconda_url##*/}" - local miniconda_dir="$HOME/miniconda" + local miniconda_target="${HOME}/${miniconda_url##*/}" curl -s -L -o "${miniconda_target}" "${miniconda_url}" chmod +x "${miniconda_target}" case "${OSTYPE}" in msys*) - miniconda_dir="${miniconda_dir}/bin" # HACK: Compensate for python.exe being in the installation root on Windows MSYS2_ARG_CONV_EXCL="*" "${miniconda_target}" /S /D="$(cygpath -w -- "${miniconda_dir}")" ;; *) "${miniconda_target}" -b -p "${miniconda_dir}" | grep --line-buffered -v "^\(installing: \|installation finished\.\)" ;; esac - { local set_x="${-//[^x]/}"; } 2> /dev/null # save set -x to suppress noise - set +x - local source_line='PYTHON3_BIN_PATH=python; PATH="$HOME/miniconda/bin:$PATH"; export PYTHON3_BIN_PATH PATH;' - test -f ~/.bashrc && grep -x -q -F "${source_line}" -- ~/.bashrc || echo "${source_line}" >> ~/.bashrc - test -z "${set_x}" || set -x # restore set -x - eval "${source_line}" + reload_env python -m pip install --upgrade --quiet pip } @@ -121,6 +123,15 @@ run_npm() { npm ci } +install_npm_project() { + if [ "${OSTYPE}" = msys ]; then + # Not Windows-compatible: https://github.com/npm/cli/issues/558#issuecomment-584673763 + { echo "WARNING: Skipping running NPM due to module incompatibilities with Windows"; } 2> /dev/null + else + run_npm + fi +} + install_dependencies() { install_base @@ -150,13 +161,15 @@ install_dependencies() { pip_packages+=(blist) # https://github.com/DanielStutzbach/blist/issues/81#issue-391460716 fi CC=gcc pip install "${pip_packages[@]}" - elif [ "${LINT-}" = 1 ]; then + fi + + if [ "${LINT-}" = 1 ]; then install_miniconda - pip install flake8==3.7.7 flake8-comprehensions flake8-quotes==2.0.0 # Python linters - pushd "${WORKSPACE_DIR}/python/ray/dashboard/client" # TypeScript & HTML linters - install_node - run_npm - popd + pip install flake8==3.7.7 flake8-comprehensions flake8-quotes==2.0.0 yapf==0.23.0 # Python linters + # readthedocs has an antiquated build env. + # This is a best effort to reproduce it locally to avoid doc build failures and hidden errors. + pip install -r "${WORKSPACE_DIR}"/doc/requirements-rtd.txt + pip install -r "${WORKSPACE_DIR}"/doc/requirements-doc.txt fi # Install modules needed in all jobs. @@ -173,7 +186,7 @@ install_dependencies() { pip install msgpack>=0.6.2 fi - if [ -n "${PYTHON-}" ] || [ "${MAC_WHEELS-}" = 1 ]; then + if [ -n "${PYTHON-}" ] || [ -n "${LINT-}" ] || [ "${MAC_WHEELS-}" = 1 ]; then install_node fi diff --git a/ci/travis/install-ray.sh b/ci/travis/install-ray.sh index c9737914dcfaa..a68354d545cb2 100755 --- a/ci/travis/install-ray.sh +++ b/ci/travis/install-ray.sh @@ -24,11 +24,7 @@ install_ray() { ( cd "${WORKSPACE_DIR}"/python build_dashboard_front_end - if [ "${OSTYPE}" = msys ]; then - "${WORKSPACE_DIR}"/ci/keep_alive pip install -v -e . || echo "WARNING: Ignoring Ray package build failure on Windows for now" 1>&2 - else - "${WORKSPACE_DIR}"/ci/keep_alive pip install -v -e . - fi + "${WORKSPACE_DIR}"/ci/keep_alive pip install -e . ) } diff --git a/ci/travis/test-wheels.sh b/ci/travis/test-wheels.sh index cd3276fd22d10..8a78b60742500 100755 --- a/ci/travis/test-wheels.sh +++ b/ci/travis/test-wheels.sh @@ -8,18 +8,13 @@ set -x ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) -platform="unknown" -unamestr="$(uname)" -if [[ "$unamestr" == "Linux" ]]; then - echo "Platform is linux." - platform="linux" -elif [[ "$unamestr" == "Darwin" ]]; then - echo "Platform is macosx." - platform="macosx" -else - echo "Unrecognized platform." - exit 1 -fi +platform="" +case "${OSTYPE}" in + linux*) platform="linux";; + darwin*) platform="macosx";; + msys*) platform="windows";; + *) echo "Unrecognized platform."; exit 1;; +esac TEST_DIR="$TRAVIS_BUILD_DIR/python/ray/tests" TEST_SCRIPTS=("$TEST_DIR/test_microbenchmarks.py" "$TEST_DIR/test_basic.py") @@ -49,7 +44,7 @@ if [[ "$platform" == "linux" ]]; then "3.7.6" "3.8.2") wget --quiet "https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh" -O miniconda3.sh - bash miniconda3.sh -b -p "$HOME/miniconda3" + "${ROOT_DIR}"/../suppress_output bash miniconda3.sh -b -p "$HOME/miniconda3" export PATH="$HOME/miniconda3/bin:$PATH" for ((i=0; i<${#PY_MMS[@]}; ++i)); do @@ -129,6 +124,8 @@ elif [[ "$platform" == "macosx" ]]; then fi done +elif [ "${platform}" = windows ]; then + echo "WARNING: Wheel testing not yet implemented for Windows." else echo "Unrecognized environment." exit 3 diff --git a/doc/requirements-doc.txt b/doc/requirements-doc.txt index dcefe0e61c865..b42a56a521142 100644 --- a/doc/requirements-doc.txt +++ b/doc/requirements-doc.txt @@ -1,4 +1,3 @@ -blist click colorama filelock diff --git a/python/ray/dashboard/dashboard.py b/python/ray/dashboard/dashboard.py index 1a226ecfdb54c..6b5e261915ede 100644 --- a/python/ray/dashboard/dashboard.py +++ b/python/ray/dashboard/dashboard.py @@ -13,6 +13,7 @@ import logging import os import re +import socket import threading import time import traceback @@ -1224,7 +1225,7 @@ def clean_trials(self, trial_details): args.redis_address, password=args.redis_password) traceback_str = ray.utils.format_error_message(traceback.format_exc()) message = ("The dashboard on node {} failed with the following " - "error:\n{}".format(os.uname()[1], traceback_str)) + "error:\n{}".format(socket.gethostname(), traceback_str)) ray.utils.push_error_to_driver_through_redis( redis_client, ray_constants.DASHBOARD_DIED_ERROR, message) if isinstance(e, OSError) and e.errno == errno.ENOENT: diff --git a/python/ray/log_monitor.py b/python/ray/log_monitor.py index 764eb668f8958..9f75f5bdb0e06 100644 --- a/python/ray/log_monitor.py +++ b/python/ray/log_monitor.py @@ -5,6 +5,7 @@ import logging import os import shutil +import socket import time import traceback @@ -301,7 +302,7 @@ def run(self): args.redis_address, password=args.redis_password) traceback_str = ray.utils.format_error_message(traceback.format_exc()) message = ("The log monitor on node {} failed with the following " - "error:\n{}".format(os.uname()[1], traceback_str)) + "error:\n{}".format(socket.gethostname(), traceback_str)) ray.utils.push_error_to_driver_through_redis( redis_client, ray_constants.LOG_MONITOR_DIED_ERROR, message) raise e diff --git a/python/ray/reporter.py b/python/ray/reporter.py index d01cb7a2a82df..eea3d79fbfce7 100644 --- a/python/ray/reporter.py +++ b/python/ray/reporter.py @@ -252,7 +252,7 @@ def run(self): args.redis_address, password=args.redis_password) traceback_str = ray.utils.format_error_message(traceback.format_exc()) message = ("The reporter on node {} failed with the following " - "error:\n{}".format(os.uname()[1], traceback_str)) + "error:\n{}".format(socket.gethostname(), traceback_str)) ray.utils.push_error_to_driver_through_redis( redis_client, ray_constants.REPORTER_DIED_ERROR, message) raise e diff --git a/python/setup.py b/python/setup.py index 8e5e49538ceca..087916438e149 100644 --- a/python/setup.py +++ b/python/setup.py @@ -159,12 +159,15 @@ def move_file(self, filename): source = filename destination = os.path.join(self.build_lib, filename) # Create the target directory if it doesn't already exist. - parent_directory = os.path.dirname(destination) - if not os.path.exists(parent_directory): - os.makedirs(parent_directory) + os.makedirs(os.path.dirname(destination), exist_ok=True) if not os.path.exists(destination): print("Copying {} to {}.".format(source, destination)) - shutil.copy(source, destination, follow_symlinks=True) + if sys.platform == "win32": + # Does not preserve file mode (needed to avoid read-only bit) + shutil.copyfile(source, destination, follow_symlinks=True) + else: + # Preserves file mode (needed to copy executable bit) + shutil.copy(source, destination, follow_symlinks=True) class BinaryDistribution(Distribution): diff --git a/src/ray/test/run_core_worker_tests.sh b/src/ray/test/run_core_worker_tests.sh old mode 100644 new mode 100755 diff --git a/src/ray/test/run_object_manager_tests.sh b/src/ray/test/run_object_manager_tests.sh old mode 100644 new mode 100755 diff --git a/src/ray/test/run_object_manager_valgrind.sh b/src/ray/test/run_object_manager_valgrind.sh old mode 100644 new mode 100755 diff --git a/streaming/src/test/run_streaming_queue_test.sh b/streaming/src/test/run_streaming_queue_test.sh old mode 100644 new mode 100755