Merge remote-tracking branch 'upstream/master'

.github/PULL_REQUEST_TEMPLATE.md

@@ -13,3 +13,7 @@
 - [ ] I've run `scripts/format.sh` to lint the changes in this PR.
 - [ ] I've included any doc changes needed for https://ray.readthedocs.io/en/latest/.
 - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failure rates at https://ray-travis-tracker.herokuapp.com/.
+- Testing Strategy
+   - [ ] Unit tests
+   - [ ] Release tests
+   - [ ] This PR is not tested (please justify below)

.travis.yml

@@ -49,11 +49,11 @@ matrix:
 
     - os: linux
       env:
-        - TESTSUITE=gcs_service
+        - TESTSUITE=gcs_service_disabled
         - JDK='Oracle JDK 8'
-        - RAY_GCS_SERVICE_ENABLED=true
         - PYTHON=3.6 PYTHONWARNINGS=ignore
         - RAY_INSTALL_JAVA=1
+        - RAY_GCS_SERVICE_ENABLED=false
       install:
         - eval `python $TRAVIS_BUILD_DIR/ci/travis/determine_tests_to_run.py`
         - ./ci/travis/install-bazel.sh
@@ -69,11 +69,11 @@ matrix:
 
     - os: linux
       env:
-        - TESTSUITE=gcs_service_python_testcase
+        - TESTSUITE=gcs_service_disabled_python_testcase
         - JDK='Oracle JDK 8'
-        - RAY_GCS_SERVICE_ENABLED=true
         - PYTHON=3.6 PYTHONWARNINGS=ignore
         - RAY_INSTALL_JAVA=1
+        - RAY_GCS_SERVICE_ENABLED=false
       install:
         - eval `python $TRAVIS_BUILD_DIR/ci/travis/determine_tests_to_run.py`
         - ./ci/travis/install-bazel.sh
@@ -98,9 +98,9 @@ matrix:
         - ./ci/travis/install-dependencies.sh
         - export PATH="$HOME/miniconda/bin:$PATH"
         - cd doc
-        - pip install -q -r requirements-doc.txt
-        - pip install -q yapf==0.23.0
-        - sphinx-build -W -b html -d _build/doctrees source _build/html
+        - pip install -r requirements-doc.txt
+        - pip install yapf==0.23.0
+        - sphinx-build -q -W -b html -d _build/doctrees source _build/html
         - cd ..
         # Run Python linting, ignore dict vs {} (C408), others are defaults
         - flake8 --inline-quotes '"' --no-avoid-escape --exclude=python/ray/core/generated/,streaming/python/generated,doc/source/conf.py,python/ray/cloudpickle/,python/ray/thirdparty_files --ignore=C408,E121,E123,E126,E226,E24,E704,W503,W504,W605
@@ -110,8 +110,8 @@ matrix:
         - python setup.py check --restructuredtext --strict --metadata
         - cd ..
         # Run Bazel linter Buildifier.
-        - wget https://dl.google.com/go/go1.11.linux-amd64.tar.gz
-        - tar -xf go1.11.linux-amd64.tar.gz
+        - wget -q https://dl.google.com/go/go1.12.linux-amd64.tar.gz
+        - tar -xf go1.12.linux-amd64.tar.gz
         - mkdir $HOME/go_dir
         - export GOROOT=`pwd`/go
         - export GOPATH="$HOME/go_dir"
@@ -149,7 +149,8 @@ matrix:
       script:
         - if [ $RAY_CI_LINUX_WHEELS_AFFECTED != "1" ]; then exit; fi
 
-        - ./ci/travis/test-wheels.sh
+        # Explicitly sleep 60 seconds for logs to go through
+        - ./ci/travis/test-wheels.sh || cat /tmp/ray/session_latest/logs/* || sleep 60 || false
       cache: false
 
     # Build MacOS wheels.
@@ -166,7 +167,8 @@ matrix:
       script:
         - if [ $RAY_CI_MACOS_WHEELS_AFFECTED != "1" ]; then exit; fi
 
-        - ./ci/travis/test-wheels.sh
+        # Explicitly sleep 60 seconds for logs to go through
+        - ./ci/travis/test-wheels.sh || cat /tmp/ray/session_latest/logs/* || sleep 60 || false
 
     # RLlib: Learning tests (from rllib/tuned_examples/regression_tests/*.yaml).
     - os: linux
@@ -186,7 +188,7 @@ matrix:
         - ./ci/suppress_output ./ci/travis/install-ray.sh
       script:
         - if [ $RAY_CI_RLLIB_AFFECTED != "1" ]; then exit; fi
-        - travis_wait 60 bazel test --build_tests_only --test_tag_filters=learning_tests --spawn_strategy=local --flaky_test_attempts=3 --nocache_test_results --test_verbose_timeout_warnings --progress_report_interval=100 --show_progress_rate_limit=100 --show_timestamps --test_output=errors rllib/...
+        - travis_wait 60 bazel test --build_tests_only --test_tag_filters=learning_tests --spawn_strategy=local --flaky_test_attempts=3 --nocache_test_results --test_verbose_timeout_warnings --progress_report_interval=100 --show_progress_rate_limit=100 --show_timestamps --test_output=streamed rllib/...
 
     # RLlib: Learning tests with tf=1.x (from rllib/tuned_examples/regression_tests/*.yaml).
     # Requested by Edi (MS): Test all learning capabilities with tf1.x
@@ -207,7 +209,7 @@ matrix:
         - ./ci/suppress_output ./ci/travis/install-ray.sh
       script:
         - if [ $RAY_CI_RLLIB_FULL_AFFECTED != "1" ]; then exit; fi
-        - travis_wait 60 bazel test --build_tests_only --test_tag_filters=learning_tests --spawn_strategy=local --flaky_test_attempts=3 --nocache_test_results --test_verbose_timeout_warnings --progress_report_interval=100 --show_progress_rate_limit=100 --show_timestamps --test_output=errors rllib/...
+        - travis_wait 60 bazel test --build_tests_only --test_tag_filters=learning_tests --spawn_strategy=local --flaky_test_attempts=3 --nocache_test_results --test_verbose_timeout_warnings --progress_report_interval=100 --show_progress_rate_limit=100 --show_timestamps --test_output=streamed rllib/...
 
     # RLlib: Quick Agent train.py runs (compilation & running, no(!) learning).
     # Agent single tests (compilation, loss-funcs, etc..).
@@ -253,8 +255,8 @@ matrix:
         - if [ $RAY_CI_RLLIB_FULL_AFFECTED != "1" ]; then exit; fi
         - ./ci/keep_alive bazel test --build_tests_only --test_tag_filters=examples_A,examples_B --spawn_strategy=local --flaky_test_attempts=3 --nocache_test_results --test_verbose_timeout_warnings --progress_report_interval=100 --show_progress_rate_limit=100 --show_timestamps --test_output=errors rllib/...
         - ./ci/keep_alive bazel test --build_tests_only --test_tag_filters=examples_C --spawn_strategy=local --flaky_test_attempts=3 --nocache_test_results --test_verbose_timeout_warnings --progress_report_interval=100 --show_progress_rate_limit=100 --show_timestamps --test_output=errors rllib/...
-        - ./ci/keep_alive bazel test --build_tests_only --test_tag_filters=examples_E,examples_M,examples_P --spawn_strategy=local --flaky_test_attempts=3 --nocache_test_results --test_verbose_timeout_warnings --progress_report_interval=100 --show_progress_rate_limit=100 --show_timestamps --test_output=errors rllib/...
-        - ./ci/keep_alive bazel test --build_tests_only --test_tag_filters=examples_R,examples_T --spawn_strategy=local --flaky_test_attempts=3 --nocache_test_results --test_verbose_timeout_warnings --progress_report_interval=100 --show_progress_rate_limit=100 --show_timestamps --test_output=errors rllib/...
+        - ./ci/keep_alive bazel test --build_tests_only --test_tag_filters=examples_E,examples_L,examples_M,examples_P --spawn_strategy=local --flaky_test_attempts=3 --nocache_test_results --test_verbose_timeout_warnings --progress_report_interval=100 --show_progress_rate_limit=100 --show_timestamps --test_output=errors rllib/...
+        - ./ci/keep_alive bazel test --build_tests_only --test_tag_filters=examples_U,examples_R,examples_S,examples_T --spawn_strategy=local --flaky_test_attempts=3 --nocache_test_results --test_verbose_timeout_warnings --progress_report_interval=100 --show_progress_rate_limit=100 --show_timestamps --test_output=errors rllib/...
 
     # RLlib: tests_dir: Everything in rllib/tests/ directory (A-I).
     - os: linux
@@ -274,9 +276,7 @@ matrix:
         - ./ci/suppress_output ./ci/travis/install-ray.sh
       script:
         - if [ $RAY_CI_RLLIB_FULL_AFFECTED != "1" ]; then exit; fi
-        - ./ci/keep_alive bazel test --build_tests_only --test_tag_filters=tests_dir_A,tests_dir_C,tests_dir_D --spawn_strategy=local --flaky_test_attempts=3 --nocache_test_results --test_verbose_timeout_warnings --progress_report_interval=100 --show_progress_rate_limit=100 --show_timestamps --test_output=errors rllib/...
-        - ./ci/keep_alive bazel test --build_tests_only --test_tag_filters=tests_dir_E --spawn_strategy=local --flaky_test_attempts=3 --nocache_test_results --test_verbose_timeout_warnings --progress_report_interval=100 --show_progress_rate_limit=100 --show_timestamps --test_output=errors rllib/...
-        - ./ci/keep_alive bazel test --build_tests_only --test_tag_filters=tests_dir_F,tests_dir_I --spawn_strategy=local --flaky_test_attempts=3 --nocache_test_results --test_verbose_timeout_warnings --progress_report_interval=100 --show_progress_rate_limit=100 --show_timestamps --test_output=errors rllib/...
+        - ./ci/keep_alive bazel test --build_tests_only --test_tag_filters=tests_dir_A,tests_dir_B,tests_dir_C,tests_dir_D,tests_dir_E,tests_dir_F,tests_dir_G,tests_dir_H,tests_dir_I --spawn_strategy=local --flaky_test_attempts=3 --nocache_test_results --test_verbose_timeout_warnings --progress_report_interval=100 --show_progress_rate_limit=100 --show_timestamps --test_output=errors rllib/...
 
     # RLlib: tests_dir: Everything in rllib/tests/ directory (J-Z).
     - os: linux
@@ -296,9 +296,7 @@ matrix:
         - ./ci/suppress_output ./ci/travis/install-ray.sh
       script:
         - if [ $RAY_CI_RLLIB_FULL_AFFECTED != "1" ]; then exit; fi
-        - ./ci/keep_alive bazel test --build_tests_only --test_tag_filters=tests_dir_L,tests_dir_M --spawn_strategy=local --flaky_test_attempts=3 --nocache_test_results --test_verbose_timeout_warnings --progress_report_interval=100 --show_progress_rate_limit=100 --show_timestamps --test_output=errors rllib/...
-        - ./ci/keep_alive bazel test --build_tests_only --test_tag_filters=tests_dir_N,tests_dir_O,test_dir_P --spawn_strategy=local --flaky_test_attempts=3 --nocache_test_results --test_verbose_timeout_warnings --progress_report_interval=100 --show_progress_rate_limit=100 --show_timestamps --test_output=errors rllib/...
-        - ./ci/keep_alive bazel test --build_tests_only --test_tag_filters=tests_dir_R,tests_dir_S --spawn_strategy=local --flaky_test_attempts=3 --nocache_test_results --test_verbose_timeout_warnings --progress_report_interval=100 --show_progress_rate_limit=100 --show_timestamps --test_output=errors rllib/...
+        - ./ci/keep_alive bazel test --build_tests_only --test_tag_filters=tests_dir_J,tests_dir_K,tests_dir_L,tests_dir_M,tests_dir_N,tests_dir_O,tests_dir_P,tests_dir_Q,tests_dir_R,tests_dir_S,tests_dir_T,tests_dir_U,tests_dir_V,tests_dir_W,tests_dir_X,tests_dir_Y,tests_dir_Z --spawn_strategy=local --flaky_test_attempts=3 --nocache_test_results --test_verbose_timeout_warnings --progress_report_interval=100 --show_progress_rate_limit=100 --show_timestamps --test_output=errors rllib/...
 
 install:
   - eval `python $TRAVIS_BUILD_DIR/ci/travis/determine_tests_to_run.py`
@@ -380,7 +378,3 @@ deploy:
       repo: ray-project/ray
       all_branches: true
       condition: $LINUX_WHEELS = 1 || $MAC_WHEELS = 1
-
-cache:
-  directories:
-    - $HOME/ray-bazel-cache

BUILD.bazel

@@ -384,7 +384,7 @@ cc_library(
         ],
         exclude = [
             "src/ray/raylet/mock_gcs_client.cc",
-            "src/ray/raylet/monitor_main.cc",
+            "src/ray/raylet/monitor*.cc",
             "src/ray/raylet/*_test.cc",
             "src/ray/raylet/main.cc",
         ],
@@ -843,6 +843,7 @@ cc_library(
         "@com_github_google_glog//:glog",
         "@com_google_absl//absl/synchronization",
         "@com_google_absl//absl/time",
+        "@com_google_googletest//:gtest_main",
         "@plasma//:plasma_client",
     ],
 )
@@ -890,6 +891,11 @@ cc_library(
     includes = ["src/ray/thirdparty"],
 )
 
+alias(
+    name = "hiredis",
+    actual = "@com_github_antirez_redis//:hiredis",
+)
+
 cc_library(
     name = "gcs",
     srcs = glob(
@@ -907,13 +913,13 @@ cc_library(
     copts = COPTS,
     deps = [
         ":gcs_cc_proto",
+        ":hiredis",
         ":node_manager_fbs",
         ":node_manager_rpc",
         ":ray_common",
         ":ray_util",
         ":stats_lib",
         "@boost//:asio",
-        "@redis//:hiredis",
     ],
 )
 
@@ -1180,6 +1186,7 @@ filegroup(
         "python/ray/*.py",
         "python/ray/autoscaler/*.py",
         "python/ray/autoscaler/aws/example-full.yaml",
+        "python/ray/autoscaler/azure/example-full.yaml",
         "python/ray/autoscaler/gcp/example-full.yaml",
         "python/ray/autoscaler/local/example-full.yaml",
         "python/ray/cloudpickle/*.py",
@@ -1206,10 +1213,11 @@ genrule(
     ] + select({
         # Windows is not supported yet, so make stubs for it
         "@bazel_tools//src/conditions:windows": [
-            "@win-redis-bin//file",
+            "@com_github_tporadowski_redis_bin//file",
         ],
         "//conditions:default": [
-            "@redis-src//file",
+            "@com_github_antirez_redis//:file",  # This is necessary so we can access the root of the directory
+            "@com_github_antirez_redis//:files",  # This is necessary to ensure entire directory tree is accessible
         ],
     }),
     outs = [
@@ -1219,18 +1227,21 @@ genrule(
     cmd = select({
         # Windows is not supported yet, so make stubs for it
         "@bazel_tools//src/conditions:windows": """
-            unzip -q -o -- $(location @win-redis-bin//file) redis-server.exe redis-cli.exe &&
+            unzip -q -o -- $(location @com_github_tporadowski_redis_bin//file) redis-server.exe redis-cli.exe &&
             mv -f -- redis-server.exe $(location redis-server) &&
             mv -f -- redis-cli.exe    $(location redis-cli)
         """,
         "//conditions:default": """
             set -x &&
-            tar -xzf $(location @redis-src//file) --strip-components=1 -C . &&
-            make &&
-            mv ./src/redis-server $(location redis-server) &&
+            tmpdir="redis.tmp" &&
+            path=$(location @com_github_antirez_redis//:file) &&
+            cp -p -L -R -- "$${path%/*}" "$${tmpdir}" &&
+            make -s -C "$${tmpdir}" -j"$$(getconf _NPROCESSORS_ONLN || echo 1)" &&
+            mv "$${tmpdir}"/src/redis-server $(location redis-server) &&
             chmod +x $(location redis-server) &&
-            mv ./src/redis-cli $(location redis-cli) &&
-            chmod +x $(location redis-cli)
+            mv "$${tmpdir}"/src/redis-cli $(location redis-cli) &&
+            chmod +x $(location redis-cli) &&
+            rm -r -f -- "$${tmpdir}"
         """,
     }),
     visibility = ["//java:__subpackages__"],

README.rst

@@ -182,7 +182,7 @@ RLlib Quick Start
 More Information
 ----------------
 
-- `Documentation`_, in particular `Building Ray and Contributing to Ray`_
+- `Documentation`_
 - `Tutorial`_
 - `Blog`_
 - `Ray paper`_
@@ -191,7 +191,6 @@ More Information
 - `Tune paper`_
 
 .. _`Documentation`: http://ray.readthedocs.io/en/latest/index.html
-.. _`Building Ray and Contributing to Ray`: https://ray.readthedocs.io/en/latest/development.html
 .. _`Tutorial`: https://github.com/ray-project/tutorial
 .. _`Blog`: https://ray-project.github.io/
 .. _`Ray paper`: https://arxiv.org/abs/1712.05889

bazel/BUILD.plasma

@@ -5,6 +5,8 @@ PROPAGATED_WINDOWS_DEFINES = ["ARROW_STATIC"]
 
 COPTS = [] + select({
     "@bazel_tools//src/conditions:windows": [
+        "-D" + "WIN32_REPLACE_FD_APIS",
+        "/FI" + "win32fd.h",
     ] + ["-D" + define for define in PROPAGATED_WINDOWS_DEFINES],
     "//conditions:default": [
         "-DARROW_USE_GLOG",
@@ -23,9 +25,13 @@ cc_library(
     name = "arrow",
     srcs = [
         "cpp/src/arrow/buffer.cc",
+        "cpp/src/arrow/device.cc",
         "cpp/src/arrow/io/interfaces.cc",
+        "cpp/src/arrow/io/memory.cc",
         "cpp/src/arrow/memory_pool.cc",
+        "cpp/src/arrow/result.cc",
         "cpp/src/arrow/status.cc",
+        "cpp/src/arrow/util/future.cc",
         "cpp/src/arrow/util/io_util.cc",
         "cpp/src/arrow/util/logging.cc",
         "cpp/src/arrow/util/memory.cc",
@@ -36,9 +42,12 @@ cc_library(
     ],
     hdrs = [
         "cpp/src/arrow/buffer.h",
+        "cpp/src/arrow/device.h",
         "cpp/src/arrow/io/concurrency.h",
         "cpp/src/arrow/io/interfaces.h",
+        "cpp/src/arrow/io/memory.h",
         "cpp/src/arrow/io/mman.h",
+        "cpp/src/arrow/io/type_fwd.h",
         "cpp/src/arrow/io/util_internal.h",
         "cpp/src/arrow/memory_pool.h",
         "cpp/src/arrow/result.h",
@@ -48,12 +57,14 @@ cc_library(
         "cpp/src/arrow/util/checked_cast.h",
         "cpp/src/arrow/util/compare.h",
         "cpp/src/arrow/util/functional.h",
+        "cpp/src/arrow/util/future.h",
         "cpp/src/arrow/util/io_util.h",
         "cpp/src/arrow/util/iterator.h",
         "cpp/src/arrow/util/logging.h",
         "cpp/src/arrow/util/macros.h",
+        "cpp/src/arrow/util/make_unique.h",
         "cpp/src/arrow/util/memory.h",
-        "cpp/src/arrow/util/stl.h",
+        "cpp/src/arrow/util/optional.h",
         "cpp/src/arrow/util/string.h",
         "cpp/src/arrow/util/string_builder.h",
         "cpp/src/arrow/util/string_view.h",
@@ -64,6 +75,8 @@ cc_library(
         "cpp/src/arrow/util/variant.h",
         "cpp/src/arrow/util/visibility.h",
         "cpp/src/arrow/util/windows_compatibility.h",
+        "cpp/src/arrow/util/windows_fixup.h",
+        "cpp/src/arrow/vendored/optional.hpp",
         "cpp/src/arrow/vendored/string_view.hpp",
         "cpp/src/arrow/vendored/utf8cpp/checked.h",
         "cpp/src/arrow/vendored/utf8cpp/core.h",
@@ -77,6 +90,7 @@ cc_library(
     linkopts = LINKOPTS,
     strip_include_prefix = "cpp/src",
     deps = [
+        "@//:platform_shims",
         "@boost//:filesystem",
         "@com_github_google_glog//:glog",
     ],
@@ -118,8 +132,8 @@ cc_library(
         ":arrow",
         ":common_fbs",
         ":plasma_fbs",
+        "@//:platform_shims",
         "@com_github_google_glog//:glog",
-        "@com_github_ray_project_ray//:platform_shims",
     ],
 )
 
@@ -187,7 +201,7 @@ cc_library(
     strip_include_prefix = "cpp/src",
     visibility = ["//visibility:public"],
     deps = [
-        "@com_github_ray_project_ray//:platform_shims",
+        "@//:platform_shims",
     ],
 )
 
@@ -216,8 +230,8 @@ cc_library(
     deps = [
         ":ae",
         ":plasma_client",
+        "@//:platform_shims",
         "@com_github_google_glog//:glog",
-        "@com_github_ray_project_ray//:platform_shims",
     ],
 )
 
@@ -226,8 +240,12 @@ cc_binary(
     srcs = [
         "cpp/src/plasma/store.cc",
     ],
+    copts = COPTS,
     visibility = ["//visibility:public"],
-    deps = [":plasma_lib"],
+    deps = [
+        ":plasma_lib",
+        "@//:platform_shims",
+    ],
 )
 
 FLATC_ARGS = [