Merge branch 'master' into redis_impl_kv_store_step_one

.github/PULL_REQUEST_TEMPLATE.md

@@ -13,3 +13,7 @@
 - [ ] I've run `scripts/format.sh` to lint the changes in this PR.
 - [ ] I've included any doc changes needed for https://ray.readthedocs.io/en/latest/.
 - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failure rates at https://ray-travis-tracker.herokuapp.com/.
+- Testing Strategy
+   - [ ] Unit tests
+   - [ ] Release tests
+   - [ ] This PR is not tested (please justify below)

.travis.yml

@@ -110,8 +110,8 @@ matrix:
         - python setup.py check --restructuredtext --strict --metadata
         - cd ..
         # Run Bazel linter Buildifier.
-        - wget -q https://dl.google.com/go/go1.11.linux-amd64.tar.gz
-        - tar -xf go1.11.linux-amd64.tar.gz
+        - wget -q https://dl.google.com/go/go1.12.linux-amd64.tar.gz
+        - tar -xf go1.12.linux-amd64.tar.gz
         - mkdir $HOME/go_dir
         - export GOROOT=`pwd`/go
         - export GOPATH="$HOME/go_dir"
@@ -186,7 +186,7 @@ matrix:
         - ./ci/suppress_output ./ci/travis/install-ray.sh
       script:
         - if [ $RAY_CI_RLLIB_AFFECTED != "1" ]; then exit; fi
-        - travis_wait 60 bazel test --build_tests_only --test_tag_filters=learning_tests --spawn_strategy=local --flaky_test_attempts=3 --nocache_test_results --test_verbose_timeout_warnings --progress_report_interval=100 --show_progress_rate_limit=100 --show_timestamps --test_output=errors rllib/...
+        - travis_wait 60 bazel test --build_tests_only --test_tag_filters=learning_tests --spawn_strategy=local --flaky_test_attempts=3 --nocache_test_results --test_verbose_timeout_warnings --progress_report_interval=100 --show_progress_rate_limit=100 --show_timestamps --test_output=streamed rllib/...
 
     # RLlib: Learning tests with tf=1.x (from rllib/tuned_examples/regression_tests/*.yaml).
     # Requested by Edi (MS): Test all learning capabilities with tf1.x
@@ -207,7 +207,7 @@ matrix:
         - ./ci/suppress_output ./ci/travis/install-ray.sh
       script:
         - if [ $RAY_CI_RLLIB_FULL_AFFECTED != "1" ]; then exit; fi
-        - travis_wait 60 bazel test --build_tests_only --test_tag_filters=learning_tests --spawn_strategy=local --flaky_test_attempts=3 --nocache_test_results --test_verbose_timeout_warnings --progress_report_interval=100 --show_progress_rate_limit=100 --show_timestamps --test_output=errors rllib/...
+        - travis_wait 60 bazel test --build_tests_only --test_tag_filters=learning_tests --spawn_strategy=local --flaky_test_attempts=3 --nocache_test_results --test_verbose_timeout_warnings --progress_report_interval=100 --show_progress_rate_limit=100 --show_timestamps --test_output=streamed rllib/...
 
     # RLlib: Quick Agent train.py runs (compilation & running, no(!) learning).
     # Agent single tests (compilation, loss-funcs, etc..).
@@ -253,8 +253,8 @@ matrix:
         - if [ $RAY_CI_RLLIB_FULL_AFFECTED != "1" ]; then exit; fi
         - ./ci/keep_alive bazel test --build_tests_only --test_tag_filters=examples_A,examples_B --spawn_strategy=local --flaky_test_attempts=3 --nocache_test_results --test_verbose_timeout_warnings --progress_report_interval=100 --show_progress_rate_limit=100 --show_timestamps --test_output=errors rllib/...
         - ./ci/keep_alive bazel test --build_tests_only --test_tag_filters=examples_C --spawn_strategy=local --flaky_test_attempts=3 --nocache_test_results --test_verbose_timeout_warnings --progress_report_interval=100 --show_progress_rate_limit=100 --show_timestamps --test_output=errors rllib/...
-        - ./ci/keep_alive bazel test --build_tests_only --test_tag_filters=examples_E,examples_M,examples_P --spawn_strategy=local --flaky_test_attempts=3 --nocache_test_results --test_verbose_timeout_warnings --progress_report_interval=100 --show_progress_rate_limit=100 --show_timestamps --test_output=errors rllib/...
-        - ./ci/keep_alive bazel test --build_tests_only --test_tag_filters=examples_R,examples_T --spawn_strategy=local --flaky_test_attempts=3 --nocache_test_results --test_verbose_timeout_warnings --progress_report_interval=100 --show_progress_rate_limit=100 --show_timestamps --test_output=errors rllib/...
+        - ./ci/keep_alive bazel test --build_tests_only --test_tag_filters=examples_E,examples_L,examples_M,examples_P --spawn_strategy=local --flaky_test_attempts=3 --nocache_test_results --test_verbose_timeout_warnings --progress_report_interval=100 --show_progress_rate_limit=100 --show_timestamps --test_output=errors rllib/...
+        - ./ci/keep_alive bazel test --build_tests_only --test_tag_filters=examples_R,examples_S,examples_T --spawn_strategy=local --flaky_test_attempts=3 --nocache_test_results --test_verbose_timeout_warnings --progress_report_interval=100 --show_progress_rate_limit=100 --show_timestamps --test_output=errors rllib/...
 
     # RLlib: tests_dir: Everything in rllib/tests/ directory (A-I).
     - os: linux
@@ -274,9 +274,7 @@ matrix:
         - ./ci/suppress_output ./ci/travis/install-ray.sh
       script:
         - if [ $RAY_CI_RLLIB_FULL_AFFECTED != "1" ]; then exit; fi
-        - ./ci/keep_alive bazel test --build_tests_only --test_tag_filters=tests_dir_A,tests_dir_C,tests_dir_D --spawn_strategy=local --flaky_test_attempts=3 --nocache_test_results --test_verbose_timeout_warnings --progress_report_interval=100 --show_progress_rate_limit=100 --show_timestamps --test_output=errors rllib/...
-        - ./ci/keep_alive bazel test --build_tests_only --test_tag_filters=tests_dir_E --spawn_strategy=local --flaky_test_attempts=3 --nocache_test_results --test_verbose_timeout_warnings --progress_report_interval=100 --show_progress_rate_limit=100 --show_timestamps --test_output=errors rllib/...
-        - ./ci/keep_alive bazel test --build_tests_only --test_tag_filters=tests_dir_F,tests_dir_I --spawn_strategy=local --flaky_test_attempts=3 --nocache_test_results --test_verbose_timeout_warnings --progress_report_interval=100 --show_progress_rate_limit=100 --show_timestamps --test_output=errors rllib/...
+        - ./ci/keep_alive bazel test --build_tests_only --test_tag_filters=tests_dir_A,tests_dir_B,tests_dir_C,tests_dir_D,tests_dir_E,tests_dir_F,tests_dir_G,tests_dir_H,tests_dir_I --spawn_strategy=local --flaky_test_attempts=3 --nocache_test_results --test_verbose_timeout_warnings --progress_report_interval=100 --show_progress_rate_limit=100 --show_timestamps --test_output=errors rllib/...
 
     # RLlib: tests_dir: Everything in rllib/tests/ directory (J-Z).
     - os: linux
@@ -296,9 +294,7 @@ matrix:
         - ./ci/suppress_output ./ci/travis/install-ray.sh
       script:
         - if [ $RAY_CI_RLLIB_FULL_AFFECTED != "1" ]; then exit; fi
-        - ./ci/keep_alive bazel test --build_tests_only --test_tag_filters=tests_dir_L,tests_dir_M --spawn_strategy=local --flaky_test_attempts=3 --nocache_test_results --test_verbose_timeout_warnings --progress_report_interval=100 --show_progress_rate_limit=100 --show_timestamps --test_output=errors rllib/...
-        - ./ci/keep_alive bazel test --build_tests_only --test_tag_filters=tests_dir_N,tests_dir_O,test_dir_P --spawn_strategy=local --flaky_test_attempts=3 --nocache_test_results --test_verbose_timeout_warnings --progress_report_interval=100 --show_progress_rate_limit=100 --show_timestamps --test_output=errors rllib/...
-        - ./ci/keep_alive bazel test --build_tests_only --test_tag_filters=tests_dir_R,tests_dir_S --spawn_strategy=local --flaky_test_attempts=3 --nocache_test_results --test_verbose_timeout_warnings --progress_report_interval=100 --show_progress_rate_limit=100 --show_timestamps --test_output=errors rllib/...
+        - ./ci/keep_alive bazel test --build_tests_only --test_tag_filters=tests_dir_J,tests_dir_K,tests_dir_L,tests_dir_M,tests_dir_N,tests_dir_O,tests_dir_P,tests_dir_Q,tests_dir_R,tests_dir_S,tests_dir_T,tests_dir_U,tests_dir_V,tests_dir_W,tests_dir_X,tests_dir_Y,tests_dir_Z --spawn_strategy=local --flaky_test_attempts=3 --nocache_test_results --test_verbose_timeout_warnings --progress_report_interval=100 --show_progress_rate_limit=100 --show_timestamps --test_output=errors rllib/...
 
 install:
   - eval `python $TRAVIS_BUILD_DIR/ci/travis/determine_tests_to_run.py`
@@ -380,7 +376,3 @@ deploy:
       repo: ray-project/ray
       all_branches: true
       condition: $LINUX_WHEELS = 1 || $MAC_WHEELS = 1
-
-cache:
-  directories:
-    - $HOME/ray-bazel-cache

BUILD.bazel

@@ -384,7 +384,7 @@ cc_library(
         ],
         exclude = [
             "src/ray/raylet/mock_gcs_client.cc",
-            "src/ray/raylet/monitor_main.cc",
+            "src/ray/raylet/monitor*.cc",
             "src/ray/raylet/*_test.cc",
             "src/ray/raylet/main.cc",
         ],
@@ -843,6 +843,7 @@ cc_library(
         "@com_github_google_glog//:glog",
         "@com_google_absl//absl/synchronization",
         "@com_google_absl//absl/time",
+        "@com_google_googletest//:gtest_main",
         "@plasma//:plasma_client",
     ],
 )
@@ -890,6 +891,11 @@ cc_library(
     includes = ["src/ray/thirdparty"],
 )
 
+alias(
+    name = "hiredis",
+    actual = "@com_github_antirez_redis//:hiredis",
+)
+
 cc_library(
     name = "redis_client",
     srcs = [
@@ -910,7 +916,7 @@ cc_library(
         ":ray_util",
         ":stats_lib",
         "@boost//:asio",
-        "@redis//:hiredis",
+        ":hiredis",
     ],
 )
 
@@ -974,13 +980,13 @@ cc_library(
     copts = COPTS,
     deps = [
         ":gcs_cc_proto",
+        ":hiredis",
         ":node_manager_fbs",
         ":node_manager_rpc",
         ":ray_common",
         ":ray_util",
         ":stats_lib",
         "@boost//:asio",
-        "@redis//:hiredis",
     ],
 )
 
@@ -1247,6 +1253,7 @@ filegroup(
         "python/ray/*.py",
         "python/ray/autoscaler/*.py",
         "python/ray/autoscaler/aws/example-full.yaml",
+        "python/ray/autoscaler/azure/example-full.yaml",
         "python/ray/autoscaler/gcp/example-full.yaml",
         "python/ray/autoscaler/local/example-full.yaml",
         "python/ray/cloudpickle/*.py",
@@ -1273,10 +1280,11 @@ genrule(
     ] + select({
         # Windows is not supported yet, so make stubs for it
         "@bazel_tools//src/conditions:windows": [
-            "@win-redis-bin//file",
+            "@com_github_tporadowski_redis_bin//file",
         ],
         "//conditions:default": [
-            "@redis-src//file",
+            "@com_github_antirez_redis//:file",  # This is necessary so we can access the root of the directory
+            "@com_github_antirez_redis//:files",  # This is necessary to ensure entire directory tree is accessible
         ],
     }),
     outs = [
@@ -1286,18 +1294,21 @@ genrule(
     cmd = select({
         # Windows is not supported yet, so make stubs for it
         "@bazel_tools//src/conditions:windows": """
-            unzip -q -o -- $(location @win-redis-bin//file) redis-server.exe redis-cli.exe &&
+            unzip -q -o -- $(location @com_github_tporadowski_redis_bin//file) redis-server.exe redis-cli.exe &&
             mv -f -- redis-server.exe $(location redis-server) &&
             mv -f -- redis-cli.exe    $(location redis-cli)
         """,
         "//conditions:default": """
             set -x &&
-            tar -xzf $(location @redis-src//file) --strip-components=1 -C . &&
-            make &&
-            mv ./src/redis-server $(location redis-server) &&
+            tmpdir="redis.tmp" &&
+            path=$(location @com_github_antirez_redis//:file) &&
+            cp -p -L -R -- "$${path%/*}" "$${tmpdir}" &&
+            make -s -C "$${tmpdir}" -j"$$(getconf _NPROCESSORS_ONLN || echo 1)" &&
+            mv "$${tmpdir}"/src/redis-server $(location redis-server) &&
             chmod +x $(location redis-server) &&
-            mv ./src/redis-cli $(location redis-cli) &&
-            chmod +x $(location redis-cli)
+            mv "$${tmpdir}"/src/redis-cli $(location redis-cli) &&
+            chmod +x $(location redis-cli) &&
+            rm -r -f -- "$${tmpdir}"
         """,
     }),
     visibility = ["//java:__subpackages__"],

bazel/BUILD.plasma

@@ -23,9 +23,13 @@ cc_library(
     name = "arrow",
     srcs = [
         "cpp/src/arrow/buffer.cc",
+        "cpp/src/arrow/device.cc",
         "cpp/src/arrow/io/interfaces.cc",
+        "cpp/src/arrow/io/memory.cc",
         "cpp/src/arrow/memory_pool.cc",
+        "cpp/src/arrow/result.cc",
         "cpp/src/arrow/status.cc",
+        "cpp/src/arrow/util/future.cc",
         "cpp/src/arrow/util/io_util.cc",
         "cpp/src/arrow/util/logging.cc",
         "cpp/src/arrow/util/memory.cc",
@@ -36,9 +40,12 @@ cc_library(
     ],
     hdrs = [
         "cpp/src/arrow/buffer.h",
+        "cpp/src/arrow/device.h",
         "cpp/src/arrow/io/concurrency.h",
         "cpp/src/arrow/io/interfaces.h",
+        "cpp/src/arrow/io/memory.h",
         "cpp/src/arrow/io/mman.h",
+        "cpp/src/arrow/io/type_fwd.h",
         "cpp/src/arrow/io/util_internal.h",
         "cpp/src/arrow/memory_pool.h",
         "cpp/src/arrow/result.h",
@@ -48,12 +55,14 @@ cc_library(
         "cpp/src/arrow/util/checked_cast.h",
         "cpp/src/arrow/util/compare.h",
         "cpp/src/arrow/util/functional.h",
+        "cpp/src/arrow/util/future.h",
         "cpp/src/arrow/util/io_util.h",
         "cpp/src/arrow/util/iterator.h",
         "cpp/src/arrow/util/logging.h",
         "cpp/src/arrow/util/macros.h",
+        "cpp/src/arrow/util/make_unique.h",
         "cpp/src/arrow/util/memory.h",
-        "cpp/src/arrow/util/stl.h",
+        "cpp/src/arrow/util/optional.h",
         "cpp/src/arrow/util/string.h",
         "cpp/src/arrow/util/string_builder.h",
         "cpp/src/arrow/util/string_view.h",
@@ -64,6 +73,8 @@ cc_library(
         "cpp/src/arrow/util/variant.h",
         "cpp/src/arrow/util/visibility.h",
         "cpp/src/arrow/util/windows_compatibility.h",
+        "cpp/src/arrow/util/windows_fixup.h",
+        "cpp/src/arrow/vendored/optional.hpp",
         "cpp/src/arrow/vendored/string_view.hpp",
         "cpp/src/arrow/vendored/utf8cpp/checked.h",
         "cpp/src/arrow/vendored/utf8cpp/core.h",

bazel/BUILD.redis

@@ -1,13 +1,34 @@
 COPTS = [] + select({
     "@bazel_tools//src/conditions:windows": [
-        "-Dssize_t=ptrdiff_t",  # no ssize_t on Windows
-        "-fcolor-diagnostics",
-        "-fansi-escape-codes",
+        "-D_CRT_DECLARE_NONSTDC_NAMES=0",  # don't define off_t, to avoid conflicts
     ],
     "//conditions:default": [
     ],
 })
 
+filegroup(
+    name = "files",
+    srcs = glob(["**"]),
+    visibility = ["//visibility:public"],
+)
+
+filegroup(
+    name = "file",
+    srcs = ["Makefile"],
+    visibility = ["//visibility:public"],
+)
+
+# This library is for internal hiredis use, because hiredis assumes a
+# different include prefix for itself than external libraries do.
+cc_library(
+    name = "_hiredis",
+    hdrs = [
+        "deps/hiredis/dict.c",
+    ],
+    copts = COPTS,
+    strip_include_prefix = "deps/hiredis",
+)
+
 cc_library(
     name = "hiredis",
     srcs = glob(
@@ -23,13 +44,13 @@ cc_library(
     hdrs = glob([
         "deps/hiredis/*.h",
         "deps/hiredis/adapters/*.h",
-        "deps/hiredis/dict.c",
     ]),
     copts = COPTS,
-    includes = ["deps"],
+    includes = ["deps/hiredis"],
     strip_include_prefix = "deps",
     deps = [
-        "@com_github_ray_project_ray//:platform_shims",
+        ":_hiredis",
+        "@//:platform_shims",
     ],
     visibility = ["//visibility:public"],
 )

bazel/ray_deps_setup.bzl

@@ -73,34 +73,28 @@ def auto_http_archive(*, name=None, url=None, urls=True,
 
 def ray_deps_setup():
     auto_http_archive(
-        name = "redis",
-        build_file = True,
+        name = "com_github_antirez_redis",
+        build_file = "//bazel:BUILD.redis",
         url = "https://github.com/antirez/redis/archive/5.0.3.tar.gz",
         sha256 = "7084e8bd9e5dedf2dbb2a1e1d862d0c46e66cc0872654bdc677f4470d28d84c5",
         patches = [
-            "//thirdparty/patches:hiredis-async-include-dict.patch",
             "//thirdparty/patches:hiredis-casts.patch",
             "//thirdparty/patches:hiredis-connect-rename.patch",
             "//thirdparty/patches:hiredis-windows-sigpipe.patch",
             "//thirdparty/patches:hiredis-windows-sockets.patch",
             "//thirdparty/patches:hiredis-windows-strerror.patch",
             "//thirdparty/patches:hiredis-windows-poll.patch",
+            "//thirdparty/patches:hiredis-windows-translations.patch",
             "//thirdparty/patches:redis-windows-poll.patch",
         ],
     )
 
     http_file(
-        name = "win-redis-bin",
+        name = "com_github_tporadowski_redis_bin",
         sha256 = "6fac443543244c803311de5883b714a7ae3c4fa0594cad51d75b24c4ef45b353",
         urls = ["https://github.com/tporadowski/redis/releases/download/v4.0.14.2/Redis-x64-4.0.14.2.zip"],
     )
 
-    http_file(
-        name = "redis-src",
-        sha256 = "7084e8bd9e5dedf2dbb2a1e1d862d0c46e66cc0872654bdc677f4470d28d84c5",
-        urls = ["https://github.com/antirez/redis/archive/5.0.3.tar.gz"],
-    )
-
     auto_http_archive(
         name = "rules_jvm_external",
         url = "https://github.com/bazelbuild/rules_jvm_external/archive/2.10.tar.gz",
@@ -172,8 +166,8 @@ def ray_deps_setup():
     auto_http_archive(
         name = "plasma",
         build_file = True,
-        url = "https://github.com/apache/arrow/archive/86f34aa07e611787d9cc98c6a33b0a0a536dce57.tar.gz",
-        sha256 = "6b5f55d10681a3938bbf8f07eee52c4eb6e761da6ba27490f55ccb89ce645ac8",
+        url = "https://github.com/apache/arrow/archive/af45b9212156980f55c399e2e88b4e19b4bb8ec1.tar.gz",
+        sha256 = "2f0aaa50053792aa274b402f2530e63c1542085021cfef83beee9281412c12f6",
         patches = [
             "//thirdparty/patches:arrow-headers-unused.patch",
             "//thirdparty/patches:arrow-windows-export.patch",