[dist] Fix travis deploy for java dist

.gitignore

@@ -10,6 +10,7 @@
 /python/python-driver-*
 /thirdparty/pkg/
 /build/java
+.jar
 
 # Files generated by flatc should be ignored
 /src/ray/gcs/format/*_generated.h

.travis.yml

@@ -99,33 +99,37 @@ matrix:
       script:
         - true  # we still need this block to exist, otherwise it will fall back to the global one
 
-    # Build Linux wheels.
-    - os: linux
+    # Build MacOS wheels and MacOS jars
+    - os: osx
+      osx_image: xcode7
       env:
-        - LINUX_WHEELS=1
+        - MAC_WHEELS=1 MAC_JARS=1
         - PYTHONWARNINGS=ignore
         - RAY_INSTALL_JAVA=1
       install:
-        - . ./ci/travis/ci.sh init RAY_CI_LINUX_WHEELS_AFFECTED
+        - . ./ci/travis/ci.sh init RAY_CI_MACOS_WHEELS_AFFECTED,RAY_CI_JAVA_AFFECTED,RAY_CI_STREAMING_JAVA_AFFECTED
       before_script:
         - . ./ci/travis/ci.sh build
       script:
         - . ./ci/travis/ci.sh test_wheels
-      cache: false
+        - bash ./java/build-jar-multiplatform.sh darwin
 
-    # Build MacOS wheels.
-    - os: osx
-      osx_image: xcode7
+    # Build Linux wheels and multi-platform jars.
+    - os: linux
       env:
-        - MAC_WHEELS=1
+        - LINUX_WHEELS=1 MULTIPLATFORM_JARS=1
         - PYTHONWARNINGS=ignore
         - RAY_INSTALL_JAVA=1
       install:
-        - . ./ci/travis/ci.sh init RAY_CI_MACOS_WHEELS_AFFECTED
+        - . ./ci/travis/ci.sh init RAY_CI_LINUX_WHEELS_AFFECTED,RAY_CI_JAVA_AFFECTED,RAY_CI_STREAMING_JAVA_AFFECTED
       before_script:
         - . ./ci/travis/ci.sh build
       script:
         - . ./ci/travis/ci.sh test_wheels
+        - bash ./java/build-jar-multiplatform.sh linux
+        - bash ./java/build-jar-multiplatform.sh multiplatform
+        #- bash ./java/build-jar-multiplatform.sh deploy
+      cache: false
 
     # RLlib: Learning tests (from rllib/tuned_examples/*.yaml).
     - os: linux
@@ -392,6 +396,39 @@ deploy:
       repo: ray-project/ray
       condition: $LINUX_WHEELS = 1 || $MAC_WHEELS = 1
 
+  # Upload jars so that we can debug locally for every commit
+  - provider: s3
+    edge: true # This supposedly opts in to deploy v2.
+    access_key_id: AKIAU6DMUCJUFL3EX3SM
+    secret_access_key:
+      secure: J1sX71fKFPQhgWzColllxfzcF877ScBZ1cIl71krZ6SO0LKnwsCScpQck5eZOyQo/Iverwye0iKtE87qNsiRi3+V2D9iulSr18T09j7+FjPKfxAmXmjfrNafoMXTDQroSJblCri5vl+DysISPqImJkWTNaYhGJ9QakoSd5djnAopLNWj6PCR3S50baS49+nB5nSIY3jMhtUzlaBdniFPFC81Cxyuafr4pv6McGRfR/dK+ZnPhdGtMnVeIJXB+ooZKQ26mDJKBPka4jm3u1Oa72b/Atu2RO3MwxTg79LTrMxXKh2OcCqhtD2Z3lz1OltvNSunCuwY8AejCJsfSLbM9mGDoz+xhNUWmYNy48YFf+61OY8PXi8S/9Q817yb3GpLbb2l/P+KMgq9eSEiELIOwuYsDxPX5TuAg6dx0wCNgDEBJoThSQjYl6MgJrLrs7p+JBxp3giedHiy0TLa5hCVKTj3euONAXDArYnnT+DvUIOkaeTk5DClRZbZ0sUXhLy//HuT5WJvjFBJJZ0u0f4RLVb5D7DI4uMZr7+yJPDR2AXCyW9YMaBEbmEYbPaKi283jlEyn7R33+AZlnXv0THHwZ4xvjKKG3/fBSXsOUmv5wmUveEqVGDj1mKPGj9NF8iA5qMm2AaZuJpEEBVBZtSlTZt6ZG7rzAJZGNL52t7xuMo=
+    bucket: ray-wheels
+    acl: public_read
+    region: us-west-2
+    local_dir: .jar
+    upload-dir: "jars/$TRAVIS_BRANCH/$TRAVIS_COMMIT"
+    skip_cleanup: true
+    on:
+      repo: ray-project/ray
+      all_branches: true
+      condition: $MULTIPLATFORM_JARS = 1 || $MAC_JARS = 1
+
+  - provider: s3
+    edge: true # This supposedly opts in to deploy v2.
+    access_key_id: AKIAU6DMUCJUFL3EX3SM
+    secret_access_key:
+      secure: J1sX71fKFPQhgWzColllxfzcF877ScBZ1cIl71krZ6SO0LKnwsCScpQck5eZOyQo/Iverwye0iKtE87qNsiRi3+V2D9iulSr18T09j7+FjPKfxAmXmjfrNafoMXTDQroSJblCri5vl+DysISPqImJkWTNaYhGJ9QakoSd5djnAopLNWj6PCR3S50baS49+nB5nSIY3jMhtUzlaBdniFPFC81Cxyuafr4pv6McGRfR/dK+ZnPhdGtMnVeIJXB+ooZKQ26mDJKBPka4jm3u1Oa72b/Atu2RO3MwxTg79LTrMxXKh2OcCqhtD2Z3lz1OltvNSunCuwY8AejCJsfSLbM9mGDoz+xhNUWmYNy48YFf+61OY8PXi8S/9Q817yb3GpLbb2l/P+KMgq9eSEiELIOwuYsDxPX5TuAg6dx0wCNgDEBJoThSQjYl6MgJrLrs7p+JBxp3giedHiy0TLa5hCVKTj3euONAXDArYnnT+DvUIOkaeTk5DClRZbZ0sUXhLy//HuT5WJvjFBJJZ0u0f4RLVb5D7DI4uMZr7+yJPDR2AXCyW9YMaBEbmEYbPaKi283jlEyn7R33+AZlnXv0THHwZ4xvjKKG3/fBSXsOUmv5wmUveEqVGDj1mKPGj9NF8iA5qMm2AaZuJpEEBVBZtSlTZt6ZG7rzAJZGNL52t7xuMo=
+    bucket: ray-wheels
+    acl: public_read
+    region: us-west-2
+    local_dir: .jar
+    upload-dir: "jars/latest"
+    skip_cleanup: true
+    on:
+      repo: ray-project/ray
+      branch: master
+      condition: $MULTIPLATFORM_JARS = 1 || $MAC_JARS = 1
+
   - provider: script
     edge: true # This supposedly opts in to deploy v2.
     script: ./ci/keep_alive bash $TRAVIS_BUILD_DIR/ci/travis/build-autoscaler-images.sh || true

BUILD.bazel

@@ -1834,6 +1834,7 @@ cc_binary(
     }),
     linkshared = 1,
     linkstatic = 1,
+    visibility = ["//java:__subpackages__"],
     deps = [
         "//:core_worker_lib",
         "//:global_state_accessor_lib",
@@ -1844,23 +1845,6 @@ cc_binary(
     ],
 )
 
-genrule(
-    name = "core_worker-jni-darwin-compat",
-    srcs = [":libcore_worker_library_java.so"],
-    outs = ["libcore_worker_library_java.dylib"],
-    cmd = "cp $< $@",
-    output_to_bindir = 1,
-)
-
-filegroup(
-    name = "core_worker_library_java",
-    srcs = select({
-        "@bazel_tools//src/conditions:darwin": [":libcore_worker_library_java.dylib"],
-        "//conditions:default": [":libcore_worker_library_java.so"],
-    }),
-    visibility = ["//java:__subpackages__"],
-)
-
 filegroup(
     name = "python_sources",
     srcs = glob([

bazel/ray.bzl

@@ -1,5 +1,6 @@
 load("@com_github_google_flatbuffers//:build_defs.bzl", "flatbuffer_library_public")
 load("@com_github_checkstyle_java//checkstyle:checkstyle.bzl", "checkstyle_test")
+load("@bazel_skylib//rules:copy_file.bzl", "copy_file")
 load("@bazel_common//tools/maven:pom_file.bzl", "pom_file")
 
 COPTS = ["-DRAY_USE_GLOG"] + select({
@@ -128,7 +129,7 @@ def copy_to_workspace(name, srcs, dstdir = ""):
             dstdir = "." + ("/" + dstdir.replace("\\", "/")).rstrip("/") + "/",
         ),
         # Keep this batch script equivalent to the Bash script above (or take out the batch script)
-        cmd_bat = r"""
+        cmd_bat = """
             (
                 if not exist {dstdir} mkdir {dstdir}
             ) && (
@@ -143,3 +144,57 @@ def copy_to_workspace(name, srcs, dstdir = ""):
         ),
         local = 1,
     )
+
+def native_java_binary(module_name, name, native_binary_name):
+    """Copy native binary file to different path based on operating systems"""
+    copy_file(
+        name = name + "_darwin",
+        src = native_binary_name,
+        out = module_name + "/src/main/resources/native/darwin/" + name,
+    )
+
+    copy_file(
+        name = name + "_linux",
+        src = native_binary_name,
+        out = module_name + "/src/main/resources/native/linux/" + name,
+    )
+
+    copy_file(
+        name = name + "_windows",
+        src = native_binary_name,
+        out = module_name + "/src/main/resources/native/windows/" + name,
+    )
+
+    native.filegroup(
+        name = name,
+        srcs = select({
+            "@bazel_tools//src/conditions:darwin": [name + "_darwin"],
+            "@bazel_tools//src/conditions:windows": [name + "_windows"],
+            "//conditions:default": [name + "_linux"],
+        }),
+        visibility = ["//visibility:public"],
+    )
+
+def native_java_library(module_name, name, native_library_name):
+    """Copy native library file to different path based on operating systems"""
+    copy_file(
+        name = name + "_darwin",
+        src = native_library_name,
+        out = module_name + "/src/main/resources/native/darwin/lib{}.dylib".format(name),
+    )
+
+    copy_file(
+        name = name + "_linux",
+        src = native_library_name,
+        out = module_name + "/src/main/resources/native/linux/lib{}.so".format(name),
+    )
+
+    native.filegroup(
+        name = name,
+        srcs = select({
+            "@bazel_tools//src/conditions:darwin": [name + "_darwin"],
+            "@bazel_tools//src/conditions:windows": [],
+            "//conditions:default": [name + "_linux"],
+        }),
+        visibility = ["//visibility:public"],
+    )

java/BUILD.bazel

@@ -1,5 +1,6 @@
 load("//bazel:ray.bzl", "define_java_module")
-load("@bazel_skylib//rules:copy_file.bzl", "copy_file")
+load("//bazel:ray.bzl", "native_java_binary")
+load("//bazel:ray.bzl", "native_java_library")
 load("@rules_proto_grpc//java:defs.bzl", "java_proto_compile")
 
 exports_files([
@@ -143,52 +144,69 @@ filegroup(
     ],
 )
 
-# plasma_store_server binary should be located in jar root path
-# and keep in sync with mvn resource
-copy_file(
-    name = "cp_plasma_store_server",
-    src = "//:plasma_store_server",
-    out = "plasma_store_server",
-)
+native_java_binary("runtime", "raylet", "//:raylet")
 
-# `//:redis-server`'s full path is `external/com_github_antirez_redis/redis-server`,
-# This rule removes the prefix, and only keeps `redis-server`.
-copy_file(
-    name = "redis-server-without-prefix",
-    src = "//:redis-server",
-    out = "redis-server",
-)
+native_java_binary("runtime", "plasma_store_server", "//:plasma_store_server")
+
+native_java_binary("runtime", "redis-server", "//:redis-server")
+
+native_java_binary("runtime", "gcs_server", "//:gcs_server")
+
+native_java_binary("runtime", "libray_redis_module.so", "//:libray_redis_module.so")
+
+native_java_library("runtime", "core_worker_library_java", "//:libcore_worker_library_java.so")
 
 filegroup(
     name = "java_native_deps",
     srcs = [
-        ":cp_plasma_store_server",
-        ":redis-server-without-prefix",
-        "//:core_worker_library_java",
-        "//:gcs_server",
-        "//:libray_redis_module.so",
-        "//:raylet",
+        ":core_worker_library_java",
+        ":gcs_server",
+        ":libray_redis_module.so",
+        ":plasma_store_server",
+        ":raylet",
+        ":redis-server",
     ],
 )
 
 # Generates the dependencies needed by maven.
 genrule(
-    name = "gen_maven_deps",
+    name = "cp_java_generated",
     srcs = [
         ":all_java_proto",
-        ":java_native_deps",
         ":copy_pom_file",
     ],
-    outs = ["gen_maven_deps.out"],
+    outs = ["cp_java_generated.out"],
     cmd = """
         WORK_DIR="$$(pwd)"
         # Copy protobuf-generated files.
         rm -rf "$$WORK_DIR/java/runtime/src/main/java/io/ray/runtime/generated"
         for f in $(locations //java:all_java_proto); do
             unzip "$$f" -x META-INF/MANIFEST.MF -d "$$WORK_DIR/java/runtime/src/main/java"
         done
-        # Copy native dependecies.
-        NATIVE_DEPS_DIR="$$WORK_DIR/java/runtime/native_dependencies/"
+        date > $@
+    """,
+    local = 1,
+    tags = ["no-cache"],
+)
+
+# Generates the dependencies needed by maven.
+genrule(
+    name = "gen_maven_deps",
+    srcs = [
+        ":cp_java_generated",
+        ":java_native_deps",
+    ],
+    outs = ["gen_maven_deps.out"],
+    cmd = """
+        WORK_DIR="$${PWD}"
+        # Copy native dependencies.
+        OS_NAME=""
+        case "$${OSTYPE}" in
+          linux*) OS_NAME="linux";;
+          darwin*) OS_NAME="darwin";;
+          *) echo "$${OSTYPE} is not supported currently"; exit 1;;
+        esac
+        NATIVE_DEPS_DIR="$$WORK_DIR/java/runtime/native_dependencies/native/$$OS_NAME"
         rm -rf "$$NATIVE_DEPS_DIR"
         mkdir -p "$$NATIVE_DEPS_DIR"
         for f in $(locations //java:java_native_deps); do