Revert "[dist] swap mac/linux wheel build order (#9746)" and "Fix package and upload ray jar (#9742)" (#9758)

* Revert "[dist] swap mac/linux wheel build order (#9746)"

This reverts commit a934056.

* Revert "Fix package and upload ray jar (#9742)"

This reverts commit c290c30.

Author: rkooo567

.gitignore

@@ -10,7 +10,6 @@
 /python/python-driver-*
 /thirdparty/pkg/
 /build/java
-.jar
 
 # Files generated by flatc should be ignored
 /src/ray/gcs/format/*_generated.h

.travis.yml

@@ -99,37 +99,33 @@ matrix:
       script:
         - true  # we still need this block to exist, otherwise it will fall back to the global one
 
-    # Build MacOS wheels and MacOS jars
-    - os: osx
-      osx_image: xcode7
+    # Build Linux wheels.
+    - os: linux
       env:
-        - MAC_WHEELS=1 MAC_JARS=1
+        - LINUX_WHEELS=1
         - PYTHONWARNINGS=ignore
         - RAY_INSTALL_JAVA=1
       install:
-        - . ./ci/travis/ci.sh init RAY_CI_MACOS_WHEELS_AFFECTED,RAY_CI_JAVA_AFFECTED,RAY_CI_STREAMING_JAVA_AFFECTED
+        - . ./ci/travis/ci.sh init RAY_CI_LINUX_WHEELS_AFFECTED
       before_script:
         - . ./ci/travis/ci.sh build
       script:
         - . ./ci/travis/ci.sh test_wheels
-        - bash ./java/build-jar-multiplatform.sh darwin
+      cache: false
 
-    # Build Linux wheels and multi-platform jars.
-    - os: linux
+    # Build MacOS wheels.
+    - os: osx
+      osx_image: xcode7
       env:
-        - LINUX_WHEELS=1 MULTIPLATFORM_JARS=1
+        - MAC_WHEELS=1
         - PYTHONWARNINGS=ignore
         - RAY_INSTALL_JAVA=1
       install:
-        - . ./ci/travis/ci.sh init RAY_CI_LINUX_WHEELS_AFFECTED,RAY_CI_JAVA_AFFECTED,RAY_CI_STREAMING_JAVA_AFFECTED
+        - . ./ci/travis/ci.sh init RAY_CI_MACOS_WHEELS_AFFECTED
       before_script:
         - . ./ci/travis/ci.sh build
       script:
         - . ./ci/travis/ci.sh test_wheels
-        - bash ./java/build-jar-multiplatform.sh linux
-        - bash ./java/build-jar-multiplatform.sh multiplatform
-        #- bash ./java/build-jar-multiplatform.sh deploy
-      cache: false
 
     # RLlib: Learning tests (from rllib/tuned_examples/*.yaml).
     - os: linux
@@ -396,39 +392,6 @@ deploy:
       repo: ray-project/ray
       condition: $LINUX_WHEELS = 1 || $MAC_WHEELS = 1
 
-  # Upload jars so that we can debug locally for every commit
-  - provider: s3
-    edge: true # This supposedly opts in to deploy v2.
-    access_key_id: AKIAU6DMUCJUFL3EX3SM
-    secret_access_key:
-      secure: J1sX71fKFPQhgWzColllxfzcF877ScBZ1cIl71krZ6SO0LKnwsCScpQck5eZOyQo/Iverwye0iKtE87qNsiRi3+V2D9iulSr18T09j7+FjPKfxAmXmjfrNafoMXTDQroSJblCri5vl+DysISPqImJkWTNaYhGJ9QakoSd5djnAopLNWj6PCR3S50baS49+nB5nSIY3jMhtUzlaBdniFPFC81Cxyuafr4pv6McGRfR/dK+ZnPhdGtMnVeIJXB+ooZKQ26mDJKBPka4jm3u1Oa72b/Atu2RO3MwxTg79LTrMxXKh2OcCqhtD2Z3lz1OltvNSunCuwY8AejCJsfSLbM9mGDoz+xhNUWmYNy48YFf+61OY8PXi8S/9Q817yb3GpLbb2l/P+KMgq9eSEiELIOwuYsDxPX5TuAg6dx0wCNgDEBJoThSQjYl6MgJrLrs7p+JBxp3giedHiy0TLa5hCVKTj3euONAXDArYnnT+DvUIOkaeTk5DClRZbZ0sUXhLy//HuT5WJvjFBJJZ0u0f4RLVb5D7DI4uMZr7+yJPDR2AXCyW9YMaBEbmEYbPaKi283jlEyn7R33+AZlnXv0THHwZ4xvjKKG3/fBSXsOUmv5wmUveEqVGDj1mKPGj9NF8iA5qMm2AaZuJpEEBVBZtSlTZt6ZG7rzAJZGNL52t7xuMo=
-    bucket: ray-wheels
-    acl: public_read
-    region: us-west-2
-    local_dir: .jar
-    upload-dir: "jars/$TRAVIS_BRANCH/$TRAVIS_COMMIT"
-    skip_cleanup: true
-    on:
-      repo: ray-project/ray
-      all_branches: true
-      condition: $MULTIPLATFORM_JARS = 1 || $MAC_JARS=1
-
-  - provider: s3
-    edge: true # This supposedly opts in to deploy v2.
-    access_key_id: AKIAU6DMUCJUFL3EX3SM
-    secret_access_key:
-      secure: J1sX71fKFPQhgWzColllxfzcF877ScBZ1cIl71krZ6SO0LKnwsCScpQck5eZOyQo/Iverwye0iKtE87qNsiRi3+V2D9iulSr18T09j7+FjPKfxAmXmjfrNafoMXTDQroSJblCri5vl+DysISPqImJkWTNaYhGJ9QakoSd5djnAopLNWj6PCR3S50baS49+nB5nSIY3jMhtUzlaBdniFPFC81Cxyuafr4pv6McGRfR/dK+ZnPhdGtMnVeIJXB+ooZKQ26mDJKBPka4jm3u1Oa72b/Atu2RO3MwxTg79LTrMxXKh2OcCqhtD2Z3lz1OltvNSunCuwY8AejCJsfSLbM9mGDoz+xhNUWmYNy48YFf+61OY8PXi8S/9Q817yb3GpLbb2l/P+KMgq9eSEiELIOwuYsDxPX5TuAg6dx0wCNgDEBJoThSQjYl6MgJrLrs7p+JBxp3giedHiy0TLa5hCVKTj3euONAXDArYnnT+DvUIOkaeTk5DClRZbZ0sUXhLy//HuT5WJvjFBJJZ0u0f4RLVb5D7DI4uMZr7+yJPDR2AXCyW9YMaBEbmEYbPaKi283jlEyn7R33+AZlnXv0THHwZ4xvjKKG3/fBSXsOUmv5wmUveEqVGDj1mKPGj9NF8iA5qMm2AaZuJpEEBVBZtSlTZt6ZG7rzAJZGNL52t7xuMo=
-    bucket: ray-wheels
-    acl: public_read
-    region: us-west-2
-    local_dir: .jar
-    upload-dir: "jars/latest"
-    skip_cleanup: true
-    on:
-      repo: ray-project/ray
-      branch: master
-      condition: $MULTIPLATFORM_JARS = 1 || $MAC_JARS=1
-
   - provider: script
     edge: true # This supposedly opts in to deploy v2.
     script: ./ci/keep_alive bash $TRAVIS_BUILD_DIR/ci/travis/build-autoscaler-images.sh || true

BUILD.bazel

@@ -1829,7 +1829,6 @@ cc_binary(
     }),
     linkshared = 1,
     linkstatic = 1,
-    visibility = ["//java:__subpackages__"],
     deps = [
         "//:core_worker_lib",
         "//:global_state_accessor_lib",
@@ -1840,6 +1839,23 @@ cc_binary(
     ],
 )
 
+genrule(
+    name = "core_worker-jni-darwin-compat",
+    srcs = [":libcore_worker_library_java.so"],
+    outs = ["libcore_worker_library_java.dylib"],
+    cmd = "cp $< $@",
+    output_to_bindir = 1,
+)
+
+filegroup(
+    name = "core_worker_library_java",
+    srcs = select({
+        "@bazel_tools//src/conditions:darwin": [":libcore_worker_library_java.dylib"],
+        "//conditions:default": [":libcore_worker_library_java.so"],
+    }),
+    visibility = ["//java:__subpackages__"],
+)
+
 filegroup(
     name = "python_sources",
     srcs = glob([

bazel/ray.bzl

@@ -1,6 +1,5 @@
 load("@com_github_google_flatbuffers//:build_defs.bzl", "flatbuffer_library_public")
 load("@com_github_checkstyle_java//checkstyle:checkstyle.bzl", "checkstyle_test")
-load("@bazel_skylib//rules:copy_file.bzl", "copy_file")
 load("@bazel_common//tools/maven:pom_file.bzl", "pom_file")
 
 COPTS = ["-DRAY_USE_GLOG"] + select({
@@ -126,7 +125,7 @@ def copy_to_workspace(name, srcs, dstdir = ""):
             dstdir = "." + ("/" + dstdir.replace("\\", "/")).rstrip("/") + "/",
         ),
         # Keep this batch script equivalent to the Bash script above (or take out the batch script)
-        cmd_bat = """
+        cmd_bat = r"""
             (
                 if not exist {dstdir} mkdir {dstdir}
             ) && (
@@ -141,57 +140,3 @@ def copy_to_workspace(name, srcs, dstdir = ""):
         ),
         local = 1,
     )
-
-def native_java_binary(module_name, name, native_binary_name):
-    """Copy native binary file to different path based on operating systems"""
-    copy_file(
-        name = name + "_darwin",
-        src = native_binary_name,
-        out = module_name + "/src/main/resources/native/darwin/" + name,
-    )
-
-    copy_file(
-        name = name + "_linux",
-        src = native_binary_name,
-        out = module_name + "/src/main/resources/native/linux/" + name,
-    )
-
-    copy_file(
-        name = name + "_windows",
-        src = native_binary_name,
-        out = module_name + "/src/main/resources/native/windows/" + name,
-    )
-
-    native.filegroup(
-        name = name,
-        srcs = select({
-            "@bazel_tools//src/conditions:darwin": [name + "_darwin"],
-            "@bazel_tools//src/conditions:windows": [name + "_windows"],
-            "//conditions:default": [name + "_linux"],
-        }),
-        visibility = ["//visibility:public"],
-    )
-
-def native_java_library(module_name, name, native_library_name):
-    """Copy native library file to different path based on operating systems"""
-    copy_file(
-        name = name + "_darwin",
-        src = native_library_name,
-        out = module_name + "/src/main/resources/native/darwin/lib{}.dylib".format(name),
-    )
-
-    copy_file(
-        name = name + "_linux",
-        src = native_library_name,
-        out = module_name + "/src/main/resources/native/linux/lib{}.so".format(name),
-    )
-
-    native.filegroup(
-        name = name,
-        srcs = select({
-            "@bazel_tools//src/conditions:darwin": [name + "_darwin"],
-            "@bazel_tools//src/conditions:windows": [],
-            "//conditions:default": [name + "_linux"],
-        }),
-        visibility = ["//visibility:public"],
-    )

java/BUILD.bazel

@@ -1,6 +1,5 @@
 load("//bazel:ray.bzl", "define_java_module")
-load("//bazel:ray.bzl", "native_java_binary")
-load("//bazel:ray.bzl", "native_java_library")
+load("@bazel_skylib//rules:copy_file.bzl", "copy_file")
 load("@rules_proto_grpc//java:defs.bzl", "java_proto_compile")
 
 exports_files([
@@ -144,69 +143,52 @@ filegroup(
     ],
 )
 
-native_java_binary("runtime", "raylet", "//:raylet")
-
-native_java_binary("runtime", "plasma_store_server", "//:plasma_store_server")
-
-native_java_binary("runtime", "redis-server", "//:redis-server")
-
-native_java_binary("runtime", "gcs_server", "//:gcs_server")
-
-native_java_binary("runtime", "libray_redis_module.so", "//:libray_redis_module.so")
+# plasma_store_server binary should be located in jar root path
+# and keep in sync with mvn resource
+copy_file(
+    name = "cp_plasma_store_server",
+    src = "//:plasma_store_server",
+    out = "plasma_store_server",
+)
 
-native_java_library("runtime", "core_worker_library_java", "//:libcore_worker_library_java.so")
+# `//:redis-server`'s full path is `external/com_github_antirez_redis/redis-server`,
+# This rule removes the prefix, and only keeps `redis-server`.
+copy_file(
+    name = "redis-server-without-prefix",
+    src = "//:redis-server",
+    out = "redis-server",
+)
 
 filegroup(
     name = "java_native_deps",
     srcs = [
-        ":core_worker_library_java",
-        ":gcs_server",
-        ":libray_redis_module.so",
-        ":plasma_store_server",
-        ":raylet",
-        ":redis-server",
+        ":cp_plasma_store_server",
+        ":redis-server-without-prefix",
+        "//:core_worker_library_java",
+        "//:gcs_server",
+        "//:libray_redis_module.so",
+        "//:raylet",
     ],
 )
 
 # Generates the dependencies needed by maven.
 genrule(
-    name = "cp_java_generated",
+    name = "gen_maven_deps",
     srcs = [
         ":all_java_proto",
+        ":java_native_deps",
         ":copy_pom_file",
     ],
-    outs = ["cp_java_generated.out"],
+    outs = ["gen_maven_deps.out"],
     cmd = """
         WORK_DIR="$$(pwd)"
         # Copy protobuf-generated files.
         rm -rf "$$WORK_DIR/java/runtime/src/main/java/io/ray/runtime/generated"
         for f in $(locations //java:all_java_proto); do
             unzip "$$f" -x META-INF/MANIFEST.MF -d "$$WORK_DIR/java/runtime/src/main/java"
         done
-        date > $@
-    """,
-    local = 1,
-    tags = ["no-cache"],
-)
-
-# Generates the dependencies needed by maven.
-genrule(
-    name = "gen_maven_deps",
-    srcs = [
-        ":cp_java_generated",
-        ":java_native_deps",
-    ],
-    outs = ["gen_maven_deps.out"],
-    cmd = """
-        WORK_DIR="$${PWD}"
-        # Copy native dependencies.
-        OS_NAME=""
-        case "$${OSTYPE}" in
-          linux*) OS_NAME="linux";;
-          darwin*) OS_NAME="darwin";;
-          *) echo "$${OSTYPE} is not supported currently"; exit 1;;
-        esac
-        NATIVE_DEPS_DIR="$$WORK_DIR/java/runtime/native_dependencies/native/$$OS_NAME"
+        # Copy native dependecies.
+        NATIVE_DEPS_DIR="$$WORK_DIR/java/runtime/native_dependencies/"
         rm -rf "$$NATIVE_DEPS_DIR"
         mkdir -p "$$NATIVE_DEPS_DIR"
         for f in $(locations //java:java_native_deps); do