[Java] Build Java code with Bazel (#4284)

Author: ruifangChen

.bazelrc

@@ -0,0 +1,2 @@
+# build config
+build --compilation_mode=opt

.travis.yml

@@ -25,18 +25,16 @@ matrix:
       env:
         - JDK='Oracle JDK 8'
         - PYTHON=3.5 PYTHONWARNINGS=ignore
-        - RAY_USE_CMAKE=1
         - RAY_INSTALL_JAVA=1
       install:
         - eval `python $TRAVIS_BUILD_DIR/ci/travis/determine_tests_to_run.py`
         - if [ $RAY_CI_JAVA_AFFECTED != "1" ]; then exit; fi
-
-        - ./ci/travis/install-dependencies.sh
+        - ./ci/suppress_output ./ci/travis/install-bazel.sh
+        - ./ci/suppress_output ./ci/travis/install-dependencies.sh
         - export PATH="$HOME/miniconda/bin:$PATH"
-        - ./ci/travis/install-ray.sh
+        - ./ci/suppress_output ./ci/travis/install-ray.sh
       script:
         - if [ $RAY_CI_JAVA_AFFECTED != "1" ]; then exit; fi
-
         - ./java/test.sh
 
     - os: linux
@@ -167,7 +165,7 @@ install:
   - ./ci/suppress_output bash src/ray/test/run_gcs_tests.sh
   # Raylet tests.
   - ./ci/suppress_output bash src/ray/test/run_object_manager_tests.sh
-  - ./ci/suppress_output bazel test --build_tests_only --test_lang_filters=cc ... -c opt
+  - ./ci/suppress_output bazel test --build_tests_only --test_lang_filters=cc //:all
 
 script:
   - export PATH="$HOME/miniconda/bin:$PATH"

BUILD.bazel

@@ -11,6 +11,7 @@ cc_binary(
     name = "raylet",
     srcs = ["src/ray/raylet/main.cc"],
     copts = COPTS,
+    visibility = ["//java:__subpackages__"],
     deps = [
         ":ray_util",
         ":raylet_lib",
@@ -100,6 +101,7 @@ cc_test(
     copts = COPTS,
     deps = [
         ":ray_util",
+        "@boost//:asio",
         "@com_google_googletest//:gtest_main",
     ],
 )
@@ -372,7 +374,7 @@ pyx_library(
 )
 
 cc_binary(
-    name = "raylet_library_java.so",
+    name = "libraylet_library_java.so",
     srcs = [
         "src/ray/raylet/lib/java/org_ray_runtime_raylet_RayletClientImpl.h",
         "src/ray/raylet/lib/java/org_ray_runtime_raylet_RayletClientImpl.cc",
@@ -394,7 +396,7 @@ cc_binary(
         "//conditions:default": ["external/bazel_tools/tools/jdk/include/linux"],
     }),
     linkshared = 1,
-    linkstatic = 0,
+    linkstatic = 1,
     deps = [
         "//:raylet_lib",
         "@plasma//:plasma_client",
@@ -403,25 +405,31 @@ cc_binary(
 
 genrule(
     name = "raylet-jni-darwin-compat",
-    srcs = [":raylet_library_java.so"],
-    outs = ["raylet_library_java.dylib"],
+    srcs = [":libraylet_library_java.so"],
+    outs = ["libraylet_library_java.dylib"],
     cmd = "cp $< $@",
     output_to_bindir = 1,
 )
 
 filegroup(
     name = "raylet_library_java",
     srcs = select({
-        "@bazel_tools//src/conditions:darwin": [":raylet_library_java.dylib"],
-        "//conditions:default": [":raylet_library_java.so"],
+        "@bazel_tools//src/conditions:darwin": [":libraylet_library_java.dylib"],
+        "//conditions:default": [":libraylet_library_java.so"],
     }),
-    visibility = ["//visibility:public"],
+    visibility = ["//java:__subpackages__"],
+)
+
+filegroup(
+    name = "gcs_fbs_file",
+    srcs = ["src/ray/gcs/format/gcs.fbs"],
+    visibility = ["//java:__subpackages__"],
 )
 
 flatbuffer_py_library(
     name = "python_gcs_fbs",
     srcs = [
-        "src/ray/gcs/format/gcs.fbs",
+        ":gcs_fbs_file",
     ],
     outs = [
         "ActorCheckpointIdData.py",
@@ -526,10 +534,11 @@ genrule(
         mv ./src/redis-cli $(location redis-cli) &&
         chmod +x $(location redis-cli)
     """,
+    visibility = ["//java:__subpackages__"],
 )
 
 cc_binary(
-    name = "ray_redis_module.so",
+    name = "libray_redis_module.so",
     srcs = [
         "src/ray/gcs/redis_module/ray_redis_module.cc",
         "src/ray/gcs/redis_module/redis_string.h",
@@ -538,6 +547,7 @@ cc_binary(
     copts = COPTS,
     linkshared = 1,
     linkstatic = 1,
+    visibility = ["//java:__subpackages__"],
     deps = [
         ":ray_common",
     ],
@@ -552,29 +562,30 @@ genrule(
         "//:python_node_manager_fbs",
         "//:redis-server",
         "//:redis-cli",
-        "//:ray_redis_module.so",
+        "//:libray_redis_module.so",
         "//:raylet",
         "//:raylet_monitor",
         "@plasma//:plasma_store_server",
     ],
-    outs = ["ray_pkg"],
+    outs = ["ray_pkg.out"],
     cmd = """
         set -x &&
-        mkdir -p python/ray &&
-        cp $(location python/ray/_raylet.so) python/ray &&
-        mkdir -p python/ray/core/src/ray/thirdparty/redis/src/ &&
-        cp $(location //:redis-server) python/ray/core/src/ray/thirdparty/redis/src/ &&
-        cp $(location //:redis-cli) python/ray/core/src/ray/thirdparty/redis/src/ &&
-        mkdir -p python/ray/core/src/ray/gcs/redis_module/ &&
-        cp $(locations //:ray_redis_module.so) python/ray/core/src/ray/gcs/redis_module/libray_redis_module.so &&
-        mkdir -p python/ray/core/src/ray/raylet/ &&
-        cp $(location //:raylet_monitor) python/ray/core/src/ray/raylet/ &&
-        mkdir -p python/ray/core/src/plasma &&
-        cp $(location @plasma//:plasma_store_server) python/ray/core/src/plasma/ &&
-        cp $(location //:raylet) python/ray/core/src/ray/raylet/ &&
-        for f in $(locations //:python_gcs_fbs); do cp $$f python/ray/core/generated/; done &&
-        mkdir -p python/ray/core/generated/ray/protocol/ &&
-        for f in $(locations //:python_node_manager_fbs); do cp $$f python/ray/core/generated/ray/protocol/; done &&
-        mv python $(location ray_pkg)
-        """,
+        WORK_DIR=$$(pwd) &&
+        cp -f $(location python/ray/_raylet.so) $$WORK_DIR/python/ray &&
+        mkdir -p $$WORK_DIR/python/ray/core/src/ray/thirdparty/redis/src/ &&
+        cp -f $(location //:redis-server) $$WORK_DIR/python/ray/core/src/ray/thirdparty/redis/src/ &&
+        cp -f $(location //:redis-cli) $$WORK_DIR/python/ray/core/src/ray/thirdparty/redis/src/ &&
+        mkdir -p $$WORK_DIR/python/ray/core/src/ray/gcs/redis_module/ &&
+        cp -f $(locations //:libray_redis_module.so) $$WORK_DIR/python/ray/core/src/ray/gcs/redis_module/ &&
+        cp -f $(location //:raylet_monitor) $$WORK_DIR/python/ray/core/src/ray/raylet/ &&
+        cp -f $(location @plasma//:plasma_store_server) $$WORK_DIR/python/ray/core/src/plasma/ &&
+        cp -f $(location //:raylet) $$WORK_DIR/python/ray/core/src/ray/raylet/ &&
+        for f in $(locations //:python_gcs_fbs); do cp -f $$f $$WORK_DIR/python/ray/core/generated/; done &&
+        mkdir -p $$WORK_DIR/python/ray/core/generated/ray/protocol/ &&
+        for f in $(locations //:python_node_manager_fbs); do 
+            cp -f $$f $$WORK_DIR/python/ray/core/generated/ray/protocol/; 
+        done &&
+        echo $$WORK_DIR > $@
+    """,
+    local = 1,
 )

WORKSPACE

@@ -1,4 +1,15 @@
+workspace(name = "com_github_ray_project_ray")
+
 load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository", "new_git_repository")
+load("//java:repo.bzl", "java_repositories")
+
+java_repositories()
+
+git_repository(
+    name = "com_github_checkstyle_java",
+    commit = "85f37871ca03b9d3fee63c69c8107f167e24e77b",
+    remote = "https://github.com/ruifangChen/checkstyle_java",
+)
 
 git_repository(
     name = "com_github_nelhage_rules_boost",
@@ -38,8 +49,8 @@ new_git_repository(
 new_git_repository(
     name = "plasma",
     build_file = "@//bazel:BUILD.plasma",
-    commit = "6a27c660ea700febf6fd73b2e851ab96e9315134",
-    remote = "https://github.com/ray-project/arrow",
+    commit = "9c33e1aee98f96a82c6a0b6bb214df172a143fb1",
+    remote = "https://github.com/apache/arrow",
 )
 
 new_git_repository(

bazel/BUILD.plasma

@@ -1,5 +1,26 @@
 load("@com_github_google_flatbuffers//:build_defs.bzl", "flatbuffer_cc_library")
 
+java_library(
+    name = "org_apache_arrow_arrow_plasma",
+    srcs = glob(["java/plasma/src/main/java/**/*.java"]),
+    data = [":plasma_client_java"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "@org_slf4j_slf4j_api//jar",
+    ],
+)
+
+java_binary(
+    name = "org_apache_arrow_arrow_plasma_test",
+    srcs = ["java/plasma/src/test/java/org/apache/arrow/plasma/PlasmaClientTest.java"],
+    main_class = "org.apache.arrow.plasma.PlasmaClientTest",
+    visibility = ["//visibility:public"],
+    deps = [
+        ":org_apache_arrow_arrow_plasma",
+        "@junit_junit//jar",
+    ],
+)
+
 cc_library(
     name = "arrow",
     srcs = [
@@ -8,6 +29,7 @@ cc_library(
         "cpp/src/arrow/status.cc",
         "cpp/src/arrow/util/io-util.cc",
         "cpp/src/arrow/util/logging.cc",
+        "cpp/src/arrow/util/memory.cc",
         "cpp/src/arrow/util/thread-pool.cc",
     ],
     hdrs = [
@@ -19,6 +41,7 @@ cc_library(
         "cpp/src/arrow/util/io-util.h",
         "cpp/src/arrow/util/logging.h",
         "cpp/src/arrow/util/macros.h",
+        "cpp/src/arrow/util/memory.h",
         "cpp/src/arrow/util/string_builder.h",
         "cpp/src/arrow/util/string_view.h",
         "cpp/src/arrow/util/thread-pool.h",
@@ -48,6 +71,7 @@ cc_library(
         "cpp/src/plasma/common.h",
         "cpp/src/plasma/common_generated.h",
         "cpp/src/plasma/compat.h",
+        "cpp/src/plasma/external_store.h",
         "cpp/src/plasma/fling.h",
         "cpp/src/plasma/io.h",
         "cpp/src/plasma/malloc.h",
@@ -65,7 +89,7 @@ cc_library(
 )
 
 cc_binary(
-    name = "plasma_client_java.so",
+    name = "libplasma_java.so",
     srcs = [
         "cpp/src/plasma/lib/java/org_apache_arrow_plasma_PlasmaClientJNI.cc",
         "cpp/src/plasma/lib/java/org_apache_arrow_plasma_PlasmaClientJNI.h",
@@ -77,7 +101,7 @@ cc_binary(
         "cpp/src",
     ],
     linkshared = 1,
-    linkstatic = 0,
+    linkstatic = 1,
     deps = [":plasma_client"],
 )
 
@@ -101,17 +125,17 @@ genrule(
 
 genrule(
     name = "plasma-jni-darwin-compat",
-    srcs = [":plasma_client_java.so"],
-    outs = ["plasma_client_java.dylib"],
+    srcs = [":libplasma_java.so"],
+    outs = ["libplasma_java.dylib"],
     cmd = "cp $< $@",
     output_to_bindir = 1,
 )
 
 filegroup(
     name = "plasma_client_java",
     srcs = select({
-        "@bazel_tools//src/conditions:darwin": [":plasma_client_java.dylib"],
-        "//conditions:default": [":plasma_client_java.so"],
+        "@bazel_tools//src/conditions:darwin": [":libplasma_java.dylib"],
+        "//conditions:default": [":libplasma_java.so"],
     }),
     visibility = ["//visibility:public"],
 )
@@ -122,10 +146,14 @@ cc_library(
         "cpp/src/plasma/dlmalloc.cc",
         "cpp/src/plasma/events.cc",
         "cpp/src/plasma/eviction_policy.cc",
+        "cpp/src/plasma/external_store.cc",
+        "cpp/src/plasma/plasma_allocator.cc",
     ],
     hdrs = [
         "cpp/src/plasma/events.h",
         "cpp/src/plasma/eviction_policy.h",
+        "cpp/src/plasma/external_store.h",
+        "cpp/src/plasma/plasma_allocator.h",
         "cpp/src/plasma/store.h",
         "cpp/src/plasma/thirdparty/ae/ae.h",
         "cpp/src/plasma/thirdparty/ae/ae_epoll.c",

bazel/ray.bzl

@@ -1,4 +1,5 @@
 load("@com_github_google_flatbuffers//:build_defs.bzl", "flatbuffer_library_public")
+load("@com_github_checkstyle_java//checkstyle:checkstyle.bzl", "checkstyle_test")
 
 def flatbuffer_py_library(name, srcs, outs, out_prefix, includes = [], include_paths = []):
     flatbuffer_library_public(
@@ -10,3 +11,44 @@ def flatbuffer_py_library(name, srcs, outs, out_prefix, includes = [], include_p
         include_paths = include_paths,
         includes = includes,
     )
+
+def flatbuffer_java_library(name, srcs, outs, out_prefix, includes = [], include_paths = []):
+    flatbuffer_library_public(
+        name = name,
+        srcs = srcs,
+        outs = outs,
+        language_flag = "-j",
+        out_prefix = out_prefix,
+        include_paths = include_paths,
+        includes = includes,
+    )
+
+def define_java_module(name, additional_srcs = [], additional_resources = [], define_test_lib = False, test_deps = [], **kwargs):
+    native.java_library(
+        name = "org_ray_ray_" + name,
+        srcs = additional_srcs + native.glob([name + "/src/main/java/**/*.java"]),
+        resources = native.glob([name + "/src/main/resources/**"]) + additional_resources,
+        **kwargs
+    )
+    checkstyle_test(
+        name = "org_ray_ray_" + name + "-checkstyle",
+        target = "//java:org_ray_ray_" + name,
+        config = "//java:checkstyle.xml",
+        suppressions = "//java:checkstyle-suppressions.xml",
+        size = "small",
+        tags = ["checkstyle"],
+    )
+    if define_test_lib:
+        native.java_library(
+            name = "org_ray_ray_" + name + "_test",
+            srcs = native.glob([name + "/src/test/java/**/*.java"]),
+            deps = test_deps,
+        )
+        checkstyle_test(
+            name = "org_ray_ray_" + name + "_test-checkstyle",
+            target = "//java:org_ray_ray_" + name + "_test",
+            config = "//java:checkstyle.xml",
+            suppressions = "//java:checkstyle-suppressions.xml",
+            size = "small",
+            tags = ["checkstyle"],
+        )

build.sh

@@ -121,14 +121,15 @@ else
   $PYTHON_EXECUTABLE -m pip install \
       --target=$ROOT_DIR/python/ray/pyarrow_files pyarrow==0.12.0.RAY \
       --find-links https://s3-us-west-2.amazonaws.com/arrow-wheels/9357dc130789ee42f8181d8724bee1d5d1509060/index.html
-  bazel build //:ray_pkg -c opt --verbose_failures --action_env=PYTHON_BIN_PATH=$PYTHON_EXECUTABLE
-  # Copy files and keep them writeable. This is a workaround, as Bazel
-  # marks all generated files non-writeable. If we would just copy them
-  # over without adding write permission, the copy would fail the next time.
-  # TODO(pcm): It would be great to have a solution here that does not
-  # require us to copy the files.
-  find $ROOT_DIR/bazel-genfiles/ray_pkg/ -exec chmod +w {} \;
-  cp -r $ROOT_DIR/bazel-genfiles/ray_pkg/ray $ROOT_DIR/python || true
+
+  if [ "$RAY_BUILD_JAVA" == "YES" ]; then
+    bazel run //java:bazel_deps -- generate -r $ROOT_DIR -s java/third_party/workspace.bzl -d java/dependencies.yaml
+    bazel build //java:all --verbose_failures --action_env=PATH
+  fi
+
+  if [ "$RAY_BUILD_PYTHON" == "YES" ]; then
+    bazel build //:ray_pkg --verbose_failures --action_env=PYTHON_BIN_PATH=$PYTHON_EXECUTABLE
+  fi
 fi
 
 popd