Upgrade arrow to include more detailed flushing message

build.sh

@@ -117,9 +117,9 @@ make -j${PARALLEL}
 popd
 
 # Move stuff from Arrow to Ray.
-cp $ROOT_DIR/thirdparty/pkg/arrow/cpp/build/cpp-install/bin/plasma_store $BUILD_DIR/src/plasma/
+cp $ROOT_DIR/thirdparty/pkg/arrow/cpp/build/cpp-install/bin/plasma_store_server $BUILD_DIR/src/plasma/
 if [[ "$RAY_BUILD_PYTHON" == "YES" ]]; then
-  cp $ROOT_DIR/thirdparty/pkg/arrow/cpp/build/cpp-install/bin/plasma_store $BUILD_DIR/../python/ray/core/src/plasma/
+  cp $ROOT_DIR/thirdparty/pkg/arrow/cpp/build/cpp-install/bin/plasma_store_server $BUILD_DIR/../python/ray/core/src/plasma/
 fi
 if [[ "$RAY_BUILD_JAVA" == "YES" ]]; then
   cp $ROOT_DIR/thirdparty/build/arrow/cpp/build/release/libplasma_java.* $BUILD_DIR/src/plasma/

java/cleanup.sh

@@ -1,7 +1,7 @@
 pkill -9 python
 pkill -9 local_scheduler
 pkill -9 plasma_manager
-pkill -9 plasma_store
+pkill -9 plasma_store_server
 pkill -9 global_scheduler
 pkill -9 redis-server
 pkill -9 redis

java/prepare.sh

@@ -43,7 +43,7 @@ fi
 
 declare -a nativeBinaries=(
     "./src/common/thirdparty/redis/src/redis-server"
-    "./src/plasma/plasma_store"
+    "./src/plasma/plasma_store_server"
     "./src/plasma/plasma_manager"
     "./src/local_scheduler/local_scheduler"
     "./src/global_scheduler/global_scheduler"

java/ray.config.ini

@@ -121,7 +121,7 @@ driver_args =
 [ray.java.path.source]
 redis_server = %CONFIG_FILE_DIR%/../build/src/common/thirdparty/redis/src/redis-server
 redis_module = %CONFIG_FILE_DIR%/../build/src/common/redis_module/libray_redis_module.so
-store = %CONFIG_FILE_DIR%/../build/src/plasma/plasma_store
+store = %CONFIG_FILE_DIR%/../build/src/plasma/plasma_store_server
 store_manager = %CONFIG_FILE_DIR%/../build/src/plasma/plasma_manager
 local_scheduler = %CONFIG_FILE_DIR%/../build/src/local_scheduler/local_scheduler
 global_scheduler = %CONFIG_FILE_DIR%/../build/src/global_scheduler/global_scheduler
@@ -134,7 +134,7 @@ java_jnilib_paths = ray.java.path.jni.package
 [ray.java.path.package]
 redis_server = %CONFIG_FILE_DIR%/../build/src/common/thirdparty/redis/src/redis-server
 redis_module = %CONFIG_FILE_DIR%/../build/src/common/redis_module/libray_redis_module.so
-store = %CONFIG_FILE_DIR%/../build/src/plasma/plasma_store
+store = %CONFIG_FILE_DIR%/../build/src/plasma/plasma_store_server
 store_manager = %CONFIG_FILE_DIR%/../build/src/plasma/plasma_manager
 local_scheduler = %CONFIG_FILE_DIR%/../build/src/local_scheduler/local_scheduler
 global_scheduler = %CONFIG_FILE_DIR%/../build/src/global_scheduler/global_scheduler
@@ -147,7 +147,7 @@ java_jnilib_paths = ray.java.path.jni.package
 [ray.java.path.deploy]
 redis_server = %CONFIG_FILE_DIR%/native/bin/redis-server
 redis_module = %CONFIG_FILE_DIR%/native/lib/libray_redis_module.so
-store = %CONFIG_FILE_DIR%/native/bin/plasma_store
+store = %CONFIG_FILE_DIR%/native/bin/plasma_store_server
 store_manager = %CONFIG_FILE_DIR%/native/bin/plasma_manager
 local_scheduler = %CONFIG_FILE_DIR%/native/bin/local_scheduler
 global_scheduler = %CONFIG_FILE_DIR%/native/bin/global_scheduler
@@ -156,4 +156,3 @@ python_dir = %CONFIG_FILE_DIR%/python
 java_runtime_rewritten_jars_dir = %CONFIG_FILE_DIR%/java/lib/
 java_class_paths = ray.java.path.classes.deploy
 java_jnilib_paths = ray.java.path.jni.deploy
-

python/ray/plasma/plasma.py

@@ -65,7 +65,7 @@ def start_plasma_store(plasma_store_memory=DEFAULT_PLASMA_STORE_MEMORY,
 
     plasma_store_executable = os.path.join(
         os.path.abspath(os.path.dirname(__file__)),
-        "../core/src/plasma/plasma_store")
+        "../core/src/plasma/plasma_store_server")
     plasma_store_name = "/tmp/plasma_store{}".format(random_name())
     command = [
         plasma_store_executable, "-s", plasma_store_name, "-m",

python/ray/plasma/test/test.py

@@ -87,7 +87,8 @@ def start_plasma_store(plasma_store_memory=DEFAULT_PLASMA_STORE_MEMORY,
     """
     if use_valgrind and use_profiler:
         raise Exception("Cannot use valgrind and profiler at the same time.")
-    plasma_store_executable = os.path.join(pa.__path__[0], "plasma_store")
+    plasma_store_executable = os.path.join(pa.__path__[0],
+                                           "plasma_store_server")
     plasma_store_name = "/tmp/plasma_store{}".format(random_name())
     command = [
         plasma_store_executable, "-s", plasma_store_name, "-m",

python/setup.py

@@ -21,7 +21,8 @@
 ray_files = [
     "ray/core/src/common/thirdparty/redis/src/redis-server",
     "ray/core/src/common/redis_module/libray_redis_module.so",
-    "ray/core/src/plasma/plasma_store", "ray/core/src/plasma/plasma_manager",
+    "ray/core/src/plasma/plasma_store_server",
+    "ray/core/src/plasma/plasma_manager",
     "ray/core/src/local_scheduler/local_scheduler",
     "ray/core/src/local_scheduler/liblocal_scheduler_library_python.so",
     "ray/core/src/global_scheduler/global_scheduler",

src/local_scheduler/test/run_tests.sh

@@ -30,9 +30,9 @@ sleep 1s
 ./src/common/thirdparty/redis/src/redis-cli set NumRedisShards 1
 ./src/common/thirdparty/redis/src/redis-cli rpush RedisShards 127.0.0.1:6380
 
-./src/plasma/plasma_store -s /tmp/plasma_store_socket_1 -m 100000000 &
+./src/plasma/plasma_store_server -s /tmp/plasma_store_socket_1 -m 100000000 &
 sleep 0.5s
 ./src/local_scheduler/local_scheduler_tests
 ./src/common/thirdparty/redis/src/redis-cli shutdown
 ./src/common/thirdparty/redis/src/redis-cli -p 6380 shutdown
-killall plasma_store
+killall plasma_store_server

src/local_scheduler/test/run_valgrind.sh

@@ -33,9 +33,9 @@ sleep 1s
 ./src/common/thirdparty/redis/src/redis-cli set NumRedisShards 1
 ./src/common/thirdparty/redis/src/redis-cli rpush RedisShards 127.0.0.1:6380
 
-./src/plasma/plasma_store -s /tmp/plasma_store_socket_1 -m 100000000 &
+./src/plasma/plasma_store_server -s /tmp/plasma_store_socket_1 -m 100000000 &
 sleep 0.5s
 valgrind --track-origins=yes --leak-check=full --show-leak-kinds=all --leak-check-heuristics=stdstring --error-exitcode=1 ./src/local_scheduler/local_scheduler_tests
 ./src/common/thirdparty/redis/src/redis-cli shutdown
 ./src/common/thirdparty/redis/src/redis-cli -p 6380 shutdown
-killall plasma_store
+killall plasma_store_server

src/plasma/test/run_tests.sh

@@ -3,10 +3,10 @@
 # Cause the script to exit if a single command fails.
 set -e
 
-./src/plasma/plasma_store -s /tmp/plasma_store_socket_1 -m 0 &
+./src/plasma/plasma_store_server -s /tmp/plasma_store_socket_1 -m 0 &
 sleep 1
 ./src/plasma/manager_tests
-killall plasma_store
+killall plasma_store_server
 
 LaunchRedis() {
     port=$1
@@ -37,11 +37,11 @@ sleep 1s
 ./src/common/thirdparty/redis/src/redis-cli set NumRedisShards 1
 ./src/common/thirdparty/redis/src/redis-cli rpush RedisShards 127.0.0.1:6380
 sleep 1
-./src/plasma/plasma_store -s /tmp/store1 -m 1000000000 &
+./src/plasma/plasma_store_server -s /tmp/store1 -m 1000000000 &
 plasma1_pid=$!
 ./src/plasma/plasma_manager -m /tmp/manager1 -s /tmp/store1 -h 127.0.0.1 -p 11111 -r 127.0.0.1:6379 &
 plasma2_pid=$!
-./src/plasma/plasma_store -s /tmp/store2 -m 1000000000 &
+./src/plasma/plasma_store_server -s /tmp/store2 -m 1000000000 &
 plasma3_pid=$!
 ./src/plasma/plasma_manager -m /tmp/manager2 -s /tmp/store2 -h 127.0.0.1 -p 22222 -r 127.0.0.1:6379 &
 plasma4_pid=$!

src/plasma/test/run_valgrind.sh

@@ -5,7 +5,7 @@ set -x
 # Cause the script to exit if a single command fails.
 set -e
 
-./src/plasma/plasma_store -s /tmp/plasma_store_socket_1 -m 0 &
+./src/plasma/plasma_store_server -s /tmp/plasma_store_socket_1 -m 0 &
 sleep 1
 valgrind --track-origins=yes --leak-check=full --show-leak-kinds=all --leak-check-heuristics=stdstring --error-exitcode=1 ./src/plasma/manager_tests
-killall plasma_store
+killall plasma_store_server

src/ray/raylet/object_manager_integration_test.cc

@@ -83,7 +83,7 @@ class TestObjectManagerBase : public ::testing::Test {
     this->server1.reset();
     this->server2.reset();
 
-    int s = system("killall plasma_store &");
+    int s = system("killall plasma_store_server &");
     ASSERT_TRUE(!s);
 
     std::string cmd_str = test_executable.substr(0, test_executable.find_last_of("/"));

src/ray/test/run_object_manager_tests.sh

@@ -34,7 +34,7 @@ else
     LOAD_MODULE_ARGS="--loadmodule ${REDIS_MODULE}"
 fi
 
-STORE_EXEC="$CORE_DIR/src/plasma/plasma_store"
+STORE_EXEC="$CORE_DIR/src/plasma/plasma_store_server"
 
 # Allow cleanup commands to fail.
 $REDIS_DIR/redis-cli -p 6379 shutdown || true

src/ray/test/run_object_manager_valgrind.sh

@@ -23,7 +23,7 @@ fi
 CORE_DIR="$RAY_ROOT/build"
 REDIS_DIR="$CORE_DIR/src/common/thirdparty/redis/src"
 REDIS_MODULE="$CORE_DIR/src/common/redis_module/libray_redis_module.so"
-STORE_EXEC="$CORE_DIR/src/plasma/plasma_store"
+STORE_EXEC="$CORE_DIR/src/plasma/plasma_store_server"
 VALGRIND_CMD="valgrind --track-origins=yes --leak-check=full --show-leak-kinds=all --leak-check-heuristics=stdstring --error-exitcode=1"
 
 if [[ "${RAY_USE_NEW_GCS}" = "on" ]]; then

src/ray/test/start_raylet.sh

@@ -21,7 +21,7 @@ if [[ `stat $STORE_SOCKET_NAME` ]]; then
   rm $STORE_SOCKET_NAME
 fi
 
-./src/plasma/plasma_store -m 1000000000 -s $STORE_SOCKET_NAME &
+./src/plasma/plasma_store_server -m 1000000000 -s $STORE_SOCKET_NAME &
 ./src/ray/raylet/raylet $RAYLET_SOCKET_NAME $STORE_SOCKET_NAME 127.0.0.1 127.0.0.1 6379 &
 
 echo

src/ray/test/start_raylets.sh

@@ -43,7 +43,7 @@ for i in `seq 1 $NUM_RAYLETS`; do
     rm $STORE_SOCKET_NAME
   fi
 
-  ./src/plasma/plasma_store -m 1000000000 -s $STORE_SOCKET_NAME &
+  ./src/plasma/plasma_store_server -m 1000000000 -s $STORE_SOCKET_NAME &
   ./src/ray/raylet/raylet $RAYLET_SOCKET_NAME $STORE_SOCKET_NAME 127.0.0.1 127.0.0.1 6379 &
 
   echo

thirdparty/scripts/build_arrow.sh

@@ -34,10 +34,10 @@ else
   exit 1
 fi
 
-# The PR for this commit is https://github.com/apache/arrow/pull/2368. We
+# The PR for this commit is https://github.com/apache/arrow/pull/2458. We
 # include the link here to make it easier to find the right commit because
 # Arrow often rewrites git history and invalidates certain commits.
-TARGET_COMMIT_ID=4660833b2c5ef63a97445e304b8f72a2e0170f9c
+TARGET_COMMIT_ID=fda4b3dcfc773612b12973df5053193f236fc696
 build_arrow() {
   echo "building arrow"
   # Make sure arrow will be built again when building ray for java later than python