Pull Plasma from Apache Arrow and remove Plasma store from Ray. (ray-…

…project#692)

* Rebase Ray on top of Plasma in Apache Arrow

* add thirdparty building scripts

* use rebased arrow

* fix

* fix build

* fix python visibility

* comment out C tests for now

* fix multithreading

* fix

* reduce logging

* fix plasma manager multithreading

* make sure old and new object IDs can coexist peacefully

* more rebasing

* update

* fixes

* fix

* install pyarrow

* install cython

* fix

* install newer cmake

* fix

* rebase on top of latest arrow

* getting runtest.py run locally (needed to comment out a test for that to work)

* work on plasma tests

* more fixes

* fix local scheduler tests

* fix global scheduler test

* more fixes

* fix python 3 bytes vs string

* fix manager tests valgrind

* fix documentation building

* fix linting

* fix c++ linting

* fix linting

* add tests back in

* Install without sudo.

* Set PKG_CONFIG_PATH in build.sh so that Ray can find plasma.

* Install pkg-config

* Link -lpthread, note that find_package(Threads) doesn't seem to work reliably.

* Comment in testGPUIDs in runtest.py.

* Set PKG_CONFIG_PATH when building pyarrow.

* Pull apache/arrow and not pcmoritz/arrow.

* Fix installation in docker image.

* adapt to changes of the plasma api

* Fix installation of pyarrow module.

* Fix linting.

* Use correct python executable to build pyarrow.

.travis/install-dependencies.sh

@@ -19,20 +19,20 @@ fi
 
 if [[ "$PYTHON" == "2.7" ]] && [[ "$platform" == "linux" ]]; then
   sudo apt-get update
-  sudo apt-get install -y cmake build-essential autoconf curl libtool python-dev python-numpy python-pip libboost-all-dev unzip
+  sudo apt-get install -y cmake pkg-config build-essential autoconf curl libtool python-dev python-numpy python-pip libboost-all-dev unzip
   # Install miniconda.
   wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -O miniconda.sh
   bash miniconda.sh -b -p $HOME/miniconda
   export PATH="$HOME/miniconda/bin:$PATH"
-  pip install numpy cloudpickle funcsigs click colorama psutil redis tensorflow flatbuffers
+  pip install numpy cloudpickle cython cmake funcsigs click colorama psutil redis tensorflow flatbuffers
 elif [[ "$PYTHON" == "3.5" ]] && [[ "$platform" == "linux" ]]; then
   sudo apt-get update
-  sudo apt-get install -y cmake python-dev python-numpy build-essential autoconf curl libtool libboost-all-dev unzip
+  sudo apt-get install -y cmake pkg-config python-dev python-numpy build-essential autoconf curl libtool libboost-all-dev unzip
   # Install miniconda.
   wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh
   bash miniconda.sh -b -p $HOME/miniconda
   export PATH="$HOME/miniconda/bin:$PATH"
-  pip install numpy cloudpickle funcsigs click colorama psutil redis tensorflow flatbuffers
+  pip install numpy cloudpickle cython cmake funcsigs click colorama psutil redis tensorflow flatbuffers
 elif [[ "$PYTHON" == "2.7" ]] && [[ "$platform" == "macosx" ]]; then
   # check that brew is installed
   which -s brew
@@ -43,12 +43,12 @@ elif [[ "$PYTHON" == "2.7" ]] && [[ "$platform" == "macosx" ]]; then
     echo "Updating brew."
     brew update
   fi
-  brew install cmake automake autoconf libtool boost
+  brew install cmake pkg-config automake autoconf libtool boost
   # Install miniconda.
   wget https://repo.continuum.io/miniconda/Miniconda2-latest-MacOSX-x86_64.sh -O miniconda.sh
   bash miniconda.sh -b -p $HOME/miniconda
   export PATH="$HOME/miniconda/bin:$PATH"
-  pip install numpy cloudpickle funcsigs click colorama psutil redis tensorflow flatbuffers
+  pip install numpy cloudpickle cython cmake funcsigs click colorama psutil redis tensorflow flatbuffers
 elif [[ "$PYTHON" == "3.5" ]] && [[ "$platform" == "macosx" ]]; then
   # check that brew is installed
   which -s brew
@@ -59,12 +59,12 @@ elif [[ "$PYTHON" == "3.5" ]] && [[ "$platform" == "macosx" ]]; then
     echo "Updating brew."
     brew update
   fi
-  brew install cmake automake autoconf libtool boost
+  brew install cmake pkg-config automake autoconf libtool boost
   # Install miniconda.
   wget https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -O miniconda.sh
   bash miniconda.sh -b -p $HOME/miniconda
   export PATH="$HOME/miniconda/bin:$PATH"
-  pip install numpy cloudpickle funcsigs click colorama psutil redis tensorflow flatbuffers
+  pip install numpy cloudpickle cython cmake funcsigs click colorama psutil redis tensorflow flatbuffers
 elif [[ "$LINT" == "1" ]]; then
   sudo apt-get update
   sudo apt-get install -y cmake build-essential autoconf curl libtool libboost-all-dev unzip

CMakeLists.txt

@@ -2,6 +2,11 @@ cmake_minimum_required(VERSION 2.8)
 
 project(ray)
 
+set(ARROW_DIR "${CMAKE_CURRENT_LIST_DIR}/src/thirdparty/arrow/"
+    CACHE STRING "Path of the arrow source directory")
+
+include_directories("${ARROW_DIR}/cpp/src/")
+
 add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/src/common/)
 add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/src/plasma/)
 add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/src/local_scheduler/)

build.sh

@@ -1,10 +1,19 @@
 #!/usr/bin/env bash
 
+set -x
+
 # Cause the script to exit if a single command fails.
 set -e
 
 ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
 
+if [[ -z  "$1" ]]; then
+  PYTHON_EXECUTABLE=`which python`
+else
+  PYTHON_EXECUTABLE=$1
+fi
+echo "Using Python executable $PYTHON_EXECUTABLE."
+
 # Determine how many parallel jobs to use for make based on the number of cores
 unamestr="$(uname)"
 if [[ "$unamestr" == "Linux" ]]; then
@@ -20,17 +29,25 @@ pushd "$ROOT_DIR/src/common/thirdparty/"
   bash build-redis.sh
 popd
 
-bash "$ROOT_DIR/src/numbuf/thirdparty/download_thirdparty.sh"
-bash "$ROOT_DIR/src/numbuf/thirdparty/build_thirdparty.sh"
+bash "$ROOT_DIR/src/thirdparty/download_thirdparty.sh"
+bash "$ROOT_DIR/src/thirdparty/build_thirdparty.sh" $PYTHON_EXECUTABLE
 
 # Now build everything.
 pushd "$ROOT_DIR/python/ray/core"
+  # We use these variables to set PKG_CONFIG_PATH, which is important so that
+  # in cmake, pkg-config can find plasma.
+  TP_DIR=$ROOT_DIR/src/thirdparty
+  ARROW_HOME=$TP_DIR/arrow/cpp/build/cpp-install
   if [ "$VALGRIND" = "1" ]
   then
-    cmake -DCMAKE_BUILD_TYPE=Debug ../../..
+    PKG_CONFIG_PATH=$ARROW_HOME/lib/pkgconfig cmake -DCMAKE_BUILD_TYPE=Debug ../../..
   else
-    cmake -DCMAKE_BUILD_TYPE=Release ../../..
+    PKG_CONFIG_PATH=$ARROW_HOME/lib/pkgconfig cmake -DCMAKE_BUILD_TYPE=Release ../../..
   fi
   make clean
   make -j${PARALLEL}
 popd
+
+# Move stuff from Arrow to Ray.
+
+mv $ROOT_DIR/src/thirdparty/arrow/cpp/build/release/plasma_store $ROOT_DIR/python/ray/core/src/plasma/

doc/source/conf.py

@@ -18,7 +18,9 @@
 
 # These lines added to enable Sphinx to work without installing Ray.
 import mock
-MOCK_MODULES = ["ray.numbuf",
+MOCK_MODULES = ["pyarrow",
+                "pyarrow.plasma",
+                "ray.numbuf",
                 "ray.local_scheduler",
                 "ray.plasma",
                 "ray.core.generated.TaskInfo",

doc/source/install-on-macosx.rst

@@ -26,7 +26,7 @@ To build Ray, first install the following dependencies. We recommend using
 .. code-block:: bash
 
   brew update
-  brew install cmake automake autoconf libtool boost wget
+  brew install cmake pkg-config automake autoconf libtool boost wget
 
   pip install numpy cloudpickle funcsigs click colorama psutil redis flatbuffers --ignore-installed six
 

doc/source/install-on-ubuntu.rst

@@ -26,7 +26,7 @@ To build Ray, first install the following dependencies. We recommend using
 .. code-block:: bash
 
   sudo apt-get update
-  sudo apt-get install -y cmake build-essential autoconf curl libtool libboost-all-dev unzip
+  sudo apt-get install -y cmake pkg-config build-essential autoconf curl libtool libboost-all-dev unzip
 
   # If you are not using Anaconda, you need the following.
   sudo apt-get install python-dev  # For Python 2.

docker/base-deps/Dockerfile

@@ -4,7 +4,7 @@
 FROM ubuntu:xenial
 RUN apt-get update \
     && apt-get install -y vim git wget \
-    && apt-get install -y cmake build-essential autoconf curl libtool libboost-all-dev unzip
+    && apt-get install -y cmake pkg-config build-essential autoconf curl libtool libboost-all-dev unzip
 RUN echo 'export PATH=/opt/conda/bin:$PATH' > /etc/profile.d/conda.sh \
     && wget --quiet 'https://repo.continuum.io/archive/Anaconda2-4.2.0-Linux-x86_64.sh' -O /tmp/anaconda.sh \
     && /bin/bash /tmp/anaconda.sh -b -p /opt/conda \

python/ray/experimental/state.py

@@ -465,7 +465,7 @@ def task_profiles(self, start=None, end=None, num_tasks=None, fwd=True):
                         **params)
 
             for (event, score) in event_list:
-                event_dict = json.loads(event)
+                event_dict = json.loads(event.decode())
                 task_id = ""
                 for event in event_dict:
                     if "task_id" in event[3]:

python/ray/global_scheduler/test/test.py

@@ -10,6 +10,7 @@
 import time
 import unittest
 
+import pyarrow as pa
 import ray.global_scheduler as global_scheduler
 import ray.local_scheduler as local_scheduler
 import ray.plasma as plasma
@@ -87,8 +88,8 @@ def setUp(self):
             self.plasma_manager_pids.append(p3)
             plasma_address = "{}:{}".format(self.node_ip_address,
                                             plasma_manager_port)
-            plasma_client = plasma.PlasmaClient(plasma_store_name,
-                                                plasma_manager_name)
+            plasma_client = pa.plasma.connect(plasma_store_name,
+                                              plasma_manager_name, 64)
             self.plasma_clients.append(plasma_client)
             # Start the local scheduler.
             local_scheduler_name, p4 = local_scheduler.start_local_scheduler(
@@ -203,9 +204,10 @@ def test_integration_single_task(self):
         # Sleep before submitting task to local scheduler.
         time.sleep(0.1)
         # Submit a task to Redis.
-        task = local_scheduler.Task(random_driver_id(), random_function_id(),
-                                    [local_scheduler.ObjectID(object_dep)],
-                                    num_return_vals[0], random_task_id(), 0)
+        task = local_scheduler.Task(
+            random_driver_id(), random_function_id(),
+            [local_scheduler.ObjectID(object_dep.binary())],
+            num_return_vals[0], random_task_id(), 0)
         self.local_scheduler_clients[0].submit(task)
         time.sleep(0.1)
         # There should now be a task in Redis, and it should get assigned to
@@ -256,11 +258,11 @@ def integration_many_tasks_helper(self, timesync=True):
                 # Give 10ms for object info handler to fire (long enough to
                 # yield CPU).
                 time.sleep(0.010)
-            task = local_scheduler.Task(random_driver_id(),
-                                        random_function_id(),
-                                        [local_scheduler.ObjectID(object_dep)],
-                                        num_return_vals[0], random_task_id(),
-                                        0)
+            task = local_scheduler.Task(
+                random_driver_id(),
+                random_function_id(),
+                [local_scheduler.ObjectID(object_dep.binary())],
+                num_return_vals[0], random_task_id(), 0)
             self.local_scheduler_clients[0].submit(task)
         # Check that there are the correct number of tasks in Redis and that
         # they all get assigned to the local scheduler.

python/ray/local_scheduler/test/test.py

@@ -12,6 +12,7 @@
 
 import ray.local_scheduler as local_scheduler
 import ray.plasma as plasma
+import pyarrow as pa
 
 USE_VALGRIND = False
 ID_SIZE = 20
@@ -41,8 +42,7 @@ class TestLocalSchedulerClient(unittest.TestCase):
     def setUp(self):
         # Start Plasma store.
         plasma_store_name, self.p1 = plasma.start_plasma_store()
-        self.plasma_client = plasma.PlasmaClient(plasma_store_name,
-                                                 release_delay=0)
+        self.plasma_client = pa.plasma.connect(plasma_store_name, "", 0)
         # Start a local scheduler.
         scheduler_name, self.p2 = local_scheduler.start_local_scheduler(
             plasma_store_name, use_valgrind=USE_VALGRIND)
@@ -72,8 +72,8 @@ def test_submit_and_get_task(self):
         # Create and seal the objects in the object store so that we can
         # schedule all of the subsequent tasks.
         for object_id in object_ids:
-            self.plasma_client.create(object_id.id(), 0)
-            self.plasma_client.seal(object_id.id())
+            self.plasma_client.create(pa.plasma.ObjectID(object_id.id()), 0)
+            self.plasma_client.seal(pa.plasma.ObjectID(object_id.id()))
         # Define some arguments to use for the tasks.
         args_list = [
             [],
@@ -153,8 +153,8 @@ def get_task():
         time.sleep(0.1)
         # Create and seal the object ID in the object store. This should
         # trigger a scheduling event.
-        self.plasma_client.create(object_id.id(), 0)
-        self.plasma_client.seal(object_id.id())
+        self.plasma_client.create(pa.plasma.ObjectID(object_id.id()), 0)
+        self.plasma_client.seal(pa.plasma.ObjectID(object_id.id()))
         # Wait until the thread finishes so that we know the task was
         # scheduled.
         t.join()
@@ -175,8 +175,8 @@ def get_task():
         t.start()
 
         # Make one of the dependencies available.
-        buf = self.plasma_client.create(object_id1.id(), 1)
-        self.plasma_client.seal(object_id1.id())
+        buf = self.plasma_client.create(pa.plasma.ObjectID(object_id1.id()), 1)
+        self.plasma_client.seal(pa.plasma.ObjectID(object_id1.id()))
         # Release the object.
         del buf
         # Check that the thread is still waiting for a task.
@@ -188,23 +188,24 @@ def get_task():
         time.sleep(0.1)
         self.assertTrue(t.is_alive())
         # Check that the first object dependency was evicted.
-        object1 = self.plasma_client.get([object_id1.id()], timeout_ms=0)
+        object1 = self.plasma_client.get([pa.plasma.ObjectID(object_id1.id())],
+                                         timeout_ms=0)
         self.assertEqual(object1, [None])
         # Check that the thread is still waiting for a task.
         time.sleep(0.1)
         self.assertTrue(t.is_alive())
 
         # Create the second dependency.
-        self.plasma_client.create(object_id2.id(), 1)
-        self.plasma_client.seal(object_id2.id())
+        self.plasma_client.create(pa.plasma.ObjectID(object_id2.id()), 1)
+        self.plasma_client.seal(pa.plasma.ObjectID(object_id2.id()))
         # Check that the thread is still waiting for a task.
         time.sleep(0.1)
         self.assertTrue(t.is_alive())
 
         # Create the first dependency again. Both dependencies are now
         # available.
-        self.plasma_client.create(object_id1.id(), 1)
-        self.plasma_client.seal(object_id1.id())
+        self.plasma_client.create(pa.plasma.ObjectID(object_id1.id()), 1)
+        self.plasma_client.seal(pa.plasma.ObjectID(object_id1.id()))
 
         # Wait until the thread finishes so that we know the task was
         # scheduled.

python/ray/plasma/__init__.py

@@ -2,13 +2,8 @@
 from __future__ import division
 from __future__ import print_function
 
-from ray.plasma.plasma import (PlasmaBuffer, buffers_equal, PlasmaClient,
-                               start_plasma_store, start_plasma_manager,
-                               plasma_object_exists_error,
-                               plasma_out_of_memory_error,
+from ray.plasma.plasma import (start_plasma_store, start_plasma_manager,
                                DEFAULT_PLASMA_STORE_MEMORY)
 
-__all__ = ["PlasmaBuffer", "buffers_equal", "PlasmaClient",
-           "start_plasma_store", "start_plasma_manager",
-           "plasma_object_exists_error", "plasma_out_of_memory_error",
+__all__ = ["start_plasma_store", "start_plasma_manager",
            "DEFAULT_PLASMA_STORE_MEMORY"]