Only install ray python packages. (ray-project#330)

* Only install ray python packages.

* Add some __init__.py files.

* Install Ray before building documentation.

* Fix install-ray.sh.

* Fix.

.gitignore

@@ -1,5 +1,5 @@
 # The build output should clearly not be checked in
-/python/core
+/python/ray/core
 /src/common/thirdparty/redis
 /numbuf/thirdparty/arrow
 

.travis.yml

@@ -28,10 +28,10 @@ matrix:
       install: []
       script:
         - .travis/check-git-clang-format-output.sh
-        # Try generating Sphinx documentation. To do this, we need to setup some
-        # Python stuff.
-        - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh
-        - bash miniconda.sh -b -p $HOME/miniconda
+        # Try generating Sphinx documentation. To do this, we need to install
+        # Ray first.
+        - ./.travis/install-dependencies.sh
+        - ./.travis/install-ray.sh
         - export PATH="$HOME/miniconda/bin:$PATH"
         - cd doc
         - pip install -r requirements-doc.txt
@@ -46,39 +46,39 @@ matrix:
         - ./.travis/install-dependencies.sh
         - ./.travis/install-ray.sh
 
-        - cd python/core
-        - bash ../../src/common/test/run_valgrind.sh
-        - bash ../../src/plasma/test/run_valgrind.sh
-        - bash ../../src/local_scheduler/test/run_valgrind.sh
-        - cd ../..
+        - cd python/ray/core
+        - bash ../../../src/common/test/run_valgrind.sh
+        - bash ../../../src/plasma/test/run_valgrind.sh
+        - bash ../../../src/local_scheduler/test/run_valgrind.sh
+        - cd ../../..
 
       script:
-        - python ./python/plasma/test/test.py valgrind
-        - python ./python/local_scheduler/test/test.py valgrind
-        - python ./python/global_scheduler/test/test.py valgrind
+        - python ./python/ray/plasma/test/test.py valgrind
+        - python ./python/ray/local_scheduler/test/test.py valgrind
+        - python ./python/ray/global_scheduler/test/test.py valgrind
 
 install:
   - ./.travis/install-dependencies.sh
   - ./.travis/install-ray.sh
 
   - if [[ "$PYTHON" == "3.5" ]]; then export PATH="$HOME/miniconda/bin:$PATH"; fi
 
-  - cd python/core
-  - bash ../../src/common/test/run_tests.sh
-  - bash ../../src/plasma/test/run_tests.sh
-  - bash ../../src/local_scheduler/test/run_tests.sh
-  - cd ../..
+  - cd python/ray/core
+  - bash ../../../src/common/test/run_tests.sh
+  - bash ../../../src/plasma/test/run_tests.sh
+  - bash ../../../src/local_scheduler/test/run_tests.sh
+  - cd ../../..
 
 script:
   - if [[ "$PYTHON" == "3.5" ]]; then export PATH="$HOME/miniconda/bin:$PATH"; fi
 
   - python src/numbuf/python/test/runtest.py
 
-  - python python/common/test/test.py
-  - python python/common/redis_module/runtest.py
-  - python python/plasma/test/test.py
-  - python python/local_scheduler/test/test.py
-  - python python/global_scheduler/test/test.py
+  - python python/ray/common/test/test.py
+  - python python/ray/common/redis_module/runtest.py
+  - python python/ray/plasma/test/test.py
+  - python python/ray/local_scheduler/test/test.py
+  - python python/ray/global_scheduler/test/test.py
 
   - python test/runtest.py
   - python test/array_test.py

.travis/install-dependencies.sh

@@ -58,6 +58,14 @@ elif [[ "$PYTHON" == "3.5" ]] && [[ "$platform" == "macosx" ]]; then
   bash miniconda.sh -b -p $HOME/miniconda
   export PATH="$HOME/miniconda/bin:$PATH"
   pip install numpy cloudpickle funcsigs colorama psutil redis tensorflow
+elif [[ "$LINT" == "1" ]]; then
+  sudo apt-get update
+  sudo apt-get install -y cmake build-essential autoconf curl libtool libboost-all-dev unzip
+  # Install miniconda.
+  wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh
+  bash miniconda.sh -b -p $HOME/miniconda
+  export PATH="$HOME/miniconda/bin:$PATH"
+  pip install numpy cloudpickle funcsigs colorama psutil redis
 else
   echo "Unrecognized environment."
   exit 1

.travis/install-ray.sh

@@ -20,6 +20,13 @@ elif [[ "$PYTHON" == "3.5" ]]; then
     python setup.py install --user
   popd
 
+elif [[ "$LINT" == "1" ]]; then
+  export PATH="$HOME/miniconda/bin:$PATH"
+
+  pushd "$ROOT_DIR/../python"
+    python setup.py install --user
+  popd
+
 else
   echo "Unrecognized Python version."
   exit 1

build.sh

@@ -24,8 +24,8 @@ bash "$ROOT_DIR/src/numbuf/thirdparty/download_thirdparty.sh"
 bash "$ROOT_DIR/src/numbuf/thirdparty/build_thirdparty.sh"
 
 # Now build everything.
-pushd "$ROOT_DIR/python/core"
-  cmake -DCMAKE_BUILD_TYPE=Release ../..
+pushd "$ROOT_DIR/python/ray/core"
+  cmake -DCMAKE_BUILD_TYPE=Release ../../..
   make clean
   make
 popd

doc/README.md

@@ -1,6 +1,7 @@
 # Ray Documentation
 
 To compile the documentation, run the following commands from this directory.
+Note that Ray must be installed first.
 
 ```
 pip install -r requirements-doc.txt

doc/source/conf.py

@@ -16,12 +16,6 @@
 import os
 import shlex
 
-# These lines added to enable Sphinx to work without installing Ray.
-import mock
-MOCK_MODULES = ["global_scheduler", "numbuf", "local_scheduler", "plasma"]
-for mod_name in MOCK_MODULES:
-  sys.modules[mod_name] = mock.Mock()
-
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.

python/ray/actor.py

@@ -6,10 +6,10 @@
 import inspect
 import json
 import numpy as np
-import local_scheduler
 import random
 import traceback
 
+import ray.local_scheduler as local_scheduler
 import ray.pickling as pickling
 import ray.worker
 import ray.experimental.state as state

python/common/redis_module/runtest.py → python/ray/common/redis_module/runtest.py

@@ -11,15 +11,6 @@
 import redis
 import ray.services
 
-# Check if the redis-server binary is present.
-redis_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), "../../core/src/common/thirdparty/redis/src/redis-server")
-if not os.path.exists(redis_path):
-  raise Exception("You do not have the redis-server binary. Run `make test` in the plasma directory to get it.")
-
-# Absolute path of the ray redis module.
-module_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), "../../core/src/common/redis_module/libray_redis_module.so")
-print("path to the redis module is {}".format(module_path))
-
 OBJECT_INFO_PREFIX = "OI:"
 OBJECT_LOCATION_PREFIX = "OL:"
 OBJECT_SUBSCRIBE_PREFIX = "OS:"

python/common/test/test.py → python/ray/common/test/test.py

@@ -7,7 +7,7 @@
 import sys
 import unittest
 
-import local_scheduler
+import ray.local_scheduler as local_scheduler
 
 ID_SIZE = 20
 

python/global_scheduler/test/test.py → python/ray/global_scheduler/test/test.py

@@ -13,10 +13,10 @@
 import time
 import unittest
 
-import global_scheduler
-import local_scheduler
-import plasma
-from plasma.utils import random_object_id, generate_metadata, write_to_data_buffer, create_object_with_id, create_object
+import ray.global_scheduler as global_scheduler
+import ray.local_scheduler as local_scheduler
+import ray.plasma as plasma
+from ray.plasma.utils import random_object_id, generate_metadata, write_to_data_buffer, create_object_with_id, create_object
 
 from ray import services
 

python/local_scheduler/__init__.py → python/ray/local_scheduler/__init__.py

@@ -2,5 +2,5 @@
 from __future__ import division
 from __future__ import print_function
 
-from core.src.local_scheduler.liblocal_scheduler_library import *
+from ray.core.src.local_scheduler.liblocal_scheduler_library import *
 from .local_scheduler_services import *

python/local_scheduler/test/test.py → python/ray/local_scheduler/test/test.py

@@ -12,8 +12,8 @@
 import time
 import unittest
 
-import local_scheduler
-import plasma
+import ray.local_scheduler as local_scheduler
+import ray.plasma as plasma
 
 USE_VALGRIND = False
 ID_SIZE = 20

python/numbuf/__init__.py → python/ray/numbuf/__init__.py

@@ -11,7 +11,7 @@
 """
 
 try:
-  from core.src.numbuf.libnumbuf import *
+  from ray.core.src.numbuf.libnumbuf import *
 except ImportError as e:
   if hasattr(e, "msg") and isinstance(e.msg, str) and ("libstdc++" in e.msg or "CXX" in e.msg):
     # This code path should be taken with Python 3.

python/plasma/__init__.py → python/ray/plasma/__init__.py

@@ -2,4 +2,4 @@
 from __future__ import division
 from __future__ import print_function
 
-from plasma.plasma import *
+from ray.plasma.plasma import *

python/plasma/plasma.py → python/ray/plasma/plasma.py

@@ -8,9 +8,9 @@
 import sys
 import time
 
-import core.src.plasma.libplasma as libplasma
-from core.src.plasma.libplasma import plasma_object_exists_error
-from core.src.plasma.libplasma import plasma_out_of_memory_error
+import ray.core.src.plasma.libplasma as libplasma
+from ray.core.src.plasma.libplasma import plasma_object_exists_error
+from ray.core.src.plasma.libplasma import plasma_out_of_memory_error
 
 PLASMA_ID_SIZE = 20
 PLASMA_WAIT_TIMEOUT = 2 ** 30

python/plasma/test/test.py → python/ray/plasma/test/test.py

@@ -15,8 +15,8 @@
 import time
 import unittest
 
-import plasma
-from plasma.utils import random_object_id, generate_metadata, write_to_data_buffer, create_object_with_id, create_object
+import ray.plasma as plasma
+from ray.plasma.utils import random_object_id, generate_metadata, write_to_data_buffer, create_object_with_id, create_object
 from ray import services
 
 USE_VALGRIND = False

python/ray/serialization.py

@@ -3,8 +3,8 @@
 from __future__ import print_function
 
 import numpy as np
-import numbuf
 
+import ray.numbuf as numbuf
 import ray.pickling as pickling
 
 def check_serializable(cls):

python/ray/services.py

@@ -17,9 +17,9 @@
 import threading
 
 # Ray modules
-import local_scheduler
-import plasma
-import global_scheduler
+import ray.local_scheduler as local_scheduler
+import ray.plasma as plasma
+import ray.global_scheduler as global_scheduler
 
 PROCESS_TYPE_WORKER = "worker"
 PROCESS_TYPE_LOCAL_SCHEDULER = "local_scheduler"
@@ -217,8 +217,8 @@ def start_redis(port=None, num_retries=20, stdout_file=None, stderr_file=None,
   Raises:
     Exception: An exception is raised if Redis could not be started.
   """
-  redis_filepath = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../core/src/common/thirdparty/redis/src/redis-server")
-  redis_module = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../core/src/common/redis_module/libray_redis_module.so")
+  redis_filepath = os.path.join(os.path.dirname(os.path.abspath(__file__)), "./core/src/common/thirdparty/redis/src/redis-server")
+  redis_module = os.path.join(os.path.dirname(os.path.abspath(__file__)), "./core/src/common/redis_module/libray_redis_module.so")
   assert os.path.isfile(redis_filepath)
   assert os.path.isfile(redis_module)
   counter = 0

python/ray/worker.py

@@ -25,9 +25,9 @@
 import ray.pickling as pickling
 import ray.serialization as serialization
 import ray.services as services
-import numbuf
-import local_scheduler
-import plasma
+import ray.numbuf as numbuf
+import ray.local_scheduler as local_scheduler
+import ray.plasma as plasma
 
 SCRIPT_MODE = 0
 WORKER_MODE = 1

python/ray/workers/default_worker.py

@@ -37,7 +37,7 @@ def random_string():
   error_explanation = """
 This error is unexpected and should not have happened. Somehow a worker crashed
 in an unanticipated way causing the main_loop to throw an exception, which is
-being caught in "lib/python/ray/workers/default_worker.py".
+being caught in "python/ray/workers/default_worker.py".
 """
 
   while True:

python/setup.py

@@ -19,16 +19,16 @@ def run(self):
 setup(name="ray",
       version="0.0.1",
       packages=find_packages(),
-      package_data={"core": ["src/common/thirdparty/redis/src/redis-server",
-                             "src/common/redis_module/libray_redis_module.so",
-                             "src/plasma/plasma_store",
-                             "src/plasma/plasma_manager",
-                             "src/plasma/libplasma.so",
-                             "src/local_scheduler/local_scheduler",
-                             "src/local_scheduler/liblocal_scheduler_library.so",
-                             "src/numbuf/libarrow.so",
-                             "src/numbuf/libnumbuf.so",
-                             "src/global_scheduler/global_scheduler"]},
+      package_data={"ray": ["core/src/common/thirdparty/redis/src/redis-server",
+                            "core/src/common/redis_module/libray_redis_module.so",
+                            "core/src/plasma/plasma_store",
+                            "core/src/plasma/plasma_manager",
+                            "core/src/plasma/libplasma.so",
+                            "core/src/local_scheduler/local_scheduler",
+                            "core/src/local_scheduler/liblocal_scheduler_library.so",
+                            "core/src/numbuf/libarrow.so",
+                            "core/src/numbuf/libnumbuf.so",
+                            "core/src/global_scheduler/global_scheduler"]},
       cmdclass={"install": install},
       install_requires=["numpy",
                         "funcsigs",

src/local_scheduler/test/local_scheduler_tests.cc

@@ -74,7 +74,7 @@ LocalSchedulerMock *LocalSchedulerMock_init(int num_workers,
   UT_string *worker_command;
   utstring_new(worker_command);
   utstring_printf(worker_command,
-                  "python ../../python/ray/workers/default_worker.py "
+                  "python ../../../python/ray/workers/default_worker.py "
                   "--node-ip-address=%s --object-store-name=%s "
                   "--object-store-manager-name=%s --local-scheduler-name=%s "
                   "--redis-address=%s:%d",

src/numbuf/python/test/runtest.py

@@ -3,7 +3,7 @@
 from __future__ import print_function
 
 import unittest
-import numbuf
+import ray.numbuf as numbuf
 import numpy as np
 from numpy.testing import assert_equal
 import os