Remove legacy Ray code. (ray-project#3121)

* Remove legacy Ray code. * Fix cmake and simplify monitor. * Fix linting * Updates * Fix * Implement some methods. * Remove more plasma manager references. * Fix * Linting * Fix * Fix * Make sure class IDs are strings. * Some path fixes * Fix * Path fixes and update arrow * Fixes. * linting * Fixes * Java fixes * Some java fixes * TaskLanguage -> Language * Minor * Fix python test and remove unused method signature. * Fix java tests * Fix jenkins tests * Remove commented out code.
jbn · Oct 26, 2018 · 658c142 · 658c142
1 parent 055daf1
commit 658c142
Show file tree

Hide file tree

Showing 289 changed files with 2,471 additions and 40,719 deletions.
diff --git a/.clang-format b/.clang-format
@@ -1,6 +1,5 @@
-BasedOnStyle: Chromium
-ColumnLimit: 80
+BasedOnStyle: Google
+ColumnLimit: 90
 DerivePointerAlignment: false
 IndentCaseLabels: false
 PointerAlignment: Right
-SpaceAfterCStyleCast: true
diff --git a/.gitignore b/.gitignore
@@ -4,22 +4,10 @@
 /python/build
 /python/dist
 /python/flatbuffers-1.7.1/
-/src/common/thirdparty/redis
-/src/thirdparty/arrow
 /flatbuffers-1.7.1/
-/src/thirdparty/boost/
-/src/thirdparty/boost_1_65_1/
-/src/thirdparty/boost_1_60_0/
-/src/thirdparty/catapult/
-/src/thirdparty/flatbuffers/
-/src/thirdparty/parquet-cpp
 /thirdparty/pkg/
 
 # Files generated by flatc should be ignored
-/src/common/format/*.py
-/src/common/format/*_generated.h
-/src/plasma/format/
-/src/local_scheduler/format/*_generated.h
 /src/ray/gcs/format/*_generated.h
 /src/ray/object_manager/format/*_generated.h
 /src/ray/raylet/format/*_generated.h
@@ -54,9 +42,6 @@ python/.eggs
 *.dylib
 *.dll
 
-# Cython-generated files
-*.c
-
 # Incremental linking files
 *.ilk
 

diff --git a/.travis.yml b/.travis.yml
@@ -53,7 +53,7 @@ matrix:
         - sphinx-build -W -b html -d _build/doctrees source _build/html
         - cd ..
         # Run Python linting, ignore dict vs {} (C408), others are defaults
-        - flake8 --exclude=python/ray/core/src/common/flatbuffers_ep-prefix/,python/ray/core/generated/,src/common/format/,doc/source/conf.py,python/ray/cloudpickle/ --ignore=C408,E121,E123,E126,E226,E24,E704,W503,W504,W605
+        - flake8 --exclude=python/ray/core/generated/,doc/source/conf.py,python/ray/cloudpickle/ --ignore=C408,E121,E123,E126,E226,E24,E704,W503,W504,W605
         - .travis/format.sh --all
 
     - os: linux
@@ -69,16 +69,9 @@ matrix:
 
       script:
         - cd build
-        # - bash ../src/common/test/run_valgrind.sh
-        # - bash ../src/plasma/test/run_valgrind.sh
-        # - bash ../src/local_scheduler/test/run_valgrind.sh
         - bash ../src/ray/test/run_object_manager_valgrind.sh
         - cd ..
 
-        # - python ./python/ray/plasma/test/test.py valgrind
-        # - python ./python/ray/local_scheduler/test/test.py valgrind
-        # - python ./python/ray/global_scheduler/test/test.py valgrind
-
     # Build Linux wheels.
     - os: linux
       dist: trusty
@@ -108,75 +101,6 @@ matrix:
         - PYTHON=3.5
         - RAY_USE_NEW_GCS=on
 
-    # Test legacy Ray.
-    - os: linux
-      dist: trusty
-      env: PYTHON=3.5 RAY_USE_XRAY=0
-      install:
-        - ./.travis/install-dependencies.sh
-        - export PATH="$HOME/miniconda/bin:$PATH"
-        - ./.travis/install-ray.sh
-        - ./.travis/install-cython-examples.sh
-
-        - cd build
-        - bash ../src/common/test/run_tests.sh
-        - bash ../src/plasma/test/run_tests.sh
-        - bash ../src/local_scheduler/test/run_tests.sh
-        - cd ..
-
-      script:
-        - export PATH="$HOME/miniconda/bin:$PATH"
-        # The following is needed so cloudpickle can find some of the
-        # class definitions: The main module of tests that are run
-        # with pytest have the same name as the test file -- and this
-        # module is only found if the test directory is in the PYTHONPATH.
-        - export PYTHONPATH="$PYTHONPATH:./test/"
-
-        - python -m pytest -v python/ray/common/test/test.py
-        - python -m pytest -v python/ray/common/redis_module/runtest.py
-        - python -m pytest -v python/ray/plasma/test/test.py
-        - python -m pytest -v python/ray/local_scheduler/test/test.py
-        - python -m pytest -v python/ray/global_scheduler/test/test.py
-
-        - python -m pytest -v python/ray/test/test_global_state.py
-        - python -m pytest -v python/ray/test/test_queue.py
-        - python -m pytest -v python/ray/test/test_ray_init.py
-        - python -m pytest -v test/xray_test.py
-
-        - python -m pytest -v test/runtest.py
-        - python -m pytest -v test/array_test.py
-        - python -m pytest -v test/actor_test.py
-        - python -m pytest -v test/autoscaler_test.py
-        - python -m pytest -v test/tensorflow_test.py
-        - python -m pytest -v test/failure_test.py
-        - python -m pytest -v test/microbenchmarks.py
-        - python -m pytest -v test/stress_tests.py
-        - pytest test/component_failures_test.py
-        - python test/multi_node_test.py
-        - python -m pytest -v test/multi_node_test_2.py
-        - python -m pytest -v test/recursion_test.py
-        - pytest test/monitor_test.py
-        - python -m pytest -v test/cython_test.py
-        - python -m pytest -v test/credis_test.py
-
-        # ray tune tests
-        - python python/ray/tune/test/dependency_test.py
-        - python -m pytest -v python/ray/tune/test/trial_runner_test.py
-        - python -m pytest -v python/ray/tune/test/trial_scheduler_test.py
-        - python -m pytest -v python/ray/tune/test/experiment_test.py
-        - python -m pytest -v python/ray/tune/test/tune_server_test.py
-        - python -m pytest -v python/ray/tune/test/ray_trial_executor_test.py
-        - python -m pytest -v python/ray/tune/test/automl_searcher_test.py
-
-        # ray rllib tests
-        - python -m pytest -v python/ray/rllib/test/test_catalog.py
-        - python -m pytest -v python/ray/rllib/test/test_filters.py
-        - python -m pytest -v python/ray/rllib/test/test_optimizers.py
-        - python -m pytest -v python/ray/rllib/test/test_evaluators.py
-
-        # ray temp file tests
-        - python -m pytest -v test/tempfile_test.py
-
 
 install:
   - ./.travis/install-dependencies.sh
@@ -207,12 +131,6 @@ script:
   # module is only found if the test directory is in the PYTHONPATH.
   - export PYTHONPATH="$PYTHONPATH:./test/"
 
-  # - python -m pytest -v python/ray/common/test/test.py
-  # - python -m pytest -v python/ray/common/redis_module/runtest.py
-  # - python -m pytest -v python/ray/plasma/test/test.py
-  # - python -m pytest -v python/ray/local_scheduler/test/test.py
-  # - python -m pytest -v python/ray/global_scheduler/test/test.py
-
   - python -m pytest -v python/ray/test/test_global_state.py
   - python -m pytest -v python/ray/test/test_queue.py
   - python -m pytest -v python/ray/test/test_ray_init.py
@@ -227,7 +145,7 @@ script:
   - python -m pytest -v test/microbenchmarks.py
   - python -m pytest -v test/stress_tests.py
   - python -m pytest -v test/component_failures_test.py
-  - python test/multi_node_test.py
+  - python -m pytest -v test/multi_node_test.py
   - python -m pytest -v test/multi_node_test_2.py
   - python -m pytest -v test/recursion_test.py
   - python -m pytest -v test/monitor_test.py

diff --git a/.travis/format.sh b/.travis/format.sh
@@ -30,7 +30,6 @@ YAPF_EXCLUDES=(
     '--exclude' 'python/build/*'
     '--exclude' 'python/ray/pyarrow_files/*'
     '--exclude' 'python/ray/core/src/ray/gcs/*'
-    '--exclude' 'python/ray/common/thirdparty/*'
 )
 
 # Format specified files

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -82,18 +82,15 @@ include_directories(SYSTEM ${PLASMA_INCLUDE_DIR})
 include_directories("${CMAKE_CURRENT_LIST_DIR}/src/")
 
 add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/src/ray/)
-add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/src/common/)
-add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/src/plasma/)
-add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/src/local_scheduler/)
-add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/src/global_scheduler/)
 
 # final target copy_ray
 add_custom_target(copy_ray ALL)
 
 # copy plasma_store_server
 add_custom_command(TARGET copy_ray POST_BUILD
+  COMMAND mkdir -p ${CMAKE_CURRENT_BINARY_DIR}/src/plasma
   COMMAND ${CMAKE_COMMAND} -E
-    copy ${ARROW_HOME}/bin/plasma_store_server ${CMAKE_CURRENT_BINARY_DIR}/src/plasma)
+    copy ${ARROW_HOME}/bin/plasma_store_server ${CMAKE_CURRENT_BINARY_DIR}/src/plasma/)
 
 if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES")
   # add pyarrow as the dependency
@@ -102,12 +99,9 @@ if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES")
   # NOTE: The lists below must be kept in sync with ray/python/setup.py.
 
   set(ray_file_list
-        "src/common/thirdparty/redis/src/redis-server"
-        "src/common/redis_module/libray_redis_module.so"
-        "src/plasma/plasma_manager"
-        "src/local_scheduler/local_scheduler"
-        "src/local_scheduler/liblocal_scheduler_library_python.so"
-        "src/global_scheduler/global_scheduler"
+        "src/ray/thirdparty/redis/src/redis-server"
+        "src/ray/gcs/redis_module/libray_redis_module.so"
+        "src/ray/raylet/liblocal_scheduler_library_python.so"
         "src/ray/raylet/raylet_monitor"
         "src/ray/raylet/raylet")
 
@@ -154,5 +148,6 @@ if ("${CMAKE_RAY_LANG_JAVA}" STREQUAL "YES")
 
   # copy libplasma_java files
   add_custom_command(TARGET copy_ray POST_BUILD
-    COMMAND bash -c "cp ${ARROW_LIBRARY_DIR}/libplasma_java.* ${CMAKE_CURRENT_BINARY_DIR}/src/plasma")
+    COMMAND bash -c "mkdir -p ${CMAKE_CURRENT_BINARY_DIR}/src/plasma"
+    COMMAND bash -c "cp ${ARROW_LIBRARY_DIR}/libplasma_java.* ${CMAKE_CURRENT_BINARY_DIR}/src/plasma/")
 endif()
diff --git a/cmake/Modules/ArrowExternalProject.cmake b/cmake/Modules/ArrowExternalProject.cmake
@@ -15,10 +15,10 @@
 #  - PLASMA_SHARED_LIB
 
 set(arrow_URL https://github.com/apache/arrow.git)
-# The PR for this commit is https://github.com/apache/arrow/pull/2792. We
+# The PR for this commit is https://github.com/apache/arrow/pull/2826. We
 # include the link here to make it easier to find the right commit because
 # Arrow often rewrites git history and invalidates certain commits.
-set(arrow_TAG 2d0d3d0dc51999fbaafb15d8b8362a1ef3de2ef7)
+set(arrow_TAG b4f7ed6d6ed5cdb6dd136bac3181a438f35c8ea0)
 
 set(ARROW_INSTALL_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/external/arrow-install)
 set(ARROW_HOME ${ARROW_INSTALL_PREFIX})

diff --git a/cmake/Modules/Common.cmake b/cmake/Modules/Common.cmake
@@ -41,6 +41,3 @@ if ("${CMAKE_RAY_LANG_JAVA}" STREQUAL "YES")
     message (WARNING "NOT FIND JNI")
   endif()
 endif()
-
-include_directories(${CMAKE_SOURCE_DIR}/src/common)
-include_directories(${CMAKE_SOURCE_DIR}/src/common/thirdparty)
diff --git a/doc/source/actors.rst b/doc/source/actors.rst
@@ -65,8 +65,7 @@ When ``a1.increment.remote()`` is called, the following events happens.
 
 1. A task is created.
 2. The task is assigned directly to the local scheduler responsible for the
-   actor by the driver's local scheduler. Thus, this scheduling procedure
-   bypasses the global scheduler.
+   actor by the driver's local scheduler.
 3. An object ID is returned.
 
 We can then call ``ray.get`` on the object ID to retrieve the actual value.