Fix TensorFlow and PyTorch compatibility (#3574)

* remove tensorflow workaround
* update docker
* add boost threads
* add date_time, too
* change link order
* cosmetics

cmake/Modules/ArrowExternalProject.cmake

@@ -14,11 +14,15 @@
 #  - PLASMA_STATIC_LIB
 #  - PLASMA_SHARED_LIB
 
-set(arrow_URL https://github.com/apache/arrow.git)
-# The PR for this commit is https://github.com/apache/arrow/pull/3154. We
+set(arrow_URL https://github.com/ray-project/arrow.git)
+# This commit is based on https://github.com/apache/arrow/pull/3197. We
 # include the link here to make it easier to find the right commit because
 # Arrow often rewrites git history and invalidates certain commits.
-set(arrow_TAG 8c413036775796d9bcc52be56373bbb45de8c0ae)
+# It has been patched to fix an upstream symbol clash with TensorFlow,
+# the patch is available at
+# https://github.com/ray-project/arrow/commit/c347cd571e51723fc8512922f1b3a8e45e45b169
+# See the discussion in https://github.com/apache/arrow/pull/3177
+set(arrow_TAG c347cd571e51723fc8512922f1b3a8e45e45b169)
 
 set(ARROW_INSTALL_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/external/arrow-install)
 set(ARROW_HOME ${ARROW_INSTALL_PREFIX})

cmake/Modules/BoostExternalProject.cmake

@@ -27,14 +27,15 @@ else()
   set(BOOST_ROOT ${Boost_INSTALL_PREFIX})
   set(Boost_LIBRARY_DIR ${Boost_INSTALL_PREFIX}/lib)
   set(Boost_SYSTEM_LIBRARY ${Boost_LIBRARY_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_system${CMAKE_STATIC_LIBRARY_SUFFIX})
+  set(Boost_THREAD_LIBRARY ${Boost_LIBRARY_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_thread${CMAKE_STATIC_LIBRARY_SUFFIX})
   set(Boost_FILESYSTEM_LIBRARY ${Boost_LIBRARY_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_filesystem${CMAKE_STATIC_LIBRARY_SUFFIX})
 
   #set(boost_URL https://github.com/boostorg/boost.git)
-  #set(boost_TAG boost-1.65.1)
+  #set(boost_TAG boost-1.68.0)
 
-  set(Boost_TAR_GZ_URL http://dl.bintray.com/boostorg/release/1.65.1/source/boost_1_65_1.tar.gz)
+  set(Boost_TAR_GZ_URL http://dl.bintray.com/boostorg/release/1.68.0/source/boost_1_68_0.tar.gz)
   set(Boost_BUILD_PRODUCTS ${Boost_SYSTEM_LIBRARY} ${Boost_FILESYSTEM_LIBRARY})
-  set(Boost_URL_MD5 "ee64fd29a3fe42232c6ac3c419e523cf")
+  set(Boost_URL_MD5 "5d8b4503582fffa9eefdb9045359c239")
 
   set(Boost_USE_STATIC_LIBS ON)
 
@@ -48,6 +49,6 @@ else()
     BUILD_IN_SOURCE 1
     BUILD_BYPRODUCTS ${Boost_BUILD_PRODUCTS}
     CONFIGURE_COMMAND ./bootstrap.sh
-    BUILD_COMMAND bash -c "./b2 cxxflags=-fPIC cflags=-fPIC variant=release link=static --with-filesystem --with-system --with-regex -j8 install --prefix=${Boost_INSTALL_PREFIX} > /dev/null"
+    BUILD_COMMAND bash -c "./b2 cxxflags=-fPIC cflags=-fPIC variant=release link=static --with-filesystem --with-system --with-thread --with-atomic --with-chrono --with-date_time --with-regex -j8 install --prefix=${Boost_INSTALL_PREFIX} > /dev/null"
     INSTALL_COMMAND "")
 endif ()

cmake/Modules/ThirdpartyToolchain.cmake

@@ -54,11 +54,14 @@ ADD_THIRDPARTY_LIB(boost_system
   STATIC_LIB ${Boost_SYSTEM_LIBRARY})
 ADD_THIRDPARTY_LIB(boost_filesystem
   STATIC_LIB ${Boost_FILESYSTEM_LIBRARY})
+ADD_THIRDPARTY_LIB(boost_thread
+  STATIC_LIB ${Boost_THREAD_LIBRARY})
 
 add_dependencies(boost_system boost_ep)
 add_dependencies(boost_filesystem boost_ep)
+add_dependencies(boost_thread boost_ep)
 
-add_custom_target(boost DEPENDS boost_system boost_filesystem)
+add_custom_target(boost DEPENDS boost_system boost_filesystem boost_thread)
 
 # flatbuffers
 include(FlatBuffersExternalProject)
@@ -120,6 +123,7 @@ if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES")
       "PYARROW_WITH_TENSORFLOW=1"
       "PYARROW_BUNDLE_ARROW_CPP=1"
       "PARQUET_HOME=${PARQUET_HOME}"
+      "BOOST_ROOT=${BOOST_ROOT}"
       "PYARROW_WITH_PARQUET=1"
       "PYARROW_PARALLEL=")
 

src/ray/CMakeLists.txt

@@ -1,11 +1,6 @@
 cmake_minimum_required(VERSION 3.2)
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -Wall -Werror -std=c++11")
 
-add_subdirectory(util)
-add_subdirectory(gcs)
-add_subdirectory(object_manager)
-add_subdirectory(raylet)
-
 include_directories(thirdparty/ae)
 
 set(HIREDIS_SRCS
@@ -56,8 +51,15 @@ set(RAY_SRCS
   raylet/raylet.cc
 )
 
-set(RAY_LIB_STATIC_LINK_LIBS ${PLASMA_STATIC_LIB} ${ARROW_STATIC_LIB})
+set(RAY_SYSTEM_LIBS ${Boost_SYSTEM_LIBRARY} ${Boost_THREAD_LIBRARY} pthread)
+if(UNIX AND NOT APPLE)
+  set(RAY_SYSTEM_LIBS ${RAY_SYSTEM_LIBS} -lrt)
+endif()
+
+set(RAY_LIB_STATIC_LINK_LIBS ${PLASMA_STATIC_LIB} ${ARROW_STATIC_LIB} ${RAY_SYSTEM_LIBS})
+
 set(RAY_LIB_DEPENDENCIES
+    boost_thread
     arrow_ep
     gen_gcs_fbs
     gen_object_manager_fbs
@@ -89,6 +91,13 @@ ADD_RAY_LIB(ray
     SHARED_LINK_LIBS ""
     STATIC_LINK_LIBS ${RAY_LIB_STATIC_LINK_LIBS})
 
+set(RAY_TEST_LIBS ray_static ${PLASMA_STATIC_LIB} ${ARROW_STATIC_LIB} gtest gtest_main gmock_main ${RAY_SYSTEM_LIBS})
+
+add_subdirectory(util)
+add_subdirectory(gcs)
+add_subdirectory(object_manager)
+add_subdirectory(raylet)
+
 add_custom_target(copy_redis ALL)
 foreach(file "redis-cli" "redis-server")
 add_custom_command(TARGET copy_redis POST_BUILD

src/ray/object_manager/CMakeLists.txt

@@ -17,11 +17,11 @@ add_custom_command(
 
 add_custom_target(gen_object_manager_fbs DEPENDS ${OBJECT_MANAGER_FBS_OUTPUT_FILES})
 
-ADD_RAY_TEST(test/object_manager_test STATIC_LINK_LIBS ray_static ${PLASMA_STATIC_LIB} ${ARROW_STATIC_LIB} gtest gtest_main pthread ${Boost_SYSTEM_LIBRARY})
-ADD_RAY_TEST(test/object_manager_stress_test STATIC_LINK_LIBS ray_static ${PLASMA_STATIC_LIB} ${ARROW_STATIC_LIB} gtest gtest_main pthread ${Boost_SYSTEM_LIBRARY})
+ADD_RAY_TEST(test/object_manager_test STATIC_LINK_LIBS ${RAY_TEST_LIBS})
+ADD_RAY_TEST(test/object_manager_stress_test STATIC_LINK_LIBS ${RAY_TEST_LIBS})
 
 add_library(object_manager object_manager.cc object_manager.h ${OBJECT_MANAGER_FBS_OUTPUT_FILES})
-target_link_libraries(object_manager common ray_static ${PLASMA_STATIC_LIB} ${ARROW_STATIC_LIB} ${Boost_SYSTEM_LIBRARY})
+target_link_libraries(object_manager common ray_static ${PLASMA_STATIC_LIB} ${ARROW_STATIC_LIB} ${Boost_SYSTEM_LIBRARY} ${Boost_THREAD_LIBRARY})
 
 install(FILES
         object_manager

src/ray/raylet/CMakeLists.txt

@@ -30,19 +30,17 @@ add_custom_command(
 
 add_dependencies(gen_node_manager_fbs flatbuffers_ep)
 
-ADD_RAY_TEST(object_manager_integration_test STATIC_LINK_LIBS ray_static ${PLASMA_STATIC_LIB} ${ARROW_STATIC_LIB} gtest gtest_main pthread ${Boost_SYSTEM_LIBRARY})
-
-ADD_RAY_TEST(worker_pool_test STATIC_LINK_LIBS ray_static ${PLASMA_STATIC_LIB} ${ARROW_STATIC_LIB} gtest gtest_main gmock_main pthread ${Boost_SYSTEM_LIBRARY})
-
-ADD_RAY_TEST(client_connection_test STATIC_LINK_LIBS ray_static gtest gtest_main gmock_main pthread ${Boost_SYSTEM_LIBRARY})
-ADD_RAY_TEST(task_test STATIC_LINK_LIBS ray_static gtest gtest_main gmock_main pthread ${Boost_SYSTEM_LIBRARY})
-ADD_RAY_TEST(lineage_cache_test STATIC_LINK_LIBS ray_static gtest gtest_main gmock_main pthread ${Boost_SYSTEM_LIBRARY})
-ADD_RAY_TEST(task_dependency_manager_test STATIC_LINK_LIBS ray_static gtest gtest_main gmock_main pthread ${Boost_SYSTEM_LIBRARY})
-ADD_RAY_TEST(reconstruction_policy_test STATIC_LINK_LIBS ray_static gtest gtest_main gmock_main pthread ${Boost_SYSTEM_LIBRARY})
+ADD_RAY_TEST(object_manager_integration_test STATIC_LINK_LIBS ${RAY_TEST_LIBS})
+ADD_RAY_TEST(worker_pool_test STATIC_LINK_LIBS ${RAY_TEST_LIBS})
+ADD_RAY_TEST(client_connection_test STATIC_LINK_LIBS ${RAY_TEST_LIBS})
+ADD_RAY_TEST(task_test STATIC_LINK_LIBS ${RAY_TEST_LIBS})
+ADD_RAY_TEST(lineage_cache_test STATIC_LINK_LIBS ${RAY_TEST_LIBS})
+ADD_RAY_TEST(task_dependency_manager_test STATIC_LINK_LIBS ${RAY_TEST_LIBS})
+ADD_RAY_TEST(reconstruction_policy_test STATIC_LINK_LIBS ${RAY_TEST_LIBS})
 
 include_directories(${GCS_FBS_OUTPUT_DIRECTORY})
 add_library(rayletlib raylet.cc ${NODE_MANAGER_FBS_OUTPUT_FILES})
-target_link_libraries(rayletlib ray_static ${Boost_SYSTEM_LIBRARY})
+target_link_libraries(rayletlib ray_static ${Boost_SYSTEM_LIBRARY} ${Boost_THREAD_LIBRARY})
 
 add_library(raylet_client STATIC raylet_client.cc)
 
@@ -52,10 +50,10 @@ add_dependencies(rayletlib gen_gcs_fbs)
 add_dependencies(raylet_client gen_gcs_fbs arrow_ep gen_node_manager_fbs)
 
 add_executable(raylet main.cc)
-target_link_libraries(raylet rayletlib ${Boost_SYSTEM_LIBRARY} pthread)
+target_link_libraries(raylet rayletlib ${RAY_SYSTEM_LIBS})
 
 add_executable(raylet_monitor monitor_main.cc)
-target_link_libraries(raylet_monitor rayletlib ${Boost_SYSTEM_LIBRARY} pthread)
+target_link_libraries(raylet_monitor rayletlib ${Boost_SYSTEM_LIBRARY} ${Boost_THREAD_LIBRARY} pthread)
 
 install(FILES
         raylet
@@ -78,9 +76,9 @@ macro(set_raylet_library LANG)
       if ("${LANG}" STREQUAL "python")
         SET_TARGET_PROPERTIES(${RAYLET_LIBRARY_LANG} PROPERTIES SUFFIX .so)
       endif()
-      target_link_libraries(${RAYLET_LIBRARY_LANG} "-undefined dynamic_lookup" raylet_client ray_static ${PLASMA_STATIC_LIB} ${ARROW_STATIC_LIB} ${Boost_SYSTEM_LIBRARY})
+      target_link_libraries(${RAYLET_LIBRARY_LANG} "-undefined dynamic_lookup" raylet_client ray_static ${PLASMA_STATIC_LIB} ${ARROW_STATIC_LIB} ${Boost_SYSTEM_LIBRARY} ${Boost_THREAD_LIBRARY})
     else(APPLE)
-      target_link_libraries(${RAYLET_LIBRARY_LANG} raylet_client ray_static ${PLASMA_STATIC_LIB} ${ARROW_STATIC_LIB} ${Boost_SYSTEM_LIBRARY})
+      target_link_libraries(${RAYLET_LIBRARY_LANG} raylet_client ray_static ${PLASMA_STATIC_LIB} ${ARROW_STATIC_LIB} ${Boost_SYSTEM_LIBRARY} ${Boost_THREAD_LIBRARY})
     endif(APPLE)
 
   add_dependencies(${RAYLET_LIBRARY_LANG} gen_node_manager_fbs)

thirdparty/scripts/build_boost.sh

@@ -28,6 +28,6 @@ if [[ ! -d $TP_DIR/pkg/boost ]]; then
   # Compile boost.
   pushd $TP_DIR/build/boost_$BOOST_VERSION_UNDERSCORE
     ./bootstrap.sh
-    ./bjam cxxflags=-fPIC cflags=-fPIC variant=release link=static --prefix=$TP_DIR/pkg/boost --with-filesystem --with-system --with-regex install > /dev/null
+    ./bjam cxxflags=-fPIC cflags=-fPIC variant=release link=static --prefix=$TP_DIR/pkg/boost --with-filesystem --with-system --with-thread --with-regex install > /dev/null
   popd
 fi