Upgrade pybind and refine pybind/protobuf.cc

CMakeLists.txt

@@ -140,6 +140,7 @@ include(external/pybind11)  # download pybind11
 include(external/nccl)
 include(external/cares)
 include(external/grpc)
+include(external/boost)
 
 include(cudnn)              # set cudnn libraries, must before configure
 include(configure)          # add paddle env configuration

cmake/external/boost.cmake

@@ -0,0 +1,73 @@
+# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+INCLUDE(ExternalProject)
+
+set(BOOST_PROJECT       "extern_boost")
+set(BOOST_VER           "1.66.0")
+set(BOOST_TAR           "boost_1_66_0")
+set(BOOST_URL           "https://dl.bintray.com/boostorg/release/${BOOST_VER}/source/${BOOST_TAR}.tar.gz")
+set(BOOST_SOURCES_DIR ${THIRD_PARTY_PATH}/boost)
+set(BOOST_DOWNLOAD_DIR  "${BOOST_SOURCES_DIR}/src/${BOOST_PROJECT}")
+set(BOOST_INSTALL_DIR ${THIRD_PARTY_PATH}/install/boost)
+set(BOOST_ROOT ${BOOST_INSTALL_DIR} CACHE FILEPATH "boost root directory." FORCE)
+set(BOOST_INCLUDE_DIR "${BOOST_INSTALL_DIR}/include" CACHE PATH "boost include directory." FORCE)
+
+set(BOOST_BOOTSTRAP_CMD)
+set(BOOST_B2_CMD)
+
+if(UNIX)
+  set(BOOST_BOOTSTRAP_CMD ./bootstrap.sh)
+  set(BOOST_B2_CMD ./b2)
+else()
+  if(WIN32)
+    set(BOOST_BOOTSTRAP_CMD bootstrap.bat)
+    set(BOOST_B2_CMD b2.exe)
+  endif()
+endif()
+
+INCLUDE_DIRECTORIES(${BOOST_INCLUDE_DIR})
+
+ExternalProject_Add(
+    ${BOOST_PROJECT}
+    ${EXTERNAL_PROJECT_LOG_ARGS}
+    DOWNLOAD_DIR          ${BOOST_DOWNLOAD_DIR}
+    DOWNLOAD_COMMAND      wget --no-check-certificate ${BOOST_URL} -c -q -O ${BOOST_TAR}.tar.gz
+                          && tar zxf ${BOOST_TAR}.tar.gz
+    DOWNLOAD_NO_PROGRESS  1
+    PREFIX                ${BOOST_SOURCES_DIR}
+    CONFIGURE_COMMAND     sh -c "cd <SOURCE_DIR>/${BOOST_TAR} && ${BOOST_BOOTSTRAP_CMD} --prefix=${BOOST_INSTALL_DIR}"
+    BUILD_COMMAND         cd <SOURCE_DIR>/${BOOST_TAR} && ${BOOST_B2_CMD} install --prefix=${BOOST_INSTALL_DIR} --with-program_options
+    INSTALL_COMMAND       ""
+    UPDATE_COMMAND        ""
+)
+
+if (${CMAKE_VERSION} VERSION_LESS "3.3.0")
+    set(dummyfile ${CMAKE_CURRENT_BINARY_DIR}/boost_dummy.c)
+    file(WRITE ${dummyfile} "const char *dummy = \"${dummyfile}\";")
+    add_library(boost STATIC ${dummyfile})
+else()
+    add_library(boost INTERFACE)
+endif()
+
+ADD_DEPENDENCIES(boost ${BOOST_PROJECT})
+LIST(APPEND external_project_dependencies boost)
+if(WIN32)
+  set(Boost_INCLUDE_DIR ${BOOST_INSTALL_DIR}/include/boost)
+  set(BOOST_ROOT ${BOOST_INSTALL_DIR} )
+else()
+  set(Boost_INCLUDE_DIR ${BOOST_INSTALL_DIR}/include)
+endif()
+
+set(Boost_LIBRARY_DIR ${BOOST_INSTALL_DIR}/lib)

cmake/external/pybind11.cmake

@@ -26,7 +26,7 @@ ExternalProject_Add(
         extern_pybind
         ${EXTERNAL_PROJECT_LOG_ARGS}
         GIT_REPOSITORY  "https://github.com/pybind/pybind11.git"
-        GIT_TAG         "v2.1.1"
+        GIT_TAG         "v2.2.1"
         PREFIX          ${PYBIND_SOURCE_DIR}
         UPDATE_COMMAND  ""
         CONFIGURE_COMMAND ""

paddle/framework/CMakeLists.txt

@@ -1,7 +1,7 @@
 # ddim lib
 proto_library(framework_proto SRCS framework.proto)
 
-cc_library(ddim SRCS ddim.cc DEPS eigen3)
+cc_library(ddim SRCS ddim.cc DEPS eigen3 boost)
 cc_test(ddim_test SRCS ddim_test.cc DEPS ddim)
 nv_test(dim_test SRCS dim_test.cu DEPS ddim)
 

paddle/pybind/protobuf.cc

@@ -21,74 +21,24 @@ limitations under the License. */
 #include "paddle/framework/program_desc.h"
 #include "paddle/framework/var_desc.h"
 
-// Cast boost::variant for PyBind.
-// Copy from
-// https://github.com/pybind/pybind11/issues/576#issuecomment-269563199
+using boost::variant;
+
 namespace pybind11 {
 namespace detail {
 
-// Can be replaced by a generic lambda in C++14
-struct variant_caster_visitor : public boost::static_visitor<handle> {
-  return_value_policy policy;
-  handle parent;
-
-  variant_caster_visitor(return_value_policy policy, handle parent)
-      : policy(policy), parent(parent) {}
-
-  template <class T>
-  handle operator()(T const &src) const {
-    return make_caster<T>::cast(src, policy, parent);
-  }
-};
-
-template <class Variant>
-struct variant_caster;
-
-template <template <class...> class V, class... Ts>
-struct variant_caster<V<Ts...>> {
-  using Type = V<Ts...>;
-
-  template <typename T>
-  typename std::enable_if<
-      !std::is_same<T, boost::detail::variant::void_>::value, bool>::type
-  try_load(handle src, bool convert) {
-    auto caster = make_caster<T>();
-    if (!load_success_ && caster.load(src, convert)) {
-      load_success_ = true;
-      value = cast_op<T>(caster);
-      return true;
-    }
-    return false;
-  }
-
-  template <typename T>
-  typename std::enable_if<std::is_same<T, boost::detail::variant::void_>::value,
-                          bool>::type
-  try_load(handle src, bool convert) {
-    return false;
-  }
-
-  bool load(handle src, bool convert) {
-    auto unused = {false, try_load<Ts>(src, convert)...};
-    (void)(unused);
-    return load_success_;
-  }
-
-  static handle cast(Type const &src, return_value_policy policy,
-                     handle parent) {
-    variant_caster_visitor visitor(policy, parent);
-    return boost::apply_visitor(visitor, src);
-  }
-
-  PYBIND11_TYPE_CASTER(Type, _("Variant"));
-  bool load_success_{false};
-};
-
 // Add specialization for concrete variant type
 template <class... Args>
 struct type_caster<boost::variant<Args...>>
     : variant_caster<boost::variant<Args...>> {};
 
+template <>
+struct visit_helper<boost::variant> {
+  template <typename... Args>
+  static auto call(Args &&... args) -> decltype(boost::apply_visitor(args...)) {
+    return boost::apply_visitor(args...);
+  }
+};
+
 }  // namespace detail
 }  // namespace pybind11
 

paddle/pybind/pybind.cc

@@ -60,8 +60,8 @@ bool IsCompileGPU() {
 #endif
 }
 
-PYBIND11_PLUGIN(core) {
-  py::module m("core", "C++ core of PaddlePaddle");
+PYBIND11_MODULE(core, m) {
+  m.doc() = "C++ core of PaddlePaddle";
 
   // using framework in this function. Since it is inside a function, it will
   // not cause namespace pollution.
@@ -475,8 +475,6 @@ All parameter, weight, gradient are variables in Paddle.
   m.def("nvprof_start", platform::CudaProfilerStart);
   m.def("nvprof_stop", platform::CudaProfilerStop);
 #endif
-
-  return m.ptr();
 }
 }  // namespace pybind
 }  // namespace paddle