Add default cuda system path (#192)

* DYLD_LIBRARY_PATH is disable after Mac OS X 10.11
* fix clang + gpu compile error on Mac OS
* fix some words and errors in build docs

Author: gangliao

cmake/flags.cmake

@@ -21,12 +21,6 @@ function(safe_set_flag is_c src_list flag_name)
     endif()
     if(${safe_name})
         set(${src_list} "${${src_list}} ${flag_name}" PARENT_SCOPE)
-        if(is_c)
-          set(CUDA_NVCC_FLAGS
-              --compiler-options;${flag_name}
-              ${CUDA_NVCC_FLAGS}
-              PARENT_SCOPE)
-        endif()
     endif()
 endfunction()
 
@@ -40,6 +34,20 @@ macro(safe_set_cxxflag src_list flag_name)
     safe_set_flag(OFF ${src_list} ${flag_name})
 endmacro()
 
+# helper macro to set nvcc flag
+macro(safe_set_nvflag flag_name)
+    string(REPLACE "-" "_" safe_name ${flag_name})
+    string(REPLACE "=" "_" safe_name ${safe_name})
+    CHECK_C_COMPILER_FLAG(${flag_name} C_COMPILER_SUPPORT_FLAG_${safe_name})
+    set(safe_name C_COMPILER_SUPPORT_FLAG_${safe_name})
+    if(${safe_name})
+        set(CUDA_NVCC_FLAGS
+            --compiler-options;${flag_name}
+            ${CUDA_NVCC_FLAGS})
+    endif()
+endmacro()
+
+
 CHECK_CXX_SYMBOL_EXISTS(UINT64_MAX "stdint.h" UINT64_MAX_EXISTS)
 if(NOT UINT64_MAX_EXISTS)
   set(CMAKE_REQUIRED_DEFINITIONS -D__STDC_LIMIT_MACROS)
@@ -63,20 +71,43 @@ set(COMMON_FLAGS
     -Wnon-virtual-dtor
     -Wdelete-non-virtual-dtor
     -Wno-unused-parameter
+    -Wno-unused-function
+    -Wno-error=literal-suffix
+    -Wno-error=unused-local-typedefs)
+
+set(GPU_COMMON_FLAGS
+    -fPIC
+    -fno-omit-frame-pointer
+    -Wnon-virtual-dtor
+    -Wdelete-non-virtual-dtor
+    -Wno-unused-parameter
+    -Wno-unused-function
     -Wno-error=literal-suffix
     -Wno-error=unused-local-typedefs
     -Wno-error=unused-function  # Warnings in Numpy Header.
 )
 
+if (APPLE)
+    # On Mac OS X build fat binaries with x86_64 architectures by default.
+    set (CMAKE_OSX_ARCHITECTURES "x86_64" CACHE STRING "Build architectures for OSX" FORCE)
+else()
+    set(GPU_COMMON_FLAGS
+        -Wall
+        -Wextra
+        -Werror
+        ${GPU_COMMON_FLAGS})
+endif()
+
+
 foreach(flag ${COMMON_FLAGS})
     safe_set_cflag(CMAKE_C_FLAGS ${flag})
     safe_set_cxxflag(CMAKE_CXX_FLAGS ${flag})
 endforeach()
 
-# On Mac OS X build fat binaries with x86_64 architectures by default.
-if (APPLE)
-    set (CMAKE_OSX_ARCHITECTURES "x86_64" CACHE STRING "Build architectures for OSX" FORCE)
-endif ()
+foreach(flag ${GPU_COMMON_FLAGS})
+    safe_set_nvflag(${flag})
+endforeach()
+
 
 # Release/Debug flags set by cmake. Such as -O3 -g -DNDEBUG etc.
 # So, don't set these flags here.

doc/build/build_from_source.md

@@ -153,12 +153,12 @@ As a simple example, consider the following:
 - **Only CPU**
 
   ```bash
-  cmake  .. -DWITH_GPU=OFF -DWITH_DOC=OFF
+  cmake  .. -DWITH_GPU=OFF
   ```
 - **GPU**
 
   ```bash
-  cmake .. -DWITH_GPU=ON -DWITH_DOC=OFF
+  cmake .. -DWITH_GPU=ON
   ```
 
 - **GPU with doc and swig**
@@ -171,7 +171,7 @@ Finally, you can build PaddlePaddle:
 
 ```bash
 # you can add build option here, such as:    
-cmake .. -DWITH_GPU=ON -DWITH_DOC=OFF -DCMAKE_INSTALL_PREFIX=<path to install>
+cmake .. -DWITH_GPU=ON -DCMAKE_INSTALL_PREFIX=<path to install>
 # please use sudo make install, if you want to install PaddlePaddle into the system
 make -j `nproc` && make install
 # set PaddlePaddle installation path in ~/.bashrc
@@ -246,7 +246,7 @@ easy_install pip
 
         ```bash
         sudo tar -xzf cudnn-7.5-osx-x64-v5.0-ga.tgz -C /usr/local
-        sudo chmod a+r /usr/local/cuda/include/cudnn.h /usr/local/cuda/lib64/libcudnn*
+        sudo chmod a+r /usr/local/cuda/include/cudnn.h /usr/local/cuda/lib/libcudnn*
         ```
     2. Then you need to set DYLD\_LIBRARY\_PATH, PATH environment variables in ~/.bashrc.
 
@@ -273,12 +273,12 @@ As a simple example, consider the following:
 - **Only CPU**
 
   ```bash
-  cmake  .. -DWITH_GPU=OFF -DWITH_DOC=OFF
+  cmake  .. -DWITH_GPU=OFF
   ```
 - **GPU**
 
   ```bash
-  cmake .. -DWITH_GPU=ON -DWITH_DOC=OFF
+  cmake .. -DWITH_GPU=ON
   ```
 
 - **GPU with doc and swig**
@@ -291,9 +291,9 @@ Finally, you can build PaddlePaddle:
 
 ```bash
 # you can add build option here, such as:    
-cmake .. -DWITH_GPU=ON -DWITH_DOC=OFF -DCMAKE_INSTALL_PREFIX=<installation path>
+cmake .. -DWITH_GPU=ON -DCMAKE_INSTALL_PREFIX=<installation path>
 # please use sudo make install, if you want to install PaddlePaddle into the system
-make -j `nproc` && make install
+make -j `sysctl -n hw.ncpu` && make install
 # set PaddlePaddle installation path in ~/.bashrc
 export PATH=<installation path>/bin:$PATH
 ```

paddle/cuda/src/hl_dso_loader.cc

@@ -46,63 +46,100 @@ static inline std::string join(const std::string& part1, const std::string& part
   return ret;
 }
 
-static inline void GetDsoHandleWithSearchPath(
+static inline void GetDsoHandleFromDefaultPath(
+        std::string& dso_path, void** dso_handle, int dynload_flags) {
+    LOG(INFO) << "Try to find cuda library: " << dso_path
+              << "from default system path.";
+    // default search from LD_LIBRARY_PATH/DYLD_LIBRARY_PATH 
+    *dso_handle = dlopen(dso_path.c_str(), dynload_flags);
+    
+    // DYLD_LIBRARY_PATH is disabled after Mac OS 10.11 to
+    // bring System Integrity Projection (SIP), if dso_handle
+    // is null, search from default package path in Mac OS.
+    #if defined(__APPLE__) or defined(__OSX__)
+    if (nullptr == *dso_handle) {
+        dso_path = join("/usr/local/cuda/lib/", dso_path);
+        *dso_handle = dlopen(dso_path.c_str(), dynload_flags);
+        if (nullptr == *dso_handle) {
+            if (dso_path == "libcudnn.dylib") {
+                LOG(FATAL) << "Note: [Recommend] copy cudnn into /usr/local/cuda/ \n"
+                << "For instance, sudo tar -xzf cudnn-7.5-osx-x64-v5.0-ga.tgz -C "
+                << "/usr/local \n sudo chmod a+r /usr/local/cuda/include/cudnn.h "
+                << "/usr/local/cuda/lib/libcudnn*";
+            }
+        } 
+    }   
+    #endif
+}
+
+static inline void GetDsoHandleFromSearchPath(
         const std::string& search_root,
-        const std::string& dso_path,
+        const std::string& dso_name,
         void** dso_handle) {
     int dynload_flags = RTLD_LAZY | RTLD_LOCAL;
     *dso_handle = nullptr;
 
-    std::string dlPath = dso_path;
+    std::string dlPath = dso_name;
     if (search_root.empty()) {
-        // default search xxx.so from LD_LIBRARY_PATH
-        *dso_handle = dlopen(dlPath.c_str(), dynload_flags);
+        GetDsoHandleFromDefaultPath(dlPath, dso_handle, dynload_flags);
     } else {
         // search xxx.so from custom path
-        dlPath = join(search_root, dso_path);
+        dlPath = join(search_root, dso_name);
         *dso_handle = dlopen(dlPath.c_str(), dynload_flags);
-        // then, search xxx.so from LD_LIBRARY_PATH
-        if (nullptr == *dso_handle) {
-            *dso_handle = dlopen(dso_path.c_str(), dynload_flags);
+        // if not found, search from default path
+        if (nullptr == dso_handle) {
+            LOG(WARNING) << "Failed to find cuda library: " << dlPath;
+            dlPath = dso_name;
+            GetDsoHandleFromDefaultPath(dlPath, dso_handle, dynload_flags);
         }
     }
 
     CHECK(nullptr != *dso_handle)
-      << "For Gpu version of PaddlePaddle, it couldn't find CUDA library: "
-      << dlPath.c_str() << ". Please make sure you already specify its path. "
-      << "Note: for training data on Cpu using Gpu version of PaddlePaddle, "
-      << "you must specify libcudart via export LD_LIBRARY_PATH for Linux or "
-      << "export DYLD_LIBRARY_PATH for MAC OS.";
+      << "Failed to find cuda library: " << dlPath << std::endl
+      << "Please specify its path correctly using one of the following ideas: \n"
+
+      << "Idea 1. set cuda and cudnn lib path at runtime. "
+      << "http://www.paddlepaddle.org/doc/ui/cmd_argument/argument_outline.html \n"
+      << "For instance, issue command: paddle train --use_gpu=1 "
+      << "--cuda_dir=/usr/local/cudnn/lib --cudnn_dir=/usr/local/cudnn/lib ...\n"
+
+      << "Idea 2. set environment variable LD_LIBRARY_PATH on Linux or "
+      << "DYLD_LIBRARY_PATH on Mac OS. \n"
+      << "For instance, issue command: export LD_LIBRARY_PATH=... \n"
+
+      << "Note: After Mac OS 10.11, using the DYLD_LIBRARY_PATH is impossible "
+      << "unless System Integrity Protection (SIP) is disabled. However, @Idea 1"
+      << "always work well.";
 }
 
 void GetCublasDsoHandle(void** dso_handle) {
 #if defined(__APPLE__) || defined(__OSX__)
-    GetDsoHandleWithSearchPath(FLAGS_cuda_dir, "libcublas.dylib", dso_handle);
+    GetDsoHandleFromSearchPath(FLAGS_cuda_dir, "libcublas.dylib", dso_handle);
 #else
-    GetDsoHandleWithSearchPath(FLAGS_cuda_dir, "libcublas.so", dso_handle);
+    GetDsoHandleFromSearchPath(FLAGS_cuda_dir, "libcublas.so", dso_handle);
 #endif
 }
 
 void GetCudnnDsoHandle(void** dso_handle) {
 #if defined(__APPLE__) || defined(__OSX__)
-    GetDsoHandleWithSearchPath(FLAGS_cudnn_dir, "libcudnn.dylib", dso_handle);
+    GetDsoHandleFromSearchPath(FLAGS_cudnn_dir, "libcudnn.dylib", dso_handle);
 #else
-    GetDsoHandleWithSearchPath(FLAGS_cudnn_dir, "libcudnn.so", dso_handle);
+    GetDsoHandleFromSearchPath(FLAGS_cudnn_dir, "libcudnn.so", dso_handle);
 #endif
 }
 
 void GetCudartDsoHandle(void** dso_handle) {
 #if defined(__APPLE__) || defined(__OSX__)
-    GetDsoHandleWithSearchPath("", "libcudart.dylib", dso_handle);
+    GetDsoHandleFromSearchPath("", "libcudart.dylib", dso_handle);
 #else
-    GetDsoHandleWithSearchPath("", "libcudart.so", dso_handle);
+    GetDsoHandleFromSearchPath("", "libcudart.so", dso_handle);
 #endif
 }
 
 void GetCurandDsoHandle(void** dso_handle) {
 #if defined(__APPLE__) || defined(__OSX__)
-    GetDsoHandleWithSearchPath(FLAGS_cuda_dir, "libcurand.dylib", dso_handle);
+    GetDsoHandleFromSearchPath(FLAGS_cuda_dir, "libcurand.dylib", dso_handle);
 #else
-    GetDsoHandleWithSearchPath(FLAGS_cuda_dir, "libcurand.so", dso_handle);
+    GetDsoHandleFromSearchPath(FLAGS_cuda_dir, "libcurand.so", dso_handle);
 #endif
 }