Dev rel legacy cuda (#3343)

jackalcooper · tsai · guo-ran · web-flow · commit 4078da22e353 · 2020-07-31T06:15:37.000+08:00
* add rel script

* support customized package name

* one wheel dir

* update docs

* add more info

* add more info

* add from img

* fix

* use oneflow_cu102 in ci

* fix

* default rdma OFF

* fix typo

Co-authored-by: tsai &lt;caishenghang@1f-dev.kbaeegfb1x0ubnoznzequyxzve.bx.internal.cloudapp.net&gt;
Co-authored-by: guo ran &lt;360112263@qq.com&gt;
diff --git a/.dockerignore b/.dockerignore
@@ -29,7 +29,7 @@ dist
 
 build/third_party/re2/src/re2-*
 
-/wheelhouse
+/wheelhouse*
 /.DS_Store
 /tmp_wheel
 /manylinux*
diff --git a/.gitignore b/.gitignore
@@ -15,5 +15,6 @@ core.*
 /.idea
 /manylinux*
 wheelhouse/
+wheelhouse*
 .DS_Store
 /tmp_wheel
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -9,7 +9,7 @@ if (NOT THIRD_PARTY AND NOT ONEFLOW)
 endif()
 
 option(USE_CLANG_FORMAT "" OFF)
-option(BUILD_RDMA "" ON)
+option(BUILD_RDMA "" OFF)
 option(BUILD_CUDA "" ON)
 option(BUILD_TESTING "" ON)
 option(PY3 "" ON)
diff --git a/README.md b/README.md
@@ -8,18 +8,33 @@
 
   #### Install Pip package
 
-  - To install latest release of OneFlow:
+  - To install latest release of OneFlow with CUDA support:
 
     ```
-    python3 -m pip install oneflow
+    python3 -m pip install oneflow_cu102
     ```
 
+  - To install OneFlow with legacy CUDA support, run one of:
+    ```
+    python3 -m pip install oneflow_cu101
+    python3 -m pip install oneflow_cu100
+    python3 -m pip install oneflow_cu92
+    python3 -m pip install oneflow_cu91
+    python3 -m pip install oneflow_cu90
+    ```
+
+  - Support for latest stable version of CUDA will be prioritized. Please upgrade your Nvidia driver to version 440.33 or above and install `oneflow_cu102` if possible. For more information, please refer to [CUDA compatibility documentation](https://docs.nvidia.com/deploy/cuda-compatibility/index.html).
+
   - To install nightly release of OneFlow:
 
     ```
-    python3 -m pip install --find-links https://oneflow-inc.github.io/nightly oneflow
+    python3 -m pip install --find-links https://oneflow-inc.github.io/nightly oneflow_cu102
     ```
 
+  - CPU-only OneFlow is not available for now.
+
+  - Releases are built with G++/GCC 4.8.5, cuDNN 7 and MKL 2020.0-088.
+
 ### Build OneFlow from Source
 
 1. #### System Requirements to Build OneFlow
diff --git a/ci/build/clean.sh b/ci/build/clean.sh
@@ -2,3 +2,4 @@ set -ex
 docker run --rm \
     -v $HOME/ci-tmp:/ci-tmp \
     -w $HOME/ci-tmp:/ci-tmp busybox rm -rf /ci-tmp/wheelhouse
+docker run --rm -v $PWD:/p -w /p busybox rm -rf build
diff --git a/ci/build/make.sh b/ci/build/make.sh
@@ -18,6 +18,7 @@ fi
 # build manylinux image
 cd $src_dir
 docker build -f $src_dir/docker/package/manylinux/Dockerfile \
+    --build-arg from=nvidia/cuda:10.2-cudnn7-devel-centos7 \
     $docker_proxy_build_args -t $docker_tag .
 
 cd -
@@ -32,7 +33,9 @@ function build() {
         -v $tmp_dir/py-build-lib:/oneflow-src/build/lib/ \
         -w /ci-tmp \
         "$docker_tag" \
-        /oneflow-src/docker/package/manylinux/build_wheel.sh --python3.6
+        /oneflow-src/docker/package/manylinux/build_wheel.sh \
+            --python3.6 \
+            --package-name oneflow_cu102
 }
 
 set +e
diff --git a/docker/package/manylinux/Dockerfile b/docker/package/manylinux/Dockerfile
@@ -1,4 +1,5 @@
-FROM nvidia/cuda:10.2-cudnn7-devel-centos7
+ARG from
+FROM ${from}
 LABEL maintainer="OneFlow Maintainers"
 
 # manylinux2014
diff --git a/docker/package/manylinux/README.md b/docker/package/manylinux/README.md
@@ -4,7 +4,7 @@
 
 在 OneFlow 源码根目录下运行:
 ```
-docker build -f docker/package/manylinux/Dockerfile -t oneflow:manylinux2014-cuda10.2 .
+docker build -f docker/package/manylinux/Dockerfile --build-arg from=nvidia/cuda:10.2-cudnn7-devel-centos7 -t oneflow:manylinux2014-cuda10.2 .
 ```
 
 ### 打包 manylinux python wheel
diff --git a/docker/package/manylinux/build_wheel.sh b/docker/package/manylinux/build_wheel.sh
@@ -13,6 +13,7 @@ while [[ "$#" > 0 ]]; do
         --skip-third-party) SKIP_THIRD_PARTY=1; ;;
         --cache-dir) CACHE_DIR=$2; shift ;;
         --house-dir) HOUSE_DIR=$2; shift ;;
+        --package-name) PACKAGE_NAME=$2; shift ;;
         --python3.5) PY_VERS+=( "35" ) ;;
         --python3.6) PY_VERS+=( "36" ) ;;
         --python3.7) PY_VERS+=( "37" ) ;;
@@ -32,6 +33,11 @@ then
     HOUSE_DIR=$PWD/wheelhouse
 fi
 
+if [[ ! -v PACKAGE_NAME ]]
+then
+    PACKAGE_NAME=oneflow
+fi
+
 ONEFLOW_SRC_DIR=`cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd`
 ONEFLOW_SRC_DIR=$ONEFLOW_SRC_DIR/../../..
 
@@ -44,13 +50,14 @@ cd $ONEFLOW_SRC_DIR
 
 THIRD_PARTY_BUILD_DIR=$CACHE_DIR/build-third-party
 THIRD_PARTY_INSTALL_DIR=$CACHE_DIR/build-third-party-install
+COMMON_CMAKE_ARGS="-DCMAKE_BUILD_TYPE=Release -DBUILD_RDMA=ON -DTHIRD_PARTY_DIR=$THIRD_PARTY_INSTALL_DIR"
 if [[ $SKIP_THIRD_PARTY != 1 ]]; then
     mkdir -p $THIRD_PARTY_BUILD_DIR
     pushd $THIRD_PARTY_BUILD_DIR
 
-    cmake -DTHIRD_PARTY=ON -DCMAKE_BUILD_TYPE=Release \
+    cmake -DTHIRD_PARTY=ON \
+        $COMMON_CMAKE_ARGS \
         -DONEFLOW=OFF \
-        -DTHIRD_PARTY_DIR=$THIRD_PARTY_INSTALL_DIR   \
         $ONEFLOW_SRC_DIR
     make -j nccl
     make -j`nproc` prepare_oneflow_third_party
@@ -77,15 +84,15 @@ do
     PY_ROOT=/opt/python/${PY_ABI}
     PY_BIN=${PY_ROOT}/bin/python
     cmake -DTHIRD_PARTY=OFF -DONEFLOW=ON\
+        $COMMON_CMAKE_ARGS \
         -DPython3_ROOT_DIR=$PY_ROOT \
-        -DCMAKE_BUILD_TYPE=Release  \
-        -DTHIRD_PARTY_DIR=$THIRD_PARTY_INSTALL_DIR   \
         $EXTRA_ONEFLOW_CMAKE_ARGS   \
         $ONEFLOW_SRC_DIR
     cmake --build . -j `nproc`
     popd
     trap cleanup EXIT
-    $PY_BIN setup.py bdist_wheel -d tmp_wheel --build_dir $ONEFLOW_BUILD_DIR
+    rm -rf $ONEFLOW_BUILD_DIR/python_scripts/*.egg-info
+    $PY_BIN setup.py bdist_wheel -d tmp_wheel --build_dir $ONEFLOW_BUILD_DIR --package_name $PACKAGE_NAME
     auditwheel repair tmp_wheel/*.whl --wheel-dir $HOUSE_DIR
     cleanup
 done
diff --git a/docker/package/manylinux/make_release.sh b/docker/package/manylinux/make_release.sh
@@ -0,0 +1,19 @@
+set -ex
+
+function release() {
+    set -ex
+    docker_tag=oneflow:rel-manylinux2014-cuda-$1
+    package_name=oneflow_cu`echo $1 | tr -d .`
+    docker build --build-arg from=nvidia/cuda:$1-cudnn7-devel-centos7 -f docker/package/manylinux/Dockerfile -t $docker_tag .
+    docker run --rm -it -v `pwd`:/oneflow-src -w /oneflow-src $docker_tag \
+        /oneflow-src/docker/package/manylinux/build_wheel.sh --cache-dir /oneflow-src/manylinux2014-build-cache-cuda-$1 \
+        --house-dir /oneflow-src/wheelhouse \
+        --package-name $package_name
+}
+
+release 10.2
+release 10.1
+release 10.0
+release 9.2
+release 9.1
+release 9.0
diff --git a/docs/source/troubleshooting.md b/docs/source/troubleshooting.md
@@ -1,12 +1,17 @@
 # Troubleshooting
 
 - `CUDNN_STATUS_NOT_INITIALIZED`
-    - You might see error message like this:
+    - You might see error message like these:
+        ```
+        I0729 22:37:45.483937439   56788 ev_epoll_linux.c:82]        Use of signals is disabled. Epoll enginll not be used
+        E0729 22:37:45.515343 56788 version.cpp:82] Failed to get cuda runtime version: CUDA driver version nsufficient for CUDA runtime version
+        F0729 22:38:31.209002 56788 improver.cpp:535] Check failed: mem_size > 0 (-524288000 vs. 0)
+        ```
         ```
         F0723 19:05:56.194067 40970 cuda_util.cpp:82] Check failed: error == CUDNN_STATUS_SUCCESS (1 vs. 0) CUDNN_STATUS_NOT_INITIALIZED
         ```
-    - Please upgrade to Nvidia Linux x86_64 driver version >= 440.33
-    - For more information, please refer to [cuda compatibility docs](https://docs.nvidia.com/deploy/cuda-compatibility/index.html).
+    - Please upgrade to Nvidia Linux x86_64 driver. Version >= 440.33 is recommended.
+    - For more information, please refer to [CUDA compatibility documentation](https://docs.nvidia.com/deploy/cuda-compatibility/index.html).
 
 - Failed to compile `.cu` files
     - Please refer to [CUDA System Requirements](https://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html#system-requirements) . Make sure your linux distribution and libraries shipped with it meet the requirements.
diff --git a/setup.py b/setup.py
@@ -30,6 +30,7 @@ def finalize_options(self):
     help="Package xla libraries if true, otherwise not."
 )
 parser.add_argument('--build_dir', type=str, default='build')
+parser.add_argument('--package_name', type=str, default='oneflow')
 args, remain_args = parser.parse_known_args()
 sys.argv = ['setup.py'] + remain_args
 
@@ -72,7 +73,7 @@ def has_ext_modules(self):
         raise Exception("Patchelf set rpath failed. command is: %s" % command)
 
 setup(
-    name='oneflow',
+    name=args.package_name,
     version='0.1.6',
     url='https://www.oneflow.org/',
     install_requires=REQUIRED_PACKAGES,
