Updated docker build again

yephm · Apr 23, 2021 · 3bf0e14 · 3bf0e14
1 parent fadf10d
commit 3bf0e14
Show file tree

Hide file tree

Showing 12 changed files with 109 additions and 258 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -63,15 +63,15 @@ endif()
 
 
 set(CUDA_ARCH
-    # -gencode=arch=compute_35,code=sm_35
+    -gencode=arch=compute_35,code=sm_35
     # -gencode=arch=compute_50,code=sm_50
     # -gencode=arch=compute_52,code=sm_52
     # -gencode=arch=compute_60,code=sm_60
     # -gencode=arch=compute_61,code=sm_61
     # -gencode=arch=compute_70,code=sm_70
     # -gencode=arch=compute_75,code=sm_75
-    -gencode=arch=compute_80,code=sm_80
-    -gencode=arch=compute_86,code=sm_86
+    # -gencode=arch=compute_80,code=sm_80
+    # -gencode=arch=compute_86,code=sm_86
 )
 
 set(CUDA_OPTS
@@ -97,12 +97,6 @@ set(CMAKE_CUDA_FLAGS "${NVCC_FLAGS_STR}")
 find_package(OpenMP REQUIRED)
 #-------------------------------------------------------------------------------
 
-# BLAS configuration
-#-------------------------------------------------------------------------------
-find_package(BLAS REQUIRED)
-find_package(LAPACK REQUIRED)
-#-------------------------------------------------------------------------------
-
 # GTEST Configuration
 #-------------------------------------------------------------------------------
 find_package(gflags REQUIRED)
@@ -185,22 +179,33 @@ add_library(tsnecuda SHARED ${SOURCES})
 target_link_libraries(tsnecuda
     gflags
     gtest
-    # glog
     CUDA::cudart
     CUDA::cublas
     CUDA::cublasLt
     CUDA::cufft
     CUDA::cufftw
     CUDA::cusparse
-    ${BLAS_LIBRARIES}
-    LAPACK::LAPACK
     OpenMP::OpenMP_CXX
     pthread
     -Wl,--allow-multiple-definition
     ${ZMQ_LIBRARIES}
     ${FAISS_LIBRARIES}
 )
 
+# BLAS configuration
+#-------------------------------------------------------------------------------
+find_package(MKL)
+if(MKL_FOUND)
+  target_link_libraries(tsnecuda PRIVATE ${MKL_LIBRARIES})
+else()
+  find_package(BLAS REQUIRED)
+  target_link_libraries(tsnecuda PRIVATE ${BLAS_LIBRARIES})
+
+  find_package(LAPACK REQUIRED)
+  target_link_libraries(tsnecuda PRIVATE ${LAPACK_LIBRARIES})
+endif()
+#-------------------------------------------------------------------------------
+
 #-------------------------------------------------------------------------------
 # Main executable
 #-------------------------------------------------------------------------------

diff --git a/conda/Dockerfile.cuda10.1 b/conda/Dockerfile.cuda10.1
@@ -0,0 +1,25 @@
+# Copyright (c) 2021 Regents of the University of California
+#
+# This software is released under the MIT License.
+# https://opensource.org/licenses/MIT
+
+
+FROM nvidia/cuda:10.1-devel-ubuntu18.04
+
+RUN apt-get update && apt-get install -y --allow-downgrades --allow-change-held-packages wget git libcublas-dev=10.1.0.105-1
+
+RUN wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \
+    bash Miniconda3-latest-Linux-x86_64.sh -b -p ~/miniconda3
+
+ENV PATH="/root/miniconda3/condabin:${PATH}"
+
+RUN conda install -y -q conda-build anaconda-client
+RUN conda config --set anaconda_upload yes
+RUN conda init
+
+RUN sh -c 'cp /usr/include/cublas*.h /usr/local/cuda/include/'
+
+ADD ./ /tsnecuda/
+WORKDIR /tsnecuda/conda
+
+ENTRYPOINT conda-build ./tsnecuda/ --variants '{ "cudatoolkit": "10.1", "python":"3.8" }' -c pytorch -c conda-forge
diff --git a/conda/Dockerfile.cuda10.2 b/conda/Dockerfile.cuda10.2
@@ -2,3 +2,21 @@
 #
 # This software is released under the MIT License.
 # https://opensource.org/licenses/MIT
+
+FROM nvidia/cuda:10.2-devel-ubuntu18.04
+
+RUN apt-get update && apt-get install -y wget git libcublas-dev
+
+RUN wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \
+    bash Miniconda3-latest-Linux-x86_64.sh -b -p ~/miniconda3
+
+ENV PATH="/root/miniconda3/condabin:${PATH}"
+
+RUN conda install -y -q conda-build anaconda-client
+RUN conda config --set anaconda_upload yes
+RUN conda init
+
+ADD ./ /tsnecuda/
+WORKDIR /tsnecuda/conda
+
+# ENTRYPOINT conda-build ./tsnecuda/ --variants '{ "cudatoolkit": "10.2", "python":"3.8" }' -c pytorch -c conda-forge
diff --git a/conda/Dockerfile.cuda11.0 b/conda/Dockerfile.cuda11.0
@@ -2,3 +2,21 @@
 #
 # This software is released under the MIT License.
 # https://opensource.org/licenses/MIT
+
+FROM nvidia/cuda:11.0.3-devel-ubuntu18.04
+
+RUN apt-get update && apt-get install -y wget git libcublas-dev-11-0
+
+RUN wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \
+        bash Miniconda3-latest-Linux-x86_64.sh -b -p ~/miniconda3
+
+ENV PATH="/root/miniconda3/condabin:${PATH}"
+
+RUN conda install -y -q conda-build anaconda-client
+RUN conda config --set anaconda_upload yes
+RUN conda init
+
+ADD ./ /tsnecuda/
+WORKDIR /tsnecuda/conda
+
+# ENTRYPOINT conda-build ./tsnecuda/ --variants '{ "cudatoolkit": "11.0", "python":"3.8" }' -c pytorch -c conda-forge
diff --git a/conda/Dockerfile.cuda11.2 b/conda/Dockerfile.cuda11.2
@@ -2,17 +2,3 @@
 #
 # This software is released under the MIT License.
 # https://opensource.org/licenses/MIT
-
-FROM nvidia/cuda:11.0.3-devel-ubuntu18.04
-
-RUN apt-get update && apt-get install -y wget git libcublas-dev-11-0
-
-RUN wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \
-        bash Miniconda3-latest-Linux-x86_64.sh -b -p ~/miniconda3
-ENV PATH="/root/miniconda3/condabin:${PATH}"
-
-RUN conda install -y -q conda-build anaconda-client
-RUN conda config --set anaconda_upload yes
-
-COPY ./ tsnecuda
-WORKDIR /tsnecuda/conda
diff --git a/conda/README.md b/conda/README.md
@@ -6,3 +6,20 @@
 -->
 
 # Building/Packaging for Anaconda
+
+## Step 1 - Build the docker images
+
+```
+docker build . -f ./conda/Dockerfile.cuda11.0 -t tsnecuda-docker-11.0
+docker run -i -t tsnecuda-docker-11.0:latest
+```
+
+## Step 2 - Build the conda package
+
+```
+conda-build ./tsnecuda/ --variants '{ "cudatoolkit": "11.0","python":"3.8" }' -c pytorch -c conda-forge
+```
+
+```
+conda debug ./tsnecuda/ --variants '{ "cudatoolkit": "11.0","python":"3.8" }' -c pytorch -c conda-forge
+```
diff --git a/conda/tsnecuda/build.sh b/conda/tsnecuda/build.sh
@@ -6,10 +6,8 @@
 set -e
 set -x
 
-git submodule init
-git submodule update
-
-cmake -B _build .
+CMAKE_PLATFORM_FLAGS+=(-DCMAKE_TOOLCHAIN_FILE="${RECIPE_DIR}/cross-linux.cmake")
+cmake -B _build . -DCMAKE_INSTALL_PREFIX=${PREFIX} ${CMAKE_PLATFORM_FLAGS[@]}
 make -C _build -j $CPU_COUNT
 
 cd _build/python/

diff --git a/cross-linux.cmake → conda/tsnecuda/cross-linux.cmake b/cross-linux.cmake → conda/tsnecuda/cross-linux.cmake
@@ -5,7 +5,7 @@ set(CMAKE_PLATFORM Linux)
 set(CMAKE_SYSTEM_VERSION 1)
 
 # specify the cross compiler
-# set(CMAKE_C_COMPILER $ENV{CC})
+set(CMAKE_C_COMPILER $ENV{CC})
 
 # where is the target environment
 set(CMAKE_FIND_ROOT_PATH $ENV{PREFIX} $ENV{BUILD_PREFIX}/$ENV{HOST}/sysroot)

diff --git a/conda/tsnecuda/meta.yaml b/conda/tsnecuda/meta.yaml
@@ -39,24 +39,31 @@ outputs:
       string: "h{{ PKG_HASH }}_{{ number }}_cuda{{ cudatoolkit }}"
       run_exports:
         - {{ pin_compatible('tsnecuda', exact=True) }}
+
     requirements:
       build:
         - {{ compiler('cxx') }}
-        - python {{ python }}
-        - cmake >=3.20
         - make
         - git
-      host:
+        - cmake >=3.20
         - mkl-devel >=2018
         - blas =*=mkl
-        - cudatoolkit {{ cudatoolkit }}
+        - cudatoolkit-dev {{ cudatoolkit }}
         - faiss-gpu ==1.6.5
         - python {{ python }}
         - numpy >=1.14
+        - gflags
+        - gtest
+        - gmock
+        - libxml2
       run:
         - mkl >=2018
         - blas =*=mkl
         - faiss-gpu ==1.6.5
         - python {{ python }}
+        - gflags
+        - gtest
+        - gmock
+        - libxml2
         - {{ pin_compatible('cudatoolkit', max_pin='x.x') }}
         - {{ pin_compatible('numpy') }}