Merge pull request Kaggle#517 from Kaggle/upgrade-tf-cuda-ci

Upgrade to cuda 10.0 and Tensorflow 1.13

Author: rosbo

Dockerfile

@@ -1,6 +1,6 @@
 ARG BASE_TAG=5.2.0
 
-FROM gcr.io/kaggle-images/python-tensorflow-whl:1.12.0-py36 as tensorflow_whl
+FROM gcr.io/kaggle-images/python-tensorflow-whl:1.13.1-py36 as tensorflow_whl
 FROM continuumio/anaconda3:${BASE_TAG}
 
 ADD clean-layer.sh  /tmp/clean-layer.sh
@@ -21,7 +21,7 @@ RUN sed -i "s/httpredir.debian.org/debian.uchicago.edu/" /etc/apt/sources.list &
     apt-get -y install cmake && \
     /tmp/clean-layer.sh
 
-# Tensorflow doesn't support python 3.7 yet. See https://github.com/tensorflow/tensorflow/issues/20517
+# Tensorflow doesn't support python 3.7 yet. See https://github.com/tensorflow/tensorflow/issues/20517	
 # Fix to install Tensorflow: Downgrade python 3.7->3.6.6.
 RUN conda install -y python=3.6.6 && \
     /tmp/clean-layer.sh

gpu.Dockerfile

@@ -1,7 +1,7 @@
 ARG BASE_TAG=staging
 
 FROM nvidia/cuda:9.2-cudnn7-devel-ubuntu16.04 AS nvidia
-FROM gcr.io/kaggle-images/python-tensorflow-whl:1.12.0-py36 as tensorflow_whl
+FROM gcr.io/kaggle-images/python-tensorflow-whl:1.13.1-py36 as tensorflow_whl
 FROM gcr.io/kaggle-images/python:${BASE_TAG}
 
 ADD clean-layer.sh  /tmp/clean-layer.sh
@@ -13,8 +13,8 @@ COPY --from=nvidia /etc/apt/trusted.gpg /etc/apt/trusted.gpg.d/cuda.gpg
 
 # Ensure the cuda libraries are compatible with the custom Tensorflow wheels.
 # TODO(b/120050292): Use templating to keep in sync or COPY installed binaries from it.
-ENV CUDA_VERSION=9.2.148
-ENV CUDA_PKG_VERSION=9-2=$CUDA_VERSION-1
+ENV CUDA_VERSION=10.0.130
+ENV CUDA_PKG_VERSION=10-0=$CUDA_VERSION-1
 LABEL com.nvidia.volumes.needed="nvidia_driver"
 LABEL com.nvidia.cuda.version="${CUDA_VERSION}"
 ENV PATH=/usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH}
@@ -26,7 +26,7 @@ ENV PATH=/usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH}
 ENV LD_LIBRARY_PATH="/usr/local/nvidia/lib64:/usr/local/cuda/lib64:/usr/local/cuda/lib64/stubs"
 ENV NVIDIA_VISIBLE_DEVICES=all
 ENV NVIDIA_DRIVER_CAPABILITIES=compute,utility
-ENV NVIDIA_REQUIRE_CUDA="cuda>=9.2"
+ENV NVIDIA_REQUIRE_CUDA="cuda>=10.0"
 RUN apt-get update && apt-get install -y --no-install-recommends \
       cuda-cupti-$CUDA_PKG_VERSION \
       cuda-cudart-$CUDA_PKG_VERSION \
@@ -36,11 +36,11 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
       cuda-nvml-dev-$CUDA_PKG_VERSION \
       cuda-minimal-build-$CUDA_PKG_VERSION \
       cuda-command-line-tools-$CUDA_PKG_VERSION \
-      libcudnn7=7.4.1.5-1+cuda9.2 \
-      libcudnn7-dev=7.4.1.5-1+cuda9.2 \
-      libnccl2=2.3.7-1+cuda9.2 \
-      libnccl-dev=2.3.7-1+cuda9.2 && \
-    ln -s /usr/local/cuda-9.2 /usr/local/cuda && \
+      libcudnn7=7.4.2.24-1+cuda10.0 \
+      libcudnn7-dev=7.4.2.24-1+cuda10.0 \
+      libnccl2=2.4.2-1+cuda10.0 \
+      libnccl-dev=2.4.2-1+cuda10.0 && \
+    ln -s /usr/local/cuda-10.0 /usr/local/cuda && \
     ln -s /usr/local/cuda/lib64/stubs/libcuda.so /usr/local/cuda/lib64/stubs/libcuda.so.1 && \
     /tmp/clean-layer.sh
 
@@ -51,15 +51,15 @@ RUN pip uninstall -y tensorflow && \
     pip install /tmp/tensorflow_gpu/tensorflow*.whl && \
     rm -rf /tmp/tensorflow_gpu && \
     conda uninstall -y pytorch-cpu torchvision-cpu && \
-    conda install -y pytorch torchvision cudatoolkit=9.2 -c pytorch && \
+    conda install -y pytorch torchvision cudatoolkit=10.0 -c pytorch && \
     pip uninstall -y mxnet && \
     # b/126259508 --no-deps prevents numpy from being downgraded.
-    pip install --no-deps mxnet-cu92 && \
+    pip install --no-deps mxnet-cu100 && \
     /tmp/clean-layer.sh
 
 # Install GPU-only packages
 RUN pip install pycuda && \
-    pip install cupy-cuda92 && \
+    pip install cupy-cuda100 && \
     pip install pynvrtc && \
     /tmp/clean-layer.sh
 

tensorflow-whl/CHANGELOG.md

@@ -1,2 +1,3 @@
 1.11.0-py36: Tensorflow 1.11.0 wheels built with python 3.6
 1.12.0-py36: Tensorflow 1.12.0 wheels with Cuda 9.2
+1.13.1-py36: Tensorflow 1.13.1 wheels with Cuda 10.0

tensorflow-whl/Dockerfile

@@ -1,4 +1,4 @@
-FROM nvidia/cuda:9.2-cudnn7-devel-ubuntu16.04 AS nvidia
+FROM nvidia/cuda:10.0-cudnn7-devel-ubuntu16.04 AS nvidia
 FROM continuumio/anaconda3:5.2.0
 
 # Avoid interactive configuration prompts/dialogs during apt-get.
@@ -15,8 +15,8 @@ COPY --from=nvidia /etc/apt/trusted.gpg /etc/apt/trusted.gpg.d/cuda.gpg
 
 # Ensure the cuda libraries are compatible with the GPU image.
 # TODO(b/120050292): Use templating to keep in sync.
-ENV CUDA_VERSION=9.2.148
-ENV CUDA_PKG_VERSION=9-2=$CUDA_VERSION-1
+ENV CUDA_VERSION=10.0.130
+ENV CUDA_PKG_VERSION=10-0=$CUDA_VERSION-1
 LABEL com.nvidia.volumes.needed="nvidia_driver"
 LABEL com.nvidia.cuda.version="${CUDA_VERSION}"
 ENV PATH=/usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH}
@@ -28,7 +28,7 @@ ENV PATH=/usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH}
 ENV LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/local/nvidia/lib64:/usr/local/cuda/lib64:/usr/local/cuda/lib64/stubs"
 ENV NVIDIA_VISIBLE_DEVICES=all
 ENV NVIDIA_DRIVER_CAPABILITIES=compute,utility
-ENV NVIDIA_REQUIRE_CUDA="cuda>=9.0"
+ENV NVIDIA_REQUIRE_CUDA="cuda>=10.0"
 RUN apt-get update && apt-get install -y --no-install-recommends \
       cuda-cupti-$CUDA_PKG_VERSION \
       cuda-cudart-$CUDA_PKG_VERSION \
@@ -38,18 +38,17 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
       cuda-nvml-dev-$CUDA_PKG_VERSION \
       cuda-minimal-build-$CUDA_PKG_VERSION \
       cuda-command-line-tools-$CUDA_PKG_VERSION \
-      libcudnn7=7.4.1.5-1+cuda9.2 \
-      libcudnn7-dev=7.4.1.5-1+cuda9.2 \
-      libnccl2=2.3.7-1+cuda9.2 \
-      libnccl-dev=2.3.7-1+cuda9.2 && \
-    ln -s /usr/local/cuda-9.2 /usr/local/cuda && \
-    ln -s /usr/local/cuda/lib64/stubs/libcuda.so /usr/local/cuda/lib64/stubs/libcuda.so.1 && \
-    rm -rf /var/lib/apt/lists/*
+      libcudnn7=7.5.0.56-1+cuda10.0 \
+      libcudnn7-dev=7.5.0.56-1+cuda10.0 \
+      libnccl2=2.4.2-1+cuda10.0 \
+      libnccl-dev=2.4.2-1+cuda10.0 && \
+    ln -s /usr/local/cuda-10.0 /usr/local/cuda && \
+    ln -s /usr/local/cuda/lib64/stubs/libcuda.so /usr/local/cuda/lib64/stubs/libcuda.so.1
 
 # Install bazel
-# Tensorflow 1.11 requires the Bazel 0.15: https://www.tensorflow.org/install/source
-ENV BAZEL_VERSION=0.15.0
-RUN apt-get update && apt-get install -y gnupg zip && \
+# Tensorflow 1.13 requires the Bazel 0.19.2: https://www.tensorflow.org/install/source
+ENV BAZEL_VERSION=0.19.2
+RUN apt-get install -y gnupg zip && \
     echo "deb http://ppa.launchpad.net/webupd8team/java/ubuntu precise main" | tee -a /etc/apt/sources.list && \
     echo "deb-src http://ppa.launchpad.net/webupd8team/java/ubuntu precise main" | tee -a /etc/apt/sources.list && \
     apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys --no-tty EEA14886 C857C906 2B90D010 && \
@@ -68,11 +67,13 @@ RUN apt-get update && apt-get install -y gnupg zip && \
 # Tensorflow doesn't support python 3.7 yet. See https://github.com/tensorflow/tensorflow/issues/20517
 RUN conda install -y python=3.6.6
 
-# Fetch tensorflow
+# Fetch tensorflow & install dependencies.
 RUN cd /usr/local/src && \
     git clone https://github.com/tensorflow/tensorflow && \
     cd tensorflow && \
-    git checkout r1.12
+    git checkout r1.13 && \
+    pip install keras_applications --no-deps && \
+    pip install keras_preprocessing --no-deps
 
 # Create a tensorflow wheel for CPU
 RUN cd /usr/local/src/tensorflow && \
@@ -83,9 +84,9 @@ RUN cd /usr/local/src/tensorflow && \
 
 # Create a tensorflow wheel for GPU/cuda
 ENV TF_NEED_CUDA=1
-ENV TF_CUDA_VERSION=9.2
-# 3.7 is for the K80 and 6.0 is for the P100: https://developer.nvidia.com/cuda-gpus
-ENV TF_CUDA_COMPUTE_CAPABILITIES=3.7,6.0
+ENV TF_CUDA_VERSION=10.0
+# 3.7 is for the K80 and 6.0 is for the P100, 7.5 is for the T4: https://developer.nvidia.com/cuda-gpus
+ENV TF_CUDA_COMPUTE_CAPABILITIES=3.7,6.0,7.5
 ENV TF_CUDNN_VERSION=7
 ENV TF_NCCL_VERSION=2
 ENV NCCL_INSTALL_PATH=/usr/