[CD] Adds python docker pipeline (apache#16547)

* Adds python docker pipeline

* Refactors docker login functionality out of docker_cache script

tweak

* Refactors docker run functionality out of build.py

Test safe docker run tweaks

Fix up

* Adds CD docker utils

* Updates logging configuration to file config

* Adds restore static and dynamic methods, instead of making the library type a parameter

* Removes CUDA 9.1 from base images script

* Adds explanatory comment to signal masking in safe_docker_run

* Adds push success message with pull command and link to repository

* Fixes logging

cd/Jenkinsfile_cd_pipeline

@@ -61,9 +61,17 @@ pipeline {
               stage("Build") {
                 cd_utils.trigger_release_job("Build static libmxnet", "mxnet_lib/static", params.MXNET_VARIANTS)    
               }
-              stage("PyPI Release") {
-                echo "Building PyPI Release"
-                cd_utils.trigger_release_job("Release PyPI Packages", "python/pypi", params.MXNET_VARIANTS)
+              stage("Releases") {
+                cd_utils.error_checked_parallel([
+                  "PyPI Release": {
+                    echo "Building PyPI Release"
+                    cd_utils.trigger_release_job("Release PyPI Packages", "python/pypi", params.MXNET_VARIANTS)
+                  },
+                  "Python Docker Release": {
+                    echo "Building Python Docker Release"
+                    cd_utils.trigger_release_job("Release Python Docker Images", "python/docker", params.MXNET_VARIANTS)
+                  }
+                ])
               }
             },
 

cd/Jenkinsfile_release_job

@@ -92,7 +92,8 @@ pipeline {
           def valid_job_types = [
             "mxnet_lib/static",
             "mxnet_lib/dynamic", 
-            "python/pypi"
+            "python/pypi",
+            "python/docker"
           ]
 
           // Convert mxnet variants to a list

cd/Jenkinsfile_utils.groovy

@@ -160,6 +160,18 @@ def restore_artifact(variant, libtype) {
   }
 }
 
+
+// Restores the statically linked libmxnet for the given variant
+def restore_static_libmxnet(variant) {
+  restore_artifact(variant, 'static')
+}
+
+
+// Restores the dynamically linked libmxnet for the given variant
+def restore_dynamic_libmxnet(variant) {
+  restore_artifact(variant, 'dynamic')
+}
+
 // A generic pipeline that can be used by *most* CD jobs
 // It can be used when implementing the pipeline steps in the Jenkins_steps.groovy
 // script for a particular delivery channel. However, it should also implement the

cd/python/docker/Dockerfile

@@ -0,0 +1,40 @@
+# -*- mode: dockerfile -*-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Python MXNet Dockerfile
+
+# NOTE: Assumes wheel_build directory is the context root when building
+
+ARG BASE_IMAGE
+FROM ${BASE_IMAGE}
+
+ARG PYTHON_CMD=python
+RUN apt-get update && \
+    apt-get install -y wget ${PYTHON_CMD}-dev gcc && \
+    wget https://bootstrap.pypa.io/get-pip.py && \
+    ${PYTHON_CMD} get-pip.py
+
+ARG MXNET_COMMIT_ID
+ENV MXNET_COMMIT_ID=${MXNET_COMMIT_ID}
+
+RUN mkdir -p /mxnet
+COPY dist/*.whl /mxnet/.
+
+WORKDIR /mxnet
+RUN WHEEL_FILE=$(ls -t /mxnet | head -n 1) && pip install ${WHEEL_FILE} && rm -f ${WHEEL_FILE}
+

cd/python/docker/Dockerfile.test

@@ -0,0 +1,39 @@
+# -*- mode: dockerfile -*-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Python MXNet Dockerfile
+
+# NOTE: Assumes 'ci' directory is root of the context when building
+
+ARG BASE_IMAGE
+FROM ${BASE_IMAGE}
+
+# Install test dependencies
+RUN pip install nose
+
+ARG USER_ID=1001
+ARG GROUP_ID=1001
+
+COPY ./docker/install/ubuntu_adduser.sh /work/ubuntu_adduser.sh
+COPY ./docker/install/requirements /work/requirements
+
+RUN mkdir -p /work
+RUN /work/ubuntu_adduser.sh
+RUN pip install -r /work/requirements
+
+WORKDIR /work/mxnet

cd/python/docker/Jenkins_pipeline.groovy

@@ -0,0 +1,74 @@
+// -*- mode: groovy -*-
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+// Jenkins pipeline
+// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/
+
+// NOTE: 
+// ci_utils and cd_utils are loaded by the originating Jenkins job, e.g. jenkins/Jenkinsfile_release_job
+
+def get_pipeline(mxnet_variant) {
+  def node_type = mxnet_variant.startsWith('cu') ? NODE_LINUX_GPU : NODE_LINUX_CPU
+  return cd_utils.generic_pipeline(mxnet_variant, this, node_type)
+}
+
+// Returns the (Docker) environment for the given variant
+// The environment corresponds to the docker files in the 'docker' directory
+def get_environment(mxnet_variant) {
+  if (mxnet_variant.startsWith("cu")) {
+    // Remove 'mkl' suffix from variant to properly format test environment
+    return "ubuntu_gpu_${mxnet_variant.replace('mkl', '')}"
+  }
+  return "ubuntu_cpu"
+}
+
+
+def build(mxnet_variant) {
+  ws("workspace/python_docker/${mxnet_variant}/${env.BUILD_NUMBER}") {
+    ci_utils.init_git()
+    cd_utils.restore_static_libmxnet(mxnet_variant)
+
+    // package wheel file
+    def nvidia_docker = mxnet_variant.startsWith('cu')
+    def environment = get_environment(mxnet_variant)
+    ci_utils.docker_run(environment, "cd_package_pypi ${mxnet_variant}", nvidia_docker)
+
+    // build python docker images
+    sh "./cd/python/docker/python_images.sh build ${mxnet_variant} py3"
+    sh "./cd/python/docker/python_images.sh build ${mxnet_variant} py2"
+  }
+}
+
+def test(mxnet_variant) {
+  ws("workspace/python_docker/${mxnet_variant}/${env.BUILD_NUMBER}") {
+    // test python docker images
+    sh "./cd/python/docker/python_images.sh test ${mxnet_variant} py3"
+    sh "./cd/python/docker/python_images.sh test ${mxnet_variant} py2"
+  }
+}
+
+def push(mxnet_variant) {
+  ws("workspace/python_docker/${mxnet_variant}/${env.BUILD_NUMBER}") {
+    // push python docker images
+    sh "./cd/python/docker/python_images.sh push ${mxnet_variant} py3"
+    sh "./cd/python/docker/python_images.sh push ${mxnet_variant} py2"
+  }
+}
+
+return this

cd/python/docker/python_images.sh

@@ -0,0 +1,128 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Executes mxnet python images pipeline functions: build, test, publish
+# Assumes script is run from the root of the mxnet repository
+# Assumes script is being run within MXNet CD infrastructure
+
+set -xe
+
+usage="Usage: python_images.sh <build|test|publish> MXNET-VARIANT <py2|py3>"
+
+command=${1:?$usage}
+mxnet_variant=${2:?$usage}
+python_version=${3:?usage}
+
+cd_utils='cd/utils'
+ci_utils='ci/'
+
+case ${python_version} in
+    py3)
+        python_cmd="python3"
+        ;;
+    py2)
+        python_cmd="python"
+        ;;
+    *)
+        echo "Error: specify python version with either 'py2' or 'py3'"
+        exit 1
+        ;;
+esac
+
+docker_tags=($(./${cd_utils}/docker_tag.sh ${mxnet_variant}))
+main_tag="${docker_tags[0]}_${python_version}"
+base_image=$(./${cd_utils}/mxnet_base_image.sh ${mxnet_variant})
+repository="python"
+image_name="${repository}:${main_tag}"
+
+resources_path='cd/python/docker'
+
+if [ ! -z "${RELEASE_DOCKERHUB_REPOSITORY}" ]; then
+    image_name="${RELEASE_DOCKERHUB_REPOSITORY}/${image_name}"
+fi
+
+build() {
+    # NOTE: Ensure the correct context root is passed in when building - Dockerfile expects ./wheel_build
+    docker build -t "${image_name}" --build-arg PYTHON_CMD=${python_cmd} --build-arg BASE_IMAGE="${base_image}" --build-arg MXNET_COMMIT_ID=${GIT_COMMIT} -f ${resources_path}/Dockerfile ./wheel_build
+}
+
+test() {
+    local runtime_param=""
+    if [[ ${mxnet_variant} == cu* ]]; then
+        runtime_param="--runtime=nvidia"
+    fi
+    local test_image_name="${image_name}_test"
+
+    # Ensure the correct context root is passed in when building - Dockerfile.test expects ci directory
+    docker build -t "${test_image_name}" --build-arg USER_ID=`id -u` --build-arg GROUP_ID=`id -g` --build-arg BASE_IMAGE="${image_name}" -f ${resources_path}/Dockerfile.test ./ci
+    ./ci/safe_docker_run.py ${runtime_param} --cap-add "SYS_PTRACE" -u `id -u`:`id -g` -v `pwd`:/work/mxnet "${test_image_name}" ${resources_path}/test_python_image.sh "${mxnet_variant}" "${python_cmd}"
+}
+
+push() {
+    if [ -z "${RELEASE_DOCKERHUB_REPOSITORY}" ]; then
+        echo "Cannot publish image without RELEASE_DOCKERHUB_REPOSITORY environment variable being set."
+        exit 1
+    fi
+
+    # The secret name env var is set in the Jenkins configuration
+    # Manage Jenkins -> Configure System
+    ./${ci_utils}/docker_login.py --secret-name "${RELEASE_DOCKERHUB_SECRET_NAME}"
+
+    # Push image
+    docker push "${image_name}"
+
+    # Iterate over remaining tags, if any
+    for ((i=1;i<${#docker_tags[@]};i++)); do
+        local docker_tag="${docker_tags[${i}]}"
+        local latest_image_name="${RELEASE_DOCKERHUB_REPOSITORY}/${repository}:${docker_tag}"
+
+        # latest and latest gpu should only be pushed for py3
+        if [[ ${docker_tag} == "latest" || ${docker_tag} == "latest_gpu" ]]; then
+            if [[ ${python_version} == "py2" ]]; then
+                continue
+            fi
+        else
+            latest_image_name="${latest_image_name}_${python_version}"
+        fi
+
+        docker tag "${image_name}" "${latest_image_name}"
+        docker push "${latest_image_name}"
+        echo "Successfully pushed ${latest_image_name}. Pull it with:"
+        echo "docker pull ${latest_image_name}"
+        echo "For a complete list of tags see https://hub.docker.com/u/${RELEASE_DOCKERHUB_REPOSITORY}/${repository}"
+    done    
+}
+
+case ${command} in
+    "build")
+        build
+        ;;
+
+    "test")
+        test
+        ;;
+
+    "push")
+        push
+        ;;
+
+    *)
+        echo $usage
+        exit 1
+esac

cd/python/docker/test_python_image.sh

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# To be run _within_ a runtime image
+# Tests the Runtime docker image
+# Assumes the mxnet source directory is mounted on /mxnet and cwd is /mxnet
+
+set -ex
+
+# Variant parameter should be passed in
+mxnet_variant=${1:?"Missing mxnet variant"}
+python_cmd=${2:?"Missing python version (python or python3)"}
+
+if [ -z "${MXNET_COMMIT_ID}" ]; then
+    echo "MXNET_COMMIT_ID environment variable is empty. Please rebuild the image with MXNET_COMMIT_ID build-arg specified."
+    exit 1
+fi
+
+# Execute tests
+if [[ $mxnet_variant == cu* ]]; then
+    mnist_params="--gpu 0"
+    test_conv_params="--gpu"
+fi
+
+if [[ $mxnet_variant == *mkl ]]; then
+    ${python_cmd} tests/python/mkl/test_mkldnn.py
+fi
+
+${python_cmd} tests/python/train/test_conv.py ${test_conv_params}
+${python_cmd} example/image-classification/train_mnist.py ${mnist_params}
+

cd/python/pypi/Jenkins_pipeline.groovy

@@ -45,7 +45,7 @@ def get_environment(mxnet_variant) {
 def build(mxnet_variant) {
   ws("workspace/python_pypi/${mxnet_variant}/${env.BUILD_NUMBER}") {
     ci_utils.init_git()
-    cd_utils.restore_artifact(mxnet_variant, 'static')
+    cd_utils.restore_static_libmxnet(mxnet_variant)
 
     // create wheel file
     def environment = get_environment(mxnet_variant)