Update docker build script to optionally publish images to DockerHub

bench/build_bench_docker_images.sh

@@ -26,15 +26,15 @@ fi
 
 docker build -t clipper/py-rpc:$tag -f ./RPCDockerfile ./
 if [ $# -eq 0 ]; then
-	time docker build build --build-arg CODE_VERSION=$tag -t clipper/sum-bench:$tag -f SumBenchDockerfile ./
-	time docker build build --build-arg CODE_VERSION=$tag -t clipper/noop-bench:$tag -f NoopBenchDockerfile ./
+	time docker build build --build-arg CODE_VERSION=$tag -t clipper/sum-bench:$tag -f dockerfiles/SumBenchDockerfile ./
+	time docker build build --build-arg CODE_VERSION=$tag -t clipper/noop-bench:$tag -f dockerfiles/NoopBenchDockerfile ./
 else
 	echo $1
 	echo $2
 	echo $3
 	echo $4
-	time docker build build --build-arg CODE_VERSION=$tag -t clipper/sum-bench:$tag -f SumBenchDockerfile ./ --build-arg MODEL_NAME="$1" --build-arg MODEL_VERSION="$2"
-	time docker build build --build-arg CODE_VERSION=$tag -t clipper/noop-bench:$tag -f NoopBenchDockerfile ./ --build-arg MODEL_NAME="$3" --build-arg MODEL_VERSION="$4"
+	time docker build build --build-arg CODE_VERSION=$tag -t clipper/sum-bench:$tag -f dockerfiles/SumBenchDockerfile ./ --build-arg MODEL_NAME="$1" --build-arg MODEL_VERSION="$2"
+	time docker build build --build-arg CODE_VERSION=$tag -t clipper/noop-bench:$tag -f dockerfiles/NoopBenchDockerfile ./ --build-arg MODEL_NAME="$3" --build-arg MODEL_VERSION="$4"
 fi
 
 cd -

bin/build_docker_images.sh

@@ -1,5 +1,14 @@
 #!/usr/bin/env bash
 
+# This script builds all the Clipper Docker images and publishes the useful ones to
+# Docker Hub (images that we don't publish are things like the unittest image and the
+# shared dependency base that don't really make sense to distribute on their own).
+# Each image that is built is tagged with two tags:
+# + The current Git SHA: <image_name>:git_hash
+# + The current version as read from VERSION.txt: <image_name>:version
+# For the images that we publish, both tags will be pushed.
+
+
 set -e
 set -u
 set -o pipefail
@@ -10,39 +19,102 @@ unset CDPATH
 # the script.
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 
-
 # Let the user start this script from anywhere in the filesystem.
-cd $DIR/..
-
-tag=$(<VERSION.txt)
-
-# Build the Clipper Docker images
-time docker build -t clipper/lib_base:$tag -f ./ClipperLibBaseDockerfile ./
-time docker build --build-arg CODE_VERSION=$tag -t clipper/query_frontend:$tag -f QueryFrontendDockerfile ./
-time docker build --build-arg CODE_VERSION=$tag -t clipper/management_frontend:$tag -f ManagementFrontendDockerfile ./
-time docker build --build-arg CODE_VERSION=$tag -t clipper/unittests:$tag -f ClipperTestsDockerfile ./
-
-# Tag and push the latest version of the Clipper Docker images to the container registry
-# for the Kubernetes testing cluster
-docker tag clipper/query_frontend:$tag 568959175238.dkr.ecr.us-west-1.amazonaws.com/clipper/query_frontend:$tag
-docker push 568959175238.dkr.ecr.us-west-1.amazonaws.com/clipper/query_frontend:$tag
-docker tag clipper/management_frontend:$tag 568959175238.dkr.ecr.us-west-1.amazonaws.com/clipper/management_frontend:$tag
-docker push 568959175238.dkr.ecr.us-west-1.amazonaws.com/clipper/management_frontend:$tag
-cd -
-
-# Build Spark JVM Container
-cd $DIR/../containers/jvm
-time docker build -t clipper/spark-scala-container:$tag -f SparkScalaContainerDockerfile ./
-cd -
-
-# Build the Python model containers
-cd $DIR/..
-
-# first build base image
-time docker build -t clipper/py-rpc:$tag -f ./RPCDockerfile ./
-time docker build --build-arg CODE_VERSION=$tag -t clipper/sum-container:$tag -f ./SumDockerfile ./
-time docker build --build-arg CODE_VERSION=$tag -t clipper/noop-container:$tag -f ./NoopDockerfile ./
-time docker build --build-arg CODE_VERSION=$tag -t clipper/python-closure-container:$tag -f ./PyClosureContainerDockerfile ./
-time docker build --build-arg CODE_VERSION=$tag -t clipper/pyspark-container:$tag -f ./PySparkContainerDockerfile ./
-time docker build --build-arg CODE_VERSION=$tag -t clipper/tf_cifar_container:$tag -f ./TensorFlowCifarDockerfile ./
-cd -
+CLIPPER_ROOT=$DIR/..
+cd $CLIPPER_ROOT
+
+# Initialize tags
+version_tag=$(<VERSION.txt)
+sha_tag=`git rev-parse --verify --short HEAD`
+
+namespace="clipper"
+
+# We build images with the SHA tag to try to prevent clobbering other images
+# being built from different branches on the same machine. This is particularly
+# useful for running these scripts on the Jenkins build cluster.
+create_image () {
+    local image=$1          # Param: The name of the image to build
+
+    local dockerfile=$2     # the Dockerfile name within the
+                            # <clipper_root>/dockerfiles directory
+
+    local public=$3        # Push the built images to Docker Hub under
+                            # the clipper namespace. Must have credentials.
+
+    echo "Building $namespace/$image:$sha_tag from file $dockerfile"
+    time docker build --build-arg CODE_VERSION=$sha_tag -t $namespace/$image:$sha_tag \
+        -f dockerfiles/$dockerfile $CLIPPER_ROOT
+    docker tag $namespace/$image:$sha_tag $namespace/$image:$version_tag
+
+    if [ "$publish" = true ] && [ "$public" = true ] ; then
+        echo "Publishing $namespace/$image:$sha_tag"
+        docker push $namespace/$image:$sha_tag
+        echo "Publishing $namespace/$image:$version_tag"
+        docker push $namespace/$image:$version_tag
+    fi
+}
+
+
+# Build the Clipper Docker images.
+build_images () {
+    # True and false indicate whether or not to publish the image. We publish public images.
+    local private=false
+    local public=true
+
+    ########################$$####### WARNING: #################################
+    # Some of these images depend on each other prior images. Do not change
+    # the order in which the images are built
+    ###########################################################################
+
+    # Build Clipper core images
+    create_image lib_base ClipperLibBaseDockerfile $private
+    create_image query_frontend QueryFrontendDockerfile $public
+    create_image management_frontend ManagementFrontendDockerfile $public
+    create_image unittests ClipperTestsDockerfile  $private
+
+    # Build containers
+    create_image spark-scala-container SparkScalaContainerDockerfile $public
+
+    # First build Python base image
+    create_image py-rpc RPCDockerfile $public
+    create_image sum-container SumDockerfile  $private
+    create_image noop-container NoopDockerfile $public
+    create_image python-closure-container PyClosureContainerDockerfile $public
+    create_image pyspark-container PySparkContainerDockerfile $public
+    create_image tf_cifar_container TensorFlowCifarDockerfile $public
+}
+
+usage () {
+    cat <<EOF
+    usage: build docker_images.sh [--publish]
+
+    Build and optionally publish the Clipper Docker images.
+
+    Options:
+
+    -p, --publish               Publish images to DockerHub
+    -h, --help                  Display this message and exit.
+
+$@
+EOF
+}
+
+
+if [ "$#" == 0 ]
+then
+  args="--nopublish"
+else
+  args=$1
+fi
+
+case $args in
+    -p | --publish )        publish=true
+                            build_images
+                            ;;
+    -n | --nopublish )      publish=false
+                            build_images
+                            ;;
+    -h | --help )           usage
+                            ;;
+    * )                     usage
+esac

bin/run_ci.sh

@@ -15,5 +15,16 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 cd $DIR/..
 tag=$(<VERSION.txt)
 
+# Build docker images
+./bin/build_docker_images.sh
+
+# Tag and push the latest version of the Clipper Docker images to the container registry
+# for the Kubernetes testing cluster
+docker tag clipper/query_frontend:$tag 568959175238.dkr.ecr.us-west-1.amazonaws.com/clipper/query_frontend:$tag
+docker push 568959175238.dkr.ecr.us-west-1.amazonaws.com/clipper/query_frontend:$tag
+docker tag clipper/management_frontend:$tag 568959175238.dkr.ecr.us-west-1.amazonaws.com/clipper/management_frontend:$tag
+docker push 568959175238.dkr.ecr.us-west-1.amazonaws.com/clipper/management_frontend:$tag
+
+# Run tests
 docker run --rm --network=host -v /var/run/docker.sock:/var/run/docker.sock -v /tmp:/tmp \
     -v /home/jenkins/.docker:/root/.docker clipper/unittests:$tag

ClipperLibBaseDockerfile → dockerfiles/ClipperLibBaseDockerfile

@@ -1,3 +1,5 @@
+# This ARG isn't used but prevents warnings in the build script
+ARG CODE_VERSION
 FROM ubuntu:16.04
 
 RUN apt-get update && apt-get install -y g++ automake autoconf autoconf-archive libtool libboost-all-dev \

ClipperTestsDockerfile → dockerfiles/ClipperTestsDockerfile

@@ -1,4 +1,4 @@
-ARG CODE_VERSION=develop
+ARG CODE_VERSION
 FROM clipper/lib_base:${CODE_VERSION}
 
 # install docker

ManagementFrontendDockerfile → dockerfiles/ManagementFrontendDockerfile

@@ -1,4 +1,4 @@
-ARG CODE_VERSION=develop
+ARG CODE_VERSION
 FROM clipper/lib_base:${CODE_VERSION}
 
 COPY ./ /clipper

NoopBenchDockerfile → dockerfiles/NoopBenchDockerfile

@@ -1,4 +1,4 @@
-ARG CODE_VERSION=develop
+ARG CODE_VERSION
 FROM clipper/py-rpc:${CODE_VERSION}
 
 COPY containers/python/noop_container.py /container/

NoopDockerfile → dockerfiles/NoopDockerfile

@@ -1,4 +1,4 @@
-ARG CODE_VERSION=develop
+ARG CODE_VERSION
 FROM clipper/py-rpc:${CODE_VERSION}
 
 MAINTAINER Dan Crankshaw <dscrankshaw@gmail.com>

PyClosureContainerDockerfile → dockerfiles/PyClosureContainerDockerfile

@@ -1,4 +1,4 @@
-ARG CODE_VERSION=develop
+ARG CODE_VERSION
 FROM clipper/py-rpc:${CODE_VERSION}
 
 COPY clipper_admin/clipper_admin/python_container_conda_deps.txt /lib/

PySparkContainerDockerfile → dockerfiles/PySparkContainerDockerfile

@@ -1,4 +1,4 @@
-ARG CODE_VERSION=develop
+ARG CODE_VERSION
 FROM clipper/py-rpc:${CODE_VERSION}
 
 COPY clipper_admin/clipper_admin/python_container_conda_deps.txt /lib/

QueryFrontendDockerfile → dockerfiles/QueryFrontendDockerfile

@@ -1,4 +1,4 @@
-ARG CODE_VERSION=develop
+ARG CODE_VERSION
 FROM clipper/lib_base:${CODE_VERSION}
 
 # Build Clipper

dockerfiles/README.md

@@ -0,0 +1,14 @@
+Dockerfiles for the Clipper project.
+
+Note that while these Dockerfiles are in the `dockerfiles` subdirectory, they must be built with the root
+of the Clipper repo as the build context. For example:
+
+*From within this directory*
+```
+docker build -t <tag> -f QueryFrontendDockerfile ../
+```
+
+*From the Clipper root directory*
+```
+docker build -t <tag> -f dockerfiles/QueryFrontendDockerfile ./
+```

RPCDockerfile → dockerfiles/RPCDockerfile

@@ -1,3 +1,5 @@
+# This ARG isn't used but prevents warnings in the build script
+ARG CODE_VERSION
 FROM continuumio/anaconda:latest
 
 MAINTAINER Dan Crankshaw <dscrankshaw@gmail.com>

containers/jvm/SparkScalaContainerDockerfile → dockerfiles/SparkScalaContainerDockerfile

@@ -1,3 +1,5 @@
+# This ARG isn't used but prevents warnings in the build script
+ARG CODE_VERSION
 FROM openjdk:8-jdk
 
 # First set up maven
@@ -24,7 +26,7 @@ RUN mkdir -p /usr/share/maven /usr/share/maven/ref \
 ENV MAVEN_HOME /usr/share/maven
 ENV MAVEN_CONFIG /root/.m2
 
-COPY . /root/container
+COPY ./containers/jvm/ /root/container
 
 WORKDIR /root/container
 

SumBenchDockerfile → dockerfiles/SumBenchDockerfile

@@ -1,4 +1,4 @@
-ARG CODE_VERSION=develop
+ARG CODE_VERSION
 FROM clipper/py-rpc:${CODE_VERSION}
 
 COPY containers/python/sum_container.py /container/

SumDockerfile → dockerfiles/SumDockerfile

@@ -1,4 +1,4 @@
-ARG CODE_VERSION=develop
+ARG CODE_VERSION
 FROM clipper/py-rpc:${CODE_VERSION}
 
 COPY containers/python/sum_container.py /container/

TensorFlowCifarDockerfile → dockerfiles/TensorFlowCifarDockerfile

@@ -1,4 +1,4 @@
-ARG CODE_VERSION=develop
+ARG CODE_VERSION
 FROM clipper/py-rpc:${CODE_VERSION}
 
 MAINTAINER Dan Crankshaw <dscrankshaw@gmail.com>