Split off bootstrap into separate Docker image.

This makes it faster to use Docker images to test new changes, as
suggested in vitessio#492.

Since we don't need to re-bootstrap for every push to master, this also
makes our automated builds faster and more reliable (they often fail due
to network lag when installing dependencies with apt-get).

Lastly, this will make it easier to swap out bootstrap images built for
different flavors, such as MariaDB 10.0 vs MySQL 5.6.

Dockerfile

@@ -1,71 +1,12 @@
-FROM golang:1.4-wheezy
+FROM vitess/bootstrap:mariadb
 
-# Install Vitess build dependencies
-RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
-    automake \
-    bison \
-    bzip2 \
-    curl \
-    g++ \
-    git \
-    libssl-dev \
-    libtool \
-    make \
-    memcached \
-    mercurial \
-    openjdk-7-jre-headless \
-    pkg-config \
-    python-dev \
-    python-mysqldb \
-    python-software-properties \
-    python-pip \
-    python-virtualenv \
-    unzip \
-    && rm -rf /var/lib/apt/lists/*
-
-# Install MariaDB 10.0.x
-RUN apt-key adv --recv-keys --keyserver keyserver.ubuntu.com 0xcbcb082a1bb943db && \
-    add-apt-repository 'deb http://sfo1.mirrors.digitalocean.com/mariadb/repo/10.0/debian wheezy main' && \
-    apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y mariadb-server libmariadbclient-dev
-
-# Add VOLUME
-VOLUME /vt/vtdataroot
-
-# Load files from directory containing Dockerfile
+# Re-copy sources from working tree
 COPY . /vt/src/github.com/youtube/vitess
 
-# Create vitess user
-RUN groupadd -r vitess && useradd -r -g vitess vitess && \
-    chown -R vitess:vitess /vt
-
-# Compile and install required packages as root
-WORKDIR /vt/src/github.com/youtube/vitess
-RUN ./travis/install_grpc.sh
-
-# Bootstrap Vitess
-ENV MYSQL_FLAVOR MariaDB
+# Fix permissions
+USER root
+RUN chown -R vitess:vitess /vt
 USER vitess
-RUN ./bootstrap.sh --skip_root_installs
-
-# Set up environment (equivalent to '. dev.env')
-ENV VTTOP /vt/src/github.com/youtube/vitess
-ENV VTROOT /vt
-ENV GOTOP $VTTOP/go
-ENV PYTOP $VTTOP/py
-ENV VTDATAROOT $VTROOT/vtdataroot
-ENV VTPORTSTART 15000
-ENV PYTHONPATH $VTROOT/dist/py-cbson/lib/python2.7/site-packages:$VTROOT/dist/py-vt-bson-0.3.2/lib/python2.7/site-packages:$VTROOT/py-vtdb
-ENV GOBIN $VTROOT/bin
-ENV GOPATH $VTROOT
-ENV PATH $VTROOT/bin:$PATH
-ENV VT_MYSQL_ROOT /usr
-ENV PKG_CONFIG_PATH $VTROOT/lib
-ENV CGO_CFLAGS -I$VTROOT/dist/vt-zookeeper-3.3.5/include/c-client-src
-ENV CGO_LDFLAGS -L$VTROOT/dist/vt-zookeeper-3.3.5/lib
-ENV LD_LIBRARY_PATH $VTROOT/dist/vt-zookeeper-3.3.5/lib
 
 # Build Vitess
 RUN make build
-
-# If the user doesn't specify a command, load a shell.
-CMD ["/bin/bash"]

Makefile

@@ -4,7 +4,7 @@
 
 MAKEFLAGS = -s
 
-.PHONY: all build test clean unit_test unit_test_cover unit_test_race queryservice_test integration_test bson proto site_test site_integration_test
+.PHONY: all build test clean unit_test unit_test_cover unit_test_race queryservice_test integration_test bson proto site_test site_integration_test docker_bootstrap
 
 all: build test
 
@@ -180,3 +180,8 @@ proto:
 	cd go/vt/proto/tabletmanager && $$VTROOT/dist/protobuf/bin/protoc -I../../../../proto ../../../../proto/tabletmanager.proto --go_out=plugins=grpc:.
 	find go/vt/proto -name "*.pb.go" | xargs sed --in-place -r -e 's,"([a-z0-9_]+).pb","github.com/youtube/vitess/go/vt/proto/\1",g'
 	cd py/vtctl && $$VTROOT/dist/protobuf/bin/protoc -I../../proto ../../proto/vtctl.proto --python_out=. --grpc_out=. --plugin=protoc-gen-grpc=$$VTROOT/dist/grpc/bin/grpc_python_plugin
+
+# This rule builds a bootstrap image of the given flavor.
+# Example: $ make docker_bootstrap flavor=mariadb
+docker_bootstrap:
+	docker/bootstrap/build.sh $(flavor)

docker/bootstrap/Dockerfile.common

@@ -0,0 +1,63 @@
+FROM golang:1.4-wheezy
+
+# Install Vitess build dependencies
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+    automake \
+    bison \
+    bzip2 \
+    curl \
+    g++ \
+    git \
+    libssl-dev \
+    libtool \
+    make \
+    memcached \
+    mercurial \
+    openjdk-7-jre-headless \
+    pkg-config \
+    python-dev \
+    python-mysqldb \
+    python-software-properties \
+    python-pip \
+    python-virtualenv \
+    unzip \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install gRPC as root
+RUN cd /tmp && \
+    git clone https://github.com/grpc/grpc.git && cd grpc && \
+    git submodule update --init && \
+    make install && \
+    CONFIG=opt tools/run_tests/build_python.sh && \
+    pip install -r src/python/requirements.txt && \
+    pip install src/python/src
+
+# Create mount point for actual data (e.g. MySQL data dir)
+VOLUME /vt/vtdataroot
+
+# Set up Vitess environment (equivalent to '. dev.env')
+ENV VTTOP /vt/src/github.com/youtube/vitess
+ENV VTROOT /vt
+ENV GOTOP $VTTOP/go
+ENV PYTOP $VTTOP/py
+ENV VTDATAROOT $VTROOT/vtdataroot
+ENV VTPORTSTART 15000
+ENV PYTHONPATH $VTROOT/dist/py-cbson/lib/python2.7/site-packages:$VTROOT/dist/py-vt-bson-0.3.2/lib/python2.7/site-packages:$VTROOT/py-vtdb
+ENV GOBIN $VTROOT/bin
+ENV GOPATH $VTROOT
+ENV PATH $VTROOT/bin:$PATH
+ENV VT_MYSQL_ROOT /usr
+ENV PKG_CONFIG_PATH $VTROOT/lib
+ENV CGO_CFLAGS -I$VTROOT/dist/vt-zookeeper-3.3.5/include/c-client-src
+ENV CGO_LDFLAGS -L$VTROOT/dist/vt-zookeeper-3.3.5/lib
+ENV LD_LIBRARY_PATH $VTROOT/dist/vt-zookeeper-3.3.5/lib
+
+# Copy files needed for bootstrap
+COPY . /vt/src/github.com/youtube/vitess
+
+# Create vitess user
+RUN groupadd -r vitess && useradd -r -g vitess vitess && \
+    chown -R vitess:vitess /vt
+
+# If the user doesn't specify a command, load a shell.
+CMD ["/bin/bash"]

docker/bootstrap/Dockerfile.mariadb

@@ -0,0 +1,14 @@
+FROM vitess/bootstrap:common
+
+# Install MariaDB 10.0.x
+RUN apt-key adv --recv-keys --keyserver keyserver.ubuntu.com 0xcbcb082a1bb943db && \
+    add-apt-repository 'deb http://sfo1.mirrors.digitalocean.com/mariadb/repo/10.0/debian wheezy main' && \
+    apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y mariadb-server libmariadbclient-dev && \
+    rm -rf /var/lib/apt/lists/*
+
+# Bootstrap Vitess
+WORKDIR /vt/src/github.com/youtube/vitess
+USER vitess
+ENV MYSQL_FLAVOR MariaDB
+RUN ./bootstrap.sh --skip_root_installs
+

docker/bootstrap/README.md

@@ -0,0 +1,19 @@
+# Bootstrap Images
+
+These Dockerfiles create images that contain everything Vitess expects to have
+after successfully running `bootstrap.sh` and `dev.env`.
+
+The `vitess/bootstrap` image comes in different flavors:
+* `vitess/bootstrap:common` - dependencies that are common to all flavors
+* `vitess/bootstrap:mariadb` - bootstrap image for MariaDB
+
+**NOTE: Unlike the base image that builds Vitess itself, this bootstrap image
+will NOT be rebuilt automatically on every push to the Vitess master branch.**
+
+To build a new bootstrap image, use the `docker_bootstrap` make rule.
+For example:
+
+```
+~/src/github.com/youtube/vitess$ make docker_bootstrap flavor=mariadb
+```
+

docker/bootstrap/build.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+flavor=$1
+
+if [[ -z "$flavor" ]]; then
+  echo "Flavor must be specified as first argument."
+  exit 1
+fi
+
+if [[ ! -f bootstrap.sh ]]; then
+  echo "This script should be run from the root of the Vitess source tree - e.g. ~/src/github.com/youtube/vitess"
+  exit 1
+fi
+
+# To avoid AUFS permission issues, files must allow access by "other"
+chmod -R o=g *
+
+docker build -f docker/bootstrap/Dockerfile.$flavor -t vitess/bootstrap:$flavor .