From 2268d5622bcdc6a43355da1a36985ac1d9fed0b7 Mon Sep 17 00:00:00 2001 From: Dan Kozlowski Date: Tue, 5 Mar 2019 10:29:23 -0800 Subject: [PATCH 1/2] Overhaul the Dockerfiles This commit overhauls the dockerfiles, specifically it 1. Converts the lite builds to use a multi-stage build to allow easier automation 2. Adds in new versions of mysql and mariadb 3. Updates the dockerfiles to use the full gpg key hash to provent collisions 4. Updates the make file with new targets Signed-off-by: Dan Kozlowski --- Makefile | 26 +++++++++++- docker/base/Dockerfile.mariadb103 | 13 ++++++ docker/base/Dockerfile.mysql80 | 13 ++++++ docker/base/Dockerfile.percona80 | 12 ++++++ docker/bootstrap/Dockerfile.mariadb | 14 ++++--- docker/bootstrap/Dockerfile.mysql80 | 16 ++++++++ docker/bootstrap/Dockerfile.percona | 2 +- docker/bootstrap/Dockerfile.percona57 | 2 +- docker/bootstrap/Dockerfile.percona80 | 24 +++++++++++ docker/lite/Dockerfile.mariadb | 39 +++++++++++++----- docker/lite/Dockerfile.mariadb103 | 49 ++++++++++++++++++++++ docker/lite/Dockerfile.mysql56 | 36 +++++++++++----- docker/lite/Dockerfile.mysql57 | 52 +++++++++++++++++++++++ docker/lite/Dockerfile.mysql80 | 52 +++++++++++++++++++++++ docker/lite/Dockerfile.percona | 49 ++++++++++++++-------- docker/lite/Dockerfile.percona57 | 51 +++++++++++++++-------- docker/lite/Dockerfile.percona80 | 59 +++++++++++++++++++++++++++ docker/lite/build.sh | 41 +------------------ 18 files changed, 449 insertions(+), 101 deletions(-) create mode 100644 docker/base/Dockerfile.mariadb103 create mode 100644 docker/base/Dockerfile.mysql80 create mode 100644 docker/base/Dockerfile.percona80 create mode 100644 docker/bootstrap/Dockerfile.mysql80 create mode 100644 docker/bootstrap/Dockerfile.percona80 create mode 100644 docker/lite/Dockerfile.mariadb103 create mode 100644 docker/lite/Dockerfile.mysql57 create mode 100644 docker/lite/Dockerfile.mysql80 create mode 100644 docker/lite/Dockerfile.percona80 diff --git a/Makefile b/Makefile index 06446dd88f9..4c3a7b72bed 100644 --- a/Makefile +++ b/Makefile @@ -160,7 +160,7 @@ $(PROTO_GO_OUTS): install_protoc-gen-go proto/*.proto # Please read docker/README.md to understand the different available images. # This rule builds the bootstrap images for all flavors. -DOCKER_IMAGES_FOR_TEST = mariadb mysql56 mysql57 percona percona57 +DOCKER_IMAGES_FOR_TEST = mariadb mariadb103 mysql56 mysql57 mysql80 percona percona57 percona80 DOCKER_IMAGES = common $(DOCKER_IMAGES_FOR_TEST) docker_bootstrap: for i in $(DOCKER_IMAGES); do echo "building bootstrap image: $$i"; docker/bootstrap/build.sh $$i || exit 1; done @@ -184,10 +184,18 @@ docker_base_mysql56: chmod -R o=g * docker build -f docker/base/Dockerfile.mysql56 -t vitess/base:mysql56 . +docker_base_mysql80: + chmod -R o=g * + docker build -f docker/base/Dockerfile.mysql56 -t vitess/base:mysql80 . + docker_base_mariadb: chmod -R o=g * docker build -f docker/base/Dockerfile.mariadb -t vitess/base:mariadb . +docker_base_mariadb103: + chmod -R o=g * + docker build -f docker/base/Dockerfile.mariadb -t vitess/base:mariadb103 . + docker_base_percona: chmod -R o=g * docker build -f docker/base/Dockerfile.percona -t vitess/base:percona . @@ -196,6 +204,10 @@ docker_base_percona57: chmod -R o=g * docker build -f docker/base/Dockerfile.percona57 -t vitess/base:percona57 . +docker_base_percona80: + chmod -R o=g * + docker build -f docker/base/Dockerfile.percona57 -t vitess/base:percona80 . + # Run "make docker_lite PROMPT_NOTICE=false" to avoid that the script # prompts you to press ENTER and confirm that the vitess/base image is not # rebuild by this target as well. @@ -205,15 +217,27 @@ docker_lite: docker_lite_mysql56: cd docker/lite && ./build.sh --prompt=$(PROMPT_NOTICE) mysql56 +docker_lite_mysql57: + cd docker/lite && ./build.sh --prompt=$(PROMPT_NOTICE) mysql57 + +docker_lite_mysql80: + cd docker/lite && ./build.sh --prompt=$(PROMPT_NOTICE) mysql80 + docker_lite_mariadb: cd docker/lite && ./build.sh --prompt=$(PROMPT_NOTICE) mariadb +docker_lite_mariadb103: + cd docker/lite && ./build.sh --prompt=$(PROMPT_NOTICE) mariadb103 + docker_lite_percona: cd docker/lite && ./build.sh --prompt=$(PROMPT_NOTICE) percona docker_lite_percona57: cd docker/lite && ./build.sh --prompt=$(PROMPT_NOTICE) percona57 +docker_lite_percona80: + cd docker/lite && ./build.sh --prompt=$(PROMPT_NOTICE) percona80 + docker_lite_alpine: cd docker/lite && ./build.sh --prompt=$(PROMPT_NOTICE) alpine diff --git a/docker/base/Dockerfile.mariadb103 b/docker/base/Dockerfile.mariadb103 new file mode 100644 index 00000000000..75811dc68ba --- /dev/null +++ b/docker/base/Dockerfile.mariadb103 @@ -0,0 +1,13 @@ +FROM vitess/bootstrap:mariadb103 + +# Re-copy sources from working tree +USER root +COPY . /vt/src/vitess.io/vitess + +# Build Vitess +RUN make build + +# Fix permissions +RUN chown -R vitess:vitess /vt +USER vitess + diff --git a/docker/base/Dockerfile.mysql80 b/docker/base/Dockerfile.mysql80 new file mode 100644 index 00000000000..5b2fc5c91ae --- /dev/null +++ b/docker/base/Dockerfile.mysql80 @@ -0,0 +1,13 @@ +FROM vitess/bootstrap:mysql80 + +# Re-copy sources from working tree +USER root +COPY . /vt/src/vitess.io/vitess + +# Build Vitess +RUN make build + +# Fix permissions +RUN chown -R vitess:vitess /vt +USER vitess + diff --git a/docker/base/Dockerfile.percona80 b/docker/base/Dockerfile.percona80 new file mode 100644 index 00000000000..83672111166 --- /dev/null +++ b/docker/base/Dockerfile.percona80 @@ -0,0 +1,12 @@ +FROM vitess/bootstrap:percona80 + +# Re-copy sources from working tree +USER root +COPY . /vt/src/vitess.io/vitess + +# Fix permissions +RUN chown -R vitess:vitess /vt +USER vitess + +# Build Vitess +RUN make build diff --git a/docker/bootstrap/Dockerfile.mariadb b/docker/bootstrap/Dockerfile.mariadb index bc76702faa9..ca40724c12f 100644 --- a/docker/bootstrap/Dockerfile.mariadb +++ b/docker/bootstrap/Dockerfile.mariadb @@ -1,10 +1,14 @@ FROM vitess/bootstrap:common -# Install MariaDB 10. -RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ - mariadb-server \ - libmariadbclient-dev \ - && rm -rf /var/lib/apt/lists/* +# Install MariaDB 10.3 +RUN apt-key adv --no-tty --recv-keys --keyserver keyserver.ubuntu.com 0xF1656F24C74CD1D8 \ + && echo 'deb http://sfo1.mirrors.digitalocean.com/mariadb/repo/10.3/debian stretch main' > /etc/apt/sources.list.d/mariadb.list \ + && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + bzip2 \ + mariadb-server \ + libmariadbclient-dev \ + && rm -rf /var/lib/apt/lists/* # Bootstrap Vitess WORKDIR /vt/src/vitess.io/vitess diff --git a/docker/bootstrap/Dockerfile.mysql80 b/docker/bootstrap/Dockerfile.mysql80 new file mode 100644 index 00000000000..c82ee741dec --- /dev/null +++ b/docker/bootstrap/Dockerfile.mysql80 @@ -0,0 +1,16 @@ +FROM vitess/bootstrap:common + +# Install MySQL 5.7 +RUN for i in $(seq 1 10); do apt-key adv --no-tty --recv-keys --keyserver ha.pool.sks-keyservers.net 8C718D3B5072E1F5 && break; done && \ + add-apt-repository 'deb http://repo.mysql.com/apt/debian/ stretch mysql-8.0' && \ + apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y mysql-server libmysqlclient-dev && \ + rm -rf /var/lib/apt/lists/* + +# Bootstrap Vitess +WORKDIR /vt/src/vitess.io/vitess + + +ENV MYSQL_FLAVOR MySQL56 +USER vitess +RUN ./bootstrap.sh \ No newline at end of file diff --git a/docker/bootstrap/Dockerfile.percona b/docker/bootstrap/Dockerfile.percona index 422dacffb6b..427ab2670c0 100644 --- a/docker/bootstrap/Dockerfile.percona +++ b/docker/bootstrap/Dockerfile.percona @@ -1,7 +1,7 @@ FROM vitess/bootstrap:common # Install Percona 5.6 -RUN for i in $(seq 1 10); do apt-key adv --no-tty --keyserver keys.gnupg.net --recv-keys 8507EFA5 && break; done && \ +RUN for i in $(seq 1 10); do apt-key adv --no-tty --keyserver keys.gnupg.net --recv-keys 9334A25F8507EFA5 && break; done && \ add-apt-repository 'deb http://repo.percona.com/apt stretch main' && \ { \ echo debconf debconf/frontend select Noninteractive; \ diff --git a/docker/bootstrap/Dockerfile.percona57 b/docker/bootstrap/Dockerfile.percona57 index 054e0d3216a..371890d116f 100644 --- a/docker/bootstrap/Dockerfile.percona57 +++ b/docker/bootstrap/Dockerfile.percona57 @@ -1,7 +1,7 @@ FROM vitess/bootstrap:common # Install Percona 5.7 -RUN for i in $(seq 1 10); do apt-key adv --no-tty --keyserver keys.gnupg.net --recv-keys 8507EFA5 && break; done && \ +RUN for i in $(seq 1 10); do apt-key adv --no-tty --keyserver keys.gnupg.net --recv-keys 9334A25F8507EFA5 && break; done && \ add-apt-repository 'deb http://repo.percona.com/apt stretch main' && \ { \ echo debconf debconf/frontend select Noninteractive; \ diff --git a/docker/bootstrap/Dockerfile.percona80 b/docker/bootstrap/Dockerfile.percona80 new file mode 100644 index 00000000000..246c6519843 --- /dev/null +++ b/docker/bootstrap/Dockerfile.percona80 @@ -0,0 +1,24 @@ +FROM vitess/bootstrap:common + +# Install Percona 8.0 +RUN for i in $(seq 1 10); do apt-key adv --no-tty --keyserver keys.gnupg.net --recv-keys 9334A25F8507EFA5 && break; done && + && echo 'deb http://repo.percona.com/ps-80/apt stretch main' > /etc/apt/sources.list.d/percona.list && \ + { \ + echo debconf debconf/frontend select Noninteractive; \ + echo percona-server-server-8.0 percona-server-server/root_password password 'unused'; \ + echo percona-server-server-8.0 percona-server-server/root_password_again password 'unused'; \ + } | debconf-set-selections \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + percona-server-server \ + libperconaserverclient21 \ + percona-server-tokudb \ + percona-server-rocksdb \ + bzip2 + +# Bootstrap Vitess +WORKDIR /vt/src/vitess.io/vitess + +ENV MYSQL_FLAVOR MySQL56 +USER vitess +RUN ./bootstrap.sh diff --git a/docker/lite/Dockerfile.mariadb b/docker/lite/Dockerfile.mariadb index 7df86adb0e0..5c947ab804c 100644 --- a/docker/lite/Dockerfile.mariadb +++ b/docker/lite/Dockerfile.mariadb @@ -1,14 +1,35 @@ -# This image is only meant to be built from within the build.sh script. -FROM debian:jessie +FROM vitess/base AS builder +FROM debian:stretch-slim AS staging + +RUN mkdir -p /vt/vtdataroot/ \ + && mkdir -p /vt/bin \ + && mkdir -p /vt/src/vitess.io/vitess/web/vtctld2 \ + && groupadd -r vitess && useradd -r -g vitess vitess + +COPY --from=builder /vt/src/vitess.io/vitess/web/vtctld /vt/src/vitess.io/vitess/web/vtctld +COPY --from=builder /vt/src/vitess.io/vitess/web/vtctld2/app /vt/src/vitess.io/vitess/web/vtctld2/app +COPY --from=builder /vt/src/vitess.io/vitess/config /vt/config +COPY --from=builder /vt/bin/mysqlctld /vt/bin/ +COPY --from=builder /vt/bin/vtctld /vt/bin/ +COPY --from=builder /vt/bin/vtctlclient /vt/bin/ +COPY --from=builder /vt/bin/vtgate /vt/bin/ +COPY --from=builder /vt/bin/vttablet /vt/bin/ +COPY --from=builder /vt/bin/vtworker /vt/bin/ + +RUN chown -R vitess:vitess /vt + +FROM debian:stretch-slim # Install dependencies -RUN apt-key adv --recv-keys --keyserver keyserver.ubuntu.com 0xcbcb082a1bb943db \ - && echo 'deb http://sfo1.mirrors.digitalocean.com/mariadb/repo/10.0/debian jessie main' > /etc/apt/sources.list.d/mariadb.list \ +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends gnupg dirmngr ca-certificates \ + && apt-key adv --no-tty --recv-keys --keyserver keyserver.ubuntu.com 0xF1656F24C74CD1D8 \ + && echo 'deb http://sfo1.mirrors.digitalocean.com/mariadb/repo/10.2/debian stretch main' > /etc/apt/sources.list.d/mariadb.list \ && apt-get update \ && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ bzip2 \ mariadb-server \ - && rm -rf /var/lib/apt/lists/* + && rm -rf /var/lib/apt/lists/* \ + && groupadd -r vitess && useradd -r -g vitess vitess # Set up Vitess environment (just enough to run pre-built Go binaries) ENV VTTOP /vt/src/vitess.io/vitess @@ -22,12 +43,8 @@ ENV VT_MYSQL_ROOT /usr ENV PKG_CONFIG_PATH $VTROOT/lib # Copy binaries (placed by build.sh) -COPY lite/vt /vt - -# Create vitess user -RUN groupadd -r vitess && useradd -r -g vitess vitess && \ - mkdir -p /vt/vtdataroot && chown -R vitess:vitess /vt +COPY --from=staging /vt/ /vt/ # Create mount point for actual data (e.g. MySQL data dir) VOLUME /vt/vtdataroot -USER vitess \ No newline at end of file +USER vitess diff --git a/docker/lite/Dockerfile.mariadb103 b/docker/lite/Dockerfile.mariadb103 new file mode 100644 index 00000000000..2290a77fbc9 --- /dev/null +++ b/docker/lite/Dockerfile.mariadb103 @@ -0,0 +1,49 @@ +FROM vitess/base AS builder +FROM debian:stretch-slim AS staging + +RUN mkdir -p /vt/vtdataroot/ \ + && mkdir -p /vt/bin \ + && mkdir -p /vt/src/vitess.io/vitess/web/vtctld2 \ + && groupadd -r vitess && useradd -r -g vitess vitess + +COPY --from=builder /vt/src/vitess.io/vitess/web/vtctld /vt/src/vitess.io/vitess/web/vtctld +COPY --from=builder /vt/src/vitess.io/vitess/web/vtctld2/app /vt/src/vitess.io/vitess/web/vtctld2/app +COPY --from=builder /vt/src/vitess.io/vitess/config /vt/config +COPY --from=builder /vt/bin/mysqlctld /vt/bin/ +COPY --from=builder /vt/bin/vtctld /vt/bin/ +COPY --from=builder /vt/bin/vtctlclient /vt/bin/ +COPY --from=builder /vt/bin/vtgate /vt/bin/ +COPY --from=builder /vt/bin/vttablet /vt/bin/ +COPY --from=builder /vt/bin/vtworker /vt/bin/ +RUN chown -R vitess:vitess /vt + +FROM debian:stretch-slim + +# Install dependencies +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends gnupg dirmngr ca-certificates \ + && apt-key adv --no-tty --recv-keys --keyserver keyserver.ubuntu.com 0xF1656F24C74CD1D8 \ + && echo 'deb http://sfo1.mirrors.digitalocean.com/mariadb/repo/10.3/debian stretch main' > /etc/apt/sources.list.d/mariadb.list \ + && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + bzip2 \ + mariadb-server \ + && rm -rf /var/lib/apt/lists/* \ + && groupadd -r vitess && useradd -r -g vitess vitess + +# Set up Vitess environment (just enough to run pre-built Go binaries) +ENV VTTOP /vt/src/vitess.io/vitess +ENV VTROOT /vt +ENV GOTOP $VTTOP/go +ENV VTDATAROOT $VTROOT/vtdataroot +ENV GOBIN $VTROOT/bin +ENV GOPATH $VTROOT +ENV PATH $VTROOT/bin:$PATH +ENV VT_MYSQL_ROOT /usr +ENV PKG_CONFIG_PATH $VTROOT/lib + +# Copy binaries (placed by build.sh) +COPY --from=staging /vt/ /vt/ + +# Create mount point for actual data (e.g. MySQL data dir) +VOLUME /vt/vtdataroot +USER vitess diff --git a/docker/lite/Dockerfile.mysql56 b/docker/lite/Dockerfile.mysql56 index 79e5381f909..89432bccc59 100644 --- a/docker/lite/Dockerfile.mysql56 +++ b/docker/lite/Dockerfile.mysql56 @@ -1,9 +1,28 @@ -# This image is only meant to be built from within the build.sh script. -FROM debian:jessie +FROM vitess/base AS builder +FROM debian:stretch-slim AS staging + +RUN mkdir -p /vt/vtdataroot/ \ + && mkdir -p /vt/bin \ + && mkdir -p /vt/src/vitess.io/vitess/web/vtctld2 \ + && groupadd -r vitess && useradd -r -g vitess vitess + +COPY --from=builder /vt/src/vitess.io/vitess/web/vtctld /vt/src/vitess.io/vitess/web/vtctld +COPY --from=builder /vt/src/vitess.io/vitess/web/vtctld2/app /vt/src/vitess.io/vitess/web/vtctld2/app +COPY --from=builder /vt/src/vitess.io/vitess/config /vt/config +COPY --from=builder /vt/bin/mysqlctld /vt/bin/ +COPY --from=builder /vt/bin/vtctld /vt/bin/ +COPY --from=builder /vt/bin/vtctlclient /vt/bin/ +COPY --from=builder /vt/bin/vtgate /vt/bin/ +COPY --from=builder /vt/bin/vttablet /vt/bin/ +COPY --from=builder /vt/bin/vtworker /vt/bin/ + +RUN chown -R vitess:vitess /vt + +FROM debian:stretch-slim # Install dependencies -RUN apt-key adv --recv-keys --keyserver keyserver.ubuntu.com 5072E1F5 \ - && echo 'deb http://repo.mysql.com/apt/debian/ jessie mysql-5.6' > /etc/apt/sources.list.d/mysql.list \ +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends gnupg dirmngr ca-certificates && apt-key adv --no-tty --recv-keys --keyserver keyserver.ubuntu.com 8C718D3B5072E1F5 \ + && echo 'deb http://repo.mysql.com/apt/debian/ stretch mysql-5.6' > /etc/apt/sources.list.d/mysql.list \ && apt-get update \ && DEBIAN_FRONTEND=noninteractive \ apt-get install -y --no-install-recommends \ @@ -11,7 +30,8 @@ RUN apt-key adv --recv-keys --keyserver keyserver.ubuntu.com 5072E1F5 \ libmysqlclient18 \ mysql-client \ mysql-server \ - && rm -rf /var/lib/apt/lists/* + && rm -rf /var/lib/apt/lists/* \ + && groupadd -r vitess && useradd -r -g vitess vitess # Set up Vitess environment (just enough to run pre-built Go binaries) ENV VTTOP /vt/src/vitess.io/vitess @@ -25,11 +45,7 @@ ENV VT_MYSQL_ROOT /usr ENV PKG_CONFIG_PATH $VTROOT/lib # Copy binaries (placed by build.sh) -COPY lite/vt /vt - -# Create vitess user -RUN groupadd -r vitess && useradd -r -g vitess vitess && \ - mkdir -p /vt/vtdataroot && chown -R vitess:vitess /vt +COPY --from=staging /vt/ /vt/ # Create mount point for actual data (e.g. MySQL data dir) VOLUME /vt/vtdataroot diff --git a/docker/lite/Dockerfile.mysql57 b/docker/lite/Dockerfile.mysql57 new file mode 100644 index 00000000000..5d442d2e5ca --- /dev/null +++ b/docker/lite/Dockerfile.mysql57 @@ -0,0 +1,52 @@ +FROM vitess/base AS builder +FROM debian:stretch-slim AS staging + +RUN mkdir -p /vt/vtdataroot/ \ + && mkdir -p /vt/bin \ + && mkdir -p /vt/src/vitess.io/vitess/web/vtctld2 \ + && groupadd -r vitess && useradd -r -g vitess vitess + +COPY --from=builder /vt/src/vitess.io/vitess/web/vtctld /vt/src/vitess.io/vitess/web/vtctld +COPY --from=builder /vt/src/vitess.io/vitess/web/vtctld2/app /vt/src/vitess.io/vitess/web/vtctld2/app +COPY --from=builder /vt/src/vitess.io/vitess/config /vt/config +COPY --from=builder /vt/bin/mysqlctld /vt/bin/ +COPY --from=builder /vt/bin/vtctld /vt/bin/ +COPY --from=builder /vt/bin/vtctlclient /vt/bin/ +COPY --from=builder /vt/bin/vtgate /vt/bin/ +COPY --from=builder /vt/bin/vttablet /vt/bin/ +COPY --from=builder /vt/bin/vtworker /vt/bin/ + +RUN chown -R vitess:vitess /vt + +FROM debian:stretch-slim + +# Install dependencies +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends gnupg dirmngr ca-certificates && apt-key adv --no-tty --recv-keys --keyserver keyserver.ubuntu.com 8C718D3B5072E1F5 \ + && echo 'deb http://repo.mysql.com/apt/debian/ stretch mysql-5.7' > /etc/apt/sources.list.d/mysql.list \ + && apt-get update \ + && DEBIAN_FRONTEND=noninteractive \ + apt-get install -y --no-install-recommends \ + bzip2 \ + libmysqlclient20 \ + mysql-client \ + mysql-server \ + && rm -rf /var/lib/apt/lists/* \ + && groupadd -r vitess && useradd -r -g vitess vitess + +# Set up Vitess environment (just enough to run pre-built Go binaries) +ENV VTTOP /vt/src/vitess.io/vitess +ENV VTROOT /vt +ENV GOTOP $VTTOP/go +ENV VTDATAROOT $VTROOT/vtdataroot +ENV GOBIN $VTROOT/bin +ENV GOPATH $VTROOT +ENV PATH $VTROOT/bin:$PATH +ENV VT_MYSQL_ROOT /usr +ENV PKG_CONFIG_PATH $VTROOT/lib + +# Copy binaries (placed by build.sh) +COPY --from=staging /vt/ /vt/ + +# Create mount point for actual data (e.g. MySQL data dir) +VOLUME /vt/vtdataroot +USER vitess \ No newline at end of file diff --git a/docker/lite/Dockerfile.mysql80 b/docker/lite/Dockerfile.mysql80 new file mode 100644 index 00000000000..4dc5f40f2d0 --- /dev/null +++ b/docker/lite/Dockerfile.mysql80 @@ -0,0 +1,52 @@ +FROM vitess/base AS builder +FROM debian:stretch-slim AS staging + +RUN mkdir -p /vt/vtdataroot/ \ + && mkdir -p /vt/bin \ + && mkdir -p /vt/src/vitess.io/vitess/web/vtctld2 \ + && groupadd -r vitess && useradd -r -g vitess vitess + +COPY --from=builder /vt/src/vitess.io/vitess/web/vtctld /vt/src/vitess.io/vitess/web/vtctld +COPY --from=builder /vt/src/vitess.io/vitess/web/vtctld2/app /vt/src/vitess.io/vitess/web/vtctld2/app +COPY --from=builder /vt/src/vitess.io/vitess/config /vt/config +COPY --from=builder /vt/bin/mysqlctld /vt/bin/ +COPY --from=builder /vt/bin/vtctld /vt/bin/ +COPY --from=builder /vt/bin/vtctlclient /vt/bin/ +COPY --from=builder /vt/bin/vtgate /vt/bin/ +COPY --from=builder /vt/bin/vttablet /vt/bin/ +COPY --from=builder /vt/bin/vtworker /vt/bin/ + +RUN chown -R vitess:vitess /vt + +FROM debian:stretch-slim + +# Install dependencies +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends gnupg dirmngr ca-certificates && apt-key adv --no-tty --recv-keys --keyserver keyserver.ubuntu.com 8C718D3B5072E1F5 \ + && echo 'deb http://repo.mysql.com/apt/debian/ stretch mysql-8.0' > /etc/apt/sources.list.d/mysql.list \ + && apt-get update \ + && DEBIAN_FRONTEND=noninteractive \ + apt-get install -y --no-install-recommends \ + bzip2 \ + libmysqlclient21 \ + mysql-client \ + mysql-server \ + && rm -rf /var/lib/apt/lists/* \ + && groupadd -r vitess && useradd -r -g vitess vitess + +# Set up Vitess environment (just enough to run pre-built Go binaries) +ENV VTTOP /vt/src/vitess.io/vitess +ENV VTROOT /vt +ENV GOTOP $VTTOP/go +ENV VTDATAROOT $VTROOT/vtdataroot +ENV GOBIN $VTROOT/bin +ENV GOPATH $VTROOT +ENV PATH $VTROOT/bin:$PATH +ENV VT_MYSQL_ROOT /usr +ENV PKG_CONFIG_PATH $VTROOT/lib + +# Copy binaries (placed by build.sh) +COPY --from=staging /vt/ /vt/ + +# Create mount point for actual data (e.g. MySQL data dir) +VOLUME /vt/vtdataroot +USER vitess \ No newline at end of file diff --git a/docker/lite/Dockerfile.percona b/docker/lite/Dockerfile.percona index 47a7430b825..c27944b637e 100644 --- a/docker/lite/Dockerfile.percona +++ b/docker/lite/Dockerfile.percona @@ -1,19 +1,40 @@ -# This image is only meant to be built from within the build.sh script. -FROM debian:jessie +FROM vitess/base AS builder +FROM debian:stretch-slim AS staging + +RUN mkdir -p /vt/vtdataroot/ \ + && mkdir -p /vt/bin \ + && mkdir -p /vt/src/vitess.io/vitess/web/vtctld2 \ + && groupadd -r vitess && useradd -r -g vitess vitess + +COPY --from=builder /vt/src/vitess.io/vitess/web/vtctld /vt/src/vitess.io/vitess/web/vtctld +COPY --from=builder /vt/src/vitess.io/vitess/web/vtctld2/app /vt/src/vitess.io/vitess/web/vtctld2/app +COPY --from=builder /vt/src/vitess.io/vitess/config /vt/config +COPY --from=builder /vt/bin/mysqlctld /vt/bin/ +COPY --from=builder /vt/bin/vtctld /vt/bin/ +COPY --from=builder /vt/bin/vtctlclient /vt/bin/ +COPY --from=builder /vt/bin/vtgate /vt/bin/ +COPY --from=builder /vt/bin/vttablet /vt/bin/ +COPY --from=builder /vt/bin/vtworker /vt/bin/ + +RUN chown -R vitess:vitess /vt + +FROM debian:stretch-slim # Install dependencies -RUN apt-key adv --keyserver keys.gnupg.net \ - --recv-keys 8507EFA5 && \ - echo 'deb http://repo.percona.com/apt jessie main' > /etc/apt/sources.list.d/mysql.list && \ +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends gnupg dirmngr ca-certificates \ + && apt-key adv --no-tty --recv-keys --keyserver keys.gnupg.net 9334A25F8507EFA5 \ + && echo 'deb http://repo.percona.com/apt stretch main' > /etc/apt/sources.list.d/percona.list && \ { \ echo debconf debconf/frontend select Noninteractive; \ echo percona-server-server-5.6 percona-server-server/root_password password 'unused'; \ echo percona-server-server-5.6 percona-server-server/root_password_again password 'unused'; \ - } | debconf-set-selections && \ - apt-get update && \ - apt-get install -y --no-install-recommends \ - percona-server-server-5.6 bzip2 && \ - rm -rf /var/lib/apt/lists/* + } | debconf-set-selections \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + percona-server-server-5.6 \ + bzip2 \ + && rm -rf /var/lib/apt/lists/* \ + && groupadd -r vitess && useradd -r -g vitess vitess # Set up Vitess environment (just enough to run pre-built Go binaries) ENV VTTOP /vt/src/vitess.io/vitess @@ -27,12 +48,8 @@ ENV VT_MYSQL_ROOT /usr ENV PKG_CONFIG_PATH $VTROOT/lib # Copy binaries (placed by build.sh) -COPY lite/vt /vt - -# Create vitess user -RUN groupadd -r vitess && useradd -r -g vitess vitess && \ - mkdir -p /vt/vtdataroot && chown -R vitess:vitess /vt +COPY --from=staging /vt/ /vt/ # Create mount point for actual data (e.g. MySQL data dir) VOLUME /vt/vtdataroot -USER vitess \ No newline at end of file +USER vitess diff --git a/docker/lite/Dockerfile.percona57 b/docker/lite/Dockerfile.percona57 index 1b824cec101..def68853c31 100644 --- a/docker/lite/Dockerfile.percona57 +++ b/docker/lite/Dockerfile.percona57 @@ -1,19 +1,41 @@ -# This image is only meant to be built from within the build.sh script. -FROM debian:jessie +FROM vitess/base AS builder +FROM debian:stretch-slim AS staging + +RUN mkdir -p /vt/vtdataroot/ \ + && mkdir -p /vt/bin \ + && mkdir -p /vt/src/vitess.io/vitess/web/vtctld2 \ + && groupadd -r vitess && useradd -r -g vitess vitess + +COPY --from=builder /vt/src/vitess.io/vitess/web/vtctld /vt/src/vitess.io/vitess/web/vtctld +COPY --from=builder /vt/src/vitess.io/vitess/web/vtctld2/app /vt/src/vitess.io/vitess/web/vtctld2/app +COPY --from=builder /vt/src/vitess.io/vitess/config /vt/config +COPY --from=builder /vt/bin/mysqlctld /vt/bin/ +COPY --from=builder /vt/bin/vtctld /vt/bin/ +COPY --from=builder /vt/bin/vtctlclient /vt/bin/ +COPY --from=builder /vt/bin/vtgate /vt/bin/ +COPY --from=builder /vt/bin/vttablet /vt/bin/ +COPY --from=builder /vt/bin/vtworker /vt/bin/ + +RUN chown -R vitess:vitess /vt + +FROM debian:stretch-slim # Install dependencies -RUN apt-key adv --keyserver keys.gnupg.net \ - --recv-keys 8507EFA5 && \ - echo 'deb http://repo.percona.com/apt jessie main' > /etc/apt/sources.list.d/mysql.list && \ +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends gnupg dirmngr ca-certificates \ + && apt-key adv --no-tty --recv-keys --keyserver keys.gnupg.net 9334A25F8507EFA5 \ + && echo 'deb http://repo.percona.com/apt stretch main' > /etc/apt/sources.list.d/percona.list && \ { \ echo debconf debconf/frontend select Noninteractive; \ echo percona-server-server-5.7 percona-server-server/root_password password 'unused'; \ echo percona-server-server-5.7 percona-server-server/root_password_again password 'unused'; \ - } | debconf-set-selections && \ - apt-get update && \ - apt-get install -y --no-install-recommends \ - percona-server-server-5.7 libperconaserverclient18.1 bzip2 && \ - rm -rf /var/lib/apt/lists/* + } | debconf-set-selections \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + percona-server-server-5.7 \ + libperconaserverclient20 \ + bzip2 \ + && rm -rf /var/lib/apt/lists/* \ + && groupadd -r vitess && useradd -r -g vitess vitess # Set up Vitess environment (just enough to run pre-built Go binaries) ENV VTTOP /vt/src/vitess.io/vitess @@ -26,13 +48,10 @@ ENV PATH $VTROOT/bin:$PATH ENV VT_MYSQL_ROOT /usr ENV PKG_CONFIG_PATH $VTROOT/lib -# Copy binaries (placed by build.sh) -COPY lite/vt /vt -# Create vitess user -RUN groupadd -r vitess && useradd -r -g vitess vitess && \ - mkdir -p /vt/vtdataroot && chown -R vitess:vitess /vt +# Copy binaries (placed by build.sh) +COPY --from=staging /vt/ /vt/ # Create mount point for actual data (e.g. MySQL data dir) VOLUME /vt/vtdataroot -USER vitess \ No newline at end of file +USER vitess diff --git a/docker/lite/Dockerfile.percona80 b/docker/lite/Dockerfile.percona80 new file mode 100644 index 00000000000..658b52c4902 --- /dev/null +++ b/docker/lite/Dockerfile.percona80 @@ -0,0 +1,59 @@ +FROM vitess/base AS builder +FROM debian:stretch-slim AS staging + +RUN mkdir -p /vt/vtdataroot/ \ + && mkdir -p /vt/bin \ + && mkdir -p /vt/src/vitess.io/vitess/web/vtctld2 \ + && groupadd -r vitess && useradd -r -g vitess vitess + +COPY --from=builder /vt/src/vitess.io/vitess/web/vtctld /vt/src/vitess.io/vitess/web/vtctld +COPY --from=builder /vt/src/vitess.io/vitess/web/vtctld2/app /vt/src/vitess.io/vitess/web/vtctld2/app +COPY --from=builder /vt/src/vitess.io/vitess/config /vt/config +COPY --from=builder /vt/bin/mysqlctld /vt/bin/ +COPY --from=builder /vt/bin/vtctld /vt/bin/ +COPY --from=builder /vt/bin/vtctlclient /vt/bin/ +COPY --from=builder /vt/bin/vtgate /vt/bin/ +COPY --from=builder /vt/bin/vttablet /vt/bin/ +COPY --from=builder /vt/bin/vtworker /vt/bin/ + +RUN chown -R vitess:vitess /vt + +FROM debian:stretch-slim + +# Install dependencies +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends gnupg dirmngr ca-certificates \ + && apt-key adv --no-tty --recv-keys --keyserver keys.gnupg.net 9334A25F8507EFA5 \ + && echo 'deb http://repo.percona.com/ps-80/apt stretch main' > /etc/apt/sources.list.d/percona.list && \ + { \ + echo debconf debconf/frontend select Noninteractive; \ + echo percona-server-server-8.0 percona-server-server/root_password password 'unused'; \ + echo percona-server-server-8.0 percona-server-server/root_password_again password 'unused'; \ + } | debconf-set-selections \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + percona-server-server \ + libperconaserverclient21 \ + percona-server-tokudb \ + percona-server-rocksdb \ + bzip2 \ + && rm -rf /var/lib/apt/lists/* \ + && groupadd -r vitess && useradd -r -g vitess vitess + +# Set up Vitess environment (just enough to run pre-built Go binaries) +ENV VTTOP /vt/src/vitess.io/vitess +ENV VTROOT /vt +ENV GOTOP $VTTOP/go +ENV VTDATAROOT $VTROOT/vtdataroot +ENV GOBIN $VTROOT/bin +ENV GOPATH $VTROOT +ENV PATH $VTROOT/bin:$PATH +ENV VT_MYSQL_ROOT /usr +ENV PKG_CONFIG_PATH $VTROOT/lib + + +# Copy binaries (placed by build.sh) +COPY --from=staging /vt/ /vt/ + +# Create mount point for actual data (e.g. MySQL data dir) +VOLUME /vt/vtdataroot +USER vitess diff --git a/docker/lite/build.sh b/docker/lite/build.sh index 66f61ec3e6d..700412f6faf 100755 --- a/docker/lite/build.sh +++ b/docker/lite/build.sh @@ -29,15 +29,12 @@ if [[ "$1" == "--prompt"* ]]; then fi flavor=$1 - base_image=vitess/base -make_target=docker_base lite_image=vitess/lite dockerfile=Dockerfile tag=latest + if [[ -n "$flavor" ]]; then - base_image=vitess/base:$flavor - make_target=docker_base_$flavor lite_image=vitess/lite:$flavor dockerfile=Dockerfile.$flavor tag=$flavor @@ -71,40 +68,4 @@ END read fi -# cleanup both temporary folders when exiting, even in case of errors -function cleanup { - rm -rf $PWD/base $PWD/lite -} -trap cleanup EXIT - -# Extract files from vitess/base image -mkdir base -# Ignore permission errors. They occur for directories we do not care e.g. ".git". -# (Copying them fails because they are owned by root and not $UID and have stricter permissions.) -docker run --rm -v $PWD/base:/base -u $UID $base_image bash -c 'cp -R /vt /base/ 2>&1 | grep -v "Permission denied"' || true - -# Grab only what we need -lite=$PWD/lite -vttop=vt/src/vitess.io/vitess -mkdir -p $lite/vt/vtdataroot - -mkdir -p $lite/vt/bin -(cd base/vt/bin; cp mysqlctld vtctld vtctlclient vtgate vttablet vtworker $lite/vt/bin/) - -mkdir -p $lite/$vttop/web -cp -R base/$vttop/web/vtctld $lite/$vttop/web/ -mkdir $lite/$vttop/web/vtctld2 -cp -R base/$vttop/web/vtctld2/app $lite/$vttop/web/vtctld2/ - -mkdir -p $lite/$vttop/config -cp -R base/$vttop/config/* $lite/$vttop/config/ -ln -s /$vttop/config $lite/vt/config - -# remove the base folder now -- not needed for the final build -rm -rf $PWD/base - -# Fix permissions for AUFS workaround -chmod -R o=g lite - -# Build vitess/lite image docker build --no-cache -f $dockerfile -t $lite_image . From b3674f2233d1c25fee65c5b314774bea31f78be3 Mon Sep 17 00:00:00 2001 From: Dan Kozlowski Date: Tue, 5 Mar 2019 11:02:16 -0800 Subject: [PATCH 2/2] Additional changes to explicitly state flavor Signed-off-by: Dan Kozlowski --- docker/bootstrap/Dockerfile.mariadb | 6 ++---- docker/bootstrap/Dockerfile.mariadb103 | 16 ++++++++++++++++ docker/lite/Dockerfile.mariadb | 1 + docker/lite/Dockerfile.mariadb103 | 1 + docker/lite/Dockerfile.mysql56 | 1 + docker/lite/Dockerfile.mysql57 | 1 + docker/lite/Dockerfile.mysql80 | 1 + docker/lite/Dockerfile.percona | 1 + docker/lite/Dockerfile.percona57 | 2 +- docker/lite/Dockerfile.percona80 | 2 +- 10 files changed, 26 insertions(+), 6 deletions(-) create mode 100644 docker/bootstrap/Dockerfile.mariadb103 diff --git a/docker/bootstrap/Dockerfile.mariadb b/docker/bootstrap/Dockerfile.mariadb index ca40724c12f..83dfdff8b74 100644 --- a/docker/bootstrap/Dockerfile.mariadb +++ b/docker/bootstrap/Dockerfile.mariadb @@ -1,9 +1,7 @@ FROM vitess/bootstrap:common -# Install MariaDB 10.3 -RUN apt-key adv --no-tty --recv-keys --keyserver keyserver.ubuntu.com 0xF1656F24C74CD1D8 \ - && echo 'deb http://sfo1.mirrors.digitalocean.com/mariadb/repo/10.3/debian stretch main' > /etc/apt/sources.list.d/mariadb.list \ - && apt-get update \ +# Install MariaDB 10 +RUN apt-get update \ && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ bzip2 \ mariadb-server \ diff --git a/docker/bootstrap/Dockerfile.mariadb103 b/docker/bootstrap/Dockerfile.mariadb103 new file mode 100644 index 00000000000..e626cf4f2e6 --- /dev/null +++ b/docker/bootstrap/Dockerfile.mariadb103 @@ -0,0 +1,16 @@ +FROM vitess/bootstrap:common + +# Install MariaDB 10. +RUN apt-key adv --recv-keys --keyserver keyserver.ubuntu.com 0xF1656F24C74CD1D8 &&\ + add-apt-repository 'deb [arch=amd64] http://ftp.osuosl.org/pub/mariadb/repo/10.3/debian stretch main' + apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + mariadb-server \ + libmariadbclient-dev \ + && rm -rf /var/lib/apt/lists/* + +# Bootstrap Vitess +WORKDIR /vt/src/vitess.io/vitess + +ENV MYSQL_FLAVOR MariaDB103 +USER vitess +RUN ./bootstrap.sh diff --git a/docker/lite/Dockerfile.mariadb b/docker/lite/Dockerfile.mariadb index 5c947ab804c..d25e274c46c 100644 --- a/docker/lite/Dockerfile.mariadb +++ b/docker/lite/Dockerfile.mariadb @@ -41,6 +41,7 @@ ENV GOPATH $VTROOT ENV PATH $VTROOT/bin:$PATH ENV VT_MYSQL_ROOT /usr ENV PKG_CONFIG_PATH $VTROOT/lib +ENV MYSQL_FLAVOR MariaDB # Copy binaries (placed by build.sh) COPY --from=staging /vt/ /vt/ diff --git a/docker/lite/Dockerfile.mariadb103 b/docker/lite/Dockerfile.mariadb103 index 2290a77fbc9..e01e394813f 100644 --- a/docker/lite/Dockerfile.mariadb103 +++ b/docker/lite/Dockerfile.mariadb103 @@ -40,6 +40,7 @@ ENV GOPATH $VTROOT ENV PATH $VTROOT/bin:$PATH ENV VT_MYSQL_ROOT /usr ENV PKG_CONFIG_PATH $VTROOT/lib +ENV MYSQL_FLAVOR MariaDB103 # Copy binaries (placed by build.sh) COPY --from=staging /vt/ /vt/ diff --git a/docker/lite/Dockerfile.mysql56 b/docker/lite/Dockerfile.mysql56 index 89432bccc59..a005a21cb46 100644 --- a/docker/lite/Dockerfile.mysql56 +++ b/docker/lite/Dockerfile.mysql56 @@ -43,6 +43,7 @@ ENV GOPATH $VTROOT ENV PATH $VTROOT/bin:$PATH ENV VT_MYSQL_ROOT /usr ENV PKG_CONFIG_PATH $VTROOT/lib +ENV MYSQL_FLAVOR MySQL56 # Copy binaries (placed by build.sh) COPY --from=staging /vt/ /vt/ diff --git a/docker/lite/Dockerfile.mysql57 b/docker/lite/Dockerfile.mysql57 index 5d442d2e5ca..ebfdb21d5c4 100644 --- a/docker/lite/Dockerfile.mysql57 +++ b/docker/lite/Dockerfile.mysql57 @@ -43,6 +43,7 @@ ENV GOPATH $VTROOT ENV PATH $VTROOT/bin:$PATH ENV VT_MYSQL_ROOT /usr ENV PKG_CONFIG_PATH $VTROOT/lib +ENV MYSQL_FLAVOR MySQL56 # Copy binaries (placed by build.sh) COPY --from=staging /vt/ /vt/ diff --git a/docker/lite/Dockerfile.mysql80 b/docker/lite/Dockerfile.mysql80 index 4dc5f40f2d0..1b98aae565f 100644 --- a/docker/lite/Dockerfile.mysql80 +++ b/docker/lite/Dockerfile.mysql80 @@ -43,6 +43,7 @@ ENV GOPATH $VTROOT ENV PATH $VTROOT/bin:$PATH ENV VT_MYSQL_ROOT /usr ENV PKG_CONFIG_PATH $VTROOT/lib +ENV MYSQL_FLAVOR MySQL56 # Copy binaries (placed by build.sh) COPY --from=staging /vt/ /vt/ diff --git a/docker/lite/Dockerfile.percona b/docker/lite/Dockerfile.percona index c27944b637e..5dd69b7b2cc 100644 --- a/docker/lite/Dockerfile.percona +++ b/docker/lite/Dockerfile.percona @@ -46,6 +46,7 @@ ENV GOPATH $VTROOT ENV PATH $VTROOT/bin:$PATH ENV VT_MYSQL_ROOT /usr ENV PKG_CONFIG_PATH $VTROOT/lib +ENV MYSQL_FLAVOR MySQL56 # Copy binaries (placed by build.sh) COPY --from=staging /vt/ /vt/ diff --git a/docker/lite/Dockerfile.percona57 b/docker/lite/Dockerfile.percona57 index def68853c31..3dc21bcf573 100644 --- a/docker/lite/Dockerfile.percona57 +++ b/docker/lite/Dockerfile.percona57 @@ -47,7 +47,7 @@ ENV GOPATH $VTROOT ENV PATH $VTROOT/bin:$PATH ENV VT_MYSQL_ROOT /usr ENV PKG_CONFIG_PATH $VTROOT/lib - +ENV MYSQL_FLAVOR MySQL56 # Copy binaries (placed by build.sh) COPY --from=staging /vt/ /vt/ diff --git a/docker/lite/Dockerfile.percona80 b/docker/lite/Dockerfile.percona80 index 658b52c4902..b48b8db5c99 100644 --- a/docker/lite/Dockerfile.percona80 +++ b/docker/lite/Dockerfile.percona80 @@ -49,7 +49,7 @@ ENV GOPATH $VTROOT ENV PATH $VTROOT/bin:$PATH ENV VT_MYSQL_ROOT /usr ENV PKG_CONFIG_PATH $VTROOT/lib - +ENV MYSQL_FLAVOR MySQL56 # Copy binaries (placed by build.sh) COPY --from=staging /vt/ /vt/