From 74a4a8c90f4d62c785a28f8ceba9c27e2b7042ac Mon Sep 17 00:00:00 2001 From: Victor Lin <13424970+victorlin@users.noreply.github.com> Date: Tue, 13 Sep 2022 19:35:01 +0000 Subject: [PATCH] copy-images: Use skopeo to "push" images MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Instead of pushing images that are available locally (different from a local registry) to Docker Hub, the proper way to make them available in the destination is to copy from the source registry directly without unnecessary pull/push. This is what skopeo can do. A containerized version of skopeo is used since the version pre-installed on the GitHub Actions Ubuntu runners is stuck in the past¹ and does not support --multi-arch², which may be used in the future. Also note that the tag-latest script is replaced by a conditional block in the new script which copies the tag from one registry to `latest` on another another. ¹ https://github.com/actions/runner-images/issues/6180#issuecomment-1245858668 ² https://github.com/containers/skopeo/releases/tag/v1.6.0 --- .github/workflows/ci.yml | 9 ++-- README.md | 9 ++-- devel/copy-images | 107 +++++++++++++++++++++++++++++++++++++++ devel/push | 30 ----------- devel/tag-latest | 19 ------- 5 files changed, 115 insertions(+), 59 deletions(-) create mode 100755 devel/copy-images delete mode 100755 devel/push delete mode 100755 devel/tag-latest diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f911b7ec..69e96d3e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,8 +33,8 @@ jobs: password: ${{ secrets.DOCKER_PASSWORD }} - if: github.event_name != 'pull_request' && startsWith(env.TAG, 'branch-') - name: Push $TAG (non-default branch) - run: ./devel/push $TAG + name: Copy $TAG images to Docker Hub (non-default branch) + run: ./devel/copy-images -i localhost:$REGISTRY_PORT -o docker.io -t $TAG - uses: actions/setup-python@v4 with: @@ -46,10 +46,9 @@ jobs: nextstrain build --image localhost:$REGISTRY_PORT/nextstrain/base:$TAG zika-tutorial -F - if: startsWith(env.TAG, 'build-') - name: Push $TAG + latest (default branch) + name: Copy $TAG + latest images to Docker Hub (default branch) run: | - ./devel/tag-latest $TAG - ./devel/push latest $TAG + ./devel/copy-images -i localhost:$REGISTRY_PORT -o docker.io -t $TAG -l - if: always() run: ./devel/stop-localhost-registry diff --git a/README.md b/README.md index 437b6cf3..16eabf77 100644 --- a/README.md +++ b/README.md @@ -66,12 +66,11 @@ during development iterations. To push images you've built locally to Docker Hub, you can run: - ./devel/push latest + ./devel/copy-images -t -This will publish your local `nextstrain/base:latest` image. This is also what -happens if you run `./devel/push` with no tags specified. If you have images -with other tags, you may provide those tags in addition to or instead of -`latest`. +This will copy the `nextstrain/base:` and `nextstrain/base-builder:` +images from the local Docker registry to Docker Hub. See instructions at the top +of the script for more options. ### Best practices diff --git a/devel/copy-images b/devel/copy-images new file mode 100755 index 00000000..4897e35c --- /dev/null +++ b/devel/copy-images @@ -0,0 +1,107 @@ +#!/bin/bash +# +# Copy the Nextstrain images from one Docker registry (-i ) to +# another (-o ). +# +# If authentication is required for a registry, make the file available as +# ~/.docker/-config.json. +# +# This copies just the tag specified by -t . If the boolean -l flag is +# specified, the tag will also be copied to "latest" on the destnation. +# +# Errors if any of the tagged images have already been pushed. +# +set -euo pipefail + +# Set default values. +registry_in=localhost:5000 +registry_out=docker.io +tag="" +push_latest=false + +# Read command-line arguments. +while getopts "i:o:t:l" opt; do + case "$opt" in + i) registry_in="$OPTARG";; + o) registry_out="$OPTARG";; + t) tag="$OPTARG";; + l) push_latest=true;; + *) echo "Usage: $0 [-i ] [-o ] [-t ] [-l]" 1>&2; exit 1;; + esac +done + +if [[ "$tag" = "" ]]; then + echo "Please provide a tag." >&2 + exit 1 +fi + +BUILDER_IMAGE=nextstrain/base-builder +FINAL_IMAGE=nextstrain/base + +if [[ $(docker image inspect --format "{{.RepoDigests}}" $FINAL_IMAGE:$tag) != '[]' || $(docker image inspect --format "{{.RepoDigests}}" $BUILDER_IMAGE:$tag) != '[]' ]]; then + echo "At least one of $BUILDER_IMAGE:$tag and $FINAL_IMAGE:$tag has already been pushed. This can happen if the newly built image is not available in the local registry." >&2 + exit 1 +fi + + +# Use Skopeo via a Docker container¹ to copy a tagged image. +# +# If a registry starts with localhost, do not require HTTPS or verify certificates, and +# access the registry without authentication. +# +# ¹ https://github.com/containers/skopeo/blob/07da29fd371dd88615a0b86e91c6824237484172/install.md#container-images +copy-image() { + local src_registry="$1" + local src_image="$2" + local dest_registry="$3" + local dest_image="$4" + + docker_run_params=(--rm --network=host) + skopeo_copy_params=(--multi-arch=all) + + if [[ "$src_registry" == localhost* ]]; then + skopeo_copy_params+=(--src-tls-verify=false) + else + docker_run_params+=(-v "$HOME/.docker/${src_registry}-config.json:/${src_registry}-config.json:ro") + skopeo_copy_params+=(--src-authfile "${src_registry}-config.json") + fi + + if [[ "$dest_registry" == localhost* ]]; then + skopeo_copy_params+=(--dest-tls-verify=false) + else + docker_run_params+=(-v "$HOME/.docker/${dest_registry}-config.json:/${dest_registry}-config.json:ro") + skopeo_copy_params+=(--dest-authfile "${dest_registry}-config.json") + fi + + docker run "${docker_run_params[@]}" \ + quay.io/skopeo/stable \ + copy "${skopeo_copy_params[@]}" \ + "docker://$src_registry/$src_image" \ + "docker://$dest_registry/$dest_image" +} + +# copy source registry $tag to destination registry $tag +copy-image \ + "$registry_in" \ + "$FINAL_IMAGE:$tag" \ + "$registry_out" \ + "$FINAL_IMAGE:$tag" +copy-image \ + "$registry_in" \ + "$BUILDER_IMAGE:$tag" \ + "$registry_out" \ + "$BUILDER_IMAGE:$tag" + +if [[ "$push_latest" = true ]]; then + # copy source registry $tag to destination registry latest + copy-image \ + "$registry_in" \ + "$FINAL_IMAGE:$tag" \ + "$registry_out" \ + "$FINAL_IMAGE:latest" + copy-image \ + "$registry_in" \ + "$BUILDER_IMAGE:$tag" \ + "$registry_out" \ + "$BUILDER_IMAGE:latest" +fi diff --git a/devel/push b/devel/push deleted file mode 100755 index 028bcdac..00000000 --- a/devel/push +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash -# -# Push the nextstrain/base and nextstrain/base-builder images to Docker Hub. -# -# By default this publishes the "latest" tag, but you can provide other tags in -# addition to or instead of "latest". -# -# Errors if any of the tagged images have already been pushed. -# -set -euo pipefail - -# If no tags are given, default to "latest". -if [[ $# -eq 0 ]]; then - set -- latest -fi - -BUILDER_IMAGE=nextstrain/base-builder -FINAL_IMAGE=nextstrain/base - -for tag in "$@"; do - if [[ $(docker image inspect --format "{{.RepoDigests}}" $BUILDER_IMAGE:$tag) != '[]' || $(docker image inspect --format "{{.RepoDigests}}" $FINAL_IMAGE:$tag) != '[]' ]]; then - echo "At least one of $BUILDER_IMAGE:$tag and $FINAL_IMAGE:$tag has already been pushed. This can happen if the newly built image is not available in the local registry." >&2 - exit 1 - fi -done - -for tag in "$@"; do - docker push "$BUILDER_IMAGE:$tag" - docker push "$FINAL_IMAGE:$tag" -done diff --git a/devel/tag-latest b/devel/tag-latest deleted file mode 100755 index 2901fa11..00000000 --- a/devel/tag-latest +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash -# -# Assign the latest tag to the nextstrain/base:$tag and -# nextstrain/base-builder:$tag images. -# -set -euo pipefail - -if [[ $# -ne 1 ]]; then - echo "Please provide a tag." >&2 - exit 1 -fi - -tag="$1" - -BUILDER_IMAGE=nextstrain/base-builder -FINAL_IMAGE=nextstrain/base - -docker tag "$BUILDER_IMAGE":{"$tag",latest} -docker tag "$FINAL_IMAGE":{"$tag",latest}