WIP: add scripts for disaster recovery

This is a rough cut of some tooling for basic backups from prod to a
backup.

TODO: replace $cip_tag with something actually valid from the real world

infra/backup_tools/backup.sh

@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+#
+# Copyright 2019 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# USAGE NOTES
+#
+# Backs up prod registries. This is a thin orchestrator as all the heavy lifting
+# is done by copy.sh and record_consistency.sh.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+if (( $# != 1 )); then
+    "usage: ./record_consistency.sh <svc_acct_key_path>"
+    exit 1
+fi
+
+prod_repos=(
+    asia.gcr.io/k8s-artifacts-prod
+    eu.gcr.io/k8s-artifacts-prod
+    us.gcr.io/k8s-artifacts-prod
+)
+sa_key_path="${1}"
+
+SCRIPT_ROOT="$(dirname "$(readlink -f "$0")")"
+cd "${SCRIPT_ROOT}"
+
+# Activate creds.
+gcloud auth activate-service-account --key-file "${sa_key_path}"
+
+# Copy each region to its backup.
+for prod_repo in "${prod_repos[@]}"; do
+    ./copy.sh "${prod_repo}" "${prod-repo}-backup" "${sa_key_path}"
+done
+
+# Write snapshots for all prod repos and their backups.
+for prod_repo in "${prod_repos[@]}"; do
+    ./record_consistency "${prod_repo}" "${prod-repo}-backup"
+done
+
+# Save snapshots to backup.
+tarball="snapshots.tar.xz"
+tar cvJf "${tarball}" snapshot*
+gsutil cp "${tarball}" "gs://${prod-repo}-backup"

infra/backup_tools/copy.sh

@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+#
+# Copyright 2019 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# USAGE NOTES
+#
+# This script backs up images in <SOUCE-GCR-REPO> with the "gcrane" binary.
+#
+# It is expected that this script will be called from an outer loop, looping
+# through all GCRs that need to be backed up (e.g., once for each prod registry
+# in each region).
+
+set -o errexit
+set -o nounset
+set -o pipefail
+set -o xtrace
+
+# Check arguments.
+if (( $# != 3 )); then
+    "usage: ./record.sh <SOURCE-GCR-REPO> <BACKUP-GCR-REPO> <svc_acct_key_path>"
+    exit 1
+fi
+
+# Configure variables.
+# We use a timestamp of the form YYYY/MM/DD because this makes the backup
+# folders more easily traversable from a human perspective.
+timestamp="$(date -u +"%Y/%m/%d")"
+source_gcr_repo="${1}" # "us.gcr.io/k8s-artifacts-prod"
+backup_gcr_repo="${2}" # "us.gcr.io/k8s-artifacts-prod-backup"
+sa_key_path="${3}"
+
+# Perform backup by copying all images recursively over.
+docker run \
+    -v "${sa_key_path}":/auth.json \
+    --env GOOGLE_APPLICATION_CREDENTIALS=/auth.json \
+    gcr.io/go-containerregistry/gcrane \
+        cp -r "${source_gcr_repo}" "${backup_gcr_repo}/${timestamp}"

infra/backup_tools/record_consistency.sh

@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+#
+# Copyright 2019 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# USAGE NOTES
+#
+# This script performs a consistency check between two GCRs and saves the
+# results to files locally.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+if (( $# != 2 )); then
+    "usage: ./record_consistency.sh <GCR-REPO-1> <GCR-REPO-2>"
+    exit 1
+fi
+
+get_snapshot_from_gcr()
+{
+    local repo_src
+
+    repo_src="${1}"
+
+    docker run \
+        "${cip}" \
+        "cip -snapshot=${repo_src} -minimal-snapshot -output-format=CSV -no-service-account"
+}
+
+#cip_tag="???"
+#cip_image="us.gcr.io/k8s-artifacts-prod/artifact-promoter/cip"
+cip_tag="latest"
+cip_image="gcr.io/k8s-staging-artifact-promoter/cip"
+cip="${cip_image}:${cip_tag}"
+
+gcr_repo_1="$1"
+gcr_repo_2="$2"
+
+snapshot_from_gcr_1="snapshot.${gcr_repo_1//\//__}.csv"
+snapshot_from_gcr_2="snapshot.${gcr_repo_2//\//__}.csv"
+
+get_snapshot_from_gcr "${gcr_repo_1}" \
+    > "${snapshot_from_gcr_1}"
+get_snapshot_from_gcr "${gcr_repo_2}" \
+    > "${snapshot_from_gcr_2}"
+
+# Now compare the two.
+difference="$(diff -u "${snapshot_from_gcr_1}" "${snapshot_from_gcr_2}" || true)"
+if [[ -n "${difference}" ]]; then
+    echo "ERROR: images differ between ${gcr_repo_1} and ${gcr_repo_2}"
+    echo "${difference}"
+    exit 1
+fi
+echo "OK (no differences detected between ${gcr_repo_1} and ${gcr_repo_2})"