Regional registries: Hack the manifests!

Allow the gcr.io/google_containers registry to be overridden
regionally by just blasting a new KUBE_ADDON_REGISTRY out. Instead of
adding every addon to Salt and asking all of the other consumers
(Trusty, Juju, Mesos, etc) to change, just script the sed ourselves.

This is probably the 9th grossest thing I've ever done, but it works
well, and it works quickly. I kind of wish it didn't.

cluster/common.sh

@@ -500,6 +500,7 @@ OPENCONTRAIL_PUBLIC_SUBNET: $(yaml-quote ${OPENCONTRAIL_PUBLIC_SUBNET:-})
 E2E_STORAGE_TEST_ENVIRONMENT: $(yaml-quote ${E2E_STORAGE_TEST_ENVIRONMENT:-})
 KUBE_IMAGE_TAG: $(yaml-quote ${KUBE_IMAGE_TAG:-})
 KUBE_DOCKER_REGISTRY: $(yaml-quote ${KUBE_DOCKER_REGISTRY:-})
+KUBE_ADDON_REGISTRY: $(yaml-quote ${KUBE_ADDON_REGISTRY:-})
 MULTIZONE: $(yaml-quote ${MULTIZONE:-})
 NON_MASQUERADE_CIDR: $(yaml-quote ${NON_MASQUERADE_CIDR:-})
 EOF

cluster/gce/config-default.sh

@@ -22,6 +22,7 @@ source "${KUBE_ROOT}/cluster/gce/config-common.sh"
 GCLOUD=gcloud
 ZONE=${KUBE_GCE_ZONE:-us-central1-b}
 RELEASE_REGION_FALLBACK=${RELEASE_REGION_FALLBACK:-false}
+REGIONAL_KUBE_ADDONS=${REGIONAL_KUBE_ADDONS:-true}
 NODE_SIZE=${NODE_SIZE:-n1-standard-2}
 NUM_NODES=${NUM_NODES:-3}
 MASTER_SIZE=${MASTER_SIZE:-n1-standard-$(get-master-size)}

cluster/gce/config-test.sh

@@ -22,6 +22,7 @@ source "${KUBE_ROOT}/cluster/gce/config-common.sh"
 GCLOUD=gcloud
 ZONE=${KUBE_GCE_ZONE:-us-central1-b}
 RELEASE_REGION_FALLBACK=${RELEASE_REGION_FALLBACK:-false}
+REGIONAL_KUBE_ADDONS=${REGIONAL_KUBE_ADDONS:-true}
 NODE_SIZE=${NODE_SIZE:-n1-standard-2}
 NUM_NODES=${NUM_NODES:-3}
 MASTER_SIZE=${MASTER_SIZE:-n1-standard-$(get-master-size)}

cluster/gce/util.sh

@@ -133,10 +133,12 @@ function copy-to-staging() {
 # give us an API for this, so we hardcode it.
 #
 # Assumed vars:
-#   REGIONAL_RELEASE
+#   RELEASE_REGION_FALLBACK
+#   REGIONAL_KUBE_ADDONS
 #   ZONE
 # Vars set:
 #   PREFERRED_REGION
+#   KUBE_ADDON_REGISTRY
 function set-preferred-region() {
   case ${ZONE} in
     asia-*)
@@ -149,9 +151,19 @@ function set-preferred-region() {
       PREFERRED_REGION=("us" "eu" "asia")
       ;;
   esac
+  local -r preferred="${PREFERRED_REGION[0]}"
 
   if [[ "${RELEASE_REGION_FALLBACK}" != "true" ]]; then
-    PREFERRED_REGION=( "${PREFERRED_REGION[0]}" )
+    PREFERRED_REGION=( "${preferred}" )
+  fi
+
+  # If we're using regional GCR, and we're outside the US, go to the
+  # regional registry. The gcr.io/google_containers registry is
+  # appropriate for US (for now).
+  if [[ "${REGIONAL_KUBE_ADDONS}" == "true" ]] && [[ "${preferred}" != "us" ]]; then
+    KUBE_ADDON_REGISTRY="${preferred}.gcr.io/google_containers"
+  else
+    KUBE_ADDON_REGISTRY="gcr.io/google_containers"
   fi
 }
 

cluster/saltbase/install.sh

@@ -85,6 +85,19 @@ cat <<EOF >>"${docker_images_sls_file}"
 kube_docker_registry: '$(echo ${KUBE_DOCKER_REGISTRY:-gcr.io/google_containers})'
 EOF
 
+# TODO(zmerlynn): Forgive me, this is really gross. But in order to
+# avoid breaking the non-Salt deployments, which already painfully
+# have to templatize a couple of the add-ons anyways, manually
+# templatize the addon registry for regional support. When we get
+# better templating, we can fix this.
+readonly kube_addon_registry="${KUBE_ADDON_REGISTRY:-gcr.io/google_containers}"
+if [[ "${kube_addon_registry}" != "gcr.io/google_containers" ]]; then
+  find /srv/salt-new -name \*.yaml -or -name \*.yaml.in | \
+    xargs sed -ri "s@(image:\s.*)gcr.io/google_containers@\1${kube_addon_registry}@"
+  # All the legacy .manifest files with hardcoded gcr.io are JSON.
+  find /srv/salt-new -name \*.manifest -or -name \*.json | \
+    xargs sed -ri "s@(image\":\s+\")gcr.io/google_containers@\1${kube_addon_registry}@"
+fi
 
 echo "+++ Swapping in new configs"
 for dir in "${SALTDIRS[@]}"; do