[gardener-local] Add pull-through cache registry (gardener#6591)

* Switch to registry.k8s.io

With this, we reduce the number of upstream registries we use.

Co-Authored-By: Rafael Franzke <rafael.franzke@sap.com>

* Deploy pull through caches and use it in local seeds and shoots

The local registry is now also exposed in the KIND_ENV=local setup for consistency reasons.
It is not used by the setup currently, though.

Co-Authored-By: Rafael Franzke <rafael.franzke@sap.com>

* Parallelize image pulls in local clusters

Co-Authored-By: Rafael Franzke <rafael.franzke@sap.com>

* Enhance docs

* Allow configuring `KubeletConfiguration.Registry{PullQPS,Burst}`

If the params are given in the shoot spec, use them.
Otherwise, don't set them in the config file and let kubelet default them.

* Increase image pull rate in local clusters

* Add validation for new fields

Co-authored-by: Rafael Franzke <rafael.franzke@sap.com>

Makefile

@@ -276,33 +276,38 @@ kind-up kind-down gardener-up gardener-down register-local-env tear-down-local-e
 
 kind2-up kind2-down gardenlet-kind2-up gardenlet-kind2-down: export KUBECONFIG = $(GARDENER_LOCAL2_KUBECONFIG)
 
-kind2-up kind2-down: TARGET_SUFFIX := 2
-
-kind-up kind2-up: $(KIND) $(KUBECTL)
-ifeq ($(MAKECMDGOALS), kind-up)
+kind-up: $(KIND) $(KUBECTL)
 	mkdir -m 775 -p $(REPO_ROOT)/dev/local-backupbuckets $(REPO_ROOT)/dev/local-registry
-endif
-	$(KIND) create cluster --name gardener-local$(TARGET_SUFFIX) --config $(REPO_ROOT)/example/gardener-local/kind$(TARGET_SUFFIX)/cluster-$(KIND_ENV).yaml --kubeconfig $(KUBECONFIG)
-	docker exec gardener-local$(TARGET_SUFFIX)-control-plane sh -c "sysctl fs.inotify.max_user_instances=8192" # workaround https://kind.sigs.k8s.io/docs/user/known-issues/#pod-errors-due-to-too-many-open-files
-	cp $(KUBECONFIG) $(REPO_ROOT)/example/provider-local/seed-kind$(TARGET_SUFFIX)/base/kubeconfig
+	$(KIND) create cluster --name gardener-local --config $(REPO_ROOT)/example/gardener-local/kind/cluster-$(KIND_ENV).yaml --kubeconfig $(KUBECONFIG)
+	docker exec gardener-local-control-plane sh -c "sysctl fs.inotify.max_user_instances=8192" # workaround https://kind.sigs.k8s.io/docs/user/known-issues/#pod-errors-due-to-too-many-open-files
+	cp $(KUBECONFIG) $(REPO_ROOT)/example/provider-local/seed-kind/base/kubeconfig
+	$(KUBECTL) apply -k $(REPO_ROOT)/example/gardener-local/registry --server-side
+	$(KUBECTL) wait --for=condition=available deployment -l app=registry -n registry --timeout 5m
 	$(KUBECTL) apply -k $(REPO_ROOT)/example/gardener-local/calico --server-side
 	$(KUBECTL) apply -k $(REPO_ROOT)/example/gardener-local/metrics-server --server-side
 
-kind-down kind2-down: $(KIND)
-	$(KIND) delete cluster --name gardener-local$(TARGET_SUFFIX)
-	rm -f $(REPO_ROOT)/example/provider-local/seed-kind$(TARGET_SUFFIX)/base/kubeconfig
-ifeq ($(MAKECMDGOALS), kind-down)
+kind2-up: $(KIND) $(KUBECTL)
+	$(KIND) create cluster --name gardener-local2 --config $(REPO_ROOT)/example/gardener-local/kind2/cluster-$(KIND_ENV).yaml --kubeconfig $(KUBECONFIG)
+	docker exec gardener-local2-control-plane sh -c "sysctl fs.inotify.max_user_instances=8192" # workaround https://kind.sigs.k8s.io/docs/user/known-issues/#pod-errors-due-to-too-many-open-files
+	cp $(KUBECONFIG) $(REPO_ROOT)/example/provider-local/seed-kind2/base/kubeconfig
+	$(KUBECTL) apply -k $(REPO_ROOT)/example/gardener-local/calico --server-side
+	$(KUBECTL) apply -k $(REPO_ROOT)/example/gardener-local/metrics-server --server-side
+
+kind-down: $(KIND)
+	$(KIND) delete cluster --name gardener-local
+	rm -f $(REPO_ROOT)/example/provider-local/seed-kind/base/kubeconfig
 	rm -rf dev/local-backupbuckets
-endif
+
+kind2-down: $(KIND)
+	$(KIND) delete cluster --name gardener-local2
+	rm -f $(REPO_ROOT)/example/provider-local/seed-kind2/base/kubeconfig
 
 # speed-up skaffold deployments by building all images concurrently
 export SKAFFOLD_BUILD_CONCURRENCY = 0
 # use static label for skaffold to prevent rolling all gardener components on every `skaffold` invocation
 gardener-up gardener-down gardenlet-kind2-up gardenlet-kind2-down: export SKAFFOLD_LABEL = skaffold.dev/run-id=gardener-local
 
 gardener-up: $(SKAFFOLD) $(HELM) $(KUBECTL)
-	$(KUBECTL) apply -k $(REPO_ROOT)/example/gardener-local/registry --server-side
-	$(KUBECTL) wait --for=condition=available deployment -l app=registry -n registry --timeout=2m
 	SKAFFOLD_DEFAULT_REPO=localhost:5001 SKAFFOLD_PUSH=true $(SKAFFOLD) run
 
 gardener-down: $(SKAFFOLD) $(HELM) $(KUBECTL)

charts/images.yaml

@@ -18,7 +18,7 @@ images:
 # Seed bootstrap
 - name: pause-container
   sourceRepository: github.com/kubernetes/kubernetes/blob/master/build/pause/Dockerfile
-  repository: k8s.gcr.io/pause
+  repository: registry.k8s.io/pause
   tag: "3.7"
   labels:
   - name: cloud.gardener.cnudie/dso/scanning-hints/binary_id/v1
@@ -37,7 +37,7 @@ images:
   tag: "v0.8.0"
 - name: nginx-ingress-controller-seed
   sourceRepository: github.com/kubernetes/ingress-nginx
-  repository: k8s.gcr.io/ingress-nginx/controller
+  repository: registry.k8s.io/ingress-nginx/controller
   tag: "v0.49.3"
   targetVersion: "< 1.22"
 - name: nginx-ingress-controller-seed
@@ -54,24 +54,24 @@ images:
 #   hyperkube is used for kubectl + kubelet binaries on the worker nodes
 - name: hyperkube
   sourceRepository: github.com/kubernetes/kubernetes
-  repository: k8s.gcr.io/hyperkube
+  repository: registry.k8s.io/hyperkube
   targetVersion: "< 1.19"
 - name: hyperkube
   sourceRepository: github.com/gardener/hyperkube
   repository: eu.gcr.io/gardener-project/hyperkube
   targetVersion: ">= 1.19"
 - name: kube-apiserver
   sourceRepository: github.com/kubernetes/kubernetes
-  repository: k8s.gcr.io/kube-apiserver
+  repository: registry.k8s.io/kube-apiserver
 - name: kube-controller-manager
   sourceRepository: github.com/kubernetes/kubernetes
-  repository: k8s.gcr.io/kube-controller-manager
+  repository: registry.k8s.io/kube-controller-manager
 - name: kube-scheduler
   sourceRepository: github.com/kubernetes/kubernetes
-  repository: k8s.gcr.io/kube-scheduler
+  repository: registry.k8s.io/kube-scheduler
 - name: kube-proxy
   sourceRepository: github.com/kubernetes/kubernetes
-  repository: k8s.gcr.io/kube-proxy
+  repository: registry.k8s.io/kube-proxy
 - name: cluster-autoscaler
   sourceRepository: github.com/gardener/autoscaler
   repository: eu.gcr.io/gardener-project/gardener/autoscaler/cluster-autoscaler
@@ -143,12 +143,12 @@ images:
   tag: v0.22.0
 - name: metrics-server
   sourceRepository: github.com/kubernetes-sigs/metrics-server
-  repository: k8s.gcr.io/metrics-server/metrics-server
+  repository: registry.k8s.io/metrics-server/metrics-server
   tag: v0.5.2
   targetVersion: "< 1.19"
 - name: metrics-server
   sourceRepository: github.com/kubernetes-sigs/metrics-server
-  repository: k8s.gcr.io/metrics-server/metrics-server
+  repository: registry.k8s.io/metrics-server/metrics-server
   tag: v0.6.1
   targetVersion: ">= 1.19"
 
@@ -167,7 +167,7 @@ images:
   tag: "1.9.3"
 - name: node-local-dns
   sourceRepository: github.com/kubernetes/kubernetes/blob/master/cluster/addons/dns/nodelocaldns
-  repository: k8s.gcr.io/dns/k8s-dns-node-cache
+  repository: registry.k8s.io/dns/k8s-dns-node-cache
   tag: "1.22.8"
 - name: node-problem-detector
   sourceRepository: github.com/gardener/node-problem-detector
@@ -212,13 +212,13 @@ images:
   labels: *optionalAddonLabels
 - name: nginx-ingress-controller
   sourceRepository: github.com/kubernetes/ingress-nginx
-  repository: k8s.gcr.io/ingress-nginx/controller
+  repository: registry.k8s.io/ingress-nginx/controller
   tag: "v0.49.3"
   targetVersion: ">= 1.20, < 1.22"
   labels: *optionalAddonLabels
 - name: nginx-ingress-controller
   sourceRepository: github.com/kubernetes/ingress-nginx
-  repository: k8s.gcr.io/ingress-nginx/controller-chroot
+  repository: registry.k8s.io/ingress-nginx/controller-chroot
   tag: "v1.2.1"
   targetVersion: ">= 1.22"
   labels: *optionalAddonLabels
@@ -269,15 +269,15 @@ images:
 # VPA
 - name: vpa-admission-controller
   sourceRepository: github.com/kubernetes/autoscaler
-  repository: k8s.gcr.io/autoscaling/vpa-admission-controller
+  repository: registry.k8s.io/autoscaling/vpa-admission-controller
   tag: "0.11.0"
 - name: vpa-recommender
   sourceRepository: github.com/kubernetes/autoscaler
-  repository: k8s.gcr.io/autoscaling/vpa-recommender
+  repository: registry.k8s.io/autoscaling/vpa-recommender
   tag: "0.11.0"
 - name: vpa-updater
   sourceRepository: github.com/kubernetes/autoscaler
-  repository: k8s.gcr.io/autoscaling/vpa-updater
+  repository: registry.k8s.io/autoscaling/vpa-updater
   tag: "0.11.0"
 - name: vpa-exporter
   sourceRepository: github.com/gardener/vpa-exporter
@@ -293,7 +293,7 @@ images:
 # Horizontal cluster-proportional-autoscaler
 - name: cluster-proportional-autoscaler
   sourceRepository: https://github.com/kubernetes-sigs/cluster-proportional-autoscaler
-  repository: k8s.gcr.io/cpa/cluster-proportional-autoscaler
+  repository: registry.k8s.io/cpa/cluster-proportional-autoscaler
   tag: "1.8.6"
 
 # Istio

docs/api-reference/core.md

@@ -4693,6 +4693,35 @@ bool
 Default: true</p>
 </td>
 </tr>
+<tr>
+<td>
+<code>registryPullQPS</code></br>
+<em>
+int32
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>RegistryPullQPS is the limit of registry pulls per second. The value must not be a negative number.
+Setting it to 0 means no limit.
+Default: 5</p>
+</td>
+</tr>
+<tr>
+<td>
+<code>registryBurst</code></br>
+<em>
+int32
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>RegistryBurst is the maximum size of bursty pulls, temporarily allows pulls to burst to this number,
+while still not exceeding registryPullQPS. The value must not be a negative number.
+Only used if registryPullQPS is greater than 0.
+Default: 10</p>
+</td>
+</tr>
 </tbody>
 </table>
 <h3 id="core.gardener.cloud/v1beta1.KubeletConfigEviction">KubeletConfigEviction

docs/deployment/getting_started_locally.md

@@ -30,7 +30,16 @@ This command sets up a new KinD cluster named `gardener-local` and stores the ku
 > It might be helpful to copy this file to `$HOME/.kube/config` since you will need to target this KinD cluster multiple times.
 Alternatively, make sure to set your `KUBECONFIG` environment variable to `./example/gardener-local/kind/kubeconfig` for all future steps via `export KUBECONFIG=example/gardener-local/kind/kubeconfig`.
 
-All following steps assume that your are using this kubeconfig.
+All following steps assume that you are using this kubeconfig.
+
+Additionally, this command also deploys a local container registry to the cluster as well as a few registry mirrors, that are set up as a pull-through cache for all upstream registries Gardener uses by default.
+This is done to speed up image pulls across local clusters.
+The local registry can be accessed as `localhost:5001` for pushing and pulling.
+The storage directories of the registries are mounted to the host machine under `dev/local-registry`.
+With this, mirrored images don't have to be pulled again after recreating the cluster.
+
+The command also deploys a default [calico](https://github.com/projectcalico/calico) installation as the cluster's CNI implementation with `NetworkPolicy` support (the default `kindnet` CNI doesn't provide `NetworkPolicy` support).
+Furthermore, it deploys the [metrics-server](https://github.com/kubernetes-sigs/metrics-server) in order to support HPA and VPA on the seed cluster.
 
 ## Setting up Gardener
 

docs/deployment/image_vector.md

@@ -33,19 +33,19 @@ That means that the Gardenlet will use the `pause-container` in with tag `3.0` f
 images:
 - name: pause-container
   sourceRepository: github.com/kubernetes/kubernetes/blob/master/build/pause/Dockerfile
-  repository: k8s.gcr.io/pause
+  repository: registry.k8s.io/pause
   tag: "3.0"
   architectures:
   - amd64
 - name: pause-container
   sourceRepository: github.com/kubernetes/kubernetes/blob/master/build/pause/Dockerfile
-  repository: k8s.gcr.io/pause
+  repository: registry.k8s.io/pause
   tag: "3.0"
   architectures:
   - arm64
 - name: pause-container
   sourceRepository: github.com/kubernetes/kubernetes/blob/master/build/pause/Dockerfile
-  repository: k8s.gcr.io/pause
+  repository: registry.k8s.io/pause
   tag: "3.0"
   architectures:
   - amd64