diff --git a/docs/development/testmachinery_tests.md b/docs/development/testmachinery_tests.md index 513979239e1..e5cc8f0cb1c 100644 --- a/docs/development/testmachinery_tests.md +++ b/docs/development/testmachinery_tests.md @@ -10,6 +10,7 @@ This manual gives an overview about test machinery tests in Gardener. - [Add a new test](#add-a-new-test) - [Test Labels](#test-labels) - [Framework](#framework) +- [Container Images](#container-images) ## Structure @@ -315,3 +316,19 @@ go test -mod=vendor -timeout=0 ./test/testmachinery/system/complete_reconcile \ -project-namespace=$PROJECT_NAMESPACE \ -gardenerVersion=$GARDENER_VERSION # needed to validate the last acted gardener version of a shoot ``` + +## Container Images + +Test machinery tests usually deploy a workload to the Shoot cluster as part of the test execution. When introducing a new container image, consider the following: + +- Make sure the container image is multi-arch. + - Tests are executed against `amd64` and `arm64` based worker Nodes. +- Do not use container images from Docker Hub. + - Docker Hub has rate limiting (see [Download rate limit](https://docs.docker.com/docker-hub/download-rate-limit/)). For anonymous users, the rate limit is set to 100 pulls per 6 hours per IP address. In some fenced environments the network setup can be such that all egress connections are issued from single IP (or set of IPs). In such scenarios the allowed rate limit can be exhausted too fast. See https://github.com/gardener/gardener/issues/4160. + - Docker Hub registry doesn't support pulling images over IPv6 (see [Beta IPv6 Support on Docker Hub Registry](https://www.docker.com/blog/beta-ipv6-support-on-docker-hub-registry/)). + - Avoid manually copying Docker Hub images to Gardener GCR (`eu.gcr.io/gardener-project/3rd/`). Use the existing prow job for this (see [Copy Images](https://github.com/gardener/ci-infra/tree/master/config/images)). + - If possible, use a Kubernetes e2e image (`registry.k8s.io/e2e-test-images/`). + - In some cases, there is already a Kubernetes e2e image alternative of the Docker Hub image. + - For example, use `registry.k8s.io/e2e-test-images/busybox` instead of `eu.gcr.io/gardener-project/3rd/busybox` or `docker.io/busybox`. + - Kubernetes has multiple test images - see https://github.com/kubernetes/kubernetes/tree/v1.27.0/test/images. `agnhost` is the most widely used image in Kubernetes e2e tests. It contains multiple testing related binaries inside such as `pause`, `logs-generator`, `serve-hostname`, `webhook` and others. See all of them in the [agnhost's README.md](https://github.com/kubernetes/kubernetes/blob/v1.27.0/test/images/agnhost/README.md). + - The list of available Kubernetes e2e images and tags can be checked in [this page](https://github.com/kubernetes/k8s.io/blob/main/registry.k8s.io/images/k8s-staging-e2e-test-images/images.yaml). diff --git a/test/framework/k8s_utils.go b/test/framework/k8s_utils.go index 8929f0ae0f7..82777cdca35 100644 --- a/test/framework/k8s_utils.go +++ b/test/framework/k8s_utils.go @@ -543,7 +543,7 @@ func DeployRootPod(ctx context.Context, c client.Client, namespace string, noden Containers: []corev1.Container{ { Name: "root-container", - Image: "eu.gcr.io/gardener-project/3rd/busybox:1.29.3", + Image: "registry.k8s.io/e2e-test-images/busybox:1.29-4", Command: []string{ "sleep", "10000000", diff --git a/test/framework/resources/templates/logger-app.yaml.tpl b/test/framework/resources/templates/logger-app.yaml.tpl index 418d74873c8..0a795ce85a8 100644 --- a/test/framework/resources/templates/logger-app.yaml.tpl +++ b/test/framework/resources/templates/logger-app.yaml.tpl @@ -18,6 +18,8 @@ spec: spec: containers: - name: logger + # A custom agnhost image (eu.gcr.io/gardener-project/3rd/agnhost) is used instead of the upstream one (registry.k8s.io/e2e-test-images/agnhost) + # because this Deployment is created in a Seed cluster and the image needs to be signed with particular keys. image: eu.gcr.io/gardener-project/3rd/agnhost:2.40 command: ["/bin/sh"] args: diff --git a/test/framework/resources/templates/network-nginx-daemonset.yaml.tpl b/test/framework/resources/templates/network-nginx-daemonset.yaml.tpl index 91996cebf27..a909c562b21 100644 --- a/test/framework/resources/templates/network-nginx-daemonset.yaml.tpl +++ b/test/framework/resources/templates/network-nginx-daemonset.yaml.tpl @@ -14,12 +14,12 @@ spec: app: net-nginx spec: containers: - - image: eu.gcr.io/gardener-project/3rd/nginx:1.17.6 - name: net-nginx + - name: net-nginx + image: registry.k8s.io/e2e-test-images/nginx:1.15-4 ports: - containerPort: 80 - - image: eu.gcr.io/gardener-project/3rd/curlimages/curl:7.70.0 - name: net-curl - command: ["sh", "-c"] - args: ["sleep 300"] + - name: pause + image: registry.k8s.io/e2e-test-images/agnhost:2.40 + args: + - pause serviceAccountName: {{ .name }} diff --git a/test/framework/resources/templates/simple-load-deployment.yaml.tpl b/test/framework/resources/templates/simple-load-deployment.yaml.tpl index cb6796d8a81..76e22e4636e 100644 --- a/test/framework/resources/templates/simple-load-deployment.yaml.tpl +++ b/test/framework/resources/templates/simple-load-deployment.yaml.tpl @@ -14,7 +14,7 @@ spec: app: load spec: containers: - - image: eu.gcr.io/gardener-project/3rd/alpine:3.16.1 + - image: registry.k8s.io/e2e-test-images/busybox:1.29-4 name: load command: ["sh", "-c"] args: diff --git a/test/framework/resources/templates/vpntunnel-copy.yaml.tpl b/test/framework/resources/templates/vpntunnel-copy.yaml.tpl index 0b9416f6fd1..87cdffa92ec 100644 --- a/test/framework/resources/templates/vpntunnel-copy.yaml.tpl +++ b/test/framework/resources/templates/vpntunnel-copy.yaml.tpl @@ -17,7 +17,7 @@ spec: app: {{ .AppLabel }} spec: initContainers: - - image: eu.gcr.io/gardener-project/3rd/alpine:3.16.1 + - image: registry.k8s.io/e2e-test-images/busybox:1.29-4 name: data-generator command: - dd @@ -28,7 +28,9 @@ spec: volumeMounts: - name: source-data mountPath: /data - - image: eu.gcr.io/gardener-project/3rd/alpine:3.16.1 + # TODO(ialidzhikov): There is a kubectl image (registry.k8s.io/kubectl), available in K8s 1.28+. + # In future, use the kubectl image, instead of downloading kubectl via init container. + - image: registry.k8s.io/e2e-test-images/busybox:1.29-4 name: install-kubectl command: - /bin/sh @@ -40,7 +42,7 @@ spec: - name: source-data mountPath: /data containers: - - image: eu.gcr.io/gardener-project/3rd/busybox:1.29.3 + - image: registry.k8s.io/e2e-test-images/busybox:1.29-4 name: source-container command: - sleep @@ -53,7 +55,7 @@ spec: mountPath: /data - name: kubecfg mountPath: /secret - - image: eu.gcr.io/gardener-project/3rd/busybox:1.29.3 + - image: registry.k8s.io/e2e-test-images/busybox:1.29-4 name: target-container command: - sleep diff --git a/test/framework/resources/templates/vpntunnel.yaml.tpl b/test/framework/resources/templates/vpntunnel.yaml.tpl index ecbe01587b1..9c0afdbe57b 100644 --- a/test/framework/resources/templates/vpntunnel.yaml.tpl +++ b/test/framework/resources/templates/vpntunnel.yaml.tpl @@ -17,22 +17,17 @@ spec: app: {{ .AppLabel }} spec: containers: - - image: eu.gcr.io/gardener-project/3rd/curlimages/curl:7.70.0 - name: net-curl + - name: pause + image: registry.k8s.io/e2e-test-images/agnhost:2.40 args: - - /bin/sh - - -c - - |- - while true; do - sleep 3600; - done + - pause - name: logger image: registry.k8s.io/e2e-test-images/agnhost:2.40 args: - - logs-generator - - --logtostderr - - --log-lines-total={{ .LogsCount }} - - --run-duration={{ .LogsDuration }} + - logs-generator + - --logtostderr + - --log-lines-total={{ .LogsCount }} + - --run-duration={{ .LogsDuration }} securityContext: fsGroup: 65532 runAsUser: 65532 diff --git a/test/testmachinery/shoots/applications/networking.go b/test/testmachinery/shoots/applications/networking.go index 0baf8a1182b..8cab581c764 100644 --- a/test/testmachinery/shoots/applications/networking.go +++ b/test/testmachinery/shoots/applications/networking.go @@ -87,7 +87,7 @@ var _ = ginkgo.Describe("Shoot network testing", func() { for _, from := range pods.Items { for _, to := range pods.Items { ginkgo.By(fmt.Sprintf("Testing %s to %s", from.GetName(), to.GetName())) - reader, err := podExecutor.Execute(ctx, from.Namespace, from.Name, "net-curl", fmt.Sprintf("curl -L %s:80 --fail -m 10", to.Status.PodIP)) + reader, err := podExecutor.Execute(ctx, from.Namespace, from.Name, "pause", fmt.Sprintf("curl -L %s:80 --fail -m 10", to.Status.PodIP)) if err != nil { allErrs = multierror.Append(allErrs, fmt.Errorf("%s to %s: %w", from.GetName(), to.GetName(), err)) continue diff --git a/test/testmachinery/shoots/vpntunnel/vpntunnel.go b/test/testmachinery/shoots/vpntunnel/vpntunnel.go index f7ddfe44856..2cc6f01afbd 100644 --- a/test/testmachinery/shoots/vpntunnel/vpntunnel.go +++ b/test/testmachinery/shoots/vpntunnel/vpntunnel.go @@ -103,7 +103,7 @@ var _ = ginkgo.Describe("Shoot vpn tunnel testing", func() { ctx, pod.Namespace, pod.Name, - "net-curl", + "pause", fmt.Sprintf("curl -k -v -XGET -H \"Accept: application/json, */*\" -H \"Authorization: Bearer %s\" \"%s/api/v1/namespaces/%s/pods/%s/log?container=logger\"", token, f.Shoot.Status.AdvertisedAddresses[0].URL, pod.Namespace, pod.Name), ) if apierrors.IsNotFound(err) {