feat(test): creating local cluster for e2e

.gitignore

@@ -37,10 +37,13 @@ dist/**
 # Apple files 
 .DS_Store
 
+# e2e local cluster kubeconfig files
+test/e2e/local-cluster/*.kubeconfig
+
 # act artifacts needed for testing workflows
 .secrets
 .env
 act_*.json
 
 # gen-crd-api-reference-docs artifacts
-*/gen-crd-api-reference-docs/tmp
+*/gen-crd-api-reference-docs/tmp

Makefile

@@ -71,13 +71,21 @@ e2e:
 	go test ./test/e2e/... -coverprofile cover.out -v
 
 .PHONY: e2e-local
-e2e-local: generate-manifests generate envtest ## Run e2e tests.
+e2e-local: generate-manifests generate envtest ## Run e2e tests against mock api.
 	unset USE_EXISTING_CLUSTER && KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" go test ./test/e2e/... -coverprofile cover.out -v
 
 .PHONY: e2e-remote
-e2e-remote: 
+e2e-remote: ## Run e2e tests against a remote Greenhouse cluster. E2E_KUBECONFIG must be set.
 	USE_EXISTING_CLUSTER=true go test ./test/e2e/... -coverprofile cover.out -v
 
+.PHONY: e2e-local-cluster
+e2e-local-cluster: e2e-local-cluster-create  ## Run e2e tests on a local KIND cluster.
+	USE_EXISTING_CLUSTER=true E2E_KUBECONFIG=$(shell pwd)/test/e2e/local-cluster/e2e.kubeconfig INTERNAL_KUBECONFIG=$(shell pwd)/test/e2e/local-cluster/e2e.internal.kubeconfig go test ./test/e2e/... -coverprofile cover.out -v
+
+.PHONY: e2e-local-cluster-create
+e2e-local-cluster-create:
+	cd test/e2e/local-cluster && go run . --dockerImagePlatform=$(PLATFORM)
+
 
 .PHONY: fmt
 fmt: goimports golint
@@ -173,4 +181,4 @@ golint: $(GOLINT)
 $(GOLINT): $(LOCALBIN)
 	GOBIN=$(LOCALBIN) go install github.com/golangci/golangci-lint/cmd/golangci-lint@$(GOLINT_VERSION)
 	GOBIN=$(LOCALBIN) go install github.com/nunnatsa/ginkgolinter/cmd/ginkgolinter@$(GINKGOLINTER_VERSION)
-	
+

go.mod

@@ -1,6 +1,6 @@
 module github.com/cloudoperators/greenhouse
 
-go 1.22.0
+go 1.22.3
 
 toolchain go1.22.5
 
@@ -46,6 +46,7 @@ require (
 	k8s.io/kubectl v0.30.2
 	k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0
 	sigs.k8s.io/controller-runtime v0.18.4
+	sigs.k8s.io/e2e-framework v0.4.0
 	sigs.k8s.io/yaml v1.4.0
 	tailscale.com v1.38.4
 )
@@ -75,6 +76,7 @@ require (
 	github.com/tidwall/match v1.1.1 // indirect
 	github.com/tidwall/pretty v1.2.1 // indirect
 	github.com/tidwall/sjson v1.2.5 // indirect
+	github.com/vladimirvivien/gexe v0.2.0 // indirect
 	github.com/x448/float16 v0.8.4 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect
 	go4.org/mem v0.0.0-20220726221520-4f986261bf13 // indirect

go.sum

@@ -52,6 +52,8 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
+github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
 github.com/bshuster-repo/logrus-logstash-hook v1.0.0 h1:e+C0SB5R1pu//O4MQ3f9cFuPGoOVeF2fE4Og9otCc70=
 github.com/bshuster-repo/logrus-logstash-hook v1.0.0/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk=
 github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd h1:rFt+Y/IK1aEZkEHchZRSq9OQbsSzIT/OrI8YFFmRIng=
@@ -477,6 +479,8 @@ github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4=
 github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
 github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY=
 github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28=
+github.com/vladimirvivien/gexe v0.2.0 h1:nbdAQ6vbZ+ZNsolCgSVb9Fno60kzSuvtzVh6Ytqi/xY=
+github.com/vladimirvivien/gexe v0.2.0/go.mod h1:LHQL00w/7gDUKIak24n801ABp8C+ni6eBht9vGVst8w=
 github.com/wI2L/jsondiff v0.6.0 h1:zrsH3FbfVa3JO9llxrcDy/XLkYPLgoMX6Mz3T2PP2AI=
 github.com/wI2L/jsondiff v0.6.0/go.mod h1:D6aQ5gKgPF9g17j+E9N7aasmU1O+XvfmWm1y8UMmNpw=
 github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
@@ -714,6 +718,8 @@ oras.land/oras-go v1.2.5 h1:XpYuAwAb0DfQsunIyMfeET92emK8km3W4yEzZvUbsTo=
 oras.land/oras-go v1.2.5/go.mod h1:PuAwRShRZCsZb7g8Ar3jKKQR/2A/qN+pkYxIOd/FAoo=
 sigs.k8s.io/controller-runtime v0.18.4 h1:87+guW1zhvuPLh1PHybKdYFLU0YJp4FhJRmiHvm5BZw=
 sigs.k8s.io/controller-runtime v0.18.4/go.mod h1:TVoGrfdpbA9VRFaRnKgk9P5/atA0pMwq+f+msb9M8Sg=
+sigs.k8s.io/e2e-framework v0.4.0 h1:4yYmFDNNoTnazqmZJXQ6dlQF1vrnDbutmxlyvBpC5rY=
+sigs.k8s.io/e2e-framework v0.4.0/go.mod h1:JilFQPF1OL1728ABhMlf9huse7h+uBJDXl9YeTs49A8=
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo=
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=
 sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3 h1:XX3Ajgzov2RKUdc5jW3t5jwY7Bo7dcRm+tFxT+NfgY0=

pkg/test/env.go

@@ -86,16 +86,26 @@ const (
 )
 
 var (
-	Cfg              *rest.Config
+	//Cfg is the rest.Config to access the cluster the tests are running against.
+	Cfg *rest.Config
+	//RestClientGetter is the clientutil.RestClientGetter to access the cluster the tests are running against.
 	RestClientGetter *clientutil.RestClientGetter
-	K8sClient        client.Client
-	K8sManager       ctrl.Manager
-	KubeConfig       []byte
-	testEnv          *envtest.Environment
-	Ctx              context.Context
-	cancel           context.CancelFunc
-	pollInterval     = 1 * time.Second
-	updateTimeout    = 30 * time.Second
+	//K8sClient is the client.Client to access the cluster the tests are running against.
+	K8sClient client.Client
+	//K8sManager is the ctrl.Manager the controllers are run by.
+	K8sManager ctrl.Manager
+	//KubeConfig is the raw kubeconfig to access the cluster the tests are running against.
+	KubeConfig []byte
+	//Ctx is the context to use for the tests.
+	Ctx context.Context
+	//IsUseExistingCluster is true if the tests are running against an existing cluster.
+	IsUseExistingCluster = useExistingGreenhouseCluster
+	testEnv              *envtest.Environment
+	cancel               context.CancelFunc
+	pollInterval         = 1 * time.Second
+	updateTimeout        = 30 * time.Second
+
+	persisted_kubeconfig = os.Getenv("KUBECONFIG")
 
 	// TestBeforeSuite configures the test suite.
 	TestBeforeSuite = func() {
@@ -108,9 +118,11 @@ var (
 		installWebhooks := len(allRegisterWebhookFuncs) > 0 && os.Getenv("TEST_INSTALL_WEBHOOKS") != "false"
 		if useExistingGreenhouseCluster {
 			// we are making use of https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/envtest#pkg-constants to prevent starting a new control plane
-			kubeconfig := os.Getenv("KUBECONFIG")
-			Expect(kubeconfig).NotTo(BeEmpty(), "the environment variable KUBECONFIG must be set to run the tests against a remote cluster")
-			fmt.Printf("using existing cluster with kubeconfig: %s\n", kubeconfig)
+			e2e_kubeconfig := os.Getenv("E2E_KUBECONFIG")
+			Expect(e2e_kubeconfig).NotTo(BeEmpty(), "the environment variable E2E_KUBECONFIG must be set to run the tests against a remote cluster")
+			// we overwrite the KUBECONFIG env var expected by envtest to make sure tests are not accidentally running against existing k8s context
+			os.Setenv("KUBECONFIG", e2e_kubeconfig)
+			fmt.Printf("Running tests against existing cluster with kubeconfig: %s\n", e2e_kubeconfig)
 			installCRDs = false
 			installWebhooks = false
 		} else {
@@ -197,6 +209,11 @@ var (
 		Eventually(func() error {
 			return testEnv.Stop()
 		}).Should(Succeed(), "there should be no error stopping the test environment")
+
+		if useExistingGreenhouseCluster {
+			// we reset the KUBECONFIG env var to its original value
+			os.Setenv("KUBECONFIG", persisted_kubeconfig)
+		}
 	}
 )
 
@@ -245,9 +262,7 @@ func StartControlPlane(port string, installCRDs, installWebhooks bool) (*rest.Co
 	var kubeConfig []byte
 	// we extract the kubeconfig from env var if we are using an existing cluster
 	if useExistingGreenhouseCluster {
-		kubeConfigLocation := os.Getenv("KUBECONFIG")
-		Expect(kubeConfigLocation).NotTo(BeEmpty(), "the environment variable KUBECONFIG must be set")
-		kubeConfig, err = os.ReadFile(kubeConfigLocation)
+		kubeConfig, err = KubeconfigFromEnvVar("KUBECONFIG")
 		Expect(err).NotTo(HaveOccurred())
 	} else {
 		// we add a user to the control plane to easily get a kubeconfig

pkg/test/util.go

@@ -6,6 +6,8 @@ package test
 import (
 	"context"
 	"encoding/json"
+	"fmt"
+	"os"
 	"time"
 
 	. "github.com/onsi/gomega"
@@ -104,3 +106,16 @@ func GreenhouseV1Alpha1Scheme() *runtime.Scheme {
 	utilruntime.Must(greenhousev1alpha1.AddToScheme(scheme))
 	return scheme
 }
+
+// KubeconfigFromEnvVar returns the kubeconfig []byte from the path specified in the environment variable
+func KubeconfigFromEnvVar(envVar string) ([]byte, error) {
+	kubeconfigPath := os.Getenv(envVar)
+	if kubeconfigPath == "" {
+		return nil, fmt.Errorf("kubeconfig path is empty")
+	}
+	kubeconfig, err := os.ReadFile(kubeconfigPath)
+	if err != nil {
+		return nil, err
+	}
+	return kubeconfig, nil
+}

test/e2e/README.md

@@ -12,9 +12,15 @@ make e2e
 
 will run the e2e test suite without making assumptions on the infrastructure to test against.
 
-Leveraging envtest, we will have basically two different test scenarios:
+Leveraging envtest, we will basically have three different test scenarios. The following env vars steer these:
 
-## Run everything local
+| Env Var                | Meaning                                                                                                                                                                                                                                                                                                                                                               |
+| ---------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --- |
+| `USE_EXISTING_CLUSTER` | If set to `true`, the e2e test suite will not spin up a local apiserver and etcd. Instead, it will expect an existing greenhouse installation on the cluster inferred from the `E2E_KUBECONFIG` environment variable.                                                                                                                                                 |
+| `E2E_KUBECONFIG`       | Required when `USE_EXISTING_CLUSTER` is `true`. Points to the remote cluster the e2e test suite is running against.                                                                                                                                                                                                                                                   |     |
+| `INTERNAL_KUBECONFIG`  | The path to the kubeconfig file for accessing the Greenhouse cluster itself from the running instance. This is used when `USE_EXISTING_CLUSTER` is set to `true`. KIND makes it necessary to set this separately to the `E2E_KUBECONFIG` as the internal api server adress differs to the external. Other setups may not use this. If unset `E2E_KUBECONFIG` is used. |
+
+## Run everything local a.k.a. `USE_EXISTING_CLUSTER = false` or unset
 
 Just running the tests via:
 
@@ -24,18 +30,18 @@ make e2e-local
 
 will spin up a local apiserver and etcd together with a local greenhouse controller. The e2e test suite will assert against this setup.
 
-## Run against an existing greenhouse installation
+## Run against an existing greenhouse installation a.k.a. `USE_EXISTING_CLUSTER = true`
 
 We can run our e2e test suite against a running greenhouse installation by exposing some env vars (also see [envtest package](https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/envtest#pkg-constants)):
 
 ```bash
 export USE_EXISTING_CLUSTER=true
 ```
 
-This will stop envtest from spinning up a local apiserver and etcd and expect an existing greenhouse installation on the cluster infered from the set `KUBECONFIG` environment variable:
+This will stop envtest from spinning up a local apiserver and etcd and expect an existing greenhouse installation on the cluster infered from the set `E2E_KUBECONFIG` environment variable:
 
 ```bash
-export KUBECONFIG=/path/to/greenhouse.kubeconfig
+export E2E_KUBECONFIG=/path/to/greenhouse.kubeconfig
 ```
 
 To run the e2e test suite against a remote installation:
@@ -44,9 +50,9 @@ To run the e2e test suite against a remote installation:
 make e2e-remote
 ```
 
-Test setup asserts `KUBECONFIG` is set and working and will fail otherwise.
+Test setup asserts `E2E_KUBECONFIG` is set and working and will fail otherwise.
 
-### Under construction:
+### Run against a local Greenhouse installation in KIND cluster a.k.a. `USE_EXISTING_CLUSTER = true` and `INTERNAL_KUBECONFIG` set
 
 We provide a convenience method to run the e2e test suite against a local KIND cluster with a greenhouse installation by running: