feat(test): creating local cluster for e2e

Makefile

@@ -71,13 +71,21 @@ e2e:
 	go test ./test/e2e/... -coverprofile cover.out -v
 
 .PHONY: e2e-local
-e2e-local: generate-manifests generate envtest ## Run e2e tests.
+e2e-local: generate-manifests generate envtest ## Run e2e tests against mock api.
 	unset USE_EXISTING_CLUSTER && KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" go test ./test/e2e/... -coverprofile cover.out -v
 
 .PHONY: e2e-remote
-e2e-remote: 
+e2e-remote: ## Run e2e tests against a remote Greenhouse cluster.
 	USE_EXISTING_CLUSTER=true go test ./test/e2e/... -coverprofile cover.out -v
 
+.PHONY: e2e-local-cluster
+e2e-local-cluster: e2e-local-cluster-create  ## Run e2e tests on a local KIND cluster.
+	USE_EXISTING_CLUSTER=true KUBECONFIG=$(shell pwd)/test/e2e/local-cluster/e2e.kubeconfig INTERNAL_KUBECONFIG=$(shell pwd)/test/e2e/local-cluster/e2e.internal.kubeconfig go test ./test/e2e/... -coverprofile cover.out -v
+
+.PHONY: e2e-local-cluster-create
+e2e-local-cluster-create:
+	cd test/e2e/local-cluster && go run .
+
 
 .PHONY: fmt
 fmt: goimports golint
@@ -174,7 +182,3 @@ $(GOLINT): $(LOCALBIN)
 	GOBIN=$(LOCALBIN) go install github.com/golangci/golangci-lint/cmd/golangci-lint@$(GOLINT_VERSION)
 	GOBIN=$(LOCALBIN) go install github.com/nunnatsa/ginkgolinter/cmd/ginkgolinter@$(GINKGOLINTER_VERSION)
 
-## E2E tests
-.PHONY: e2e-local-cluster-create
-e2e-local-cluster-create:
-	cd test/e2e/local-cluster && go run .

go.mod

@@ -2,7 +2,7 @@ module github.com/cloudoperators/greenhouse
 
 go 1.22.3
 
-toolchain go1.22.4
+toolchain go1.22.5
 
 replace (
 	// DEX import matches version v2.39.1.
@@ -37,15 +37,15 @@ require (
 	golang.org/x/text v0.16.0
 	gopkg.in/yaml.v3 v3.0.1
 	gotest.tools/v3 v3.5.1
-	helm.sh/helm/v3 v3.14.4
-	k8s.io/api v0.30.1
-	k8s.io/apiextensions-apiserver v0.29.5
-	k8s.io/apimachinery v0.30.1
-	k8s.io/cli-runtime v0.29.5
-	k8s.io/client-go v0.30.1
-	k8s.io/kubectl v0.29.5
+	helm.sh/helm/v3 v3.15.2
+	k8s.io/api v0.30.2
+	k8s.io/apiextensions-apiserver v0.30.2
+	k8s.io/apimachinery v0.30.2
+	k8s.io/cli-runtime v0.30.2
+	k8s.io/client-go v0.30.2
+	k8s.io/kubectl v0.30.2
 	k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0
-	sigs.k8s.io/controller-runtime v0.18.2
+	sigs.k8s.io/controller-runtime v0.18.4
 	sigs.k8s.io/e2e-framework v0.4.0
 	sigs.k8s.io/yaml v1.4.0
 	tailscale.com v1.38.4
@@ -71,10 +71,7 @@ require (
 	github.com/mdlayher/netlink v1.7.1 // indirect
 	github.com/mdlayher/socket v0.4.0 // indirect
 	github.com/mitchellh/go-ps v1.0.0 // indirect
-	github.com/moby/patternmatcher v0.6.0 // indirect
-	github.com/moby/sys/sequential v0.5.0 // indirect
 	github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
-	github.com/opencontainers/runc v1.1.5 // indirect
 	github.com/tidwall/gjson v1.17.1 // indirect
 	github.com/tidwall/match v1.1.1 // indirect
 	github.com/tidwall/pretty v1.2.1 // indirect
@@ -112,9 +109,9 @@ require (
 	github.com/cyphar/filepath-securejoin v0.2.4 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dexidp/dex/api/v2 v2.1.1-0.20240409110506-3705207f0190 // indirect
-	github.com/docker/cli v24.0.6+incompatible // indirect
-	github.com/docker/distribution v2.8.2+incompatible // indirect
-	github.com/docker/docker v24.0.9+incompatible
+	github.com/docker/cli v25.0.1+incompatible // indirect
+	github.com/docker/distribution v2.8.3+incompatible // indirect
+	github.com/docker/docker v25.0.5+incompatible // indirect
 	github.com/docker/docker-credential-helpers v0.7.0 // indirect
 	github.com/docker/go-connections v0.5.0 // indirect
 	github.com/docker/go-metrics v0.0.1 // indirect
@@ -220,11 +217,11 @@ require (
 	google.golang.org/protobuf v1.34.2
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	k8s.io/apiserver v0.29.5 // indirect
-	k8s.io/component-base v0.30.1 // indirect
+	k8s.io/apiserver v0.30.2 // indirect
+	k8s.io/component-base v0.30.2 // indirect
 	k8s.io/klog/v2 v2.120.1
 	k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect
-	oras.land/oras-go v1.2.4 // indirect
+	oras.land/oras-go v1.2.5 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
 	sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3 // indirect
 	sigs.k8s.io/kustomize/kyaml v0.14.3-0.20230601165947-6ce0bf390ce3 // indirect

pkg/test/env.go

@@ -86,16 +86,24 @@ const (
 )
 
 var (
-	Cfg              *rest.Config
+	//Cfg is the rest.Config to access the cluster the tests are running against.
+	Cfg *rest.Config
+	//RestClientGetter is the clientutil.RestClientGetter to access the cluster the tests are running against.
 	RestClientGetter *clientutil.RestClientGetter
-	K8sClient        client.Client
-	K8sManager       ctrl.Manager
-	KubeConfig       []byte
-	testEnv          *envtest.Environment
-	Ctx              context.Context
-	cancel           context.CancelFunc
-	pollInterval     = 1 * time.Second
-	updateTimeout    = 30 * time.Second
+	//K8sClient is the client.Client to access the cluster the tests are running against.
+	K8sClient client.Client
+	//K8sManager is the ctrl.Manager the controllers are run by.
+	K8sManager ctrl.Manager
+	//KubeConfig is the raw kubeconfig to access the cluster the tests are running against.
+	KubeConfig []byte
+	//Ctx is the context to use for the tests.
+	Ctx context.Context
+	//IsUseExistingCluster is true if the tests are running against an existing cluster.
+	IsUseExistingCluster = useExistingGreenhouseCluster
+	testEnv              *envtest.Environment
+	cancel               context.CancelFunc
+	pollInterval         = 1 * time.Second
+	updateTimeout        = 30 * time.Second
 
 	// TestBeforeSuite configures the test suite.
 	TestBeforeSuite = func() {
@@ -245,9 +253,7 @@ func StartControlPlane(port string, installCRDs, installWebhooks bool) (*rest.Co
 	var kubeConfig []byte
 	// we extract the kubeconfig from env var if we are using an existing cluster
 	if useExistingGreenhouseCluster {
-		kubeConfigLocation := os.Getenv("KUBECONFIG")
-		Expect(kubeConfigLocation).NotTo(BeEmpty(), "the environment variable KUBECONFIG must be set")
-		kubeConfig, err = os.ReadFile(kubeConfigLocation)
+		kubeConfig, err = KubeconfigFromEnvVar("KUBECONFIG")
 		Expect(err).NotTo(HaveOccurred())
 	} else {
 		// we add a user to the control plane to easily get a kubeconfig

pkg/test/util.go

@@ -6,6 +6,8 @@ package test
 import (
 	"context"
 	"encoding/json"
+	"fmt"
+	"os"
 	"time"
 
 	. "github.com/onsi/gomega"
@@ -104,3 +106,16 @@ func GreenhouseV1Alpha1Scheme() *runtime.Scheme {
 	utilruntime.Must(greenhousev1alpha1.AddToScheme(scheme))
 	return scheme
 }
+
+// KubeconfigFromEnvVar returns the kubeconfig []byte from the path specified in the environment variable
+func KubeconfigFromEnvVar(envVar string) ([]byte, error) {
+	kubeconfigPath := os.Getenv(envVar)
+	if kubeconfigPath == "" {
+		return nil, fmt.Errorf("kubeconfig path is empty")
+	}
+	kubeconfig, err := os.ReadFile(kubeconfigPath)
+	if err != nil {
+		return nil, err
+	}
+	return kubeconfig, nil
+}

test/e2e/README.md

@@ -12,9 +12,14 @@ make e2e
 
 will run the e2e test suite without making assumptions on the infrastructure to test against.
 
-Leveraging envtest, we will have basically two different test scenarios:
+Leveraging envtest, we will basically have three different test scenarios. The following env vars steer these:
 
-## Run everything local
+| Env Var                | Meaning                                                                                                                                                                                                                                                                                                                                                       |
+| ---------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `USE_EXISTING_CLUSTER` | If set to `true`, the e2e test suite will not spin up a local apiserver and etcd. Instead, it will expect an existing greenhouse installation on the cluster inferred from the `KUBECONFIG` environment variable.                                                                                                                                             |
+| `INTERNAL_KUBECONFIG`  | The path to the kubeconfig file for accessing the Greenhouse cluster itself from the running instance. This is used when `USE_EXISTING_CLUSTER` is set to `true`. KIND makes it necessary to set this separately to the `KUBECONFIG` as the internal api server adress differs to the external. Other setups may not use this. If unset `KUBECONFIG` is used. |
+
+## Run everything local a.k.a. `USE_EXISTING_CLUSTER = false` or unset
 
 Just running the tests via:
 
@@ -24,7 +29,7 @@ make e2e-local
 
 will spin up a local apiserver and etcd together with a local greenhouse controller. The e2e test suite will assert against this setup.
 
-## Run against an existing greenhouse installation
+## Run against an existing greenhouse installation a.k.a. `USE_EXISTING_CLUSTER = true`
 
 We can run our e2e test suite against a running greenhouse installation by exposing some env vars (also see [envtest package](https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/envtest#pkg-constants)):
 
@@ -46,7 +51,7 @@ make e2e-remote
 
 Test setup asserts `KUBECONFIG` is set and working and will fail otherwise.
 
-### Under construction:
+### Run against a local Greenhouse installation in KIND cluster a.k.a. `USE_EXISTING_CLUSTER = true` and `INTERNAL_KUBECONFIG` set
 
 We provide a convenience method to run the e2e test suite against a local KIND cluster with a greenhouse installation by running: