feat(test): e2e test setup & cluster onboard test

go.mod

@@ -43,7 +43,8 @@ require (
 	k8s.io/client-go v0.29.2
 	k8s.io/kubectl v0.29.2
 	k8s.io/utils v0.0.0-20230726121419-3b25d923346b
-	sigs.k8s.io/controller-runtime v0.15.0
+	sigs.k8s.io/controller-runtime v0.15.1
+	sigs.k8s.io/e2e-framework v0.3.0
 	sigs.k8s.io/yaml v1.4.0
 	tailscale.com v1.38.4
 )
@@ -70,6 +71,7 @@ require (
 	github.com/tidwall/match v1.1.1 // indirect
 	github.com/tidwall/pretty v1.2.0 // indirect
 	github.com/tidwall/sjson v1.2.5 // indirect
+	github.com/vladimirvivien/gexe v0.2.0 // indirect
 	github.com/x448/float16 v0.8.4 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 // indirect
 	go4.org/mem v0.0.0-20220726221520-4f986261bf13 // indirect

go.sum

@@ -484,6 +484,8 @@ github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
 github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
 github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY=
 github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28=
+github.com/vladimirvivien/gexe v0.2.0 h1:nbdAQ6vbZ+ZNsolCgSVb9Fno60kzSuvtzVh6Ytqi/xY=
+github.com/vladimirvivien/gexe v0.2.0/go.mod h1:LHQL00w/7gDUKIak24n801ABp8C+ni6eBht9vGVst8w=
 github.com/wI2L/jsondiff v0.5.0 h1:RRMTi/mH+R2aXcPe1VYyvGINJqQfC3R+KSEakuU1Ikw=
 github.com/wI2L/jsondiff v0.5.0/go.mod h1:qqG6hnK0Lsrz2BpIVCxWiK9ItsBCpIZQiv0izJjOZ9s=
 github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
@@ -731,6 +733,8 @@ oras.land/oras-go v1.2.4 h1:djpBY2/2Cs1PV87GSJlxv4voajVOMZxqqtq9AB8YNvY=
 oras.land/oras-go v1.2.4/go.mod h1:DYcGfb3YF1nKjcezfX2SNlDAeQFKSXmf+qrFmrh4324=
 sigs.k8s.io/controller-runtime v0.17.2 h1:FwHwD1CTUemg0pW2otk7/U5/i5m2ymzvOXdbeGOUvw0=
 sigs.k8s.io/controller-runtime v0.17.2/go.mod h1:+MngTvIQQQhfXtwfdGw/UOQ/aIaqsYywfCINOtwMO/s=
+sigs.k8s.io/e2e-framework v0.3.0 h1:eqQALBtPCth8+ulTs6lcPK7ytV5rZSSHJzQHZph4O7U=
+sigs.k8s.io/e2e-framework v0.3.0/go.mod h1:C+ef37/D90Dc7Xq1jQnNbJYscrUGpxrWog9bx2KIa+c=
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo=
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=
 sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3 h1:XX3Ajgzov2RKUdc5jW3t5jwY7Bo7dcRm+tFxT+NfgY0=

test/e2e/README.md

@@ -0,0 +1,33 @@
+## End-to-End Tests for Greenhouse
+
+- Tests are designed by following the guideline for [E2E Test Framework for Kubernetes](https://github.com/kubernetes-sigs/e2e-framework/blob/v0.3.0/docs/design/README.md)
+
+- `e2e_test.go` is the main entrypoint to run the tests that are defined the other files under the same folder.
+
+### Requirements
+- Docker
+- [kind](https://kind.sigs.k8s.io/)
+
+##3 Example Run
+```
+$ go test -v
+INFO[0031] Initializing the test environment variables  
+INFO[0031] Building docker image: greenhouse:e2e-latest 
+INFO[0045] Docker image built: greenhouse:e2e-latest    
+INFO[0052] Installing Greenhouse Controller Manager to the central cluster 
+INFO[0064] Installing Greenhouse Controller Manager to the central cluster: Done! 
+INFO[0064] Deploying organization resource to the central cluster 
+INFO[0067] Namespace is created automatically for organization 
+=== RUN   TestClusterBootstrap
+=== RUN   TestClusterBootstrap/Cluster_onboarding
+INFO[0067] Creating temporary file for central cluster kubeconfig (with external access) 
+INFO[0067] Exporting kubeconfig file for central cluster 
+INFO[0067] Creating kubeconfig secret for cluster       
+=== RUN   TestClusterBootstrap/Cluster_onboarding/Cluster_with_ready_status
+INFO[0070] Cluster status is ready                      
+--- PASS: TestClusterBootstrap (3.10s)
+    --- PASS: TestClusterBootstrap/Cluster_onboarding (3.10s)
+        --- PASS: TestClusterBootstrap/Cluster_onboarding/Cluster_with_ready_status (3.01s)
+PASS
+ok      github.com/cloudoperators/greenhouse/test/e2e   71.990s
+```

test/e2e/cluster_bootstrap_test.go

@@ -0,0 +1,110 @@
+// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Greenhouse contributors
+// SPDX-License-Identifier: Apache-2.0
+
+package e2e
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"os"
+	"os/exec"
+	"testing"
+
+	greenhouseapis "github.com/cloudoperators/greenhouse/pkg/apis"
+	greenhousev1alpha1 "github.com/cloudoperators/greenhouse/pkg/apis/greenhouse/v1alpha1"
+	"github.com/sirupsen/logrus"
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/e2e-framework/klient/wait"
+	"sigs.k8s.io/e2e-framework/pkg/envconf"
+	"sigs.k8s.io/e2e-framework/pkg/features"
+)
+
+var (
+	remotClusterKubeconfigFilePath = flag.String("remoteClusterKubeConfig", "", "path to the kubeconfig file for the remote cluster")
+)
+
+func TestClusterBootstrap(t *testing.T) {
+
+	feature := features.New("Cluster onboarding")
+	feature.Setup(
+		func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
+
+			if *remotClusterKubeconfigFilePath == "" {
+				logrus.Info("Creating temporary file for central cluster kubeconfig (with external access)")
+				f, err := os.CreateTemp("", "greenhouse-central")
+				if err != nil {
+					t.Fatal(err)
+				}
+				defer os.Remove(f.Name())
+
+				logrus.Info("Exporting kubeconfig file for central cluster")
+				args := []string{"export", "kubeconfig", "--name", centralClusterName, "--internal", "--kubeconfig", f.Name()}
+				cmd := exec.Command("kind", args...)
+				output, err := cmd.Output()
+				if err != nil {
+					logrus.Error("Error during kind export kubeconfig:", output)
+					t.Fatal(err)
+				}
+				*remotClusterKubeconfigFilePath = f.Name()
+			}
+			kubeconfigFileContents, err := os.ReadFile(*remotClusterKubeconfigFilePath)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			logrus.Info("Creating kubeconfig secret for cluster")
+			err = centralClusterK8sClient.Create(ctx,
+				&corev1.Secret{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      centralClusterName,
+						Namespace: greenhouseOrganizationName,
+					},
+					Type: greenhouseapis.SecretTypeKubeConfig,
+					Data: map[string][]byte{
+						greenhouseapis.KubeConfigKey: kubeconfigFileContents,
+					},
+				},
+			)
+			if err != nil {
+				t.Fatal(err)
+			}
+			return ctx
+		})
+
+	feature.Assess("Cluster with ready status", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
+		// wait for the cluster to be ready
+		if err := wait.For(
+			func(context.Context) (done bool, err error) {
+				cluster := &greenhousev1alpha1.Cluster{}
+				err = centralClusterK8sClient.Get(ctx, types.NamespacedName{Namespace: greenhouseOrganizationName, Name: centralClusterName}, cluster)
+				if apierrors.IsNotFound(err) {
+					return false, nil
+				} else if err != nil {
+					return false, err
+				}
+				clusterReady := cluster.Status.GetConditionByType(greenhousev1alpha1.ReadyCondition)
+				if clusterReady == nil {
+					return false, nil
+				}
+				if clusterReady.IsTrue() {
+					logrus.Info("Cluster status is ready")
+					return true, nil
+				}
+				return false, errors.New(clusterReady.Message)
+			},
+			wait.WithTimeout(TEST_TIMEOUT),
+			wait.WithInterval(TEST_RETRY_INTERVAL),
+		); err != nil {
+			t.Fatal(err)
+		}
+		return ctx
+	},
+	)
+
+	// submit the feature to be tested
+	testEnv.Test(t, feature.Feature())
+}