Refactor testrunner to use controller-runtime client

cmd/testrunner/cmd/runtemplate/run_template.go

@@ -18,6 +18,10 @@ import (
 	"fmt"
 	"os"
 
+	"github.com/gardener/gardener/pkg/client/kubernetes"
+	"github.com/gardener/test-infra/pkg/testmachinery"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
 	"github.com/gardener/test-infra/pkg/testrunner"
 	"github.com/gardener/test-infra/pkg/testrunner/result"
 	testrunnerTemplate "github.com/gardener/test-infra/pkg/testrunner/template"
@@ -87,12 +91,19 @@ var runCmd = &cobra.Command{
 			log.Debugf("Error loading .env file: %s", err.Error())
 		}
 
+		tmClient, err := kubernetes.NewClientFromFile(tmKubeconfigPath, nil, client.Options{
+			Scheme: testmachinery.TestMachineryScheme,
+		})
+		if err != nil {
+			log.Fatalf("Cannot build kubernetes client from %s: %s", tmKubeconfigPath, err.Error())
+		}
+
 		testrunName := fmt.Sprintf("%s-%s-", testrunNamePrefix, cloudprovider)
 		config := &testrunner.Config{
-			TmKubeconfigPath: tmKubeconfigPath,
-			Namespace:        namespace,
-			Timeout:          timeout,
-			Interval:         interval,
+			TmClient:  tmClient,
+			Namespace: namespace,
+			Timeout:   timeout,
+			Interval:  interval,
 		}
 
 		rsConfig := &result.Config{
@@ -129,13 +140,13 @@ var runCmd = &cobra.Command{
 			CloudProvider:     parameters.Cloudprovider,
 			KubernetesVersion: parameters.K8sVersion,
 		}
-		testruns, err := testrunnerTemplate.Render(config.TmKubeconfigPath, parameters, metadata)
+		testruns, err := testrunnerTemplate.Render(tmClient, parameters, metadata)
 		if err != nil {
 			log.Fatal(err)
 		}
 
 		finishedTestruns, err := testrunner.Run(config, testruns, testrunName)
-		failed, err := result.Collect(rsConfig, config.TmKubeconfigPath, config.Namespace, finishedTestruns, metadata)
+		failed, err := result.Collect(rsConfig, tmClient, config.Namespace, finishedTestruns, metadata)
 		if err != nil {
 			log.Fatal(err)
 		}

cmd/testrunner/cmd/runtestrun/run_testrun.go

@@ -17,6 +17,10 @@ package runtestrun
 import (
 	"fmt"
 
+	"github.com/gardener/gardener/pkg/client/kubernetes"
+	"github.com/gardener/test-infra/pkg/testmachinery"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
 	"github.com/gardener/test-infra/pkg/util"
 	"github.com/joho/godotenv"
 
@@ -64,11 +68,18 @@ var runTestrunCmd = &cobra.Command{
 			log.Debugf("Error loading .env file: %s", err.Error())
 		}
 
+		tmClient, err := kubernetes.NewClientFromFile(tmKubeconfigPath, nil, client.Options{
+			Scheme: testmachinery.TestMachineryScheme,
+		})
+		if err != nil {
+			log.Fatalf("Cannot build kubernetes client from %s: %s", tmKubeconfigPath, err.Error())
+		}
+
 		config := &testrunner.Config{
-			TmKubeconfigPath: tmKubeconfigPath,
-			Namespace:        namespace,
-			Timeout:          timeout,
-			Interval:         interval,
+			TmClient:  tmClient,
+			Namespace: namespace,
+			Timeout:   timeout,
+			Interval:  interval,
 		}
 
 		tr, err := util.ParseTestrunFromFile(testrunPath)

pkg/testmachinery/types.go

@@ -14,6 +14,14 @@
 
 package testmachinery
 
+import (
+	argoscheme "github.com/argoproj/argo/pkg/client/clientset/versioned/scheme"
+	tmscheme "github.com/gardener/test-infra/pkg/client/testmachinery/clientset/versioned/scheme"
+	"k8s.io/apimachinery/pkg/runtime"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	corescheme "k8s.io/client-go/kubernetes/scheme"
+)
+
 const (
 	// TM_KUBECONFIG_PATH is the path where kubeconfigs are mounted to tests.
 	TM_KUBECONFIG_PATH = "/tmp/env/kubeconfig"
@@ -82,3 +90,16 @@ type TechnicalUser struct {
 	Password  string `yaml:"password"`
 	AuthToken string `yaml:"authToken"`
 }
+
+// TestMachineryScheme is the scheme used in the testmachinery and testrunner.
+var TestMachineryScheme = runtime.NewScheme()
+
+func init() {
+	testmachinerySchemeBuilder := runtime.NewSchemeBuilder(
+		corescheme.AddToScheme,
+		tmscheme.AddToScheme,
+		argoscheme.AddToScheme,
+	)
+
+	utilruntime.Must(testmachinerySchemeBuilder.AddToScheme(TestMachineryScheme))
+}

pkg/testrunner/result/collect.go

@@ -17,17 +17,18 @@ package result
 import (
 	"fmt"
 
+	"github.com/gardener/gardener/pkg/client/kubernetes"
 	tmv1beta1 "github.com/gardener/test-infra/pkg/apis/testmachinery/v1beta1"
 	"github.com/gardener/test-infra/pkg/util"
 	log "github.com/sirupsen/logrus"
 )
 
 // Collect collects results of all testruns and write them to a file.
 // It returns wheter there are failed testruns or not.
-func Collect(config *Config, tmKubeconfigPath, namespace string, testruns []*tmv1beta1.Testrun, metadata *Metadata) (bool, error) {
+func Collect(config *Config, tmClient kubernetes.Interface, namespace string, testruns []*tmv1beta1.Testrun, metadata *Metadata) (bool, error) {
 	testrunsFailed := false
 	for _, tr := range testruns {
-		err := Output(config, tmKubeconfigPath, namespace, tr, metadata)
+		err := Output(config, tmClient, namespace, tr, metadata)
 		if err != nil {
 			return false, err
 		}
@@ -36,7 +37,7 @@ func Collect(config *Config, tmKubeconfigPath, namespace string, testruns []*tmv
 		if err != nil {
 			log.Errorf("Cannot persist file %s: %s", config.OutputFile, err.Error())
 		} else {
-			err := MarkTestrunsAsIngested(tmKubeconfigPath, testruns)
+			err := MarkTestrunsAsIngested(tmClient, testruns)
 			if err != nil {
 				log.Warn(err.Error())
 			}

pkg/testrunner/result/ingest.go

@@ -15,14 +15,14 @@
 package result
 
 import (
-	"fmt"
+	"context"
 	"os"
 	"os/exec"
 
+	"github.com/gardener/gardener/pkg/client/kubernetes"
+
 	tmv1beta1 "github.com/gardener/test-infra/pkg/apis/testmachinery/v1beta1"
-	tmclientset "github.com/gardener/test-infra/pkg/client/testmachinery/clientset/versioned"
 	log "github.com/sirupsen/logrus"
-	"k8s.io/client-go/tools/clientcmd"
 )
 
 const (
@@ -46,18 +46,12 @@ func IngestFile(file, esCfgName string) error {
 }
 
 // MarkTestrunsAsIngested sets the ingest status of testruns to true
-func MarkTestrunsAsIngested(tmKubeconfigPath string, testruns []*tmv1beta1.Testrun) error {
-	tmConfig, err := clientcmd.BuildConfigFromFlags("", tmKubeconfigPath)
-	if err != nil {
-		return fmt.Errorf("Cannot build kubernetes client from %s: %s", tmKubeconfigPath, err.Error())
-	}
-	tmClient, err := tmclientset.NewForConfig(tmConfig)
-	if err != nil {
-		return fmt.Errorf("Canno build testmachinery kubernetes client: %s", err.Error())
-	}
+func MarkTestrunsAsIngested(tmClient kubernetes.Interface, testruns []*tmv1beta1.Testrun) error {
+	ctx := context.Background()
+	defer ctx.Done()
 	for _, tr := range testruns {
 		tr.Status.Ingested = true
-		_, err := tmClient.Testmachinery().Testruns(tr.Namespace).UpdateStatus(tr)
+		err := tmClient.Client().Update(ctx, tr)
 		if err != nil {
 			log.Errorf("Cannot update status off testrun %s in namespace %s: %s", tr.Name, tr.Namespace, err.Error())
 			continue

pkg/testrunner/result/output.go

@@ -18,6 +18,7 @@ import (
 	"archive/tar"
 	"bytes"
 	"compress/gzip"
+	"context"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -33,10 +34,11 @@ import (
 	tmv1beta1 "github.com/gardener/test-infra/pkg/apis/testmachinery/v1beta1"
 	minio "github.com/minio/minio-go"
 	log "github.com/sirupsen/logrus"
+	corev1 "k8s.io/api/core/v1"
 )
 
 // Output takes a completed testrun status and writes the results to elastic search bulk json file.
-func Output(config *Config, tmKubeconfigPath, namespace string, tr *tmv1beta1.Testrun, metadata *Metadata) error {
+func Output(config *Config, tmClient kubernetes.Interface, namespace string, tr *tmv1beta1.Testrun, metadata *Metadata) error {
 	if config.OutputFile == "" {
 		return nil
 	}
@@ -58,7 +60,7 @@ func Output(config *Config, tmKubeconfigPath, namespace string, tr *tmv1beta1.Te
 		return err
 	}
 
-	osConfig, err := getOSConfig(tmKubeconfigPath, namespace, config.S3Endpoint, config.S3SSL)
+	osConfig, err := getOSConfig(tmClient, namespace, config.S3Endpoint, config.S3SSL)
 	if err != nil {
 		log.Warnf("Cannot get exported Test results of steps: %s", err.Error())
 	} else {
@@ -225,16 +227,17 @@ func writeToFile(fielPath string, data []byte) error {
 	return nil
 }
 
-func getOSConfig(tmKubeconfigPath, namespace, minioEndpoint string, ssl bool) (*testmachinery.ObjectStoreConfig, error) {
-	clusterClient, err := kubernetes.NewClientFromFile(tmKubeconfigPath, nil, client.Options{})
-	if err != nil {
-		return nil, fmt.Errorf("Cannot create client for %s: %s", tmKubeconfigPath, err.Error())
-	}
-	minioConfig, err := clusterClient.GetConfigMap(namespace, "tm-config")
+func getOSConfig(tmClient kubernetes.Interface, namespace, minioEndpoint string, ssl bool) (*testmachinery.ObjectStoreConfig, error) {
+	ctx := context.Background()
+	defer ctx.Done()
+
+	minioConfig := &corev1.ConfigMap{}
+	err := tmClient.Client().Get(ctx, client.ObjectKey{Namespace: namespace, Name: "tm-config"}, minioConfig)
 	if err != nil {
 		return nil, fmt.Errorf("Cannot get ConfigMap 'tm-config': %s", err.Error())
 	}
-	minioSecrets, err := clusterClient.GetSecret(namespace, minioConfig.Data["objectstore.secretName"])
+	minioSecrets := &corev1.Secret{}
+	err = tmClient.Client().Get(ctx, client.ObjectKey{Namespace: namespace, Name: minioConfig.Data["objectstore.secretName"]}, minioSecrets)
 	if err != nil {
 		return nil, fmt.Errorf("Cannot get Secret '%s': %s", minioConfig.Data["objectstore.secretName"], err.Error())
 	}

pkg/testrunner/run.go

@@ -15,23 +15,28 @@
 package testrunner
 
 import (
+	"context"
 	"fmt"
 	"time"
 
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	"github.com/gardener/gardener/pkg/client/kubernetes"
+
 	tmv1beta1 "github.com/gardener/test-infra/pkg/apis/testmachinery/v1beta1"
-	tmclientset "github.com/gardener/test-infra/pkg/client/testmachinery/clientset/versioned"
 	"github.com/gardener/test-infra/pkg/util"
 	log "github.com/sirupsen/logrus"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
 )
 
-func runTestrun(tmClient *tmclientset.Clientset, tr *tmv1beta1.Testrun, namespace, name string) (*tmv1beta1.Testrun, error) {
+func runTestrun(tmClient kubernetes.Interface, tr *tmv1beta1.Testrun, namespace, name string) (*tmv1beta1.Testrun, error) {
+	ctx := context.Background()
+	defer ctx.Done()
 	// TODO: Remove legacy name attribute. Instead enforce usage of generateName.
 	tr.Name = ""
 	tr.GenerateName = name
 	tr.Namespace = namespace
-	tr, err := tmClient.Testmachinery().Testruns(namespace).Create(tr)
+	err := tmClient.Client().Create(ctx, tr)
 	if err != nil {
 		return nil, fmt.Errorf("Cannot create testrun: %s", err.Error())
 	}
@@ -41,7 +46,7 @@ func runTestrun(tmClient *tmclientset.Clientset, tr *tmv1beta1.Testrun, namespac
 	interval := time.Duration(pollIntervalSeconds) * time.Second
 	timeout := time.Duration(maxWaitTimeSeconds) * time.Second
 	err = wait.PollImmediate(interval, timeout, func() (bool, error) {
-		tr, err = tmClient.Testmachinery().Testruns(namespace).Get(tr.Name, metav1.GetOptions{})
+		err := tmClient.Client().Get(ctx, client.ObjectKey{Namespace: namespace, Name: tr.Name}, tr)
 		if err != nil {
 			log.Errorf("Cannot get testrun: %s", err.Error())
 		}

pkg/testrunner/template/render.go

@@ -8,19 +8,14 @@ import (
 	"github.com/gardener/gardener/pkg/client/kubernetes"
 	"github.com/gardener/test-infra/pkg/util"
 	log "github.com/sirupsen/logrus"
-	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 // RenderChart renders the provided helm chart with testruns, adds the testrun parameters and returns the templated files.
-func RenderChart(tmKubeconfigPath string, parameters *TestrunParameters, versions []string) ([]string, error) {
+func RenderChart(tmClient kubernetes.Interface, parameters *TestrunParameters, versions []string) ([]string, error) {
 	log.Debugf("Parameters: %+v", util.PrettyPrintStruct(parameters))
 	log.Debugf("Render chart from %s", parameters.TestrunChartPath)
 
-	tmClusterClient, err := kubernetes.NewClientFromFile(tmKubeconfigPath, nil, client.Options{})
-	if err != nil {
-		return nil, fmt.Errorf("couldn't create k8s client from kubeconfig filepath %s: %v", tmKubeconfigPath, err)
-	}
-	tmChartRenderer, err := chartrenderer.New(tmClusterClient)
+	tmChartRenderer, err := chartrenderer.New(tmClient)
 	if err != nil {
 		return nil, fmt.Errorf("Cannot create chartrenderer for gardener: %s", err.Error())
 	}

pkg/testrunner/template/template.go

@@ -18,6 +18,7 @@ import (
 	"fmt"
 	"io/ioutil"
 
+	"github.com/gardener/gardener/pkg/client/kubernetes"
 	tmv1beta1 "github.com/gardener/test-infra/pkg/apis/testmachinery/v1beta1"
 	"github.com/gardener/test-infra/pkg/testrunner/componentdescriptor"
 	"github.com/gardener/test-infra/pkg/testrunner/result"
@@ -26,7 +27,7 @@ import (
 )
 
 // Render renders a helm chart with containing testruns, adds the provided parameters and values, and returns the parsed and modified testruns.
-func Render(tmKubeconfigPath string, parameters *TestrunParameters, metadata *result.Metadata) ([]*tmv1beta1.Testrun, error) {
+func Render(tmClient kubernetes.Interface, parameters *TestrunParameters, metadata *result.Metadata) ([]*tmv1beta1.Testrun, error) {
 	versions, err := getK8sVersions(parameters)
 	if err != nil {
 		log.Fatal(err.Error())
@@ -35,17 +36,16 @@ func Render(tmKubeconfigPath string, parameters *TestrunParameters, metadata *re
 	if parameters.ComponentDescriptorPath != "" {
 		data, err := ioutil.ReadFile(parameters.ComponentDescriptorPath)
 		if err != nil {
-			log.Warnf("Cannot read component descriptor file %s: %s", parameters.ComponentDescriptorPath, err.Error())
+			return nil, fmt.Errorf("Cannot read component descriptor file %s: %s", parameters.ComponentDescriptorPath, err.Error())
 		}
 		components, err := componentdescriptor.GetComponents(data)
 		if err != nil {
-			log.Warnf("Cannot decode and parse BOM %s", err.Error())
-		} else {
-			metadata.BOM = components
+			return nil, fmt.Errorf("Cannot decode and parse BOM %s", err.Error())
 		}
+		metadata.BOM = components
 	}
 
-	files, err := RenderChart(tmKubeconfigPath, parameters, versions)
+	files, err := RenderChart(tmClient, parameters, versions)
 	if err != nil {
 		return nil, err
 	}
@@ -58,7 +58,7 @@ func Render(tmKubeconfigPath string, parameters *TestrunParameters, metadata *re
 			log.Warnf("Cannot parse rendered file: %s", err.Error())
 		}
 
-		// Add current dependency repositories to the testrun location.
+		// Add all repositories defined in the component descriptor to the testrun locations.
 		// This gives us all dependent repositories as well as there deployed version.
 		addBOMLocationsToTestrun(&tr, metadata.BOM)
 

pkg/testrunner/testrunner.go

@@ -16,15 +16,13 @@ package testrunner
 
 import (
 	"errors"
-	"fmt"
 	"sync"
 
+	"github.com/gardener/gardener/pkg/client/kubernetes"
+
 	tmv1beta1 "github.com/gardener/test-infra/pkg/apis/testmachinery/v1beta1"
 	"github.com/gardener/test-infra/pkg/util"
 
-	"k8s.io/client-go/tools/clientcmd"
-
-	tmclientset "github.com/gardener/test-infra/pkg/client/testmachinery/clientset/versioned"
 	log "github.com/sirupsen/logrus"
 )
 
@@ -39,16 +37,7 @@ func Run(config *Config, testruns []*tmv1beta1.Testrun, testrunNamePrefix string
 	maxWaitTimeSeconds = config.Timeout
 	pollIntervalSeconds = config.Interval
 
-	tmConfig, err := clientcmd.BuildConfigFromFlags("", config.TmKubeconfigPath)
-	if err != nil {
-		return nil, fmt.Errorf("Cannot build kubernetes client from %s: %s", config.TmKubeconfigPath, err.Error())
-	}
-	tmClient, err := tmclientset.NewForConfig(tmConfig)
-	if err != nil {
-		return nil, err
-	}
-
-	finishedTestruns := runChart(tmClient, testruns, config.Namespace, testrunNamePrefix)
+	finishedTestruns := runChart(config.TmClient, testruns, config.Namespace, testrunNamePrefix)
 
 	if len(finishedTestruns) == 0 {
 		return nil, errors.New("No testruns finished")
@@ -59,7 +48,7 @@ func Run(config *Config, testruns []*tmv1beta1.Testrun, testrunNamePrefix string
 
 // runChart tries to parse each rendered file of a chart into a testrun.
 // If a filecontent is a testrun then it is deployed into the testmachinery.
-func runChart(tmClient *tmclientset.Clientset, testruns []*tmv1beta1.Testrun, namespace, testrunNamePrefix string) []*tmv1beta1.Testrun {
+func runChart(tmClient kubernetes.Interface, testruns []*tmv1beta1.Testrun, namespace, testrunNamePrefix string) []*tmv1beta1.Testrun {
 	var wg sync.WaitGroup
 	mutex := &sync.Mutex{}
 	finishedTestruns := []*tmv1beta1.Testrun{}