diff --git a/cmd/testrunner/cmd/runtemplate/run_template.go b/cmd/testrunner/cmd/runtemplate/run_template.go index 77b8a3bb35..5a2f40d84d 100644 --- a/cmd/testrunner/cmd/runtemplate/run_template.go +++ b/cmd/testrunner/cmd/runtemplate/run_template.go @@ -18,6 +18,10 @@ import ( "fmt" "os" + "github.com/gardener/gardener/pkg/client/kubernetes" + "github.com/gardener/test-infra/pkg/testmachinery" + "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/gardener/test-infra/pkg/testrunner" "github.com/gardener/test-infra/pkg/testrunner/result" testrunnerTemplate "github.com/gardener/test-infra/pkg/testrunner/template" @@ -87,12 +91,19 @@ var runCmd = &cobra.Command{ log.Debugf("Error loading .env file: %s", err.Error()) } + tmClient, err := kubernetes.NewClientFromFile(tmKubeconfigPath, nil, client.Options{ + Scheme: testmachinery.TestMachineryScheme, + }) + if err != nil { + log.Fatalf("Cannot build kubernetes client from %s: %s", tmKubeconfigPath, err.Error()) + } + testrunName := fmt.Sprintf("%s-%s-", testrunNamePrefix, cloudprovider) config := &testrunner.Config{ - TmKubeconfigPath: tmKubeconfigPath, - Namespace: namespace, - Timeout: timeout, - Interval: interval, + TmClient: tmClient, + Namespace: namespace, + Timeout: timeout, + Interval: interval, } rsConfig := &result.Config{ @@ -129,13 +140,13 @@ var runCmd = &cobra.Command{ CloudProvider: parameters.Cloudprovider, KubernetesVersion: parameters.K8sVersion, } - testruns, err := testrunnerTemplate.Render(config.TmKubeconfigPath, parameters, metadata) + testruns, err := testrunnerTemplate.Render(tmClient, parameters, metadata) if err != nil { log.Fatal(err) } finishedTestruns, err := testrunner.Run(config, testruns, testrunName) - failed, err := result.Collect(rsConfig, config.TmKubeconfigPath, config.Namespace, finishedTestruns, metadata) + failed, err := result.Collect(rsConfig, tmClient, config.Namespace, finishedTestruns, metadata) if err != nil { log.Fatal(err) } diff --git a/cmd/testrunner/cmd/runtestrun/run_testrun.go b/cmd/testrunner/cmd/runtestrun/run_testrun.go index 2cd6459af9..b9cb115c55 100644 --- a/cmd/testrunner/cmd/runtestrun/run_testrun.go +++ b/cmd/testrunner/cmd/runtestrun/run_testrun.go @@ -17,6 +17,10 @@ package runtestrun import ( "fmt" + "github.com/gardener/gardener/pkg/client/kubernetes" + "github.com/gardener/test-infra/pkg/testmachinery" + "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/gardener/test-infra/pkg/util" "github.com/joho/godotenv" @@ -64,11 +68,18 @@ var runTestrunCmd = &cobra.Command{ log.Debugf("Error loading .env file: %s", err.Error()) } + tmClient, err := kubernetes.NewClientFromFile(tmKubeconfigPath, nil, client.Options{ + Scheme: testmachinery.TestMachineryScheme, + }) + if err != nil { + log.Fatalf("Cannot build kubernetes client from %s: %s", tmKubeconfigPath, err.Error()) + } + config := &testrunner.Config{ - TmKubeconfigPath: tmKubeconfigPath, - Namespace: namespace, - Timeout: timeout, - Interval: interval, + TmClient: tmClient, + Namespace: namespace, + Timeout: timeout, + Interval: interval, } tr, err := util.ParseTestrunFromFile(testrunPath) diff --git a/pkg/testmachinery/types.go b/pkg/testmachinery/types.go index a3d796fecf..6a82173c35 100644 --- a/pkg/testmachinery/types.go +++ b/pkg/testmachinery/types.go @@ -14,6 +14,14 @@ package testmachinery +import ( + argoscheme "github.com/argoproj/argo/pkg/client/clientset/versioned/scheme" + tmscheme "github.com/gardener/test-infra/pkg/client/testmachinery/clientset/versioned/scheme" + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + corescheme "k8s.io/client-go/kubernetes/scheme" +) + const ( // TM_KUBECONFIG_PATH is the path where kubeconfigs are mounted to tests. TM_KUBECONFIG_PATH = "/tmp/env/kubeconfig" @@ -82,3 +90,16 @@ type TechnicalUser struct { Password string `yaml:"password"` AuthToken string `yaml:"authToken"` } + +// TestMachineryScheme is the scheme used in the testmachinery and testrunner. +var TestMachineryScheme = runtime.NewScheme() + +func init() { + testmachinerySchemeBuilder := runtime.NewSchemeBuilder( + corescheme.AddToScheme, + tmscheme.AddToScheme, + argoscheme.AddToScheme, + ) + + utilruntime.Must(testmachinerySchemeBuilder.AddToScheme(TestMachineryScheme)) +} diff --git a/pkg/testrunner/result/collect.go b/pkg/testrunner/result/collect.go index 603976ef80..b747959b47 100644 --- a/pkg/testrunner/result/collect.go +++ b/pkg/testrunner/result/collect.go @@ -17,6 +17,7 @@ package result import ( "fmt" + "github.com/gardener/gardener/pkg/client/kubernetes" tmv1beta1 "github.com/gardener/test-infra/pkg/apis/testmachinery/v1beta1" "github.com/gardener/test-infra/pkg/util" log "github.com/sirupsen/logrus" @@ -24,10 +25,10 @@ import ( // Collect collects results of all testruns and write them to a file. // It returns wheter there are failed testruns or not. -func Collect(config *Config, tmKubeconfigPath, namespace string, testruns []*tmv1beta1.Testrun, metadata *Metadata) (bool, error) { +func Collect(config *Config, tmClient kubernetes.Interface, namespace string, testruns []*tmv1beta1.Testrun, metadata *Metadata) (bool, error) { testrunsFailed := false for _, tr := range testruns { - err := Output(config, tmKubeconfigPath, namespace, tr, metadata) + err := Output(config, tmClient, namespace, tr, metadata) if err != nil { return false, err } @@ -36,7 +37,7 @@ func Collect(config *Config, tmKubeconfigPath, namespace string, testruns []*tmv if err != nil { log.Errorf("Cannot persist file %s: %s", config.OutputFile, err.Error()) } else { - err := MarkTestrunsAsIngested(tmKubeconfigPath, testruns) + err := MarkTestrunsAsIngested(tmClient, testruns) if err != nil { log.Warn(err.Error()) } diff --git a/pkg/testrunner/result/ingest.go b/pkg/testrunner/result/ingest.go index 13344d8c6c..1522c3ca3a 100644 --- a/pkg/testrunner/result/ingest.go +++ b/pkg/testrunner/result/ingest.go @@ -15,14 +15,14 @@ package result import ( - "fmt" + "context" "os" "os/exec" + "github.com/gardener/gardener/pkg/client/kubernetes" + tmv1beta1 "github.com/gardener/test-infra/pkg/apis/testmachinery/v1beta1" - tmclientset "github.com/gardener/test-infra/pkg/client/testmachinery/clientset/versioned" log "github.com/sirupsen/logrus" - "k8s.io/client-go/tools/clientcmd" ) const ( @@ -46,18 +46,12 @@ func IngestFile(file, esCfgName string) error { } // MarkTestrunsAsIngested sets the ingest status of testruns to true -func MarkTestrunsAsIngested(tmKubeconfigPath string, testruns []*tmv1beta1.Testrun) error { - tmConfig, err := clientcmd.BuildConfigFromFlags("", tmKubeconfigPath) - if err != nil { - return fmt.Errorf("Cannot build kubernetes client from %s: %s", tmKubeconfigPath, err.Error()) - } - tmClient, err := tmclientset.NewForConfig(tmConfig) - if err != nil { - return fmt.Errorf("Canno build testmachinery kubernetes client: %s", err.Error()) - } +func MarkTestrunsAsIngested(tmClient kubernetes.Interface, testruns []*tmv1beta1.Testrun) error { + ctx := context.Background() + defer ctx.Done() for _, tr := range testruns { tr.Status.Ingested = true - _, err := tmClient.Testmachinery().Testruns(tr.Namespace).UpdateStatus(tr) + err := tmClient.Client().Update(ctx, tr) if err != nil { log.Errorf("Cannot update status off testrun %s in namespace %s: %s", tr.Name, tr.Namespace, err.Error()) continue diff --git a/pkg/testrunner/result/output.go b/pkg/testrunner/result/output.go index 97f2ddb673..8206bb22cc 100644 --- a/pkg/testrunner/result/output.go +++ b/pkg/testrunner/result/output.go @@ -18,6 +18,7 @@ import ( "archive/tar" "bytes" "compress/gzip" + "context" "encoding/json" "fmt" "io" @@ -33,10 +34,11 @@ import ( tmv1beta1 "github.com/gardener/test-infra/pkg/apis/testmachinery/v1beta1" minio "github.com/minio/minio-go" log "github.com/sirupsen/logrus" + corev1 "k8s.io/api/core/v1" ) // Output takes a completed testrun status and writes the results to elastic search bulk json file. -func Output(config *Config, tmKubeconfigPath, namespace string, tr *tmv1beta1.Testrun, metadata *Metadata) error { +func Output(config *Config, tmClient kubernetes.Interface, namespace string, tr *tmv1beta1.Testrun, metadata *Metadata) error { if config.OutputFile == "" { return nil } @@ -58,7 +60,7 @@ func Output(config *Config, tmKubeconfigPath, namespace string, tr *tmv1beta1.Te return err } - osConfig, err := getOSConfig(tmKubeconfigPath, namespace, config.S3Endpoint, config.S3SSL) + osConfig, err := getOSConfig(tmClient, namespace, config.S3Endpoint, config.S3SSL) if err != nil { log.Warnf("Cannot get exported Test results of steps: %s", err.Error()) } else { @@ -225,16 +227,17 @@ func writeToFile(fielPath string, data []byte) error { return nil } -func getOSConfig(tmKubeconfigPath, namespace, minioEndpoint string, ssl bool) (*testmachinery.ObjectStoreConfig, error) { - clusterClient, err := kubernetes.NewClientFromFile(tmKubeconfigPath, nil, client.Options{}) - if err != nil { - return nil, fmt.Errorf("Cannot create client for %s: %s", tmKubeconfigPath, err.Error()) - } - minioConfig, err := clusterClient.GetConfigMap(namespace, "tm-config") +func getOSConfig(tmClient kubernetes.Interface, namespace, minioEndpoint string, ssl bool) (*testmachinery.ObjectStoreConfig, error) { + ctx := context.Background() + defer ctx.Done() + + minioConfig := &corev1.ConfigMap{} + err := tmClient.Client().Get(ctx, client.ObjectKey{Namespace: namespace, Name: "tm-config"}, minioConfig) if err != nil { return nil, fmt.Errorf("Cannot get ConfigMap 'tm-config': %s", err.Error()) } - minioSecrets, err := clusterClient.GetSecret(namespace, minioConfig.Data["objectstore.secretName"]) + minioSecrets := &corev1.Secret{} + err = tmClient.Client().Get(ctx, client.ObjectKey{Namespace: namespace, Name: minioConfig.Data["objectstore.secretName"]}, minioSecrets) if err != nil { return nil, fmt.Errorf("Cannot get Secret '%s': %s", minioConfig.Data["objectstore.secretName"], err.Error()) } diff --git a/pkg/testrunner/run.go b/pkg/testrunner/run.go index 9a59a86691..52dd95f722 100644 --- a/pkg/testrunner/run.go +++ b/pkg/testrunner/run.go @@ -15,23 +15,28 @@ package testrunner import ( + "context" "fmt" "time" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/gardener/gardener/pkg/client/kubernetes" + tmv1beta1 "github.com/gardener/test-infra/pkg/apis/testmachinery/v1beta1" - tmclientset "github.com/gardener/test-infra/pkg/client/testmachinery/clientset/versioned" "github.com/gardener/test-infra/pkg/util" log "github.com/sirupsen/logrus" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" ) -func runTestrun(tmClient *tmclientset.Clientset, tr *tmv1beta1.Testrun, namespace, name string) (*tmv1beta1.Testrun, error) { +func runTestrun(tmClient kubernetes.Interface, tr *tmv1beta1.Testrun, namespace, name string) (*tmv1beta1.Testrun, error) { + ctx := context.Background() + defer ctx.Done() // TODO: Remove legacy name attribute. Instead enforce usage of generateName. tr.Name = "" tr.GenerateName = name tr.Namespace = namespace - tr, err := tmClient.Testmachinery().Testruns(namespace).Create(tr) + err := tmClient.Client().Create(ctx, tr) if err != nil { return nil, fmt.Errorf("Cannot create testrun: %s", err.Error()) } @@ -41,7 +46,7 @@ func runTestrun(tmClient *tmclientset.Clientset, tr *tmv1beta1.Testrun, namespac interval := time.Duration(pollIntervalSeconds) * time.Second timeout := time.Duration(maxWaitTimeSeconds) * time.Second err = wait.PollImmediate(interval, timeout, func() (bool, error) { - tr, err = tmClient.Testmachinery().Testruns(namespace).Get(tr.Name, metav1.GetOptions{}) + err := tmClient.Client().Get(ctx, client.ObjectKey{Namespace: namespace, Name: tr.Name}, tr) if err != nil { log.Errorf("Cannot get testrun: %s", err.Error()) } diff --git a/pkg/testrunner/template/render.go b/pkg/testrunner/template/render.go index 9dcd9aafda..da819cf68d 100644 --- a/pkg/testrunner/template/render.go +++ b/pkg/testrunner/template/render.go @@ -8,19 +8,14 @@ import ( "github.com/gardener/gardener/pkg/client/kubernetes" "github.com/gardener/test-infra/pkg/util" log "github.com/sirupsen/logrus" - "sigs.k8s.io/controller-runtime/pkg/client" ) // RenderChart renders the provided helm chart with testruns, adds the testrun parameters and returns the templated files. -func RenderChart(tmKubeconfigPath string, parameters *TestrunParameters, versions []string) ([]string, error) { +func RenderChart(tmClient kubernetes.Interface, parameters *TestrunParameters, versions []string) ([]string, error) { log.Debugf("Parameters: %+v", util.PrettyPrintStruct(parameters)) log.Debugf("Render chart from %s", parameters.TestrunChartPath) - tmClusterClient, err := kubernetes.NewClientFromFile(tmKubeconfigPath, nil, client.Options{}) - if err != nil { - return nil, fmt.Errorf("couldn't create k8s client from kubeconfig filepath %s: %v", tmKubeconfigPath, err) - } - tmChartRenderer, err := chartrenderer.New(tmClusterClient) + tmChartRenderer, err := chartrenderer.New(tmClient) if err != nil { return nil, fmt.Errorf("Cannot create chartrenderer for gardener: %s", err.Error()) } diff --git a/pkg/testrunner/template/template.go b/pkg/testrunner/template/template.go index a35956cea5..3d8475d022 100644 --- a/pkg/testrunner/template/template.go +++ b/pkg/testrunner/template/template.go @@ -18,6 +18,7 @@ import ( "fmt" "io/ioutil" + "github.com/gardener/gardener/pkg/client/kubernetes" tmv1beta1 "github.com/gardener/test-infra/pkg/apis/testmachinery/v1beta1" "github.com/gardener/test-infra/pkg/testrunner/componentdescriptor" "github.com/gardener/test-infra/pkg/testrunner/result" @@ -26,7 +27,7 @@ import ( ) // Render renders a helm chart with containing testruns, adds the provided parameters and values, and returns the parsed and modified testruns. -func Render(tmKubeconfigPath string, parameters *TestrunParameters, metadata *result.Metadata) ([]*tmv1beta1.Testrun, error) { +func Render(tmClient kubernetes.Interface, parameters *TestrunParameters, metadata *result.Metadata) ([]*tmv1beta1.Testrun, error) { versions, err := getK8sVersions(parameters) if err != nil { log.Fatal(err.Error()) @@ -35,17 +36,16 @@ func Render(tmKubeconfigPath string, parameters *TestrunParameters, metadata *re if parameters.ComponentDescriptorPath != "" { data, err := ioutil.ReadFile(parameters.ComponentDescriptorPath) if err != nil { - log.Warnf("Cannot read component descriptor file %s: %s", parameters.ComponentDescriptorPath, err.Error()) + return nil, fmt.Errorf("Cannot read component descriptor file %s: %s", parameters.ComponentDescriptorPath, err.Error()) } components, err := componentdescriptor.GetComponents(data) if err != nil { - log.Warnf("Cannot decode and parse BOM %s", err.Error()) - } else { - metadata.BOM = components + return nil, fmt.Errorf("Cannot decode and parse BOM %s", err.Error()) } + metadata.BOM = components } - files, err := RenderChart(tmKubeconfigPath, parameters, versions) + files, err := RenderChart(tmClient, parameters, versions) if err != nil { return nil, err } @@ -58,7 +58,7 @@ func Render(tmKubeconfigPath string, parameters *TestrunParameters, metadata *re log.Warnf("Cannot parse rendered file: %s", err.Error()) } - // Add current dependency repositories to the testrun location. + // Add all repositories defined in the component descriptor to the testrun locations. // This gives us all dependent repositories as well as there deployed version. addBOMLocationsToTestrun(&tr, metadata.BOM) diff --git a/pkg/testrunner/testrunner.go b/pkg/testrunner/testrunner.go index 0ff99303d6..0362c9d3e3 100644 --- a/pkg/testrunner/testrunner.go +++ b/pkg/testrunner/testrunner.go @@ -16,15 +16,13 @@ package testrunner import ( "errors" - "fmt" "sync" + "github.com/gardener/gardener/pkg/client/kubernetes" + tmv1beta1 "github.com/gardener/test-infra/pkg/apis/testmachinery/v1beta1" "github.com/gardener/test-infra/pkg/util" - "k8s.io/client-go/tools/clientcmd" - - tmclientset "github.com/gardener/test-infra/pkg/client/testmachinery/clientset/versioned" log "github.com/sirupsen/logrus" ) @@ -39,16 +37,7 @@ func Run(config *Config, testruns []*tmv1beta1.Testrun, testrunNamePrefix string maxWaitTimeSeconds = config.Timeout pollIntervalSeconds = config.Interval - tmConfig, err := clientcmd.BuildConfigFromFlags("", config.TmKubeconfigPath) - if err != nil { - return nil, fmt.Errorf("Cannot build kubernetes client from %s: %s", config.TmKubeconfigPath, err.Error()) - } - tmClient, err := tmclientset.NewForConfig(tmConfig) - if err != nil { - return nil, err - } - - finishedTestruns := runChart(tmClient, testruns, config.Namespace, testrunNamePrefix) + finishedTestruns := runChart(config.TmClient, testruns, config.Namespace, testrunNamePrefix) if len(finishedTestruns) == 0 { return nil, errors.New("No testruns finished") @@ -59,7 +48,7 @@ func Run(config *Config, testruns []*tmv1beta1.Testrun, testrunNamePrefix string // runChart tries to parse each rendered file of a chart into a testrun. // If a filecontent is a testrun then it is deployed into the testmachinery. -func runChart(tmClient *tmclientset.Clientset, testruns []*tmv1beta1.Testrun, namespace, testrunNamePrefix string) []*tmv1beta1.Testrun { +func runChart(tmClient kubernetes.Interface, testruns []*tmv1beta1.Testrun, namespace, testrunNamePrefix string) []*tmv1beta1.Testrun { var wg sync.WaitGroup mutex := &sync.Mutex{} finishedTestruns := []*tmv1beta1.Testrun{} diff --git a/pkg/testrunner/types.go b/pkg/testrunner/types.go index 27ff995faf..63c2c36b3c 100644 --- a/pkg/testrunner/types.go +++ b/pkg/testrunner/types.go @@ -14,11 +14,13 @@ package testrunner +import "github.com/gardener/gardener/pkg/client/kubernetes" + // Config are configuration of the evironment like the testmachinery cluster or S3 store // where the testrunner executes the testrun. type Config struct { - // Path to the kubeconfig where the testmachinery is running. - TmKubeconfigPath string + // Kubernetes client for the testmachinery k8s cluster + TmClient kubernetes.Interface // Namespace where the testrun is deployed. Namespace string diff --git a/test/controller/garbagecollection/garbagecollection_test.go b/test/controller/garbagecollection/garbagecollection_test.go index 66309db315..4481d863e7 100644 --- a/test/controller/garbagecollection/garbagecollection_test.go +++ b/test/controller/garbagecollection/garbagecollection_test.go @@ -15,11 +15,14 @@ package garbagecollection_test import ( + "context" "fmt" "net/http" "os" "time" + "github.com/gardener/test-infra/pkg/testmachinery" + "github.com/gardener/gardener/pkg/client/kubernetes" "sigs.k8s.io/controller-runtime/pkg/client" @@ -29,14 +32,9 @@ import ( . "github.com/onsi/gomega" "k8s.io/apimachinery/pkg/api/errors" - argoclientset "github.com/argoproj/argo/pkg/client/clientset/versioned" - tmclientset "github.com/gardener/test-infra/pkg/client/testmachinery/clientset/versioned" "github.com/gardener/test-infra/pkg/util" "github.com/gardener/test-infra/test/resources" "github.com/gardener/test-infra/test/utils" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "k8s.io/client-go/tools/clientcmd" ) var ( @@ -48,8 +46,7 @@ var _ = Describe("Garbage collection tests", func() { var ( commitSha string namespace string - tmClient *tmclientset.Clientset - argoClient *argoclientset.Clientset + tmClient kubernetes.Interface minioClient *minio.Client minioBucket string ) @@ -61,18 +58,13 @@ var _ = Describe("Garbage collection tests", func() { namespace = os.Getenv("TM_NAMESPACE") minioEndpoint := os.Getenv("S3_ENDPOINT") - tmConfig, err := clientcmd.BuildConfigFromFlags("", tmKubeconfig) - Expect(err).ToNot(HaveOccurred(), "couldn't create k8s client from kubeconfig filepath %s", tmKubeconfig) - - tmClient = tmclientset.NewForConfigOrDie(tmConfig) - - argoClient = argoclientset.NewForConfigOrDie(tmConfig) - - clusterClient, err := kubernetes.NewClientFromFile(tmKubeconfig, nil, client.Options{}) + tmClient, err := kubernetes.NewClientFromFile(tmKubeconfig, nil, client.Options{ + Scheme: testmachinery.TestMachineryScheme, + }) Expect(err).ToNot(HaveOccurred()) - utils.WaitForClusterReadiness(clusterClient, namespace, maxWaitTime) - osConfig := utils.WaitForMinioService(clusterClient, minioEndpoint, namespace, maxWaitTime) + utils.WaitForClusterReadiness(tmClient, namespace, maxWaitTime) + osConfig := utils.WaitForMinioService(tmClient, minioEndpoint, namespace, maxWaitTime) minioBucket = osConfig.BucketName minioClient, err = minio.New(osConfig.Endpoint, osConfig.AccessKey, osConfig.SecretKey, false) @@ -80,9 +72,11 @@ var _ = Describe("Garbage collection tests", func() { }) It("should cleanup all artifacts when a TestDef is deleted", func() { + ctx := context.Background() + defer ctx.Done() tr := resources.GetBasicTestrun(namespace, commitSha) - tr, wf, err := utils.RunTestrun(tmClient, argoClient, tr, argov1.NodeSucceeded, namespace, maxWaitTime) + tr, wf, err := utils.RunTestrun(tmClient, tr, argov1.NodeSucceeded, namespace, maxWaitTime) Expect(err).ToNot(HaveOccurred()) utils.DeleteTestrun(tmClient, tr) @@ -90,7 +84,7 @@ var _ = Describe("Garbage collection tests", func() { for { Expect(util.MaxTimeExceeded(startTime, maxWaitTime)).To(BeFalse(), "Max Wait time exceeded.") - if _, err := tmClient.Testmachinery().Testruns(namespace).Get(tr.Name, metav1.GetOptions{}); err != nil { + if err := tmClient.Client().Get(ctx, client.ObjectKey{Namespace: namespace, Name: tr.Name}, nil); err != nil { Expect(errors.IsNotFound(err)).To(BeTrue(), "Testrun: %s", tr.Name) break } diff --git a/test/controller/resultcollection/resultcollection_test.go b/test/controller/resultcollection/resultcollection_test.go index b81935b176..29aa879cf0 100644 --- a/test/controller/resultcollection/resultcollection_test.go +++ b/test/controller/resultcollection/resultcollection_test.go @@ -15,23 +15,21 @@ package resultcollection_test import ( + "context" "os" "time" + "github.com/gardener/test-infra/pkg/testmachinery" + argov1 "github.com/argoproj/argo/pkg/apis/workflow/v1alpha1" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "sigs.k8s.io/controller-runtime/pkg/client" - argoclientset "github.com/argoproj/argo/pkg/client/clientset/versioned" "github.com/gardener/gardener/pkg/client/kubernetes" tmv1beta1 "github.com/gardener/test-infra/pkg/apis/testmachinery/v1beta1" - tmclientset "github.com/gardener/test-infra/pkg/client/testmachinery/clientset/versioned" "github.com/gardener/test-infra/test/resources" "github.com/gardener/test-infra/test/utils" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "k8s.io/client-go/tools/clientcmd" ) var ( @@ -41,10 +39,9 @@ var ( var _ = Describe("Result collection tests", func() { var ( - commitSha string - namespace string - tmClient *tmclientset.Clientset - argoClient *argoclientset.Clientset + commitSha string + namespace string + tmClient kubernetes.Interface ) BeforeSuite(func() { @@ -53,30 +50,27 @@ var _ = Describe("Result collection tests", func() { tmKubeconfig := os.Getenv("TM_KUBECONFIG_PATH") namespace = os.Getenv("TM_NAMESPACE") - tmConfig, err := clientcmd.BuildConfigFromFlags("", tmKubeconfig) - Expect(err).ToNot(HaveOccurred(), "couldn't create k8s client from kubeconfig filepath %s", tmKubeconfig) - - tmClient = tmclientset.NewForConfigOrDie(tmConfig) - - argoClient = argoclientset.NewForConfigOrDie(tmConfig) - - clusterClient, err := kubernetes.NewClientFromFile(tmKubeconfig, nil, client.Options{}) + tmClient, err := kubernetes.NewClientFromFile(tmKubeconfig, nil, client.Options{ + Scheme: testmachinery.TestMachineryScheme, + }) Expect(err).ToNot(HaveOccurred()) - utils.WaitForClusterReadiness(clusterClient, namespace, maxWaitTime) + utils.WaitForClusterReadiness(tmClient, namespace, maxWaitTime) }) Context("step status", func() { It("should update status immediately with all steps of the generated testflow", func() { + ctx := context.Background() + defer ctx.Done() tr := resources.GetBasicTestrun(namespace, commitSha) - tr, err := tmClient.Testmachinery().Testruns(tr.Namespace).Create(tr) + err := tmClient.Client().Create(ctx, tr) Expect(err).ToNot(HaveOccurred()) defer utils.DeleteTestrun(tmClient, tr) time.Sleep(5 * time.Second) - tr, err = tmClient.Testmachinery().Testruns(namespace).Get(tr.Name, metav1.GetOptions{}) + err = tmClient.Client().Get(ctx, client.ObjectKey{Namespace: namespace, Name: tr.Name}, tr) Expect(err).ToNot(HaveOccurred()) Expect(utils.TestflowLen(tr.Status.Steps)).To(Equal(1), "Should be one steps status") @@ -86,9 +80,11 @@ var _ = Describe("Result collection tests", func() { }) It("should collect status of all workflow nodes", func() { + ctx := context.Background() + defer ctx.Done() tr := resources.GetBasicTestrun(namespace, commitSha) - tr, _, err := utils.RunTestrun(tmClient, argoClient, tr, argov1.NodeSucceeded, namespace, maxWaitTime) + tr, _, err := utils.RunTestrun(ctx, tmClient, tr, argov1.NodeSucceeded, namespace, maxWaitTime) Expect(err).ToNot(HaveOccurred()) defer utils.DeleteTestrun(tmClient, tr) @@ -102,6 +98,8 @@ var _ = Describe("Result collection tests", func() { }) It("should collect status of multiple workflow nodes", func() { + ctx := context.Background() + defer ctx.Done() tr := resources.GetBasicTestrun(namespace, commitSha) tr.Spec.TestFlow = [][]tmv1beta1.TestflowStep{ { @@ -119,7 +117,7 @@ var _ = Describe("Result collection tests", func() { }, } - tr, _, err := utils.RunTestrun(tmClient, argoClient, tr, argov1.NodeSucceeded, namespace, maxWaitTime) + tr, _, err := utils.RunTestrun(ctx, tmClient, tr, argov1.NodeSucceeded, namespace, maxWaitTime) Expect(err).ToNot(HaveOccurred()) defer utils.DeleteTestrun(tmClient, tr) @@ -138,6 +136,8 @@ var _ = Describe("Result collection tests", func() { }) It("should collect status of multiple workflow nodes with a failing step", func() { + ctx := context.Background() + defer ctx.Done() tr := resources.GetBasicTestrun(namespace, commitSha) tr.Spec.TestFlow = [][]tmv1beta1.TestflowStep{ { @@ -155,7 +155,7 @@ var _ = Describe("Result collection tests", func() { }, } - tr, _, err := utils.RunTestrun(tmClient, argoClient, tr, argov1.NodeFailed, namespace, maxWaitTime) + tr, _, err := utils.RunTestrun(ctx, tmClient, tr, argov1.NodeFailed, namespace, maxWaitTime) Expect(err).ToNot(HaveOccurred()) defer utils.DeleteTestrun(tmClient, tr) diff --git a/test/controller/testdefinition/testdefinition_suite_test.go b/test/controller/testdefinition/testdefinition_suite_test.go index dd310d1a9d..2faf592504 100644 --- a/test/controller/testdefinition/testdefinition_suite_test.go +++ b/test/controller/testdefinition/testdefinition_suite_test.go @@ -15,9 +15,12 @@ package testdefinition_test import ( + "context" "os" "testing" + "github.com/gardener/test-infra/pkg/testmachinery" + argov1 "github.com/argoproj/argo/pkg/apis/workflow/v1alpha1" "github.com/gardener/gardener/pkg/client/kubernetes" tmv1beta1 "github.com/gardener/test-infra/pkg/apis/testmachinery/v1beta1" @@ -26,12 +29,8 @@ import ( corev1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" - argoclientset "github.com/argoproj/argo/pkg/client/clientset/versioned" - tmclientset "github.com/gardener/test-infra/pkg/client/testmachinery/clientset/versioned" "github.com/gardener/test-infra/test/resources" "github.com/gardener/test-infra/test/utils" - - "k8s.io/client-go/tools/clientcmd" ) var ( @@ -46,11 +45,9 @@ func TestValidationWebhook(t *testing.T) { var _ = Describe("Testflow execution tests", func() { var ( - commitSha string - namespace string - tmClient *tmclientset.Clientset - argoClient *argoclientset.Clientset - clusterClient kubernetes.Interface + commitSha string + namespace string + tmClient kubernetes.Interface ) BeforeSuite(func() { @@ -59,22 +56,19 @@ var _ = Describe("Testflow execution tests", func() { namespace = os.Getenv("TM_NAMESPACE") tmKubeconfig := os.Getenv("TM_KUBECONFIG_PATH") - tmConfig, err := clientcmd.BuildConfigFromFlags("", tmKubeconfig) - Expect(err).ToNot(HaveOccurred(), "couldn't create k8s client from kubeconfig filepath %s", tmKubeconfig) - - tmClient = tmclientset.NewForConfigOrDie(tmConfig) - - argoClient = argoclientset.NewForConfigOrDie(tmConfig) - - clusterClient, err = kubernetes.NewClientFromFile(tmKubeconfig, nil, client.Options{}) + tmClient, err := kubernetes.NewClientFromFile(tmKubeconfig, nil, client.Options{ + Scheme: testmachinery.TestMachineryScheme, + }) Expect(err).ToNot(HaveOccurred()) - utils.WaitForClusterReadiness(clusterClient, namespace, maxWaitTime) + utils.WaitForClusterReadiness(tmClient, namespace, maxWaitTime) }) Context("config", func() { Context("type environment variable", func() { It("should run a TesDef with a environment variable defined by value", func() { + ctx := context.Background() + defer ctx.Done() tr := resources.GetBasicTestrun(namespace, commitSha) tr.Spec.TestFlow = [][]tmv1beta1.TestflowStep{ { @@ -84,12 +78,14 @@ var _ = Describe("Testflow execution tests", func() { }, } - tr, _, err := utils.RunTestrun(tmClient, argoClient, tr, argov1.NodeSucceeded, namespace, maxWaitTime) + tr, _, err := utils.RunTestrun(ctx, tmClient, tr, argov1.NodeSucceeded, namespace, maxWaitTime) defer utils.DeleteTestrun(tmClient, tr) Expect(err).ToNot(HaveOccurred()) }) It("should run a TesDef with a evironment variable defined by a secret", func() { + ctx := context.Background() + defer ctx.Done() tr := resources.GetBasicTestrun(namespace, commitSha) tr.Spec.TestFlow = [][]tmv1beta1.TestflowStep{ { @@ -99,16 +95,16 @@ var _ = Describe("Testflow execution tests", func() { }, } - _, err := clusterClient.CreateSecret(namespace, "test-secret", corev1.SecretTypeOpaque, map[string][]byte{ + _, err := tmClient.CreateSecret(namespace, "test-secret", corev1.SecretTypeOpaque, map[string][]byte{ "test": []byte("test"), }, false) Expect(err).ToNot(HaveOccurred()) defer func() { - err := clusterClient.DeleteSecret(namespace, "test-secret") + err := tmClient.DeleteSecret(namespace, "test-secret") Expect(err).ToNot(HaveOccurred(), "Cannot delete secret") }() - tr, _, err = utils.RunTestrun(tmClient, argoClient, tr, argov1.NodeSucceeded, namespace, maxWaitTime) + tr, _, err = utils.RunTestrun(ctx, tmClient, tr, argov1.NodeSucceeded, namespace, maxWaitTime) defer utils.DeleteTestrun(tmClient, tr) Expect(err).ToNot(HaveOccurred()) }) @@ -116,6 +112,8 @@ var _ = Describe("Testflow execution tests", func() { Context("type file", func() { It("should run a TesDef with a file defined by a secret", func() { + ctx := context.Background() + defer ctx.Done() tr := resources.GetBasicTestrun(namespace, commitSha) tr.Spec.TestFlow = [][]tmv1beta1.TestflowStep{ { @@ -125,16 +123,16 @@ var _ = Describe("Testflow execution tests", func() { }, } - _, err := clusterClient.CreateSecret(namespace, "test-secret-file", corev1.SecretTypeOpaque, map[string][]byte{ + _, err := tmClient.CreateSecret(namespace, "test-secret-file", corev1.SecretTypeOpaque, map[string][]byte{ "test": []byte("test"), }, false) Expect(err).ToNot(HaveOccurred()) defer func() { - err := clusterClient.DeleteSecret(namespace, "test-secret-file") + err := tmClient.DeleteSecret(namespace, "test-secret-file") Expect(err).ToNot(HaveOccurred(), "Cannot delete secret") }() - tr, _, err = utils.RunTestrun(tmClient, argoClient, tr, argov1.NodeSucceeded, namespace, maxWaitTime) + tr, _, err = utils.RunTestrun(ctx, tmClient, tr, argov1.NodeSucceeded, namespace, maxWaitTime) defer utils.DeleteTestrun(tmClient, tr) Expect(err).ToNot(HaveOccurred()) }) diff --git a/test/controller/testflow/testflow_suite_test.go b/test/controller/testflow/testflow_suite_test.go index 01386c688a..0249427c83 100644 --- a/test/controller/testflow/testflow_suite_test.go +++ b/test/controller/testflow/testflow_suite_test.go @@ -21,6 +21,8 @@ import ( "testing" "time" + "github.com/gardener/test-infra/pkg/testmachinery" + "github.com/gardener/gardener/pkg/client/kubernetes" "sigs.k8s.io/controller-runtime/pkg/client" @@ -29,9 +31,7 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - argoclientset "github.com/argoproj/argo/pkg/client/clientset/versioned" tmv1beta1 "github.com/gardener/test-infra/pkg/apis/testmachinery/v1beta1" - tmclientset "github.com/gardener/test-infra/pkg/client/testmachinery/clientset/versioned" "github.com/gardener/test-infra/pkg/util" "github.com/gardener/test-infra/test/resources" "github.com/gardener/test-infra/test/utils" @@ -39,7 +39,6 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/tools/clientcmd" ) var ( @@ -54,11 +53,9 @@ func TestValidationWebhook(t *testing.T) { var _ = Describe("Testflow execution tests", func() { var ( - commitSha string - namespace string - tmClient *tmclientset.Clientset - argoClient *argoclientset.Clientset - clusterClient kubernetes.Interface + commitSha string + namespace string + tmClient kubernetes.Interface ) BeforeSuite(func() { @@ -67,23 +64,20 @@ var _ = Describe("Testflow execution tests", func() { tmKubeconfig := os.Getenv("TM_KUBECONFIG_PATH") namespace = os.Getenv("TM_NAMESPACE") - tmConfig, err := clientcmd.BuildConfigFromFlags("", tmKubeconfig) - Expect(err).ToNot(HaveOccurred(), "couldn't create k8s client from kubeconfig filepath %s", tmKubeconfig) - - tmClient = tmclientset.NewForConfigOrDie(tmConfig) - - argoClient = argoclientset.NewForConfigOrDie(tmConfig) - - clusterClient, err = kubernetes.NewClientFromFile(tmKubeconfig, nil, client.Options{}) + tmClient, err := kubernetes.NewClientFromFile(tmKubeconfig, nil, client.Options{ + Scheme: testmachinery.TestMachineryScheme, + }) Expect(err).ToNot(HaveOccurred()) - utils.WaitForClusterReadiness(clusterClient, namespace, maxWaitTime) + utils.WaitForClusterReadiness(tmClient, namespace, maxWaitTime) }) Context("testflow", func() { It("should run a test with TestDefs defined by name", func() { + ctx := context.Background() + defer ctx.Done() tr := resources.GetBasicTestrun(namespace, commitSha) - tr, wf, err := utils.RunTestrun(tmClient, argoClient, tr, argov1.NodeSucceeded, namespace, maxWaitTime) + tr, wf, err := utils.RunTestrun(ctx, tmClient, tr, argov1.NodeSucceeded, namespace, maxWaitTime) defer utils.DeleteTestrun(tmClient, tr) Expect(err).ToNot(HaveOccurred()) @@ -98,6 +92,8 @@ var _ = Describe("Testflow execution tests", func() { }) It("should run a test with TestDefs defined by label and name", func() { + ctx := context.Background() + defer ctx.Done() tr := resources.GetBasicTestrun(namespace, commitSha) tr.Spec.TestFlow = append(tr.Spec.TestFlow, []tmv1beta1.TestflowStep{ { @@ -105,7 +101,7 @@ var _ = Describe("Testflow execution tests", func() { }, }) - tr, wf, err := utils.RunTestrun(tmClient, argoClient, tr, argov1.NodeSucceeded, namespace, maxWaitTime) + tr, wf, err := utils.RunTestrun(ctx, tmClient, tr, argov1.NodeSucceeded, namespace, maxWaitTime) defer utils.DeleteTestrun(tmClient, tr) Expect(err).ToNot(HaveOccurred()) @@ -120,6 +116,8 @@ var _ = Describe("Testflow execution tests", func() { }) It("should execute all tests in right order when no testdefs for a label can be found", func() { + ctx := context.Background() + defer ctx.Done() tr := resources.GetBasicTestrun(namespace, commitSha) tr.Spec.TestFlow = append(tr.Spec.TestFlow, []tmv1beta1.TestflowStep{ { @@ -133,7 +131,7 @@ var _ = Describe("Testflow execution tests", func() { }, }) - tr, wf, err := utils.RunTestrun(tmClient, argoClient, tr, argov1.NodeSucceeded, namespace, maxWaitTime) + tr, wf, err := utils.RunTestrun(ctx, tmClient, tr, argov1.NodeSucceeded, namespace, maxWaitTime) defer utils.DeleteTestrun(tmClient, tr) Expect(err).ToNot(HaveOccurred()) @@ -148,6 +146,8 @@ var _ = Describe("Testflow execution tests", func() { }) It("should execute serial steps after parallel steps", func() { + ctx := context.Background() + defer ctx.Done() tr := resources.GetBasicTestrun(namespace, commitSha) tr.Spec.TestFlow = [][]tmv1beta1.TestflowStep{ { @@ -157,7 +157,7 @@ var _ = Describe("Testflow execution tests", func() { }, } - tr, wf, err := utils.RunTestrun(tmClient, argoClient, tr, argov1.NodeSucceeded, namespace, maxWaitTime) + tr, wf, err := utils.RunTestrun(ctx, tmClient, tr, argov1.NodeSucceeded, namespace, maxWaitTime) defer utils.DeleteTestrun(tmClient, tr) Expect(err).ToNot(HaveOccurred()) @@ -180,6 +180,8 @@ var _ = Describe("Testflow execution tests", func() { Context("config", func() { Context("type environment variable", func() { It("should mount a config as environment variable", func() { + ctx := context.Background() + defer ctx.Done() tr := resources.GetBasicTestrun(namespace, commitSha) tr.Spec.TestFlow = [][]tmv1beta1.TestflowStep{ { @@ -196,13 +198,15 @@ var _ = Describe("Testflow execution tests", func() { }, } - tr, _, err := utils.RunTestrun(tmClient, argoClient, tr, argov1.NodeSucceeded, namespace, maxWaitTime) + tr, _, err := utils.RunTestrun(ctx, tmClient, tr, argov1.NodeSucceeded, namespace, maxWaitTime) defer utils.DeleteTestrun(tmClient, tr) Expect(err).ToNot(HaveOccurred()) }) It("should mount a global config as environement variable", func() { + ctx := context.Background() + defer ctx.Done() tr := resources.GetBasicTestrun(namespace, commitSha) tr.Spec.TestFlow = [][]tmv1beta1.TestflowStep{ { @@ -219,7 +223,7 @@ var _ = Describe("Testflow execution tests", func() { }, } - tr, _, err := utils.RunTestrun(tmClient, argoClient, tr, argov1.NodeSucceeded, namespace, maxWaitTime) + tr, _, err := utils.RunTestrun(ctx, tmClient, tr, argov1.NodeSucceeded, namespace, maxWaitTime) defer utils.DeleteTestrun(tmClient, tr) Expect(err).ToNot(HaveOccurred()) }) @@ -227,6 +231,8 @@ var _ = Describe("Testflow execution tests", func() { Context("type file", func() { It("should mount a config with value as file to a specific path", func() { + ctx := context.Background() + defer ctx.Done() tr := resources.GetBasicTestrun(namespace, commitSha) tr.Spec.TestFlow = [][]tmv1beta1.TestflowStep{ { @@ -249,7 +255,7 @@ var _ = Describe("Testflow execution tests", func() { }, } - tr, _, err := utils.RunTestrun(tmClient, argoClient, tr, argov1.NodeSucceeded, namespace, maxWaitTime) + tr, _, err := utils.RunTestrun(ctx, tmClient, tr, argov1.NodeSucceeded, namespace, maxWaitTime) defer utils.DeleteTestrun(tmClient, tr) Expect(err).ToNot(HaveOccurred()) }) @@ -267,10 +273,10 @@ var _ = Describe("Testflow execution tests", func() { "test": []byte("test"), }, } - err := clusterClient.Client().Create(ctx, secret) + err := tmClient.Client().Create(ctx, secret) Expect(err).ToNot(HaveOccurred()) defer func() { - err := clusterClient.Client().Delete(ctx, secret) + err := tmClient.Client().Delete(ctx, secret) Expect(err).ToNot(HaveOccurred(), "Cannot delete secret") }() @@ -303,7 +309,7 @@ var _ = Describe("Testflow execution tests", func() { }, } - tr, _, err = utils.RunTestrun(tmClient, argoClient, tr, argov1.NodeSucceeded, namespace, maxWaitTime) + tr, _, err = utils.RunTestrun(ctx, tmClient, tr, argov1.NodeSucceeded, namespace, maxWaitTime) defer utils.DeleteTestrun(tmClient, tr) Expect(err).ToNot(HaveOccurred()) @@ -321,10 +327,10 @@ var _ = Describe("Testflow execution tests", func() { "test": "test", }, } - err := clusterClient.Client().Create(ctx, configmap) + err := tmClient.Client().Create(ctx, configmap) Expect(err).ToNot(HaveOccurred()) defer func() { - err := clusterClient.Client().Delete(ctx, configmap) + err := tmClient.Client().Delete(ctx, configmap) Expect(err).ToNot(HaveOccurred(), "Cannot delete configmap %s", configmap.Name) }() @@ -357,7 +363,7 @@ var _ = Describe("Testflow execution tests", func() { }, } - tr, _, err = utils.RunTestrun(tmClient, argoClient, tr, argov1.NodeSucceeded, namespace, maxWaitTime) + tr, _, err = utils.RunTestrun(ctx, tmClient, tr, argov1.NodeSucceeded, namespace, maxWaitTime) defer utils.DeleteTestrun(tmClient, tr) Expect(err).ToNot(HaveOccurred()) @@ -368,9 +374,11 @@ var _ = Describe("Testflow execution tests", func() { Context("onExit", func() { It("should run ExitHandlerTestDef when testflow succeeds", func() { + ctx := context.Background() + defer ctx.Done() tr := resources.GetTestrunWithExitHandler(resources.GetBasicTestrun(namespace, commitSha), tmv1beta1.ConditionTypeSuccess) - tr, wf, err := utils.RunTestrun(tmClient, argoClient, tr, argov1.NodeSucceeded, namespace, maxWaitTime) + tr, wf, err := utils.RunTestrun(ctx, tmClient, tr, argov1.NodeSucceeded, namespace, maxWaitTime) defer utils.DeleteTestrun(tmClient, tr) Expect(err).ToNot(HaveOccurred()) @@ -385,9 +393,11 @@ var _ = Describe("Testflow execution tests", func() { }) It("should not run exit-handler-testdef when testflow succeeds", func() { + ctx := context.Background() + defer ctx.Done() tr := resources.GetTestrunWithExitHandler(resources.GetBasicTestrun(namespace, commitSha), tmv1beta1.ConditionTypeError) - tr, wf, err := utils.RunTestrun(tmClient, argoClient, tr, argov1.NodeSucceeded, namespace, maxWaitTime) + tr, wf, err := utils.RunTestrun(ctx, tmClient, tr, argov1.NodeSucceeded, namespace, maxWaitTime) defer utils.DeleteTestrun(tmClient, tr) Expect(err).ToNot(HaveOccurred()) @@ -402,9 +412,11 @@ var _ = Describe("Testflow execution tests", func() { }) It("should run exit-handler-testdef when testflow fails", func() { + ctx := context.Background() + defer ctx.Done() tr := resources.GetTestrunWithExitHandler(resources.GetFailingTestrun(namespace, commitSha), tmv1beta1.ConditionTypeError) - tr, wf, err := utils.RunTestrun(tmClient, argoClient, tr, argov1.NodeFailed, namespace, maxWaitTime) + tr, wf, err := utils.RunTestrun(ctx, tmClient, tr, argov1.NodeFailed, namespace, maxWaitTime) defer utils.DeleteTestrun(tmClient, tr) Expect(err).ToNot(HaveOccurred()) @@ -421,47 +433,53 @@ var _ = Describe("Testflow execution tests", func() { Context("TTL", func() { It("should delete the testrun after ttl has finished", func() { + ctx := context.Background() + defer ctx.Done() + var ttl int32 = 90 var maxWaitTime int64 = 600 tr := resources.GetBasicTestrun(namespace, commitSha) tr.Spec.TTLSecondsAfterFinished = &ttl - tr, err := tmClient.Testmachinery().Testruns(tr.Namespace).Create(tr) + err := tmClient.Client().Create(ctx, tr) defer utils.DeleteTestrun(tmClient, tr) Expect(err).ToNot(HaveOccurred()) startTime := time.Now() for !util.MaxTimeExceeded(startTime, maxWaitTime) { - _, err := tmClient.Testmachinery().Testruns(tr.Namespace).Get(tr.Name, metav1.GetOptions{}) + err = tmClient.Client().Get(ctx, client.ObjectKey{Namespace: tr.Namespace, Name: tr.Name}, nil) if errors.IsNotFound(err) { return } time.Sleep(5 * time.Second) } - _, err = tmClient.Testmachinery().Testruns(tr.Namespace).Get(tr.Name, metav1.GetOptions{}) + err = tmClient.Client().Get(ctx, client.ObjectKey{Namespace: tr.Namespace, Name: tr.Name}, nil) Expect(errors.IsNotFound(err)).To(BeTrue(), "Testrun %s was not deleted in %d seconds", tr.Name, maxWaitTime) }) It("should delete the testrun after workflow has finished", func() { + ctx := context.Background() + defer ctx.Done() + var ttl int32 = 1 var maxWaitTime int64 = 600 tr := resources.GetBasicTestrun(namespace, commitSha) tr.Spec.TTLSecondsAfterFinished = &ttl - tr, err := tmClient.Testmachinery().Testruns(tr.Namespace).Create(tr) + err := tmClient.Client().Create(ctx, tr) defer utils.DeleteTestrun(tmClient, tr) Expect(err).ToNot(HaveOccurred()) startTime := time.Now() for !util.MaxTimeExceeded(startTime, maxWaitTime) { - _, err := tmClient.Testmachinery().Testruns(tr.Namespace).Get(tr.Name, metav1.GetOptions{}) + err = tmClient.Client().Get(ctx, client.ObjectKey{Namespace: tr.Namespace, Name: tr.Name}, nil) if errors.IsNotFound(err) { return } time.Sleep(5 * time.Second) } - _, err = tmClient.Testmachinery().Testruns(tr.Namespace).Get(tr.Name, metav1.GetOptions{}) + err = tmClient.Client().Get(ctx, client.ObjectKey{Namespace: tr.Namespace, Name: tr.Name}, nil) Expect(errors.IsNotFound(err)).To(BeTrue(), "Testrun %s was not deleted in %d seconds", tr.Name, maxWaitTime) }) }) diff --git a/test/testrunner/output/testrunner_output_execution_test.go b/test/testrunner/output/testrunner_output_execution_test.go index 54762d129f..fc198ec9eb 100644 --- a/test/testrunner/output/testrunner_output_execution_test.go +++ b/test/testrunner/output/testrunner_output_execution_test.go @@ -15,12 +15,15 @@ package testrunner_output_test import ( - "github.com/gardener/test-infra/pkg/testrunner/result" "bufio" + "context" "encoding/json" "os" "testing" + "github.com/gardener/test-infra/pkg/testmachinery" + "github.com/gardener/test-infra/pkg/testrunner/result" + "github.com/gardener/test-infra/pkg/util" "sigs.k8s.io/controller-runtime/pkg/client" @@ -29,12 +32,8 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - argoclientset "github.com/argoproj/argo/pkg/client/clientset/versioned" - tmclientset "github.com/gardener/test-infra/pkg/client/testmachinery/clientset/versioned" "github.com/gardener/test-infra/test/resources" "github.com/gardener/test-infra/test/utils" - - "k8s.io/client-go/tools/clientcmd" ) var ( @@ -52,8 +51,7 @@ var _ = Describe("Testrunner execution tests", func() { commitSha string namespace string tmKubeconfig string - tmClient *tmclientset.Clientset - argoClient *argoclientset.Clientset + tmClient kubernetes.Interface outputFilePath = "./out-" testrunConfig result.Config s3Endpoint string @@ -66,18 +64,13 @@ var _ = Describe("Testrunner execution tests", func() { namespace = os.Getenv("TM_NAMESPACE") s3Endpoint = os.Getenv("S3_ENDPOINT") - tmConfig, err := clientcmd.BuildConfigFromFlags("", tmKubeconfig) - Expect(err).ToNot(HaveOccurred(), "couldn't create k8s client from kubeconfig filepath %s", tmKubeconfig) - - tmClient = tmclientset.NewForConfigOrDie(tmConfig) - - argoClient = argoclientset.NewForConfigOrDie(tmConfig) - - clusterClient, err := kubernetes.NewClientFromFile(tmKubeconfig, nil, client.Options{}) + tmClient, err := kubernetes.NewClientFromFile(tmKubeconfig, nil, client.Options{ + Scheme: testmachinery.TestMachineryScheme, + }) Expect(err).ToNot(HaveOccurred()) - utils.WaitForClusterReadiness(clusterClient, namespace, maxWaitTime) - utils.WaitForMinioService(clusterClient, s3Endpoint, namespace, maxWaitTime) + utils.WaitForClusterReadiness(tmClient, namespace, maxWaitTime) + utils.WaitForMinioService(tmClient, s3Endpoint, namespace, maxWaitTime) }) BeforeEach(func() { @@ -90,15 +83,17 @@ var _ = Describe("Testrunner execution tests", func() { }) It("should output a summary of the testrun as elasticsearch bulk request", func() { + ctx := context.Background() + defer ctx.Done() testrunConfig.OutputFile = outputFilePath + util.RandomString(3) tr := resources.GetBasicTestrun(namespace, commitSha) - tr, _, err := utils.RunTestrun(tmClient, argoClient, tr, argov1.NodeSucceeded, namespace, maxWaitTime) + tr, _, err := utils.RunTestrun(ctx, tmClient, tr, argov1.NodeSucceeded, namespace, maxWaitTime) Expect(err).ToNot(HaveOccurred()) if err == nil { defer utils.DeleteTestrun(tmClient, tr) } - err = result.Output(&testrunConfig, tmKubeconfig, namespace, tr, &result.Metadata{}) + err = result.Output(&testrunConfig, tmClient, namespace, tr, &result.Metadata{}) Expect(err).ToNot(HaveOccurred()) file, err := os.Open(testrunConfig.OutputFile) @@ -127,15 +122,17 @@ var _ = Describe("Testrunner execution tests", func() { }) It("should add exported artifacts to the elasticsearch bulk output", func() { + ctx := context.Background() + defer ctx.Done() testrunConfig.OutputFile = outputFilePath + util.RandomString(3) tr := resources.GetBasicTestrun(namespace, commitSha) - tr, _, err := utils.RunTestrun(tmClient, argoClient, tr, argov1.NodeSucceeded, namespace, maxWaitTime) + tr, _, err := utils.RunTestrun(ctx, tmClient, tr, argov1.NodeSucceeded, namespace, maxWaitTime) Expect(err).ToNot(HaveOccurred()) if err == nil { defer utils.DeleteTestrun(tmClient, tr) } - err = result.Output(&testrunConfig, tmKubeconfig, namespace, tr, &result.Metadata{}) + err = result.Output(&testrunConfig, tmClient, namespace, tr, &result.Metadata{}) Expect(err).ToNot(HaveOccurred()) file, err := os.Open(testrunConfig.OutputFile) diff --git a/test/testrunner/run/testrunner_run_test.go b/test/testrunner/run/testrunner_run_test.go index 2224cd00c5..721076ad15 100644 --- a/test/testrunner/run/testrunner_run_test.go +++ b/test/testrunner/run/testrunner_run_test.go @@ -18,7 +18,8 @@ import ( "os" "testing" - "k8s.io/client-go/tools/clientcmd" + "github.com/gardener/test-infra/pkg/testmachinery" + "sigs.k8s.io/controller-runtime/pkg/client" "github.com/gardener/gardener/pkg/client/kubernetes" @@ -27,7 +28,6 @@ import ( . "github.com/onsi/gomega" tmv1beta1 "github.com/gardener/test-infra/pkg/apis/testmachinery/v1beta1" - tmclientset "github.com/gardener/test-infra/pkg/client/testmachinery/clientset/versioned" "github.com/gardener/test-infra/test/resources" "github.com/gardener/test-infra/test/utils" ) @@ -47,7 +47,7 @@ var _ = Describe("Testrunner execution tests", func() { commitSha string namespace string tmKubeconfig string - tmClient *tmclientset.Clientset + tmClient kubernetes.Interface testrunConfig testrunner.Config s3Endpoint string ) @@ -59,24 +59,21 @@ var _ = Describe("Testrunner execution tests", func() { namespace = os.Getenv("TM_NAMESPACE") s3Endpoint = os.Getenv("S3_ENDPOINT") - tmConfig, err := clientcmd.BuildConfigFromFlags("", tmKubeconfig) - Expect(err).ToNot(HaveOccurred(), "couldn't create k8s client from kubeconfig filepath %s", tmKubeconfig) - - tmClient = tmclientset.NewForConfigOrDie(tmConfig) - - clusterClient, err := kubernetes.NewClientFromFile(tmKubeconfig, nil, client.Options{}) + tmClient, err := kubernetes.NewClientFromFile(tmKubeconfig, nil, client.Options{ + Scheme: testmachinery.TestMachineryScheme, + }) Expect(err).ToNot(HaveOccurred()) - utils.WaitForClusterReadiness(clusterClient, namespace, maxWaitTime) - utils.WaitForMinioService(clusterClient, s3Endpoint, namespace, maxWaitTime) + utils.WaitForClusterReadiness(tmClient, namespace, maxWaitTime) + utils.WaitForMinioService(tmClient, s3Endpoint, namespace, maxWaitTime) }) BeforeEach(func() { testrunConfig = testrunner.Config{ - TmKubeconfigPath: tmKubeconfig, - Namespace: namespace, - Timeout: maxWaitTime, - Interval: 5, + TmClient: tmClient, + Namespace: namespace, + Timeout: maxWaitTime, + Interval: 5, } }) diff --git a/test/utils/utils.go b/test/utils/utils.go index e905b78669..574c4e820d 100644 --- a/test/utils/utils.go +++ b/test/utils/utils.go @@ -15,6 +15,7 @@ package utils import ( + "context" "errors" "fmt" "net/http" @@ -22,6 +23,8 @@ import ( "strings" "time" + "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/gardener/gardener/pkg/utils/kubernetes/health" "github.com/gardener/test-infra/pkg/testmachinery" @@ -33,34 +36,29 @@ import ( argov1 "github.com/argoproj/argo/pkg/apis/workflow/v1alpha1" tmv1beta1 "github.com/gardener/test-infra/pkg/apis/testmachinery/v1beta1" apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - argoclientset "github.com/argoproj/argo/pkg/client/clientset/versioned" - tmclientset "github.com/gardener/test-infra/pkg/client/testmachinery/clientset/versioned" . "github.com/onsi/gomega" ) // RunTestrun executes a testrun on a cluster and returns the corresponding executed testrun and workflow. -func RunTestrun(tmClient *tmclientset.Clientset, argoClient *argoclientset.Clientset, tr *tmv1beta1.Testrun, phase argov1.NodePhase, namespace string, maxWaitTime int64) (*tmv1beta1.Testrun, *argov1.Workflow, error) { - tr, err := tmClient.Testmachinery().Testruns(tr.Namespace).Create(tr) +// Note: Deletion of the workflow on error should be handled by the calling test. +func RunTestrun(ctx context.Context, tmClient kubernetes.Interface, tr *tmv1beta1.Testrun, phase argov1.NodePhase, namespace string, maxWaitTime int64) (*tmv1beta1.Testrun, *argov1.Workflow, error) { + err := tmClient.Client().Create(ctx, tr) if err != nil { return nil, nil, err } - foundTestrun, err := WatchTestrun(tmClient, tr, namespace, maxWaitTime) + foundTestrun, err := WatchTestrun(ctx, tmClient, tr, namespace, maxWaitTime) if err != nil { - DeleteTestrun(tmClient, tr) return nil, nil, fmt.Errorf("Error watching Testrun: %s\n%s", tr.Name, err.Error()) } - wf, err := GetWorkflow(argoClient, foundTestrun) + wf, err := GetWorkflow(ctx, tmClient, foundTestrun) if err != nil { - DeleteTestrun(tmClient, tr) return nil, nil, fmt.Errorf("Cannot get Workflow for Testrun: %s\n%s", tr.Name, err.Error()) } if reflect.DeepEqual(foundTestrun.Status, tmv1beta1.TestrunStatus{}) { - DeleteTestrun(tmClient, tr) return nil, nil, fmt.Errorf("Testrun %s status is empty", tr.Name) } if foundTestrun.Status.Phase != phase { @@ -80,7 +78,6 @@ func RunTestrun(tmClient *tmclientset.Clientset, argoClient *argoclientset.Clien errMsg = fmt.Sprintf("%s.\nAdditional Errors: %s", errMsg, strings.Join(errMsgs, "; ")) } - DeleteTestrun(tmClient, tr) return nil, nil, errors.New(errMsg) } @@ -88,8 +85,9 @@ func RunTestrun(tmClient *tmclientset.Clientset, argoClient *argoclientset.Clien } // GetWorkflow returns the argo workflow of a testrun. -func GetWorkflow(argoClient *argoclientset.Clientset, tr *tmv1beta1.Testrun) (*argov1.Workflow, error) { - wf, err := argoClient.Argoproj().Workflows(tr.Namespace).Get(tr.Status.Workflow, metav1.GetOptions{}) +func GetWorkflow(ctx context.Context, tmClient kubernetes.Interface, tr *tmv1beta1.Testrun) (*argov1.Workflow, error) { + wf := &argov1.Workflow{} + err := tmClient.Client().Get(ctx, client.ObjectKey{Namespace: tr.Namespace, Name: tr.Status.Workflow}, wf) if err != nil { return nil, err } @@ -97,7 +95,7 @@ func GetWorkflow(argoClient *argoclientset.Clientset, tr *tmv1beta1.Testrun) (*a } // WatchTestrun watches a testrun to finish and returns the newest testrun object. -func WatchTestrun(tmClient *tmclientset.Clientset, tr *tmv1beta1.Testrun, namespace string, maxWaitTime int64) (*tmv1beta1.Testrun, error) { +func WatchTestrun(ctx context.Context, tmClient kubernetes.Interface, tr *tmv1beta1.Testrun, namespace string, maxWaitTime int64) (*tmv1beta1.Testrun, error) { var foundTestrun *tmv1beta1.Testrun var testrunPhase argov1.NodePhase startTime := time.Now() @@ -107,7 +105,7 @@ func WatchTestrun(tmClient *tmclientset.Clientset, tr *tmv1beta1.Testrun, namesp return nil, fmt.Errorf("Maximum wait time exceeded") } - foundTestrun, err = tmClient.Testmachinery().Testruns(namespace).Get(tr.Name, metav1.GetOptions{}) + err = tmClient.Client().Get(ctx, client.ObjectKey{Namespace: tr.Namespace, Name: tr.Name}, foundTestrun) if err != nil { return nil, err } @@ -120,12 +118,12 @@ func WatchTestrun(tmClient *tmclientset.Clientset, tr *tmv1beta1.Testrun, namesp } // DeleteTestrun deletes a testrun and expects to be successful. -func DeleteTestrun(tmClient *tmclientset.Clientset, tr *tmv1beta1.Testrun) { +func DeleteTestrun(tmClient kubernetes.Interface, tr *tmv1beta1.Testrun) { // wf is not deleted if testrun is triggered but deleted before wf can be deployed. // Strange timing in validation test with kubeconfig. // needs further investigation time.Sleep(3 * time.Second) - err := tmClient.Testmachinery().Testruns(tr.Namespace).Delete(tr.Name, &metav1.DeleteOptions{}) + err := tmClient.Client().Delete(context.TODO(), tr) if !apierrors.IsNotFound(err) { Expect(err).To(BeNil(), "Error deleting Testrun: %s", tr.Name) } diff --git a/test/validationwebhook/testrun_validation_test.go b/test/validationwebhook/testrun_validation_test.go index b55e78612d..598da39f03 100644 --- a/test/validationwebhook/testrun_validation_test.go +++ b/test/validationwebhook/testrun_validation_test.go @@ -15,21 +15,21 @@ package validationwebhook_test import ( + "context" "os" + "github.com/gardener/test-infra/pkg/testmachinery" + "github.com/gardener/test-infra/test/utils" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/gardener/gardener/pkg/client/kubernetes" tmv1beta1 "github.com/gardener/test-infra/pkg/apis/testmachinery/v1beta1" - tmclientset "github.com/gardener/test-infra/pkg/client/testmachinery/clientset/versioned" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/gardener/test-infra/test/resources" - "k8s.io/client-go/tools/clientcmd" ) var _ = Describe("Testrun validation tests", func() { @@ -37,7 +37,7 @@ var _ = Describe("Testrun validation tests", func() { var ( commitSha string namespace string - tmClient *tmclientset.Clientset + tmClient kubernetes.Interface maxWaitTime int64 = 300 ) @@ -47,19 +47,18 @@ var _ = Describe("Testrun validation tests", func() { tmKubeconfig := os.Getenv("TM_KUBECONFIG_PATH") namespace = os.Getenv("TM_NAMESPACE") - tmConfig, err := clientcmd.BuildConfigFromFlags("", tmKubeconfig) - Expect(err).ToNot(HaveOccurred(), "couldn't create k8s client from kubeconfig filepath %s", tmKubeconfig) - - tmClient = tmclientset.NewForConfigOrDie(tmConfig) - - clusterClient, err := kubernetes.NewClientFromFile(tmKubeconfig, nil, client.Options{}) + tmClient, err := kubernetes.NewClientFromFile(tmKubeconfig, nil, client.Options{ + Scheme: testmachinery.TestMachineryScheme, + }) Expect(err).ToNot(HaveOccurred()) - utils.WaitForClusterReadiness(clusterClient, namespace, maxWaitTime) + utils.WaitForClusterReadiness(tmClient, namespace, maxWaitTime) }) Context("Metadata", func() { It("should reject when name contains '.'", func() { + ctx := context.Background() + defer ctx.Done() tr := resources.GetBasicTestrun(namespace, commitSha) tr.Spec.TestFlow = [][]tmv1beta1.TestflowStep{ { @@ -68,7 +67,7 @@ var _ = Describe("Testrun validation tests", func() { }, }, } - _, err := tmClient.Testmachinery().Testruns(namespace).Create(tr) + err := tmClient.Client().Create(ctx, tr) if err == nil { defer utils.DeleteTestrun(tmClient, tr) } @@ -79,11 +78,12 @@ var _ = Describe("Testrun validation tests", func() { Context("TestLocations", func() { It("should reject when no locations are defined", func() { - + ctx := context.Background() + defer ctx.Done() tr := resources.GetBasicTestrun(namespace, commitSha) tr.Spec.TestLocations = []tmv1beta1.TestLocation{} - _, err := tmClient.Testmachinery().Testruns(namespace).Create(tr) + err := tmClient.Client().Create(ctx, tr) if err == nil { defer utils.DeleteTestrun(tmClient, tr) } @@ -92,13 +92,14 @@ var _ = Describe("Testrun validation tests", func() { }) It("should reject when a local location is defined", func() { - + ctx := context.Background() + defer ctx.Done() tr := resources.GetBasicTestrun(namespace, commitSha) tr.Spec.TestLocations = append(tr.Spec.TestLocations, tmv1beta1.TestLocation{ Type: tmv1beta1.LocationTypeLocal, }) - tr, err := tmClient.Testmachinery().Testruns(namespace).Create(tr) + err := tmClient.Client().Create(ctx, tr) if err == nil { defer utils.DeleteTestrun(tmClient, tr) } @@ -109,11 +110,12 @@ var _ = Describe("Testrun validation tests", func() { Context("Testflow", func() { It("should reject when no locations can be found", func() { - + ctx := context.Background() + defer ctx.Done() tr := resources.GetBasicTestrun(namespace, commitSha) tr.Spec.TestFlow = [][]tmv1beta1.TestflowStep{} - tr, err := tmClient.Testmachinery().Testruns(namespace).Create(tr) + err := tmClient.Client().Create(ctx, tr) if err == nil { defer utils.DeleteTestrun(tmClient, tr) } @@ -122,7 +124,8 @@ var _ = Describe("Testrun validation tests", func() { }) It("should reject when a no locations for only one label in the testrun can be found", func() { - + ctx := context.Background() + defer ctx.Done() tr := resources.GetBasicTestrun(namespace, commitSha) tr.Spec.TestFlow = [][]tmv1beta1.TestflowStep{ { @@ -132,7 +135,7 @@ var _ = Describe("Testrun validation tests", func() { }, } - tr, err := tmClient.Testmachinery().Testruns(namespace).Create(tr) + err := tmClient.Client().Create(ctx, tr) if err == nil { defer utils.DeleteTestrun(tmClient, tr) } @@ -143,11 +146,12 @@ var _ = Describe("Testrun validation tests", func() { Context("Kubeconfigs", func() { It("should reject when a invalid kubeconfig is provided", func() { - + ctx := context.Background() + defer ctx.Done() tr := resources.GetBasicTestrun(namespace, commitSha) tr.Spec.Kubeconfigs.Gardener = "dGVzdGluZwo=" - tr, err := tmClient.Testmachinery().Testruns(namespace).Create(tr) + err := tmClient.Client().Create(ctx, tr) if err == nil { defer utils.DeleteTestrun(tmClient, tr) } @@ -158,15 +162,16 @@ var _ = Describe("Testrun validation tests", func() { Context("OnExit", func() { It("should accept when no steps are defined", func() { - + ctx := context.Background() + defer ctx.Done() tr := resources.GetBasicTestrun(namespace, commitSha) - tr, err := tmClient.Testmachinery().Testruns(namespace).Create(tr) + err := tmClient.Client().Create(ctx, tr) defer utils.DeleteTestrun(tmClient, tr) Expect(err).ToNot(HaveOccurred()) - _, err = tmClient.Testmachinery().Testruns(namespace).Get(tr.Name, metav1.GetOptions{}) + err = tmClient.Client().Get(ctx, client.ObjectKey{Namespace: namespace, Name: tr.Name}, nil) Expect(err).ToNot(HaveOccurred()) })