Introduce command line flags: kubeconfig, k8sVersion, cloudprovider, …

…flakeAttempts, testcasegroup.

Fix: setting up test utilities
Fix: switching to antoher k8sRelese

test/e2etest/config/config.go

@@ -40,13 +40,19 @@ var (
 	GardenerVersion          string
 	RetestFlaggedOnly        bool
 	TestcaseGroup            []string
+	TestcaseGroupString      string
 )
 
 var WORKING_DESC_FILE = "working.json"
 var Debug bool
 
 func init() {
 	flag.BoolVar(&Debug, "debug", false, "Run e2etest in debug mode")
+	flag.StringVar(&ShootKubeconfigPath, "kubeconfig", "", "Kubeconfig file path of cluster to test")
+	flag.StringVar(&K8sRelease, "k8sVersion", "", "Kubernetes release version e.g. 1.14.0")
+	flag.StringVar(&CloudProvider, "cloudprovider", "", "Cluster cloud provider (aws, gcp, azure, alicloud, openstack)")
+	flag.IntVar(&FlakeAttempts, "flakeAttemps", 2, "Testcase flake attempts. Will run testcase n times, until it is successful")
+	flag.StringVar(&TestcaseGroupString, "testcasegroup", "", "Testcase groups to run (conformance, fast, slow")
 	flag.Parse()
 	if Debug {
 		log.SetLevel(log.DebugLevel)
@@ -73,22 +79,32 @@ func init() {
 	K8sRoot = filepath.Join(GoPath, "src/k8s.io")
 	KubernetesPath = filepath.Join(K8sRoot, "kubernetes")
 	TestInfraPath = filepath.Join(K8sRoot, "test-infra")
-	ShootKubeconfigPath = tiutil.Getenv("E2E_KUBECONFIG_PATH", filepath.Join(ExportPath, "shoot.config"))
+	if ShootKubeconfigPath == "" {
+		ShootKubeconfigPath = tiutil.Getenv("E2E_KUBECONFIG_PATH", filepath.Join(ExportPath, "shoot.config"))
+	}
 	if _, err := os.Stat(ShootKubeconfigPath); err != nil {
 		log.Fatal(errors.Wrapf(err, "file %s does not exist: ", ShootKubeconfigPath))
 	}
 	GinkgoParallel = tiutil.GetenvBool("GINKGO_PARALLEL", true)
 	DescriptionFile = tiutil.Getenv("DESCRIPTION_FILE", WORKING_DESC_FILE)
-	K8sRelease = os.Getenv("K8S_VERSION")
+	if K8sRelease == "" {
+		K8sRelease = os.Getenv("K8S_VERSION")
+	}
 	if K8sRelease == "" {
 		log.Fatal("K8S_VERSION environment variable not found")
 	}
-	TestcaseGroup = strings.Split(os.Getenv("TESTCASE_GROUPS"), ",")
+	if TestcaseGroupString == "" {
+		TestcaseGroup = strings.Split(os.Getenv("TESTCASE_GROUPS"), ",")
+	} else {
+		TestcaseGroup = strings.Split(TestcaseGroupString, ",")
+	}
 	sort.Strings(TestcaseGroup)
 	if len(TestcaseGroup) == 0 {
 		log.Fatal("TESTCASE_GROUP environment variable not found")
 	}
-	CloudProvider = os.Getenv("CLOUDPROVIDER")
+	if CloudProvider == "" {
+		CloudProvider = os.Getenv("CLOUDPROVIDER")
+	}
 	if CloudProvider == "" {
 		log.Fatal("CLOUDPROVIDER environment variable not found")
 	}
@@ -100,7 +116,9 @@ func init() {
 	if _, err := os.Stat(DescriptionFilePath); err != nil {
 		log.Fatal(errors.Wrapf(err, "file %s does not exist: ", DescriptionFilePath))
 	}
-	FlakeAttempts, _ = strconv.Atoi(tiutil.Getenv("FLAKE_ATTEMPTS", "2"))
+	if FlakeAttempts == 0 || FlakeAttempts == 2 {
+		FlakeAttempts, _ = strconv.Atoi(tiutil.Getenv("FLAKE_ATTEMPTS", "2"))
+	}
 	PublishResultsToTestgrid = tiutil.GetenvBool("PUBLISH_RESULTS_TO_TESTGRID", false)
 	IgnoreSkipList = tiutil.GetenvBool("IGNORE_SKIP_LIST", false)
 	RetestFlaggedOnly = tiutil.GetenvBool("RETEST_FLAGGED_ONLY", false)

test/e2etest/kubetest/setup/setup.go

@@ -22,9 +22,11 @@ var k8sOutputBinDir string = filepath.Join(config.KubernetesPath, "_output/bin")
 func Setup() error {
 	cleanUpPreviousRuns()
 	if areTestUtilitiesReady() {
-		// nothing to do here
+		log.Info("all test utilities were already ready")
 		return nil
 	}
+
+	log.Info("test utilities are not ready. Install...")
 	if err := downloadKubernetes(config.K8sRelease); err != nil {
 		return err
 	}
@@ -61,15 +63,22 @@ func areTestUtilitiesReady() bool {
 	}
 	e2eTestPath := path.Join(k8sOutputBinDir, "e2e.test")
 	if _, err := os.Stat(e2eTestPath); os.IsNotExist(err) {
-		log.Infof("test utility not ready: %s", e2eTestPath)
+		log.Infof("test utility not ready: %s doesn't exist", e2eTestPath)
 		return false // path does not exist
 	}
 	ginkgoPath := path.Join(k8sOutputBinDir, "ginkgo")
 	if _, err := os.Stat(ginkgoPath); os.IsNotExist(err) {
-		log.Infof("test utility not ready: %s", ginkgoPath)
+		log.Infof("test utility not ready: %s doesn't exist", ginkgoPath)
 		return false // path does not exist
 	}
-	log.Info("all test utilities were already ready")
+	if out, err := util.RunCmd("git describe", config.KubernetesPath); err != nil {
+		log.Errorf("failed to run 'git describe' in %s", config.KubernetesPath, err)
+		return false
+	} else if strings.TrimSpace(out.StdOut) != fmt.Sprintf("v%s", config.K8sRelease) {
+		log.Infof("test utility not ready: current k8s release version is %s, but the requested version is v%s", out.StdOut, config.K8sRelease)
+		return false
+	}
+
 	return true
 }
 
@@ -87,15 +96,14 @@ func downloadKubernetes(k8sVersion string) error {
 		}
 	} else if os.IsNotExist(err) {
 		log.Infof("directory %s does not exist. Run go get -d k8s.io/kubernetes", config.KubernetesPath)
-		if stderr, err := util.RunCmd("go get -d k8s.io/kubernetes", ""); err != nil && !isNoGoFilesErr(stderr) {
+		if out, err := util.RunCmd("go get -d k8s.io/kubernetes", ""); err != nil && !isNoGoFilesErr(out.StdErr) {
 			log.Error("failed to go get k8s.io/kubernetes", err)
 			return err
 		}
 	} else {
 		return err
 	}
 
-
 	if _, err := util.RunCmd(fmt.Sprintf("git checkout v%s", k8sVersion), config.KubernetesPath); err != nil {
 		return err
 	}
@@ -107,7 +115,10 @@ func compileOrGetTestUtilities(k8sVersion string) error {
 	resp, err := http.Get(k8sTestBinariesVersionURL)
 
 	if err != nil || resp.StatusCode != http.StatusOK || (runtime.GOOS != "linux" && runtime.GOOS != "darwin") {
-		log.Info("no precompiled kubernetes test binaries available, or operating system is not linux/darwin, build e2e.test and ginko")
+		log.Info("no precompiled kubernetes test binaries available, or operating system is not linux/darwin, build e2e.test and ginkgo")
+		if err := os.RemoveAll(k8sOutputBinDir); err != nil {
+			return err
+		}
 		if _, err = util.RunCmd("make WHAT=test/e2e/e2e.test", config.KubernetesPath); err != nil {
 			return err
 		}
@@ -147,7 +158,7 @@ func compileOrGetTestUtilities(k8sVersion string) error {
 	}
 
 	log.Info("get k8s examples")
-	if stderr, err := util.RunCmd("go get -u k8s.io/examples", ""); err != nil && !isNoGoFilesErr(stderr) {
+	if out, err := util.RunCmd("go get -u k8s.io/examples", ""); err != nil && !isNoGoFilesErr(out.StdErr) {
 		return err
 	}
 	return nil

test/e2etest/main.go

@@ -8,7 +8,6 @@ import (
 	log "github.com/sirupsen/logrus"
 )
 
-
 func main() {
 	if err := setup.Setup(); err != nil {
 		log.Fatal(errors.Wrapf(err, "Initial setup invocation failed"))
@@ -19,4 +18,4 @@ func main() {
 	if config.PublishResultsToTestgrid == true && resultSummary.TestsuiteSuccessful == true {
 		kubetest.Publish(config.ExportPath, resultSummary)
 	}
-}
+}

test/e2etest/util/util.go

@@ -32,7 +32,7 @@ func DownloadFile(url, dir string) (filePath string, err error) {
 	return filePath, nil
 }
 
-func RunCmd(command, execPath string) (stderrString string, err error) {
+func RunCmd(command, execPath string) (output CmdOutput, err error) {
 	separator := " "
 	parts := strings.Split(command, separator)
 
@@ -49,18 +49,22 @@ func RunCmd(command, execPath string) (stderrString string, err error) {
 	cmd.Stdout = &out
 	cmd.Stderr = &stderr
 
+	log.Infof("run command: '%s'", command)
 	err = cmd.Run()
 
 	//	Output our results
 	if out.String() != "" {
-		log.Info(out.String())
+		stdoutString := out.String()
+		output.StdOut = stdoutString
+		log.Info(stdoutString)
 	}
-	stderrString = stderr.String()
 	if stderr.Len() != 0 {
+		stderrString := stderr.String()
+		output.StdErr = stderrString
 		log.Error(stderrString)
 	}
 
-	return stderrString, err
+	return output, err
 }
 
 // Contains tells whether a contains x.
@@ -99,3 +103,8 @@ func GetFilesByPattern(rootDir, filenamePattern string) []string {
 	}
 	return files
 }
+
+type CmdOutput struct {
+	StdOut string
+	StdErr string
+}