Add support for testcase groups. Merge test results of junit.xml file…

…s and e2e.log files.

.test-defs/Testgrid.yaml → .test-defs/allE2eTestgrid.yaml

@@ -27,7 +27,7 @@ spec:
 
   config:
   - type: env
-    name: PUBLISH_TO_TESTGRID
+    name: PUBLISH_RESULTS_TO_TESTGRID
     value: "true"
   - type: file
     name: GCLOUD_ACCOUNT_SECRET
@@ -36,7 +36,13 @@ spec:
       secretKeyRef:
         name: testgrid-gcs-secret
         key: gcloud.json
+  - type: env
+    name: TESTCASE_GROUPS
+    value: 'fast,slow'
+  - type: env
+    name: DESCRIPTION_FILE
+    value: "working.json"
 
   command: [bash, -c]
-  args: ["TESTDESCRIPTION_NAME=conformance.desc ./test/k8s-e2e/e2e-test-execute"]
+  args: go run $GOPATH/src/github.com/gardener/test-infra/test/e2etest
   image: eu.gcr.io/gardener-project/gardener/testmachinery/base-step

.test-defs/conformanceTestgrid.yaml

@@ -0,0 +1,51 @@
+# Copyright 2019 Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+kind: TestDefinition
+metadata:
+  name: testgrid
+spec:
+  owner: DL_5C5BE3E2970B9F404D0E2F50@sap.com
+  recipientsOnFailure:
+  - DL_5C5BE3E2970B9F404D0E2F50@sap.com
+
+  description: Run kubernetes conformance tests and push result files (e2e.log and junit_01.xml) to testgrid repository.
+
+  activeDeadlineSeconds: 10800
+  behavior: ["serial"]
+
+  config:
+  - type: env
+    name: PUBLISH_RESULTS_TO_TESTGRID
+    value: "true"
+  - type: file
+    name: GCLOUD_ACCOUNT_SECRET
+    path: /tmp/secrets/gardener-logs-conformance-tests.json
+    valueFrom:
+      secretKeyRef:
+        name: testgrid-gcs-secret
+        key: gcloud.json
+  - type: env
+    name: TESTCASE_GROUPS
+    value: 'conformance'
+  - type: env
+    name: GINKGO_PARALLEL
+    value: "false"
+  - type: env
+    name: DESCRIPTION_FILE
+    value: "working.json"
+
+  command: [bash, -c]
+  args: go run $GOPATH/src/github.com/gardener/test-infra/test/e2etest
+  image: eu.gcr.io/gardener-project/gardener/testmachinery/base-step

.test-defs/e2eFast.yaml

@@ -16,9 +16,9 @@ kind: TestDefinition
 metadata:
   name: e2e-fast
 spec:
-  owner: DL_5C5BE3E2970B9F404D0E2F50@sap.com
+  owner: gardener-oq@listserv.sap.com
   recipientsOnFailure:
-  - DL_5C5BE3E2970B9F404D0E2F50@sap.com
+  - gardener-oq@listserv.sap.com
 
   description: Run fast kubernetes e2e tests.
 
@@ -27,12 +27,12 @@ spec:
 
   config:
   - type: env
-    name: CONFORMANCE_SKIP
-    value: 'Serial\|Slow\|\[k8s.io\]\sProbing\scontainer\sshould\sbe\srestarted\swith\sa\s\/healthz\shttp\sliveness\sprobe\s\[NodeConformance\]\s\[Conformance\]\|\[k8s.io\]\sProbing\scontainer\sshould\s\*not\*\sbe\srestarted\swith\sa\s\/healthz\shttp\sliveness\sprobe\s\[NodeConformance\]\s\[Conformance\]\|\[k8s.io\]\sProbing\scontainer\sshould\shave\smonotonically\sincreasing\srestart\scount\s\[NodeConformance\]\s\[Conformance\]\|\[k8s.io\]\sBasic\sStatefulSet\sfunctionality\s\[StatefulSetBasic\]\sBurst\sscaling\sshould\srun\sto\scompletion\seven\swith\sunhealthy\spods\s\[Conformance\]\|\[k8s.io\]\sBasic\sStatefulSet\sfunctionality\s\[StatefulSetBasic\]\sScaling\sshould\shappen\sin\spredictable\sorder\sand\shalt\sif\sany\sstateful\spod\sis\sunhealthy\s\[Conformance\]\|\[k8s.io\]\sBasic\sStatefulSet\sfunctionality\s\[StatefulSetBasic\]\sshould\sperform\scanary\supdates\sand\sphased\srolling\supdates\sof\stemplate\smodifications\s\[Conformance\]\|\[k8s.io\]\sBasic\sStatefulSet\sfunctionality\s\[StatefulSetBasic\]\sshould\sperform\srolling\supdates\sand\sroll\sbacks\sof\stemplate\smodifications\s\[Conformance\]\|\[k8s.io\]\sGuestbook\sapplication\sshould\screate\sand\sstop\sa\sworking\sapplication\s\s\[Conformance\]'
+    name: TESTCASE_GROUPS
+    value: 'fast'
   - type: env
-    name: GINKGO_PARALLEL
-    value: "true"
+    name: DESCRIPTION_FILE
+    value: "working.json"
 
   command: [bash, -c]
-  args: ["TESTDESCRIPTION_NAME=e2e_base_fast.desc ./test/k8s-e2e/e2e-test-execute"]
+  args: go run $GOPATH/src/github.com/gardener/test-infra/test/e2etest
   image: eu.gcr.io/gardener-project/gardener/testmachinery/base-step

.test-defs/e2eSlow.yaml

@@ -16,9 +16,9 @@ kind: TestDefinition
 metadata:
   name: e2e-slow
 spec:
-  owner: DL_5C5BE3E2970B9F404D0E2F50@sap.com
+  owner: gardener-oq@listserv.sap.com
   recipientsOnFailure:
-  - DL_5C5BE3E2970B9F404D0E2F50@sap.com
+  - gardener-oq@listserv.sap.com
 
   description: Run kubernetes e2e test with tags Slow and Serial.
 
@@ -27,13 +27,13 @@ spec:
   behavior: ["serial"]
 
   config:
-  - type: env
-    name: CONFORMANCE_FOCUS
-    value: 'Serial\|Slow'
-  - type: env
-    name: GINKGO_PARALLEL
-    value: "true"
+    - type: env
+      name: TESTCASE_GROUPS
+      value: 'slow'
+    - type: env
+      name: DESCRIPTION_FILE
+      value: "working.json"
 
   command: [bash, -c]
-  args: ["TESTDESCRIPTION_NAME=e2e_base_slow.desc ./test/k8s-e2e/e2e-test-execute"]
+  args: go run $GOPATH/src/github.com/gardener/test-infra/test/e2etest
   image: eu.gcr.io/gardener-project/gardener/testmachinery/base-step

test/e2etest/README.md

@@ -0,0 +1,59 @@
+# E2E Test Runner
+
+The e2e test runner leverages kubetest to execute e2e tests and has a few additional features:
+
+- Define description files consisting of e2e testcases to run
+- Annotate testcases to run only for dedicated cloud providers
+- Evaluate test results of the kubetest run and provide elastic search documents
+
+## Usage
+
+Ensure all required environment variables have been set. Create a `shoot.config` file in `EXPORT_PATH` directory and paste the kubeconfig of the kubernetes cluster to test in it. Run `e2etest` in command line to execute the e2e tests.
+
+### Prerequisites:
+
+- Go installed
+- Git installed
+- (only for publishing results) environment variable `GCLOUD_ACCOUNT_SECRET` should point to a google cloud storage secret file
+
+### Parameters:
+
+| Environment Variable  | Default | Description  |
+|---|---|---|
+| DESCRIPTION_FILE |  | **[Required]** Path to description json file, which lists the testcases to run |
+| K8S_VERSION |  | **[Required]** Kubernetes cluster version |
+| TESTCASE_GROUPS |  | **[Required]** testcases groups to run (comma separated). E.g. `fast,slow` |
+| CLOUDPROVIDER |  | **[Required]** Cloud provider (supported: aws, gcp, azure, alicloud, openstack) |
+| EXPORT_PATH  | /tmp/e2e/export  | Location of `shoot.config` file and test results |
+| GINKGO_PARALLEL | true | Whether to run kubetest in parallel way. Testcases that consist of the `[Serial] tag are executed serially. |
+| IGNORE_FALSE_POSITIVE_LIST | false | Ignores exclusion of testcases that are listed in `false_positive.json` |
+| IGNORE_SKIP_LIST | false | Ignores exclusion of testcases that are listed in `skip.json`  |
+| INCLUDE_UNTRACKED_TESTS | false | Executes testcases that are not mentioned in description files for given provider and kubernetes release version |
+| FLAKE_ATTEMPTS | 2 | Flake attempts for kubetest: how many time a failed test should be rerun |
+| PUBLISH_RESULTS_TO_TESTGRID | false | Whether to push test results to google cloud storage, for testgrid |
+| RETEST_FLAGGED_ONLY | false | Runs testcases with retest flag only. Value of `DESCRIPTION_FILE` is ignored |
+
+### Description Files
+Example:
+```json
+[
+  { "testcase": "[k8s.io] Sysctls [NodeFeature:Sysctls] should reject invalid sysctls", "groups": ["slow", "conformance"], "only": ["aws", "gcp"], "retest": ["aws"], "comment": "Some comment"},
+  { "testcase": "[k8s.io] Sysctls [NodeFeature:Sysctls] should support sysctls", "groups": ["slow"], "exclude": ["aws"]}
+]
+```
+| Field  | Description  |
+|---|---|
+| testcase | testcase name. Can be a substring. All testcases that has this as substring will be executed |
+| groups | assigns the testcase to testcase groups |
+| only | will consider the testcase only for given cloud provider |
+| exclude | will not consider the tetscase for given cloud provider |
+| comment | is not evaluated in any way in code. Use only for additional information |
+| retest | testcase will be excluded from all general test runs for given providers. Testcases with retest flag can be executed by setting `RETEST_ONLY=true`  |
+
+Existing description files:
+- `working.json` consists of all working e2e testcases separated in different groups
+- `skip.json` consists of testcases that are always skipped by kubetest due to reasons like: driver not supported, requires >1 nodes, etc.
+- `false_positive.json` consists of testcases that are failing because of different reasons like bad code, which makes sense to test with next kubernetes release version
+
+### Output
+You find the kubetest dump results (like e2e.log and junit_*.xml files) in the `/tmp/e2e/artifacts` directory. These artifacts are evaluated and stored as *.json files in the `EXPORT_PATH` directory.

test/e2etest/config/config.go

@@ -9,7 +9,9 @@ import (
 	"path/filepath"
 	"regexp"
 	"runtime"
+	"sort"
 	"strconv"
+	"strings"
 )
 
 var (
@@ -23,8 +25,6 @@ var (
 	TmpDir                   string
 	ShootKubeconfigPath      string
 	GinkgoParallel           bool
-	ConformanceTestsSkip     string
-	ConformanceTestsFocus    string
 	DescriptionFile          string
 	K8sRelease               string
 	CloudProvider            string
@@ -38,8 +38,11 @@ var (
 	K8sReleaseMajorMinor     string
 	GardenerVersion          string
 	RetestFlaggedOnly        bool
+	TestcaseGroup            []string
 )
 
+var WORKING_DESC_FILE = "working.json"
+
 func init() {
 	//log.SetLevel(log.DebugLevel)
 
@@ -69,13 +72,16 @@ func init() {
 		log.Fatal(errors.Wrapf(err, "file %s does not exist: ", ShootKubeconfigPath))
 	}
 	GinkgoParallel, _ = strconv.ParseBool(util.GetEnv("GINKGO_PARALLEL", "true"))
-	ConformanceTestsSkip = os.Getenv("CONFORMANCE_SKIP")
-	ConformanceTestsFocus = os.Getenv("CONFORMANCE_FOCUS")
-	DescriptionFile = os.Getenv("DESCRIPTION_FILE")
+	DescriptionFile = util.GetEnv("DESCRIPTION_FILE", WORKING_DESC_FILE)
 	K8sRelease = os.Getenv("K8S_VERSION")
 	if K8sRelease == "" {
 		log.Fatal("K8S_VERSION environment variable not found")
 	}
+	TestcaseGroup = strings.Split(os.Getenv("TESTCASE_GROUPS"), ",")
+	sort.Strings(TestcaseGroup)
+	if len(TestcaseGroup) == 0 {
+		log.Fatal("TESTCASE_GROUP environment variable not found")
+	}
 	CloudProvider = os.Getenv("CLOUDPROVIDER")
 	if CloudProvider == "" {
 		log.Fatal("CLOUDPROVIDER environment variable not found")
@@ -85,6 +91,9 @@ func init() {
 	K8sReleaseMajorMinor = string(regexp.MustCompile(`^(\d+\.\d+)`).FindSubmatch([]byte(K8sRelease))[1])
 	DescriptionsPath = path.Join(OwnDir, "kubetest", "description", K8sReleaseMajorMinor)
 	DescriptionFilePath = path.Join(DescriptionsPath, DescriptionFile)
+	if _, err := os.Stat(DescriptionFilePath); err != nil {
+		log.Fatal(errors.Wrapf(err, "file %s does not exist: ", DescriptionFilePath))
+	}
 	FlakeAttempts, _ = strconv.Atoi(util.GetEnv("FLAKE_ATTEMPTS", "2"))
 	PublishResultsToTestgrid, _ = strconv.ParseBool(util.GetEnv("PUBLISH_RESULTS_TO_TESTGRID", "false"))
 	IgnoreSkipList, _ = strconv.ParseBool(util.GetEnv("IGNORE_SKIP_LIST", "false"))
@@ -99,8 +108,6 @@ func init() {
 	log.Debugf("TestInfraPath: %s", TestInfraPath)
 	log.Debugf("ShootKubeconfigPath: %s", ShootKubeconfigPath)
 	log.Debugf("GinkgoParallel: %t", GinkgoParallel)
-	log.Debugf("ConformanceTestsSkip: %s", ConformanceTestsSkip)
-	log.Debugf("ConformanceTestsFocus: %s", ConformanceTestsFocus)
 	log.Debugf("K8sRelease: %s", K8sRelease)
 	log.Debugf("CloudProvider: %s", CloudProvider)
 	log.Debugf("IgnoreFalsePositiveList: %t", IgnoreFalsePositiveList)
@@ -112,4 +119,5 @@ func init() {
 	log.Debugf("FlakeAttempts: %o", FlakeAttempts)
 	log.Debugf("GardenerVersion: %o", GardenerVersion)
 	log.Debugf("RetestFlaggedOnly: %o", RetestFlaggedOnly)
+	log.Debugf("TestcaseGroup: %o", TestcaseGroup)
 }