Merge pull request #248 from embano1/issue-194

fix: Label handling

examples/skip_flags/README.md

@@ -1,11 +1,14 @@
 
 # CLI flags
 
-The test harness framework supports several CLI flags that can be used to influence how tests are executed. This example shows how to create tests that are configured using the CLI flags.
+The test harness framework supports several CLI flags that can be used to influence how tests are executed. This example
+shows how to run or skip specific tests (features) using `--skip-features`, `--labels`, and `--skip-labels`. See
+[examples/flags](../flags/README.md) for a full list of supported flags.
 
 ## Configure tests with CLI flags
 
-To drive your tests with CLI flags, you must initialize a test environment using the passed in CLI flags. This is done by calling `envconf.NewFromFlags` function to create configuration for the environment as shown below:
+To drive your tests with CLI flags, you must initialize a test environment using the passed in CLI flags. This is done
+by calling `envconf.NewFromFlags` function to create configuration for the environment as shown below:
 
 ```go
 var test env.Environment
@@ -20,22 +23,17 @@ func TestMain(m *testing.M) {
 }
 ```
 
-### Supported flags
+### Use `Labels` in your tests
 
-There are several supported flags (for more accurate list, see package `pkg/flag`):
-
-* `assess`
-* `features`
-* `labels`
-* `kubeconfig`
-* `namespace`
-* `skip-assessment`
-* `skip-features`
-* `skip-labels`
+To have more fine-grained control over which feature is run/skipped, you can use `WithLabel()` to add labels to your
+tests and use those with the `--labels` and `--skip-labels` CLI flags.
 
 ### Running tests with flags
 
-The tests can be executed using the normal `go test` tools steps. For instance, to pass the flags to your tests, do the followings:
+#### Using `--skip-features`
+
+The tests can be executed using the normal `go test` tools steps. For instance, to skip the `pod list` feature, pass the
+`--skip-features` flag to your tests:
 
 ```shell
 go test -v . -args --skip-features "pod list"
@@ -50,20 +48,46 @@ go test -c -o skipflags.test .
 Then execute the test binary passing the CLI arguments:
 
 ```shell
-./skipflags.test --labels "env=dev"
+./skipflags.test --skip-features "pod list"
+```
+
+#### Using `--labels` and `--skip-labels`
+
+Adding one or more labels to your features using `WithLabels()` gives you more control over which features are executed.
+
+To skip features labeled with `"env"` is `"dev"`:
+
+```shell
+go test -v . -args --skip-labels=env=dev
+```
+
+To only run features labeled with `"type"` `"k8score"`:
+
+```shell
+go test -v . -args --labels=type=k8score
+```
+
+To be even more explicit, `--labels` and `--skip-labels` allows for multiple labels to be passed. For example, to run
+all tests labeled with `"type"` `"k8score"` **and** labeled with `"env"` `"prod"`:
+
+```shell
+go test -v . -args --labels=type=k8score,env=prod
 ```
 
-To skip a particular labeled feature , do the following
+You can combine `--labels` and `--skip-labels` for even more control. For example, to run all features labeled with
+`"type"` `"k8score"` but exclude those labeled with `"env"` is `"dev"`:
 
 ```shell
-./skipflags.test --skip-labels "env=prod"
+go test -v . -args --labels=type=k8score --skip-labels=env=dev
 ```
 
+> **Note**
+> Features without a label or where the specified `--labels` do not exactly match are also excluded from the tests.
+
 ### Skip tests using built in -skip flag in go test 
 
 Go 1.20 introduces the `-skip` flag for `go test` command to skip tests. 
 
-
 Tests can also be skipped based on test function name, feature name and assesment name with `-skip` flag
 
 ```shell
@@ -76,35 +100,35 @@ To skip a test by test function name `TestSkipFlags`, do the following
 go test -v . -skip TestSkipFlags
 ```
 
-
 To skip a feature with name `pod list` within test function `TestSkipFlags`, do the following
 
 ```shell
 go test -v . -skip TestSkipFlags/pod list
 ```
 
-
-To skip a assesment with name `pods from kube-system` within feature `pod list` within test function `TestSkipFlags`,  do the following
+To skip a assesment with name `pods from kube-system` within feature `pod list` within test function `TestSkipFlags`,
+do the following
 
 ```shell
 go test -v . -skip TestSkipFlags/pod list/pods from kube-system
 ``` 
 
 It is not possible to skip features by label name with this option
 
-
 ### Skip tests using both -skip flag and --skip-xxx flags
 
-We can also use the combination of `-skip` flag built in `go test` and `-skip-xxx` flags provided by the e2e-framework to skip multiple tests
-
+We can also use the combination of `-skip` flag built in `go test` and `-skip-xxx` flags provided by the e2e-framework
+to skip multiple tests
 
-To skip a feature `pod list` within test function `TestSkipFlags` and feature `appsv1/deployment` within test function `TestSkipFlags`, do the following
+To skip a feature `pod list` within test function `TestSkipFlags` and feature `appsv1/deployment` within test function
+`TestSkipFlags`, do the following
 
 ```shell
 go test -v . -skip TestSkipFlags/appsv1/deployment -args --skip-features "pod list"
 ```
 
-To skip a particular labeled feature with label `env=prod` and assesment `deployment creation` within feature `appsv1/deployment` within test function `TestSkipFlags`, do the following
+To skip a particular labeled feature with label `env=prod` and assesment `deployment creation` within feature
+`appsv1/deployment` within test function `TestSkipFlags`, do the following
 
 ```shell
 go test -v . -skip TestSkipFlags/appsv1/deployment/deployment_creation -args --skip-labels "env=prod"

examples/skip_flags/k8s_test.go

@@ -29,7 +29,9 @@ import (
 )
 
 func TestSkipFlags(t *testing.T) {
-	podFeature := features.New("pod list").WithLabel("env", "prod").
+	podFeature := features.New("pod list").
+		WithLabel("type", "k8score").
+		WithLabel("env", "prod").
 		Assess("pods from kube-system", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
 			var pods corev1.PodList
 			client, err := cfg.NewClient()
@@ -48,7 +50,9 @@ func TestSkipFlags(t *testing.T) {
 		}).Feature()
 
 	// feature uses pre-generated namespace (see TestMain)
-	depFeature := features.New("appsv1/deployment").WithLabel("env", "dev").
+	depFeature := features.New("appsv1/deployment").
+		WithLabel("type", "k8score").
+		WithLabel("env", "dev").
 		Setup(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
 			// insert a deployment
 			deployment := newDeployment(cfg.Namespace(), "test-deployment", 1)

pkg/env/env.go

@@ -513,16 +513,28 @@ func (e *testEnv) requireProcessing(kind, testName string, requiredRegexp, skipR
 	}
 
 	if labels != nil {
+		// only run a feature if all its label keys and values match those specified
+		// with --labels
+		matches := 0
 		for key, vals := range e.cfg.Labels() {
 			for _, v := range vals {
-				if !labels.Contains(key, v) {
-					skip = true
-					message = fmt.Sprintf(`Skipping feature "%s": unmatched label "%s=%s"`, testName, key, v)
-					return skip, message
+				if labels.Contains(key, v) {
+					matches++
+					break // continue with next key
 				}
 			}
 		}
 
+		if len(e.cfg.Labels()) != matches {
+			skip = true
+			var kvs []string
+			for k, v := range labels {
+				kvs = append(kvs, fmt.Sprintf("%s=%s", k, v)) // prettify output
+			}
+			message = fmt.Sprintf(`Skipping feature "%s": unmatched labels "%s"`, testName, kvs)
+			return skip, message
+		}
+
 		// skip running a feature if labels matches with --skip-labels
 		for key, vals := range e.cfg.SkipLabels() {
 			for _, v := range vals {