From 204abaf2cfebcdee287fcecfd9beaaffc0befafe Mon Sep 17 00:00:00 2001 From: Mattia Lavacca Date: Thu, 28 Mar 2024 14:32:38 +0100 Subject: [PATCH] Experimental conformance promotion to standard (#2868) * chore: experimental conformance promotion Signed-off-by: Mattia Lavacca * file renamed Signed-off-by: Mattia Lavacca * Allow single test to be run Signed-off-by: Mattia Lavacca Co-authored-by: Dave Protasowski * address review comments Signed-off-by: Mattia Lavacca * chore: minor changes in the CLI Signed-off-by: Mattia Lavacca --------- Signed-off-by: Mattia Lavacca Co-authored-by: Dave Protasowski --- Makefile | 7 +- .../{v1alpha1 => v1}/conformancereport.go | 2 +- conformance/apis/{v1alpha1 => v1}/doc.go | 17 +- .../apis/{v1alpha1 => v1}/profilereport.go | 2 +- conformance/apis/{v1alpha1 => v1}/result.go | 2 +- .../apis/{v1alpha1 => v1}/statistics.go | 2 +- conformance/conformance_test.go | 145 ++++-- conformance/experimental_conformance_test.go | 160 ------- conformance/utils/flags/experimental_flags.go | 41 -- conformance/utils/flags/flags.go | 14 + conformance/utils/suite/conformance.go | 104 +++++ conformance/utils/suite/conformance_test.go | 69 +++ conformance/utils/suite/experimental_suite.go | 415 ------------------ .../utils/suite/experimental_suite_test.go | 179 -------- .../{experimental_profiles.go => profiles.go} | 5 +- .../{experimental_reports.go => reports.go} | 36 +- ...mental_reports_test.go => reports_test.go} | 60 +-- conformance/utils/suite/suite.go | 415 ++++++++++++++---- conformance/utils/suite/suite_test.go | 196 +++++++-- 19 files changed, 827 insertions(+), 1044 deletions(-) rename conformance/apis/{v1alpha1 => v1}/conformancereport.go (99%) rename conformance/apis/{v1alpha1 => v1}/doc.go (51%) rename conformance/apis/{v1alpha1 => v1}/profilereport.go (99%) rename conformance/apis/{v1alpha1 => v1}/result.go (98%) rename conformance/apis/{v1alpha1 => v1}/statistics.go (98%) delete mode 100644 conformance/experimental_conformance_test.go delete mode 100644 conformance/utils/flags/experimental_flags.go create mode 100644 conformance/utils/suite/conformance.go create mode 100644 conformance/utils/suite/conformance_test.go delete mode 100644 conformance/utils/suite/experimental_suite.go delete mode 100644 conformance/utils/suite/experimental_suite_test.go rename conformance/utils/suite/{experimental_profiles.go => profiles.go} (96%) rename conformance/utils/suite/{experimental_reports.go => reports.go} (87%) rename conformance/utils/suite/{experimental_reports_test.go => reports_test.go} (66%) diff --git a/Makefile b/Makefile index 7cf7c0bb87..39cd316631 100644 --- a/Makefile +++ b/Makefile @@ -87,12 +87,7 @@ test: .PHONY: conformance conformance: go test ${GO_TEST_FLAGS} -v ./conformance -run TestConformance -args ${CONFORMANCE_FLAGS} - -# Run experimental conformance tests against controller implementation -.PHONY: conformance.experimental -conformance.experimental: - go test ${GO_TEST_FLAGS} -v ./conformance -run TestExperimentalConformance -args ${CONFORMANCE_FLAGS} - + # Install CRD's and example resources to a pre-existing cluster. .PHONY: install install: crd example diff --git a/conformance/apis/v1alpha1/conformancereport.go b/conformance/apis/v1/conformancereport.go similarity index 99% rename from conformance/apis/v1alpha1/conformancereport.go rename to conformance/apis/v1/conformancereport.go index 3a6e4d80ba..159736d248 100644 --- a/conformance/apis/v1alpha1/conformancereport.go +++ b/conformance/apis/v1/conformancereport.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package v1alpha1 +package v1 import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/conformance/apis/v1alpha1/doc.go b/conformance/apis/v1/doc.go similarity index 51% rename from conformance/apis/v1alpha1/doc.go rename to conformance/apis/v1/doc.go index a51862f386..9635118b08 100644 --- a/conformance/apis/v1alpha1/doc.go +++ b/conformance/apis/v1/doc.go @@ -14,23 +14,10 @@ See the License for the specific language governing permissions and limitations under the License. */ -// V1alpha1 includes alpha maturity API types and utilities for creating and +// v1 includes GA maturity API types and utilities for creating and // handling the results of conformance test runs. These types are _only_ // intended for use by the conformance test suite OR external test suites that // are written in Golang and execute the conformance test suite as a Golang // library. -// -// Note that currently all sub-packages are considered "experimental" in that -// they aren't intended for general use or to be distributed as part of a -// release so there is no way to use them by default when using the Golang -// library at this time. If you don't know for sure that you want to use these -// features, then you should not use them. If you would like to opt into these -// unreleased features use Go build tags to enable them, e.g.: -// -// $ go test ./conformance/... -args ${CONFORMANCE_ARGS} -// -// Please note that everything here is considered experimental and subject to -// change. Expect breaking changes and/or complete removals if you start using -// them. -package v1alpha1 +package v1 diff --git a/conformance/apis/v1alpha1/profilereport.go b/conformance/apis/v1/profilereport.go similarity index 99% rename from conformance/apis/v1alpha1/profilereport.go rename to conformance/apis/v1/profilereport.go index 6e377ffddf..fdcaf6d29f 100644 --- a/conformance/apis/v1alpha1/profilereport.go +++ b/conformance/apis/v1/profilereport.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package v1alpha1 +package v1 // ProfileReport is the generated report for the test results of a specific // named conformance profile. diff --git a/conformance/apis/v1alpha1/result.go b/conformance/apis/v1/result.go similarity index 98% rename from conformance/apis/v1alpha1/result.go rename to conformance/apis/v1/result.go index ec7ed44445..f3a826c4a2 100644 --- a/conformance/apis/v1alpha1/result.go +++ b/conformance/apis/v1/result.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package v1alpha1 +package v1 // Result is a simple high-level summary describing the conclusion of a test // run. diff --git a/conformance/apis/v1alpha1/statistics.go b/conformance/apis/v1/statistics.go similarity index 98% rename from conformance/apis/v1alpha1/statistics.go rename to conformance/apis/v1/statistics.go index cc20d3a5e0..40095aa4c7 100644 --- a/conformance/apis/v1alpha1/statistics.go +++ b/conformance/apis/v1/statistics.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package v1alpha1 +package v1 // Statistics includes numerical summaries of the number of conformance tests // that passed, failed or were intentionally skipped. diff --git a/conformance/conformance_test.go b/conformance/conformance_test.go index bbd656347f..e2b069d107 100644 --- a/conformance/conformance_test.go +++ b/conformance/conformance_test.go @@ -14,70 +14,141 @@ See the License for the specific language governing permissions and limitations under the License. */ -// conformance_test contains code to run the conformance tests. This is in its own package to avoid circular imports. package conformance_test import ( + "os" "testing" - v1 "sigs.k8s.io/gateway-api/apis/v1" - "sigs.k8s.io/gateway-api/apis/v1alpha2" - "sigs.k8s.io/gateway-api/apis/v1beta1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/config" + "sigs.k8s.io/yaml" + + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" + gatewayv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" + confv1 "sigs.k8s.io/gateway-api/conformance/apis/v1" "sigs.k8s.io/gateway-api/conformance/tests" "sigs.k8s.io/gateway-api/conformance/utils/flags" "sigs.k8s.io/gateway-api/conformance/utils/suite" +) - "k8s.io/client-go/kubernetes" - _ "k8s.io/client-go/plugin/pkg/client/auth" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/config" +var ( + cfg *rest.Config + k8sClientset *kubernetes.Clientset + mgrClient client.Client + supportedFeatures sets.Set[suite.SupportedFeature] + exemptFeatures sets.Set[suite.SupportedFeature] + namespaceLabels map[string]string + namespaceAnnotations map[string]string + implementation *confv1.Implementation + mode string + allowCRDsMismatch bool + conformanceProfiles sets.Set[suite.ConformanceProfileName] + skipTests []string ) func TestConformance(t *testing.T) { - cfg, err := config.GetConfig() + var err error + cfg, err = config.GetConfig() if err != nil { t.Fatalf("Error loading Kubernetes config: %v", err) } - client, err := client.New(cfg, client.Options{}) + mgrClient, err = client.New(cfg, client.Options{}) if err != nil { t.Fatalf("Error initializing Kubernetes client: %v", err) } - clientset, err := kubernetes.NewForConfig(cfg) + k8sClientset, err = kubernetes.NewForConfig(cfg) if err != nil { t.Fatalf("Error initializing Kubernetes REST client: %v", err) } - v1alpha2.AddToScheme(client.Scheme()) - v1beta1.AddToScheme(client.Scheme()) - v1.AddToScheme(client.Scheme()) + gatewayv1alpha2.AddToScheme(mgrClient.Scheme()) + gatewayv1beta1.AddToScheme(mgrClient.Scheme()) + gatewayv1.AddToScheme(mgrClient.Scheme()) + apiextensionsv1.AddToScheme(mgrClient.Scheme()) + + // conformance flags + supportedFeatures = suite.ParseSupportedFeatures(*flags.SupportedFeatures) + exemptFeatures = suite.ParseSupportedFeatures(*flags.ExemptFeatures) + skipTests = suite.ParseSkipTests(*flags.SkipTests) + namespaceLabels = suite.ParseKeyValuePairs(*flags.NamespaceLabels) + namespaceAnnotations = suite.ParseKeyValuePairs(*flags.NamespaceAnnotations) + conformanceProfiles = suite.ParseConformanceProfiles(*flags.ConformanceProfiles) + if len(conformanceProfiles) == 0 { + t.Fatal("conformance profiles need to be given") + } + mode = *flags.Mode + allowCRDsMismatch = *flags.AllowCRDsMismatch - supportedFeatures := suite.ParseSupportedFeatures(*flags.SupportedFeatures) - exemptFeatures := suite.ParseSupportedFeatures(*flags.ExemptFeatures) - skipTests := suite.ParseSkipTests(*flags.SkipTests) - namespaceLabels := suite.ParseKeyValuePairs(*flags.NamespaceLabels) - namespaceAnnotations := suite.ParseKeyValuePairs(*flags.NamespaceAnnotations) + implementation, err = suite.ParseImplementation( + *flags.ImplementationOrganization, + *flags.ImplementationProject, + *flags.ImplementationURL, + *flags.ImplementationVersion, + *flags.ImplementationContact, + ) + if err != nil { + t.Fatalf("Error parsing implementation's details: %v", err) + } + testConformance(t) +} +func testConformance(t *testing.T) { t.Logf("Running conformance tests with %s GatewayClass\n cleanup: %t\n debug: %t\n enable all features: %t \n supported features: [%v]\n exempt features: [%v]", *flags.GatewayClassName, *flags.CleanupBaseResources, *flags.ShowDebug, *flags.EnableAllSupportedFeatures, *flags.SupportedFeatures, *flags.ExemptFeatures) - cSuite := suite.New(suite.Options{ - Client: client, - RestConfig: cfg, - // This clientset is needed in addition to the client only because - // controller-runtime client doesn't support non CRUD sub-resources yet (https://github.com/kubernetes-sigs/controller-runtime/issues/452). - Clientset: clientset, - GatewayClassName: *flags.GatewayClassName, - Debug: *flags.ShowDebug, - CleanupBaseResources: *flags.CleanupBaseResources, - SupportedFeatures: supportedFeatures, - ExemptFeatures: exemptFeatures, - EnableAllSupportedFeatures: *flags.EnableAllSupportedFeatures, - NamespaceLabels: namespaceLabels, - NamespaceAnnotations: namespaceAnnotations, - SkipTests: skipTests, - RunTest: *flags.RunTest, - }) - cSuite.Setup(t) + cSuite, err := suite.NewConformanceTestSuite( + suite.ConformanceOptions{ + Client: mgrClient, + RestConfig: cfg, + // This clientset is needed in addition to the client only because + // controller-runtime client doesn't support non CRUD sub-resources yet (https://github.com/kubernetes-sigs/controller-runtime/issues/452). + Clientset: k8sClientset, + GatewayClassName: *flags.GatewayClassName, + Debug: *flags.ShowDebug, + CleanupBaseResources: *flags.CleanupBaseResources, + SupportedFeatures: supportedFeatures, + ExemptFeatures: exemptFeatures, + EnableAllSupportedFeatures: *flags.EnableAllSupportedFeatures, + NamespaceLabels: namespaceLabels, + NamespaceAnnotations: namespaceAnnotations, + SkipTests: skipTests, + RunTest: *flags.RunTest, + Mode: mode, + AllowCRDsMismatch: allowCRDsMismatch, + Implementation: *implementation, + ConformanceProfiles: conformanceProfiles, + }) + if err != nil { + t.Fatalf("error creating conformance test suite: %v", err) + } + cSuite.Setup(t) cSuite.Run(t, tests.ConformanceTests) + report, err := cSuite.Report() + if err != nil { + t.Fatalf("error generating conformance profile report: %v", err) + } + writeReport(t.Logf, *report, *flags.ReportOutput) +} + +func writeReport(logf func(string, ...any), report confv1.ConformanceReport, output string) error { + rawReport, err := yaml.Marshal(report) + if err != nil { + return err + } + + if output != "" { + if err = os.WriteFile(output, rawReport, 0o600); err != nil { + return err + } + } + logf("Conformance report:\n%s", string(rawReport)) + + return nil } diff --git a/conformance/experimental_conformance_test.go b/conformance/experimental_conformance_test.go deleted file mode 100644 index bc5a78089e..0000000000 --- a/conformance/experimental_conformance_test.go +++ /dev/null @@ -1,160 +0,0 @@ -/* -Copyright 2023 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package conformance_test - -import ( - "os" - "testing" - - apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/config" - "sigs.k8s.io/yaml" - - gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" - "sigs.k8s.io/gateway-api/apis/v1alpha2" - "sigs.k8s.io/gateway-api/apis/v1beta1" - confv1a1 "sigs.k8s.io/gateway-api/conformance/apis/v1alpha1" - "sigs.k8s.io/gateway-api/conformance/tests" - "sigs.k8s.io/gateway-api/conformance/utils/flags" - "sigs.k8s.io/gateway-api/conformance/utils/suite" -) - -var ( - cfg *rest.Config - k8sClientset *kubernetes.Clientset - mgrClient client.Client - supportedFeatures sets.Set[suite.SupportedFeature] - exemptFeatures sets.Set[suite.SupportedFeature] - namespaceLabels map[string]string - namespaceAnnotations map[string]string - implementation *confv1a1.Implementation - mode string - allowCRDsMismatch bool - conformanceProfiles sets.Set[suite.ConformanceProfileName] - skipTests []string -) - -func TestExperimentalConformance(t *testing.T) { - var err error - cfg, err = config.GetConfig() - if err != nil { - t.Fatalf("Error loading Kubernetes config: %v", err) - } - mgrClient, err = client.New(cfg, client.Options{}) - if err != nil { - t.Fatalf("Error initializing Kubernetes client: %v", err) - } - k8sClientset, err = kubernetes.NewForConfig(cfg) - if err != nil { - t.Fatalf("Error initializing Kubernetes REST client: %v", err) - } - - v1alpha2.AddToScheme(mgrClient.Scheme()) - v1beta1.AddToScheme(mgrClient.Scheme()) - gatewayv1.AddToScheme(mgrClient.Scheme()) - apiextensionsv1.AddToScheme(mgrClient.Scheme()) - - // standard conformance flags - supportedFeatures = suite.ParseSupportedFeatures(*flags.SupportedFeatures) - exemptFeatures = suite.ParseSupportedFeatures(*flags.ExemptFeatures) - skipTests = suite.ParseSkipTests(*flags.SkipTests) - namespaceLabels = suite.ParseKeyValuePairs(*flags.NamespaceLabels) - namespaceAnnotations = suite.ParseKeyValuePairs(*flags.NamespaceAnnotations) - - // experimental conformance flags - conformanceProfiles = suite.ParseConformanceProfiles(*flags.ConformanceProfiles) - mode = *flags.Mode - allowCRDsMismatch = *flags.AllowCRDsMismatch - - if conformanceProfiles.Len() > 0 { - // if some conformance profiles have been set, run the experimental conformance suite... - implementation, err = suite.ParseImplementation( - *flags.ImplementationOrganization, - *flags.ImplementationProject, - *flags.ImplementationURL, - *flags.ImplementationVersion, - *flags.ImplementationContact, - ) - if err != nil { - t.Fatalf("Error parsing implementation's details: %v", err) - } - testExperimentalConformance(t) - } else { - // ...otherwise run the standard conformance suite. - t.Run("standard conformance tests", TestConformance) - } -} - -func testExperimentalConformance(t *testing.T) { - t.Logf("Running experimental conformance tests with %s GatewayClass\n cleanup: %t\n debug: %t\n enable all features: %t \n supported features: [%v]\n exempt features: [%v]", - *flags.GatewayClassName, *flags.CleanupBaseResources, *flags.ShowDebug, *flags.EnableAllSupportedFeatures, *flags.SupportedFeatures, *flags.ExemptFeatures) - - cSuite, err := suite.NewExperimentalConformanceTestSuite( - suite.ExperimentalConformanceOptions{ - Options: suite.Options{ - Client: mgrClient, - RestConfig: cfg, - // This clientset is needed in addition to the client only because - // controller-runtime client doesn't support non CRUD sub-resources yet (https://github.com/kubernetes-sigs/controller-runtime/issues/452). - Clientset: k8sClientset, - GatewayClassName: *flags.GatewayClassName, - Debug: *flags.ShowDebug, - CleanupBaseResources: *flags.CleanupBaseResources, - SupportedFeatures: supportedFeatures, - ExemptFeatures: exemptFeatures, - EnableAllSupportedFeatures: *flags.EnableAllSupportedFeatures, - NamespaceLabels: namespaceLabels, - NamespaceAnnotations: namespaceAnnotations, - SkipTests: skipTests, - }, - Mode: mode, - AllowCRDsMismatch: allowCRDsMismatch, - Implementation: *implementation, - ConformanceProfiles: conformanceProfiles, - }) - if err != nil { - t.Fatalf("error creating experimental conformance test suite: %v", err) - } - - cSuite.Setup(t) - cSuite.Run(t, tests.ConformanceTests) - report, err := cSuite.Report() - if err != nil { - t.Fatalf("error generating conformance profile report: %v", err) - } - writeReport(t.Logf, *report, *flags.ReportOutput) -} - -func writeReport(logf func(string, ...any), report confv1a1.ConformanceReport, output string) error { - rawReport, err := yaml.Marshal(report) - if err != nil { - return err - } - - if output != "" { - if err = os.WriteFile(output, rawReport, 0o600); err != nil { - return err - } - } - logf("Conformance report:\n%s", string(rawReport)) - - return nil -} diff --git a/conformance/utils/flags/experimental_flags.go b/conformance/utils/flags/experimental_flags.go deleted file mode 100644 index 3bf1cae4cd..0000000000 --- a/conformance/utils/flags/experimental_flags.go +++ /dev/null @@ -1,41 +0,0 @@ -/* -Copyright 2023 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// flags contains command-line flag definitions for the conformance -// profile certification. They're in this package so they can be shared -// among the various suites that are all run by the same Makefile invocation. -package flags - -import ( - "flag" -) - -const ( - // DefaultMode is the operating mode to default to in case no mode is specified. - DefaultMode = "default" -) - -var ( - ImplementationOrganization = flag.String("organization", "", "Implementation's Organization to issue conformance to") - ImplementationProject = flag.String("project", "", "Implementation's project to issue conformance to") - ImplementationURL = flag.String("url", "", "Implementation's url to issue conformance to") - ImplementationVersion = flag.String("version", "", "Implementation's version to issue conformance to") - ImplementationContact = flag.String("contact", "", "Comma-separated list of contact information for the maintainers") - Mode = flag.String("mode", DefaultMode, "The operating mode of the implementation.") - AllowCRDsMismatch = flag.Bool("allow-crds-mismatch", false, "Flag to allow the suite not to fail in case there is a mismatch between CRDs versions and channels.") - ConformanceProfiles = flag.String("conformance-profiles", "", "Comma-separated list of the conformance profiles to run") - ReportOutput = flag.String("report-output", "", "The file where to write the conformance report") -) diff --git a/conformance/utils/flags/flags.go b/conformance/utils/flags/flags.go index afdf4ea544..f97eeba044 100644 --- a/conformance/utils/flags/flags.go +++ b/conformance/utils/flags/flags.go @@ -23,6 +23,11 @@ import ( "flag" ) +const ( + // DefaultMode is the operating mode to default to in case no mode is specified. + DefaultMode = "default" +) + var ( GatewayClassName = flag.String("gateway-class", "gateway-conformance", "Name of GatewayClass to use for tests") ShowDebug = flag.Bool("debug", false, "Whether to print debug logs") @@ -34,4 +39,13 @@ var ( EnableAllSupportedFeatures = flag.Bool("all-features", false, "Whether to enable all supported features for conformance tests") NamespaceLabels = flag.String("namespace-labels", "", "Comma-separated list of name=value labels to add to test namespaces") NamespaceAnnotations = flag.String("namespace-annotations", "", "Comma-separated list of name=value annotations to add to test namespaces") + ImplementationOrganization = flag.String("organization", "", "Implementation's Organization") + ImplementationProject = flag.String("project", "", "Implementation's project") + ImplementationURL = flag.String("url", "", "Implementation's url") + ImplementationVersion = flag.String("version", "", "Implementation's version") + ImplementationContact = flag.String("contact", "", "Comma-separated list of contact information for the maintainers") + Mode = flag.String("mode", DefaultMode, "The operating mode of the implementation.") + AllowCRDsMismatch = flag.Bool("allow-crds-mismatch", false, "Flag to allow the suite not to fail in case there is a mismatch between CRDs versions and channels.") + ConformanceProfiles = flag.String("conformance-profiles", "", "Comma-separated list of the conformance profiles to run") + ReportOutput = flag.String("report-output", "", "The file where to write the conformance report") ) diff --git a/conformance/utils/suite/conformance.go b/conformance/utils/suite/conformance.go new file mode 100644 index 0000000000..25fbdfbc9f --- /dev/null +++ b/conformance/utils/suite/conformance.go @@ -0,0 +1,104 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package suite + +import ( + "strings" + "testing" + + "k8s.io/apimachinery/pkg/util/sets" +) + +// ConformanceTest is used to define each individual conformance test. +type ConformanceTest struct { + ShortName string + Description string + Features []SupportedFeature + Manifests []string + Slow bool + Parallel bool + Test func(*testing.T, *ConformanceTestSuite) +} + +// Run runs an individual tests, applying and cleaning up the required manifests +// before calling the Test function. +func (test *ConformanceTest) Run(t *testing.T, suite *ConformanceTestSuite) { + if test.Parallel { + t.Parallel() + } + + // Test against features if the user hasn't focused on a single test + if suite.RunTest == "" { + // Check that all features exercised by the test have been opted into by + // the suite. + for _, feature := range test.Features { + if !suite.SupportedFeatures.Has(feature) { + t.Skipf("Skipping %s: suite does not support %s", test.ShortName, feature) + } + } + } + + // check that the test should not be skipped + if suite.SkipTests.Has(test.ShortName) || suite.RunTest != "" && suite.RunTest != test.ShortName { + t.Skipf("Skipping %s: test explicitly skipped", test.ShortName) + } + + for _, manifestLocation := range test.Manifests { + t.Logf("Applying %s", manifestLocation) + suite.Applier.MustApplyWithCleanup(t, suite.Client, suite.TimeoutConfig, manifestLocation, true) + } + + test.Test(t, suite) +} + +// ParseSupportedFeatures parses flag arguments and converts the string to +// sets.Set[suite.SupportedFeature] +func ParseSupportedFeatures(f string) sets.Set[SupportedFeature] { + if f == "" { + return nil + } + res := sets.Set[SupportedFeature]{} + for _, value := range strings.Split(f, ",") { + res.Insert(SupportedFeature(value)) + } + return res +} + +// ParseKeyValuePairs parses flag arguments and converts the string to +// map[string]string containing label key/value pairs. +func ParseKeyValuePairs(f string) map[string]string { + if f == "" { + return nil + } + res := map[string]string{} + for _, kv := range strings.Split(f, ",") { + parts := strings.Split(kv, "=") + if len(parts) == 2 { + res[parts[0]] = parts[1] + } + } + return res +} + +// ParseSkipTests parses flag arguments and converts the string to +// []string containing the tests to be skipped. +func ParseSkipTests(t string) []string { + if t == "" { + return nil + } + return strings.Split(t, ",") +} diff --git a/conformance/utils/suite/conformance_test.go b/conformance/utils/suite/conformance_test.go new file mode 100644 index 0000000000..ddb73c7c53 --- /dev/null +++ b/conformance/utils/suite/conformance_test.go @@ -0,0 +1,69 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package suite + +import ( + "reflect" + "testing" + + "k8s.io/apimachinery/pkg/util/sets" +) + +func TestParseSupportedFeatures(t *testing.T) { + flags := []string{ + "", + "a", + "b,c,d", + } + + s1 := sets.Set[SupportedFeature]{} + s1.Insert(SupportedFeature("a")) + s2 := sets.Set[SupportedFeature]{} + s2.Insert(SupportedFeature("b")) + s2.Insert(SupportedFeature("c")) + s2.Insert(SupportedFeature("d")) + features := []sets.Set[SupportedFeature]{nil, s1, s2} + + for i, f := range flags { + expect := features[i] + got := ParseSupportedFeatures(f) + if !reflect.DeepEqual(got, expect) { + t.Errorf("Unexpected features from flags '%s', expected: %v, got: %v", f, expect, got) + } + } +} + +func TestParseKeyValuePairs(t *testing.T) { + flags := []string{ + "", + "a=b", + "b=c,d=e,f=g", + } + labels := []map[string]string{ + nil, + {"a": "b"}, + {"b": "c", "d": "e", "f": "g"}, + } + + for i, f := range flags { + expect := labels[i] + got := ParseKeyValuePairs(f) + if !reflect.DeepEqual(got, expect) { + t.Errorf("Unexpected labels from flags '%s', expected: %v, got: %v", f, expect, got) + } + } +} diff --git a/conformance/utils/suite/experimental_suite.go b/conformance/utils/suite/experimental_suite.go deleted file mode 100644 index 20044ff84a..0000000000 --- a/conformance/utils/suite/experimental_suite.go +++ /dev/null @@ -1,415 +0,0 @@ -/* -Copyright 2023 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package suite - -import ( - "context" - "errors" - "fmt" - "sort" - "strings" - "sync" - "testing" - "time" - - apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/sets" - - "sigs.k8s.io/gateway-api/conformance" - confv1a1 "sigs.k8s.io/gateway-api/conformance/apis/v1alpha1" - "sigs.k8s.io/gateway-api/conformance/utils/config" - "sigs.k8s.io/gateway-api/conformance/utils/flags" - "sigs.k8s.io/gateway-api/conformance/utils/kubernetes" - "sigs.k8s.io/gateway-api/conformance/utils/roundtripper" - "sigs.k8s.io/gateway-api/pkg/consts" -) - -// ----------------------------------------------------------------------------- -// Conformance Test Suite - Public Types -// ----------------------------------------------------------------------------- - -// ConformanceTestSuite defines the test suite used to run Gateway API -// conformance tests. -// This is experimental for now and can be used as an alternative to the -// ConformanceTestSuite. Once this won't be experimental any longer, -// the two of them will be merged. -type ExperimentalConformanceTestSuite struct { - ConformanceTestSuite - - // mode is the operating mode of the implementation. - // The default value for it is "default". - mode string - - // implementation contains the details of the implementation, such as - // organization, project, etc. - implementation confv1a1.Implementation - - // apiVersion is the version of the Gateway API installed in the cluster - // and is extracted by the annotation gateway.networking.k8s.io/bundle-version - // in the Gateway API CRDs. - apiVersion string - - // apiChannel is the channel of the Gateway API installed in the cluster - // and is extracted by the annotation gateway.networking.k8s.io/channel - // in the Gateway API CRDs. - apiChannel string - - // conformanceProfiles is a compiled list of profiles to check - // conformance against. - conformanceProfiles sets.Set[ConformanceProfileName] - - // running indicates whether the test suite is currently running - running bool - - // results stores the pass or fail results of each test that was run by - // the test suite, organized by the tests unique name. - results map[string]testResult - - // extendedSupportedFeatures is a compiled list of named features that were - // marked as supported, and is used for reporting the test results. - extendedSupportedFeatures map[ConformanceProfileName]sets.Set[SupportedFeature] - - // extendedUnsupportedFeatures is a compiled list of named features that were - // marked as not supported, and is used for reporting the test results. - extendedUnsupportedFeatures map[ConformanceProfileName]sets.Set[SupportedFeature] - - // lock is a mutex to help ensure thread safety of the test suite object. - lock sync.RWMutex -} - -// Options can be used to initialize a ConformanceTestSuite. -type ExperimentalConformanceOptions struct { - Options - - Mode string - AllowCRDsMismatch bool - Implementation confv1a1.Implementation - ConformanceProfiles sets.Set[ConformanceProfileName] -} - -const ( - undefinedKeyword = "UNDEFINED" -) - -// NewExperimentalConformanceTestSuite is a helper to use for creating a new ExperimentalConformanceTestSuite. -func NewExperimentalConformanceTestSuite(options ExperimentalConformanceOptions) (*ExperimentalConformanceTestSuite, error) { - config.SetupTimeoutConfig(&options.TimeoutConfig) - - roundTripper := options.RoundTripper - if roundTripper == nil { - roundTripper = &roundtripper.DefaultRoundTripper{Debug: options.Debug, TimeoutConfig: options.TimeoutConfig} - } - - installedCRDs := &apiextensionsv1.CustomResourceDefinitionList{} - err := options.Client.List(context.TODO(), installedCRDs) - if err != nil { - return nil, err - } - apiVersion, apiChannel, err := getAPIVersionAndChannel(installedCRDs.Items) - if err != nil { - // in case an error is returned and the AllowCRDsMismatch flag is false, the suite fails. - // This is the default behavior but can be customized in case one wants to experiment - // with mixed versions/channels of the API. - if !options.AllowCRDsMismatch { - return nil, err - } - apiVersion = undefinedKeyword - apiChannel = undefinedKeyword - } - - mode := flags.DefaultMode - if options.Mode != "" { - mode = options.Mode - } - - suite := &ExperimentalConformanceTestSuite{ - results: make(map[string]testResult), - extendedUnsupportedFeatures: make(map[ConformanceProfileName]sets.Set[SupportedFeature]), - extendedSupportedFeatures: make(map[ConformanceProfileName]sets.Set[SupportedFeature]), - conformanceProfiles: options.ConformanceProfiles, - implementation: options.Implementation, - mode: mode, - apiVersion: apiVersion, - apiChannel: apiChannel, - } - - // test suite callers are required to provide a conformance profile OR at - // minimum a list of features which they support. - if options.SupportedFeatures == nil && options.ConformanceProfiles.Len() == 0 && !options.EnableAllSupportedFeatures { - return nil, fmt.Errorf("no conformance profile was selected for test run, and no supported features were provided so no tests could be selected") - } - - // test suite callers can potentially just run all tests by saying they - // cover all features, if they don't they'll need to have provided a - // conformance profile or at least some specific features they support. - if options.EnableAllSupportedFeatures { - options.SupportedFeatures = AllFeatures - } else if options.SupportedFeatures == nil { - options.SupportedFeatures = sets.New[SupportedFeature]() - } - - for _, conformanceProfileName := range options.ConformanceProfiles.UnsortedList() { - conformanceProfile, err := getConformanceProfileForName(conformanceProfileName) - if err != nil { - return nil, fmt.Errorf("failed to retrieve conformance profile: %w", err) - } - // the use of a conformance profile implicitly enables any features of - // that profile which are supported at a Core level of support. - for _, f := range conformanceProfile.CoreFeatures.UnsortedList() { - if !options.SupportedFeatures.Has(f) { - options.SupportedFeatures.Insert(f) - } - } - for _, f := range conformanceProfile.ExtendedFeatures.UnsortedList() { - if options.SupportedFeatures.Has(f) { - if suite.extendedSupportedFeatures[conformanceProfileName] == nil { - suite.extendedSupportedFeatures[conformanceProfileName] = sets.New[SupportedFeature]() - } - suite.extendedSupportedFeatures[conformanceProfileName].Insert(f) - } else { - if suite.extendedUnsupportedFeatures[conformanceProfileName] == nil { - suite.extendedUnsupportedFeatures[conformanceProfileName] = sets.New[SupportedFeature]() - } - suite.extendedUnsupportedFeatures[conformanceProfileName].Insert(f) - } - // Add Exempt Features into unsupported features list - if options.ExemptFeatures.Has(f) { - suite.extendedUnsupportedFeatures[conformanceProfileName].Insert(f) - } - } - } - - for feature := range options.ExemptFeatures { - options.SupportedFeatures.Delete(feature) - } - - if options.FS == nil { - options.FS = &conformance.Manifests - } - - suite.ConformanceTestSuite = ConformanceTestSuite{ - Client: options.Client, - Clientset: options.Clientset, - RestConfig: options.RestConfig, - RoundTripper: roundTripper, - GatewayClassName: options.GatewayClassName, - Debug: options.Debug, - Cleanup: options.CleanupBaseResources, - BaseManifests: options.BaseManifests, - MeshManifests: options.MeshManifests, - Applier: kubernetes.Applier{ - NamespaceLabels: options.NamespaceLabels, - NamespaceAnnotations: options.NamespaceAnnotations, - }, - SupportedFeatures: options.SupportedFeatures, - TimeoutConfig: options.TimeoutConfig, - SkipTests: sets.New(options.SkipTests...), - FS: *options.FS, - UsableNetworkAddresses: options.UsableNetworkAddresses, - UnusableNetworkAddresses: options.UnusableNetworkAddresses, - } - - // apply defaults - if suite.BaseManifests == "" { - suite.BaseManifests = "base/manifests.yaml" - } - if suite.MeshManifests == "" { - suite.MeshManifests = "mesh/manifests.yaml" - } - - return suite, nil -} - -// ----------------------------------------------------------------------------- -// Conformance Test Suite - Public Methods -// ----------------------------------------------------------------------------- - -// Setup ensures the base resources required for conformance tests are installed -// in the cluster. It also ensures that all relevant resources are ready. -func (suite *ExperimentalConformanceTestSuite) Setup(t *testing.T) { - suite.ConformanceTestSuite.Setup(t) -} - -// Run runs the provided set of conformance tests. -func (suite *ExperimentalConformanceTestSuite) Run(t *testing.T, tests []ConformanceTest) error { - // verify that the test suite isn't already running, don't start a new run - // until the previous run finishes - suite.lock.Lock() - if suite.running { - suite.lock.Unlock() - return fmt.Errorf("can't run the test suite multiple times in parallel: the test suite is already running") - } - - // if the test suite is not currently running, reset reporting and start a - // new test run. - suite.running = true - suite.results = nil - suite.lock.Unlock() - - // run all tests and collect the test results for conformance reporting - results := make(map[string]testResult) - for _, test := range tests { - succeeded := t.Run(test.ShortName, func(t *testing.T) { - test.Run(t, &suite.ConformanceTestSuite) - }) - res := testSucceeded - if suite.SkipTests.Has(test.ShortName) { - res = testSkipped - } - if !suite.SupportedFeatures.HasAll(test.Features...) { - res = testNotSupported - } - - if !succeeded { - res = testFailed - } - - results[test.ShortName] = testResult{ - test: test, - result: res, - } - } - - // now that the tests have completed, mark the test suite as not running - // and report the test results. - suite.lock.Lock() - suite.running = false - suite.results = results - suite.lock.Unlock() - - return nil -} - -// Report emits a ConformanceReport for the previously completed test run. -// If no run completed prior to running the report, and error is emitted. -func (suite *ExperimentalConformanceTestSuite) Report() (*confv1a1.ConformanceReport, error) { - suite.lock.RLock() - if suite.running { - suite.lock.RUnlock() - return nil, fmt.Errorf("can't generate report: the test suite is currently running") - } - defer suite.lock.RUnlock() - - testNames := make([]string, 0, len(suite.results)) - for tN := range suite.results { - testNames = append(testNames, tN) - } - sort.Strings(testNames) - profileReports := newReports() - for _, tN := range testNames { - testResult := suite.results[tN] - conformanceProfiles := getConformanceProfilesForTest(testResult.test, suite.conformanceProfiles).UnsortedList() - sort.Slice(conformanceProfiles, func(i, j int) bool { - return conformanceProfiles[i].Name < conformanceProfiles[j].Name - }) - for _, profile := range conformanceProfiles { - profileReports.addTestResults(*profile, testResult) - } - } - - profileReports.compileResults(suite.extendedSupportedFeatures, suite.extendedUnsupportedFeatures) - - return &confv1a1.ConformanceReport{ - TypeMeta: v1.TypeMeta{ - APIVersion: "gateway.networking.k8s.io/v1alpha1", - Kind: "ConformanceReport", - }, - Date: time.Now().Format(time.RFC3339), - Mode: suite.mode, - Implementation: suite.implementation, - GatewayAPIVersion: suite.apiVersion, - GatewayAPIChannel: suite.apiChannel, - ProfileReports: profileReports.list(), - }, nil -} - -// ParseImplementation parses implementation-specific flag arguments and -// creates a *confv1a1.Implementation. -func ParseImplementation(org, project, url, version, contact string) (*confv1a1.Implementation, error) { - if org == "" { - return nil, errors.New("implementation's organization can not be empty") - } - if project == "" { - return nil, errors.New("implementation's project can not be empty") - } - if url == "" { - return nil, errors.New("implementation's url can not be empty") - } - if version == "" { - return nil, errors.New("implementation's version can not be empty") - } - contacts := strings.Split(contact, ",") - if len(contacts) == 0 { - return nil, errors.New("implementation's contact can not be empty") - } - - // TODO: add data validation https://github.com/kubernetes-sigs/gateway-api/issues/2178 - - return &confv1a1.Implementation{ - Organization: org, - Project: project, - URL: url, - Version: version, - Contact: contacts, - }, nil -} - -// ParseConformanceProfiles parses flag arguments and converts the string to -// sets.Set[ConformanceProfileName]. -func ParseConformanceProfiles(p string) sets.Set[ConformanceProfileName] { - res := sets.Set[ConformanceProfileName]{} - if p == "" { - return res - } - - for _, value := range strings.Split(p, ",") { - res.Insert(ConformanceProfileName(value)) - } - return res -} - -// getAPIVersionAndChannel iterates over all the crds installed in the cluster and check the version and channel annotations. -// In case the annotations are not found or there are crds with different versions or channels, an error is returned. -func getAPIVersionAndChannel(crds []apiextensionsv1.CustomResourceDefinition) (version string, channel string, err error) { - for _, crd := range crds { - v, okv := crd.Annotations[consts.BundleVersionAnnotation] - c, okc := crd.Annotations[consts.ChannelAnnotation] - if !okv && !okc { - continue - } - if !okv || !okc { - return "", "", errors.New("detected CRDs with partial version and channel annotations") - } - if version != "" && v != version { - return "", "", errors.New("multiple gateway API CRDs versions detected") - } - if channel != "" && c != channel { - return "", "", errors.New("multiple gateway API CRDs channels detected") - } - version = v - channel = c - } - if version == "" || channel == "" { - return "", "", errors.New("no Gateway API CRDs with the proper annotations found in the cluster") - } - if version != consts.BundleVersion { - return "", "", errors.New("the installed CRDs version is different from the suite version") - } - - return version, channel, nil -} diff --git a/conformance/utils/suite/experimental_suite_test.go b/conformance/utils/suite/experimental_suite_test.go deleted file mode 100644 index 24bf10c16c..0000000000 --- a/conformance/utils/suite/experimental_suite_test.go +++ /dev/null @@ -1,179 +0,0 @@ -/* -Copyright 2024 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package suite - -import ( - "errors" - "testing" - - "github.com/stretchr/testify/assert" - apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "sigs.k8s.io/gateway-api/pkg/consts" -) - -func TestGetAPIVersionAndChannel(t *testing.T) { - testCases := []struct { - name string - crds []apiextensionsv1.CustomResourceDefinition - expectedVersion string - expectedChannel string - err error - }{ - { - name: "no Gateway API CRDs", - err: errors.New("no Gateway API CRDs with the proper annotations found in the cluster"), - }, - { - name: "properly installed Gateway API CRDs", - crds: []apiextensionsv1.CustomResourceDefinition{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "gateways.gateway.networking.k8s.io", - Annotations: map[string]string{ - consts.BundleVersionAnnotation: "v1.0.0", - consts.ChannelAnnotation: "standard", - }, - }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Name: "httproutes.gateway.networking.k8s.io", - Annotations: map[string]string{ - consts.BundleVersionAnnotation: "v1.0.0", - consts.ChannelAnnotation: "standard", - }, - }, - }, - }, - expectedVersion: "v1.0.0", - expectedChannel: "standard", - }, - { - name: "properly installed Gateway API CRDs, with additional CRDs", - crds: []apiextensionsv1.CustomResourceDefinition{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "gateways.gateway.networking.k8s.io", - Annotations: map[string]string{ - consts.BundleVersionAnnotation: "v1.0.0", - consts.ChannelAnnotation: "standard", - }, - }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Name: "httproutes.gateway.networking.k8s.io", - Annotations: map[string]string{ - consts.BundleVersionAnnotation: "v1.0.0", - consts.ChannelAnnotation: "standard", - }, - }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Name: "crd.fake.group.k8s.io", - }, - }, - }, - expectedVersion: "v1.0.0", - expectedChannel: "standard", - }, - { - name: "installed Gateway API CRDs having multiple versions", - crds: []apiextensionsv1.CustomResourceDefinition{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "gateways.gateway.networking.k8s.io", - Annotations: map[string]string{ - consts.BundleVersionAnnotation: "v1.0.0", - consts.ChannelAnnotation: "standard", - }, - }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Name: "httproutes.gateway.networking.k8s.io", - Annotations: map[string]string{ - consts.BundleVersionAnnotation: "v2.0.0", - consts.ChannelAnnotation: "standard", - }, - }, - }, - }, - err: errors.New("multiple gateway API CRDs versions detected"), - }, - { - name: "installed Gateway API CRDs having multiple channels", - crds: []apiextensionsv1.CustomResourceDefinition{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "gateways.gateway.networking.k8s.io", - Annotations: map[string]string{ - consts.BundleVersionAnnotation: "v1.0.0", - consts.ChannelAnnotation: "standard", - }, - }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Name: "httproutes.gateway.networking.k8s.io", - Annotations: map[string]string{ - consts.BundleVersionAnnotation: "v1.0.0", - consts.ChannelAnnotation: "experimental", - }, - }, - }, - }, - err: errors.New("multiple gateway API CRDs channels detected"), - }, - { - name: "installed Gateway API CRDs having partial annotations", - crds: []apiextensionsv1.CustomResourceDefinition{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "gateways.gateway.networking.k8s.io", - Annotations: map[string]string{ - consts.BundleVersionAnnotation: "v1.0.0", - consts.ChannelAnnotation: "standard", - }, - }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Name: "httproutes.gateway.networking.k8s.io", - Annotations: map[string]string{ - consts.BundleVersionAnnotation: "v1.0.0", - }, - }, - }, - }, - err: errors.New("detected CRDs with partial version and channel annotations"), - }, - } - - for _, tc := range testCases { - tc := tc - t.Run(tc.name, func(t *testing.T) { - version, channel, err := getAPIVersionAndChannel(tc.crds) - assert.Equal(t, tc.expectedVersion, version) - assert.Equal(t, tc.expectedChannel, channel) - assert.Equal(t, tc.err, err) - }) - } -} diff --git a/conformance/utils/suite/experimental_profiles.go b/conformance/utils/suite/profiles.go similarity index 96% rename from conformance/utils/suite/experimental_profiles.go rename to conformance/utils/suite/profiles.go index 7632e64e9b..db7efe7a5a 100644 --- a/conformance/utils/suite/experimental_profiles.go +++ b/conformance/utils/suite/profiles.go @@ -71,7 +71,9 @@ var ( SupportReferenceGrant, SupportHTTPRoute, ), - ExtendedFeatures: HTTPRouteExtendedFeatures, + ExtendedFeatures: sets.New[SupportedFeature](). + Insert(GatewayExtendedFeatures.UnsortedList()...). + Insert(HTTPRouteExtendedFeatures.UnsortedList()...), } // TLSConformanceProfile is a ConformanceProfile that covers testing TLS @@ -83,6 +85,7 @@ var ( SupportReferenceGrant, SupportTLSRoute, ), + ExtendedFeatures: GatewayExtendedFeatures, } // GRPCConformanceProfile is a ConformanceProfile that covers testing GRPC diff --git a/conformance/utils/suite/experimental_reports.go b/conformance/utils/suite/reports.go similarity index 87% rename from conformance/utils/suite/experimental_reports.go rename to conformance/utils/suite/reports.go index 5b9b1c773a..a2339d5dae 100644 --- a/conformance/utils/suite/experimental_reports.go +++ b/conformance/utils/suite/reports.go @@ -22,7 +22,7 @@ import ( "k8s.io/apimachinery/pkg/util/sets" - confv1a1 "sigs.k8s.io/gateway-api/conformance/apis/v1alpha1" + confv1 "sigs.k8s.io/gateway-api/conformance/apis/v1" ) // ----------------------------------------------------------------------------- @@ -43,7 +43,7 @@ var ( testNotSupported resultType = "NOT_SUPPORTED" ) -type profileReportsMap map[ConformanceProfileName]confv1a1.ProfileReport +type profileReportsMap map[ConformanceProfileName]confv1.ProfileReport func newReports() profileReportsMap { return make(profileReportsMap) @@ -52,7 +52,7 @@ func newReports() profileReportsMap { func (p profileReportsMap) addTestResults(conformanceProfile ConformanceProfile, result testResult) { // initialize the profile report if not already initialized if _, ok := p[conformanceProfile.Name]; !ok { - p[conformanceProfile.Name] = confv1a1.ProfileReport{ + p[conformanceProfile.Name] = confv1.ProfileReport{ Name: string(conformanceProfile.Name), } } @@ -64,7 +64,7 @@ func (p profileReportsMap) addTestResults(conformanceProfile ConformanceProfile, case testSucceeded: if testIsExtended { if report.Extended == nil { - report.Extended = &confv1a1.ExtendedStatus{} + report.Extended = &confv1.ExtendedStatus{} } report.Extended.Statistics.Passed++ } else { @@ -73,7 +73,7 @@ func (p profileReportsMap) addTestResults(conformanceProfile ConformanceProfile, case testFailed: if testIsExtended { if report.Extended == nil { - report.Extended = &confv1a1.ExtendedStatus{} + report.Extended = &confv1.ExtendedStatus{} } report.Extended.FailedTests = append(report.Extended.FailedTests, result.test.ShortName) report.Extended.Statistics.Failed++ @@ -87,7 +87,7 @@ func (p profileReportsMap) addTestResults(conformanceProfile ConformanceProfile, case testSkipped: if testIsExtended { if report.Extended == nil { - report.Extended = &confv1a1.ExtendedStatus{} + report.Extended = &confv1.ExtendedStatus{} } report.Extended.Statistics.Skipped++ report.Extended.SkippedTests = append(report.Extended.SkippedTests, result.test.ShortName) @@ -99,7 +99,7 @@ func (p profileReportsMap) addTestResults(conformanceProfile ConformanceProfile, p[conformanceProfile.Name] = report } -func (p profileReportsMap) list() (profileReports []confv1a1.ProfileReport) { +func (p profileReportsMap) list() (profileReports []confv1.ProfileReport) { for _, profileReport := range p { profileReports = append(profileReports, profileReport) } @@ -111,22 +111,22 @@ func (p profileReportsMap) compileResults(supportedFeaturesMap map[ConformancePr // report the overall result for core features switch { case report.Core.Failed > 0: - report.Core.Result = confv1a1.Failure + report.Core.Result = confv1.Failure case report.Core.Skipped > 0: - report.Core.Result = confv1a1.Partial + report.Core.Result = confv1.Partial default: - report.Core.Result = confv1a1.Success + report.Core.Result = confv1.Success } if report.Extended != nil { // report the overall result for extended features switch { case report.Extended.Failed > 0: - report.Extended.Result = confv1a1.Failure + report.Extended.Result = confv1.Failure case report.Extended.Skipped > 0: - report.Extended.Result = confv1a1.Partial + report.Extended.Result = confv1.Partial default: - report.Extended.Result = confv1a1.Success + report.Extended.Result = confv1.Success } } report.Summary = buildSummary(report) @@ -187,7 +187,7 @@ func isTestExtended(profile ConformanceProfile, test ConformanceTest) bool { } // buildSummary creates a human-readable message about each profile's test outcomes. -func buildSummary(report confv1a1.ProfileReport) (reportSummary string) { +func buildSummary(report confv1.ProfileReport) (reportSummary string) { reportSummary = fmt.Sprintf("Core tests %s", buildReportSummary(report.Core)) if report.Extended != nil { reportSummary = fmt.Sprintf("%s. Extended tests %s", reportSummary, buildReportSummary(report.Extended.Status)) @@ -195,14 +195,14 @@ func buildSummary(report confv1a1.ProfileReport) (reportSummary string) { return fmt.Sprintf("%s.", reportSummary) } -func buildReportSummary(status confv1a1.Status) string { +func buildReportSummary(status confv1.Status) string { var message string switch status.Result { - case confv1a1.Success: + case confv1.Success: message = "succeeded" - case confv1a1.Partial: + case confv1.Partial: message = fmt.Sprintf("partially succeeded with %d test skips", status.Statistics.Skipped) - case confv1a1.Failure: + case confv1.Failure: message = fmt.Sprintf("failed with %d test failures", status.Statistics.Failed) } return message diff --git a/conformance/utils/suite/experimental_reports_test.go b/conformance/utils/suite/reports_test.go similarity index 66% rename from conformance/utils/suite/experimental_reports_test.go rename to conformance/utils/suite/reports_test.go index 311bf5bffd..fe5b21bbc0 100644 --- a/conformance/utils/suite/experimental_reports_test.go +++ b/conformance/utils/suite/reports_test.go @@ -21,22 +21,22 @@ import ( "github.com/stretchr/testify/require" - confv1a1 "sigs.k8s.io/gateway-api/conformance/apis/v1alpha1" + confv1 "sigs.k8s.io/gateway-api/conformance/apis/v1" ) func TestBuildSummary(t *testing.T) { testCases := []struct { name string - report confv1a1.ProfileReport + report confv1.ProfileReport expectedSummary string }{ { name: "core tests failed, no extended tests", - report: confv1a1.ProfileReport{ + report: confv1.ProfileReport{ Name: string(HTTPConformanceProfileName), - Core: confv1a1.Status{ - Result: confv1a1.Failure, - Statistics: confv1a1.Statistics{ + Core: confv1.Status{ + Result: confv1.Failure, + Statistics: confv1.Statistics{ Passed: 5, Failed: 3, }, @@ -46,18 +46,18 @@ func TestBuildSummary(t *testing.T) { }, { name: "core tests succeeded, extended tests failed", - report: confv1a1.ProfileReport{ + report: confv1.ProfileReport{ Name: string(HTTPConformanceProfileName), - Core: confv1a1.Status{ - Result: confv1a1.Success, - Statistics: confv1a1.Statistics{ + Core: confv1.Status{ + Result: confv1.Success, + Statistics: confv1.Statistics{ Passed: 8, }, }, - Extended: &confv1a1.ExtendedStatus{ - Status: confv1a1.Status{ - Result: confv1a1.Failure, - Statistics: confv1a1.Statistics{ + Extended: &confv1.ExtendedStatus{ + Status: confv1.Status{ + Result: confv1.Failure, + Statistics: confv1.Statistics{ Passed: 2, Failed: 1, }, @@ -68,19 +68,19 @@ func TestBuildSummary(t *testing.T) { }, { name: "core tests partially succeeded, extended tests succeeded", - report: confv1a1.ProfileReport{ + report: confv1.ProfileReport{ Name: string(HTTPConformanceProfileName), - Core: confv1a1.Status{ - Result: confv1a1.Partial, - Statistics: confv1a1.Statistics{ + Core: confv1.Status{ + Result: confv1.Partial, + Statistics: confv1.Statistics{ Passed: 6, Skipped: 2, }, }, - Extended: &confv1a1.ExtendedStatus{ - Status: confv1a1.Status{ - Result: confv1a1.Success, - Statistics: confv1a1.Statistics{ + Extended: &confv1.ExtendedStatus{ + Status: confv1.Status{ + Result: confv1.Success, + Statistics: confv1.Statistics{ Passed: 2, }, }, @@ -90,18 +90,18 @@ func TestBuildSummary(t *testing.T) { }, { name: "core tests succeeded, extended tests partially succeeded", - report: confv1a1.ProfileReport{ + report: confv1.ProfileReport{ Name: string(HTTPConformanceProfileName), - Core: confv1a1.Status{ - Result: confv1a1.Success, - Statistics: confv1a1.Statistics{ + Core: confv1.Status{ + Result: confv1.Success, + Statistics: confv1.Statistics{ Passed: 8, }, }, - Extended: &confv1a1.ExtendedStatus{ - Status: confv1a1.Status{ - Result: confv1a1.Partial, - Statistics: confv1a1.Statistics{ + Extended: &confv1.ExtendedStatus{ + Status: confv1.Status{ + Result: confv1.Partial, + Statistics: confv1.Statistics{ Passed: 2, Skipped: 1, }, diff --git a/conformance/utils/suite/suite.go b/conformance/utils/suite/suite.go index 58a55624c6..fdb1c64345 100644 --- a/conformance/utils/suite/suite.go +++ b/conformance/utils/suite/suite.go @@ -1,5 +1,5 @@ /* -Copyright 2022 The Kubernetes Authors. +Copyright 2023 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -17,10 +17,18 @@ limitations under the License. package suite import ( + "context" "embed" + "errors" + "fmt" + "sort" "strings" + "sync" "testing" + "time" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" @@ -28,11 +36,18 @@ import ( "sigs.k8s.io/gateway-api/apis/v1beta1" "sigs.k8s.io/gateway-api/conformance" + confv1 "sigs.k8s.io/gateway-api/conformance/apis/v1" "sigs.k8s.io/gateway-api/conformance/utils/config" + "sigs.k8s.io/gateway-api/conformance/utils/flags" "sigs.k8s.io/gateway-api/conformance/utils/kubernetes" "sigs.k8s.io/gateway-api/conformance/utils/roundtripper" + "sigs.k8s.io/gateway-api/pkg/consts" ) +// ----------------------------------------------------------------------------- +// Conformance Test Suite - Public Types +// ----------------------------------------------------------------------------- + // ConformanceTestSuite defines the test suite used to run Gateway API // conformance tests. type ConformanceTestSuite struct { @@ -55,10 +70,52 @@ type ConformanceTestSuite struct { FS embed.FS UsableNetworkAddresses []v1beta1.GatewayAddress UnusableNetworkAddresses []v1beta1.GatewayAddress + + // mode is the operating mode of the implementation. + // The default value for it is "default". + mode string + + // implementation contains the details of the implementation, such as + // organization, project, etc. + implementation confv1.Implementation + + // apiVersion is the version of the Gateway API installed in the cluster + // and is extracted by the annotation gateway.networking.k8s.io/bundle-version + // in the Gateway API CRDs. + apiVersion string + + // apiChannel is the channel of the Gateway API installed in the cluster + // and is extracted by the annotation gateway.networking.k8s.io/channel + // in the Gateway API CRDs. + apiChannel string + + // conformanceProfiles is a compiled list of profiles to check + // conformance against. + conformanceProfiles sets.Set[ConformanceProfileName] + + // running indicates whether the test suite is currently running. + // Through this flag we prevent a Run() execution to happen in case + // another Run() is ongoing. + running bool + + // results stores the pass or fail results of each test that was run by + // the test suite, organized by the tests unique name. + results map[string]testResult + + // extendedSupportedFeatures is a compiled list of named features that were + // marked as supported, and is used for reporting the test results. + extendedSupportedFeatures map[ConformanceProfileName]sets.Set[SupportedFeature] + + // extendedUnsupportedFeatures is a compiled list of named features that were + // marked as not supported, and is used for reporting the test results. + extendedUnsupportedFeatures map[ConformanceProfileName]sets.Set[SupportedFeature] + + // lock is a mutex to help ensure thread safety of the test suite object. + lock sync.RWMutex } // Options can be used to initialize a ConformanceTestSuite. -type Options struct { +type ConformanceOptions struct { Client client.Client Clientset clientset.Interface RestConfig *rest.Config @@ -93,57 +150,138 @@ type Options struct { // Gateways for tests which need to test failures with manual Gateway // address assignment. UnusableNetworkAddresses []v1beta1.GatewayAddress + + Mode string + AllowCRDsMismatch bool + Implementation confv1.Implementation + ConformanceProfiles sets.Set[ConformanceProfileName] } -// New returns a new ConformanceTestSuite. -func New(s Options) *ConformanceTestSuite { - config.SetupTimeoutConfig(&s.TimeoutConfig) +const ( + // undefinedKeyword is set in the ConformanceReport "GatewayAPIVersion" and + // "GatewayAPIChannel" fields in case it's not possible to figure out the actual + // values in the cluster, due to multiple versions of CRDs installed. + undefinedKeyword = "UNDEFINED" +) + +// NewConformanceTestSuite is a helper to use for creating a new ConformanceTestSuite. +func NewConformanceTestSuite(options ConformanceOptions) (*ConformanceTestSuite, error) { + if err := apiextensionsv1.AddToScheme(options.Client.Scheme()); err != nil { + return nil, err + } + + config.SetupTimeoutConfig(&options.TimeoutConfig) - roundTripper := s.RoundTripper + roundTripper := options.RoundTripper if roundTripper == nil { - roundTripper = &roundtripper.DefaultRoundTripper{Debug: s.Debug, TimeoutConfig: s.TimeoutConfig} + roundTripper = &roundtripper.DefaultRoundTripper{Debug: options.Debug, TimeoutConfig: options.TimeoutConfig} } - switch { - case s.EnableAllSupportedFeatures: - s.SupportedFeatures = AllFeatures - case s.SupportedFeatures == nil: - s.SupportedFeatures = GatewayCoreFeatures - default: - for feature := range GatewayCoreFeatures { - s.SupportedFeatures.Insert(feature) + installedCRDs := &apiextensionsv1.CustomResourceDefinitionList{} + err := options.Client.List(context.TODO(), installedCRDs) + if err != nil { + return nil, err + } + apiVersion, apiChannel, err := getAPIVersionAndChannel(installedCRDs.Items) + if err != nil { + // in case an error is returned and the AllowCRDsMismatch flag is false, the suite fails. + // This is the default behavior but can be customized in case one wants to experiment + // with mixed versions/channels of the API. + if !options.AllowCRDsMismatch { + return nil, err } + apiVersion = undefinedKeyword + apiChannel = undefinedKeyword + } + + mode := flags.DefaultMode + if options.Mode != "" { + mode = options.Mode + } + + if options.FS == nil { + options.FS = &conformance.Manifests + } + + // test suite callers are required to provide a conformance profile OR at + // minimum a list of features which they support. + if options.SupportedFeatures == nil && options.ConformanceProfiles.Len() == 0 && !options.EnableAllSupportedFeatures { + return nil, fmt.Errorf("no conformance profile was selected for test run, and no supported features were provided so no tests could be selected") } - for feature := range s.ExemptFeatures { - s.SupportedFeatures.Delete(feature) + // test suite callers can potentially just run all tests by saying they + // cover all features, if they don't they'll need to have provided a + // conformance profile or at least some specific features they support. + if options.EnableAllSupportedFeatures { + options.SupportedFeatures = AllFeatures + } else if options.SupportedFeatures == nil { + options.SupportedFeatures = sets.New[SupportedFeature]() } - if s.FS == nil { - s.FS = &conformance.Manifests + for feature := range options.ExemptFeatures { + options.SupportedFeatures.Delete(feature) } suite := &ConformanceTestSuite{ - Client: s.Client, - Clientset: s.Clientset, - RestConfig: s.RestConfig, + Client: options.Client, + Clientset: options.Clientset, + RestConfig: options.RestConfig, RoundTripper: roundTripper, - GatewayClassName: s.GatewayClassName, - Debug: s.Debug, - Cleanup: s.CleanupBaseResources, - BaseManifests: s.BaseManifests, - MeshManifests: s.MeshManifests, + GatewayClassName: options.GatewayClassName, + Debug: options.Debug, + Cleanup: options.CleanupBaseResources, + BaseManifests: options.BaseManifests, + MeshManifests: options.MeshManifests, Applier: kubernetes.Applier{ - NamespaceLabels: s.NamespaceLabels, - NamespaceAnnotations: s.NamespaceAnnotations, + NamespaceLabels: options.NamespaceLabels, + NamespaceAnnotations: options.NamespaceAnnotations, }, - SupportedFeatures: s.SupportedFeatures, - TimeoutConfig: s.TimeoutConfig, - SkipTests: sets.New(s.SkipTests...), - RunTest: s.RunTest, - FS: *s.FS, - UsableNetworkAddresses: s.UsableNetworkAddresses, - UnusableNetworkAddresses: s.UnusableNetworkAddresses, + SupportedFeatures: options.SupportedFeatures, + TimeoutConfig: options.TimeoutConfig, + SkipTests: sets.New(options.SkipTests...), + RunTest: options.RunTest, + FS: *options.FS, + UsableNetworkAddresses: options.UsableNetworkAddresses, + UnusableNetworkAddresses: options.UnusableNetworkAddresses, + results: make(map[string]testResult), + extendedUnsupportedFeatures: make(map[ConformanceProfileName]sets.Set[SupportedFeature]), + extendedSupportedFeatures: make(map[ConformanceProfileName]sets.Set[SupportedFeature]), + conformanceProfiles: options.ConformanceProfiles, + implementation: options.Implementation, + mode: mode, + apiVersion: apiVersion, + apiChannel: apiChannel, + } + + for _, conformanceProfileName := range options.ConformanceProfiles.UnsortedList() { + conformanceProfile, err := getConformanceProfileForName(conformanceProfileName) + if err != nil { + return nil, fmt.Errorf("failed to retrieve conformance profile: %w", err) + } + // the use of a conformance profile implicitly enables any features of + // that profile which are supported at a Core level of support. + for _, f := range conformanceProfile.CoreFeatures.UnsortedList() { + if !options.SupportedFeatures.Has(f) { + suite.SupportedFeatures.Insert(f) + } + } + for _, f := range conformanceProfile.ExtendedFeatures.UnsortedList() { + if options.SupportedFeatures.Has(f) { + if suite.extendedSupportedFeatures[conformanceProfileName] == nil { + suite.extendedSupportedFeatures[conformanceProfileName] = sets.New[SupportedFeature]() + } + suite.extendedSupportedFeatures[conformanceProfileName].Insert(f) + } else { + if suite.extendedUnsupportedFeatures[conformanceProfileName] == nil { + suite.extendedUnsupportedFeatures[conformanceProfileName] = sets.New[SupportedFeature]() + } + suite.extendedUnsupportedFeatures[conformanceProfileName].Insert(f) + } + // Add Exempt Features into unsupported features list + if options.ExemptFeatures.Has(f) { + suite.extendedUnsupportedFeatures[conformanceProfileName].Insert(f) + } + } } // apply defaults @@ -154,9 +292,13 @@ func New(s Options) *ConformanceTestSuite { suite.MeshManifests = "mesh/manifests.yaml" } - return suite + return suite, nil } +// ----------------------------------------------------------------------------- +// Conformance Test Suite - Public Methods +// ----------------------------------------------------------------------------- + // Setup ensures the base resources required for conformance tests are installed // in the cluster. It also ensures that all relevant resources are ready. func (suite *ConformanceTestSuite) Setup(t *testing.T) { @@ -207,87 +349,170 @@ func (suite *ConformanceTestSuite) Setup(t *testing.T) { } // Run runs the provided set of conformance tests. -func (suite *ConformanceTestSuite) Run(t *testing.T, tests []ConformanceTest) { +func (suite *ConformanceTestSuite) Run(t *testing.T, tests []ConformanceTest) error { + // verify that the test suite isn't already running, don't start a new run + // until the previous run finishes + suite.lock.Lock() + if suite.running { + suite.lock.Unlock() + return fmt.Errorf("can't run the test suite multiple times in parallel: the test suite is already running") + } + + // if the test suite is not currently running, reset reporting and start a + // new test run. + suite.running = true + suite.results = nil + suite.lock.Unlock() + + // run all tests and collect the test results for conformance reporting + results := make(map[string]testResult) for _, test := range tests { - t.Run(test.ShortName, func(t *testing.T) { + succeeded := t.Run(test.ShortName, func(t *testing.T) { test.Run(t, suite) }) + res := testSucceeded + if suite.SkipTests.Has(test.ShortName) { + res = testSkipped + } + if !suite.SupportedFeatures.HasAll(test.Features...) { + res = testNotSupported + } + + if !succeeded { + res = testFailed + } + + results[test.ShortName] = testResult{ + test: test, + result: res, + } } -} -// ConformanceTest is used to define each individual conformance test. -type ConformanceTest struct { - ShortName string - Description string - Features []SupportedFeature - Manifests []string - Slow bool - Parallel bool - Test func(*testing.T, *ConformanceTestSuite) + // now that the tests have completed, mark the test suite as not running + // and report the test results. + suite.lock.Lock() + suite.running = false + suite.results = results + suite.lock.Unlock() + + return nil } -// Run runs an individual tests, applying and cleaning up the required manifests -// before calling the Test function. -func (test *ConformanceTest) Run(t *testing.T, suite *ConformanceTestSuite) { - if test.Parallel { - t.Parallel() +// Report emits a ConformanceReport for the previously completed test run. +// If no run completed prior to running the report, and error is emitted. +func (suite *ConformanceTestSuite) Report() (*confv1.ConformanceReport, error) { + suite.lock.RLock() + if suite.running { + suite.lock.RUnlock() + return nil, fmt.Errorf("can't generate report: the test suite is currently running") } + defer suite.lock.RUnlock() - // Check that all features exercised by the test have been opted into by - // the suite. - for _, feature := range test.Features { - if !suite.SupportedFeatures.Has(feature) { - t.Skipf("Skipping %s: suite does not support %s", test.ShortName, feature) - } + testNames := make([]string, 0, len(suite.results)) + for tN := range suite.results { + testNames = append(testNames, tN) } - - // check that the test should not be skipped - if suite.SkipTests.Has(test.ShortName) || suite.RunTest != "" && suite.RunTest != test.ShortName { - t.Skipf("Skipping %s: test explicitly skipped", test.ShortName) + sort.Strings(testNames) + profileReports := newReports() + for _, tN := range testNames { + testResult := suite.results[tN] + conformanceProfiles := getConformanceProfilesForTest(testResult.test, suite.conformanceProfiles).UnsortedList() + sort.Slice(conformanceProfiles, func(i, j int) bool { + return conformanceProfiles[i].Name < conformanceProfiles[j].Name + }) + for _, profile := range conformanceProfiles { + profileReports.addTestResults(*profile, testResult) + } } - for _, manifestLocation := range test.Manifests { - t.Logf("Applying %s", manifestLocation) - suite.Applier.MustApplyWithCleanup(t, suite.Client, suite.TimeoutConfig, manifestLocation, true) - } + profileReports.compileResults(suite.extendedSupportedFeatures, suite.extendedUnsupportedFeatures) - test.Test(t, suite) + return &confv1.ConformanceReport{ + TypeMeta: v1.TypeMeta{ + APIVersion: "gateway.networking.k8s.io/v1alpha1", + Kind: "ConformanceReport", + }, + Date: time.Now().Format(time.RFC3339), + Mode: suite.mode, + Implementation: suite.implementation, + GatewayAPIVersion: suite.apiVersion, + GatewayAPIChannel: suite.apiChannel, + ProfileReports: profileReports.list(), + }, nil } -// ParseSupportedFeatures parses flag arguments and converts the string to -// sets.Set[suite.SupportedFeature] -func ParseSupportedFeatures(f string) sets.Set[SupportedFeature] { - if f == "" { - return nil +// ParseImplementation parses implementation-specific flag arguments and +// creates a *confv1a1.Implementation. +func ParseImplementation(org, project, url, version, contact string) (*confv1.Implementation, error) { + if org == "" { + return nil, errors.New("implementation's organization can not be empty") } - res := sets.Set[SupportedFeature]{} - for _, value := range strings.Split(f, ",") { - res.Insert(SupportedFeature(value)) + if project == "" { + return nil, errors.New("implementation's project can not be empty") } - return res + if url == "" { + return nil, errors.New("implementation's url can not be empty") + } + if version == "" { + return nil, errors.New("implementation's version can not be empty") + } + contacts := strings.Split(contact, ",") + if len(contacts) == 0 { + return nil, errors.New("implementation's contact can not be empty") + } + + // TODO: add data validation https://github.com/kubernetes-sigs/gateway-api/issues/2178 + + return &confv1.Implementation{ + Organization: org, + Project: project, + URL: url, + Version: version, + Contact: contacts, + }, nil } -// ParseKeyValuePairs parses flag arguments and converts the string to -// map[string]string containing label key/value pairs. -func ParseKeyValuePairs(f string) map[string]string { - if f == "" { - return nil +// ParseConformanceProfiles parses flag arguments and converts the string to +// sets.Set[ConformanceProfileName]. +func ParseConformanceProfiles(p string) sets.Set[ConformanceProfileName] { + res := sets.Set[ConformanceProfileName]{} + if p == "" { + return res } - res := map[string]string{} - for _, kv := range strings.Split(f, ",") { - parts := strings.Split(kv, "=") - if len(parts) == 2 { - res[parts[0]] = parts[1] - } + + for _, value := range strings.Split(p, ",") { + res.Insert(ConformanceProfileName(value)) } return res } -// ParseSkipTests parses flag arguments and converts the string to -// []string containing the tests to be skipped. -func ParseSkipTests(t string) []string { - if t == "" { - return nil +// getAPIVersionAndChannel iterates over all the crds installed in the cluster and check the version and channel annotations. +// In case the annotations are not found or there are crds with different versions or channels, an error is returned. +func getAPIVersionAndChannel(crds []apiextensionsv1.CustomResourceDefinition) (version string, channel string, err error) { + for _, crd := range crds { + v, okv := crd.Annotations[consts.BundleVersionAnnotation] + c, okc := crd.Annotations[consts.ChannelAnnotation] + if !okv && !okc { + continue + } + if !okv || !okc { + return "", "", errors.New("detected CRDs with partial version and channel annotations") + } + if version != "" && v != version { + return "", "", errors.New("multiple gateway API CRDs versions detected") + } + if channel != "" && c != channel { + return "", "", errors.New("multiple gateway API CRDs channels detected") + } + version = v + channel = c } - return strings.Split(t, ",") + if version == "" || channel == "" { + return "", "", errors.New("no Gateway API CRDs with the proper annotations found in the cluster") + } + if version != consts.BundleVersion { + return "", "", errors.New("the installed CRDs version is different from the suite version") + } + + return version, channel, nil } diff --git a/conformance/utils/suite/suite_test.go b/conformance/utils/suite/suite_test.go index 738b4a3ba2..24bf10c16c 100644 --- a/conformance/utils/suite/suite_test.go +++ b/conformance/utils/suite/suite_test.go @@ -1,5 +1,5 @@ /* -Copyright 2023 The Kubernetes Authors. +Copyright 2024 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -17,53 +17,163 @@ limitations under the License. package suite import ( - "reflect" + "errors" "testing" - "k8s.io/apimachinery/pkg/util/sets" -) - -func TestParseSupportedFeatures(t *testing.T) { - flags := []string{ - "", - "a", - "b,c,d", - } - - s1 := sets.Set[SupportedFeature]{} - s1.Insert(SupportedFeature("a")) - s2 := sets.Set[SupportedFeature]{} - s2.Insert(SupportedFeature("b")) - s2.Insert(SupportedFeature("c")) - s2.Insert(SupportedFeature("d")) - features := []sets.Set[SupportedFeature]{nil, s1, s2} + "github.com/stretchr/testify/assert" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - for i, f := range flags { - expect := features[i] - got := ParseSupportedFeatures(f) - if !reflect.DeepEqual(got, expect) { - t.Errorf("Unexpected features from flags '%s', expected: %v, got: %v", f, expect, got) - } - } -} + "sigs.k8s.io/gateway-api/pkg/consts" +) -func TestParseKeyValuePairs(t *testing.T) { - flags := []string{ - "", - "a=b", - "b=c,d=e,f=g", - } - labels := []map[string]string{ - nil, - {"a": "b"}, - {"b": "c", "d": "e", "f": "g"}, +func TestGetAPIVersionAndChannel(t *testing.T) { + testCases := []struct { + name string + crds []apiextensionsv1.CustomResourceDefinition + expectedVersion string + expectedChannel string + err error + }{ + { + name: "no Gateway API CRDs", + err: errors.New("no Gateway API CRDs with the proper annotations found in the cluster"), + }, + { + name: "properly installed Gateway API CRDs", + crds: []apiextensionsv1.CustomResourceDefinition{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "gateways.gateway.networking.k8s.io", + Annotations: map[string]string{ + consts.BundleVersionAnnotation: "v1.0.0", + consts.ChannelAnnotation: "standard", + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "httproutes.gateway.networking.k8s.io", + Annotations: map[string]string{ + consts.BundleVersionAnnotation: "v1.0.0", + consts.ChannelAnnotation: "standard", + }, + }, + }, + }, + expectedVersion: "v1.0.0", + expectedChannel: "standard", + }, + { + name: "properly installed Gateway API CRDs, with additional CRDs", + crds: []apiextensionsv1.CustomResourceDefinition{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "gateways.gateway.networking.k8s.io", + Annotations: map[string]string{ + consts.BundleVersionAnnotation: "v1.0.0", + consts.ChannelAnnotation: "standard", + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "httproutes.gateway.networking.k8s.io", + Annotations: map[string]string{ + consts.BundleVersionAnnotation: "v1.0.0", + consts.ChannelAnnotation: "standard", + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "crd.fake.group.k8s.io", + }, + }, + }, + expectedVersion: "v1.0.0", + expectedChannel: "standard", + }, + { + name: "installed Gateway API CRDs having multiple versions", + crds: []apiextensionsv1.CustomResourceDefinition{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "gateways.gateway.networking.k8s.io", + Annotations: map[string]string{ + consts.BundleVersionAnnotation: "v1.0.0", + consts.ChannelAnnotation: "standard", + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "httproutes.gateway.networking.k8s.io", + Annotations: map[string]string{ + consts.BundleVersionAnnotation: "v2.0.0", + consts.ChannelAnnotation: "standard", + }, + }, + }, + }, + err: errors.New("multiple gateway API CRDs versions detected"), + }, + { + name: "installed Gateway API CRDs having multiple channels", + crds: []apiextensionsv1.CustomResourceDefinition{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "gateways.gateway.networking.k8s.io", + Annotations: map[string]string{ + consts.BundleVersionAnnotation: "v1.0.0", + consts.ChannelAnnotation: "standard", + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "httproutes.gateway.networking.k8s.io", + Annotations: map[string]string{ + consts.BundleVersionAnnotation: "v1.0.0", + consts.ChannelAnnotation: "experimental", + }, + }, + }, + }, + err: errors.New("multiple gateway API CRDs channels detected"), + }, + { + name: "installed Gateway API CRDs having partial annotations", + crds: []apiextensionsv1.CustomResourceDefinition{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "gateways.gateway.networking.k8s.io", + Annotations: map[string]string{ + consts.BundleVersionAnnotation: "v1.0.0", + consts.ChannelAnnotation: "standard", + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "httproutes.gateway.networking.k8s.io", + Annotations: map[string]string{ + consts.BundleVersionAnnotation: "v1.0.0", + }, + }, + }, + }, + err: errors.New("detected CRDs with partial version and channel annotations"), + }, } - for i, f := range flags { - expect := labels[i] - got := ParseKeyValuePairs(f) - if !reflect.DeepEqual(got, expect) { - t.Errorf("Unexpected labels from flags '%s', expected: %v, got: %v", f, expect, got) - } + for _, tc := range testCases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + version, channel, err := getAPIVersionAndChannel(tc.crds) + assert.Equal(t, tc.expectedVersion, version) + assert.Equal(t, tc.expectedChannel, channel) + assert.Equal(t, tc.err, err) + }) } }