chore: experimental conformance promotion

Signed-off-by: Mattia Lavacca <lavacca.mattia@gmail.com>

conformance/apis/v1alpha1/conformancereport.go → conformance/apis/v1beta1/conformancereport.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package v1alpha1
+package v1beta1
 
 import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

conformance/apis/v1alpha1/doc.go → conformance/apis/v1beta1/doc.go

@@ -14,23 +14,10 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-// V1alpha1 includes alpha maturity API types and utilities for creating and
+// v1beta1 includes beta maturity API types and utilities for creating and
 // handling the results of conformance test runs. These types are _only_
 // intended for use by the conformance test suite OR external test suites that
 // are written in Golang and execute the conformance test suite as a Golang
 // library.
-//
-// Note that currently all sub-packages are considered "experimental" in that
-// they aren't intended for general use or to be distributed as part of a
-// release so there is no way to use them by default when using the Golang
-// library at this time. If you don't know for sure that you want to use these
-// features, then you should not use them. If you would like to opt into these
-// unreleased features use Go build tags to enable them, e.g.:
-//
-//   $ go test ./conformance/... -args ${CONFORMANCE_ARGS}
-//
-// Please note that everything here is considered experimental and subject to
-// change. Expect breaking changes and/or complete removals if you start using
-// them.
 
-package v1alpha1
+package v1beta1

conformance/apis/v1alpha1/profilereport.go → conformance/apis/v1beta1/profilereport.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package v1alpha1
+package v1beta1
 
 // ProfileReport is the generated report for the test results of a specific
 // named conformance profile.

conformance/apis/v1alpha1/result.go → conformance/apis/v1beta1/result.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package v1alpha1
+package v1beta1
 
 // Result is a simple high-level summary describing the conclusion of a test
 // run.

conformance/apis/v1alpha1/statistics.go → conformance/apis/v1beta1/statistics.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package v1alpha1
+package v1beta1
 
 // Statistics includes numerical summaries of the number of conformance tests
 // that passed, failed or were intentionally skipped.

conformance/conformance_test.go

@@ -1,5 +1,5 @@
 /*
-Copyright 2022 The Kubernetes Authors.
+Copyright 2023 The Kubernetes Authors.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -14,70 +14,138 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-// conformance_test contains code to run the conformance tests. This is in its own package to avoid circular imports.
 package conformance_test
 
 import (
+	"os"
 	"testing"
 
-	v1 "sigs.k8s.io/gateway-api/apis/v1"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/config"
+	"sigs.k8s.io/yaml"
+
+	gatewayv1 "sigs.k8s.io/gateway-api/apis/v1"
 	"sigs.k8s.io/gateway-api/apis/v1alpha2"
 	"sigs.k8s.io/gateway-api/apis/v1beta1"
+	confv1b1 "sigs.k8s.io/gateway-api/conformance/apis/v1beta1"
 	"sigs.k8s.io/gateway-api/conformance/tests"
 	"sigs.k8s.io/gateway-api/conformance/utils/flags"
 	"sigs.k8s.io/gateway-api/conformance/utils/suite"
+)
 
-	"k8s.io/client-go/kubernetes"
-	_ "k8s.io/client-go/plugin/pkg/client/auth"
-	"sigs.k8s.io/controller-runtime/pkg/client"
-	"sigs.k8s.io/controller-runtime/pkg/client/config"
+var (
+	cfg                  *rest.Config
+	k8sClientset         *kubernetes.Clientset
+	mgrClient            client.Client
+	supportedFeatures    sets.Set[suite.SupportedFeature]
+	exemptFeatures       sets.Set[suite.SupportedFeature]
+	namespaceLabels      map[string]string
+	namespaceAnnotations map[string]string
+	implementation       *confv1b1.Implementation
+	mode                 string
+	allowCRDsMismatch    bool
+	conformanceProfiles  sets.Set[suite.ConformanceProfileName]
+	skipTests            []string
 )
 
 func TestConformance(t *testing.T) {
-	cfg, err := config.GetConfig()
+	var err error
+	cfg, err = config.GetConfig()
 	if err != nil {
 		t.Fatalf("Error loading Kubernetes config: %v", err)
 	}
-	client, err := client.New(cfg, client.Options{})
+	mgrClient, err = client.New(cfg, client.Options{})
 	if err != nil {
 		t.Fatalf("Error initializing Kubernetes client: %v", err)
 	}
-	clientset, err := kubernetes.NewForConfig(cfg)
+	k8sClientset, err = kubernetes.NewForConfig(cfg)
 	if err != nil {
 		t.Fatalf("Error initializing Kubernetes REST client: %v", err)
 	}
 
-	v1alpha2.AddToScheme(client.Scheme())
-	v1beta1.AddToScheme(client.Scheme())
-	v1.AddToScheme(client.Scheme())
+	v1alpha2.AddToScheme(mgrClient.Scheme())
+	v1beta1.AddToScheme(mgrClient.Scheme())
+	gatewayv1.AddToScheme(mgrClient.Scheme())
+
+	// conformance flags
+	supportedFeatures = suite.ParseSupportedFeatures(*flags.SupportedFeatures)
+	exemptFeatures = suite.ParseSupportedFeatures(*flags.ExemptFeatures)
+	skipTests = suite.ParseSkipTests(*flags.SkipTests)
+	namespaceLabels = suite.ParseKeyValuePairs(*flags.NamespaceLabels)
+	namespaceAnnotations = suite.ParseKeyValuePairs(*flags.NamespaceAnnotations)
+	conformanceProfiles = suite.ParseConformanceProfiles(*flags.ConformanceProfiles)
+	if len(conformanceProfiles) == 0 {
+		t.Fatal("conformance profiles need to be given")
+	}
+	mode = *flags.Mode
+	allowCRDsMismatch = *flags.AllowCRDsMismatch
 
-	supportedFeatures := suite.ParseSupportedFeatures(*flags.SupportedFeatures)
-	exemptFeatures := suite.ParseSupportedFeatures(*flags.ExemptFeatures)
-	skipTests := suite.ParseSkipTests(*flags.SkipTests)
-	namespaceLabels := suite.ParseKeyValuePairs(*flags.NamespaceLabels)
-	namespaceAnnotations := suite.ParseKeyValuePairs(*flags.NamespaceAnnotations)
+	implementation, err = suite.ParseImplementation(
+		*flags.ImplementationOrganization,
+		*flags.ImplementationProject,
+		*flags.ImplementationURL,
+		*flags.ImplementationVersion,
+		*flags.ImplementationContact,
+	)
+	if err != nil {
+		t.Fatalf("Error parsing implementation's details: %v", err)
+	}
+	testConformance(t)
+}
 
+func testConformance(t *testing.T) {
 	t.Logf("Running conformance tests with %s GatewayClass\n cleanup: %t\n debug: %t\n enable all features: %t \n supported features: [%v]\n exempt features: [%v]",
 		*flags.GatewayClassName, *flags.CleanupBaseResources, *flags.ShowDebug, *flags.EnableAllSupportedFeatures, *flags.SupportedFeatures, *flags.ExemptFeatures)
 
-	cSuite := suite.New(suite.Options{
-		Client:     client,
-		RestConfig: cfg,
-		// This clientset is needed in addition to the client only because
-		// controller-runtime client doesn't support non CRUD sub-resources yet (https://github.com/kubernetes-sigs/controller-runtime/issues/452).
-		Clientset:                  clientset,
-		GatewayClassName:           *flags.GatewayClassName,
-		Debug:                      *flags.ShowDebug,
-		CleanupBaseResources:       *flags.CleanupBaseResources,
-		SupportedFeatures:          supportedFeatures,
-		ExemptFeatures:             exemptFeatures,
-		EnableAllSupportedFeatures: *flags.EnableAllSupportedFeatures,
-		NamespaceLabels:            namespaceLabels,
-		NamespaceAnnotations:       namespaceAnnotations,
-		SkipTests:                  skipTests,
-		RunTest:                    *flags.RunTest,
-	})
-	cSuite.Setup(t)
+	cSuite, err := suite.NewConformanceTestSuite(
+		suite.ConformanceOptions{
+			Client:     mgrClient,
+			RestConfig: cfg,
+			// This clientset is needed in addition to the client only because
+			// controller-runtime client doesn't support non CRUD sub-resources yet (https://github.com/kubernetes-sigs/controller-runtime/issues/452).
+			Clientset:                  k8sClientset,
+			GatewayClassName:           *flags.GatewayClassName,
+			Debug:                      *flags.ShowDebug,
+			CleanupBaseResources:       *flags.CleanupBaseResources,
+			SupportedFeatures:          supportedFeatures,
+			ExemptFeatures:             exemptFeatures,
+			EnableAllSupportedFeatures: *flags.EnableAllSupportedFeatures,
+			NamespaceLabels:            namespaceLabels,
+			NamespaceAnnotations:       namespaceAnnotations,
+			SkipTests:                  skipTests,
+			Mode:                       mode,
+			AllowCRDsMismatch:          allowCRDsMismatch,
+			Implementation:             *implementation,
+			ConformanceProfiles:        conformanceProfiles,
+		})
+	if err != nil {
+		t.Fatalf("error creating conformance test suite: %v", err)
+	}
 
+	cSuite.Setup(t)
 	cSuite.Run(t, tests.ConformanceTests)
+	report, err := cSuite.Report()
+	if err != nil {
+		t.Fatalf("error generating conformance profile report: %v", err)
+	}
+	writeReport(t.Logf, *report, *flags.ReportOutput)
+}
+
+func writeReport(logf func(string, ...any), report confv1b1.ConformanceReport, output string) error {
+	rawReport, err := yaml.Marshal(report)
+	if err != nil {
+		return err
+	}
+
+	if output != "" {
+		if err = os.WriteFile(output, rawReport, 0o600); err != nil {
+			return err
+		}
+	}
+	logf("Conformance report:\n%s", string(rawReport))
+
+	return nil
 }