Skip to content

Commit

Permalink
Experimental conformance promotion to standard (#2868)
Browse files Browse the repository at this point in the history 
* chore: experimental conformance promotion

Signed-off-by: Mattia Lavacca <lavacca.mattia@gmail.com>

* file renamed

Signed-off-by: Mattia Lavacca <lavacca.mattia@gmail.com>

* Allow single test to be run

Signed-off-by: Mattia Lavacca <lavacca.mattia@gmail.com>
Co-authored-by: Dave Protasowski <dprotaso@gmail.com>

* address review comments

Signed-off-by: Mattia Lavacca <lavacca.mattia@gmail.com>

* chore: minor changes in the CLI

Signed-off-by: Mattia Lavacca <lavacca.mattia@gmail.com>

---------

Signed-off-by: Mattia Lavacca <lavacca.mattia@gmail.com>
Co-authored-by: Dave Protasowski <dprotaso@gmail.com>
  • Loading branch information
@mlavacca @dprotaso
mlavacca and dprotaso authored Mar 28, 2024
1 parent 9183329 commit 204abaf
Show file tree
Hide file tree
Showing 19 changed files with 827 additions and 1,044 deletions.
7 changes: 1 addition & 6 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,12 +87,7 @@ test:
.PHONY: conformance
conformance:
go test ${GO_TEST_FLAGS} -v ./conformance -run TestConformance -args ${CONFORMANCE_FLAGS}

# Run experimental conformance tests against controller implementation
.PHONY: conformance.experimental
conformance.experimental:
go test ${GO_TEST_FLAGS} -v ./conformance -run TestExperimentalConformance -args ${CONFORMANCE_FLAGS}


# Install CRD's and example resources to a pre-existing cluster.
.PHONY: install
install: crd example
Expand Down
2 changes: 1 addition & 1 deletion ...rmance/apis/v1alpha1/conformancereport.go → conformance/apis/v1/conformancereport.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/

package v1alpha1
package v1

import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
Expand Down
17 changes: 2 additions & 15 deletions conformance/apis/v1alpha1/doc.go → conformance/apis/v1/doc.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,23 +14,10 @@ See the License for the specific language governing permissions and
limitations under the License.
*/

// V1alpha1 includes alpha maturity API types and utilities for creating and
// v1 includes GA maturity API types and utilities for creating and
// handling the results of conformance test runs. These types are _only_
// intended for use by the conformance test suite OR external test suites that
// are written in Golang and execute the conformance test suite as a Golang
// library.
//
// Note that currently all sub-packages are considered "experimental" in that
// they aren't intended for general use or to be distributed as part of a
// release so there is no way to use them by default when using the Golang
// library at this time. If you don't know for sure that you want to use these
// features, then you should not use them. If you would like to opt into these
// unreleased features use Go build tags to enable them, e.g.:
//
// $ go test ./conformance/... -args ${CONFORMANCE_ARGS}
//
// Please note that everything here is considered experimental and subject to
// change. Expect breaking changes and/or complete removals if you start using
// them.

package v1alpha1
package v1
2 changes: 1 addition & 1 deletion conformance/apis/v1alpha1/profilereport.go → conformance/apis/v1/profilereport.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/

package v1alpha1
package v1

// ProfileReport is the generated report for the test results of a specific
// named conformance profile.
Expand Down
2 changes: 1 addition & 1 deletion conformance/apis/v1alpha1/result.go → conformance/apis/v1/result.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/

package v1alpha1
package v1

// Result is a simple high-level summary describing the conclusion of a test
// run.
Expand Down
2 changes: 1 addition & 1 deletion conformance/apis/v1alpha1/statistics.go → conformance/apis/v1/statistics.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/

package v1alpha1
package v1

// Statistics includes numerical summaries of the number of conformance tests
// that passed, failed or were intentionally skipped.
Expand Down
145 changes: 108 additions & 37 deletions conformance/conformance_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,70 +14,141 @@ See the License for the specific language governing permissions and
limitations under the License.
*/

// conformance_test contains code to run the conformance tests. This is in its own package to avoid circular imports.
package conformance_test

import (
"os"
"testing"

v1 "sigs.k8s.io/gateway-api/apis/v1"
"sigs.k8s.io/gateway-api/apis/v1alpha2"
"sigs.k8s.io/gateway-api/apis/v1beta1"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/config"
"sigs.k8s.io/yaml"

gatewayv1 "sigs.k8s.io/gateway-api/apis/v1"
gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2"
gatewayv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1"
confv1 "sigs.k8s.io/gateway-api/conformance/apis/v1"
"sigs.k8s.io/gateway-api/conformance/tests"
"sigs.k8s.io/gateway-api/conformance/utils/flags"
"sigs.k8s.io/gateway-api/conformance/utils/suite"
)

"k8s.io/client-go/kubernetes"
_ "k8s.io/client-go/plugin/pkg/client/auth"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/config"
var (
cfg *rest.Config
k8sClientset *kubernetes.Clientset
mgrClient client.Client
supportedFeatures sets.Set[suite.SupportedFeature]
exemptFeatures sets.Set[suite.SupportedFeature]
namespaceLabels map[string]string
namespaceAnnotations map[string]string
implementation *confv1.Implementation
mode string
allowCRDsMismatch bool
conformanceProfiles sets.Set[suite.ConformanceProfileName]
skipTests []string
)

func TestConformance(t *testing.T) {
cfg, err := config.GetConfig()
var err error
cfg, err = config.GetConfig()
if err != nil {
t.Fatalf("Error loading Kubernetes config: %v", err)
}
client, err := client.New(cfg, client.Options{})
mgrClient, err = client.New(cfg, client.Options{})
if err != nil {
t.Fatalf("Error initializing Kubernetes client: %v", err)
}
clientset, err := kubernetes.NewForConfig(cfg)
k8sClientset, err = kubernetes.NewForConfig(cfg)
if err != nil {
t.Fatalf("Error initializing Kubernetes REST client: %v", err)
}

v1alpha2.AddToScheme(client.Scheme())
v1beta1.AddToScheme(client.Scheme())
v1.AddToScheme(client.Scheme())
gatewayv1alpha2.AddToScheme(mgrClient.Scheme())
gatewayv1beta1.AddToScheme(mgrClient.Scheme())
gatewayv1.AddToScheme(mgrClient.Scheme())
apiextensionsv1.AddToScheme(mgrClient.Scheme())

// conformance flags
supportedFeatures = suite.ParseSupportedFeatures(*flags.SupportedFeatures)
exemptFeatures = suite.ParseSupportedFeatures(*flags.ExemptFeatures)
skipTests = suite.ParseSkipTests(*flags.SkipTests)
namespaceLabels = suite.ParseKeyValuePairs(*flags.NamespaceLabels)
namespaceAnnotations = suite.ParseKeyValuePairs(*flags.NamespaceAnnotations)
conformanceProfiles = suite.ParseConformanceProfiles(*flags.ConformanceProfiles)
if len(conformanceProfiles) == 0 {
t.Fatal("conformance profiles need to be given")
}
mode = *flags.Mode
allowCRDsMismatch = *flags.AllowCRDsMismatch

supportedFeatures := suite.ParseSupportedFeatures(*flags.SupportedFeatures)
exemptFeatures := suite.ParseSupportedFeatures(*flags.ExemptFeatures)
skipTests := suite.ParseSkipTests(*flags.SkipTests)
namespaceLabels := suite.ParseKeyValuePairs(*flags.NamespaceLabels)
namespaceAnnotations := suite.ParseKeyValuePairs(*flags.NamespaceAnnotations)
implementation, err = suite.ParseImplementation(
*flags.ImplementationOrganization,
*flags.ImplementationProject,
*flags.ImplementationURL,
*flags.ImplementationVersion,
*flags.ImplementationContact,
)
if err != nil {
t.Fatalf("Error parsing implementation's details: %v", err)
}
testConformance(t)
}

func testConformance(t *testing.T) {
t.Logf("Running conformance tests with %s GatewayClass\n cleanup: %t\n debug: %t\n enable all features: %t \n supported features: [%v]\n exempt features: [%v]",
*flags.GatewayClassName, *flags.CleanupBaseResources, *flags.ShowDebug, *flags.EnableAllSupportedFeatures, *flags.SupportedFeatures, *flags.ExemptFeatures)

cSuite := suite.New(suite.Options{
Client: client,
RestConfig: cfg,
// This clientset is needed in addition to the client only because
// controller-runtime client doesn't support non CRUD sub-resources yet (https://github.com/kubernetes-sigs/controller-runtime/issues/452).
Clientset: clientset,
GatewayClassName: *flags.GatewayClassName,
Debug: *flags.ShowDebug,
CleanupBaseResources: *flags.CleanupBaseResources,
SupportedFeatures: supportedFeatures,
ExemptFeatures: exemptFeatures,
EnableAllSupportedFeatures: *flags.EnableAllSupportedFeatures,
NamespaceLabels: namespaceLabels,
NamespaceAnnotations: namespaceAnnotations,
SkipTests: skipTests,
RunTest: *flags.RunTest,
})
cSuite.Setup(t)
cSuite, err := suite.NewConformanceTestSuite(
suite.ConformanceOptions{
Client: mgrClient,
RestConfig: cfg,
// This clientset is needed in addition to the client only because
// controller-runtime client doesn't support non CRUD sub-resources yet (https://github.com/kubernetes-sigs/controller-runtime/issues/452).
Clientset: k8sClientset,
GatewayClassName: *flags.GatewayClassName,
Debug: *flags.ShowDebug,
CleanupBaseResources: *flags.CleanupBaseResources,
SupportedFeatures: supportedFeatures,
ExemptFeatures: exemptFeatures,
EnableAllSupportedFeatures: *flags.EnableAllSupportedFeatures,
NamespaceLabels: namespaceLabels,
NamespaceAnnotations: namespaceAnnotations,
SkipTests: skipTests,
RunTest: *flags.RunTest,
Mode: mode,
AllowCRDsMismatch: allowCRDsMismatch,
Implementation: *implementation,
ConformanceProfiles: conformanceProfiles,
})
if err != nil {
t.Fatalf("error creating conformance test suite: %v", err)
}

cSuite.Setup(t)
cSuite.Run(t, tests.ConformanceTests)
report, err := cSuite.Report()
if err != nil {
t.Fatalf("error generating conformance profile report: %v", err)
}
writeReport(t.Logf, *report, *flags.ReportOutput)
}

func writeReport(logf func(string, ...any), report confv1.ConformanceReport, output string) error {
rawReport, err := yaml.Marshal(report)
if err != nil {
return err
}

if output != "" {
if err = os.WriteFile(output, rawReport, 0o600); err != nil {
return err
}
}
logf("Conformance report:\n%s", string(rawReport))

return nil
}
Loading

0 comments on commit 204abaf

Please sign in to comment.