Skip to content

Commit

Permalink
feat: conformance profiles cli
Browse files Browse the repository at this point in the history
cobra and viper added. The Configuration can now be set either via flags or
configuration file.

Signed-off-by: Mattia Lavacca <lavacca.mattia@gmail.com>
  • Loading branch information
mlavacca committed Jul 7, 2023
1 parent bb7ede6 commit 9fb4107
Show file tree
Hide file tree
Showing 8 changed files with 244 additions and 56 deletions.
20 changes: 0 additions & 20 deletions cmd/certification/main.go

This file was deleted.

141 changes: 141 additions & 0 deletions conformance/experimental_conformance_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,141 @@
//go:build experimental
// +build experimental

/*
Copyright 2023 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package conformance_test

import (
"fmt"
"os"
"testing"

"gopkg.in/yaml.v2"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
"k8s.io/utils/pointer"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/config"
"sigs.k8s.io/gateway-api/apis/v1alpha2"
"sigs.k8s.io/gateway-api/apis/v1beta1"
confv1a1 "sigs.k8s.io/gateway-api/conformance/apis/v1alpha1"
"sigs.k8s.io/gateway-api/conformance/tests"
"sigs.k8s.io/gateway-api/conformance/utils/flags"
"sigs.k8s.io/gateway-api/conformance/utils/suite"
)

func init() {
flags.ConformanceProfiles = pointer.String("HTTP")
flags.ImplementationOrganization = pointer.String("acme.org")
flags.ImplementationProject = pointer.String("test")
flags.ImplementationUrl = pointer.String("acme.org/test")
flags.ImplementationVersion = pointer.String("0.1.0")
flags.ImplementationContact = pointer.String("@mlavacca")

flags.ConformanceProfiles = pointer.String("HTTP")
flags.GatewayClassName = pointer.String("kong")
//flags.SupportedFeatures = pointer.String("HTTPRoute,Gateway,ReferenceGrant")
flags.ShowDebug = pointer.Bool(true)
}

func TestExperimentalConformance(t *testing.T) {
cfg, err := config.GetConfig()
if err != nil {
t.Fatalf("Error loading Kubernetes config: %v", err)
}
client, err := client.New(cfg, client.Options{})
if err != nil {
t.Fatalf("Error initializing Kubernetes client: %v", err)
}
clientset, err := kubernetes.NewForConfig(cfg)
if err != nil {
t.Fatalf("Error initializing Kubernetes REST client: %v", err)
}

v1alpha2.AddToScheme(client.Scheme())
v1beta1.AddToScheme(client.Scheme())

supportedFeatures := suite.ParseSupportedFeatures(*flags.SupportedFeatures)
exemptFeatures := suite.ParseSupportedFeatures(*flags.ExemptFeatures)
implementation, err := suite.ParseImplementation(
*flags.ImplementationOrganization,
*flags.ImplementationProject,
*flags.ImplementationUrl,
*flags.ImplementationVersion,
*flags.ImplementationContact,
)
if err != nil {
t.Fatalf("Error parsing implementation's details: %v", err)
}
conformanceProfiles, err := suite.ParseConformanceProfiles(*flags.ConformanceProfiles)
if err != nil {
t.Fatalf("Error parsing conformance profiles: %v", err)
}

namespaceLabels := suite.ParseNamespaceLabels(*flags.NamespaceLabels)

t.Logf("Running conformance tests with %s GatewayClass\n cleanup: %t\n debug: %t\n enable all features: %t \n supported features: [%v]\n exempt features: [%v]",
*flags.GatewayClassName, *flags.CleanupBaseResources, *flags.ShowDebug, *flags.EnableAllSupportedFeatures, *flags.SupportedFeatures, *flags.ExemptFeatures)

cSuite, err := suite.NewExperimentalConformanceTestSuite(
suite.ExperimentalConformanceOptions{
Options: suite.Options{
Client: client,
RESTClient: clientset.CoreV1().RESTClient().(*rest.RESTClient),
RestConfig: cfg,
// This clientset is needed in addition to the client only because
// controller-runtime client doesn't support non CRUD sub-resources yet (https://github.com/kubernetes-sigs/controller-runtime/issues/452).
Clientset: clientset,
GatewayClassName: *flags.GatewayClassName,
Debug: *flags.ShowDebug,
CleanupBaseResources: *flags.CleanupBaseResources,
SupportedFeatures: supportedFeatures,
ExemptFeatures: exemptFeatures,
EnableAllSupportedFeatures: *flags.EnableAllSupportedFeatures,
NamespaceLabels: namespaceLabels,
},
Implementation: *implementation,
ConformanceProfiles: conformanceProfiles,
})
if err != nil {
t.Fatalf("error creating experimental conformance test suite: %v", err)
}

cSuite.Setup(t)
cSuite.Run(t, tests.ConformanceTests)
report, err := cSuite.Report()
if err != nil {
t.Fatalf("error generating conformance profile report: %v", err)
}
writeReport(*report, *flags.ReportOutput)
}

func writeReport(report confv1a1.ConformanceReport, output string) error {
rawReport, err := yaml.Marshal(report)
if err != nil {
return err
}

if output != "" {
if err = os.WriteFile(output, rawReport, 0666); err != nil {
return err
}
}
fmt.Printf("report profiles report:\n %s", rawReport)

return nil
}
35 changes: 35 additions & 0 deletions conformance/utils/flags/experimental_flags.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
//go:build experimental
// +build experimental

/*
Copyright 2023 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// flags contains command-line flag definitions for the conformance
// profile certification. They're in this package so they can be shared
// among the various suites that are all run by the same Makefile invocation.
package flags

import "flag"

var (
ImplementationOrganization = flag.String("organization", "", "Implementation's Organization to issue conformance to")
ImplementationProject = flag.String("project", "", "Implementation's project to issue conformance to")
ImplementationUrl = flag.String("url", "", "Implementation's url to issue conformance to")
ImplementationVersion = flag.String("version", "", "Implementation's version to issue conformance to")
ImplementationContact = flag.String("contact", "", "Comma-separated list of contact information for the maintainers")
ConformanceProfiles = flag.String("conformance-profiles", "", "Comma-separated list of the conformance profiles to run")
ReportOutput = flag.String("report-output", "STDOUT", "The file where to write the conformance report")
)
1 change: 1 addition & 0 deletions conformance/utils/flags/flags.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ var (
CleanupBaseResources = flag.Bool("cleanup-base-resources", true, "Whether to cleanup base test resources after the run")
SupportedFeatures = flag.String("supported-features", "", "Supported features included in conformance tests suites")
ExemptFeatures = flag.String("exempt-features", "", "Exempt Features excluded from conformance tests suites")
SkipTests = flag.String("skip-tests", "", "Comma-separated list of conformance tests to be skipped")
EnableAllSupportedFeatures = flag.Bool("all-features", false, "Whether to enable all supported features for conformance tests")
NamespaceLabels = flag.String("namespace-labels", "", "Comma-separated list of name=value labels to add to test namespaces")
)
56 changes: 54 additions & 2 deletions conformance/utils/suite/experimental_suite.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,9 @@ limitations under the License.
package suite

import (
"errors"
"fmt"
"strings"
"sync"
"testing"
"time"
Expand Down Expand Up @@ -114,13 +116,19 @@ func NewExperimentalConformanceTestSuite(s ExperimentalConformanceOptions) (*Exp
if s.SupportedFeatures == nil {
s.SupportedFeatures = sets.New[SupportedFeature]()
}
// the use of a conformance profile implicitly enables any features of
// that profile which are supported at a Core level of support.

for _, conformanceProfileName := range s.ConformanceProfiles.UnsortedList() {
conformanceProfile, err := getConformanceProfileForName(conformanceProfileName)
if err != nil {
return nil, fmt.Errorf("failed to retrieve conformance profile: %w", err)
}
// the use of a conformance profile implicitly enables any features of
// that profile which are supported at a Core level of support.
for _, f := range conformanceProfile.CoreFeatures.UnsortedList() {
if !s.SupportedFeatures.Has(f) {
s.SupportedFeatures.Insert(f)
}
}
for _, f := range conformanceProfile.ExtendedFeatures.UnsortedList() {
if s.SupportedFeatures.Has(f) {
if suite.extendedSupportedFeatures[conformanceProfileName] == nil {
Expand Down Expand Up @@ -282,3 +290,47 @@ func (suite *ExperimentalConformanceTestSuite) Report() (*confv1a1.ConformanceRe
ProfileReports: profileReports.list(),
}, nil
}

// TODO: comment
func ParseImplementation(org, project, url, version, contact string) (*confv1a1.Implementation, error) {
if org == "" {
return nil, errors.New("implementation's organization can not be empty")
}
if project == "" {
return nil, errors.New("implementation's project can not be empty")
}
if url == "" {
return nil, errors.New("implementation's url can not be empty")
}
if version == "" {
return nil, errors.New("implementation's version can not be empty")
}
contacts := strings.SplitN(contact, ",", -1)
if len(contacts) == 0 {
return nil, errors.New("implementation's contact can not be empty")
}

// TODO: we may want to add validation for url
// TODO: we may want to add validation for version
// TODO: we may want to add validation for contact (e.g., @someone or someone@acme.org)

return &confv1a1.Implementation{
Organization: org,
Project: project,
URL: url,
Version: version,
Contact: contacts,
}, nil
}

// TODO: comment
func ParseConformanceProfiles(p string) (sets.Set[ConformanceProfileName], error) {
// if p == "" {
// return nil, errors.New("at least one conformance profile must be set")
// }
res := sets.Set[ConformanceProfileName]{}
for _, value := range strings.Split(p, ",") {
res.Insert(ConformanceProfileName(value))
}
return res, nil
}
31 changes: 0 additions & 31 deletions docker/Dockerfile.certification

This file was deleted.

7 changes: 6 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ require (
github.com/ahmetb/gen-crd-api-reference-docs v0.3.0
github.com/lithammer/dedent v1.1.0
github.com/stretchr/testify v1.8.4
gopkg.in/yaml.v2 v2.4.0
k8s.io/api v0.27.3
k8s.io/apiextensions-apiserver v0.27.3
k8s.io/apimachinery v0.27.3
Expand All @@ -18,6 +19,11 @@ require (
sigs.k8s.io/yaml v1.3.0
)

require (
go.uber.org/atomic v1.9.0 // indirect
go.uber.org/multierr v1.8.0 // indirect
)

require (
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/emicklei/go-restful/v3 v3.9.0 // indirect
Expand Down Expand Up @@ -63,7 +69,6 @@ require (
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/protobuf v1.30.0 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
k8s.io/gengo v0.0.0-20220902162205-c0856e24416d // indirect
k8s.io/klog v0.2.0 // indirect
Expand Down
9 changes: 7 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -140,15 +140,19 @@ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSS
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4=
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE=
go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8=
go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak=
go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
Expand Down Expand Up @@ -249,6 +253,7 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
Expand Down

0 comments on commit 9fb4107

Please sign in to comment.