Skip to content

Commit

Permalink
Consolidate api clients for e2e tests (vmware-tanzu#3764)
Browse files Browse the repository at this point in the history
* Consolidate api clients
* Adress Nolan reviews
* Adding back output warning for consistency
* Remove unnecessary documentation
* Address Bridget's reviews
* Update go.sum files

Signed-off-by: Carlisia <carlisia@grokkingtech.io>
Co-authored-by: Bridget McErlean <bmcerlean@vmware.com>
  • Loading branch information
Carlisia Thompson and zubron authored Jun 9, 2021
1 parent 4ce33c5 commit 81f1f21
Show file tree
Hide file tree
Showing 14 changed files with 210 additions and 148 deletions.
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ require (
github.com/hashicorp/go-plugin v0.0.0-20190610192547-a1bc61569a26
github.com/joho/godotenv v1.3.0
github.com/kubernetes-csi/external-snapshotter/client/v4 v4.0.0
github.com/onsi/ginkgo v1.16.1
github.com/onsi/ginkgo v1.16.2
github.com/onsi/gomega v1.10.2
github.com/pkg/errors v0.9.1
github.com/prometheus/client_golang v1.7.1
Expand Down
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -433,8 +433,8 @@ github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W
github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
github.com/onsi/ginkgo v1.14.1/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
github.com/onsi/ginkgo v1.16.1 h1:foqVmeWDD6yYpK+Yz3fHyNIxFYNxswxqNFjSKe+vI54=
github.com/onsi/ginkgo v1.16.1/go.mod h1:CObGmKUOKaSC0RjmoAK7tKyn4Azo5P2IWuoMnvwxz1E=
github.com/onsi/ginkgo v1.16.2 h1:HFB2fbVIlhIfCfOW81bZFbiC/RvnpXSdhbF2/DJr134=
github.com/onsi/ginkgo v1.16.2/go.mod h1:CObGmKUOKaSC0RjmoAK7tKyn4Azo5P2IWuoMnvwxz1E=
github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
Expand Down
7 changes: 6 additions & 1 deletion pkg/client/factory.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@ package client
import (
"os"

apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
k8scheme "k8s.io/client-go/kubernetes/scheme"
kbclient "sigs.k8s.io/controller-runtime/pkg/client"

"github.com/pkg/errors"
Expand All @@ -45,7 +47,8 @@ type Factory interface {
// DynamicClient returns a Kubernetes dynamic client. It uses the following priority to specify the cluster
// configuration: --kubeconfig flag, KUBECONFIG environment variable, in-cluster configuration.
DynamicClient() (dynamic.Interface, error)
// KubebuilderClient returns a Kubernetes dynamic client. It uses the following priority to specify the cluster
// KubebuilderClient returns a client for the controller runtime framework. It adds Kubernetes and Velero
// types to its scheme. It uses the following priority to specify the cluster
// configuration: --kubeconfig flag, KUBECONFIG environment variable, in-cluster configuration.
KubebuilderClient() (kbclient.Client, error)
// SetBasename changes the basename for an already-constructed client.
Expand Down Expand Up @@ -151,6 +154,8 @@ func (f *factory) KubebuilderClient() (kbclient.Client, error) {

scheme := runtime.NewScheme()
velerov1api.AddToScheme(scheme)
k8scheme.AddToScheme(scheme)
apiextv1beta1.AddToScheme(scheme)
kubebuilderClient, err := kbclient.New(clientConfig, kbclient.Options{
Scheme: scheme,
})
Expand Down
69 changes: 41 additions & 28 deletions pkg/cmd/cli/uninstall/uninstall.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,20 +26,18 @@ import (
"github.com/spf13/pflag"

corev1 "k8s.io/api/core/v1"
apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
kubeerrs "k8s.io/apimachinery/pkg/util/errors"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/kubernetes"

apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
kbclient "sigs.k8s.io/controller-runtime/pkg/client"

"github.com/vmware-tanzu/velero/pkg/client"
"github.com/vmware-tanzu/velero/pkg/cmd"
"github.com/vmware-tanzu/velero/pkg/cmd/cli"
"github.com/vmware-tanzu/velero/pkg/install"
"github.com/vmware-tanzu/velero/pkg/util/kube"
)

// uninstallOptions collects all the options for uninstalling Velero from a Kubernetes cluster.
Expand Down Expand Up @@ -78,9 +76,9 @@ Use '--force' to skip the prompt confirming if you want to uninstall Velero.
}
}

client, extCl, err := kube.GetClusterClient()
kbClient, err := f.KubebuilderClient()
cmd.CheckError(err)
cmd.CheckError(Run(context.Background(), client, extCl, f.Namespace(), o.wait))
cmd.CheckError(Run(context.Background(), kbClient, f.Namespace(), o.wait))
},
}

Expand All @@ -89,53 +87,68 @@ Use '--force' to skip the prompt confirming if you want to uninstall Velero.
}

// Run removes all components that were deployed using the Velero install command
func Run(ctx context.Context, client *kubernetes.Clientset, extensionsClient *apiextensionsclientset.Clientset, namespace string, waitToTerminate bool) error {
func Run(ctx context.Context, kbClient kbclient.Client, namespace string, waitToTerminate bool) error {
var errs []error

// namespace
ns, err := client.CoreV1().Namespaces().Get(ctx, namespace, metav1.GetOptions{})
if err != nil {
ns := &corev1.Namespace{}
key := kbclient.ObjectKey{Name: namespace}
if err := kbClient.Get(ctx, key, ns); err != nil {
if apierrors.IsNotFound(err) {
fmt.Printf("Velero installation namespace %q does not exist, skipping.\n", namespace)
fmt.Printf("Velero namespace %q does not exist, skipping.\n", namespace)
} else {
errs = append(errs, errors.WithStack(err))
}
} else {
if ns.Status.Phase == corev1.NamespaceTerminating {
fmt.Printf("Velero installation namespace %q is terminating.\n", namespace)
fmt.Printf("Velero namespace %q is terminating.\n", namespace)
} else {
err = client.CoreV1().Namespaces().Delete(ctx, ns.Name, metav1.DeleteOptions{})
if err != nil {
if err := kbClient.Delete(ctx, ns); err != nil {
errs = append(errs, errors.WithStack(err))
}
}
}

// rolebinding
// ClusterRoleBinding
crb := install.ClusterRoleBinding(namespace)
if err := client.RbacV1().ClusterRoleBindings().Delete(ctx, crb.Name, metav1.DeleteOptions{}); err != nil {
key = kbclient.ObjectKey{Name: crb.Name}
if err := kbClient.Get(ctx, key, crb); err != nil {
if apierrors.IsNotFound(err) {
fmt.Printf("Velero installation clusterrolebinding %q does not exist, skipping.\n", crb.Name)
fmt.Printf("Velero ClusterRoleBinding %q does not exist, skipping.\n", crb.Name)
} else {
errs = append(errs, errors.WithStack(err))
}
} else {
if err := kbClient.Delete(ctx, crb); err != nil {
errs = append(errs, errors.WithStack(err))
}
}

// CRDs
veleroLabels := labels.FormatLabels(install.Labels())
crds, err := extensionsClient.ApiextensionsV1().CustomResourceDefinitions().List(ctx, metav1.ListOptions{
LabelSelector: veleroLabels,
})
if err != nil {
errs = append(errs, errors.WithStack(err))
crdList := apiextv1beta1.CustomResourceDefinitionList{}
opts := kbclient.ListOptions{
Namespace: namespace,
Raw: &metav1.ListOptions{
LabelSelector: veleroLabels,
},
}
if len(crds.Items) == 0 {
fmt.Print("Velero CRDs do not exist, skipping.\n")
if err := kbClient.List(context.Background(), &crdList, &opts); err != nil {
errs = append(errs, errors.WithStack(err))
} else {
for _, removeCRD := range crds.Items {
if err = extensionsClient.ApiextensionsV1().CustomResourceDefinitions().Delete(ctx, removeCRD.ObjectMeta.Name, metav1.DeleteOptions{}); err != nil {
err2 := errors.WithMessagef(err, "Uninstall failed removing CRD %s", removeCRD.ObjectMeta.Name)
errs = append(errs, errors.WithStack(err2))
if len(crdList.Items) == 0 {
fmt.Print("Velero CRDs do not exist, skipping.\n")
} else {
veleroLabelSelector := labels.SelectorFromSet(install.Labels())
opts := []kbclient.DeleteAllOfOption{
kbclient.InNamespace(namespace),
kbclient.MatchingLabelsSelector{
Selector: veleroLabelSelector,
},
}
crd := &apiextv1beta1.CustomResourceDefinition{}
if err := kbClient.DeleteAllOf(ctx, crd, opts...); err != nil {
errs = append(errs, errors.WithStack(err))
}
}
}
Expand All @@ -147,7 +160,7 @@ func Run(ctx context.Context, client *kubernetes.Clientset, extensionsClient *ap
defer cancel()

checkFunc := func() {
_, err := client.CoreV1().Namespaces().Get(ctx, namespace, metav1.GetOptions{})
err := kbClient.Get(ctx, key, ns)
if err != nil {
if apierrors.IsNotFound(err) {
fmt.Print("\n")
Expand Down
26 changes: 0 additions & 26 deletions pkg/util/kube/utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,15 +24,12 @@ import (
"github.com/pkg/errors"
corev1api "k8s.io/api/core/v1"
apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/kubernetes"
corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
corev1listers "k8s.io/client-go/listers/core/v1"
"k8s.io/client-go/tools/clientcmd"
)

// NamespaceAndName returns a string in the format <namespace>/<name>
Expand Down Expand Up @@ -220,26 +217,3 @@ func IsUnstructuredCRDReady(crd *unstructured.Unstructured) (bool, error) {

return (isEstablished && namesAccepted), nil
}

// GetClusterClient instantiates and returns a client for the cluster.
func GetClusterClient() (*kubernetes.Clientset, *apiextensionsclientset.Clientset, error) {
loadingRules := clientcmd.NewDefaultClientConfigLoadingRules()
configOverrides := &clientcmd.ConfigOverrides{}
kubeConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, configOverrides)
clientConfig, err := kubeConfig.ClientConfig()
if err != nil {
return nil, nil, errors.WithStack(err)
}

client, err := kubernetes.NewForConfig(clientConfig)
if err != nil {
return nil, nil, errors.WithStack(err)
}

extensionClientSet, err := apiextensionsclientset.NewForConfig(clientConfig)
if err != nil {
return nil, nil, errors.WithStack(err)
}

return client, extensionClientSet, nil
}
2 changes: 1 addition & 1 deletion test/e2e/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ OUTPUT_DIR := _output/$(GOOS)/$(GOARCH)/bin
GINKGO_FOCUS ?=
VELERO_CLI ?=$$(pwd)/../../_output/bin/$(GOOS)/$(GOARCH)/velero
VELERO_IMAGE ?= velero/velero:main
VELERO_NAMESPACE ?=
VELERO_NAMESPACE ?= velero
CREDS_FILE ?=
BSL_BUCKET ?=
BSL_PREFIX ?=
Expand Down
8 changes: 8 additions & 0 deletions test/e2e/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -105,3 +105,11 @@ For example, E2E tests can be run from Velero repository roots using the command
Velero E2E tests uses [Ginkgo](https://onsi.github.io/ginkgo/) testing framework which allows a subset of the tests to be run using the [`-focus` and `-skip`](https://onsi.github.io/ginkgo/#focused-specs) flags to ginkgo.

The `-focus` flag is passed to ginkgo using the `GINKGO_FOCUS` make variable. This can be used to focus on specific tests.

## Adding tests

### API clients
When adding a test, aim to instantiate an API client only once at the beginning of the test. There is a constructor `newTestClient` that facilitates the configuration and instantiation of clients. Also, please use the `kubebuilder` runtime controller client for any new test, as we will phase out usage of `client-go` API clients.

### Tips
Look for the ⛵ emoji printed at the end of each install and uninstall log. There should not be two install/unintall in a row, and there should be tests between an install and an uninstall.
24 changes: 8 additions & 16 deletions test/e2e/backup_test.go
Original file line number Diff line number Diff line change
@@ -1,31 +1,28 @@
/*
Copyright the Velero contributors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
"context"
"flag"
"fmt"
"time"

"github.com/google/uuid"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
"k8s.io/client-go/kubernetes"

"github.com/vmware-tanzu/velero/pkg/util/kube"
)

var (
Expand All @@ -47,12 +44,12 @@ func backup_restore_with_restic() {

func backup_restore_test(useVolumeSnapshots bool) {
var (
client *kubernetes.Clientset
extensionsClient *apiextensionsclientset.Clientset
backupName string
restoreName string
backupName, restoreName string
)

client, err := newTestClient()
Expect(err).To(Succeed(), "Failed to instantiate cluster client for backup tests")

BeforeEach(func() {
if useVolumeSnapshots && cloudProvider == "kind" {
Skip("Volume snapshots not supported on kind")
Expand All @@ -64,19 +61,14 @@ func backup_restore_test(useVolumeSnapshots bool) {
if installVelero {
Expect(veleroInstall(context.Background(), veleroImage, veleroNamespace, cloudProvider, objectStoreProvider, useVolumeSnapshots,
cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, vslConfig, "")).To(Succeed())

}
client, extensionsClient, err = kube.GetClusterClient()
Expect(err).To(Succeed(), "Failed to instantiate cluster client")
})

AfterEach(func() {
if installVelero {
timeoutCTX, _ := context.WithTimeout(context.Background(), time.Minute)
err := veleroUninstall(timeoutCTX, client, extensionsClient, veleroNamespace)
err = veleroUninstall(context.Background(), client.kubebuilder, installVelero, veleroNamespace)
Expect(err).To(Succeed())
}

})

When("kibishii is the sample workload", func() {
Expand Down
79 changes: 79 additions & 0 deletions test/e2e/client.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
/*
Copyright the Velero contributors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
"k8s.io/client-go/kubernetes"
kbclient "sigs.k8s.io/controller-runtime/pkg/client"

"github.com/vmware-tanzu/velero/pkg/client"
)

// testClient contains different API clients that are in use throughout
// the e2e tests.

type testClient struct {
kubebuilder kbclient.Client

// clientGo returns a client-go API client.
//
// Deprecated, TODO(2.0): presuming all controllers and resources are converted to the
// controller runtime framework by v2.0, it is the intent to remove all
// client-go API clients. Please use the controller runtime to make API calls for tests.
clientGo kubernetes.Interface

// dynamicFactory returns a client-go API client for retrieving dynamic clients
// for GroupVersionResources and GroupVersionKinds.
//
// Deprecated, TODO(2.0): presuming all controllers and resources are converted to the
// controller runtime framework by v2.0, it is the intent to remove all
// client-go API clients. Please use the controller runtime to make API calls for tests.
dynamicFactory client.DynamicFactory
}

// newTestClient returns a set of ready-to-use API clients.
func newTestClient() (testClient, error) {
config, err := client.LoadConfig()
if err != nil {
return testClient{}, err
}

f := client.NewFactory("e2e", config)

clientGo, err := f.KubeClient()
if err != nil {
return testClient{}, err
}

kb, err := f.KubebuilderClient()
if err != nil {
return testClient{}, err
}

dynamicClient, err := f.DynamicClient()
if err != nil {
return testClient{}, err
}

factory := client.NewDynamicFactory(dynamicClient)

return testClient{
kubebuilder: kb,
clientGo: clientGo,
dynamicFactory: factory,
}, nil
}
Loading

0 comments on commit 81f1f21

Please sign in to comment.