Skip to content

Commit

Permalink
Use a separate namespace for each e2e test
Browse files Browse the repository at this point in the history 
Fixes operator-framework#1307

Create and use a new namespace for every e2e test. This means that
extension resources are placed in their own namespace. The tests
deletes the namespace, and then waits until completion. This ensures
that _most_ of an extension's resources are deleted.

It does guarantee that global resources (e.g. CRDs, CRs, CRBs) are
deleted.

And would eventually allow the tests to be run in parallel (assuming
the installed extensions allow for that).

Signed-off-by: Todd Short <tshort@redhat.com>
  • Loading branch information
@tmshort
tmshort committed Nov 6, 2024
1 parent 6bda277 commit 7b79ace
Show file tree
Hide file tree
Showing 2 changed files with 99 additions and 33 deletions.
130 changes: 97 additions & 33 deletions test/e2e/cluster_extension_install_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,11 @@ import (
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
rbacv1 "k8s.io/api/rbac/v1"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
"k8s.io/apimachinery/pkg/api/errors"
apimeta "k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/rand"
kubeclient "k8s.io/client-go/kubernetes"
Expand All @@ -38,6 +40,20 @@ const (
var pollDuration = time.Minute
var pollInterval = time.Second

func createNamespace(ctx context.Context) (*corev1.Namespace, error) {
namespaceName := fmt.Sprintf("namespace-%s", rand.String(8))
ns := &corev1.Namespace{
ObjectMeta: metav1.ObjectMeta{
Name: namespaceName,
},
}
err := c.Create(ctx, ns)
if err != nil {
return nil, err
}
return ns, nil
}

func createServiceAccount(ctx context.Context, name types.NamespacedName, clusterExtensionName string) (*corev1.ServiceAccount, error) {
sa := &corev1.ServiceAccount{
ObjectMeta: metav1.ObjectMeta{
Expand Down Expand Up @@ -177,8 +193,12 @@ func createClusterRoleAndBindingForSA(ctx context.Context, name string, sa *core
return nil
}

func testInit(t *testing.T) (*ocv1alpha1.ClusterExtension, *catalogd.ClusterCatalog, *corev1.ServiceAccount) {
func testInit(t *testing.T) (*ocv1alpha1.ClusterExtension, *catalogd.ClusterCatalog, *corev1.ServiceAccount, *corev1.Namespace) {
var err error

ns, err := createNamespace(context.Background())
require.NoError(t, err)

extensionCatalog, err := createTestCatalog(context.Background(), testCatalogName, os.Getenv(testCatalogRefEnvVar))
require.NoError(t, err)

Expand All @@ -191,30 +211,72 @@ func testInit(t *testing.T) (*ocv1alpha1.ClusterExtension, *catalogd.ClusterCata

defaultNamespace := types.NamespacedName{
Name: clusterExtensionName,
Namespace: "default",
Namespace: ns.GetName(),
}

sa, err := createServiceAccount(context.Background(), defaultNamespace, clusterExtensionName)
require.NoError(t, err)
return clusterExtension, extensionCatalog, sa
return clusterExtension, extensionCatalog, sa, ns
}

func testCleanup(t *testing.T, cat *catalogd.ClusterCatalog, clusterExtension *ocv1alpha1.ClusterExtension, sa *corev1.ServiceAccount) {
func ensureNoExtensionResources(t *testing.T, clusterExtensionName string) {
ls := labels.Set{"olm.operatorframework.io/owner-name": clusterExtensionName}

t.Logf("By waiting for CustomResourceDefinitions of %q to be deleted", clusterExtensionName)
require.EventuallyWithT(t, func(ct *assert.CollectT) {
list := &apiextensionsv1.CustomResourceDefinitionList{}
err := c.List(context.Background(), list, client.MatchingLabelsSelector{Selector: ls.AsSelector()})
assert.NoError(ct, err)
assert.Empty(ct, list.Items)
}, 2*pollDuration, pollInterval)

t.Logf("By waiting for ClusterRoleBindings of %q to be deleted", clusterExtensionName)
require.EventuallyWithT(t, func(ct *assert.CollectT) {
list := &rbacv1.ClusterRoleBindingList{}
err := c.List(context.Background(), list, client.MatchingLabelsSelector{Selector: ls.AsSelector()})
assert.NoError(ct, err)
assert.Empty(ct, list.Items)
}, 2*pollDuration, pollInterval)

t.Logf("By waiting for ClusterRoles of %q to be deleted", clusterExtensionName)
require.EventuallyWithT(t, func(ct *assert.CollectT) {
list := &rbacv1.ClusterRoleList{}
err := c.List(context.Background(), list, client.MatchingLabelsSelector{Selector: ls.AsSelector()})
assert.NoError(ct, err)
assert.Empty(ct, list.Items)
}, 2*pollDuration, pollInterval)
}

func testCleanup(t *testing.T, cat *catalogd.ClusterCatalog, clusterExtension *ocv1alpha1.ClusterExtension, sa *corev1.ServiceAccount, ns *corev1.Namespace) {
t.Logf("By deleting ClusterCatalog %q", cat.Name)
require.NoError(t, c.Delete(context.Background(), cat))
require.Eventually(t, func() bool {
err := c.Get(context.Background(), types.NamespacedName{Name: cat.Name}, &catalogd.ClusterCatalog{})
return errors.IsNotFound(err)
}, pollDuration, pollInterval)

t.Logf("By deleting ClusterExtension %q", clusterExtension.Name)
require.NoError(t, c.Delete(context.Background(), clusterExtension))
require.Eventually(t, func() bool {
err := c.Get(context.Background(), types.NamespacedName{Name: clusterExtension.Name}, &ocv1alpha1.ClusterExtension{})
return errors.IsNotFound(err)
}, pollDuration, pollInterval)

t.Logf("By deleting ServiceAccount %q", sa.Name)
require.NoError(t, c.Delete(context.Background(), sa))
require.Eventually(t, func() bool {
err := c.Get(context.Background(), types.NamespacedName{Name: sa.Name, Namespace: sa.Namespace}, &corev1.ServiceAccount{})
return errors.IsNotFound(err)
}, pollDuration, pollInterval)

t.Logf("By deleting Namespace %q", ns.Name)
require.NoError(t, c.Delete(context.Background(), ns))
require.Eventually(t, func() bool {
err := c.Get(context.Background(), types.NamespacedName{Name: ns.Name}, &corev1.Namespace{})
return errors.IsNotFound(err)
}, pollDuration, pollInterval)

ensureNoExtensionResources(t, clusterExtension.Name)
}

func TestClusterExtensionInstallRegistry(t *testing.T) {
Expand All @@ -240,8 +302,8 @@ func TestClusterExtensionInstallRegistry(t *testing.T) {
t.Log("When a cluster extension is installed from a catalog")
t.Log("When the extension bundle format is registry+v1")

clusterExtension, extensionCatalog, sa := testInit(t)
defer testCleanup(t, extensionCatalog, clusterExtension, sa)
clusterExtension, extensionCatalog, sa, ns := testInit(t)
defer testCleanup(t, extensionCatalog, clusterExtension, sa, ns)
defer getArtifactsOutput(t)

clusterExtension.Spec = ocv1alpha1.ClusterExtensionSpec{
Expand All @@ -255,7 +317,7 @@ func TestClusterExtensionInstallRegistry(t *testing.T) {
},
},
Install: ocv1alpha1.ClusterExtensionInstallConfig{
Namespace: "default",
Namespace: ns.Name,
ServiceAccount: ocv1alpha1.ServiceAccountReference{
Name: sa.Name,
},
Expand Down Expand Up @@ -298,8 +360,8 @@ func TestClusterExtensionInstallRegistry(t *testing.T) {
func TestClusterExtensionInstallRegistryMultipleBundles(t *testing.T) {
t.Log("When a cluster extension is installed from a catalog")

clusterExtension, extensionCatalog, sa := testInit(t)
defer testCleanup(t, extensionCatalog, clusterExtension, sa)
clusterExtension, extensionCatalog, sa, ns := testInit(t)
defer testCleanup(t, extensionCatalog, clusterExtension, sa, ns)
defer getArtifactsOutput(t)

clusterExtension.Spec = ocv1alpha1.ClusterExtensionSpec{
Expand All @@ -310,7 +372,7 @@ func TestClusterExtensionInstallRegistryMultipleBundles(t *testing.T) {
},
},
Install: ocv1alpha1.ClusterExtensionInstallConfig{
Namespace: "default",
Namespace: ns.Name,
ServiceAccount: ocv1alpha1.ServiceAccountReference{
Name: sa.Name,
},
Expand Down Expand Up @@ -341,8 +403,8 @@ func TestClusterExtensionBlockInstallNonSuccessorVersion(t *testing.T) {
t.Log("When a cluster extension is installed from a catalog")
t.Log("When resolving upgrade edges")

clusterExtension, extensionCatalog, sa := testInit(t)
defer testCleanup(t, extensionCatalog, clusterExtension, sa)
clusterExtension, extensionCatalog, sa, ns := testInit(t)
defer testCleanup(t, extensionCatalog, clusterExtension, sa, ns)
defer getArtifactsOutput(t)

t.Log("By creating an ClusterExtension at a specified version")
Expand All @@ -356,7 +418,7 @@ func TestClusterExtensionBlockInstallNonSuccessorVersion(t *testing.T) {
},
},
Install: ocv1alpha1.ClusterExtensionInstallConfig{
Namespace: "default",
Namespace: ns.Name,
ServiceAccount: ocv1alpha1.ServiceAccountReference{
Name: sa.Name,
},
Expand Down Expand Up @@ -406,8 +468,8 @@ func TestClusterExtensionForceInstallNonSuccessorVersion(t *testing.T) {
t.Log("When a cluster extension is installed from a catalog")
t.Log("When resolving upgrade edges")

clusterExtension, extensionCatalog, sa := testInit(t)
defer testCleanup(t, extensionCatalog, clusterExtension, sa)
clusterExtension, extensionCatalog, sa, ns := testInit(t)
defer testCleanup(t, extensionCatalog, clusterExtension, sa, ns)
defer getArtifactsOutput(t)

t.Log("By creating an ClusterExtension at a specified version")
Expand All @@ -420,7 +482,7 @@ func TestClusterExtensionForceInstallNonSuccessorVersion(t *testing.T) {
},
},
Install: ocv1alpha1.ClusterExtensionInstallConfig{
Namespace: "default",
Namespace: ns.Name,
ServiceAccount: ocv1alpha1.ServiceAccountReference{
Name: sa.Name,
},
Expand Down Expand Up @@ -457,8 +519,8 @@ func TestClusterExtensionForceInstallNonSuccessorVersion(t *testing.T) {
func TestClusterExtensionInstallSuccessorVersion(t *testing.T) {
t.Log("When a cluster extension is installed from a catalog")
t.Log("When resolving upgrade edges")
clusterExtension, extensionCatalog, sa := testInit(t)
defer testCleanup(t, extensionCatalog, clusterExtension, sa)
clusterExtension, extensionCatalog, sa, ns := testInit(t)
defer testCleanup(t, extensionCatalog, clusterExtension, sa, ns)
defer getArtifactsOutput(t)

t.Log("By creating an ClusterExtension at a specified version")
Expand All @@ -471,7 +533,7 @@ func TestClusterExtensionInstallSuccessorVersion(t *testing.T) {
},
},
Install: ocv1alpha1.ClusterExtensionInstallConfig{
Namespace: "default",
Namespace: ns.Name,
ServiceAccount: ocv1alpha1.ServiceAccountReference{
Name: sa.Name,
},
Expand Down Expand Up @@ -507,8 +569,8 @@ func TestClusterExtensionInstallSuccessorVersion(t *testing.T) {
func TestClusterExtensionInstallReResolvesWhenCatalogIsPatched(t *testing.T) {
t.Log("When a cluster extension is installed from a catalog")
t.Log("It resolves again when a catalog is patched with new ImageRef")
clusterExtension, extensionCatalog, sa := testInit(t)
defer testCleanup(t, extensionCatalog, clusterExtension, sa)
clusterExtension, extensionCatalog, sa, ns := testInit(t)
defer testCleanup(t, extensionCatalog, clusterExtension, sa, ns)
defer getArtifactsOutput(t)

clusterExtension.Spec = ocv1alpha1.ClusterExtensionSpec{
Expand All @@ -528,7 +590,7 @@ func TestClusterExtensionInstallReResolvesWhenCatalogIsPatched(t *testing.T) {
},
},
Install: ocv1alpha1.ClusterExtensionInstallConfig{
Namespace: "default",
Namespace: ns.Name,
ServiceAccount: ocv1alpha1.ServiceAccountReference{
Name: sa.Name,
},
Expand Down Expand Up @@ -593,9 +655,11 @@ func TestClusterExtensionInstallReResolvesWhenNewCatalog(t *testing.T) {
Name: clusterExtensionName,
},
}
sa, err := createServiceAccount(context.Background(), types.NamespacedName{Name: clusterExtensionName, Namespace: "default"}, clusterExtensionName)
ns, err := createNamespace(context.Background())
require.NoError(t, err)
defer testCleanup(t, extensionCatalog, clusterExtension, sa)
sa, err := createServiceAccount(context.Background(), types.NamespacedName{Name: clusterExtensionName, Namespace: ns.Name}, clusterExtensionName)
require.NoError(t, err)
defer testCleanup(t, extensionCatalog, clusterExtension, sa, ns)
defer getArtifactsOutput(t)

clusterExtension.Spec = ocv1alpha1.ClusterExtensionSpec{
Expand All @@ -609,7 +673,7 @@ func TestClusterExtensionInstallReResolvesWhenNewCatalog(t *testing.T) {
},
},
Install: ocv1alpha1.ClusterExtensionInstallConfig{
Namespace: "default",
Namespace: ns.Name,
ServiceAccount: ocv1alpha1.ServiceAccountReference{
Name: sa.Name,
},
Expand Down Expand Up @@ -657,8 +721,8 @@ func TestClusterExtensionInstallReResolvesWhenNewCatalog(t *testing.T) {
func TestClusterExtensionInstallReResolvesWhenManagedContentChanged(t *testing.T) {
t.Log("When a cluster extension is installed from a catalog")
t.Log("It resolves again when managed content is changed")
clusterExtension, extensionCatalog, sa := testInit(t)
defer testCleanup(t, extensionCatalog, clusterExtension, sa)
clusterExtension, extensionCatalog, sa, ns := testInit(t)
defer testCleanup(t, extensionCatalog, clusterExtension, sa, ns)
defer getArtifactsOutput(t)

clusterExtension.Spec = ocv1alpha1.ClusterExtensionSpec{
Expand All @@ -672,7 +736,7 @@ func TestClusterExtensionInstallReResolvesWhenManagedContentChanged(t *testing.T
},
},
Install: ocv1alpha1.ClusterExtensionInstallConfig{
Namespace: "default",
Namespace: ns.Name,
ServiceAccount: ocv1alpha1.ServiceAccountReference{
Name: sa.Name,
},
Expand Down Expand Up @@ -712,18 +776,18 @@ func TestClusterExtensionRecoversFromInitialInstallFailedWhenFailureFixed(t *tes
t.Log("When a cluster extension is installed from a catalog")
t.Log("When the extension bundle format is registry+v1")

clusterExtension, extensionCatalog, _ := testInit(t)
clusterExtension, extensionCatalog, _, ns := testInit(t)

name := rand.String(10)
sa := &corev1.ServiceAccount{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: "default",
Namespace: ns.Name,
},
}
err := c.Create(context.Background(), sa)
require.NoError(t, err)

defer testCleanup(t, extensionCatalog, clusterExtension, sa)
defer testCleanup(t, extensionCatalog, clusterExtension, sa, ns)
defer getArtifactsOutput(t)

clusterExtension.Spec = ocv1alpha1.ClusterExtensionSpec{
Expand All @@ -737,7 +801,7 @@ func TestClusterExtensionRecoversFromInitialInstallFailedWhenFailureFixed(t *tes
},
},
Install: ocv1alpha1.ClusterExtensionInstallConfig{
Namespace: "default",
Namespace: ns.Name,
ServiceAccount: ocv1alpha1.ServiceAccountReference{
Name: sa.Name,
},
Expand Down
2 changes: 2 additions & 0 deletions test/e2e/e2e_suite_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"testing"
"time"

apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/client-go/rest"
Expand All @@ -32,6 +33,7 @@ func TestMain(m *testing.M) {
cfg = ctrl.GetConfigOrDie()

var err error
utilruntime.Must(apiextensionsv1.AddToScheme(scheme.Scheme))
c, err = client.New(cfg, client.Options{Scheme: scheme.Scheme})
utilruntime.Must(err)

Expand Down

0 comments on commit 7b79ace

Please sign in to comment.