Merge pull request #771 from andyzhangx/restart-driver-test

test: add restart driver e2e test

Author: andyzhangx

test/e2e/driver/blob_csi_driver.go

@@ -39,9 +39,9 @@ func InitBlobCSIDriver() PVTestDriver {
 	}
 }
 
-func (d *blobCSIDriver) GetDynamicProvisionStorageClass(parameters map[string]string, mountOptions []string, reclaimPolicy *v1.PersistentVolumeReclaimPolicy, bindingMode *storagev1.VolumeBindingMode, allowedTopologyValues []string, namespace string) *storagev1.StorageClass {
+func (d *blobCSIDriver) GetProvisionStorageClass(parameters map[string]string, mountOptions []string, reclaimPolicy *v1.PersistentVolumeReclaimPolicy, bindingMode *storagev1.VolumeBindingMode, allowedTopologyValues []string, namespace string) *storagev1.StorageClass {
 	provisioner := d.driverName
-	generateName := fmt.Sprintf("%s-%s-dynamic-sc-", namespace, provisioner)
+	generateName := fmt.Sprintf("%s-%s-sc-", namespace, provisioner)
 	return getStorageClass(generateName, provisioner, parameters, mountOptions, reclaimPolicy, bindingMode, nil)
 }
 
@@ -87,9 +87,3 @@ func (d *blobCSIDriver) GetPersistentVolume(volumeID string, fsType string, size
 		},
 	}
 }
-
-func (d *blobCSIDriver) GetPreProvisionStorageClass(parameters map[string]string, mountOptions []string, reclaimPolicy *v1.PersistentVolumeReclaimPolicy, bindingMode *storagev1.VolumeBindingMode, allowedTopologyValues []string, namespace string) *storagev1.StorageClass {
-	provisioner := d.driverName
-	generateName := fmt.Sprintf("%s-%s-pre-provisioned-sc-", namespace, provisioner)
-	return getStorageClass(generateName, provisioner, parameters, mountOptions, reclaimPolicy, bindingMode, nil)
-}

test/e2e/driver/driver.go

@@ -29,16 +29,16 @@ type PVTestDriver interface {
 
 // DynamicPVTestDriver represents an interface for a CSI driver that supports DynamicPV
 type DynamicPVTestDriver interface {
-	// GetDynamicProvisionStorageClass returns a StorageClass dynamic provision Persistent Volume
-	GetDynamicProvisionStorageClass(parameters map[string]string, mountOptions []string, reclaimPolicy *v1.PersistentVolumeReclaimPolicy, bindingMode *storagev1.VolumeBindingMode, allowedTopologyValues []string, namespace string) *storagev1.StorageClass
+	// GetProvisionStorageClass returns a StorageClass dynamic provision Persistent Volume
+	GetProvisionStorageClass(parameters map[string]string, mountOptions []string, reclaimPolicy *v1.PersistentVolumeReclaimPolicy, bindingMode *storagev1.VolumeBindingMode, allowedTopologyValues []string, namespace string) *storagev1.StorageClass
 }
 
 // PreProvisionedVolumeTestDriver represents an interface for a CSI driver that supports pre-provisioned volume
 type PreProvisionedVolumeTestDriver interface {
 	// GetPersistentVolume returns a PersistentVolume with pre-provisioned volumeHandle
 	GetPersistentVolume(volumeID string, fsType string, size string, reclaimPolicy *v1.PersistentVolumeReclaimPolicy, namespace string, attrib map[string]string, nodeStageSecretRef string) *v1.PersistentVolume
-	// GetPreProvisionStorageClass returns a StorageClass with existing container
-	GetPreProvisionStorageClass(parameters map[string]string, mountOptions []string, reclaimPolicy *v1.PersistentVolumeReclaimPolicy, bindingMode *storagev1.VolumeBindingMode, allowedTopologyValues []string, namespace string) *storagev1.StorageClass
+	// GetProvisionStorageClass returns a StorageClass with existing container
+	GetProvisionStorageClass(parameters map[string]string, mountOptions []string, reclaimPolicy *v1.PersistentVolumeReclaimPolicy, bindingMode *storagev1.VolumeBindingMode, allowedTopologyValues []string, namespace string) *storagev1.StorageClass
 }
 
 func getStorageClass(

test/e2e/pre_provisioning_test.go

@@ -19,6 +19,7 @@ package e2e
 import (
 	"context"
 	"fmt"
+	"os"
 	"time"
 
 	"sigs.k8s.io/blob-csi-driver/test/e2e/driver"
@@ -330,6 +331,94 @@ var _ = ginkgo.Describe("[blob-csi-e2e] Pre-Provisioned", func() {
 		}
 		test.Run(cs, ns)
 	})
+
+	ginkgo.It("nfs volume mount is still valid after driver restart [blob.csi.azure.com]", func() {
+		// print driver logs before driver restart
+		blobLog := testCmd{
+			command:  "bash",
+			args:     []string{"test/utils/blob_log.sh"},
+			startLog: "===================blob log (before restart)===================",
+			endLog:   "====================================================================",
+		}
+		execTestCmd([]testCmd{blobLog})
+
+		pod := testsuites.PodDetails{
+			Cmd: "echo 'hello world' >> /mnt/test-1/data && while true; do sleep 3600; done",
+			Volumes: []testsuites.VolumeDetails{
+				{
+					ClaimSize: "10Gi",
+					VolumeMount: testsuites.VolumeMountDetails{
+						NameGenerate:      "test-volume-",
+						MountPathGenerate: "/mnt/test-",
+					},
+				},
+			},
+		}
+
+		podCheckCmd := []string{"cat", "/mnt/test-1/data"}
+		expectedString := "hello world\n"
+		test := testsuites.DynamicallyProvisionedRestartDriverTest{
+			CSIDriver: testDriver,
+			Pod:       pod,
+			PodCheck: &testsuites.PodExecCheck{
+				Cmd:            podCheckCmd,
+				ExpectedString: expectedString,
+			},
+			StorageClassParameters: map[string]string{"protocol": "nfs"},
+			RestartDriverFunc: func() {
+				restartDriver := testCmd{
+					command:  "bash",
+					args:     []string{"test/utils/restart_driver_daemonset.sh"},
+					startLog: "Restart driver node daemonset ...",
+					endLog:   "Restart driver node daemonset done successfully",
+				}
+				execTestCmd([]testCmd{restartDriver})
+			},
+		}
+		test.Run(cs, ns)
+	})
+
+	ginkgo.It("blobfuse volume mount is still valid after driver restart [blob.csi.azure.com]", func() {
+		_, useBlobfuseProxy := os.LookupEnv("ENABLE_BLOBFUSE_PROXY")
+		if !useBlobfuseProxy {
+			ginkgo.Skip("skip this test since blobfuse-proxy is not enabled")
+		}
+
+		pod := testsuites.PodDetails{
+			Cmd: "echo 'hello world' >> /mnt/test-1/data && while true; do sleep 3600; done",
+			Volumes: []testsuites.VolumeDetails{
+				{
+					ClaimSize: "10Gi",
+					VolumeMount: testsuites.VolumeMountDetails{
+						NameGenerate:      "test-volume-",
+						MountPathGenerate: "/mnt/test-",
+					},
+				},
+			},
+		}
+
+		podCheckCmd := []string{"cat", "/mnt/test-1/data"}
+		expectedString := "hello world\n"
+		test := testsuites.DynamicallyProvisionedRestartDriverTest{
+			CSIDriver: testDriver,
+			Pod:       pod,
+			PodCheck: &testsuites.PodExecCheck{
+				Cmd:            podCheckCmd,
+				ExpectedString: expectedString,
+			},
+			StorageClassParameters: map[string]string{"skuName": "Standard_LRS"},
+			RestartDriverFunc: func() {
+				restartDriver := testCmd{
+					command:  "bash",
+					args:     []string{"test/utils/restart_driver_daemonset.sh"},
+					startLog: "Restart driver node daemonset ...",
+					endLog:   "Restart driver node daemonset done successfully",
+				}
+				execTestCmd([]testCmd{restartDriver})
+			},
+		}
+		test.Run(cs, ns)
+	})
 })
 
 func makeCreateVolumeReq(volumeName, secretNamespace string) *csi.CreateVolumeRequest {

test/e2e/suite_test.go

@@ -133,19 +133,13 @@ var _ = ginkgo.AfterSuite(func() {
 	}
 	execTestCmd([]testCmd{createExampleDeployment})
 
-	blobLog := testCmd{
-		command:  "bash",
-		args:     []string{"test/utils/blob_log.sh"},
-		startLog: "===================blob log===================",
-		endLog:   "==================================================",
-	}
 	e2eTeardown := testCmd{
 		command:  "make",
 		args:     []string{"e2e-teardown"},
 		startLog: "Uninstalling Azure Blob Storage CSI driver...",
 		endLog:   "Azure Blob Storage CSI driver uninstalled",
 	}
-	execTestCmd([]testCmd{blobLog, e2eTeardown})
+	execTestCmd([]testCmd{e2eTeardown})
 
 	// install/uninstall CSI Driver deployment scripts test
 	installDriver := testCmd{

test/e2e/testsuites/dynamically_provisioned_restart_driver_tester.go

@@ -0,0 +1,64 @@
+/*
+Copyright 2021 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package testsuites
+
+import (
+	"github.com/onsi/ginkgo"
+	v1 "k8s.io/api/core/v1"
+	clientset "k8s.io/client-go/kubernetes"
+
+	"sigs.k8s.io/blob-csi-driver/test/e2e/driver"
+)
+
+// DynamicallyProvisionedRestartDriverTest will test to ensure that restarting driver doesn't affect pod mounting.
+// It will mount a pod, restart the driver daemonset and ensure that the pod still has access to original volume.
+type DynamicallyProvisionedRestartDriverTest struct {
+	CSIDriver              driver.DynamicPVTestDriver
+	Pod                    PodDetails
+	PodCheck               *PodExecCheck
+	StorageClassParameters map[string]string
+	RestartDriverFunc      func()
+}
+
+func (t *DynamicallyProvisionedRestartDriverTest) Run(client clientset.Interface, namespace *v1.Namespace) {
+	tDeployment, cleanup := t.Pod.SetupDeployment(client, namespace, t.CSIDriver, t.StorageClassParameters)
+	// defer must be called here for resources not get removed before using them
+	for i := range cleanup {
+		defer cleanup[i]()
+	}
+
+	ginkgo.By("creating the deployment for the pod")
+	tDeployment.Create()
+
+	ginkgo.By("checking that the pod is running")
+	tDeployment.WaitForPodReady()
+
+	if t.PodCheck != nil {
+		ginkgo.By("checking if pod is able to access volume")
+		tDeployment.PollForStringInPodsExec(t.PodCheck.Cmd, t.PodCheck.ExpectedString)
+	}
+
+	// restart the driver
+	ginkgo.By("restarting the driver daemonset")
+	t.RestartDriverFunc()
+
+	// check if original pod could still access volume
+	if t.PodCheck != nil {
+		ginkgo.By("checking if pod still has access to volume after driver restart")
+		tDeployment.PollForStringInPodsExec(t.PodCheck.Cmd, t.PodCheck.ExpectedString)
+	}
+}

test/e2e/testsuites/pre_provisioned_existing_credentials_tester.go

@@ -51,7 +51,7 @@ func (t *PreProvisionedExistingCredentialsTest) Run(client clientset.Interface,
 			}
 
 			ginkgo.By("creating the storageclass with existing credentials")
-			sc := t.CSIDriver.GetPreProvisionStorageClass(parameters, volume.MountOptions, volume.ReclaimPolicy, volume.VolumeBindingMode, volume.AllowedTopologyValues, namespace.Name)
+			sc := t.CSIDriver.GetProvisionStorageClass(parameters, volume.MountOptions, volume.ReclaimPolicy, volume.VolumeBindingMode, volume.AllowedTopologyValues, namespace.Name)
 			tsc := NewTestStorageClass(client, namespace, sc)
 			createdStorageClass := tsc.Create()
 			defer tsc.Cleanup()

test/e2e/testsuites/specs.go

@@ -152,7 +152,7 @@ func (pod *PodDetails) SetupDeployment(client clientset.Interface, namespace *v1
 	cleanupFuncs := make([]func(), 0)
 	volume := pod.Volumes[0]
 	ginkgo.By("setting up the StorageClass")
-	storageClass := csiDriver.GetDynamicProvisionStorageClass(storageClassParameters, volume.MountOptions, volume.ReclaimPolicy, volume.VolumeBindingMode, volume.AllowedTopologyValues, namespace.Name)
+	storageClass := csiDriver.GetProvisionStorageClass(storageClassParameters, volume.MountOptions, volume.ReclaimPolicy, volume.VolumeBindingMode, volume.AllowedTopologyValues, namespace.Name)
 	tsc := NewTestStorageClass(client, namespace, storageClass)
 	createdStorageClass := tsc.Create()
 	cleanupFuncs = append(cleanupFuncs, tsc.Cleanup)
@@ -172,7 +172,7 @@ func (pod *PodDetails) SetupDeployment(client clientset.Interface, namespace *v1
 func (volume *VolumeDetails) SetupDynamicPersistentVolumeClaim(client clientset.Interface, namespace *v1.Namespace, csiDriver driver.DynamicPVTestDriver, storageClassParameters map[string]string) (*TestPersistentVolumeClaim, []func()) {
 	cleanupFuncs := make([]func(), 0)
 	ginkgo.By("setting up the StorageClass")
-	storageClass := csiDriver.GetDynamicProvisionStorageClass(storageClassParameters, volume.MountOptions, volume.ReclaimPolicy, volume.VolumeBindingMode, volume.AllowedTopologyValues, namespace.Name)
+	storageClass := csiDriver.GetProvisionStorageClass(storageClassParameters, volume.MountOptions, volume.ReclaimPolicy, volume.VolumeBindingMode, volume.AllowedTopologyValues, namespace.Name)
 	tsc := NewTestStorageClass(client, namespace, storageClass)
 	createdStorageClass := tsc.Create()
 	cleanupFuncs = append(cleanupFuncs, tsc.Cleanup)

test/e2e/testsuites/testsuites.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"fmt"
 	"math/rand"
+	"strings"
 	"time"
 
 	"sigs.k8s.io/blob-csi-driver/pkg/blob"
@@ -35,6 +36,8 @@ import (
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/apimachinery/pkg/util/errors"
+	"k8s.io/apimachinery/pkg/util/wait"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/kubernetes/pkg/kubelet/events"
 	"k8s.io/kubernetes/test/e2e/framework"
@@ -54,8 +57,9 @@ const (
 	// Description that will printed during tests
 	failedConditionDescription = "Error status code"
 
-	poll            = 2 * time.Second
-	pollLongTimeout = 5 * time.Minute
+	poll                 = 2 * time.Second
+	pollLongTimeout      = 5 * time.Minute
+	pollForStringTimeout = 1 * time.Minute
 )
 
 type TestStorageClass struct {
@@ -379,6 +383,10 @@ func (t *TestDeployment) Exec(command []string, expectedString string) {
 	framework.ExpectNoError(err)
 }
 
+func (t *TestDeployment) PollForStringInPodsExec(command []string, expectedString string) {
+	pollForStringInPodsExec(t.namespace.Name, []string{t.podName}, command, expectedString)
+}
+
 func (t *TestDeployment) DeletePodAndWait() {
 	e2elog.Logf("Deleting pod %q in namespace %q", t.podName, t.namespace.Name)
 	err := t.client.CoreV1().Pods(t.namespace.Name).Delete(context.TODO(), t.podName, metav1.DeleteOptions{})
@@ -629,3 +637,33 @@ func waitForPersistentVolumeClaimDeleted(c clientset.Interface, ns string, pvcNa
 	}
 	return fmt.Errorf("PersistentVolumeClaim %s is not removed from the system within %v", pvcName, timeout)
 }
+
+func pollForStringWorker(namespace string, pod string, command []string, expectedString string, ch chan<- error) {
+	args := append([]string{"exec", pod, "--"}, command...)
+	err := wait.PollImmediate(poll, pollForStringTimeout, func() (bool, error) {
+		stdout, err := framework.RunKubectl(namespace, args...)
+		if err != nil {
+			framework.Logf("Error waiting for output %q in pod %q: %v.", expectedString, pod, err)
+			return false, nil
+		}
+		if !strings.Contains(stdout, expectedString) {
+			framework.Logf("The stdout did not contain output %q in pod %q, found: %q.", expectedString, pod, stdout)
+			return false, nil
+		}
+		return true, nil
+	})
+	ch <- err
+}
+
+// Execute the command for all pods in the namespace, looking for expectedString in stdout
+func pollForStringInPodsExec(namespace string, pods []string, command []string, expectedString string) {
+	ch := make(chan error, len(pods))
+	for _, pod := range pods {
+		go pollForStringWorker(namespace, pod, command, expectedString, ch)
+	}
+	errs := make([]error, 0, len(pods))
+	for range pods {
+		errs = append(errs, <-ch)
+	}
+	framework.ExpectNoError(errors.NewAggregate(errs), "Failed to find %q in at least one pod's output.", expectedString)
+}

test/utils/restart_driver_daemonset.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+# Copyright 2021 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -euo pipefail
+
+echo "restart driver node daemonset ..."
+kubectl rollout restart ds csi-blob-node -n kube-system
+
+sleep 10