Add snapshot creation, deletion and restore from snapshot, Clone from PVC support for CephFS volumes.

charts/ceph-csi-cephfs/Chart.yaml

@@ -2,7 +2,7 @@
 apiVersion: v1
 appVersion: canary
 description: "Container Storage Interface (CSI) driver,
-provisioner, and attacher for Ceph cephfs"
+provisioner, snapshotter and attacher for Ceph cephfs"
 name: ceph-csi-cephfs
 version: 1.3.0-canary
 keywords:

charts/ceph-csi-cephfs/templates/provisioner-deployment.yaml

@@ -48,6 +48,24 @@ spec:
               mountPath: /csi
           resources:
 {{ toYaml .Values.provisioner.provisioner.resources | indent 12 }}
+        - name: csi-snapshotter
+          image: {{ .Values.provisioner.snapshotter.image.repository }}:{{ .Values.provisioner.snapshotter.image.tag }}
+          imagePullPolicy: {{ .Values.provisioner.snapshotter.image.pullPolicy }}
+          args:
+            - "--csi-address=$(ADDRESS)"
+            - "--v=5"
+            - "--timeout={{ .Values.provisioner.timeout }}"
+            - "--leader-election=true"
+          env:
+            - name: ADDRESS
+              value: "unix:///csi/{{ .Values.provisionerSocketFile }}"
+          securityContext:
+            privileged: true
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+          resources:
+{{ toYaml .Values.provisioner.snapshotter.resources | indent 12 }}
 {{- if .Values.provisioner.attacher.enabled }}
         - name: csi-attacher
           image: "{{ .Values.provisioner.attacher.image.repository }}:{{ .Values.provisioner.attacher.image.tag }}"

charts/ceph-csi-cephfs/templates/provisioner-rules-clusterrole.yaml

@@ -26,6 +26,18 @@ rules:
   - apiGroups: [""]
     resources: ["events"]
     verbs: ["list", "watch", "create", "update", "patch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshots"]
+    verbs: ["get", "list"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents"]
+    verbs: ["create", "get", "list", "watch", "update", "delete"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents/status"]
+    verbs: ["update"]
 {{- if .Values.provisioner.attacher.enabled }}
   - apiGroups: ["storage.k8s.io"]
     resources: ["volumeattachments"]

charts/ceph-csi-cephfs/values.yaml

@@ -134,7 +134,7 @@ provisioner:
   provisioner:
     image:
       repository: quay.io/k8scsi/csi-provisioner
-      tag: v1.4.0
+      tag: v1.6.0
       pullPolicy: IfNotPresent
     resources: {}
 
@@ -156,6 +156,13 @@ provisioner:
       pullPolicy: IfNotPresent
     resources: {}
 
+  snapshotter:
+    image:
+      repository: quay.io/k8scsi/csi-snapshotter
+      tag: v2.1.1
+      pullPolicy: IfNotPresent
+    resources: {}
+
   nodeSelector: {}
 
   tolerations: []

deploy/cephfs/kubernetes/csi-cephfsplugin-provisioner.yaml

@@ -32,7 +32,7 @@ spec:
       serviceAccount: cephfs-csi-provisioner
       containers:
         - name: csi-provisioner
-          image: quay.io/k8scsi/csi-provisioner:v1.4.0
+          image: quay.io/k8scsi/csi-provisioner:v1.6.0
           args:
             - "--csi-address=$(ADDRESS)"
             - "--v=5"
@@ -63,6 +63,22 @@ spec:
           volumeMounts:
             - name: socket-dir
               mountPath: /csi
+        - name: csi-snapshotter
+          image: quay.io/k8scsi/csi-snapshotter:v2.1.1
+          args:
+            - "--csi-address=$(ADDRESS)"
+            - "--v=5"
+            - "--timeout=150s"
+            - "--leader-election=true"
+          env:
+            - name: ADDRESS
+              value: unix:///csi/csi-provisioner.sock
+          imagePullPolicy: "IfNotPresent"
+          securityContext:
+            privileged: true
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
         - name: csi-cephfsplugin-attacher
           image: quay.io/k8scsi/csi-attacher:v2.1.1
           args:

deploy/cephfs/kubernetes/csi-provisioner-rbac.yaml

@@ -40,6 +40,15 @@ rules:
   - apiGroups: ["storage.k8s.io"]
     resources: ["storageclasses"]
     verbs: ["get", "list", "watch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshots"]
+    verbs: ["get", "list"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents"]
+    verbs: ["create", "get", "list", "watch", "update", "delete"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotclasses"]
+    verbs: ["get", "list", "watch"]
   - apiGroups: ["storage.k8s.io"]
     resources: ["volumeattachments"]
     verbs: ["get", "list", "watch", "update", "patch"]
@@ -49,7 +58,9 @@ rules:
   - apiGroups: ["storage.k8s.io"]
     resources: ["csinodes"]
     verbs: ["get", "list", "watch"]
-
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents/status"]
+    verbs: ["update"]
 ---
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1

e2e/cephfs.go

@@ -3,7 +3,9 @@ package e2e
 import (
 	"fmt"
 	"strings"
+	"sync"
 
+	vs "github.com/kubernetes-csi/external-snapshotter/v2/pkg/apis/volumesnapshot/v1beta1"
 	. "github.com/onsi/ginkgo" // nolint
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -163,6 +165,11 @@ var _ = Describe("cephfs", func() {
 		It("Test cephfs CSI", func() {
 			pvcPath := cephfsExamplePath + "pvc.yaml"
 			appPath := cephfsExamplePath + "pod.yaml"
+			pvcClonePath := cephfsExamplePath + "pvc-restore.yaml"
+			pvcSmartClonePath := cephfsExamplePath + "pvc-clone.yaml"
+			appClonePath := cephfsExamplePath + "pod-restore.yaml"
+			appSmartClonePath := cephfsExamplePath + "pod-clone.yaml"
+			snapshotPath := cephfsExamplePath + "snapshot.yaml"
 
 			By("checking provisioner deployment is running")
 			var err error
@@ -362,6 +369,237 @@ var _ = Describe("cephfs", func() {
 					}
 				})
 
+				By("create a PVC clone and bind it to an app", func() {
+					v, err := f.ClientSet.Discovery().ServerVersion()
+					if err != nil {
+						e2elog.Logf("failed to get server version with error %v", err)
+						Fail(err.Error())
+					}
+					// snapshot beta is only supported from v1.17+
+					if v.Major > "1" || (v.Major == "1" && v.Minor >= "17") {
+						var wg sync.WaitGroup
+						totalCount := 3
+						wg.Add(totalCount)
+						createCephFSSnapshotClass(f)
+						pvc, err := loadPVC(pvcPath)
+						if err != nil {
+							Fail(err.Error())
+						}
+
+						pvc.Namespace = f.UniqueName
+						e2elog.Logf("The PVC template %+v", pvc)
+						err = createPVCAndvalidatePV(f.ClientSet, pvc, deployTimeout)
+						if err != nil {
+							Fail(err.Error())
+						}
+
+						app, err := loadApp(appPath)
+						if err != nil {
+							Fail(err.Error())
+						}
+
+						app.Namespace = f.UniqueName
+						app.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = pvc.Name
+						wErr := writeDataInPod(app, f)
+						if wErr != nil {
+							Fail(wErr.Error())
+						}
+
+						snap := getSnapshot(snapshotPath)
+						snap.Namespace = f.UniqueName
+						snap.Spec.Source.PersistentVolumeClaimName = &pvc.Name
+						// create snapshot
+						for i := 0; i < totalCount; i++ {
+							go func(w *sync.WaitGroup, n int, s vs.VolumeSnapshot) {
+								s.Name = fmt.Sprintf("%s%d", f.UniqueName, n)
+								err = createSnapshot(&s, deployTimeout)
+								if err != nil {
+									e2elog.Logf("failed to create snapshot %v", err)
+									Fail(err.Error())
+								}
+								w.Done()
+							}(&wg, i, snap)
+						}
+						wg.Wait()
+
+						pvcClone, err := loadPVC(pvcClonePath)
+						if err != nil {
+							Fail(err.Error())
+						}
+						appClone, err := loadApp(appClonePath)
+						if err != nil {
+							Fail(err.Error())
+						}
+						pvcClone.Namespace = f.UniqueName
+						appClone.Namespace = f.UniqueName
+						pvcClone.Spec.DataSource.Name = fmt.Sprintf("%s%d", f.UniqueName, 0)
+
+						// create multiple PVC from same snapshot
+						wg.Add(totalCount)
+						for i := 0; i < totalCount; i++ {
+
+							go func(w *sync.WaitGroup, n int, p v1.PersistentVolumeClaim, a v1.Pod) {
+								name := fmt.Sprintf("%s%d", f.UniqueName, n)
+								err = createPVCAndApp(name, f, &p, &a, deployTimeout)
+								if err != nil {
+									e2elog.Logf("failed to create pvc and app %v", err)
+									Fail(err.Error())
+								}
+								w.Done()
+							}(&wg, i, *pvcClone, *appClone)
+						}
+						wg.Wait()
+
+						wg.Add(totalCount)
+						// delete clone and app
+						for i := 0; i < totalCount; i++ {
+							go func(w *sync.WaitGroup, n int, p v1.PersistentVolumeClaim, a v1.Pod) {
+								name := fmt.Sprintf("%s%d", f.UniqueName, n)
+								p.Spec.DataSource.Name = name
+								err = deletePVCAndApp(name, f, &p, &a)
+								if err != nil {
+									e2elog.Logf("failed to delete pvc and app %v", err)
+									Fail(err.Error())
+								}
+								w.Done()
+							}(&wg, i, *pvcClone, *appClone)
+						}
+						wg.Wait()
+
+						// create clones from different snapshosts and bind it to an
+						// app
+						wg.Add(totalCount)
+						for i := 0; i < totalCount; i++ {
+							go func(w *sync.WaitGroup, n int, p v1.PersistentVolumeClaim, a v1.Pod) {
+								name := fmt.Sprintf("%s%d", f.UniqueName, n)
+								p.Spec.DataSource.Name = name
+								err = createPVCAndApp(name, f, &p, &a, deployTimeout)
+								if err != nil {
+									e2elog.Logf("failed to create pvc and app %v", err)
+									Fail(err.Error())
+								}
+								w.Done()
+							}(&wg, i, *pvcClone, *appClone)
+						}
+						wg.Wait()
+
+						wg.Add(totalCount)
+						// delete snapshot
+						for i := 0; i < totalCount; i++ {
+							go func(w *sync.WaitGroup, n int, s vs.VolumeSnapshot) {
+								s.Name = fmt.Sprintf("%s%d", f.UniqueName, n)
+								err = deleteSnapshot(&s, deployTimeout)
+								if err != nil {
+									e2elog.Logf("failed to delete snapshot %v", err)
+									Fail(err.Error())
+								}
+								w.Done()
+							}(&wg, i, snap)
+						}
+						wg.Wait()
+
+						wg.Add(totalCount)
+						// delete clone and app
+						for i := 0; i < totalCount; i++ {
+							go func(w *sync.WaitGroup, n int, p v1.PersistentVolumeClaim, a v1.Pod) {
+								name := fmt.Sprintf("%s%d", f.UniqueName, n)
+								p.Spec.DataSource.Name = name
+								err = deletePVCAndApp(name, f, &p, &a)
+								if err != nil {
+									e2elog.Logf("failed to delete pvc and app %v", err)
+									Fail(err.Error())
+								}
+								w.Done()
+							}(&wg, i, *pvcClone, *appClone)
+						}
+						wg.Wait()
+
+						// delete parent pvc
+						err = deletePVCAndApp("", f, pvc, app)
+						if err != nil {
+							Fail(err.Error())
+						}
+					}
+				})
+
+				By("create a PVC-PVC clone and bind it to an app", func() {
+					v, err := f.ClientSet.Discovery().ServerVersion()
+					if err != nil {
+						e2elog.Logf("failed to get server version with error %v", err)
+						Fail(err.Error())
+					}
+					// pvc clone is only supported from v1.16+
+					if v.Major > "1" || (v.Major == "1" && v.Minor >= "16") {
+						var wg sync.WaitGroup
+						totalCount := 3
+						pvc, err := loadPVC(pvcPath)
+						if err != nil {
+							Fail(err.Error())
+						}
+
+						pvc.Namespace = f.UniqueName
+						err = createPVCAndvalidatePV(f.ClientSet, pvc, deployTimeout)
+						if err != nil {
+							Fail(err.Error())
+						}
+						app, err := loadApp(appPath)
+						if err != nil {
+							Fail(err.Error())
+						}
+						app.Namespace = f.UniqueName
+						app.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = pvc.Name
+						wErr := writeDataInPod(app, f)
+						if wErr != nil {
+							Fail(wErr.Error())
+						}
+
+						pvcClone, err := loadPVC(pvcSmartClonePath)
+						if err != nil {
+							Fail(err.Error())
+						}
+						pvcClone.Spec.DataSource.Name = pvc.Name
+						pvcClone.Namespace = f.UniqueName
+						appClone, err := loadApp(appSmartClonePath)
+						if err != nil {
+							Fail(err.Error())
+						}
+						appClone.Namespace = f.UniqueName
+						wg.Add(totalCount)
+						// create clone and bind it to an app
+						for i := 0; i < totalCount; i++ {
+							go func(w *sync.WaitGroup, n int, p v1.PersistentVolumeClaim, a v1.Pod) {
+								name := fmt.Sprintf("%s%d", f.UniqueName, n)
+								err = createPVCAndApp(name, f, &p, &a, deployTimeout)
+								if err != nil {
+									Fail(err.Error())
+								}
+								w.Done()
+							}(&wg, i, *pvcClone, *appClone)
+						}
+						wg.Wait()
+						// delete parent pvc
+						err = deletePVCAndApp("", f, pvc, app)
+						if err != nil {
+							Fail(err.Error())
+						}
+
+						wg.Add(totalCount)
+						// delete clone and app
+						for i := 0; i < totalCount; i++ {
+							go func(w *sync.WaitGroup, n int, p v1.PersistentVolumeClaim, a v1.Pod) {
+								name := fmt.Sprintf("%s%d", f.UniqueName, n)
+								p.Spec.DataSource.Name = name
+								err = deletePVCAndApp(name, f, &p, &a)
+								if err != nil {
+									Fail(err.Error())
+								}
+								w.Done()
+							}(&wg, i, *pvcClone, *appClone)
+						}
+						wg.Wait()
+					}
+				})
+
 				By("Create ROX PVC and Bind it to an app", func() {
 					// create pvc and bind it to an app
 					pvc, err := loadPVC(pvcPath)