Skip to content

Commit

Permalink
New volume snapshot e2e tests
Browse files Browse the repository at this point in the history
  • Loading branch information
dkoshkin committed Mar 10, 2019
1 parent 70852ac commit c4fe59e
Show file tree
Hide file tree
Showing 9 changed files with 357 additions and 41 deletions.
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ require (
github.com/jonboulle/clockwork v0.1.0 // indirect
github.com/json-iterator/go v1.1.5 // indirect
github.com/kubernetes-csi/csi-test v1.1.0
github.com/kubernetes-csi/external-snapshotter v1.0.1
github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.1 // indirect
Expand Down
3 changes: 3 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,8 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGi
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kubernetes-csi/csi-test v1.1.0 h1:a7CfGqhGDs0h7AZt1f6LTIUzBazcRf6eBdTUBXB4xE4=
github.com/kubernetes-csi/csi-test v1.1.0/go.mod h1:YxJ4UiuPWIhMBkxUKY5c267DyA0uDZ/MtAimhx/2TA0=
github.com/kubernetes-csi/external-snapshotter v1.0.1 h1:mR9hun6TViPyNDa+SFoRxnIXQmITIVM840ODiuNG784=
github.com/kubernetes-csi/external-snapshotter v1.0.1/go.mod h1:oYfxnsuh48V1UDYORl77YQxQbbdokNy7D73phuFpksY=
github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329 h1:2gxZ0XQIU/5z3Z3bUBu+FXuk2pFbkN6tcwi/pjyaDic=
github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
Expand Down Expand Up @@ -221,6 +223,7 @@ gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81
honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
k8s.io/api v0.0.0-20181204000039-89a74a8d264d h1:HQoGWsWUe/FmRcX9BU440AAMnzBFEf+DBo4nbkQlNzs=
k8s.io/api v0.0.0-20181204000039-89a74a8d264d/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA=
k8s.io/api v0.0.0-20190226013409-f951fa7b8c72 h1:4wrH+M4Kpkbbjs4UzT7XOvuhgBczsd4eXfBn9T5WqKk=
k8s.io/apiextensions-apiserver v0.0.0-20181204003920-20c909e7c8c3 h1:bpFjd6jnk/2Wi5ZqnJaTbaHt4GhCYwc2UlhcyYrOx24=
k8s.io/apiextensions-apiserver v0.0.0-20181204003920-20c909e7c8c3/go.mod h1:IxkesAMoaCRoLrPJdZNZUQp9NfZnzqaVzLhb2VEQzXE=
k8s.io/apimachinery v0.0.0-20181127025237-2b1284ed4c93 h1:tT6oQBi0qwLbbZSfDkdIsb23EwaLY85hoAV4SpXfdao=
Expand Down
1 change: 1 addition & 0 deletions hack/feature-gates.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
CSIDriverRegistry: "true"
CSINodeInfo: "true"
CSIBlockVolume: "true"
VolumeSnapshotDataSource: "true"
kubelet:
featureGates:
CSIDriverRegistry: "true"
Expand Down
25 changes: 25 additions & 0 deletions tests/e2e/driver/driver.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,14 +15,21 @@ limitations under the License.
package driver

import (
"github.com/kubernetes-csi/external-snapshotter/pkg/apis/volumesnapshot/v1alpha1"
"k8s.io/api/core/v1"
storagev1 "k8s.io/api/storage/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

const (
VolumeSnapshotClassKind = "VolumeSnapshotClass"
SnapshotAPIVersion = "snapshot.storage.k8s.io/v1alpha1"
)

type PVTestDriver interface {
DynamicPVTestDriver
PreProvisionedVolumeTestDriver
VolumeSnapshotTestDriver
}

// DynamicPVTestDriver represents an interface for a CSI driver that supports DynamicPV
Expand All @@ -37,6 +44,10 @@ type PreProvisionedVolumeTestDriver interface {
GetPersistentVolume(volumeID string, fsType string, size string, reclaimPolicy *v1.PersistentVolumeReclaimPolicy, namespace string) *v1.PersistentVolume
}

type VolumeSnapshotTestDriver interface {
GetVolumeSnapshotClass(namespace string) *v1alpha1.VolumeSnapshotClass
}

func getStorageClass(
generateName string,
provisioner string,
Expand Down Expand Up @@ -66,3 +77,17 @@ func getStorageClass(
AllowedTopologies: allowedTopologies,
}
}

func getVolumeSnapshotClass(generateName string, provisioner string) *v1alpha1.VolumeSnapshotClass {
return &v1alpha1.VolumeSnapshotClass{
TypeMeta: metav1.TypeMeta{
Kind: VolumeSnapshotClassKind,
APIVersion: SnapshotAPIVersion,
},

ObjectMeta: metav1.ObjectMeta{
GenerateName: generateName,
},
Snapshotter: provisioner,
}
}
11 changes: 9 additions & 2 deletions tests/e2e/driver/ebs_csi_driver.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ package driver

import (
"fmt"
"github.com/kubernetes-csi/external-snapshotter/pkg/apis/volumesnapshot/v1alpha1"
ebscsidriver "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/driver"
"k8s.io/api/core/v1"
storagev1 "k8s.io/api/storage/v1"
Expand All @@ -41,7 +42,7 @@ func InitEbsCSIDriver() PVTestDriver {

func (d *ebsCSIDriver) GetDynamicProvisionStorageClass(parameters map[string]string, mountOptions []string, reclaimPolicy *v1.PersistentVolumeReclaimPolicy, bindingMode *storagev1.VolumeBindingMode, allowedTopologyValues []string, namespace string) *storagev1.StorageClass {
provisioner := d.driverName
generatedName := fmt.Sprintf("%s-%s-dynamic-sc-", namespace, provisioner)
generateName := fmt.Sprintf("%s-%s-dynamic-sc-", namespace, provisioner)
allowedTopologies := []v1.TopologySelectorTerm{}
if len(allowedTopologyValues) > 0 {
allowedTopologies = []v1.TopologySelectorTerm{
Expand All @@ -55,7 +56,13 @@ func (d *ebsCSIDriver) GetDynamicProvisionStorageClass(parameters map[string]str
},
}
}
return getStorageClass(generatedName, provisioner, parameters, mountOptions, reclaimPolicy, bindingMode, allowedTopologies)
return getStorageClass(generateName, provisioner, parameters, mountOptions, reclaimPolicy, bindingMode, allowedTopologies)
}

func (d *ebsCSIDriver) GetVolumeSnapshotClass(namespace string) *v1alpha1.VolumeSnapshotClass {
provisioner := d.driverName
generateName := fmt.Sprintf("%s-%s-dynamic-sc-", namespace, provisioner)
return getVolumeSnapshotClass(generateName, provisioner)
}

func (d *ebsCSIDriver) GetPersistentVolume(volumeID string, fsType string, size string, reclaimPolicy *v1.PersistentVolumeReclaimPolicy, namespace string) *v1.PersistentVolume {
Expand Down
79 changes: 78 additions & 1 deletion tests/e2e/dynamic_provisioning.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"k8s.io/api/core/v1"
storagev1 "k8s.io/api/storage/v1"
clientset "k8s.io/client-go/kubernetes"
restclientset "k8s.io/client-go/rest"
"k8s.io/kubernetes/test/e2e/framework"
"math/rand"
"os"
Expand All @@ -30,6 +31,9 @@ import (

awscloud "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/cloud"
ebscsidriver "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/driver"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/runtime/serializer"
)

var _ = Describe("[ebs-csi-e2e] [single-az] Dynamic Provisioning", func() {
Expand All @@ -38,7 +42,7 @@ var _ = Describe("[ebs-csi-e2e] [single-az] Dynamic Provisioning", func() {
var (
cs clientset.Interface
ns *v1.Namespace
ebsDriver driver.DynamicPVTestDriver
ebsDriver driver.PVTestDriver
)

BeforeEach(func() {
Expand Down Expand Up @@ -399,6 +403,66 @@ var _ = Describe("[ebs-csi-e2e] [single-az] Dynamic Provisioning", func() {
})
})

var _ = Describe("[ebs-csi-e2e] [single-az] Snapshot", func() {
f := framework.NewDefaultFramework("ebs")

var (
cs clientset.Interface
snapshotrcs restclientset.Interface
ns *v1.Namespace
ebsDriver driver.PVTestDriver
)

BeforeEach(func() {
cs = f.ClientSet
var err error
snapshotrcs, err = restClient(testsuites.SnapshotAPIGroup, testsuites.APIVersionv1alpha1)
if err != nil {
Fail(fmt.Sprintf("could not get rest clientset: %v", err))
}
ns = f.Namespace
ebsDriver = driver.InitEbsCSIDriver()
})

It("should create a pod, write and read to it, take a volume snapshot, and create another pod from the snapshot", func() {
pod := testsuites.PodDetails{
// sync before taking a snapshot so that any cached data is written to the EBS volume
Cmd: "echo 'hello world' >> /mnt/test-1/data && grep 'hello world' /mnt/test-1/data && sync",
Volumes: []testsuites.VolumeDetails{
{
VolumeType: awscloud.VolumeTypeGP2,
FSType: ebscsidriver.FSTypeExt4,
ClaimSize: driver.MinimumSizeForVolumeType(awscloud.VolumeTypeGP2),
VolumeMount: testsuites.VolumeMountDetails{
NameGenerate: "test-volume-",
MountPathGenerate: "/mnt/test-",
},
},
},
}
restoredPod := testsuites.PodDetails{
Cmd: "grep 'hello world' /mnt/test-1/data",
Volumes: []testsuites.VolumeDetails{
{
VolumeType: awscloud.VolumeTypeGP2,
FSType: ebscsidriver.FSTypeExt4,
ClaimSize: driver.MinimumSizeForVolumeType(awscloud.VolumeTypeGP2),
VolumeMount: testsuites.VolumeMountDetails{
NameGenerate: "test-volume-",
MountPathGenerate: "/mnt/test-",
},
},
},
}
test := testsuites.DynamicallyProvisionedVolumeSnapshotTest{
CSIDriver: ebsDriver,
Pod: pod,
RestoredPod: restoredPod,
}
test.Run(cs, snapshotrcs, ns)
})
})

var _ = Describe("[ebs-csi-e2e] [multi-az] Dynamic Provisioning", func() {
f := framework.NewDefaultFramework("ebs")

Expand Down Expand Up @@ -472,3 +536,16 @@ var _ = Describe("[ebs-csi-e2e] [multi-az] Dynamic Provisioning", func() {
test.Run(cs, ns)
})
})

func restClient(group string, version string) (restclientset.Interface, error) {
// setup rest client
config, err := framework.LoadConfig()
if err != nil {
Fail(fmt.Sprintf("could not load config: %v", err))
}
gv := schema.GroupVersion{Group: group, Version: version}
config.GroupVersion = &gv
config.APIPath = "/apis"
config.NegotiatedSerializer = serializer.DirectCodecFactory{CodecFactory: serializer.NewCodecFactory(runtime.NewScheme())}
return restclientset.RESTClientFor(config)
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package testsuites

import (
"github.com/kubernetes-sigs/aws-ebs-csi-driver/tests/e2e/driver"

"k8s.io/api/core/v1"
clientset "k8s.io/client-go/kubernetes"
restclientset "k8s.io/client-go/rest"

. "github.com/onsi/ginkgo"
)

// DynamicallyProvisionedVolumeSnapshotTest will provision required StorageClass(es),VolumeSnapshotClass(es), PVC(s) and Pod(s)
// Waiting for the PV provisioner to create a new PV
// Testing if the Pod(s) can write and read to mounted volumes
// Create a snapshot, validate the data is still on the disk, and then write and read to it again
// And finally delete the snapshot
// This test only supports a single volume
type DynamicallyProvisionedVolumeSnapshotTest struct {
CSIDriver driver.PVTestDriver
Pod PodDetails
RestoredPod PodDetails
}

func (t *DynamicallyProvisionedVolumeSnapshotTest) Run(client clientset.Interface, restclient restclientset.Interface, namespace *v1.Namespace) {
tpod := NewTestPod(client, namespace, t.Pod.Cmd)
volume := t.Pod.Volumes[0]
tpvc, pvcCleanup := volume.SetupDynamicPersistentVolumeClaim(client, namespace, t.CSIDriver)
for i := range pvcCleanup {
defer pvcCleanup[i]()
}
tpod.SetupVolume(tpvc.persistentVolumeClaim, volume.VolumeMount.NameGenerate+"1", volume.VolumeMount.MountPathGenerate+"1", volume.VolumeMount.ReadOnly)

By("deploying the pod")
tpod.Create()
defer tpod.Cleanup()
By("checking that the pods command exits with no error")
tpod.WaitForSuccess()

By("taking snapshots")
tvsc, cleanup := CreateVolumeSnapshotClass(restclient, namespace, t.CSIDriver)
defer cleanup()

snapshot := tvsc.CreateSnapshot(tpvc.persistentVolumeClaim)
defer tvsc.DeleteSnapshot(snapshot)
tvsc.ReadyToUse(snapshot)

t.RestoredPod.Volumes[0].DataSource = &DataSource{Name: snapshot.Name}
trpod := NewTestPod(client, namespace, t.RestoredPod.Cmd)
rvolume := t.RestoredPod.Volumes[0]
trpvc, rpvcCleanup := rvolume.SetupDynamicPersistentVolumeClaim(client, namespace, t.CSIDriver)
for i := range rpvcCleanup {
defer rpvcCleanup[i]()
}
trpod.SetupVolume(trpvc.persistentVolumeClaim, rvolume.VolumeMount.NameGenerate+"1", rvolume.VolumeMount.MountPathGenerate+"1", rvolume.VolumeMount.ReadOnly)

By("deploying a second pod with a volume restored from the snapshot")
trpod.Create()
defer trpod.Cleanup()
By("checking that the pods command exits with no error")
trpod.WaitForSuccess()
}
38 changes: 37 additions & 1 deletion tests/e2e/testsuites/specs.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (
"k8s.io/api/core/v1"
storagev1 "k8s.io/api/storage/v1"
clientset "k8s.io/client-go/kubernetes"
restclientset "k8s.io/client-go/rest"

. "github.com/onsi/ginkgo"
)
Expand All @@ -45,6 +46,8 @@ type VolumeDetails struct {
VolumeDevice VolumeDeviceDetails
// Optional, used with pre-provisioned volumes
VolumeID string
// Optional, used with PVCs created from snapshots
DataSource *DataSource
}

type VolumeMode int
Expand All @@ -54,6 +57,16 @@ const (
Block
)

const (
VolumeSnapshotKind = "VolumeSnapshot"
SnapshotAPIVersion = "snapshot.storage.k8s.io/v1alpha1"
APIVersionv1alpha1 = "v1alpha1"
)

var (
SnapshotAPIGroup = "snapshot.storage.k8s.io"
)

type VolumeMountDetails struct {
NameGenerate string
MountPathGenerate string
Expand All @@ -65,6 +78,10 @@ type VolumeDeviceDetails struct {
DevicePath string
}

type DataSource struct {
Name string
}

func (pod *PodDetails) SetupWithDynamicVolumes(client clientset.Interface, namespace *v1.Namespace, csiDriver driver.DynamicPVTestDriver) (*TestPod, []func()) {
tpod := NewTestPod(client, namespace, pod.Cmd)
cleanupFuncs := make([]func(), 0)
Expand Down Expand Up @@ -126,7 +143,17 @@ func (volume *VolumeDetails) SetupDynamicPersistentVolumeClaim(client clientset.
createdStorageClass := tsc.Create()
cleanupFuncs = append(cleanupFuncs, tsc.Cleanup)
By("setting up the PVC and PV")
tpvc := NewTestPersistentVolumeClaim(client, namespace, volume.ClaimSize, volume.VolumeMode, &createdStorageClass)
var tpvc *TestPersistentVolumeClaim
if volume.DataSource != nil {
dataSource := &v1.TypedLocalObjectReference{
Name: volume.DataSource.Name,
Kind: VolumeSnapshotKind,
APIGroup: &SnapshotAPIGroup,
}
tpvc = NewTestPersistentVolumeClaimWithDataSource(client, namespace, volume.ClaimSize, volume.VolumeMode, &createdStorageClass, dataSource)
} else {
tpvc = NewTestPersistentVolumeClaim(client, namespace, volume.ClaimSize, volume.VolumeMode, &createdStorageClass)
}
tpvc.Create()
cleanupFuncs = append(cleanupFuncs, tpvc.Cleanup)
// PV will not be ready until PVC is used in a pod when volumeBindingMode: WaitForFirstConsumer
Expand Down Expand Up @@ -154,3 +181,12 @@ func (volume *VolumeDetails) SetupPreProvisionedPersistentVolumeClaim(client cli

return tpvc, cleanupFuncs
}

func CreateVolumeSnapshotClass(client restclientset.Interface, namespace *v1.Namespace, csiDriver driver.VolumeSnapshotTestDriver) (*TestVolumeSnapshotClass, func()) {
By("setting up the VolumeSnapshotClass")
volumeSnapshotClass := csiDriver.GetVolumeSnapshotClass(namespace.Name)
tvsc := NewTestVolumeSnapshotClass(client, namespace, volumeSnapshotClass)
tvsc.Create()

return tvsc, tvsc.Cleanup
}
Loading

0 comments on commit c4fe59e

Please sign in to comment.