From 6a8e716059e4049ca19d20acd10f31aaea557dfa Mon Sep 17 00:00:00 2001 From: danfengl Date: Sat, 21 Aug 2021 12:57:49 +0000 Subject: [PATCH] Add upgrade test into E2E tests Signed-off-by: danfengl --- test/e2e/Makefile | 5 +- test/e2e/e2e_suite_test.go | 2 + test/e2e/kibishii.go | 77 +++++++++++++++ .../{kibishii_tests.go => kibishii_utils.go} | 52 ++-------- test/e2e/upgrade_test.go | 87 +++++++++++++++++ test/e2e/upgrade_test_test.go | 95 +++++++++++++++++++ test/e2e/velero_utils.go | 92 ++++++++++++++++++ 7 files changed, 365 insertions(+), 45 deletions(-) create mode 100644 test/e2e/kibishii.go rename test/e2e/{kibishii_tests.go => kibishii_utils.go} (74%) create mode 100644 test/e2e/upgrade_test.go create mode 100644 test/e2e/upgrade_test_test.go diff --git a/test/e2e/Makefile b/test/e2e/Makefile index ed47614b29..1d2315979e 100644 --- a/test/e2e/Makefile +++ b/test/e2e/Makefile @@ -48,6 +48,8 @@ OUTPUT_DIR := _output/$(GOOS)/$(GOARCH)/bin GINKGO_FOCUS ?= VELERO_CLI ?=$$(pwd)/../../_output/bin/$(GOOS)/$(GOARCH)/velero VELERO_IMAGE ?= velero/velero:main +#Released version only +UPGRADE_FROM_VERSION ?= v1.6.2 CRDS_VERSION ?= v1 VELERO_NAMESPACE ?= velero CREDS_FILE ?= @@ -78,9 +80,10 @@ run: ginkgo @[ "${BSL_BUCKET}" ] && echo "Using bucket ${BSL_BUCKET} to store backups from E2E tests" || \ (echo "Bucket to store the backups from E2E tests is required, please re-run with BSL_BUCKET="; exit 1 ) @[ "${CLOUD_PROVIDER}" ] && echo "Using cloud provider ${CLOUD_PROVIDER}" || \ - (echo "Cloud provider for target cloud/plug-in provider is required, please rerun with CLOUD_PROVIDER="; exit 1) + (echo "Cloud provider for target cloud/plug-in provider is required, please rerun with CLOUD_PROVIDER="; exit 1) @$(GINKGO) -v -focus="$(GINKGO_FOCUS)" . -- -velerocli=$(VELERO_CLI) \ -velero-image=$(VELERO_IMAGE) \ + -upgrade-from-version=$(UPGRADE_FROM_VERSION) \ -velero-namespace=$(VELERO_NAMESPACE) \ -crds-version=$(CRDS_VERSION) \ -credentials-file=$(CREDS_FILE) \ diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 7f0712e310..c4b374e42c 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -28,6 +28,7 @@ import ( var ( veleroCLI, veleroImage, cloudCredentialsFile, bslConfig, bslBucket, bslPrefix, vslConfig, cloudProvider, objectStoreProvider, veleroNamespace, crdsVersion string additionalBSLProvider, additionalBSLBucket, additionalBSLPrefix, additionalBSLConfig, additionalBSLCredentials, registryCredentialFile string + upgradeFromVersion string installVelero bool ) @@ -38,6 +39,7 @@ func init() { flag.StringVar(&cloudCredentialsFile, "credentials-file", "", "file containing credentials for backup and volume provider. Required.") flag.StringVar(&veleroCLI, "velerocli", "velero", "path to the velero application to use.") flag.StringVar(&veleroImage, "velero-image", "velero/velero:main", "image for the velero server to be tested.") + flag.StringVar(&upgradeFromVersion, "upgrade-from-version", "v1.6.1", "image for the upgrade test.") flag.StringVar(&bslConfig, "bsl-config", "", "configuration to use for the backup storage location. Format is key1=value1,key2=value2") flag.StringVar(&bslPrefix, "prefix", "", "prefix under which all Velero data should be stored within the bucket. Optional.") flag.StringVar(&vslConfig, "vsl-config", "", "configuration to use for the volume snapshot location. Format is key1=value1,key2=value2") diff --git a/test/e2e/kibishii.go b/test/e2e/kibishii.go new file mode 100644 index 0000000000..ea13fc9246 --- /dev/null +++ b/test/e2e/kibishii.go @@ -0,0 +1,77 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "fmt" + "time" + + "github.com/pkg/errors" + "golang.org/x/net/context" +) + +const ( + kibishiiNamespace = "kibishii-workload" + jumpPadPod = "jump-pad" +) + +// runKibishiiTests runs kibishii tests on the provider. +func runKibishiiTests(client testClient, providerName, veleroCLI, veleroNamespace, backupName, restoreName, backupLocation string, + useVolumeSnapshots bool, registryCredentialFile string) error { + oneHourTimeout, _ := context.WithTimeout(context.Background(), time.Minute*60) + serviceAccountName := "default" + + if err := kibishiiPrepareBeforeBackup(client, providerName, oneHourTimeout, kibishiiNamespace, serviceAccountName, registryCredentialFile); err != nil { + return errors.Wrapf(err, "Failed to install and prepare data for kibishii %s", kibishiiNamespace) + } + + if err := veleroBackupNamespace(oneHourTimeout, veleroCLI, veleroNamespace, backupName, kibishiiNamespace, backupLocation, useVolumeSnapshots); err != nil { + veleroBackupLogs(oneHourTimeout, veleroCLI, veleroNamespace, backupName) + return errors.Wrapf(err, "Failed to backup kibishii namespace %s", kibishiiNamespace) + } + + if providerName == "vsphere" && useVolumeSnapshots { + // Wait for uploads started by the Velero Plug-in for vSphere to complete + // TODO - remove after upload progress monitoring is implemented + fmt.Println("Waiting for vSphere uploads to complete") + if err := waitForVSphereUploadCompletion(oneHourTimeout, time.Hour, kibishiiNamespace); err != nil { + return errors.Wrapf(err, "Error waiting for uploads to complete") + } + } + fmt.Printf("Simulating a disaster by removing namespace %s\n", kibishiiNamespace) + if err := deleteNamespace(oneHourTimeout, client, kibishiiNamespace, true); err != nil { + return errors.Wrapf(err, "failed to delete namespace %s", kibishiiNamespace) + } + + if err := veleroRestore(oneHourTimeout, veleroCLI, veleroNamespace, restoreName, backupName); err != nil { + veleroRestoreLogs(oneHourTimeout, veleroCLI, veleroNamespace, restoreName) + return errors.Wrapf(err, "Restore %s failed from backup %s", restoreName, backupName) + } + + if err := kibishiiVerifyAfterRestore(client, kibishiiNamespace, oneHourTimeout); err != nil { + return errors.Wrapf(err, "Error verifying kibishii after restore") + } + time.Sleep(time.Duration(1) * time.Hour) + defer func() { + if err := deleteNamespace(oneHourTimeout, client, kibishiiNamespace, true); err != nil { + fmt.Println(errors.Wrapf(err, "failed to delete the namespace %q", kibishiiNamespace)) + } + }() + + fmt.Printf("kibishii test completed successfully\n") + return nil +} diff --git a/test/e2e/kibishii_tests.go b/test/e2e/kibishii_utils.go similarity index 74% rename from test/e2e/kibishii_tests.go rename to test/e2e/kibishii_utils.go index 27c32b8e92..d3f1618bea 100644 --- a/test/e2e/kibishii_tests.go +++ b/test/e2e/kibishii_utils.go @@ -28,11 +28,6 @@ import ( veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec" ) -const ( - kibishiiNamespace = "kibishii-workload" - jumpPadPod = "jump-pad" -) - func installKibishii(ctx context.Context, namespace string, cloudPlatform string) error { // We use kustomize to generate YAML for Kibishii from the checked-in yaml directories kibishiiInstallCmd := exec.CommandContext(ctx, "kubectl", "apply", "-n", namespace, "-k", @@ -88,19 +83,14 @@ func verifyData(ctx context.Context, namespace string, levels int, filesPerLevel return nil } -// runKibishiiTests runs kibishii tests on the provider. -func runKibishiiTests(client testClient, providerName, veleroCLI, veleroNamespace, backupName, restoreName, backupLocation string, - useVolumeSnapshots bool, registryCredentialFile string) error { - oneHourTimeout, _ := context.WithTimeout(context.Background(), time.Minute*60) - serviceAccountName := "default" +func waitForKibishiiPods(ctx context.Context, client testClient, kibishiiNamespace string) error { + return waitForPods(ctx, client, kibishiiNamespace, []string{"jump-pad", "etcd0", "etcd1", "etcd2", "kibishii-deployment-0", "kibishii-deployment-1"}) +} + +func kibishiiPrepareBeforeBackup(client testClient, providerName string, oneHourTimeout context.Context, kibishiiNamespace, serviceAccountName, registryCredentialFile string) error { if err := createNamespace(oneHourTimeout, client, kibishiiNamespace); err != nil { return errors.Wrapf(err, "Failed to create namespace %s to install Kibishii workload", kibishiiNamespace) } - defer func() { - if err := deleteNamespace(oneHourTimeout, client, kibishiiNamespace, true); err != nil { - fmt.Println(errors.Wrapf(err, "failed to delete the namespace %q", kibishiiNamespace)) - } - }() // wait until the service account is created before patch the image pull secret if err := waitUntilServiceAccountCreated(oneHourTimeout, client, kibishiiNamespace, serviceAccountName, 10*time.Minute); err != nil { @@ -125,30 +115,10 @@ func runKibishiiTests(client testClient, providerName, veleroCLI, veleroNamespac if err := generateData(oneHourTimeout, kibishiiNamespace, 2, 10, 10, 1024, 1024, 0, 2); err != nil { return errors.Wrap(err, "Failed to generate data") } + return nil +} - if err := veleroBackupNamespace(oneHourTimeout, veleroCLI, veleroNamespace, backupName, kibishiiNamespace, backupLocation, useVolumeSnapshots); err != nil { - veleroBackupLogs(oneHourTimeout, veleroCLI, veleroNamespace, backupName) - return errors.Wrapf(err, "Failed to backup kibishii namespace %s", kibishiiNamespace) - } - - if providerName == "vsphere" && useVolumeSnapshots { - // Wait for uploads started by the Velero Plug-in for vSphere to complete - // TODO - remove after upload progress monitoring is implemented - fmt.Println("Waiting for vSphere uploads to complete") - if err := waitForVSphereUploadCompletion(oneHourTimeout, time.Hour, kibishiiNamespace); err != nil { - return errors.Wrapf(err, "Error waiting for uploads to complete") - } - } - fmt.Printf("Simulating a disaster by removing namespace %s\n", kibishiiNamespace) - if err := deleteNamespace(oneHourTimeout, client, kibishiiNamespace, true); err != nil { - return errors.Wrapf(err, "failed to delete namespace %s", kibishiiNamespace) - } - - if err := veleroRestore(oneHourTimeout, veleroCLI, veleroNamespace, restoreName, backupName); err != nil { - veleroRestoreLogs(oneHourTimeout, veleroCLI, veleroNamespace, restoreName) - return errors.Wrapf(err, "Restore %s failed from backup %s", restoreName, backupName) - } - +func kibishiiVerifyAfterRestore(client testClient, kibishiiNamespace string, oneHourTimeout context.Context) error { // wait for kibishii pod startup // TODO - Fix kibishii so we can check that it is ready to go fmt.Printf("Waiting for kibishii pods to be ready\n") @@ -161,11 +131,5 @@ func runKibishiiTests(client testClient, providerName, veleroCLI, veleroNamespac if err := verifyData(oneHourTimeout, kibishiiNamespace, 2, 10, 10, 1024, 1024, 0, 2); err != nil { return errors.Wrap(err, "Failed to verify data generated by kibishii") } - - fmt.Printf("kibishii test completed successfully\n") return nil } - -func waitForKibishiiPods(ctx context.Context, client testClient, kibishiiNamespace string) error { - return waitForPods(ctx, client, kibishiiNamespace, []string{"jump-pad", "etcd0", "etcd1", "etcd2", "kibishii-deployment-0", "kibishii-deployment-1"}) -} diff --git a/test/e2e/upgrade_test.go b/test/e2e/upgrade_test.go new file mode 100644 index 0000000000..3d84e40abd --- /dev/null +++ b/test/e2e/upgrade_test.go @@ -0,0 +1,87 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package e2e + +import ( + "context" + "flag" + "time" + + "github.com/google/uuid" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +// Test backup and restore of Kibishi using restic +var _ = Describe("[Upgrade-Restic] Velero tests on cluster using the plugin provider for object storage and Restic for volume backups", backup_upgrade_restore_with_restic) + +var _ = Describe("[Upgrade-Snapshot] Velero tests on cluster using the plugin provider for object storage and snapshots for volume backups", backup_upgrade_restore_with_snapshots) + +func backup_upgrade_restore_with_snapshots() { + backup_upgrade_restore_test(true) +} + +func backup_upgrade_restore_with_restic() { + backup_upgrade_restore_test(false) +} + +func backup_upgrade_restore_test(useVolumeSnapshots bool) { + var ( + backupName, restoreName string + veleroImageTemp string + ) + veleroImageProfix := "velero/velero:" + client, err := newTestClient() + Expect(err).To(Succeed(), "Failed to instantiate cluster client for backup tests") + + BeforeEach(func() { + if useVolumeSnapshots && cloudProvider == "kind" { + Skip("Volume snapshots not supported on kind") + } + var err error + flag.Parse() + uuidgen, err = uuid.NewRandom() + Expect(err).To(Succeed()) + err = veleroUninstall(context.Background(), veleroCLI, veleroNamespace) + Expect(err).To(Succeed()) + + veleroImageTemp = veleroImage + veleroImage = veleroImageProfix + upgradeFromVersion + cli, err := installVeleroCLI(upgradeFromVersion) + Expect(err).To(Succeed()) + Expect(veleroInstall(context.Background(), cli, veleroImage, veleroNamespace, cloudProvider, objectStoreProvider, useVolumeSnapshots, + cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, vslConfig, "", "", registryCredentialFile)).To(Succeed()) + time.Sleep(10 * time.Second) + Expect(checkVeleroVersion(context.Background(), cli, upgradeFromVersion)).To(Succeed()) + + }) + + AfterEach(func() { + + }) + + When("kibishii is the sample workload", func() { + FIt("should be successfully backed up and restored to the default BackupStorageLocation", func() { + backupName = "backup-" + uuidgen.String() + restoreName = "restore-" + uuidgen.String() + // Even though we are using Velero's CloudProvider plugin for object storage, the kubernetes cluster is running on + // KinD. So use the kind installation for Kibishii. + veleroImage = veleroImageTemp + Expect(runUpgradeTests(client, veleroImage, cloudProvider, veleroCLI, veleroNamespace, backupName, restoreName, "", useVolumeSnapshots, registryCredentialFile)).To(Succeed(), + "Failed to successfully backup and restore Kibishii namespace") + }) + }) +} diff --git a/test/e2e/upgrade_test_test.go b/test/e2e/upgrade_test_test.go new file mode 100644 index 0000000000..dfc3df23f3 --- /dev/null +++ b/test/e2e/upgrade_test_test.go @@ -0,0 +1,95 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "fmt" + "strings" + "time" + + "github.com/pkg/errors" + "golang.org/x/net/context" +) + +const ( + upgradeNamespace = "upgrade-workload" +) + +// runKibishiiTests runs kibishii tests on the provider. +func runUpgradeTests(client testClient, toVeleroImage, providerName, veleroCLI, veleroNamespace, backupName, restoreName, backupLocation string, + useVolumeSnapshots bool, registryCredentialFile string) error { + oneHourTimeout, _ := context.WithTimeout(context.Background(), time.Minute*60) + serviceAccountName := "default" + if err := kibishiiPrepareBeforeBackup(client, providerName, oneHourTimeout, upgradeNamespace, serviceAccountName, registryCredentialFile); err != nil { + return errors.Wrapf(err, "Failed to install and prepare data for kibishii %s", upgradeNamespace) + } + + defer func() { + if err := deleteNamespace(oneHourTimeout, client, upgradeNamespace, true); err != nil { + fmt.Println(errors.Wrapf(err, "failed to delete the namespace %q", upgradeNamespace)) + } + }() + + if err := veleroBackupNamespace(oneHourTimeout, veleroCLI, veleroNamespace, backupName, upgradeNamespace, backupLocation, useVolumeSnapshots); err != nil { + veleroBackupLogs(oneHourTimeout, veleroCLI, veleroNamespace, backupName) + return errors.Wrapf(err, "Failed to backup kibishii namespace %s", upgradeNamespace) + } + + if providerName == "vsphere" && useVolumeSnapshots { + // Wait for uploads started by the Velero Plug-in for vSphere to complete + // TODO - remove after upload progress monitoring is implemented + fmt.Println("Waiting for vSphere uploads to complete") + if err := waitForVSphereUploadCompletion(oneHourTimeout, time.Hour, upgradeNamespace); err != nil { + return errors.Wrapf(err, "Error waiting for uploads to complete") + } + } + fmt.Printf("Simulating a disaster by removing namespace %s\n", upgradeNamespace) + if err := deleteNamespace(oneHourTimeout, client, upgradeNamespace, true); err != nil { + return errors.Wrapf(err, "failed to delete namespace %s", upgradeNamespace) + } + var cli string + var err error + main_tag := "main" + upgradeToVersion := veleroImage[strings.Index(toVeleroImage, ":")+1:] + if upgradeToVersion == main_tag { + cli = veleroCLI + } else { + cli, err = installVeleroCLI(upgradeToVersion) + if err != nil { + return errors.Wrapf(err, "Fail to install velero CLI.") + } + } + if err := veleroInstall(context.Background(), cli, toVeleroImage, veleroNamespace, cloudProvider, objectStoreProvider, useVolumeSnapshots, + cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, vslConfig, crdsVersion, "", registryCredentialFile); err != nil { + return errors.Wrapf(err, "Failed to install velero from image %s", toVeleroImage) + } + time.Sleep(10 * time.Second) + if err := checkVeleroVersion(context.Background(), cli, upgradeToVersion); err != nil { + return errors.Wrapf(err, "Velero install version mismatch.") + } + if err := veleroRestore(oneHourTimeout, cli, veleroNamespace, restoreName, backupName); err != nil { + veleroRestoreLogs(oneHourTimeout, cli, veleroNamespace, restoreName) + return errors.Wrapf(err, "Restore %s failed from backup %s", restoreName, backupName) + } + + if err := kibishiiVerifyAfterRestore(client, upgradeNamespace, oneHourTimeout); err != nil { + return errors.Wrapf(err, "Error verifying kibishii after restore") + } + + fmt.Printf("kibishii test completed successfully\n") + return nil +} diff --git a/test/e2e/velero_utils.go b/test/e2e/velero_utils.go index fcdfec0b5e..f8c58312db 100644 --- a/test/e2e/velero_utils.go +++ b/test/e2e/velero_utils.go @@ -22,9 +22,11 @@ import ( "encoding/json" "fmt" "io" + "net/http" "os" "os/exec" "path/filepath" + "regexp" "strings" "time" @@ -391,3 +393,93 @@ func waitForVSphereUploadCompletion(ctx context.Context, timeout time.Duration, return err } + +func getVeleroVersion(ctx context.Context, veleroCLI string) (string, error) { + checkCMD := exec.CommandContext(ctx, veleroCLI, "version") + + stdoutPipe, err := checkCMD.StdoutPipe() + if err != nil { + return "", err + } + + jsonBuf := make([]byte, 1*1024) + + err = checkCMD.Start() + if err != nil { + return "", err + } + + bytesRead, err := io.ReadFull(stdoutPipe, jsonBuf) + + if err != nil && err != io.ErrUnexpectedEOF { + return "", err + } + if bytesRead == len(jsonBuf) { + errors.New("Velero version command returned bigger than expected") + } + output := strings.Replace(string(jsonBuf), "\n", " ", -1) + fmt.Println(output) + regCompiler := regexp.MustCompile(`(?i)client\s*:\s*version\s*:\s*([\w|\.]+).+server\s*:\s*version\s*:\s*([\w|\.]+)`) + versionMatches := regCompiler.FindStringSubmatch(output) + if len(versionMatches) < 3 { + return "", errors.New("Velero version command returned null version") + } + if versionMatches[1] != versionMatches[2] { + return "", errors.New("Velero server and client version are not matched") + } + return versionMatches[1], nil +} + +func checkVeleroVersion(ctx context.Context, veleroCLI string, expectedVer string) error { + tag := expectedVer + tagInstalled, err := getVeleroVersion(ctx, veleroCLI) + if strings.Trim(tag, " ") != strings.Trim(tagInstalled, " ") { + return errors.New(fmt.Sprintf("Velero version %s is not as expected %s", tagInstalled, tag)) + } + if err != nil { + return errors.WithMessagef(err, "Failed to get Velero version") + } + fmt.Printf("Velero version %s is as expected %s\n", tagInstalled, tag) + return nil +} +func installVeleroCLI(version string) (string, error) { + name := "velero-" + version + "-linux-amd64" + postfix := ".tar.gz" + tarball := name + postfix + cliTarballUrl := "https://github.com/vmware-tanzu/velero/releases/download/" + version + "/" + tarball + + if err := getVeleroCliTarball(tarball, cliTarballUrl); err != nil { + return "", errors.WithMessagef(err, "Failed to get Velero CLI tarball") + } + + // 执行单个shell命令时, 直接运行即可 + cmd := exec.Command("tar", "-xvf", tarball, "-C", ".") + if _, err := cmd.Output(); err != nil { + return "", errors.WithMessagef(err, "Failed to extract file from velero CLI tarball") + } + return "./" + name + "/velero", nil +} + +func getVeleroCliTarball(tarball string, cliTarballUrl string) error { + // Get the data + resp, err := http.Get(cliTarballUrl) + if err != nil { + return errors.WithMessagef(err, "Failed to access Velero CLI tarball") + } + defer resp.Body.Close() + + // Create the file + out, err := os.Create(tarball) + if err != nil { + return errors.WithMessagef(err, "Failed to create file") + } + defer out.Close() + + // Write the body to file + _, err = io.Copy(out, resp.Body) + + if err != nil { + return errors.WithMessagef(err, "Failed to extract file from velero CLI tarball") + } + return err +}