Merge pull request #5524 from danfengliu/update-upgrade-test

Update upgrade test

Makefile

@@ -209,7 +209,7 @@ endif
 	--build-arg=RESTIC_VERSION=$(RESTIC_VERSION) \
 	-f $(VELERO_DOCKERFILE) .
 	@echo "container: $(IMAGE):$(VERSION)"
-ifeq ($(BUILDX_OUTPUT_TYPE), "registry")
+ifeq ($(BUILDX_OUTPUT_TYPE), registry)
 	@docker pull $(IMAGE):$(VERSION)
 	@docker save $(IMAGE):$(VERSION) -o $(BIN)-$(VERSION).tar
 	@gzip $(BIN)-$(VERSION).tar

test/e2e/Makefile

@@ -55,7 +55,7 @@ VELERO_VERSION ?= $(VERSION)
 PLUGINS ?=
 RESTORE_HELPER_IMAGE ?=
 #Released version only
-UPGRADE_FROM_VELERO_VERSION ?= v1.7.1,v1.8.1
+UPGRADE_FROM_VELERO_VERSION ?= v1.8.1,v1.9.2
 # UPGRADE_FROM_VELERO_CLI can has the same format(a list divided by comma) with UPGRADE_FROM_VELERO_VERSION
 # Upgrade tests will be executed sequently according to the list by UPGRADE_FROM_VELERO_VERSION
 # So although length of UPGRADE_FROM_VELERO_CLI list is not equal with UPGRADE_FROM_VELERO_VERSION

test/e2e/backups/deletion.go

@@ -86,7 +86,7 @@ func backup_deletion_test(useVolumeSnapshots bool) {
 }
 
 // runUpgradeTests runs upgrade test on the provider by kibishii.
-func runBackupDeletionTests(client TestClient, veleroCfg VerleroConfig, backupName, backupLocation string,
+func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupName, backupLocation string,
 	useVolumeSnapshots bool, kibishiiDirectory string) error {
 	oneHourTimeout, _ := context.WithTimeout(context.Background(), time.Minute*60)
 	veleroCLI := VeleroCfg.VeleroCLI

test/e2e/types.go

@@ -26,9 +26,9 @@ import (
 
 var UUIDgen uuid.UUID
 
-var VeleroCfg VerleroConfig
+var VeleroCfg VeleroConfig
 
-type VerleroConfig struct {
+type VeleroConfig struct {
 	VeleroCLI                string
 	VeleroImage              string
 	VeleroVersion            string

test/e2e/upgrade/upgrade.go

@@ -29,6 +29,7 @@ import (
 	. "github.com/vmware-tanzu/velero/test/e2e"
 	. "github.com/vmware-tanzu/velero/test/e2e/util/k8s"
 	. "github.com/vmware-tanzu/velero/test/e2e/util/kibishii"
+
 	. "github.com/vmware-tanzu/velero/test/e2e/util/providers"
 	. "github.com/vmware-tanzu/velero/test/e2e/util/velero"
 )
@@ -91,7 +92,6 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC
 			UUIDgen, err = uuid.NewRandom()
 			Expect(err).To(Succeed())
 			oneHourTimeout, _ := context.WithTimeout(context.Background(), time.Minute*60)
-
 			if veleroCLI2Version.VeleroCLI == "" {
 				//Assume tag of velero server image is identical to velero CLI version
 				//Download velero CLI if it's empty according to velero CLI version
@@ -200,7 +200,8 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC
 				tmpCfg.GCFrequency = ""
 				tmpCfg.UseNodeAgent = !useVolumeSnapshots
 				tmpCfg.UseRestic = false
-				Expect(VeleroInstall(context.Background(), &tmpCfg, useVolumeSnapshots)).To(Succeed())
+				tmpCfg.UploaderType = "restic"
+				Expect(VeleroUpgrade(context.Background(), tmpCfg)).To(Succeed())
 				Expect(CheckVeleroVersion(context.Background(), tmpCfg.VeleroCLI,
 					tmpCfg.VeleroVersion)).To(Succeed())
 			})

test/e2e/util/common/common.go

@@ -13,33 +13,36 @@ type OsCommandLine struct {
 	Args []string
 }
 
-func GetListBy2Pipes(ctx context.Context, cmdline1, cmdline2, cmdline3 OsCommandLine) ([]string, error) {
-	var b2 bytes.Buffer
-	var errVelero, errAwk error
-
-	c1 := exec.CommandContext(ctx, cmdline1.Cmd, cmdline1.Args...)
-	c2 := exec.Command(cmdline2.Cmd, cmdline2.Args...)
-	c3 := exec.Command(cmdline3.Cmd, cmdline3.Args...)
-	fmt.Println(c1)
-	fmt.Println(c2)
-	fmt.Println(c3)
-	c2.Stdin, errVelero = c1.StdoutPipe()
-	if errVelero != nil {
-		return nil, errVelero
+func GetListByCmdPipes(ctx context.Context, cmdlines []*OsCommandLine) ([]string, error) {
+	var buf bytes.Buffer
+	var err error
+	var cmds []*exec.Cmd
+	for _, cmdline := range cmdlines {
+		cmd := exec.Command(cmdline.Cmd, cmdline.Args...)
+		cmds = append(cmds, cmd)
+		fmt.Println(cmd)
+	}
+	for i := 0; i < len(cmds); i++ {
+		if i == len(cmds)-1 {
+			break
+		}
+		cmds[i+1].Stdin, err = cmds[i].StdoutPipe()
+		if err != nil {
+			return nil, err
+		}
+	}
+	cmds[len(cmds)-1].Stdout = &buf
+	for i := len(cmds) - 1; i >= 0; i-- {
+		_ = cmds[i].Start()
+		if i == 0 {
+			_ = cmds[i].Run()
+		}
 	}
-	c3.Stdin, errAwk = c2.StdoutPipe()
-	if errAwk != nil {
-		return nil, errAwk
+	for i := 1; i < len(cmds); i++ {
+		_ = cmds[i].Wait()
 	}
-	c3.Stdout = &b2
-	_ = c3.Start()
-	_ = c2.Start()
-	_ = c1.Run()
-	_ = c2.Wait()
-	_ = c3.Wait()
 
-	//fmt.Println(&b2)
-	scanner := bufio.NewScanner(&b2)
+	scanner := bufio.NewScanner(&buf)
 	var ret []string
 	for scanner.Scan() {
 		fmt.Printf("line: %s\n", scanner.Text())

test/e2e/util/k8s/common.go

@@ -31,7 +31,7 @@ import (
 
 	"github.com/vmware-tanzu/velero/pkg/builder"
 	veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec"
-	common "github.com/vmware-tanzu/velero/test/e2e/util/common"
+	"github.com/vmware-tanzu/velero/test/e2e/util/common"
 )
 
 // ensureClusterExists returns whether or not a kubernetes cluster exists for tests to be run on.
@@ -86,42 +86,52 @@ func GetPvcByPodName(ctx context.Context, namespace, podName string) ([]string,
 	// Example:
 	//    NAME                                  STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS             AGE
 	//    kibishii-data-kibishii-deployment-0   Bound    pvc-94b9fdf2-c30f-4a7b-87bf-06eadca0d5b6   1Gi        RWO            kibishii-storage-class   115s
-	CmdLine1 := &common.OsCommandLine{
+	cmds := []*common.OsCommandLine{}
+	cmd := &common.OsCommandLine{
 		Cmd:  "kubectl",
 		Args: []string{"get", "pvc", "-n", namespace},
 	}
-	CmdLine2 := &common.OsCommandLine{
+	cmds = append(cmds, cmd)
+
+	cmd = &common.OsCommandLine{
 		Cmd:  "grep",
 		Args: []string{podName},
 	}
-	CmdLine3 := &common.OsCommandLine{
+	cmds = append(cmds, cmd)
+
+	cmd = &common.OsCommandLine{
 		Cmd:  "awk",
 		Args: []string{"{print $1}"},
 	}
+	cmds = append(cmds, cmd)
 
-	return common.GetListBy2Pipes(ctx, *CmdLine1, *CmdLine2, *CmdLine3)
+	return common.GetListByCmdPipes(ctx, cmds)
 }
 
 func GetPvByPvc(ctx context.Context, namespace, pvc string) ([]string, error) {
 	// Example:
 	// 	  NAME                                       CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS   CLAIM                                              STORAGECLASS             REASON   AGE
 	//    pvc-3f784366-58db-40b2-8fec-77307807e74b   1Gi        RWO            Delete           Bound    bsl-deletion/kibishii-data-kibishii-deployment-0   kibishii-storage-class            6h41m
-	CmdLine1 := &common.OsCommandLine{
+	cmds := []*common.OsCommandLine{}
+	cmd := &common.OsCommandLine{
 		Cmd:  "kubectl",
 		Args: []string{"get", "pv"},
 	}
+	cmds = append(cmds, cmd)
 
-	CmdLine2 := &common.OsCommandLine{
+	cmd = &common.OsCommandLine{
 		Cmd:  "grep",
 		Args: []string{namespace + "/" + pvc},
 	}
+	cmds = append(cmds, cmd)
 
-	CmdLine3 := &common.OsCommandLine{
+	cmd = &common.OsCommandLine{
 		Cmd:  "awk",
 		Args: []string{"{print $1}"},
 	}
+	cmds = append(cmds, cmd)
 
-	return common.GetListBy2Pipes(ctx, *CmdLine1, *CmdLine2, *CmdLine3)
+	return common.GetListByCmdPipes(ctx, cmds)
 }
 
 func CRDShouldExist(ctx context.Context, name string) error {
@@ -145,22 +155,26 @@ func CRDCountShouldBe(ctx context.Context, name string, count int) error {
 }
 
 func GetCRD(ctx context.Context, name string) ([]string, error) {
-	CmdLine1 := &common.OsCommandLine{
+	cmds := []*common.OsCommandLine{}
+	cmd := &common.OsCommandLine{
 		Cmd:  "kubectl",
 		Args: []string{"get", "crd"},
 	}
+	cmds = append(cmds, cmd)
 
-	CmdLine2 := &common.OsCommandLine{
+	cmd = &common.OsCommandLine{
 		Cmd:  "grep",
 		Args: []string{name},
 	}
+	cmds = append(cmds, cmd)
 
-	CmdLine3 := &common.OsCommandLine{
+	cmd = &common.OsCommandLine{
 		Cmd:  "awk",
 		Args: []string{"{print $1}"},
 	}
+	cmds = append(cmds, cmd)
 
-	return common.GetListBy2Pipes(ctx, *CmdLine1, *CmdLine2, *CmdLine3)
+	return common.GetListByCmdPipes(ctx, cmds)
 }
 
 func AddLabelToPv(ctx context.Context, pv, label string) error {
@@ -282,3 +296,34 @@ func ReadFileFromPodVolume(ctx context.Context, namespace, podName, volume, file
 	fmt.Print(stderr)
 	return stdout, err
 }
+
+func KubectlGetInfo(cmdName string, arg []string) {
+	cmd := exec.CommandContext(context.Background(), cmdName, arg...)
+	fmt.Printf("Kubectl exec cmd =%v\n", cmd)
+	stdout, stderr, err := veleroexec.RunCommand(cmd)
+	fmt.Println(stdout)
+	if err != nil {
+		fmt.Println(stderr)
+		fmt.Println(err)
+	}
+}
+
+func KubectlGetDsJson(veleroNamespace string) (string, error) {
+	arg := []string{"get", "ds", "-n", veleroNamespace, "-ojson"}
+	cmd := exec.CommandContext(context.Background(), "kubectl", arg...)
+	fmt.Printf("Kubectl exec cmd =%v\n", cmd)
+	stdout, stderr, err := veleroexec.RunCommand(cmd)
+	fmt.Println(stdout)
+	if err != nil {
+		fmt.Println(stderr)
+		fmt.Println(err)
+		return "", err
+	}
+	return stdout, nil
+}
+
+func DeleteVeleroDs(ctx context.Context) error {
+	args := []string{"delete", "ds", "-n", "velero", "--all", "--force", "--grace-period", "0"}
+	fmt.Println(args)
+	return exec.CommandContext(ctx, "kubectl", args...).Run()
+}

test/e2e/util/kibishii/kibishii_utils.go

@@ -51,7 +51,7 @@ var DefaultKibishiiData = &KibishiiData{2, 10, 10, 1024, 1024, 0, 2}
 var KibishiiPodNameList = []string{"kibishii-deployment-0", "kibishii-deployment-1"}
 
 // RunKibishiiTests runs kibishii tests on the provider.
-func RunKibishiiTests(client TestClient, veleroCfg VerleroConfig, backupName, restoreName, backupLocation, kibishiiNamespace string,
+func RunKibishiiTests(client TestClient, veleroCfg VeleroConfig, backupName, restoreName, backupLocation, kibishiiNamespace string,
 	useVolumeSnapshots bool) error {
 	oneHourTimeout, _ := context.WithTimeout(context.Background(), time.Minute*60)
 	veleroCLI := VeleroCfg.VeleroCLI
@@ -174,7 +174,8 @@ func installKibishii(ctx context.Context, namespace string, cloudPlatform, veler
 }
 
 func generateData(ctx context.Context, namespace string, kibishiiData *KibishiiData) error {
-	kibishiiGenerateCmd := exec.CommandContext(ctx, "kubectl", "exec", "-n", namespace, "jump-pad", "--",
+	timeout, _ := context.WithTimeout(context.Background(), time.Minute*10)
+	kibishiiGenerateCmd := exec.CommandContext(timeout, "kubectl", "exec", "-n", namespace, "jump-pad", "--",
 		"/usr/local/bin/generate.sh", strconv.Itoa(kibishiiData.Levels), strconv.Itoa(kibishiiData.DirsPerLevel),
 		strconv.Itoa(kibishiiData.FilesPerLevel), strconv.Itoa(kibishiiData.FileLength),
 		strconv.Itoa(kibishiiData.BlockSize), strconv.Itoa(kibishiiData.PassNum), strconv.Itoa(kibishiiData.ExpectedNodes))
@@ -189,7 +190,7 @@ func generateData(ctx context.Context, namespace string, kibishiiData *KibishiiD
 }
 
 func verifyData(ctx context.Context, namespace string, kibishiiData *KibishiiData) error {
-	timeout, _ := context.WithTimeout(context.Background(), time.Minute*5)
+	timeout, _ := context.WithTimeout(context.Background(), time.Minute*10)
 	kibishiiVerifyCmd := exec.CommandContext(timeout, "kubectl", "exec", "-n", namespace, "jump-pad", "--",
 		"/usr/local/bin/verify.sh", strconv.Itoa(kibishiiData.Levels), strconv.Itoa(kibishiiData.DirsPerLevel),
 		strconv.Itoa(kibishiiData.FilesPerLevel), strconv.Itoa(kibishiiData.FileLength),

test/e2e/util/velero/install.go

@@ -49,7 +49,7 @@ type installOptions struct {
 	RestoreHelperImage     string
 }
 
-func VeleroInstall(ctx context.Context, veleroCfg *VerleroConfig, useVolumeSnapshots bool) error {
+func VeleroInstall(ctx context.Context, veleroCfg *VeleroConfig, useVolumeSnapshots bool) error {
 	if veleroCfg.CloudProvider != "kind" {
 		if veleroCfg.ObjectStoreProvider != "" {
 			return errors.New("For cloud platforms, object store plugin cannot be overridden") // Can't set an object store provider that is different than your cloud
@@ -103,6 +103,7 @@ func VeleroInstall(ctx context.Context, veleroCfg *VerleroConfig, useVolumeSnaps
 		RestoreHelperImage:     veleroCfg.RestoreHelperImage,
 	})
 	if err != nil {
+		RunDebug(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, "", "")
 		return errors.WithMessagef(err, "Failed to install Velero in the cluster")
 	}