fix: instance is always pending when the component replicas is 1 and …

…enable the pitr and using the cloud disk (#8289)
apecloud · Oct 17, 2024 · 6af7b76 · 6af7b76
1 parent 1548d52
commit 6af7b76
Show file tree

Hide file tree

Showing 3 changed files with 61 additions and 4 deletions.
diff --git a/controllers/dataprotection/backup_controller.go b/controllers/dataprotection/backup_controller.go
@@ -454,10 +454,20 @@ func (r *BackupReconciler) prepareRequestTargetInfo(reqCtx intctrlutil.RequestCt
 	}
 	targetPods, err := GetTargetPods(reqCtx, r.Client,
 		selectedPods, request.BackupPolicy, target, backupType)
-	if err != nil || len(targetPods) == 0 {
+	if err != nil {
+		return err
+	}
+	if len(targetPods) == 0 {
+		if backupType == dpv1alpha1.BackupTypeContinuous {
+			// stop the sts to un-bound the pvcs when the continuous backup is failed.
+			if err = dpbackup.StopStatefulSetsWhenFailed(reqCtx.Ctx, r.Client, request.Backup, target.Name); err != nil {
+				return err
+			}
+		}
 		return fmt.Errorf("failed to get target pods by backup policy %s/%s",
 			request.BackupPolicy.Namespace, request.BackupPolicy.Name)
 	}
+
 	request.TargetPods = targetPods
 	saName := target.ServiceAccountName
 	if saName == "" {

diff --git a/controllers/dataprotection/backup_controller_test.go b/controllers/dataprotection/backup_controller_test.go
@@ -28,6 +28,7 @@ import (
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+	appsv1 "k8s.io/api/apps/v1"
 
 	vsv1 "github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1"
 	batchv1 "k8s.io/api/batch/v1"
@@ -37,7 +38,7 @@ import (
 	"k8s.io/utils/pointer"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
-	appsv1 "github.com/apecloud/kubeblocks/apis/apps/v1"
+	kbappsv1 "github.com/apecloud/kubeblocks/apis/apps/v1"
 	dpv1alpha1 "github.com/apecloud/kubeblocks/apis/dataprotection/v1alpha1"
 	"github.com/apecloud/kubeblocks/pkg/constant"
 	intctrlutil "github.com/apecloud/kubeblocks/pkg/controllerutil"
@@ -101,7 +102,7 @@ var _ = Describe("Backup Controller test", func() {
 		var (
 			backupPolicy *dpv1alpha1.BackupPolicy
 			repoPVCName  string
-			cluster      *appsv1.Cluster
+			cluster      *kbappsv1.Cluster
 			pvcName      string
 			targetPod    *corev1.Pod
 		)
@@ -878,6 +879,36 @@ var _ = Describe("Backup Controller test", func() {
 					g.Expect(fetched.Status.Phase).Should(Equal(dpv1alpha1.BackupPhaseRunning))
 					g.Expect(fetched.Status.PersistentVolumeClaimName).Should(Equal(repoPVCName))
 				})).Should(Succeed())
+
+				By("mock no target pod found and expect backup is Failed")
+				Expect(testapps.ChangeObj(&testCtx, clusterInfo.TargetPod, func(pod *corev1.Pod) {
+					delete(clusterInfo.TargetPod.Labels, constant.RoleLabelKey)
+				}))
+				Eventually(testapps.CheckObj(&testCtx, backupKey, func(g Gomega, fetched *dpv1alpha1.Backup) {
+					g.Expect(fetched.Status.Phase).Should(Equal(dpv1alpha1.BackupPhaseFailed))
+					g.Expect(fetched.Status.FailureReason).Should(ContainSubstring("failed to get target pods by backup policy"))
+				})).Should(Succeed())
+
+				By("expect the replicas of statefulSet is 0")
+				backup := &dpv1alpha1.Backup{}
+				Expect(k8sClient.Get(ctx, backupKey, backup)).Should(Succeed())
+				stsKey := client.ObjectKey{
+					Name:      dpbackup.GenerateBackupStatefulSetName(backup, "", dpbackup.BackupDataJobNamePrefix),
+					Namespace: testCtx.DefaultNamespace,
+				}
+				Eventually(testapps.CheckObj(&testCtx, stsKey, func(g Gomega, sts *appsv1.StatefulSet) {
+					g.Expect(*sts.Spec.Replicas).Should(BeEquivalentTo(0))
+				}))
+
+				By("mock target pod exists")
+				Expect(testapps.ChangeObj(&testCtx, clusterInfo.TargetPod, func(pod *corev1.Pod) {
+					clusterInfo.TargetPod.Labels[constant.RoleLabelKey] = constant.Leader
+				}))
+
+				By("expect the replicas of statefulSet is 1")
+				Eventually(testapps.CheckObj(&testCtx, stsKey, func(g Gomega, sts *appsv1.StatefulSet) {
+					g.Expect(*sts.Spec.Replicas).Should(BeEquivalentTo(1))
+				}))
 			})
 		})
 	})
@@ -1041,7 +1072,7 @@ var _ = Describe("Backup Controller test", func() {
 		var (
 			backupPolicy *dpv1alpha1.BackupPolicy
 			repoPVCName  string
-			cluster      *appsv1.Cluster
+			cluster      *kbappsv1.Cluster
 		)
 
 		BeforeEach(func() {

diff --git a/pkg/dataprotection/backup/utils.go b/pkg/dataprotection/backup/utils.go
@@ -27,8 +27,10 @@ import (
 	"strings"
 
 	"github.com/rogpeppe/go-internal/semver"
+	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/utils/pointer"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	dpv1alpha1 "github.com/apecloud/kubeblocks/apis/dataprotection/v1alpha1"
@@ -286,3 +288,17 @@ func BuildCronJobSchedule(cronExpression string) (*string, string) {
 	}
 	return nil, fmt.Sprintf("CRON_TZ=%s %s", timeZone, cronExpression)
 }
+
+// StopStatefulSetsWhenFailed stops the sts to un-bound the pvcs.
+func StopStatefulSetsWhenFailed(ctx context.Context, cli client.Client, backup *dpv1alpha1.Backup, targetName string) error {
+	if backup.Status.Phase != dpv1alpha1.BackupPhaseFailed {
+		return nil
+	}
+	sts := &appsv1.StatefulSet{}
+	stsName := GenerateBackupStatefulSetName(backup, targetName, BackupDataJobNamePrefix)
+	if err := cli.Get(ctx, client.ObjectKey{Name: stsName, Namespace: backup.Namespace}, sts); client.IgnoreNotFound(err) != nil {
+		return nil
+	}
+	sts.Spec.Replicas = pointer.Int32(0)
+	return cli.Update(ctx, sts)
+}