diff --git a/controllers/dataprotection/backup_controller.go b/controllers/dataprotection/backup_controller.go index ad85f5afd88..2459c324413 100644 --- a/controllers/dataprotection/backup_controller.go +++ b/controllers/dataprotection/backup_controller.go @@ -454,10 +454,20 @@ func (r *BackupReconciler) prepareRequestTargetInfo(reqCtx intctrlutil.RequestCt } targetPods, err := GetTargetPods(reqCtx, r.Client, selectedPods, request.BackupPolicy, target, backupType) - if err != nil || len(targetPods) == 0 { + if err != nil { + return err + } + if len(targetPods) == 0 { + if backupType == dpv1alpha1.BackupTypeContinuous { + // stop the sts to un-bound the pvcs when the continuous backup is failed. + if err = dpbackup.StopStatefulSetsWhenFailed(reqCtx.Ctx, r.Client, request.Backup, target.Name); err != nil { + return err + } + } return fmt.Errorf("failed to get target pods by backup policy %s/%s", request.BackupPolicy.Namespace, request.BackupPolicy.Name) } + request.TargetPods = targetPods saName := target.ServiceAccountName if saName == "" { diff --git a/controllers/dataprotection/backup_controller_test.go b/controllers/dataprotection/backup_controller_test.go index e78b631de6d..37f6a1a30e4 100644 --- a/controllers/dataprotection/backup_controller_test.go +++ b/controllers/dataprotection/backup_controller_test.go @@ -28,6 +28,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" vsv1 "github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1" batchv1 "k8s.io/api/batch/v1" @@ -37,7 +38,7 @@ import ( "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/client" - appsv1 "github.com/apecloud/kubeblocks/apis/apps/v1" + kbappsv1 "github.com/apecloud/kubeblocks/apis/apps/v1" dpv1alpha1 "github.com/apecloud/kubeblocks/apis/dataprotection/v1alpha1" "github.com/apecloud/kubeblocks/pkg/constant" intctrlutil "github.com/apecloud/kubeblocks/pkg/controllerutil" @@ -101,7 +102,7 @@ var _ = Describe("Backup Controller test", func() { var ( backupPolicy *dpv1alpha1.BackupPolicy repoPVCName string - cluster *appsv1.Cluster + cluster *kbappsv1.Cluster pvcName string targetPod *corev1.Pod ) @@ -878,6 +879,36 @@ var _ = Describe("Backup Controller test", func() { g.Expect(fetched.Status.Phase).Should(Equal(dpv1alpha1.BackupPhaseRunning)) g.Expect(fetched.Status.PersistentVolumeClaimName).Should(Equal(repoPVCName)) })).Should(Succeed()) + + By("mock no target pod found and expect backup is Failed") + Expect(testapps.ChangeObj(&testCtx, clusterInfo.TargetPod, func(pod *corev1.Pod) { + delete(clusterInfo.TargetPod.Labels, constant.RoleLabelKey) + })) + Eventually(testapps.CheckObj(&testCtx, backupKey, func(g Gomega, fetched *dpv1alpha1.Backup) { + g.Expect(fetched.Status.Phase).Should(Equal(dpv1alpha1.BackupPhaseFailed)) + g.Expect(fetched.Status.FailureReason).Should(ContainSubstring("failed to get target pods by backup policy")) + })).Should(Succeed()) + + By("expect the replicas of statefulSet is 0") + backup := &dpv1alpha1.Backup{} + Expect(k8sClient.Get(ctx, backupKey, backup)).Should(Succeed()) + stsKey := client.ObjectKey{ + Name: dpbackup.GenerateBackupStatefulSetName(backup, "", dpbackup.BackupDataJobNamePrefix), + Namespace: testCtx.DefaultNamespace, + } + Eventually(testapps.CheckObj(&testCtx, stsKey, func(g Gomega, sts *appsv1.StatefulSet) { + g.Expect(*sts.Spec.Replicas).Should(BeEquivalentTo(0)) + })) + + By("mock target pod exists") + Expect(testapps.ChangeObj(&testCtx, clusterInfo.TargetPod, func(pod *corev1.Pod) { + clusterInfo.TargetPod.Labels[constant.RoleLabelKey] = constant.Leader + })) + + By("expect the replicas of statefulSet is 1") + Eventually(testapps.CheckObj(&testCtx, stsKey, func(g Gomega, sts *appsv1.StatefulSet) { + g.Expect(*sts.Spec.Replicas).Should(BeEquivalentTo(1)) + })) }) }) }) @@ -1041,7 +1072,7 @@ var _ = Describe("Backup Controller test", func() { var ( backupPolicy *dpv1alpha1.BackupPolicy repoPVCName string - cluster *appsv1.Cluster + cluster *kbappsv1.Cluster ) BeforeEach(func() { diff --git a/pkg/dataprotection/backup/utils.go b/pkg/dataprotection/backup/utils.go index bf61e24befa..5735ff2453e 100644 --- a/pkg/dataprotection/backup/utils.go +++ b/pkg/dataprotection/backup/utils.go @@ -27,8 +27,10 @@ import ( "strings" "github.com/rogpeppe/go-internal/semver" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/client" dpv1alpha1 "github.com/apecloud/kubeblocks/apis/dataprotection/v1alpha1" @@ -286,3 +288,17 @@ func BuildCronJobSchedule(cronExpression string) (*string, string) { } return nil, fmt.Sprintf("CRON_TZ=%s %s", timeZone, cronExpression) } + +// StopStatefulSetsWhenFailed stops the sts to un-bound the pvcs. +func StopStatefulSetsWhenFailed(ctx context.Context, cli client.Client, backup *dpv1alpha1.Backup, targetName string) error { + if backup.Status.Phase != dpv1alpha1.BackupPhaseFailed { + return nil + } + sts := &appsv1.StatefulSet{} + stsName := GenerateBackupStatefulSetName(backup, targetName, BackupDataJobNamePrefix) + if err := cli.Get(ctx, client.ObjectKey{Name: stsName, Namespace: backup.Namespace}, sts); client.IgnoreNotFound(err) != nil { + return nil + } + sts.Spec.Replicas = pointer.Int32(0) + return cli.Update(ctx, sts) +}