From e9f41ed5ad8dce82e032efee4fa3640dc96b7121 Mon Sep 17 00:00:00 2001 From: Shuo Wu Date: Tue, 20 Feb 2024 17:07:11 +0800 Subject: [PATCH] controller: DR volume should wait for the latest backup restored before activation Longhorn 7945 Signed-off-by: Shuo Wu (cherry picked from commit 50acc8136199f5042afcf2b5b3c7285edb9b7820) --- controller/volume_controller.go | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/controller/volume_controller.go b/controller/volume_controller.go index 6b73f1e376..e504610b3c 100644 --- a/controller/volume_controller.go +++ b/controller/volume_controller.go @@ -3157,6 +3157,9 @@ func (c *VolumeController) checkAndFinishVolumeRestore(v *longhorn.Volume, e *lo // 3) The restore/DR volume is // 3.1) it's state `Healthy`; // 3.2) or it's state `Degraded` with all the scheduled replica included in the engine + if v.Spec.FromBackup == "" || !v.Status.IsStandby { + return nil + } isPurging := false for _, status := range e.Status.PurgeStatus { if status.IsPurging { @@ -3169,6 +3172,26 @@ func (c *VolumeController) checkAndFinishVolumeRestore(v *longhorn.Volume, e *lo return nil } + // Make sure the backup volume is up-to-date and the latest backup is restored + _, bvName, _, err := backupstore.DecodeBackupURL(v.Spec.FromBackup) + if err != nil { + return errors.Wrapf(err, "failed to get backup name from volume %s backup URL %v", v.Name, v.Spec.FromBackup) + } + bv, err := c.ds.GetBackupVolumeRO(bvName) + if err != nil { + return err + } + if !bv.Status.LastSyncedAt.IsZero() && + bv.Spec.SyncRequestedAt.After(bv.Status.LastSyncedAt.Time) { + log.Infof("Restore/DR volume needs to wait for backup volume %s update", bvName) + return nil + } + if e.Status.LastRestoredBackup != bv.Status.LastBackupName { + log.Infof("Restore/DR volume needs to restore the latest backup %s, and the current restored backup is %s", bv.Status.LastBackupName, e.Status.LastRestoredBackup) + c.enqueueVolume(v) + return nil + } + allScheduledReplicasIncluded, err := c.checkAllScheduledReplicasIncluded(v, e, rs) if err != nil { return err @@ -3180,7 +3203,7 @@ func (c *VolumeController) checkAndFinishVolumeRestore(v *longhorn.Volume, e *lo } if !isPurging && ((v.Status.Robustness == longhorn.VolumeRobustnessHealthy && allScheduledReplicasIncluded) || (v.Status.Robustness == longhorn.VolumeRobustnessDegraded && degradedVolumeSupported)) { - log.Info("Restore/DR volume finished") + log.Infof("Restore/DR volume finished with the last restored backup %s", e.Status.LastRestoredBackup) v.Status.IsStandby = false v.Status.RestoreRequired = false }