Make garbage collection for expired backups configurable

Make garbage collection for expired backups configurable

Fixes #4875

Signed-off-by: Wenkai Yin(尹文开) <yinw@vmware.com>

changelogs/unreleased/4897-ywk253100

@@ -0,0 +1 @@
+Make garbage collection for expired backups configurable

pkg/cmd/cli/install/install.go

@@ -67,6 +67,7 @@ type InstallOptions struct {
 	Wait                              bool
 	UseVolumeSnapshots                bool
 	DefaultResticMaintenanceFrequency time.Duration
+	GarbageCollectionFrequency        time.Duration
 	Plugins                           flag.StringArray
 	NoDefaultBackupLocation           bool
 	CRDsOnly                          bool
@@ -103,6 +104,7 @@ func (o *InstallOptions) BindFlags(flags *pflag.FlagSet) {
 	flags.BoolVar(&o.UseRestic, "use-restic", o.UseRestic, "Create restic daemonset. Optional.")
 	flags.BoolVar(&o.Wait, "wait", o.Wait, "Wait for Velero deployment to be ready. Optional.")
 	flags.DurationVar(&o.DefaultResticMaintenanceFrequency, "default-restic-prune-frequency", o.DefaultResticMaintenanceFrequency, "How often 'restic prune' is run for restic repositories by default. Optional.")
+	flags.DurationVar(&o.GarbageCollectionFrequency, "garbage-collection-frequency", o.GarbageCollectionFrequency, "How often the garbage collection runs for expired backups.(default 1h)")
 	flags.Var(&o.Plugins, "plugins", "Plugin container images to install into the Velero Deployment")
 	flags.BoolVar(&o.CRDsOnly, "crds-only", o.CRDsOnly, "Only generate CustomResourceDefinition resources. Useful for updating CRDs for an existing Velero install.")
 	flags.StringVar(&o.CACertFile, "cacert", o.CACertFile, "File containing a certificate bundle to use when verifying TLS connections to the object store. Optional.")
@@ -187,6 +189,7 @@ func (o *InstallOptions) AsVeleroOptions() (*install.VeleroOptions, error) {
 		BSLConfig:                         o.BackupStorageConfig.Data(),
 		VSLConfig:                         o.VolumeSnapshotConfig.Data(),
 		DefaultResticMaintenanceFrequency: o.DefaultResticMaintenanceFrequency,
+		GarbageCollectionFrequency:        o.GarbageCollectionFrequency,
 		Plugins:                           o.Plugins,
 		NoDefaultBackupLocation:           o.NoDefaultBackupLocation,
 		CACertData:                        caCertData,
@@ -395,5 +398,9 @@ func (o *InstallOptions) Validate(c *cobra.Command, args []string, f client.Fact
 		return errors.New("--default-restic-prune-frequency must be non-negative")
 	}
 
+	if o.GarbageCollectionFrequency < 0 {
+		return errors.New("--garbage-collection-frequency must be non-negative")
+	}
+
 	return nil
 }

pkg/cmd/server/server.go

@@ -121,6 +121,7 @@ type serverConfig struct {
 	profilerAddress                                                         string
 	formatFlag                                                              *logging.FormatFlag
 	defaultResticMaintenanceFrequency                                       time.Duration
+	garbageCollectionFrequency                                              time.Duration
 	defaultVolumesToRestic                                                  bool
 }
 
@@ -214,6 +215,7 @@ func NewCommand(f client.Factory) *cobra.Command {
 	command.Flags().DurationVar(&config.resourceTerminatingTimeout, "terminating-resource-timeout", config.resourceTerminatingTimeout, "How long to wait on persistent volumes and namespaces to terminate during a restore before timing out.")
 	command.Flags().DurationVar(&config.defaultBackupTTL, "default-backup-ttl", config.defaultBackupTTL, "How long to wait by default before backups can be garbage collected.")
 	command.Flags().DurationVar(&config.defaultResticMaintenanceFrequency, "default-restic-prune-frequency", config.defaultResticMaintenanceFrequency, "How often 'restic prune' is run for restic repositories by default.")
+	command.Flags().DurationVar(&config.garbageCollectionFrequency, "garbage-collection-frequency", config.garbageCollectionFrequency, "How often garbage collection is run for expired backups.")
 	command.Flags().BoolVar(&config.defaultVolumesToRestic, "default-volumes-to-restic", config.defaultVolumesToRestic, "Backup all volumes with restic by default.")
 
 	return command
@@ -669,6 +671,7 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string
 			s.sharedInformerFactory.Velero().V1().DeleteBackupRequests().Lister(),
 			s.veleroClient.VeleroV1(),
 			s.mgr.GetClient(),
+			s.config.garbageCollectionFrequency,
 		)
 
 		return controllerRunInfo{

pkg/controller/gc_controller.go

@@ -39,7 +39,7 @@ import (
 )
 
 const (
-	GCSyncPeriod             = 60 * time.Minute
+	defaultGCFrequency       = 60 * time.Minute
 	garbageCollectionFailure = "velero.io/gc-failure"
 	gcFailureBSLNotFound     = "BSLNotFound"
 	gcFailureBSLCannotGet    = "BSLCannotGet"
@@ -54,6 +54,7 @@ type gcController struct {
 	deleteBackupRequestLister velerov1listers.DeleteBackupRequestLister
 	deleteBackupRequestClient velerov1client.DeleteBackupRequestsGetter
 	kbClient                  client.Client
+	frequency                 time.Duration
 
 	clock clock.Clock
 }
@@ -65,6 +66,7 @@ func NewGCController(
 	deleteBackupRequestLister velerov1listers.DeleteBackupRequestLister,
 	deleteBackupRequestClient velerov1client.DeleteBackupRequestsGetter,
 	kbClient client.Client,
+	frequency time.Duration,
 ) Interface {
 	c := &gcController{
 		genericController:         newGenericController(GarbageCollection, logger),
@@ -76,7 +78,11 @@ func NewGCController(
 	}
 
 	c.syncHandler = c.processQueueItem
-	c.resyncPeriod = GCSyncPeriod
+	c.resyncPeriod = frequency
+	if c.resyncPeriod < 0 {
+		c.resyncPeriod = defaultGCFrequency
+	}
+	logger.Infof("Garbage collection frequency: %s", c.resyncPeriod.String())
 	c.resyncFunc = c.enqueueAllBackups
 
 	backupInformer.Informer().AddEventHandler(

pkg/controller/gc_controller_test.go

@@ -53,6 +53,7 @@ func TestGCControllerEnqueueAllBackups(t *testing.T) {
 			sharedInformers.Velero().V1().DeleteBackupRequests().Lister(),
 			client.VeleroV1(),
 			nil,
+			defaultGCFrequency,
 		).(*gcController)
 	)
 
@@ -114,6 +115,7 @@ func TestGCControllerHasUpdateFunc(t *testing.T) {
 		sharedInformers.Velero().V1().DeleteBackupRequests().Lister(),
 		client.VeleroV1(),
 		nil,
+		defaultGCFrequency,
 	).(*gcController)
 
 	keys := make(chan string)
@@ -262,6 +264,7 @@ func TestGCControllerProcessQueueItem(t *testing.T) {
 				sharedInformers.Velero().V1().DeleteBackupRequests().Lister(),
 				client.VeleroV1(),
 				fakeClient,
+				defaultGCFrequency,
 			).(*gcController)
 			controller.clock = fakeClock
 

pkg/install/deployment.go

@@ -40,6 +40,7 @@ type podTemplateConfig struct {
 	resources                         corev1.ResourceRequirements
 	withSecret                        bool
 	defaultResticMaintenanceFrequency time.Duration
+	garbageCollectionFrequency        time.Duration
 	plugins                           []string
 	features                          []string
 	defaultVolumesToRestic            bool
@@ -104,6 +105,12 @@ func WithDefaultResticMaintenanceFrequency(val time.Duration) podTemplateOption
 	}
 }
 
+func WithGarbageCollectionFrequency(val time.Duration) podTemplateOption {
+	return func(c *podTemplateConfig) {
+		c.garbageCollectionFrequency = val
+	}
+}
+
 func WithPlugins(plugins []string) podTemplateOption {
 	return func(c *podTemplateConfig) {
 		c.plugins = plugins
@@ -275,6 +282,10 @@ func Deployment(namespace string, opts ...podTemplateOption) *appsv1.Deployment
 		deployment.Spec.Template.Spec.Containers[0].Args = append(deployment.Spec.Template.Spec.Containers[0].Args, fmt.Sprintf("--default-restic-prune-frequency=%v", c.defaultResticMaintenanceFrequency))
 	}
 
+	if c.garbageCollectionFrequency > 0 {
+		deployment.Spec.Template.Spec.Containers[0].Args = append(deployment.Spec.Template.Spec.Containers[0].Args, fmt.Sprintf("--garbage-collection-frequency=%v", c.garbageCollectionFrequency))
+	}
+
 	if len(c.plugins) > 0 {
 		for _, image := range c.plugins {
 			container := *builder.ForPluginContainer(image, pullPolicy).Result()

pkg/install/deployment_test.go

@@ -50,6 +50,10 @@ func TestDeployment(t *testing.T) {
 	assert.Len(t, deploy.Spec.Template.Spec.Containers[0].Args, 2)
 	assert.Equal(t, "--default-restic-prune-frequency=24h0m0s", deploy.Spec.Template.Spec.Containers[0].Args[1])
 
+	deploy = Deployment("velero", WithGarbageCollectionFrequency(24*time.Hour))
+	assert.Len(t, deploy.Spec.Template.Spec.Containers[0].Args, 2)
+	assert.Equal(t, "--garbage-collection-frequency=24h0m0s", deploy.Spec.Template.Spec.Containers[0].Args[1])
+
 	deploy = Deployment("velero", WithFeatures([]string{"EnableCSI", "foo", "bar", "baz"}))
 	assert.Len(t, deploy.Spec.Template.Spec.Containers[0].Args, 2)
 	assert.Equal(t, "--features=EnableCSI,foo,bar,baz", deploy.Spec.Template.Spec.Containers[0].Args[1])

pkg/install/resources.go

@@ -226,6 +226,7 @@ type VeleroOptions struct {
 	BSLConfig                         map[string]string
 	VSLConfig                         map[string]string
 	DefaultResticMaintenanceFrequency time.Duration
+	GarbageCollectionFrequency        time.Duration
 	Plugins                           []string
 	NoDefaultBackupLocation           bool
 	CACertData                        []byte
@@ -285,6 +286,7 @@ func AllResources(o *VeleroOptions) *unstructured.UnstructuredList {
 		WithResources(o.VeleroPodResources),
 		WithSecret(secretPresent),
 		WithDefaultResticMaintenanceFrequency(o.DefaultResticMaintenanceFrequency),
+		WithGarbageCollectionFrequency(o.GarbageCollectionFrequency),
 	}
 
 	if len(o.Features) > 0 {