Change restic prune default interval to 7d and make user-configurable (…

…#1864)

* change restic prune default interval to 7d, add server flag for it

Signed-off-by: Steve Kriss <krisss@vmware.com>

pkg/cmd/cli/install/install.go

@@ -22,6 +22,7 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"time"
 
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
@@ -38,29 +39,30 @@ import (
 
 // InstallOptions collects all the options for installing Velero into a Kubernetes cluster.
 type InstallOptions struct {
-	Namespace            string
-	Image                string
-	BucketName           string
-	Prefix               string
-	ProviderName         string
-	PodAnnotations       flag.Map
-	VeleroPodCPURequest  string
-	VeleroPodMemRequest  string
-	VeleroPodCPULimit    string
-	VeleroPodMemLimit    string
-	ResticPodCPURequest  string
-	ResticPodMemRequest  string
-	ResticPodCPULimit    string
-	ResticPodMemLimit    string
-	RestoreOnly          bool
-	SecretFile           string
-	NoSecret             bool
-	DryRun               bool
-	BackupStorageConfig  flag.Map
-	VolumeSnapshotConfig flag.Map
-	UseRestic            bool
-	Wait                 bool
-	UseVolumeSnapshots   bool
+	Namespace                         string
+	Image                             string
+	BucketName                        string
+	Prefix                            string
+	ProviderName                      string
+	PodAnnotations                    flag.Map
+	VeleroPodCPURequest               string
+	VeleroPodMemRequest               string
+	VeleroPodCPULimit                 string
+	VeleroPodMemLimit                 string
+	ResticPodCPURequest               string
+	ResticPodMemRequest               string
+	ResticPodCPULimit                 string
+	ResticPodMemLimit                 string
+	RestoreOnly                       bool
+	SecretFile                        string
+	NoSecret                          bool
+	DryRun                            bool
+	BackupStorageConfig               flag.Map
+	VolumeSnapshotConfig              flag.Map
+	UseRestic                         bool
+	Wait                              bool
+	UseVolumeSnapshots                bool
+	DefaultResticMaintenanceFrequency time.Duration
 }
 
 // BindFlags adds command line values to the options struct.
@@ -87,6 +89,7 @@ func (o *InstallOptions) BindFlags(flags *pflag.FlagSet) {
 	flags.BoolVar(&o.DryRun, "dry-run", o.DryRun, "generate resources, but don't send them to the cluster. Use with -o. Optional.")
 	flags.BoolVar(&o.UseRestic, "use-restic", o.UseRestic, "create restic deployment. Optional.")
 	flags.BoolVar(&o.Wait, "wait", o.Wait, "wait for Velero deployment to be ready. Optional.")
+	flags.DurationVar(&o.DefaultResticMaintenanceFrequency, "default-restic-prune-frequency", o.DefaultResticMaintenanceFrequency, "how often 'restic prune' is run for restic repositories by default. Optional.")
 }
 
 // NewInstallOptions instantiates a new, default InstallOptions struct.
@@ -133,20 +136,21 @@ func (o *InstallOptions) AsVeleroOptions() (*install.VeleroOptions, error) {
 	}
 
 	return &install.VeleroOptions{
-		Namespace:          o.Namespace,
-		Image:              o.Image,
-		ProviderName:       o.ProviderName,
-		Bucket:             o.BucketName,
-		Prefix:             o.Prefix,
-		PodAnnotations:     o.PodAnnotations.Data(),
-		VeleroPodResources: veleroPodResources,
-		ResticPodResources: resticPodResources,
-		SecretData:         secretData,
-		RestoreOnly:        o.RestoreOnly,
-		UseRestic:          o.UseRestic,
-		UseVolumeSnapshots: o.UseVolumeSnapshots,
-		BSLConfig:          o.BackupStorageConfig.Data(),
-		VSLConfig:          o.VolumeSnapshotConfig.Data(),
+		Namespace:                         o.Namespace,
+		Image:                             o.Image,
+		ProviderName:                      o.ProviderName,
+		Bucket:                            o.BucketName,
+		Prefix:                            o.Prefix,
+		PodAnnotations:                    o.PodAnnotations.Data(),
+		VeleroPodResources:                veleroPodResources,
+		ResticPodResources:                resticPodResources,
+		SecretData:                        secretData,
+		RestoreOnly:                       o.RestoreOnly,
+		UseRestic:                         o.UseRestic,
+		UseVolumeSnapshots:                o.UseVolumeSnapshots,
+		BSLConfig:                         o.BackupStorageConfig.Data(),
+		VSLConfig:                         o.VolumeSnapshotConfig.Data(),
+		DefaultResticMaintenanceFrequency: o.DefaultResticMaintenanceFrequency,
 	}, nil
 }
 
@@ -291,5 +295,9 @@ func (o *InstallOptions) Validate(c *cobra.Command, args []string, f client.Fact
 		return errors.New("Cannot use both --secret-file and --no-secret")
 	}
 
+	if o.DefaultResticMaintenanceFrequency < 0 {
+		return errors.New("--default-restic-prune-frequency must be non-negative")
+	}
+
 	return nil
 }

pkg/cmd/server/server.go

@@ -118,6 +118,7 @@ type serverConfig struct {
 	clientBurst                                                             int
 	profilerAddress                                                         string
 	formatFlag                                                              *logging.FormatFlag
+	defaultResticMaintenanceFrequency                                       time.Duration
 }
 
 type controllerRunInfo struct {
@@ -130,19 +131,20 @@ func NewCommand(f client.Factory) *cobra.Command {
 		volumeSnapshotLocations = flag.NewMap().WithKeyValueDelimiter(":")
 		logLevelFlag            = logging.LogLevelFlag(logrus.InfoLevel)
 		config                  = serverConfig{
-			pluginDir:                      "/plugins",
-			metricsAddress:                 defaultMetricsAddress,
-			defaultBackupLocation:          "default",
-			defaultVolumeSnapshotLocations: make(map[string]string),
-			backupSyncPeriod:               defaultBackupSyncPeriod,
-			defaultBackupTTL:               defaultBackupTTL,
-			podVolumeOperationTimeout:      defaultPodVolumeOperationTimeout,
-			restoreResourcePriorities:      defaultRestorePriorities,
-			clientQPS:                      defaultClientQPS,
-			clientBurst:                    defaultClientBurst,
-			profilerAddress:                defaultProfilerAddress,
-			resourceTerminatingTimeout:     defaultResourceTerminatingTimeout,
-			formatFlag:                     logging.NewFormatFlag(),
+			pluginDir:                         "/plugins",
+			metricsAddress:                    defaultMetricsAddress,
+			defaultBackupLocation:             "default",
+			defaultVolumeSnapshotLocations:    make(map[string]string),
+			backupSyncPeriod:                  defaultBackupSyncPeriod,
+			defaultBackupTTL:                  defaultBackupTTL,
+			podVolumeOperationTimeout:         defaultPodVolumeOperationTimeout,
+			restoreResourcePriorities:         defaultRestorePriorities,
+			clientQPS:                         defaultClientQPS,
+			clientBurst:                       defaultClientBurst,
+			profilerAddress:                   defaultProfilerAddress,
+			resourceTerminatingTimeout:        defaultResourceTerminatingTimeout,
+			formatFlag:                        logging.NewFormatFlag(),
+			defaultResticMaintenanceFrequency: restic.DefaultMaintenanceFrequency,
 		}
 	)
 
@@ -198,6 +200,7 @@ func NewCommand(f client.Factory) *cobra.Command {
 	command.Flags().StringVar(&config.profilerAddress, "profiler-address", config.profilerAddress, "the address to expose the pprof profiler")
 	command.Flags().DurationVar(&config.resourceTerminatingTimeout, "terminating-resource-timeout", config.resourceTerminatingTimeout, "how long to wait on persistent volumes and namespaces to terminate during a restore before timing out")
 	command.Flags().DurationVar(&config.defaultBackupTTL, "default-backup-ttl", config.defaultBackupTTL, "how long to wait by default before backups can be garbage collected")
+	command.Flags().DurationVar(&config.defaultResticMaintenanceFrequency, "default-restic-prune-frequency", config.defaultResticMaintenanceFrequency, "how often 'restic prune' is run for restic repositories by default")
 
 	return command
 }
@@ -689,6 +692,7 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string
 			s.veleroClient.VeleroV1(),
 			s.sharedInformerFactory.Velero().V1().BackupStorageLocations(),
 			s.resticManager,
+			s.config.defaultResticMaintenanceFrequency,
 		)
 
 		return controllerRunInfo{

pkg/controller/restic_repository_controller.go

@@ -41,10 +41,11 @@ import (
 type resticRepositoryController struct {
 	*genericController
 
-	resticRepositoryClient velerov1client.ResticRepositoriesGetter
-	resticRepositoryLister listers.ResticRepositoryLister
-	backupLocationLister   listers.BackupStorageLocationLister
-	repositoryManager      restic.RepositoryManager
+	resticRepositoryClient      velerov1client.ResticRepositoriesGetter
+	resticRepositoryLister      listers.ResticRepositoryLister
+	backupLocationLister        listers.BackupStorageLocationLister
+	repositoryManager           restic.RepositoryManager
+	defaultMaintenanceFrequency time.Duration
 
 	clock clock.Clock
 }
@@ -56,14 +57,22 @@ func NewResticRepositoryController(
 	resticRepositoryClient velerov1client.ResticRepositoriesGetter,
 	backupLocationInformer informers.BackupStorageLocationInformer,
 	repositoryManager restic.RepositoryManager,
+	defaultMaintenanceFrequency time.Duration,
 ) Interface {
 	c := &resticRepositoryController{
-		genericController:      newGenericController("restic-repository", logger),
-		resticRepositoryClient: resticRepositoryClient,
-		resticRepositoryLister: resticRepositoryInformer.Lister(),
-		backupLocationLister:   backupLocationInformer.Lister(),
-		repositoryManager:      repositoryManager,
-		clock:                  &clock.RealClock{},
+		genericController:           newGenericController("restic-repository", logger),
+		resticRepositoryClient:      resticRepositoryClient,
+		resticRepositoryLister:      resticRepositoryInformer.Lister(),
+		backupLocationLister:        backupLocationInformer.Lister(),
+		repositoryManager:           repositoryManager,
+		defaultMaintenanceFrequency: defaultMaintenanceFrequency,
+
+		clock: &clock.RealClock{},
+	}
+
+	if c.defaultMaintenanceFrequency <= 0 {
+		logger.Infof("Invalid default restic maintenance frequency, setting to %v", restic.DefaultMaintenanceFrequency)
+		c.defaultMaintenanceFrequency = restic.DefaultMaintenanceFrequency
 	}
 
 	c.syncHandler = c.processQueueItem
@@ -159,7 +168,7 @@ func (c *resticRepositoryController) initializeRepo(req *v1.ResticRepository, lo
 			r.Status.Phase = v1.ResticRepositoryPhaseNotReady
 
 			if r.Spec.MaintenanceFrequency.Duration <= 0 {
-				r.Spec.MaintenanceFrequency = metav1.Duration{Duration: restic.DefaultMaintenanceFrequency}
+				r.Spec.MaintenanceFrequency = metav1.Duration{Duration: c.defaultMaintenanceFrequency}
 			}
 		})
 	}
@@ -169,7 +178,7 @@ func (c *resticRepositoryController) initializeRepo(req *v1.ResticRepository, lo
 		r.Spec.ResticIdentifier = repoIdentifier
 
 		if r.Spec.MaintenanceFrequency.Duration <= 0 {
-			r.Spec.MaintenanceFrequency = metav1.Duration{Duration: restic.DefaultMaintenanceFrequency}
+			r.Spec.MaintenanceFrequency = metav1.Duration{Duration: c.defaultMaintenanceFrequency}
 		}
 	}); err != nil {
 		return err

pkg/install/deployment.go

@@ -17,7 +17,9 @@ limitations under the License.
 package install
 
 import (
+	"fmt"
 	"strings"
+	"time"
 
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
@@ -27,12 +29,13 @@ import (
 type podTemplateOption func(*podTemplateConfig)
 
 type podTemplateConfig struct {
-	image       string
-	envVars     []corev1.EnvVar
-	restoreOnly bool
-	annotations map[string]string
-	resources   corev1.ResourceRequirements
-	withSecret  bool
+	image                             string
+	envVars                           []corev1.EnvVar
+	restoreOnly                       bool
+	annotations                       map[string]string
+	resources                         corev1.ResourceRequirements
+	withSecret                        bool
+	defaultResticMaintenanceFrequency time.Duration
 }
 
 func WithImage(image string) podTemplateOption {
@@ -81,6 +84,12 @@ func WithResources(resources corev1.ResourceRequirements) podTemplateOption {
 	}
 }
 
+func WithDefaultResticMaintenanceFrequency(val time.Duration) podTemplateOption {
+	return func(c *podTemplateConfig) {
+		c.defaultResticMaintenanceFrequency = val
+	}
+}
+
 func Deployment(namespace string, opts ...podTemplateOption) *appsv1.Deployment {
 	// TODO: Add support for server args
 	c := &podTemplateConfig{
@@ -218,5 +227,9 @@ func Deployment(namespace string, opts ...podTemplateOption) *appsv1.Deployment
 		deployment.Spec.Template.Spec.Containers[0].Args = append(deployment.Spec.Template.Spec.Containers[0].Args, "--restore-only")
 	}
 
+	if c.defaultResticMaintenanceFrequency > 0 {
+		deployment.Spec.Template.Spec.Containers[0].Args = append(deployment.Spec.Template.Spec.Containers[0].Args, fmt.Sprintf("--default-restic-prune-frequency=%v", c.defaultResticMaintenanceFrequency))
+	}
+
 	return deployment
 }

pkg/install/deployment_test.go

@@ -18,6 +18,7 @@ package install
 
 import (
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/assert"
 	corev1 "k8s.io/api/core/v1"
@@ -44,4 +45,8 @@ func TestDeployment(t *testing.T) {
 	deploy = Deployment("velero", WithSecret(true))
 	assert.Equal(t, 5, len(deploy.Spec.Template.Spec.Containers[0].Env))
 	assert.Equal(t, 3, len(deploy.Spec.Template.Spec.Volumes))
+
+	deploy = Deployment("velero", WithDefaultResticMaintenanceFrequency(24*time.Hour))
+	assert.Len(t, deploy.Spec.Template.Spec.Containers[0].Args, 2)
+	assert.Equal(t, "--default-restic-prune-frequency=24h0m0s", deploy.Spec.Template.Spec.Containers[0].Args[1])
 }

pkg/install/resources.go

@@ -17,6 +17,8 @@ limitations under the License.
 package install
 
 import (
+	"time"
+
 	corev1 "k8s.io/api/core/v1"
 	rbacv1beta1 "k8s.io/api/rbac/v1beta1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -193,20 +195,21 @@ func appendUnstructured(list *unstructured.UnstructuredList, obj runtime.Object)
 }
 
 type VeleroOptions struct {
-	Namespace          string
-	Image              string
-	ProviderName       string
-	Bucket             string
-	Prefix             string
-	PodAnnotations     map[string]string
-	VeleroPodResources corev1.ResourceRequirements
-	ResticPodResources corev1.ResourceRequirements
-	SecretData         []byte
-	RestoreOnly        bool
-	UseRestic          bool
-	UseVolumeSnapshots bool
-	BSLConfig          map[string]string
-	VSLConfig          map[string]string
+	Namespace                         string
+	Image                             string
+	ProviderName                      string
+	Bucket                            string
+	Prefix                            string
+	PodAnnotations                    map[string]string
+	VeleroPodResources                corev1.ResourceRequirements
+	ResticPodResources                corev1.ResourceRequirements
+	SecretData                        []byte
+	RestoreOnly                       bool
+	UseRestic                         bool
+	UseVolumeSnapshots                bool
+	BSLConfig                         map[string]string
+	VSLConfig                         map[string]string
+	DefaultResticMaintenanceFrequency time.Duration
 }
 
 // AllResources returns a list of all resources necessary to install Velero, in the appropriate order, into a Kubernetes cluster.
@@ -245,20 +248,20 @@ func AllResources(o *VeleroOptions) (*unstructured.UnstructuredList, error) {
 
 	secretPresent := o.SecretData != nil
 
-	deploy := Deployment(o.Namespace,
+	deployOpts := []podTemplateOption{
 		WithAnnotations(o.PodAnnotations),
 		WithImage(o.Image),
 		WithResources(o.VeleroPodResources),
 		WithSecret(secretPresent),
-	)
+		WithDefaultResticMaintenanceFrequency(o.DefaultResticMaintenanceFrequency),
+	}
+
 	if o.RestoreOnly {
-		deploy = Deployment(o.Namespace,
-			WithAnnotations(o.PodAnnotations),
-			WithImage(o.Image),
-			WithSecret(secretPresent),
-			WithRestoreOnly(),
-		)
+		deployOpts = append(deployOpts, WithRestoreOnly())
 	}
+
+	deploy := Deployment(o.Namespace, deployOpts...)
+
 	appendUnstructured(resources, deploy)
 
 	if o.UseRestic {

pkg/restic/common.go

@@ -43,8 +43,8 @@ const (
 	InitContainer = "restic-wait"
 
 	// DefaultMaintenanceFrequency is the default time interval
-	// at which restic check & prune are run.
-	DefaultMaintenanceFrequency = 24 * time.Hour
+	// at which restic prune is run.
+	DefaultMaintenanceFrequency = 7 * 24 * time.Hour
 
 	// PVCNameAnnotation is the key for the annotation added to
 	// pod volume backups when they're for a PVC.