From 8e967225319753f59f2e19c6cfa7da9fa1375cf8 Mon Sep 17 00:00:00 2001 From: Bridget McErlean Date: Tue, 19 Jan 2021 09:09:52 -0500 Subject: [PATCH] Load credentials and pass through via config Update NewObjectBackupStore to take a CredentialsGetter which can be used to get the credentials for a BackupStorageLocation if it has been configured with a Credential. If the BSL has a credential, use that SecretKeySelector to fetch the secret, write the contents to a temp file and then pass that file through to the plugin via the config map. This relies on the plugin being able to use the config field. This does not yet handle VolumeSnapshotLocations or ResticRepositories. Signed-off-by: Bridget McErlean --- pkg/cmd/server/server.go | 15 +++- pkg/controller/backup_controller.go | 10 ++- pkg/controller/backup_deletion_controller.go | 8 +- .../backup_storage_location_controller.go | 7 +- pkg/controller/backup_sync_controller.go | 8 +- pkg/controller/download_request_controller.go | 8 +- pkg/controller/restore_controller.go | 10 ++- pkg/credentials/credentials.go | 83 +++++++++++++++++++ pkg/persistence/object_store.go | 19 ++++- 9 files changed, 148 insertions(+), 20 deletions(-) create mode 100644 pkg/credentials/credentials.go diff --git a/pkg/cmd/server/server.go b/pkg/cmd/server/server.go index 8a51eeb031..4f75e59571 100644 --- a/pkg/cmd/server/server.go +++ b/pkg/cmd/server/server.go @@ -56,6 +56,7 @@ import ( "github.com/vmware-tanzu/velero/pkg/cmd" "github.com/vmware-tanzu/velero/pkg/cmd/util/flag" "github.com/vmware-tanzu/velero/pkg/cmd/util/signals" + "github.com/vmware-tanzu/velero/pkg/credentials" "github.com/vmware-tanzu/velero/pkg/controller" velerodiscovery "github.com/vmware-tanzu/velero/pkg/discovery" @@ -567,6 +568,8 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string } csiVSLister, csiVSCLister := s.getCSISnapshotListers() + credentialsGetter := credentials.NewCredentialsGetter(s.kubeClient, s.namespace) + backupSyncControllerRunInfo := func() controllerRunInfo { backupSyncContoller := controller.NewBackupSyncController( s.veleroClient.VeleroV1(), @@ -579,6 +582,7 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string s.kubeClient, s.config.defaultBackupLocation, newPluginManager, + credentialsGetter, s.logger, ) @@ -621,6 +625,7 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string s.config.formatFlag.Parse(), csiVSLister, csiVSCLister, + credentialsGetter, ) return controllerRunInfo{ @@ -679,6 +684,7 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string newPluginManager, s.metrics, s.discoveryHelper, + credentialsGetter, ) return controllerRunInfo{ @@ -716,6 +722,7 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string newPluginManager, s.metrics, s.config.formatFlag.Parse(), + credentialsGetter, ) return controllerRunInfo{ @@ -748,6 +755,7 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string s.mgr.GetClient(), s.sharedInformerFactory.Velero().V1().Backups().Lister(), newPluginManager, + credentialsGetter, s.logger, ) @@ -834,9 +842,10 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string StorageLocation: s.config.defaultBackupLocation, ServerValidationFrequency: s.config.storeValidationFrequency, }, - NewPluginManager: newPluginManager, - NewBackupStore: persistence.NewObjectBackupStore, - Log: s.logger, + NewPluginManager: newPluginManager, + NewBackupStore: persistence.NewObjectBackupStore, + Log: s.logger, + CredentialsGetter: credentialsGetter, } if err := bslr.SetupWithManager(s.mgr); err != nil { s.logger.Fatal(err, "unable to create controller", "controller", controller.BackupStorageLocation) diff --git a/pkg/controller/backup_controller.go b/pkg/controller/backup_controller.go index c29bab22bd..ab3a745041 100644 --- a/pkg/controller/backup_controller.go +++ b/pkg/controller/backup_controller.go @@ -44,6 +44,7 @@ import ( "github.com/vmware-tanzu/velero/internal/storage" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" pkgbackup "github.com/vmware-tanzu/velero/pkg/backup" + "github.com/vmware-tanzu/velero/pkg/credentials" "github.com/vmware-tanzu/velero/pkg/discovery" "github.com/vmware-tanzu/velero/pkg/features" velerov1client "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/typed/velero/v1" @@ -80,10 +81,11 @@ type backupController struct { snapshotLocationLister velerov1listers.VolumeSnapshotLocationLister defaultSnapshotLocations map[string]string metrics *metrics.ServerMetrics - newBackupStore func(*velerov1api.BackupStorageLocation, persistence.ObjectStoreGetter, logrus.FieldLogger) (persistence.BackupStore, error) + newBackupStore func(*velerov1api.BackupStorageLocation, persistence.ObjectStoreGetter, credentials.Getter, logrus.FieldLogger) (persistence.BackupStore, error) formatFlag logging.Format volumeSnapshotLister snapshotv1beta1listers.VolumeSnapshotLister volumeSnapshotContentLister snapshotv1beta1listers.VolumeSnapshotContentLister + credentialsGetter credentials.Getter } func NewBackupController( @@ -105,6 +107,7 @@ func NewBackupController( formatFlag logging.Format, volumeSnapshotLister snapshotv1beta1listers.VolumeSnapshotLister, volumeSnapshotContentLister snapshotv1beta1listers.VolumeSnapshotContentLister, + credentialsGetter credentials.Getter, ) Interface { c := &backupController{ genericController: newGenericController(Backup, logger), @@ -126,6 +129,7 @@ func NewBackupController( formatFlag: formatFlag, volumeSnapshotLister: volumeSnapshotLister, volumeSnapshotContentLister: volumeSnapshotContentLister, + credentialsGetter: credentialsGetter, newBackupStore: persistence.NewObjectBackupStore, } @@ -570,7 +574,7 @@ func (c *backupController) runBackup(backup *pkgbackup.Request) error { } backupLog.Info("Setting up backup store to check for backup existence") - backupStore, err := c.newBackupStore(backup.StorageLocation, pluginManager, backupLog) + backupStore, err := c.newBackupStore(backup.StorageLocation, pluginManager, c.credentialsGetter, backupLog) if err != nil { return err } @@ -651,7 +655,7 @@ func (c *backupController) runBackup(backup *pkgbackup.Request) error { // re-instantiate the backup store because credentials could have changed since the original // instantiation, if this was a long-running backup backupLog.Info("Setting up backup store to persist the backup") - backupStore, err = c.newBackupStore(backup.StorageLocation, pluginManager, backupLog) + backupStore, err = c.newBackupStore(backup.StorageLocation, pluginManager, c.credentialsGetter, backupLog) if err != nil { return err } diff --git a/pkg/controller/backup_deletion_controller.go b/pkg/controller/backup_deletion_controller.go index ffc0f8b9a3..1acbe3b129 100644 --- a/pkg/controller/backup_deletion_controller.go +++ b/pkg/controller/backup_deletion_controller.go @@ -39,6 +39,7 @@ import ( "github.com/vmware-tanzu/velero/internal/delete" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" pkgbackup "github.com/vmware-tanzu/velero/pkg/backup" + "github.com/vmware-tanzu/velero/pkg/credentials" "github.com/vmware-tanzu/velero/pkg/discovery" "github.com/vmware-tanzu/velero/pkg/features" velerov1client "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/typed/velero/v1" @@ -77,9 +78,10 @@ type backupDeletionController struct { processRequestFunc func(*velerov1api.DeleteBackupRequest) error clock clock.Clock newPluginManager func(logrus.FieldLogger) clientmgmt.Manager - newBackupStore func(*velerov1api.BackupStorageLocation, persistence.ObjectStoreGetter, logrus.FieldLogger) (persistence.BackupStore, error) + newBackupStore func(*velerov1api.BackupStorageLocation, persistence.ObjectStoreGetter, credentials.Getter, logrus.FieldLogger) (persistence.BackupStore, error) metrics *metrics.ServerMetrics helper discovery.Helper + credentialsGetter credentials.Getter } // NewBackupDeletionController creates a new backup deletion controller. @@ -101,6 +103,7 @@ func NewBackupDeletionController( newPluginManager func(logrus.FieldLogger) clientmgmt.Manager, metrics *metrics.ServerMetrics, helper discovery.Helper, + credentialsGetter credentials.Getter, ) Interface { c := &backupDeletionController{ genericController: newGenericController(BackupDeletion, logger), @@ -119,6 +122,7 @@ func NewBackupDeletionController( csiSnapshotClient: csiSnapshotClient, metrics: metrics, helper: helper, + credentialsGetter: credentialsGetter, // use variables to refer to these functions so they can be // replaced with fakes for testing. newPluginManager: newPluginManager, @@ -290,7 +294,7 @@ func (c *backupDeletionController) processRequest(req *velerov1api.DeleteBackupR pluginManager := c.newPluginManager(log) defer pluginManager.CleanupClients() - backupStore, err := c.newBackupStore(location, pluginManager, log) + backupStore, err := c.newBackupStore(location, pluginManager, c.credentialsGetter, log) if err != nil { errs = append(errs, err.Error()) } diff --git a/pkg/controller/backup_storage_location_controller.go b/pkg/controller/backup_storage_location_controller.go index bdac891e01..6f1c4a7a7d 100644 --- a/pkg/controller/backup_storage_location_controller.go +++ b/pkg/controller/backup_storage_location_controller.go @@ -32,6 +32,7 @@ import ( "github.com/vmware-tanzu/velero/internal/storage" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "github.com/vmware-tanzu/velero/pkg/credentials" "github.com/vmware-tanzu/velero/pkg/persistence" "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt" ) @@ -42,10 +43,12 @@ type BackupStorageLocationReconciler struct { Client client.Client Scheme *runtime.Scheme DefaultBackupLocationInfo storage.DefaultBackupLocationInfo + CredentialsGetter credentials.Getter + // use variables to refer to these functions so they can be // replaced with fakes for testing. NewPluginManager func(logrus.FieldLogger) clientmgmt.Manager - NewBackupStore func(*velerov1api.BackupStorageLocation, persistence.ObjectStoreGetter, logrus.FieldLogger) (persistence.BackupStore, error) + NewBackupStore func(*velerov1api.BackupStorageLocation, persistence.ObjectStoreGetter, credentials.Getter, logrus.FieldLogger) (persistence.BackupStore, error) Log logrus.FieldLogger } @@ -95,7 +98,7 @@ func (r *BackupStorageLocationReconciler) Reconcile(req ctrl.Request) (ctrl.Resu continue } - backupStore, err := r.NewBackupStore(location, pluginManager, log) + backupStore, err := r.NewBackupStore(location, pluginManager, r.CredentialsGetter, log) if err != nil { log.WithError(err).Error("Error getting a backup store") continue diff --git a/pkg/controller/backup_sync_controller.go b/pkg/controller/backup_sync_controller.go index c76fbada27..129882c614 100644 --- a/pkg/controller/backup_sync_controller.go +++ b/pkg/controller/backup_sync_controller.go @@ -31,6 +31,7 @@ import ( "github.com/vmware-tanzu/velero/internal/storage" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "github.com/vmware-tanzu/velero/pkg/credentials" "github.com/vmware-tanzu/velero/pkg/features" velerov1client "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/typed/velero/v1" velerov1listers "github.com/vmware-tanzu/velero/pkg/generated/listers/velero/v1" @@ -54,7 +55,8 @@ type backupSyncController struct { defaultBackupLocation string defaultBackupSyncPeriod time.Duration newPluginManager func(logrus.FieldLogger) clientmgmt.Manager - newBackupStore func(*velerov1api.BackupStorageLocation, persistence.ObjectStoreGetter, logrus.FieldLogger) (persistence.BackupStore, error) + newBackupStore func(*velerov1api.BackupStorageLocation, persistence.ObjectStoreGetter, credentials.Getter, logrus.FieldLogger) (persistence.BackupStore, error) + credentialsGetter credentials.Getter } func NewBackupSyncController( @@ -68,6 +70,7 @@ func NewBackupSyncController( kubeClient kubernetes.Interface, defaultBackupLocation string, newPluginManager func(logrus.FieldLogger) clientmgmt.Manager, + credentialsGetter credentials.Getter, logger logrus.FieldLogger, ) Interface { if syncPeriod <= 0 { @@ -86,6 +89,7 @@ func NewBackupSyncController( backupLister: backupLister, csiSnapshotClient: csiSnapshotClient, kubeClient: kubeClient, + credentialsGetter: credentialsGetter, // use variables to refer to these functions so they can be // replaced with fakes for testing. @@ -169,7 +173,7 @@ func (c *backupSyncController) run() { log.Debug("Checking backup location for backups to sync into cluster") - backupStore, err := c.newBackupStore(&location, pluginManager, log) + backupStore, err := c.newBackupStore(&location, pluginManager, c.credentialsGetter, log) if err != nil { log.WithError(err).Error("Error getting backup store for this location") continue diff --git a/pkg/controller/download_request_controller.go b/pkg/controller/download_request_controller.go index 3ebd385ef7..59fcadda49 100644 --- a/pkg/controller/download_request_controller.go +++ b/pkg/controller/download_request_controller.go @@ -33,6 +33,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "github.com/vmware-tanzu/velero/pkg/credentials" velerov1client "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/typed/velero/v1" velerov1informers "github.com/vmware-tanzu/velero/pkg/generated/informers/externalversions/velero/v1" velerov1listers "github.com/vmware-tanzu/velero/pkg/generated/listers/velero/v1" @@ -50,8 +51,9 @@ type downloadRequestController struct { clock clock.Clock kbClient client.Client backupLister velerov1listers.BackupLister + credentialsGetter credentials.Getter newPluginManager func(logrus.FieldLogger) clientmgmt.Manager - newBackupStore func(*velerov1api.BackupStorageLocation, persistence.ObjectStoreGetter, logrus.FieldLogger) (persistence.BackupStore, error) + newBackupStore func(*velerov1api.BackupStorageLocation, persistence.ObjectStoreGetter, credentials.Getter, logrus.FieldLogger) (persistence.BackupStore, error) } // NewDownloadRequestController creates a new DownloadRequestController. @@ -62,6 +64,7 @@ func NewDownloadRequestController( kbClient client.Client, backupLister velerov1listers.BackupLister, newPluginManager func(logrus.FieldLogger) clientmgmt.Manager, + credentialsGetter credentials.Getter, logger logrus.FieldLogger, ) Interface { c := &downloadRequestController{ @@ -71,6 +74,7 @@ func NewDownloadRequestController( restoreLister: restoreLister, kbClient: kbClient, backupLister: backupLister, + credentialsGetter: credentialsGetter, // use variables to refer to these functions so they can be // replaced with fakes for testing. @@ -172,7 +176,7 @@ func (c *downloadRequestController) generatePreSignedURL(downloadRequest *velero pluginManager := c.newPluginManager(log) defer pluginManager.CleanupClients() - backupStore, err := c.newBackupStore(backupLocation, pluginManager, log) + backupStore, err := c.newBackupStore(backupLocation, pluginManager, c.credentialsGetter, log) if err != nil { return errors.WithStack(err) } diff --git a/pkg/controller/restore_controller.go b/pkg/controller/restore_controller.go index fe79f00831..3d5fa5edd7 100644 --- a/pkg/controller/restore_controller.go +++ b/pkg/controller/restore_controller.go @@ -40,6 +40,7 @@ import ( api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "github.com/vmware-tanzu/velero/pkg/credentials" velerov1client "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/typed/velero/v1" velerov1informers "github.com/vmware-tanzu/velero/pkg/generated/informers/externalversions/velero/v1" velerov1listers "github.com/vmware-tanzu/velero/pkg/generated/listers/velero/v1" @@ -91,9 +92,10 @@ type restoreController struct { metrics *metrics.ServerMetrics logFormat logging.Format clock clock.Clock + credentialsGetter credentials.Getter newPluginManager func(logger logrus.FieldLogger) clientmgmt.Manager - newBackupStore func(*velerov1api.BackupStorageLocation, persistence.ObjectStoreGetter, logrus.FieldLogger) (persistence.BackupStore, error) + newBackupStore func(*velerov1api.BackupStorageLocation, persistence.ObjectStoreGetter, credentials.Getter, logrus.FieldLogger) (persistence.BackupStore, error) } func NewRestoreController( @@ -110,6 +112,7 @@ func NewRestoreController( newPluginManager func(logrus.FieldLogger) clientmgmt.Manager, metrics *metrics.ServerMetrics, logFormat logging.Format, + credentialsGetter credentials.Getter, ) Interface { c := &restoreController{ genericController: newGenericController(Restore, logger), @@ -125,6 +128,7 @@ func NewRestoreController( metrics: metrics, logFormat: logFormat, clock: &clock.RealClock{}, + credentialsGetter: credentialsGetter, // use variables to refer to these functions so they can be // replaced with fakes for testing. @@ -410,7 +414,7 @@ func (c *restoreController) fetchBackupInfo(backupName string, pluginManager cli return backupInfo{}, errors.WithStack(err) } - backupStore, err := c.newBackupStore(location, pluginManager, c.logger) + backupStore, err := c.newBackupStore(location, pluginManager, c.credentialsGetter, c.logger) if err != nil { return backupInfo{}, err } @@ -480,7 +484,7 @@ func (c *restoreController) runValidatedRestore(restore *api.Restore, info backu // re-instantiate the backup store because credentials could have changed since the original // instantiation, if this was a long-running restore - info.backupStore, err = c.newBackupStore(info.location, pluginManager, c.logger) + info.backupStore, err = c.newBackupStore(info.location, pluginManager, c.credentialsGetter, c.logger) if err != nil { return errors.Wrap(err, "error setting up backup store to persist log and results files") } diff --git a/pkg/credentials/credentials.go b/pkg/credentials/credentials.go new file mode 100644 index 0000000000..ef24adfa85 --- /dev/null +++ b/pkg/credentials/credentials.go @@ -0,0 +1,83 @@ +/* +Copyright 2021 the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package credentials + +import ( + "context" + "fmt" + "io/ioutil" + + "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" +) + +type Getter interface { + Get(selector *corev1.SecretKeySelector) ([]byte, error) + + // GetAsFile gets the secret data specified by the selector and returns a filepath + // where the contents have been written to + GetAsFile(selector *corev1.SecretKeySelector) (string, error) +} + +type credentialsGetter struct { + client kubernetes.Interface + namespace string + credentialsFiles map[corev1.SecretKeySelector]string +} + +func (c *credentialsGetter) Get(selector *corev1.SecretKeySelector) ([]byte, error) { + secret, err := c.client.CoreV1().Secrets(c.namespace).Get(context.TODO(), selector.Name, metav1.GetOptions{}) + if err != nil { + return []byte{}, err + } + + data, ok := secret.Data[selector.Key] + if !ok { + return []byte{}, fmt.Errorf("invalid key selector, key %q does not exist in secret %q", selector.Key, selector.Name) + } + return data, nil +} + +func (c *credentialsGetter) GetAsFile(selector *corev1.SecretKeySelector) (string, error) { + creds, err := c.Get(selector) + if err != nil { + return "", err + } + + filePath, ok := c.credentialsFiles[*selector] + if !ok { + tmpFile, err := ioutil.TempFile("", "creds") + if err != nil { + return "", errors.Wrap(err, "unable to create temp file to write credentials to") + } + filePath = tmpFile.Name() + } + + if err := ioutil.WriteFile(filePath, creds, 0644); err != nil { + return "", errors.Wrap(err, "unable to write credentials to temp file") + } + return filePath, nil +} + +func NewCredentialsGetter(client kubernetes.Interface, namespace string) Getter { + return &credentialsGetter{ + client: client, + namespace: namespace, + } +} diff --git a/pkg/persistence/object_store.go b/pkg/persistence/object_store.go index 545f3f0423..216ce8f63a 100644 --- a/pkg/persistence/object_store.go +++ b/pkg/persistence/object_store.go @@ -27,12 +27,12 @@ import ( snapshotv1beta1api "github.com/kubernetes-csi/external-snapshotter/v2/pkg/apis/volumesnapshot/v1beta1" "github.com/pkg/errors" "github.com/sirupsen/logrus" - kerrors "k8s.io/apimachinery/pkg/util/errors" - velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "github.com/vmware-tanzu/velero/pkg/credentials" "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/scheme" "github.com/vmware-tanzu/velero/pkg/plugin/velero" "github.com/vmware-tanzu/velero/pkg/volume" + kerrors "k8s.io/apimachinery/pkg/util/errors" ) type BackupInfo struct { @@ -90,7 +90,7 @@ type ObjectStoreGetter interface { GetObjectStore(provider string) (velero.ObjectStore, error) } -func NewObjectBackupStore(location *velerov1api.BackupStorageLocation, objectStoreGetter ObjectStoreGetter, logger logrus.FieldLogger) (BackupStore, error) { +func NewObjectBackupStore(location *velerov1api.BackupStorageLocation, objectStoreGetter ObjectStoreGetter, credentialsGetter credentials.Getter, logger logrus.FieldLogger) (BackupStore, error) { if location.Spec.ObjectStorage == nil { return nil, errors.New("backup storage location does not use object storage") } @@ -125,6 +125,19 @@ func NewObjectBackupStore(location *velerov1api.BackupStorageLocation, objectSto } } + if location.Spec.Credential != nil { + if location.Spec.Config == nil { + location.Spec.Config = make(map[string]string) + } + + credsFile, err := credentialsGetter.GetAsFile(location.Spec.Credential) + if err != nil { + return nil, errors.Wrap(err, "unable to get credentials") + } + + location.Spec.Config["credentialsFile"] = credsFile + } + objectStore, err := objectStoreGetter.GetObjectStore(location.Spec.Provider) if err != nil { return nil, err