feat(backup): multiple backup stores support

api/backup.go

@@ -3,23 +3,129 @@ package api
 import (
 	"fmt"
 	"net/http"
+	"strconv"
+	"time"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	"github.com/gorilla/mux"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 
 	"github.com/rancher/go-rancher/api"
 	"github.com/rancher/go-rancher/client"
+
+	longhorn "github.com/longhorn/longhorn-manager/k8s/pkg/apis/longhorn/v1beta2"
+	"github.com/longhorn/longhorn-manager/util"
 )
 
+const (
+	BackupTargetDefaultPollInterval = 300
+)
+
+func (s *Server) BackupTargetGet(w http.ResponseWriter, req *http.Request) error {
+	apiContext := api.GetApiContext(req)
+
+	backupTargetName := mux.Vars(req)["name"]
+	backupTarget, err := s.m.GetBackupTarget(backupTargetName)
+	if err != nil {
+		return errors.Wrapf(err, "failed to get backup target %v", backupTargetName)
+	}
+	apiContext.Write(toBackupTargetResource(backupTarget, apiContext))
+	return nil
+}
+
 func (s *Server) BackupTargetList(w http.ResponseWriter, req *http.Request) error {
 	apiContext := api.GetApiContext(req)
 
-	backupTargets, err := s.m.ListBackupTargetsSorted()
+	bts, err := s.backupTargetList(apiContext)
 	if err != nil {
 		return errors.Wrap(err, "failed to list backup targets")
 	}
-	apiContext.Write(toBackupTargetCollection(backupTargets, apiContext))
+
+	apiContext.Write(bts)
+	return nil
+}
+
+func (s *Server) backupTargetList(apiContext *api.ApiContext) (*client.GenericCollection, error) {
+	bts, err := s.m.ListBackupTargetsSorted()
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to list backup target")
+	}
+	return toBackupTargetCollection(bts, apiContext), nil
+}
+
+func (s *Server) BackupTargetCreate(rw http.ResponseWriter, req *http.Request) error {
+	var input BackupTarget
+	apiContext := api.GetApiContext(req)
+
+	if err := apiContext.Read(&input); err != nil {
+		return err
+	}
+
+	backupTargetSpec, err := newBackupTarget(input)
+	if err != nil {
+		return err
+	}
+
+	obj, err := s.m.CreateBackupTarget(input.Name, backupTargetSpec)
+	if err != nil {
+		return errors.Wrapf(err, "failed to create backup target %v", input.Name)
+	}
+	apiContext.Write(toBackupTargetResource(obj, apiContext))
+	return nil
+}
+
+func newBackupTarget(input BackupTarget) (*longhorn.BackupTargetSpec, error) {
+	pollInterval, err := strconv.ParseInt(input.PollInterval, 10, 64)
+	if err != nil {
+		return nil, errors.Wrapf(err, "invalid backup target polling interval '%s'", input.PollInterval)
+	}
+
+	return &longhorn.BackupTargetSpec{
+		BackupTargetURL:  input.BackupTargetURL,
+		CredentialSecret: input.CredentialSecret,
+		PollInterval:     metav1.Duration{Duration: time.Duration(pollInterval) * time.Second},
+		ReadOnly:         input.ReadOnly}, nil
+}
+
+func (s *Server) BackupTargetUpdate(rw http.ResponseWriter, req *http.Request) error {
+	var input BackupTarget
+
+	apiContext := api.GetApiContext(req)
+	if err := apiContext.Read(&input); err != nil {
+		return err
+	}
+
+	name := mux.Vars(req)["name"]
+
+	backupTargetSpec, err := newBackupTarget(input)
+	if err != nil {
+		return err
+	}
+
+	obj, err := util.RetryOnConflictCause(func() (interface{}, error) {
+		return s.m.UpdateBackupTarget(name, backupTargetSpec)
+	})
+	if err != nil {
+		return errors.Wrapf(err, "failed to update backup target %v", name)
+	}
+
+	backupTarget, ok := obj.(*longhorn.BackupTarget)
+	if !ok {
+		return fmt.Errorf("failed to convert %v to backup target", name)
+	}
+
+	apiContext.Write(toBackupTargetResource(backupTarget, apiContext))
+	return nil
+}
+
+func (s *Server) BackupTargetDelete(rw http.ResponseWriter, req *http.Request) error {
+	backupTargetName := mux.Vars(req)["name"]
+	if err := s.m.DeleteBackupTarget(backupTargetName); err != nil {
+		return errors.Wrapf(err, "failed to delete backup target %v", backupTargetName)
+	}
+
 	return nil
 }
 
@@ -174,6 +280,17 @@ func (s *Server) BackupVolumeDelete(w http.ResponseWriter, req *http.Request) er
 func (s *Server) BackupList(w http.ResponseWriter, req *http.Request) error {
 	apiContext := api.GetApiContext(req)
 
+	bs, err := s.m.ListAllBackupsSorted()
+	if err != nil {
+		return errors.Wrapf(err, "failed to list all backups")
+	}
+	apiContext.Write(toBackupCollection(bs))
+	return nil
+}
+
+func (s *Server) BackupListByVolumeName(w http.ResponseWriter, req *http.Request) error {
+	apiContext := api.GetApiContext(req)
+
 	volName := mux.Vars(req)["volName"]
 
 	bs, err := s.m.ListBackupsForVolumeSorted(volName)

api/backupbackingimage.go

@@ -63,9 +63,16 @@ func (s *Server) BackupBackingImageRestore(w http.ResponseWriter, req *http.Requ
 }
 
 func (s *Server) BackupBackingImageCreate(w http.ResponseWriter, req *http.Request) error {
-	backupBackingImageName := mux.Vars(req)["name"]
-	if err := s.m.CreateBackupBackingImage(backupBackingImageName); err != nil {
-		return errors.Wrapf(err, "failed to create backup backing image '%s'", backupBackingImageName)
+	var input BackupBackingImage
+
+	apiContext := api.GetApiContext(req)
+	if err := apiContext.Read(&input); err != nil {
+		return err
+	}
+
+	backingImageName := mux.Vars(req)["name"]
+	if err := s.m.CreateBackupBackingImage(input.Name, backingImageName, input.BackupTargetName); err != nil {
+		return errors.Wrapf(err, "failed to create backup backing image '%s'", input.Name)
 	}
 	return nil
 }

api/model.go

@@ -61,6 +61,7 @@ type Volume struct {
 	SnapshotMaxCount            int                                    `json:"snapshotMaxCount"`
 	SnapshotMaxSize             string                                 `json:"snapshotMaxSize"`
 	FreezeFilesystemForSnapshot longhorn.FreezeFilesystemForSnapshot   `json:"freezeFilesystemForSnapshot"`
+	BackupTargetName            string                                 `json:"backupTargetName"`
 
 	DiskSelector         []string                      `json:"diskSelector"`
 	NodeSelector         []string                      `json:"nodeSelector"`
@@ -139,6 +140,8 @@ type BackupVolume struct {
 	BackingImageName     string            `json:"backingImageName"`
 	BackingImageChecksum string            `json:"backingImageChecksum"`
 	StorageClassName     string            `json:"storageClassName"`
+	BackupTargetName     string            `json:"backupTargetName"`
+	VolumeName           string            `json:"volumeName"`
 }
 
 // SyncBackupResource is used for the Backup*Sync* actions
@@ -171,6 +174,7 @@ type Backup struct {
 	CompressionMethod      string               `json:"compressionMethod"`
 	NewlyUploadedDataSize  string               `json:"newlyUploadDataSize"`
 	ReUploadedDataSize     string               `json:"reUploadedDataSize"`
+	BackupTargetName       string               `json:"backupTargetName"`
 }
 
 type BackupBackingImage struct {
@@ -187,6 +191,8 @@ type BackupBackingImage struct {
 	CompressionMethod string               `json:"compressionMethod"`
 	Secret            string               `json:"secret"`
 	SecretNamespace   string               `json:"secretNamespace"`
+	BackingImageName  string               `json:"backingImageName"`
+	BackupTargetName  string               `json:"backupTargetName"`
 }
 
 type Setting struct {
@@ -871,14 +877,44 @@ func kubernetesStatusSchema(status *client.Schema) {
 }
 
 func backupTargetSchema(backupTarget *client.Schema) {
-	backupTarget.CollectionMethods = []string{"GET"}
-	backupTarget.ResourceMethods = []string{"GET", "PUT"}
+	backupTarget.CollectionMethods = []string{"GET", "POST"}
+	backupTarget.ResourceMethods = []string{"GET", "PUT", "DELETE"}
+
+	backupTargetName := backupTarget.ResourceFields["name"]
+	backupTargetName.Required = true
+	backupTargetName.Unique = true
+	backupTargetName.Create = true
+	backupTarget.ResourceFields["name"] = backupTargetName
+
+	backupTargetURL := backupTarget.ResourceFields["backupTargetURL"]
+	backupTargetURL.Create = true
+	backupTargetURL.Default = ""
+	backupTarget.ResourceFields["backupTargetURL"] = backupTargetURL
+
+	credentialSecret := backupTarget.ResourceFields["credentialSecret"]
+	credentialSecret.Create = true
+	credentialSecret.Default = ""
+	backupTarget.ResourceFields["credentialSecret"] = credentialSecret
+
+	backupTargetPollInterval := backupTarget.ResourceFields["pollInterval"]
+	backupTargetPollInterval.Create = true
+	backupTargetPollInterval.Default = "300"
+	backupTarget.ResourceFields["pollInterval"] = backupTargetPollInterval
+
+	backupTargetReadOnly := backupTarget.ResourceFields["readOnly"]
+	backupTargetReadOnly.Create = true
+	backupTargetReadOnly.Default = false
+	backupTarget.ResourceFields["readOnly"] = backupTargetReadOnly
 
 	backupTarget.ResourceActions = map[string]client.Action{
 		"backupTargetSync": {
 			Input:  "syncBackupResource",
 			Output: "backupTargetListOutput",
 		},
+		"backupTargetUpdate": {
+			Input:  "BackupTarget",
+			Output: "backupTargetListOutput",
+		},
 	}
 }
 
@@ -1562,6 +1598,7 @@ func toVolumeResource(v *longhorn.Volume, ves []*longhorn.Engine, vrs []*longhor
 		NodeSelector:                v.Spec.NodeSelector,
 		RestoreVolumeRecurringJob:   v.Spec.RestoreVolumeRecurringJob,
 		FreezeFilesystemForSnapshot: v.Spec.FreezeFilesystemForSnapshot,
+		BackupTargetName:            v.Spec.BackupTargetName,
 
 		State:                       v.Status.State,
 		Robustness:                  v.Status.Robustness,
@@ -1797,11 +1834,13 @@ func toBackupTargetResource(bt *longhorn.BackupTarget, apiContext *api.ApiContex
 			CredentialSecret: bt.Spec.CredentialSecret,
 			PollInterval:     bt.Spec.PollInterval.Duration.String(),
 			Available:        bt.Status.Available,
+			ReadOnly:         bt.Spec.ReadOnly,
 			Message:          types.GetCondition(bt.Status.Conditions, longhorn.BackupTargetConditionTypeUnavailable).Message,
 		},
 	}
 	res.Actions = map[string]string{
-		"backupTargetSync": apiContext.UrlBuilder.ActionLink(res.Resource, "backupTargetSync"),
+		"backupTargetSync":   apiContext.UrlBuilder.ActionLink(res.Resource, "backupTargetSync"),
+		"backupTargetUpdate": apiContext.UrlBuilder.ActionLink(res.Resource, "backupTargetUpdate"),
 	}
 
 	return res
@@ -1828,6 +1867,8 @@ func toBackupVolumeResource(bv *longhorn.BackupVolume, apiContext *api.ApiContex
 		BackingImageName:     bv.Status.BackingImageName,
 		BackingImageChecksum: bv.Status.BackingImageChecksum,
 		StorageClassName:     bv.Status.StorageClassName,
+		BackupTargetName:     bv.Spec.BackupTargetName,
+		VolumeName:           bv.Spec.VolumeName,
 	}
 	b.Actions = map[string]string{
 		"backupList":       apiContext.UrlBuilder.ActionLink(b.Resource, "backupList"),
@@ -1877,6 +1918,8 @@ func toBackupBackingImageResource(bbi *longhorn.BackupBackingImage, apiContext *
 		CompressionMethod: string(bbi.Status.CompressionMethod),
 		Secret:            bbi.Status.Secret,
 		SecretNamespace:   bbi.Status.SecretNamespace,
+		BackingImageName:  bbi.Spec.BackingImage,
+		BackupTargetName:  bbi.Spec.BackupTargetName,
 	}
 
 	backupBackingImage.Actions = map[string]string{
@@ -1932,6 +1975,7 @@ func toBackupResource(b *longhorn.Backup) *Backup {
 		CompressionMethod:      string(b.Status.CompressionMethod),
 		NewlyUploadedDataSize:  b.Status.NewlyUploadedDataSize,
 		ReUploadedDataSize:     b.Status.ReUploadedDataSize,
+		BackupTargetName:       b.Spec.BackupTargetName,
 	}
 	// Set the volume name from backup CR's label if it's empty.
 	// This field is empty probably because the backup state is not Ready

api/router.go

@@ -122,11 +122,15 @@ func NewRouter(s *Server) *mux.Router {
 	r.Methods("GET").Path("/v1/backuptargets").Handler(f(schemas, s.BackupTargetList))
 	r.Methods("PUT").Path("/v1/backuptargets").Handler(f(schemas, s.BackupTargetSyncAll))
 	backupTargetActions := map[string]func(http.ResponseWriter, *http.Request) error{
-		"backupTargetSync": s.fwd.Handler(s.fwd.HandleProxyRequestByNodeID, s.fwd.GetHTTPAddressByNodeID(NodeHasDefaultEngineImage(s.m)), s.BackupTargetSync),
+		"backupTargetSync":   s.fwd.Handler(s.fwd.HandleProxyRequestByNodeID, s.fwd.GetHTTPAddressByNodeID(NodeHasDefaultEngineImage(s.m)), s.BackupTargetSync),
+		"backupTargetUpdate": s.fwd.Handler(s.fwd.HandleProxyRequestByNodeID, s.fwd.GetHTTPAddressByNodeID(NodeHasDefaultEngineImage(s.m)), s.BackupTargetUpdate),
 	}
 	for name, action := range backupTargetActions {
 		r.Methods("POST").Path("/v1/backuptargets/{backupTargetName}").Queries("action", name).Handler(f(schemas, action))
 	}
+	r.Methods("GET").Path("/v1/backuptargets/{name}").Handler(f(schemas, s.BackupTargetGet))
+	r.Methods("POST").Path("/v1/backuptargets").Handler(f(schemas, s.BackupTargetCreate))
+	r.Methods("DELETE").Path("/v1/backuptargets/{name}").Handler(f(schemas, s.BackupTargetDelete))
 	r.Methods("GET").Path("/v1/backupvolumes").Handler(f(schemas, s.fwd.Handler(s.fwd.HandleProxyRequestByNodeID, s.fwd.GetHTTPAddressByNodeID(NodeHasDefaultEngineImage(s.m)), s.BackupVolumeList)))
 	r.Methods("PUT").Path("/v1/backupvolumes").Handler(f(schemas, s.fwd.Handler(s.fwd.HandleProxyRequestByNodeID, s.fwd.GetHTTPAddressByNodeID(NodeHasDefaultEngineImage(s.m)), s.BackupVolumeSyncAll)))
 	r.Methods("GET").Path("/v1/backupvolumes/{volName}").Handler(f(schemas, s.fwd.Handler(s.fwd.HandleProxyRequestByNodeID, s.fwd.GetHTTPAddressByNodeID(NodeHasDefaultEngineImage(s.m)), s.BackupVolumeGet)))
@@ -255,6 +259,10 @@ func NewRouter(s *Server) *mux.Router {
 	r.Path("/v1/ws/backupvolumes").Handler(f(schemas, backupVolumeStream))
 	r.Path("/v1/ws/{period}/backupvolumes").Handler(f(schemas, backupVolumeStream))
 
+	backupTargetStream := NewStreamHandlerFunc("backuptargets", s.wsc.NewWatcher("backupTarget"), s.backupTargetList)
+	r.Path("/v1/ws/backuptargets").Handler(f(schemas, backupTargetStream))
+	r.Path("/v1/ws/{period}/backuptargets").Handler(f(schemas, backupTargetStream))
+
 	// TODO:
 	// We haven't found a way to allow passing the volume name as a parameter to filter
 	// per-backup volume's backups change thru. WebSocket endpoint. Either by:

api/snapshot.go

@@ -195,7 +195,7 @@ func (s *Server) SnapshotBackup(w http.ResponseWriter, req *http.Request) (err e
 		labels[types.KubernetesStatusLabel] = string(kubeStatus)
 	}
 
-	if err := s.m.BackupSnapshot(bsutil.GenerateName("backup"), volName, input.Name, labels, input.BackupMode); err != nil {
+	if err := s.m.BackupSnapshot(bsutil.GenerateName("backup"), vol.Spec.BackupTargetName, volName, input.Name, labels, input.BackupMode); err != nil {
 		return err
 	}
 

api/volume.go

@@ -104,7 +104,7 @@ func (s *Server) responseWithVolume(rw http.ResponseWriter, req *http.Request, i
 	if err != nil {
 		return err
 	}
-	backups, err := s.m.ListBackupsForVolumeSorted(id)
+	backups, err := s.m.ListBackupsForVolumeSorted(v.Name)
 	if err != nil {
 		return err
 	}

app/recurring_job.go

@@ -677,7 +677,7 @@ func (job *Job) doRecurringBackup() (err error) {
 		}
 	}()
 
-	backupVolume, err := job.api.BackupVolume.ById(job.volumeName)
+	backupVolume, err := job.getBackupVolume(volume.BackupTargetName)
 	if err != nil {
 		return err
 	}
@@ -701,6 +701,26 @@ func (job *Job) doRecurringBackup() (err error) {
 	return nil
 }
 
+func (job *Job) getBackupVolume(backupTargetName string) (*longhornclient.BackupVolume, error) {
+	list, err := job.api.BackupVolume.List(&longhornclient.ListOpts{
+		Filters: map[string]interface{}{
+			types.LonghornLabelBackupTarget: backupTargetName,
+			types.LonghornLabelBackupVolume: job.volumeName,
+		}})
+	if err != nil {
+		return nil, err
+	}
+
+	if len(list.Data) >= 2 {
+		return nil, fmt.Errorf("found more than one backup volume %v of the backup target %v", job.volumeName, backupTargetName)
+	}
+	if len(list.Data) == 0 {
+		return nil, fmt.Errorf("failed to find backup volume %s of the backup target %s", job.volumeName, backupTargetName)
+	}
+
+	return &list.Data[0], nil
+}
+
 func (job *Job) doRecurringFilesystemTrim(volume *longhornclient.Volume) (err error) {
 	defer func() {
 		err = errors.Wrapf(err, "failed to complete filesystem-trim for %v", volume.Name)
@@ -754,7 +774,7 @@ func (job *Job) getLastBackup() (*longhornclient.Backup, error) {
 	if volume.LastBackup == "" {
 		return nil, nil
 	}
-	backupVolume, err := job.api.BackupVolume.ById(job.volumeName)
+	backupVolume, err := job.getBackupVolume(volume.BackupTargetName)
 	if err != nil {
 		return nil, err
 	}

client/generated_backup.go

@@ -7,6 +7,8 @@ const (
 type Backup struct {
 	Resource `yaml:"-"`
 
+	BackupTargetName string `json:"backupTargetName,omitempty" yaml:"backup_target_name,omitempty"`
+
 	CompressionMethod string `json:"compressionMethod,omitempty" yaml:"compression_method,omitempty"`
 
 	Created string `json:"created,omitempty" yaml:"created,omitempty"`