Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(backup): multiple backup stores support #2182

Open
wants to merge 30 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
81b406e
feat(backup): new field of Backup CRD
mantissahz Oct 22, 2024
acd50aa
feat(backup): new fields of BackupVolume CRD
mantissahz Oct 22, 2024
a07a302
feat(backup): new field of BackupTarget CRD
mantissahz Oct 22, 2024
20b0856
feat(backup): new filed of Volume CR
mantissahz Oct 22, 2024
f37caff
feat(backup): methods to get backup volume CRs
mantissahz Oct 30, 2024
5dfebd2
feat(backup): backup target mutation
mantissahz Oct 29, 2024
dab50a4
feat(backup): volume mutation and validation for mutiple backup targets
mantissahz Oct 29, 2024
44c2241
feat(backup): new fields of BackupBackingImage CRD
mantissahz Oct 28, 2024
c9ad360
feat(backup): modify the backup target controller
mantissahz Oct 29, 2024
32bfa63
feat(backup): move out backup target logic from setting controller
mantissahz Oct 29, 2024
92abea1
feat(backup): add backup target validator
mantissahz Oct 29, 2024
6a49632
feat(backup): modify setting and uninstall controllers
mantissahz Oct 22, 2024
1d35163
feat(backup): new filed of Volume CR
mantissahz Oct 22, 2024
abaa6d0
feat(backup): modify backup volume controller
mantissahz Oct 29, 2024
559c31e
feat(backup): modify backup controller
mantissahz Oct 30, 2024
5b53968
feat(backup): add backup target label when creating a backup
mantissahz Oct 22, 2024
87d4702
feat(backup): handle the restoration
mantissahz Oct 30, 2024
2770fcf
feat(backup): handle triggering the backup volume synchronization
mantissahz Oct 24, 2024
3863ecf
feat(backup): recurringjob for multiple backupstores
mantissahz Oct 28, 2024
a7e9e1c
feat(backup): add backup target APIs
mantissahz Oct 30, 2024
6036180
feat(backup): backup validation
mantissahz Oct 22, 2024
ffab803
feat(backup): modify volume controller unit tests for backup targets
mantissahz Oct 22, 2024
f5d6cc3
feat(backup): modify getting BackupVolume for csi backup
mantissahz Oct 30, 2024
f190b0c
feat(backup): modify controllers related to backup backing image
mantissahz Oct 24, 2024
789bf78
feat(backup): modify backing image data source controller
mantissahz Oct 29, 2024
4a9652c
feat(backup): add backup target name when mutating BackingImage for t…
mantissahz Oct 29, 2024
ef8e0b3
feat(backup): modify system backup controller
mantissahz Oct 22, 2024
1cf1bda
feat(backup): upgrade CRs for multiple backup store
mantissahz Oct 29, 2024
39595e1
feat(backup): add websocket for backup targets
mantissahz Sep 30, 2024
97553ea
feat(backup): update k8s/crds.yaml
mantissahz Oct 28, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
121 changes: 119 additions & 2 deletions api/backup.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,23 +3,129 @@ package api
import (
"fmt"
"net/http"
"strconv"
"time"

metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

"github.com/gorilla/mux"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"

"github.com/rancher/go-rancher/api"
"github.com/rancher/go-rancher/client"

longhorn "github.com/longhorn/longhorn-manager/k8s/pkg/apis/longhorn/v1beta2"
"github.com/longhorn/longhorn-manager/util"
)

const (
BackupTargetDefaultPollInterval = 300
)

func (s *Server) BackupTargetGet(w http.ResponseWriter, req *http.Request) error {
apiContext := api.GetApiContext(req)

backupTargetName := mux.Vars(req)["name"]
backupTarget, err := s.m.GetBackupTarget(backupTargetName)
if err != nil {
return errors.Wrapf(err, "failed to get backup target %v", backupTargetName)
}
apiContext.Write(toBackupTargetResource(backupTarget, apiContext))
return nil
}

func (s *Server) BackupTargetList(w http.ResponseWriter, req *http.Request) error {
apiContext := api.GetApiContext(req)

backupTargets, err := s.m.ListBackupTargetsSorted()
bts, err := s.backupTargetList(apiContext)
if err != nil {
return errors.Wrap(err, "failed to list backup targets")
}
apiContext.Write(toBackupTargetCollection(backupTargets, apiContext))

apiContext.Write(bts)
return nil
}

func (s *Server) backupTargetList(apiContext *api.ApiContext) (*client.GenericCollection, error) {
bts, err := s.m.ListBackupTargetsSorted()
if err != nil {
return nil, errors.Wrap(err, "failed to list backup target")
}
return toBackupTargetCollection(bts, apiContext), nil
}

func (s *Server) BackupTargetCreate(rw http.ResponseWriter, req *http.Request) error {
var input BackupTarget
apiContext := api.GetApiContext(req)

if err := apiContext.Read(&input); err != nil {
return err
}

backupTargetSpec, err := newBackupTarget(input)
if err != nil {
return err
}

obj, err := s.m.CreateBackupTarget(input.Name, backupTargetSpec)
if err != nil {
return errors.Wrapf(err, "failed to create backup target %v", input.Name)
}
apiContext.Write(toBackupTargetResource(obj, apiContext))
return nil
}

func newBackupTarget(input BackupTarget) (*longhorn.BackupTargetSpec, error) {
pollInterval, err := strconv.ParseInt(input.PollInterval, 10, 64)
if err != nil {
return nil, errors.Wrapf(err, "invalid backup target polling interval '%s'", input.PollInterval)
}

return &longhorn.BackupTargetSpec{
BackupTargetURL: input.BackupTargetURL,
CredentialSecret: input.CredentialSecret,
PollInterval: metav1.Duration{Duration: time.Duration(pollInterval) * time.Second},
ReadOnly: input.ReadOnly}, nil
}
mantissahz marked this conversation as resolved.
Show resolved Hide resolved
mantissahz marked this conversation as resolved.
Show resolved Hide resolved

func (s *Server) BackupTargetUpdate(rw http.ResponseWriter, req *http.Request) error {
var input BackupTarget

apiContext := api.GetApiContext(req)
if err := apiContext.Read(&input); err != nil {
return err
}

name := mux.Vars(req)["name"]

backupTargetSpec, err := newBackupTarget(input)
if err != nil {
return err
}
mantissahz marked this conversation as resolved.
Show resolved Hide resolved

obj, err := util.RetryOnConflictCause(func() (interface{}, error) {
return s.m.UpdateBackupTarget(name, backupTargetSpec)
})
if err != nil {
return errors.Wrapf(err, "failed to update backup target %v", name)
}

backupTarget, ok := obj.(*longhorn.BackupTarget)
if !ok {
return fmt.Errorf("failed to convert %v to backup target", name)
}
mantissahz marked this conversation as resolved.
Show resolved Hide resolved

apiContext.Write(toBackupTargetResource(backupTarget, apiContext))
return nil
}

func (s *Server) BackupTargetDelete(rw http.ResponseWriter, req *http.Request) error {
backupTargetName := mux.Vars(req)["name"]
if err := s.m.DeleteBackupTarget(backupTargetName); err != nil {
return errors.Wrapf(err, "failed to delete backup target %v", backupTargetName)
}

mantissahz marked this conversation as resolved.
Show resolved Hide resolved
return nil
}

Expand Down Expand Up @@ -174,6 +280,17 @@ func (s *Server) BackupVolumeDelete(w http.ResponseWriter, req *http.Request) er
func (s *Server) BackupList(w http.ResponseWriter, req *http.Request) error {
apiContext := api.GetApiContext(req)

bs, err := s.m.ListAllBackupsSorted()
if err != nil {
return errors.Wrapf(err, "failed to list all backups")
}
apiContext.Write(toBackupCollection(bs))
return nil
}

func (s *Server) BackupListByVolumeName(w http.ResponseWriter, req *http.Request) error {
apiContext := api.GetApiContext(req)

volName := mux.Vars(req)["volName"]

bs, err := s.m.ListBackupsForVolumeSorted(volName)
Expand Down
13 changes: 10 additions & 3 deletions api/backupbackingimage.go
Original file line number Diff line number Diff line change
Expand Up @@ -63,9 +63,16 @@ func (s *Server) BackupBackingImageRestore(w http.ResponseWriter, req *http.Requ
}

func (s *Server) BackupBackingImageCreate(w http.ResponseWriter, req *http.Request) error {
backupBackingImageName := mux.Vars(req)["name"]
if err := s.m.CreateBackupBackingImage(backupBackingImageName); err != nil {
return errors.Wrapf(err, "failed to create backup backing image '%s'", backupBackingImageName)
var input BackupBackingImage

apiContext := api.GetApiContext(req)
if err := apiContext.Read(&input); err != nil {
return err
}

backingImageName := mux.Vars(req)["name"]
if err := s.m.CreateBackupBackingImage(input.Name, backingImageName, input.BackupTargetName); err != nil {
return errors.Wrapf(err, "failed to create backup backing image '%s'", input.Name)
mantissahz marked this conversation as resolved.
Show resolved Hide resolved
}
return nil
}
50 changes: 47 additions & 3 deletions api/model.go
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ type Volume struct {
SnapshotMaxCount int `json:"snapshotMaxCount"`
SnapshotMaxSize string `json:"snapshotMaxSize"`
FreezeFilesystemForSnapshot longhorn.FreezeFilesystemForSnapshot `json:"freezeFilesystemForSnapshot"`
BackupTargetName string `json:"backupTargetName"`

DiskSelector []string `json:"diskSelector"`
NodeSelector []string `json:"nodeSelector"`
Expand Down Expand Up @@ -139,6 +140,8 @@ type BackupVolume struct {
BackingImageName string `json:"backingImageName"`
BackingImageChecksum string `json:"backingImageChecksum"`
StorageClassName string `json:"storageClassName"`
BackupTargetName string `json:"backupTargetName"`
VolumeName string `json:"volumeName"`
}

// SyncBackupResource is used for the Backup*Sync* actions
Expand Down Expand Up @@ -171,6 +174,7 @@ type Backup struct {
CompressionMethod string `json:"compressionMethod"`
NewlyUploadedDataSize string `json:"newlyUploadDataSize"`
ReUploadedDataSize string `json:"reUploadedDataSize"`
BackupTargetName string `json:"backupTargetName"`
}

type BackupBackingImage struct {
Expand All @@ -187,6 +191,8 @@ type BackupBackingImage struct {
CompressionMethod string `json:"compressionMethod"`
Secret string `json:"secret"`
SecretNamespace string `json:"secretNamespace"`
BackingImageName string `json:"backingImageName"`
BackupTargetName string `json:"backupTargetName"`
}

type Setting struct {
Expand Down Expand Up @@ -871,14 +877,44 @@ func kubernetesStatusSchema(status *client.Schema) {
}

func backupTargetSchema(backupTarget *client.Schema) {
backupTarget.CollectionMethods = []string{"GET"}
backupTarget.ResourceMethods = []string{"GET", "PUT"}
backupTarget.CollectionMethods = []string{"GET", "POST"}
backupTarget.ResourceMethods = []string{"GET", "PUT", "DELETE"}

backupTargetName := backupTarget.ResourceFields["name"]
backupTargetName.Required = true
backupTargetName.Unique = true
backupTargetName.Create = true
backupTarget.ResourceFields["name"] = backupTargetName

backupTargetURL := backupTarget.ResourceFields["backupTargetURL"]
backupTargetURL.Create = true
backupTargetURL.Default = ""
backupTarget.ResourceFields["backupTargetURL"] = backupTargetURL

credentialSecret := backupTarget.ResourceFields["credentialSecret"]
credentialSecret.Create = true
credentialSecret.Default = ""
backupTarget.ResourceFields["credentialSecret"] = credentialSecret

backupTargetPollInterval := backupTarget.ResourceFields["pollInterval"]
backupTargetPollInterval.Create = true
backupTargetPollInterval.Default = "300"
backupTarget.ResourceFields["pollInterval"] = backupTargetPollInterval

backupTargetReadOnly := backupTarget.ResourceFields["readOnly"]
backupTargetReadOnly.Create = true
backupTargetReadOnly.Default = false
backupTarget.ResourceFields["readOnly"] = backupTargetReadOnly

backupTarget.ResourceActions = map[string]client.Action{
"backupTargetSync": {
Input: "syncBackupResource",
Output: "backupTargetListOutput",
},
"backupTargetUpdate": {
Input: "BackupTarget",
Output: "backupTargetListOutput",
},
mantissahz marked this conversation as resolved.
Show resolved Hide resolved
}
}

Expand Down Expand Up @@ -1562,6 +1598,7 @@ func toVolumeResource(v *longhorn.Volume, ves []*longhorn.Engine, vrs []*longhor
NodeSelector: v.Spec.NodeSelector,
RestoreVolumeRecurringJob: v.Spec.RestoreVolumeRecurringJob,
FreezeFilesystemForSnapshot: v.Spec.FreezeFilesystemForSnapshot,
BackupTargetName: v.Spec.BackupTargetName,

State: v.Status.State,
Robustness: v.Status.Robustness,
Expand Down Expand Up @@ -1797,11 +1834,13 @@ func toBackupTargetResource(bt *longhorn.BackupTarget, apiContext *api.ApiContex
CredentialSecret: bt.Spec.CredentialSecret,
PollInterval: bt.Spec.PollInterval.Duration.String(),
Available: bt.Status.Available,
ReadOnly: bt.Spec.ReadOnly,
Message: types.GetCondition(bt.Status.Conditions, longhorn.BackupTargetConditionTypeUnavailable).Message,
},
}
res.Actions = map[string]string{
"backupTargetSync": apiContext.UrlBuilder.ActionLink(res.Resource, "backupTargetSync"),
"backupTargetSync": apiContext.UrlBuilder.ActionLink(res.Resource, "backupTargetSync"),
"backupTargetUpdate": apiContext.UrlBuilder.ActionLink(res.Resource, "backupTargetUpdate"),
}

return res
Expand All @@ -1828,6 +1867,8 @@ func toBackupVolumeResource(bv *longhorn.BackupVolume, apiContext *api.ApiContex
BackingImageName: bv.Status.BackingImageName,
BackingImageChecksum: bv.Status.BackingImageChecksum,
StorageClassName: bv.Status.StorageClassName,
BackupTargetName: bv.Spec.BackupTargetName,
VolumeName: bv.Spec.VolumeName,
}
b.Actions = map[string]string{
"backupList": apiContext.UrlBuilder.ActionLink(b.Resource, "backupList"),
Expand Down Expand Up @@ -1877,6 +1918,8 @@ func toBackupBackingImageResource(bbi *longhorn.BackupBackingImage, apiContext *
CompressionMethod: string(bbi.Status.CompressionMethod),
Secret: bbi.Status.Secret,
SecretNamespace: bbi.Status.SecretNamespace,
BackingImageName: bbi.Spec.BackingImage,
BackupTargetName: bbi.Spec.BackupTargetName,
}

backupBackingImage.Actions = map[string]string{
Expand Down Expand Up @@ -1932,6 +1975,7 @@ func toBackupResource(b *longhorn.Backup) *Backup {
CompressionMethod: string(b.Status.CompressionMethod),
NewlyUploadedDataSize: b.Status.NewlyUploadedDataSize,
ReUploadedDataSize: b.Status.ReUploadedDataSize,
BackupTargetName: b.Spec.BackupTargetName,
}
// Set the volume name from backup CR's label if it's empty.
// This field is empty probably because the backup state is not Ready
Expand Down
10 changes: 9 additions & 1 deletion api/router.go
Original file line number Diff line number Diff line change
Expand Up @@ -122,11 +122,15 @@ func NewRouter(s *Server) *mux.Router {
r.Methods("GET").Path("/v1/backuptargets").Handler(f(schemas, s.BackupTargetList))
r.Methods("PUT").Path("/v1/backuptargets").Handler(f(schemas, s.BackupTargetSyncAll))
backupTargetActions := map[string]func(http.ResponseWriter, *http.Request) error{
"backupTargetSync": s.fwd.Handler(s.fwd.HandleProxyRequestByNodeID, s.fwd.GetHTTPAddressByNodeID(NodeHasDefaultEngineImage(s.m)), s.BackupTargetSync),
"backupTargetSync": s.fwd.Handler(s.fwd.HandleProxyRequestByNodeID, s.fwd.GetHTTPAddressByNodeID(NodeHasDefaultEngineImage(s.m)), s.BackupTargetSync),
"backupTargetUpdate": s.fwd.Handler(s.fwd.HandleProxyRequestByNodeID, s.fwd.GetHTTPAddressByNodeID(NodeHasDefaultEngineImage(s.m)), s.BackupTargetUpdate),
}
for name, action := range backupTargetActions {
r.Methods("POST").Path("/v1/backuptargets/{backupTargetName}").Queries("action", name).Handler(f(schemas, action))
}
r.Methods("GET").Path("/v1/backuptargets/{name}").Handler(f(schemas, s.BackupTargetGet))
r.Methods("POST").Path("/v1/backuptargets").Handler(f(schemas, s.BackupTargetCreate))
r.Methods("DELETE").Path("/v1/backuptargets/{name}").Handler(f(schemas, s.BackupTargetDelete))
mantissahz marked this conversation as resolved.
Show resolved Hide resolved
r.Methods("GET").Path("/v1/backupvolumes").Handler(f(schemas, s.fwd.Handler(s.fwd.HandleProxyRequestByNodeID, s.fwd.GetHTTPAddressByNodeID(NodeHasDefaultEngineImage(s.m)), s.BackupVolumeList)))
r.Methods("PUT").Path("/v1/backupvolumes").Handler(f(schemas, s.fwd.Handler(s.fwd.HandleProxyRequestByNodeID, s.fwd.GetHTTPAddressByNodeID(NodeHasDefaultEngineImage(s.m)), s.BackupVolumeSyncAll)))
r.Methods("GET").Path("/v1/backupvolumes/{volName}").Handler(f(schemas, s.fwd.Handler(s.fwd.HandleProxyRequestByNodeID, s.fwd.GetHTTPAddressByNodeID(NodeHasDefaultEngineImage(s.m)), s.BackupVolumeGet)))
Expand Down Expand Up @@ -255,6 +259,10 @@ func NewRouter(s *Server) *mux.Router {
r.Path("/v1/ws/backupvolumes").Handler(f(schemas, backupVolumeStream))
r.Path("/v1/ws/{period}/backupvolumes").Handler(f(schemas, backupVolumeStream))

backupTargetStream := NewStreamHandlerFunc("backuptargets", s.wsc.NewWatcher("backupTarget"), s.backupTargetList)
r.Path("/v1/ws/backuptargets").Handler(f(schemas, backupTargetStream))
r.Path("/v1/ws/{period}/backuptargets").Handler(f(schemas, backupTargetStream))
mantissahz marked this conversation as resolved.
Show resolved Hide resolved

mantissahz marked this conversation as resolved.
Show resolved Hide resolved
// TODO:
// We haven't found a way to allow passing the volume name as a parameter to filter
// per-backup volume's backups change thru. WebSocket endpoint. Either by:
Expand Down
2 changes: 1 addition & 1 deletion api/snapshot.go
Original file line number Diff line number Diff line change
Expand Up @@ -195,7 +195,7 @@ func (s *Server) SnapshotBackup(w http.ResponseWriter, req *http.Request) (err e
labels[types.KubernetesStatusLabel] = string(kubeStatus)
}

if err := s.m.BackupSnapshot(bsutil.GenerateName("backup"), volName, input.Name, labels, input.BackupMode); err != nil {
if err := s.m.BackupSnapshot(bsutil.GenerateName("backup"), vol.Spec.BackupTargetName, volName, input.Name, labels, input.BackupMode); err != nil {
return err
}

Expand Down
2 changes: 1 addition & 1 deletion api/volume.go
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ func (s *Server) responseWithVolume(rw http.ResponseWriter, req *http.Request, i
if err != nil {
return err
}
backups, err := s.m.ListBackupsForVolumeSorted(id)
backups, err := s.m.ListBackupsForVolumeSorted(v.Name)
if err != nil {
return err
}
Expand Down
24 changes: 22 additions & 2 deletions app/recurring_job.go
Original file line number Diff line number Diff line change
Expand Up @@ -677,7 +677,7 @@ func (job *Job) doRecurringBackup() (err error) {
}
}()

backupVolume, err := job.api.BackupVolume.ById(job.volumeName)
backupVolume, err := job.getBackupVolume(volume.BackupTargetName)
if err != nil {
return err
}
Expand All @@ -701,6 +701,26 @@ func (job *Job) doRecurringBackup() (err error) {
return nil
}

func (job *Job) getBackupVolume(backupTargetName string) (*longhornclient.BackupVolume, error) {
list, err := job.api.BackupVolume.List(&longhornclient.ListOpts{
Filters: map[string]interface{}{
types.LonghornLabelBackupTarget: backupTargetName,
types.LonghornLabelBackupVolume: job.volumeName,
}})
if err != nil {
return nil, err
}

if len(list.Data) >= 2 {
return nil, fmt.Errorf("found more than one backup volume %v of the backup target %v", job.volumeName, backupTargetName)
}
if len(list.Data) == 0 {
return nil, fmt.Errorf("failed to find backup volume %s of the backup target %s", job.volumeName, backupTargetName)
}

return &list.Data[0], nil
}
mantissahz marked this conversation as resolved.
Show resolved Hide resolved

func (job *Job) doRecurringFilesystemTrim(volume *longhornclient.Volume) (err error) {
defer func() {
err = errors.Wrapf(err, "failed to complete filesystem-trim for %v", volume.Name)
Expand Down Expand Up @@ -754,7 +774,7 @@ func (job *Job) getLastBackup() (*longhornclient.Backup, error) {
if volume.LastBackup == "" {
return nil, nil
}
backupVolume, err := job.api.BackupVolume.ById(job.volumeName)
backupVolume, err := job.getBackupVolume(volume.BackupTargetName)
if err != nil {
return nil, err
}
Expand Down
2 changes: 2 additions & 0 deletions client/generated_backup.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ const (
type Backup struct {
Resource `yaml:"-"`

BackupTargetName string `json:"backupTargetName,omitempty" yaml:"backup_target_name,omitempty"`

CompressionMethod string `json:"compressionMethod,omitempty" yaml:"compression_method,omitempty"`

Created string `json:"created,omitempty" yaml:"created,omitempty"`
Expand Down
Loading