feat: add snapshot max count and size

api/model.go

@@ -330,6 +330,14 @@ type UpdateReplicaDiskSoftAntiAffinityInput struct {
 	ReplicaDiskSoftAntiAffinity string `json:"replicaDiskSoftAntiAffinity"`
 }
 
+type UpdateSnapshotMaxCount struct {
+	SnapshotMaxCount int `json:"snapshotMaxCount"`
+}
+
+type UpdateSnapshotMaxSize struct {
+	SnapshotMaxSize int64 `json:"snapshotMaxSize"`
+}
+
 type PVCreateInput struct {
 	PVName string `json:"pvName"`
 	FSType string `json:"fsType"`

api/router.go

@@ -75,6 +75,8 @@ func NewRouter(s *Server) *mux.Router {
 		"updateDataLocality":                s.VolumeUpdateDataLocality,
 		"updateAccessMode":                  s.VolumeUpdateAccessMode,
 		"updateUnmapMarkSnapChainRemoved":   s.VolumeUpdateUnmapMarkSnapChainRemoved,
+		"updateSnapshotMaxCount":            s.VolumeUpdateSnapshotMaxCount,
+		"updateSnapshotMaxSize":             s.VolumeUpdateSnapshotMaxSize,
 		"updateReplicaSoftAntiAffinity":     s.VolumeUpdateReplicaSoftAntiAffinity,
 		"updateReplicaZoneSoftAntiAffinity": s.VolumeUpdateReplicaZoneSoftAntiAffinity,
 		"updateReplicaDiskSoftAntiAffinity": s.VolumeUpdateReplicaDiskSoftAntiAffinity,

api/volume.go

@@ -779,3 +779,47 @@ func (s *Server) EngineUpgrade(rw http.ResponseWriter, req *http.Request) error
 
 	return s.responseWithVolume(rw, req, id, v)
 }
+
+func (s *Server) VolumeUpdateSnapshotMaxCount(rw http.ResponseWriter, req *http.Request) error {
+	var input UpdateSnapshotMaxCount
+	id := mux.Vars(req)["name"]
+
+	apiContext := api.GetApiContext(req)
+	if err := apiContext.Read(&input); err != nil {
+		return errors.Wrap(err, "failed to read SnapshotMaxCount input")
+	}
+
+	obj, err := util.RetryOnConflictCause(func() (interface{}, error) {
+		return s.m.UpdateSnapshotMaxCount(id, input.SnapshotMaxCount)
+	})
+	if err != nil {
+		return err
+	}
+	v, ok := obj.(*longhorn.Volume)
+	if !ok {
+		return fmt.Errorf("failed to convert to volume %v object", id)
+	}
+	return s.responseWithVolume(rw, req, "", v)
+}
+
+func (s *Server) VolumeUpdateSnapshotMaxSize(rw http.ResponseWriter, req *http.Request) error {
+	var input UpdateSnapshotMaxSize
+	id := mux.Vars(req)["name"]
+
+	apiContext := api.GetApiContext(req)
+	if err := apiContext.Read(&input); err != nil {
+		return errors.Wrap(err, "failed to read SnapshotMaxSize input")
+	}
+
+	obj, err := util.RetryOnConflictCause(func() (interface{}, error) {
+		return s.m.UpdateSnapshotMaxSize(id, input.SnapshotMaxSize)
+	})
+	if err != nil {
+		return err
+	}
+	v, ok := obj.(*longhorn.Volume)
+	if !ok {
+		return fmt.Errorf("failed to convert to volume %v object", id)
+	}
+	return s.responseWithVolume(rw, req, "", v)
+}

controller/engine_controller.go

@@ -986,6 +986,24 @@ func (m *EngineMonitor) refresh(engine *longhorn.Engine) error {
 						volumeInfo.UnmapMarkSnapChainRemoved, engine.Spec.UnmapMarkSnapChainRemovedEnabled)
 				}
 			}
+
+			engine.Status.SnapshotMaxCount = volumeInfo.SnapshotMaxCount
+			if engine.Spec.SnapshotMaxCount != volumeInfo.SnapshotMaxCount {
+				logrus.Infof("Correcting flag SnapshotMaxCount from %d to %d", volumeInfo.SnapshotMaxCount, engine.Spec.SnapshotMaxCount)
+				if err := engineClientProxy.VolumeSnapshotMaxCountSet(engine); err != nil {
+					return errors.Wrapf(err, "failed to correct flag SnapshotMaxCount from %d to %d",
+						volumeInfo.SnapshotMaxCount, engine.Spec.SnapshotMaxCount)
+				}
+			}
+
+			engine.Status.SnapshotMaxSize = volumeInfo.SnapshotMaxSize
+			if engine.Spec.SnapshotMaxSize != volumeInfo.SnapshotMaxSize {
+				logrus.Infof("Correcting flag SnapshotMaxSize from %d to %d", volumeInfo.SnapshotMaxSize, engine.Spec.SnapshotMaxSize)
+				if err := engineClientProxy.VolumeSnapshotMaxSizeSet(engine); err != nil {
+					return errors.Wrapf(err, "failed to correct flag SnapshotMaxSize from %d to %d",
+						volumeInfo.SnapshotMaxSize, engine.Spec.SnapshotMaxSize)
+				}
+			}
 		}
 	} else {
 		// For incompatible running engine, the current size is always `engine.Spec.VolumeSize`.

controller/volume_controller.go

@@ -459,6 +459,10 @@ func (c *VolumeController) syncVolume(key string) (err error) {
 		return err
 	}
 
+	if err := c.syncVolumeSnapshotSetting(volume, engines, replicas); err != nil {
+		return err
+	}
+
 	if err := c.updateRecurringJobs(volume); err != nil {
 		return err
 	}
@@ -1188,6 +1192,23 @@ func (c *VolumeController) syncVolumeUnmapMarkSnapChainRemovedSetting(v *longhor
 	return nil
 }
 
+func (c *VolumeController) syncVolumeSnapshotSetting(v *longhorn.Volume, es map[string]*longhorn.Engine, rs map[string]*longhorn.Replica) error {
+	if es == nil && rs == nil {
+		return nil
+	}
+
+	for _, e := range es {
+		e.Spec.SnapshotMaxCount = v.Spec.SnapshotMaxCount
+		e.Spec.SnapshotMaxSize = v.Spec.SnapshotMaxSize
+	}
+	for _, r := range rs {
+		r.Spec.SnapshotMaxCount = v.Spec.SnapshotMaxCount
+		r.Spec.SnapshotMaxSize = v.Spec.SnapshotMaxSize
+	}
+
+	return nil
+}
+
 // ReconcileVolumeState handles the attaching and detaching of volume
 func (c *VolumeController) ReconcileVolumeState(v *longhorn.Volume, es map[string]*longhorn.Engine, rs map[string]*longhorn.Replica) (err error) {
 	defer func() {
@@ -3191,6 +3212,8 @@ func (c *VolumeController) createEngine(v *longhorn.Volume, currentEngineName st
 			ReplicaAddressMap:         map[string]string{},
 			UpgradedReplicaAddressMap: map[string]string{},
 			RevisionCounterDisabled:   v.Spec.RevisionCounterDisabled,
+			SnapshotMaxCount:          v.Spec.SnapshotMaxCount,
+			SnapshotMaxSize:           v.Spec.SnapshotMaxSize,
 		},
 	}
 
@@ -3242,6 +3265,8 @@ func (c *VolumeController) createReplica(v *longhorn.Volume, e *longhorn.Engine,
 			HardNodeAffinity:                 hardNodeAffinity,
 			RevisionCounterDisabled:          v.Spec.RevisionCounterDisabled,
 			UnmapMarkDiskChainRemovedEnabled: e.Spec.UnmapMarkSnapChainRemovedEnabled,
+			SnapshotMaxCount:                 v.Spec.SnapshotMaxCount,
+			SnapshotMaxSize:                  v.Spec.SnapshotMaxSize,
 		},
 	}
 	if isRebuildingReplica {

datastore/longhorn.go

@@ -411,6 +411,14 @@ func (s *DataStore) ValidateSetting(name, value string) (err error) {
 		if value == "true" && autoCleanupValue {
 			return errors.Errorf("cannot set %v setting to true when %v setting is true", name, types.SettingNameAutoCleanupSystemGeneratedSnapshot)
 		}
+	case types.SettingNameSnapshotMaxCount:
+		v, err := strconv.Atoi(value)
+		if err != nil {
+			return err
+		}
+		if v < 2 || v > 250 {
+			return fmt.Errorf("%s should be between 2 and 250", name)
+		}
 	}
 	return nil
 }

engineapi/engine.go

@@ -290,6 +290,26 @@ func (e *EngineBinary) VolumeUnmapMarkSnapChainRemovedSet(engine *longhorn.Engin
 	return nil
 }
 
+// VolumeSnapshotMaxCountSet calls engine binary
+// TODO: Deprecated, replaced by gRPC proxy
+func (e *EngineBinary) VolumeSnapshotMaxCountSet(engine *longhorn.Engine) error {
+	cmdline := []string{"snapshot-max-count", strconv.Itoa(engine.Spec.SnapshotMaxCount)}
+	if _, err := e.ExecuteEngineBinary(cmdline...); err != nil {
+		return errors.Wrapf(err, "error setting volume flag SnapshotMaxCount to %d", engine.Spec.SnapshotMaxCount)
+	}
+	return nil
+}
+
+// VolumeSnapshotMaxSizeSet calls engine binary
+// TODO: Deprecated, replaced by gRPC proxy
+func (e *EngineBinary) VolumeSnapshotMaxSizeSet(engine *longhorn.Engine) error {
+	cmdline := []string{"snapshot-max-size", strconv.FormatInt(engine.Spec.SnapshotMaxSize, 10)}
+	if _, err := e.ExecuteEngineBinary(cmdline...); err != nil {
+		return errors.Wrapf(err, "error setting volume flag SnapshotMaxSize to %d", engine.Spec.SnapshotMaxSize)
+	}
+	return nil
+}
+
 // ReplicaRebuildVerify calls engine binary
 // TODO: Deprecated, replaced by gRPC proxy
 func (e *EngineBinary) ReplicaRebuildVerify(engine *longhorn.Engine, replicaName, url string) error {

engineapi/enginesim.go

@@ -245,6 +245,14 @@ func (e *EngineSimulator) VolumeUnmapMarkSnapChainRemovedSet(*longhorn.Engine) e
 	return fmt.Errorf(ErrNotImplement)
 }
 
+func (e *EngineSimulator) VolumeSnapshotMaxCountSet(*longhorn.Engine) error {
+	return fmt.Errorf(ErrNotImplement)
+}
+
+func (e *EngineSimulator) VolumeSnapshotMaxSizeSet(*longhorn.Engine) error {
+	return fmt.Errorf(ErrNotImplement)
+}
+
 func (e *EngineSimulator) ReplicaRebuildVerify(engine *longhorn.Engine, replicaName, url string) error {
 	return fmt.Errorf(ErrNotImplement)
 }

engineapi/instance_manager.go

@@ -313,6 +313,8 @@ func getBinaryAndArgsForEngineProcessCreation(e *longhorn.Engine,
 
 	if engineCLIAPIVersion >= 9 {
 		args = append([]string{"--engine-instance-name", e.Name}, args...)
+		args = append(args, "--snapshot-max-count", strconv.Itoa(e.Spec.SnapshotMaxCount))
+		args = append(args, "--snapshot-max-size", strconv.FormatInt(e.Spec.SnapshotMaxSize, 10))
 	}
 
 	for _, addr := range e.Status.CurrentReplicaAddressMap {
@@ -354,6 +356,8 @@ func getBinaryAndArgsForReplicaProcessCreation(r *longhorn.Replica,
 	if engineCLIAPIVersion >= 9 {
 		args = append(args, "--replica-instance-name", r.Name)
 		args = append([]string{"--volume-name", r.Spec.VolumeName}, args...)
+		args = append(args, "--snapshot-max-count", strconv.Itoa(r.Spec.SnapshotMaxCount))
+		args = append(args, "--snapshot-max-size", strconv.FormatInt(r.Spec.SnapshotMaxSize, 10))
 	}
 
 	// 3 ports are already used by replica server, data server and syncagent server

engineapi/proxy_volume.go

@@ -43,3 +43,13 @@ func (p *Proxy) VolumeUnmapMarkSnapChainRemovedSet(e *longhorn.Engine) error {
 	return p.grpcClient.VolumeUnmapMarkSnapChainRemovedSet(string(e.Spec.DataEngine), e.Name, e.Spec.VolumeName,
 		p.DirectToURL(e), e.Spec.UnmapMarkSnapChainRemovedEnabled)
 }
+
+func (p *Proxy) VolumeSnapshotMaxCountSet(e *longhorn.Engine) error {
+	return p.grpcClient.VolumeSnapshotMaxCountSet(string(e.Spec.BackendStoreDriver), e.Name, e.Spec.VolumeName,
+		p.DirectToURL(e), e.Spec.SnapshotMaxCount)
+}
+
+func (p *Proxy) VolumeSnapshotMaxSizeSet(e *longhorn.Engine) error {
+	return p.grpcClient.VolumeSnapshotMaxSizeSet(string(e.Spec.BackendStoreDriver), e.Name, e.Spec.VolumeName,
+		p.DirectToURL(e), e.Spec.SnapshotMaxSize)
+}

engineapi/types.go

@@ -75,6 +75,8 @@ type EngineClient interface {
 	VolumeFrontendStart(*longhorn.Engine) error
 	VolumeFrontendShutdown(*longhorn.Engine) error
 	VolumeUnmapMarkSnapChainRemovedSet(engine *longhorn.Engine) error
+	VolumeSnapshotMaxCountSet(engine *longhorn.Engine) error
+	VolumeSnapshotMaxSizeSet(engine *longhorn.Engine) error
 
 	ReplicaList(*longhorn.Engine) (map[string]*Replica, error)
 	ReplicaAdd(engine *longhorn.Engine, replicaName, url string, isRestoreVolume, fastSync bool, replicaFileSyncHTTPClientTimeout int64) error
@@ -128,6 +130,8 @@ type Volume struct {
 	LastExpansionError        string `json:"lastExpansionError"`
 	LastExpansionFailedAt     string `json:"lastExpansionFailedAt"`
 	UnmapMarkSnapChainRemoved bool   `json:"unmapMarkSnapChainRemoved"`
+	SnapshotMaxCount          int    `json:"snapshotMaxCount"`
+	SnapshotMaxSize           int64  `json:"SnapshotMaxSize"`
 }
 
 type BackupTarget struct {

go.mod

@@ -48,8 +48,8 @@ require (
 	github.com/longhorn/go-common-libs v0.0.0-20240103081802-3993c5908447
 	github.com/longhorn/go-iscsi-helper v0.0.0-20231220091133-38e296e68ad2
 	github.com/longhorn/go-spdk-helper v0.0.0-20231225123608-84cedbc12ba3
-	github.com/longhorn/longhorn-engine v1.6.0-dev-20231217.0.20231229031622-5efed2b8a02f
-	github.com/longhorn/longhorn-instance-manager v0.0.0-20240102035444-cbade98f23e1
+	github.com/longhorn/longhorn-engine v1.6.0-dev-20231217.0.20240103105254-12e92e847462
+	github.com/longhorn/longhorn-instance-manager v0.0.0-20240103134309-e09a1a821a26
 	github.com/longhorn/longhorn-share-manager v1.6.0-dev-20231217.0.20231226052309-99d57c1695ea
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.16.0

go.sum

@@ -1042,10 +1042,10 @@ github.com/longhorn/go-iscsi-helper v0.0.0-20231220091133-38e296e68ad2 h1:gUNsw0
 github.com/longhorn/go-iscsi-helper v0.0.0-20231220091133-38e296e68ad2/go.mod h1:ZKPflEa33TpR3/Q8mrwZm+j2OW1v/lPMudEQ+9+fEj0=
 github.com/longhorn/go-spdk-helper v0.0.0-20231225123608-84cedbc12ba3 h1:RuDfgGeFxp+/EYeiTU4R8pR5mu9IEnro72w5ysRlLi8=
 github.com/longhorn/go-spdk-helper v0.0.0-20231225123608-84cedbc12ba3/go.mod h1:4KsWeze+GVhcjtIE39ho0qp0SisqYx03Ie9Qj0Bzuow=
-github.com/longhorn/longhorn-engine v1.6.0-dev-20231217.0.20231229031622-5efed2b8a02f h1:WvjOdqhjZ2HXr7FN0GWSEo6a26+1EEuIsg3/pw5LrMQ=
-github.com/longhorn/longhorn-engine v1.6.0-dev-20231217.0.20231229031622-5efed2b8a02f/go.mod h1:c2xk2KtroFqwewJ5TNuvCgCDRnWU4YdWo+p60TmptCc=
-github.com/longhorn/longhorn-instance-manager v0.0.0-20240102035444-cbade98f23e1 h1:9EFqKctA1jOQzyRyHnwNzqaHK1DjVkxNFWzyUg7cwns=
-github.com/longhorn/longhorn-instance-manager v0.0.0-20240102035444-cbade98f23e1/go.mod h1:UrlSbl7LAIrCMY7B8NHpyDnXamWu9p5jU5luYSXD6B4=
+github.com/longhorn/longhorn-engine v1.6.0-dev-20231217.0.20240103105254-12e92e847462 h1:5zHxDud6RazYlUqOCDI5n97bZgyxGsbTwqfTaMM23r8=
+github.com/longhorn/longhorn-engine v1.6.0-dev-20231217.0.20240103105254-12e92e847462/go.mod h1:EJs9hAdSRGS5CeC8ZYtEpUIOZmKu+3Tear8/oKO/oHw=
+github.com/longhorn/longhorn-instance-manager v0.0.0-20240103134309-e09a1a821a26 h1:xSWiam6TZcMT5eMnL9oUdC68Ob8xoLcCsQLWbgzpP08=
+github.com/longhorn/longhorn-instance-manager v0.0.0-20240103134309-e09a1a821a26/go.mod h1:dzlUIt67XEh3q8RWP01ePhyY14PWhrdTEuqSf3cwaik=
 github.com/longhorn/longhorn-share-manager v1.6.0-dev-20231217.0.20231226052309-99d57c1695ea h1:bYNEyl0yAh76fjzTQEo7+u2G80XaVi8aXfMlwyPiZmI=
 github.com/longhorn/longhorn-share-manager v1.6.0-dev-20231217.0.20231226052309-99d57c1695ea/go.mod h1:sDo6d4FV5nALnXkZHRY31H6R6KTK0LNo0kMVULlhYiE=
 github.com/longhorn/longhorn-spdk-engine v0.0.0-20231225122448-779b72c5fcf8 h1:xCCeskYtBhpybefILyIHnVLxSbKmmSIuDu0KvFljwqE=

k8s/crds.yaml

@@ -1358,6 +1358,11 @@ spec:
                 type: boolean
               salvageRequested:
                 type: boolean
+              snapshotMaxCount:
+                type: integer
+              snapshotMaxSize:
+                format: int64
+                type: string
               unmapMarkSnapChainRemovedEnabled:
                 type: boolean
               upgradedReplicaAddressMap:
@@ -1524,6 +1529,11 @@ spec:
                 type: object
               salvageExecuted:
                 type: boolean
+              snapshotMaxCount:
+                type: integer
+              snapshotMaxSize:
+                format: int64
+                type: string
               snapshots:
                 additionalProperties:
                   properties:
@@ -2499,6 +2509,11 @@ spec:
                 type: boolean
               salvageRequested:
                 type: boolean
+              snapshotMaxCount:
+                type: integer
+              snapshotMaxSize:
+                format: int64
+                type: string
               unmapMarkDiskChainRemovedEnabled:
                 type: boolean
               volumeName:
@@ -3605,6 +3620,11 @@ spec:
                 - enabled
                 - fast-check
                 type: string
+              snapshotMaxCount:
+                type: integer
+              snapshotMaxSize:
+                format: int64
+                type: string
               staleReplicaTimeout:
                 type: integer
               unmapMarkSnapChainRemoved:

k8s/pkg/apis/longhorn/v1beta2/engine.go

@@ -138,6 +138,11 @@ type EngineSpec struct {
 	UnmapMarkSnapChainRemovedEnabled bool `json:"unmapMarkSnapChainRemovedEnabled"`
 	// +optional
 	Active bool `json:"active"`
+	// +optional
+	SnapshotMaxCount int `json:"snapshotMaxCount"`
+	// +kubebuilder:validation:Type=string
+	// +optional
+	SnapshotMaxSize int64 `json:"snapshotMaxSize,string"`
 }
 
 // EngineStatus defines the observed state of the Longhorn engine
@@ -184,6 +189,11 @@ type EngineStatus struct {
 	LastExpansionFailedAt string `json:"lastExpansionFailedAt"`
 	// +optional
 	UnmapMarkSnapChainRemovedEnabled bool `json:"unmapMarkSnapChainRemovedEnabled"`
+	// +optional
+	SnapshotMaxCount int `json:"snapshotMaxCount"`
+	// +kubebuilder:validation:Type=string
+	// +optional
+	SnapshotMaxSize int64 `json:"snapshotMaxSize,string"`
 }
 
 // +genclient

k8s/pkg/apis/longhorn/v1beta2/replica.go

@@ -48,6 +48,11 @@ type ReplicaSpec struct {
 	RebuildRetryCount int `json:"rebuildRetryCount"`
 	// +optional
 	EvictionRequested bool `json:"evictionRequested"`
+	// +optional
+	SnapshotMaxCount int `json:"snapshotMaxCount"`
+	// +kubebuilder:validation:Type=string
+	// +optional
+	SnapshotMaxSize int64 `json:"snapshotMaxSize,string"`
 }
 
 // ReplicaStatus defines the observed state of the Longhorn replica

k8s/pkg/apis/longhorn/v1beta2/volume.go

@@ -293,6 +293,11 @@ type VolumeSpec struct {
 	// +kubebuilder:validation:Enum=ignored;disabled;enabled
 	// +optional
 	OfflineReplicaRebuilding OfflineReplicaRebuilding `json:"offlineReplicaRebuilding"`
+	// +optional
+	SnapshotMaxCount int `json:"snapshotMaxCount"`
+	// +kubebuilder:validation:Type=string
+	// +optional
+	SnapshotMaxSize int64 `json:"snapshotMaxSize,string"`
 }
 
 // VolumeStatus defines the observed state of the Longhorn volume