refactor: implement backup plugin hook (cloudnative-pg#3808)

This update empowers instance manager plugins to control the
backup process.

Partially closes: cloudnative-pg#3699

Signed-off-by: Leonardo Cecchi <leonardo.cecchi@enterprisedb.com>
Signed-off-by: Armando Ruocco <armando.ruocco@enterprisedb.com>
Signed-off-by: Marco Nenciarini <marco.nenciarini@enterprisedb.com>
Co-authored-by: Armando Ruocco <armando.ruocco@enterprisedb.com>
Co-authored-by: Marco Nenciarini <marco.nenciarini@enterprisedb.com>

.wordlist-en-custom.txt

@@ -21,12 +21,14 @@ AzureCredentials
 AzurePVCUpdateEnabled
 Azurite
 BDR
+BackupCapabilities
 BackupConfiguration
 BackupFrom
 BackupLabelFile
 BackupList
 BackupMethod
 BackupPhase
+BackupPluginConfiguration
 BackupSnapshotElementStatus
 BackupSnapshotStatus
 BackupSource
@@ -427,6 +429,7 @@ VolumeSnapshots
 WAL
 WAL's
 WALBackupConfiguration
+WALCapabilities
 WALs
 Wadle
 WalBackupConfiguration
@@ -491,6 +494,7 @@ backends
 backport
 backported
 backporting
+backupCapabilities
 backupID
 backupId
 backupLabelFile
@@ -935,6 +939,7 @@ pid
 pitr
 plpgsql
 pluggable
+pluginConfiguration
 pluginStatus
 png
 podAffinityTerm
@@ -1203,6 +1208,7 @@ volumeSnapshots
 volumesnapshot
 waitForArchive
 wal
+walCapabilities
 walClassName
 walSegmentSize
 walStorage

api/v1/backup_types.go

@@ -73,6 +73,10 @@ const (
 	// BackupMethodBarmanObjectStore means using barman to backup the
 	// PostgreSQL cluster
 	BackupMethodBarmanObjectStore BackupMethod = "barmanObjectStore"
+
+	// BackupMethodPlugin means that this backup should be handled by
+	// a plugin
+	BackupMethodPlugin BackupMethod = "plugin"
 )
 
 // BackupSpec defines the desired state of Backup
@@ -90,13 +94,17 @@ type BackupSpec struct {
 	// +kubebuilder:validation:Enum=primary;prefer-standby
 	Target BackupTarget `json:"target,omitempty"`
 
-	// The backup method to be used, possible options are `barmanObjectStore`
-	// and `volumeSnapshot`. Defaults to: `barmanObjectStore`.
+	// The backup method to be used, possible options are `barmanObjectStore`,
+	// `volumeSnapshot` or `plugin`. Defaults to: `barmanObjectStore`.
 	// +optional
-	// +kubebuilder:validation:Enum=barmanObjectStore;volumeSnapshot
+	// +kubebuilder:validation:Enum=barmanObjectStore;volumeSnapshot;plugin
 	// +kubebuilder:default:=barmanObjectStore
 	Method BackupMethod `json:"method,omitempty"`
 
+	// Configuration parameters passed to the plugin managing this backup
+	// +optional
+	PluginConfiguration *BackupPluginConfiguration `json:"pluginConfiguration,omitempty"`
+
 	// Whether the default type of backup with volume snapshots is
 	// online/hot (`true`, default) or offline/cold (`false`)
 	// Overrides the default setting specified in the cluster field '.spec.backup.volumeSnapshot.online'
@@ -109,6 +117,18 @@ type BackupSpec struct {
 	OnlineConfiguration *OnlineConfiguration `json:"onlineConfiguration,omitempty"`
 }
 
+// BackupPluginConfiguration contains the backup configuration used by
+// the backup plugin
+type BackupPluginConfiguration struct {
+	// Name is the name of the plugin managing this backup
+	Name string `json:"name"`
+
+	// Parameters are the configuration parameters passed to the backup
+	// plugin for this backup
+	// +optional
+	Parameters map[string]string `json:"parameters,omitempty"`
+}
+
 // BackupSnapshotStatus the fields exclusive to the volumeSnapshot method backup
 type BackupSnapshotStatus struct {
 	// The elements list, populated with the gathered volume snapshots
@@ -479,6 +499,11 @@ func (backup *Backup) GetVolumeSnapshotConfiguration(
 	return config
 }
 
+// IsEmpty checks if the plugin configuration is empty or not
+func (configuration *BackupPluginConfiguration) IsEmpty() bool {
+	return configuration == nil || len(configuration.Name) == 0
+}
+
 func init() {
 	SchemeBuilder.Register(&Backup{}, &BackupList{})
 }

api/v1/backup_webhook.go

@@ -108,5 +108,13 @@ func (r *Backup) validate() field.ErrorList {
 		))
 	}
 
+	if r.Spec.Method == BackupMethodPlugin && r.Spec.PluginConfiguration.IsEmpty() {
+		result = append(result, field.Invalid(
+			field.NewPath("spec", "pluginConfiguration"),
+			r.Spec.OnlineConfiguration,
+			"cannot be empty when the backup method is plugin",
+		))
+	}
+
 	return result
 }

api/v1/cluster_types.go

@@ -2382,8 +2382,12 @@ type PluginStatus struct {
 	OperatorCapabilities []string `json:"operatorCapabilities,omitempty"`
 
 	// WALCapabilities are the list of capabilities of the
-	// plugin regarding the reconciler
+	// plugin regarding the WAL management
 	WALCapabilities []string `json:"walCapabilities,omitempty"`
+
+	// BackupCapabilities are the list of capabilities of the
+	// plugin regarding the Backup management
+	BackupCapabilities []string `json:"backupCapabilities,omitempty"`
 }
 
 // RoleConfiguration is the representation, in Kubernetes, of a PostgreSQL role

api/v1/scheduledbackup_types.go

@@ -67,6 +67,10 @@ type ScheduledBackupSpec struct {
 	// +kubebuilder:default:=barmanObjectStore
 	Method BackupMethod `json:"method,omitempty"`
 
+	// Configuration parameters passed to the plugin managing this backup
+	// +optional
+	PluginConfiguration *BackupPluginConfiguration `json:"pluginConfiguration,omitempty"`
+
 	// Whether the default type of backup with volume snapshots is
 	// online/hot (`true`, default) or offline/cold (`false`)
 	// Overrides the default setting specified in the cluster field '.spec.backup.volumeSnapshot.online'
@@ -180,6 +184,7 @@ func (scheduledBackup *ScheduledBackup) CreateBackup(name string) *Backup {
 			Method:              scheduledBackup.Spec.Method,
 			Online:              scheduledBackup.Spec.Online,
 			OnlineConfiguration: scheduledBackup.Spec.OnlineConfiguration,
+			PluginConfiguration: scheduledBackup.Spec.PluginConfiguration,
 		},
 	}
 	utils.InheritAnnotations(&backup.ObjectMeta, scheduledBackup.Annotations, nil, configuration.Current)

config/crd/bases/postgresql.cnpg.io_backups.yaml

@@ -69,11 +69,12 @@ spec:
               method:
                 default: barmanObjectStore
                 description: |-
-                  The backup method to be used, possible options are `barmanObjectStore`
-                  and `volumeSnapshot`. Defaults to: `barmanObjectStore`.
+                  The backup method to be used, possible options are `barmanObjectStore`,
+                  `volumeSnapshot` or `plugin`. Defaults to: `barmanObjectStore`.
                 enum:
                 - barmanObjectStore
                 - volumeSnapshot
+                - plugin
                 type: string
               online:
                 description: |-
@@ -108,6 +109,23 @@ spec:
                       an immediate segment switch.
                     type: boolean
                 type: object
+              pluginConfiguration:
+                description: Configuration parameters passed to the plugin managing
+                  this backup
+                properties:
+                  name:
+                    description: Name is the name of the plugin managing this backup
+                    type: string
+                  parameters:
+                    additionalProperties:
+                      type: string
+                    description: |-
+                      Parameters are the configuration parameters passed to the backup
+                      plugin for this backup
+                    type: object
+                required:
+                - name
+                type: object
               target:
                 description: |-
                   The policy to decide which instance should perform this backup. If empty,

config/crd/bases/postgresql.cnpg.io_clusters.yaml

@@ -3211,11 +3211,13 @@ spec:
                     type: boolean
                 type: object
               plugins:
-                description: The plugins configuration, containing any plugin to be
-                  loaded with the corresponding configuration
+                description: |-
+                  The plugins configuration, containing
+                  any plugin to be loaded with the corresponding configuration
                 items:
-                  description: PluginConfiguration specifies a plugin that need to
-                    be loaded for this cluster to be reconciled
+                  description: |-
+                    PluginConfiguration specifies a plugin that need to be loaded for this
+                    cluster to be reconciled
                   properties:
                     name:
                       description: Name is the plugin name
@@ -5165,8 +5167,16 @@ spec:
                 items:
                   description: PluginStatus is the status of a loaded plugin
                   properties:
+                    backupCapabilities:
+                      description: |-
+                        BackupCapabilities are the list of capabilities of the
+                        plugin regarding the Backup management
+                      items:
+                        type: string
+                      type: array
                     capabilities:
-                      description: Capabilities are the list of capabilities of the
+                      description: |-
+                        Capabilities are the list of capabilities of the
                         plugin
                       items:
                         type: string
@@ -5175,18 +5185,21 @@ spec:
                       description: Name is the name of the plugin
                       type: string
                     operatorCapabilities:
-                      description: OperatorCapabilities are the list of capabilities
-                        of the plugin regarding the reconciler
+                      description: |-
+                        OperatorCapabilities are the list of capabilities of the
+                        plugin regarding the reconciler
                       items:
                         type: string
                       type: array
                     version:
-                      description: Version is the version of the plugin loaded by
-                        the latest reconciliation loop
+                      description: |-
+                        Version is the version of the plugin loaded by the
+                        latest reconciliation loop
                       type: string
                     walCapabilities:
-                      description: WALCapabilities are the list of capabilities of
-                        the plugin regarding the reconciler
+                      description: |-
+                        WALCapabilities are the list of capabilities of the
+                        plugin regarding the WAL management
                       items:
                         type: string
                       type: array

config/crd/bases/postgresql.cnpg.io_scheduledbackups.yaml

@@ -118,6 +118,23 @@ spec:
                       an immediate segment switch.
                     type: boolean
                 type: object
+              pluginConfiguration:
+                description: Configuration parameters passed to the plugin managing
+                  this backup
+                properties:
+                  name:
+                    description: Name is the name of the plugin managing this backup
+                    type: string
+                  parameters:
+                    additionalProperties:
+                      type: string
+                    description: |-
+                      Parameters are the configuration parameters passed to the backup
+                      plugin for this backup
+                    type: object
+                required:
+                - name
+                type: object
               schedule:
                 description: |-
                   The schedule does not follow the same format used in Kubernetes CronJobs