Add ability to upload logical backup to gcs (#1173)

Support logical backup provider/storage S3 and GCS equivalent

Author: Pavel Tumik

docker/logical-backup/Dockerfile

@@ -13,7 +13,10 @@ RUN apt-get update     \
         curl \
         jq \
         gnupg \
+        gcc \
+        libffi-dev \
     && pip3 install --no-cache-dir awscli --upgrade \
+    && pip3 install --no-cache-dir gsutil --upgrade \
     && echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list \
     && cat /etc/apt/sources.list.d/pgdg.list \
     && curl --silent https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - \

docker/logical-backup/dump.sh

@@ -46,6 +46,23 @@ function aws_upload {
     aws s3 cp - "$PATH_TO_BACKUP" "${args[@]//\'/}"
 }
 
+function gcs_upload {
+    PATH_TO_BACKUP=gs://$LOGICAL_BACKUP_S3_BUCKET"/spilo/"$SCOPE$LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX"/logical_backups/"$(date +%s).sql.gz
+
+    gsutil -o Credentials:gs_service_key_file=$LOGICAL_BACKUP_GOOGLE_APPLICATION_CREDENTIALS cp - "$PATH_TO_BACKUP"
+}
+
+function upload {
+    case $LOGICAL_BACKUP_PROVIDER in
+        "gcs")
+            gcs_upload
+            ;;
+        *)
+            aws_upload $(($(estimate_size) / DUMP_SIZE_COEFF))
+            ;;
+    esac
+}
+
 function get_pods {
     declare -r SELECTOR="$1"
 
@@ -93,7 +110,7 @@ for search in "${search_strategy[@]}"; do
 done
 
 set -x
-dump | compress | aws_upload $(($(estimate_size) / DUMP_SIZE_COEFF))
+dump | compress | upload
 [[ ${PIPESTATUS[0]} != 0 || ${PIPESTATUS[1]} != 0 || ${PIPESTATUS[2]} != 0 ]] && (( ERRORCOUNT += 1 ))
 set +x
 

docs/reference/operator_parameters.md

@@ -563,6 +563,10 @@ grouped under the `logical_backup` key.
   The default image is the same image built with the Zalando-internal CI
   pipeline. Default: "registry.opensource.zalan.do/acid/logical-backup"
 
+* **logical_backup_provider**
+  Specifies the storage provider to which the backup should be uploaded (`s3` or `gcs`).
+  Default: "s3"
+
 * **logical_backup_s3_bucket**
   S3 bucket to store backup results. The bucket has to be present and
   accessible by Postgres pods. Default: empty.
@@ -583,6 +587,9 @@ grouped under the `logical_backup` key.
 * **logical_backup_s3_secret_access_key**
   When set, value will be in AWS_SECRET_ACCESS_KEY env variable. The Default is empty.
 
+* **logical_backup_google_application_credentials**
+  Specifies the path of the google cloud service account json file. Default is empty.
+
 ## Debugging the operator
 
 Options to aid debugging of the operator itself. Grouped under the `debug` key.

pkg/apis/acid.zalan.do/v1/operator_configuration_type.go

@@ -186,14 +186,16 @@ type ConnectionPoolerConfiguration struct {
 
 // OperatorLogicalBackupConfiguration defines configuration for logical backup
 type OperatorLogicalBackupConfiguration struct {
-	Schedule          string `json:"logical_backup_schedule,omitempty"`
-	DockerImage       string `json:"logical_backup_docker_image,omitempty"`
-	S3Bucket          string `json:"logical_backup_s3_bucket,omitempty"`
-	S3Region          string `json:"logical_backup_s3_region,omitempty"`
-	S3Endpoint        string `json:"logical_backup_s3_endpoint,omitempty"`
-	S3AccessKeyID     string `json:"logical_backup_s3_access_key_id,omitempty"`
-	S3SecretAccessKey string `json:"logical_backup_s3_secret_access_key,omitempty"`
-	S3SSE             string `json:"logical_backup_s3_sse,omitempty"`
+	Schedule                     string `json:"logical_backup_schedule,omitempty"`
+	DockerImage                  string `json:"logical_backup_docker_image,omitempty"`
+	BackupProvider               string `json:"logical_backup_provider,omitempty"`
+	S3Bucket                     string `json:"logical_backup_s3_bucket,omitempty"`
+	S3Region                     string `json:"logical_backup_s3_region,omitempty"`
+	S3Endpoint                   string `json:"logical_backup_s3_endpoint,omitempty"`
+	S3AccessKeyID                string `json:"logical_backup_s3_access_key_id,omitempty"`
+	S3SecretAccessKey            string `json:"logical_backup_s3_secret_access_key,omitempty"`
+	S3SSE                        string `json:"logical_backup_s3_sse,omitempty"`
+	GoogleApplicationCredentials string `json:"logical_backup_google_application_credentials,omitempty"`
 }
 
 // OperatorConfigurationData defines the operation config

pkg/cluster/k8sres.go

@@ -1988,6 +1988,10 @@ func (c *Cluster) generateLogicalBackupPodEnvVars() []v1.EnvVar {
 			},
 		},
 		// Bucket env vars
+		{
+			Name:  "LOGICAL_BACKUP_PROVIDER",
+			Value: c.OpConfig.LogicalBackup.LogicalBackupProvider,
+		},
 		{
 			Name:  "LOGICAL_BACKUP_S3_BUCKET",
 			Value: c.OpConfig.LogicalBackup.LogicalBackupS3Bucket,
@@ -2008,6 +2012,10 @@ func (c *Cluster) generateLogicalBackupPodEnvVars() []v1.EnvVar {
 			Name:  "LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX",
 			Value: getBucketScopeSuffix(string(c.Postgresql.GetUID())),
 		},
+		{
+			Name:  "LOGICAL_BACKUP_GOOGLE_APPLICATION_CREDENTIALS",
+			Value: c.OpConfig.LogicalBackup.LogicalBackupGoogleApplicationCredentials,
+		},
 		// Postgres env vars
 		{
 			Name:  "PG_VERSION",

pkg/controller/operator_config.go

@@ -146,12 +146,14 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur
 	// logical backup config
 	result.LogicalBackupSchedule = util.Coalesce(fromCRD.LogicalBackup.Schedule, "30 00 * * *")
 	result.LogicalBackupDockerImage = util.Coalesce(fromCRD.LogicalBackup.DockerImage, "registry.opensource.zalan.do/acid/logical-backup")
+	result.LogicalBackupProvider = util.Coalesce(fromCRD.LogicalBackup.BackupProvider, "s3")
 	result.LogicalBackupS3Bucket = fromCRD.LogicalBackup.S3Bucket
 	result.LogicalBackupS3Region = fromCRD.LogicalBackup.S3Region
 	result.LogicalBackupS3Endpoint = fromCRD.LogicalBackup.S3Endpoint
 	result.LogicalBackupS3AccessKeyID = fromCRD.LogicalBackup.S3AccessKeyID
 	result.LogicalBackupS3SecretAccessKey = fromCRD.LogicalBackup.S3SecretAccessKey
 	result.LogicalBackupS3SSE = fromCRD.LogicalBackup.S3SSE
+	result.LogicalBackupGoogleApplicationCredentials = fromCRD.LogicalBackup.GoogleApplicationCredentials
 
 	// debug config
 	result.DebugLogging = fromCRD.OperatorDebug.DebugLogging

pkg/util/config/config.go

@@ -111,14 +111,16 @@ type Scalyr struct {
 
 // LogicalBackup defines configuration for logical backup
 type LogicalBackup struct {
-	LogicalBackupSchedule          string `name:"logical_backup_schedule" default:"30 00 * * *"`
-	LogicalBackupDockerImage       string `name:"logical_backup_docker_image" default:"registry.opensource.zalan.do/acid/logical-backup"`
-	LogicalBackupS3Bucket          string `name:"logical_backup_s3_bucket" default:""`
-	LogicalBackupS3Region          string `name:"logical_backup_s3_region" default:""`
-	LogicalBackupS3Endpoint        string `name:"logical_backup_s3_endpoint" default:""`
-	LogicalBackupS3AccessKeyID     string `name:"logical_backup_s3_access_key_id" default:""`
-	LogicalBackupS3SecretAccessKey string `name:"logical_backup_s3_secret_access_key" default:""`
-	LogicalBackupS3SSE             string `name:"logical_backup_s3_sse" default:""`
+	LogicalBackupSchedule                     string `name:"logical_backup_schedule" default:"30 00 * * *"`
+	LogicalBackupDockerImage                  string `name:"logical_backup_docker_image" default:"registry.opensource.zalan.do/acid/logical-backup"`
+	LogicalBackupProvider                     string `name:"logical_backup_provider" default:"s3"`
+	LogicalBackupS3Bucket                     string `name:"logical_backup_s3_bucket" default:""`
+	LogicalBackupS3Region                     string `name:"logical_backup_s3_region" default:""`
+	LogicalBackupS3Endpoint                   string `name:"logical_backup_s3_endpoint" default:""`
+	LogicalBackupS3AccessKeyID                string `name:"logical_backup_s3_access_key_id" default:""`
+	LogicalBackupS3SecretAccessKey            string `name:"logical_backup_s3_secret_access_key" default:""`
+	LogicalBackupS3SSE                        string `name:"logical_backup_s3_sse" default:""`
+	LogicalBackupGoogleApplicationCredentials string `name:"logical_backup_google_application_credentials" default:""`
 }
 
 // Operator options for connection pooler