Override clone s3 bucket path (zalando#487)

Override clone s3 bucket path

Add possibility to use a custom s3 bucket path for cloning a cluster
from an arbitrary bucket (e.g. from another k8s cluster). For that
a new config options is introduced `s3_wal_path`, that should point
to a location that spilo would understand.

Author: erthalion

manifests/complete-postgres-manifest.yaml

@@ -63,6 +63,7 @@ spec:
   #  uid: "efd12e58-5786-11e8-b5a7-06148230260c"
   #  cluster: "acid-batman"
   #  timestamp: "2017-12-19T12:40:33+01:00" # timezone required (offset relative to UTC, see RFC 3339 section 5.6)
+  #  s3_wal_path: "s3://custom/path/to/bucket"
   maintenanceWindows:
   - 01:00-06:00 #UTC
   - Sat:00:00-04:00

pkg/apis/acid.zalan.do/v1/postgresql_type.go

@@ -114,6 +114,7 @@ type CloneDescription struct {
 	ClusterName  string `json:"cluster,omitempty"`
 	UID          string `json:"uid,omitempty"`
 	EndTimestamp string `json:"timestamp,omitempty"`
+	S3WalPath    string `json:"s3_wal_path,omitempty"`
 }
 
 // Sidecar defines a container to be run in the same pod as the Postgres container.

pkg/apis/acid.zalan.do/v1/util_test.go

@@ -61,12 +61,12 @@ var cloneClusterDescriptions = []struct {
 	in  *CloneDescription
 	err error
 }{
-	{&CloneDescription{"foo+bar", "", "NotEmpty"}, nil},
-	{&CloneDescription{"foo+bar", "", ""},
+	{&CloneDescription{"foo+bar", "", "NotEmpty", ""}, nil},
+	{&CloneDescription{"foo+bar", "", "", ""},
 		errors.New(`clone cluster name must confirm to DNS-1035, regex used for validation is "^[a-z]([-a-z0-9]*[a-z0-9])?$"`)},
-	{&CloneDescription{"foobar123456789012345678901234567890123456789012345678901234567890", "", ""},
+	{&CloneDescription{"foobar123456789012345678901234567890123456789012345678901234567890", "", "", ""},
 		errors.New("clone cluster name must be no longer than 63 characters")},
-	{&CloneDescription{"foobar", "", ""}, nil},
+	{&CloneDescription{"foobar", "", "", ""}, nil},
 }
 
 var maintenanceWindows = []struct {

pkg/cluster/k8sres.go

@@ -1216,10 +1216,37 @@ func (c *Cluster) generateCloneEnvironment(description *acidv1.CloneDescription)
 			})
 	} else {
 		// cloning with S3, find out the bucket to clone
+		msg := "Clone from S3 bucket"
+		c.logger.Info(msg, description.S3WalPath)
+
+		if description.S3WalPath == "" {
+			msg := "Figure out which S3 bucket to use from env"
+			c.logger.Info(msg, description.S3WalPath)
+
+			envs := []v1.EnvVar{
+				v1.EnvVar{
+					Name:  "CLONE_WAL_S3_BUCKET",
+					Value: c.OpConfig.WALES3Bucket,
+				},
+				v1.EnvVar{
+					Name:  "CLONE_WAL_BUCKET_SCOPE_SUFFIX",
+					Value: getBucketScopeSuffix(description.UID),
+				},
+			}
+
+			result = append(result, envs...)
+		} else {
+			msg := "Use custom parsed S3WalPath %s from the manifest"
+			c.logger.Warningf(msg, description.S3WalPath)
+
+			result = append(result, v1.EnvVar{
+				Name:  "CLONE_WALE_S3_PREFIX",
+				Value: description.S3WalPath,
+			})
+		}
+
 		result = append(result, v1.EnvVar{Name: "CLONE_METHOD", Value: "CLONE_WITH_WALE"})
-		result = append(result, v1.EnvVar{Name: "CLONE_WAL_S3_BUCKET", Value: c.OpConfig.WALES3Bucket})
 		result = append(result, v1.EnvVar{Name: "CLONE_TARGET_TIME", Value: description.EndTimestamp})
-		result = append(result, v1.EnvVar{Name: "CLONE_WAL_BUCKET_SCOPE_SUFFIX", Value: getBucketScopeSuffix(description.UID)})
 		result = append(result, v1.EnvVar{Name: "CLONE_WAL_BUCKET_SCOPE_PREFIX", Value: ""})
 	}
 

pkg/cluster/k8sres_test.go

@@ -129,3 +129,82 @@ func TestShmVolume(t *testing.T) {
 		}
 	}
 }
+
+func TestCloneEnv(t *testing.T) {
+	testName := "TestCloneEnv"
+	tests := []struct {
+		subTest   string
+		cloneOpts *acidv1.CloneDescription
+		env       v1.EnvVar
+		envPos    int
+	}{
+		{
+			subTest: "custom s3 path",
+			cloneOpts: &acidv1.CloneDescription{
+				ClusterName:  "test-cluster",
+				S3WalPath:    "s3://some/path/",
+				EndTimestamp: "somewhen",
+			},
+			env: v1.EnvVar{
+				Name:  "CLONE_WALE_S3_PREFIX",
+				Value: "s3://some/path/",
+			},
+			envPos: 1,
+		},
+		{
+			subTest: "generated s3 path, bucket",
+			cloneOpts: &acidv1.CloneDescription{
+				ClusterName:  "test-cluster",
+				EndTimestamp: "somewhen",
+				UID:          "0000",
+			},
+			env: v1.EnvVar{
+				Name:  "CLONE_WAL_S3_BUCKET",
+				Value: "wale-bucket",
+			},
+			envPos: 1,
+		},
+		{
+			subTest: "generated s3 path, target time",
+			cloneOpts: &acidv1.CloneDescription{
+				ClusterName:  "test-cluster",
+				EndTimestamp: "somewhen",
+				UID:          "0000",
+			},
+			env: v1.EnvVar{
+				Name:  "CLONE_TARGET_TIME",
+				Value: "somewhen",
+			},
+			envPos: 4,
+		},
+	}
+
+	var cluster = New(
+		Config{
+			OpConfig: config.Config{
+				WALES3Bucket:   "wale-bucket",
+				ProtectedRoles: []string{"admin"},
+				Auth: config.Auth{
+					SuperUsername:       superUserName,
+					ReplicationUsername: replicationUserName,
+				},
+			},
+		}, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger)
+
+	for _, tt := range tests {
+		envs := cluster.generateCloneEnvironment(tt.cloneOpts)
+
+		env := envs[tt.envPos]
+
+		if env.Name != tt.env.Name {
+			t.Errorf("%s %s: Expected env name %s, have %s instead",
+				testName, tt.subTest, tt.env.Name, env.Name)
+		}
+
+		if env.Value != tt.env.Value {
+			t.Errorf("%s %s: Expected env value %s, have %s instead",
+				testName, tt.subTest, tt.env.Value, env.Value)
+		}
+
+	}
+}