Add S3 KMS support to blocks storage client (#3810)

* Add S3 KMS support to blocks storage client

Signed-off-by: Marco Pracucci <marco@pracucci.com>

* Fixed integration test

Signed-off-by: Marco Pracucci <marco@pracucci.com>

* Removed named return arguments from parseKMSEncryptionContext()

Signed-off-by: Marco Pracucci <marco@pracucci.com>

* Update pkg/storage/bucket/s3/config.go

Signed-off-by: Marco Pracucci <marco@pracucci.com>

Co-authored-by: Jacob Lisi <jlisi@grafana.com>

* Rebuilt doc

Signed-off-by: Marco Pracucci <marco@pracucci.com>

Co-authored-by: Jacob Lisi <jlisi@grafana.com>

CHANGELOG.md

@@ -4,7 +4,7 @@
 
 * [CHANGE] Ingester: don't update internal "last updated" timestamp of TSDB if tenant only sends invalid samples. This affects how "idle" time is computed. #3727
 * [CHANGE] Require explicit flag `-<prefix>.tls-enabled` to enable TLS in GRPC clients. Previously it was enough to specify a TLS flag to enable TLS validation. #3156
-* [FEATURE] Adds support to S3 server side encryption using KMS. Deprecated `-<prefix>.s3.sse-encryption`, you should use the following CLI flags that have been added. #3651
+* [FEATURE] Adds support to S3 server side encryption using KMS. Deprecated `-<prefix>.s3.sse-encryption`, you should use the following CLI flags that have been added. #3651 #3810
   - `-<prefix>.s3.sse.type`
   - `-<prefix>.s3.sse.kms-key-id`
   - `-<prefix>.s3.sse.kms-encryption-context`

docs/blocks-storage/querier.md

@@ -259,6 +259,20 @@ blocks_storage:
     # CLI flag: -blocks-storage.s3.signature-version
     [signature_version: <string> | default = "v4"]
 
+    sse:
+      # Enable AWS Server Side Encryption. Only SSE-S3 and SSE-KMS are supported
+      # CLI flag: -blocks-storage.s3.sse.type
+      [type: <string> | default = ""]
+
+      # KMS Key ID used to encrypt objects in S3
+      # CLI flag: -blocks-storage.s3.sse.kms-key-id
+      [kms_key_id: <string> | default = ""]
+
+      # KMS Encryption Context used for object encryption. It expects JSON
+      # formatted string.
+      # CLI flag: -blocks-storage.s3.sse.kms-encryption-context
+      [kms_encryption_context: <string> | default = ""]
+
     http:
       # The time an idle connection will remain idle before closing.
       # CLI flag: -blocks-storage.s3.http.idle-conn-timeout

docs/blocks-storage/store-gateway.md

@@ -306,6 +306,20 @@ blocks_storage:
     # CLI flag: -blocks-storage.s3.signature-version
     [signature_version: <string> | default = "v4"]
 
+    sse:
+      # Enable AWS Server Side Encryption. Only SSE-S3 and SSE-KMS are supported
+      # CLI flag: -blocks-storage.s3.sse.type
+      [type: <string> | default = ""]
+
+      # KMS Key ID used to encrypt objects in S3
+      # CLI flag: -blocks-storage.s3.sse.kms-key-id
+      [kms_key_id: <string> | default = ""]
+
+      # KMS Encryption Context used for object encryption. It expects JSON
+      # formatted string.
+      # CLI flag: -blocks-storage.s3.sse.kms-encryption-context
+      [kms_encryption_context: <string> | default = ""]
+
     http:
       # The time an idle connection will remain idle before closing.
       # CLI flag: -blocks-storage.s3.http.idle-conn-timeout

docs/configuration/config-file-reference.md

@@ -1312,8 +1312,8 @@ storage:
       # CLI flag: -ruler.storage.s3.sse.kms-key-id
       [kms_key_id: <string> | default = ""]
 
-      # KMS Encryption Context used for object encryption. It expects a JSON as
-      # a string.
+      # KMS Encryption Context used for object encryption. It expects JSON
+      # formatted string.
       # CLI flag: -ruler.storage.s3.sse.kms-encryption-context
       [kms_encryption_context: <string> | default = ""]
 
@@ -1801,8 +1801,8 @@ storage:
       # CLI flag: -alertmanager.storage.s3.sse.kms-key-id
       [kms_key_id: <string> | default = ""]
 
-      # KMS Encryption Context used for object encryption. It expects a JSON as
-      # a string.
+      # KMS Encryption Context used for object encryption. It expects JSON
+      # formatted string.
       # CLI flag: -alertmanager.storage.s3.sse.kms-encryption-context
       [kms_encryption_context: <string> | default = ""]
 
@@ -2333,8 +2333,8 @@ aws:
     # CLI flag: -s3.sse.kms-key-id
     [kms_key_id: <string> | default = ""]
 
-    # KMS Encryption Context used for object encryption. It expects a JSON as a
-    # string.
+    # KMS Encryption Context used for object encryption. It expects JSON
+    # formatted string.
     # CLI flag: -s3.sse.kms-encryption-context
     [kms_encryption_context: <string> | default = ""]
 
@@ -3840,6 +3840,20 @@ s3:
   # CLI flag: -blocks-storage.s3.signature-version
   [signature_version: <string> | default = "v4"]
 
+  sse:
+    # Enable AWS Server Side Encryption. Only SSE-S3 and SSE-KMS are supported
+    # CLI flag: -blocks-storage.s3.sse.type
+    [type: <string> | default = ""]
+
+    # KMS Key ID used to encrypt objects in S3
+    # CLI flag: -blocks-storage.s3.sse.kms-key-id
+    [kms_key_id: <string> | default = ""]
+
+    # KMS Encryption Context used for object encryption. It expects JSON
+    # formatted string.
+    # CLI flag: -blocks-storage.s3.sse.kms-encryption-context
+    [kms_encryption_context: <string> | default = ""]
+
   http:
     # The time an idle connection will remain idle before closing.
     # CLI flag: -blocks-storage.s3.http.idle-conn-timeout

integration/s3_storage_client_test.go

@@ -14,6 +14,7 @@ import (
 	"github.com/cortexproject/cortex/integration/e2e"
 	e2edb "github.com/cortexproject/cortex/integration/e2e/db"
 	s3 "github.com/cortexproject/cortex/pkg/chunk/aws"
+	cortex_s3 "github.com/cortexproject/cortex/pkg/storage/bucket/s3"
 	"github.com/cortexproject/cortex/pkg/util/flagext"
 )
 
@@ -84,7 +85,7 @@ func TestS3Client(t *testing.T) {
 				Insecure:         true,
 				AccessKeyID:      e2edb.MinioAccessKey,
 				SecretAccessKey:  e2edb.MinioSecretKey,
-				SSEConfig: s3.SSEConfig{
+				SSEConfig: cortex_s3.SSEConfig{
 					Type: "SSE-S3",
 				},
 			},

pkg/chunk/aws/s3_storage_client.go

@@ -27,6 +27,7 @@ import (
 	"github.com/weaveworks/common/instrument"
 
 	"github.com/cortexproject/cortex/pkg/chunk"
+	cortex_s3 "github.com/cortexproject/cortex/pkg/storage/bucket/s3"
 	"github.com/cortexproject/cortex/pkg/util"
 	"github.com/cortexproject/cortex/pkg/util/flagext"
 )
@@ -64,15 +65,15 @@ type S3Config struct {
 	S3ForcePathStyle bool
 
 	BucketNames      string
-	Endpoint         string     `yaml:"endpoint"`
-	Region           string     `yaml:"region"`
-	AccessKeyID      string     `yaml:"access_key_id"`
-	SecretAccessKey  string     `yaml:"secret_access_key"`
-	Insecure         bool       `yaml:"insecure"`
-	SSEEncryption    bool       `yaml:"sse_encryption"`
-	HTTPConfig       HTTPConfig `yaml:"http_config"`
-	SignatureVersion string     `yaml:"signature_version"`
-	SSEConfig        SSEConfig  `yaml:"sse"`
+	Endpoint         string              `yaml:"endpoint"`
+	Region           string              `yaml:"region"`
+	AccessKeyID      string              `yaml:"access_key_id"`
+	SecretAccessKey  string              `yaml:"secret_access_key"`
+	Insecure         bool                `yaml:"insecure"`
+	SSEEncryption    bool                `yaml:"sse_encryption"`
+	HTTPConfig       HTTPConfig          `yaml:"http_config"`
+	SignatureVersion string              `yaml:"signature_version"`
+	SSEConfig        cortex_s3.SSEConfig `yaml:"sse"`
 
 	Inject InjectRequestMiddleware `yaml:"-"`
 }
@@ -165,8 +166,8 @@ func buildSSEParsedConfig(cfg S3Config) (*SSEParsedConfig, error) {
 
 	// deprecated, but if used it assumes SSE-S3 type
 	if cfg.SSEEncryption {
-		return NewSSEParsedConfig(SSEConfig{
-			Type: SSES3,
+		return NewSSEParsedConfig(cortex_s3.SSEConfig{
+			Type: cortex_s3.SSES3,
 		})
 	}
 

pkg/chunk/aws/sse_config.go

@@ -3,20 +3,15 @@ package aws
 import (
 	"encoding/base64"
 	"encoding/json"
-	"flag"
 
 	"github.com/pkg/errors"
+
+	cortex_s3 "github.com/cortexproject/cortex/pkg/storage/bucket/s3"
 )
 
 const (
-	// SSEKMS config type constant to configure S3 server side encryption using KMS
-	// https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingKMSEncryption.html
-	SSEKMS     = "SSE-KMS"
 	sseKMSType = "aws:kms"
-	// SSES3 config type constant to configure S3 server side encryption with AES-256
-	// https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingServerSideEncryption.html
-	SSES3     = "SSE-S3"
-	sseS3Type = "AES256"
+	sseS3Type  = "AES256"
 )
 
 // SSEParsedConfig configures server side encryption (SSE)
@@ -27,29 +22,14 @@ type SSEParsedConfig struct {
 	KMSEncryptionContext *string
 }
 
-// SSEConfig configures S3 server side encryption
-// struct that is going to receive user input (through config file or CLI)
-type SSEConfig struct {
-	Type                 string `yaml:"type"`
-	KMSKeyID             string `yaml:"kms_key_id"`
-	KMSEncryptionContext string `yaml:"kms_encryption_context"`
-}
-
-// RegisterFlagsWithPrefix adds the flags required to config this to the given FlagSet
-func (cfg *SSEConfig) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) {
-	f.StringVar(&cfg.Type, prefix+"type", "", "Enable AWS Server Side Encryption. Only SSE-S3 and SSE-KMS are supported")
-	f.StringVar(&cfg.KMSKeyID, prefix+"kms-key-id", "", "KMS Key ID used to encrypt objects in S3")
-	f.StringVar(&cfg.KMSEncryptionContext, prefix+"kms-encryption-context", "", "KMS Encryption Context used for object encryption. It expects a JSON as a string.")
-}
-
 // NewSSEParsedConfig creates a struct to configure server side encryption (SSE)
-func NewSSEParsedConfig(cfg SSEConfig) (*SSEParsedConfig, error) {
+func NewSSEParsedConfig(cfg cortex_s3.SSEConfig) (*SSEParsedConfig, error) {
 	switch cfg.Type {
-	case SSES3:
+	case cortex_s3.SSES3:
 		return &SSEParsedConfig{
 			ServerSideEncryption: sseS3Type,
 		}, nil
-	case SSEKMS:
+	case cortex_s3.SSEKMS:
 		if cfg.KMSKeyID == "" {
 			return nil, errors.New("KMS key id must be passed when SSE-KMS encryption is selected")
 		}

pkg/chunk/aws/sse_config_test.go

@@ -5,6 +5,8 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/stretchr/testify/assert"
+
+	cortex_s3 "github.com/cortexproject/cortex/pkg/storage/bucket/s3"
 )
 
 func TestNewSSEParsedConfig(t *testing.T) {
@@ -15,23 +17,23 @@ func TestNewSSEParsedConfig(t *testing.T) {
 
 	tests := []struct {
 		name        string
-		params      SSEConfig
+		params      cortex_s3.SSEConfig
 		expected    *SSEParsedConfig
 		expectedErr error
 	}{
 		{
 			name: "Test SSE encryption with SSES3 type",
-			params: SSEConfig{
-				Type: SSES3,
+			params: cortex_s3.SSEConfig{
+				Type: cortex_s3.SSES3,
 			},
 			expected: &SSEParsedConfig{
 				ServerSideEncryption: sseS3Type,
 			},
 		},
 		{
 			name: "Test SSE encryption with SSEKMS type without context",
-			params: SSEConfig{
-				Type:     SSEKMS,
+			params: cortex_s3.SSEConfig{
+				Type:     cortex_s3.SSEKMS,
 				KMSKeyID: kmsKeyID,
 			},
 			expected: &SSEParsedConfig{
@@ -41,8 +43,8 @@ func TestNewSSEParsedConfig(t *testing.T) {
 		},
 		{
 			name: "Test SSE encryption with SSEKMS type with context",
-			params: SSEConfig{
-				Type:                 SSEKMS,
+			params: cortex_s3.SSEConfig{
+				Type:                 cortex_s3.SSEKMS,
 				KMSKeyID:             kmsKeyID,
 				KMSEncryptionContext: kmsEncryptionContext,
 			},
@@ -54,23 +56,23 @@ func TestNewSSEParsedConfig(t *testing.T) {
 		},
 		{
 			name: "Test invalid SSE type",
-			params: SSEConfig{
+			params: cortex_s3.SSEConfig{
 				Type: "invalid",
 			},
 			expectedErr: errors.New("SSE type is empty or invalid"),
 		},
 		{
 			name: "Test SSE encryption with SSEKMS type without KMS Key ID",
-			params: SSEConfig{
-				Type:     SSEKMS,
+			params: cortex_s3.SSEConfig{
+				Type:     cortex_s3.SSEKMS,
 				KMSKeyID: "",
 			},
 			expectedErr: errors.New("KMS key id must be passed when SSE-KMS encryption is selected"),
 		},
 		{
 			name: "Test SSE with invalid KMS encryption context JSON",
-			params: SSEConfig{
-				Type:                 SSEKMS,
+			params: cortex_s3.SSEConfig{
+				Type:                 cortex_s3.SSEKMS,
 				KMSKeyID:             kmsKeyID,
 				KMSEncryptionContext: `INVALID_JSON`,
 			},

pkg/storage/bucket/s3/bucket_client.go

@@ -9,21 +9,37 @@ import (
 
 // NewBucketClient creates a new S3 bucket client
 func NewBucketClient(cfg Config, name string, logger log.Logger) (objstore.Bucket, error) {
-	return s3.NewBucketWithConfig(logger, newS3Config(cfg), name)
+	s3Cfg, err := newS3Config(cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	return s3.NewBucketWithConfig(logger, s3Cfg, name)
 }
 
 // NewBucketReaderClient creates a new S3 bucket client
 func NewBucketReaderClient(cfg Config, name string, logger log.Logger) (objstore.BucketReader, error) {
-	return s3.NewBucketWithConfig(logger, newS3Config(cfg), name)
+	s3Cfg, err := newS3Config(cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	return s3.NewBucketWithConfig(logger, s3Cfg, name)
 }
 
-func newS3Config(cfg Config) s3.Config {
+func newS3Config(cfg Config) (s3.Config, error) {
+	sseCfg, err := cfg.SSE.BuildThanosConfig()
+	if err != nil {
+		return s3.Config{}, err
+	}
+
 	return s3.Config{
 		Bucket:    cfg.BucketName,
 		Endpoint:  cfg.Endpoint,
 		AccessKey: cfg.AccessKeyID,
 		SecretKey: cfg.SecretAccessKey.Value,
 		Insecure:  cfg.Insecure,
+		SSEConfig: sseCfg,
 		HTTPConfig: s3.HTTPConfig{
 			IdleConnTimeout:       model.Duration(cfg.HTTP.IdleConnTimeout),
 			ResponseHeaderTimeout: model.Duration(cfg.HTTP.ResponseHeaderTimeout),
@@ -37,5 +53,5 @@ func newS3Config(cfg Config) s3.Config {
 		},
 		// Enforce signature version 2 if CLI flag is set
 		SignatureV2: cfg.SignatureVersion == SignatureVersionV2,
-	}
+	}, nil
 }