Make s3 bucket lookup type configurable

CHANGELOG.md

@@ -52,6 +52,7 @@
 * [FEATURE] Create OpenTelemetry Bridge for Tracing. Now cortex can send traces to multiple destinations using OTEL Collectors. #4834
 * [FEATURE] Added `-api.http-request-headers-to-log` allowing for the addition of HTTP Headers to logs #4803
 * [FEATURE] Distributor: Added a new limit `-validation.max-labels-size-bytes` allowing to limit the combined size of labels for each timeseries. #4848
+* [FEATURE] Storage/Bucket: Added `-*.s3.bucket-lookup-type` allowing to configure the s3 bucket lookup type. #4794
 * [BUGFIX] Memberlist: Add join with no retrying when starting service. #4804
 * [BUGFIX] Ruler: Fix /ruler/rule_groups returns YAML with extra fields. #4767
 

docs/blocks-storage/querier.md

@@ -264,6 +264,11 @@ blocks_storage:
     # CLI flag: -blocks-storage.s3.signature-version
     [signature_version: <string> | default = "v4"]
 
+    # The s3 bucket lookup style. Supported values are: auto, virtual-hosted,
+    # path.
+    # CLI flag: -blocks-storage.s3.bucket-lookup-type
+    [bucket_lookup_type: <string> | default = "auto"]
+
     # The s3_sse_config configures the S3 server-side encryption.
     # The CLI flags prefix for this block config is: blocks-storage
     [sse: <s3_sse_config>]

docs/blocks-storage/store-gateway.md

@@ -329,6 +329,11 @@ blocks_storage:
     # CLI flag: -blocks-storage.s3.signature-version
     [signature_version: <string> | default = "v4"]
 
+    # The s3 bucket lookup style. Supported values are: auto, virtual-hosted,
+    # path.
+    # CLI flag: -blocks-storage.s3.bucket-lookup-type
+    [bucket_lookup_type: <string> | default = "auto"]
+
     # The s3_sse_config configures the S3 server-side encryption.
     # The CLI flags prefix for this block config is: blocks-storage
     [sse: <s3_sse_config>]

docs/configuration/config-file-reference.md

@@ -1397,6 +1397,11 @@ s3:
   # CLI flag: -ruler-storage.s3.signature-version
   [signature_version: <string> | default = "v4"]
 
+  # The s3 bucket lookup style. Supported values are: auto, virtual-hosted,
+  # path.
+  # CLI flag: -ruler-storage.s3.bucket-lookup-type
+  [bucket_lookup_type: <string> | default = "auto"]
+
   # The s3_sse_config configures the S3 server-side encryption.
   # The CLI flags prefix for this block config is: ruler-storage
   [sse: <s3_sse_config>]
@@ -1839,6 +1844,11 @@ s3:
   # CLI flag: -alertmanager-storage.s3.signature-version
   [signature_version: <string> | default = "v4"]
 
+  # The s3 bucket lookup style. Supported values are: auto, virtual-hosted,
+  # path.
+  # CLI flag: -alertmanager-storage.s3.bucket-lookup-type
+  [bucket_lookup_type: <string> | default = "auto"]
+
   # The s3_sse_config configures the S3 server-side encryption.
   # The CLI flags prefix for this block config is: alertmanager-storage
   [sse: <s3_sse_config>]
@@ -3081,6 +3091,11 @@ s3:
   # CLI flag: -blocks-storage.s3.signature-version
   [signature_version: <string> | default = "v4"]
 
+  # The s3 bucket lookup style. Supported values are: auto, virtual-hosted,
+  # path.
+  # CLI flag: -blocks-storage.s3.bucket-lookup-type
+  [bucket_lookup_type: <string> | default = "auto"]
+
   # The s3_sse_config configures the S3 server-side encryption.
   # The CLI flags prefix for this block config is: blocks-storage
   [sse: <s3_sse_config>]

integration/e2ecortex/storage.go

@@ -39,11 +39,12 @@ func NewS3Client(cfg s3.Config) (*S3Client, error) {
 
 func NewS3ClientForMinio(minio *e2e.HTTPService, bucketName string) (*S3Client, error) {
 	return NewS3Client(s3.Config{
-		Endpoint:        minio.HTTPEndpoint(),
-		BucketName:      bucketName,
-		SecretAccessKey: flagext.Secret{Value: e2edb.MinioSecretKey},
-		AccessKeyID:     e2edb.MinioAccessKey,
-		Insecure:        true,
+		Endpoint:         minio.HTTPEndpoint(),
+		BucketName:       bucketName,
+		SecretAccessKey:  flagext.Secret{Value: e2edb.MinioSecretKey},
+		AccessKeyID:      e2edb.MinioAccessKey,
+		Insecure:         true,
+		BucketLookupType: s3.BucketAutoLookup,
 	})
 }
 

pkg/cortex/cortex_test.go

@@ -53,7 +53,8 @@ func TestCortex(t *testing.T) {
 				Bucket: bucket.Config{
 					Backend: bucket.S3,
 					S3: s3.Config{
-						Endpoint: "localhost",
+						Endpoint:         "localhost",
+						BucketLookupType: s3.BucketPathLookup,
 					},
 				},
 			},
@@ -71,7 +72,8 @@ func TestCortex(t *testing.T) {
 			Bucket: bucket.Config{
 				Backend: bucket.S3,
 				S3: s3.Config{
-					Endpoint: "localhost",
+					Endpoint:         "localhost",
+					BucketLookupType: s3.BucketPathLookup,
 				},
 			},
 			BucketStore: tsdb.BucketStoreConfig{

pkg/storage/bucket/s3/bucket_client.go

@@ -32,6 +32,10 @@ func newS3Config(cfg Config) (s3.Config, error) {
 	if err != nil {
 		return s3.Config{}, err
 	}
+	bucketLookupType, err := cfg.bucketLookupType()
+	if err != nil {
+		return s3.Config{}, err
+	}
 
 	return s3.Config{
 		Bucket:    cfg.BucketName,
@@ -53,6 +57,7 @@ func newS3Config(cfg Config) (s3.Config, error) {
 			Transport:             cfg.HTTP.Transport,
 		},
 		// Enforce signature version 2 if CLI flag is set
-		SignatureV2: cfg.SignatureVersion == SignatureVersionV2,
+		SignatureV2:      cfg.SignatureVersion == SignatureVersionV2,
+		BucketLookupType: bucketLookupType,
 	}, nil
 }

pkg/storage/bucket/s3/config.go

@@ -27,14 +27,20 @@ const (
 	// SSES3 config type constant to configure S3 server side encryption with AES-256
 	// https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingServerSideEncryption.html
 	SSES3 = "SSE-S3"
+
+	BucketAutoLookup        = "auto"
+	BucketVirtualHostLookup = "virtual-hosted"
+	BucketPathLookup        = "path"
 )
 
 var (
 	supportedSignatureVersions     = []string{SignatureVersionV4, SignatureVersionV2}
 	supportedSSETypes              = []string{SSEKMS, SSES3}
+	supportedBucketLookupTypes     = []string{BucketAutoLookup, BucketVirtualHostLookup, BucketPathLookup}
 	errUnsupportedSignatureVersion = errors.New("unsupported signature version")
 	errUnsupportedSSEType          = errors.New("unsupported S3 SSE type")
 	errInvalidSSEContext           = errors.New("invalid S3 SSE encryption context")
+	errInvalidBucketLookupType     = errors.New("invalid bucket lookup type")
 )
 
 // HTTPConfig stores the http.Transport configuration for the s3 minio client.
@@ -59,6 +65,7 @@ type Config struct {
 	AccessKeyID      string         `yaml:"access_key_id"`
 	Insecure         bool           `yaml:"insecure"`
 	SignatureVersion string         `yaml:"signature_version"`
+	BucketLookupType string         `yaml:"bucket_lookup_type"`
 
 	SSE  SSEConfig  `yaml:"sse"`
 	HTTP HTTPConfig `yaml:"http"`
@@ -78,6 +85,7 @@ func (cfg *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) {
 	f.StringVar(&cfg.Endpoint, prefix+"s3.endpoint", "", "The S3 bucket endpoint. It could be an AWS S3 endpoint listed at https://docs.aws.amazon.com/general/latest/gr/s3.html or the address of an S3-compatible service in hostname:port format.")
 	f.BoolVar(&cfg.Insecure, prefix+"s3.insecure", false, "If enabled, use http:// for the S3 endpoint instead of https://. This could be useful in local dev/test environments while using an S3-compatible backend storage, like Minio.")
 	f.StringVar(&cfg.SignatureVersion, prefix+"s3.signature-version", SignatureVersionV4, fmt.Sprintf("The signature version to use for authenticating against S3. Supported values are: %s.", strings.Join(supportedSignatureVersions, ", ")))
+	f.StringVar(&cfg.BucketLookupType, prefix+"s3.bucket-lookup-type", BucketAutoLookup, fmt.Sprintf("The s3 bucket lookup style. Supported values are: %s.", strings.Join(supportedBucketLookupTypes, ", ")))
 	cfg.SSE.RegisterFlagsWithPrefix(prefix+"s3.sse.", f)
 	cfg.HTTP.RegisterFlagsWithPrefix(prefix, f)
 }
@@ -87,6 +95,9 @@ func (cfg *Config) Validate() error {
 	if !util.StringsContain(supportedSignatureVersions, cfg.SignatureVersion) {
 		return errUnsupportedSignatureVersion
 	}
+	if !util.StringsContain(supportedBucketLookupTypes, cfg.BucketLookupType) {
+		return errInvalidBucketLookupType
+	}
 
 	if err := cfg.SSE.Validate(); err != nil {
 		return err
@@ -95,6 +106,19 @@ func (cfg *Config) Validate() error {
 	return nil
 }
 
+func (cfg *Config) bucketLookupType() (s3.BucketLookupType, error) {
+	switch cfg.BucketLookupType {
+	case BucketVirtualHostLookup:
+		return s3.VirtualHostLookup, nil
+	case BucketPathLookup:
+		return s3.PathLookup, nil
+	case BucketAutoLookup:
+		return s3.AutoLookup, nil
+	default:
+		return s3.AutoLookup, errInvalidBucketLookupType
+	}
+}
+
 // SSEConfig configures S3 server side encryption
 // struct that is going to receive user input (through config file or CLI)
 type SSEConfig struct {

pkg/storage/bucket/s3/config_test.go

@@ -17,6 +17,7 @@ import (
 // defaultConfig should match the default flag values defined in RegisterFlagsWithPrefix.
 var defaultConfig = Config{
 	SignatureVersion: SignatureVersionV4,
+	BucketLookupType: BucketAutoLookup,
 	HTTP: HTTPConfig{
 		Config: bucket_http.Config{
 			IdleConnTimeout:       90 * time.Second,
@@ -53,6 +54,7 @@ secret_access_key: test-secret-access-key
 access_key_id: test-access-key-id
 insecure: true
 signature_version: test-signature-version
+bucket_lookup_type: virtual-hosted
 sse:
   type: test-type
   kms_key_id: test-kms-key-id
@@ -75,6 +77,7 @@ http:
 				AccessKeyID:      "test-access-key-id",
 				Insecure:         true,
 				SignatureVersion: "test-signature-version",
+				BucketLookupType: BucketVirtualHostLookup,
 				SSE: SSEConfig{
 					Type:                 "test-type",
 					KMSKeyID:             "test-kms-key-id",

pkg/storage/bucket/sse_bucket_client_test.go

@@ -48,12 +48,13 @@ func TestSSEBucketClient_Upload_ShouldInjectCustomSSEConfig(t *testing.T) {
 			defer srv.Close()
 
 			s3Cfg := s3.Config{
-				Endpoint:        srv.Listener.Addr().String(),
-				Region:          "test",
-				BucketName:      "test-bucket",
-				SecretAccessKey: flagext.Secret{Value: "test"},
-				AccessKeyID:     "test",
-				Insecure:        true,
+				Endpoint:         srv.Listener.Addr().String(),
+				Region:           "test",
+				BucketName:       "test-bucket",
+				SecretAccessKey:  flagext.Secret{Value: "test"},
+				AccessKeyID:      "test",
+				Insecure:         true,
+				BucketLookupType: s3.BucketPathLookup,
 			}
 
 			s3Client, err := s3.NewBucketClient(s3Cfg, "test", log.NewNopLogger())