Skip to content

Commit 44a5d25

Browse files
yeya24yeya24
authored
Add S3 send_content_md5 config and default to use MD5 (#5870)
* change s3 bucket client to put objects using md5 checksum algorithm Signed-off-by: Ben Ye <benye@amazon.com> * update docs Signed-off-by: Ben Ye <benye@amazon.com> * changelog Signed-off-by: Ben Ye <benye@amazon.com> * fix tests Signed-off-by: Ben Ye <benye@amazon.com> --------- Signed-off-by: Ben Ye <benye@amazon.com>
1 parent 00ffb3c commit 44a5d25

File tree

7 files changed

+49
-7
lines changed

7 files changed

+49
-7
lines changed

‎CHANGELOG.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
* [CHANGE] Querier: Mark `-querier.ingester-streaming` flag as deprecated. Now query ingester streaming is always enabled. #5817
1212
* [CHANGE] Compactor/Bucket Store: Added `-blocks-storage.bucket-store.block-discovery-strategy` to configure different block listing strategy. Reverted the current recursive block listing mechanism and use the strategy `Concurrent` as in 1.15. #5828
1313
* [CHANGE] Compactor: Don't halt compactor when overlapped source blocks detected. #5854
14+
* [CHANGE] S3 Bucket Client: Expose `-blocks-storage.s3.send-content-md5` flag and set default checksum algorithm to MD5. #5870
1415
* [FEATURE] OTLP ingestion experimental. #5813
1516
* [FEATURE] Ingester: Add per-tenant new metric `cortex_ingester_tsdb_data_replay_duration_seconds`. #5477
1617
* [FEATURE] Query Frontend/Scheduler: Add query priority support. #5605

‎docs/blocks-storage/querier.md

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -288,6 +288,12 @@ blocks_storage:
288288
# CLI flag: -blocks-storage.s3.bucket-lookup-type
289289
[bucket_lookup_type: <string> | default = "auto"]
290290
291+
# If true, attach MD5 checksum when upload objects and S3 uses MD5 checksum
292+
# algorithm to verify the provided digest. If false, use CRC32C algorithm
293+
# instead.
294+
# CLI flag: -blocks-storage.s3.send-content-md5
295+
[send_content_md5: <boolean> | default = true]
296+
291297
# The s3_sse_config configures the S3 server-side encryption.
292298
# The CLI flags prefix for this block config is: blocks-storage
293299
[sse: <s3_sse_config>]

‎docs/blocks-storage/store-gateway.md

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -397,6 +397,12 @@ blocks_storage:
397397
# CLI flag: -blocks-storage.s3.bucket-lookup-type
398398
[bucket_lookup_type: <string> | default = "auto"]
399399
400+
# If true, attach MD5 checksum when upload objects and S3 uses MD5 checksum
401+
# algorithm to verify the provided digest. If false, use CRC32C algorithm
402+
# instead.
403+
# CLI flag: -blocks-storage.s3.send-content-md5
404+
[send_content_md5: <boolean> | default = true]
405+
400406
# The s3_sse_config configures the S3 server-side encryption.
401407
# The CLI flags prefix for this block config is: blocks-storage
402408
[sse: <s3_sse_config>]

‎docs/configuration/config-file-reference.md

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -559,6 +559,12 @@ s3:
559559
# CLI flag: -alertmanager-storage.s3.bucket-lookup-type
560560
[bucket_lookup_type: <string> | default = "auto"]
561561
562+
# If true, attach MD5 checksum when upload objects and S3 uses MD5 checksum
563+
# algorithm to verify the provided digest. If false, use CRC32C algorithm
564+
# instead.
565+
# CLI flag: -alertmanager-storage.s3.send-content-md5
566+
[send_content_md5: <boolean> | default = true]
567+
562568
# The s3_sse_config configures the S3 server-side encryption.
563569
# The CLI flags prefix for this block config is: alertmanager-storage
564570
[sse: <s3_sse_config>]
@@ -832,6 +838,12 @@ s3:
832838
# CLI flag: -blocks-storage.s3.bucket-lookup-type
833839
[bucket_lookup_type: <string> | default = "auto"]
834840
841+
# If true, attach MD5 checksum when upload objects and S3 uses MD5 checksum
842+
# algorithm to verify the provided digest. If false, use CRC32C algorithm
843+
# instead.
844+
# CLI flag: -blocks-storage.s3.send-content-md5
845+
[send_content_md5: <boolean> | default = true]
846+
835847
# The s3_sse_config configures the S3 server-side encryption.
836848
# The CLI flags prefix for this block config is: blocks-storage
837849
[sse: <s3_sse_config>]
@@ -4326,6 +4338,12 @@ s3:
43264338
# CLI flag: -ruler-storage.s3.bucket-lookup-type
43274339
[bucket_lookup_type: <string> | default = "auto"]
43284340
4341+
# If true, attach MD5 checksum when upload objects and S3 uses MD5 checksum
4342+
# algorithm to verify the provided digest. If false, use CRC32C algorithm
4343+
# instead.
4344+
# CLI flag: -ruler-storage.s3.send-content-md5
4345+
[send_content_md5: <boolean> | default = true]
4346+
43294347
# The s3_sse_config configures the S3 server-side encryption.
43304348
# The CLI flags prefix for this block config is: ruler-storage
43314349
[sse: <s3_sse_config>]
@@ -4607,6 +4625,12 @@ s3:
46074625
# CLI flag: -runtime-config.s3.bucket-lookup-type
46084626
[bucket_lookup_type: <string> | default = "auto"]
46094627
4628+
# If true, attach MD5 checksum when upload objects and S3 uses MD5 checksum
4629+
# algorithm to verify the provided digest. If false, use CRC32C algorithm
4630+
# instead.
4631+
# CLI flag: -runtime-config.s3.send-content-md5
4632+
[send_content_md5: <boolean> | default = true]
4633+
46104634
# The s3_sse_config configures the S3 server-side encryption.
46114635
# The CLI flags prefix for this block config is: runtime-config
46124636
[sse: <s3_sse_config>]

‎pkg/storage/bucket/s3/bucket_client.go

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -81,13 +81,14 @@ func newS3Config(cfg Config) (s3.Config, error) {
8181
}
8282

8383
return s3.Config{
84-
Bucket: cfg.BucketName,
85-
Endpoint: cfg.Endpoint,
86-
Region: cfg.Region,
87-
AccessKey: cfg.AccessKeyID,
88-
SecretKey: cfg.SecretAccessKey.Value,
89-
Insecure: cfg.Insecure,
90-
SSEConfig: sseCfg,
84+
Bucket: cfg.BucketName,
85+
Endpoint: cfg.Endpoint,
86+
Region: cfg.Region,
87+
AccessKey: cfg.AccessKeyID,
88+
SecretKey: cfg.SecretAccessKey.Value,
89+
Insecure: cfg.Insecure,
90+
SSEConfig: sseCfg,
91+
SendContentMd5: cfg.SendContentMd5,
9192
HTTPConfig: s3.HTTPConfig{
9293
IdleConnTimeout: model.Duration(cfg.HTTP.IdleConnTimeout),
9394
ResponseHeaderTimeout: model.Duration(cfg.HTTP.ResponseHeaderTimeout),

‎pkg/storage/bucket/s3/config.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,7 @@ type Config struct {
6666
Insecure bool `yaml:"insecure"`
6767
SignatureVersion string `yaml:"signature_version"`
6868
BucketLookupType string `yaml:"bucket_lookup_type"`
69+
SendContentMd5 bool `yaml:"send_content_md5"`
6970

7071
SSE SSEConfig `yaml:"sse"`
7172
HTTP HTTPConfig `yaml:"http"`
@@ -86,6 +87,7 @@ func (cfg *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) {
8687
f.BoolVar(&cfg.Insecure, prefix+"s3.insecure", false, "If enabled, use http:// for the S3 endpoint instead of https://. This could be useful in local dev/test environments while using an S3-compatible backend storage, like Minio.")
8788
f.StringVar(&cfg.SignatureVersion, prefix+"s3.signature-version", SignatureVersionV4, fmt.Sprintf("The signature version to use for authenticating against S3. Supported values are: %s.", strings.Join(supportedSignatureVersions, ", ")))
8889
f.StringVar(&cfg.BucketLookupType, prefix+"s3.bucket-lookup-type", BucketAutoLookup, fmt.Sprintf("The s3 bucket lookup style. Supported values are: %s.", strings.Join(supportedBucketLookupTypes, ", ")))
90+
f.BoolVar(&cfg.SendContentMd5, prefix+"s3.send-content-md5", true, "If true, attach MD5 checksum when upload objects and S3 uses MD5 checksum algorithm to verify the provided digest. If false, use CRC32C algorithm instead.")
8991
cfg.SSE.RegisterFlagsWithPrefix(prefix+"s3.sse.", f)
9092
cfg.HTTP.RegisterFlagsWithPrefix(prefix, f)
9193
}

‎pkg/storage/bucket/s3/config_test.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import (
1818
var defaultConfig = Config{
1919
SignatureVersion: SignatureVersionV4,
2020
BucketLookupType: BucketAutoLookup,
21+
SendContentMd5: true,
2122
HTTP: HTTPConfig{
2223
Config: bucket_http.Config{
2324
IdleConnTimeout: 90 * time.Second,
@@ -78,6 +79,7 @@ http:
7879
Insecure: true,
7980
SignatureVersion: "test-signature-version",
8081
BucketLookupType: BucketVirtualHostLookup,
82+
SendContentMd5: true,
8183
SSE: SSEConfig{
8284
Type: "test-type",
8385
KMSKeyID: "test-kms-key-id",

0 commit comments

Comments
 (0)