Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,12 @@ version numbers.
for S3 compatible providers. The default behaviour is to require checksum
verification.

* `checksum_algorithm`: *Optional.* Specifies the checksum algorithm to use
when uploading objects to S3. Valid values are `CRC32`, `CRC32C`, `SHA1`,
`SHA256`, or `CRC64NVME`. If not specified, S3 will use its default algorithm.
This setting is ignored if `skip_s3_checksums` is set to `true`. Note that
not all S3-compatible providers support all algorithms.

### File Names

One of the following two options must be specified:
Expand Down
1 change: 1 addition & 0 deletions cmd/check/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ func main() {
request.Source.DisableSSL,
request.Source.UsePathStyle,
request.Source.SkipS3Checksums,
request.Source.ChecksumAlgorithm,
)
if err != nil {
s3resource.Fatal("error creating s3 client", err)
Expand Down
1 change: 1 addition & 0 deletions cmd/in/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ func main() {
request.Source.DisableSSL,
request.Source.UsePathStyle,
request.Source.SkipS3Checksums,
request.Source.ChecksumAlgorithm,
)
if err != nil {
s3resource.Fatal("error creating s3 client", err)
Expand Down
1 change: 1 addition & 0 deletions cmd/out/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ func main() {
request.Source.DisableSSL,
request.Source.UsePathStyle,
request.Source.SkipS3Checksums,
request.Source.ChecksumAlgorithm,
)
if err != nil {
s3resource.Fatal("error creating s3 client", err)
Expand Down
2 changes: 2 additions & 0 deletions integration/integration_suite_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,7 @@ func getSessionTokenS3Client(awsConfig *aws.Config) (*s3.Client, s3resource.S3Cl
false,
pathStyle,
skipS3Checksums,
"",
)
Ω(err).ShouldNot(HaveOccurred())

Expand Down Expand Up @@ -145,6 +146,7 @@ var _ = SynchronizedBeforeSuite(func() []byte {
false,
pathStyle,
skipS3Checksums,
"",
)
Ω(err).ShouldNot(HaveOccurred())
}
Expand Down
19 changes: 19 additions & 0 deletions models.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ type Source struct {
DisableMultipart bool `json:"disable_multipart"`
UsePathStyle bool `json:"use_path_style"`
SkipS3Checksums bool `json:"skip_s3_checksums"`
ChecksumAlgorithm string `json:"checksum_algorithm"`
}

func (source Source) IsValid() (bool, string) {
Expand Down Expand Up @@ -56,6 +57,24 @@ func (source Source) IsValid() (bool, string) {
return false, "please specify initial_version or initial_path if initial content is set"
}

// Validate checksum algorithm if specified
if source.ChecksumAlgorithm != "" {
validAlgorithms := map[string]bool{
"CRC32": true,
"CRC32C": true,
"SHA1": true,
"SHA256": true,
"CRC64NVME": true,
}
if !validAlgorithms[source.ChecksumAlgorithm] {
return false, "checksum_algorithm must be one of: CRC32, CRC32C, SHA1, SHA256, CRC64NVME"
}

if source.SkipS3Checksums {
return false, "checksum_algorithm cannot be used when skip_s3_checksums is true"
}
}

return true, ""
}

Expand Down
8 changes: 8 additions & 0 deletions s3client.go
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@ type UploadFileOptions struct {
KmsKeyId string
ContentType string
DisableMultipart bool
ChecksumAlgorithm string
}

func NewUploadFileOptions() UploadFileOptions {
Expand All @@ -75,6 +76,7 @@ func NewS3Client(
awsConfig *aws.Config,
endpoint string,
disableSSL, usePathStyle, skipS3Checksums bool,
checksumAlgorithm string,
) (S3Client, error) {
s3Opts := []func(*s3.Options){}

Expand All @@ -100,6 +102,9 @@ func NewS3Client(
o.RequestChecksumCalculation = aws.RequestChecksumCalculationWhenRequired
o.ResponseChecksumValidation = aws.ResponseChecksumValidationWhenRequired
}
if checksumAlgorithm != "" {
o.RequestChecksumCalculation = aws.RequestChecksumCalculationWhenSupported
}
})
}

Expand Down Expand Up @@ -342,6 +347,9 @@ func (client *s3client) UploadFile(bucketName string, remotePath string, localPa
if options.ContentType != "" {
uploadInput.ContentType = aws.String(options.ContentType)
}
if options.ChecksumAlgorithm != "" {
uploadInput.ChecksumAlgorithm = types.ChecksumAlgorithm(options.ChecksumAlgorithm)
}

uploadOutput, err := uploader.Upload(context.TODO(), uploadInput)
if err != nil {
Expand Down