s3: storage class for files in result storage bucket (#402)

* s3: storage class for files in result storage bucket

* s3: review fixes. add missed storage class

* s3: review fixes, add tests for s3storageclass

Author: f100024

config/awsconfig/awsconfig.go

@@ -2,6 +2,7 @@ package awsconfig
 
 import (
 	"flag"
+
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/credentials"
 	"github.com/aws/aws-sdk-go/aws/session"
@@ -90,6 +91,8 @@ func WithAWS(fs *flag.FlagSet, cb func() (*zap.Logger, bool)) imagor.Option {
 			"Upload ACL for S3 Result Storage")
 		s3ResultStorageExpiration = fs.Duration("s3-result-storage-expiration", 0,
 			"S3 Result Storage expiration duration e.g. 24h. Default no expiration")
+		s3StorageClass = fs.String("s3-storage-class", "STANDARD",
+			"S3 File Storage Class. Available values: REDUCED_REDUNDANCY, STANDARD_IA, ONEZONE_IA, INTELLIGENT_TIERING, GLACIER, DEEP_ARCHIVE. Default: STANDARD.")
 
 		_, _ = cb()
 	)
@@ -160,6 +163,7 @@ func WithAWS(fs *flag.FlagSet, cb func() (*zap.Logger, bool)) imagor.Option {
 					s3storage.WithACL(*s3StorageACL),
 					s3storage.WithSafeChars(*s3SafeChars),
 					s3storage.WithExpiration(*s3StorageExpiration),
+					s3storage.WithStorageClass(*s3StorageClass),
 				),
 			)
 		}
@@ -182,6 +186,7 @@ func WithAWS(fs *flag.FlagSet, cb func() (*zap.Logger, bool)) imagor.Option {
 					s3storage.WithACL(*s3ResultStorageACL),
 					s3storage.WithSafeChars(*s3SafeChars),
 					s3storage.WithExpiration(*s3ResultStorageExpiration),
+					s3storage.WithStorageClass(*s3StorageClass),
 				),
 			)
 		}

config/awsconfig/awsconfig_test.go

@@ -1,11 +1,12 @@
 package awsconfig
 
 import (
+	"testing"
+
 	"github.com/cshum/imagor"
 	"github.com/cshum/imagor/config"
 	"github.com/cshum/imagor/storage/s3storage"
 	"github.com/stretchr/testify/assert"
-	"testing"
 )
 
 func TestS3Empty(t *testing.T) {
@@ -120,3 +121,44 @@ func TestS3SessionOverride(t *testing.T) {
 	assert.Equal(t, "/bcda/", resultStorage.PathPrefix)
 	assert.Equal(t, "!", resultStorage.SafeChars)
 }
+
+func TestS3StorageClassWithResultStorageBucket(t *testing.T) {
+	srv := config.CreateServer([]string{
+		"-s3-storage-class", "asdf",
+		"-s3-storage-bucket", "a",
+		"-s3-result-storage-bucket", "b",
+	}, WithAWS)
+	app := srv.App.(*imagor.Imagor)
+	storage := app.Storages[0].(*s3storage.S3Storage)
+	assert.Equal(t, "STANDARD", storage.StorageClass)
+
+}
+
+func TestS3StorageClassWithoutResultStorageBucket(t *testing.T) {
+	srv := config.CreateServer([]string{
+		"-s3-storage-class", "asdf",
+		"-s3-storage-bucket", "a",
+	}, WithAWS)
+	app := srv.App.(*imagor.Imagor)
+	storage := app.Storages[0].(*s3storage.S3Storage)
+	assert.Equal(t, "STANDARD", storage.StorageClass)
+
+}
+
+func TestS3StorageClass(t *testing.T) {
+	srv := config.CreateServer([]string{
+		"-s3-storage-class", "asdf",
+		"-s3-storage-bucket", "a",
+	}, WithAWS)
+	app := srv.App.(*imagor.Imagor)
+	storage := app.Storages[0].(*s3storage.S3Storage)
+	assert.Equal(t, "STANDARD", storage.StorageClass)
+
+	srv = config.CreateServer([]string{
+		"-s3-storage-class", "REDUCED_REDUNDANCY",
+		"-s3-storage-bucket", "a",
+	}, WithAWS)
+	app = srv.App.(*imagor.Imagor)
+	storage = app.Storages[0].(*s3storage.S3Storage)
+	assert.Equal(t, "REDUCED_REDUNDANCY", storage.StorageClass)
+}

storage/s3storage/option.go

@@ -1,9 +1,10 @@
 package s3storage
 
 import (
-	"github.com/aws/aws-sdk-go/service/s3"
 	"strings"
 	"time"
+
+	"github.com/aws/aws-sdk-go/service/s3"
 )
 
 // Option S3Storage option
@@ -70,3 +71,18 @@ func WithExpiration(exp time.Duration) Option {
 		}
 	}
 }
+
+// WithFileStorageClass with storage storage class option
+func WithStorageClass(storageClass string) Option {
+	return func(h *S3Storage) {
+		allowedStorageClasses := [6]string{"REDUCED_REDUNDANCY", "STANDARD_IA", "ONEZONE_IA",
+			"INTELLIGENT_TIERING", "GLACIER", "DEEP_ARCHIVE"}
+		h.StorageClass = "STANDARD"
+		for _, allowedStorageClass := range allowedStorageClasses {
+			if storageClass == allowedStorageClass {
+				h.StorageClass = storageClass
+				break
+			}
+		}
+	}
+}

storage/s3storage/s3storage.go

@@ -2,19 +2,20 @@ package s3storage
 
 import (
 	"context"
+	"io"
+	"net/http"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/awserr"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/aws/aws-sdk-go/service/s3/s3manager"
 	"github.com/cshum/imagor"
 	"github.com/cshum/imagor/imagorpath"
-	"io"
-	"net/http"
-	"path/filepath"
-	"strings"
-	"sync"
-	"time"
 )
 
 // S3Storage AWS S3 Storage implements imagor.Storage interface
@@ -24,11 +25,12 @@ type S3Storage struct {
 	Downloader *s3manager.Downloader
 	Bucket     string
 
-	BaseDir    string
-	PathPrefix string
-	ACL        string
-	SafeChars  string
-	Expiration time.Duration
+	BaseDir      string
+	PathPrefix   string
+	ACL          string
+	SafeChars    string
+	StorageClass string
+	Expiration   time.Duration
 
 	safeChars imagorpath.SafeChars
 }
@@ -128,12 +130,13 @@ func (s *S3Storage) Put(ctx context.Context, image string, blob *imagor.Blob) er
 	}()
 	var metadata map[string]*string
 	input := &s3manager.UploadInput{
-		ACL:         aws.String(s.ACL),
-		Body:        reader,
-		Bucket:      aws.String(s.Bucket),
-		ContentType: aws.String(blob.ContentType()),
-		Metadata:    metadata,
-		Key:         aws.String(image),
+		ACL:          aws.String(s.ACL),
+		Body:         reader,
+		Bucket:       aws.String(s.Bucket),
+		ContentType:  aws.String(blob.ContentType()),
+		Metadata:     metadata,
+		Key:          aws.String(image),
+		StorageClass: aws.String(s.StorageClass),
 	}
 	_, err = s.Uploader.UploadWithContext(ctx, input)
 	return err