Add support for OCS S3 snapstore

.gitignore

@@ -25,6 +25,7 @@ default.etcd*
 
 # IDE config
 .vscode
+.idea/
 
 # developers workspace
 tmp

pkg/snapstore/ecs_s3_snapstore.go

@@ -14,19 +14,8 @@
 
 package snapstore
 
-import (
-	"crypto/tls"
-	"fmt"
-	"net/http"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/aws/credentials"
-	"github.com/aws/aws-sdk-go/aws/session"
-	"github.com/aws/aws-sdk-go/service/s3"
-)
-
 const (
-	// ECS does not support regions and always returns the default region: US-Standard.
+	// ECS does not support regions and always uses the default region US-Standard.
 	ecsDefaultRegion             string = "US-Standard"
 	ecsDefaultDisableSSL         bool   = false
 	ecsDefaultInsecureSkipVerify bool   = false
@@ -38,62 +27,28 @@ const (
 	ecsSecretAccessKey    string = "ECS_SECRET_ACCESS_KEY"
 )
 
-// A ecsAuthOptions groups all the required options to authenticate again ECS
-type ecsAuthOptions struct {
-	endpoint           string
-	region             string
-	disableSSL         bool
-	insecureSkipVerify bool
-	accessKeyID        string
-	secretAccessKey    string
-}
-
-// NewECSSnapStore create new S3SnapStore from shared configuration with specified bucket
+// NewECSSnapStore creates a new S3SnapStore from shared configuration with the specified bucket.
 func NewECSSnapStore(bucket, prefix, tempDir string, maxParallelChunkUploads uint) (*S3SnapStore, error) {
 	ao, err := ecsAuthOptionsFromEnv()
 	if err != nil {
 		return nil, err
 	}
-	return newECSFromAuthOpt(bucket, prefix, tempDir, maxParallelChunkUploads, ao)
-}
-
-// newECSFromAuthOpt will create the new S3 snapstore object from ECS S3 authentication options
-func newECSFromAuthOpt(bucket, prefix, tempDir string, maxParallelChunkUploads uint, ao ecsAuthOptions) (*S3SnapStore, error) {
-	httpClient := http.DefaultClient
-	if !ao.disableSSL {
-		httpClient.Transport = &http.Transport{
-			TLSClientConfig: &tls.Config{InsecureSkipVerify: ao.insecureSkipVerify},
-		}
-	}
-
-	sess, err := session.NewSession(&aws.Config{
-		Credentials:      credentials.NewStaticCredentials(ao.accessKeyID, ao.secretAccessKey, ""),
-		Endpoint:         aws.String(ao.endpoint),
-		Region:           aws.String(ecsDefaultRegion),
-		DisableSSL:       aws.Bool(ao.disableSSL),
-		S3ForcePathStyle: aws.Bool(true),
-		HTTPClient:       httpClient,
-	})
-	if err != nil {
-		return nil, fmt.Errorf("new ECS S3 session failed: %v", err)
-	}
-	cli := s3.New(sess)
-	return NewS3FromClient(bucket, prefix, tempDir, maxParallelChunkUploads, cli), nil
+	return newGenericS3FromAuthOpt(bucket, prefix, tempDir, maxParallelChunkUploads, ao)
 }
 
-// ecsAuthOptionsFromEnv will get provider configuration from environment variables
-func ecsAuthOptionsFromEnv() (ecsAuthOptions, error) {
+// ecsAuthOptionsFromEnv gets ECS provider configuration from environment variables.
+func ecsAuthOptionsFromEnv() (s3AuthOptions, error) {
 	endpoint, err := GetEnvVarOrError(ecsEndPoint)
 	if err != nil {
-		return ecsAuthOptions{}, err
+		return s3AuthOptions{}, err
 	}
 	accessKeyID, err := GetEnvVarOrError(ecsAccessKeyID)
 	if err != nil {
-		return ecsAuthOptions{}, err
+		return s3AuthOptions{}, err
 	}
 	secretAccessKey, err := GetEnvVarOrError(ecsSecretAccessKey)
 	if err != nil {
-		return ecsAuthOptions{}, err
+		return s3AuthOptions{}, err
 	}
 	disableSSL, err := GetEnvVarToBool(ecsDisableSSL)
 	if err != nil {
@@ -104,8 +59,9 @@ func ecsAuthOptionsFromEnv() (ecsAuthOptions, error) {
 		insecureSkipVerify = ecsDefaultInsecureSkipVerify
 	}
 
-	ao := ecsAuthOptions{
+	ao := s3AuthOptions{
 		endpoint:           endpoint,
+		region:             ecsDefaultRegion,
 		disableSSL:         disableSSL,
 		insecureSkipVerify: insecureSkipVerify,
 		accessKeyID:        accessKeyID,

pkg/snapstore/generic_s3_snapstore.go

@@ -0,0 +1,60 @@
+// Copyright (c) 2020 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package snapstore
+
+import (
+	"crypto/tls"
+	"fmt"
+	"net/http"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+)
+
+// s3AuthOptions contains all needed options to authenticate against a S3-compatible store.
+type s3AuthOptions struct {
+	endpoint           string
+	region             string
+	disableSSL         bool
+	insecureSkipVerify bool
+	accessKeyID        string
+	secretAccessKey    string
+}
+
+// newGenericS3FromAuthOpt creates a new S3 snapstore object from the specified authentication options.
+func newGenericS3FromAuthOpt(bucket, prefix, tempDir string, maxParallelChunkUploads uint, ao s3AuthOptions) (*S3SnapStore, error) {
+	httpClient := http.DefaultClient
+	if !ao.disableSSL {
+		httpClient.Transport = &http.Transport{
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: ao.insecureSkipVerify},
+		}
+	}
+
+	sess, err := session.NewSession(&aws.Config{
+		Credentials:      credentials.NewStaticCredentials(ao.accessKeyID, ao.secretAccessKey, ""),
+		Endpoint:         aws.String(ao.endpoint),
+		Region:           aws.String(ao.region),
+		DisableSSL:       aws.Bool(ao.disableSSL),
+		S3ForcePathStyle: aws.Bool(true),
+		HTTPClient:       httpClient,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("could not create S3 session: %v", err)
+	}
+	cli := s3.New(sess)
+	return NewS3FromClient(bucket, prefix, tempDir, maxParallelChunkUploads, cli), nil
+}

pkg/snapstore/ocs_s3_snapstore.go

@@ -0,0 +1,75 @@
+// Copyright (c) 2020 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package snapstore
+
+const (
+	ocsDefaultDisableSSL         bool = false
+	ocsDefaultInsecureSkipVerify bool = false
+
+	ocsEndpoint           string = "OCS_ENDPOINT"
+	ocsRegion             string = "OCS_REGION"
+	ocsDisableSSL         string = "OCS_DISABLE_SSL"
+	ocsInsecureSkipVerify string = "OCS_INSECURE_SKIP_VERIFY"
+	ocsAccessKeyID        string = "OCS_ACCESS_KEY_ID"
+	ocsSecretAccessKey    string = "OCS_SECRET_ACCESS_KEY"
+)
+
+// NewOCSSnapStore creates a new S3SnapStore from shared configuration with the specified bucket.
+func NewOCSSnapStore(bucket, prefix, tempDir string, maxParallelChunkUploads uint) (*S3SnapStore, error) {
+	ao, err := ocsAuthOptionsFromEnv()
+	if err != nil {
+		return nil, err
+	}
+	return newGenericS3FromAuthOpt(bucket, prefix, tempDir, maxParallelChunkUploads, ao)
+}
+
+// ocsAuthOptionsFromEnv gets OCS provider configuration from environment variables.
+func ocsAuthOptionsFromEnv() (s3AuthOptions, error) {
+	endpoint, err := GetEnvVarOrError(ocsEndpoint)
+	if err != nil {
+		return s3AuthOptions{}, err
+	}
+	accessKeyID, err := GetEnvVarOrError(ocsAccessKeyID)
+	if err != nil {
+		return s3AuthOptions{}, err
+	}
+	secretAccessKey, err := GetEnvVarOrError(ocsSecretAccessKey)
+	if err != nil {
+		return s3AuthOptions{}, err
+	}
+	region, err := GetEnvVarOrError(ocsRegion)
+	if err != nil {
+		return s3AuthOptions{}, err
+	}
+	disableSSL, err := GetEnvVarToBool(ocsDisableSSL)
+	if err != nil {
+		disableSSL = ocsDefaultDisableSSL
+	}
+	insecureSkipVerify, err := GetEnvVarToBool(ocsInsecureSkipVerify)
+	if err != nil {
+		insecureSkipVerify = ocsDefaultInsecureSkipVerify
+	}
+
+	ao := s3AuthOptions{
+		endpoint:           endpoint,
+		region:             region,
+		disableSSL:         disableSSL,
+		insecureSkipVerify: insecureSkipVerify,
+		accessKeyID:        accessKeyID,
+		secretAccessKey:    secretAccessKey,
+	}
+
+	return ao, nil
+}

pkg/snapstore/snapstore_test.go

@@ -99,6 +99,11 @@ var _ = Describe("Snapstore", func() {
 				prefix:           prefix,
 				multiPartUploads: map[string]*[][]byte{},
 			}),
+			"OCS": NewS3FromClient(bucket, prefix, "/tmp", 5, &mockS3Client{
+				objects:          objectMap,
+				prefix:           prefix,
+				multiPartUploads: map[string]*[][]byte{},
+			}),
 		}
 	})
 

pkg/snapstore/types.go

@@ -51,6 +51,8 @@ const (
 	SnapstoreProviderOSS = "OSS"
 	// SnapstoreProviderECS is constant for Dell EMC ECS S3 storage provider.
 	SnapstoreProviderECS = "ECS"
+	// SnapstoreProviderOCS is constant for OpenShift Container Storage S3 storage provider.
+	SnapstoreProviderOCS = "OCS"
 	// SnapstoreProviderFakeFailed is constant for fake failed storage provider.
 	SnapstoreProviderFakeFailed = "FAILED"
 

pkg/snapstore/utils.go

@@ -88,6 +88,11 @@ func GetSnapstore(config *Config) (SnapStore, error) {
 			return nil, fmt.Errorf("storage container name not specified")
 		}
 		return NewECSSnapStore(config.Container, config.Prefix, config.TempDir, config.MaxParallelChunkUploads)
+	case SnapstoreProviderOCS:
+		if config.Container == "" {
+			return nil, fmt.Errorf("storage container name not specified")
+		}
+		return NewOCSSnapStore(config.Container, config.Prefix, config.TempDir, config.MaxParallelChunkUploads)
 	case SnapstoreProviderFakeFailed:
 		return NewFailedSnapStore(), nil
 	default: