Skip to content

Minio S3 backend (#4) #7

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Jul 20, 2022
Merged

Minio S3 backend (#4) #7

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
58909a6
use minio SDK for S3 backend per #4
ahouene Jul 6, 2022
7b59c3b
automatic version definition for minio client
ahouene Jul 13, 2022
c340764
fix RemoveObject not being checked for tests
ahouene Jul 13, 2022
a8ecf89
keep previous configuration format, and adapt it to how minio operates
ahouene Jul 20, 2022
2105f82
use lighter io.ReadAll instead of bytes.Buffer for Load
ahouene Jul 20, 2022
c5c8551
Merge branch 'main' into minio-s3-backend
ahouene Jul 20, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
180 changes: 93 additions & 87 deletions backends/s3/s3.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,26 +3,27 @@ package s3
import (
"bytes"
"context"
"errors"
"fmt"
"net/http"
"io"
"net/url"
"os"
"runtime/debug"
"sort"
"strings"
"sync"
"time"

"github.com/PowerDNS/go-tlsconfig"
"github.com/aws/aws-sdk-go-v2/aws"
awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http"
s3config "github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
"github.com/aws/aws-sdk-go-v2/service/s3"
minio "github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"

"github.com/PowerDNS/simpleblob"
)

const (
// DefaultEndpointURL is the default S3 endpoint to use if none is set.
// Here, no custom endpoint assumes AWS endpoint.
DefaultEndpointURL = "s3.amazonaws.com"
// DefaultRegion is the default S3 region to use, if none is configured
DefaultRegion = "us-east-1"
// DefaultInitTimeout is the time we allow for initialisation, like credential
Expand All @@ -49,7 +50,7 @@ type Options struct {
CreateBucket bool `yaml:"create_bucket"`

// EndpointURL can be set to something like "http://localhost:9000" when using Minio
// instead of AWS S3.
// or "s3.amazonaws.com" for AWS S3.
EndpointURL string `yaml:"endpoint_url"`

// TLS allows customising the TLS configuration
Expand Down Expand Up @@ -92,9 +93,9 @@ func (o Options) Check() error {
}

type Backend struct {
opt Options
s3config aws.Config
client *s3.Client
opt Options
config *minio.Options
client *minio.Client

mu sync.Mutex
lastMarker string
Expand Down Expand Up @@ -143,32 +144,17 @@ func (b *Backend) List(ctx context.Context, prefix string) (simpleblob.BlobList,
func (b *Backend) doList(ctx context.Context, prefix string) (simpleblob.BlobList, error) {
var blobs simpleblob.BlobList

paginator := s3.NewListObjectsV2Paginator(b.client, &s3.ListObjectsV2Input{
Bucket: aws.String(b.opt.Bucket),
Prefix: aws.String(prefix),
Delimiter: aws.String("/"),
objCh := b.client.ListObjects(ctx, b.opt.Bucket, minio.ListObjectsOptions{
Prefix: prefix,
Recursive: false,
})

for paginator.HasMorePages() {
for obj := range objCh {
metricCalls.WithLabelValues("list").Inc()
metricLastCallTimestamp.WithLabelValues("list").SetToCurrentTime()
page, err := paginator.NextPage(ctx)
if err != nil {
metricCallErrors.WithLabelValues("list").Inc()
return nil, err
}
for _, obj := range page.Contents {
if obj.Key == nil {
continue // just in case, should not happen
}
if *obj.Key == UpdateMarkerFilename {
continue
}
blobs = append(blobs, simpleblob.Blob{
Name: *obj.Key,
Size: obj.Size,
})
if obj.Key == UpdateMarkerFilename {
continue
}
blobs = append(blobs, simpleblob.Blob{Name: obj.Key, Size: obj.Size})
}

// Minio appears to return them sorted, but maybe not all implementations
Expand All @@ -179,22 +165,23 @@ func (b *Backend) doList(ctx context.Context, prefix string) (simpleblob.BlobLis
}

func (b *Backend) Load(ctx context.Context, name string) ([]byte, error) {
buf := manager.NewWriteAtBuffer(nil)
downloader := manager.NewDownloader(b.client)
metricCalls.WithLabelValues("load").Inc()
metricLastCallTimestamp.WithLabelValues("load").SetToCurrentTime()
_, err := downloader.Download(ctx, buf, &s3.GetObjectInput{
Bucket: aws.String(b.opt.Bucket),
Key: aws.String(name),
})

obj, err := b.client.GetObject(ctx, b.opt.Bucket, name, minio.GetObjectOptions{})
if err != nil {
metricCallErrors.WithLabelValues("load").Inc()
if isResponseError(err, http.StatusNotFound) {
return nil, os.ErrNotExist
if err = handleErrorResponse(err); err != nil {
return nil, err
}
} else if obj == nil {
return nil, os.ErrNotExist
}

p, err := io.ReadAll(obj)
if err = handleErrorResponse(err); err != nil {
return nil, err
}
return buf.Bytes(), nil
return p, nil
}

func (b *Backend) Store(ctx context.Context, name string, data []byte) error {
Expand All @@ -212,11 +199,9 @@ func (b *Backend) Store(ctx context.Context, name string, data []byte) error {
func (b *Backend) doStore(ctx context.Context, name string, data []byte) error {
metricCalls.WithLabelValues("store").Inc()
metricLastCallTimestamp.WithLabelValues("store").SetToCurrentTime()
uploader := manager.NewUploader(b.client)
_, err := uploader.Upload(ctx, &s3.PutObjectInput{
Bucket: aws.String(b.opt.Bucket),
Key: aws.String(name),
Body: bytes.NewReader(data),

_, err := b.client.PutObject(ctx, b.opt.Bucket, name, bytes.NewReader(data), int64(len(data)), minio.PutObjectOptions{
NumThreads: 3,
})
if err != nil {
metricCallErrors.WithLabelValues("store").Inc()
Expand All @@ -228,23 +213,13 @@ func (b *Backend) Delete(ctx context.Context, name string) error {
metricCalls.WithLabelValues("delete").Inc()
metricLastCallTimestamp.WithLabelValues("delete").SetToCurrentTime()

bu, k := aws.String(b.opt.Bucket), aws.String(name)

_, err := b.client.DeleteObject(ctx, &s3.DeleteObjectInput{Bucket: bu, Key: k})
if err != nil {
err := b.client.RemoveObject(ctx, b.opt.Bucket, name, minio.RemoveObjectOptions{})
if err = handleErrorResponse(err); err != nil {
metricCallErrors.WithLabelValues("delete").Inc()
}
return err
}

func isResponseError(err error, statusCode int) bool {
var responseError *awshttp.ResponseError
if !errors.As(err, &responseError) {
return false
}
return responseError.ResponseError.HTTPStatusCode() == statusCode
}

func New(ctx context.Context, opt Options) (*Backend, error) {
if opt.Region == "" {
opt.Region = DefaultRegion
Expand All @@ -255,6 +230,9 @@ func New(ctx context.Context, opt Options) (*Backend, error) {
if opt.UpdateMarkerForceListInterval == 0 {
opt.UpdateMarkerForceListInterval = DefaultUpdateMarkerForceListInterval
}
if opt.EndpointURL == "" {
opt.EndpointURL = DefaultEndpointURL
}
if err := opt.Check(); err != nil {
return nil, err
}
Expand Down Expand Up @@ -286,52 +264,80 @@ func New(ctx context.Context, opt Options) (*Backend, error) {
ctx, cancel := context.WithTimeout(ctx, opt.InitTimeout)
defer cancel()

creds := credentials.NewStaticCredentialsProvider(opt.AccessKey, opt.SecretKey, "")
cfg, err := s3config.LoadDefaultConfig(
ctx,
s3config.WithCredentialsProvider(creds),
s3config.WithRegion(opt.Region),
s3config.WithHTTPClient(hc))
// Minio takes only the Host value as the endpoint URL.
// The connection being secure depends on a key in minio.Option.
u, err := url.Parse(opt.EndpointURL)
if err != nil {
return nil, err
}
var useSSL bool
switch u.Scheme {
case "http":
// Ok, no SSL
case "https":
useSSL = true
default:
return nil, fmt.Errorf("Unsupported scheme for S3: '%s'", u.Scheme)
}

if opt.EndpointURL != "" {
cfg.EndpointResolverWithOptions = aws.EndpointResolverWithOptionsFunc(
func(service, region string, options ...interface{}) (aws.Endpoint, error) {
return aws.Endpoint{
URL: opt.EndpointURL,
HostnameImmutable: true,
SigningRegion: region,
}, nil
},
)
cfg := &minio.Options{
Creds: credentials.NewStaticV4(opt.AccessKey, opt.SecretKey, ""),
Secure: useSSL,
Transport: hc.Transport,
Region: opt.Region,
}

client := s3.NewFromConfig(cfg)
// Remove scheme from URL.
// Leave remaining validation to Minio client.
endpoint := opt.EndpointURL[len(u.Scheme)+1:] // Remove scheme and colon
endpoint = strings.TrimLeft(endpoint, "/") // Remove slashes if any

client, err := minio.New(endpoint, cfg)
if err != nil {
return nil, err
}

if info, ok := debug.ReadBuildInfo(); ok {
client.SetAppInfo("simpleblob", info.Main.Version)
}

if opt.CreateBucket {
// Create bucket if it does not exist
metricCalls.WithLabelValues("create-bucket").Inc()
metricLastCallTimestamp.WithLabelValues("create-bucket").SetToCurrentTime()
_, err = client.CreateBucket(ctx, &s3.CreateBucketInput{
Bucket: aws.String(opt.Bucket),
})
if err != nil && !isResponseError(err, http.StatusConflict) {
metricCallErrors.WithLabelValues("create-bucket").Inc()
return nil, err

err := client.MakeBucket(ctx, opt.Bucket, minio.MakeBucketOptions{Region: opt.Region})
if err != nil {
if err := handleErrorResponse(err); err != nil {
return nil, err
}
}
}

b := &Backend{
opt: opt,
s3config: cfg,
client: client,
opt: opt,
config: cfg,
client: client,
}

return b, nil
}

// handleErrorResponse takes an error, possibly a minio.ErrorResponse
// and turns it into a well known error when possible.
// If error is not well known, it is returned as is.
// If error is considered to be ignorable, nil is returned.
func handleErrorResponse(err error) error {
errResp := minio.ToErrorResponse(err)
if errResp.StatusCode == 404 {
return os.ErrNotExist
}
if errResp.Code != "BucketAlreadyOwnedByYou" {
return err
}
return nil
}

func init() {
simpleblob.RegisterBackend("s3", func(ctx context.Context, p simpleblob.InitParams) (simpleblob.Interface, error) {
var opt Options
Expand Down
14 changes: 4 additions & 10 deletions backends/s3/s3_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,7 @@ import (
"testing"
"time"

"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/minio/minio-go/v7"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gopkg.in/yaml.v2"
Expand Down Expand Up @@ -60,19 +59,14 @@ func getBackend(ctx context.Context, t *testing.T) (b *Backend) {
return
}
for _, blob := range blobs {
_, err := b.client.DeleteObject(ctx, &s3.DeleteObjectInput{
Bucket: aws.String(b.opt.Bucket),
Key: aws.String(blob.Name),
})
err := b.client.RemoveObject(ctx, b.opt.Bucket, blob.Name, minio.RemoveObjectOptions{})
if err != nil {
t.Logf("Object delete error: %s", err)
}
}
// This one is not returned by the List command
_, _ = b.client.DeleteObject(ctx, &s3.DeleteObjectInput{
Bucket: aws.String(b.opt.Bucket),
Key: aws.String(UpdateMarkerFilename),
})
err = b.client.RemoveObject(ctx, b.opt.Bucket, UpdateMarkerFilename, minio.RemoveObjectOptions{})
require.NoError(t, err)
}
t.Cleanup(cleanup)
cleanup()
Expand Down
39 changes: 21 additions & 18 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,44 +3,47 @@ module github.com/PowerDNS/simpleblob
go 1.17

require (
github.com/aws/aws-sdk-go-v2 v1.13.0
github.com/aws/aws-sdk-go-v2/config v1.13.1
github.com/aws/aws-sdk-go-v2/credentials v1.8.0
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.9.1
github.com/aws/aws-sdk-go-v2/service/s3 v1.24.1
github.com/PowerDNS/go-tlsconfig v0.0.0-20201014142732-fe6ff56e2a95
github.com/minio/minio-go/v7 v7.0.30
github.com/prometheus/client_golang v1.12.0
github.com/stretchr/testify v1.7.0
gopkg.in/yaml.v2 v2.4.0
)

require (
github.com/PowerDNS/go-tlsconfig v0.0.0-20201014142732-fe6ff56e2a95 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.2.0 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.10.0 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.4 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.2.0 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.5 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.7.0 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.7.0 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.11.0 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.9.0 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.14.0 // indirect
github.com/aws/smithy-go v1.10.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dustin/go-humanize v1.0.0 // indirect
github.com/go-logr/logr v0.2.1 // indirect
github.com/golang/protobuf v1.5.2 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/google/go-cmp v0.5.6 // indirect
github.com/google/uuid v1.1.1 // indirect
github.com/gopherjs/gopherjs v1.17.2 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/jtolds/gls v4.20.0+incompatible // indirect
github.com/klauspost/compress v1.13.5 // indirect
github.com/klauspost/cpuid v1.3.1 // indirect
github.com/kr/text v0.2.0 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
github.com/minio/md5-simd v1.1.0 // indirect
github.com/minio/sha256-simd v0.1.1 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_model v0.2.0 // indirect
github.com/prometheus/common v0.32.1 // indirect
github.com/prometheus/procfs v0.7.3 // indirect
github.com/rs/xid v1.2.1 // indirect
github.com/sirupsen/logrus v1.8.1 // indirect
github.com/smartystreets/assertions v1.13.0 // indirect
golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 // indirect
golang.org/x/net v0.0.0-20210525063256-abc453219eb5 // indirect
golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 // indirect
golang.org/x/text v0.3.6 // indirect
google.golang.org/protobuf v1.27.1 // indirect
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect
gopkg.in/ini.v1 v1.57.0 // indirect
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
)
Loading