From 984f42ea00f1e62d599709382408e6c1861212b2 Mon Sep 17 00:00:00 2001 From: Michael Dai Date: Wed, 19 Sep 2018 20:04:36 +0800 Subject: [PATCH] Refactor bucket configuration flags for Object Store (#500) * Add unified flags for bucket configuration Signed-off-by: jojohappy * Use the unified bucket config for components Signed-off-by: jojohappy * Use double quotes instead of single quotes Signed-off-by: jojohappy * Fixed missing flags in man page Signed-off-by: jojohappy * To use value of bucket config instead of pointer to get value from command-line arguments Signed-off-by: jojohappy * Remove useless code Signed-off-by: jojohappy * Update documents Signed-off-by: jojohappy * Update - Rename provider to objstore for flags - To use objProvider instead of string for Provider Type - Get rid of bucket configuration errors Signed-off-by: jojohappy * Change errors package Signed-off-by: jojohappy * To validate the configuration in each provider client Signed-off-by: jojohappy * To support to make bucket configuration flag using suffix Signed-off-by: jojohappy * Update documents Signed-off-by: jojohappy * Refactor: - Remove all flags of objstore - Add flag as objstore.config to pass the configuration for bucket with yaml - To define the configuration for each provider - Add new method to get the name of bucket Signed-off-by: jojohappy * Update documents * Fixed missing method for inmem bucket * Update the describe for objstore.config * To setup bucket flags required for component bucket and downsample * Rename Content to Config for Bucket.Config Signed-off-by: jojohappy * To change error handler idiom Signed-off-by: jojohappy * Update describe for component store Signed-off-by: jojohappy * To setup secret-key just use envvar Signed-off-by: jojohappy * Update the placeholder of flags and documents Signed-off-by: jojohappy * Update example for bucket Signed-off-by: jojohappy * Update documents Signed-off-by: jojohappy * Update CHANGELOG Signed-off-by: jojohappy * Fixed something nits * To distinguish no bucket is configured or not supported Signed-off-by: jojohappy * Update CHANGELOG Signed-off-by: jojohappy * Remove unrequired unit test Signed-off-by: jojohappy * To set bucket flag required for component store and compact Signed-off-by: jojohappy * Rename GetBucket to Name & Set context as first argument Signed-off-by: jojohappy * Wrap error to give more information Signed-off-by: jojohappy * Rename field bucket to name for the struct of Bucket Signed-off-by: jojohappy * Update documents Signed-off-by: jojohappy * Change the bucket configuration flags to reference to YAML file * Do not get the secret-key from envvar * Update documents * Update CHANGELOG Signed-off-by: jojohappy * Update test case for package objstore client Signed-off-by: jojohappy * Rename flag objstore.config.file to objstore.config-file Signed-off-by: jojohappy * Update CHANGELOG * To wrap errors for loading and parsing bucket configuration file Signed-off-by: jojohappy * Rename objstore flag for consistency Signed-off-by: jojohappy --- CHANGELOG.md | 19 +++ Makefile | 2 +- cmd/thanos/bucket.go | 22 +-- cmd/thanos/compact.go | 15 +- cmd/thanos/downsample.go | 14 +- cmd/thanos/rule.go | 17 +- cmd/thanos/sidecar.go | 19 +-- cmd/thanos/store.go | 19 +-- docs/components/bucket.md | 144 +++++++---------- docs/components/compact.md | 70 ++++---- docs/components/rule.md | 27 ++-- docs/components/sidecar.md | 24 ++- docs/components/store.md | 31 ++-- docs/storage.md | 40 +++-- pkg/objstore/client/factory.go | 76 ++++++--- pkg/objstore/client/factory_test.go | 23 +++ .../client/testconf/blank-gcs.conf.yml | 1 + .../client/testconf/fake-gcs.conf.yml | 3 + pkg/objstore/gcs/gcs.go | 72 ++++++--- pkg/objstore/inmem/inmem.go | 5 + pkg/objstore/objstore.go | 7 + pkg/objstore/s3/s3.go | 151 ++++++++---------- 22 files changed, 429 insertions(+), 372 deletions(-) create mode 100644 pkg/objstore/client/factory_test.go create mode 100644 pkg/objstore/client/testconf/blank-gcs.conf.yml create mode 100644 pkg/objstore/client/testconf/fake-gcs.conf.yml diff --git a/CHANGELOG.md b/CHANGELOG.md index fc8787cddb..bccdcf874e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,25 @@ NOTE: As semantic versioning states all 0.y.z releases can contain breaking chan ## Unreleased +- Remove support of those flags for bucket + - --gcs-bucket=\ + - --s3.bucket=\ + - --s3.endpoint=\ + - --s3.access-key=\ + - --s3.insecure + - --s3.signature-version2 + - --s3.encrypt-sse + - --gcs-backup-bucket=\ + - --s3-backup-bucket=\ +- Remove support of those environment variables for bucket + * S3_BUCKET + * S3_ENDPOINT + * S3_ACCESS_KEY + * S3_INSECURE + * S3_SIGNATURE_VERSION2 + * S3_SECRET_KEY +- Add flag `--objstore.config-file` to reference to the bucket configuration file in yaml format. Note that detailed information in document [storage](docs/storage.md). + ## [v0.1.0](https://github.com/improbable-eng/thanos/releases/tag/v0.1.0) - 2018.09.14 Initial version to have a stable reference before [gossip protocol removal](https://github.com/improbable-eng/thanos/blob/master/docs/proposals/gossip-removal.md). diff --git a/Makefile b/Makefile index 23beaab07b..fe98a7b6d6 100644 --- a/Makefile +++ b/Makefile @@ -139,7 +139,7 @@ tarballs-release: $(PROMU) # test runs all Thanos golang tests against each supported version of Prometheus. .PHONY: test test: test-deps - @echo ">> running all tests. Do export THANOS_SKIP_GCS_TESTS="true" or/and export THANOS_SKIP_S3_AWS_TESTS="true" if you want to skip e2e tests against real store buckets" + @echo ">> running all tests. Do export THANOS_SKIP_GCS_TESTS='true' or/and export THANOS_SKIP_S3_AWS_TESTS='true' if you want to skip e2e tests against real store buckets" @for ver in $(SUPPORTED_PROM_VERSIONS); do \ THANOS_TEST_PROMETHEUS_PATH="prometheus-$$ver" THANOS_TEST_ALERTMANAGER_PATH="alertmanager-$(ALERTMANAGER_VERSION)" go test $(shell go list ./... | grep -v /vendor/ | grep -v /benchmark/); \ done diff --git a/cmd/thanos/bucket.go b/cmd/thanos/bucket.go index fa8c75797e..a51a4d14dc 100644 --- a/cmd/thanos/bucket.go +++ b/cmd/thanos/bucket.go @@ -12,7 +12,6 @@ import ( "github.com/go-kit/kit/log" "github.com/improbable-eng/thanos/pkg/block" "github.com/improbable-eng/thanos/pkg/objstore/client" - "github.com/improbable-eng/thanos/pkg/objstore/s3" "github.com/improbable-eng/thanos/pkg/runutil" "github.com/improbable-eng/thanos/pkg/verifier" "github.com/oklog/run" @@ -42,34 +41,27 @@ var ( func registerBucket(m map[string]setupFunc, app *kingpin.Application, name string) { cmd := app.Command(name, "inspect metric data in an object storage bucket") - gcsBucket := cmd.Flag("gcs-bucket", "Google Cloud Storage bucket name for stored blocks."). - PlaceHolder("").String() - - s3Config := s3.RegisterS3Params(cmd) + bucketConfFile := cmd.Flag("objstore.config-file", "The object store configuration file path."). + PlaceHolder("").Required().String() // Verify command. verify := cmd.Command("verify", "verify all blocks in the bucket against specified issues") verifyRepair := verify.Flag("repair", "attempt to repair blocks for which issues were detected"). Short('r').Default("false").Bool() - // NOTE(bplotka): Currently we support backup buckets only in the same project. - verifyBackupGCSBucket := cmd.Flag("gcs-backup-bucket", "Google Cloud Storage bucket name to backup blocks on repair operations."). - PlaceHolder("").String() - verifyBackupS3Bucket := cmd.Flag("s3-backup-bucket", "S3 bucket name to backup blocks on repair operations."). - PlaceHolder("").String() + backupBucketConfFile := verify.Flag("objstore-backup.config-file", "The backup object store configuration file path."). + PlaceHolder("").String() verifyIssues := verify.Flag("issues", fmt.Sprintf("Issues to verify (and optionally repair). Possible values: %v", allIssues())). Short('i').Default(verifier.IndexIssueID, verifier.OverlappedBlocksIssueID).Strings() verifyIDWhitelist := verify.Flag("id-whitelist", "Block IDs to verify (and optionally repair) only. "+ "If none is specified, all blocks will be verified. Repeated field").Strings() m[name+" verify"] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, _ opentracing.Tracer, _ bool) error { - bkt, err := client.NewBucket(logger, gcsBucket, *s3Config, reg, name) + bkt, err := client.NewBucket(logger, *bucketConfFile, reg, name) if err != nil { return err } defer runutil.CloseWithLogOnErr(logger, bkt, "bucket client") - backupS3Config := *s3Config - backupS3Config.Bucket = *verifyBackupS3Bucket - backupBkt, err := client.NewBucket(logger, verifyBackupGCSBucket, backupS3Config, reg, name) + backupBkt, err := client.NewBucket(logger, *backupBucketConfFile, reg, name) if err == client.ErrNotFound { if *verifyRepair { return errors.Wrap(err, "repair is specified, so backup client is required") @@ -129,7 +121,7 @@ func registerBucket(m map[string]setupFunc, app *kingpin.Application, name strin lsOutput := ls.Flag("output", "Format in which to print each block's information. May be 'json' or custom template."). Short('o').Default("").String() m[name+" ls"] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, _ opentracing.Tracer, _ bool) error { - bkt, err := client.NewBucket(logger, gcsBucket, *s3Config, reg, name) + bkt, err := client.NewBucket(logger, *bucketConfFile, reg, name) if err != nil { return err } diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go index 0781285523..42f5a1a344 100644 --- a/cmd/thanos/compact.go +++ b/cmd/thanos/compact.go @@ -11,7 +11,6 @@ import ( "github.com/improbable-eng/thanos/pkg/compact" "github.com/improbable-eng/thanos/pkg/compact/downsample" "github.com/improbable-eng/thanos/pkg/objstore/client" - "github.com/improbable-eng/thanos/pkg/objstore/s3" "github.com/improbable-eng/thanos/pkg/runutil" "github.com/oklog/run" "github.com/opentracing/opentracing-go" @@ -32,10 +31,8 @@ func registerCompact(m map[string]setupFunc, app *kingpin.Application, name stri dataDir := cmd.Flag("data-dir", "Data directory in which to cache blocks and process compactions."). Default("./data").String() - gcsBucket := cmd.Flag("gcs.bucket", "Google Cloud Storage bucket name for stored blocks."). - PlaceHolder("").String() - - s3config := s3.RegisterS3Params(cmd) + bucketConfFile := cmd.Flag("objstore.config-file", "The object store configuration file path."). + PlaceHolder("").Required().String() syncDelay := modelDuration(cmd.Flag("sync-delay", "Minimum age of fresh (non-compacted) blocks before they are being processed."). Default("30m")) @@ -56,8 +53,7 @@ func registerCompact(m map[string]setupFunc, app *kingpin.Application, name stri return runCompact(g, logger, reg, *httpAddr, *dataDir, - *gcsBucket, - s3config, + *bucketConfFile, time.Duration(*syncDelay), *haltOnError, *wait, @@ -78,8 +74,7 @@ func runCompact( reg *prometheus.Registry, httpBindAddr string, dataDir string, - gcsBucket string, - s3Config *s3.Config, + bucketConfFile string, syncDelay time.Duration, haltOnError bool, wait bool, @@ -99,7 +94,7 @@ func runCompact( reg.MustRegister(halted) - bkt, err := client.NewBucket(logger, &gcsBucket, *s3Config, reg, component) + bkt, err := client.NewBucket(logger, bucketConfFile, reg, component) if err != nil { return err } diff --git a/cmd/thanos/downsample.go b/cmd/thanos/downsample.go index 6ea50f8a07..a6e92a696f 100644 --- a/cmd/thanos/downsample.go +++ b/cmd/thanos/downsample.go @@ -16,7 +16,6 @@ import ( "github.com/improbable-eng/thanos/pkg/compact/downsample" "github.com/improbable-eng/thanos/pkg/objstore" "github.com/improbable-eng/thanos/pkg/objstore/client" - "github.com/improbable-eng/thanos/pkg/objstore/s3" "github.com/improbable-eng/thanos/pkg/runutil" "github.com/oklog/run" "github.com/oklog/ulid" @@ -33,13 +32,11 @@ func registerDownsample(m map[string]setupFunc, app *kingpin.Application, name s dataDir := cmd.Flag("data-dir", "Data directory in which to cache blocks and process downsamplings."). Default("./data").String() - gcsBucket := cmd.Flag("gcs.bucket", "Google Cloud Storage bucket name for stored blocks."). - PlaceHolder("").Required().String() - - s3Config := s3.RegisterS3Params(cmd) + bucketConfFile := cmd.Flag("objstore.config-file", "The object store configuration file path."). + PlaceHolder("").Required().String() m[name] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, _ bool) error { - return runDownsample(g, logger, reg, *dataDir, *gcsBucket, s3Config, name) + return runDownsample(g, logger, reg, *dataDir, *bucketConfFile, name) } } @@ -48,12 +45,11 @@ func runDownsample( logger log.Logger, reg *prometheus.Registry, dataDir string, - gcsBucket string, - s3Config *s3.Config, + bucketConfFile string, component string, ) error { - bkt, err := client.NewBucket(logger, &gcsBucket, *s3Config, reg, component) + bkt, err := client.NewBucket(logger, bucketConfFile, reg, component) if err != nil { return err } diff --git a/cmd/thanos/rule.go b/cmd/thanos/rule.go index 2bb8e16944..dd85b0630c 100644 --- a/cmd/thanos/rule.go +++ b/cmd/thanos/rule.go @@ -25,7 +25,6 @@ import ( "github.com/improbable-eng/thanos/pkg/block" "github.com/improbable-eng/thanos/pkg/cluster" "github.com/improbable-eng/thanos/pkg/objstore/client" - "github.com/improbable-eng/thanos/pkg/objstore/s3" "github.com/improbable-eng/thanos/pkg/runutil" "github.com/improbable-eng/thanos/pkg/shipper" "github.com/improbable-eng/thanos/pkg/store" @@ -72,12 +71,10 @@ func registerRule(m map[string]setupFunc, app *kingpin.Application, name string) alertmgrs := cmd.Flag("alertmanagers.url", "Alertmanager URLs to push firing alerts to. The scheme may be prefixed with 'dns+' or 'dnssrv+' to detect Alertmanager IPs through respective DNS lookups. The port defaults to 9093 or the SRV record's value. The URL path is used as a prefix for the regular Alertmanager API path."). Strings() - gcsBucket := cmd.Flag("gcs.bucket", "Google Cloud Storage bucket name for stored blocks. If empty, ruler won't store any block inside Google Cloud Storage."). - PlaceHolder("").String() - alertQueryURL := cmd.Flag("alert.query-url", "The external Thanos Query URL that would be set in all alerts 'Source' field").String() - s3Config := s3.RegisterS3Params(cmd) + bucketConfFile := cmd.Flag("objstore.config-file", "The object store configuration file path."). + PlaceHolder("").String() m[name] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, _ bool) error { lset, err := parseFlagLabels(*labelStrs) @@ -112,8 +109,7 @@ func registerRule(m map[string]setupFunc, app *kingpin.Application, name string) *dataDir, *ruleFiles, peer, - *gcsBucket, - s3Config, + *bucketConfFile, tsdbOpts, name, alertQueryURL, @@ -136,8 +132,7 @@ func runRule( dataDir string, ruleFiles []string, peer *cluster.Peer, - gcsBucket string, - s3Config *s3.Config, + bucketConfFile string, tsdbOpts *tsdb.Options, component string, alertQueryURL *url.URL, @@ -415,13 +410,13 @@ func runRule( // The background shipper continuously scans the data directory and uploads // new blocks to Google Cloud Storage or an S3-compatible storage service. - bkt, err := client.NewBucket(logger, &gcsBucket, *s3Config, reg, component) + bkt, err := client.NewBucket(logger, bucketConfFile, reg, component) if err != nil && err != client.ErrNotFound { return err } if err == client.ErrNotFound { - level.Info(logger).Log("msg", "No GCS or S3 bucket was configured, uploads will be disabled") + level.Info(logger).Log("msg", "No supported bucket was configured, uploads will be disabled") uploads = false } diff --git a/cmd/thanos/sidecar.go b/cmd/thanos/sidecar.go index 284c2fc651..3c78f03597 100644 --- a/cmd/thanos/sidecar.go +++ b/cmd/thanos/sidecar.go @@ -16,7 +16,6 @@ import ( "github.com/improbable-eng/thanos/pkg/block" "github.com/improbable-eng/thanos/pkg/cluster" "github.com/improbable-eng/thanos/pkg/objstore/client" - "github.com/improbable-eng/thanos/pkg/objstore/s3" "github.com/improbable-eng/thanos/pkg/reloader" "github.com/improbable-eng/thanos/pkg/runutil" "github.com/improbable-eng/thanos/pkg/shipper" @@ -43,11 +42,6 @@ func registerSidecar(m map[string]setupFunc, app *kingpin.Application, name stri dataDir := cmd.Flag("tsdb.path", "Data directory of TSDB."). Default("./data").String() - gcsBucket := cmd.Flag("gcs.bucket", "Google Cloud Storage bucket name for stored blocks. If empty, sidecar won't store any block inside Google Cloud Storage."). - PlaceHolder("").String() - - s3Config := s3.RegisterS3Params(cmd) - reloaderCfgFile := cmd.Flag("reloader.config-file", "Config file watched by the reloader."). Default("").String() @@ -56,6 +50,9 @@ func registerSidecar(m map[string]setupFunc, app *kingpin.Application, name stri reloaderRuleDirs := cmd.Flag("reloader.rule-dir", "Rule directories for the reloader to refresh (repeated field).").Strings() + bucketConfFile := cmd.Flag("objstore.config-file", "The object store configuration file path."). + PlaceHolder("").String() + m[name] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, _ bool) error { rl := reloader.New( log.With(logger, "component", "reloader"), @@ -77,8 +74,7 @@ func registerSidecar(m map[string]setupFunc, app *kingpin.Application, name stri *httpBindAddr, *promURL, *dataDir, - *gcsBucket, - s3Config, + *bucketConfFile, peer, rl, name, @@ -95,8 +91,7 @@ func runSidecar( httpBindAddr string, promURL *url.URL, dataDir string, - gcsBucket string, - s3Config *s3.Config, + bucketConfFile string, peer *cluster.Peer, reloader *reloader.Reloader, component string, @@ -224,13 +219,13 @@ func runSidecar( // The background shipper continuously scans the data directory and uploads // new blocks to Google Cloud Storage or an S3-compatible storage service. - bkt, err := client.NewBucket(logger, &gcsBucket, *s3Config, reg, component) + bkt, err := client.NewBucket(logger, bucketConfFile, reg, component) if err != nil && err != client.ErrNotFound { return err } if err == client.ErrNotFound { - level.Info(logger).Log("msg", "No GCS or S3 bucket was configured, uploads will be disabled") + level.Info(logger).Log("msg", "No supported bucket was configured, uploads will be disabled") uploads = false } diff --git a/cmd/thanos/store.go b/cmd/thanos/store.go index 0e92c8c774..4110736e78 100644 --- a/cmd/thanos/store.go +++ b/cmd/thanos/store.go @@ -10,7 +10,6 @@ import ( "github.com/go-kit/kit/log/level" "github.com/improbable-eng/thanos/pkg/cluster" "github.com/improbable-eng/thanos/pkg/objstore/client" - "github.com/improbable-eng/thanos/pkg/objstore/s3" "github.com/improbable-eng/thanos/pkg/runutil" "github.com/improbable-eng/thanos/pkg/store" "github.com/improbable-eng/thanos/pkg/store/storepb" @@ -24,17 +23,15 @@ import ( // registerStore registers a store command. func registerStore(m map[string]setupFunc, app *kingpin.Application, name string) { - cmd := app.Command(name, "store node giving access to blocks in a GCS bucket") + cmd := app.Command(name, "store node giving access to blocks in a bucket provider. Now supported GCS / S3.") grpcBindAddr, httpBindAddr, newPeerFn := regCommonServerFlags(cmd) dataDir := cmd.Flag("data-dir", "Data directory in which to cache remote blocks."). Default("./data").String() - gcsBucket := cmd.Flag("gcs.bucket", "Google Cloud Storage bucket name for stored blocks. If empty sidecar won't store any block inside Google Cloud Storage."). - PlaceHolder("").String() - - s3Config := s3.RegisterS3Params(cmd) + bucketConfFile := cmd.Flag("objstore.config-file", "The object store configuration file path."). + PlaceHolder("").Required().String() indexCacheSize := cmd.Flag("index-cache-size", "Maximum size of items held in the index cache."). Default("250MB").Bytes() @@ -51,8 +48,7 @@ func registerStore(m map[string]setupFunc, app *kingpin.Application, name string logger, reg, tracer, - *gcsBucket, - s3Config, + *bucketConfFile, *dataDir, *grpcBindAddr, *httpBindAddr, @@ -71,8 +67,7 @@ func runStore( logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, - gcsBucket string, - s3Config *s3.Config, + bucketConfFile string, dataDir string, grpcBindAddr string, httpBindAddr string, @@ -83,9 +78,9 @@ func runStore( verbose bool, ) error { { - bkt, err := client.NewBucket(logger, &gcsBucket, *s3Config, reg, component) + bkt, err := client.NewBucket(logger, bucketConfFile, reg, component) if err != nil { - return err + return errors.Wrap(err, "create bucket client") } // Ensure we close up everything properly. diff --git a/docs/components/bucket.md b/docs/components/bucket.md index 5a37dafaa2..5a7c87be78 100644 --- a/docs/components/bucket.md +++ b/docs/components/bucket.md @@ -6,7 +6,15 @@ It is normally run as a stand alone command to aid with troubleshooting. Example: ``` -$ thanos bucket verify --gcs.bucket example-bucket +$ thanos bucket verify --objstore.config-file=bucket.yml +``` + +The content of `bucket.yml`: + +```yaml +type: GCS +config: + bucket: example-bucket ``` Bucket can be extended to add more subcommands that will be helpful when working with object storage buckets @@ -18,39 +26,24 @@ by adding a new command within `/cmd/thanos/bucket.go` [embedmd]:# (flags/bucket.txt $) ```$ -usage: thanos bucket [] [ ...] +usage: thanos bucket --objstore.config-file= [ ...] inspect metric data in an object storage bucket Flags: - -h, --help Show context-sensitive help (also try --help-long - and --help-man). - --version Show application version. - --log.level=info Log filtering level. + -h, --help Show context-sensitive help (also try --help-long and + --help-man). + --version Show application version. + --log.level=info Log filtering level. --gcloudtrace.project=GCLOUDTRACE.PROJECT - GCP project to send Google Cloud Trace tracings - to. If empty, tracing will be disabled. + GCP project to send Google Cloud Trace tracings to. If + empty, tracing will be disabled. --gcloudtrace.sample-factor=1 - How often we send traces (1/). If - 0 no trace will be sent periodically, unless - forced by baggage item. See - `pkg/tracing/tracing.go` for details. - --gcs-bucket= Google Cloud Storage bucket name for stored - blocks. - --s3.bucket= S3-Compatible API bucket name for stored blocks. - --s3.endpoint= S3-Compatible API endpoint for stored blocks. - --s3.access-key= Access key for an S3-Compatible API. - --s3.insecure Whether to use an insecure connection with an - S3-Compatible API. - --s3.signature-version2 Whether to use S3 Signature Version 2; otherwise - Signature Version 4 will be used. - --s3.encrypt-sse Whether to use Server Side Encryption - --gcs-backup-bucket= - Google Cloud Storage bucket name to backup blocks - on repair operations. - --s3-backup-bucket= - S3 bucket name to backup blocks on repair - operations. + How often we send traces (1/). If 0 no + trace will be sent periodically, unless forced by + baggage item. See `pkg/tracing/tracing.go` for details. + --objstore.config-file= + The object store configuration file path. Subcommands: bucket verify [] @@ -79,44 +72,30 @@ usage: thanos bucket verify [] verify all blocks in the bucket against specified issues Flags: - -h, --help Show context-sensitive help (also try --help-long - and --help-man). - --version Show application version. - --log.level=info Log filtering level. + -h, --help Show context-sensitive help (also try --help-long and + --help-man). + --version Show application version. + --log.level=info Log filtering level. --gcloudtrace.project=GCLOUDTRACE.PROJECT - GCP project to send Google Cloud Trace tracings - to. If empty, tracing will be disabled. + GCP project to send Google Cloud Trace tracings to. If + empty, tracing will be disabled. --gcloudtrace.sample-factor=1 - How often we send traces (1/). If - 0 no trace will be sent periodically, unless - forced by baggage item. See - `pkg/tracing/tracing.go` for details. - --gcs-bucket= Google Cloud Storage bucket name for stored - blocks. - --s3.bucket= S3-Compatible API bucket name for stored blocks. - --s3.endpoint= S3-Compatible API endpoint for stored blocks. - --s3.access-key= Access key for an S3-Compatible API. - --s3.insecure Whether to use an insecure connection with an - S3-Compatible API. - --s3.signature-version2 Whether to use S3 Signature Version 2; otherwise - Signature Version 4 will be used. - --s3.encrypt-sse Whether to use Server Side Encryption - --gcs-backup-bucket= - Google Cloud Storage bucket name to backup blocks - on repair operations. - --s3-backup-bucket= - S3 bucket name to backup blocks on repair - operations. - -r, --repair attempt to repair blocks for which issues were - detected + How often we send traces (1/). If 0 no + trace will be sent periodically, unless forced by + baggage item. See `pkg/tracing/tracing.go` for details. + --objstore.config-file= + The object store configuration file path. + -r, --repair attempt to repair blocks for which issues were detected + --objstore-backup.config-file= + The backup object store configuration file path. -i, --issues=index_issue... ... - Issues to verify (and optionally repair). - Possible values: [duplicated_compaction - index_issue overlapped_blocks] + Issues to verify (and optionally repair). Possible + values: [duplicated_compaction index_issue + overlapped_blocks] --id-whitelist=ID-WHITELIST ... - Block IDs to verify (and optionally repair) only. - If none is specified, all blocks will be - verified. Repeated field + Block IDs to verify (and optionally repair) only. If + none is specified, all blocks will be verified. Repeated + field ``` @@ -137,36 +116,21 @@ usage: thanos bucket ls [] list all blocks in the bucket Flags: - -h, --help Show context-sensitive help (also try --help-long - and --help-man). - --version Show application version. - --log.level=info Log filtering level. + -h, --help Show context-sensitive help (also try --help-long and + --help-man). + --version Show application version. + --log.level=info Log filtering level. --gcloudtrace.project=GCLOUDTRACE.PROJECT - GCP project to send Google Cloud Trace tracings - to. If empty, tracing will be disabled. + GCP project to send Google Cloud Trace tracings to. If + empty, tracing will be disabled. --gcloudtrace.sample-factor=1 - How often we send traces (1/). If - 0 no trace will be sent periodically, unless - forced by baggage item. See - `pkg/tracing/tracing.go` for details. - --gcs-bucket= Google Cloud Storage bucket name for stored - blocks. - --s3.bucket= S3-Compatible API bucket name for stored blocks. - --s3.endpoint= S3-Compatible API endpoint for stored blocks. - --s3.access-key= Access key for an S3-Compatible API. - --s3.insecure Whether to use an insecure connection with an - S3-Compatible API. - --s3.signature-version2 Whether to use S3 Signature Version 2; otherwise - Signature Version 4 will be used. - --s3.encrypt-sse Whether to use Server Side Encryption - --gcs-backup-bucket= - Google Cloud Storage bucket name to backup blocks - on repair operations. - --s3-backup-bucket= - S3 bucket name to backup blocks on repair - operations. - -o, --output="" Format in which to print each block's - information. May be 'json' or custom template. + How often we send traces (1/). If 0 no + trace will be sent periodically, unless forced by + baggage item. See `pkg/tracing/tracing.go` for details. + --objstore.config-file= + The object store configuration file path. + -o, --output="" Format in which to print each block's information. May + be 'json' or custom template. ``` diff --git a/docs/components/compact.md b/docs/components/compact.md index 45e730b87c..df738f747f 100644 --- a/docs/components/compact.md +++ b/docs/components/compact.md @@ -6,7 +6,15 @@ It is generally not semantically concurrency safe and must be deployed as a sing Example: ``` -$ thanos compact --gcs.bucket example-bucket --data-dir /tmp/thanos-compact +$ thanos compact --data-dir /tmp/thanos-compact --objstore.config-file=bucket.yml +``` + +The content of `bucket.yml`: + +```yaml +type: GCS +config: + bucket: example-bucket ``` The compactor needs local disk space to store intermediate data for its processing. Generally, about 100GB are recommended for it to keep working as the compacted time ranges grow over time. @@ -18,49 +26,41 @@ On-disk data is safe to delete between restarts and should be the first attempt [embedmd]:# (flags/compact.txt $) ```$ -usage: thanos compact [] +usage: thanos compact --objstore.config-file= [] continuously compacts blocks in an object store bucket Flags: - -h, --help Show context-sensitive help (also try --help-long - and --help-man). - --version Show application version. - --log.level=info Log filtering level. + -h, --help Show context-sensitive help (also try --help-long and + --help-man). + --version Show application version. + --log.level=info Log filtering level. --gcloudtrace.project=GCLOUDTRACE.PROJECT - GCP project to send Google Cloud Trace tracings - to. If empty, tracing will be disabled. + GCP project to send Google Cloud Trace tracings to. + If empty, tracing will be disabled. --gcloudtrace.sample-factor=1 - How often we send traces (1/). If - 0 no trace will be sent periodically, unless - forced by baggage item. See - `pkg/tracing/tracing.go` for details. + How often we send traces (1/). If 0 no + trace will be sent periodically, unless forced by + baggage item. See `pkg/tracing/tracing.go` for + details. --http-address="0.0.0.0:10902" - Listen host:port for HTTP endpoints. - --data-dir="./data" Data directory in which to cache blocks and - process compactions. - --gcs.bucket= Google Cloud Storage bucket name for stored - blocks. - --s3.bucket= S3-Compatible API bucket name for stored blocks. - --s3.endpoint= S3-Compatible API endpoint for stored blocks. - --s3.access-key= Access key for an S3-Compatible API. - --s3.insecure Whether to use an insecure connection with an - S3-Compatible API. - --s3.signature-version2 Whether to use S3 Signature Version 2; otherwise - Signature Version 4 will be used. - --s3.encrypt-sse Whether to use Server Side Encryption - --sync-delay=30m Minimum age of fresh (non-compacted) blocks - before they are being processed. + Listen host:port for HTTP endpoints. + --data-dir="./data" Data directory in which to cache blocks and process + compactions. + --objstore.config-file= + The object store configuration file path. + --sync-delay=30m Minimum age of fresh (non-compacted) blocks before + they are being processed. --retention.resolution-raw=0d - How long to retain raw samples in bucket. 0d - - disables this retention + How long to retain raw samples in bucket. 0d - + disables this retention --retention.resolution-5m=0d - How long to retain samples of resolution 1 (5 - minutes) in bucket. 0d - disables this retention + How long to retain samples of resolution 1 (5 + minutes) in bucket. 0d - disables this retention --retention.resolution-1h=0d - How long to retain samples of resolution 2 (1 - hour) in bucket. 0d - disables this retention - -w, --wait Do not exit after all compactions have been - processed and wait for new work. + How long to retain samples of resolution 2 (1 hour) + in bucket. 0d - disables this retention + -w, --wait Do not exit after all compactions have been processed + and wait for new work. ``` diff --git a/docs/components/rule.md b/docs/components/rule.md index 4f82600266..0237812397 100644 --- a/docs/components/rule.md +++ b/docs/components/rule.md @@ -14,9 +14,17 @@ $ thanos rule \ --eval-interval "30s" \ --rule-file "/path/to/rules/*.rules.yaml" \ --alert.query-url "http://0.0.0.0:9090" \ - --alertmanagers.url "alert.thanos.io" - --gcs.bucket "example-bucket" \ - --cluster.peers "thanos-cluster.example.org" + --alertmanagers.url "alert.thanos.io" \ + --cluster.peers "thanos-cluster.example.org" \ + --objstore.config-file "bucket.yml" +``` + +The content of `bucket.yml`: + +```yaml +type: GCS +config: + bucket: example-bucket ``` As rule nodes outsource query processing to query nodes, they should generally experience little load. If necessary, functional sharding can be applied by splitting up the sets of rules between HA pairs. @@ -108,19 +116,10 @@ Flags: DNS lookups. The port defaults to 9093 or the SRV record's value. The URL path is used as a prefix for the regular Alertmanager API path. - --gcs.bucket= Google Cloud Storage bucket name for stored - blocks. If empty, ruler won't store any block - inside Google Cloud Storage. --alert.query-url=ALERT.QUERY-URL The external Thanos Query URL that would be set in all alerts 'Source' field - --s3.bucket= S3-Compatible API bucket name for stored blocks. - --s3.endpoint= S3-Compatible API endpoint for stored blocks. - --s3.access-key= Access key for an S3-Compatible API. - --s3.insecure Whether to use an insecure connection with an - S3-Compatible API. - --s3.signature-version2 Whether to use S3 Signature Version 2; otherwise - Signature Version 4 will be used. - --s3.encrypt-sse Whether to use Server Side Encryption + --objstore.config-file= + The object store configuration file path. ``` diff --git a/docs/components/sidecar.md b/docs/components/sidecar.md index 2fd00b147e..dbd92bc088 100644 --- a/docs/components/sidecar.md +++ b/docs/components/sidecar.md @@ -15,8 +15,16 @@ The retention is recommended to not be lower than three times the block duration $ thanos sidecar \ --tsdb.path "/path/to/prometheus/data/dir" \ --prometheus.url "http://localhost:9090" \ - --gcs.bucket "example-bucket" \ --cluster.peers "thanos-cluster.example.org" \ + --objstore.config-file "bucket.yml" +``` + +The content of `bucket.yml`: + +```yaml +type: GCS +config: + bucket: example-bucket ``` ## Deployment @@ -92,18 +100,6 @@ Flags: URL at which to reach Prometheus's API. For better performance use local network. --tsdb.path="./data" Data directory of TSDB. - --gcs.bucket= Google Cloud Storage bucket name for stored - blocks. If empty, sidecar won't store any block - inside Google Cloud Storage. - --s3.bucket= S3-Compatible API bucket name for stored - blocks. - --s3.endpoint= S3-Compatible API endpoint for stored blocks. - --s3.access-key= Access key for an S3-Compatible API. - --s3.insecure Whether to use an insecure connection with an - S3-Compatible API. - --s3.signature-version2 Whether to use S3 Signature Version 2; - otherwise Signature Version 4 will be used. - --s3.encrypt-sse Whether to use Server Side Encryption --reloader.config-file="" Config file watched by the reloader. --reloader.config-envsubst-file="" Output file for environment variable @@ -111,6 +107,8 @@ Flags: --reloader.rule-dir=RELOADER.RULE-DIR ... Rule directories for the reloader to refresh (repeated field). + --objstore.config-file= + The object store configuration file path. ``` diff --git a/docs/components/store.md b/docs/components/store.md index 55d417304b..eae9af5437 100644 --- a/docs/components/store.md +++ b/docs/components/store.md @@ -5,9 +5,17 @@ It keeps a small amount of information about all remote blocks on local disk and ``` $ thanos store \ - --data-dir "/local/state/data/dir" \ - --gcs.bucket "example-bucket" \ - --cluster.peers "thanos-cluster.example.org" + --data-dir "/local/state/data/dir" \ + --cluster.peers "thanos-cluster.example.org" \ + --objstore.config-file "bucket.yml" +``` + +The content of `bucket.yml`: + +```yaml +type: GCS +config: + bucket: example-bucket ``` In general about 1MB of local disk space is required per TSDB block stored in the object storage bucket. @@ -17,9 +25,9 @@ In general about 1MB of local disk space is required per TSDB block stored in th [embedmd]:# (flags/store.txt $) ```$ -usage: thanos store [] +usage: thanos store --objstore.config-file= [] -store node giving access to blocks in a GCS bucket +store node giving access to blocks in a bucket provider. Now supported GCS / S3. Flags: -h, --help Show context-sensitive help (also try @@ -81,17 +89,8 @@ Flags: accounting the latency differences between network types: local, lan, wan. --data-dir="./data" Data directory in which to cache remote blocks. - --gcs.bucket= Google Cloud Storage bucket name for stored - blocks. If empty sidecar won't store any block - inside Google Cloud Storage. - --s3.bucket= S3-Compatible API bucket name for stored blocks. - --s3.endpoint= S3-Compatible API endpoint for stored blocks. - --s3.access-key= Access key for an S3-Compatible API. - --s3.insecure Whether to use an insecure connection with an - S3-Compatible API. - --s3.signature-version2 Whether to use S3 Signature Version 2; otherwise - Signature Version 4 will be used. - --s3.encrypt-sse Whether to use Server Side Encryption + --objstore.config-file= + The object store configuration file path. --index-cache-size=250MB Maximum size of items held in the index cache. --chunk-pool-size=2GB Maximum size of concurrently allocatable bytes for chunks. diff --git a/docs/storage.md b/docs/storage.md index 4a4ff7e1bd..2d8b4ac209 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -26,23 +26,26 @@ At that point, anyone can use your provider! Thanos uses minio client to upload Prometheus data into AWS s3. -To configure S3 bucket as an object store you need to set these mandatory S3 flags: -- --s3.endpoint -- --s3.bucket - -Instead of using flags you can pass all the configuration via environment variables: -- `S3_BUCKET` -- `S3_ENDPOINT` -- `S3_ACCESS_KEY` -- `S3_SECRET_KEY` -- `S3_INSECURE` -- `S3_SIGNATURE_VERSION2` +To configure S3 bucket as an object store you need to set these mandatory S3 variables in yaml format stored in a file: +``` +type: S3 +config: + bucket: + endpoint: + access-key: + insecure: + signature-version2: + encrypt-sse: + secret-key: +``` + +Set the flags `--objstore.config-file` to reference to the configuration file. AWS region to endpoint mapping can be found in this [link](https://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region) -Make sure you use a correct signature version with `--s3.signature-version2`, otherwise, you will get Access Denied error. +Make sure you use a correct signature version to set `signature-version2: true`, otherwise, you will get Access Denied error. -For debug purposes you can `--s3.insecure` to switch to plain insecure HTTP instead of HTTPS +For debug purposes you can set `insecure: true` to switch to plain insecure HTTP instead of HTTPS ### Credentials Credentials will by default try to retrieve from the following sources: @@ -122,7 +125,16 @@ Details about AWS policies: https://docs.aws.amazon.com/AmazonS3/latest/dev/usin ## GCP Configuration -To configure Google Cloud Storage bucket as an object store you need to set `--gcs.bucket` with GCS bucket name and configure Google Application credentials. +To configure Google Cloud Storage bucket as an object store you need to set `bucket` with GCS bucket name and configure Google Application credentials. + +For example: +``` +type: GCS +config: + bucket: +``` + +Set the flags `--objstore.config-file` to reference to the configuration file. Application credentials are configured via JSON file, the client looks for: diff --git a/pkg/objstore/client/factory.go b/pkg/objstore/client/factory.go index 7324bfc5e3..3a50a38d92 100644 --- a/pkg/objstore/client/factory.go +++ b/pkg/objstore/client/factory.go @@ -3,39 +3,75 @@ package client import ( "context" "fmt" - "runtime" + "io/ioutil" - "cloud.google.com/go/storage" "github.com/go-kit/kit/log" + "github.com/go-kit/kit/log/level" "github.com/improbable-eng/thanos/pkg/objstore" "github.com/improbable-eng/thanos/pkg/objstore/gcs" "github.com/improbable-eng/thanos/pkg/objstore/s3" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/common/version" - "google.golang.org/api/option" + yaml "gopkg.in/yaml.v2" ) -var ErrNotFound = errors.New("no valid GCS or S3 configuration supplied") +type objProvider string + +const ( + GCS objProvider = "GCS" + S3 objProvider = "S3" +) + +type BucketConfig struct { + Type objProvider `yaml:"type"` + Config interface{} `yaml:"config"` +} + +var ErrNotFound = errors.New("not found bucket") + +func loadFile(confFile string) (*BucketConfig, error) { + content, err := ioutil.ReadFile(confFile) + if err != nil { + return nil, errors.Wrap(err, fmt.Sprintf("loading YAML file %s", confFile)) + } + + bucketConf := &BucketConfig{} + if err := yaml.UnmarshalStrict(content, bucketConf); err != nil { + return nil, errors.Wrap(err, fmt.Sprintf("parsing YAML file %s", confFile)) + } + return bucketConf, nil +} // NewBucket initializes and returns new object storage clients. -func NewBucket(logger log.Logger, gcsBucket *string, s3Config s3.Config, reg *prometheus.Registry, component string) (objstore.Bucket, error) { - if *gcsBucket != "" { - gcsOptions := option.WithUserAgent(fmt.Sprintf("thanos-%s/%s (%s)", component, version.Version, runtime.Version())) - gcsClient, err := storage.NewClient(context.Background(), gcsOptions) - if err != nil { - return nil, errors.Wrap(err, "create GCS client") - } - return objstore.BucketWithMetrics(*gcsBucket, gcs.NewBucket(*gcsBucket, gcsClient, reg), reg), nil +func NewBucket(logger log.Logger, confFile string, reg *prometheus.Registry, component string) (objstore.Bucket, error) { + level.Info(logger).Log("msg", "loading bucket configuration file", "filename", confFile) + + var err error + if confFile == "" { + return nil, ErrNotFound } - if s3Config.Validate() == nil { - b, err := s3.NewBucket(logger, &s3Config, reg, component) - if err != nil { - return nil, errors.Wrap(err, "create s3 client") - } - return objstore.BucketWithMetrics(s3Config.Bucket, b, reg), nil + bucketConf, err := loadFile(confFile) + if err != nil { + return nil, errors.Wrap(err, "parsing objstore.config-file") } - return nil, ErrNotFound + config, err := yaml.Marshal(bucketConf.Config) + if err != nil { + return nil, errors.Wrap(err, "marshal content of bucket configuration") + } + + var bucket objstore.Bucket + switch bucketConf.Type { + case GCS: + bucket, err = gcs.NewBucket(context.Background(), logger, config, reg, component) + case S3: + bucket, err = s3.NewBucket(logger, config, reg, component) + default: + return nil, errors.Errorf("bucket with type %s is not supported", bucketConf.Type) + } + if err != nil { + return nil, errors.Wrap(err, fmt.Sprintf("create %s client", bucketConf.Type)) + } + return objstore.BucketWithMetrics(bucket.Name(), bucket, reg), nil } diff --git a/pkg/objstore/client/factory_test.go b/pkg/objstore/client/factory_test.go new file mode 100644 index 0000000000..319c1e20ac --- /dev/null +++ b/pkg/objstore/client/factory_test.go @@ -0,0 +1,23 @@ +package client + +import ( + "testing" + + "github.com/improbable-eng/thanos/pkg/testutil" + + "github.com/go-kit/kit/log" +) + +func TestErrorBucketConfig(t *testing.T) { + conf := "testconf/fake-gcs.conf.yml" + _, err := NewBucket(log.NewNopLogger(), conf, nil, "bkt-client-test") + testutil.NotOk(t, err) + testutil.Assert(t, err != ErrNotFound, "it should not error with not found") +} + +func TestBlankBucketConfigContent(t *testing.T) { + conf := "testconf/blank-gcs.conf.yml" + _, err := NewBucket(log.NewNopLogger(), conf, nil, "bkt-client-test") + testutil.NotOk(t, err) + testutil.Assert(t, err != ErrNotFound, "it should not error with not found") +} diff --git a/pkg/objstore/client/testconf/blank-gcs.conf.yml b/pkg/objstore/client/testconf/blank-gcs.conf.yml new file mode 100644 index 0000000000..cb5ef588cc --- /dev/null +++ b/pkg/objstore/client/testconf/blank-gcs.conf.yml @@ -0,0 +1 @@ +type: GCS \ No newline at end of file diff --git a/pkg/objstore/client/testconf/fake-gcs.conf.yml b/pkg/objstore/client/testconf/fake-gcs.conf.yml new file mode 100644 index 0000000000..538c832788 --- /dev/null +++ b/pkg/objstore/client/testconf/fake-gcs.conf.yml @@ -0,0 +1,3 @@ +type: FAKE-GCS +config: + bucket: test-bucket \ No newline at end of file diff --git a/pkg/objstore/gcs/gcs.go b/pkg/objstore/gcs/gcs.go index 227c47c051..e06be4f374 100644 --- a/pkg/objstore/gcs/gcs.go +++ b/pkg/objstore/gcs/gcs.go @@ -3,18 +3,23 @@ package gcs import ( "context" - "io" - "strings" - "fmt" + "io" "math/rand" + "runtime" + "strings" "testing" "time" "cloud.google.com/go/storage" + "github.com/go-kit/kit/log" "github.com/improbable-eng/thanos/pkg/objstore" + "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/common/version" "google.golang.org/api/iterator" + "google.golang.org/api/option" + yaml "gopkg.in/yaml.v2" ) const ( @@ -32,29 +37,55 @@ const ( // DirDelim is the delimiter used to model a directory structure in an object store bucket. const DirDelim = "/" +// gcsConfig stores the configuration for gcs bucket. +type gcsConfig struct { + Bucket string `yaml:"bucket"` +} + // Bucket implements the store.Bucket and shipper.Bucket interfaces against GCS. type Bucket struct { + logger log.Logger bkt *storage.BucketHandle opsTotal *prometheus.CounterVec + name string closer io.Closer } // NewBucket returns a new Bucket against the given bucket handle. -func NewBucket(name string, cl *storage.Client, reg prometheus.Registerer) *Bucket { +func NewBucket(ctx context.Context, logger log.Logger, conf []byte, reg prometheus.Registerer, component string) (*Bucket, error) { + var gc gcsConfig + if err := yaml.Unmarshal(conf, &gc); err != nil { + return nil, err + } + if gc.Bucket == "" { + return nil, errors.New("missing Google Cloud Storage bucket name for stored blocks") + } + gcsOptions := option.WithUserAgent(fmt.Sprintf("thanos-%s/%s (%s)", component, version.Version, runtime.Version())) + gcsClient, err := storage.NewClient(ctx, gcsOptions) + if err != nil { + return nil, err + } bkt := &Bucket{ - bkt: cl.Bucket(name), + logger: logger, + bkt: gcsClient.Bucket(gc.Bucket), opsTotal: prometheus.NewCounterVec(prometheus.CounterOpts{ Name: "thanos_objstore_gcs_bucket_operations_total", Help: "Total number of operations that were executed against a Google Compute Storage bucket.", - ConstLabels: prometheus.Labels{"bucket": name}, + ConstLabels: prometheus.Labels{"bucket": gc.Bucket}, }, []string{"operation"}), - closer: cl, + closer: gcsClient, + name: gc.Bucket, } if reg != nil { reg.MustRegister() } - return bkt + return bkt, nil +} + +// Name returns the bucket name for gcs. +func (b *Bucket) Name() string { + return b.name } // Iter calls f for each entry in the given directory. The argument to f is the full @@ -151,27 +182,32 @@ func (b *Bucket) Close() error { // In a close function it empties and deletes the bucket. func NewTestBucket(t testing.TB, project string) (objstore.Bucket, func(), error) { ctx, cancel := context.WithCancel(context.Background()) - gcsClient, err := storage.NewClient(ctx) + src := rand.NewSource(time.Now().UnixNano()) + gTestConfig := gcsConfig{ + Bucket: fmt.Sprintf("test_%s_%x", strings.ToLower(t.Name()), src.Int63()), + } + + bc, err := yaml.Marshal(gTestConfig) if err != nil { - cancel() return nil, nil, err } - src := rand.NewSource(time.Now().UnixNano()) - name := fmt.Sprintf("test_%s_%x", strings.ToLower(t.Name()), src.Int63()) - bkt := gcsClient.Bucket(name) - if err = bkt.Create(ctx, project, nil); err != nil { + b, err := NewBucket(ctx, log.NewNopLogger(), bc, nil, "thanos-e2e-test") + if err != nil { cancel() - _ = gcsClient.Close() return nil, nil, err } - b := NewBucket(name, gcsClient, nil) + if err = b.bkt.Create(ctx, project, nil); err != nil { + cancel() + _ = b.Close() + return nil, nil, err + } - t.Log("created temporary GCS bucket for GCS tests with name", name, "in project", project) + t.Log("created temporary GCS bucket for GCS tests with name", b.name, "in project", project) return b, func() { objstore.EmptyBucket(t, ctx, b) - if err := bkt.Delete(ctx); err != nil { + if err := b.bkt.Delete(ctx); err != nil { t.Logf("deleting bucket failed: %s", err) } cancel() diff --git a/pkg/objstore/inmem/inmem.go b/pkg/objstore/inmem/inmem.go index 57e9e0562b..29f95d7b8e 100644 --- a/pkg/objstore/inmem/inmem.go +++ b/pkg/objstore/inmem/inmem.go @@ -145,3 +145,8 @@ func (b *Bucket) IsObjNotFoundErr(err error) bool { } func (b *Bucket) Close() error { return nil } + +// Name returns the bucket name. +func (b *Bucket) Name() string { + return "inmem" +} diff --git a/pkg/objstore/objstore.go b/pkg/objstore/objstore.go index 379de95e33..81ae6fbb8f 100644 --- a/pkg/objstore/objstore.go +++ b/pkg/objstore/objstore.go @@ -26,6 +26,9 @@ type Bucket interface { // Delete removes the object with the given name. Delete(ctx context.Context, name string) error + + // Name returns the bucket name for the provider. + Name() string } // BucketReader provides read access to an object storage bucket. @@ -313,6 +316,10 @@ func (b *metricBucket) Close() error { return b.bkt.Close() } +func (b *metricBucket) Name() string { + return b.bkt.Name() +} + type timingReadCloser struct { io.ReadCloser diff --git a/pkg/objstore/s3/s3.go b/pkg/objstore/s3/s3.go index 9678cd288b..3794f23607 100644 --- a/pkg/objstore/s3/s3.go +++ b/pkg/objstore/s3/s3.go @@ -24,7 +24,7 @@ import ( "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/version" - "gopkg.in/alecthomas/kingpin.v2" + yaml "gopkg.in/yaml.v2" ) const ( @@ -38,87 +38,46 @@ const ( // DirDelim is the delimiter used to model a directory structure in an object store bucket. const DirDelim = "/" +// s3Config stores the configuration for s3 bucket. +type s3Config struct { + Bucket string `yaml:"bucket"` + Endpoint string `yaml:"endpoint"` + AccessKey string `yaml:"access-key"` + Insecure bool `yaml:"insecure"` + SignatureV2 bool `yaml:"signature-version2"` + SSEEncryption bool `yaml:"encrypt-sse"` + SecretKey string `yaml:"secret-key"` +} + // Bucket implements the store.Bucket interface against s3-compatible APIs. type Bucket struct { logger log.Logger - bucket string + name string client *minio.Client sse encrypt.ServerSide opsTotal *prometheus.CounterVec } -// Config encapsulates the necessary config values to instantiate an s3 client. -type Config struct { - Bucket string - Endpoint string - AccessKey string - secretKey string - Insecure bool - SignatureV2 bool - SSEEncryption bool -} - -// RegisterS3Params registers the s3 flags and returns an initialized Config struct. -func RegisterS3Params(cmd *kingpin.CmdClause) *Config { - var s3config Config - - cmd.Flag("s3.bucket", "S3-Compatible API bucket name for stored blocks."). - PlaceHolder("").Envar("S3_BUCKET").StringVar(&s3config.Bucket) - - cmd.Flag("s3.endpoint", "S3-Compatible API endpoint for stored blocks."). - PlaceHolder("").Envar("S3_ENDPOINT").StringVar(&s3config.Endpoint) - - cmd.Flag("s3.access-key", "Access key for an S3-Compatible API."). - PlaceHolder("").Envar("S3_ACCESS_KEY").StringVar(&s3config.AccessKey) - - s3config.secretKey = os.Getenv("S3_SECRET_KEY") - - cmd.Flag("s3.insecure", "Whether to use an insecure connection with an S3-Compatible API."). - Default("false").Envar("S3_INSECURE").BoolVar(&s3config.Insecure) - - cmd.Flag("s3.signature-version2", "Whether to use S3 Signature Version 2; otherwise Signature Version 4 will be used."). - Default("false").Envar("S3_SIGNATURE_VERSION2").BoolVar(&s3config.SignatureV2) - - cmd.Flag("s3.encrypt-sse", "Whether to use Server Side Encryption"). - Default("false").Envar("S3_SSE_ENCRYPTION").BoolVar(&s3config.SSEEncryption) - - return &s3config -} - -// Validate checks to see if mandatory s3 config options are set. -func (conf *Config) Validate() error { - if conf.Bucket == "" || - conf.Endpoint == "" || - (conf.AccessKey == "" && conf.secretKey != "") || - (conf.AccessKey != "" && conf.secretKey == "") { - return errors.New("insufficient s3 configuration information") - } - return nil -} - -// ValidateForTests checks to see if mandatory s3 config options for tests are set. -func (conf *Config) ValidateForTests() error { - if conf.Endpoint == "" || - conf.AccessKey == "" || - conf.secretKey == "" { - return errors.New("insufficient s3 test configuration information") - } - return nil -} - // NewBucket returns a new Bucket using the provided s3 config values. -func NewBucket(logger log.Logger, conf *Config, reg prometheus.Registerer, component string) (*Bucket, error) { +func NewBucket(logger log.Logger, conf []byte, reg prometheus.Registerer, component string) (*Bucket, error) { var chain []credentials.Provider - if conf.AccessKey != "" { + var config s3Config + if err := yaml.Unmarshal(conf, &config); err != nil { + return nil, err + } + if err := Validate(config); err != nil { + return nil, err + } + if config.AccessKey != "" { signature := credentials.SignatureV4 - if conf.SignatureV2 { + if config.SignatureV2 { signature = credentials.SignatureV2 } chain = []credentials.Provider{&credentials.Static{ Value: credentials.Value{ - AccessKeyID: conf.AccessKey, - SecretAccessKey: conf.secretKey, + AccessKeyID: config.AccessKey, + SecretAccessKey: config.SecretKey, SignerType: signature, }, }} @@ -134,7 +93,7 @@ func NewBucket(logger log.Logger, conf *Config, reg prometheus.Registerer, compo } } - client, err := minio.NewWithCredentials(conf.Endpoint, credentials.NewChainCredentials(chain), !conf.Insecure, "") + client, err := minio.NewWithCredentials(config.Endpoint, credentials.NewChainCredentials(chain), !config.Insecure, "") if err != nil { return nil, errors.Wrap(err, "initialize s3 client") } @@ -164,19 +123,19 @@ func NewBucket(logger log.Logger, conf *Config, reg prometheus.Registerer, compo }) var sse encrypt.ServerSide - if conf.SSEEncryption { + if config.SSEEncryption { sse = encrypt.NewSSE() } bkt := &Bucket{ logger: logger, - bucket: conf.Bucket, + name: config.Bucket, client: client, sse: sse, opsTotal: prometheus.NewCounterVec(prometheus.CounterOpts{ Name: "thanos_objstore_s3_bucket_operations_total", Help: "Total number of operations that were executed against an s3 bucket.", - ConstLabels: prometheus.Labels{"bucket": conf.Bucket}, + ConstLabels: prometheus.Labels{"bucket": config.Bucket}, }, []string{"operation"}), } if reg != nil { @@ -185,6 +144,31 @@ func NewBucket(logger log.Logger, conf *Config, reg prometheus.Registerer, compo return bkt, nil } +// Name returns the bucket name for s3. +func (b *Bucket) Name() string { + return b.name +} + +// Validate checks to see the config options are set. +func Validate(conf s3Config) error { + if conf.Endpoint == "" || + (conf.AccessKey == "" && conf.SecretKey != "") || + (conf.AccessKey != "" && conf.SecretKey == "") { + return errors.New("insufficient s3 test configuration information") + } + return nil +} + +// ValidateForTests checks to see the config options for tests are set. +func ValidateForTests(conf s3Config) error { + if conf.Endpoint == "" || + conf.AccessKey == "" || + conf.SecretKey == "" { + return errors.New("insufficient s3 test configuration information") + } + return nil +} + // Iter calls f for each entry in the given directory. The argument to f is the full // object name including the prefix of the inspected directory. func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error) error { @@ -195,7 +179,7 @@ func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error) err dir = strings.TrimSuffix(dir, DirDelim) + DirDelim } - for object := range b.client.ListObjects(b.bucket, dir, false, ctx.Done()) { + for object := range b.client.ListObjects(b.name, dir, false, ctx.Done()) { // Catch the error when failed to list objects. if object.Err != nil { return object.Err @@ -220,7 +204,7 @@ func (b *Bucket) getRange(ctx context.Context, name string, off, length int64) ( return nil, err } } - r, err := b.client.GetObjectWithContext(ctx, b.bucket, name, *opts) + r, err := b.client.GetObjectWithContext(ctx, b.name, name, *opts) if err != nil { return nil, err } @@ -250,7 +234,7 @@ func (b *Bucket) GetRange(ctx context.Context, name string, off, length int64) ( // Exists checks if the given object exists. func (b *Bucket) Exists(ctx context.Context, name string) (bool, error) { b.opsTotal.WithLabelValues(opObjectHead).Inc() - _, err := b.client.StatObject(b.bucket, name, minio.StatObjectOptions{}) + _, err := b.client.StatObject(b.name, name, minio.StatObjectOptions{}) if err != nil { if b.IsObjNotFoundErr(err) { return false, nil @@ -265,7 +249,7 @@ func (b *Bucket) Exists(ctx context.Context, name string) (bool, error) { func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error { b.opsTotal.WithLabelValues(opObjectInsert).Inc() - _, err := b.client.PutObjectWithContext(ctx, b.bucket, name, r, -1, + _, err := b.client.PutObjectWithContext(ctx, b.name, name, r, -1, minio.PutObjectOptions{ServerSideEncryption: b.sse}, ) @@ -275,7 +259,7 @@ func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error { // Delete removes the object with the given name. func (b *Bucket) Delete(ctx context.Context, name string) error { b.opsTotal.WithLabelValues(opObjectDelete).Inc() - return b.client.RemoveObject(b.bucket, name) + return b.client.RemoveObject(b.name, name) } // IsObjNotFoundErr returns true if error means that object is not found. Relevant to Get operations. @@ -285,12 +269,12 @@ func (b *Bucket) IsObjNotFoundErr(err error) bool { func (b *Bucket) Close() error { return nil } -func configFromEnv() *Config { - c := &Config{ +func configFromEnv() s3Config { + c := s3Config{ Bucket: os.Getenv("S3_BUCKET"), Endpoint: os.Getenv("S3_ENDPOINT"), AccessKey: os.Getenv("S3_ACCESS_KEY"), - secretKey: os.Getenv("S3_SECRET_KEY"), + SecretKey: os.Getenv("S3_SECRET_KEY"), } insecure, err := strconv.ParseBool(os.Getenv("S3_INSECURE")) @@ -308,11 +292,14 @@ func configFromEnv() *Config { // In a close function it empties and deletes the bucket. func NewTestBucket(t testing.TB, location string) (objstore.Bucket, func(), error) { c := configFromEnv() - if err := c.ValidateForTests(); err != nil { + if err := ValidateForTests(c); err != nil { return nil, nil, err } - - b, err := NewBucket(log.NewNopLogger(), c, nil, "thanos-e2e-test") + bc, err := yaml.Marshal(c) + if err != nil { + return nil, nil, err + } + b, err := NewBucket(log.NewNopLogger(), bc, nil, "thanos-e2e-test") if err != nil { return nil, nil, err } @@ -346,7 +333,7 @@ func NewTestBucket(t testing.TB, location string) (objstore.Bucket, func(), erro if err := b.client.MakeBucket(tmpBucketName, location); err != nil { return nil, nil, err } - b.bucket = tmpBucketName + b.name = tmpBucketName t.Log("created temporary AWS bucket for AWS tests with name", tmpBucketName, "in", location) return b, func() {