-
Notifications
You must be signed in to change notification settings - Fork 3.4k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Feat: add Baidu Cloud BOS as storage backends for Loki #4788 #5848
Changes from 3 commits
9d58e8f
23a8c8e
cd077b7
655ab0e
b293942
5876eaf
cbe43d6
a38578f
fb30c1e
9a51421
737190b
01a83ce
6fb42b4
9a0a0be
ae5c2ef
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,25 @@ | ||
schema_config: | ||
configs: | ||
- from: 2020-05-15 | ||
store: boltdb-shipper | ||
object_store: bos | ||
schema: v11 | ||
index: | ||
prefix: index_ | ||
period: 24h | ||
|
||
storage_config: | ||
boltdb_shipper: | ||
active_index_directory: /loki/index | ||
cache_location: /loki/index_cache | ||
shared_store: bos | ||
|
||
bos: | ||
bucket_name: bucket_name_1 | ||
endpoint: bj.bcebos.com | ||
access_key_id: access_key_id | ||
secret_access_key: secret_access_key | ||
|
||
compactor: | ||
working_directory: /tmp/loki/compactor | ||
shared_store: bos |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -110,6 +110,8 @@ require ( | |
k8s.io/klog v1.0.0 | ||
) | ||
|
||
require github.com/baidubce/bce-sdk-go v0.9.81 | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I'm not an expert around why we have this file organized the way we do, but I imagine we want this in one of the other require sections rather than sitting on it's own? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. yes! I move it. |
||
|
||
require ( | ||
cloud.google.com/go v0.100.2 // indirect | ||
cloud.google.com/go/compute v1.3.0 // indirect | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -21,6 +21,7 @@ import ( | |
"github.com/grafana/loki/pkg/storage/chunk/client" | ||
"github.com/grafana/loki/pkg/storage/chunk/client/aws" | ||
"github.com/grafana/loki/pkg/storage/chunk/client/azure" | ||
"github.com/grafana/loki/pkg/storage/chunk/client/baidubce" | ||
"github.com/grafana/loki/pkg/storage/chunk/client/gcp" | ||
"github.com/grafana/loki/pkg/storage/chunk/client/hedging" | ||
"github.com/grafana/loki/pkg/storage/chunk/client/openstack" | ||
|
@@ -33,11 +34,12 @@ type RuleStoreConfig struct { | |
ConfigDB configClient.Config `yaml:"configdb"` | ||
|
||
// Object Storage Configs | ||
Azure azure.BlobStorageConfig `yaml:"azure"` | ||
GCS gcp.GCSConfig `yaml:"gcs"` | ||
S3 aws.S3Config `yaml:"s3"` | ||
Swift openstack.SwiftConfig `yaml:"swift"` | ||
Local local.Config `yaml:"local"` | ||
Azure azure.BlobStorageConfig `yaml:"azure"` | ||
GCS gcp.GCSConfig `yaml:"gcs"` | ||
S3 aws.S3Config `yaml:"s3"` | ||
BOS baidubce.BosStorageConfig `yaml:"bos"` | ||
Swift openstack.SwiftConfig `yaml:"swift"` | ||
Local local.Config `yaml:"local"` | ||
|
||
mock rulestore.RuleStore `yaml:"-"` | ||
} | ||
|
@@ -50,7 +52,7 @@ func (cfg *RuleStoreConfig) RegisterFlags(f *flag.FlagSet) { | |
cfg.S3.RegisterFlagsWithPrefix("ruler.storage.", f) | ||
cfg.Swift.RegisterFlagsWithPrefix("ruler.storage.", f) | ||
cfg.Local.RegisterFlagsWithPrefix("ruler.storage.", f) | ||
|
||
cfg.BOS.RegisterFlagsWithPrefix("ruler.storage.", f) | ||
f.StringVar(&cfg.Type, "ruler.storage.type", "configdb", "Method to use for backend rule storage (configdb, azure, gcs, s3, swift, local)") | ||
} | ||
|
||
|
@@ -65,6 +67,9 @@ func (cfg *RuleStoreConfig) Validate() error { | |
if err := cfg.S3.Validate(); err != nil { | ||
return errors.Wrap(err, "invalid S3 Storage config") | ||
} | ||
if err := cfg.BOS.Validate(); err != nil { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. this function always returns There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I delete the function |
||
return errors.Wrap(err, "invalid BOS Storage config") | ||
} | ||
return nil | ||
} | ||
|
||
|
@@ -101,6 +106,8 @@ func NewLegacyRuleStore(cfg RuleStoreConfig, hedgeCfg hedging.Config, clientMetr | |
client, err = gcp.NewGCSObjectClient(context.Background(), cfg.GCS, hedgeCfg) | ||
case "s3": | ||
client, err = aws.NewS3ObjectClient(cfg.S3, hedgeCfg) | ||
case "bos": | ||
client, err = baidubce.NewBosObjectStorage(&cfg.BOS) | ||
case "swift": | ||
client, err = openstack.NewSwiftObjectClient(cfg.Swift, hedgeCfg) | ||
case "local": | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
should add an option to the comment above on
type
as well to includebos
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This comment. I agree.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Thanks, I was careless, this has been fixed.