RuleStore: implementation using thanos objstore bucket client

CHANGELOG.md

@@ -7,6 +7,7 @@
 * [CHANGE] Query-frontend: removed `-querier.split-queries-by-day` (deprecated in Cortex 0.4.0). You should use `-querier.split-queries-by-interval` instead. #3813
 * [CHANGE] Store-gateway: the chunks pool controlled by `-blocks-storage.bucket-store.max-chunk-pool-bytes` is now shared across all tenants. #3830
 * [CHANGE] Ingester: return error code 400 instead of 429 when per-user/per-tenant series/metadata limits are reached. #3833
+* [FEATURE] Experimental Ruler Storage: Add a separate set of configuration options to configure the ruler storage backend under the `-ruler-storage.` flag prefix. All blocks storage bucket clients and the config service are currently supported. Clients using this implementation will only be enabled if the existing `-ruler.storage` flags are left unset. #3805
 * [FEATURE] Adds support to S3 server-side encryption using KMS. The S3 server-side encryption config can be overridden on a per-tenant basis. Deprecated `-<prefix>.s3.sse-encryption`, you should use the following CLI flags that have been added. #3651 #3810 #3811
   - `-<prefix>.s3.sse.type`
   - `-<prefix>.s3.sse.kms-key-id`

docs/configuration/config-file-reference.md

@@ -153,6 +153,209 @@ tenant_federation:
 # The ruler_config configures the Cortex ruler.
 [ruler: <ruler_config>]
 
+ruler_storage:
+  # Backend storage to use. Supported backends are: s3, gcs, azure, swift,
+  # filesystem, configdb.
+  # CLI flag: -ruler-storage.backend
+  [backend: <string> | default = "s3"]
+
+  s3:
+    # The S3 bucket endpoint. It could be an AWS S3 endpoint listed at
+    # https://docs.aws.amazon.com/general/latest/gr/s3.html or the address of an
+    # S3-compatible service in hostname:port format.
+    # CLI flag: -ruler-storage.s3.endpoint
+    [endpoint: <string> | default = ""]
+
+    # S3 region. If unset, the client will issue a S3 GetBucketLocation API call
+    # to autodetect it.
+    # CLI flag: -ruler-storage.s3.region
+    [region: <string> | default = ""]
+
+    # S3 bucket name
+    # CLI flag: -ruler-storage.s3.bucket-name
+    [bucket_name: <string> | default = ""]
+
+    # S3 secret access key
+    # CLI flag: -ruler-storage.s3.secret-access-key
+    [secret_access_key: <string> | default = ""]
+
+    # S3 access key ID
+    # CLI flag: -ruler-storage.s3.access-key-id
+    [access_key_id: <string> | default = ""]
+
+    # If enabled, use http:// for the S3 endpoint instead of https://. This
+    # could be useful in local dev/test environments while using an
+    # S3-compatible backend storage, like Minio.
+    # CLI flag: -ruler-storage.s3.insecure
+    [insecure: <boolean> | default = false]
+
+    # The signature version to use for authenticating against S3. Supported
+    # values are: v4, v2.
+    # CLI flag: -ruler-storage.s3.signature-version
+    [signature_version: <string> | default = "v4"]
+
+    # The s3_sse_config configures the S3 server-side encryption.
+    # The CLI flags prefix for this block config is: ruler-storage
+    [sse: <s3_sse_config>]
+
+    http:
+      # The time an idle connection will remain idle before closing.
+      # CLI flag: -ruler-storage.s3.http.idle-conn-timeout
+      [idle_conn_timeout: <duration> | default = 1m30s]
+
+      # The amount of time the client will wait for a servers response headers.
+      # CLI flag: -ruler-storage.s3.http.response-header-timeout
+      [response_header_timeout: <duration> | default = 2m]
+
+      # If the client connects to S3 via HTTPS and this option is enabled, the
+      # client will accept any certificate and hostname.
+      # CLI flag: -ruler-storage.s3.http.insecure-skip-verify
+      [insecure_skip_verify: <boolean> | default = false]
+
+      # Maximum time to wait for a TLS handshake. 0 means no limit.
+      # CLI flag: -ruler-storage.s3.tls-handshake-timeout
+      [tls_handshake_timeout: <duration> | default = 10s]
+
+      # The time to wait for a server's first response headers after fully
+      # writing the request headers if the request has an Expect header. 0 to
+      # send the request body immediately.
+      # CLI flag: -ruler-storage.s3.expect-continue-timeout
+      [expect_continue_timeout: <duration> | default = 1s]
+
+      # Maximum number of idle (keep-alive) connections across all hosts. 0
+      # means no limit.
+      # CLI flag: -ruler-storage.s3.max-idle-connections
+      [max_idle_connections: <int> | default = 100]
+
+      # Maximum number of idle (keep-alive) connections to keep per-host. If 0,
+      # a built-in default value is used.
+      # CLI flag: -ruler-storage.s3.max-idle-connections-per-host
+      [max_idle_connections_per_host: <int> | default = 100]
+
+      # Maximum number of connections per host. 0 means no limit.
+      # CLI flag: -ruler-storage.s3.max-connections-per-host
+      [max_connections_per_host: <int> | default = 0]
+
+  gcs:
+    # GCS bucket name
+    # CLI flag: -ruler-storage.gcs.bucket-name
+    [bucket_name: <string> | default = ""]
+
+    # JSON representing either a Google Developers Console
+    # client_credentials.json file or a Google Developers service account key
+    # file. If empty, fallback to Google default logic.
+    # CLI flag: -ruler-storage.gcs.service-account
+    [service_account: <string> | default = ""]
+
+  azure:
+    # Azure storage account name
+    # CLI flag: -ruler-storage.azure.account-name
+    [account_name: <string> | default = ""]
+
+    # Azure storage account key
+    # CLI flag: -ruler-storage.azure.account-key
+    [account_key: <string> | default = ""]
+
+    # Azure storage container name
+    # CLI flag: -ruler-storage.azure.container-name
+    [container_name: <string> | default = ""]
+
+    # Azure storage endpoint suffix without schema. The account name will be
+    # prefixed to this value to create the FQDN
+    # CLI flag: -ruler-storage.azure.endpoint-suffix
+    [endpoint_suffix: <string> | default = ""]
+
+    # Number of retries for recoverable errors
+    # CLI flag: -ruler-storage.azure.max-retries
+    [max_retries: <int> | default = 20]
+
+  swift:
+    # OpenStack Swift authentication API version. 0 to autodetect.
+    # CLI flag: -ruler-storage.swift.auth-version
+    [auth_version: <int> | default = 0]
+
+    # OpenStack Swift authentication URL
+    # CLI flag: -ruler-storage.swift.auth-url
+    [auth_url: <string> | default = ""]
+
+    # OpenStack Swift username.
+    # CLI flag: -ruler-storage.swift.username
+    [username: <string> | default = ""]
+
+    # OpenStack Swift user's domain name.
+    # CLI flag: -ruler-storage.swift.user-domain-name
+    [user_domain_name: <string> | default = ""]
+
+    # OpenStack Swift user's domain ID.
+    # CLI flag: -ruler-storage.swift.user-domain-id
+    [user_domain_id: <string> | default = ""]
+
+    # OpenStack Swift user ID.
+    # CLI flag: -ruler-storage.swift.user-id
+    [user_id: <string> | default = ""]
+
+    # OpenStack Swift API key.
+    # CLI flag: -ruler-storage.swift.password
+    [password: <string> | default = ""]
+
+    # OpenStack Swift user's domain ID.
+    # CLI flag: -ruler-storage.swift.domain-id
+    [domain_id: <string> | default = ""]
+
+    # OpenStack Swift user's domain name.
+    # CLI flag: -ruler-storage.swift.domain-name
+    [domain_name: <string> | default = ""]
+
+    # OpenStack Swift project ID (v2,v3 auth only).
+    # CLI flag: -ruler-storage.swift.project-id
+    [project_id: <string> | default = ""]
+
+    # OpenStack Swift project name (v2,v3 auth only).
+    # CLI flag: -ruler-storage.swift.project-name
+    [project_name: <string> | default = ""]
+
+    # ID of the OpenStack Swift project's domain (v3 auth only), only needed if
+    # it differs the from user domain.
+    # CLI flag: -ruler-storage.swift.project-domain-id
+    [project_domain_id: <string> | default = ""]
+
+    # Name of the OpenStack Swift project's domain (v3 auth only), only needed
+    # if it differs from the user domain.
+    # CLI flag: -ruler-storage.swift.project-domain-name
+    [project_domain_name: <string> | default = ""]
+
+    # OpenStack Swift Region to use (v2,v3 auth only).
+    # CLI flag: -ruler-storage.swift.region-name
+    [region_name: <string> | default = ""]
+
+    # Name of the OpenStack Swift container to put chunks in.
+    # CLI flag: -ruler-storage.swift.container-name
+    [container_name: <string> | default = ""]
+
+    # Max retries on requests error.
+    # CLI flag: -ruler-storage.swift.max-retries
+    [max_retries: <int> | default = 3]
+
+    # Time after which a connection attempt is aborted.
+    # CLI flag: -ruler-storage.swift.connect-timeout
+    [connect_timeout: <duration> | default = 10s]
+
+    # Time after which an idle request is aborted. The timeout watchdog is reset
+    # each time some data is received, so the timeout triggers after X time no
+    # data is received on a request.
+    # CLI flag: -ruler-storage.swift.request-timeout
+    [request_timeout: <duration> | default = 5s]
+
+  filesystem:
+    # Local filesystem storage directory.
+    # CLI flag: -ruler-storage.filesystem.dir
+    [dir: <string> | default = ""]
+
+  # The configstore_config configures the config database storing rules and
+  # alerts, and is used by the Cortex alertmanager.
+  # The CLI flags prefix for this block config is: ruler-storage
+  [configdb: <configstore_config>]
+
 # The configs_config configures the Cortex Configs DB and API.
 [configs: <configs_config>]
 
@@ -3781,6 +3984,7 @@ The `configstore_config` configures the config database storing rules and alerts
 
 - `alertmanager`
 - `ruler`
+- `ruler-storage`
 
 &nbsp;
 
@@ -4677,6 +4881,7 @@ The `s3_sse_config` configures the S3 server-side encryption. The supported CLI
 - _no prefix_
 - `alertmanager.storage`
 - `blocks-storage`
+- `ruler-storage`
 - `ruler.storage`
 
 &nbsp;

docs/configuration/v1-guarantees.md

@@ -70,3 +70,4 @@ Currently experimental features are:
 - Alertmanager: Sharding of tenants across multiple instances
 - The thanosconvert tool for converting Thanos block metadata to Cortex
 - HA Tracker: cleanup of old replicas from KV Store.
+- Ruler storage: backend client configuration options using a config fields similar to the TSDB object storage clients.

integration/configs.go

@@ -21,6 +21,7 @@ type storeConfig struct {
 const (
 	defaultNetworkName     = "e2e-cortex-test"
 	bucketName             = "cortex"
+	rulestoreBucketName    = "cortex-rules"
 	cortexConfigFile       = "config.yaml"
 	cortexSchemaConfigFile = "schema.yaml"
 	blocksStorageEngine    = "blocks"
@@ -128,16 +129,30 @@ var (
 		}
 	}
 
-	RulerFlags = func() map[string]string {
+	RulerFlags = func(legacy bool) map[string]string {
+		if legacy {
+			return map[string]string{
+				"-api.response-compression-enabled":  "true",
+				"-ruler.enable-sharding":             "false",
+				"-ruler.poll-interval":               "2s",
+				"-experimental.ruler.enable-api":     "true",
+				"-ruler.storage.type":                "s3",
+				"-ruler.storage.s3.buckets":          rulestoreBucketName,
+				"-ruler.storage.s3.force-path-style": "true",
+				"-ruler.storage.s3.url":              fmt.Sprintf("s3://%s:%s@%s-minio-9000.:9000", e2edb.MinioAccessKey, e2edb.MinioSecretKey, networkName),
+			}
+		}
 		return map[string]string{
-			"-api.response-compression-enabled":  "true",
-			"-ruler.enable-sharding":             "false",
-			"-ruler.poll-interval":               "2s",
-			"-experimental.ruler.enable-api":     "true",
-			"-ruler.storage.type":                "s3",
-			"-ruler.storage.s3.buckets":          "cortex-rules",
-			"-ruler.storage.s3.force-path-style": "true",
-			"-ruler.storage.s3.url":              fmt.Sprintf("s3://%s:%s@%s-minio-9000.:9000", e2edb.MinioAccessKey, e2edb.MinioSecretKey, networkName),
+			"-api.response-compression-enabled":   "true",
+			"-ruler.enable-sharding":              "false",
+			"-ruler.poll-interval":                "2s",
+			"-experimental.ruler.enable-api":      "true",
+			"-ruler-storage.backend":              "s3",
+			"-ruler-storage.s3.access-key-id":     e2edb.MinioAccessKey,
+			"-ruler-storage.s3.secret-access-key": e2edb.MinioSecretKey,
+			"-ruler-storage.s3.bucket-name":       rulestoreBucketName,
+			"-ruler-storage.s3.endpoint":          fmt.Sprintf("%s-minio-9000:9000", networkName),
+			"-ruler-storage.s3.insecure":          "true",
 		}
 	}