Fixed ruler to run with blocks storage (cortexproject#2101)

* Fixed ruler to run with blocks storage

Signed-off-by: Marco Pracucci <marco@pracucci.com>

Author: pracucci

CHANGELOG.md

@@ -28,6 +28,7 @@
   * `-server.grpc.keepalive.timeout`
 * [ENHANCEMENT] PostgreSQL: Bump up `github.com/lib/pq` from `v1.0.0` to `v1.3.0` to support PostgreSQL SCRAM-SHA-256 authentication. #2097
 * [BUGFIX] Experimental TSDB: fixed `/all_user_stats` and `/api/prom/user_stats` endpoints when using the experimental TSDB blocks storage. #2042
+* [BUGFIX] Experimental TSDB: fixed ruler to correctly work with the experimental TSDB blocks storage. #2101
 * [BUGFIX] Azure Blob ChunkStore: Fixed issue causing `invalid chunk checksum` errors. #2074
 
 Cortex 0.4.0 is the last version that can *write* denormalised tokens. Cortex 0.5.0 and above always write normalised tokens.

development/tsdb-blocks-storage-s3/.data-configstore/api/prom/configs/rules

@@ -0,0 +1,14 @@
+{
+    "since": 2,
+    "configs": {
+        "fake": {
+            "id": 1,
+            "config": {
+                "rule_format_version": "2",
+                "rules_files": {
+                    "test": "groups:\n    - name: group-1\n      rules:\n      - record: avalanche_metric_mmmmm_0_0:count\n        expr: count(avalanche_metric_mmmmm_0_0)\n    - name: group-2\n      rules:\n      - record: avalanche_metric_mmmmm_0:count\n        expr: |\n          count(avalanche_metric_mmmmm_0_0) +\n          count(avalanche_metric_mmmmm_0_1)\n      - alert: alert-1\n        expr: |\n          count(avalanche_metric_mmmmm_0_0) > 1000\n      - alert: alert-2\n        expr:  count(avalanche_metric_mmmmm_0_0) > 100\n        for:   5m\n        labels:\n          label1: value1\n          label2: value2\n        annotations:\n          annotation1: value1\n          annotation2: value2\n"
+                }
+            }
+        }
+    }
+}

development/tsdb-blocks-storage-s3/.data-configstore/api/prom/configs/rules.yaml

@@ -0,0 +1,23 @@
+groups:
+    - name: group-1
+      rules:
+      - record: avalanche_metric_mmmmm_0_0:count
+        expr: count(avalanche_metric_mmmmm_0_0)
+    - name: group-2
+      rules:
+      - record: avalanche_metric_mmmmm_0:count
+        expr: |
+          count(avalanche_metric_mmmmm_0_0) +
+          count(avalanche_metric_mmmmm_0_1)
+      - alert: alert-1
+        expr: |
+          count(avalanche_metric_mmmmm_0_0) > 1000
+      - alert: alert-2
+        expr:  count(avalanche_metric_mmmmm_0_0) > 100
+        for:   5m
+        labels:
+          label1: value1
+          label2: value2
+        annotations:
+          annotation1: value1
+          annotation2: value2

development/tsdb-blocks-storage-s3/config/cortex.yaml

@@ -46,5 +46,12 @@ tsdb:
     secret_access_key: supersecret
     insecure:          true
 
+ruler:
+  enable_api: true
+  storeconfig:
+    type: configdb
+    configdb:
+      configsapiurl: http://configstore:80/
+
 storage:
   engine: tsdb

development/tsdb-blocks-storage-s3/docker-compose.yml

@@ -18,6 +18,11 @@ services:
     volumes:
       - .data-minio:/data:delegated
 
+  configstore:
+    image: nginx
+    volumes:
+      - .data-configstore:/usr/share/nginx/html/private:ro
+
   distributor:
     build:
       context:    .
@@ -75,3 +80,17 @@ services:
       - 8004:8004
     volumes:
       - ./config:/cortex/config
+    
+  ruler:
+    build:
+      context:    .
+      dockerfile: dev.dockerfile
+    image: cortex
+    command: ["sh", "-c", "sleep 3 && exec ./cortex -config.file=./config/cortex.yaml -target=ruler -server.http-listen-port=8005 -server.grpc-listen-port=9005"]
+    depends_on:
+      - consul
+      - minio
+    ports:
+      - 8005:8005
+    volumes:
+      - ./config:/cortex/config

pkg/cortex/cortex.go

@@ -176,6 +176,10 @@ type Cortex struct {
 	configDB     db.DB
 	alertmanager *alertmanager.MultitenantAlertmanager
 	compactor    *compactor.Compactor
+
+	// The chunk store that the querier should use to query the long
+	// term storage. It depends on the storage engine used.
+	querierChunkStore querier.ChunkStore
 }
 
 // New makes a new Cortex.

pkg/cortex/modules.go

@@ -48,6 +48,7 @@ const (
 	Distributor
 	Ingester
 	Querier
+	QuerierChunkStore
 	QueryFrontend
 	Store
 	TableManager
@@ -76,6 +77,8 @@ func (m moduleName) String() string {
 		return "ingester"
 	case Querier:
 		return "querier"
+	case QuerierChunkStore:
+		return "querier-chunk-store"
 	case QueryFrontend:
 		return "query-frontend"
 	case TableManager:
@@ -118,6 +121,9 @@ func (m *moduleName) Set(s string) error {
 	case "querier":
 		*m = Querier
 		return nil
+	case "querier-chunk-store":
+		*m = QuerierChunkStore
+		return nil
 	case "query-frontend":
 		*m = QueryFrontend
 		return nil
@@ -226,18 +232,7 @@ func (t *Cortex) stopDistributor() (err error) {
 }
 
 func (t *Cortex) initQuerier(cfg *Config) (err error) {
-	var store querier.ChunkStore
-
-	if cfg.Storage.Engine == storage.StorageEngineTSDB {
-		store, err = querier.NewBlockQuerier(cfg.TSDB, cfg.Server.LogLevel, prometheus.DefaultRegisterer)
-		if err != nil {
-			return err
-		}
-	} else {
-		store = t.store
-	}
-
-	queryable, engine := querier.New(cfg.Querier, t.distributor, store)
+	queryable, engine := querier.New(cfg.Querier, t.distributor, t.querierChunkStore)
 	api := v1.NewAPI(
 		engine,
 		queryable,
@@ -286,6 +281,29 @@ func (t *Cortex) stopQuerier() error {
 	return nil
 }
 
+func (t *Cortex) initQuerierChunkStore(cfg *Config) error {
+	if cfg.Storage.Engine == storage.StorageEngineChunks {
+		t.querierChunkStore = t.store
+		return nil
+	}
+
+	if cfg.Storage.Engine == storage.StorageEngineTSDB {
+		store, err := querier.NewBlockQuerier(cfg.TSDB, cfg.Server.LogLevel, prometheus.DefaultRegisterer)
+		if err != nil {
+			return err
+		}
+
+		t.querierChunkStore = store
+		return nil
+	}
+
+	return fmt.Errorf("unknown storage engine '%s'", cfg.Storage.Engine)
+}
+
+func (t *Cortex) stopQuerierChunkStore() error {
+	return nil
+}
+
 func (t *Cortex) initIngester(cfg *Config) (err error) {
 	cfg.Ingester.LifecyclerConfig.RingConfig.KVStore.Multi.ConfigProvider = multiClientRuntimeConfigChannel(t.runtimeConfig)
 	cfg.Ingester.LifecyclerConfig.ListenPort = &cfg.Server.GRPCListenPort
@@ -413,7 +431,7 @@ func (t *Cortex) stopTableManager() error {
 
 func (t *Cortex) initRuler(cfg *Config) (err error) {
 	cfg.Ruler.Ring.ListenPort = cfg.Server.GRPCListenPort
-	queryable, engine := querier.New(cfg.Querier, t.distributor, t.store)
+	queryable, engine := querier.New(cfg.Querier, t.distributor, t.querierChunkStore)
 
 	t.ruler, err = ruler.NewRuler(cfg.Ruler, engine, queryable, t.distributor, prometheus.DefaultRegisterer, util.Logger)
 	if err != nil {
@@ -527,11 +545,17 @@ var modules = map[moduleName]module{
 	},
 
 	Querier: {
-		deps: []moduleName{Distributor, Store, Ring, Server},
+		deps: []moduleName{Distributor, Store, Ring, Server, QuerierChunkStore},
 		init: (*Cortex).initQuerier,
 		stop: (*Cortex).stopQuerier,
 	},
 
+	QuerierChunkStore: {
+		deps: []moduleName{Store},
+		init: (*Cortex).initQuerierChunkStore,
+		stop: (*Cortex).stopQuerierChunkStore,
+	},
+
 	QueryFrontend: {
 		deps: []moduleName{Server, Overrides},
 		init: (*Cortex).initQueryFrontend,
@@ -545,7 +569,7 @@ var modules = map[moduleName]module{
 	},
 
 	Ruler: {
-		deps: []moduleName{Distributor, Store},
+		deps: []moduleName{Distributor, Store, QuerierChunkStore},
 		init: (*Cortex).initRuler,
 		stop: (*Cortex).stopRuler,
 	},