Update prometheus to v2.16.0

.gitignore

@@ -7,4 +7,6 @@ docker-images/
 website/public
 website/resources
 website/content/en/docs
-e2e_integration_test*
+e2e_integration_test*
+active-query-tracker
+

CHANGELOG.md

@@ -22,6 +22,7 @@
 * [CHANGE] Experimental Memberlist KV store can now be used in single-binary Cortex. Attempts to use it previously would fail with panic. This change also breaks existing binary protocol used to exchange gossip messages, so this version will not be able to understand gossiped Ring when used in combination with the previous version of Cortex. Easiest way to upgrade is to shutdown old Cortex installation, and restart it with new version. Incremental rollout works too, but with reduced functionality until all components run the same version. #2016
 * [CHANGE] Renamed the cache configuration setting `defaul_validity` to `default_validity`. #2140
 * [CHANGE] Removed unused /validate_expr endpoint. #2152
+* [CHANGE] Updated Prometheus dependency to v2.16.0. This Prometheus version uses Active Query Tracker to limit concurrent queries. In order to keep `-querier.max-concurrent` working, Active Query Tracker is enabled by default, and is configured to store its data to `active-query-tracker` directory (relative to current directory when Cortex started). This can be changed by using `-querier.active-query-tracker-dir` option. Purpose of Active Query Tracker is to log queries that were running when Cortex crashes. This logging happens on next Cortex start. #2088
 * [FEATURE] Added a read-only local alertmanager config store using files named corresponding to their tenant id. #2125
 * [FEATURE] Added user sub rings to distribute users to a subset of ingesters. #1947
   * `--experimental.distributor.user-subring-size`

docs/configuration/config-file-reference.md

@@ -569,6 +569,13 @@ The `querier_config` configures the Cortex querier.
 # The default evaluation interval or step size for subqueries.
 # CLI flag: -querier.default-evaluation-interval
 [defaultevaluationinterval: <duration> | default = 1m0s]
+
+# Active query tracker monitors active queries, and writes them to the file in
+# given directory. If Cortex discovers any queries in this log during startup,
+# it will log them to the log file. Setting to empty value disables active query
+# tracker, which also disables -querier.max-concurrent option.
+# CLI flag: -querier.active-query-tracker-dir
+[active_query_tracker_dir: <string> | default = "./active-query-tracker"]
 ```
 
 ## `query_frontend_config`

go.mod

@@ -52,8 +52,8 @@ require (
 	github.com/prometheus/alertmanager v0.19.0
 	github.com/prometheus/client_golang v1.2.1
 	github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4
-	github.com/prometheus/common v0.7.0
-	github.com/prometheus/prometheus v1.8.2-0.20200107122003-4708915ac6ef
+	github.com/prometheus/common v0.8.0
+	github.com/prometheus/prometheus v1.8.2-0.20200213233353-b90be6f32a33
 	github.com/rafaeljusto/redigomock v0.0.0-20190202135759-257e089e14a1
 	github.com/segmentio/fasthash v0.0.0-20180216231524-a72b379d632e
 	github.com/spf13/afero v1.2.2
@@ -71,7 +71,7 @@ require (
 	golang.org/x/time v0.0.0-20191024005414-555d28b269f0
 	google.golang.org/api v0.14.0
 	google.golang.org/grpc v1.25.1
-	gopkg.in/yaml.v2 v2.2.5
+	gopkg.in/yaml.v2 v2.2.7
 	sigs.k8s.io/yaml v1.1.0
 )
 

go.sum

@@ -671,6 +671,8 @@ github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8
 github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc=
 github.com/prometheus/common v0.7.0 h1:L+1lyG48J1zAQXA3RBX/nG/B3gjlHq0zTt2tlbJLyCY=
 github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA=
+github.com/prometheus/common v0.8.0 h1:bLkjvFe2ZRX1DpcgZcdf7j/+MnusEps5hktST/FHA34=
+github.com/prometheus/common v0.8.0/go.mod h1:PC/OgXc+UN7B4ALwvn1yzVZmVwvhXp5JsbBv6wSv6i0=
 github.com/prometheus/procfs v0.0.0-20180612222113-7d6f385de8be/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
@@ -686,6 +688,8 @@ github.com/prometheus/prometheus v0.0.0-20180315085919-58e2a31db8de/go.mod h1:oA
 github.com/prometheus/prometheus v0.0.0-20190818123050-43acd0e2e93f/go.mod h1:rMTlmxGCvukf2KMu3fClMDKLLoJ5hl61MhcJ7xKakf0=
 github.com/prometheus/prometheus v1.8.2-0.20200107122003-4708915ac6ef h1:pYYKXo/zGx25kyViw+Gdbxd0ItIg+vkVKpwgWUEyIc4=
 github.com/prometheus/prometheus v1.8.2-0.20200107122003-4708915ac6ef/go.mod h1:7U90zPoLkWjEIQcy/rweQla82OCTUzxVHE51G3OhJbI=
+github.com/prometheus/prometheus v1.8.2-0.20200213233353-b90be6f32a33 h1:HBYrMJj5iosUjUkAK9L5GO+5eEQXbcrzdjkqY9HV5W4=
+github.com/prometheus/prometheus v1.8.2-0.20200213233353-b90be6f32a33/go.mod h1:fkIPPkuZnkXyopYHmXPxf9rgiPkVgZCN8w9o8+UgBlY=
 github.com/rafaeljusto/redigomock v0.0.0-20190202135759-257e089e14a1 h1:+kGqA4dNN5hn7WwvKdzHl0rdN5AEkbNZd0VjRltAiZg=
 github.com/rafaeljusto/redigomock v0.0.0-20190202135759-257e089e14a1/go.mod h1:JaY6n2sDr+z2WTsXkOmNRUfDy6FN0L6Nk7x06ndm4tY=
 github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
@@ -1064,6 +1068,10 @@ gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.5 h1:ymVxjfMaHvXD8RqPRmzHHsB3VvucivSkIAvJFDI5O3c=
 gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.7 h1:VUgggvou5XRW9mHwD/yXxIYSMtY0zoKQf/v226p2nyo=
+gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.0-20191120175047-4206685974f2 h1:XZx7nhd5GMaZpmDaEHFVafUZC7ya0fuo7cSJ3UCKYmM=
+gopkg.in/yaml.v3 v3.0.0-20191120175047-4206685974f2/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
 gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
 honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

pkg/configs/configs.go

@@ -8,11 +8,11 @@ import (
 	"github.com/go-kit/kit/log"
 	"github.com/prometheus/common/model"
 	"github.com/prometheus/prometheus/pkg/labels"
-	"github.com/prometheus/prometheus/pkg/rulefmt"
 	"github.com/prometheus/prometheus/promql"
 	"github.com/prometheus/prometheus/rules"
 
 	legacy_promql "github.com/cortexproject/cortex/pkg/configs/legacy_promql"
+	legacy_rulefmt "github.com/cortexproject/cortex/pkg/ruler/legacy_rulefmt"
 	"github.com/cortexproject/cortex/pkg/util"
 )
 
@@ -185,7 +185,7 @@ func (c RulesConfig) Parse() (map[string][]rules.Rule, error) {
 
 // ParseFormatted returns the rulefmt map of a users rules configs. It allows
 // for rules to be mapped to disk and read by the prometheus rules manager.
-func (c RulesConfig) ParseFormatted() (map[string]rulefmt.RuleGroups, error) {
+func (c RulesConfig) ParseFormatted() (map[string]legacy_rulefmt.RuleGroups, error) {
 	switch c.FormatVersion {
 	case RuleFormatV1:
 		return c.parseV1Formatted()
@@ -198,11 +198,11 @@ func (c RulesConfig) ParseFormatted() (map[string]rulefmt.RuleGroups, error) {
 
 // parseV2 parses and validates the content of the rule files in a RulesConfig
 // according to the Prometheus 2.x rule format.
-func (c RulesConfig) parseV2Formatted() (map[string]rulefmt.RuleGroups, error) {
-	ruleMap := map[string]rulefmt.RuleGroups{}
+func (c RulesConfig) parseV2Formatted() (map[string]legacy_rulefmt.RuleGroups, error) {
+	ruleMap := map[string]legacy_rulefmt.RuleGroups{}
 
 	for fn, content := range c.Files {
-		rgs, errs := rulefmt.Parse([]byte(content))
+		rgs, errs := legacy_rulefmt.Parse([]byte(content))
 		for _, err := range errs { // return just the first error, if any
 			return nil, err
 		}
@@ -214,25 +214,25 @@ func (c RulesConfig) parseV2Formatted() (map[string]rulefmt.RuleGroups, error) {
 
 // parseV1 parses and validates the content of the rule files in a RulesConfig
 // according to the Prometheus 1.x rule format.
-func (c RulesConfig) parseV1Formatted() (map[string]rulefmt.RuleGroups, error) {
-	result := map[string]rulefmt.RuleGroups{}
+func (c RulesConfig) parseV1Formatted() (map[string]legacy_rulefmt.RuleGroups, error) {
+	result := map[string]legacy_rulefmt.RuleGroups{}
 	for fn, content := range c.Files {
 		stmts, err := legacy_promql.ParseStmts(content)
 		if err != nil {
 			return nil, fmt.Errorf("error parsing %s: %s", fn, err)
 		}
 
-		ra := []rulefmt.Rule{}
+		ra := []legacy_rulefmt.Rule{}
 		for _, stmt := range stmts {
-			var rule rulefmt.Rule
+			var rule legacy_rulefmt.Rule
 			switch r := stmt.(type) {
 			case *legacy_promql.AlertStmt:
 				_, err := promql.ParseExpr(r.Expr.String())
 				if err != nil {
 					return nil, err
 				}
 
-				rule = rulefmt.Rule{
+				rule = legacy_rulefmt.Rule{
 					Alert:       r.Name,
 					Expr:        r.Expr.String(),
 					For:         model.Duration(r.Duration),
@@ -246,7 +246,7 @@ func (c RulesConfig) parseV1Formatted() (map[string]rulefmt.RuleGroups, error) {
 					return nil, err
 				}
 
-				rule = rulefmt.Rule{
+				rule = legacy_rulefmt.Rule{
 					Record: r.Name,
 					Expr:   r.Expr.String(),
 					Labels: r.Labels.Map(),
@@ -257,8 +257,8 @@ func (c RulesConfig) parseV1Formatted() (map[string]rulefmt.RuleGroups, error) {
 			}
 			ra = append(ra, rule)
 		}
-		result[fn] = rulefmt.RuleGroups{
-			Groups: []rulefmt.RuleGroup{
+		result[fn] = legacy_rulefmt.RuleGroups{
+			Groups: []legacy_rulefmt.RuleGroup{
 				{
 					Name:  "rg:" + fn,
 					Rules: ra,
@@ -286,7 +286,7 @@ func (c RulesConfig) parseV2() (map[string][]rules.Rule, error) {
 	groups := map[string][]rules.Rule{}
 
 	for fn, content := range c.Files {
-		rgs, errs := rulefmt.Parse([]byte(content))
+		rgs, errs := legacy_rulefmt.Parse([]byte(content))
 		if len(errs) > 0 {
 			return nil, fmt.Errorf("error parsing %s: %v", fn, errs[0])
 		}

pkg/configs/configs_test.go

@@ -9,12 +9,12 @@ import (
 	"github.com/go-kit/kit/log"
 	"github.com/prometheus/common/model"
 	"github.com/prometheus/prometheus/pkg/labels"
-	"github.com/prometheus/prometheus/pkg/rulefmt"
 	"github.com/prometheus/prometheus/promql"
 	"github.com/prometheus/prometheus/rules"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
+	legacy_rulefmt "github.com/cortexproject/cortex/pkg/ruler/legacy_rulefmt"
 	"github.com/cortexproject/cortex/pkg/util"
 )
 
@@ -130,7 +130,7 @@ func TestParseFormatted(t *testing.T) {
 	dur, err := model.ParseDuration("5m")
 	require.NoError(t, err)
 
-	rules := []rulefmt.Rule{
+	rules := []legacy_rulefmt.Rule{
 		{
 			Alert: "TestAlert",
 			Expr:  "up == 0",
@@ -146,7 +146,7 @@ func TestParseFormatted(t *testing.T) {
 
 	for i, tc := range []struct {
 		cfg      RulesConfig
-		expected map[string]rulefmt.RuleGroups
+		expected map[string]legacy_rulefmt.RuleGroups
 	}{
 		{
 			cfg: RulesConfig{
@@ -155,9 +155,9 @@ func TestParseFormatted(t *testing.T) {
 					"legacy.rules": legacyRulesFile,
 				},
 			},
-			expected: map[string]rulefmt.RuleGroups{
+			expected: map[string]legacy_rulefmt.RuleGroups{
 				"legacy.rules": {
-					Groups: []rulefmt.RuleGroup{
+					Groups: []legacy_rulefmt.RuleGroup{
 						{
 							Name:  "rg:legacy.rules",
 							Rules: rules,
@@ -173,9 +173,9 @@ func TestParseFormatted(t *testing.T) {
 					"alerts.yaml": ruleFile,
 				},
 			},
-			expected: map[string]rulefmt.RuleGroups{
+			expected: map[string]legacy_rulefmt.RuleGroups{
 				"alerts.yaml": {
-					Groups: []rulefmt.RuleGroup{
+					Groups: []legacy_rulefmt.RuleGroup{
 						{
 							Name:  "example",
 							Rules: rules,

pkg/configs/legacy_promql/engine_test.go

@@ -132,6 +132,9 @@ type errQuerier struct {
 func (q *errQuerier) Select(*storage.SelectParams, ...*labels.Matcher) (storage.SeriesSet, storage.Warnings, error) {
 	return errSeriesSet{err: q.err}, nil, q.err
 }
+func (q *errQuerier) SelectSorted(*storage.SelectParams, ...*labels.Matcher) (storage.SeriesSet, storage.Warnings, error) {
+	return errSeriesSet{err: q.err}, nil, q.err
+}
 func (q *errQuerier) LabelValues(name string) ([]string, storage.Warnings, error) {
 	return nil, nil, q.err
 }

pkg/cortex/modules.go

@@ -3,7 +3,6 @@ package cortex
 import (
 	"errors"
 	"fmt"
-	"math"
 	"net/http"
 	"os"
 	"regexp"
@@ -262,7 +261,7 @@ func (t *Cortex) initQuerier(cfg *Config) (err error) {
 	api.Register(promRouter)
 
 	subrouter := t.server.HTTP.PathPrefix("/api/prom").Subrouter()
-	subrouter.PathPrefix("/api/v1").Handler(t.httpAuthMiddleware.Wrap(promRouter))
+	subrouter.PathPrefix("/api/v1").Handler(fakeRemoteAddr(t.httpAuthMiddleware.Wrap(promRouter)))
 	subrouter.Path("/read").Handler(t.httpAuthMiddleware.Wrap(querier.RemoteReadHandler(queryable)))
 	subrouter.Path("/chunks").Handler(t.httpAuthMiddleware.Wrap(querier.ChunksHandler(queryable)))
 	subrouter.Path("/user_stats").Handler(middleware.AuthenticateUser.Wrap(http.HandlerFunc(t.distributor.UserStatsHandler)))
@@ -283,6 +282,20 @@ func (t *Cortex) initQuerier(cfg *Config) (err error) {
 	return
 }
 
+// Latest Prometheus requires r.RemoteAddr to be set to addr:port, otherwise it reject the request.
+// Requests to Querier sometimes doesn't have that (if they are fetched from Query-Frontend).
+// Prometheus uses this when logging queries to QueryLogger, but Cortex doesn't call engine.SetQueryLogger to set one.
+//
+// Can be removed when (if) https://github.com/prometheus/prometheus/pull/6840 is merged.
+func fakeRemoteAddr(handler http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.RemoteAddr == "" {
+			r.RemoteAddr = "127.0.0.1:8888"
+		}
+		handler.ServeHTTP(w, r)
+	})
+}
+
 func (t *Cortex) stopQuerier() error {
 	t.worker.Stop()
 	return nil
@@ -374,11 +387,10 @@ func (t *Cortex) initQueryFrontend(cfg *Config) (err error) {
 		queryrange.PrometheusResponseExtractor,
 		cfg.Schema,
 		promql.EngineOpts{
-			Logger:        util.Logger,
-			Reg:           prometheus.DefaultRegisterer,
-			MaxConcurrent: int(math.MaxInt64), // the frontend's promql engine should not set any concurrency controls (these are handled by middleware)
-			MaxSamples:    cfg.Querier.MaxSamples,
-			Timeout:       cfg.Querier.Timeout,
+			Logger:     util.Logger,
+			Reg:        prometheus.DefaultRegisterer,
+			MaxSamples: cfg.Querier.MaxSamples,
+			Timeout:    cfg.Querier.Timeout,
 		},
 		cfg.Querier.QueryIngestersWithin,
 	)

pkg/querier/astmapper/astmapper_test.go

@@ -24,8 +24,8 @@ func TestCloneNode(t *testing.T) {
 			},
 			&promql.BinaryExpr{
 				Op:  promql.ADD,
-				LHS: &promql.NumberLiteral{Val: 1},
-				RHS: &promql.NumberLiteral{Val: 1},
+				LHS: &promql.NumberLiteral{Val: 1, PosRange: promql.PositionRange{Start: 0, End: 1}},
+				RHS: &promql.NumberLiteral{Val: 1, PosRange: promql.PositionRange{Start: 4, End: 5}},
 			},
 		},
 		{
@@ -48,8 +48,16 @@ func TestCloneNode(t *testing.T) {
 					LabelMatchers: []*labels.Matcher{
 						mustLabelMatcher(labels.MatchEqual, string(model.MetricNameLabel), "some_metric"),
 					},
+					PosRange: promql.PositionRange{
+						Start: 18,
+						End:   29,
+					},
 				},
 				Grouping: []string{"foo"},
+				PosRange: promql.PositionRange{
+					Start: 0,
+					End:   30,
+				},
 			},
 		},
 	}

pkg/querier/astmapper/shard_summer.go

@@ -222,15 +222,23 @@ func shardMatrixSelector(curshard, shards int, selector *promql.MatrixSelector)
 		return nil, err
 	}
 
-	return &promql.MatrixSelector{
-		Name:   selector.Name,
-		Range:  selector.Range,
-		Offset: selector.Offset,
-		LabelMatchers: append(
-			[]*labels.Matcher{shardMatcher},
-			selector.LabelMatchers...,
-		),
-	}, nil
+	if vs, ok := selector.VectorSelector.(*promql.VectorSelector); ok {
+		return &promql.MatrixSelector{
+			VectorSelector: &promql.VectorSelector{
+				Name:   vs.Name,
+				Offset: vs.Offset,
+				LabelMatchers: append(
+					[]*labels.Matcher{shardMatcher},
+					vs.LabelMatchers...,
+				),
+				PosRange: vs.PosRange,
+			},
+			Range:  selector.Range,
+			EndPos: selector.EndPos,
+		}, nil
+	}
+
+	return nil, fmt.Errorf("invalid selector type: %T", selector.VectorSelector)
 }
 
 // ParseShard will extract the shard information encoded in ShardLabelFmt

pkg/querier/astmapper/subtree_folder.go

@@ -47,10 +47,7 @@ func isEmbedded(node promql.Node) (bool, error) {
 		}
 
 	case *promql.MatrixSelector:
-		if n.Name == EmbeddedQueriesMetricName {
-			return true, nil
-		}
-
+		return isEmbedded(n.VectorSelector)
 	}
 	return false, nil
 }

pkg/querier/block.go

@@ -78,6 +78,10 @@ func (b *blocksQuerier) addUserToContext(ctx context.Context) context.Context {
 }
 
 func (b *blocksQuerier) Select(sp *storage.SelectParams, matchers ...*labels.Matcher) (storage.SeriesSet, storage.Warnings, error) {
+	return b.SelectSorted(sp, matchers...)
+}
+
+func (b *blocksQuerier) SelectSorted(sp *storage.SelectParams, matchers ...*labels.Matcher) (storage.SeriesSet, storage.Warnings, error) {
 	log, ctx := spanlogger.New(b.ctx, "blocksQuerier.Select")
 	defer log.Span.Finish()