Emit max concurrency metric

Signed-off-by: 🌲 Harry 🌊 John 🏔 <johrry@amazon.com>

Author: harry671003

CHANGELOG.md

@@ -14,6 +14,7 @@
 * [ENHANCEMENT] Update Go version to 1.20.4. #5299
 * [ENHANCEMENT] Log: Avoid expensive log.Valuer evaluation for disallowed levels. #5297
 * [ENHANCEMENT] Improving Performance on the API Gzip Handler. #5347
+* [ENHANCEMENT] Emit querier `max_concurrent` as a metric. #5362
 * [BUGFIX] Ruler: Validate if rule group can be safely converted back to rule group yaml from protobuf message #5265
 * [BUGFIX] Querier: Convert gRPC `ResourceExhausted` status code from store gateway to 422 limit error. #5286
 * [BUGFIX] Alertmanager: Route web-ui requests to the alertmanager distributor when sharding is enabled. #5293

pkg/querier/querier.go

@@ -12,6 +12,7 @@ import (
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 	"github.com/prometheus/common/model"
 	"github.com/prometheus/prometheus/model/labels"
 	"github.com/prometheus/prometheus/promql"
@@ -173,6 +174,14 @@ func New(cfg Config, limits *validation.Overrides, distributor Distributor, stor
 		return lazyquery.NewLazyQuerier(querier), nil
 	})
 
+	// Emit max_concurrent config as a metric.
+	maxConcurrentMetric := promauto.With(reg).NewGauge(prometheus.GaugeOpts{
+		Namespace: "cortex",
+		Name:      "max_concurrent_queries",
+		Help:      "The maximum number of concurrent queries.",
+	})
+	maxConcurrentMetric.Set(float64(cfg.MaxConcurrent))
+
 	var queryEngine v1.QueryEngine
 	opts := promql.EngineOpts{
 		Logger:               logger,

pkg/querier/querier_test.go

@@ -38,6 +38,9 @@ import (
 	"github.com/cortexproject/cortex/pkg/util/chunkcompat"
 	"github.com/cortexproject/cortex/pkg/util/flagext"
 	"github.com/cortexproject/cortex/pkg/util/validation"
+
+	"github.com/prometheus/client_golang/prometheus"
+	promutil "github.com/prometheus/client_golang/prometheus/testutil"
 )
 
 const (
@@ -363,6 +366,28 @@ func TestQuerier(t *testing.T) {
 	}
 }
 
+func TestQuerierMetric(t *testing.T) {
+	var cfg Config
+	flagext.DefaultValues(&cfg)
+	cfg.MaxConcurrent = 120
+
+	overrides, err := validation.NewOverrides(DefaultLimitsConfig(), nil)
+	require.NoError(t, err)
+
+	chunkStore, through := makeMockChunkStore(t, 24, promchunk.PrometheusXorChunk)
+	distributor := mockDistibutorFor(t, chunkStore, through)
+
+	queryables := []QueryableWithFilter{}
+	r := prometheus.NewRegistry()
+	reg := prometheus.WrapRegistererWith(prometheus.Labels{"engine": "querier"}, r)
+	New(cfg, overrides, distributor, queryables, purger.NewNoopTombstonesLoader(), reg, log.NewNopLogger())
+	assert.NoError(t, promutil.GatherAndCompare(r, strings.NewReader(`
+		# HELP cortex_max_concurrent_queries The maximum number of concurrent queries.
+		# TYPE cortex_max_concurrent_queries gauge
+		cortex_max_concurrent_queries{engine="querier"} 120
+	`), "cortex_max_concurrent_queries"))
+}
+
 func mockTSDB(t *testing.T, labels []labels.Labels, mint model.Time, samples int, step, chunkOffset time.Duration, samplesPerChunk int) (storage.Queryable, []cortexpb.Sample) {
 	//parallel testing causes data race
 	opts := tsdb.DefaultHeadOptions()