querier panics when query exemplars #4469
Description
Describe the bug
querier panics when query exemplars
To Reproduce
- set ShardByAllLabels = false for querier
- call /api/v1/query_exemplars
Expected behavior
querier dose not panic
Environment:
- Kubernetes
Storage Engine
- Blocks
Additional Context
cortex/pkg/distributor/query.go
Line 62 in 74055d8
replicationSet, err := d.GetIngestersForQuery(ctx, nil)
should not pass nil here. will result in panics later in MetricNameMatcherFromMatchers
cortex/pkg/util/extract/extract.go
Line 58 in 74055d8
stack info:
panic: runtime error: invalid memory address or nil pointer dereference [recovered]
panic: runtime error: invalid memory address or nil pointer dereference
[signal SIGSEGV: segmentation violation code=0x1 addr=0x8 pc=0x14497b5]
goroutine 1295 [running]:
github.com/opentracing-contrib/go-stdlib/nethttp.MiddlewareFunc.func5.1(0xc000a1fc08, 0x2d46780, 0xc00113f560)
/data/fuxi_ci_workspace/612d9ff956bed87ff117ab16/cmd/pkg/mod/github.com/opentracing-contrib/go-stdlib@v1.0.0/nethttp/server.go:150 +0x1ab
panic(0x242fb40, 0x3e2f400)
/data/fuxi_ci_workspace/612d9ff956bed87ff117ab16_cache/buildbox/go-1.16.3/src/runtime/panic.go:965 +0x1b9
github.com/cortexproject/cortex/pkg/util/extract.MetricNameMatcherFromMatchers(0xc000942578, 0x1, 0x1, 0xc000af2e80, 0x20, 0x0, 0x0, 0x14f9b45)
/data/fuxi_ci_workspace/612d9ff956bed87ff117ab16/pkg/util/extract/extract.go:58 +0x75
github.com/cortexproject/cortex/pkg/distributor.(*Distributor).GetIngestersForQuery(0xc000664000, 0x2d18a20, 0xc0010b66c0, 0xc000942578, 0x1, 0x1, 0x0, 0x0, 0x0, 0x0, ...)
/data/fuxi_ci_workspace/612d9ff956bed87ff117ab16/pkg/distributor/query.go:138 +0x133
github.com/cortexproject/cortex/pkg/distributor.(*Distributor).QueryExemplars.func1(0x2d18a20, 0xc0010b66c0, 0x1a, 0xc0469b556a0f3025)
/data/fuxi_ci_workspace/612d9ff956bed87ff117ab16/pkg/distributor/query.go:67 +0x119
github.com/weaveworks/common/instrument.CollectedRequest(0x2d18978, 0xc001159bc0, 0x28526e8, 0x1a, 0x2d07e30, 0xc0001a1338, 0x29b7fd0, 0xc000942790, 0x10, 0xc000b45b30)
/data/fuxi_ci_workspace/612d9ff956bed87ff117ab16/cmd/pkg/mod/github.com/weaveworks/common@v0.0.0-20210419092856-009d1eebd624/instrument/instrument.go:152 +0x271
github.com/cortexproject/cortex/pkg/distributor.(*Distributor).QueryExemplars(0xc000664000, 0x2d18978, 0xc001159bc0, 0x17bca307ae0, 0x17bca676960, 0xc000a1fd40, 0x1, 0x1, 0xc0007a0600, 0x20, ...)
/data/fuxi_ci_workspace/612d9ff956bed87ff117ab16/pkg/distributor/query.go:60 +0x125
github.com/cortexproject/cortex/pkg/querier.(*distributorExemplarQuerier).Select(0xc0007a0600, 0x17bca307ae0, 0x17bca676960, 0xc000a1fd40, 0x1, 0x1, 0x0, 0xed8cbec9c, 0x0, 0x0, ...)
/data/fuxi_ci_workspace/612d9ff956bed87ff117ab16/pkg/querier/distributor_queryable.go:228 +0x9e
github.com/prometheus/prometheus/web/api/v1.(*API).queryExemplars(0xc000b49440, 0xc00031f800, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/data/fuxi_ci_workspace/612d9ff956bed87ff117ab16/cmd/pkg/mod/github.com/prometheus/prometheus@v1.8.2-0.20210510213326-e313ffa8abf6/web/api/v1/api.go:509 +0x9cb
github.com/prometheus/prometheus/web/api/v1.(*API).Register.func1.1(0x2d06c60, 0xc0007a0540, 0xc00031f800)
/data/fuxi_ci_workspace/612d9ff956bed87ff117ab16/cmd/pkg/mod/github.com/prometheus/prometheus@v1.8.2-0.20210510213326-e313ffa8abf6/web/api/v1/api.go:264 +0xa5
net/http.HandlerFunc.ServeHTTP(0xc000b77c50, 0x2d06c60, 0xc0007a0540, 0xc00031f800)
/data/fuxi_ci_workspace/612d9ff956bed87ff117ab16_cache/buildbox/go-1.16.3/src/net/http/server.go:2069 +0x44
github.com/prometheus/prometheus/util/httputil.CompressionHandler.ServeHTTP(0x2cd3560, 0xc000b77c50, 0x2d0c450, 0xc0007a04e0, 0xc00031f800)
/data/fuxi_ci_workspace/612d9ff956bed87ff117ab16/cmd/pkg/mod/github.com/prometheus/prometheus@v1.8.2-0.20210510213326-e313ffa8abf6/util/httputil/compression.go:90 +0x7e
github.com/prometheus/common/route.(*Router).handle.func1(0x2d0c450, 0xc0007a04e0, 0xc00031f700, 0x0, 0x0, 0x0)
/data/fuxi_ci_workspace/612d9ff956bed87ff117ab16/cmd/pkg/mod/github.com/prometheus/common@v0.26.1-0.20210603143733-6ef301f414bf/route/route.go:83 +0x27f
github.com/julienschmidt/httprouter.(*Router).ServeHTTP(0xc000b7bf80, 0x2d0c450, 0xc0007a04e0, 0xc00031f700)
/data/fuxi_ci_workspace/612d9ff956bed87ff117ab16/cmd/pkg/mod/github.com/julienschmidt/httprouter@v1.3.0/router.go:387 +0xc7e
github.com/prometheus/common/route.(*Router).ServeHTTP(0xc000b78e20, 0x2d0c450, 0xc0007a04e0, 0xc00031f700)
/data/fuxi_ci_workspace/612d9ff956bed87ff117ab16/cmd/pkg/mod/github.com/prometheus/common@v0.26.1-0.20210603143733-6ef301f414bf/route/route.go:121 +0x4c
github.com/cortexproject/cortex/pkg/api.getHTTPCacheGenNumberHeaderSetterMiddleware.func1.1(0x2d0c450, 0xc0007a04e0, 0xc00031f700)
/data/fuxi_ci_workspace/612d9ff956bed87ff117ab16/pkg/api/middlewares.go:26 +0x206
net/http.HandlerFunc.ServeHTTP(0xc0007a0320, 0x2d0c450, 0xc0007a04e0, 0xc00031f700)
/data/fuxi_ci_workspace/612d9ff956bed87ff117ab16_cache/buildbox/go-1.16.3/src/net/http/server.go:2069 +0x44
github.com/weaveworks/common/middleware.Instrument.Wrap.func1.2(0x2d0c450, 0xc0007a04e0)
/data/fuxi_ci_workspace/612d9ff956bed87ff117ab16/cmd/pkg/mod/github.com/weaveworks/common@v0.0.0-20210419092856-009d1eebd624/middleware/instrument.go:68 +0x4c
github.com/felixge/httpsnoop.CaptureMetricsFn(0x2d0c450, 0xc0007a02e0, 0xc000549300, 0x2, 0x2d41c90, 0xc001159b80)
/data/fuxi_ci_workspace/612d9ff956bed87ff117ab16/cmd/pkg/mod/github.com/felixge/httpsnoop@v1.0.1/capture_metrics.go:81 +0x24b
github.com/weaveworks/common/middleware.Instrument.Wrap.func1(0x2d0c450, 0xc0007a02e0, 0xc00031f700)
/data/fuxi_ci_workspace/612d9ff956bed87ff117ab16/cmd/pkg/mod/github.com/weaveworks/common@v0.0.0-20210419092856-009d1eebd624/middleware/instrument.go:67 +0x325
net/http.HandlerFunc.ServeHTTP(0xc0009a84b0, 0x2d0c450, 0xc0007a02e0, 0xc00031f700)
/data/fuxi_ci_workspace/612d9ff956bed87ff117ab16_cache/buildbox/go-1.16.3/src/net/http/server.go:2069 +0x44
github.com/gorilla/mux.(*Router).ServeHTTP(0xc0008b4300, 0x2d0c450, 0xc0007a02e0, 0xc00031f400)
/data/fuxi_ci_workspace/612d9ff956bed87ff117ab16/cmd/pkg/mod/github.com/gorilla/mux@v1.7.3/mux.go:212 +0xd3
github.com/cortexproject/cortex/pkg/querier/stats.WallTimeMiddleware.Wrap.func1(0x2d0c450, 0xc0007a02e0, 0xc00031f400)
/data/fuxi_ci_workspace/612d9ff956bed87ff117ab16/pkg/querier/stats/time_middleware.go:25 +0xca
net/http.HandlerFunc.ServeHTTP(0xc000690d80, 0x2d0c450, 0xc0007a02e0, 0xc00031f400)
/data/fuxi_ci_workspace/612d9ff956bed87ff117ab16_cache/buildbox/go-1.16.3/src/net/http/server.go:2069 +0x44
github.com/weaveworks/common/middleware.glob..func1.1(0x2d0c450, 0xc0007a02e0, 0xc00031f300)
/data/fuxi_ci_workspace/612d9ff956bed87ff117ab16/cmd/pkg/mod/github.com/weaveworks/common@v0.0.0-20210419092856-009d1eebd624/middleware/http_auth.go:17 +0x147
net/http.HandlerFunc.ServeHTTP(0xc000690e28, 0x2d0c450, 0xc0007a02e0, 0xc00031f300)
/data/fuxi_ci_workspace/612d9ff956bed87ff117ab16_cache/buildbox/go-1.16.3/src/net/http/server.go:2069 +0x44
github.com/gorilla/mux.(*Router).ServeHTTP(0xc0008b4a80, 0x2d0c450, 0xc0007a02e0, 0xc00031f100)
/data/fuxi_ci_workspace/612d9ff956bed87ff117ab16/cmd/pkg/mod/github.com/gorilla/mux@v1.7.3/mux.go:212 +0xd3
github.com/weaveworks/common/middleware.Instrument.Wrap.func1.2(0x2d0c450, 0xc0007a02e0)
/data/fuxi_ci_workspace/612d9ff956bed87ff117ab16/cmd/pkg/mod/github.com/weaveworks/common@v0.0.0-20210419092856-009d1eebd624/middleware/instrument.go:68 +0x4c
github.com/felixge/httpsnoop.CaptureMetricsFn(0x2d07e60, 0xc0009a8410, 0xc000549990, 0x2, 0x2d41c90, 0xc001159b40)
/data/fuxi_ci_workspace/612d9ff956bed87ff117ab16/cmd/pkg/mod/github.com/felixge/httpsnoop@v1.0.1/capture_metrics.go:81 +0x24b
github.com/weaveworks/common/middleware.Instrument.Wrap.func1(0x2d07e60, 0xc0009a8410, 0xc00031f100)
/data/fuxi_ci_workspace/612d9ff956bed87ff117ab16/cmd/pkg/mod/github.com/weaveworks/common@v0.0.0-20210419092856-009d1eebd624/middleware/instrument.go:67 +0x325
net/http.HandlerFunc.ServeHTTP(0xc000a18960, 0x2d07e60, 0xc0009a8410, 0xc00031f100)
/data/fuxi_ci_workspace/612d9ff956bed87ff117ab16_cache/buildbox/go-1.16.3/src/net/http/server.go:2069 +0x44
github.com/weaveworks/common/middleware.Log.Wrap.func1(0x2d0c150, 0xc0007a0220, 0xc00031f100)
/data/fuxi_ci_workspace/612d9ff956bed87ff117ab16/cmd/pkg/mod/github.com/weaveworks/common@v0.0.0-20210419092856-009d1eebd624/middleware/logging.go:52 +0x1a9
net/http.HandlerFunc.ServeHTTP(0xc000959000, 0x2d0c150, 0xc0007a0220, 0xc00031f100)
/data/fuxi_ci_workspace/612d9ff956bed87ff117ab16_cache/buildbox/go-1.16.3/src/net/http/server.go:2069 +0x44
github.com/opentracing-contrib/go-stdlib/nethttp.MiddlewareFunc.func5(0x2d0a5f0, 0xc001159a40, 0xc00031f000)
/data/fuxi_ci_workspace/612d9ff956bed87ff117ab16/cmd/pkg/mod/github.com/opentracing-contrib/go-stdlib@v1.0.0/nethttp/server.go:154 +0x5e4
net/http.HandlerFunc.ServeHTTP(0xc000959040, 0x2d0a5f0, 0xc001159a40, 0xc00031f000)
/data/fuxi_ci_workspace/612d9ff956bed87ff117ab16_cache/buildbox/go-1.16.3/src/net/http/server.go:2069 +0x44
github.com/weaveworks/common/httpgrpc/server.Server.Handle(0x2cd3560, 0xc000959040, 0x2d18a20, 0xc000515710, 0xc0009a8280, 0xc000515710, 0x231c1e0, 0x3ddfd60)
/data/fuxi_ci_workspace/612d9ff956bed87ff117ab16/cmd/pkg/mod/github.com/weaveworks/common@v0.0.0-20210419092856-009d1eebd624/httpgrpc/server/server.go:61 +0x2f1
github.com/cortexproject/cortex/pkg/querier/worker.(*frontendProcessor).runRequest(0xc000b26e40, 0x2d18978, 0xc0000d4580, 0xc0009a8280, 0x1, 0xc000a1fbc0)
/data/fuxi_ci_workspace/612d9ff956bed87ff117ab16/pkg/querier/worker/frontend_processor.go:123 +0x89
created by github.com/cortexproject/cortex/pkg/querier/worker.(*frontendProcessor).process
/data/fuxi_ci_workspace/612d9ff956bed87ff117ab16/pkg/querier/worker/frontend_processor.go:98 +0x16a