Skip to content

Commit 02a62ed

Browse files
erlan-zerlan-z
committed
retrieve peakSamples and processedSamples query stats from results_cache as well
Signed-off-by: Erlan Zholdubai uulu <erlanz@amazon.com>
1 parent 5b53a48 commit 02a62ed

12 files changed

+159
-11
lines changed

‎.config/go/telemetry/local/asm@go1.23.2-go1.23.2-linux-arm64-2025-02-14.v1.count

16 KB
Binary file not shown.

‎.config/go/telemetry/local/compile@go1.23.2-go1.23.2-linux-arm64-2025-02-14.v1.count

16 KB
Binary file not shown.

‎.config/go/telemetry/local/go@go1.23.2-go1.23.2-linux-arm64-2025-02-14.v1.count

16 KB
Binary file not shown.

‎.config/go/telemetry/local/link@go1.23.2-go1.23.2-linux-arm64-2025-02-14.v1.count

16 KB
Binary file not shown.

‎.config/go/telemetry/local/local.2025-02-14.json

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
{
2+
"Week": "2025-02-14",
3+
"LastWeek": "",
4+
"X": 0.7210633463536673,
5+
"Programs": [
6+
{
7+
"Program": "cmd/asm",
8+
"Version": "go1.23.2",
9+
"GoVersion": "go1.23.2",
10+
"GOOS": "linux",
11+
"GOARCH": "arm64",
12+
"Counters": {
13+
"asm/flag:D": 255,
14+
"asm/flag:I": 255,
15+
"asm/flag:gensymabis": 88,
16+
"asm/flag:o": 255,
17+
"asm/flag:p": 255,
18+
"asm/flag:trimpath": 255,
19+
"asm/invocations": 255
20+
},
21+
"Stacks": {}
22+
},
23+
{
24+
"Program": "cmd/compile",
25+
"Version": "go1.23.2",
26+
"GoVersion": "go1.23.2",
27+
"GOOS": "linux",
28+
"GOARCH": "arm64",
29+
"Counters": {
30+
"compile/flag:asmhdr": 88,
31+
"compile/flag:buildid": 2875,
32+
"compile/flag:c": 2875,
33+
"compile/flag:complete": 2769,
34+
"compile/flag:embedcfg": 8,
35+
"compile/flag:goversion": 2875,
36+
"compile/flag:importcfg": 2875,
37+
"compile/flag:lang": 2875,
38+
"compile/flag:nolocalimports": 2875,
39+
"compile/flag:o": 2875,
40+
"compile/flag:p": 2875,
41+
"compile/flag:pack": 2875,
42+
"compile/flag:std": 392,
43+
"compile/flag:symabis": 88,
44+
"compile/flag:trimpath": 2875,
45+
"compile/invocations": 3285
46+
},
47+
"Stacks": {}
48+
},
49+
{
50+
"Program": "cmd/go",
51+
"Version": "go1.23.2",
52+
"GoVersion": "go1.23.2",
53+
"GOOS": "linux",
54+
"GOARCH": "arm64",
55+
"Counters": {
56+
"go/build/flag:ldflags": 42,
57+
"go/build/flag:o": 42,
58+
"go/build/flag:tags": 42,
59+
"go/buildcache/hit": 49952,
60+
"go/buildcache/miss": 2915,
61+
"go/buildcache/stdlib-recompiled": 2,
62+
"go/goroot:usr-local-go": 42,
63+
"go/invocations": 42,
64+
"go/mode:module": 42,
65+
"go/platform/host/linux/major-version:6": 42,
66+
"go/platform/host/linux/version:6-10": 42,
67+
"go/platform/target/goamd64:v1": 21,
68+
"go/platform/target/goarch:amd64": 21,
69+
"go/platform/target/goarch:arm64": 21,
70+
"go/platform/target/goarm64:v8.0": 21,
71+
"go/platform/target/goos:linux": 42,
72+
"go/subcommand:build": 42
73+
},
74+
"Stacks": {}
75+
},
76+
{
77+
"Program": "cmd/link",
78+
"Version": "go1.23.2",
79+
"GoVersion": "go1.23.2",
80+
"GOOS": "linux",
81+
"GOARCH": "arm64",
82+
"Counters": {
83+
"link/flag:X": 40,
84+
"link/flag:buildid": 40,
85+
"link/flag:buildmode": 40,
86+
"link/flag:extld": 40,
87+
"link/flag:extldflags": 40,
88+
"link/flag:importcfg": 40,
89+
"link/flag:o": 40,
90+
"link/flag:s": 40,
91+
"link/flag:w": 40,
92+
"link/invocations": 82
93+
},
94+
"Stacks": {}
95+
}
96+
],
97+
"Config": "v0.0.0-0"
98+
}

‎.config/go/telemetry/local/upload.token

Whitespace-only changes.

‎.config/go/telemetry/local/weekends

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
5

‎pkg/frontend/transport/handler.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -409,6 +409,7 @@ func (f *Handler) reportQueryStats(r *http.Request, source, userID string, query
409409
"split_queries", splitQueries,
410410
"status_code", statusCode,
411411
"response_size", contentLength,
412+
"samples_processed", numScannedSamples,
412413
}, stats.LoadExtraFields()...)
413414

414415
if numStoreGatewayTouchedPostings > 0 {
@@ -458,7 +459,7 @@ func (f *Handler) reportQueryStats(r *http.Request, source, userID string, query
458459
if error != nil {
459460
level.Error(util_log.WithContext(r.Context(), f.log)).Log(logMessage...)
460461
} else {
461-
level.Info(util_log.WithContext(r.Context(), f.log)).Log(logMessage...)
462+
level.Error(util_log.WithContext(r.Context(), f.log)).Log(logMessage...)
462463
}
463464

464465
var reason string

‎pkg/querier/tripperware/queryrange/results_cache.go

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"context"
55
"flag"
66
"fmt"
7+
"github.com/cortexproject/cortex/pkg/querier/stats"
78
"net/http"
89
"slices"
910
"sort"
@@ -439,7 +440,7 @@ func (s resultsCache) handleHit(ctx context.Context, r tripperware.Request, exte
439440

440441
level.Debug(util_log.WithContext(ctx, log)).Log("msg", "handle hit", "start", r.GetStart(), "spanID", jaegerSpanID(ctx))
441442

442-
requests, responses, err := s.partition(r, extents)
443+
requests, responses, err := s.partition(ctx, r, extents)
443444
if err != nil {
444445
return nil, nil, err
445446
}
@@ -636,7 +637,7 @@ func convertFromTripperwarePrometheusResponse(resp tripperware.Response) tripper
636637

637638
// partition calculates the required requests to satisfy req given the cached data.
638639
// extents must be in order by start time.
639-
func (s resultsCache) partition(req tripperware.Request, extents []tripperware.Extent) ([]tripperware.Request, []tripperware.Response, error) {
640+
func (s resultsCache) partition(ctx context.Context, req tripperware.Request, extents []tripperware.Extent) ([]tripperware.Request, []tripperware.Response, error) {
640641
var requests []tripperware.Request
641642
var cachedResponses []tripperware.Response
642643
start := req.GetStart()
@@ -667,7 +668,14 @@ func (s resultsCache) partition(req tripperware.Request, extents []tripperware.E
667668
return nil, nil, err
668669
}
669670
// extract the overlap from the cached extent.
670-
cachedResponses = append(cachedResponses, s.extractor.Extract(start, req.GetEnd(), res))
671+
promRes := s.extractor.Extract(start, req.GetEnd(), res).(*tripperware.PrometheusResponse)
672+
cachedResponses = append(cachedResponses, promRes)
673+
674+
if queryStats := stats.FromContext(ctx); queryStats != nil && promRes.Data.Stats != nil {
675+
queryStats.AddScannedSamples(uint64(promRes.Data.Stats.Samples.TotalQueryableSamples))
676+
queryStats.SetPeakSamples(max(queryStats.LoadPeakSamples(), uint64(promRes.Data.Stats.Samples.PeakSamples)))
677+
}
678+
671679
start = extent.End
672680
}
673681

@@ -796,6 +804,7 @@ func extractStats(start, end int64, stats *tripperware.PrometheusResponseStats)
796804
if start <= s.TimestampMs && s.TimestampMs <= end {
797805
result.Samples.TotalQueryableSamplesPerStep = append(result.Samples.TotalQueryableSamplesPerStep, s)
798806
result.Samples.TotalQueryableSamples += s.Value
807+
result.Samples.PeakSamples = max(result.Samples.PeakSamples, s.Value)
799808
}
800809
}
801810
return result

‎pkg/querier/tripperware/queryrange/results_cache_test.go

Lines changed: 38 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package queryrange
33
import (
44
"context"
55
"fmt"
6+
querier_stats "github.com/cortexproject/cortex/pkg/querier/stats"
67
"net/http"
78
"strconv"
89
"testing"
@@ -173,6 +174,7 @@ func mkAPIResponseWithStats(start, end, step int64, withStats bool, oldFormat bo
173174
})
174175

175176
stats.Samples.TotalQueryableSamples += i
177+
stats.Samples.PeakSamples = max(stats.Samples.PeakSamples, i)
176178
}
177179
}
178180

@@ -597,11 +599,13 @@ func TestShouldCache(t *testing.T) {
597599
func TestPartition(t *testing.T) {
598600
t.Parallel()
599601
for _, tc := range []struct {
600-
name string
601-
input tripperware.Request
602-
prevCachedResponse []tripperware.Extent
603-
expectedRequests []tripperware.Request
604-
expectedCachedResponse []tripperware.Response
602+
name string
603+
input tripperware.Request
604+
prevCachedResponse []tripperware.Extent
605+
expectedRequests []tripperware.Request
606+
expectedCachedResponse []tripperware.Response
607+
expectedScannedSamplesFromCachedResponse uint64
608+
expectedPeakSamplesFromCachedResponse uint64
605609
}{
606610
{
607611
name: "Test a complete hit.",
@@ -822,6 +826,8 @@ func TestPartition(t *testing.T) {
822826
expectedCachedResponse: []tripperware.Response{
823827
mkAPIResponseWithStats(0, 100, 10, true, false),
824828
},
829+
expectedPeakSamplesFromCachedResponse: getPeakSamples(0, 100, 10),
830+
expectedScannedSamplesFromCachedResponse: getScannedSamples(0, 100, 10),
825831
},
826832

827833
{
@@ -836,6 +842,8 @@ func TestPartition(t *testing.T) {
836842
expectedCachedResponse: []tripperware.Response{
837843
mkAPIResponseWithStats(0, 100, 10, true, false),
838844
},
845+
expectedPeakSamplesFromCachedResponse: getPeakSamples(0, 100, 10),
846+
expectedScannedSamplesFromCachedResponse: getScannedSamples(0, 100, 10),
839847
},
840848

841849
{
@@ -886,6 +894,8 @@ func TestPartition(t *testing.T) {
886894
expectedCachedResponse: []tripperware.Response{
887895
mkAPIResponseWithStats(50, 100, 10, true, false),
888896
},
897+
expectedPeakSamplesFromCachedResponse: getPeakSamples(50, 100, 10),
898+
expectedScannedSamplesFromCachedResponse: getScannedSamples(50, 100, 10),
889899
},
890900
{
891901
name: "[stats] Test multiple partial hits.",
@@ -907,6 +917,8 @@ func TestPartition(t *testing.T) {
907917
mkAPIResponseWithStats(100, 120, 10, true, false),
908918
mkAPIResponseWithStats(160, 200, 10, true, false),
909919
},
920+
expectedPeakSamplesFromCachedResponse: max(getPeakSamples(100, 120, 10), getPeakSamples(160, 200, 10)),
921+
expectedScannedSamplesFromCachedResponse: getScannedSamples(100, 120, 10) + getScannedSamples(160, 200, 10),
910922
},
911923
{
912924
name: "[stats] Partial hits with tiny gap.",
@@ -927,7 +939,8 @@ func TestPartition(t *testing.T) {
927939
expectedCachedResponse: []tripperware.Response{
928940
mkAPIResponseWithStats(100, 120, 10, true, false),
929941
},
930-
},
942+
expectedPeakSamplesFromCachedResponse: getPeakSamples(100, 120, 10),
943+
expectedScannedSamplesFromCachedResponse: getScannedSamples(100, 120, 10)},
931944
{
932945
name: "[stats] Extent is outside the range and the request has a single step (same start and end).",
933946
input: &tripperware.PrometheusRequest{
@@ -957,6 +970,8 @@ func TestPartition(t *testing.T) {
957970
expectedCachedResponse: []tripperware.Response{
958971
mkAPIResponseWithStats(100, 105, 10, true, false),
959972
},
973+
expectedPeakSamplesFromCachedResponse: getPeakSamples(100, 105, 10),
974+
expectedScannedSamplesFromCachedResponse: getScannedSamples(100, 105, 10),
960975
},
961976
{
962977
name: "[stats] Test when hit has a large step and only a single sample extent with old format.",
@@ -971,6 +986,8 @@ func TestPartition(t *testing.T) {
971986
expectedCachedResponse: []tripperware.Response{
972987
mkAPIResponseWithStats(100, 105, 10, true, false),
973988
},
989+
expectedPeakSamplesFromCachedResponse: getPeakSamples(100, 105, 10),
990+
expectedScannedSamplesFromCachedResponse: getScannedSamples(100, 105, 10),
974991
},
975992
} {
976993
tc := tc
@@ -980,10 +997,13 @@ func TestPartition(t *testing.T) {
980997
extractor: PrometheusResponseExtractor{},
981998
minCacheExtent: 10,
982999
}
983-
reqs, resps, err := s.partition(tc.input, tc.prevCachedResponse)
1000+
stats, ctx := querier_stats.ContextWithEmptyStats(context.Background())
1001+
reqs, resps, err := s.partition(ctx, tc.input, tc.prevCachedResponse)
9841002
require.Nil(t, err)
9851003
require.Equal(t, tc.expectedRequests, reqs)
9861004
require.Equal(t, tc.expectedCachedResponse, resps)
1005+
require.Equal(t, tc.expectedScannedSamplesFromCachedResponse, stats.ScannedSamples)
1006+
require.Equal(t, tc.expectedPeakSamplesFromCachedResponse, stats.PeakSamples)
9871007
})
9881008
}
9891009
}
@@ -1580,3 +1600,14 @@ func TestResultsCacheFillCompatibility(t *testing.T) {
15801600
func toMs(t time.Duration) int64 {
15811601
return int64(t / time.Millisecond)
15821602
}
1603+
1604+
func getScannedSamples(start, end, step uint64) uint64 {
1605+
lastTerm := start + ((end-start)/step)*step
1606+
n := (lastTerm-start)/step + 1
1607+
1608+
return (n * (2*start + (n-1)*step)) / 2
1609+
}
1610+
1611+
func getPeakSamples(start, end, step uint64) uint64 {
1612+
return start + ((end-start)/step)*step
1613+
}

‎pkg/ruler/user2/namespace

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
groups:
2+
- name: first
3+
interval: 1m
4+
rules: []

‎pkg/ruler/user3/namespace

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
groups:
2+
- name: first
3+
interval: 1m
4+
rules: []

0 commit comments

Comments
 (0)