Skip to content

Add new metric for store gateway chunk refetches #5532

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Aug 25, 2023
Merged

Add new metric for store gateway chunk refetches #5532

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
70dd253
add new metric cortex_bucket_store_chunk_refetches_total for store ga…
yeya24 Aug 24, 2023
cd62bc5
add new cortex_bucket_store_chunk_refetches_total
yeya24 Aug 24, 2023
1c82c36
update changelog
yeya24 Aug 24, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@
* [ENHANCEMENT] Querier: Retry store gateway on different zones when zone awareness is enabled. #5476
* [ENHANCEMENT] DDBKV: Change metric name from dynamodb_kv_read_capacity_total to dynamodb_kv_consumed_capacity_total and include Delete, Put, Batch dimension. #5481
* [ENHANCEMENT] Compactor: allow unregisteronshutdown to be configurable. #5503
* [ENHANCEMENT] Store Gateway: add metric `cortex_bucket_store_chunk_refetches_total` for number of chunk refetches. #5532
* [BUGFIX] Ruler: Validate if rule group can be safely converted back to rule group yaml from protobuf message #5265
* [BUGFIX] Querier: Convert gRPC `ResourceExhausted` status code from store gateway to 422 limit error. #5286
* [BUGFIX] Alertmanager: Route web-ui requests to the alertmanager distributor when sharding is enabled. #5293
Expand Down
7 changes: 7 additions & 0 deletions pkg/storegateway/bucket_store_metrics.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ type BucketStoreMetrics struct {
seriesGetAllDuration *prometheus.Desc
seriesMergeDuration *prometheus.Desc
seriesRefetches *prometheus.Desc
chunkRefetches *prometheus.Desc
resultSeriesCount *prometheus.Desc
queriesDropped *prometheus.Desc
chunkSizeBytes *prometheus.Desc
Expand Down Expand Up @@ -105,6 +106,10 @@ func NewBucketStoreMetrics() *BucketStoreMetrics {
"cortex_bucket_store_series_refetches_total",
"Total number of cases where the built-in max series size was not enough to fetch series from index, resulting in refetch.",
nil, nil),
chunkRefetches: prometheus.NewDesc(
"cortex_bucket_store_chunk_refetches_total",
"Total number of cases where configured estimated chunk bytes was not enough was to fetch chunks from object store, resulting in refetch.",
nil, nil),
resultSeriesCount: prometheus.NewDesc(
"cortex_bucket_store_series_result_series",
"Number of series observed in the final result of a query.",
Expand Down Expand Up @@ -205,6 +210,7 @@ func (m *BucketStoreMetrics) Describe(out chan<- *prometheus.Desc) {
out <- m.seriesGetAllDuration
out <- m.seriesMergeDuration
out <- m.seriesRefetches
out <- m.chunkRefetches
out <- m.resultSeriesCount
out <- m.queriesDropped
out <- m.chunkSizeBytes
Expand Down Expand Up @@ -247,6 +253,7 @@ func (m *BucketStoreMetrics) Collect(out chan<- prometheus.Metric) {
data.SendSumOfHistograms(out, m.seriesGetAllDuration, "thanos_bucket_store_series_get_all_duration_seconds")
data.SendSumOfHistograms(out, m.seriesMergeDuration, "thanos_bucket_store_series_merge_duration_seconds")
data.SendSumOfCounters(out, m.seriesRefetches, "thanos_bucket_store_series_refetches_total")
data.SendSumOfCounters(out, m.chunkRefetches, "thanos_bucket_store_chunk_refetches_total")
data.SendSumOfHistograms(out, m.resultSeriesCount, "thanos_bucket_store_series_result_series")
data.SendSumOfCounters(out, m.queriesDropped, "thanos_bucket_store_queries_dropped_total")
data.SendSumOfHistograms(out, m.chunkSizeBytes, "thanos_bucket_store_sent_chunk_size_bytes")
Expand Down
9 changes: 9 additions & 0 deletions pkg/storegateway/bucket_store_metrics_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -393,6 +393,10 @@ func TestBucketStoreMetrics(t *testing.T) {
# TYPE cortex_bucket_store_cached_postings_original_size_bytes_total counter
cortex_bucket_store_cached_postings_original_size_bytes_total 1261064

# HELP cortex_bucket_store_chunk_refetches_total Total number of cases where configured estimated chunk bytes was not enough was to fetch chunks from object store, resulting in refetch.
# TYPE cortex_bucket_store_chunk_refetches_total counter
cortex_bucket_store_chunk_refetches_total 0

# HELP cortex_bucket_store_cached_postings_compressed_size_bytes_total Compressed size of postings stored into cache.
# TYPE cortex_bucket_store_cached_postings_compressed_size_bytes_total counter
cortex_bucket_store_cached_postings_compressed_size_bytes_total 1283583
Expand Down Expand Up @@ -634,6 +638,7 @@ type mockedBucketStoreMetrics struct {
seriesGetAllDuration prometheus.Histogram
seriesMergeDuration prometheus.Histogram
seriesRefetches prometheus.Counter
chunkRefetches prometheus.Counter
resultSeriesCount prometheus.Histogram
chunkSizeBytes prometheus.Histogram
postingsSizeBytes prometheus.Histogram
Expand Down Expand Up @@ -748,6 +753,10 @@ func newMockedBucketStoreMetrics(reg prometheus.Registerer) *mockedBucketStoreMe
Name: "thanos_bucket_store_series_refetches_total",
Help: fmt.Sprintf("Total number of cases where %v bytes was not enough was to fetch series from index, resulting in refetch.", 64*1024),
})
m.chunkRefetches = promauto.With(reg).NewCounter(prometheus.CounterOpts{
Name: "thanos_bucket_store_chunk_refetches_total",
Help: "Total number of cases where configured estimated chunk bytes was not enough was to fetch chunks from object store, resulting in refetch",
})

m.cachedPostingsCompressions = promauto.With(reg).NewCounterVec(prometheus.CounterOpts{
Name: "thanos_bucket_store_cached_postings_compressions_total",
Expand Down