Skip to content

Fix flaky on thanos_memcached_operations_total metrics #5482

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Jul 25, 2023
Merged

Fix flaky on thanos_memcached_operations_total metrics #5482

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
cfba5a0
fix flaky on thanos_memcached_operations_total metrics
yeya24 Jul 25, 2023
4f65546
update comment
yeya24 Jul 25, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions integration/e2e/metrics.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,21 @@ func LessOrEqual(value float64) func(sums ...float64) bool {
}
}

// EqualsAmong is an isExpected function for WaitSumMetrics that returns true if the first sum is equal to any value provided.
func EqualsAmong(values ...float64) func(sums ...float64) bool {
return func(sums ...float64) bool {
if len(sums) != 1 {
panic("equals among: expected one value")
}
for _, value := range values {
if sums[0] == value {
return true
}
}
return false
}
}

// EqualsAmongTwo is an isExpected function for WaitSumMetrics that returns true if first sum is equal to the second.
// NOTE: Be careful on scrapes in between of process that changes two metrics. Those are
// usually not atomic.
Expand Down
17 changes: 13 additions & 4 deletions integration/querier_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -270,18 +270,27 @@ func TestQuerierWithBlocksStorageRunningInMicroservicesMode(t *testing.T) {
require.Equal(t, model.ValVector, result.Type())
assert.Equal(t, expectedVector1, result.(model.Vector))

var l0CacheHits float64
if numberOfCacheBackends > 1 {
// 6 requests for Expanded Postings, 5 for Postings and 3 for Series.
require.NoError(t, storeGateways.WaitSumMetricsWithOptions(e2e.Equals(float64(6+5+3)), []string{"thanos_store_index_cache_requests_total"}, e2e.WithLabelMatchers(
labels.MustNewMatcher(labels.MatchEqual, "level", "L0"),
)))
// In case of L0 cache hits, store gateway might send fewer requests. Should be within range 12 ~ 14.
require.NoError(t, storeGateways.WaitSumMetricsWithOptions(e2e.GreaterOrEqual(float64(12)), []string{"thanos_store_index_cache_requests_total"}, e2e.WithLabelMatchers(
require.NoError(t, storeGateways.WaitSumMetricsWithOptions(e2e.EqualsAmong(float64(12), float64(14)), []string{"thanos_store_index_cache_requests_total"}, e2e.WithLabelMatchers(
labels.MustNewMatcher(labels.MatchEqual, "level", "L1"),
)))
require.NoError(t, storeGateways.WaitSumMetricsWithOptions(e2e.LessOrEqual(float64(14)), []string{"thanos_store_index_cache_requests_total"}, e2e.WithLabelMatchers(
l1IndexCacheRequests, err := storeGateways.SumMetrics([]string{"thanos_store_index_cache_requests_total"}, e2e.WithLabelMatchers(
labels.MustNewMatcher(labels.MatchEqual, "level", "L1"),
)))
))
require.NoError(t, err)
l0IndexCacheHits, err := storeGateways.SumMetrics([]string{"thanos_store_index_cache_hits_total"}, e2e.WithLabelMatchers(
labels.MustNewMatcher(labels.MatchEqual, "level", "L0"),
))
require.NoError(t, err)
// Make sure l1 cache requests + l0 cache hits is 14.
require.Equal(t, float64(14), l1IndexCacheRequests[0]+l0IndexCacheHits[0])
l0CacheHits = l0IndexCacheHits[0]
} else {
// 6 requests for Expanded Postings, 5 for Postings and 3 for Series.
require.NoError(t, storeGateways.WaitSumMetrics(e2e.Equals(float64(6+5+3)), "thanos_store_index_cache_requests_total"))
Expand All @@ -293,7 +302,7 @@ func TestQuerierWithBlocksStorageRunningInMicroservicesMode(t *testing.T) {
require.NoError(t, storeGateways.WaitSumMetrics(e2e.Equals(9), "thanos_store_index_cache_items_added_total")) // as before
}
if strings.Contains(testCfg.indexCacheBackend, tsdb.IndexCacheBackendMemcached) {
require.NoError(t, storeGateways.WaitSumMetrics(e2e.Equals(23), "thanos_memcached_operations_total")) // as before + 2 gets
require.NoError(t, storeGateways.WaitSumMetrics(e2e.Equals(23-l0CacheHits), "thanos_memcached_operations_total")) // as before + 2 gets - cache hits
}

// Query metadata.
Expand Down