Skip to content

Commit 6ceb38d

Browse files
ultrotterGuido Trotter
ultrotter
and
Guido Trotter
authored
compile silence matchers when the silence is added (#4695)
* compile silence matchers when the silence is added If QMatches doesn't update the map, multiple queries can execute concurrently, with just a read lock. We have had this in production for over a year, and it significantly improves performance for cases where many silences need to be processed against many alerts. This makes load time a bit slower, but a snapshot file with about 120.000 silences can still load in about 2 seconds, so we can accept this, in exchange for the ability to run queries in parallel. goos: linux goarch: amd64 pkg: github.com/prometheus/alertmanager/cluster cpu: AMD EPYC Processor (with IBPB) │ bench-before.txt │ bench-after.txt │ │ sec/op │ sec/op vs base │ WriteTo-40 0.003794n ± ∞ ¹ 0.004227n ± ∞ ¹ ~ (p=1.000 n=1) ² ¹ need >= 6 samples for confidence interval at level 0.95 ² need >= 4 samples to detect a difference at alpha level 0.05 │ bench-before.txt │ bench-after.txt │ │ B/op │ B/op vs base │ WriteTo-40 0.000 ± ∞ ¹ 0.000 ± ∞ ¹ ~ (p=1.000 n=1) ² ¹ need >= 6 samples for confidence interval at level 0.95 ² all samples are equal │ bench-before.txt │ bench-after.txt │ │ allocs/op │ allocs/op vs base │ WriteTo-40 0.000 ± ∞ ¹ 0.000 ± ∞ ¹ ~ (p=1.000 n=1) ² ¹ need >= 6 samples for confidence interval at level 0.95 ² all samples are equal pkg: github.com/prometheus/alertmanager/inhibit │ bench-before.txt │ bench-after.txt │ │ sec/op │ sec/op vs base │ Mutes/1_inhibition_rule,_1_inhibiting_alert-40 1.248µ ± ∞ ¹ 1.340µ ± ∞ ¹ ~ (p=1.000 n=1) ² Mutes/10_inhibition_rules,_1_inhibiting_alert-40 1.298µ ± ∞ ¹ 1.326µ ± ∞ ¹ ~ (p=1.000 n=1) ² Mutes/100_inhibition_rules,_1_inhibiting_alert-40 1.347µ ± ∞ ¹ 1.391µ ± ∞ ¹ ~ (p=1.000 n=1) ² Mutes/1000_inhibition_rules,_1_inhibiting_alert-40 1.498µ ± ∞ ¹ 1.578µ ± ∞ ¹ ~ (p=1.000 n=1) ² Mutes/10000_inhibition_rules,_1_inhibiting_alert-40 1.685µ ± ∞ ¹ 1.502µ ± ∞ ¹ ~ (p=1.000 n=1) ² Mutes/1_inhibition_rule,_10_inhibiting_alerts-40 1.443µ ± ∞ ¹ 1.521µ ± ∞ ¹ ~ (p=1.000 n=1) ² Mutes/1_inhibition_rule,_100_inhibiting_alerts-40 1.453µ ± ∞ ¹ 1.561µ ± ∞ ¹ ~ (p=1.000 n=1) ² Mutes/1_inhibition_rule,_1000_inhibiting_alerts-40 1.423µ ± ∞ ¹ 1.511µ ± ∞ ¹ ~ (p=1.000 n=1) ² Mutes/1_inhibition_rule,_10000_inhibiting_alerts-40 1.389µ ± ∞ ¹ 1.441µ ± ∞ ¹ ~ (p=1.000 n=1) ² Mutes/100_inhibition_rules,_1000_inhibiting_alerts-40 1.218µ ± ∞ ¹ 1.306µ ± ∞ ¹ ~ (p=1.000 n=1) ² Mutes/10_inhibition_rules,_last_rule_matches-40 3.054µ ± ∞ ¹ 3.114µ ± ∞ ¹ ~ (p=1.000 n=1) ² Mutes/100_inhibition_rules,_last_rule_matches-40 19.57µ ± ∞ ¹ 19.47µ ± ∞ ¹ ~ (p=1.000 n=1) ² Mutes/1000_inhibition_rules,_last_rule_matches-40 187.4µ ± ∞ ¹ 188.9µ ± ∞ ¹ ~ (p=1.000 n=1) ² Mutes/10000_inhibition_rules,_last_rule_matches-40 1.944m ± ∞ ¹ 1.955m ± ∞ ¹ ~ (p=1.000 n=1) ² geomean 4.239µ 4.356µ +2.75% ¹ need >= 6 samples for confidence interval at level 0.95 ² need >= 4 samples to detect a difference at alpha level 0.05 │ bench-before.txt │ bench-after.txt │ │ B/op │ B/op vs base │ Mutes/1_inhibition_rule,_1_inhibiting_alert-40 488.0 ± ∞ ¹ 488.0 ± ∞ ¹ ~ (p=1.000 n=1) ² Mutes/10_inhibition_rules,_1_inhibiting_alert-40 488.0 ± ∞ ¹ 488.0 ± ∞ ¹ ~ (p=1.000 n=1) ² Mutes/100_inhibition_rules,_1_inhibiting_alert-40 489.0 ± ∞ ¹ 489.0 ± ∞ ¹ ~ (p=1.000 n=1) ² Mutes/1000_inhibition_rules,_1_inhibiting_alert-40 488.0 ± ∞ ¹ 488.0 ± ∞ ¹ ~ (p=1.000 n=1) ² Mutes/10000_inhibition_rules,_1_inhibiting_alert-40 489.0 ± ∞ ¹ 489.0 ± ∞ ¹ ~ (p=1.000 n=1) ² Mutes/1_inhibition_rule,_10_inhibiting_alerts-40 488.0 ± ∞ ¹ 488.0 ± ∞ ¹ ~ (p=1.000 n=1) ² Mutes/1_inhibition_rule,_100_inhibiting_alerts-40 488.0 ± ∞ ¹ 488.0 ± ∞ ¹ ~ (p=1.000 n=1) ² Mutes/1_inhibition_rule,_1000_inhibiting_alerts-40 488.0 ± ∞ ¹ 488.0 ± ∞ ¹ ~ (p=1.000 n=1) ² Mutes/1_inhibition_rule,_10000_inhibiting_alerts-40 488.0 ± ∞ ¹ 488.0 ± ∞ ¹ ~ (p=1.000 n=1) ² Mutes/100_inhibition_rules,_1000_inhibiting_alerts-40 488.0 ± ∞ ¹ 488.0 ± ∞ ¹ ~ (p=1.000 n=1) ² Mutes/10_inhibition_rules,_last_rule_matches-40 472.0 ± ∞ ¹ 472.0 ± ∞ ¹ ~ (p=1.000 n=1) ² Mutes/100_inhibition_rules,_last_rule_matches-40 472.0 ± ∞ ¹ 472.0 ± ∞ ¹ ~ (p=1.000 n=1) ² Mutes/1000_inhibition_rules,_last_rule_matches-40 473.0 ± ∞ ¹ 473.0 ± ∞ ¹ ~ (p=1.000 n=1) ² Mutes/10000_inhibition_rules,_last_rule_matches-40 481.0 ± ∞ ¹ 481.0 ± ∞ ¹ ~ (p=1.000 n=1) ² geomean 484.2 484.2 +0.00% ¹ need >= 6 samples for confidence interval at level 0.95 ² all samples are equal │ bench-before.txt │ bench-after.txt │ │ allocs/op │ allocs/op vs base │ Mutes/1_inhibition_rule,_1_inhibiting_alert-40 10.00 ± ∞ ¹ 10.00 ± ∞ ¹ ~ (p=1.000 n=1) ² Mutes/10_inhibition_rules,_1_inhibiting_alert-40 10.00 ± ∞ ¹ 10.00 ± ∞ ¹ ~ (p=1.000 n=1) ² Mutes/100_inhibition_rules,_1_inhibiting_alert-40 10.00 ± ∞ ¹ 10.00 ± ∞ ¹ ~ (p=1.000 n=1) ² Mutes/1000_inhibition_rules,_1_inhibiting_alert-40 10.00 ± ∞ ¹ 10.00 ± ∞ ¹ ~ (p=1.000 n=1) ² Mutes/10000_inhibition_rules,_1_inhibiting_alert-40 10.00 ± ∞ ¹ 10.00 ± ∞ ¹ ~ (p=1.000 n=1) ² Mutes/1_inhibition_rule,_10_inhibiting_alerts-40 10.00 ± ∞ ¹ 10.00 ± ∞ ¹ ~ (p=1.000 n=1) ² Mutes/1_inhibition_rule,_100_inhibiting_alerts-40 10.00 ± ∞ ¹ 10.00 ± ∞ ¹ ~ (p=1.000 n=1) ² Mutes/1_inhibition_rule,_1000_inhibiting_alerts-40 10.00 ± ∞ ¹ 10.00 ± ∞ ¹ ~ (p=1.000 n=1) ² Mutes/1_inhibition_rule,_10000_inhibiting_alerts-40 10.00 ± ∞ ¹ 10.00 ± ∞ ¹ ~ (p=1.000 n=1) ² Mutes/100_inhibition_rules,_1000_inhibiting_alerts-40 10.00 ± ∞ ¹ 10.00 ± ∞ ¹ ~ (p=1.000 n=1) ² Mutes/10_inhibition_rules,_last_rule_matches-40 10.00 ± ∞ ¹ 10.00 ± ∞ ¹ ~ (p=1.000 n=1) ² Mutes/100_inhibition_rules,_last_rule_matches-40 10.00 ± ∞ ¹ 10.00 ± ∞ ¹ ~ (p=1.000 n=1) ² Mutes/1000_inhibition_rules,_last_rule_matches-40 10.00 ± ∞ ¹ 10.00 ± ∞ ¹ ~ (p=1.000 n=1) ² Mutes/10000_inhibition_rules,_last_rule_matches-40 10.00 ± ∞ ¹ 10.00 ± ∞ ¹ ~ (p=1.000 n=1) ² geomean 10.00 10.00 +0.00% ¹ need >= 6 samples for confidence interval at level 0.95 ² all samples are equal pkg: github.com/prometheus/alertmanager/matcher/parse │ bench-before.txt │ bench-after.txt │ │ sec/op │ sec/op vs base │ ParseSimple-40 1.046µ ± ∞ ¹ 1.096µ ± ∞ ¹ ~ (p=1.000 n=1) ² ParseComplex-40 4.855µ ± ∞ ¹ 4.769µ ± ∞ ¹ ~ (p=1.000 n=1) ² geomean 2.254µ 2.286µ +1.45% ¹ need >= 6 samples for confidence interval at level 0.95 ² need >= 4 samples to detect a difference at alpha level 0.05 │ bench-before.txt │ bench-after.txt │ │ B/op │ B/op vs base │ ParseSimple-40 232.0 ± ∞ ¹ 232.0 ± ∞ ¹ ~ (p=1.000 n=1) ² ParseComplex-40 2.602Ki ± ∞ ¹ 2.602Ki ± ∞ ¹ ~ (p=1.000 n=1) ² geomean 786.2 786.2 +0.00% ¹ need >= 6 samples for confidence interval at level 0.95 ² all samples are equal │ bench-before.txt │ bench-after.txt │ │ allocs/op │ allocs/op vs base │ ParseSimple-40 7.000 ± ∞ ¹ 7.000 ± ∞ ¹ ~ (p=1.000 n=1) ² ParseComplex-40 42.00 ± ∞ ¹ 42.00 ± ∞ ¹ ~ (p=1.000 n=1) ² geomean 17.15 17.15 +0.00% ¹ need >= 6 samples for confidence interval at level 0.95 ² all samples are equal pkg: github.com/prometheus/alertmanager/notify │ bench-before.txt │ bench-after.txt │ │ sec/op │ sec/op vs base │ HashAlert-40 239.8n ± ∞ ¹ 244.2n ± ∞ ¹ ~ (p=1.000 n=1) ² ¹ need >= 6 samples for confidence interval at level 0.95 ² need >= 4 samples to detect a difference at alpha level 0.05 │ bench-before.txt │ bench-after.txt │ │ B/op │ B/op vs base │ HashAlert-40 72.00 ± ∞ ¹ 72.00 ± ∞ ¹ ~ (p=1.000 n=1) ² ¹ need >= 6 samples for confidence interval at level 0.95 ² all samples are equal │ bench-before.txt │ bench-after.txt │ │ allocs/op │ allocs/op vs base │ HashAlert-40 2.000 ± ∞ ¹ 2.000 ± ∞ ¹ ~ (p=1.000 n=1) ² ¹ need >= 6 samples for confidence interval at level 0.95 ² all samples are equal pkg: github.com/prometheus/alertmanager/silence │ bench-before.txt │ bench-after.txt │ │ sec/op │ sec/op vs base │ Mutes/1_silence_mutes_alert-40 10.50µ ± ∞ ¹ 10.23µ ± ∞ ¹ ~ (p=1.000 n=1) ² Mutes/10_silences_mute_alert-40 12.15µ ± ∞ ¹ 13.85µ ± ∞ ¹ ~ (p=1.000 n=1) ² Mutes/100_silences_mute_alert-40 35.81µ ± ∞ ¹ 39.36µ ± ∞ ¹ ~ (p=1.000 n=1) ² Mutes/1000_silences_mute_alert-40 430.7µ ± ∞ ¹ 474.4µ ± ∞ ¹ ~ (p=1.000 n=1) ² Mutes/10000_silences_mute_alert-40 5.054m ± ∞ ¹ 5.211m ± ∞ ¹ ~ (p=1.000 n=1) ² Query/100_silences-40 31.09µ ± ∞ ¹ Query/1000_silences-40 314.3µ ± ∞ ¹ Query/10000_silences-40 6.531m ± ∞ ¹ QueryParallel/100_silences,_1_goroutine-40 38.32µ ± ∞ ¹ 18.78µ ± ∞ ¹ ~ (p=1.000 n=1) ² QueryParallel/100_silences,_2_goroutines-40 37.57µ ± ∞ ¹ 17.50µ ± ∞ ¹ ~ (p=1.000 n=1) ² QueryParallel/100_silences,_4_goroutines-40 39.29µ ± ∞ ¹ 17.52µ ± ∞ ¹ ~ (p=1.000 n=1) ² QueryParallel/100_silences,_8_goroutines-40 37.88µ ± ∞ ¹ 18.83µ ± ∞ ¹ ~ (p=1.000 n=1) ² QueryParallel/1000_silences,_1_goroutine-40 351.65µ ± ∞ ¹ 46.81µ ± ∞ ¹ ~ (p=1.000 n=1) ² QueryParallel/1000_silences,_2_goroutines-40 335.03µ ± ∞ ¹ 44.99µ ± ∞ ¹ ~ (p=1.000 n=1) ² QueryParallel/1000_silences,_4_goroutines-40 362.03µ ± ∞ ¹ 46.17µ ± ∞ ¹ ~ (p=1.000 n=1) ² QueryParallel/1000_silences,_8_goroutines-40 338.11µ ± ∞ ¹ 46.33µ ± ∞ ¹ ~ (p=1.000 n=1) ² QueryParallel/10000_silences,_1_goroutine-40 6979.9µ ± ∞ ¹ 354.1µ ± ∞ ¹ ~ (p=1.000 n=1) ² QueryParallel/10000_silences,_2_goroutines-40 6454.0µ ± ∞ ¹ 363.8µ ± ∞ ¹ ~ (p=1.000 n=1) ² QueryParallel/10000_silences,_4_goroutines-40 6607.6µ ± ∞ ¹ 346.9µ ± ∞ ¹ ~ (p=1.000 n=1) ² QueryParallel/10000_silences,_8_goroutines-40 9666.4µ ± ∞ ¹ 409.3µ ± ∞ ¹ ~ (p=1.000 n=1) ² QueryWithConcurrentAdds/1000_initial_silences,_10%_add_rate-40 493.8µ ± ∞ ¹ 154.1µ ± ∞ ¹ ~ (p=1.000 n=1) ² QueryWithConcurrentAdds/1000_initial_silences,_1%_add_rate-40 373.19µ ± ∞ ¹ 70.64µ ± ∞ ¹ ~ (p=1.000 n=1) ² QueryWithConcurrentAdds/1000_initial_silences,_0.1%_add_rate-40 358.16µ ± ∞ ¹ 49.32µ ± ∞ ¹ ~ (p=1.000 n=1) ² QueryWithConcurrentAdds/10000_initial_silences,_1%_add_rate-40 7028.4µ ± ∞ ¹ 524.5µ ± ∞ ¹ ~ (p=1.000 n=1) ² QueryWithConcurrentAdds/10000_initial_silences,_0.1%_add_rate-40 8035.2µ ± ∞ ¹ 374.5µ ± ∞ ¹ ~ (p=1.000 n=1) ² MutesParallel/100_silences,_4_goroutines-40 53.23µ ± ∞ ¹ 39.80µ ± ∞ ¹ ~ (p=1.000 n=1) ² MutesParallel/1000_silences,_4_goroutines-40 484.7µ ± ∞ ¹ 173.0µ ± ∞ ¹ ~ (p=1.000 n=1) ² MutesParallel/10000_silences,_4_goroutines-40 4.185m ± ∞ ¹ 1.112m ± ∞ ¹ ~ (p=1.000 n=1) ² MutesParallel/10000_silences,_8_goroutines-40 4.365m ± ∞ ¹ 1.077m ± ∞ ¹ ~ (p=1.000 n=1) ² geomean 451.9µ 109.5µ -76.10% ³ ¹ need >= 6 samples for confidence interval at level 0.95 ² need >= 4 samples to detect a difference at alpha level 0.05 ³ benchmark set differs from baseline; geomeans may not be comparable │ bench-before.txt │ bench-after.txt │ │ B/op │ B/op vs base │ Mutes/1_silence_mutes_alert-40 3.835Ki ± ∞ ¹ 3.718Ki ± ∞ ¹ ~ (p=1.000 n=1) ² Mutes/10_silences_mute_alert-40 6.740Ki ± ∞ ¹ 6.799Ki ± ∞ ¹ ~ (p=1.000 n=1) ² Mutes/100_silences_mute_alert-40 35.99Ki ± ∞ ¹ 35.87Ki ± ∞ ¹ ~ (p=1.000 n=1) ² Mutes/1000_silences_mute_alert-40 308.2Ki ± ∞ ¹ 308.1Ki ± ∞ ¹ ~ (p=1.000 n=1) ² Mutes/10000_silences_mute_alert-40 3.536Mi ± ∞ ¹ 3.527Mi ± ∞ ¹ ~ (p=1.000 n=1) ² Query/100_silences-40 6.142Ki ± ∞ ¹ Query/1000_silences-40 41.30Ki ± ∞ ¹ Query/10000_silences-40 525.2Ki ± ∞ ¹ QueryParallel/100_silences,_1_goroutine-40 6.052Ki ± ∞ ¹ 6.136Ki ± ∞ ¹ ~ (p=1.000 n=1) ² QueryParallel/100_silences,_2_goroutines-40 6.048Ki ± ∞ ¹ 6.049Ki ± ∞ ¹ ~ (p=1.000 n=1) ² QueryParallel/100_silences,_4_goroutines-40 6.048Ki ± ∞ ¹ 6.050Ki ± ∞ ¹ ~ (p=1.000 n=1) ² QueryParallel/100_silences,_8_goroutines-40 6.053Ki ± ∞ ¹ 6.039Ki ± ∞ ¹ ~ (p=1.000 n=1) ² QueryParallel/1000_silences,_1_goroutine-40 41.30Ki ± ∞ ¹ 41.42Ki ± ∞ ¹ ~ (p=1.000 n=1) ² QueryParallel/1000_silences,_2_goroutines-40 41.27Ki ± ∞ ¹ 41.42Ki ± ∞ ¹ ~ (p=1.000 n=1) ² QueryParallel/1000_silences,_4_goroutines-40 41.32Ki ± ∞ ¹ 41.34Ki ± ∞ ¹ ~ (p=1.000 n=1) ² QueryParallel/1000_silences,_8_goroutines-40 41.29Ki ± ∞ ¹ 41.44Ki ± ∞ ¹ ~ (p=1.000 n=1) ² QueryParallel/10000_silences,_1_goroutine-40 525.0Ki ± ∞ ¹ 524.9Ki ± ∞ ¹ ~ (p=1.000 n=1) ² QueryParallel/10000_silences,_2_goroutines-40 524.9Ki ± ∞ ¹ 524.9Ki ± ∞ ¹ ~ (p=1.000 n=1) ² QueryParallel/10000_silences,_4_goroutines-40 524.9Ki ± ∞ ¹ 525.0Ki ± ∞ ¹ ~ (p=1.000 n=1) ² QueryParallel/10000_silences,_8_goroutines-40 525.0Ki ± ∞ ¹ 524.9Ki ± ∞ ¹ ~ (p=1.000 n=1) ² QueryWithConcurrentAdds/1000_initial_silences,_10%_add_rate-40 100.7Ki ± ∞ ¹ 226.4Ki ± ∞ ¹ ~ (p=1.000 n=1) ² QueryWithConcurrentAdds/1000_initial_silences,_1%_add_rate-40 61.88Ki ± ∞ ¹ 86.27Ki ± ∞ ¹ ~ (p=1.000 n=1) ² QueryWithConcurrentAdds/1000_initial_silences,_0.1%_add_rate-40 44.36Ki ± ∞ ¹ 46.04Ki ± ∞ ¹ ~ (p=1.000 n=1) ² QueryWithConcurrentAdds/10000_initial_silences,_1%_add_rate-40 1058.7Ki ± ∞ ¹ 533.3Ki ± ∞ ¹ ~ (p=1.000 n=1) ² QueryWithConcurrentAdds/10000_initial_silences,_0.1%_add_rate-40 1118.4Ki ± ∞ ¹ 525.1Ki ± ∞ ¹ ~ (p=1.000 n=1) ² MutesParallel/100_silences,_4_goroutines-40 36.10Ki ± ∞ ¹ 36.05Ki ± ∞ ¹ ~ (p=1.000 n=1) ² MutesParallel/1000_silences,_4_goroutines-40 307.6Ki ± ∞ ¹ 307.4Ki ± ∞ ¹ ~ (p=1.000 n=1) ² MutesParallel/10000_silences,_4_goroutines-40 3.487Mi ± ∞ ¹ 3.517Mi ± ∞ ¹ ~ (p=1.000 n=1) ² MutesParallel/10000_silences,_8_goroutines-40 3.492Mi ± ∞ ¹ 3.514Mi ± ∞ ¹ ~ (p=1.000 n=1) ² geomean 94.43Ki 100.4Ki -0.98% ³ ¹ need >= 6 samples for confidence interval at level 0.95 ² need >= 4 samples to detect a difference at alpha level 0.05 ³ benchmark set differs from baseline; geomeans may not be comparable │ bench-before.txt │ bench-after.txt │ │ allocs/op │ allocs/op vs base │ Mutes/1_silence_mutes_alert-40 39.00 ± ∞ ¹ 39.00 ± ∞ ¹ ~ (p=1.000 n=1) ² Mutes/10_silences_mute_alert-40 59.00 ± ∞ ¹ 59.00 ± ∞ ¹ ~ (p=1.000 n=1) ² Mutes/100_silences_mute_alert-40 158.0 ± ∞ ¹ 158.0 ± ∞ ¹ ~ (p=1.000 n=1) ² Mutes/1000_silences_mute_alert-40 1.071k ± ∞ ¹ 1.071k ± ∞ ¹ ~ (p=1.000 n=1) ² Mutes/10000_silences_mute_alert-40 10.18k ± ∞ ¹ 10.10k ± ∞ ¹ ~ (p=1.000 n=1) ³ Query/100_silences-40 45.00 ± ∞ ¹ Query/1000_silences-40 141.0 ± ∞ ¹ Query/10000_silences-40 1.051k ± ∞ ¹ QueryParallel/100_silences,_1_goroutine-40 42.00 ± ∞ ¹ 42.00 ± ∞ ¹ ~ (p=1.000 n=1) ² QueryParallel/100_silences,_2_goroutines-40 42.00 ± ∞ ¹ 42.00 ± ∞ ¹ ~ (p=1.000 n=1) ² QueryParallel/100_silences,_4_goroutines-40 42.00 ± ∞ ¹ 42.00 ± ∞ ¹ ~ (p=1.000 n=1) ² QueryParallel/100_silences,_8_goroutines-40 42.00 ± ∞ ¹ 42.00 ± ∞ ¹ ~ (p=1.000 n=1) ² QueryParallel/1000_silences,_1_goroutine-40 138.0 ± ∞ ¹ 139.0 ± ∞ ¹ ~ (p=1.000 n=1) ³ QueryParallel/1000_silences,_2_goroutines-40 138.0 ± ∞ ¹ 139.0 ± ∞ ¹ ~ (p=1.000 n=1) ³ QueryParallel/1000_silences,_4_goroutines-40 138.0 ± ∞ ¹ 139.0 ± ∞ ¹ ~ (p=1.000 n=1) ³ QueryParallel/1000_silences,_8_goroutines-40 138.0 ± ∞ ¹ 139.0 ± ∞ ¹ ~ (p=1.000 n=1) ³ QueryParallel/10000_silences,_1_goroutine-40 1.049k ± ∞ ¹ 1.050k ± ∞ ¹ ~ (p=1.000 n=1) ³ QueryParallel/10000_silences,_2_goroutines-40 1.049k ± ∞ ¹ 1.049k ± ∞ ¹ ~ (p=1.000 n=1) ² QueryParallel/10000_silences,_4_goroutines-40 1.049k ± ∞ ¹ 1.050k ± ∞ ¹ ~ (p=1.000 n=1) ³ QueryParallel/10000_silences,_8_goroutines-40 1.049k ± ∞ ¹ 1.049k ± ∞ ¹ ~ (p=1.000 n=1) ² QueryWithConcurrentAdds/1000_initial_silences,_10%_add_rate-40 400.0 ± ∞ ¹ 876.0 ± ∞ ¹ ~ (p=1.000 n=1) ³ QueryWithConcurrentAdds/1000_initial_silences,_1%_add_rate-40 204.0 ± ∞ ¹ 294.0 ± ∞ ¹ ~ (p=1.000 n=1) ³ QueryWithConcurrentAdds/1000_initial_silences,_0.1%_add_rate-40 175.0 ± ∞ ¹ 162.0 ± ∞ ¹ ~ (p=1.000 n=1) ³ QueryWithConcurrentAdds/10000_initial_silences,_1%_add_rate-40 8.363k ± ∞ ¹ 1.065k ± ∞ ¹ ~ (p=1.000 n=1) ³ QueryWithConcurrentAdds/10000_initial_silences,_0.1%_add_rate-40 9.138k ± ∞ ¹ 1.052k ± ∞ ¹ ~ (p=1.000 n=1) ³ MutesParallel/100_silences,_4_goroutines-40 158.0 ± ∞ ¹ 158.0 ± ∞ ¹ ~ (p=1.000 n=1) ² MutesParallel/1000_silences,_4_goroutines-40 1.073k ± ∞ ¹ 1.072k ± ∞ ¹ ~ (p=1.000 n=1) ³ MutesParallel/10000_silences,_4_goroutines-40 10.18k ± ∞ ¹ 10.10k ± ∞ ¹ ~ (p=1.000 n=1) ³ MutesParallel/10000_silences,_8_goroutines-40 10.16k ± ∞ ¹ 10.10k ± ∞ ¹ ~ (p=1.000 n=1) ³ geomean 380.8 366.0 -11.39% ⁴ ¹ need >= 6 samples for confidence interval at level 0.95 ² all samples are equal ³ need >= 4 samples to detect a difference at alpha level 0.05 ⁴ benchmark set differs from baseline; geomeans may not be comparable Signed-off-by: Guido Trotter <guido@hudson-trading.com> * Make 'get' private Signed-off-by: Guido Trotter <guido@hudson-trading.com> * Add metrics for the case where the matches fails to compile/add to the cache Before the silences would be in the status, but querying would result in query errors. Now they are still in the status (since we have to calculate after calling Merge), but we also know that some broken ones exist. At load snapshot time we can also avoid importing those, since they would not be usable anyway, and we have different labels to distinguish the cases/behaviors. Signed-off-by: Guido Trotter <guido@hudson-trading.com> * Rename silenceAdded to cacheSilence Signed-off-by: Guido Trotter <guido@hudson-trading.com> * Use matcherCompile.*Error metrics with pre-set labels Signed-off-by: Guido Trotter <guido@hudson-trading.com> --------- Signed-off-by: Guido Trotter <guido@hudson-trading.com> Co-authored-by: Guido Trotter <guido@hudson-trading.com>
1 parent ce8ce3b commit 6ceb38d

File tree

2 files changed

+58
-34
lines changed

2 files changed

+58
-34
lines changed

‎silence/silence.go‎

Lines changed: 48 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -51,14 +51,12 @@ var ErrInvalidState = errors.New("invalid state")
5151

5252
type matcherCache map[string]labels.Matchers
5353

54-
// Get retrieves the matchers for a given silence. If it is a missed cache
55-
// access, it compiles and adds the matchers of the requested silence to the
56-
// cache.
57-
func (c matcherCache) Get(s *pb.Silence) (labels.Matchers, error) {
54+
// get retrieves the matchers for a given silence.
55+
func (c matcherCache) get(s *pb.Silence) (labels.Matchers, error) {
5856
if m, ok := c[s.Id]; ok {
5957
return m, nil
6058
}
61-
return c.add(s)
59+
return nil, ErrNotFound
6260
}
6361

6462
// add compiles a silences' matchers and adds them to the cache.
@@ -217,18 +215,20 @@ type Limits struct {
217215
type MaintenanceFunc func() (int64, error)
218216

219217
type metrics struct {
220-
gcDuration prometheus.Summary
221-
snapshotDuration prometheus.Summary
222-
snapshotSize prometheus.Gauge
223-
queriesTotal prometheus.Counter
224-
queryErrorsTotal prometheus.Counter
225-
queryDuration prometheus.Histogram
226-
silencesActive prometheus.GaugeFunc
227-
silencesPending prometheus.GaugeFunc
228-
silencesExpired prometheus.GaugeFunc
229-
propagatedMessagesTotal prometheus.Counter
230-
maintenanceTotal prometheus.Counter
231-
maintenanceErrorsTotal prometheus.Counter
218+
gcDuration prometheus.Summary
219+
snapshotDuration prometheus.Summary
220+
snapshotSize prometheus.Gauge
221+
queriesTotal prometheus.Counter
222+
queryErrorsTotal prometheus.Counter
223+
queryDuration prometheus.Histogram
224+
silencesActive prometheus.GaugeFunc
225+
silencesPending prometheus.GaugeFunc
226+
silencesExpired prometheus.GaugeFunc
227+
propagatedMessagesTotal prometheus.Counter
228+
maintenanceTotal prometheus.Counter
229+
maintenanceErrorsTotal prometheus.Counter
230+
matcherCompileCacheSilenceErrorsTotal prometheus.Counter
231+
matcherCompileLoadSnapshotErrorsTotal prometheus.Counter
232232
}
233233

234234
func newSilenceMetricByState(s *Silences, st types.SilenceState) prometheus.GaugeFunc {
@@ -273,6 +273,15 @@ func newMetrics(r prometheus.Registerer, s *Silences) *metrics {
273273
Name: "alertmanager_silences_maintenance_errors_total",
274274
Help: "How many maintenances were executed for silences that failed.",
275275
})
276+
matcherCompileErrorsTotal := prometheus.NewCounterVec(
277+
prometheus.CounterOpts{
278+
Name: "alertmanager_silences_matcher_compile_errors_total",
279+
Help: "How many silence matcher compilations failed.",
280+
},
281+
[]string{"stage"},
282+
)
283+
m.matcherCompileCacheSilenceErrorsTotal = matcherCompileErrorsTotal.WithLabelValues("cache_silence")
284+
m.matcherCompileLoadSnapshotErrorsTotal = matcherCompileErrorsTotal.WithLabelValues("load_snapshot")
276285
m.queriesTotal = prometheus.NewCounter(prometheus.CounterOpts{
277286
Name: "alertmanager_silences_queries_total",
278287
Help: "How many silence queries were received.",
@@ -313,6 +322,7 @@ func newMetrics(r prometheus.Registerer, s *Silences) *metrics {
313322
m.propagatedMessagesTotal,
314323
m.maintenanceTotal,
315324
m.maintenanceErrorsTotal,
325+
matcherCompileErrorsTotal,
316326
)
317327
}
318328
return m
@@ -562,6 +572,15 @@ func (s *Silences) checkSizeLimits(msil *pb.MeshSilence) error {
562572
return nil
563573
}
564574

575+
func (s *Silences) cacheSilence(sil *pb.Silence) {
576+
s.version++
577+
_, err := s.mc.add(sil)
578+
if err != nil {
579+
s.metrics.matcherCompileCacheSilenceErrorsTotal.Inc()
580+
s.logger.Error("Failed to compile silence matchers", "silence_id", sil.Id, "err", err)
581+
}
582+
}
583+
565584
func (s *Silences) getSilence(id string) (*pb.Silence, bool) {
566585
msil, ok := s.st[id]
567586
if !ok {
@@ -584,7 +603,7 @@ func (s *Silences) setSilence(msil *pb.MeshSilence, now time.Time) error {
584603
}
585604
_, added := s.st.merge(msil, now)
586605
if added {
587-
s.version++
606+
s.cacheSilence(msil.Silence)
588607
}
589608
s.broadcast(b)
590609
return nil
@@ -738,7 +757,7 @@ func QIDs(ids ...string) QueryParam {
738757
func QMatches(set model.LabelSet) QueryParam {
739758
return func(q *query) error {
740759
f := func(sil *pb.Silence, s *Silences, _ time.Time) (bool, error) {
741-
m, err := s.mc.Get(sil)
760+
m, err := s.mc.get(sil)
742761
if err != nil {
743762
return true, err
744763
}
@@ -833,8 +852,8 @@ func (s *Silences) query(q *query, now time.Time) ([]*pb.Silence, int, error) {
833852
// the use of post-filter functions is the trivial solution for now.
834853
var res []*pb.Silence
835854

836-
s.mtx.Lock()
837-
defer s.mtx.Unlock()
855+
s.mtx.RLock()
856+
defer s.mtx.RUnlock()
838857

839858
if q.ids != nil {
840859
for _, id := range q.ids {
@@ -883,7 +902,13 @@ func (s *Silences) loadSnapshot(r io.Reader) error {
883902
e.Silence.CreatedBy = e.Silence.Comments[0].Author
884903
e.Silence.Comments = nil
885904
}
886-
st[e.Silence.Id] = e
905+
// Add to matcher cache, and only if successful, to the new state.
906+
if _, err := s.mc.add(e.Silence); err != nil {
907+
s.metrics.matcherCompileLoadSnapshotErrorsTotal.Inc()
908+
s.logger.Error("Failed to compile silence matchers during snapshot load", "silence_id", e.Silence.Id, "err", err)
909+
} else {
910+
st[e.Silence.Id] = e
911+
}
887912
}
888913
s.mtx.Lock()
889914
s.st = st
@@ -933,7 +958,7 @@ func (s *Silences) Merge(b []byte) error {
933958
merged, added := s.st.merge(e, now)
934959
if merged {
935960
if added {
936-
s.version++
961+
s.cacheSilence(e.Silence)
937962
}
938963
if !cluster.OversizedMessage(b) {
939964
// If this is the first we've seen the message and it's

‎silence/silence_test.go‎

Lines changed: 10 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -90,11 +90,15 @@ func TestSilenceGCOverTime(t *testing.T) {
9090
require.NoError(t, err)
9191
s.clock = quartz.NewMock(t)
9292
now := s.nowUTC()
93-
s.st = state{
93+
initialState := state{
9494
"1": &pb.MeshSilence{Silence: &pb.Silence{Id: "1"}, ExpiresAt: now},
9595
"2": &pb.MeshSilence{Silence: &pb.Silence{Id: "2"}, ExpiresAt: now.Add(-time.Second)},
9696
"3": &pb.MeshSilence{Silence: &pb.Silence{Id: "3"}, ExpiresAt: now.Add(time.Second)},
9797
}
98+
for _, sil := range initialState {
99+
s.st[sil.Silence.Id] = sil
100+
s.cacheSilence(sil.Silence)
101+
}
98102
want := state{
99103
"3": &pb.MeshSilence{Silence: &pb.Silence{Id: "3"}, ExpiresAt: now.Add(time.Second)},
100104
}
@@ -119,8 +123,6 @@ func TestSilenceGCOverTime(t *testing.T) {
119123
EndsAt: clock.Now().Add(time.Minute),
120124
}
121125
require.NoError(t, s.Set(sil1))
122-
// Need to query the silence to populate the matcher cache.
123-
s.Query(QMatches(model.LabelSet{"foo": "bar"}))
124126
require.Len(t, s.st, 1)
125127
require.Len(t, s.mc, 1)
126128
// Move time forward and both silence and cache entry should be garbage
@@ -148,8 +150,6 @@ func TestSilenceGCOverTime(t *testing.T) {
148150
EndsAt: clock.Now().Add(time.Minute),
149151
}
150152
require.NoError(t, s.Set(sil1))
151-
// Need to query the silence to populate the matcher cache.
152-
s.Query(QMatches(model.LabelSet{"foo": "bar"}))
153153
require.Len(t, s.st, 1)
154154
require.Len(t, s.mc, 1)
155155
// must clone sil1 before replacing it.
@@ -160,8 +160,6 @@ func TestSilenceGCOverTime(t *testing.T) {
160160
Pattern: "baz",
161161
}}
162162
require.NoError(t, s.Set(sil2))
163-
// Need to query the silence to populate the matcher cache.
164-
s.Query(QMatches(model.LabelSet{"bar": "baz"}))
165163
require.Len(t, s.st, 2)
166164
require.Len(t, s.mc, 2)
167165
// Move time forward and both silence and cache entry should be garbage
@@ -176,7 +174,7 @@ func TestSilenceGCOverTime(t *testing.T) {
176174

177175
// This test checks for a memory leak that occurred in the matcher cache when
178176
// updating an existing silence.
179-
t.Run("updating a silences does not leak cache entries", func(t *testing.T) {
177+
t.Run("updating a silence does not leak cache entries", func(t *testing.T) {
180178
s, err := New(Options{})
181179
require.NoError(t, err)
182180
clock := quartz.NewMock(t)
@@ -192,8 +190,7 @@ func TestSilenceGCOverTime(t *testing.T) {
192190
EndsAt: clock.Now().Add(time.Minute),
193191
}
194192
s.st["1"] = &pb.MeshSilence{Silence: sil1, ExpiresAt: clock.Now().Add(time.Minute)}
195-
// Need to query the silence to populate the matcher cache.
196-
s.Query(QMatches(model.LabelSet{"foo": "bar"}))
193+
s.cacheSilence(sil1)
197194
require.Len(t, s.mc, 1)
198195
// must clone sil1 before updating it.
199196
sil2 := cloneSilence(sil1)
@@ -942,7 +939,9 @@ func TestQMatches(t *testing.T) {
942939
},
943940
}
944941
for _, c := range cases {
945-
drop, err := f(c.sil, &Silences{mc: matcherCache{}, st: state{}}, time.Time{})
942+
silences := &Silences{mc: matcherCache{}, st: state{}}
943+
silences.mc.add(c.sil)
944+
drop, err := f(c.sil, silences, time.Time{})
946945
require.NoError(t, err)
947946
require.Equal(t, c.drop, drop, "unexpected filter result")
948947
}

0 commit comments

Comments
 (0)