fix: improve inhibition performance

inhibit/index.go

@@ -0,0 +1,57 @@
+// Copyright The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package inhibit
+
+import (
+	"sync"
+
+	"github.com/prometheus/common/model"
+)
+
+// index contains map of fingerprints to fingerprints.
+// The keys are fingerprints of the equal labels of source alerts.
+// The values are fingerprints of the source alerts.
+// For more info see comments on inhibitor and InhibitRule.
+type index struct {
+	mtx   sync.RWMutex
+	items map[model.Fingerprint]model.Fingerprint
+}
+
+func newIndex() *index {
+	return &index{
+		items: make(map[model.Fingerprint]model.Fingerprint),
+	}
+}
+
+func (c *index) Get(key model.Fingerprint) (model.Fingerprint, bool) {
+	c.mtx.RLock()
+	defer c.mtx.RUnlock()
+
+	fp, ok := c.items[key]
+	return fp, ok
+}
+
+func (c *index) Set(key, value model.Fingerprint) {
+	c.mtx.Lock()
+	defer c.mtx.Unlock()
+
+	c.items[key] = value
+}
+
+func (c *index) Delete(key model.Fingerprint) {
+	c.mtx.Lock()
+	defer c.mtx.Unlock()
+
+	delete(c.items, key)
+}

inhibit/inhibit.go

@@ -1,4 +1,4 @@
-// Copyright 2015 Prometheus Team
+// Copyright The Prometheus Authors
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
@@ -74,7 +74,10 @@ func (ih *Inhibitor) run(ctx context.Context) {
 				if r.SourceMatchers.Matches(a.Labels) {
 					if err := r.scache.Set(a); err != nil {
 						ih.logger.Error("error on set alert", "err", err)
+						continue
 					}
+
+					r.updateIndex(a)
 				}
 			}
 		}
@@ -162,6 +165,12 @@ type InhibitRule struct {
 
 	// Cache of alerts matching source labels.
 	scache *store.Alerts
+
+	// Index of fingerprints of source alert equal labels to fingerprint of source alert.
+	// The index helps speed up source alert lookups from scache significantely in scenarios with 100s of source alerts cached.
+	// The index items might overwrite eachother if multiple source alerts have exact equal labels.
+	// Overwrites only happen if the new source alert has bigger EndsAt value.
+	sindex *index
 }
 
 // NewInhibitRule returns a new InhibitRule based on a configuration definition.
@@ -217,11 +226,85 @@ func NewInhibitRule(cr config.InhibitRule) *InhibitRule {
 		equal[model.LabelName(ln)] = struct{}{}
 	}
 
-	return &InhibitRule{
+	rule := &InhibitRule{
 		SourceMatchers: sourcem,
 		TargetMatchers: targetm,
 		Equal:          equal,
 		scache:         store.NewAlerts(),
+		sindex:         newIndex(),
+	}
+	rule.scache.SetGCCallback(rule.gcCallback)
+
+	return rule
+}
+
+// fingerprintEquals returns the fingerprint of the equal labels of the given label set.
+func (r *InhibitRule) fingerprintEquals(lset model.LabelSet) model.Fingerprint {
+	equalSet := model.LabelSet{}
+	for n := range r.Equal {
+		equalSet[n] = lset[n]
+	}
+	return equalSet.Fingerprint()
+}
+
+// updateIndex updates the source alert index if necessary.
+func (r *InhibitRule) updateIndex(alert *types.Alert) {
+	fp := alert.Fingerprint()
+	// Calculate source labelset subset which is in equals.
+	eq := r.fingerprintEquals(alert.Labels)
+
+	// Check if the equal labelset is already in the index.
+	indexed, ok := r.sindex.Get(eq)
+	if !ok {
+		// If not, add it.
+		r.sindex.Set(eq, fp)
+		return
+	}
+	// If the indexed fingerprint is the same as the new fingerprint, do nothing.
+	if indexed == fp {
+		return
+	}
+
+	// New alert and existing index are not the same, compare them.
+	existing, err := r.scache.Get(indexed)
+	if err != nil {
+		// failed to get the existing alert, overwrite the index.
+		r.sindex.Set(eq, fp)
+		return
+	}
+
+	// If the new alert resolves after the existing alert, replace the index.
+	if existing.ResolvedAt(alert.EndsAt) {
+		r.sindex.Set(eq, fp)
+		return
+	}
+	// If the existing alert resolves after the new alert, do nothing.
+}
+
+// findEqualSourceAlert returns the source alert that matches the equal labels of the given label set.
+func (r *InhibitRule) findEqualSourceAlert(lset model.LabelSet, now time.Time) (*types.Alert, bool) {
+	equalsFP := r.fingerprintEquals(lset)
+	sourceFP, ok := r.sindex.Get(equalsFP)
+	if ok {
+		alert, err := r.scache.Get(sourceFP)
+		if err != nil {
+			return nil, false
+		}
+
+		if alert.ResolvedAt(now) {
+			return nil, false
+		}
+
+		return alert, true
+	}
+
+	return nil, false
+}
+
+func (r *InhibitRule) gcCallback(alerts []types.Alert) {
+	for _, a := range alerts {
+		fp := r.fingerprintEquals(a.Labels)
+		r.sindex.Delete(fp)
 	}
 }
 
@@ -231,21 +314,13 @@ func NewInhibitRule(cr config.InhibitRule) *InhibitRule {
 // source and the target side of the rule are disregarded.
 func (r *InhibitRule) hasEqual(lset model.LabelSet, excludeTwoSidedMatch bool) (model.Fingerprint, bool) {
 	now := time.Now()
-Outer:
-	for _, a := range r.scache.List() {
-		// The cache might be stale and contain resolved alerts.
-		if a.ResolvedAt(now) {
-			continue
+	equal, found := r.findEqualSourceAlert(lset, now)
+	if found {
+		if excludeTwoSidedMatch && r.TargetMatchers.Matches(equal.Labels) {
+			return model.Fingerprint(0), false
 		}
-		for n := range r.Equal {
-			if a.Labels[n] != lset[n] {
-				continue Outer
-			}
-		}
-		if excludeTwoSidedMatch && r.TargetMatchers.Matches(a.Labels) {
-			continue Outer
-		}
-		return a.Fingerprint(), true
+		return equal.Fingerprint(), found
 	}
+
 	return model.Fingerprint(0), false
 }

inhibit/inhibit_bench_test.go

@@ -1,4 +1,4 @@
-// Copyright 2024 The Prometheus Authors
+// Copyright The Prometheus Authors
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at

inhibit/inhibit_test.go

@@ -1,4 +1,4 @@
-// Copyright 2016 Prometheus Team
+// Copyright The Prometheus Authors
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
@@ -35,25 +35,26 @@ func TestInhibitRuleHasEqual(t *testing.T) {
 
 	now := time.Now()
 	cases := []struct {
+		name    string
 		initial map[model.Fingerprint]*types.Alert
 		equal   model.LabelNames
 		input   model.LabelSet
 		result  bool
 	}{
 		{
-			// No source alerts at all.
+			name:    "no source alerts",
 			initial: map[model.Fingerprint]*types.Alert{},
 			input:   model.LabelSet{"a": "b"},
 			result:  false,
 		},
 		{
-			// No equal labels, any source alerts satisfies the requirement.
+			name:    "no equal labels, any source alerts satisfies the requirement",
 			initial: map[model.Fingerprint]*types.Alert{1: {}},
 			input:   model.LabelSet{"a": "b"},
 			result:  true,
 		},
 		{
-			// Matching but already resolved.
+			name: "matching but already resolved",
 			initial: map[model.Fingerprint]*types.Alert{
 				1: {
 					Alert: model.Alert{
@@ -75,7 +76,7 @@ func TestInhibitRuleHasEqual(t *testing.T) {
 			result: false,
 		},
 		{
-			// Matching and unresolved.
+			name: "matching and unresolved",
 			initial: map[model.Fingerprint]*types.Alert{
 				1: {
 					Alert: model.Alert{
@@ -97,7 +98,7 @@ func TestInhibitRuleHasEqual(t *testing.T) {
 			result: true,
 		},
 		{
-			// Equal label does not match.
+			name: "equal label does not match",
 			initial: map[model.Fingerprint]*types.Alert{
 				1: {
 					Alert: model.Alert{
@@ -121,20 +122,24 @@ func TestInhibitRuleHasEqual(t *testing.T) {
 	}
 
 	for _, c := range cases {
-		r := &InhibitRule{
-			Equal:  map[model.LabelName]struct{}{},
-			scache: store.NewAlerts(),
-		}
-		for _, ln := range c.equal {
-			r.Equal[ln] = struct{}{}
-		}
-		for _, v := range c.initial {
-			r.scache.Set(v)
-		}
+		t.Run(c.name, func(t *testing.T) {
+			r := &InhibitRule{
+				Equal:  map[model.LabelName]struct{}{},
+				scache: store.NewAlerts(),
+				sindex: newIndex(),
+			}
+			for _, ln := range c.equal {
+				r.Equal[ln] = struct{}{}
+			}
+			for _, v := range c.initial {
+				r.scache.Set(v)
+				r.updateIndex(v)
+			}
 
-		if _, have := r.hasEqual(c.input, false); have != c.result {
-			t.Errorf("Unexpected result %t, expected %t", have, c.result)
-		}
+			if _, have := r.hasEqual(c.input, false); have != c.result {
+				t.Errorf("Unexpected result %t, expected %t", have, c.result)
+			}
+		})
 	}
 }
 
@@ -174,8 +179,13 @@ func TestInhibitRuleMatches(t *testing.T) {
 
 	ih.rules[0].scache = store.NewAlerts()
 	ih.rules[0].scache.Set(sourceAlert1)
+	ih.rules[0].sindex = newIndex()
+	ih.rules[0].updateIndex(sourceAlert1)
+
 	ih.rules[1].scache = store.NewAlerts()
 	ih.rules[1].scache.Set(sourceAlert2)
+	ih.rules[1].sindex = newIndex()
+	ih.rules[1].updateIndex(sourceAlert2)
 
 	cases := []struct {
 		target   model.LabelSet
@@ -270,8 +280,13 @@ func TestInhibitRuleMatchers(t *testing.T) {
 
 	ih.rules[0].scache = store.NewAlerts()
 	ih.rules[0].scache.Set(sourceAlert1)
+	ih.rules[0].sindex = newIndex()
+	ih.rules[0].updateIndex(sourceAlert1)
+
 	ih.rules[1].scache = store.NewAlerts()
 	ih.rules[1].scache.Set(sourceAlert2)
+	ih.rules[1].sindex = newIndex()
+	ih.rules[1].updateIndex(sourceAlert2)
 
 	cases := []struct {
 		target   model.LabelSet