fix ruler send empty alerts (#5377)

Signed-off-by: Ben Ye <ben.ye@bytedance.com>

pkg/alert/alert.go

@@ -170,7 +170,8 @@ func (q *Queue) Push(alerts []*notifier.Alert) {
 
 	q.pushed.Add(float64(len(alerts)))
 
-	// Attach external labels and drop excluded labels before sending.
+	// Attach external labels, drop excluded labels and process relabeling before sending.
+	var relabeledAlerts []*notifier.Alert
 	for _, a := range alerts {
 		lb := labels.NewBuilder(labels.Labels{})
 		for _, l := range a.Labels {
@@ -183,8 +184,15 @@ func (q *Queue) Push(alerts []*notifier.Alert) {
 			lb.Set(l.Name, l.Value)
 		}
 		a.Labels = relabel.Process(lb.Labels(), q.alertRelabelConfigs...)
+		if a.Labels != nil {
+			relabeledAlerts = append(relabeledAlerts, a)
+		}
 	}
 
+	alerts = relabeledAlerts
+	if len(alerts) == 0 {
+		return
+	}
 	// Queue capacity should be significantly larger than a single alert
 	// batch could be.
 	if d := len(alerts) - q.capacity; d > 0 {

pkg/alert/alert_test.go

@@ -91,6 +91,27 @@ func TestQueue_Push_Relabelled_Alerts(t *testing.T) {
 	)
 }
 
+func TestQueue_Push_RelabelDropAlerts(t *testing.T) {
+	q := NewQueue(nil, nil, 10, 10, nil, nil,
+		[]*relabel.Config{
+			{
+				SourceLabels: model.LabelNames{"a"},
+				Regex:        relabel.MustNewRegexp("1"),
+				Action:       relabel.Drop,
+			},
+		})
+
+	q.Push([]*notifier.Alert{
+		{Labels: labels.FromStrings("a", "1")},
+		{Labels: labels.FromStrings("a", "2")},
+		{Labels: labels.FromStrings("b", "3")},
+	})
+
+	testutil.Equals(t, 2, len(q.queue))
+	testutil.Equals(t, labels.FromStrings("a", "2"), q.queue[0].Labels)
+	testutil.Equals(t, labels.FromStrings("b", "3"), q.queue[1].Labels)
+}
+
 func assertSameHosts(t *testing.T, expected, found []*url.URL) {
 	testutil.Equals(t, len(expected), len(found))