Add debug logs for muted alerts (prometheus#3558)

This commit adds debug logs to MuteStage that logs when an alert
is muted. This can help operators root cause missing notifications
when alerts are silenced by mistake or purpose but then forgotten
about.

Signed-off-by: George Robinson <george.robinson@grafana.com>

Author: grobinson-grafana

notify/notify.go

@@ -539,16 +539,24 @@ func NewMuteStage(m types.Muter) *MuteStage {
 }
 
 // Exec implements the Stage interface.
-func (n *MuteStage) Exec(ctx context.Context, _ log.Logger, alerts ...*types.Alert) (context.Context, []*types.Alert, error) {
-	var filtered []*types.Alert
+func (n *MuteStage) Exec(ctx context.Context, logger log.Logger, alerts ...*types.Alert) (context.Context, []*types.Alert, error) {
+	var (
+		filtered []*types.Alert
+		muted    []*types.Alert
+	)
 	for _, a := range alerts {
 		// TODO(fabxc): increment total alerts counter.
 		// Do not send the alert if muted.
-		if !n.muter.Mutes(a.Labels) {
+		if n.muter.Mutes(a.Labels) {
+			muted = append(muted, a)
+		} else {
 			filtered = append(filtered, a)
 		}
 		// TODO(fabxc): increment muted alerts counter if muted.
 	}
+	if len(muted) > 0 {
+		level.Debug(logger).Log("msg", "Notifications will not be sent for muted alerts", "alerts", fmt.Sprintf("%v", muted))
+	}
 	return ctx, filtered, nil
 }