Add duration to the notify success message (prometheus#3559)

This commit updates Alertmanager to add a duration to the notify
success message. It complements the existing histogram to offer
fine-grained information about notification attempts. This can be
useful when debuggin duplicate notifications, for example, when
the duration exceeds peer_timeout.

Signed-off-by: George Robinson <george.robinson@grafana.com>

notify/notify.go

@@ -813,12 +813,11 @@ func (r RetryStage) exec(ctx context.Context, l log.Logger, alerts ...*types.Ale
 		case <-tick.C:
 			now := time.Now()
 			retry, err := r.integration.Notify(ctx, sent...)
-
-			duration := time.Since(now)
-			r.metrics.notificationLatencySeconds.WithLabelValues(r.labelValues...).Observe(duration.Seconds())
+			dur := time.Since(now)
+			r.metrics.notificationLatencySeconds.WithLabelValues(r.labelValues...).Observe(dur.Seconds())
 			r.metrics.numNotificationRequestsTotal.WithLabelValues(r.labelValues...).Inc()
 
-			r.integration.Report(now, model.Duration(duration), err)
+			r.integration.Report(now, model.Duration(dur), err)
 			if err != nil {
 				r.metrics.numNotificationRequestsFailedTotal.WithLabelValues(r.labelValues...).Inc()
 				if !retry {
@@ -838,7 +837,7 @@ func (r RetryStage) exec(ctx context.Context, l log.Logger, alerts ...*types.Ale
 					lvl = level.Debug(log.With(l, "alerts", fmt.Sprintf("%v", alerts)))
 				}
 
-				lvl.Log("msg", "Notify success", "attempts", i)
+				lvl.Log("msg", "Notify success", "attempts", i, "duration", dur)
 				return ctx, alerts, nil
 			}
 		case <-ctx.Done():