tsdb: record a histogram of replay tsdb times (#2362)

Signed-off-by: Thor <thansen@digitalocean.com>

Co-authored-by: Thor <thansen@digitalocean.com>

Author: thorfour

pkg/ingester/ingester_v2.go

@@ -75,6 +75,7 @@ type TSDBState struct {
 	// Head compactions metrics.
 	compactionsTriggered prometheus.Counter
 	compactionsFailed    prometheus.Counter
+	walReplayTime        prometheus.Histogram
 }
 
 // NewV2 returns a new Ingester that uses prometheus block storage instead of chunk storage
@@ -110,6 +111,11 @@ func NewV2(cfg Config, clientConfig client.Config, limits *validation.Overrides,
 				Name: "cortex_ingester_tsdb_compactions_failed_total",
 				Help: "Total number of compactions that failed.",
 			}),
+			walReplayTime: promauto.With(registerer).NewHistogram(prometheus.HistogramOpts{
+				Name:    "cortex_ingester_tsdb_wal_replay_duration_seconds",
+				Help:    "The total time it takes to open and replay a TSDB WAL.",
+				Buckets: prometheus.DefBuckets,
+			}),
 		},
 	}
 
@@ -857,6 +863,10 @@ func (i *Ingester) openExistingTSDB(ctx context.Context) error {
 		go func(userID string) {
 			defer wg.Done()
 			defer openGate.Done()
+			defer func(ts time.Time) {
+				i.TSDBState.walReplayTime.Observe(time.Since(ts).Seconds())
+			}(time.Now())
+
 			db, err := i.createTSDB(userID)
 			if err != nil {
 				level.Error(util.Logger).Log("msg", "unable to open user TSDB", "err", err, "user", userID)