register notifier metrics under per-user metric registry

Signed-off-by: Roy Chiang <roychi@amazon.com>

Author: roystchiang

pkg/ruler/manager.go

@@ -166,16 +166,16 @@ func (r *DefaultMultiTenantManager) syncRulesToManager(ctx context.Context, user
 // newManager creates a prometheus rule manager wrapped with a user id
 // configured storage, appendable, notifier, and instrumentation
 func (r *DefaultMultiTenantManager) newManager(ctx context.Context, userID string) (RulesManager, error) {
-	notifier, err := r.getOrCreateNotifier(userID)
-	if err != nil {
-		return nil, err
-	}
-
 	// Create a new Prometheus registry and register it within
 	// our metrics struct for the provided user.
 	reg := prometheus.NewRegistry()
 	r.userManagerMetrics.AddUserRegistry(userID, reg)
 
+	notifier, err := r.getOrCreateNotifier(userID, reg)
+	if err != nil {
+		return nil, err
+	}
+
 	return r.managerFactory(ctx, userID, notifier, r.logger, reg), nil
 }
 
@@ -191,7 +191,7 @@ func (r *DefaultMultiTenantManager) removeNotifier(userID string) *rulerNotifier
 	return n
 }
 
-func (r *DefaultMultiTenantManager) getOrCreateNotifier(userID string) (*notifier.Manager, error) {
+func (r *DefaultMultiTenantManager) getOrCreateNotifier(userID string, userManagerRegistry prometheus.Registerer) (*notifier.Manager, error) {
 	r.notifiersMtx.Lock()
 	defer r.notifiersMtx.Unlock()
 
@@ -200,7 +200,7 @@ func (r *DefaultMultiTenantManager) getOrCreateNotifier(userID string) (*notifie
 		return n.notifier, nil
 	}
 
-	reg := prometheus.WrapRegistererWith(prometheus.Labels{"user": userID}, r.registry)
+	reg := prometheus.WrapRegistererWith(prometheus.Labels{"user": userID}, userManagerRegistry)
 	reg = prometheus.WrapRegistererWithPrefix("cortex_", reg)
 	n = newRulerNotifier(&notifier.Options{
 		QueueCapacity: r.cfg.NotificationQueueCapacity,

pkg/ruler/ruler_test.go

@@ -274,7 +274,7 @@ func TestNotifierSendsUserIDHeader(t *testing.T) {
 	defer rcleanup()
 	defer manager.Stop()
 
-	n, err := manager.getOrCreateNotifier("1")
+	n, err := manager.getOrCreateNotifier("1", manager.registry)
 	require.NoError(t, err)
 
 	// Loop until notifier discovery syncs up