add total reconciliations counter, change bucket sizes for histogram

Signed-off-by: Jaideep Rao <jaideep.r97@gmail.com>

controllers/argocd/argocd_controller.go

@@ -22,6 +22,7 @@ import (
 	"time"
 
 	argoproj "github.com/argoproj-labs/argocd-operator/api/v1alpha1"
+	"github.com/prometheus/client_golang/prometheus"
 
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
@@ -110,8 +111,8 @@ func (r *ReconcileArgoCD) Reconcile(ctx context.Context, request ctrl.Request) (
 	if _, ok := ActiveInstanceMap[request.Namespace]; !ok {
 		if newPhase != "" {
 			ActiveInstanceMap[request.Namespace] = newPhase
-			ActiveInstancesByPhase.WithLabelValues(newPhase).Add(1)
-			ActiveInstancesTotal.Add(1)
+			ActiveInstancesByPhase.WithLabelValues(newPhase).Inc()
+			ActiveInstancesTotal.Inc()
 		}
 	} else {
 		// If we discover an existing instance's phase has changed since we last saw it
@@ -120,18 +121,21 @@ func (r *ReconcileArgoCD) Reconcile(ctx context.Context, request ctrl.Request) (
 		// total instance count remains the same
 		if oldPhase := ActiveInstanceMap[argocd.Namespace]; oldPhase != newPhase {
 			ActiveInstanceMap[argocd.Namespace] = newPhase
-			ActiveInstancesByPhase.WithLabelValues(newPhase).Add(1)
-			ActiveInstancesByPhase.WithLabelValues(oldPhase).Add(-1)
+			ActiveInstancesByPhase.WithLabelValues(newPhase).Inc()
+			ActiveInstancesByPhase.WithLabelValues(oldPhase).Dec()
 		}
+		ActiveInstanceReconciliationCount.WithLabelValues(argocd.Namespace).Inc()
 	}
 
 	if argocd.GetDeletionTimestamp() != nil {
 
 		// Argo CD instance marked for deletion; remove entry from activeInstances map and decrement active instance count
 		// by phase as well as total
 		delete(ActiveInstanceMap, argocd.Namespace)
-		ActiveInstancesByPhase.WithLabelValues(newPhase).Add(-1)
-		ActiveInstancesTotal.Add(-1)
+		ActiveInstancesByPhase.WithLabelValues(newPhase).Dec()
+		ActiveInstancesTotal.Dec()
+		ActiveInstanceReconciliationCount.DeleteLabelValues(argocd.Namespace)
+		ReconcileTime.DeletePartialMatch(prometheus.Labels{"namespace": argocd.Namespace})
 
 		if argocd.IsDeletionFinalizerPresent() {
 			if err := r.deleteClusterResources(argocd); err != nil {

controllers/argocd/metrics.go

@@ -25,13 +25,20 @@ var (
 		},
 	)
 
+	ActiveInstanceReconciliationCount = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "active_argocd_instance_reconciliation_count",
+			Help: "Number of reconciliations performed for a given instance",
+		},
+		[]string{"namespace"},
+	)
+
 	// ReconcileTime is a prometheus metric which keeps track of the duration
 	// of reconciliations for a given instance
 	ReconcileTime = prometheus.NewHistogramVec(prometheus.HistogramOpts{
-		Name: "controller_runtime_reconcile_time_seconds_per_instance",
-		Help: "Length of time per reconciliation per instance",
-		Buckets: []float64{0.005, 0.01, 0.025, 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, 0.45, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0,
-			1.25, 1.5, 1.75, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5, 6, 7, 8, 9, 10, 15, 20, 25, 30, 40, 50, 60},
+		Name:    "controller_runtime_reconcile_time_seconds_per_instance",
+		Help:    "Length of time per reconciliation per instance",
+		Buckets: []float64{0.05, 0.075, 0.1, 0.15, 0.2, 0.22, 0.24, 0.26, 0.28, 0.3, 0.32, 0.34, 0.37, 0.4, 0.42, 0.44, 0.48, 0.5, 0.55, 0.6, 0.75, 0.9, 1.00},
 	}, []string{"namespace"})
 )
 
@@ -47,5 +54,5 @@ func StartMetricsServer(port int) chan error {
 }
 
 func init() {
-	metrics.Registry.MustRegister(ActiveInstancesTotal, ActiveInstancesByPhase, ReconcileTime)
+	metrics.Registry.MustRegister(ActiveInstancesTotal, ActiveInstancesByPhase, ActiveInstanceReconciliationCount, ReconcileTime)
 }