inject all metrics into default registry, remove extra metrics server…

… exposed at 8085

Signed-off-by: Jaideep Rao <jaideep.r97@gmail.com>

bundle/manifests/argocd-operator-controller-manager-metrics-monitor_monitoring.coreos.com_v1_servicemonitor.yaml

@@ -0,0 +1,17 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  labels:
+    control-plane: argocd-operator
+  name: argocd-operator-controller-manager-metrics-monitor
+spec:
+  endpoints:
+  - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+    path: /metrics
+    port: https
+    scheme: https
+    tlsConfig:
+      insecureSkipVerify: true
+  selector:
+    matchLabels:
+      control-plane: argocd-operator

bundle/manifests/argocd-operator-controller-manager-metrics-service_v1_service.yaml

@@ -7,12 +7,9 @@ metadata:
   name: argocd-operator-controller-manager-metrics-service
 spec:
   ports:
-  - name: default-metrics
-    port: 8080
-    targetPort: 8080
-  - name: custom-metrics
-    port: 8085
-    targetPort: 8085
+  - name: https
+    port: 8443
+    targetPort: https
   selector:
     control-plane: argocd-operator
 status:

bundle/manifests/argocd-operator.clusterserviceversion.yaml

@@ -549,8 +549,8 @@ spec:
         - urn:alm:descriptor:com.tectonic.ui:fieldGroup:RBAC
         - urn:alm:descriptor:com.tectonic.ui:text
       - description: 'Policy is CSV containing user-defined RBAC policies and role
-          definitions. Policy rules are in the form:   p, subject, resource, action,
-          object, effect Role definitions and bindings are in the form:   g, subject,
+          definitions. Policy rules are in the form: p, subject, resource, action,
+          object, effect Role definitions and bindings are in the form: g, subject,
           inherited-subject See https://github.com/argoproj/argo-cd/blob/master/docs/operator-manual/rbac.md
           for additional information.'
         displayName: Policy
@@ -1004,6 +1004,7 @@ spec:
           - monitoring.coreos.com
           resources:
           - prometheuses
+          - prometheusrules
           - servicemonitors
           verbs:
           - '*'
@@ -1067,7 +1068,9 @@ spec:
           - create
         serviceAccountName: argocd-operator-controller-manager
       deployments:
-      - name: argocd-operator-controller-manager
+      - label:
+          control-plane: argocd-operator
+        name: argocd-operator-controller-manager
         spec:
           replicas: 1
           selector:

common/defaults.go

@@ -299,11 +299,8 @@ vs-ssh.visualstudio.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC7Hr1oTWqNqOlzGJOf
 github.com ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEmKSENjQEezOmxkZMy7opKgwFB9nkt5YRrYMjNuG5N87uRgg6CLrbo5wAdT/y6v0mKV0U2w0WZ2YB/++Tpockg=
 github.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl
 `
-	// OperatorDefaultMetricsPort is the port that is used to expose default controller-runtime metrics for the operator pod.
-	OperatorDefaultMetricsPort = 8080
-
-	// OperatorDefaultMetricsPort is the port that is used to expose custom metrics implemented by the Argo CD Operator.
-	OperatorCustomMetricsPort = 8085
+	// OperatorMetricsPort is the port that is used to expose default controller-runtime metrics for the operator pod.
+	OperatorMetricsPort = 8080
 )
 
 // DefaultLabels returns the default set of labels for controllers.

config/default/kustomization.yaml

@@ -22,7 +22,7 @@ bases:
 # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required.
 #- ../certmanager
 # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'.
-#- ../prometheus
+- ../prometheus
 
 patchesStrategicMerge:
 # Protect the /metrics endpoint by putting it behind auth.

config/prometheus/monitor.yaml

@@ -10,15 +10,7 @@ metadata:
 spec:
   endpoints:
     - path: /metrics
-      interval: 30s
-      port: default-metrics
-      scheme: https
-      bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
-      tlsConfig:
-        insecureSkipVerify: true
-    - path: /metrics
-      interval: 10s
-      port: cr-metrics
+      port: https
       scheme: https
       bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
       tlsConfig:

config/rbac/auth_proxy_service.yaml

@@ -7,11 +7,8 @@ metadata:
   namespace: system
 spec:
   ports:
-  - name: default-metrics
-    port: 8080
-    targetPort: 8080
-  - name: custom-metrics
-    port: 8085
-    targetPort: 8085
+  - name: https
+    port: 8443
+    targetPort: https
   selector:
     control-plane: argocd-operator

config/rbac/role.yaml

@@ -99,6 +99,7 @@ rules:
   - monitoring.coreos.com
   resources:
   - prometheuses
+  - prometheusrules
   - servicemonitors
   verbs:
   - '*'

controllers/argocd/argocd_controller.go

@@ -64,7 +64,7 @@ var ActiveInstanceMap = make(map[string]string)
 //+kubebuilder:rbac:groups=batch,resources=cronjobs;jobs,verbs=*
 //+kubebuilder:rbac:groups=config.openshift.io,resources=clusterversions,verbs=get;list;watch
 //+kubebuilder:rbac:groups=networking.k8s.io,resources=ingresses,verbs=*
-//+kubebuilder:rbac:groups=monitoring.coreos.com,resources=prometheuses;servicemonitors,verbs=*
+//+kubebuilder:rbac:groups=monitoring.coreos.com,resources=prometheuses;servicemonitors;prometheusrules,verbs=*
 //+kubebuilder:rbac:groups=route.openshift.io,resources=routes;routes/custom-host,verbs=*
 //+kubebuilder:rbac:groups=argoproj.io,resources=applications;appprojects,verbs=*
 //+kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=*,verbs=*

controllers/argocd/metrics.go

@@ -5,20 +5,20 @@ import (
 	"net/http"
 
 	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promauto"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"sigs.k8s.io/controller-runtime/pkg/metrics"
 )
 
 var (
-	ActiveInstancesByPhase = promauto.NewGaugeVec(
+	ActiveInstancesByPhase = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
 			Name: "active_argocd_instances_by_phase",
 			Help: "Number of active argocd instances by phase",
 		},
 		[]string{"phase"},
 	)
 
-	ActiveInstancesTotal = promauto.NewGauge(
+	ActiveInstancesTotal = prometheus.NewGauge(
 		prometheus.GaugeOpts{
 			Name: "active_argocd_instances_total",
 			Help: "Total number of active argocd instances",
@@ -27,8 +27,8 @@ var (
 
 	// ReconcileTime is a prometheus metric which keeps track of the duration
 	// of reconciliations for a given instance
-	ReconcileTime = promauto.NewHistogramVec(prometheus.HistogramOpts{
-		Name: "controller_runtime_reconcile_time_seconds",
+	ReconcileTime = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Name: "controller_runtime_reconcile_time_seconds_per_instance",
 		Help: "Length of time per reconciliation per instance",
 		Buckets: []float64{0.005, 0.01, 0.025, 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, 0.45, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0,
 			1.25, 1.5, 1.75, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5, 6, 7, 8, 9, 10, 15, 20, 25, 30, 40, 50, 60},
@@ -45,3 +45,7 @@ func StartMetricsServer(port int) chan error {
 	}()
 	return errCh
 }
+
+func init() {
+	metrics.Registry.MustRegister(ActiveInstancesTotal, ActiveInstancesByPhase, ReconcileTime)
+}

deploy/olm-catalog/argocd-operator/0.7.0/argocd-operator-controller-manager-metrics-monitor_monitoring.coreos.com_v1_servicemonitor.yaml

@@ -0,0 +1,17 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  labels:
+    control-plane: argocd-operator
+  name: argocd-operator-controller-manager-metrics-monitor
+spec:
+  endpoints:
+  - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+    path: /metrics
+    port: https
+    scheme: https
+    tlsConfig:
+      insecureSkipVerify: true
+  selector:
+    matchLabels:
+      control-plane: argocd-operator

deploy/olm-catalog/argocd-operator/0.7.0/argocd-operator.v0.7.0.clusterserviceversion.yaml

@@ -549,8 +549,8 @@ spec:
         - urn:alm:descriptor:com.tectonic.ui:fieldGroup:RBAC
         - urn:alm:descriptor:com.tectonic.ui:text
       - description: 'Policy is CSV containing user-defined RBAC policies and role
-          definitions. Policy rules are in the form:   p, subject, resource, action,
-          object, effect Role definitions and bindings are in the form:   g, subject,
+          definitions. Policy rules are in the form: p, subject, resource, action,
+          object, effect Role definitions and bindings are in the form: g, subject,
           inherited-subject See https://github.com/argoproj/argo-cd/blob/master/docs/operator-manual/rbac.md
           for additional information.'
         displayName: Policy
@@ -1004,6 +1004,7 @@ spec:
           - monitoring.coreos.com
           resources:
           - prometheuses
+          - prometheusrules
           - servicemonitors
           verbs:
           - '*'
@@ -1067,7 +1068,9 @@ spec:
           - create
         serviceAccountName: argocd-operator-controller-manager
       deployments:
-      - name: argocd-operator-controller-manager
+      - label:
+          control-plane: argocd-operator
+        name: argocd-operator-controller-manager
         spec:
           replicas: 1
           selector:

main.go

@@ -77,7 +77,7 @@ func main() {
 	var metricsAddr string
 	var enableLeaderElection bool
 	var probeAddr string
-	flag.StringVar(&metricsAddr, "metrics-bind-address", fmt.Sprintf(":%d", common.OperatorDefaultMetricsPort), "The address the metric endpoint binds to.")
+	flag.StringVar(&metricsAddr, "metrics-bind-address", fmt.Sprintf(":%d", common.OperatorMetricsPort), "The address the metric endpoint binds to.")
 	flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
 	flag.BoolVar(&enableLeaderElection, "leader-elect", false,
 		"Enable leader election for controller manager. "+
@@ -202,16 +202,6 @@ func main() {
 		os.Exit(1)
 	}
 
-	// start a new metrics server at a different port to serve custom implemented metrics
-	// This step is being taken as a workaround since custom metrics were not getting registered
-	// within the same default registry as is being used out of the box
-	go func() {
-		msErrCh := argocd.StartMetricsServer(common.OperatorCustomMetricsPort)
-		if err = <-msErrCh; err != nil {
-			setupLog.Error(err, "metrics server exited with error: %v", err)
-		}
-	}()
-
 	setupLog.Info("starting manager")
 	if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
 		setupLog.Error(err, "problem running manager")