feat: Notification service monitor (#1187)

* feat: Expose notifications controller metrics to prometheus monitoring

---------

Signed-off-by: iam-veeramalla <abhishek.veeramalla@gmail.com>

build/util/Dockerfile

@@ -48,4 +48,4 @@ ENV USER_NAME=argocd
 ENV HOME=/home/argocd
 
 USER argocd
-WORKDIR /home/argocd
+WORKDIR /home/argocd

common/defaults.go

@@ -306,6 +306,9 @@ vs-ssh.visualstudio.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC7Hr1oTWqNqOlzGJOf
 `
 	// OperatorMetricsPort is the port that is used to expose default controller-runtime metrics for the operator pod.
 	OperatorMetricsPort = 8080
+
+	// NotificationsControllerMetricsPort is the port that is used to expose notifications controller metrics.
+	NotificationsControllerMetricsPort = 9001
 )
 
 // DefaultLabels returns the default set of labels for controllers.

controllers/argocd/notifications.go

@@ -6,10 +6,12 @@ import (
 	"reflect"
 	"time"
 
+	monitoringv1 "github.com/coreos/prometheus-operator/pkg/apis/monitoring/v1"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
@@ -53,6 +55,18 @@ func (r *ReconcileArgoCD) reconcileNotificationsController(cr *argoproj.ArgoCD)
 		return err
 	}
 
+	log.Info("reconciling notifications metrics service")
+	if err := r.reconcileNotificationsMetricsService(cr); err != nil {
+		return err
+	}
+
+	if prometheusAPIFound {
+		log.Info("reconciling notifications metrics service monitor")
+		if err := r.reconcileNotificationsServiceMonitor(cr); err != nil {
+			return err
+		}
+	}
+
 	return nil
 }
 
@@ -82,6 +96,16 @@ func (r *ReconcileArgoCD) deleteNotificationsResources(cr *argoproj.ArgoCD) erro
 		return err
 	}
 
+	log.Info("reconciling notifications service")
+	if err := r.reconcileNotificationsMetricsService(cr); err != nil {
+		return err
+	}
+
+	log.Info("reconciling notifications service monitor")
+	if err := r.reconcileNotificationsServiceMonitor(cr); err != nil {
+		return err
+	}
+
 	log.Info("reconciling notifications secret")
 	if err := r.reconcileNotificationsSecret(cr); err != nil {
 		return err
@@ -432,6 +456,64 @@ func (r *ReconcileArgoCD) reconcileNotificationsDeployment(cr *argoproj.ArgoCD,
 
 }
 
+// reconcileNotificationsService will ensure that the Service for the Notifications controller metrics is present.
+func (r *ReconcileArgoCD) reconcileNotificationsMetricsService(cr *argoproj.ArgoCD) error {
+
+	var component = "notifications-controller"
+	var suffix = "notifications-controller-metrics"
+
+	svc := newServiceWithSuffix(suffix, component, cr)
+	if argoutil.IsObjectFound(r.Client, cr.Namespace, svc.Name, svc) {
+		// Service found, do nothing
+		return nil
+	}
+
+	svc.Spec.Selector = map[string]string{
+		common.ArgoCDKeyName: nameWithSuffix(component, cr),
+	}
+
+	svc.Spec.Ports = []corev1.ServicePort{
+		{
+			Name:       "metrics",
+			Port:       common.NotificationsControllerMetricsPort,
+			Protocol:   corev1.ProtocolTCP,
+			TargetPort: intstr.FromInt(common.NotificationsControllerMetricsPort),
+		},
+	}
+
+	if err := controllerutil.SetControllerReference(cr, svc, r.Scheme); err != nil {
+		return err
+	}
+	return r.Client.Create(context.TODO(), svc)
+}
+
+// reconcileNotificationsServiceMonitor will ensure that the ServiceMonitor for the Notifications controller metrics is present.
+func (r *ReconcileArgoCD) reconcileNotificationsServiceMonitor(cr *argoproj.ArgoCD) error {
+
+	name := fmt.Sprintf("%s-%s", cr.Name, "notifications-controller-metrics")
+	serviceMonitor := newServiceMonitorWithName(name, cr)
+	if argoutil.IsObjectFound(r.Client, cr.Namespace, serviceMonitor.Name, serviceMonitor) {
+		// Service found, do nothing
+		return nil
+	}
+
+	serviceMonitor.Spec.Selector = v1.LabelSelector{
+		MatchLabels: map[string]string{
+			common.ArgoCDKeyName: name,
+		},
+	}
+
+	serviceMonitor.Spec.Endpoints = []monitoringv1.Endpoint{
+		{
+			Port:     "metrics",
+			Scheme:   "http",
+			Interval: "30s",
+		},
+	}
+
+	return r.Client.Create(context.TODO(), serviceMonitor)
+}
+
 // reconcileNotificationsConfigMap only creates/deletes the argocd-notifications-cm based on whether notifications is enabled/disabled in the CR
 // It does not reconcile/overwrite any fields or information in the configmap itself
 func (r *ReconcileArgoCD) reconcileNotificationsConfigMap(cr *argoproj.ArgoCD) error {

controllers/argocd/notifications_test.go

@@ -2,12 +2,15 @@ package argocd
 
 import (
 	"context"
+	"fmt"
 	"testing"
 
+	monitoringv1 "github.com/coreos/prometheus-operator/pkg/apis/monitoring/v1"
 	"github.com/google/go-cmp/cmp"
 	"github.com/stretchr/testify/assert"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -252,6 +255,90 @@ func TestReconcileNotifications_CreateDeployments(t *testing.T) {
 	assert.True(t, errors.IsNotFound(err))
 }
 
+func TestReconcileNotifications_CreateMetricsService(t *testing.T) {
+	a := makeTestArgoCD(func(a *argoproj.ArgoCD) {
+		a.Spec.Notifications.Enabled = true
+	})
+
+	resObjs := []client.Object{a}
+	subresObjs := []client.Object{a}
+	runtimeObjs := []runtime.Object{}
+	sch := makeTestReconcilerScheme(argoproj.AddToScheme)
+	cl := makeTestReconcilerClient(sch, resObjs, subresObjs, runtimeObjs)
+	r := makeTestReconciler(cl, sch)
+
+	err := monitoringv1.AddToScheme(r.Scheme)
+	assert.NoError(t, err)
+
+	err = r.reconcileNotificationsMetricsService(a)
+	assert.NoError(t, err)
+
+	testService := &corev1.Service{}
+	assert.NoError(t, r.Client.Get(context.TODO(), types.NamespacedName{
+		Name:      fmt.Sprintf("%s-%s", a.Name, "notifications-controller-metrics"),
+		Namespace: a.Namespace,
+	}, testService))
+
+	assert.Equal(t, testService.ObjectMeta.Labels["app.kubernetes.io/name"],
+		fmt.Sprintf("%s-%s", a.Name, "notifications-controller-metrics"))
+
+	assert.Equal(t, testService.Spec.Selector["app.kubernetes.io/name"],
+		fmt.Sprintf("%s-%s", a.Name, "notifications-controller"))
+
+	assert.Equal(t, testService.Spec.Ports[0].Port, int32(9001))
+	assert.Equal(t, testService.Spec.Ports[0].TargetPort, intstr.IntOrString{
+		IntVal: int32(9001),
+	})
+	assert.Equal(t, testService.Spec.Ports[0].Protocol, v1.Protocol("TCP"))
+	assert.Equal(t, testService.Spec.Ports[0].Name, "metrics")
+}
+
+func TestReconcileNotifications_CreateServiceMonitor(t *testing.T) {
+
+	a := makeTestArgoCD(func(a *argoproj.ArgoCD) {
+		a.Spec.Notifications.Enabled = true
+	})
+
+	resObjs := []client.Object{a}
+	subresObjs := []client.Object{a}
+	runtimeObjs := []runtime.Object{}
+	sch := makeTestReconcilerScheme(argoproj.AddToScheme)
+	monitoringv1.AddToScheme(sch)
+
+	cl := makeTestReconcilerClient(sch, resObjs, subresObjs, runtimeObjs)
+	r := makeTestReconciler(cl, sch)
+
+	// Notifications controller service monitor should not be created when Prometheus API is not found.
+	prometheusAPIFound = false
+	err := r.reconcileNotificationsController(a)
+	assert.NoError(t, err)
+
+	testServiceMonitor := &monitoringv1.ServiceMonitor{}
+	assert.Error(t, r.Client.Get(context.TODO(), types.NamespacedName{
+		Name:      fmt.Sprintf("%s-%s", a.Name, "notifications-controller-metrics"),
+		Namespace: a.Namespace,
+	}, testServiceMonitor))
+
+	// Prometheus API found, Verify notification controller service monitor exists.
+	prometheusAPIFound = true
+	err = r.reconcileNotificationsController(a)
+	assert.NoError(t, err)
+
+	testServiceMonitor = &monitoringv1.ServiceMonitor{}
+	assert.NoError(t, r.Client.Get(context.TODO(), types.NamespacedName{
+		Name:      fmt.Sprintf("%s-%s", a.Name, "notifications-controller-metrics"),
+		Namespace: a.Namespace,
+	}, testServiceMonitor))
+
+	assert.Equal(t, testServiceMonitor.ObjectMeta.Labels["release"], "prometheus-operator")
+
+	assert.Equal(t, testServiceMonitor.Spec.Endpoints[0].Port, "metrics")
+	assert.Equal(t, testServiceMonitor.Spec.Endpoints[0].Scheme, "http")
+	assert.Equal(t, testServiceMonitor.Spec.Endpoints[0].Interval, "30s")
+	assert.Equal(t, testServiceMonitor.Spec.Selector.MatchLabels["app.kubernetes.io/name"],
+		fmt.Sprintf("%s-%s", a.Name, "notifications-controller-metrics"))
+}
+
 func TestReconcileNotifications_CreateSecret(t *testing.T) {
 	logf.SetLogger(ZapLogger(true))
 	a := makeTestArgoCD(func(a *argoproj.ArgoCD) {

examples/argocd-notifications.yaml

@@ -1,10 +1,16 @@
-apiVersion: argoproj.io/v1alpha1
+apiVersion: argoproj.io/v1beta1
 kind: ArgoCD
 metadata:
   name: example-argocd
+  labels:
+    example: route
 spec:
   notifications:
-    enabled: True
+    enabled: true
+  prometheus:
+    enabled: true
+    route:
+      enabled: true
   server:
-    ingress:
-      enabled: true
+    route:
+      enabled: true

tests/k8s/1-021_validate_notification_controller/02-assert.yaml

@@ -6,6 +6,22 @@ status:
   phase: Available
   notificationsController: Running
 ---
+kind: Service
+apiVersion: v1
+metadata:
+  name: example-argocd-notifications-controller-metrics
+---
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  name: example-argocd-notifications-controller
+status:
+  conditions:
+    - type: Available
+      status: 'True'
+    - type: Progressing
+      status: 'True'
+---
 kind: Deployment
 apiVersion: apps/v1
 metadata:

tests/k8s/1-027_validate_applicationset_status/03-change-appset-image.yaml

@@ -4,4 +4,4 @@ metadata:
   name: example-argocd
 spec:
   applicationSet:
-    image: quay.io/argoproj/argocd@sha256:8576d347f30fa4c56a0129d1c0a0f5ed1e75662f0499f1ed7e917c405fd909dc
+    image: quay.io/argoproj/argocd@sha256:8576d347f30fa4c56a0129d1c0a0f5ed1e75662f0499f1ed7e917c405fd909dc