fix: add labels to metrics and discover servicemonitors in all namesp…

…aces (#821)

Signed-off-by: Lenin Mehedy <lenin.mehedy@swirldslabs.com>

charts/fullstack-cluster-setup/values.yaml

@@ -24,4 +24,21 @@ cert-manager:
 
 minio-operator:
   operator:
-    replicaCount: 1
+    replicaCount: 1
+
+prometheus-stack:
+  prometheus:
+    prometheusSpec:
+      scrapeInterval: 5s
+      podMonitorNamespaceSelector:
+        any: true # fetch metrics from other namespaces
+      podMonitorSelector: { }
+      podMonitorSelectorNilUsesHelmValues: false
+      ruleNamespaceSelector:
+        any: true # fetch metrics from other namespaces
+      ruleSelector: { }
+      ruleSelectorNilUsesHelmValues: false
+      serviceMonitorNamespaceSelector:
+        any: true  # fetch metrics from other namespaces
+      serviceMonitorSelector: { }
+      serviceMonitorSelectorNilUsesHelmValues: false

charts/fullstack-deployment/config-files/otel-collector-config.yaml

@@ -15,6 +15,25 @@ receivers:
 
 processors:
   batch:
+  # Add standard label sets to both logs and metrics.
+  attributes/addLabels:
+    actions:
+      - key: environment
+        action: insert
+        value: "{{ .environment_name }}"
+      - key: metrics_node_id
+        action: insert
+        value: "{{ .otel_node_id | add1 }}"
+      - key: node_id
+        action: insert
+        value: "{{ .otel_node_id }}"
+      # TODO how to pass otel_instance_hostname and otel_instance_ip?
+      #      - key: inventory_name
+      #        action: insert
+      #        value: "{{ .otel_instance_hostname }} - {{ .otel_instance_ip }}"
+      - key: instance_type
+        action: insert
+        value: "{{ .otel_instance_type }}"
 
 exporters:
   prometheus:
@@ -51,8 +70,11 @@ service:
       processors: [batch]
       exporters: [otlp]
     metrics:
-      receivers: [prometheus]
-      processors: [batch]
+      receivers:
+        - prometheus
+      processors:
+        - attributes/addLabels
+        - batch
       exporters:
         - prometheus
         {{- if .otelDefaults.exporters.prometheusRemoteWrite.enabled }}

charts/fullstack-deployment/templates/configmaps/otel-collector-cm.yaml

@@ -1,10 +1,19 @@
+{{- range $index, $node := ($.Values.hedera.nodes) }}
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: otel-collector-cm
+  name: otel-cm-{{ $node.name }}
   namespace: {{ default $.Release.Namespace $.Values.global.namespaceOverride }}
   labels:
     {{- include "fullstack.testLabels" $ | nindent 4 }}
 data:
   config.yaml: |
-  {{- tpl (.Files.Get "config-files/otel-collector-config.yaml") ( dict "otelDefaults" $.Values.defaults.sidecars.otelCollector "Template" $.Template )  | nindent 4 }}
+  {{- $params := dict "" "" -}}
+  {{- $_ignore := set $params "otelDefaults" $.Values.defaults.sidecars.otelCollector  -}}
+  {{- $_ignore = set $params "Template" $.Template -}}
+  {{- $_ignore = set $params "environment_name" $.Release.Namespace -}}
+  {{- $_ignore = set $params "otel_node_id" $index -}}
+  {{- $_ignore = set $params "otel_instance_type" "hedera-node" -}}
+  {{- tpl ( $.Files.Get "config-files/otel-collector-config.yaml" ) $params | nindent 4 }}
+---
+{{ end }}

charts/fullstack-deployment/templates/network-node-statefulset.yaml

@@ -80,7 +80,7 @@ spec:
         {{- if $otelCollector.enabled }}
         - name: otel-collector-volume
           configMap:
-            name: otel-collector-cm
+            name: otel-cm-{{ $node.name }}
         {{- end }}
       containers:
       # Root Container: {{ $node.name }}-root-container

charts/fullstack-deployment/templates/telemetry/prometheus-svc-monitor.yaml

@@ -13,5 +13,5 @@ spec:
       fullstack.hedera.com/prometheus-endpoint: active
   endpoints:
     - port: prometheus # must match the prometheus port-name in network-node-svc.yaml
-      interval: 10s # ideally it should be higher than the node-metrics-scraper interval set in otel-collector-config.yaml
+      interval: 5s
 {{- end }}