Fix k8s monitoring (#17)

* Add k8sattributes processor to otel collector

* Fix monitoring part of kubernetes internals

---------

Co-authored-by: Frédéric Collonval <fcollonval@users.noreply.github.com>

charts/datalayer-observer/templates/collector-cluster-role.yaml

@@ -0,0 +1,29 @@
+{{- if index .Values "opentelemetry-operator" "enabled" }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: datalayer-collector
+rules:
+- apiGroups: [""]
+  resources: ["pods", "namespaces", "nodes"]
+  verbs: ["get", "watch", "list"]
+- apiGroups: ["apps"]
+  resources: ["replicasets"]
+  verbs: ["get", "list", "watch"]
+- apiGroups: ["extensions"]
+  resources: ["replicasets"]
+  verbs: ["get", "list", "watch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: datalayer-collector
+subjects:
+- kind: ServiceAccount
+  name: datalayer-collector-collector
+  namespace: {{ default .Values.observer.namespace .Release.Namespace }}
+roleRef:
+  kind: ClusterRole
+  name: datalayer-collector
+  apiGroup: rbac.authorization.k8s.io
+{{- end }}

charts/datalayer-observer/templates/collector.yaml

@@ -16,6 +16,8 @@ spec:
             endpoint: 0.0.0.0:4317
           http:
             endpoint: 0.0.0.0:4318
+
+    # https://github.com/open-telemetry/opentelemetry-collector/tree/main/processor#recommended-processors
     processors:
       # https://github.com/open-telemetry/opentelemetry-collector/tree/main/processor/batchprocessor
       batch: {}
@@ -24,6 +26,8 @@ spec:
         check_interval: 1s
         limit_mib: 4000
         spike_limit_mib: 800
+      # https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/k8sattributesprocessor
+      k8sattributes: {}
 
     exporters:
       # https://github.com/open-telemetry/opentelemetry-collector/tree/main/exporter/debugexporter
@@ -54,10 +58,10 @@ spec:
       pipelines:
         traces:
           receivers: [otlp]
-          processors: [memory_limiter, batch]
+          processors: [memory_limiter, k8sattributes, batch]
           exporters: [otlp]
         metrics:
           receivers: [otlp]
-          processors: [memory_limiter, batch]
+          processors: [memory_limiter, k8sattributes, batch]
           exporters: [prometheus]
 {{- end }}

charts/datalayer-observer/values.yaml

@@ -212,6 +212,47 @@ kube-prometheus-stack:
         zookeeper:
           url: https://raw.githubusercontent.com/streamnative/apache-pulsar-grafana-dashboard/master/dashboards.kubernetes/zookeeper-3.6.json
           datasource: Prometheus
+  kubeApiServer:
+    serviceMonitor:
+      additionalLabels:
+        monitoring.datalayer.io/instance: "observer"
+        monitoring.datalayer.io/enabled: "true"
+  kubelet:
+    serviceMonitor:
+      additionalLabels:
+        monitoring.datalayer.io/instance: "observer"
+        monitoring.datalayer.io/enabled: "true"
+  kubeControllerManager:
+    serviceMonitor:
+      additionalLabels:
+        monitoring.datalayer.io/instance: "observer"
+        monitoring.datalayer.io/enabled: "true"
+  coreDns:
+    serviceMonitor:
+      additionalLabels:
+        monitoring.datalayer.io/instance: "observer"
+        monitoring.datalayer.io/enabled: "true"
+  kubeEtcd:
+    serviceMonitor:
+      additionalLabels:
+        monitoring.datalayer.io/instance: "observer"
+        monitoring.datalayer.io/enabled: "true"
+  kubeScheduler:
+    serviceMonitor:
+      additionalLabels:
+        monitoring.datalayer.io/instance: "observer"
+        monitoring.datalayer.io/enabled: "true"
+  kubeProxy:
+    serviceMonitor:
+      additionalLabels:
+        monitoring.datalayer.io/instance: "observer"
+        monitoring.datalayer.io/enabled: "true"
+  kube-state-metrics:
+    prometheus:
+      monitor:
+        additionalLabels:
+          monitoring.datalayer.io/instance: "observer"
+          monitoring.datalayer.io/enabled: "true"
   prometheus:
     enabled: true
     prometheusSpec: