From 3dcb038feb4f1b1104022bbab136bdb310f480f0 Mon Sep 17 00:00:00 2001 From: Matej Gera Date: Mon, 7 Aug 2023 18:41:05 +0200 Subject: [PATCH 1/4] Limit KSM scraping to only chart instance Signed-off-by: Matej Gera --- charts/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/values.yaml b/charts/values.yaml index 2771330..b05406f 100644 --- a/charts/values.yaml +++ b/charts/values.yaml @@ -71,7 +71,7 @@ opentelemetry-collector-agent: watch_observers: [k8s_observer] receivers: prometheus_simple: - rule: type == "port" && port == 8080 && pod.name contains "kube-state-metrics" + rule: type == "port" && port == 8080 && pod.name contains "{{ .Release.Name }}-kube-state-metrics" config: endpoint: '`endpoint`' prometheus/kube_extra_metrics: From 018ba66a0506d9563981b94a08604ffff349fbee Mon Sep 17 00:00:00 2001 From: Matej Gera Date: Mon, 7 Aug 2023 18:41:50 +0200 Subject: [PATCH 2/4] Replace k8s node name with target node name for node info metric Signed-off-by: Matej Gera --- charts/values.yaml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/charts/values.yaml b/charts/values.yaml index b05406f..49724e3 100644 --- a/charts/values.yaml +++ b/charts/values.yaml @@ -132,7 +132,7 @@ opentelemetry-collector-agent: enabled: true metricstransform: transforms: - include: .* + - include: .* match_type: regexp action: update operations: @@ -142,6 +142,14 @@ opentelemetry-collector-agent: - action: add_label new_label: cx.otel_integration.name new_value: "{{ .Chart.Name }}" + # Replace node name for kube node info with the name of the target node. + - include: kube_node_info + match_type: strict + action: update + operations: + - action: update_label + label: node + new_label: k8s.node.name filter/metrics: metrics: include: From c51842aedffe74494b9090c62283107a0533c9f0 Mon Sep 17 00:00:00 2001 From: Matej Gera Date: Mon, 7 Aug 2023 18:43:46 +0200 Subject: [PATCH 3/4] Move extra kube metrics to events collector Signed-off-by: Matej Gera --- charts/values.yaml | 116 ++++++++++++++++++++++----------------------- 1 file changed, 56 insertions(+), 60 deletions(-) diff --git a/charts/values.yaml b/charts/values.yaml index 49724e3..17ede14 100644 --- a/charts/values.yaml +++ b/charts/values.yaml @@ -39,25 +39,6 @@ opentelemetry-collector-agent: # Best used with mode = daemonset. kubeletMetrics: enabled: true - - # Extra role rules are required in order to fetch kubelet and kuberenetes API metrics. - clusterRole: - rules: - - apiGroups: - - "" - resources: - - nodes - - nodes/metrics - - endpoints - - services - verbs: - - get - - list - - watch - - nonResourceURLs: - - "/metrics" - verbs: - - get config: extensions: @@ -74,40 +55,6 @@ opentelemetry-collector-agent: rule: type == "port" && port == 8080 && pod.name contains "{{ .Release.Name }}-kube-state-metrics" config: endpoint: '`endpoint`' - prometheus/kube_extra_metrics: - config: - scrape_configs: - - job_name: kubernetes-apiserver - honor_timestamps: true - scheme: https - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token - tls_config: - ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt - insecure_skip_verify: true - kubernetes_sd_configs: - - role: endpoints - relabel_configs: - - source_labels: - [ - __meta_kubernetes_namespace, - __meta_kubernetes_service_name, - __meta_kubernetes_endpoint_port_name, - ] - action: keep - regex: default;kubernetes;https - - job_name: kubernetes-cadvisor - honor_timestamps: true - metrics_path: /metrics/cadvisor - scheme: https - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token - tls_config: - ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt - insecure_skip_verify: true - kubernetes_sd_configs: - - role: node - relabel_configs: - - action: labelmap - regex: __meta_kubernetes_node_label_(.+) processors: resourcedetection/env: @@ -172,12 +119,6 @@ opentelemetry-collector-agent: - kube_node_info - kube_pod_status_reason - kube_pod_status_qos_class - - kubernetes_build_info - - container_fs_writes_total - - container_fs_writes_bytes_total - - container_fs_reads_total - - container_fs_reads_bytes_total - - container_fs_usage_bytes exporters: coralogix: @@ -225,7 +166,6 @@ opentelemetry-collector-agent: - prometheus - hostmetrics - receiver_creator/ksm_prometheus - - prometheus/kube_extra_metrics opentelemetry-collector-events: enabled: true @@ -260,9 +200,58 @@ opentelemetry-collector-events: - apiGroups: ["", "events.k8s.io"] resources: ["events"] verbs: ["watch", "list"] + - apiGroups: + - "" + resources: + - nodes + - nodes/metrics + - endpoints + - services + verbs: + - get + - list + - watch + - nonResourceURLs: + - "/metrics" + verbs: + - get config: receivers: + prometheus/kube_extra_metrics: + config: + scrape_configs: + - job_name: kubernetes-apiserver + honor_timestamps: true + scheme: https + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecure_skip_verify: true + kubernetes_sd_configs: + - role: endpoints + relabel_configs: + - source_labels: + [ + __meta_kubernetes_namespace, + __meta_kubernetes_service_name, + __meta_kubernetes_endpoint_port_name, + ] + action: keep + regex: default;kubernetes;https + - job_name: kubernetes-cadvisor + honor_timestamps: true + metrics_path: /metrics/cadvisor + scheme: https + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecure_skip_verify: true + kubernetes_sd_configs: + - role: node + relabel_configs: + - action: labelmap + regex: __meta_kubernetes_node_label_(.+) k8sobjects: objects: - group: events.k8s.io @@ -325,6 +314,12 @@ opentelemetry-collector-events: - k8s.container.cpu_request - k8s.container.memory_limit - k8s.container.memory_request + - kubernetes_build_info + - container_fs_writes_total + - container_fs_writes_bytes_total + - container_fs_reads_total + - container_fs_reads_bytes_total + - container_fs_usage_bytes exporters: coralogix: @@ -367,6 +362,7 @@ opentelemetry-collector-events: receivers: - otlp - prometheus + - prometheus/kube_extra_metrics - k8s_cluster logs/kube-events: exporters: From 886de75ed1919c957c0c5b38d93e1c8a95f291bd Mon Sep 17 00:00:00 2001 From: Matej Gera Date: Tue, 8 Aug 2023 09:10:54 +0200 Subject: [PATCH 4/4] Bump chart and add CHANGELOG Signed-off-by: Matej Gera --- CHANGELOG.md | 5 +++++ charts/Chart.yaml | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b6e14c..c3c4ff4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ ## Coralogix Opentelemetry Integration +### v0.9.0 / 2023-08-08 +* [FIX] Limit kube-state-metrics scraping to chart's instance only +* [FIX] Move extra Kubernetes metrics to collector instead of agent +* [FEATURE] Make `k8s.node.name` label the target node for Kubernetes node info metric + ### v0.8.0 / 2023-08-03 * [FEATURE] Add cluster metrics related to allocatable resources (CPU, memory) diff --git a/charts/Chart.yaml b/charts/Chart.yaml index 5ed7d5f..4228009 100644 --- a/charts/Chart.yaml +++ b/charts/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: coralogix-opentelemetry-integration description: OpenTelemetry barebones to have Coralogix Kubernetes Monitoring working out-of-the-box. -version: 0.8.0 +version: 0.9.0 kubeVersion: ">=1.16.0-0" keywords: - OpenTelemetry Collector