From 6c7ddadabab1f696b173a4cdb6da30d3ac7ec8e6 Mon Sep 17 00:00:00 2001 From: Rafael Franzke Date: Thu, 23 May 2024 16:07:06 +0200 Subject: [PATCH] [GEP-19] Adapt monitoring configuration (#307) * Adapt Prometheus scrape configs and rules and Plutono dashboards * Address PR review feedback --- .github/renovate.json5 | 2 +- .../templates/rbac.yaml | 4 ++ .../cilium-agent-metrics-dashboard.json | 4 +- .../cilium-operator-metrics-dashboard.json | 4 +- .../hubble-metrics-dashboard.json | 4 +- .../templates/cilium-monitoring-config.yaml | 2 + .../templates/configmap-dashboards.yaml | 18 +++++ .../templates/prometheusrule-agent.yaml | 17 +++++ .../templates/scrapeconfig-agent.yaml | 67 +++++++++++++++++++ .../templates/scrapeconfig-hubble.yaml | 67 +++++++++++++++++++ .../templates/scrapeconfig-operator.yaml | 67 +++++++++++++++++++ charts/internal/cilium-monitoring/values.yaml | 3 + .../app/app.go | 9 +++ example/controller-registration.yaml | 2 +- go.mod | 2 +- pkg/controller/actuator_reconcile.go | 35 +++++++++- 16 files changed, 297 insertions(+), 10 deletions(-) create mode 100644 charts/internal/cilium-monitoring/templates/configmap-dashboards.yaml create mode 100644 charts/internal/cilium-monitoring/templates/prometheusrule-agent.yaml create mode 100644 charts/internal/cilium-monitoring/templates/scrapeconfig-agent.yaml create mode 100644 charts/internal/cilium-monitoring/templates/scrapeconfig-hubble.yaml create mode 100644 charts/internal/cilium-monitoring/templates/scrapeconfig-operator.yaml diff --git a/.github/renovate.json5 b/.github/renovate.json5 index bf486a6ec..39092f0ad 100644 --- a/.github/renovate.json5 +++ b/.github/renovate.json5 @@ -17,7 +17,7 @@ "separateMinorPatch": true, "packageRules": [ { - // Group calico image updates in one PR. + // Group cilium image updates in one PR. "groupName": "cilium images", "matchDatasources": ["docker"], "matchPackagePatterns": ["quay\\.io\/cilium\/.+"], diff --git a/charts/gardener-extension-networking-cilium/templates/rbac.yaml b/charts/gardener-extension-networking-cilium/templates/rbac.yaml index 491f849a4..bd77c9074 100644 --- a/charts/gardener-extension-networking-cilium/templates/rbac.yaml +++ b/charts/gardener-extension-networking-cilium/templates/rbac.yaml @@ -65,6 +65,7 @@ rules: - admissionregistration.k8s.io - apiextensions.k8s.io - networking.k8s.io + - monitoring.coreos.com resources: - namespaces - events @@ -85,6 +86,9 @@ rules: - mutatingwebhookconfigurations - customresourcedefinitions - networkpolicies + - scrapeconfigs + - prometheusrules + - statefulsets # TODO(rfranzke): Remove this after August 2024. verbs: - "*" --- diff --git a/charts/internal/cilium-monitoring/cilium-agent-metrics-dashboard.json b/charts/internal/cilium-monitoring/cilium-agent-metrics-dashboard.json index 466b423f7..257531f67 100644 --- a/charts/internal/cilium-monitoring/cilium-agent-metrics-dashboard.json +++ b/charts/internal/cilium-monitoring/cilium-agent-metrics-dashboard.json @@ -3,7 +3,7 @@ "list": [ { "builtIn": 1, - "datasource": "-- Grafana --", + "datasource": "-- Plutono --", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", @@ -8327,4 +8327,4 @@ "title": "Cilium v1.12 Agent Metrics", "uid": "dtas", "version": 3 -} \ No newline at end of file +} diff --git a/charts/internal/cilium-monitoring/cilium-operator-metrics-dashboard.json b/charts/internal/cilium-monitoring/cilium-operator-metrics-dashboard.json index b86e3d93d..298213834 100644 --- a/charts/internal/cilium-monitoring/cilium-operator-metrics-dashboard.json +++ b/charts/internal/cilium-monitoring/cilium-operator-metrics-dashboard.json @@ -3,7 +3,7 @@ "list": [ { "builtIn": 1, - "datasource": "-- Grafana --", + "datasource": "-- Plutono --", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", @@ -1005,4 +1005,4 @@ "title": "Cilium v1.12 Operator Metrics", "uid": "fsafdsf", "version": 1 -} \ No newline at end of file +} diff --git a/charts/internal/cilium-monitoring/hubble-metrics-dashboard.json b/charts/internal/cilium-monitoring/hubble-metrics-dashboard.json index e415bb96b..1827bee4f 100644 --- a/charts/internal/cilium-monitoring/hubble-metrics-dashboard.json +++ b/charts/internal/cilium-monitoring/hubble-metrics-dashboard.json @@ -3,7 +3,7 @@ "list": [ { "builtIn": 1, - "datasource": "-- Grafana --", + "datasource": "-- Plutono --", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", @@ -3568,4 +3568,4 @@ "title": "Cilium v1.12 Hubble Metrics", "uid": "seafadsfdsa", "version": 1 -} \ No newline at end of file +} diff --git a/charts/internal/cilium-monitoring/templates/cilium-monitoring-config.yaml b/charts/internal/cilium-monitoring/templates/cilium-monitoring-config.yaml index 758502af6..6d5b002d8 100644 --- a/charts/internal/cilium-monitoring/templates/cilium-monitoring-config.yaml +++ b/charts/internal/cilium-monitoring/templates/cilium-monitoring-config.yaml @@ -1,3 +1,4 @@ +{{- if not .Values.gep19Monitoring }} apiVersion: v1 kind: ConfigMap metadata: @@ -125,3 +126,4 @@ data: expr: histogram_quantile(0.99,sum by(verb, method, path, le) (rate(cilium_agent_api_process_time_seconds_bucket[10m]))) labels: quantile: "0.99" +{{- end }} diff --git a/charts/internal/cilium-monitoring/templates/configmap-dashboards.yaml b/charts/internal/cilium-monitoring/templates/configmap-dashboards.yaml new file mode 100644 index 000000000..70deec303 --- /dev/null +++ b/charts/internal/cilium-monitoring/templates/configmap-dashboards.yaml @@ -0,0 +1,18 @@ +{{- if .Values.gep19Monitoring }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: cilium-dashboards + namespace: {{ .Release.Namespace }} + labels: + dashboard.monitoring.gardener.cloud/shoot: "true" +data: + cilium-agent-metrics-dashboard.json: |- + {{- .Files.Get "cilium-agent-metrics-dashboard.json" | nindent 4 }} + + cilium-operator-metrics--dashboard.json: |- + {{- .Files.Get "cilium-operator-metrics-dashboard.json" | nindent 4 }} + + hubble-dashboard-metrics-dashboard.json: |- + {{- .Files.Get "hubble-metrics-dashboard.json" | nindent 4 }} +{{- end }} diff --git a/charts/internal/cilium-monitoring/templates/prometheusrule-agent.yaml b/charts/internal/cilium-monitoring/templates/prometheusrule-agent.yaml new file mode 100644 index 000000000..61492187f --- /dev/null +++ b/charts/internal/cilium-monitoring/templates/prometheusrule-agent.yaml @@ -0,0 +1,17 @@ +{{- if .Values.gep19Monitoring }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: shoot-cilium-agent + namespace: {{ .Release.Namespace }} + labels: + prometheus: shoot +spec: + groups: + - name: recording-rules.rules + rules: + - record: cilium:api_latency + expr: histogram_quantile(0.99,sum by(verb, method, path, le) (rate(cilium_agent_api_process_time_seconds_bucket[10m]))) + labels: + quantile: "0.99" +{{- end }} diff --git a/charts/internal/cilium-monitoring/templates/scrapeconfig-agent.yaml b/charts/internal/cilium-monitoring/templates/scrapeconfig-agent.yaml new file mode 100644 index 000000000..dd440ef44 --- /dev/null +++ b/charts/internal/cilium-monitoring/templates/scrapeconfig-agent.yaml @@ -0,0 +1,67 @@ +{{- if .Values.gep19Monitoring }} +apiVersion: monitoring.coreos.com/v1alpha1 +kind: ScrapeConfig +metadata: + name: shoot-cilium-agent + namespace: {{ .Release.Namespace }} + labels: + prometheus: shoot +spec: + authorization: + credentials: + name: shoot-access-prometheus-shoot + key: token + scheme: HTTPS + tlsConfig: + # This is needed because the kubelets' certificates are not are generated + # for a specific pod IP + insecureSkipVerify: true + kubernetesSDConfigs: + - apiServer: https://kube-apiserver + authorization: + credentials: + name: shoot-access-prometheus-shoot + key: token + followRedirects: true + namespaces: + names: + - kube-system + role: endpoints + tlsConfig: + # This is needed because we do not fetch the correct cluster CA bundle right now + insecureSkipVerify: true + metricRelabelings: + - sourceLabels: + - __name__ + action: keep + regex: ^(cilium_process_virtual_memory_bytes|cilium_process_resident_memory_bytes|cilium_process_open_fds|cilium_bpf_maps_virtual_memory_max_bytes|cilium_bpf_progs_virtual_memory_max_bytes|cilium_agent_api_process_time_seconds_count|cilium_agent_api_process_time_seconds_sum|cilium_agent_api_process_time_seconds_bucket|cilium_bpf_syscall_duration_seconds_count|cilium_bpf_syscall_duration_seconds_sum|cilium_bpf_map_ops_total|kvstore_operations_total|cilium_kvstore_operations_duration_seconds_sum|cilium_kvstore_events_queue_seconds_count|cilium_forward_count_total|cilium_forward_bytes_total|cilium_datapath_conntrack_gc_entries|cilium_ip_addresses|cilium_datapath_errors_total|cilium_services_events_total|cilium_unreachable_health_endpoints|cilium_unreachable_nodes|cilium_unreachable_health_endpoints|cilium_drop_count_total|cilium_nodes_all_events_received_total|cilium_drop_bytes_total|cilium_nodes_all_num|cilium_policy_l7_denied_total|cilium_policy_l7_forwarded_total|cilium_policy_l7_received_total|cilium_policy_l7_parse_errors_total|cilium_proxy_upstream_reply_seconds_sum|cilium_proxy_upstream_reply_seconds_count|cilium_triggers_policy_update_call_duration_seconds_sum|cilium_policy_endpoint_enforcement_status|cilium_proxy_redirects|cilium_triggers_policy_update_total|cilium_policy_count|cilium_policy_import_errors|cilium_policy_max_revision|cilium_endfpoint_regeneration_time_stats_seconds_bucket|cilium_endpoint_regenerations|cilium_endpoint_state|cilium_controllers_runs_total|cilium_controllers_failing|cilium_controllers_runs_duration_seconds_sum|cilium_controllers_runs_duration_seconds_count|cilium_k8s_client_api_latency_time_seconds_sum|cilium_k8s_client_api_latency_time_seconds_count|cilium_k8s_client_api_calls_counter|cilium_kubernetes_events_received_total|cilium_kubernetes_events_total|cilium_process_cpu_seconds_total|cilium_errors_warnings_total|cilium_endpoint_regeneration_time_stats_seconds_bucket)$ + - sourceLabels: + - namespace + action: keep + regex: kube-system + relabelings: + - action: replace + replacement: cilium-agent-metrics + targetLabel: job + - sourceLabels: [__meta_kubernetes_pod_label_k8s_app,__meta_kubernetes_service_annotation_prometheus_io_scrape] + separator: ; + regex: cilium;true + replacement: $1 + action: keep + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - sourceLabels: [__meta_kubernetes_pod_name] + separator: ; + regex: (.*) + targetLabel: pod + replacement: $1 + action: replace + - targetLabel: __address__ + replacement: kube-apiserver:443 + - sourceLabels: [__meta_kubernetes_pod_name] + separator: ; + regex: (.+);(.+) + targetLabel: __metrics_path__ + replacement: /api/v1/namespaces/kube-system/pods/${1}:9090/proxy/metrics + action: replace +{{- end }} diff --git a/charts/internal/cilium-monitoring/templates/scrapeconfig-hubble.yaml b/charts/internal/cilium-monitoring/templates/scrapeconfig-hubble.yaml new file mode 100644 index 000000000..1e7c9b8e0 --- /dev/null +++ b/charts/internal/cilium-monitoring/templates/scrapeconfig-hubble.yaml @@ -0,0 +1,67 @@ +{{- if .Values.gep19Monitoring }} +apiVersion: monitoring.coreos.com/v1alpha1 +kind: ScrapeConfig +metadata: + name: shoot-cilium-hubble + namespace: {{ .Release.Namespace }} + labels: + prometheus: shoot +spec: + authorization: + credentials: + name: shoot-access-prometheus-shoot + key: token + scheme: HTTPS + tlsConfig: + # This is needed because the kubelets' certificates are not are generated + # for a specific pod IP + insecureSkipVerify: true + kubernetesSDConfigs: + - apiServer: https://kube-apiserver + authorization: + credentials: + name: shoot-access-prometheus-shoot + key: token + followRedirects: true + namespaces: + names: + - kube-system + role: endpoints + tlsConfig: + # This is needed because we do not fetch the correct cluster CA bundle right now + insecureSkipVerify: true + metricRelabelings: + - sourceLabels: + - __name__ + action: keep + regex: ^(hubble_flows_processed_total|hubble_drop_total|hubble_port_distribution_total|hubble_tcp_flags_total|hubble_icmp_total)$ + - sourceLabels: + - namespace + action: keep + regex: kube-system + relabelings: + - action: replace + replacement: hubble-metrics + targetLabel: job + - sourceLabels: [__meta_kubernetes_pod_label_k8s_app,__meta_kubernetes_service_annotation_prometheus_io_scrape] + separator: ; + regex: cilium;true + replacement: $1 + action: keep + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - sourceLabels: [__meta_kubernetes_pod_name] + separator: ; + regex: (.*) + targetLabel: pod + replacement: $1 + action: replace + - targetLabel: __address__ + replacement: kube-apiserver:443 + - sourceLabels: [__meta_kubernetes_pod_name] + separator: ; + regex: (.+);(.+) + targetLabel: __metrics_path__ + replacement: /api/v1/namespaces/kube-system/pods/${1}:9091/proxy/metrics + action: replace +{{- end }} diff --git a/charts/internal/cilium-monitoring/templates/scrapeconfig-operator.yaml b/charts/internal/cilium-monitoring/templates/scrapeconfig-operator.yaml new file mode 100644 index 000000000..f9487d4f3 --- /dev/null +++ b/charts/internal/cilium-monitoring/templates/scrapeconfig-operator.yaml @@ -0,0 +1,67 @@ +{{- if .Values.gep19Monitoring }} +apiVersion: monitoring.coreos.com/v1alpha1 +kind: ScrapeConfig +metadata: + name: shoot-cilium-operator + namespace: {{ .Release.Namespace }} + labels: + prometheus: shoot +spec: + authorization: + credentials: + name: shoot-access-prometheus-shoot + key: token + scheme: HTTPS + tlsConfig: + # This is needed because the kubelets' certificates are not are generated + # for a specific pod IP + insecureSkipVerify: true + kubernetesSDConfigs: + - apiServer: https://kube-apiserver + authorization: + credentials: + name: shoot-access-prometheus-shoot + key: token + followRedirects: true + namespaces: + names: + - kube-system + role: endpoints + tlsConfig: + # This is needed because we do not fetch the correct cluster CA bundle right now + insecureSkipVerify: true + metricRelabelings: + - sourceLabels: + - __name__ + action: keep + regex: ^(cilium_operator_process_cpu_seconds_total|cilium_operator_process_resident_memory_bytes)$ + - sourceLabels: + - namespace + action: keep + regex: kube-system + relabelings: + - action: replace + replacement: cilium-operator-metrics + targetLabel: job + - sourceLabels: [__meta_kubernetes_pod_label_io_cilium_app,__meta_kubernetes_pod_annotation_prometheus_io_scrape] + separator: ; + regex: operator;true + replacement: $1 + action: keep + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - sourceLabels: [__meta_kubernetes_pod_name] + separator: ; + regex: (.*) + targetLabel: pod + replacement: $1 + action: replace + - targetLabel: __address__ + replacement: kube-apiserver:443 + - sourceLabels: [__meta_kubernetes_pod_name,__meta_kubernetes_pod_annotation_prometheus_io_port] + separator: ; + regex: (.+);(.+) + targetLabel: __metrics_path__ + replacement: /api/v1/namespaces/kube-system/pods/${1}:${2}/proxy/metrics + action: replace +{{- end }} diff --git a/charts/internal/cilium-monitoring/values.yaml b/charts/internal/cilium-monitoring/values.yaml index 5fe0ad832..319fbc971 100644 --- a/charts/internal/cilium-monitoring/values.yaml +++ b/charts/internal/cilium-monitoring/values.yaml @@ -68,3 +68,6 @@ allowedMetrics: ciliumOperator: - cilium_operator_process_cpu_seconds_total - cilium_operator_process_resident_memory_bytes + +# TODO(rfranzke): Remove this field after August 2024. +gep19Monitoring: false diff --git a/cmd/gardener-extension-networking-cilium/app/app.go b/cmd/gardener-extension-networking-cilium/app/app.go index 470e2532b..15a414771 100644 --- a/cmd/gardener-extension-networking-cilium/app/app.go +++ b/cmd/gardener-extension-networking-cilium/app/app.go @@ -18,6 +18,8 @@ import ( v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" "github.com/gardener/gardener/pkg/logger" "github.com/pkg/errors" + monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + monitoringv1alpha1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1alpha1" "github.com/spf13/cobra" corev1 "k8s.io/api/core/v1" "k8s.io/component-base/version" @@ -140,6 +142,13 @@ func NewControllerManagerCommand(ctx context.Context) *cobra.Command { return fmt.Errorf("could not update manager scheme: %w", err) } + if err := monitoringv1.AddToScheme(mgr.GetScheme()); err != nil { + return fmt.Errorf("could not update manager scheme: %w", err) + } + if err := monitoringv1alpha1.AddToScheme(mgr.GetScheme()); err != nil { + return fmt.Errorf("could not update manager scheme: %w", err) + } + reconcileOpts.Completed().Apply(&ciliumcontroller.DefaultAddOptions.IgnoreOperationAnnotation) ciliumCtrlOpts.Completed().Apply(&ciliumcontroller.DefaultAddOptions.Controller) configFileOpts.Completed().ApplyHealthCheckConfig(&healthcheck.AddOptions.HealthCheckConfig) diff --git a/example/controller-registration.yaml b/example/controller-registration.yaml index c88cffe00..f0102f466 100644 --- a/example/controller-registration.yaml +++ b/example/controller-registration.yaml @@ -5,7 +5,7 @@ metadata: name: networking-cilium type: helm providerConfig: - chart: H4sIAAAAAAAAA+0ca3PbNjKf+StwSjt9TEiKkiznNJObc2039TSxNbbPdzc3Nx6IhCXUJMESpBw1zf32Wzz4FGWJTuLWLXcyMQViFwtgsS8AnOPYIyGJTfIuISGnLDRDktyx+JaGc9OlPk0D+9nHQR9gf29P/gWo/5XPznDkDPYG47Eod/b7A+cZ2vvIdneClCc4RuhZzFhyX71t758ozHeZf2tB/IDOQxaTh7QhJng8Gm2cf5j26vwPHGdv/Az1P3Vnm+BPPv/P0RQnCYlDjhKG1ByjuwUJ0SylvgdCgCLs3uI54ZbxHF0uKEc8jSIWJ/AAcuGjuc9mKMCJu4DaL1BMfJzQJQG8ZFEqx6EHBEIyh7csRF9HMbmh74iH7ijU+8s3FjoL/RViocQULKGIxMinIbEM6+ji+iIB3oDEIQsCIHB1eIE8GnPDmtPElv8r9g1r9ktsy/+zgsXcFv9lP/kytAtCM+hfGqEb6hNufGvxuwj+n+Fb+D8J4Pl/UPUKx5SlHJ0cHUODUcx+Im5iWNQj2Fb1oMiwltxlHrGN33pWd4ed1v/hAseJtcKB/6A2tq3/wXC/tv7BXPS79f8YgCN6RWIx7xO0dAwcRfnPnmP1e4ZHuBvTKJFFB+gHsATIFfKAbliMkgVBr7UIoUMpLehUyQ86ziTKCHFAJmgXUTOWWet9C5p/QgvpicJO699jrjVnD25j2/ofg29YW//OYNit/8cA20YX06N/md+D9Ttk0QpM5iK5BGGYoEF/MEIXB1N0cYxgqeNQ/sA3YCgpTghyWRDhcCUMe6EDXBYmMZ2lYKu5YdtGRv8NdUG8iHkC1RJ6Q0kM2gQ8iwUxB7DOod6cTeaChCDNF8h0UW+G4eGL1wfnR8enx+fXPxwc/nh9dHJuZ/VM2RrzfZDfmMwpT2LpXFiAtibFyEJffO3iBFmWDf+ujs8vTs5Ov9E/yTscRD6xN1EUxi/TbBNFsCe4BvdJekdaJ5IQz8CNQJXOKHdJ6ktdKNwqoTpdFsfgSKCiVVRp1YjK1D+HNtxp/ScExga6wh8WCbaO/wYOhIBd/PcY0HL+r8HjB6+cW0m0uy+4Rf87zsipzf9wOOziv0eB9+9N5EEgBlFXT7hpPWR++GDs5KoJVAK6XyAYZTo+nhGfW+BLWrdkpSjKH+kMojoCcmRRZovWKjQ2kFhiP9VsvX+PaOj6qZczayGNeA8j67h1BgWVCdpQQ7cvW1rvBQ1BfkKXSHTrnPgEc2KdAnONnOWs0QDUuuIMIfGG3qAF5lMZFKMeX+DB3ngCzV6J5qEpUd9K8BzlGFFMw+QG9b7kf/+S12vGJGKcghVe3UcC+kiaCE4eTBA6W+o3PP7WAt7BvdBS/4OjckPnAY5MOflLcF5YbDKI2u5impDmHME2/380Hlb1/xAigC7+fxTQqqeypK/krJ5lk6oUXyVNAMLhTdChFIa3ODICkmAPJ3gCakAF+82qullqNBIHZ7dBj8pipWGUVp406HJB/lcoFNEFGonaGTuyRX5dFdEJ+lUQubfXVXJ/VI320PXfJhu4zf8bO2v+33i0163/x4BPtbBzwfisi1m1ki9hBGCapvxb7ogSW6sQZCuXbm5pGpngW67PUs9eOtiPFtiRtPJR0EG5Go9UBeVGTWVqeq5PgV2oGYIeERscspPAcq18IkuBX9clkSgHxpLLVUS4HK2Y/JzSmHiot4W+tU4AUZ7j97bx14SvWZbjnJW25KqE2Y6dMmLOx89R21EBjHbtCoS8vVka86RlixKnXZsK5XdhVVrqf49EPlsF0J0WBmCL/t8HA1DT/6P+fuf/PQqU1SZEt9zOjcBRPtUNVmCnBEFbS6DUeFOeYOcGERJHFSy+sGXGtAVe29BetkXnCxMvMYUeAKVkpc0gBMucpbELdGpmJpEalZMY3EyDR8QVvY7JkgrOfqBcxNdvaECB8758E/nUxUoJZlpFFx6yNEwUJxx4E86rGkK55f6mNKafYFQfMj6Z1tBclWRI0gtDlkiDmnOZJ0K2RSNZdXdB3FsOCqoISzX7zYFGxXv5WqY70BfWpebT+g74n4pDC72d4tzeN9JPUaka4KLMWUmxb+gbjEdMXW6pHYMLN4aVGM7LCFHMoNKCpKUJ6zUMdW8DBhc0BU4Sp2RTJbEzocnWGJN7FmXqa33yKzL2SaTsYetQQMnZq685ZsawVGhATBhjtfZA2fk+uyPebvgeCOl9GJvXOzMBzeQLsB7cFP0qeDATNzJHo2GVcqYU1FRRBuK2OvQx56fVseQrDgvM/Gu/rysLotQlB64rFMNp25EX/hemUDefULOdrlcgl0s1StCpTqs8W7Jsmvr+lIEuW1X0m0orRvnLyopnQYDBOuUFJrJb8WdCwBDgd0JRu2kcg3UzYyJ+iNM/r0pcFFtyVlH3YhW6vMyPILcg2E8WUhe1Jl3C3aGZOJkRnJi5VX21yaiiDZhAnNyZoPZAVjCIpWDPu4c3hWZJtBONdaGQ1lpRh6tMFhEVJ5mFht/QgMI4yxAO8vr1rittbIoTWq9skrh2cxCqtbZdihIrZO7IDNbhbUZu+zDeh62XmynUZLl/upoKGS1da6p06TZ6IMPbyZG4iVq+CPQBmjKNXCfpd2VcEi7Li0mt+DfHB0fH59fHb44PL0/OTq9PD94eX0wPDo/zmgjJbZHvwZZMSoUI3VDie+fkplqqy4VxneSegJVPwEPtf8bvyduD18dXwOzZ+fXZ1fH5P89PLtd4nSDlE5ZygXZjcvA+Iy7PPa4PWDaX2rErWs616lTa2ZYTK411wlzmT9Dl4bSeecitThknL5SZlYT9WxxcWMf4FYU6weL0a5FoPmrMTwPyVliThi4rsS2xGoiKaoa3r9GPnfFNeeQmZtZmvVQvJtgTx04nSDhJm2dejcWaaax1xs3yZmXp3zltloFHbnDqJ2+ZB3ijQWbfW43UbuPUnt9t434P73/E7PnTh5b5n4h5HuVxKs+DzlJvTnZIBG3L//fH9fO/8H+X/38UKOd/IuluFxmgKfOO8sn+Tk72k0oFtQ0ls9ALnPh/hDqt40Nd53eaYtEWgZNgKfYoggjHoK3/9go51mBs9kFlH+JIZaYo0PsRSOu5tq7KnmAaqjBkBRN+DO6qmO4sLjvw7/CKH4gA9XeQre7gU0NL/R/PsNv6IsgW/T/cXzv/PRgPRp3+fwwQ26dlGyAnGKfJgsX0F3UI+fal1EXF7rAPY0bic+aTh5uDJ5nzj1NfxACm2Gx+HbM0kh0ApVjsLlfzf0YlHhNVXTV4HH6A0p7pUmFZxV+fcvVwJwzMRzWke8wrP2zoXZLu1rh4ivKnNIJpJuscuYzFHg3LkrLOixzFWqsuhF4ydFANo6Jl9LGtZOXSt6iMRjH/JtT2hIj4ak9WjcS64ORJsOYx2zQsmxLC61wHOISQystLq+3kc1IaLg941o/3zFTG5RprvZ78Izb75MMsx9y49hWCF1AuxqRyQ6FcIaIl+Sy9KCXKN81d7h0qniDEDBP1yAl0XT/n0bKuFXoRo1nFYmt67bfNXewTTU4m5Hj5B1Ypc15eoDHzyVrBDPQf9EKVFzXWXv3EZuoBwqXiwfbZXE15msgrIDoF5JZPl+g2oUkWZGMkzwzT4q0eT+mu07rE9L7tGR+n1L9TXfmz6Xbouc5bZgvmnoEz8kNCJWPYaph4OhN3iKU5UaQuKhs4nyO+arD/Lf0/vWLauYDb/L/x/nDt/sde5/89CjSe/9OSuPPpv53D+7VN/132UG9iFphQy/fMhJkqA4+++s/7XpYc7016l4fT3oueeNeb7JZk//Dfr9pxIPdxCfFMta9ugtiAfeWm3rytMFbno7an/qLO+oOYyTYd7mPksUYon34zS5RoDkqZEmi3V9XIlZ0g4FER7cEArSHW2hPaGhDkvnqvFadgiU1poHJOi41AGEIqDtsUboyRZa8rqXe5b3le333Z2KSqXzmIUdtgqOX2c3whb5ZIS8FTIlwGbiUM7D6brw7uMPDA0kQeb5WHSOrkGhND+y0SQ/mhgpolzXgwA5nn74GFZKV+5ZeKtuJj0QlzIXy4dTLV4Sn9fI4uz47OJupjJHIuhbWPGYbYAUrA8YuJC+bKk1eTQ4Z8Fs5JDOIBrpQHCkzevb1JkzQmFjonAVsSURQgzBFn0Hv4m81Bfrzg70vH2t8Tl7TQjJBQfOdEqDjPardyVZLVXBdDeQykScM8iP6n00ptzml/BlnOnHyT+2K7XcrK5iEAX5pTT5zvaV5sWYpXncPTztvJtJLg3dLRseIs3wk2UbR9b7d2BqC6rStK1Fy32iX+JPb/Yf6fjph2dAO33f/aG47q53/3B4PO/3sMuM//yyKRp7Tn81jhorBV8pBBdawu2S2BkbzBYAOfxCZJy/W/jHD770BtWf/iOyP1/P9ovN+t/8eAmrUW81sywGXtIN0zkUgr8nj20ulplQHVEgovp8w70PVI/EDNYQIXO2qPzJ43dCPzEcqHSKtlSuWUjmipfJoqNosX6iztV9+qIGOHxqyAhgfKASuO5BRlxWmbAHzPeLXhwtG9dC2F2nzvaCfMnLG1w0479A+/W+9fXlb0z43SFp3LCViAt3vPqmilw2KtR7eg1HZ01zCbR7fyQ7l8+VnJpps4onztNo4MB9pYRbUjUJZ5VaJOafUOZNjVbe7/KWEn+79Ugv/QD0Bus/+jUf38lzNwuu+/PQqo+xrSNGbfd5kgksYsIqbH3FuIrKPbueWRZXHPQn/+1I7SGaiUvNwu0mZ2kxpK8HyCpBspdFhUuvxxcnPKkinoVKHejGJrEL3/YIC2FexppyQ/p1u35M/rJvZ51QY4g5dvqdFsp6SVGhk1hP5rA4g0HbMVV88FeXV6XDWVHbiNcMrVd+mk66DZULyel4Z4TpNFOrNcFthF2FF+nPlsZgdY5Cds+SleW5K2j+SkiKsQmnZ54uZuLNyyOWNzn1wXV3oUrokDbzzSaHI2ekPxiU9VkH9507Ecx3r3tHvlrPVKpT+dgXphWZZhFFdR9LcFytdvJmjPEOFkfj8n9xYba6lDAhOkM8Hr92UmaNgXbzdeesmiRkPtB0uWGr8d0PTlAHAYxJ1QUcn+icvjDLVb/I015P16R18h05ffnWHfyD8ANjHKQ9iTebJs4ZRvqmUpvcL1a8rtlY+Z5wta97qS6ZoYeepYpcRGo6Euyq5SOP3BHjD6/HnetvzAos5pvkDEmluIZ9cqZyvJuFlcfjR0zUK5ZHcwcx2j8nov+y+779B20EEHHXTQQQcddNBBBx100EEHHXTQQQcddNBBB08I/g+lHS/FAHgAAA== + chart: H4sIAAAAAAAAA+0ca2/bRjKf+Sv2lBZNipIUJdnOCejhVNtNjSa2Yft8dzgcjBW5krYmuSyXlKOmud9+sw8+RVmik7h1y0EQU8ud2dnXvHaWcxx7JCSxSd4lJOSUhWZIkjsW39JwbrrUp2lgP/s46AMc7O3JvwD1v/LZGY6cwd5gf1+UOwf9gfMM7X1kuztByhMcI/QsZiy5r962908U5rvMv7UgfkDnIYvJQ9oQE7w/Gm2cf5j26vwPHGdv/xnqf+rONsGffP6fo3OcJCQOOUoYUnOM7hYkRNOU+h4sAhRh9xbPCbeM5+hqQTniaRSxOIEHWBc+mvtsigKcuAuo/Q2KiY8TuiSAlyxK5Tj0gEBI5vCWhehFFJMZfUc8dEeh3l9eWugs9FeIhRJTsIQiEiOfhsQyrKPLm8sEeAMShywIgMD14SXyaMwNa04TW/6v2Des6S+xLf/PChZzW/yX/eTL0C4ITaF/aYRm1Cfc+NridxH8P8W38H8SwPP/oOo1jilLOTo5OoYGo5j9RNzEsKhHsK3qQZFhLbnLPGIbv/Ws7g477f/DBY4Ta4UD/0FtbNv/g+FBbf+Duuh3+/8xAEf0msRi3sdo6Rg4ivKfPcfq9wyPcDemUSKLJugH0ATIFesBzViMkgVBr/USQodytaBTtX7QcbaijBAHZIx2WWrGMmu9b0HzT2gjPVHYaf97zLXm7MFtbNv/+2Ab1va/Mxh2+/8xwLbR5fnRv8zvQfsdsmgFKnORXMFiGKNBfzBCl5NzdHmMYKvjUP7AM1CUFCcEuSyIcLgSir2QAS4Lk5hOU9DV3LBtI6P/hrqwvIh5AtUSOqMkBmkClsWCmAPY51BvzsZzQUKQ5gtkuqg3xfDwxevJxdHx6fHFzQ+Twx9vjk4u7KyeKVtjvg/rNyZzypNYGhcWoK2tYmShL164OEGWZcO/6+OLy5Oz05f6J3mHg8gn9iaKQvllkm2sCPYE12A+SetIy0QS4imYEajSGWUuSXmpC4VZJUSny+IYDAlUtIoqrRpRmfrnkIY77f+EwNhAV/jDPMHW/t/AARew8/8eA1rO/w1Y/GCVcyuJdrcFt8h/xxk5tfkfDoed//co8P69iTxwxMDr6gkzrYfMDx+MnUw1gUpA9gsEo0zHx1PicwtsSeuWrBRF+SOdgldHYB1ZlNmitQqNDSSW2E81W+/fIxq6furlzFpII97DyDpunUFBZYw21NDty5bWe0FDWD+hSyS6dUF8gjmxToG5Rs5y1mgAYl1xhpB4Q2dogfm5dIpRjy/wYG9/DM1ei+ahKVHfSvAc5RhRTMNkhnpf8r9/yes1YxIxTkELr+4jAX0kTQTHDyYInS31Gx5/6wXewb3QUv6DoTKj8wBHppz8JRgvLDYZeG13MU1Ic4xgm/0/2h9W5f8QPIDO/38U0KKnsqWv5ayeZZOqBF8lTACLwxujQ7kY3uLICEiCPZzgMYgB5ew3i+rmVaOROBi7DXJUFisJo6TyuEGWC/K/QqHwLtBI1M7YkS3ym+oSHaNfBZF7e10l90eVaA/d/22igdvsv31nzf7bH+11+/8x4FNt7HxhfNbNrFrJtzACME1T/i13RC1bq1jIVr66uaVpZAvfcn2WevbSwX60wI6klY+CdsrVeKTKKTdqIlPTc30K7ELNEOSIOOCQnQSWa+VjWQr8ui6JRDkwllytIsLlaMXk55TGxEO9LfStdQKI8hy/t42/JnzNshznrLQlVyXMduyUEXM+fo7ajgpgtGtXIOTtTdOYJy1blDjt2lQovwut0lL+eyTy2SqA7rRQAFvk/wEogJr8H/UPOvvvUaAsNsG75XauBI7yqW7QAjsFCNpqAiXGm+IEOzeIkEhVsPjClhHTFnhtXXvZFp0vTLzEFHoAlJKVVoPgLHOWxi7QqamZREpUTmIwMw0eEVf0OiZLKjj7gXLhX7+hAQXO+/JN5FMXKyGYSRVdeMjSMFGccOBNGK9qCOWR+5vSmH6CUX3I+GRSQ3NVWkOSXhiyRCrUnMs8ELLNG8mquwvi3nIQUIVbqtlvdjQq1ssLGe5AX1hXmk/rO+D/XCQt9Hbyc3svpZ2iQjXARZmzkmDf0DcYj5i63FInBpduDDsxnJcRophBpQVJSxPWaxjq3gYMLmgKnCROyaZK4mRCk60xJs8sytTX+uRX1tgnWWUP24cCSsZefc8xM4atQgNiwhirvQfCzvfZHfF2w/dgkd6HsXm/MxPQTL4A7cFN0a+CBzNxI3M0GlYpZ0JBTRVlsNxWhz7m/LQ6lnzFYYOZf+33dWVBlLpk4rpCMJy2HXlhf2EKdfMJNdvJegVyu1S9BB3qtMqzJcvOU98/ZyDLVhX5psKKUf6ysuNZEGDQTnmBiexW/JngMAT4nRDUbhrHoN3MmIgfIvvn2xIXxZGcVdS9XIUuL/MjyC0I9pOFlEWtSZdwd2gmTqYEJ2auVb/dpFTRBkwgTu5MEHuwVjAsS8Gedw9vCs2SaCca61IhrbWikqtMFhHlJ5mFhN/QgMI4yxAmef1615U0NkWG1rc2SVy72QnVUtsueYkVMndkCvvwNiO3fRjvw9bbzRRistw/XU25jJauda5k6TZ6sIa3kyNxE7V8E+gEmjKNXCbpd2VcEi7Lm0nt+DfHk6Pji5vjN8eHVydnpzenk7fHl+eTw+O8JkLyWOR70CXjUiFCM0p874LMqqW6XCjXcW4JWPkEPFT/Z/yevJ28Pr4GZs8ubs6ujy/+eXFytcbrGCmbsBQLtBuDg/cpcZn3uD5g2Vxqw65oOZeq51LPtpxYqawT5jJ/jK4Oz+uRh1zrlHHyQhlZSdi/ReLCOsavKNQBFqdf80TzUWN+GpC3Qps0dFkt2xKrgaioZnj7Hv3YGd8UR25iZm3WS/Vigj2RdjpGwkjaPPNqLNZUY60zbhY3K6/+ncNmGXhkhlM/ecs8wBsNMv3eaqR2G6f2/G4b93t4/yNGz58+tIz/RMzzKI9TmQ86Tb052SEQtC3+39+v5//C/138/1GgHP+JpLldRIDOmXeUT/Z3crKfVCiorSuZuV5gxP8j1GEdH+o6v9MQi9YInARLcUYRRDgGaf23b5FjDfbNPojsQxypyBQFej8CaT3X1nXZEkxD5YasYMKPwVwV0535ZRP/Dq/4RDiov4NodQefGlrK/3iK3dYXQbbI/5GzV5f/g31x/6+T/58fxPFpWQfICcZpsmAx/UUlId++krKoOB32YcxIfMF88nB18CRj/nHqCx/AFIfNr2OWRrIDIBSL0+Vq/M+o+GOiqqsGj8MPENpTXSo0q/jrU64e7oSC+aiGdI955YcNvUvS3RoXT1H+lEYwzWSdI5ex2KNheaWs8yJHsdaqC66XdB1Uw6hoGX1sK1m5tC0qo1HMvwm1PbFEfHUmq0ZifeHkQbDmMds0LJsCwutcBzgEl8rLS6vt5HNSGi4PeNaP98xUxuUaa72e/CMO++TDNMfcuPcVghdQLsakckOhXCGipfVZelEKlJdKAxaKdFJR6rKYMBEaDBrWcWYgKm7B+QwT9cgJDIp+zv1oXSv0IkazisWh9dpvm7vYJ5qcDNXx8g+sgum8vHVj5pO1gilIRuiJKi9qrL36iU3VAzhSxYPts7kakzSRl0N0cMgt553oNqFJFmRjJLOJafFWj7Q05GnWFXkMpEjpJvMTICnQVC1omMxSn8OIoufo6uzo7EU8i3H4yy15OUYXJAB3HyXi+i+eQafRJJ0DK/KSkFVds72ve8bHqZXv1JD92bQL9FxHTrMte8/AGXmaUkkdtxomnk7FLWap0BSpy8oR0ufw8H5rc6eDGrS0/7VcbOcCbLH/h/sHw7X7P3ujzv5/DGjM/9RyYOfsz53DO2tJH7ucoc9AXZlQy/fMhJnqBAZ99Z/3vexwpDfuXR2e977piXe98W6HLB/++1U7DuQ5PiGeqRSqCcsG7Ctu6sP7CmN1Pmo5Fd/UWX8QM9mh032MPNYI5dNvZoEyzUEpUgbt9qr6sHISCDwqoj0YoDXEWntCVwKCzKvoteIU7C1Tmgc5p8VBMAwhFclWhRlrZKcXlaMXeW59UT9929ikql9JxKkdMNXOdnJ8sd4sEZaEp0QYhtxKGFh3bL6a3GHggaWJTG+WSUR1co2BwYMWgcE8qaRmx2Q8mIE85+mBfcJK/covlW3Fx6IT5kJY6utkqsNT+qnM07H6GI2cS2FrxQyD7wglYN7HxAV15cmr6SFDPgvnYLKGMJpQSEN593qWJmlMrMK0JQHCHHEGvYe/2Rzk6SV/XzrWwZ64pIemhITiOzdCxHlWu52rguzm+jKUaUBNEuZB9D+dVGqTp/8Z1nLmypncF+kWcq1sHgJwczj1RH5X82bLQvwqD1ObzifnlQD/lo7uK87yTAATRdvP9ms5INVjfVGi5rpVlsAn0f8Ps/+0X7yjGbjt/t/ecFTP/z4YDDr77zHgPvsv8wOf0pnfYznrQlfJJJPqWF2xWwIjOcOgA5+Es9ty/y8j3P47YFv2vwgh1c9/RvsH3f5/DKhpazG/JQVclg7SPBPh0iKOay+dnhYZUC2h8PKceRNdj8QPlBwmcLGj9Mj0eUM3MhuhnERcLVMip5Sip6KmqtgsXqhc6q++Vk7GDo1ZAQ0nygArUrKKsiLbKgDbM15tuHB2L11LoTbfO9sJM2dsLdlth/7hd+v9y8uK/rlR2qJzOQEL8HbvWRWtlCzYenQLSm1Hdw2zeXQrP5TJl+fKNt3EEuVrt7GkO9BGK6oTofKaVyUqS683kW5Xl9zxp4Sd9P9SLfyHfgB0m/4fjer5H87A6b7/9yig7utI1Zh932eMSBqziJgec2/Bs45u55ZHlsU9G/35WztKpyBS8nK7CJvZTWIowfMxkmakkGFR6fLPyeyUJecgU4V4M4oDYPT+gwHSVrCnjZI8T7uuyZ/XVezzqg5wBq/eUqNZT0ktNTJqCP3XBhBpSrMWnx4Q5NXtAdVUlnAd4ZSr7xJK00GzoXi9KA3xnCaLdCqOvO3C7Sg/Tn02tQMs4hO2/BSzLUnbR3JSxFUYTbs8cXM3FmbZnLG5T26KK10K18SBtz/SaHI2ekPxiVdVkH951bEcx3r3tHvlrPVKhT+dgXphWZZhFFeR9LclytevxmjPEO5kfj8rtxYba6kkkTHSkeD1+1JjNOyLtxsvPWVeo6GO6iVLjd+OaPpyBBgM4k6wqGT/xGU6S+0rDo015PcVHH2FUH/8wBn2jfwDcGOjPIQ9GSfLNk75pmIW0itMv6bYXvmaQb6hda8rka6xkYeOVUhsNBrqouwqjdMf7AGjz5/nbcsPbOqY5jeIWHNL5T6Ia7XTlWTcLFIfDF2zEC7ZHdxcxqi43qv+q+47xB100EEHHXTQQQcddNBBBx100EEHHXTQQQcdPDH4P7tAqE4AeAAA values: image: tag: v1.36.0-dev diff --git a/go.mod b/go.mod index 1b6853179..0cfc039f9 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/onsi/ginkgo/v2 v2.17.3 github.com/onsi/gomega v1.33.1 github.com/pkg/errors v0.9.1 + github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.73.2 github.com/spf13/cobra v1.8.0 github.com/spf13/pflag v1.0.5 go.uber.org/mock v0.4.0 @@ -79,7 +80,6 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect - github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.73.2 // indirect github.com/prometheus/client_golang v1.18.0 // indirect github.com/prometheus/client_model v0.6.0 // indirect github.com/prometheus/common v0.45.0 // indirect diff --git a/pkg/controller/actuator_reconcile.go b/pkg/controller/actuator_reconcile.go index a26368bf0..7e0eaaea7 100644 --- a/pkg/controller/actuator_reconcile.go +++ b/pkg/controller/actuator_reconcile.go @@ -18,9 +18,14 @@ import ( gardenerkubernetes "github.com/gardener/gardener/pkg/client/kubernetes" "github.com/gardener/gardener/pkg/utils" "github.com/gardener/gardener/pkg/utils/chart" + kubernetesutils "github.com/gardener/gardener/pkg/utils/kubernetes" "github.com/gardener/gardener/pkg/utils/managedresources" "github.com/go-logr/logr" + monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + monitoringv1alpha1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1alpha1" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/client" @@ -48,6 +53,26 @@ func applyMonitoringConfig(ctx context.Context, seedClient client.Client, chartA Type: &corev1.ConfigMap{}, Name: cilium.MonitoringName, }, + { + Type: &corev1.ConfigMap{}, + Name: "cilium-dashboards", + }, + { + Type: &monitoringv1alpha1.ScrapeConfig{}, + Name: "shoot-cilium-agent", + }, + { + Type: &monitoringv1alpha1.ScrapeConfig{}, + Name: "shoot-cilium-hubble", + }, + { + Type: &monitoringv1alpha1.ScrapeConfig{}, + Name: "shoot-cilium-operator", + }, + { + Type: &monitoringv1.PrometheusRule{}, + Name: "shoot-cilium-agent", + }, }, } @@ -55,7 +80,15 @@ func applyMonitoringConfig(ctx context.Context, seedClient client.Client, chartA return client.IgnoreNotFound(ciliumControlPlaneMonitoringChart.Delete(ctx, seedClient, network.Namespace)) } - return ciliumControlPlaneMonitoringChart.Apply(ctx, chartApplier, network.Namespace, nil, "", "", nil) + // TODO(rfranzke): Delete this after August 2024. + gep19Monitoring := seedClient.Get(ctx, client.ObjectKey{Name: "prometheus-shoot", Namespace: network.Namespace}, &appsv1.StatefulSet{}) == nil + if gep19Monitoring { + if err := kubernetesutils.DeleteObject(ctx, seedClient, &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cilium-monitoring-config", Namespace: network.Namespace}}); err != nil { + return fmt.Errorf("failed deleting cilium-monitoring-config ConfigMap: %w", err) + } + } + + return ciliumControlPlaneMonitoringChart.Apply(ctx, chartApplier, network.Namespace, nil, "", "", map[string]interface{}{"gep19Monitoring": gep19Monitoring}) } // Reconcile implements Network.Actuator.