From cbbe93c31e74298af9dcd0c7c698b718d5ee4328 Mon Sep 17 00:00:00 2001 From: JooYoung Park Date: Mon, 3 Jun 2024 01:17:17 +0800 Subject: [PATCH] fix setup and configs to expose kube-proxy and etcd metrics replace stale configs files to repo URLs Signed-off-by: JooYoung Park --- config/kubeadm_init.yaml | 8 + config/metrics_server_components.yaml | 201 -------------------------- config/prometh_kn.yaml | 169 ---------------------- config/prometh_values_kn.yaml | 7 +- scripts/setup/expose_infra_metrics.sh | 14 +- 5 files changed, 20 insertions(+), 379 deletions(-) delete mode 100644 config/metrics_server_components.yaml delete mode 100644 config/prometh_kn.yaml diff --git a/config/kubeadm_init.yaml b/config/kubeadm_init.yaml index 4ef80e4c2..7193a13b2 100644 --- a/config/kubeadm_init.yaml +++ b/config/kubeadm_init.yaml @@ -16,6 +16,8 @@ dns: {} etcd: local: dataDir: /var/lib/etcd + extraArgs: + listen-metrics-urls: http://0.0.0.0:2381 imageRepository: registry.k8s.io kubernetesVersion: v1.29.1 networking: @@ -36,3 +38,9 @@ kind: KubeletConfiguration kubeAPIQPS: 50 # Default: 5 kubeAPIBurst: 100 # Default: 10 configMapAndSecretChangeDetectionStrategy: "Watch" + + +--- +apiVersion: kubeproxy.config.k8s.io/v1alpha1 +kind: KubeProxyConfiguration +metricsBindAddress: 0.0.0.0:10249 \ No newline at end of file diff --git a/config/metrics_server_components.yaml b/config/metrics_server_components.yaml deleted file mode 100644 index 59a02723c..000000000 --- a/config/metrics_server_components.yaml +++ /dev/null @@ -1,201 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - labels: - k8s-app: metrics-server - name: metrics-server - namespace: kube-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - k8s-app: metrics-server - rbac.authorization.k8s.io/aggregate-to-admin: "true" - rbac.authorization.k8s.io/aggregate-to-edit: "true" - rbac.authorization.k8s.io/aggregate-to-view: "true" - name: system:aggregated-metrics-reader -rules: -- apiGroups: - - metrics.k8s.io - resources: - - pods - - nodes - verbs: - - get - - list - - watch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - k8s-app: metrics-server - name: system:metrics-server -rules: -- apiGroups: - - "" - resources: - - nodes/metrics - verbs: - - get -- apiGroups: - - "" - resources: - - pods - - nodes - verbs: - - get - - list - - watch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - labels: - k8s-app: metrics-server - name: metrics-server-auth-reader - namespace: kube-system -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: extension-apiserver-authentication-reader -subjects: -- kind: ServiceAccount - name: metrics-server - namespace: kube-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - k8s-app: metrics-server - name: metrics-server:system:auth-delegator -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:auth-delegator -subjects: -- kind: ServiceAccount - name: metrics-server - namespace: kube-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - k8s-app: metrics-server - name: system:metrics-server -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:metrics-server -subjects: -- kind: ServiceAccount - name: metrics-server - namespace: kube-system ---- -apiVersion: v1 -kind: Service -metadata: - labels: - k8s-app: metrics-server - name: metrics-server - namespace: kube-system -spec: - ports: - - name: https - port: 443 - protocol: TCP - targetPort: https - selector: - k8s-app: metrics-server ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - k8s-app: metrics-server - name: metrics-server - namespace: kube-system -spec: - selector: - matchLabels: - k8s-app: metrics-server - strategy: - rollingUpdate: - maxUnavailable: 0 - template: - metadata: - labels: - k8s-app: metrics-server - spec: - containers: - - args: - - --cert-dir=/tmp - - --secure-port=4443 - - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname - - --kubelet-use-node-status-port - - --metric-resolution=15s - - --kubelet-insecure-tls=true - - --logtostderr - image: registry.k8s.io/metrics-server/metrics-server:v0.6.1 - imagePullPolicy: IfNotPresent - livenessProbe: - failureThreshold: 3 - httpGet: - path: /livez - port: https - scheme: HTTPS - periodSeconds: 10 - name: metrics-server - ports: - - containerPort: 4443 - name: https - protocol: TCP - readinessProbe: - failureThreshold: 3 - httpGet: - path: /readyz - port: https - scheme: HTTPS - initialDelaySeconds: 20 - periodSeconds: 10 - resources: - limits: - cpu: 1 - memory: 1Gi - requests: - cpu: 100m - memory: 200Mi - securityContext: - allowPrivilegeEscalation: false - readOnlyRootFilesystem: true - runAsNonRoot: true - runAsUser: 1000 - volumeMounts: - - mountPath: /tmp - name: tmp-dir - nodeSelector: - kubernetes.io/os: linux - priorityClassName: system-cluster-critical - serviceAccountName: metrics-server - volumes: - - emptyDir: {} - name: tmp-dir ---- -apiVersion: apiregistration.k8s.io/v1 -kind: APIService -metadata: - labels: - k8s-app: metrics-server - name: v1beta1.metrics.k8s.io -spec: - group: metrics.k8s.io - groupPriorityMinimum: 100 - insecureSkipTLSVerify: true - service: - name: metrics-server - namespace: kube-system - version: v1beta1 - versionPriority: 100 diff --git a/config/prometh_kn.yaml b/config/prometh_kn.yaml deleted file mode 100644 index 82d4eaeeb..000000000 --- a/config/prometh_kn.yaml +++ /dev/null @@ -1,169 +0,0 @@ ---- -apiVersion: monitoring.coreos.com/v1 -kind: ServiceMonitor -metadata: - labels: - app: controller - name: controller - namespace: knative-serving -spec: - endpoints: - - interval: 2s - port: http-metrics - namespaceSelector: - matchNames: - - knative-serving - selector: - matchLabels: - app: controller ---- -apiVersion: monitoring.coreos.com/v1 -kind: ServiceMonitor -metadata: - labels: - app: autoscaler - name: autoscaler - namespace: knative-serving -spec: - endpoints: - - interval: 2s - port: http-metrics - namespaceSelector: - matchNames: - - knative-serving - selector: - matchLabels: - app: autoscaler ---- -apiVersion: monitoring.coreos.com/v1 -kind: ServiceMonitor -metadata: - labels: - app: activator - name: activator - namespace: knative-serving -spec: - endpoints: - - interval: 2s - port: http-metrics - namespaceSelector: - matchNames: - - knative-serving - selector: - matchLabels: - app: activator ---- -apiVersion: monitoring.coreos.com/v1 -kind: ServiceMonitor -metadata: - labels: - app: webhook - name: webhook - namespace: knative-serving -spec: - endpoints: - - interval: 2s - port: http-metrics - namespaceSelector: - matchNames: - - knative-serving - selector: - matchLabels: - app: activator ---- -apiVersion: monitoring.coreos.com/v1 -kind: ServiceMonitor -metadata: - labels: - app: broker-filter - name: broker-filter - namespace: knative-eventing -spec: - endpoints: - - interval: 2s - port: http-metrics - namespaceSelector: - matchNames: - - knative-eventing - selector: - matchLabels: - eventing.knative.dev/brokerRole: filter ---- -apiVersion: monitoring.coreos.com/v1 -kind: ServiceMonitor -metadata: - labels: - app: broker-ingress - name: broker-ingress - namespace: knative-eventing -spec: - endpoints: - - interval: 2s - port: http-metrics - namespaceSelector: - matchNames: - - knative-eventing - selector: - matchLabels: - eventing.knative.dev/brokerRole: ingress ---- -apiVersion: monitoring.coreos.com/v1 -kind: PodMonitor -metadata: - name: eventing-controller - labels: - app: eventing-controller - namespace: knative-eventing -spec: - selector: - matchLabels: - app: eventing-controller - namespaceSelector: - matchNames: - - knative-eventing - podMetricsEndpoints: - - port: metrics ---- -apiVersion: monitoring.coreos.com/v1 -kind: PodMonitor -metadata: - name: imc-controller - namespace: knative-eventing -spec: - selector: - matchLabels: - messaging.knative.dev/role: controller - namespaceSelector: - matchNames: - - knative-eventing - podMetricsEndpoints: - - port: metrics ---- -apiVersion: monitoring.coreos.com/v1 -kind: PodMonitor -metadata: - name: ping-source - namespace: knative-eventing -spec: - selector: - matchLabels: - eventing.knative.dev/source: ping-source-controller - namespaceSelector: - matchNames: - - knative-eventing - podMetricsEndpoints: - - port: metrics ---- -apiVersion: monitoring.coreos.com/v1 -kind: PodMonitor -metadata: - name: apiserver-source - namespace: knative-eventing -spec: - namespaceSelector: - any: true - selector: - matchLabels: - eventing.knative.dev/source: apiserver-source-controller - podMetricsEndpoints: - - port: metrics \ No newline at end of file diff --git a/config/prometh_values_kn.yaml b/config/prometh_values_kn.yaml index 7cc923c0f..af935950b 100644 --- a/config/prometh_values_kn.yaml +++ b/config/prometh_values_kn.yaml @@ -2,8 +2,6 @@ alertmanager: config: route: group_by: ['job'] - - serviceMonitor: interval: "15s" alertmanagerSpec: @@ -53,11 +51,7 @@ coreDns: serviceMonitor: interval: "15s" - kubeEtcd: - service: - port: 2379 - targetPort: 2379 serviceMonitor: interval: "15s" @@ -68,6 +62,7 @@ kubeScheduler: interval: "15s" https: true insecureSkipVerify: true + kubeProxy: serviceMonitor: interval: "15s" diff --git a/scripts/setup/expose_infra_metrics.sh b/scripts/setup/expose_infra_metrics.sh index 60be7b15a..7858e9270 100755 --- a/scripts/setup/expose_infra_metrics.sh +++ b/scripts/setup/expose_infra_metrics.sh @@ -34,7 +34,9 @@ server_exec() { server_exec 'sudo apt install htop' #* Deploy Metrics Server to k8s in namespace kube-system. - server_exec 'cd loader; kubectl apply -f config/metrics_server_components.yaml' + metrics_server_version="v0.7.1" + server_exec "kubectl apply -f https://github.com/kubernetes-sigs/metrics-server/releases/download/${metrics_server_version}/components.yaml" + server_exec "kubectl patch deployment metrics-server -n kube-system --type='json' -p='[{"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "--kubelet-insecure-tls=true"}]'" #* Install helm. server_exec 'curl https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash' @@ -44,15 +46,21 @@ server_exec() { server_exec 'kubectl create namespace monitoring' release_label="prometheus" - prometheus_chart_version="58.5.0" + prometheus_chart_version="60.1.0" server_exec "cd loader; helm install -n monitoring $release_label --version $prometheus_chart_version prometheus-community/kube-prometheus-stack -f config/prometh_values_kn.yaml" + #* Apply the ServiceMonitors/PodMonitors to collect metrics from Knative. #* The ports of the control manager and scheduler are mapped in a way that prometheus default installation can find them. - server_exec 'cd loader; kubectl apply -f config/prometh_kn.yaml' + #* Also apply the grafana dashboards for Knative. + server_exec "curl -s https://raw.githubusercontent.com/knative-extensions/monitoring/main/servicemonitor.yaml | sed 's/interval: 30s/interval: 2s/g' | kubectl apply -f -" + server_exec 'kubectl apply -f https://raw.githubusercontent.com/knative-extensions/monitoring/main/grafana/dashboards.yaml' #* Bind addresses of the control manager and scheduler to "0.0.0.0" so that prometheus can scrape them from any domains. server_exec 'cd loader; sudo kubeadm upgrade apply --config config/kubeadm_init.yaml --ignore-preflight-errors all --force --v=7' + #* Restart the kube-proxy to apply the changes. + server_exec 'kubectl delete pod -l k8s-app=kube-proxy -n kube-system' + sleep 5 #* Set up port prometheus panel (infinite loops are important to circumvent kubectl timeout in the middle of experiments).