update 1.30

helm_karpenter.tf

@@ -61,7 +61,7 @@ spec:
   blockDeviceMappings:
     - deviceName: /dev/xvda
       ebs:
-        volumeSize: 30Gi
+        volumeSize: 20Gi
         volumeType: gp3
         iops: 3000
         deleteOnTermination: true
@@ -105,10 +105,11 @@ spec:
         kind: EC2NodeClass
         name: ${var.cluster_name}-default
   limits:
-    cpu: 1000
+    cpu: 50
+    memory: 100Gi
   disruption:
     consolidationPolicy: WhenEmptyOrUnderutilized
-    consolidateAfter: 72h
+    consolidateAfter: 2h
 YAML
 
   depends_on = [

helm_prometheus.tf

@@ -7,6 +7,10 @@ resource "helm_release" "prometheus" {
 
   version = "62.3.1"
 
+  values = [
+    "${file("./prometheus/values.yaml")}"
+  ]
+
 
   depends_on = [
     aws_eks_cluster.eks_cluster,
@@ -19,6 +23,38 @@ resource "helm_release" "prometheus" {
   ]
 }
 
+resource "kubectl_manifest" "prometheus_all_pod_monitor" {
+
+  count = 0
+
+  yaml_body = <<YAML
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  name: generic-stats-monitor
+  namespace: prometheus
+  labels:
+    monitoring: istio-proxies
+    release: istio
+spec:
+  selector:
+    matchExpressions:
+    - {key: istio-prometheus-ignore, operator: DoesNotExist}
+  namespaceSelector:
+    any: true
+  jobLabel: generic-stats
+  podMetricsEndpoints:
+  - path: /metrics
+    interval: 15s
+    relabelings:
+    - action: keep
+YAML
+
+  depends_on = [
+    helm_release.prometheus
+  ]
+}
+
 resource "kubectl_manifest" "grafana_gateway" {
   yaml_body = <<YAML
 apiVersion: networking.istio.io/v1alpha3

prometheus/values.yaml

@@ -0,0 +1,70 @@
+prometheus:
+  prometheusSpec:
+    ruleSelectorNilUsesHelmValues: false
+    serviceMonitorSelectorNilUsesHelmValues: false
+    podMonitorSelectorNilUsesHelmValues: false
+    probeSelectorNilUsesHelmValues: false
+    podMonitorNamespaceSelector:
+      any: true
+    podMonitorSelector: {}
+    ruleNamespaceSelector:
+      any: true
+    ruleSelector: {}
+    serviceMonitorNamespaceSelector:
+      any: true
+    serviceMonitorSelector: {}
+    additionalScrapeConfigs:
+      - job_name: 'kubernetes-pods'
+        honor_labels: true
+        kubernetes_sd_configs:
+          - role: pod
+        relabel_configs:
+          - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
+            action: keep
+            regex: true
+          - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape_slow]
+            action: drop
+            regex: true
+          - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scheme]
+            action: replace
+            regex: (https?)
+            target_label: __scheme__
+          - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
+            action: replace
+            target_label: __metrics_path__
+            regex: (.+)
+          - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_port, __meta_kubernetes_pod_ip]
+            action: replace
+            regex: (\d+);(([A-Fa-f0-9]{1,4}::?){1,7}[A-Fa-f0-9]{1,4})
+            replacement: '[$2]:$1'
+            target_label: __address__
+          - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_port, __meta_kubernetes_pod_ip]
+            action: replace
+            regex: (\d+);((([0-9]+?)(\.|$)){4})
+            replacement: $2:$1
+            target_label: __address__
+          - action: labelmap
+            regex: __meta_kubernetes_pod_annotation_prometheus_io_param_(.+)
+            replacement: __param_$1
+          - action: labelmap
+            regex: __meta_kubernetes_pod_label_(.+)
+          - source_labels: [__meta_kubernetes_namespace]
+            action: replace
+            target_label: namespace
+          - source_labels: [__meta_kubernetes_pod_name]
+            action: replace
+            target_label: pod
+          - source_labels: [__meta_kubernetes_pod_phase]
+            regex: Pending|Succeeded|Failed|Completed
+            action: drop
+          - source_labels: [__meta_kubernetes_pod_node_name]
+            action: replace
+            target_label: node
+
+prometheusOperator:
+  enabled: true
+  namespaces: ''
+  denyNamespaces: ''
+  prometheusInstanceNamespaces: ''
+  alertmanagerInstanceNamespaces: ''
+  thanosRulerInstanceNamespaces: ''