feat(k8s): base with google-kubernetes-engine overlay example (#6041)

re: https://discord.com/channels/687972960811745322/738982866670714901/1041381487754354689 Introduces a more modular base with a google-kubernetes-engine overlay example.
erigontech · Nov 17, 2022 · 0d854fe · 0d854fe
1 parent 874d96f
commit 0d854fe
Show file tree

Hide file tree

Showing 15 changed files with 231 additions and 97 deletions.
diff --git a/k8s/base/kustomization.yaml b/k8s/base/kustomization.yaml
@@ -1,8 +1,3 @@
+---
 resources:
-- eth66-peering-tcp.yaml
-- eth66-peering-udp.yaml
-- http.yaml
-- metrics.yaml
-- snap-sync-tcp.yaml
-- snap-sync-udp.yaml
-- statefulset.yaml
+  - statefulset.yaml
diff --git a/k8s/base/eth66-peering-tcp.yaml → k8s/base/services/eth66-peering-tcp.yaml b/k8s/base/eth66-peering-tcp.yaml → k8s/base/services/eth66-peering-tcp.yaml
@@ -1,3 +1,4 @@
+---
 apiVersion: v1
 kind: Service
 metadata:
@@ -6,13 +7,13 @@ metadata:
   name: eth66-peering-tcp
 spec:
   ipFamilies:
-  - IPv4
+    - IPv4
   ipFamilyPolicy: SingleStack
   ports:
-  - name: eth66-pr-tcp
-    port: 30303
-    protocol: TCP
-    targetPort: 30303
+    - name: eth66-pr-tcp
+      port: 30303
+      protocol: TCP
+      targetPort: 30303
   selector:
     app: erigon
   type: LoadBalancer
diff --git a/k8s/base/eth66-peering-udp.yaml → k8s/base/services/eth66-peering-udp.yaml b/k8s/base/eth66-peering-udp.yaml → k8s/base/services/eth66-peering-udp.yaml
@@ -1,3 +1,4 @@
+---
 apiVersion: v1
 kind: Service
 metadata:
@@ -6,13 +7,13 @@ metadata:
   name: eth66-peering-udp
 spec:
   ipFamilies:
-  - IPv4
+    - IPv4
   ipFamilyPolicy: SingleStack
   ports:
-  - name: eth66-pr-udp
-    port: 30303
-    protocol: UDP
-    targetPort: 30303
+    - name: eth66-pr-udp
+      port: 30303
+      protocol: UDP
+      targetPort: 30303
   selector:
     app: erigon
   type: LoadBalancer
diff --git a/k8s/base/http.yaml → k8s/base/services/http.yaml b/k8s/base/http.yaml → k8s/base/services/http.yaml
@@ -1,3 +1,4 @@
+---
 apiVersion: v1
 kind: Service
 metadata:
@@ -6,13 +7,13 @@ metadata:
   name: http
 spec:
   ipFamilies:
-  - IPv4
+    - IPv4
   ipFamilyPolicy: SingleStack
   ports:
-  - name: http
-    port: 8545
-    protocol: TCP
-    targetPort: 8545
+    - name: http
+      port: 8545
+      protocol: TCP
+      targetPort: 8545
   selector:
     app: erigon
-  type: LoadBalancer
+  type: LoadBalancer
diff --git a/k8s/base/services/kustomization.yaml b/k8s/base/services/kustomization.yaml
@@ -0,0 +1,8 @@
+---
+resources:
+  - eth66-peering-tcp.yaml
+  - eth66-peering-udp.yaml
+  - http.yaml
+  - metrics.yaml
+  - snap-sync-tcp.yaml
+  - snap-sync-udp.yaml
diff --git a/k8s/base/metrics.yaml → k8s/base/services/metrics.yaml b/k8s/base/metrics.yaml → k8s/base/services/metrics.yaml
@@ -1,3 +1,4 @@
+---
 apiVersion: v1
 kind: Service
 metadata:
@@ -6,12 +7,12 @@ metadata:
   name: metrics
 spec:
   ipFamilies:
-  - IPv4
+    - IPv4
   ipFamilyPolicy: SingleStack
   ports:
-  - name: metrics
-    port: 6060
-    protocol: TCP
-    targetPort: 6060
+    - name: metrics
+      port: 6060
+      protocol: TCP
+      targetPort: 6060
   selector:
     app: erigon
diff --git a/k8s/base/snap-sync-tcp.yaml → k8s/base/services/snap-sync-tcp.yaml b/k8s/base/snap-sync-tcp.yaml → k8s/base/services/snap-sync-tcp.yaml
@@ -1,3 +1,4 @@
+---
 apiVersion: v1
 kind: Service
 metadata:
@@ -6,13 +7,13 @@ metadata:
   name: snap-sync-tcp
 spec:
   ipFamilies:
-  - IPv4
+    - IPv4
   ipFamilyPolicy: SingleStack
   ports:
-  - name: snap-sync-tcp
-    port: 42069
-    protocol: TCP
-    targetPort: 42069
+    - name: snap-sync-tcp
+      port: 42069
+      protocol: TCP
+      targetPort: 42069
   selector:
     app: erigon
   type: LoadBalancer
diff --git a/k8s/base/snap-sync-udp.yaml → k8s/base/services/snap-sync-udp.yaml b/k8s/base/snap-sync-udp.yaml → k8s/base/services/snap-sync-udp.yaml
@@ -1,3 +1,4 @@
+---
 apiVersion: v1
 kind: Service
 metadata:
@@ -6,13 +7,13 @@ metadata:
   name: snap-sync-udp
 spec:
   ipFamilies:
-  - IPv4
+    - IPv4
   ipFamilyPolicy: SingleStack
   ports:
-  - name: snap-sync-udp
-    port: 42069
-    protocol: UDP
-    targetPort: 42069
+    - name: snap-sync-udp
+      port: 42069
+      protocol: UDP
+      targetPort: 42069
   selector:
     app: erigon
   type: LoadBalancer
diff --git a/k8s/base/statefulset.yaml b/k8s/base/statefulset.yaml
@@ -1,3 +1,4 @@
+---
 apiVersion: apps/v1
 kind: StatefulSet
 metadata:
@@ -17,65 +18,48 @@ spec:
         app: erigon
     spec:
       containers:
-      - args:
-        - '--authrpc.vhosts=*'
-        - '--datadir=/home/erigon/.local/share/erigon'
-        - '--healthcheck'
-        - '--log.json'
-        - '--metrics'
-        - '--metrics.addr=0.0.0.0'
-        - '--metrics.port=6060'
-        - '--nat=none'
-        - '--private.api.ratelimit=50000'
-        - '--private.api.addr=0.0.0.0:9090'
-        - '--torrent.download.rate=3000mb'
-        - '--torrent.download.slots=200'
-        - '--verbosity=3'
-        - —-batchSize=8000M
-        command:
-        - erigon
-        image: erigon-image
-        livenessProbe:
-          initialDelaySeconds: 1800
-          periodSeconds: 20
-          tcpSocket:
-            port: 9090
-        name: erigon
-        ports:
-        - containerPort: 9090
-          name: private-api
-        - containerPort: 8551
-          name: engine
-        - containerPort: 30303
-          name: eth66-pr-tcp
-          protocol: TCP
-        - containerPort: 30303
-          name: eth66-pr-udp
-          protocol: UDP
-        - containerPort: 6060
-          name: metrics
-        - containerPort: 6070
-          name: pprof
-        - containerPort: 42069
-          name: snap-sync-tcp
-          protocol: TCP
-        - containerPort: 42069
-          name: snap-sync-udp
-          protocol: UDP
-        readinessProbe:
-          initialDelaySeconds: 1800
-          periodSeconds: 20
-          tcpSocket:
-            port: 9090
-        resources:
-          requests:
-            cpu: 2462m
-            memory: 16Gi
-        securityContext:
-          allowPrivilegeEscalation: false
-          runAsGroup: 1000
-          runAsUser: 1000
-        volumeMounts: []
+        - args: []
+          command:
+            - erigon
+          env: []
+          image: thorax/erigon
+          name: erigon
+          ports:
+            - containerPort: 9090
+              name: private-api
+            - containerPort: 8551
+              name: engine
+            - containerPort: 30303
+              name: eth66-pr-tcp
+              protocol: TCP
+            - containerPort: 30303
+              name: eth66-pr-udp
+              protocol: UDP
+            - containerPort: 8545
+              name: http
+              protocol: TCP
+            - containerPort: 6060
+              name: metrics
+            - containerPort: 6070
+              name: pprof
+            - containerPort: 42069
+              name: snap-sync-tcp
+              protocol: TCP
+            - containerPort: 42069
+              name: snap-sync-udp
+              protocol: UDP
+          readinessProbe:
+            tcpSocket:
+              port: 8545
+          resources:
+            requests:
+              cpu: 2462m
+              memory: 16Gi
+          securityContext:
+            allowPrivilegeEscalation: false
+            runAsGroup: 1000
+            runAsUser: 1000
+          volumeMounts: []
       initContainers: []
       volumes: []
   volumeClaimTemplates: []
diff --git a/k8s/google-kubernetes-engine/kustomization.yaml b/k8s/google-kubernetes-engine/kustomization.yaml
@@ -0,0 +1,12 @@
+---
+patchesJson6902:
+  - path: statefulset-erigon-patch.yaml
+    target:
+      group: apps
+      kind: StatefulSet
+      name: erigon
+      version: v1
+resources:
+  - ../base
+  - ../base/services
+  - podmonitoring.yaml
diff --git a/k8s/google-kubernetes-engine/podmonitoring.yaml b/k8s/google-kubernetes-engine/podmonitoring.yaml
@@ -0,0 +1,13 @@
+---
+apiVersion: monitoring.googleapis.com/v1
+kind: PodMonitoring
+metadata:
+  name: erigon
+spec:
+  endpoints:
+    - interval: 30s
+      path: /debug/metrics/prometheus
+      port: metrics
+  selector:
+    matchLabels:
+      app: erigon
diff --git a/k8s/google-kubernetes-engine/statefulset-erigon-patch.yaml b/k8s/google-kubernetes-engine/statefulset-erigon-patch.yaml
@@ -0,0 +1,70 @@
+---
+- op: replace
+  path: /spec/replicas
+  value: 3
+- op: add
+  path: /spec/template/spec/containers/0/resources/limits
+  value:
+    cpu: '9'
+    memory: 110Gi
+- op: replace
+  path: /spec/template/spec/containers/0/resources/requests/cpu
+  value: '9'
+- op: replace
+  path: /spec/template/spec/containers/0/resources/requests/memory
+  value: 110Gi
+- op: replace
+  path: /spec/template/spec/containers/0/args
+  value:
+    - '--chain=mainnet'
+    - '--datadir=/home/erigon/.local/share/erigon'
+    - '--db.pagesize=64KB'
+    - '--healthcheck'
+    - '--http'
+    - '--http.addr=0.0.0.0'
+    - '--http.api=eth,erigon,web3,net,debug,ots,trace,txpool'
+    - '--http.corsdomain=*'
+    - '--http.vhosts=*'
+    - '--log.console.verbosity=1'
+    - '--log.json'
+    - '--metrics'
+    - '--metrics.addr=0.0.0.0'
+    - '--metrics.port=6060'
+    - '--nat=none'
+    - '--torrent.download.rate=3000mb'
+    - '--torrent.download.slots=200'
+    - '--ws'
+    - '--ws.compression'
+- op: replace
+  path: /spec/template/spec/containers/0/volumeMounts
+  value:
+    - mountPath: /home/erigon/.local/share/erigon
+      name: mainnet
+- op: add
+  path: /spec/template/spec/initContainers/-
+  value:
+    command:
+      - sh
+      - '-c'
+      - chown 1000 -R /home/erigon/.local/share/erigon
+    image: busybox
+    name: chown-datadir
+    securityContext:
+      capabilities:
+        add:
+          - CHOWN
+    volumeMounts:
+      - mountPath: /home/erigon/.local/share/erigon
+        name: mainnet
+- op: add
+  path: /spec/volumeClaimTemplates/-
+  value:
+    metadata:
+      name: mainnet
+    spec:
+      accessModes:
+        - ReadWriteOnce
+      resources:
+        requests:
+          storage: 3000Gi
+      storageClassName: premium-rwo