feat: implement mirror.py container support (via side car) (#135)

Signed-off-by: Deepak Mishra <deepak@swirldslabs.com>
hashgraph · Jun 29, 2023 · c8ea9e2 · c8ea9e2
1 parent dc0a27a
commit c8ea9e2
Show file tree

Hide file tree

Showing 4 changed files with 179 additions and 49 deletions.
diff --git a/charts/hedera-network/templates/configmaps.yaml b/charts/hedera-network/templates/configmaps.yaml
@@ -1,11 +1,19 @@
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: uploader-mirror-config
+ name: uploader-config
 data:
-  # add your data here
-  property1: value1
-  property2: value2
+  DEBUG: "{{ $.Values.mirror.debug }}"
+  REAPER_ENABLE: "{{ $.Values.mirror.reaper.enable }}"
+  REAPER_MIN_KEEP: "{{ $.Values.mirror.reaper.minKeep }}"
+  REAPER_INTERVAL: "{{ $.Values.mirror.reaper.interval }}"
+  REAPER_DEFAULT_BACKOFF: "{{ $.Values.mirror.reaper.defaultBackoff }}"
+  S3_ENABLE: "{{ $.Values.mirror.s3.enable }}"
+  GCS_ENABLE: "{{ $.Values.mirror.gcs.enable }}"
+  SIG_REQUIRE: "{{ $.Values.mirror.sig.require }}"
+  SIG_PRIORITIZE: "{{ $.Values.mirror.sig.prioritize }}"
+  BUCKET_PATH: "{{ $.Values.mirror.accountBalance.bucketPath }}"
+  BUCKET_NAME: "{{ $.Values.mirror.accountBalance.bucketName }}"
 ---
 apiVersion: v1
 kind: ConfigMap

diff --git a/charts/hedera-network/templates/network-node-statefulset.yaml b/charts/hedera-network/templates/network-node-statefulset.yaml
@@ -1,52 +1,128 @@
 
-{{ range $nodeName, $nodeConfig := ($.Values.hedera.nodes) }}
+{{ range $node := ($.Values.hedera.nodes) }}
 ---
 apiVersion: apps/v1
 kind: StatefulSet
 metadata:
-  name: network-{{ $nodeName }}
+  name: network-{{ $node.name }}
   labels:
-    app: network-{{ $nodeName }}
+    app: network-{{ $node.name }}
 spec:
   replicas: 1
-  serviceName: "network-node-{{ $nodeName }}"
+  serviceName: "network-node-{{ $node.name }}"
   selector:
     matchLabels:
-      app: network-{{ $nodeName }}
+      app: network-{{ $node.name }}
   template:
     metadata:
       labels:
-        app: network-{{ $nodeName }}
+        app: network-{{ $node.name }}
     spec:
+      volumes:
+        - name: hedera-storage
+          emptyDir: {}
+
       containers:
       - name: root-container
         image: {{ $.Values.infrastructure.docker.registry }}/{{ $.Values.infrastructure.docker.images.root }}
         resources:
           requests:
-            memory: 50Mi
-            cpu: {{ index (index $.Values.hedera.nodes $nodeName) "cpu" }}
+            cpu: {{ $node.requests.cpu }}
+            memory: {{ $node.requests.memory }}
           limits:
-            memory: 100Mi
-            cpu: 100m
-      - name: record-stream-uploader
-        image: {{ $.Values.infrastructure.docker.registry }}/{{ $.Values.infrastructure.docker.images.streamuploader }}
+            cpu: {{ $node.limits.cpu }}
+            memory: {{ $node.limits.memory }}
+        volumeMounts:
+          - name: hedera-storage
+            mountPath: /opt/hgcapp/
+
+  {{ if $node.uploaderSidecars }}
+
+      # Account Balance Uploader
+      - name: account-balances-uploader
+        image: {{ $.Values.infrastructure.docker.images.mirroruploader }}
+        securityContext:
+          runAsUser: {{ $.Values.mirror.user }}
+        command:
+          - /usr/bin/env
+          - python3.7
+          - /usr/local/bin/mirror.py
+          - --linux
+          - --watch-directory
+          - {{ $.Values.mirror.accountBalance.watchDir }}
+        volumeMounts:
+          - name: hedera-storage
+            mountPath: /opt/hgcapp/
         envFrom:
-        - configMapRef:
-            name: uploader-mirror-config
-        - secretRef:
-            name: uploader-mirror-secrets
+          - configMapRef:
+                name: uploader-config
+          - secretRef:
+              name: uploader-secrets
+
+        # Event Stream Uploader
       - name: event-stream-uploader
-        image: {{ $.Values.infrastructure.docker.registry }}/{{ $.Values.infrastructure.docker.images.streamuploader }}
+        image: {{ $.Values.infrastructure.docker.images.mirroruploader }}
+        securityContext:
+          runAsUser: {{ $.Values.mirror.user }}
+        command:
+          - /usr/bin/env
+          - python3.7
+          - /usr/local/bin/mirror.py
+          - --linux
+          - --watch-directory
+          - {{ $.Values.mirror.eventStream.watchDir }}
+        volumeMounts:
+          - name: hedera-storage
+            mountPath: /opt/hgcapp/
         envFrom:
-        - configMapRef:
-            name: uploader-mirror-config
-        - secretRef:
-            name: uploader-mirror-secrets
-      - name: backup-uploader
-        image: {{ $.Values.infrastructure.docker.registry }}/{{ $.Values.infrastructure.docker.images.backupuploader }}
+          - configMapRef:
+              name: uploader-config
+          - secretRef:
+              name: uploader-secrets
+
+        # Record Stream Uploader
+      - name: record-stream-uploader
+        image: {{ $.Values.infrastructure.docker.images.mirroruploader }}
+        securityContext:
+          runAsUser: {{ $.Values.mirror.user }}
+        command:
+          - /usr/bin/env
+          - python3.7
+          - /usr/local/bin/mirror.py
+          - --linux
+          - --watch-directory
+          - {{ $.Values.mirror.recordStream.watchDir }}
+          - --csv-stats-directory
+          - {{ $.Values.mirror.recordStream.csvStatsDir }}
+        volumeMounts:
+          - name: hedera-storage
+            mountPath: /opt/hgcapp/
         envFrom:
-        - configMapRef:
-            name: backup-config
-        - secretRef:
-            name: backup-secrets
+          - configMapRef:
+              name: uploader-config
+          - secretRef:
+              name: uploader-secrets
+
+      # Record Stream Sidecar Uploader
+      - name: record-sidecar-uploader
+        image: {{ $.Values.infrastructure.docker.images.mirroruploader }}
+        securityContext:
+          runAsUser: {{ $.Values.mirror.user }}
+        command:
+          - /usr/bin/env
+          - python3.7
+          - /usr/local/bin/mirror.py
+          - --linux
+          - --watch-directory
+          - {{ $.Values.mirror.recordStreamSideCar.watchDir }}
+        volumeMounts:
+          - name: hedera-storage
+            mountPath: /opt/hgcapp/
+        envFrom:
+          - configMapRef:
+              name: uploader-config
+          - secretRef:
+              name: uploader-secrets
+  {{ end }}
+
 {{ end }}
diff --git a/charts/hedera-network/templates/secrets.yaml b/charts/hedera-network/templates/secrets.yaml
@@ -1,13 +1,13 @@
 apiVersion: v1
 kind: Secret
 metadata:
-  name: uploader-mirror-secrets
+  name: uploader-secrets
 type: Opaque
-data:
-  # add your secrets here
-  # Note: Kubernetes secrets should be base64 encoded
-  secret1: "c2VjcmV0"
-  secret2: "c2VjcmV0"
+data: # TODO: These secrets should be got from minio secrets
+  S3_ACCESS_KEY: ""
+  S3_SECRET_KEY: ""
+  GCS_ACCESS_KEY: ""
+  GCS_SECRET_KEY: ""
 ---
 apiVersion: v1
 kind: Secret

diff --git a/charts/hedera-network/values.yaml b/charts/hedera-network/values.yaml
@@ -1,25 +1,41 @@
 hedera:
   # TODO: do we need an easy way (to avoid repetition) to create multiple notes if they all have the same configuration?
   nodes:
-    node-1:
+    - name: node-1
       # TODO: Configuration like address ( and similar ), can be autogenerated by helm, do we need to think about this ?
       address: 0.0.1
       cpu: 1
-      memory: 1GB
+      requests:
+        cpu: 1
+        memory: 1G
+      limits:
+        cpu: 1
+        memory: 1G
       latency: 10ms
       bandwidth: 1 Mbps
-    node-2:
+      uploaderSidecars: true
+    - name: node-2
       address: 0.0.2
-      cpu: 1
-      memory: 1GB
+      requests:
+        cpu: 1
+        memory: 1G
+      limits:
+        cpu: 1
+        memory: 1G
       latency: 10ms
       bandwidth: 2 Mbps
-    node-3:
+      uploaderSidecars: true
+    - name: node-3
       address: 0.0.3
-      cpu: 1
-      memory: 1GB
+      requests:
+        cpu: 1
+        memory: 1G
+      limits:
+        cpu: 1
+        memory: 1G
       latency: 10ms
       bandwidth: 3 Mbps
+      uploaderSidecars: true
 
 infrastructure:
   docker:
@@ -29,15 +45,45 @@ infrastructure:
       envoyproxy: envoyproxy/envoy:v1.26-latest
       haproxy: haproxy:lts-alpine3.18
       jsonrpcrelay: jsonrpcrelay
+      mirroruploader: gcr.io/hedera-registry/uploader-mirror:1.3.0
       streamuploader: ubuntu
       backupuploader: ubuntu
 
+mirror:
+  # TODO: Use accurate folder names
+  bucketName: bucketName
+  accountBalance:
+    watchDir: /opt/hgcapp/accountbalance
+    bucketPath: /accountbalance
+  recordStream:
+    watchDir: /opt/hgcapp/recordstream
+    csvStatsDir: /uploader-stats/recordstreams/gcs
+    bucketPath: /recordstream
+  recordStreamSideCar:
+    watchDir: /opt/hgcapp/sidecar
+    bucketPath: /recordstreamsidecar
+  eventStream:
+    watchDir: /opt/hgcapp/events
+    bucketPath: /events
+  user: 1000
+  debug: true
+  reaper:
+    enable: true
+    minKeep: 1
+    interval: 1
+    defaultBackoff: 1
+  sig:
+    require: true
+    prioritize: true
+  s3:
+    enable: true
+  gcs:
+    enable: false
+  bucket:
+    path: bucket
+
   proxies:
     # 1 Haproxy Deployment is created for 1 hedera network node (StatefulSet)
     # This controls how many replicas within a deployment are needed, in production we use 3
     haproxyReplicas: 1
     envoyProxyReplicas: 1
-
-
-
-