goto · ishanarya0 · Sep 11, 2024 · Sep 9, 2024 · Sep 11, 2024 · Sep 11, 2024
diff --git a/stable/dagger/.helmignore b/stable/dagger/.helmignore
@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/
diff --git a/stable/dagger/Chart.yaml b/stable/dagger/Chart.yaml
@@ -0,0 +1,5 @@
+apiVersion: v2
+name: dagger-deployment-chart
+description: A Helm chart for dagger deployment
+version: 0.1.0
+appVersion: "1.0"
diff --git a/stable/dagger/templates/NOTES.txt b/stable/dagger/templates/NOTES.txt
@@ -0,0 +1 @@
+Thank you for installing {{ .Chart.Name }}.
diff --git a/stable/dagger/templates/dagger.yaml b/stable/dagger/templates/dagger.yaml
@@ -0,0 +1,141 @@
+apiVersion: flink.apache.org/v1beta1
+kind: FlinkDeployment
+metadata:
+  name: {{ .Values.deployment_id }}
+spec:
+  image: {{ .Values.image }}
+  flinkVersion: v1_14
+  flinkConfiguration:
+    blob.server.port: "6124"
+    taskmanager.rpc.port: "6122"
+    taskmanager.numberOfTaskSlots: "1"
+    metrics.latency.interval: "0"
+    web.timeout: "60000"
+    akka.ask.timeout: 10 min
+    rest.server.numThreads: "20"
+    slot.request.timeout: "300000"
+    cluster.evenly-spread-out-slots: "false"
+    rest.flamegraph.enabled: "false"
+    taskmanager.memory.managed.fraction: "0.2"
+    taskmanager.memory.managed.consumer-weights: OPERATOR:70,STATE_BACKEND:70,PYTHON:30
+    classloader.resolve-order: parent-first
+
+    #HA CONFIGS
+    state.backend: filesystem
+    state.backend.fs.checkpointdir: {{ .Values.storage_protocol }}://{{ .Values.projectID }}-dagger-checkpoints/flink-checkpoints/{{ .Values.name }}
+    state.savepoints.dir: {{ .Values.storage_protocol }}://{{ .Values.projectID }}-dagger-checkpoints/flink-savepoints/{{ .Values.name }}
+    blob.storage.directory: /tmp/flink-blobs
+    jobmanager.web.tmpdir: /tmp/flink-web
+    web.upload.dir: /tmp/flink-web/flink-uploads
+
+    high-availability: org.apache.flink.kubernetes.highavailability.KubernetesHaServicesFactory
+    high-availability.storageDir: {{ .Values.storage_protocol }}://{{ .Values.projectID }}-dagger-checkpoints/ha
+
+    metrics.reporters: stsd
+    metrics.reporter.stsd.class: com.gojek.de.dagger.metrics.StatsDReporter
+    metrics.reporter.stsd.host: localhost
+    metrics.reporter.stsd.port: "8125"
+    metrics.reporter.stsd.tags: dagger_name={{ .Values.name }},team_name={{ .Values.team }}
+    metrics.reporter.stsd.blacklistedTags: task_name,operator_name,task_attempt_id,metric_type
+    metrics.reporter.stsd.blacklistedMetrics: .*KafkaConsumer_current_offsets_.*,.*KafkaConsumer_committed_offsets.*,.*Shuffle_Netty.*
+    metrics.reporter.stsd.interval: 5 SECONDS
+    akka.framesize: 10485760b
+    client.timeout: 10 min
+    metrics.scope.jm: jobmanager.godata-id-flink-operator
+    metrics.scope.jm.job: jobmanager-job.godata-id-flink-operator.<job_name>.<job_id>
+    metrics.scope.tm: taskmanager.godata-id-flink-operator.<tm_id>
+    metrics.scope.tm.job: taskmanager-job.godata-id-flink-operator.<tm_id>.<job_name>
+    metrics.scope.task: task.godata-id-flink-operator.<tm_id>.<job_name>.<task_name>.<subtask_index>
+    metrics.scope.operator: operator.godata-id-flink-operator.<tm_id>.<job_name>.<subtask_index>
+  serviceAccount: flink
+  podTemplate:
+    apiVersion: v1
+    kind: Pod
+    metadata:
+      name: common-pod-template
+      labels:
+        application: dagger
+        app-id: 02408523-06e0-4b4a-a3f4-8df6d429e21e
+        instance-id: {{ .Values.deployment_id }}
+        {{- range $key, $value := .Values.extra_labels }}
+        {{ $key }}: '{{ $value }}'
+        {{- end }}
+    spec:
+      serviceAccount: flink
+      containers:
+        - name: flink-main-container
+          env:
+            - name: GOOGLE_APPLICATION_CREDENTIALS
+              value: /var/secrets/google/gcp_key.json
+          volumeMounts:
+            - mountPath: /etc/hadoop/conf
+              name: hadoop-config
+            - mountPath: /var/secrets/google
+              name: google-cloud-key
+        - name: telegrafd
+          image: telegraf:1.18.3-alpine
+          imagePullPolicy: IfNotPresent
+          env:
+            - name: CLUSTER_NAME
+              value: {{ .Values.flink_name }}
+            - name: PROMETHEUS_URL
+              value: {{ .Values.prometheus_url }}
+          terminationMessagePath: /dev/termination-log
+          terminationMessagePolicy: File
+          resources:
+            limits:
+              cpu: 100m
+              memory: 100Mi
+            requests:
+              cpu: 100m
+              memory: 100Mi
+          volumeMounts:
+            - mountPath: /etc/telegraf
+              name: telegraf-config
+      volumes:
+        - name: hadoop-config
+          configMap:
+            defaultMode: 420
+            name: flink-operator-hadoop-config
+        - name: telegraf-config
+          configMap:
+            defaultMode: 420
+            name: flink-operator-telegraf-config
+        - name: google-cloud-key
+          secret:
+            defaultMode: 420
+            secretName: flink-operator-gcp
+  jobManager:
+    resource:
+      cpu: {{ .Values.resources.jobmanager.cpu }}
+      memory: {{ .Values.resources.jobmanager.memory }}
+  taskManager:
+    resource:
+      cpu: {{ .Values.resources.taskmanager.cpu }}
+      memory: {{ .Values.resources.taskmanager.memory }}
+  job:
+    jarURI: {{ .Values.jarURI }}
+    parallelism: {{ .Values.configuration.FLINK_PARALLELISM }}
+    upgradeMode: stateless
+    state: {{ .Values.state }}
+    entryClass: com.gotocompany.dagger.core.KafkaProtoSQLProcessor
+    args:
+      {{  range .Values.programArgs }}
+      - {{ . }}
+      {{ end }}
+  logConfiguration:
+    "log4j-console.properties": |
+      rootLogger.level=INFO
+      rootLogger.appenderRef.console.ref=ConsoleAppender
+      logger.flink.name=org.apache.flink
+      logger.flink.level=INFO
+      logger.akka.name=akka
+      logger.akka.level=INFO
+      logger.kafka.name=org.apache.kafka
+      logger.kafka.level=INFO
+      logger.hadoop.name=org.apache.hadoop
+      logger.hadoop.level=INFO
+      appender.console.name=ConsoleAppender
+      appender.console.type=CONSOLE
+      appender.console.layout.type=PatternLayout
+      appender.console.layout.pattern=%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c - %m%n
diff --git a/stable/dagger/values.yaml b/stable/dagger/values.yaml
@@ -0,0 +1,23 @@
+deployment_id: "dagger-test-deployment"
+image: "dagger-flink-operator-image"
+projectID: "g-pilotdata-gl"
+name: "dagger-test"
+team: "streaming"
+flink_name: "flink-cluster"
+prometheus_url: "http://prometheus:9090"
+storage_protocol: gs
+resources:
+  jobmanager:
+    cpu: "1"
+    memory: "1024m"
+  taskmanager:
+    cpu: "1"
+    memory: "1024m"
+jarURI: "dagger-minimal-jar"
+configuration:
+  FLINK_PARALLELISM: 1
+state: "running"
+programArgs:
+  - "--encodedArgs"
+  - WyItLVNPVVJDRV9LQUZLQV9DT05TVU1FX0xBUkdFX01FU1NBR0VfRU5BQkxFIixmYWxzZSwiLS1FTkFCTEVfU1RFTkNJTF9VUkwiLHRydWUsIi0tRkxJTktfSk9CX0lEIiwiZy1waWxvdGRhdGEtZ2wtdGVzdC1kYWdnZXIiLCItLVNJTktfSU5GTFVYX0JBVENIX1NJWkUiLDEwMCwiLS1TSU5LX0lORkxVWF9EQl9OQU1FIiwiIiwiLS1TSU5LX0lORkxVWF9GTFVTSF9EVVJBVElPTl9NUyIsMTAwMCwiLS1TSU5LX0lORkxVWF9NRUFTVVJFTUVOVF9OQU1FIiwiIiwiLS1TSU5LX0lORkxVWF9QQVNTV09SRCIsInJvb3QiLCItLVNJTktfSU5GTFVYX1JFVEVOVElPTl9QT0xJQ1kiLCJhdXRvZ2VuIiwiLS1TSU5LX0lORkxVWF9VUkwiLCIiLCItLVNJTktfSU5GTFVYX1VTRVJOQU1FIiwicm9vdCIsIi0tUFJPQ0VTU09SX0xPTkdCT1dfR0NQX0lOU1RBTkNFX0lEIiwidGVzdCIsIi0tUFJPQ0VTU09SX0xPTkdCT1dfR0NQX1BST0pFQ1RfSUQiLCJ0ZXN0IiwiLS1TSU5LX0tBRktBX0JST0tFUlMiLCIiLCItLVNJTktfS0FGS0FfVE9QSUMiLCIiLCItLU9VVFBVVF9QUk9UT19DTEFTU19QUkVGSVgiLCIiLCItLVNJTktfS0FGS0FfUFJPVE9fS0VZIiwiIiwiLS1TSU5LX0tBRktBX1BST1RPX01FU1NBR0UiLCIiLCItLVNJTktfS0FGS0FfU1RSRUFNIiwiIiwiLS1GTElOS19QQVJBTExFTElTTSIsMSwiLS1QT1JUQUxfVkVSU0lPTiIsIjIiLCItLVBST0NFU1NPUl9QUkVQUk9DRVNTT1JfQ09ORklHIiwiIiwiLS1QUk9DRVNTT1JfUFJFUFJPQ0VTU09SX0VOQUJMRSIsZmFsc2UsIi0tUFJPQ0VTU09SX1BPU1RQUk9DRVNTT1JfQ09ORklHIiwiIiwiLS1QUk9DRVNTT1JfUE9TVFBST0NFU1NPUl9FTkFCTEUiLGZhbHNlLCItLVNJTktfS0FGS0FfUFJPRFVDRV9MQVJHRV9NRVNTQUdFX0VOQUJMRSIsZmFsc2UsIi0tUkVESVNfU0VSVkVSIiwiIiwiLS1GTElOS19ST1dUSU1FX0FUVFJJQlVURV9OQU1FIiwicm93dGltZSIsIi0tU0lOS19UWVBFIiwiaW5mbHV4IiwiLS1GTElOS19TUUxfUVVFUlkiLCJTRUxFQ1QgKiBGUk9NIGRhdGFfc3RyZWFtc18wIiwiLS1TQ0hFTUFfUkVHSVNUUllfU1RFTkNJTF9FTkFCTEUiLHRydWUsIi0tU0NIRU1BX1JFR0lTVFJZX1NURU5DSUxfVVJMUyIsIiIsIi0tU1RSRUFNUyIsIlt7XCJJTlBVVF9TQ0hFTUFfRVZFTlRfVElNRVNUQU1QX0ZJRUxEX0lOREVYXCI6XCI1XCIsXCJJTlBVVF9TQ0hFTUFfUFJPVE9fQ0xBU1NcIjpcImdvamVrLmVzYi5ib29raW5nLkJvb2tpbmdMb2dNZXNzYWdlXCIsXCJJTlBVVF9TQ0hFTUFfVEFCTEVcIjpcImRhdGFfc3RyZWFtc18wXCIsXCJTT1VSQ0VfREVUQUlMU1wiOlt7XCJTT1VSQ0VfTkFNRVwiOlwiS0FGS0FfQ09OU1VNRVJcIixcIlNPVVJDRV9UWVBFXCI6XCJVTkJPVU5ERURcIn1dLFwiU09VUkNFX0tBRktBX0NPTlNVTUVSX0NPTkZJR19BVVRPX0NPTU1JVF9FTkFCTEVcIjpcImZhbHNlXCIsXCJTT1VSQ0VfS0FGS0FfQ09OU1VNRVJfQ09ORklHX0FVVE9fT0ZGU0VUX1JFU0VUXCI6XCJsYXRlc3RcIixcIlNPVVJDRV9LQUZLQV9DT05TVU1FUl9DT05GSUdfQk9PVFNUUkFQX1NFUlZFUlNcIjpcIlwiLFwiU09VUkNFX0tBRktBX0NPTlNVTUVSX0NPTkZJR19HUk9VUF9JRFwiOlwiXCIsXCJTT1VSQ0VfS0FGS0FfTkFNRVwiOlwiXCIsXCJTT1VSQ0VfS0FGS0FfVE9QSUNfTkFNRVNcIjpcIlwiLFwiU09VUkNFX1BBUlFVRVRfRklMRV9EQVRFX1JBTkdFXCI6bnVsbCxcIlNPVVJDRV9QQVJRVUVUX0ZJTEVfUEFUSFNcIjpudWxsfV0iLCItLUZMSU5LX1dBVEVSTUFSS19ERUxBWV9NUyIsMTAwMCwiLS1GTElOS19XQVRFUk1BUktfSU5URVJWQUxfTVMiLDYwMDAwLCItLVVERl9EQVJUX0dDU19CVUNLRVRfSUQiLCJwLWdvZGF0YS1kYWdnZXJzLWRhcnRzLXN0b3JhZ2UiLCItLVVERl9EQVJUX0dDU19QUk9KRUNUX0lEIiwiZ29kYXRhLXByb2R1Y3Rpb24iLCItLUZVTkNUSU9OX0ZBQ1RPUllfQ0xBU1NFUyIsImNvbS5nb3RvY29tcGFueS5kYWdnZXIuZnVuY3Rpb25zLnVkZnMuZmFjdG9yaWVzLkZ1bmN0aW9uRmFjdG9yeSxjb20uZ29qZWsuZGUuZmx1ZC51ZGZzLmZhY3Rvcmllcy5GdW5jdGlvbkZhY3RvcnkiLCItLVBZVEhPTl9VREZfRU5BQkxFIixmYWxzZSwiLS1QWVRIT05fVURGX0NPTkZJRyIsIntcIlBZVEhPTl9GSUxFU1wiOiBcImdzOi8vZ29kYXRhLWRhZ2dlci9weXRob24vbWFzdGVyL2xhdGVzdC9weXRob25fdWRmcy56aXBcIixcIlBZVEhPTl9SRVFVSVJFTUVOVFNcIjogXCJnczovL2dvZGF0YS1kYWdnZXIvcHl0aG9uL21hc3Rlci9sYXRlc3QvcmVxdWlyZW1lbnRzLnR4dFwiLFwiUFlUSE9OX0FSQ0hJVkVTXCI6IFwiZ3M6Ly9nb2RhdGEtZGFnZ2VyL3B5dGhvbi9tYXN0ZXIvbGF0ZXN0L2RhdGEuemlwI2RhdGFcIixcIlBZVEhPTl9GTl9FWEVDVVRJT05fQVJST1dfQkFUQ0hfU0laRVwiOiBcIjEwMDAwXCIsXCJQWVRIT05fRk5fRVhFQ1VUSU9OX0JVTkRMRV9TSVpFXCI6IFwiMTAwMDAwXCIsXCJQWVRIT05fRk5fRVhFQ1VUSU9OX0JVTkRMRV9USU1FXCI6IFwiMTAwMFwifSIsIi0tU0lOS19CSUdRVUVSWV9HT09HTEVfQ0xPVURfUFJPSkVDVF9JRCIsIiIsIi0tU0lOS19CSUdRVUVSWV9UQUJMRV9OQU1FIiwiIiwiLS1TSU5LX0JJR1FVRVJZX0RBVEFTRVRfTEFCRUxTIiwiIiwiLS1TSU5LX0JJR1FVRVJZX1RBQkxFX0xBQkVMUyIsIiIsIi0tU0lOS19CSUdRVUVSWV9EQVRBU0VUX05BTUUiLCIiLCItLVNJTktfQklHUVVFUllfQ1JFREVOVElBTF9QQVRIIiwiL3Zhci9zZWNyZXRzL2dvb2dsZS9nY3Bfa2V5Lmpzb24iLCItLVNJTktfQklHUVVFUllfVEFCTEVfUEFSVElUSU9OSU5HX0VOQUJMRSIsZmFsc2UsIi0tU0lOS19CSUdRVUVSWV9UQUJMRV9QQVJUSVRJT05fS0VZIiwiIiwiLS1TSU5LX0JJR1FVRVJZX1JPV19JTlNFUlRfSURfRU5BQkxFIixmYWxzZSwiLS1TSU5LX0JJR1FVRVJZX0NMSUVOVF9SRUFEX1RJTUVPVVRfTVMiLC0xLCItLVNJTktfQklHUVVFUllfQ0xJRU5UX0NPTk5FQ1RfVElNRU9VVF9NUyIsLTEsIi0tU0lOS19CSUdRVUVSWV9UQUJMRV9QQVJUSVRJT05fRVhQSVJZX01TIiwtMSwiLS1TSU5LX0JJR1FVRVJZX0RBVEFTRVRfTE9DQVRJT04iLCJhc2lhLXNvdXRoZWFzdDEiLCItLVNJTktfQklHUVVFUllfTUVUQURBVEFfTkFNRVNQQUNFIiwiIiwiLS1TSU5LX0JJR1FVRVJZX0FERF9NRVRBREFUQV9FTkFCTEVEIixmYWxzZSwiLS1TSU5LX0JJR1FVRVJZX01FVEFEQVRBX0NPTFVNTlNfVFlQRVMiLCIiLCItLVNJTktfTUVUUklDU19BUFBMSUNBVElPTl9QUkVGSVgiLCJkYWdnZXJfIiwiLS1TSU5LX0JJR1FVRVJZX0JBVENIX1NJWkUiLCIiLCItLVNJTktfQ09OTkVDVE9SX1NDSEVNQV9QUk9UT19NRVNTQUdFX0NMQVNTIiwiIiwiLS1TSU5LX0NPTk5FQ1RPUl9TQ0hFTUFfUFJPVE9fS0VZX0NMQVNTIiwiIiwiLS1TSU5LX0NPTk5FQ1RPUl9TQ0hFTUFfREFUQV9UWVBFIiwiUFJPVE9CVUYiLCItLVNJTktfQ09OTkVDVE9SX1NDSEVNQV9NRVNTQUdFX01PREUiLCJMT0dfTUVTU0FHRSIsIi0tU0lOS19DT05ORUNUT1JfU0NIRU1BX1BST1RPX0FMTE9XX1VOS05PV05fRklFTERTX0VOQUJMRSIsZmFsc2UsIi0tU0lOS19FUlJPUl9UWVBFU19GT1JfRkFJTFVSRSIsIiIsIi0tU0lOS19CSUdRVUVSWV9UQUJMRV9DTFVTVEVSSU5HX0VOQUJMRSIsZmFsc2UsIi0tU0lOS19CSUdRVUVSWV9UQUJMRV9DTFVTVEVSSU5HX0tFWVMiLCIiLCItLVNDSEVNQV9SRUdJU1RSWV9TVEVOQ0lMX0NBQ0hFX0FVVE9fUkVGUkVTSCIsZmFsc2UsIi0tU0NIRU1BX1JFR0lTVFJZX1NURU5DSUxfUkVGUkVTSF9TVFJBVEVHWSIsIkxPTkdfUE9MTElORyIsIi0tU0NIRU1BX1JFR0lTVFJZX1NURU5DSUxfQ0FDSEVfVFRMX01TIiwiOTAwMDAwIiwiLS1TSU5LX0tBRktBX0xJTkdFUl9NUyIsIjAiLCItLVNDSEVNQV9SRUdJU1RSWV9TVEVOQ0lMX0ZFVENIX1JFVFJJRVMiLCI0IiwiLS1TQ0hFTUFfUkVHSVNUUllfU1RFTkNJTF9GRVRDSF9CQUNLT0ZGX01JTl9NUyIsIjYwMDAwIiwiLS1TQ0hFTUFfUkVHSVNUUllfU1RFTkNJTF9GRVRDSF9USU1FT1VUX01TIiwiMTAwMCJd
+extra_labels: {}