Merge pull request #86 from trifork/feature/flink-ssl

Always ssl between flink-containers

charts/flink-job/Chart.yaml

@@ -7,7 +7,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.3.9
+version: 0.4.0
 
 dependencies:
   - name: image-automation

charts/flink-job/README.md

@@ -1,6 +1,6 @@
 # flink-job
 
-![Version: 0.3.9](https://img.shields.io/badge/Version-0.3.9-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square)
+![Version: 0.4.0](https://img.shields.io/badge/Version-0.4.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square)
 
 A Helm chart for handling Cheetah Data Platform Flink jobs
 
@@ -129,6 +129,7 @@ Read more about Flink and highly available job-managers [here](https://nightlies
 | global.imagePullSecrets | list | `[]` | Set the global image pull secrets If image automation is enabled, this is useful to reduce configuration duplication |
 | imagePullSecrets | list | `[]` | Array of image pull secrets. Each entry follows the `name: <secret-name>` format |
 | version | string | `"v1_16"` | Which Flink version to use |
+| internalSsl.enabled | bool | `true` | Whether to use SSL between the job- and taskmanager |
 | flinkConfiguration | object | (see [values.yaml](values.yaml)) | Flink configuration For more configuration options, see here: <https://nightlies.apache.org/flink/flink-docs-master/docs/deployment/config/> For specific metrics configuration, see here:  <https://nightlies.apache.org/flink/flink-docs-master/docs/deployment/metric_reporters/> |
 | restartNonce | int | `0` | change this to force a restart of the job, see <https://nightlies.apache.org/flink/flink-kubernetes-operator-docs-main/docs/custom-resource/job-management/> for more info |
 | logConfiguration | object | (see [values.yaml](values.yaml)) | Custom logging configuration |

charts/flink-job/templates/_helpers.tpl

@@ -169,7 +169,8 @@ Calculate the flinkConfiguration
   {{- $configs = fromJson (include "flink-job.haConfiguration" (dict "configs" $configs "global" $.Values "fullname" $fullname)) -}}
   {{- $configs = fromJson (include "flink-job.storageConfiguration" (dict "configs" $configs "global" $.Values "fullname" $fullname)) -}}
   {{- $configs = fromJson (include "flink-job.istioConfiguration" (dict "configs" $configs "global" $.Values "fullname" $fullname)) -}}
-  {{- toYaml $configs -}}
+  {{- $configs = fromJson (include "flink-job.sslConfiguration" (dict "configs" $configs "global" $.Values "fullname" $fullname)) -}}
+  {{ toYaml $configs }}
 {{- end -}}
 
 {{/*
@@ -189,6 +190,24 @@ Add necessary metrics configuration
   {{- $configs | toJson -}}
 {{- end -}}
 
+{{/*
+Add necessary ssl configuration
+*/}}
+{{- define "flink-job.sslConfiguration" -}}
+  {{- $configs := .configs -}}
+  {{- $password := sha1sum (nospace (toString .global.image)) | trunc 10 }}
+  {{- if .global.internalSsl.enabled -}}
+    {{- $configs = fromJson (include "flink-job._dictSet" (list $configs "security.ssl.internal.enabled" "true")) -}}
+    {{- $configs = fromJson (include "flink-job._dictSet" (list $configs "security.ssl.internal.keystore" "/flinkkeystore/keystore.jks")) -}}
+    {{- $configs = fromJson (include "flink-job._dictSet" (list $configs "security.ssl.internal.truststore" "/flinkkeystore/truststore.jks")) -}}
+    {{- $configs = fromJson (include "flink-job._dictSet" (list $configs "security.ssl.internal.keystore-password" (toString $password))) -}}
+    {{- $configs = fromJson (include "flink-job._dictSet" (list $configs "security.ssl.internal.truststore-password" (toString $password))) -}}
+    {{- $configs = fromJson (include "flink-job._dictSet" (list $configs "security.ssl.internal.key-password" (toString $password))) -}}
+  {{- end -}}
+  {{- $configs | toJson -}}
+{{- end -}}
+
+
 {{/*
 Add necessary istio configuration
 */}}
@@ -243,6 +262,7 @@ Validate the configuration
   {{- end -}}
   {{- $configs | toJson -}}
 {{- end -}}
+
 {{/*
 Set a key=value in a dictionary, if the key is not defined
 */}}
@@ -255,3 +275,15 @@ Set a key=value in a dictionary, if the key is not defined
   {{- end -}}
   {{- $dict | toJson -}}
 {{- end -}}
+
+{{- define "flink-job.sslVolumes" -}}
+  {{- if $.Values.internalSsl.enabled -}}
+  {{ (dict "name" "truststore" "secret" (dict "secretName" (print (include "flink-job.fullname" . ) "-mtls-secret"))) | toYaml }}
+  {{- end -}}
+{{- end -}}
+
+{{- define "flink-job.sslVolumeMounts" -}}
+  {{- if $.Values.internalSsl.enabled -}}
+    {{ (dict "name" "truststore" "mountPath" "/flinkkeystore" "readOnly" "true") | toYaml}}
+  {{- end -}}
+{{- end -}}

charts/flink-job/templates/cert.yaml

@@ -0,0 +1,24 @@
+{{- if .Values.internalSsl.enabled -}}
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+  name: {{ printf "%s-mtls-issuer" ( include "flink-job.fullname" . ) | quote }}
+spec:
+  selfSigned: {}
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: {{ printf "%s-mtls-crt" ( include "flink-job.fullname" . ) | quote }}
+spec:
+  secretName: {{ printf "%s-mtls-secret" ( include "flink-job.fullname" . ) | quote }}
+  commonName: {{ printf "%s-mtls-crt" ( include "flink-job.fullname" . ) | quote }}
+  issuerRef:
+    name: {{ printf "%s-mtls-issuer" ( include "flink-job.fullname" . ) | quote }}
+  keystores:
+    jks:
+      create: true
+      passwordSecretRef: # Password used to encrypt the keystore
+        key: password
+        name: {{ printf "%s-mtls-password" ( include "flink-job.fullname" . ) | quote }}
+{{- end -}}

charts/flink-job/templates/flink-deployment.yaml

@@ -24,6 +24,7 @@ spec:
   flinkConfiguration:
     {{- include "flink-job.calculateConfigurations" . | nindent 4 -}}
 
+
   {{- with .Values.podTemplate }}
   podTemplate:
     {{- tpl . $ | nindent 4 -}}

charts/flink-job/templates/secret.yaml

@@ -12,4 +12,14 @@ data:
   tls.crt: {{ $cert.Cert | b64enc }}
   tls.key: {{ $cert.Key | b64enc }}
   ca.crt: {{ $ca.Cert | b64enc }}
-{{- end -}}
+{{- end }}
+
+{{- if .Values.internalSsl.enabled }}
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ printf "%s-mtls-password" ( include "flink-job.fullname" . ) | quote }}
+stringData:
+  password: {{ sha1sum (nospace (toString .Values.image)) | trunc 10 }}
+{{- end -}}

charts/flink-job/values.yaml

@@ -29,6 +29,10 @@ imagePullSecrets: []
 # -- Which Flink version to use
 version: v1_16
 
+internalSsl:
+  # -- Whether to use SSL between the job- and taskmanager
+  enabled: true
+
 # -- Flink configuration
 # For more configuration options, see here: <https://nightlies.apache.org/flink/flink-docs-master/docs/deployment/config/>
 # For specific metrics configuration, see here:  <https://nightlies.apache.org/flink/flink-docs-master/docs/deployment/metric_reporters/>
@@ -176,7 +180,7 @@ podTemplate: |-
     {{- end }}
     {{- with .Values.initContainers }}
     initContainers:
-      {{- toYaml . | nindent 4 }}
+      {{- toYaml .  | nindent 4 }}
     {{- end }}
 
 taskManager:
@@ -235,11 +239,11 @@ taskManager:
           envFrom:
             {{- toYaml . | nindent 8 }}
           {{- end }}
-          {{- with (concat .Values.volumeMounts .Values.taskManager.volumeMounts) }}
+          {{- with (concat .Values.volumeMounts .Values.taskManager.volumeMounts (list (include "flink-job.sslVolumeMounts" . | fromYaml))) }}
           volumeMounts:
             {{- toYaml . | nindent 8 -}}
           {{- end }}
-      {{- with (concat .Values.volumes .Values.taskManager.volumes) }}
+      {{- with (concat .Values.volumes .Values.taskManager.volumes  (list (include "flink-job.sslVolumes" . | fromYaml))) }}
       volumes:
         {{- toYaml . | nindent 4 -}}
       {{- end }}
@@ -304,11 +308,11 @@ jobManager:
           envFrom:
             {{- toYaml . | nindent 8 }}
           {{- end }}
-          {{- with (concat .Values.volumeMounts .Values.jobManager.volumeMounts) }}
+          {{- with (concat .Values.volumeMounts .Values.jobManager.volumeMounts (list (include "flink-job.sslVolumeMounts" . | fromYaml))) }}
           volumeMounts:
             {{- toYaml . | nindent 8 -}}
           {{- end }}
-      {{- with (concat .Values.volumes .Values.jobManager.volumes) }}
+      {{- with (concat .Values.volumes .Values.jobManager.volumes  (list (include "flink-job.sslVolumes" . | fromYaml))) }}
       volumes:
         {{- toYaml . | nindent 4 -}}
       {{- end }}