chart(kafka-ha): release 18.0.0

Author: itboon

charts/kafka-ha/Chart.yaml

@@ -7,4 +7,4 @@ keywords:
 - kraft
 name: kafka-ha
 type: application
-version: 17.0.2
+version: 18.0.0

charts/kafka-ha/README.md

@@ -98,4 +98,16 @@ broker:
 
 ## 集群外访问
 
-请参考 <https://github.com/itboon/kafka-helm/blob/main/docs/external.md>
+```yaml
+## NodePort example
+broker:
+  external:
+    enabled: true
+    service:
+      type: "NodePort"
+      annotations: {}
+    autoDiscovery:
+      enabled: true
+```
+
+更多集群外案例请参考 <https://github.com/itboon/kafka-helm/blob/main/docs/external.md>

charts/kafka-ha/templates/broker/_helpers.tpl

@@ -56,6 +56,17 @@ broker.externalDns.domain
 {{- end -}}
 {{- end -}}
 
+{{/*
+broker.externalDns.hostnamePrefix
+*/}}
+{{- define "broker.externalDns.hostnamePrefix" -}}
+{{- if .Values.broker.external.externalDns.hostnamePrefix -}}
+{{- .Values.broker.external.externalDns.hostnamePrefix -}}
+{{- else -}}
+{{- printf "%s-%s" (include "kafka.fullname" .) .Release.Namespace -}}
+{{- end -}}
+{{- end -}}
+
 {{/*
 broker.advertisedListeners.internal
 */}}
@@ -79,8 +90,7 @@ broker.advertisedListeners.external
   {{- $port = .hostPort | default .containerPort -}}
 {{- end -}}
 {{- if eq (include "broker.externalDns.enabled" $) "true" -}}
-  {{- $domain := (include "broker.externalDns.domain" $) -}}
-  {{- $addr = printf "$(POD_NAME).%s" $domain -}}
+  {{- $addr = printf "$(POD_NAME).%s.%s" (include "broker.externalDns.hostnamePrefix" $) (include "broker.externalDns.domain" $) -}}
 {{- end -}}
 {{- end -}}
 {{- printf "EXTERNAL://%s:%d" $addr ($port | int) -}}
@@ -179,12 +189,11 @@ broker.externalEnv
   value: {{ include "broker.fullNodePorts" $ | quote }}
 {{- end }}
 {{- if eq .type "LoadBalancer" }}
-  {{- $externalDnsEnabled := eq (include "broker.externalDns.enabled" $) "true" }}
-  {{- if and $externalDnsEnabled .externalDns.hostnamePrefixOverride }}
+  {{- if eq (include "broker.externalDns.enabled" $) "true" }}
 - name: KAFKA_EXTERNAL_DOMAIN
   value: {{ include "broker.externalDns.domain" $ }}
 - name: KAFKA_EXTERNAL_HOSTNAME_PREFIX
-  value: {{ .externalDns.hostnamePrefixOverride | quote }}
+  value: {{ include "broker.externalDns.hostnamePrefix" $ }}
   {{- end }}
 {{- end }}
 {{- end }}

charts/kafka-ha/templates/broker/statefulset.yaml

@@ -87,23 +87,17 @@ spec:
           {{- toYaml . | nindent 10 }}
         {{- end }}
         volumeMounts:
-        - mountPath: /opt/kafka/data
+        - mountPath: {{ .persistence.mountPath }}
           name: data
           subPath: data
-        - mountPath: /opt/kafka/logs
+        - mountPath: {{ .persistence.logsMountPath | default "/opt/kafka/logs" }}
           name: data
           subPath: logs
         - mountPath: "/etc/kafka/base-cm"
           name: base-cm
-        # - mountPath: "kafka.broker.baseConfigFile"
-        #   name: base-config
-        #   subPath: server.properties
         - mountPath: /entrypoint.sh
           name: entrypoint-sh
           subPath: entrypoint.sh
-        # - mountPath: /etc/kafka/broker/config-init.sh
-        #   name: config-init-sh
-        #   subPath: config-init.sh
         {{- if eq (include "broker.external.autoDiscovery.enabled" $) "true" }}
         - name: auto-discovery-shared
           mountPath: /init-shared

charts/kafka-ha/templates/broker/svc-external.yaml

@@ -26,10 +26,7 @@ metadata:
   {{- end }}
   {{- if eq (include "broker.externalDns.enabled" $) "true" }}
   {{- if eq $.Values.broker.external.type "LoadBalancer" }}
-    {{- $hostname := printf "%s.%s" $targetPod (include "broker.externalDns.domain" $) }}
-    {{- with $.Values.broker.external.externalDns.hostnamePrefixOverride }}
-    {{- $hostname = printf "%s-%d.%s" . (int $i) (include "broker.externalDns.domain" $) }}
-    {{- end }}
+    {{- $hostname := printf "%s-%d.%s" (include "broker.externalDns.hostnamePrefix" $) (int $i) (include "broker.externalDns.domain" $) }}
     external-dns.alpha.kubernetes.io/hostname: {{ $hostname | quote }}
     {{- with $.Values.broker.external.externalDns.annotations }}
     {{- toYaml . | nindent 4 }}

charts/kafka-ha/templates/broker/svc-headless.yaml

@@ -19,7 +19,8 @@ metadata:
   {{- $validExternalType := list "PodIP" "NodePort" "HostPort" }}
   {{- $externalType := $.Values.broker.external.type }}
   {{- if has $externalType $validExternalType }}
-    external-dns.alpha.kubernetes.io/hostname: {{ include "broker.externalDns.domain" $ }}
+    {{- $hostname := printf "%s.%s" (include "broker.externalDns.hostnamePrefix" $) (include "broker.externalDns.domain" $) }}
+    external-dns.alpha.kubernetes.io/hostname: {{ $hostname | quote }}
     {{- if or (eq $externalType "NodePort") (eq $externalType "HostPort") }}
     external-dns.alpha.kubernetes.io/endpoints-type: HostIP
     {{- end }}

charts/kafka-ha/templates/cm-entrypoint.yaml

@@ -11,26 +11,20 @@ data:
   entrypoint.sh: |
     #!/bin/sh
     
+    export KAFKA_HOME="${KAFKA_HOME:=/opt/kafka}"
+    
     export KAFKA_CONF_FILE="${KAFKA_CONF_FILE:=/etc/kafka/server.properties}"
     export KAFKA_BROKER_LISTENER_PORT="${KAFKA_BROKER_LISTENER_PORT:=9092}"
     export KAFKA_CONTROLLER_LISTENER_PORT="${KAFKA_CONTROLLER_LISTENER_PORT:=19091}"
     export KAFKA_BASE_CONF_FILE="${KAFKA_BASE_CONF_FILE:=/etc/kafka/base-cm/server.properties}"
     
-    export KAFKA_CFG_LOG_DIR="$KAFKA_HOME/data"
-    
-    if [ -z "$KAFKA_HOME" ]; then
-      export KAFKA_HOME="/opt/kafka"
-      export KAFKA_CFG_LOG_DIR="$KAFKA_HOME/data"
-    fi
-    if [ ! -d "$KAFKA_HOME" ]; then
-      mkdir -p "$KAFKA_HOME"
-    fi
+    export KAFKA_CFG_LOG_DIR="${KAFKA_CFG_LOG_DIR:=/var/lib/kafka/data}"
     
     check_runtime() {
       java -version
       if [ $? -ne 0 ]; then
         echo "[ERROR] Missing java"
-        exit "500"
+        exit "50"
       fi
     }
     
@@ -84,15 +78,6 @@ data:
       fi
     }
     
-    take_file_ownership() {
-      if [ "$(id -u)" = "0" ]; then
-        chown -R 1000:1000 "$KAFKA_HOME"
-        if [ -d "$KAFKA_CFG_LOG_DIR" ]; then
-          chown -R 1000:1000 "$KAFKA_CFG_LOG_DIR"
-        fi
-      fi
-    }
-    
     update_server_conf() {
       local key=$1
       local value=$2
@@ -169,7 +154,7 @@ data:
       if [ ! -f "$KAFKA_CONF_FILE" ]; then
         mkdir -p "$(dirname $KAFKA_CONF_FILE)"
         if [ -f "$KAFKA_BASE_CONF_FILE" ]; then
-          cat "$KAFKA_BASE_CONF_FILE" > $KAFKA_CONF_FILE
+          cat "$KAFKA_BASE_CONF_FILE" | grep -Ev '^log.dirs? *=' > $KAFKA_CONF_FILE
         fi
         touch "$KAFKA_CONF_FILE"
       fi
@@ -182,22 +167,43 @@ data:
       done
     }
     
-    reset_log_dirs() {
-      ## protect log.dirs
-      if [ "$KAFKA_LOG_DIR_LOCKED" = "false" ]; then
+    take_logdir_ownership_if_needed() {
+      local dir="$1"
+      if [ -d "$dir" ]; then
+        if [ "$KAFKA_LOGDIR_CHOWN_FORCE" = "true" ]; then
+          chown -R 1000:1000 "$dir"
+          echo "ls -alh $dir" ; ls -alh "$dir"
+        elif [ "$(stat -c "%u" $dir)" != "1000" ]; then
+          chown -R 1000:1000 "$dir"
+          echo "ls -alh $dir" ; ls -alh "$dir"
+        fi
+      fi
+    }
+    
+    init_kafka_dirs_ownership() {
+      if [ "$(id -u)" != "0" ]; then
         return
       fi
-      sed -i "/^log.dir *=/d" "$KAFKA_CONF_FILE"
-      update_server_conf "log.dirs" "$KAFKA_CFG_LOG_DIR"
+      if [ -n "$KAFKA_CFG_LOG_DIRS" ]; then
+        unset IFS
+        echo "$KAFKA_CFG_LOG_DIRS" | tr ',' '\n' | while read dir ; do
+          take_logdir_ownership_if_needed "$dir"
+        done
+      elif [ -d "$KAFKA_CFG_LOG_DIR" ]; then
+        take_logdir_ownership_if_needed "$KAFKA_CFG_LOG_DIR"
+      fi
+      if [ "$KAFKA_HOMEDIR_CHOWN_FORCE" = "true" ]; then
+        chown -R 1000:1000 "$KAFKA_HOME"
+      fi
+      take_logdir_ownership_if_needed "$KAFKA_HOME/logs"
     }
     
-    start_server() {
-      check_runtime
-      reset_log_dirs
-      if [ -n "$KAFKA_HEAP_OPTS" ]; then
-        export JAVA_TOOL_OPTIONS="${JAVA_TOOL_OPTIONS} ${KAFKA_HEAP_OPTS}"
+    init_storage_format_if_needed() {
+      local logdir="$KAFKA_CFG_LOG_DIR"
+      if [ -n "$KAFKA_CFG_LOG_DIRS" ]; then
+        logdir=$(echo "$KAFKA_CFG_LOG_DIRS" | cut -d "," -f 1)
       fi
-      if [ ! -f "$KAFKA_CFG_LOG_DIR/meta.properties" ]; then
+      if [ ! -f "$logdir/meta.properties" ]; then
         echo ">>> Format Log Directories <<<"
         if [ -z "$KAFKA_CLUSTER_ID" ]; then
           echo "Generate a Cluster UUID"
@@ -212,11 +218,19 @@ data:
             -t $KAFKA_CLUSTER_ID -c "$KAFKA_CONF_FILE"
         fi
       fi
+    }
+    
+    start_server() {
+      check_runtime
+      init_server_conf
+      init_kafka_dirs_ownership
+      if [ -n "$KAFKA_HEAP_OPTS" ]; then
+        export JAVA_TOOL_OPTIONS="${JAVA_TOOL_OPTIONS} ${KAFKA_HEAP_OPTS}"
+      fi
+      init_storage_format_if_needed
       run_as_other_user_if_needed "${KAFKA_HOME}/bin/kafka-server-start.sh" "$KAFKA_CONF_FILE"
     }
     
-    init_server_conf
-    take_file_ownership
     if [ "$@" = "start" ]; then
       start_server
     else

charts/kafka-ha/values.yaml

@@ -60,6 +60,8 @@ broker:
   persistence:
     enabled: true
     size: 20Gi
+    mountPath: /var/lib/kafka/data
+    logsMountPath: /opt/kafka/logs
     # storageClass: "gp2"
   ## broker.resources
   resources:
@@ -108,8 +110,8 @@ broker:
     nodePorts: []
     externalDns:
       enabled: false
-      # domain: ""
-      # hostnamePrefixOverride: ""
+      domain: ""
+      hostnamePrefix: ""
       annotations: {}
     autoDiscovery:
       enabled: false