Implement DB cleanup in Helm

chart/templates/_helpers.yaml

@@ -760,6 +760,11 @@ server_tls_key_file = /etc/pgbouncer/server.key
   {{- include "_serviceAccountName" (merge (dict "key" "cleanup") .) -}}
 {{- end }}
 
+{{/* Create the name of the DB cleanup service account to use */}}
+{{- define "dbCleanup.serviceAccountName" -}}
+  {{- include "_serviceAccountName" (merge (dict "key" "dbCleanup") .) -}}
+{{- end }}
+
 {{- define "wait-for-migrations-command" -}}
   {{- if semverCompare ">=2.0.0" .Values.airflowVersion }}
   - airflow
@@ -1142,3 +1147,26 @@ capabilities:
             app: keda-operator
   {{- end }}
 {{- end }}
+
+{{- define "airflow.db_clean_command" }}
+  - airflow
+  - db
+  - clean
+  - --yes
+  - --clean-before-timestamp
+  - $( date --utc --date='now - {{ int .Values.dbCleanup.cleanBeforeSeconds }} seconds' +'%Y-%m-%d %H:%M:%S%:z' )
+  {{- if .Values.dbCleanup.skipArchive }}
+  - --skip-archive
+  {{- end }}
+  {{- if .Values.dbCleanup.verbose }}
+  - --verbose
+  {{- end }}
+  {{- with .Values.dbCleanup.batchSize }}
+  - --batch-size
+  - "{{ . }}"
+  {{- end }}
+  {{- with .Values.dbCleanup.tables }}
+  - --tables
+  - "{{ . | join "," }}"
+  {{- end }}
+{{- end }}

chart/templates/db-cleanup/db-cleanup-cronjob.yaml

@@ -0,0 +1,125 @@
+{{/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied.  See the License for the
+ specific language governing permissions and limitations
+ under the License.
+*/}}
+
+################################
+## Airflow DB Cleanup CronJob
+#################################
+{{- if .Values.dbCleanup.enabled }}
+{{- $nodeSelector := or .Values.dbCleanup.nodeSelector .Values.nodeSelector }}
+{{- $affinity := or .Values.dbCleanup.affinity .Values.affinity }}
+{{- $tolerations := or .Values.dbCleanup.tolerations .Values.tolerations }}
+{{- $topologySpreadConstraints := or .Values.dbCleanup.topologySpreadConstraints .Values.topologySpreadConstraints }}
+{{- $securityContext := include "airflowPodSecurityContext" (list . .Values.dbCleanup) }}
+{{- $containerSecurityContext := include "containerSecurityContext" (list . .Values.dbCleanup) }}
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: {{ include "airflow.fullname" . }}-db-cleanup
+  labels:
+    tier: airflow
+    component: airflow-db-cleanup
+    release: {{ .Release.Name }}
+    chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+    heritage: {{ .Release.Service }}
+    {{- with .Values.labels }}
+      {{- toYaml . | nindent 4 }}
+    {{- end }}
+  {{- with .Values.dbCleanup.jobAnnotations }}
+  annotations: {{- toYaml . | nindent 4 }}
+  {{- end }}
+spec:
+  schedule: "{{ tpl .Values.dbCleanup.schedule . }}"
+  concurrencyPolicy: Forbid
+  {{- if not ( eq .Values.dbCleanup.failedJobsHistoryLimit nil) }}
+  failedJobsHistoryLimit: {{ .Values.dbCleanup.failedJobsHistoryLimit }}
+  {{- end }}
+  {{- if not (eq .Values.dbCleanup.successfulJobsHistoryLimit nil) }}
+  successfulJobsHistoryLimit: {{ .Values.dbCleanup.successfulJobsHistoryLimit }}
+  {{- end }}
+  jobTemplate:
+    spec:
+      backoffLimit: 1
+      template:
+        metadata:
+          labels:
+            tier: airflow
+            component: airflow-db-cleanup
+            release: {{ .Release.Name }}
+            {{- if or (.Values.labels) (.Values.dbCleanup.labels) }}
+              {{- mustMerge .Values.dbCleanup.labels .Values.labels | toYaml | nindent 12 }}
+            {{- end }}
+          annotations:
+            sidecar.istio.io/inject: "false"
+            {{- if .Values.airflowPodAnnotations }}
+              {{- toYaml .Values.airflowPodAnnotations | nindent 12 }}
+            {{- end }}
+            {{- if .Values.dbCleanup.podAnnotations }}
+              {{- toYaml .Values.dbCleanup.podAnnotations | nindent 12 }}
+            {{- end }}
+        spec:
+          restartPolicy: Never
+          automountServiceAccountToken: false
+          {{- if .Values.dbCleanup.priorityClassName }}
+          priorityClassName: {{ .Values.dbCleanup.priorityClassName }}
+          {{- end }}
+          nodeSelector: {{- toYaml $nodeSelector | nindent 12 }}
+          affinity: {{- toYaml $affinity | nindent 12 }}
+          {{- if .Values.schedulerName }}
+          schedulerName: {{ .Values.schedulerName }}
+          {{- end }}
+          tolerations: {{- toYaml $tolerations | nindent 12 }}
+          topologySpreadConstraints: {{- toYaml $topologySpreadConstraints | nindent 12 }}
+          imagePullSecrets: {{- include "image_pull_secrets" . | nindent 12 }}
+          securityContext: {{ $securityContext | nindent 12 }}
+          containers:
+            - name: airflow-db-cleanup
+              image: {{ template "airflow_image" . }}
+              imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
+              securityContext: {{ $containerSecurityContext | nindent 16 }}
+              {{- if .Values.dbCleanup.command }}
+              command: {{ tpl (toYaml .Values.dbCleanup.command) . | nindent 16 }}
+              {{- end }}
+              {{- if .Values.dbCleanup.args }}
+              args: {{ tpl (toYaml .Values.dbCleanup.args) . | nindent 16 }}
+              {{- end }}
+              env:
+                - name: PYTHONUNBUFFERED
+                  value: "1"
+                {{- include "standard_airflow_environment" . | indent 14 }}
+                {{- include "container_extra_envs" (list . .Values.dbCleanup.env) | indent 14 }}
+              volumeMounts:
+                {{- include "airflow_config_mount" . | nindent 16 }}
+                {{- if .Values.volumeMounts }}
+                  {{- toYaml .Values.volumeMounts | nindent 16 }}
+                {{- end }}
+                {{- if .Values.dbCleanup.extraVolumeMounts }}
+                  {{- tpl (toYaml .Values.dbCleanup.extraVolumeMounts) . | nindent 16 }}
+                {{- end }}
+              resources: {{- toYaml .Values.dbCleanup.resources | nindent 16 }}
+          volumes:
+            - name: config
+              configMap:
+                name: {{ template "airflow_config" . }}
+            {{- if .Values.volumes }}
+              {{- toYaml .Values.volumes | nindent 12 }}
+            {{- end }}
+            {{- if .Values.dbCleanup.extraVolumes }}
+              {{- tpl (toYaml .Values.dbCleanup.extraVolumes) . | nindent 12 }}
+            {{- end }}
+{{- end }}

chart/templates/db-cleanup/db-cleanup-serviceaccount.yaml

@@ -0,0 +1,41 @@
+{{/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied.  See the License for the
+ specific language governing permissions and limitations
+ under the License.
+*/}}
+
+################################
+## Airflow DB Cleanup ServiceAccount
+#################################
+{{- if and .Values.dbCleanup.serviceAccount.create .Values.dbCleanup.enabled }}
+apiVersion: v1
+kind: ServiceAccount
+automountServiceAccountToken: {{ .Values.dbCleanup.serviceAccount.automountServiceAccountToken }}
+metadata:
+  name: {{ include "dbCleanup.serviceAccountName" . }}
+  labels:
+    tier: airflow
+    component: airflow-db-cleanup
+    release: {{ .Release.Name }}
+    chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+    heritage: {{ .Release.Service }}
+    {{- if or (.Values.labels) (.Values.dbCleanup.labels) }}
+      {{- mustMerge .Values.dbCleanup.labels .Values.labels | toYaml | nindent 4 }}
+    {{- end }}
+  {{- with .Values.dbCleanup.serviceAccount.annotations }}
+  annotations: {{- toYaml . | nindent 4 }}
+  {{- end }}
+{{- end }}