Add database cleanup cronjob to clean old records

chart/templates/_helpers.yaml

@@ -760,6 +760,11 @@ server_tls_key_file = /etc/pgbouncer/server.key
   {{- include "_serviceAccountName" (merge (dict "key" "cleanup") .) -}}
 {{- end }}
 
+{{/* Create the name of the database cleanup service account to use */}}
+{{- define "databaseCleanup.serviceAccountName" -}}
+  {{- include "_serviceAccountName" (merge (dict "key" "databaseCleanup" "nameSuffix" "database-cleanup") .) -}}
+{{- end }}
+
 {{- define "wait-for-migrations-command" -}}
   {{- if semverCompare ">=2.0.0" .Values.airflowVersion }}
   - airflow

chart/templates/database-cleanup/database-cleanup-cronjob.yaml

@@ -0,0 +1,120 @@
+{{/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied.  See the License for the
+ specific language governing permissions and limitations
+ under the License.
+*/}}
+
+################################
+## Airflow Database Cleanup CronJob
+#################################
+{{- if and .Values.databaseCleanup.enabled (or .Values.databaseCleanup.command .Values.databaseCleanup.args) }}
+{{- $nodeSelector := or .Values.databaseCleanup.nodeSelector .Values.nodeSelector }}
+{{- $affinity := or .Values.databaseCleanup.affinity .Values.affinity }}
+{{- $tolerations := or .Values.databaseCleanup.tolerations .Values.tolerations }}
+{{- $topologySpreadConstraints := or .Values.databaseCleanup.topologySpreadConstraints .Values.topologySpreadConstraints }}
+{{- $securityContext := include "airflowPodSecurityContext" (list . .Values.databaseCleanup) }}
+{{- $containerSecurityContext := include "containerSecurityContext" (list . .Values.databaseCleanup) }}
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: {{ include "airflow.fullname" . }}-database-cleanup
+  labels:
+    tier: airflow
+    component: database-cleanup
+    release: {{ .Release.Name }}
+    chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+    heritage: {{ .Release.Service }}
+    {{- with .Values.labels }}
+      {{- toYaml . | nindent 4 }}
+    {{- end }}
+  {{- with .Values.databaseCleanup.jobAnnotations }}
+  annotations: {{- toYaml . | nindent 4 }}
+  {{- end }}
+spec:
+  schedule: "{{ tpl .Values.databaseCleanup.schedule . }}"
+  # The cron job does not allow concurrent runs; if it is time for a new job run and the previous job run hasn't finished yet, the cron job skips the new job run
+  concurrencyPolicy: Forbid
+  {{- if not ( eq .Values.databaseCleanup.failedJobsHistoryLimit nil) }}
+  failedJobsHistoryLimit: {{ .Values.databaseCleanup.failedJobsHistoryLimit }}
+  {{- end }}
+  {{- if not (eq .Values.databaseCleanup.successfulJobsHistoryLimit nil) }}
+  successfulJobsHistoryLimit: {{ .Values.databaseCleanup.successfulJobsHistoryLimit }}
+  {{- end }}
+  jobTemplate:
+    spec:
+      backoffLimit: 1
+      template:
+        metadata:
+          labels:
+            tier: airflow
+            component: database-cleanup
+            release: {{ .Release.Name }}
+            {{- if or (.Values.labels) (.Values.databaseCleanup.labels) }}
+              {{- mustMerge .Values.databaseCleanup.labels .Values.labels | toYaml | nindent 12 }}
+            {{- end }}
+          annotations:
+            {{- if .Values.airflowPodAnnotations }}
+              {{- toYaml .Values.airflowPodAnnotations | nindent 12 }}
+            {{- end }}
+            {{- if .Values.databaseCleanup.podAnnotations }}
+              {{- toYaml .Values.databaseCleanup.podAnnotations | nindent 12 }}
+            {{- end }}
+        spec:
+          restartPolicy: Never
+          {{- if .Values.databaseCleanup.priorityClassName }}
+          priorityClassName: {{ .Values.databaseCleanup.priorityClassName }}
+          {{- end }}
+          nodeSelector: {{- toYaml $nodeSelector | nindent 12 }}
+          affinity: {{- toYaml $affinity | nindent 12 }}
+          {{- if .Values.schedulerName }}
+          schedulerName: {{ .Values.schedulerName }}
+          {{- end }}
+          tolerations: {{- toYaml $tolerations | nindent 12 }}
+          topologySpreadConstraints: {{- toYaml $topologySpreadConstraints | nindent 12 }}
+          serviceAccountName: {{ include "databaseCleanup.serviceAccountName" . }}
+          {{- if or .Values.registry.secretName .Values.registry.connection }}
+          imagePullSecrets:
+            - name: {{ template "registry_secret" . }}
+          {{- end }}
+          securityContext: {{ $securityContext | nindent 12 }}
+          containers:
+            - name: database-cleanup
+              image: {{ template "airflow_image" . }}
+              imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
+              securityContext: {{ $containerSecurityContext | nindent 16 }}
+              {{- if .Values.databaseCleanup.command }}
+              command: {{ tpl (toYaml .Values.databaseCleanup.command) . | nindent 16 }}
+              {{- end }}
+              {{- if .Values.databaseCleanup.args }}
+              args: {{ tpl (toYaml .Values.databaseCleanup.args) . | nindent 16 }}
+              {{- end }}
+              env:
+                {{- include "standard_airflow_environment" . | indent 12 }}
+                {{- include "container_extra_envs" (list . .Values.databaseCleanup.env) | indent 12 }}
+              volumeMounts:
+                {{- include "airflow_config_mount" . | nindent 16 }}
+                {{- if .Values.volumeMounts }}
+                  {{- toYaml .Values.volumeMounts | nindent 16 }}
+                {{- end }}
+              resources: {{- toYaml .Values.databaseCleanup.resources | nindent 16 }}
+          volumes:
+            - name: config
+              configMap:
+                name: {{ template "airflow_config" . }}
+            {{- if .Values.volumes }}
+              {{- toYaml .Values.volumes | nindent 12 }}
+            {{- end }}
+{{- end }}

chart/templates/database-cleanup/database-cleanup-serviceaccount.yaml

@@ -0,0 +1,41 @@
+{{/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied.  See the License for the
+ specific language governing permissions and limitations
+ under the License.
+*/}}
+
+################################
+## Airflow Database Cleanup ServiceAccount
+#################################
+{{- if and .Values.databaseCleanup.serviceAccount.create .Values.databaseCleanup.enabled }}
+apiVersion: v1
+kind: ServiceAccount
+automountServiceAccountToken: {{ .Values.databaseCleanup.serviceAccount.automountServiceAccountToken }}
+metadata:
+  name: {{ include "databaseCleanup.serviceAccountName" . }}
+  labels:
+    tier: airflow
+    component: database-cleanup
+    release: {{ .Release.Name }}
+    chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+    heritage: {{ .Release.Service }}
+    {{- if or (.Values.labels) (.Values.databaseCleanup.labels) }}
+      {{- mustMerge .Values.databaseCleanup.labels .Values.labels | toYaml | nindent 4 }}
+    {{- end }}
+  {{- with .Values.databaseCleanup.serviceAccount.annotations }}
+  annotations: {{- toYaml . | nindent 4 }}
+  {{- end }}
+{{- end }}

chart/templates/rbac/pod-database-cleanup-role.yaml

@@ -0,0 +1,37 @@
+{{/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied.  See the License for the
+ specific language governing permissions and limitations
+ under the License.
+*/}}
+
+################################
+## Airflow Database Cleanup Role
+#################################
+{{- if and .Values.rbac.create .Values.databaseCleanup.enabled }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: {{ include "airflow.fullname" . }}-database-cleanup-role
+  labels:
+    tier: airflow
+    release: {{ .Release.Name }}
+    chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+    heritage: {{ .Release.Service }}
+    {{- with .Values.labels }}
+      {{- toYaml . | nindent 4 }}
+    {{- end }}
+rules: []
+{{- end }}

chart/templates/rbac/pod-database-cleanup-rolebinding.yaml

@@ -0,0 +1,44 @@
+{{/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied.  See the License for the
+ specific language governing permissions and limitations
+ under the License.
+*/}}
+
+################################
+## Airflow Database Cleanup Role Binding
+#################################
+{{- if and .Values.rbac.create .Values.databaseCleanup.enabled }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: {{ include "airflow.fullname" . }}-database-cleanup-rolebinding
+  labels:
+    tier: airflow
+    release: {{ .Release.Name }}
+    chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+    heritage: {{ .Release.Service }}
+    {{- with .Values.labels }}
+      {{- toYaml . | nindent 4 }}
+    {{- end }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ include "airflow.fullname" . }}-database-cleanup-role
+subjects:
+  - kind: ServiceAccount
+    name: {{ include "databaseCleanup.serviceAccountName" . }}
+    namespace: "{{ .Release.Namespace }}"
+{{- end }}

chart/templates/rbac/security-context-constraint-rolebinding.yaml

@@ -107,6 +107,11 @@ subjects:
     name: {{ include "cleanup.serviceAccountName" . }}
     namespace: "{{ .Release.Namespace }}"
   {{- end }}
+  {{- if .Values.databaseCleanup.enabled }}
+  - kind: ServiceAccount
+    name: {{ include "databaseCleanup.serviceAccountName" . }}
+    namespace: "{{ .Release.Namespace }}"
+  {{- end }}
   {{- if .Values.dagProcessor.enabled }}
   - kind: ServiceAccount
     name: {{ include "dagProcessor.serviceAccountName" . }}