Skip to content

helm: add webserver_config.py to api server #50432

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
GrumpyCat51 wants to merge 8 commits into from
Closed

helm: add webserver_config.py to api server #50432

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
d0f74c0
add webserver_config.py to api server
May 10, 2025
b586bac
Update test_webserver.py
May 10, 2025
7ac60da
fix test, standardize some names
May 12, 2025
945d62f
linting fix
May 12, 2025
4c258f1
add config changes to all other deployments
May 16, 2025
71efb59
fix tests
May 18, 2025
25218bd
fix pod template tests
May 18, 2025
1a1e072
Merge branch 'main' into add-webserver-config-to-api-server
GrumpyCat51 May 18, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 13 additions & 3 deletions chart/files/pod-template-file.kubernetes-helm-yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,8 +79,10 @@ spec:
{{- if .Values.workers.extraVolumeMounts }}
{{- tpl (toYaml .Values.workers.extraVolumeMounts) . | nindent 8 }}
{{- end }}
{{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }}
{{- if and (semverCompare "<3.0.0" .Values.airflowVersion) (or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName) }}
{{- include "airflow_webserver_config_mount" . | nindent 8 }}
{{- else if and (semverCompare ">=3.0.0" .Values.airflowVersion) (or .Values.apiServer.apiServerConfig .Values.apiServer.apiServerConfigConfigMapName) }}
{{- include "airflow_api_server_config_mount" . | nindent 8 }}
{{- end }}
envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 6 }}
env:
Expand Down Expand Up @@ -173,8 +175,10 @@ spec:
{{- if .Values.workers.extraVolumeMounts }}
{{- tpl (toYaml .Values.workers.extraVolumeMounts) . | nindent 8 }}
{{- end }}
{{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }}
{{- if and (semverCompare "<3.0.0" .Values.airflowVersion) (or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName) }}
{{- include "airflow_webserver_config_mount" . | nindent 8 }}
{{- else if and (semverCompare ">=3.0.0" .Values.airflowVersion) (or .Values.apiServer.apiServerConfig .Values.apiServer.apiServerConfigConfigMapName) }}
{{- include "airflow_api_server_config_mount" . | nindent 8 }}
{{- end }}
envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 6 }}
env:
Expand Down Expand Up @@ -235,10 +239,16 @@ spec:
- configMap:
name: {{ include "airflow_config" . }}
name: config
{{- if and (or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName) (or .Values.workers.kerberosInitContainer.enabled .Values.workers.kerberosSidecar.enabled)}}
{{- if (or .Values.workers.kerberosInitContainer.enabled .Values.workers.kerberosSidecar.enabled) }}
{{- if and (semverCompare "<3.0.0" .Values.airflowVersion) (or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName) }}
- name: webserver-config
configMap:
name: {{ template "airflow_webserver_config_configmap_name" . }}
{{- else if and (semverCompare ">=3.0.0" .Values.airflowVersion) (or .Values.apiServer.apiServerConfig .Values.apiServer.apiServerConfigConfigMapName) }}
- name: api-server-config
configMap:
name: {{ template "airflow_api_server_config_configmap_name" . }}
{{- end }}
{{- end }}
{{- if .Values.volumes }}
{{- toYaml .Values.volumes | nindent 2 }}
Expand Down
15 changes: 15 additions & 0 deletions chart/templates/_helpers.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -571,17 +571,32 @@ server_tls_key_file = /etc/pgbouncer/server.key
{{- printf "%s/webserver_config.py" .Values.airflowHome | quote }}
{{- end }}

{{- define "airflow_api_server_config_path" -}}
{{- printf "%s/webserver_config.py" .Values.airflowHome | quote }}
{{- end }}

{{- define "airflow_webserver_config_configmap_name" -}}
{{- default (printf "%s-webserver-config" .Release.Name) .Values.webserver.webserverConfigConfigMapName }}
{{- end }}

{{- define "airflow_api_server_config_configmap_name" -}}
{{- default (printf "%s-api-server-config" .Release.Name) .Values.apiServer.apiServerConfigConfigMapName }}
{{- end }}

{{- define "airflow_webserver_config_mount" -}}
- name: webserver-config
mountPath: {{ template "airflow_webserver_config_path" . }}
subPath: webserver_config.py
readOnly: True
{{- end }}

{{- define "airflow_api_server_config_mount" -}}
- name: api-server-config
mountPath: {{ template "airflow_api_server_config_path" . }}
subPath: webserver_config.py
readOnly: True
{{- end }}

{{- define "airflow_local_setting_path" -}}
{{- printf "%s/config/airflow_local_settings.py" .Values.airflowHome | quote }}
{{- end }}
Expand Down
8 changes: 7 additions & 1 deletion chart/templates/api-server/api-server-deployment.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -170,6 +170,9 @@ spec:
resources: {{- toYaml .Values.apiServer.resources | nindent 12 }}
volumeMounts:
{{- include "airflow_config_mount" . | nindent 12 }}
{{- if or .Values.apiServer.apiServerConfig .Values.apiServer.apiServerConfigConfigMapName }}
Copy link
Contributor

@nhuantho nhuantho May 14, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think you can integrate this logic into the schedule, workers, and trigger deployment.

Copy link
Contributor

@nhuantho nhuantho May 14, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

example in schedule depoyment:
{{- if or .Values.apiServer.apiServerConfig .Values.apiServer.apiServerConfigConfigMapName (semverCompare ">=3.0.0" .Values.airflowVersion) }}
{{- include "airflow_api_server_config_mount" . | nindent 12 }}
{{- end }}

Copy link
Contributor Author

@GrumpyCat51 GrumpyCat51 May 14, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

But will it have an actual effect on the scheduler/worker/triggerer, or is it just making the chart unnecessarily more complex?

Copy link
Contributor

@albundy83 albundy83 May 14, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In version 2.10.5, those are mounted also (here for example) but I don't think there are really needed.

Maybe you should not include them ?

Copy link
Contributor

@nhuantho nhuantho May 14, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, I checked gain, so it is unnecessary. Thanks!

{{- include "airflow_api_server_config_mount" . | nindent 12 }}
{{- end }}
{{- if .Values.logs.persistence.enabled }}
- name: logs
mountPath: {{ template "airflow_logs" . }}
Expand Down Expand Up @@ -225,7 +228,10 @@ spec:
- name: config
configMap:
name: {{ template "airflow_config" . }}
{{- if (semverCompare "<2.0.0" .Values.airflowVersion) }}
{{- if or .Values.apiServer.apiServerConfig .Values.apiServer.apiServerConfigConfigMapName }}
- name: api-server-config
configMap:
name: {{ template "airflow_api_server_config_configmap_name" . }}
{{- end }}
{{- if .Values.logs.persistence.enabled }}
- name: logs
Expand Down
23 changes: 22 additions & 1 deletion chart/templates/configmaps/webserver-configmap.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
################################
## Airflow ConfigMap
#################################
{{- if and .Values.webserver.webserverConfig (not .Values.webserver.webserverConfigConfigMapName) }}
{{- if and .Values.webserver.webserverConfig (not .Values.webserver.webserverConfigConfigMapName) (semverCompare "<3.0.0" .Values.airflowVersion) }}
apiVersion: v1
kind: ConfigMap
metadata:
Expand All @@ -41,4 +41,25 @@ metadata:
data:
webserver_config.py: |-
{{- tpl .Values.webserver.webserverConfig . | nindent 4 }}
{{- else if and .Values.apiServer.apiServerConfig (not .Values.apiServer.apiServerConfigConfigMapName) (semverCompare ">=3.0.0" .Values.airflowVersion) }}
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ template "airflow_api_server_config_configmap_name" . }}
labels:
tier: airflow
component: config
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
{{- with .Values.apiServer.configMapAnnotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
data:
webserver_config.py: |-
{{- tpl .Values.apiServer.apiServerConfig . | nindent 4 }}
{{- end }}
10 changes: 8 additions & 2 deletions chart/templates/dag-processor/dag-processor-deployment.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -233,8 +233,10 @@ spec:
{{- if .Values.dagProcessor.extraVolumeMounts }}
{{- tpl (toYaml .Values.dagProcessor.extraVolumeMounts) . | nindent 12 }}
{{- end }}
{{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }}
{{- if and (semverCompare "<3.0.0" .Values.airflowVersion) (or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName) }}
{{- include "airflow_webserver_config_mount" . | nindent 12 }}
{{- else if and (semverCompare ">=3.0.0" .Values.airflowVersion) (or .Values.apiServer.apiServerConfig .Values.apiServer.apiServerConfigConfigMapName) }}
{{- include "airflow_api_server_config_mount" . | nindent 12 }}
{{- end }}
{{- end }}
{{- if .Values.dagProcessor.extraContainers }}
Expand All @@ -244,10 +246,14 @@ spec:
- name: config
configMap:
name: {{ template "airflow_config" . }}
{{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }}
{{- if and (semverCompare "<3.0.0" .Values.airflowVersion) (or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName) }}
- name: webserver-config
configMap:
name: {{ template "airflow_webserver_config_configmap_name" . }}
{{- else if and (semverCompare ">=3.0.0" .Values.airflowVersion) (or .Values.apiServer.apiServerConfig .Values.apiServer.apiServerConfigConfigMapName) }}
- name: api-server-config
configMap:
name: {{ template "airflow_api_server_config_configmap_name" . }}
{{- end }}
{{- if .Values.dags.persistence.enabled }}
- name: dags
Expand Down
18 changes: 14 additions & 4 deletions chart/templates/scheduler/scheduler-deployment.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -160,8 +160,10 @@ spec:
{{- if .Values.scheduler.extraVolumeMounts }}
{{- tpl (toYaml .Values.scheduler.extraVolumeMounts) . | nindent 12 }}
{{- end }}
{{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }}
{{- if and (semverCompare "<3.0.0" .Values.airflowVersion) (or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName) }}
{{- include "airflow_webserver_config_mount" . | nindent 12 }}
{{- else if and (semverCompare ">=3.0.0" .Values.airflowVersion) (or .Values.apiServer.apiServerConfig .Values.apiServer.apiServerConfigConfigMapName) }}
{{- include "airflow_api_server_config_mount" . | nindent 12 }}
{{- end }}
args: {{- include "wait-for-migrations-command" . | indent 10 }}
envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 10 }}
Expand Down Expand Up @@ -239,8 +241,10 @@ spec:
- name: logs
mountPath: {{ template "airflow_logs" . }}
{{- include "airflow_config_mount" . | nindent 12 }}
{{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }}
{{- if and (semverCompare "<3.0.0" .Values.airflowVersion) (or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName) }}
{{- include "airflow_webserver_config_mount" . | nindent 12 }}
{{- else if and (semverCompare ">=3.0.0" .Values.airflowVersion) (or .Values.apiServer.apiServerConfig .Values.apiServer.apiServerConfigConfigMapName) }}
{{- include "airflow_api_server_config_mount" . | nindent 12 }}
{{- end }}
{{- if and $localOrDagProcessorDisabled (or .Values.dags.persistence.enabled .Values.dags.gitSync.enabled) }}
{{- include "airflow_dags_mount" . | nindent 12 }}
Expand Down Expand Up @@ -292,8 +296,10 @@ spec:
{{- if .Values.scheduler.extraVolumeMounts }}
{{- tpl (toYaml .Values.scheduler.extraVolumeMounts) . | nindent 12 }}
{{- end }}
{{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }}
{{- if and (semverCompare "<3.0.0" .Values.airflowVersion) (or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName) }}
{{- include "airflow_webserver_config_mount" . | nindent 12 }}
{{- else if and (semverCompare ">=3.0.0" .Values.airflowVersion) (or .Values.apiServer.apiServerConfig .Values.apiServer.apiServerConfigConfigMapName) }}
{{- include "airflow_api_server_config_mount" . | nindent 12 }}
{{- end }}
{{- end }}
{{- if .Values.scheduler.extraContainers }}
Expand All @@ -303,10 +309,14 @@ spec:
- name: config
configMap:
name: {{ template "airflow_config" . }}
{{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }}
{{- if and (semverCompare "<3.0.0" .Values.airflowVersion) (or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName) }}
- name: webserver-config
configMap:
name: {{ template "airflow_webserver_config_configmap_name" . }}
{{- else if and (semverCompare ">=3.0.0" .Values.airflowVersion) (or .Values.apiServer.apiServerConfig .Values.apiServer.apiServerConfigConfigMapName) }}
- name: api-server-config
configMap:
name: {{ template "airflow_api_server_config_configmap_name" . }}
{{- end }}
{{- if $localOrDagProcessorDisabled }}
{{- if .Values.dags.persistence.enabled }}
Expand Down
18 changes: 14 additions & 4 deletions chart/templates/triggerer/triggerer-deployment.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -150,8 +150,10 @@ spec:
{{- if .Values.triggerer.extraVolumeMounts }}
{{- tpl (toYaml .Values.triggerer.extraVolumeMounts) . | nindent 12 }}
{{- end }}
{{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }}
{{- if and (semverCompare "<3.0.0" .Values.airflowVersion) (or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName) }}
{{- include "airflow_webserver_config_mount" . | nindent 12 }}
{{- else if and (semverCompare ">=3.0.0" .Values.airflowVersion) (or .Values.apiServer.apiServerConfig .Values.apiServer.apiServerConfigConfigMapName) }}
{{- include "airflow_api_server_config_mount" . | nindent 12 }}
{{- end }}
args: {{- include "wait-for-migrations-command" . | indent 10 }}
envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 10 }}
Expand Down Expand Up @@ -193,8 +195,10 @@ spec:
- name: logs
mountPath: {{ template "airflow_logs" . }}
{{- include "airflow_config_mount" . | nindent 12 }}
{{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }}
{{- if and (semverCompare "<3.0.0" .Values.airflowVersion) (or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName) }}
{{- include "airflow_webserver_config_mount" . | nindent 12 }}
{{- else if and (semverCompare ">=3.0.0" .Values.airflowVersion) (or .Values.apiServer.apiServerConfig .Values.apiServer.apiServerConfigConfigMapName) }}
{{- include "airflow_api_server_config_mount" . | nindent 12 }}
{{- end }}
{{- if or .Values.dags.persistence.enabled .Values.dags.gitSync.enabled }}
{{- include "airflow_dags_mount" . | nindent 12 }}
Expand Down Expand Up @@ -263,8 +267,10 @@ spec:
{{- if .Values.triggerer.extraVolumeMounts }}
{{- tpl (toYaml .Values.triggerer.extraVolumeMounts) . | nindent 12 }}
{{- end }}
{{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }}
{{- if and (semverCompare "<3.0.0" .Values.airflowVersion) (or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName) }}
{{- include "airflow_webserver_config_mount" . | nindent 12 }}
{{- else if and (semverCompare ">=3.0.0" .Values.airflowVersion) (or .Values.apiServer.apiServerConfig .Values.apiServer.apiServerConfigConfigMapName) }}
{{- include "airflow_api_server_config_mount" . | nindent 12 }}
{{- end }}
{{- end }}
{{- if .Values.triggerer.extraContainers }}
Expand All @@ -274,10 +280,14 @@ spec:
- name: config
configMap:
name: {{ template "airflow_config" . }}
{{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }}
{{- if and (semverCompare "<3.0.0" .Values.airflowVersion) (or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName) }}
- name: webserver-config
configMap:
name: {{ template "airflow_webserver_config_configmap_name" . }}
{{- else if and (semverCompare ">=3.0.0" .Values.airflowVersion) (or .Values.apiServer.apiServerConfig .Values.apiServer.apiServerConfigConfigMapName) }}
- name: api-server-config
configMap:
name: {{ template "airflow_api_server_config_configmap_name" . }}
{{- end }}
{{- if .Values.dags.persistence.enabled }}
- name: dags
Expand Down
6 changes: 5 additions & 1 deletion chart/templates/webserver/webserver-deployment.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -267,10 +267,14 @@ spec:
- name: config
configMap:
name: {{ template "airflow_config" . }}
{{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }}
{{- if and (semverCompare "<3.0.0" .Values.airflowVersion) (or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName) }}
- name: webserver-config
configMap:
name: {{ template "airflow_webserver_config_configmap_name" . }}
{{- else if and (semverCompare ">=3.0.0" .Values.airflowVersion) (or .Values.apiServer.apiServerConfig .Values.apiServer.apiServerConfigConfigMapName) }}
- name: api-server-config
configMap:
name: {{ template "airflow_api_server_config_configmap_name" . }}
{{- end }}
{{- if (semverCompare "<2.0.0" .Values.airflowVersion) }}
{{- if .Values.dags.persistence.enabled }}
Expand Down
Loading