Skip to content

Extra tests for db cleanup and fix cronjob image pull secrets #58626

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
potiuk merged 1 commit into from
Nov 25, 2025
Merged

Extra tests for db cleanup and fix cronjob image pull secrets #58626

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 1 addition & 4 deletions chart/templates/database-cleanup/database-cleanup-cronjob.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,10 +85,7 @@ spec:
tolerations: {{- toYaml $tolerations | nindent 12 }}
topologySpreadConstraints: {{- toYaml $topologySpreadConstraints | nindent 12 }}
serviceAccountName: {{ include "databaseCleanup.serviceAccountName" . }}
{{- if or .Values.registry.secretName .Values.registry.connection }}
imagePullSecrets:
- name: {{ template "registry_secret" . }}
{{- end }}
imagePullSecrets: {{- include "image_pull_secrets" . | nindent 12 }}
securityContext: {{ $securityContext | nindent 12 }}
containers:
- name: database-cleanup
Expand Down
20 changes: 14 additions & 6 deletions helm-tests/tests/helm_tests/airflow_aux/test_airflow_common.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -194,6 +194,7 @@ def test_annotations(self):
values={
"airflowPodAnnotations": {"test-annotation/safe-to-evict": "true"},
"cleanup": {"enabled": True},
"databaseCleanup": {"enabled": True},
"flower": {"enabled": True},
"dagProcessor": {"enabled": True},
},
Expand All @@ -206,11 +207,12 @@ def test_annotations(self):
"templates/triggerer/triggerer-deployment.yaml",
"templates/dag-processor/dag-processor-deployment.yaml",
"templates/cleanup/cleanup-cronjob.yaml",
"templates/database-cleanup/database-cleanup-cronjob.yaml",
],
)

# Objects in show_only are 8 but only one of Webserver or API server is created so we have 7 objects
assert len(k8s_objects) == 7
# Objects in show_only are 9 but only one of Webserver or API server is created so we have 8 objects
assert len(k8s_objects) == 8

for k8s_object in k8s_objects:
if k8s_object["kind"] == "CronJob":
Expand All @@ -226,6 +228,7 @@ def test_global_affinity_tolerations_topology_spread_constraints_and_node_select
k8s_objects = render_chart(
values={
"cleanup": {"enabled": True},
"databaseCleanup": {"enabled": True},
"flower": {"enabled": True},
"pgbouncer": {"enabled": True},
"dagProcessor": {"enabled": True},
Expand Down Expand Up @@ -257,6 +260,7 @@ def test_global_affinity_tolerations_topology_spread_constraints_and_node_select
},
show_only=[
"templates/cleanup/cleanup-cronjob.yaml",
"templates/database-cleanup/database-cleanup-cronjob.yaml",
"templates/flower/flower-deployment.yaml",
"templates/jobs/create-user-job.yaml",
"templates/jobs/migrate-database-job.yaml",
Expand All @@ -272,8 +276,8 @@ def test_global_affinity_tolerations_topology_spread_constraints_and_node_select
],
)

# Objects in show_only are 13 but only one of Webserver or API server is created so we have 12 objects
assert len(k8s_objects) == 12
# Objects in show_only are 14 but only one of Webserver or API server is created so we have 13 objects
assert len(k8s_objects) == 13

for k8s_object in k8s_objects:
if k8s_object["kind"] == "CronJob":
Expand Down Expand Up @@ -464,6 +468,7 @@ def test_priority_class_name(self):
"webserver": {"priorityClassName": "low-priority-webserver"},
"workers": {"priorityClassName": "low-priority-worker"},
"cleanup": {"enabled": True, "priorityClassName": "low-priority-airflow-cleanup-pods"},
"databaseCleanup": {"enabled": True, "priorityClassName": "low-priority-database-cleanup"},
"migrateDatabaseJob": {"priorityClassName": "low-priority-run-airflow-migrations"},
"createUserJob": {"priorityClassName": "low-priority-create-user-job"},
},
Expand All @@ -477,15 +482,16 @@ def test_priority_class_name(self):
"templates/webserver/webserver-deployment.yaml",
"templates/workers/worker-deployment.yaml",
"templates/cleanup/cleanup-cronjob.yaml",
"templates/database-cleanup/database-cleanup-cronjob.yaml",
"templates/jobs/migrate-database-job.yaml",
"templates/jobs/create-user-job.yaml",
],
)

assert len(docs) == 10
assert len(docs) == 11
for doc in docs:
component = doc["metadata"]["labels"]["component"]
if component == "airflow-cleanup-pods":
if component in ["airflow-cleanup-pods", "database-cleanup"]:
priority = doc["spec"]["jobTemplate"]["spec"]["template"]["spec"]["priorityClassName"]
else:
priority = doc["spec"]["template"]["spec"]["priorityClassName"]
Expand Down Expand Up @@ -525,6 +531,7 @@ def test_image_pull_secrets(
"flower": {"enabled": True},
"pgbouncer": {"enabled": True},
"cleanup": {"enabled": True},
"databaseCleanup": {"enabled": True},
},
show_only=[
"templates/flower/flower-deployment.yaml",
Expand All @@ -536,6 +543,7 @@ def test_image_pull_secrets(
"templates/webserver/webserver-deployment.yaml",
"templates/workers/worker-deployment.yaml",
"templates/cleanup/cleanup-cronjob.yaml",
"templates/database-cleanup/database-cleanup-cronjob.yaml",
"templates/jobs/migrate-database-job.yaml",
"templates/jobs/create-user-job.yaml",
],
Expand Down
30 changes: 30 additions & 0 deletions helm-tests/tests/helm_tests/airflow_aux/test_annotations.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,22 @@ class TestServiceAccountAnnotations:
"example": "cleanup",
},
),
(
{
"databaseCleanup": {
"enabled": True,
"serviceAccount": {
"annotations": {
"example": "database-cleanup",
},
},
},
},
"templates/database-cleanup/database-cleanup-serviceaccount.yaml",
{
"example": "database-cleanup",
},
),
(
{
"scheduler": {
Expand Down Expand Up @@ -446,6 +462,20 @@ def test_annotations_on_webserver(self):
"example": "cleanup",
},
),
(
{
"databaseCleanup": {
"enabled": True,
"podAnnotations": {
"example": "database-cleanup",
},
}
},
"templates/database-cleanup/database-cleanup-cronjob.yaml",
{
"example": "database-cleanup",
},
),
(
{
"redis": {
Expand Down
6 changes: 6 additions & 0 deletions helm-tests/tests/helm_tests/airflow_aux/test_basic_helm_chart.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -390,6 +390,7 @@ def test_labels_are_valid(self, airflow_version):
"ingress": {"enabled": True},
"networkPolicies": {"enabled": True},
"cleanup": {"enabled": True},
"databaseCleanup": {"enabled": True},
"flower": {"enabled": True},
"dagProcessor": {"enabled": True},
"logs": {"persistence": {"enabled": True}},
Expand All @@ -408,6 +409,7 @@ def test_labels_are_valid(self, airflow_version):

kind_names_tuples = [
(f"{release_name}-airflow-cleanup", "ServiceAccount", "airflow-cleanup-pods"),
(f"{release_name}-airflow-database-cleanup", "ServiceAccount", "database-cleanup"),
(f"{release_name}-config", "ConfigMap", "config"),
(f"{release_name}-airflow-create-user-job", "ServiceAccount", "create-user-job"),
(f"{release_name}-airflow-flower", "ServiceAccount", "flower"),
Expand All @@ -425,6 +427,9 @@ def test_labels_are_valid(self, airflow_version):
(f"{release_name}-cleanup", "CronJob", "airflow-cleanup-pods"),
(f"{release_name}-cleanup-role", "Role", None),
(f"{release_name}-cleanup-rolebinding", "RoleBinding", None),
(f"{release_name}-database-cleanup", "CronJob", "database-cleanup"),
(f"{release_name}-database-cleanup-role", "Role", None),
(f"{release_name}-database-cleanup-rolebinding", "RoleBinding", None),
(f"{release_name}-create-user", "Job", "create-user-job"),
(f"{release_name}-fernet-key", "Secret", None),
(f"{release_name}-flower", "Deployment", "flower"),
Expand Down Expand Up @@ -509,6 +514,7 @@ def test_labels_are_valid_on_job_templates(self):
"redis": {"enabled": True},
"networkPolicies": {"enabled": True},
"cleanup": {"enabled": True},
"databaseCleanup": {"enabled": True},
"flower": {"enabled": True},
"postgresql": {"enabled": False}, # We won't check the objects created by the postgres chart
},
Expand Down
31 changes: 31 additions & 0 deletions helm-tests/tests/helm_tests/security/test_rbac.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,22 +50,26 @@
("Job", "test-rbac-create-user"),
("Job", "test-rbac-run-airflow-migrations"),
("CronJob", "test-rbac-cleanup"),
("CronJob", "test-rbac-database-cleanup"),
]

RBAC_ENABLED_KIND_NAME_TUPLES = [
("Role", "test-rbac-pod-launcher-role"),
("Role", "test-rbac-cleanup-role"),
("Role", "test-rbac-database-cleanup-role"),
("Role", "test-rbac-pod-log-reader-role"),
("RoleBinding", "test-rbac-pod-launcher-rolebinding"),
("RoleBinding", "test-rbac-pod-log-reader-rolebinding"),
("RoleBinding", "test-rbac-cleanup-rolebinding"),
("RoleBinding", "test-rbac-database-cleanup-rolebinding"),
]

SERVICE_ACCOUNT_NAME_TUPLES = [
("ServiceAccount", "test-rbac-cleanup"),
("ServiceAccount", "test-rbac-scheduler"),
("ServiceAccount", "test-rbac-triggerer"),
("ServiceAccount", "test-rbac-pgbouncer"),
("ServiceAccount", "test-rbac-database-cleanup"),
("ServiceAccount", "test-rbac-flower"),
("ServiceAccount", "test-rbac-statsd"),
("ServiceAccount", "test-rbac-create-user-job"),
Expand All @@ -80,6 +84,7 @@
(CUSTOM_WORKER_NAME := "TestWorker"),
(CUSTOM_TRIGGERER_NAME := "TestTriggerer"),
(CUSTOM_CLEANUP_NAME := "TestCleanup"),
(CUSTOM_DATABASE_CLEANUP_NAME := "TestDatabaseCleanup"),
(CUSTOM_FLOWER_NAME := "TestFlower"),
(CUSTOM_PGBOUNCER_NAME := "TestPGBouncer"),
(CUSTOM_STATSD_NAME := "TestStatsd"),
Expand Down Expand Up @@ -174,6 +179,12 @@ def test_deployments_no_rbac_no_sa(self, version, workers_values):
"create": False,
},
},
"databaseCleanup": {
"enabled": True,
"serviceAccount": {
"create": False,
},
},
"pgbouncer": {
"enabled": True,
"serviceAccount": {
Expand Down Expand Up @@ -211,6 +222,7 @@ def test_deployments_no_rbac_with_sa(self, version, dedicated_workers_sa):
"executor": "CeleryExecutor,KubernetesExecutor",
"rbac": {"create": False},
"cleanup": {"enabled": True},
"databaseCleanup": {"enabled": True},
"flower": {"enabled": True},
"pgbouncer": {"enabled": True},
"workers": {"useWorkerDedicatedServiceAccounts": dedicated_workers_sa},
Expand Down Expand Up @@ -252,6 +264,12 @@ def test_deployments_with_rbac_no_sa(self, version, workers_values):
"create": False,
},
},
"databaseCleanup": {
"enabled": True,
"serviceAccount": {
"create": False,
},
},
"scheduler": {"serviceAccount": {"create": False}},
"dagProcessor": {"serviceAccount": {"create": False}},
"webserver": {"serviceAccount": {"create": False}},
Expand Down Expand Up @@ -289,6 +307,7 @@ def test_deployments_with_rbac_with_sa(self, version, dedicated_workers_sa):
"fullnameOverride": "test-rbac",
"executor": "CeleryExecutor,KubernetesExecutor",
"cleanup": {"enabled": True},
"databaseCleanup": {"enabled": True},
"flower": {"enabled": True},
"pgbouncer": {"enabled": True},
"workers": {"useWorkerDedicatedServiceAccounts": dedicated_workers_sa},
Expand Down Expand Up @@ -318,6 +337,12 @@ def test_service_account_custom_names(self):
"name": CUSTOM_CLEANUP_NAME,
},
},
"databaseCleanup": {
"enabled": True,
"serviceAccount": {
"name": CUSTOM_DATABASE_CLEANUP_NAME,
},
},
"scheduler": {"serviceAccount": {"name": CUSTOM_SCHEDULER_NAME}},
"dagProcessor": {"serviceAccount": {"name": CUSTOM_DAG_PROCESSOR_NAME}},
"apiServer": {"serviceAccount": {"name": CUSTOM_API_SERVER_NAME}},
Expand Down Expand Up @@ -396,6 +421,12 @@ def test_service_account_custom_names_in_objects(self):
"name": CUSTOM_CLEANUP_NAME,
},
},
"databaseCleanup": {
"enabled": True,
"serviceAccount": {
"name": CUSTOM_DATABASE_CLEANUP_NAME,
},
},
"scheduler": {"serviceAccount": {"name": CUSTOM_SCHEDULER_NAME}},
"dagProcessor": {"serviceAccount": {"name": CUSTOM_DAG_PROCESSOR_NAME}},
"apiServer": {"serviceAccount": {"name": CUSTOM_API_SERVER_NAME}},
Expand Down