Skip to content

fix(helm): dagprocessor livenessProbe include --local and --job-type args #32426

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
potiuk merged 5 commits into from
Jul 9, 2023
Merged

fix(helm): dagprocessor livenessProbe include --local and --job-type args #32426

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
ff07ce5
fix: dagprocessor livenessProbe include --local and --job-type args
detvdl Jul 7, 2023
c605f11
test: add helm test for versioned dagProcessor livenessProbe
detvdl Jul 7, 2023
1e464d8
test: add versioned livenessProbe tests for scheduler & triggerer
detvdl Jul 7, 2023
0ad2e88
Merge branch 'main' into fix/standalone-dag-processor-probe
detvdl Jul 7, 2023
5fc9961
test: fix static checks
Jul 7, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 11 additions & 2 deletions chart/templates/_helpers.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -690,16 +690,25 @@ server_tls_key_file = /etc/pgbouncer/server.key
- -c
- |
CONNECTION_CHECK_MAX_COUNT=0 AIRFLOW__LOGGING__LOGGING_LEVEL=ERROR exec /entrypoint \
airflow jobs check --job-type TriggererJob --hostname $(hostname)
airflow jobs check --job-type TriggererJob --hostname $(hostname)
{{- end }}
{{- end }}

{{- define "dag_processor_liveness_check_command" }}
{{- $commandArgs := (list) -}}
{{- if semverCompare ">=2.5.0" .Values.airflowVersion }}
{{- $commandArgs = append $commandArgs "--local" -}}
{{- if semverCompare ">=2.5.2" .Values.airflowVersion }}
{{- $commandArgs = concat $commandArgs (list "--job-type" "DagProcessorJob") -}}
{{- end }}
{{- else }}
{{- $commandArgs = concat $commandArgs (list "--hostname" "$(hostname)") -}}
{{- end }}
- sh
- -c
- |
CONNECTION_CHECK_MAX_COUNT=0 AIRFLOW__LOGGING__LOGGING_LEVEL=ERROR exec /entrypoint \
airflow jobs check --hostname $(hostname)
airflow jobs check {{ join " " $commandArgs }}
{{- end }}

{{- define "registry_docker_config" }}
Expand Down
18 changes: 18 additions & 0 deletions helm_tests/airflow_core/test_dag_processor.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -338,6 +338,24 @@ def test_livenessprobe_values_are_configurable(self):
"spec.template.spec.containers[0].livenessProbe.exec.command", docs[0]
)

@pytest.mark.parametrize(
"airflow_version, probe_command",
[
("2.4.9", "airflow jobs check --hostname $(hostname)"),
("2.5.0", "airflow jobs check --local"),
("2.5.2", "airflow jobs check --local --job-type DagProcessorJob"),
],
)
def test_livenessprobe_command_depends_on_airflow_version(self, airflow_version, probe_command):
docs = render_chart(
values={"airflowVersion": f"{airflow_version}", "dagProcessor": {"enabled": True}},
show_only=["templates/dag-processor/dag-processor-deployment.yaml"],
)
assert (
probe_command
in jmespath.search("spec.template.spec.containers[0].livenessProbe.exec.command", docs[0])[-1]
)

@pytest.mark.parametrize(
"log_persistence_values, expected_volume",
[
Expand Down
18 changes: 18 additions & 0 deletions helm_tests/airflow_core/test_scheduler.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -357,6 +357,24 @@ def test_livenessprobe_values_are_configurable(self):
"spec.template.spec.containers[0].livenessProbe.exec.command", docs[0]
)

@pytest.mark.parametrize(
"airflow_version, probe_command",
[
("1.9.0", "from airflow.jobs.scheduler_job import SchedulerJob"),
("2.1.0", "airflow jobs check --job-type SchedulerJob --hostname $(hostname)"),
("2.5.0", "airflow jobs check --job-type SchedulerJob --local"),
],
)
def test_livenessprobe_command_depends_on_airflow_version(self, airflow_version, probe_command):
docs = render_chart(
values={"airflowVersion": f"{airflow_version}"},
show_only=["templates/scheduler/scheduler-deployment.yaml"],
)
assert (
probe_command
in jmespath.search("spec.template.spec.containers[0].livenessProbe.exec.command", docs[0])[-1]
)

@pytest.mark.parametrize(
"log_persistence_values, expected_volume",
[
Expand Down
17 changes: 17 additions & 0 deletions helm_tests/airflow_core/test_triggerer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -359,6 +359,23 @@ def test_livenessprobe_values_are_configurable(self):
"spec.template.spec.containers[0].livenessProbe.exec.command", docs[0]
)

@pytest.mark.parametrize(
"airflow_version, probe_command",
[
("2.4.9", "airflow jobs check --job-type TriggererJob --hostname $(hostname)"),
("2.5.0", "airflow jobs check --job-type TriggererJob --local"),
],
)
def test_livenessprobe_command_depends_on_airflow_version(self, airflow_version, probe_command):
docs = render_chart(
values={"airflowVersion": f"{airflow_version}"},
show_only=["templates/triggerer/triggerer-deployment.yaml"],
)
assert (
probe_command
in jmespath.search("spec.template.spec.containers[0].livenessProbe.exec.command", docs[0])[-1]
)

@pytest.mark.parametrize(
"log_persistence_values, expected_volume",
[
Expand Down