Add workers.celery.livenessProbe section

chart/newsfragments/60186.significant.rst

@@ -0,0 +1,3 @@
+``workers.livenessProbe`` section is now deprecated in favor of ``workers.celery.livenessProbe``.
+
+Along the upgrade of Helm Chart version, change the configuration of values in section ``workers.livenessProbe`` to ``workers.celery.livenessProbe`` section or unset all values under ``workers.celery.livenessProbe`` to preserve previous behaviour of the Helm Chart.

chart/templates/NOTES.txt

@@ -212,6 +212,54 @@ DEPRECATION WARNING:
 
 {{- end }}
 
+{{- if not .Values.workers.livenessProbe.enabled }}
+
+ DEPRECATION WARNING:
+    `workers.livenessProbe.enabled` has been renamed to `workers.celery.livenessProbe.enabled`.
+    Please change your values as support for the old name will be dropped in a future release.
+
+{{- end }}
+
+{{- if ne (int .Values.workers.livenessProbe.initialDelaySeconds) 10 }}
+
+ DEPRECATION WARNING:
+    `workers.livenessProbe.initialDelaySeconds` has been renamed to `workers.celery.livenessProbe.initialDelaySeconds`.
+    Please change your values as support for the old name will be dropped in a future release.
+
+{{- end }}
+
+{{- if ne (int .Values.workers.livenessProbe.timeoutSeconds) 20 }}
+
+ DEPRECATION WARNING:
+    `workers.livenessProbe.timeoutSeconds` has been renamed to `workers.celery.livenessProbe.timeoutSeconds`.
+    Please change your values as support for the old name will be dropped in a future release.
+
+{{- end }}
+
+{{- if ne (int .Values.workers.livenessProbe.failureThreshold) 5 }}
+
+ DEPRECATION WARNING:
+    `workers.livenessProbe.failureThreshold` has been renamed to `workers.celery.livenessProbe.failureThreshold`.
+    Please change your values as support for the old name will be dropped in a future release.
+
+{{- end }}
+
+{{- if ne (int .Values.workers.livenessProbe.periodSeconds) 60 }}
+
+ DEPRECATION WARNING:
+    `workers.livenessProbe.periodSeconds` has been renamed to `workers.celery.livenessProbe.periodSeconds`.
+    Please change your values as support for the old name will be dropped in a future release.
+
+{{- end }}
+
+{{- if not (empty .Values.workers.livenessProbe.command) }}
+
+ DEPRECATION WARNING:
+    `workers.livenessProbe.command` has been renamed to `workers.celery.livenessProbe.command`.
+    Please change your values as support for the old name will be dropped in a future release.
+
+{{- end }}
+
 {{ if (semverCompare ">=3.0.0" .Values.airflowVersion) }}
 #####################################################
 #  WARNING: You should set a static API secret key  #

chart/templates/workers/worker-deployment.yaml

@@ -256,16 +256,16 @@ spec:
           args: {{ tpl (toYaml (.Values.workers.celery.args | default .Values.workers.args)) . | nindent 12 }}
           {{- end }}
           resources: {{- toYaml .Values.workers.resources | nindent 12 }}
-          {{- if .Values.workers.livenessProbe.enabled }}
+          {{- if or .Values.workers.celery.livenessProbe.enabled (and (not (has .Values.workers.celery.livenessProbe.enabled (list true false))) .Values.workers.livenessProbe.enabled) }}
           livenessProbe:
-            initialDelaySeconds: {{ .Values.workers.livenessProbe.initialDelaySeconds }}
-            timeoutSeconds: {{ .Values.workers.livenessProbe.timeoutSeconds }}
-            failureThreshold: {{ .Values.workers.livenessProbe.failureThreshold }}
-            periodSeconds: {{ .Values.workers.livenessProbe.periodSeconds }}
+            initialDelaySeconds: {{ .Values.workers.celery.livenessProbe.initialDelaySeconds | default .Values.workers.livenessProbe.initialDelaySeconds }}
+            timeoutSeconds: {{ .Values.workers.celery.livenessProbe.timeoutSeconds | default .Values.workers.livenessProbe.timeoutSeconds }}
+            failureThreshold: {{ .Values.workers.celery.livenessProbe.failureThreshold | default .Values.workers.livenessProbe.failureThreshold }}
+            periodSeconds: {{ .Values.workers.celery.livenessProbe.periodSeconds | default .Values.workers.livenessProbe.periodSeconds }}
             exec:
               command:
-                {{- if .Values.workers.livenessProbe.command }}
-                  {{- toYaml .Values.workers.livenessProbe.command  | nindent 16 }}
+                {{- if or .Values.workers.celery.livenessProbe.command .Values.workers.livenessProbe.command }}
+                  {{- toYaml (.Values.workers.celery.livenessProbe.command | default .Values.workers.livenessProbe.command)  | nindent 16 }}
                 {{- else }}
                 - sh
                 - -c

chart/values.schema.json

@@ -1675,37 +1675,37 @@
                     ]
                 },
                 "livenessProbe": {
-                    "description": "Liveness probe configuration for Airflow Celery worker containers.",
+                    "description": "Liveness probe configuration for Airflow Celery worker containers (deprecated, use `workers.celery.livenessProbe` section instead).",
                     "type": "object",
                     "additionalProperties": false,
                     "properties": {
                         "enabled": {
-                            "description": "Enable liveness probe for Airflow Celery workers.",
+                            "description": "Enable liveness probe for Airflow Celery workers (deprecated, use `workers.celery.livenessProbe.enabled` instead).",
                             "type": "boolean",
                             "default": true
                         },
                         "initialDelaySeconds": {
-                            "description": "Number of seconds after the container has started before liveness probes are initiated.",
+                            "description": "Number of seconds after the container has started before liveness probes are initiated (deprecated, use `workers.celery.livenessProbe.initialDelaySeconds` instead).",
                             "type": "integer",
                             "default": 10
                         },
                         "timeoutSeconds": {
-                            "description": "Number of seconds after which the probe times out. Minimum value is 1 seconds.",
+                            "description": "Number of seconds after which the probe times out. Minimum value is 1 seconds (deprecated, use `workers.celery.livenessProbe.timeoutSeconds` instead).",
                             "type": "integer",
                             "default": 20
                         },
                         "failureThreshold": {
-                            "description": "Minimum consecutive failures for the probe to be considered failed after having succeeded. Minimum value is 1.",
+                            "description": "Minimum consecutive failures for the probe to be considered failed after having succeeded. Minimum value is 1 (deprecated, use `workers.celery.livenessProbe.failureThreshold` instead).",
                             "type": "integer",
                             "default": 5
                         },
                         "periodSeconds": {
-                            "description": "How often (in seconds) to perform the probe. Minimum value is 1.",
+                            "description": "How often (in seconds) to perform the probe. Minimum value is 1 (deprecated, use `workers.celery.livenessProbe.periodSeconds` instead).",
                             "type": "integer",
                             "default": 60
                         },
                         "command": {
-                            "description": "Command for livenessProbe",
+                            "description": "Command for livenessProbe (deprecated, use `workers.celery.livenessProbe.command` instead)",
                             "type": [
                                 "array",
                                 "null"
@@ -2648,6 +2648,48 @@
                                 "exec \\\nairflow {{ semverCompare \">=2.0.0\" .Values.airflowVersion | ternary \"celery worker\" \"worker\" }}"
                             ]
                         },
+                        "livenessProbe": {
+                            "description": "Liveness probe configuration for Airflow Celery worker containers.",
+                            "type": "object",
+                            "additionalProperties": false,
+                            "properties": {
+                                "enabled": {
+                                    "description": "Enable liveness probe for Airflow Celery workers.",
+                                    "type": "boolean",
+                                    "default": true
+                                },
+                                "initialDelaySeconds": {
+                                    "description": "Number of seconds after the container has started before liveness probes are initiated.",
+                                    "type": "integer",
+                                    "default": 10
+                                },
+                                "timeoutSeconds": {
+                                    "description": "Number of seconds after which the probe times out. Minimum value is 1 seconds.",
+                                    "type": "integer",
+                                    "default": 20
+                                },
+                                "failureThreshold": {
+                                    "description": "Minimum consecutive failures for the probe to be considered failed after having succeeded. Minimum value is 1.",
+                                    "type": "integer",
+                                    "default": 5
+                                },
+                                "periodSeconds": {
+                                    "description": "How often (in seconds) to perform the probe. Minimum value is 1.",
+                                    "type": "integer",
+                                    "default": 60
+                                },
+                                "command": {
+                                    "description": "Command for livenessProbe",
+                                    "type": [
+                                        "array",
+                                        "null"
+                                    ],
+                                    "items": {
+                                        "type": "string"
+                                    }
+                                }
+                            }
+                        },
                         "serviceAccount": {
                             "description": "Create ServiceAccount.",
                             "type": "object",

chart/values.yaml

@@ -661,12 +661,19 @@ workers:
 
   # If the Airflow Celery worker stops responding for 5 minutes (5*60s)
   # kill the worker and let Kubernetes restart it
+  # (deprecated, use `workers.celery.livenessProbe` section instead)
   livenessProbe:
+    # (deprecated, use `workers.celery.livenessProbe.enabled` instead)
     enabled: true
+    # (deprecated, use `workers.celery.livenessProbe.initialDelaySeconds` instead)
     initialDelaySeconds: 10
+    # (deprecated, use `workers.celery.livenessProbe.timeoutSeconds` instead)
     timeoutSeconds: 20
+    # (deprecated, use `workers.celery.livenessProbe.failureThreshold` instead)
     failureThreshold: 5
+    # (deprecated, use `workers.celery.livenessProbe.periodSeconds` instead)
     periodSeconds: 60
+    # (deprecated, use `workers.celery.livenessProbe.command` instead)
     command: ~
 
   # Update Strategy when Airflow Celery worker is deployed as a StatefulSet
@@ -1033,6 +1040,16 @@ workers:
         exec \
         airflow {{ semverCompare ">=2.0.0" .Values.airflowVersion | ternary "celery worker" "worker" }}
 
+    # If the Airflow Celery worker stops responding for 5 minutes (5*60s)
+    # kill the worker and let Kubernetes restart it
+    livenessProbe:
+      enabled: true
+      initialDelaySeconds: 10
+      timeoutSeconds: 20
+      failureThreshold: 5
+      periodSeconds: 60
+      command: ~
+
     # Create ServiceAccount for Airflow Celery workers
     serviceAccount:
       # default value is true

helm-tests/tests/helm_tests/airflow_core/test_worker.py

@@ -583,10 +583,29 @@ def test_livenessprobe_default_command(self, airflow_version, default_cmd):
         )
         assert default_cmd in livenessprobe_cmd[-1]
 
-    def test_livenessprobe_values_are_configurable(self):
-        docs = render_chart(
-            values={
-                "workers": {
+    @pytest.mark.parametrize(
+        "workers_values",
+        [
+            {
+                "celery": {
+                    "livenessProbe": {
+                        "initialDelaySeconds": 111,
+                        "timeoutSeconds": 222,
+                        "failureThreshold": 333,
+                        "periodSeconds": 444,
+                        "command": ["sh", "-c", "echo", "wow such test"],
+                    }
+                }
+            },
+            {
+                "livenessProbe": {
+                    "initialDelaySeconds": 11,
+                    "timeoutSeconds": 22,
+                    "failureThreshold": 33,
+                    "periodSeconds": 44,
+                    "command": ["test"],
+                },
+                "celery": {
                     "livenessProbe": {
                         "initialDelaySeconds": 111,
                         "timeoutSeconds": 222,
@@ -596,6 +615,28 @@ def test_livenessprobe_values_are_configurable(self):
                     }
                 },
             },
+            {
+                "livenessProbe": {
+                    "initialDelaySeconds": 111,
+                    "timeoutSeconds": 222,
+                    "failureThreshold": 333,
+                    "periodSeconds": 444,
+                    "command": ["sh", "-c", "echo", "wow such test"],
+                },
+                "celery": {
+                    "livenessProbe": {
+                        "initialDelaySeconds": None,
+                        "timeoutSeconds": None,
+                        "failureThreshold": None,
+                        "periodSeconds": None,
+                    }
+                },
+            },
+        ],
+    )
+    def test_livenessprobe_celery_values_overwrite(self, workers_values):
+        docs = render_chart(
+            values={"workers": workers_values},
             show_only=["templates/workers/worker-deployment.yaml"],
         )
 
@@ -610,14 +651,46 @@ def test_livenessprobe_values_are_configurable(self):
             },
         }
 
-    def test_disable_livenessprobe(self):
+    def test_livenessprobe_values_overwrite(self):
         docs = render_chart(
             values={
-                "workers": {"livenessProbe": {"enabled": False}},
+                "workers": {
+                    "livenessProbe": {
+                        "initialDelaySeconds": 111,
+                        "timeoutSeconds": 222,
+                        "failureThreshold": 333,
+                        "periodSeconds": 444,
+                        "command": ["sh", "-c", "echo", "wow such test"],
+                    }
+                }
             },
             show_only=["templates/workers/worker-deployment.yaml"],
         )
 
+        livenessprobe = jmespath.search("spec.template.spec.containers[0].livenessProbe", docs[0])
+        assert livenessprobe == {
+            "initialDelaySeconds": 10,
+            "timeoutSeconds": 20,
+            "failureThreshold": 5,
+            "periodSeconds": 60,
+            "exec": {
+                "command": ["sh", "-c", "echo", "wow such test"],
+            },
+        }
+
+    @pytest.mark.parametrize(
+        "workers_values",
+        [
+            {"livenessProbe": {"enabled": False}, "celery": {"livenessProbe": {"enabled": None}}},
+            {"celery": {"livenessProbe": {"enabled": False}}},
+        ],
+    )
+    def test_disable_livenessprobe(self, workers_values):
+        docs = render_chart(
+            values={"workers": workers_values},
+            show_only=["templates/workers/worker-deployment.yaml"],
+        )
+
         livenessprobe = jmespath.search("spec.template.spec.containers[0].livenessProbe", docs[0])
         assert livenessprobe is None