Add Horizontal Pod Autoscaler (HPA) Support for Airflow API Server

chart/templates/api-server/api-server-hpa.yaml

@@ -0,0 +1,49 @@
+{{/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied.  See the License for the
+ specific language governing permissions and limitations
+ under the License.
+*/}}
+
+################################
+## Airflow Api-Server HPA
+#################################
+{{- if .Values.apiServer.hpa.enabled }}
+apiVersion: autoscaling/v2
+kind: HorizontalPodAutoscaler
+metadata:
+  name: {{ include "airflow.fullname" . }}-api-server
+  labels:
+    tier: airflow
+    component: api-server-horizontalpodautoscaler
+    release: {{ .Release.Name }}
+    chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+    heritage: {{ .Release.Service }}
+    deploymentName: {{ .Release.Name }}-api-server
+    {{- if or (.Values.labels) (.Values.apiServer.labels) }}
+      {{- mustMerge .Values.apiServer.labels .Values.labels | toYaml | nindent 4 }}
+    {{- end }}
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: {{ include "airflow.fullname" . }}-api-server
+  minReplicas: {{ .Values.apiServer.hpa.minReplicaCount }}
+  maxReplicas: {{ .Values.apiServer.hpa.maxReplicaCount }}
+  metrics: {{- toYaml .Values.apiServer.hpa.metrics | nindent 4 }}
+  {{- with .Values.apiServer.hpa.behavior }}
+  behavior: {{- toYaml . | nindent 4 }}
+  {{- end }}
+{{- end }}

chart/values.schema.json

@@ -5077,7 +5077,7 @@
                     }
                 },
                 "replicas": {
-                    "description": "How many Airflow API server replicas should run.",
+                    "description": "How many Airflow API server replicas should run. This setting is ignored when HPA (Horizontal Pod Autoscaler) is enabled",
                     "type": "integer",
                     "default": 1
                 },
@@ -5124,6 +5124,53 @@
                     ],
                     "default": null
                 },
+                "hpa": {
+                    "description": "Horizontal Pod Autoscaler (HPA) configuration for apiServer. (optional)",
+                    "type": "object",
+                    "additionalProperties": false,
+                    "properties": {
+                        "enabled": {
+                            "description": "Enable HPA autoscaling for API server",
+                            "type": "boolean",
+                            "default": false
+                        },
+                        "minReplicaCount": {
+                            "description": "Minimum number of API server replicas created by HPA if HPA is enabled.",
+                            "type": "integer",
+                            "default": 1
+                        },
+                        "maxReplicaCount": {
+                            "description": "Maximum number of API server replicas created by HPA if HPA is enabled.",
+                            "type": "integer",
+                            "default": 5
+                        },
+                        "metrics": {
+                            "description": "Specifications for which to use to calculate the desired replica count.",
+                            "type": "array",
+                            "default": [
+                                {
+                                    "type": "Resource",
+                                    "resource": {
+                                        "name": "cpu",
+                                        "target": {
+                                            "type": "Utilization",
+                                            "averageUtilization": 50
+                                        }
+                                    }
+                                }
+                            ],
+                            "items": {
+                                "$ref": "#/definitions/io.k8s.api.autoscaling.v2.MetricSpec"
+                            }
+                        },
+                        "behavior": {
+                            "description": "HorizontalPodAutoscalerBehavior configures the scaling behavior of the target.",
+                            "type": "object",
+                            "default": {},
+                            "$ref": "#/definitions/io.k8s.api.autoscaling.v2.HorizontalPodAutoscalerBehavior"
+                        }
+                    }
+                },
                 "serviceAccount": {
                     "description": "Create ServiceAccount.",
                     "type": "object",

chart/values.yaml

@@ -1416,6 +1416,8 @@ migrateDatabaseJob:
 apiServer:
   enabled: true
   # Number of Airflow API servers in the deployment
+  # This setting is ignored when HPA (Horizontal Pod Autoscaler) is enabled,
+  # as HPA will automatically manage the number of replicas based on the configured metrics.
   replicas: 1
   # Max number of old replicasets to retain
   revisionHistoryLimit: ~
@@ -1429,6 +1431,31 @@ apiServer:
   args: ["bash", "-c", "exec airflow api-server"]
   allowPodLogReading: true
   env: []
+
+  # Allow Horizontal Pod Autoscaler (HPA) configuration for apiServer. (optional)
+  # HPA automatically scales the number of apiServer pods based on observed metrics.
+  # HPA automatically adjusts apiServer replicas between minReplicaCount and maxReplicaCount based on metrics.
+  hpa:
+    enabled: false
+
+    # Minimum number of api-servers created by HPA
+    minReplicaCount: 1
+
+    # Maximum number of api-servers created by HPA
+    maxReplicaCount: 5
+
+    # Specifications for which to use to calculate the desired replica count
+    metrics:
+      - type: Resource
+        resource:
+          name: cpu
+          target:
+            type: Utilization
+            averageUtilization: 50
+
+    # Scaling behavior of the target in both Up and Down directions
+    behavior: {}
+
   serviceAccount:
     # default value is true
     # ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/

helm-tests/tests/helm_tests/apiserver/test_hpa_apiserver.py

@@ -0,0 +1,139 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+import jmespath
+import pytest
+from chart_utils.helm_template_generator import render_chart
+
+
+class TestAPIServerHPA:
+    """Tests HPA."""
+
+    def test_hpa_disabled_by_default(self):
+        """Disabled by default."""
+        docs = render_chart(
+            values={},
+            show_only=["templates/api-server/api-server-hpa.yaml"],
+        )
+        assert docs == []
+
+    def test_should_add_component_specific_labels(self):
+        docs = render_chart(
+            values={
+                "airflowVersion": "3.0.2",
+                "apiServer": {
+                    "hpa": {"enabled": True},
+                    "labels": {"test_label": "test_label_value"},
+                },
+            },
+            show_only=["templates/api-server/api-server-hpa.yaml"],
+        )
+
+        assert "test_label" in jmespath.search("metadata.labels", docs[0])
+        assert jmespath.search("metadata.labels", docs[0])["test_label"] == "test_label_value"
+
+    @pytest.mark.parametrize(
+        ("min_replicas", "max_replicas"),
+        [
+            (None, None),
+            (2, 8),
+        ],
+    )
+    def test_min_max_replicas(self, min_replicas, max_replicas):
+        """Verify minimum and maximum replicas."""
+        docs = render_chart(
+            values={
+                "airflowVersion": "3.0.2",
+                "apiServer": {
+                    "hpa": {
+                        "enabled": True,
+                        **({"minReplicaCount": min_replicas} if min_replicas else {}),
+                        **({"maxReplicaCount": max_replicas} if max_replicas else {}),
+                    }
+                },
+            },
+            show_only=["templates/api-server/api-server-hpa.yaml"],
+        )
+        assert jmespath.search("spec.minReplicas", docs[0]) == 1 if min_replicas is None else min_replicas
+        assert jmespath.search("spec.maxReplicas", docs[0]) == 5 if max_replicas is None else max_replicas
+
+    def test_hpa_behavior(self):
+        """Verify HPA behavior."""
+        expected_behavior = {
+            "scaleDown": {
+                "stabilizationWindowSeconds": 300,
+                "policies": [{"type": "Percent", "value": 100, "periodSeconds": 15}],
+            }
+        }
+        docs = render_chart(
+            values={
+                "airflowVersion": "3.0.2",
+                "apiServer": {
+                    "hpa": {
+                        "enabled": True,
+                        "behavior": expected_behavior,
+                    },
+                },
+            },
+            show_only=["templates/api-server/api-server-hpa.yaml"],
+        )
+        assert jmespath.search("spec.behavior", docs[0]) == expected_behavior
+
+    @pytest.mark.parametrize(
+        ("metrics", "expected_metrics"),
+        [
+            # default metrics
+            (
+                None,
+                {
+                    "type": "Resource",
+                    "resource": {"name": "cpu", "target": {"type": "Utilization", "averageUtilization": 50}},
+                },
+            ),
+            # custom metric
+            (
+                [
+                    {
+                        "type": "Pods",
+                        "pods": {
+                            "metric": {"name": "custom"},
+                            "target": {"type": "Utilization", "averageUtilization": 50},
+                        },
+                    }
+                ],
+                {
+                    "type": "Pods",
+                    "pods": {
+                        "metric": {"name": "custom"},
+                        "target": {"type": "Utilization", "averageUtilization": 50},
+                    },
+                },
+            ),
+        ],
+    )
+    def test_should_use_hpa_metrics(self, metrics, expected_metrics):
+        docs = render_chart(
+            values={
+                "airflowVersion": "3.0.2",
+                "apiServer": {
+                    "hpa": {"enabled": True, **({"metrics": metrics} if metrics else {})},
+                },
+            },
+            show_only=["templates/api-server/api-server-hpa.yaml"],
+        )
+        assert jmespath.search("spec.metrics[0]", docs[0]) == expected_metrics