diff --git a/api/v1alpha1/envoyproxy_helpers.go b/api/v1alpha1/envoyproxy_helpers.go index 2a0bd91ac0a..d446df9f054 100644 --- a/api/v1alpha1/envoyproxy_helpers.go +++ b/api/v1alpha1/envoyproxy_helpers.go @@ -9,6 +9,11 @@ import ( "fmt" "sort" "strings" + + autoscalingv2 "k8s.io/api/autoscaling/v2" + v1 "k8s.io/api/core/v1" + + "github.com/envoyproxy/gateway/internal/utils/ptr" ) // DefaultEnvoyProxyProvider returns a new EnvoyProxyProvider with default settings. @@ -37,6 +42,21 @@ func DefaultEnvoyProxyKubeProvider() *EnvoyProxyKubernetesProvider { } } +func DefaultEnvoyProxyHpaMetrics() []autoscalingv2.MetricSpec { + return []autoscalingv2.MetricSpec{ + { + Resource: &autoscalingv2.ResourceMetricSource{ + Name: v1.ResourceCPU, + Target: autoscalingv2.MetricTarget{ + Type: autoscalingv2.UtilizationMetricType, + AverageUtilization: ptr.To[int32](80), + }, + }, + Type: autoscalingv2.ResourceMetricSourceType, + }, + } +} + // GetEnvoyProxyKubeProvider returns the EnvoyProxyKubernetesProvider of EnvoyProxyProvider or // a default EnvoyProxyKubernetesProvider if unspecified. If EnvoyProxyProvider is not of // type "Kubernetes", a nil EnvoyProxyKubernetesProvider is returned. @@ -64,6 +84,10 @@ func (r *EnvoyProxyProvider) GetEnvoyProxyKubeProvider() *EnvoyProxyKubernetesPr r.Kubernetes.EnvoyService.Type = GetKubernetesServiceType(ServiceTypeLoadBalancer) } + if r.Kubernetes.EnvoyHpa != nil { + r.Kubernetes.EnvoyHpa.setDefault() + } + return r.Kubernetes } diff --git a/api/v1alpha1/envoyproxy_types.go b/api/v1alpha1/envoyproxy_types.go index 4aa7156cb8c..acf34417c76 100644 --- a/api/v1alpha1/envoyproxy_types.go +++ b/api/v1alpha1/envoyproxy_types.go @@ -127,6 +127,15 @@ type EnvoyProxyKubernetesProvider struct { // +kubebuilder:validation:XValidation:message="loadBalancerIP can only be set for LoadBalancer type",rule="!has(self.loadBalancerIP) || self.type == 'LoadBalancer'" // +kubebuilder:validation:XValidation:message="loadBalancerIP must be a valid IPv4 address",rule="!has(self.loadBalancerIP) || self.loadBalancerIP.matches(r\"^(((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\\.|$)){4})\")" EnvoyService *KubernetesServiceSpec `json:"envoyService,omitempty"` + + // EnvoyHpa defines the Horizontal Pod Autoscaler settings for Envoy Proxy Deployment. + // Once the HPA is being set, Replicas field from EnvoyDeployment will be ignored. + // + // +optional + // +kubebuilder:validation:XValidation:message="minReplicas must be greater than 0",rule="!has(self.minReplicas) || self.minReplicas > 0" + // +kubebuilder:validation:XValidation:message="maxReplicas must be greater than 0",rule="!has(self.maxReplicas) || self.maxReplicas > 0" + // +kubebuilder:validation:XValidation:message="maxReplicas cannot be less than or equal to minReplicas",rule="!has(self.minReplicas) || self.maxReplicas > self.minReplicas" + EnvoyHpa *KubernetesHorizontalPodAutoscalerSpec `json:"envoyHpa,omitempty"` } // ProxyLogging defines logging parameters for managed proxies. diff --git a/api/v1alpha1/kubernetes_helpers.go b/api/v1alpha1/kubernetes_helpers.go index 7b6c131e6c9..90c75873cbb 100644 --- a/api/v1alpha1/kubernetes_helpers.go +++ b/api/v1alpha1/kubernetes_helpers.go @@ -106,3 +106,9 @@ func (deployment *KubernetesDeploymentSpec) defaultKubernetesDeploymentSpec(imag deployment.Container.Image = DefaultKubernetesContainerImage(image) } } + +func (hpa *KubernetesHorizontalPodAutoscalerSpec) setDefault() { + if len(hpa.Metrics) == 0 { + hpa.Metrics = DefaultEnvoyProxyHpaMetrics() + } +} diff --git a/api/v1alpha1/shared_types.go b/api/v1alpha1/shared_types.go index 1764558d706..e6d19f960ae 100644 --- a/api/v1alpha1/shared_types.go +++ b/api/v1alpha1/shared_types.go @@ -7,6 +7,7 @@ package v1alpha1 import ( appv1 "k8s.io/api/apps/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" ) @@ -275,3 +276,34 @@ const ( // https://github.com/google/re2/wiki/Syntax. StringMatchRegularExpression StringMatchType = "RegularExpression" ) + +// KubernetesHorizontalPodAutoscalerSpec defines Kubernetes Horizontal Pod Autoscaler settings of Envoy Proxy Deployment +// See k8s.io.autoscaling.v2.HorizontalPodAutoScalerSpec +type KubernetesHorizontalPodAutoscalerSpec struct { + // minReplicas is the lower limit for the number of replicas to which the autoscaler + // can scale down. It defaults to 1 replica. + // + // +optional + MinReplicas *int32 `json:"minReplicas,omitempty"` + + // maxReplicas is the upper limit for the number of replicas to which the autoscaler can scale up. + // It cannot be less that minReplicas. + // + MaxReplicas *int32 `json:"maxReplicas"` + + // metrics contains the specifications for which to use to calculate the + // desired replica count (the maximum replica count across all metrics will + // be used). + // If left empty, it defaults to being based on CPU utilization with average on 80% usage. + // + // +optional + Metrics []autoscalingv2.MetricSpec `json:"metrics,omitempty"` + + // behavior configures the scaling behavior of the target + // in both Up and Down directions (scaleUp and scaleDown fields respectively). + // If not set, the default HPAScalingRules for scale up and scale down are used. + // See k8s.io.autoscaling.v2.HorizontalPodAutoScalerBehavior. + // + // +optional + Behavior *autoscalingv2.HorizontalPodAutoscalerBehavior `json:"behavior,omitempty"` +} diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 88bc53a3ae5..2654e070a27 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -11,6 +11,7 @@ package v1alpha1 import ( appsv1 "k8s.io/api/apps/v1" + "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" @@ -925,6 +926,11 @@ func (in *EnvoyProxyKubernetesProvider) DeepCopyInto(out *EnvoyProxyKubernetesPr *out = new(KubernetesServiceSpec) (*in).DeepCopyInto(*out) } + if in.EnvoyHpa != nil { + in, out := &in.EnvoyHpa, &out.EnvoyHpa + *out = new(KubernetesHorizontalPodAutoscalerSpec) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EnvoyProxyKubernetesProvider. @@ -1428,6 +1434,43 @@ func (in *KubernetesDeploymentSpec) DeepCopy() *KubernetesDeploymentSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KubernetesHorizontalPodAutoscalerSpec) DeepCopyInto(out *KubernetesHorizontalPodAutoscalerSpec) { + *out = *in + if in.MinReplicas != nil { + in, out := &in.MinReplicas, &out.MinReplicas + *out = new(int32) + **out = **in + } + if in.MaxReplicas != nil { + in, out := &in.MaxReplicas, &out.MaxReplicas + *out = new(int32) + **out = **in + } + if in.Metrics != nil { + in, out := &in.Metrics, &out.Metrics + *out = make([]v2.MetricSpec, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Behavior != nil { + in, out := &in.Behavior, &out.Behavior + *out = new(v2.HorizontalPodAutoscalerBehavior) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubernetesHorizontalPodAutoscalerSpec. +func (in *KubernetesHorizontalPodAutoscalerSpec) DeepCopy() *KubernetesHorizontalPodAutoscalerSpec { + if in == nil { + return nil + } + out := new(KubernetesHorizontalPodAutoscalerSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *KubernetesPodSpec) DeepCopyInto(out *KubernetesPodSpec) { *out = *in diff --git a/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml b/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml index 2b937371634..7bff9e4149a 100644 --- a/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml +++ b/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml @@ -5230,6 +5230,690 @@ spec: type: string type: object type: object + envoyHpa: + description: EnvoyHpa defines the Horizontal Pod Autoscaler + settings for Envoy Proxy Deployment. Once the HPA is being + set, Replicas field from EnvoyDeployment will be ignored. + properties: + behavior: + description: behavior configures the scaling behavior + of the target in both Up and Down directions (scaleUp + and scaleDown fields respectively). If not set, the + default HPAScalingRules for scale up and scale down + are used. See k8s.io.autoscaling.v2.HorizontalPodAutoScalerBehavior. + properties: + scaleDown: + description: scaleDown is scaling policy for scaling + Down. If not set, the default value is to allow + to scale down to minReplicas pods, with a 300 second + stabilization window (i.e., the highest recommendation + for the last 300sec is used). + properties: + policies: + description: policies is a list of potential scaling + polices which can be used during scaling. At + least one policy must be specified, otherwise + the HPAScalingRules will be discarded as invalid + items: + description: HPAScalingPolicy is a single policy + which must hold true for a specified past + interval. + properties: + periodSeconds: + description: periodSeconds specifies the + window of time for which the policy should + hold true. PeriodSeconds must be greater + than zero and less than or equal to 1800 + (30 min). + format: int32 + type: integer + type: + description: type is used to specify the + scaling policy. + type: string + value: + description: value contains the amount of + change which is permitted by the policy. + It must be greater than zero + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + x-kubernetes-list-type: atomic + selectPolicy: + description: selectPolicy is used to specify which + policy should be used. If not set, the default + value Max is used. + type: string + stabilizationWindowSeconds: + description: 'stabilizationWindowSeconds is the + number of seconds for which past recommendations + should be considered while scaling up or scaling + down. StabilizationWindowSeconds must be greater + than or equal to zero and less than or equal + to 3600 (one hour). If not set, use the default + values: - For scale up: 0 (i.e. no stabilization + is done). - For scale down: 300 (i.e. the stabilization + window is 300 seconds long).' + format: int32 + type: integer + type: object + scaleUp: + description: 'scaleUp is scaling policy for scaling + Up. If not set, the default value is the higher + of: * increase no more than 4 pods per 60 seconds + * double the number of pods per 60 seconds No stabilization + is used.' + properties: + policies: + description: policies is a list of potential scaling + polices which can be used during scaling. At + least one policy must be specified, otherwise + the HPAScalingRules will be discarded as invalid + items: + description: HPAScalingPolicy is a single policy + which must hold true for a specified past + interval. + properties: + periodSeconds: + description: periodSeconds specifies the + window of time for which the policy should + hold true. PeriodSeconds must be greater + than zero and less than or equal to 1800 + (30 min). + format: int32 + type: integer + type: + description: type is used to specify the + scaling policy. + type: string + value: + description: value contains the amount of + change which is permitted by the policy. + It must be greater than zero + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + x-kubernetes-list-type: atomic + selectPolicy: + description: selectPolicy is used to specify which + policy should be used. If not set, the default + value Max is used. + type: string + stabilizationWindowSeconds: + description: 'stabilizationWindowSeconds is the + number of seconds for which past recommendations + should be considered while scaling up or scaling + down. StabilizationWindowSeconds must be greater + than or equal to zero and less than or equal + to 3600 (one hour). If not set, use the default + values: - For scale up: 0 (i.e. no stabilization + is done). - For scale down: 300 (i.e. the stabilization + window is 300 seconds long).' + format: int32 + type: integer + type: object + type: object + maxReplicas: + description: maxReplicas is the upper limit for the number + of replicas to which the autoscaler can scale up. It + cannot be less that minReplicas. + format: int32 + type: integer + metrics: + description: metrics contains the specifications for which + to use to calculate the desired replica count (the maximum + replica count across all metrics will be used). If left + empty, it defaults to being based on CPU utilization + with average on 80% usage. + items: + description: MetricSpec specifies how to scale based + on a single metric (only `type` and one other matching + field should be set at once). + properties: + containerResource: + description: containerResource refers to a resource + metric (such as those specified in requests and + limits) known to Kubernetes describing a single + container in each pod of the current scale target + (e.g. CPU or memory). Such metrics are built in + to Kubernetes, and have special scaling options + on top of those available to normal per-pod metrics + using the "pods" source. This is an alpha feature + and can be enabled by the HPAContainerMetrics + feature flag. + properties: + container: + description: container is the name of the container + in the pods of the scaling target + type: string + name: + description: name is the name of the resource + in question. + type: string + target: + description: target specifies the target value + for the given metric + properties: + averageUtilization: + description: averageUtilization is the target + value of the average of the resource metric + across all relevant pods, represented + as a percentage of the requested value + of the resource for the pods. Currently + only valid for Resource metric source + type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: averageValue is the target + value of the average of the metric across + all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the + metric type is Utilization, Value, or + AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of + the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - container + - name + - target + type: object + external: + description: external refers to a global metric + that is not associated with any Kubernetes object. + It allows autoscaling based on information coming + from components running outside of cluster (for + example length of queue in cloud messaging service, + or QPS from loadbalancer running outside of cluster). + properties: + metric: + description: metric identifies the target metric + by name and selector + properties: + name: + description: name is the name of the given + metric + type: string + selector: + description: selector is the string-encoded + form of a standard kubernetes label selector + for the given metric When set, it is passed + as an additional parameter to the metrics + server for more specific metrics scoping. + When unset, just the metricName will be + used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: A label selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: operator represents + a key's relationship to a set + of values. Valid operators are + In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array + of string values. If the operator + is In or NotIn, the values array + must be non-empty. If the operator + is Exists or DoesNotExist, the + values array must be empty. + This array is replaced during + a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of + {key,value} pairs. A single {key,value} + in the matchLabels map is equivalent + to an element of matchExpressions, + whose key field is "key", the operator + is "In", and the values array contains + only "value". The requirements are + ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value + for the given metric + properties: + averageUtilization: + description: averageUtilization is the target + value of the average of the resource metric + across all relevant pods, represented + as a percentage of the requested value + of the resource for the pods. Currently + only valid for Resource metric source + type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: averageValue is the target + value of the average of the metric across + all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the + metric type is Utilization, Value, or + AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of + the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - metric + - target + type: object + object: + description: object refers to a metric describing + a single kubernetes object (for example, hits-per-second + on an Ingress object). + properties: + describedObject: + description: describedObject specifies the descriptions + of a object,such as kind,name apiVersion + properties: + apiVersion: + description: apiVersion is the API version + of the referent + type: string + kind: + description: 'kind is the kind of the referent; + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'name is the name of the referent; + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + required: + - kind + - name + type: object + metric: + description: metric identifies the target metric + by name and selector + properties: + name: + description: name is the name of the given + metric + type: string + selector: + description: selector is the string-encoded + form of a standard kubernetes label selector + for the given metric When set, it is passed + as an additional parameter to the metrics + server for more specific metrics scoping. + When unset, just the metricName will be + used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: A label selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: operator represents + a key's relationship to a set + of values. Valid operators are + In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array + of string values. If the operator + is In or NotIn, the values array + must be non-empty. If the operator + is Exists or DoesNotExist, the + values array must be empty. + This array is replaced during + a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of + {key,value} pairs. A single {key,value} + in the matchLabels map is equivalent + to an element of matchExpressions, + whose key field is "key", the operator + is "In", and the values array contains + only "value". The requirements are + ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value + for the given metric + properties: + averageUtilization: + description: averageUtilization is the target + value of the average of the resource metric + across all relevant pods, represented + as a percentage of the requested value + of the resource for the pods. Currently + only valid for Resource metric source + type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: averageValue is the target + value of the average of the metric across + all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the + metric type is Utilization, Value, or + AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of + the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - describedObject + - metric + - target + type: object + pods: + description: pods refers to a metric describing + each pod in the current scale target (for example, + transactions-processed-per-second). The values + will be averaged together before being compared + to the target value. + properties: + metric: + description: metric identifies the target metric + by name and selector + properties: + name: + description: name is the name of the given + metric + type: string + selector: + description: selector is the string-encoded + form of a standard kubernetes label selector + for the given metric When set, it is passed + as an additional parameter to the metrics + server for more specific metrics scoping. + When unset, just the metricName will be + used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: A label selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: operator represents + a key's relationship to a set + of values. Valid operators are + In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array + of string values. If the operator + is In or NotIn, the values array + must be non-empty. If the operator + is Exists or DoesNotExist, the + values array must be empty. + This array is replaced during + a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of + {key,value} pairs. A single {key,value} + in the matchLabels map is equivalent + to an element of matchExpressions, + whose key field is "key", the operator + is "In", and the values array contains + only "value". The requirements are + ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value + for the given metric + properties: + averageUtilization: + description: averageUtilization is the target + value of the average of the resource metric + across all relevant pods, represented + as a percentage of the requested value + of the resource for the pods. Currently + only valid for Resource metric source + type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: averageValue is the target + value of the average of the metric across + all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the + metric type is Utilization, Value, or + AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of + the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - metric + - target + type: object + resource: + description: resource refers to a resource metric + (such as those specified in requests and limits) + known to Kubernetes describing each pod in the + current scale target (e.g. CPU or memory). Such + metrics are built in to Kubernetes, and have special + scaling options on top of those available to normal + per-pod metrics using the "pods" source. + properties: + name: + description: name is the name of the resource + in question. + type: string + target: + description: target specifies the target value + for the given metric + properties: + averageUtilization: + description: averageUtilization is the target + value of the average of the resource metric + across all relevant pods, represented + as a percentage of the requested value + of the resource for the pods. Currently + only valid for Resource metric source + type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: averageValue is the target + value of the average of the metric across + all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the + metric type is Utilization, Value, or + AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of + the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - name + - target + type: object + type: + description: 'type is the type of metric source. It + should be one of "ContainerResource", "External", + "Object", "Pods" or "Resource", each mapping to + a matching field in the object. Note: "ContainerResource" + type is available on when the feature-gate HPAContainerMetrics + is enabled' + type: string + required: + - type + type: object + type: array + minReplicas: + description: minReplicas is the lower limit for the number + of replicas to which the autoscaler can scale down. + It defaults to 1 replica. + format: int32 + type: integer + required: + - maxReplicas + type: object + x-kubernetes-validations: + - message: minReplicas must be greater than 0 + rule: '!has(self.minReplicas) || self.minReplicas > 0' + - message: maxReplicas must be greater than 0 + rule: '!has(self.maxReplicas) || self.maxReplicas > 0' + - message: maxReplicas cannot be less than or equal to minReplicas + rule: '!has(self.minReplicas) || self.maxReplicas > self.minReplicas' envoyService: description: EnvoyService defines the desired state of the Envoy service resource. If unspecified, default settings diff --git a/charts/gateway-helm/templates/infra-manager-rbac.yaml b/charts/gateway-helm/templates/infra-manager-rbac.yaml index 6f3e5a4677f..3929524f484 100644 --- a/charts/gateway-helm/templates/infra-manager-rbac.yaml +++ b/charts/gateway-helm/templates/infra-manager-rbac.yaml @@ -25,6 +25,15 @@ rules: - get - update - delete +- apiGroups: + - autoscaling + resources: + - horizontalpodautoscalers + verbs: + - create + - get + - update + - delete --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding diff --git a/internal/infrastructure/kubernetes/infra.go b/internal/infrastructure/kubernetes/infra.go index f6b27d26702..0d1f6e18c03 100644 --- a/internal/infrastructure/kubernetes/infra.go +++ b/internal/infrastructure/kubernetes/infra.go @@ -10,6 +10,7 @@ import ( "github.com/pkg/errors" appsv1 "k8s.io/api/apps/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" @@ -25,6 +26,7 @@ type ResourceRender interface { Service() (*corev1.Service, error) ConfigMap() (*corev1.ConfigMap, error) Deployment() (*appsv1.Deployment, error) + HorizontalPodAutoscaler() (*autoscalingv2.HorizontalPodAutoscaler, error) } // Infra manages the creation and deletion of Kubernetes infrastructure @@ -68,6 +70,10 @@ func (i *Infra) createOrUpdate(ctx context.Context, r ResourceRender) error { return errors.Wrapf(err, "failed to create or update service %s/%s", i.Namespace, r.Name()) } + if err := i.createOrUpdateHPA(ctx, r); err != nil { + return errors.Wrapf(err, "failed to create or update hpa %s/%s", i.Namespace, r.Name()) + } + return nil } @@ -89,5 +95,9 @@ func (i *Infra) delete(ctx context.Context, r ResourceRender) error { return errors.Wrapf(err, "failed to delete service %s/%s", i.Namespace, r.Name()) } + if err := i.deleteHPA(ctx, r); err != nil { + return errors.Wrapf(err, "failed to delete hpa %s/%s", i.Namespace, r.Name()) + } + return nil } diff --git a/internal/infrastructure/kubernetes/infra_resource.go b/internal/infrastructure/kubernetes/infra_resource.go index af041ad3313..d0e21be628e 100644 --- a/internal/infrastructure/kubernetes/infra_resource.go +++ b/internal/infrastructure/kubernetes/infra_resource.go @@ -9,7 +9,10 @@ import ( "context" "reflect" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" appsv1 "k8s.io/api/apps/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -74,8 +77,44 @@ func (i *Infra) createOrUpdateDeployment(ctx context.Context, r ResourceRender) Name: deployment.Name, } + hpa, err := r.HorizontalPodAutoscaler() + if err != nil { + return err + } + + var opts cmp.Options + if hpa != nil { + opts = append(opts, cmpopts.IgnoreFields(appsv1.DeploymentSpec{}, "Replicas")) + } + return i.Client.CreateOrUpdate(ctx, key, current, deployment, func() bool { - return !reflect.DeepEqual(deployment.Spec, current.Spec) + return !cmp.Equal(current.Spec, deployment.Spec, opts...) + }) +} + +// createOrUpdateHPA creates HorizontalPodAutoscaler object in the kube api server based on +// the provided ResourceRender, if it doesn't exist and updates it if it does, +// and delete hpa if not set. +func (i *Infra) createOrUpdateHPA(ctx context.Context, r ResourceRender) error { + hpa, err := r.HorizontalPodAutoscaler() + if err != nil { + return err + } + + // when HorizontalPodAutoscaler is not set, + // then delete the object in the kube api server if any. + if hpa == nil { + return i.deleteHPA(ctx, r) + } + + current := &autoscalingv2.HorizontalPodAutoscaler{} + key := types.NamespacedName{ + Namespace: hpa.Namespace, + Name: hpa.Name, + } + + return i.Client.CreateOrUpdate(ctx, key, current, hpa, func() bool { + return !cmp.Equal(hpa.Spec, current.Spec) }) } @@ -145,3 +184,15 @@ func (i *Infra) deleteService(ctx context.Context, r ResourceRender) error { return i.Client.Delete(ctx, svc) } + +// deleteHpa deletes the Horizontal Pod Autoscaler associated to its renderer, if it exists. +func (i *Infra) deleteHPA(ctx context.Context, r ResourceRender) error { + hpa := &autoscalingv2.HorizontalPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: i.Namespace, + Name: r.Name(), + }, + } + + return i.Client.Delete(ctx, hpa) +} diff --git a/internal/infrastructure/kubernetes/proxy/resource_provider.go b/internal/infrastructure/kubernetes/proxy/resource_provider.go index bc446f79f9e..6ab47c0b334 100644 --- a/internal/infrastructure/kubernetes/proxy/resource_provider.go +++ b/internal/infrastructure/kubernetes/proxy/resource_provider.go @@ -11,10 +11,12 @@ import ( "golang.org/x/exp/maps" appsv1 "k8s.io/api/apps/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/utils/pointer" + "k8s.io/utils/ptr" egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1" "github.com/envoyproxy/gateway/internal/gatewayapi" @@ -56,7 +58,7 @@ func (r *ResourceRender) ServiceAccount() (*corev1.ServiceAccount, error) { }, ObjectMeta: metav1.ObjectMeta{ Namespace: r.Namespace, - Name: ExpectedResourceHashedName(r.infra.Name), + Name: r.Name(), Labels: labels, }, }, nil @@ -110,7 +112,7 @@ func (r *ResourceRender) Service() (*corev1.Service, error) { }, ObjectMeta: metav1.ObjectMeta{ Namespace: r.Namespace, - Name: ExpectedResourceHashedName(r.infra.Name), + Name: r.Name(), Labels: labels, Annotations: annotations, }, @@ -135,7 +137,7 @@ func (r *ResourceRender) ConfigMap() (*corev1.ConfigMap, error) { }, ObjectMeta: metav1.ObjectMeta{ Namespace: r.Namespace, - Name: ExpectedResourceHashedName(r.infra.Name), + Name: r.Name(), Labels: labels, }, Data: map[string]string{ @@ -192,7 +194,7 @@ func (r *ResourceRender) Deployment() (*appsv1.Deployment, error) { }, ObjectMeta: metav1.ObjectMeta{ Namespace: r.Namespace, - Name: ExpectedResourceHashedName(r.infra.Name), + Name: r.Name(), Labels: dpLabels, }, Spec: appsv1.DeploymentSpec{ @@ -224,5 +226,46 @@ func (r *ResourceRender) Deployment() (*appsv1.Deployment, error) { }, } + // omit the deployment replicas if HPA is being set + if provider.GetEnvoyProxyKubeProvider().EnvoyHpa != nil { + deployment.Spec.Replicas = nil + } + return deployment, nil } + +func (r *ResourceRender) HorizontalPodAutoscaler() (*autoscalingv2.HorizontalPodAutoscaler, error) { + provider := r.infra.GetProxyConfig().GetEnvoyProxyProvider() + if provider.Type != egv1a1.ProviderTypeKubernetes { + return nil, fmt.Errorf("invalid provider type %v for Kubernetes infra manager", provider.Type) + } + + hpaConfig := provider.GetEnvoyProxyKubeProvider().EnvoyHpa + if hpaConfig == nil { + return nil, nil + } + + hpa := &autoscalingv2.HorizontalPodAutoscaler{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "autoscaling/v2", + Kind: "HorizontalPodAutoscaler", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: r.Namespace, + Name: r.Name(), + }, + Spec: autoscalingv2.HorizontalPodAutoscalerSpec{ + ScaleTargetRef: autoscalingv2.CrossVersionObjectReference{ + APIVersion: "apps/v1", + Kind: "Deployment", + Name: r.Name(), + }, + MinReplicas: hpaConfig.MinReplicas, + MaxReplicas: ptr.Deref[int32](hpaConfig.MaxReplicas, 1), + Metrics: hpaConfig.Metrics, + Behavior: hpaConfig.Behavior, + }, + } + + return hpa, nil +} diff --git a/internal/infrastructure/kubernetes/proxy/resource_provider_test.go b/internal/infrastructure/kubernetes/proxy/resource_provider_test.go index 1c784927385..f4603fc370c 100644 --- a/internal/infrastructure/kubernetes/proxy/resource_provider_test.go +++ b/internal/infrastructure/kubernetes/proxy/resource_provider_test.go @@ -14,6 +14,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" appsv1 "k8s.io/api/apps/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/utils/pointer" @@ -23,6 +24,7 @@ import ( "github.com/envoyproxy/gateway/internal/envoygateway/config" "github.com/envoyproxy/gateway/internal/gatewayapi" "github.com/envoyproxy/gateway/internal/ir" + "github.com/envoyproxy/gateway/internal/utils/ptr" ) const ( @@ -340,6 +342,7 @@ func TestDeployment(t *testing.T) { if tc.deploy != nil { kube.EnvoyDeployment = tc.deploy } + replace := egv1a1.BootstrapTypeReplace if tc.bootstrap != "" { tc.infra.Proxy.Config.Spec.Bootstrap = &egv1a1.ProxyBootstrap{ @@ -506,3 +509,85 @@ func loadServiceAccount() (*corev1.ServiceAccount, error) { _ = yaml.Unmarshal(saYAML, sa) return sa, nil } + +func TestHorizontalPodAutoscaler(t *testing.T) { + cfg, err := config.New() + require.NoError(t, err) + + cases := []struct { + caseName string + infra *ir.Infra + hpa *egv1a1.KubernetesHorizontalPodAutoscalerSpec + }{ + { + caseName: "default", + infra: newTestInfra(), + hpa: &egv1a1.KubernetesHorizontalPodAutoscalerSpec{ + MaxReplicas: ptr.To[int32](1), + }, + }, + { + caseName: "custom", + infra: newTestInfra(), + hpa: &egv1a1.KubernetesHorizontalPodAutoscalerSpec{ + MinReplicas: ptr.To[int32](5), + MaxReplicas: ptr.To[int32](10), + Metrics: []autoscalingv2.MetricSpec{ + { + Resource: &autoscalingv2.ResourceMetricSource{ + Name: corev1.ResourceCPU, + Target: autoscalingv2.MetricTarget{ + Type: autoscalingv2.UtilizationMetricType, + AverageUtilization: ptr.To[int32](60), + }, + }, + Type: autoscalingv2.ResourceMetricSourceType, + }, + { + Resource: &autoscalingv2.ResourceMetricSource{ + Name: corev1.ResourceMemory, + Target: autoscalingv2.MetricTarget{ + Type: autoscalingv2.UtilizationMetricType, + AverageUtilization: ptr.To[int32](70), + }, + }, + Type: autoscalingv2.ResourceMetricSourceType, + }, + }, + }, + }, + } + + for _, tc := range cases { + t.Run(tc.caseName, func(t *testing.T) { + provider := tc.infra.GetProxyInfra().GetProxyConfig().GetEnvoyProxyProvider() + provider.Kubernetes = egv1a1.DefaultEnvoyProxyKubeProvider() + + if tc.hpa != nil { + provider.Kubernetes.EnvoyHpa = tc.hpa + } + + provider.GetEnvoyProxyKubeProvider() + + r := NewResourceRender(cfg.Namespace, tc.infra.GetProxyInfra()) + hpa, err := r.HorizontalPodAutoscaler() + require.NoError(t, err) + + want, err := loadHPA(tc.caseName) + require.NoError(t, err) + + assert.Equal(t, want, hpa) + }) + } +} + +func loadHPA(caseName string) (*autoscalingv2.HorizontalPodAutoscaler, error) { + hpaYAML, err := os.ReadFile(fmt.Sprintf("testdata/hpa/%s.yaml", caseName)) + if err != nil { + return nil, err + } + + hpa := &autoscalingv2.HorizontalPodAutoscaler{} + _ = yaml.Unmarshal(hpaYAML, hpa) + return hpa, nil +} diff --git a/internal/infrastructure/kubernetes/proxy/testdata/hpa/custom.yaml b/internal/infrastructure/kubernetes/proxy/testdata/hpa/custom.yaml new file mode 100644 index 00000000000..17171f8abb3 --- /dev/null +++ b/internal/infrastructure/kubernetes/proxy/testdata/hpa/custom.yaml @@ -0,0 +1,25 @@ +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: envoy-default-37a8eec1 + namespace: envoy-gateway-system +spec: + maxReplicas: 10 + metrics: + - resource: + name: cpu + target: + averageUtilization: 60 + type: Utilization + type: Resource + - resource: + name: memory + target: + averageUtilization: 70 + type: Utilization + type: Resource + minReplicas: 5 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: envoy-default-37a8eec1 diff --git a/internal/infrastructure/kubernetes/proxy/testdata/hpa/default.yaml b/internal/infrastructure/kubernetes/proxy/testdata/hpa/default.yaml new file mode 100644 index 00000000000..d11b7e47636 --- /dev/null +++ b/internal/infrastructure/kubernetes/proxy/testdata/hpa/default.yaml @@ -0,0 +1,18 @@ +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: envoy-default-37a8eec1 + namespace: envoy-gateway-system +spec: + metrics: + - resource: + name: cpu + target: + averageUtilization: 80 + type: Utilization + type: Resource + maxReplicas: 1 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: envoy-default-37a8eec1 diff --git a/internal/infrastructure/kubernetes/ratelimit/resource_provider.go b/internal/infrastructure/kubernetes/ratelimit/resource_provider.go index f39961795cc..bb7f2ee598d 100644 --- a/internal/infrastructure/kubernetes/ratelimit/resource_provider.go +++ b/internal/infrastructure/kubernetes/ratelimit/resource_provider.go @@ -7,6 +7,7 @@ package ratelimit import ( appsv1 "k8s.io/api/apps/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -202,3 +203,7 @@ func (r *ResourceRender) Deployment() (*appsv1.Deployment, error) { return deployment, nil } + +func (r *ResourceRender) HorizontalPodAutoscaler() (*autoscalingv2.HorizontalPodAutoscaler, error) { + return nil, nil +} diff --git a/site/content/en/latest/api/extension_types.md b/site/content/en/latest/api/extension_types.md index 7379bf3cef7..4fa146d29a9 100644 --- a/site/content/en/latest/api/extension_types.md +++ b/site/content/en/latest/api/extension_types.md @@ -647,6 +647,7 @@ _Appears in:_ | --- | --- | | `envoyDeployment` _[KubernetesDeploymentSpec](#kubernetesdeploymentspec)_ | EnvoyDeployment defines the desired state of the Envoy deployment resource. If unspecified, default settings for the managed Envoy deployment resource are applied. | | `envoyService` _[KubernetesServiceSpec](#kubernetesservicespec)_ | EnvoyService defines the desired state of the Envoy service resource. If unspecified, default settings for the managed Envoy service resource are applied. | +| `envoyHpa` _[KubernetesHorizontalPodAutoscalerSpec](#kuberneteshorizontalpodautoscalerspec)_ | EnvoyHpa defines the Horizontal Pod Autoscaler settings for Envoy Proxy Deployment. Once the HPA is being set, Replicas field from EnvoyDeployment will be ignored. | #### EnvoyProxyProvider @@ -976,6 +977,23 @@ _Appears in:_ | `initContainers` _[Container](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.26/#container-v1-core) array_ | List of initialization containers belonging to the pod. More info: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ | +#### KubernetesHorizontalPodAutoscalerSpec + + + +KubernetesHorizontalPodAutoscalerSpec defines Kubernetes Horizontal Pod Autoscaler settings of Envoy Proxy Deployment See k8s.io.autoscaling.v2.HorizontalPodAutoScalerSpec + +_Appears in:_ +- [EnvoyProxyKubernetesProvider](#envoyproxykubernetesprovider) + +| Field | Description | +| --- | --- | +| `minReplicas` _integer_ | minReplicas is the lower limit for the number of replicas to which the autoscaler can scale down. It defaults to 1 replica. | +| `maxReplicas` _integer_ | maxReplicas is the upper limit for the number of replicas to which the autoscaler can scale up. It cannot be less that minReplicas. | +| `metrics` _[MetricSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.26/#metricspec-v2-autoscaling) array_ | metrics contains the specifications for which to use to calculate the desired replica count (the maximum replica count across all metrics will be used). If left empty, it defaults to being based on CPU utilization with average on 80% usage. | +| `behavior` _[HorizontalPodAutoscalerBehavior](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.26/#horizontalpodautoscalerbehavior-v2-autoscaling)_ | behavior configures the scaling behavior of the target in both Up and Down directions (scaleUp and scaleDown fields respectively). If not set, the default HPAScalingRules for scale up and scale down are used. See k8s.io.autoscaling.v2.HorizontalPodAutoScalerBehavior. | + + #### KubernetesPodSpec diff --git a/test/cel-validation/envoyproxy_test.go b/test/cel-validation/envoyproxy_test.go index ac949ff971d..86b5a8ddc5e 100644 --- a/test/cel-validation/envoyproxy_test.go +++ b/test/cel-validation/envoyproxy_test.go @@ -419,6 +419,86 @@ func TestEnvoyProxyProvider(t *testing.T) { }, wantErrors: []string{}, }, + { + desc: "ProxyHpa-maxReplicas-is-required", + mutate: func(envoy *egv1a1.EnvoyProxy) { + envoy.Spec = egv1a1.EnvoyProxySpec{ + Provider: &egv1a1.EnvoyProxyProvider{ + Type: egv1a1.ProviderTypeKubernetes, + Kubernetes: &egv1a1.EnvoyProxyKubernetesProvider{ + EnvoyHpa: &egv1a1.KubernetesHorizontalPodAutoscalerSpec{}, + }, + }, + } + }, + wantErrors: []string{"spec.provider.kubernetes.envoyHpa.maxReplicas: Required value"}, + }, + { + desc: "ProxyHpa-minReplicas-less-than-0", + mutate: func(envoy *egv1a1.EnvoyProxy) { + envoy.Spec = egv1a1.EnvoyProxySpec{ + Provider: &egv1a1.EnvoyProxyProvider{ + Type: egv1a1.ProviderTypeKubernetes, + Kubernetes: &egv1a1.EnvoyProxyKubernetesProvider{ + EnvoyHpa: &egv1a1.KubernetesHorizontalPodAutoscalerSpec{ + MinReplicas: ptr.To[int32](-1), + MaxReplicas: ptr.To[int32](2), + }, + }, + }, + } + }, + wantErrors: []string{"minReplicas must be greater than 0"}, + }, + { + desc: "ProxyHpa-maxReplicas-less-than-0", + mutate: func(envoy *egv1a1.EnvoyProxy) { + envoy.Spec = egv1a1.EnvoyProxySpec{ + Provider: &egv1a1.EnvoyProxyProvider{ + Type: egv1a1.ProviderTypeKubernetes, + Kubernetes: &egv1a1.EnvoyProxyKubernetesProvider{ + EnvoyHpa: &egv1a1.KubernetesHorizontalPodAutoscalerSpec{ + MaxReplicas: ptr.To[int32](-1), + }, + }, + }, + } + }, + wantErrors: []string{"maxReplicas must be greater than 0"}, + }, + { + desc: "ProxyHpa-maxReplicas-less-than-minReplicas", + mutate: func(envoy *egv1a1.EnvoyProxy) { + envoy.Spec = egv1a1.EnvoyProxySpec{ + Provider: &egv1a1.EnvoyProxyProvider{ + Type: egv1a1.ProviderTypeKubernetes, + Kubernetes: &egv1a1.EnvoyProxyKubernetesProvider{ + EnvoyHpa: &egv1a1.KubernetesHorizontalPodAutoscalerSpec{ + MinReplicas: ptr.To[int32](5), + MaxReplicas: ptr.To[int32](2), + }, + }, + }, + } + }, + wantErrors: []string{"maxReplicas cannot be less than or equal to minReplicas"}, + }, + { + desc: "ProxyHpa-valid", + mutate: func(envoy *egv1a1.EnvoyProxy) { + envoy.Spec = egv1a1.EnvoyProxySpec{ + Provider: &egv1a1.EnvoyProxyProvider{ + Type: egv1a1.ProviderTypeKubernetes, + Kubernetes: &egv1a1.EnvoyProxyKubernetesProvider{ + EnvoyHpa: &egv1a1.KubernetesHorizontalPodAutoscalerSpec{ + MinReplicas: ptr.To[int32](5), + MaxReplicas: ptr.To[int32](10), + }, + }, + }, + } + }, + }, } for _, tc := range cases {