Fix AWS cluster when scaling up (#1841)

Author: RobertLucian

manager/manifests/istio.yaml.j2

@@ -42,6 +42,7 @@ spec:
           {% if env['CORTEX_PROVIDER'] == "aws" %}
           serviceAnnotations:
             service.beta.kubernetes.io/aws-load-balancer-type: "nlb"
+            service.beta.kubernetes.io/aws-load-balancer-cross-zone-load-balancing-enabled: "true"
             service.beta.kubernetes.io/aws-load-balancer-backend-protocol: "tcp"
             service.beta.kubernetes.io/aws-load-balancer-additional-resource-tags: "{{ env['CORTEX_OPERATOR_LOAD_BALANCER_TAGS'] }}"
             {% if config.get('operator_load_balancer_scheme') == 'internal' %}
@@ -53,7 +54,7 @@ spec:
           {% endif %}
           service:
             type: LoadBalancer
-            externalTrafficPolicy: Local # https://medium.com/pablo-perez/k8s-externaltrafficpolicy-local-or-cluster-40b259a19404, https://www.asykim.com/blog/deep-dive-into-kubernetes-external-traffic-policies
+            externalTrafficPolicy: Cluster # https://medium.com/pablo-perez/k8s-externaltrafficpolicy-local-or-cluster-40b259a19404, https://www.asykim.com/blog/deep-dive-into-kubernetes-external-traffic-policies
             selector:
               app: operator-istio-gateway
               istio: ingressgateway-operator
@@ -100,6 +101,7 @@ spec:
           {% if env['CORTEX_PROVIDER'] == "aws" %}
           serviceAnnotations:
             service.beta.kubernetes.io/aws-load-balancer-type: "nlb"
+            service.beta.kubernetes.io/aws-load-balancer-cross-zone-load-balancing-enabled: "true"
             service.beta.kubernetes.io/aws-load-balancer-additional-resource-tags: "{{ env['CORTEX_API_LOAD_BALANCER_TAGS'] }}"
             service.beta.kubernetes.io/aws-load-balancer-backend-protocol: "tcp"
             service.beta.kubernetes.io/aws-load-balancer-ssl-ports: "https"  # "https" is the name of the https port below
@@ -115,7 +117,7 @@ spec:
           {% endif %}
           service:
             type: LoadBalancer
-            externalTrafficPolicy: Local # https://medium.com/pablo-perez/k8s-externaltrafficpolicy-local-or-cluster-40b259a19404, https://www.asykim.com/blog/deep-dive-into-kubernetes-external-traffic-policies
+            externalTrafficPolicy: Cluster # https://medium.com/pablo-perez/k8s-externaltrafficpolicy-local-or-cluster-40b259a19404, https://www.asykim.com/blog/deep-dive-into-kubernetes-external-traffic-policies
             selector:
               app: apis-istio-gateway
               istio: ingressgateway-apis