Support private load balancers on GCP

docs/clusters/gcp/install.md

@@ -45,6 +45,13 @@ max_instances: 5
 
 # the name of the subnetwork in which to create your cluster
 # subnet: default
+
+# API load balancer scheme [internet-facing | internal]
+api_load_balancer_scheme: internet-facing
+
+# operator load balancer scheme [internet-facing | internal]
+# note: if using "internal", you must be within the cluster's VPC or configure VPC Peering to connect your CLI to your cluster operator
+operator_load_balancer_scheme: internet-facing
 ```
 
 The docker images used by the Cortex cluster can also be overridden, although this is not common. They can be configured by adding any of these keys to your cluster configuration file (default values are shown):

manager/install.sh

@@ -706,12 +706,14 @@ function validate_cortex_gcp() {
       api_load_balancer_endpoint=$(kubectl -n=istio-system get service ingressgateway-apis -o json | tr -d '[:space:]' | sed 's/.*{\"ip\":\"\(.*\)\".*/\1/')
     fi
 
-    operator_endpoint_reachable="false"  # don't cache this result
-    if ! curl --max-time 3 "${operator_endpoint}/verifycortex" >/dev/null 2>&1; then
-      success_cycles=0
-      continue
+    if [ "$CORTEX_OPERATOR_LOAD_BALANCER_SCHEME" == "internet-facing" ]; then
+      operator_endpoint_reachable="false"  # don't cache this result
+      if ! curl --max-time 3 "${operator_endpoint}/verifycortex" >/dev/null 2>&1; then
+        success_cycles=0
+        continue
+      fi
+      operator_endpoint_reachable="true"
     fi
-    operator_endpoint_reachable="true"
 
     if [[ $success_cycles -lt 1 ]]; then
       ((success_cycles++))

manager/manifests/istio.yaml.j2

@@ -47,6 +47,9 @@ spec:
             {% if config.get('operator_load_balancer_scheme') == 'internal' %}
             service.beta.kubernetes.io/aws-load-balancer-internal: "true"
             {% endif %}
+          {% elif env['CORTEX_PROVIDER'] == "gcp" and config.get('operator_load_balancer_scheme') == 'internal' %}
+          serviceAnnotations:
+            cloud.google.com/load-balancer-type: "Internal"
           {% endif %}
           service:
             type: LoadBalancer
@@ -106,6 +109,9 @@ spec:
             {% if config.get('ssl_certificate_arn', '') != '' %}
             service.beta.kubernetes.io/aws-load-balancer-ssl-cert: "{{ config['ssl_certificate_arn'] }}"
             {% endif %}
+          {% elif env['CORTEX_PROVIDER'] == "gcp" and config.get('api_load_balancer_scheme') == 'internal' %}
+          serviceAnnotations:
+            cloud.google.com/load-balancer-type: "Internal"
           {% endif %}
           service:
             type: LoadBalancer

pkg/types/clusterconfig/cluster_config_gcp.go

@@ -33,24 +33,26 @@ import (
 )
 
 type GCPConfig struct {
-	Provider         types.ProviderType `json:"provider" yaml:"provider"`
-	Project          *string            `json:"project" yaml:"project"`
-	Zone             *string            `json:"zone" yaml:"zone"`
-	InstanceType     *string            `json:"instance_type" yaml:"instance_type"`
-	AcceleratorType  *string            `json:"accelerator_type" yaml:"accelerator_type"`
-	Network          *string            `json:"network" yaml:"network"`
-	Subnet           *string            `json:"subnet" yaml:"subnet"`
-	MinInstances     *int64             `json:"min_instances" yaml:"min_instances"`
-	MaxInstances     *int64             `json:"max_instances" yaml:"max_instances"`
-	ClusterName      string             `json:"cluster_name" yaml:"cluster_name"`
-	Telemetry        bool               `json:"telemetry" yaml:"telemetry"`
-	ImageOperator    string             `json:"image_operator" yaml:"image_operator"`
-	ImageManager     string             `json:"image_manager" yaml:"image_manager"`
-	ImageDownloader  string             `json:"image_downloader" yaml:"image_downloader"`
-	ImageFluentBit   string             `json:"image_fluent_bit" yaml:"image_fluent_bit"`
-	ImageIstioProxy  string             `json:"image_istio_proxy" yaml:"image_istio_proxy"`
-	ImageIstioPilot  string             `json:"image_istio_pilot" yaml:"image_istio_pilot"`
-	ImageGooglePause string             `json:"image_google_pause" yaml:"image_google_pause"`
+	Provider                   types.ProviderType `json:"provider" yaml:"provider"`
+	Project                    *string            `json:"project" yaml:"project"`
+	Zone                       *string            `json:"zone" yaml:"zone"`
+	InstanceType               *string            `json:"instance_type" yaml:"instance_type"`
+	AcceleratorType            *string            `json:"accelerator_type" yaml:"accelerator_type"`
+	Network                    *string            `json:"network" yaml:"network"`
+	Subnet                     *string            `json:"subnet" yaml:"subnet"`
+	APILoadBalancerScheme      LoadBalancerScheme `json:"api_load_balancer_scheme" yaml:"api_load_balancer_scheme"`
+	OperatorLoadBalancerScheme LoadBalancerScheme `json:"operator_load_balancer_scheme" yaml:"operator_load_balancer_scheme"`
+	MinInstances               *int64             `json:"min_instances" yaml:"min_instances"`
+	MaxInstances               *int64             `json:"max_instances" yaml:"max_instances"`
+	ClusterName                string             `json:"cluster_name" yaml:"cluster_name"`
+	Telemetry                  bool               `json:"telemetry" yaml:"telemetry"`
+	ImageOperator              string             `json:"image_operator" yaml:"image_operator"`
+	ImageManager               string             `json:"image_manager" yaml:"image_manager"`
+	ImageDownloader            string             `json:"image_downloader" yaml:"image_downloader"`
+	ImageFluentBit             string             `json:"image_fluent_bit" yaml:"image_fluent_bit"`
+	ImageIstioProxy            string             `json:"image_istio_proxy" yaml:"image_istio_proxy"`
+	ImageIstioPilot            string             `json:"image_istio_pilot" yaml:"image_istio_pilot"`
+	ImageGooglePause           string             `json:"image_google_pause" yaml:"image_google_pause"`
 }
 
 type InternalGCPConfig struct {
@@ -148,6 +150,26 @@ var UserGCPValidation = &cr.StructValidation{
 				AllowExplicitNull: true,
 			},
 		},
+		{
+			StructField: "APILoadBalancerScheme",
+			StringValidation: &cr.StringValidation{
+				AllowedValues: LoadBalancerSchemeStrings(),
+				Default:       InternetFacingLoadBalancerScheme.String(),
+			},
+			Parser: func(str string) (interface{}, error) {
+				return LoadBalancerSchemeFromString(str), nil
+			},
+		},
+		{
+			StructField: "OperatorLoadBalancerScheme",
+			StringValidation: &cr.StringValidation{
+				AllowedValues: LoadBalancerSchemeStrings(),
+				Default:       InternetFacingLoadBalancerScheme.String(),
+			},
+			Parser: func(str string) (interface{}, error) {
+				return LoadBalancerSchemeFromString(str), nil
+			},
+		},
 		{
 			StructField: "MinInstances",
 			Int64PtrValidation: &cr.Int64PtrValidation{
@@ -501,6 +523,8 @@ func (cc *GCPConfig) UserTable() table.KeyValuePairs {
 	if cc.Subnet != nil {
 		items.Add(SubnetUserKey, *cc.Subnet)
 	}
+	items.Add(APILoadBalancerSchemeUserKey, cc.APILoadBalancerScheme)
+	items.Add(OperatorLoadBalancerSchemeUserKey, cc.OperatorLoadBalancerScheme)
 	items.Add(TelemetryUserKey, cc.Telemetry)
 	items.Add(ImageOperatorUserKey, cc.ImageOperator)
 	items.Add(ImageManagerUserKey, cc.ImageManager)
@@ -536,6 +560,8 @@ func (cc *GCPConfig) TelemetryEvent() map[string]interface{} {
 	if cc.Subnet != nil {
 		event["subnet._is_defined"] = true
 	}
+	event["api_load_balancer_scheme"] = cc.APILoadBalancerScheme
+	event["operator_load_balancer_scheme"] = cc.OperatorLoadBalancerScheme
 	if cc.MinInstances != nil {
 		event["min_instances._is_defined"] = true
 		event["min_instances"] = *cc.MinInstances