Skip to content

Commit cb303f3

Browse files
deliahuRobertLucian
deliahu
and
RobertLucian
authored
Support private load balancers on GCP (#1786)
* Support private networking on GCP * Remove node visibility configuration Co-authored-by: Robert Lucian Chiriac <robert.lucian.chiriac@gmail.com>
1 parent 61f6ed9 commit cb303f3

File tree

4 files changed

+64
-23
lines changed

4 files changed

+64
-23
lines changed

‎docs/clusters/gcp/install.md

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,13 @@ max_instances: 5
4545

4646
# the name of the subnetwork in which to create your cluster
4747
# subnet: default
48+
49+
# API load balancer scheme [internet-facing | internal]
50+
api_load_balancer_scheme: internet-facing
51+
52+
# operator load balancer scheme [internet-facing | internal]
53+
# note: if using "internal", you must be within the cluster's VPC or configure VPC Peering to connect your CLI to your cluster operator
54+
operator_load_balancer_scheme: internet-facing
4855
```
4956
5057
The docker images used by the Cortex cluster can also be overridden, although this is not common. They can be configured by adding any of these keys to your cluster configuration file (default values are shown):

‎manager/install.sh

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -706,12 +706,14 @@ function validate_cortex_gcp() {
706706
api_load_balancer_endpoint=$(kubectl -n=istio-system get service ingressgateway-apis -o json | tr -d '[:space:]' | sed 's/.*{\"ip\":\"\(.*\)\".*/\1/')
707707
fi
708708

709-
operator_endpoint_reachable="false" # don't cache this result
710-
if ! curl --max-time 3 "${operator_endpoint}/verifycortex" >/dev/null 2>&1; then
711-
success_cycles=0
712-
continue
709+
if [ "$CORTEX_OPERATOR_LOAD_BALANCER_SCHEME" == "internet-facing" ]; then
710+
operator_endpoint_reachable="false" # don't cache this result
711+
if ! curl --max-time 3 "${operator_endpoint}/verifycortex" >/dev/null 2>&1; then
712+
success_cycles=0
713+
continue
714+
fi
715+
operator_endpoint_reachable="true"
713716
fi
714-
operator_endpoint_reachable="true"
715717

716718
if [[ $success_cycles -lt 1 ]]; then
717719
((success_cycles++))

‎manager/manifests/istio.yaml.j2

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,9 @@ spec:
4747
{% if config.get('operator_load_balancer_scheme') == 'internal' %}
4848
service.beta.kubernetes.io/aws-load-balancer-internal: "true"
4949
{% endif %}
50+
{% elif env['CORTEX_PROVIDER'] == "gcp" and config.get('operator_load_balancer_scheme') == 'internal' %}
51+
serviceAnnotations:
52+
cloud.google.com/load-balancer-type: "Internal"
5053
{% endif %}
5154
service:
5255
type: LoadBalancer
@@ -106,6 +109,9 @@ spec:
106109
{% if config.get('ssl_certificate_arn', '') != '' %}
107110
service.beta.kubernetes.io/aws-load-balancer-ssl-cert: "{{ config['ssl_certificate_arn'] }}"
108111
{% endif %}
112+
{% elif env['CORTEX_PROVIDER'] == "gcp" and config.get('api_load_balancer_scheme') == 'internal' %}
113+
serviceAnnotations:
114+
cloud.google.com/load-balancer-type: "Internal"
109115
{% endif %}
110116
service:
111117
type: LoadBalancer

‎pkg/types/clusterconfig/cluster_config_gcp.go

Lines changed: 44 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -33,24 +33,26 @@ import (
3333
)
3434

3535
type GCPConfig struct {
36-
Provider types.ProviderType `json:"provider" yaml:"provider"`
37-
Project *string `json:"project" yaml:"project"`
38-
Zone *string `json:"zone" yaml:"zone"`
39-
InstanceType *string `json:"instance_type" yaml:"instance_type"`
40-
AcceleratorType *string `json:"accelerator_type" yaml:"accelerator_type"`
41-
Network *string `json:"network" yaml:"network"`
42-
Subnet *string `json:"subnet" yaml:"subnet"`
43-
MinInstances *int64 `json:"min_instances" yaml:"min_instances"`
44-
MaxInstances *int64 `json:"max_instances" yaml:"max_instances"`
45-
ClusterName string `json:"cluster_name" yaml:"cluster_name"`
46-
Telemetry bool `json:"telemetry" yaml:"telemetry"`
47-
ImageOperator string `json:"image_operator" yaml:"image_operator"`
48-
ImageManager string `json:"image_manager" yaml:"image_manager"`
49-
ImageDownloader string `json:"image_downloader" yaml:"image_downloader"`
50-
ImageFluentBit string `json:"image_fluent_bit" yaml:"image_fluent_bit"`
51-
ImageIstioProxy string `json:"image_istio_proxy" yaml:"image_istio_proxy"`
52-
ImageIstioPilot string `json:"image_istio_pilot" yaml:"image_istio_pilot"`
53-
ImageGooglePause string `json:"image_google_pause" yaml:"image_google_pause"`
36+
Provider types.ProviderType `json:"provider" yaml:"provider"`
37+
Project *string `json:"project" yaml:"project"`
38+
Zone *string `json:"zone" yaml:"zone"`
39+
InstanceType *string `json:"instance_type" yaml:"instance_type"`
40+
AcceleratorType *string `json:"accelerator_type" yaml:"accelerator_type"`
41+
Network *string `json:"network" yaml:"network"`
42+
Subnet *string `json:"subnet" yaml:"subnet"`
43+
APILoadBalancerScheme LoadBalancerScheme `json:"api_load_balancer_scheme" yaml:"api_load_balancer_scheme"`
44+
OperatorLoadBalancerScheme LoadBalancerScheme `json:"operator_load_balancer_scheme" yaml:"operator_load_balancer_scheme"`
45+
MinInstances *int64 `json:"min_instances" yaml:"min_instances"`
46+
MaxInstances *int64 `json:"max_instances" yaml:"max_instances"`
47+
ClusterName string `json:"cluster_name" yaml:"cluster_name"`
48+
Telemetry bool `json:"telemetry" yaml:"telemetry"`
49+
ImageOperator string `json:"image_operator" yaml:"image_operator"`
50+
ImageManager string `json:"image_manager" yaml:"image_manager"`
51+
ImageDownloader string `json:"image_downloader" yaml:"image_downloader"`
52+
ImageFluentBit string `json:"image_fluent_bit" yaml:"image_fluent_bit"`
53+
ImageIstioProxy string `json:"image_istio_proxy" yaml:"image_istio_proxy"`
54+
ImageIstioPilot string `json:"image_istio_pilot" yaml:"image_istio_pilot"`
55+
ImageGooglePause string `json:"image_google_pause" yaml:"image_google_pause"`
5456
}
5557

5658
type InternalGCPConfig struct {
@@ -148,6 +150,26 @@ var UserGCPValidation = &cr.StructValidation{
148150
AllowExplicitNull: true,
149151
},
150152
},
153+
{
154+
StructField: "APILoadBalancerScheme",
155+
StringValidation: &cr.StringValidation{
156+
AllowedValues: LoadBalancerSchemeStrings(),
157+
Default: InternetFacingLoadBalancerScheme.String(),
158+
},
159+
Parser: func(str string) (interface{}, error) {
160+
return LoadBalancerSchemeFromString(str), nil
161+
},
162+
},
163+
{
164+
StructField: "OperatorLoadBalancerScheme",
165+
StringValidation: &cr.StringValidation{
166+
AllowedValues: LoadBalancerSchemeStrings(),
167+
Default: InternetFacingLoadBalancerScheme.String(),
168+
},
169+
Parser: func(str string) (interface{}, error) {
170+
return LoadBalancerSchemeFromString(str), nil
171+
},
172+
},
151173
{
152174
StructField: "MinInstances",
153175
Int64PtrValidation: &cr.Int64PtrValidation{
@@ -501,6 +523,8 @@ func (cc *GCPConfig) UserTable() table.KeyValuePairs {
501523
if cc.Subnet != nil {
502524
items.Add(SubnetUserKey, *cc.Subnet)
503525
}
526+
items.Add(APILoadBalancerSchemeUserKey, cc.APILoadBalancerScheme)
527+
items.Add(OperatorLoadBalancerSchemeUserKey, cc.OperatorLoadBalancerScheme)
504528
items.Add(TelemetryUserKey, cc.Telemetry)
505529
items.Add(ImageOperatorUserKey, cc.ImageOperator)
506530
items.Add(ImageManagerUserKey, cc.ImageManager)
@@ -536,6 +560,8 @@ func (cc *GCPConfig) TelemetryEvent() map[string]interface{} {
536560
if cc.Subnet != nil {
537561
event["subnet._is_defined"] = true
538562
}
563+
event["api_load_balancer_scheme"] = cc.APILoadBalancerScheme
564+
event["operator_load_balancer_scheme"] = cc.OperatorLoadBalancerScheme
539565
if cc.MinInstances != nil {
540566
event["min_instances._is_defined"] = true
541567
event["min_instances"] = *cc.MinInstances

0 commit comments

Comments
 (0)