Skip to content
This repository has been archived by the owner on Feb 27, 2023. It is now read-only.

Commit

Permalink
Adds Support for Managing AWS Load Balancer Type (#366)
Browse files Browse the repository at this point in the history 
* Adds AWS Load Balancer Type API
* Implements AWS NLB Load Balancer Type

Signed-off-by: Daneyon Hansen <daneyonhansen@gmail.com>
  • Loading branch information
@danehans
danehans authored May 26, 2021
1 parent 2468c8e commit 33ef23c
Show file tree
Hide file tree
Showing 5 changed files with 151 additions and 35 deletions.
42 changes: 42 additions & 0 deletions api/v1alpha1/contour_types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -289,6 +289,15 @@ type ProviderLoadBalancerParameters struct {
// +unionDiscriminator
// +kubebuilder:default=AWS
Type LoadBalancerProviderType `json:"type,omitempty"`

// AWS provides configuration settings that are specific to AWS
// load balancers.
//
// If empty, defaults will be applied. See specific aws fields for
// details about their defaults.
//
// +optional
AWS *AWSLoadBalancerParameters `json:"aws,omitempty"`
}

// LoadBalancerProviderType is the underlying infrastructure provider for the
Expand All @@ -303,6 +312,39 @@ const (
GCPLoadBalancerProvider LoadBalancerProviderType = "GCP"
)

// AWSLoadBalancerParameters provides configuration settings that are specific to
// AWS load balancers.
type AWSLoadBalancerParameters struct {
// Type is the type of AWS load balancer to manage.
//
// Valid values are:
//
// * "Classic": A Classic load balancer makes routing decisions at either the
// transport layer (TCP/SSL) or the application layer (HTTP/HTTPS). See
// the following for additional details:
//
// https://docs.aws.amazon.com/AmazonECS/latest/developerguide/load-balancer-types.html#clb
//
// * "NLB": A Network load balancer makes routing decisions at the transport
// layer (TCP/SSL). See the following for additional details:
//
// https://docs.aws.amazon.com/AmazonECS/latest/developerguide/load-balancer-types.html#nlb
//
// If unset, defaults to "Classic".
//
// +kubebuilder:default=Classic
Type AWSLoadBalancerType `json:"type,omitempty"`
}

// AWSLoadBalancerType is the type of AWS load balancer to manage.
// +kubebuilder:validation:Enum=Classic;NLB
type AWSLoadBalancerType string

const (
AWSClassicLoadBalancer AWSLoadBalancerType = "Classic"
AWSNetworkLoadBalancer AWSLoadBalancerType = "NLB"
)

// NodePort is the schema to specify a network port for a NodePort Service.
type NodePort struct {
// Name is an IANA_SVC_NAME within the NodePort Service.
Expand Down
24 changes: 22 additions & 2 deletions api/v1alpha1/zz_generated.deepcopy.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

24 changes: 24 additions & 0 deletions config/crd/bases/operator.projectcontour.io_contours.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -175,6 +175,30 @@ spec:
information specific to the underlying infrastructure
provider.
properties:
aws:
description: "AWS provides configuration settings
that are specific to AWS load balancers. \n If empty,
defaults will be applied. See specific aws fields
for details about their defaults."
properties:
type:
default: Classic
description: "Type is the type of AWS load balancer
to manage. \n Valid values are: \n * \"Classic\":
A Classic load balancer makes routing decisions
at either the transport layer (TCP/SSL) or
the application layer (HTTP/HTTPS). See the
following for additional details: \n https://docs.aws.amazon.com/AmazonECS/latest/developerguide/load-balancer-types.html#clb
\n * \"NLB\": A Network load balancer makes
routing decisions at the transport layer (TCP/SSL).
See the following for additional details: \n
\ https://docs.aws.amazon.com/AmazonECS/latest/developerguide/load-balancer-types.html#nlb
\n If unset, defaults to \"Classic\"."
enum:
- Classic
- NLB
type: string
type: object
type:
default: AWS
description: Type is the underlying infrastructure
Expand Down
63 changes: 37 additions & 26 deletions internal/objects/service/service.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,12 +49,10 @@ const (
// TODO [danehans]: Make proxy protocol configurable or automatically enabled. See
// https://github.com/projectcontour/contour-operator/issues/49 for details.
awsLbBackendProtoAnnotation = "service.beta.kubernetes.io/aws-load-balancer-backend-protocol"
// awsProviderType is the name of the Amazon Web Services provider.
awsProviderType = "AWS"
// azureProviderType is the name of the Microsoft Azure provider.
azureProviderType = "Azure"
// gcpProviderType is the name of the Google Cloud Platform provider.
gcpProviderType = "GCP"
// awsLBTypeAnnotation is a Service annotation used to specify an AWS load
// balancer type. See the following for additional details:
// https://kubernetes.io/docs/concepts/services-networking/service/#aws-nlb-support
awsLBTypeAnnotation = "service.beta.kubernetes.io/aws-load-balancer-type"
// awsInternalLBAnnotation is the annotation used on a service to specify an AWS
// load balancer as being internal.
awsInternalLBAnnotation = "service.beta.kubernetes.io/aws-load-balancer-internal"
Expand All @@ -77,31 +75,20 @@ const (
)

var (
// LbAnnotations maps cloud providers to the provider's annotation
// key/value pair used for managing a load balancer. For additional
// details see:
// https://kubernetes.io/docs/concepts/services-networking/service/#internal-load-balancer
//
LbAnnotations = map[operatorv1alpha1.LoadBalancerProviderType]map[string]string{
awsProviderType: {
awsLbBackendProtoAnnotation: "tcp",
},
}

// InternalLBAnnotations maps cloud providers to the provider's annotation
// key/value pair used for managing an internal load balancer. For additional
// details see:
// https://kubernetes.io/docs/concepts/services-networking/service/#internal-load-balancer
//
InternalLBAnnotations = map[operatorv1alpha1.LoadBalancerProviderType]map[string]string{
awsProviderType: {
awsInternalLBAnnotation: "0.0.0.0/0",
operatorv1alpha1.AWSLoadBalancerProvider: {
awsInternalLBAnnotation: "true",
},
azureProviderType: {
operatorv1alpha1.AzureLoadBalancerProvider: {
// Azure load balancers are not customizable and are set to (2 fail @ 5s interval, 2 healthy)
azureInternalLBAnnotation: "true",
},
gcpProviderType: {
operatorv1alpha1.GCPLoadBalancerProvider: {
gcpLBTypeAnnotation: "Internal",
},
}
Expand Down Expand Up @@ -252,6 +239,16 @@ func DesiredEnvoyService(contour *operatorv1alpha1.Contour) *corev1.Service {
SessionAffinity: corev1.ServiceAffinityNone,
},
}

// Add the NLB annotation if specified by AWS provider parameters.
if nlbNeeded(&contour.Spec) {
svc.Annotations[awsLBTypeAnnotation] = "nlb"
}
// Add the TCP backend protocol annotation for AWS classic load balancers.
if backendTCPNeeded(&contour.Spec) {
svc.Annotations[awsLbBackendProtoAnnotation] = "tcp"
}

epType := contour.Spec.NetworkPublishing.Envoy.Type
if epType == operatorv1alpha1.LoadBalancerServicePublishingType ||
epType == operatorv1alpha1.NodePortServicePublishingType {
Expand All @@ -260,13 +257,9 @@ func DesiredEnvoyService(contour *operatorv1alpha1.Contour) *corev1.Service {
switch epType {
case operatorv1alpha1.LoadBalancerServicePublishingType:
svc.Spec.Type = corev1.ServiceTypeLoadBalancer
provider := contour.Spec.NetworkPublishing.Envoy.LoadBalancer.ProviderParameters.Type
lbAnnotations := LbAnnotations[provider]
for name, value := range lbAnnotations {
svc.Annotations[name] = value
}
isInternal := contour.Spec.NetworkPublishing.Envoy.LoadBalancer.Scope == operatorv1alpha1.InternalLoadBalancer
if isInternal {
provider := contour.Spec.NetworkPublishing.Envoy.LoadBalancer.ProviderParameters.Type
internalAnnotations := InternalLBAnnotations[provider]
for name, value := range internalAnnotations {
svc.Annotations[name] = value
Expand Down Expand Up @@ -368,3 +361,21 @@ func updateEnvoyServiceIfNeeded(ctx context.Context, cli client.Client, contour
}
return nil
}

// nlbNeeded returns true if the "service.beta.kubernetes.io/aws-load-balancer-type"
// annotation is needed based on the provided spec.
func nlbNeeded(spec *operatorv1alpha1.ContourSpec) bool {
return spec.NetworkPublishing.Envoy.Type == operatorv1alpha1.LoadBalancerServicePublishingType &&
spec.NetworkPublishing.Envoy.LoadBalancer.ProviderParameters.Type == operatorv1alpha1.AWSLoadBalancerProvider &&
spec.NetworkPublishing.Envoy.LoadBalancer.ProviderParameters.AWS != nil &&
spec.NetworkPublishing.Envoy.LoadBalancer.ProviderParameters.AWS.Type == operatorv1alpha1.AWSNetworkLoadBalancer
}

// backendTCPNeeded returns true if the "service.beta.kubernetes.io/aws-load-balancer-backend-protocol"
// annotation is needed based on the provided spec.
func backendTCPNeeded(spec *operatorv1alpha1.ContourSpec) bool {
return spec.NetworkPublishing.Envoy.Type == operatorv1alpha1.LoadBalancerServicePublishingType &&
spec.NetworkPublishing.Envoy.LoadBalancer.ProviderParameters.Type == operatorv1alpha1.AWSLoadBalancerProvider &&
(spec.NetworkPublishing.Envoy.LoadBalancer.ProviderParameters.AWS == nil ||
spec.NetworkPublishing.Envoy.LoadBalancer.ProviderParameters.AWS.Type == operatorv1alpha1.AWSClassicLoadBalancer)
}
33 changes: 26 additions & 7 deletions internal/objects/service/service_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,19 +81,29 @@ func checkServiceHasPortProtocol(t *testing.T, svc *corev1.Service, protocol cor
t.Errorf("service is missing port protocol %q", protocol)
}

func checkServiceHasAnnotation(t *testing.T, svc *corev1.Service, key string) {
func checkServiceHasAnnotation(t *testing.T, svc *corev1.Service, expect bool, key string) {
t.Helper()

if svc.Annotations == nil {
if !expect {
return
}
t.Errorf("service is missing annotations")
}

found := false
for k := range svc.Annotations {
if k == key {
return
found = true
}
}

t.Errorf("service is missing annotation %q", key)
if found && !expect {
t.Errorf("service contains annotation %q", key)
}
if !found && expect {
t.Errorf("service is missing annotation %q", key)
}
}

func checkServiceHasType(t *testing.T, svc *corev1.Service, svcType corev1.ServiceType) {
Expand Down Expand Up @@ -161,16 +171,25 @@ func TestDesiredEnvoyService(t *testing.T) {
svc = DesiredEnvoyService(cntr)
checkServiceHasType(t, svc, corev1.ServiceTypeLoadBalancer)
checkServiceHasExternalTrafficPolicy(t, svc, corev1.ServiceExternalTrafficPolicyTypeLocal)
checkServiceHasAnnotation(t, svc, awsLbBackendProtoAnnotation)
checkServiceHasAnnotation(t, svc, true, awsLbBackendProtoAnnotation)
// Check AWS NLB load balancer type.
awsParams := operatorv1alpha1.ProviderLoadBalancerParameters{
Type: operatorv1alpha1.AWSLoadBalancerProvider,
AWS: &operatorv1alpha1.AWSLoadBalancerParameters{Type: operatorv1alpha1.AWSNetworkLoadBalancer},
}
cntr.Spec.NetworkPublishing.Envoy.LoadBalancer.ProviderParameters = awsParams
svc = DesiredEnvoyService(cntr)
checkServiceHasAnnotation(t, svc, true, awsLBTypeAnnotation)
checkServiceHasAnnotation(t, svc, false, awsLbBackendProtoAnnotation)
cntr.Spec.NetworkPublishing.Envoy.LoadBalancer.Scope = operatorv1alpha1.InternalLoadBalancer
svc = DesiredEnvoyService(cntr)
checkServiceHasAnnotation(t, svc, awsInternalLBAnnotation)
checkServiceHasAnnotation(t, svc, true, awsInternalLBAnnotation)
cntr.Spec.NetworkPublishing.Envoy.LoadBalancer.ProviderParameters.Type = operatorv1alpha1.AzureLoadBalancerProvider
svc = DesiredEnvoyService(cntr)
checkServiceHasAnnotation(t, svc, azureInternalLBAnnotation)
checkServiceHasAnnotation(t, svc, true, azureInternalLBAnnotation)
cntr.Spec.NetworkPublishing.Envoy.LoadBalancer.ProviderParameters.Type = operatorv1alpha1.GCPLoadBalancerProvider
svc = DesiredEnvoyService(cntr)
checkServiceHasAnnotation(t, svc, gcpLBTypeAnnotation)
checkServiceHasAnnotation(t, svc, true, gcpLBTypeAnnotation)
// Set network publishing type to ClusterIPService and verify the service type is as expected.
cntr.Spec.NetworkPublishing.Envoy.Type = operatorv1alpha1.ClusterIPServicePublishingType
svc = DesiredEnvoyService(cntr)
Expand Down

0 comments on commit 33ef23c

Please sign in to comment.