diff --git a/api/v1alpha1/contour_types.go b/api/v1alpha1/contour_types.go index 3c181948..3efe892e 100644 --- a/api/v1alpha1/contour_types.go +++ b/api/v1alpha1/contour_types.go @@ -289,6 +289,15 @@ type ProviderLoadBalancerParameters struct { // +unionDiscriminator // +kubebuilder:default=AWS Type LoadBalancerProviderType `json:"type,omitempty"` + + // AWS provides configuration settings that are specific to AWS + // load balancers. + // + // If empty, defaults will be applied. See specific aws fields for + // details about their defaults. + // + // +optional + AWS *AWSLoadBalancerParameters `json:"aws,omitempty"` } // LoadBalancerProviderType is the underlying infrastructure provider for the @@ -303,6 +312,39 @@ const ( GCPLoadBalancerProvider LoadBalancerProviderType = "GCP" ) +// AWSLoadBalancerParameters provides configuration settings that are specific to +// AWS load balancers. +type AWSLoadBalancerParameters struct { + // Type is the type of AWS load balancer to manage. + // + // Valid values are: + // + // * "Classic": A Classic load balancer makes routing decisions at either the + // transport layer (TCP/SSL) or the application layer (HTTP/HTTPS). See + // the following for additional details: + // + // https://docs.aws.amazon.com/AmazonECS/latest/developerguide/load-balancer-types.html#clb + // + // * "NLB": A Network load balancer makes routing decisions at the transport + // layer (TCP/SSL). See the following for additional details: + // + // https://docs.aws.amazon.com/AmazonECS/latest/developerguide/load-balancer-types.html#nlb + // + // If unset, defaults to "Classic". + // + // +kubebuilder:default=Classic + Type AWSLoadBalancerType `json:"type,omitempty"` +} + +// AWSLoadBalancerType is the type of AWS load balancer to manage. +// +kubebuilder:validation:Enum=Classic;NLB +type AWSLoadBalancerType string + +const ( + AWSClassicLoadBalancer AWSLoadBalancerType = "Classic" + AWSNetworkLoadBalancer AWSLoadBalancerType = "NLB" +) + // NodePort is the schema to specify a network port for a NodePort Service. type NodePort struct { // Name is an IANA_SVC_NAME within the NodePort Service. diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index df12ead1..093f7523 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -22,6 +22,21 @@ import ( runtime "k8s.io/apimachinery/pkg/runtime" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AWSLoadBalancerParameters) DeepCopyInto(out *AWSLoadBalancerParameters) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AWSLoadBalancerParameters. +func (in *AWSLoadBalancerParameters) DeepCopy() *AWSLoadBalancerParameters { + if in == nil { + return nil + } + out := new(AWSLoadBalancerParameters) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ContainerPort) DeepCopyInto(out *ContainerPort) { *out = *in @@ -148,7 +163,7 @@ func (in *ContourStatus) DeepCopy() *ContourStatus { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *EnvoyNetworkPublishing) DeepCopyInto(out *EnvoyNetworkPublishing) { *out = *in - out.LoadBalancer = in.LoadBalancer + in.LoadBalancer.DeepCopyInto(&out.LoadBalancer) if in.NodePorts != nil { in, out := &in.NodePorts, &out.NodePorts *out = make([]NodePort, len(*in)) @@ -176,7 +191,7 @@ func (in *EnvoyNetworkPublishing) DeepCopy() *EnvoyNetworkPublishing { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *LoadBalancerStrategy) DeepCopyInto(out *LoadBalancerStrategy) { *out = *in - out.ProviderParameters = in.ProviderParameters + in.ProviderParameters.DeepCopyInto(&out.ProviderParameters) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LoadBalancerStrategy. @@ -243,6 +258,11 @@ func (in *NodePort) DeepCopy() *NodePort { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ProviderLoadBalancerParameters) DeepCopyInto(out *ProviderLoadBalancerParameters) { *out = *in + if in.AWS != nil { + in, out := &in.AWS, &out.AWS + *out = new(AWSLoadBalancerParameters) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProviderLoadBalancerParameters. diff --git a/config/crd/bases/operator.projectcontour.io_contours.yaml b/config/crd/bases/operator.projectcontour.io_contours.yaml index b30fcfc4..dee66c51 100644 --- a/config/crd/bases/operator.projectcontour.io_contours.yaml +++ b/config/crd/bases/operator.projectcontour.io_contours.yaml @@ -175,6 +175,30 @@ spec: information specific to the underlying infrastructure provider. properties: + aws: + description: "AWS provides configuration settings + that are specific to AWS load balancers. \n If empty, + defaults will be applied. See specific aws fields + for details about their defaults." + properties: + type: + default: Classic + description: "Type is the type of AWS load balancer + to manage. \n Valid values are: \n * \"Classic\": + A Classic load balancer makes routing decisions + at either the transport layer (TCP/SSL) or + the application layer (HTTP/HTTPS). See the + following for additional details: \n https://docs.aws.amazon.com/AmazonECS/latest/developerguide/load-balancer-types.html#clb + \n * \"NLB\": A Network load balancer makes + routing decisions at the transport layer (TCP/SSL). + See the following for additional details: \n + \ https://docs.aws.amazon.com/AmazonECS/latest/developerguide/load-balancer-types.html#nlb + \n If unset, defaults to \"Classic\"." + enum: + - Classic + - NLB + type: string + type: object type: default: AWS description: Type is the underlying infrastructure diff --git a/internal/objects/service/service.go b/internal/objects/service/service.go index 8671f845..58658de6 100644 --- a/internal/objects/service/service.go +++ b/internal/objects/service/service.go @@ -49,12 +49,10 @@ const ( // TODO [danehans]: Make proxy protocol configurable or automatically enabled. See // https://github.com/projectcontour/contour-operator/issues/49 for details. awsLbBackendProtoAnnotation = "service.beta.kubernetes.io/aws-load-balancer-backend-protocol" - // awsProviderType is the name of the Amazon Web Services provider. - awsProviderType = "AWS" - // azureProviderType is the name of the Microsoft Azure provider. - azureProviderType = "Azure" - // gcpProviderType is the name of the Google Cloud Platform provider. - gcpProviderType = "GCP" + // awsLBTypeAnnotation is a Service annotation used to specify an AWS load + // balancer type. See the following for additional details: + // https://kubernetes.io/docs/concepts/services-networking/service/#aws-nlb-support + awsLBTypeAnnotation = "service.beta.kubernetes.io/aws-load-balancer-type" // awsInternalLBAnnotation is the annotation used on a service to specify an AWS // load balancer as being internal. awsInternalLBAnnotation = "service.beta.kubernetes.io/aws-load-balancer-internal" @@ -77,31 +75,20 @@ const ( ) var ( - // LbAnnotations maps cloud providers to the provider's annotation - // key/value pair used for managing a load balancer. For additional - // details see: - // https://kubernetes.io/docs/concepts/services-networking/service/#internal-load-balancer - // - LbAnnotations = map[operatorv1alpha1.LoadBalancerProviderType]map[string]string{ - awsProviderType: { - awsLbBackendProtoAnnotation: "tcp", - }, - } - // InternalLBAnnotations maps cloud providers to the provider's annotation // key/value pair used for managing an internal load balancer. For additional // details see: // https://kubernetes.io/docs/concepts/services-networking/service/#internal-load-balancer // InternalLBAnnotations = map[operatorv1alpha1.LoadBalancerProviderType]map[string]string{ - awsProviderType: { - awsInternalLBAnnotation: "0.0.0.0/0", + operatorv1alpha1.AWSLoadBalancerProvider: { + awsInternalLBAnnotation: "true", }, - azureProviderType: { + operatorv1alpha1.AzureLoadBalancerProvider: { // Azure load balancers are not customizable and are set to (2 fail @ 5s interval, 2 healthy) azureInternalLBAnnotation: "true", }, - gcpProviderType: { + operatorv1alpha1.GCPLoadBalancerProvider: { gcpLBTypeAnnotation: "Internal", }, } @@ -252,6 +239,16 @@ func DesiredEnvoyService(contour *operatorv1alpha1.Contour) *corev1.Service { SessionAffinity: corev1.ServiceAffinityNone, }, } + + // Add the NLB annotation if specified by AWS provider parameters. + if nlbNeeded(&contour.Spec) { + svc.Annotations[awsLBTypeAnnotation] = "nlb" + } + // Add the TCP backend protocol annotation for AWS classic load balancers. + if backendTCPNeeded(&contour.Spec) { + svc.Annotations[awsLbBackendProtoAnnotation] = "tcp" + } + epType := contour.Spec.NetworkPublishing.Envoy.Type if epType == operatorv1alpha1.LoadBalancerServicePublishingType || epType == operatorv1alpha1.NodePortServicePublishingType { @@ -260,13 +257,9 @@ func DesiredEnvoyService(contour *operatorv1alpha1.Contour) *corev1.Service { switch epType { case operatorv1alpha1.LoadBalancerServicePublishingType: svc.Spec.Type = corev1.ServiceTypeLoadBalancer - provider := contour.Spec.NetworkPublishing.Envoy.LoadBalancer.ProviderParameters.Type - lbAnnotations := LbAnnotations[provider] - for name, value := range lbAnnotations { - svc.Annotations[name] = value - } isInternal := contour.Spec.NetworkPublishing.Envoy.LoadBalancer.Scope == operatorv1alpha1.InternalLoadBalancer if isInternal { + provider := contour.Spec.NetworkPublishing.Envoy.LoadBalancer.ProviderParameters.Type internalAnnotations := InternalLBAnnotations[provider] for name, value := range internalAnnotations { svc.Annotations[name] = value @@ -368,3 +361,21 @@ func updateEnvoyServiceIfNeeded(ctx context.Context, cli client.Client, contour } return nil } + +// nlbNeeded returns true if the "service.beta.kubernetes.io/aws-load-balancer-type" +// annotation is needed based on the provided spec. +func nlbNeeded(spec *operatorv1alpha1.ContourSpec) bool { + return spec.NetworkPublishing.Envoy.Type == operatorv1alpha1.LoadBalancerServicePublishingType && + spec.NetworkPublishing.Envoy.LoadBalancer.ProviderParameters.Type == operatorv1alpha1.AWSLoadBalancerProvider && + spec.NetworkPublishing.Envoy.LoadBalancer.ProviderParameters.AWS != nil && + spec.NetworkPublishing.Envoy.LoadBalancer.ProviderParameters.AWS.Type == operatorv1alpha1.AWSNetworkLoadBalancer +} + +// backendTCPNeeded returns true if the "service.beta.kubernetes.io/aws-load-balancer-backend-protocol" +// annotation is needed based on the provided spec. +func backendTCPNeeded(spec *operatorv1alpha1.ContourSpec) bool { + return spec.NetworkPublishing.Envoy.Type == operatorv1alpha1.LoadBalancerServicePublishingType && + spec.NetworkPublishing.Envoy.LoadBalancer.ProviderParameters.Type == operatorv1alpha1.AWSLoadBalancerProvider && + (spec.NetworkPublishing.Envoy.LoadBalancer.ProviderParameters.AWS == nil || + spec.NetworkPublishing.Envoy.LoadBalancer.ProviderParameters.AWS.Type == operatorv1alpha1.AWSClassicLoadBalancer) +} diff --git a/internal/objects/service/service_test.go b/internal/objects/service/service_test.go index 22f65d8d..73da8735 100644 --- a/internal/objects/service/service_test.go +++ b/internal/objects/service/service_test.go @@ -81,19 +81,29 @@ func checkServiceHasPortProtocol(t *testing.T, svc *corev1.Service, protocol cor t.Errorf("service is missing port protocol %q", protocol) } -func checkServiceHasAnnotation(t *testing.T, svc *corev1.Service, key string) { +func checkServiceHasAnnotation(t *testing.T, svc *corev1.Service, expect bool, key string) { t.Helper() if svc.Annotations == nil { + if !expect { + return + } t.Errorf("service is missing annotations") } + + found := false for k := range svc.Annotations { if k == key { - return + found = true } } - t.Errorf("service is missing annotation %q", key) + if found && !expect { + t.Errorf("service contains annotation %q", key) + } + if !found && expect { + t.Errorf("service is missing annotation %q", key) + } } func checkServiceHasType(t *testing.T, svc *corev1.Service, svcType corev1.ServiceType) { @@ -161,16 +171,25 @@ func TestDesiredEnvoyService(t *testing.T) { svc = DesiredEnvoyService(cntr) checkServiceHasType(t, svc, corev1.ServiceTypeLoadBalancer) checkServiceHasExternalTrafficPolicy(t, svc, corev1.ServiceExternalTrafficPolicyTypeLocal) - checkServiceHasAnnotation(t, svc, awsLbBackendProtoAnnotation) + checkServiceHasAnnotation(t, svc, true, awsLbBackendProtoAnnotation) + // Check AWS NLB load balancer type. + awsParams := operatorv1alpha1.ProviderLoadBalancerParameters{ + Type: operatorv1alpha1.AWSLoadBalancerProvider, + AWS: &operatorv1alpha1.AWSLoadBalancerParameters{Type: operatorv1alpha1.AWSNetworkLoadBalancer}, + } + cntr.Spec.NetworkPublishing.Envoy.LoadBalancer.ProviderParameters = awsParams + svc = DesiredEnvoyService(cntr) + checkServiceHasAnnotation(t, svc, true, awsLBTypeAnnotation) + checkServiceHasAnnotation(t, svc, false, awsLbBackendProtoAnnotation) cntr.Spec.NetworkPublishing.Envoy.LoadBalancer.Scope = operatorv1alpha1.InternalLoadBalancer svc = DesiredEnvoyService(cntr) - checkServiceHasAnnotation(t, svc, awsInternalLBAnnotation) + checkServiceHasAnnotation(t, svc, true, awsInternalLBAnnotation) cntr.Spec.NetworkPublishing.Envoy.LoadBalancer.ProviderParameters.Type = operatorv1alpha1.AzureLoadBalancerProvider svc = DesiredEnvoyService(cntr) - checkServiceHasAnnotation(t, svc, azureInternalLBAnnotation) + checkServiceHasAnnotation(t, svc, true, azureInternalLBAnnotation) cntr.Spec.NetworkPublishing.Envoy.LoadBalancer.ProviderParameters.Type = operatorv1alpha1.GCPLoadBalancerProvider svc = DesiredEnvoyService(cntr) - checkServiceHasAnnotation(t, svc, gcpLBTypeAnnotation) + checkServiceHasAnnotation(t, svc, true, gcpLBTypeAnnotation) // Set network publishing type to ClusterIPService and verify the service type is as expected. cntr.Spec.NetworkPublishing.Envoy.Type = operatorv1alpha1.ClusterIPServicePublishingType svc = DesiredEnvoyService(cntr)