Skip to content

Commit 8d08bde

Browse files
rahulaitrahulait
authored
[feat] : add support for nodebalancers with UDP ports (#387)
* use linodego with udp support * add UDP support to CCM * add unittests * update linodego for udp_check_port fix * update documentation for UDP support * use supported linodego version * partially run the udp e2e test * fix review comments
1 parent 8cad101 commit 8d08bde

File tree

15 files changed

+1004
-99
lines changed

15 files changed

+1004
-99
lines changed

‎cloud/annotations/annotations.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@ const (
66
AnnLinodeDefaultProtocol = "service.beta.kubernetes.io/linode-loadbalancer-default-protocol"
77
AnnLinodePortConfigPrefix = "service.beta.kubernetes.io/linode-loadbalancer-port-"
88
AnnLinodeDefaultProxyProtocol = "service.beta.kubernetes.io/linode-loadbalancer-default-proxy-protocol"
9+
AnnLinodeDefaultAlgorithm = "service.beta.kubernetes.io/linode-loadbalancer-default-algorithm"
10+
AnnLinodeDefaultStickiness = "service.beta.kubernetes.io/linode-loadbalancer-default-stickiness"
911

1012
AnnLinodeCheckPath = "service.beta.kubernetes.io/linode-loadbalancer-check-path"
1113
AnnLinodeCheckBody = "service.beta.kubernetes.io/linode-loadbalancer-check-body"
@@ -16,6 +18,8 @@ const (
1618
AnnLinodeHealthCheckAttempts = "service.beta.kubernetes.io/linode-loadbalancer-check-attempts"
1719
AnnLinodeHealthCheckPassive = "service.beta.kubernetes.io/linode-loadbalancer-check-passive"
1820

21+
AnnLinodeUDPCheckPort = "service.beta.kubernetes.io/linode-loadbalancer-udp-check-port"
22+
1923
// AnnLinodeThrottle is the annotation specifying the value of the Client Connection
2024
// Throttle, which limits the number of subsequent new connections per second from the
2125
// same client IP. Options are a number between 1-20, or 0 to disable. Defaults to 20.

‎cloud/linode/loadbalancers.go

Lines changed: 128 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,57 @@ import (
3535
var (
3636
errNoNodesAvailable = errors.New("no nodes available for nodebalancer")
3737
maxConnThrottleStringLen int = 20
38+
39+
// validProtocols is a map of valid protocols
40+
validProtocols = map[string]bool{
41+
string(linodego.ProtocolTCP): true,
42+
string(linodego.ProtocolUDP): true,
43+
string(linodego.ProtocolHTTP): true,
44+
string(linodego.ProtocolHTTPS): true,
45+
}
46+
// validProxyProtocols is a map of valid proxy protocols
47+
validProxyProtocols = map[string]bool{
48+
string(linodego.ProxyProtocolNone): true,
49+
string(linodego.ProxyProtocolV1): true,
50+
string(linodego.ProxyProtocolV2): true,
51+
}
52+
// validTCPAlgorithms is a map of valid TCP algorithms
53+
validTCPAlgorithms = map[string]bool{
54+
string(linodego.AlgorithmRoundRobin): true,
55+
string(linodego.AlgorithmLeastConn): true,
56+
string(linodego.AlgorithmSource): true,
57+
}
58+
// validUDPAlgorithms is a map of valid UDP algorithms
59+
validUDPAlgorithms = map[string]bool{
60+
string(linodego.AlgorithmRoundRobin): true,
61+
string(linodego.AlgorithmRingHash): true,
62+
string(linodego.AlgorithmLeastConn): true,
63+
}
64+
// validHTTPStickiness is a map of valid HTTP stickiness options
65+
validHTTPStickiness = map[string]bool{
66+
string(linodego.StickinessNone): true,
67+
string(linodego.StickinessHTTPCookie): true,
68+
string(linodego.StickinessTable): true,
69+
}
70+
// validHTTPSStickiness is the same as validHTTPStickiness, but for HTTPS
71+
validHTTPSStickiness = map[string]bool{
72+
string(linodego.StickinessNone): true,
73+
string(linodego.StickinessHTTPCookie): true,
74+
string(linodego.StickinessTable): true,
75+
}
76+
// validUDPStickiness is a map of valid UDP stickiness options
77+
validUDPStickiness = map[string]bool{
78+
string(linodego.StickinessNone): true,
79+
string(linodego.StickinessSession): true,
80+
string(linodego.StickinessSourceIP): true,
81+
}
82+
// validNBConfigChecks is a map of valid NodeBalancer config checks
83+
validNBConfigChecks = map[string]bool{
84+
string(linodego.CheckNone): true,
85+
string(linodego.CheckHTTP): true,
86+
string(linodego.CheckHTTPBody): true,
87+
string(linodego.CheckConnection): true,
88+
}
3889
)
3990

4091
type lbNotFoundError struct {
@@ -61,13 +112,19 @@ type portConfigAnnotation struct {
61112
TLSSecretName string `json:"tls-secret-name"`
62113
Protocol string `json:"protocol"`
63114
ProxyProtocol string `json:"proxy-protocol"`
115+
Algorithm string `json:"algorithm"`
116+
Stickiness string `json:"stickiness"`
117+
UDPCheckPort string `json:"udp-check-port"`
64118
}
65119

66120
type portConfig struct {
67121
TLSSecretName string
68122
Protocol linodego.ConfigProtocol
69123
ProxyProtocol linodego.ConfigProxyProtocol
70124
Port int
125+
Algorithm linodego.ConfigAlgorithm
126+
Stickiness linodego.ConfigStickiness
127+
UDPCheckPort int
71128
}
72129

73130
// newLoadbalancers returns a cloudprovider.LoadBalancer whose concrete type is a *loadbalancer.
@@ -351,14 +408,8 @@ func (l *loadbalancers) updateNodeBalancer(
351408

352409
// Add or overwrite configs for each of the Service's ports
353410
for _, port := range service.Spec.Ports {
354-
if port.Protocol == v1.ProtocolUDP {
355-
err := fmt.Errorf("error updating NodeBalancer Config: ports with the UDP protocol are not supported")
356-
sentry.CaptureError(ctx, err)
357-
return err
358-
}
359-
360411
// Construct a new config for this port
361-
newNBCfg, err := l.buildNodeBalancerConfig(ctx, service, int(port.Port))
412+
newNBCfg, err := l.buildNodeBalancerConfig(ctx, service, port)
362413
if err != nil {
363414
sentry.CaptureError(ctx, err)
364415
return err
@@ -408,7 +459,7 @@ func (l *loadbalancers) updateNodeBalancer(
408459
subnetID = id
409460
}
410461
for _, node := range nodes {
411-
newNodeOpts := l.buildNodeBalancerNodeConfigRebuildOptions(node, port.NodePort, subnetID)
462+
newNodeOpts := l.buildNodeBalancerNodeConfigRebuildOptions(node, port.NodePort, subnetID, newNBCfg.Protocol)
412463
oldNodeID, ok := oldNBNodeIDs[newNodeOpts.Address]
413464
if ok {
414465
newNodeOpts.ID = oldNodeID
@@ -726,22 +777,31 @@ func (l *loadbalancers) createNodeBalancer(ctx context.Context, clusterName stri
726777
return l.client.CreateNodeBalancer(ctx, createOpts)
727778
}
728779

729-
func (l *loadbalancers) buildNodeBalancerConfig(ctx context.Context, service *v1.Service, port int) (linodego.NodeBalancerConfig, error) {
780+
func (l *loadbalancers) buildNodeBalancerConfig(ctx context.Context, service *v1.Service, port v1.ServicePort) (linodego.NodeBalancerConfig, error) {
730781
portConfigResult, err := getPortConfig(service, port)
731782
if err != nil {
732783
return linodego.NodeBalancerConfig{}, err
733784
}
734785

735-
health, err := getHealthCheckType(service)
786+
health, err := getHealthCheckType(service, port)
736787
if err != nil {
737788
return linodego.NodeBalancerConfig{}, err
738789
}
739790

740791
config := linodego.NodeBalancerConfig{
741-
Port: port,
792+
Port: int(port.Port),
742793
Protocol: portConfigResult.Protocol,
743794
ProxyProtocol: portConfigResult.ProxyProtocol,
744795
Check: health,
796+
Algorithm: portConfigResult.Algorithm,
797+
}
798+
799+
if portConfigResult.Stickiness != "" {
800+
config.Stickiness = portConfigResult.Stickiness
801+
}
802+
803+
if portConfigResult.UDPCheckPort != 0 {
804+
config.UDPCheckPort = portConfigResult.UDPCheckPort
745805
}
746806

747807
if health == linodego.CheckHTTP || health == linodego.CheckHTTPBody {
@@ -784,7 +844,9 @@ func (l *loadbalancers) buildNodeBalancerConfig(ctx context.Context, service *v1
784844
config.CheckAttempts = checkAttempts
785845

786846
checkPassive := true
787-
if cp, ok := service.GetAnnotations()[annotations.AnnLinodeHealthCheckPassive]; ok {
847+
if config.Protocol == linodego.ProtocolUDP {
848+
checkPassive = false
849+
} else if cp, ok := service.GetAnnotations()[annotations.AnnLinodeHealthCheckPassive]; ok {
788850
if checkPassive, err = strconv.ParseBool(cp); err != nil {
789851
return config, err
790852
}
@@ -858,18 +920,14 @@ func (l *loadbalancers) buildLoadBalancerRequest(ctx context.Context, clusterNam
858920
}
859921

860922
for _, port := range ports {
861-
if port.Protocol == v1.ProtocolUDP {
862-
return nil, fmt.Errorf("error creating NodeBalancer Config: ports with the UDP protocol are not supported")
863-
}
864-
865-
config, err := l.buildNodeBalancerConfig(ctx, service, int(port.Port))
923+
config, err := l.buildNodeBalancerConfig(ctx, service, port)
866924
if err != nil {
867925
return nil, err
868926
}
869927
createOpt := config.GetCreateOptions()
870928

871929
for _, n := range nodes {
872-
createOpt.Nodes = append(createOpt.Nodes, l.buildNodeBalancerNodeConfigRebuildOptions(n, port.NodePort, subnetID).NodeBalancerNodeCreateOptions)
930+
createOpt.Nodes = append(createOpt.Nodes, l.buildNodeBalancerNodeConfigRebuildOptions(n, port.NodePort, subnetID, config.Protocol).NodeBalancerNodeCreateOptions)
873931
}
874932

875933
configs = append(configs, &createOpt)
@@ -889,17 +947,20 @@ func coerceString(str string, minLen, maxLen int, padding string) string {
889947
return str
890948
}
891949

892-
func (l *loadbalancers) buildNodeBalancerNodeConfigRebuildOptions(node *v1.Node, nodePort int32, subnetID int) linodego.NodeBalancerConfigRebuildNodeOptions {
950+
func (l *loadbalancers) buildNodeBalancerNodeConfigRebuildOptions(node *v1.Node, nodePort int32, subnetID int, protocol linodego.ConfigProtocol) linodego.NodeBalancerConfigRebuildNodeOptions {
893951
nodeOptions := linodego.NodeBalancerConfigRebuildNodeOptions{
894952
NodeBalancerNodeCreateOptions: linodego.NodeBalancerNodeCreateOptions{
895953
Address: fmt.Sprintf("%v:%v", getNodePrivateIP(node, subnetID), nodePort),
896954
// NodeBalancer backends must be 3-32 chars in length
897955
// If < 3 chars, pad node name with "node-" prefix
898956
Label: coerceString(node.Name, 3, 32, "node-"),
899-
Mode: "accept",
900957
Weight: 100,
901958
},
902959
}
960+
// Mode is not set for UDP protocol
961+
if protocol != linodego.ProtocolUDP {
962+
nodeOptions.Mode = "accept"
963+
}
903964
if subnetID != 0 {
904965
nodeOptions.SubnetID = subnetID
905966
}
@@ -937,57 +998,73 @@ func (l *loadbalancers) retrieveKubeClient() error {
937998
return nil
938999
}
9391000

940-
func getPortConfig(service *v1.Service, port int) (portConfig, error) {
1001+
func getPortConfig(service *v1.Service, port v1.ServicePort) (portConfig, error) {
9411002
portConfigResult := portConfig{}
942-
portConfigAnnotationResult, err := getPortConfigAnnotation(service, port)
1003+
portConfigResult.Port = int(port.Port)
1004+
1005+
portConfigAnnotationResult, err := getPortConfigAnnotation(service, int(port.Port))
9431006
if err != nil {
9441007
return portConfigResult, err
9451008
}
946-
protocol := portConfigAnnotationResult.Protocol
947-
if protocol == "" {
948-
protocol = "tcp"
949-
if p, ok := service.GetAnnotations()[annotations.AnnLinodeDefaultProtocol]; ok {
950-
protocol = p
951-
}
952-
}
953-
protocol = strings.ToLower(protocol)
9541009

955-
proxyProtocol := portConfigAnnotationResult.ProxyProtocol
956-
if proxyProtocol == "" {
957-
proxyProtocol = string(linodego.ProxyProtocolNone)
958-
for _, ann := range []string{annotations.AnnLinodeDefaultProxyProtocol, annLinodeProxyProtocolDeprecated} {
959-
if pp, ok := service.GetAnnotations()[ann]; ok {
960-
proxyProtocol = pp
961-
break
962-
}
963-
}
1010+
// validate and set protocol
1011+
protocol, err := getPortProtocol(portConfigAnnotationResult, service, port)
1012+
if err != nil {
1013+
return portConfigResult, err
9641014
}
1015+
portConfigResult.Protocol = linodego.ConfigProtocol(protocol)
9651016

966-
if protocol != "tcp" && protocol != "http" && protocol != "https" {
967-
return portConfigResult, fmt.Errorf("invalid protocol: %q specified", protocol)
1017+
// validate and set proxy protocol
1018+
proxyProtocol, err := getPortProxyProtocol(portConfigAnnotationResult, service, portConfigResult.Protocol)
1019+
if err != nil {
1020+
return portConfigResult, err
9681021
}
1022+
portConfigResult.ProxyProtocol = linodego.ConfigProxyProtocol(proxyProtocol)
9691023

970-
switch proxyProtocol {
971-
case string(linodego.ProxyProtocolNone), string(linodego.ProxyProtocolV1), string(linodego.ProxyProtocolV2):
972-
break
973-
default:
974-
return portConfigResult, fmt.Errorf("invalid NodeBalancer proxy protocol value '%s'", proxyProtocol)
1024+
// validate and set algorithm
1025+
algorithm, err := getPortAlgorithm(portConfigAnnotationResult, service, portConfigResult.Protocol)
1026+
if err != nil {
1027+
return portConfigResult, err
9751028
}
1029+
portConfigResult.Algorithm = linodego.ConfigAlgorithm(algorithm)
9761030

977-
portConfigResult.Port = port
978-
portConfigResult.Protocol = linodego.ConfigProtocol(protocol)
979-
portConfigResult.ProxyProtocol = linodego.ConfigProxyProtocol(proxyProtocol)
1031+
// set TLS secret name. Its only used for TCP and HTTPS protocols
1032+
if protocol == string(linodego.ProtocolUDP) && portConfigAnnotationResult.TLSSecretName != "" {
1033+
return portConfigResult, fmt.Errorf("specifying TLS secret name is not supported for UDP")
1034+
}
9801035
portConfigResult.TLSSecretName = portConfigAnnotationResult.TLSSecretName
9811036

1037+
// validate and set udp check port
1038+
udpCheckPort, err := getPortUDPCheckPort(portConfigAnnotationResult, service, portConfigResult.Protocol)
1039+
if err != nil {
1040+
return portConfigResult, err
1041+
}
1042+
if protocol == string(linodego.ProtocolUDP) {
1043+
portConfigResult.UDPCheckPort = udpCheckPort
1044+
}
1045+
1046+
// validate and set stickiness
1047+
stickiness, err := getPortStickiness(portConfigAnnotationResult, service, portConfigResult.Protocol)
1048+
if err != nil {
1049+
return portConfigResult, err
1050+
}
1051+
// Stickiness is not supported for TCP protocol
1052+
if protocol != string(linodego.ProtocolTCP) {
1053+
portConfigResult.Stickiness = linodego.ConfigStickiness(stickiness)
1054+
}
1055+
9821056
return portConfigResult, nil
9831057
}
9841058

985-
func getHealthCheckType(service *v1.Service) (linodego.ConfigCheck, error) {
1059+
func getHealthCheckType(service *v1.Service, port v1.ServicePort) (linodego.ConfigCheck, error) {
9861060
hType, ok := service.GetAnnotations()[annotations.AnnLinodeHealthCheckType]
9871061
if !ok {
1062+
if port.Protocol == v1.ProtocolUDP {
1063+
return linodego.CheckNone, nil
1064+
}
9881065
return linodego.CheckConnection, nil
9891066
}
990-
if hType != "none" && hType != "connection" && hType != "http" && hType != "http_body" {
1067+
if !validNBConfigChecks[hType] {
9911068
return "", fmt.Errorf("invalid health check type: %q specified in annotation: %q", hType, annotations.AnnLinodeHealthCheckType)
9921069
}
9931070
return linodego.ConfigCheck(hType), nil

0 commit comments

Comments
 (0)