Support lb's listener configuration (#120)

* Skip lb update when nothing changed Signed-off-by: jacksontong <jacksontong@yunify.com> Co-authored-by: jacksontong <jacksontong@yunify.com> * add lsn healthycheck config * add lsn balance mode config * update loadbalancer_test.go * update docs for lb's listener config Co-authored-by: jacksontong <jacksontong@yunify.com>
yunify · Jun 1, 2022 · 94c25ce · 94c25ce
1 parent eb4e273
commit 94c25ce
Show file tree

Hide file tree

Showing 7 changed files with 285 additions and 20 deletions.
diff --git a/docs/configure.md b/docs/configure.md
@@ -111,6 +111,39 @@ spec:
     targetPort:  80
 ```
 
+## 四、配置LB的监听器属性
+
+### 如何配置
+1. 设置监听器的健康检查方式，`service.beta.kubernetes.io/qingcloud-lb-listener-healthycheckmethod`，对于 tcp 协议默认是 tcp 方式，对于 udp 协议默认是 udp 方式
+2. 设置监听器的健康检查参数，`service.beta.kubernetes.io/qingcloud-lb-listener-healthycheckoption`，默认是 "10|5|2|5"
+3. 支持 roundrobin/leastconn/source 三种负载均衡方式，`service.beta.kubernetes.io/qingcloud-lb-listener-balancemode`，默认是 roundrobin
+
+因为一个LB会有多个监听器，所以进行service注解设置时，通过如下格式区分不同监听器：`80:xxx,443:xxx`。
+
+### 参考Service
+```yaml
+kind: Service
+apiVersion: v1
+metadata:
+  name:  reuse-lb
+  annotations:
+    service.beta.kubernetes.io/qingcloud-load-balancer-eip-strategy: "reuse-lb"
+    service.beta.kubernetes.io/qingcloud-load-balancer-id: "lb-oglqftju"
+    service.beta.kubernetes.io/qingcloud-lb-listener-healthycheckmethod: "8090:tcp"
+    service.beta.kubernetes.io/qingcloud-lb-listener-healthycheckoption: "8090:10|5|2|5"
+    service.beta.kubernetes.io/qingcloud-lb-listener-balancemode: "8090:source"
+spec:
+  selector:
+    app:  mylbapp
+  type:  LoadBalancer
+  ports:
+  - name:  http
+    port:  8090
+    protocol: TCP
+    targetPort:  80
+```
+监听器参数说明：https://docsv3.qingcloud.com/network/loadbalancer/api/listener/modify_listener_attribute/
+
 ## 配置内网负载均衡器
 ### 已知问题
 k8s在ipvs模式下，kube-proxy会把内网负载均衡器的ip绑定在ipvs接口上，这样会导致从LB过来的包被drop（进来的是主网卡，但是出去的时候发现ipvs有这么一个ip，路由不一致）故目前无法在IPVS模式下使用内网负载均衡器。参考[issue](https://github.com/kubernetes/kubernetes/issues/79783)

diff --git a/pkg/apis/types.go b/pkg/apis/types.go
@@ -85,6 +85,9 @@ type LoadBalancerListenerSpec struct {
 	ListenerProtocol         *string `json:"listener_protocol" name:"listener_protocol"`
 	LoadBalancerListenerName *string `json:"loadbalancer_listener_name" name:"loadbalancer_listener_name"`
 	LoadBalancerID           *string `json:"loadbalancer_id" name:"loadbalancer_id"`
+	HealthyCheckMethod       *string `json:"healthy_check_method" name:"healthy_check_method"`
+	HealthyCheckOption       *string `json:"healthy_check_option" name:"healthy_check_option"`
+	BalanceMode              *string `json:"balance_mode" name:"balance_mode"`
 }
 
 type LoadBalancerListenerStatus struct {

diff --git a/pkg/executor/listener.go b/pkg/executor/listener.go
@@ -44,6 +44,9 @@ func convertLoadBalancerListener(inputs []*qcservice.LoadBalancerListener) []*ap
 				ListenerProtocol:         input.ListenerProtocol,
 				LoadBalancerListenerName: input.LoadBalancerListenerName,
 				LoadBalancerID:           input.LoadBalancerID,
+				HealthyCheckMethod:       input.HealthyCheckMethod,
+				HealthyCheckOption:       input.HealthyCheckOption,
+				BalanceMode:              input.BalanceMode,
 			},
 			Status: apis.LoadBalancerListenerStatus{
 				LoadBalancerListenerID: input.LoadBalancerListenerID,
@@ -64,7 +67,9 @@ func convertFromLoadBalancerListener(inputs []*apis.LoadBalancerListener) []*qcs
 			ListenerPort:             input.Spec.ListenerPort,
 			ListenerProtocol:         input.Spec.BackendProtocol,
 			LoadBalancerListenerName: input.Spec.LoadBalancerListenerName,
-			HealthyCheckMethod:       input.Spec.BackendProtocol,
+			HealthyCheckMethod:       input.Spec.HealthyCheckMethod,
+			HealthyCheckOption:       input.Spec.HealthyCheckOption,
+			BalanceMode:              input.Spec.BalanceMode,
 		})
 	}
 

diff --git a/pkg/qingcloud/annotations.go b/pkg/qingcloud/annotations.go
@@ -68,6 +68,12 @@ const (
 	//3. Configure sg
 
 	//4. Configure listener
+	// tcp or http, such as "80:tcp,443:tcp"
+	ServiceAnnotationListenerHealthyCheckMethod = "service.beta.kubernetes.io/qingcloud-lb-listener-healthycheckmethod"
+	// inter | timeout | fall | rise , such as "80:10|5|2|5,443:10|5|2|5", default is "*:10|5|2|5"
+	ServiceAnnotationListenerHealthyCheckOption = "service.beta.kubernetes.io/qingcloud-lb-listener-healthycheckoption"
+	// roundrobin / leastconn / source
+	ServiceAnnotationListenerBalanceMode = "service.beta.kubernetes.io/qingcloud-lb-listener-balancemode"
 )
 
 type LoadBalancerConfig struct {
@@ -81,6 +87,11 @@ type LoadBalancerConfig struct {
 	NodeCount        *int
 	InternalIP       *string
 
+	//listener attrs
+	healthyCheckMethod *string
+	healthyCheckOption *string
+	balanceMode        *string
+
 	//It's just for defining names, nothing more.
 	NetworkType      string
 	Policy           string
@@ -127,6 +138,15 @@ func (qc *QingCloud) ParseServiceLBConfig(cluster string, service *v1.Service) (
 	if internalReuseID, ok := annotation[ServiceAnnotationLoadBalancerInternalReuseID]; ok {
 		config.InternalReuseID = &internalReuseID
 	}
+	if healthyCheckMethod, ok := annotation[ServiceAnnotationListenerHealthyCheckMethod]; ok {
+		config.healthyCheckMethod = &healthyCheckMethod
+	}
+	if healthyCheckOption, ok := annotation[ServiceAnnotationListenerHealthyCheckOption]; ok {
+		config.healthyCheckOption = &healthyCheckOption
+	}
+	if balanceMode, ok := annotation[ServiceAnnotationListenerBalanceMode]; ok {
+		config.balanceMode = &balanceMode
+	}
 
 	networkType := annotation[ServiceAnnotationLoadBalancerNetworkType]
 	switch networkType {

diff --git a/pkg/qingcloud/loadbalancer_test.go b/pkg/qingcloud/loadbalancer_test.go
@@ -1,12 +1,14 @@
 package qingcloud
 
 import (
-	"github.com/yunify/qingcloud-cloud-controller-manager/pkg/apis"
+	"reflect"
+	"testing"
+
 	qcservice "github.com/yunify/qingcloud-sdk-go/service"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"reflect"
-	"testing"
+
+	"github.com/yunify/qingcloud-cloud-controller-manager/pkg/apis"
 )
 
 func TestConvertLoadBalancerStatus(t *testing.T) {
@@ -134,13 +136,17 @@ func TestDiffListeners(t *testing.T) {
 		ports     []v1.ServicePort
 		toDelete  []*string
 		toAdd     []v1.ServicePort
+		conf      *LoadBalancerConfig
 	}{
 		{
 			listeners: []*apis.LoadBalancerListener{
 				{
 					Spec: apis.LoadBalancerListenerSpec{
-						ListenerPort:     qcservice.Int(8080),
-						ListenerProtocol: qcservice.String("tcp"),
+						ListenerPort:       qcservice.Int(8080),
+						ListenerProtocol:   qcservice.String("tcp"),
+						HealthyCheckMethod: qcservice.String("tcp"),
+						HealthyCheckOption: qcservice.String("10|5|2|5"),
+						BalanceMode:        qcservice.String("roundrobin"),
 					},
 					Status: apis.LoadBalancerListenerStatus{
 						LoadBalancerListenerID: qcservice.String("testListener"),
@@ -173,8 +179,11 @@ func TestDiffListeners(t *testing.T) {
 			listeners: []*apis.LoadBalancerListener{
 				{
 					Spec: apis.LoadBalancerListenerSpec{
-						ListenerPort:     qcservice.Int(8080),
-						ListenerProtocol: qcservice.String("tcp"),
+						ListenerPort:       qcservice.Int(8080),
+						ListenerProtocol:   qcservice.String("tcp"),
+						HealthyCheckMethod: qcservice.String("tcp"),
+						HealthyCheckOption: qcservice.String("10|5|2|5"),
+						BalanceMode:        qcservice.String("roundrobin"),
 					},
 					Status: apis.LoadBalancerListenerStatus{
 						LoadBalancerListenerID: qcservice.String("testListener"),
@@ -213,8 +222,11 @@ func TestDiffListeners(t *testing.T) {
 			listeners: []*apis.LoadBalancerListener{
 				{
 					Spec: apis.LoadBalancerListenerSpec{
-						ListenerPort:     qcservice.Int(8080),
-						ListenerProtocol: qcservice.String("tcp"),
+						ListenerPort:       qcservice.Int(8080),
+						ListenerProtocol:   qcservice.String("tcp"),
+						HealthyCheckMethod: qcservice.String("tcp"),
+						HealthyCheckOption: qcservice.String("10|5|2|5"),
+						BalanceMode:        qcservice.String("roundrobin"),
 					},
 					Status: apis.LoadBalancerListenerStatus{
 						LoadBalancerListenerID: qcservice.String("testListener"),
@@ -253,8 +265,11 @@ func TestDiffListeners(t *testing.T) {
 			listeners: []*apis.LoadBalancerListener{
 				{
 					Spec: apis.LoadBalancerListenerSpec{
-						ListenerPort:     qcservice.Int(8080),
-						ListenerProtocol: qcservice.String("tcp"),
+						ListenerPort:       qcservice.Int(8080),
+						ListenerProtocol:   qcservice.String("tcp"),
+						HealthyCheckMethod: qcservice.String("tcp"),
+						HealthyCheckOption: qcservice.String("10|5|2|5"),
+						BalanceMode:        qcservice.String("roundrobin"),
 					},
 					Status: apis.LoadBalancerListenerStatus{
 						LoadBalancerListenerID: qcservice.String("testListener"),
@@ -289,10 +304,56 @@ func TestDiffListeners(t *testing.T) {
 			},
 			toDelete: []*string{qcservice.String("testListener")},
 		},
+		{
+			listeners: []*apis.LoadBalancerListener{
+				{
+					Spec: apis.LoadBalancerListenerSpec{
+						ListenerPort:       qcservice.Int(8080),
+						ListenerProtocol:   qcservice.String("tcp"),
+						HealthyCheckMethod: qcservice.String("tcp"),
+						HealthyCheckOption: qcservice.String("10|5|2|5"),
+						BalanceMode:        qcservice.String("roundrobin"),
+					},
+					Status: apis.LoadBalancerListenerStatus{
+						LoadBalancerListenerID: qcservice.String("testListenerBalanceMode"),
+						LoadBalancerBackends: []*apis.LoadBalancerBackend{
+							{
+								Spec: apis.LoadBalancerBackendSpec{
+									LoadBalancerBackendName: qcservice.String("instance1"),
+									Port:                    qcservice.Int(9090),
+									ResourceID:              qcservice.String("instance1"),
+								},
+								Status: apis.LoadBalancerBackendStatus{
+									LoadBalancerBackendID: qcservice.String("testBackend"),
+								},
+							},
+						},
+					},
+				},
+			},
+			ports: []v1.ServicePort{
+				{
+					Protocol: v1.ProtocolTCP,
+					Port:     8080,
+					NodePort: 9090,
+				},
+			},
+			conf: &LoadBalancerConfig{
+				balanceMode: qcservice.String("8080:source"), // change balanceMode
+			},
+			toAdd: []v1.ServicePort{
+				{
+					Protocol: v1.ProtocolTCP,
+					Port:     8080,
+					NodePort: 9090,
+				},
+			},
+			toDelete: []*string{qcservice.String("testListenerBalanceMode")},
+		},
 	}
 
 	for _, tc := range testCases {
-		toDelete, toAdd := diffListeners(tc.listeners, tc.ports)
+		toDelete, toAdd := diffListeners(tc.listeners, tc.conf, tc.ports)
 		//fmt.Printf("delete=%s, add=%s", spew.Sdump(toDelete), spew.Sdump(toAdd))
 		if !reflect.DeepEqual(toDelete, tc.toDelete) || !reflect.DeepEqual(toAdd, tc.toAdd) {
 			t.Fail()