Support https protocol and certificate for lb listener; Signed-off-by gangqiangwang <gangqiangwang@yunify.com>;

docs/configure.md

@@ -117,7 +117,8 @@ spec:
 1. 设置监听器的健康检查方式，`service.beta.kubernetes.io/qingcloud-lb-listener-healthycheckmethod`，对于 tcp 协议默认是 tcp 方式，对于 udp 协议默认是 udp 方式
 2. 设置监听器的健康检查参数，`service.beta.kubernetes.io/qingcloud-lb-listener-healthycheckoption`，默认是 "10|5|2|5"
 3. 支持 roundrobin/leastconn/source 三种负载均衡方式，`service.beta.kubernetes.io/qingcloud-lb-listener-balancemode`，默认是 roundrobin
-4. 支持https协议及证书的配置，`service.beta.kubernetes.io/qingcloud-lb-listener-cert`，如果配置了证书，则监听器使用 https 协议，没有此注解则默认使用 Service 所用协议
+4. 支持 http/https 协议的配置，`service.beta.kubernetes.io/qingcloud-lb-listener-protocol`，没有此注解则默认使用 Service 所用协议
+5. 支持 https 协议证书的配置，`service.beta.kubernetes.io/qingcloud-lb-listener-cert`，如果配置的 https 协议，则必须配置证书
 
 因为一个LB会有多个监听器，所以进行service注解设置时，通过如下格式区分不同监听器：`80:xxx,443:xxx`。
 
@@ -133,6 +134,7 @@ metadata:
     service.beta.kubernetes.io/qingcloud-lb-listener-healthycheckmethod: "8090:tcp"
     service.beta.kubernetes.io/qingcloud-lb-listener-healthycheckoption: "8090:10|5|2|5"
     service.beta.kubernetes.io/qingcloud-lb-listener-balancemode: "8090:source"
+    service.beta.kubernetes.io/qingcloud-lb-listener-protocol: "8090:https"
     service.beta.kubernetes.io/qingcloud-lb-listener-cert: "8090:sc-77oko7zj"
 spec:
   selector:

pkg/errors/errors.go

@@ -31,6 +31,7 @@ func NewResourceNotFoundError(resource, name string, message ...string) error {
 		Type:         ResourceNotFound,
 		ResourceType: resource,
 		Action:       "GetResource",
+		ResouceName:  name,
 	}
 	if len(message) > 0 {
 		e.Message = message[0]
@@ -52,6 +53,7 @@ func NewCommonServerError(resource, name, action, message string) error {
 		ResourceType: resource,
 		Message:      message,
 		Action:       action,
+		ResouceName:  name,
 	}
 }
 

pkg/qingcloud/annotations.go

@@ -78,6 +78,8 @@ const (
 	ServiceAnnotationListenerBalanceMode = "service.beta.kubernetes.io/qingcloud-lb-listener-balancemode"
 	// port:certificate, such as "6443:sc-77oko7zj,8443:sc-77oko7zj"
 	ServiceAnnotationListenerServerCertificate = "service.beta.kubernetes.io/qingcloud-lb-listener-cert"
+	// port:protocol, such as "443:https,80:http"
+	ServiceAnnotationListenerProtocol = "service.beta.kubernetes.io/qingcloud-lb-listener-protocol"
 )
 
 type LoadBalancerConfig struct {
@@ -96,6 +98,7 @@ type LoadBalancerConfig struct {
 	healthyCheckOption *string
 	balanceMode        *string
 	ServerCertificate  *string
+	Protocol           *string
 
 	//It's just for defining names, nothing more.
 	NetworkType      string
@@ -155,6 +158,9 @@ func (qc *QingCloud) ParseServiceLBConfig(cluster string, service *v1.Service) (
 	if serverCertificate, ok := annotation[ServiceAnnotationListenerServerCertificate]; ok {
 		config.ServerCertificate = &serverCertificate
 	}
+	if protocol, ok := annotation[ServiceAnnotationListenerProtocol]; ok {
+		config.Protocol = &protocol
+	}
 
 	networkType := annotation[ServiceAnnotationLoadBalancerNetworkType]
 	switch networkType {

pkg/qingcloud/loadbalancer_test.go

@@ -361,7 +361,7 @@ func TestDiffListeners(t *testing.T) {
 						BalanceMode:        qcservice.String("roundrobin"),
 					},
 					Status: apis.LoadBalancerListenerStatus{
-						LoadBalancerListenerID: qcservice.String("testListenerCert"),
+						LoadBalancerListenerID: qcservice.String("testListenerProtocol"),
 						LoadBalancerBackends: []*apis.LoadBalancerBackend{
 							{
 								Spec: apis.LoadBalancerBackendSpec{
@@ -385,6 +385,7 @@ func TestDiffListeners(t *testing.T) {
 				},
 			},
 			conf: &LoadBalancerConfig{
+				Protocol:          qcservice.String("8080:https"),       //change protocol
 				ServerCertificate: qcservice.String("8080:sc-llluxekm"), // change cert
 			},
 			toAdd: []v1.ServicePort{
@@ -394,13 +395,13 @@ func TestDiffListeners(t *testing.T) {
 					NodePort: 9090,
 				},
 			},
-			toDelete: []*string{qcservice.String("testListenerCert")},
+			toDelete: []*string{qcservice.String("testListenerProtocol")},
 		},
 	}
 
 	for _, tc := range testCases {
 		toDelete, toAdd := diffListeners(tc.listeners, tc.conf, tc.ports)
-		//fmt.Printf("delete=%s, add=%s", spew.Sdump(toDelete), spew.Sdump(toAdd))
+		// fmt.Printf("delete=%s, add=%s", spew.Sdump(toDelete), spew.Sdump(toAdd))
 		if !reflect.DeepEqual(toDelete, tc.toDelete) || !reflect.DeepEqual(toAdd, tc.toAdd) {
 			t.Fail()
 		}

pkg/qingcloud/loadbalancer_utils.go

@@ -128,6 +128,30 @@ func getCertificate(annotationConf map[int]string, port int) *string {
 	return nil
 }
 
+func getProtocol(annotationConf map[int]string, port int) *string {
+	var protocol string
+	if annotationConf != nil {
+		protocolConf := annotationConf[port]
+		switch protocolConf {
+		case "https", "HTTPS":
+			protocol = "https"
+		case "http", "HTTP":
+			protocol = "http"
+		case "tcp", "TCP":
+			protocol = "tcp"
+		case "udp", "UDP":
+			protocol = "upd"
+		default:
+			protocol = ""
+		}
+	}
+	if protocol != "" {
+		return &protocol
+	} else {
+		return nil
+	}
+}
+
 func diffListeners(listeners []*apis.LoadBalancerListener, conf *LoadBalancerConfig, ports []v1.ServicePort) (toDelete []*string, toAdd []v1.ServicePort) {
 	svcNodePort := make(map[string]int)
 	for _, listener := range listeners {
@@ -144,11 +168,10 @@ func diffListeners(listeners []*apis.LoadBalancerListener, conf *LoadBalancerCon
 		balanceMode := getBalanceMode(bms, int(port.Port))
 		for _, listener := range listeners {
 			if *listener.Spec.ListenerPort == int(port.Port) &&
-				strings.EqualFold(*listener.Spec.ListenerProtocol, string(port.Protocol)) &&
 				svcNodePort[*listener.Status.LoadBalancerListenerID] == int(port.NodePort) &&
 				*balanceMode == *listener.Spec.BalanceMode &&
 				(*healthyCheck.option == *listener.Spec.HealthyCheckOption && *healthyCheck.method == *listener.Spec.HealthyCheckMethod) &&
-				equalCertificate(listener, conf, port) {
+				equalProtocol(listener, conf, port) {
 				add = false
 				break
 			}
@@ -164,11 +187,10 @@ func diffListeners(listeners []*apis.LoadBalancerListener, conf *LoadBalancerCon
 			healthyCheck := getHealthyCheck(hcs, int(port.Port), strings.ToLower(string(port.Protocol)))
 			balanceMode := getBalanceMode(bms, int(port.Port))
 			if *listener.Spec.ListenerPort == int(port.Port) &&
-				strings.EqualFold(*listener.Spec.ListenerProtocol, string(port.Protocol)) &&
 				svcNodePort[*listener.Status.LoadBalancerListenerID] == int(port.NodePort) &&
 				*balanceMode == *listener.Spec.BalanceMode &&
 				(*healthyCheck.option == *listener.Spec.HealthyCheckOption && *healthyCheck.method == *listener.Spec.HealthyCheckMethod) &&
-				equalCertificate(listener, conf, port) {
+				equalProtocol(listener, conf, port) {
 				delete = false
 				break
 			}
@@ -190,7 +212,7 @@ func getLoadBalancerListenerNodePort(listener *apis.LoadBalancerListener, ports
 
 	for _, port := range ports {
 		lsnProtocol := listener.Spec.ListenerProtocol
-		if *listener.Spec.ListenerProtocol == "https" && port.Protocol == v1.ProtocolTCP {
+		if (*lsnProtocol == "https" || *lsnProtocol == "http") && port.Protocol == v1.ProtocolTCP {
 			*lsnProtocol = "tcp"
 		}
 
@@ -233,10 +255,15 @@ type healthyChek struct {
 //  1)healthycheckmethod: "80:tcp,443:tcp"
 //  2)healthycheckoption: "80:10|5|2|5,443:10|5|2|5"
 //  3)balancemode: "80:roundrobin,443:leastconn,8080:source"
+//  4)cert: "443:sc-77oko7zj,80:sc-77oko7zj"
+//  5)protocol: "443:https,80:http"
 func parseLsnAnnotaionData(data string) (map[int]string, error) {
 	methods := strings.Split(data, ",")
 	rst := make(map[int]string, len(methods))
 	for _, method := range methods {
+		if method == "" {
+			continue
+		}
 		m := strings.Split(method, ":")
 		if len(m) != 2 {
 			return nil, fmt.Errorf("wrong format: (%s)", data)
@@ -306,6 +333,13 @@ func parseCertificate(conf *LoadBalancerConfig) (map[int]string, error) {
 	return parseLsnAnnotaionData(*conf.ServerCertificate)
 }
 
+func parseProtocol(conf *LoadBalancerConfig) (map[int]string, error) {
+	if conf == nil || conf.Protocol == nil {
+		return nil, nil
+	}
+	return parseLsnAnnotaionData(*conf.Protocol)
+}
+
 func generateLoadBalancerListeners(conf *LoadBalancerConfig, lb *apis.LoadBalancer, ports []v1.ServicePort) ([]*apis.LoadBalancerListener, error) {
 	hcs, err := parseHeathyCheck(conf)
 	if err != nil {
@@ -322,6 +356,11 @@ func generateLoadBalancerListeners(conf *LoadBalancerConfig, lb *apis.LoadBalanc
 		return nil, err
 	}
 
+	protocols, err := parseProtocol(conf)
+	if err != nil {
+		return nil, err
+	}
+
 	var result []*apis.LoadBalancerListener
 	for _, port := range ports {
 		protocol := ""
@@ -337,16 +376,25 @@ func generateLoadBalancerListeners(conf *LoadBalancerConfig, lb *apis.LoadBalanc
 
 		healthyCheck := getHealthyCheck(hcs, int(port.Port), strings.ToLower(string(port.Protocol)))
 		balanceMode := getBalanceMode(bms, int(port.Port))
+		listenerProtocol := getProtocol(protocols, int(port.Port))
 		certID := getCertificate(certs, int(port.Port))
-		listenerProtocol := protocol
-		if protocol == "tcp" && certID != nil {
-			listenerProtocol = "https"
-			serverCertificate = append(serverCertificate, certID)
+
+		if listenerProtocol == nil {
+			listenerProtocol = &protocol
+		} else if *listenerProtocol == "https" {
+			protocol = "http"
+			if certID != nil {
+				serverCertificate = append(serverCertificate, certID)
+			} else {
+				return nil, fmt.Errorf("loadbalance listener with https protocol must config certificate")
+			}
+		} else if *listenerProtocol == "http" {
+			protocol = "http"
 		}
 		result = append(result, &apis.LoadBalancerListener{
 			Spec: apis.LoadBalancerListenerSpec{
 				BackendProtocol:          &protocol,
-				ListenerProtocol:         &listenerProtocol,
+				ListenerProtocol:         listenerProtocol,
 				ListenerPort:             qcservice.Int(int(port.Port)),
 				LoadBalancerListenerName: &conf.listenerName,
 				LoadBalancerID:           lb.Status.LoadBalancerID,
@@ -418,3 +466,34 @@ func equalCertificate(listener *apis.LoadBalancerListener, conf *LoadBalancerCon
 
 	return false
 }
+
+func equalProtocol(listener *apis.LoadBalancerListener, conf *LoadBalancerConfig, port v1.ServicePort) bool {
+	protocols, _ := parseProtocol(conf)
+	lsnProtocol := getProtocol(protocols, int(port.Port))
+
+	if lsnProtocol == nil {
+		lsnProtocol = qcservice.String(strings.ToLower(string(port.Protocol)))
+	}
+
+	if *lsnProtocol == *listener.Spec.ListenerProtocol {
+		if *lsnProtocol == "https" {
+			// check cert
+			certs, _ := parseCertificate(conf)
+			certIDConf := getCertificate(certs, int(port.Port))
+			if certIDConf == nil && len(listener.Spec.ServerCertificateID) == 0 {
+				return true
+			}
+
+			if certIDConf != nil {
+				for _, cert := range listener.Spec.ServerCertificateID {
+					if *cert == *certIDConf {
+						return true
+					}
+				}
+			}
+			return false
+		}
+		return true
+	}
+	return false
+}

pkg/qingcloud/qingcloud.go

@@ -210,6 +210,7 @@ func (qc *QingCloud) EnsureLoadBalancer(ctx context.Context, _ string, service *
 		if len(listenerIDs) <= 0 {
 			klog.Infof("create listeners for loadbalancers %s, service ports %s", *lb.Status.LoadBalancerID, spew.Sdump(service.Spec.Ports))
 			if err = qc.createListenersAndBackends(conf, lb, service.Spec.Ports, nodes); err != nil {
+				klog.Errorf("createListenersAndBackends for loadbalancer %s error: %v", *lb.Status.LoadBalancerID, err)
 				return nil, err
 			}
 		} else {
@@ -338,6 +339,7 @@ func (qc *QingCloud) EnsureLoadBalancer(ctx context.Context, _ string, service *
 // Parameter 'clusterName' is the name of the cluster as presented to kube-controller-manager
 func (qc *QingCloud) UpdateLoadBalancer(ctx context.Context, _ string, service *v1.Service, nodes []*v1.Node) error {
 	conf, lb, err := qc.getLoadBalancer(service)
+	klog.V(4).Infof("==== UpdateLoadBalancer %s config %s ====", spew.Sdump(lb), spew.Sdump(conf))
 	if err != nil {
 		return err
 	}
@@ -381,10 +383,12 @@ func (qc *QingCloud) UpdateLoadBalancer(ctx context.Context, _ string, service *
 func (qc *QingCloud) createListenersAndBackends(conf *LoadBalancerConfig, status *apis.LoadBalancer, ports []v1.ServicePort, nodes []*v1.Node) error {
 	listeners, err := generateLoadBalancerListeners(conf, status, ports)
 	if err != nil {
+		klog.Errorf("generateLoadBalancerListeners for loadbalancer %s error: %v", *status.Status.LoadBalancerID, err)
 		return err
 	}
 	listeners, err = qc.Client.CreateListener(listeners)
 	if err != nil {
+		klog.Errorf("CreateListener for loadbalancer %s error: %v", *status.Status.LoadBalancerID, err)
 		return err
 	}
 
@@ -393,6 +397,7 @@ func (qc *QingCloud) createListenersAndBackends(conf *LoadBalancerConfig, status
 		backends := generateLoadBalancerBackends(nodes, listener, ports)
 		_, err = qc.Client.CreateBackends(backends)
 		if err != nil {
+			klog.Errorf("CreateBackends for loadbalancer %s error: %v", *status.Status.LoadBalancerID, err)
 			return err
 		}
 
@@ -411,6 +416,7 @@ func (qc *QingCloud) createListenersAndBackends(conf *LoadBalancerConfig, status
 // Parameter 'clusterName' is the name of the cluster as presented to kube-controller-manager
 func (qc *QingCloud) EnsureLoadBalancerDeleted(ctx context.Context, _ string, service *v1.Service) error {
 	lbConfig, lb, err := qc.getLoadBalancer(service)
+	klog.V(4).Infof("==== EnsureLoadBalancerDeleted %s config %s ====", spew.Sdump(lb), spew.Sdump(lbConfig))
 	if errors.IsResourceNotFound(err) {
 		return nil
 	}