✨ support internal ip (#89)

docs/configure.md

@@ -166,6 +166,7 @@ spec:
 ### 注意事项
 1. 必须手动指定`service.beta.kubernetes.io/qingcloud-load-balancer-network-type`为`internal`，如果不指定或者填写其他值，都默认为公网LB，需要配置EIP
 2. 可选指定LB所在的Vxnet，默认为创建LB插件配置文件中的`defaultVxnet`，手动配置vxnet的annotation为`service.beta.kubernetes.io/qingcloud-load-balancer-vxnet-id`
+3. 可选指定LB 内网ip，通过`service.beta.kubernetes.io/qingcloud-load-balancer-internal-ip`指定。**注意，当前不支持更换内网IP**
 ### 参考Service
 ```yaml
 kind: Service
@@ -176,6 +177,7 @@ metadata:
   annotations:
     service.beta.kubernetes.io/qingcloud-load-balancer-type: "0"
     service.beta.kubernetes.io/qingcloud-load-balancer-network-type: "internal"
+    service.beta.kubernetes.io/qingcloud-load-balancer-internal-ip: "192.168.0.1" ##如果要路由器自动分配删掉这一行
 spec:
   selector:
     app:  mylbapp

go.mod

@@ -19,7 +19,7 @@ require (
 	k8s.io/client-go v0.0.0-20190409092706-ca8df85b1798
 	k8s.io/cloud-provider v0.0.0-20190409133700-fca63a37790d
 	k8s.io/component-base v0.0.0-20190409093041-e34633071963
-	k8s.io/klog v0.2.0
+	k8s.io/klog v0.4.0
 	k8s.io/kube-controller-manager v0.0.0-20190409093943-63563cf7ef9a // indirect
 	k8s.io/kubernetes v1.14.1
 )

go.sum

@@ -50,6 +50,7 @@ github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0
 github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
+github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
 github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI=
 github.com/go-openapi/analysis v0.17.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik=
 github.com/go-openapi/analysis v0.17.2/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik=
@@ -293,6 +294,8 @@ k8s.io/gengo v0.0.0-20190116091435-f8a0810f38af/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8
 k8s.io/klog v0.0.0-20190306015804-8e90cee79f82/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
 k8s.io/klog v0.2.0 h1:0ElL0OHzF3N+OhoJTL0uca20SxtYt4X4+bzHeqrB83c=
 k8s.io/klog v0.2.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
+k8s.io/klog v0.4.0 h1:lCJCxf/LIowc2IGS9TPjWDyXY4nOmdGdfcwwDQCOURQ=
+k8s.io/klog v0.4.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I=
 k8s.io/kube-controller-manager v0.0.0-20190409093943-63563cf7ef9a h1:rNaiSRjwZw/EtkFBuR1OVWvdQwpWhDfO205T/gn5+9c=
 k8s.io/kube-controller-manager v0.0.0-20190409093943-63563cf7ef9a/go.mod h1:TmTxiKBdfDjRwRlSS1VHhjE4Y3VcalaK1jkUgqGUwck=
 k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30 h1:TRb4wNWoBVrH9plmkp2q86FIDppkbrEXdXlxU3a3BMI=

hack/debug.sh

@@ -53,5 +53,5 @@ if [ $SKIP_BUILD == "no" ]; then
 fi
 
 echo "Generating yaml"
-sed -e 's@image: .*@image: '"${IMG}"'@' deploy/kube-cloud-controller-manager.yaml > $DEST
+sed -e 's@image: .*@image: '"${IMG}"'@' -e "s/IfNotPresent/Always/g" deploy/kube-cloud-controller-manager.yaml > $DEST
 kubectl apply -f $DEST

pkg/executor/lb.go

@@ -107,6 +107,7 @@ func (q *qingCloudLoadBalanceExecutor) Stop(id string) error {
 }
 
 func (q *qingCloudLoadBalanceExecutor) Create(input *qcservice.CreateLoadBalancerInput) (*qcservice.LoadBalancer, error) {
+	klog.V(2).Infof("Creating LB: %+v", *input)
 	name := *input.LoadBalancerName
 	output, err := q.lbapi.CreateLoadBalancer(input)
 	if err != nil {

pkg/loadbalance/annotations.go

@@ -15,8 +15,8 @@ const (
 	// ServiceAnnotationLoadBalancerVxnetId is the annotation which indicates the very vxnet where load
 	// balancer resides. This annotation should NOT be used when ServiceAnnotationLoadBalancerEipIds is
 	// set.
-	ServiceAnnotationLoadBalancerVxnetID = "service.beta.kubernetes.io/qingcloud-load-balancer-vxnet-id"
-
+	ServiceAnnotationLoadBalancerVxnetID    = "service.beta.kubernetes.io/qingcloud-load-balancer-vxnet-id"
+	ServiceAnnotationLoadBalancerInternalIP = "service.beta.kubernetes.io/qingcloud-load-balancer-internal-ip"
 	// ServiceAnnotationLoadBalancerType is the annotation used on the
 	// service to indicate that we want a qingcloud loadBalancer type.
 	// value "0" means the LB can max support 5000 concurrency connections, it's default type.

pkg/loadbalance/loadbalancer.go

@@ -43,6 +43,7 @@ type LoadBalancerSpec struct {
 	LoadBalancerID    string
 	NetworkType       string
 	VxnetID           string
+	InternalIP        string
 }
 
 type LoadBalancerStatus struct {
@@ -135,6 +136,9 @@ func NewLoadBalancer(opt *NewLoadBalancerOption) (*LoadBalancer, error) {
 		} else {
 			result.VxnetID = opt.DefaultVxnet
 		}
+		if ip, ok := opt.K8sService.Annotations[ServiceAnnotationLoadBalancerInternalIP]; ok {
+			result.InternalIP = ip
+		}
 	}
 
 	lbType := opt.K8sService.Annotations[ServiceAnnotationLoadBalancerType]
@@ -329,6 +333,10 @@ func (l *LoadBalancer) CreateQingCloudLB() error {
 		createInput.EIPs = qcservice.StringSlice(l.EIPs)
 	} else {
 		createInput.VxNet = &l.VxnetID
+		if l.InternalIP != "" {
+			klog.V(1).Infof("Set %s for lb %s", l.InternalIP, l.Name)
+			createInput.PrivateIP = &l.InternalIP
+		}
 	}
 	lb, err := l.lbExec.Create(createInput)
 	if err != nil {
@@ -353,7 +361,11 @@ func (l *LoadBalancer) CreateQingCloudLB() error {
 		klog.Errorf("Failed to make loadbalancer %s go into effect", l.Name)
 		return err
 	}
-	l.GenerateK8sLoadBalancer()
+	err = l.GenerateK8sLoadBalancer()
+	if err != nil {
+		klog.Errorf("Failed to get ip of loadBalancer %s", l.Name)
+		return err
+	}
 	klog.V(1).Infof("Loadbalancer %s created succeefully", l.Name)
 	return nil
 }
@@ -562,6 +574,9 @@ func (l *LoadBalancer) GenerateK8sLoadBalancer() error {
 
 	if l.NetworkType == NetworkModeInternal {
 		for _, ip := range l.Status.QcLoadBalancer.PrivateIPs {
+			if l.InternalIP != "" && l.InternalIP != *ip {
+				return fmt.Errorf("Specify ip %s but got %s of lb %s", l.InternalIP, *ip, l.Name)
+			}
 			status.Ingress = append(status.Ingress, corev1.LoadBalancerIngress{IP: *ip})
 		}
 	} else {

test/test_cases/internal.yaml

@@ -6,6 +6,7 @@ metadata:
   annotations:
     service.beta.kubernetes.io/qingcloud-load-balancer-type: "0"
     service.beta.kubernetes.io/qingcloud-load-balancer-network-type: "internal"
+    service.beta.kubernetes.io/qingcloud-load-balancer-internal-ip: "192.168.98.200"
 spec:
   selector:
     app:  mylbapp