Merge pull request #134 from qiangzii/master

update log; fix ccm deployment yaml

Makefile

@@ -2,7 +2,7 @@
 # GitHub viewer defaults to 8, change with ?ts=4 in URL
 
 GIT_REPOSITORY= github.com/yunify/qingcloud-cloud-controller-manager
-IMG?= qingcloud/cloud-controller-manager:v1.4.18
+IMG?= qingcloud/cloud-controller-manager:v1.4.19
 #Debug level: 0, 1, 2 (1 true, 2 use bash)
 DEBUG?= 0
 DOCKERFILE?= deploy/Dockerfile
@@ -31,9 +31,9 @@ endif
 publish: test build
 	docker build -t ${IMG}  -f ${DOCKERFILE} bin/
 	@echo "updating kustomize image patch file for manager resource"
-	sed -i'' -e 's@image: .*@image: '"${IMG}"'@' config/${TARGET}/manager_image_patch.yaml
+	#sed -i'' -e 's@image: .*@image: '"${IMG}"'@' config/${TARGET}/manager_image_patch.yaml
 	docker push ${IMG}
-	kustomize build config/${TARGET} > ${DEPLOY}
+	#kustomize build config/${TARGET} > ${DEPLOY}
 
 clean:
 	rm -rf bin/

deploy/kube-cloud-controller-manager.yaml

@@ -137,7 +137,7 @@ spec:
         - -v=3
         - --cloud-provider=qingcloud
         - --cloud-config=/etc/kubernetes/qingcloud.yaml
-        image: qingcloud/cloud-controller-manager:v1.4.18
+        image: qingcloud/cloud-controller-manager:v1.4.19
         imagePullPolicy: IfNotPresent
         name: qingcloud-cloud-controller-manager
         resources:
@@ -167,8 +167,10 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
+          defaultMode: 420
           name: lbconfig
         name: lbconfig
-      - name: qingcloud
-        secret:
-          secretName: qcsecret
+      - configMap:
+          defaultMode: 420
+          name: ccm-qingcloud
+        name: qingcloud

go.mod

@@ -1,6 +1,6 @@
 module github.com/yunify/qingcloud-cloud-controller-manager
 
-go 1.16
+go 1.19
 
 require (
 	github.com/davecgh/go-spew v1.1.1
@@ -13,11 +13,80 @@ require (
 	k8s.io/cloud-provider v0.21.1
 	k8s.io/component-base v0.21.1
 	k8s.io/controller-manager v0.21.1
-	k8s.io/klog v1.0.0
 	k8s.io/klog/v2 v2.9.0
 )
 
+require (
+	github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 // indirect
+	github.com/NYTimes/gziphandler v1.1.1 // indirect
+	github.com/PuerkitoBio/purell v1.1.1 // indirect
+	github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/blang/semver v3.5.1+incompatible // indirect
+	github.com/cespare/xxhash/v2 v2.1.1 // indirect
+	github.com/coreos/go-semver v0.3.0 // indirect
+	github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e // indirect
+	github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect
+	github.com/emicklei/go-restful v2.9.5+incompatible // indirect
+	github.com/evanphx/json-patch v4.9.0+incompatible // indirect
+	github.com/go-logr/logr v0.4.0 // indirect
+	github.com/go-openapi/jsonpointer v0.19.3 // indirect
+	github.com/go-openapi/jsonreference v0.19.3 // indirect
+	github.com/go-openapi/spec v0.19.5 // indirect
+	github.com/go-openapi/swag v0.19.5 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
+	github.com/golang/protobuf v1.4.3 // indirect
+	github.com/google/go-cmp v0.5.2 // indirect
+	github.com/google/gofuzz v1.1.0 // indirect
+	github.com/google/uuid v1.1.2 // indirect
+	github.com/googleapis/gnostic v0.4.1 // indirect
+	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
+	github.com/hashicorp/golang-lru v0.5.1 // indirect
+	github.com/imdario/mergo v0.3.5 // indirect; indirectsd
+	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/json-iterator/go v1.1.10 // indirect
+	github.com/mailru/easyjson v0.7.0 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
+	github.com/moby/term v0.0.0-20201216013528-df9cb8a40635 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.1 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/prometheus/client_golang v1.7.1 // indirect
+	github.com/prometheus/client_model v0.2.0 // indirect
+	github.com/prometheus/common v0.10.0 // indirect
+	github.com/prometheus/procfs v0.2.0 // indirect
+	github.com/sirupsen/logrus v1.7.0 // indirect
+	github.com/spf13/cobra v1.1.1 // indirect
+	go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489 // indirect
+	go.uber.org/atomic v1.4.0 // indirect
+	go.uber.org/multierr v1.1.0 // indirect
+	go.uber.org/zap v1.10.0 // indirect
+	golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 // indirect
+	golang.org/x/net v0.0.0-20210224082022-3d97a244fca7 // indirect
+	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d // indirect
+	golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 // indirect
+	golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073 // indirect
+	golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d // indirect
+	golang.org/x/text v0.3.4 // indirect
+	golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect
+	google.golang.org/appengine v1.6.5 // indirect
+	google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a // indirect
+	google.golang.org/grpc v1.27.1 // indirect
+	google.golang.org/protobuf v1.25.0 // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
+	k8s.io/apiserver v0.21.1 // indirect
+	k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7 // indirect
+	k8s.io/utils v0.0.0-20201110183641-67b214c5f920 // indirect
+	sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15 // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.1.0 // indirect
+	sigs.k8s.io/yaml v1.2.0 // indirect
+)
+
 replace (
+	// github.com/yunify/qingcloud-sdk-go => ../qingcloud-sdk-go
 	k8s.io/api => k8s.io/api v0.21.1
 	k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.21.1
 	k8s.io/apimachinery => k8s.io/apimachinery v0.21.1

go.sum

@@ -59,7 +59,6 @@ github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJm
 github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ=
 github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
-github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
@@ -685,8 +684,6 @@ k8s.io/component-base v0.21.1/go.mod h1:NgzFZ2qu4m1juby4TnrmpR8adRk6ka62YdH5DkII
 k8s.io/controller-manager v0.21.1 h1:IFbukN4M0xl3OHEasNQ91h2MLEAMk3uQrBU4+Edka8w=
 k8s.io/controller-manager v0.21.1/go.mod h1:8ugs8DCcHqybiwdVERhnnyGoS5Ksq/ea1p2B0CosHyc=
 k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
-k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8=
-k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I=
 k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=
 k8s.io/klog/v2 v2.8.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec=
 k8s.io/klog/v2 v2.9.0 h1:D7HV+n1V57XeZ0m6tdRkfknthUaM06VFbWldOFh8kzM=

pkg/controllers/clusternode/cluster_node_controller.go

@@ -19,7 +19,7 @@ import (
 	cloudproviderapp "k8s.io/cloud-provider/app"
 	cloudcontrollerconfig "k8s.io/cloud-provider/app/config"
 	genericcontrollermanager "k8s.io/controller-manager/app"
-	"k8s.io/klog"
+	"k8s.io/klog/v2"
 
 	"github.com/yunify/qingcloud-cloud-controller-manager/pkg/controllers/utils"
 	"github.com/yunify/qingcloud-cloud-controller-manager/pkg/qingcloud"
@@ -167,7 +167,7 @@ func (cnc *ClusterNodeController) processNextWorkItem() bool {
 	return true
 }
 
-// handleNodesUpdate handle service backend according to node lables
+// handleNodesUpdate handle node labels change for service which has node lable annotation
 func (cnc *ClusterNodeController) handleNodesUpdate(key string) error {
 	startTime := time.Now()
 	defer func() {
@@ -197,7 +197,8 @@ func (cnc *ClusterNodeController) handleNodesUpdate(key string) error {
 		_, ok := svc.Annotations[qingcloud.ServiceAnnotationBackendLabel]
 		if ok && svc.Spec.Type == corev1.ServiceTypeLoadBalancer &&
 			svc.Spec.ExternalTrafficPolicy == corev1.ServiceExternalTrafficPolicyTypeCluster {
-			klog.Infof("service %s serviceType = %s, externalTrafficPolicy = %s, also has backend label annotation , going to update loadbalancer", svc.Name, svc.Spec.Type, svc.Spec.ExternalTrafficPolicy)
+			klog.Infof("node lable changed and service has backend label annotation, going to update loadbalancer for service %s/%s serviceType = %s, externalTrafficPolicy = %s, ",
+				svc.Namespace, svc.Name, svc.Spec.Type, svc.Spec.ExternalTrafficPolicy)
 
 			// 4. update lb
 			lbInterface, _ := cnc.cloud.LoadBalancer()

pkg/controllers/endpoint/endpoint_controller.go

@@ -20,7 +20,7 @@ import (
 	cloudproviderapp "k8s.io/cloud-provider/app"
 	cloudcontrollerconfig "k8s.io/cloud-provider/app/config"
 	genericcontrollermanager "k8s.io/controller-manager/app"
-	"k8s.io/klog"
+	"k8s.io/klog/v2"
 
 	"github.com/yunify/qingcloud-cloud-controller-manager/pkg/controllers/utils"
 )
@@ -160,6 +160,7 @@ func (epc *EndpointController) processNextWorkItem() bool {
 	return true
 }
 
+// handleEndpointsUpdate handle endpoint change for loadlanbacer service which externalTrafficPolicy is local
 func (epc *EndpointController) handleEndpointsUpdate(key string) error {
 	startTime := time.Now()
 	defer func() {
@@ -182,10 +183,12 @@ func (epc *EndpointController) handleEndpointsUpdate(key string) error {
 	}
 	// ignore service which service type != loadbalancer or externalTrafficPolicy != Local
 	if svc.Spec.Type != corev1.ServiceTypeLoadBalancer || svc.Spec.ExternalTrafficPolicy != corev1.ServiceExternalTrafficPolicyTypeLocal {
-		klog.V(4).Infof("service %s serviceType = %s, externalTrafficPolicy = %s, skip handle endpoint update", svc.Name, svc.Spec.Type, svc.Spec.ExternalTrafficPolicy)
+		klog.V(4).Infof("skip handle endpoint update for service %s/%s serviceType = %s, externalTrafficPolicy = %s",
+			svc.Namespace, svc.Name, svc.Spec.Type, svc.Spec.ExternalTrafficPolicy)
 		return nil
 	}
-	klog.Infof("service %s serviceType = %s, externalTrafficPolicy = %s, going to handle endpoint update", svc.Name, svc.Spec.Type, svc.Spec.ExternalTrafficPolicy)
+	klog.Infof("going to handle endpoint update for service %s/%s serviceType = %s, externalTrafficPolicy = %s",
+		svc.Namespace, svc.Name, svc.Spec.Type, svc.Spec.ExternalTrafficPolicy)
 
 	// 2. get node list
 	var nodes []*corev1.Node
@@ -203,9 +206,9 @@ func (epc *EndpointController) handleEndpointsUpdate(key string) error {
 	cloudLbIntf, _ := epc.cloud.LoadBalancer()
 	err = cloudLbIntf.UpdateLoadBalancer(context.TODO(), "", svc, nodes)
 	if err != nil {
-		return fmt.Errorf("update loadbalancer for service %s/%s error: %v", svc.Namespace, svc.Name, err)
+		return fmt.Errorf("update loadbalancer for service %s/%s error: %v", namespace, name, err)
 	}
-	klog.Infof("update loadbalancer for service %s/%s success", svc.Namespace, svc.Name)
+	klog.Infof("update loadbalancer for service %s/%s success", namespace, name)
 
 	return nil
 }

pkg/controllers/service/cloud_service_controller.go

@@ -18,7 +18,7 @@ import (
 	cloudproviderapp "k8s.io/cloud-provider/app"
 	cloudcontrollerconfig "k8s.io/cloud-provider/app/config"
 	genericcontrollermanager "k8s.io/controller-manager/app"
-	"k8s.io/klog"
+	"k8s.io/klog/v2"
 
 	"github.com/yunify/qingcloud-cloud-controller-manager/pkg/qingcloud"
 )
@@ -182,10 +182,11 @@ func (sc *ServiceController) processNextWorkItem() bool {
 	return true
 }
 
+// handleServiceUpdate clean lb listener for service which new svc del the lb annotation
 func (sc *ServiceController) handleServiceUpdate(svc *corev1.Service) error {
 	startTime := time.Now()
 	defer func() {
-		klog.V(4).Infof("Finished handleEndpointsUpdate  %s/%s (%v)", svc.Namespace, svc.Name, time.Since(startTime))
+		klog.V(4).Infof("Finished handleServiceUpdate  %s/%s (%v)", svc.Namespace, svc.Name, time.Since(startTime))
 	}()
 
 	cloudLbIntf, _ := sc.cloud.LoadBalancer()

pkg/controllers/utils/node.go

@@ -2,7 +2,7 @@ package utils
 
 import (
 	corev1 "k8s.io/api/core/v1"
-	"k8s.io/klog"
+	"k8s.io/klog/v2"
 )
 
 // check if the node should be backend or not

pkg/executor/backend.go

@@ -5,7 +5,7 @@ import (
 
 	"github.com/davecgh/go-spew/spew"
 	qcservice "github.com/yunify/qingcloud-sdk-go/service"
-	"k8s.io/klog"
+	"k8s.io/klog/v2"
 
 	"github.com/yunify/qingcloud-cloud-controller-manager/pkg/apis"
 	"github.com/yunify/qingcloud-cloud-controller-manager/pkg/errors"
@@ -20,13 +20,18 @@ func (q *QingCloudClient) DeleteBackends(ids []*string) error {
 		klog.Warningln("No backends to delete, pls check the inputs")
 		return nil
 	}
-	output, err := q.LBService.DeleteLoadBalancerBackends(&qcservice.DeleteLoadBalancerBackendsInput{
+	input := &qcservice.DeleteLoadBalancerBackendsInput{
 		LoadBalancerBackends: ids,
-	})
-	if *output.RetCode != 0 {
-		return errors.NewCommonServerError(ResourceNameBackend, spew.Sdump(ids), "DeleteBackends", *output.Message)
 	}
-	return err
+	output, err := q.LBService.DeleteLoadBalancerBackends(input)
+	if err != nil || *output.RetCode != 0 {
+		klog.V(4).Infof("failed to delete backends, err=%s, input=%s, output=%s", spew.Sdump(err), spew.Sdump(input), spew.Sdump(output))
+		if err != nil {
+			return fmt.Errorf("failed to delete backends,err=%v", err)
+		}
+		return fmt.Errorf("failed to delete backends, code=%d, msg=%s", *output.RetCode, *output.Message)
+	}
+	return nil
 }
 
 // need update lb
@@ -48,8 +53,13 @@ func (q *QingCloudClient) CreateBackends(backends []*apis.LoadBalancerBackend) (
 	}
 
 	output, err := q.LBService.AddLoadBalancerBackends(input)
+
 	if err != nil || *output.RetCode != 0 {
-		return nil, fmt.Errorf("failed to create backends, err=%s, input=%s, output=%s", spew.Sdump(err), spew.Sdump(input), spew.Sdump(output))
+		klog.V(4).Infof("failed to create backends, err=%s, input=%s, output=%s", spew.Sdump(err), spew.Sdump(input), spew.Sdump(output))
+		if err != nil {
+			return nil, fmt.Errorf("failed to create backends,err=%v", err)
+		}
+		return nil, fmt.Errorf("failed to create backends, code=%d, msg=%s", *output.RetCode, *output.Message)
 	}
 
 	return output.LoadBalancerBackends, nil

pkg/executor/eip.go

@@ -9,7 +9,7 @@ import (
 	qcservice "github.com/yunify/qingcloud-sdk-go/service"
 	"github.com/yunify/qingcloud-sdk-go/utils"
 	"k8s.io/apimachinery/pkg/util/wait"
-	"k8s.io/klog"
+	"k8s.io/klog/v2"
 
 	"github.com/yunify/qingcloud-cloud-controller-manager/pkg/apis"
 )
@@ -25,7 +25,11 @@ func (q *QingCloudClient) getEIPByID(id string) (*qcservice.EIP, error) {
 		EIPs: []*string{&id},
 	})
 	if err != nil || *output.RetCode != 0 {
-		return nil, fmt.Errorf("failed to get eip by id %s, err=%s, output=%s", id, spew.Sdump(err), spew.Sdump(output))
+		klog.V(4).Infof("failed to get eip by id %s, err=%s, output=%s", id, spew.Sdump(err), spew.Sdump(output))
+		if err != nil {
+			return nil, fmt.Errorf("failed to get eip by id %s, err=%v", id, err)
+		}
+		return nil, fmt.Errorf("failed to get eip by id %s, code=%d, msg=%s", id, *output.RetCode, *output.Message)
 	}
 
 	return output.EIPSet[0], nil
@@ -50,7 +54,8 @@ func (q *QingCloudClient) ReleaseEIP(ids []*string) error {
 		//QingCloud Error: Code (1400), Message (PermissionDenied, resource [eip-5ywkioa5] lease info not ready yet, please try later)
 		if (err != nil && !strings.Contains(err.Error(), "QingCloud Error: Code (1400)")) ||
 			(output != nil && *output.RetCode != 0 && *output.RetCode != 1400) {
-			return false, fmt.Errorf("failed to release eip, err=%s, input=%s, output=%s", spew.Sdump(err), spew.Sdump(input), spew.Sdump(output))
+			klog.V(4).Infof("failed to release eip, err=%s, input=%s, output=%s", spew.Sdump(err), spew.Sdump(input), spew.Sdump(output))
+			return false, fmt.Errorf("failed to release eip, err=%v", err)
 		} else if err == nil && *output.RetCode == 0 {
 			return true, nil
 		}
@@ -89,7 +94,11 @@ func (q *QingCloudClient) AllocateEIP(eip *apis.EIP) (*apis.EIP, error) {
 
 	output, err := q.EIPService.AllocateEIPs(input)
 	if err != nil || *output.RetCode != 0 {
-		return nil, fmt.Errorf("failed to allocate eip, err=%s, input=%s, output=%s", spew.Sdump(err), spew.Sdump(input), spew.Sdump(output))
+		klog.V(4).Infof("failed to allocate eip, err=%s, input=%s, output=%s", spew.Sdump(err), spew.Sdump(input), spew.Sdump(output))
+		if err != nil {
+			return nil, fmt.Errorf("failed to allocate eip, err=%v", err)
+		}
+		return nil, fmt.Errorf("failed to allocate eip, code=%d, msg=%s", *output.RetCode, *output.Message)
 	}
 
 	id := output.EIPs[0]
@@ -120,13 +129,18 @@ func convertEIP(eip *qcservice.EIP) *apis.EIP {
 }
 
 func (q *QingCloudClient) GetAvaliableEIPs() ([]*apis.EIP, error) {
-	output, err := q.EIPService.DescribeEIPs(&qcservice.DescribeEIPsInput{
+	input := &qcservice.DescribeEIPsInput{
 		Owner:  &q.Config.UserID,
 		Status: []*string{qcservice.String(EIPStatusAvailable)},
-	})
+	}
+	output, err := q.EIPService.DescribeEIPs(input)
 
 	if err != nil || *output.RetCode != 0 {
-		return nil, fmt.Errorf("failed to get avaliable eips, err=%s, output=%s", spew.Sdump(err), spew.Sdump(output))
+		klog.V(4).Infof("failed to get avaliable eips, err=%s, input=%s, output=%s", spew.Sdump(err), spew.Sdump(input), spew.Sdump(output))
+		if err != nil {
+			return nil, fmt.Errorf("failed to get avaliable eips, err=%v", err)
+		}
+		return nil, fmt.Errorf("failed to get avaliable eips, code=%d, msg=%s", *output.RetCode, *output.Message)
 	}
 
 	result := make([]*apis.EIP, 0)