implement k8s service discovery (zeromicro#988)

* implement k8s service discovery

* simplify code

* use default namespace if not provided

* disable codecov bot comment

* ignore adhoc dir

* simplify building target in NewClient

* reformat code

* Fix filepath (zeromicro#990)

* format code, and reorg imports (zeromicro#991)

* add more unit test

Co-authored-by: anqiansong <anqiansong@gmail.com>

.codecov.yml

@@ -1,4 +1,5 @@
+comment: false
 ignore:
   - "doc"
   - "example"
-  - "tools"
+  - "tools"

.gitignore

@@ -10,10 +10,14 @@
 !*/
 !api
 
+# ignore
 .idea
 **/.DS_Store
 **/logs
 
+# ignore adhoc test code
+**/adhoc
+
 # gitlab ci
 .cache
 

core/proc/signals.go

@@ -1,3 +1,4 @@
+//go:build linux || darwin
 // +build linux darwin
 
 package proc
@@ -12,6 +13,8 @@ import (
 
 const timeFormat = "0102150405"
 
+var done = make(chan struct{})
+
 func init() {
 	go func() {
 		var profiler Stopper
@@ -33,10 +36,21 @@ func init() {
 					profiler = nil
 				}
 			case syscall.SIGTERM:
+				select {
+				case <-done:
+					// already closed
+				default:
+					close(done)
+				}
+
 				gracefulStop(signals)
 			default:
 				logx.Error("Got unregistered signal:", v)
 			}
 		}
 	}()
 }
+
+func Done() <-chan struct{} {
+	return done
+}

core/proc/signals_test.go

@@ -0,0 +1,16 @@
+package proc
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestDone(t *testing.T) {
+	select {
+	case <-Done():
+		assert.Fail(t, "should run")
+	default:
+	}
+	assert.NotNil(t, Done())
+}

go.mod

@@ -18,18 +18,13 @@ require (
 	github.com/go-xorm/builder v0.3.4
 	github.com/golang/mock v1.4.3
 	github.com/golang/protobuf v1.5.2
-	github.com/google/go-cmp v0.5.6 // indirect
 	github.com/google/uuid v1.1.2
 	github.com/iancoleman/strcase v0.1.2
 	github.com/justinas/alice v1.2.0
-	github.com/kr/pretty v0.2.0 // indirect
-	github.com/kr/text v0.2.0 // indirect
 	github.com/lib/pq v1.3.0
 	github.com/logrusorgru/aurora v2.0.3+incompatible
 	github.com/mattn/go-colorable v0.1.6 // indirect
 	github.com/olekukonko/tablewriter v0.0.5
-	github.com/onsi/ginkgo v1.7.0 // indirect
-	github.com/onsi/gomega v1.5.0 // indirect
 	github.com/pierrec/lz4 v2.5.1+incompatible // indirect
 	github.com/prometheus/client_golang v1.11.0
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
@@ -41,18 +36,20 @@ require (
 	go.etcd.io/etcd/api/v3 v3.5.0
 	go.etcd.io/etcd/client/v3 v3.5.0
 	go.opentelemetry.io/otel v1.0.0-RC2
-	go.opentelemetry.io/otel/exporters/jaeger v1.0.0-RC2 // indirect
-	go.opentelemetry.io/otel/metric v0.20.0 // indirect
+	go.opentelemetry.io/otel/exporters/jaeger v1.0.0-RC2
 	go.opentelemetry.io/otel/sdk v1.0.0-RC2
 	go.opentelemetry.io/otel/trace v1.0.0-RC2
 	go.uber.org/automaxprocs v1.3.0
 	golang.org/x/net v0.0.0-20210716203947-853a461950ff
 	golang.org/x/sys v0.0.0-20210820121016-41cdb8703e55 // indirect
-	golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba
+	golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac
 	google.golang.org/genproto v0.0.0-20210722135532-667f2b7c528f // indirect
 	google.golang.org/grpc v1.39.0
 	google.golang.org/protobuf v1.27.1
 	gopkg.in/cheggaaa/pb.v1 v1.0.28
 	gopkg.in/h2non/gock.v1 v1.0.15
 	gopkg.in/yaml.v2 v2.4.0
+	k8s.io/api v0.22.1
+	k8s.io/apimachinery v0.22.1
+	k8s.io/client-go v0.22.1
 )

zrpc/client.go

@@ -55,13 +55,21 @@ func NewClient(c RpcClientConf, options ...ClientOption) (Client, error) {
 	}
 	opts = append(opts, options...)
 
-	var client Client
+	var target string
 	var err error
 	if len(c.Endpoints) > 0 {
-		client, err = internal.NewClient(internal.BuildDirectTarget(c.Endpoints), opts...)
-	} else if err = c.Etcd.Validate(); err == nil {
-		client, err = internal.NewClient(internal.BuildDiscovTarget(c.Etcd.Hosts, c.Etcd.Key), opts...)
+		target = internal.BuildDirectTarget(c.Endpoints)
+	} else if len(c.Target) > 0 {
+		target = c.Target
+	} else {
+		if err = c.Etcd.Validate(); err != nil {
+			return nil, err
+		}
+
+		target = internal.BuildDiscovTarget(c.Etcd.Hosts, c.Etcd.Key)
 	}
+
+	client, err := internal.NewClient(target, opts...)
 	if err != nil {
 		return nil, err
 	}

zrpc/config.go

@@ -23,7 +23,8 @@ type (
 	// A RpcClientConf is a rpc client config.
 	RpcClientConf struct {
 		Etcd      discov.EtcdConf `json:",optional"`
-		Endpoints []string        `json:",optional=!Etcd"`
+		Endpoints []string        `json:",optional"`
+		Target    string          `json:",optional"`
 		App       string          `json:",optional"`
 		Token     string          `json:",optional"`
 		Timeout   int64           `json:",default=2000"`

zrpc/internal/resolver/directbuilder.go

@@ -20,9 +20,11 @@ func (d *directBuilder) Build(target resolver.Target, cc resolver.ClientConn, op
 			Addr: val,
 		})
 	}
-	cc.UpdateState(resolver.State{
+	if err := cc.UpdateState(resolver.State{
 		Addresses: addrs,
-	})
+	}); err != nil {
+		return nil, err
+	}
 
 	return &nopResolver{cc: cc}, nil
 }

zrpc/internal/resolver/discovbuilder.go

@@ -4,12 +4,13 @@ import (
 	"strings"
 
 	"github.com/tal-tech/go-zero/core/discov"
+	"github.com/tal-tech/go-zero/core/logx"
 	"google.golang.org/grpc/resolver"
 )
 
 type discovBuilder struct{}
 
-func (d *discovBuilder) Build(target resolver.Target, cc resolver.ClientConn, opts resolver.BuildOptions) (
+func (b *discovBuilder) Build(target resolver.Target, cc resolver.ClientConn, opts resolver.BuildOptions) (
 	resolver.Resolver, error) {
 	hosts := strings.FieldsFunc(target.Authority, func(r rune) bool {
 		return r == EndpointSepChar
@@ -26,16 +27,18 @@ func (d *discovBuilder) Build(target resolver.Target, cc resolver.ClientConn, op
 				Addr: val,
 			})
 		}
-		cc.UpdateState(resolver.State{
+		if err := cc.UpdateState(resolver.State{
 			Addresses: addrs,
-		})
+		}); err != nil {
+			logx.Error(err)
+		}
 	}
 	sub.AddListener(update)
 	update()
 
 	return &nopResolver{cc: cc}, nil
 }
 
-func (d *discovBuilder) Scheme() string {
+func (b *discovBuilder) Scheme() string {
 	return DiscovScheme
 }

zrpc/internal/resolver/discovbuilder_test.go

@@ -0,0 +1,12 @@
+package resolver
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestDiscovBuilder_Scheme(t *testing.T) {
+	var b discovBuilder
+	assert.Equal(t, DiscovScheme, b.Scheme())
+}

zrpc/internal/resolver/kube/eventhandler.go

@@ -0,0 +1,133 @@
+package kube
+
+import (
+	"sync"
+
+	"github.com/tal-tech/go-zero/core/lang"
+	"github.com/tal-tech/go-zero/core/logx"
+	v1 "k8s.io/api/core/v1"
+)
+
+type EventHandler struct {
+	update    func([]string)
+	endpoints map[string]lang.PlaceholderType
+	lock      sync.Mutex
+}
+
+func NewEventHandler(update func([]string)) *EventHandler {
+	return &EventHandler{
+		update:    update,
+		endpoints: make(map[string]lang.PlaceholderType),
+	}
+}
+
+func (h *EventHandler) OnAdd(obj interface{}) {
+	endpoints, ok := obj.(*v1.Endpoints)
+	if !ok {
+		logx.Errorf("%v is not an object with type *v1.Endpoints", obj)
+		return
+	}
+
+	h.lock.Lock()
+	defer h.lock.Unlock()
+
+	var changed bool
+	for _, sub := range endpoints.Subsets {
+		for _, point := range sub.Addresses {
+			if _, ok := h.endpoints[point.IP]; !ok {
+				h.endpoints[point.IP] = lang.Placeholder
+				changed = true
+			}
+		}
+	}
+
+	if changed {
+		h.notify()
+	}
+}
+
+func (h *EventHandler) OnDelete(obj interface{}) {
+	endpoints, ok := obj.(*v1.Endpoints)
+	if !ok {
+		logx.Errorf("%v is not an object with type *v1.Endpoints", obj)
+		return
+	}
+
+	h.lock.Lock()
+	defer h.lock.Unlock()
+
+	var changed bool
+	for _, sub := range endpoints.Subsets {
+		for _, point := range sub.Addresses {
+			if _, ok := h.endpoints[point.IP]; ok {
+				delete(h.endpoints, point.IP)
+				changed = true
+			}
+		}
+	}
+
+	if changed {
+		h.notify()
+	}
+}
+
+func (h *EventHandler) OnUpdate(oldObj, newObj interface{}) {
+	oldEndpoints, ok := oldObj.(*v1.Endpoints)
+	if !ok {
+		logx.Errorf("%v is not an object with type *v1.Endpoints", oldObj)
+		return
+	}
+
+	newEndpoints, ok := newObj.(*v1.Endpoints)
+	if !ok {
+		logx.Errorf("%v is not an object with type *v1.Endpoints", newObj)
+		return
+	}
+
+	if oldEndpoints.ResourceVersion == newEndpoints.ResourceVersion {
+		return
+	}
+
+	h.Update(newEndpoints)
+}
+
+func (h *EventHandler) Update(endpoints *v1.Endpoints) {
+	h.lock.Lock()
+	defer h.lock.Unlock()
+
+	old := h.endpoints
+	h.endpoints = make(map[string]lang.PlaceholderType)
+	for _, sub := range endpoints.Subsets {
+		for _, point := range sub.Addresses {
+			h.endpoints[point.IP] = lang.Placeholder
+		}
+	}
+
+	if diff(old, h.endpoints) {
+		h.notify()
+	}
+}
+
+func (h *EventHandler) notify() {
+	var targets []string
+
+	for k := range h.endpoints {
+		targets = append(targets, k)
+	}
+
+	h.update(targets)
+}
+
+func diff(o, n map[string]lang.PlaceholderType) bool {
+	if len(o) != len(n) {
+		return true
+	}
+
+	for k := range o {
+		if _, ok := n[k]; !ok {
+			return true
+		}
+	}
+
+	return false
+}