feat: opentelemetry integration, removed self designed tracing (zerom…

…icro#1111)

* feat: opentelemetry integration, removed self designed tracing

* feat: support zipkin on opentelemetry integration

* feat: support zipkin on opentelemetry integration, enable it in conf

* style: format code

* fix: support logx without exporter configured

* fix: check return values

* refactor: simplify code

* refactor: simplify opentelemetry integration

* ci: fix staticcheck errors

.gitignore

@@ -15,9 +15,6 @@
 **/.DS_Store
 **/logs
 
-# ignore adhoc test code
-**/adhoc
-
 # gitlab ci
 .cache
 

core/logx/tracelogger.go

@@ -7,7 +7,6 @@ import (
 	"time"
 
 	"github.com/tal-tech/go-zero/core/timex"
-	"github.com/tal-tech/go-zero/core/trace/tracespec"
 	"go.opentelemetry.io/otel/trace"
 )
 
@@ -94,35 +93,19 @@ func WithContext(ctx context.Context) Logger {
 }
 
 func spanIdFromContext(ctx context.Context) string {
-	span := trace.SpanFromContext(ctx)
-	if span.IsRecording() {
-		spanCtx := span.SpanContext()
-		if spanCtx.IsValid() {
-			return spanCtx.SpanID().String()
-		}
+	spanCtx := trace.SpanContextFromContext(ctx)
+	if spanCtx.HasSpanID() {
+		return spanCtx.SpanID().String()
 	}
 
-	t, ok := ctx.Value(tracespec.TracingKey).(tracespec.Trace)
-	if !ok {
-		return ""
-	}
-
-	return t.SpanId()
+	return ""
 }
 
 func traceIdFromContext(ctx context.Context) string {
-	span := trace.SpanFromContext(ctx)
-	if span.IsRecording() {
-		spanCtx := span.SpanContext()
-		if spanCtx.IsValid() {
-			return span.SpanContext().TraceID().String()
-		}
-	}
-
-	t, ok := ctx.Value(tracespec.TracingKey).(tracespec.Trace)
-	if !ok {
-		return ""
+	spanCtx := trace.SpanContextFromContext(ctx)
+	if spanCtx.HasTraceID() {
+		return spanCtx.TraceID().String()
 	}
 
-	return t.TraceId()
+	return ""
 }

core/logx/tracelogger_test.go

@@ -9,71 +9,90 @@ import (
 	"time"
 
 	"github.com/stretchr/testify/assert"
-	"github.com/tal-tech/go-zero/core/trace/tracespec"
+	"go.opentelemetry.io/otel"
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
 )
 
 const (
-	mockTraceID = "mock-trace-id"
-	mockSpanID  = "mock-span-id"
+	traceKey = "trace"
+	spanKey  = "span"
 )
 
-var mock tracespec.Trace = new(mockTrace)
-
 func TestTraceLog(t *testing.T) {
 	var buf mockWriter
 	atomic.StoreUint32(&initialized, 1)
-	ctx := context.WithValue(context.Background(), tracespec.TracingKey, mock)
+	otp := otel.GetTracerProvider()
+	tp := sdktrace.NewTracerProvider(sdktrace.WithSampler(sdktrace.AlwaysSample()))
+	otel.SetTracerProvider(tp)
+	defer otel.SetTracerProvider(otp)
+
+	ctx, _ := tp.Tracer("foo").Start(context.Background(), "bar")
 	WithContext(ctx).(*traceLogger).write(&buf, levelInfo, testlog)
-	assert.True(t, strings.Contains(buf.String(), mockTraceID))
-	assert.True(t, strings.Contains(buf.String(), mockSpanID))
+	assert.True(t, strings.Contains(buf.String(), traceKey))
+	assert.True(t, strings.Contains(buf.String(), spanKey))
 }
 
 func TestTraceError(t *testing.T) {
 	var buf mockWriter
 	atomic.StoreUint32(&initialized, 1)
 	errorLog = newLogWriter(log.New(&buf, "", flags))
-	ctx := context.WithValue(context.Background(), tracespec.TracingKey, mock)
+	otp := otel.GetTracerProvider()
+	tp := sdktrace.NewTracerProvider(sdktrace.WithSampler(sdktrace.AlwaysSample()))
+	otel.SetTracerProvider(tp)
+	defer otel.SetTracerProvider(otp)
+
+	ctx, _ := tp.Tracer("foo").Start(context.Background(), "bar")
 	l := WithContext(ctx).(*traceLogger)
 	SetLevel(InfoLevel)
 	l.WithDuration(time.Second).Error(testlog)
-	assert.True(t, strings.Contains(buf.String(), mockTraceID))
-	assert.True(t, strings.Contains(buf.String(), mockSpanID))
+	assert.True(t, strings.Contains(buf.String(), traceKey))
+	assert.True(t, strings.Contains(buf.String(), spanKey))
 	buf.Reset()
 	l.WithDuration(time.Second).Errorf(testlog)
-	assert.True(t, strings.Contains(buf.String(), mockTraceID))
-	assert.True(t, strings.Contains(buf.String(), mockSpanID))
+	assert.True(t, strings.Contains(buf.String(), traceKey))
+	assert.True(t, strings.Contains(buf.String(), spanKey))
 }
 
 func TestTraceInfo(t *testing.T) {
 	var buf mockWriter
 	atomic.StoreUint32(&initialized, 1)
 	infoLog = newLogWriter(log.New(&buf, "", flags))
-	ctx := context.WithValue(context.Background(), tracespec.TracingKey, mock)
+	otp := otel.GetTracerProvider()
+	tp := sdktrace.NewTracerProvider(sdktrace.WithSampler(sdktrace.AlwaysSample()))
+	otel.SetTracerProvider(tp)
+	defer otel.SetTracerProvider(otp)
+
+	ctx, _ := tp.Tracer("foo").Start(context.Background(), "bar")
 	l := WithContext(ctx).(*traceLogger)
 	SetLevel(InfoLevel)
 	l.WithDuration(time.Second).Info(testlog)
-	assert.True(t, strings.Contains(buf.String(), mockTraceID))
-	assert.True(t, strings.Contains(buf.String(), mockSpanID))
+	assert.True(t, strings.Contains(buf.String(), traceKey))
+	assert.True(t, strings.Contains(buf.String(), spanKey))
 	buf.Reset()
 	l.WithDuration(time.Second).Infof(testlog)
-	assert.True(t, strings.Contains(buf.String(), mockTraceID))
-	assert.True(t, strings.Contains(buf.String(), mockSpanID))
+	assert.True(t, strings.Contains(buf.String(), traceKey))
+	assert.True(t, strings.Contains(buf.String(), spanKey))
 }
 
 func TestTraceSlow(t *testing.T) {
 	var buf mockWriter
 	atomic.StoreUint32(&initialized, 1)
 	slowLog = newLogWriter(log.New(&buf, "", flags))
-	ctx := context.WithValue(context.Background(), tracespec.TracingKey, mock)
+	otp := otel.GetTracerProvider()
+	tp := sdktrace.NewTracerProvider(sdktrace.WithSampler(sdktrace.AlwaysSample()))
+	otel.SetTracerProvider(tp)
+	defer otel.SetTracerProvider(otp)
+
+	ctx, _ := tp.Tracer("foo").Start(context.Background(), "bar")
 	l := WithContext(ctx).(*traceLogger)
 	SetLevel(InfoLevel)
 	l.WithDuration(time.Second).Slow(testlog)
-	assert.True(t, strings.Contains(buf.String(), mockTraceID))
-	assert.True(t, strings.Contains(buf.String(), mockSpanID))
+	assert.True(t, strings.Contains(buf.String(), traceKey))
+	assert.True(t, strings.Contains(buf.String(), spanKey))
 	buf.Reset()
 	l.WithDuration(time.Second).Slowf(testlog)
-	assert.True(t, strings.Contains(buf.String(), mockTraceID))
-	assert.True(t, strings.Contains(buf.String(), mockSpanID))
+	assert.True(t, strings.Contains(buf.String(), traceKey))
+	assert.True(t, strings.Contains(buf.String(), spanKey))
 }
 
 func TestTraceWithoutContext(t *testing.T) {
@@ -83,34 +102,10 @@ func TestTraceWithoutContext(t *testing.T) {
 	l := WithContext(context.Background()).(*traceLogger)
 	SetLevel(InfoLevel)
 	l.WithDuration(time.Second).Info(testlog)
-	assert.False(t, strings.Contains(buf.String(), mockTraceID))
-	assert.False(t, strings.Contains(buf.String(), mockSpanID))
+	assert.False(t, strings.Contains(buf.String(), traceKey))
+	assert.False(t, strings.Contains(buf.String(), spanKey))
 	buf.Reset()
 	l.WithDuration(time.Second).Infof(testlog)
-	assert.False(t, strings.Contains(buf.String(), mockTraceID))
-	assert.False(t, strings.Contains(buf.String(), mockSpanID))
-}
-
-type mockTrace struct{}
-
-func (t mockTrace) TraceId() string {
-	return mockTraceID
-}
-
-func (t mockTrace) SpanId() string {
-	return mockSpanID
-}
-
-func (t mockTrace) Finish() {
-}
-
-func (t mockTrace) Fork(ctx context.Context, serviceName, operationName string) (context.Context, tracespec.Trace) {
-	return nil, nil
-}
-
-func (t mockTrace) Follow(ctx context.Context, serviceName, operationName string) (context.Context, tracespec.Trace) {
-	return nil, nil
-}
-
-func (t mockTrace) Visit(fn func(key, val string) bool) {
+	assert.False(t, strings.Contains(buf.String(), traceKey))
+	assert.False(t, strings.Contains(buf.String(), spanKey))
 }

core/service/serviceconf.go

@@ -7,7 +7,7 @@ import (
 	"github.com/tal-tech/go-zero/core/logx"
 	"github.com/tal-tech/go-zero/core/prometheus"
 	"github.com/tal-tech/go-zero/core/stat"
-	"github.com/tal-tech/go-zero/core/trace/opentelemetry"
+	"github.com/tal-tech/go-zero/core/trace"
 )
 
 const (
@@ -27,10 +27,10 @@ const (
 type ServiceConf struct {
 	Name       string
 	Log        logx.LogConf
-	Mode       string               `json:",default=pro,options=dev|test|rt|pre|pro"`
-	MetricsUrl string               `json:",optional"`
-	Prometheus prometheus.Config    `json:",optional"`
-	Telemetry  opentelemetry.Config `json:",optional"`
+	Mode       string            `json:",default=pro,options=dev|test|rt|pre|pro"`
+	MetricsUrl string            `json:",optional"`
+	Prometheus prometheus.Config `json:",optional"`
+	Telemetry  trace.Config      `json:",optional"`
 }
 
 // MustSetUp sets up the service, exits on error.
@@ -55,7 +55,7 @@ func (sc ServiceConf) SetUp() error {
 	if len(sc.Telemetry.Name) == 0 {
 		sc.Telemetry.Name = sc.Name
 	}
-	opentelemetry.StartAgent(sc.Telemetry)
+	trace.StartAgent(sc.Telemetry)
 
 	if len(sc.MetricsUrl) > 0 {
 		stat.SetReportWriter(stat.NewRemoteWriter(sc.MetricsUrl))

core/trace/agent.go

@@ -0,0 +1,69 @@
+package trace
+
+import (
+	"fmt"
+	"sync"
+
+	"github.com/tal-tech/go-zero/core/logx"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/exporters/jaeger"
+	"go.opentelemetry.io/otel/exporters/zipkin"
+	"go.opentelemetry.io/otel/propagation"
+	"go.opentelemetry.io/otel/sdk/resource"
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+	semconv "go.opentelemetry.io/otel/semconv/v1.4.0"
+)
+
+const (
+	kindJaeger = "jaeger"
+	kindZipkin = "zipkin"
+)
+
+var once sync.Once
+
+// StartAgent starts a opentelemetry agent.
+func StartAgent(c Config) {
+	once.Do(func() {
+		startAgent(c)
+	})
+}
+
+func createExporter(c Config) (sdktrace.SpanExporter, error) {
+	// Just support jaeger now, more for later
+	switch c.Batcher {
+	case kindJaeger:
+		return jaeger.New(jaeger.WithCollectorEndpoint(jaeger.WithEndpoint(c.Endpoint)))
+	case kindZipkin:
+		return zipkin.New(c.Endpoint)
+	default:
+		return nil, fmt.Errorf("unknown exporter: %s", c.Batcher)
+	}
+}
+
+func startAgent(c Config) {
+	opts := []sdktrace.TracerProviderOption{
+		// Set the sampling rate based on the parent span to 100%
+		sdktrace.WithSampler(sdktrace.ParentBased(sdktrace.TraceIDRatioBased(c.Sampler))),
+		// Record information about this application in an Resource.
+		sdktrace.WithResource(resource.NewSchemaless(semconv.ServiceNameKey.String(c.Name))),
+	}
+
+	if len(c.Endpoint) > 0 {
+		exp, err := createExporter(c)
+		if err != nil {
+			logx.Error(err)
+			return
+		}
+
+		// Always be sure to batch in production.
+		opts = append(opts, sdktrace.WithBatcher(exp))
+	}
+
+	tp := sdktrace.NewTracerProvider(opts...)
+	otel.SetTracerProvider(tp)
+	otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(
+		propagation.TraceContext{}, propagation.Baggage{}))
+	otel.SetErrorHandler(otel.ErrorHandlerFunc(func(err error) {
+		logx.Errorf("[otel] error: %v", err)
+	}))
+}

core/trace/opentelemetry/attributes.go → core/trace/attributes.go

@@ -1,4 +1,4 @@
-package opentelemetry
+package trace
 
 import (
 	"go.opentelemetry.io/otel/attribute"