feat: Introduce connect traces (#624)

## This PR

Partially address issue #620

Introduce connect interceptor[1] to add distributed tracing support for
flagd.

With interceptor and sdk tracing, we can get out of the box distributed
tracing. Consider following screens,


![image](https://user-images.githubusercontent.com/8186721/232918928-c4c75290-d1fd-4021-9435-d5e079084bc7.png)


![image](https://user-images.githubusercontent.com/8186721/232918838-0dbb488b-e7df-4c42-9bd1-960879291541.png)

note - follow up pr is ready at
https://github.com/open-feature/go-sdk-contrib


[1] - https://connect.build/docs/go/observability/

---------

Signed-off-by: Kavindu Dodanduwa <kavindudodanduwa@gmail.com>

Author: Kavindu-Dodan

core/go.mod

@@ -7,6 +7,7 @@ require (
 	buf.build/gen/go/open-feature/flagd/grpc/go v1.3.0-20230317150644-afd1cc2ef580.1
 	buf.build/gen/go/open-feature/flagd/protocolbuffers/go v1.29.1-20230317150644-afd1cc2ef580.1
 	github.com/bufbuild/connect-go v1.7.0
+	github.com/bufbuild/connect-opentelemetry-go v0.1.0
 	github.com/diegoholiveira/jsonlogic/v3 v3.2.7
 	github.com/fsnotify/fsnotify v1.6.0
 	github.com/golang/mock v1.6.0
@@ -41,6 +42,8 @@ require (
 )
 
 require (
+	github.com/Masterminds/goutils v1.1.1 // indirect
+	github.com/Masterminds/semver v1.5.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cenkalti/backoff/v4 v4.2.0 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
@@ -53,6 +56,7 @@ require (
 	github.com/go-openapi/jsonpointer v0.19.6 // indirect
 	github.com/go-openapi/jsonreference v0.20.2 // indirect
 	github.com/go-openapi/swag v0.22.3 // indirect
+	github.com/go-task/slim-sprig v2.20.0+incompatible // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
@@ -61,11 +65,11 @@ require (
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/uuid v1.3.0 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3 // indirect
+	github.com/huandu/xstrings v1.4.0 // indirect
 	github.com/imdario/mergo v0.3.13 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.4 // indirect
-	github.com/kr/pretty v0.3.1 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/mitchellh/copystructure v1.2.0 // indirect
@@ -78,6 +82,7 @@ require (
 	github.com/prometheus/client_model v0.3.0 // indirect
 	github.com/prometheus/common v0.42.0 // indirect
 	github.com/prometheus/procfs v0.9.0 // indirect
+	github.com/rogpeppe/go-internal v1.9.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
 	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
@@ -91,6 +96,7 @@ require (
 	golang.org/x/term v0.7.0 // indirect
 	golang.org/x/text v0.9.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
+	golang.org/x/tools v0.7.0 // indirect
 	gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/genproto v0.0.0-20230221151758-ace64dc21148 // indirect

core/go.sum

@@ -134,6 +134,7 @@ func FromConfig(logger *logger.Logger, version string, config Config) (*Runtime,
 			CertPath:    config.ServiceCertPath,
 			SocketPath:  config.ServiceSocketPath,
 			CORS:        config.CORS,
+			Options:     telemetry.BuildConnectOptions(telCfg),
 		},
 		SyncImpl: iSyncs,
 	}, nil

core/pkg/runtime/from_config.go

@@ -116,7 +116,7 @@ func (s *ConnectService) setupServer(svcConf service.Configuration) (net.Listene
 		s.eventingConfiguration,
 		s.metrics,
 	)
-	path, handler := schemaConnectV1.NewServiceHandler(fes)
+	path, handler := schemaConnectV1.NewServiceHandler(fes, svcConf.Options...)
 	mux.Handle(path, handler)
 
 	s.serverMtx.Lock()

core/pkg/service/flag-evaluation/connect_service.go

@@ -2,6 +2,8 @@ package service
 
 import (
 	"context"
+
+	"github.com/bufbuild/connect-go"
 )
 
 type NotificationType string
@@ -28,6 +30,7 @@ type Configuration struct {
 	KeyPath        string
 	SocketPath     string
 	CORS           []string
+	Options        []connect.HandlerOption
 }
 
 /*

core/pkg/service/iservice.go

@@ -5,6 +5,11 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/bufbuild/connect-go"
+	otelconnect "github.com/bufbuild/connect-opentelemetry-go"
+
+	"go.opentelemetry.io/otel/propagation"
+
 	"github.com/open-feature/flagd/core/pkg/logger"
 	"go.opentelemetry.io/otel"
 	"go.opentelemetry.io/otel/sdk/trace"
@@ -50,14 +55,14 @@ func BuildMetricsRecorder(
 	return NewOTelRecorder(mReader, rsc, svcName), nil
 }
 
-// BuildTraceProvider build and register the trace provider for the caller runtime. This method attempt to register
-// a global TracerProvider backed by batch SpanProcessor.Config. CollectorTarget can be used to provide the grpc
-// collector target. Providing empty target results in skipping provider registration. This results in tracers having
-// NoopTracerProvider performing no action
+// BuildTraceProvider build and register the trace provider and propagator for the caller runtime. This method
+// attempt to register a global TracerProvider backed by batch SpanProcessor.Config. CollectorTarget can be used to
+// provide the grpc collector target. Providing empty target results in skipping provider & propagator registration.
+// This results in tracers having NoopTracerProvider and propagator having No-Op TextMapPropagator performing no action
 func BuildTraceProvider(ctx context.Context, logger *logger.Logger, svc string, svcVersion string, cfg Config) error {
 	if cfg.CollectorTarget == "" {
 		logger.Warn("skipping trace provider setup as collector target is not set." +
-			" Traces will use NoopTracerProvider provider")
+			" Traces will use NoopTracerProvider provider and propagator will use no-Op TextMapPropagator")
 		return nil
 	}
 
@@ -77,10 +82,24 @@ func BuildTraceProvider(ctx context.Context, logger *logger.Logger, svc string,
 		trace.WithResource(res))
 
 	otel.SetTracerProvider(provider)
-
+	otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}, propagation.Baggage{}))
 	return nil
 }
 
+// BuildConnectOptions is a helper to build connect options based on telemetry configurations
+func BuildConnectOptions(cfg Config) []connect.HandlerOption {
+	options := []connect.HandlerOption{}
+
+	// add interceptor if configuration is available for collector
+	if cfg.CollectorTarget != "" {
+		options = append(options, connect.WithInterceptors(
+			otelconnect.NewInterceptor(otelconnect.WithTrustRemote()),
+		))
+	}
+
+	return options
+}
+
 // buildMetricReader builds a metric reader based on provided configurations
 func buildMetricReader(ctx context.Context, cfg Config) (metric.Reader, error) {
 	if cfg.MetricsExporter == "" {

core/pkg/telemetry/builder.go

@@ -107,6 +107,33 @@ func TestBuildSpanProcessor(t *testing.T) {
 	}
 }
 
+func TestBuildConnectOptions(t *testing.T) {
+	tests := []struct {
+		name        string
+		cfg         Config
+		optionCount int
+	}{
+		{
+			name:        "No options for empty/default configurations",
+			cfg:         Config{},
+			optionCount: 0,
+		},
+		{
+			name: "Connect option is set when telemetry target is set",
+			cfg: Config{
+				CollectorTarget: "localhost:8080",
+			},
+			optionCount: 1,
+		},
+	}
+
+	for _, test := range tests {
+		options := BuildConnectOptions(test.cfg)
+
+		require.Len(t, options, test.optionCount, "option count mismatch for test %s", test.name)
+	}
+}
+
 func TestBuildResourceFor(t *testing.T) {
 	svc := "testSvc"
 	svcVersion := "0.0.1"

core/pkg/telemetry/builder_test.go

@@ -15,8 +15,10 @@ require (
 	buf.build/gen/go/open-feature/flagd/bufbuild/connect-go v1.5.2-20230222100723-491ee098dd92.1 // indirect
 	buf.build/gen/go/open-feature/flagd/grpc/go v1.3.0-20230317150644-afd1cc2ef580.1 // indirect
 	buf.build/gen/go/open-feature/flagd/protocolbuffers/go v1.29.1-20230317150644-afd1cc2ef580.1 // indirect
+	github.com/Masterminds/goutils v1.1.1 // indirect
+	github.com/Masterminds/semver v1.5.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/bufbuild/connect-go v1.5.2 // indirect
+	github.com/bufbuild/connect-go v1.7.0 // indirect
 	github.com/cenkalti/backoff/v4 v4.2.0 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/common-nighthawk/go-figure v0.0.0-20210622060536-734e95fb86be // indirect
@@ -30,6 +32,7 @@ require (
 	github.com/go-openapi/jsonpointer v0.19.6 // indirect
 	github.com/go-openapi/jsonreference v0.20.2 // indirect
 	github.com/go-openapi/swag v0.22.3 // indirect
+	github.com/go-task/slim-sprig v2.20.0+incompatible // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
@@ -39,6 +42,7 @@ require (
 	github.com/google/uuid v1.3.0 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
+	github.com/huandu/xstrings v1.4.0 // indirect
 	github.com/imdario/mergo v0.3.13 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
@@ -54,17 +58,18 @@ require (
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
-	github.com/open-feature/open-feature-operator v0.2.32 // indirect
+	github.com/open-feature/open-feature-operator v0.2.34 // indirect
 	github.com/open-feature/schemas v0.2.8 // indirect
 	github.com/pelletier/go-toml/v2 v2.0.6 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
-	github.com/prometheus/client_golang v1.14.0 // indirect
+	github.com/prometheus/client_golang v1.15.0 // indirect
 	github.com/prometheus/client_model v0.3.0 // indirect
-	github.com/prometheus/common v0.40.0 // indirect
+	github.com/prometheus/common v0.42.0 // indirect
 	github.com/prometheus/procfs v0.9.0 // indirect
 	github.com/robfig/cron v1.2.0 // indirect
-	github.com/rs/cors v1.8.3 // indirect
-	github.com/rs/xid v1.4.0 // indirect
+	github.com/rogpeppe/go-internal v1.9.0 // indirect
+	github.com/rs/cors v1.9.0 // indirect
+	github.com/rs/xid v1.5.0 // indirect
 	github.com/spf13/afero v1.9.3 // indirect
 	github.com/spf13/cast v1.5.0 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
@@ -87,16 +92,16 @@ require (
 	go.opentelemetry.io/otel/trace v1.14.0 // indirect
 	go.opentelemetry.io/proto/otlp v0.19.0 // indirect
 	go.uber.org/atomic v1.10.0 // indirect
-	go.uber.org/goleak v1.2.1 // indirect
 	go.uber.org/multierr v1.9.0 // indirect
-	golang.org/x/crypto v0.7.0 // indirect
-	golang.org/x/net v0.8.0 // indirect
+	golang.org/x/crypto v0.8.0 // indirect
+	golang.org/x/net v0.9.0 // indirect
 	golang.org/x/oauth2 v0.5.0 // indirect
 	golang.org/x/sync v0.1.0 // indirect
 	golang.org/x/sys v0.7.0 // indirect
 	golang.org/x/term v0.7.0 // indirect
 	golang.org/x/text v0.9.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
+	golang.org/x/tools v0.7.0 // indirect
 	gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/genproto v0.0.0-20230221151758-ace64dc21148 // indirect