feat: otel tracing

cmd/vela-server/main.go

@@ -17,6 +17,7 @@ import (
 	"github.com/go-vela/server/queue"
 	"github.com/go-vela/server/scm"
 	"github.com/go-vela/server/secret"
+	"github.com/go-vela/server/tracing"
 	"github.com/go-vela/server/version"
 	"github.com/go-vela/types/constants"
 )
@@ -279,6 +280,9 @@ func main() {
 	// Add Source Flags
 	app.Flags = append(app.Flags, scm.Flags...)
 
+	// Add Tracing Flags
+	app.Flags = append(app.Flags, tracing.Flags...)
+
 	if err = app.Run(os.Args); err != nil {
 		logrus.Fatal(err)
 	}

cmd/vela-server/server.go

@@ -26,6 +26,7 @@ import (
 	"github.com/go-vela/server/queue"
 	"github.com/go-vela/server/router"
 	"github.com/go-vela/server/router/middleware"
+	"github.com/go-vela/server/tracing"
 )
 
 //nolint:funlen,gocyclo // ignore function length and cyclomatic complexity
@@ -78,7 +79,21 @@ func server(c *cli.Context) error {
 		return err
 	}
 
-	database, err := database.FromCLIContext(c)
+	tc, err := tracing.FromCLIContext(c)
+	if err != nil {
+		return err
+	}
+
+	if tc.EnableTracing {
+		defer func() {
+			err := tc.TracerProvider.Shutdown(context.Background())
+			if err != nil {
+				logrus.Errorf("unable to shutdown tracer provider: %v", err)
+			}
+		}()
+	}
+
+	database, err := database.FromCLIContext(c, tc)
 	if err != nil {
 		return err
 	}
@@ -189,6 +204,8 @@ func server(c *cli.Context) error {
 		middleware.DefaultRepoEventsMask(c.Int64("default-repo-events-mask")),
 		middleware.DefaultRepoApproveBuild(c.String("default-repo-approve-build")),
 		middleware.ScheduleFrequency(c.Duration("schedule-minimum-frequency")),
+		middleware.TracingClient(tc),
+		middleware.TracingInstrumentation(tc),
 	)
 
 	addr, err := url.Parse(c.String("server-addr"))

database/context.go

@@ -7,6 +7,8 @@ import (
 
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli/v2"
+
+	"github.com/go-vela/server/tracing"
 )
 
 const key = "database"
@@ -38,7 +40,7 @@ func ToContext(c Setter, d Interface) {
 }
 
 // FromCLIContext creates and returns a database engine from the urfave/cli context.
-func FromCLIContext(c *cli.Context) (Interface, error) {
+func FromCLIContext(c *cli.Context, tc *tracing.Client) (Interface, error) {
 	logrus.Debug("creating database engine from CLI configuration")
 
 	return New(
@@ -54,5 +56,6 @@ func FromCLIContext(c *cli.Context) (Interface, error) {
 		WithLogSlowThreshold(c.Duration("database.log.slow_threshold")),
 		WithLogShowSQL(c.Bool("database.log.show_sql")),
 		WithSkipCreation(c.Bool("database.skip_creation")),
+		WithTracing(tc),
 	)
 }

database/context_test.go

@@ -10,6 +10,8 @@ import (
 
 	"github.com/gin-gonic/gin"
 	"github.com/urfave/cli/v2"
+
+	"github.com/go-vela/server/tracing"
 )
 
 func TestDatabase_FromContext(t *testing.T) {
@@ -132,7 +134,7 @@ func TestDatabase_FromCLIContext(t *testing.T) {
 	// run tests
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			_, err := FromCLIContext(test.context)
+			_, err := FromCLIContext(test.context, &tracing.Client{Config: tracing.Config{EnableTracing: false}})
 
 			if test.failure {
 				if err == nil {

database/database.go

@@ -8,6 +8,7 @@ import (
 	"time"
 
 	"github.com/sirupsen/logrus"
+	"github.com/uptrace/opentelemetry-go-extra/otelgorm"
 	"gorm.io/driver/postgres"
 	"gorm.io/driver/sqlite"
 	"gorm.io/gorm"
@@ -28,6 +29,7 @@ import (
 	"github.com/go-vela/server/database/step"
 	"github.com/go-vela/server/database/user"
 	"github.com/go-vela/server/database/worker"
+	"github.com/go-vela/server/tracing"
 	"github.com/go-vela/types/constants"
 )
 
@@ -70,6 +72,8 @@ type (
 		ctx context.Context
 		// sirupsen/logrus logger used in database functions
 		logger *logrus.Entry
+		// configurations related to telemetry/tracing
+		tracing *tracing.Client
 
 		settings.SettingsInterface
 		build.BuildInterface
@@ -105,7 +109,7 @@ func New(opts ...EngineOpt) (Interface, error) {
 	e.client = new(gorm.DB)
 	e.config = new(config)
 	e.logger = new(logrus.Entry)
-	e.ctx = context.TODO()
+	e.ctx = context.Background()
 
 	// apply all provided configuration options
 	for _, opt := range opts {
@@ -191,6 +195,19 @@ func New(opts ...EngineOpt) (Interface, error) {
 		return nil, err
 	}
 
+	// initialize otel tracing if enabled
+	if e.tracing.EnableTracing {
+		otelPlugin := otelgorm.NewPlugin(
+			otelgorm.WithTracerProvider(e.tracing.TracerProvider),
+			otelgorm.WithoutQueryVariables(),
+		)
+
+		err := e.client.Use(otelPlugin)
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	// set the maximum amount of time a connection may be reused
 	db.SetConnMaxLifetime(e.config.ConnectionLife)
 	// set the maximum number of connections in the idle connection pool
@@ -230,5 +247,6 @@ func NewTest() (Interface, error) {
 		WithLogShowSQL(false),
 		WithLogSkipNotFound(true),
 		WithLogSlowThreshold(200*time.Millisecond),
+		WithTracing(&tracing.Client{Config: tracing.Config{EnableTracing: false}}),
 	)
 }

database/database_test.go

@@ -11,6 +11,8 @@ import (
 	"gorm.io/driver/postgres"
 	"gorm.io/driver/sqlite"
 	"gorm.io/gorm"
+
+	"github.com/go-vela/server/tracing"
 )
 
 func TestDatabase_New(t *testing.T) {
@@ -110,6 +112,7 @@ func TestDatabase_New(t *testing.T) {
 				WithLogSlowThreshold(test.config.LogSlowThreshold),
 				WithEncryptionKey(test.config.EncryptionKey),
 				WithSkipCreation(test.config.SkipCreation),
+				WithTracing(&tracing.Client{Config: tracing.Config{EnableTracing: false}}),
 			)
 
 			if test.failure {

database/integration_test.go

@@ -33,6 +33,7 @@ import (
 	"github.com/go-vela/server/database/testutils"
 	"github.com/go-vela/server/database/user"
 	"github.com/go-vela/server/database/worker"
+	"github.com/go-vela/server/tracing"
 	"github.com/go-vela/types/constants"
 	"github.com/go-vela/types/library"
 	"github.com/go-vela/types/raw"
@@ -114,6 +115,7 @@ func TestDatabase_Integration(t *testing.T) {
 				WithDriver(test.config.Driver),
 				WithEncryptionKey(test.config.EncryptionKey),
 				WithSkipCreation(test.config.SkipCreation),
+				WithTracing(&tracing.Client{Config: tracing.Config{EnableTracing: false}}),
 			)
 			if err != nil {
 				t.Errorf("unable to create new database engine for %s: %v", test.name, err)

database/opts.go

@@ -5,6 +5,8 @@ package database
 import (
 	"context"
 	"time"
+
+	"github.com/go-vela/server/tracing"
 )
 
 // EngineOpt represents a configuration option to initialize the database engine.
@@ -130,6 +132,15 @@ func WithSkipCreation(skipCreation bool) EngineOpt {
 	}
 }
 
+// WithTracing sets the shared tracing config in the database engine.
+func WithTracing(tracing *tracing.Client) EngineOpt {
+	return func(e *engine) error {
+		e.tracing = tracing
+
+		return nil
+	}
+}
+
 // WithContext sets the context in the database engine.
 func WithContext(ctx context.Context) EngineOpt {
 	return func(e *engine) error {

docker-compose.yml

@@ -44,6 +44,13 @@ services:
       VELA_ENABLE_SECURE_COOKIE: 'false'
       VELA_REPO_ALLOWLIST: '*'
       VELA_SCHEDULE_ALLOWLIST: '*'
+      VELA_OTEL_EXPORTER_OTLP_ENDPOINT: http://jaeger:4318
+      VELA_OTEL_TRACING_ENDPOINT: jaeger:4318
+      VELA_OTEL_TRACING_ENABLE: true
+      VELA_OTEL_TRACING_SERVICE_NAME: vela-server
+      VELA_OTEL_TRACING_RESOURCE_ATTRIBUTES: "process.runtime.name=go"
+      VELA_OTEL_TRACING_RESOURCE_ENV_ATTRIBUTES: "deployment.environment=CLOUD_ENVIRONMENT"
+      VELA_OTEL_TRACING_SAMPLER_RATELIMIT_PER_SECOND: 0.2
     env_file:
       - .env
     restart: always
@@ -152,18 +159,29 @@ services:
   #
   # https://www.vaultproject.io/
   vault:
-    image: hashicorp/vault:latest
-    container_name: vault
-    command: server -dev
+   image: hashicorp/vault:latest
+   container_name: vault
+   command: server -dev
+   networks:
+     - vela
+   environment:
+     VAULT_DEV_LISTEN_ADDRESS: 0.0.0.0:8200
+     VAULT_DEV_ROOT_TOKEN_ID: vela
+   ports:
+     - '8200:8200'
+   cap_add:
+     - IPC_LOCK
+
+  jaeger:
+    image: jaegertracing/all-in-one:latest
+    container_name: jaeger
     networks:
       - vela
     environment:
-      VAULT_DEV_LISTEN_ADDRESS: 0.0.0.0:8200
-      VAULT_DEV_ROOT_TOKEN_ID: vela
+      COLLECTOR_OTLP_ENABLED: true
     ports:
-      - '8200:8200'
-    cap_add:
-      - IPC_LOCK
+      - '16686:16686'
+      - '4318:4318'
 
 networks:
   vela:

go.mod

@@ -36,11 +36,21 @@ require (
 	github.com/redis/go-redis/v9 v9.6.0
 	github.com/sirupsen/logrus v1.9.3
 	github.com/spf13/afero v1.11.0
+	github.com/uptrace/opentelemetry-go-extra/otelgorm v0.3.1
 	github.com/urfave/cli/v2 v2.27.2
+	go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin v0.53.0
+	go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.42.0
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0
+	go.opentelemetry.io/otel v1.28.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.28.0
+	go.opentelemetry.io/otel/sdk v1.28.0
+	go.opentelemetry.io/otel/trace v1.28.0
 	go.starlark.net v0.0.0-20240314022150-ee8ed142361c
 	golang.org/x/crypto v0.25.0
 	golang.org/x/oauth2 v0.21.0
 	golang.org/x/sync v0.7.0
+	golang.org/x/time v0.5.0
 	gopkg.in/yaml.v3 v3.0.1
 	gorm.io/driver/postgres v1.5.9
 	gorm.io/driver/sqlite v1.5.6
@@ -55,28 +65,32 @@ require (
 	github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a // indirect
 	github.com/aymerick/douceur v0.2.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/bytedance/sonic v1.11.6 // indirect
+	github.com/bytedance/sonic v1.11.9 // indirect
 	github.com/bytedance/sonic/loader v0.1.1 // indirect
 	github.com/cenkalti/backoff/v3 v3.0.0 // indirect
-	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/cloudwego/base64x v0.1.4 // indirect
 	github.com/cloudwego/iasm v0.2.0 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
-	github.com/gabriel-vasile/mimetype v1.4.3 // indirect
+	github.com/felixge/httpsnoop v1.0.4 // indirect
+	github.com/gabriel-vasile/mimetype v1.4.4 // indirect
 	github.com/gin-contrib/sse v0.1.0 // indirect
 	github.com/go-jose/go-jose/v4 v4.0.1 // indirect
-	github.com/go-logr/logr v1.4.1 // indirect
+	github.com/go-logr/logr v1.4.2 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-playground/locales v0.14.1 // indirect
 	github.com/go-playground/universal-translator v0.18.1 // indirect
-	github.com/go-playground/validator/v10 v10.20.0 // indirect
+	github.com/go-playground/validator/v10 v10.22.0 // indirect
 	github.com/goccy/go-json v0.10.3 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/gomodule/redigo v2.0.0+incompatible // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/gorilla/css v1.0.1 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.21.0 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-rootcerts v1.0.2 // indirect
 	github.com/hashicorp/go-secure-stdlib/parseutil v0.1.6 // indirect
@@ -93,8 +107,7 @@ require (
 	github.com/jinzhu/now v1.1.5 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/klauspost/cpuid/v2 v2.2.7 // indirect
-	github.com/kr/text v0.2.0 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.8 // indirect
 	github.com/leodido/go-urn v1.4.0 // indirect
 	github.com/lestrrat-go/blackmagic v1.0.2 // indirect
 	github.com/lestrrat-go/httpcc v1.0.1 // indirect
@@ -121,14 +134,19 @@ require (
 	github.com/spf13/cast v1.3.1 // indirect
 	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
 	github.com/ugorji/go/codec v1.2.12 // indirect
+	github.com/uptrace/opentelemetry-go-extra/otelsql v0.3.1 // indirect
 	github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 // indirect
 	github.com/yuin/gopher-lua v1.1.1 // indirect
+	go.opentelemetry.io/otel/metric v1.28.0 // indirect
+	go.opentelemetry.io/proto/otlp v1.3.1 // indirect
 	golang.org/x/arch v0.8.0 // indirect
-	golang.org/x/net v0.26.0 // indirect
+	golang.org/x/net v0.27.0 // indirect
 	golang.org/x/sys v0.22.0 // indirect
 	golang.org/x/text v0.16.0 // indirect
-	golang.org/x/time v0.5.0 // indirect
-	google.golang.org/protobuf v1.34.1 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20240730163845-b1a4ccb954bf // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20240730163845-b1a4ccb954bf // indirect
+	google.golang.org/grpc v1.65.0 // indirect
+	google.golang.org/protobuf v1.34.2 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	k8s.io/klog/v2 v2.120.1 // indirect
 	k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect