Initial attempt at OpenTelemetry support

BUILD.bazel

@@ -17,6 +17,7 @@ go_library(
         "//cache/disk:go_default_library",
         "//config:go_default_library",
         "//ldap:go_default_library",
+        "//otel:go_default_library",
         "//server:go_default_library",
         "//utils/flags:go_default_library",
         "//utils/idle:go_default_library",
@@ -28,6 +29,8 @@ go_library(
         "@com_github_slok_go_http_metrics//middleware:go_default_library",
         "@com_github_slok_go_http_metrics//middleware/std:go_default_library",
         "@com_github_urfave_cli_v2//:go_default_library",
+        "@io_opentelemetry_go_contrib_instrumentation_google_golang_org_grpc_otelgrpc//:go_default_library",
+        "@io_opentelemetry_go_contrib_instrumentation_net_http_otelhttp//:go_default_library",
         "@org_golang_google_grpc//:go_default_library",
         "@org_golang_google_grpc//credentials:go_default_library",
         "@org_golang_x_sync//errgroup:go_default_library",

MODULE.bazel

@@ -128,6 +128,11 @@ use_repo(
     "in_gopkg_mgo_v2",
     "in_gopkg_yaml_v3",
     "io_etcd_go_bbolt",
+    "io_opentelemetry_go_contrib_instrumentation_google_golang_org_grpc_otelgrpc",
+    "io_opentelemetry_go_contrib_instrumentation_net_http_otelhttp",
+    "io_opentelemetry_go_otel",
+    "io_opentelemetry_go_otel_exporters_otlp_otlptrace_otlptracegrpc",
+    "io_opentelemetry_go_otel_sdk",
     "org_golang_google_genproto_googleapis_api",
     "org_golang_google_genproto_googleapis_bytestream",
     "org_golang_google_genproto_googleapis_rpc",

cache/disk/disk_test.go

@@ -890,7 +890,7 @@ func TestHttpProxyBackend(t *testing.T) {
 	accessLogger := testutils.NewSilentLogger()
 	errorLogger := testutils.NewSilentLogger()
 
-	proxy, err := httpproxy.New(url, "zstd", &http.Client{}, accessLogger, errorLogger, 100, 1000000)
+	proxy, err := httpproxy.New(url, "zstd", &http.Client{}, accessLogger, errorLogger, 100, 1000000, false)
 	if err != nil {
 		t.Fatal(err)
 	}

cache/gcsproxy/BUILD.bazel

@@ -8,6 +8,7 @@ go_library(
     deps = [
         "//cache:go_default_library",
         "//cache/httpproxy:go_default_library",
+        "@io_opentelemetry_go_contrib_instrumentation_net_http_otelhttp//:go_default_library",
         "@org_golang_x_oauth2//:go_default_library",
         "@org_golang_x_oauth2//google:go_default_library",
     ],

cache/gcsproxy/gcsproxy.go

@@ -9,6 +9,8 @@ import (
 	"net/url"
 	"os"
 
+	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
+
 	"github.com/buchgr/bazel-remote/v2/cache"
 	"github.com/buchgr/bazel-remote/v2/cache/httpproxy"
 
@@ -18,7 +20,7 @@ import (
 
 // New creates a cache that proxies requests to Google Cloud Storage.
 func New(bucket string, useDefaultCredentials bool, jsonCredentialsFile string, storageMode string,
-	accessLogger cache.Logger, errorLogger cache.Logger, numUploaders, maxQueuedUploads int) (cache.Proxy, error) {
+	accessLogger cache.Logger, errorLogger cache.Logger, numUploaders, maxQueuedUploads int, otelEnabled bool) (cache.Proxy, error) {
 	var remoteClient *http.Client
 	var err error
 
@@ -47,6 +49,14 @@ func New(bucket string, useDefaultCredentials bool, jsonCredentialsFile string,
 			"credentials or a json credentials file %v", useDefaultCredentials)
 	}
 
+	// Wrap OAuth2 client transport with OTEL instrumentation if enabled
+	if otelEnabled {
+		if remoteClient.Transport == nil {
+			remoteClient.Transport = http.DefaultTransport
+		}
+		remoteClient.Transport = otelhttp.NewTransport(remoteClient.Transport)
+	}
+
 	errorLogger.Printf("Proxying artifacts to GCS bucket '%s'.\n", bucket)
 
 	baseURL := url.URL{
@@ -55,5 +65,5 @@ func New(bucket string, useDefaultCredentials bool, jsonCredentialsFile string,
 		Path:   bucket,
 	}
 
-	return httpproxy.New(&baseURL, storageMode, remoteClient, accessLogger, errorLogger, numUploaders, maxQueuedUploads)
+	return httpproxy.New(&baseURL, storageMode, remoteClient, accessLogger, errorLogger, numUploaders, maxQueuedUploads, otelEnabled)
 }

cache/httpproxy/BUILD.bazel

@@ -11,6 +11,7 @@ go_library(
         "//utils/backendproxy:go_default_library",
         "@com_github_prometheus_client_golang//prometheus:go_default_library",
         "@com_github_prometheus_client_golang//prometheus/promauto:go_default_library",
+        "@io_opentelemetry_go_contrib_instrumentation_net_http_otelhttp//:go_default_library",
     ],
 )
 

cache/httpproxy/httpproxy.go

@@ -12,6 +12,8 @@ import (
 	"strconv"
 	"strings"
 
+	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
+
 	"github.com/buchgr/bazel-remote/v2/cache"
 	"github.com/buchgr/bazel-remote/v2/cache/disk/casblob"
 	"github.com/buchgr/bazel-remote/v2/utils/backendproxy"
@@ -98,7 +100,15 @@ func (r *remoteHTTPProxyCache) UploadFile(item backendproxy.UploadReq) {
 // CAS blobs) or "zstd" (which expects cas.v2 blobs).
 func New(baseURL *url.URL, storageMode string, remote *http.Client,
 	accessLogger cache.Logger, errorLogger cache.Logger,
-	numUploaders, maxQueuedUploads int) (cache.Proxy, error) {
+	numUploaders, maxQueuedUploads int, otelEnabled bool) (cache.Proxy, error) {
+
+	// Wrap HTTP client transport with OTEL instrumentation if enabled
+	if otelEnabled {
+		if remote.Transport == nil {
+			remote.Transport = http.DefaultTransport
+		}
+		remote.Transport = otelhttp.NewTransport(remote.Transport)
+	}
 
 	proxy := &remoteHTTPProxyCache{
 		remote:       remote,

cache/httpproxy/httpproxy_test.go

@@ -119,7 +119,7 @@ func TestEverything(t *testing.T) {
 	t.Log("cas HASH:", hash)
 	acData := []byte{1, 2, 3, 4}
 
-	proxyCache, err := New(url, "zstd", &http.Client{}, accessLogger, errorLogger, 100, 10000)
+	proxyCache, err := New(url, "zstd", &http.Client{}, accessLogger, errorLogger, 100, 10000, false)
 	if err != nil {
 		t.Fatal(err)
 	}

cache/s3proxy/BUILD.bazel

@@ -16,6 +16,7 @@ go_library(
         "@com_github_minio_minio_go_v7//pkg/credentials:go_default_library",
         "@com_github_prometheus_client_golang//prometheus:go_default_library",
         "@com_github_prometheus_client_golang//prometheus/promauto:go_default_library",
+        "@io_opentelemetry_go_contrib_instrumentation_net_http_otelhttp//:go_default_library",
     ],
 )
 

cache/s3proxy/s3proxy.go

@@ -6,8 +6,11 @@ import (
 	"fmt"
 	"io"
 	"log"
+	"net/http"
 	"path"
 
+	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
+
 	"github.com/buchgr/bazel-remote/v2/cache"
 	"github.com/buchgr/bazel-remote/v2/cache/disk/casblob"
 	"github.com/buchgr/bazel-remote/v2/utils/backendproxy"
@@ -58,7 +61,7 @@ func New(
 	MaxIdleConns int,
 
 	storageMode string, accessLogger cache.Logger,
-	errorLogger cache.Logger, numUploaders, maxQueuedUploads int) cache.Proxy {
+	errorLogger cache.Logger, numUploaders, maxQueuedUploads int, otelEnabled bool) cache.Proxy {
 
 	fmt.Println("Using S3 backend.")
 
@@ -78,14 +81,20 @@ func New(
 	tr.MaxIdleConns = MaxIdleConns
 	tr.MaxIdleConnsPerHost = MaxIdleConns
 
+	// Wrap transport with OTEL instrumentation if enabled
+	var transport http.RoundTripper = tr
+	if otelEnabled {
+		transport = otelhttp.NewTransport(tr)
+	}
+
 	// Initialize minio client with credentials
 	opts := &minio.Options{
 		Creds:        Credentials,
 		BucketLookup: BucketLookupType,
 
 		Region:    Region,
 		Secure:    secure,
-		Transport: tr,
+		Transport: transport,
 	}
 	minioCore, err = minio.NewCore(Endpoint, opts)
 	if err != nil {

config/BUILD.bazel

@@ -27,6 +27,7 @@ go_library(
         "@com_github_prometheus_client_golang//prometheus:go_default_library",
         "@com_github_urfave_cli_v2//:go_default_library",
         "@in_gopkg_yaml_v3//:go_default_library",
+        "@io_opentelemetry_go_contrib_instrumentation_google_golang_org_grpc_otelgrpc//:go_default_library",
         "@org_golang_google_grpc//:go_default_library",
         "@org_golang_google_grpc//credentials:go_default_library",
         "@org_golang_google_grpc//credentials/insecure:go_default_library",

config/config.go

@@ -23,6 +23,19 @@ import (
 	yaml "gopkg.in/yaml.v3"
 )
 
+// OtelTracingConfig stores the OpenTelemetry tracing configuration.
+type OtelTracingConfig struct {
+	Enabled          bool    `yaml:"enabled"`
+	ExporterEndpoint string  `yaml:"exporter_endpoint"`
+	ServiceName      string  `yaml:"service_name"`
+	SampleRate       float64 `yaml:"sample_rate"`
+}
+
+// OtelConfig stores the OpenTelemetry configuration.
+type OtelConfig struct {
+	Tracing *OtelTracingConfig `yaml:"tracing,omitempty"`
+}
+
 // GoogleCloudStorageConfig stores the configuration of a GCS proxy backend.
 type GoogleCloudStorageConfig struct {
 	Bucket                string `yaml:"bucket"`
@@ -127,6 +140,7 @@ type Config struct {
 	LogTimezone                 string                    `yaml:"log_timezone"`
 	MaxBlobSize                 int64                     `yaml:"max_blob_size"`
 	MaxProxyBlobSize            int64                     `yaml:"max_proxy_blob_size"`
+	Otel                        *OtelConfig               `yaml:"otel,omitempty"`
 
 	// Fields that are created by combinations of the flags above.
 	ProxyBackend cache.Proxy
@@ -173,6 +187,7 @@ func newFromArgs(dir string, maxSize int, storageMode string, zstdImplementation
 	ldap *LDAPConfig,
 	s3 *S3CloudStorageConfig,
 	azblob *AzBlobStorageConfig,
+	otel *OtelConfig,
 	disableHTTPACValidation bool,
 	disableGRPCACDepsCheck bool,
 	enableACKeyInstanceMangling bool,
@@ -210,6 +225,7 @@ func newFromArgs(dir string, maxSize int, storageMode string, zstdImplementation
 		HTTPBackend:                 hc,
 		GRPCBackend:                 grpcb,
 		LDAP:                        ldap,
+		Otel:                        otel,
 		IdleTimeout:                 idleTimeout,
 		DisableHTTPACValidation:     disableHTTPACValidation,
 		DisableGRPCACDepsCheck:      disableGRPCACDepsCheck,
@@ -500,6 +516,36 @@ func validateConfig(c *Config) error {
 		return errors.New("'log_timezone' must be set to either \"UTC\", \"local\" or \"none\"")
 	}
 
+	// Validate OpenTelemetry tracing configuration
+	if c.Otel != nil && c.Otel.Tracing != nil && c.Otel.Tracing.Enabled {
+		if c.Otel.Tracing.ExporterEndpoint == "" {
+			// Check standard OTEL env vars
+			if endpoint := os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT"); endpoint != "" {
+				c.Otel.Tracing.ExporterEndpoint = endpoint
+			} else if endpoint := os.Getenv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT"); endpoint != "" {
+				c.Otel.Tracing.ExporterEndpoint = endpoint
+			} else {
+				return errors.New("when 'otel.tracing.enabled' is true, either 'otel.tracing.exporter_endpoint' must be set or OTEL_EXPORTER_OTLP_ENDPOINT env var must be defined")
+			}
+		}
+
+		if c.Otel.Tracing.ServiceName == "" {
+			if name := os.Getenv("OTEL_SERVICE_NAME"); name != "" {
+				c.Otel.Tracing.ServiceName = name
+			} else {
+				c.Otel.Tracing.ServiceName = "bazel-remote"
+			}
+		}
+
+		if c.Otel.Tracing.SampleRate == 0 {
+			c.Otel.Tracing.SampleRate = 1.0 // Default to 100% sampling
+		}
+
+		if c.Otel.Tracing.SampleRate < 0.0 || c.Otel.Tracing.SampleRate > 1.0 {
+			return errors.New("'otel.tracing.sample_rate' must be between 0.0 and 1.0")
+		}
+	}
+
 	return nil
 }
 
@@ -643,6 +689,18 @@ func get(ctx *cli.Context) (*Config, error) {
 		}
 	}
 
+	var otel *OtelConfig
+	if ctx.Bool("otel.tracing.enabled") {
+		otel = &OtelConfig{
+			Tracing: &OtelTracingConfig{
+				Enabled:          ctx.Bool("otel.tracing.enabled"),
+				ExporterEndpoint: ctx.String("otel.tracing.exporter_endpoint"),
+				ServiceName:      ctx.String("otel.tracing.service_name"),
+				SampleRate:       ctx.Float64("otel.tracing.sample_rate"),
+			},
+		}
+	}
+
 	return newFromArgs(
 		ctx.String("dir"),
 		ctx.Int("max_size"),
@@ -666,6 +724,7 @@ func get(ctx *cli.Context) (*Config, error) {
 		ldap,
 		s3,
 		azblob,
+		otel,
 		ctx.Bool("disable_http_ac_validation"),
 		ctx.Bool("disable_grpc_ac_deps_check"),
 		ctx.Bool("enable_ac_key_instance_mangling"),