OpenTelemetry with logger exporter and Prometheus

cmd/pbapi/main.go

@@ -12,11 +12,14 @@ import (
 	_ "github.com/RHEnVision/provisioning-backend/internal/clients/http/azure"
 	_ "github.com/RHEnVision/provisioning-backend/internal/clients/http/ec2"
 	_ "github.com/RHEnVision/provisioning-backend/internal/clients/http/gcp"
+	"github.com/RHEnVision/provisioning-backend/internal/random"
 
 	// HTTP client implementations
 	_ "github.com/RHEnVision/provisioning-backend/internal/clients/http/image_builder"
 	_ "github.com/RHEnVision/provisioning-backend/internal/clients/http/sources"
 	"github.com/RHEnVision/provisioning-backend/internal/config/parser"
+	"github.com/RHEnVision/provisioning-backend/internal/telemetry"
+	"github.com/RHEnVision/provisioning-backend/internal/version"
 
 	// Job queue implementation
 	"github.com/RHEnVision/provisioning-backend/internal/jobs/queue/dejq"
@@ -37,12 +40,17 @@ import (
 	"github.com/rs/zerolog/log"
 )
 
+func init() {
+	random.SeedGlobal()
+}
+
 func statusOk(w http.ResponseWriter, _ *http.Request) {
 	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
 	w.WriteHeader(http.StatusOK)
 }
 
 func main() {
+	ctx := context.Background()
 	config.Initialize()
 
 	// initialize stdout logging and AWS clients first
@@ -64,14 +72,16 @@ func main() {
 		logger.Warn().Msgf("Unknown ENV variables, add them in the codebase: %+v", unknown)
 	}
 
+	tel := telemetry.Initialize(&log.Logger)
+	defer tel.Close(ctx)
+
 	// initialize the rest
 	err = db.Initialize("public")
 	if err != nil {
 		log.Fatal().Err(err).Msg("Error initializing database")
 	}
 
 	// initialize the job queue
-	ctx := context.Background()
 	err = dejq.Initialize(ctx, &logger)
 	if err != nil {
 		log.Fatal().Err(err).Msg("Error initializing dejq queue")
@@ -83,10 +93,10 @@ func main() {
 
 	// Routes for the main service
 	r := chi.NewRouter()
+	r.Use(m.NewPatternMiddleware(version.PrometheusLabelName))
+	r.Use(telemetry.Middleware(r))
 	r.Use(m.VersionMiddleware)
-	r.Use(m.RequestID)
-	r.Use(m.RequestNum)
-	r.Use(m.MetricsMiddleware)
+	r.Use(m.TraceID)
 	r.Use(m.LoggerMiddleware(&log.Logger))
 
 	// Set Content-Type to JSON for chi renderer. Warning: Non-chi routes

cmd/pbmigrate/main.go

@@ -7,9 +7,14 @@ import (
 	"github.com/RHEnVision/provisioning-backend/internal/config"
 	"github.com/RHEnVision/provisioning-backend/internal/db"
 	"github.com/RHEnVision/provisioning-backend/internal/logging"
+	"github.com/RHEnVision/provisioning-backend/internal/random"
 	"github.com/rs/zerolog/log"
 )
 
+func init() {
+	random.SeedGlobal()
+}
+
 func main() {
 	config.Initialize()
 

cmd/pbworker/main.go

@@ -8,6 +8,7 @@ import (
 
 	// Job queue implementation
 	"github.com/RHEnVision/provisioning-backend/internal/jobs/queue/dejq"
+	"github.com/RHEnVision/provisioning-backend/internal/random"
 
 	"github.com/RHEnVision/provisioning-backend/internal/clients/http/cloudwatchlogs"
 	"github.com/RHEnVision/provisioning-backend/internal/config"
@@ -24,6 +25,10 @@ import (
 	"github.com/rs/zerolog/log"
 )
 
+func init() {
+	random.SeedGlobal()
+}
+
 func main() {
 	config.Initialize()
 

configs/local_example.yaml

@@ -11,8 +11,9 @@ database:
   maxIdleConn: 20
   maxLifetime: 5m
   maxIdletime: 10m
-  # 1-none, 2-error, 3-warn, 4-info, 5-debug, 6-trace
-  loglevel: 1
+  # Database tracing level, not effective if telemetry is enabled. Possible values:
+  # none, error, warn, info, debug, trace
+  #loglevel: trace
 
 logging:
   # Overall log level (-1 trace, 0 debug, 1 info ... 5 panic)
@@ -24,6 +25,18 @@ logging:
   # to disable.
   maxField: 0
 
+telemetry:
+  # OpenTelemetry tracing support
+  #enabled: true
+  # Jaeger Tracing exporter
+  jaeger:
+    #enabled: true
+    #endpoint: http://localhost:14268/api/traces
+  # Send OpenTelemetry output to logger as trace log level. No other exporting
+  # is currently supported, Jaeger exporting is planned for the platform.
+  logger:
+    #enabled: true
+
 app:
   # HTTP transparent gzip compression
   compression: false

docs/opentelemetry.md

@@ -0,0 +1,54 @@
+# OpenTelemetry
+
+While logging is done through Zerolog with ability to export to AWS CloudWatch, the project supports tracing via OpenTelemetry and handover of W3C Trace Context.
+
+# Configuration
+
+To enable OpenTelemetry, use the following configuration:
+
+```yaml
+telemetry:
+  enabled: true
+```
+
+By default, OpenTelemetry data is not exported because full OpenTelemetry Jaeger tracing support is planned for the platform. As a temporary solution, tho, it is possible to export tracing data into the application logger (Zerolog/CloudWatch):
+
+```yaml
+telemetry:
+    enabled: true
+    loggerExporter: true
+
+logging:
+    level: -1
+```
+
+Keep in mind that logging level must be set to "trace" (-1) as all the tracing data is reported at this level. All the OpenTelemetry log messages start with `otel:` and also have "otel" boolean flag set to true. Span attributes are exported as regular Zerolog fields. Example output:
+
+```
+11:19AM TRC otel: /api/provisioning/v1/ready/{SRV} duration=6.689334 hostname=mone.home.lan http.flavor=1.1 http.host=localhost:8000 http.method=GET http.route=/api/provisioning/v1/ready/{SRV} http.scheme=http http.server_name=provisioning-backend http.target=/api/provisioning/v1/ready/sources http.user_agent="Apache-HttpClient/4.5.13 (Java/17.0.3)" net.host.name=localhost net.peer.ip=127.0.0.1 net.transport=ip_tcp otel=true span_id=0a557df38618c977 trace_id=6c87886d623eb0a9c5e3ec03bb64edfb
+11:19AM TRC otel: HTTP GET duration=2418.531833 hostname=mone.home.lan http.flavor=1.1 http.host=console.stage.redhat.com http.method=GET http.scheme=http http.url=https://console.stage.redhat.com/api/image-builder... otel=true span_id=8fb56982443c1ae0 span_id_parent=a9c1040b9686ba5d trace_id=a58e792bb7754c7132e9812374cf0243
+11:19AM TRC otel: Ready duration=2419.063708 hostname=mone.home.lan otel=true span_id=72638bd6a72a23ed span_id_parent=0b9434ba2311ac5f trace_id=a58e792bb7754c7132e9812374cf0243
+11:19AM INF Completed GET request /api/provisioning/v1/ready/ib in 2.491s ms with 200 bytes_in=0 bytes_out=0 hostname=mone.home.lan latency_ms=2490.723792 method=GET remote_ip=127.0.0.1:50180 status=200 trace_id=a58e792bb7754c7132e9812374cf0243 url=/api/provisioning/v1/ready/ib
+11:19AM TRC otel: /api/provisioning/v1/ready/{SRV} duration=2490.834875 hostname=mone.home.lan http.flavor=1.1 http.host=localhost:8000 http.method=GET http.route=/api/provisioning/v1/ready/{SRV} http.scheme=http http.server_name=provisioning-backend http.target=/api/provisioning/v1/ready/ib http.user_agent="Apache-HttpClient/4.5.13 (Java/17.0.3)" net.host.name=localhost net.peer.ip=127.0.0.1 net.transport=ip_tcp otel=true span_id=0b9434ba2311ac5f trace_id=a58e792bb7754c7132e9812374cf0243
+```
+
+## Features
+
+Tracing ID is parsed from the W3C Trace Context header or generated when missing for each incoming request. The Trace ID is generated even if tracing feature is turned off because this field is used for correlation of log messages for each request on the application level.
+
+Spans are created for each Chi route with the route being the name of the span (e.g. `/api/provisioning/v1/ready/{SRV}`).
+
+Spans are created for each HTTP client call being made via `telemetry.HTTPClient`. Name is set to "HTTP" and followed by HTTP method.
+
+Spans are created for all SQL operations made through the `pgx` SQL driver.
+
+Spans are created for custom instrumentation points. An example:
+
+```go
+func Function() {
+    ctx, span := otel.Tracer(TraceName).Start(ctx, "Function")
+    defer span.End()
+    // ...
+}
+```
+

go.mod

@@ -35,10 +35,16 @@ require (
 	github.com/prometheus/client_golang v1.13.0
 	github.com/redhatinsights/app-common-go v1.6.3
 	github.com/redhatinsights/platform-go-middlewares v0.20.0
+	github.com/riandyrn/otelchi v0.4.0
 	github.com/rs/xid v1.4.0
 	github.com/rs/zerolog v1.28.0
 	github.com/spf13/viper v1.12.0
 	github.com/stretchr/testify v1.8.0
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.36.0
+	go.opentelemetry.io/otel v1.10.0
+	go.opentelemetry.io/otel/exporters/jaeger v1.10.0
+	go.opentelemetry.io/otel/sdk v1.10.0
+	go.opentelemetry.io/otel/trace v1.10.0
 	golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90
 	google.golang.org/api v0.81.0
 	google.golang.org/genproto v0.0.0-20220519153652-3a47de7e79bd
@@ -66,8 +72,10 @@ require (
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/felixge/httpsnoop v1.0.3 // indirect
 	github.com/fsnotify/fsnotify v1.5.4 // indirect
 	github.com/go-logr/logr v1.2.3 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-openapi/analysis v0.21.4 // indirect
 	github.com/go-openapi/errors v0.20.3 // indirect
 	github.com/go-openapi/jsonpointer v0.19.5 // indirect
@@ -122,6 +130,8 @@ require (
 	github.com/subosito/gotenv v1.4.1 // indirect
 	go.mongodb.org/mongo-driver v1.10.1 // indirect
 	go.opencensus.io v0.23.0 // indirect
+	go.opentelemetry.io/contrib v1.0.0 // indirect
+	go.opentelemetry.io/otel/metric v0.32.0 // indirect
 	golang.org/x/net v0.0.0-20220708220712-1185a9018129 // indirect
 	golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 // indirect
 	golang.org/x/sys v0.0.0-20220829200755-d48e67d00261 // indirect

go.sum

@@ -196,6 +196,9 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.m
 github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0=
 github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/felixge/httpsnoop v1.0.2/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
+github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk=
+github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI=
@@ -225,9 +228,13 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
 github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
 github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
+github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/logr v1.2.1/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0=
 github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/stdr v1.2.0/go.mod h1:YkVgnZu1ZjjL7xTxrfm/LLZBfkhTqSR1ydtm6jTKKwI=
+github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
 github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
 github.com/go-logr/zerologr v1.2.2 h1:nKJ1glUZQPURRpe20GaqCBgNyGYg9cylaerwrwKoogE=
 github.com/go-logr/zerologr v1.2.2/go.mod h1:eIsB+dwGuN3lAGytcpbXyBeiY8GKInIxy+Qwe+gI5lI=
@@ -677,6 +684,8 @@ github.com/redhatinsights/app-common-go v1.6.3 h1:HhjDKLBqQM5i8Ii58WLi5hG+lTNaKg
 github.com/redhatinsights/app-common-go v1.6.3/go.mod h1:6gzRyg8ZyejwMCksukeAhh2ZXOB3uHSmBsbP06fG2PQ=
 github.com/redhatinsights/platform-go-middlewares v0.20.0 h1:qwK9ArGYRlORsZ56PXXLJrGvzTsMe3bk2lR+WN5aIjM=
 github.com/redhatinsights/platform-go-middlewares v0.20.0/go.mod h1:i5gVDZJ/quCQhs5AW5CwkRPXlz1HfDBvyNtXHnlXZfM=
+github.com/riandyrn/otelchi v0.4.0 h1:btWguNWE2NluzNqRNaqAdJ809GKP1CjZEi8UP4AdYGw=
+github.com/riandyrn/otelchi v0.4.0/go.mod h1:gYlFYJYyLUmgxgJFKE/NFno7ljcu2lCd8+aOlD8oLTw=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
@@ -731,6 +740,7 @@ github.com/spf13/viper v1.12.0/go.mod h1:b6COn30jlNxbm/V2IqWiNWkJ+vZNiMNksliPCiu
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
+github.com/stretchr/objx v0.4.0 h1:M2gUjqZET1qApGOWNSnZ49BAIMX4F/1plDv3+l31EJ4=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
@@ -784,6 +794,24 @@ go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
 go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M=
 go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
+go.opentelemetry.io/contrib v1.0.0 h1:khwDCxdSspjOLmFnvMuSHd/5rPzbTx0+l6aURwtQdfE=
+go.opentelemetry.io/contrib v1.0.0/go.mod h1:EH4yDYeNoaTqn/8yCWQmfNB78VHfGX2Jt2bvnvzBlGM=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.36.0 h1:qZ3KzA4qPzLBDtQyPk4ydjlg8zvXbNysnFHaVMKJbVo=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.36.0/go.mod h1:14Oo79mRwusSI02L0EfG3Gp1uF3+1wSL+D4zDysxyqs=
+go.opentelemetry.io/otel v1.3.0/go.mod h1:PWIKzi6JCp7sM0k9yZ43VX+T345uNbAkDKwHVjb2PTs=
+go.opentelemetry.io/otel v1.10.0 h1:Y7DTJMR6zs1xkS/upamJYk0SxxN4C9AqRd77jmZnyY4=
+go.opentelemetry.io/otel v1.10.0/go.mod h1:NbvWjCthWHKBEUMpf0/v8ZRZlni86PpGFEMA9pnQSnQ=
+go.opentelemetry.io/otel/exporters/jaeger v1.10.0 h1:7W3aVVjEYayu/GOqOVF4mbTvnCuxF1wWu3eRxFGQXvw=
+go.opentelemetry.io/otel/exporters/jaeger v1.10.0/go.mod h1:n9IGyx0fgyXXZ/i0foLHNxtET9CzXHzZeKCucvRBFgA=
+go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.3.0/go.mod h1:PQLM+xJ3EMSZU9rMevmw+4nH1efyp23CW/nD9BlB3sg=
+go.opentelemetry.io/otel/metric v0.32.0 h1:lh5KMDB8xlMM4kwE38vlZJ3rZeiWrjw3As1vclfC01k=
+go.opentelemetry.io/otel/metric v0.32.0/go.mod h1:PVDNTt297p8ehm949jsIzd+Z2bIZJYQQG/uuHTeWFHY=
+go.opentelemetry.io/otel/sdk v1.3.0/go.mod h1:rIo4suHNhQwBIPg9axF8V9CA72Wz2mKF1teNrup8yzs=
+go.opentelemetry.io/otel/sdk v1.10.0 h1:jZ6K7sVn04kk/3DNUdJ4mqRlGDiXAVuIG+MMENpTNdY=
+go.opentelemetry.io/otel/sdk v1.10.0/go.mod h1:vO06iKzD5baltJz1zarxMCNHFpUlUiOy4s65ECtn6kE=
+go.opentelemetry.io/otel/trace v1.3.0/go.mod h1:c/VDhno8888bvQYmbYLqe41/Ldmr/KKunbvWM4/fEjk=
+go.opentelemetry.io/otel/trace v1.10.0 h1:npQMbR8o7mum8uF95yFbOEJffhs1sbCOfDh8zAJiH5E=
+go.opentelemetry.io/otel/trace v1.10.0/go.mod h1:Sij3YYczqAdz+EhmGhE6TpTxUO5/F/AzrK+kxfGqySM=
 go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
 go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=

internal/cache/account_cache.go

@@ -6,23 +6,27 @@ import (
 	"github.com/RHEnVision/provisioning-backend/internal/cache/memcache"
 	"github.com/RHEnVision/provisioning-backend/internal/config"
 	"github.com/RHEnVision/provisioning-backend/internal/models"
+	"github.com/RHEnVision/provisioning-backend/internal/version"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
 )
 
 var (
 	accountCache = memcache.New[AccountKey, *models.Account](config.Application.Cache.CleanupInterval)
 	metricHit    = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "app_cache_account_hits",
-		Help: "The total number of cache hits for account ID",
+		Name:        "app_cache_account_hits",
+		Help:        "The total number of cache hits for account ID",
+		ConstLabels: prometheus.Labels{"service": version.PrometheusLabelName},
 	})
 	metricMiss = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "app_cache_account_miss",
-		Help: "The total number of cache misses for account ID",
+		Name:        "app_cache_account_miss",
+		Help:        "The total number of cache misses for account ID",
+		ConstLabels: prometheus.Labels{"service": version.PrometheusLabelName},
 	})
 	_ = promauto.NewGaugeFunc(prometheus.GaugeOpts{
-		Name: "app_cache_account_items",
-		Help: "The total number of cache items for account ID",
+		Name:        "app_cache_account_items",
+		Help:        "The total number of cache items for account ID",
+		ConstLabels: prometheus.Labels{"service": version.PrometheusLabelName},
 	}, func() float64 {
 		return float64(accountCache.Count())
 	})

internal/clients/http/gcp/gcp_client.go

@@ -36,7 +36,7 @@ func newGCPClient(ctx context.Context, auth *clients.Authentication) (clients.GC
 	options := []option.ClientOption{
 		option.WithCredentialsJSON([]byte(config.GCP.JSON)),
 		option.WithQuotaProject(auth.Payload),
-		option.WithRequestReason(ctxval.RequestId(ctx)),
+		option.WithRequestReason(ctxval.TraceId(ctx)),
 	}
 	return &gcpClient{
 		auth:    auth,

internal/clients/http/image_builder/image_client.go

@@ -10,9 +10,13 @@ import (
 	"github.com/RHEnVision/provisioning-backend/internal/config"
 	"github.com/RHEnVision/provisioning-backend/internal/ctxval"
 	"github.com/RHEnVision/provisioning-backend/internal/headers"
+	"github.com/RHEnVision/provisioning-backend/internal/telemetry"
 	"github.com/rs/zerolog"
+	"go.opentelemetry.io/otel"
 )
 
+const TraceName = "github.com/EnVision/provisioning/internal/clients/http/image_builder"
+
 type ibClient struct {
 	client *ClientWithResponses
 }
@@ -27,20 +31,15 @@ func logger(ctx context.Context) zerolog.Logger {
 
 func newImageBuilderClient(ctx context.Context) (clients.ImageBuilder, error) {
 	c, err := NewClientWithResponses(config.ImageBuilder.URL, func(c *Client) error {
-		if config.ImageBuilder.Proxy.URL != "" {
-			var client HttpRequestDoer
-			if config.Features.Environment != "development" {
-				return http.ClientProxyProductionUseErr
-			}
-			client, err := http.NewProxyDoer(ctx, config.ImageBuilder.Proxy.URL)
-			if err != nil {
-				return fmt.Errorf("cannot create proxy doer: %w", err)
-			}
-			if config.RestEndpoints.TraceData {
-				client = http.NewLoggingDoer(ctx, client)
-			}
-			c.Client = client
+		var doer HttpRequestDoer
+		doer, err := telemetry.HTTPClient(ctx, config.StringToURL(ctx, config.ImageBuilder.Proxy.URL))
+		if err != nil {
+			return fmt.Errorf("cannot HTTP client: %w", err)
+		}
+		if config.RestEndpoints.TraceData {
+			doer = http.NewLoggingDoer(ctx, doer)
 		}
+		c.Client = doer
 		return nil
 	})
 	if err != nil {
@@ -50,6 +49,9 @@ func newImageBuilderClient(ctx context.Context) (clients.ImageBuilder, error) {
 }
 
 func (c *ibClient) Ready(ctx context.Context) error {
+	ctx, span := otel.Tracer(TraceName).Start(ctx, "Ready")
+	defer span.End()
+
 	logger := logger(ctx)
 	resp, err := c.client.GetReadiness(ctx, headers.AddImageBuilderIdentityHeader)
 	if err != nil {