Enhance Traces with pod information (#4898)

* Enhance Tracing

Signed-off-by: Alan Protasio <approtas@amazon.com>

* changelog

Signed-off-by: Alan Protasio <approtas@amazon.com>

* lint

Signed-off-by: Alan Protasio <approtas@amazon.com>

* go mod vendor

Signed-off-by: Alan Protasio <approtas@amazon.com>

* removing k8s detector + allowing extra detectors

Signed-off-by: Alan Protasio <approtas@amazon.com>

Signed-off-by: Alan Protasio <approtas@amazon.com>

Author: alanprot

CHANGELOG.md

@@ -48,6 +48,7 @@
 * [ENHANCEMENT] Ingester: Prevent ingesters to become unhealthy during wall replay. #4847
 * [ENHANCEMENT] Compactor: Introduced visit marker file for blocks so blocks are under compaction will not be picked up by another compactor. #4805
 * [ENHANCEMENT] Distributor: Add label name to labelValueTooLongError. #4855
+* [ENHANCEMENT] Enhance traces with hostname information. #4898
 * [FEATURE] Compactor: Added `-compactor.block-files-concurrency` allowing to configure number of go routines for download/upload block files during compaction. #4784
 * [FEATURE] Compactor: Added -compactor.blocks-fetch-concurrency` allowing to configure number of go routines for blocks during compaction. #4787
 * [FEATURE] Compactor: Added configurations for Azure MSI in blocks-storage, ruler-storage and alertmanager-storage. #4818

pkg/tracing/tracing.go

@@ -21,7 +21,7 @@ import (
 	"go.opentelemetry.io/otel/propagation"
 	"go.opentelemetry.io/otel/sdk/resource"
 	sdktrace "go.opentelemetry.io/otel/sdk/trace"
-	semconv "go.opentelemetry.io/otel/semconv/v1.10.0"
+	semconv "go.opentelemetry.io/otel/semconv/v1.12.0"
 
 	"github.com/cortexproject/cortex/pkg/tracing/migration"
 	"github.com/cortexproject/cortex/pkg/tracing/sampler"
@@ -40,11 +40,12 @@ type Config struct {
 }
 
 type Otel struct {
-	OltpEndpoint string           `yaml:"oltp_endpoint" json:"oltp_endpoint"`
-	ExporterType string           `yaml:"exporter_type" json:"exporter_type"`
-	SampleRatio  float64          `yaml:"sample_ratio" json:"sample_ratio"`
-	TLSEnabled   bool             `yaml:"tls_enabled"`
-	TLS          tls.ClientConfig `yaml:"tls"`
+	OltpEndpoint   string              `yaml:"oltp_endpoint" json:"oltp_endpoint"`
+	ExporterType   string              `yaml:"exporter_type" json:"exporter_type"`
+	SampleRatio    float64             `yaml:"sample_ratio" json:"sample_ratio"`
+	TLSEnabled     bool                `yaml:"tls_enabled"`
+	TLS            tls.ClientConfig    `yaml:"tls"`
+	ExtraDetectors []resource.Detector `yaml:"-"`
 }
 
 // RegisterFlags registers flag.
@@ -103,7 +104,13 @@ func SetupTracing(ctx context.Context, name string, c Config) (func(context.Cont
 			return nil, fmt.Errorf("creating OTLP trace exporter: %w", err)
 		}
 
-		tracerProvider := newTraceProvider(name, c, exporter)
+		r, err := newResource(ctx, name, c.Otel.ExtraDetectors)
+
+		if err != nil {
+			return nil, fmt.Errorf("creating tracing resource: %w", err)
+		}
+
+		tracerProvider := newTraceProvider(r, c, exporter)
 
 		bridge, wrappedProvider := migration.NewCortexBridgeTracerWrapper(tracerProvider.Tracer("github.com/cortexproject/cortex/cmd/cortex"))
 		bridge.SetTextMapPropagator(propagation.TraceContext{})
@@ -118,10 +125,10 @@ func SetupTracing(ctx context.Context, name string, c Config) (func(context.Cont
 	}, nil
 }
 
-func newTraceProvider(name string, c Config, exporter *otlptrace.Exporter) *sdktrace.TracerProvider {
+func newTraceProvider(r *resource.Resource, c Config, exporter *otlptrace.Exporter) *sdktrace.TracerProvider {
 	options := []sdktrace.TracerProviderOption{
 		sdktrace.WithBatcher(exporter),
-		sdktrace.WithResource(newResource(name)),
+		sdktrace.WithResource(r),
 	}
 
 	switch strings.ToLower(c.Otel.ExporterType) {
@@ -135,9 +142,15 @@ func newTraceProvider(name string, c Config, exporter *otlptrace.Exporter) *sdkt
 	return sdktrace.NewTracerProvider(options...)
 }
 
-func newResource(target string) *resource.Resource {
-	return resource.NewWithAttributes(
+func newResource(ctx context.Context, target string, detectors []resource.Detector) (*resource.Resource, error) {
+	r, err := resource.New(ctx, resource.WithHost(), resource.WithDetectors(detectors...))
+
+	if err != nil {
+		return nil, err
+	}
+
+	return resource.Merge(r, resource.NewWithAttributes(
 		semconv.SchemaURL,
 		semconv.ServiceNameKey.String(target),
-	)
+	))
 }