Description
Version of dd-trace-go
gopkg.in/DataDog/dd-trace-go.v1 v1.52.0
Describe what happened:
Configuring the Datadog tracer provider as demonstrated here: https://github.com/DataDog/dd-trace-go/blob/main/ddtrace/opentelemetry/example_test.go results in poor application behavior. The default OTEL http tracing wrappers create new tracers constantly. This results in a huge amount of log spam in the service as the datadog provider constantly logs its configuration.
The log spam can be silenced with the tracer.WithLogStartup(false) option, but this still results in a huge number of datadog tracers being created which destroys application performance (my application which runs at like 30% CPU immediately starts failing health checks if I do this).
Describe what you expected:
I expected the Datadog tracer provider to wrap itself such that its not instantiated millions of times.
Steps to reproduce the issue:
Setup tracing like in this example: https://github.com/DataDog/dd-trace-go/blob/main/ddtrace/opentelemetry/example_test.go but then also wrap http clients and servers in the OTEL libraries and issues hundreds or thousands of requests and see all the datadog tracers being created.
I was able to work around this issue using the code below, but its a ton of code and should probably just be the "default" way the Datadog tracer provider works:
func StartDatadogTracer(
serviceName string,
rate float64,
) (func(), error) {
slog.Info("starting tracer")
rules := []tracer.SamplingRule{tracer.RateRule(0.1)}
provider := ddotel.NewTracerProvider(
tracer.WithSamplingRules(rules),
tracer.WithService(serviceName),
tracer.WithEnv(os.Getenv("ENVIRONMENT")),
// These are incredibly expensive to collect, we never want them by
// default.
tracer.WithDebugStack(false))
tracer := provider.Tracer("")
// Make sure OTel always gets the same trace provider regardless of how many times it
// calls GetTracer() otherwise it has tons of overhead and generates unbelievable amounts
// of log spam.
constantProvider := &constantProvider{}
constantTracer := newConstantTracer(tracer, constantProvider)
constantProvider.tracer = constantTracer
otel.SetTracerProvider(constantProvider)
tracer = otel.Tracer("")
if tracer != constantTracer {
panic("otel did not returned datadog tracer provider after setting it as provider")
}
_, span := tracer.Start(context.Background(), "test")
if span.TracerProvider() != constantProvider {
panic("test span tracer provider was not constant provider")
}
slog.Info("started tracer")
return func() {
provider.Shutdown()
}, nil
}
type constantProvider struct {
tracer trace.Tracer
}
func (cp *constantProvider) Tracer(name string, options ...trace.TracerOption) trace.Tracer {
return cp.tracer
}
// constantTracer wraps trace.Tracer such that the spans it generates will always return itself
// for the result of TracerProvider.
type constantTracer struct {
tracer trace.Tracer
provider *constantProvider
}
func newConstantTracer(tracer trace.Tracer, provider *constantProvider) trace.Tracer {
return &constantTracer{
tracer: tracer,
provider: provider,
}
}
func (c *constantTracer) Start(
ctx context.Context,
spanName string,
opts ...trace.SpanStartOption,
) (context.Context, trace.Span) {
ctx, span := c.tracer.Start(ctx, spanName, opts...)
span = newConstantSpan(span, c.provider)
return trace.ContextWithSpan(ctx, span), span
}
// constantSpan wraps a trace.Span such that we can override the TracerProvider()
// method to always return the same trace.TracerProvider.
type constantSpan struct {
span trace.Span
provider *constantProvider
}
func newConstantSpan(span trace.Span, provider *constantProvider) trace.Span {
return &constantSpan{
span: span,
provider: provider,
}
}
func (c *constantSpan) End(options ...trace.SpanEndOption) {
c.span.End(options...)
}
func (c *constantSpan) AddEvent(name string, options ...trace.EventOption) {
c.span.AddEvent(name, options...)
}
func (c *constantSpan) IsRecording() bool {
return c.span.IsRecording()
}
func (c *constantSpan) RecordError(err error, options ...trace.EventOption) {
c.span.RecordError(err, options...)
}
func (c *constantSpan) SpanContext() trace.SpanContext {
return c.span.SpanContext()
}
func (c *constantSpan) SetStatus(code codes.Code, description string) {
c.span.SetStatus(code, description)
}
func (c *constantSpan) SetName(name string) {
c.span.SetName(name)
}
func (c *constantSpan) SetAttributes(kv ...attribute.KeyValue) {
c.span.SetAttributes(kv...)
}
func (c *constantSpan) TracerProvider() trace.TracerProvider {
return c.provider
}