Merge branch 'main' into kubeletstats_fix_defaults

.chloggen/dd-conn-client-producer-stats.yaml

@@ -0,0 +1,31 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: bug_fix
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: datadogconnector
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: "Produce stats for non-root client and producer spans when `connector.datadogconnector.NativeIngest` and `compute_top_level_by_span_kind` are enabled"
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [34197]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext: You should have only run into this bug when ALL the conditions below are met
+  | 1. feature gate `connector.datadogconnector.NativeIngest` is enabled
+  | 2. config `compute_top_level_by_span_kind` is set to true
+  | 3. config `compute_stats_by_span_kind` is unset or set to false
+  | 4. you have child spans with client or producer span kind
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: []

.chloggen/dd-conn-measured.yaml

@@ -0,0 +1,27 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: bug_fix
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: datadogconnector
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: "Respect `_dd.measured` when `connector.datadogconnector.NativeIngest` is enabled"
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [34197]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext: "Spans with attribute `_dd.measured` set to 1 will always get Datadog APM stats"
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: []

cmd/otelcontribcol/go.mod

@@ -329,7 +329,7 @@ require (
 	github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.56.0-rc.6 // indirect
 	github.com/DataDog/datadog-agent/pkg/status/health v0.56.0-rc.6 // indirect
 	github.com/DataDog/datadog-agent/pkg/telemetry v0.56.0-rc.6 // indirect
-	github.com/DataDog/datadog-agent/pkg/trace v0.57.0-devel.0.20240716154421-697b001e760f // indirect
+	github.com/DataDog/datadog-agent/pkg/trace v0.57.0-devel.0.20240722160158-ad956a31a730 // indirect
 	github.com/DataDog/datadog-agent/pkg/util/backoff v0.56.0-rc.6 // indirect
 	github.com/DataDog/datadog-agent/pkg/util/cgroups v0.56.0-rc.6 // indirect
 	github.com/DataDog/datadog-agent/pkg/util/executable v0.56.0-rc.6 // indirect

connector/datadogconnector/connector_native_test.go

@@ -5,20 +5,24 @@ package datadogconnector
 
 import (
 	"context"
+	"sort"
 	"testing"
 	"time"
 
 	pb "github.com/DataDog/datadog-agent/pkg/proto/pbgo/trace"
+	"github.com/google/go-cmp/cmp"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"go.opentelemetry.io/collector/component"
 	"go.opentelemetry.io/collector/component/componenttest"
 	"go.opentelemetry.io/collector/connector/connectortest"
 	"go.opentelemetry.io/collector/consumer/consumertest"
 	"go.opentelemetry.io/collector/featuregate"
+	"go.opentelemetry.io/collector/pdata/ptrace"
 	semconv "go.opentelemetry.io/collector/semconv/v1.5.0"
 	"go.uber.org/zap"
 	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/testing/protocmp"
 )
 
 var _ component.Component = (*traceToMetricConnectorNative)(nil) // testing that the connectorImp properly implements the type Component interface
@@ -63,6 +67,12 @@ func TestTraceToTraceConnectorNative(t *testing.T) {
 }
 
 func creteConnectorNative(t *testing.T) (*traceToMetricConnectorNative, *consumertest.MetricsSink) {
+	cfg := NewFactory().CreateDefaultConfig().(*Config)
+	cfg.Traces.ResourceAttributesAsContainerTags = []string{semconv.AttributeCloudAvailabilityZone, semconv.AttributeCloudRegion, "az"}
+	return creteConnectorNativeWithCfg(t, cfg)
+}
+
+func creteConnectorNativeWithCfg(t *testing.T, cfg *Config) (*traceToMetricConnectorNative, *consumertest.MetricsSink) {
 	err := featuregate.GlobalRegistry().Set(NativeIngestFeatureGate.ID(), true)
 	assert.NoError(t, err)
 	defer func() {
@@ -72,9 +82,6 @@ func creteConnectorNative(t *testing.T) (*traceToMetricConnectorNative, *consume
 	factory := NewFactory()
 
 	creationParams := connectortest.NewNopSettings()
-	cfg := factory.CreateDefaultConfig().(*Config)
-	cfg.Traces.ResourceAttributesAsContainerTags = []string{semconv.AttributeCloudAvailabilityZone, semconv.AttributeCloudRegion, "az"}
-
 	metricsSink := &consumertest.MetricsSink{}
 
 	tconn, err := factory.CreateTracesToMetrics(context.Background(), creationParams, cfg, metricsSink)
@@ -131,3 +138,128 @@ func TestContainerTagsNative(t *testing.T) {
 	assert.Equal(t, 3, len(tags))
 	assert.ElementsMatch(t, []string{"region:my-region", "zone:my-zone", "az:my-az"}, tags)
 }
+
+var (
+	testTraceID = [16]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}
+	testSpanID1 = [8]byte{1, 2, 3, 4, 5, 6, 7, 8}
+	testSpanID2 = [8]byte{2, 2, 3, 4, 5, 6, 7, 8}
+	testSpanID3 = [8]byte{3, 2, 3, 4, 5, 6, 7, 8}
+	testSpanID4 = [8]byte{4, 2, 3, 4, 5, 6, 7, 8}
+)
+
+func TestMeasuredAndClientKindNative(t *testing.T) {
+	cfg := NewFactory().CreateDefaultConfig().(*Config)
+	cfg.Traces.ComputeTopLevelBySpanKind = true
+	connector, metricsSink := creteConnectorNativeWithCfg(t, cfg)
+	err := connector.Start(context.Background(), componenttest.NewNopHost())
+	if err != nil {
+		t.Errorf("Error starting connector: %v", err)
+		return
+	}
+	defer func() {
+		require.NoError(t, connector.Shutdown(context.Background()))
+	}()
+
+	td := ptrace.NewTraces()
+	res := td.ResourceSpans().AppendEmpty().Resource()
+	res.Attributes().PutStr("service.name", "svc")
+	ss := td.ResourceSpans().At(0).ScopeSpans().AppendEmpty().Spans()
+	// Root span
+	s1 := ss.AppendEmpty()
+	s1.SetName("parent")
+	s1.SetKind(ptrace.SpanKindServer)
+	s1.SetTraceID(testTraceID)
+	s1.SetSpanID(testSpanID1)
+	// Child span with internal kind does not get stats
+	s2 := ss.AppendEmpty()
+	s2.SetName("child1")
+	s2.SetKind(ptrace.SpanKindInternal)
+	s2.SetTraceID(testTraceID)
+	s2.SetSpanID(testSpanID2)
+	s2.SetParentSpanID(testSpanID1)
+	// Child span with internal kind and the _dd.measured key gets stats
+	s3 := ss.AppendEmpty()
+	s3.SetName("child2")
+	s3.SetKind(ptrace.SpanKindInternal)
+	s3.SetTraceID(testTraceID)
+	s3.SetSpanID(testSpanID3)
+	s3.SetParentSpanID(testSpanID1)
+	s3.Attributes().PutInt("_dd.measured", 1)
+	// Child span with client kind gets stats
+	s4 := ss.AppendEmpty()
+	s4.SetName("child3")
+	s4.SetKind(ptrace.SpanKindClient)
+	s4.SetTraceID(testTraceID)
+	s4.SetSpanID(testSpanID4)
+	s4.SetParentSpanID(testSpanID1)
+
+	err = connector.ConsumeTraces(context.Background(), td)
+	assert.NoError(t, err)
+
+	timeout := time.Now().Add(1 * time.Minute)
+	for time.Now().Before(timeout) {
+		if len(metricsSink.AllMetrics()) > 0 {
+			break
+		}
+		time.Sleep(100 * time.Millisecond)
+	}
+
+	metrics := metricsSink.AllMetrics()
+	require.Equal(t, 1, len(metrics))
+
+	ch := make(chan []byte, 100)
+	tr := newTranslatorWithStatsChannel(t, zap.NewNop(), ch)
+	_, err = tr.MapMetrics(context.Background(), metrics[0], nil)
+	require.NoError(t, err)
+	msg := <-ch
+	sp := &pb.StatsPayload{}
+
+	err = proto.Unmarshal(msg, sp)
+	require.NoError(t, err)
+	assert.Len(t, sp.Stats, 1)
+	assert.Len(t, sp.Stats[0].Stats, 1)
+	assert.Len(t, sp.Stats[0].Stats[0].Stats, 3)
+	cgss := sp.Stats[0].Stats[0].Stats
+	sort.Slice(cgss, func(i, j int) bool {
+		return cgss[i].Resource < cgss[j].Resource
+	})
+	expected := []*pb.ClientGroupedStats{
+		{
+			Service:      "svc",
+			Name:         "opentelemetry.internal",
+			Resource:     "child2",
+			Type:         "custom",
+			Hits:         1,
+			TopLevelHits: 0,
+			SpanKind:     "internal",
+			IsTraceRoot:  pb.Trilean_FALSE,
+		},
+		{
+			Service:      "svc",
+			Name:         "opentelemetry.client",
+			Resource:     "child3",
+			Type:         "http",
+			Hits:         1,
+			TopLevelHits: 0,
+			SpanKind:     "client",
+			IsTraceRoot:  pb.Trilean_FALSE,
+		},
+		{
+			Service:      "svc",
+			Name:         "opentelemetry.server",
+			Resource:     "parent",
+			Type:         "web",
+			Hits:         1,
+			TopLevelHits: 1,
+			SpanKind:     "server",
+			IsTraceRoot:  pb.Trilean_TRUE,
+		},
+	}
+	if diff := cmp.Diff(
+		cgss,
+		expected,
+		protocmp.Transform(),
+		protocmp.IgnoreFields(&pb.ClientGroupedStats{}, "duration", "okSummary", "errorSummary")); diff != "" {
+		t.Errorf("Diff between APM stats -want +got:\n%v", diff)
+	}
+}

connector/datadogconnector/go.mod

@@ -6,10 +6,11 @@ require (
 	github.com/DataDog/datadog-agent/comp/otelcol/otlp/components/metricsclient v0.56.0-rc.6
 	github.com/DataDog/datadog-agent/comp/otelcol/otlp/components/statsprocessor v0.56.0-rc.6
 	github.com/DataDog/datadog-agent/pkg/proto v0.56.0-rc.6
-	github.com/DataDog/datadog-agent/pkg/trace v0.57.0-devel.0.20240716154421-697b001e760f
+	github.com/DataDog/datadog-agent/pkg/trace v0.57.0-devel.0.20240722160158-ad956a31a730
 	github.com/DataDog/datadog-go/v5 v5.5.0
 	github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.17.0
 	github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/metrics v0.17.0
+	github.com/google/go-cmp v0.6.0
 	github.com/open-telemetry/opentelemetry-collector-contrib/exporter/datadogexporter v0.105.0
 	github.com/open-telemetry/opentelemetry-collector-contrib/processor/tailsamplingprocessor v0.105.0
 	github.com/patrickmn/go-cache v2.1.0+incompatible
@@ -138,7 +139,6 @@ require (
 	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/gnostic-models v0.6.8 // indirect
-	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7 // indirect
 	github.com/google/uuid v1.6.0 // indirect

exporter/datadogexporter/go.mod

@@ -25,7 +25,7 @@ require (
 	github.com/DataDog/datadog-agent/pkg/logs/status/statusinterface v0.56.0-rc.6 // indirect
 	github.com/DataDog/datadog-agent/pkg/proto v0.56.0-rc.6
 	github.com/DataDog/datadog-agent/pkg/status/health v0.56.0-rc.6 // indirect
-	github.com/DataDog/datadog-agent/pkg/trace v0.57.0-devel.0.20240716154421-697b001e760f
+	github.com/DataDog/datadog-agent/pkg/trace v0.57.0-devel.0.20240722160158-ad956a31a730
 	github.com/DataDog/datadog-agent/pkg/util/hostname/validate v0.56.0-rc.6
 	github.com/DataDog/datadog-agent/pkg/util/startstop v0.56.0-rc.6 // indirect
 	github.com/DataDog/datadog-api-client-go/v2 v2.28.0

exporter/datadogexporter/integrationtest/go.mod

@@ -83,7 +83,7 @@ require (
 	github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.56.0-rc.6 // indirect
 	github.com/DataDog/datadog-agent/pkg/status/health v0.56.0-rc.6 // indirect
 	github.com/DataDog/datadog-agent/pkg/telemetry v0.56.0-rc.6 // indirect
-	github.com/DataDog/datadog-agent/pkg/trace v0.57.0-devel.0.20240716154421-697b001e760f // indirect
+	github.com/DataDog/datadog-agent/pkg/trace v0.57.0-devel.0.20240722160158-ad956a31a730 // indirect
 	github.com/DataDog/datadog-agent/pkg/util/backoff v0.56.0-rc.6 // indirect
 	github.com/DataDog/datadog-agent/pkg/util/cgroups v0.56.0-rc.6 // indirect
 	github.com/DataDog/datadog-agent/pkg/util/executable v0.56.0-rc.6 // indirect