Proposed Internal Datadog Module Component

.github/CODEOWNERS

@@ -112,6 +112,7 @@ extension/storage/filestorage/                           @open-telemetry/collect
 
 internal/aws/                                            @open-telemetry/collector-contrib-approvers @Aneurysm9 @mxiamxia
 internal/coreinternal/                                   @open-telemetry/collector-contrib-approvers @open-telemetry/collector-approvers
+internal/datadog/                                        @open-telemetry/collector-contrib-approvers @mx-psi @gbbr @dineshg13
 internal/docker/                                         @open-telemetry/collector-contrib-approvers @rmfitzpatrick @jamesmoessis
 internal/filter/                                         @open-telemetry/collector-contrib-approvers @open-telemetry/collector-approvers
 internal/k8sconfig/                                      @open-telemetry/collector-contrib-approvers @dmitryax

.github/ISSUE_TEMPLATE/bug_report.yaml

@@ -103,6 +103,7 @@ body:
       - extension/storage/filestorage
       - internal/aws
       - internal/core
+      - internal/datadog
       - internal/docker
       - internal/filter
       - internal/k8sconfig

.github/ISSUE_TEMPLATE/feature_request.yaml

@@ -97,6 +97,7 @@ body:
       - extension/storage/filestorage
       - internal/aws
       - internal/core
+      - internal/datadog
       - internal/docker
       - internal/filter
       - internal/k8sconfig

.github/ISSUE_TEMPLATE/other.yaml

@@ -97,6 +97,7 @@ body:
       - extension/storage/filestorage
       - internal/aws
       - internal/core
+      - internal/datadog
       - internal/docker
       - internal/filter
       - internal/k8sconfig

.github/dependabot.yml

@@ -472,6 +472,11 @@ updates:
     schedule:
       interval: "weekly"
       day: "wednesday"
+  - package-ecosystem: "gomod"
+    directory: "/internal/datadog"
+    schedule:
+      interval: "weekly"
+      day: "wednesday"
   - package-ecosystem: "gomod"
     directory: "/internal/docker"
     schedule:
@@ -1097,8 +1102,3 @@ updates:
     schedule:
       interval: "weekly"
       day: "wednesday"
-  - package-ecosystem: "gomod"
-    directory: "/receiver/sqlqueryreceiver"
-    schedule:
-      interval: "weekly"
-      day: "wednesday"

cmd/configschema/go.mod

@@ -509,6 +509,7 @@ require (
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/proxy v0.82.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/xray v0.82.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/common v0.82.0 // indirect
+	github.com/open-telemetry/opentelemetry-collector-contrib/internal/datadog v0.82.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/sharedcomponent v0.82.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/experimentalmetricmetadata v0.82.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/winperfcounters v0.82.0 // indirect
@@ -1119,4 +1120,6 @@ replace github.com/open-telemetry/opentelemetry-collector-contrib/extension/obse
 
 replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/k8stest => ../../internal/k8stest
 
+replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/datadog => ../../internal/datadog
+
 replace github.com/open-telemetry/opentelemetry-collector-contrib/extension/jaegerremotesampling => ../../extension/jaegerremotesampling

cmd/otelcontribcol/builder-config.yaml

@@ -241,6 +241,7 @@ replaces:
   - github.com/open-telemetry/opentelemetry-collector-contrib/exporter/cassandraexporter => ../../exporter/cassandraexporter
   - github.com/open-telemetry/opentelemetry-collector-contrib/receiver/udplogreceiver => ../../receiver/udplogreceiver
   - github.com/open-telemetry/opentelemetry-collector-contrib/internal/splunk => ../../internal/splunk
+  - github.com/open-telemetry/opentelemetry-collector-contrib/internal/datadog => ../../internal/datadog
   - github.com/open-telemetry/opentelemetry-collector-contrib/receiver/podmanreceiver => ../../receiver/podmanreceiver
   - github.com/open-telemetry/opentelemetry-collector-contrib/receiver/flinkmetricsreceiver => ../../receiver/flinkmetricsreceiver
   - github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsecscontainermetricsreceiver => ../../receiver/awsecscontainermetricsreceiver

cmd/otelcontribcol/go.mod

@@ -518,6 +518,7 @@ require (
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/metrics v0.82.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/proxy v0.82.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/xray v0.82.0 // indirect
+	github.com/open-telemetry/opentelemetry-collector-contrib/internal/datadog v0.82.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/docker v0.82.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/filter v0.82.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/k8sconfig v0.82.0 // indirect
@@ -782,6 +783,8 @@ replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/udplo
 
 replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/splunk => ../../internal/splunk
 
+replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/datadog => ../../internal/datadog
+
 replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/podmanreceiver => ../../receiver/podmanreceiver
 
 replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/flinkmetricsreceiver => ../../receiver/flinkmetricsreceiver

go.mod

@@ -499,6 +499,7 @@ require (
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/xray v0.82.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/common v0.82.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.82.0 // indirect
+	github.com/open-telemetry/opentelemetry-collector-contrib/internal/datadog v0.82.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/docker v0.82.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/filter v0.82.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/k8sconfig v0.82.0 // indirect
@@ -1093,6 +1094,8 @@ replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/zooke
 
 replace github.com/open-telemetry/opentelemetry-collector-contrib/extension/observer/dockerobserver => ./extension/observer/dockerobserver
 
+replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/datadog => ./internal/datadog
+
 replace github.com/open-telemetry/opentelemetry-collector-contrib/extension/jaegerremotesampling => ./extension/jaegerremotesampling
 
 // see https://github.com/google/gnostic/issues/262

internal/datadog/Makefile

@@ -0,0 +1 @@
+include ../../Makefile.Common

processor/datadogprocessor/agent.go → internal/datadog/agent.go

@@ -1,7 +1,7 @@
 // Copyright The OpenTelemetry Authors
 // SPDX-License-Identifier: Apache-2.0
 
-package datadogprocessor // import "github.com/open-telemetry/opentelemetry-collector-contrib/processor/datadogprocessor"
+package datadog // import "github.com/open-telemetry/opentelemetry-collector-contrib/internal/datadog"
 
 import (
 	"context"
@@ -24,8 +24,8 @@ import (
 // computed for the resource spans.
 const keyStatsComputed = "_dd.stats_computed"
 
-// traceagent specifies a minimal trace agent instance that is able to process traces and output stats.
-type traceagent struct {
+// TraceAgent specifies a minimal trace agent instance that is able to process traces and output stats.
+type TraceAgent struct {
 	*agent.Agent
 
 	// pchan specifies the channel that will be used to output Datadog Trace Agent API Payloads
@@ -41,12 +41,12 @@ type traceagent struct {
 
 // newAgent creates a new unstarted traceagent using the given context. Call Start to start the traceagent.
 // The out channel will receive outoing stats payloads resulting from spans ingested using the Ingest method.
-func newAgent(ctx context.Context, out chan pb.StatsPayload) *traceagent {
-	return newAgentWithConfig(ctx, traceconfig.New(), out)
+func NewAgent(ctx context.Context, out chan pb.StatsPayload) *TraceAgent {
+	return NewAgentWithConfig(ctx, traceconfig.New(), out)
 }
 
 // newAgentWithConfig creates a new traceagent with the given config cfg. Used in tests; use newAgent instead.
-func newAgentWithConfig(ctx context.Context, cfg *traceconfig.AgentConfig, out chan pb.StatsPayload) *traceagent {
+func NewAgentWithConfig(ctx context.Context, cfg *traceconfig.AgentConfig, out chan pb.StatsPayload) *TraceAgent {
 	// disable the HTTP receiver
 	cfg.ReceiverPort = 0
 	// set the API key to succeed startup; it is never used nor needed
@@ -68,15 +68,15 @@ func newAgentWithConfig(ctx context.Context, cfg *traceconfig.AgentConfig, out c
 	// lastly, start the OTLP receiver, which will be used to introduce ResourceSpans into the traceagent,
 	// so that we can transform them to Datadog spans and receive stats.
 	a.OTLPReceiver = api.NewOTLPReceiver(pchan, cfg)
-	return &traceagent{
+	return &TraceAgent{
 		Agent: a,
 		exit:  make(chan struct{}),
 		pchan: pchan,
 	}
 }
 
 // Start starts the traceagent, making it ready to ingest spans.
-func (p *traceagent) Start() {
+func (p *TraceAgent) Start() {
 	// we don't need to start the full agent, so we only start a set of minimal
 	// components needed to compute stats:
 	for _, starter := range []interface{ Start() }{
@@ -98,7 +98,7 @@ func (p *traceagent) Start() {
 }
 
 // Stop stops the traceagent, making it unable to ingest spans. Do not call Ingest after Stop.
-func (p *traceagent) Stop() {
+func (p *TraceAgent) Stop() {
 	for _, stopper := range []interface{ Stop() }{
 		p.Concentrator,
 		p.ClientStatsAggregator,
@@ -115,7 +115,7 @@ func (p *traceagent) Stop() {
 
 // goDrain drains the TraceWriter channel, ensuring it won't block. We don't need the traces,
 // nor do we have a running TraceWrite. We just want the outgoing stats.
-func (p *traceagent) goDrain() {
+func (p *TraceAgent) goDrain() {
 	p.wg.Add(1)
 	go func() {
 		defer p.wg.Done()
@@ -132,7 +132,7 @@ func (p *traceagent) goDrain() {
 
 // Ingest processes the given spans within the traceagent and outputs stats through the output channel
 // provided to newAgent. Do not call Ingest on an unstarted or stopped traceagent.
-func (p *traceagent) Ingest(ctx context.Context, traces ptrace.Traces) {
+func (p *TraceAgent) Ingest(ctx context.Context, traces ptrace.Traces) {
 	rspanss := traces.ResourceSpans()
 	for i := 0; i < rspanss.Len(); i++ {
 		rspans := rspanss.At(i)
@@ -148,7 +148,7 @@ func (p *traceagent) Ingest(ctx context.Context, traces ptrace.Traces) {
 
 // goProcesses runs the main loop which takes incoming payloads, processes them and generates stats.
 // It then picks up those stats and converts them to metrics.
-func (p *traceagent) goProcess() {
+func (p *TraceAgent) goProcess() {
 	for i := 0; i < runtime.NumCPU(); i++ {
 		p.wg.Add(1)
 		go func() {
@@ -167,10 +167,10 @@ func (p *traceagent) goProcess() {
 	}
 }
 
-var _ ingester = (*traceagent)(nil)
+var _ Ingester = (*TraceAgent)(nil)
 
-// An ingester is able to ingest traces. Implemented by traceagent.
-type ingester interface {
+// An Ingester is able to ingest traces. Implemented by traceagent.
+type Ingester interface {
 	// Start starts the ingester.
 	Start()
 

processor/datadogprocessor/agent_test.go → internal/datadog/agent_test.go

@@ -1,7 +1,7 @@
 // Copyright The OpenTelemetry Authors
 // SPDX-License-Identifier: Apache-2.0
 
-package datadogprocessor // import "github.com/open-telemetry/opentelemetry-collector-contrib/processor/datadogprocessor"
+package datadog // import "github.com/open-telemetry/opentelemetry-collector-contrib/processor/datadog"
 
 import (
 	"context"
@@ -21,7 +21,7 @@ func TestTraceAgentConfig(t *testing.T) {
 	require.NotZero(t, cfg.ReceiverPort)
 
 	out := make(chan pb.StatsPayload)
-	agnt := newAgentWithConfig(context.Background(), cfg, out)
+	agnt := NewAgentWithConfig(context.Background(), cfg, out)
 	require.Zero(t, cfg.ReceiverPort)
 	require.NotEmpty(t, cfg.Endpoints[0].APIKey)
 	require.Equal(t, metrics.UnsetHostnamePlaceholder, cfg.Hostname)
@@ -33,7 +33,7 @@ func TestTraceAgent(t *testing.T) {
 	cfg.BucketInterval = 50 * time.Millisecond
 	out := make(chan pb.StatsPayload, 10)
 	ctx := context.Background()
-	a := newAgentWithConfig(ctx, cfg, out)
+	a := NewAgentWithConfig(ctx, cfg, out)
 	a.Start()
 	defer a.Stop()
 

internal/datadog/go.mod

@@ -0,0 +1,72 @@
+module github.com/open-telemetry/opentelemetry-collector-contrib/internal/datadog
+
+go 1.19
+
+require (
+	github.com/DataDog/datadog-agent/pkg/trace v0.48.0-devel
+	github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/metrics v0.5.2
+	github.com/stretchr/testify v1.8.4
+	go.opentelemetry.io/collector/pdata v1.0.0-rcv0014
+)
+
+require (
+	github.com/DataDog/datadog-agent/pkg/obfuscate v0.47.0-rc.3 // indirect
+	github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.47.0-rc.3 // indirect
+	github.com/DataDog/datadog-agent/pkg/util/cgroups v0.47.0-rc.3 // indirect
+	github.com/DataDog/datadog-agent/pkg/util/log v0.47.0-rc.3 // indirect
+	github.com/DataDog/datadog-agent/pkg/util/pointer v0.47.0-rc.3 // indirect
+	github.com/DataDog/datadog-agent/pkg/util/scrubber v0.47.0-rc.3 // indirect
+	github.com/DataDog/datadog-go/v5 v5.1.1 // indirect
+	github.com/DataDog/go-tuf v0.3.0--fix-localmeta-fork // indirect
+	github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.5.2 // indirect
+	github.com/DataDog/opentelemetry-mapping-go/pkg/quantile v0.5.2 // indirect
+	github.com/DataDog/sketches-go v1.4.2 // indirect
+	github.com/Microsoft/go-winio v0.6.0 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 // indirect
+	github.com/containerd/cgroups v1.0.4 // indirect
+	github.com/coreos/go-systemd/v22 v22.3.2 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/docker/go-units v0.4.0 // indirect
+	github.com/dustin/go-humanize v1.0.1 // indirect
+	github.com/go-ole/go-ole v1.2.6 // indirect
+	github.com/godbus/dbus/v5 v5.0.6 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/mock v1.6.0 // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/google/uuid v1.3.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/karrick/godirwalk v1.17.0 // indirect
+	github.com/lufia/plan9stats v0.0.0-20220913051719-115f729f3c8c // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417 // indirect
+	github.com/outcaste-io/ristretto v0.2.1 // indirect
+	github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
+	github.com/philhofer/fwd v1.1.2 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/power-devops/perfstat v0.0.0-20220216144756-c35f1ee13d7c // indirect
+	github.com/secure-systems-lab/go-securesystemslib v0.5.0 // indirect
+	github.com/shirou/gopsutil/v3 v3.23.2 // indirect
+	github.com/tinylib/msgp v1.1.8 // indirect
+	github.com/tklauser/go-sysconf v0.3.11 // indirect
+	github.com/tklauser/numcpus v0.6.0 // indirect
+	github.com/yusufpapurcu/wmi v1.2.2 // indirect
+	go.opentelemetry.io/collector/semconv v0.82.0 // indirect
+	go.uber.org/atomic v1.11.0 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	go.uber.org/zap v1.24.0 // indirect
+	golang.org/x/exp v0.0.0-20230321023759-10a507213a29 // indirect
+	golang.org/x/mod v0.10.0 // indirect
+	golang.org/x/net v0.12.0 // indirect
+	golang.org/x/sys v0.10.0 // indirect
+	golang.org/x/text v0.11.0 // indirect
+	golang.org/x/time v0.3.0 // indirect
+	golang.org/x/tools v0.9.1 // indirect
+	google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect
+	google.golang.org/grpc v1.56.2 // indirect
+	google.golang.org/protobuf v1.31.0 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)