Datadog Connector Component

.github/CODEOWNERS

@@ -112,6 +112,7 @@ extension/storage/filestorage/                           @open-telemetry/collect
 
 internal/aws/                                            @open-telemetry/collector-contrib-approvers @Aneurysm9 @mxiamxia
 internal/coreinternal/                                   @open-telemetry/collector-contrib-approvers @open-telemetry/collector-approvers
+internal/datadog/                                        @open-telemetry/collector-contrib-approvers @mx-psi @gbbr @dineshg13
 internal/docker/                                         @open-telemetry/collector-contrib-approvers @rmfitzpatrick @jamesmoessis
 internal/filter/                                         @open-telemetry/collector-contrib-approvers @open-telemetry/collector-approvers
 internal/k8sconfig/                                      @open-telemetry/collector-contrib-approvers @dmitryax

.github/ISSUE_TEMPLATE/bug_report.yaml

@@ -103,6 +103,7 @@ body:
       - extension/storage/filestorage
       - internal/aws
       - internal/core
+      - internal/datadog
       - internal/docker
       - internal/filter
       - internal/k8sconfig

.github/ISSUE_TEMPLATE/feature_request.yaml

@@ -97,6 +97,7 @@ body:
       - extension/storage/filestorage
       - internal/aws
       - internal/core
+      - internal/datadog
       - internal/docker
       - internal/filter
       - internal/k8sconfig

.github/ISSUE_TEMPLATE/other.yaml

@@ -97,6 +97,7 @@ body:
       - extension/storage/filestorage
       - internal/aws
       - internal/core
+      - internal/datadog
       - internal/docker
       - internal/filter
       - internal/k8sconfig

.github/dependabot.yml

@@ -472,6 +472,11 @@ updates:
     schedule:
       interval: "weekly"
       day: "wednesday"
+  - package-ecosystem: "gomod"
+    directory: "/internal/datadog"
+    schedule:
+      interval: "weekly"
+      day: "wednesday"
   - package-ecosystem: "gomod"
     directory: "/internal/docker"
     schedule:
@@ -1097,8 +1102,3 @@ updates:
     schedule:
       interval: "weekly"
       day: "wednesday"
-  - package-ecosystem: "gomod"
-    directory: "/receiver/sqlqueryreceiver"
-    schedule:
-      interval: "weekly"
-      day: "wednesday"

cmd/configschema/go.mod

@@ -509,6 +509,7 @@ require (
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/proxy v0.82.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/xray v0.82.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/common v0.82.0 // indirect
+	github.com/open-telemetry/opentelemetry-collector-contrib/internal/datadog v0.82.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/sharedcomponent v0.82.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/experimentalmetricmetadata v0.82.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/winperfcounters v0.82.0 // indirect
@@ -1119,4 +1120,6 @@ replace github.com/open-telemetry/opentelemetry-collector-contrib/extension/obse
 
 replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/k8stest => ../../internal/k8stest
 
+replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/datadog => ../../internal/datadog
+
 replace github.com/open-telemetry/opentelemetry-collector-contrib/extension/jaegerremotesampling => ../../extension/jaegerremotesampling

cmd/otelcontribcol/builder-config.yaml

@@ -241,6 +241,7 @@ replaces:
   - github.com/open-telemetry/opentelemetry-collector-contrib/exporter/cassandraexporter => ../../exporter/cassandraexporter
   - github.com/open-telemetry/opentelemetry-collector-contrib/receiver/udplogreceiver => ../../receiver/udplogreceiver
   - github.com/open-telemetry/opentelemetry-collector-contrib/internal/splunk => ../../internal/splunk
+  - github.com/open-telemetry/opentelemetry-collector-contrib/internal/datadog => ../../internal/datadog
   - github.com/open-telemetry/opentelemetry-collector-contrib/receiver/podmanreceiver => ../../receiver/podmanreceiver
   - github.com/open-telemetry/opentelemetry-collector-contrib/receiver/flinkmetricsreceiver => ../../receiver/flinkmetricsreceiver
   - github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsecscontainermetricsreceiver => ../../receiver/awsecscontainermetricsreceiver

cmd/otelcontribcol/go.mod

@@ -518,6 +518,7 @@ require (
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/metrics v0.82.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/proxy v0.82.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/xray v0.82.0 // indirect
+	github.com/open-telemetry/opentelemetry-collector-contrib/internal/datadog v0.82.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/docker v0.82.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/filter v0.82.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/k8sconfig v0.82.0 // indirect
@@ -782,6 +783,8 @@ replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/udplo
 
 replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/splunk => ../../internal/splunk
 
+replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/datadog => ../../internal/datadog
+
 replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/podmanreceiver => ../../receiver/podmanreceiver
 
 replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/flinkmetricsreceiver => ../../receiver/flinkmetricsreceiver

connector/datadogconnector/Makefile

@@ -0,0 +1 @@
+include ../../Makefile.Common

connector/datadogconnector/connector.go

@@ -0,0 +1,119 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package datadogconnector // import "github.com/open-telemetry/opentelemetry-collector-contrib/connector/datadogconnector"
+
+import (
+	"context"
+
+	"github.com/DataDog/datadog-agent/pkg/trace/pb"
+	"github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/metrics"
+	"go.opentelemetry.io/collector/component"
+	"go.opentelemetry.io/collector/consumer"
+	"go.opentelemetry.io/collector/pdata/ptrace"
+	"go.uber.org/zap"
+
+	"github.com/open-telemetry/opentelemetry-collector-contrib/internal/datadog"
+)
+
+// schema for connector
+type connectorImp struct {
+	metricsConsumer consumer.Metrics // the next component in the pipeline to ingest data after connector
+	count           int
+	logger          *zap.Logger
+	started         bool
+
+	// agent specifies the agent used to ingest traces and output APM Stats.
+	// It is implemented by the traceagent structure; replaced in tests.
+	agent datadog.Ingester
+
+	// translator specifies the translator used to transform APM Stats Payloads
+	// from the agent to OTLP Metrics.
+	translator *metrics.Translator
+
+	// in specifies the channel through which the agent will output Stats Payloads
+	// resulting from ingested traces.
+	in chan pb.StatsPayload
+
+	// exit specifies the exit channel, which will be closed upon shutdown.
+	exit chan struct{}
+}
+
+// function to create a new connector
+func newConnector(logger *zap.Logger, config component.Config) (*connectorImp, error) {
+	logger.Info("Building datadog connector")
+	in := make(chan pb.StatsPayload, 100)
+	trans, err := metrics.NewTranslator(logger)
+
+	ctx := context.Background()
+	if err != nil {
+		return nil, err
+	}
+	return &connectorImp{
+		logger:     logger,
+		agent:      datadog.NewAgent(ctx, in),
+		translator: trans,
+		in:         in,
+		exit:       make(chan struct{}),
+	}, nil
+}
+
+// Start implements the component.Component interface.
+func (c *connectorImp) Start(ctx context.Context, _ component.Host) error {
+	c.logger.Info("Starting datadogconnector")
+	c.started = true
+	c.agent.Start()
+	go c.run()
+	return nil
+}
+
+// Shutdown implements the component.Component interface.
+func (c *connectorImp) Shutdown(context.Context) error {
+	c.logger.Info("Shutting down datadog connector")
+	if !c.started {
+		return nil
+	}
+	c.started = false
+	c.agent.Stop()
+	c.exit <- struct{}{} // signal exit
+	<-c.exit             // wait for close
+	return nil
+}
+
+// Capabilities implements the consumer interface.
+// tells use whether the component(connector) will mutate the data passed into it. if set to true the processor does modify the data
+func (c *connectorImp) Capabilities() consumer.Capabilities {
+	return consumer.Capabilities{MutatesData: false}
+}
+
+func (c *connectorImp) ConsumeTraces(ctx context.Context, traces ptrace.Traces) error {
+	c.logger.Debug("Received traces.", zap.Int("spans", traces.SpanCount()))
+	c.agent.Ingest(ctx, traces)
+	return nil
+}
+
+// run awaits incoming stats resulting from the agent's ingestion, converts them
+// to metrics and flushes them using the configured metrics exporter.
+func (c *connectorImp) run() {
+	defer close(c.exit)
+	for {
+		select {
+		case stats := <-c.in:
+			if len(stats.Stats) == 0 {
+				continue
+			}
+			// APM stats as metrics
+			mx := c.translator.StatsPayloadToMetrics(stats)
+			ctx := context.TODO()
+			c.logger.Debug("Exporting APM Stats metrics.", zap.Int("count", mx.MetricCount()))
+
+			// send metrics to the consumer or next compoonet in pipeline
+			if err := c.metricsConsumer.ConsumeMetrics(ctx, mx); err != nil {
+				c.logger.Error("Failed ConsumeMetrics", zap.Error(err))
+				return
+			}
+		case <-c.exit:
+			return
+		}
+	}
+}

connector/datadogconnector/connector_test.go

@@ -0,0 +1,27 @@
+package datadogconnector
+
+import (
+	"go.opentelemetry.io/collector/component"
+	"go.uber.org/zap"
+	"reflect"
+	"testing"
+)
+
+var _ component.Component = (*connectorImp)(nil) // testing that the connectorImp properly implements the type Component interface
+
+// create test to create a connector, check that basic code compiles
+func TestNewConnector(t *testing.T) {
+	logger := zap.NewNop()
+	config := createDefaultConfig()
+
+	connector, err := newConnector(logger, config)
+	if err != nil {
+		t.Errorf("Failed to create new connector: %v", err)
+	}
+
+	if reflect.TypeOf(*connector) != reflect.TypeOf(connectorImp{}) {
+		t.Errorf("Wrong type for created connector")
+	}
+}
+
+// test for flag

connector/datadogconnector/factory.go

@@ -0,0 +1,39 @@
+package datadogconnector // import "github.com/open-telemetry/opentelemetry-collector-contrib/connector/datadogconnector"
+
+import (
+	"context"
+
+	"go.opentelemetry.io/collector/component"
+	"go.opentelemetry.io/collector/connector"
+	"go.opentelemetry.io/collector/consumer"
+)
+
+const (
+	defaultVal = 0
+	// this is the name used to refer to the connector in the config.yaml
+	typeStr = "datadog"
+)
+
+// NewFactory creates a factory for tailtracer connector.
+func NewFactory() connector.Factory {
+	//  OTel connector factory to make a factory for connectors
+	return connector.NewFactory(
+		typeStr,
+		createDefaultConfig,
+		connector.WithTracesToMetrics(createTracesToMetricsConnector, component.StabilityLevelAlpha))
+}
+
+func createDefaultConfig() component.Config {
+	return &struct{}{}
+}
+
+// defines the consumer type of the connector
+// we want to consume traces and export metrics therefore define nextConsumer as metrics, consumer is the next component in the pipeline
+func createTracesToMetricsConnector(ctx context.Context, params connector.CreateSettings, cfg component.Config, nextConsumer consumer.Metrics) (connector.Traces, error) {
+	c, err := newConnector(params.Logger, cfg)
+	if err != nil {
+		return nil, err
+	}
+	c.metricsConsumer = nextConsumer
+	return c, nil
+}

connector/datadogconnector/go.mod

@@ -0,0 +1,86 @@
+module github.com/open-telemetry/opentelemetry-collector-contrib/connector/datadogconnector
+
+go 1.20
+
+require (
+	github.com/DataDog/datadog-agent/pkg/trace v0.48.0-devel
+	github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/metrics v0.5.2
+	github.com/open-telemetry/opentelemetry-collector-contrib/internal/datadog v0.82.0
+	go.opentelemetry.io/collector/component v0.82.0
+	go.opentelemetry.io/collector/connector v0.82.0
+	go.opentelemetry.io/collector/consumer v0.82.0
+	go.opentelemetry.io/collector/pdata v1.0.0-rcv0014
+	go.uber.org/zap v1.24.0
+)
+
+require (
+	github.com/DataDog/datadog-agent/pkg/obfuscate v0.47.0-rc.3 // indirect
+	github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.47.0-rc.3 // indirect
+	github.com/DataDog/datadog-agent/pkg/util/cgroups v0.47.0-rc.3 // indirect
+	github.com/DataDog/datadog-agent/pkg/util/log v0.47.0-rc.3 // indirect
+	github.com/DataDog/datadog-agent/pkg/util/pointer v0.47.0-rc.3 // indirect
+	github.com/DataDog/datadog-agent/pkg/util/scrubber v0.47.0-rc.3 // indirect
+	github.com/DataDog/datadog-go/v5 v5.1.1 // indirect
+	github.com/DataDog/go-tuf v0.3.0--fix-localmeta-fork // indirect
+	github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.5.2 // indirect
+	github.com/DataDog/opentelemetry-mapping-go/pkg/quantile v0.5.2 // indirect
+	github.com/DataDog/sketches-go v1.4.2 // indirect
+	github.com/Microsoft/go-winio v0.6.0 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 // indirect
+	github.com/containerd/cgroups v1.0.4 // indirect
+	github.com/coreos/go-systemd/v22 v22.3.2 // indirect
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
+	github.com/docker/go-units v0.5.0 // indirect
+	github.com/dustin/go-humanize v1.0.1 // indirect
+	github.com/go-ole/go-ole v1.2.6 // indirect
+	github.com/godbus/dbus/v5 v5.0.6 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/mock v1.6.0 // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/google/uuid v1.3.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/karrick/godirwalk v1.17.0 // indirect
+	github.com/knadh/koanf v1.5.0 // indirect
+	github.com/knadh/koanf/v2 v2.0.1 // indirect
+	github.com/lufia/plan9stats v0.0.0-20220913051719-115f729f3c8c // indirect
+	github.com/mitchellh/copystructure v1.2.0 // indirect
+	github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4 // indirect
+	github.com/mitchellh/reflectwalk v1.0.2 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417 // indirect
+	github.com/outcaste-io/ristretto v0.2.1 // indirect
+	github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
+	github.com/philhofer/fwd v1.1.2 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/power-devops/perfstat v0.0.0-20220216144756-c35f1ee13d7c // indirect
+	github.com/secure-systems-lab/go-securesystemslib v0.5.0 // indirect
+	github.com/shirou/gopsutil/v3 v3.23.2 // indirect
+	github.com/tinylib/msgp v1.1.8 // indirect
+	github.com/tklauser/go-sysconf v0.3.11 // indirect
+	github.com/tklauser/numcpus v0.6.0 // indirect
+	github.com/yusufpapurcu/wmi v1.2.2 // indirect
+	go.opentelemetry.io/collector/config/configtelemetry v0.82.0 // indirect
+	go.opentelemetry.io/collector/confmap v0.82.0 // indirect
+	go.opentelemetry.io/collector/featuregate v1.0.0-rcv0014 // indirect
+	go.opentelemetry.io/collector/semconv v0.82.0 // indirect
+	go.opentelemetry.io/otel v1.16.0 // indirect
+	go.opentelemetry.io/otel/metric v1.16.0 // indirect
+	go.opentelemetry.io/otel/trace v1.16.0 // indirect
+	go.uber.org/atomic v1.11.0 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df // indirect
+	golang.org/x/mod v0.11.0 // indirect
+	golang.org/x/net v0.12.0 // indirect
+	golang.org/x/sys v0.10.0 // indirect
+	golang.org/x/text v0.11.0 // indirect
+	golang.org/x/time v0.3.0 // indirect
+	golang.org/x/tools v0.9.1 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20230629202037-9506855d4529 // indirect
+	google.golang.org/grpc v1.57.0 // indirect
+	google.golang.org/protobuf v1.31.0 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+)
+
+replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/datadog => ../../internal/datadog