Datadog Connector Component

.chloggen/datadogconnector.yaml

@@ -0,0 +1,27 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: new_component
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: datadogconnector
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: This is a new component that computes Datadog APM Stats in the event that trace pipelines are sampled.
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [19740]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: [user]

.github/CODEOWNERS

@@ -32,6 +32,7 @@ cmd/telemetrygen/                                        @open-telemetry/collect
 confmap/provider/s3provider/                             @open-telemetry/collector-contrib-approvers @Aneurysm9
 
 connector/countconnector/                                @open-telemetry/collector-contrib-approvers @djaglowski @jpkrohling
+connector/datadogconnector/                              @open-telemetry/collector-contrib-approvers @mx-psi @gbbr @dineshg13
 connector/exceptionsconnector/                           @open-telemetry/collector-contrib-approvers @jpkrohling
 connector/routingconnector/                              @open-telemetry/collector-contrib-approvers @jpkrohling @mwear
 connector/servicegraphconnector/                         @open-telemetry/collector-contrib-approvers @jpkrohling @mapno

.github/ISSUE_TEMPLATE/bug_report.yaml

@@ -27,6 +27,7 @@ body:
       - cmd/telemetrygen
       - confmap/provider/s3provider
       - connector/count
+      - connector/datadog
       - connector/exceptions
       - connector/routing
       - connector/servicegraph

.github/ISSUE_TEMPLATE/feature_request.yaml

@@ -21,6 +21,7 @@ body:
       - cmd/telemetrygen
       - confmap/provider/s3provider
       - connector/count
+      - connector/datadog
       - connector/exceptions
       - connector/routing
       - connector/servicegraph

.github/ISSUE_TEMPLATE/other.yaml

@@ -21,6 +21,7 @@ body:
       - cmd/telemetrygen
       - confmap/provider/s3provider
       - connector/count
+      - connector/datadog
       - connector/exceptions
       - connector/routing
       - connector/servicegraph

.github/dependabot.yml

@@ -47,6 +47,11 @@ updates:
     schedule:
       interval: "weekly"
       day: "wednesday"
+  - package-ecosystem: "gomod"
+    directory: "/connector/datadogconnector"
+    schedule:
+      interval: "weekly"
+      day: "wednesday"
   - package-ecosystem: "gomod"
     directory: "/connector/exceptionsconnector"
     schedule:
@@ -1097,8 +1102,3 @@ updates:
     schedule:
       interval: "weekly"
       day: "wednesday"
-  - package-ecosystem: "gomod"
-    directory: "/receiver/splunkhecreceiver"
-    schedule:
-      interval: "weekly"
-      day: "wednesday"

cmd/otelcontribcol/builder-config.yaml

@@ -197,6 +197,7 @@ receivers:
 connectors:
   - gomod: go.opentelemetry.io/collector/connector/forwardconnector v0.82.0
   - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/connector/countconnector v0.82.0
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/connector/datadogconnector v0.82.0
   - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/connector/exceptionsconnector v0.82.0
   - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/connector/routingconnector v0.82.0
   - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/connector/servicegraphconnector v0.82.0
@@ -412,6 +413,7 @@ replaces:
   - github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatatest => ../../pkg/pdatatest
   - github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil => ../../pkg/pdatautil
   - github.com/open-telemetry/opentelemetry-collector-contrib/connector/countconnector => ../../connector/countconnector
+  - github.com/open-telemetry/opentelemetry-collector-contrib/connector/datadogconnector => ../../connector/datadogconnector
   - github.com/open-telemetry/opentelemetry-collector-contrib/connector/exceptionsconnector => ../../connector/exceptionsconnector
   - github.com/open-telemetry/opentelemetry-collector-contrib/connector/routingconnector => ../../connector/routingconnector
   - github.com/open-telemetry/opentelemetry-collector-contrib/connector/servicegraphconnector => ../../connector/servicegraphconnector

cmd/otelcontribcol/go.mod

@@ -6,6 +6,7 @@ go 1.19
 
 require (
 	github.com/open-telemetry/opentelemetry-collector-contrib/connector/countconnector v0.82.0
+	github.com/open-telemetry/opentelemetry-collector-contrib/connector/datadogconnector v0.82.0
 	github.com/open-telemetry/opentelemetry-collector-contrib/connector/exceptionsconnector v0.82.0
 	github.com/open-telemetry/opentelemetry-collector-contrib/connector/routingconnector v0.82.0
 	github.com/open-telemetry/opentelemetry-collector-contrib/connector/servicegraphconnector v0.82.0
@@ -1124,6 +1125,8 @@ replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil
 
 replace github.com/open-telemetry/opentelemetry-collector-contrib/connector/countconnector => ../../connector/countconnector
 
+replace github.com/open-telemetry/opentelemetry-collector-contrib/connector/datadogconnector => ../../connector/datadogconnector
+
 replace github.com/open-telemetry/opentelemetry-collector-contrib/connector/exceptionsconnector => ../../connector/exceptionsconnector
 
 replace github.com/open-telemetry/opentelemetry-collector-contrib/connector/routingconnector => ../../connector/routingconnector

connector/datadogconnector/Makefile

@@ -0,0 +1 @@
+include ../../Makefile.Common

connector/datadogconnector/README.md

@@ -0,0 +1,85 @@
+# Datadog Connector
+
+## Description
+
+The Datadog Connector is a connector component that computes Datadog APM Stats pre-sampling in the event that your traces pipeline is sampled using components such as the tailsamplingprocessor or probabilisticsamplerprocessor.
+
+The connector is most applicable when using the sampling components such as the [tailsamplingprocessor](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/tailsamplingprocessor#tail-sampling-processor), or the [probabilisticsamplerprocessor](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/probabilisticsamplerprocessor) in one of your pipelines. The sampled pipeline should be duplicated and the `datadog` connector should be added to the the pipeline that is not being sampled to ensure that Datadog APM Stats are accurate in the backend.
+
+## Usage
+
+To use the Datadog Connector, add the connector to one set of the duplicated pipelines while sampling the other. The Datadog Connector will compute APM Stats on all spans that it sees. Here is an example on how to add it to a pipeline using the [probabilisticsampler]:
+
+<table>
+<tr>
+<td> Before </td> <td> After </td>
+</tr>
+<tr>
+<td valign="top">
+
+```yaml
+# ...
+processors:
+  # ...
+  probabilistic_sampler:
+    sampling_percentage: 20
+  # add the "datadog" processor definition
+  datadog:
+
+exporters:
+  datadog:
+    api:
+      key: ${env:DD_API_KEY}
+
+service:
+  pipelines:
+    traces:
+      receivers: [otlp]
+      # prepend it to the sampler in your pipeline:
+      processors: [batch, datadog, probabilistic_sampler]
+      exporters: [datadog]
+
+    metrics:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [datadog]
+```
+
+</td><td valign="top">
+
+```yaml
+# ...
+processors:
+  probabilistic_sampler:
+    sampling_percentage: 20
+
+connectors:
+    # add the "datadog" connector definition and further configurations
+    datadog/connector:
+
+exporters:
+  datadog:
+    api:
+      key: ${env:DD_API_KEY}
+
+service:
+  pipelines:
+   traces:
+     receivers: [otlp]
+     processors: [batch]
+     exporters: [datadog/connector]
+
+   traces/2: # this pipeline uses sampling
+     receivers: [otlp]
+     processors: [batch, probabilistic_sampler]
+     exporters: [datadog]
+
+  metrics:
+    receivers: [datadog/connector]
+    processors: [batch]
+    exporters: [datadog]
+```
+</tr></table>
+
+Here we have two traces pipelines that ingest the same data but one is being sampled. The one that is sampled has its data sent to the datadog backend for you to see the sampled subset of the total traces sent across. The other non-sampled pipeline of traces sends its data to the metrics pipeline to be used in the APM stats. This unsampled pipeline gives the full picture of how much data the application emits in traces. 
+

connector/datadogconnector/connector.go

@@ -0,0 +1,114 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package datadogconnector // import "github.com/open-telemetry/opentelemetry-collector-contrib/connector/datadogconnector"
+
+import (
+	"context"
+
+	pb "github.com/DataDog/datadog-agent/pkg/proto/pbgo/trace"
+	"github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/metrics"
+	"go.opentelemetry.io/collector/component"
+	"go.opentelemetry.io/collector/consumer"
+	"go.opentelemetry.io/collector/pdata/ptrace"
+	"go.uber.org/zap"
+
+	"github.com/open-telemetry/opentelemetry-collector-contrib/internal/datadog"
+)
+
+// connectorImp is the schema for connector
+type connectorImp struct {
+	metricsConsumer consumer.Metrics // the next component in the pipeline to ingest data after connector
+	logger          *zap.Logger
+
+	// agent specifies the agent used to ingest traces and output APM Stats.
+	// It is implemented by the traceagent structure; replaced in tests.
+	agent datadog.Ingester
+
+	// translator specifies the translator used to transform APM Stats Payloads
+	// from the agent to OTLP Metrics.
+	translator *metrics.Translator
+
+	// in specifies the channel through which the agent will output Stats Payloads
+	// resulting from ingested traces.
+	in chan *pb.StatsPayload
+
+	// exit specifies the exit channel, which will be closed upon shutdown.
+	exit chan struct{}
+}
+
+var _ component.Component = (*connectorImp)(nil) // testing that the connectorImp properly implements the type Component interface
+
+// function to create a new connector
+func newConnector(logger *zap.Logger, _ component.Config, nextConsumer consumer.Metrics) (*connectorImp, error) {
+	logger.Info("Building datadog connector")
+
+	in := make(chan *pb.StatsPayload, 100)
+	trans, err := metrics.NewTranslator(logger)
+
+	ctx := context.Background()
+	if err != nil {
+		return nil, err
+	}
+	return &connectorImp{
+		logger:          logger,
+		agent:           datadog.NewAgent(ctx, in),
+		translator:      trans,
+		in:              in,
+		metricsConsumer: nextConsumer,
+		exit:            make(chan struct{}),
+	}, nil
+}
+
+// Start implements the component.Component interface.
+func (c *connectorImp) Start(_ context.Context, _ component.Host) error {
+	c.logger.Info("Starting datadogconnector")
+	c.agent.Start()
+	go c.run()
+	return nil
+}
+
+// Shutdown implements the component.Component interface.
+func (c *connectorImp) Shutdown(context.Context) error {
+	c.logger.Info("Shutting down datadog connector")
+	c.agent.Stop()
+	c.exit <- struct{}{} // signal exit
+	<-c.exit             // wait for close
+	return nil
+}
+
+// Capabilities implements the consumer interface.
+// tells use whether the component(connector) will mutate the data passed into it. if set to true the processor does modify the data
+func (c *connectorImp) Capabilities() consumer.Capabilities {
+	return consumer.Capabilities{MutatesData: false}
+}
+
+func (c *connectorImp) ConsumeTraces(ctx context.Context, traces ptrace.Traces) error {
+	c.agent.Ingest(ctx, traces)
+	return nil
+}
+
+// run awaits incoming stats resulting from the agent's ingestion, converts them
+// to metrics and flushes them using the configured metrics exporter.
+func (c *connectorImp) run() {
+	defer close(c.exit)
+	for {
+		select {
+		case stats := <-c.in:
+			if len(stats.Stats) == 0 {
+				continue
+			}
+			// APM stats as metrics
+			mx := c.translator.StatsPayloadToMetrics(stats)
+			ctx := context.TODO()
+
+			// send metrics to the consumer or next component in pipeline
+			if err := c.metricsConsumer.ConsumeMetrics(ctx, mx); err != nil {
+				c.logger.Error("Failed ConsumeMetrics", zap.Error(err))
+				return
+			}
+		case <-c.exit:
+			return
+		}
+	}
+}

connector/datadogconnector/connector_test.go

@@ -0,0 +1,31 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package datadogconnector
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"go.opentelemetry.io/collector/component"
+	"go.opentelemetry.io/collector/connector/connectortest"
+	"go.opentelemetry.io/collector/consumer/consumertest"
+)
+
+var _ component.Component = (*connectorImp)(nil) // testing that the connectorImp properly implements the type Component interface
+
+// create test to create a connector, check that basic code compiles
+func TestNewConnector(t *testing.T) {
+
+	factory := NewFactory()
+
+	creationParams := connectortest.NewNopCreateSettings()
+	cfg := factory.CreateDefaultConfig().(*Config)
+
+	traceConnector, err := factory.CreateTracesToMetrics(context.Background(), creationParams, cfg, consumertest.NewNop())
+	assert.NoError(t, err)
+
+	_, ok := traceConnector.(*connectorImp)
+	assert.True(t, ok) // checks if the created connector implements the connectorImp struct
+}

connector/datadogconnector/doc.go

@@ -0,0 +1,6 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+//go:generate mdatagen metadata.yaml
+
+package datadogconnector // import "github.com/open-telemetry/opentelemetry-collector-contrib/connector/datadogconnector"