From f442b9f17a0852e3e9cce8c4536253ac80c3135b Mon Sep 17 00:00:00 2001 From: Yang Song Date: Tue, 2 Jul 2024 04:42:13 -0400 Subject: [PATCH] [chore][exporter/datadog] Move integration test configs to files (#33832) Move the test configs from hard-coded strings to yaml files. Also set the fake server URL via env vars, and remove batch processor to speed up the tests. Makes it a bit easier to read the configs and add more tests. --- .../datadogexporter/integrationtest/go.mod | 2 +- .../integrationtest/integration_test.go | 146 ++---------------- .../integration_test_config.yaml | 58 +++++++ .../integration_test_toplevel_config.yaml | 45 ++++++ 4 files changed, 116 insertions(+), 135 deletions(-) create mode 100644 exporter/datadogexporter/integrationtest/integration_test_config.yaml create mode 100644 exporter/datadogexporter/integrationtest/integration_test_toplevel_config.yaml diff --git a/exporter/datadogexporter/integrationtest/go.mod b/exporter/datadogexporter/integrationtest/go.mod index 83c5d40e1992..e37eacd84692 100644 --- a/exporter/datadogexporter/integrationtest/go.mod +++ b/exporter/datadogexporter/integrationtest/go.mod @@ -11,6 +11,7 @@ require ( github.com/tinylib/msgp v1.1.9 go.opentelemetry.io/collector/component v0.104.0 go.opentelemetry.io/collector/confmap v0.104.0 + go.opentelemetry.io/collector/confmap/provider/envprovider v0.104.0 go.opentelemetry.io/collector/confmap/provider/fileprovider v0.104.0 go.opentelemetry.io/collector/connector v0.104.0 go.opentelemetry.io/collector/exporter v0.104.0 @@ -220,7 +221,6 @@ require ( go.opentelemetry.io/collector/config/configtls v0.104.0 // indirect go.opentelemetry.io/collector/config/internal v0.104.0 // indirect go.opentelemetry.io/collector/confmap/converter/expandconverter v0.104.0 // indirect - go.opentelemetry.io/collector/confmap/provider/envprovider v0.104.0 // indirect go.opentelemetry.io/collector/confmap/provider/httpprovider v0.104.0 // indirect go.opentelemetry.io/collector/confmap/provider/yamlprovider v0.104.0 // indirect go.opentelemetry.io/collector/consumer v0.104.0 // indirect diff --git a/exporter/datadogexporter/integrationtest/integration_test.go b/exporter/datadogexporter/integrationtest/integration_test.go index c9a0f131b223..c3438707eb09 100644 --- a/exporter/datadogexporter/integrationtest/integration_test.go +++ b/exporter/datadogexporter/integrationtest/integration_test.go @@ -9,7 +9,6 @@ import ( "context" "fmt" "io" - "os" "strings" "testing" "time" @@ -20,6 +19,7 @@ import ( "github.com/tinylib/msgp/msgp" "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/confmap" + "go.opentelemetry.io/collector/confmap/provider/envprovider" "go.opentelemetry.io/collector/confmap/provider/fileprovider" "go.opentelemetry.io/collector/connector" "go.opentelemetry.io/collector/exporter" @@ -45,124 +45,6 @@ import ( "github.com/open-telemetry/opentelemetry-collector-contrib/processor/tailsamplingprocessor" ) -const collectorConfig = ` -receivers: - otlp: - protocols: - http: - endpoint: "localhost:4318" - grpc: - endpoint: "localhost:4317" - -processors: - batch: - send_batch_size: 10 - timeout: 5s - tail_sampling: - decision_wait: 1s - policies: [ - { - name: sample_flag, - type: boolean_attribute, - boolean_attribute: { key: sampled, value: true }, - } - ] - -connectors: - datadog/connector: - traces: - compute_stats_by_span_kind: true - peer_tags_aggregation: true - peer_tags: ["extra_peer_tag"] - -exporters: - debug: - verbosity: detailed - datadog: - api: - key: "key" - tls: - insecure_skip_verify: true - host_metadata: - enabled: false - traces: - endpoint: %q - trace_buffer: 10 - metrics: - endpoint: %q - -service: - telemetry: - metrics: - level: none - pipelines: - traces: - receivers: [otlp] - processors: [batch] - exporters: [datadog/connector] - traces/2: # this pipeline uses sampling - receivers: [datadog/connector] - processors: [tail_sampling, batch] - exporters: [datadog, debug] - metrics: - receivers: [datadog/connector] - processors: [batch] - exporters: [datadog, debug]` - -const collectorConfigComputeTopLevelBySpanKind = ` -receivers: - otlp: - protocols: - http: - endpoint: "localhost:4318" - grpc: - endpoint: "localhost:4317" - -processors: - batch: - send_batch_size: 10 - timeout: 5s - -connectors: - datadog/connector: - traces: - compute_top_level_by_span_kind: true - -exporters: - debug: - verbosity: detailed - datadog: - api: - key: "key" - tls: - insecure_skip_verify: true - host_metadata: - enabled: false - traces: - endpoint: %q - trace_buffer: 10 - compute_top_level_by_span_kind: true - metrics: - endpoint: %q - -service: - telemetry: - metrics: - level: none - pipelines: - traces: - receivers: [otlp] - processors: [batch] - exporters: [datadog/connector] - traces/2: - receivers: [datadog/connector] - processors: [batch] - exporters: [datadog, debug] - metrics: - receivers: [datadog/connector] - processors: [batch] - exporters: [datadog, debug]` - func TestIntegration_NativeOTelAPMStatsIngest(t *testing.T) { previousVal := datadogconnector.NativeIngestFeatureGate.IsEnabled() err := featuregate.GlobalRegistry().Set(datadogconnector.NativeIngestFeatureGate.ID(), true) @@ -186,15 +68,16 @@ func testIntegration(t *testing.T) { tracesRec := &testutil.HTTPRequestRecorderWithChan{Pattern: testutil.TraceEndpoint, ReqChan: make(chan []byte)} server := testutil.DatadogServerMock(apmstatsRec.HandlerFunc, tracesRec.HandlerFunc) defer server.Close() + t.Setenv("SERVER_URL", server.URL) // 2. Start in-process collector factories := getIntegrationTestComponents(t) - app, confFilePath := getIntegrationTestCollector(t, collectorConfig, server.URL, factories) + app := getIntegrationTestCollector(t, "integration_test_config.yaml", factories) go func() { assert.NoError(t, app.Run(context.Background())) }() defer app.Shutdown() - defer os.Remove(confFilePath) + waitForReadiness(app) // 3. Generate and send traces @@ -278,24 +161,18 @@ func getIntegrationTestComponents(t *testing.T) otelcol.Factories { return factories } -func getIntegrationTestCollector(t *testing.T, cfgStr string, url string, factories otelcol.Factories) (*otelcol.Collector, string) { - cfg := fmt.Sprintf(cfgStr, url, url) - - confFile, err := os.CreateTemp(os.TempDir(), "conf-") - require.NoError(t, err) - _, err = confFile.Write([]byte(cfg)) - require.NoError(t, err) +func getIntegrationTestCollector(t *testing.T, cfgFile string, factories otelcol.Factories) *otelcol.Collector { // https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/33594 // nolint:staticcheck - _, err = otelcoltest.LoadConfigAndValidate(confFile.Name(), factories) + _, err := otelcoltest.LoadConfigAndValidate(cfgFile, factories) require.NoError(t, err, "All yaml config must be valid.") appSettings := otelcol.CollectorSettings{ Factories: func() (otelcol.Factories, error) { return factories, nil }, ConfigProviderSettings: otelcol.ConfigProviderSettings{ ResolverSettings: confmap.ResolverSettings{ - URIs: []string{confFile.Name()}, - ProviderFactories: []confmap.ProviderFactory{fileprovider.NewFactory()}, + URIs: []string{cfgFile}, + ProviderFactories: []confmap.ProviderFactory{fileprovider.NewFactory(), envprovider.NewFactory()}, }, }, BuildInfo: component.BuildInfo{ @@ -307,7 +184,7 @@ func getIntegrationTestCollector(t *testing.T, cfgStr string, url string, factor app, err := otelcol.NewCollector(appSettings) require.NoError(t, err) - return app, confFile.Name() + return app } func waitForReadiness(app *otelcol.Collector) { @@ -382,15 +259,16 @@ func TestIntegrationComputeTopLevelBySpanKind(t *testing.T) { tracesRec := &testutil.HTTPRequestRecorderWithChan{Pattern: testutil.TraceEndpoint, ReqChan: make(chan []byte)} server := testutil.DatadogServerMock(apmstatsRec.HandlerFunc, tracesRec.HandlerFunc) defer server.Close() + t.Setenv("SERVER_URL", server.URL) // 2. Start in-process collector factories := getIntegrationTestComponents(t) - app, confFilePath := getIntegrationTestCollector(t, collectorConfigComputeTopLevelBySpanKind, server.URL, factories) + app := getIntegrationTestCollector(t, "integration_test_toplevel_config.yaml", factories) go func() { assert.NoError(t, app.Run(context.Background())) }() defer app.Shutdown() - defer os.Remove(confFilePath) + waitForReadiness(app) // 3. Generate and send traces diff --git a/exporter/datadogexporter/integrationtest/integration_test_config.yaml b/exporter/datadogexporter/integrationtest/integration_test_config.yaml new file mode 100644 index 000000000000..19aa31595cf8 --- /dev/null +++ b/exporter/datadogexporter/integrationtest/integration_test_config.yaml @@ -0,0 +1,58 @@ + +receivers: + otlp: + protocols: + http: + endpoint: "localhost:4318" + grpc: + endpoint: "localhost:4317" + +processors: + tail_sampling: + decision_wait: 1s + policies: [ + { + name: sample_flag, + type: boolean_attribute, + boolean_attribute: { key: sampled, value: true }, + } + ] + +connectors: + datadog/connector: + traces: + compute_stats_by_span_kind: true + peer_tags_aggregation: true + peer_tags: ["extra_peer_tag"] + +exporters: + debug: + verbosity: detailed + datadog: + api: + key: "key" + tls: + insecure_skip_verify: true + host_metadata: + enabled: false + traces: + endpoint: ${env:SERVER_URL} + trace_buffer: 10 + metrics: + endpoint: ${env:SERVER_URL} + +service: + telemetry: + metrics: + level: none + pipelines: + traces: + receivers: [otlp] + exporters: [datadog/connector] + traces/2: # this pipeline uses sampling + receivers: [datadog/connector] + processors: [tail_sampling] + exporters: [datadog, debug] + metrics: + receivers: [datadog/connector] + exporters: [datadog, debug] \ No newline at end of file diff --git a/exporter/datadogexporter/integrationtest/integration_test_toplevel_config.yaml b/exporter/datadogexporter/integrationtest/integration_test_toplevel_config.yaml new file mode 100644 index 000000000000..f0fac370e413 --- /dev/null +++ b/exporter/datadogexporter/integrationtest/integration_test_toplevel_config.yaml @@ -0,0 +1,45 @@ + +receivers: + otlp: + protocols: + http: + endpoint: "localhost:4318" + grpc: + endpoint: "localhost:4317" + +connectors: + datadog/connector: + traces: + compute_top_level_by_span_kind: true + +exporters: + debug: + verbosity: detailed + datadog: + api: + key: "key" + tls: + insecure_skip_verify: true + host_metadata: + enabled: false + traces: + endpoint: ${env:SERVER_URL} + trace_buffer: 10 + compute_top_level_by_span_kind: true + metrics: + endpoint: ${env:SERVER_URL} + +service: + telemetry: + metrics: + level: none + pipelines: + traces: + receivers: [otlp] + exporters: [datadog/connector] + traces/2: + receivers: [datadog/connector] + exporters: [datadog, debug] + metrics: + receivers: [datadog/connector] + exporters: [datadog, debug] \ No newline at end of file