From 2aaeb8ee763cf2dcec43f89f173927c9c6f70852 Mon Sep 17 00:00:00 2001 From: Dmitry Anoshin Date: Thu, 29 Jun 2023 11:49:22 -0700 Subject: [PATCH] Split the logs pipeline in the default configuration Split the logs pipeline in the default configuration in a way that profiling data is always sent to Splunk Observability endpoint while other logs can be sent to another hec endpoint configured with `SPLUNK_HEC_URL` and `SPLUNK_HEC_TOKEN` environment variables. --- CHANGELOG.md | 3 ++ .../config/collector/agent_config.yaml | 10 ++++-- .../config/collector/ecs_ec2_config.yaml | 16 +++++---- .../config/collector/fargate_config.yaml | 16 +++++---- .../config/collector/full_config_linux.yaml | 13 ++++++- .../config/collector/gateway_config.yaml | 23 +++++++++--- .../collector/upstream_agent_config.yaml | 7 +++- tests/general/default_config_test.go | 35 +++++++++++++++---- 8 files changed, 95 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e7ba0ceed5..138a5f936e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,9 @@ - (Splunk) Add bundled collectd/nginx Smart Agent receiver discovery rules ([#3321](https://github.com/signalfx/splunk-otel-collector/pull/3321)) - (Splunk) Support custom `--discovery-properties` file ([#3334](https://github.com/signalfx/splunk-otel-collector/pull/3334)) - (Splunk) Add `--discovery` to the Linux installer script ([#3365](https://github.com/signalfx/splunk-otel-collector/pull/3365)) +- (Splunk) Starting from this version the logs pipeline is split in the default configuration in a way that profiling + data is always sent to Splunk Observability endpoint while other logs can be sent to another hec endpoint configured + with `SPLUNK_HEC_URL` and `SPLUNK_HEC_TOKEN` environment variables ([#3330](https://github.com/signalfx/splunk-otel-collector/pull/3330)) ## v0.80.0 diff --git a/cmd/otelcol/config/collector/agent_config.yaml b/cmd/otelcol/config/collector/agent_config.yaml index 6f15235eb0..3a829a27d8 100644 --- a/cmd/otelcol/config/collector/agent_config.yaml +++ b/cmd/otelcol/config/collector/agent_config.yaml @@ -143,6 +143,12 @@ exporters: endpoint: "${SPLUNK_HEC_URL}" source: "otel" sourcetype: "otel" + profiling_data_enabled: false + # Profiling + splunk_hec/profiling: + token: "${SPLUNK_ACCESS_TOKEN}" + endpoint: "${SPLUNK_INGEST_URL}/v1/log" + log_data_enabled: false # Send to gateway otlp: endpoint: "${SPLUNK_GATEWAY_URL}:4317" @@ -181,8 +187,6 @@ service: receivers: [signalfx, smartagent/processlist] processors: [memory_limiter, batch, resourcedetection] exporters: [signalfx] - # Use instead when sending to gateway - #exporters: [otlp] logs: receivers: [fluentforward, otlp] processors: @@ -190,6 +194,6 @@ service: - batch - resourcedetection #- resource/add_environment - exporters: [splunk_hec] + exporters: [splunk_hec, splunk_hec/profiling] # Use instead when sending to gateway #exporters: [otlp] diff --git a/cmd/otelcol/config/collector/ecs_ec2_config.yaml b/cmd/otelcol/config/collector/ecs_ec2_config.yaml index 2b0aee010c..6e0654a49d 100644 --- a/cmd/otelcol/config/collector/ecs_ec2_config.yaml +++ b/cmd/otelcol/config/collector/ecs_ec2_config.yaml @@ -129,12 +129,18 @@ exporters: access_token: "${SPLUNK_ACCESS_TOKEN}" realm: "${SPLUNK_REALM}" correlation: - # Logs + Profiling + # Logs splunk_hec: token: "${SPLUNK_HEC_TOKEN}" endpoint: "${SPLUNK_HEC_URL}" source: "otel" sourcetype: "otel" + profiling_data_enabled: false + # Profiling + splunk_hec/profiling: + token: "${SPLUNK_ACCESS_TOKEN}" + endpoint: "${SPLUNK_INGEST_URL}/v1/log" + log_data_enabled: false service: extensions: [health_check, http_forwarder, zpages, memory_ballast] @@ -145,7 +151,7 @@ service: - memory_limiter - batch - resourcedetection - # - resource/add_environment + #- resource/add_environment exporters: [sapm, signalfx] metrics: receivers: [hostmetrics, signalfx, smartagent/signalfx-forwarder, smartagent/ecs-metadata] @@ -161,7 +167,5 @@ service: - memory_limiter - batch - resourcedetection - #- resource/add_environment - exporters: [splunk_hec] - # Use instead when sending to gateway - #exporters: [otlp] + #- resource/add_environment + exporters: [splunk_hec, splunk_hec/profiling] diff --git a/cmd/otelcol/config/collector/fargate_config.yaml b/cmd/otelcol/config/collector/fargate_config.yaml index dc002425e7..0c4ebeb22f 100644 --- a/cmd/otelcol/config/collector/fargate_config.yaml +++ b/cmd/otelcol/config/collector/fargate_config.yaml @@ -103,12 +103,18 @@ exporters: access_token: "${SPLUNK_ACCESS_TOKEN}" realm: "${SPLUNK_REALM}" correlation: - # Logs + Profiling + # Logs splunk_hec: token: "${SPLUNK_HEC_TOKEN}" endpoint: "${SPLUNK_HEC_URL}" source: "otel" sourcetype: "otel" + profiling_data_enabled: false + # Profiling + splunk_hec/profiling: + token: "${SPLUNK_ACCESS_TOKEN}" + endpoint: "${SPLUNK_INGEST_URL}/v1/log" + log_data_enabled: false service: extensions: [health_check, http_forwarder, zpages, memory_ballast] @@ -119,7 +125,7 @@ service: - memory_limiter - batch - resourcedetection -# - resource/add_environment + #- resource/add_environment exporters: [sapm, signalfx] metrics: receivers: [signalfx, smartagent/signalfx-forwarder, smartagent/ecs-metadata, prometheus/internal] @@ -131,7 +137,5 @@ service: - memory_limiter - batch - resourcedetection - #- resource/add_environment - exporters: [splunk_hec] - # Use instead when sending to gateway - #exporters: [otlp] + #- resource/add_environment + exporters: [splunk_hec, splunk_hec/profiling] diff --git a/cmd/otelcol/config/collector/full_config_linux.yaml b/cmd/otelcol/config/collector/full_config_linux.yaml index af57c262d6..c1b75d77e4 100644 --- a/cmd/otelcol/config/collector/full_config_linux.yaml +++ b/cmd/otelcol/config/collector/full_config_linux.yaml @@ -386,6 +386,17 @@ exporters: timeout: 10s # Whether to skip checking the certificate of the HEC endpoint when sending data over HTTPS. Defaults to false. insecure_skip_verify: false + # Profiling data should always be sent to Splunk Observability Cloud which is configured in the next section. + profiling_data_enabled: false + + # Enables the splunk hec exporter to send profiling data to Splunk Observability Cloud. + splunk_hec/profiling: + # Splunk Observability Cloud access token. + token: "${SPLUNK_ACCESS_TOKEN}" + # Splunk Observability Cloud endpoint. + endpoint: "https://ingest.${SPLUNK_REALM}.signalfx.com/v1/log" + # Disabling logs as only profiling data should be sent to Splunk Observability Cloud. + logs_data_enabled: false ############################################################################# # Traces @@ -568,4 +579,4 @@ service: logs: receivers: [fluentforward, otlp] processors: [memory_limiter, batch] - exporters: [splunk_hec] + exporters: [splunk_hec, splunk_hec/profiling] diff --git a/cmd/otelcol/config/collector/gateway_config.yaml b/cmd/otelcol/config/collector/gateway_config.yaml index ac72a09b17..b0c8ff8383 100644 --- a/cmd/otelcol/config/collector/gateway_config.yaml +++ b/cmd/otelcol/config/collector/gateway_config.yaml @@ -107,9 +107,18 @@ exporters: access_token: "${SPLUNK_ACCESS_TOKEN}" realm: "${SPLUNK_REALM}" sync_host_metadata: true - # Debug - #logging: - #verbosity: detailed + # Logs + splunk_hec: + token: "${SPLUNK_HEC_TOKEN}" + endpoint: "${SPLUNK_HEC_URL}" + source: "otel" + sourcetype: "otel" + profiling_data_enabled: false + # Profiling + splunk_hec/profiling: + token: "${SPLUNK_ACCESS_TOKEN}" + endpoint: "${SPLUNK_INGEST_URL}/v1/log" + log_data_enabled: false service: extensions: [health_check, http_forwarder, zpages, memory_ballast] @@ -129,7 +138,11 @@ service: receivers: [prometheus/internal] processors: [memory_limiter, batch, resourcedetection/internal] exporters: [signalfx/internal] - logs: - receivers: [otlp, signalfx] + logs/signalfx: + receivers: [signalfx] processors: [memory_limiter, batch] exporters: [signalfx] + logs: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [splunk_hec, splunk_hec/profiling] diff --git a/cmd/otelcol/config/collector/upstream_agent_config.yaml b/cmd/otelcol/config/collector/upstream_agent_config.yaml index 11c032f23d..c983acef8b 100644 --- a/cmd/otelcol/config/collector/upstream_agent_config.yaml +++ b/cmd/otelcol/config/collector/upstream_agent_config.yaml @@ -137,6 +137,11 @@ exporters: endpoint: "${SPLUNK_HEC_URL}" source: "otel" sourcetype: "otel" + profiling_data_enabled: false + splunk_hec/profiling: + token: "${SPLUNK_ACCESS_TOKEN}" + endpoint: "https://ingest.${SPLUNK_REALM}.signalfx.com/v1/log" + logs_data_enabled: false # Send to gateway otlp: endpoint: "${SPLUNK_GATEWAY_URL}:4317" @@ -189,6 +194,6 @@ service: - batch - resourcedetection #- resource/add_environment - exporters: [splunk_hec] + exporters: [splunk_hec, splunk_hec/profiling] # Use instead when sending to gateway #exporters: [otlp] diff --git a/tests/general/default_config_test.go b/tests/general/default_config_test.go index 53ad838c23..3ba72e5087 100644 --- a/tests/general/default_config_test.go +++ b/tests/general/default_config_test.go @@ -64,6 +64,18 @@ func TestDefaultGatewayConfig(t *testing.T) { "realm": "not.real", "sync_host_metadata": true, }, + "splunk_hec": map[string]any{ + "endpoint": "https://ingest.not.real.signalfx.com/v1/log", + "source": "otel", + "sourcetype": "otel", + "token": "", + "profiling_data_enabled": false, + }, + "splunk_hec/profiling": map[string]any{ + "endpoint": "https://ingest.not.real.signalfx.com/v1/log", + "token": "", + "logs_data_enabled": false, + }, }, "extensions": map[string]any{ "health_check": map[string]any{ @@ -145,9 +157,14 @@ func TestDefaultGatewayConfig(t *testing.T) { "extensions": []any{"health_check", "http_forwarder", "zpages", "memory_ballast"}, "pipelines": map[string]any{ "logs": map[string]any{ + "exporters": []any{"otlp"}, + "processors": []any{"memory_limiter", "batch"}, + "receivers": []any{"splunk_hec", "splunk_hec/profiling"}, + }, + "logs/signalfx": map[string]any{ "exporters": []any{"signalfx"}, "processors": []any{"memory_limiter", "batch"}, - "receivers": []any{"otlp", "signalfx"}, + "receivers": []any{"signalfx"}, }, "metrics": map[string]any{ "exporters": []any{"signalfx"}, @@ -211,10 +228,16 @@ func TestDefaultAgentConfig(t *testing.T) { "sync_host_metadata": true, }, "splunk_hec": map[string]any{ - "endpoint": "https://ingest.not.real.signalfx.com/v1/log", - "source": "otel", - "sourcetype": "otel", - "token": "", + "endpoint": "https://ingest.not.real.signalfx.com/v1/log", + "source": "otel", + "sourcetype": "otel", + "token": "", + "profiling_data_enabled": false, + }, + "splunk_hec/profiling": map[string]any{ + "endpoint": "https://ingest.not.real.signalfx.com/v1/log", + "token": "", + "logs_data_enabled": false, }, }, "extensions": map[string]any{ @@ -305,7 +328,7 @@ func TestDefaultAgentConfig(t *testing.T) { "extensions": []any{"health_check", "http_forwarder", "zpages", "memory_ballast", "smartagent"}, "pipelines": map[string]any{ "logs": map[string]any{ - "exporters": []any{"splunk_hec"}, + "exporters": []any{"splunk_hec", "splunk_hec/profiling"}, "processors": []any{"memory_limiter", "batch", "resourcedetection"}, "receivers": []any{"fluentforward", "otlp"}}, "logs/signalfx": map[string]any{