Split the logs pipeline in the default configuration

CHANGELOG.md

@@ -8,6 +8,9 @@
 - (Splunk) Add bundled collectd/nginx Smart Agent receiver discovery rules ([#3321](https://github.com/signalfx/splunk-otel-collector/pull/3321))
 - (Splunk) Support custom `--discovery-properties` file ([#3334](https://github.com/signalfx/splunk-otel-collector/pull/3334))
 - (Splunk) Add `--discovery` to the Linux installer script ([#3365](https://github.com/signalfx/splunk-otel-collector/pull/3365))
+- (Splunk) Starting from this version the logs pipeline is split in the default configuration in a way that profiling 
+  data is always sent to Splunk Observability endpoint while other logs can be sent to another hec endpoint configured
+  with `SPLUNK_HEC_URL` and `SPLUNK_HEC_TOKEN` environment variables ([#3330](https://github.com/signalfx/splunk-otel-collector/pull/3330))
 
 ## v0.80.0
 

cmd/otelcol/config/collector/agent_config.yaml

@@ -143,6 +143,12 @@ exporters:
     endpoint: "${SPLUNK_HEC_URL}"
     source: "otel"
     sourcetype: "otel"
+    profiling_data_enabled: false
+  # Profiling
+  splunk_hec/profiling:
+    token: "${SPLUNK_ACCESS_TOKEN}"
+    endpoint: "${SPLUNK_INGEST_URL}/v1/log"
+    log_data_enabled: false
   # Send to gateway
   otlp:
     endpoint: "${SPLUNK_GATEWAY_URL}:4317"
@@ -181,15 +187,13 @@ service:
       receivers: [signalfx, smartagent/processlist]
       processors: [memory_limiter, batch, resourcedetection]
       exporters: [signalfx]
-      # Use instead when sending to gateway
-      #exporters: [otlp]
     logs:
       receivers: [fluentforward, otlp]
       processors:
       - memory_limiter
       - batch
       - resourcedetection
       #- resource/add_environment
-      exporters: [splunk_hec]
+      exporters: [splunk_hec, splunk_hec/profiling]
       # Use instead when sending to gateway
       #exporters: [otlp]

cmd/otelcol/config/collector/ecs_ec2_config.yaml

@@ -129,12 +129,18 @@ exporters:
     access_token: "${SPLUNK_ACCESS_TOKEN}"
     realm: "${SPLUNK_REALM}"
     correlation:
-  # Logs + Profiling
+  # Logs
   splunk_hec:
     token: "${SPLUNK_HEC_TOKEN}"
     endpoint: "${SPLUNK_HEC_URL}"
     source: "otel"
     sourcetype: "otel"
+    profiling_data_enabled: false
+  # Profiling
+  splunk_hec/profiling:
+    token: "${SPLUNK_ACCESS_TOKEN}"
+    endpoint: "${SPLUNK_INGEST_URL}/v1/log"
+    log_data_enabled: false
 
 service:
   extensions: [health_check, http_forwarder, zpages, memory_ballast]
@@ -145,7 +151,7 @@ service:
         - memory_limiter
         - batch
         - resourcedetection
-      #        - resource/add_environment
+        #- resource/add_environment
       exporters: [sapm, signalfx]
     metrics:
       receivers: [hostmetrics, signalfx, smartagent/signalfx-forwarder, smartagent/ecs-metadata]
@@ -161,7 +167,5 @@ service:
         - memory_limiter
         - batch
         - resourcedetection
-      #- resource/add_environment
-      exporters: [splunk_hec]
-      # Use instead when sending to gateway
-      #exporters: [otlp]
+        #- resource/add_environment
+      exporters: [splunk_hec, splunk_hec/profiling]

cmd/otelcol/config/collector/fargate_config.yaml

@@ -103,12 +103,18 @@ exporters:
     access_token: "${SPLUNK_ACCESS_TOKEN}"
     realm: "${SPLUNK_REALM}"
     correlation:
-  # Logs + Profiling
+  # Logs
   splunk_hec:
     token: "${SPLUNK_HEC_TOKEN}"
     endpoint: "${SPLUNK_HEC_URL}"
     source: "otel"
     sourcetype: "otel"
+    profiling_data_enabled: false
+  # Profiling
+  splunk_hec/profiling:
+    token: "${SPLUNK_ACCESS_TOKEN}"
+    endpoint: "${SPLUNK_INGEST_URL}/v1/log"
+    log_data_enabled: false
 
 service:
   extensions: [health_check, http_forwarder, zpages, memory_ballast]
@@ -119,7 +125,7 @@ service:
         - memory_limiter
         - batch
         - resourcedetection
-#        - resource/add_environment
+        #- resource/add_environment
       exporters: [sapm, signalfx]
     metrics:
       receivers: [signalfx, smartagent/signalfx-forwarder, smartagent/ecs-metadata, prometheus/internal]
@@ -131,7 +137,5 @@ service:
         - memory_limiter
         - batch
         - resourcedetection
-      #- resource/add_environment
-      exporters: [splunk_hec]
-      # Use instead when sending to gateway
-      #exporters: [otlp]
+        #- resource/add_environment
+      exporters: [splunk_hec, splunk_hec/profiling]

cmd/otelcol/config/collector/full_config_linux.yaml

@@ -386,6 +386,17 @@ exporters:
     timeout: 10s
     # Whether to skip checking the certificate of the HEC endpoint when sending data over HTTPS. Defaults to false.
     insecure_skip_verify: false
+    # Profiling data should always be sent to Splunk Observability Cloud which is configured in the next section.
+    profiling_data_enabled: false
+
+  # Enables the splunk hec exporter to send profiling data to Splunk Observability Cloud.
+  splunk_hec/profiling:
+    # Splunk Observability Cloud access token.
+    token: "${SPLUNK_ACCESS_TOKEN}"
+    # Splunk Observability Cloud endpoint.
+    endpoint: "https://ingest.${SPLUNK_REALM}.signalfx.com/v1/log"
+    # Disabling logs as only profiling data should be sent to Splunk Observability Cloud.
+    log_data_enabled: false
 
   #############################################################################
   # Traces
@@ -568,4 +579,4 @@ service:
     logs:
       receivers: [fluentforward, otlp]
       processors: [memory_limiter, batch]
-      exporters: [splunk_hec]
+      exporters: [splunk_hec, splunk_hec/profiling]

cmd/otelcol/config/collector/gateway_config.yaml

@@ -107,9 +107,18 @@ exporters:
     access_token: "${SPLUNK_ACCESS_TOKEN}"
     realm: "${SPLUNK_REALM}"
     sync_host_metadata: true
-  # Debug
-  #logging:
-    #verbosity: detailed
+  # Logs
+  splunk_hec:
+    token: "${SPLUNK_HEC_TOKEN}"
+    endpoint: "${SPLUNK_HEC_URL}"
+    source: "otel"
+    sourcetype: "otel"
+    profiling_data_enabled: false
+  # Profiling
+  splunk_hec/profiling:
+    token: "${SPLUNK_ACCESS_TOKEN}"
+    endpoint: "${SPLUNK_INGEST_URL}/v1/log"
+    log_data_enabled: false
 
 service:
   extensions: [health_check, http_forwarder, zpages, memory_ballast]
@@ -129,7 +138,11 @@ service:
       receivers: [prometheus/internal]
       processors: [memory_limiter, batch, resourcedetection/internal]
       exporters: [signalfx/internal]
-    logs:
-      receivers: [otlp, signalfx]
+    logs/signalfx:
+      receivers: [signalfx]
       processors: [memory_limiter, batch]
       exporters: [signalfx]
+    logs:
+      receivers: [otlp]
+      processors: [memory_limiter, batch]
+      exporters: [splunk_hec, splunk_hec/profiling]

cmd/otelcol/config/collector/upstream_agent_config.yaml

@@ -137,6 +137,11 @@ exporters:
     endpoint: "${SPLUNK_HEC_URL}"
     source: "otel"
     sourcetype: "otel"
+    profiling_data_enabled: false
+  splunk_hec/profiling:
+    token: "${SPLUNK_ACCESS_TOKEN}"
+    endpoint: "https://ingest.${SPLUNK_REALM}.signalfx.com/v1/log"
+    log_data_enabled: false
   # Send to gateway
   otlp:
     endpoint: "${SPLUNK_GATEWAY_URL}:4317"
@@ -189,6 +194,6 @@ service:
       - batch
       - resourcedetection
       #- resource/add_environment
-      exporters: [splunk_hec]
+      exporters: [splunk_hec, splunk_hec/profiling]
       # Use instead when sending to gateway
       #exporters: [otlp]

tests/general/default_config_test.go

@@ -64,6 +64,18 @@ func TestDefaultGatewayConfig(t *testing.T) {
 				"realm":              "not.real",
 				"sync_host_metadata": true,
 			},
+			"splunk_hec": map[string]any{
+				"endpoint":               "https://ingest.not.real.signalfx.com/v1/log",
+				"source":                 "otel",
+				"sourcetype":             "otel",
+				"token":                  "<redacted>",
+				"profiling_data_enabled": false,
+			},
+			"splunk_hec/profiling": map[string]any{
+				"endpoint":         "https://ingest.not.real.signalfx.com/v1/log",
+				"token":            "<redacted>",
+				"log_data_enabled": false,
+			},
 		},
 		"extensions": map[string]any{
 			"health_check": map[string]any{
@@ -145,9 +157,14 @@ func TestDefaultGatewayConfig(t *testing.T) {
 			"extensions": []any{"health_check", "http_forwarder", "zpages", "memory_ballast"},
 			"pipelines": map[string]any{
 				"logs": map[string]any{
+					"exporters":  []any{"splunk_hec", "splunk_hec/profiling"},
+					"processors": []any{"memory_limiter", "batch"},
+					"receivers":  []any{"otlp"},
+				},
+				"logs/signalfx": map[string]any{
 					"exporters":  []any{"signalfx"},
 					"processors": []any{"memory_limiter", "batch"},
-					"receivers":  []any{"otlp", "signalfx"},
+					"receivers":  []any{"signalfx"},
 				},
 				"metrics": map[string]any{
 					"exporters":  []any{"signalfx"},
@@ -211,10 +228,16 @@ func TestDefaultAgentConfig(t *testing.T) {
 				"sync_host_metadata": true,
 			},
 			"splunk_hec": map[string]any{
-				"endpoint":   "https://ingest.not.real.signalfx.com/v1/log",
-				"source":     "otel",
-				"sourcetype": "otel",
-				"token":      "<redacted>",
+				"endpoint":               "https://ingest.not.real.signalfx.com/v1/log",
+				"source":                 "otel",
+				"sourcetype":             "otel",
+				"token":                  "<redacted>",
+				"profiling_data_enabled": false,
+			},
+			"splunk_hec/profiling": map[string]any{
+				"endpoint":         "https://ingest.not.real.signalfx.com/v1/log",
+				"token":            "<redacted>",
+				"log_data_enabled": false,
 			},
 		},
 		"extensions": map[string]any{
@@ -305,7 +328,7 @@ func TestDefaultAgentConfig(t *testing.T) {
 			"extensions": []any{"health_check", "http_forwarder", "zpages", "memory_ballast", "smartagent"},
 			"pipelines": map[string]any{
 				"logs": map[string]any{
-					"exporters":  []any{"splunk_hec"},
+					"exporters":  []any{"splunk_hec", "splunk_hec/profiling"},
 					"processors": []any{"memory_limiter", "batch", "resourcedetection"},
 					"receivers":  []any{"fluentforward", "otlp"}},
 				"logs/signalfx": map[string]any{