honeycomb: adjust the sample rate using an attribute on the span (#1162)

Allows setting Honeycomb's sample rate using an attribute on span.

This allows either a custom processor (my use case) or the application sdks to adjust the sample rate value for honeycomb. Once the sampling otel is finalized this implementation could be adjusted to use that instead of a custom attribute.

exporter/honeycombexporter/README.md

@@ -8,6 +8,7 @@ The following configuration options are supported:
 * `dataset` (Required): The Honeycomb dataset that you want to send events to.
 * `api_url` (Optional): You can set the hostname to send events to. Useful for debugging, defaults to `https://api.honeycomb.io`
 * `sample_rate` (Optional): Constant sample rate. Can be used to send 1 / x events to Honeycomb. Defaults to 1 (always sample).
+* `sample_rate_attribute` (Optional): The name of an attribute that contains the sample_rate for each span. If the attribute is on the span, it takes precedence over the static sample_rate configuration
 * `debug` (Optional): Set this to true to get debug logs from the honeycomb SDK. Defaults to false.
 Example:
 
@@ -18,5 +19,6 @@ exporters:
     dataset: "my-dataset"
     api_url: "https://api.testhost.io"
     sample_rate: 25
+    sample_rate_attribute: "hny.sample_rate"
     debug: true
 ```

exporter/honeycombexporter/config.go

@@ -28,6 +28,9 @@ type Config struct {
 	// meaning no sampling. If you want to send one event out of every 250
 	// times Send() is called, you would specify 250 here.
 	SampleRate uint `mapstructure:"sample_rate"`
+	// The name of an attribute that contains the sample_rate for each span.
+	// If the attribute is on the span, it takes precedence over the static sample_rate configuration
+	SampleRateAttribute string `mapstructure:"sample_rate_attribute"`
 	// Debug enables more verbose logging from the Honeycomb SDK. It defaults to false.
 	Debug bool `mapstructure:"debug"`
 }

exporter/honeycombexporter/config_test.go

@@ -38,7 +38,7 @@ func TestLoadConfig(t *testing.T) {
 	require.NoError(t, err)
 	require.NotNil(t, cfg)
 
-	assert.Equal(t, len(cfg.Exporters), 2)
+	assert.Equal(t, len(cfg.Exporters), 3)
 
 	r0 := cfg.Exporters["honeycomb"]
 	assert.Equal(t, r0, factory.CreateDefaultConfig())
@@ -51,4 +51,12 @@ func TestLoadConfig(t *testing.T) {
 		APIURL:           "https://api.testhost.io",
 		SampleRate:       1,
 	})
+
+	r2 := cfg.Exporters["honeycomb/sample_rate"].(*Config)
+	assert.Equal(t, r2, &Config{
+		ExporterSettings:    configmodels.ExporterSettings{TypeVal: configmodels.Type(typeStr), NameVal: "honeycomb/sample_rate"},
+		APIURL:              "https://api.honeycomb.io",
+		SampleRate:          5,
+		SampleRateAttribute: "custom.sample_rate",
+	})
 }

exporter/honeycombexporter/factory.go

@@ -41,11 +41,12 @@ func createDefaultConfig() configmodels.Exporter {
 			TypeVal: configmodels.Type(typeStr),
 			NameVal: typeStr,
 		},
-		APIKey:     "",
-		Dataset:    "",
-		APIURL:     "https://api.honeycomb.io",
-		SampleRate: 1,
-		Debug:      false,
+		APIKey:              "",
+		Dataset:             "",
+		APIURL:              "https://api.honeycomb.io",
+		SampleRate:          1,
+		SampleRateAttribute: "",
+		Debug:               false,
 	}
 }
 

exporter/honeycombexporter/honeycomb.go

@@ -38,9 +38,10 @@ const (
 
 // honeycombExporter is the object that sends events to honeycomb.
 type honeycombExporter struct {
-	builder *libhoney.Builder
-	onError func(error)
-	logger  *zap.Logger
+	builder             *libhoney.Builder
+	onError             func(error)
+	logger              *zap.Logger
+	sampleRateAttribute string
 }
 
 // event represents a honeycomb event.
@@ -105,6 +106,7 @@ func newHoneycombTraceExporter(cfg *Config, logger *zap.Logger) (component.Trace
 		onError: func(err error) {
 			logger.Warn(err.Error())
 		},
+		sampleRateAttribute: cfg.SampleRateAttribute,
 	}
 
 	return exporterhelper.NewTraceExporter(
@@ -159,6 +161,8 @@ func (e *honeycombExporter) pushTraceData(ctx context.Context, td pdata.Traces)
 				for k, v := range attrs {
 					ev.AddField(k, v)
 				}
+
+				e.addSampleRate(ev, attrs)
 			}
 
 			ev.Timestamp = timestampToTime(span.GetStartTime())
@@ -217,6 +221,8 @@ func (e *honeycombExporter) sendSpanLinks(span *tracepb.Span) {
 		for k, v := range attrs {
 			ev.AddField(k, v)
 		}
+		e.addSampleRate(ev, attrs)
+
 		if err := ev.SendPresampled(); err != nil {
 			e.onError(err)
 		}
@@ -256,6 +262,8 @@ func (e *honeycombExporter) sendMessageEvents(td consumerdata.TraceData, span *t
 		for k, v := range attrs {
 			ev.AddField(k, v)
 		}
+		e.addSampleRate(ev, attrs)
+
 		ev.Timestamp = ts
 		ev.Add(spanEvent{
 			Name:           name,
@@ -298,3 +306,16 @@ func (e *honeycombExporter) RunErrorLogger(ctx context.Context, responses chan t
 		}
 	}
 }
+
+func (e *honeycombExporter) addSampleRate(event *libhoney.Event, attrs map[string]interface{}) {
+	if e.sampleRateAttribute != "" && attrs != nil {
+		if value, ok := attrs[e.sampleRateAttribute]; ok {
+			switch v := value.(type) {
+			case int64:
+				event.SampleRate = uint(v)
+			default:
+				return
+			}
+		}
+	}
+}

exporter/honeycombexporter/honeycomb_test.go

@@ -38,7 +38,8 @@ import (
 )
 
 type honeycombData struct {
-	Data map[string]interface{} `json:"data"`
+	Data       map[string]interface{} `json:"data"`
+	SampleRate int                    `json:"samplerate"`
 }
 
 func testingServer(callback func(data []honeycombData)) *httptest.Server {
@@ -66,22 +67,17 @@ func testingServer(callback func(data []honeycombData)) *httptest.Server {
 	}))
 }
 
-func testTraceExporter(td pdata.Traces, t *testing.T) []honeycombData {
+func testTraceExporter(td pdata.Traces, t *testing.T, cfg *Config) []honeycombData {
 	var got []honeycombData
 	server := testingServer(func(data []honeycombData) {
 		got = append(got, data...)
 	})
 	defer server.Close()
-	cfg := Config{
-		APIKey:     "test",
-		Dataset:    "test",
-		APIURL:     server.URL,
-		Debug:      false,
-		SampleRate: 1,
-	}
+
+	cfg.APIURL = server.URL
 
 	params := component.ExporterCreateParams{Logger: zap.NewNop()}
-	exporter, err := createTraceExporter(context.Background(), params, &cfg)
+	exporter, err := createTraceExporter(context.Background(), params, cfg)
 	require.NoError(t, err)
 
 	ctx := context.Background()
@@ -92,6 +88,16 @@ func testTraceExporter(td pdata.Traces, t *testing.T) []honeycombData {
 	return got
 }
 
+func baseConfig() *Config {
+	return &Config{
+		APIKey:              "test",
+		Dataset:             "test",
+		Debug:               false,
+		SampleRate:          1,
+		SampleRateAttribute: "",
+	}
+}
+
 func TestExporter(t *testing.T) {
 	td := consumerdata.TraceData{
 		Node: &commonpb.Node{
@@ -190,7 +196,7 @@ func TestExporter(t *testing.T) {
 			},
 		},
 	}
-	got := testTraceExporter(internaldata.OCToTraceData(td), t)
+	got := testTraceExporter(internaldata.OCToTraceData(td), t, baseConfig())
 	want := []honeycombData{
 		{
 			Data: map[string]interface{}{
@@ -281,6 +287,126 @@ func TestExporter(t *testing.T) {
 	}
 }
 
+func TestSampleRateAttribute(t *testing.T) {
+	td := consumerdata.TraceData{
+		Node: nil,
+		Spans: []*tracepb.Span{
+			{
+				TraceId:                 []byte{0x01},
+				SpanId:                  []byte{0x02},
+				Name:                    &tracepb.TruncatableString{Value: "root"},
+				Kind:                    tracepb.Span_SERVER,
+				SameProcessAsParentSpan: &wrapperspb.BoolValue{Value: true},
+				Attributes: &tracepb.Span_Attributes{
+					AttributeMap: map[string]*tracepb.AttributeValue{
+						"some_attribute": {
+							Value: &tracepb.AttributeValue_StringValue{
+								StringValue: &tracepb.TruncatableString{Value: "A value"},
+							},
+						},
+						"hc.sample.rate": {
+							Value: &tracepb.AttributeValue_IntValue{
+								IntValue: 13,
+							},
+						},
+					},
+				},
+			},
+			{
+				TraceId:                 []byte{0x01},
+				SpanId:                  []byte{0x02},
+				Name:                    &tracepb.TruncatableString{Value: "root"},
+				Kind:                    tracepb.Span_SERVER,
+				SameProcessAsParentSpan: &wrapperspb.BoolValue{Value: true},
+				Attributes: &tracepb.Span_Attributes{
+					AttributeMap: map[string]*tracepb.AttributeValue{
+						"no_sample_rate": {
+							Value: &tracepb.AttributeValue_StringValue{
+								StringValue: &tracepb.TruncatableString{Value: "gets_default"},
+							},
+						},
+					},
+				},
+			},
+			{
+				TraceId:                 []byte{0x01},
+				SpanId:                  []byte{0x02},
+				Name:                    &tracepb.TruncatableString{Value: "root"},
+				Kind:                    tracepb.Span_SERVER,
+				SameProcessAsParentSpan: &wrapperspb.BoolValue{Value: true},
+				Attributes: &tracepb.Span_Attributes{
+					AttributeMap: map[string]*tracepb.AttributeValue{
+						"hc.sample.rate": {
+							Value: &tracepb.AttributeValue_StringValue{
+								StringValue: &tracepb.TruncatableString{Value: "wrong_type"},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	cfg := baseConfig()
+	cfg.SampleRate = 2 // default sample rate
+	cfg.SampleRateAttribute = "hc.sample.rate"
+
+	got := testTraceExporter(internaldata.OCToTraceData(td), t, cfg)
+
+	want := []honeycombData{
+		{
+			SampleRate: 13,
+			Data: map[string]interface{}{
+				"duration_ms":                            float64(0),
+				"has_remote_parent":                      false,
+				"hc.sample.rate":                         float64(13),
+				"name":                                   "root",
+				"source_format":                          "otlp_trace",
+				"status.code":                            float64(0),
+				"status.message":                         "OK",
+				"trace.span_id":                          "02",
+				"trace.trace_id":                         "01",
+				"opencensus.same_process_as_parent_span": true,
+				"some_attribute":                         "A value",
+			},
+		},
+		{
+			SampleRate: 2,
+			Data: map[string]interface{}{
+				"duration_ms":                            float64(0),
+				"has_remote_parent":                      false,
+				"name":                                   "root",
+				"source_format":                          "otlp_trace",
+				"status.code":                            float64(0),
+				"status.message":                         "OK",
+				"trace.span_id":                          "02",
+				"trace.trace_id":                         "01",
+				"opencensus.same_process_as_parent_span": true,
+				"no_sample_rate":                         "gets_default",
+			},
+		},
+		{
+			SampleRate: 2,
+			Data: map[string]interface{}{
+				"duration_ms":                            float64(0),
+				"has_remote_parent":                      false,
+				"hc.sample.rate":                         "wrong_type",
+				"name":                                   "root",
+				"source_format":                          "otlp_trace",
+				"status.code":                            float64(0),
+				"status.message":                         "OK",
+				"trace.span_id":                          "02",
+				"trace.trace_id":                         "01",
+				"opencensus.same_process_as_parent_span": true,
+			},
+		},
+	}
+
+	if diff := cmp.Diff(want, got); diff != "" {
+		t.Errorf("otel span: (-want +got):\n%s", diff)
+	}
+}
+
 func TestEmptyNode(t *testing.T) {
 	td := consumerdata.TraceData{
 		Node: nil,
@@ -295,7 +421,7 @@ func TestEmptyNode(t *testing.T) {
 		},
 	}
 
-	got := testTraceExporter(internaldata.OCToTraceData(td), t)
+	got := testTraceExporter(internaldata.OCToTraceData(td), t, baseConfig())
 
 	want := []honeycombData{
 		{
@@ -424,7 +550,7 @@ func TestNode(t *testing.T) {
 				},
 			}
 
-			got := testTraceExporter(internaldata.OCToTraceData(td), t)
+			got := testTraceExporter(internaldata.OCToTraceData(td), t, baseConfig())
 
 			want := []honeycombData{
 				{

exporter/honeycombexporter/testdata/config.yaml

@@ -10,6 +10,9 @@ exporters:
     api_key: "test-apikey"
     dataset: "test-dataset"
     api_url: "https://api.testhost.io"
+  honeycomb/sample_rate:
+    sample_rate: 5
+    sample_rate_attribute: "custom.sample_rate"
 
 service:
   pipelines: