feat(hpsf): add ability to produce otlp exporter to honeycomb

Makefile

@@ -21,4 +21,4 @@ test_all: test_results
 	$(call GOTESTCMD,$@) -tags all --timeout 60s -v ./...
 
 test_results:
-	@mkdir -p test_results
+	@mkdir -p test_results

cmd/hpsf/main.go

@@ -173,7 +173,7 @@ func main() {
 		}
 		cfg, err := tr.GenerateConfig(hpsf, ct, userdata)
 		if err != nil {
-			log.Fatalf("error translating refinery config: %v", err)
+			log.Fatalf("error translating config: %v", err)
 		}
 		data, _, err := cfg.RenderYAML()
 		if err != nil {

go.mod

@@ -4,7 +4,12 @@ go 1.23.0
 
 require (
 	github.com/jessevdk/go-flags v1.6.1
+	github.com/stretchr/testify v1.10.0
 	gopkg.in/yaml.v3 v3.0.1
 )
 
-require golang.org/x/sys v0.21.0 // indirect
+require (
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	golang.org/x/sys v0.21.0 // indirect
+)

go.sum

@@ -1,5 +1,11 @@
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/jessevdk/go-flags v1.6.1 h1:Cvu5U8UGrLay1rZfv/zP7iLpSHGUZ/Ou68T0iX1bBK4=
 github.com/jessevdk/go-flags v1.6.1/go.mod h1:Mk8T1hIAWpOiJiHa9rJASDK2UGWji0EuPGBnNLMooyc=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
+github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws=
 golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=

pkg/config/collector.go

@@ -0,0 +1,24 @@
+package config
+
+import (
+	"github.com/honeycombio/hpsf/pkg/hpsf"
+	"github.com/honeycombio/hpsf/pkg/yaml"
+)
+
+// This base component is used to make sure that the config will be valid
+// even if it stands alone. This is likely to be a temporary solution until we have a
+// database of components.
+type CollectorBaseComponent struct {
+	Component hpsf.Component
+}
+
+var _ Component = CollectorBaseComponent{}
+
+func (c CollectorBaseComponent) GenerateConfig(ct Type, userdata map[string]any) (yaml.DottedConfig, error) {
+	return yaml.DottedConfig{
+		"processors": map[string]any{},
+		"receivers":  map[string]any{},
+		"extensions": map[string]any{},
+		"service":    map[string]any{},
+	}, nil
+}

pkg/config/collector_test.go

@@ -0,0 +1,22 @@
+package config
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/honeycombio/hpsf/pkg/yaml"
+)
+
+func TestCollectorBaseComponent(t *testing.T) {
+	c := CollectorBaseComponent{}
+	got, err := c.GenerateConfig(CollectorConfigType, map[string]any{})
+	want := yaml.DottedConfig{
+		"processors": map[string]interface{}{},
+		"receivers":  map[string]interface{}{},
+		"extensions": map[string]interface{}{},
+		"service":    map[string]interface{}{},
+	}
+	require.NoError(t, err)
+	require.Equal(t, want, got)
+}

pkg/config/components/honeycombExporter.yaml

@@ -23,6 +23,14 @@ properties:
     type: string
     validations: [nonblank, url]
     default: https://api.honeycomb.io
+  - name: Compression
+    summary: The compression to use when sending data to Honeycomb
+    description: |
+      The compression to use when sending data to Honeycomb.
+      This is normally gzip, but can be overridden.
+    type: string
+    validations: [nonblank]
+    default: gzip
 templates:
   - kind: refinery_config
     name: HoneycombExporter_RefineryConfig
@@ -31,5 +39,17 @@ templates:
       - key: Network.HoneycombAPI
         value: "{{ firstNonblank .User.APIEndpoint .Props.APIEndpoint.Default }}"
       - key: AccessKeys.SendKey
-        value: "{{ .User.APIKey }}"
-
+        value: "{{ .User.API_Key }}"
+  - kind: collector_config
+    name: HoneycombExporter_CollectorConfig
+    format: dottedConfig
+    data:
+      - key: exporters.otlphttp.endpoint
+        value: "{{ firstNonblank .User.API_URL .Props.API_URL.Default }}"
+      - key: exporters.otlphttp.compression
+        value: "{{ firstNonblank .User.Compression .Props.Compression.Default }}"
+      # TODO: headers must be an array
+      - key: exporters.otlphttp.headers
+        value:
+          - name: x-honeycomb-team
+            value: "{{ .User.API_Key }}"

pkg/config/loadComponents_test.go

@@ -1,7 +1,11 @@
 package config
 
 import (
+	"os"
+	"path"
 	"testing"
+
+	"github.com/stretchr/testify/require"
 )
 
 func TestLoadTemplateComponents(t *testing.T) {
@@ -13,3 +17,43 @@ func TestLoadTemplateComponents(t *testing.T) {
 		t.Errorf("LoadTemplateComponents() = %v, want non-empty", got)
 	}
 }
+
+func TestTemplateComponents(t *testing.T) {
+	components, err := LoadTemplateComponents()
+	require.NoError(t, err)
+	// for test component type
+	tests := []struct {
+		name       string
+		kind       string
+		cType      Type
+		wantOutput string
+	}{
+		{
+			name:       "HoneycombExporter to refinery config",
+			kind:       "HoneycombExporter",
+			cType:      RefineryConfigType,
+			wantOutput: "HoneycombExporter_output_refinery_config.yaml",
+		},
+		{
+			name:       "HoneycombExporter to collector config",
+			kind:       "HoneycombExporter",
+			cType:      CollectorConfigType,
+			wantOutput: "HoneycombExporter_output_collector_config.yaml",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			want, err := os.ReadFile(path.Join("testdata", tt.wantOutput))
+			require.NoError(t, err)
+			c, ok := components[tt.kind]
+			require.True(t, ok)
+			conf, err := c.GenerateConfig(tt.cType, map[string]any{
+				"API_Key": "test",
+			})
+			require.NoError(t, err)
+			got, _, err := conf.RenderYAML()
+			require.NoError(t, err)
+			require.Equal(t, string(want), string(got))
+		})
+	}
+}

pkg/config/refinery.go

@@ -75,10 +75,6 @@ type DeterministicSampler struct {
 var _ Component = DeterministicSampler{}
 
 func (c DeterministicSampler) GenerateConfig(ct Type, userdata map[string]any) (yaml.DottedConfig, error) {
-	if ct != RefineryRulesType {
-		return nil, nil
-	}
-
 	if c.Component.Properties == nil {
 		return nil, nil
 	}
@@ -95,8 +91,18 @@ func (c DeterministicSampler) GenerateConfig(ct Type, userdata map[string]any) (
 	}
 	e := yaml.AsString(env.Value)
 
-	return yaml.DottedConfig{
-		fmt.Sprintf("Samplers.%s.DeterministicSampler.SampleRate", e): r,
-		"Samplers." + e + ".DeterministicSampler.SampleRate":          r,
-	}, nil
+	switch ct {
+	case CollectorConfigType:
+		return yaml.DottedConfig{
+			"probabilistic_sampler": map[string]interface{}{
+				"sampling_percentage": r,
+			},
+		}, nil
+	default:
+		return yaml.DottedConfig{
+			fmt.Sprintf("Samplers.%s.DeterministicSampler.SampleRate", e): r,
+			"Samplers." + e + ".DeterministicSampler.SampleRate":          r,
+		}, nil
+	}
+
 }

pkg/config/templateComponent.go

@@ -82,7 +82,7 @@ func (t *TemplateComponent) Props() map[string]TemplateProperty {
 
 type dottedConfigTemplateKV struct {
 	key   string
-	value string
+	value any
 }
 
 type dottedConfigTemplate []dottedConfigTemplateKV
@@ -94,7 +94,8 @@ func buildDottedConfigTemplate(data []any) (dottedConfigTemplate, error) {
 		if !ok {
 			return nil, fmt.Errorf("expected map, got %T", v)
 		}
-		var sk, sv string
+		var sk string
+		var sv any
 		if mk, ok := m["key"]; !ok {
 			return nil, fmt.Errorf("missing key in template data")
 		} else {
@@ -106,10 +107,15 @@ func buildDottedConfigTemplate(data []any) (dottedConfigTemplate, error) {
 		if _, ok := m["value"]; !ok {
 			return nil, fmt.Errorf("missing value in template data")
 		} else {
-			if _, ok := m["value"].(string); !ok {
-				return nil, fmt.Errorf("expected string for v, got %T", m["value"])
+			switch val := m["value"].(type) {
+			case string:
+				sv = val
+			case []any:
+				sv = val
+			default:
+				return nil, fmt.Errorf("unexpected type for v, got %T", m["value"])
 			}
-			sv = m["value"].(string)
+
 		}
 		d = append(d, dottedConfigTemplateKV{key: sk, value: sv})
 	}
@@ -137,15 +143,11 @@ func (t *TemplateComponent) GenerateConfig(cfgType Type, userdata map[string]any
 	return nil, nil
 }
 
-func (t *TemplateComponent) applyTemplate(tmplText string, userdata map[string]any) (string, error) {
-	if tmplText == "" || !strings.Contains(tmplText, "{{") {
-		return tmplText, nil
-	}
-	tmpl, err := template.New("template").Funcs(helpers()).Parse(tmplText)
+func (t *TemplateComponent) expandTemplateVariable(variable string, userdata map[string]any) (string, error) {
+	tmpl, err := template.New("template").Funcs(helpers()).Parse(variable)
 	if err != nil {
 		return "", fmt.Errorf("error %w parsing template", err)
 	}
-
 	t.User = userdata
 	var b bytes.Buffer
 	err = tmpl.Execute(&b, t)
@@ -155,12 +157,39 @@ func (t *TemplateComponent) applyTemplate(tmplText string, userdata map[string]a
 	return b.String(), nil
 }
 
+func (t *TemplateComponent) applyTemplate(tmplVal any, userdata map[string]any) (any, error) {
+	switch k := tmplVal.(type) {
+	case string:
+		if tmplVal == "" || !strings.Contains(k, "{{") {
+			return k, nil
+		}
+		return t.expandTemplateVariable(k, userdata)
+	case []any:
+		for _, s := range k {
+			stringMap, ok := s.(map[string]any)
+			if !ok {
+				return "", fmt.Errorf("invalid templated variable type %T", s)
+			}
+			for k, v := range stringMap {
+				expandedVariable, err := t.applyTemplate(v, userdata)
+				if err != nil {
+					return "", err
+				}
+				stringMap[k] = expandedVariable
+			}
+		}
+		return k, nil
+	default:
+		return "", fmt.Errorf("invalid templated variable type %T", k)
+	}
+}
+
 func (t *TemplateComponent) generateDottedConfig(dct dottedConfigTemplate, userdata map[string]any) (yaml.DottedConfig, error) {
 	// we have to fill in the template with the default values
 	// and the values from the properties
 	config := make(yaml.DottedConfig)
 	for _, kv := range dct {
-		key, err := t.applyTemplate(kv.key, userdata)
+		key, err := t.expandTemplateVariable(kv.key, userdata)
 		if err != nil {
 			return nil, err
 		}

pkg/config/testdata/HoneycombExporter_output_collector_config.yaml

@@ -0,0 +1,10 @@
+API_Key: ${HONEYCOMB_TRACES_APIKEY}
+API_URL: https://api.honeycomb.io
+Compression: gzip
+exporters:
+    otlphttp:
+        compression: gzip
+        endpoint: https://api.honeycomb.io
+        headers:
+            - name: x-honeycomb-team
+              value: test

pkg/config/testdata/HoneycombExporter_output_refinery_config.yaml

@@ -0,0 +1,7 @@
+API_Key: ${HONEYCOMB_TRACES_APIKEY}
+API_URL: https://api.honeycomb.io
+AccessKeys:
+    SendKey: test
+Compression: gzip
+Network:
+    HoneycombAPI: https://api.honeycomb.io

pkg/translator/translator.go

@@ -44,7 +44,14 @@ func (t *Translator) GenerateConfig(h *hpsf.HPSF, ct config.Type, userdata map[s
 	// Add base component to the config so we can make a valid config
 	// this may be temporary until we have a database of components
 	dummy := hpsf.Component{Name: "dummy", Kind: "dummy"}
-	base := config.RefineryBaseComponent{Component: dummy}
+	var base config.Component
+	switch ct {
+	case config.RefineryConfigType, config.RefineryRulesType:
+		base = config.RefineryBaseComponent{Component: dummy}
+	case config.CollectorConfigType:
+		base = config.CollectorBaseComponent{Component: dummy}
+	}
+
 	cfg, err := base.GenerateConfig(ct, userdata)
 	if err != nil {
 		return nil, err