Tracing: Add additional options to align with the upstream exporter (p…

…rometheus#10276)

* Enhance configuration

Signed-off-by: Matej Gera <matejgera@gmail.com>

config/config.go

@@ -510,6 +510,8 @@ type TracingClientType string
 const (
 	TracingClientHTTP TracingClientType = "http"
 	TracingClientGRPC TracingClientType = "grpc"
+
+	GzipCompression = "gzip"
 )
 
 // UnmarshalYAML implements the yaml.Unmarshaler interface.
@@ -536,6 +538,9 @@ type TracingConfig struct {
 	SamplingFraction float64           `yaml:"sampling_fraction,omitempty"`
 	Insecure         bool              `yaml:"insecure,omitempty"`
 	TLSConfig        config.TLSConfig  `yaml:"tls_config,omitempty"`
+	Headers          map[string]string `yaml:"headers,omitempty"`
+	Compression      string            `yaml:"compression,omitempty"`
+	Timeout          model.Duration    `yaml:"timeout,omitempty"`
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -547,17 +552,25 @@ func (t *TracingConfig) SetDirectory(dir string) {
 func (t *TracingConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	*t = TracingConfig{
 		ClientType: TracingClientGRPC,
-		Insecure:   true,
 	}
 	type plain TracingConfig
 	if err := unmarshal((*plain)(t)); err != nil {
 		return err
 	}
 
+	if err := validateHeaders(t.Headers); err != nil {
+		return err
+	}
+
 	if t.Endpoint == "" {
 		return errors.New("tracing endpoint must be set")
 	}
 
+	if t.Compression != "" && t.Compression != GzipCompression {
+		return fmt.Errorf("invalid compression type %s provided, valid options: %s",
+			t.Compression, GzipCompression)
+	}
+
 	return nil
 }
 

config/config_test.go

@@ -988,9 +988,17 @@ var expectedConf = &Config{
 		},
 	},
 	TracingConfig: TracingConfig{
-		Endpoint:   "localhost:4317",
-		ClientType: TracingClientGRPC,
-		Insecure:   true,
+		Endpoint:    "localhost:4317",
+		ClientType:  TracingClientGRPC,
+		Insecure:    false,
+		Compression: "gzip",
+		Timeout:     model.Duration(5 * time.Second),
+		Headers:     map[string]string{"foo": "bar"},
+		TLSConfig: config.TLSConfig{
+			CertFile:           "testdata/valid_cert_file",
+			KeyFile:            "testdata/valid_key_file",
+			InsecureSkipVerify: true,
+		},
 	},
 }
 
@@ -1449,9 +1457,17 @@ var expectedErrors = []struct {
 		errMsg:   "relabel action cannot be empty",
 	},
 	{
-		filename: "tracing.bad.yml",
+		filename: "tracing_missing_endpoint.bad.yml",
 		errMsg:   "tracing endpoint must be set",
 	},
+	{
+		filename: "tracing_invalid_header.bad.yml",
+		errMsg:   "x-prometheus-remote-write-version is a reserved header. It must not be changed",
+	},
+	{
+		filename: "tracing_invalid_compression.bad.yml",
+		errMsg:   "invalid compression type foo provided, valid options: gzip",
+	},
 	{
 		filename: "uyuni_no_server.bad.yml",
 		errMsg:   "Uyuni SD configuration requires server host",

config/testdata/conf.good.yml

@@ -370,3 +370,11 @@ alerting:
 tracing:
   endpoint: "localhost:4317"
   client_type: "grpc"
+  headers:
+    foo: "bar"
+  timeout: 5s
+  compression: "gzip"
+  tls_config:
+    cert_file: valid_cert_file
+    key_file: valid_key_file
+    insecure_skip_verify: true

config/testdata/tracing_invalid_compression.bad.yml

@@ -0,0 +1,4 @@
+tracing:
+  sampling_fraction: 1
+  endpoint: "localhost:4317"
+  compression: foo

config/testdata/tracing_invalid_header.bad.yml

@@ -0,0 +1,5 @@
+tracing:
+  sampling_fraction: 1
+  endpoint: "localhost:4317"
+  headers:
+    "x-prometheus-remote-write-version": foo

docs/configuration/configuration.md

@@ -2668,14 +2668,24 @@ relabel_configs:
 # Client used to export the traces. Options are 'http' or 'grpc'.
 [ client_type: <string> | default = grpc ]
 
-# Endpoint to send the traces to.
+# Endpoint to send the traces to. Should be provided in format <host>:<port>.
 [ endpoint: <string> ]
 
 # Sets the probability a given trace will be sampled. Must be a float from 0 through 1.
 [ sampling_fraction: <float> | default = 0 ]
 
 # If disabled, the client will use a secure connection.
-[ insecure: <boolean> | default = true ]
+[ insecure: <boolean> | default = false ]
+
+# Key-value pairs to be used as headers associated with gRPC or HTTP requests.
+headers:
+  [ <string>: <string> ... ]
+
+# Compression key for supported compression types. Supported compression: gzip.
+[ compression: <string> ]
+
+# Maximum time the exporter will wait for each batch export.
+[ timeout: <duration> | default = 10s ]
 
 # TLS configuration.
 tls_config:

tracing/tracing.go

@@ -15,6 +15,7 @@ package tracing
 
 import (
 	"context"
+	"reflect"
 	"time"
 
 	"github.com/go-kit/log"
@@ -72,7 +73,7 @@ func (m *Manager) ApplyConfig(cfg *config.Config) error {
 	// set, we have to restart the manager to make sure that new TLS
 	// certificates are picked up.
 	var blankTLSConfig config_util.TLSConfig
-	if m.config == cfg.TracingConfig && m.config.TLSConfig == blankTLSConfig {
+	if reflect.DeepEqual(m.config, cfg.TracingConfig) && m.config.TLSConfig == blankTLSConfig {
 		return nil
 	}
 
@@ -183,25 +184,52 @@ func getClient(tracingCfg config.TracingConfig) (otlptrace.Client, error) {
 		if tracingCfg.Insecure {
 			opts = append(opts, otlptracegrpc.WithInsecure())
 		}
+		if tracingCfg.Compression != "" {
+			opts = append(opts, otlptracegrpc.WithCompressor(tracingCfg.Compression))
+		}
+		if len(tracingCfg.Headers) != 0 {
+			opts = append(opts, otlptracegrpc.WithHeaders(tracingCfg.Headers))
+		}
+		if tracingCfg.Timeout != 0 {
+			opts = append(opts, otlptracegrpc.WithTimeout(time.Duration(tracingCfg.Timeout)))
+		}
 
-		tlsConf, err := config_util.NewTLSConfig(&tracingCfg.TLSConfig)
-		if err != nil {
-			return nil, err
+		// Configure TLS only if config is not empty.
+		var blankTLSConfig config_util.TLSConfig
+		if tracingCfg.TLSConfig != blankTLSConfig {
+			tlsConf, err := config_util.NewTLSConfig(&tracingCfg.TLSConfig)
+			if err != nil {
+				return nil, err
+			}
+			opts = append(opts, otlptracegrpc.WithTLSCredentials(credentials.NewTLS(tlsConf)))
 		}
-		opts = append(opts, otlptracegrpc.WithTLSCredentials(credentials.NewTLS(tlsConf)))
 
 		client = otlptracegrpc.NewClient(opts...)
 	case config.TracingClientHTTP:
 		opts := []otlptracehttp.Option{otlptracehttp.WithEndpoint(tracingCfg.Endpoint)}
 		if tracingCfg.Insecure {
 			opts = append(opts, otlptracehttp.WithInsecure())
 		}
+		// Currently, OTEL supports only gzip compression.
+		if tracingCfg.Compression == config.GzipCompression {
+			opts = append(opts, otlptracehttp.WithCompression(otlptracehttp.GzipCompression))
+		}
+		if len(tracingCfg.Headers) != 0 {
+			opts = append(opts, otlptracehttp.WithHeaders(tracingCfg.Headers))
+		}
+		if tracingCfg.Timeout != 0 {
+			opts = append(opts, otlptracehttp.WithTimeout(time.Duration(tracingCfg.Timeout)))
+		}
 
-		tlsConf, err := config_util.NewTLSConfig(&tracingCfg.TLSConfig)
-		if err != nil {
-			return nil, err
+		// Configure TLS only if config is not empty.
+		var blankTLSConfig config_util.TLSConfig
+		if tracingCfg.TLSConfig != blankTLSConfig {
+			tlsConf, err := config_util.NewTLSConfig(&tracingCfg.TLSConfig)
+			if err != nil {
+				return nil, err
+			}
+			opts = append(opts, otlptracehttp.WithTLSClientConfig(tlsConf))
 		}
-		opts = append(opts, otlptracehttp.WithTLSClientConfig(tlsConf))
 
 		client = otlptracehttp.NewClient(opts...)
 	}

tracing/tracing_test.go

@@ -46,6 +46,7 @@ func TestReinstallingTracerProvider(t *testing.T) {
 		TracingConfig: config.TracingConfig{
 			Endpoint:   "localhost:1234",
 			ClientType: config.TracingClientGRPC,
+			Headers:    map[string]string{"foo": "bar"},
 		},
 	}
 
@@ -60,11 +61,18 @@ func TestReinstallingTracerProvider(t *testing.T) {
 		TracingConfig: config.TracingConfig{
 			Endpoint:   "localhost:1234",
 			ClientType: config.TracingClientHTTP,
+			Headers:    map[string]string{"bar": "foo"},
 		},
 	}
 
 	require.NoError(t, m.ApplyConfig(&cfg2))
 	require.NotEqual(t, tpFirstConfig, otel.GetTracerProvider())
+	tpSecondConfig := otel.GetTracerProvider()
+
+	// Setting previously unset option should reinstall provider.
+	cfg2.TracingConfig.Compression = "gzip"
+	require.NoError(t, m.ApplyConfig(&cfg2))
+	require.NotEqual(t, tpSecondConfig, otel.GetTracerProvider())
 }
 
 func TestReinstallingTracerProviderWithTLS(t *testing.T) {