Promtail: Add basic tracing support (#7414)

We've been using Promtail's HTTP/gRPC server-based targets with real
workloads for some weeks now. The server that these spin out is based on
[weaveworks commons](https://github.com/weaveworks/common) HTTP/gRPC
servers. These include by default some middlewares that if enabled,
inject tracing support.

We found this useful when tracking down request handling time, or even
more, if the caller also support Jaeger tracing support,
cross-correlated requests between the upstream server and Promtail's.

This PR adds support for enabling tracing the same way Loki does. At the
moment, this will just generate traces for the before-mentioned targets.

In the future, the quality and completeness of tracing support can grow
organically to enrich spans.

CHANGELOG.md

@@ -54,6 +54,7 @@
 * [6828](https://github.com/grafana/loki/pull/6828) **alexandre1984rj** Add the BotScore and BotScoreSrc fields once the Cloudflare API returns those two fields on the list of all available log fields.
 * [6656](https://github.com/grafana/loki/pull/6656) **carlospeon**: Allow promtail to add matches to the journal reader
 * [7401](https://github.com/grafana/loki/pull/7401) **thepalbi**: Add timeout to GCP Logs push target
+* [7414](https://github.com/grafana/loki/pull/7414) **thepalbi**: Add basic tracing support
 
 ##### Fixes
 * [6766](https://github.com/grafana/loki/pull/6766) **kavirajk**: fix(logql): Make `LabelSampleExtractor` ignore processing the line if it doesn't contain that specific label. Fixes unwrap behavior explained in the issue https://github.com/grafana/loki/issues/6713

clients/cmd/promtail/main.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"reflect"
 	"sync"
+
 	// embed time zone data
 	_ "time/tzdata"
 
@@ -16,6 +17,7 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/version"
 	"github.com/weaveworks/common/logging"
+	"github.com/weaveworks/common/tracing"
 
 	"github.com/grafana/loki/clients/pkg/logentry/stages"
 	"github.com/grafana/loki/clients/pkg/promtail"
@@ -133,6 +135,22 @@ func main() {
 		}
 	}
 
+	if config.Tracing.Enabled {
+		// Setting the environment variable JAEGER_AGENT_HOST enables tracing
+		trace, err := tracing.NewFromEnv("promtail")
+		if err != nil {
+			level.Error(util_log.Logger).Log("msg", "error in initializing tracing. tracing will not be enabled", "err", err)
+		}
+
+		defer func() {
+			if trace != nil {
+				if err := trace.Close(); err != nil {
+					level.Error(util_log.Logger).Log("msg", "error closing tracing", "err", err)
+				}
+			}
+		}()
+	}
+
 	clientMetrics := client.NewMetrics(prometheus.DefaultRegisterer, config.Config.Options.StreamLagLabels)
 	newConfigFunc := func() (*promtail_config.Config, error) {
 		mtx.Lock()

clients/pkg/promtail/config/config.go

@@ -8,7 +8,9 @@ import (
 	"github.com/go-kit/log/level"
 	dskit_flagext "github.com/grafana/dskit/flagext"
 
-	yaml "gopkg.in/yaml.v2"
+	"github.com/grafana/loki/pkg/tracing"
+
+	"gopkg.in/yaml.v2"
 
 	"github.com/grafana/loki/clients/pkg/promtail/client"
 	"github.com/grafana/loki/clients/pkg/promtail/limit"
@@ -36,6 +38,7 @@ type Config struct {
 	TargetConfig    file.Config           `yaml:"target_config,omitempty"`
 	LimitsConfig    limit.Config          `yaml:"limits_config,omitempty"`
 	Options         Options               `yaml:"options,omitempty"`
+	Tracing         tracing.Config        `yaml:"tracing"`
 }
 
 // RegisterFlags with prefix registers flags where every name is prefixed by
@@ -46,6 +49,7 @@ func (c *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) {
 	c.PositionsConfig.RegisterFlagsWithPrefix(prefix, f)
 	c.TargetConfig.RegisterFlagsWithPrefix(prefix, f)
 	c.LimitsConfig.RegisterFlagsWithPrefix(prefix, f)
+	c.Tracing.RegisterFlagsWithPrefix(prefix, f)
 }
 
 // RegisterFlags registers flags.

docs/sources/clients/promtail/configuration.md

@@ -108,6 +108,9 @@ scrape_configs:
 
 # Configures additional promtail configurations.
 [options: <options_config>]
+
+# Configures tracing support
+[tracing: <tracing_config>]
 ```
 
 ## server
@@ -1935,6 +1938,15 @@ sync_period: "10s"
 [stream_lag_labels: <string> | default = "filename"]
 ```
 
+## tracing_config
+
+The `tracing` block configures tracing for Jaeger. Currently, limited to configuration per [environment variables](https://www.jaegertracing.io/docs/1.16/client-features/) only.
+
+```yaml
+# When true, 
+[enabled: <boolean> | default = false]
+```
+
 ## Example Docker Config
 
 It's fairly difficult to tail Docker files on a standalone machine because they are in different locations for every OS.  We recommend the [Docker logging driver](../../docker-driver/) for local Docker installs or Docker Compose.

pkg/tracing/config.go

@@ -11,3 +11,7 @@ type Config struct {
 func (cfg *Config) RegisterFlags(f *flag.FlagSet) {
 	f.BoolVar(&cfg.Enabled, "tracing.enabled", true, "Set to false to disable tracing.")
 }
+
+func (cfg *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) {
+	f.BoolVar(&cfg.Enabled, prefix+"tracing.enabled", true, "Set to false to disable tracing.")
+}