Add support for opentelemetry tracing

.github/workflows/ci.yml

@@ -386,11 +386,36 @@ jobs:
       - name: Build example bpf probes
         run: make -j $(nproc) -C examples build
 
+  build-tracing-demos-x86_64:
+    name: Build tracing demos (x86_64)
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-go@v5
+        with:
+          go-version: ^1.22
+
+      - name: Build
+        run: make -j $(nproc) tracing-demos
+
+      - name: Extract tracing-demos.x86_64.tar.gz
+        run: |
+          tar -cvzf tracing-demos.x86_64.tar.gz tracing/demos/*/demo
+
+      - name: Upload tracing-demos.x86_64.tar.gz
+        uses: actions/upload-artifact@v3
+        with:
+          name: tracing-demos.x86_64.tar.gz
+          path: tracing-demos.x86_64.tar.gz
+          if-no-files-found: error
+
   check-configs-x86_64:
     name: Check examples
     runs-on: ubuntu-22.04
     needs:
       - build-ebpf-exporter-docker-x86_64
+      - build-tracing-demos-x86_64
     steps:
       - uses: actions/checkout@v4
 
@@ -402,6 +427,14 @@ jobs:
       - name: Extract ebpf_exporter_with_examples.x86_64.tar.gz
         run: tar --strip-components 1 -xzvf ebpf_exporter_with_examples.x86_64.tar.gz
 
+      - name: Download tracing-demos.x86_64.tar.gz
+        uses: actions/download-artifact@v3
+        with:
+          name: tracing-demos.x86_64.tar.gz
+
+      - name: Extract tracing-demos.x86_64.tar.gz
+        run: tar -xzvf tracing-demos.x86_64.tar.gz
+
       - name: Run configuration check
         run: make config-check
 

.gitignore

@@ -1,2 +1,4 @@
 /ebpf_exporter
 /libbpf
+
+.DS_Store

.vscode/config-schema.yaml

@@ -2,6 +2,11 @@ title: ebpf_exporter config schema
 $schema: https://json-schema.org/draft/2020-12/schema
 type: object
 additionalProperties: false
+anyOf:
+  - required:
+      - metrics
+  - required:
+      - tracing
 properties:
   metrics:
     type: object
@@ -66,6 +71,27 @@ properties:
                 type: number
             labels:
               $ref: "#/definitions/labels"
+  tracing:
+    type: object
+    additionalProperties: false
+    properties:
+      spans:
+        type: array
+        items:
+          type: object
+          additionalProperties: false
+          required:
+            - ringbuf
+            - labels
+          properties:
+            name:
+              type: string
+            ringbuf:
+              type: string
+            service:
+              type: string
+            labels:
+              $ref: "#/definitions/labels"
   kaddrs:
     type: array
     items:
@@ -97,6 +123,7 @@ definitions:
                 enum:
                   - cgroup
                   - dname
+                  - hex
                   - ifname
                   - inet_ip
                   - kstack

Makefile

@@ -30,7 +30,7 @@ CLANG_FORMAT_FILES = ${wildcard examples/*.c examples/*.h benchmark/probes/*.c b
 CONFIGS_TO_IGNORE_IN_CHECK := cachestat kfree_skb llcstat pci unix-socket-backlog
 CONFIGS_TO_CHECK := $(filter-out $(CONFIGS_TO_IGNORE_IN_CHECK), ${patsubst examples/%.yaml, %, ${wildcard examples/*.yaml}})
 
-export CGO_LDFLAGS := -l bpf
+CGO_LDFLAGS := -l bpf
 
 include Makefile.libbpf
 
@@ -55,7 +55,7 @@ clang-format-check:
 
 .PHONY: test
 test: $(LIBBPF_DEPS)
-	go test -ldflags='-extldflags "-static"' $(GO_TEST_ARGS) ./...
+	CGO_LDFLAGS="$(CGO_LDFLAGS)" CGO_CFLAGS="$(CGO_CFLAGS)" go test -ldflags='-extldflags "-static"' $(GO_TEST_ARGS) ./...
 
 .PHONY: test-privileged
 test-privileged:
@@ -78,8 +78,23 @@ build-dynamic:
 
 .PHONY: build-binary
 build-binary: $(LIBBPF_DEPS)
-	go build -o ebpf_exporter -v -ldflags="$(GO_LDFLAGS) $(GO_LDFLAGS_VARS)" ./cmd/ebpf_exporter
+	CGO_LDFLAGS="$(CGO_LDFLAGS)" CGO_CFLAGS="$(CGO_CFLAGS)" go build -o ebpf_exporter -v -ldflags="$(GO_LDFLAGS) $(GO_LDFLAGS_VARS)" ./cmd/ebpf_exporter
+
+.PHONY: examples
+examples:
+	$(MAKE) -C examples
+
+.PHONY: tracing-demos
+tracing-demos:
+	$(MAKE) -C tracing/demos
 
 .PHONY: syscalls
 syscalls:
 	go run ./scripts/mksyscalls --strace.version v6.4
+
+.PHONY: clean
+clean:
+	rm -rf ebpf_exporter libbpf
+	$(MAKE) -C tracing/demos clean
+	$(MAKE) -C examples clean
+	$(MAKE) -C benchmark clean

Makefile.libbpf

@@ -8,8 +8,8 @@ ifneq ($(BUILD_LIBBPF),0)
 LIBBPF_DEPS := $(LIBBPF_PATH)
 LIBBPF_CFLAGS := -I$(LIBBPF_TOP)/libbpf/dest/usr/include
 LIBBPF_LDFLAGS := -L$(LIBBPF_TOP)/libbpf/dest/usr/lib
-export CGO_LDFLAGS := $(CGO_LDFLAGS) $(LIBBPF_LDFLAGS)
-export CGO_CFLAGS := $(LIBBPF_CFLAGS)
+CGO_LDFLAGS := $(CGO_LDFLAGS) $(LIBBPF_LDFLAGS)
+CGO_CFLAGS := $(LIBBPF_CFLAGS)
 endif
 
 .PHONY: clean-libbpf

README.md

@@ -1,6 +1,14 @@
 # ebpf_exporter
 
-Prometheus exporter for custom eBPF metrics.
+Prometheus exporter for custom eBPF metrics and OpenTelemetry traces.
+
+* Metrics:
+
+![metrics](./examples/biolatency.png)
+
+* [Traces](./tracing):
+
+![tracing](./tracing/demos/exec/trace.png)
 
 Motivation of this exporter is to allow you to write eBPF code and export
 metrics that are not otherwise accessible from the Linux kernel.
@@ -20,6 +28,9 @@ We use libbpf rather than legacy bcc driven code, so it's more like libbpf-tools
 
 * https://github.com/iovisor/bcc/tree/master/libbpf-tools
 
+Producing [OpenTelemetry](https://opentelemetry.io/) compatible traces is also
+supported, see [Tracing docs](./tracing/) for more information on that.
+
 ## Reading material
 
 * https://www.brendangregg.com/ebpf.html
@@ -377,7 +388,7 @@ ebpf_exporter_bio_latency_seconds_count{device="nvme1n1",operation="write"} 1
 
 You can nicely plot this with Grafana:
 
-![Histogram](examples/bio.write.latency.png)
+![Histogram](./examples/biolatency.png)
 
 ## Configuration concepts
 
@@ -585,6 +596,10 @@ could be used after `string` decode, like the following example:
     - name: dname
 ```
 
+### `hex`
+
+Hex decoder turns bytes into their hex representation.
+
 #### `inet_ip`
 
 Network IP decoded can turn byte encoded IPv4 and IPv6 addresses

benchmark/Makefile

@@ -1,6 +1,6 @@
 .DEFAULT_GOAL := build
 
-export CGO_LDFLAGS := -l bpf
+CGO_LDFLAGS := -l bpf
 
 include ../Makefile.libbpf
 

cmd/ebpf_exporter/main.go

@@ -4,13 +4,15 @@ import (
 	"fmt"
 	"log"
 	"net/http"
+	"os"
 	"strconv"
 	"strings"
 	"time"
 
 	"github.com/aquasecurity/libbpfgo"
 	"github.com/cloudflare/ebpf_exporter/v2/config"
 	"github.com/cloudflare/ebpf_exporter/v2/exporter"
+	"github.com/cloudflare/ebpf_exporter/v2/tracing"
 	"github.com/coreos/go-systemd/activation"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
@@ -58,7 +60,16 @@ func main() {
 		log.Fatalf("Error parsing configs: %v", err)
 	}
 
-	e, err := exporter.New(configs, *btfPath)
+	if os.Getenv("OTEL_SERVICE_NAME") == "" {
+		os.Setenv("OTEL_SERVICE_NAME", "ebpf_exporter")
+	}
+
+	processor, err := tracing.NewProcessor()
+	if err != nil {
+		log.Fatalf("Error creating tracing processor: %v", err)
+	}
+
+	e, err := exporter.New(configs, tracing.NewProvider(processor), *btfPath)
 	if err != nil {
 		log.Fatalf("Error creating exporter: %s", err)
 	}

config/config.go

@@ -13,6 +13,7 @@ import (
 type Config struct {
 	Name    string   `yaml:"name"`
 	Metrics Metrics  `yaml:"metrics"`
+	Tracing Tracing  `yaml:"tracing"`
 	Kaddrs  []string `yaml:"kaddrs"`
 	BPFPath string
 }
@@ -44,6 +45,19 @@ type Histogram struct {
 	Labels           []Label             `yaml:"labels"`
 }
 
+// Tracing is a collection of spans attached to a program
+type Tracing struct {
+	Spans []Span `yaml:"spans"`
+}
+
+// Span describes how a span is decoded from the kernel
+type Span struct {
+	RingBuf string  `yaml:"ringbuf"`
+	Name    string  `yaml:"name"`
+	Service string  `yaml:"service"`
+	Labels  []Label `yaml:"labels"`
+}
+
 // Label defines how to decode an element from eBPF map key
 // with the list of decoders
 type Label struct {
@@ -107,8 +121,8 @@ func ParseConfigs(dir string, names []string) ([]Config, error) {
 }
 
 func validateConfig(cfg *Config) error {
-	if cfg.Metrics.Counters == nil && cfg.Metrics.Histograms == nil {
-		return fmt.Errorf("metrics are not defined for config %q", cfg.Name)
+	if cfg.Metrics.Counters == nil && cfg.Metrics.Histograms == nil && cfg.Tracing.Spans == nil {
+		return fmt.Errorf("neither metrics nor tracing are defined for config %q", cfg.Name)
 	}
 
 	for _, counter := range cfg.Metrics.Counters {

decoder/decoder.go

@@ -42,6 +42,7 @@ func NewSet() (*Set, error) {
 		decoders: map[string]Decoder{
 			"cgroup":       cgroup,
 			"dname":        &Dname{},
+			"hex":          &Hex{},
 			"ifname":       &IfName{},
 			"inet_ip":      &InetIP{},
 			"kstack":       &KStack{ksym},

decoder/hex.go

@@ -0,0 +1,15 @@
+package decoder
+
+import (
+	"fmt"
+
+	"github.com/cloudflare/ebpf_exporter/v2/config"
+)
+
+// Hex is a decoder that decodes raw bytes into their hex string representation
+type Hex struct{}
+
+// Decode transforms bytes into their hex string representation
+func (u *Hex) Decode(in []byte, _ config.Decoder) ([]byte, error) {
+	return []byte(fmt.Sprintf("%x", in)), nil
+}

decoder/hex_test.go

@@ -0,0 +1,42 @@
+package decoder
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/cloudflare/ebpf_exporter/v2/config"
+)
+
+func TestHexDecoder(t *testing.T) {
+	cases := []struct {
+		in  []byte
+		out []byte
+	}{
+		{
+			in:  []byte{0x1},
+			out: []byte("01"),
+		},
+		{
+			in:  []byte{0x1, 0x2},
+			out: []byte("0102"),
+		},
+		{
+			in:  []byte{0xde, 0xad, 0xbe, 0xef},
+			out: []byte("deadbeef"),
+		},
+	}
+
+	for _, c := range cases {
+		d := &Hex{}
+
+		out, err := d.Decode(c.in, config.Decoder{})
+		if err != nil {
+			t.Errorf("Error decoding %#v to %q: %v", c.in, c.out, err)
+			continue
+		}
+
+		if !bytes.Equal(out, c.out) {
+			t.Errorf("Expected %q, got %q", string(c.out), string(out))
+		}
+	}
+}