Print more information on failing flaky tests (#1451)

* fix flaky K8s network integration test

* Print more information on failing flaky tests

* fix flaky check

pkg/internal/exec/proclang_darwin.go

@@ -10,6 +10,6 @@ func FindProcLanguage(_ int32, _ *elf.File, _ string) svc.InstrumentableType {
 	return svc.InstrumentableGeneric
 }
 
-func FindExeSymbols(_ *elf.File, _ map[string]struct{}) (map[string]Sym, error) {
+func FindExeSymbols(_ *elf.File, _ []string) (map[string]Sym, error) {
 	return nil, nil
 }

test/integration/components/jaeger/jaeger.go

@@ -9,6 +9,10 @@ import (
 	"strings"
 )
 
+type Services struct {
+	Data []string `json:"data"`
+}
+
 type TracesQuery struct {
 	Data []Trace `json:"data"`
 }

test/integration/k8s/common/k8s_metrics_testfuncs.go

@@ -4,7 +4,10 @@ package k8s
 
 import (
 	"context"
+	"encoding/json"
+	"fmt"
 	"net/http"
+	"os"
 	"slices"
 	"testing"
 	"time"
@@ -15,6 +18,7 @@ import (
 	"sigs.k8s.io/e2e-framework/pkg/envconf"
 	"sigs.k8s.io/e2e-framework/pkg/features"
 
+	"github.com/grafana/beyla/test/integration/components/jaeger"
 	"github.com/grafana/beyla/test/integration/components/kube"
 	"github.com/grafana/beyla/test/integration/components/prom"
 )
@@ -287,3 +291,66 @@ func testMetricsDecoration(
 		return ctx
 	}
 }
+
+func DumpMetricsAfterFail(t *testing.T, queryURL string) {
+	if !t.Failed() {
+		return
+	}
+	fmt.Printf("===== Dumping metrics from %s ====\n", queryURL)
+	pq := prom.Client{HostPort: queryURL}
+	results, err := pq.Query(`{__name__!=""}`)
+	if err != nil {
+		fmt.Printf("ERROR: %s\n", err)
+		return
+	}
+	for _, res := range results {
+		fmt.Printf(res.Metric["__name__"])
+		fmt.Printf("{")
+		for k, v := range res.Metric {
+			if k == "__name__" {
+				continue
+			}
+			fmt.Printf(`%s="%s",`, k, v)
+		}
+		fmt.Print("} ")
+		for _, v := range res.Value {
+			fmt.Printf("%s ", v)
+		}
+		fmt.Println()
+	}
+}
+
+func DumpTracesAfterFail(t *testing.T, hostURL string) {
+	if !t.Failed() {
+		return
+	}
+	fmt.Printf("===== Dumping traces from %s ====\n", hostURL)
+	// get services
+	res, err := http.Get(hostURL + "/api/services")
+	if err != nil {
+		fmt.Println("ERROR getting services:", err)
+		return
+	}
+	svcs := jaeger.Services{}
+	if err := json.NewDecoder(res.Body).Decode(&svcs); err != nil {
+		fmt.Println("ERROR decoding services:", err)
+		return
+	}
+	for _, svcName := range svcs.Data {
+		fmt.Printf("---- Service: %s ----\n", svcName)
+		res, err := http.Get(hostURL + "/api/traces?service=" + svcName)
+		if err != nil {
+			fmt.Println("ERROR getting service:", err)
+			return
+		}
+		tq := jaeger.TracesQuery{}
+		if err := json.NewDecoder(res.Body).Decode(&tq); err != nil {
+			fmt.Println("ERROR decoding service:", err)
+			continue
+		}
+		for _, trace := range tq.Data {
+			json.NewEncoder(os.Stdout).Encode(trace)
+			fmt.Println()
+		}
+	}
+}

test/integration/k8s/daemonset/k8s_daemonset_main_test.go

@@ -17,7 +17,8 @@ import (
 const (
 	testTimeout = 3 * time.Minute
 
-	jaegerQueryURL = "http://localhost:36686/api/traces"
+	jaegerHost     = "http://localhost:36686"
+	jaegerQueryURL = jaegerHost + "/api/traces"
 )
 
 var cluster *kube.Kind

test/integration/k8s/daemonset/k8s_daemonset_traces_test.go

@@ -27,6 +27,7 @@ func TestBasicTracing(t *testing.T) {
 	feat := features.New("Beyla is able to instrument an arbitrary process").
 		Assess("it sends traces for that service",
 			func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
+				defer k8s.DumpTracesAfterFail(t, jaegerHost)
 				var podID string
 				test.Eventually(t, testTimeout, func(t require.TestingT) {
 					// Invoking both service instances, but we will expect that only one
@@ -125,7 +126,9 @@ func TestBasicTracing(t *testing.T) {
 					require.NoError(t, json.NewDecoder(resp.Body).Decode(&tq))
 					traces := tq.FindBySpan(jaeger.Tag{Key: "url.path", Type: "string", Value: "/pingpongtoo"})
 					require.NotEmpty(t, traces)
-					trace := traces[0]
+					// get the last trace, to avoid that the old instance captured any request
+					// before being restarted
+					trace := traces[len(traces)-1]
 					require.NotEmpty(t, trace.Spans)
 
 					// Check that the service.namespace is set from the K8s namespace
@@ -154,7 +157,7 @@ func TestBasicTracing(t *testing.T) {
 					}, trace.Processes[parent.ProcessID].Tags)
 					require.Empty(t, sd)
 
-					// ensure the pod really restarted
+					// ensure the pod really restarted, comparing the current uid with the previous pod uid
 					tag, found := jaeger.FindIn(trace.Processes[parent.ProcessID].Tags, "k8s.pod.uid")
 					assert.True(t, found)
 

test/integration/k8s/netolly/k8s_netolly_network_metrics.go

@@ -46,6 +46,8 @@ func FeatureNetworkFlowBytes() features.Feature {
 }
 
 func testNetFlowBytesForExistingConnections(ctx context.Context, t *testing.T, _ *envconf.Config) context.Context {
+	defer k8s.DumpMetricsAfterFail(t, prometheusHostPort)
+
 	pq := prom.Client{HostPort: prometheusHostPort}
 	// testing request flows (to testserver as Service)
 	test.Eventually(t, testTimeout, func(t require.TestingT) {