From 0a0bb6fa77465442689287f7071421497903d8f5 Mon Sep 17 00:00:00 2001 From: Mario Macias Date: Fri, 13 Dec 2024 09:31:29 +0100 Subject: [PATCH] Print more information on failing flaky tests (#1451) * fix flaky K8s network integration test * Print more information on failing flaky tests * fix flaky check --- pkg/internal/exec/proclang_darwin.go | 2 +- test/integration/components/jaeger/jaeger.go | 4 ++ .../k8s/common/k8s_metrics_testfuncs.go | 67 +++++++++++++++++++ .../k8s/daemonset/k8s_daemonset_main_test.go | 3 +- .../daemonset/k8s_daemonset_traces_test.go | 7 +- .../netolly/k8s_netolly_network_metrics.go | 2 + 6 files changed, 81 insertions(+), 4 deletions(-) diff --git a/pkg/internal/exec/proclang_darwin.go b/pkg/internal/exec/proclang_darwin.go index 1cd3d9e11..5e830a2b3 100644 --- a/pkg/internal/exec/proclang_darwin.go +++ b/pkg/internal/exec/proclang_darwin.go @@ -10,6 +10,6 @@ func FindProcLanguage(_ int32, _ *elf.File, _ string) svc.InstrumentableType { return svc.InstrumentableGeneric } -func FindExeSymbols(_ *elf.File, _ map[string]struct{}) (map[string]Sym, error) { +func FindExeSymbols(_ *elf.File, _ []string) (map[string]Sym, error) { return nil, nil } diff --git a/test/integration/components/jaeger/jaeger.go b/test/integration/components/jaeger/jaeger.go index db1d2b5fb..e01151281 100644 --- a/test/integration/components/jaeger/jaeger.go +++ b/test/integration/components/jaeger/jaeger.go @@ -9,6 +9,10 @@ import ( "strings" ) +type Services struct { + Data []string `json:"data"` +} + type TracesQuery struct { Data []Trace `json:"data"` } diff --git a/test/integration/k8s/common/k8s_metrics_testfuncs.go b/test/integration/k8s/common/k8s_metrics_testfuncs.go index 5c6b0a277..7595b6d32 100644 --- a/test/integration/k8s/common/k8s_metrics_testfuncs.go +++ b/test/integration/k8s/common/k8s_metrics_testfuncs.go @@ -4,7 +4,10 @@ package k8s import ( "context" + "encoding/json" + "fmt" "net/http" + "os" "slices" "testing" "time" @@ -15,6 +18,7 @@ import ( "sigs.k8s.io/e2e-framework/pkg/envconf" "sigs.k8s.io/e2e-framework/pkg/features" + "github.com/grafana/beyla/test/integration/components/jaeger" "github.com/grafana/beyla/test/integration/components/kube" "github.com/grafana/beyla/test/integration/components/prom" ) @@ -287,3 +291,66 @@ func testMetricsDecoration( return ctx } } + +func DumpMetricsAfterFail(t *testing.T, queryURL string) { + if !t.Failed() { + return + } + fmt.Printf("===== Dumping metrics from %s ====\n", queryURL) + pq := prom.Client{HostPort: queryURL} + results, err := pq.Query(`{__name__!=""}`) + if err != nil { + fmt.Printf("ERROR: %s\n", err) + return + } + for _, res := range results { + fmt.Printf(res.Metric["__name__"]) + fmt.Printf("{") + for k, v := range res.Metric { + if k == "__name__" { + continue + } + fmt.Printf(`%s="%s",`, k, v) + } + fmt.Print("} ") + for _, v := range res.Value { + fmt.Printf("%s ", v) + } + fmt.Println() + } +} + +func DumpTracesAfterFail(t *testing.T, hostURL string) { + if !t.Failed() { + return + } + fmt.Printf("===== Dumping traces from %s ====\n", hostURL) + // get services + res, err := http.Get(hostURL + "/api/services") + if err != nil { + fmt.Println("ERROR getting services:", err) + return + } + svcs := jaeger.Services{} + if err := json.NewDecoder(res.Body).Decode(&svcs); err != nil { + fmt.Println("ERROR decoding services:", err) + return + } + for _, svcName := range svcs.Data { + fmt.Printf("---- Service: %s ----\n", svcName) + res, err := http.Get(hostURL + "/api/traces?service=" + svcName) + if err != nil { + fmt.Println("ERROR getting service:", err) + return + } + tq := jaeger.TracesQuery{} + if err := json.NewDecoder(res.Body).Decode(&tq); err != nil { + fmt.Println("ERROR decoding service:", err) + continue + } + for _, trace := range tq.Data { + json.NewEncoder(os.Stdout).Encode(trace) + fmt.Println() + } + } +} diff --git a/test/integration/k8s/daemonset/k8s_daemonset_main_test.go b/test/integration/k8s/daemonset/k8s_daemonset_main_test.go index be06ac28f..f38c917a2 100644 --- a/test/integration/k8s/daemonset/k8s_daemonset_main_test.go +++ b/test/integration/k8s/daemonset/k8s_daemonset_main_test.go @@ -17,7 +17,8 @@ import ( const ( testTimeout = 3 * time.Minute - jaegerQueryURL = "http://localhost:36686/api/traces" + jaegerHost = "http://localhost:36686" + jaegerQueryURL = jaegerHost + "/api/traces" ) var cluster *kube.Kind diff --git a/test/integration/k8s/daemonset/k8s_daemonset_traces_test.go b/test/integration/k8s/daemonset/k8s_daemonset_traces_test.go index c5d07fd60..0143d9f3a 100644 --- a/test/integration/k8s/daemonset/k8s_daemonset_traces_test.go +++ b/test/integration/k8s/daemonset/k8s_daemonset_traces_test.go @@ -27,6 +27,7 @@ func TestBasicTracing(t *testing.T) { feat := features.New("Beyla is able to instrument an arbitrary process"). Assess("it sends traces for that service", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + defer k8s.DumpTracesAfterFail(t, jaegerHost) var podID string test.Eventually(t, testTimeout, func(t require.TestingT) { // Invoking both service instances, but we will expect that only one @@ -125,7 +126,9 @@ func TestBasicTracing(t *testing.T) { require.NoError(t, json.NewDecoder(resp.Body).Decode(&tq)) traces := tq.FindBySpan(jaeger.Tag{Key: "url.path", Type: "string", Value: "/pingpongtoo"}) require.NotEmpty(t, traces) - trace := traces[0] + // get the last trace, to avoid that the old instance captured any request + // before being restarted + trace := traces[len(traces)-1] require.NotEmpty(t, trace.Spans) // Check that the service.namespace is set from the K8s namespace @@ -154,7 +157,7 @@ func TestBasicTracing(t *testing.T) { }, trace.Processes[parent.ProcessID].Tags) require.Empty(t, sd) - // ensure the pod really restarted + // ensure the pod really restarted, comparing the current uid with the previous pod uid tag, found := jaeger.FindIn(trace.Processes[parent.ProcessID].Tags, "k8s.pod.uid") assert.True(t, found) diff --git a/test/integration/k8s/netolly/k8s_netolly_network_metrics.go b/test/integration/k8s/netolly/k8s_netolly_network_metrics.go index f7b5269fc..13fce689a 100644 --- a/test/integration/k8s/netolly/k8s_netolly_network_metrics.go +++ b/test/integration/k8s/netolly/k8s_netolly_network_metrics.go @@ -46,6 +46,8 @@ func FeatureNetworkFlowBytes() features.Feature { } func testNetFlowBytesForExistingConnections(ctx context.Context, t *testing.T, _ *envconf.Config) context.Context { + defer k8s.DumpMetricsAfterFail(t, prometheusHostPort) + pq := prom.Client{HostPort: prometheusHostPort} // testing request flows (to testserver as Service) test.Eventually(t, testTimeout, func(t require.TestingT) {