From abb47a2eba106fdbe790005ed01099927529f637 Mon Sep 17 00:00:00 2001 From: lukaszcl <120112546+lukaszcl@users.noreply.github.com> Date: Fri, 18 Oct 2024 12:46:46 +0200 Subject: [PATCH] Fix re-running tests n times --- tools/flakeguard/cmd/run.go | 16 +++-- tools/flakeguard/runner/runner.go | 110 +++++++++++++++--------------- 2 files changed, 65 insertions(+), 61 deletions(-) diff --git a/tools/flakeguard/cmd/run.go b/tools/flakeguard/cmd/run.go index 255715191..392dc4201 100644 --- a/tools/flakeguard/cmd/run.go +++ b/tools/flakeguard/cmd/run.go @@ -18,7 +18,7 @@ var RunTestsCmd = &cobra.Command{ repoPath, _ := cmd.Flags().GetString("repo-path") testPackagesJson, _ := cmd.Flags().GetString("test-packages-json") testPackage, _ := cmd.Flags().GetString("test-package") - count, _ := cmd.Flags().GetInt("count") + runCount, _ := cmd.Flags().GetInt("run-count") useRace, _ := cmd.Flags().GetBool("race") failFast, _ := cmd.Flags().GetBool("fail-fast") outputPath, _ := cmd.Flags().GetString("output-json") @@ -38,13 +38,16 @@ var RunTestsCmd = &cobra.Command{ runner := runner.Runner{ Verbose: true, Dir: repoPath, - Count: count, + RunCount: runCount, UseRace: useRace, FailFast: failFast, } - testResults, _ := runner.RunTests(testPackages) - // TODO: Handle error + testResults, err := runner.RunTests(testPackages) + if err != nil { + fmt.Printf("Error running tests: %v\n", err) + os.Exit(1) + } // Filter out failed tests based on the threshold failedTests := reports.FilterFailedTests(testResults, threshold) @@ -54,7 +57,6 @@ var RunTestsCmd = &cobra.Command{ log.Fatalf("Error marshaling test results to JSON: %v", err) } fmt.Printf("Threshold for flaky tests: %.2f\nFailed tests:\n%s\n", threshold, string(jsonData)) - fmt.Println(string(jsonData)) } // Save the test results in JSON format @@ -66,7 +68,7 @@ var RunTestsCmd = &cobra.Command{ if err := os.WriteFile(outputPath, jsonData, 0644); err != nil { log.Fatalf("Error writing test results to file: %v", err) } - fmt.Printf("Test results saved to %s\n", outputPath) + fmt.Printf("All test results saved to %s\n", outputPath) } if len(failedTests) > 0 { @@ -81,7 +83,7 @@ func init() { RunTestsCmd.Flags().StringP("repo-path", "r", ".", "Path to the Git repository") RunTestsCmd.Flags().String("test-packages-json", "", "JSON-encoded string of test packages") RunTestsCmd.Flags().String("test-package", "", "Single test package to run") - RunTestsCmd.Flags().IntP("count", "c", 1, "Number of times to run the tests") + RunTestsCmd.Flags().IntP("run-count", "c", 1, "Number of times to run the tests") RunTestsCmd.Flags().Bool("race", false, "Enable the race detector") RunTestsCmd.Flags().Bool("fail-fast", false, "Stop on the first test failure") RunTestsCmd.Flags().String("output-json", "", "Path to output the test results in JSON format") diff --git a/tools/flakeguard/runner/runner.go b/tools/flakeguard/runner/runner.go index 29a577c75..6316852bc 100644 --- a/tools/flakeguard/runner/runner.go +++ b/tools/flakeguard/runner/runner.go @@ -4,6 +4,7 @@ import ( "bufio" "bytes" "encoding/json" + "errors" "fmt" "os/exec" "strings" @@ -14,7 +15,7 @@ import ( type Runner struct { Verbose bool // If true, provides detailed logging. Dir string // Directory to run commands in. - Count int // Number of times to run the tests. + RunCount int // Number of times to run the tests. UseRace bool // Enable race detector. FailFast bool // Stop on first test failure. } @@ -22,31 +23,31 @@ type Runner struct { // RunTests executes the tests for each provided package and aggregates all results. // It returns all test results and any error encountered during testing. func (r *Runner) RunTests(packages []string) ([]reports.TestResult, error) { - var allResults []reports.TestResult - var errors []string + var jsonOutputs [][]byte for _, p := range packages { - testResults, err := r.runTestPackage(p) - allResults = append(allResults, testResults...) - - if err != nil { - errors = append(errors, fmt.Sprintf("failed to run tests in package %s: %v", p, err)) + for i := 0; i < r.RunCount; i++ { + jsonOutput, err := r.runTestPackage(p) + if err != nil { + return nil, fmt.Errorf("failed to run tests in package %s: %w", p, err) + } + jsonOutputs = append(jsonOutputs, jsonOutput) } } - if len(errors) > 0 { - return allResults, fmt.Errorf("some tests failed: %s", strings.Join(errors, "; ")) - } + return parseTestResults(jsonOutputs) +} - return allResults, nil +type exitCoder interface { + ExitCode() int } // runTestPackage executes the test command for a single test package. -func (r *Runner) runTestPackage(testPackage string) ([]reports.TestResult, error) { - args := []string{"test", "-json"} // Enable JSON output - if r.Count > 0 { - args = append(args, fmt.Sprintf("-count=%d", r.Count)) - } +func (r *Runner) runTestPackage(testPackage string) ([]byte, error) { + args := []string{"test", "-json", "-count=1"} // Enable JSON output + // if r.Count > 0 { + // args = append(args, fmt.Sprintf("-count=%d", r.Count)) + // } if r.UseRace { args = append(args, "-race") } @@ -68,55 +69,56 @@ func (r *Runner) runTestPackage(testPackage string) ([]reports.TestResult, error // Run the command err := cmd.Run() - - // Parse results - results, parseErr := parseTestResults(out.Bytes()) - if parseErr != nil { - return results, fmt.Errorf("failed to parse test results for %s: %v", testPackage, parseErr) - } - if err != nil { - return results, fmt.Errorf("test command failed at %s: %w", testPackage, err) + var exErr exitCoder + if errors.As(err, &exErr) && exErr.ExitCode() == 0 { + return nil, fmt.Errorf("test command failed at %s: %w", testPackage, err) + } } - return results, nil + return out.Bytes(), nil } -// parseTestResults analyzes the JSON output from 'go test -json' to determine test results -func parseTestResults(data []byte) ([]reports.TestResult, error) { - scanner := bufio.NewScanner(bytes.NewReader(data)) +// parseTestResults analyzes multiple JSON outputs from 'go test -json' commands to determine test results. +// It accepts a slice of []byte where each []byte represents a separate JSON output from a test run. +// This function aggregates results across multiple test runs, summing runs and passes for each test. +func parseTestResults(datas [][]byte) ([]reports.TestResult, error) { testDetails := make(map[string]map[string]int) // Holds run and pass counts for each test - for scanner.Scan() { - var entry struct { - Action string `json:"Action"` - Test string `json:"Test"` - } - if err := json.Unmarshal(scanner.Bytes(), &entry); err != nil { - return nil, fmt.Errorf("failed to parse json test output: %w", err) - } - - // Skip processing if the test name is empty - if entry.Test == "" { - continue - } - - if _, exists := testDetails[entry.Test]; !exists { - testDetails[entry.Test] = map[string]int{"run": 0, "pass": 0} + // Process each data set + for _, data := range datas { + scanner := bufio.NewScanner(bytes.NewReader(data)) + for scanner.Scan() { + var entry struct { + Action string `json:"Action"` + Test string `json:"Test"` + } + if err := json.Unmarshal(scanner.Bytes(), &entry); err != nil { + return nil, fmt.Errorf("failed to parse json test output: %w", err) + } + + // Skip processing if the test name is empty + if entry.Test == "" { + continue + } + + if _, exists := testDetails[entry.Test]; !exists { + testDetails[entry.Test] = map[string]int{"run": 0, "pass": 0} + } + + if entry.Action == "run" { + testDetails[entry.Test]["run"]++ + } + if entry.Action == "pass" { + testDetails[entry.Test]["pass"]++ + } } - if entry.Action == "run" { - testDetails[entry.Test]["run"]++ - } - if entry.Action == "pass" { - testDetails[entry.Test]["pass"]++ + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("reading standard input: %w", err) } } - if err := scanner.Err(); err != nil { - return nil, fmt.Errorf("reading standard input: %w", err) - } - var results []reports.TestResult for testName, counts := range testDetails { runs := counts["run"]