Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

testgrid: print out all failed tests for visibility. #17785

Merged
merged 1 commit into from
Apr 13, 2024
Merged

testgrid: print out all failed tests for visibility. #17785

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 8 additions & 2 deletions scripts/measure-testgrid-flakiness.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,12 @@ then
fi

pushd ./tools/testgrid-analysis
go run main.go flaky --create-issue --dashboard=sig-etcd-periodics --tab=ci-etcd-e2e-amd64
go run main.go flaky --create-issue --dashboard=sig-etcd-periodics --tab=ci-etcd-unit-test-amd64
# ci-etcd-e2e-amd64 and ci-etcd-unit-test-amd64 runs 6 times a day. Keeping a rolling window of 14 days.
siyuanfoundation marked this conversation as resolved.
Show resolved Hide resolved
go run main.go flaky --create-issue --dashboard=sig-etcd-periodics --tab=ci-etcd-e2e-amd64 --max-days=14
go run main.go flaky --create-issue --dashboard=sig-etcd-periodics --tab=ci-etcd-unit-test-amd64 --max-days=14

# do not create issues for presubmit tests
go run main.go flaky --dashboard=sig-etcd-presubmits --tab=pull-etcd-e2e-amd64
go run main.go flaky --dashboard=sig-etcd-presubmits --tab=pull-etcd-unit-test

popd
11 changes: 10 additions & 1 deletion tools/testgrid-analysis/cmd/data.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"net/http"
"os"
"strings"
"time"

apipb "github.com/GoogleCloudPlatform/testgrid/pb/api/v1"
statuspb "github.com/GoogleCloudPlatform/testgrid/pb/test_status"
Expand Down Expand Up @@ -77,6 +78,7 @@ func processRow(dashboard, tab string, row *apipb.ListRowsResponse_Row, allTests
if !strings.HasPrefix(row.Name, "go.etcd.io") {
return &t
}
earliestTimeToConsider := time.Now().AddDate(0, 0, -1*maxDays)
total := 0
failed := 0
logs := []string{}
Expand All @@ -89,13 +91,19 @@ func processRow(dashboard, tab string, row *apipb.ListRowsResponse_Row, allTests
}
continue
}
header := headers[i]
if maxDays > 0 && header.Started.AsTime().Before(earliestTimeToConsider) {
continue
}
total += 1
if _, ok := failureTestStatusesInt[cell.Result]; ok {
failed += 1
header := headers[i]
// markdown table format of | commit | log |
logs = append(logs, fmt.Sprintf("| %s | %s | https://prow.k8s.io/view/gs/kubernetes-jenkins/logs/%s/%s |", strings.Join(header.Extra, ","), header.Started.AsTime().String(), tab, header.Build))
}
if maxRuns > 0 && total >= maxRuns {
break
}
}
t.FailedRuns = failed
t.TotalRuns = total
Expand All @@ -106,6 +114,7 @@ func processRow(dashboard, tab string, row *apipb.ListRowsResponse_Row, allTests
t.IssueBody = fmt.Sprintf("## %s Test: %s \nTest failed %.1f%% (%d/%d) of the time\n\nfailure logs are:\n| commit | started | log |\n| --- | --- | --- |\n%s\n",
dashboardUrl, t.FullName, t.FailureRate*100, t.FailedRuns, t.TotalRuns, strings.Join(t.FailureLogs, "\n"))
t.IssueBody += "\nPlease follow the [instructions in the contributing guide](https://github.com/etcd-io/etcd/blob/main/CONTRIBUTING.md#check-for-flaky-tests) to reproduce the issue.\n"
fmt.Printf("%s failed %.1f%% (%d/%d) of the time\n", t.FullName, t.FailureRate*100, t.FailedRuns, t.TotalRuns)
}
return &t
}
Expand Down
6 changes: 5 additions & 1 deletion tools/testgrid-analysis/cmd/flaky.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@ var flakyCmd = &cobra.Command{
var (
flakyThreshold float32
minRuns int
maxRuns int
maxDays int
createGithubIssue bool
githubOwner string
githubRepo string
Expand All @@ -44,6 +46,8 @@ func init() {
flakyCmd.Flags().BoolVar(&createGithubIssue, "create-issue", false, "create Github issue for each flaky test")
flakyCmd.Flags().Float32Var(&flakyThreshold, "flaky-threshold", 0.1, "fraction threshold of test failures for a test to be considered flaky")
flakyCmd.Flags().IntVar(&minRuns, "min-runs", 20, "minimum test runs for a test to be included in flaky analysis")
flakyCmd.Flags().IntVar(&maxRuns, "max-runs", 0, "maximum test runs for a test to be included in flaky analysis, 0 to include all")
flakyCmd.Flags().IntVar(&maxDays, "max-days", 0, "maximum days of results before today to be included in flaky analysis, 0 to include all")
flakyCmd.Flags().StringVar(&githubOwner, "github-owner", "etcd-io", "the github organization to create the issue for")
flakyCmd.Flags().StringVar(&githubRepo, "github-repo", "etcd", "the github repo to create the issue for")
}
Expand All @@ -59,7 +63,7 @@ func flakyFunc(cmd *cobra.Command, args []string) {
}
}
fmt.Println(lineSep)
fmt.Printf("Detected total %d flaky tests for %s#%s\n", len(flakyTests), dashboard, tab)
fmt.Printf("Detected total %d flaky tests above the %.0f%% threshold for %s#%s\n", len(flakyTests), flakyThreshold*100, dashboard, tab)
fmt.Println(lineSep)
if len(flakyTests) == 0 {
return
Expand Down
Loading