Test runner improvements (#6865)

## What changed?

- Do not rerun tests if there are more than 10 failures in a single
suite. This threshold can later become configurable as needed.
- Add a sanity check to ensure test-runner reruns all failed tests as
expected.

## Why?

For confidence, this is a new tool and I don't trust it yet. The
threshold is

tools/testrunner/testrunner.go

@@ -242,6 +242,12 @@ func (r *runner) combineAttempts() junit.Testsuites {
 			continue
 		}
 
+		// Just a sanity check for this tool since it's new and we want to make sure we actually rerun what we
+		// expect.
+		if attempt.suites.Tests != r.attempts[i-1].suites.Failures {
+			log.Fatalf("expected a rerun of all failures from the previous attempt, got (%d/%d)", attempt.suites.Tests, r.attempts[i-1].suites.Failures)
+		}
+
 		for _, suite := range attempt.suites.Suites {
 			cpy := suite
 			cpy.Name += fmt.Sprintf(" (retry %d)", i)
@@ -286,6 +292,11 @@ func Main() {
 		if len(failures) == 0 {
 			log.Fatalf("tests failed but no failures have been detected, not rerunning tests")
 		}
+		// Don't rerun if there's more than 10 failures in a single suite.
+		if len(failures) > 10 && retry < r.retries {
+			log.Printf("will not rerun tests, number of failures exceeds configured threshold (%d/%d)", len(failures), 10)
+			break
+		}
 		args = stripRunFromArgs(args)
 		for i, failure := range failures {
 			failures[i] = goTestNameToRunFlagRegexp(failure)