Makefile: run stable cases and unstable cases seperately (#32525)

close #32524

Makefile

@@ -132,7 +132,15 @@ gotest_in_verify_ci: tools/bin/xprog tools/bin/ut failpoint-enable
 	@echo "Running gotest_in_verify_ci"
 	@mkdir -p $(TEST_COVERAGE_DIR)
 	@export TZ='Asia/Shanghai'; \
-	tools/bin/ut --junitfile "$(TEST_COVERAGE_DIR)/tidb-junit-report.xml" --coverprofile "$(TEST_COVERAGE_DIR)/tidb_cov.unit_test.out" || { $(FAILPOINT_DISABLE); exit 1; }
+	tools/bin/ut --junitfile "$(TEST_COVERAGE_DIR)/tidb-junit-report.xml" --coverprofile "$(TEST_COVERAGE_DIR)/tidb_cov.unit_test.out" --except unstable.txt || { $(FAILPOINT_DISABLE); exit 1; }
+	@$(FAILPOINT_DISABLE)
+	@$(CLEAN_UT_BINARY)
+
+gotest_unstable_in_verify_ci: tools/bin/xprog tools/bin/ut failpoint-enable
+	@echo "Running gotest_in_verify_ci"
+	@mkdir -p $(TEST_COVERAGE_DIR)
+	@export TZ='Asia/Shanghai'; \
+	tools/bin/ut --junitfile "$(TEST_COVERAGE_DIR)/tidb-junit-report.xml" --coverprofile "$(TEST_COVERAGE_DIR)/tidb_cov.unit_test.out" --only unstable.txt || { $(FAILPOINT_DISABLE); exit 1; }
 	@$(FAILPOINT_DISABLE)
 	@$(CLEAN_UT_BINARY)
 

tools/check/ut.go

@@ -30,7 +30,7 @@ import (
 	"sync"
 	"time"
 
-	// Set the correct when it runs inside docker.
+	// Set the correct value when it runs inside docker.
 	_ "go.uber.org/automaxprocs"
 	"golang.org/x/tools/cover"
 )
@@ -78,6 +78,10 @@ type task struct {
 	old  bool
 }
 
+func (t *task) String() string {
+	return t.pkg + " " + t.test
+}
+
 var P int
 var workDir string
 
@@ -254,6 +258,37 @@ func cmdRun(args ...string) bool {
 		}
 		tasks = tmp
 	}
+
+	if except != "" {
+		list, err := parseCaseListFromFile(except)
+		if err != nil {
+			fmt.Println("parse --except file error", err)
+			return false
+		}
+		tmp := tasks[:0]
+		for _, task := range tasks {
+			if _, ok := list[task.String()]; !ok {
+				tmp = append(tmp, task)
+			}
+		}
+		tasks = tmp
+	}
+
+	if only != "" {
+		list, err := parseCaseListFromFile(only)
+		if err != nil {
+			fmt.Println("parse --only file error", err)
+			return false
+		}
+		tmp := tasks[:0]
+		for _, task := range tasks {
+			if _, ok := list[task.String()]; ok {
+				tmp = append(tmp, task)
+			}
+		}
+		tasks = tmp
+	}
+
 	fmt.Printf("building task finish, count=%d, takes=%v\n", len(tasks), time.Since(start))
 
 	taskCh := make(chan task, 100)
@@ -298,6 +333,25 @@ func cmdRun(args ...string) bool {
 	return true
 }
 
+func parseCaseListFromFile(fileName string) (map[string]struct{}, error) {
+	f, err := os.Open(fileName)
+	if err != nil {
+		return nil, withTrace(err)
+	}
+	defer f.Close()
+
+	ret := make(map[string]struct{})
+	s := bufio.NewScanner(f)
+	for s.Scan() {
+		line := s.Bytes()
+		ret[string(line)] = struct{}{}
+	}
+	if err := s.Err(); err != nil {
+		return nil, withTrace(err)
+	}
+	return ret, nil
+}
+
 // handleFlags strip the '--flag xxx' from the command line os.Args
 // Example of the os.Args changes
 // Before: ut run sessoin TestXXX --coverprofile xxx --junitfile yyy
@@ -334,9 +388,14 @@ var junitfile string
 var coverprofile string
 var coverFileTempDir string
 
+var except string
+var only string
+
 func main() {
 	junitfile = handleFlags("--junitfile")
 	coverprofile = handleFlags("--coverprofile")
+	except = handleFlags("--except")
+	only = handleFlags("--only")
 	if coverprofile != "" {
 		var err error
 		coverFileTempDir, err = os.MkdirTemp(os.TempDir(), "cov")

unstable.txt

@@ -0,0 +1,12 @@
+ddl TestTiFlashReplicaPartitionTableNormal
+ddl TestTestSerialStatSuite
+ddl testSerialDBSuite.TestModifyColumnTypeWhenInterception
+session testPessimisticSuite.TestSelectForUpdateNoWait
+server TestGetSchemaStorage
+server TestUptime
+executor TestSpillToDisk
+executor TestHashRowContainer
+executor TestExplainAnalyzeCTEMemoryAndDiskInfo
+executor TestDefaultValForAnalyze
+executor TestPBMemoryLeak
+statistics/handle TestAnalyzeGlobalStatsWithOpts1