[DNM] support chaos mesh

ddl/ddl.go

@@ -88,8 +88,12 @@ func (c *DDLCase) Execute(ctx context.Context, dbss [][]*sql.DB, exeDDLFunc Exec
 							disableTiKVGC(db)
 						}
 					}
-					// os.Exit(-1)
-					log.Fatalf("[error] [instance %d] ERROR: %s", i, errors.ErrorStack(err))
+					if Chaos {
+						log.Warnf("[error] [instance %d] ERROR: %s", i, errors.ErrorStack(err))
+					} else {
+						// os.Exit(-1)
+						log.Fatalf("[error] [instance %d] ERROR: %s", i, errors.ErrorStack(err))
+					}
 				}
 			}
 		}(i)
@@ -249,12 +253,18 @@ ParallelExecuteOperations executes process:
 func ParallelExecuteOperations(c *testCase, ops []ddlTestOpExecutor, postOp func() error) error {
 	perm := rand.Perm(len(ops))
 	taskCh := make(chan *ddlJobTask, len(ops))
+	var probability map[int]float64
+	if isIndexMode {
+		probability = mapOfDDLKindProbabilityInIndexMode
+	} else {
+		probability = mapOfDDLKindProbability
+	}
 	for _, idx := range perm {
 		if c.isStop() {
 			return nil
 		}
 		op := ops[idx]
-		if rand.Float64() > mapOfDDLKindProbability[op.ddlKind] {
+		if rand.Float64() > probability[op.ddlKind] {
 			continue
 		}
 		op.executeFunc(op.config, taskCh)

ddl/ddl_ops.go

@@ -206,6 +206,33 @@ var mapOfDDLKindProbability = map[DDLKind]float64{
 	ddlSetTiflashReplica:            0.30,
 }
 
+// mapOfDDLKindProbabilityInIndexMode use to control every kind of ddl request execute probability in index mode.
+var mapOfDDLKindProbabilityInIndexMode = map[DDLKind]float64{
+	ddlAddTable:  0.05,
+	ddlDropTable: 0.05,
+
+	ddlAddIndex:  0.85,
+	ddlDropIndex: 0.5,
+
+	ddlAddColumn:     0.2,
+	ddlModifyColumn:  0.1,
+	ddlModifyColumn2: 0.1,
+	ddlDropColumn:    0.1,
+
+	ddlCreateView: 0.07,
+
+	ddlCreateSchema:                 0.02,
+	ddlDropSchema:                   0.02,
+	ddlRenameTable:                  0.12,
+	ddlRenameIndex:                  0.12,
+	ddlTruncateTable:                0.12,
+	ddlShardRowID:                   0.07,
+	ddlRebaseAutoID:                 0.03,
+	ddlSetDefaultValue:              0.07,
+	ddlModifyTableComment:           0.07,
+	ddlModifyTableCharsetAndCollate: 0.07,
+}
+
 type ddlJob struct {
 	id         int
 	schemaName string

ddl/run.go

@@ -17,6 +17,8 @@ import (
 var defaultPushMetricsInterval = 15 * time.Second
 var enableTransactionTestFlag = "0"
 var enableTransactionTest = false
+var isIndexMode = false
+var Chaos bool = false
 
 func init() {
 	if enableTransactionTestFlag == "1" {
@@ -35,7 +37,9 @@ func OpenDB(dsn string, maxIdleConns int) (*sql.DB, error) {
 	return db, nil
 }
 
-func Run(dbAddr string, dbName string, concurrency int, tablesToCreate int, mysqlCompatible bool, testTp DDLTestType, testTime time.Duration) {
+func Run(dbAddr string, dbName string, concurrency int, tablesToCreate int, mysqlCompatible bool, testTp DDLTestType, testTime time.Duration, indexMode, chaos bool) {
+	Chaos = chaos
+	isIndexMode = indexMode
 	wrapCtx := context.WithCancel
 	if testTime > 0 {
 		wrapCtx = func(ctx context.Context) (context.Context, context.CancelFunc) {
@@ -149,6 +153,9 @@ func ddlIgnoreError(err error) bool {
 		return true
 	}
 	errStr := err.Error()
+	if Chaos && strings.Contains(errStr, "invalid connection") {
+		return true
+	}
 	log.Warnf("check DDL err:%s", errStr)
 	if strings.Contains(errStr, "Information schema is changed") {
 		return true

main.go

@@ -32,8 +32,10 @@ var (
 	concurrency     = flag.Int("concurrency", 20, "concurrency")
 	tablesToCreate  = flag.Int("tables", 1, "the number of the tables to create")
 	mysqlCompatible = flag.Bool("mysql-compatible", false, "disable TiDB-only features")
+	chaos           = flag.Bool("chaos", true, "whether to test in the chaos mode")
 	testTime        = flag.Duration("time", 2*time.Hour, "test time")
 	output          = flag.String("output", "", "output file")
+	indexMode       = flag.Bool("index-mode", false, "test more index operator")
 )
 
 func prepareEnv() {
@@ -73,6 +75,10 @@ func main() {
 	default:
 		log.Fatalf("unknown test mode: %s", *mode)
 	}
-	prepareEnv()
-	Run(*dbAddr, *dbName, *concurrency, *tablesToCreate, *mysqlCompatible, testType, *testTime)
+	if !*chaos {
+		prepareEnv()
+	} else {
+		log.Warnf("in the chaos mode, Please do the prepare env by yourself")
+	}
+	Run(*dbAddr, *dbName, *concurrency, *tablesToCreate, *mysqlCompatible, testType, *testTime, *indexMode, *chaos)
 }