dm: add more event for incompatible ddl changes (#9978)

close #9692

dm/syncer/ddl.go

@@ -1102,10 +1102,10 @@ func (ddl *DDLWorker) handleModifyColumn(qec *queryEventContext, info *ddlInfo,
 	switch {
 	case mysql.HasAutoIncrementFlag(oldCol.GetFlag()) && !mysql.HasAutoIncrementFlag(newCol.GetFlag()):
 		return bf.RemoveAutoIncrement, nil
-	case mysql.HasPriKeyFlag(oldCol.GetFlag()) != mysql.HasPriKeyFlag(newCol.GetFlag()):
-		return bf.ModifyPK, nil
-	case mysql.HasUniKeyFlag(oldCol.GetFlag()) != mysql.HasUniKeyFlag(newCol.GetFlag()):
-		return bf.ModifyUK, nil
+	case mysql.HasPriKeyFlag(oldCol.GetFlag()) && !mysql.HasPriKeyFlag(newCol.GetFlag()):
+		return bf.DropPrimaryKey, nil
+	case mysql.HasUniKeyFlag(oldCol.GetFlag()) && !mysql.HasUniKeyFlag(newCol.GetFlag()):
+		return bf.DropUniqueKey, nil
 	case oldCol.GetDefaultValue() != newCol.GetDefaultValue():
 		return bf.ModifyDefaultValue, nil
 	case oldCol.GetCharset() != newCol.GetCharset():
@@ -1115,7 +1115,7 @@ func (ddl *DDLWorker) handleModifyColumn(qec *queryEventContext, info *ddlInfo,
 	case spec.Position != nil && spec.Position.Tp != ast.ColumnPositionNone:
 		return bf.ModifyColumnsOrder, nil
 	case oldCol.Name.L != newCol.Name.L:
-		return bf.Rename, nil
+		return bf.RenameColumn, nil
 	default:
 		return bf.AlterTable, nil
 	}
@@ -1142,14 +1142,22 @@ func (ddl *DDLWorker) AstToDDLEvent(qec *queryEventContext, info *ddlInfo) (et b
 					ddl.logger.Warn("handle modify column failed", zap.Error(err))
 				}
 				return et
-			case ast.AlterTableRenameColumn, ast.AlterTableRenameIndex, ast.AlterTableRenameTable:
-				return bf.Rename
-			case ast.AlterTableDropColumn, ast.AlterTableDropIndex, ast.AlterTableDropPartition:
-				return bf.Drop
+			case ast.AlterTableRenameColumn:
+				return bf.RenameColumn
+			case ast.AlterTableRenameIndex:
+				return bf.RenameIndex
+			case ast.AlterTableRenameTable:
+				return bf.RenameTable
+			case ast.AlterTableDropColumn:
+				return bf.DropColumn
+			case ast.AlterTableDropIndex:
+				return bf.DropIndex
+			case ast.AlterTableDropPartition:
+				return bf.DropTablePartition
 			case ast.AlterTableDropPrimaryKey:
-				return bf.ModifyPK
+				return bf.DropPrimaryKey
 			case ast.AlterTableTruncatePartition:
-				return bf.Truncate
+				return bf.TruncateTablePartition
 			case ast.AlterTableAlterColumn:
 				return bf.ModifyDefaultValue
 			case ast.AlterTableAddConstraint:

dm/tests/incompatible_ddl_changes/conf/dm-task1.yaml

@@ -23,26 +23,34 @@ filters:
     table-pattern: "*"
     events:
       [
+        "drop database",
+        "drop table",
+        "drop index",
+        "rename table",
+        "truncate table",
+        "drop table partition",
+        "truncate table partition",
         "value range decrease",
         "precision decrease",
         "modify column",
-        "rename",
-        "drop",
-        "truncate",
-        "modify pk",
-        "modify uk",
+        "rename column",
+        "rename index",
+        "drop column",
+        "truncate table partition",
+        "drop primary key",
+        "drop unique key",
         "modify default value",
         "modify constaints",
         "modify columns order",
         "modify charset",
         "modify collation",
         "remove auto increment",
         "modify storage engine",
-        "reorganize partition",
-        "coalesce partition",
-        "split partition",
-        "exchange partition",
-        "rebuild partition",
+        "reorganize table partition",
+        "coalesce table partition",
+        "split table partition",
+        "exchange table partition",
+        "rebuild table partition",
       ]
     sql-pattern: []
     action: Error

dm/tests/incompatible_ddl_changes/run.sh

@@ -74,23 +74,23 @@ function incompatible_ddl() {
 	run_sql_source1 "alter table incompatible_ddl_changes.t1 change c_mediumint c_mediumint_new mediumint(7);"
 	run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \
 		"query-status test" \
-		"event rename" 1
+		"event rename column" 1
 	run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \
 		"binlog skip test" \
 		"\"result\": true" 2
 	run_sql_source1 "alter table incompatible_ddl_changes.t1 change c_mediumint_new c_mediumint mediumint(7);"
 	run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \
 		"query-status test" \
-		"event rename" 1
+		"event rename column" 1
 	run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \
 		"binlog skip test" \
 		"\"result\": true" 2
 
-	# drop
+	# drop column
 	run_sql_source1 "alter table incompatible_ddl_changes.t1 drop column c_json;"
 	run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \
 		"query-status test" \
-		"event drop" 1
+		"event drop column" 1
 	run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \
 		"binlog skip test" \
 		"\"result\": true" 2
@@ -105,11 +105,11 @@ function incompatible_ddl() {
 		"binlog skip test" \
 		"\"result\": true" 2
 
-	# modify pk
+	# drop pk
 	run_sql_source1 "alter table incompatible_ddl_changes.t1 drop primary key"
 	run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \
 		"query-status test" \
-		"event modify pk" 1
+		"event drop primary key" 1
 	run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \
 		"binlog skip test" \
 		"\"result\": true" 2
@@ -132,18 +132,19 @@ function incompatible_ddl() {
 		"binlog skip test" \
 		"\"result\": true" 2
 
-	# modify uk
+	# drop uk
 	run_sql_source1 "alter table incompatible_ddl_changes.t1 change c_int c_int int"
 	run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \
 		"query-status test" \
-		"modify uk" 1
+		"drop unique key" 1
 	run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \
 		"binlog skip test" \
 		"\"result\": true" 2
+	# drop index
 	run_sql_source1 "alter table incompatible_ddl_changes.t1 drop index c_int_unique"
 	run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \
 		"query-status test" \
-		"event drop" 1
+		"event drop index" 1
 	run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \
 		"binlog skip test" \
 		"\"result\": true" 2
@@ -189,16 +190,16 @@ function incompatible_ddl() {
 	run_sql_source1 "alter table incompatible_ddl_changes.t1 reorganize partition p0 into ( partition n0 values less than (5), partition n1 values less than (100000));"
 	run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \
 		"query-status test" \
-		"event reorganize partition" 1
+		"event reorganize table partition" 1
 	run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \
 		"binlog skip test" \
 		"\"result\": true" 2
 
-	# truncate
+	# truncate partition
 	run_sql_source1 "alter table incompatible_ddl_changes.t1 truncate partition n0"
 	run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \
 		"query-status test" \
-		"event truncate" 1
+		"event truncate table partition" 1
 	run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \
 		"binlog skip test" \
 		"\"result\": true" 2
@@ -207,18 +208,18 @@ function incompatible_ddl() {
 	run_sql_source1 "alter table incompatible_ddl_changes.t1 rebuild partition n0,n1;"
 	run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \
 		"query-status test" \
-		"event rebuild partition" 1
+		"event rebuild table partition" 1
 	run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \
 		"binlog skip test" \
 		"\"result\": true" 2
 
 	# exchange partition
-	run_sql_source1 "create table incompatible_ddl_changes.tb1(id int) partition by range(id)(partition p0 values less than (100000));"
+	run_sql_source1 "create table incompatible_ddl_changes.tb1(id int) partition by range(id)(partition p0 values less than (100000), partition p1 values less than(1000000));"
 	run_sql_source1 "create table incompatible_ddl_changes.tb2(id int);"
 	run_sql_source1 "alter table incompatible_ddl_changes.tb1 exchange partition p0 with table incompatible_ddl_changes.tb2"
 	run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \
 		"query-status test" \
-		"event exchange partition" 1
+		"event exchange table partition" 1
 	run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \
 		"binlog skip test" \
 		"\"result\": true" 2
@@ -228,7 +229,68 @@ function incompatible_ddl() {
 	run_sql_source1 "alter table incompatible_ddl_changes.tb3 coalesce partition 2;"
 	run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \
 		"query-status test" \
-		"event coalesce partition" 1
+		"event coalesce table partition" 1
+	run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \
+		"binlog skip test" \
+		"\"result\": true" 2
+
+	# rename index
+	run_sql_source1 "alter table incompatible_ddl_changes.tb3 add index idx(id);"
+	run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \
+		"query-status test" \
+		"event modify constaints" 1
+	run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \
+		"binlog skip test" \
+		"\"result\": true" 2
+	run_sql_source1 "alter table incompatible_ddl_changes.tb3 rename index idx to idx1;"
+	run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \
+		"query-status test" \
+		"event rename index" 1
+	run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \
+		"binlog skip test" \
+		"\"result\": true" 2
+
+	# drop partition
+	run_sql_source1 "alter table incompatible_ddl_changes.tb1 drop partition p0;"
+	run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \
+		"query-status test" \
+		"event drop table partition" 1
+	run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \
+		"binlog skip test" \
+		"\"result\": true" 2
+
+	# rename table
+	run_sql_source1 "rename table incompatible_ddl_changes.tb3 to incompatible_ddl_changes.tb4;"
+	run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \
+		"query-status test" \
+		"event rename table" 1
+	run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \
+		"binlog skip test" \
+		"\"result\": true" 2
+
+	# truncate table
+	run_sql_source1 "truncate table incompatible_ddl_changes.tb4;"
+	run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \
+		"query-status test" \
+		"event truncate table" 1
+	run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \
+		"binlog skip test" \
+		"\"result\": true" 2
+
+	# drop table
+	run_sql_source1 "drop table incompatible_ddl_changes.tb4;"
+	run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \
+		"query-status test" \
+		"event drop table" 1
+	run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \
+		"binlog skip test" \
+		"\"result\": true" 2
+
+	# drop database
+	run_sql_source1 "drop database incompatible_ddl_changes;"
+	run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \
+		"query-status test" \
+		"event drop database" 1
 	run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \
 		"binlog skip test" \
 		"\"result\": true" 2

go.mod

@@ -69,7 +69,7 @@ require (
 	github.com/pingcap/kvproto v0.0.0-20230925123611-87bebcc0d071
 	github.com/pingcap/log v1.1.1-0.20230317032135-a0d097d16e22
 	github.com/pingcap/tidb v1.1.0-beta.0.20231010115255-ec2731b8f539
-	github.com/pingcap/tidb-tools v0.0.0-20231010140451-6189b1bea2fd
+	github.com/pingcap/tidb-tools v0.0.0-20231031074420-a020d2cdfc5a
 	github.com/pingcap/tidb/parser v0.0.0-20231010115255-ec2731b8f539
 	github.com/prometheus/client_golang v1.16.0
 	github.com/prometheus/client_model v0.4.0

go.sum

@@ -1047,8 +1047,8 @@ github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21/go.mod h1:QYnjfA
 github.com/pingcap/tidb v1.1.0-beta.0.20220511160835-98c31070d958/go.mod h1:luW4sIZoLHY3bCWuKqyqk2QgMvF+/M7nWOXf/me0+fY=
 github.com/pingcap/tidb v1.1.0-beta.0.20231010115255-ec2731b8f539 h1:Pw+tqB5BeNxn8uRQh6XZgU/MmdaSQiXtioStsxiu7M4=
 github.com/pingcap/tidb v1.1.0-beta.0.20231010115255-ec2731b8f539/go.mod h1:K9GzvaQZ9ht52B81kDXqvtW8p5rkQmP8rgJ8AW8yXCo=
-github.com/pingcap/tidb-tools v0.0.0-20231010140451-6189b1bea2fd h1:0BfNSfxiQgTD+YA4a3dmb7zR7nyVMOZB4qknn1y8t9s=
-github.com/pingcap/tidb-tools v0.0.0-20231010140451-6189b1bea2fd/go.mod h1:Awem9vQiEdvfdGLEbO6HWjR1jh2vFOCjKkwZdoHXfVI=
+github.com/pingcap/tidb-tools v0.0.0-20231031074420-a020d2cdfc5a h1:9MXDt1cSyG8eLDZj0dv1NZrT86BKjU8KxPalIS7KQRQ=
+github.com/pingcap/tidb-tools v0.0.0-20231031074420-a020d2cdfc5a/go.mod h1:Awem9vQiEdvfdGLEbO6HWjR1jh2vFOCjKkwZdoHXfVI=
 github.com/pingcap/tidb/parser v0.0.0-20211011031125-9b13dc409c5e/go.mod h1:e1MGCA9Sg3T8jid8PKAEq5eYVuMMCq4n8gJ+Kqp4Plg=
 github.com/pingcap/tidb/parser v0.0.0-20220511160835-98c31070d958/go.mod h1:ElJiub4lRy6UZDb+0JHDkGEdr6aOli+ykhyej7VCLoI=
 github.com/pingcap/tidb/parser v0.0.0-20231010115255-ec2731b8f539 h1:q9m7W3fJRAgmIwRckzxxrqWX/QRuQrXnVJSc5do1New=