expression_filter(dm): fix append update filter for non-update item (#…

…7851) (#7906)

close #7831

dm/syncer/expr_filter_group.go

@@ -112,6 +112,9 @@ func (g *ExprFilterGroup) GetUpdateExprs(table *filter.Table, ti *model.TableInf
 
 	if _, ok := g.hasUpdateFilter[tableID]; ok {
 		for _, c := range g.configs[tableID] {
+			if c.UpdateOldValueExpr == "" && c.UpdateNewValueExpr == "" {
+				continue
+			}
 			if c.UpdateOldValueExpr != "" {
 				expr, err := getSimpleExprOfTable(g.ctx, c.UpdateOldValueExpr, ti)
 				if err != nil {

dm/syncer/expr_filter_group_test.go

@@ -18,7 +18,6 @@ import (
 
 	. "github.com/pingcap/check"
 	"github.com/pingcap/tidb/util/filter"
-
 	"github.com/pingcap/tiflow/dm/dm/config"
 	"github.com/pingcap/tiflow/dm/pkg/log"
 	"github.com/pingcap/tiflow/dm/pkg/schema"
@@ -443,3 +442,67 @@ create table t (
 	c.Assert(err, IsNil)
 	c.Assert(skip, Equals, false)
 }
+
+func (s *testFilterSuite) TestGetUpdateExprsSameLength(c *C) {
+	var (
+		ctx     = context.Background()
+		dbName  = "test"
+		tblName = "t"
+		table   = &filter.Table{
+			Schema: dbName,
+			Name:   tblName,
+		}
+		tableStr = `
+create table t (
+	c varchar(20)
+);`
+		exprStr = "c > 1"
+		sessCtx = utils.NewSessionCtx(map[string]string{"time_zone": "UTC"})
+	)
+
+	cases := []*config.ExpressionFilter{
+		{
+			Schema:          dbName,
+			Table:           tblName,
+			InsertValueExpr: exprStr,
+		},
+		{
+			Schema:             dbName,
+			Table:              tblName,
+			UpdateOldValueExpr: exprStr,
+		},
+		{
+			Schema:             dbName,
+			Table:              tblName,
+			UpdateNewValueExpr: exprStr,
+		},
+		{
+			Schema:             dbName,
+			Table:              tblName,
+			UpdateOldValueExpr: exprStr,
+			UpdateNewValueExpr: exprStr,
+		},
+	}
+
+	dbConn := dbconn.NewDBConn(&config.SubTaskConfig{}, s.baseConn)
+	schemaTracker, err := schema.NewTracker(ctx, "unit-test", defaultTestSessionCfg, dbConn)
+	c.Assert(err, IsNil)
+	c.Assert(schemaTracker.CreateSchemaIfNotExists(dbName), IsNil)
+	c.Assert(schemaTracker.Exec(ctx, dbName, tableStr), IsNil)
+
+	tableInfo, err := schemaTracker.GetTableInfo(table)
+	c.Assert(err, IsNil)
+
+	for i, ca := range cases {
+		c.Logf("case #%d", i)
+		g := NewExprFilterGroup(sessCtx, []*config.ExpressionFilter{ca})
+		oldExprs, newExprs, err2 := g.GetUpdateExprs(table, tableInfo)
+		c.Assert(err2, IsNil)
+		c.Assert(len(oldExprs), Equals, len(newExprs))
+	}
+	g := NewExprFilterGroup(sessCtx, cases)
+	oldExprs, newExprs, err := g.GetUpdateExprs(table, tableInfo)
+	c.Assert(err, IsNil)
+	c.Assert(len(oldExprs), Equals, len(newExprs))
+	c.Assert(len(oldExprs), Equals, 3)
+}

dm/tests/expression_filter/conf/dm-task2.yaml

@@ -13,6 +13,7 @@ mysql-instances:
     black-white-list: "instance"
     expression-filters:
       - "even_c"
+      - "always_false"
       - "future_date"
       - "pythagorean"
       - "update_old_lt_100"
@@ -26,6 +27,11 @@ expression-filter:
     schema: "expr_filter"
     table: "t2"
     insert-value-expr: "c % 2 = 0"
+  always_false:
+    schema: "expr_filter"
+    table: "t2"
+    update-old-value-expr: "1 = 0"
+    update-new-value-expr: "1 = 0"
   future_date:
     schema: "expr_filter"
     table: "t3"

dm/tests/expression_filter/data/db1.increment2.sql

@@ -10,7 +10,8 @@ alter table t2 drop column c;
 insert into t2 (id, should_skip) values (5, 0), (6, 0);
 -- test filter become valid again, and the checking column is a generated column
 alter table t2 add column c int as (id + 1);
-insert into t2 (id, should_skip, d) values (7, 1, 100), (8, 0, 200);
+insert into t2 (id, should_skip, d) values (7, 1, 100), (8, 0, 200), (10, 2, 300);
+update t2 set should_skip = 0 where id = 10;
 
 -- test a new created table
 create table t3 (id int primary key,

dm/tests/expression_filter/run.sh

@@ -40,8 +40,10 @@ function complex_behaviour() {
 		"\"result\": true" 2 \
 		"\"synced\": true" 1
 
-	run_sql_tidb "select count(1) from expr_filter.t2"
-	check_contains "count(1): 5"
+	run_sql_tidb "select count(0) from expr_filter.t2"
+	check_contains "count(0): 6"
+	run_sql_tidb "select count(1) from expr_filter.t2 where should_skip = 0"
+	check_contains "count(1): 6"
 	run_sql_tidb "select count(2) from expr_filter.t2 where should_skip = 1"
 	check_contains "count(2): 0"