Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

expression_filter(dm): fix append update filter for non-update item #7851

Merged
merged 4 commits into from
Dec 14, 2022
Merged

expression_filter(dm): fix append update filter for non-update item #7851

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
60fe6a6
expression_filter(dm): fix append update filter for insert-only item
lance6716 Dec 8, 2022
3c90190
fix lint
lance6716 Dec 8, 2022
5f097cd
Merge branch 'master' into fix7831
ti-chi-bot Dec 14, 2022
b4a3411
Merge branch 'master' into fix7831
ti-chi-bot Dec 14, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions dm/syncer/expr_filter_group.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,9 @@ func (g *ExprFilterGroup) GetUpdateExprs(table *filter.Table, ti *model.TableInf

if _, ok := g.hasUpdateFilter[tableID]; ok {
for _, c := range g.configs[tableID] {
if c.UpdateOldValueExpr == "" && c.UpdateNewValueExpr == "" {
continue
}
if c.UpdateOldValueExpr != "" {
expr, err := getSimpleExprOfTable(g.tidbCtx, c.UpdateOldValueExpr, ti, g.logCtx.L())
if err != nil {
Expand Down
9 changes: 7 additions & 2 deletions dm/syncer/expr_filter_group_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -497,8 +497,13 @@ create table t (
for i, c := range cases {
t.Logf("case #%d", i)
g := NewExprFilterGroup(tcontext.Background(), sessCtx, []*config.ExpressionFilter{c})
oldExprs, newExprs, err := g.GetUpdateExprs(table, tableInfo)
require.NoError(t, err)
oldExprs, newExprs, err2 := g.GetUpdateExprs(table, tableInfo)
require.NoError(t, err2)
require.Equal(t, len(oldExprs), len(newExprs))
}
g := NewExprFilterGroup(tcontext.Background(), sessCtx, cases)
oldExprs, newExprs, err := g.GetUpdateExprs(table, tableInfo)
require.NoError(t, err)
require.Equal(t, len(oldExprs), len(newExprs))
require.Len(t, oldExprs, 3)
}
6 changes: 6 additions & 0 deletions dm/tests/expression_filter/conf/dm-task2.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ mysql-instances:
black-white-list: "instance"
expression-filters:
- "even_c"
- "always_false"
- "future_date"
- "pythagorean"
- "update_old_lt_100"
Expand All @@ -26,6 +27,11 @@ expression-filter:
schema: "expr_filter"
table: "t2"
insert-value-expr: "c % 2 = 0"
always_false:
schema: "expr_filter"
table: "t2"
update-old-value-expr: "1 = 0"
update-new-value-expr: "1 = 0"
Comment on lines +30 to +34
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you explain this configuration? I did not figure out why added it.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this is a FALSE expression, so it should not skip any UPDATE. But before we fix the issue it will skip

future_date:
schema: "expr_filter"
table: "t3"
Expand Down
3 changes: 2 additions & 1 deletion dm/tests/expression_filter/data/db1.increment2.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,8 @@ alter table t2 drop column c;
insert into t2 (id, should_skip) values (5, 0), (6, 0);
-- test filter become valid again, and the checking column is a generated column
alter table t2 add column c int as (id + 1);
insert into t2 (id, should_skip, d) values (7, 1, 100), (8, 0, 200);
insert into t2 (id, should_skip, d) values (7, 1, 100), (8, 0, 200), (10, 2, 300);
update t2 set should_skip = 0 where id = 10;

-- test a new created table
create table t3 (id int primary key,
Expand Down
6 changes: 4 additions & 2 deletions dm/tests/expression_filter/run.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,8 +40,10 @@ function complex_behaviour() {
"\"result\": true" 2 \
"\"synced\": true" 1

run_sql_tidb "select count(1) from expr_filter.t2"
check_contains "count(1): 5"
run_sql_tidb "select count(0) from expr_filter.t2"
check_contains "count(0): 6"
run_sql_tidb "select count(1) from expr_filter.t2 where should_skip = 0"
check_contains "count(1): 6"
run_sql_tidb "select count(2) from expr_filter.t2 where should_skip = 1"
check_contains "count(2): 0"

Expand Down