Skip to content

Commit

Permalink
[fix](delete) Fix delete stmt on MOW table doesn't use partial update…
Browse files Browse the repository at this point in the history 
… in Nereids planner (apache#38751)

## Proposed changes

1. Fix nereids planner don't use partial update for delete statement on
merge-on-write table introduced in
apache#36782
2. add `IS_PARTIAL_UPDATE: true` in `OlapTableSink.getExplainString()`
3. don't convert the delete stmt to partial update if the table has
cluster key.
  • Loading branch information
@bobhan1 @wyxxxcat
bobhan1 authored and wyxxxcat committed Aug 14, 2024
1 parent 1af96f1 commit 8e22b34
Show file tree
Hide file tree
Showing 27 changed files with 164 additions and 107 deletions.
20 changes: 13 additions & 7 deletions ...e-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DeleteFromCommand.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -400,26 +400,32 @@ public LogicalPlan completeQueryPlan(ConnectContext ctx, LogicalPlan logicalQuer
List<String> cols = Lists.newArrayList();
boolean isMow = targetTable.getEnableUniqueKeyMergeOnWrite();
String tableName = tableAlias != null ? tableAlias : targetTable.getName();
boolean hasClusterKey = targetTable.getBaseSchema().stream().anyMatch(Column::isClusterKey);
// currently cluster key doesn't support partial update, so we can't convert
// a delete stmt to partial update load if the table has cluster key
for (Column column : targetTable.getFullSchema()) {
NamedExpression expr = null;
if (column.getName().equalsIgnoreCase(Column.DELETE_SIGN)) {
selectLists.add(new UnboundAlias(new TinyIntLiteral(((byte) 1)), Column.DELETE_SIGN));
expr = new UnboundAlias(new TinyIntLiteral(((byte) 1)), Column.DELETE_SIGN);
} else if (column.getName().equalsIgnoreCase(Column.SEQUENCE_COL)
&& targetTable.getSequenceMapCol() != null) {
selectLists.add(new UnboundSlot(tableName, targetTable.getSequenceMapCol()));
expr = new UnboundSlot(tableName, targetTable.getSequenceMapCol());
} else if (column.isKey()) {
selectLists.add(new UnboundSlot(tableName, column.getName()));
expr = new UnboundSlot(tableName, column.getName());
} else if (!isMow && (!column.isVisible() || (!column.isAllowNull() && !column.hasDefaultValue()))) {
selectLists.add(new UnboundSlot(tableName, column.getName()));
expr = new UnboundSlot(tableName, column.getName());
} else if (hasClusterKey) {
expr = new UnboundSlot(tableName, column.getName());
} else {
selectLists.add(new UnboundSlot(tableName, column.getName()));
continue;
}
selectLists.add(expr);
cols.add(column.getName());
}

logicalQuery = new LogicalProject<>(selectLists, logicalQuery);

boolean isPartialUpdate = targetTable.getEnableUniqueKeyMergeOnWrite()
&& cols.size() < targetTable.getColumns().size();
boolean isPartialUpdate = isMow && !hasClusterKey && cols.size() < targetTable.getColumns().size();
logicalQuery = handleCte(logicalQuery);
// make UnboundTableSink
return UnboundTableSinkCreator.createUnboundTableSink(nameParts, cols, ImmutableList.of(),
Expand Down
1 change: 1 addition & 0 deletions fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -242,6 +242,7 @@ public String getExplainString(String prefix, TExplainLevel explainLevel) {
}
strBuilder.append(prefix + " TUPLE ID: " + tupleDescriptor.getId() + "\n");
strBuilder.append(prefix + " " + DataPartition.RANDOM.getExplainString(explainLevel));
strBuilder.append(prefix + " IS_PARTIAL_UPDATE: " + isPartialUpdate);
return strBuilder.toString();
}

Expand Down
4 changes: 2 additions & 2 deletions regression-test/data/compaction/test_full_compaction.out
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,12 +32,12 @@
2 2
2 20
2 200
3 100
3 0
3 100
3 300

-- !select_final --
1 100
2 200
3 100
3 0

4 changes: 2 additions & 2 deletions regression-test/data/compaction/test_full_compaction_by_table_id.out
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,12 +32,12 @@
2 2
2 20
2 200
3 100
3 0
3 100
3 300

-- !select_final --
1 100
2 200
3 100
3 0

48 changes: 20 additions & 28 deletions ...t/data/delete_p0/test_delete_on_value.out → ...ete_p0/test_delete_predicate_on_value.out
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !sql --
-- !sql_1 --
1 1 1
2 2 2
3 3 3
Expand All @@ -10,7 +10,7 @@
8 8 8
9 9 9

-- !sql --
-- !sql_1 --
1 1 1
2 2 2
3 3 3
Expand All @@ -20,77 +20,69 @@
8 8 8
9 9 9

-- !sql --
-- !sql_1 --
1 1 1
2 2 2
8 8 8
9 9 9

-- !sql --
-- !skip_delete_predicate_sql_1 --
1 1 1 0
2 2 2 0
3 3 3 0
4 4 4 0
5 5 5 0
6 6 6 0
7 7 7 0
8 8 8 0
9 9 9 0

-- !sql --
-- !sql_1 --
1 1 1
2 2 2
4 4 4
5 5 5
8 8 8
9 9 9

-- !sql --
-- !sql_1 --
1 1 1
2 2 2
4 4 4
8 8 8
9 9 9

-- !sql --
-- !skip_delete_predicate_sql_1 --
1 1 1 0
2 2 2 0
3 3 3 0
3 3 3 1
4 4 4 0
4 4 4 0
4 4 4 1
5 5 5 0
5 5 5 0
5 5 5 1
5 5 5 1
6 6 6 0
6 6 6 1
7 7 7 0
7 7 7 1
8 8 8 0
9 9 9 0

-- !sql --
-- !sql_3 --
1 1 10

-- !sql --
-- !skip_delete_predicate_sql_3 --
1 1 5 0 3 5
1 1 10 0 2 10

-- !sql --
-- !sql_3 --

-- !sql --
-- !skip_delete_predicate_sql_3 --
1 1 5 0 3 5
1 1 10 0 2 10
1 1 10 1 4 10

-- !sql --
-- !sql_4 --
1 1 10

-- !sql --
1 1 5 0 3 5
1 1 10 0 2 10

-- !sql --
-- !sql_4 --

-- !sql --
-- !skip_delete_predicate_sql_4 --
1 \N \N 1 4 10
1 1 5 0 3 5
1 1 10 0 2 10
1 1 10 1 4 10

24 changes: 18 additions & 6 deletions regression-test/data/nereids_p0/delete/delete_mow_partial_update.out
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,20 +7,26 @@
5 5

-- !sql --
2 2
4 4
5 5

-- !sql_skip_delete_predicate --
2 2
4 4
5 5

-- !sql --
2 2
5 5

-- !sql --
1 \N 1
1 1 0
1 1 1
2 2 0
2 2 1
3 \N 1
3 3 0
3 3 1
4 \N 1
4 4 0
5 5 0

Expand Down Expand Up @@ -53,20 +59,26 @@
5 5

-- !sql --
2 2
4 4
5 5

-- !sql_skip_delete_predicate --
2 2
4 4
5 5

-- !sql --
2 2
5 5

-- !sql --
1 \N 1
1 1 0
1 1 1
2 2 0
2 2 1
3 \N 1
3 3 0
3 3 1
4 \N 1
4 4 0
5 5 0

Expand Down
4 changes: 2 additions & 2 deletions regression-test/data/unique_with_mow_p0/partial_update/test_new_partial_update_delete.out
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
-- !sql2 --

-- !sql3 --
1 1 1 1 1 1
1 \N \N \N \N 1

-- !sql4 --
1 2 \N \N \N
Expand Down Expand Up @@ -57,7 +57,7 @@
-- !sql2 --

-- !sql3 --
1 1 1 1 1 1
1 \N \N \N \N 1

-- !sql4 --
1 2 \N \N \N
Expand Down
22 changes: 16 additions & 6 deletions regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_delete.out
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,17 +6,22 @@
4 4 4 4 4
5 5 5 5 5

-- !sql --
2 2 2 2 2
4 4 4 4 4
5 5 5 5 5

-- !sql --
4 4 4 4 4
5 5 5 5 5

-- !with_delete_sign --
1 \N \N 0 \N 1
1 1 1 1 1 0
1 1 1 1 1 1
2 \N \N 0 \N 1
2 2 2 2 2 0
2 2 2 2 2 1
3 \N \N 0 \N 1
3 3 3 3 3 0
3 3 3 3 3 1
4 4 4 4 4 0
5 5 5 5 5 0

Expand Down Expand Up @@ -48,17 +53,22 @@
4 4 4 4 4
5 5 5 5 5

-- !sql --
2 2 2 2 2
4 4 4 4 4
5 5 5 5 5

-- !sql --
4 4 4 4 4
5 5 5 5 5

-- !with_delete_sign --
1 \N \N 0 \N 1
1 1 1 1 1 0
1 1 1 1 1 1
2 \N \N 0 \N 1
2 2 2 2 2 0
2 2 2 2 2 1
3 \N \N 0 \N 1
3 3 3 3 3 0
3 3 3 3 3 1
4 4 4 4 4 0
5 5 5 5 5 0

Expand Down
3 changes: 2 additions & 1 deletion regression-test/suites/compaction/test_compaction_uniq_cluster_keys_with_delete.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,8 @@ suite("test_compaction_uniq_cluster_keys_with_delete") {
DISTRIBUTED BY HASH(`user_id`)
PROPERTIES (
"replication_num" = "1",
"enable_unique_key_merge_on_write" = "true"
"enable_unique_key_merge_on_write" = "true",
"enable_mow_light_delete" = "true"
);
"""

Expand Down
Loading

0 comments on commit 8e22b34

Please sign in to comment.