Skip to content

Commit 2f19e95

Browse files
morrySnowmorrySnow
authored
branch-4.0: [feat](load) support merge into #57044 #58091 (#58219)
picked from #57044 #58091
1 parent 516e652 commit 2f19e95

17 files changed

Lines changed: 2066 additions & 10 deletions

File tree

‎fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -340,6 +340,7 @@ LOW_PRIORITY: 'LOW_PRIORITY';
340340
MANUAL: 'MANUAL';
341341
MAP: 'MAP';
342342
MATCH: 'MATCH';
343+
MATCHED: 'MATCHED';
343344
MATCH_ALL: 'MATCH_ALL';
344345
MATCH_ANY: 'MATCH_ANY';
345346
MATCH_NAME: 'MATCH_NAME';

‎fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -145,6 +145,10 @@ supportedDmlStatement
145145
partitionSpec? tableAlias
146146
(USING relations)?
147147
whereClause? #delete
148+
| explain? cte? MERGE INTO targetTable=multipartIdentifier
149+
(AS? identifier)? USING srcRelation=relationPrimary
150+
ON expression
151+
(mergeMatchedClause | mergeNotMatchedClause)+ #mergeInto
148152
| LOAD LABEL lableName=multipartIdentifier
149153
LEFT_PAREN dataDescs+=dataDesc (COMMA dataDescs+=dataDesc)* RIGHT_PAREN
150154
(withRemoteStorageSystem)?
@@ -164,6 +168,16 @@ supportedDmlStatement
164168
| TRUNCATE TABLE multipartIdentifier specifiedPartition? FORCE? #truncateTable
165169
;
166170

171+
mergeMatchedClause
172+
: WHEN MATCHED (AND casePredicate=expression)? THEN
173+
(UPDATE SET updateAssignmentSeq | DELETE)
174+
;
175+
176+
mergeNotMatchedClause
177+
: WHEN NOT MATCHED (AND casePredicate=expression)? THEN
178+
INSERT cols=identifierList? VALUES rowConstructor
179+
;
180+
167181
supportedCreateStatement
168182
: CREATE (EXTERNAL | TEMPORARY)? TABLE (IF NOT EXISTS)? name=multipartIdentifier
169183
((ctasCols=identifierList)? | (LEFT_PAREN columnDefs (COMMA indexDefs)? COMMA? RIGHT_PAREN))
@@ -2057,6 +2071,7 @@ nonReserved
20572071
| LOGICAL
20582072
| MANUAL
20592073
| MAP
2074+
| MATCHED
20602075
| MATCH_ALL
20612076
| MATCH_ANY
20622077
| MATCH_PHRASE

‎fe/fe-core/src/main/java/org/apache/doris/analysis/StmtType.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ public enum StmtType {
3838
INSTALL,
3939
KILL,
4040
LOAD,
41+
MERGE_INTO,
4142
OTHER,
4243
OPTIMIZE,
4344
PAUSE,

‎fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1288,6 +1288,10 @@ public String tryGetBaseColumnName() {
12881288
return colName;
12891289
}
12901290

1291+
public boolean isGeneratedColumn() {
1292+
return generatedColumnInfo != null;
1293+
}
1294+
12911295
public GeneratedColumnInfo getGeneratedColumnInfo() {
12921296
return generatedColumnInfo;
12931297
}

‎fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1691,6 +1691,10 @@ public boolean hasHiddenColumn() {
16911691
return getBaseSchema().stream().anyMatch(column -> !column.isVisible());
16921692
}
16931693

1694+
public boolean hasGeneratedColumn() {
1695+
return getBaseSchema().stream().anyMatch(Column::isGeneratedColumn);
1696+
}
1697+
16941698
public Type getSequenceType() {
16951699
if (getSequenceCol() == null) {
16961700
return null;

‎fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java‎

Lines changed: 50 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -256,6 +256,9 @@
256256
import org.apache.doris.nereids.DorisParser.LogicalBinaryContext;
257257
import org.apache.doris.nereids.DorisParser.LogicalNotContext;
258258
import org.apache.doris.nereids.DorisParser.MapLiteralContext;
259+
import org.apache.doris.nereids.DorisParser.MergeIntoContext;
260+
import org.apache.doris.nereids.DorisParser.MergeMatchedClauseContext;
261+
import org.apache.doris.nereids.DorisParser.MergeNotMatchedClauseContext;
259262
import org.apache.doris.nereids.DorisParser.ModifyColumnClauseContext;
260263
import org.apache.doris.nereids.DorisParser.ModifyColumnCommentClauseContext;
261264
import org.apache.doris.nereids.DorisParser.ModifyDistributionClauseContext;
@@ -1000,6 +1003,9 @@
10001003
import org.apache.doris.nereids.trees.plans.commands.load.ResumeRoutineLoadCommand;
10011004
import org.apache.doris.nereids.trees.plans.commands.load.ShowCreateRoutineLoadCommand;
10021005
import org.apache.doris.nereids.trees.plans.commands.load.StopRoutineLoadCommand;
1006+
import org.apache.doris.nereids.trees.plans.commands.merge.MergeIntoCommand;
1007+
import org.apache.doris.nereids.trees.plans.commands.merge.MergeMatchedClause;
1008+
import org.apache.doris.nereids.trees.plans.commands.merge.MergeNotMatchedClause;
10031009
import org.apache.doris.nereids.trees.plans.commands.refresh.RefreshCatalogCommand;
10041010
import org.apache.doris.nereids.trees.plans.commands.refresh.RefreshDatabaseCommand;
10051011
import org.apache.doris.nereids.trees.plans.commands.refresh.RefreshDictionaryCommand;
@@ -1372,6 +1378,49 @@ public LogicalPlan visitInsertTable(InsertTableContext ctx) {
13721378
return withExplain(command, ctx.explain());
13731379
}
13741380

1381+
@Override
1382+
public Object visitMergeInto(MergeIntoContext ctx) {
1383+
return ParserUtils.withOrigin(ctx, () -> {
1384+
List<String> targetNameParts = visitMultipartIdentifier(ctx.targetTable);
1385+
Optional<String> targetAlias = Optional.ofNullable(
1386+
ctx.identifier() != null ? ctx.identifier().getText() : null);
1387+
LogicalPlan source = plan(ctx.relationPrimary());
1388+
Expression onClause = typedVisit(ctx.expression());
1389+
List<MergeMatchedClause> matchedClauses = visit(ctx.mergeMatchedClause(), MergeMatchedClause.class);
1390+
List<MergeNotMatchedClause> notMatchedClauses = visit(ctx.mergeNotMatchedClause(),
1391+
MergeNotMatchedClause.class);
1392+
Optional<LogicalPlan> cte = Optional.empty();
1393+
if (ctx.cte() != null) {
1394+
cte = Optional.ofNullable(withCte(source, ctx.cte()));
1395+
}
1396+
return withExplain(new MergeIntoCommand(targetNameParts, targetAlias, cte,
1397+
source, onClause, matchedClauses, notMatchedClauses), ctx.explain());
1398+
});
1399+
}
1400+
1401+
@Override
1402+
public MergeMatchedClause visitMergeMatchedClause(MergeMatchedClauseContext ctx) {
1403+
return ParserUtils.withOrigin(ctx, () -> {
1404+
Optional<Expression> casePredicate = Optional.ofNullable(
1405+
ctx.casePredicate != null ? typedVisit(ctx.casePredicate) : null);
1406+
boolean isDelete = ctx.DELETE() != null;
1407+
List<EqualTo> updateAssignments = isDelete ? ImmutableList.of() :
1408+
visitUpdateAssignmentSeq(ctx.updateAssignmentSeq());
1409+
return new MergeMatchedClause(casePredicate, updateAssignments, isDelete);
1410+
});
1411+
}
1412+
1413+
@Override
1414+
public MergeNotMatchedClause visitMergeNotMatchedClause(MergeNotMatchedClauseContext ctx) {
1415+
return ParserUtils.withOrigin(ctx, () -> {
1416+
Optional<Expression> casePredicate = Optional.ofNullable(
1417+
ctx.casePredicate != null ? typedVisit(ctx.casePredicate) : null);
1418+
List<String> cols = ctx.cols != null ? visitIdentifierList(ctx.cols) : ImmutableList.of();
1419+
List<NamedExpression> row = visitRowConstructor(ctx.rowConstructor());
1420+
return new MergeNotMatchedClause(casePredicate, cols, row);
1421+
});
1422+
}
1423+
13751424
/**
13761425
* return a pair, first will be true if partitions is temp partition, select is a list to present partition list.
13771426
*/
@@ -2445,7 +2494,7 @@ public LogicalPlan visitRegularQuerySpecification(RegularQuerySpecificationConte
24452494
}
24462495

24472496
@Override
2448-
public LogicalPlan visitInlineTable(InlineTableContext ctx) {
2497+
public UnboundInlineTable visitInlineTable(InlineTableContext ctx) {
24492498
List<RowConstructorContext> rowConstructorContexts = ctx.rowConstructor();
24502499
ImmutableList.Builder<List<NamedExpression>> rows
24512500
= ImmutableList.builderWithExpectedSize(rowConstructorContexts.size());

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -354,7 +354,7 @@ private static Map<String, NamedExpression> getColumnToOutput(
354354
List<Column> shadowColumns = Lists.newArrayList();
355355
// generate slots not mentioned in sql, mv slots and shaded slots.
356356
for (Column column : boundSink.getTargetTable().getFullSchema()) {
357-
if (column.getGeneratedColumnInfo() != null) {
357+
if (column.isGeneratedColumn()) {
358358
generatedColumns.add(column);
359359
continue;
360360
} else if (column.isMaterializedViewColumn()) {
@@ -814,7 +814,7 @@ private Pair<List<Column>, Integer> bindTargetColumns(OlapTable table, List<Stri
814814
++extraColumnsNum;
815815
processedColsName.add(col.getName());
816816
}
817-
} else if (col.getGeneratedColumnInfo() != null) {
817+
} else if (col.isGeneratedColumn()) {
818818
++extraColumnsNum;
819819
processedColsName.add(col.getName());
820820
}

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -147,6 +147,7 @@ public enum PlanType {
147147
DROP_DICTIONARY_COMMAND,
148148
CREATE_SQL_BLOCK_RULE_COMMAND,
149149
DELETE_COMMAND,
150+
MERGE_INTO_COMMAND,
150151
EXPLAIN_COMMAND,
151152
EXPLAIN_DICTIONARY_COMMAND,
152153
EXPORT_COMMAND,

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateCommand.java‎

Lines changed: 15 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -111,10 +111,10 @@ public LogicalPlan completeQueryPlan(ConnectContext ctx, LogicalPlan logicalQuer
111111
Map<String, Expression> colNameToExpression = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER);
112112
Map<String, Expression> partialUpdateColNameToExpression = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER);
113113
for (EqualTo equalTo : assignments) {
114-
List<String> nameParts = ((UnboundSlot) equalTo.left()).getNameParts();
115-
checkAssignmentColumn(ctx, nameParts);
116-
colNameToExpression.put(nameParts.get(nameParts.size() - 1), equalTo.right());
117-
partialUpdateColNameToExpression.put(nameParts.get(nameParts.size() - 1), equalTo.right());
114+
List<String> colNameParts = ((UnboundSlot) equalTo.left()).getNameParts();
115+
checkAssignmentColumn(ctx, colNameParts, this.nameParts, this.tableAlias);
116+
colNameToExpression.put(colNameParts.get(colNameParts.size() - 1), equalTo.right());
117+
partialUpdateColNameToExpression.put(colNameParts.get(colNameParts.size() - 1), equalTo.right());
118118
}
119119
// check if any key in update clause
120120
if (targetTable.getFullSchema().stream().filter(Column::isKey)
@@ -198,7 +198,15 @@ public LogicalPlan completeQueryPlan(ConnectContext ctx, LogicalPlan logicalQuer
198198
DMLCommandType.UPDATE, logicalQuery);
199199
}
200200

201-
private void checkAssignmentColumn(ConnectContext ctx, List<String> columnNameParts) {
201+
/**
202+
* check assignment column valid or not.
203+
* @param ctx connect context
204+
* @param columnNameParts qualified column name
205+
* @param tableNameParts qualified target table name
206+
* @param tableAlias target table alias
207+
*/
208+
public static void checkAssignmentColumn(ConnectContext ctx, List<String> columnNameParts,
209+
List<String> tableNameParts, String tableAlias) {
202210
if (columnNameParts.size() <= 1) {
203211
return;
204212
}
@@ -212,10 +220,10 @@ private void checkAssignmentColumn(ConnectContext ctx, List<String> columnNamePa
212220
} else {
213221
throw new AnalysisException("column in assignment list is invalid, " + String.join(".", columnNameParts));
214222
}
215-
if (dbName != null && this.tableAlias != null) {
223+
if (dbName != null && tableAlias != null) {
216224
throw new AnalysisException("column in assignment list is invalid, " + String.join(".", columnNameParts));
217225
}
218-
List<String> tableQualifier = RelationUtil.getQualifierName(ctx, nameParts);
226+
List<String> tableQualifier = RelationUtil.getQualifierName(ctx, tableNameParts);
219227
if (!ExpressionAnalyzer.sameTableName(tableAlias == null ? tableQualifier.get(2) : tableAlias, tableName)
220228
|| (dbName != null
221229
&& !ExpressionAnalyzer.compareDbNameIgnoreClusterName(tableQualifier.get(1), dbName))) {

0 commit comments

Comments
 (0)