Skip to content

Commit

Permalink
[Enhancement] Automatically create partition table to support insert …
Browse files Browse the repository at this point in the history 
…overwrite by partition name (#34777)

Signed-off-by: Astralidea <astralidea@163.com>
  • Loading branch information
@Astralidea
Astralidea authored Nov 14, 2023
1 parent 000f4a7 commit 7c6884e
Show file tree
Hide file tree
Showing 3 changed files with 100 additions and 12 deletions.
54 changes: 42 additions & 12 deletions fe/fe-core/src/main/java/com/starrocks/load/InsertOverwriteJobRunner.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import com.starrocks.analysis.Expr;
import com.starrocks.analysis.LiteralExpr;
import com.starrocks.catalog.Database;
import com.starrocks.catalog.ListPartitionInfo;
import com.starrocks.catalog.MaterializedIndex;
import com.starrocks.catalog.OlapTable;
import com.starrocks.catalog.Partition;
Expand Down Expand Up @@ -228,21 +229,49 @@ private void createPartitionByValue(InsertStmt insertStmt) {
return;
}
OlapTable olapTable = (OlapTable) insertStmt.getTargetTable();
List<List<String>> partitionValues = Lists.newArrayList();
if (!olapTable.getPartitionInfo().isAutomaticPartition()) {
return;
}
List<Expr> partitionColValues = insertStmt.getTargetPartitionNames().getPartitionColValues();
List<List<String>> partitionValues = Lists.newArrayList();
// Currently we only support overwriting one partition at a time
List<String> firstValues = Lists.newArrayList();
partitionValues.add(firstValues);
for (Expr expr : partitionColValues) {
if (expr instanceof LiteralExpr) {
firstValues.add(((LiteralExpr) expr).getStringValue());
} else {
throw new SemanticException("Only support literal value for partition column.");

if (insertStmt.isSpecifyPartitionNames()) {
List<String> partitionNames = insertStmt.getTargetPartitionNames().getPartitionNames();
PartitionInfo partitionInfo = olapTable.getPartitionInfo();
for (String partitionName : partitionNames) {
Partition partition = olapTable.getPartition(partitionName);
if (partition == null) {
throw new RuntimeException("Partition '" + partitionName
+ "' does not exist in table '" + olapTable.getName() + "'.");
}
if (partitionInfo instanceof ListPartitionInfo) {
ListPartitionInfo listPartitionInfo = (ListPartitionInfo) partitionInfo;
List<List<LiteralExpr>> lists = listPartitionInfo.getMultiLiteralExprValues().get(partition.getId());
for (List<LiteralExpr> list : lists) {
List<String> values = Lists.newArrayList();
for (LiteralExpr literalExpr : list) {
values.add(literalExpr.getStringValue());
}
partitionValues.add(values);
}
} else {
throw new RuntimeException("Specify the partition name, and automatically create partition names. " +
"Currently, only List partitions are supported.");
}
}
} else {
List<Expr> partitionColValues = insertStmt.getTargetPartitionNames().getPartitionColValues();
// Currently we only support overwriting one partition at a time
List<String> firstValues = Lists.newArrayList();
partitionValues.add(firstValues);
for (Expr expr : partitionColValues) {
if (expr instanceof LiteralExpr) {
firstValues.add(((LiteralExpr) expr).getStringValue());
} else {
throw new SemanticException("Only support literal value for partition column.");
}
}
}

try {
addPartitionClause = AnalyzerUtils.getAddPartitionClauseFromPartitionValues(olapTable, partitionValues);
} catch (AnalysisException ex) {
Expand Down Expand Up @@ -270,9 +299,10 @@ private void createPartitionByValue(InsertStmt insertStmt) {
}
for (String partitionColName : partitionColNames) {
Partition partition = olapTable.getPartition(partitionColName);
sourcePartitionIds.add(partition.getId());
if (!sourcePartitionIds.contains(partition.getId())) {
sourcePartitionIds.add(partition.getId());
}
}

}

private void executeInsert() throws Exception {
Expand Down
35 changes: 35 additions & 0 deletions test/sql/test_automatic_partition/R/test_insert_overwrite
View file
Original file line number Diff line number Diff line change
Expand Up @@ -161,4 +161,39 @@ insert overwrite duplicate_table_with_null1 select * from duplicate_table_with_n
select count(1) from duplicate_table_with_null1;
-- result:
3
-- !result
CREATE TABLE `t_recharge_detail2` (
`id` bigint(20) NULL COMMENT "",
`user_id` bigint(20) NULL COMMENT "",
`recharge_money` decimal128(32, 2) NULL COMMENT "",
`city` varchar(20) NOT NULL COMMENT "",
`dt` varchar(20) NOT NULL COMMENT ""
) ENGINE=OLAP
DUPLICATE KEY(`id`)
PARTITION BY (`dt`,`city`)
DISTRIBUTED BY HASH(`id`)
PROPERTIES (
"replication_num" = "1",
"in_memory" = "false",
"enable_persistent_index" = "false",
"replicated_storage" = "true",
"light_schema_change" = "true",
"partition_live_number" = "1",
"compression" = "LZ4"
);
-- result:
-- !result
insert into t_recharge_detail2 values (1, 2, 1.1, "Menlo Park", "2023-11-02");
-- result:
-- !result
insert overwrite t_recharge_detail2 partition(p20231101_MenloPark) values (2, 4, 2.2, "Menlo Park", "2023-11-01");
-- result:
E: (1064, "Getting analyzing error. Detail message: Unknown partition 'p20231101_MenloPark' in table 't_recharge_detail2'.")
-- !result
insert overwrite t_recharge_detail2 partition(p20231102_MenloPark) values (3, 6, 3.3, "Menlo Park", "2023-11-02");
-- result:
-- !result
select * from t_recharge_detail2;
-- result:
3 6 3.30 Menlo Park 2023-11-02
-- !result
23 changes: 23 additions & 0 deletions test/sql/test_automatic_partition/T/test_insert_overwrite
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,3 +88,26 @@ insert into duplicate_table_with_null1 values('2020-01-01', '2020-01-01 12:30:30
insert into duplicate_table_with_null2 values('2021-01-01', '2020-01-01 12:30:30', 'asfgrgte', 'wergergqer', 'stringaaa', 0, 11, 111, 1111, 11111, 111111, 11.11, 1), ('2021-01-02', '0000-01-01 00:00:00', 'asfgrgtf', 'wergergqes', 'stringbbb',1, -128, -32768, -2147483648, -9223372036854775808, -170141183460469231731687303715884105728, 22.22, 2), ('2021-01-03', '9999-12-31 23:59:59', 'asfgrgtf', 'wergergqes', 'stringbbb',1, 127, 32767, 2147483647, 9223372036854775807, 170141183460469231731687303715884105727, 22.22, 2);
insert overwrite duplicate_table_with_null1 select * from duplicate_table_with_null2;
select count(1) from duplicate_table_with_null1;
CREATE TABLE `t_recharge_detail2` (
`id` bigint(20) NULL COMMENT "",
`user_id` bigint(20) NULL COMMENT "",
`recharge_money` decimal128(32, 2) NULL COMMENT "",
`city` varchar(20) NOT NULL COMMENT "",
`dt` varchar(20) NOT NULL COMMENT ""
) ENGINE=OLAP
DUPLICATE KEY(`id`)
PARTITION BY (`dt`,`city`)
DISTRIBUTED BY HASH(`id`)
PROPERTIES (
"replication_num" = "1",
"in_memory" = "false",
"enable_persistent_index" = "false",
"replicated_storage" = "true",
"light_schema_change" = "true",
"partition_live_number" = "1",
"compression" = "LZ4"
);
insert into t_recharge_detail2 values (1, 2, 1.1, "Menlo Park", "2023-11-02");
insert overwrite t_recharge_detail2 partition(p20231101_MenloPark) values (2, 4, 2.2, "Menlo Park", "2023-11-01");
insert overwrite t_recharge_detail2 partition(p20231102_MenloPark) values (3, 6, 3.3, "Menlo Park", "2023-11-02");
select * from t_recharge_detail2;

0 comments on commit 7c6884e

Please sign in to comment.