Skip to content

Commit

Permalink
[Dynamic Partition] Support for automatically drop partitions (apache…
Browse files Browse the repository at this point in the history 
…#3081)
  • Loading branch information
@WingsGo
WingsGo authored Mar 25, 2020
1 parent e794bb6 commit 3cff89d
Show file tree
Hide file tree
Showing 10 changed files with 287 additions and 87 deletions.
4 changes: 3 additions & 1 deletion fe/src/main/java/org/apache/doris/analysis/ShowDynamicPartitionStmt.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,13 +33,15 @@ public class ShowDynamicPartitionStmt extends ShowStmt {
.addColumn(new Column("TableName", ScalarType.createVarchar(20)))
.addColumn(new Column("Enable", ScalarType.createVarchar(20)))
.addColumn(new Column("TimeUnit", ScalarType.createVarchar(20)))
.addColumn(new Column("Start", ScalarType.createVarchar(20)))
.addColumn(new Column("End", ScalarType.createVarchar(20)))
.addColumn(new Column("Prefix", ScalarType.createVarchar(20)))
.addColumn(new Column("Buckets", ScalarType.createVarchar(20)))
.addColumn(new Column("LastUpdateTime", ScalarType.createVarchar(20)))
.addColumn(new Column("LastSchedulerTime", ScalarType.createVarchar(20)))
.addColumn(new Column("State", ScalarType.createVarchar(20)))
.addColumn(new Column("Msg", ScalarType.createVarchar(20)))
.addColumn(new Column("LastCreatePartitionMsg", ScalarType.createVarchar(20)))
.addColumn(new Column("LastDropPartitionMsg", ScalarType.createVarchar(20)))
.build();

ShowDynamicPartitionStmt(String db) {
Expand Down
8 changes: 8 additions & 0 deletions fe/src/main/java/org/apache/doris/catalog/DynamicPartitionProperty.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@

public class DynamicPartitionProperty{
public static final String TIME_UNIT = "dynamic_partition.time_unit";
public static final String START = "dynamic_partition.start";
public static final String END = "dynamic_partition.end";
public static final String PREFIX = "dynamic_partition.prefix";
public static final String BUCKETS = "dynamic_partition.buckets";
Expand All @@ -30,6 +31,7 @@ public class DynamicPartitionProperty{

private boolean enable;
private String timeUnit;
private int start;
private int end;
private String prefix;
private int buckets;
Expand All @@ -39,6 +41,7 @@ public class DynamicPartitionProperty{
this.exist = true;
this.enable = Boolean.parseBoolean(properties.get(ENABLE));
this.timeUnit = properties.get(TIME_UNIT);
this.start = Integer.parseInt(properties.get(START));
this.end = Integer.parseInt(properties.get(END));
this.prefix = properties.get(PREFIX);
this.buckets = Integer.parseInt(properties.get(BUCKETS));
Expand All @@ -55,6 +58,10 @@ public String getTimeUnit() {
return timeUnit;
}

public int getStart() {
return start;
}

public int getEnd() {
return end;
}
Expand All @@ -75,6 +82,7 @@ public boolean getEnable() {
public String toString() {
return ",\n\"" + ENABLE + "\" = \"" + enable + "\"" +
",\n\"" + TIME_UNIT + "\" = \"" + timeUnit + "\"" +
",\n\"" + START + "\" = \"" + start + "\"" +
",\n\"" + END + "\" = \"" + end + "\"" +
",\n\"" + PREFIX + "\" = \"" + prefix + "\"" +
",\n\"" + BUCKETS + "\" = \"" + buckets + "\"";
Expand Down
249 changes: 165 additions & 84 deletions fe/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java
View file

Large diffs are not rendered by default.

4 changes: 4 additions & 0 deletions fe/src/main/java/org/apache/doris/common/ErrorCode.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -211,6 +211,10 @@ public enum ErrorCode {
ERR_INVALID_OPERATION(5065, new byte[] { '4', '2', '0', '0', '0' }, "Operation %s is invalid"),
ERROR_DYNAMIC_PARTITION_TIME_UNIT(5065, new byte[] {'4', '2', '0', '0', '0'},
"Unsupported time unit %s. Expect DAY WEEK MONTH."),
ERROR_DYNAMIC_PARTITION_START_ZERO(5066, new byte[] {'4', '2', '0', '0', '0'},
"Dynamic partition start must less than 0"),
ERROR_DYNAMIC_PARTITION_START_FORMAT(5066, new byte[] {'4', '2', '0', '0', '0'},
"Invalid dynamic partition start %s"),
ERROR_DYNAMIC_PARTITION_END_ZERO(5066, new byte[] {'4', '2', '0', '0', '0'},
"Dynamic partition end must greater than 0"),
ERROR_DYNAMIC_PARTITION_END_FORMAT(5066, new byte[] {'4', '2', '0', '0', '0'},
Expand Down
23 changes: 23 additions & 0 deletions fe/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,16 @@ private static void checkPrefix(String prefix) throws DdlException {
}
}

private static void checkStart(String start) throws DdlException {
try {
if (Integer.parseInt(start) >= 0) {
ErrorReport.reportDdlException(ErrorCode.ERROR_DYNAMIC_PARTITION_START_ZERO, start);
}
} catch (NumberFormatException e) {
ErrorReport.reportDdlException(ErrorCode.ERROR_DYNAMIC_PARTITION_START_FORMAT, start);
}
}

private static void checkEnd(String end) throws DdlException {
if (Strings.isNullOrEmpty(end)) {
ErrorReport.reportDdlException(ErrorCode.ERROR_DYNAMIC_PARTITION_END_EMPTY);
Expand Down Expand Up @@ -103,6 +113,7 @@ public static boolean checkDynamicPartitionPropertiesExist(Map<String, String> p
return false;
}
return properties.containsKey(DynamicPartitionProperty.TIME_UNIT) ||
properties.containsKey(DynamicPartitionProperty.START) ||
properties.containsKey(DynamicPartitionProperty.END) ||
properties.containsKey(DynamicPartitionProperty.PREFIX) ||
properties.containsKey(DynamicPartitionProperty.BUCKETS) ||
Expand All @@ -118,12 +129,14 @@ public static boolean checkInputDynamicPartitionProperties(Map<String, String> p
}
String timeUnit = properties.get(DynamicPartitionProperty.TIME_UNIT);
String prefix = properties.get(DynamicPartitionProperty.PREFIX);
String start = properties.get(DynamicPartitionProperty.START);
String end = properties.get(DynamicPartitionProperty.END);
String buckets = properties.get(DynamicPartitionProperty.BUCKETS);
String enable = properties.get(DynamicPartitionProperty.ENABLE);
if (!((Strings.isNullOrEmpty(enable) &&
Strings.isNullOrEmpty(timeUnit) &&
Strings.isNullOrEmpty(prefix) &&
Strings.isNullOrEmpty(start) &&
Strings.isNullOrEmpty(end) &&
Strings.isNullOrEmpty(buckets)))) {
if (Strings.isNullOrEmpty(enable)) {
Expand All @@ -135,6 +148,9 @@ public static boolean checkInputDynamicPartitionProperties(Map<String, String> p
if (Strings.isNullOrEmpty(prefix)) {
throw new DdlException("Must assign dynamic_partition.prefix properties");
}
if (Strings.isNullOrEmpty(start)) {
properties.put(DynamicPartitionProperty.START, String.valueOf(Integer.MIN_VALUE));
}
if (Strings.isNullOrEmpty(end)) {
throw new DdlException("Must assign dynamic_partition.end properties");
}
Expand Down Expand Up @@ -189,6 +205,13 @@ public static Map<String, String> analyzeDynamicPartition(Map<String, String> pr
properties.remove(DynamicPartitionProperty.ENABLE);
analyzedProperties.put(DynamicPartitionProperty.ENABLE, enableValue);
}
// If dynamic property is not specified.Use Integer.MIN_VALUE as default
if (properties.containsKey(DynamicPartitionProperty.START)) {
String startValue = properties.get(DynamicPartitionProperty.START);
checkStart(properties.get(DynamicPartitionProperty.START));
properties.remove(DynamicPartitionProperty.START);
analyzedProperties.put(DynamicPartitionProperty.START, startValue);
}
return analyzedProperties;
}

Expand Down
6 changes: 4 additions & 2 deletions fe/src/main/java/org/apache/doris/qe/ShowExecutor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1486,13 +1486,15 @@ private void handleShowDynamicPartition() {
tableName,
String.valueOf(dynamicPartitionProperty.getEnable()),
dynamicPartitionProperty.getTimeUnit().toUpperCase(),
String.valueOf(dynamicPartitionProperty.getStart()),
String.valueOf(dynamicPartitionProperty.getEnd()),
dynamicPartitionProperty.getPrefix(),
String.valueOf(dynamicPartitionProperty.getBuckets()),
dynamicPartitionScheduler.getRuntimeInfo(tableName, DynamicPartitionScheduler.LAST_UPDATE_TIME),
dynamicPartitionScheduler.getRuntimeInfo(tableName, DynamicPartitionScheduler.LAST_SCHEDULER_TIME),
dynamicPartitionScheduler.getRuntimeInfo(tableName, DynamicPartitionScheduler.DYNAMIC_PARTITION_STATE),
dynamicPartitionScheduler.getRuntimeInfo(tableName, DynamicPartitionScheduler.MSG)));
dynamicPartitionScheduler.getRuntimeInfo(tableName, DynamicPartitionScheduler.CREATE_PARTITION_MSG),
dynamicPartitionScheduler.getRuntimeInfo(tableName, DynamicPartitionScheduler.DROP_PARTITION_MSG)));
}
} finally {
db.readUnlock();
Expand All @@ -1506,7 +1508,7 @@ private void handleShowTransaction() throws AnalysisException {
ShowTransactionStmt showStmt = (ShowTransactionStmt) stmt;
Database db = ctx.getCatalog().getDb(showStmt.getDbName());
if (db == null) {
ErrorReport.reportAnalysisException(ErrorCode.ERR_BAD_DB_ERROR, showStmt.getDbName().toString());
ErrorReport.reportAnalysisException(ErrorCode.ERR_BAD_DB_ERROR, showStmt.getDbName());
}

long txnId = showStmt.getTxnId();
Expand Down
45 changes: 45 additions & 0 deletions fe/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@ public void setUp() throws Exception {
properties.put(DynamicPartitionProperty.ENABLE, "true");
properties.put(DynamicPartitionProperty.PREFIX, "p");
properties.put(DynamicPartitionProperty.TIME_UNIT, "day");
properties.put(DynamicPartitionProperty.START, "-3");
properties.put(DynamicPartitionProperty.END, "3");
properties.put(DynamicPartitionProperty.BUCKETS, "30");

Expand Down Expand Up @@ -293,6 +294,50 @@ public void testMissTimeUnit(@Injectable SystemInfoService systemInfoService,
catalog.createTable(stmt);
}

@Test
public void testMissSTART(@Injectable SystemInfoService systemInfoService,
@Injectable PaloAuth paloAuth,
@Injectable EditLog editLog) throws UserException {
new Expectations(catalog) {
{
catalog.getDb(dbTableName.getDb());
minTimes = 0;
result = db;

Catalog.getCurrentSystemInfo();
minTimes = 0;
result = systemInfoService;

systemInfoService.checkClusterCapacity(anyString);
minTimes = 0;
systemInfoService.seqChooseBackendIds(anyInt, true, true, anyString);
minTimes = 0;
result = beIds;

catalog.getAuth();
minTimes = 0;
result = paloAuth;
paloAuth.checkTblPriv((ConnectContext) any, anyString, anyString, PrivPredicate.CREATE);
minTimes = 0;
result = true;

catalog.getEditLog();
minTimes = 0;
result = editLog;
}
};

properties.remove(DynamicPartitionProperty.START);

CreateTableStmt stmt = new CreateTableStmt(false, false, dbTableName, columnDefs, "olap",
new KeysDesc(KeysType.AGG_KEYS, columnNames),
new RangePartitionDesc(Lists.newArrayList("key1"), singleRangePartitionDescs),
new HashDistributionDesc(1, Lists.newArrayList("key1")), properties, null, "");
stmt.analyze(analyzer);

catalog.createTable(stmt);
}

@Test
public void testMissEnd(@Injectable SystemInfoService systemInfoService,
@Injectable PaloAuth paloAuth,
Expand Down
2 changes: 2 additions & 0 deletions fe/src/test/java/org/apache/doris/catalog/TablePropertyTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ public void testNormal() throws IOException {
HashMap<String, String> properties = new HashMap<>();
properties.put(DynamicPartitionProperty.ENABLE, "true");
properties.put(DynamicPartitionProperty.TIME_UNIT, "day");
properties.put(DynamicPartitionProperty.START, "-3");
properties.put(DynamicPartitionProperty.END, "3");
properties.put(DynamicPartitionProperty.PREFIX, "p");
properties.put(DynamicPartitionProperty.BUCKETS, "30");
Expand All @@ -67,6 +68,7 @@ public void testNormal() throws IOException {
Assert.assertEquals(readDynamicPartitionProperty.getEnable(), dynamicPartitionProperty.getEnable());
Assert.assertEquals(readDynamicPartitionProperty.getBuckets(), dynamicPartitionProperty.getBuckets());
Assert.assertEquals(readDynamicPartitionProperty.getPrefix(), dynamicPartitionProperty.getPrefix());
Assert.assertEquals(readDynamicPartitionProperty.getStart(), dynamicPartitionProperty.getStart());
Assert.assertEquals(readDynamicPartitionProperty.getEnd(), dynamicPartitionProperty.getEnd());
Assert.assertEquals(readDynamicPartitionProperty.getTimeUnit(), dynamicPartitionProperty.getTimeUnit());
in.close();
Expand Down
1 change: 1 addition & 0 deletions fe/src/test/java/org/apache/doris/persist/ModifyDynamicPartitionInfoTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ public void testNormal() throws IOException {
HashMap<String, String> properties = new HashMap<>();
properties.put(DynamicPartitionProperty.ENABLE, "true");
properties.put(DynamicPartitionProperty.TIME_UNIT, "day");
properties.put(DynamicPartitionProperty.START, "-3");
properties.put(DynamicPartitionProperty.END, "3");
properties.put(DynamicPartitionProperty.PREFIX, "p");
properties.put(DynamicPartitionProperty.BUCKETS, "30");
Expand Down
32 changes: 32 additions & 0 deletions fe/src/test/java/org/apache/doris/planner/QueryPlanTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,38 @@ public static void beforeClass() throws Exception {
"PROPERTIES (\n" +
"\"replication_num\" = \"1\"\n" +
");");

createTable("CREATE TABLE test.`dynamic_partition` (\n" +
" `k1` date NULL COMMENT \"\",\n" +
" `k2` smallint(6) NULL COMMENT \"\",\n" +
" `k3` int(11) NULL COMMENT \"\",\n" +
" `k4` bigint(20) NULL COMMENT \"\",\n" +
" `k5` decimal(9, 3) NULL COMMENT \"\",\n" +
" `k6` char(5) NULL COMMENT \"\",\n" +
" `k10` date NULL COMMENT \"\",\n" +
" `k11` datetime NULL COMMENT \"\",\n" +
" `k7` varchar(20) NULL COMMENT \"\",\n" +
" `k8` double MAX NULL COMMENT \"\",\n" +
" `k9` float SUM NULL COMMENT \"\"\n" +
") ENGINE=OLAP\n" +
"AGGREGATE KEY(`k1`, `k2`, `k3`, `k4`, `k5`, `k6`, `k10`, `k11`, `k7`)\n" +
"COMMENT \"OLAP\"\n" +
"PARTITION BY RANGE (k1)\n" +
"(\n" +
"PARTITION p1 VALUES LESS THAN (\"2014-01-01\"),\n" +
"PARTITION p2 VALUES LESS THAN (\"2014-06-01\"),\n" +
"PARTITION p3 VALUES LESS THAN (\"2014-12-01\")\n" +
")\n" +
"DISTRIBUTED BY HASH(`k1`) BUCKETS 5\n" +
"PROPERTIES (\n" +
"\"replication_num\" = \"1\",\n" +
"\"dynamic_partition.enable\" = \"true\",\n" +
"\"dynamic_partition.start\" = \"-3\",\n" +
"\"dynamic_partition.end\" = \"3\",\n" +
"\"dynamic_partition.time_unit\" = \"day\",\n" +
"\"dynamic_partition.prefix\" = \"p\",\n" +
"\"dynamic_partition.buckets\" = \"1\"\n" +
");");
}

@AfterClass
Expand Down

0 comments on commit 3cff89d

Please sign in to comment.