Skip to content

Commit

Permalink
[BugFix] Fix partition live number not register (#34842)
Browse files Browse the repository at this point in the history 
Why I'm doing:
Creating the table first and then changing the partition_live_number attribute does not take effect.
What I'm doing:
This is caused by not registering the schedule info in DynamicPartitionScheduler when modifying the properties. Adding this schedule info when modifying partition_live_number can solve the problem.

Signed-off-by: Astralidea <astralidea@163.com>
  • Loading branch information
@Astralidea
Astralidea authored Nov 16, 2023
1 parent ddccb12 commit 035e033
Show file tree
Hide file tree
Showing 3 changed files with 147 additions and 32 deletions.
13 changes: 11 additions & 2 deletions fe/fe-core/src/main/java/com/starrocks/clone/DynamicPartitionScheduler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@
import com.starrocks.catalog.DynamicPartitionProperty;
import com.starrocks.catalog.HashDistributionInfo;
import com.starrocks.catalog.OlapTable;
import com.starrocks.catalog.Partition;
import com.starrocks.catalog.PartitionInfo;
import com.starrocks.catalog.PartitionKey;
import com.starrocks.catalog.RandomDistributionInfo;
Expand Down Expand Up @@ -139,6 +140,11 @@ public void removeTtlPartitionTable(Long dbId, Long tableId) {
ttlPartitionInfo.remove(new Pair<>(dbId, tableId));
}

@VisibleForTesting
public Set<Pair<Long, Long>> getTtlPartitionInfo() {
return ttlPartitionInfo;
}

public String getRuntimeInfo(String tableName, String key) {
Map<String, String> tableRuntimeInfo = runtimeInfos.getOrDefault(tableName, createDefaultRuntimeInfo());
return tableRuntimeInfo.getOrDefault(key, DEFAULT_RUNTIME_VALUE);
Expand Down Expand Up @@ -610,8 +616,11 @@ private ArrayList<DropPartitionClause> getDropPartitionClauseByTTL(OlapTable ola
int dropSize = allPartitionNumber - ttlNumber;
for (int i = 0; i < dropSize; i++) {
Long checkDropPartitionId = candidatePartitionList.get(i).getKey();
String dropPartitionName = olapTable.getPartition(checkDropPartitionId).getName();
dropPartitionClauses.add(new DropPartitionClause(false, dropPartitionName, false, true));
Partition partition = olapTable.getPartition(checkDropPartitionId);
if (partition != null) {
String dropPartitionName = partition.getName();
dropPartitionClauses.add(new DropPartitionClause(false, dropPartitionName, false, true));
}
}
}
return dropPartitionClauses;
Expand Down
96 changes: 66 additions & 30 deletions fe/fe-core/src/main/java/com/starrocks/server/LocalMetastore.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
import com.google.common.base.Preconditions;
import com.google.common.collect.HashMultimap;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Multimap;
Expand Down Expand Up @@ -3986,66 +3987,101 @@ public void modifyTableDynamicPartition(Database db, OlapTable table, Map<String

public void alterTableProperties(Database db, OlapTable table, Map<String, String> properties)
throws DdlException {
Map<String, String> logProperties = new HashMap<>(properties);
int partitionLiveNumber = -1;
Map<String, String> propertiesToPersist = new HashMap<>(properties);
Map<String, Object> results = validateToBeModifiedProps(properties);

TableProperty tableProperty = table.getTableProperty();
for (String key : results.keySet()) {
if (propertiesToPersist.containsKey(PropertyAnalyzer.PROPERTIES_PARTITION_LIVE_NUMBER)) {
int partitionLiveNumber = (int) results.get(key);
tableProperty.getProperties().put(PropertyAnalyzer.PROPERTIES_PARTITION_LIVE_NUMBER,
String.valueOf(partitionLiveNumber));
if (partitionLiveNumber == TableProperty.INVALID) {
GlobalStateMgr.getCurrentState().getDynamicPartitionScheduler().removeTtlPartitionTable(db.getId(),
table.getId());
} else {
GlobalStateMgr.getCurrentState().getDynamicPartitionScheduler().registerTtlPartitionTable(db.getId(),
table.getId());
}
tableProperty.setPartitionTTLNumber(partitionLiveNumber);
ModifyTablePropertyOperationLog info =
new ModifyTablePropertyOperationLog(db.getId(), table.getId(),
ImmutableMap.of(key, propertiesToPersist.get(key)));
GlobalStateMgr.getCurrentState().getEditLog().logAlterTableProperties(info);
}
if (propertiesToPersist.containsKey(PropertyAnalyzer.PROPERTIES_STORAGE_MEDIUM)) {
DataProperty dataProperty = (DataProperty) results.get(key);
TStorageMedium storageMedium = dataProperty.getStorageMedium();
table.setStorageMedium(storageMedium);
tableProperty.getProperties()
.put(PropertyAnalyzer.PROPERTIES_STORAGE_COOLDOWN_TIME,
String.valueOf(dataProperty.getCooldownTimeMs()));
ModifyTablePropertyOperationLog info =
new ModifyTablePropertyOperationLog(db.getId(), table.getId(),
ImmutableMap.of(key, propertiesToPersist.get(key)));
GlobalStateMgr.getCurrentState().getEditLog().logAlterTableProperties(info);
}
if (propertiesToPersist.containsKey(PropertyAnalyzer.PROPERTIES_STORAGE_COOLDOWN_TTL)) {
String storageCoolDownTTL = propertiesToPersist.get(PropertyAnalyzer.PROPERTIES_STORAGE_COOLDOWN_TTL);
tableProperty.getProperties().put(PropertyAnalyzer.PROPERTIES_STORAGE_COOLDOWN_TTL, storageCoolDownTTL);
tableProperty.buildStorageCoolDownTTL();
ModifyTablePropertyOperationLog info =
new ModifyTablePropertyOperationLog(db.getId(), table.getId(),
ImmutableMap.of(key, propertiesToPersist.get(key)));
GlobalStateMgr.getCurrentState().getEditLog().logAlterTableProperties(info);
}
if (propertiesToPersist.containsKey(PropertyAnalyzer.PROPERTIES_DATACACHE_PARTITION_DURATION)) {
String partitionDuration = propertiesToPersist.get(PropertyAnalyzer.PROPERTIES_DATACACHE_PARTITION_DURATION);
tableProperty.getProperties().put(PropertyAnalyzer.PROPERTIES_DATACACHE_PARTITION_DURATION, partitionDuration);
tableProperty.buildDataCachePartitionDuration();
ModifyTablePropertyOperationLog info =
new ModifyTablePropertyOperationLog(db.getId(), table.getId(),
ImmutableMap.of(key, propertiesToPersist.get(key)));
GlobalStateMgr.getCurrentState().getEditLog().logAlterTableProperties(info);
}

}
}

private Map<String, Object> validateToBeModifiedProps(Map<String, String> properties) throws DdlException {
Map<String, Object> results = Maps.newHashMap();
if (properties.containsKey(PropertyAnalyzer.PROPERTIES_PARTITION_LIVE_NUMBER)) {
try {
partitionLiveNumber = PropertyAnalyzer.analyzePartitionLiveNumber(properties, true);
int partitionLiveNumber = PropertyAnalyzer.analyzePartitionLiveNumber(properties, true);
results.put(PropertyAnalyzer.PROPERTIES_PARTITION_LIVE_NUMBER, partitionLiveNumber);
} catch (AnalysisException ex) {
throw new DdlException(ex.getMessage());
}
}
DataProperty dataProperty = DataProperty.getInferredDefaultDataProperty();
if (properties.containsKey(PropertyAnalyzer.PROPERTIES_STORAGE_MEDIUM)) {
try {
DataProperty dataProperty = DataProperty.getInferredDefaultDataProperty();
dataProperty = PropertyAnalyzer.analyzeDataProperty(properties, dataProperty, false);
results.put(PropertyAnalyzer.PROPERTIES_STORAGE_MEDIUM, dataProperty);
} catch (AnalysisException ex) {
throw new RuntimeException(ex.getMessage());
}
}
if (properties.containsKey(PropertyAnalyzer.PROPERTIES_STORAGE_COOLDOWN_TTL)) {
try {
PropertyAnalyzer.analyzeStorageCoolDownTTL(properties, true);
results.put(PropertyAnalyzer.PROPERTIES_STORAGE_COOLDOWN_TTL, null);
} catch (AnalysisException ex) {
throw new RuntimeException(ex.getMessage());
}
}
if (properties.containsKey(PropertyAnalyzer.PROPERTIES_DATACACHE_PARTITION_DURATION)) {
try {
PropertyAnalyzer.analyzeDataCachePartitionDuration(properties);
results.put(PropertyAnalyzer.PROPERTIES_DATACACHE_PARTITION_DURATION, null);
} catch (AnalysisException ex) {
throw new RuntimeException(ex.getMessage());
}
}
if (!properties.isEmpty()) {
throw new DdlException("Modify failed because unknown properties: " + properties);
}
TableProperty tableProperty = table.getTableProperty();
if (logProperties.containsKey(PropertyAnalyzer.PROPERTIES_PARTITION_LIVE_NUMBER)) {
tableProperty.getProperties().put(PropertyAnalyzer.PROPERTIES_PARTITION_LIVE_NUMBER,
String.valueOf(partitionLiveNumber));
tableProperty.setPartitionTTLNumber(partitionLiveNumber);
} else if (logProperties.containsKey(PropertyAnalyzer.PROPERTIES_STORAGE_MEDIUM)) {
TStorageMedium storageMedium = dataProperty.getStorageMedium();
table.setStorageMedium(storageMedium);
tableProperty.getProperties()
.put(PropertyAnalyzer.PROPERTIES_STORAGE_COOLDOWN_TIME,
String.valueOf(dataProperty.getCooldownTimeMs()));
} else if (logProperties.containsKey(PropertyAnalyzer.PROPERTIES_STORAGE_COOLDOWN_TTL)) {
String storageCoolDownTTL = logProperties.get(PropertyAnalyzer.PROPERTIES_STORAGE_COOLDOWN_TTL);
tableProperty.getProperties().put(PropertyAnalyzer.PROPERTIES_STORAGE_COOLDOWN_TTL, storageCoolDownTTL);
tableProperty.buildStorageCoolDownTTL();
} else if (logProperties.containsKey(PropertyAnalyzer.PROPERTIES_DATACACHE_PARTITION_DURATION)) {
String partitionDuration = logProperties.get(PropertyAnalyzer.PROPERTIES_DATACACHE_PARTITION_DURATION);
tableProperty.getProperties().put(PropertyAnalyzer.PROPERTIES_DATACACHE_PARTITION_DURATION, partitionDuration);
tableProperty.buildDataCachePartitionDuration();
} else {
throw new DdlException("Modify failed because unknown properties: " + properties);
}

ModifyTablePropertyOperationLog info =
new ModifyTablePropertyOperationLog(db.getId(), table.getId(), logProperties);
GlobalStateMgr.getCurrentState().getEditLog().logAlterTableProperties(info);
return results;
}

/**
Expand Down
70 changes: 70 additions & 0 deletions fe/fe-core/src/test/java/com/starrocks/alter/AlterTableTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,12 +15,14 @@
package com.starrocks.alter;

import com.starrocks.catalog.DataProperty;
import com.starrocks.catalog.Database;
import com.starrocks.catalog.OlapTable;
import com.starrocks.catalog.RangePartitionInfo;
import com.starrocks.catalog.Table;
import com.starrocks.common.AnalysisException;
import com.starrocks.common.Config;
import com.starrocks.common.FeConstants;
import com.starrocks.common.Pair;
import com.starrocks.common.util.TimeUtils;
import com.starrocks.qe.ConnectContext;
import com.starrocks.server.GlobalStateMgr;
Expand All @@ -32,6 +34,8 @@
import org.junit.Test;
import org.threeten.extra.PeriodDuration;

import java.util.Set;

public class AlterTableTest {
private static ConnectContext connectContext;
private static StarRocksAssert starRocksAssert;
Expand Down Expand Up @@ -164,4 +168,70 @@ public void testAlterTableStorageCoolDownTTLPartition() throws Exception {

}

@Test
public void testAlterTablePartitionTTLInvalid() throws Exception {
starRocksAssert.useDatabase("test").withTable("CREATE TABLE test_partition_live_number (\n" +
"event_day DATE,\n" +
"site_id INT DEFAULT '10',\n" +
"city_code VARCHAR(100),\n" +
"user_name VARCHAR(32) DEFAULT '',\n" +
"pv BIGINT DEFAULT '0'\n" +
")\n" +
"DUPLICATE KEY(event_day, site_id, city_code, user_name)\n" +
"PARTITION BY RANGE(event_day)(\n" +
"PARTITION p20200321 VALUES LESS THAN (\"2020-03-22\"),\n" +
"PARTITION p20200322 VALUES LESS THAN (\"2020-03-23\"),\n" +
"PARTITION p20200323 VALUES LESS THAN (\"2020-03-24\"),\n" +
"PARTITION p20200324 VALUES LESS THAN MAXVALUE\n" +
")\n" +
"DISTRIBUTED BY HASH(event_day, site_id)\n" +
"PROPERTIES(\n" +
"\t\"replication_num\" = \"1\",\n" +
" \"storage_medium\" = \"SSD\",\n" +
" \"partition_live_number\" = \"2\"\n" +
");");
ConnectContext ctx = starRocksAssert.getCtx();
String sql = "ALTER TABLE test_partition_live_number SET(\"partition_live_number\" = \"-1\");";
AlterTableStmt alterTableStmt = (AlterTableStmt) UtFrameUtils.parseStmtWithNewParser(sql, ctx);
GlobalStateMgr.getCurrentState().alterTable(alterTableStmt);
Set<Pair<Long, Long>> ttlPartitionInfo = GlobalStateMgr.getCurrentState()
.getDynamicPartitionScheduler().getTtlPartitionInfo();
Database db = GlobalStateMgr.getCurrentState().getDb("test");
Table table = db.getTable("test_partition_live_number");
Assert.assertFalse(ttlPartitionInfo.contains(new Pair<>(db.getId(), table.getId())));
sql = "ALTER TABLE test_partition_live_number SET(\"partition_live_number\" = \"1\");";
alterTableStmt = (AlterTableStmt) UtFrameUtils.parseStmtWithNewParser(sql, ctx);
GlobalStateMgr.getCurrentState().alterTable(alterTableStmt);
Assert.assertTrue(ttlPartitionInfo.contains(new Pair<>(db.getId(), table.getId())));
}

@Test
public void testAlterTablePartitionStorageMedium() throws Exception {
starRocksAssert.useDatabase("test").withTable("CREATE TABLE test_partition_storage_medium (\n" +
"event_day DATE,\n" +
"site_id INT DEFAULT '10',\n" +
"city_code VARCHAR(100),\n" +
"user_name VARCHAR(32) DEFAULT '',\n" +
"pv BIGINT DEFAULT '0'\n" +
")\n" +
"DUPLICATE KEY(event_day, site_id, city_code, user_name)\n" +
"PARTITION BY RANGE(event_day)(\n" +
"PARTITION p20200321 VALUES LESS THAN (\"2020-03-22\"),\n" +
"PARTITION p20200322 VALUES LESS THAN (\"2020-03-23\"),\n" +
"PARTITION p20200323 VALUES LESS THAN (\"2020-03-24\"),\n" +
"PARTITION p20200324 VALUES LESS THAN MAXVALUE\n" +
")\n" +
"DISTRIBUTED BY HASH(event_day, site_id)\n" +
"PROPERTIES(\n" +
"\"replication_num\" = \"1\"\n" +
");");
ConnectContext ctx = starRocksAssert.getCtx();
String sql = "ALTER TABLE test_partition_storage_medium SET(\"default.storage_medium\" = \"SSD\");";
AlterTableStmt alterTableStmt = (AlterTableStmt) UtFrameUtils.parseStmtWithNewParser(sql, ctx);
GlobalStateMgr.getCurrentState().alterTable(alterTableStmt);
Database db = GlobalStateMgr.getCurrentState().getDb("test");
OlapTable olapTable = (OlapTable) db.getTable("test_partition_storage_medium");
Assert.assertTrue(olapTable.getStorageMedium().equals("SSD"));
}

}

0 comments on commit 035e033

Please sign in to comment.