Skip to content

Commit

Permalink
[Refactor] Rename light schema change to fast schema evolution (#34472)
Browse files Browse the repository at this point in the history 
Signed-off-by: zhangqiang <qiangzh95@gmail.com>
  • Loading branch information
@sevev
sevev authored Nov 13, 2023
1 parent 9c76539 commit b24c6ed
Show file tree
Hide file tree
Showing 33 changed files with 155 additions and 151 deletions.
62 changes: 31 additions & 31 deletions fe/fe-core/src/main/java/com/starrocks/alter/SchemaChangeHandler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -269,7 +269,7 @@ private boolean processAddColumns(AddColumnsClause alterClause, OlapTable olapTa
private boolean processDropColumn(DropColumnClause alterClause, OlapTable olapTable,
Map<Long, LinkedList<Column>> indexSchemaMap, List<Index> indexes) throws DdlException {

boolean lightSchemaChange = olapTable.getUseLightSchemaChange();
boolean fastSchemaEvolution = olapTable.getUseFastSchemaEvolution();
String dropColName = alterClause.getColName();
String targetIndexName = alterClause.getRollupName();
checkIndexExists(olapTable, targetIndexName);
Expand All @@ -292,7 +292,7 @@ private boolean processDropColumn(DropColumnClause alterClause, OlapTable olapTa
if (isKey) {
throw new DdlException("Can not drop key column in primary data model table");
}
lightSchemaChange &= !isKey;
fastSchemaEvolution &= !isKey;
MaterializedIndexMeta indexMeta = olapTable.getIndexMetaByIndexId(olapTable.getBaseIndexId());
if (indexMeta.getSortKeyIdxes() != null) {
for (Integer sortKeyIdx : indexMeta.getSortKeyIdxes()) {
Expand All @@ -305,7 +305,7 @@ private boolean processDropColumn(DropColumnClause alterClause, OlapTable olapTa
long baseIndexId = olapTable.getBaseIndexId();
List<Column> baseSchema = indexSchemaMap.get(baseIndexId);
boolean isKey = baseSchema.stream().anyMatch(c -> c.isKey() && c.getName().equalsIgnoreCase(dropColName));
lightSchemaChange &= !isKey;
fastSchemaEvolution &= !isKey;
if (isKey) {
throw new DdlException("Can not drop key column in Unique data model table");
}
Expand All @@ -315,7 +315,7 @@ private boolean processDropColumn(DropColumnClause alterClause, OlapTable olapTa
long baseIndexId = olapTable.getBaseIndexId();
List<Column> baseSchema = indexSchemaMap.get(baseIndexId);
boolean isKey = baseSchema.stream().anyMatch(c -> c.isKey() && c.getName().equalsIgnoreCase(dropColName));
lightSchemaChange &= !isKey;
fastSchemaEvolution &= !isKey;
boolean hasReplaceColumn = baseSchema.stream().map(Column::getAggregationType)
.anyMatch(agg -> agg == AggregateType.REPLACE || agg == AggregateType.REPLACE_IF_NOT_NULL);
if (isKey && hasReplaceColumn) {
Expand All @@ -326,7 +326,7 @@ private boolean processDropColumn(DropColumnClause alterClause, OlapTable olapTa
long targetIndexId = olapTable.getIndexIdByName(targetIndexName);
List<Column> targetIndexSchema = indexSchemaMap.get(targetIndexId);
boolean isKey = targetIndexSchema.stream().anyMatch(c -> c.isKey() && c.getName().equalsIgnoreCase(dropColName));
lightSchemaChange &= !isKey;
fastSchemaEvolution &= !isKey;
boolean hasReplaceColumn = targetIndexSchema.stream().map(Column::getAggregationType)
.anyMatch(agg -> agg == AggregateType.REPLACE || agg == AggregateType.REPLACE_IF_NOT_NULL);
if (isKey && hasReplaceColumn) {
Expand All @@ -339,7 +339,7 @@ private boolean processDropColumn(DropColumnClause alterClause, OlapTable olapTa
List<Column> baseSchema = indexSchemaMap.get(baseIndexId);
for (Column column : baseSchema) {
if (column.isKey() && column.getName().equalsIgnoreCase(dropColName)) {
lightSchemaChange = false;
fastSchemaEvolution = false;
break;
}
}
Expand All @@ -360,7 +360,7 @@ private boolean processDropColumn(DropColumnClause alterClause, OlapTable olapTa
columnIterator.remove();
removed = true;
if (column.isKey()) {
lightSchemaChange = false;
fastSchemaEvolution = false;
}
}
}
Expand All @@ -377,7 +377,7 @@ private boolean processDropColumn(DropColumnClause alterClause, OlapTable olapTa
if (column.getName().equalsIgnoreCase(dropColName)) {
columnIterator.remove();
if (column.isKey()) {
lightSchemaChange = false;
fastSchemaEvolution = false;
}
}
}
Expand All @@ -394,7 +394,7 @@ private boolean processDropColumn(DropColumnClause alterClause, OlapTable olapTa
columnIterator.remove();
removed = true;
if (column.isKey()) {
lightSchemaChange = false;
fastSchemaEvolution = false;
}
}
}
Expand All @@ -403,7 +403,7 @@ private boolean processDropColumn(DropColumnClause alterClause, OlapTable olapTa
throw new DdlException("Column does not exists: " + dropColName);
}
}
return lightSchemaChange;
return fastSchemaEvolution;
}

// User can modify column type and column position
Expand Down Expand Up @@ -766,10 +766,10 @@ private boolean addColumnInternal(OlapTable olapTable, Column newColumn, ColumnP
throw new DdlException("unsupported default expr:" + newColumn.getDefaultExpr().getExpr());
}

boolean lightSchemaChange = olapTable.getUseLightSchemaChange();
boolean fastSchemaEvolution = olapTable.getUseFastSchemaEvolution();
// if column is generated column, need to rewrite table data, so we can not use light schema change
if (newColumn.isAutoIncrement() || newColumn.isGeneratedColumn()) {
lightSchemaChange = false;
fastSchemaEvolution = false;
}

String newColName = newColumn.getName();
Expand Down Expand Up @@ -840,7 +840,7 @@ private boolean addColumnInternal(OlapTable olapTable, Column newColumn, ColumnP

//type key column do not allow light schema change.
if (newColumn.isKey()) {
lightSchemaChange = false;
fastSchemaEvolution = false;
}

// check if the new column already exist in base schema.
Expand Down Expand Up @@ -888,10 +888,10 @@ private boolean addColumnInternal(OlapTable olapTable, Column newColumn, ColumnP
modIndexSchema = indexSchemaMap.get(baseIndexId);
checkAndAddColumn(modIndexSchema, newColumn, columnPos, newColNameSet, true);
if (targetIndexId == -1L) {
return lightSchemaChange;
return fastSchemaEvolution;
}
// 2. add to rollup
lightSchemaChange = false;
fastSchemaEvolution = false;
modIndexSchema = indexSchemaMap.get(targetIndexId);
checkAndAddColumn(modIndexSchema, newColumn, columnPos, newColNameSet, false);
}
Expand All @@ -901,10 +901,10 @@ private boolean addColumnInternal(OlapTable olapTable, Column newColumn, ColumnP
List<Column> modIndexSchema = indexSchemaMap.get(baseIndexId);
checkAndAddColumn(modIndexSchema, newColumn, columnPos, newColNameSet, true);
// no specified target index. return
return lightSchemaChange;
return fastSchemaEvolution;
} else {
// add to rollup index
lightSchemaChange = false;
fastSchemaEvolution = false;
List<Column> modIndexSchema = indexSchemaMap.get(targetIndexId);
checkAndAddColumn(modIndexSchema, newColumn, columnPos, newColNameSet, false);

Expand All @@ -928,15 +928,15 @@ private boolean addColumnInternal(OlapTable olapTable, Column newColumn, ColumnP

if (targetIndexId == -1L) {
// no specified target index. return
return lightSchemaChange;
return fastSchemaEvolution;
}

lightSchemaChange = false;
fastSchemaEvolution = false;
// 2. add to rollup index
modIndexSchema = indexSchemaMap.get(targetIndexId);
checkAndAddColumn(modIndexSchema, newColumn, columnPos, newColNameSet, false);
}
return lightSchemaChange;
return fastSchemaEvolution;
}

/*
Expand Down Expand Up @@ -1424,7 +1424,7 @@ public Optional<Long> getActiveTxnIdOfTable(long tableId) {
@Nullable
public AlterJobV2 analyzeAndCreateJob(List<AlterClause> alterClauses, Database db, OlapTable olapTable) throws UserException {
//alterClauses can or cannot light schema change
boolean lightSchemaChange = true;
boolean fastSchemaEvolution = true;
//for multi add colmuns clauses
IntSupplier colUniqueIdSupplier = new IntSupplier() {
private int pendingMaxColUniqueId = olapTable.getMaxColUniqueId();
Expand Down Expand Up @@ -1523,25 +1523,25 @@ public int getAsInt() {

if (alterClause instanceof AddColumnClause) {
// add column
lightSchemaChange =
fastSchemaEvolution =
processAddColumn((AddColumnClause) alterClause, olapTable, indexSchemaMap,
colUniqueIdSupplier);
} else if (alterClause instanceof AddColumnsClause) {
// add columns
lightSchemaChange =
fastSchemaEvolution =
processAddColumns((AddColumnsClause) alterClause, olapTable, indexSchemaMap, colUniqueIdSupplier);
} else if (alterClause instanceof DropColumnClause) {
// drop column and drop indexes on this column
lightSchemaChange =
fastSchemaEvolution =
processDropColumn((DropColumnClause) alterClause, olapTable, indexSchemaMap,
newIndexes);
} else if (alterClause instanceof ModifyColumnClause) {
// modify column
processModifyColumn((ModifyColumnClause) alterClause, olapTable, indexSchemaMap);
lightSchemaChange = false;
fastSchemaEvolution = false;
} else if (alterClause instanceof ReorderColumnsClause) {
// reorder column
lightSchemaChange = false;
fastSchemaEvolution = false;
if (olapTable.getKeysType() == KeysType.PRIMARY_KEYS) {
List<Integer> sortKeyIdxes = new ArrayList<>();
List<Integer> sortKeyUniqueIds = new ArrayList<>();
Expand All @@ -1555,12 +1555,12 @@ public int getAsInt() {
} else if (alterClause instanceof ModifyTablePropertiesClause) {
// modify table properties
// do nothing, properties are already in propertyMap
lightSchemaChange = false;
fastSchemaEvolution = false;
} else if (alterClause instanceof CreateIndexClause) {
lightSchemaChange = false;
fastSchemaEvolution = false;
processAddIndex((CreateIndexClause) alterClause, olapTable, newIndexes);
} else if (alterClause instanceof DropIndexClause) {
lightSchemaChange = false;
fastSchemaEvolution = false;
processDropIndex((DropIndexClause) alterClause, olapTable, newIndexes);
} else if (alterClause instanceof OptimizeClause) {
return createOptimizeTableJob((OptimizeClause) alterClause, db, olapTable, propertyMap);
Expand All @@ -1570,9 +1570,9 @@ public int getAsInt() {
} // end for alter clauses

LOG.debug("processAddColumns, table: {}({}), ligthSchemaChange: {}", olapTable.getName(), olapTable.getId(),
lightSchemaChange);
fastSchemaEvolution);

if (lightSchemaChange) {
if (fastSchemaEvolution) {
long jobId = GlobalStateMgr.getCurrentState().getNextId();
long txnId = GlobalStateMgr.getCurrentGlobalTransactionMgr().getTransactionIDGenerator().getNextTransactionId();
//for schema change add/drop value column optimize, direct modify table meta.
Expand Down
5 changes: 2 additions & 3 deletions fe/fe-core/src/main/java/com/starrocks/catalog/MaterializedView.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,13 +111,12 @@ public enum RefreshMoment {
}

@Override
public Boolean getUseLightSchemaChange() {
public Boolean getUseFastSchemaEvolution() {
return false;
}

@Override
public void setUseLightSchemaChange(boolean useLightSchemaChange) {
}
public void setUseFastSchemaEvolution(boolean useFastSchemaEvolution) {}

public static class BasePartitionInfo {

Expand Down
12 changes: 6 additions & 6 deletions fe/fe-core/src/main/java/com/starrocks/catalog/OlapTable.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2701,21 +2701,21 @@ public void setForeignKeyConstraints(List<ForeignKeyConstraint> foreignKeyConstr
tableProperty.setForeignKeyConstraints(foreignKeyConstraints);
}

public Boolean getUseLightSchemaChange() {
public Boolean getUseFastSchemaEvolution() {
if (tableProperty != null) {
return tableProperty.getUseSchemaLightChange();
return tableProperty.getUseFastSchemaEvolution();
}
// property is set false by default
return false;
}

public void setUseLightSchemaChange(boolean useLightSchemaChange) {
public void setUseFastSchemaEvolution(boolean useFastSchemaEvolution) {
if (tableProperty == null) {
tableProperty = new TableProperty(new HashMap<>());
}
tableProperty.modifyTableProperties(PropertyAnalyzer.PROPERTIES_USE_LIGHT_SCHEMA_CHANGE,
Boolean.valueOf(useLightSchemaChange).toString());
tableProperty.buildUseLightSchemaChange();
tableProperty.modifyTableProperties(PropertyAnalyzer.PROPERTIES_USE_FAST_SCHEMA_EVOLUTION,
Boolean.valueOf(useFastSchemaEvolution).toString());
tableProperty.buildUseFastSchemaEvolution();
}


Expand Down
14 changes: 7 additions & 7 deletions fe/fe-core/src/main/java/com/starrocks/catalog/TableProperty.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -212,7 +212,7 @@ public static String valueList() {
// foreign key constraint for mv rewrite
private List<ForeignKeyConstraint> foreignKeyConstraints;

private Boolean useSchemaLightChange;
private Boolean useFastSchemaEvolution;

private PeriodDuration dataCachePartitionDuration;

Expand Down Expand Up @@ -794,14 +794,14 @@ public void clearBinlogAvailableVersion() {
}
}

public TableProperty buildUseLightSchemaChange() {
useSchemaLightChange = Boolean.parseBoolean(
properties.getOrDefault(PropertyAnalyzer.PROPERTIES_USE_LIGHT_SCHEMA_CHANGE, "false"));
public TableProperty buildUseFastSchemaEvolution() {
useFastSchemaEvolution = Boolean.parseBoolean(
properties.getOrDefault(PropertyAnalyzer.PROPERTIES_USE_FAST_SCHEMA_EVOLUTION, "false"));
return this;
}

public Boolean getUseSchemaLightChange() {
return useSchemaLightChange;
public Boolean getUseFastSchemaEvolution() {
return useFastSchemaEvolution;
}

@Override
Expand Down Expand Up @@ -841,6 +841,6 @@ public void gsonPostProcess() throws IOException {
buildBinlogAvailableVersion();
buildConstraint();
buildDataCachePartitionDuration();
buildUseLightSchemaChange();
buildUseFastSchemaEvolution();
}
}
2 changes: 1 addition & 1 deletion fe/fe-core/src/main/java/com/starrocks/common/Config.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2545,7 +2545,7 @@ public class Config extends ConfigBase {
public static int external_table_commit_timeout_ms = 10000; // 10s

@ConfField(mutable = true)
public static boolean allow_default_light_schema_change = true;
public static boolean enable_fast_schema_evolution = true;

@ConfField(mutable = false)
public static int pipe_listener_interval_millis = 1000;
Expand Down
17 changes: 11 additions & 6 deletions fe/fe-core/src/main/java/com/starrocks/common/util/PropertyAnalyzer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -196,7 +196,8 @@ public class PropertyAnalyzer {
*/
public static final String PROPERTY_MV_SORT_KEYS = "mv_sort_keys";

// light schema change
// fast schema evolution
public static final String PROPERTIES_USE_FAST_SCHEMA_EVOLUTION = "fast_schema_evolution";
public static final String PROPERTIES_USE_LIGHT_SCHEMA_CHANGE = "light_schema_change";

public static final String PROPERTIES_DEFAULT_PREFIX = "default.";
Expand Down Expand Up @@ -600,21 +601,25 @@ public static int analyzeSchemaVersion(Map<String, String> properties) throws An
return schemaVersion;
}

public static Boolean analyzeUseLightSchemaChange(Map<String, String> properties) throws AnalysisException {
public static Boolean analyzeUseFastSchemaEvolution(Map<String, String> properties) throws AnalysisException {
if (properties == null || properties.isEmpty()) {
return Config.allow_default_light_schema_change;
return Config.enable_fast_schema_evolution;
}
String value = properties.get(PROPERTIES_USE_LIGHT_SCHEMA_CHANGE);
String value = properties.get(PROPERTIES_USE_FAST_SCHEMA_EVOLUTION);
if (null == value) {
return Config.allow_default_light_schema_change;
value = properties.get(PROPERTIES_USE_LIGHT_SCHEMA_CHANGE);
if (null == value) {
return Config.enable_fast_schema_evolution;
}
}
properties.remove(PROPERTIES_USE_FAST_SCHEMA_EVOLUTION);
properties.remove(PROPERTIES_USE_LIGHT_SCHEMA_CHANGE);
if (Boolean.TRUE.toString().equalsIgnoreCase(value)) {
return true;
} else if (Boolean.FALSE.toString().equalsIgnoreCase(value)) {
return false;
}
throw new AnalysisException(PROPERTIES_USE_LIGHT_SCHEMA_CHANGE
throw new AnalysisException(PROPERTIES_USE_FAST_SCHEMA_EVOLUTION
+ " must be `true` or `false`");
}

Expand Down
8 changes: 4 additions & 4 deletions fe/fe-core/src/main/java/com/starrocks/server/GlobalStateMgr.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2768,10 +2768,10 @@ public static void getDdlStmt(String dbName, Table table, List<String> createTab
sb.append(WriteQuorum.writeQuorumToName(olapTable.writeQuorum())).append("\"");
}

// show lightSchemaChange only when it is set true
if (olapTable.getUseLightSchemaChange()) {
sb.append(",\n\"").append(PropertyAnalyzer.PROPERTIES_USE_LIGHT_SCHEMA_CHANGE).append("\" = \"");
sb.append(olapTable.getUseLightSchemaChange()).append("\"");
// show fastSchemaEvolution only when it is set true
if (olapTable.getUseFastSchemaEvolution()) {
sb.append(",\n\"").append(PropertyAnalyzer.PROPERTIES_USE_FAST_SCHEMA_EVOLUTION).append("\" = \"");
sb.append(olapTable.getUseFastSchemaEvolution()).append("\"");
}

// storage media
Expand Down
Loading

0 comments on commit b24c6ed

Please sign in to comment.