diff --git a/fe/fe-core/pom.xml b/fe/fe-core/pom.xml index b47cac79c9d33ff..c5e31fcaccacf99 100644 --- a/fe/fe-core/pom.xml +++ b/fe/fe-core/pom.xml @@ -557,12 +557,6 @@ under the License. ${paimon.version} - - io.delta - delta-standalone_2.12 - ${delta.version} - - software.amazon.awssdk glue diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index d558f88692ac03c..09e5b1ef0dd4a50 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -4360,7 +4360,8 @@ public void replayRenamePartition(TableInfo tableInfo) throws MetaNotFoundExcept } private void renameColumn(Database db, OlapTable table, String colName, - String newColName, boolean isReplay) throws DdlException { + String newColName, Map indexIdToSchemaVersion, + boolean isReplay) throws DdlException { table.checkNormalStateForAlter(); if (colName.equalsIgnoreCase(newColName)) { throw new DdlException("Same column name"); @@ -4406,6 +4407,12 @@ private void renameColumn(Database db, OlapTable table, String colName, Env.getCurrentEnv().getQueryStats() .rename(Env.getCurrentEnv().getCurrentCatalog().getId(), db.getId(), table.getId(), entry.getKey(), colName, newColName); + if (!isReplay) { + indexIdToSchemaVersion.put(entry.getKey(), entry.getValue().getSchemaVersion() + 1); + } + if (indexIdToSchemaVersion != null) { + entry.getValue().setSchemaVersion(indexIdToSchemaVersion.get(entry.getKey())); + } } } if (!hasColumn) { @@ -4466,7 +4473,8 @@ private void renameColumn(Database db, OlapTable table, String colName, if (!isReplay) { // log - TableRenameColumnInfo info = new TableRenameColumnInfo(db.getId(), table.getId(), colName, newColName); + TableRenameColumnInfo info = new TableRenameColumnInfo(db.getId(), table.getId(), colName, newColName, + indexIdToSchemaVersion); editLog.logColumnRename(info); LOG.info("rename coloumn[{}] to {}", colName, newColName); } @@ -4477,7 +4485,8 @@ public void renameColumn(Database db, OlapTable table, ColumnRenameClause rename try { String colName = renameClause.getColName(); String newColName = renameClause.getNewColName(); - renameColumn(db, table, colName, newColName, false); + Map indexIdToSchemaVersion = new HashMap(); + renameColumn(db, table, colName, newColName, indexIdToSchemaVersion, false); } finally { table.writeUnlock(); } @@ -4489,12 +4498,13 @@ public void replayRenameColumn(TableRenameColumnInfo info) throws MetaNotFoundEx long tableId = info.getTableId(); String colName = info.getColName(); String newColName = info.getNewColName(); + Map indexIdToSchemaVersion = info.getIndexIdToSchemaVersion(); Database db = getCurrentEnv().getInternalCatalog().getDbOrMetaException(dbId); OlapTable table = (OlapTable) db.getTableOrMetaException(tableId, TableType.OLAP); table.writeLock(); try { - renameColumn(db, table, colName, newColName, true); + renameColumn(db, table, colName, newColName, indexIdToSchemaVersion, true); } catch (DdlException e) { // should not happen LOG.warn("failed to replay rename column", e); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java index e5f14a8a31ef2f8..3539d17e269a53e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java @@ -156,7 +156,7 @@ enum TableType { MYSQL, ODBC, OLAP, SCHEMA, INLINE_VIEW, VIEW, BROKER, ELASTICSEARCH, HIVE, ICEBERG, @Deprecated HUDI, JDBC, TABLE_VALUED_FUNCTION, HMS_EXTERNAL_TABLE, ES_EXTERNAL_TABLE, MATERIALIZED_VIEW, JDBC_EXTERNAL_TABLE, ICEBERG_EXTERNAL_TABLE, TEST_EXTERNAL_TABLE, PAIMON_EXTERNAL_TABLE, MAX_COMPUTE_EXTERNAL_TABLE, - HUDI_EXTERNAL_TABLE, DELTALAKE_EXTERNAL_TABLE; + HUDI_EXTERNAL_TABLE; public String toEngineName() { switch (this) { @@ -193,8 +193,6 @@ public String toEngineName() { return "iceberg"; case HUDI_EXTERNAL_TABLE: return "hudi"; - case DELTALAKE_EXTERNAL_TABLE: - return "deltalake"; default: return null; } @@ -223,7 +221,6 @@ public String toMysqlType() { case ES_EXTERNAL_TABLE: case ICEBERG_EXTERNAL_TABLE: case PAIMON_EXTERNAL_TABLE: - case DELTALAKE_EXTERNAL_TABLE: return "EXTERNAL TABLE"; default: return null; diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/DeltaLakeExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/DeltaLakeExternalTable.java deleted file mode 100644 index cd5bb162001bc1c..000000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/DeltaLakeExternalTable.java +++ /dev/null @@ -1,159 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.catalog.external; - -import org.apache.doris.catalog.ArrayType; -import org.apache.doris.catalog.Column; -import org.apache.doris.catalog.HiveMetaStoreClientHelper; -import org.apache.doris.catalog.MapType; -import org.apache.doris.catalog.ScalarType; -import org.apache.doris.catalog.StructType; -import org.apache.doris.catalog.Type; -import org.apache.doris.datasource.HMSExternalCatalog; -import org.apache.doris.datasource.deltalake.DeltaLakeExternalCatalog; - -import com.google.common.collect.Lists; -import io.delta.standalone.DeltaLog; -import io.delta.standalone.actions.Metadata; -import io.delta.standalone.types.DataType; -import io.delta.standalone.types.StructField; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.metastore.api.FieldSchema; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -public class DeltaLakeExternalTable extends HMSExternalTable { - public DeltaLakeExternalTable(long id, String name, String dbName, - DeltaLakeExternalCatalog catalog) { - super(id, name, dbName, catalog, TableType.DELTALAKE_EXTERNAL_TABLE); - } - - @Override - protected synchronized void makeSureInitialized() { - super.makeSureInitialized(); - if (!objectCreated) { - remoteTable = ((HMSExternalCatalog) catalog).getClient().getTable(dbName, name); - if (remoteTable == null) { - dlaType = DLAType.UNKNOWN; - } else { - if (supportedDeltaLakeTable()) { - dlaType = DLAType.DELTALAKE; - } else { - dlaType = DLAType.UNKNOWN; - } - } - objectCreated = true; - } - } - - private boolean supportedDeltaLakeTable() { - Map parameters = remoteTable.getParameters(); - if (parameters == null) { - return false; - } - // Check that the 'spark.sql.sources.provider' parameter exists and has a value of 'delta' - return "delta".equalsIgnoreCase(parameters.get("spark.sql.sources.provider")); - } - - @Override - public List initSchema() { - makeSureInitialized(); - List columns; - List schema = ((DeltaLakeExternalCatalog) catalog).getClient().getSchema(dbName, name); - io.delta.standalone.types.StructType deltaSchema = getDeltaTableSchema(this); - List tmpSchema = Lists.newArrayListWithCapacity(schema.size()); - for (StructField field : deltaSchema.getFields()) { - String columnName = field.getName(); - tmpSchema.add(new Column(columnName, fromDeltaTypeToDorisType(field.getDataType()), - true, null, true, null, "", true, null, -1, null)); - } - columns = tmpSchema; - initPartitionColumns(columns); - return columns; - } - - private static io.delta.standalone.types.StructType getDeltaTableSchema(DeltaLakeExternalTable table) { - String path = table.getRemoteTable().getSd().getLocation(); - Configuration conf = HiveMetaStoreClientHelper.getConfiguration(table); - DeltaLog deltaLog = DeltaLog.forTable(conf, path); - Metadata metadata = deltaLog.snapshot().getMetadata(); - io.delta.standalone.types.StructType tableSchema = metadata.getSchema(); - return tableSchema; - } - - private static Type fromDeltaTypeToDorisType(DataType dataType) { - String typeName = dataType.getTypeName(); - switch (typeName) { - case "boolean": - return Type.BOOLEAN; - case "byte": - case "tinyint": - return Type.TINYINT; - case "smallint": - return Type.SMALLINT; - case "integer": - return Type.INT; - case "long": - return Type.BIGINT; - case "float": - return Type.FLOAT; - case "double": - return Type.DOUBLE; - case "date": - return Type.DATEV2; - case "timestamp": - return ScalarType.createDatetimeV2Type(6); - case "string": - return Type.STRING; - case "decimal": - int precision = ((io.delta.standalone.types.DecimalType) dataType).getPrecision(); - int scale = ((io.delta.standalone.types.DecimalType) dataType).getScale(); - return ScalarType.createDecimalV3Type(precision, scale); - case "array": - io.delta.standalone.types.ArrayType arrayType = (io.delta.standalone.types.ArrayType) dataType; - Type innerType = fromDeltaTypeToDorisType(arrayType.getElementType()); - return ArrayType.create(innerType, true); - case "map": - io.delta.standalone.types.MapType mapType = (io.delta.standalone.types.MapType) dataType; - return new MapType(Type.STRING, fromDeltaTypeToDorisType(mapType.getValueType())); - case "struct": - io.delta.standalone.types.StructType deltaStructType = (io.delta.standalone.types.StructType) dataType; - ArrayList dorisFields = new ArrayList<>(); - for (io.delta.standalone.types.StructField deltaField : deltaStructType.getFields()) { - // Convert the Delta field type to a Doris type - Type dorisFieldType = fromDeltaTypeToDorisType(deltaField.getDataType()); - - // Create a Doris struct field with the same name and type - org.apache.doris.catalog.StructField dorisField = new org.apache.doris.catalog.StructField( - deltaField.getName(), dorisFieldType); - - // Add the Doris field to the list - dorisFields.add(dorisField); - } - // Create a Doris struct type with the converted fields - return new StructType(dorisFields); - case "null": - return Type.NULL; - case "binary": - default: - return Type.UNSUPPORTED; - } - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalDatabase.java index 318ea06f3413c2e..d75f86bd0883dac 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalDatabase.java @@ -46,10 +46,6 @@ public HMSExternalDatabase(ExternalCatalog extCatalog, long id, String name) { super(extCatalog, id, name, InitDatabaseLog.Type.HMS); } - public HMSExternalDatabase(ExternalCatalog extCatalog, long id, String name, InitDatabaseLog.Type type) { - super(extCatalog, id, name, type); - } - @Override protected HMSExternalTable getExternalTable(String tableName, long tblId, ExternalCatalog catalog) { return new HMSExternalTable(tblId, tableName, name, (HMSExternalCatalog) extCatalog); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java index 0243ad12f757708..38f42562764d0de 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java @@ -108,10 +108,10 @@ public class HMSExternalTable extends ExternalTable { SUPPORTED_HUDI_FILE_FORMATS.add("com.uber.hoodie.hadoop.realtime.HoodieRealtimeInputFormat"); } - protected volatile org.apache.hadoop.hive.metastore.api.Table remoteTable = null; - protected List partitionColumns; + private volatile org.apache.hadoop.hive.metastore.api.Table remoteTable = null; + private List partitionColumns; - protected DLAType dlaType = DLAType.UNKNOWN; + private DLAType dlaType = DLAType.UNKNOWN; // No as precise as row count in TableStats, but better than none. private long estimatedRowCount = -1; @@ -120,7 +120,7 @@ public class HMSExternalTable extends ExternalTable { protected volatile long eventUpdateTime; public enum DLAType { - UNKNOWN, HIVE, HUDI, ICEBERG, DELTALAKE + UNKNOWN, HIVE, HUDI, ICEBERG } /** @@ -135,10 +135,6 @@ public HMSExternalTable(long id, String name, String dbName, HMSExternalCatalog super(id, name, catalog, dbName, TableType.HMS_EXTERNAL_TABLE); } - public HMSExternalTable(long id, String name, String dbName, HMSExternalCatalog catalog, TableType type) { - super(id, name, catalog, dbName, type); - } - public boolean isSupportedHmsTable() { makeSureInitialized(); return dlaType != DLAType.UNKNOWN; @@ -493,7 +489,7 @@ private List getIcebergSchema(List hmsSchema) { return tmpSchema; } - protected void initPartitionColumns(List schema) { + private void initPartitionColumns(List schema) { List partitionKeys = remoteTable.getPartitionKeys().stream().map(FieldSchema::getName) .collect(Collectors.toList()); partitionColumns = Lists.newArrayListWithCapacity(partitionKeys.size()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogFactory.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogFactory.java index d3dd3fa5fbebb5a..09ad69ec8b661d1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogFactory.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogFactory.java @@ -28,7 +28,6 @@ import org.apache.doris.catalog.Resource; import org.apache.doris.common.DdlException; import org.apache.doris.common.FeConstants; -import org.apache.doris.datasource.deltalake.DeltaLakeExternalCatalog; import org.apache.doris.datasource.iceberg.IcebergExternalCatalogFactory; import org.apache.doris.datasource.jdbc.JdbcExternalCatalog; import org.apache.doris.datasource.paimon.PaimonExternalCatalogFactory; @@ -131,9 +130,6 @@ private static CatalogIf createCatalog(long catalogId, String name, String resou case "max_compute": catalog = new MaxComputeExternalCatalog(catalogId, name, resource, props, comment); break; - case "deltalake": - catalog = new DeltaLakeExternalCatalog(catalogId, name, resource, props, comment); - break; case "test": if (!FeConstants.runningUnitTest) { throw new DdlException("test catalog is only for FE unit test"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index 7393a75bac71f88..413f6554c88c5ce 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -22,7 +22,6 @@ import org.apache.doris.catalog.Env; import org.apache.doris.catalog.Resource; import org.apache.doris.catalog.TableIf; -import org.apache.doris.catalog.external.DeltaLakeExternalDataBase; import org.apache.doris.catalog.external.EsExternalDatabase; import org.apache.doris.catalog.external.ExternalDatabase; import org.apache.doris.catalog.external.ExternalTable; @@ -497,8 +496,6 @@ protected ExternalDatabase getDbForInit(String dbName, return new TestExternalDatabase(this, dbId, dbName); case PAIMON: return new PaimonExternalDatabase(this, dbId, dbName); - case DELTALAKE: - return new DeltaLakeExternalDataBase(this, dbId, dbName); default: break; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/HMSExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/HMSExternalCatalog.java index 408504f5cc2404e..6e3543dfccef94a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/HMSExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/HMSExternalCatalog.java @@ -84,13 +84,6 @@ public HMSExternalCatalog(long catalogId, String name, String resource, Map props, - String comment, InitCatalogLog.Type type) { - super(catalogId, name, type, comment); - props = PropertyConverter.convertToMetaProperties(props); - catalogProperty = new CatalogProperty(resource, props); - } - @Override public void checkProperties() throws DdlException { super.checkProperties(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InitCatalogLog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InitCatalogLog.java index e766324a726befe..dd30fbf43c95acc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InitCatalogLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InitCatalogLog.java @@ -40,7 +40,6 @@ public enum Type { PAIMON, MAX_COMPUTE, HUDI, - DELTALAKE, TEST, UNKNOWN; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InitDatabaseLog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InitDatabaseLog.java index 1659a38cc030435..3a85fb1edc54c4c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InitDatabaseLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InitDatabaseLog.java @@ -40,7 +40,6 @@ public enum Type { MAX_COMPUTE, HUDI, PAIMON, - DELTALAKE, TEST, UNKNOWN; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/deltalake/DeltaLakeExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/deltalake/DeltaLakeExternalCatalog.java deleted file mode 100644 index af142ea731592a6..000000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/deltalake/DeltaLakeExternalCatalog.java +++ /dev/null @@ -1,69 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.datasource.deltalake; - -import org.apache.doris.catalog.external.HMSExternalDatabase; -import org.apache.doris.catalog.external.HMSExternalTable; -import org.apache.doris.datasource.HMSExternalCatalog; -import org.apache.doris.datasource.InitCatalogLog; -import org.apache.doris.datasource.SessionContext; - -import com.google.common.collect.Lists; -import org.apache.hadoop.hive.metastore.api.Table; - -import java.util.List; -import java.util.Map; - -public class DeltaLakeExternalCatalog extends HMSExternalCatalog { - - public DeltaLakeExternalCatalog(long catalogId, String name, String resource, Map props, - String comment) { - super(catalogId, name, resource, props, comment, InitCatalogLog.Type.DELTALAKE); - } - - @Override - public List listTableNames(SessionContext ctx, String dbName) { - makeSureInitialized(); - HMSExternalDatabase hmsExternalDatabase = (HMSExternalDatabase) idToDb.get(dbNameToId.get(dbName)); - if (hmsExternalDatabase != null && hmsExternalDatabase.isInitialized()) { - List names = Lists.newArrayList(); - for (HMSExternalTable table : hmsExternalDatabase.getTables()) { - String tableName = table.getName(); - Table tableDetails = client.getTable(dbName, tableName); - Map parameters = tableDetails.getParameters(); - String provider = parameters.get("spark.sql.sources.provider"); - if ("delta".equalsIgnoreCase(provider)) { - names.add(tableName); - } - } - return names; - } else { - List allTableNames = client.getAllTables(getRealTableName(dbName)); - List deltaTableNames = Lists.newArrayList(); - for (String tableName : allTableNames) { - Table tableDetails = client.getTable(dbName, tableName); - Map parameters = tableDetails.getParameters(); - String provider = parameters.get("spark.sql.sources.provider"); - if ("delta".equalsIgnoreCase(provider)) { - deltaTableNames.add(tableName); - } - } - return deltaTableNames; - } - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java index 606233201285dfa..d4a8849aee85cfd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java @@ -69,7 +69,7 @@ import org.apache.doris.nereids.trees.UnaryNode; import org.apache.doris.nereids.trees.expressions.AggregateExpression; import org.apache.doris.nereids.trees.expressions.CTEId; -import org.apache.doris.nereids.trees.expressions.EqualTo; +import org.apache.doris.nereids.trees.expressions.EqualPredicate; import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; @@ -1138,7 +1138,7 @@ public PlanFragment visitPhysicalHashJoin( JoinType joinType = hashJoin.getJoinType(); List execEqConjuncts = hashJoin.getHashJoinConjuncts().stream() - .map(EqualTo.class::cast) + .map(EqualPredicate.class::cast) .map(e -> JoinUtils.swapEqualToForChildrenOrder(e, hashJoin.left().getOutputSet())) .map(e -> ExpressionTranslator.translate(e, context)) .collect(Collectors.toList()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterGenerator.java index 579fe7485ababfd..acfd924940842df 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterGenerator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterGenerator.java @@ -363,9 +363,10 @@ private void pushDownRuntimeFilterCommon(PhysicalHashJoin legalTypes = Arrays.stream(TRuntimeFilterType.values()) .filter(type -> (type.getValue() & ctx.getSessionVariable().getRuntimeFilterType()) > 0) .collect(Collectors.toList()); - for (int i = 0; i < join.getHashJoinConjuncts().size(); i++) { + List hashJoinConjuncts = join.getEqualToConjuncts(); + for (int i = 0; i < hashJoinConjuncts.size(); i++) { EqualTo equalTo = ((EqualTo) JoinUtils.swapEqualToForChildrenOrder( - (EqualTo) join.getHashJoinConjuncts().get(i), join.left().getOutputSet())); + hashJoinConjuncts.get(i), join.left().getOutputSet())); for (TRuntimeFilterType type : legalTypes) { //bitmap rf is generated by nested loop join. if (type == TRuntimeFilterType.BITMAP) { @@ -487,7 +488,7 @@ private void analyzeRuntimeFilterPushDownIntoCTEInfos(PhysicalHashJoin conditions = curJoin.getHashJoinConjuncts(); boolean inSameEqualSet = false; - for (Expression e : conditions) { + for (EqualTo e : curJoin.getEqualToConjuncts()) { if (e instanceof EqualTo) { - SlotReference oneSide = (SlotReference) ((EqualTo) e).left(); - SlotReference anotherSide = (SlotReference) ((EqualTo) e).right(); + SlotReference oneSide = (SlotReference) e.left(); + SlotReference anotherSide = (SlotReference) e.right(); if (anotherSideSlotSet.contains(oneSide) && anotherSideSlotSet.contains(anotherSide)) { inSameEqualSet = true; break; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateOuterJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateOuterJoin.java index 440e5d73ae31f0c..09e38ed056ec000 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateOuterJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateOuterJoin.java @@ -19,6 +19,7 @@ import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.trees.expressions.EqualPredicate; import org.apache.doris.nereids.trees.expressions.EqualTo; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.IsNull; @@ -91,22 +92,18 @@ public Rule build() { */ conjunctsChanged |= join.getHashJoinConjuncts().stream() .map(EqualTo.class::cast) - .map(equalTo -> - (EqualTo) JoinUtils.swapEqualToForChildrenOrder(equalTo, join.left().getOutputSet())) - .map(equalTo -> createIsNotNullIfNecessary(equalTo, conjuncts) - ).anyMatch(Boolean::booleanValue); + .map(equalTo -> JoinUtils.swapEqualToForChildrenOrder(equalTo, join.left().getOutputSet())) + .anyMatch(equalTo -> createIsNotNullIfNecessary(equalTo, conjuncts)); JoinUtils.JoinSlotCoverageChecker checker = new JoinUtils.JoinSlotCoverageChecker( join.left().getOutput(), join.right().getOutput()); - conjunctsChanged |= join.getOtherJoinConjuncts().stream().filter(EqualTo.class::isInstance) - .map(EqualTo.class::cast) - .filter(equalTo -> checker.isHashJoinCondition(equalTo)) - .map(equalTo -> (EqualTo) JoinUtils.swapEqualToForChildrenOrder(equalTo, + conjunctsChanged |= join.getOtherJoinConjuncts().stream() + .filter(EqualTo.class::isInstance) + .filter(equalTo -> checker.isHashJoinCondition((EqualPredicate) equalTo)) + .map(equalTo -> JoinUtils.swapEqualToForChildrenOrder((EqualPredicate) equalTo, join.left().getOutputSet())) - .map(equalTo -> - createIsNotNullIfNecessary(equalTo, conjuncts)) - .anyMatch(Boolean::booleanValue); + .anyMatch(equalTo -> createIsNotNullIfNecessary(equalTo, conjuncts)); } if (conjunctsChanged) { return filter.withConjuncts(conjuncts.stream().collect(ImmutableSet.toImmutableSet())) @@ -135,7 +132,7 @@ private JoinType tryEliminateOuterJoin(JoinType joinType, boolean canFilterLeftN return joinType; } - private boolean createIsNotNullIfNecessary(EqualTo swapedEqualTo, Collection container) { + private boolean createIsNotNullIfNecessary(EqualPredicate swapedEqualTo, Collection container) { boolean containerChanged = false; if (swapedEqualTo.left().nullable()) { Not not = new Not(new IsNull(swapedEqualTo.left())); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushdownExpressionsInHashCondition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushdownExpressionsInHashCondition.java index 05da591526cd834..df7acb4553c6aee 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushdownExpressionsInHashCondition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushdownExpressionsInHashCondition.java @@ -20,7 +20,7 @@ import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.trees.expressions.Alias; -import org.apache.doris.nereids.trees.expressions.EqualTo; +import org.apache.doris.nereids.trees.expressions.EqualPredicate; import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; @@ -77,11 +77,10 @@ public Rule build() { Set rightProjectExprs = Sets.newHashSet(); Map exprReplaceMap = Maps.newHashMap(); join.getHashJoinConjuncts().forEach(conjunct -> { - Preconditions.checkArgument(conjunct instanceof EqualTo); + Preconditions.checkArgument(conjunct instanceof EqualPredicate); // sometimes: t1 join t2 on t2.a + 1 = t1.a + 2, so check the situation, but actually it // doesn't swap the two sides. - conjunct = JoinUtils.swapEqualToForChildrenOrder( - (EqualTo) conjunct, join.left().getOutputSet()); + conjunct = JoinUtils.swapEqualToForChildrenOrder((EqualPredicate) conjunct, join.left().getOutputSet()); generateReplaceMapAndProjectExprs(conjunct.child(0), exprReplaceMap, leftProjectExprs); generateReplaceMapAndProjectExprs(conjunct.child(1), exprReplaceMap, rightProjectExprs); }); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractSelectMaterializedIndexRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractSelectMaterializedIndexRule.java index 012dec4c91cf49f..c1550cb5bd5d5bf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractSelectMaterializedIndexRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractSelectMaterializedIndexRule.java @@ -27,13 +27,12 @@ import org.apache.doris.nereids.trees.expressions.CaseWhen; import org.apache.doris.nereids.trees.expressions.Cast; import org.apache.doris.nereids.trees.expressions.ComparisonPredicate; -import org.apache.doris.nereids.trees.expressions.EqualTo; +import org.apache.doris.nereids.trees.expressions.EqualPredicate; import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.InPredicate; import org.apache.doris.nereids.trees.expressions.IsNull; import org.apache.doris.nereids.trees.expressions.NamedExpression; -import org.apache.doris.nereids.trees.expressions.NullSafeEqual; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.WhenClause; @@ -306,7 +305,7 @@ public PrefixIndexCheckResult visitInPredicate(InPredicate in, Map context) { - if (cp instanceof EqualTo || cp instanceof NullSafeEqual) { + if (cp instanceof EqualPredicate) { return check(cp, context, PrefixIndexCheckResult::createEqual); } else { return check(cp, context, PrefixIndexCheckResult::createNonEqual); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java index 055f3b88a07eff5..e2d7f40622fd01a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java @@ -23,6 +23,7 @@ import org.apache.doris.nereids.trees.expressions.And; import org.apache.doris.nereids.trees.expressions.ComparisonPredicate; import org.apache.doris.nereids.trees.expressions.CompoundPredicate; +import org.apache.doris.nereids.trees.expressions.EqualPredicate; import org.apache.doris.nereids.trees.expressions.EqualTo; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.GreaterThan; @@ -33,7 +34,6 @@ import org.apache.doris.nereids.trees.expressions.LessThanEqual; import org.apache.doris.nereids.trees.expressions.Like; import org.apache.doris.nereids.trees.expressions.Not; -import org.apache.doris.nereids.trees.expressions.NullSafeEqual; import org.apache.doris.nereids.trees.expressions.Or; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; @@ -210,7 +210,7 @@ private Statistics calculateWhenLiteralRight(ComparisonPredicate cp, return context.statistics.withSel(DEFAULT_INEQUALITY_COEFFICIENT); } - if (cp instanceof EqualTo || cp instanceof NullSafeEqual) { + if (cp instanceof EqualPredicate) { return estimateEqualTo(cp, statsForLeft, statsForRight, context); } else { if (cp instanceof LessThan || cp instanceof LessThanEqual) { @@ -255,7 +255,7 @@ private Statistics calculateWhenBothColumn(ComparisonPredicate cp, EstimationCon ColumnStatistic statsForLeft, ColumnStatistic statsForRight) { Expression left = cp.left(); Expression right = cp.right(); - if (cp instanceof EqualTo || cp instanceof NullSafeEqual) { + if (cp instanceof EqualPredicate) { return estimateColumnEqualToColumn(left, statsForLeft, right, statsForRight, context); } if (cp instanceof GreaterThan || cp instanceof GreaterThanEqual) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/JoinEstimation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/JoinEstimation.java index 3b7797439f3edec..d43171375b8a936 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/JoinEstimation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/JoinEstimation.java @@ -19,7 +19,7 @@ import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.trees.expressions.Cast; -import org.apache.doris.nereids.trees.expressions.EqualTo; +import org.apache.doris.nereids.trees.expressions.EqualPredicate; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.JoinType; @@ -45,14 +45,14 @@ public class JoinEstimation { private static double DEFAULT_ANTI_JOIN_SELECTIVITY_COEFFICIENT = 0.3; - private static EqualTo normalizeHashJoinCondition(EqualTo equalTo, Statistics leftStats, Statistics rightStats) { - boolean changeOrder = equalTo.left().getInputSlots().stream().anyMatch( - slot -> rightStats.findColumnStatistics(slot) != null - ); + private static EqualPredicate normalizeHashJoinCondition(EqualPredicate equal, Statistics leftStats, + Statistics rightStats) { + boolean changeOrder = equal.left().getInputSlots().stream() + .anyMatch(slot -> rightStats.findColumnStatistics(slot) != null); if (changeOrder) { - return new EqualTo(equalTo.right(), equalTo.left()); + return equal.commute(); } else { - return equalTo; + return equal; } } @@ -81,18 +81,18 @@ private static Statistics estimateHashJoin(Statistics leftStats, Statistics righ * In order to avoid error propagation, for unTrustEquations, we only use the biggest selectivity. */ List unTrustEqualRatio = Lists.newArrayList(); - List unTrustableCondition = Lists.newArrayList(); + List unTrustableCondition = Lists.newArrayList(); boolean leftBigger = leftStats.getRowCount() > rightStats.getRowCount(); double rightStatsRowCount = StatsMathUtil.nonZeroDivisor(rightStats.getRowCount()); double leftStatsRowCount = StatsMathUtil.nonZeroDivisor(leftStats.getRowCount()); - List trustableConditions = join.getHashJoinConjuncts().stream() - .map(expression -> (EqualTo) expression) + List trustableConditions = join.getHashJoinConjuncts().stream() + .map(expression -> (EqualPredicate) expression) .filter( expression -> { // since ndv is not accurate, if ndv/rowcount < almostUniqueThreshold, // this column is regarded as unique. double almostUniqueThreshold = 0.9; - EqualTo equal = normalizeHashJoinCondition(expression, leftStats, rightStats); + EqualPredicate equal = normalizeHashJoinCondition(expression, leftStats, rightStats); ColumnStatistic eqLeftColStats = ExpressionEstimation.estimate(equal.left(), leftStats); ColumnStatistic eqRightColStats = ExpressionEstimation.estimate(equal.right(), rightStats); boolean trustable = eqRightColStats.ndv / rightStatsRowCount > almostUniqueThreshold @@ -204,7 +204,7 @@ private static double estimateJoinConditionSel(Statistics crossJoinStats, Expres } private static double estimateSemiOrAntiRowCountBySlotsEqual(Statistics leftStats, - Statistics rightStats, Join join, EqualTo equalTo) { + Statistics rightStats, Join join, EqualPredicate equalTo) { Expression eqLeft = equalTo.left(); Expression eqRight = equalTo.right(); ColumnStatistic probColStats = leftStats.findColumnStatistics(eqLeft); @@ -261,7 +261,7 @@ private static Statistics estimateSemiOrAnti(Statistics leftStats, Statistics ri double rowCount = Double.POSITIVE_INFINITY; for (Expression conjunct : join.getHashJoinConjuncts()) { double eqRowCount = estimateSemiOrAntiRowCountBySlotsEqual(leftStats, rightStats, - join, (EqualTo) conjunct); + join, (EqualPredicate) conjunct); if (rowCount > eqRowCount) { rowCount = eqRowCount; } @@ -336,7 +336,7 @@ public static Statistics estimate(Statistics leftStats, Statistics rightStats, J private static Statistics updateJoinResultStatsByHashJoinCondition(Statistics innerStats, Join join) { Map updatedCols = new HashMap<>(); for (Expression expr : join.getHashJoinConjuncts()) { - EqualTo equalTo = (EqualTo) expr; + EqualPredicate equalTo = (EqualPredicate) expr; ColumnStatistic leftColStats = ExpressionEstimation.estimate(equalTo.left(), innerStats); ColumnStatistic rightColStats = ExpressionEstimation.estimate(equalTo.right(), innerStats); double minNdv = Math.min(leftColStats.ndv, rightColStats.ndv); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/DeltaLakeExternalDataBase.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/EqualPredicate.java similarity index 55% rename from fe/fe-core/src/main/java/org/apache/doris/catalog/external/DeltaLakeExternalDataBase.java rename to fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/EqualPredicate.java index 2db5c4eb8357184..3f61bd3cf621a5a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/DeltaLakeExternalDataBase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/EqualPredicate.java @@ -15,20 +15,22 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.catalog.external; +package org.apache.doris.nereids.trees.expressions; -import org.apache.doris.datasource.ExternalCatalog; -import org.apache.doris.datasource.InitDatabaseLog; -import org.apache.doris.datasource.deltalake.DeltaLakeExternalCatalog; +import java.util.List; -public class DeltaLakeExternalDataBase extends HMSExternalDatabase { +/** + * EqualPredicate + */ +public abstract class EqualPredicate extends ComparisonPredicate { - public DeltaLakeExternalDataBase(ExternalCatalog extCatalog, long id, String name) { - super(extCatalog, id, name, InitDatabaseLog.Type.DELTALAKE); + protected EqualPredicate(List children, String symbol) { + super(children, symbol); } @Override - protected DeltaLakeExternalTable getExternalTable(String tableName, long tblId, ExternalCatalog catalog) { - return new DeltaLakeExternalTable(tblId, tableName, name, (DeltaLakeExternalCatalog) extCatalog); + public EqualPredicate commute() { + return null; } } + diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/EqualTo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/EqualTo.java index 065f6b9340312de..3faccff6d99651f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/EqualTo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/EqualTo.java @@ -29,7 +29,7 @@ /** * Equal to expression: a = b. */ -public class EqualTo extends ComparisonPredicate implements PropagateNullable { +public class EqualTo extends EqualPredicate implements PropagateNullable { public EqualTo(Expression left, Expression right) { super(ImmutableList.of(left, right), "="); @@ -55,7 +55,7 @@ public R accept(ExpressionVisitor visitor, C context) { } @Override - public ComparisonPredicate commute() { + public EqualTo commute() { return new EqualTo(right(), left()); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/NullSafeEqual.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/NullSafeEqual.java index c2b63aebbd793a7..48d05364fa3441c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/NullSafeEqual.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/NullSafeEqual.java @@ -29,13 +29,7 @@ * Null safe equal expression: a <=> b. * Unlike normal equal to expression, null <=> null is true. */ -public class NullSafeEqual extends ComparisonPredicate implements AlwaysNotNullable { - /** - * Constructor of Null Safe Equal ComparisonPredicate. - * - * @param left left child of Null Safe Equal - * @param right right child of Null Safe Equal - */ +public class NullSafeEqual extends EqualPredicate implements AlwaysNotNullable { public NullSafeEqual(Expression left, Expression right) { super(ImmutableList.of(left, right), "<=>"); } @@ -61,8 +55,7 @@ public NullSafeEqual withChildren(List children) { } @Override - public ComparisonPredicate commute() { + public NullSafeEqual commute() { return new NullSafeEqual(right(), left()); } - } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/AbstractPhysicalJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/AbstractPhysicalJoin.java index c6bdd7b6b200130..67cdef940bf2201 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/AbstractPhysicalJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/AbstractPhysicalJoin.java @@ -20,6 +20,7 @@ import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.PhysicalProperties; +import org.apache.doris.nereids.trees.expressions.EqualTo; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.MarkJoinSlotReference; import org.apache.doris.nereids.trees.expressions.Slot; @@ -114,6 +115,11 @@ public List getHashJoinConjuncts() { return hashJoinConjuncts; } + public List getEqualToConjuncts() { + return hashJoinConjuncts.stream().filter(EqualTo.class::isInstance).map(EqualTo.class::cast) + .collect(Collectors.toList()); + } + public boolean isShouldTranslateOutput() { return shouldTranslateOutput; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashJoin.java index 0041812796a2f23..183ccaabfa82ed6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashJoin.java @@ -25,7 +25,7 @@ import org.apache.doris.nereids.processor.post.RuntimeFilterGenerator; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.PhysicalProperties; -import org.apache.doris.nereids.trees.expressions.EqualTo; +import org.apache.doris.nereids.trees.expressions.EqualPredicate; import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.MarkJoinSlotReference; @@ -213,7 +213,7 @@ public boolean pushDownRuntimeFilter(CascadesContext context, IdGenerator left, List right) { rightExprIds = right.stream().map(Slot::getExprId).collect(Collectors.toSet()); } - JoinSlotCoverageChecker(Set left, Set right) { - leftExprIds = left; - rightExprIds = right; - } - - /** - * PushDownExpressionInHashConjuncts ensure the "slots" is only one slot. - */ - boolean isCoveredByLeftSlots(ExprId slot) { - return leftExprIds.contains(slot); - } - - boolean isCoveredByRightSlots(ExprId slot) { - return rightExprIds.contains(slot); - } - /** * consider following cases: * 1# A=1 => not for hash table @@ -112,25 +96,20 @@ boolean isCoveredByRightSlots(ExprId slot) { * 4# t1.a=t2.a or t1.b=t2.b not for hash table * 5# t1.a > 1 not for hash table * - * @param equalTo a conjunct in on clause condition + * @param equal a conjunct in on clause condition * @return true if the equal can be used as hash join condition */ - public boolean isHashJoinCondition(EqualTo equalTo) { - Set equalLeft = equalTo.left().getInputSlots(); - if (equalLeft.isEmpty()) { + public boolean isHashJoinCondition(EqualPredicate equal) { + Set equalLeftExprIds = equal.left().getInputSlotExprIds(); + if (equalLeftExprIds.isEmpty()) { return false; } - Set equalRight = equalTo.right().getInputSlots(); - if (equalRight.isEmpty()) { + Set equalRightExprIds = equal.right().getInputSlotExprIds(); + if (equalRightExprIds.isEmpty()) { return false; } - List equalLeftExprIds = equalLeft.stream() - .map(Slot::getExprId).collect(Collectors.toList()); - - List equalRightExprIds = equalRight.stream() - .map(Slot::getExprId).collect(Collectors.toList()); return leftExprIds.containsAll(equalLeftExprIds) && rightExprIds.containsAll(equalRightExprIds) || leftExprIds.containsAll(equalRightExprIds) && rightExprIds.containsAll(equalLeftExprIds); } @@ -147,9 +126,8 @@ public boolean isHashJoinCondition(EqualTo equalTo) { public static Pair, List> extractExpressionForHashTable(List leftSlots, List rightSlots, List onConditions) { JoinSlotCoverageChecker checker = new JoinSlotCoverageChecker(leftSlots, rightSlots); - Map> mapper = onConditions.stream() - .collect(Collectors.groupingBy( - expr -> (expr instanceof EqualTo) && checker.isHashJoinCondition((EqualTo) expr))); + Map> mapper = onConditions.stream().collect(Collectors.groupingBy( + expr -> (expr instanceof EqualPredicate) && checker.isHashJoinCondition((EqualPredicate) expr))); return Pair.of( mapper.getOrDefault(true, ImmutableList.of()), mapper.getOrDefault(false, ImmutableList.of()) @@ -205,7 +183,7 @@ public static boolean shouldNestedLoopJoin(JoinType joinType, List h * The left child of origin predicate is t2.id and the right child of origin predicate is t1.id. * In this situation, the children of predicate need to be swap => t1.id=t2.id. */ - public static Expression swapEqualToForChildrenOrder(EqualTo equalTo, Set leftOutput) { + public static EqualPredicate swapEqualToForChildrenOrder(EqualPredicate equalTo, Set leftOutput) { if (leftOutput.containsAll(equalTo.left().getInputSlots())) { return equalTo; } else { diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/TableRenameColumnInfo.java b/fe/fe-core/src/main/java/org/apache/doris/persist/TableRenameColumnInfo.java index aec3a56a4fc52ba..eafdb943e1164b1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/TableRenameColumnInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/TableRenameColumnInfo.java @@ -26,6 +26,7 @@ import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; +import java.util.Map; /** * PersistInfo for Table rename column info @@ -39,13 +40,16 @@ public class TableRenameColumnInfo implements Writable { private String colName; @SerializedName(value = "newColName") private String newColName; + @SerializedName(value = "indexIdToSchemaVersion") + private Map indexIdToSchemaVersion; public TableRenameColumnInfo(long dbId, long tableId, - String colName, String newColName) { + String colName, String newColName, Map indexIdToSchemaVersion) { this.dbId = dbId; this.tableId = tableId; this.colName = colName; this.newColName = newColName; + this.indexIdToSchemaVersion = indexIdToSchemaVersion; } public long getDbId() { @@ -64,6 +68,10 @@ public String getNewColName() { return newColName; } + public Map getIndexIdToSchemaVersion() { + return indexIdToSchemaVersion; + } + @Override public void write(DataOutput out) throws IOException { Text.writeString(out, GsonUtils.GSON.toJson(this)); @@ -86,7 +94,8 @@ public boolean equals(Object obj) { TableRenameColumnInfo info = (TableRenameColumnInfo) obj; return (dbId == info.dbId && tableId == info.tableId - && colName.equals(info.colName) && newColName.equals(info.newColName)); + && colName.equals(info.colName) && newColName.equals(info.newColName) + && indexIdToSchemaVersion.equals(info.indexIdToSchemaVersion)); } @Override @@ -96,6 +105,7 @@ public String toString() { sb.append(" tableId: ").append(tableId); sb.append(" colName: ").append(colName); sb.append(" newColName: ").append(newColName); + sb.append(" indexIdToSchemaVersion: ").append(indexIdToSchemaVersion.toString()); return sb.toString(); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java b/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java index 8b6f85d511eef54..f467128780114e1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java @@ -44,8 +44,6 @@ import org.apache.doris.catalog.SparkResource; import org.apache.doris.catalog.StructType; import org.apache.doris.catalog.TableIf; -import org.apache.doris.catalog.external.DeltaLakeExternalDataBase; -import org.apache.doris.catalog.external.DeltaLakeExternalTable; import org.apache.doris.catalog.external.EsExternalDatabase; import org.apache.doris.catalog.external.EsExternalTable; import org.apache.doris.catalog.external.ExternalDatabase; @@ -66,7 +64,6 @@ import org.apache.doris.datasource.HMSExternalCatalog; import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.datasource.MaxComputeExternalCatalog; -import org.apache.doris.datasource.deltalake.DeltaLakeExternalCatalog; import org.apache.doris.datasource.iceberg.IcebergDLFExternalCatalog; import org.apache.doris.datasource.iceberg.IcebergExternalCatalog; import org.apache.doris.datasource.iceberg.IcebergGlueExternalCatalog; @@ -214,8 +211,7 @@ public class GsonUtils { .registerSubtype(PaimonExternalCatalog.class, PaimonExternalCatalog.class.getSimpleName()) .registerSubtype(PaimonHMSExternalCatalog.class, PaimonHMSExternalCatalog.class.getSimpleName()) .registerSubtype(PaimonFileExternalCatalog.class, PaimonFileExternalCatalog.class.getSimpleName()) - .registerSubtype(MaxComputeExternalCatalog.class, MaxComputeExternalCatalog.class.getSimpleName()) - .registerSubtype(DeltaLakeExternalCatalog.class, DeltaLakeExternalCatalog.class.getSimpleName()); + .registerSubtype(MaxComputeExternalCatalog.class, MaxComputeExternalCatalog.class.getSimpleName()); // routine load data source private static RuntimeTypeAdapterFactory rdsTypeAdapterFactory = RuntimeTypeAdapterFactory.of( @@ -234,8 +230,7 @@ public class GsonUtils { .registerSubtype(JdbcExternalDatabase.class, JdbcExternalDatabase.class.getSimpleName()) .registerSubtype(IcebergExternalDatabase.class, IcebergExternalDatabase.class.getSimpleName()) .registerSubtype(PaimonExternalDatabase.class, PaimonExternalDatabase.class.getSimpleName()) - .registerSubtype(MaxComputeExternalDatabase.class, MaxComputeExternalDatabase.class.getSimpleName()) - .registerSubtype(DeltaLakeExternalDataBase.class, DeltaLakeExternalDataBase.class.getSimpleName()); + .registerSubtype(MaxComputeExternalDatabase.class, MaxComputeExternalDatabase.class.getSimpleName()); private static RuntimeTypeAdapterFactory tblTypeAdapterFactory = RuntimeTypeAdapterFactory.of( TableIf.class, "clazz").registerSubtype(ExternalTable.class, ExternalTable.class.getSimpleName()) @@ -245,8 +240,7 @@ public class GsonUtils { .registerSubtype(JdbcExternalTable.class, JdbcExternalTable.class.getSimpleName()) .registerSubtype(IcebergExternalTable.class, IcebergExternalTable.class.getSimpleName()) .registerSubtype(PaimonExternalTable.class, PaimonExternalTable.class.getSimpleName()) - .registerSubtype(MaxComputeExternalTable.class, MaxComputeExternalTable.class.getSimpleName()) - .registerSubtype(DeltaLakeExternalTable.class, DeltaLakeExternalTable.class.getSimpleName()); + .registerSubtype(MaxComputeExternalTable.class, MaxComputeExternalTable.class.getSimpleName()); // runtime adapter for class "PartitionInfo" private static RuntimeTypeAdapterFactory partitionInfoTypeAdapterFactory diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java b/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java index 8e128b3b4a05efd..1ba48b0b9df9a6d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java @@ -1926,7 +1926,7 @@ private PlanNode createScanNode(Analyzer analyzer, TableRef tblRef, SelectStmt s ((HiveScanNode) scanNode).setTableSample(tblRef.getTableSample()); break; default: - throw new UserException("Not supported table type: " + ((HMSExternalTable) table).getDlaType()); + throw new UserException("Not supported table type" + table.getType()); } break; case ICEBERG_EXTERNAL_TABLE: @@ -1950,7 +1950,7 @@ private PlanNode createScanNode(Analyzer analyzer, TableRef tblRef, SelectStmt s scanNode = new TestExternalTableScanNode(ctx.getNextNodeId(), tblRef.getDesc()); break; default: - throw new UserException("Not supported table type: " + tblRef.getTable().getType()); + throw new UserException("Not supported table type" + tblRef.getTable().getType()); } if (scanNode instanceof OlapScanNode || scanNode instanceof EsScanNode || scanNode instanceof OdbcScanNode || scanNode instanceof JdbcScanNode diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java index a278200e5c724ef..f3fa143b5286225 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java @@ -72,8 +72,8 @@ public abstract class BaseAnalysisTask { + "${idxId} AS `idx_id`, " + "'${colId}' AS `col_id`, " + "NULL AS `part_id`, " - + "ROUND(COUNT(1) * ${scaleFactor}) AS `row_count`, " - + "ROUND(NDV(`${colName}`) * ${scaleFactor}) as `ndv`, " + + "${rowCount} AS `row_count`, " + + "${ndvFunction} as `ndv`, " + "ROUND(SUM(CASE WHEN `${colName}` IS NULL THEN 1 ELSE 0 END) * ${scaleFactor}) AS `null_count`, " + "${min} AS `min`, " + "${max} AS `max`, " diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/HMSAnalysisTask.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/HMSAnalysisTask.java index 812bd615a6949f0..4c12236fb40412a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/HMSAnalysisTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/HMSAnalysisTask.java @@ -145,6 +145,7 @@ private void getOrdinaryColumnStats() throws Exception { if (distributionColumns.size() == 1 && distributionColumns.contains(col.getName().toLowerCase())) { bucketFlag = true; sb.append(LINEAR_ANALYZE_TEMPLATE); + params.put("ndvFunction", "ROUND(NDV(`${colName}`) * ${scaleFactor})"); params.put("rowCount", "ROUND(count(1) * ${scaleFactor})"); } else { sb.append(DUJ1_ANALYZE_TEMPLATE); diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java index d7037580595adcd..97cb10c520c7d89 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java @@ -17,6 +17,7 @@ package org.apache.doris.statistics; +import org.apache.doris.catalog.KeysType; import org.apache.doris.catalog.MaterializedIndex; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Partition; @@ -129,21 +130,26 @@ protected void doSample() throws Exception { } StringSubstitutor stringSubstitutor = new StringSubstitutor(params); String sql; - // Single distribution column is not fit for DUJ1 estimator, use linear estimator. - Set distributionColumns = tbl.getDistributionColumnNames(); - if (distributionColumns.size() == 1 && distributionColumns.contains(col.getName().toLowerCase())) { + if (useLinearAnalyzeTemplate()) { params.put("min", StatisticsUtil.quote(min)); params.put("max", StatisticsUtil.quote(max)); + // For single unique key, use count as ndv. + if (isSingleUniqueKey()) { + params.put("ndvFunction", String.valueOf(rowCount)); + } else { + params.put("ndvFunction", "ROUND(NDV(`${colName}`) * ${scaleFactor})"); + } sql = stringSubstitutor.replace(LINEAR_ANALYZE_TEMPLATE); } else { params.put("dataSizeFunction", getDataSizeFunction(col, true)); sql = stringSubstitutor.replace(DUJ1_ANALYZE_TEMPLATE); } LOG.info("Sample for column [{}]. Total rows [{}], rows to sample [{}], scale factor [{}], " - + "limited [{}], distribute column [{}], partition column [{}], key column [{}]", + + "limited [{}], distribute column [{}], partition column [{}], key column [{}], " + + "is single unique key [{}]", col.getName(), params.get("rowCount"), rowsToSample, params.get("scaleFactor"), limitFlag, tbl.isDistributionColumn(col.getName()), - tbl.isPartitionColumn(col.getName()), col.isKey()); + tbl.isPartitionColumn(col.getName()), col.isKey(), isSingleUniqueKey()); runQuery(sql, false); } } @@ -278,4 +284,28 @@ protected long getSampleRows() { } return sampleRows; } + + /** + * Check if the task should use linear analyze template. + * @return True for single unique key column and single distribution column. + */ + protected boolean useLinearAnalyzeTemplate() { + if (isSingleUniqueKey()) { + return true; + } + Set distributionColumns = tbl.getDistributionColumnNames(); + return distributionColumns.size() == 1 && distributionColumns.contains(col.getName().toLowerCase()); + } + + /** + * Check if the olap table has a single unique key. + * @return True if the table has a single unique/agg key. False otherwise. + */ + protected boolean isSingleUniqueKey() { + int keysNum = ((OlapTable) tbl).getKeysNum(); + KeysType keysType = ((OlapTable) tbl).getKeysType(); + return col.isKey() + && keysNum == 1 + && (keysType.equals(KeysType.UNIQUE_KEYS) || keysType.equals(KeysType.AGG_KEYS)); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/statistics/OlapAnalysisTaskTest.java b/fe/fe-core/src/test/java/org/apache/doris/statistics/OlapAnalysisTaskTest.java index 9437d2d07876603..8e30519e8c4fff5 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/statistics/OlapAnalysisTaskTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/statistics/OlapAnalysisTaskTest.java @@ -218,7 +218,7 @@ public ResultRow collectBasicStat(AutoCloseConnectContext context) { @Mock public void runQuery(String sql, boolean needEncode) { Assertions.assertFalse(needEncode); - Assertions.assertEquals(" SELECT CONCAT(30001, '-', -1, '-', 'null') AS `id`, 10001 AS `catalog_id`, 20001 AS `db_id`, 30001 AS `tbl_id`, -1 AS `idx_id`, 'null' AS `col_id`, NULL AS `part_id`, ROUND(COUNT(1) * 5.0) AS `row_count`, ROUND(NDV(`${colName}`) * 5.0) as `ndv`, ROUND(SUM(CASE WHEN `${colName}` IS NULL THEN 1 ELSE 0 END) * 5.0) AS `null_count`, 'MQ==' AS `min`, 'Mg==' AS `max`, SUM(LENGTH(`${colName}`)) * 5.0 AS `data_size`, NOW() FROM `catalogName`.`${dbName}`.`${tblName}` limit 100", sql); + Assertions.assertEquals(" SELECT CONCAT(30001, '-', -1, '-', 'null') AS `id`, 10001 AS `catalog_id`, 20001 AS `db_id`, 30001 AS `tbl_id`, -1 AS `idx_id`, 'null' AS `col_id`, NULL AS `part_id`, 500 AS `row_count`, ROUND(NDV(`${colName}`) * 5.0) as `ndv`, ROUND(SUM(CASE WHEN `${colName}` IS NULL THEN 1 ELSE 0 END) * 5.0) AS `null_count`, 'MQ==' AS `min`, 'Mg==' AS `max`, SUM(LENGTH(`${colName}`)) * 5.0 AS `data_size`, NOW() FROM `catalogName`.`${dbName}`.`${tblName}` limit 100", sql); return; } }; diff --git a/fe/pom.xml b/fe/pom.xml index d00487952337cc9..18990f4fc241ef0 100644 --- a/fe/pom.xml +++ b/fe/pom.xml @@ -303,7 +303,6 @@ under the License. 1.1.0 - 3.0.0rc1 0.45.2-public 1.11.2 13.0.0 diff --git a/fs_brokers/apache_hdfs_broker/src/main/java/org/apache/doris/broker/hdfs/HDFSBrokerServiceImpl.java b/fs_brokers/apache_hdfs_broker/src/main/java/org/apache/doris/broker/hdfs/HDFSBrokerServiceImpl.java index 816462ecb340e7d..e01bf31cba8c347 100644 --- a/fs_brokers/apache_hdfs_broker/src/main/java/org/apache/doris/broker/hdfs/HDFSBrokerServiceImpl.java +++ b/fs_brokers/apache_hdfs_broker/src/main/java/org/apache/doris/broker/hdfs/HDFSBrokerServiceImpl.java @@ -69,13 +69,13 @@ private TBrokerOperationStatus generateOKStatus() { @Override public TBrokerListResponse listPath(TBrokerListPathRequest request) throws TException { - logger.info("received a list path request, request detail: " + request); TBrokerListResponse response = new TBrokerListResponse(); try { boolean fileNameOnly = false; if (request.isSetFileNameOnly()) { fileNameOnly = request.isFileNameOnly(); } + logger.info("received a list path request, request path: " + request.path + ", fileNameOnly: " + fileNameOnly); List fileStatuses = fileSystemManager.listPath(request.path, fileNameOnly, request.properties); response.setOpStatus(generateOKStatus()); @@ -92,7 +92,6 @@ public TBrokerListResponse listPath(TBrokerListPathRequest request) @Override public TBrokerListResponse listLocatedFiles(TBrokerListPathRequest request) throws TException { - logger.info("received a listLocatedFiles request, request detail: " + request); TBrokerListResponse response = new TBrokerListResponse(); try { boolean recursive = request.isIsRecursive(); @@ -100,6 +99,8 @@ public TBrokerListResponse listLocatedFiles(TBrokerListPathRequest request) if (request.isSetOnlyFiles()) { onlyFiles = request.isOnlyFiles(); } + logger.info("received a listLocatedFiles request, request path: " + + request.path + ", onlyFiles: " + onlyFiles + ", recursive: " + recursive); List fileStatuses = fileSystemManager.listLocatedFiles(request.path, onlyFiles, recursive, request.properties); response.setOpStatus(generateOKStatus()); @@ -115,7 +116,8 @@ public TBrokerListResponse listLocatedFiles(TBrokerListPathRequest request) @Override public TBrokerIsSplittableResponse isSplittable(TBrokerIsSplittableRequest request) throws TException { - logger.info("received a isSplittable request, request detail: " + request); + logger.info("received a isSplittable request, request path: " + + request.path + ", inputFormat: " +request.inputFormat); TBrokerIsSplittableResponse response = new TBrokerIsSplittableResponse(); try { boolean isSplittable = HiveUtils.isSplittable(request.path, request.inputFormat, request.properties); @@ -133,7 +135,7 @@ public TBrokerIsSplittableResponse isSplittable(TBrokerIsSplittableRequest reque @Override public TBrokerOperationStatus deletePath(TBrokerDeletePathRequest request) throws TException { - logger.info("receive a delete path request, request detail: " + request); + logger.info("receive a delete path request, request path: " + request.path); try { fileSystemManager.deletePath(request.path, request.properties); } catch (BrokerException e) { @@ -147,7 +149,8 @@ public TBrokerOperationStatus deletePath(TBrokerDeletePathRequest request) @Override public TBrokerOperationStatus renamePath(TBrokerRenamePathRequest request) throws TException { - logger.info("receive a rename path request, request detail: " + request); + logger.info("receive a rename path request, request srcPath: " + + request.srcPath + ", destPath: " + request.destPath); try { fileSystemManager.renamePath(request.srcPath, request.destPath, request.properties); } catch (BrokerException e) { @@ -161,7 +164,7 @@ public TBrokerOperationStatus renamePath(TBrokerRenamePathRequest request) @Override public TBrokerCheckPathExistResponse checkPathExist( TBrokerCheckPathExistRequest request) throws TException { - logger.info("receive a check path request, request detail: " + request); + logger.info("receive a check path request, request path: " + request.path); TBrokerCheckPathExistResponse response = new TBrokerCheckPathExistResponse(); try { boolean isPathExist = fileSystemManager.checkPathExist(request.path, request.properties); @@ -178,7 +181,8 @@ public TBrokerCheckPathExistResponse checkPathExist( @Override public TBrokerOpenReaderResponse openReader(TBrokerOpenReaderRequest request) throws TException { - logger.info("receive a open reader request, request detail: " + request); + logger.info("receive a open reader request, request client id: " + + request.clientId + ", path: " + request.path + ", startOffset: " +request.startOffset); TBrokerOpenReaderResponse response = new TBrokerOpenReaderResponse(); try { TBrokerFD fd = fileSystemManager.openReader(request.clientId, request.path, @@ -196,7 +200,7 @@ public TBrokerOpenReaderResponse openReader(TBrokerOpenReaderRequest request) @Override public TBrokerReadResponse pread(TBrokerPReadRequest request) throws TException { - logger.debug("receive a read request, request detail: " + request); + logger.info("receive a read request, request detail: " + request); Stopwatch stopwatch = BrokerPerfMonitor.startWatch(); TBrokerReadResponse response = new TBrokerReadResponse(); try { @@ -220,7 +224,7 @@ public TBrokerReadResponse pread(TBrokerPReadRequest request) @Override public TBrokerOperationStatus seek(TBrokerSeekRequest request) throws TException { - logger.debug("receive a seek request, request detail: " + request); + logger.info("receive a seek request, request detail: " + request); try { fileSystemManager.seek(request.fd, request.offset); } catch (BrokerException e) { @@ -248,7 +252,8 @@ public TBrokerOperationStatus closeReader(TBrokerCloseReaderRequest request) @Override public TBrokerOpenWriterResponse openWriter(TBrokerOpenWriterRequest request) throws TException { - logger.info("receive a open writer request, request detail: " + request); + logger.info("receive a open writer request, request client id: " + + request.clientId + ", path: " +request.path); TBrokerOpenWriterResponse response = new TBrokerOpenWriterResponse(); try { TBrokerFD fd = fileSystemManager.openWriter(request.clientId, request.path, request.properties); @@ -265,7 +270,7 @@ public TBrokerOpenWriterResponse openWriter(TBrokerOpenWriterRequest request) @Override public TBrokerOperationStatus pwrite(TBrokerPWriteRequest request) throws TException { - logger.debug("receive a pwrite request, request detail: " + request); + logger.info("receive a pwrite request, request detail: " + request); Stopwatch stopwatch = BrokerPerfMonitor.startWatch(); try { fileSystemManager.pwrite(request.fd, request.offset, request.getData()); @@ -299,7 +304,7 @@ public TBrokerOperationStatus closeWriter(TBrokerCloseWriterRequest request) @Override public TBrokerOperationStatus ping(TBrokerPingBrokerRequest request) throws TException { - logger.debug("receive a ping request, request detail: " + request); + logger.info("receive a ping request, request detail: " + request); try { fileSystemManager.ping(request.clientId); } catch (BrokerException e) { @@ -313,7 +318,7 @@ public TBrokerOperationStatus ping(TBrokerPingBrokerRequest request) @Override public TBrokerFileSizeResponse fileSize( TBrokerFileSizeRequest request) throws TException { - logger.debug("receive a file size request, request detail: " + request); + logger.info("receive a file size request, request path: " + request.path); TBrokerFileSizeResponse response = new TBrokerFileSizeResponse(); try { long fileSize = fileSystemManager.fileSize(request.path, request.properties); diff --git a/regression-test/suites/schema_change_p0/test_alter_table_column_rename.groovy b/regression-test/suites/schema_change_p0/test_alter_table_column_rename.groovy new file mode 100644 index 000000000000000..3650b93ccbb2826 --- /dev/null +++ b/regression-test/suites/schema_change_p0/test_alter_table_column_rename.groovy @@ -0,0 +1,49 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_alter_table_column_rename") { + def tbName = "alter_table_column_rename" + + sql "DROP TABLE IF EXISTS ${tbName}" + sql """ + CREATE TABLE IF NOT EXISTS ${tbName} ( + k1 INT NOT NULL, + value1 varchar(16) NOT NULL, + value2 int NOT NULL + ) + DUPLICATE KEY (k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 properties("replication_num" = "1"); + """ + + sql """ insert into ${tbName} values (1, 'a', 2) """ + sql """ select * from ${tbName} """ + + // rename column name + sql """ ALTER TABLE ${tbName} RENAME COLUMN value2 new_col """ + + List> result = sql """ show frontends """ + for (row : result) { + //println row + String jdbcUrl = "jdbc:mysql://" + row[1] + ":" + row[4] + def result1 = connect(user = 'root', password = '', jdbcUrl) { + sql """ SYNC """ + sql """ use regression_test_schema_change_p0 """ + sql """ select * from ${tbName} where new_col = 2 """ + } + } + +}