Skip to content

Commit

Permalink
[BugFix] Fix count(*) error during adding smaller type column (StarRo…
Browse files Browse the repository at this point in the history 
…cks#33243)

Signed-off-by: zihe.liu <ziheliu1024@gmail.com>
  • Loading branch information
@ZiheLiu
ZiheLiu authored Oct 20, 2023
1 parent 76f3831 commit d86f3ba
Show file tree
Hide file tree
Showing 7 changed files with 320 additions and 14 deletions.
11 changes: 11 additions & 0 deletions fe/fe-core/src/main/java/com/starrocks/sql/optimizer/Utils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@
import java.util.BitSet;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
Expand Down Expand Up @@ -424,6 +425,16 @@ public static long convertBitSetToLong(BitSet bitSet, int length) {
return gid;
}

public static ColumnRefOperator findSmallestColumnRefFromTable(Map<ColumnRefOperator, Column> colRefToColumnMetaMap,
Table table) {
Set<Column> baseSchema = new HashSet<>(table.getBaseSchema());
List<ColumnRefOperator> visibleColumnRefs = colRefToColumnMetaMap.entrySet().stream()
.filter(e -> baseSchema.contains(e.getValue()))
.map(Map.Entry::getKey)
.collect(Collectors.toList());
return findSmallestColumnRef(visibleColumnRefs);
}

public static ColumnRefOperator findSmallestColumnRef(List<ColumnRefOperator> columnRefOperatorList) {
if (CollectionUtils.isEmpty(columnRefOperatorList)) {
return null;
Expand Down
7 changes: 3 additions & 4 deletions ...re/src/main/java/com/starrocks/sql/optimizer/rule/transformation/PruneScanColumnRule.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@
import com.starrocks.sql.optimizer.operator.scalar.ColumnRefOperator;
import com.starrocks.sql.optimizer.rule.RuleType;

import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
Expand Down Expand Up @@ -62,9 +61,9 @@ public List<OptExpression> transform(OptExpression input, OptimizerContext conte
.collect(Collectors.toSet());
outputColumns.addAll(Utils.extractColumnRef(scanOperator.getPredicate()));
boolean canUseAnyColumn = false;
if (outputColumns.size() == 0) {
outputColumns.add(Utils.findSmallestColumnRef(
new ArrayList<>(scanOperator.getColRefToColumnMetaMap().keySet())));
if (outputColumns.isEmpty()) {
outputColumns.add(
Utils.findSmallestColumnRefFromTable(scanOperator.getColRefToColumnMetaMap(), scanOperator.getTable()));
canUseAnyColumn = true;
}

Expand Down
7 changes: 3 additions & 4 deletions .../java/com/starrocks/sql/optimizer/rule/transformation/materialization/MVColumnPruner.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,6 @@
import com.starrocks.sql.optimizer.operator.scalar.CallOperator;
import com.starrocks.sql.optimizer.operator.scalar.ColumnRefOperator;

import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;
Expand Down Expand Up @@ -64,9 +63,9 @@ public OptExpression visitLogicalTableScan(OptExpression optExpression, Void con
scanOperator.getColRefToColumnMetaMap().keySet().stream().filter(requiredOutputColumns::contains)
.collect(Collectors.toSet());
outputColumns.addAll(Utils.extractColumnRef(scanOperator.getPredicate()));
if (outputColumns.size() == 0) {
outputColumns.add(Utils.findSmallestColumnRef(
new ArrayList<>(scanOperator.getColRefToColumnMetaMap().keySet())));
if (outputColumns.isEmpty()) {
outputColumns.add(
Utils.findSmallestColumnRefFromTable(scanOperator.getColRefToColumnMetaMap(), scanOperator.getTable()));
}

ImmutableMap.Builder<ColumnRefOperator, Column> columnRefColumnMapBuilder = new ImmutableMap.Builder<>();
Expand Down
192 changes: 186 additions & 6 deletions fe/fe-core/src/test/java/com/starrocks/sql/optimizer/OptimizerTaskTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -145,16 +145,26 @@ public void testTaskScheduler(@Mocked OlapTable olapTable1,
scan1ColumnMap.put(column1, new Column("t1", ScalarType.INT, true));
scan1ColumnMap.put(column2, new Column("t2", ScalarType.INT, true));

Map<ColumnRefOperator, Column> scan2ColumnMap = Maps.newHashMap();
scan2ColumnMap.put(column3, new Column("t3", ScalarType.INT, true));
scan2ColumnMap.put(column4, new Column("t4", ScalarType.INT, true));

OptExpression logicOperatorTree = OptExpression.create(new LogicalJoinOperator(),
OptExpression.create(new LogicalOlapScanOperator(olapTable1,
scan1ColumnMap, Maps.newHashMap(), null, -1, null)),
OptExpression.create(new LogicalOlapScanOperator(olapTable2,
scan1ColumnMap, Maps.newHashMap(), null, -1, null)));

new Expectations() {
{
olapTable1.getBaseSchema();
result = new ArrayList<>(scan1ColumnMap.values());
minTimes = 0;
}

{
olapTable2.getBaseSchema();
result = new ArrayList<>(scan1ColumnMap.values());
minTimes = 0;
}
};

Optimizer optimizer = new Optimizer();
optimizer.optimize(ctx, logicOperatorTree, new PhysicalPropertySet(), new ColumnRefSet(),
columnRefFactory);
Expand Down Expand Up @@ -223,6 +233,26 @@ public void testTwoJoin(@Mocked OlapTable olapTable1,
new OptExpression(new LogicalOlapScanOperator(olapTable3,
scanColumnMap, Maps.newHashMap(), null, -1, null)));

new Expectations() {
{
olapTable1.getBaseSchema();
result = new ArrayList<>(scanColumnMap.values());
minTimes = 0;
}

{
olapTable2.getBaseSchema();
result = new ArrayList<>(scanColumnMap.values());
minTimes = 0;
}

{
olapTable3.getBaseSchema();
result = new ArrayList<>(scanColumnMap.values());
minTimes = 0;
}
};

Optimizer optimizer = new Optimizer();
optimizer.optimize(ctx, topJoin, new PhysicalPropertySet(), new ColumnRefSet(),
columnRefFactory);
Expand Down Expand Up @@ -301,6 +331,32 @@ public void testThreeJoin(@Mocked OlapTable olapTable1,
Map<ColumnRefOperator, Column> scan4ColumnMap = Maps.newHashMap();
scan4ColumnMap.put(column4, new Column("t4", ScalarType.INT, true));

new Expectations() {
{
olapTable1.getBaseSchema();
result = new ArrayList<>(scan1ColumnMap.values());
minTimes = 0;
}

{
olapTable2.getBaseSchema();
result = new ArrayList<>(scan2ColumnMap.values());
minTimes = 0;
}

{
olapTable3.getBaseSchema();
result = new ArrayList<>(scan3ColumnMap.values());
minTimes = 0;
}

{
olapTable4.getBaseSchema();
result = new ArrayList<>(scan4ColumnMap.values());
minTimes = 0;
}
};

OptExpression bottomJoin = OptExpression.create(new LogicalJoinOperator(),
OptExpression.create(new LogicalOlapScanOperator(olapTable1,
scan1ColumnMap, Maps.newHashMap(), null, -1, null)),
Expand Down Expand Up @@ -383,6 +439,34 @@ public void testFourJoin(@Mocked OlapTable olapTable1,
new LogicalOlapScanOperator(olapTable4, scanColumnMap,
Maps.newHashMap(), null, -1, null)));

new Expectations() {
{
olapTable1.getBaseSchema();
result = new ArrayList<>(scanColumnMap.values());
minTimes = 0;
}
{
olapTable2.getBaseSchema();
result = new ArrayList<>(scanColumnMap.values());
minTimes = 0;
}
{
olapTable3.getBaseSchema();
result = new ArrayList<>(scanColumnMap.values());
minTimes = 0;
}
{
olapTable4.getBaseSchema();
result = new ArrayList<>(scanColumnMap.values());
minTimes = 0;
}
{
olapTable5.getBaseSchema();
result = new ArrayList<>(scanColumnMap.values());
minTimes = 0;
}
};

Optimizer optimizer = new Optimizer();
optimizer.optimize(ctx, topJoin, new PhysicalPropertySet(), new ColumnRefSet(),
columnRefFactory);
Expand Down Expand Up @@ -480,6 +564,56 @@ public void testSevenJoin(@Mocked OlapTable olapTable1,
new LogicalOlapScanOperator(olapTable4, scanColumnMap,
Maps.newHashMap(), null, -1, null)));

new Expectations() {
{
olapTable1.getBaseSchema();
result = new ArrayList<>(scanColumnMap.values());
minTimes = 0;
}

{
olapTable2.getBaseSchema();
result = new ArrayList<>(scanColumnMap.values());
minTimes = 0;
}

{
olapTable3.getBaseSchema();
result = new ArrayList<>(scanColumnMap.values());
minTimes = 0;
}

{
olapTable4.getBaseSchema();
result = new ArrayList<>(scanColumnMap.values());
minTimes = 0;
}

{
olapTable5.getBaseSchema();
result = new ArrayList<>(scanColumnMap.values());
minTimes = 0;
}

{
olapTable6.getBaseSchema();
result = new ArrayList<>(scanColumnMap.values());
minTimes = 0;
}

{
olapTable7.getBaseSchema();
result = new ArrayList<>(scanColumnMap.values());
minTimes = 0;
}

{
olapTable8.getBaseSchema();
result = new ArrayList<>(scanColumnMap.values());
minTimes = 0;
}
};

Optimizer optimizer = new Optimizer();
optimizer.optimize(ctx, topJoin, new PhysicalPropertySet(), new ColumnRefSet(),
columnRefFactory);
Expand Down Expand Up @@ -600,6 +734,14 @@ public void testTopDownRewrite(@Mocked OlapTable olapTable1) {
OptExpression.create(new LogicalOlapScanOperator(olapTable1,
scanColumnMap, Maps.newHashMap(), null, -1, null)));

new Expectations() {
{
olapTable1.getBaseSchema();
result = new ArrayList<>(scanColumnMap.values());
minTimes = 0;
}
};

Optimizer optimizer = new Optimizer();
OptExpression physicalTree = optimizer.optimize(ctx, expression, new PhysicalPropertySet(), new ColumnRefSet(),
columnRefFactory);
Expand Down Expand Up @@ -671,6 +813,14 @@ public void testPruneOlapScanColumnsRuleWithConstant(@Mocked OlapTable olapTable
new LogicalOlapScanOperator(olapTable1, scanColumnMap, Maps.newHashMap(), null,
-1, null)));

new Expectations() {
{
olapTable1.getBaseSchema();
result = new ArrayList<>(scanColumnMap.values());
minTimes = 0;
}
};

Optimizer optimizer = new Optimizer();
OptExpression physicalTree = optimizer.optimize(ctx, expression, new PhysicalPropertySet(),
new ColumnRefSet(outputColumns1), columnRefFactory);
Expand Down Expand Up @@ -814,6 +964,14 @@ public void testPruneCountStarRule(@Mocked OlapTable olapTable1) {
scanColumnMap.put(column3, new Column("t3", ScalarType.INT, true));
scanColumnMap.put(column4, new Column("t4", ScalarType.INT, true));

new Expectations() {
{
olapTable1.getBaseSchema();
result = new ArrayList<>(scanColumnMap.values());
minTimes = 0;
}
};

Map<ColumnRefOperator, CallOperator> map = Maps.newHashMap();
map.put(column5, call);
LogicalAggregationOperator aggregationOperator =
Expand Down Expand Up @@ -873,6 +1031,14 @@ public void testPruneAggregateConstantRule(@Mocked OlapTable olapTable1) {

ColumnRefSet outputColumns = new ColumnRefSet(column4.getId());

new Expectations() {
{
olapTable1.getBaseSchema();
result = new ArrayList<>(scanColumnMap.values());
minTimes = 0;
}
};

Optimizer optimizer = new Optimizer();
try {
optimizer.optimize(ctx, expression, new PhysicalPropertySet(), outputColumns, columnRefFactory);
Expand Down Expand Up @@ -922,6 +1088,14 @@ public void testMergeAggregateWithLimitRule(@Mocked OlapTable olapTable1) {

ColumnRefSet outputColumns = new ColumnRefSet(column4.getId());

new Expectations() {
{
olapTable1.getBaseSchema();
result = new ArrayList<>(scanColumnMap.values());
minTimes = 0;
}
};

Optimizer optimizer = new Optimizer();
try {
optimizer.optimize(ctx, expression, new PhysicalPropertySet(), outputColumns, columnRefFactory);
Expand Down Expand Up @@ -1076,8 +1250,6 @@ public void testSplitAggregateRuleNoGroupBy(@Mocked OlapTable olapTable1) {
}
};

List<ColumnRefOperator> scanColumns = Lists.newArrayList(column1, column2);

Map<ColumnRefOperator, Column> scanColumnMap = Maps.newHashMap();
scanColumnMap.put(column1, new Column("t1", ScalarType.INT, true));
scanColumnMap.put(column2, new Column("t2", ScalarType.INT, true));
Expand All @@ -1093,6 +1265,14 @@ public void testSplitAggregateRuleNoGroupBy(@Mocked OlapTable olapTable1) {

ColumnRefSet outputColumns = new ColumnRefSet(Lists.newArrayList(column3));

new Expectations() {
{
olapTable1.getBaseSchema();
result = new ArrayList<>(scanColumnMap.values());
minTimes = 0;
}
};

Optimizer optimizer = new Optimizer();
try {
optimizer.optimize(ctx, expression, new PhysicalPropertySet(), outputColumns, columnRefFactory);
Expand Down
18 changes: 18 additions & 0 deletions test/lib/sr_sql_lib.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -981,6 +981,24 @@ def wait_alter_table_finish(self, alter_type="COLUMN"):
sleep_time += 0.5
tools.assert_equal("FINISHED", status, "wait alter table finish error")

def wait_alter_table_not_pending(self, alter_type="COLUMN"):
"""
wait until the status of the latest alter table job becomes from PNEDING to others
"""
status = ""
while True:
res = self.execute_sql(
"SHOW ALTER TABLE %s ORDER BY CreateTime DESC LIMIT 1" % alter_type,
True,
)
if (not res["status"]) or len(res["result"]) <= 0:
return None

status = res["result"][0][9]
if status != "PENDING":
break
time.sleep(0.5)

def wait_optimize_table_finish(self, alter_type="OPTIMIZE"):
"""
wait alter table job finish and return status
Expand Down
Loading

0 comments on commit d86f3ba

Please sign in to comment.