Skip to content

Commit

Permalink
Disable Bitmap or Hll type in keys or in values with incorrect agg-ty…
Browse files Browse the repository at this point in the history 
…pe (apache#3768)

Bitmap and Hll type can not be used with incorrect aggregate functions, which will cause to BE crush.
Add some logical checks in FE's ColumnDef#analyze to avoid creating tables or changing schemas incorrectly.

Keys never be bitmap or hll type
values with bitmap or hll type have to be associated with bitmap_union or hll_union
  • Loading branch information
@spaces-X @morningman-cmy
spaces-X authored and morningman-cmy committed Jun 7, 2020
1 parent d193b30 commit 0f3d9ef
Show file tree
Hide file tree
Showing 4 changed files with 103 additions and 19 deletions.
10 changes: 10 additions & 0 deletions fe/src/main/java/org/apache/doris/analysis/ColumnDef.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,16 @@ public void analyze(boolean isOlap) throws AnalysisException {

Type type = typeDef.getType();

// disable Bitmap Hll type in keys, values without aggregate function.
if (type.isBitmapType() || type.isHllType()) {
if (isKey) {
throw new AnalysisException("Key column can not set bitmap or hll type:" + name);
}
if (aggregateType == null) {
throw new AnalysisException("Bitmap and hll type have to use aggregate function" + name);
}
}

// A column is a key column if and only if isKey is true.
// aggregateType == null does not mean that this is a key column,
// because when creating a UNIQUE KEY table, aggregateType is implicit.
Expand Down
14 changes: 1 addition & 13 deletions fe/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -322,24 +322,12 @@ public void analyze(Analyzer analyzer) throws UserException {
columnDef.analyze(engineName.equals("olap"));

if (columnDef.getType().isHllType()) {
if (columnDef.isKey()) {
throw new AnalysisException("HLL can't be used as keys, " +
"please specify the aggregation type HLL_UNION");
}
hasHll = true;
}

if (columnDef.getType().isBitmapType()) {
if (columnDef.isKey()) {
throw new AnalysisException("BITMAP can't be used as keys, ");
}
}

if (columnDef.getAggregateType() == BITMAP_UNION) {
if (columnDef.isKey()) {
throw new AnalysisException("Key column can't has the BITMAP_UNION aggregation type");
}
hasBitmap = true;
hasBitmap = columnDef.getType().isBitmapType();
}

if (!columnSet.add(columnDef.getName())) {
Expand Down
15 changes: 9 additions & 6 deletions fe/src/main/java/org/apache/doris/catalog/AggregateType.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,20 +85,23 @@ public enum AggregateType {
compatibilityMap.put(MAX, EnumSet.copyOf(primitiveTypeList));

primitiveTypeList.clear();
compatibilityMap.put(REPLACE, EnumSet.allOf(PrimitiveType.class));
// all types except bitmap and hll.
EnumSet<PrimitiveType> exc_bitmap_hll = EnumSet.allOf(PrimitiveType.class);
exc_bitmap_hll.remove(PrimitiveType.BITMAP);
exc_bitmap_hll.remove(PrimitiveType.HLL);
compatibilityMap.put(REPLACE, EnumSet.copyOf(exc_bitmap_hll));

compatibilityMap.put(REPLACE_IF_NOT_NULL, EnumSet.copyOf(exc_bitmap_hll));

primitiveTypeList.clear();
compatibilityMap.put(REPLACE_IF_NOT_NULL, EnumSet.allOf(PrimitiveType.class));

primitiveTypeList.clear();
primitiveTypeList.add(PrimitiveType.HLL);
compatibilityMap.put(HLL_UNION, EnumSet.copyOf(primitiveTypeList));

primitiveTypeList.clear();
primitiveTypeList.add(PrimitiveType.BITMAP);
compatibilityMap.put(BITMAP_UNION, EnumSet.copyOf(primitiveTypeList));
compatibilityMap.put(NONE, EnumSet.allOf(PrimitiveType.class));

compatibilityMap.put(NONE, EnumSet.copyOf(exc_bitmap_hll));
}
private final String sqlName;

Expand Down
83 changes: 83 additions & 0 deletions fe/src/test/java/org/apache/doris/analysis/CreateTableStmtTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
package org.apache.doris.analysis;

import mockit.Expectations;
import org.apache.doris.catalog.AggregateType;
import org.apache.doris.catalog.KeysType;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.ScalarType;
Expand All @@ -31,7 +32,9 @@

import org.junit.Assert;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand All @@ -55,6 +58,8 @@ public class CreateTableStmtTest {
private PaloAuth auth;
@Mocked
private ConnectContext ctx;
@Rule
public ExpectedException expectedEx = ExpectedException.none();

// set default db is 'db1'
// table name is table1
Expand Down Expand Up @@ -163,4 +168,82 @@ public void testDupCol() throws UserException, AnalysisException {
new RandomDistributionDesc(10), null, null, "");
stmt.analyze(analyzer);
}


@Test
public void testBmpHllKey() throws Exception {

ColumnDef bitmap = new ColumnDef("col3", new TypeDef(ScalarType.createType(PrimitiveType.BITMAP)));
cols.add(bitmap);
colsName.add("col3");

CreateTableStmt stmt = new CreateTableStmt(false, false, tblNameNoDb, cols, "olap",
new KeysDesc(KeysType.AGG_KEYS, colsName), null,
new RandomDistributionDesc(10), null, null, "");
expectedEx.expect(AnalysisException.class);
expectedEx.expectMessage("Key column can not set bitmap or hll type:col3");
stmt.analyze(analyzer);

cols.remove(bitmap);

ColumnDef hll = new ColumnDef("col3", new TypeDef(ScalarType.createType(PrimitiveType.HLL)));
cols.add(hll);
stmt = new CreateTableStmt(false, false, tblNameNoDb, cols, "olap",
new KeysDesc(KeysType.AGG_KEYS, colsName), null,
new RandomDistributionDesc(10), null, null, "");
expectedEx.expect(AnalysisException.class);
expectedEx.expectMessage("Key column can not set bitmap or hll type:col3");
stmt.analyze(analyzer);
}

@Test
public void testBmpHllNoAggTab() throws Exception {
ColumnDef bitmap = new ColumnDef("col3", new TypeDef(ScalarType.createType(PrimitiveType.BITMAP)));
cols.add(bitmap);
CreateTableStmt stmt = new CreateTableStmt(false, false, tblNameNoDb, cols, "olap",
new KeysDesc(KeysType.DUP_KEYS, colsName), null,
new RandomDistributionDesc(10), null, null, "");
expectedEx.expect(AnalysisException.class);
expectedEx.expectMessage("Aggregate type `col3` bitmap NONE NOT NULL COMMENT \"\" is not compatible with primitive type bitmap");
stmt.analyze(analyzer);

cols.remove(bitmap);
ColumnDef hll = new ColumnDef("col3", new TypeDef(ScalarType.createType(PrimitiveType.HLL)));
cols.add(hll);
stmt = new CreateTableStmt(false, false, tblNameNoDb, cols, "olap",
new KeysDesc(KeysType.DUP_KEYS, colsName), null,
new RandomDistributionDesc(10), null, null, "");
expectedEx.expect(AnalysisException.class);
expectedEx.expectMessage("Aggregate type `col3` hll NONE NOT NULL COMMENT \"\" is not compatible with primitive type hll");
stmt.analyze(analyzer);
}

@Test
public void testBmpHllIncAgg() throws Exception {
ColumnDef bitmap = new ColumnDef("col3", new TypeDef(ScalarType.createType(PrimitiveType.BITMAP)));
bitmap.setAggregateType(AggregateType.SUM);

cols.add(bitmap);
CreateTableStmt stmt = new CreateTableStmt(false, false, tblNameNoDb, cols, "olap",
new KeysDesc(KeysType.AGG_KEYS, colsName), null,
new RandomDistributionDesc(10), null, null, "");

expectedEx.expect(AnalysisException.class);
expectedEx.expectMessage(String.format("Aggregate type %s is not compatible with primitive type %s",
bitmap.toString(), bitmap.getTypeDef().getType().toSql()));
stmt.analyze(analyzer);

cols.remove(bitmap);
ColumnDef hll = new ColumnDef("col3", new TypeDef(ScalarType.createType(PrimitiveType.HLL)));
hll.setAggregateType(AggregateType.SUM);
cols.add(hll);
stmt = new CreateTableStmt(false, false, tblNameNoDb, cols, "olap",
new KeysDesc(KeysType.AGG_KEYS, colsName), null,
new RandomDistributionDesc(10), null, null, "");

expectedEx.expect(AnalysisException.class);
expectedEx.expectMessage(String.format("Aggregate type %s is not compatible with primitive type %s",
hll.toString(), hll.getTypeDef().getType().toSql()));
stmt.analyze(analyzer);
}
}

0 comments on commit 0f3d9ef

Please sign in to comment.