Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -507,8 +507,16 @@ protected void testGeneratedQueries(boolean withMultiValues, boolean useMultista
public void testQueryExceptions()
throws Exception {
testQueryException("POTATO", QueryException.SQL_PARSING_ERROR_CODE);
testQueryException("SELECT COUNT(*) FROM potato", QueryException.TABLE_DOES_NOT_EXIST_ERROR_CODE);
testQueryException("SELECT POTATO(ArrTime) FROM mytable", QueryException.QUERY_EXECUTION_ERROR_CODE);

// Ideally, we should attempt to unify the error codes returned by the two query engines if possible
testQueryException("SELECT COUNT(*) FROM potato",
useMultiStageQueryEngine()
? QueryException.QUERY_PLANNING_ERROR_CODE : QueryException.TABLE_DOES_NOT_EXIST_ERROR_CODE);

testQueryException("SELECT POTATO(ArrTime) FROM mytable",
useMultiStageQueryEngine()
? QueryException.QUERY_PLANNING_ERROR_CODE : QueryException.QUERY_EXECUTION_ERROR_CODE);

testQueryException("SELECT COUNT(*) FROM mytable where ArrTime = 'potato'",
QueryException.QUERY_EXECUTION_ERROR_CODE);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -153,14 +153,15 @@ private void testDictionaryBasedFunctions(String column)
public void testHardcodedQueries(boolean useMultiStageQueryEngine)
throws Exception {
setUseMultiStageQueryEngine(useMultiStageQueryEngine);
notSupportedInV2();
super.testHardcodedQueries();
}

@Test(dataProvider = "useBothQueryEngines")
public void testQueriesFromQueryFile(boolean useMultiStageQueryEngine)
throws Exception {
setUseMultiStageQueryEngine(useMultiStageQueryEngine);
// Some of the hardcoded queries in the query file need to be adapted for v2 (for instance, using the arrayToMV
// with multi-value columns in filters / aggregations)
notSupportedInV2();
super.testQueriesFromQueryFile();
}
Expand All @@ -169,15 +170,13 @@ public void testQueriesFromQueryFile(boolean useMultiStageQueryEngine)
public void testGeneratedQueries(boolean useMultiStageQueryEngine)
throws Exception {
setUseMultiStageQueryEngine(useMultiStageQueryEngine);
notSupportedInV2();
testGeneratedQueries(true, useMultiStageQueryEngine);
}

@Test(dataProvider = "useBothQueryEngines")
public void testQueryExceptions(boolean useMultiStageQueryEngine)
throws Exception {
setUseMultiStageQueryEngine(useMultiStageQueryEngine);
notSupportedInV2();
super.testQueryExceptions();
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -278,6 +278,7 @@ public void testBrokerDebugRoutingTableSQL(boolean useMultiStageQueryEngine)
public void testQueryTracing(boolean useMultiStageQueryEngine)
throws Exception {
setUseMultiStageQueryEngine(useMultiStageQueryEngine);
// Tracing is a v1 only concept and the v2 query engine has separate multi-stage stats that are enabled by default
notSupportedInV2();
JsonNode jsonNode = postQuery("SET trace = true; SELECT COUNT(*) FROM " + getTableName());
Assert.assertEquals(jsonNode.get("resultTable").get("rows").get(0).get(0).asLong(), getCountStarResult());
Expand All @@ -292,6 +293,7 @@ public void testQueryTracing(boolean useMultiStageQueryEngine)
public void testQueryTracingWithLiteral(boolean useMultiStageQueryEngine)
throws Exception {
setUseMultiStageQueryEngine(useMultiStageQueryEngine);
// Tracing is a v1 only concept and the v2 query engine has separate multi-stage stats that are enabled by default
notSupportedInV2();
JsonNode jsonNode =
postQuery("SET trace = true; SELECT 1, \'test\', ArrDelay FROM " + getTableName() + " LIMIT 10");
Expand Down Expand Up @@ -330,14 +332,15 @@ public void testDropResults(boolean useMultiStageQueryEngine)
public void testHardcodedQueries(boolean useMultiStageQueryEngine)
throws Exception {
setUseMultiStageQueryEngine(useMultiStageQueryEngine);
notSupportedInV2();
super.testHardcodedQueries();
}

@Test(dataProvider = "useBothQueryEngines")
public void testQueriesFromQueryFile(boolean useMultiStageQueryEngine)
throws Exception {
setUseMultiStageQueryEngine(useMultiStageQueryEngine);
// Some of the hardcoded queries in the query file need to be adapted for v2 (for instance, using the arrayToMV
// with multi-value columns in filters / aggregations)
notSupportedInV2();
super.testQueriesFromQueryFile();
}
Expand All @@ -346,15 +349,13 @@ public void testQueriesFromQueryFile(boolean useMultiStageQueryEngine)
public void testGeneratedQueries(boolean useMultiStageQueryEngine)
throws Exception {
setUseMultiStageQueryEngine(useMultiStageQueryEngine);
notSupportedInV2();
super.testGeneratedQueries();
super.testGeneratedQueries(true, useMultiStageQueryEngine);
}

@Test(dataProvider = "useBothQueryEngines")
public void testQueryExceptions(boolean useMultiStageQueryEngine)
throws Exception {
setUseMultiStageQueryEngine(useMultiStageQueryEngine);
notSupportedInV2();
super.testQueryExceptions();
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1491,7 +1491,6 @@ public void testStarTreeTriggering()
public void testDefaultColumns(boolean useMultiStageQueryEngine)
throws Exception {
setUseMultiStageQueryEngine(useMultiStageQueryEngine);
notSupportedInV2();
long numTotalDocs = getCountStarResult();

reloadWithExtraColumns();
Expand All @@ -1500,7 +1499,11 @@ public void testDefaultColumns(boolean useMultiStageQueryEngine)
assertEquals(queryResponse.get("resultTable").get("dataSchema").get("columnNames").size(), 103);

testNewAddedColumns();
testExpressionOverride();

// The multi-stage query engine doesn't support expression overrides currently
if (!useMultiStageQueryEngine()) {
testExpressionOverride();
}

reloadWithMissingColumns();
queryResponse = postQuery(SELECT_STAR_QUERY);
Expand Down Expand Up @@ -1692,66 +1695,126 @@ private void testNewAddedColumns()
String h2Query = "SELECT COUNT(*) FROM mytable";
String pinotQuery = "SELECT COUNT(*) FROM mytable WHERE NewAddedIntMetric = 1";
testQuery(pinotQuery, h2Query);

pinotQuery = "SELECT COUNT(*) FROM mytable WHERE NewAddedLongMetric = 1";
testQuery(pinotQuery, h2Query);

pinotQuery = "SELECT COUNT(*) FROM mytable WHERE NewAddedFloatMetric = 0";
testQuery(pinotQuery, h2Query);

pinotQuery = "SELECT COUNT(*) FROM mytable WHERE NewAddedDoubleMetric = 0";
testQuery(pinotQuery, h2Query);

pinotQuery = "SELECT COUNT(*) FROM mytable WHERE NewAddedBigDecimalMetric = 0";
testQuery(pinotQuery, h2Query);
pinotQuery = "SELECT COUNT(*) FROM mytable WHERE NewAddedBytesMetric = ''";

pinotQuery = "SELECT COUNT(*) FROM mytable WHERE NewAddedBytesMetric = "
+ (useMultiStageQueryEngine() ? "X''" : "''");
testQuery(pinotQuery, h2Query);
pinotQuery = "SELECT COUNT(*) FROM mytable WHERE NewAddedMVIntDimension < 0";

pinotQuery = "SELECT COUNT(*) FROM mytable WHERE "
+ (useMultiStageQueryEngine() ? "arrayToMV(NewAddedMVIntDimension)" : "NewAddedMVIntDimension") + " < 0";
testQuery(pinotQuery, h2Query);
pinotQuery = "SELECT COUNT(*) FROM mytable WHERE NewAddedMVLongDimension < 0";

pinotQuery = "SELECT COUNT(*) FROM mytable WHERE "
+ (useMultiStageQueryEngine() ? "arrayToMV(NewAddedMVLongDimension)" : "NewAddedMVLongDimension") + " < 0";
testQuery(pinotQuery, h2Query);
pinotQuery = "SELECT COUNT(*) FROM mytable WHERE NewAddedMVFloatDimension < 0.0";

pinotQuery = "SELECT COUNT(*) FROM mytable WHERE "
+ (useMultiStageQueryEngine() ? "arrayToMV(NewAddedMVFloatDimension)" : "NewAddedMVFloatDimension") + " < 0.0";
testQuery(pinotQuery, h2Query);
pinotQuery = "SELECT COUNT(*) FROM mytable WHERE NewAddedMVDoubleDimension < 0.0";

pinotQuery = "SELECT COUNT(*) FROM mytable WHERE "
+ (useMultiStageQueryEngine() ? "arrayToMV(NewAddedMVDoubleDimension)" : "NewAddedMVDoubleDimension")
+ " < 0.0";
testQuery(pinotQuery, h2Query);
pinotQuery = "SELECT COUNT(*) FROM mytable WHERE NewAddedMVBooleanDimension = false";

pinotQuery = "SELECT COUNT(*) FROM mytable WHERE "
+ (useMultiStageQueryEngine() ? "arrayToMV(NewAddedMVBooleanDimension)" : "NewAddedMVBooleanDimension")
+ " = false";
testQuery(pinotQuery, h2Query);
pinotQuery = "SELECT COUNT(*) FROM mytable WHERE NewAddedMVTimestampDimension = 0";

pinotQuery = "SELECT COUNT(*) FROM mytable WHERE "
+ (useMultiStageQueryEngine()
? "arrayToMV(CAST(NewAddedMVTimestampDimension AS BIGINT ARRAY))"
: "NewAddedMVTimestampDimension")
+ " = 0";
testQuery(pinotQuery, h2Query);
pinotQuery = "SELECT COUNT(*) FROM mytable WHERE NewAddedMVStringDimension = 'null'";

pinotQuery = "SELECT COUNT(*) FROM mytable WHERE "
+ (useMultiStageQueryEngine() ? "arrayToMV(NewAddedMVStringDimension)" : "NewAddedMVStringDimension")
+ " = 'null'";
testQuery(pinotQuery, h2Query);

pinotQuery = "SELECT COUNT(*) FROM mytable WHERE NewAddedSVJSONDimension = 'null'";
testQuery(pinotQuery, h2Query);
pinotQuery = "SELECT COUNT(*) FROM mytable WHERE NewAddedSVBytesDimension = ''";

pinotQuery = "SELECT COUNT(*) FROM mytable WHERE NewAddedSVBytesDimension = "
+ (useMultiStageQueryEngine() ? "X''" : "''");
testQuery(pinotQuery, h2Query);

pinotQuery = "SELECT COUNT(*) FROM mytable WHERE NewAddedDerivedHoursSinceEpoch = 392232";
h2Query = "SELECT COUNT(*) FROM mytable WHERE DaysSinceEpoch = 16343";
testQuery(pinotQuery, h2Query);
pinotQuery = "SELECT COUNT(*) FROM mytable WHERE NewAddedDerivedTimestamp = 1411862400000";

pinotQuery = "SELECT COUNT(*) FROM mytable WHERE "
+ (useMultiStageQueryEngine() ? "CAST(NewAddedDerivedTimestamp AS BIGINT)" : "NewAddedDerivedTimestamp")
+ " = 1411862400000";
h2Query = "SELECT COUNT(*) FROM mytable WHERE DaysSinceEpoch = 16341";
testQuery(pinotQuery, h2Query);

pinotQuery = "SELECT COUNT(*) FROM mytable WHERE NewAddedDerivedSVBooleanDimension = true";
h2Query = "SELECT COUNT(*) FROM mytable WHERE ActualElapsedTime > 0";
testQuery(pinotQuery, h2Query);
pinotQuery = "SELECT COUNT(*) FROM mytable WHERE NewAddedDerivedMVStringDimension = 'CA'";

pinotQuery = "SELECT COUNT(*) FROM mytable WHERE "
+ (useMultiStageQueryEngine()
? "arrayToMV(NewAddedDerivedMVStringDimension)"
: "NewAddedDerivedMVStringDimension")
+ " = 'CA'";
h2Query = "SELECT COUNT(*) FROM mytable WHERE DestState = 'CA'";
testQuery(pinotQuery, h2Query);

pinotQuery = "SELECT COUNT(*) FROM mytable WHERE NewAddedRawDerivedStringDimension = 'Washington, DC'";
h2Query = "SELECT COUNT(*) FROM mytable WHERE DestCityName = 'CD ,notgnihsaW'";
testQuery(pinotQuery, h2Query);
pinotQuery = "SELECT COUNT(*) FROM mytable WHERE NewAddedRawDerivedMVIntDimension = 332";

pinotQuery = "SELECT COUNT(*) FROM mytable WHERE "
+ (useMultiStageQueryEngine()
? "arrayToMV(NewAddedRawDerivedMVIntDimension)"
: "NewAddedRawDerivedMVIntDimension")
+ " = 332";
h2Query = "SELECT COUNT(*) FROM mytable WHERE ActualElapsedTime = 332";
testQuery(pinotQuery, h2Query);
pinotQuery = "SELECT COUNT(*) FROM mytable WHERE NewAddedDerivedMVDoubleDimension = 110.0";

pinotQuery = "SELECT COUNT(*) FROM mytable WHERE "
+ (useMultiStageQueryEngine()
? "arrayToMV(NewAddedDerivedMVDoubleDimension)"
: "NewAddedDerivedMVDoubleDimension")
+ " = 110.0";
h2Query = "SELECT COUNT(*) FROM mytable WHERE ArrDelayMinutes = 110.0";
testQuery(pinotQuery, h2Query);

pinotQuery = "SELECT COUNT(*) FROM mytable WHERE DivAirportSeqIDs > 1100000";
pinotQuery = "SELECT COUNT(*) FROM mytable WHERE "
+ (useMultiStageQueryEngine() ? "arrayToMV(DivAirportSeqIDs)" : "DivAirportSeqIDs")
+ " > 1100000";
JsonNode response = postQuery(pinotQuery);
JsonNode rows = response.get("resultTable").get("rows");
long count = rows.get(0).get(0).asLong();
pinotQuery = "SELECT COUNT(*) FROM mytable WHERE NewAddedDerivedDivAirportSeqIDs > 1100000";
pinotQuery = "SELECT COUNT(*) FROM mytable WHERE "
+ (useMultiStageQueryEngine()
? "arrayToMV(NewAddedDerivedDivAirportSeqIDs)"
: "NewAddedDerivedDivAirportSeqIDs")
+ " > 1100000";
response = postQuery(pinotQuery);
rows = response.get("resultTable").get("rows");
assertEquals(rows.get(0).get(0).asLong(), count);
pinotQuery = "SELECT COUNT(*) FROM mytable WHERE CAST(NewAddedDerivedDivAirportSeqIDsString AS INT) > 1100000";

pinotQuery = "SELECT COUNT(*) FROM mytable WHERE "
+ (useMultiStageQueryEngine()
? "arrayToMV(NewAddedDerivedDivAirportSeqIDsString)"
: "CAST(NewAddedDerivedDivAirportSeqIDsString AS INT)")
+ " > 1100000";
response = postQuery(pinotQuery);
rows = response.get("resultTable").get("rows");
assertEquals(rows.get(0).get(0).asLong(), count);
Expand All @@ -1760,44 +1823,66 @@ private void testNewAddedColumns()
pinotQuery = "SELECT SUM(NewAddedIntMetric) FROM mytable WHERE DaysSinceEpoch <= 16312";
h2Query = "SELECT COUNT(*) FROM mytable WHERE DaysSinceEpoch <= 16312";
testQuery(pinotQuery, h2Query);

pinotQuery = "SELECT SUM(NewAddedIntMetric) FROM mytable WHERE DaysSinceEpoch > 16312";
h2Query = "SELECT COUNT(*) FROM mytable WHERE DaysSinceEpoch > 16312";
testQuery(pinotQuery, h2Query);

pinotQuery = "SELECT SUM(NewAddedLongMetric) FROM mytable WHERE DaysSinceEpoch <= 16312";
h2Query = "SELECT COUNT(*) FROM mytable WHERE DaysSinceEpoch <= 16312";
testQuery(pinotQuery, h2Query);

pinotQuery = "SELECT SUM(NewAddedLongMetric) FROM mytable WHERE DaysSinceEpoch > 16312";
h2Query = "SELECT COUNT(*) FROM mytable WHERE DaysSinceEpoch > 16312";
testQuery(pinotQuery, h2Query);

// Test other query forms with new added columns
pinotQuery =
"SELECT NewAddedMVStringDimension, SUM(NewAddedFloatMetric) FROM mytable GROUP BY NewAddedMVStringDimension";
"SELECT "
+ (useMultiStageQueryEngine() ? "arrayToMV(NewAddedMVStringDimension)" : "NewAddedMVStringDimension")
+ ", SUM(NewAddedFloatMetric) FROM mytable GROUP BY "
+ (useMultiStageQueryEngine() ? "arrayToMV(NewAddedMVStringDimension)" : "NewAddedMVStringDimension");
response = postQuery(pinotQuery);
rows = response.get("resultTable").get("rows");
assertEquals(rows.size(), 1);
JsonNode row = rows.get(0);
assertEquals(row.size(), 2);
assertEquals(row.get(0).asText(), "null");
assertEquals(row.get(1).asDouble(), 0.0);

// The multi-stage query engine doesn't support BIG_DECIMAL data type currently.
if (!useMultiStageQueryEngine()) {
pinotQuery =
"SELECT NewAddedSVBytesDimension, SUM(NewAddedBigDecimalMetric) FROM mytable "
+ "GROUP BY NewAddedSVBytesDimension";
response = postQuery(pinotQuery);
rows = response.get("resultTable").get("rows");
assertEquals(rows.size(), 1);
row = rows.get(0);
assertEquals(row.size(), 2);
assertEquals(row.get(0).asText(), "");
assertEquals(row.get(1).asDouble(), 0.0);
}

pinotQuery =
"SELECT NewAddedSVBytesDimension, SUM(NewAddedBigDecimalMetric) FROM mytable GROUP BY NewAddedSVBytesDimension";
response = postQuery(pinotQuery);
rows = response.get("resultTable").get("rows");
assertEquals(rows.size(), 1);
row = rows.get(0);
assertEquals(row.size(), 2);
assertEquals(row.get(0).asText(), "");
assertEquals(row.get(1).asDouble(), 0.0);
pinotQuery = "SELECT NewAddedMVLongDimension, SUM(NewAddedIntMetric) FROM mytable GROUP BY NewAddedMVLongDimension";
"SELECT "
+ (useMultiStageQueryEngine() ? "arrayToMV(NewAddedMVLongDimension)" : "NewAddedMVLongDimension")
+ ", SUM(NewAddedIntMetric) FROM mytable GROUP BY "
+ (useMultiStageQueryEngine() ? "arrayToMV(NewAddedMVLongDimension)" : "NewAddedMVLongDimension");
response = postQuery(pinotQuery);
rows = response.get("resultTable").get("rows");
assertEquals(rows.size(), 1);
row = rows.get(0);
assertEquals(row.size(), 2);
assertEquals(row.get(0).asLong(), Long.MIN_VALUE);
assertEquals(row.get(1).asDouble(), numTotalDocsInDouble);
String newAddedDimensions =

String newAddedDimensions = useMultiStageQueryEngine()
? "arrayToMV(NewAddedMVIntDimension), arrayToMV(NewAddedMVLongDimension), arrayToMV(NewAddedMVFloatDimension), "
+ "arrayToMV(NewAddedMVDoubleDimension), arrayToMV(NewAddedMVBooleanDimension), "
+ "arrayToMV(NewAddedMVTimestampDimension), arrayToMV(NewAddedMVStringDimension), "
+ "NewAddedSVJSONDimension, NewAddedSVBytesDimension"
:
"NewAddedMVIntDimension, NewAddedMVLongDimension, NewAddedMVFloatDimension, NewAddedMVDoubleDimension, "
+ "NewAddedMVBooleanDimension, NewAddedMVTimestampDimension, NewAddedMVStringDimension, "
+ "NewAddedSVJSONDimension, NewAddedSVBytesDimension";
Expand Down Expand Up @@ -1917,7 +2002,6 @@ public void testInBuiltVirtualColumns(boolean useMultiStageQueryEngine)
public void testGroupByUDF(boolean useMultiStageQueryEngine)
throws Exception {
setUseMultiStageQueryEngine(useMultiStageQueryEngine);
notSupportedInV2();
String query = "SELECT timeConvert(DaysSinceEpoch,'DAYS','SECONDS'), COUNT(*) FROM mytable "
+ "GROUP BY timeConvert(DaysSinceEpoch,'DAYS','SECONDS') ORDER BY COUNT(*) DESC";
JsonNode response = postQuery(query);
Expand Down Expand Up @@ -2040,8 +2124,13 @@ public void testGroupByUDF(boolean useMultiStageQueryEngine)
assertEquals(row.get(0).asInt(), 2);
assertEquals(row.get(1).asLong(), 2);

query = "SELECT valueIn(DivAirports,'DFW','ORD'), COUNT(*) FROM mytable "
+ "GROUP BY valueIn(DivAirports,'DFW','ORD') ORDER BY COUNT(*) DESC";
if (useMultiStageQueryEngine()) {
query = "SELECT arrayToMV(valueIn(DivAirports,'DFW','ORD')), COUNT(*) FROM mytable "
+ "GROUP BY arrayToMV(valueIn(DivAirports,'DFW','ORD')) ORDER BY COUNT(*) DESC";
} else {
query = "SELECT valueIn(DivAirports,'DFW','ORD'), COUNT(*) FROM mytable "
+ "GROUP BY valueIn(DivAirports,'DFW','ORD') ORDER BY COUNT(*) DESC";
}
response = postQuery(query);
resultTable = response.get("resultTable");
dataSchema = resultTable.get("dataSchema");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,9 @@


/**
* Integration test for heap size based server query killing, this works only for xmx4G
* Integration test for heap size based server query killing, this works only for xmx4G.
* <p>
* Query killing isn't currently supported in the v2 multi-stage query engine so these tests only run on the v1 engine.
*/
public class OfflineClusterMemBasedServerQueryKillingTest extends BaseClusterIntegrationTestSet {
public static final String STRING_DIM_SV1 = "stringDimSV1";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,8 @@

/**
* Integration test for heap size based server query killing, this works only for xmx4G
* <p>
* Query killing isn't currently supported in the v2 multi-stage query engine so these tests only run on the v1 engine.
*/
public class OfflineClusterServerCPUTimeQueryKillingTest extends BaseClusterIntegrationTestSet {
public static final String STRING_DIM_SV1 = "stringDimSV1";
Expand Down