Skip to content

[multistage][test] add ignored test capability #11703

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
walterddr merged 1 commit into from
Sep 28, 2023
Merged

[multistage][test] add ignored test capability #11703

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 23 additions & 14 deletions ...untime/src/test/java/org/apache/pinot/query/runtime/queries/ResourceBasedQueriesTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,9 +71,11 @@ public class ResourceBasedQueriesTest extends QueryRunnerTestBase {
private static final String QUERY_TEST_RESOURCE_FOLDER = "queries";
private static final Random RANDOM = new Random(42);
private static final String FILE_FILTER_PROPERTY = "pinot.fileFilter";
private static final String IGNORE_FILTER_PROPERTY = "pinot.runIgnored";
private static final int NUM_PARTITIONS = 4;

private final Map<String, Set<String>> _tableToSegmentMap = new HashMap<>();
private boolean _isRunIgnored;
private TimeZone _currentSystemTimeZone;

@BeforeClass
Expand All @@ -82,6 +84,9 @@ public void setUp()
// Save the original default timezone
_currentSystemTimeZone = TimeZone.getDefault();

String runIgnoredProp = System.getProperty(IGNORE_FILTER_PROPERTY);
_isRunIgnored = runIgnoredProp != null;

// Change the default timezone
TimeZone.setDefault(TimeZone.getTimeZone("America/Los_Angeles"));

Expand Down Expand Up @@ -245,7 +250,7 @@ public void tearDown() {

// TODO: name the test using testCaseName for testng reports
@Test(dataProvider = "testResourceQueryTestCaseProviderInputOnly")
public void testQueryTestCasesWithH2(String testCaseName, String sql, String h2Sql, String expect,
public void testQueryTestCasesWithH2(String testCaseName, boolean isIgnored, String sql, String h2Sql, String expect,
boolean keepOutputRowOrder)
throws Exception {
// query pinot
Expand All @@ -259,15 +264,15 @@ public void testQueryTestCasesWithH2(String testCaseName, String sql, String h2S
}

@Test(dataProvider = "testResourceQueryTestCaseProviderBoth")
public void testQueryTestCasesWithOutput(String testCaseName, String sql, String h2Sql, List<Object[]> expectedRows,
String expect, boolean keepOutputRowOrder)
public void testQueryTestCasesWithOutput(String testCaseName, boolean isIgnored, String sql, String h2Sql,
List<Object[]> expectedRows, String expect, boolean keepOutputRowOrder)
throws Exception {
runQuery(sql, expect, null).ifPresent(rows -> compareRowEquals(rows, expectedRows, keepOutputRowOrder));
}

@Test(dataProvider = "testResourceQueryTestCaseProviderWithMetadata")
public void testQueryTestCasesWithMetadata(String testCaseName, String sql, String h2Sql, String expect,
int numSegments)
public void testQueryTestCasesWithMetadata(String testCaseName, boolean isIgnored, String sql, String h2Sql,
String expect, int numSegments)
throws Exception {
Map<Integer, ExecutionStatsAggregator> executionStatsAggregatorMap = new HashMap<>();
runQuery(sql, expect, executionStatsAggregatorMap).ifPresent(rows -> {
Expand Down Expand Up @@ -358,7 +363,7 @@ private Object[][] testResourceQueryTestCaseProviderBoth()

List<QueryTestCase.Query> queryCases = testCaseEntry.getValue()._queries;
for (QueryTestCase.Query queryCase : queryCases) {
if (queryCase._ignored) {
if (queryCase._ignored && !_isRunIgnored) {
continue;
}

Expand All @@ -372,7 +377,8 @@ private Object[][] testResourceQueryTestCaseProviderBoth()
expectedRows.add(objs.toArray());
}
Object[] testEntry = new Object[]{
testCaseName, sql, h2Sql, expectedRows, queryCase._expectedException, queryCase._keepOutputRowOrder
testCaseName, queryCase._ignored, sql, h2Sql, expectedRows, queryCase._expectedException,
queryCase._keepOutputRowOrder
};
providerContent.add(testEntry);
}
Expand Down Expand Up @@ -402,7 +408,7 @@ private Object[][] testResourceQueryTestCaseProviderWithMetadata()
List<QueryTestCase.Query> queryCases = testCaseEntry.getValue()._queries;
for (QueryTestCase.Query queryCase : queryCases) {

if (queryCase._ignored) {
if (queryCase._ignored && !_isRunIgnored) {
continue;
}

Expand All @@ -418,7 +424,9 @@ private Object[][] testResourceQueryTestCaseProviderWithMetadata()
throw new RuntimeException("Unable to test metadata without expected num segments configuration!");
}

Object[] testEntry = new Object[]{testCaseName, sql, h2Sql, queryCase._expectedException, segmentCount};
Object[] testEntry = new Object[]{
testCaseName, queryCase._ignored, sql, h2Sql, queryCase._expectedException, segmentCount
};
providerContent.add(testEntry);
}
}
Expand All @@ -439,15 +447,16 @@ private Object[][] testResourceQueryTestCaseProviderInputOnly()
String testCaseName = testCaseEntry.getKey();
List<QueryTestCase.Query> queryCases = testCaseEntry.getValue()._queries;
for (QueryTestCase.Query queryCase : queryCases) {
if (queryCase._ignored) {
if (queryCase._ignored && !_isRunIgnored) {
continue;
}
if (queryCase._outputs == null) {
String sql = replaceTableName(testCaseName, queryCase._sql);
String h2Sql = queryCase._h2Sql != null ? replaceTableName(testCaseName, queryCase._h2Sql)
: replaceTableName(testCaseName, queryCase._sql);
Object[] testEntry =
new Object[]{testCaseName, sql, h2Sql, queryCase._expectedException, queryCase._keepOutputRowOrder};
Object[] testEntry = new Object[]{
testCaseName, queryCase._ignored, sql, h2Sql, queryCase._expectedException, queryCase._keepOutputRowOrder
};
providerContent.add(testEntry);
}
}
Expand Down Expand Up @@ -476,11 +485,11 @@ private Map<String, QueryTestCase> getTestCases()
}

// get filter if set
String property = System.getProperty(FILE_FILTER_PROPERTY);
String fileFilterProp = System.getProperty(FILE_FILTER_PROPERTY);

// Load each test file.
for (String testCaseName : testFilenames) {
if (property != null && !testCaseName.toLowerCase().contains(property.toLowerCase())) {
if (fileFilterProp != null && !testCaseName.toLowerCase().contains(fileFilterProp.toLowerCase())) {
continue;
}

Expand Down
8 changes: 0 additions & 8 deletions pinot-query-runtime/src/test/resources/queries/Aggregates.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -729,8 +729,6 @@
"sql": "SELECT sum(int_col) FROM {tbl} WHERE string_col IN ( SELECT string_col FROM {tbl} WHERE int_col BETWEEN 1 AND 0 GROUP BY string_col )"
},
{
"ignored": true,
"comment": "min empty returns [infinity] instead of [null] at the moment when leaf aggregation is performed by v1 engine.",
"description": "Return empty for min",
"sql": "SELECT min(int_col) FROM {tbl} WHERE string_col IN ('foo', 'bar')"
},
Expand All @@ -739,8 +737,6 @@
"sql": "SELECT min(int_col) FROM {tbl} WHERE string_col IN ( SELECT string_col FROM {tbl} WHERE int_col BETWEEN 1 AND 0 GROUP BY string_col )"
},
{
"ignored": true,
"comment": "max empty returns [-infinity] instead of [null] at the moment when leaf aggregation is performed by v1 engine.",
"description": "Return empty for max",
"sql": "SELECT max(int_col) FROM {tbl} WHERE string_col IN ('foo', 'bar')"
},
Expand Down Expand Up @@ -774,8 +770,6 @@
"sql": "SELECT count(*) FROM {tbl} WHERE string_col IN ( SELECT string_col FROM {tbl} WHERE int_col BETWEEN 1 AND 0 GROUP BY string_col )"
},
{
"ignored": true,
"comment": "bool_and empty returns [true] instead of [null] at the moment when leaf aggregation is performed by v1 engine.",
"description": "Return empty for bool_and",
"sql": "SELECT bool_and(bool_col) FROM {tbl} WHERE string_col IN ('foo', 'bar')"
},
Expand All @@ -784,8 +778,6 @@
"sql": "SELECT bool_and(bool_col) FROM {tbl} WHERE string_col IN ( SELECT string_col FROM {tbl} WHERE int_col BETWEEN 1 AND 0 GROUP BY string_col )"
},
{
"ignored": true,
"comment": "bool_or empty returns [false] instead of [null] at the moment when leaf aggregation is performed by v1 engine.",
"description": "Return empty for bool_or",
"sql": "SELECT bool_or(bool_col) FROM {tbl} WHERE string_col IN ('foo', 'bar')"
},
Expand Down
14 changes: 3 additions & 11 deletions pinot-query-runtime/src/test/resources/queries/Comparisons.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -400,19 +400,11 @@
{ "sql": "SELECT 3 BETWEEN small AND big FROM {tbl}" },
{ "sql": "SELECT val NOT BETWEEN small AND big FROM {tbl}" },
{ "sql": "SELECT val NOT BETWEEN big AND small FROM {tbl}" },
{
"ignored": true,
"sql": "SELECT val NOT BETWEEN 2 AND 3 FROM {tbl}",
"comment": "Somehow it failed with Range is not implemented yet"
},
{ "sql": "SELECT val NOT BETWEEN 2 AND 3 FROM {tbl}" },
{ "sql": "SELECT val NOT BETWEEN 3 AND 2 FROM {tbl}" },
{ "sql": "SELECT 3 NOT BETWEEN small AND big FROM {tbl}" },
{
"sql": "SELECT val BETWEEN SYMMETRIC small AND big FROM {tbl}"
},
{
"sql": "SELECT val BETWEEN SYMMETRIC big AND small FROM {tbl}"
}
{ "sql": "SELECT val BETWEEN SYMMETRIC small AND big FROM {tbl}" },
{ "sql": "SELECT val BETWEEN SYMMETRIC big AND small FROM {tbl}" }
]
},
"between_bigints": {
Expand Down
8 changes: 0 additions & 8 deletions pinot-query-runtime/src/test/resources/queries/MathFuncs.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -270,8 +270,6 @@
"sql": "SELECT intCol % 10 FROM {numTbl}"
},
{
"ignored": true,
"comment": "should return 0 for 0 value but we are not",
"description": "test mod on floating point literal with columns",
"sql": "SELECT intCol % 1.2 FROM {numTbl}"
},
Expand Down Expand Up @@ -345,8 +343,6 @@
"sql": "SELECT exp(floatCol) FROM {numTbl}"
},
{
"ignored": true,
"comment": "Argument cannot be literal for transform function: exp:",
"description": "test exp on literal",
"sql": "SELECT exp(2.0) FROM {numTbl}"
}
Expand Down Expand Up @@ -387,8 +383,6 @@
"sql": "SELECT floor(floatCol) FROM {numTbl}"
},
{
"ignored": true,
"comment": "Caught exception while initializing transform function: floor",
"description": "test exp on literal",
"sql": "SELECT floor(2.0) FROM {numTbl}"
}
Expand Down Expand Up @@ -429,8 +423,6 @@
"sql": "SELECT ceil(floatCol) FROM {numTbl}"
},
{
"ignored": true,
"comment": "Caught exception while initializing transform function: ceil",
"description": "test exp on literal",
"sql": "SELECT ceil(2.0) FROM {numTbl}"
}
Expand Down