Skip to content

Commit

Permalink
Support verifying structured types containing DATE and UNKNOWN
Browse files Browse the repository at this point in the history
  • Loading branch information
@caithagoras0
caithagoras0 committed Mar 23, 2020
1 parent ed7a6ef commit 1bbd131
Show file tree
Hide file tree
Showing 4 changed files with 82 additions and 2 deletions.
1 change: 0 additions & 1 deletion presto-verifier/src/main/java/com/facebook/presto/verifier/framework/VerifierUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,6 @@
import java.util.function.Function;
import java.util.stream.IntStream;

import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature;
import static com.facebook.presto.sql.parser.ParsingOptions.DecimalLiteralTreatment.AS_DOUBLE;
import static com.google.common.base.Functions.identity;
import static com.google.common.collect.ImmutableList.toImmutableList;
Expand Down
36 changes: 35 additions & 1 deletion presto-verifier/src/main/java/com/facebook/presto/verifier/rewrite/QueryRewriter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,13 @@
*/
package com.facebook.presto.verifier.rewrite;

import com.facebook.presto.spi.type.ArrayType;
import com.facebook.presto.spi.type.MapType;
import com.facebook.presto.spi.type.RowType;
import com.facebook.presto.spi.type.Type;
import com.facebook.presto.spi.type.TypeManager;
import com.facebook.presto.spi.type.TypeSignature;
import com.facebook.presto.spi.type.TypeSignatureParameter;
import com.facebook.presto.sql.parser.SqlParser;
import com.facebook.presto.sql.tree.AllColumns;
import com.facebook.presto.sql.tree.Cast;
Expand Down Expand Up @@ -54,6 +59,8 @@

import static com.facebook.presto.spi.type.BigintType.BIGINT;
import static com.facebook.presto.spi.type.DateType.DATE;
import static com.facebook.presto.spi.type.RowType.Field;
import static com.facebook.presto.spi.type.StandardTypes.MAP;
import static com.facebook.presto.spi.type.TimestampType.TIMESTAMP;
import static com.facebook.presto.sql.tree.LikeClause.PropertiesOption.INCLUDING;
import static com.facebook.presto.type.UnknownType.UNKNOWN;
Expand Down Expand Up @@ -265,14 +272,41 @@ private Query rewriteNonStorableColumns(Query query, ResultSetMetaData metadata)
query.getLimit());
}

private static Optional<Type> getColumnTypeRewrite(Type type)
private Optional<Type> getColumnTypeRewrite(Type type)
{
if (type.equals(DATE)) {
return Optional.of(TIMESTAMP);
}
if (type.equals(UNKNOWN)) {
return Optional.of(BIGINT);
}
if (type instanceof ArrayType) {
return getColumnTypeRewrite(((ArrayType) type).getElementType()).map(ArrayType::new);
}
if (type instanceof MapType) {
Type keyType = ((MapType) type).getKeyType();
Type valueType = ((MapType) type).getValueType();
Optional<Type> keyTypeRewrite = getColumnTypeRewrite(keyType);
Optional<Type> valueTypeRewrite = getColumnTypeRewrite(valueType);
if (keyTypeRewrite.isPresent() || valueTypeRewrite.isPresent()) {
return Optional.of(typeManager.getType(new TypeSignature(
MAP,
TypeSignatureParameter.of(keyTypeRewrite.orElse(keyType).getTypeSignature()),
TypeSignatureParameter.of(valueTypeRewrite.orElse(valueType).getTypeSignature()))));
}
return Optional.empty();
}
if (type instanceof RowType) {
List<Field> fields = ((RowType) type).getFields();
List<Field> fieldsRewrite = new ArrayList<>();
boolean rewrite = false;
for (Field field : fields) {
Optional<Type> fieldTypeRewrite = getColumnTypeRewrite(field.getType());
rewrite = rewrite || fieldTypeRewrite.isPresent();
fieldsRewrite.add(new Field(field.getName(), fieldTypeRewrite.orElse(field.getType())));
}
return rewrite ? Optional.of(RowType.from(fieldsRewrite)) : Optional.empty();
}
return Optional.empty();
}

Expand Down
17 changes: 17 additions & 0 deletions ...o-verifier/src/test/java/com/facebook/presto/verifier/framework/TestDataVerification.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -303,6 +303,23 @@ public void testSelectUnknown()
assertEvent(event.get(), SUCCEEDED, Optional.empty(), Optional.empty(), Optional.empty());
}

@Test
public void testSelectNonStorableStructuredColumns()
{
String query = "SELECT\n" +
" ARRAY[DATE '2020-01-01'],\n" +
" ARRAY[NULL],\n" +
" MAP(\n" +
" ARRAY[DATE '2020-01-01'], ARRAY[\n" +
" CAST(ROW(1, 'a', DATE '2020-01-01') AS ROW(x int, y VARCHAR, z date))\n" +
" ]\n" +
" ),\n" +
" ROW(NULL)";
Optional<VerifierQueryEvent> event = createVerification(query, query).run();
assertTrue(event.isPresent());
assertEvent(event.get(), SUCCEEDED, Optional.empty(), Optional.empty(), Optional.empty());
}

@Test
public void testChecksumQueryCompilerError()
{
Expand Down
30 changes: 30 additions & 0 deletions presto-verifier/src/test/java/com/facebook/presto/verifier/rewrite/TestQueryRewriter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -187,6 +187,36 @@ public void testRewriteUnknown()
", CAST(null AS bigint) unknown");
}

@Test
public void testRewriteNonStorableStructuredTypes()
{
QueryBundle queryBundle = getQueryRewriter(DEFAULT_PREFIX).rewriteQuery(
"SELECT\n" +
" ARRAY[DATE '2020-01-01'],\n" +
" ARRAY[NULL],\n" +
" MAP(\n" +
" ARRAY[DATE '2020-01-01'], ARRAY[\n" +
" CAST(ROW(1, 'a', DATE '2020-01-01') AS ROW(x int, y VARCHAR, z date))\n" +
" ]\n" +
" ),\n" +
" ROW(NULL)",
CONTROL);
assertCreateTableAs(
queryBundle.getQuery(),
"SELECT\n" +
" CAST(ARRAY[DATE '2020-01-01'] AS ARRAY(timestamp)),\n" +
" CAST(ARRAY[NULL] AS ARRAY(BIGINT)),\n" +
" CAST(\n" +
" \"map\"(\n" +
" ARRAY[DATE '2020-01-01'],\n" +
" ARRAY[\n" +
" CAST(ROW (1, 'a', DATE '2020-01-01') AS ROW(x int, y VARCHAR, z date))\n" +
" ]\n" +
" ) AS MAP(timestamp, ROW(x INTEGER, y VARCHAR, z timestamp))\n" +
" ),\n" +
" CAST(ROW (NULL) AS ROW(BIGINT))");
}

private void assertShadowed(
QueryRewriter queryRewriter,
@Language("SQL") String query,
Expand Down

0 comments on commit 1bbd131

Please sign in to comment.