diff --git a/LICENSE.bin b/LICENSE.bin index a32729253d6..ce86e8edec7 100644 --- a/LICENSE.bin +++ b/LICENSE.bin @@ -256,7 +256,6 @@ Airlift The Netty Project Open Telemetry - Substrait Java Trino Jakarta Dependency Injection Jakarta Bean Validation diff --git a/api/build.gradle.kts b/api/build.gradle.kts index 231a88916eb..a25068320c2 100644 --- a/api/build.gradle.kts +++ b/api/build.gradle.kts @@ -10,14 +10,6 @@ plugins { } dependencies { - implementation(libs.substrait.java.core) { - exclude("com.fasterxml.jackson.core") - exclude("com.fasterxml.jackson.datatype") - exclude("com.fasterxml.jackson.dataformat") - exclude("com.google.protobuf") - exclude("com.google.code.findbugs") - exclude("org.slf4j") - } implementation(libs.guava) implementation(libs.slf4j.api) diff --git a/api/src/main/java/com/datastrato/gravitino/rel/Column.java b/api/src/main/java/com/datastrato/gravitino/rel/Column.java index bbc1342987e..8399766fa29 100644 --- a/api/src/main/java/com/datastrato/gravitino/rel/Column.java +++ b/api/src/main/java/com/datastrato/gravitino/rel/Column.java @@ -5,7 +5,7 @@ package com.datastrato.gravitino.rel; import com.datastrato.gravitino.NameIdentifier; -import io.substrait.type.Type; +import com.datastrato.gravitino.rel.types.Type; import java.util.Map; /** diff --git a/api/src/main/java/com/datastrato/gravitino/rel/TableChange.java b/api/src/main/java/com/datastrato/gravitino/rel/TableChange.java index 40a688a226d..5fb6c842cf1 100644 --- a/api/src/main/java/com/datastrato/gravitino/rel/TableChange.java +++ b/api/src/main/java/com/datastrato/gravitino/rel/TableChange.java @@ -20,7 +20,7 @@ package com.datastrato.gravitino.rel; -import io.substrait.type.Type; +import com.datastrato.gravitino.rel.types.Type; import lombok.EqualsAndHashCode; import lombok.Getter; diff --git a/api/src/main/java/com/datastrato/gravitino/rel/expressions/Literal.java b/api/src/main/java/com/datastrato/gravitino/rel/expressions/Literal.java index d8260e717df..2a4a5c773aa 100644 --- a/api/src/main/java/com/datastrato/gravitino/rel/expressions/Literal.java +++ b/api/src/main/java/com/datastrato/gravitino/rel/expressions/Literal.java @@ -20,8 +20,8 @@ package com.datastrato.gravitino.rel.expressions; -import io.substrait.type.Type; -import io.substrait.type.TypeCreator; +import com.datastrato.gravitino.rel.types.Type; +import com.datastrato.gravitino.rel.types.Types; import java.util.Objects; /** @@ -60,7 +60,7 @@ static LiteralImpl of(T value, Type dataType) { * @return a new {@link Literal} instance */ static LiteralImpl integer(Integer value) { - return of(value, TypeCreator.REQUIRED.I32); + return of(value, Types.IntegerType.get()); } /** @@ -70,7 +70,7 @@ static LiteralImpl integer(Integer value) { * @return a new {@link Literal} instance */ static LiteralImpl string(String value) { - return of(value, TypeCreator.REQUIRED.STRING); + return of(value, Types.StringType.get()); } final class LiteralImpl implements Literal { diff --git a/api/src/test/java/com/datastrato/gravitino/TestTableChange.java b/api/src/test/java/com/datastrato/gravitino/TestTableChange.java index dd46e813419..2826ad3421e 100644 --- a/api/src/test/java/com/datastrato/gravitino/TestTableChange.java +++ b/api/src/test/java/com/datastrato/gravitino/TestTableChange.java @@ -23,7 +23,8 @@ import com.datastrato.gravitino.rel.TableChange.UpdateColumnPosition; import com.datastrato.gravitino.rel.TableChange.UpdateColumnType; import com.datastrato.gravitino.rel.TableChange.UpdateComment; -import io.substrait.type.Type; +import com.datastrato.gravitino.rel.types.Type; +import com.datastrato.gravitino.rel.types.Types; import org.junit.jupiter.api.Test; public class TestTableChange { @@ -67,7 +68,7 @@ public void testColumnPosition() { @Test public void testAddColumn() { String[] fieldNames = {"Name"}; - Type dataType = Type.withNullability(false).STRING; + Type dataType = Types.StringType.get(); String comment = "Person name"; AddColumn addColumn = (AddColumn) TableChange.addColumn(fieldNames, dataType, comment); @@ -80,7 +81,7 @@ public void testAddColumn() { @Test public void testAddColumnWithPosition() { String[] fieldNames = {"Full Name", "First Name"}; - Type dataType = Type.withNullability(false).STRING; + Type dataType = Types.StringType.get(); String comment = "First or given name"; TableChange.ColumnPosition position = TableChange.ColumnPosition.after("Address"); AddColumn addColumn = @@ -95,7 +96,7 @@ public void testAddColumnWithPosition() { @Test public void testAddColumnWithNullCommentAndPosition() { String[] fieldNames = {"Middle Name"}; - Type dataType = Type.withNullability(false).STRING; + Type dataType = Types.StringType.get(); AddColumn addColumn = (AddColumn) TableChange.addColumn(fieldNames, dataType, null, null); assertArrayEquals(fieldNames, addColumn.fieldNames()); @@ -191,7 +192,7 @@ public void testDeleteNestedColumn() { @Test public void testUpdateColumnType() { String[] fieldNames = {"existing_column"}; - Type dataType = Type.withNullability(false).STRING; + Type dataType = Types.StringType.get(); UpdateColumnType updateColumnType = (UpdateColumnType) TableChange.updateColumnType(fieldNames, dataType); @@ -202,7 +203,7 @@ public void testUpdateColumnType() { @Test public void testUpdateNestedColumnType() { String[] fieldNames = {"nested", "existing_column"}; - Type dataType = Type.withNullability(false).STRING; + Type dataType = Types.StringType.get(); UpdateColumnType updateColumnType = (UpdateColumnType) TableChange.updateColumnType(fieldNames, dataType); @@ -347,7 +348,7 @@ void testColumnRenameNotEqualsAndHashCode() { @Test void testColumnUpdateTypeEqualsAndHashCode() { String[] nameA = {"First Name"}; - Type dataType = Type.withNullability(false).STRING; + Type dataType = Types.StringType.get(); UpdateColumnType columnA = (UpdateColumnType) TableChange.updateColumnType(nameA, dataType); String[] nameB = {"First Name"}; UpdateColumnType columnB = (UpdateColumnType) TableChange.updateColumnType(nameB, dataType); @@ -360,7 +361,7 @@ void testColumnUpdateTypeEqualsAndHashCode() { @Test void testColumnUpdateTypeNotEqualsAndHashCode() { String[] nameA = {"First Name"}; - Type dataType = Type.withNullability(false).STRING; + Type dataType = Types.StringType.get(); UpdateColumnType columnA = (UpdateColumnType) TableChange.updateColumnType(nameA, dataType); String[] nameB = {"Given Name"}; UpdateColumnType columnB = (UpdateColumnType) TableChange.updateColumnType(nameB, dataType); @@ -492,11 +493,11 @@ void testSetPropertyNotEqualsAndHashCode() { @Test void testAddColumnEqualsAndHashCode() { String[] fieldNamesA = {"Name"}; - Type dataTypeA = Type.withNullability(false).STRING; + Type dataTypeA = Types.StringType.get(); String commentA = "Person name"; AddColumn columnA = (AddColumn) TableChange.addColumn(fieldNamesA, dataTypeA, commentA); String[] fieldNamesB = {"Name"}; - Type dataTypeB = Type.withNullability(false).STRING; + Type dataTypeB = Types.StringType.get(); String commentB = "Person name"; AddColumn columnB = (AddColumn) TableChange.addColumn(fieldNamesB, dataTypeB, commentB); @@ -508,11 +509,11 @@ void testAddColumnEqualsAndHashCode() { @Test void testAddColumnNotEqualsAndHashCode() { String[] fieldNamesA = {"Name"}; - Type dataTypeA = Type.withNullability(false).STRING; + Type dataTypeA = Types.StringType.get(); String commentA = "Person name"; AddColumn columnA = (AddColumn) TableChange.addColumn(fieldNamesA, dataTypeA, commentA); String[] fieldNamesB = {"First Name"}; - Type dataTypeB = Type.withNullability(false).STRING; + Type dataTypeB = Types.StringType.get(); String commentB = "Person name"; AddColumn columnB = (AddColumn) TableChange.addColumn(fieldNamesB, dataTypeB, commentB); diff --git a/api/src/test/java/com/datastrato/gravitino/rel/TestTransforms.java b/api/src/test/java/com/datastrato/gravitino/rel/TestTransforms.java index 7110da9992f..9b0db806165 100644 --- a/api/src/test/java/com/datastrato/gravitino/rel/TestTransforms.java +++ b/api/src/test/java/com/datastrato/gravitino/rel/TestTransforms.java @@ -21,8 +21,8 @@ import com.datastrato.gravitino.rel.expressions.Literal; import com.datastrato.gravitino.rel.expressions.NamedReference; import com.datastrato.gravitino.rel.expressions.transforms.Transform; -import io.substrait.type.Type; -import io.substrait.type.TypeCreator; +import com.datastrato.gravitino.rel.types.Type; +import com.datastrato.gravitino.rel.types.Types; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -38,7 +38,7 @@ public String name() { @Override public Type dataType() { - return TypeCreator.NULLABLE.I8; + return Types.ByteType.get(); } @Override @@ -85,7 +85,7 @@ public String name() { @Override public Type dataType() { - return TypeCreator.NULLABLE.I8; + return Types.ByteType.get(); } @Override diff --git a/catalogs/catalog-hive/build.gradle.kts b/catalogs/catalog-hive/build.gradle.kts index 53f267307d4..0e37d84490a 100644 --- a/catalogs/catalog-hive/build.gradle.kts +++ b/catalogs/catalog-hive/build.gradle.kts @@ -65,12 +65,6 @@ dependencies { exclude("*") } - implementation(libs.substrait.java.core) { - exclude("org.slf4j") - exclude("com.fasterxml.jackson.core") - exclude("com.fasterxml.jackson.datatype") - } - implementation(libs.slf4j.api) implementation(libs.guava) diff --git a/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveCatalogOperations.java b/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveCatalogOperations.java index b62ece4fcbb..35b5645e2dd 100644 --- a/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveCatalogOperations.java +++ b/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveCatalogOperations.java @@ -778,7 +778,7 @@ private void doAddColumn(List cols, TableChange.AddColumn change) { targetPosition, new FieldSchema( change.fieldNames()[0], - change.getDataType().accept(ToHiveType.INSTANCE).getQualifiedName(), + ToHiveType.convert(change.getDataType()).getQualifiedName(), change.getComment())); } @@ -826,8 +826,7 @@ private void doUpdateColumnType(List cols, TableChange.UpdateColumn if (indexOfColumn == -1) { throw new IllegalArgumentException("UpdateColumnType does not exist: " + columnName); } - cols.get(indexOfColumn) - .setType(change.getNewDataType().accept(ToHiveType.INSTANCE).getQualifiedName()); + cols.get(indexOfColumn).setType(ToHiveType.convert(change.getNewDataType()).getQualifiedName()); } /** diff --git a/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveTable.java b/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveTable.java index 19b4d84b8b0..1ac9a16ccb8 100644 --- a/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveTable.java +++ b/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveTable.java @@ -213,7 +213,7 @@ private FieldSchema getPartitionKey(String[] fieldName) { .collect(Collectors.toList()); return new FieldSchema( partitionColumns.get(0).name(), - partitionColumns.get(0).dataType().accept(ToHiveType.INSTANCE).getQualifiedName(), + ToHiveType.convert(partitionColumns.get(0).dataType()).getQualifiedName(), partitionColumns.get(0).comment()); } @@ -228,9 +228,7 @@ private StorageDescriptor buildStorageDescriptor( .map( c -> new FieldSchema( - c.name(), - c.dataType().accept(ToHiveType.INSTANCE).getQualifiedName(), - c.comment())) + c.name(), ToHiveType.convert(c.dataType()).getQualifiedName(), c.comment())) .collect(Collectors.toList())); // `location` must not be null, otherwise it will result in an NPE when calling HMS `alterTable` diff --git a/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/converter/FromHiveType.java b/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/converter/FromHiveType.java index 2456d32a4b8..13bd38d7fdd 100644 --- a/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/converter/FromHiveType.java +++ b/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/converter/FromHiveType.java @@ -5,23 +5,24 @@ package com.datastrato.gravitino.catalog.hive.converter; import static org.apache.hadoop.hive.serde.serdeConstants.BIGINT_TYPE_NAME; +import static org.apache.hadoop.hive.serde.serdeConstants.BINARY_TYPE_NAME; import static org.apache.hadoop.hive.serde.serdeConstants.BOOLEAN_TYPE_NAME; -import static org.apache.hadoop.hive.serde.serdeConstants.CHAR_TYPE_NAME; import static org.apache.hadoop.hive.serde.serdeConstants.DATE_TYPE_NAME; -import static org.apache.hadoop.hive.serde.serdeConstants.DECIMAL_TYPE_NAME; import static org.apache.hadoop.hive.serde.serdeConstants.DOUBLE_TYPE_NAME; import static org.apache.hadoop.hive.serde.serdeConstants.FLOAT_TYPE_NAME; +import static org.apache.hadoop.hive.serde.serdeConstants.INTERVAL_DAY_TIME_TYPE_NAME; +import static org.apache.hadoop.hive.serde.serdeConstants.INTERVAL_YEAR_MONTH_TYPE_NAME; import static org.apache.hadoop.hive.serde.serdeConstants.INT_TYPE_NAME; import static org.apache.hadoop.hive.serde.serdeConstants.SMALLINT_TYPE_NAME; import static org.apache.hadoop.hive.serde.serdeConstants.STRING_TYPE_NAME; import static org.apache.hadoop.hive.serde.serdeConstants.TIMESTAMP_TYPE_NAME; import static org.apache.hadoop.hive.serde.serdeConstants.TINYINT_TYPE_NAME; -import static org.apache.hadoop.hive.serde.serdeConstants.VARCHAR_TYPE_NAME; import static org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils.getTypeInfoFromTypeString; -import io.substrait.type.Type; -import io.substrait.type.TypeCreator; -import java.util.stream.Collectors; +import com.datastrato.gravitino.rel.types.Type; +import com.datastrato.gravitino.rel.types.Types; +import java.util.ArrayList; +import java.util.stream.IntStream; import org.apache.hadoop.hive.serde2.typeinfo.CharTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo; @@ -31,84 +32,101 @@ import org.apache.hadoop.hive.serde2.typeinfo.UnionTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo; -/** Converts Hive data types to corresponding Substrait data types. */ +/** Converts Hive data types to corresponding Gravitino data types. */ public class FromHiveType { /** - * Converts a Hive data type string to the corresponding Substrait data type. + * Converts a Hive data type string to the corresponding Gravitino data type. * * @param hiveType The Hive data type string to convert. - * @return The equivalent Substrait data type. + * @return The equivalent Gravitino data type. * @throws IllegalArgumentException If the Hive data type is unknown or unsupported. */ public static Type convert(String hiveType) throws IllegalArgumentException { TypeInfo hiveTypeInfo = getTypeInfoFromTypeString(hiveType); - return toSubstraitType(hiveTypeInfo); + return toGravitinoType(hiveTypeInfo); } /** - * Converts a Hive TypeInfo object to the corresponding Substrait Type. + * Converts a Hive TypeInfo object to the corresponding Gravitino Type. * * @param hiveTypeInfo The Hive TypeInfo object to convert. - * @return The equivalent Substrait Type. + * @return The equivalent Gravitino Type. * @throws IllegalArgumentException if the Hive data type category is unknown or unsupported. */ - private static Type toSubstraitType(TypeInfo hiveTypeInfo) throws IllegalArgumentException { + private static Type toGravitinoType(TypeInfo hiveTypeInfo) throws IllegalArgumentException { switch (hiveTypeInfo.getCategory()) { case PRIMITIVE: switch (hiveTypeInfo.getTypeName()) { case BOOLEAN_TYPE_NAME: - return TypeCreator.NULLABLE.BOOLEAN; + return Types.BooleanType.get(); case TINYINT_TYPE_NAME: - return TypeCreator.NULLABLE.I8; + return Types.ByteType.get(); case SMALLINT_TYPE_NAME: - return TypeCreator.NULLABLE.I16; + return Types.ShortType.get(); case INT_TYPE_NAME: - return TypeCreator.NULLABLE.I32; + return Types.IntegerType.get(); case BIGINT_TYPE_NAME: - return TypeCreator.NULLABLE.I64; + return Types.LongType.get(); case FLOAT_TYPE_NAME: - return TypeCreator.NULLABLE.FP32; + return Types.FloatType.get(); case DOUBLE_TYPE_NAME: - return TypeCreator.NULLABLE.FP64; + return Types.DoubleType.get(); case STRING_TYPE_NAME: - return TypeCreator.NULLABLE.STRING; + return Types.StringType.get(); case DATE_TYPE_NAME: - return TypeCreator.NULLABLE.DATE; + return Types.DateType.get(); case TIMESTAMP_TYPE_NAME: - return TypeCreator.NULLABLE.TIMESTAMP; - case DECIMAL_TYPE_NAME: - DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo) hiveTypeInfo; - return TypeCreator.NULLABLE.decimal( - decimalTypeInfo.precision(), decimalTypeInfo.getScale()); - case CHAR_TYPE_NAME: - return TypeCreator.NULLABLE.fixedChar(((CharTypeInfo) hiveTypeInfo).getLength()); - case VARCHAR_TYPE_NAME: - return TypeCreator.NULLABLE.varChar(((VarcharTypeInfo) hiveTypeInfo).getLength()); + return Types.TimestampType.withoutTimeZone(); + case BINARY_TYPE_NAME: + return Types.BinaryType.get(); + case INTERVAL_YEAR_MONTH_TYPE_NAME: + return Types.IntervalYearType.get(); + case INTERVAL_DAY_TIME_TYPE_NAME: + return Types.IntervalDayType.get(); default: + if (hiveTypeInfo instanceof CharTypeInfo) { + return Types.FixedCharType.of(((CharTypeInfo) hiveTypeInfo).getLength()); + } + + if (hiveTypeInfo instanceof VarcharTypeInfo) { + return Types.VarCharType.of(((VarcharTypeInfo) hiveTypeInfo).getLength()); + } + + if (hiveTypeInfo instanceof DecimalTypeInfo) { + DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo) hiveTypeInfo; + return Types.DecimalType.of(decimalTypeInfo.precision(), decimalTypeInfo.scale()); + } + throw new IllegalArgumentException( "Unknown Hive type: " + hiveTypeInfo.getQualifiedName()); } case LIST: - return TypeCreator.NULLABLE.list( - toSubstraitType(((ListTypeInfo) hiveTypeInfo).getListElementTypeInfo())); + return Types.ListType.nullable( + toGravitinoType(((ListTypeInfo) hiveTypeInfo).getListElementTypeInfo())); case MAP: MapTypeInfo mapTypeInfo = (MapTypeInfo) hiveTypeInfo; - return TypeCreator.NULLABLE.map( - toSubstraitType(mapTypeInfo.getMapKeyTypeInfo()), - toSubstraitType(mapTypeInfo.getMapKeyTypeInfo())); + return Types.MapType.valueNullable( + toGravitinoType(mapTypeInfo.getMapKeyTypeInfo()), + toGravitinoType(mapTypeInfo.getMapValueTypeInfo())); case STRUCT: - return TypeCreator.NULLABLE.struct( - ((StructTypeInfo) hiveTypeInfo) - .getAllStructFieldTypeInfos().stream() - .map(FromHiveType::toSubstraitType) - .collect(Collectors.toList())); + StructTypeInfo structTypeInfo = (StructTypeInfo) hiveTypeInfo; + ArrayList fieldNames = structTypeInfo.getAllStructFieldNames(); + ArrayList typeInfos = structTypeInfo.getAllStructFieldTypeInfos(); + Types.StructType.Field[] fields = + IntStream.range(0, fieldNames.size()) + .mapToObj( + i -> + Types.StructType.Field.nullableField( + fieldNames.get(i), toGravitinoType(typeInfos.get(i)))) + .toArray(Types.StructType.Field[]::new); + return Types.StructType.of(fields); case UNION: - return TypeCreator.NULLABLE.struct( - ((UnionTypeInfo) hiveTypeInfo) - .getAllUnionObjectTypeInfos().stream() - .map(FromHiveType::toSubstraitType) - .collect(Collectors.toList())); + UnionTypeInfo unionTypeInfo = (UnionTypeInfo) hiveTypeInfo; + return Types.UnionType.of( + unionTypeInfo.getAllUnionObjectTypeInfos().stream() + .map(FromHiveType::toGravitinoType) + .toArray(Type[]::new)); default: throw new IllegalArgumentException( "Unknown category of Hive type: " + hiveTypeInfo.getCategory()); diff --git a/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/converter/ToHiveType.java b/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/converter/ToHiveType.java index ff03ee2c962..c0606f53eaf 100644 --- a/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/converter/ToHiveType.java +++ b/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/converter/ToHiveType.java @@ -23,126 +23,76 @@ import static org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory.getMapTypeInfo; import static org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory.getPrimitiveTypeInfo; import static org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory.getStructTypeInfo; +import static org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory.getUnionTypeInfo; import static org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory.getVarcharTypeInfo; -import io.substrait.function.ParameterizedTypeVisitor; -import io.substrait.type.Type; +import com.datastrato.gravitino.rel.types.Type; +import com.datastrato.gravitino.rel.types.Types; +import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; -import java.util.stream.IntStream; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; -/** Converts Substrait data types to corresponding Hive data types. */ -public class ToHiveType - extends ParameterizedTypeVisitor.ParameterizedTypeThrowsVisitor { - - public static ToHiveType INSTANCE = new ToHiveType(); - - private ToHiveType() { - super("Only support type literals and parameterized types."); - } - - // Visit methods for each Substrait data type - @Override - public TypeInfo visit(Type.Bool type) throws RuntimeException { - return getPrimitiveTypeInfo(BOOLEAN_TYPE_NAME); - } - - @Override - public TypeInfo visit(Type.I8 type) throws RuntimeException { - return getPrimitiveTypeInfo(TINYINT_TYPE_NAME); - } - - @Override - public TypeInfo visit(Type.I16 type) throws RuntimeException { - return getPrimitiveTypeInfo(SMALLINT_TYPE_NAME); - } - - @Override - public TypeInfo visit(Type.I32 type) throws RuntimeException { - return getPrimitiveTypeInfo(INT_TYPE_NAME); - } - - @Override - public TypeInfo visit(Type.I64 type) throws RuntimeException { - return getPrimitiveTypeInfo(BIGINT_TYPE_NAME); - } - - @Override - public TypeInfo visit(Type.FP32 type) throws RuntimeException { - return getPrimitiveTypeInfo(FLOAT_TYPE_NAME); - } - - @Override - public TypeInfo visit(Type.FP64 type) throws RuntimeException { - return getPrimitiveTypeInfo(DOUBLE_TYPE_NAME); - } - - @Override - public TypeInfo visit(Type.Str type) throws RuntimeException { - return getPrimitiveTypeInfo(STRING_TYPE_NAME); - } - - @Override - public TypeInfo visit(Type.VarChar type) throws RuntimeException { - return getVarcharTypeInfo(type.length()); - } - - @Override - public TypeInfo visit(Type.FixedChar type) throws RuntimeException { - return getCharTypeInfo(type.length()); - } - - @Override - public TypeInfo visit(Type.Date type) throws RuntimeException { - return getPrimitiveTypeInfo(DATE_TYPE_NAME); - } - - @Override - public TypeInfo visit(Type.Timestamp type) throws RuntimeException { - return getPrimitiveTypeInfo(TIMESTAMP_TYPE_NAME); - } - - @Override - public TypeInfo visit(Type.Decimal type) throws RuntimeException { - return getDecimalTypeInfo(type.precision(), type.scale()); - } - - @Override - public TypeInfo visit(Type.Binary type) throws RuntimeException { - return getPrimitiveTypeInfo(BINARY_TYPE_NAME); - } - - @Override - public TypeInfo visit(Type.IntervalYear type) throws RuntimeException { - return getPrimitiveTypeInfo(INTERVAL_YEAR_MONTH_TYPE_NAME); - } - - @Override - public TypeInfo visit(Type.IntervalDay type) throws RuntimeException { - return getPrimitiveTypeInfo(INTERVAL_DAY_TIME_TYPE_NAME); - } - - @Override - public TypeInfo visit(Type.ListType type) throws RuntimeException { - return getListTypeInfo(type.elementType().accept(INSTANCE)); - } - - @Override - public TypeInfo visit(Type.Map type) throws RuntimeException { - return getMapTypeInfo(type.key().accept(INSTANCE), type.value().accept(INSTANCE)); - } - - @Override - public TypeInfo visit(Type.Struct type) throws RuntimeException { - List typeInfos = - type.fields().stream().map(t -> t.accept(INSTANCE)).collect(Collectors.toList()); - List names = - IntStream.range(0, typeInfos.size()).mapToObj(String::valueOf).collect(Collectors.toList()); - // TODO: Actually, Hive's Struct type should correspond to Substrait's NamedStruct type. - // However, NamedStruct is a Pseudo-type, not an implementation of io.substrait.type.Type. - // I haven't figured out a good way to resolve this yet. For now, I'm using an index instead of - // name as a temporary workaround. - return getStructTypeInfo(names, typeInfos); +/** Converts Gravitino data types to corresponding Hive data types. */ +public class ToHiveType { + public static TypeInfo convert(Type type) { + switch (type.name()) { + case BOOLEAN: + return getPrimitiveTypeInfo(BOOLEAN_TYPE_NAME); + case BYTE: + return getPrimitiveTypeInfo(TINYINT_TYPE_NAME); + case SHORT: + return getPrimitiveTypeInfo(SMALLINT_TYPE_NAME); + case INTEGER: + return getPrimitiveTypeInfo(INT_TYPE_NAME); + case LONG: + return getPrimitiveTypeInfo(BIGINT_TYPE_NAME); + case FLOAT: + return getPrimitiveTypeInfo(FLOAT_TYPE_NAME); + case DOUBLE: + return getPrimitiveTypeInfo(DOUBLE_TYPE_NAME); + case STRING: + return getPrimitiveTypeInfo(STRING_TYPE_NAME); + case VARCHAR: + return getVarcharTypeInfo(((Types.VarCharType) type).length()); + case FIXEDCHAR: + return getCharTypeInfo(((Types.FixedCharType) type).length()); + case DATE: + return getPrimitiveTypeInfo(DATE_TYPE_NAME); + case TIMESTAMP: + return getPrimitiveTypeInfo(TIMESTAMP_TYPE_NAME); + case DECIMAL: + Types.DecimalType decimalType = (Types.DecimalType) type; + return getDecimalTypeInfo(decimalType.precision(), decimalType.scale()); + case BINARY: + return getPrimitiveTypeInfo(BINARY_TYPE_NAME); + case INTERVAL_YEAR: + return getPrimitiveTypeInfo(INTERVAL_YEAR_MONTH_TYPE_NAME); + case INTERVAL_DAY: + return getPrimitiveTypeInfo(INTERVAL_DAY_TIME_TYPE_NAME); + case LIST: + return getListTypeInfo(convert(((Types.ListType) type).elementType())); + case MAP: + Types.MapType mapType = (Types.MapType) type; + return getMapTypeInfo(convert(mapType.keyType()), convert(mapType.valueType())); + case STRUCT: + Types.StructType structType = (Types.StructType) type; + List typeInfos = + Arrays.stream(structType.fields()) + .map(t -> convert(t.type())) + .collect(Collectors.toList()); + List names = + Arrays.stream(structType.fields()) + .map(Types.StructType.Field::name) + .collect(Collectors.toList()); + return getStructTypeInfo(names, typeInfos); + case UNION: + return getUnionTypeInfo( + Arrays.stream(((Types.UnionType) type).types()) + .map(ToHiveType::convert) + .collect(Collectors.toList())); + default: + throw new UnsupportedOperationException("Unsupported conversion to Hive type: " + type); + } } } diff --git a/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/converter/TestTypeConverter.java b/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/converter/TestTypeConverter.java new file mode 100644 index 00000000000..05c34eb3374 --- /dev/null +++ b/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/converter/TestTypeConverter.java @@ -0,0 +1,79 @@ +/* + * Copyright 2023 Datastrato. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.catalog.converter; + +import static org.apache.hadoop.hive.serde.serdeConstants.BIGINT_TYPE_NAME; +import static org.apache.hadoop.hive.serde.serdeConstants.BINARY_TYPE_NAME; +import static org.apache.hadoop.hive.serde.serdeConstants.BOOLEAN_TYPE_NAME; +import static org.apache.hadoop.hive.serde.serdeConstants.DATE_TYPE_NAME; +import static org.apache.hadoop.hive.serde.serdeConstants.DOUBLE_TYPE_NAME; +import static org.apache.hadoop.hive.serde.serdeConstants.FLOAT_TYPE_NAME; +import static org.apache.hadoop.hive.serde.serdeConstants.INTERVAL_DAY_TIME_TYPE_NAME; +import static org.apache.hadoop.hive.serde.serdeConstants.INTERVAL_YEAR_MONTH_TYPE_NAME; +import static org.apache.hadoop.hive.serde.serdeConstants.INT_TYPE_NAME; +import static org.apache.hadoop.hive.serde.serdeConstants.SMALLINT_TYPE_NAME; +import static org.apache.hadoop.hive.serde.serdeConstants.STRING_TYPE_NAME; +import static org.apache.hadoop.hive.serde.serdeConstants.TIMESTAMP_TYPE_NAME; +import static org.apache.hadoop.hive.serde.serdeConstants.TINYINT_TYPE_NAME; +import static org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory.getCharTypeInfo; +import static org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory.getDecimalTypeInfo; +import static org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory.getListTypeInfo; +import static org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory.getMapTypeInfo; +import static org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory.getPrimitiveTypeInfo; +import static org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory.getStructTypeInfo; +import static org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory.getUnionTypeInfo; +import static org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory.getVarcharTypeInfo; +import static org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils.getTypeInfoFromTypeString; + +import com.datastrato.gravitino.catalog.hive.converter.FromHiveType; +import com.datastrato.gravitino.catalog.hive.converter.ToHiveType; +import java.util.Arrays; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class TestTypeConverter { + @Test + public void testTypeConverter() { + testConverter(BOOLEAN_TYPE_NAME); + testConverter(TINYINT_TYPE_NAME); + testConverter(SMALLINT_TYPE_NAME); + testConverter(INT_TYPE_NAME); + testConverter(BIGINT_TYPE_NAME); + testConverter(FLOAT_TYPE_NAME); + testConverter(DOUBLE_TYPE_NAME); + testConverter(STRING_TYPE_NAME); + testConverter(DATE_TYPE_NAME); + testConverter(TIMESTAMP_TYPE_NAME); + testConverter(BINARY_TYPE_NAME); + testConverter(INTERVAL_YEAR_MONTH_TYPE_NAME); + testConverter(INTERVAL_DAY_TIME_TYPE_NAME); + testConverter(getVarcharTypeInfo(10).getTypeName()); + testConverter(getCharTypeInfo(20).getTypeName()); + testConverter(getDecimalTypeInfo(10, 2).getTypeName()); + + testConverter(getListTypeInfo(getPrimitiveTypeInfo(STRING_TYPE_NAME)).getTypeName()); + testConverter( + getMapTypeInfo(getPrimitiveTypeInfo(INT_TYPE_NAME), getPrimitiveTypeInfo(DATE_TYPE_NAME)) + .getTypeName()); + testConverter( + getStructTypeInfo( + Arrays.asList("a", "b"), + Arrays.asList( + getPrimitiveTypeInfo(STRING_TYPE_NAME), getPrimitiveTypeInfo(INT_TYPE_NAME))) + .getTypeName()); + testConverter( + getUnionTypeInfo( + Arrays.asList( + getPrimitiveTypeInfo(STRING_TYPE_NAME), getPrimitiveTypeInfo(INT_TYPE_NAME))) + .getTypeName()); + } + + private void testConverter(String typeName) { + TypeInfo hiveType = getTypeInfoFromTypeString(typeName); + TypeInfo convertedType = ToHiveType.convert(FromHiveType.convert(hiveType.getTypeName())); + Assertions.assertEquals(hiveType, convertedType); + } +} diff --git a/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/TestHiveTable.java b/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/TestHiveTable.java index 401af0eef65..af057520b5a 100644 --- a/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/TestHiveTable.java +++ b/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/TestHiveTable.java @@ -27,8 +27,8 @@ import com.datastrato.gravitino.rel.expressions.sorts.SortOrder; import com.datastrato.gravitino.rel.expressions.sorts.SortOrders; import com.datastrato.gravitino.rel.expressions.transforms.Transform; +import com.datastrato.gravitino.rel.types.Types; import com.google.common.collect.Maps; -import io.substrait.type.TypeCreator; import java.time.Instant; import java.util.Arrays; import java.util.Map; @@ -127,13 +127,13 @@ public void testCreateHiveTable() { HiveColumn col1 = new HiveColumn.Builder() .withName("col_1") - .withType(TypeCreator.NULLABLE.I8) + .withType(Types.ByteType.get()) .withComment(HIVE_COMMENT) .build(); HiveColumn col2 = new HiveColumn.Builder() .withName("col_2") - .withType(TypeCreator.NULLABLE.DATE) + .withType(Types.DateType.get()) .withComment(HIVE_COMMENT) .build(); Column[] columns = new Column[] {col1, col2}; @@ -201,7 +201,7 @@ public void testCreateHiveTable() { HiveColumn illegalColumn = new HiveColumn.Builder() .withName("col_3") - .withType(TypeCreator.REQUIRED.I8) + .withType(Types.ByteType.get()) .withComment(HIVE_COMMENT) .withNullable(false) .build(); @@ -239,13 +239,13 @@ public void testCreatePartitionedHiveTable() { HiveColumn col1 = new HiveColumn.Builder() .withName("city") - .withType(TypeCreator.NULLABLE.I8) + .withType(Types.ByteType.get()) .withComment(HIVE_COMMENT) .build(); HiveColumn col2 = new HiveColumn.Builder() .withName("dt") - .withType(TypeCreator.NULLABLE.DATE) + .withType(Types.DateType.get()) .withComment(HIVE_COMMENT) .build(); Column[] columns = new Column[] {col1, col2}; @@ -337,13 +337,13 @@ public void testDropHiveTable() { HiveColumn col1 = new HiveColumn.Builder() .withName("col_1") - .withType(TypeCreator.NULLABLE.I8) + .withType(Types.ByteType.get()) .withComment(HIVE_COMMENT) .build(); HiveColumn col2 = new HiveColumn.Builder() .withName("col_2") - .withType(TypeCreator.NULLABLE.DATE) + .withType(Types.DateType.get()) .withComment(HIVE_COMMENT) .build(); Column[] columns = new Column[] {col1, col2}; @@ -385,13 +385,13 @@ public void testAlterHiveTable() { HiveColumn col1 = new HiveColumn.Builder() .withName("col_1") - .withType(TypeCreator.NULLABLE.I8) + .withType(Types.ByteType.get()) .withComment(HIVE_COMMENT) .build(); HiveColumn col2 = new HiveColumn.Builder() .withName("col_2") - .withType(TypeCreator.NULLABLE.DATE) + .withType(Types.DateType.get()) .withComment(HIVE_COMMENT) .build(); Column[] columns = new Column[] {col1, col2}; @@ -460,7 +460,7 @@ public void testAlterHiveTable() { // .alterTable( // tableIdentifier, // TableChange.addColumn(new String[] {"col_3"}, - // TypeCreator.REQUIRED.I8))); + // Types.ByteType.get()))); // Assertions.assertTrue( // exception // .getMessage() @@ -476,7 +476,7 @@ public void testAlterHiveTable() { .asTableCatalog() .alterTable( tableIdentifier, - TableChange.addColumn(new String[] {"col_1"}, TypeCreator.NULLABLE.I8))); + TableChange.addColumn(new String[] {"col_1"}, Types.ByteType.get()))); Assertions.assertTrue( exception.getMessage().contains("Cannot add column after partition column")); @@ -490,7 +490,7 @@ public void testAlterHiveTable() { tableIdentifier, TableChange.addColumn( new String[] {col1.name()}, - TypeCreator.NULLABLE.I8, + Types.ByteType.get(), "comment", TableChange.ColumnPosition.after(col1.name())))); Assertions.assertTrue(exception.getMessage().contains("Cannot add column with duplicate name")); @@ -507,7 +507,7 @@ public void testAlterHiveTable() { // columns current format: [col_1:I8:comment, col_2:DATE:comment] TableChange.addColumn( new String[] {"col_3"}, - TypeCreator.NULLABLE.STRING, + Types.StringType.get(), null, TableChange.ColumnPosition.after(col1.name())), // columns current format: [col_1:I8:comment, col_3:STRING:null, col_2:DATE:comment] @@ -516,7 +516,7 @@ public void testAlterHiveTable() { TableChange.updateColumnComment(new String[] {"col_1"}, HIVE_COMMENT + "_new"), // columns current format: [col_1:I8:comment_new, col_3_new:STRING:null, // col_2:DATE:comment] - TableChange.updateColumnType(new String[] {"col_1"}, TypeCreator.NULLABLE.I32), + TableChange.updateColumnType(new String[] {"col_1"}, Types.IntegerType.get()), // columns current format: [col_1:I32:comment_new, col_3_new:STRING:null, // col_2:DATE:comment] TableChange.updateColumnPosition( @@ -542,17 +542,17 @@ public void testAlterHiveTable() { new Column[] { new HiveColumn.Builder() .withName("col_3_new") - .withType(TypeCreator.NULLABLE.STRING) + .withType(Types.StringType.get()) .withComment(null) .build(), new HiveColumn.Builder() .withName("col_1") - .withType(TypeCreator.NULLABLE.I32) + .withType(Types.IntegerType.get()) .withComment(HIVE_COMMENT + "_new") .build(), new HiveColumn.Builder() .withName("col_2") - .withType(TypeCreator.NULLABLE.DATE) + .withType(Types.DateType.get()) .withComment(HIVE_COMMENT) .build() }; diff --git a/catalogs/catalog-jdbc-common/build.gradle.kts b/catalogs/catalog-jdbc-common/build.gradle.kts index b10623396eb..9000b0f3e92 100644 --- a/catalogs/catalog-jdbc-common/build.gradle.kts +++ b/catalogs/catalog-jdbc-common/build.gradle.kts @@ -24,14 +24,6 @@ dependencies { implementation(libs.commons.lang3) implementation(libs.commons.collections4) implementation(libs.commons.dbcp2) - implementation(libs.substrait.java.core) { - exclude("com.fasterxml.jackson.core") - exclude("com.fasterxml.jackson.datatype") - exclude("com.fasterxml.jackson.dataformat") - exclude("com.google.protobuf") - exclude("com.google.code.findbugs") - exclude("org.slf4j") - } testImplementation(libs.commons.io) testImplementation(libs.sqlite.jdbc) diff --git a/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/converter/JdbcTypeConverter.java b/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/converter/JdbcTypeConverter.java index 8406624fffa..6572c049f96 100644 --- a/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/converter/JdbcTypeConverter.java +++ b/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/converter/JdbcTypeConverter.java @@ -4,7 +4,7 @@ */ package com.datastrato.gravitino.catalog.jdbc.converter; -import io.substrait.type.Type; +import com.datastrato.gravitino.rel.types.Type; public abstract class JdbcTypeConverter { diff --git a/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/converter/SqliteTypeConverter.java b/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/converter/SqliteTypeConverter.java index 4438300d87b..ce3367a19fc 100644 --- a/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/converter/SqliteTypeConverter.java +++ b/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/converter/SqliteTypeConverter.java @@ -4,8 +4,8 @@ */ package com.datastrato.gravitino.catalog.jdbc.converter; -import io.substrait.type.Type; -import io.substrait.type.TypeCreator; +import com.datastrato.gravitino.rel.types.Type; +import com.datastrato.gravitino.rel.types.Types; import java.util.Collection; import java.util.HashMap; import java.util.Map; @@ -16,10 +16,10 @@ public class SqliteTypeConverter extends JdbcTypeConverter { protected static final Map GRAVITINO_TO_SQLITE_MAPPING = new HashMap<>(); static { - GRAVITINO_TO_SQLITE_MAPPING.put(TypeCreator.NULLABLE.I8, "TINYINT"); - GRAVITINO_TO_SQLITE_MAPPING.put(TypeCreator.NULLABLE.I32, "INTEGER"); - GRAVITINO_TO_SQLITE_MAPPING.put(TypeCreator.NULLABLE.STRING, "TEXT"); - GRAVITINO_TO_SQLITE_MAPPING.put(TypeCreator.NULLABLE.BINARY, "BLOB"); + GRAVITINO_TO_SQLITE_MAPPING.put(Types.ByteType.get(), "TINYINT"); + GRAVITINO_TO_SQLITE_MAPPING.put(Types.IntegerType.get(), "INTEGER"); + GRAVITINO_TO_SQLITE_MAPPING.put(Types.StringType.get(), "TEXT"); + GRAVITINO_TO_SQLITE_MAPPING.put(Types.BinaryType.get(), "BLOB"); } @Override diff --git a/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/operation/TestJdbcTableOperations.java b/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/operation/TestJdbcTableOperations.java index c048e957412..c58c5091fa6 100644 --- a/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/operation/TestJdbcTableOperations.java +++ b/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/operation/TestJdbcTableOperations.java @@ -15,9 +15,9 @@ import com.datastrato.gravitino.exceptions.NoSuchTableException; import com.datastrato.gravitino.rel.Column; import com.datastrato.gravitino.rel.TableChange; +import com.datastrato.gravitino.rel.types.Type; +import com.datastrato.gravitino.rel.types.Types; import com.google.common.collect.Maps; -import io.substrait.type.Type; -import io.substrait.type.TypeCreator; import java.io.File; import java.io.IOException; import java.nio.file.Files; @@ -98,7 +98,7 @@ public void testOperationTable() { new JdbcColumn.Builder() .withName("col_a") .withNullable(true) - .withType(TypeCreator.NULLABLE.I32) + .withType(Types.IntegerType.get()) .withComment(null) .withDefaultValue(null) .build(); @@ -106,7 +106,7 @@ public void testOperationTable() { new JdbcColumn.Builder() .withName("col_b") .withNullable(false) - .withType(TypeCreator.NULLABLE.STRING) + .withType(Types.StringType.get()) .withComment(null) .withDefaultValue(null) .build(); @@ -163,8 +163,7 @@ public void testOperationTable() { JDBC_TABLE_OPERATIONS.alterTable( DATABASE_NAME, newName, - TableChange.updateColumnType( - new String[] {col_a.name()}, TypeCreator.NULLABLE.STRING))); + TableChange.updateColumnType(new String[] {col_a.name()}, Types.StringType.get()))); // delete table. JDBC_TABLE_OPERATIONS.drop(DATABASE_NAME, newName); diff --git a/catalogs/catalog-lakehouse-iceberg/build.gradle.kts b/catalogs/catalog-lakehouse-iceberg/build.gradle.kts index d2ea3101913..ae2ff75bc5b 100644 --- a/catalogs/catalog-lakehouse-iceberg/build.gradle.kts +++ b/catalogs/catalog-lakehouse-iceberg/build.gradle.kts @@ -29,14 +29,6 @@ dependencies { implementation(libs.commons.lang3) implementation(libs.commons.io) implementation(libs.commons.collections4) - implementation(libs.substrait.java.core) { - exclude("com.fasterxml.jackson.core") - exclude("com.fasterxml.jackson.datatype") - exclude("com.fasterxml.jackson.dataformat") - exclude("com.google.protobuf") - exclude("com.google.code.findbugs") - exclude("org.slf4j") - } implementation(libs.iceberg.hive.metastore) implementation(libs.sqlite.jdbc) diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/ConvertUtil.java b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/ConvertUtil.java index 60124b9711e..e0fa5259fe7 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/ConvertUtil.java +++ b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/ConvertUtil.java @@ -31,7 +31,8 @@ public static Schema toIcebergSchema(IcebergTable icebergTable) { * @param gravitinoType Gravitino type. * @return Iceberg type. */ - public static Type toIcebergType(boolean nullable, io.substrait.type.Type gravitinoType) { + public static Type toIcebergType( + boolean nullable, com.datastrato.gravitino.rel.types.Type gravitinoType) { return ToIcebergTypeVisitor.visit(gravitinoType, new ToIcebergType(nullable)); } @@ -41,7 +42,7 @@ public static Type toIcebergType(boolean nullable, io.substrait.type.Type gravit * @param type Iceberg type of field. * @return */ - public static io.substrait.type.Type formIcebergType(Type type) { + public static com.datastrato.gravitino.rel.types.Type formIcebergType(Type type) { return TypeUtil.visit(type, new FromIcebergType()); } diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/FromIcebergType.java b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/FromIcebergType.java index f829c337b89..2adf6bae643 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/FromIcebergType.java +++ b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/FromIcebergType.java @@ -4,8 +4,7 @@ */ package com.datastrato.gravitino.catalog.lakehouse.iceberg.converter; -import io.substrait.type.Type; -import io.substrait.type.TypeCreator; +import com.datastrato.gravitino.rel.types.Type; import java.util.List; import org.apache.iceberg.Schema; import org.apache.iceberg.types.TypeUtil; @@ -37,58 +36,53 @@ public Type field(Types.NestedField field, Type fieldResult) { @Override public Type list(Types.ListType list, Type elementResult) { - if (list.isElementOptional()) { - return TypeCreator.NULLABLE.list(elementResult); - } else { - return TypeCreator.REQUIRED.list(elementResult); - } + return com.datastrato.gravitino.rel.types.Types.ListType.of( + elementResult, list.isElementOptional()); } @Override public Type map(Types.MapType map, Type keyResult, Type valueResult) { - if (map.isValueOptional()) { - return TypeCreator.NULLABLE.map(keyResult, valueResult); - } else { - return TypeCreator.REQUIRED.map(keyResult, valueResult); - } + return com.datastrato.gravitino.rel.types.Types.MapType.of( + keyResult, valueResult, map.isValueOptional()); } @Override public Type primitive(org.apache.iceberg.types.Type.PrimitiveType primitive) { switch (primitive.typeId()) { case BOOLEAN: - return TypeCreator.NULLABLE.BOOLEAN; + return com.datastrato.gravitino.rel.types.Types.BooleanType.get(); case INTEGER: - return TypeCreator.NULLABLE.I32; + return com.datastrato.gravitino.rel.types.Types.IntegerType.get(); case LONG: - return TypeCreator.NULLABLE.I64; + return com.datastrato.gravitino.rel.types.Types.LongType.get(); case FLOAT: - return TypeCreator.NULLABLE.FP32; + return com.datastrato.gravitino.rel.types.Types.FloatType.get(); case DOUBLE: - return TypeCreator.NULLABLE.FP64; + return com.datastrato.gravitino.rel.types.Types.DoubleType.get(); case DATE: - return TypeCreator.NULLABLE.DATE; + return com.datastrato.gravitino.rel.types.Types.DateType.get(); case TIME: - return TypeCreator.NULLABLE.TIME; + return com.datastrato.gravitino.rel.types.Types.TimeType.get(); case TIMESTAMP: Types.TimestampType ts = (Types.TimestampType) primitive; if (ts.shouldAdjustToUTC()) { - return TypeCreator.NULLABLE.TIMESTAMP_TZ; + return com.datastrato.gravitino.rel.types.Types.TimestampType.withoutTimeZone(); } else { - return TypeCreator.NULLABLE.TIMESTAMP; + return com.datastrato.gravitino.rel.types.Types.TimestampType.withTimeZone(); } case STRING: - return TypeCreator.NULLABLE.STRING; + return com.datastrato.gravitino.rel.types.Types.StringType.get(); case UUID: - return TypeCreator.NULLABLE.UUID; + return com.datastrato.gravitino.rel.types.Types.UUIDType.get(); case FIXED: Types.FixedType fixedType = (Types.FixedType) primitive; - return TypeCreator.NULLABLE.fixedChar(fixedType.length()); + return com.datastrato.gravitino.rel.types.Types.FixedType.of(fixedType.length()); case BINARY: - return TypeCreator.NULLABLE.BINARY; + return com.datastrato.gravitino.rel.types.Types.BinaryType.get(); case DECIMAL: Types.DecimalType decimal = (Types.DecimalType) primitive; - return TypeCreator.NULLABLE.decimal(decimal.precision(), decimal.scale()); + return com.datastrato.gravitino.rel.types.Types.DecimalType.of( + decimal.precision(), decimal.scale()); default: throw new UnsupportedOperationException( "Cannot convert unknown type to Gravitino: " + primitive); diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/ToIcebergType.java b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/ToIcebergType.java index 0a03943f269..0b7e89c840b 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/ToIcebergType.java +++ b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/ToIcebergType.java @@ -68,7 +68,7 @@ public Type field(IcebergColumn field, Type typeResult) { } @Override - public Type array(io.substrait.type.Type.ListType array, Type elementType) { + public Type array(com.datastrato.gravitino.rel.types.Types.ListType array, Type elementType) { if (nullable) { return Types.ListType.ofOptional(getNextId(), elementType); } else { @@ -77,7 +77,8 @@ public Type array(io.substrait.type.Type.ListType array, Type elementType) { } @Override - public Type map(io.substrait.type.Type.Map map, Type keyType, Type valueType) { + public Type map( + com.datastrato.gravitino.rel.types.Types.MapType map, Type keyType, Type valueType) { if (nullable) { return Types.MapType.ofOptional(getNextId(), getNextId(), keyType, valueType); } else { @@ -86,39 +87,41 @@ public Type map(io.substrait.type.Type.Map map, Type keyType, Type valueType) { } @Override - public Type atomic(io.substrait.type.Type primitive) { - if (primitive instanceof io.substrait.type.Type.Bool) { + public Type atomic(com.datastrato.gravitino.rel.types.Type.PrimitiveType primitive) { + if (primitive instanceof com.datastrato.gravitino.rel.types.Types.BooleanType) { return Types.BooleanType.get(); - } else if (primitive instanceof io.substrait.type.Type.I8 - || primitive instanceof io.substrait.type.Type.I16 - || primitive instanceof io.substrait.type.Type.I32) { + } else if (primitive instanceof com.datastrato.gravitino.rel.types.Types.ByteType + || primitive instanceof com.datastrato.gravitino.rel.types.Types.ShortType + || primitive instanceof com.datastrato.gravitino.rel.types.Types.IntegerType) { return Types.IntegerType.get(); - } else if (primitive instanceof io.substrait.type.Type.I64) { + } else if (primitive instanceof com.datastrato.gravitino.rel.types.Types.LongType) { return Types.LongType.get(); - } else if (primitive instanceof io.substrait.type.Type.FP32) { + } else if (primitive instanceof com.datastrato.gravitino.rel.types.Types.FloatType) { return Types.FloatType.get(); - } else if (primitive instanceof io.substrait.type.Type.FP64) { + } else if (primitive instanceof com.datastrato.gravitino.rel.types.Types.DoubleType) { return Types.DoubleType.get(); - } else if (primitive instanceof io.substrait.type.Type.Str - || primitive instanceof io.substrait.type.Type.VarChar) { + } else if (primitive instanceof com.datastrato.gravitino.rel.types.Types.StringType) { return Types.StringType.get(); - } else if (primitive instanceof io.substrait.type.Type.Date) { + } else if (primitive instanceof com.datastrato.gravitino.rel.types.Types.DateType) { return Types.DateType.get(); - } else if (primitive instanceof io.substrait.type.Type.Time) { + } else if (primitive instanceof com.datastrato.gravitino.rel.types.Types.TimeType) { return Types.TimeType.get(); - } else if (primitive instanceof io.substrait.type.Type.TimestampTZ) { - return Types.TimestampType.withZone(); - } else if (primitive instanceof io.substrait.type.Type.Timestamp) { - return Types.TimestampType.withoutZone(); - } else if (primitive instanceof io.substrait.type.Type.Decimal) { + } else if (primitive instanceof com.datastrato.gravitino.rel.types.Types.TimestampType) { + if (((com.datastrato.gravitino.rel.types.Types.TimestampType) primitive).hasTimeZone()) { + return Types.TimestampType.withZone(); + } else { + return Types.TimestampType.withoutZone(); + } + } else if (primitive instanceof com.datastrato.gravitino.rel.types.Types.DecimalType) { return Types.DecimalType.of( - ((io.substrait.type.Type.Decimal) primitive).precision(), - ((io.substrait.type.Type.Decimal) primitive).scale()); - } else if (primitive instanceof io.substrait.type.Type.FixedChar) { - return Types.FixedType.ofLength(((io.substrait.type.Type.FixedChar) primitive).length()); - } else if (primitive instanceof io.substrait.type.Type.Binary) { + ((com.datastrato.gravitino.rel.types.Types.DecimalType) primitive).precision(), + ((com.datastrato.gravitino.rel.types.Types.DecimalType) primitive).scale()); + } else if (primitive instanceof com.datastrato.gravitino.rel.types.Types.FixedType) { + return Types.FixedType.ofLength( + ((com.datastrato.gravitino.rel.types.Types.FixedType) primitive).length()); + } else if (primitive instanceof com.datastrato.gravitino.rel.types.Types.BinaryType) { return Types.BinaryType.get(); - } else if (primitive instanceof io.substrait.type.Type.UUID) { + } else if (primitive instanceof com.datastrato.gravitino.rel.types.Types.UUIDType) { return Types.UUIDType.get(); } throw new UnsupportedOperationException("Not a supported type: " + primitive.toString()); diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/ToIcebergTypeVisitor.java b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/ToIcebergTypeVisitor.java index 6a6a6bffcfc..97edffc9bb8 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/ToIcebergTypeVisitor.java +++ b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/ToIcebergTypeVisitor.java @@ -7,8 +7,9 @@ import com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergColumn; import com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergTable; import com.datastrato.gravitino.rel.Column; +import com.datastrato.gravitino.rel.types.Type; +import com.datastrato.gravitino.rel.types.Types; import com.google.common.collect.Lists; -import io.substrait.type.Type; import java.util.List; /** @@ -45,14 +46,14 @@ public static T visit(IcebergTable table, ToIcebergTypeVisitor visitor) { * @param */ public static T visit(Type type, ToIcebergTypeVisitor visitor) { - if (type instanceof Type.Map) { - Type.Map map = (Type.Map) type; - return visitor.map(map, visit(map.key(), visitor), visit(map.value(), visitor)); - } else if (type instanceof Type.ListType) { - Type.ListType list = (Type.ListType) type; + if (type instanceof Types.MapType) { + Types.MapType map = (Types.MapType) type; + return visitor.map(map, visit(map.keyType(), visitor), visit(map.valueType(), visitor)); + } else if (type instanceof Types.ListType) { + Types.ListType list = (Types.ListType) type; return visitor.array(list, visit(list.elementType(), visitor)); } else { - return visitor.atomic(type); + return visitor.atomic((Type.PrimitiveType) type); } } @@ -64,15 +65,15 @@ public T field(IcebergColumn field, T typeResult) { throw new UnsupportedOperationException(); } - public T array(Type.ListType array, T elementResult) { + public T array(Types.ListType array, T elementResult) { throw new UnsupportedOperationException(); } - public T map(Type.Map map, T keyResult, T valueResult) { + public T map(Types.MapType map, T keyResult, T valueResult) { throw new UnsupportedOperationException(); } - public T atomic(Type primitive) { + public T atomic(Type.PrimitiveType primitive) { throw new UnsupportedOperationException(); } } diff --git a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/TestIcebergTable.java b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/TestIcebergTable.java index ba6cbdfe575..acd9ba68d34 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/TestIcebergTable.java +++ b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/TestIcebergTable.java @@ -27,8 +27,8 @@ import com.datastrato.gravitino.rel.expressions.sorts.SortOrder; import com.datastrato.gravitino.rel.expressions.sorts.SortOrders; import com.datastrato.gravitino.rel.expressions.transforms.Transform; +import com.datastrato.gravitino.rel.types.Types; import com.google.common.collect.Maps; -import io.substrait.type.TypeCreator; import java.io.IOException; import java.time.Instant; import java.util.Arrays; @@ -128,14 +128,14 @@ public void testCreateIcebergTable() throws IOException { IcebergColumn col1 = new IcebergColumn.Builder() .withName("col_1") - .withType(TypeCreator.NULLABLE.I8) + .withType(Types.ByteType.get()) .withComment(ICEBERG_COMMENT) .withNullable(true) .build(); IcebergColumn col2 = new IcebergColumn.Builder() .withName("col_2") - .withType(TypeCreator.NULLABLE.DATE) + .withType(Types.DateType.get()) .withComment(ICEBERG_COMMENT) .withNullable(false) .build(); @@ -208,13 +208,13 @@ public void testCreatePartitionedIcebergTable() { IcebergColumn col1 = new IcebergColumn.Builder() .withName("city") - .withType(TypeCreator.NULLABLE.I8) + .withType(Types.ByteType.get()) .withComment(ICEBERG_COMMENT) .build(); IcebergColumn col2 = new IcebergColumn.Builder() .withName("date") - .withType(TypeCreator.NULLABLE.DATE) + .withType(Types.DateType.get()) .withComment(ICEBERG_COMMENT) .build(); Column[] columns = new Column[] {col1, col2}; @@ -310,13 +310,13 @@ public void testDropIcebergTable() { IcebergColumn col1 = new IcebergColumn.Builder() .withName("col_1") - .withType(TypeCreator.NULLABLE.I8) + .withType(Types.ByteType.get()) .withComment(ICEBERG_COMMENT) .build(); IcebergColumn col2 = new IcebergColumn.Builder() .withName("col_2") - .withType(TypeCreator.NULLABLE.DATE) + .withType(Types.DateType.get()) .withComment(ICEBERG_COMMENT) .build(); Column[] columns = new Column[] {col1, col2}; @@ -359,13 +359,13 @@ public void testAlterIcebergTable() { IcebergColumn col1 = new IcebergColumn.Builder() .withName("col_1") - .withType(TypeCreator.NULLABLE.I8) + .withType(Types.ByteType.get()) .withComment(ICEBERG_COMMENT) .build(); IcebergColumn col2 = new IcebergColumn.Builder() .withName("col_2") - .withType(TypeCreator.NULLABLE.DATE) + .withType(Types.DateType.get()) .withComment(ICEBERG_COMMENT) .build(); Column[] columns = new Column[] {col1, col2}; @@ -408,14 +408,14 @@ public void testAlterIcebergTable() { TableChange.removeProperty("key1"), TableChange.setProperty("key2", "val2_new"), // columns current format: [col_1:I8:comment, col_2:DATE:comment] - TableChange.addColumn(new String[] {"col_3"}, TypeCreator.NULLABLE.STRING), + TableChange.addColumn(new String[] {"col_3"}, Types.StringType.get()), // columns current format: [col_1:I8:comment, col_2:DATE:comment, col_3:STRING:null] TableChange.renameColumn(new String[] {"col_2"}, "col_2_new"), // columns current format: [col_1:I8:comment, col_2_new:DATE:comment, col_3:STRING:null] TableChange.updateColumnComment(new String[] {"col_1"}, ICEBERG_COMMENT + "_new"), // columns current format: [col_1:I8:comment_new, col_2_new:DATE:comment, // col_3:STRING:null] - TableChange.updateColumnType(new String[] {"col_1"}, TypeCreator.NULLABLE.I32), + TableChange.updateColumnType(new String[] {"col_1"}, Types.IntegerType.get()), // columns current format: [col_1:I32:comment_new, col_2_new:DATE:comment, // col_3:STRING:null] TableChange.updateColumnPosition( @@ -442,17 +442,17 @@ public void testAlterIcebergTable() { new Column[] { new IcebergColumn.Builder() .withName("col_2_new") - .withType(TypeCreator.NULLABLE.DATE) + .withType(Types.DateType.get()) .withComment(ICEBERG_COMMENT) .build(), new IcebergColumn.Builder() .withName("col_1") - .withType(TypeCreator.NULLABLE.I32) + .withType(Types.IntegerType.get()) .withComment(ICEBERG_COMMENT + "_new") .build(), new IcebergColumn.Builder() .withName("col_3") - .withType(TypeCreator.NULLABLE.STRING) + .withType(Types.StringType.get()) .withComment(null) .build() }; diff --git a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/TestBaseConvert.java b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/TestBaseConvert.java index 2de80e20216..d1da592c3ab 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/TestBaseConvert.java +++ b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/TestBaseConvert.java @@ -13,9 +13,8 @@ import com.datastrato.gravitino.rel.expressions.sorts.SortDirection; import com.datastrato.gravitino.rel.expressions.sorts.SortOrder; import com.datastrato.gravitino.rel.expressions.sorts.SortOrders; +import com.datastrato.gravitino.rel.types.Type; import com.google.common.collect.Lists; -import io.substrait.type.Type; -import io.substrait.type.TypeCreator; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -35,21 +34,22 @@ public class TestBaseConvert { protected static final Map ICEBERG_TYPE = new HashMap<>(); static { - GRAVITINO_TYPE.put("BOOLEAN", TypeCreator.NULLABLE.BOOLEAN); - // Types not supported by iceberg - // GRAVITINO_TYPE.put("I8", TypeCreator.NULLABLE.I8); - // GRAVITINO_TYPE.put("I16", TypeCreator.NULLABLE.I16); - GRAVITINO_TYPE.put("I32", TypeCreator.NULLABLE.I32); - GRAVITINO_TYPE.put("I64", TypeCreator.NULLABLE.I64); - GRAVITINO_TYPE.put("FP32", TypeCreator.NULLABLE.FP32); - GRAVITINO_TYPE.put("FP64", TypeCreator.NULLABLE.FP64); - GRAVITINO_TYPE.put("STRING", TypeCreator.NULLABLE.STRING); - GRAVITINO_TYPE.put("BINARY", TypeCreator.NULLABLE.BINARY); - GRAVITINO_TYPE.put("TIMESTAMP", TypeCreator.NULLABLE.TIMESTAMP); - GRAVITINO_TYPE.put("TIMESTAMP_TZ", TypeCreator.NULLABLE.TIMESTAMP_TZ); - GRAVITINO_TYPE.put("DATE", TypeCreator.NULLABLE.DATE); - GRAVITINO_TYPE.put("TIME", TypeCreator.NULLABLE.TIME); - GRAVITINO_TYPE.put("UUID", TypeCreator.NULLABLE.UUID); + GRAVITINO_TYPE.put("BOOLEAN", com.datastrato.gravitino.rel.types.Types.BooleanType.get()); + GRAVITINO_TYPE.put("I8", com.datastrato.gravitino.rel.types.Types.ByteType.get()); + GRAVITINO_TYPE.put("I16", com.datastrato.gravitino.rel.types.Types.ShortType.get()); + GRAVITINO_TYPE.put("I32", com.datastrato.gravitino.rel.types.Types.IntegerType.get()); + GRAVITINO_TYPE.put("I64", com.datastrato.gravitino.rel.types.Types.LongType.get()); + GRAVITINO_TYPE.put("FP32", com.datastrato.gravitino.rel.types.Types.FloatType.get()); + GRAVITINO_TYPE.put("FP64", com.datastrato.gravitino.rel.types.Types.DoubleType.get()); + GRAVITINO_TYPE.put("STRING", com.datastrato.gravitino.rel.types.Types.StringType.get()); + GRAVITINO_TYPE.put("BINARY", com.datastrato.gravitino.rel.types.Types.BinaryType.get()); + GRAVITINO_TYPE.put( + "TIMESTAMP", com.datastrato.gravitino.rel.types.Types.TimestampType.withoutTimeZone()); + GRAVITINO_TYPE.put( + "TIMESTAMP_TZ", com.datastrato.gravitino.rel.types.Types.TimestampType.withTimeZone()); + GRAVITINO_TYPE.put("DATE", com.datastrato.gravitino.rel.types.Types.DateType.get()); + GRAVITINO_TYPE.put("TIME", com.datastrato.gravitino.rel.types.Types.TimeType.get()); + GRAVITINO_TYPE.put("UUID", com.datastrato.gravitino.rel.types.Types.UUIDType.get()); // Types not supported by iceberg // ICEBERG_TYPE.put("INTERVAL_DAY", TypeCreator.NULLABLE.INTERVAL_DAY); // ICEBERG_TYPE.put("INTERVAL_YEAR", TypeCreator.NULLABLE.INTERVAL_YEAR); diff --git a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/TestConvertUtil.java b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/TestConvertUtil.java index 35ade5abd7e..cfc9991555c 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/TestConvertUtil.java +++ b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/TestConvertUtil.java @@ -10,7 +10,6 @@ import com.datastrato.gravitino.rel.Column; import com.datastrato.gravitino.rel.expressions.sorts.SortOrder; import com.google.common.collect.Maps; -import io.substrait.type.TypeCreator; import java.time.Instant; import java.util.Arrays; import java.util.List; @@ -34,10 +33,12 @@ public void testToIcebergSchema() { new IcebergColumn.Builder() .withName(col5Name) .withType( - TypeCreator.NULLABLE.map( - TypeCreator.NULLABLE.list(TypeCreator.NULLABLE.TIMESTAMP_TZ), - TypeCreator.NULLABLE.map( - TypeCreator.NULLABLE.STRING, TypeCreator.NULLABLE.DATE))) + com.datastrato.gravitino.rel.types.Types.MapType.valueNullable( + com.datastrato.gravitino.rel.types.Types.ListType.nullable( + com.datastrato.gravitino.rel.types.Types.TimestampType.withTimeZone()), + com.datastrato.gravitino.rel.types.Types.MapType.valueNullable( + com.datastrato.gravitino.rel.types.Types.StringType.get(), + com.datastrato.gravitino.rel.types.Types.DateType.get()))) .withComment(TEST_COMMENT) .build(); columns = ArrayUtils.add(columns, col5); @@ -84,49 +85,71 @@ public void testToIcebergSchema() { @Test public void testToPrimitiveType() { Assertions.assertTrue( - ConvertUtil.toIcebergType(true, TypeCreator.NULLABLE.BOOLEAN) instanceof Types.BooleanType); + ConvertUtil.toIcebergType(true, com.datastrato.gravitino.rel.types.Types.BooleanType.get()) + instanceof Types.BooleanType); Assertions.assertTrue( - ConvertUtil.toIcebergType(true, TypeCreator.NULLABLE.STRING) instanceof Types.StringType); + ConvertUtil.toIcebergType(true, com.datastrato.gravitino.rel.types.Types.StringType.get()) + instanceof Types.StringType); Assertions.assertTrue( - ConvertUtil.toIcebergType(true, TypeCreator.NULLABLE.I8) instanceof Types.IntegerType); + ConvertUtil.toIcebergType(true, com.datastrato.gravitino.rel.types.Types.ByteType.get()) + instanceof Types.IntegerType); Assertions.assertTrue( - ConvertUtil.toIcebergType(true, TypeCreator.NULLABLE.I16) instanceof Types.IntegerType); + ConvertUtil.toIcebergType(true, com.datastrato.gravitino.rel.types.Types.ShortType.get()) + instanceof Types.IntegerType); Assertions.assertTrue( - ConvertUtil.toIcebergType(true, TypeCreator.NULLABLE.I32) instanceof Types.IntegerType); + ConvertUtil.toIcebergType(true, com.datastrato.gravitino.rel.types.Types.IntegerType.get()) + instanceof Types.IntegerType); Assertions.assertTrue( - ConvertUtil.toIcebergType(true, TypeCreator.NULLABLE.I64) instanceof Types.LongType); + ConvertUtil.toIcebergType(true, com.datastrato.gravitino.rel.types.Types.LongType.get()) + instanceof Types.LongType); Assertions.assertTrue( - ConvertUtil.toIcebergType(true, TypeCreator.NULLABLE.FP32) instanceof Types.FloatType); + ConvertUtil.toIcebergType(true, com.datastrato.gravitino.rel.types.Types.FloatType.get()) + instanceof Types.FloatType); Assertions.assertTrue( - ConvertUtil.toIcebergType(true, TypeCreator.NULLABLE.FP64) instanceof Types.DoubleType); + ConvertUtil.toIcebergType(true, com.datastrato.gravitino.rel.types.Types.DoubleType.get()) + instanceof Types.DoubleType); Assertions.assertTrue( - ConvertUtil.toIcebergType(true, TypeCreator.NULLABLE.DATE) instanceof Types.DateType); + ConvertUtil.toIcebergType(true, com.datastrato.gravitino.rel.types.Types.DateType.get()) + instanceof Types.DateType); Assertions.assertTrue( - ConvertUtil.toIcebergType(true, TypeCreator.NULLABLE.TIME) instanceof Types.TimeType); + ConvertUtil.toIcebergType(true, com.datastrato.gravitino.rel.types.Types.TimeType.get()) + instanceof Types.TimeType); Assertions.assertTrue( - ConvertUtil.toIcebergType(true, TypeCreator.NULLABLE.BINARY) instanceof Types.BinaryType); + ConvertUtil.toIcebergType(true, com.datastrato.gravitino.rel.types.Types.BinaryType.get()) + instanceof Types.BinaryType); Assertions.assertTrue( - ConvertUtil.toIcebergType(true, TypeCreator.NULLABLE.UUID) instanceof Types.UUIDType); + ConvertUtil.toIcebergType(true, com.datastrato.gravitino.rel.types.Types.UUIDType.get()) + instanceof Types.UUIDType); - Type timestampTZ = ConvertUtil.toIcebergType(true, TypeCreator.NULLABLE.TIMESTAMP_TZ); + Type timestampTZ = + ConvertUtil.toIcebergType( + true, com.datastrato.gravitino.rel.types.Types.TimestampType.withTimeZone()); Assertions.assertTrue(timestampTZ instanceof Types.TimestampType); Assertions.assertTrue(((Types.TimestampType) timestampTZ).shouldAdjustToUTC()); - Type timestamp = ConvertUtil.toIcebergType(true, TypeCreator.NULLABLE.TIMESTAMP); + Type timestamp = + ConvertUtil.toIcebergType( + true, com.datastrato.gravitino.rel.types.Types.TimestampType.withoutTimeZone()); Assertions.assertTrue(timestamp instanceof Types.TimestampType); Assertions.assertFalse(((Types.TimestampType) timestamp).shouldAdjustToUTC()); - Type decimalType = ConvertUtil.toIcebergType(true, TypeCreator.NULLABLE.decimal(9, 2)); + Type decimalType = + ConvertUtil.toIcebergType( + true, com.datastrato.gravitino.rel.types.Types.DecimalType.of(9, 2)); Assertions.assertTrue(decimalType instanceof Types.DecimalType); Assertions.assertEquals(((Types.DecimalType) decimalType).precision(), 9); Assertions.assertEquals(((Types.DecimalType) decimalType).scale(), 2); - Type fixedCharType = ConvertUtil.toIcebergType(true, TypeCreator.NULLABLE.fixedChar(9)); + Type fixedCharType = + ConvertUtil.toIcebergType(true, com.datastrato.gravitino.rel.types.Types.FixedType.of(9)); Assertions.assertTrue(fixedCharType instanceof Types.FixedType); Assertions.assertEquals(((Types.FixedType) fixedCharType).length(), 9); - io.substrait.type.Type mapType = - TypeCreator.NULLABLE.map(TypeCreator.NULLABLE.STRING, TypeCreator.NULLABLE.I8); + com.datastrato.gravitino.rel.types.Type mapType = + com.datastrato.gravitino.rel.types.Types.MapType.of( + com.datastrato.gravitino.rel.types.Types.StringType.get(), + com.datastrato.gravitino.rel.types.Types.IntegerType.get(), + true); Type convertedMapType = ConvertUtil.toIcebergType(true, mapType); Assertions.assertTrue(convertedMapType instanceof Types.MapType); Assertions.assertTrue(((Types.MapType) convertedMapType).keyType() instanceof Types.StringType); @@ -134,65 +157,85 @@ public void testToPrimitiveType() { ((Types.MapType) convertedMapType).valueType() instanceof Types.IntegerType); Type listType = - ConvertUtil.toIcebergType(true, TypeCreator.NULLABLE.list(TypeCreator.NULLABLE.FP64)); + ConvertUtil.toIcebergType( + true, + com.datastrato.gravitino.rel.types.Types.ListType.of( + com.datastrato.gravitino.rel.types.Types.FloatType.get(), true)); Assertions.assertTrue(listType instanceof Types.ListType); - Assertions.assertTrue(listType.asListType().elementType() instanceof Types.DoubleType); + Assertions.assertTrue(listType.asListType().elementType() instanceof Types.FloatType); } @Test public void testFormIcebergType() { Assertions.assertTrue( - ConvertUtil.formIcebergType(Types.StringType.get()) instanceof io.substrait.type.Type.Str); + ConvertUtil.formIcebergType(Types.StringType.get()) + instanceof com.datastrato.gravitino.rel.types.Types.StringType); Assertions.assertTrue( ConvertUtil.formIcebergType(Types.BinaryType.get()) - instanceof io.substrait.type.Type.Binary); + instanceof com.datastrato.gravitino.rel.types.Types.BinaryType); Assertions.assertTrue( - ConvertUtil.formIcebergType(Types.TimeType.get()) instanceof io.substrait.type.Type.Time); + ConvertUtil.formIcebergType(Types.TimeType.get()) + instanceof com.datastrato.gravitino.rel.types.Types.TimeType); Assertions.assertTrue( ConvertUtil.formIcebergType(Types.TimestampType.withoutZone()) - instanceof io.substrait.type.Type.Timestamp); + instanceof com.datastrato.gravitino.rel.types.Types.TimestampType); Assertions.assertTrue( ConvertUtil.formIcebergType(Types.TimestampType.withZone()) - instanceof io.substrait.type.Type.TimestampTZ); + instanceof com.datastrato.gravitino.rel.types.Types.TimestampType); Assertions.assertTrue( - ConvertUtil.formIcebergType(Types.DoubleType.get()) instanceof io.substrait.type.Type.FP64); + ConvertUtil.formIcebergType(Types.DoubleType.get()) + instanceof com.datastrato.gravitino.rel.types.Types.DoubleType); Assertions.assertTrue( - ConvertUtil.formIcebergType(Types.FloatType.get()) instanceof io.substrait.type.Type.FP32); + ConvertUtil.formIcebergType(Types.FloatType.get()) + instanceof com.datastrato.gravitino.rel.types.Types.FloatType); Assertions.assertTrue( - ConvertUtil.formIcebergType(Types.IntegerType.get()) instanceof io.substrait.type.Type.I32); + ConvertUtil.formIcebergType(Types.IntegerType.get()) + instanceof com.datastrato.gravitino.rel.types.Types.IntegerType); Assertions.assertTrue( - ConvertUtil.formIcebergType(Types.LongType.get()) instanceof io.substrait.type.Type.I64); + ConvertUtil.formIcebergType(Types.LongType.get()) + instanceof com.datastrato.gravitino.rel.types.Types.LongType); Assertions.assertTrue( - ConvertUtil.formIcebergType(Types.UUIDType.get()) instanceof io.substrait.type.Type.UUID); + ConvertUtil.formIcebergType(Types.UUIDType.get()) + instanceof com.datastrato.gravitino.rel.types.Types.UUIDType); Assertions.assertTrue( - ConvertUtil.formIcebergType(Types.DateType.get()) instanceof io.substrait.type.Type.Date); + ConvertUtil.formIcebergType(Types.DateType.get()) + instanceof com.datastrato.gravitino.rel.types.Types.DateType); - io.substrait.type.Type decimalType = ConvertUtil.formIcebergType(Types.DecimalType.of(9, 2)); - Assertions.assertTrue(decimalType instanceof io.substrait.type.Type.Decimal); - Assertions.assertEquals(((io.substrait.type.Type.Decimal) decimalType).precision(), 9); - Assertions.assertEquals(((io.substrait.type.Type.Decimal) decimalType).scale(), 2); + com.datastrato.gravitino.rel.types.Type decimalType = + ConvertUtil.formIcebergType(Types.DecimalType.of(9, 2)); + Assertions.assertTrue( + decimalType instanceof com.datastrato.gravitino.rel.types.Types.DecimalType); + Assertions.assertEquals( + ((com.datastrato.gravitino.rel.types.Types.DecimalType) decimalType).precision(), 9); + Assertions.assertEquals( + ((com.datastrato.gravitino.rel.types.Types.DecimalType) decimalType).scale(), 2); - io.substrait.type.Type fixedType = ConvertUtil.formIcebergType(Types.FixedType.ofLength(2)); - Assertions.assertTrue(fixedType instanceof io.substrait.type.Type.FixedChar); - Assertions.assertEquals(((io.substrait.type.Type.FixedChar) fixedType).length(), 2); + com.datastrato.gravitino.rel.types.Type fixedType = + ConvertUtil.formIcebergType(Types.FixedType.ofLength(2)); + Assertions.assertTrue(fixedType instanceof com.datastrato.gravitino.rel.types.Types.FixedType); + Assertions.assertEquals( + ((com.datastrato.gravitino.rel.types.Types.FixedType) fixedType).length(), 2); Types.MapType mapType = Types.MapType.ofOptional(1, 2, Types.StringType.get(), Types.IntegerType.get()); - io.substrait.type.Type gravitinoMapType = ConvertUtil.formIcebergType(mapType); - Assertions.assertTrue(gravitinoMapType instanceof io.substrait.type.Type.Map); + com.datastrato.gravitino.rel.types.Type gravitinoMapType = ConvertUtil.formIcebergType(mapType); + Assertions.assertTrue( + gravitinoMapType instanceof com.datastrato.gravitino.rel.types.Types.MapType); Assertions.assertTrue( - ((io.substrait.type.Type.Map) gravitinoMapType).key() - instanceof io.substrait.type.Type.Str); + ((com.datastrato.gravitino.rel.types.Types.MapType) gravitinoMapType).keyType() + instanceof com.datastrato.gravitino.rel.types.Types.StringType); Assertions.assertTrue( - ((io.substrait.type.Type.Map) gravitinoMapType).value() - instanceof io.substrait.type.Type.I32); + ((com.datastrato.gravitino.rel.types.Types.MapType) gravitinoMapType).valueType() + instanceof com.datastrato.gravitino.rel.types.Types.IntegerType); Types.ListType listType = Types.ListType.ofOptional(1, Types.StringType.get()); - io.substrait.type.Type gravitinoListType = ConvertUtil.formIcebergType(listType); - Assertions.assertTrue(gravitinoListType instanceof io.substrait.type.Type.ListType); + com.datastrato.gravitino.rel.types.Type gravitinoListType = + ConvertUtil.formIcebergType(listType); Assertions.assertTrue( - ((io.substrait.type.Type.ListType) gravitinoListType).elementType() - instanceof io.substrait.type.Type.Str); + gravitinoListType instanceof com.datastrato.gravitino.rel.types.Types.ListType); + Assertions.assertTrue( + ((com.datastrato.gravitino.rel.types.Types.ListType) gravitinoListType).elementType() + instanceof com.datastrato.gravitino.rel.types.Types.StringType); } @Test @@ -205,7 +248,8 @@ public void testFromNestedField() { Assertions.assertEquals(icebergColumn.name(), colName); Assertions.assertTrue(icebergColumn.nullable()); Assertions.assertEquals(icebergColumn.comment(), doc); - Assertions.assertTrue(icebergColumn.dataType() instanceof io.substrait.type.Type.I32); + Assertions.assertTrue( + icebergColumn.dataType() instanceof com.datastrato.gravitino.rel.types.Types.IntegerType); colName = RandomStringUtils.randomAlphabetic(10); doc = RandomStringUtils.randomAlphabetic(20); @@ -214,7 +258,8 @@ public void testFromNestedField() { Assertions.assertEquals(icebergColumn.name(), colName); Assertions.assertFalse(icebergColumn.nullable()); Assertions.assertEquals(icebergColumn.comment(), doc); - Assertions.assertTrue(icebergColumn.dataType() instanceof io.substrait.type.Type.Str); + Assertions.assertTrue( + icebergColumn.dataType() instanceof com.datastrato.gravitino.rel.types.Types.StringType); colName = RandomStringUtils.randomAlphabetic(10); doc = RandomStringUtils.randomAlphabetic(20); @@ -223,7 +268,8 @@ public void testFromNestedField() { Assertions.assertEquals(icebergColumn.name(), colName); Assertions.assertFalse(icebergColumn.nullable()); Assertions.assertEquals(icebergColumn.comment(), doc); - Assertions.assertTrue(icebergColumn.dataType() instanceof io.substrait.type.Type.Date); + Assertions.assertTrue( + icebergColumn.dataType() instanceof com.datastrato.gravitino.rel.types.Types.DateType); colName = RandomStringUtils.randomAlphabetic(10); doc = RandomStringUtils.randomAlphabetic(20); @@ -235,34 +281,37 @@ public void testFromNestedField() { 1, 2, Types.ListType.ofOptional(3, Types.StringType.get()), - Types.DecimalType.of(1, 13)), + Types.DecimalType.of(13, 1)), doc); icebergColumn = ConvertUtil.fromNestedField(colField); Assertions.assertEquals(icebergColumn.name(), colName); Assertions.assertFalse(icebergColumn.nullable()); Assertions.assertEquals(icebergColumn.comment(), doc); - Assertions.assertTrue(icebergColumn.dataType() instanceof io.substrait.type.Type.Map); + Assertions.assertTrue( + icebergColumn.dataType() instanceof com.datastrato.gravitino.rel.types.Types.MapType); } - private static void checkType(Type type, io.substrait.type.Type expected) { - io.substrait.type.Type actual = ConvertUtil.formIcebergType(type); + private static void checkType(Type type, com.datastrato.gravitino.rel.types.Type expected) { + com.datastrato.gravitino.rel.types.Type actual = ConvertUtil.formIcebergType(type); checkType(actual, expected); } - private static void checkType(io.substrait.type.Type actual, io.substrait.type.Type expected) { - if (actual instanceof io.substrait.type.Type.Map) { - Assertions.assertTrue(expected instanceof io.substrait.type.Type.Map); + private static void checkType( + com.datastrato.gravitino.rel.types.Type actual, + com.datastrato.gravitino.rel.types.Type expected) { + if (actual instanceof com.datastrato.gravitino.rel.types.Types.MapType) { + Assertions.assertTrue(expected instanceof com.datastrato.gravitino.rel.types.Types.MapType); checkType( - ((io.substrait.type.Type.Map) actual).key(), - ((io.substrait.type.Type.Map) expected).key()); + ((com.datastrato.gravitino.rel.types.Types.MapType) actual).keyType(), + ((com.datastrato.gravitino.rel.types.Types.MapType) expected).keyType()); checkType( - ((io.substrait.type.Type.Map) actual).value(), - ((io.substrait.type.Type.Map) expected).value()); - } else if (actual instanceof io.substrait.type.Type.ListType) { - Assertions.assertTrue(expected instanceof io.substrait.type.Type.ListType); + ((com.datastrato.gravitino.rel.types.Types.MapType) actual).valueType(), + ((com.datastrato.gravitino.rel.types.Types.MapType) expected).valueType()); + } else if (actual instanceof com.datastrato.gravitino.rel.types.Types.ListType) { + Assertions.assertTrue(expected instanceof com.datastrato.gravitino.rel.types.Types.ListType); checkType( - ((io.substrait.type.Type.ListType) actual).elementType(), - ((io.substrait.type.Type.ListType) expected).elementType()); + ((com.datastrato.gravitino.rel.types.Types.ListType) actual).elementType(), + ((com.datastrato.gravitino.rel.types.Types.ListType) expected).elementType()); } else { Assertions.assertEquals(expected.getClass(), actual.getClass()); } diff --git a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/ops/TestIcebergTableUpdate.java b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/ops/TestIcebergTableUpdate.java index 62adc9f1e5b..fead0088c11 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/ops/TestIcebergTableUpdate.java +++ b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/ops/TestIcebergTableUpdate.java @@ -8,8 +8,7 @@ import com.datastrato.gravitino.catalog.lakehouse.iceberg.ops.IcebergTableOpsHelper.IcebergTableChange; import com.datastrato.gravitino.rel.TableChange; import com.datastrato.gravitino.rel.TableChange.ColumnPosition; -import io.substrait.type.Type.I32; -import io.substrait.type.Type.I64; +import com.datastrato.gravitino.rel.types.Types; import java.util.List; import java.util.stream.Collectors; import org.apache.iceberg.Schema; @@ -158,7 +157,7 @@ public void testAddColumn() { TableChange addColumn = TableChange.addColumn( new String[] {addColumnNameAfter}, - I32.builder().nullable(true).build(), + Types.IntegerType.get(), "", ColumnPosition.after(firstField[0])); LoadTableResponse loadTableResponse = updateTable(identifier, addColumn); @@ -169,19 +168,14 @@ public void testAddColumn() { String addColumnNameFirst = "add_column_first"; addColumn = TableChange.addColumn( - new String[] {addColumnNameFirst}, - I32.builder().nullable(true).build(), - "", - ColumnPosition.first()); + new String[] {addColumnNameFirst}, Types.IntegerType.get(), "", ColumnPosition.first()); loadTableResponse = updateTable(identifier, addColumn); columns = getColumnNames(loadTableResponse); Assertions.assertEquals(columns.get(0), addColumnNameFirst); // add to last String addColumnNameLast = "add_column_last"; - addColumn = - TableChange.addColumn( - new String[] {addColumnNameLast}, I32.builder().nullable(true).build()); + addColumn = TableChange.addColumn(new String[] {addColumnNameLast}, Types.IntegerType.get()); loadTableResponse = updateTable(identifier, addColumn); columns = getColumnNames(loadTableResponse); Assertions.assertEquals(columns.get(columns.size() - 1), addColumnNameLast); @@ -190,7 +184,7 @@ public void testAddColumn() { addColumn = TableChange.addColumn( new String[] {fourthField[0], "element", "struct_after"}, - I32.builder().nullable(true).build(), + Types.IntegerType.get(), "", ColumnPosition.after("struct_int")); loadTableResponse = updateTable(identifier, addColumn); @@ -206,7 +200,7 @@ public void testAddColumn() { addColumn = TableChange.addColumn( new String[] {fourthField[0], "element", "struct_first"}, - I32.builder().nullable(true).build(), + Types.IntegerType.get(), "", ColumnPosition.first()); loadTableResponse = updateTable(identifier, addColumn); @@ -221,8 +215,7 @@ public void testAddColumn() { // add to struct last addColumn = TableChange.addColumn( - new String[] {fourthField[0], "element", "struct_last"}, - I32.builder().nullable(true).build()); + new String[] {fourthField[0], "element", "struct_last"}, Types.IntegerType.get()); loadTableResponse = updateTable(identifier, addColumn); t = (StructType) @@ -236,8 +229,7 @@ public void testAddColumn() { Assertions.assertThrowsExactly( IllegalArgumentException.class, () -> { - TableChange addColumn1 = - TableChange.addColumn(firstField, I32.builder().nullable(true).build(), ""); + TableChange addColumn1 = TableChange.addColumn(firstField, Types.IntegerType.get(), ""); updateTable(identifier, addColumn1); }); @@ -247,10 +239,7 @@ public void testAddColumn() { () -> { TableChange addColumn1 = TableChange.addColumn( - firstField, - I32.builder().nullable(true).build(), - "", - ColumnPosition.after("not_exits")); + firstField, Types.IntegerType.get(), "", ColumnPosition.after("not_exits")); updateTable(identifier, addColumn1); }); } @@ -291,8 +280,7 @@ public void testUpdateColumnComment() { @Test public void testUpdateColumnType() { - TableChange updateColumnType = - TableChange.updateColumnType(thirdField, I64.builder().nullable(true).build()); + TableChange updateColumnType = TableChange.updateColumnType(thirdField, Types.LongType.get()); LoadTableResponse loadTableResponse = updateTable(identifier, updateColumnType); Assertions.assertEquals( LongType.get(), loadTableResponse.tableMetadata().schema().columns().get(2).type()); @@ -300,8 +288,7 @@ public void testUpdateColumnType() { // update struct_int from int to long updateColumnType = TableChange.updateColumnType( - new String[] {fourthField[0], "element", "struct_int"}, - I64.builder().nullable(true).build()); + new String[] {fourthField[0], "element", "struct_int"}, Types.LongType.get()); loadTableResponse = updateTable(identifier, updateColumnType); StructType t = (StructType) @@ -312,7 +299,7 @@ public void testUpdateColumnType() { Assertions.assertEquals(LongType.get(), t.fields().get(0).type()); TableChange updateColumnType2 = - TableChange.updateColumnType(notExistField, I32.builder().nullable(true).build()); + TableChange.updateColumnType(notExistField, Types.IntegerType.get()); Assertions.assertThrowsExactly( IllegalArgumentException.class, () -> updateTable(identifier, updateColumnType2)); } diff --git a/clients/client-java-runtime/build.gradle.kts b/clients/client-java-runtime/build.gradle.kts index 7ddacbd303a..22d5585ce4b 100644 --- a/clients/client-java-runtime/build.gradle.kts +++ b/clients/client-java-runtime/build.gradle.kts @@ -20,14 +20,6 @@ tasks.withType(ShadowJar::class.java) { archiveClassifier.set("") // Relocate dependencies to avoid conflicts - relocate("io.substrait", "com.datastrato.gravitino.shaded.io.substrait") { - exclude("org.slf4j") - exclude("com.fasterxml.jackson.core") - exclude("com.fasterxml.jackson.datatype") - exclude("com.fasterxml.jackson.dataformat") - exclude("com.google.code.findbugs") - exclude("com.google.protobuf") - } relocate("com.google", "com.datastrato.gravitino.shaded.com.google") relocate("com.fasterxml", "com.datastrato.gravitino.shaded.com.fasterxml") relocate("org.apache.httpcomponents", "com.datastrato.gravitino.shaded.org.apache.httpcomponents") diff --git a/clients/client-java/build.gradle.kts b/clients/client-java/build.gradle.kts index acdef958146..0609e73ea29 100644 --- a/clients/client-java/build.gradle.kts +++ b/clients/client-java/build.gradle.kts @@ -16,14 +16,6 @@ dependencies { exclude("com.google.guava", "guava") .because("Brings in Guava for Andriod, which we don't want (and breaks multimaps).") } - implementation(libs.substrait.java.core) { - exclude("org.slf4j") - exclude("com.fasterxml.jackson.core") - exclude("com.fasterxml.jackson.datatype") - exclude("com.fasterxml.jackson.dataformat") - exclude("com.google.code.findbugs") - exclude("com.google.protobuf") - } implementation(libs.jackson.databind) implementation(libs.jackson.annotations) implementation(libs.jackson.datatype.jdk8) diff --git a/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestRelationalCatalog.java b/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestRelationalCatalog.java index 10979954624..aa30a2815da 100644 --- a/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestRelationalCatalog.java +++ b/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestRelationalCatalog.java @@ -53,11 +53,11 @@ import com.datastrato.gravitino.rel.expressions.distributions.Strategy; import com.datastrato.gravitino.rel.expressions.sorts.SortDirection; import com.datastrato.gravitino.rel.expressions.sorts.SortOrder; +import com.datastrato.gravitino.rel.types.Type; +import com.datastrato.gravitino.rel.types.Types; import com.fasterxml.jackson.core.JsonProcessingException; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import io.substrait.type.Type; -import io.substrait.type.TypeCreator; import java.time.Instant; import java.util.Arrays; import java.util.Collections; @@ -335,8 +335,8 @@ public void testCreateTable() throws JsonProcessingException { ColumnDTO[] columns = new ColumnDTO[] { - createMockColumn("col1", TypeCreator.NULLABLE.I8, "comment1"), - createMockColumn("col2", TypeCreator.NULLABLE.STRING, "comment2") + createMockColumn("col1", Types.ByteType.get(), "comment1"), + createMockColumn("col2", Types.StringType.get(), "comment2") }; SortOrderDTO[] sortOrderDTOs = createMockSortOrderDTO("col2", DESCENDING); @@ -431,8 +431,8 @@ public void testCreatePartitionedTable() throws JsonProcessingException { ColumnDTO[] columns = new ColumnDTO[] { - createMockColumn("city", TypeCreator.NULLABLE.I32, "comment1"), - createMockColumn("dt", TypeCreator.NULLABLE.DATE, "comment2") + createMockColumn("city", Types.IntegerType.get(), "comment1"), + createMockColumn("dt", Types.DateType.get(), "comment2") }; // Test empty partitioning @@ -559,8 +559,8 @@ public void testLoadTable() throws JsonProcessingException { RelationalCatalog.formatTableRequestPath(tableId.namespace()) + "/" + tableId.name()); ColumnDTO[] columns = new ColumnDTO[] { - createMockColumn("col1", TypeCreator.NULLABLE.I8, "comment1"), - createMockColumn("col2", TypeCreator.NULLABLE.STRING, "comment2") + createMockColumn("col1", Types.ByteType.get(), "comment1"), + createMockColumn("col2", Types.StringType.get(), "comment2") }; DistributionDTO distributionDTO = createMockDistributionDTO("col1", 10); @@ -597,7 +597,7 @@ public void testLoadTable() throws JsonProcessingException { public void testRenameTable() throws JsonProcessingException { NameIdentifier tableId = NameIdentifier.of(metalakeName, catalogName, "schema1", "table1"); ColumnDTO[] columns = - new ColumnDTO[] {createMockColumn("col1", TypeCreator.NULLABLE.I8, "comment1")}; + new ColumnDTO[] {createMockColumn("col1", Types.ByteType.get(), "comment1")}; DistributionDTO distributionDTO = createMockDistributionDTO("col1", 10); SortOrderDTO[] sortOrderDTOs = createMockSortOrderDTO("col1", DESCENDING); @@ -621,7 +621,7 @@ public void testRenameTable() throws JsonProcessingException { public void testUpdateTableComment() throws JsonProcessingException { NameIdentifier tableId = NameIdentifier.of(metalakeName, catalogName, "schema1", "table1"); ColumnDTO[] columns = - new ColumnDTO[] {createMockColumn("col1", TypeCreator.NULLABLE.I8, "comment1")}; + new ColumnDTO[] {createMockColumn("col1", Types.ByteType.get(), "comment1")}; DistributionDTO distributionDTO = createMockDistributionDTO("col1", 10); SortOrderDTO[] sortOrderDTOs = createMockSortOrderDTO("col1", DESCENDING); @@ -645,7 +645,7 @@ public void testUpdateTableComment() throws JsonProcessingException { public void testSetTableProperty() throws JsonProcessingException { NameIdentifier tableId = NameIdentifier.of(metalakeName, catalogName, "schema1", "table1"); ColumnDTO[] columns = - new ColumnDTO[] {createMockColumn("col1", TypeCreator.NULLABLE.I8, "comment1")}; + new ColumnDTO[] {createMockColumn("col1", Types.ByteType.get(), "comment1")}; Map properties = ImmutableMap.of("k1", "v1"); DistributionDTO distributionDTO = createMockDistributionDTO("col1", 10); @@ -670,7 +670,7 @@ public void testSetTableProperty() throws JsonProcessingException { public void testRemoveTableProperty() throws JsonProcessingException { NameIdentifier tableId = NameIdentifier.of(metalakeName, catalogName, "schema1", "table1"); ColumnDTO[] columns = - new ColumnDTO[] {createMockColumn("col1", TypeCreator.NULLABLE.I8, "comment1")}; + new ColumnDTO[] {createMockColumn("col1", Types.ByteType.get(), "comment1")}; DistributionDTO distributionDTO = createMockDistributionDTO("col1", 10); SortOrderDTO[] sortOrderDTOs = createMockSortOrderDTO("col1", DESCENDING); @@ -694,8 +694,8 @@ public void testAddTableColumn() throws JsonProcessingException { NameIdentifier tableId = NameIdentifier.of(metalakeName, catalogName, "schema1", "table1"); ColumnDTO[] columns = new ColumnDTO[] { - createMockColumn("col1", TypeCreator.NULLABLE.I8, "comment1"), - createMockColumn("col2", TypeCreator.NULLABLE.STRING, "comment2") + createMockColumn("col1", Types.ByteType.get(), "comment1"), + createMockColumn("col2", Types.StringType.get(), "comment2") }; DistributionDTO distributionDTO = createMockDistributionDTO("col2", 10); @@ -714,7 +714,7 @@ public void testAddTableColumn() throws JsonProcessingException { TableUpdateRequest.AddTableColumnRequest req = new TableUpdateRequest.AddTableColumnRequest( new String[] {"col2"}, - TypeCreator.NULLABLE.STRING, + Types.StringType.get(), "comment2", TableChange.ColumnPosition.after("col1")); @@ -726,9 +726,9 @@ public void testRenameTableColumn() throws JsonProcessingException { NameIdentifier tableId = NameIdentifier.of(metalakeName, catalogName, "schema1", "table1"); ColumnDTO[] columns = new ColumnDTO[] { - createMockColumn("col1", TypeCreator.NULLABLE.I8, "comment1"), - createMockColumn("col2", TypeCreator.NULLABLE.STRING, "comment2"), - createMockColumn("col3", TypeCreator.NULLABLE.STRING, "comment3") + createMockColumn("col1", Types.ByteType.get(), "comment1"), + createMockColumn("col2", Types.StringType.get(), "comment2"), + createMockColumn("col3", Types.StringType.get(), "comment3") }; DistributionDTO distributionDTO = createMockDistributionDTO("col1", 10); @@ -753,7 +753,7 @@ public void testRenameTableColumn() throws JsonProcessingException { public void testUpdateTableColumnComment() throws JsonProcessingException { NameIdentifier tableId = NameIdentifier.of(metalakeName, catalogName, "schema1", "table1"); ColumnDTO[] columns = - new ColumnDTO[] {createMockColumn("col1", TypeCreator.NULLABLE.I8, "comment2")}; + new ColumnDTO[] {createMockColumn("col1", Types.ByteType.get(), "comment2")}; DistributionDTO distributionDTO = createMockDistributionDTO("col1", 10); SortOrderDTO[] sortOrderDTOs = createMockSortOrderDTO("col1", DESCENDING); @@ -777,7 +777,7 @@ public void testUpdateTableColumnComment() throws JsonProcessingException { public void testUpdateTableColumnDataType() throws JsonProcessingException { NameIdentifier tableId = NameIdentifier.of(metalakeName, catalogName, "schema1", "table1"); ColumnDTO[] columns = - new ColumnDTO[] {createMockColumn("col1", TypeCreator.NULLABLE.STRING, "comment1")}; + new ColumnDTO[] {createMockColumn("col1", Types.StringType.get(), "comment1")}; DistributionDTO distributionDTO = createMockDistributionDTO("col1", 10); SortOrderDTO[] sortOrderDTOs = createMockSortOrderDTO("col1", DESCENDING); @@ -792,7 +792,7 @@ public void testUpdateTableColumnDataType() throws JsonProcessingException { sortOrderDTOs); TableUpdateRequest.UpdateTableColumnTypeRequest req = new TableUpdateRequest.UpdateTableColumnTypeRequest( - new String[] {"col1"}, TypeCreator.NULLABLE.STRING); + new String[] {"col1"}, Types.StringType.get()); testAlterTable(tableId, req, expectedTable); } @@ -802,8 +802,8 @@ public void testUpdateTableColumnPosition() throws JsonProcessingException { NameIdentifier tableId = NameIdentifier.of(metalakeName, catalogName, "schema1", "table1"); ColumnDTO[] columns = new ColumnDTO[] { - createMockColumn("col1", TypeCreator.NULLABLE.I8, "comment1"), - createMockColumn("col2", TypeCreator.NULLABLE.STRING, "comment2") + createMockColumn("col1", Types.ByteType.get(), "comment1"), + createMockColumn("col2", Types.StringType.get(), "comment2") }; DistributionDTO distributionDTO = createMockDistributionDTO("col1", 10); @@ -848,7 +848,7 @@ private SortOrderDTO[] createMockSortOrderDTO(String columnName, SortDirection d public void testDeleteTableColumn() throws JsonProcessingException { NameIdentifier tableId = NameIdentifier.of(metalakeName, catalogName, "schema1", "table1"); ColumnDTO[] columns = - new ColumnDTO[] {createMockColumn("col2", TypeCreator.NULLABLE.STRING, "comment2")}; + new ColumnDTO[] {createMockColumn("col2", Types.StringType.get(), "comment2")}; DistributionDTO distributionDTO = createMockDistributionDTO("col2", 10); SortOrderDTO[] sortOrderDTOs = createMockSortOrderDTO("col2", DESCENDING); diff --git a/common/build.gradle.kts b/common/build.gradle.kts index 4bd88989ac3..311bd5ba96b 100644 --- a/common/build.gradle.kts +++ b/common/build.gradle.kts @@ -11,14 +11,6 @@ plugins { dependencies { implementation(project(":api")) - implementation(libs.substrait.java.core) { - exclude("org.slf4j") - exclude("com.fasterxml.jackson.core") - exclude("com.fasterxml.jackson.datatype") - exclude("com.fasterxml.jackson.dataformat") - exclude("com.google.protobuf") - exclude("com.google.code.findbugs") - } implementation(libs.jackson.databind) implementation(libs.jackson.annotations) diff --git a/common/src/main/java/com/datastrato/gravitino/dto/rel/ColumnDTO.java b/common/src/main/java/com/datastrato/gravitino/dto/rel/ColumnDTO.java index 779a31fda73..c4aee785add 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/rel/ColumnDTO.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/rel/ColumnDTO.java @@ -6,11 +6,11 @@ import com.datastrato.gravitino.json.JsonUtils; import com.datastrato.gravitino.rel.Column; +import com.datastrato.gravitino.rel.types.Type; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonSerialize; import com.google.common.base.Preconditions; -import io.substrait.type.Type; import lombok.EqualsAndHashCode; import lombok.ToString; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/rel/expressions/LiteralDTO.java b/common/src/main/java/com/datastrato/gravitino/dto/rel/expressions/LiteralDTO.java index bc7588b4dba..e0bee2955c7 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/rel/expressions/LiteralDTO.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/rel/expressions/LiteralDTO.java @@ -5,7 +5,7 @@ package com.datastrato.gravitino.dto.rel.expressions; import com.datastrato.gravitino.rel.expressions.Literal; -import io.substrait.type.Type; +import com.datastrato.gravitino.rel.types.Type; import lombok.EqualsAndHashCode; @EqualsAndHashCode diff --git a/common/src/main/java/com/datastrato/gravitino/dto/requests/TableUpdateRequest.java b/common/src/main/java/com/datastrato/gravitino/dto/requests/TableUpdateRequest.java index 698ce504e66..8d6af905a7c 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/requests/TableUpdateRequest.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/requests/TableUpdateRequest.java @@ -6,6 +6,7 @@ import com.datastrato.gravitino.json.JsonUtils; import com.datastrato.gravitino.rel.TableChange; +import com.datastrato.gravitino.rel.types.Type; import com.datastrato.gravitino.rest.RESTRequest; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; @@ -14,7 +15,6 @@ import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonSerialize; import com.google.common.base.Preconditions; -import io.substrait.type.Type; import java.util.Arrays; import javax.annotation.Nullable; import lombok.EqualsAndHashCode; diff --git a/common/src/main/java/com/datastrato/gravitino/json/JsonUtils.java b/common/src/main/java/com/datastrato/gravitino/json/JsonUtils.java index b1a09e10d79..f300bc013fc 100644 --- a/common/src/main/java/com/datastrato/gravitino/json/JsonUtils.java +++ b/common/src/main/java/com/datastrato/gravitino/json/JsonUtils.java @@ -48,9 +48,6 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import io.substrait.type.StringTypeVisitor; -import io.substrait.type.parser.ParseToPojo; -import io.substrait.type.parser.TypeStringParser; import java.io.IOException; import java.util.Iterator; import java.util.List; @@ -272,8 +269,7 @@ public static FunctionArg readFunctionArg(JsonNode node) { node.has(LITERAL_VALUE), "Cannot parse literal arg from missing literal value: %s", node); - io.substrait.type.Type dataType = - TypeStringParser.parse(getString(DATA_TYPE, node), ParseToPojo::type); + Type dataType = readDataType(node.get(DATA_TYPE)); String value = getString(LITERAL_VALUE, node); return new LiteralDTO.Builder().withDataType(dataType).withValue(value).build(); case FIELD: @@ -309,8 +305,8 @@ public static void writeFunctionArg(FunctionArg arg, JsonGenerator gen) throws I gen.writeStringField(EXPRESSION_TYPE, arg.argType().name().toLowerCase()); switch (arg.argType()) { case LITERAL: - gen.writeStringField( - DATA_TYPE, ((LiteralDTO) arg).dataType().accept(new StringTypeVisitor())); + gen.writeFieldName(DATA_TYPE); + writeDataType(((LiteralDTO) arg).dataType(), gen); gen.writeStringField(LITERAL_VALUE, ((LiteralDTO) arg).value()); break; case FIELD: @@ -610,7 +606,7 @@ private static Types.StructType.Field readStructField(JsonNode node) { // Nested classes for custom serialization and deserialization /** Custom JSON serializer for Gravitino Type objects. */ - public static class GravitinoTypeSerializer extends JsonSerializer { + public static class TypeSerializer extends JsonSerializer { @Override public void serialize(Type value, JsonGenerator gen, SerializerProvider serializers) throws IOException { @@ -619,7 +615,7 @@ public void serialize(Type value, JsonGenerator gen, SerializerProvider serializ } /** Custom JSON deserializer for Gravitino Type objects. */ - public static class GravitinoTypeDeserializer extends JsonDeserializer { + public static class TypeDeserializer extends JsonDeserializer { @Override public Type deserialize(JsonParser p, DeserializationContext ctxt) throws IOException { @@ -627,39 +623,6 @@ public Type deserialize(JsonParser p, DeserializationContext ctxt) throws IOExce } } - public static class TypeSerializer extends JsonSerializer { - - private final StringTypeVisitor visitor = new StringTypeVisitor(); - - @Override - public void serialize( - io.substrait.type.Type value, JsonGenerator gen, SerializerProvider serializers) - throws IOException { - try { - gen.writeString(value.accept(visitor)); - } catch (Exception e) { - LOG.warn("Unable to serialize type {}.", value, e); - throw new IOException("Unable to serialize type " + value, e); - } - } - } - - /** Custom JSON deserializer for Substrait Type objects. */ - public static class TypeDeserializer extends JsonDeserializer { - - @Override - public io.substrait.type.Type deserialize(JsonParser p, DeserializationContext ctxt) - throws IOException { - String s = p.getValueAsString(); - try { - return TypeStringParser.parse(s, ParseToPojo::type); - } catch (Exception e) { - LOG.warn("Unable to parse string {}.", s.replace("\n", " \\n"), e); - throw new IOException("Unable to parse string " + s.replace("\n", " \\n"), e); - } - } - } - /** Custom JSON serializer for NameIdentifier objects. */ public static class NameIdentifierSerializer extends JsonSerializer { diff --git a/common/src/test/java/com/datastrato/gravitino/dto/responses/TestResponses.java b/common/src/test/java/com/datastrato/gravitino/dto/responses/TestResponses.java index 0bf4c5bc1a1..d27f82640bc 100644 --- a/common/src/test/java/com/datastrato/gravitino/dto/responses/TestResponses.java +++ b/common/src/test/java/com/datastrato/gravitino/dto/responses/TestResponses.java @@ -18,7 +18,7 @@ import com.datastrato.gravitino.dto.rel.SchemaDTO; import com.datastrato.gravitino.dto.rel.TableDTO; import com.datastrato.gravitino.dto.rel.partitions.Partitioning; -import io.substrait.type.TypeCreator; +import com.datastrato.gravitino.rel.types.Types; import java.time.Instant; import org.junit.jupiter.api.Test; @@ -144,7 +144,7 @@ void testTableResponse() throws IllegalArgumentException { AuditDTO audit = new AuditDTO.Builder().withCreator("creator").withCreateTime(Instant.now()).build(); ColumnDTO column = - new ColumnDTO.Builder().withName("ColumnA").withDataType(TypeCreator.NULLABLE.I8).build(); + new ColumnDTO.Builder().withName("ColumnA").withDataType(Types.ByteType.get()).build(); TableDTO table = new TableDTO.Builder() .withName("TableA") diff --git a/common/src/test/java/com/datastrato/gravitino/json/TestDTOJsonSerDe.java b/common/src/test/java/com/datastrato/gravitino/json/TestDTOJsonSerDe.java index b2d93d5c35e..f65b0319391 100644 --- a/common/src/test/java/com/datastrato/gravitino/json/TestDTOJsonSerDe.java +++ b/common/src/test/java/com/datastrato/gravitino/json/TestDTOJsonSerDe.java @@ -25,11 +25,11 @@ import com.datastrato.gravitino.dto.rel.partitions.RangePartitioningDTO; import com.datastrato.gravitino.dto.rel.partitions.TruncatePartitioningDTO; import com.datastrato.gravitino.dto.rel.partitions.YearPartitioningDTO; +import com.datastrato.gravitino.rel.types.Type; +import com.datastrato.gravitino.rel.types.Types; import com.fasterxml.jackson.databind.cfg.EnumFeature; import com.fasterxml.jackson.databind.json.JsonMapper; import com.google.common.collect.ImmutableMap; -import io.substrait.type.StringTypeVisitor; -import io.substrait.type.TypeCreator; import java.time.Instant; import java.util.Map; import org.junit.jupiter.api.Assertions; @@ -170,7 +170,7 @@ public void testCatalogDTOSerDe() throws Exception { @Test public void testColumnDTOSerDe() throws Exception { String name = "column"; - io.substrait.type.Type type = TypeCreator.NULLABLE.I8; + Type type = Types.ByteType.get(); String comment = "comment"; // Test required fields @@ -181,7 +181,7 @@ public void testColumnDTOSerDe() throws Exception { String.format( columnJson, withQuotes(name), - withQuotes(type.accept(new StringTypeVisitor())), + withQuotes(type.simpleString()), withQuotes(comment), column.nullable()); Assertions.assertEquals(expectedJson, serJson); @@ -192,7 +192,7 @@ public void testColumnDTOSerDe() throws Exception { @Test public void testTableDTOSerDe() throws Exception { String name = "column"; - io.substrait.type.Type type = TypeCreator.NULLABLE.I8; + Type type = Types.ByteType.get(); String comment = "comment"; String creator = "creator"; Instant now = Instant.now(); @@ -227,7 +227,7 @@ public void testTableDTOSerDe() throws Exception { String.format( columnJson, withQuotes(name), - withQuotes(type.accept(new StringTypeVisitor())), + withQuotes(type.simpleString()), withQuotes(comment), column.nullable()), JsonUtils.objectMapper().writeValueAsString(properties), @@ -275,7 +275,7 @@ public void testPartitioningDTOSerDe() throws Exception { FunctionArg arg1 = FieldReferenceDTO.of(field1); FunctionArg arg2 = new LiteralDTO.Builder() - .withDataType(TypeCreator.REQUIRED.STRING) + .withDataType(Types.StringType.get()) .withValue("Asia/Shanghai") .build(); FunctionArg toDateFunc = diff --git a/common/src/test/java/com/datastrato/gravitino/json/TestJsonUtils.java b/common/src/test/java/com/datastrato/gravitino/json/TestJsonUtils.java index 6e0b9f61c92..0452149ba2b 100644 --- a/common/src/test/java/com/datastrato/gravitino/json/TestJsonUtils.java +++ b/common/src/test/java/com/datastrato/gravitino/json/TestJsonUtils.java @@ -25,10 +25,9 @@ public class TestJsonUtils { @BeforeAll static void setUp() { objectMapper = JsonUtils.objectMapper(); - // TODO(minghuang): remove this once the old type serializer/deserializer is replaced. SimpleModule module = new SimpleModule(); - module.addSerializer(Type.class, new JsonUtils.GravitinoTypeSerializer()); - module.addDeserializer(Type.class, new JsonUtils.GravitinoTypeDeserializer()); + module.addSerializer(Type.class, new JsonUtils.TypeSerializer()); + module.addDeserializer(Type.class, new JsonUtils.TypeDeserializer()); objectMapper.registerModule(module); } diff --git a/core/build.gradle.kts b/core/build.gradle.kts index 72769b67145..036bc5ad040 100644 --- a/core/build.gradle.kts +++ b/core/build.gradle.kts @@ -17,11 +17,6 @@ dependencies { exclude("com.google.guava", "guava") .because("Brings in Guava for Andriod, which we don't want (and breaks multimaps).") } - implementation(libs.substrait.java.core) { - exclude("org.slf4j") - exclude("com.fasterxml.jackson.core") - exclude("com.fasterxml.jackson.datatype") - } implementation(libs.guava) implementation(libs.bundles.log4j) implementation(libs.commons.lang3) diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/rel/BaseColumn.java b/core/src/main/java/com/datastrato/gravitino/catalog/rel/BaseColumn.java index 800e65c8c20..ccb24a53b9f 100644 --- a/core/src/main/java/com/datastrato/gravitino/catalog/rel/BaseColumn.java +++ b/core/src/main/java/com/datastrato/gravitino/catalog/rel/BaseColumn.java @@ -6,7 +6,7 @@ package com.datastrato.gravitino.catalog.rel; import com.datastrato.gravitino.rel.Column; -import io.substrait.type.Type; +import com.datastrato.gravitino.rel.types.Type; import javax.annotation.Nullable; import lombok.EqualsAndHashCode; import lombok.ToString; diff --git a/core/src/test/java/com/datastrato/gravitino/catalog/TestCatalogOperationDispatcher.java b/core/src/test/java/com/datastrato/gravitino/catalog/TestCatalogOperationDispatcher.java index 262d8b915f0..674ab0da286 100644 --- a/core/src/test/java/com/datastrato/gravitino/catalog/TestCatalogOperationDispatcher.java +++ b/core/src/test/java/com/datastrato/gravitino/catalog/TestCatalogOperationDispatcher.java @@ -39,10 +39,10 @@ import com.datastrato.gravitino.rel.Table; import com.datastrato.gravitino.rel.TableChange; import com.datastrato.gravitino.rel.expressions.transforms.Transform; +import com.datastrato.gravitino.rel.types.Types; import com.datastrato.gravitino.storage.IdGenerator; import com.datastrato.gravitino.storage.RandomIdGenerator; import com.google.common.collect.ImmutableMap; -import io.substrait.type.TypeCreator; import java.io.IOException; import java.time.Instant; import java.util.Arrays; @@ -326,8 +326,8 @@ public void testCreateAndListTables() throws IOException { NameIdentifier tableIdent1 = NameIdentifier.of(tableNs, "table1"); Column[] columns = new Column[] { - new TestColumn.Builder().withName("col1").withType(TypeCreator.NULLABLE.STRING).build(), - new TestColumn.Builder().withName("col2").withType(TypeCreator.NULLABLE.STRING).build() + new TestColumn.Builder().withName("col1").withType(Types.StringType.get()).build(), + new TestColumn.Builder().withName("col2").withType(Types.StringType.get()).build() }; Table table1 = dispatcher.createTable(tableIdent1, columns, "comment", props, new Transform[0]); @@ -402,8 +402,8 @@ public void testCreateAndLoadTable() throws IOException { NameIdentifier tableIdent1 = NameIdentifier.of(tableNs, "table11"); Column[] columns = new Column[] { - new TestColumn.Builder().withName("col1").withType(TypeCreator.NULLABLE.STRING).build(), - new TestColumn.Builder().withName("col2").withType(TypeCreator.NULLABLE.STRING).build() + new TestColumn.Builder().withName("col1").withType(Types.StringType.get()).build(), + new TestColumn.Builder().withName("col2").withType(Types.StringType.get()).build() }; Table table1 = dispatcher.createTable(tableIdent1, columns, "comment", props, new Transform[0]); @@ -458,8 +458,8 @@ public void testCreateAndAlterTable() throws IOException { NameIdentifier tableIdent = NameIdentifier.of(tableNs, "table21"); Column[] columns = new Column[] { - new TestColumn.Builder().withName("col1").withType(TypeCreator.NULLABLE.STRING).build(), - new TestColumn.Builder().withName("col2").withType(TypeCreator.NULLABLE.STRING).build() + new TestColumn.Builder().withName("col1").withType(Types.StringType.get()).build(), + new TestColumn.Builder().withName("col2").withType(Types.StringType.get()).build() }; Table table = dispatcher.createTable(tableIdent, columns, "comment", props, new Transform[0]); @@ -525,8 +525,8 @@ public void testCreateAndDropTable() throws IOException { Map props = ImmutableMap.of("k1", "v1", "k2", "v2"); Column[] columns = new Column[] { - new TestColumn.Builder().withName("col1").withType(TypeCreator.NULLABLE.STRING).build(), - new TestColumn.Builder().withName("col2").withType(TypeCreator.NULLABLE.STRING).build() + new TestColumn.Builder().withName("col1").withType(Types.StringType.get()).build(), + new TestColumn.Builder().withName("col2").withType(Types.StringType.get()).build() }; Table table = dispatcher.createTable(tableIdent, columns, "comment", props, new Transform[0]); diff --git a/core/src/test/java/com/datastrato/gravitino/catalog/rel/TestBaseColumn.java b/core/src/test/java/com/datastrato/gravitino/catalog/rel/TestBaseColumn.java index e4bc7a651a8..84f7805f6c7 100644 --- a/core/src/test/java/com/datastrato/gravitino/catalog/rel/TestBaseColumn.java +++ b/core/src/test/java/com/datastrato/gravitino/catalog/rel/TestBaseColumn.java @@ -7,7 +7,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotEquals; -import io.substrait.type.TypeCreator; +import com.datastrato.gravitino.rel.types.Types; import org.junit.jupiter.api.Test; final class BaseColumnExtension extends BaseColumn { @@ -35,12 +35,12 @@ void testColumnFields() { new BaseColumnExtension.Builder() .withName("testColumnName") .withComment("testColumnComment") - .withType(TypeCreator.NULLABLE.STRING) + .withType(Types.IntegerType.get()) .build(); assertEquals("testColumnName", column.name()); assertEquals("testColumnComment", column.comment()); - assertEquals(TypeCreator.NULLABLE.STRING, column.dataType()); + assertEquals(Types.IntegerType.get(), column.dataType()); } @Test @@ -49,28 +49,28 @@ void testEqualsAndHashCode() { new BaseColumnExtension.Builder() .withName("testColumnName") .withComment("testColumnComment") - .withType(TypeCreator.NULLABLE.STRING) + .withType(Types.StringType.get()) .build(); BaseColumn column2 = new BaseColumnExtension.Builder() .withName("testColumnName") .withComment("testColumnComment") - .withType(TypeCreator.NULLABLE.STRING) + .withType(Types.StringType.get()) .build(); BaseColumn column3 = new BaseColumnExtension.Builder() .withName("differentColumnName") .withComment("testColumnComment") - .withType(TypeCreator.NULLABLE.STRING) + .withType(Types.StringType.get()) .build(); BaseColumn column4 = new BaseColumnExtension.Builder() .withName("testColumnName") .withComment("testColumnComment") - .withType(TypeCreator.NULLABLE.I32) + .withType(Types.IntegerType.get()) .build(); assertEquals(column1, column2); diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index ede0239631b..32741c771ac 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -5,7 +5,6 @@ [versions] junit = "5.8.1" protoc = "3.17.3" -substrait = "0.9.0" jackson = "2.15.2" guava = "29.0-jre" lombok = "1.18.20" @@ -48,7 +47,6 @@ metrics = "4.2.19" [libraries] protobuf-java = { group = "com.google.protobuf", name = "protobuf-java", version.ref = "protoc" } protobuf-java-util = { group = "com.google.protobuf", name = "protobuf-java-util", version.ref = "protoc" } -substrait-java-core = { group = "io.substrait", name = "core", version.ref = "substrait" } jackson-databind = { group = "com.fasterxml.jackson.core", name = "jackson-databind", version.ref = "jackson" } jackson-annotations = { group = "com.fasterxml.jackson.core", name = "jackson-annotations", version.ref = "jackson" } jackson-datatype-jdk8 = { group = "com.fasterxml.jackson.datatype", name = "jackson-datatype-jdk8", version.ref = "jackson" } diff --git a/integration-test/build.gradle.kts b/integration-test/build.gradle.kts index 26bd694f966..7dc09a15b40 100644 --- a/integration-test/build.gradle.kts +++ b/integration-test/build.gradle.kts @@ -85,12 +85,6 @@ dependencies { exclude("*") } - testImplementation(libs.substrait.java.core) { - exclude("org.slf4j") - exclude("com.fasterxml.jackson.core") - exclude("com.fasterxml.jackson.datatype") - } - testCompileOnly(libs.lombok) testAnnotationProcessor(libs.lombok) testImplementation(libs.guava) diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/hive/CatalogHiveIT.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/hive/CatalogHiveIT.java index 1280f5dddc1..f30bc48f1fb 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/hive/CatalogHiveIT.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/hive/CatalogHiveIT.java @@ -53,9 +53,9 @@ import com.datastrato.gravitino.rel.expressions.sorts.SortDirection; import com.datastrato.gravitino.rel.expressions.sorts.SortOrder; import com.datastrato.gravitino.rel.expressions.transforms.Transform; +import com.datastrato.gravitino.rel.types.Types; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; -import io.substrait.type.TypeCreator; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -214,19 +214,19 @@ private ColumnDTO[] createColumns() { ColumnDTO col1 = new ColumnDTO.Builder<>() .withName(HIVE_COL_NAME1) - .withDataType(TypeCreator.NULLABLE.I8) + .withDataType(Types.ByteType.get()) .withComment("col_1_comment") .build(); ColumnDTO col2 = new ColumnDTO.Builder<>() .withName(HIVE_COL_NAME2) - .withDataType(TypeCreator.NULLABLE.DATE) + .withDataType(Types.DateType.get()) .withComment("col_2_comment") .build(); ColumnDTO col3 = new ColumnDTO.Builder<>() .withName(HIVE_COL_NAME3) - .withDataType(TypeCreator.NULLABLE.STRING) + .withDataType(Types.StringType.get()) .withComment("col_3_comment") .build(); return new ColumnDTO[] {col1, col2, col3}; @@ -674,13 +674,13 @@ public void testAlterHiveTable() throws TException, InterruptedException { TableChange.setProperty("key2", "val2_new"), TableChange.addColumn( new String[] {"col_4"}, - TypeCreator.NULLABLE.STRING, + Types.StringType.get(), null, TableChange.ColumnPosition.after(columns[1].name())), TableChange.renameColumn(new String[] {HIVE_COL_NAME2}, "col_2_new"), TableChange.updateColumnComment(new String[] {HIVE_COL_NAME1}, "comment_new"), TableChange.updateColumnType( - new String[] {HIVE_COL_NAME1}, TypeCreator.NULLABLE.I32)); + new String[] {HIVE_COL_NAME1}, Types.IntegerType.get())); // Direct get table from hive metastore to check if the table is altered successfully. org.apache.hadoop.hive.metastore.api.Table hiveTab = @@ -716,7 +716,7 @@ public void testAlterHiveTable() throws TException, InterruptedException { .alterTable( NameIdentifier.of(metalakeName, catalogName, schemaName, ALTER_TABLE_NAME), TableChange.updateColumnType( - new String[] {HIVE_COL_NAME3}, TypeCreator.NULLABLE.I32)); + new String[] {HIVE_COL_NAME3}, Types.IntegerType.get())); }); Assertions.assertTrue(exception.getMessage().contains("Cannot alter partition column")); @@ -724,19 +724,19 @@ public void testAlterHiveTable() throws TException, InterruptedException { ColumnDTO col1 = new ColumnDTO.Builder() .withName("name") - .withDataType(TypeCreator.NULLABLE.STRING) + .withDataType(Types.StringType.get()) .withComment("comment") .build(); ColumnDTO col2 = new ColumnDTO.Builder() .withName("address") - .withDataType(TypeCreator.NULLABLE.STRING) + .withDataType(Types.StringType.get()) .withComment("comment") .build(); ColumnDTO col3 = new ColumnDTO.Builder() .withName("date_of_birth") - .withDataType(TypeCreator.NULLABLE.DATE) + .withDataType(Types.DateType.get()) .withComment("comment") .build(); ColumnDTO[] newColumns = new ColumnDTO[] {col1, col2, col3}; diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/lakehouse/iceberg/CatalogIcebergIT.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/lakehouse/iceberg/CatalogIcebergIT.java index 2a1e939d270..2ccc1bfb434 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/lakehouse/iceberg/CatalogIcebergIT.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/lakehouse/iceberg/CatalogIcebergIT.java @@ -38,9 +38,9 @@ import com.datastrato.gravitino.rel.expressions.sorts.NullOrdering; import com.datastrato.gravitino.rel.expressions.sorts.SortDirection; import com.datastrato.gravitino.rel.expressions.sorts.SortOrder; +import com.datastrato.gravitino.rel.types.Types; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; -import io.substrait.type.TypeCreator; import java.time.LocalDate; import java.time.format.DateTimeFormatter; import java.util.ArrayList; @@ -201,19 +201,19 @@ private ColumnDTO[] createColumns() { ColumnDTO col1 = new ColumnDTO.Builder() .withName(ICEBERG_COL_NAME1) - .withDataType(TypeCreator.NULLABLE.I32) + .withDataType(Types.IntegerType.get()) .withComment("col_1_comment") .build(); ColumnDTO col2 = new ColumnDTO.Builder() .withName(ICEBERG_COL_NAME2) - .withDataType(TypeCreator.NULLABLE.DATE) + .withDataType(Types.DateType.get()) .withComment("col_2_comment") .build(); ColumnDTO col3 = new ColumnDTO.Builder() .withName(ICEBERG_COL_NAME3) - .withDataType(TypeCreator.NULLABLE.STRING) + .withDataType(Types.StringType.get()) .withComment("col_3_comment") .build(); return new ColumnDTO[] {col1, col2, col3}; @@ -506,11 +506,11 @@ public void testAlterIcebergTable() { TableChange.updateComment(table_comment + "_new"), TableChange.removeProperty("key1"), TableChange.setProperty("key2", "val2_new"), - TableChange.addColumn(new String[] {"col_4"}, TypeCreator.NULLABLE.STRING), + TableChange.addColumn(new String[] {"col_4"}, Types.StringType.get()), TableChange.renameColumn(new String[] {ICEBERG_COL_NAME2}, "col_2_new"), TableChange.updateColumnComment(new String[] {ICEBERG_COL_NAME1}, "comment_new"), TableChange.updateColumnType( - new String[] {ICEBERG_COL_NAME1}, TypeCreator.NULLABLE.I32)); + new String[] {ICEBERG_COL_NAME1}, Types.IntegerType.get())); Table table = catalog @@ -520,19 +520,19 @@ public void testAlterIcebergTable() { Assertions.assertEquals("val2_new", table.properties().get("key2")); Assertions.assertEquals(ICEBERG_COL_NAME1, table.columns()[0].name()); - Assertions.assertEquals(TypeCreator.NULLABLE.I32, table.columns()[0].dataType()); + Assertions.assertEquals(Types.IntegerType.get(), table.columns()[0].dataType()); Assertions.assertEquals("comment_new", table.columns()[0].comment()); Assertions.assertEquals("col_2_new", table.columns()[1].name()); - Assertions.assertEquals(TypeCreator.NULLABLE.DATE, table.columns()[1].dataType()); + Assertions.assertEquals(Types.DateType.get(), table.columns()[1].dataType()); Assertions.assertEquals("col_2_comment", table.columns()[1].comment()); Assertions.assertEquals(ICEBERG_COL_NAME3, table.columns()[2].name()); - Assertions.assertEquals(TypeCreator.NULLABLE.STRING, table.columns()[2].dataType()); + Assertions.assertEquals(Types.StringType.get(), table.columns()[2].dataType()); Assertions.assertEquals("col_3_comment", table.columns()[2].comment()); Assertions.assertEquals("col_4", table.columns()[3].name()); - Assertions.assertEquals(TypeCreator.NULLABLE.STRING, table.columns()[3].dataType()); + Assertions.assertEquals(Types.StringType.get(), table.columns()[3].dataType()); Assertions.assertNull(table.columns()[3].comment()); Assertions.assertEquals(1, table.partitioning().length); @@ -543,19 +543,19 @@ public void testAlterIcebergTable() { ColumnDTO col1 = new ColumnDTO.Builder() .withName("name") - .withDataType(TypeCreator.NULLABLE.STRING) + .withDataType(Types.StringType.get()) .withComment("comment") .build(); ColumnDTO col2 = new ColumnDTO.Builder() .withName("address") - .withDataType(TypeCreator.NULLABLE.STRING) + .withDataType(Types.StringType.get()) .withComment("comment") .build(); ColumnDTO col3 = new ColumnDTO.Builder() .withName("date_of_birth") - .withDataType(TypeCreator.NULLABLE.DATE) + .withDataType(Types.DateType.get()) .withComment("comment") .build(); ColumnDTO[] newColumns = new ColumnDTO[] {col1, col2, col3}; diff --git a/meta/build.gradle.kts b/meta/build.gradle.kts index 93297767805..6ffe360d718 100644 --- a/meta/build.gradle.kts +++ b/meta/build.gradle.kts @@ -13,11 +13,6 @@ plugins { dependencies { implementation(libs.protobuf.java) - implementation(libs.substrait.java.core) { - exclude("org.slf4j") - exclude("com.fasterxml.jackson.core") - exclude("com.fasterxml.jackson.datatype") - } } sourceSets { diff --git a/rfc/rfc-1/rfc-1.md b/rfc/rfc-1/rfc-1.md index f8ef0dce394..489fb1d5231 100644 --- a/rfc/rfc-1/rfc-1.md +++ b/rfc/rfc-1/rfc-1.md @@ -20,29 +20,11 @@ ## Design choices -1. We choose to leverage Substrait’s type system into our entities’ type system. The main considerations are here: - 1. The goal of Substrait is to build a generic intermediate layer, its type system is relatively complete. - 2. We will further use Substrait to represent our logical plans (for example, like view, function and others), so using Substrait’s type system will reduce some converting works later on. -2. We choose JSON protocol as our user-faced protocol, which is easy to debug for users and systems. -3. We choose Protobuf binary layout to store the schema, the main considerations are here: +1. We choose JSON protocol as our user-faced protocol, which is easy to debug for users and systems. +2. We choose Protobuf binary layout to store the schema, the main considerations are here: 1. Binary layout is much more concise compared to HMS’s schema layout. 2. A general-purpose NoSQL database can be used to store the schema, which will get better performance and scalability compared to Hive Metastore. -## Type System - -We use Substrait’s type system to define our schema’s type, the advantages of using Substrait’s type system is that: - -1. Substrait’s type system is generic enough to cover major type kinds of DWS, DBS, and other type system. -2. Substrait supports extending the type systems to user-defined types. -3. Substrait support serializing and deserializing using Google protobuf. - -Disadvantages: - -1. Substrait is still in fast iteration, the spec is not fully defined yet, also the project is not mature enough. So we potentially have the risk of breaking backward compatibility. -2. Also Substrait lacks best practices, the learning curve is relatively high. - -Substrait’s type system can be referred here https://substrait.io/types/type_system/. - ## Meta System ### Prerequisites diff --git a/server/build.gradle.kts b/server/build.gradle.kts index f44865635e9..92a371e9700 100644 --- a/server/build.gradle.kts +++ b/server/build.gradle.kts @@ -26,11 +26,6 @@ dependencies { implementation(libs.bundles.log4j) implementation(libs.bundles.jetty) implementation(libs.bundles.jersey) - implementation(libs.substrait.java.core) { - exclude("org.slf4j") - exclude("com.fasterxml.jackson.core") - exclude("com.fasterxml.jackson.datatype") - } compileOnly(libs.lombok) annotationProcessor(libs.lombok) diff --git a/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestTableOperations.java b/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestTableOperations.java index 38cacf8a145..3cb9d4bbd88 100644 --- a/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestTableOperations.java +++ b/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestTableOperations.java @@ -41,11 +41,11 @@ import com.datastrato.gravitino.rel.expressions.sorts.SortDirection; import com.datastrato.gravitino.rel.expressions.sorts.SortOrder; import com.datastrato.gravitino.rel.expressions.transforms.Transform; +import com.datastrato.gravitino.rel.types.Type; +import com.datastrato.gravitino.rel.types.Types; import com.datastrato.gravitino.rest.RESTUtils; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import io.substrait.type.Type; -import io.substrait.type.TypeCreator; import java.io.IOException; import java.time.Instant; import java.util.Arrays; @@ -183,8 +183,7 @@ private SortOrderDTO[] createMockSortOrderDTO(String columnName, SortDirection d public void testCreateTable() { Column[] columns = new Column[] { - mockColumn("col1", TypeCreator.NULLABLE.STRING), - mockColumn("col2", TypeCreator.NULLABLE.I8) + mockColumn("col1", Types.StringType.get()), mockColumn("col2", Types.ByteType.get()) }; Table table = mockTable("table1", columns, "mock comment", ImmutableMap.of("k1", "v1")); when(dispatcher.createTable(any(), any(), any(), any(), any(), any(), any())).thenReturn(table); @@ -311,8 +310,7 @@ public void testCreateTable() { public void testCreatePartitionedTable() { Column[] columns = new Column[] { - mockColumn("col1", TypeCreator.NULLABLE.STRING), - mockColumn("col2", TypeCreator.NULLABLE.I8) + mockColumn("col1", Types.StringType.get()), mockColumn("col2", Types.ByteType.get()) }; Partitioning[] partitioning = new Partitioning[] {IdentityPartitioningDTO.of(columns[0].name())}; @@ -387,8 +385,7 @@ public void testCreatePartitionedTable() { public void testLoadTable() { Column[] columns = new Column[] { - mockColumn("col1", TypeCreator.NULLABLE.STRING), - mockColumn("col2", TypeCreator.NULLABLE.I8) + mockColumn("col1", Types.StringType.get()), mockColumn("col2", Types.ByteType.get()) }; Table table = mockTable("table1", columns, "mock comment", ImmutableMap.of("k1", "v1"), new Transform[0]); @@ -459,8 +456,7 @@ public void testRenameTable() { TableUpdateRequest.RenameTableRequest req = new TableUpdateRequest.RenameTableRequest("table2"); Column[] columns = new Column[] { - mockColumn("col1", TypeCreator.NULLABLE.STRING), - mockColumn("col2", TypeCreator.NULLABLE.I8) + mockColumn("col1", Types.StringType.get()), mockColumn("col2", Types.ByteType.get()) }; Table table = mockTable("table2", columns, "mock comment", ImmutableMap.of("k1", "v1"), new Transform[0]); @@ -473,8 +469,7 @@ public void testUpdateTableComment() { new TableUpdateRequest.UpdateTableCommentRequest("new comment"); Column[] columns = new Column[] { - mockColumn("col1", TypeCreator.NULLABLE.STRING), - mockColumn("col2", TypeCreator.NULLABLE.I8) + mockColumn("col1", Types.StringType.get()), mockColumn("col2", Types.ByteType.get()) }; Table table = mockTable("table1", columns, "new comment", ImmutableMap.of("k1", "v1"), new Transform[0]); @@ -487,8 +482,7 @@ public void testSetTableProperty() { new TableUpdateRequest.SetTablePropertyRequest("k2", "v2"); Column[] columns = new Column[] { - mockColumn("col1", TypeCreator.NULLABLE.STRING), - mockColumn("col2", TypeCreator.NULLABLE.I8) + mockColumn("col1", Types.StringType.get()), mockColumn("col2", Types.ByteType.get()) }; Table table = mockTable( @@ -506,8 +500,7 @@ public void testRemoveTableProperty() { new TableUpdateRequest.RemoveTablePropertyRequest("k1"); Column[] columns = new Column[] { - mockColumn("col1", TypeCreator.NULLABLE.STRING), - mockColumn("col2", TypeCreator.NULLABLE.I8) + mockColumn("col1", Types.StringType.get()), mockColumn("col2", Types.ByteType.get()) }; Table table = mockTable("table1", columns, "mock comment", ImmutableMap.of(), new Transform[0]); testAlterTableRequest(req, table); @@ -518,14 +511,14 @@ public void testAddTableColumnFirst() { TableUpdateRequest.AddTableColumnRequest req = new TableUpdateRequest.AddTableColumnRequest( new String[] {"col3"}, - TypeCreator.NULLABLE.STRING, + Types.StringType.get(), "mock comment", TableChange.ColumnPosition.first()); Column[] columns = new Column[] { - mockColumn("col3", TypeCreator.NULLABLE.STRING), - mockColumn("col1", TypeCreator.NULLABLE.STRING), - mockColumn("col2", TypeCreator.NULLABLE.I8) + mockColumn("col3", Types.StringType.get()), + mockColumn("col1", Types.StringType.get()), + mockColumn("col2", Types.ByteType.get()) }; Table table = mockTable("table1", columns, "mock comment", ImmutableMap.of("k1", "v1"), new Transform[0]); @@ -537,14 +530,14 @@ public void testAddTableColumnAfter() { TableUpdateRequest.AddTableColumnRequest req = new TableUpdateRequest.AddTableColumnRequest( new String[] {"col1"}, - TypeCreator.NULLABLE.STRING, + Types.StringType.get(), "mock comment", TableChange.ColumnPosition.after("col2")); Column[] columns = new Column[] { - mockColumn("col1", TypeCreator.NULLABLE.STRING), - mockColumn("col2", TypeCreator.NULLABLE.I8), - mockColumn("col3", TypeCreator.NULLABLE.STRING) + mockColumn("col1", Types.StringType.get()), + mockColumn("col2", Types.ByteType.get()), + mockColumn("col3", Types.StringType.get()) }; Table table = mockTable("table1", columns, "mock comment", ImmutableMap.of("k1", "v1"), new Transform[0]); @@ -557,8 +550,7 @@ public void testRenameTableColumn() { new TableUpdateRequest.RenameTableColumnRequest(new String[] {"col1"}, "col3"); Column[] columns = new Column[] { - mockColumn("col3", TypeCreator.NULLABLE.STRING), - mockColumn("col2", TypeCreator.NULLABLE.I8) + mockColumn("col3", Types.StringType.get()), mockColumn("col2", Types.ByteType.get()) }; Table table = mockTable("table1", columns, "mock comment", ImmutableMap.of("k1", "v1"), new Transform[0]); @@ -569,10 +561,10 @@ public void testRenameTableColumn() { public void testUpdateTableColumnType() { TableUpdateRequest.UpdateTableColumnTypeRequest req = new TableUpdateRequest.UpdateTableColumnTypeRequest( - new String[] {"col1"}, TypeCreator.NULLABLE.I8); + new String[] {"col1"}, Types.ByteType.get()); Column[] columns = new Column[] { - mockColumn("col1", TypeCreator.NULLABLE.I8), mockColumn("col2", TypeCreator.NULLABLE.I8) + mockColumn("col1", Types.ByteType.get()), mockColumn("col2", Types.ByteType.get()) }; Table table = mockTable("table1", columns, "mock comment", ImmutableMap.of("k1", "v1"), new Transform[0]); @@ -586,8 +578,8 @@ public void testUpdateTableColumnComment() { new String[] {"col1"}, "new comment"); Column[] columns = new Column[] { - mockColumn("col1", TypeCreator.NULLABLE.STRING, "new comment"), - mockColumn("col2", TypeCreator.NULLABLE.I8) + mockColumn("col1", Types.StringType.get(), "new comment"), + mockColumn("col2", Types.ByteType.get()) }; Table table = mockTable("table1", columns, "mock comment", ImmutableMap.of("k1", "v1"), new Transform[0]); @@ -601,8 +593,7 @@ public void testUpdateTableColumnPosition() { new String[] {"col1"}, TableChange.ColumnPosition.after("col2")); Column[] columns = new Column[] { - mockColumn("col2", TypeCreator.NULLABLE.I8), - mockColumn("col1", TypeCreator.NULLABLE.STRING) + mockColumn("col2", Types.ByteType.get()), mockColumn("col1", Types.StringType.get()) }; Table table = mockTable("table1", columns, "mock comment", ImmutableMap.of("k1", "v1"), new Transform[0]); diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogConnectorMetadata.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogConnectorMetadata.java index 2fa5c660390..23ff9b7f6c1 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogConnectorMetadata.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogConnectorMetadata.java @@ -243,14 +243,9 @@ public void renameColumn(SchemaTableName schemaTableName, String columnName, Str } public void setColumnType(SchemaTableName schemaTableName, String columnName, Type type) { - boolean isNullable = - getTable(schemaTableName.getSchemaName(), schemaTableName.getTableName()) - .getColumn(columnName) - .isNullable(); String[] columnNames = {columnName}; applyAlter( schemaTableName, - TableChange.updateColumnType( - columnNames, DataTypeTransformer.getGravitinoType(type, isNullable))); + TableChange.updateColumnType(columnNames, DataTypeTransformer.getGravitinoType(type))); } } diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogConnectorMetadataAdapter.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogConnectorMetadataAdapter.java index 06853d03825..ded99a7498e 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogConnectorMetadataAdapter.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogConnectorMetadataAdapter.java @@ -71,9 +71,10 @@ public GravitinoTable createTable(ConnectorTableMetadata tableMetadata) { columns.add( new GravitinoColumn( column.getName(), - DataTypeTransformer.getGravitinoType(column.getType(), column.isNullable()), + DataTypeTransformer.getGravitinoType(column.getType()), i, - column.getComment())); + column.getComment(), + column.isNullable())); } return new GravitinoTable(schemaName, tableName, columns, comment, properties); } @@ -144,8 +145,9 @@ private Map removeUnsetProperties(Map properties public GravitinoColumn createColumn(ColumnMetadata column) { return new GravitinoColumn( column.getName(), - DataTypeTransformer.getGravitinoType(column.getType(), column.isNullable()), + DataTypeTransformer.getGravitinoType(column.getType()), -1, - column.getComment()); + column.getComment(), + column.isNullable()); } } diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/metadata/GravitinoColumn.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/metadata/GravitinoColumn.java index 764444ea896..e9500322c8c 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/metadata/GravitinoColumn.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/metadata/GravitinoColumn.java @@ -4,10 +4,8 @@ */ package com.datastrato.gravitino.trino.connector.metadata; -import static java.util.Objects.requireNonNull; - import com.datastrato.gravitino.rel.Column; -import com.datastrato.gravitino.shaded.io.substrait.type.Type; +import com.datastrato.gravitino.rel.types.Type; import java.util.Map; /** Help Gravitino connector access ColumnMetadata from gravitino client. */ @@ -16,20 +14,18 @@ public final class GravitinoColumn { private final Type dataType; private final int index; private final String comment; + private final boolean nullable; public GravitinoColumn(Column column, int columnIndex) { - this.name = column.name(); - this.dataType = column.dataType(); - this.index = columnIndex; - this.comment = column.comment(); - requireNonNull(column, "column is null or is empty"); + this(column.name(), column.dataType(), columnIndex, column.comment(), column.nullable()); } - public GravitinoColumn(String name, Type dataType, int index, String comment) { + public GravitinoColumn(String name, Type dataType, int index, String comment, boolean nullable) { this.name = name; this.dataType = dataType; this.index = index; this.comment = comment; + this.nullable = nullable; } public int getIndex() { @@ -53,7 +49,7 @@ public String getComment() { } public boolean isNullable() { - return dataType.nullable(); + return nullable; } public boolean isHidden() { diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/util/DataTypeTransformer.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/util/DataTypeTransformer.java index 5eb9975fad9..17d691355d9 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/util/DataTypeTransformer.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/util/DataTypeTransformer.java @@ -12,63 +12,62 @@ import static io.trino.spi.type.DoubleType.DOUBLE; import static io.trino.spi.type.IntegerType.INTEGER; import static io.trino.spi.type.TimestampType.TIMESTAMP_SECONDS; -import static io.trino.spi.type.TimestampType.createTimestampType; import static io.trino.spi.type.VarbinaryType.VARBINARY; import static io.trino.spi.type.VarcharType.VARCHAR; -import static io.trino.spi.type.VarcharType.createUnboundedVarcharType; -import com.datastrato.gravitino.shaded.io.substrait.type.TypeCreator; +import com.datastrato.gravitino.rel.types.Types; import io.trino.spi.TrinoException; import io.trino.spi.type.Type; +import io.trino.spi.type.VarcharType; /** This class is used to transform datatype between gravitino and trino */ public class DataTypeTransformer { - public static Type getTrinoType(com.datastrato.gravitino.shaded.io.substrait.type.Type type) { - if (type.equals(TypeCreator.REQUIRED.STRING) || type.equals(TypeCreator.NULLABLE.STRING)) { - return createUnboundedVarcharType(); - } else if (type.equals(TypeCreator.REQUIRED.BOOLEAN) - || type.equals(TypeCreator.NULLABLE.BOOLEAN)) { - return BOOLEAN; - } else if (type.equals(TypeCreator.REQUIRED.I32) || type.equals(TypeCreator.NULLABLE.I32)) { - return INTEGER; - } else if (type.equals(TypeCreator.REQUIRED.I64) || type.equals(TypeCreator.NULLABLE.I64)) { - return BIGINT; - } else if (type.equals(TypeCreator.REQUIRED.FP64) || type.equals(TypeCreator.NULLABLE.FP64)) { - return DOUBLE; - } else if (type.equals(TypeCreator.REQUIRED.BINARY) - || type.equals(TypeCreator.NULLABLE.BINARY)) { - return VARBINARY; - } else if (type.equals(TypeCreator.REQUIRED.DATE) || type.equals(TypeCreator.NULLABLE.DATE)) { - return DATE; - } else if (type.equals(TypeCreator.REQUIRED.TIMESTAMP) - || type.equals(TypeCreator.NULLABLE.TIMESTAMP)) { - return createTimestampType(TIMESTAMP_SECONDS.getPrecision()); + public static Type getTrinoType(com.datastrato.gravitino.rel.types.Type type) { + switch (type.name()) { + case BOOLEAN: + return BOOLEAN; + case INTEGER: + return INTEGER; + case LONG: + return BIGINT; + case DOUBLE: + return DOUBLE; + case BINARY: + return VARBINARY; + case DATE: + return DATE; + case TIMESTAMP: + return TIMESTAMP_SECONDS; + case STRING: + return VARCHAR; + default: + throw new TrinoException( + GRAVITINO_UNSUPPORTED_GRAVITINO_DATATYPE, "Unsupported gravitino datatype: " + type); } - throw new TrinoException( - GRAVITINO_UNSUPPORTED_GRAVITINO_DATATYPE, "Unsupported gravitino datatype: " + type); } - public static com.datastrato.gravitino.shaded.io.substrait.type.Type getGravitinoType( - Type type, boolean nullable) { + public static com.datastrato.gravitino.rel.types.Type getGravitinoType(Type type) { if (type.equals(VARCHAR)) { - return nullable ? TypeCreator.NULLABLE.STRING : TypeCreator.REQUIRED.STRING; + return ((VarcharType) type).getLength().isPresent() + ? Types.VarCharType.of(((VarcharType) type).getLength().get()) + : Types.StringType.get(); } else if (type.equals(BOOLEAN)) { - return nullable ? TypeCreator.NULLABLE.BOOLEAN : TypeCreator.REQUIRED.BOOLEAN; + return Types.BooleanType.get(); } else if (type.equals(INTEGER)) { - return nullable ? TypeCreator.NULLABLE.I32 : TypeCreator.REQUIRED.I32; + return Types.IntegerType.get(); } else if (type.equals(BIGINT)) { - return nullable ? TypeCreator.NULLABLE.I64 : TypeCreator.REQUIRED.I64; + return Types.LongType.get(); } else if (type.equals(DOUBLE)) { - return nullable ? TypeCreator.NULLABLE.FP64 : TypeCreator.REQUIRED.FP64; + return Types.DoubleType.get(); } else if (type.equals(VARBINARY)) { - return nullable ? TypeCreator.NULLABLE.BINARY : TypeCreator.REQUIRED.BINARY; + return Types.BinaryType.get(); } else if (type.equals(DATE)) { - return nullable ? TypeCreator.NULLABLE.DATE : TypeCreator.REQUIRED.DATE; + return Types.DateType.get(); } else if (type.equals(TIMESTAMP_SECONDS)) { - return nullable ? TypeCreator.NULLABLE.TIMESTAMP : TypeCreator.REQUIRED.TIMESTAMP; + return Types.TimestampType.withoutTimeZone(); } throw new TrinoException( - GRAVITINO_UNSUPPORTED_TRINO_DATATYPE, "Unsupported trino datatype: " + type); + GRAVITINO_UNSUPPORTED_TRINO_DATATYPE, "Unsupported Trino datatype: " + type); } } diff --git a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/GravitinoMockServer.java b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/GravitinoMockServer.java index 50d1beca875..5acd0301641 100644 --- a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/GravitinoMockServer.java +++ b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/GravitinoMockServer.java @@ -394,7 +394,8 @@ void doAlterTable( } else if (tableChange instanceof TableChange.AddColumn) { TableChange.AddColumn addColumn = (TableChange.AddColumn) tableChange; String fieldName = addColumn.fieldNames()[0]; - GravitinoColumn column = new GravitinoColumn(fieldName, addColumn.getDataType(), -1, ""); + GravitinoColumn column = + new GravitinoColumn(fieldName, addColumn.getDataType(), -1, "", true); CatalogConnectorMetadataAdapter metadataAdapter = catalogConnectorManager.getCatalogConnector(catalogName.toString()).getMetadataAdapter(); metadata.addColumn(null, tableHandle, metadataAdapter.getColumnMetadata(column)); diff --git a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/metadata/TestGravitinoColumn.java b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/metadata/TestGravitinoColumn.java index 40d968bc93e..6917d0c1c61 100644 --- a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/metadata/TestGravitinoColumn.java +++ b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/metadata/TestGravitinoColumn.java @@ -7,7 +7,7 @@ import static org.testng.Assert.assertEquals; import com.datastrato.gravitino.dto.rel.ColumnDTO; -import com.datastrato.gravitino.shaded.io.substrait.type.TypeCreator; +import com.datastrato.gravitino.rel.types.Types; import org.testng.annotations.Test; public class TestGravitinoColumn { @@ -18,7 +18,7 @@ public void testGravitinoColumn() { new ColumnDTO.Builder() .withName("f1") .withComment("test column") - .withDataType(TypeCreator.NULLABLE.STRING) + .withDataType(Types.StringType.get()) .build(); GravitinoColumn column = new GravitinoColumn(columnDTO, 0); diff --git a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/metadata/TestGravitinoTable.java b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/metadata/TestGravitinoTable.java index 1d056406d43..6bd28378704 100644 --- a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/metadata/TestGravitinoTable.java +++ b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/metadata/TestGravitinoTable.java @@ -9,7 +9,7 @@ import com.datastrato.gravitino.dto.AuditDTO; import com.datastrato.gravitino.dto.rel.ColumnDTO; import com.datastrato.gravitino.dto.rel.TableDTO; -import com.datastrato.gravitino.shaded.io.substrait.type.TypeCreator; +import com.datastrato.gravitino.rel.types.Types; import java.time.Instant; import java.util.HashMap; import java.util.Map; @@ -23,12 +23,12 @@ public void testGravitinoTable() { new ColumnDTO[] { new ColumnDTO.Builder() .withName("f1") - .withDataType(TypeCreator.NULLABLE.STRING) + .withDataType(Types.StringType.get()) .withComment("f1 column") .build(), new ColumnDTO.Builder() .withName("f2") - .withDataType(TypeCreator.NULLABLE.I32) + .withDataType(Types.IntegerType.get()) .withComment("f2 column") .build() }; diff --git a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/util/TestDataTypeTransformer.java b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/util/TestDataTypeTransformer.java index 80f526f3d51..307fcf70c29 100644 --- a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/util/TestDataTypeTransformer.java +++ b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/util/TestDataTypeTransformer.java @@ -17,7 +17,7 @@ import static io.trino.spi.type.VarcharType.VARCHAR; import static org.testng.Assert.assertEquals; -import com.datastrato.gravitino.shaded.io.substrait.type.TypeCreator; +import com.datastrato.gravitino.rel.types.Types; import io.trino.spi.TrinoException; import org.testng.annotations.Test; @@ -25,39 +25,26 @@ public class TestDataTypeTransformer { @Test public void testGetGravitinoType() { - assertEquals(DataTypeTransformer.getGravitinoType(VARCHAR, true), TypeCreator.NULLABLE.STRING); - assertEquals(DataTypeTransformer.getGravitinoType(VARCHAR, false), TypeCreator.REQUIRED.STRING); + assertEquals(DataTypeTransformer.getGravitinoType(VARCHAR), Types.StringType.get()); - assertEquals(DataTypeTransformer.getGravitinoType(BOOLEAN, true), TypeCreator.NULLABLE.BOOLEAN); - assertEquals( - DataTypeTransformer.getGravitinoType(BOOLEAN, false), TypeCreator.REQUIRED.BOOLEAN); + assertEquals(DataTypeTransformer.getGravitinoType(BOOLEAN), Types.BooleanType.get()); - assertEquals(DataTypeTransformer.getGravitinoType(INTEGER, true), TypeCreator.NULLABLE.I32); - assertEquals(DataTypeTransformer.getGravitinoType(INTEGER, false), TypeCreator.REQUIRED.I32); + assertEquals(DataTypeTransformer.getGravitinoType(INTEGER), Types.IntegerType.get()); - assertEquals(DataTypeTransformer.getGravitinoType(BIGINT, true), TypeCreator.NULLABLE.I64); - assertEquals(DataTypeTransformer.getGravitinoType(BIGINT, false), TypeCreator.REQUIRED.I64); + assertEquals(DataTypeTransformer.getGravitinoType(BIGINT), Types.LongType.get()); - assertEquals(DataTypeTransformer.getGravitinoType(DOUBLE, true), TypeCreator.NULLABLE.FP64); - assertEquals(DataTypeTransformer.getGravitinoType(DOUBLE, false), TypeCreator.REQUIRED.FP64); + assertEquals(DataTypeTransformer.getGravitinoType(DOUBLE), Types.DoubleType.get()); - assertEquals( - DataTypeTransformer.getGravitinoType(VARBINARY, true), TypeCreator.NULLABLE.BINARY); - assertEquals( - DataTypeTransformer.getGravitinoType(VARBINARY, false), TypeCreator.REQUIRED.BINARY); + assertEquals(DataTypeTransformer.getGravitinoType(VARBINARY), Types.BinaryType.get()); - assertEquals(DataTypeTransformer.getGravitinoType(DATE, true), TypeCreator.NULLABLE.DATE); - assertEquals(DataTypeTransformer.getGravitinoType(DATE, false), TypeCreator.REQUIRED.DATE); + assertEquals(DataTypeTransformer.getGravitinoType(DATE), Types.DateType.get()); assertEquals( - DataTypeTransformer.getGravitinoType(TIMESTAMP_SECONDS, true), - TypeCreator.NULLABLE.TIMESTAMP); - assertEquals( - DataTypeTransformer.getGravitinoType(TIMESTAMP_SECONDS, false), - TypeCreator.REQUIRED.TIMESTAMP); + DataTypeTransformer.getGravitinoType(TIMESTAMP_SECONDS), + Types.TimestampType.withoutTimeZone()); try { - DataTypeTransformer.getGravitinoType(HYPER_LOG_LOG, true); + DataTypeTransformer.getGravitinoType(HYPER_LOG_LOG); } catch (TrinoException e) { if (e.getErrorCode() != GRAVITINO_UNSUPPORTED_TRINO_DATATYPE.toErrorCode()) { throw e; @@ -67,34 +54,25 @@ public void testGetGravitinoType() { @Test public void testGetTrinoType() { - assertEquals(DataTypeTransformer.getTrinoType(TypeCreator.NULLABLE.STRING), VARCHAR); - assertEquals(DataTypeTransformer.getTrinoType(TypeCreator.REQUIRED.STRING), VARCHAR); + assertEquals(DataTypeTransformer.getTrinoType(Types.StringType.get()), VARCHAR); - assertEquals(DataTypeTransformer.getTrinoType(TypeCreator.NULLABLE.BOOLEAN), BOOLEAN); - assertEquals(DataTypeTransformer.getTrinoType(TypeCreator.REQUIRED.BOOLEAN), BOOLEAN); + assertEquals(DataTypeTransformer.getTrinoType(Types.BooleanType.get()), BOOLEAN); - assertEquals(DataTypeTransformer.getTrinoType(TypeCreator.NULLABLE.I32), INTEGER); - assertEquals(DataTypeTransformer.getTrinoType(TypeCreator.REQUIRED.I32), INTEGER); + assertEquals(DataTypeTransformer.getTrinoType(Types.IntegerType.get()), INTEGER); - assertEquals(DataTypeTransformer.getTrinoType(TypeCreator.NULLABLE.I64), BIGINT); - assertEquals(DataTypeTransformer.getTrinoType(TypeCreator.REQUIRED.I64), BIGINT); + assertEquals(DataTypeTransformer.getTrinoType(Types.LongType.get()), BIGINT); - assertEquals(DataTypeTransformer.getTrinoType(TypeCreator.NULLABLE.I64), BIGINT); - assertEquals(DataTypeTransformer.getTrinoType(TypeCreator.REQUIRED.I64), BIGINT); + assertEquals(DataTypeTransformer.getTrinoType(Types.LongType.get()), BIGINT); - assertEquals(DataTypeTransformer.getTrinoType(TypeCreator.NULLABLE.FP64), DOUBLE); - assertEquals(DataTypeTransformer.getTrinoType(TypeCreator.REQUIRED.FP64), DOUBLE); + assertEquals(DataTypeTransformer.getTrinoType(Types.DoubleType.get()), DOUBLE); - assertEquals(DataTypeTransformer.getTrinoType(TypeCreator.NULLABLE.DATE), DATE); - assertEquals(DataTypeTransformer.getTrinoType(TypeCreator.REQUIRED.DATE), DATE); + assertEquals(DataTypeTransformer.getTrinoType(Types.DateType.get()), DATE); assertEquals( - DataTypeTransformer.getTrinoType(TypeCreator.NULLABLE.TIMESTAMP), TIMESTAMP_SECONDS); - assertEquals( - DataTypeTransformer.getTrinoType(TypeCreator.REQUIRED.TIMESTAMP), TIMESTAMP_SECONDS); + DataTypeTransformer.getTrinoType(Types.TimestampType.withoutTimeZone()), TIMESTAMP_SECONDS); try { - DataTypeTransformer.getTrinoType(TypeCreator.NULLABLE.BINARY); + DataTypeTransformer.getTrinoType(Types.BinaryType.get()); } catch (TrinoException e) { if (e.getErrorCode() != GRAVITINO_UNSUPPORTED_GRAVITINO_DATATYPE.toErrorCode()) { throw e;