[apache#800] feat(all): Replace the old type system with a new one (a…

…pache#801) ### What changes were proposed in this pull request? - Replace the old type system with a new one - modify the type converter between Iceberg and Gravitino - modify the type converter between Hive and Gravitino - modify the type converter between Trino and Gravitino ### Why are the changes needed? Fix: apache#800 ### Does this PR introduce _any_ user-facing change? yes, type API use Gravitino type instead of Substrait ### How was this patch tested? UTs and ITs --------- Co-authored-by: Jerry Shao <jerryshao@datastrato.com>
yuqi1129 · Nov 24, 2023 · c03911f · c03911f
1 parent a03d24f
commit c03911f
Show file tree

Hide file tree

Showing 58 changed files with 673 additions and 766 deletions.
diff --git a/LICENSE.bin b/LICENSE.bin
@@ -256,7 +256,6 @@
    Airlift
    The Netty Project
    Open Telemetry
-   Substrait Java
    Trino
    Jakarta Dependency Injection
    Jakarta Bean Validation

diff --git a/api/build.gradle.kts b/api/build.gradle.kts
@@ -10,14 +10,6 @@ plugins {
 }
 
 dependencies {
-  implementation(libs.substrait.java.core) {
-    exclude("com.fasterxml.jackson.core")
-    exclude("com.fasterxml.jackson.datatype")
-    exclude("com.fasterxml.jackson.dataformat")
-    exclude("com.google.protobuf")
-    exclude("com.google.code.findbugs")
-    exclude("org.slf4j")
-  }
   implementation(libs.guava)
   implementation(libs.slf4j.api)
 

diff --git a/api/src/main/java/com/datastrato/gravitino/rel/Column.java b/api/src/main/java/com/datastrato/gravitino/rel/Column.java
@@ -5,7 +5,7 @@
 package com.datastrato.gravitino.rel;
 
 import com.datastrato.gravitino.NameIdentifier;
-import io.substrait.type.Type;
+import com.datastrato.gravitino.rel.types.Type;
 import java.util.Map;
 
 /**

diff --git a/api/src/main/java/com/datastrato/gravitino/rel/TableChange.java b/api/src/main/java/com/datastrato/gravitino/rel/TableChange.java
@@ -20,7 +20,7 @@
 
 package com.datastrato.gravitino.rel;
 
-import io.substrait.type.Type;
+import com.datastrato.gravitino.rel.types.Type;
 import lombok.EqualsAndHashCode;
 import lombok.Getter;
 

diff --git a/api/src/main/java/com/datastrato/gravitino/rel/expressions/Literal.java b/api/src/main/java/com/datastrato/gravitino/rel/expressions/Literal.java
@@ -20,8 +20,8 @@
 
 package com.datastrato.gravitino.rel.expressions;
 
-import io.substrait.type.Type;
-import io.substrait.type.TypeCreator;
+import com.datastrato.gravitino.rel.types.Type;
+import com.datastrato.gravitino.rel.types.Types;
 import java.util.Objects;
 
 /**
@@ -60,7 +60,7 @@ static <T> LiteralImpl<T> of(T value, Type dataType) {
    * @return a new {@link Literal} instance
    */
   static LiteralImpl<Integer> integer(Integer value) {
-    return of(value, TypeCreator.REQUIRED.I32);
+    return of(value, Types.IntegerType.get());
   }
 
   /**
@@ -70,7 +70,7 @@ static LiteralImpl<Integer> integer(Integer value) {
    * @return a new {@link Literal} instance
    */
   static LiteralImpl<String> string(String value) {
-    return of(value, TypeCreator.REQUIRED.STRING);
+    return of(value, Types.StringType.get());
   }
 
   final class LiteralImpl<T> implements Literal<T> {

diff --git a/api/src/test/java/com/datastrato/gravitino/TestTableChange.java b/api/src/test/java/com/datastrato/gravitino/TestTableChange.java
@@ -23,7 +23,8 @@
 import com.datastrato.gravitino.rel.TableChange.UpdateColumnPosition;
 import com.datastrato.gravitino.rel.TableChange.UpdateColumnType;
 import com.datastrato.gravitino.rel.TableChange.UpdateComment;
-import io.substrait.type.Type;
+import com.datastrato.gravitino.rel.types.Type;
+import com.datastrato.gravitino.rel.types.Types;
 import org.junit.jupiter.api.Test;
 
 public class TestTableChange {
@@ -67,7 +68,7 @@ public void testColumnPosition() {
   @Test
   public void testAddColumn() {
     String[] fieldNames = {"Name"};
-    Type dataType = Type.withNullability(false).STRING;
+    Type dataType = Types.StringType.get();
     String comment = "Person name";
     AddColumn addColumn = (AddColumn) TableChange.addColumn(fieldNames, dataType, comment);
 
@@ -80,7 +81,7 @@ public void testAddColumn() {
   @Test
   public void testAddColumnWithPosition() {
     String[] fieldNames = {"Full Name", "First Name"};
-    Type dataType = Type.withNullability(false).STRING;
+    Type dataType = Types.StringType.get();
     String comment = "First or given name";
     TableChange.ColumnPosition position = TableChange.ColumnPosition.after("Address");
     AddColumn addColumn =
@@ -95,7 +96,7 @@ public void testAddColumnWithPosition() {
   @Test
   public void testAddColumnWithNullCommentAndPosition() {
     String[] fieldNames = {"Middle Name"};
-    Type dataType = Type.withNullability(false).STRING;
+    Type dataType = Types.StringType.get();
     AddColumn addColumn = (AddColumn) TableChange.addColumn(fieldNames, dataType, null, null);
 
     assertArrayEquals(fieldNames, addColumn.fieldNames());
@@ -191,7 +192,7 @@ public void testDeleteNestedColumn() {
   @Test
   public void testUpdateColumnType() {
     String[] fieldNames = {"existing_column"};
-    Type dataType = Type.withNullability(false).STRING;
+    Type dataType = Types.StringType.get();
     UpdateColumnType updateColumnType =
         (UpdateColumnType) TableChange.updateColumnType(fieldNames, dataType);
 
@@ -202,7 +203,7 @@ public void testUpdateColumnType() {
   @Test
   public void testUpdateNestedColumnType() {
     String[] fieldNames = {"nested", "existing_column"};
-    Type dataType = Type.withNullability(false).STRING;
+    Type dataType = Types.StringType.get();
     UpdateColumnType updateColumnType =
         (UpdateColumnType) TableChange.updateColumnType(fieldNames, dataType);
 
@@ -347,7 +348,7 @@ void testColumnRenameNotEqualsAndHashCode() {
   @Test
   void testColumnUpdateTypeEqualsAndHashCode() {
     String[] nameA = {"First Name"};
-    Type dataType = Type.withNullability(false).STRING;
+    Type dataType = Types.StringType.get();
     UpdateColumnType columnA = (UpdateColumnType) TableChange.updateColumnType(nameA, dataType);
     String[] nameB = {"First Name"};
     UpdateColumnType columnB = (UpdateColumnType) TableChange.updateColumnType(nameB, dataType);
@@ -360,7 +361,7 @@ void testColumnUpdateTypeEqualsAndHashCode() {
   @Test
   void testColumnUpdateTypeNotEqualsAndHashCode() {
     String[] nameA = {"First Name"};
-    Type dataType = Type.withNullability(false).STRING;
+    Type dataType = Types.StringType.get();
     UpdateColumnType columnA = (UpdateColumnType) TableChange.updateColumnType(nameA, dataType);
     String[] nameB = {"Given Name"};
     UpdateColumnType columnB = (UpdateColumnType) TableChange.updateColumnType(nameB, dataType);
@@ -492,11 +493,11 @@ void testSetPropertyNotEqualsAndHashCode() {
   @Test
   void testAddColumnEqualsAndHashCode() {
     String[] fieldNamesA = {"Name"};
-    Type dataTypeA = Type.withNullability(false).STRING;
+    Type dataTypeA = Types.StringType.get();
     String commentA = "Person name";
     AddColumn columnA = (AddColumn) TableChange.addColumn(fieldNamesA, dataTypeA, commentA);
     String[] fieldNamesB = {"Name"};
-    Type dataTypeB = Type.withNullability(false).STRING;
+    Type dataTypeB = Types.StringType.get();
     String commentB = "Person name";
     AddColumn columnB = (AddColumn) TableChange.addColumn(fieldNamesB, dataTypeB, commentB);
 
@@ -508,11 +509,11 @@ void testAddColumnEqualsAndHashCode() {
   @Test
   void testAddColumnNotEqualsAndHashCode() {
     String[] fieldNamesA = {"Name"};
-    Type dataTypeA = Type.withNullability(false).STRING;
+    Type dataTypeA = Types.StringType.get();
     String commentA = "Person name";
     AddColumn columnA = (AddColumn) TableChange.addColumn(fieldNamesA, dataTypeA, commentA);
     String[] fieldNamesB = {"First Name"};
-    Type dataTypeB = Type.withNullability(false).STRING;
+    Type dataTypeB = Types.StringType.get();
     String commentB = "Person name";
     AddColumn columnB = (AddColumn) TableChange.addColumn(fieldNamesB, dataTypeB, commentB);
 

diff --git a/api/src/test/java/com/datastrato/gravitino/rel/TestTransforms.java b/api/src/test/java/com/datastrato/gravitino/rel/TestTransforms.java
@@ -21,8 +21,8 @@
 import com.datastrato.gravitino.rel.expressions.Literal;
 import com.datastrato.gravitino.rel.expressions.NamedReference;
 import com.datastrato.gravitino.rel.expressions.transforms.Transform;
-import io.substrait.type.Type;
-import io.substrait.type.TypeCreator;
+import com.datastrato.gravitino.rel.types.Type;
+import com.datastrato.gravitino.rel.types.Types;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
 
@@ -38,7 +38,7 @@ public String name() {
 
           @Override
           public Type dataType() {
-            return TypeCreator.NULLABLE.I8;
+            return Types.ByteType.get();
           }
 
           @Override
@@ -85,7 +85,7 @@ public String name() {
 
           @Override
           public Type dataType() {
-            return TypeCreator.NULLABLE.I8;
+            return Types.ByteType.get();
           }
 
           @Override

diff --git a/catalogs/catalog-hive/build.gradle.kts b/catalogs/catalog-hive/build.gradle.kts
@@ -65,12 +65,6 @@ dependencies {
         exclude("*")
     }
 
-    implementation(libs.substrait.java.core) {
-        exclude("org.slf4j")
-        exclude("com.fasterxml.jackson.core")
-        exclude("com.fasterxml.jackson.datatype")
-    }
-
     implementation(libs.slf4j.api)
     implementation(libs.guava)
 

diff --git a/...talog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveCatalogOperations.java b/...talog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveCatalogOperations.java
@@ -778,7 +778,7 @@ private void doAddColumn(List<FieldSchema> cols, TableChange.AddColumn change) {
         targetPosition,
         new FieldSchema(
             change.fieldNames()[0],
-            change.getDataType().accept(ToHiveType.INSTANCE).getQualifiedName(),
+            ToHiveType.convert(change.getDataType()).getQualifiedName(),
             change.getComment()));
   }
 
@@ -826,8 +826,7 @@ private void doUpdateColumnType(List<FieldSchema> cols, TableChange.UpdateColumn
     if (indexOfColumn == -1) {
       throw new IllegalArgumentException("UpdateColumnType does not exist: " + columnName);
     }
-    cols.get(indexOfColumn)
-        .setType(change.getNewDataType().accept(ToHiveType.INSTANCE).getQualifiedName());
+    cols.get(indexOfColumn).setType(ToHiveType.convert(change.getNewDataType()).getQualifiedName());
   }
 
   /**

diff --git a/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveTable.java b/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveTable.java
@@ -213,7 +213,7 @@ private FieldSchema getPartitionKey(String[] fieldName) {
             .collect(Collectors.toList());
     return new FieldSchema(
         partitionColumns.get(0).name(),
-        partitionColumns.get(0).dataType().accept(ToHiveType.INSTANCE).getQualifiedName(),
+        ToHiveType.convert(partitionColumns.get(0).dataType()).getQualifiedName(),
         partitionColumns.get(0).comment());
   }
 
@@ -228,9 +228,7 @@ private StorageDescriptor buildStorageDescriptor(
             .map(
                 c ->
                     new FieldSchema(
-                        c.name(),
-                        c.dataType().accept(ToHiveType.INSTANCE).getQualifiedName(),
-                        c.comment()))
+                        c.name(), ToHiveType.convert(c.dataType()).getQualifiedName(), c.comment()))
             .collect(Collectors.toList()));
 
     // `location` must not be null, otherwise it will result in an NPE when calling HMS `alterTable`