Support 'SELECT NULL null_field' and union schemas with NULL type fie…

…ld in coral-schema (#71) * Support 'SELECT NULL null_field' and union schemas with NULL type field in coral-schema
linkedin · Apr 13, 2021 · d948118 · d948118
1 parent 1f16d96
commit d948118
Show file tree

Hide file tree

Showing 5 changed files with 38 additions and 2 deletions.
diff --git a/coral-schema/src/main/java/com/linkedin/coral/schema/avro/SchemaUtilities.java b/coral-schema/src/main/java/com/linkedin/coral/schema/avro/SchemaUtilities.java
@@ -171,7 +171,7 @@ static void appendField(@Nonnull String fieldName, @Nonnull RelDataType fieldRel
     Schema fieldSchema = RelDataTypeToAvroType.relDataTypeToAvroTypeNonNullable(fieldRelDataType, fieldName);
 
     // TODO: handle default value properly
-    if (isNullable) {
+    if (isNullable && fieldSchema.getType() != Schema.Type.NULL) {
       Schema fieldSchemaNullable = Schema.createUnion(Arrays.asList(fieldSchema, Schema.create(Schema.Type.NULL)));
       fieldAssembler.name(fieldName).type(fieldSchemaNullable).noDefault();
     } else {
@@ -436,6 +436,7 @@ private static boolean isUnionSchemaCompatible(@Nonnull Schema leftSchema, @Nonn
       case INT:
       case LONG:
       case STRING:
+      case NULL:
         return leftSchema.getType() == rightSchema.getType();
       case FIXED:
         boolean isSameType = leftSchema.getType() == rightSchema.getType();
@@ -526,6 +527,7 @@ private static Schema setupNestedNamespaceForRecord(@Nonnull Schema schema, @Non
         case LONG:
         case STRING:
         case FIXED:
+        case NULL:
           // TODO: verify whether FIXED type has namespace
           appendField(field, fieldAssembler);
           break;

diff --git a/coral-schema/src/test/java/com/linkedin/coral/schema/avro/TestUtils.java b/coral-schema/src/test/java/com/linkedin/coral/schema/avro/TestUtils.java
@@ -1,5 +1,5 @@
 /**
- * Copyright 2019-2020 LinkedIn Corporation. All rights reserved.
+ * Copyright 2019-2021 LinkedIn Corporation. All rights reserved.
  * Licensed under the BSD-2 Clause license.
  * See LICENSE in the project root for license information.
  */
@@ -82,12 +82,14 @@ private static void initializeTables() {
     String baseCasePreservation = loadSchema("base-casepreservation.avsc");
     String baseComplexFieldSchema = loadSchema("base-complex-fieldschema");
     String baseNestedComplexSchema = loadSchema("base-nested-complex.avsc");
+    String baseNullTypeFieldSchema = loadSchema("base-null-type-field.avsc");
 
     executeCreateTableQuery("default", "basecomplex", baseComplexSchema);
     executeCreateTableQuery("default", "basecomplexunioncompatible", baseComplexUnionCompatible);
     executeCreateTableQuery("default", "baseenum", baseEnumSchema);
     executeCreateTableQuery("default", "baselateralview", baseLateralViewSchema);
     executeCreateTableQuery("default", "basenullability", baseNullabilitySchema);
+    executeCreateTableQuery("default", "basenulltypefield", baseNullTypeFieldSchema);
     executeCreateTableWithPartitionQuery("default", "basecasepreservation", baseCasePreservation);
     executeCreateTableWithPartitionFieldSchemaQuery("default", "basecomplexfieldschema", baseComplexFieldSchema);
     executeCreateTableWithPartitionQuery("default", "basenestedcomplex", baseNestedComplexSchema);

diff --git a/...l-schema/src/test/java/com/linkedin/coral/schema/avro/ViewToAvroSchemaConverterTests.java b/...l-schema/src/test/java/com/linkedin/coral/schema/avro/ViewToAvroSchemaConverterTests.java
@@ -617,5 +617,19 @@ public void testSubQueryWhere() {
     // TODO: implement this test
   }
 
+  @Test
+  public void testSelectNullUnionNullField() {
+    String viewSql =
+        "CREATE VIEW v AS SELECT NULL Null_Field FROM basecomplex UNION ALL SELECT Null_Field FROM basenulltypefield";
+
+    TestUtils.executeCreateViewQuery("default", "v", viewSql);
+
+    ViewToAvroSchemaConverter viewToAvroSchemaConverter = ViewToAvroSchemaConverter.create(hiveMetastoreClient);
+    Schema actualSchema = viewToAvroSchemaConverter.toAvroSchema("default", "v", false);
+
+    Assert.assertEquals(actualSchema.toString(true),
+        TestUtils.loadSchema("testSelectNullUnionNullField-expected.avsc"));
+  }
+
   // TODO: add more unit tests
 }
diff --git a/coral-schema/src/test/resources/base-null-type-field.avsc b/coral-schema/src/test/resources/base-null-type-field.avsc
@@ -0,0 +1,9 @@
+{
+  "type" : "record",
+  "name" : "basenulltypefield",
+  "namespace" : "coral.schema.avro.base.null.type.field",
+  "fields" : [ {
+    "name" : "Null_Field",
+    "type" : "null"
+  } ]
+}
diff --git a/coral-schema/src/test/resources/testSelectNullUnionNullField-expected.avsc b/coral-schema/src/test/resources/testSelectNullUnionNullField-expected.avsc
@@ -0,0 +1,9 @@
+{
+  "type" : "record",
+  "name" : "v",
+  "namespace" : "default.v",
+  "fields" : [ {
+    "name" : "Null_Field",
+    "type" : "null"
+  } ]
+}