HIVE-21737: Make Avro use in Hive compatible with Avro 1.9.x

Co-Authored-By: Ismaël Mejía <iemejia@gmail.com>

Author: Fokko

pom.xml

@@ -502,7 +502,6 @@
       <dependency>
         <groupId>org.apache.avro</groupId>
         <artifactId>avro-mapred</artifactId>
-        <classifier>hadoop2</classifier>
         <version>${avro.version}</version>
         <exclusions>
           <exclusion>

ql/pom.xml

@@ -220,7 +220,7 @@
     <dependency>
       <groupId>org.apache.avro</groupId>
       <artifactId>avro-mapred</artifactId>
-      <classifier>hadoop2</classifier>
+      <version>${avro.version}</version>
     </dependency>
     <dependency>
       <groupId>org.apache.ant</groupId>

serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroDeserializer.java

@@ -278,7 +278,7 @@ private Object deserializePrimitive(Object datum, Schema fileSchema, Schema reco
 
       int scale = 0;
       try {
-        scale = fileSchema.getJsonProp(AvroSerDe.AVRO_PROP_SCALE).asInt();
+        scale = (int) fileSchema.getObjectProp(AvroSerDe.AVRO_PROP_SCALE);
       } catch(Exception ex) {
         throw new AvroSerdeException("Failed to obtain scale value from file schema: " + fileSchema, ex);
       }
@@ -294,7 +294,7 @@ private Object deserializePrimitive(Object datum, Schema fileSchema, Schema reco
 
       int maxLength = 0;
       try {
-        maxLength = fileSchema.getJsonProp(AvroSerDe.AVRO_PROP_MAX_LENGTH).getValueAsInt();
+        maxLength = (int) fileSchema.getObjectProp(AvroSerDe.AVRO_PROP_MAX_LENGTH);
       } catch (Exception ex) {
         throw new AvroSerdeException("Failed to obtain maxLength value for char field from file schema: " + fileSchema, ex);
       }
@@ -309,7 +309,7 @@ private Object deserializePrimitive(Object datum, Schema fileSchema, Schema reco
 
       maxLength = 0;
       try {
-        maxLength = fileSchema.getJsonProp(AvroSerDe.AVRO_PROP_MAX_LENGTH).getValueAsInt();
+        maxLength = (int) fileSchema.getObjectProp(AvroSerDe.AVRO_PROP_MAX_LENGTH);
       } catch (Exception ex) {
         throw new AvroSerdeException("Failed to obtain maxLength value for varchar field from file schema: " + fileSchema, ex);
       }

serde/src/java/org/apache/hadoop/hive/serde2/avro/SchemaToTypeInfo.java

@@ -136,8 +136,8 @@ public static TypeInfo generateTypeInfo(Schema schema,
       int precision = 0;
       int scale = 0;
       try {
-        precision = schema.getJsonProp(AvroSerDe.AVRO_PROP_PRECISION).getIntValue();
-        scale = schema.getJsonProp(AvroSerDe.AVRO_PROP_SCALE).getIntValue();
+        precision = (int) schema.getObjectProp(AvroSerDe.AVRO_PROP_PRECISION);
+        scale = (int) schema.getObjectProp(AvroSerDe.AVRO_PROP_SCALE);
       } catch (Exception ex) {
         throw new AvroSerdeException("Failed to obtain scale value from file schema: " + schema, ex);
       }
@@ -155,7 +155,7 @@ public static TypeInfo generateTypeInfo(Schema schema,
       AvroSerDe.CHAR_TYPE_NAME.equalsIgnoreCase(schema.getProp(AvroSerDe.AVRO_PROP_LOGICAL_TYPE))) {
       int maxLength = 0;
       try {
-        maxLength = schema.getJsonProp(AvroSerDe.AVRO_PROP_MAX_LENGTH).getValueAsInt();
+        maxLength = (int) schema.getObjectProp(AvroSerDe.AVRO_PROP_MAX_LENGTH);
       } catch (Exception ex) {
         throw new AvroSerdeException("Failed to obtain maxLength value from file schema: " + schema, ex);
       }
@@ -166,7 +166,7 @@ public static TypeInfo generateTypeInfo(Schema schema,
       .equalsIgnoreCase(schema.getProp(AvroSerDe.AVRO_PROP_LOGICAL_TYPE))) {
       int maxLength = 0;
       try {
-        maxLength = schema.getJsonProp(AvroSerDe.AVRO_PROP_MAX_LENGTH).getValueAsInt();
+        maxLength = (int) schema.getObjectProp(AvroSerDe.AVRO_PROP_MAX_LENGTH);
       } catch (Exception ex) {
         throw new AvroSerdeException("Failed to obtain maxLength value from file schema: " + schema, ex);
       }

serde/src/java/org/apache/hadoop/hive/serde2/avro/TypeInfoToSchema.java

@@ -28,8 +28,6 @@
 import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
 import org.apache.hadoop.hive.serde2.typeinfo.UnionTypeInfo;
 import org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo;
-import org.codehaus.jackson.JsonNode;
-import org.codehaus.jackson.node.JsonNodeFactory;
 
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -42,6 +40,13 @@ public class TypeInfoToSchema {
 
   private long recordCounter = 0;
 
+  /**
+   * This constant value must be equal to org.apache.avro.Schema.Field.NULL_DEFAULT_VALUE
+   * starting with Avro 1.9.x, but it is not available in Avro 1.8.x so we define it here
+   * for backwards compatibility.
+   */
+  private static final Object NULL_DEFAULT_VALUE = new Object();
+
   /**
    * Converts Hive schema to avro schema
    *
@@ -235,14 +240,13 @@ private Schema createAvroArray(TypeInfo typeInfo) {
   private List<Schema.Field> getFields(Schema.Field schemaField) {
     List<Schema.Field> fields = new ArrayList<Schema.Field>();
 
-    JsonNode nullDefault = JsonNodeFactory.instance.nullNode();
     if (schemaField.schema().getType() == Schema.Type.RECORD) {
       for (Schema.Field field : schemaField.schema().getFields()) {
-        fields.add(new Schema.Field(field.name(), field.schema(), field.doc(), nullDefault));
+        fields.add(new Schema.Field(field.name(), field.schema(), field.doc(), NULL_DEFAULT_VALUE));
       }
     } else {
       fields.add(new Schema.Field(schemaField.name(), schemaField.schema(), schemaField.doc(),
-          nullDefault));
+          NULL_DEFAULT_VALUE));
     }
 
     return fields;