Taking out AvroParquet test for now to remove Avro dependency

Author: AndreSchumacher

project/SparkBuild.scala

@@ -503,9 +503,6 @@ object SparkBuild extends Build {
     libraryDependencies ++= Seq(
       "com.twitter"                  % "parquet-column"             % parquetVersion,
       "com.twitter"                  % "parquet-hadoop"             % parquetVersion,
-      "com.twitter"                  % "parquet-avro"               % parquetVersion      % "test",
-      // here we need version >= 1.7.5 because of AVRO-1274
-      "org.apache.avro"              % "avro"                       % "1.7.6"             % "test",
       "com.fasterxml.jackson.core"   % "jackson-databind"           % "2.3.0" // json4s-jackson 3.2.6 requires jackson-databind 2.3.0.
     ),
     initialCommands in console :=

sql/core/src/test/scala/org/apache/spark/sql/parquet/ParquetQuerySuite.scala

@@ -17,17 +17,11 @@
 
 package org.apache.spark.sql.parquet
 
-import java.io.File
-
 import org.scalatest.{BeforeAndAfterAll, FunSuiteLike}
 
-import org.apache.avro.{SchemaBuilder, Schema}
-import org.apache.avro.generic.{GenericData, GenericRecord}
-
 import org.apache.hadoop.fs.{Path, FileSystem}
 import org.apache.hadoop.mapreduce.Job
 
-import parquet.avro.AvroParquetWriter
 import parquet.hadoop.ParquetFileWriter
 import parquet.hadoop.util.ContextUtil
 import parquet.schema.MessageTypeParser
@@ -679,147 +673,6 @@ class ParquetQuerySuite extends QueryTest with FunSuiteLike with BeforeAndAfterA
     assert(result3(0)(1) === "the answer")
     Utils.deleteRecursively(tmpdir)
   }
-
-  test("Importing data generated with Avro") {
-    val tmpdir = Utils.createTempDir()
-    val file: File = new File(tmpdir, "test.avro")
-
-    val primitiveArrayType: Schema = SchemaBuilder.array.items.intType
-    val complexArrayType: Schema = SchemaBuilder.array.items.map.values.stringType
-    val primitiveMapType: Schema = SchemaBuilder.map.values.booleanType
-    val complexMapType: Schema = SchemaBuilder.map.values.array.items.floatType
-    val schema: Schema = SchemaBuilder
-      .record("TestRecord")
-      .namespace("")
-      .fields
-        .name("testInt")
-          .`type`.
-          intType
-          .noDefault
-      .name("testDouble")
-        .`type`
-        .doubleType
-        .noDefault
-      .name("testString")
-        .`type`
-        .nullable
-        .stringType
-        .stringDefault("")
-      .name("testPrimitiveArray")
-        .`type`(primitiveArrayType)
-        .noDefault
-      .name("testComplexArray")
-        .`type`(complexArrayType)
-        .noDefault
-      .name("testPrimitiveMap")
-        .`type`(primitiveMapType)
-        .noDefault
-      .name("testComplexMap")
-        .`type`(complexMapType)
-        .noDefault
-      .endRecord
-
-    val record1: GenericRecord = new GenericData.Record(schema)
-
-    // primitive fields
-    record1.put("testInt", 256)
-    record1.put("testDouble", 0.5)
-    record1.put("testString", "foo")
-
-    val primitiveArrayData = new GenericData.Array[Integer](10, primitiveArrayType)
-    val complexArrayData: GenericData.Array[java.util.Map[String, String]] =
-      new GenericData.Array[java.util.Map[String, String]](10, SchemaBuilder.array.items.map.values.stringType)
-
-    // two arrays: one primitive (array of ints), one complex (array of string->string maps)
-    primitiveArrayData.add(1)
-    primitiveArrayData.add(2)
-    primitiveArrayData.add(3)
-    val map1 = new java.util.HashMap[String, String]
-    map1.put("key11", "data11")
-    map1.put("key12", "data12")
-    val map2 = new java.util.HashMap[String, String]
-    map2.put("key21", "data21")
-    map2.put("key22", "data22")
-    complexArrayData.add(0, map1)
-    complexArrayData.add(1, map2)
-    
-    record1.put("testPrimitiveArray", primitiveArrayData)
-    record1.put("testComplexArray", complexArrayData)
-
-    // two maps: one primitive (string->boolean), one complex (string->array of floats)
-    val primitiveMap = new java.util.HashMap[String, Boolean](10)
-    primitiveMap.put("key1", true)
-    primitiveMap.put("key2", false)
-    val complexMap = new java.util.HashMap[String, GenericData.Array[Float]](10)
-    val value1: GenericData.Array[Float] = new GenericData.Array[Float](10, SchemaBuilder.array.items.floatType)
-    value1.add(0.1f)
-    value1.add(0.2f)
-    value1.add(0.3f)
-    complexMap.put("compKey1", value1)
-    val value2: GenericData.Array[Float] = new GenericData.Array[Float](10, SchemaBuilder.array.items.floatType)
-    value2.add(1.1f)
-    value2.add(1.2f)
-    value2.add(1.3f)
-    complexMap.put("compKey2", value2)
-
-    record1.put("testPrimitiveMap", primitiveMap)
-    record1.put("testComplexMap", complexMap)
-
-    // TODO: test array or map with value type Avro record
-
-    val writer = new AvroParquetWriter[GenericRecord](new Path(file.toString), schema)
-    writer.write(record1)
-    writer.close()
-
-    val data = TestSQLContext
-      .parquetFile(tmpdir.toString)
-      .toSchemaRDD
-    data.registerAsTable("avroTable")
-    val resultPrimitives = sql("SELECT testInt, testDouble, testString FROM avroTable").collect()
-    assert(resultPrimitives(0)(0) === 256)
-    assert(resultPrimitives(0)(1) === 0.5)
-    assert(resultPrimitives(0)(2) === "foo")
-    val resultPrimitiveArray = sql("SELECT testPrimitiveArray FROM avroTable").collect()
-    assert(resultPrimitiveArray(0)(0)
-      .asInstanceOf[CatalystConverter.ArrayScalaType[_]](0) === 1)
-    assert(resultPrimitiveArray(0)(0)
-      .asInstanceOf[CatalystConverter.ArrayScalaType[_]](1) === 2)
-    assert(resultPrimitiveArray(0)(0)
-      .asInstanceOf[CatalystConverter.ArrayScalaType[_]](2) === 3)
-    val resultComplexArray = sql("SELECT testComplexArray FROM avroTable").collect()
-    assert(resultComplexArray(0)(0)
-      .asInstanceOf[CatalystConverter.ArrayScalaType[_]].size === 2)
-    assert(
-      resultComplexArray(0)(0)
-      .asInstanceOf[CatalystConverter.ArrayScalaType[_]](0)
-      .asInstanceOf[CatalystConverter.MapScalaType[String, String]]
-        .get("key11").get.equals("data11"))
-    assert(
-      resultComplexArray(0)(0)
-      .asInstanceOf[CatalystConverter.ArrayScalaType[_]](1)
-      .asInstanceOf[CatalystConverter.MapScalaType[String, String]]
-        .get("key22").get.equals("data22"))
-    val resultPrimitiveMap = sql("SELECT testPrimitiveMap FROM avroTable").collect()
-    assert(
-      resultPrimitiveMap(0)(0)
-      .asInstanceOf[CatalystConverter.MapScalaType[String, Boolean]].get("key1").get === true)
-    assert(
-      resultPrimitiveMap(0)(0)
-      .asInstanceOf[CatalystConverter.MapScalaType[String, Boolean]].get("key2").get === false)
-    val resultComplexMap = sql("SELECT testComplexMap FROM avroTable").collect()
-    val mapResult1 =
-      resultComplexMap(0)(0)
-      .asInstanceOf[CatalystConverter.MapScalaType[String, CatalystConverter.ArrayScalaType[_]]]
-      .get("compKey1").get
-    val mapResult2 =
-      resultComplexMap(0)(0)
-        .asInstanceOf[CatalystConverter.MapScalaType[String, CatalystConverter.ArrayScalaType[_]]]
-        .get("compKey2").get
-    assert(mapResult1(0) === 0.1f)
-    assert(mapResult1(2) === 0.3f)
-    assert(mapResult2(0) === 1.1f)
-    assert(mapResult2(2) === 1.3f)
-  }
 }
 
 // TODO: the code below is needed temporarily until the standard parser is able to parse