[DO-NOT-MERGE] Test compatibility for Parquet 1.11.1, Avro 1.10.1 and Hive 2.3.8

core/pom.xml

@@ -35,18 +35,13 @@
   </properties>
 
   <dependencies>
-    <dependency>
-      <groupId>com.thoughtworks.paranamer</groupId>
-      <artifactId>paranamer</artifactId>
-    </dependency>
     <dependency>
       <groupId>org.apache.avro</groupId>
       <artifactId>avro</artifactId>
     </dependency>
     <dependency>
       <groupId>org.apache.avro</groupId>
       <artifactId>avro-mapred</artifactId>
-      <classifier>${avro.mapred.classifier}</classifier>
     </dependency>
     <dependency>
       <groupId>com.google.guava</groupId>

dev/deps/spark-deps-hadoop-2.7-hive-2.3

@@ -22,9 +22,9 @@ arrow-memory-netty/2.0.0//arrow-memory-netty-2.0.0.jar
 arrow-vector/2.0.0//arrow-vector-2.0.0.jar
 audience-annotations/0.5.0//audience-annotations-0.5.0.jar
 automaton/1.11-8//automaton-1.11-8.jar
-avro-ipc/1.8.2//avro-ipc-1.8.2.jar
-avro-mapred/1.8.2/hadoop2/avro-mapred-1.8.2-hadoop2.jar
-avro/1.8.2//avro-1.8.2.jar
+avro-ipc/1.10.1//avro-ipc-1.10.1.jar
+avro-mapred/1.10.1//avro-mapred-1.10.1.jar
+avro/1.10.1//avro-1.10.1.jar
 bonecp/0.8.0.RELEASE//bonecp-0.8.0.RELEASE.jar
 breeze-macros_2.12/1.0//breeze-macros_2.12-1.0.jar
 breeze_2.12/1.0//breeze_2.12-1.0.jar
@@ -81,21 +81,21 @@ hadoop-yarn-client/2.7.4//hadoop-yarn-client-2.7.4.jar
 hadoop-yarn-common/2.7.4//hadoop-yarn-common-2.7.4.jar
 hadoop-yarn-server-common/2.7.4//hadoop-yarn-server-common-2.7.4.jar
 hadoop-yarn-server-web-proxy/2.7.4//hadoop-yarn-server-web-proxy-2.7.4.jar
-hive-beeline/2.3.7//hive-beeline-2.3.7.jar
-hive-cli/2.3.7//hive-cli-2.3.7.jar
-hive-common/2.3.7//hive-common-2.3.7.jar
-hive-exec/2.3.7/core/hive-exec-2.3.7-core.jar
-hive-jdbc/2.3.7//hive-jdbc-2.3.7.jar
-hive-llap-common/2.3.7//hive-llap-common-2.3.7.jar
-hive-metastore/2.3.7//hive-metastore-2.3.7.jar
-hive-serde/2.3.7//hive-serde-2.3.7.jar
+hive-beeline/2.3.8//hive-beeline-2.3.8.jar
+hive-cli/2.3.8//hive-cli-2.3.8.jar
+hive-common/2.3.8//hive-common-2.3.8.jar
+hive-exec/2.3.8/core/hive-exec-2.3.8-core.jar
+hive-jdbc/2.3.8//hive-jdbc-2.3.8.jar
+hive-llap-common/2.3.8//hive-llap-common-2.3.8.jar
+hive-metastore/2.3.8//hive-metastore-2.3.8.jar
+hive-serde/2.3.8//hive-serde-2.3.8.jar
 hive-service-rpc/3.1.2//hive-service-rpc-3.1.2.jar
-hive-shims-0.23/2.3.7//hive-shims-0.23-2.3.7.jar
-hive-shims-common/2.3.7//hive-shims-common-2.3.7.jar
-hive-shims-scheduler/2.3.7//hive-shims-scheduler-2.3.7.jar
-hive-shims/2.3.7//hive-shims-2.3.7.jar
+hive-shims-0.23/2.3.8//hive-shims-0.23-2.3.8.jar
+hive-shims-common/2.3.8//hive-shims-common-2.3.8.jar
+hive-shims-scheduler/2.3.8//hive-shims-scheduler-2.3.8.jar
+hive-shims/2.3.8//hive-shims-2.3.8.jar
 hive-storage-api/2.7.2//hive-storage-api-2.7.2.jar
-hive-vector-code-gen/2.3.7//hive-vector-code-gen-2.3.7.jar
+hive-vector-code-gen/2.3.8//hive-vector-code-gen-2.3.8.jar
 hk2-api/2.6.1//hk2-api-2.6.1.jar
 hk2-locator/2.6.1//hk2-locator-2.6.1.jar
 hk2-utils/2.6.1//hk2-utils-2.6.1.jar
@@ -125,6 +125,7 @@ jakarta.ws.rs-api/2.1.6//jakarta.ws.rs-api-2.1.6.jar
 jakarta.xml.bind-api/2.3.2//jakarta.xml.bind-api-2.3.2.jar
 janino/3.0.16//janino-3.0.16.jar
 javassist/3.25.0-GA//javassist-3.25.0-GA.jar
+javax.annotation-api/1.3.2//javax.annotation-api-1.3.2.jar
 javax.inject/1//javax.inject-1.jar
 javax.jdo/3.2.0-m3//javax.jdo-3.2.0-m3.jar
 javolution/5.5.1//javolution-5.5.1.jar
@@ -202,12 +203,12 @@ orc-shims/1.6.6//orc-shims-1.6.6.jar
 oro/2.0.8//oro-2.0.8.jar
 osgi-resource-locator/1.0.3//osgi-resource-locator-1.0.3.jar
 paranamer/2.8//paranamer-2.8.jar
-parquet-column/1.10.1//parquet-column-1.10.1.jar
-parquet-common/1.10.1//parquet-common-1.10.1.jar
-parquet-encoding/1.10.1//parquet-encoding-1.10.1.jar
-parquet-format/2.4.0//parquet-format-2.4.0.jar
-parquet-hadoop/1.10.1//parquet-hadoop-1.10.1.jar
-parquet-jackson/1.10.1//parquet-jackson-1.10.1.jar
+parquet-column/1.11.1//parquet-column-1.11.1.jar
+parquet-common/1.11.1//parquet-common-1.11.1.jar
+parquet-encoding/1.11.1//parquet-encoding-1.11.1.jar
+parquet-format-structures/1.11.1//parquet-format-structures-1.11.1.jar
+parquet-hadoop/1.11.1//parquet-hadoop-1.11.1.jar
+parquet-jackson/1.11.1//parquet-jackson-1.11.1.jar
 protobuf-java/2.5.0//protobuf-java-2.5.0.jar
 py4j/0.10.9.1//py4j-0.10.9.1.jar
 pyrolite/4.30//pyrolite-4.30.jar
@@ -234,12 +235,13 @@ super-csv/2.2.0//super-csv-2.2.0.jar
 threeten-extra/1.5.0//threeten-extra-1.5.0.jar
 transaction-api/1.1//transaction-api-1.1.jar
 univocity-parsers/2.9.0//univocity-parsers-2.9.0.jar
+velocity-engine-core/2.2//velocity-engine-core-2.2.jar
 velocity/1.5//velocity-1.5.jar
 xbean-asm7-shaded/4.15//xbean-asm7-shaded-4.15.jar
 xercesImpl/2.12.0//xercesImpl-2.12.0.jar
 xml-apis/1.4.01//xml-apis-1.4.01.jar
 xmlenc/0.52//xmlenc-0.52.jar
-xz/1.5//xz-1.5.jar
+xz/1.8//xz-1.8.jar
 zjsonpatch/0.3.0//zjsonpatch-0.3.0.jar
 zookeeper/3.4.14//zookeeper-3.4.14.jar
 zstd-jni/1.4.8-1//zstd-jni-1.4.8-1.jar

dev/deps/spark-deps-hadoop-3.2-hive-2.3

@@ -19,9 +19,9 @@ arrow-memory-netty/2.0.0//arrow-memory-netty-2.0.0.jar
 arrow-vector/2.0.0//arrow-vector-2.0.0.jar
 audience-annotations/0.5.0//audience-annotations-0.5.0.jar
 automaton/1.11-8//automaton-1.11-8.jar
-avro-ipc/1.8.2//avro-ipc-1.8.2.jar
-avro-mapred/1.8.2/hadoop2/avro-mapred-1.8.2-hadoop2.jar
-avro/1.8.2//avro-1.8.2.jar
+avro-ipc/1.10.1//avro-ipc-1.10.1.jar
+avro-mapred/1.10.1//avro-mapred-1.10.1.jar
+avro/1.10.1//avro-1.10.1.jar
 bonecp/0.8.0.RELEASE//bonecp-0.8.0.RELEASE.jar
 breeze-macros_2.12/1.0//breeze-macros_2.12-1.0.jar
 breeze_2.12/1.0//breeze_2.12-1.0.jar
@@ -80,21 +80,21 @@ hadoop-yarn-common/3.2.0//hadoop-yarn-common-3.2.0.jar
 hadoop-yarn-registry/3.2.0//hadoop-yarn-registry-3.2.0.jar
 hadoop-yarn-server-common/3.2.0//hadoop-yarn-server-common-3.2.0.jar
 hadoop-yarn-server-web-proxy/3.2.0//hadoop-yarn-server-web-proxy-3.2.0.jar
-hive-beeline/2.3.7//hive-beeline-2.3.7.jar
-hive-cli/2.3.7//hive-cli-2.3.7.jar
-hive-common/2.3.7//hive-common-2.3.7.jar
-hive-exec/2.3.7/core/hive-exec-2.3.7-core.jar
-hive-jdbc/2.3.7//hive-jdbc-2.3.7.jar
-hive-llap-common/2.3.7//hive-llap-common-2.3.7.jar
-hive-metastore/2.3.7//hive-metastore-2.3.7.jar
-hive-serde/2.3.7//hive-serde-2.3.7.jar
+hive-beeline/2.3.8//hive-beeline-2.3.8.jar
+hive-cli/2.3.8//hive-cli-2.3.8.jar
+hive-common/2.3.8//hive-common-2.3.8.jar
+hive-exec/2.3.8/core/hive-exec-2.3.8-core.jar
+hive-jdbc/2.3.8//hive-jdbc-2.3.8.jar
+hive-llap-common/2.3.8//hive-llap-common-2.3.8.jar
+hive-metastore/2.3.8//hive-metastore-2.3.8.jar
+hive-serde/2.3.8//hive-serde-2.3.8.jar
 hive-service-rpc/3.1.2//hive-service-rpc-3.1.2.jar
-hive-shims-0.23/2.3.7//hive-shims-0.23-2.3.7.jar
-hive-shims-common/2.3.7//hive-shims-common-2.3.7.jar
-hive-shims-scheduler/2.3.7//hive-shims-scheduler-2.3.7.jar
-hive-shims/2.3.7//hive-shims-2.3.7.jar
+hive-shims-0.23/2.3.8//hive-shims-0.23-2.3.8.jar
+hive-shims-common/2.3.8//hive-shims-common-2.3.8.jar
+hive-shims-scheduler/2.3.8//hive-shims-scheduler-2.3.8.jar
+hive-shims/2.3.8//hive-shims-2.3.8.jar
 hive-storage-api/2.7.2//hive-storage-api-2.7.2.jar
-hive-vector-code-gen/2.3.7//hive-vector-code-gen-2.3.7.jar
+hive-vector-code-gen/2.3.8//hive-vector-code-gen-2.3.8.jar
 hk2-api/2.6.1//hk2-api-2.6.1.jar
 hk2-locator/2.6.1//hk2-locator-2.6.1.jar
 hk2-utils/2.6.1//hk2-utils-2.6.1.jar
@@ -124,6 +124,7 @@ jakarta.ws.rs-api/2.1.6//jakarta.ws.rs-api-2.1.6.jar
 jakarta.xml.bind-api/2.3.2//jakarta.xml.bind-api-2.3.2.jar
 janino/3.0.16//janino-3.0.16.jar
 javassist/3.25.0-GA//javassist-3.25.0-GA.jar
+javax.annotation-api/1.3.2//javax.annotation-api-1.3.2.jar
 javax.inject/1//javax.inject-1.jar
 javax.jdo/3.2.0-m3//javax.jdo-3.2.0-m3.jar
 javax.servlet-api/3.1.0//javax.servlet-api-3.1.0.jar
@@ -217,12 +218,12 @@ orc-shims/1.6.6//orc-shims-1.6.6.jar
 oro/2.0.8//oro-2.0.8.jar
 osgi-resource-locator/1.0.3//osgi-resource-locator-1.0.3.jar
 paranamer/2.8//paranamer-2.8.jar
-parquet-column/1.10.1//parquet-column-1.10.1.jar
-parquet-common/1.10.1//parquet-common-1.10.1.jar
-parquet-encoding/1.10.1//parquet-encoding-1.10.1.jar
-parquet-format/2.4.0//parquet-format-2.4.0.jar
-parquet-hadoop/1.10.1//parquet-hadoop-1.10.1.jar
-parquet-jackson/1.10.1//parquet-jackson-1.10.1.jar
+parquet-column/1.11.1//parquet-column-1.11.1.jar
+parquet-common/1.11.1//parquet-common-1.11.1.jar
+parquet-encoding/1.11.1//parquet-encoding-1.11.1.jar
+parquet-format-structures/1.11.1//parquet-format-structures-1.11.1.jar
+parquet-hadoop/1.11.1//parquet-hadoop-1.11.1.jar
+parquet-jackson/1.11.1//parquet-jackson-1.11.1.jar
 protobuf-java/2.5.0//protobuf-java-2.5.0.jar
 py4j/0.10.9.1//py4j-0.10.9.1.jar
 pyrolite/4.30//pyrolite-4.30.jar
@@ -251,10 +252,11 @@ threeten-extra/1.5.0//threeten-extra-1.5.0.jar
 token-provider/1.0.1//token-provider-1.0.1.jar
 transaction-api/1.1//transaction-api-1.1.jar
 univocity-parsers/2.9.0//univocity-parsers-2.9.0.jar
+velocity-engine-core/2.2//velocity-engine-core-2.2.jar
 velocity/1.5//velocity-1.5.jar
 woodstox-core/5.0.3//woodstox-core-5.0.3.jar
 xbean-asm7-shaded/4.15//xbean-asm7-shaded-4.15.jar
-xz/1.5//xz-1.5.jar
+xz/1.8//xz-1.8.jar
 zjsonpatch/0.3.0//zjsonpatch-0.3.0.jar
 zookeeper/3.4.14//zookeeper-3.4.14.jar
 zstd-jni/1.4.8-1//zstd-jni-1.4.8-1.jar

docs/building-spark.md

@@ -83,9 +83,9 @@ Example:
 
 To enable Hive integration for Spark SQL along with its JDBC server and CLI,
 add the `-Phive` and `-Phive-thriftserver` profiles to your existing build options.
-By default Spark will build with Hive 2.3.7.
+By default Spark will build with Hive 2.3.8.
 
-    # With Hive 2.3.7 support
+    # With Hive 2.3.8 support
     ./build/mvn -Pyarn -Phive -Phive-thriftserver -DskipTests clean package
 
 ## Packaging without Hadoop Dependencies for YARN

docs/sql-data-sources-avro.md

@@ -339,7 +339,7 @@ applications. Read the [Advanced Dependency Management](https://spark.apache
 Submission Guide for more details. 
 
 ## Supported types for Avro -> Spark SQL conversion
-Currently Spark supports reading all [primitive types](https://avro.apache.org/docs/1.8.2/spec.html#schema_primitive) and [complex types](https://avro.apache.org/docs/1.8.2/spec.html#schema_complex) under records of Avro.
+Currently Spark supports reading all [primitive types](https://avro.apache.org/docs/1.10.1/spec.html#schema_primitive) and [complex types](https://avro.apache.org/docs/1.8.2/spec.html#schema_complex) under records of Avro.
 <table class="table">
   <tr><th><b>Avro type</b></th><th><b>Spark SQL type</b></th></tr>
   <tr>
@@ -403,7 +403,7 @@ In addition to the types listed above, it supports reading `union` types. The fo
 3. `union(something, null)`, where something is any supported Avro type. This will be mapped to the same Spark SQL type as that of something, with nullable set to true.
 All other union types are considered complex. They will be mapped to StructType where field names are member0, member1, etc., in accordance with members of the union. This is consistent with the behavior when converting between Avro and Parquet.
 
-It also supports reading the following Avro [logical types](https://avro.apache.org/docs/1.8.2/spec.html#Logical+Types):
+It also supports reading the following Avro [logical types](https://avro.apache.org/docs/1.10.1/spec.html#Logical+Types):
 
 <table class="table">
   <tr><th><b>Avro logical type</b></th><th><b>Avro type</b></th><th><b>Spark SQL type</b></th></tr>

docs/sql-data-sources-hive-tables.md

@@ -127,10 +127,10 @@ The following options can be used to configure the version of Hive that is used
   <tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
   <tr>
     <td><code>spark.sql.hive.metastore.version</code></td>
-    <td><code>2.3.7</code></td>
+    <td><code>2.3.8</code></td>
     <td>
       Version of the Hive metastore. Available
-      options are <code>0.12.0</code> through <code>2.3.7</code> and <code>3.0.0</code> through <code>3.1.2</code>.
+      options are <code>0.12.0</code> through <code>2.3.8</code> and <code>3.0.0</code> through <code>3.1.2</code>.
     </td>
     <td>1.4.0</td>
   </tr>
@@ -142,9 +142,9 @@ The following options can be used to configure the version of Hive that is used
       property can be one of four options:
       <ol>
         <li><code>builtin</code></li>
-        Use Hive 2.3.7, which is bundled with the Spark assembly when <code>-Phive</code> is
+        Use Hive 2.3.8, which is bundled with the Spark assembly when <code>-Phive</code> is
         enabled. When this option is chosen, <code>spark.sql.hive.metastore.version</code> must be
-        either <code>2.3.7</code> or not defined.
+        either <code>2.3.8</code> or not defined.
         <li><code>maven</code></li>
         Use Hive jars of specified version downloaded from Maven repositories. This configuration
         is not generally recommended for production deployments.

docs/sql-migration-guide.md

@@ -854,7 +854,7 @@ Python UDF registration is unchanged.
 Spark SQL is designed to be compatible with the Hive Metastore, SerDes and UDFs.
 Currently, Hive SerDes and UDFs are based on built-in Hive,
 and Spark SQL can be connected to different versions of Hive Metastore
-(from 0.12.0 to 2.3.7 and 3.0.0 to 3.1.2. Also see [Interacting with Different Versions of Hive Metastore](sql-data-sources-hive-tables.html#interacting-with-different-versions-of-hive-metastore)).
+(from 0.12.0 to 2.3.8 and 3.0.0 to 3.1.2. Also see [Interacting with Different Versions of Hive Metastore](sql-data-sources-hive-tables.html#interacting-with-different-versions-of-hive-metastore)).
 
 #### Deploying in Existing Hive Warehouses
 {:.no_toc}

external/avro/src/main/scala/org/apache/spark/sql/avro/AvroOptions.scala

@@ -51,14 +51,14 @@ private[sql] class AvroOptions(
 
   /**
    * Top level record name in write result, which is required in Avro spec.
-   * See https://avro.apache.org/docs/1.8.2/spec.html#schema_record .
+   * See https://avro.apache.org/docs/1.10.1/spec.html#schema_record .
    * Default value is "topLevelRecord"
    */
   val recordName: String = parameters.getOrElse("recordName", "topLevelRecord")
 
   /**
    * Record namespace in write result. Default value is "".
-   * See Avro spec for details: https://avro.apache.org/docs/1.8.2/spec.html#schema_record .
+   * See Avro spec for details: https://avro.apache.org/docs/1.10.1/spec.html#schema_record .
    */
   val recordNamespace: String = parameters.getOrElse("recordNamespace", "")
 

external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala

@@ -1015,7 +1015,7 @@ abstract class AvroSuite
           .save(s"$tempDir/${UUID.randomUUID()}")
       }.getCause.getMessage
       assert(message.contains("Caused by: java.lang.NullPointerException: " +
-        "in test_schema in string null of string in field Name"))
+        "null of string in string in field Name of test_schema in test_schema"))
     }
   }
 

external/kafka-0-10-assembly/pom.xml

@@ -77,7 +77,6 @@
     <dependency>
       <groupId>org.apache.avro</groupId>
       <artifactId>avro-mapred</artifactId>
-      <classifier>${avro.mapred.classifier}</classifier>
       <scope>provided</scope>
     </dependency>
     <dependency>

external/kinesis-asl-assembly/pom.xml

@@ -94,15 +94,9 @@
       <artifactId>hadoop-client</artifactId>
       <scope>provided</scope>
     </dependency>
-    <dependency>
-      <groupId>org.apache.avro</groupId>
-      <artifactId>avro-ipc</artifactId>
-      <scope>provided</scope>
-    </dependency>
     <dependency>
       <groupId>org.apache.avro</groupId>
       <artifactId>avro-mapred</artifactId>
-      <classifier>${avro.mapred.classifier}</classifier>
       <scope>provided</scope>
     </dependency>
     <dependency>