Upgrade built-in Hive to 2.3.8

Author: wangyum

dev/deps/spark-deps-hadoop-2.7-hive-2.3

@@ -80,21 +80,21 @@ hadoop-yarn-client/2.7.4//hadoop-yarn-client-2.7.4.jar
 hadoop-yarn-common/2.7.4//hadoop-yarn-common-2.7.4.jar
 hadoop-yarn-server-common/2.7.4//hadoop-yarn-server-common-2.7.4.jar
 hadoop-yarn-server-web-proxy/2.7.4//hadoop-yarn-server-web-proxy-2.7.4.jar
-hive-beeline/2.3.7//hive-beeline-2.3.7.jar
-hive-cli/2.3.7//hive-cli-2.3.7.jar
-hive-common/2.3.7//hive-common-2.3.7.jar
-hive-exec/2.3.7/core/hive-exec-2.3.7-core.jar
-hive-jdbc/2.3.7//hive-jdbc-2.3.7.jar
-hive-llap-common/2.3.7//hive-llap-common-2.3.7.jar
-hive-metastore/2.3.7//hive-metastore-2.3.7.jar
-hive-serde/2.3.7//hive-serde-2.3.7.jar
+hive-beeline/2.3.8//hive-beeline-2.3.8.jar
+hive-cli/2.3.8//hive-cli-2.3.8.jar
+hive-common/2.3.8//hive-common-2.3.8.jar
+hive-exec/2.3.8/core/hive-exec-2.3.8-core.jar
+hive-jdbc/2.3.8//hive-jdbc-2.3.8.jar
+hive-llap-common/2.3.8//hive-llap-common-2.3.8.jar
+hive-metastore/2.3.8//hive-metastore-2.3.8.jar
+hive-serde/2.3.8//hive-serde-2.3.8.jar
 hive-service-rpc/3.1.2//hive-service-rpc-3.1.2.jar
-hive-shims-0.23/2.3.7//hive-shims-0.23-2.3.7.jar
-hive-shims-common/2.3.7//hive-shims-common-2.3.7.jar
-hive-shims-scheduler/2.3.7//hive-shims-scheduler-2.3.7.jar
-hive-shims/2.3.7//hive-shims-2.3.7.jar
+hive-shims-0.23/2.3.8//hive-shims-0.23-2.3.8.jar
+hive-shims-common/2.3.8//hive-shims-common-2.3.8.jar
+hive-shims-scheduler/2.3.8//hive-shims-scheduler-2.3.8.jar
+hive-shims/2.3.8//hive-shims-2.3.8.jar
 hive-storage-api/2.7.2//hive-storage-api-2.7.2.jar
-hive-vector-code-gen/2.3.7//hive-vector-code-gen-2.3.7.jar
+hive-vector-code-gen/2.3.8//hive-vector-code-gen-2.3.8.jar
 hk2-api/2.6.1//hk2-api-2.6.1.jar
 hk2-locator/2.6.1//hk2-locator-2.6.1.jar
 hk2-utils/2.6.1//hk2-utils-2.6.1.jar

dev/deps/spark-deps-hadoop-3.2-hive-2.3

@@ -79,21 +79,21 @@ hadoop-yarn-common/3.2.0//hadoop-yarn-common-3.2.0.jar
 hadoop-yarn-registry/3.2.0//hadoop-yarn-registry-3.2.0.jar
 hadoop-yarn-server-common/3.2.0//hadoop-yarn-server-common-3.2.0.jar
 hadoop-yarn-server-web-proxy/3.2.0//hadoop-yarn-server-web-proxy-3.2.0.jar
-hive-beeline/2.3.7//hive-beeline-2.3.7.jar
-hive-cli/2.3.7//hive-cli-2.3.7.jar
-hive-common/2.3.7//hive-common-2.3.7.jar
-hive-exec/2.3.7/core/hive-exec-2.3.7-core.jar
-hive-jdbc/2.3.7//hive-jdbc-2.3.7.jar
-hive-llap-common/2.3.7//hive-llap-common-2.3.7.jar
-hive-metastore/2.3.7//hive-metastore-2.3.7.jar
-hive-serde/2.3.7//hive-serde-2.3.7.jar
+hive-beeline/2.3.8//hive-beeline-2.3.8.jar
+hive-cli/2.3.8//hive-cli-2.3.8.jar
+hive-common/2.3.8//hive-common-2.3.8.jar
+hive-exec/2.3.8/core/hive-exec-2.3.8-core.jar
+hive-jdbc/2.3.8//hive-jdbc-2.3.8.jar
+hive-llap-common/2.3.8//hive-llap-common-2.3.8.jar
+hive-metastore/2.3.8//hive-metastore-2.3.8.jar
+hive-serde/2.3.8//hive-serde-2.3.8.jar
 hive-service-rpc/3.1.2//hive-service-rpc-3.1.2.jar
-hive-shims-0.23/2.3.7//hive-shims-0.23-2.3.7.jar
-hive-shims-common/2.3.7//hive-shims-common-2.3.7.jar
-hive-shims-scheduler/2.3.7//hive-shims-scheduler-2.3.7.jar
-hive-shims/2.3.7//hive-shims-2.3.7.jar
+hive-shims-0.23/2.3.8//hive-shims-0.23-2.3.8.jar
+hive-shims-common/2.3.8//hive-shims-common-2.3.8.jar
+hive-shims-scheduler/2.3.8//hive-shims-scheduler-2.3.8.jar
+hive-shims/2.3.8//hive-shims-2.3.8.jar
 hive-storage-api/2.7.2//hive-storage-api-2.7.2.jar
-hive-vector-code-gen/2.3.7//hive-vector-code-gen-2.3.7.jar
+hive-vector-code-gen/2.3.8//hive-vector-code-gen-2.3.8.jar
 hk2-api/2.6.1//hk2-api-2.6.1.jar
 hk2-locator/2.6.1//hk2-locator-2.6.1.jar
 hk2-utils/2.6.1//hk2-utils-2.6.1.jar

docs/building-spark.md

@@ -83,9 +83,9 @@ Example:
 
 To enable Hive integration for Spark SQL along with its JDBC server and CLI,
 add the `-Phive` and `-Phive-thriftserver` profiles to your existing build options.
-By default Spark will build with Hive 2.3.7.
+By default Spark will build with Hive 2.3.8.
 
-    # With Hive 2.3.7 support
+    # With Hive 2.3.8 support
     ./build/mvn -Pyarn -Phive -Phive-thriftserver -DskipTests clean package
 
 ## Packaging without Hadoop Dependencies for YARN

docs/sql-data-sources-hive-tables.md

@@ -127,10 +127,10 @@ The following options can be used to configure the version of Hive that is used
   <tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
   <tr>
     <td><code>spark.sql.hive.metastore.version</code></td>
-    <td><code>2.3.7</code></td>
+    <td><code>2.3.8</code></td>
     <td>
       Version of the Hive metastore. Available
-      options are <code>0.12.0</code> through <code>2.3.7</code> and <code>3.0.0</code> through <code>3.1.2</code>.
+      options are <code>0.12.0</code> through <code>2.3.8</code> and <code>3.0.0</code> through <code>3.1.2</code>.
     </td>
     <td>1.4.0</td>
   </tr>
@@ -142,9 +142,9 @@ The following options can be used to configure the version of Hive that is used
       property can be one of three options:
       <ol>
         <li><code>builtin</code></li>
-        Use Hive 2.3.7, which is bundled with the Spark assembly when <code>-Phive</code> is
+        Use Hive 2.3.8, which is bundled with the Spark assembly when <code>-Phive</code> is
         enabled. When this option is chosen, <code>spark.sql.hive.metastore.version</code> must be
-        either <code>2.3.7</code> or not defined.
+        either <code>2.3.8</code> or not defined.
         <li><code>maven</code></li>
         Use Hive jars of specified version downloaded from Maven repositories. This configuration
         is not generally recommended for production deployments.

docs/sql-migration-guide.md

@@ -824,7 +824,7 @@ Python UDF registration is unchanged.
 Spark SQL is designed to be compatible with the Hive Metastore, SerDes and UDFs.
 Currently, Hive SerDes and UDFs are based on built-in Hive,
 and Spark SQL can be connected to different versions of Hive Metastore
-(from 0.12.0 to 2.3.7 and 3.0.0 to 3.1.2. Also see [Interacting with Different Versions of Hive Metastore](sql-data-sources-hive-tables.html#interacting-with-different-versions-of-hive-metastore)).
+(from 0.12.0 to 2.3.8 and 3.0.0 to 3.1.2. Also see [Interacting with Different Versions of Hive Metastore](sql-data-sources-hive-tables.html#interacting-with-different-versions-of-hive-metastore)).
 
 #### Deploying in Existing Hive Warehouses
 {:.no_toc}

pom.xml

@@ -128,8 +128,8 @@
     <hive.group>org.apache.hive</hive.group>
     <hive.classifier>core</hive.classifier>
     <!-- Version used in Maven Hive dependency -->
-    <hive.version>2.3.7</hive.version>
-    <hive23.version>2.3.7</hive23.version>
+    <hive.version>2.3.8</hive.version>
+    <hive23.version>2.3.8</hive23.version>
     <!-- Version used for internal directory structure -->
     <hive.version.short>2.3</hive.version.short>
     <!-- note that this should be compatible with Kafka brokers version 0.10 and up -->
@@ -286,6 +286,17 @@
         <enabled>false</enabled>
       </snapshots>
     </repository>
+    <repository>
+      <id>staged</id>
+      <name>hive-staged-releases</name>
+      <url>https://repository.apache.org/content/repositories/staging/</url>
+      <releases>
+        <enabled>true</enabled>
+      </releases>
+      <snapshots>
+        <enabled>true</enabled>
+      </snapshots>
+    </repository>
   </repositories>
   <pluginRepositories>
     <pluginRepository>

project/SparkBuild.scala

@@ -263,6 +263,9 @@ object SparkBuild extends PomBuild {
       DefaultMavenRepository,
       Resolver.mavenLocal,
       Resolver.file("ivyLocal", file(Path.userHome.absolutePath + "/.ivy2/local"))(Resolver.ivyStylePatterns)
+    ) ++ Seq(
+      "hive-staged-releases-mirror" at "https://repository.apache.org/content/repositories/staging/",
+      Resolver.file("local", file(Path.userHome.absolutePath + "/.ivy2/local"))(Resolver.ivyStylePatterns)
     ),
     externalResolvers := resolvers.value,
     otherResolvers := SbtPomKeys.mvnLocalRepository(dotM2 => Seq(Resolver.file("dotM2", dotM2))).value,

sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala

@@ -2669,7 +2669,8 @@ object SQLConf {
       .stringConf
       .createWithDefault(
         sys.env.getOrElse("DEFAULT_ARTIFACT_REPOSITORY",
-          "https://maven-central.storage-download.googleapis.com/maven2/"))
+          "https://maven-central.storage-download.googleapis.com/maven2/," +
+            "https://repository.apache.org/content/repositories/staging/"))
 
   val LEGACY_FROM_DAYTIME_STRING =
     buildConf("spark.sql.legacy.fromDayTimeString.enabled")

sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala

@@ -545,7 +545,7 @@ class HiveThriftBinaryServerSuite extends HiveThriftJdbcTest {
         conf += resultSet.getString(1) -> resultSet.getString(2)
       }
 
-      assert(conf.get(HiveUtils.FAKE_HIVE_VERSION.key) === Some("2.3.7"))
+      assert(conf.get(HiveUtils.FAKE_HIVE_VERSION.key) === Some("2.3.8"))
     }
   }
 
@@ -558,7 +558,7 @@ class HiveThriftBinaryServerSuite extends HiveThriftJdbcTest {
         conf += resultSet.getString(1) -> resultSet.getString(2)
       }
 
-      assert(conf.get(HiveUtils.FAKE_HIVE_VERSION.key) === Some("2.3.7"))
+      assert(conf.get(HiveUtils.FAKE_HIVE_VERSION.key) === Some("2.3.8"))
     }
   }
 

sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala

@@ -60,7 +60,7 @@ private[spark] object HiveUtils extends Logging {
 
   val HIVE_METASTORE_VERSION = buildStaticConf("spark.sql.hive.metastore.version")
     .doc("Version of the Hive metastore. Available options are " +
-        "<code>0.12.0</code> through <code>2.3.7</code> and " +
+        "<code>0.12.0</code> through <code>2.3.8</code> and " +
         "<code>3.0.0</code> through <code>3.1.2</code>.")
     .version("1.4.0")
     .stringConf

sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala

@@ -98,8 +98,8 @@ private[hive] object IsolatedClientLoader extends Logging {
     case "2.0" | "2.0.0" | "2.0.1" => hive.v2_0
     case "2.1" | "2.1.0" | "2.1.1" => hive.v2_1
     case "2.2" | "2.2.0" => hive.v2_2
-    case "2.3" | "2.3.0" | "2.3.1" | "2.3.2" | "2.3.3" | "2.3.4" | "2.3.5" | "2.3.6" | "2.3.7" =>
-      hive.v2_3
+    case "2.3" | "2.3.0" | "2.3.1" | "2.3.2" | "2.3.3" | "2.3.4" | "2.3.5" | "2.3.6" | "2.3.7" |
+         "2.3.8" => hive.v2_3
     case "3.0" | "3.0.0" => hive.v3_0
     case "3.1" | "3.1.0" | "3.1.1" | "3.1.2" => hive.v3_1
     case version =>

sql/hive/src/main/scala/org/apache/spark/sql/hive/client/package.scala

@@ -102,7 +102,7 @@ package object client {
 
     // Since HIVE-14496, Hive materialized view need calcite-core.
     // For spark, only VersionsSuite currently creates a hive materialized view for testing.
-    case object v2_3 extends HiveVersion("2.3.7",
+    case object v2_3 extends HiveVersion("2.3.8",
       exclusions = Seq("org.apache.calcite:calcite-druid",
         "org.apache.calcite.avatica:avatica",
         "org.apache.curator:*",

sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala

@@ -60,7 +60,7 @@ class HiveExternalCatalogVersionsSuite extends SparkSubmitTestUtils {
       .map(new File(_)).getOrElse(Utils.createTempDir(namePrefix = "test-spark"))
   private val unusedJar = TestUtils.createJarWithClasses(Seq.empty)
   val hiveVersion = if (SystemUtils.isJavaVersionAtLeast(JavaVersion.JAVA_9)) {
-    "2.3.7"
+    "2.3.8"
   } else {
     "1.2.1"
   }

sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala

@@ -668,7 +668,7 @@ private[sql] class TestHiveSessionStateBuilder(
 }
 
 private[hive] object HiveTestJars {
-  private val repository = SQLConf.ADDITIONAL_REMOTE_REPOSITORIES.defaultValueString.split(",")(0)
+  private val repository = SQLConf.ADDITIONAL_REMOTE_REPOSITORIES.defaultValueString.split(",").last
   private val hiveTestJarsDir = Utils.createTempDir()
 
   def getHiveContribJar(version: String = HiveUtils.builtinHiveVersion): File =