[WIP][SPARK-29250][BUILD][test-hadoop3.2][test-maven] Upgrade to Hadoop 3.2.1

dev/deps/spark-deps-hadoop-3.2-hive-2.3

@@ -7,6 +7,7 @@ accessors-smart-1.2.jar
 activation-1.1.1.jar
 aircompressor-0.10.jar
 algebra_2.12-2.0.0-M2.jar
+animal-sniffer-annotations-1.17.jar
 antlr-runtime-3.5.2.jar
 antlr4-runtime-4.7.1.jar
 aopalliance-1.0.jar
@@ -20,10 +21,13 @@ automaton-1.11-8.jar
 avro-1.8.2.jar
 avro-ipc-1.8.2.jar
 avro-mapred-1.8.2-hadoop2.jar
+bcpkix-jdk15on-1.60.jar
+bcprov-jdk15on-1.60.jar
 bonecp-0.8.0.RELEASE.jar
 breeze-macros_2.12-1.0.jar
 breeze_2.12-1.0.jar
 cats-kernel_2.12-2.0.0-M4.jar
+checker-qual-2.5.2.jar
 chill-java-0.9.3.jar
 chill_2.12-0.9.3.jar
 commons-beanutils-1.9.4.jar
@@ -57,27 +61,29 @@ derby-10.12.1.1.jar
 dnsjava-2.1.7.jar
 dropwizard-metrics-hadoop-metrics2-reporter-0.1.2.jar
 ehcache-3.3.1.jar
+error_prone_annotations-2.2.0.jar
+failureaccess-1.0.jar
 flatbuffers-java-1.9.0.jar
 generex-1.0.2.jar
 geronimo-jcache_1.0_spec-1.0-alpha-1.jar
 gson-2.2.4.jar
-guava-14.0.1.jar
+guava-27.0-jre.jar
 guice-4.0.jar
 guice-servlet-4.0.jar
-hadoop-annotations-3.2.0.jar
-hadoop-auth-3.2.0.jar
-hadoop-client-3.2.0.jar
-hadoop-common-3.2.0.jar
-hadoop-hdfs-client-3.2.0.jar
-hadoop-mapreduce-client-common-3.2.0.jar
-hadoop-mapreduce-client-core-3.2.0.jar
-hadoop-mapreduce-client-jobclient-3.2.0.jar
-hadoop-yarn-api-3.2.0.jar
-hadoop-yarn-client-3.2.0.jar
-hadoop-yarn-common-3.2.0.jar
-hadoop-yarn-registry-3.2.0.jar
-hadoop-yarn-server-common-3.2.0.jar
-hadoop-yarn-server-web-proxy-3.2.0.jar
+hadoop-annotations-3.2.1.jar
+hadoop-auth-3.2.1.jar
+hadoop-client-3.2.1.jar
+hadoop-common-3.2.1.jar
+hadoop-hdfs-client-3.2.1.jar
+hadoop-mapreduce-client-common-3.2.1.jar
+hadoop-mapreduce-client-core-3.2.1.jar
+hadoop-mapreduce-client-jobclient-3.2.1.jar
+hadoop-yarn-api-3.2.1.jar
+hadoop-yarn-client-3.2.1.jar
+hadoop-yarn-common-3.2.1.jar
+hadoop-yarn-registry-3.2.1.jar
+hadoop-yarn-server-common-3.2.1.jar
+hadoop-yarn-server-web-proxy-3.2.1.jar
 hive-beeline-2.3.6.jar
 hive-cli-2.3.6.jar
 hive-common-2.3.6.jar
@@ -101,13 +107,14 @@ httpclient-4.5.6.jar
 httpcore-4.4.12.jar
 istack-commons-runtime-3.0.8.jar
 ivy-2.4.0.jar
+j2objc-annotations-1.1.jar
 jackson-annotations-2.10.0.jar
 jackson-core-2.10.0.jar
 jackson-core-asl-1.9.13.jar
 jackson-databind-2.10.0.jar
 jackson-dataformat-yaml-2.10.0.jar
-jackson-jaxrs-base-2.9.5.jar
-jackson-jaxrs-json-provider-2.9.5.jar
+jackson-jaxrs-base-2.9.8.jar
+jackson-jaxrs-json-provider-2.9.8.jar
 jackson-mapper-asl-1.9.13.jar
 jackson-module-jaxb-annotations-2.10.0.jar
 jackson-module-paranamer-2.10.0.jar
@@ -171,6 +178,7 @@ kubernetes-model-common-4.6.4.jar
 leveldbjni-all-1.8.jar
 libfb303-0.9.3.jar
 libthrift-0.12.0.jar
+listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar
 log4j-1.2.17.jar
 logging-interceptor-3.12.6.jar
 lz4-java-1.7.0.jar

pom.xml

@@ -199,6 +199,7 @@
     and ./python/setup.py too.
     -->
     <arrow.version>0.15.1</arrow.version>
+    <guava.version>14.0.1</guava.version>
     <!-- org.fusesource.leveldbjni will be used except on arm64 platform. -->
     <leveldbjni.group>org.fusesource.leveldbjni</leveldbjni.group>
 
@@ -411,7 +412,7 @@
       <dependency>
         <groupId>com.google.guava</groupId>
         <artifactId>guava</artifactId>
-        <version>14.0.1</version>
+        <version>${guava.version}</version>
         <scope>provided</scope>
       </dependency>
       <dependency>
@@ -2938,8 +2939,9 @@
     <profile>
       <id>hadoop-3.2</id>
       <properties>
-        <hadoop.version>3.2.0</hadoop.version>
+        <hadoop.version>3.2.1</hadoop.version>
         <curator.version>2.13.0</curator.version>
+        <guava.version>27.0-jre</guava.version>
       </properties>
     </profile>
 

sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala

@@ -192,7 +192,6 @@ private[hive] class IsolatedClientLoader(
     name.startsWith("org.apache.spark.") ||
     (sharesHadoopClasses && isHadoopClass) ||
     name.startsWith("scala.") ||
-    (name.startsWith("com.google") && !name.startsWith("com.google.cloud")) ||
     name.startsWith("java.") ||
     name.startsWith("javax.sql.") ||
     sharedPrefixes.exists(name.startsWith)