Merge branch 'trunk' into possible-resource-leak

Author: Nargeshdb

BUILDING.txt

@@ -139,7 +139,7 @@ Maven build goals:
  * Compile                   : mvn compile [-Pnative]
  * Run tests                 : mvn test [-Pnative] [-Pshelltest]
  * Create JAR                : mvn package
- * Run findbugs              : mvn compile findbugs:findbugs
+ * Run spotbugs              : mvn compile spotbugs:spotbugs
  * Run checkstyle            : mvn compile checkstyle:checkstyle
  * Install JAR in M2 cache   : mvn install
  * Deploy JAR to Maven repo  : mvn deploy

LICENSE-binary

@@ -225,7 +225,7 @@ com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:2.9.9
 com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:2.9.9
 com.fasterxml.jackson.module:jackson-module-jaxb-annotations:2.9.9
 com.fasterxml.uuid:java-uuid-generator:3.1.4
-com.fasterxml.woodstox:woodstox-core:5.0.3
+com.fasterxml.woodstox:woodstox-core:5.3.0
 com.github.davidmoten:rxjava-extras:0.8.0.17
 com.github.stephenc.jcip:jcip-annotations:1.0-1
 com.google:guice:4.0
@@ -366,7 +366,7 @@ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanage
 
 com.github.luben:zstd-jni:1.4.3-1
 dnsjava:dnsjava:2.1.7
-org.codehaus.woodstox:stax2-api:3.1.4
+org.codehaus.woodstox:stax2-api:4.2.1
 
 
 BSD 3-Clause
@@ -468,8 +468,8 @@ com.microsoft.azure:azure-cosmosdb-gateway:2.4.5
 com.microsoft.azure:azure-data-lake-store-sdk:2.3.3
 com.microsoft.azure:azure-keyvault-core:1.0.0
 com.microsoft.sqlserver:mssql-jdbc:6.2.1.jre7
-org.bouncycastle:bcpkix-jdk15on:1.60
-org.bouncycastle:bcprov-jdk15on:1.60
+org.bouncycastle:bcpkix-jdk15on:1.68
+org.bouncycastle:bcprov-jdk15on:1.68
 org.checkerframework:checker-qual:2.5.2
 org.codehaus.mojo:animal-sniffer-annotations:1.17
 org.jruby.jcodings:jcodings:1.0.13

dev-support/Jenkinsfile

@@ -35,7 +35,7 @@ pipeline {
         DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile"
         YETUS='yetus'
         // Branch or tag name.  Yetus release tags are 'rel/X.Y.Z'
-        YETUS_VERSION='rel/0.13.0'
+        YETUS_VERSION='11eb9b09786e401fbdeaa3be83a19a4066fd7813'
     }
 
     parameters {
@@ -60,7 +60,7 @@ pipeline {
         stage ('precommit-run') {
             steps {
                 withCredentials(
-                    [usernamePassword(credentialsId: '683f5dcf-5552-4b28-9fb1-6a6b77cf53dd',
+                    [usernamePassword(credentialsId: 'apache-hadoop-at-github.com',
                                   passwordVariable: 'GITHUB_TOKEN',
                                   usernameVariable: 'GITHUB_USER'),
                     usernamePassword(credentialsId: 'hadoopqa-at-asf-jira',
@@ -120,11 +120,11 @@ pipeline {
                         YETUS_ARGS+=("--proclimit=5500")
                         YETUS_ARGS+=("--dockermemlimit=22g")
 
-                        # -1 findbugs issues that show up prior to the patch being applied
-                        YETUS_ARGS+=("--findbugs-strict-precheck")
+                        # -1 spotbugs issues that show up prior to the patch being applied
+                        YETUS_ARGS+=("--spotbugs-strict-precheck")
 
                         # rsync these files back into the archive dir
-                        YETUS_ARGS+=("--archive-list=checkstyle-errors.xml,findbugsXml.xml")
+                        YETUS_ARGS+=("--archive-list=checkstyle-errors.xml,spotbugsXml.xml")
 
                         # URL for user-side presentation in reports and such to our artifacts
                         # (needs to match the archive bits below)
@@ -157,6 +157,10 @@ pipeline {
                         # custom javadoc goals
                         YETUS_ARGS+=("--mvn-javadoc-goals=process-sources,javadoc:javadoc-no-fork")
 
+                        # write Yetus report as GitHub comment (YETUS-1102)
+                        YETUS_ARGS+=("--github-write-comment")
+                        YETUS_ARGS+=("--github-use-emoji-vote")
+
                         "${TESTPATCHBIN}" "${YETUS_ARGS[@]}"
                         '''
                 }

dev-support/bin/hadoop.sh

@@ -482,7 +482,7 @@ function personality_file_tests
   fi
 
   if [[ ${filename} =~ \.java$ ]]; then
-    add_test findbugs
+    add_test spotbugs
   fi
 }
 
@@ -550,7 +550,7 @@ function shadedclient_rebuild
   echo_and_redirect "${logfile}" \
     "${MAVEN}" "${MAVEN_ARGS[@]}" verify -fae --batch-mode -am \
       "${modules[@]}" \
-      -Dtest=NoUnitTests -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true -Dfindbugs.skip=true
+      -Dtest=NoUnitTests -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true -Dspotbugs.skip=true
 
   count=$("${GREP}" -c '\[ERROR\]' "${logfile}")
   if [[ ${count} -gt 0 ]]; then

dev-support/bin/test-patch

@@ -15,4 +15,4 @@
 # limitations under the License.
 
 BINDIR=$(cd -P -- "$(dirname -- "${BASH_SOURCE-0}")" >/dev/null && pwd -P)
-exec "${BINDIR}/yetus-wrapper" test-patch --project=hadoop --skip-dir=dev-support "$@"
+exec "${BINDIR}/yetus-wrapper" test-patch --project=hadoop --skip-dirs=dev-support "$@"

dev-support/docker/Dockerfile

@@ -44,7 +44,6 @@ RUN apt-get -q update \
         cmake \
         curl \
         doxygen \
-        findbugs \
         fuse \
         g++ \
         gcc \
@@ -92,7 +91,16 @@ RUN apt-get -q update \
 ENV MAVEN_HOME /usr
 # JAVA_HOME must be set in Maven >= 3.5.0 (MNG-6003)
 ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64
-ENV FINDBUGS_HOME /usr
+
+#######
+# Install SpotBugs 4.2.2
+#######
+RUN mkdir -p /opt/spotbugs \
+    && curl -L -s -S https://github.com/spotbugs/spotbugs/releases/download/4.2.2/spotbugs-4.2.2.tgz \
+      -o /opt/spotbugs.tgz \
+    && tar xzf /opt/spotbugs.tgz --strip-components 1 -C /opt/spotbugs \
+    && chmod +x /opt/spotbugs/bin/*
+ENV SPOTBUGS_HOME /opt/spotbugs
 
 #######
 # Install Boost 1.72 (1.71 ships with Focal)

dev-support/docker/Dockerfile_aarch64

@@ -48,7 +48,6 @@ RUN apt-get -q update \
         cmake \
         curl \
         doxygen \
-        findbugs \
         fuse \
         g++ \
         gcc \
@@ -96,7 +95,16 @@ RUN apt-get -q update \
 ENV MAVEN_HOME /usr
 # JAVA_HOME must be set in Maven >= 3.5.0 (MNG-6003)
 ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-arm64
-ENV FINDBUGS_HOME /usr
+
+#######
+# Install SpotBugs 4.2.2
+#######
+RUN mkdir -p /opt/spotbugs \
+    && curl -L -s -S https://github.com/spotbugs/spotbugs/releases/download/4.2.2/spotbugs-4.2.2.tgz \
+      -o /opt/spotbugs.tgz \
+    && tar xzf /opt/spotbugs.tgz --strip-components 1 -C /opt/spotbugs \
+    && chmod +x /opt/spotbugs/bin/*
+ENV SPOTBUGS_HOME /opt/spotbugs
 
 #######
 # Install Boost 1.72 (1.71 ships with Focal)

hadoop-cloud-storage-project/hadoop-cos/pom.xml

@@ -64,10 +64,9 @@
   <build>
     <plugins>
       <plugin>
-        <groupId>org.codehaus.mojo</groupId>
-        <artifactId>findbugs-maven-plugin</artifactId>
+        <groupId>com.github.spotbugs</groupId>
+        <artifactId>spotbugs-maven-plugin</artifactId>
         <configuration>
-          <findbugsXmlOutput>true</findbugsXmlOutput>
           <xmlOutput>true</xmlOutput>
           <excludeFilterFile>${basedir}/dev-support/findbugs-exclude.xml
           </excludeFilterFile>

hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/CosNFileSystem.java

@@ -28,11 +28,11 @@
 import java.util.HashMap;
 import java.util.Set;
 import java.util.TreeSet;
+import java.util.concurrent.ExecutorService;
 import java.util.concurrent.TimeUnit;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ListeningExecutorService;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
@@ -71,8 +71,8 @@ public class CosNFileSystem extends FileSystem {
   private String owner = "Unknown";
   private String group = "Unknown";
 
-  private ListeningExecutorService boundedIOThreadPool;
-  private ListeningExecutorService boundedCopyThreadPool;
+  private ExecutorService boundedIOThreadPool;
+  private ExecutorService boundedCopyThreadPool;
 
   public CosNFileSystem() {
   }

hadoop-cloud-storage-project/hadoop-huaweicloud/src/main/java/org/apache/hadoop/fs/obs/OBSFileSystem.java

@@ -19,7 +19,6 @@
 package org.apache.hadoop.fs.obs;
 
 import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ListeningExecutorService;
 import com.obs.services.ObsClient;
 import com.obs.services.exception.ObsException;
 import com.obs.services.model.AccessControlList;
@@ -53,6 +52,7 @@
 import java.io.IOException;
 import java.net.URI;
 import java.util.EnumSet;
+import java.util.concurrent.ExecutorService;
 import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
@@ -169,7 +169,7 @@ public final class OBSFileSystem extends FileSystem {
   /**
    * Bounded thread pool for multipart upload.
    */
-  private ListeningExecutorService boundedMultipartUploadThreadPool;
+  private ExecutorService boundedMultipartUploadThreadPool;
 
   /**
    * Bounded thread pool for copy.