Merge branch 'trunk' into YARN-10561

Author: aajisaka

.github/pull_request_template.md

@@ -1,6 +1,19 @@
-## NOTICE
+<!--
+  Thanks for sending a pull request!
+    1. If this is your first time, please read our contributor guidelines: https://cwiki.apache.org/confluence/display/HADOOP/How+To+Contribute
+    2. Make sure your PR title starts with JIRA issue id, e.g., 'HADOOP-17799. Your PR title ...'.
+-->
+
+### Description of PR
+
+
+### How was this patch tested?
+
+
+### For code changes:
+
+- [ ] Does the title or this PR starts with the corresponding JIRA issue id (e.g. 'HADOOP-17799. Your PR title ...')?
+- [ ] Object storage: have the integration tests been executed and the endpoint declared according to the connector-specific documentation?
+- [ ] If adding new dependencies to the code, are these dependencies licensed in a way that is compatible for inclusion under [ASF 2.0](http://www.apache.org/legal/resolved.html#category-a)?
+- [ ] If applicable, have you updated the `LICENSE`, `LICENSE-binary`, `NOTICE-binary` files?
 
-Please create an issue in ASF JIRA before opening a pull request,
-and you need to set the title of the pull request which starts with
-the corresponding JIRA issue number. (e.g. HADOOP-XXXXX. Fix a typo in YYY.)
-For more details, please see https://cwiki.apache.org/confluence/display/HADOOP/How+To+Contribute

BUILDING.txt

@@ -57,7 +57,7 @@ Refer to  dev-support/docker/Dockerfile):
 
 * Open JDK 1.8
   $ sudo apt-get update
-  $ sudo apt-get -y install java-8-openjdk
+  $ sudo apt-get -y install openjdk-8-jdk
 * Maven
   $ sudo apt-get -y install maven
 * Native libraries
@@ -326,40 +326,35 @@ to update SNAPSHOTs from external repos.
 ----------------------------------------------------------------------------------
 Importing projects to eclipse
 
-When you import the project to eclipse, install hadoop-maven-plugins at first.
+At first, install artifacts including hadoop-maven-plugins at the top of the source tree.
 
-  $ cd hadoop-maven-plugins
-  $ mvn install
+  $ mvn clean install -DskipTests -DskipShade
 
-Then, generate eclipse project files.
-
-  $ mvn eclipse:eclipse -DskipTests
-
-At last, import to eclipse by specifying the root directory of the project via
-[File] > [Import] > [Existing Projects into Workspace].
+Then, import to eclipse by specifying the root directory of the project via
+[File] > [Import] > [Maven] > [Existing Maven Projects].
 
 ----------------------------------------------------------------------------------
 Building distributions:
 
-Create binary distribution without native code and without documentation:
+Create binary distribution without native code and without Javadocs:
 
   $ mvn package -Pdist -DskipTests -Dtar -Dmaven.javadoc.skip=true
 
-Create binary distribution with native code and with documentation:
+Create binary distribution with native code:
 
-  $ mvn package -Pdist,native,docs -DskipTests -Dtar
+  $ mvn package -Pdist,native -DskipTests -Dtar
 
 Create source distribution:
 
   $ mvn package -Psrc -DskipTests
 
-Create source and binary distributions with native code and documentation:
+Create source and binary distributions with native code:
 
-  $ mvn package -Pdist,native,docs,src -DskipTests -Dtar
+  $ mvn package -Pdist,native,src -DskipTests -Dtar
 
 Create a local staging version of the website (in /tmp/hadoop-site)
 
-  $ mvn clean site -Preleasedocs; mvn site:stage -DstagingDirectory=/tmp/hadoop-site
+  $ mvn site site:stage -Preleasedocs,docs -DstagingDirectory=/tmp/hadoop-site
 
 Note that the site needs to be built in a second pass after other artifacts.
 

LICENSE-binary

@@ -214,16 +214,16 @@ com.aliyun:aliyun-java-sdk-core:3.4.0
 com.aliyun:aliyun-java-sdk-ecs:4.2.0
 com.aliyun:aliyun-java-sdk-ram:3.0.0
 com.aliyun:aliyun-java-sdk-sts:3.0.0
-com.aliyun.oss:aliyun-sdk-oss:3.4.1
+com.aliyun.oss:aliyun-sdk-oss:3.13.0
 com.amazonaws:aws-java-sdk-bundle:1.11.901
 com.cedarsoftware:java-util:1.9.0
 com.cedarsoftware:json-io:2.5.1
-com.fasterxml.jackson.core:jackson-annotations:2.9.9
-com.fasterxml.jackson.core:jackson-core:2.9.9
-com.fasterxml.jackson.core:jackson-databind:2.9.9.2
-com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:2.9.9
-com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:2.9.9
-com.fasterxml.jackson.module:jackson-module-jaxb-annotations:2.9.9
+com.fasterxml.jackson.core:jackson-annotations:2.13.0
+com.fasterxml.jackson.core:jackson-core:2.13.0
+com.fasterxml.jackson.core:jackson-databind:2.13.0
+com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:2.13.0
+com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:2.13.0
+com.fasterxml.jackson.module:jackson-module-jaxb-annotations:2.13.0
 com.fasterxml.uuid:java-uuid-generator:3.1.4
 com.fasterxml.woodstox:woodstox-core:5.3.0
 com.github.davidmoten:rxjava-extras:0.8.0.17
@@ -250,7 +250,6 @@ commons-codec:commons-codec:1.11
 commons-collections:commons-collections:3.2.2
 commons-daemon:commons-daemon:1.0.13
 commons-io:commons-io:2.8.0
-commons-lang:commons-lang:2.6
 commons-logging:commons-logging:1.1.3
 commons-net:commons-net:3.6
 de.ruedigermoeller:fst:2.50
@@ -283,30 +282,30 @@ javax.inject:javax.inject:1
 log4j:log4j:1.2.17
 net.java.dev.jna:jna:5.2.0
 net.minidev:accessors-smart:1.2
-net.minidev:json-smart:2.4.2
+net.minidev:json-smart:2.4.7
 org.apache.avro:avro:1.7.7
 org.apache.commons:commons-collections4:4.2
-org.apache.commons:commons-compress:1.19
+org.apache.commons:commons-compress:1.21
 org.apache.commons:commons-configuration2:2.1.1
 org.apache.commons:commons-csv:1.0
 org.apache.commons:commons-digester:1.8.1
-org.apache.commons:commons-lang3:3.7
+org.apache.commons:commons-lang3:3.12.0
 org.apache.commons:commons-math3:3.1.1
 org.apache.commons:commons-text:1.4
 org.apache.commons:commons-validator:1.6
 org.apache.curator:curator-client:5.2.0
 org.apache.curator:curator-framework:5.2.0
 org.apache.curator:curator-recipes:5.2.0
 org.apache.geronimo.specs:geronimo-jcache_1.0_spec:1.0-alpha-1
-org.apache.hbase:hbase-annotations:1.4.8
-org.apache.hbase:hbase-client:1.4.8
-org.apache.hbase:hbase-common:1.4.8
-org.apache.hbase:hbase-protocol:1.4.8
+org.apache.hbase:hbase-annotations:1.7.1
+org.apache.hbase:hbase-client:1.7.1
+org.apache.hbase:hbase-common:1.7.1
+org.apache.hbase:hbase-protocol:1.7.1
 org.apache.htrace:htrace-core:3.1.0-incubating
 org.apache.htrace:htrace-core4:4.1.0-incubating
 org.apache.httpcomponents:httpclient:4.5.6
 org.apache.httpcomponents:httpcore:4.4.10
-org.apache.kafka:kafka-clients:2.4.0
+org.apache.kafka:kafka-clients:2.8.1
 org.apache.kerby:kerb-admin:1.0.1
 org.apache.kerby:kerb-client:1.0.1
 org.apache.kerby:kerb-common:1.0.1
@@ -329,22 +328,22 @@ org.codehaus.jackson:jackson-jaxrs:1.9.13
 org.codehaus.jackson:jackson-mapper-asl:1.9.13
 org.codehaus.jackson:jackson-xc:1.9.13
 org.codehaus.jettison:jettison:1.1
-org.eclipse.jetty:jetty-annotations:9.3.27.v20190418
-org.eclipse.jetty:jetty-http:9.3.27.v20190418
-org.eclipse.jetty:jetty-io:9.3.27.v20190418
-org.eclipse.jetty:jetty-jndi:9.3.27.v20190418
-org.eclipse.jetty:jetty-plus:9.3.27.v20190418
-org.eclipse.jetty:jetty-security:9.3.27.v20190418
-org.eclipse.jetty:jetty-server:9.3.27.v20190418
-org.eclipse.jetty:jetty-servlet:9.3.27.v20190418
-org.eclipse.jetty:jetty-util:9.3.27.v20190418
-org.eclipse.jetty:jetty-util-ajax:9.3.27.v20190418
-org.eclipse.jetty:jetty-webapp:9.3.27.v20190418
-org.eclipse.jetty:jetty-xml:9.3.27.v20190418
-org.eclipse.jetty.websocket:javax-websocket-client-impl:9.3.27.v20190418
-org.eclipse.jetty.websocket:javax-websocket-server-impl:9.3.27.v20190418
+org.eclipse.jetty:jetty-annotations:9.4.44.v20210927
+org.eclipse.jetty:jetty-http:9.4.44.v20210927
+org.eclipse.jetty:jetty-io:9.4.44.v20210927
+org.eclipse.jetty:jetty-jndi:9.4.44.v20210927
+org.eclipse.jetty:jetty-plus:9.4.44.v20210927
+org.eclipse.jetty:jetty-security:9.4.44.v20210927
+org.eclipse.jetty:jetty-server:9.4.44.v20210927
+org.eclipse.jetty:jetty-servlet:9.4.44.v20210927
+org.eclipse.jetty:jetty-util:9.4.44.v20210927
+org.eclipse.jetty:jetty-util-ajax:9.4.44.v20210927
+org.eclipse.jetty:jetty-webapp:9.4.44.v20210927
+org.eclipse.jetty:jetty-xml:9.4.44.v20210927
+org.eclipse.jetty.websocket:javax-websocket-client-impl:9.4.44.v20210927
+org.eclipse.jetty.websocket:javax-websocket-server-impl:9.4.44.v20210927
 org.ehcache:ehcache:3.3.1
-org.lz4:lz4-java:1.6.0
+org.lz4:lz4-java:1.7.1
 org.objenesis:objenesis:2.6
 org.xerial.snappy:snappy-java:1.0.5
 org.yaml:snakeyaml:1.16:
@@ -364,7 +363,7 @@ hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/com
 hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/util/tree.h
 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/compat/{fstatat|openat|unlinkat}.h
 
-com.github.luben:zstd-jni:1.4.3-1
+com.github.luben:zstd-jni:1.4.9-1
 dnsjava:dnsjava:2.1.7
 org.codehaus.woodstox:stax2-api:4.2.1
 
@@ -405,7 +404,7 @@ hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dataTables.bootstrap.css
 hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dataTables.bootstrap.js
 hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dust-full-2.0.0.min.js
 hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dust-helpers-1.1.1.min.js
-hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery-3.5.1.min.js
+hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery-3.6.0.min.js
 hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery.dataTables.min.js
 hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/moment.min.js
 hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/bootstrap.min.js
@@ -495,6 +494,7 @@ javax.annotation:javax.annotation-api:1.3.2
 javax.servlet:javax.servlet-api:3.1.0
 javax.servlet.jsp:jsp-api:2.1
 javax.websocket:javax.websocket-api:1.0
+javax.ws.rs:javax.ws.rs-api:2.1.1
 javax.ws.rs:jsr311-api:1.1.1
 javax.xml.bind:jaxb-api:2.2.11
 

LICENSE.txt

@@ -245,7 +245,7 @@ hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dataTables.bootstrap.css
 hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dataTables.bootstrap.js
 hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dust-full-2.0.0.min.js
 hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dust-helpers-1.1.1.min.js
-hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery-3.5.1.min.js
+hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery-3.6.0.min.js
 hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery.dataTables.min.js
 hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/moment.min.js
 hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/bootstrap.min.js

NOTICE-binary

@@ -66,7 +66,7 @@ available from http://www.digip.org/jansson/.
 
 
 AWS SDK for Java
-Copyright 2010-2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+Copyright 2010-2022 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 
 This product includes software developed by
 Amazon Technologies, Inc (http://www.amazon.com/).

dev-support/Jenkinsfile

@@ -15,13 +15,28 @@
 // specific language governing permissions and limitations
 // under the License.
 
-def getGithubAndJiraCreds() {
+def getGithubCreds() {
     return [usernamePassword(credentialsId: 'apache-hadoop-at-github.com',
                                 passwordVariable: 'GITHUB_TOKEN',
-                                usernameVariable: 'GITHUB_USER'),
-            usernamePassword(credentialsId: 'hadoopqa-at-asf-jira',
-                                passwordVariable: 'JIRA_PASSWORD',
-                                usernameVariable: 'JIRA_USER')]
+                                usernameVariable: 'GITHUB_USER')]
+}
+
+// Publish JUnit results only if there are XML files under surefire-reports
+def publishJUnitResults() {
+    def findCmdExitCode = sh script: "find ${SOURCEDIR} -wholename */target/surefire-reports/*.xml | egrep .", returnStatus: true
+    boolean surefireReportsExist = findCmdExitCode == 0
+    if (surefireReportsExist) {
+        echo "XML files found under surefire-reports, running junit"
+        // The path should be relative to WORKSPACE for the junit.
+        SRC = "${SOURCEDIR}/**/target/surefire-reports/*.xml".replace("$WORKSPACE/","")
+        try {
+            junit "${SRC}"
+        } catch(e) {
+            echo 'junit processing: ' + e.toString()
+        }
+    } else {
+        echo "No XML files found under surefire-reports, skipping junit"
+    }
 }
 
 pipeline {
@@ -112,7 +127,7 @@ pipeline {
             }
 
             steps {
-                withCredentials(getGithubAndJiraCreds()) {
+                withCredentials(getGithubCreds()) {
                     sh '''#!/usr/bin/env bash
 
                     chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
@@ -158,7 +173,7 @@ pipeline {
             }
 
             steps {
-                withCredentials(getGithubAndJiraCreds()) {
+                withCredentials(getGithubCreds()) {
                         sh '''#!/usr/bin/env bash
 
                         chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
@@ -204,7 +219,7 @@ pipeline {
             }
 
             steps {
-                withCredentials(getGithubAndJiraCreds()) {
+                withCredentials(getGithubCreds()) {
                     sh '''#!/usr/bin/env bash
 
                     chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
@@ -247,7 +262,7 @@ pipeline {
             }
 
             steps {
-                withCredentials(getGithubAndJiraCreds()) {
+                withCredentials(getGithubCreds()) {
                     sh '''#!/usr/bin/env bash
 
                     chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
@@ -290,12 +305,7 @@ pipeline {
                                       reportName: 'Yetus Report'
                         ])
 
-                        // Publish JUnit results
-                        try {
-                            junit "${SOURCEDIR}/**/target/surefire-reports/*.xml"
-                        } catch(e) {
-                            echo 'junit processing: ' + e.toString()
-                        }
+                        publishJUnitResults()
                     }
                 }
 

dev-support/bin/hadoop.sh

@@ -513,7 +513,7 @@ function shadedclient_initialize
   maven_add_install shadedclient
 }
 
-## @description build client facing shaded artifacts and test them
+## @description build client facing shaded and non-shaded artifacts and test them
 ## @audience private
 ## @stability evolving
 ## @param repostatus
@@ -546,13 +546,20 @@ function shadedclient_rebuild
     return 0
   fi
 
-  big_console_header "Checking client artifacts on ${repostatus}"
+  big_console_header "Checking client artifacts on ${repostatus} with shaded clients"
 
   echo_and_redirect "${logfile}" \
     "${MAVEN}" "${MAVEN_ARGS[@]}" verify -fae --batch-mode -am \
       "${modules[@]}" \
       -Dtest=NoUnitTests -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true -Dspotbugs.skip=true
 
+  big_console_header "Checking client artifacts on ${repostatus} with non-shaded clients"
+
+  echo_and_redirect "${logfile}" \
+    "${MAVEN}" "${MAVEN_ARGS[@]}" verify -fae --batch-mode -am \
+      "${modules[@]}" \
+      -DskipShade -Dtest=NoUnitTests -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true -Dspotbugs.skip=true
+
   count=$("${GREP}" -c '\[ERROR\]' "${logfile}")
   if [[ ${count} -gt 0 ]]; then
     add_vote_table -1 shadedclient "${repostatus} has errors when building and testing our client artifacts."