apache
diff --git a/‎BUILDING.txt
Lines changed: 1 addition & 1 deletion b/‎BUILDING.txt
Lines changed: 1 addition & 1 deletion
diff --git a/‎LICENSE-binary
Lines changed: 4 additions & 4 deletions b/‎LICENSE-binary
Lines changed: 4 additions & 4 deletions
diff --git a/‎dev-support/Jenkinsfile
Lines changed: 61 additions & 91 deletions b/‎dev-support/Jenkinsfile
Lines changed: 61 additions & 91 deletions
diff --git a/‎dev-support/bin/checkcompatibility.py
Lines changed: 1 addition & 1 deletion b/‎dev-support/bin/checkcompatibility.py
Lines changed: 1 addition & 1 deletion
diff --git a/‎dev-support/bin/create-release
Lines changed: 1 addition & 1 deletion b/‎dev-support/bin/create-release
Lines changed: 1 addition & 1 deletion
diff --git a/‎dev-support/bin/hadoop.sh
Lines changed: 2 additions & 1 deletion b/‎dev-support/bin/hadoop.sh
Lines changed: 2 additions & 1 deletion
@@ -143,7 +143,7 @@ Maven build goals:
  * Run checkstyle            : mvn compile checkstyle:checkstyle
  * Install JAR in M2 cache   : mvn install
  * Deploy JAR to Maven repo  : mvn deploy
- * Run clover                : mvn test -Pclover [-DcloverLicenseLocation=${user.name}/.clover.license]
+ * Run clover                : mvn test -Pclover
  * Run Rat                   : mvn apache-rat:check
  * Build javadocs            : mvn javadoc:javadoc
  * Build distribution        : mvn package [-Pdist][-Pdocs][-Psrc][-Pnative][-Dtar][-Preleasedocs][-Pyarn-ui]
 
@@ -249,7 +249,7 @@ commons-cli:commons-cli:1.2
 commons-codec:commons-codec:1.11
 commons-collections:commons-collections:3.2.2
 commons-daemon:commons-daemon:1.0.13
-commons-io:commons-io:2.5
+commons-io:commons-io:2.8.0
 commons-lang:commons-lang:2.6
 commons-logging:commons-logging:1.1.3
 commons-net:commons-net:3.6
@@ -468,8 +468,8 @@ com.microsoft.azure:azure-cosmosdb-gateway:2.4.5
 com.microsoft.azure:azure-data-lake-store-sdk:2.3.3
 com.microsoft.azure:azure-keyvault-core:1.0.0
 com.microsoft.sqlserver:mssql-jdbc:6.2.1.jre7
-org.bouncycastle:bcpkix-jdk15on:1.68
-org.bouncycastle:bcprov-jdk15on:1.68
+org.bouncycastle:bcpkix-jdk15on:1.60
+org.bouncycastle:bcprov-jdk15on:1.60
 org.checkerframework:checker-qual:2.5.2
 org.codehaus.mojo:animal-sniffer-annotations:1.17
 org.jruby.jcodings:jcodings:1.0.13
@@ -502,7 +502,7 @@ javax.xml.bind:jaxb-api:2.2.11
 Eclipse Public License 1.0
 --------------------------
 
-junit:junit:4.12
+junit:junit:4.13.2
 
 
 HSQL License
 
@@ -23,7 +23,7 @@ pipeline {
 
     options {
         buildDiscarder(logRotator(numToKeepStr: '5'))
-        timeout (time: 20, unit: 'HOURS')
+        timeout (time: 24, unit: 'HOURS')
         timestamps()
         checkoutToSubdirectory('src')
     }
@@ -32,7 +32,6 @@ pipeline {
         SOURCEDIR = 'src'
         // will also need to change notification section below
         PATCHDIR = 'out'
-        DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile"
         YETUS='yetus'
         // Branch or tag name.  Yetus release tags are 'rel/X.Y.Z'
         YETUS_VERSION='f9ba0170a5787a5f4662d3769804fef0226a182f'
@@ -57,7 +56,17 @@ pipeline {
             }
         }
 
-        stage ('precommit-run') {
+        // This is an optional stage which runs only when there's a change in
+        // C++/C++ build/platform.
+        // This stage serves as a means of cross platform validation, which is
+        // really needed to ensure that any C++ related/platform change doesn't
+        // break the Hadoop build on Centos 8.
+        stage ('precommit-run Centos 8') {
+            environment {
+                DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile_centos_8"
+                IS_OPTIONAL = 1
+            }
+
             steps {
                 withCredentials(
                     [usernamePassword(credentialsId: 'apache-hadoop-at-github.com',
@@ -68,100 +77,61 @@ pipeline {
                                         usernameVariable: 'JIRA_USER')]) {
                         sh '''#!/usr/bin/env bash
 
-                        set -e
-
-                        TESTPATCHBIN="${WORKSPACE}/${YETUS}/precommit/src/main/shell/test-patch.sh"
-
-                        # this must be clean for every run
-                        if [[ -d "${WORKSPACE}/${PATCHDIR}" ]]; then
-                          rm -rf "${WORKSPACE}/${PATCHDIR}"
-                        fi
-                        mkdir -p "${WORKSPACE}/${PATCHDIR}"
-
-                        # if given a JIRA issue, process it. If CHANGE_URL is set
-                        # (e.g., Github Branch Source plugin), process it.
-                        # otherwise exit, because we don't want Hadoop to do a
-                        # full build.  We wouldn't normally do this check for smaller
-                        # projects. :)
-                        if [[ -n "${JIRA_ISSUE_KEY}" ]]; then
-                            YETUS_ARGS+=("${JIRA_ISSUE_KEY}")
-                        elif [[ -z "${CHANGE_URL}" ]]; then
-                            echo "Full build skipped" > "${WORKSPACE}/${PATCHDIR}/report.html"
-                            exit 0
-                        fi
-
-                        YETUS_ARGS+=("--patch-dir=${WORKSPACE}/${PATCHDIR}")
-
-                        # where the source is located
-                        YETUS_ARGS+=("--basedir=${WORKSPACE}/${SOURCEDIR}")
-
-                        # our project defaults come from a personality file
-                        YETUS_ARGS+=("--project=hadoop")
-                        YETUS_ARGS+=("--personality=${WORKSPACE}/${SOURCEDIR}/dev-support/bin/hadoop.sh")
-
-                        # lots of different output formats
-                        YETUS_ARGS+=("--brief-report-file=${WORKSPACE}/${PATCHDIR}/brief.txt")
-                        YETUS_ARGS+=("--console-report-file=${WORKSPACE}/${PATCHDIR}/console.txt")
-                        YETUS_ARGS+=("--html-report-file=${WORKSPACE}/${PATCHDIR}/report.html")
-
-                        # enable writing back to Github
-                        YETUS_ARGS+=(--github-token="${GITHUB_TOKEN}")
-
-                        # enable writing back to ASF JIRA
-                        YETUS_ARGS+=(--jira-password="${JIRA_PASSWORD}")
-                        YETUS_ARGS+=(--jira-user="${JIRA_USER}")
-
-                        # auto-kill any surefire stragglers during unit test runs
-                        YETUS_ARGS+=("--reapermode=kill")
-
-                        # set relatively high limits for ASF machines
-                        # changing these to higher values may cause problems
-                        # with other jobs on systemd-enabled machines
-                        YETUS_ARGS+=("--proclimit=5500")
-                        YETUS_ARGS+=("--dockermemlimit=22g")
-
-                        # -1 spotbugs issues that show up prior to the patch being applied
-                        YETUS_ARGS+=("--spotbugs-strict-precheck")
-
-                        # rsync these files back into the archive dir
-                        YETUS_ARGS+=("--archive-list=checkstyle-errors.xml,spotbugsXml.xml")
-
-                        # URL for user-side presentation in reports and such to our artifacts
-                        # (needs to match the archive bits below)
-                        YETUS_ARGS+=("--build-url-artifacts=artifact/out")
-
-                        # plugins to enable
-                        YETUS_ARGS+=("--plugins=all")
-
-                        # don't let these tests cause -1s because we aren't really paying that
-                        # much attention to them
-                        YETUS_ARGS+=("--tests-filter=checkstyle")
-
-                        # run in docker mode and specifically point to our
-                        # Dockerfile since we don't want to use the auto-pulled version.
-                        YETUS_ARGS+=("--docker")
-                        YETUS_ARGS+=("--dockerfile=${DOCKERFILE}")
-                        YETUS_ARGS+=("--mvn-custom-repos")
+                        chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
+                        "${SOURCEDIR}/dev-support/jenkins.sh"
+                        '''
+                }
+            }
+        }
 
-                        # effectively treat dev-suport as a custom maven module
-                        YETUS_ARGS+=("--skip-dirs=dev-support")
+        // This is an optional stage which runs only when there's a change in
+        // C++/C++ build/platform.
+        // This stage serves as a means of cross platform validation, which is
+        // really needed to ensure that any C++ related/platform change doesn't
+        // break the Hadoop build on Debian 10.
+        stage ('precommit-run Debian 10') {
+            environment {
+                DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile_debian_10"
+                IS_OPTIONAL = 1
+            }
 
-                        # help keep the ASF boxes clean
-                        YETUS_ARGS+=("--sentinel")
+            steps {
+                withCredentials(
+                    [usernamePassword(credentialsId: 'apache-hadoop-at-github.com',
+                                  passwordVariable: 'GITHUB_TOKEN',
+                                  usernameVariable: 'GITHUB_USER'),
+                    usernamePassword(credentialsId: 'hadoopqa-at-asf-jira',
+                                        passwordVariable: 'JIRA_PASSWORD',
+                                        usernameVariable: 'JIRA_USER')]) {
+                        sh '''#!/usr/bin/env bash
 
-                        # test with Java 8 and 11
-                        YETUS_ARGS+=("--java-home=/usr/lib/jvm/java-8-openjdk-amd64")
-                        YETUS_ARGS+=("--multijdkdirs=/usr/lib/jvm/java-11-openjdk-amd64")
-                        YETUS_ARGS+=("--multijdktests=compile")
+                        chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
+                        "${SOURCEDIR}/dev-support/jenkins.sh"
+                        '''
+                }
+            }
+        }
 
-                        # custom javadoc goals
-                        YETUS_ARGS+=("--mvn-javadoc-goals=process-sources,javadoc:javadoc-no-fork")
+        // We want to use Ubuntu Focal as our main CI and thus, this stage
+        // isn't optional (runs for all the PRs).
+        stage ('precommit-run Ubuntu focal') {
+            environment {
+                DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile"
+                IS_OPTIONAL = 0
+            }
 
-                        # write Yetus report as GitHub comment (YETUS-1102)
-                        YETUS_ARGS+=("--github-write-comment")
-                        YETUS_ARGS+=("--github-use-emoji-vote")
+            steps {
+                withCredentials(
+                    [usernamePassword(credentialsId: 'apache-hadoop-at-github.com',
+                                  passwordVariable: 'GITHUB_TOKEN',
+                                  usernameVariable: 'GITHUB_USER'),
+                    usernamePassword(credentialsId: 'hadoopqa-at-asf-jira',
+                                        passwordVariable: 'JIRA_PASSWORD',
+                                        usernameVariable: 'JIRA_USER')]) {
+                        sh '''#!/usr/bin/env bash
 
-                        "${TESTPATCHBIN}" "${YETUS_ARGS[@]}"
+                        chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
+                        "${SOURCEDIR}/dev-support/jenkins.sh"
                         '''
                 }
             }
 
@@ -175,7 +175,7 @@ def run_java_acc(src_name, src_jars, dst_name, dst_jars, annotations):
 
   if annotations is not None:
     annotations_path = os.path.join(get_scratch_dir(), "annotations.txt")
-    with file(annotations_path, "w") as f:
+    with open(annotations_path, "w") as f:
       for ann in annotations:
         print(ann, file=f)
     args += ["-annotations-list", annotations_path]
 
@@ -514,7 +514,7 @@ function dockermode
 
     echo "USER ${user_name}"
     printf "\n\n"
-  ) | docker build -t "${imgname}" -
+  ) | docker build -t "${imgname}" -f - "${BASEDIR}"/dev-support/docker/
 
   run docker run -i -t \
     --privileged \
 
@@ -355,6 +355,7 @@ function personality_modules
       fi
     ;;
     unit)
+      extra="-Dsurefire.rerunFailingTestsCount=2"
       if [[ "${BUILDMODE}" = full ]]; then
         ordering=mvnsrc
       elif [[ "${CHANGED_MODULES[*]}" =~ \. ]]; then
@@ -363,7 +364,7 @@ function personality_modules
 
       if [[ ${TEST_PARALLEL} = "true" ]] ; then
         if hadoop_test_parallel; then
-          extra="-Pparallel-tests"
+          extra="${extra} -Pparallel-tests"
           if [[ -n ${TEST_THREADS:-} ]]; then
             extra="${extra} -DtestsThreadCount=${TEST_THREADS}"
           fi