Merge branch 'apache:trunk' into HADOOP-19663

Author: PeterPtroc

.github/workflows/stale.yml

@@ -0,0 +1,45 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+name: Close stale PRs
+on:
+  schedule:
+    - cron: "0 0 * * *"
+
+jobs:
+  stale:
+    if: github.repository == 'apache/hadoop'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/stale@v1
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+          stale-pr-message: >
+            We're closing this stale PR because it has been open for 100 days with
+            no activity. This isn't a judgement on the merit of the PR in any way.
+            It's just a way of keeping the PR queue manageable.
+
+            If you feel like this was a mistake, or you would like to continue
+            working on it, please feel free to re-open it and ask for a committer to
+            remove the stale tag and review again.
+
+            Thanks all for your contribution.
+          days-before-stale: 100
+          days-before-close: 0
+          close-pr-label: 'stale'

BUILDING.txt

@@ -31,7 +31,7 @@ On Linux / Mac:
 
     $ ./start-build-env.sh [OS platform]
 
-   - [OS Platform] One of [centos_7, centos_8, debian_10, ubuntu_20, ubuntu_24, windows_10].
+   - [OS Platform] One of [rockylinux_8, debian_11, ubuntu_20, ubuntu_24, windows_10].
         Default is 'ubuntu_20'.
         Note: Currently only default ('ubuntu_20') is supported on arm machine
 
@@ -279,8 +279,7 @@ Maven build goals:
    OWASP's dependency-check plugin will scan the third party dependencies
    of this project for known CVEs (security vulnerabilities against them).
    It will produce a report in target/dependency-check-report.html. To
-   invoke, run 'mvn dependency-check:aggregate'. Note that this plugin
-   requires maven 3.1.1 or greater.
+   invoke, run 'mvn dependency-check:aggregate'.
 
  PMDK library build options:
 
@@ -461,7 +460,7 @@ path. This is necessary at least for Homebrewed OpenSSL.
 
 ----------------------------------------------------------------------------------
 
-Building on CentOS 8
+Building on Rocky Linux 8
 
 ----------------------------------------------------------------------------------
 
@@ -482,7 +481,7 @@ Building on CentOS 8
   $ sudo make install
   $ cd ..
 
-* Install libraries provided by CentOS 8.
+* Install libraries provided by Rocky Linux 8.
   $ sudo dnf install libtirpc-devel zlib-devel lz4-devel bzip2-devel openssl-devel cyrus-sasl-devel libpmem-devel
 
 * Install GCC 9.3.0
@@ -529,7 +528,7 @@ Requirements:
 
 * Windows 10
 * JDK 1.8
-* Maven 3.0 or later (maven.apache.org)
+* Maven 3.3 or later (maven.apache.org)
 * Boost 1.86.0 (boost.org)
 * Protocol Buffers 3.21.12 (https://github.com/protocolbuffers/protobuf/tags)
 * CMake 3.19 or newer (cmake.org)

LICENSE-binary

@@ -229,6 +229,25 @@ com.fasterxml.woodstox:woodstox-core:5.4.0
 com.github.ben-manes.caffeine:caffeine:2.9.3
 com.github.davidmoten:rxjava-extras:0.8.0.17
 com.github.stephenc.jcip:jcip-annotations:1.0-1
+com.google.api:api-common:2.47.2
+com.google.api-client:google-api-client:2.7.2
+com.google.api:gax:2.64.2
+com.google.api:gax-grpc:2.64.2
+com.google.api:gax-httpjson:2.64.2
+com.google.api.grpc:gapic-google-cloud-storage-v2:2.52.0
+com.google.api.grpc:grpc-google-cloud-storage-v2:2.52.0
+com.google.api.grpc:proto-google-cloud-monitoring-v3:3.52.0
+com.google.api.grpc:proto-google-cloud-storage-v2:2.52.0
+com.google.api.grpc:proto-google-common-protos:2.55.2
+com.google.api.grpc:proto-google-iam-v1:1.50.2
+com.google.apis:google-api-services-storage:v1-rev20250420-2.0.0
+com.google.auto.value:auto-value-annotations:1.11.0
+com.google.cloud:google-cloud-core-grpc:2.54.2
+com.google.cloud:google-cloud-core-http:2.54.2
+com.google.cloud:google-cloud-core:2.54.2
+com.google.cloud:google-cloud-monitoring:3.52.0
+com.google.cloud.opentelemetry:exporter-metrics:0.33.0
+com.google.errorprone:error_prone_annotations:2.36.0
 com.google:guice:5.1.0
 com.google:guice-servlet:5.1.0
 com.google.api.grpc:proto-google-common-protos:1.0.0
@@ -237,11 +256,20 @@ com.google.errorprone:error_prone_annotations:2.5.1
 com.google.j2objc:j2objc-annotations:1.3
 com.google.json-simple:json-simple:1.1.1
 com.google.guava:failureaccess:1.0
+com.google.guava:failureaccess:1.0.2
 com.google.guava:guava:20.0
 com.google.guava:guava:32.0.1-jre
+com.google.guava:guava:33.1.0-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
+com.google.http-client:google-http-client-apache-v2:1.46.3
+com.google.http-client:google-http-client-appengine:1.46.3
+com.google.http-client:google-http-client-gson:1.46.3
+com.google.http-client:google-http-client-jackson2:1.46.3
+com.google.http-client:google-http-client:1.46.3
+com.google.j2objc:j2objc-annotations:3.0.0
+com.google.oauth-client:google-oauth-client:1.37.0
 com.microsoft.azure:azure-storage:7.0.0
-com.nimbusds:nimbus-jose-jwt:9.37.2
+com.nimbusds:nimbus-jose-jwt:10.4
 com.zaxxer:HikariCP:4.0.3
 commons-beanutils:commons-beanutils:1.9.4
 commons-cli:commons-cli:1.9.0
@@ -251,45 +279,73 @@ commons-daemon:commons-daemon:1.0.13
 commons-io:commons-io:2.16.1
 commons-net:commons-net:3.9.0
 de.ruedigermoeller:fst:2.50
+io.grpc:grpc-alts:1.70.0
 io.grpc:grpc-api:1.69.0
+io.grpc:grpc-api:1.70.0
+io.grpc:grpc-auth:1.70.0
 io.grpc:grpc-context:1.69.0
+io.grpc:grpc-context:1.70.0
 io.grpc:grpc-core:1.69.0
+io.grpc:grpc-core:1.70.0
+io.grpc:grpc-googleapis:1.70.0
+io.grpc:grpc-grpclb:1.70.0
+io.grpc:grpc-inprocess:1.70.0
 io.grpc:grpc-netty:1.69.0
+io.grpc:grpc-netty-shaded:1.70.0
+io.grpc:grpc-opentelemetry:1.70.0
 io.grpc:grpc-protobuf:1.69.0
+io.grpc:grpc-protobuf:1.70.0
 io.grpc:grpc-protobuf-lite:1.69.0
+io.grpc:grpc-rls:1.70.0
+io.grpc:grpc-services:1.70.0
 io.grpc:grpc-stub:1.69.0
-io.netty:netty-all:4.1.118.Final
-io.netty:netty-buffer:4.1.118.Final
-io.netty:netty-codec:4.1.118.Final
-io.netty:netty-codec-dns:4.1.118.Final
-io.netty:netty-codec-haproxy:4.1.118.Final
-io.netty:netty-codec-http:4.1.118.Final
-io.netty:netty-codec-http2:4.1.118.Final
-io.netty:netty-codec-memcache:4.1.118.Final
-io.netty:netty-codec-mqtt:4.1.118.Final
-io.netty:netty-codec-redis:4.1.118.Final
-io.netty:netty-codec-smtp:4.1.118.Final
-io.netty:netty-codec-socks:4.1.118.Final
-io.netty:netty-codec-stomp:4.1.118.Final
-io.netty:netty-codec-xml:4.1.118.Final
-io.netty:netty-common:4.1.118.Final
-io.netty:netty-handler:4.1.118.Final
-io.netty:netty-handler-proxy:4.1.118.Final
-io.netty:netty-resolver:4.1.118.Final
-io.netty:netty-resolver-dns:4.1.118.Final
-io.netty:netty-transport:4.1.118.Final
-io.netty:netty-transport-rxtx:4.1.118.Final
-io.netty:netty-transport-sctp:4.1.118.Final
-io.netty:netty-transport-udt:4.1.118.Final
-io.netty:netty-transport-classes-epoll:4.1.118.Final
-io.netty:netty-transport-native-unix-common:4.1.118.Final
-io.netty:netty-transport-classes-kqueue:4.1.118.Final
-io.netty:netty-resolver-dns-classes-macos:4.1.118.Final
-io.netty:netty-transport-native-epoll:4.1.118.Final
-io.netty:netty-transport-native-kqueue:4.1.118.Final
-io.netty:netty-resolver-dns-native-macos:4.1.118.Final
+io.grpc:grpc-stub:1.70.0
+io.grpc:grpc-util:1.70.0
+io.grpc:grpc-xds:1.70.0
+io.netty:netty-all:4.1.127.Final
+io.netty:netty-buffer:4.1.127.Final
+io.netty:netty-codec:4.1.127.Final
+io.netty:netty-codec-dns:4.1.127.Final
+io.netty:netty-codec-haproxy:4.1.127.Final
+io.netty:netty-codec-http:4.1.127.Final
+io.netty:netty-codec-http2:4.1.127.Final
+io.netty:netty-codec-memcache:4.1.127.Final
+io.netty:netty-codec-mqtt:4.1.127.Final
+io.netty:netty-codec-redis:4.1.127.Final
+io.netty:netty-codec-smtp:4.1.127.Final
+io.netty:netty-codec-socks:4.1.127.Final
+io.netty:netty-codec-stomp:4.1.127.Final
+io.netty:netty-codec-xml:4.1.127.Final
+io.netty:netty-common:4.1.127.Final
+io.netty:netty-handler:4.1.127.Final
+io.netty:netty-handler-proxy:4.1.127.Final
+io.netty:netty-resolver:4.1.127.Final
+io.netty:netty-resolver-dns:4.1.127.Final
+io.netty:netty-transport:4.1.127.Final
+io.netty:netty-transport-rxtx:4.1.127.Final
+io.netty:netty-transport-sctp:4.1.127.Final
+io.netty:netty-transport-udt:4.1.127.Final
+io.netty:netty-transport-classes-epoll:4.1.127.Final
+io.netty:netty-transport-native-unix-common:4.1.127.Final
+io.netty:netty-transport-classes-kqueue:4.1.127.Final
+io.netty:netty-resolver-dns-classes-macos:4.1.127.Final
+io.netty:netty-transport-native-epoll:4.1.127.Final
+io.netty:netty-transport-native-kqueue:4.1.127.Final
+io.netty:netty-resolver-dns-native-macos:4.1.127.Final
 io.opencensus:opencensus-api:0.12.3
+io.opencensus:opencensus-api:0.31.1
 io.opencensus:opencensus-contrib-grpc-metrics:0.12.3
+io.opencensus:opencensus-contrib-http-util:0.31.1
+io.opentelemetry.contrib:opentelemetry-gcp-resources:1.37.0-alpha
+io.opentelemetry:opentelemetry-api:1.47.0
+io.opentelemetry:opentelemetry-context:1.47.0
+io.opentelemetry:opentelemetry-sdk-common:1.47.0
+io.opentelemetry:opentelemetry-sdk-extension-autoconfigure-spi:1.47.0
+io.opentelemetry:opentelemetry-sdk:1.47.0
+io.opentelemetry:opentelemetry-sdk-logs:1.47.0
+io.opentelemetry:opentelemetry-sdk-metrics:1.47.0
+io.opentelemetry:opentelemetry-sdk-trace:1.47.0
+io.opentelemetry.semconv:opentelemetry-semconv:1.29.0-alpha
 io.reactivex:rxjava:1.3.8
 io.reactivex:rxjava-string:1.1.1
 io.reactivex:rxnetty:0.4.20
@@ -302,9 +358,9 @@ org.apache.commons:commons-compress:1.26.1
 org.apache.commons:commons-configuration2:2.10.1
 org.apache.commons:commons-csv:1.9.0
 org.apache.commons:commons-digester:1.8.1
-org.apache.commons:commons-lang3:3.17.0
+org.apache.commons:commons-lang3:3.18.0
 org.apache.commons:commons-math3:3.6.1
-org.apache.commons:commons-text:1.10.0
+org.apache.commons:commons-text:1.14.0
 org.apache.commons:commons-validator:1.6
 org.apache.curator:curator-client:5.2.0
 org.apache.curator:curator-framework:5.2.0
@@ -340,6 +396,7 @@ org.apache.solr:solr-solrj:8.11.2
 org.apache.yetus:audience-annotations:0.5.0
 org.apache.zookeeper:zookeeper:3.8.4
 org.codehaus.jettison:jettison:1.5.4
+org.conscrypt:conscrypt-openjdk-uber:2.5.2
 org.eclipse.jetty:jetty-annotations:9.4.57.v20241219
 org.eclipse.jetty:jetty-http:9.4.57.v20241219
 org.eclipse.jetty:jetty-io:9.4.57.v20241219
@@ -363,6 +420,7 @@ org.yaml:snakeyaml:2.0
 org.wildfly.openssl:wildfly-openssl:2.1.4.Final
 ro.isdc.wro4j:wro4j-maven-plugin:1.8.0
 software.amazon.awssdk:bundle:2.29.52
+software.amazon.s3.analyticsaccelerator:analyticsaccelerator-s3:1.3.0
 net.jodah:failsafe:2.4.4
 
 --------------------------------------------------------------------------------
@@ -394,18 +452,22 @@ hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/d3.v3.js
 hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/d3-3.5.17.min.js
 leveldb v1.13
 
+com.google.auth:google-auth-library-credentials:1.33.1
+com.google.auth:google-auth-library-oauth2-http:1.33.1
 com.google.protobuf:protobuf-java:2.5.0
 com.google.protobuf:protobuf-java:3.25.5
 com.google.re2j:re2j:1.1
 com.jcraft:jsch:0.1.55
 com.thoughtworks.paranamer:paranamer:2.3
 jakarta.activation:jakarta.activation-api:1.2.1
+org.checkerframework:checker-qual:3.49.0
 org.fusesource.leveldbjni:leveldbjni-all:1.8
 org.jline:jline:3.9.0
 org.hamcrest:hamcrest-core:1.3
 org.ow2.asm:asm:5.0.4
 org.ow2.asm:asm-commons:6.0
 org.ow2.asm:asm-tree:6.0
+org.threeten:threetenbp:1.7.0
 
 
 MIT License
@@ -510,6 +572,7 @@ Eclipse Distribution License (EDL) 1.0
 --------------------------
 
 jakarta.xml.bind:jakarta.xml.bind-api:2.3.3
+org.glassfish.jaxb:jaxb-runtime:2.3.9
 
 Eclipse Public License 1.0
 --------------------------

dev-support/Jenkinsfile

@@ -84,17 +84,17 @@ pipeline {
         stage ('setup sources') {
             steps {
 
-                dir("${WORKSPACE}/centos-8") {
+                dir("${WORKSPACE}/rockylinux-8") {
                     sh '''#!/usr/bin/env bash
 
-                    cp -Rp ${WORKSPACE}/src ${WORKSPACE}/centos-8
+                    cp -Rp ${WORKSPACE}/src ${WORKSPACE}/rockylinux-8
                     '''
                 }
 
-                dir("${WORKSPACE}/debian-10") {
+                dir("${WORKSPACE}/debian-11") {
                     sh '''#!/usr/bin/env bash
 
-                    cp -Rp ${WORKSPACE}/src ${WORKSPACE}/debian-10
+                    cp -Rp ${WORKSPACE}/src ${WORKSPACE}/debian-11
                     '''
                 }
 
@@ -111,12 +111,12 @@ pipeline {
         // C++/C++ build/platform.
         // This stage serves as a means of cross platform validation, which is
         // really needed to ensure that any C++ related/platform change doesn't
-        // break the Hadoop build on Centos 8.
-        stage ('precommit-run Centos 8') {
+        // break the Hadoop build on Rocky Linux 8.
+        stage ('precommit-run Rocky Linux 8') {
             environment {
-                SOURCEDIR = "${WORKSPACE}/centos-8/src"
-                PATCHDIR = "${WORKSPACE}/centos-8/out"
-                DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile_centos_8"
+                SOURCEDIR = "${WORKSPACE}/rockylinux-8/src"
+                PATCHDIR = "${WORKSPACE}/rockylinux-8/out"
+                DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile_rockylinux_8"
                 IS_OPTIONAL = 1
             }
 
@@ -136,7 +136,7 @@ pipeline {
                 failure {
                     sh '''#!/usr/bin/env bash
 
-                    cp -Rp "${WORKSPACE}/centos-8/out" "${WORKSPACE}"
+                    cp -Rp "${WORKSPACE}/rockylinux-8/out" "${WORKSPACE}"
                     '''
                     archiveArtifacts "out/**"
                 }
@@ -157,12 +157,12 @@ pipeline {
         // C++/C++ build/platform.
         // This stage serves as a means of cross platform validation, which is
         // really needed to ensure that any C++ related/platform change doesn't
-        // break the Hadoop build on Debian 10.
-        stage ('precommit-run Debian 10') {
+        // break the Hadoop build on Debian 11.
+        stage ('precommit-run Debian 11') {
             environment {
-                SOURCEDIR = "${WORKSPACE}/debian-10/src"
-                PATCHDIR = "${WORKSPACE}/debian-10/out"
-                DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile_debian_10"
+                SOURCEDIR = "${WORKSPACE}/debian-11/src"
+                PATCHDIR = "${WORKSPACE}/debian-11/out"
+                DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile_debian_11"
                 IS_OPTIONAL = 1
             }
 
@@ -182,7 +182,7 @@ pipeline {
                 failure {
                     sh '''#!/usr/bin/env bash
 
-                    cp -Rp "${WORKSPACE}/debian-10/out" "${WORKSPACE}"
+                    cp -Rp "${WORKSPACE}/debian-11/out" "${WORKSPACE}"
                     '''
                     archiveArtifacts "out/**"
                 }

dev-support/docker/Dockerfile

@@ -61,7 +61,8 @@ ENV PYTHONIOENCODING=utf-8
 ######
 # Set env vars required to build Hadoop
 ######
-ENV MAVEN_HOME /usr
+ENV MAVEN_HOME=/opt/maven
+ENV PATH="${PATH}:${MAVEN_HOME}/bin"
 # JAVA_HOME must be set in Maven >= 3.5.0 (MNG-6003)
 ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64
 
@@ -88,6 +89,7 @@ ENV HADOOP_SKIP_YETUS_VERIFICATION true
 # Install packages
 ####
 RUN pkg-resolver/install-common-pkgs.sh
+RUN pkg-resolver/install-maven.sh ubuntu:focal
 RUN pkg-resolver/install-spotbugs.sh ubuntu:focal
 RUN pkg-resolver/install-boost.sh ubuntu:focal
 RUN pkg-resolver/install-protobuf.sh ubuntu:focal