Back Merge with trunk

Author: Anuj Modi

BUILDING.txt

@@ -7,7 +7,7 @@ Requirements:
 * JDK 1.8
 * Maven 3.3 or later
 * Boost 1.72 (if compiling native code)
-* Protocol Buffers 3.7.1 (if compiling native code)
+* Protocol Buffers 3.21.12 (if compiling native code)
 * CMake 3.19 or newer (if compiling native code)
 * Zlib devel (if compiling native code)
 * Cyrus SASL devel (if compiling native code)
@@ -74,10 +74,10 @@ Refer to  dev-support/docker/Dockerfile):
   $ ./bootstrap
   $ make -j$(nproc)
   $ sudo make install
-* Protocol Buffers 3.7.1 (required to build native code)
-  $ curl -L -s -S https://github.com/protocolbuffers/protobuf/releases/download/v3.7.1/protobuf-java-3.7.1.tar.gz -o protobuf-3.7.1.tar.gz
-  $ mkdir protobuf-3.7-src
-  $ tar xzf protobuf-3.7.1.tar.gz --strip-components 1 -C protobuf-3.7-src && cd protobuf-3.7-src
+* Protocol Buffers 3.21.12 (required to build native code)
+  $ curl -L https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.21.12.tar.gz > protobuf-3.21.12.tar.gz
+  $ tar -zxvf protobuf-3.21.12.tar.gz && cd protobuf-3.21.12
+  $ ./autogen.sh
   $ ./configure
   $ make -j$(nproc)
   $ sudo make install
@@ -433,10 +433,10 @@ Installing required dependencies for clean install of macOS 10.14:
 * Install native libraries, only openssl is required to compile native code,
 you may optionally install zlib, lz4, etc.
   $ brew install openssl
-* Protocol Buffers 3.7.1 (required to compile native code)
-  $ wget https://github.com/protocolbuffers/protobuf/releases/download/v3.7.1/protobuf-java-3.7.1.tar.gz
-  $ mkdir -p protobuf-3.7 && tar zxvf protobuf-java-3.7.1.tar.gz --strip-components 1 -C protobuf-3.7
-  $ cd protobuf-3.7
+* Protocol Buffers 3.21.12 (required to compile native code)
+  $ curl -L https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.21.12.tar.gz > protobuf-3.21.12.tar.gz
+  $ tar -zxvf protobuf-3.21.12.tar.gz && cd protobuf-3.21.12
+  $ ./autogen.sh
   $ ./configure
   $ make
   $ make check
@@ -472,11 +472,10 @@ Building on CentOS 8
 * Install python2 for building documentation.
   $ sudo dnf install python2
 
-* Install Protocol Buffers v3.7.1.
-  $ git clone https://github.com/protocolbuffers/protobuf
-  $ cd protobuf
-  $ git checkout v3.7.1
-  $ autoreconf -i
+* Install Protocol Buffers v3.21.12.
+  $ curl -L https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.21.12.tar.gz > protobuf-3.21.12.tar.gz
+  $ tar -zxvf protobuf-3.21.12.tar.gz && cd protobuf-3.21.12
+  $ ./autogen.sh
   $ ./configure --prefix=/usr/local
   $ make
   $ sudo make install
@@ -531,7 +530,7 @@ Requirements:
 * JDK 1.8
 * Maven 3.0 or later (maven.apache.org)
 * Boost 1.72 (boost.org)
-* Protocol Buffers 3.7.1 (https://github.com/protocolbuffers/protobuf/releases)
+* Protocol Buffers 3.21.12 (https://github.com/protocolbuffers/protobuf/tags)
 * CMake 3.19 or newer (cmake.org)
 * Visual Studio 2019 (visualstudio.com)
 * Windows SDK 8.1 (optional, if building CPU rate control for the container executor. Get this from

LICENSE-binary

@@ -210,12 +210,13 @@ hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/nvd3-1.8.5.* (css and js
 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/AbstractFuture.java
 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/TimeoutFuture.java
 
+ch.qos.reload4j:reload4j:1.2.22
 com.aliyun:aliyun-java-sdk-core:4.5.10
 com.aliyun:aliyun-java-sdk-kms:2.11.0
 com.aliyun:aliyun-java-sdk-ram:3.1.0
 com.aliyun:aliyun-java-sdk-sts:3.0.0
 com.aliyun.oss:aliyun-sdk-oss:3.13.2
-com.amazonaws:aws-java-sdk-bundle:1.12.499
+com.amazonaws:aws-java-sdk-bundle:1.12.599
 com.cedarsoftware:java-util:1.9.0
 com.cedarsoftware:json-io:2.5.1
 com.fasterxml.jackson.core:jackson-annotations:2.12.7
@@ -250,13 +251,13 @@ commons-daemon:commons-daemon:1.0.13
 commons-io:commons-io:2.14.0
 commons-net:commons-net:3.9.0
 de.ruedigermoeller:fst:2.50
-io.grpc:grpc-api:1.26.0
-io.grpc:grpc-context:1.26.0
-io.grpc:grpc-core:1.26.0
-io.grpc:grpc-netty:1.26.0
-io.grpc:grpc-protobuf:1.26.0
-io.grpc:grpc-protobuf-lite:1.26.0
-io.grpc:grpc-stub:1.26.0
+io.grpc:grpc-api:1.53.0
+io.grpc:grpc-context:1.53.0
+io.grpc:grpc-core:1.53.0
+io.grpc:grpc-netty:1.53.0
+io.grpc:grpc-protobuf:1.53.0
+io.grpc:grpc-protobuf-lite:1.53.0
+io.grpc:grpc-stub:1.53.0
 io.netty:netty-all:4.1.100.Final
 io.netty:netty-buffer:4.1.100.Final
 io.netty:netty-codec:4.1.100.Final
@@ -294,7 +295,6 @@ io.reactivex:rxjava-string:1.1.1
 io.reactivex:rxnetty:0.4.20
 io.swagger:swagger-annotations:1.5.4
 javax.inject:javax.inject:1
-log4j:log4j:1.2.17
 net.java.dev.jna:jna:5.2.0
 net.minidev:accessors-smart:1.2
 org.apache.avro:avro:1.9.2
@@ -335,9 +335,12 @@ org.apache.kerby:kerby-pkix:2.0.3
 org.apache.kerby:kerby-util:2.0.3
 org.apache.kerby:kerby-xdr:2.0.3
 org.apache.kerby:token-provider:2.0.3
+org.apache.sshd:sshd-common:2.11.0
+org.apache.sshd:sshd-core:2.11.0
+org.apache.sshd:sshd-sftp:2.11.0
 org.apache.solr:solr-solrj:8.11.2
 org.apache.yetus:audience-annotations:0.5.0
-org.apache.zookeeper:zookeeper:3.7.2
+org.apache.zookeeper:zookeeper:3.8.3
 org.codehaus.jettison:jettison:1.5.4
 org.eclipse.jetty:jetty-annotations:9.4.53.v20231009
 org.eclipse.jetty:jetty-http:9.4.53.v20231009
@@ -360,7 +363,7 @@ org.objenesis:objenesis:2.6
 org.xerial.snappy:snappy-java:1.1.10.4
 org.yaml:snakeyaml:2.0
 org.wildfly.openssl:wildfly-openssl:1.1.3.Final
-software.amazon.awssdk:bundle:jar:2.20.160
+software.amazon.awssdk:bundle:jar:2.23.19
 
 
 --------------------------------------------------------------------------------
@@ -479,17 +482,17 @@ com.microsoft.azure:azure-cosmosdb-gateway:2.4.5
 com.microsoft.azure:azure-data-lake-store-sdk:2.3.3
 com.microsoft.azure:azure-keyvault-core:1.0.0
 com.microsoft.sqlserver:mssql-jdbc:6.2.1.jre7
-org.bouncycastle:bcpkix-jdk15on:1.68
-org.bouncycastle:bcprov-jdk15on:1.68
+org.bouncycastle:bcpkix-jdk15on:1.70
+org.bouncycastle:bcprov-jdk15on:1.70
+org.bouncycastle:bcutil-jdk15on:1.70
 org.checkerframework:checker-qual:2.5.2
-org.codehaus.mojo:animal-sniffer-annotations:1.17
+org.codehaus.mojo:animal-sniffer-annotations:1.21
 org.jruby.jcodings:jcodings:1.0.13
 org.jruby.joni:joni:2.1.2
-org.slf4j:jul-to-slf4j:jar:1.7.25
-org.ojalgo:ojalgo:43.0:compile
-org.slf4j:jul-to-slf4j:1.7.25
-org.slf4j:slf4j-api:1.7.25
-org.slf4j:slf4j-log4j12:1.7.25
+org.ojalgo:ojalgo:43.0
+org.slf4j:jul-to-slf4j:1.7.36
+org.slf4j:slf4j-api:1.7.36
+org.slf4j:slf4j-reload4j:1.7.36
 
 
 CDDL 1.1 + GPLv2 with classpath exception

NOTICE-binary

@@ -66,7 +66,7 @@ available from http://www.digip.org/jansson/.
 
 
 AWS SDK for Java
-Copyright 2010-2022 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+Copyright 2010-2024 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 
 This product includes software developed by
 Amazon Technologies, Inc (http://www.amazon.com/).

dev-support/bin/create-release

@@ -504,9 +504,9 @@ function dockermode
     echo "LABEL org.apache.hadoop.create-release=\"cr-${RANDOM}\""
 
     # setup ownerships, etc
-    echo "RUN groupadd --non-unique -g ${group_id} ${user_name}"
-    echo "RUN useradd -g ${group_id} -u ${user_id} -m ${user_name}"
-    echo "RUN chown -R ${user_name} /home/${user_name}"
+    echo "RUN groupadd --non-unique -g ${group_id} ${user_name}; exit 0;"
+    echo "RUN useradd -g ${group_id} -u ${user_id} -m ${user_name}; exit 0;"
+    echo "RUN chown -R ${user_name} /home/${user_name}; exit 0;"
     echo "ENV HOME /home/${user_name}"
     echo "RUN mkdir -p /maven"
     echo "RUN chown -R ${user_name} /maven"

dev-support/docker/Dockerfile

@@ -66,7 +66,7 @@ ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64
 ENV SPOTBUGS_HOME /opt/spotbugs
 
 #######
-# Set env vars for Google Protobuf 3.7.1
+# Set env vars for Google Protobuf 3.21.12
 #######
 ENV PROTOBUF_HOME /opt/protobuf
 ENV PATH "${PATH}:/opt/protobuf/bin"

dev-support/docker/Dockerfile_aarch64

@@ -66,7 +66,7 @@ ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-arm64
 ENV SPOTBUGS_HOME /opt/spotbugs
 
 #######
-# Set env vars for Google Protobuf 3.7.1
+# Set env vars for Google Protobuf 3.21.12
 #######
 ENV PROTOBUF_HOME /opt/protobuf
 ENV PATH "${PATH}:/opt/protobuf/bin"

dev-support/docker/Dockerfile_centos_7

@@ -76,7 +76,7 @@ ENV JAVA_HOME /usr/lib/jvm/java-1.8.0
 ENV SPOTBUGS_HOME /opt/spotbugs
 
 #######
-# Set env vars for Google Protobuf
+# Set env vars for Google Protobuf 3.21.12
 #######
 ENV PROTOBUF_HOME /opt/protobuf
 ENV PATH "${PATH}:/opt/protobuf/bin"

dev-support/docker/Dockerfile_centos_8

@@ -101,7 +101,7 @@ ENV JAVA_HOME /usr/lib/jvm/java-1.8.0
 ENV SPOTBUGS_HOME /opt/spotbugs
 
 #######
-# Set env vars for Google Protobuf
+# Set env vars for Google Protobuf 3.21.12
 #######
 ENV PROTOBUF_HOME /opt/protobuf
 ENV PATH "${PATH}:/opt/protobuf/bin"

dev-support/docker/Dockerfile_debian_10

@@ -66,7 +66,7 @@ ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-amd64
 ENV SPOTBUGS_HOME /opt/spotbugs
 
 #######
-# Set env vars for Google Protobuf 3.7.1
+# Set env vars for Google Protobuf 3.21.12
 #######
 ENV PROTOBUF_HOME /opt/protobuf
 ENV PATH "${PATH}:/opt/protobuf/bin"

dev-support/docker/pkg-resolver/install-maven.sh

@@ -40,7 +40,7 @@ fi
 
 if [ "$version_to_install" == "3.6.3" ]; then
   mkdir -p /opt/maven /tmp/maven &&
-    curl -L -s -S https://dlcdn.apache.org/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz \
+    curl -L -s -S https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.6.3/apache-maven-3.6.3-bin.tar.gz \
       -o /tmp/maven/apache-maven-3.6.3-bin.tar.gz &&
     tar xzf /tmp/maven/apache-maven-3.6.3-bin.tar.gz --strip-components 1 -C /opt/maven
 else

dev-support/docker/pkg-resolver/install-protobuf.sh

@@ -27,25 +27,26 @@ if [ $? -eq 1 ]; then
   exit 1
 fi
 
-default_version="3.7.1"
+default_version="3.21.12"
 version_to_install=$default_version
 if [ -n "$2" ]; then
   version_to_install="$2"
 fi
 
-if [ "$version_to_install" != "3.7.1" ]; then
+if [ "$version_to_install" != "3.21.12" ]; then
   echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead"
   version_to_install=$default_version
 fi
 
-if [ "$version_to_install" == "3.7.1" ]; then
+if [ "$version_to_install" == "3.21.12" ]; then
   # hadolint ignore=DL3003
   mkdir -p /opt/protobuf-src &&
     curl -L -s -S \
-      https://github.com/protocolbuffers/protobuf/releases/download/v3.7.1/protobuf-java-3.7.1.tar.gz \
+      https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.21.12.tar.gz \
       -o /opt/protobuf.tar.gz &&
     tar xzf /opt/protobuf.tar.gz --strip-components 1 -C /opt/protobuf-src &&
     cd /opt/protobuf-src &&
+    ./autogen.sh &&
     ./configure --prefix=/opt/protobuf &&
     make "-j$(nproc)" &&
     make install &&