HADOOP-19605. Upgrade Protobuf 3.25.5 for docker images

BUILDING.txt

@@ -7,7 +7,7 @@ Requirements:
 * JDK 1.8
 * Maven 3.3 or later
 * Boost 1.86.0 (if compiling native code)
-* Protocol Buffers 3.21.12 (if compiling native code)
+* Protocol Buffers 3.25.5 (if compiling native code)
 * CMake 3.19 or newer (if compiling native code)
 * Zlib devel (if compiling native code)
 * Cyrus SASL devel (if compiling native code)
@@ -78,13 +78,15 @@ Refer to  dev-support/docker/Dockerfile):
   $ ./bootstrap
   $ make -j$(nproc)
   $ sudo make install
-* Protocol Buffers 3.21.12 (required to build native code)
-  $ curl -L https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.21.12.tar.gz > protobuf-3.21.12.tar.gz
-  $ tar -zxvf protobuf-3.21.12.tar.gz && cd protobuf-3.21.12
-  $ ./autogen.sh
-  $ ./configure
-  $ make -j$(nproc)
-  $ sudo make install
+* Protocol Buffers 3.25.5 (required to build native code)
+  $ curl -L https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.25.5.tar.gz > protobuf-3.25.5.tar.gz
+  $ curl -L https://github.com/abseil/abseil-cpp/archive/refs/tags/20230802.1.tar.gz > abseil-cpp-20230802.1.tar.gz
+  $ tar -zxvf protobuf-3.25.5.tar.gz
+  $ tar -zxvf abseil-cpp-20230802.1.tar.gz --strip-components 1 -C protobuf-3.25.5/third_party/abseil-cpp
+  $ cd protobuf-3.25.5
+  $ cmake -S . -B build -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_BUILD_TESTS=OFF
+  $ cmake --build build --parallel $(nproc)
+  $ sudo cmake --install build
 * Boost
   $ curl -L https://sourceforge.net/projects/boost/files/boost/1.86.0/boost_1_86_0.tar.bz2/download > boost_1_86_0.tar.bz2
   $ tar --bzip2 -xf boost_1_86_0.tar.bz2 && cd boost_1_86_0
@@ -433,14 +435,15 @@ Installing required dependencies for clean install of macOS 10.14:
 * Install native libraries, only openssl is required to compile native code,
 you may optionally install zlib, lz4, etc.
   $ brew install openssl
-* Protocol Buffers 3.21.12 (required to compile native code)
-  $ curl -L https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.21.12.tar.gz > protobuf-3.21.12.tar.gz
-  $ tar -zxvf protobuf-3.21.12.tar.gz && cd protobuf-3.21.12
-  $ ./autogen.sh
-  $ ./configure
-  $ make
-  $ make check
-  $ make install
+* Protocol Buffers 3.25.5 (required to build native code)
+  $ curl -L https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.25.5.tar.gz > protobuf-3.25.5.tar.gz
+  $ curl -L https://github.com/abseil/abseil-cpp/archive/refs/tags/20230802.1.tar.gz > abseil-cpp-20230802.1.tar.gz
+  $ tar -zxvf protobuf-3.25.5.tar.gz
+  $ tar -zxvf abseil-cpp-20230802.1.tar.gz --strip-components 1 -C protobuf-3.25.5/third_party/abseil-cpp
+  $ cd protobuf-3.25.5
+  $ cmake -S . -B build -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_BUILD_TESTS=OFF
+  $ cmake --build build --parallel $(nproc)
+  $ cmake --install build
   $ protoc --version
 
 Note that building Hadoop 3.1.1/3.1.2/3.2.0 native code from source is broken
@@ -472,13 +475,15 @@ Building on Rocky Linux 8
 * Install python2 for building documentation.
   $ sudo dnf install python2
 
-* Install Protocol Buffers v3.21.12.
-  $ curl -L https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.21.12.tar.gz > protobuf-3.21.12.tar.gz
-  $ tar -zxvf protobuf-3.21.12.tar.gz && cd protobuf-3.21.12
-  $ ./autogen.sh
-  $ ./configure --prefix=/usr/local
-  $ make
-  $ sudo make install
+* Protocol Buffers 3.25.5 (required to build native code)
+  $ curl -L https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.25.5.tar.gz > protobuf-3.25.5.tar.gz
+  $ curl -L https://github.com/abseil/abseil-cpp/archive/refs/tags/20230802.1.tar.gz > abseil-cpp-20230802.1.tar.gz
+  $ tar -zxvf protobuf-3.25.5.tar.gz
+  $ tar -zxvf abseil-cpp-20230802.1.tar.gz --strip-components 1 -C protobuf-3.25.5/third_party/abseil-cpp
+  $ cd protobuf-3.25.5
+  $ cmake -S . -B build -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_BUILD_TESTS=OFF
+  $ cmake --build build --parallel $(nproc)
+  $ sudo cmake --install build --prefix /usr/local
   $ cd ..
 
 * Install libraries provided by Rocky Linux 8.
@@ -530,7 +535,7 @@ Requirements:
 * JDK 1.8
 * Maven 3.3 or later (maven.apache.org)
 * Boost 1.86.0 (boost.org)
-* Protocol Buffers 3.21.12 (https://github.com/protocolbuffers/protobuf/tags)
+* Protocol Buffers 3.25.5 (https://github.com/protocolbuffers/protobuf/tags)
 * CMake 3.19 or newer (cmake.org)
 * Visual Studio 2019 (visualstudio.com)
 * Windows SDK 8.1 (optional, if building CPU rate control for the container executor. Get this from

dev-support/docker/Dockerfile

@@ -72,7 +72,7 @@ ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64
 ENV SPOTBUGS_HOME /opt/spotbugs
 
 #######
-# Set env vars for Google Protobuf 3.21.12
+# Set env vars for Google Protobuf 3.25.5
 #######
 ENV PROTOBUF_HOME /opt/protobuf
 ENV PATH "${PATH}:/opt/protobuf/bin"

dev-support/docker/Dockerfile_aarch64

@@ -73,7 +73,7 @@ ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-arm64
 ENV SPOTBUGS_HOME /opt/spotbugs
 
 #######
-# Set env vars for Google Protobuf 3.21.12
+# Set env vars for Google Protobuf 3.25.5
 #######
 ENV PROTOBUF_HOME /opt/protobuf
 ENV PATH "${PATH}:/opt/protobuf/bin"

dev-support/docker/Dockerfile_debian_11

@@ -65,7 +65,7 @@ ENV JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64
 ENV SPOTBUGS_HOME=/opt/spotbugs
 
 #######
-# Set env vars for Google Protobuf 3.21.12
+# Set env vars for Google Protobuf 3.25.5
 #######
 ENV PROTOBUF_HOME=/opt/protobuf
 ENV PATH="${PATH}:/opt/protobuf/bin"

dev-support/docker/Dockerfile_rockylinux_8

@@ -94,7 +94,7 @@ ENV JAVA_HOME=/usr/lib/jvm/java-1.8.0
 ENV SPOTBUGS_HOME=/opt/spotbugs
 
 #######
-# Set env vars for Google Protobuf 3.21.12
+# Set env vars for Google Protobuf 3.25.5
 #######
 ENV PROTOBUF_HOME=/opt/protobuf
 ENV PATH="${PATH}:${PROTOBUF_HOME}/bin"

dev-support/docker/Dockerfile_ubuntu_24

@@ -72,7 +72,7 @@ ENV JAVA_HOME /usr/lib/jvm/java-17-openjdk-amd64
 ENV SPOTBUGS_HOME /opt/spotbugs
 
 #######
-# Set env vars for Google Protobuf 3.21.12
+# Set env vars for Google Protobuf 3.25.5
 #######
 ENV PROTOBUF_HOME /opt/protobuf
 ENV PATH "${PATH}:/opt/protobuf/bin"

dev-support/docker/pkg-resolver/install-protobuf.sh

@@ -27,29 +27,32 @@ if [ $? -eq 1 ]; then
   exit 1
 fi
 
-default_version="3.21.12"
+default_version="3.25.5"
 version_to_install=$default_version
 if [ -n "$2" ]; then
   version_to_install="$2"
 fi
 
-if [ "$version_to_install" != "3.21.12" ]; then
+if [ "$version_to_install" != "3.25.5" ]; then
   echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead"
   version_to_install=$default_version
 fi
 
-if [ "$version_to_install" == "3.21.12" ]; then
+if [ "$version_to_install" == "3.25.5" ]; then
   # hadolint ignore=DL3003
   mkdir -p /opt/protobuf-src &&
     curl -L -s -S \
-      https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.21.12.tar.gz \
+      https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.25.5.tar.gz \
       -o /opt/protobuf.tar.gz &&
     tar xzf /opt/protobuf.tar.gz --strip-components 1 -C /opt/protobuf-src &&
+    curl -L -s -S \
+      https://github.com/abseil/abseil-cpp/archive/refs/tags/20230802.1.tar.gz \
+      -o /opt/abseil-cpp.tar.gz &&
+    tar xzf /opt/abseil-cpp.tar.gz --strip-components 1 -C /opt/protobuf-src/third_party/abseil-cpp &&
     cd /opt/protobuf-src &&
-    ./autogen.sh &&
-    ./configure --prefix=/opt/protobuf &&
-    make "-j$(nproc)" &&
-    make install &&
+    cmake -S . -B build -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_BUILD_TESTS=OFF &&
+    cmake --build build --parallel $(nproc) &&
+    cmake --install build --prefix /opt/protobuf &&
     cd /root &&
     rm -rf /opt/protobuf-src
 else

dev-support/docker/vcpkg/vcpkg.json

@@ -10,7 +10,7 @@
   "overrides": [
     {
       "name": "protobuf",
-      "version": "3.21.12"
+      "version": "3.25.5"
     }
   ]
 }

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt

@@ -43,9 +43,70 @@ SET(CMAKE_PREFIX_PATH "${CMAKE_PREFIX_PATH};${CYRUS_SASL_DIR};${GSASL_DIR};$ENV{
 # Specify PROTOBUF_HOME so that find_package picks up the correct version
 SET(CMAKE_PREFIX_PATH "${CMAKE_PREFIX_PATH};$ENV{PROTOBUF_HOME}")
 
+if (BUILD_SHARED_LIBS AND MSVC)
+  # On MSVC Abseil is bundled into a single DLL.
+  # This condition is necessary as of abseil 20230125.3 when abseil is consumed via add_subdirectory,
+  # the abseil_dll target  is named abseil_dll, while if abseil is consumed via find_package, the target
+  # is called absl::abseil_dll
+  # Once https://github.com/abseil/abseil-cpp/pull/1466 is merged and released in the minimum version of
+  # abseil required by protobuf, it is possible to always link absl::abseil_dll and absl::abseil_test_dll
+  # and remove the if
+  if(protobuf_ABSL_PROVIDER STREQUAL "package")
+    set(protobuf_ABSL_USED_TARGETS absl::abseil_dll)
+    set(protobuf_ABSL_USED_TEST_TARGETS absl::abseil_test_dll)
+  else()
+    set(protobuf_ABSL_USED_TARGETS abseil_dll)
+    set(protobuf_ABSL_USED_TEST_TARGETS abseil_test_dll)
+  endif()
+else()
+  set(protobuf_ABSL_USED_TARGETS
+    absl::absl_check
+    absl::absl_log
+    absl::algorithm
+    absl::base
+    absl::bind_front
+    absl::bits
+    absl::btree
+    absl::cleanup
+    absl::cord
+    absl::core_headers
+    absl::debugging
+    absl::die_if_null
+    absl::dynamic_annotations
+    absl::flags
+    absl::flat_hash_map
+    absl::flat_hash_set
+    absl::function_ref
+    absl::hash
+    absl::layout
+    absl::log_initialize
+    absl::log_severity
+    absl::memory
+    absl::node_hash_map
+    absl::node_hash_set
+    absl::optional
+    absl::span
+    absl::status
+    absl::statusor
+    absl::strings
+    absl::synchronization
+    absl::time
+    absl::type_traits
+    absl::utility
+    absl::variant
+    utf8_range::utf8_validity
+    utf8_range::utf8_range
+  )
+  set(protobuf_ABSL_USED_TEST_TARGETS
+    absl::scoped_mock_log
+  )
+endif ()
+
 find_package(Doxygen)
 find_package(OpenSSL REQUIRED)
 find_package(Protobuf REQUIRED)
+find_package(absl REQUIRED)
+find_package(utf8_range REQUIRED)
 find_package(CyrusSASL)
 find_package(GSasl)
 find_package(Threads)
@@ -89,7 +150,10 @@ endif (NOT THREAD_LOCAL_SUPPORTED)
 # to compile some dummy code
 unset (PROTOC_IS_COMPATIBLE CACHE)
 set (CMAKE_REQUIRED_INCLUDES ${PROTOBUF_INCLUDE_DIRS})
-set (CMAKE_REQUIRED_LIBRARIES ${PROTOBUF_LIBRARY} ${PROTOBUF_PROTOC_LIBRARY})
+set (CMAKE_REQUIRED_LIBRARIES
+  ${PROTOBUF_LIBRARY}
+  ${PROTOBUF_PROTOC_LIBRARY}
+  ${protobuf_ABSL_USED_TARGETS})
 check_cxx_source_compiles(
     "#include <google/protobuf/io/printer.h>
     #include <string>
@@ -280,6 +344,7 @@ if (HADOOP_BUILD AND NOT MSVC)
   hadoop_target_link_dual_libraries(hdfspp
     ${LIB_DL}
     ${PROTOBUF_LIBRARY}
+    ${protobuf_ABSL_USED_TARGETS}
     ${OPENSSL_LIBRARIES}
     ${SASL_LIBRARIES}
     ${CMAKE_THREAD_LIBS_INIT}
@@ -291,6 +356,7 @@ else (HADOOP_BUILD AND NOT MSVC)
   target_link_libraries(hdfspp_static PUBLIC
     ${LIB_DL}
     ${PROTOBUF_LIBRARY}
+    ${protobuf_ABSL_USED_TARGETS}
     ${OPENSSL_LIBRARIES}
     ${SASL_LIBRARIES}
     ${CMAKE_THREAD_LIBS_INIT}

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/proto/CMakeLists.txt

@@ -38,7 +38,10 @@ protobuf_generate_cpp(PROTO_SRCS PROTO_HDRS
 )
 
 add_executable(protoc-gen-hrpc protoc_gen_hrpc.cc)
-target_link_libraries(protoc-gen-hrpc ${PROTOBUF_PROTOC_LIBRARY} ${PROTOBUF_LIBRARY})
+target_link_libraries(protoc-gen-hrpc
+  ${PROTOBUF_PROTOC_LIBRARY}
+  ${PROTOBUF_LIBRARY}
+  ${protobuf_ABSL_USED_TARGETS})
 
 function(GEN_HRPC SRCS)
   if(NOT ARGN)

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/proto/protoc_gen_hrpc.cc

@@ -48,7 +48,6 @@ class StubGenerator : public CodeGenerator {
 
 bool StubGenerator::Generate(const FileDescriptor *file, const std::string &,
                              GeneratorContext *ctx, std::string *) const {
-  namespace pb = ::google::protobuf;
   std::unique_ptr<ZeroCopyOutputStream> os(
       ctx->Open(StripProto(file->name()) + ".hrpc.inl"));
   Printer out(os.get(), '$');