Skip to content

Commit c81fdf9

Browse files
pan3793pan3793
committed
HADOOP-19605. (3.4) Upgrade Protobuf 3.25.5 for docker images (apache#7780)
* HADOOP-19605. Upgrade Protobuf 3.25.5 for docker images. Reviewed-by: Steve Loughran <stevel@apache.org> Signed-off-by: Shilun Fan <slfan1989@apache.org>
1 parent 38f9c1c commit c81fdf9

File tree

12 files changed

+137
-57
lines changed

12 files changed

+137
-57
lines changed

‎BUILDING.txt‎

Lines changed: 34 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ Requirements:
77
* JDK 1.8
88
* Maven 3.3 or later
99
* Boost 1.72 (if compiling native code)
10-
* Protocol Buffers 3.21.12 (if compiling native code)
10+
* Protocol Buffers 3.25.5 (if compiling native code)
1111
* CMake 3.19 or newer (if compiling native code)
1212
* Zlib devel (if compiling native code)
1313
* Cyrus SASL devel (if compiling native code)
@@ -74,13 +74,15 @@ Refer to dev-support/docker/Dockerfile):
7474
$ ./bootstrap
7575
$ make -j$(nproc)
7676
$ sudo make install
77-
* Protocol Buffers 3.21.12 (required to build native code)
78-
$ curl -L https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.21.12.tar.gz > protobuf-3.21.12.tar.gz
79-
$ tar -zxvf protobuf-3.21.12.tar.gz && cd protobuf-3.21.12
80-
$ ./autogen.sh
81-
$ ./configure
82-
$ make -j$(nproc)
83-
$ sudo make install
77+
* Protocol Buffers 3.25.5 (required to build native code)
78+
$ curl -L https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.25.5.tar.gz > protobuf-3.25.5.tar.gz
79+
$ curl -L https://github.com/abseil/abseil-cpp/archive/refs/tags/20230802.1.tar.gz > abseil-cpp-20230802.1.tar.gz
80+
$ tar -zxvf protobuf-3.25.5.tar.gz
81+
$ tar -zxvf abseil-cpp-20230802.1.tar.gz --strip-components 1 -C protobuf-3.25.5/third_party/abseil-cpp
82+
$ cd protobuf-3.25.5
83+
$ cmake -S . -B build -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_BUILD_TESTS=OFF
84+
$ cmake --build build --parallel $(nproc)
85+
$ sudo cmake --install build
8486
* Boost
8587
$ curl -L https://sourceforge.net/projects/boost/files/boost/1.72.0/boost_1_72_0.tar.bz2/download > boost_1_72_0.tar.bz2
8688
$ tar --bzip2 -xf boost_1_72_0.tar.bz2 && cd boost_1_72_0
@@ -430,14 +432,15 @@ Installing required dependencies for clean install of macOS 10.14:
430432
* Install native libraries, only openssl is required to compile native code,
431433
you may optionally install zlib, lz4, etc.
432434
$ brew install openssl
433-
* Protocol Buffers 3.21.12 (required to compile native code)
434-
$ curl -L https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.21.12.tar.gz > protobuf-3.21.12.tar.gz
435-
$ tar -zxvf protobuf-3.21.12.tar.gz && cd protobuf-3.21.12
436-
$ ./autogen.sh
437-
$ ./configure
438-
$ make
439-
$ make check
440-
$ make install
435+
* Protocol Buffers 3.25.5 (required to build native code)
436+
$ curl -L https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.25.5.tar.gz > protobuf-3.25.5.tar.gz
437+
$ curl -L https://github.com/abseil/abseil-cpp/archive/refs/tags/20230802.1.tar.gz > abseil-cpp-20230802.1.tar.gz
438+
$ tar -zxvf protobuf-3.25.5.tar.gz
439+
$ tar -zxvf abseil-cpp-20230802.1.tar.gz --strip-components 1 -C protobuf-3.25.5/third_party/abseil-cpp
440+
$ cd protobuf-3.25.5
441+
$ cmake -S . -B build -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_BUILD_TESTS=OFF
442+
$ cmake --build build --parallel $(nproc)
443+
$ cmake --install build
441444
$ protoc --version
442445

443446
Note that building Hadoop 3.1.1/3.1.2/3.2.0 native code from source is broken
@@ -469,13 +472,15 @@ Building on CentOS 8
469472
* Install python2 for building documentation.
470473
$ sudo dnf install python2
471474

472-
* Install Protocol Buffers v3.21.12.
473-
$ curl -L https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.21.12.tar.gz > protobuf-3.21.12.tar.gz
474-
$ tar -zxvf protobuf-3.21.12.tar.gz && cd protobuf-3.21.12
475-
$ ./autogen.sh
476-
$ ./configure --prefix=/usr/local
477-
$ make
478-
$ sudo make install
475+
* Protocol Buffers 3.25.5 (required to build native code)
476+
$ curl -L https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.25.5.tar.gz > protobuf-3.25.5.tar.gz
477+
$ curl -L https://github.com/abseil/abseil-cpp/archive/refs/tags/20230802.1.tar.gz > abseil-cpp-20230802.1.tar.gz
478+
$ tar -zxvf protobuf-3.25.5.tar.gz
479+
$ tar -zxvf abseil-cpp-20230802.1.tar.gz --strip-components 1 -C protobuf-3.25.5/third_party/abseil-cpp
480+
$ cd protobuf-3.25.5
481+
$ cmake -S . -B build -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_BUILD_TESTS=OFF
482+
$ cmake --build build --parallel $(nproc)
483+
$ sudo cmake --install build --prefix /usr/local
479484
$ cd ..
480485

481486
* Install libraries provided by CentOS 8.
@@ -525,9 +530,15 @@ Requirements:
525530

526531
* Windows 10
527532
* JDK 1.8
533+
<<<<<<< HEAD
528534
* Maven 3.0 or later (maven.apache.org)
529535
* Boost 1.72 (boost.org)
530536
* Protocol Buffers 3.21.12 (https://github.com/protocolbuffers/protobuf/tags)
537+
=======
538+
* Maven 3.3 or later (maven.apache.org)
539+
* Boost 1.86.0 (boost.org)
540+
* Protocol Buffers 3.25.5 (https://github.com/protocolbuffers/protobuf/tags)
541+
>>>>>>> d361d0ddff2 (HADOOP-19605. Upgrade Protobuf 3.25.5 for docker images (#7780))
531542
* CMake 3.19 or newer (cmake.org)
532543
* Visual Studio 2019 (visualstudio.com)
533544
* Windows SDK 8.1 (optional, if building CPU rate control for the container executor. Get this from

‎dev-support/docker/Dockerfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@ ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64
6666
ENV SPOTBUGS_HOME /opt/spotbugs
6767

6868
#######
69-
# Set env vars for Google Protobuf 3.21.12
69+
# Set env vars for Google Protobuf 3.25.5
7070
#######
7171
ENV PROTOBUF_HOME /opt/protobuf
7272
ENV PATH "${PATH}:/opt/protobuf/bin"

‎dev-support/docker/Dockerfile_aarch64‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-arm64
6767
ENV SPOTBUGS_HOME /opt/spotbugs
6868

6969
#######
70-
# Set env vars for Google Protobuf 3.21.12
70+
# Set env vars for Google Protobuf 3.25.5
7171
#######
7272
ENV PROTOBUF_HOME /opt/protobuf
7373
ENV PATH "${PATH}:/opt/protobuf/bin"

‎dev-support/docker/Dockerfile_centos_8‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,7 @@ ENV JAVA_HOME /usr/lib/jvm/java-1.8.0
101101
ENV SPOTBUGS_HOME /opt/spotbugs
102102

103103
#######
104-
# Set env vars for Google Protobuf 3.21.12
104+
# Set env vars for Google Protobuf 3.25.5
105105
#######
106106
ENV PROTOBUF_HOME /opt/protobuf
107107
ENV PATH "${PATH}:/opt/protobuf/bin"

‎dev-support/docker/Dockerfile_debian_10‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@ ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-amd64
6666
ENV SPOTBUGS_HOME /opt/spotbugs
6767

6868
#######
69-
# Set env vars for Google Protobuf 3.21.12
69+
# Set env vars for Google Protobuf 3.25.5
7070
#######
7171
ENV PROTOBUF_HOME /opt/protobuf
7272
ENV PATH "${PATH}:/opt/protobuf/bin"

‎dev-support/docker/pkg-resolver/install-protobuf.sh‎

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -27,29 +27,32 @@ if [ $? -eq 1 ]; then
2727
exit 1
2828
fi
2929

30-
default_version="3.21.12"
30+
default_version="3.25.5"
3131
version_to_install=$default_version
3232
if [ -n "$2" ]; then
3333
version_to_install="$2"
3434
fi
3535

36-
if [ "$version_to_install" != "3.21.12" ]; then
36+
if [ "$version_to_install" != "3.25.5" ]; then
3737
echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead"
3838
version_to_install=$default_version
3939
fi
4040

41-
if [ "$version_to_install" == "3.21.12" ]; then
41+
if [ "$version_to_install" == "3.25.5" ]; then
4242
# hadolint ignore=DL3003
4343
mkdir -p /opt/protobuf-src &&
4444
curl -L -s -S \
45-
https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.21.12.tar.gz \
45+
https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.25.5.tar.gz \
4646
-o /opt/protobuf.tar.gz &&
4747
tar xzf /opt/protobuf.tar.gz --strip-components 1 -C /opt/protobuf-src &&
48+
curl -L -s -S \
49+
https://github.com/abseil/abseil-cpp/archive/refs/tags/20230802.1.tar.gz \
50+
-o /opt/abseil-cpp.tar.gz &&
51+
tar xzf /opt/abseil-cpp.tar.gz --strip-components 1 -C /opt/protobuf-src/third_party/abseil-cpp &&
4852
cd /opt/protobuf-src &&
49-
./autogen.sh &&
50-
./configure --prefix=/opt/protobuf &&
51-
make "-j$(nproc)" &&
52-
make install &&
53+
cmake -S . -B build -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_BUILD_TESTS=OFF &&
54+
cmake --build build --parallel $(nproc) &&
55+
cmake --install build --prefix /opt/protobuf &&
5356
cd /root &&
5457
rm -rf /opt/protobuf-src
5558
else

‎hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt‎

Lines changed: 67 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,9 +43,70 @@ SET(CMAKE_PREFIX_PATH "${CMAKE_PREFIX_PATH};${CYRUS_SASL_DIR};${GSASL_DIR};$ENV{
4343
# Specify PROTOBUF_HOME so that find_package picks up the correct version
4444
SET(CMAKE_PREFIX_PATH "${CMAKE_PREFIX_PATH};$ENV{PROTOBUF_HOME}")
4545

46+
if (BUILD_SHARED_LIBS AND MSVC)
47+
# On MSVC Abseil is bundled into a single DLL.
48+
# This condition is necessary as of abseil 20230125.3 when abseil is consumed via add_subdirectory,
49+
# the abseil_dll target is named abseil_dll, while if abseil is consumed via find_package, the target
50+
# is called absl::abseil_dll
51+
# Once https://github.com/abseil/abseil-cpp/pull/1466 is merged and released in the minimum version of
52+
# abseil required by protobuf, it is possible to always link absl::abseil_dll and absl::abseil_test_dll
53+
# and remove the if
54+
if(protobuf_ABSL_PROVIDER STREQUAL "package")
55+
set(protobuf_ABSL_USED_TARGETS absl::abseil_dll)
56+
set(protobuf_ABSL_USED_TEST_TARGETS absl::abseil_test_dll)
57+
else()
58+
set(protobuf_ABSL_USED_TARGETS abseil_dll)
59+
set(protobuf_ABSL_USED_TEST_TARGETS abseil_test_dll)
60+
endif()
61+
else()
62+
set(protobuf_ABSL_USED_TARGETS
63+
absl::absl_check
64+
absl::absl_log
65+
absl::algorithm
66+
absl::base
67+
absl::bind_front
68+
absl::bits
69+
absl::btree
70+
absl::cleanup
71+
absl::cord
72+
absl::core_headers
73+
absl::debugging
74+
absl::die_if_null
75+
absl::dynamic_annotations
76+
absl::flags
77+
absl::flat_hash_map
78+
absl::flat_hash_set
79+
absl::function_ref
80+
absl::hash
81+
absl::layout
82+
absl::log_initialize
83+
absl::log_severity
84+
absl::memory
85+
absl::node_hash_map
86+
absl::node_hash_set
87+
absl::optional
88+
absl::span
89+
absl::status
90+
absl::statusor
91+
absl::strings
92+
absl::synchronization
93+
absl::time
94+
absl::type_traits
95+
absl::utility
96+
absl::variant
97+
utf8_range::utf8_validity
98+
utf8_range::utf8_range
99+
)
100+
set(protobuf_ABSL_USED_TEST_TARGETS
101+
absl::scoped_mock_log
102+
)
103+
endif ()
104+
46105
find_package(Doxygen)
47106
find_package(OpenSSL REQUIRED)
48107
find_package(Protobuf REQUIRED)
108+
find_package(absl REQUIRED)
109+
find_package(utf8_range REQUIRED)
49110
find_package(CyrusSASL)
50111
find_package(GSasl)
51112
find_package(Threads)
@@ -89,7 +150,10 @@ endif (NOT THREAD_LOCAL_SUPPORTED)
89150
# to compile some dummy code
90151
unset (PROTOC_IS_COMPATIBLE CACHE)
91152
set (CMAKE_REQUIRED_INCLUDES ${PROTOBUF_INCLUDE_DIRS})
92-
set (CMAKE_REQUIRED_LIBRARIES ${PROTOBUF_LIBRARY} ${PROTOBUF_PROTOC_LIBRARY})
153+
set (CMAKE_REQUIRED_LIBRARIES
154+
${PROTOBUF_LIBRARY}
155+
${PROTOBUF_PROTOC_LIBRARY}
156+
${protobuf_ABSL_USED_TARGETS})
93157
check_cxx_source_compiles(
94158
"#include <google/protobuf/io/printer.h>
95159
#include <string>
@@ -280,6 +344,7 @@ if (HADOOP_BUILD AND NOT MSVC)
280344
hadoop_target_link_dual_libraries(hdfspp
281345
${LIB_DL}
282346
${PROTOBUF_LIBRARY}
347+
${protobuf_ABSL_USED_TARGETS}
283348
${OPENSSL_LIBRARIES}
284349
${SASL_LIBRARIES}
285350
${CMAKE_THREAD_LIBS_INIT}
@@ -291,6 +356,7 @@ else (HADOOP_BUILD AND NOT MSVC)
291356
target_link_libraries(hdfspp_static PUBLIC
292357
${LIB_DL}
293358
${PROTOBUF_LIBRARY}
359+
${protobuf_ABSL_USED_TARGETS}
294360
${OPENSSL_LIBRARIES}
295361
${SASL_LIBRARIES}
296362
${CMAKE_THREAD_LIBS_INIT}

‎hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/proto/CMakeLists.txt‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,10 @@ protobuf_generate_cpp(PROTO_SRCS PROTO_HDRS
3838
)
3939

4040
add_executable(protoc-gen-hrpc protoc_gen_hrpc.cc)
41-
target_link_libraries(protoc-gen-hrpc ${PROTOBUF_PROTOC_LIBRARY} ${PROTOBUF_LIBRARY})
41+
target_link_libraries(protoc-gen-hrpc
42+
${PROTOBUF_PROTOC_LIBRARY}
43+
${PROTOBUF_LIBRARY}
44+
${protobuf_ABSL_USED_TARGETS})
4245

4346
function(GEN_HRPC SRCS)
4447
if(NOT ARGN)

‎hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/proto/protoc_gen_hrpc.cc‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,6 @@ class StubGenerator : public CodeGenerator {
4848

4949
bool StubGenerator::Generate(const FileDescriptor *file, const std::string &,
5050
GeneratorContext *ctx, std::string *) const {
51-
namespace pb = ::google::protobuf;
5251
std::unique_ptr<ZeroCopyOutputStream> os(
5352
ctx->Open(StripProto(file->name()) + ".hrpc.inl"));
5453
Printer out(os.get(), '$');

0 commit comments

Comments
 (0)