Skip to content

Commit

Permalink
[deps](libhdfs) add official hadoop libhdfs for x86 (apache#17435)
Browse files Browse the repository at this point in the history
This is the first step to introduce official hadoop libhdfs to Doris.
Because the current hdfs client libhdfs3 lacks some important feature and is hard to maintain.

Download the hadoop 3.3.4 binary from hadoop website: https://hadoop.apache.org/releases.html
Extract libs and headers which are used for libhdfs, and pack them into hadoop_lib_3.3.4-x86.tar.gz
Upload it to https://github.com/apache/doris-thirdparty/releases/tag/hadoop-libs-3.3.4

TODO:
The hadoop libs for arm is missing, we need to find a way to build it
  • Loading branch information
morningman authored Mar 6, 2023
1 parent 8e6c34d commit bc5f16d
Show file tree
Hide file tree
Showing 2 changed files with 21 additions and 0 deletions.
14 changes: 14 additions & 0 deletions thirdparty/build-thirdparty.sh
Original file line number Diff line number Diff line change
Expand Up @@ -1603,6 +1603,16 @@ build_clucene() {
cp -rf src/contribs-lib/CLucene/analysis/jieba/dict "${TP_INSTALL_DIR}"/share/
}

# hadoop_libs_x86
build_hadoop_libs_x86() {
check_if_source_exist "${HADOOP_LIBS_X86_SOURCE}"
cd "${TP_SOURCE_DIR}/${HADOOP_LIBS_X86_SOURCE}"
mkdir -p "${TP_INSTALL_DIR}/include/hadoop_hdfs/"
mkdir -p "${TP_INSTALL_DIR}/lib/hadoop_hdfs/"
cp ./include/hdfs.h "${TP_INSTALL_DIR}/include/hadoop_hdfs/"
cp ./native/libhdfs.a "${TP_INSTALL_DIR}/lib/hadoop_hdfs/"
}

if [[ "$(uname -s)" == 'Darwin' ]]; then
echo 'build for Darwin'
build_binutils
Expand Down Expand Up @@ -1668,4 +1678,8 @@ build_concurrentqueue
build_fast_float
build_clucene

if [[ "$(uname -m)" == 'x86_64' ]]; then
build_hadoop_libs_x86
fi

echo "Finished to build all thirdparties"
7 changes: 7 additions & 0 deletions thirdparty/vars.sh
Original file line number Diff line number Diff line change
Expand Up @@ -458,6 +458,12 @@ FAST_FLOAT_NAME=fast_float-3.9.0.tar.gz
FAST_FLOAT_SOURCE=fast_float-3.9.0
FAST_FLOAT_MD5SUM="5656b0d8b150a3b157cfb092d214f6ea"

# libhdfs
HADOOP_LIBS_X86_DOWNLOAD="https://github.com/apache/doris-thirdparty/releases/download/hadoop-libs-3.3.4/hadoop_lib_3.3.4-x86.tar.gz"
HADOOP_LIBS_X86_NAME="hadoop_lib_3.3.4-x86.tar.gz"
HADOOP_LIBS_X86_SOURCE="hadoop_lib_3.3.4-x86"
HADOOP_LIBS_X86_MD5SUM="96117450170487f007ffeca5ddf62f7e"

# all thirdparties which need to be downloaded is set in array TP_ARCHIVES
export TP_ARCHIVES=(
'CLUCENE'
Expand Down Expand Up @@ -525,6 +531,7 @@ export TP_ARCHIVES=(
'XXHASH'
'CONCURRENTQUEUE'
'FAST_FLOAT'
'HADOOP_LIBS_X86'
)

if [[ "$(uname -s)" == 'Darwin' ]]; then
Expand Down

0 comments on commit bc5f16d

Please sign in to comment.