Skip to content

Commit

Permalink
[chore](thirdparty) Update bitshuffle from 0.3.5 to 0.5.1 (apache#15993)
Browse files Browse the repository at this point in the history
In order to use AVX512 instructions. See apache#15972
  • Loading branch information
HackToday authored Jan 17, 2023
1 parent 388d623 commit 3810727
Show file tree
Hide file tree
Showing 2 changed files with 13 additions and 10 deletions.
15 changes: 9 additions & 6 deletions thirdparty/build-thirdparty.sh
Original file line number Diff line number Diff line change
Expand Up @@ -1035,11 +1035,11 @@ build_bitshuffle() {
cd "${TP_SOURCE_DIR}/${BITSHUFFLE_SOURCE}"
PREFIX="${TP_INSTALL_DIR}"

# This library has significant optimizations when built with -mavx2. However,
# we still need to support non-AVX2-capable hardware. So, we build it twice,
# once with the flag and once without, and use some linker tricks to
# suffix the AVX2 symbols with '_avx2'.
arches=('default' 'avx2')
# This library has significant optimizations when built with AVX2/AVX512. However,
# we still need to support non-AVX2-capable hardware. So, we build it three times,
# with the flag AVX2, AVX512 each and once without, and use some linker tricks to
# suffix the AVX2 symbols with '_avx2', AVX512 symbols with '_avx512'
arches=('default' 'avx2' 'avx512')
MACHINE_TYPE="$(uname -m)"
# Becuase aarch64 don't support avx2, disable it.
if [[ "${MACHINE_TYPE}" == "aarch64" || "${MACHINE_TYPE}" == 'arm64' ]]; then
Expand All @@ -1052,6 +1052,9 @@ build_bitshuffle() {
if [[ "${arch}" == "avx2" ]]; then
arch_flag="-mavx2"
fi
if [[ "${arch}" == "avx512" ]]; then
arch_flag="-mavx512bw -mavx512f"
fi
tmp_obj="bitshuffle_${arch}_tmp.o"
dst_obj="bitshuffle_${arch}.o"
"${CC}" ${EXTRA_CFLAGS:+${EXTRA_CFLAGS}} ${arch_flag:+${arch_flag}} -std=c99 "-I${PREFIX}/include/lz4" -O3 -DNDEBUG -c \
Expand All @@ -1061,7 +1064,7 @@ build_bitshuffle() {
# Merge the object files together to produce a combined .o file.
"${ld}" -r -o "${tmp_obj}" bitshuffle_core.o bitshuffle.o iochain.o
# For the AVX2 symbols, suffix them.
if [[ "${arch}" == "avx2" ]]; then
if [[ "${arch}" == "avx2" ]] || [[ "${arch}" == "avx512" ]]; then
local nm="${DORIS_BIN_UTILS}/nm"
local objcopy="${DORIS_BIN_UTILS}/objcopy"

Expand Down
8 changes: 4 additions & 4 deletions thirdparty/vars.sh
Original file line number Diff line number Diff line change
Expand Up @@ -263,10 +263,10 @@ S2_SOURCE=s2geometry-0.10.0
S2_MD5SUM="c68f3c5d326dde9255681b9201393a9f"

# bitshuffle
BITSHUFFLE_DOWNLOAD="https://github.com/kiyo-masui/bitshuffle/archive/0.3.5.tar.gz"
BITSHUFFLE_NAME=bitshuffle-0.3.5.tar.gz
BITSHUFFLE_SOURCE=bitshuffle-0.3.5
BITSHUFFLE_MD5SUM="2648ec7ccd0b896595c6636d926fc867"
BITSHUFFLE_DOWNLOAD="https://github.com/kiyo-masui/bitshuffle/archive/0.5.1.tar.gz"
BITSHUFFLE_NAME=bitshuffle-0.5.1.tar.gz
BITSHUFFLE_SOURCE=bitshuffle-0.5.1
BITSHUFFLE_MD5SUM="b3bf6a9838927f7eb62214981c138e2f"

# croaringbitmap
CROARINGBITMAP_DOWNLOAD="https://github.com/RoaringBitmap/CRoaring/archive/refs/tags/v0.4.0.tar.gz"
Expand Down

0 comments on commit 3810727

Please sign in to comment.