From a0e96f134133636d5bb118fe210e44be44408aba Mon Sep 17 00:00:00 2001 From: stdpain <34912776+stdpain@users.noreply.github.com> Date: Tue, 28 Nov 2023 20:31:03 +0800 Subject: [PATCH] [Enhancement] link libLLVM as single shared lib (#35955) Signed-off-by: stdpain --- bin/start_backend.sh | 3 +- build.sh | 3 +- thirdparty/build-thirdparty.sh | 77 +++++----------------------------- 3 files changed, 14 insertions(+), 69 deletions(-) diff --git a/bin/start_backend.sh b/bin/start_backend.sh index 69d06fb4625df..d659190871621 100755 --- a/bin/start_backend.sh +++ b/bin/start_backend.sh @@ -83,6 +83,8 @@ fi export ASAN_OPTIONS=abort_on_error=1:disable_coredump=0:unmap_shadow_on_exit=1 export LSAN_OPTIONS=suppressions=${STARROCKS_HOME}/conf/asan_suppressions.conf +# Dependent dynamic libraries +export LD_LIBRARY_PATH=$STARROCKS_HOME/lib:$LD_LIBRARY_PATH # ================== jvm section ======================= if [ -e $STARROCKS_HOME/conf/hadoop_env.sh ]; then @@ -97,7 +99,6 @@ fi if [ "$JAVA_HOME" = "" ]; then echo "[WARNING] JAVA_HOME env not set. Functions or features that requires jni will not work at all." - export LD_LIBRARY_PATH=$STARROCKS_HOME/lib:$LD_LIBRARY_PATH else java_version=$(jdk_version) if [[ $java_version -gt 8 ]]; then diff --git a/build.sh b/build.sh index b41593d5e2a13..6a3afb289f0d0 100755 --- a/build.sh +++ b/build.sh @@ -66,7 +66,7 @@ if [[ $OSTYPE == darwin* ]] ; then PARALLEL=$(sysctl -n hw.ncpu) # We know for sure that build-thirdparty.sh will fail on darwin platform, so just skip the step. else - if [[ ! -f ${STARROCKS_THIRDPARTY}/installed/llvm/lib/libLLVMInstCombine.a ]]; then + if [[ ! -f ${STARROCKS_THIRDPARTY}/installed/llvm/lib/libLLVM.so ]]; then echo "Thirdparty libraries need to be build ..." ${STARROCKS_THIRDPARTY}/build-thirdparty.sh fi @@ -504,6 +504,7 @@ if [ ${BUILD_BE} -eq 1 ]; then cp -p ${STARROCKS_THIRDPARTY}/installed/hadoop/share/hadoop/tools/lib/azure-* ${STARROCKS_OUTPUT}/be/lib/hadoop/hdfs cp -p ${STARROCKS_THIRDPARTY}/installed/gcs_connector/*.jar ${STARROCKS_OUTPUT}/be/lib/hadoop/hdfs cp -r -p ${STARROCKS_THIRDPARTY}/installed/hadoop/lib/native ${STARROCKS_OUTPUT}/be/lib/hadoop/ + cp ${STARROCKS_THIRDPARTY}/installed/llvm/lib/libLLVM.so ${STARROCKS_OUTPUT}/be/lib/ rm -f ${STARROCKS_OUTPUT}/be/lib/hadoop/common/lib/log4j-1.2.17.jar rm -f ${STARROCKS_OUTPUT}/be/lib/hadoop/hdfs/lib/log4j-1.2.17.jar diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh index d6423d9ae4e75..3e0ea98e545be 100755 --- a/thirdparty/build-thirdparty.sh +++ b/thirdparty/build-thirdparty.sh @@ -224,64 +224,6 @@ build_llvm() { export CFLAGS="-O3 -fno-omit-frame-pointer -std=c99 -D_POSIX_C_SOURCE=200112L" export CXXFLAGS="-O3 -fno-omit-frame-pointer -Wno-class-memaccess" - LLVM_TARGET="X86" - if [[ "${MACHINE_TYPE}" == "aarch64" ]]; then - LLVM_TARGET="AArch64" - fi - - LLVM_TARGETS_TO_BUILD=( - "LLVMBitstreamReader" - "LLVMRuntimeDyld" - "LLVMOption" - "LLVMAsmPrinter" - "LLVMProfileData" - "LLVMAsmParser" - "LLVMOrcTargetProcess" - "LLVMExecutionEngine" - "LLVMBinaryFormat" - "LLVMDebugInfoDWARF" - "LLVMObjCARCOpts" - "LLVMCodeGen" - "LLVMMCDisassembler" - "LLVMSupport" - "LLVMJITLink" - "LLVMCFGuard" - "LLVMInstrumentation" - "LLVMInstCombine" - "LLVMipo" - "LLVMVectorize" - "LLVMIRReader" - "LLVMCore" - "LLVMTarget" - "LLVMMC" - "LLVMAnalysis" - "LLVMGlobalISel" - "LLVMScalarOpts" - "LLVMTargetParser" - "LLVMDemangle" - "LLVMRemarks" - "LLVMDebugInfoCodeView" - "LLVMOrcShared" - "LLVMOrcJIT" - "LLVMTextAPI" - "LLVMBitWriter" - "LLVMBitReader" - "LLVMObject" - "LLVMTransformUtils" - "LLVMSelectionDAG" - "LLVMMCParser" - ) - if [ "${LLVM_TARGET}" == "X86" ]; then - LLVM_TARGETS_TO_BUILD+=("LLVMX86Info" "LLVMX86Desc" "LLVMX86CodeGen") - elif [ "${LLVM_TARGET}" == "AArch64" ]; then - LLVM_TARGETS_TO_BUILD+=("LLVMAArch64Info" "LLVMAArch64Desc" "LLVMAArch64CodeGen") - fi - - LLVM_TARGETS_TO_INSTALL=() - for target in ${LLVM_TARGETS_TO_BUILD[@]}; do - LLVM_TARGETS_TO_INSTALL+=("install-${target}") - done - check_if_source_exist $LLVM_SOURCE cd $TP_SOURCE_DIR @@ -295,22 +237,23 @@ build_llvm() { -DLLVM_ENABLE_RTTI:Bool=True \ -DLLVM_ENABLE_PIC:Bool=True \ -DLLVM_ENABLE_TERMINFO:Bool=False \ - -DLLVM_TARGETS_TO_BUILD=${LLVM_TARGET} \ - -DLLVM_BUILD_LLVM_DYLIB:BOOL=False \ - -DLLVM_INCLUDE_TOOLS:BOOL=False \ + `# require tools/llvm-shlib for libllvm` \ + -DLLVM_INCLUDE_TOOLS:BOOL=TRUE \ -DLLVM_BUILD_TOOLS:BOOL=False \ -DLLVM_INCLUDE_EXAMPLES:BOOL=False \ -DLLVM_INCLUDE_TESTS:BOOL=False \ -DLLVM_INCLUDE_BENCHMARKS:BOOL=False \ - -DBUILD_SHARED_LIBS:BOOL=False \ + -DLLVM_BUILD_LLVM_DYLIB=ON \ + -DLLVM_LINK_LLVM_DYLIB=ON \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_INSTALL_PREFIX=${TP_INSTALL_DIR}/llvm ../${LLVM_SOURCE} - # TODO(yueyang): Add more targets. - # This is a little bit hack, we need to minimize the build time and binary size. - ${BUILD_SYSTEM} -j$PARALLEL REQUIRES_RTTI=1 ${LLVM_TARGETS_TO_BUILD[@]} - ${BUILD_SYSTEM} install-llvm-headers - ${BUILD_SYSTEM} ${LLVM_TARGETS_TO_INSTALL[@]} + + ${BUILD_SYSTEM} -j$PARALLEL REQUIRES_RTTI=1 + ${BUILD_SYSTEM} install + # only reserve libLLVM.so + rm -f ${TP_INSTALL_DIR}/llvm/*.a + rm -rf ${TP_INSTALL_DIR}/llvm/cmake restore_compile_flags }