[Enhancement] link libLLVM as single shared lib (#35955)

Signed-off-by: stdpain <drfeng08@gmail.com>
StarRocks · Nov 28, 2023 · a0e96f1 · a0e96f1
1 parent b990226
commit a0e96f1
Show file tree

Hide file tree

Showing 3 changed files with 14 additions and 69 deletions.
diff --git a/bin/start_backend.sh b/bin/start_backend.sh
@@ -83,6 +83,8 @@ fi
 export ASAN_OPTIONS=abort_on_error=1:disable_coredump=0:unmap_shadow_on_exit=1
 export LSAN_OPTIONS=suppressions=${STARROCKS_HOME}/conf/asan_suppressions.conf
 
+# Dependent dynamic libraries
+export LD_LIBRARY_PATH=$STARROCKS_HOME/lib:$LD_LIBRARY_PATH
 
 # ================== jvm section =======================
 if [ -e $STARROCKS_HOME/conf/hadoop_env.sh ]; then
@@ -97,7 +99,6 @@ fi
 
 if [ "$JAVA_HOME" = "" ]; then
     echo "[WARNING] JAVA_HOME env not set. Functions or features that requires jni will not work at all."
-    export LD_LIBRARY_PATH=$STARROCKS_HOME/lib:$LD_LIBRARY_PATH
 else
     java_version=$(jdk_version)
     if [[ $java_version -gt 8 ]]; then

diff --git a/build.sh b/build.sh
@@ -66,7 +66,7 @@ if [[ $OSTYPE == darwin* ]] ; then
     PARALLEL=$(sysctl -n hw.ncpu)
     # We know for sure that build-thirdparty.sh will fail on darwin platform, so just skip the step.
 else
-    if [[ ! -f ${STARROCKS_THIRDPARTY}/installed/llvm/lib/libLLVMInstCombine.a ]]; then
+    if [[ ! -f ${STARROCKS_THIRDPARTY}/installed/llvm/lib/libLLVM.so ]]; then
         echo "Thirdparty libraries need to be build ..."
         ${STARROCKS_THIRDPARTY}/build-thirdparty.sh
     fi
@@ -504,6 +504,7 @@ if [ ${BUILD_BE} -eq 1 ]; then
     cp -p ${STARROCKS_THIRDPARTY}/installed/hadoop/share/hadoop/tools/lib/azure-* ${STARROCKS_OUTPUT}/be/lib/hadoop/hdfs
     cp -p ${STARROCKS_THIRDPARTY}/installed/gcs_connector/*.jar ${STARROCKS_OUTPUT}/be/lib/hadoop/hdfs
     cp -r -p ${STARROCKS_THIRDPARTY}/installed/hadoop/lib/native ${STARROCKS_OUTPUT}/be/lib/hadoop/
+    cp ${STARROCKS_THIRDPARTY}/installed/llvm/lib/libLLVM.so ${STARROCKS_OUTPUT}/be/lib/
 
     rm -f ${STARROCKS_OUTPUT}/be/lib/hadoop/common/lib/log4j-1.2.17.jar
     rm -f ${STARROCKS_OUTPUT}/be/lib/hadoop/hdfs/lib/log4j-1.2.17.jar

diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh
@@ -224,64 +224,6 @@ build_llvm() {
     export CFLAGS="-O3 -fno-omit-frame-pointer -std=c99 -D_POSIX_C_SOURCE=200112L"
     export CXXFLAGS="-O3 -fno-omit-frame-pointer -Wno-class-memaccess"
 
-    LLVM_TARGET="X86"
-    if [[ "${MACHINE_TYPE}" == "aarch64" ]]; then
-        LLVM_TARGET="AArch64"
-    fi
-
-    LLVM_TARGETS_TO_BUILD=(
-        "LLVMBitstreamReader"
-        "LLVMRuntimeDyld" 
-        "LLVMOption"
-        "LLVMAsmPrinter"
-        "LLVMProfileData"
-        "LLVMAsmParser"
-        "LLVMOrcTargetProcess"
-        "LLVMExecutionEngine"
-        "LLVMBinaryFormat"
-        "LLVMDebugInfoDWARF"
-        "LLVMObjCARCOpts"
-        "LLVMCodeGen"
-        "LLVMMCDisassembler"
-        "LLVMSupport"
-        "LLVMJITLink"
-        "LLVMCFGuard"
-        "LLVMInstrumentation"
-        "LLVMInstCombine"
-        "LLVMipo"
-        "LLVMVectorize"
-        "LLVMIRReader"
-        "LLVMCore"
-        "LLVMTarget"
-        "LLVMMC"
-        "LLVMAnalysis"
-        "LLVMGlobalISel"
-        "LLVMScalarOpts"
-        "LLVMTargetParser"
-        "LLVMDemangle"
-        "LLVMRemarks"
-        "LLVMDebugInfoCodeView"
-        "LLVMOrcShared"
-        "LLVMOrcJIT"
-        "LLVMTextAPI"
-        "LLVMBitWriter"
-        "LLVMBitReader"
-        "LLVMObject"
-        "LLVMTransformUtils"
-        "LLVMSelectionDAG"
-        "LLVMMCParser"
-    )
-    if [ "${LLVM_TARGET}" == "X86" ]; then
-        LLVM_TARGETS_TO_BUILD+=("LLVMX86Info" "LLVMX86Desc" "LLVMX86CodeGen")
-    elif [ "${LLVM_TARGET}" == "AArch64" ]; then
-        LLVM_TARGETS_TO_BUILD+=("LLVMAArch64Info" "LLVMAArch64Desc" "LLVMAArch64CodeGen")
-    fi
-
-    LLVM_TARGETS_TO_INSTALL=()
-    for target in ${LLVM_TARGETS_TO_BUILD[@]}; do
-        LLVM_TARGETS_TO_INSTALL+=("install-${target}")
-    done
-
     check_if_source_exist $LLVM_SOURCE
 
     cd $TP_SOURCE_DIR
@@ -295,22 +237,23 @@ build_llvm() {
     -DLLVM_ENABLE_RTTI:Bool=True \
     -DLLVM_ENABLE_PIC:Bool=True \
     -DLLVM_ENABLE_TERMINFO:Bool=False \
-    -DLLVM_TARGETS_TO_BUILD=${LLVM_TARGET} \
-    -DLLVM_BUILD_LLVM_DYLIB:BOOL=False \
-    -DLLVM_INCLUDE_TOOLS:BOOL=False \
+    `# require tools/llvm-shlib for libllvm` \
+    -DLLVM_INCLUDE_TOOLS:BOOL=TRUE \
     -DLLVM_BUILD_TOOLS:BOOL=False \
     -DLLVM_INCLUDE_EXAMPLES:BOOL=False \
     -DLLVM_INCLUDE_TESTS:BOOL=False \
     -DLLVM_INCLUDE_BENCHMARKS:BOOL=False \
-    -DBUILD_SHARED_LIBS:BOOL=False \
+    -DLLVM_BUILD_LLVM_DYLIB=ON \
+    -DLLVM_LINK_LLVM_DYLIB=ON \
     -DCMAKE_BUILD_TYPE=Release \
     -DCMAKE_INSTALL_PREFIX=${TP_INSTALL_DIR}/llvm ../${LLVM_SOURCE}
 
-    # TODO(yueyang): Add more targets.
-    # This is a little bit hack, we need to minimize the build time and binary size.
-    ${BUILD_SYSTEM} -j$PARALLEL REQUIRES_RTTI=1 ${LLVM_TARGETS_TO_BUILD[@]}
-    ${BUILD_SYSTEM} install-llvm-headers
-    ${BUILD_SYSTEM} ${LLVM_TARGETS_TO_INSTALL[@]}
+
+    ${BUILD_SYSTEM} -j$PARALLEL REQUIRES_RTTI=1
+    ${BUILD_SYSTEM} install
+    # only reserve libLLVM.so
+    rm -f ${TP_INSTALL_DIR}/llvm/*.a
+    rm -rf ${TP_INSTALL_DIR}/llvm/cmake
 
     restore_compile_flags
 }