[chore](ci) prepare jdk17 in cloud_p* and performance (apache#31685)

Ma1oneZhang · Mar 3, 2024 · d459ac2 · d459ac2
1 parent b71686d
commit d459ac2
Show file tree

Hide file tree

Showing 6 changed files with 45 additions and 15 deletions.
diff --git a/regression-test/pipeline/cloud_p0/deploy.sh b/regression-test/pipeline/cloud_p0/deploy.sh
@@ -69,6 +69,8 @@ exit_flag=0
     print_doris_conf
 
     echo "#### 4. start Doris"
+    JAVA_HOME="$(find /usr/lib/jvm -maxdepth 1 -type d -name 'java-17-*' | sed -n '1p')"
+    export JAVA_HOME
     if ! start_doris_ms; then exit 1; fi
     if ! start_doris_recycler; then exit 1; fi
     if ! create_warehouse; then exit 1; fi

diff --git a/regression-test/pipeline/cloud_p0/prepare.sh b/regression-test/pipeline/cloud_p0/prepare.sh
@@ -61,7 +61,7 @@ if ${DEBUG:-false}; then
 fi
 
 # shellcheck source=/dev/null
-# stop_doris, clean_fdb, install_fdb
+# stop_doris, clean_fdb, install_fdb, install_java
 source "${teamcity_build_checkoutDir}"/regression-test/pipeline/common/doris-utils.sh
 # shellcheck source=/dev/null
 # check_oss_file_exist
@@ -94,7 +94,8 @@ fi
 source "$(bash "${teamcity_build_checkoutDir}"/regression-test/pipeline/common/get-or-set-tmp-env.sh 'get')"
 if ${skip_pipeline:=false}; then echo "INFO: skip build pipline" && exit 0; else echo "INFO: no skip"; fi
 if [[ "${target_branch}" == "master" ]]; then
-    echo "INFO: PR target branch ${target_branch} is in (master)"
+    echo "INFO: PR target branch ${target_branch}"
+    install_java
 else
     echo "WARNING: PR target branch ${target_branch} is NOT in (master), skip pipeline."
     bash "${teamcity_build_checkoutDir}"/regression-test/pipeline/common/get-or-set-tmp-env.sh 'set' "export skip_pipeline=true"

diff --git a/regression-test/pipeline/cloud_p0/run.sh b/regression-test/pipeline/cloud_p0/run.sh
@@ -36,7 +36,7 @@ if [[ -z "${cos_ak}" || -z "${cos_sk}" ]]; then echo "ERROR: env cos_ak or cos_s
 source "$(bash "${teamcity_build_checkoutDir}"/regression-test/pipeline/common/get-or-set-tmp-env.sh 'get')"
 if ${skip_pipeline:=false}; then echo "INFO: skip build pipline" && exit 0; else echo "INFO: no skip"; fi
 
-echo "#### Run tpcds test on Doris ####"
+echo "#### Run cloud_p0 test on Doris ####"
 DORIS_HOME="${teamcity_build_checkoutDir}/output"
 export DORIS_HOME
 exit_flag=0
@@ -51,6 +51,8 @@ run() {
     echo "sk='${cos_sk}'" >>"${teamcity_build_checkoutDir}"/regression-test/pipeline/cloud_p0/conf/regression-conf-custom.groovy
     cp -f "${teamcity_build_checkoutDir}"/regression-test/pipeline/cloud_p0/conf/regression-conf-custom.groovy \
         "${teamcity_build_checkoutDir}"/regression-test/conf/
+    JAVA_HOME="$(find /usr/lib/jvm -maxdepth 1 -type d -name 'java-8-*' | sed -n '1p')"
+    export JAVA_HOME
     if "${teamcity_build_checkoutDir}"/run-regression-test.sh \
         --teamcity \
         --clean \

diff --git a/regression-test/pipeline/common/doris-utils.sh b/regression-test/pipeline/common/doris-utils.sh
@@ -89,14 +89,25 @@ function start_doris_recycler() {
     fi
 }
 
-function start_doris_fe() {
-    if [[ ! -d "${DORIS_HOME:-}" ]]; then return 1; fi
+function install_java() {
     if ! java -version >/dev/null ||
         [[ -z "$(find /usr/lib/jvm -maxdepth 1 -type d -name 'java-8-*')" ]]; then
         sudo apt update && sudo apt install openjdk-8-jdk -y >/dev/null
     fi
-    JAVA_HOME="$(find /usr/lib/jvm -maxdepth 1 -type d -name 'java-8-*' | sed -n '1p')"
-    export JAVA_HOME
+    # doris master branch use java-17
+    if ! java -version >/dev/null ||
+        [[ -z "$(find /usr/lib/jvm -maxdepth 1 -type d -name 'java-17-*')" ]]; then
+        sudo apt update && sudo apt install openjdk-17-jdk -y >/dev/null
+    fi
+}
+
+function start_doris_fe() {
+    if [[ ! -d "${DORIS_HOME:-}" ]]; then return 1; fi
+    if install_java && [[ -z "${JAVA_HOME}" ]]; then
+        # default to use java-8
+        JAVA_HOME="$(find /usr/lib/jvm -maxdepth 1 -type d -name 'java-8-*' | sed -n '1p')"
+        export JAVA_HOME
+    fi
     # export JACOCO_COVERAGE_OPT="-javaagent:/usr/local/jacoco/lib/jacocoagent.jar=excludes=org.apache.doris.thrift:org.apache.doris.proto:org.apache.parquet.format:com.aliyun*:com.amazonaws*:org.apache.hadoop.hive.metastore:org.apache.parquet.format,output=file,append=true,destfile=${DORIS_HOME}/fe/fe_cov.exec"
     "${DORIS_HOME}"/fe/bin/start_fe.sh --daemon
 
@@ -117,12 +128,11 @@ function start_doris_fe() {
 
 function start_doris_be() {
     if [[ ! -d "${DORIS_HOME:-}" ]]; then return 1; fi
-    if ! java -version >/dev/null ||
-        [[ -z "$(find /usr/lib/jvm -maxdepth 1 -type d -name 'java-8-*')" ]]; then
-        sudo apt update && sudo apt install openjdk-8-jdk -y >/dev/null
+    if install_java && [[ -z "${JAVA_HOME}" ]]; then
+        # default to use java-8
+        JAVA_HOME="$(find /usr/lib/jvm -maxdepth 1 -type d -name 'java-8-*' | sed -n '1p')"
+        export JAVA_HOME
     fi
-    JAVA_HOME="$(find /usr/lib/jvm -maxdepth 1 -type d -name 'java-8-*' | sed -n '1p')"
-    export JAVA_HOME
     ASAN_SYMBOLIZER_PATH="$(command -v llvm-symbolizer)"
     if [[ -z "${ASAN_SYMBOLIZER_PATH}" ]]; then ASAN_SYMBOLIZER_PATH='/var/local/ldb-toolchain/bin/llvm-symbolizer'; fi
     export ASAN_SYMBOLIZER_PATH

diff --git a/regression-test/pipeline/performance/compile.sh b/regression-test/pipeline/performance/compile.sh
@@ -109,6 +109,10 @@ if sudo docker ps -a --no-trunc | grep "${docker_name}"; then
 fi
 rm -f custom_env.sh
 cp "${teamcity_build_checkoutDir}"/regression-test/pipeline/performance/conf/custom_env.sh .
+if [[ "${target_branch}" == "master" ]]; then
+    echo "export JAVA_HOME=/usr/lib/jvm/jdk-17.0.2" >>custom_env.sh
+    jdk17_str="&& cd /usr/lib/jvm/ && wget --continue -t3 https://download.java.net/java/GA/jdk17.0.2/dfd4a8d0985749f896bed50d7138ee7f/8/GPL/openjdk-17.0.2_linux-x64_bin.tar.gz && yum install -y pigz  && tar -I pigz -xf openjdk-17.0.2_linux-x64_bin.tar.gz && cd -"
+fi
 rm -rf "${teamcity_build_checkoutDir}"/output
 set -x
 # shellcheck disable=SC2086
@@ -131,7 +135,7 @@ sudo docker run -i --rm \
                     && export CCACHE_REMOTE_STORAGE=file:///root/ccache \
                     && export EXTRA_CXX_FLAGS=-O3 \
                     && export USE_JEMALLOC='ON' \
-					&& export ENABLE_PCH=OFF \
+					&& export ENABLE_PCH=OFF ${jdk17_str}\
                     && export CUSTOM_NPM_REGISTRY=https://registry.npmjs.org \
                     && bash build.sh --fe --be --clean 2>&1 | tee build.log"
 set +x

diff --git a/regression-test/pipeline/performance/prepare.sh b/regression-test/pipeline/performance/prepare.sh
@@ -28,6 +28,7 @@ set +x
 export teamcity_build_checkoutDir="%teamcity.build.checkoutDir%"
 export commit_id_from_checkout="%build.vcs.number%"
 export target_branch='%teamcity.pullRequest.target.branch%'
+export JAVA_HOME="/usr/lib/jvm/java-17-openjdk-amd64/"
 if [[ -f "${teamcity_build_checkoutDir:-}"/regression-test/pipeline/performance/prepare.sh ]]; then
     cd "${teamcity_build_checkoutDir}"/regression-test/pipeline/performance/
     bash prepare.sh
@@ -65,11 +66,21 @@ if [[ "${commit_id_from_trigger}" != "${commit_id_from_checkout}" ]]; then
     commit_id_from_trigger is outdate"
     exit 1
 fi
+
 # shellcheck source=/dev/null
 source "$(bash "${teamcity_build_checkoutDir}"/regression-test/pipeline/common/get-or-set-tmp-env.sh 'get')"
+# shellcheck source=/dev/null
+# install_java
+source "${teamcity_build_checkoutDir}"/regression-test/pipeline/common/doris-utils.sh
+
 if ${skip_pipeline:=false}; then echo "INFO: skip build pipline" && exit 0; else echo "INFO: no skip"; fi
-if [[ "${target_branch}" == "master" || "${target_branch}" == "branch-2.0" ]]; then
-    echo "INFO: PR target branch ${target_branch} is in (master, branch-2.0)"
+if [[ "${target_branch}" == "master" ]]; then
+    echo "INFO: PR target branch ${target_branch}"
+    install_java
+    JAVA_HOME="${JAVA_HOME:-$(find /usr/lib/jvm -maxdepth 1 -type d -name 'java-17-*' | sed -n '1p')}"
+    bash "${teamcity_build_checkoutDir}"/regression-test/pipeline/common/get-or-set-tmp-env.sh 'set' "export JAVA_HOME=\"${JAVA_HOME}\""
+elif [[ "${target_branch}" == "branch-2.0" ]]; then
+    echo "INFO: PR target branch ${target_branch}"
 else
     echo "WARNING: PR target branch ${target_branch} is NOT in (master, branch-2.0), skip pipeline."
     bash "${teamcity_build_checkoutDir}"/regression-test/pipeline/common/get-or-set-tmp-env.sh 'set' "export skip_pipeline=true"