diff --git a/.github/workflows/ci-pipeline-branch.yml b/.github/workflows/ci-pipeline-branch.yml index 64f2271d42872..4effbeb1735da 100644 --- a/.github/workflows/ci-pipeline-branch.yml +++ b/.github/workflows/ci-pipeline-branch.yml @@ -387,3 +387,38 @@ jobs: rm -f ${{ steps.run_ut.outputs.RES_LOG }} rm -rf ${{ steps.run_ut.outputs.COV_DIR }} rm -rf ${{ github.workspace }}/* + + Teardown: + runs-on: [self-hosted, normal] + name: Teardown + needs: [ fe-ut, be-ut ] + if: always() + env: + PR_NUMBER: ${{ github.event.number }} + BRANCH: ${{ github.base_ref }} + CONF_FILE: ${{ needs.deploy.outputs.deploy_conf_file }} + steps: + - name: upload info + if: always() + run: | + echo $PR_NUMBER > pr_num.txt + GITHUB_SHA=$(cat $GITHUB_EVENT_PATH | jq -r .pull_request.head.sha) + echo ${GITHUB_SHA} > head_sha.txt + + - name: Upload the PR number + uses: actions/upload-artifact@v2 + with: + name: pr_num + path: ./pr_num.txt + retention-days: 3 + + - name: Upload the PR HEAD REF + uses: actions/upload-artifact@v2 + with: + name: head_sha + path: ./head_sha.txt + retention-days: 3 + + - name: Clean + run: | + rm -rf ${{ github.workspace }}/* diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index a92baa46eb68d..6d70af890914b 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -29,8 +29,6 @@ jobs: THIRDPARTY: ${{ steps.thirdparty-checker.outputs.thirdparty }} steps: - name: upload info - env: - PR_NUMBER: ${{ github.event.number }} run: | echo $PR_NUMBER > pr_num.txt GITHUB_SHA=$(cat $GITHUB_EVENT_PATH | jq -r .pull_request.head.sha) @@ -302,7 +300,6 @@ jobs: - 'run-fe-ut.sh' - 'java-extensions/**' - 'build.sh' - - 'test/**' - run: echo ${{ steps.changes.outputs.fe }} @@ -374,41 +371,28 @@ jobs: level: error sonarcloud-fe-checker: - runs-on: ubuntu-latest + runs-on: [self-hosted, normal] needs: fe-checker if: needs.fe-checker.outputs.output1 == 'true' && github.repository == 'StarRocks/starrocks' name: FE Sonarcloud Check env: PR_NUMBER: ${{ github.event.number }} + CODE_PATH: ${{ github.workspace }} + BRANCH: ${{ github.base_ref }} + GH_TOKEN: ${{ github.token }} steps: + - name: clean + run: | + rm -rf ${{ github.workspace }} + mkdir -p ${{ github.workspace }} + - uses: actions/checkout@v3 with: fetch-depth: 0 - - name: BRANCH INFO - id: branch - run: | - echo ${{github.base_ref}} - echo "branch=${{github.base_ref}}" >> $GITHUB_OUTPUT - - name: Checkout PR run: | - BRANCH=${{steps.branch.outputs.branch}} - git config --global user.name "wanpengfei-git"; - git config --global user.email "wanpengfei91@163.com"; - git checkout $BRANCH; - git pull; - BRANCH_NAME="${BRANCH}-${PR_NUMBER}"; - git fetch origin pull/${PR_NUMBER}/head:${BRANCH_NAME}; - git checkout $BRANCH_NAME; - git checkout -b merge_pr; - git merge --squash --no-edit ${BRANCH} || (echo "Merge conflict, please check." && exit -1); - - - name: Set up JDK 11 - uses: actions/setup-java@v3 - with: - java-version: 11 - distribution: 'adopt' + rm -rf ./ci-tool && cp -rf /var/lib/ci-tool ./ci-tool && cd ci-tool && git pull && source lib/init.sh && ./bin/checkout.sh - name: Cache SonarCloud packages uses: actions/cache@v3 @@ -417,30 +401,22 @@ jobs: key: ${{ runner.os }}-sonar restore-keys: ${{ runner.os }}-sonar - - name: Cache Maven packages - uses: actions/cache@v3 - with: - path: ~/.m2 - key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-maven - - - name: Setup thrift - uses: dodopizza/setup-thrift@v1 - with: - version: 0.13.0 - - name: Analyze FE env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any SONAR_TOKEN: f0fb4d25c03bae90c2e994c45c29c49dc86fc169 # ${{ secrets.SONAR_TOKEN }} run: | - thrift --version - whereis thrift + set -x + export JAVA_HOME=/var/local/env/jdk17.0.8; + export PATH=$JAVA_HOME/bin:/var/local/env/tools/maven/bin:$PATH; + export MAVEN_OPTS=-D"maven.repo.local"=/var/local/env/.m2/repository; export STARROCKS_HOME=${{ github.workspace }} source env.sh mkdir -p thirdparty/installed/bin/ - cd thirdparty/installed/bin/ && ln -s /usr/local/bin/thrift thrift + cd thirdparty/installed/bin/ && ln -s /var/local/env/tools/thrift_0.13/thrift thrift cd ${{ github.workspace }}/fe + sed -i 's###g' fe-core/pom.xml + sed -i 's###g' fe-core/pom.xml mvn -B -DskipTests verify org.sonarsource.scanner.maven:sonar-maven-plugin:sonar -Dsonar.projectKey=StarRocks_starrocks -Dsonar.pullrequest.key=${{ github.event.number }} -Dsonar.pullrequest.base=${{ github.base_ref }} -Dsonar.pullrequest.branch=${{ github.head_ref }} fe-ut: @@ -507,9 +483,30 @@ jobs: rm -rf ${{ steps.run_ut.outputs.COV_DIR }} rm -rf ${{ github.workspace }}/* + test-checker: + runs-on: ubuntu-latest + needs: sync-checker + name: TEST FILTER + if: needs.sync-checker.outputs.PASS == 'true' + outputs: + output1: ${{ steps.test-changes-info.outputs.test }} + steps: + - uses: dorny/paths-filter@v2 + id: changes + with: + filters: | + test: + - 'test/**' + + - name: FE CHECK INFO + id: test-changes-info + run: | + echo "test=${{ steps.changes.outputs.test }}" >> $GITHUB_OUTPUT + + build: runs-on: [self-hosted, normal] - needs: [be-checker, fe-checker, be-ut, clang-tidy, fe-ut, thirdparty-update] + needs: [be-checker, fe-checker, test-checker, be-ut, clang-tidy, fe-ut, thirdparty-update] name: BUILD env: PR_NUMBER: ${{ github.event.number }} @@ -517,7 +514,12 @@ jobs: outputs: build_output_tar: ${{ steps.run_build.outputs.OUTPUT_TAR }} base_version: ${{ steps.run_build.outputs.BASE_VERSION }} - if: always() && (needs.be-ut.result == 'success' && needs.clang-tidy.result == 'success' && needs.fe-ut.result == 'success') || (needs.be-checker.outputs.output1 != 'true' && needs.fe-ut.result == 'success') || (needs.fe-checker.outputs.output1 != 'true' && needs.be-ut.result == 'success' && needs.clang-tidy.result == 'success') + if: > + always() && + (needs.be-ut.result == 'success' && needs.clang-tidy.result == 'success' && needs.fe-ut.result == 'success') || + (needs.be-checker.outputs.output1 != 'true' && needs.fe-ut.result == 'success') || + (needs.fe-checker.outputs.output1 != 'true' && needs.be-ut.result == 'success' && needs.clang-tidy.result == 'success') || + (needs.be-checker.outputs.output1 != 'true' && needs.fe-checker.outputs.output1 != 'true' && needs.test-checker.outputs.output1 == 'true') steps: - name: CLEAN run: | @@ -626,6 +628,8 @@ jobs: FE_NODE: ${{ needs.deploy.outputs.fe }} BE_NODE: ${{ needs.deploy.outputs.be }} BASE_VERSION: ${{needs.build.outputs.base_version}} + outputs: + MYSQL_ECI_ID: ${{ steps.run_case.outputs.MYSQL_ECI_ID }} steps: - name: CLEAN run: | @@ -658,6 +662,7 @@ jobs: git merge --squash --no-edit ${BRANCH} || (echo "Merge conflict, please check." && exit -1); - name: Run Case (${{ needs.deploy.outputs.fe }}) + id: run_case run: | rm -rf ./ci-tool && cp -rf /var/lib/ci-tool ./ci-tool && cd ci-tool && git pull && source lib/init.sh ./bin/run-sql-tester.sh @@ -758,9 +763,35 @@ jobs: if [[ "${SQL_TESTER_RESULT}" == 'success' && "${ADMIT_RESULT}" == 'success' ]]; then ./bin/elastic-cluster.sh --delete else - ./bin/elastic-cluster.sh --renew 0.25 + ./bin/elastic-cluster.sh --renew 0.5 fi + - name: clean ECI + if: always() && needs.SQL-Tester.outputs.MYSQL_ECI_ID != '' + run: | + eci rm ${{ needs.SQL-Tester.outputs.MYSQL_ECI_ID }} + + - name: upload info + if: always() + run: | + echo $PR_NUMBER > pr_num.txt + GITHUB_SHA=$(cat $GITHUB_EVENT_PATH | jq -r .pull_request.head.sha) + echo ${GITHUB_SHA} > head_sha.txt + + - name: Upload the PR number + uses: actions/upload-artifact@v2 + with: + name: pr_num + path: ./pr_num.txt + retention-days: 3 + + - name: Upload the PR HEAD REF + uses: actions/upload-artifact@v2 + with: + name: head_sha + path: ./head_sha.txt + retention-days: 3 + - name: Clean run: | rm -f $CONF_FILE diff --git a/.github/workflows/ci-report.yml b/.github/workflows/ci-report.yml index 923d9e3751830..04f355cbe80d1 100644 --- a/.github/workflows/ci-report.yml +++ b/.github/workflows/ci-report.yml @@ -84,6 +84,7 @@ jobs: FE-REPORT: runs-on: [ self-hosted, quick ] needs: INFO + if: needs.INFO.outputs.PR_NUMBER != '' env: PR_NUMBER: ${{ needs.INFO.outputs.PR_NUMBER }} branch: ${{ needs.INFO.outputs.BASE_REF }} @@ -320,6 +321,7 @@ jobs: SQL-Tester-REPORT: runs-on: [ self-hosted, quick ] needs: INFO + if: needs.INFO.outputs.PR_NUMBER != '' env: PR_NUMBER: ${{ needs.INFO.outputs.PR_NUMBER }} branch: ${{ needs.INFO.outputs.BASE_REF }} @@ -370,6 +372,7 @@ jobs: Admit-REPORT: runs-on: [ self-hosted, quick ] needs: INFO + if: needs.INFO.outputs.PR_NUMBER != '' env: PR_NUMBER: ${{ needs.INFO.outputs.PR_NUMBER }} branch: ${{ needs.INFO.outputs.BASE_REF }} diff --git a/test/conf/sr.conf b/test/conf/sr.conf index d3d1a0f14a629..9534866fd61fb 100644 --- a/test/conf/sr.conf +++ b/test/conf/sr.conf @@ -24,3 +24,9 @@ hive_metastore_uris = hudi_hive_metastore_uris = iceberg_catalog_hive_metastore_uris = + +external_mysql_ip = +external_mysql_port = +external_mysql_user = +external_mysql_password = +jdbc_url =