[Tool] Sonarcloud FE in self runner && support MySQL in SQL-Tester (S…

…tarRocks#30225)

Signed-off-by: AndyZiYe <yeziyu@starrocks.com>

.github/workflows/ci-pipeline-branch.yml

@@ -387,3 +387,38 @@ jobs:
           rm -f ${{ steps.run_ut.outputs.RES_LOG }}
           rm -rf ${{ steps.run_ut.outputs.COV_DIR }}
           rm -rf ${{ github.workspace }}/*
+
+  Teardown:
+    runs-on: [self-hosted, normal]
+    name: Teardown
+    needs: [ fe-ut, be-ut ]
+    if: always()
+    env:
+      PR_NUMBER: ${{ github.event.number }}
+      BRANCH: ${{ github.base_ref }}
+      CONF_FILE: ${{ needs.deploy.outputs.deploy_conf_file }}
+    steps:
+      - name: upload info
+        if: always()
+        run: |
+          echo $PR_NUMBER > pr_num.txt
+          GITHUB_SHA=$(cat $GITHUB_EVENT_PATH | jq -r .pull_request.head.sha)
+          echo ${GITHUB_SHA} > head_sha.txt
+
+      - name: Upload the PR number
+        uses: actions/upload-artifact@v2
+        with:
+          name: pr_num
+          path: ./pr_num.txt
+          retention-days: 3
+
+      - name: Upload the PR HEAD REF
+        uses: actions/upload-artifact@v2
+        with:
+          name: head_sha
+          path: ./head_sha.txt
+          retention-days: 3
+
+      - name: Clean
+        run: |
+          rm -rf ${{ github.workspace }}/*

.github/workflows/ci-pipeline.yml

@@ -29,8 +29,6 @@ jobs:
       THIRDPARTY: ${{ steps.thirdparty-checker.outputs.thirdparty }}
     steps:
       - name: upload info
-        env:
-          PR_NUMBER: ${{ github.event.number }}
         run: |
           echo $PR_NUMBER > pr_num.txt
           GITHUB_SHA=$(cat $GITHUB_EVENT_PATH | jq -r .pull_request.head.sha)
@@ -302,7 +300,6 @@ jobs:
               - 'run-fe-ut.sh'
               - 'java-extensions/**'
               - 'build.sh'
-              - 'test/**'
 
       - run: echo ${{ steps.changes.outputs.fe }}
 
@@ -374,41 +371,28 @@ jobs:
           level: error
 
   sonarcloud-fe-checker:
-    runs-on: ubuntu-latest
+    runs-on: [self-hosted, normal]
     needs: fe-checker
     if: needs.fe-checker.outputs.output1 == 'true' && github.repository == 'StarRocks/starrocks'
     name: FE Sonarcloud Check
     env:
       PR_NUMBER: ${{ github.event.number }}
+      CODE_PATH: ${{ github.workspace }}
+      BRANCH: ${{ github.base_ref }}
+      GH_TOKEN: ${{ github.token }}
     steps:
+      - name: clean
+        run: |
+          rm -rf ${{ github.workspace }}
+          mkdir -p ${{ github.workspace }}
+
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0
 
-      - name: BRANCH INFO
-        id: branch
-        run: |
-          echo ${{github.base_ref}}
-          echo "branch=${{github.base_ref}}" >> $GITHUB_OUTPUT
-
       - name: Checkout PR
         run: |
-          BRANCH=${{steps.branch.outputs.branch}}
-          git config --global user.name "wanpengfei-git";
-          git config --global user.email "wanpengfei91@163.com";
-          git checkout $BRANCH;
-          git pull;
-          BRANCH_NAME="${BRANCH}-${PR_NUMBER}";
-          git fetch origin pull/${PR_NUMBER}/head:${BRANCH_NAME};
-          git checkout $BRANCH_NAME;
-          git checkout -b merge_pr;
-          git merge --squash --no-edit ${BRANCH} || (echo "Merge conflict, please check." && exit -1);
-
-      - name: Set up JDK 11
-        uses: actions/setup-java@v3
-        with:
-          java-version: 11
-          distribution: 'adopt'
+          rm -rf ./ci-tool && cp -rf /var/lib/ci-tool ./ci-tool && cd ci-tool && git pull && source lib/init.sh && ./bin/checkout.sh
 
       - name: Cache SonarCloud packages
         uses: actions/cache@v3
@@ -417,30 +401,22 @@ jobs:
           key: ${{ runner.os }}-sonar
           restore-keys: ${{ runner.os }}-sonar
 
-      - name: Cache Maven packages
-        uses: actions/cache@v3
-        with:
-          path: ~/.m2
-          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
-          restore-keys: ${{ runner.os }}-maven
-
-      - name: Setup thrift
-        uses: dodopizza/setup-thrift@v1
-        with:
-          version: 0.13.0
-
       - name: Analyze FE
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}  # Needed to get PR information, if any
           SONAR_TOKEN: f0fb4d25c03bae90c2e994c45c29c49dc86fc169 # ${{ secrets.SONAR_TOKEN }}
         run: |
-          thrift --version
-          whereis thrift
+          set -x
+          export JAVA_HOME=/var/local/env/jdk17.0.8;
+          export PATH=$JAVA_HOME/bin:/var/local/env/tools/maven/bin:$PATH;
+          export MAVEN_OPTS=-D"maven.repo.local"=/var/local/env/.m2/repository;
           export STARROCKS_HOME=${{ github.workspace }}
           source env.sh
           mkdir -p thirdparty/installed/bin/
-          cd thirdparty/installed/bin/ && ln -s /usr/local/bin/thrift thrift
+          cd thirdparty/installed/bin/ && ln -s /var/local/env/tools/thrift_0.13/thrift thrift
           cd ${{ github.workspace }}/fe
+          sed -i 's#<tasks>#<target>#g' fe-core/pom.xml
+          sed -i 's#</tasks>#</target>#g' fe-core/pom.xml
           mvn -B -DskipTests verify org.sonarsource.scanner.maven:sonar-maven-plugin:sonar -Dsonar.projectKey=StarRocks_starrocks -Dsonar.pullrequest.key=${{ github.event.number }} -Dsonar.pullrequest.base=${{ github.base_ref }} -Dsonar.pullrequest.branch=${{ github.head_ref }}
 
   fe-ut:
@@ -507,17 +483,43 @@ jobs:
           rm -rf ${{ steps.run_ut.outputs.COV_DIR }}
           rm -rf ${{ github.workspace }}/*
 
+  test-checker:
+    runs-on: ubuntu-latest
+    needs: sync-checker
+    name: TEST FILTER
+    if: needs.sync-checker.outputs.PASS == 'true'
+    outputs:
+      output1: ${{ steps.test-changes-info.outputs.test }}
+    steps:
+      - uses: dorny/paths-filter@v2
+        id: changes
+        with:
+          filters: |
+            test:
+              - 'test/**'
+
+      - name: FE CHECK INFO
+        id: test-changes-info
+        run: |
+          echo "test=${{ steps.changes.outputs.test }}" >> $GITHUB_OUTPUT
+
+
   build:
     runs-on: [self-hosted, normal]
-    needs: [be-checker, fe-checker, be-ut, clang-tidy, fe-ut, thirdparty-update]
+    needs: [be-checker, fe-checker, test-checker, be-ut, clang-tidy, fe-ut, thirdparty-update]
     name: BUILD
     env:
       PR_NUMBER: ${{ github.event.number }}
       BRANCH: ${{ github.base_ref }}
     outputs:
       build_output_tar: ${{ steps.run_build.outputs.OUTPUT_TAR }}
       base_version: ${{ steps.run_build.outputs.BASE_VERSION }}
-    if: always() && (needs.be-ut.result == 'success' && needs.clang-tidy.result == 'success' && needs.fe-ut.result == 'success') || (needs.be-checker.outputs.output1 != 'true' && needs.fe-ut.result == 'success') || (needs.fe-checker.outputs.output1 != 'true' && needs.be-ut.result == 'success' && needs.clang-tidy.result == 'success')
+    if: > 
+      always() && 
+      (needs.be-ut.result == 'success' && needs.clang-tidy.result == 'success' && needs.fe-ut.result == 'success') || 
+      (needs.be-checker.outputs.output1 != 'true' && needs.fe-ut.result == 'success') || 
+      (needs.fe-checker.outputs.output1 != 'true' && needs.be-ut.result == 'success' && needs.clang-tidy.result == 'success') ||
+      (needs.be-checker.outputs.output1 != 'true' && needs.fe-checker.outputs.output1 != 'true' && needs.test-checker.outputs.output1 == 'true')
     steps:
       - name: CLEAN
         run: |
@@ -626,6 +628,8 @@ jobs:
       FE_NODE: ${{ needs.deploy.outputs.fe }}
       BE_NODE: ${{ needs.deploy.outputs.be }}
       BASE_VERSION: ${{needs.build.outputs.base_version}}
+    outputs:
+      MYSQL_ECI_ID: ${{ steps.run_case.outputs.MYSQL_ECI_ID }}
     steps:
       - name: CLEAN
         run: |
@@ -658,6 +662,7 @@ jobs:
           git merge --squash --no-edit ${BRANCH} || (echo "Merge conflict, please check." && exit -1);
 
       - name: Run Case (${{ needs.deploy.outputs.fe }})
+        id: run_case
         run: |
           rm -rf ./ci-tool && cp -rf /var/lib/ci-tool ./ci-tool && cd ci-tool && git pull && source lib/init.sh
           ./bin/run-sql-tester.sh
@@ -758,9 +763,35 @@ jobs:
           if [[ "${SQL_TESTER_RESULT}" == 'success' && "${ADMIT_RESULT}" == 'success' ]]; then
               ./bin/elastic-cluster.sh --delete
           else
-              ./bin/elastic-cluster.sh --renew 0.25
+              ./bin/elastic-cluster.sh --renew 0.5
           fi
 
+      - name: clean ECI
+        if: always() && needs.SQL-Tester.outputs.MYSQL_ECI_ID != ''
+        run: |
+          eci rm ${{ needs.SQL-Tester.outputs.MYSQL_ECI_ID }}
+
+      - name: upload info
+        if: always()
+        run: |
+          echo $PR_NUMBER > pr_num.txt
+          GITHUB_SHA=$(cat $GITHUB_EVENT_PATH | jq -r .pull_request.head.sha)
+          echo ${GITHUB_SHA} > head_sha.txt
+
+      - name: Upload the PR number
+        uses: actions/upload-artifact@v2
+        with:
+          name: pr_num
+          path: ./pr_num.txt
+          retention-days: 3
+
+      - name: Upload the PR HEAD REF
+        uses: actions/upload-artifact@v2
+        with:
+          name: head_sha
+          path: ./head_sha.txt
+          retention-days: 3
+
       - name: Clean
         run: |
           rm -f $CONF_FILE

.github/workflows/ci-report.yml

@@ -84,6 +84,7 @@ jobs:
   FE-REPORT:
     runs-on: [ self-hosted, quick ]
     needs: INFO
+    if: needs.INFO.outputs.PR_NUMBER != ''
     env:
       PR_NUMBER: ${{ needs.INFO.outputs.PR_NUMBER }}
       branch: ${{ needs.INFO.outputs.BASE_REF }}
@@ -320,6 +321,7 @@ jobs:
   SQL-Tester-REPORT:
     runs-on: [ self-hosted, quick ]
     needs: INFO
+    if: needs.INFO.outputs.PR_NUMBER != ''
     env:
       PR_NUMBER: ${{ needs.INFO.outputs.PR_NUMBER }}
       branch: ${{ needs.INFO.outputs.BASE_REF }}
@@ -370,6 +372,7 @@ jobs:
   Admit-REPORT:
     runs-on: [ self-hosted, quick ]
     needs: INFO
+    if: needs.INFO.outputs.PR_NUMBER != ''
     env:
       PR_NUMBER: ${{ needs.INFO.outputs.PR_NUMBER }}
       branch: ${{ needs.INFO.outputs.BASE_REF }}

test/conf/sr.conf

@@ -24,3 +24,9 @@ hive_metastore_uris =
 hudi_hive_metastore_uris = 
 
 iceberg_catalog_hive_metastore_uris =
+
+external_mysql_ip =
+external_mysql_port =
+external_mysql_user =
+external_mysql_password =
+jdbc_url =