[#4811] improvement(CI): optimize CI jobs trigger conditions (#4827)

### What changes were proposed in this pull request?

 - optimize CI jobs trigger conditions to remove unnecessary jobs
 - reduce job number of FlinkIT and SparkIT on PR
 - fix skipUTs does not work

### Why are the changes needed?

Fix: #4811 

### Does this PR introduce _any_ user-facing change?

no

### How was this patch tested?

CI passed

.github/workflows/access-control-integration-test.yml

@@ -22,21 +22,32 @@ jobs:
         with:
           filters: |
             source_changes:
+              - .github/**
               - api/**
               - authorizations/**
+              - bin/**
               - catalogs/**
               - clients/client-java/**
               - clients/client-java-runtime/**
               - common/**
+              - conf/**
               - core/**
+              - dev/**
+              - gradle/**
+              - iceberg/**
               - integration-test-common/**
+              - meta/**
+              - scripts/**
               - server/**
               - server-common/**
+              - build.gradle.kts
+              - gradle.properties
+              - gradlew
+              - setting.gradle.kts
     outputs:
       source_changes: ${{ steps.filter.outputs.source_changes }}
 
-  # Integration test for AMD64 architecture
-  test-amd64-arch:
+  access-control-IT:
     needs: changes
     if: needs.changes.outputs.source_changes == 'true'
     runs-on: ubuntu-latest
@@ -46,12 +57,6 @@ jobs:
         # Integration test for AMD64 architecture
         architecture: [linux/amd64]
         java-version: [ 17 ]
-        test-mode: [ embedded, deploy ]
-        include:
-          - test-mode: 'embedded'
-            backend: 'h2'
-          - test-mode: 'deploy'
-            backend: 'mysql'
 
     env:
       PLATFORM: ${{ matrix.architecture }}
@@ -72,18 +77,19 @@ jobs:
           dev/ci/check_commands.sh
 
       - name: Package Gravitino
-        if : ${{ matrix.test-mode == 'deploy' }}
         run: |
           ./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }}
 
       - name: Free up disk space
         run: |
           dev/ci/util_free_space.sh
 
-      - name: Authorization Integration Test (JDK${{ matrix.java-version }}-${{ matrix.test-mode }}-${{ matrix.backend }})
+      - name: Authorization Integration Test (JDK${{ matrix.java-version }})
         id: integrationTest
         run: |
-          ./gradlew -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdbcBackend=${{ matrix.backend }} -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :authorizations:authorization-ranger:test --tests "org.apache.gravitino.authorization.ranger.integration.test.**"
+          ./gradlew -PskipTests -PtestMode=embedded -PjdbcBackend=h2 -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :authorizations:authorization-ranger:test --tests "org.apache.gravitino.authorization.ranger.integration.test.**"
+          ./gradlew -PskipTests -PtestMode=deploy -PjdbcBackend=mysql -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :authorizations:authorization-ranger:test --tests "org.apache.gravitino.authorization.ranger.integration.test.**"
+          ./gradlew -PskipTests -PtestMode=deploy -PjdbcBackend=postgresql -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :authorizations:authorization-ranger:test --tests "org.apache.gravitino.authorization.ranger.integration.test.**"
 
       - name: Upload integrate tests reports
         uses: actions/upload-artifact@v3

.github/workflows/backend-integration-test-action.yml

@@ -0,0 +1,77 @@
+name: Backend Integration Test Action
+
+# run backend integration test
+on:
+  workflow_call:
+    inputs:
+      architecture:
+        required: true
+        description: 'Architecture of the platform'
+        type: string
+      java-version:
+        required: true
+        description: 'Java version'
+        type: string
+      backend:
+        required: true
+        description: 'Backend storage for Gravitino'
+        type: string
+      test-mode:
+        required: true
+        description: 'run on embedded or deploy mode'
+        type: string
+
+jobs:
+  start-runner:
+    name: JDK${{ inputs.java-version }}-${{ inputs.test-mode }}-${{ inputs.backend }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 90
+    env:
+      PLATFORM: ${{ inputs.architecture }}
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-java@v4
+        with:
+          java-version: ${{ inputs.java-version }}
+          distribution: 'temurin'
+          cache: 'gradle'
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Check required command
+        run: |
+          dev/ci/check_commands.sh
+
+      - name: Package Gravitino
+        if: ${{ inputs.test-mode == 'deploy' }}
+        run: |
+          ./gradlew compileDistribution -x test -PjdkVersion=${{ inputs.java-version }}
+
+      - name: Free up disk space
+        run: |
+          dev/ci/util_free_space.sh
+
+      - name: Backend Integration Test (JDK${{ inputs.java-version }}-${{ inputs.test-mode }}-${{ inputs.backend }})
+        id: integrationTest
+        run: >
+          ./gradlew test -PskipTests -PtestMode=${{ inputs.test-mode }} -PjdkVersion=${{ inputs.java-version }} -PjdbcBackend=${{ inputs.backend }} -PskipDockerTests=false
+          -x :web:web:test -x :web:integration-test:test -x :clients:client-python:test -x :flink-connector:flink:test -x :spark-connector:spark-common:test 
+          -x :spark-connector:spark-3.3:test -x :spark-connector:spark-3.4:test -x :spark-connector:spark-3.5:test 
+          -x :spark-connector:spark-runtime-3.3:test -x :spark-connector:spark-runtime-3.4:test -x :spark-connector:spark-runtime-3.5:test
+          -x :authorizations:authorization-ranger:test -x :trino-connector:integration-test:test -x :trino-connector:trino-connector:test
+
+      - name: Upload integrate tests reports
+        uses: actions/upload-artifact@v3
+        if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }}
+        with:
+          name: integrate-test-reports-${{ inputs.java-version }}-${{ inputs.test-mode }}-${{ inputs.backend }}
+          path: |
+            build/reports
+            iceberg/iceberg-rest-server/build/*.log
+            distribution/package/logs/*.out
+            distribution/package/logs/*.log
+            catalogs/**/*.log
+            catalogs/**/*.tar
+            distribution/**/*.log

.github/workflows/backend-integration-test.yml

@@ -2,7 +2,6 @@ name: Backend Integration Test
 
 # Controls when the workflow will run
 on:
-  # Triggers the workflow on push or pull request events but only for the "main" branch
   push:
     branches: [ "main", "branch-*" ]
   pull_request:
@@ -29,36 +28,29 @@ jobs:
               - clients/client-java/**
               - clients/client-java-runtime/**
               - clients/filesystem-hadoop3/**
-              - clients/filesystem-hadoop3-runtime/**
               - common/**
               - conf/**
               - core/**
               - dev/**
               - gradle/**
-              - integration-test/**
-              - integration-test-common/**
               - iceberg/**
+              - integration-test-common/**
               - meta/**
+              - scripts/**
               - server/**
               - server-common/**
-              - trino-connector/**
-              - web/**
-              - docs/open-api/**
               - build.gradle.kts
               - gradle.properties
               - gradlew
               - setting.gradle.kts
     outputs:
       source_changes: ${{ steps.filter.outputs.source_changes }}
 
-  test-on-push:
+  BackendIT-on-push:
     needs: changes
     if: (github.event_name == 'push' && needs.changes.outputs.source_changes == 'true')
-    runs-on: ubuntu-latest
-    timeout-minutes: 90
     strategy:
       matrix:
-        # Integration test for AMD64 architecture
         architecture: [linux/amd64]
         java-version: [ 8, 11, 17 ]
         backend: [ h2, mysql, postgresql ]
@@ -70,126 +62,32 @@ jobs:
             backend: 'postgresql'
           - test-mode: 'deploy'
             backend: 'h2'
-
-    env:
-      PLATFORM: ${{ matrix.architecture }}
-    steps:
-      - uses: actions/checkout@v3
-
-      - uses: actions/setup-java@v4
-        with:
-          java-version: ${{ matrix.java-version }}
-          distribution: 'temurin'
-          cache: 'gradle'
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
-
-      - name: Check required command
-        run: |
-          dev/ci/check_commands.sh
-
-      - name: Package Gravitino
-        if : ${{ matrix.test-mode == 'deploy' }}
-        run: |
-          ./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }}
-
-      - name: Free up disk space
-        run: |
-          dev/ci/util_free_space.sh
-
-      - name: Backend Integration Test (JDK${{ matrix.java-version }}-${{ matrix.test-mode }}-${{ matrix.backend }})
-        id: integrationTest
-        run: >
-          ./gradlew test -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PjdbcBackend=${{ matrix.backend }} -PskipWebITs -PskipDockerTests=false
-          -x :web:test -x :clients:client-python:test -x :flink-connector:flink:test -x :spark-connector:test -x :spark-connector:spark-common:test 
-          -x :spark-connector:spark-3.3:test -x :spark-connector:spark-3.4:test -x :spark-connector:spark-3.5:test 
-          -x :spark-connector:spark-runtime-3.3:test -x :spark-connector:spark-runtime-3.4:test -x :spark-connector:spark-runtime-3.5:test
-          -x :authorizations:authorization-ranger:test -x :web:integration-test:test -x :trino-connector:integration-test:test -x :trino-connector:trino-connector:test
-
-      - name: Upload integrate tests reports
-        uses: actions/upload-artifact@v3
-        if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }}
-        with:
-          name: integrate-test-reports-${{ matrix.java-version }}-${{ matrix.test-mode }}-${{ matrix.backend }}
-          path: |
-            build/reports
-            iceberg/iceberg-rest-server/build/*.log
-            integration-test/build/*.log
-            integration-test/build/*.tar
-            integration-test/build/trino-ci-container-log
-            distribution/package/logs/*.out
-            distribution/package/logs/*.log
-            catalogs/**/*.log
-            catalogs/**/*.tar
-            distribution/**/*.log
-
-  test-on-pr:
+    uses: ./.github/workflows/backend-integration-test-action.yml
+    with:
+      architecture: ${{ matrix.architecture }}
+      java-version: ${{ matrix.java-version }}
+      backend: ${{ matrix.backend }}
+      test-mode: ${{ matrix.test-mode }}
+
+  BackendIT-on-pr:
     needs: changes
     if: (github.event_name == 'pull_request' && needs.changes.outputs.source_changes == 'true')
-    runs-on: ubuntu-latest
-    timeout-minutes: 90
     strategy:
       matrix:
-        # Integration test for AMD64 architecture
         architecture: [ linux/amd64 ]
         java-version: [ 17 ]
-        test-mode: [ embedded, deploy ]
         backend: [ h2, mysql, postgresql ]
+        test-mode: [ embedded, deploy ]
         exclude:
           - test-mode: 'embedded'
             backend: 'mysql'
           - test-mode: 'embedded'
             backend: 'postgresql'
           - test-mode: 'deploy'
             backend: 'h2'
-
-    env:
-      PLATFORM: ${{ matrix.architecture }}
-    steps:
-      - uses: actions/checkout@v3
-
-      - uses: actions/setup-java@v4
-        with:
-          java-version: ${{ matrix.java-version }}
-          distribution: 'temurin'
-          cache: 'gradle'
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
-
-      - name: Check required command
-        run: |
-          dev/ci/check_commands.sh
-
-      - name: Package Gravitino
-        if: ${{ matrix.test-mode == 'deploy' }}
-        run: |
-          ./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }}
-
-      - name: Free up disk space
-        run: |
-          dev/ci/util_free_space.sh
-
-      - name: Backend Integration Test (JDK${{ matrix.java-version }}-${{ matrix.test-mode }}-${{ matrix.backend }})
-        id: integrationTest
-        run: >
-          ./gradlew test -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PjdbcBackend=${{ matrix.backend }} -PskipWebITs -PskipDockerTests=false
-          -x :web:test -x :clients:client-python:test -x :flink-connector:flink:test -x :spark-connector:test -x :spark-connector:spark-common:test 
-          -x :spark-connector:spark-3.3:test -x :spark-connector:spark-3.4:test -x :spark-connector:spark-3.5:test 
-          -x :spark-connector:spark-runtime-3.3:test -x :spark-connector:spark-runtime-3.4:test -x :spark-connector:spark-runtime-3.5:test
-          -x :authorizations:authorization-ranger:test -x :web:integration-test:test -x :trino-connector:integration-test:test -x :trino-connector:test
-
-      - name: Upload integrate tests reports
-        uses: actions/upload-artifact@v3
-        if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }}
-        with:
-          name: integrate-test-reports-${{ matrix.java-version }}-${{ matrix.test-mode }}-${{ matrix.backend }}
-          path: |
-            build/reports
-            iceberg/iceberg-rest-server/build/*.log
-            distribution/package/logs/*.out
-            distribution/package/logs/*.log
-            catalogs/**/*.log
-            catalogs/**/*.tar
-            distribution/**/*.log
+    uses: ./.github/workflows/backend-integration-test-action.yml
+    with:
+      architecture: ${{ matrix.architecture }}
+      java-version: ${{ matrix.java-version }}
+      backend: ${{ matrix.backend }}
+      test-mode: ${{ matrix.test-mode }}

.github/workflows/build.yml

@@ -25,6 +25,7 @@ jobs:
             source_changes:
               - .github/**
               - api/**
+              - authorizations/**
               - bin/**
               - catalogs/**
               - clients/client-java/**
@@ -35,23 +36,28 @@ jobs:
               - conf/**
               - core/**
               - dev/**
+              - docs/open-api/**
+              - docs/build.gradle.kts
+              - flink-connector/**
               - gradle/**
-              - integration-test/**
+              - iceberg/**
+              - integration-test-common/**
               - meta/**
+              - scripts/**
               - server/**
               - server-common/**
               - spark-connector/**
-              - flink-connector/**
               - trino-connector/**
               - web/**
-              - docs/open-api/**
-              - docs/build.gradle.kts
               - build.gradle.kts
               - gradle.properties
               - gradlew
               - setting.gradle.kts
+            spark_connector_changes:
+              - spark-connector/**
     outputs:
       source_changes: ${{ steps.filter.outputs.source_changes }}
+      spark_connector_changes: ${{ steps.filter.outputs.spark_connector_changes }}
 
   compile-check:
     runs-on: ubuntu-latest
@@ -74,7 +80,7 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 30
     needs: changes
-    if: needs.changes.outputs.source_changes == 'true'
+    if: needs.changes.outputs.spark_connector_changes == 'true'
     steps:
       - uses: actions/checkout@v3