diff --git a/.github/workflows/access-control-integration-test.yml b/.github/workflows/access-control-integration-test.yml index 9f7d5f2bdc4..e9c55728ab8 100644 --- a/.github/workflows/access-control-integration-test.yml +++ b/.github/workflows/access-control-integration-test.yml @@ -22,21 +22,32 @@ jobs: with: filters: | source_changes: + - .github/** - api/** - authorizations/** + - bin/** - catalogs/** - clients/client-java/** - clients/client-java-runtime/** - common/** + - conf/** - core/** + - dev/** + - gradle/** + - iceberg/** - integration-test-common/** + - meta/** + - scripts/** - server/** - server-common/** + - build.gradle.kts + - gradle.properties + - gradlew + - setting.gradle.kts outputs: source_changes: ${{ steps.filter.outputs.source_changes }} - # Integration test for AMD64 architecture - test-amd64-arch: + access-control-IT: needs: changes if: needs.changes.outputs.source_changes == 'true' runs-on: ubuntu-latest @@ -46,12 +57,6 @@ jobs: # Integration test for AMD64 architecture architecture: [linux/amd64] java-version: [ 17 ] - test-mode: [ embedded, deploy ] - include: - - test-mode: 'embedded' - backend: 'h2' - - test-mode: 'deploy' - backend: 'mysql' env: PLATFORM: ${{ matrix.architecture }} @@ -72,7 +77,6 @@ jobs: dev/ci/check_commands.sh - name: Package Gravitino - if : ${{ matrix.test-mode == 'deploy' }} run: | ./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }} @@ -80,10 +84,12 @@ jobs: run: | dev/ci/util_free_space.sh - - name: Authorization Integration Test (JDK${{ matrix.java-version }}-${{ matrix.test-mode }}-${{ matrix.backend }}) + - name: Authorization Integration Test (JDK${{ matrix.java-version }}) id: integrationTest run: | - ./gradlew -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdbcBackend=${{ matrix.backend }} -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :authorizations:authorization-ranger:test --tests "org.apache.gravitino.authorization.ranger.integration.test.**" + ./gradlew -PskipTests -PtestMode=embedded -PjdbcBackend=h2 -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :authorizations:authorization-ranger:test --tests "org.apache.gravitino.authorization.ranger.integration.test.**" + ./gradlew -PskipTests -PtestMode=deploy -PjdbcBackend=mysql -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :authorizations:authorization-ranger:test --tests "org.apache.gravitino.authorization.ranger.integration.test.**" + ./gradlew -PskipTests -PtestMode=deploy -PjdbcBackend=postgresql -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :authorizations:authorization-ranger:test --tests "org.apache.gravitino.authorization.ranger.integration.test.**" - name: Upload integrate tests reports uses: actions/upload-artifact@v3 diff --git a/.github/workflows/backend-integration-test-action.yml b/.github/workflows/backend-integration-test-action.yml new file mode 100644 index 00000000000..69cfc3164cd --- /dev/null +++ b/.github/workflows/backend-integration-test-action.yml @@ -0,0 +1,77 @@ +name: Backend Integration Test Action + +# run backend integration test +on: + workflow_call: + inputs: + architecture: + required: true + description: 'Architecture of the platform' + type: string + java-version: + required: true + description: 'Java version' + type: string + backend: + required: true + description: 'Backend storage for Gravitino' + type: string + test-mode: + required: true + description: 'run on embedded or deploy mode' + type: string + +jobs: + start-runner: + name: JDK${{ inputs.java-version }}-${{ inputs.test-mode }}-${{ inputs.backend }} + runs-on: ubuntu-latest + timeout-minutes: 90 + env: + PLATFORM: ${{ inputs.architecture }} + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-java@v4 + with: + java-version: ${{ inputs.java-version }} + distribution: 'temurin' + cache: 'gradle' + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - name: Check required command + run: | + dev/ci/check_commands.sh + + - name: Package Gravitino + if: ${{ inputs.test-mode == 'deploy' }} + run: | + ./gradlew compileDistribution -x test -PjdkVersion=${{ inputs.java-version }} + + - name: Free up disk space + run: | + dev/ci/util_free_space.sh + + - name: Backend Integration Test (JDK${{ inputs.java-version }}-${{ inputs.test-mode }}-${{ inputs.backend }}) + id: integrationTest + run: > + ./gradlew test -PskipTests -PtestMode=${{ inputs.test-mode }} -PjdkVersion=${{ inputs.java-version }} -PjdbcBackend=${{ inputs.backend }} -PskipDockerTests=false + -x :web:web:test -x :web:integration-test:test -x :clients:client-python:test -x :flink-connector:flink:test -x :spark-connector:spark-common:test + -x :spark-connector:spark-3.3:test -x :spark-connector:spark-3.4:test -x :spark-connector:spark-3.5:test + -x :spark-connector:spark-runtime-3.3:test -x :spark-connector:spark-runtime-3.4:test -x :spark-connector:spark-runtime-3.5:test + -x :authorizations:authorization-ranger:test -x :trino-connector:integration-test:test -x :trino-connector:trino-connector:test + + - name: Upload integrate tests reports + uses: actions/upload-artifact@v3 + if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }} + with: + name: integrate-test-reports-${{ inputs.java-version }}-${{ inputs.test-mode }}-${{ inputs.backend }} + path: | + build/reports + iceberg/iceberg-rest-server/build/*.log + distribution/package/logs/*.out + distribution/package/logs/*.log + catalogs/**/*.log + catalogs/**/*.tar + distribution/**/*.log \ No newline at end of file diff --git a/.github/workflows/backend-integration-test.yml b/.github/workflows/backend-integration-test.yml index 3aa5ab7d027..b00965fbad9 100644 --- a/.github/workflows/backend-integration-test.yml +++ b/.github/workflows/backend-integration-test.yml @@ -2,7 +2,6 @@ name: Backend Integration Test # Controls when the workflow will run on: - # Triggers the workflow on push or pull request events but only for the "main" branch push: branches: [ "main", "branch-*" ] pull_request: @@ -29,21 +28,17 @@ jobs: - clients/client-java/** - clients/client-java-runtime/** - clients/filesystem-hadoop3/** - - clients/filesystem-hadoop3-runtime/** - common/** - conf/** - core/** - dev/** - gradle/** - - integration-test/** - - integration-test-common/** - iceberg/** + - integration-test-common/** - meta/** + - scripts/** - server/** - server-common/** - - trino-connector/** - - web/** - - docs/open-api/** - build.gradle.kts - gradle.properties - gradlew @@ -51,14 +46,11 @@ jobs: outputs: source_changes: ${{ steps.filter.outputs.source_changes }} - test-on-push: + BackendIT-on-push: needs: changes if: (github.event_name == 'push' && needs.changes.outputs.source_changes == 'true') - runs-on: ubuntu-latest - timeout-minutes: 90 strategy: matrix: - # Integration test for AMD64 architecture architecture: [linux/amd64] java-version: [ 8, 11, 17 ] backend: [ h2, mysql, postgresql ] @@ -70,72 +62,22 @@ jobs: backend: 'postgresql' - test-mode: 'deploy' backend: 'h2' - - env: - PLATFORM: ${{ matrix.architecture }} - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-java@v4 - with: - java-version: ${{ matrix.java-version }} - distribution: 'temurin' - cache: 'gradle' - - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - - name: Check required command - run: | - dev/ci/check_commands.sh - - - name: Package Gravitino - if : ${{ matrix.test-mode == 'deploy' }} - run: | - ./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }} - - - name: Free up disk space - run: | - dev/ci/util_free_space.sh - - - name: Backend Integration Test (JDK${{ matrix.java-version }}-${{ matrix.test-mode }}-${{ matrix.backend }}) - id: integrationTest - run: > - ./gradlew test -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PjdbcBackend=${{ matrix.backend }} -PskipWebITs -PskipDockerTests=false - -x :web:test -x :clients:client-python:test -x :flink-connector:flink:test -x :spark-connector:test -x :spark-connector:spark-common:test - -x :spark-connector:spark-3.3:test -x :spark-connector:spark-3.4:test -x :spark-connector:spark-3.5:test - -x :spark-connector:spark-runtime-3.3:test -x :spark-connector:spark-runtime-3.4:test -x :spark-connector:spark-runtime-3.5:test - -x :authorizations:authorization-ranger:test -x :web:integration-test:test -x :trino-connector:integration-test:test -x :trino-connector:trino-connector:test - - - name: Upload integrate tests reports - uses: actions/upload-artifact@v3 - if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }} - with: - name: integrate-test-reports-${{ matrix.java-version }}-${{ matrix.test-mode }}-${{ matrix.backend }} - path: | - build/reports - iceberg/iceberg-rest-server/build/*.log - integration-test/build/*.log - integration-test/build/*.tar - integration-test/build/trino-ci-container-log - distribution/package/logs/*.out - distribution/package/logs/*.log - catalogs/**/*.log - catalogs/**/*.tar - distribution/**/*.log - - test-on-pr: + uses: ./.github/workflows/backend-integration-test-action.yml + with: + architecture: ${{ matrix.architecture }} + java-version: ${{ matrix.java-version }} + backend: ${{ matrix.backend }} + test-mode: ${{ matrix.test-mode }} + + BackendIT-on-pr: needs: changes if: (github.event_name == 'pull_request' && needs.changes.outputs.source_changes == 'true') - runs-on: ubuntu-latest - timeout-minutes: 90 strategy: matrix: - # Integration test for AMD64 architecture architecture: [ linux/amd64 ] java-version: [ 17 ] - test-mode: [ embedded, deploy ] backend: [ h2, mysql, postgresql ] + test-mode: [ embedded, deploy ] exclude: - test-mode: 'embedded' backend: 'mysql' @@ -143,53 +85,9 @@ jobs: backend: 'postgresql' - test-mode: 'deploy' backend: 'h2' - - env: - PLATFORM: ${{ matrix.architecture }} - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-java@v4 - with: - java-version: ${{ matrix.java-version }} - distribution: 'temurin' - cache: 'gradle' - - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - - name: Check required command - run: | - dev/ci/check_commands.sh - - - name: Package Gravitino - if: ${{ matrix.test-mode == 'deploy' }} - run: | - ./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }} - - - name: Free up disk space - run: | - dev/ci/util_free_space.sh - - - name: Backend Integration Test (JDK${{ matrix.java-version }}-${{ matrix.test-mode }}-${{ matrix.backend }}) - id: integrationTest - run: > - ./gradlew test -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PjdbcBackend=${{ matrix.backend }} -PskipWebITs -PskipDockerTests=false - -x :web:test -x :clients:client-python:test -x :flink-connector:flink:test -x :spark-connector:test -x :spark-connector:spark-common:test - -x :spark-connector:spark-3.3:test -x :spark-connector:spark-3.4:test -x :spark-connector:spark-3.5:test - -x :spark-connector:spark-runtime-3.3:test -x :spark-connector:spark-runtime-3.4:test -x :spark-connector:spark-runtime-3.5:test - -x :authorizations:authorization-ranger:test -x :web:integration-test:test -x :trino-connector:integration-test:test -x :trino-connector:test - - - name: Upload integrate tests reports - uses: actions/upload-artifact@v3 - if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }} - with: - name: integrate-test-reports-${{ matrix.java-version }}-${{ matrix.test-mode }}-${{ matrix.backend }} - path: | - build/reports - iceberg/iceberg-rest-server/build/*.log - distribution/package/logs/*.out - distribution/package/logs/*.log - catalogs/**/*.log - catalogs/**/*.tar - distribution/**/*.log \ No newline at end of file + uses: ./.github/workflows/backend-integration-test-action.yml + with: + architecture: ${{ matrix.architecture }} + java-version: ${{ matrix.java-version }} + backend: ${{ matrix.backend }} + test-mode: ${{ matrix.test-mode }} \ No newline at end of file diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b934c4cc26a..8490b4d851f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -25,6 +25,7 @@ jobs: source_changes: - .github/** - api/** + - authorizations/** - bin/** - catalogs/** - clients/client-java/** @@ -35,23 +36,28 @@ jobs: - conf/** - core/** - dev/** + - docs/open-api/** + - docs/build.gradle.kts + - flink-connector/** - gradle/** - - integration-test/** + - iceberg/** + - integration-test-common/** - meta/** + - scripts/** - server/** - server-common/** - spark-connector/** - - flink-connector/** - trino-connector/** - web/** - - docs/open-api/** - - docs/build.gradle.kts - build.gradle.kts - gradle.properties - gradlew - setting.gradle.kts + spark_connector_changes: + - spark-connector/** outputs: source_changes: ${{ steps.filter.outputs.source_changes }} + spark_connector_changes: ${{ steps.filter.outputs.spark_connector_changes }} compile-check: runs-on: ubuntu-latest @@ -74,7 +80,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 30 needs: changes - if: needs.changes.outputs.source_changes == 'true' + if: needs.changes.outputs.spark_connector_changes == 'true' steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/flink-integration-test-action.yml b/.github/workflows/flink-integration-test-action.yml new file mode 100644 index 00000000000..f22308c1951 --- /dev/null +++ b/.github/workflows/flink-integration-test-action.yml @@ -0,0 +1,66 @@ +name: Flink Integration Test Action + +# run flink integration test +on: + workflow_call: + inputs: + architecture: + required: true + description: 'Architecture of the platform' + type: string + java-version: + required: true + description: 'Java version' + type: string + +jobs: + start-runner: + name: JDK${{ inputs.java-version }} + runs-on: ubuntu-latest + timeout-minutes: 30 + env: + PLATFORM: ${{ inputs.architecture }} + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-java@v4 + with: + java-version: ${{ inputs.java-version }} + distribution: 'temurin' + cache: 'gradle' + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - name: Check required command + run: | + dev/ci/check_commands.sh + + - name: Package Gravitino + run: | + ./gradlew compileDistribution -x test -PjdkVersion=${{ inputs.java-version }} + + - name: Free up disk space + run: | + dev/ci/util_free_space.sh + + - name: Flink Integration Test + id: integrationTest + # run embedded mode and deploy mode integration tests + run: | + ./gradlew -PskipTests -PtestMode=embedded -PjdkVersion=${{ inputs.java-version }} -PskipDockerTests=false :flink-connector:flink:test --tests "org.apache.gravitino.flink.connector.integration.test.**" + ./gradlew -PskipTests -PtestMode=deploy -PjdkVersion=${{ inputs.java-version }} -PskipDockerTests=false :flink-connector:flink:test --tests "org.apache.gravitino.flink.connector.integration.test.**" + + - name: Upload integrate tests reports + uses: actions/upload-artifact@v3 + if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }} + with: + name: flink-connector-integrate-test-reports-${{ inputs.java-version }} + path: | + build/reports + flink-connector/flink/build/*.log + flink-connector/flink/build/*.tar + distribution/package/logs/gravitino-server.out + distribution/package/logs/gravitino-server.log + catalogs/**/*.log + catalogs/**/*.tar \ No newline at end of file diff --git a/.github/workflows/flink-integration-test.yml b/.github/workflows/flink-integration-test.yml index 6e1227664cb..eb87b37c344 100644 --- a/.github/workflows/flink-integration-test.yml +++ b/.github/workflows/flink-integration-test.yml @@ -2,7 +2,6 @@ name: Flink Integration Test # Controls when the workflow will run on: - # Triggers the workflow on push or pull request events but only for the "main" branch push: branches: [ "main", "branch-*" ] pull_request: @@ -28,18 +27,19 @@ jobs: - catalogs/** - clients/client-java/** - clients/client-java-runtime/** - - clients/filesystem-hadoop3/** - - clients/filesystem-hadoop3-runtime/** - common/** - conf/** - core/** - dev/** + - flink-connector/** - gradle/** + # todo: uncomment the following line after the flink-connector support Iceberg + # - iceberg/** + - integration-test-common/** - meta/** + - scripts/** - server/** - server-common/** - - flink-connector/** - - docs/open-api/** - build.gradle.kts - gradle.properties - gradlew @@ -47,58 +47,26 @@ jobs: outputs: source_changes: ${{ steps.filter.outputs.source_changes }} - # Integration test for AMD64 architecture - test-amd64-arch: + FlinkIT-on-push: needs: changes - if: needs.changes.outputs.source_changes == 'true' - runs-on: ubuntu-latest - timeout-minutes: 30 + if: (github.event_name == 'push' && needs.changes.outputs.source_changes == 'true') strategy: matrix: architecture: [linux/amd64] java-version: [ 8, 11, 17 ] - env: - PLATFORM: ${{ matrix.architecture }} - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-java@v4 - with: - java-version: ${{ matrix.java-version }} - distribution: 'temurin' - cache: 'gradle' - - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - - name: Check required command - run: | - dev/ci/check_commands.sh + uses: ./.github/workflows/flink-integration-test-action.yml + with: + architecture: ${{ matrix.architecture }} + java-version: ${{ matrix.java-version }} - - name: Package Gravitino - run: | - ./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }} - - - name: Free up disk space - run: | - dev/ci/util_free_space.sh - - - name: Flink Integration Test - id: integrationTest - run: | - ./gradlew -PskipTests -PtestMode=embedded -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :flink-connector:flink:test --tests "org.apache.gravitino.flink.connector.integration.test.**" - ./gradlew -PskipTests -PtestMode=deploy -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :flink-connector:flink:test --tests "org.apache.gravitino.flink.connector.integration.test.**" - - - name: Upload integrate tests reports - uses: actions/upload-artifact@v3 - if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }} - with: - name: flink-connector-integrate-test-reports-${{ matrix.java-version }} - path: | - build/reports - flink-connector/flink/build/*.log - flink-connector/flink/build/*.tar - distribution/package/logs/gravitino-server.out - distribution/package/logs/gravitino-server.log - catalogs/**/*.log - catalogs/**/*.tar + FlinkIT-on-pr: + needs: changes + if: (github.event_name == 'pull_request' && needs.changes.outputs.source_changes == 'true') + strategy: + matrix: + architecture: [linux/amd64] + java-version: [ 17 ] + uses: ./.github/workflows/flink-integration-test-action.yml + with: + architecture: ${{ matrix.architecture }} + java-version: ${{ matrix.java-version }} \ No newline at end of file diff --git a/.github/workflows/frontend-integration-test.yml b/.github/workflows/frontend-integration-test.yml index 4f1bf0d568f..2dd2bee6474 100644 --- a/.github/workflows/frontend-integration-test.yml +++ b/.github/workflows/frontend-integration-test.yml @@ -2,7 +2,6 @@ name: Frontend Integration Test # Controls when the workflow will run on: - # Triggers the workflow on push or pull request events but only for the "main" branch push: branches: [ "main", "branch-*" ] pull_request: @@ -27,22 +26,18 @@ jobs: - bin/** - catalogs/** - clients/client-java/** - - clients/client-java-runtime/** - - clients/filesystem-hadoop3/** - - clients/filesystem-hadoop3-runtime/** - common/** - conf/** - core/** - dev/** - gradle/** - - integration-test/** + - iceberg/** + - integration-test-common/** - meta/** + - scripts/** - server/** - server-common/** - - spark-connector/** - - trino-connector/** - web/** - - docs/open-api/** - build.gradle.kts - gradle.properties - gradlew @@ -50,16 +45,16 @@ jobs: outputs: source_changes: ${{ steps.filter.outputs.source_changes }} - # Integration test for AMD64 architecture - test-amd64-arch: + FrontendIT: needs: changes if: needs.changes.outputs.source_changes == 'true' runs-on: ubuntu-latest timeout-minutes: 60 strategy: matrix: + # Integration test for AMD64 architecture architecture: [linux/amd64] - java-version: [ 8 ] + java-version: [ 17 ] env: PLATFORM: ${{ matrix.architecture }} steps: @@ -89,8 +84,8 @@ jobs: - name: Frontend Integration Test id: integrationTest run: | - ./gradlew -PskipTests -PtestMode=embedded -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :web:integration-test:test --tests "org.apache.gravitino.integration.test.web.ui.**" - ./gradlew -PskipTests -PtestMode=deploy -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :web:integration-test:test --tests "org.apache.gravitino.integration.test.web.ui.**" + ./gradlew -PskipTests -PtestMode=embedded -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :web:integration-test:test + ./gradlew -PskipTests -PtestMode=deploy -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :web:integration-test:test - name: Upload integrate tests reports uses: actions/upload-artifact@v3 diff --git a/.github/workflows/python-integration-test.yml b/.github/workflows/python-integration-test.yml index 4e9f96bc690..e29ed3fc46a 100644 --- a/.github/workflows/python-integration-test.yml +++ b/.github/workflows/python-integration-test.yml @@ -2,7 +2,6 @@ name: Python Client Integration Test # Controls when the workflow will run on: - # Triggers the workflow on push or pull request events but only for the "main" branch push: branches: [ "main", "branch-*" ] pull_request: @@ -24,27 +23,37 @@ jobs: source_changes: - .github/** - api/** - - catalogs/catalog-hadoop/** + - authorizations/** + - bin/** + - catalogs/** - clients/client-python/** - common/** - conf/** - core/** + - dev/** + - gradle/** + - iceberg/** - meta/** + - scripts/** - server/** - server-common/** + - build.gradle.kts + - gradle.properties + - gradlew + - setting.gradle.kts outputs: source_changes: ${{ steps.filter.outputs.source_changes }} - # Integration test for AMD64 architecture - test-amd64-arch: + PythonIT: needs: changes if: needs.changes.outputs.source_changes == 'true' runs-on: ubuntu-latest timeout-minutes: 30 strategy: matrix: + # Integration test for AMD64 architecture architecture: [linux/amd64] - java-version: [ 8 ] + java-version: [ 17 ] env: PLATFORM: ${{ matrix.architecture }} steps: diff --git a/.github/workflows/spark-integration-test-action.yml b/.github/workflows/spark-integration-test-action.yml new file mode 100644 index 00000000000..873877bc29c --- /dev/null +++ b/.github/workflows/spark-integration-test-action.yml @@ -0,0 +1,76 @@ +name: Spark Integration Test Action + +# run spark integration test +on: + workflow_call: + inputs: + architecture: + required: true + description: 'Architecture of the platform' + type: string + java-version: + required: true + description: 'Java version' + type: string + scala-version: + required: true + description: 'Scala version' + type: string + test-mode: + required: true + description: 'run on embedded or deploy mode' + type: string + +jobs: + start-runner: + name: JDK${{ inputs.java-version }}-${{ inputs.test-mode }}-Scala${{ inputs.scala-version }} + runs-on: ubuntu-latest + timeout-minutes: 60 + env: + PLATFORM: ${{ inputs.architecture }} + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-java@v4 + with: + java-version: ${{ inputs.java-version }} + distribution: 'temurin' + cache: 'gradle' + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - name: Check required command + run: | + dev/ci/check_commands.sh + + - name: Package Gravitino + if: ${{ inputs.test-mode == 'deploy' }} + run: | + ./gradlew compileDistribution -x test -PjdkVersion=${{ inputs.java-version }} + + - name: Free up disk space + run: | + dev/ci/util_free_space.sh + + - name: Spark Integration Test + id: integrationTest + run: | + if [ "${{ inputs.scala-version }}" == "2.12" ];then + ./gradlew -PskipTests -PtestMode=${{ inputs.test-mode }} -PjdkVersion=${{ inputs.java-version }} -PscalaVersion=${{ inputs.scala-version }} -PskipDockerTests=false :spark-connector:spark-3.3:test --tests "org.apache.gravitino.spark.connector.integration.test.**" + fi + ./gradlew -PskipTests -PtestMode=${{ inputs.test-mode }} -PjdkVersion=${{ inputs.java-version }} -PscalaVersion=${{ inputs.scala-version }} -PskipDockerTests=false :spark-connector:spark-3.4:test --tests "org.apache.gravitino.spark.connector.integration.test.**" + ./gradlew -PskipTests -PtestMode=${{ inputs.test-mode }} -PjdkVersion=${{ inputs.java-version }} -PscalaVersion=${{ inputs.scala-version }} -PskipDockerTests=false :spark-connector:spark-3.5:test --tests "org.apache.gravitino.spark.connector.integration.test.**" + + - name: Upload integrate tests reports + uses: actions/upload-artifact@v3 + if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }} + with: + name: spark-connector-integrate-test-reports-${{ inputs.java-version }}-${{ inputs.test-mode }} + path: | + build/reports + spark-connector/v3.3/spark/build/spark-3.3-integration-test.log + spark-connector/v3.4/spark/build/spark-3.4-integration-test.log + spark-connector/v3.5/spark/build/spark-3.5-integration-test.log + distribution/package/logs/*.out + distribution/package/logs/*.log \ No newline at end of file diff --git a/.github/workflows/spark-integration-test.yml b/.github/workflows/spark-integration-test.yml index a2f11061cda..55fd22822c3 100644 --- a/.github/workflows/spark-integration-test.yml +++ b/.github/workflows/spark-integration-test.yml @@ -2,7 +2,6 @@ name: Spark Integration Test # Controls when the workflow will run on: - # Triggers the workflow on push or pull request events but only for the "main" branch push: branches: [ "main", "branch-*" ] pull_request: @@ -28,19 +27,18 @@ jobs: - catalogs/** - clients/client-java/** - clients/client-java-runtime/** - - clients/filesystem-hadoop3/** - - clients/filesystem-hadoop3-runtime/** - common/** - conf/** - core/** - dev/** - gradle/** - iceberg/** + - integration-test-common/** - meta/** + - scripts/** - server/** - server-common/** - spark-connector/** - - docs/open-api/** - build.gradle.kts - gradle.properties - gradlew @@ -48,63 +46,36 @@ jobs: outputs: source_changes: ${{ steps.filter.outputs.source_changes }} - # Integration test for AMD64 architecture - test-amd64-arch: + SparkIT-on-push: needs: changes - if: needs.changes.outputs.source_changes == 'true' - runs-on: ubuntu-latest - timeout-minutes: 90 + if: (github.event_name == 'push' && needs.changes.outputs.source_changes == 'true') strategy: matrix: + # Integration test for AMD64 architecture architecture: [linux/amd64] java-version: [ 8, 11, 17 ] scala-version: [ 2.12 ] test-mode: [ embedded, deploy ] - env: - PLATFORM: ${{ matrix.architecture }} - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-java@v4 - with: - java-version: ${{ matrix.java-version }} - distribution: 'temurin' - cache: 'gradle' - - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - - name: Check required command - run: | - dev/ci/check_commands.sh - - - name: Package Gravitino - if : ${{ matrix.test-mode == 'deploy' }} - run: | - ./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }} - - - name: Free up disk space - run: | - dev/ci/util_free_space.sh + uses: ./.github/workflows/spark-integration-test-action.yml + with: + architecture: ${{ matrix.architecture }} + java-version: ${{ matrix.java-version }} + scala-version: ${{ matrix.scala-version }} + test-mode: ${{ matrix.test-mode }} - - name: Spark Integration Test - id: integrationTest - run: | - if [ "${{ matrix.scala-version }}" == "2.12" ];then - ./gradlew -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PscalaVersion=${{ matrix.scala-version }} -PskipDockerTests=false :spark-connector:spark-3.3:test --tests "org.apache.gravitino.spark.connector.integration.test.**" - fi - ./gradlew -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PscalaVersion=${{ matrix.scala-version }} -PskipDockerTests=false :spark-connector:spark-3.4:test --tests "org.apache.gravitino.spark.connector.integration.test.**" - ./gradlew -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PscalaVersion=${{ matrix.scala-version }} -PskipDockerTests=false :spark-connector:spark-3.5:test --tests "org.apache.gravitino.spark.connector.integration.test.**" - - - name: Upload integrate tests reports - uses: actions/upload-artifact@v3 - if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }} - with: - name: spark-connector-integrate-test-reports-${{ matrix.java-version }}-${{ matrix.test-mode }} - path: | - build/reports - spark-connector/v3.3/spark/build/spark-3.3-integration-test.log - spark-connector/v3.4/spark/build/spark-3.4-integration-test.log - spark-connector/v3.5/spark/build/spark-3.5-integration-test.log - distribution/package/logs/*.out - distribution/package/logs/*.log + SparkIT-on-pr: + needs: changes + if: (github.event_name == 'pull_request' && needs.changes.outputs.source_changes == 'true') + strategy: + matrix: + # Integration test for AMD64 architecture + architecture: [linux/amd64] + java-version: [ 17 ] + scala-version: [ 2.12 ] + test-mode: [ embedded, deploy ] + uses: ./.github/workflows/spark-integration-test-action.yml + with: + architecture: ${{ matrix.architecture }} + java-version: ${{ matrix.java-version }} + scala-version: ${{ matrix.scala-version }} + test-mode: ${{ matrix.test-mode }} \ No newline at end of file diff --git a/.github/workflows/trino-integration-test.yml b/.github/workflows/trino-integration-test.yml index b3ea3b0eff6..421b2303f93 100644 --- a/.github/workflows/trino-integration-test.yml +++ b/.github/workflows/trino-integration-test.yml @@ -2,7 +2,6 @@ name: Trino Integration Test # Controls when the workflow will run on: - # Triggers the workflow on push or pull request events but only for the "main" branch push: branches: [ "main", "branch-*" ] pull_request: @@ -33,7 +32,10 @@ jobs: - core/** - dev/** - gradle/** + - iceberg/** + - integration-test-common/** - meta/** + - scripts/** - server/** - server-common/** - trino-connector/** @@ -45,7 +47,7 @@ jobs: source_changes: ${{ steps.filter.outputs.source_changes }} # Integration test for AMD64 architecture - test-amd64-arch: + TrinoIT: needs: changes if: needs.changes.outputs.source_changes == 'true' runs-on: ubuntu-latest @@ -94,6 +96,7 @@ jobs: path: | build/reports trino-connector/integrate-test/build/*.log + trino-connector/integrate-test/build/*.tar integration-test-common/build/trino-ci-container-log distribution/package/logs/gravitino-server.out distribution/package/logs/gravitino-server.log diff --git a/authorizations/authorization-ranger/build.gradle.kts b/authorizations/authorization-ranger/build.gradle.kts index b197dc20cf4..efc20e6c8e1 100644 --- a/authorizations/authorization-ranger/build.gradle.kts +++ b/authorizations/authorization-ranger/build.gradle.kts @@ -86,6 +86,7 @@ dependencies { exclude("org.eclipse.jetty.aggregate") } testImplementation(libs.mysql.driver) + testImplementation(libs.postgresql.driver) } tasks { @@ -111,16 +112,11 @@ tasks { tasks.test { dependsOn(":catalogs:catalog-hive:jar", ":catalogs:catalog-hive:runtimeJars") - val skipUTs = project.hasProperty("skipTests") - if (skipUTs) { - // Only run integration tests - include("**/integration/**") - } val skipITs = project.hasProperty("skipITs") if (skipITs) { // Exclude integration tests - exclude("**/integration/**") + exclude("**/integration/test/**") } else { dependsOn(tasks.jar) } diff --git a/build.gradle.kts b/build.gradle.kts index da13580a32e..1737902aff6 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -204,6 +204,13 @@ allprojects { throw GradleException("Gravitino integration tests only support [-PtestMode=embedded] or [-PtestMode=deploy] mode!") } + param.useJUnitPlatform() + val skipUTs = project.hasProperty("skipTests") + if (skipUTs) { + // Only run integration tests + param.include("**/integration/test/**") + } + param.useJUnitPlatform { val dockerTest = project.rootProject.extra["dockerTest"] as? Boolean ?: false if (!dockerTest) { diff --git a/catalogs/catalog-hadoop/build.gradle.kts b/catalogs/catalog-hadoop/build.gradle.kts index 429cd405243..ba60a161d8f 100644 --- a/catalogs/catalog-hadoop/build.gradle.kts +++ b/catalogs/catalog-hadoop/build.gradle.kts @@ -149,16 +149,10 @@ tasks.test { } } - val skipUTs = project.hasProperty("skipTests") - if (skipUTs) { - // Only run integration tests - include("**/integration/**") - } - val skipITs = project.hasProperty("skipITs") if (skipITs) { // Exclude integration tests - exclude("**/integration/**") + exclude("**/integration/test/**") } else { dependsOn(tasks.jar) } diff --git a/catalogs/catalog-hive/build.gradle.kts b/catalogs/catalog-hive/build.gradle.kts index 081bfcbeb6e..84474878e8c 100644 --- a/catalogs/catalog-hive/build.gradle.kts +++ b/catalogs/catalog-hive/build.gradle.kts @@ -196,16 +196,10 @@ tasks { } tasks.test { - val skipUTs = project.hasProperty("skipTests") - if (skipUTs) { - // Only run integration tests - include("**/integration/**") - } - val skipITs = project.hasProperty("skipITs") if (skipITs) { // Exclude integration tests - exclude("**/integration/**") + exclude("**/integration/test/**") } else { dependsOn(tasks.jar) } diff --git a/catalogs/catalog-jdbc-doris/build.gradle.kts b/catalogs/catalog-jdbc-doris/build.gradle.kts index ccfaaddfb7b..7be17eb6191 100644 --- a/catalogs/catalog-jdbc-doris/build.gradle.kts +++ b/catalogs/catalog-jdbc-doris/build.gradle.kts @@ -93,15 +93,10 @@ tasks { } tasks.test { - val skipUTs = project.hasProperty("skipTests") - if (skipUTs) { - // Only run integration tests - include("**/integration/**") - } val skipITs = project.hasProperty("skipITs") if (skipITs) { // Exclude integration tests - exclude("**/integration/**") + exclude("**/integration/test/**") } else { dependsOn(tasks.jar) } diff --git a/catalogs/catalog-jdbc-mysql/build.gradle.kts b/catalogs/catalog-jdbc-mysql/build.gradle.kts index 95f0578d3bf..f469307eca0 100644 --- a/catalogs/catalog-jdbc-mysql/build.gradle.kts +++ b/catalogs/catalog-jdbc-mysql/build.gradle.kts @@ -97,16 +97,10 @@ tasks { } tasks.test { - val skipUTs = project.hasProperty("skipTests") - if (skipUTs) { - // Only run integration tests - include("**/integration/**") - } - val skipITs = project.hasProperty("skipITs") if (skipITs) { // Exclude integration tests - exclude("**/integration/**") + exclude("**/integration/test/**") } else { dependsOn(tasks.jar) } diff --git a/catalogs/catalog-jdbc-postgresql/build.gradle.kts b/catalogs/catalog-jdbc-postgresql/build.gradle.kts index 70ffcfba514..dfb2d4404b2 100644 --- a/catalogs/catalog-jdbc-postgresql/build.gradle.kts +++ b/catalogs/catalog-jdbc-postgresql/build.gradle.kts @@ -96,16 +96,10 @@ tasks { } tasks.test { - val skipUTs = project.hasProperty("skipTests") - if (skipUTs) { - // Only run integration tests - include("**/integration/**") - } - val skipITs = project.hasProperty("skipITs") if (skipITs) { // Exclude integration tests - exclude("**/integration/**") + exclude("**/integration/test/**") } else { // PG will use project jdbc-mysql/build/libs directory, so we add the task dependency here. dependsOn(":catalogs:catalog-jdbc-mysql:jar") diff --git a/catalogs/catalog-kafka/build.gradle.kts b/catalogs/catalog-kafka/build.gradle.kts index 3165758a168..fe8e6086f46 100644 --- a/catalogs/catalog-kafka/build.gradle.kts +++ b/catalogs/catalog-kafka/build.gradle.kts @@ -106,16 +106,10 @@ tasks.test { } } - val skipUTs = project.hasProperty("skipTests") - if (skipUTs) { - // Only run integration tests - include("**/integration/**") - } - val skipITs = project.hasProperty("skipITs") if (skipITs) { // Exclude integration tests - exclude("**/integration/**") + exclude("**/integration/test/**") } else { dependsOn(tasks.jar) } diff --git a/catalogs/catalog-lakehouse-iceberg/build.gradle.kts b/catalogs/catalog-lakehouse-iceberg/build.gradle.kts index 5ee05e39724..bab879f64d9 100644 --- a/catalogs/catalog-lakehouse-iceberg/build.gradle.kts +++ b/catalogs/catalog-lakehouse-iceberg/build.gradle.kts @@ -138,16 +138,10 @@ tasks { } tasks.test { - val skipUTs = project.hasProperty("skipTests") - if (skipUTs) { - // Only run integration tests - include("**/integration/**") - } - val skipITs = project.hasProperty("skipITs") if (skipITs) { // Exclude integration tests - exclude("**/integration/**") + exclude("**/integration/test/**") } else { dependsOn(tasks.jar) } diff --git a/catalogs/catalog-lakehouse-paimon/build.gradle.kts b/catalogs/catalog-lakehouse-paimon/build.gradle.kts index 930e3e3354d..3974fba61f3 100644 --- a/catalogs/catalog-lakehouse-paimon/build.gradle.kts +++ b/catalogs/catalog-lakehouse-paimon/build.gradle.kts @@ -127,16 +127,10 @@ tasks { } tasks.test { - val skipUTs = project.hasProperty("skipTests") - if (skipUTs) { - // Only run integration tests - include("**/integration/**") - } - val skipITs = project.hasProperty("skipITs") if (skipITs) { // Exclude integration tests - exclude("**/integration/**") + exclude("**/integration/test/**") } else { dependsOn(tasks.jar) } diff --git a/docs/how-to-test.md b/docs/how-to-test.md index 48bdf74d49a..99bfed7c9d8 100644 --- a/docs/how-to-test.md +++ b/docs/how-to-test.md @@ -70,7 +70,7 @@ To deploy the Gravitino server locally to run the integration tests, follow thes * Skip unit tests by using the `./gradlew build -PskipTests` command. * Skip integration tests by using the `./gradlew build -PskipITs` command. -* Skip web frontend integration tests by using the `./gradlew build -PskipWebITs` command. +* Skip web frontend integration tests by using the `./gradlew build -x :web:integration-test:test` command. * Skip both unit tests and integration tests by using the `./gradlew build -x test` or `./gradlew build -PskipTests -PskipITs` commands. ## Configuring parameters for integration tests diff --git a/flink-connector/flink/build.gradle.kts b/flink-connector/flink/build.gradle.kts index 34b09a7f0a3..c4e75200ec6 100644 --- a/flink-connector/flink/build.gradle.kts +++ b/flink-connector/flink/build.gradle.kts @@ -41,7 +41,6 @@ dependencies { implementation(project(":api")) implementation(project(":catalogs:catalog-common")) implementation(project(":common")) - implementation(project(":core")) compileOnly(libs.bundles.log4j) implementation(libs.commons.lang3) @@ -172,17 +171,10 @@ dependencies { } tasks.test { - val skipUTs = project.hasProperty("skipTests") - if (skipUTs) { - // Only run integration tests - include("**/integration/**") - } - val skipITs = project.hasProperty("skipITs") - val skipFlinkITs = project.hasProperty("skipFlinkITs") - if (skipITs || skipFlinkITs) { + if (skipITs) { // Exclude integration tests - exclude("**/integration/**") + exclude("**/integration/test/**") } else { dependsOn(tasks.jar) dependsOn(":catalogs:catalog-hive:jar") diff --git a/iceberg/iceberg-rest-server/build.gradle.kts b/iceberg/iceberg-rest-server/build.gradle.kts index 0ee6b0ff693..a90b85b96ff 100644 --- a/iceberg/iceberg-rest-server/build.gradle.kts +++ b/iceberg/iceberg-rest-server/build.gradle.kts @@ -150,16 +150,10 @@ tasks { } tasks.test { - val skipUTs = project.hasProperty("skipTests") - if (skipUTs) { - // Only run integration tests - include("**/integration/**") - } - val skipITs = project.hasProperty("skipITs") if (skipITs) { // Exclude integration tests - exclude("**/integration/**") + exclude("**/integration/test/**") } else { dependsOn(tasks.jar) } diff --git a/spark-connector/spark-common/build.gradle.kts b/spark-connector/spark-common/build.gradle.kts index daa34c811a1..2d875204628 100644 --- a/spark-connector/spark-common/build.gradle.kts +++ b/spark-connector/spark-common/build.gradle.kts @@ -128,17 +128,10 @@ dependencies { } tasks.test { - val skipUTs = project.hasProperty("skipTests") - if (skipUTs) { - // Only run integration tests - include("**/integration/**") - } - val skipITs = project.hasProperty("skipITs") - val skipSparkITs = project.hasProperty("skipSparkITs") - if (skipITs || skipSparkITs) { + if (skipITs) { // Exclude integration tests - exclude("**/integration/**") + exclude("**/integration/test/**") } else { dependsOn(tasks.jar) } diff --git a/spark-connector/v3.3/spark/build.gradle.kts b/spark-connector/v3.3/spark/build.gradle.kts index 4f60c46555e..e209da55272 100644 --- a/spark-connector/v3.3/spark/build.gradle.kts +++ b/spark-connector/v3.3/spark/build.gradle.kts @@ -136,21 +136,14 @@ dependencies { } tasks.test { - val skipUTs = project.hasProperty("skipTests") - if (skipUTs) { - // Only run integration tests - include("**/integration/**") - } - val skipITs = project.hasProperty("skipITs") - val skipSparkITs = project.hasProperty("skipSparkITs") val enableSparkSQLITs = project.hasProperty("enableSparkSQLITs") if (!enableSparkSQLITs) { exclude("**/integration/test/sql/**") } - if (skipITs || skipSparkITs) { + if (skipITs) { // Exclude integration tests - exclude("**/integration/**") + exclude("**/integration/test/**") } else { dependsOn(tasks.jar) dependsOn(":catalogs:catalog-lakehouse-iceberg:jar") diff --git a/spark-connector/v3.4/spark/build.gradle.kts b/spark-connector/v3.4/spark/build.gradle.kts index 4cdaaa628dd..73993747ff1 100644 --- a/spark-connector/v3.4/spark/build.gradle.kts +++ b/spark-connector/v3.4/spark/build.gradle.kts @@ -136,21 +136,14 @@ dependencies { } tasks.test { - val skipUTs = project.hasProperty("skipTests") - if (skipUTs) { - // Only run integration tests - include("**/integration/**") - } - val skipITs = project.hasProperty("skipITs") - val skipSparkITs = project.hasProperty("skipSparkITs") val enableSparkSQLITs = project.hasProperty("enableSparkSQLITs") if (!enableSparkSQLITs) { exclude("**/integration/test/sql/**") } - if (skipITs || skipSparkITs) { + if (skipITs) { // Exclude integration tests - exclude("**/integration/**") + exclude("**/integration/test/**") } else { dependsOn(tasks.jar) dependsOn(":catalogs:catalog-lakehouse-iceberg:jar") diff --git a/spark-connector/v3.5/spark/build.gradle.kts b/spark-connector/v3.5/spark/build.gradle.kts index a70d56b9e07..dbe5e562593 100644 --- a/spark-connector/v3.5/spark/build.gradle.kts +++ b/spark-connector/v3.5/spark/build.gradle.kts @@ -138,21 +138,14 @@ dependencies { } tasks.test { - val skipUTs = project.hasProperty("skipTests") - if (skipUTs) { - // Only run integration tests - include("**/integration/**") - } - val skipITs = project.hasProperty("skipITs") - val skipSparkITs = project.hasProperty("skipSparkITs") val enableSparkSQLITs = project.hasProperty("enableSparkSQLITs") if (!enableSparkSQLITs) { exclude("**/integration/test/sql/**") } - if (skipITs || skipSparkITs) { + if (skipITs) { // Exclude integration tests - exclude("**/integration/**") + exclude("**/integration/test/**") } else { dependsOn(tasks.jar) dependsOn(":catalogs:catalog-lakehouse-iceberg:jar") diff --git a/web/integration-test/build.gradle.kts b/web/integration-test/build.gradle.kts index ea22104cb6a..90ac2ad509c 100644 --- a/web/integration-test/build.gradle.kts +++ b/web/integration-test/build.gradle.kts @@ -54,8 +54,7 @@ dependencies { tasks.test { val skipITs = project.hasProperty("skipITs") - val skipWebITs = project.hasProperty("skipWebITs") - if (skipITs || skipWebITs) { + if (skipITs) { exclude("*") } else { dependsOn(":trino-connector:trino-connector:jar")