[#4662] improve(IT): Add ranger authorization Hive E2E test (#4677)

### What changes were proposed in this pull request?

1. Use an isolated class loader to create an authorization Ranger
plugin.
2. Add authorization for Hive E2E tests using Ranger via Gravitino
RESTful access control.
3. Copy the authorization ranger jar to the
`distribution/package/authorizations/ranger/libs`.
4. Add `Apache Ranger` to license.bin

### Why are the changes needed?

#4662 

### Does this PR introduce _any_ user-facing change?

N/A

### How was this patch tested?

CI

.github/workflows/access-control-integration-test.yml

@@ -0,0 +1,96 @@
+name: Access Control Integration Test
+
+# Controls when the workflow will run
+on:
+  # Triggers the workflow on push or pull request events but only for the "main" branch
+  push:
+    branches: [ "main", "branch-*" ]
+  pull_request:
+    branches: [ "main", "branch-*" ]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  changes:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: dorny/paths-filter@v2
+        id: filter
+        with:
+          filters: |
+            source_changes:
+              - api/**
+              - authorizations/**
+              - catalogs/**
+              - clients/client-java/**
+              - clients/client-java-runtime/**
+              - common/**
+              - core/**
+              - integration-test-common/**
+              - server/**
+              - server-common/**
+    outputs:
+      source_changes: ${{ steps.filter.outputs.source_changes }}
+
+  # Integration test for AMD64 architecture
+  test-amd64-arch:
+    needs: changes
+    if: needs.changes.outputs.source_changes == 'true'
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    strategy:
+      matrix:
+        # Integration test for AMD64 architecture
+        architecture: [linux/amd64]
+        java-version: [ 17 ]
+        test-mode: [ embedded, deploy ]
+        include:
+          - test-mode: 'embedded'
+            backend: 'h2'
+          - test-mode: 'deploy'
+            backend: 'mysql'
+
+    env:
+      PLATFORM: ${{ matrix.architecture }}
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-java@v4
+        with:
+          java-version: ${{ matrix.java-version }}
+          distribution: 'temurin'
+          cache: 'gradle'
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Check required command
+        run: |
+          dev/ci/check_commands.sh
+
+      - name: Package Gravitino
+        if : ${{ matrix.test-mode == 'deploy' }}
+        run: |
+          ./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }}
+
+      - name: Free up disk space
+        run: |
+          dev/ci/util_free_space.sh
+
+      - name: Authorization Integration Test (JDK${{ matrix.java-version }}-${{ matrix.test-mode }}-${{ matrix.backend }})
+        id: integrationTest
+        run: |
+          ./gradlew -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdbcBackend=${{ matrix.backend }} -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :authorizations:authorization-ranger:test --tests "org.apache.gravitino.authorization.ranger.integration.test.**"
+
+      - name: Upload integrate tests reports
+        uses: actions/upload-artifact@v3
+        if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }}
+        with:
+          name: authorizations-integrate-test-reports-${{ matrix.java-version }}
+          path: |
+            build/reports
+            distribution/package/logs/gravitino-server.out
+            distribution/package/logs/gravitino-server.log

.github/workflows/backend-integration-test.yml

@@ -90,13 +90,84 @@ jobs:
         run: |
           dev/ci/util_free_space.sh
 
-      - name: Backend Integration Test(JDK${{ matrix.java-version }}-${{ matrix.test-mode }}-${{ matrix.backend }})
+      - name: Backend Integration Test (JDK${{ matrix.java-version }}-${{ matrix.test-mode }}-${{ matrix.backend }})
         id: integrationTest
         run: >
           ./gradlew test -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PjdbcBackend=${{ matrix.backend }} -PskipWebITs -PskipDockerTests=false
           -x :web:test -x :clients:client-python:test -x :flink-connector:test -x :spark-connector:test -x :spark-connector:spark-common:test 
           -x :spark-connector:spark-3.3:test -x :spark-connector:spark-3.4:test -x :spark-connector:spark-3.5:test 
           -x :spark-connector:spark-runtime-3.3:test -x :spark-connector:spark-runtime-3.4:test -x :spark-connector:spark-runtime-3.5:test
+          -x :authorizations:authorization-ranger:test
+
+      - name: Upload integrate tests reports
+        uses: actions/upload-artifact@v3
+        if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }}
+        with:
+          name: integrate-test-reports-${{ matrix.java-version }}-${{ matrix.test-mode }}-${{ matrix.backend }}
+          path: |
+            build/reports
+            iceberg/iceberg-rest-server/build/*.log
+            integration-test/build/*.log
+            integration-test/build/*.tar
+            integration-test/build/trino-ci-container-log
+            distribution/package/logs/*.out
+            distribution/package/logs/*.log
+            catalogs/**/*.log
+            catalogs/**/*.tar
+            distribution/**/*.log
+
+  test-on-pr:
+    needs: changes
+    if: (github.event_name == 'pull_request' && needs.changes.outputs.source_changes == 'true')
+    runs-on: ubuntu-latest
+    timeout-minutes: 90
+    strategy:
+      matrix:
+        # Integration test for AMD64 architecture
+        architecture: [ linux/amd64 ]
+        java-version: [ 17 ]
+        test-mode: [ embedded, deploy ]
+        include:
+          - test-mode: 'embedded'
+            backend: 'h2'
+          - test-mode: 'deploy'
+            backend: 'mysql'
+
+    env:
+      PLATFORM: ${{ matrix.architecture }}
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-java@v4
+        with:
+          java-version: ${{ matrix.java-version }}
+          distribution: 'temurin'
+          cache: 'gradle'
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Check required command
+        run: |
+          dev/ci/check_commands.sh
+
+      - name: Package Gravitino
+        if: ${{ matrix.test-mode == 'deploy' }}
+        run: |
+          ./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }}
+
+      - name: Free up disk space
+        run: |
+          dev/ci/util_free_space.sh
+
+      - name: Backend Integration Test (JDK${{ matrix.java-version }}-${{ matrix.test-mode }}-${{ matrix.backend }})
+        id: integrationTest
+        run: >
+          ./gradlew test -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PjdbcBackend=${{ matrix.backend }} -PskipWebITs -PskipDockerTests=false
+          -x :web:test -x :clients:client-python:test -x :flink-connector:test -x :spark-connector:test -x :spark-connector:spark-common:test 
+          -x :spark-connector:spark-3.3:test -x :spark-connector:spark-3.4:test -x :spark-connector:spark-3.5:test 
+          -x :spark-connector:spark-runtime-3.3:test -x :spark-connector:spark-runtime-3.4:test -x :spark-connector:spark-runtime-3.5:test
+          -x :authorizations:authorization-ranger:test
 
       - name: Upload integrate tests reports
         uses: actions/upload-artifact@v3

LICENSE.bin

@@ -310,6 +310,8 @@
    Apache Yetus - Audience Annotations
    Apache Kerby
    Apache Kyuubi
+   Apache Ranger
+   Apache Ranger intg
    Jackson JSON processor
    DataNucleus
    Modernizer Maven Plugin
@@ -364,7 +366,7 @@
 
    This product bundles various third-party components also under the
    BSD license
-   
+
    JSR305
    LevelDB JNI
    RocksDB JNI
@@ -446,6 +448,7 @@
    Jakarta RESTful WS API
    Jakarta XML Binding API
    JavaServer Pages(TM) API
+   Javax WS RS API
    HK2 API Module
    HK2 Service Locator
    HK2 Utils

NOTICE.bin

@@ -22,6 +22,9 @@ Copyright 2008-2023 The Apache Software Foundation
 Apache Zeppelin
 Copyright 2016-2023 The Apache Software Foundation
 
+Apache Ranger
+Copyright 2014-2024 The Apache Software Foundation
+
 Apache Hadoop
 Copyright 2006 and onwards The Apache Software Foundation.
 

authorizations/authorization-ranger/build.gradle.kts

@@ -51,11 +51,17 @@ dependencies {
     exclude("org.apache.ranger", "ranger-plugins-audit")
     exclude("org.apache.ranger", "ranger-plugins-cred")
     exclude("org.apache.ranger", "ranger-plugins-classloader")
+    exclude("javax.ws.rs")
   }
+  implementation(libs.javax.ws.rs.api)
   implementation(libs.javax.jaxb.api) {
     exclude("*")
   }
 
+  testImplementation(project(":common"))
+  testImplementation(project(":clients:client-java"))
+  testImplementation(project(":server"))
+  testImplementation(project(":catalogs:catalog-common"))
   testImplementation(project(":integration-test-common", "testArtifacts"))
   testImplementation(libs.junit.jupiter.api)
   testImplementation(libs.mockito.core)
@@ -69,24 +75,33 @@ dependencies {
     exclude("org.elasticsearch")
     exclude("org.elasticsearch.client")
     exclude("org.elasticsearch.plugin")
+    exclude("javax.ws.rs")
   }
   testImplementation(libs.hive2.jdbc) {
     exclude("org.slf4j")
   }
+  testImplementation(libs.mysql.driver)
 }
 
 tasks {
   val runtimeJars by registering(Copy::class) {
     from(configurations.runtimeClasspath)
     into("build/libs")
   }
-}
 
-tasks.build {
-  dependsOn("runtimeJars", "javadoc")
+  val copyAuthorizationLibs by registering(Copy::class) {
+    dependsOn("jar", "runtimeJars")
+    from("build/libs")
+    into("$rootDir/distribution/package/authorizations/ranger/libs")
+  }
+
+  register("copyLibAndConfig", Copy::class) {
+    dependsOn(copyAuthorizationLibs)
+  }
 }
 
 tasks.test {
+  dependsOn(":catalogs:catalog-hive:jar", ":catalogs:catalog-hive:runtimeJars")
   val skipUTs = project.hasProperty("skipTests")
   if (skipUTs) {
     // Only run integration tests
@@ -101,7 +116,3 @@ tasks.test {
     dependsOn(tasks.jar)
   }
 }
-
-tasks.getByName("generateMetadataFileForMavenJavaPublication") {
-  dependsOn("runtimeJars")
-}