Ranger-5081: CI: Add check to verify plugin installation in ranger-service containers

.github/workflows/maven.yml

@@ -160,7 +160,7 @@ jobs:
           -f docker-compose.ranger-knox.yml \
           -f docker-compose.ranger-ozone.yml up -d
 
-      - name: Check status of containers and remove them
+      - name: Check status of containers
         run: |
           sleep 60
           containers=(ranger ranger-zk ranger-solr ranger-postgres ranger-usersync ranger-tagsync ranger-kms ranger-hadoop ranger-hbase ranger-kafka ranger-hive ranger-knox ozone-om ozone-scm ozone-datanode);
@@ -174,10 +174,23 @@ jobs:
               fi
           done
 
-          if [[ $flag == true ]]; then
-              echo "All required containers are up and running";
-              docker stop $(docker ps -q) && docker rm $(docker ps -aq);
-          else
-              docker stop $(docker ps -q) && docker rm $(docker ps -aq);
-              exit 1;
+          if [[ $flag == false ]]; then
+              echo "Some required containers are NOT running. Failing pipeline."
+              exit 1
           fi
+
+      - name: Check Ranger plugin status
+        env:
+          PLUGIN_RETRY_COUNT: 10
+          PLUGIN_RETRY_INTERVAL: 30
+        run: |
+          cd dev-support/ranger-docker
+          pip install --upgrade pip
+          pip install apache-ranger python-dotenv
+          ./scripts/ranger-plugin-status.py
+
+      - name: Clean up containers
+        if: always()
+        run: |
+          echo "Stopping and removing all containers..."
+          docker stop $(docker ps -q) && docker rm $(docker ps -aq)

dev-support/ranger-docker/.env

@@ -74,3 +74,15 @@ OZONE_PLUGIN_VERSION=3.0.0-SNAPSHOT
 DEBUG_ADMIN=false
 DEBUG_USERSYNC=false
 DEBUG_TAGSYNC=false
+
+
+# using this info to check plugin status in actions file
+RANGER_ADMIN_USER=admin
+RANGER_ADMIN_PASS=rangerR0cks!
+KNOX_USER=admin
+KNOX_PASS=admin-password
+
+PLUGIN_RETRY_COUNT=4
+PLUGIN_RETRY_INTERVAL=30
+
+

dev-support/ranger-docker/scripts/ranger-plugin-status.py

@@ -0,0 +1,146 @@
+#!/usr/bin/env python3
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import time
+from dotenv import load_dotenv
+from apache_ranger.client.ranger_client import RangerClient
+
+# Load environment variables from .env
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+
+# Path to the .env file in the parent directory
+ENV_PATH = os.path.join(SCRIPT_DIR, "..", ".env")
+
+# Load it
+load_dotenv(dotenv_path=ENV_PATH)
+
+RANGER_ADMIN_USER = os.getenv("RANGER_ADMIN_USER")
+RANGER_ADMIN_PASS = os.getenv("RANGER_ADMIN_PASS")
+KNOX_USER = os.getenv("KNOX_USER")
+KNOX_PASS = os.getenv("KNOX_PASS")
+
+# Ranger Admin URL and credentials
+ranger_url = "http://localhost:6080"
+credentials = (RANGER_ADMIN_USER, RANGER_ADMIN_PASS)
+
+# Initialize the Ranger client
+ranger = RangerClient(ranger_url, credentials)
+
+PLUGIN_INFO_ENDPOINT = "/service/public/v2/api/plugins/info"
+KNOX_ENDPOINT = "https://localhost:8443/gateway/sandbox/webhdfs/v1/?op=LISTSTATUS"
+
+RETRY_COUNT = int(os.getenv("PLUGIN_RETRY_COUNT", 4))
+RETRY_INTERVAL = int(os.getenv("PLUGIN_RETRY_INTERVAL", 30))
+
+expected_services = ["hdfs", "hbase", "kms", "yarn", "kafka", "ozone", "knox", "hive"]
+
+
+def init_knox_plugin():
+    print("\nTriggering Knox activity to ensure plugin status is updated...")
+    try:
+        response = ranger.session.get(
+            KNOX_ENDPOINT,
+            auth=(KNOX_USER,KNOX_PASS),
+            verify=False,
+            timeout=10
+        )
+        print("Knox activity triggered.")
+    except Exception as e:
+        print(f"Failed to trigger Knox activity: {e}")
+
+def fetch_plugin_info():
+    print(f"\nFetching plugin info from {PLUGIN_INFO_ENDPOINT} ...")
+    try:
+        response = ranger.session.get(ranger_url + PLUGIN_INFO_ENDPOINT)
+        response.raise_for_status()
+        return response.json()
+    except Exception as e:
+        print(f"Error fetching plugin info: {e}")
+        exit(1)
+
+def check_plugins(plugin_data):
+
+    failed_services = []
+    print("\n<---------  Plugin Status  ---------->")
+    for svc in expected_services:
+        print(f"\nChecking service type: {svc}")
+        entries = [entry for entry in plugin_data if entry.get("serviceType") == svc]
+
+        if not entries:
+            print(f"MISSING: No plugins found for service type '{svc}'.")
+            failed_services.append(svc)
+            continue
+
+        active_plugins = [
+            entry for entry in entries
+            if entry.get("info", {}).get("policyActiveVersion")
+        ]
+        print(f"🟢 Active plugins: {len(active_plugins)} / {len(entries)} total plugins found.")
+
+        if not active_plugins:
+            print(f"WARNING: Plugins present but NONE are active for '{svc}'.")
+            failed_services.append(svc)
+
+        print("Details:")
+        for entry in entries:
+            host = entry.get("hostName", "unknown")
+            app_type = entry.get("appType", "unknown")
+            version = entry.get("info", {}).get("policyActiveVersion", "null")
+            print(f"- Host: {host}, AppType: {app_type}, PolicyActiveVersion: {version}")
+    return failed_services
+
+
+def main():
+
+    print("Checking Ranger plugin status via Ranger Admin API")
+
+    # Trigger knox activity
+    init_knox_plugin()
+
+    # wait for status update
+    for i in range(RETRY_COUNT):         #retrying upto RETRY_COUNT * RETRY_INTERVAL seconds
+        plugin_data = fetch_plugin_info()
+        if all(any(entry.get("info", {}).get("policyActiveVersion") for entry in plugin_data if entry["serviceType"] == svc) for svc in expected_services):
+           break
+        print(f"Some plugins not active yet, retrying in {RETRY_INTERVAL}s...")
+        time.sleep(RETRY_INTERVAL)
+
+    else:
+        print("Timed out waiting for plugins to become active.")
+
+    # fetch plugin info through admin API
+    plugin_data = fetch_plugin_info()
+
+    if not plugin_data:
+        print("No plugin info returned from API.")
+        exit(1)
+
+    # get plugin details
+    failed_services = check_plugins(plugin_data)
+
+    print()
+    if failed_services:
+        print(f"❌ The following plugins are missing or inactive: {', '.join(failed_services)}")
+        exit(1)
+    else:
+        print("✅ All expected plugins are present and active.")
+
+if __name__ == "__main__":
+    main()
+

dev-support/ranger-docker/scripts/ranger-plugin-status.sh

@@ -0,0 +1,86 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e 
+
+echo "Checking Ranger plugin status via Ranger Admin API"
+
+# Load environment variables from .env file
+SCRIPT_DIR=$(dirname "$0")
+source "$SCRIPT_DIR/../.env"
+
+RANGER_HOST="http://localhost:6080"
+ENDPOINT="$RANGER_HOST/service/public/v2/api/plugins/info"
+
+# Trigger activity for KNOX to make plugin active
+echo
+echo "Triggering Knox activity to ensure plugin status is updated..."
+KNOX_ENDPOINT="https://localhost:8443/gateway/sandbox/webhdfs/v1/?op=LISTSTATUS"
+
+curl -k -u "$KNOX_USER:$KNOX_PASS" "$KNOX_ENDPOINT" > /dev/null 2>&1
+echo "Knox activity triggered."
+
+sleep 60
+
+echo "Fetching plugin info from $ENDPOINT ..."
+response=$(curl -s -u "$RANGER_ADMIN_USER:$RANGER_ADMIN_PASS" "$ENDPOINT")
+
+if [[ -z "$response" || "$response" == "[]" ]]; then
+  echo "No plugin info returned from API."
+  exit 1
+fi
+
+expected_services=("hdfs" "hbase" "kms" "yarn" "kafka" "ozone" "knox" "hive")
+failed=false
+
+echo
+echo "<---------  Plugin Status  ----------> "
+for svc in "${expected_services[@]}"; do
+  echo
+  echo "Checking service type: $svc"
+
+  entries=$(echo "$response" | jq --arg svc "$svc" '[.[] | select(.serviceType == $svc)]')
+  count=$(echo "$entries" | jq 'length')
+
+  if (( count == 0 )); then
+    echo "MISSING: No plugins found for service type '$svc'."
+    failed=true
+    continue
+  fi
+
+  active_count=$(echo "$entries" | jq '[.[] | select(.info.policyActiveVersion != null and .info.policyActiveVersion != "")] | length')
+
+  echo "🟢 Active plugins: $active_count / $count total plugins found."
+
+  if (( active_count == 0 )); then
+    echo "WARNING: Plugins present but NONE are active for '$svc'."
+    failed=true
+  fi
+
+  # List hostnames and plugin statuses for this service type
+  echo "Details:"
+  echo "$entries" | jq -r '.[] | "- Host: \(.hostName), AppType: \(.appType), PolicyActiveVersion: \(.info.policyActiveVersion // "null")"'
+done
+
+echo
+if $failed; then
+  echo "❌ One or more plugins are missing or inactive."
+  exit 1
+else
+  echo "✅ All expected plugins are present and active."
+fi