RANGER-4978: Synchronize Tags From Open Metadata Table Entities

.github/workflows/maven.yml

@@ -181,3 +181,128 @@ jobs:
               docker stop $(docker ps -q) && docker rm $(docker ps -aq);
               exit 1;
           fi
+
+  build-17:
+    runs-on: ubuntu-22.04
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@v4
+      - name: Cache for maven dependencies
+        uses: actions/cache/restore@v4
+        with:
+          path: |
+            ~/.m2/repository/*/*/*
+            !~/.m2/repository/org/apache/ranger
+          key: maven-repo-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            maven-repo-
+      - name: Set up JDK 17
+        uses: actions/setup-java@v4
+        with:
+          java-version: '17'
+          distribution: 'temurin'
+      - name: Install Maven
+        run: |
+          sudo apt update
+          sudo apt install -y maven
+          mvn -version
+      - name: build (17)
+        run: mvn -T 8 clean verify -pl '!knox-agent' --no-transfer-progress -B -V
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: target-17
+          path: target/*
+
+  docker-build-17:
+    needs:
+      - build-17
+    runs-on: ubuntu-22.04
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@v4
+      - name: Download build-8 artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: target-17
+
+      - name: Copy artifacts for docker build
+        run: |
+          cp ranger-*.tar.gz dev-support/ranger-docker/dist
+          cp version dev-support/ranger-docker/dist
+
+      - name: Cache downloaded archives
+        uses: actions/cache@v4
+        with:
+          path: dev-support/ranger-docker/downloads
+          key: ${{ runner.os }}-ranger-downloads-${{ hashFiles('dev-support/ranger-docker/.env') }}
+          restore-keys: |
+            ${{ runner.os }}-ranger-downloads-
+
+      - name: Run download-archives.sh
+        run: |
+          cd dev-support/ranger-docker
+          ./download-archives.sh hadoop hive hbase kafka knox ozone
+
+      - name: Clean up Docker space
+        run: docker system prune --all --force --volumes
+
+      - name: Build all ranger-service images
+        run: |
+          cd dev-support/ranger-docker
+          docker compose -f docker-compose.ranger-base.yml build
+          export DOCKER_BUILDKIT=1
+          export COMPOSE_DOCKER_CLI_BUILD=1
+          export RANGER_DB_TYPE=postgres
+          docker compose \
+          -f docker-compose.ranger-${RANGER_DB_TYPE}.yml \
+          -f docker-compose.ranger.yml \
+          -f docker-compose.ranger-usersync.yml \
+          -f docker-compose.ranger-tagsync.yml \
+          -f docker-compose.ranger-kms.yml \
+          -f docker-compose.ranger-hadoop.yml \
+          -f docker-compose.ranger-hbase.yml \
+          -f docker-compose.ranger-kafka.yml \
+          -f docker-compose.ranger-hive.yml \
+          -f docker-compose.ranger-knox.yml \
+          -f docker-compose.ranger-ozone.yml build
+
+      - name: Bring up containers
+        run: |
+          cd dev-support/ranger-docker
+          ./scripts/ozone-plugin-docker-setup.sh
+          export RANGER_DB_TYPE=postgres
+          docker compose \
+          -f docker-compose.ranger-${RANGER_DB_TYPE}.yml \
+          -f docker-compose.ranger.yml \
+          -f docker-compose.ranger-usersync.yml \
+          -f docker-compose.ranger-tagsync.yml \
+          -f docker-compose.ranger-kms.yml \
+          -f docker-compose.ranger-hadoop.yml \
+          -f docker-compose.ranger-hbase.yml \
+          -f docker-compose.ranger-kafka.yml \
+          -f docker-compose.ranger-hive.yml \
+          -f docker-compose.ranger-knox.yml \
+          -f docker-compose.ranger-ozone.yml up -d
+
+      - name: Check status of containers and remove them
+        run: |
+          sleep 60
+          containers=(ranger ranger-zk ranger-solr ranger-postgres ranger-usersync ranger-tagsync ranger-kms ranger-hadoop ranger-hbase ranger-kafka ranger-hive ranger-knox ozone-om ozone-scm ozone-datanode);
+          flag=true;
+          for container in "${containers[@]}"; do
+              if [[ $(docker inspect -f '{{.State.Running}}' $container 2>/dev/null) == "true" ]]; then
+                  echo "Container $container is running!";
+              else
+                  flag=false;
+                  echo "Container $container is NOT running!";
+              fi
+          done
+
+          if [[ $flag == true ]]; then
+              echo "All required containers are up and running";
+              docker stop $(docker ps -q) && docker rm $(docker ps -aq);
+          else
+              docker stop $(docker ps -q) && docker rm $(docker ps -aq);
+              exit 1;
+          fi

agents-common/pom.xml

@@ -176,6 +176,11 @@
             <groupId>org.mockito</groupId>
             <artifactId>mockito-core</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.openjdk.nashorn</groupId>
+            <artifactId>nashorn-core</artifactId>
+            <version>${nashorn.version}</version>
+        </dependency>
         <!-- To be added for JDK15 and above
         <dependency>
             <groupId>org.graalvm.js</groupId>

agents-common/src/main/java/org/apache/ranger/plugin/util/NashornScriptEngineCreator.java

@@ -19,8 +19,8 @@
 
 package org.apache.ranger.plugin.util;
 
-import jdk.nashorn.api.scripting.ClassFilter;
-import jdk.nashorn.api.scripting.NashornScriptEngineFactory;
+import org.openjdk.nashorn.api.scripting.ClassFilter;
+import org.openjdk.nashorn.api.scripting.NashornScriptEngineFactory;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 

plugin-nestedstructure/src/main/java/org/apache/ranger/authorization/nestedstructure/authorizer/RecordFilterJavaScript.java

@@ -18,8 +18,8 @@
 
 package org.apache.ranger.authorization.nestedstructure.authorizer;
 
-import jdk.nashorn.api.scripting.ClassFilter;
-import jdk.nashorn.api.scripting.NashornScriptEngineFactory;
+import org.openjdk.nashorn.api.scripting.ClassFilter;
+import org.openjdk.nashorn.api.scripting.NashornScriptEngineFactory;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 

pom.xml

@@ -89,7 +89,7 @@
         <dnsjava.version>3.6.2</dnsjava.version>
         <eclipse.jpa.version>2.7.12</eclipse.jpa.version>
         <elasticsearch.version>7.10.2</elasticsearch.version>
-        <enunciate.version>2.13.2</enunciate.version>
+        <enunciate.version>2.17.1</enunciate.version>
         <fasterxml.jackson.databind.version>2.17.2</fasterxml.jackson.databind.version>
         <fasterxml.jackson.version>2.17.2</fasterxml.jackson.version>
         <fasterxml.woodstox.version>5.4.0</fasterxml.woodstox.version>
@@ -170,10 +170,14 @@
         <metrics.core.version>3.0.2</metrics.core.version>
         <mockito.version>3.0.0</mockito.version>
         <mysql-connector-java.version>5.1.49</mysql-connector-java.version>
+        <nashorn.version>15.3</nashorn.version>
         <net.minidev.asm.version>1.0.2</net.minidev.asm.version>
         <netty-all.version>4.1.100.Final</netty-all.version>
         <nimbus-jose-jwt.version>10.0.1</nimbus-jose-jwt.version>
         <noggit.version>0.8</noggit.version>
+
+        <!-- openmetadata deps-->
+        <openmetadata.version>1.6.4</openmetadata.version>
         <orc.core.version>1.6.7</orc.core.version>
         <orc.version>1.5.8</orc.version>
         <org.bouncycastle.bcpkix-jdk15on>1.70</org.bouncycastle.bcpkix-jdk15on>

tagsync/conf/templates/installprop2xml.properties

@@ -69,4 +69,14 @@ hadoop_conf = hadoop_conf
 JVM_METRICS_ENABLED=ranger.tagsync.metrics.enabled
 JVM_METRICS_FILENAME=ranger.tagsync.metrics.filename
 JVM_METRICS_FILEPATH=ranger.tagsync.metrics.filepath
-JVM_METRICS_FREQUENCY_TIME_IN_MILLIS=ranger.tagsync.metrics.frequencytimeinmillis
+JVM_METRICS_FREQUENCY_TIME_IN_MILLIS=ranger.tagsync.metrics.frequencytimeinmillis
+
+#Openmetadata related properties
+TAG_SOURCE_OPENMETADATAREST_ENABLED=ranger.tagsync.source.openmetadatarest
+TAG_SOURCE_OPENMETADATAREST_CLASS=ranger.tagsync.source.openmetadatarest.class
+TAG_SOURCE_OPENMETADATAREST_TOKEN=ranger.tagsync.source.openmetadatarest.token
+TAG_SOURCE_OPENMETADATAREST_ENDPOINT=ranger.tagsync.source.openmetadatarest.endpoint
+TAG_SOURCE_OPENMETADATAREST_TABLE_MAPPER=ranger.tagsync.openmetadata.trino.instance.defaulttrinoingestionservice.ranger.service
+TAG_SOURCE_OPENMETADATAREST_SOURCE_DOWNLOAD_INTERVAL=ranger.tagsync.source.openmetadatarest.download.interval.millis
+TAGSYNC_OPENMETADATAREST_SOURCE_ENTITIES_BATCH_SIZE=ranger.tagsync.source.openmetadatarest.entities.batch.size
+TAGSYNC_OPENMETADATA_REST_SSL_CONFIG_FILENAME = ranger.tagsync.source.openmetadatarest.ssl.config.filename

tagsync/conf/templates/ranger-tagsync-template.xml

@@ -181,4 +181,38 @@
 		<name>ranger-tagsync.server.ha.https.port</name>
 		<value></value>
 	</property>
+
+	<!--openmetadata properties-->
+	<property>
+		<name>ranger.tagsync.source.openmetadatarest</name>
+		<value></value>
+	</property>
+	<property>
+		<name>ranger.tagsync.source.openmetadatarest.class</name>
+		<value></value>
+	</property>
+	<property>
+		<name>ranger.tagsync.source.openmetadatarest.token</name>
+		<value>false</value>
+	</property>
+	<property>
+		<name>ranger.tagsync.source.openmetadatarest.endpoint</name>
+		<value></value>
+	</property>
+	<property>
+		<name>ranger.tagsync.openmetadata.trino.instance.defaulttrinoingestionservice.ranger.service</name>
+		<value></value>
+	</property>
+	<property>
+		<name>ranger.tagsync.source.openmetadatarest.download.interval.millis</name>
+		<value></value>
+	</property>
+	<property>
+		<name>ranger.tagsync.source.openmetadatarest.entities.batch.size</name>
+		<value></value>
+	</property>
+	<property>
+		<name>ranger.tagsync.source.openmetadatarest.ssl.config.filename</name>
+		<value></value>
+	</property>
 </configuration>

tagsync/pom.xml

@@ -291,6 +291,16 @@
             <artifactId>jettison</artifactId>
             <version>${jettison.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.open-metadata</groupId>
+            <artifactId>openmetadata-java-client</artifactId>
+            <version>${openmetadata.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.open-metadata</groupId>
+            <artifactId>openmetadata-spec</artifactId>
+            <version>${openmetadata.version}</version>
+        </dependency>
         <dependency>
             <groupId>org.scala-lang</groupId>
             <artifactId>scala-library</artifactId>

tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSyncConfig.java

@@ -98,6 +98,19 @@ public class TagSyncConfig extends Configuration {
     private static TagSyncConfig instance;
     private static String        localHostname;
 
+    // openmetadata env variables
+    private static final String TAGSYNC_OPENMETADATASOURCE_ENDPOINT_PROP = "ranger.tagsync.source.openmetadatarest.endpoint";
+    public static final long DEFAULT_TAGSYNC_OPENMETADATAREST_SOURCE_DOWNLOAD_INTERVAL = 230000;
+    public  static final int  DEFAULT_TAGSYNC_OPENMETADATAREST_SOURCE_ENTITIES_BATCH_SIZE = 10000;
+    private static final String TAGSYNC_OPENMETADATAREST_SOURCE_DOWNLOAD_INTERVAL = "ranger.tagsync.source.openmetadatarest.download.interval.millis";
+    private static final String TAGSYNC_OPENMETADATAREST_SOURCE_ENTITIES_BATCH_SIZE = "ranger.tagsync.source.openmetadatarest.entities.batch.size";
+    private static final String TAGSYNC_OPENMETADATAREST_TOKEN_PROP = "ranger.tagsync.source.openmetadatarest.token";
+    public static final String TAGSYNC_SOURCE_OPENMETADATA_CUSTOM_RESOURCE_MAPPERS_PROP = "ranger.tagsync.openmetadatarest.custom.resource.mappers";
+    private static final String TAGSYNC_OPENMETADATA_REST_SSL_CONFIG_FILENAME = "ranger.tagsync.source.openmetadatarest.ssl.config.filename";
+    private static final String TAGSYNC_OPENMETADATAREST_KEYSTORE_PROP = "ranger.tagsync.source.openmetadatarest.keystore.filename";
+    private static final String RANGER_OPENMETADATA_TABLE_COMPONENT_NAME = "ranger.tagsync.source.openmetadatarest.component.tabletype";
+    public static final String DEFAULT_RANGER_OPENMETADATA_TABLE_COMPONENT_NAME = "trino";
+
     private Properties                 props;
 
     private TagSyncConfig() {
@@ -582,4 +595,96 @@ private void readConfigFile(String fileName) {
             localHostname = "unknown";
         }
     }
+
+    //openmetadata methods
+    static public String getOpenmetadataRESTToken(Properties prop) {
+        String token = null;
+        try{
+            if(prop!=null && prop.containsKey(TAGSYNC_OPENMETADATAREST_TOKEN_PROP)){
+                token = prop.getProperty(TAGSYNC_OPENMETADATAREST_TOKEN_PROP);
+            }
+            else if (prop != null && prop.containsKey(TAGSYNC_OPENMETADATAREST_KEYSTORE_PROP)) {
+                String path = prop.getProperty(TAGSYNC_OPENMETADATAREST_KEYSTORE_PROP);
+                if (path != null) {
+                    if (!path.trim().isEmpty()) {
+                        try {
+                            token = CredentialReader.getDecryptedString(path.trim(), TAGSYNC_OPENMETADATAREST_TOKEN_PROP, getTagsyncKeyStoreType(prop));
+                        } catch (Exception ex) {
+                            token = null;
+                        }
+                        if (token != null && !token.trim().isEmpty() && !token.trim().equalsIgnoreCase("none")) {
+                            return token;
+                        }
+                    }
+                }
+                else{
+                    LOG.info("==> Keystore Property not set for OpenMetadata token. Using the token directly from property.");
+                }
+            }
+            else{
+                token = null;
+            }
+        }
+        catch(Exception exception){
+            LOG.error("The token required to connect with Openmetadata is either null or incorrect. Expecting a valid non null token.", exception);
+        }
+        return token;
+    }
+
+    static public String getOpenmetadataRESTEndpoint(Properties prop) {
+        return prop.getProperty(TAGSYNC_OPENMETADATASOURCE_ENDPOINT_PROP);
+    }
+
+    static public int getOpenmetadataRestTagSourceEntitiesBatchSize(Properties prop) {
+        String val = prop.getProperty(TAGSYNC_OPENMETADATAREST_SOURCE_ENTITIES_BATCH_SIZE);
+        int    ret = DEFAULT_TAGSYNC_ATLASREST_SOURCE_ENTITIES_BATCH_SIZE;
+
+        if (StringUtils.isNotBlank(val)) {
+            try {
+                ret = Integer.valueOf(val);
+            } catch (NumberFormatException exception) {
+                // Ignore
+            }
+        }
+
+        return ret;
+    }
+
+    static public long getOpenmetadataRESTTagSourceDownloadIntervalInMillis(Properties prop) {
+        String val = prop.getProperty(TAGSYNC_OPENMETADATAREST_SOURCE_DOWNLOAD_INTERVAL);
+        long    ret = DEFAULT_TAGSYNC_OPENMETADATAREST_SOURCE_DOWNLOAD_INTERVAL;
+
+        if (StringUtils.isNotBlank(val)) {
+            try {
+                ret = Long.valueOf(val);
+            } catch (NumberFormatException exception) {
+                // Ignore
+            }
+        }
+
+        return ret;
+    }
+
+    static public String getOpenmetadataRESTSslConfigFile(Properties prop) {
+        return prop.getProperty(TAGSYNC_OPENMETADATA_REST_SSL_CONFIG_FILENAME);
+    }
+
+    static public String getCustomOpenmetadataRESTResourceMappers(Properties prop) {
+        return prop.getProperty(TAGSYNC_SOURCE_OPENMETADATA_CUSTOM_RESOURCE_MAPPERS_PROP);
+    }
+
+    static public String getRangerOpenmetadataTableComponentName(Properties prop) {
+        String tableComponentType = null;
+        try{
+            if(prop!=null && prop.containsKey(RANGER_OPENMETADATA_TABLE_COMPONENT_NAME)){
+                tableComponentType = prop.getProperty(RANGER_OPENMETADATA_TABLE_COMPONENT_NAME);
+            }
+        }
+        catch(Exception exception){
+            LOG.warn("Error getting table component name", exception);
+            LOG.warn("Setting property 'RANGER_OPENMETADATA_TABLE_COMPONENT_NAME' to default value");
+            tableComponentType = DEFAULT_RANGER_OPENMETADATA_TABLE_COMPONENT_NAME;
+        }
+        return tableComponentType;
+    }
 }