RANGER-2699 : JVM metrics for Ranger usersync and Ranger tagsync

Signed-off-by: Pradeep <pradeep@apache.org>

agents-common/src/main/java/org/apache/ranger/plugin/util/RangerMetricsUtil.java

@@ -0,0 +1,189 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.plugin.util;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.log4j.Logger;
+import org.apache.ranger.plugin.model.RangerMetrics;
+
+import com.google.gson.Gson;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.lang.management.ManagementFactory;
+import java.lang.management.MemoryMXBean;
+import java.lang.management.OperatingSystemMXBean;
+import java.lang.management.RuntimeMXBean;
+import java.util.Arrays;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.lang.management.MemoryPoolMXBean;
+import java.lang.management.MemoryType;
+import java.lang.management.MemoryUsage;
+
+/**
+ * Connect Worker system and runtime information.
+ */
+public class RangerMetricsUtil {
+
+    private static final Logger LOG = Logger.getLogger(RangerMetricsUtil.class);
+    private static final OperatingSystemMXBean OS;
+    private static final MemoryMXBean MEM_BEAN;
+    public static final String NL = System.getProperty("line.separator");
+
+    private static final RuntimeMXBean RUNTIME = ManagementFactory.getRuntimeMXBean();
+    private static final String JVM_MACHINE_ACTUAL_NAME = RUNTIME.getVmName();
+    private static final String VERSION = RUNTIME.getVmVersion();
+    private static final String JVM_MACHINE_REPRESENTATION_NAME = RUNTIME.getName();
+    private static final long UP_TIME_OF_JVM = RUNTIME.getUptime();
+    private static final String JVM_VENDOR_NAME =  RUNTIME.getVmVendor();
+
+
+    static {
+        OS = ManagementFactory.getOperatingSystemMXBean();
+        MEM_BEAN = ManagementFactory.getMemoryMXBean();
+    }
+
+    public Map<String, Object> getValues() {
+        if (LOG.isDebugEnabled()) {
+            LOG.debug("==> RangerMetricsUtil.getValues()");
+        }
+
+        Map<String, Object> values = new LinkedHashMap<>();
+        values.put("os.spec", StringUtils.join(Arrays.asList(addSystemInfo()), ", "));
+        values.put("os.vcpus", String.valueOf(OS.getAvailableProcessors()));
+        values.put("memory", addMemoryDetails());
+
+        if (LOG.isDebugEnabled()) {
+            LOG.debug("<== RangerMetricsUtil.getValues()" + values);
+        }
+
+        return values;
+    }
+
+    /**
+     * collect the pool division of java
+     */
+    protected Map<String, Object> getPoolDivision() {
+        if (LOG.isDebugEnabled()) {
+            LOG.debug("==> RangerMetricsUtil.getPoolDivision()");
+        }
+
+        Map<String, Object> poolDivisionValues = new LinkedHashMap<>();
+        for (MemoryPoolMXBean mpBean : ManagementFactory.getMemoryPoolMXBeans()) {
+            if (mpBean.getType() == MemoryType.HEAP) {
+                poolDivisionValues.put(mpBean.getName(), mpBean.getUsage());
+            }
+        }
+
+        if (LOG.isDebugEnabled()) {
+            LOG.debug("<== RangerMetricsUtil.getPoolDivision()" + poolDivisionValues);
+        }
+
+        return poolDivisionValues;
+    }
+
+    /**
+     * Add memory details
+     */
+    protected Map<String, Object> addMemoryDetails() {
+        if (LOG.isDebugEnabled()) {
+            LOG.debug("==> RangerMetricsUtil.addMemoryDetails()");
+        }
+
+        Map<String, Object> memory  = new LinkedHashMap<>();
+        MemoryUsage memHeapUsage = MEM_BEAN.getHeapMemoryUsage();
+        MemoryUsage nonHeapUsage = MEM_BEAN.getNonHeapMemoryUsage();
+        memory.put("heapInit", String.valueOf(memHeapUsage.getInit()));
+        memory.put("heapMax", String.valueOf(memHeapUsage.getMax()));
+        memory.put("heapCommitted", String.valueOf(memHeapUsage.getCommitted()));
+        memory.put("heapUsed", String.valueOf(memHeapUsage.getUsed()));
+        memory.put("nonHeapInit", String.valueOf(nonHeapUsage.getInit()));
+        memory.put("nonHeapMax", String.valueOf(nonHeapUsage.getMax()));
+        memory.put("nonHeapCommitted", String.valueOf(nonHeapUsage.getCommitted()));
+        memory.put("nonHeapUsed", String.valueOf(nonHeapUsage.getUsed()));
+        memory.put("memory_pool_usages", getPoolDivision());
+
+        if (LOG.isDebugEnabled()) {
+            LOG.debug("<== RangerMetricsUtil.addMemoryDetails()" + memory);
+        }
+
+        return memory;
+    }
+
+    /**
+     * Collect system information.
+     */
+    protected String[] addSystemInfo() {
+        if (LOG.isDebugEnabled()) {
+            LOG.debug("==> RangerMetricsUtil.addSystemInfo()");
+        }
+
+        String[] osInfo = { OS.getName(), OS.getArch(), OS.getVersion() };
+        if (LOG.isDebugEnabled()) {
+            LOG.debug("<== RangerMetricsUtil.addSystemInfo()" + osInfo);
+        }
+
+        return osInfo;
+    }
+
+    public RangerMetrics getVMStatus() {
+		if (LOG.isDebugEnabled()) {
+			LOG.debug("==> RangerMetricsUtil.getVMStatus()");
+		}
+
+		Map<String, Object> jvm = new LinkedHashMap<>();
+		Map<String, Object> vmDetails = new LinkedHashMap<>();
+		vmDetails.put("JVM Machine Actual Name", JVM_MACHINE_ACTUAL_NAME);
+		vmDetails.put("version", VERSION);
+		vmDetails.put("JVM Machine Representation Name", JVM_MACHINE_REPRESENTATION_NAME);
+		vmDetails.put("Up time of JVM", UP_TIME_OF_JVM);
+		vmDetails.put("JVM Vendor Name", JVM_VENDOR_NAME);
+		vmDetails.putAll(getValues());
+		jvm.put("jvm", vmDetails);
+
+		if (LOG.isDebugEnabled()) {
+			LOG.debug("<== RangerMetricsUtil.getVMStatus() " + jvm);
+		}
+
+		return new RangerMetrics(jvm);
+	}
+
+	public void writeMetricsToFile(File filePath) throws Throwable {
+
+		RangerMetrics rangerMetrics = null;
+		rangerMetrics = getVMStatus();
+		if (null == rangerMetrics || null == filePath) {
+			LOG.debug("RangerMetrics or filePath can not be null)");
+			return;
+		}
+		if (LOG.isDebugEnabled()) {
+            LOG.debug("==> RangerMetricsUtil.writeMetricsToFIle() for path: "+ filePath);
+        }
+		Gson gson = new Gson();
+		try (FileWriter file = new FileWriter(filePath)) {
+			gson.toJson(rangerMetrics, file);
+			file.flush();
+		} catch (Exception e ) {
+			LOG.error("RangerMetricsUtil.writeMetricsToFile() got an error",e);
+			throw e;
+		}
+	}
+}

tagsync/conf/templates/installprop2xml.properties

@@ -65,3 +65,8 @@ tagsync_keytab = ranger.tagsync.kerberos.keytab
 # TODO - What property in ranger-tagsync-site.xml should hadoop_conf map to??
 hadoop_conf = hadoop_conf
 
+#JVM metrics related property
+JVM_METRICS_ENABLED=ranger.tagsync.metrics.enabled
+JVM_METRICS_FILENAME=ranger.tagsync.metrics.filename
+JVM_METRICS_FILEPATH=ranger.tagsync.metrics.filepath
+JVM_METRICS_FREQUENCY_TIME_IN_MILLIS=ranger.tagsync.metrics.frequencytimeinmillis

tagsync/conf/templates/ranger-tagsync-template.xml

@@ -107,4 +107,20 @@
 		<name>ranger.tagsync.dest.ranger.session.cookie.name</name>
 		<value>RANGERADMINSESSIONID</value>
 	</property>
+	<property>
+		<name>ranger.tagsync.metrics.filepath</name>
+		<value></value>
+	</property>
+	<property>
+		<name>ranger.tagsync.metrics.filename</name>
+		<value></value>
+	</property>
+	<property>
+		<name>ranger.tagsync.metrics.frequencytimeinmillis</name>
+		<value></value>
+	</property>
+	<property>
+		<name>ranger.tagsync.metrics.enabled</name>
+		<value>false</value>
+	</property>
 </configuration>

tagsync/scripts/install.properties

@@ -109,3 +109,22 @@ tagsync_keytab=
 
 
 hadoop_conf=/etc/hadoop/conf
+
+# if you want to enable or disable jvm metrics for tagsync process
+# valid values: true, false
+# any value other than true would be treated as false
+# default value: false
+# if the value is false, jvm metrics is not created
+JVM_METRICS_ENABLED=
+
+# filename of jvm metrics created for tagsync process
+# default value: ranger_tagsync_metric.json
+JVM_METRICS_FILENAME=
+
+#file directory for jvm metrics
+# default value : logdir
+JVM_METRICS_FILEPATH=
+
+#frequency for jvm metrics to be updated
+# default value : 10000 milliseconds 
+JVM_METRICS_FREQUENCY_TIME_IN_MILLIS=

tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSyncConfig.java

@@ -31,6 +31,8 @@
 import java.io.InputStream;
 import java.net.URL;
 import java.net.UnknownHostException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
 import java.util.Enumeration;
 import java.util.Properties;
 
@@ -105,6 +107,14 @@ public class TagSyncConfig extends Configuration {
 
 	private static String LOCAL_HOSTNAME = "unknown";
 
+    private static final String  TAGSYNC_METRICS_FILEPATH =   "ranger.tagsync.metrics.filepath";
+    private static final String  DEFAULT_TAGSYNC_METRICS_FILEPATH =   "/tmp/";
+    private static final String  TAGSYNC_METRICS_FILENAME =   "ranger.tagsync.metrics.filename";
+    private static final String  DEFAULT_TAGSYNC_METRICS_FILENAME =   "ranger_tagsync_metric.json";
+    private static final String  TAGSYNC_METRICS_FREQUENCY_TIME_IN_MILLIS_PARAM = "ranger.tagsync.metrics.frequencytimeinmillis";
+    private static final long    DEFAULT_TAGSYNC_METRICS_FREQUENCY__TIME_IN_MILLIS = 10000L;
+    private static final String  TAGSYNC_METRICS_ENABLED_PROP = "ranger.tagsync.metrics.enabled";
+
 	private Properties props;
 
 	static {
@@ -460,4 +470,48 @@ private void readConfigFile(String fileName) {
 		}
 	}
 
+	public String getTagSyncMetricsFileName() {
+		String val = getProperties().getProperty(TAGSYNC_METRICS_FILEPATH);
+		if (StringUtils.isBlank(val)) {
+			if (StringUtils.isBlank(System.getProperty("logdir"))) {
+				val = DEFAULT_TAGSYNC_METRICS_FILEPATH;
+			} else {
+				val = System.getProperty("logdir");
+			}
+		}
+
+		if (Files.notExists(Paths.get(val))) {
+				return null;
+		}
+
+		StringBuilder pathAndFileName = new StringBuilder(val);
+		if (!val.endsWith("/")) {
+			pathAndFileName.append("/");
+		}
+		String fileName = getProperties().getProperty(TAGSYNC_METRICS_FILENAME);
+		if (StringUtils.isBlank(fileName)) {
+			fileName = DEFAULT_TAGSYNC_METRICS_FILENAME;
+		}
+		pathAndFileName.append(fileName);
+		return pathAndFileName.toString();
+	}
+
+	public long getTagSyncMetricsFrequency() {
+		long ret = DEFAULT_TAGSYNC_METRICS_FREQUENCY__TIME_IN_MILLIS;
+		String val = getProperties().getProperty(TAGSYNC_METRICS_FREQUENCY_TIME_IN_MILLIS_PARAM);
+		if (StringUtils.isNotBlank(val)) {
+			try {
+				ret = Long.valueOf(val);
+			} catch (NumberFormatException exception) {
+				// Ignore
+			}
+		}
+		return ret;
+	}
+
+	public static boolean isTagSyncMetricsEnabled(Properties prop) {
+		String val = prop.getProperty(TAGSYNC_METRICS_ENABLED_PROP);
+		return "true".equalsIgnoreCase(StringUtils.trimToEmpty(val));
+	}
+
 }