Prometheus (#106)

Add JMX Prometheus agent

README.md

@@ -80,6 +80,10 @@ The address to the Management Center application can be defined using the `MANCE
 $ docker run -e MANCENTER_URL=<mancenter_url> hazelcast/hazelcast-enterprise
 ```
 
+### PROMETHEUS_PORT
+
+The port of the JMX Prometheus agent. For example, if you set `PROMETHEUS_PORT=8080`, then you can access metrics at: `http://<hostname>:8080/metrics`. You can also use `PROMETHEUS_CONFIG` to set a path to the custom configuration.
+
 ### HZ_LICENSE_KEY (Hazelcast Enterprise Only)
 
 The license key for Hazelcast Enterprise can be defined using the `HZ_LICENSE_KEY` variable
@@ -161,6 +165,10 @@ Now you can connect with your remote debugger using the address: `localhost:5005
 
 ### Managing and Monitoring
 
+You can use JMX or Prometheus for the application monitoring.
+
+#### JMX
+
 You can use the standard JMX protocol to monitor your Hazelcast instance. Start Hazelcast container with the following parameters.
 
 ```
@@ -169,6 +177,16 @@ $ docker run -p 9999:9999 -e JAVA_OPTS='-Dhazelcast.jmx=true -Dcom.sun.managemen
 
 Now you can connect using the address: `localhost:9999`.
 
+#### Prometheus
+
+You can use JMX Prometheus agent and expose JVM and JMX Hazelcast metrics.
+
+```
+$ docker run -p 8080:8080 -e PROMETHEUS_PORT=8080
+```
+
+Then, the metrics are available at: `http://localhost:8080/metrics`. Note that you can add also `-e JAVA_OPTS='-Dhazelcast.jmx=true'` to expose JMX via Prometheus (otherwise just JVM metrics are visible).
+
 ## Docker Images Usages
 
 ### Hazelcast Docker Repositories

hazelcast-enterprise/Dockerfile

@@ -5,6 +5,7 @@ ARG HZ_VERSION=3.11.2
 ARG CACHE_API_VERSION=1.1.0
 ARG HZ_KUBE_VERSION=1.3.1
 ARG HZ_EUREKA_VERSION=1.0.2
+ARG JMX_PROMETHEUS_AGENT_VERSION=0.11.0
 ARG NETTY_VERSION=4.1.32.Final
 ARG NETTY_TCNATIVE_VERSION=2.0.20.Final
 
@@ -14,19 +15,22 @@ ARG HZ_JAR="hazelcast-enterprise-all-${HZ_VERSION}.jar"
 ARG CACHE_API_JAR="cache-api-${CACHE_API_VERSION}.jar"
 
 # Runtime constants / variables
-ENV CLASSPATH_DEFAULT="${HZ_HOME}/lib/*" \
+ENV HZ_HOME="${HZ_HOME}" \
+    CLASSPATH_DEFAULT="${HZ_HOME}/lib/*" \
     JAVA_OPTS_DEFAULT="-Djava.net.preferIPv4Stack=true" \
     MIN_HEAP_SIZE="" \
     MAX_HEAP_SIZE="" \
     HZ_LICENSE_KEY="" \
     MANCENTER_URL="" \
+    PROMETHEUS_PORT="" \
+    PROMETHEUS_CONFIG="${HZ_HOME}/jmx_agent_config.yaml" \
     CLASSPATH="" \
     JAVA_OPTS=""
 
 # Expose port
 EXPOSE 5701
 
-COPY *.xml *.sh ${HZ_HOME}/
+COPY *.xml *.sh *.yaml ${HZ_HOME}/
 
 # Install
 RUN echo "Updating Alpine system" \
@@ -38,26 +42,29 @@ RUN echo "Updating Alpine system" \
     && mkdir lib \
     && curl -sf -o ${HZ_HOME}/lib/${HZ_JAR} \
          -L https://repository.hazelcast.com/release/com/hazelcast/hazelcast-enterprise-all/${HZ_VERSION}/${HZ_JAR} \
-   && echo "Installing JCache" \
-   && curl -sf -o "${HZ_HOME}/lib/${CACHE_API_JAR}" \
+    && echo "Installing JCache" \
+    && curl -sf -o "${HZ_HOME}/lib/${CACHE_API_JAR}" \
          -L "https://repo1.maven.org/maven2/javax/cache/cache-api/${CACHE_API_VERSION}/${CACHE_API_JAR}" \
-   && echo "Installing Hazelcast plugins" \
-   && mvn -f dependency-copy.xml \
+    && echo "Installing Hazelcast plugins" \
+    && mvn -f dependency-copy.xml \
            -Dhazelcast-kubernetes-version=${HZ_KUBE_VERSION} \
            -Dhazelcast-eureka-version=${HZ_EUREKA_VERSION} \
            -Dnetty.version=${NETTY_VERSION} \
            -Dnetty-tcnative.version=${NETTY_TCNATIVE_VERSION} \
            dependency:copy-dependencies \
-   && echo "Maven clean-up" \
-   && apk del maven \
-   && rm -rf ~/.m2 \
-   && rm -f dependency-copy.xml \
-   && echo "Granting read permission to ${HZ_HOME}" \
-   && chmod -R +r $HZ_HOME \
-   && echo "Setting Pardot ID to 'docker'" \
-   && echo 'hazelcastDownloadId=docker' > hazelcast-download.properties \
-   && echo "Cleaning APK packages" \
-   && rm -rf /var/cache/apk/*
+    && echo "Maven clean-up" \
+    && apk del maven \
+    && rm -rf ~/.m2 \
+    && rm -f dependency-copy.xml \
+    && echo "Installing JMX Prometheus Agent" \
+    && curl -sf -o "${HZ_HOME}/lib/jmx_prometheus_javaagent.jar" \
+         -L "https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/${JMX_PROMETHEUS_AGENT_VERSION}/jmx_prometheus_javaagent-${JMX_PROMETHEUS_AGENT_VERSION}.jar" \
+    && echo "Granting read permission to ${HZ_HOME}" \
+    && chmod -R +r $HZ_HOME \
+    && echo "Setting Pardot ID to 'docker'" \
+    && echo 'hazelcastDownloadId=docker' > hazelcast-download.properties \
+    && echo "Cleaning APK packages" \
+    && rm -rf /var/cache/apk/*
 
 WORKDIR ${HZ_HOME}
 

hazelcast-enterprise/jmx_agent_config.yaml

@@ -0,0 +1,26 @@
+---
+ssl: false
+#lowercaseOutputName: false
+#lowercaseOutputLabelNames: false
+whitelistObjectNames: ["com.hazelcast:*"]
+rules:
+# HELP com_hazelcast_hzInstance_1_dev_clusterTime Cluster-wide Time (com.hazelcast<instance=_hzInstance_1_dev, name=_hzInstance_1_dev, type=HazelcastInstance><>clusterTime)
+  #com.hazelcast:instance=_hzInstance_1_dev,name=_hzInstance_1_dev,type=HazelcastInstance.PartitionServiceMBean
+- pattern: '<instance=(\w+), name=(\w+), type=(.*)><>(.*):'
+  name: hazelcast_$3_$4
+  #value: $4
+  #valueFactor: 0.001
+  labels:
+    instance: $1
+    name: $2
+    type: $3
+  help: "Hazelcast metric instance=$1 name=$2 type=$3 attribute=$4"
+  attrNameSnakeCase: false
+- pattern: '<name=(\w+), instance=(\w+), type=(\w+.*)'
+  name: hazelcast_$1_$3
+  labels:
+    instance: $2
+    name: $1
+    type: $3
+  help: "Hazelcast metric instance=$1 name=$2 type=$3"
+  attrNameSnakeCase: false

hazelcast-enterprise/start-hazelcast.sh

@@ -31,6 +31,12 @@ else
   export JAVA_OPTS="${JAVA_OPTS} -Dhazelcast.mancenter.enabled=false"
 fi
 
+if [ -n "${PROMETHEUS_PORT}" ]; then
+  export JAVA_OPTS="${JAVA_OPTS} -javaagent:${HZ_HOME}/lib/jmx_prometheus_javaagent.jar=${PROMETHEUS_PORT}:${PROMETHEUS_CONFIG}"
+else
+  export JAVA_OPTS="${JAVA_OPTS}"
+fi
+
 if [ -n "${HZ_LICENSE_KEY}" ]; then
   export JAVA_OPTS="${JAVA_OPTS} -Dhazelcast.enterprise.license.key=${HZ_LICENSE_KEY}"
 else

hazelcast-oss/Dockerfile

@@ -5,25 +5,29 @@ ARG HZ_VERSION=3.11.2
 ARG CACHE_API_VERSION=1.1.0
 ARG HZ_KUBE_VERSION=1.3.1
 ARG HZ_EUREKA_VERSION=1.0.2
+ARG JMX_PROMETHEUS_AGENT_VERSION=0.11.0
 
 # Build constants
 ARG HZ_HOME="/opt/hazelcast"
 ARG HZ_JAR="hazelcast-all-${HZ_VERSION}.jar"
 ARG CACHE_API_JAR="cache-api-${CACHE_API_VERSION}.jar"
 
 # Runtime constants / variables
-ENV CLASSPATH_DEFAULT="${HZ_HOME}/lib/*" \
+ENV HZ_HOME="${HZ_HOME}" \
+    CLASSPATH_DEFAULT="${HZ_HOME}/lib/*" \
     JAVA_OPTS_DEFAULT="-Djava.net.preferIPv4Stack=true" \
     MIN_HEAP_SIZE="" \
     MAX_HEAP_SIZE="" \
     MANCENTER_URL="" \
+    PROMETHEUS_PORT="" \
+    PROMETHEUS_CONFIG="${HZ_HOME}/jmx_agent_config.yaml" \
     CLASSPATH="" \
     JAVA_OPTS=""
 
 # Expose port
 EXPOSE 5701
 
-COPY *.xml *.sh ${HZ_HOME}/
+COPY *.xml *.sh *.yaml ${HZ_HOME}/
 
 # Install
 RUN echo "Updating Alpine system" \
@@ -35,24 +39,27 @@ RUN echo "Updating Alpine system" \
     && mkdir lib \
     && curl -sf -o ${HZ_HOME}/lib/${HZ_JAR} \
          -L https://repo1.maven.org/maven2/com/hazelcast/hazelcast-all/${HZ_VERSION}/${HZ_JAR} \
-   && echo "Installing JCache" \
-   && curl -sf -o "${HZ_HOME}/lib/${CACHE_API_JAR}" \
+    && echo "Installing JCache" \
+    && curl -sf -o "${HZ_HOME}/lib/${CACHE_API_JAR}" \
          -L "https://repo1.maven.org/maven2/javax/cache/cache-api/${CACHE_API_VERSION}/${CACHE_API_JAR}" \
-   && echo "Installing Hazelcast plugins" \
-   && mvn -f dependency-copy.xml \
+    && echo "Installing Hazelcast plugins" \
+    && mvn -f dependency-copy.xml \
            -Dhazelcast-kubernetes-version=${HZ_KUBE_VERSION} \
            -Dhazelcast-eureka-version=${HZ_EUREKA_VERSION} \
            dependency:copy-dependencies \
-   && echo "Maven clean-up" \
-   && apk del maven \
-   && rm -rf ~/.m2 \
-   && rm -f dependency-copy.xml \
-   && echo "Granting read permission to ${HZ_HOME}" \
-   && chmod -R +r $HZ_HOME \
-   && echo "Setting Pardot ID to 'docker'" \
-   && echo 'hazelcastDownloadId=docker' > hazelcast-download.properties \
-   && echo "Cleaning APK packages" \
-   && rm -rf /var/cache/apk/*
+    && echo "Maven clean-up" \
+    && apk del maven \
+    && rm -rf ~/.m2 \
+    && rm -f dependency-copy.xml \
+    && echo "Installing JMX Prometheus Agent" \
+    && curl -sf -o "${HZ_HOME}/lib/jmx_prometheus_javaagent.jar" \
+         -L "https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/${JMX_PROMETHEUS_AGENT_VERSION}/jmx_prometheus_javaagent-${JMX_PROMETHEUS_AGENT_VERSION}.jar" \
+    && echo "Granting read permission to ${HZ_HOME}" \
+    && chmod -R +r $HZ_HOME \
+    && echo "Setting Pardot ID to 'docker'" \
+    && echo 'hazelcastDownloadId=docker' > hazelcast-download.properties \
+    && echo "Cleaning APK packages" \
+    && rm -rf /var/cache/apk/*
 
 WORKDIR ${HZ_HOME}
 

hazelcast-oss/jmx_agent_config.yaml

@@ -0,0 +1,26 @@
+---
+ssl: false
+#lowercaseOutputName: false
+#lowercaseOutputLabelNames: false
+whitelistObjectNames: ["com.hazelcast:*"]
+rules:
+# HELP com_hazelcast_hzInstance_1_dev_clusterTime Cluster-wide Time (com.hazelcast<instance=_hzInstance_1_dev, name=_hzInstance_1_dev, type=HazelcastInstance><>clusterTime)
+  #com.hazelcast:instance=_hzInstance_1_dev,name=_hzInstance_1_dev,type=HazelcastInstance.PartitionServiceMBean
+- pattern: '<instance=(\w+), name=(\w+), type=(.*)><>(.*):'
+  name: hazelcast_$3_$4
+  #value: $4
+  #valueFactor: 0.001
+  labels:
+    instance: $1
+    name: $2
+    type: $3
+  help: "Hazelcast metric instance=$1 name=$2 type=$3 attribute=$4"
+  attrNameSnakeCase: false
+- pattern: '<name=(\w+), instance=(\w+), type=(\w+.*)'
+  name: hazelcast_$1_$3
+  labels:
+    instance: $2
+    name: $1
+    type: $3
+  help: "Hazelcast metric instance=$1 name=$2 type=$3"
+  attrNameSnakeCase: false

hazelcast-oss/start-hazelcast.sh

@@ -31,6 +31,12 @@ else
   export JAVA_OPTS="${JAVA_OPTS} -Dhazelcast.mancenter.enabled=false"
 fi
 
+if [ -n "${PROMETHEUS_PORT}" ]; then
+  export JAVA_OPTS="${JAVA_OPTS} -javaagent:${HZ_HOME}/lib/jmx_prometheus_javaagent.jar=${PROMETHEUS_PORT}:${PROMETHEUS_CONFIG}"
+else
+  export JAVA_OPTS="${JAVA_OPTS}"
+fi
+
 echo "########################################"
 echo "# JAVA_OPTS=${JAVA_OPTS}"
 echo "# CLASSPATH=${CLASSPATH}"