Add Histogram for GC pauses (lavalink-devs#106)

* Make lavaplayer gc warnings optional * [Metrics] Add Histogram for GC pauses
freyacoded · Jun 3, 2018 · e2b28c4 · e2b28c4
1 parent 9001c25
commit e2b28c4
Show file tree

Hide file tree

Showing 5 changed files with 70 additions and 9 deletions.
diff --git a/LavalinkServer/application.yml.example b/LavalinkServer/application.yml.example
@@ -19,6 +19,7 @@ lavalink:
     sentryDsn: ""
     bufferDurationMs: 400
     youtubePlaylistLoadLimit: 600
+    gc-warnings: true
 
 metrics:
   prometheus:

diff --git a/LavalinkServer/src/main/java/lavalink/server/config/AudioPlayerConfiguration.java b/LavalinkServer/src/main/java/lavalink/server/config/AudioPlayerConfiguration.java
@@ -23,7 +23,10 @@ public class AudioPlayerConfiguration {
     public AudioPlayerManager audioPlayerManager(AudioSourcesConfig sources, ServerConfig serverConfig) {
 
         AudioPlayerManager audioPlayerManager = new DefaultAudioPlayerManager();
-        audioPlayerManager.enableGcMonitoring();
+
+        if (serverConfig.isGcWarnings()) {
+            audioPlayerManager.enableGcMonitoring();
+        }
 
         if (sources.isYoutube()) {
             YoutubeAudioSourceManager youtube = new YoutubeAudioSourceManager();

diff --git a/LavalinkServer/src/main/java/lavalink/server/config/ServerConfig.java b/LavalinkServer/src/main/java/lavalink/server/config/ServerConfig.java
@@ -32,6 +32,12 @@
 public class ServerConfig {
 
     private String password;
+    private String sentryDsn = "";
+    @Nullable
+    private Integer bufferDurationMs;
+    @Nullable
+    private Integer youtubePlaylistLoadLimit;
+    private boolean gcWarnings = true;
 
     public String getPassword() {
         return password;
@@ -41,8 +47,6 @@ public void setPassword(String password) {
         this.password = password;
     }
 
-    private String sentryDsn = "";
-
     public String getSentryDsn() {
         return sentryDsn;
     }
@@ -51,9 +55,6 @@ public void setSentryDsn(String sentryDsn) {
         this.sentryDsn = sentryDsn;
     }
 
-    @Nullable
-    public Integer bufferDurationMs;
-
     @Nullable
     public Integer getBufferDurationMs() {
         return bufferDurationMs;
@@ -63,9 +64,6 @@ public void setBufferDurationMs(@Nullable Integer bufferDurationMs) {
         this.bufferDurationMs = bufferDurationMs;
     }
 
-    @Nullable
-    private Integer youtubePlaylistLoadLimit;
-
     @Nullable
     public Integer getYoutubePlaylistLoadLimit() {
         return youtubePlaylistLoadLimit;
@@ -74,4 +72,12 @@ public Integer getYoutubePlaylistLoadLimit() {
     public void setYoutubePlaylistLoadLimit(@Nullable Integer youtubePlaylistLoadLimit) {
         this.youtubePlaylistLoadLimit = youtubePlaylistLoadLimit;
     }
+
+    public boolean isGcWarnings() {
+        return gcWarnings;
+    }
+
+    public void setGcWarnings(boolean gcWarnings) {
+        this.gcWarnings = gcWarnings;
+    }
 }
diff --git a/LavalinkServer/src/main/java/lavalink/server/metrics/GcNotificationListener.java b/LavalinkServer/src/main/java/lavalink/server/metrics/GcNotificationListener.java
@@ -0,0 +1,39 @@
+package lavalink.server.metrics;
+
+import com.sun.management.GarbageCollectionNotificationInfo;
+import com.sun.management.GcInfo;
+import io.prometheus.client.Collector;
+import io.prometheus.client.Histogram;
+
+import javax.management.Notification;
+import javax.management.NotificationListener;
+import javax.management.openmbean.CompositeData;
+
+import static com.sun.management.GarbageCollectionNotificationInfo.GARBAGE_COLLECTION_NOTIFICATION;
+import static com.sun.management.GarbageCollectionNotificationInfo.from;
+
+/**
+ * Created by napster on 21.05.18.
+ * <p>
+ * General idea taken from {@link com.sedmelluq.discord.lavaplayer.tools.GarbageCollectionMonitor}, thanks!
+ */
+public class GcNotificationListener implements NotificationListener {
+
+    private final Histogram gcPauses = Histogram.build()
+            .name("lavalink_gc_pauses_seconds")
+            .help("Garbage collection pauses by buckets")
+            .buckets(25, 50, 100, 200, 400, 800, 1600)
+            .register();
+
+    @Override
+    public void handleNotification(Notification notification, Object handback) {
+        if (GARBAGE_COLLECTION_NOTIFICATION.equals(notification.getType())) {
+            GarbageCollectionNotificationInfo notificationInfo = from((CompositeData) notification.getUserData());
+            GcInfo info = notificationInfo.getGcInfo();
+
+            if (info != null && !"No GC".equals(notificationInfo.getGcCause())) {
+                gcPauses.observe(info.getDuration() / Collector.MILLISECONDS_PER_SECOND);
+            }
+        }
+    }
+}
diff --git a/LavalinkServer/src/main/java/lavalink/server/metrics/PrometheusMetrics.java b/LavalinkServer/src/main/java/lavalink/server/metrics/PrometheusMetrics.java
@@ -8,6 +8,10 @@
 import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
 import org.springframework.stereotype.Component;
 
+import javax.management.NotificationEmitter;
+import java.lang.management.GarbageCollectorMXBean;
+import java.lang.management.ManagementFactory;
+
 /**
  * Created by napster on 08.05.18.
  */
@@ -30,6 +34,14 @@ public PrometheusMetrics() {
         //jvm (hotspot) metrics
         DefaultExports.initialize();
 
+        //gc pause buckets
+        final GcNotificationListener gcNotificationListener = new GcNotificationListener();
+        for (GarbageCollectorMXBean gcBean : ManagementFactory.getGarbageCollectorMXBeans()) {
+            if (gcBean instanceof NotificationEmitter) {
+                ((NotificationEmitter) gcBean).addNotificationListener(gcNotificationListener, null, gcBean);
+            }
+        }
+
         log.info("Prometheus metrics set up");
     }
 }