GEODE-8977: change ThreadMonitor to reduce how long it does a "stop the world" ThreadDump vm op (#7751)

Now uses a cheaper getThreadInfo that does not get lock info by default and calls getThreadInfo for each stuck thread. These are the defaults because they have the shortest time do the the VM ThreadDump operation.
To get locks set the system property "gemfire.threadmonitor.showLocks" to "true".
To get ThreadInfo on all stuck threads with a single call set the system property "gemfire.threadmonitor.batchCalls" to "true".

(cherry picked from commit 3df1e76)

Author: dschneider-pivotal

geode-core/src/main/java/org/apache/geode/internal/monitoring/ThreadsMonitoringProcess.java

@@ -17,6 +17,7 @@
 
 import java.lang.management.ManagementFactory;
 import java.lang.management.ThreadInfo;
+import java.lang.management.ThreadMXBean;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
@@ -119,30 +120,87 @@ private int checkForStuckThreads(Collection<AbstractExecutor> executors, long cu
     return result;
   }
 
-  @VisibleForTesting
-  public static Map<Long, ThreadInfo> createThreadInfoMap(Set<Long> stuckThreadIds) {
-    /*
-     * NOTE: at least some implementations of getThreadInfo(long[], boolean, boolean)
-     * will core dump if the long array contains a duplicate value.
-     * That is why stuckThreadIds is a Set instead of a List.
-     */
+  /**
+   * If set to true, then the JVM will be asked for what locks a thread holds.
+   * This is extra expensive to ask for on some JVMs so be careful setting this to true.
+   */
+  private static final boolean SHOW_LOCKS = Boolean.getBoolean("gemfire.threadmonitor.showLocks");
+  /**
+   * If set to true, then the JVM will be asked for all potential stuck threads with one call.
+   * Since getThreadInfo on many JVMs, stops ALL threads from running, and since getting info
+   * on multiple threads with one call is additional work, setting this can cause an extra long
+   * stop the world that can then cause other problems (like a forced disconnect).
+   * So be careful setting this to true.
+   */
+  private static final boolean BATCH_CALLS = Boolean.getBoolean("gemfire.threadmonitor.batchCalls");
+
+  private static Map<Long, ThreadInfo> createThreadInfoMap(Set<Long> stuckThreadIds) {
+    return createThreadInfoMap(stuckThreadIds, SHOW_LOCKS, BATCH_CALLS);
+  }
+
+  public static Map<Long, ThreadInfo> createThreadInfoMap(Set<Long> stuckThreadIds,
+      boolean showLocks, boolean batchCalls) {
+    return createThreadInfoMap(ManagementFactory.getThreadMXBean(), stuckThreadIds, showLocks,
+        batchCalls);
+  }
+
+  static Map<Long, ThreadInfo> createThreadInfoMap(ThreadMXBean threadMXBean,
+      Set<Long> stuckThreadIds,
+      final boolean showLocks, final boolean batchCalls) {
     if (stuckThreadIds.isEmpty()) {
       return Collections.emptyMap();
     }
+    logger.info(
+        "Obtaining ThreadInfo for {} threads. Configuration: showLocks={} batchCalls={}. This is an expensive operation for the JVM and on most JVMs causes all threads to be paused.",
+        stuckThreadIds.size(), showLocks, batchCalls);
+    Map<Long, ThreadInfo> result = new HashMap<>();
+    if (batchCalls) {
+      createThreadInfoMapUsingSingleCall(threadMXBean, stuckThreadIds, showLocks, result);
+    } else {
+      for (long id : stuckThreadIds) {
+        ThreadInfo threadInfo = createThreadInfoForSingleThread(threadMXBean, showLocks, id);
+        if (threadInfo != null) {
+          result.put(threadInfo.getThreadId(), threadInfo);
+        }
+      }
+    }
+    logger.info("finished obtaining ThreadInfo");
+    return result;
+  }
+
+  private static ThreadInfo createThreadInfoForSingleThread(
+      ThreadMXBean threadMXBean, boolean showLocks, long id) {
+    ThreadInfo threadInfo;
+    if (showLocks) {
+      ThreadInfo[] threadInfos =
+          threadMXBean.getThreadInfo(new long[] {id}, true, true);
+      threadInfo = threadInfos[0];
+    } else {
+      threadInfo = threadMXBean.getThreadInfo(id, Integer.MAX_VALUE);
+    }
+    return threadInfo;
+  }
+
+  private static void createThreadInfoMapUsingSingleCall(
+      ThreadMXBean threadMXBean, Set<Long> stuckThreadIds, boolean showLocks,
+      Map<Long, ThreadInfo> result) {
     long[] ids = new long[stuckThreadIds.size()];
     int idx = 0;
     for (long id : stuckThreadIds) {
       ids[idx] = id;
       idx++;
     }
-    ThreadInfo[] threadInfos = ManagementFactory.getThreadMXBean().getThreadInfo(ids, true, true);
-    Map<Long, ThreadInfo> result = new HashMap<>();
+    /*
+     * NOTE: at least some implementations of getThreadInfo(long[], boolean, boolean)
+     * will core dump if the long array contains a duplicate value.
+     * That is why stuckThreadIds is a Set instead of a List.
+     */
+    ThreadInfo[] threadInfos = threadMXBean.getThreadInfo(ids, showLocks, showLocks);
     for (ThreadInfo threadInfo : threadInfos) {
       if (threadInfo != null) {
         result.put(threadInfo.getThreadId(), threadInfo);
       }
     }
-    return result;
   }
 
   private void addLockOwnerThreadId(Set<Long> stuckThreadIds, long threadId) {

geode-core/src/test/java/org/apache/geode/internal/monitoring/ThreadsMonitoringImplJUnitTest.java renamed to geode-core/src/test/java/org/apache/geode/internal/monitoring/ThreadsMonitoringImplTest.java

@@ -15,37 +15,35 @@
 package org.apache.geode.internal.monitoring;
 
 import static org.assertj.core.api.Assertions.assertThat;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.never;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Test;
 
 import org.apache.geode.internal.monitoring.ThreadsMonitoring.Mode;
 import org.apache.geode.internal.monitoring.executor.AbstractExecutor;
 import org.apache.geode.internal.monitoring.executor.FunctionExecutionPooledExecutorGroup;
+import org.apache.geode.internal.monitoring.executor.PooledExecutorGroup;
 
 /**
  * Contains simple tests for the {@link org.apache.geode.internal.monitoring.ThreadsMonitoringImpl}.
  *
  * @since Geode 1.5
  */
-public class ThreadsMonitoringImplJUnitTest {
+public class ThreadsMonitoringImplTest {
+  private static final int TIME_LIMIT_MILLIS = 1000;
 
   private ThreadsMonitoringImpl threadsMonitoringImpl;
 
-  @Before
-  public void before() {
-    threadsMonitoringImpl = new ThreadsMonitoringImpl(null, 100000, 0, false);
-  }
-
-  @After
+  @AfterEach
   public void after() {
     threadsMonitoringImpl.close();
   }
@@ -55,6 +53,7 @@ public void after() {
    */
   @Test
   public void testStartMonitor() {
+    threadsMonitoringImpl = new ThreadsMonitoringImpl(null, 100000, 0, false);
     assertTrue(threadsMonitoringImpl.startMonitor(Mode.FunctionExecutor));
     assertTrue(threadsMonitoringImpl.startMonitor(Mode.PooledExecutor));
     assertTrue(threadsMonitoringImpl.startMonitor(Mode.SerialQueuedExecutor));
@@ -67,6 +66,7 @@ public void testStartMonitor() {
 
   @Test
   public void verifyMonitorLifeCycle() {
+    threadsMonitoringImpl = new ThreadsMonitoringImpl(null, 100000, 0, false);
     assertFalse(threadsMonitoringImpl.isMonitoring());
     threadsMonitoringImpl.startMonitor(Mode.FunctionExecutor);
     assertTrue(threadsMonitoringImpl.isMonitoring());
@@ -76,6 +76,7 @@ public void verifyMonitorLifeCycle() {
 
   @Test
   public void verifyExecutorMonitoringLifeCycle() {
+    threadsMonitoringImpl = new ThreadsMonitoringImpl(null, 100000, 0, false);
     AbstractExecutor executor =
         threadsMonitoringImpl.createAbstractExecutor(Mode.P2PReaderExecutor);
     assertThat(threadsMonitoringImpl.isMonitoring(executor)).isFalse();
@@ -95,24 +96,28 @@ public void verifyExecutorMonitoringLifeCycle() {
 
   @Test
   public void createAbstractExecutorIsAssociatedWithCallingThread() {
+    threadsMonitoringImpl = new ThreadsMonitoringImpl(null, 100000, 0, false);
     AbstractExecutor executor = threadsMonitoringImpl.createAbstractExecutor(Mode.FunctionExecutor);
     assertThat(executor.getThreadID()).isEqualTo(Thread.currentThread().getId());
   }
 
   @Test
   public void createAbstractExecutorDoesNotSetStartTime() {
+    threadsMonitoringImpl = new ThreadsMonitoringImpl(null, 100000, 0, false);
     AbstractExecutor executor = threadsMonitoringImpl.createAbstractExecutor(Mode.FunctionExecutor);
     assertThat(executor.getStartTime()).isEqualTo(0);
   }
 
   @Test
   public void createAbstractExecutorSetsNumIterationsStuckToZero() {
+    threadsMonitoringImpl = new ThreadsMonitoringImpl(null, 100000, 0, false);
     AbstractExecutor executor = threadsMonitoringImpl.createAbstractExecutor(Mode.FunctionExecutor);
     assertThat(executor.getNumIterationsStuck()).isEqualTo((short) 0);
   }
 
   @Test
   public void createAbstractExecutorSetsExpectedGroupName() {
+    threadsMonitoringImpl = new ThreadsMonitoringImpl(null, 100000, 0, false);
     AbstractExecutor executor = threadsMonitoringImpl.createAbstractExecutor(Mode.FunctionExecutor);
     assertThat(executor.getGroupName()).isEqualTo(FunctionExecutionPooledExecutorGroup.GROUPNAME);
   }
@@ -122,16 +127,17 @@ public void createAbstractExecutorSetsExpectedGroupName() {
    */
   @Test
   public void testClosure() {
-    ThreadsMonitoringImpl liveMonitor = new ThreadsMonitoringImpl(null, 100000, 0, true);
-    assertTrue(liveMonitor.getThreadsMonitoringProcess() != null);
-    assertFalse(liveMonitor.isClosed());
-    liveMonitor.close();
-    assertTrue(liveMonitor.isClosed());
-    assertFalse(liveMonitor.getThreadsMonitoringProcess() != null);
+    threadsMonitoringImpl = new ThreadsMonitoringImpl(null, 100000, 0, true);
+    assertNotNull(threadsMonitoringImpl.getThreadsMonitoringProcess());
+    assertFalse(threadsMonitoringImpl.isClosed());
+    threadsMonitoringImpl.close();
+    assertTrue(threadsMonitoringImpl.isClosed());
+    assertNull(threadsMonitoringImpl.getThreadsMonitoringProcess());
   }
 
   @Test
   public void updateThreadStatusCallsSetStartTime() {
+    threadsMonitoringImpl = new ThreadsMonitoringImpl(null, 100000, 0, false);
     AbstractExecutor executor = mock(AbstractExecutor.class);
     long threadId = Thread.currentThread().getId();
 
@@ -142,8 +148,52 @@ public void updateThreadStatusCallsSetStartTime() {
 
   @Test
   public void updateThreadStatusWithoutExecutorInMapDoesNotCallSetStartTime() {
+    threadsMonitoringImpl = new ThreadsMonitoringImpl(null, 100000, 0, false);
     AbstractExecutor executor = mock(AbstractExecutor.class);
     threadsMonitoringImpl.updateThreadStatus();
     verify(executor, never()).setStartTime(any(Long.class));
   }
+
+  /**
+   * Tests that indeed thread is considered stuck when it should
+   */
+  @Test
+  public void testThreadIsStuck() {
+    threadsMonitoringImpl = new ThreadsMonitoringImpl(null, 100000, TIME_LIMIT_MILLIS);
+
+    final long threadID = 123456;
+
+    AbstractExecutor absExtgroup = new PooledExecutorGroup();
+    absExtgroup.setStartTime(absExtgroup.getStartTime() - TIME_LIMIT_MILLIS - 1);
+
+    threadsMonitoringImpl.getMonitorMap().put(threadID, absExtgroup);
+
+    assertTrue(threadsMonitoringImpl.getThreadsMonitoringProcess().mapValidation());
+  }
+
+  @Test
+  public void monitorHandlesDefunctThread() {
+    threadsMonitoringImpl = new ThreadsMonitoringImpl(null, 100000, TIME_LIMIT_MILLIS);
+    final long threadID = Long.MAX_VALUE;
+
+    AbstractExecutor absExtgroup = new PooledExecutorGroup(threadID);
+    absExtgroup.setStartTime(absExtgroup.getStartTime() - TIME_LIMIT_MILLIS - 1);
+
+    threadsMonitoringImpl.getMonitorMap().put(threadID, absExtgroup);
+
+    assertTrue(threadsMonitoringImpl.getThreadsMonitoringProcess().mapValidation());
+  }
+
+  /**
+   * Tests that indeed thread is NOT considered stuck when it shouldn't
+   */
+  @Test
+  public void testThreadIsNotStuck() {
+    threadsMonitoringImpl = new ThreadsMonitoringImpl(null, 100000, TIME_LIMIT_MILLIS);
+
+    threadsMonitoringImpl.startMonitor(Mode.PooledExecutor);
+
+    assertFalse(threadsMonitoringImpl.getThreadsMonitoringProcess().mapValidation());
+  }
+
 }