HBASE-29724: Backport "HBASE-25334: TestRSGroupsFallback.testFallback is flaky" into branch-2 and branch-2.6 (#7472)

Signed-off-by: Tak Lon (Stephen) Wu <taklwu@apache.org>

Author: kgeisz

hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsFallback.java

@@ -101,16 +101,13 @@ public void testFallback() throws Exception {
     assertRegionsInGroup(tableName, FALLBACK_GROUP);
 
     // add a new server to default group, regions move to default group
-    JVMClusterUtil.RegionServerThread t =
-      TEST_UTIL.getMiniHBaseCluster().startRegionServerAndWait(60000);
-    Address startRSAddress = t.getRegionServer().getServerName().getAddress();
-    TEST_UTIL.waitFor(3000,
-      () -> rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).containsServer(startRSAddress));
+    TEST_UTIL.getMiniHBaseCluster().startRegionServerAndWait(60000);
     assertTrue(master.balance().isBalancerRan());
     assertRegionsInGroup(tableName, RSGroupInfo.DEFAULT_GROUP);
 
     // add a new server to test group, regions move back
-    t = TEST_UTIL.getMiniHBaseCluster().startRegionServerAndWait(60000);
+    JVMClusterUtil.RegionServerThread t =
+      TEST_UTIL.getMiniHBaseCluster().startRegionServerAndWait(60000);
     rsGroupAdmin.moveServers(
       Collections.singleton(t.getRegionServer().getServerName().getAddress()), groupName);
     assertTrue(master.balance().isBalancerRan());
@@ -119,11 +116,11 @@ public void testFallback() throws Exception {
     TEST_UTIL.deleteTable(tableName);
   }
 
-  private void assertRegionsInGroup(TableName tableName, String group) throws IOException {
+  private void assertRegionsInGroup(TableName table, String group) throws IOException {
     ProcedureTestingUtility
       .waitAllProcedures(TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterProcedureExecutor());
     RSGroupInfo groupInfo = rsGroupAdmin.getRSGroupInfo(group);
-    master.getAssignmentManager().getRegionStates().getRegionsOfTable(tableName).forEach(region -> {
+    master.getAssignmentManager().getRegionStates().getRegionsOfTable(table).forEach(region -> {
       Address regionOnServer = master.getAssignmentManager().getRegionStates()
         .getRegionAssignments().get(region).getAddress();
       assertTrue(groupInfo.getServers().contains(regionOnServer));

hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java

@@ -549,7 +549,8 @@ public DeadServer getDeadServers() {
    * @return true if any RS are being processed as dead, false if not
    */
   public boolean areDeadServersInProgress() throws IOException {
-    return master.getProcedures().stream().anyMatch(p -> p instanceof ServerCrashProcedure);
+    return master.getProcedures().stream()
+      .anyMatch(p -> !p.isFinished() && p instanceof ServerCrashProcedure);
   }
 
   void letRegionServersShutdown() {

hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDeadServer.java

@@ -20,7 +20,6 @@
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
-import java.io.IOException;
 import java.util.List;
 import java.util.Set;
 import org.apache.hadoop.hbase.HBaseClassTestRule;
@@ -93,15 +92,19 @@ public void testIsDead() {
   }
 
   @Test
-  public void testCrashProcedureReplay() throws IOException {
+  public void testCrashProcedureReplay() throws Exception {
     HMaster master = TEST_UTIL.getHBaseCluster().getMaster();
     final ProcedureExecutor<MasterProcedureEnv> pExecutor = master.getMasterProcedureExecutor();
     ServerCrashProcedure proc =
       new ServerCrashProcedure(pExecutor.getEnvironment(), hostname123, false, false);
 
+    pExecutor.stop();
     ProcedureTestingUtility.submitAndWait(pExecutor, proc);
-
     assertTrue(master.getServerManager().areDeadServersInProgress());
+
+    ProcedureTestingUtility.restart(pExecutor);
+    ProcedureTestingUtility.waitProcedure(pExecutor, proc);
+    assertFalse(master.getServerManager().areDeadServersInProgress());
   }
 
   @Test