HBASE-29724: Backport "HBASE-25334: TestRSGroupsFallback.testFallback is flaky" into branch-2 and branch-2.6

Author: sunhelly

hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsFallback.java

@@ -33,7 +33,6 @@
 import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
 import org.apache.hadoop.hbase.testclassification.MediumTests;
 import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.JVMClusterUtil;
 import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Before;
@@ -101,32 +100,28 @@ public void testFallback() throws Exception {
     assertRegionsInGroup(tableName, FALLBACK_GROUP);
 
     // add a new server to default group, regions move to default group
-    JVMClusterUtil.RegionServerThread t =
-      TEST_UTIL.getMiniHBaseCluster().startRegionServerAndWait(60000);
-    Address startRSAddress = t.getRegionServer().getServerName().getAddress();
-    TEST_UTIL.waitFor(3000,
-      () -> rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).containsServer(startRSAddress));
-    assertTrue(master.balance().isBalancerRan());
+    TEST_UTIL.getMiniHBaseCluster().startRegionServerAndWait(60000);
+    assertTrue(MASTER.balance());
     assertRegionsInGroup(tableName, RSGroupInfo.DEFAULT_GROUP);
 
     // add a new server to test group, regions move back
     t = TEST_UTIL.getMiniHBaseCluster().startRegionServerAndWait(60000);
     rsGroupAdmin.moveServers(
       Collections.singleton(t.getRegionServer().getServerName().getAddress()), groupName);
-    assertTrue(master.balance().isBalancerRan());
+    assertTrue(MASTER.balance());
     assertRegionsInGroup(tableName, groupName);
 
     TEST_UTIL.deleteTable(tableName);
   }
 
-  private void assertRegionsInGroup(TableName tableName, String group) throws IOException {
+  private void assertRegionsInGroup(TableName table, String group) throws IOException {
     ProcedureTestingUtility
       .waitAllProcedures(TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterProcedureExecutor());
-    RSGroupInfo groupInfo = rsGroupAdmin.getRSGroupInfo(group);
-    master.getAssignmentManager().getRegionStates().getRegionsOfTable(tableName).forEach(region -> {
-      Address regionOnServer = master.getAssignmentManager().getRegionStates()
+    RSGroupInfo rsGroup = ADMIN.getRSGroup(group);
+    MASTER.getAssignmentManager().getRegionStates().getRegionsOfTable(table).forEach(region -> {
+      Address regionOnServer = MASTER.getAssignmentManager().getRegionStates()
         .getRegionAssignments().get(region).getAddress();
-      assertTrue(groupInfo.getServers().contains(regionOnServer));
+      assertTrue(rsGroup.getServers().contains(regionOnServer));
     });
   }
 

hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java

@@ -549,7 +549,8 @@ public DeadServer getDeadServers() {
    * @return true if any RS are being processed as dead, false if not
    */
   public boolean areDeadServersInProgress() throws IOException {
-    return master.getProcedures().stream().anyMatch(p -> p instanceof ServerCrashProcedure);
+    return master.getProcedures().stream()
+      .anyMatch(p -> !p.isFinished() && p instanceof ServerCrashProcedure);
   }
 
   void letRegionServersShutdown() {

hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDeadServer.java

@@ -20,7 +20,6 @@
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
-import java.io.IOException;
 import java.util.List;
 import java.util.Set;
 import org.apache.hadoop.hbase.HBaseClassTestRule;
@@ -93,15 +92,19 @@ public void testIsDead() {
   }
 
   @Test
-  public void testCrashProcedureReplay() throws IOException {
+  public void testCrashProcedureReplay() throws Exception {
     HMaster master = TEST_UTIL.getHBaseCluster().getMaster();
     final ProcedureExecutor<MasterProcedureEnv> pExecutor = master.getMasterProcedureExecutor();
     ServerCrashProcedure proc =
       new ServerCrashProcedure(pExecutor.getEnvironment(), hostname123, false, false);
 
+    pExecutor.stop();
     ProcedureTestingUtility.submitAndWait(pExecutor, proc);
-
     assertTrue(master.getServerManager().areDeadServersInProgress());
+
+    ProcedureTestingUtility.restart(pExecutor);
+    ProcedureTestingUtility.waitProcedure(pExecutor, proc);
+    assertFalse(master.getServerManager().areDeadServersInProgress());
   }
 
   @Test