HBASE-22964 Fix flaky TestClusterRestartFailover and TestClusterRestartFailoverSplitWithoutZk

Author: infraio

hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestClusterRestartFailover.java

@@ -61,54 +61,53 @@ public void test() throws Exception {
     UTIL.waitFor(60000, () -> UTIL.getMiniHBaseCluster().getMaster().isInitialized());
     // wait for all SCPs finished
     UTIL.waitFor(60000, () -> UTIL.getHBaseCluster().getMaster().getProcedures().stream()
-      .noneMatch(p -> p instanceof ServerCrashProcedure));
+        .noneMatch(p -> p instanceof ServerCrashProcedure));
     TableName tableName = TABLES[0];
     ServerName testServer = UTIL.getHBaseCluster().getRegionServer(0).getServerName();
     UTIL.waitFor(30000, () -> getServerStateNode(testServer) != null);
     ServerStateNode serverNode = getServerStateNode(testServer);
     Assert.assertNotNull(serverNode);
     Assert.assertTrue("serverNode should be ONLINE when cluster runs normally",
-      serverNode.isInState(ServerState.ONLINE));
+        serverNode.isInState(ServerState.ONLINE));
     UTIL.createMultiRegionTable(tableName, FAMILY);
     UTIL.waitTableEnabled(tableName);
     Table table = UTIL.getConnection().getTable(tableName);
     for (int i = 0; i < 100; i++) {
       UTIL.loadTable(table, FAMILY);
     }
     List<Integer> ports =
-      UTIL.getHBaseCluster().getMaster().getServerManager().getOnlineServersList().stream()
-        .map(serverName -> serverName.getPort()).collect(Collectors.toList());
+        UTIL.getHBaseCluster().getMaster().getServerManager().getOnlineServersList().stream()
+            .map(serverName -> serverName.getPort()).collect(Collectors.toList());
     LOG.info("Shutting down cluster");
     UTIL.getHBaseCluster().killAll();
     UTIL.getHBaseCluster().waitUntilShutDown();
     LOG.info("Starting cluster the second time");
     UTIL.restartHBaseCluster(3, ports);
     UTIL.waitFor(30000, () -> UTIL.getHBaseCluster().getMaster().isInitialized());
     serverNode = UTIL.getHBaseCluster().getMaster().getAssignmentManager().getRegionStates()
-      .getServerNode(testServer);
-    Assert.assertNotNull("serverNode should not be null when restart whole cluster", serverNode);
-    Assert.assertFalse(serverNode.isInState(ServerState.ONLINE));
-    LOG.info("start to find the procedure of SCP for the severName we choose");
-    UTIL.waitFor(60000,
-      () -> UTIL.getHBaseCluster().getMaster().getProcedures().stream()
-        .anyMatch(procedure -> (procedure instanceof ServerCrashProcedure) &&
-          ((ServerCrashProcedure) procedure).getServerName().equals(testServer)));
-    Assert.assertFalse("serverNode should not be ONLINE during SCP processing",
-      serverNode.isInState(ServerState.ONLINE));
-    LOG.info("start to submit the SCP for the same serverName {} which should fail", testServer);
-    Assert
-      .assertFalse(UTIL.getHBaseCluster().getMaster().getServerManager().expireServer(testServer));
-    Procedure<?> procedure = UTIL.getHBaseCluster().getMaster().getProcedures().stream()
-      .filter(p -> (p instanceof ServerCrashProcedure) &&
-        ((ServerCrashProcedure) p).getServerName().equals(testServer))
-      .findAny().get();
-    UTIL.waitFor(60000, () -> procedure.isFinished());
-    LOG.info("even when the SCP is finished, the duplicate SCP should not be scheduled for {}",
-      testServer);
-    Assert
-      .assertFalse(UTIL.getHBaseCluster().getMaster().getServerManager().expireServer(testServer));
-    serverNode = UTIL.getHBaseCluster().getMaster().getAssignmentManager().getRegionStates()
-      .getServerNode(testServer);
-    Assert.assertNull("serverNode should be deleted after SCP finished", serverNode);
+        .getServerNode(testServer);
+    if (serverNode != null) {
+      Assert.assertFalse(serverNode.isInState(ServerState.ONLINE));
+      LOG.info("start to find the procedure of SCP for the severName we choose");
+      Procedure<?> procedure = UTIL.getHBaseCluster().getMaster().getProcedures().stream().filter(
+          p -> (p instanceof ServerCrashProcedure) &&
+              ((ServerCrashProcedure) p).getServerName().equals(testServer)).findAny().get();
+      if (procedure != null) {
+        Assert.assertFalse("serverNode should not be ONLINE during SCP processing",
+            serverNode.isInState(ServerState.ONLINE));
+        LOG.info("start to submit the SCP for the same serverName {} which should fail",
+            testServer);
+        Assert.assertFalse(
+            UTIL.getHBaseCluster().getMaster().getServerManager().expireServer(testServer));
+        UTIL.waitFor(60000, () -> procedure.isFinished());
+        LOG.info("even when the SCP is finished, the duplicate SCP should not be scheduled for {}",
+            testServer);
+        Assert.assertFalse(
+            UTIL.getHBaseCluster().getMaster().getServerManager().expireServer(testServer));
+      }
+      serverNode = UTIL.getHBaseCluster().getMaster().getAssignmentManager().getRegionStates()
+          .getServerNode(testServer);
+      Assert.assertNull("serverNode should be deleted after SCP finished", serverNode);
+    }
   }
 }