HBASE-23269 Hbase crashed due to two versions of regionservers when rolling upgrading

hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java

@@ -274,6 +274,21 @@ public ServerName randomAssignment(HRegionInfo region,
     return this.internalBalancer.randomAssignment(region, filteredServers);
   }
 
+  @Override
+  public  Set<Address> getServersInDefaultOrGroup(HRegionInfo region) throws HBaseIOException {
+    try{
+      String groupName = infoManager.getRSGroupOfTable(region.getTable());
+      if (groupName == null) {
+        groupName = RSGroupInfo.DEFAULT_GROUP;
+      }
+      RSGroupInfo info = infoManager.getRSGroup(groupName);
+      Set<Address> serversInGroup = info.getServers();
+      return serversInGroup;
+    } catch (Exception e) {
+      throw new HBaseIOException("Failed to get servers ", e);
+    }
+  }
+
   private void generateGroupMaps(
     List<HRegionInfo> regions,
     List<ServerName> servers,

hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java

@@ -89,6 +89,7 @@
 import org.apache.hadoop.hbase.master.handler.DisableTableHandler;
 import org.apache.hadoop.hbase.master.handler.EnableTableHandler;
 import org.apache.hadoop.hbase.master.handler.OpenedRegionHandler;
+import org.apache.hadoop.hbase.net.Address;
 import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionStateTransition;
 import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionStateTransition.TransitionCode;
 import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos;
@@ -2636,10 +2637,28 @@ public void run() {
             for (ServerName server : getExcludedServersForSystemTable()) {
               regionsShouldMove.addAll(getCarryingSystemTables(server));
             }
+            Set<Address> remainservers = new HashSet<>();
             if (!regionsShouldMove.isEmpty()) {
               List<RegionPlan> plans = new ArrayList<>();
+              List<ServerName> names = getExcludedServersForSystemTable();
               for (HRegionInfo regionInfo : regionsShouldMove) {
                 RegionPlan plan = getRegionPlan(regionInfo, true);
+                Set<Address> serverbygroup = balancer.getServersInDefaultOrGroup(regionInfo);
+                Set<Address> excludedAddresses = getAddressFromServerName(names);
+                if (serverbygroup != null) {
+                  for (Address address : serverbygroup) {
+                    if (!excludedAddresses.contains(address) &&
+                            getAddressFromServerName(serverManager.
+                                    getOnlineServersList()).contains(address)) {
+                      remainservers.add(address);
+                    }
+                  }
+                  if (remainservers.size() == 0) {
+                    LOG.info("The region("+regionInfo.getRegionNameAsString()+")" +
+                            "do not need to move to RS with the highest version ");
+                    continue;
+                  }
+                }
                 if (regionInfo.isMetaRegion()) {
                   // Must move meta region first.
                   balance(plan);
@@ -2659,6 +2678,16 @@ public void run() {
     }).start();
   }
 
+  public HashSet<Address> getAddressFromServerName(List<ServerName> servernames) {
+    HashSet<Address> addresses = new HashSet<>();
+    if (servernames == null) {
+      return addresses;
+    }
+    for (ServerName servername : servernames) {
+      addresses.add(servername.getAddress());
+    }
+    return addresses;
+  }
 
   /**
    * Unassigns the specified region.

hbase-server/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java

@@ -20,6 +20,7 @@
 
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
 import org.apache.hadoop.hbase.conf.ConfigurationObserver;
@@ -31,6 +32,7 @@
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.Stoppable;
 import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.net.Address;
 
 /**
  * Makes decisions about the placement and movement of Regions across
@@ -127,6 +129,14 @@ ServerName randomAssignment(
     HRegionInfo regionInfo, List<ServerName> servers
   ) throws HBaseIOException;
 
+  /**
+   * Get all the servers of this region's rs_group，and now only used in RSGroupBasedLoadBalancer
+   * @param region Get the servers of region's table
+   * @return Set of Address
+   */
+  Set<Address> getServersInDefaultOrGroup(
+          HRegionInfo region) throws HBaseIOException;
+
   /**
    * Initialize the load balancer. Must be called after setters.
    * @throws HBaseIOException

hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/FavoredNodeLoadBalancer.java

@@ -19,10 +19,7 @@
 package org.apache.hadoop.hbase.master.balancer;
 
 import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -40,6 +37,7 @@
 import org.apache.hadoop.hbase.master.ServerManager;
 import org.apache.hadoop.hbase.master.SnapshotOfRegionAssignmentFromMeta;
 import org.apache.hadoop.hbase.master.balancer.FavoredNodesPlan.Position;
+import org.apache.hadoop.hbase.net.Address;
 import org.apache.hadoop.hbase.util.Pair;
 
 /**
@@ -225,6 +223,11 @@ public ServerName randomAssignment(HRegionInfo regionInfo, List<ServerName> serv
     }
   }
 
+  @Override
+  public Set<Address> getServersInDefaultOrGroup(HRegionInfo region) throws HBaseIOException {
+    return null;
+  }
+
   private Pair<Map<ServerName, List<HRegionInfo>>, List<HRegionInfo>>
   segregateRegionsAndAssignRegionsWithFavoredNodes(List<HRegionInfo> regions,
       List<ServerName> availableServers) {

hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/SimpleLoadBalancer.java

@@ -25,6 +25,7 @@
 import java.util.Map;
 import java.util.NavigableMap;
 import java.util.Random;
+import java.util.Set;
 import java.util.TreeMap;
 
 import org.apache.commons.logging.Log;
@@ -36,6 +37,7 @@
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.master.RegionPlan;
+import org.apache.hadoop.hbase.net.Address;
 
 import com.google.common.collect.MinMaxPriorityQueue;
 
@@ -437,4 +439,9 @@ public List<RegionPlan> balanceCluster(TableName tableName,
       Map<ServerName, List<HRegionInfo>> clusterState) throws HBaseIOException {
     return balanceCluster(clusterState);
   }
+
+  @Override
+  public Set<Address> getServersInDefaultOrGroup(HRegionInfo region) throws HBaseIOException {
+    return null;
+  }
 }

hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java

@@ -30,13 +30,16 @@
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Random;
+import java.util.Set;
+
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.ClusterStatus;
 import org.apache.hadoop.hbase.HBaseInterfaceAudience;
+import org.apache.hadoop.hbase.HBaseIOException;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.RegionLoad;
@@ -53,6 +56,7 @@
 import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.Cluster.SwapRegionsAction;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+import org.apache.hadoop.hbase.net.Address;
 
 import com.google.common.base.Optional;
 
@@ -442,6 +446,11 @@ public synchronized List<RegionPlan> balanceCluster(Map<ServerName,
     return null;
   }
 
+  @Override
+  public Set<Address> getServersInDefaultOrGroup(HRegionInfo region) throws HBaseIOException {
+    return null;
+  }
+
   /**
    * update costs to JMX
    */

hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestBaseLoadBalancer.java

@@ -38,6 +38,7 @@
 import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HBaseIOException;
 import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.net.Address;
 import org.apache.hadoop.hbase.testclassification.MediumTests;
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.TableName;
@@ -100,6 +101,10 @@ public static void beforeAllTests() throws Exception {
   }
 
   public static class MockBalancer extends BaseLoadBalancer {
+    @Override
+    public Set<Address> getServersInDefaultOrGroup(HRegionInfo region) throws HBaseIOException {
+      return null;
+    }
 
     @Override
     public List<RegionPlan> balanceCluster(Map<ServerName, List<HRegionInfo>> clusterState) {