HBASE-24530 Introduce a split policy similar with SteppingSplitPolicy but count all store size

Author: bsglz

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/IncreasingToUpperBoundRegionSplitPolicy.java

@@ -70,31 +70,35 @@ protected void configureForRegion(HRegion region) {
 
   @Override
   protected boolean shouldSplit() {
-    boolean foundABigStore = false;
+    // If any of the stores is unable to split (eg they contain reference files)
+    // then don't split
+    for (HStore store : region.getStores()) {
+      if (!store.canSplit()) {
+        return false;
+      }
+    }
     // Get count of regions that have the same common table as this.region
     int tableRegionsCount = getCountOfCommonTableRegions();
     // Get size to check
     long sizeToCheck = getSizeToCheck(tableRegionsCount);
+    return isExceedSize(tableRegionsCount, sizeToCheck);
+  }
 
+  /**
+   * @return true if any store's size exceed the sizeToCheck
+   */
+  protected boolean isExceedSize(int tableRegionsCount, long sizeToCheck) {
     for (HStore store : region.getStores()) {
-      // If any of the stores is unable to split (eg they contain reference files)
-      // then don't split
-      if (!store.canSplit()) {
-        return false;
-      }
-
-      // Mark if any store is big enough
       long size = store.getSize();
       if (size > sizeToCheck) {
         LOG.debug("ShouldSplit because " + store.getColumnFamilyName() +
           " size=" + StringUtils.humanSize(size) +
           ", sizeToCheck=" + StringUtils.humanSize(sizeToCheck) +
           ", regionsWithCommonTable=" + tableRegionsCount);
-        foundABigStore = true;
+        return true;
       }
     }
-
-    return foundABigStore;
+    return false;
   }
 
   /**

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SteppingAllStoresSizeSplitPolicy.java

@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import org.apache.hadoop.hbase.procedure2.util.StringUtils;
+import org.apache.yetus.audience.InterfaceAudience;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@InterfaceAudience.Private
+public class SteppingAllStoresSizeSplitPolicy extends SteppingSplitPolicy {
+  private static final Logger LOG =
+    LoggerFactory.getLogger(SteppingAllStoresSizeSplitPolicy.class);
+
+  /**
+   * @return true if sum of store's size exceed the sizeToCheck
+   */
+  @Override
+  protected boolean isExceedSize(int tableRegionsCount, long sizeToCheck) {
+    long sumSize = 0;
+    for (HStore store : region.getStores()) {
+      sumSize += store.getSize();
+    }
+    if (sumSize > sizeToCheck) {
+      LOG.debug("ShouldSplit because region size is big enough " +
+        "size={}, sizeToCheck={}, regionsWithCommonTable={}",
+        StringUtils.humanSize(sumSize), StringUtils.humanSize(sizeToCheck),
+        tableRegionsCount);
+      return true;
+    }
+    return false;
+  }
+
+}

hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionSplitPolicy.java

@@ -153,6 +153,41 @@ public void testIncreasingToUpperBoundRegionSplitPolicy() throws IOException {
     assertWithinJitter(maxSplitSize, policy.getSizeToCheck(0));
   }
 
+  @Test
+  public void testIsExceedSize() throws IOException {
+    // Configure SteppingAllStoresSizeSplitPolicy as our split policy
+    conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
+      SteppingAllStoresSizeSplitPolicy.class.getName());
+    // Now make it so the mock region has a RegionServerService that will
+    // return 'online regions'.
+    RegionServerServices rss = Mockito.mock(RegionServerServices.class);
+    final List<HRegion> regions = new ArrayList<>();
+    Mockito.doReturn(regions).when(rss).getRegions(TABLENAME);
+    Mockito.when(mockRegion.getRegionServerServices()).thenReturn(rss);
+
+    SteppingAllStoresSizeSplitPolicy policy =
+      (SteppingAllStoresSizeSplitPolicy) RegionSplitPolicy.create(mockRegion, conf);
+    regions.add(mockRegion);
+
+    HStore mockStore1 = Mockito.mock(HStore.class);
+    Mockito.doReturn(100L).when(mockStore1).getSize();
+    HStore mockStore2 = Mockito.mock(HStore.class);
+    Mockito.doReturn(924L).when(mockStore2).getSize();
+    HStore mockStore3 = Mockito.mock(HStore.class);
+    Mockito.doReturn(925L).when(mockStore3).getSize();
+
+    // test sum of store's size not greater than sizeToCheck
+    stores.add(mockStore1);
+    stores.add(mockStore2);
+    assertFalse(policy.isExceedSize(1, 1024));
+    stores.clear();
+
+    // test sum of store's size greater than sizeToCheck
+    stores.add(mockStore1);
+    stores.add(mockStore3);
+    assertTrue(policy.isExceedSize(1, 1024));
+  }
+
   @Test
   public void testBusyRegionSplitPolicy() throws Exception {
     doReturn(TableDescriptorBuilder.newBuilder(TABLENAME).build()).when(mockRegion)