|
| 1 | +/* |
| 2 | + * Licensed to the Apache Software Foundation (ASF) under one |
| 3 | + * or more contributor license agreements. See the NOTICE file |
| 4 | + * distributed with this work for additional information |
| 5 | + * regarding copyright ownership. The ASF licenses this file |
| 6 | + * to you under the Apache License, Version 2.0 (the |
| 7 | + * "License"); you may not use this file except in compliance |
| 8 | + * with the License. You may obtain a copy of the License at |
| 9 | + * |
| 10 | + * http://www.apache.org/licenses/LICENSE-2.0 |
| 11 | + * |
| 12 | + * Unless required by applicable law or agreed to in writing, software |
| 13 | + * distributed under the License is distributed on an "AS IS" BASIS, |
| 14 | + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 15 | + * See the License for the specific language governing permissions and |
| 16 | + * limitations under the License. |
| 17 | + */ |
| 18 | +package org.apache.hadoop.hbase; |
| 19 | + |
| 20 | +import static org.junit.Assert.assertEquals; |
| 21 | +import static org.junit.Assert.assertNotNull; |
| 22 | +import static org.junit.Assert.assertTrue; |
| 23 | + |
| 24 | +import java.io.IOException; |
| 25 | +import java.util.Collection; |
| 26 | +import java.util.EnumSet; |
| 27 | +import java.util.List; |
| 28 | +import java.util.Map; |
| 29 | +import java.util.TreeMap; |
| 30 | +import java.util.concurrent.TimeUnit; |
| 31 | +import java.util.stream.Collectors; |
| 32 | +import org.apache.hadoop.hbase.ClusterMetrics.Option; |
| 33 | +import org.apache.hadoop.hbase.client.Admin; |
| 34 | +import org.apache.hadoop.hbase.client.RegionInfo; |
| 35 | +import org.apache.hadoop.hbase.client.Table; |
| 36 | +import org.apache.hadoop.hbase.testclassification.MediumTests; |
| 37 | +import org.apache.hadoop.hbase.testclassification.MiscTests; |
| 38 | +import org.apache.hadoop.hbase.util.Bytes; |
| 39 | +import org.junit.AfterClass; |
| 40 | +import org.junit.BeforeClass; |
| 41 | +import org.junit.ClassRule; |
| 42 | +import org.junit.Test; |
| 43 | +import org.junit.experimental.categories.Category; |
| 44 | +import org.slf4j.Logger; |
| 45 | +import org.slf4j.LoggerFactory; |
| 46 | + |
| 47 | +import org.apache.hbase.thirdparty.com.google.common.collect.Lists; |
| 48 | +import org.apache.hbase.thirdparty.com.google.common.collect.Maps; |
| 49 | + |
| 50 | +@Category({MiscTests.class, MediumTests.class}) |
| 51 | +public class TestRegionLoad { |
| 52 | + |
| 53 | + @ClassRule |
| 54 | + public static final HBaseClassTestRule CLASS_RULE = |
| 55 | + HBaseClassTestRule.forClass(TestRegionLoad.class); |
| 56 | + |
| 57 | + private static final Logger LOG = LoggerFactory.getLogger(TestRegionLoad.class); |
| 58 | + private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); |
| 59 | + private static Admin admin; |
| 60 | + |
| 61 | + private static final TableName TABLE_1 = TableName.valueOf("table_1"); |
| 62 | + private static final TableName TABLE_2 = TableName.valueOf("table_2"); |
| 63 | + private static final TableName TABLE_3 = TableName.valueOf("table_3"); |
| 64 | + private static final TableName[] tables = new TableName[]{TABLE_1, TABLE_2, TABLE_3}; |
| 65 | + private static final int MSG_INTERVAL = 500; // ms |
| 66 | + |
| 67 | + @BeforeClass |
| 68 | + public static void beforeClass() throws Exception { |
| 69 | + // Make servers report eagerly. This test is about looking at the cluster status reported. |
| 70 | + // Make it so we don't have to wait around too long to see change. |
| 71 | + UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", MSG_INTERVAL); |
| 72 | + UTIL.startMiniCluster(4); |
| 73 | + admin = UTIL.getAdmin(); |
| 74 | + admin.balancerSwitch(false, true); |
| 75 | + createTables(); |
| 76 | + } |
| 77 | + |
| 78 | + @AfterClass |
| 79 | + public static void afterClass() throws Exception { |
| 80 | + UTIL.shutdownMiniCluster(); |
| 81 | + } |
| 82 | + |
| 83 | + private static void createTables() throws IOException, InterruptedException { |
| 84 | + byte[][] FAMILIES = new byte [][] {Bytes.toBytes("f")}; |
| 85 | + for (TableName tableName : tables) { |
| 86 | + Table table = |
| 87 | + UTIL.createTable(tableName, FAMILIES, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE); |
| 88 | + UTIL.waitTableAvailable(tableName); |
| 89 | + UTIL.loadTable(table, FAMILIES[0]); |
| 90 | + } |
| 91 | + } |
| 92 | + |
| 93 | + @Test |
| 94 | + public void testRegionLoad() throws Exception { |
| 95 | + |
| 96 | + // Check if regions match with the regionLoad from the server |
| 97 | + for (ServerName serverName : admin |
| 98 | + .getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS)).getLiveServerMetrics().keySet()) { |
| 99 | + List<RegionInfo> regions = admin.getRegions(serverName); |
| 100 | + LOG.info("serverName=" + serverName + ", regions=" + |
| 101 | + regions.stream().map(r -> r.getRegionNameAsString()).collect(Collectors.toList())); |
| 102 | + Collection<RegionLoad> regionLoads = admin.getRegionMetrics(serverName) |
| 103 | + .stream().map(r -> new RegionLoad(r)).collect(Collectors.toList()); |
| 104 | + LOG.info("serverName=" + serverName + ", regionLoads=" + |
| 105 | + regionLoads.stream().map(r -> Bytes.toString(r.getRegionName())). |
| 106 | + collect(Collectors.toList())); |
| 107 | + checkRegionsAndRegionLoads(regions, regionLoads); |
| 108 | + } |
| 109 | + |
| 110 | + // Check if regionLoad matches the table's regions and nothing is missed |
| 111 | + for (TableName table : new TableName[]{TABLE_1, TABLE_2, TABLE_3}) { |
| 112 | + List<RegionInfo> tableRegions = admin.getRegions(table); |
| 113 | + |
| 114 | + List<RegionLoad> regionLoads = Lists.newArrayList(); |
| 115 | + for (ServerName serverName : admin |
| 116 | + .getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS)).getLiveServerMetrics().keySet()) { |
| 117 | + regionLoads.addAll(admin.getRegionMetrics(serverName, table) |
| 118 | + .stream().map(r -> new RegionLoad(r)).collect(Collectors.toList())); |
| 119 | + } |
| 120 | + checkRegionsAndRegionLoads(tableRegions, regionLoads); |
| 121 | + } |
| 122 | + |
| 123 | + // Just wait here. If this fixes the test, come back and do a better job. |
| 124 | + // Would have to redo the below so can wait on cluster status changing. |
| 125 | + // Admin#getClusterMetrics retrieves data from HMaster. Admin#getRegionMetrics, by contrast, |
| 126 | + // get the data from RS. Hence, it will fail if we do the assert check before RS has done |
| 127 | + // the report. |
| 128 | + TimeUnit.MILLISECONDS.sleep(3 * MSG_INTERVAL); |
| 129 | + |
| 130 | + // Check RegionLoad matches the regionLoad from ClusterStatus |
| 131 | + ClusterStatus clusterStatus |
| 132 | + = new ClusterStatus(admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS))); |
| 133 | + for (ServerName serverName : clusterStatus.getServers()) { |
| 134 | + ServerLoad serverLoad = clusterStatus.getLoad(serverName); |
| 135 | + Map<byte[], RegionLoad> regionLoads = admin.getRegionMetrics(serverName).stream() |
| 136 | + .collect(Collectors.toMap(e -> e.getRegionName(), e -> new RegionLoad(e), |
| 137 | + (v1, v2) -> { |
| 138 | + throw new RuntimeException("impossible!!"); |
| 139 | + }, () -> new TreeMap<>(Bytes.BYTES_COMPARATOR))); |
| 140 | + LOG.debug("serverName=" + serverName + ", getRegionLoads=" + |
| 141 | + serverLoad.getRegionsLoad().keySet().stream().map(r -> Bytes.toString(r)). |
| 142 | + collect(Collectors.toList())); |
| 143 | + LOG.debug("serverName=" + serverName + ", regionLoads=" + |
| 144 | + regionLoads.keySet().stream().map(r -> Bytes.toString(r)). |
| 145 | + collect(Collectors.toList())); |
| 146 | + compareRegionLoads(serverLoad.getRegionsLoad(), regionLoads); |
| 147 | + } |
| 148 | + } |
| 149 | + |
| 150 | + private void compareRegionLoads(Map<byte[], RegionLoad> regionLoadCluster, |
| 151 | + Map<byte[], RegionLoad> regionLoads) { |
| 152 | + |
| 153 | + assertEquals("No of regionLoads from clusterStatus and regionloads from RS doesn't match", |
| 154 | + regionLoadCluster.size(), regionLoads.size()); |
| 155 | + |
| 156 | + // The contents of region load from cluster and server should match |
| 157 | + for (byte[] regionName : regionLoadCluster.keySet()) { |
| 158 | + regionLoads.remove(regionName); |
| 159 | + } |
| 160 | + assertEquals("regionLoads from SN should be empty", 0, regionLoads.size()); |
| 161 | + } |
| 162 | + |
| 163 | + private void checkRegionsAndRegionLoads(Collection<RegionInfo> regions, |
| 164 | + Collection<RegionLoad> regionLoads) { |
| 165 | + for (RegionLoad load : regionLoads) { |
| 166 | + assertNotNull(load.regionLoadPB); |
| 167 | + } |
| 168 | + |
| 169 | + assertEquals("No of regions and regionloads doesn't match", regions.size(), regionLoads.size()); |
| 170 | + |
| 171 | + Map<byte[], RegionLoad> regionLoadMap = Maps.newTreeMap(Bytes.BYTES_COMPARATOR); |
| 172 | + for (RegionLoad regionLoad : regionLoads) { |
| 173 | + regionLoadMap.put(regionLoad.getName(), regionLoad); |
| 174 | + } |
| 175 | + for (RegionInfo info : regions) { |
| 176 | + assertTrue("Region not in regionLoadMap region:" + info.getRegionNameAsString() + |
| 177 | + " regionMap: " + regionLoadMap, regionLoadMap.containsKey(info.getRegionName())); |
| 178 | + } |
| 179 | + } |
| 180 | +} |
0 commit comments