Skip to content

Commit 0085fce

Browse files
bbeaudreaultbbeaudreault
committed
Use the base BalanceRequest in RSGroup, and update coprocessor preBalance/postBalance to take the BalanceRequest
1 parent 91f0c8d commit 0085fce

File tree

19 files changed

+106
-149
lines changed

19 files changed

+106
-149
lines changed

‎hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdmin.java

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,8 @@
2222
import java.util.Map;
2323
import java.util.Set;
2424

25-
import org.apache.hadoop.hbase.client.BalanceRequest;
2625
import org.apache.hadoop.hbase.TableName;
26+
import org.apache.hadoop.hbase.client.BalanceRequest;
2727
import org.apache.hadoop.hbase.net.Address;
2828
import org.apache.yetus.audience.InterfaceAudience;
2929

@@ -69,7 +69,7 @@ public interface RSGroupAdmin {
6969
* @return boolean Whether balance ran or not
7070
*/
7171
default boolean balanceRSGroup(String groupName) throws IOException {
72-
return balanceRSGroup(BalanceRSGroupRequest.newBuilder(groupName).build());
72+
return balanceRSGroup(groupName, BalanceRequest.defaultInstance());
7373
}
7474

7575
/**
@@ -78,7 +78,7 @@ default boolean balanceRSGroup(String groupName) throws IOException {
7878
*
7979
* @return boolean Whether balance ran or not
8080
*/
81-
boolean balanceRSGroup(BalanceRSGroupRequest request) throws IOException;
81+
boolean balanceRSGroup(String groupName, BalanceRequest request) throws IOException;
8282

8383
/**
8484
* Lists current set of RegionServer groups.

‎hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminClient.java

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,12 +27,12 @@
2727
import java.util.Set;
2828
import java.util.stream.Collectors;
2929

30-
import org.apache.hadoop.hbase.client.BalanceRequest;
3130
import org.apache.hadoop.hbase.ClusterMetrics;
3231
import org.apache.hadoop.hbase.ServerName;
3332
import org.apache.hadoop.hbase.TableName;
3433
import org.apache.hadoop.hbase.TableNotFoundException;
3534
import org.apache.hadoop.hbase.client.Admin;
35+
import org.apache.hadoop.hbase.client.BalanceRequest;
3636
import org.apache.hadoop.hbase.client.Connection;
3737
import org.apache.hadoop.hbase.net.Address;
3838
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
@@ -158,10 +158,10 @@ public void removeRSGroup(String name) throws IOException {
158158
}
159159

160160
@Override
161-
public boolean balanceRSGroup(BalanceRSGroupRequest request) throws IOException {
161+
public boolean balanceRSGroup(String groupName, BalanceRequest request) throws IOException {
162162
try {
163163
return stub.balanceRSGroup(null,
164-
RSGroupProtobufUtil.toBalanceRSGroupRequest(request)).getBalanceRan();
164+
RSGroupProtobufUtil.createBalanceRSGroupRequest(groupName, request)).getBalanceRan();
165165
} catch (ServiceException e) {
166166
throw ProtobufUtil.handleRemoteException(e);
167167
}

‎hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminEndpoint.java

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,6 @@
3131
import java.util.Set;
3232
import java.util.stream.Collectors;
3333

34-
import org.apache.hadoop.hbase.client.BalanceRequest;
3534
import org.apache.hadoop.hbase.CoprocessorEnvironment;
3635
import org.apache.hadoop.hbase.HBaseIOException;
3736
import org.apache.hadoop.hbase.HConstants;
@@ -40,6 +39,7 @@
4039
import org.apache.hadoop.hbase.PleaseHoldException;
4140
import org.apache.hadoop.hbase.ServerName;
4241
import org.apache.hadoop.hbase.TableName;
42+
import org.apache.hadoop.hbase.client.BalanceRequest;
4343
import org.apache.hadoop.hbase.client.RegionInfo;
4444
import org.apache.hadoop.hbase.client.SnapshotDescription;
4545
import org.apache.hadoop.hbase.client.TableDescriptor;
@@ -59,6 +59,7 @@
5959
import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos;
6060
import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.AddRSGroupRequest;
6161
import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.AddRSGroupResponse;
62+
import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.BalanceRSGroupRequest;
6263
import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.BalanceRSGroupResponse;
6364
import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.GetRSGroupInfoOfServerRequest;
6465
import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.GetRSGroupInfoOfServerResponse;
@@ -290,21 +291,22 @@ public void removeRSGroup(RpcController controller,
290291

291292
@Override
292293
public void balanceRSGroup(RpcController controller,
293-
RSGroupAdminProtos.BalanceRSGroupRequest request, RpcCallback<BalanceRSGroupResponse> done) {
294-
BalanceRSGroupRequest balanceRequest = RSGroupProtobufUtil.toBalanceRequest(request);
294+
BalanceRSGroupRequest request, RpcCallback<BalanceRSGroupResponse> done) {
295+
BalanceRequest balanceRequest = RSGroupProtobufUtil.toBalanceRequest(request);
295296

296297
BalanceRSGroupResponse.Builder builder = BalanceRSGroupResponse.newBuilder();
297298
LOG.info(master.getClientIdAuditPrefix() + " balance rsgroup, group="
298299
+ request.getRSGroupName());
299300
try {
300-
if (master.getMasterCoprocessorHost() != null && !balanceRequest.isDryRun()) {
301-
master.getMasterCoprocessorHost().preBalanceRSGroup(request.getRSGroupName());
301+
if (master.getMasterCoprocessorHost() != null) {
302+
master.getMasterCoprocessorHost().preBalanceRSGroup(request.getRSGroupName(), balanceRequest);
302303
}
303304
checkPermission("balanceRSGroup");
304-
boolean balancerRan = groupAdminServer.balanceRSGroup(balanceRequest);
305+
boolean balancerRan = groupAdminServer.balanceRSGroup(request.getRSGroupName(), balanceRequest);
305306
builder.setBalanceRan(balancerRan);
306-
if (master.getMasterCoprocessorHost() != null && !balanceRequest.isDryRun()) {
307+
if (master.getMasterCoprocessorHost() != null) {
307308
master.getMasterCoprocessorHost().postBalanceRSGroup(request.getRSGroupName(),
309+
balanceRequest,
308310
balancerRan);
309311
}
310312
} catch (IOException e) {

‎hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminServer.java

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -29,11 +29,11 @@
2929
import java.util.concurrent.Future;
3030
import java.util.stream.Collectors;
3131
import org.apache.commons.lang3.StringUtils;
32-
import org.apache.hadoop.hbase.client.BalanceRequest;
3332
import org.apache.hadoop.hbase.HConstants;
3433
import org.apache.hadoop.hbase.NamespaceDescriptor;
3534
import org.apache.hadoop.hbase.ServerName;
3635
import org.apache.hadoop.hbase.TableName;
36+
import org.apache.hadoop.hbase.client.BalanceRequest;
3737
import org.apache.hadoop.hbase.client.RegionInfo;
3838
import org.apache.hadoop.hbase.client.TableDescriptor;
3939
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
@@ -509,7 +509,7 @@ public void removeRSGroup(String name) throws IOException {
509509
}
510510

511511
@Override
512-
public boolean balanceRSGroup(BalanceRSGroupRequest request) throws IOException {
512+
public boolean balanceRSGroup(String groupName, BalanceRequest request) throws IOException {
513513
ServerManager serverManager = master.getServerManager();
514514
LoadBalancer balancer = master.getLoadBalancer();
515515

@@ -519,12 +519,12 @@ public boolean balanceRSGroup(BalanceRSGroupRequest request) throws IOException
519519
return false;
520520
}
521521

522-
if (getRSGroupInfo(request.getGroupName()) == null) {
523-
throw new ConstraintException("RSGroup does not exist: "+ request.getGroupName());
522+
if (getRSGroupInfo(groupName) == null) {
523+
throw new ConstraintException("RSGroup does not exist: "+groupName);
524524
}
525525

526526
// Only allow one balance run at at time.
527-
Map<String, RegionState> groupRIT = rsGroupGetRegionsInTransition(request.getGroupName());
527+
Map<String, RegionState> groupRIT = rsGroupGetRegionsInTransition(groupName);
528528
if (groupRIT.size() > 0 && !request.isIgnoreRegionsInTransition()) {
529529
LOG.debug("Not running balancer because {} region(s) in transition: {}", groupRIT.size(),
530530
StringUtils.abbreviate(
@@ -541,7 +541,7 @@ public boolean balanceRSGroup(BalanceRSGroupRequest request) throws IOException
541541

542542
//We balance per group instead of per table
543543
Map<TableName, Map<ServerName, List<RegionInfo>>> assignmentsByTable =
544-
getRSGroupAssignmentsByTable(master.getTableStateManager(), request.getGroupName());
544+
getRSGroupAssignmentsByTable(master.getTableStateManager(), groupName);
545545
List<RegionPlan> plans = balancer.balanceCluster(assignmentsByTable);
546546
boolean balancerRan = !plans.isEmpty();
547547

@@ -550,9 +550,9 @@ public boolean balanceRSGroup(BalanceRSGroupRequest request) throws IOException
550550
}
551551

552552
if (balancerRan) {
553-
LOG.info("RSGroup balance {} starting with plan count: {}", request.getGroupName(), plans.size());
553+
LOG.info("RSGroup balance {} starting with plan count: {}", groupName, plans.size());
554554
master.executeRegionPlansWithThrottling(plans);
555-
LOG.info("RSGroup balance " + request.getGroupName() + " completed");
555+
LOG.info("RSGroup balance " + groupName + " completed");
556556
}
557557
return balancerRan;
558558
}

‎hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupProtobufUtil.java

Lines changed: 7 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -22,13 +22,13 @@
2222
import java.util.List;
2323
import java.util.stream.Collectors;
2424

25-
import org.apache.hadoop.hbase.client.BalanceRequest;
2625
import org.apache.hadoop.hbase.TableName;
26+
import org.apache.hadoop.hbase.client.BalanceRequest;
2727
import org.apache.hadoop.hbase.net.Address;
2828
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
2929
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
3030
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameStringPair;
31-
import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos;
31+
import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.BalanceRSGroupRequest;
3232
import org.apache.hadoop.hbase.protobuf.generated.RSGroupProtos;
3333
import org.apache.hadoop.hbase.protobuf.generated.TableProtos;
3434
import org.apache.yetus.audience.InterfaceAudience;
@@ -38,17 +38,16 @@ final class RSGroupProtobufUtil {
3838
private RSGroupProtobufUtil() {
3939
}
4040

41-
static RSGroupAdminProtos.BalanceRSGroupRequest toBalanceRSGroupRequest(BalanceRSGroupRequest request) {
42-
return RSGroupAdminProtos.BalanceRSGroupRequest.newBuilder()
43-
.setRSGroupName(request.getGroupName())
41+
static BalanceRSGroupRequest createBalanceRSGroupRequest(String groupName, BalanceRequest request) {
42+
return BalanceRSGroupRequest.newBuilder()
43+
.setRSGroupName(groupName)
4444
.setDryRun(request.isDryRun())
4545
.setIgnoreRit(request.isIgnoreRegionsInTransition())
4646
.build();
4747
}
4848

49-
static BalanceRSGroupRequest toBalanceRequest(
50-
RSGroupAdminProtos.BalanceRSGroupRequest request) {
51-
return BalanceRSGroupRequest.newBuilder(request.getRSGroupName())
49+
static BalanceRequest toBalanceRequest(BalanceRSGroupRequest request) {
50+
return BalanceRequest.newBuilder()
5251
.setDryRun(request.hasDryRun() && request.getDryRun())
5352
.setIgnoreRegionsInTransition(request.hasIgnoreRit() && request.getIgnoreRit())
5453
.build();

‎hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBalance.java

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -23,14 +23,14 @@
2323

2424
import java.util.List;
2525
import java.util.Map;
26-
import org.apache.hadoop.hbase.client.BalanceRequest;
2726
import org.apache.hadoop.hbase.HBaseClassTestRule;
2827
import org.apache.hadoop.hbase.HConstants;
2928
import org.apache.hadoop.hbase.NamespaceDescriptor;
3029
import org.apache.hadoop.hbase.ServerName;
3130
import org.apache.hadoop.hbase.TableName;
3231
import org.apache.hadoop.hbase.Waiter;
3332
import org.apache.hadoop.hbase.Waiter.Predicate;
33+
import org.apache.hadoop.hbase.client.BalanceRequest;
3434
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
3535
import org.apache.hadoop.hbase.client.RegionInfo;
3636
import org.apache.hadoop.hbase.client.TableDescriptor;
@@ -128,11 +128,7 @@ public void testGroupDryRunBalance() throws Exception {
128128

129129
// run the balancer in dry run mode. it should return true, but should not actually move any regions
130130
admin.balancerSwitch(true, true);
131-
assertTrue(rsGroupAdmin.balanceRSGroup(
132-
BalanceRSGroupRequest.newBuilder(newGroupName)
133-
.setDryRun(true)
134-
.build())
135-
);
131+
assertTrue(rsGroupAdmin.balanceRSGroup(newGroupName, BalanceRequest.newBuilder().setDryRun(true).build()));
136132
// validate imbalance still exists.
137133
assertEquals(6, getTableServerRegionMap().get(tableName).get(first).size());
138134
}

‎hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@
4343
import org.apache.hadoop.hbase.Waiter;
4444
import org.apache.hadoop.hbase.client.AbstractTestUpdateConfiguration;
4545
import org.apache.hadoop.hbase.client.Admin;
46+
import org.apache.hadoop.hbase.client.BalanceRequest;
4647
import org.apache.hadoop.hbase.client.RegionInfo;
4748
import org.apache.hadoop.hbase.client.TableDescriptor;
4849
import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
@@ -407,13 +408,13 @@ public void postMoveServers(final ObserverContext<MasterCoprocessorEnvironment>
407408

408409
@Override
409410
public void preBalanceRSGroup(final ObserverContext<MasterCoprocessorEnvironment> ctx,
410-
String groupName) throws IOException {
411+
String groupName, BalanceRequest request) throws IOException {
411412
preBalanceRSGroupCalled = true;
412413
}
413414

414415
@Override
415416
public void postBalanceRSGroup(final ObserverContext<MasterCoprocessorEnvironment> ctx,
416-
String groupName, boolean balancerRan) throws IOException {
417+
String groupName, BalanceRequest request, boolean balancerRan) throws IOException {
417418
postBalanceRSGroupCalled = true;
418419
}
419420

‎hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/VerifyingRSGroupAdminClient.java

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,10 @@
2222
import java.util.List;
2323
import java.util.Map;
2424
import java.util.Set;
25+
2526
import org.apache.hadoop.conf.Configuration;
2627
import org.apache.hadoop.hbase.TableName;
28+
import org.apache.hadoop.hbase.client.BalanceRequest;
2729
import org.apache.hadoop.hbase.client.ConnectionFactory;
2830
import org.apache.hadoop.hbase.client.Result;
2931
import org.apache.hadoop.hbase.client.Scan;
@@ -38,6 +40,7 @@
3840
import org.apache.yetus.audience.InterfaceAudience;
3941
import org.apache.zookeeper.KeeperException;
4042
import org.junit.Assert;
43+
4144
import org.apache.hbase.thirdparty.com.google.common.collect.Maps;
4245
import org.apache.hbase.thirdparty.com.google.common.collect.Sets;
4346

@@ -90,8 +93,8 @@ public void removeRSGroup(String name) throws IOException {
9093
}
9194

9295
@Override
93-
public boolean balanceRSGroup(BalanceRSGroupRequest request) throws IOException {
94-
return wrapped.balanceRSGroup(request);
96+
public boolean balanceRSGroup(String groupName, BalanceRequest request) throws IOException {
97+
return wrapped.balanceRSGroup(groupName, request);
9598
}
9699

97100
@Override

‎hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
import org.apache.hadoop.hbase.NamespaceDescriptor;
2929
import org.apache.hadoop.hbase.ServerName;
3030
import org.apache.hadoop.hbase.TableName;
31+
import org.apache.hadoop.hbase.client.BalanceRequest;
3132
import org.apache.hadoop.hbase.client.MasterSwitchType;
3233
import org.apache.hadoop.hbase.client.Mutation;
3334
import org.apache.hadoop.hbase.client.RegionInfo;
@@ -566,18 +567,20 @@ default void postRegionOffline(final ObserverContext<MasterCoprocessorEnvironmen
566567
* Called prior to requesting rebalancing of the cluster regions, though after
567568
* the initial checks for regions in transition and the balance switch flag.
568569
* @param ctx the environment to interact with the framework and master
570+
* @param request the request used to trigger the balancer
569571
*/
570-
default void preBalance(final ObserverContext<MasterCoprocessorEnvironment> ctx)
572+
default void preBalance(final ObserverContext<MasterCoprocessorEnvironment> ctx, BalanceRequest request)
571573
throws IOException {}
572574

573575
/**
574576
* Called after the balancing plan has been submitted.
575577
* @param ctx the environment to interact with the framework and master
578+
* @param request the request used to trigger the balance
576579
* @param plans the RegionPlans which master has executed. RegionPlan serves as hint
577580
* as for the final destination for the underlying region but may not represent the
578581
* final state of assignment
579582
*/
580-
default void postBalance(final ObserverContext<MasterCoprocessorEnvironment> ctx, List<RegionPlan> plans)
583+
default void postBalance(final ObserverContext<MasterCoprocessorEnvironment> ctx, BalanceRequest request, List<RegionPlan> plans)
581584
throws IOException {}
582585

583586
/**
@@ -1275,15 +1278,17 @@ default void postRemoveRSGroup(final ObserverContext<MasterCoprocessorEnvironmen
12751278
* @param groupName group name
12761279
*/
12771280
default void preBalanceRSGroup(final ObserverContext<MasterCoprocessorEnvironment> ctx,
1278-
String groupName) throws IOException {}
1281+
String groupName, BalanceRequest request) throws IOException {
1282+
}
12791283

12801284
/**
12811285
* Called after a region server group is removed
12821286
* @param ctx the environment to interact with the framework and master
12831287
* @param groupName group name
12841288
*/
12851289
default void postBalanceRSGroup(final ObserverContext<MasterCoprocessorEnvironment> ctx,
1286-
String groupName, boolean balancerRan) throws IOException {}
1290+
String groupName, BalanceRequest request, boolean balancerRan) throws IOException {
1291+
}
12871292

12881293
/**
12891294
* Called before servers are removed from rsgroup

‎hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java

Lines changed: 12 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1783,9 +1783,9 @@ public boolean balance(BalanceRequest request) throws IOException {
17831783
return false;
17841784
}
17851785

1786-
if (this.cpHost != null && !request.isDryRun()) {
1786+
if (this.cpHost != null) {
17871787
try {
1788-
if (this.cpHost.preBalance()) {
1788+
if (this.cpHost.preBalance(request)) {
17891789
LOG.debug("Coprocessor bypassing balancer request");
17901790
return false;
17911791
}
@@ -1807,20 +1807,22 @@ public boolean balance(BalanceRequest request) throws IOException {
18071807

18081808
List<RegionPlan> plans = this.balancer.balanceCluster(assignments);
18091809

1810-
if (request.isDryRun()) {
1811-
return true;
1812-
}
1813-
1810+
// make one last check that the cluster isn't shutting down before proceeding.
18141811
if (skipRegionManagementAction("balancer")) {
1815-
// make one last check that the cluster isn't shutting down before proceeding.
1816-
return false;
1812+
// return isDryRun because in this case the dry run did run, and the user could see
1813+
// the results in the master logs.
1814+
return request.isDryRun();
18171815
}
18181816

1819-
List<RegionPlan> sucRPs = executeRegionPlansWithThrottling(plans);
1817+
// For dry run we don't actually want to execute the moves, but we do want
1818+
// to execute the coprocessor below
1819+
List<RegionPlan> sucRPs = request.isDryRun()
1820+
? Collections.emptyList()
1821+
: executeRegionPlansWithThrottling(plans);
18201822

18211823
if (this.cpHost != null) {
18221824
try {
1823-
this.cpHost.postBalance(sucRPs);
1825+
this.cpHost.postBalance(request, sucRPs);
18241826
} catch (IOException ioe) {
18251827
// balancing already succeeded so don't change the result
18261828
LOG.error("Error invoking master coprocessor postBalance()", ioe);

0 commit comments

Comments
 (0)