Skip to content

Commit 6de2cf6

Browse files
slfan1989slfan1989
committed
YARN-11536. Fix CheckStyle.
1 parent 04e7aa8 commit 6de2cf6

File tree

12 files changed

+231
-39
lines changed

12 files changed

+231
-39
lines changed

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ResourceManagerAdministrationProtocol.java‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,8 @@
6060
import org.apache.hadoop.yarn.server.api.protocolrecords.DeregisterSubClusterResponse;
6161
import org.apache.hadoop.yarn.server.api.protocolrecords.SaveFederationQueuePolicyRequest;
6262
import org.apache.hadoop.yarn.server.api.protocolrecords.SaveFederationQueuePolicyResponse;
63-
63+
import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesRequest;
64+
import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesResponse;
6465

6566
@Private
6667
public interface ResourceManagerAdministrationProtocol extends GetUserMappingsProtocol {
@@ -189,4 +190,17 @@ DeregisterSubClusterResponse deregisterSubCluster(DeregisterSubClusterRequest re
189190
@Idempotent
190191
SaveFederationQueuePolicyResponse saveFederationQueuePolicy(
191192
SaveFederationQueuePolicyRequest request) throws YarnException, IOException;
193+
194+
/**
195+
* In YARN-Federation mode, this method provides a way to save queue policies in batches.
196+
*
197+
* @param request BatchSaveFederationQueuePolicies Request
198+
* @return Response from batchSaveFederationQueuePolicies.
199+
* @throws YarnException exceptions from yarn servers.
200+
* @throws IOException if an IO error occurred.
201+
*/
202+
@Private
203+
@Idempotent
204+
BatchSaveFederationQueuePoliciesResponse batchSaveFederationQueuePolicies(
205+
BatchSaveFederationQueuePoliciesRequest request) throws YarnException, IOException;
192206
}

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/ResourceManagerAdministrationProtocolPBClientImpl.java‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
import org.apache.hadoop.ipc.RPC;
3030
import org.apache.hadoop.yarn.exceptions.YarnException;
3131
import org.apache.hadoop.yarn.ipc.RPCUtil;
32+
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos;
3233
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.AddToClusterNodeLabelsRequestProto;
3334
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.CheckForDecommissioningNodesRequestProto;
3435
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.GetGroupsForUserRequestProto;
@@ -47,6 +48,7 @@
4748
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.UpdateNodeResourceRequestProto;
4849
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.DeregisterSubClusterRequestProto;
4950
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.SaveFederationQueuePolicyRequestProto;
51+
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.BatchSaveFederationQueuePoliciesRequestProto;
5052
import org.apache.hadoop.yarn.server.api.ResourceManagerAdministrationProtocol;
5153
import org.apache.hadoop.yarn.server.api.ResourceManagerAdministrationProtocolPB;
5254
import org.apache.hadoop.yarn.server.api.protocolrecords.AddToClusterNodeLabelsRequest;
@@ -81,6 +83,8 @@
8183
import org.apache.hadoop.yarn.server.api.protocolrecords.DeregisterSubClusterResponse;
8284
import org.apache.hadoop.yarn.server.api.protocolrecords.SaveFederationQueuePolicyRequest;
8385
import org.apache.hadoop.yarn.server.api.protocolrecords.SaveFederationQueuePolicyResponse;
86+
import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesRequest;
87+
import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesResponse;
8488
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.AddToClusterNodeLabelsRequestPBImpl;
8589
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.AddToClusterNodeLabelsResponsePBImpl;
8690
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.CheckForDecommissioningNodesRequestPBImpl;
@@ -113,6 +117,8 @@
113117
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.DeregisterSubClusterResponsePBImpl;
114118
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.SaveFederationQueuePolicyRequestPBImpl;
115119
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.SaveFederationQueuePolicyResponsePBImpl;
120+
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.BatchSaveFederationQueuePoliciesRequestPBImpl;
121+
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.BatchSaveFederationQueuePoliciesResponsePBImpl;
116122

117123
import org.apache.hadoop.thirdparty.protobuf.ServiceException;
118124

@@ -381,4 +387,18 @@ public SaveFederationQueuePolicyResponse saveFederationQueuePolicy(
381387
return null;
382388
}
383389
}
390+
391+
@Override
392+
public BatchSaveFederationQueuePoliciesResponse batchSaveFederationQueuePolicies(
393+
BatchSaveFederationQueuePoliciesRequest request) throws YarnException, IOException {
394+
BatchSaveFederationQueuePoliciesRequestProto requestProto =
395+
((BatchSaveFederationQueuePoliciesRequestPBImpl) request).getProto();
396+
try {
397+
return new BatchSaveFederationQueuePoliciesResponsePBImpl(
398+
proxy.batchSaveFederationQueuePolicies(null, requestProto));
399+
} catch (ServiceException e) {
400+
RPCUtil.unwrapAndThrowException(e);
401+
return null;
402+
}
403+
}
384404
}

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/ResourceManagerAdministrationProtocolPBServiceImpl.java‎

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,10 @@
2222

2323
import org.apache.hadoop.classification.InterfaceAudience.Private;
2424
import org.apache.hadoop.yarn.exceptions.YarnException;
25-
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos;
2625
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.AddToClusterNodeLabelsRequestProto;
2726
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.AddToClusterNodeLabelsResponseProto;
27+
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.BatchSaveFederationQueuePoliciesRequestProto;
28+
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.BatchSaveFederationQueuePoliciesResponseProto;
2829
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.CheckForDecommissioningNodesRequestProto;
2930
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.CheckForDecommissioningNodesResponseProto;
3031
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.GetGroupsForUserRequestProto;
@@ -80,6 +81,8 @@
8081
import org.apache.hadoop.yarn.server.api.protocolrecords.DeregisterSubClusterResponse;
8182
import org.apache.hadoop.yarn.server.api.protocolrecords.SaveFederationQueuePolicyRequest;
8283
import org.apache.hadoop.yarn.server.api.protocolrecords.SaveFederationQueuePolicyResponse;
84+
import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesRequest;
85+
import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesResponse;
8386
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.AddToClusterNodeLabelsRequestPBImpl;
8487
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.AddToClusterNodeLabelsResponsePBImpl;
8588
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.CheckForDecommissioningNodesRequestPBImpl;
@@ -112,6 +115,8 @@
112115
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.DeregisterSubClusterResponsePBImpl;
113116
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.SaveFederationQueuePolicyRequestPBImpl;
114117
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.SaveFederationQueuePolicyResponsePBImpl;
118+
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.BatchSaveFederationQueuePoliciesRequestPBImpl;
119+
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.BatchSaveFederationQueuePoliciesResponsePBImpl;
115120

116121
import org.apache.hadoop.thirdparty.protobuf.RpcController;
117122
import org.apache.hadoop.thirdparty.protobuf.ServiceException;
@@ -402,7 +407,17 @@ public SaveFederationQueuePolicyResponseProto saveFederationQueuePolicy(RpcContr
402407
}
403408

404409
@Override
405-
public YarnServerResourceManagerServiceProtos.BatchSaveFederationQueuePoliciesResponseProto batchSaveFederationQueuePolicies(RpcController controller, YarnServerResourceManagerServiceProtos.BatchSaveFederationQueuePoliciesRequestProto request) throws ServiceException {
406-
return null;
410+
public BatchSaveFederationQueuePoliciesResponseProto batchSaveFederationQueuePolicies(
411+
RpcController controller, BatchSaveFederationQueuePoliciesRequestProto proto)
412+
throws ServiceException {
413+
BatchSaveFederationQueuePoliciesRequest request = new BatchSaveFederationQueuePoliciesRequestPBImpl(proto);
414+
try {
415+
BatchSaveFederationQueuePoliciesResponse response = real.batchSaveFederationQueuePolicies(request);
416+
return ((BatchSaveFederationQueuePoliciesResponsePBImpl) response).getProto();
417+
} catch (YarnException e) {
418+
throw new ServiceException(e);
419+
} catch (IOException e) {
420+
throw new ServiceException(e);
421+
}
407422
}
408423
}

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/BatchSaveFederationQueuePoliciesRequestPBImpl.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,9 @@
3030
import java.util.ArrayList;
3131
import java.util.List;
3232

33+
/**
34+
* The class is responsible for batch-saving queue policies requests.
35+
*/
3336
@Private
3437
@Unstable
3538
public class BatchSaveFederationQueuePoliciesRequestPBImpl

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/BatchSaveFederationQueuePoliciesResponsePBImpl.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,11 +18,13 @@
1818
package org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb;
1919

2020
import org.apache.hadoop.thirdparty.protobuf.TextFormat;
21-
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos;
2221
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.BatchSaveFederationQueuePoliciesResponseProto;
2322
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.BatchSaveFederationQueuePoliciesResponseProtoOrBuilder;
2423
import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesResponse;
2524

25+
/**
26+
* The class is responsible for batch-saving queue policies responses.
27+
*/
2628
public class BatchSaveFederationQueuePoliciesResponsePBImpl
2729
extends BatchSaveFederationQueuePoliciesResponse {
2830

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -177,6 +177,8 @@
177177
import org.apache.hadoop.yarn.server.api.protocolrecords.DeregisterSubClusterResponse;
178178
import org.apache.hadoop.yarn.server.api.protocolrecords.SaveFederationQueuePolicyRequest;
179179
import org.apache.hadoop.yarn.server.api.protocolrecords.SaveFederationQueuePolicyResponse;
180+
import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesRequest;
181+
import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesResponse;
180182
import org.apache.hadoop.thirdparty.com.google.common.base.Strings;
181183

182184
/**
@@ -972,6 +974,12 @@ public SaveFederationQueuePolicyResponse saveFederationQueuePolicy(
972974
return null;
973975
}
974976

977+
@Override
978+
public BatchSaveFederationQueuePoliciesResponse batchSaveFederationQueuePolicies(
979+
BatchSaveFederationQueuePoliciesRequest request) throws YarnException, IOException {
980+
return null;
981+
}
982+
975983
@VisibleForTesting
976984
public HashMap<ApplicationId, List<ContainerId>> getApplicationContainerIdMap() {
977985
return applicationContainerIdMap;

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,8 @@
9999
import org.apache.hadoop.yarn.server.api.protocolrecords.DeregisterSubClusterResponse;
100100
import org.apache.hadoop.yarn.server.api.protocolrecords.SaveFederationQueuePolicyRequest;
101101
import org.apache.hadoop.yarn.server.api.protocolrecords.SaveFederationQueuePolicyResponse;
102+
import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesRequest;
103+
import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesResponse;
102104
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NodeLabelsUtils;
103105
import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSystem;
104106
import org.apache.hadoop.yarn.server.resourcemanager.resource.DynamicResourceConfiguration;
@@ -1060,7 +1062,15 @@ public DeregisterSubClusterResponse deregisterSubCluster(
10601062
public SaveFederationQueuePolicyResponse saveFederationQueuePolicy(
10611063
SaveFederationQueuePolicyRequest request) throws YarnException, IOException {
10621064
throw new YarnException("It is not allowed to call the RM's saveFederationQueuePolicy. " +
1063-
" Please call Router's deregisterSubCluster to set Policy.");
1065+
" Please call Router's saveFederationQueuePolicy to set Policy.");
1066+
}
1067+
1068+
@Override
1069+
public BatchSaveFederationQueuePoliciesResponse batchSaveFederationQueuePolicies(
1070+
BatchSaveFederationQueuePoliciesRequest request) throws YarnException, IOException {
1071+
throw new YarnException("It is not allowed to call the RM's " +
1072+
" batchSaveFederationQueuePolicies. " +
1073+
" Please call Router's batchSaveFederationQueuePolicies to set Policies.");
10641074
}
10651075

10661076
private void validateAttributesExists(

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterMetrics.java‎

Lines changed: 33 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -151,6 +151,8 @@ public final class RouterMetrics {
151151
private MutableGaugeInt numDeregisterSubClusterFailedRetrieved;
152152
@Metric("# of saveFederationQueuePolicy failed to be retrieved")
153153
private MutableGaugeInt numSaveFederationQueuePolicyFailedRetrieved;
154+
@Metric("# of batchSaveFederationQueuePolicies failed to be retrieved")
155+
private MutableGaugeInt numBatchSaveFederationQueuePoliciesFailedRetrieved;
154156
@Metric("# of refreshAdminAcls failed to be retrieved")
155157
private MutableGaugeInt numRefreshAdminAclsFailedRetrieved;
156158
@Metric("# of refreshServiceAcls failed to be retrieved")
@@ -299,6 +301,8 @@ public final class RouterMetrics {
299301
private MutableRate totalSucceededDeregisterSubClusterRetrieved;
300302
@Metric("Total number of successful Retrieved SaveFederationQueuePolicy and latency(ms)")
301303
private MutableRate totalSucceededSaveFederationQueuePolicyRetrieved;
304+
@Metric("Total number of successful Retrieved BatchSaveFederationQueuePolicies and latency(ms)")
305+
private MutableRate totalSucceededBatchSaveFederationQueuePoliciesRetrieved;
302306
@Metric("Total number of successful Retrieved RefreshAdminAcls and latency(ms)")
303307
private MutableRate totalSucceededRefreshAdminAclsRetrieved;
304308
@Metric("Total number of successful Retrieved RefreshServiceAcls and latency(ms)")
@@ -386,6 +390,7 @@ public final class RouterMetrics {
386390
private MutableQuantiles refreshUserToGroupsMappingsLatency;
387391
private MutableQuantiles refreshDeregisterSubClusterLatency;
388392
private MutableQuantiles saveFederationQueuePolicyLatency;
393+
private MutableQuantiles batchSaveFederationQueuePoliciesLatency;
389394
private MutableQuantiles refreshAdminAclsLatency;
390395
private MutableQuantiles refreshServiceAclsLatency;
391396
private MutableQuantiles replaceLabelsOnNodesLatency;
@@ -598,7 +603,11 @@ private RouterMetrics() {
598603
"latency of deregister subcluster timeouts", "ops", "latency", 10);
599604

600605
saveFederationQueuePolicyLatency = registry.newQuantiles("saveFederationQueuePolicyLatency",
601-
"latency of refresh subcluster timeouts", "ops", "latency", 10);
606+
"latency of save federation queue policy timeouts", "ops", "latency", 10);
607+
608+
batchSaveFederationQueuePoliciesLatency = registry.newQuantiles(
609+
"batchSaveFederationQueuePoliciesLatency",
610+
"latency of batch save federationqueuepolicies timeouts", "ops", "latency", 10);
602611

603612
refreshAdminAclsLatency = registry.newQuantiles("refreshAdminAclsLatency",
604613
"latency of refresh admin acls timeouts", "ops", "latency", 10);
@@ -934,6 +943,11 @@ public long getNumSucceededSaveFederationQueuePolicyRetrieved() {
934943
return totalSucceededSaveFederationQueuePolicyRetrieved.lastStat().numSamples();
935944
}
936945

946+
@VisibleForTesting
947+
public long getNumSucceededBatchSaveFederationQueuePoliciesRetrieved() {
948+
return totalSucceededBatchSaveFederationQueuePoliciesRetrieved.lastStat().numSamples();
949+
}
950+
937951
@VisibleForTesting
938952
public long getNumSucceededRefreshAdminAclsRetrieved() {
939953
return totalSucceededRefreshAdminAclsRetrieved.lastStat().numSamples();
@@ -1284,6 +1298,11 @@ public double getLatencySucceededSaveFederationQueuePolicyRetrieved() {
12841298
return totalSucceededSaveFederationQueuePolicyRetrieved.lastStat().mean();
12851299
}
12861300

1301+
@VisibleForTesting
1302+
public double getLatencySucceededBatchSaveFederationQueuePoliciesRetrieved() {
1303+
return totalSucceededBatchSaveFederationQueuePoliciesRetrieved.lastStat().mean();
1304+
}
1305+
12871306
@VisibleForTesting
12881307
public double getLatencySucceededRefreshAdminAclsRetrieved() {
12891308
return totalSucceededRefreshAdminAclsRetrieved.lastStat().mean();
@@ -1583,6 +1602,10 @@ public int getSaveFederationQueuePolicyFailedRetrieved() {
15831602
return numSaveFederationQueuePolicyFailedRetrieved.value();
15841603
}
15851604

1605+
public int getBatchSaveFederationQueuePoliciesFailedRetrieved() {
1606+
return numBatchSaveFederationQueuePoliciesFailedRetrieved.value();
1607+
}
1608+
15861609
public int getNumRefreshAdminAclsFailedRetrieved() {
15871610
return numRefreshAdminAclsFailedRetrieved.value();
15881611
}
@@ -1940,6 +1963,11 @@ public void succeededSaveFederationQueuePolicyRetrieved(long duration) {
19401963
saveFederationQueuePolicyLatency.add(duration);
19411964
}
19421965

1966+
public void succeededBatchSaveFederationQueuePoliciesRetrieved(long duration) {
1967+
totalSucceededBatchSaveFederationQueuePoliciesRetrieved.add(duration);
1968+
batchSaveFederationQueuePoliciesLatency.add(duration);
1969+
}
1970+
19431971
public void succeededRefreshAdminAclsRetrieved(long duration) {
19441972
totalSucceededRefreshAdminAclsRetrieved.add(duration);
19451973
refreshAdminAclsLatency.add(duration);
@@ -2222,6 +2250,10 @@ public void incrSaveFederationQueuePolicyFailedRetrieved() {
22222250
numSaveFederationQueuePolicyFailedRetrieved.incr();
22232251
}
22242252

2253+
public void incrBatchSaveFederationQueuePoliciesFailedRetrieved() {
2254+
numBatchSaveFederationQueuePoliciesFailedRetrieved.incr();
2255+
}
2256+
22252257
public void incrRefreshAdminAclsFailedRetrieved() {
22262258
numRefreshAdminAclsFailedRetrieved.incr();
22272259
}

0 commit comments

Comments
 (0)