Skip to content

Commit df0381f

Browse files
slfan1989slfan1989
authored
YARN-11536. [Federation] Router CLI Supports Batch Save the SubClusterPolicyConfiguration Of Queues. (#5862) Contributed by Shilun Fan.
Reviewed-by: Inigo Goiri <inigoiri@apache.org> Signed-off-by: Shilun Fan <slfan1989@apache.org>
1 parent ba32ea7 commit df0381f

File tree

26 files changed

+1330
-13
lines changed

26 files changed

+1330
-13
lines changed

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ResourceManagerAdministrationProtocol.java‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,8 @@
6060
import org.apache.hadoop.yarn.server.api.protocolrecords.DeregisterSubClusterResponse;
6161
import org.apache.hadoop.yarn.server.api.protocolrecords.SaveFederationQueuePolicyRequest;
6262
import org.apache.hadoop.yarn.server.api.protocolrecords.SaveFederationQueuePolicyResponse;
63-
63+
import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesRequest;
64+
import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesResponse;
6465

6566
@Private
6667
public interface ResourceManagerAdministrationProtocol extends GetUserMappingsProtocol {
@@ -189,4 +190,17 @@ DeregisterSubClusterResponse deregisterSubCluster(DeregisterSubClusterRequest re
189190
@Idempotent
190191
SaveFederationQueuePolicyResponse saveFederationQueuePolicy(
191192
SaveFederationQueuePolicyRequest request) throws YarnException, IOException;
193+
194+
/**
195+
* In YARN-Federation mode, this method provides a way to save queue policies in batches.
196+
*
197+
* @param request BatchSaveFederationQueuePolicies Request
198+
* @return Response from batchSaveFederationQueuePolicies.
199+
* @throws YarnException exceptions from yarn servers.
200+
* @throws IOException if an IO error occurred.
201+
*/
202+
@Private
203+
@Idempotent
204+
BatchSaveFederationQueuePoliciesResponse batchSaveFederationQueuePolicies(
205+
BatchSaveFederationQueuePoliciesRequest request) throws YarnException, IOException;
192206
}

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/BatchSaveFederationQueuePoliciesRequest.java‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
/**
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
package org.apache.hadoop.yarn.server.api.protocolrecords;
19+
20+
import org.apache.hadoop.classification.InterfaceAudience.Private;
21+
import org.apache.hadoop.classification.InterfaceAudience.Public;
22+
import org.apache.hadoop.classification.InterfaceStability.Unstable;
23+
import org.apache.hadoop.yarn.util.Records;
24+
25+
import java.util.List;
26+
27+
/**
28+
* In Federation mode,
29+
* we will support batch save queues policies to FederationStateStore.
30+
*/
31+
@Private
32+
@Unstable
33+
public abstract class BatchSaveFederationQueuePoliciesRequest {
34+
35+
@Private
36+
@Unstable
37+
public static BatchSaveFederationQueuePoliciesRequest newInstance(
38+
List<FederationQueueWeight> federationQueueWeights) {
39+
BatchSaveFederationQueuePoliciesRequest request =
40+
Records.newRecord(BatchSaveFederationQueuePoliciesRequest.class);
41+
request.setFederationQueueWeights(federationQueueWeights);
42+
return request;
43+
}
44+
45+
@Public
46+
@Unstable
47+
public abstract List<FederationQueueWeight> getFederationQueueWeights();
48+
49+
@Private
50+
@Unstable
51+
public abstract void setFederationQueueWeights(
52+
List<FederationQueueWeight> federationQueueWeights);
53+
}

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/BatchSaveFederationQueuePoliciesResponse.java‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
/**
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
package org.apache.hadoop.yarn.server.api.protocolrecords;
19+
20+
import org.apache.hadoop.classification.InterfaceAudience.Public;
21+
import org.apache.hadoop.classification.InterfaceAudience.Private;
22+
import org.apache.hadoop.classification.InterfaceStability.Unstable;
23+
import org.apache.hadoop.yarn.util.Records;
24+
25+
@Private
26+
@Unstable
27+
public abstract class BatchSaveFederationQueuePoliciesResponse {
28+
29+
public static BatchSaveFederationQueuePoliciesResponse newInstance() {
30+
return Records.newRecord(BatchSaveFederationQueuePoliciesResponse.class);
31+
}
32+
33+
public static BatchSaveFederationQueuePoliciesResponse newInstance(String msg) {
34+
BatchSaveFederationQueuePoliciesResponse response =
35+
Records.newRecord(BatchSaveFederationQueuePoliciesResponse.class);
36+
response.setMessage(msg);
37+
return response;
38+
}
39+
40+
@Public
41+
@Unstable
42+
public abstract String getMessage();
43+
44+
@Public
45+
@Unstable
46+
public abstract void setMessage(String msg);
47+
}

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/FederationQueueWeight.java‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,19 @@ public static FederationQueueWeight newInstance(String routerWeight,
7575
return federationQueueWeight;
7676
}
7777

78+
@Private
79+
@Unstable
80+
public static FederationQueueWeight newInstance(String routerWeight,
81+
String amrmWeight, String headRoomAlpha, String queue, String policyManagerClassName) {
82+
FederationQueueWeight federationQueueWeight = Records.newRecord(FederationQueueWeight.class);
83+
federationQueueWeight.setRouterWeight(routerWeight);
84+
federationQueueWeight.setAmrmWeight(amrmWeight);
85+
federationQueueWeight.setHeadRoomAlpha(headRoomAlpha);
86+
federationQueueWeight.setQueue(queue);
87+
federationQueueWeight.setPolicyManagerClassName(policyManagerClassName);
88+
return federationQueueWeight;
89+
}
90+
7891
@Public
7992
@Unstable
8093
public abstract String getRouterWeight();
@@ -166,4 +179,32 @@ public static void checkHeadRoomAlphaValid(String headRoomAlpha) throws YarnExce
166179
protected static boolean isNumeric(String value) {
167180
return NumberUtils.isCreatable(value);
168181
}
182+
183+
@Public
184+
@Unstable
185+
public abstract String getQueue();
186+
187+
@Public
188+
@Unstable
189+
public abstract void setQueue(String queue);
190+
191+
@Public
192+
@Unstable
193+
public abstract String getPolicyManagerClassName();
194+
195+
@Public
196+
@Unstable
197+
public abstract void setPolicyManagerClassName(String policyManagerClassName);
198+
199+
@Override
200+
public String toString() {
201+
StringBuilder builder = new StringBuilder();
202+
builder.append("FederationQueueWeight { ");
203+
builder.append("Queue: ").append(getQueue()).append(", ");
204+
builder.append("RouterWeight: ").append(getRouterWeight()).append(", ");
205+
builder.append("AmrmWeight: ").append(getAmrmWeight()).append(", ");
206+
builder.append("PolicyManagerClassName: ").append(getPolicyManagerClassName());
207+
builder.append(" }");
208+
return builder.toString();
209+
}
169210
}

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/resourcemanager_administration_protocol.proto‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,4 +49,5 @@ service ResourceManagerAdministrationProtocolService {
4949
rpc mapAttributesToNodes(NodesToAttributesMappingRequestProto) returns (NodesToAttributesMappingResponseProto);
5050
rpc deregisterSubCluster(DeregisterSubClusterRequestProto) returns (DeregisterSubClusterResponseProto);
5151
rpc saveFederationQueuePolicy(SaveFederationQueuePolicyRequestProto) returns (SaveFederationQueuePolicyResponseProto);
52+
rpc batchSaveFederationQueuePolicies(BatchSaveFederationQueuePoliciesRequestProto) returns (BatchSaveFederationQueuePoliciesResponseProto);
5253
}

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -180,6 +180,14 @@ message SaveFederationQueuePolicyResponseProto {
180180
required string message = 1;
181181
}
182182

183+
message BatchSaveFederationQueuePoliciesRequestProto {
184+
repeated FederationQueueWeightProto federationQueueWeights = 1;
185+
}
186+
187+
message BatchSaveFederationQueuePoliciesResponseProto {
188+
required string message = 1;
189+
}
190+
183191
//////////////////////////////////////////////////////////////////
184192
///////////// RM Failover related records ////////////////////////
185193
//////////////////////////////////////////////////////////////////

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -444,6 +444,8 @@ message FederationQueueWeightProto {
444444
optional string routerWeight = 1;
445445
optional string amrmWeight = 2;
446446
optional string headRoomAlpha = 3;
447+
optional string queue = 4;
448+
optional string policyManagerClassName = 5;
447449
}
448450

449451
////////////////////////////////////////////////////////////////////////

0 commit comments

Comments
 (0)