Skip to content

Commit e330bb4

Browse files
committed
YARN-11511 Improve TestRMWebServices test config and data
5. CSConfigGenerator static class makes the tests hard to read.
1 parent 0c8a746 commit e330bb4

File tree

1 file changed

+119
-127
lines changed

1 file changed

+119
-127
lines changed

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java

Lines changed: 119 additions & 127 deletions
Original file line numberDiff line numberDiff line change
@@ -58,8 +58,7 @@ public TestRMWebServicesCapacitySchedDynamicConfig() {
5858
@Test
5959
public void testSchedulerResponsePercentageMode()
6060
throws Exception {
61-
Configuration config = CSConfigGenerator
62-
.createPercentageConfig();
61+
Configuration config = createPercentageConfig();
6362

6463
initResourceManager(config);
6564

@@ -75,8 +74,7 @@ public void testSchedulerResponsePercentageMode()
7574
@Test
7675
public void testSchedulerResponsePercentageModeLegacyAutoCreation()
7776
throws Exception {
78-
Configuration config = CSConfigGenerator
79-
.createPercentageConfigLegacyAutoCreation();
77+
Configuration config = createPercentageConfigLegacyAutoCreation();
8078

8179
initResourceManager(config);
8280

@@ -93,8 +91,7 @@ public void testSchedulerResponsePercentageModeLegacyAutoCreation()
9391
@Test
9492
public void testSchedulerResponseAbsoluteModeLegacyAutoCreation()
9593
throws Exception {
96-
Configuration config = CSConfigGenerator
97-
.createAbsoluteConfigLegacyAutoCreation();
94+
Configuration config = createAbsoluteConfigLegacyAutoCreation();
9895

9996
initResourceManager(config);
10097
initAutoQueueHandler(8192);
@@ -107,8 +104,7 @@ public void testSchedulerResponseAbsoluteModeLegacyAutoCreation()
107104
@Test
108105
public void testSchedulerResponseAbsoluteMode()
109106
throws Exception {
110-
Configuration config = CSConfigGenerator
111-
.createAbsoluteConfig();
107+
Configuration config = createAbsoluteConfig();
112108

113109
initResourceManager(config);
114110

@@ -124,8 +120,7 @@ public void testSchedulerResponseAbsoluteMode()
124120
@Test
125121
public void testSchedulerResponseWeightMode()
126122
throws Exception {
127-
Configuration config = CSConfigGenerator
128-
.createWeightConfig();
123+
Configuration config = createWeightConfig();
129124

130125
initResourceManager(config);
131126

@@ -143,8 +138,7 @@ public void testSchedulerResponseWeightMode()
143138
@Test
144139
public void testSchedulerResponseWeightModeWithAutoCreatedQueues()
145140
throws Exception {
146-
Configuration config = CSConfigGenerator
147-
.createWeightConfigWithAutoQueueCreationEnabled();
141+
Configuration config = createWeightConfigWithAutoQueueCreationEnabled();
148142
config.setInt(CapacitySchedulerConfiguration
149143
.getQueuePrefix("root.autoParent1") +
150144
AutoCreatedQueueTemplate.AUTO_QUEUE_TEMPLATE_PREFIX +
@@ -186,132 +180,130 @@ private ClientResponse sendRequest() {
186180
.get(ClientResponse.class);
187181
}
188182

189-
private static class CSConfigGenerator {
190-
public static Configuration createPercentageConfig() {
191-
Map<String, String> conf = new HashMap<>();
192-
conf.put("yarn.scheduler.capacity.root.queues", "default, test1, test2");
193-
conf.put("yarn.scheduler.capacity.root.test1.capacity", "50");
194-
conf.put("yarn.scheduler.capacity.root.test2.capacity", "50");
195-
conf.put("yarn.scheduler.capacity.root.test1.maximum-capacity", "100");
196-
conf.put("yarn.scheduler.capacity.root.test1.state", "RUNNING");
197-
conf.put("yarn.scheduler.capacity.root.test2.state", "RUNNING");
198-
return createConfiguration(conf);
199-
}
183+
private Configuration createPercentageConfig() {
184+
Map<String, String> conf = new HashMap<>();
185+
conf.put("yarn.scheduler.capacity.root.queues", "default, test1, test2");
186+
conf.put("yarn.scheduler.capacity.root.test1.capacity", "50");
187+
conf.put("yarn.scheduler.capacity.root.test2.capacity", "50");
188+
conf.put("yarn.scheduler.capacity.root.test1.maximum-capacity", "100");
189+
conf.put("yarn.scheduler.capacity.root.test1.state", "RUNNING");
190+
conf.put("yarn.scheduler.capacity.root.test2.state", "RUNNING");
191+
return createConfiguration(conf);
192+
}
200193

201-
public static Configuration createPercentageConfigLegacyAutoCreation() {
202-
Map<String, String> conf = new HashMap<>();
203-
conf.put("yarn.scheduler.capacity.root.queues", "default, test1, " +
204-
"managedtest2");
205-
conf.put("yarn.scheduler.capacity.root.test1.capacity", "50");
206-
conf.put("yarn.scheduler.capacity.root.managedtest2.capacity", "50");
207-
conf.put("yarn.scheduler.capacity.root.test1.maximum-capacity", "100");
208-
conf.put("yarn.scheduler.capacity.root.test1.state", "RUNNING");
209-
conf.put("yarn.scheduler.capacity.root.managedtest2.state", "RUNNING");
210-
conf.put("yarn.scheduler.capacity.root.managedtest2." +
211-
"auto-create-child-queue.enabled", "true");
212-
return createConfiguration(conf);
213-
}
194+
private Configuration createPercentageConfigLegacyAutoCreation() {
195+
Map<String, String> conf = new HashMap<>();
196+
conf.put("yarn.scheduler.capacity.root.queues", "default, test1, " +
197+
"managedtest2");
198+
conf.put("yarn.scheduler.capacity.root.test1.capacity", "50");
199+
conf.put("yarn.scheduler.capacity.root.managedtest2.capacity", "50");
200+
conf.put("yarn.scheduler.capacity.root.test1.maximum-capacity", "100");
201+
conf.put("yarn.scheduler.capacity.root.test1.state", "RUNNING");
202+
conf.put("yarn.scheduler.capacity.root.managedtest2.state", "RUNNING");
203+
conf.put("yarn.scheduler.capacity.root.managedtest2." +
204+
"auto-create-child-queue.enabled", "true");
205+
return createConfiguration(conf);
206+
}
214207

215-
public static Configuration createAbsoluteConfigLegacyAutoCreation() {
216-
Map<String, String> conf = new HashMap<>();
217-
conf.put("yarn.scheduler.capacity.root.queues", "default, managed");
218-
conf.put("yarn.scheduler.capacity.root.default.state", "STOPPED");
219-
conf.put("yarn.scheduler.capacity.root.default.capacity", "[memory=0,vcores=0]");
220-
conf.put("yarn.scheduler.capacity.root.managed.capacity", "[memory=4096,vcores=4]");
221-
conf.put("yarn.scheduler.capacity.root.managed.leaf-queue-template.capacity",
222-
"[memory=2048,vcores=2]");
223-
conf.put("yarn.scheduler.capacity.root.managed.state", "RUNNING");
224-
conf.put("yarn.scheduler.capacity.root.managed." +
225-
"auto-create-child-queue.enabled", "true");
226-
conf.put("yarn.scheduler.capacity.root.managed.leaf-queue-template.acl_submit_applications",
227-
"user");
228-
conf.put("yarn.scheduler.capacity.root.managed.leaf-queue-template.acl_administer_queue",
229-
"admin");
230-
return createConfiguration(conf);
231-
}
208+
private Configuration createAbsoluteConfigLegacyAutoCreation() {
209+
Map<String, String> conf = new HashMap<>();
210+
conf.put("yarn.scheduler.capacity.root.queues", "default, managed");
211+
conf.put("yarn.scheduler.capacity.root.default.state", "STOPPED");
212+
conf.put("yarn.scheduler.capacity.root.default.capacity", "[memory=0,vcores=0]");
213+
conf.put("yarn.scheduler.capacity.root.managed.capacity", "[memory=4096,vcores=4]");
214+
conf.put("yarn.scheduler.capacity.root.managed.leaf-queue-template.capacity",
215+
"[memory=2048,vcores=2]");
216+
conf.put("yarn.scheduler.capacity.root.managed.state", "RUNNING");
217+
conf.put("yarn.scheduler.capacity.root.managed." +
218+
"auto-create-child-queue.enabled", "true");
219+
conf.put("yarn.scheduler.capacity.root.managed.leaf-queue-template.acl_submit_applications",
220+
"user");
221+
conf.put("yarn.scheduler.capacity.root.managed.leaf-queue-template.acl_administer_queue",
222+
"admin");
223+
return createConfiguration(conf);
224+
}
232225

233-
public static Configuration createAbsoluteConfig() {
234-
Map<String, String> conf = new HashMap<>();
235-
conf.put("yarn.scheduler.capacity.root.queues", "default, test1, test2");
236-
conf.put("yarn.scheduler.capacity.root.capacity",
237-
"[memory=6136,vcores=30]");
238-
conf.put("yarn.scheduler.capacity.root.default.capacity",
239-
"[memory=3064,vcores=15]");
240-
conf.put("yarn.scheduler.capacity.root.test1.capacity",
241-
"[memory=2048,vcores=10]");
242-
conf.put("yarn.scheduler.capacity.root.test2.capacity",
243-
"[memory=1024,vcores=5]");
244-
conf.put("yarn.scheduler.capacity.root.test1.state", "RUNNING");
245-
conf.put("yarn.scheduler.capacity.root.test2.state", "RUNNING");
246-
return createConfiguration(conf);
247-
}
226+
private Configuration createAbsoluteConfig() {
227+
Map<String, String> conf = new HashMap<>();
228+
conf.put("yarn.scheduler.capacity.root.queues", "default, test1, test2");
229+
conf.put("yarn.scheduler.capacity.root.capacity",
230+
"[memory=6136,vcores=30]");
231+
conf.put("yarn.scheduler.capacity.root.default.capacity",
232+
"[memory=3064,vcores=15]");
233+
conf.put("yarn.scheduler.capacity.root.test1.capacity",
234+
"[memory=2048,vcores=10]");
235+
conf.put("yarn.scheduler.capacity.root.test2.capacity",
236+
"[memory=1024,vcores=5]");
237+
conf.put("yarn.scheduler.capacity.root.test1.state", "RUNNING");
238+
conf.put("yarn.scheduler.capacity.root.test2.state", "RUNNING");
239+
return createConfiguration(conf);
240+
}
248241

249-
public static Configuration createWeightConfig() {
250-
return createWeightConfigInternal(false);
251-
}
242+
private Configuration createWeightConfig() {
243+
return createWeightConfigInternal(false);
244+
}
252245

253-
public static Configuration createWeightConfigWithAutoQueueCreationEnabled() {
254-
return createWeightConfigInternal(true);
255-
}
246+
private Configuration createWeightConfigWithAutoQueueCreationEnabled() {
247+
return createWeightConfigInternal(true);
248+
}
256249

257-
private static Configuration createWeightConfigInternal(boolean enableAqc) {
258-
Map<String, String> conf = new HashMap<>();
259-
conf.put("yarn.scheduler.capacity.root.queues", "default, test1, test2, parent");
260-
conf.put("yarn.scheduler.capacity.root.capacity", "1w");
261-
conf.put("yarn.scheduler.capacity.root.default.capacity", "10w");
262-
conf.put("yarn.scheduler.capacity.root.test1.capacity", "5w");
263-
conf.put("yarn.scheduler.capacity.root.test2.capacity", "10w");
264-
conf.put("yarn.scheduler.capacity.root.parent.capacity", "20w");
265-
conf.put("yarn.scheduler.capacity.root.test1.state", "RUNNING");
266-
conf.put("yarn.scheduler.capacity.root.test2.state", "RUNNING");
267-
268-
if (enableAqc) {
269-
final String root = "yarn.scheduler.capacity.root.";
270-
conf.put(root + "auto-queue-creation-v2.enabled", "true");
271-
272-
conf.put(root + "auto-queue-creation-v2.parent-template.acl_submit_applications",
273-
"parentUser1");
274-
conf.put(root + "auto-queue-creation-v2.parent-template.acl_administer_queue",
275-
"parentAdmin1");
276-
277-
conf.put(root + "autoParent1.auto-queue-creation-v2.leaf-template.acl_submit_applications",
278-
"user1");
279-
conf.put(root + "autoParent1.auto-queue-creation-v2.leaf-template.acl_administer_queue",
280-
"admin1");
281-
282-
conf.put(root + "*.auto-queue-creation-v2.leaf-template.acl_submit_applications",
283-
"wildUser1");
284-
conf.put(root + "*.auto-queue-creation-v2.leaf-template.acl_administer_queue",
285-
"wildAdmin1");
286-
287-
288-
conf.put(root + "parent.auto-queue-creation-v2.enabled", "true");
289-
conf.put(root + "parent.auto-queue-creation-v2.parent-template.acl_submit_applications",
290-
"parentUser2");
291-
conf.put(root + "parent.auto-queue-creation-v2.parent-template.acl_administer_queue",
292-
"parentAdmin2");
293-
294-
conf.put(root + "parent.*.auto-queue-creation-v2.leaf-template.acl_submit_applications",
295-
"wildUser2");
296-
conf.put(root + "parent.*.auto-queue-creation-v2.leaf-template.acl_administer_queue",
297-
"wildAdmin2");
298-
}
299-
return createConfiguration(conf);
250+
private Configuration createWeightConfigInternal(boolean enableAqc) {
251+
Map<String, String> conf = new HashMap<>();
252+
conf.put("yarn.scheduler.capacity.root.queues", "default, test1, test2, parent");
253+
conf.put("yarn.scheduler.capacity.root.capacity", "1w");
254+
conf.put("yarn.scheduler.capacity.root.default.capacity", "10w");
255+
conf.put("yarn.scheduler.capacity.root.test1.capacity", "5w");
256+
conf.put("yarn.scheduler.capacity.root.test2.capacity", "10w");
257+
conf.put("yarn.scheduler.capacity.root.parent.capacity", "20w");
258+
conf.put("yarn.scheduler.capacity.root.test1.state", "RUNNING");
259+
conf.put("yarn.scheduler.capacity.root.test2.state", "RUNNING");
260+
261+
if (enableAqc) {
262+
final String root = "yarn.scheduler.capacity.root.";
263+
conf.put(root + "auto-queue-creation-v2.enabled", "true");
264+
265+
conf.put(root + "auto-queue-creation-v2.parent-template.acl_submit_applications",
266+
"parentUser1");
267+
conf.put(root + "auto-queue-creation-v2.parent-template.acl_administer_queue",
268+
"parentAdmin1");
269+
270+
conf.put(root + "autoParent1.auto-queue-creation-v2.leaf-template.acl_submit_applications",
271+
"user1");
272+
conf.put(root + "autoParent1.auto-queue-creation-v2.leaf-template.acl_administer_queue",
273+
"admin1");
274+
275+
conf.put(root + "*.auto-queue-creation-v2.leaf-template.acl_submit_applications",
276+
"wildUser1");
277+
conf.put(root + "*.auto-queue-creation-v2.leaf-template.acl_administer_queue",
278+
"wildAdmin1");
279+
280+
281+
conf.put(root + "parent.auto-queue-creation-v2.enabled", "true");
282+
conf.put(root + "parent.auto-queue-creation-v2.parent-template.acl_submit_applications",
283+
"parentUser2");
284+
conf.put(root + "parent.auto-queue-creation-v2.parent-template.acl_administer_queue",
285+
"parentAdmin2");
286+
287+
conf.put(root + "parent.*.auto-queue-creation-v2.leaf-template.acl_submit_applications",
288+
"wildUser2");
289+
conf.put(root + "parent.*.auto-queue-creation-v2.leaf-template.acl_administer_queue",
290+
"wildAdmin2");
300291
}
292+
return createConfiguration(conf);
293+
}
301294

302-
public static Configuration createConfiguration(
303-
Map<String, String> configs) {
304-
Configuration config = new Configuration();
295+
private Configuration createConfiguration(
296+
Map<String, String> configs) {
297+
Configuration config = new Configuration();
305298

306-
for (Map.Entry<String, String> entry : configs.entrySet()) {
307-
config.set(entry.getKey(), entry.getValue());
308-
}
299+
for (Map.Entry<String, String> entry : configs.entrySet()) {
300+
config.set(entry.getKey(), entry.getValue());
301+
}
309302

310-
config.set(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS,
311-
YarnConfiguration.MEMORY_CONFIGURATION_STORE);
303+
config.set(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS,
304+
YarnConfiguration.MEMORY_CONFIGURATION_STORE);
312305

313-
return config;
314-
}
306+
return config;
315307
}
316308

317309
private void initResourceManager(Configuration conf) throws IOException {

0 commit comments

Comments
 (0)