Skip to content

Commit ffa239b

Browse files
brumi1024brumi1024
committed
YARN-11608. Fix QueueCapacityVectorInfo NPE when accessible labels config is used.
1 parent 2b1378c commit ffa239b

File tree

7 files changed

+1277
-7
lines changed

7 files changed

+1277
-7
lines changed

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueCapacitiesInfo.java

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
2828
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
2929
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities;
30+
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacityVector;
3031

3132
/**
3233
* DAO which wraps PartitionQueueCapacitiesInfo applicable for a queue
@@ -57,8 +58,10 @@ public QueueCapacitiesInfo(CSQueue queue, boolean considerAMUsage) {
5758
float weight;
5859
float normalizedWeight;
5960
for (String partitionName : capacities.getExistingNodeLabels()) {
60-
queueCapacityVectorInfo = new QueueCapacityVectorInfo(
61-
queue.getConfiguredCapacityVector(partitionName));
61+
QueueCapacityVector queueCapacityVector = queue.getConfiguredCapacityVector(partitionName);
62+
queueCapacityVectorInfo = queueCapacityVector == null ?
63+
new QueueCapacityVectorInfo(new QueueCapacityVector()) :
64+
new QueueCapacityVectorInfo(queue.getConfiguredCapacityVector(partitionName));
6265
usedCapacity = capacities.getUsedCapacity(partitionName) * 100;
6366
capacity = capacities.getCapacity(partitionName) * 100;
6467
maxCapacity = capacities.getMaximumCapacity(partitionName);

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -150,6 +150,7 @@ private Configuration createConfig() {
150150
conf.set("yarn.scheduler.capacity.legacy-queue-mode.enabled", String.valueOf(legacyQueueMode));
151151
conf.set("yarn.scheduler.capacity.root.queues", "a, b, c");
152152
conf.set("yarn.scheduler.capacity.root.a.capacity", "12.5");
153+
conf.set("yarn.scheduler.capacity.root.a.accessible-node-labels", "root-a-default-label");
153154
conf.set("yarn.scheduler.capacity.root.a.maximum-capacity", "50");
154155
conf.set("yarn.scheduler.capacity.root.a.max-parallel-app", "42");
155156
conf.set("yarn.scheduler.capacity.root.b.capacity", "50");

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-NodeLabelDefaultAPI.xml

Lines changed: 259 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@
6161
</resourceInformations>
6262
</resourcesUsed>
6363
<hideReservationQueues>false</hideReservationQueues>
64-
<nodeLabels>*</nodeLabels>
64+
<nodeLabels>root-a-default-label</nodeLabels>
6565
<allocatedContainers>0</allocatedContainers>
6666
<reservedContainers>0</reservedContainers>
6767
<pendingContainers>0</pendingContainers>
@@ -185,6 +185,117 @@
185185
</resourceInformations>
186186
</effectiveMaxResource>
187187
</queueCapacitiesByPartition>
188+
<queueCapacitiesByPartition>
189+
<partitionName>root-a-default-label</partitionName>
190+
<queueCapacityVectorInfo>
191+
<configuredCapacityVector>[]</configuredCapacityVector>
192+
</queueCapacityVectorInfo>
193+
<capacity>0.0</capacity>
194+
<usedCapacity>0.0</usedCapacity>
195+
<maxCapacity>100.0</maxCapacity>
196+
<absoluteCapacity>0.0</absoluteCapacity>
197+
<absoluteUsedCapacity>0.0</absoluteUsedCapacity>
198+
<absoluteMaxCapacity>0.0</absoluteMaxCapacity>
199+
<maxAMLimitPercentage>0.0</maxAMLimitPercentage>
200+
<weight>-1.0</weight>
201+
<normalizedWeight>0.0</normalizedWeight>
202+
<configuredMinResource>
203+
<memory>0</memory>
204+
<vCores>0</vCores>
205+
<resourceInformations>
206+
<resourceInformation>
207+
<attributes/>
208+
<maximumAllocation>8192</maximumAllocation>
209+
<minimumAllocation>1024</minimumAllocation>
210+
<name>memory-mb</name>
211+
<resourceType>COUNTABLE</resourceType>
212+
<units>Mi</units>
213+
<value>0</value>
214+
</resourceInformation>
215+
<resourceInformation>
216+
<attributes/>
217+
<maximumAllocation>4</maximumAllocation>
218+
<minimumAllocation>1</minimumAllocation>
219+
<name>vcores</name>
220+
<resourceType>COUNTABLE</resourceType>
221+
<units/>
222+
<value>0</value>
223+
</resourceInformation>
224+
</resourceInformations>
225+
</configuredMinResource>
226+
<configuredMaxResource>
227+
<memory>0</memory>
228+
<vCores>0</vCores>
229+
<resourceInformations>
230+
<resourceInformation>
231+
<attributes/>
232+
<maximumAllocation>8192</maximumAllocation>
233+
<minimumAllocation>1024</minimumAllocation>
234+
<name>memory-mb</name>
235+
<resourceType>COUNTABLE</resourceType>
236+
<units>Mi</units>
237+
<value>0</value>
238+
</resourceInformation>
239+
<resourceInformation>
240+
<attributes/>
241+
<maximumAllocation>4</maximumAllocation>
242+
<minimumAllocation>1</minimumAllocation>
243+
<name>vcores</name>
244+
<resourceType>COUNTABLE</resourceType>
245+
<units/>
246+
<value>0</value>
247+
</resourceInformation>
248+
</resourceInformations>
249+
</configuredMaxResource>
250+
<effectiveMinResource>
251+
<memory>0</memory>
252+
<vCores>0</vCores>
253+
<resourceInformations>
254+
<resourceInformation>
255+
<attributes/>
256+
<maximumAllocation>8192</maximumAllocation>
257+
<minimumAllocation>1024</minimumAllocation>
258+
<name>memory-mb</name>
259+
<resourceType>COUNTABLE</resourceType>
260+
<units>Mi</units>
261+
<value>0</value>
262+
</resourceInformation>
263+
<resourceInformation>
264+
<attributes/>
265+
<maximumAllocation>4</maximumAllocation>
266+
<minimumAllocation>1</minimumAllocation>
267+
<name>vcores</name>
268+
<resourceType>COUNTABLE</resourceType>
269+
<units/>
270+
<value>0</value>
271+
</resourceInformation>
272+
</resourceInformations>
273+
</effectiveMinResource>
274+
<effectiveMaxResource>
275+
<memory>0</memory>
276+
<vCores>0</vCores>
277+
<resourceInformations>
278+
<resourceInformation>
279+
<attributes/>
280+
<maximumAllocation>8192</maximumAllocation>
281+
<minimumAllocation>1024</minimumAllocation>
282+
<name>memory-mb</name>
283+
<resourceType>COUNTABLE</resourceType>
284+
<units>Mi</units>
285+
<value>0</value>
286+
</resourceInformation>
287+
<resourceInformation>
288+
<attributes/>
289+
<maximumAllocation>4</maximumAllocation>
290+
<minimumAllocation>1</minimumAllocation>
291+
<name>vcores</name>
292+
<resourceType>COUNTABLE</resourceType>
293+
<units/>
294+
<value>0</value>
295+
</resourceInformation>
296+
</resourceInformations>
297+
</effectiveMaxResource>
298+
</queueCapacitiesByPartition>
188299
</capacities>
189300
<resources>
190301
<resourceUsagesByPartition>
@@ -334,6 +445,153 @@
334445
</resourceInformations>
335446
</userAmLimit>
336447
</resourceUsagesByPartition>
448+
<resourceUsagesByPartition>
449+
<partitionName>root-a-default-label</partitionName>
450+
<used>
451+
<memory>0</memory>
452+
<vCores>0</vCores>
453+
<resourceInformations>
454+
<resourceInformation>
455+
<attributes/>
456+
<maximumAllocation>9223372036854775807</maximumAllocation>
457+
<minimumAllocation>0</minimumAllocation>
458+
<name>memory-mb</name>
459+
<resourceType>COUNTABLE</resourceType>
460+
<units>Mi</units>
461+
<value>0</value>
462+
</resourceInformation>
463+
<resourceInformation>
464+
<attributes/>
465+
<maximumAllocation>9223372036854775807</maximumAllocation>
466+
<minimumAllocation>0</minimumAllocation>
467+
<name>vcores</name>
468+
<resourceType>COUNTABLE</resourceType>
469+
<units/>
470+
<value>0</value>
471+
</resourceInformation>
472+
</resourceInformations>
473+
</used>
474+
<reserved>
475+
<memory>0</memory>
476+
<vCores>0</vCores>
477+
<resourceInformations>
478+
<resourceInformation>
479+
<attributes/>
480+
<maximumAllocation>9223372036854775807</maximumAllocation>
481+
<minimumAllocation>0</minimumAllocation>
482+
<name>memory-mb</name>
483+
<resourceType>COUNTABLE</resourceType>
484+
<units>Mi</units>
485+
<value>0</value>
486+
</resourceInformation>
487+
<resourceInformation>
488+
<attributes/>
489+
<maximumAllocation>9223372036854775807</maximumAllocation>
490+
<minimumAllocation>0</minimumAllocation>
491+
<name>vcores</name>
492+
<resourceType>COUNTABLE</resourceType>
493+
<units/>
494+
<value>0</value>
495+
</resourceInformation>
496+
</resourceInformations>
497+
</reserved>
498+
<pending>
499+
<memory>0</memory>
500+
<vCores>0</vCores>
501+
<resourceInformations>
502+
<resourceInformation>
503+
<attributes/>
504+
<maximumAllocation>9223372036854775807</maximumAllocation>
505+
<minimumAllocation>0</minimumAllocation>
506+
<name>memory-mb</name>
507+
<resourceType>COUNTABLE</resourceType>
508+
<units>Mi</units>
509+
<value>0</value>
510+
</resourceInformation>
511+
<resourceInformation>
512+
<attributes/>
513+
<maximumAllocation>9223372036854775807</maximumAllocation>
514+
<minimumAllocation>0</minimumAllocation>
515+
<name>vcores</name>
516+
<resourceType>COUNTABLE</resourceType>
517+
<units/>
518+
<value>0</value>
519+
</resourceInformation>
520+
</resourceInformations>
521+
</pending>
522+
<amUsed>
523+
<memory>0</memory>
524+
<vCores>0</vCores>
525+
<resourceInformations>
526+
<resourceInformation>
527+
<attributes/>
528+
<maximumAllocation>9223372036854775807</maximumAllocation>
529+
<minimumAllocation>0</minimumAllocation>
530+
<name>memory-mb</name>
531+
<resourceType>COUNTABLE</resourceType>
532+
<units>Mi</units>
533+
<value>0</value>
534+
</resourceInformation>
535+
<resourceInformation>
536+
<attributes/>
537+
<maximumAllocation>9223372036854775807</maximumAllocation>
538+
<minimumAllocation>0</minimumAllocation>
539+
<name>vcores</name>
540+
<resourceType>COUNTABLE</resourceType>
541+
<units/>
542+
<value>0</value>
543+
</resourceInformation>
544+
</resourceInformations>
545+
</amUsed>
546+
<amLimit>
547+
<memory>0</memory>
548+
<vCores>0</vCores>
549+
<resourceInformations>
550+
<resourceInformation>
551+
<attributes/>
552+
<maximumAllocation>9223372036854775807</maximumAllocation>
553+
<minimumAllocation>0</minimumAllocation>
554+
<name>memory-mb</name>
555+
<resourceType>COUNTABLE</resourceType>
556+
<units>Mi</units>
557+
<value>0</value>
558+
</resourceInformation>
559+
<resourceInformation>
560+
<attributes/>
561+
<maximumAllocation>9223372036854775807</maximumAllocation>
562+
<minimumAllocation>0</minimumAllocation>
563+
<name>vcores</name>
564+
<resourceType>COUNTABLE</resourceType>
565+
<units/>
566+
<value>0</value>
567+
</resourceInformation>
568+
</resourceInformations>
569+
</amLimit>
570+
<userAmLimit>
571+
<memory>0</memory>
572+
<vCores>0</vCores>
573+
<resourceInformations>
574+
<resourceInformation>
575+
<attributes/>
576+
<maximumAllocation>9223372036854775807</maximumAllocation>
577+
<minimumAllocation>0</minimumAllocation>
578+
<name>memory-mb</name>
579+
<resourceType>COUNTABLE</resourceType>
580+
<units>Mi</units>
581+
<value>0</value>
582+
</resourceInformation>
583+
<resourceInformation>
584+
<attributes/>
585+
<maximumAllocation>9223372036854775807</maximumAllocation>
586+
<minimumAllocation>0</minimumAllocation>
587+
<name>vcores</name>
588+
<resourceType>COUNTABLE</resourceType>
589+
<units/>
590+
<value>0</value>
591+
</resourceInformation>
592+
</resourceInformations>
593+
</userAmLimit>
594+
</resourceUsagesByPartition>
337595
</resources>
338596
<minEffectiveCapacity>
339597
<memory>4096</memory>

0 commit comments

Comments
 (0)