Skip to content

Commit

Permalink
YARN-11608. Fix QueueCapacityVectorInfo NPE when accessible labels co…
Browse files Browse the repository at this point in the history 
…nfig is used. (apache#6250)

Co-authored-by: Benjamin Teke <bteke@cloudera.com>
  • Loading branch information
@brumi1024 @bteke @jiajunmao
2 people authored and jiajunmao committed Feb 6, 2024
1 parent d541e99 commit 3999be8
Show file tree
Hide file tree
Showing 7 changed files with 1,277 additions and 7 deletions.
7 changes: 5 additions & 2 deletions ...in/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueCapacitiesInfo.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacityVector;

/**
* DAO which wraps PartitionQueueCapacitiesInfo applicable for a queue
Expand Down Expand Up @@ -57,8 +58,10 @@ public QueueCapacitiesInfo(CSQueue queue, boolean considerAMUsage) {
float weight;
float normalizedWeight;
for (String partitionName : capacities.getExistingNodeLabels()) {
queueCapacityVectorInfo = new QueueCapacityVectorInfo(
queue.getConfiguredCapacityVector(partitionName));
QueueCapacityVector queueCapacityVector = queue.getConfiguredCapacityVector(partitionName);
queueCapacityVectorInfo = queueCapacityVector == null ?
new QueueCapacityVectorInfo(new QueueCapacityVector()) :
new QueueCapacityVectorInfo(queue.getConfiguredCapacityVector(partitionName));
usedCapacity = capacities.getUsedCapacity(partitionName) * 100;
capacity = capacities.getCapacity(partitionName) * 100;
maxCapacity = capacities.getMaximumCapacity(partitionName);
Expand Down
1 change: 1 addition & 0 deletions .../org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,7 @@ private Configuration createConfig() {
conf.set("yarn.scheduler.capacity.legacy-queue-mode.enabled", String.valueOf(legacyQueueMode));
conf.set("yarn.scheduler.capacity.root.queues", "a, b, c");
conf.set("yarn.scheduler.capacity.root.a.capacity", "12.5");
conf.set("yarn.scheduler.capacity.root.a.accessible-node-labels", "root-a-default-label");
conf.set("yarn.scheduler.capacity.root.a.maximum-capacity", "50");
conf.set("yarn.scheduler.capacity.root.a.max-parallel-app", "42");
conf.set("yarn.scheduler.capacity.root.b.capacity", "50");
Expand Down
260 changes: 259 additions & 1 deletion ...rver-resourcemanager/src/test/resources/webapp/scheduler-response-NodeLabelDefaultAPI.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@
</resourceInformations>
</resourcesUsed>
<hideReservationQueues>false</hideReservationQueues>
<nodeLabels>*</nodeLabels>
<nodeLabels>root-a-default-label</nodeLabels>
<allocatedContainers>0</allocatedContainers>
<reservedContainers>0</reservedContainers>
<pendingContainers>0</pendingContainers>
Expand Down Expand Up @@ -185,6 +185,117 @@
</resourceInformations>
</effectiveMaxResource>
</queueCapacitiesByPartition>
<queueCapacitiesByPartition>
<partitionName>root-a-default-label</partitionName>
<queueCapacityVectorInfo>
<configuredCapacityVector>[]</configuredCapacityVector>
</queueCapacityVectorInfo>
<capacity>0.0</capacity>
<usedCapacity>0.0</usedCapacity>
<maxCapacity>100.0</maxCapacity>
<absoluteCapacity>0.0</absoluteCapacity>
<absoluteUsedCapacity>0.0</absoluteUsedCapacity>
<absoluteMaxCapacity>0.0</absoluteMaxCapacity>
<maxAMLimitPercentage>0.0</maxAMLimitPercentage>
<weight>-1.0</weight>
<normalizedWeight>0.0</normalizedWeight>
<configuredMinResource>
<memory>0</memory>
<vCores>0</vCores>
<resourceInformations>
<resourceInformation>
<attributes/>
<maximumAllocation>8192</maximumAllocation>
<minimumAllocation>1024</minimumAllocation>
<name>memory-mb</name>
<resourceType>COUNTABLE</resourceType>
<units>Mi</units>
<value>0</value>
</resourceInformation>
<resourceInformation>
<attributes/>
<maximumAllocation>4</maximumAllocation>
<minimumAllocation>1</minimumAllocation>
<name>vcores</name>
<resourceType>COUNTABLE</resourceType>
<units/>
<value>0</value>
</resourceInformation>
</resourceInformations>
</configuredMinResource>
<configuredMaxResource>
<memory>0</memory>
<vCores>0</vCores>
<resourceInformations>
<resourceInformation>
<attributes/>
<maximumAllocation>8192</maximumAllocation>
<minimumAllocation>1024</minimumAllocation>
<name>memory-mb</name>
<resourceType>COUNTABLE</resourceType>
<units>Mi</units>
<value>0</value>
</resourceInformation>
<resourceInformation>
<attributes/>
<maximumAllocation>4</maximumAllocation>
<minimumAllocation>1</minimumAllocation>
<name>vcores</name>
<resourceType>COUNTABLE</resourceType>
<units/>
<value>0</value>
</resourceInformation>
</resourceInformations>
</configuredMaxResource>
<effectiveMinResource>
<memory>0</memory>
<vCores>0</vCores>
<resourceInformations>
<resourceInformation>
<attributes/>
<maximumAllocation>8192</maximumAllocation>
<minimumAllocation>1024</minimumAllocation>
<name>memory-mb</name>
<resourceType>COUNTABLE</resourceType>
<units>Mi</units>
<value>0</value>
</resourceInformation>
<resourceInformation>
<attributes/>
<maximumAllocation>4</maximumAllocation>
<minimumAllocation>1</minimumAllocation>
<name>vcores</name>
<resourceType>COUNTABLE</resourceType>
<units/>
<value>0</value>
</resourceInformation>
</resourceInformations>
</effectiveMinResource>
<effectiveMaxResource>
<memory>0</memory>
<vCores>0</vCores>
<resourceInformations>
<resourceInformation>
<attributes/>
<maximumAllocation>8192</maximumAllocation>
<minimumAllocation>1024</minimumAllocation>
<name>memory-mb</name>
<resourceType>COUNTABLE</resourceType>
<units>Mi</units>
<value>0</value>
</resourceInformation>
<resourceInformation>
<attributes/>
<maximumAllocation>4</maximumAllocation>
<minimumAllocation>1</minimumAllocation>
<name>vcores</name>
<resourceType>COUNTABLE</resourceType>
<units/>
<value>0</value>
</resourceInformation>
</resourceInformations>
</effectiveMaxResource>
</queueCapacitiesByPartition>
</capacities>
<resources>
<resourceUsagesByPartition>
Expand Down Expand Up @@ -334,6 +445,153 @@
</resourceInformations>
</userAmLimit>
</resourceUsagesByPartition>
<resourceUsagesByPartition>
<partitionName>root-a-default-label</partitionName>
<used>
<memory>0</memory>
<vCores>0</vCores>
<resourceInformations>
<resourceInformation>
<attributes/>
<maximumAllocation>9223372036854775807</maximumAllocation>
<minimumAllocation>0</minimumAllocation>
<name>memory-mb</name>
<resourceType>COUNTABLE</resourceType>
<units>Mi</units>
<value>0</value>
</resourceInformation>
<resourceInformation>
<attributes/>
<maximumAllocation>9223372036854775807</maximumAllocation>
<minimumAllocation>0</minimumAllocation>
<name>vcores</name>
<resourceType>COUNTABLE</resourceType>
<units/>
<value>0</value>
</resourceInformation>
</resourceInformations>
</used>
<reserved>
<memory>0</memory>
<vCores>0</vCores>
<resourceInformations>
<resourceInformation>
<attributes/>
<maximumAllocation>9223372036854775807</maximumAllocation>
<minimumAllocation>0</minimumAllocation>
<name>memory-mb</name>
<resourceType>COUNTABLE</resourceType>
<units>Mi</units>
<value>0</value>
</resourceInformation>
<resourceInformation>
<attributes/>
<maximumAllocation>9223372036854775807</maximumAllocation>
<minimumAllocation>0</minimumAllocation>
<name>vcores</name>
<resourceType>COUNTABLE</resourceType>
<units/>
<value>0</value>
</resourceInformation>
</resourceInformations>
</reserved>
<pending>
<memory>0</memory>
<vCores>0</vCores>
<resourceInformations>
<resourceInformation>
<attributes/>
<maximumAllocation>9223372036854775807</maximumAllocation>
<minimumAllocation>0</minimumAllocation>
<name>memory-mb</name>
<resourceType>COUNTABLE</resourceType>
<units>Mi</units>
<value>0</value>
</resourceInformation>
<resourceInformation>
<attributes/>
<maximumAllocation>9223372036854775807</maximumAllocation>
<minimumAllocation>0</minimumAllocation>
<name>vcores</name>
<resourceType>COUNTABLE</resourceType>
<units/>
<value>0</value>
</resourceInformation>
</resourceInformations>
</pending>
<amUsed>
<memory>0</memory>
<vCores>0</vCores>
<resourceInformations>
<resourceInformation>
<attributes/>
<maximumAllocation>9223372036854775807</maximumAllocation>
<minimumAllocation>0</minimumAllocation>
<name>memory-mb</name>
<resourceType>COUNTABLE</resourceType>
<units>Mi</units>
<value>0</value>
</resourceInformation>
<resourceInformation>
<attributes/>
<maximumAllocation>9223372036854775807</maximumAllocation>
<minimumAllocation>0</minimumAllocation>
<name>vcores</name>
<resourceType>COUNTABLE</resourceType>
<units/>
<value>0</value>
</resourceInformation>
</resourceInformations>
</amUsed>
<amLimit>
<memory>0</memory>
<vCores>0</vCores>
<resourceInformations>
<resourceInformation>
<attributes/>
<maximumAllocation>9223372036854775807</maximumAllocation>
<minimumAllocation>0</minimumAllocation>
<name>memory-mb</name>
<resourceType>COUNTABLE</resourceType>
<units>Mi</units>
<value>0</value>
</resourceInformation>
<resourceInformation>
<attributes/>
<maximumAllocation>9223372036854775807</maximumAllocation>
<minimumAllocation>0</minimumAllocation>
<name>vcores</name>
<resourceType>COUNTABLE</resourceType>
<units/>
<value>0</value>
</resourceInformation>
</resourceInformations>
</amLimit>
<userAmLimit>
<memory>0</memory>
<vCores>0</vCores>
<resourceInformations>
<resourceInformation>
<attributes/>
<maximumAllocation>9223372036854775807</maximumAllocation>
<minimumAllocation>0</minimumAllocation>
<name>memory-mb</name>
<resourceType>COUNTABLE</resourceType>
<units>Mi</units>
<value>0</value>
</resourceInformation>
<resourceInformation>
<attributes/>
<maximumAllocation>9223372036854775807</maximumAllocation>
<minimumAllocation>0</minimumAllocation>
<name>vcores</name>
<resourceType>COUNTABLE</resourceType>
<units/>
<value>0</value>
</resourceInformation>
</resourceInformations>
</userAmLimit>
</resourceUsagesByPartition>
</resources>
<minEffectiveCapacity>
<memory>4096</memory>
Expand Down
Loading

0 comments on commit 3999be8

Please sign in to comment.