From 9f68fc2ea550d84e1790e64c48362f228287cd60 Mon Sep 17 00:00:00 2001 From: Peter Szucs Date: Tue, 29 Nov 2022 15:23:19 +0100 Subject: [PATCH] YARN-10946. Refactorings in QueueInfo creation Change-Id: I9202e4125fa4b35791db6c443e85d04f813953a7 --- .../scheduler/capacity/AbstractCSQueue.java | 28 ++--------- .../capacity/CSQueueInfoProvider.java | 49 ++++++++++++------- 2 files changed, 34 insertions(+), 43 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java index 9912237060ac9..7a3ec4f6f4e5e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java @@ -198,28 +198,16 @@ public float getCapacity() { return queueCapacities.getCapacity(); } - public float getCapacityByNodeLabel(String nodeLabel) { - return queueCapacities.getCapacity(nodeLabel); - } - @Override public float getAbsoluteCapacity() { return queueCapacities.getAbsoluteCapacity(); } - public float getAbsoluteCapacityByNodeLabel(String nodeLabel) { - return queueCapacities.getAbsoluteCapacity(nodeLabel); - } - @Override public float getAbsoluteMaximumCapacity() { return queueCapacities.getAbsoluteMaximumCapacity(); } - public float getAbsoluteMaximumCapacityByNodeLabel(String nodeLabel) { - return queueCapacities.getAbsoluteMaximumCapacity(nodeLabel); - } - @Override public float getAbsoluteUsedCapacity() { return queueCapacities.getAbsoluteUsedCapacity(); @@ -230,23 +218,11 @@ public float getMaximumCapacity() { return queueCapacities.getMaximumCapacity(); } - public float getMaximumCapacityByNodeLabel(String nodeLabel) { - return queueCapacities.getMaximumCapacity(nodeLabel); - } - - public float getMaxAMResourcePercentageByNodeLabel(String nodeLabel) { - return queueCapacities.getMaxAMResourcePercentage(nodeLabel); - } - @Override public float getUsedCapacity() { return queueCapacities.getUsedCapacity(); } - public float getWeight() { - return queueCapacities.getWeight(); - } - @Override public Resource getUsedResources() { return usageTracker.getQueueUsage().getUsed(); @@ -602,6 +578,10 @@ public QueueCapacityVector getConfiguredCapacityVector( } protected QueueInfo getQueueInfo() { + // Deliberately doesn't use lock here, because this method will be invoked + // from schedulerApplicationAttempt, to avoid deadlock, sacrifice + // consistency here. + // TODO, improve this return CSQueueInfoProvider.getQueueInfo(this); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueInfoProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueInfoProvider.java index 008ee19cfe7be..8daca2bc26be6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueInfoProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueInfoProvider.java @@ -1,3 +1,21 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; import org.apache.hadoop.yarn.api.records.QueueConfigurations; @@ -13,18 +31,14 @@ public final class CSQueueInfoProvider { - private static final RecordFactory recordFactory = + private static final RecordFactory RECORD_FACTORY = RecordFactoryProvider.getRecordFactory(null); - public CSQueueInfoProvider() { + private CSQueueInfoProvider() { } public static QueueInfo getQueueInfo(AbstractCSQueue csQueue) { - // Deliberately doesn't use lock here, because this method will be invoked - // from schedulerApplicationAttempt, to avoid deadlock, sacrifice - // consistency here. - // TODO, improve this - QueueInfo queueInfo = recordFactory.newRecordInstance(QueueInfo.class); + QueueInfo queueInfo = RECORD_FACTORY.newRecordInstance(QueueInfo.class); queueInfo.setQueueName(csQueue.getQueuePathObject().getLeafName()); queueInfo.setQueuePath(csQueue.getQueuePathObject().getFullPath()); queueInfo.setAccessibleNodeLabels(csQueue.getAccessibleNodeLabels()); @@ -38,17 +52,13 @@ public static QueueInfo getQueueInfo(AbstractCSQueue csQueue) { queueInfo.setIntraQueuePreemptionDisabled( csQueue.getIntraQueuePreemptionDisabled()); queueInfo.setQueueConfigurations(getQueueConfigurations(csQueue)); - queueInfo.setWeight(csQueue.getWeight()); + queueInfo.setWeight(csQueue.getQueueCapacities().getWeight()); queueInfo.setMaxParallelApps(csQueue.getMaxParallelApps()); return queueInfo; } private static QueueStatistics getQueueStatistics(AbstractCSQueue csQueue) { - // Deliberately doesn't use lock here, because this method will be invoked - // from schedulerApplicationAttempt, to avoid deadlock, sacrifice - // consistency here. - // TODO, improve this - QueueStatistics stats = recordFactory.newRecordInstance( + QueueStatistics stats = RECORD_FACTORY.newRecordInstance( QueueStatistics.class); CSQueueMetrics queueMetrics = csQueue.getMetrics(); stats.setNumAppsSubmitted(queueMetrics.getAppsSubmitted()); @@ -78,14 +88,15 @@ private static Map getQueueConfigurations(AbstractC QueueResourceQuotas queueResourceQuotas = csQueue.getQueueResourceQuotas(); for (String nodeLabel : nodeLabels) { QueueConfigurations queueConfiguration = - recordFactory.newRecordInstance(QueueConfigurations.class); - float capacity = csQueue.getCapacityByNodeLabel(nodeLabel); - float absoluteCapacity = csQueue.getAbsoluteCapacityByNodeLabel(nodeLabel); - float maxCapacity = csQueue.getMaximumCapacityByNodeLabel(nodeLabel); + RECORD_FACTORY.newRecordInstance(QueueConfigurations.class); + QueueCapacities queueCapacities = csQueue.getQueueCapacities(); + float capacity = queueCapacities.getCapacity(nodeLabel); + float absoluteCapacity = queueCapacities.getAbsoluteCapacity(nodeLabel); + float maxCapacity = queueCapacities.getMaximumCapacity(nodeLabel); float absMaxCapacity = - csQueue.getAbsoluteMaximumCapacityByNodeLabel(nodeLabel); + queueCapacities.getAbsoluteMaximumCapacity(nodeLabel); float maxAMPercentage = - csQueue.getMaxAMResourcePercentageByNodeLabel(nodeLabel); + queueCapacities.getMaxAMResourcePercentage(nodeLabel); queueConfiguration.setCapacity(capacity); queueConfiguration.setAbsoluteCapacity(absoluteCapacity); queueConfiguration.setMaxCapacity(maxCapacity);