Skip to content

Commit f097d68

Browse files
ASHISH-RANJAN59ASHISH-RANJAN59
authored
YARN-11808. RM memory leak due to Opportunistic container request cancellation at App level. (#7583)
1 parent fbb049a commit f097d68

File tree

2 files changed

+204
-2
lines changed

2 files changed

+204
-2
lines changed

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/OpportunisticContainerContext.java‎

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
import java.util.Map;
3737
import java.util.Set;
3838
import java.util.TreeMap;
39+
import java.util.Objects;
3940

4041
import static org.apache.hadoop.yarn.server.scheduler.OpportunisticContainerAllocator.Allocation;
4142
import static org.apache.hadoop.yarn.server.scheduler.OpportunisticContainerAllocator.AllocationParams;
@@ -132,10 +133,25 @@ public void addToOutstandingReqs(List<ResourceRequest> resourceAsks) {
132133
SchedulerRequestKey schedulerKey = SchedulerRequestKey.create(request);
133134

134135
Map<Resource, EnrichedResourceRequest> reqMap =
135-
outstandingOpReqs.get(schedulerKey);
136+
getOutstandingOpReqs().get(schedulerKey);
137+
138+
if (request.getNumContainers() == 0) {
139+
if (Objects.nonNull(reqMap) &&
140+
ResourceRequest.isAnyLocation(request.getResourceName())) {
141+
reqMap.remove(request.getCapability());
142+
if (reqMap.isEmpty()) {
143+
outstandingOpReqs.remove(schedulerKey);
144+
}
145+
continue;
146+
} else if (Objects.isNull(reqMap) || Objects.isNull(
147+
reqMap.get(request.getCapability()))) {
148+
continue;
149+
}
150+
}
151+
136152
if (reqMap == null) {
137153
reqMap = new HashMap<>();
138-
outstandingOpReqs.put(schedulerKey, reqMap);
154+
getOutstandingOpReqs().put(schedulerKey, reqMap);
139155
}
140156

141157
EnrichedResourceRequest eReq = reqMap.get(request.getCapability());

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/scheduler/TestOpportunisticContainerContext.java‎

Lines changed: 186 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,186 @@
1+
/**
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
* <p>
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
* <p>
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
package org.apache.hadoop.yarn.server.scheduler;
19+
20+
import org.apache.hadoop.yarn.api.records.Priority;
21+
import org.apache.hadoop.yarn.api.records.Resource;
22+
import org.apache.hadoop.yarn.api.records.ResourceRequest;
23+
24+
import org.junit.Assert;
25+
import org.junit.Before;
26+
import org.junit.Test;
27+
import org.mockito.Mockito;
28+
import org.mockito.Spy;
29+
30+
import java.util.HashMap;
31+
import java.util.List;
32+
import java.util.Map;
33+
import java.util.TreeMap;
34+
import java.util.ArrayList;
35+
36+
/**
37+
* Test cases for OpportunisticContainerContext.
38+
*/
39+
public class TestOpportunisticContainerContext {
40+
41+
@Spy
42+
private OpportunisticContainerContext opportunisticContainerContext;
43+
private Map<Resource, OpportunisticContainerAllocator.EnrichedResourceRequest> reqMap =
44+
new HashMap<>();
45+
private TreeMap<SchedulerRequestKey,
46+
Map<Resource, OpportunisticContainerAllocator.EnrichedResourceRequest>> outstandingOpReqs;
47+
48+
@Before
49+
public void setUp() {
50+
opportunisticContainerContext = Mockito.spy(new OpportunisticContainerContext());
51+
outstandingOpReqs = new TreeMap<>();
52+
}
53+
54+
/**
55+
* Resource Request - {
56+
* Location = ANY
57+
* No of container != 0
58+
* }.
59+
*/
60+
@Test
61+
public void testAddToOutstandingReqsWithANYRequest() {
62+
ResourceRequest request = getResourceRequest(ResourceRequest.ANY, 1);
63+
List<ResourceRequest> resourceRequestList = new ArrayList<>();
64+
resourceRequestList.add(request);
65+
opportunisticContainerContext.addToOutstandingReqs(resourceRequestList);
66+
Assert.assertEquals(opportunisticContainerContext.getOutstandingOpReqs().size(), 1);
67+
}
68+
69+
/**
70+
* Resource Request - {
71+
* Location != ANY
72+
* No of container = 0
73+
* }.
74+
*/
75+
@Test
76+
public void testAddToOutstandingReqsWithZeroContainer() {
77+
ResourceRequest request = getResourceRequest("resource", 0);
78+
createOutstandingOpReqs(request, getResource());
79+
Mockito.doReturn(outstandingOpReqs).when(opportunisticContainerContext)
80+
.getOutstandingOpReqs();
81+
List<ResourceRequest> resourceRequestList = new ArrayList<>();
82+
resourceRequestList.add(request);
83+
opportunisticContainerContext.addToOutstandingReqs(resourceRequestList);
84+
Assert.assertEquals(opportunisticContainerContext.getOutstandingOpReqs().size(), 1);
85+
}
86+
87+
/**
88+
* Resource Request - [
89+
* {Location != ANY, No of Container = 0}
90+
* {Location = ANY, No of Container = 0}
91+
* ].
92+
*/
93+
@Test
94+
public void testAddToOutstandingReqsWithZeroContainerAndMultipleSchedulerKey() {
95+
ResourceRequest req1 = getResourceRequest("resource", 0);
96+
ResourceRequest req2 = getResourceRequest(ResourceRequest.ANY, 0);
97+
createOutstandingOpReqs(req1, getResource());
98+
createOutstandingOpReqs(req2, getResource());
99+
Mockito.doReturn(outstandingOpReqs).when(opportunisticContainerContext)
100+
.getOutstandingOpReqs();
101+
List<ResourceRequest> resourceRequestList = new ArrayList<>();
102+
resourceRequestList.add(req1);
103+
resourceRequestList.add(req2);
104+
opportunisticContainerContext.addToOutstandingReqs(resourceRequestList);
105+
Assert.assertEquals(opportunisticContainerContext.getOutstandingOpReqs().size(), 1);
106+
}
107+
108+
/**
109+
* Resource Request - [
110+
* {Location != ANY, No of Container = 0}
111+
* {Location = ANY, No of Container != 0}
112+
* ].
113+
*/
114+
@Test
115+
public void testAddToOutstandingReqsWithMultipleSchedulerKey() {
116+
ResourceRequest req1 = getResourceRequest("resource", 0);
117+
ResourceRequest req2 = getResourceRequest(ResourceRequest.ANY, 1);
118+
createOutstandingOpReqs(req1, getResource());
119+
createOutstandingOpReqs(req2, getResource());
120+
Mockito.doReturn(outstandingOpReqs).when(opportunisticContainerContext)
121+
.getOutstandingOpReqs();
122+
List<ResourceRequest> resourceRequestList = new ArrayList<>();
123+
resourceRequestList.add(req1);
124+
resourceRequestList.add(req2);
125+
opportunisticContainerContext.addToOutstandingReqs(resourceRequestList);
126+
Assert.assertEquals(opportunisticContainerContext.getOutstandingOpReqs().size(), 1);
127+
}
128+
129+
/**
130+
* Resource Request - {
131+
* Location != ANY
132+
* No of container = 0
133+
* Capability = NULL
134+
* }.
135+
*/
136+
@Test
137+
public void testAddToOutstandingReqsWithZeroContainerAndNullCapability() {
138+
ResourceRequest request = getResourceRequestWithoutCapability();
139+
createOutstandingOpReqs(request, getResource());
140+
Mockito.doReturn(outstandingOpReqs).when(opportunisticContainerContext)
141+
.getOutstandingOpReqs();
142+
List<ResourceRequest> resourceRequestList = new ArrayList<>();
143+
resourceRequestList.add(request);
144+
opportunisticContainerContext.addToOutstandingReqs(resourceRequestList);
145+
Assert.assertEquals(opportunisticContainerContext.getOutstandingOpReqs().size(), 1);
146+
}
147+
148+
/**
149+
* Resource Request - {
150+
* Location != ANY
151+
* No of container = 0
152+
* Req map is NULL
153+
* }.
154+
*/
155+
@Test
156+
public void testAddToOutstandingReqsWithEmptyReqMap() {
157+
ResourceRequest request = getResourceRequest("resource", 0);
158+
Mockito.doReturn(new TreeMap<>()).when(opportunisticContainerContext)
159+
.getOutstandingOpReqs();
160+
List<ResourceRequest> resourceRequestList = new ArrayList<>();
161+
resourceRequestList.add(request);
162+
opportunisticContainerContext.addToOutstandingReqs(resourceRequestList);
163+
Assert.assertEquals(opportunisticContainerContext.getOutstandingOpReqs().size(), 0);
164+
}
165+
166+
private void createOutstandingOpReqs(ResourceRequest req, Resource resource) {
167+
SchedulerRequestKey schedulerRequestKey = SchedulerRequestKey.create(req);
168+
reqMap.put(resource, new OpportunisticContainerAllocator.EnrichedResourceRequest(req));
169+
outstandingOpReqs.put(schedulerRequestKey, reqMap);
170+
}
171+
172+
private ResourceRequest getResourceRequest(String resourceName, int numContainer) {
173+
return ResourceRequest.newBuilder().resourceName(resourceName).numContainers(numContainer)
174+
.allocationRequestId(1).priority(Priority.newInstance(1)).capability(getResource())
175+
.build();
176+
}
177+
178+
private ResourceRequest getResourceRequestWithoutCapability() {
179+
return ResourceRequest.newBuilder().resourceName("resource").numContainers(0)
180+
.allocationRequestId(1).priority(Priority.newInstance(1)).build();
181+
}
182+
183+
private Resource getResource() {
184+
return Resource.newInstance(1024, 2);
185+
}
186+
}

0 commit comments

Comments
 (0)