Skip to content

Commit 133e8aa

Browse files
slfan1989slfan1989
authored
YARN-11227. [Federation] Add getAppTimeout, getAppTimeouts, updateApplicationTimeout REST APIs for Router. (#4715)
1 parent ffa9ed9 commit 133e8aa

File tree

4 files changed

+273
-15
lines changed

4 files changed

+273
-15
lines changed

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppTimeoutInfo.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import javax.xml.bind.annotation.XmlElement;
2323
import javax.xml.bind.annotation.XmlRootElement;
2424

25+
import org.apache.hadoop.yarn.api.records.ApplicationTimeout;
2526
import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType;
2627

2728
/**
@@ -45,6 +46,12 @@ public AppTimeoutInfo() {
4546
remainingTimeInSec = -1;
4647
}
4748

49+
public AppTimeoutInfo(ApplicationTimeout applicationTimeout) {
50+
this.expiryTime = applicationTimeout.getExpiryTime();
51+
this.remainingTimeInSec = applicationTimeout.getRemainingTime();
52+
this.timeoutType = applicationTimeout.getTimeoutType();
53+
}
54+
4855
public ApplicationTimeoutType getTimeoutType() {
4956
return timeoutType;
5057
}

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationInterceptorREST.java‎

Lines changed: 61 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1342,41 +1342,96 @@ public Response listReservation(String queue, String reservationId,
13421342
@Override
13431343
public AppTimeoutInfo getAppTimeout(HttpServletRequest hsr, String appId,
13441344
String type) throws AuthorizationException {
1345-
throw new NotImplementedException("Code is not implemented");
1345+
1346+
if (appId == null || appId.isEmpty()) {
1347+
throw new IllegalArgumentException("Parameter error, the appId is empty or null.");
1348+
}
1349+
1350+
if (type == null || type.isEmpty()) {
1351+
throw new IllegalArgumentException("Parameter error, the type is empty or null.");
1352+
}
1353+
1354+
try {
1355+
SubClusterInfo subClusterInfo = getHomeSubClusterInfoByAppId(appId);
1356+
DefaultRequestInterceptorREST interceptor = getOrCreateInterceptorForSubCluster(
1357+
subClusterInfo.getSubClusterId(), subClusterInfo.getRMWebServiceAddress());
1358+
return interceptor.getAppTimeout(hsr, appId, type);
1359+
} catch (IllegalArgumentException e) {
1360+
RouterServerUtil.logAndThrowRunTimeException(e,
1361+
"Unable to get the getAppTimeout appId: %s.", appId);
1362+
} catch (YarnException e) {
1363+
RouterServerUtil.logAndThrowRunTimeException("getAppTimeout Failed.", e);
1364+
}
1365+
return null;
13461366
}
13471367

13481368
@Override
13491369
public AppTimeoutsInfo getAppTimeouts(HttpServletRequest hsr, String appId)
13501370
throws AuthorizationException {
1351-
throw new NotImplementedException("Code is not implemented");
1371+
1372+
if (appId == null || appId.isEmpty()) {
1373+
throw new IllegalArgumentException("Parameter error, the appId is empty or null.");
1374+
}
1375+
1376+
try {
1377+
SubClusterInfo subClusterInfo = getHomeSubClusterInfoByAppId(appId);
1378+
DefaultRequestInterceptorREST interceptor = getOrCreateInterceptorForSubCluster(
1379+
subClusterInfo.getSubClusterId(), subClusterInfo.getRMWebServiceAddress());
1380+
return interceptor.getAppTimeouts(hsr, appId);
1381+
} catch (IllegalArgumentException e) {
1382+
RouterServerUtil.logAndThrowRunTimeException(e,
1383+
"Unable to get the getAppTimeouts appId: %s.", appId);
1384+
} catch (YarnException e) {
1385+
RouterServerUtil.logAndThrowRunTimeException("getAppTimeouts Failed.", e);
1386+
}
1387+
return null;
13521388
}
13531389

13541390
@Override
13551391
public Response updateApplicationTimeout(AppTimeoutInfo appTimeout,
13561392
HttpServletRequest hsr, String appId) throws AuthorizationException,
13571393
YarnException, InterruptedException, IOException {
1358-
throw new NotImplementedException("Code is not implemented");
1394+
1395+
if (appId == null || appId.isEmpty()) {
1396+
throw new IllegalArgumentException("Parameter error, the appId is empty or null.");
1397+
}
1398+
1399+
if (appTimeout == null) {
1400+
throw new IllegalArgumentException("Parameter error, the appTimeout is null.");
1401+
}
1402+
1403+
try {
1404+
SubClusterInfo subClusterInfo = getHomeSubClusterInfoByAppId(appId);
1405+
DefaultRequestInterceptorREST interceptor = getOrCreateInterceptorForSubCluster(
1406+
subClusterInfo.getSubClusterId(), subClusterInfo.getRMWebServiceAddress());
1407+
return interceptor.updateApplicationTimeout(appTimeout, hsr, appId);
1408+
} catch (IllegalArgumentException e) {
1409+
RouterServerUtil.logAndThrowRunTimeException(e,
1410+
"Unable to get the updateApplicationTimeout appId: %s.", appId);
1411+
} catch (YarnException e) {
1412+
RouterServerUtil.logAndThrowRunTimeException("updateApplicationTimeout Failed.", e);
1413+
}
1414+
return null;
13591415
}
13601416

13611417
@Override
13621418
public AppAttemptsInfo getAppAttempts(HttpServletRequest hsr, String appId) {
1419+
13631420
if (appId == null || appId.isEmpty()) {
13641421
throw new IllegalArgumentException("Parameter error, the appId is empty or null.");
13651422
}
13661423

13671424
try {
13681425
SubClusterInfo subClusterInfo = getHomeSubClusterInfoByAppId(appId);
1369-
13701426
DefaultRequestInterceptorREST interceptor = getOrCreateInterceptorForSubCluster(
13711427
subClusterInfo.getSubClusterId(), subClusterInfo.getRMWebServiceAddress());
13721428
return interceptor.getAppAttempts(hsr, appId);
13731429
} catch (IllegalArgumentException e) {
13741430
RouterServerUtil.logAndThrowRunTimeException(e,
13751431
"Unable to get the AppAttempt appId: %s.", appId);
13761432
} catch (YarnException e) {
1377-
RouterServerUtil.logAndThrowRunTimeException("getContainer Failed.", e);
1433+
RouterServerUtil.logAndThrowRunTimeException("getAppAttempts Failed.", e);
13781434
}
1379-
13801435
return null;
13811436
}
13821437

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/MockDefaultRequestInterceptorREST.java‎

Lines changed: 124 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@
2121
import java.io.IOException;
2222
import java.net.ConnectException;
2323
import java.util.ArrayList;
24-
import java.util.HashSet;
2524
import java.util.Set;
2625
import java.util.Map;
2726
import java.util.HashMap;
@@ -54,6 +53,8 @@
5453
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
5554
import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
5655
import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
56+
import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType;
57+
import org.apache.hadoop.yarn.api.records.ApplicationTimeout;
5758
import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
5859
import org.apache.hadoop.yarn.exceptions.YarnException;
5960
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
@@ -73,6 +74,8 @@
7374
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeToLabelsInfo;
7475
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeLabelInfo;
7576
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptsInfo;
77+
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppTimeoutInfo;
78+
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppTimeoutsInfo;
7679
import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo;
7780
import org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo;
7881
import org.apache.hadoop.yarn.server.webapp.dao.ContainersInfo;
@@ -93,7 +96,7 @@ public class MockDefaultRequestInterceptorREST
9396
// This property allows us to write tests for specific scenario as YARN RM
9497
// down e.g. network issue, failover.
9598
private boolean isRunning = true;
96-
private HashSet<ApplicationId> applicationMap = new HashSet<>();
99+
private Map<ApplicationId, ApplicationReport> applicationMap = new HashMap<>();
97100
public static final String APP_STATE_RUNNING = "RUNNING";
98101

99102
private void validateRunning() throws ConnectException {
@@ -123,7 +126,22 @@ public Response submitApplication(ApplicationSubmissionContextInfo newApp,
123126

124127
ApplicationId appId = ApplicationId.fromString(newApp.getApplicationId());
125128
LOG.info("Application submitted: " + appId);
126-
applicationMap.add(appId);
129+
130+
// Initialize appReport
131+
ApplicationReport appReport = ApplicationReport.newInstance(
132+
appId, ApplicationAttemptId.newInstance(appId, 1), null, newApp.getQueue(), null, null, 0,
133+
null, YarnApplicationState.ACCEPTED, "", null, 0, 0, null, null, null, 0, null, null, null,
134+
false, Priority.newInstance(newApp.getPriority()), null, null);
135+
136+
// Initialize appTimeoutsMap
137+
HashMap<ApplicationTimeoutType, ApplicationTimeout> appTimeoutsMap = new HashMap<>();
138+
ApplicationTimeoutType timeoutType = ApplicationTimeoutType.LIFETIME;
139+
ApplicationTimeout appTimeOut =
140+
ApplicationTimeout.newInstance(ApplicationTimeoutType.LIFETIME, "UNLIMITED", 10);
141+
appTimeoutsMap.put(timeoutType, appTimeOut);
142+
appReport.setApplicationTimeouts(appTimeoutsMap);
143+
144+
applicationMap.put(appId, appReport);
127145
return Response.status(Status.ACCEPTED).header(HttpHeaders.LOCATION, "")
128146
.entity(getSubClusterId()).build();
129147
}
@@ -136,7 +154,7 @@ public AppInfo getApp(HttpServletRequest hsr, String appId,
136154
}
137155

138156
ApplicationId applicationId = ApplicationId.fromString(appId);
139-
if (!applicationMap.contains(applicationId)) {
157+
if (!applicationMap.containsKey(applicationId)) {
140158
throw new NotFoundException("app with id: " + appId + " not found");
141159
}
142160

@@ -171,7 +189,7 @@ public Response updateAppState(AppState targetState, HttpServletRequest hsr,
171189
validateRunning();
172190

173191
ApplicationId applicationId = ApplicationId.fromString(appId);
174-
if (!applicationMap.remove(applicationId)) {
192+
if (applicationMap.remove(applicationId) == null) {
175193
throw new ApplicationNotFoundException(
176194
"Trying to kill an absent application: " + appId);
177195
}
@@ -244,7 +262,7 @@ public AppState getAppState(HttpServletRequest hsr, String appId)
244262
}
245263

246264
ApplicationId applicationId = ApplicationId.fromString(appId);
247-
if (!applicationMap.contains(applicationId)) {
265+
if (!applicationMap.containsKey(applicationId)) {
248266
throw new NotFoundException("app with id: " + appId + " not found");
249267
}
250268

@@ -428,7 +446,7 @@ public AppAttemptInfo getAppAttempt(HttpServletRequest req, HttpServletResponse
428446
}
429447

430448
ApplicationId applicationId = ApplicationId.fromString(appId);
431-
if (!applicationMap.contains(applicationId)) {
449+
if (!applicationMap.containsKey(applicationId)) {
432450
throw new NotFoundException("app with id: " + appId + " not found");
433451
}
434452

@@ -454,7 +472,7 @@ public AppAttemptsInfo getAppAttempts(HttpServletRequest hsr, String appId) {
454472
}
455473

456474
ApplicationId applicationId = ApplicationId.fromString(appId);
457-
if (!applicationMap.contains(applicationId)) {
475+
if (!applicationMap.containsKey(applicationId)) {
458476
throw new NotFoundException("app with id: " + appId + " not found");
459477
}
460478

@@ -463,4 +481,102 @@ public AppAttemptsInfo getAppAttempts(HttpServletRequest hsr, String appId) {
463481
infos.add(TestRouterWebServiceUtil.generateAppAttemptInfo(1));
464482
return infos;
465483
}
484+
485+
@Override
486+
public AppTimeoutInfo getAppTimeout(HttpServletRequest hsr,
487+
String appId, String type) throws AuthorizationException {
488+
489+
if (!isRunning) {
490+
throw new RuntimeException("RM is stopped");
491+
}
492+
493+
ApplicationId applicationId = ApplicationId.fromString(appId);
494+
if (!applicationMap.containsKey(applicationId)) {
495+
throw new NotFoundException("app with id: " + appId + " not found");
496+
}
497+
498+
ApplicationReport appReport = applicationMap.get(applicationId);
499+
Map<ApplicationTimeoutType, ApplicationTimeout> timeouts = appReport.getApplicationTimeouts();
500+
ApplicationTimeoutType paramType = ApplicationTimeoutType.valueOf(type);
501+
502+
if (paramType == null) {
503+
throw new NotFoundException("application timeout type not found");
504+
}
505+
506+
if (!timeouts.containsKey(paramType)) {
507+
throw new NotFoundException("timeout with id: " + appId + " not found");
508+
}
509+
510+
ApplicationTimeout applicationTimeout = timeouts.get(paramType);
511+
512+
AppTimeoutInfo timeoutInfo = new AppTimeoutInfo();
513+
timeoutInfo.setExpiryTime(applicationTimeout.getExpiryTime());
514+
timeoutInfo.setTimeoutType(applicationTimeout.getTimeoutType());
515+
timeoutInfo.setRemainingTime(applicationTimeout.getRemainingTime());
516+
517+
return timeoutInfo;
518+
}
519+
520+
@Override
521+
public AppTimeoutsInfo getAppTimeouts(HttpServletRequest hsr, String appId)
522+
throws AuthorizationException {
523+
524+
if (!isRunning) {
525+
throw new RuntimeException("RM is stopped");
526+
}
527+
528+
ApplicationId applicationId = ApplicationId.fromString(appId);
529+
530+
if (!applicationMap.containsKey(applicationId)) {
531+
throw new NotFoundException("app with id: " + appId + " not found");
532+
}
533+
534+
ApplicationReport appReport = applicationMap.get(applicationId);
535+
Map<ApplicationTimeoutType, ApplicationTimeout> timeouts = appReport.getApplicationTimeouts();
536+
537+
AppTimeoutsInfo timeoutsInfo = new AppTimeoutsInfo();
538+
539+
for (ApplicationTimeout timeout : timeouts.values()) {
540+
AppTimeoutInfo timeoutInfo = new AppTimeoutInfo();
541+
timeoutInfo.setExpiryTime(timeout.getExpiryTime());
542+
timeoutInfo.setTimeoutType(timeout.getTimeoutType());
543+
timeoutInfo.setRemainingTime(timeout.getRemainingTime());
544+
timeoutsInfo.add(timeoutInfo);
545+
}
546+
547+
return timeoutsInfo;
548+
}
549+
550+
@Override
551+
public Response updateApplicationTimeout(AppTimeoutInfo appTimeout, HttpServletRequest hsr,
552+
String appId) throws AuthorizationException,
553+
YarnException, InterruptedException, IOException {
554+
555+
if (!isRunning) {
556+
throw new RuntimeException("RM is stopped");
557+
}
558+
559+
ApplicationId applicationId = ApplicationId.fromString(appId);
560+
561+
if (!applicationMap.containsKey(applicationId)) {
562+
throw new NotFoundException("app with id: " + appId + " not found");
563+
}
564+
565+
ApplicationReport appReport = applicationMap.get(applicationId);
566+
Map<ApplicationTimeoutType, ApplicationTimeout> timeouts = appReport.getApplicationTimeouts();
567+
568+
ApplicationTimeoutType paramTimeoutType = appTimeout.getTimeoutType();
569+
if (!timeouts.containsKey(paramTimeoutType)) {
570+
throw new NotFoundException("TimeOutType with id: " + appId + " not found");
571+
}
572+
573+
ApplicationTimeout applicationTimeout = timeouts.get(paramTimeoutType);
574+
applicationTimeout.setTimeoutType(appTimeout.getTimeoutType());
575+
applicationTimeout.setExpiryTime(appTimeout.getExpireTime());
576+
applicationTimeout.setRemainingTime(appTimeout.getRemainingTimeInSec());
577+
578+
AppTimeoutInfo result = new AppTimeoutInfo(applicationTimeout);
579+
580+
return Response.status(Status.OK).entity(result).build();
581+
}
466582
}

0 commit comments

Comments
 (0)