Skip to content

Commit b2bca94

Browse files
slfan1989pull[bot]
slfan1989
authored and
pull[bot]
committed
YARN-11580. YARN Router Web supports displaying information for Non-Federation. (apache#6127) Contributed by Shilun Fan.
Reviewed-by: Inigo Goiri <inigoiri@apache.org> Signed-off-by: Shilun Fan <slfan1989@apache.org>
1 parent dc245b2 commit b2bca94

File tree

12 files changed

+439
-140
lines changed

12 files changed

+439
-140
lines changed

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterInfo.java

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,13 @@ public static SubClusterInfo newInstance(SubClusterId subClusterId,
7676
return subClusterInfo;
7777
}
7878

79+
public static SubClusterInfo newInstance(SubClusterId subClusterId,
80+
String rmWebServiceAddress, SubClusterState state, long lastStartTime, long lastHeartBeat,
81+
String capability) {
82+
return newInstance(subClusterId, null, null, null,
83+
rmWebServiceAddress, lastHeartBeat, state, lastStartTime, capability);
84+
}
85+
7986
/**
8087
* Get the {@link SubClusterId} representing the unique identifier of the
8188
* subcluster.

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterServerUtil.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -661,6 +661,7 @@ public static ReservationDefinition convertReservationDefinition(
661661
* - if its size is within limits.
662662
*
663663
* @param appContext the app context to check.
664+
* @param conf Configuration.
664665
* @throws IOException if an IO error occurred.
665666
* @throws YarnException yarn exception.
666667
*/

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AppsBlock.java

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -94,11 +94,13 @@ private static String escape(String str) {
9494
}
9595

9696
private AppsInfo getYarnFederationAppsInfo(boolean isEnabled) {
97+
String webAddress = null;
9798
if (isEnabled) {
98-
String webAddress = WebAppUtils.getRouterWebAppURLWithScheme(this.conf);
99-
return getSubClusterAppsInfoByWebAddress(webAddress, StringUtils.EMPTY);
99+
webAddress = WebAppUtils.getRouterWebAppURLWithScheme(this.conf);
100+
} else {
101+
webAddress = WebAppUtils.getRMWebAppURLWithScheme(this.conf);
100102
}
101-
return null;
103+
return getSubClusterAppsInfoByWebAddress(webAddress, StringUtils.EMPTY);
102104
}
103105

104106
private AppsInfo getSubClusterAppsInfo(String subCluster, String states) {
@@ -110,7 +112,7 @@ private AppsInfo getSubClusterAppsInfo(String subCluster, String states) {
110112
if (subClusterInfo != null) {
111113
// Prepare webAddress
112114
String webAddress = subClusterInfo.getRMWebServiceAddress();
113-
String herfWebAppAddress = "";
115+
String herfWebAppAddress;
114116
if (webAddress != null && !webAddress.isEmpty()) {
115117
herfWebAppAddress = WebAppUtils.getHttpSchemePrefix(conf) + webAddress;
116118
return getSubClusterAppsInfoByWebAddress(herfWebAppAddress, states);

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationBlock.java

Lines changed: 129 additions & 74 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
import java.util.Date;
2727

2828
import com.google.gson.Gson;
29+
import org.apache.hadoop.conf.Configuration;
2930
import org.apache.hadoop.util.StringUtils;
3031
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
3132
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
@@ -67,7 +68,7 @@ public void render(Block html) {
6768
* @param capability metric json obtained from RM.
6869
* @return ClusterMetricsInfo Object
6970
*/
70-
private ClusterMetricsInfo getClusterMetricsInfo(String capability) {
71+
protected ClusterMetricsInfo getClusterMetricsInfo(String capability) {
7172
try {
7273
if (capability != null && !capability.isEmpty()) {
7374
JSONJAXBContext jc = new JSONJAXBContext(
@@ -125,79 +126,10 @@ private void initHtmlPageFederation(Block html, boolean isEnabled) {
125126
.__().__().tbody();
126127

127128
try {
128-
129-
// Sort the SubClusters
130-
List<SubClusterInfo> subclusters = getSubClusterInfoList();
131-
132-
for (SubClusterInfo subcluster : subclusters) {
133-
134-
Map<String, String> subclusterMap = new HashMap<>();
135-
136-
// Prepare subCluster
137-
SubClusterId subClusterId = subcluster.getSubClusterId();
138-
String subClusterIdText = subClusterId.getId();
139-
140-
// Prepare WebAppAddress
141-
String webAppAddress = subcluster.getRMWebServiceAddress();
142-
String herfWebAppAddress = "";
143-
if (webAppAddress != null && !webAppAddress.isEmpty()) {
144-
herfWebAppAddress =
145-
WebAppUtils.getHttpSchemePrefix(this.router.getConfig()) + webAppAddress;
146-
}
147-
148-
// Prepare Capability
149-
String capability = subcluster.getCapability();
150-
ClusterMetricsInfo subClusterInfo = getClusterMetricsInfo(capability);
151-
152-
// Prepare LastStartTime & LastHeartBeat
153-
Date lastStartTime = new Date(subcluster.getLastStartTime());
154-
Date lastHeartBeat = new Date(subcluster.getLastHeartBeat());
155-
156-
// Prepare Resource
157-
long totalMB = subClusterInfo.getTotalMB();
158-
String totalMBDesc = StringUtils.byteDesc(totalMB * BYTES_IN_MB);
159-
long totalVirtualCores = subClusterInfo.getTotalVirtualCores();
160-
String resources = String.format("<memory:%s, vCores:%s>", totalMBDesc, totalVirtualCores);
161-
162-
// Prepare Node
163-
long totalNodes = subClusterInfo.getTotalNodes();
164-
long activeNodes = subClusterInfo.getActiveNodes();
165-
String nodes = String.format("<totalNodes:%s, activeNodes:%s>", totalNodes, activeNodes);
166-
167-
// Prepare HTML Table
168-
String stateStyle = "color:#dc3545;font-weight:bolder";
169-
SubClusterState state = subcluster.getState();
170-
if (SubClusterState.SC_RUNNING == state) {
171-
stateStyle = "color:#28a745;font-weight:bolder";
172-
}
173-
174-
tbody.tr().$id(subClusterIdText)
175-
.td().$class("details-control").a(herfWebAppAddress, subClusterIdText).__()
176-
.td().$style(stateStyle).__(state.name()).__()
177-
.td().__(lastStartTime).__()
178-
.td().__(lastHeartBeat).__()
179-
.td(resources)
180-
.td(nodes)
181-
.__();
182-
183-
// Formatted memory information
184-
long allocatedMB = subClusterInfo.getAllocatedMB();
185-
String allocatedMBDesc = StringUtils.byteDesc(allocatedMB * BYTES_IN_MB);
186-
long availableMB = subClusterInfo.getAvailableMB();
187-
String availableMBDesc = StringUtils.byteDesc(availableMB * BYTES_IN_MB);
188-
long pendingMB = subClusterInfo.getPendingMB();
189-
String pendingMBDesc = StringUtils.byteDesc(pendingMB * BYTES_IN_MB);
190-
long reservedMB = subClusterInfo.getReservedMB();
191-
String reservedMBDesc = StringUtils.byteDesc(reservedMB * BYTES_IN_MB);
192-
193-
subclusterMap.put("totalmemory", totalMBDesc);
194-
subclusterMap.put("allocatedmemory", allocatedMBDesc);
195-
subclusterMap.put("availablememory", availableMBDesc);
196-
subclusterMap.put("pendingmemory", pendingMBDesc);
197-
subclusterMap.put("reservedmemory", reservedMBDesc);
198-
subclusterMap.put("subcluster", subClusterId.getId());
199-
subclusterMap.put("capability", capability);
200-
lists.add(subclusterMap);
129+
if (isEnabled) {
130+
initSubClusterPage(tbody, lists);
131+
} else {
132+
initLocalClusterPage(tbody, lists);
201133
}
202134
} catch (Exception e) {
203135
LOG.error("Cannot render Router Federation.", e);
@@ -210,4 +142,127 @@ private void initHtmlPageFederation(Block html, boolean isEnabled) {
210142
tbody.__().__().div().p().$style("color:red")
211143
.__("*The application counts are local per subcluster").__().__();
212144
}
145+
146+
/**
147+
* Initialize the Federation page of the local-cluster.
148+
*
149+
* @param tbody HTML tbody.
150+
* @param lists subCluster page data list.
151+
*/
152+
private void initLocalClusterPage(TBODY<TABLE<Hamlet>> tbody, List<Map<String, String>> lists) {
153+
Configuration config = this.router.getConfig();
154+
SubClusterInfo localCluster = getSubClusterInfoByLocalCluster(config);
155+
if (localCluster != null) {
156+
try {
157+
initSubClusterPageItem(tbody, localCluster, lists);
158+
} catch (Exception e) {
159+
LOG.error("init LocalCluster = {} page data error.", localCluster, e);
160+
}
161+
}
162+
}
163+
164+
/**
165+
* Initialize the Federation page of the sub-cluster.
166+
*
167+
* @param tbody HTML tbody.
168+
* @param lists subCluster page data list.
169+
*/
170+
private void initSubClusterPage(TBODY<TABLE<Hamlet>> tbody, List<Map<String, String>> lists) {
171+
// Sort the SubClusters
172+
List<SubClusterInfo> subClusters = getSubClusterInfoList();
173+
174+
// Iterate through the sub-clusters and display data for each sub-cluster.
175+
// If a sub-cluster cannot display data, skip it.
176+
for (SubClusterInfo subCluster : subClusters) {
177+
try {
178+
initSubClusterPageItem(tbody, subCluster, lists);
179+
} catch (Exception e) {
180+
LOG.error("init subCluster = {} page data error.", subCluster, e);
181+
}
182+
}
183+
}
184+
185+
/**
186+
* We will initialize the specific SubCluster's data within this method.
187+
*
188+
* @param tbody HTML TBody.
189+
* @param subClusterInfo Sub-cluster information.
190+
* @param lists Used to record data that needs to be displayed in JS.
191+
*/
192+
private void initSubClusterPageItem(TBODY<TABLE<Hamlet>> tbody,
193+
SubClusterInfo subClusterInfo, List<Map<String, String>> lists) {
194+
195+
Map<String, String> subClusterMap = new HashMap<>();
196+
197+
// Prepare subCluster
198+
SubClusterId subClusterId = subClusterInfo.getSubClusterId();
199+
String subClusterIdText = subClusterId.getId();
200+
201+
// Prepare WebAppAddress
202+
String webAppAddress = subClusterInfo.getRMWebServiceAddress();
203+
String herfWebAppAddress = "";
204+
if (webAppAddress != null && !webAppAddress.isEmpty()) {
205+
herfWebAppAddress =
206+
WebAppUtils.getHttpSchemePrefix(this.router.getConfig()) + webAppAddress;
207+
}
208+
209+
// Prepare Capability
210+
String capability = subClusterInfo.getCapability();
211+
ClusterMetricsInfo subClusterMetricsInfo = getClusterMetricsInfo(capability);
212+
213+
if (subClusterMetricsInfo == null) {
214+
return;
215+
}
216+
217+
// Prepare LastStartTime & LastHeartBeat
218+
Date lastStartTime = new Date(subClusterInfo.getLastStartTime());
219+
Date lastHeartBeat = new Date(subClusterInfo.getLastHeartBeat());
220+
221+
// Prepare Resource
222+
long totalMB = subClusterMetricsInfo.getTotalMB();
223+
String totalMBDesc = StringUtils.byteDesc(totalMB * BYTES_IN_MB);
224+
long totalVirtualCores = subClusterMetricsInfo.getTotalVirtualCores();
225+
String resources = String.format("<memory:%s, vCores:%s>", totalMBDesc, totalVirtualCores);
226+
227+
// Prepare Node
228+
long totalNodes = subClusterMetricsInfo.getTotalNodes();
229+
long activeNodes = subClusterMetricsInfo.getActiveNodes();
230+
String nodes = String.format("<totalNodes:%s, activeNodes:%s>", totalNodes, activeNodes);
231+
232+
// Prepare HTML Table
233+
String stateStyle = "color:#dc3545;font-weight:bolder";
234+
SubClusterState state = subClusterInfo.getState();
235+
if (SubClusterState.SC_RUNNING == state) {
236+
stateStyle = "color:#28a745;font-weight:bolder";
237+
}
238+
239+
tbody.tr().$id(subClusterIdText)
240+
.td().$class("details-control").a(herfWebAppAddress, subClusterIdText).__()
241+
.td().$style(stateStyle).__(state.name()).__()
242+
.td().__(lastStartTime).__()
243+
.td().__(lastHeartBeat).__()
244+
.td(resources)
245+
.td(nodes)
246+
.__();
247+
248+
// Formatted memory information
249+
long allocatedMB = subClusterMetricsInfo.getAllocatedMB();
250+
String allocatedMBDesc = StringUtils.byteDesc(allocatedMB * BYTES_IN_MB);
251+
long availableMB = subClusterMetricsInfo.getAvailableMB();
252+
String availableMBDesc = StringUtils.byteDesc(availableMB * BYTES_IN_MB);
253+
long pendingMB = subClusterMetricsInfo.getPendingMB();
254+
String pendingMBDesc = StringUtils.byteDesc(pendingMB * BYTES_IN_MB);
255+
long reservedMB = subClusterMetricsInfo.getReservedMB();
256+
String reservedMBDesc = StringUtils.byteDesc(reservedMB * BYTES_IN_MB);
257+
258+
subClusterMap.put("totalmemory", totalMBDesc);
259+
subClusterMap.put("allocatedmemory", allocatedMBDesc);
260+
subClusterMap.put("availablememory", availableMBDesc);
261+
subClusterMap.put("pendingmemory", pendingMBDesc);
262+
subClusterMap.put("reservedmemory", reservedMBDesc);
263+
subClusterMap.put("subcluster", subClusterId.getId());
264+
subClusterMap.put("capability", capability);
265+
lists.add(subClusterMap);
266+
}
267+
213268
}

0 commit comments

Comments
 (0)