Skip to content

Commit 475932c

Browse files
slfan1989slfan1989
authored
YARN-11562. [Federation] GPG Support Query Policies In Web. (#6023)
1 parent 4652d22 commit 475932c

File tree

7 files changed

+221
-2
lines changed

7 files changed

+221
-2
lines changed

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/pom.xml

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,13 @@
9191
<scope>test</scope>
9292
</dependency>
9393

94+
<dependency>
95+
<groupId>org.apache.hadoop</groupId>
96+
<artifactId>hadoop-yarn-common</artifactId>
97+
<type>test-jar</type>
98+
<scope>test</scope>
99+
</dependency>
100+
94101
<dependency>
95102
<groupId>org.hsqldb</groupId>
96103
<artifactId>hsqldb</artifactId>

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/GPGController.java

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,12 @@ public void index() {
3838
}
3939

4040
public void overview() {
41-
setTitle("GPG Details");
41+
setTitle("GPG");
4242
render(GPGOverviewPage.class);
4343
}
44+
45+
public void policies() {
46+
setTitle("Global Policy Generator Policies");
47+
render(GPGPoliciesPage.class);
48+
}
4449
}

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/GPGPoliciesBlock.java

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
/**
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
package org.apache.hadoop.yarn.server.globalpolicygenerator.webapp;
19+
20+
import com.google.inject.Inject;
21+
import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
22+
import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
23+
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
24+
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
25+
import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
26+
import org.apache.hadoop.yarn.server.globalpolicygenerator.GlobalPolicyGenerator;
27+
import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet;
28+
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
29+
30+
import java.nio.ByteBuffer;
31+
import java.util.Collection;
32+
import java.util.Map;
33+
34+
/**
35+
* Overview block for the GPG Policies Web UI.
36+
*/
37+
public class GPGPoliciesBlock extends HtmlBlock {
38+
39+
private final GlobalPolicyGenerator gpg;
40+
41+
private final FederationStateStoreFacade facade;
42+
43+
@Inject
44+
GPGPoliciesBlock(GlobalPolicyGenerator gpg, ViewContext ctx) {
45+
super(ctx);
46+
this.gpg = gpg;
47+
this.facade = FederationStateStoreFacade.getInstance(gpg.getConfig());
48+
}
49+
50+
@Override
51+
protected void render(Block html) {
52+
try {
53+
Collection<SubClusterPolicyConfiguration> policies =
54+
facade.getPoliciesConfigurations().values();
55+
initYarnFederationPolicies(policies, html);
56+
} catch (Exception e) {
57+
LOG.error("Get GPGPolicies Error.", e);
58+
}
59+
}
60+
61+
private void initYarnFederationPolicies(Collection<SubClusterPolicyConfiguration> policies,
62+
Block html) throws FederationPolicyInitializationException {
63+
64+
Hamlet.TBODY<Hamlet.TABLE<Hamlet>> tbody = html.table("#policies").
65+
thead().
66+
tr().
67+
th(".queue", "Queue Name").
68+
th(".policyType", "Policy Type").
69+
th(".routerPolicyWeights", "Router PolicyWeights").
70+
th(".amrmPolicyWeights", "Router AMRMPolicyWeights").
71+
th(".headroomAlpha", "Router Headroom Alpha").
72+
__().__().
73+
tbody();
74+
75+
if (policies != null) {
76+
for (SubClusterPolicyConfiguration policy : policies) {
77+
Hamlet.TR<Hamlet.TBODY<Hamlet.TABLE<Hamlet>>> row = tbody.tr().td(policy.getQueue());
78+
// Policy Type
79+
String type = policy.getType();
80+
row = row.td(type);
81+
82+
// WeightedPolicyInfo
83+
ByteBuffer params = policy.getParams();
84+
WeightedPolicyInfo weightedPolicyInfo = WeightedPolicyInfo.fromByteBuffer(params);
85+
row = row.td(policyWeight2String(weightedPolicyInfo.getRouterPolicyWeights()));
86+
row = row.td(policyWeight2String(weightedPolicyInfo.getAMRMPolicyWeights()));
87+
row.td(String.valueOf(weightedPolicyInfo.getHeadroomAlpha())).__();
88+
}
89+
}
90+
91+
tbody.__().__();
92+
}
93+
94+
/**
95+
* We will convert the PolicyWeight to string format.
96+
*
97+
* @param weights PolicyWeight.
98+
* @return string format PolicyWeight. example: SC-1:0.91, SC-2:0.09
99+
*/
100+
private String policyWeight2String(Map<SubClusterIdInfo, Float> weights) {
101+
StringBuilder sb = new StringBuilder();
102+
for (Map.Entry<SubClusterIdInfo, Float> entry : weights.entrySet()) {
103+
sb.append(entry.getKey().toId()).append(": ").append(entry.getValue()).append(", ");
104+
}
105+
if (sb.length() > 2) {
106+
sb.setLength(sb.length() - 2);
107+
}
108+
return sb.toString();
109+
}
110+
}

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/GPGPoliciesPage.java

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
/**
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
package org.apache.hadoop.yarn.server.globalpolicygenerator.webapp;
19+
20+
import org.apache.hadoop.yarn.webapp.SubView;
21+
import org.apache.hadoop.yarn.webapp.view.TwoColumnLayout;
22+
23+
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.ACCORDION_ID;
24+
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.ACCORDION;
25+
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES_ID;
26+
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID;
27+
28+
/**
29+
* Overview page for the GPG Policies Web UI.
30+
*/
31+
public class GPGPoliciesPage extends TwoColumnLayout {
32+
33+
@Override
34+
protected void preHead(Page.HTML<__> html) {
35+
commonPreHead(html);
36+
}
37+
38+
protected void commonPreHead(Page.HTML<__> html) {
39+
setTitle("Global Policy Generator Policies");
40+
set(ACCORDION_ID, "nav");
41+
set(initID(ACCORDION, "nav"), "{autoHeight:false, active:0}");
42+
set(DATATABLES_ID, "policies");
43+
}
44+
45+
@Override
46+
protected Class<? extends SubView> content() {
47+
return GPGPoliciesBlock.class;
48+
}
49+
50+
@Override
51+
protected Class<? extends SubView> nav() {
52+
return NavBlock.class;
53+
}
54+
55+
}

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/GPGWebApp.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@
2525
/**
2626
* The GPG webapp.
2727
*/
28-
public class GPGWebApp extends WebApp{
28+
public class GPGWebApp extends WebApp {
2929
private GlobalPolicyGenerator gpg;
3030

3131
public GPGWebApp(GlobalPolicyGenerator gpg) {
@@ -41,5 +41,6 @@ public void setup() {
4141
bind(GlobalPolicyGenerator.class).toInstance(gpg);
4242
}
4343
route("/", GPGController.class, "overview");
44+
route("/policies", GPGController.class, "policies");
4445
}
4546
}

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/NavBlock.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ public void render(Block html) {
3131
h3("GPG").
3232
ul().
3333
li().a(url(""), "Overview").__().
34+
li().a(url("policies"), "Policies").__().
3435
__().
3536
h3("Tools").
3637
ul().

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/TestGPGWebApp.java

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
/**
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
package org.apache.hadoop.yarn.server.globalpolicygenerator.webapp;
19+
20+
import org.apache.hadoop.yarn.exceptions.YarnException;
21+
import org.apache.hadoop.yarn.server.globalpolicygenerator.GlobalPolicyGenerator;
22+
import org.apache.hadoop.yarn.webapp.test.WebAppTests;
23+
import org.junit.Test;
24+
import org.slf4j.Logger;
25+
import org.slf4j.LoggerFactory;
26+
27+
import java.io.IOException;
28+
29+
public class TestGPGWebApp {
30+
31+
private static final Logger LOG = LoggerFactory.getLogger(TestGPGWebApp.class);
32+
33+
@Test
34+
public void testGPGPoliciesPageWebView()
35+
throws InterruptedException, YarnException, IOException {
36+
LOG.info("testGPGPoliciesPageWebView.");
37+
WebAppTests.testPage(GPGPoliciesPage.class, GlobalPolicyGenerator.class,
38+
new GlobalPolicyGenerator());
39+
}
40+
}

0 commit comments

Comments
 (0)