Skip to content

Commit 6e3d705

Browse files
szilard-nemethszilard-nemeth
committed
YARN-10386. Create new JSON schema for Placement Rules. Contributed by Peter Bacsko and Benjamin Teke
1 parent c392d90 commit 6e3d705

File tree

7 files changed

+221
-0
lines changed

7 files changed

+221
-0
lines changed

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ patchprocess/
5151
.history/
5252
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/package-lock.json
5353
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/yarn-error.log
54+
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/placement/schema
5455

5556
#robotframework outputs
5657
log.html

‎hadoop-project/pom.xml‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -207,6 +207,7 @@
207207
<powermock.version>1.5.6</powermock.version>
208208
<solr.version>7.7.0</solr.version>
209209
<openssl-wildfly.version>1.0.7.Final</openssl-wildfly.version>
210+
<jsonschema2pojo.version>1.0.2</jsonschema2pojo.version>
210211
</properties>
211212

212213
<dependencyManagement>
@@ -1165,6 +1166,11 @@
11651166
<artifactId>jackson-dataformat-cbor</artifactId>
11661167
<version>${jackson2.version}</version>
11671168
</dependency>
1169+
<dependency>
1170+
<groupId>com.fasterxml.jackson.dataformat</groupId>
1171+
<artifactId>jackson-dataformat-yaml</artifactId>
1172+
<version>${jackson2.version}</version>
1173+
</dependency>
11681174
<dependency>
11691175
<groupId>org.mockito</groupId>
11701176
<artifactId>mockito-core</artifactId>
@@ -1728,6 +1734,49 @@
17281734
<artifactId>jna</artifactId>
17291735
<version>${jna.version}</version>
17301736
</dependency>
1737+
<dependency>
1738+
<groupId>org.jsonschema2pojo</groupId>
1739+
<artifactId>jsonschema2pojo-core</artifactId>
1740+
<version>${jsonschema2pojo.version}</version>
1741+
<exclusions>
1742+
<exclusion>
1743+
<groupId>com.fasterxml.jackson.core</groupId>
1744+
<artifactId>*</artifactId>
1745+
</exclusion>
1746+
<exclusion>
1747+
<groupId>com.fasterxml.jackson.dataformat</groupId>
1748+
<artifactId>*</artifactId>
1749+
</exclusion>
1750+
<exclusion>
1751+
<groupId>org.codehaus.jackson</groupId>
1752+
<artifactId>*</artifactId>
1753+
</exclusion>
1754+
<exclusion>
1755+
<groupId>com.google.code.gson</groupId>
1756+
<artifactId>gson</artifactId>
1757+
</exclusion>
1758+
<exclusion>
1759+
<groupId>com.google.code.findbugs</groupId>
1760+
<artifactId>annotations</artifactId>
1761+
</exclusion>
1762+
<exclusion>
1763+
<groupId>org.scala-lang</groupId>
1764+
<artifactId>scala-library</artifactId>
1765+
</exclusion>
1766+
<exclusion>
1767+
<groupId>org.jsonschema2pojo</groupId>
1768+
<artifactId>jsonschema2pojo-scalagen</artifactId>
1769+
</exclusion>
1770+
<exclusion>
1771+
<groupId>com.google.code.javaparser</groupId>
1772+
<artifactId>javaparser</artifactId>
1773+
</exclusion>
1774+
<exclusion>
1775+
<groupId>javax.validation</groupId>
1776+
<artifactId>validation-api</artifactId>
1777+
</exclusion>
1778+
</exclusions>
1779+
</dependency>
17311780
</dependencies>
17321781
</dependencyManagement>
17331782

‎hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -711,4 +711,9 @@
711711
<Class name="~org\.apache\.hadoop\.yarn\.server\.timelineservice\.reader\.TestTimelineReaderWebServicesHBaseStorage.*" />
712712
<Bug pattern="UMAC_UNCALLABLE_METHOD_OF_ANONYMOUS_CLASS" />
713713
</Match>
714+
715+
<!-- Generated files from JSON schema -->
716+
<Match>
717+
<Package name="org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.placement.schema" />
718+
</Match>
714719
</FindBugsFilter>

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -250,6 +250,21 @@
250250
<scope>test</scope>
251251
</dependency>
252252

253+
<!-- Necessary to include jackson separately to avoid issues with
254+
maven enforcer plugin in "hadoop-client-check-test-invariants"
255+
-->
256+
<dependency>
257+
<groupId>org.jsonschema2pojo</groupId>
258+
<artifactId>jsonschema2pojo-core</artifactId>
259+
</dependency>
260+
<dependency>
261+
<groupId>com.fasterxml.jackson.core</groupId>
262+
<artifactId>jackson-databind</artifactId>
263+
</dependency>
264+
<dependency>
265+
<groupId>com.fasterxml.jackson.dataformat</groupId>
266+
<artifactId>jackson-dataformat-yaml</artifactId>
267+
</dependency>
253268
</dependencies>
254269

255270
<build>
@@ -380,6 +395,22 @@
380395
</excludes>
381396
</configuration>
382397
</plugin>
398+
<plugin>
399+
<groupId>org.jsonschema2pojo</groupId>
400+
<artifactId>jsonschema2pojo-maven-plugin</artifactId>
401+
<configuration>
402+
<sourceDirectory>${basedir}/src/main/json_schema</sourceDirectory>
403+
<targetPackage>org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.placement.schema</targetPackage>
404+
<outputDirectory>${basedir}/src/main/java</outputDirectory>
405+
</configuration>
406+
<executions>
407+
<execution>
408+
<goals>
409+
<goal>generate</goal>
410+
</goals>
411+
</execution>
412+
</executions>
413+
</plugin>
383414
</plugins>
384415
</build>
385416

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/placement/GeneratePojos.java‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
/**
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.placement;
20+
21+
import java.io.File;
22+
import java.io.IOException;
23+
import java.net.URL;
24+
import java.nio.file.Paths;
25+
26+
import org.jsonschema2pojo.DefaultGenerationConfig;
27+
import org.jsonschema2pojo.GenerationConfig;
28+
import org.jsonschema2pojo.Jackson2Annotator;
29+
import org.jsonschema2pojo.SchemaGenerator;
30+
import org.jsonschema2pojo.SchemaMapper;
31+
import org.jsonschema2pojo.SchemaStore;
32+
import org.jsonschema2pojo.rules.RuleFactory;
33+
34+
import com.sun.codemodel.JCodeModel;
35+
36+
/**
37+
* Helper class to re-generate java POJOs based on the JSON schema.
38+
*/
39+
public final class GeneratePojos {
40+
@SuppressWarnings("checkstyle:linelength")
41+
private static final String TARGET_PACKAGE =
42+
"org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.placement.schema";
43+
44+
private GeneratePojos() {
45+
// no instances
46+
}
47+
48+
public static void main(String[] args) throws IOException {
49+
JCodeModel codeModel = new JCodeModel();
50+
URL schemaURL = Paths.get(
51+
"src/main/json_schema/MappingRulesDescription.json").toUri().toURL();
52+
53+
GenerationConfig config = new DefaultGenerationConfig() {
54+
@Override
55+
public boolean isGenerateBuilders() {
56+
return false;
57+
}
58+
59+
@Override
60+
public boolean isUsePrimitives() {
61+
return true;
62+
}
63+
};
64+
65+
SchemaMapper mapper =
66+
new SchemaMapper(
67+
new RuleFactory(config,
68+
new Jackson2Annotator(config),
69+
new SchemaStore()),
70+
new SchemaGenerator());
71+
72+
mapper.generate(codeModel, "ignore", TARGET_PACKAGE, schemaURL);
73+
74+
codeModel.build(new File("src/main/java"));
75+
}
76+
}

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/placement/package-info.java‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
/**
20+
* Contains classes that are related to the newer
21+
* version of CS placement engine.
22+
*/
23+
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.placement;

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/json_schema/MappingRulesDescription.json‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
{
2+
"$schema": "http://json-schema.org/draft/2019-09/schema#",
3+
4+
"type" : "object",
5+
"properties" : {
6+
"rules" : {
7+
"type" : "array",
8+
"items": {
9+
"type": "object",
10+
"properties": {
11+
"type": { "type": "string",
12+
"enum": ["user", "group", "application"] },
13+
"matches": { "type": "string" },
14+
"policy": { "type": "string",
15+
"enum": [
16+
"specified",
17+
"reject",
18+
"defaultQueue",
19+
"user",
20+
"primaryGroup",
21+
"secondaryGroup",
22+
"primaryGroupUser",
23+
"secondaryGroupUser",
24+
"setDefaultQueue",
25+
"custom" ] },
26+
"parentQueue": { "type": "string" },
27+
"fallbackResult": { "type": "string",
28+
"enum": ["skip", "reject", "placeDefault"] },
29+
"create": { "type": "boolean" },
30+
"value": { "type": "string" },
31+
"customPlacement": { "type" : "string" }
32+
}
33+
}
34+
}
35+
}
36+
}

0 commit comments

Comments
 (0)