Skip to content

Commit 8584f82

Browse files
tangzhankunAjay Kumar
tangzhankun
authored and
Ajay Kumar
committed
SUBMARINE-58. Submarine client needs to generate fat jar. Contributed by Zac Zhou.
1 parent 9dfb397 commit 8584f82

File tree

42 files changed

+1424
-183
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

42 files changed

+1424
-183
lines changed

‎hadoop-submarine/hadoop-submarine-all/pom.xml‎

Lines changed: 183 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,183 @@
1+
<?xml version="1.0"?>
2+
<!--
3+
Licensed under the Apache License, Version 2.0 (the "License");
4+
you may not use this file except in compliance with the License.
5+
You may obtain a copy of the License at
6+
7+
http://www.apache.org/licenses/LICENSE-2.0
8+
9+
Unless required by applicable law or agreed to in writing, software
10+
distributed under the License is distributed on an "AS IS" BASIS,
11+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
See the License for the specific language governing permissions and
13+
limitations under the License. See accompanying LICENSE file.
14+
-->
15+
<project xmlns="http://maven.apache.org/POM/4.0.0"
16+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
17+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
18+
http://maven.apache.org/xsd/maven-4.0.0.xsd">
19+
<modelVersion>4.0.0</modelVersion>
20+
<parent>
21+
<artifactId>hadoop-submarine</artifactId>
22+
<groupId>org.apache.hadoop</groupId>
23+
<version>0.2.0-SNAPSHOT</version>
24+
</parent>
25+
<artifactId>${project.artifactId}</artifactId>
26+
<version>${project.version}</version>
27+
<name>Hadoop Submarine All</name>
28+
29+
<properties>
30+
<!-- Needed for generating FindBugs warnings using parent pom -->
31+
<yarn.basedir>${project.parent.parent.basedir}</yarn.basedir>
32+
<project.artifactId>hadoop-submarine-all</project.artifactId>
33+
<project.version>0.2.0-SNAPSHOT</project.version>
34+
</properties>
35+
36+
<dependencies>
37+
38+
<!-- Dependencies for Hadoop commons -->
39+
40+
<dependency>
41+
<groupId>org.apache.hadoop</groupId>
42+
<artifactId>hadoop-hdfs</artifactId>
43+
</dependency>
44+
45+
<dependency>
46+
<groupId>org.apache.hadoop</groupId>
47+
<artifactId>hadoop-common</artifactId>
48+
</dependency>
49+
50+
<dependency>
51+
<groupId>org.apache.hadoop</groupId>
52+
<artifactId>hadoop-submarine-core</artifactId>
53+
<version>${project.version}</version>
54+
</dependency>
55+
</dependencies>
56+
57+
<profiles>
58+
59+
<profile>
60+
<id>hadoop-3.2</id>
61+
<dependencies>
62+
<dependency>
63+
<groupId>org.apache.hadoop</groupId>
64+
<artifactId>hadoop-submarine-yarnservice-runtime</artifactId>
65+
<version>${project.version}</version>
66+
</dependency>
67+
<dependency>
68+
<groupId>org.apache.hadoop</groupId>
69+
<artifactId>hadoop-submarine-tony-runtime</artifactId>
70+
<version>${project.version}</version>
71+
</dependency>
72+
<dependency>
73+
<groupId>org.apache.hadoop</groupId>
74+
<artifactId>hadoop-hdfs-client</artifactId>
75+
<version>${hadoop.version}</version>
76+
</dependency>
77+
</dependencies>
78+
</profile>
79+
80+
<!-- Default profile-->
81+
<profile>
82+
<id>hadoop-3.1</id>
83+
<activation>
84+
<activeByDefault>true</activeByDefault>
85+
</activation>
86+
<dependencies>
87+
<dependency>
88+
<groupId>org.apache.hadoop</groupId>
89+
<artifactId>hadoop-submarine-yarnservice-runtime</artifactId>
90+
<version>${project.version}</version>
91+
</dependency>
92+
<dependency>
93+
<groupId>org.apache.hadoop</groupId>
94+
<artifactId>hadoop-submarine-tony-runtime</artifactId>
95+
<version>${project.version}</version>
96+
</dependency>
97+
<dependency>
98+
<groupId>org.apache.hadoop</groupId>
99+
<artifactId>hadoop-hdfs-client</artifactId>
100+
<version>${hadoop.version}</version>
101+
</dependency>
102+
</dependencies>
103+
</profile>
104+
105+
<profile>
106+
<id>hadoop-2.9</id>
107+
<dependencies>
108+
<dependency>
109+
<groupId>org.apache.hadoop</groupId>
110+
<artifactId>hadoop-submarine-tony-runtime</artifactId>
111+
<version>${project.version}</version>
112+
</dependency>
113+
<dependency>
114+
<groupId>org.apache.hadoop</groupId>
115+
<artifactId>hadoop-hdfs-client</artifactId>
116+
<version>${hadoop.version}</version>
117+
</dependency>
118+
</dependencies>
119+
</profile>
120+
121+
<profile>
122+
<id>hadoop-2.7</id>
123+
<dependencies>
124+
<dependency>
125+
<groupId>org.apache.hadoop</groupId>
126+
<artifactId>hadoop-submarine-tony-runtime</artifactId>
127+
<version>${project.version}</version>
128+
</dependency>
129+
</dependencies>
130+
</profile>
131+
</profiles>
132+
133+
<build>
134+
<plugins>
135+
<plugin>
136+
<groupId>org.apache.maven.plugins</groupId>
137+
<artifactId>maven-shade-plugin</artifactId>
138+
<version>3.2.1</version>
139+
<executions>
140+
<execution>
141+
<phase>package</phase>
142+
<goals>
143+
<goal>shade</goal>
144+
</goals>
145+
<configuration>
146+
<!--
147+
<shadedArtifactAttached>true</shadedArtifactAttached>
148+
<shadedClassifierName>with-all-dependencies</shadedClassifierName>
149+
-->
150+
<outputFile>target/${project.artifactId}-${project.version}-${project.activeProfiles[0].id}.jar</outputFile>
151+
<artifactSet>
152+
<excludes>
153+
<exclude>classworlds:classworlds</exclude>
154+
<exclude>junit:junit</exclude>
155+
<exclude>jmock:*</exclude>
156+
<exclude>*:xml-apis</exclude>
157+
<exclude>org.apache.maven:lib:tests</exclude>
158+
</excludes>
159+
</artifactSet>
160+
<filters>
161+
<filter>
162+
<artifact>*:*</artifact>
163+
<excludes>
164+
<exclude>META-INF/*.SF</exclude>
165+
<exclude>META-INF/*.DSA</exclude>
166+
<exclude>META-INF/*.RSA</exclude>
167+
</excludes>
168+
</filter>
169+
</filters>
170+
<transformers>
171+
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
172+
<mainClass>org.apache.hadoop.yarn.submarine.client.cli.Cli</mainClass>
173+
</transformer>
174+
<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
175+
</transformers>
176+
</configuration>
177+
</execution>
178+
</executions>
179+
</plugin>
180+
</plugins>
181+
</build>
182+
183+
</project>

‎hadoop-submarine/hadoop-submarine-core/pom.xml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,10 @@
6767
<groupId>org.yaml</groupId>
6868
<artifactId>snakeyaml</artifactId>
6969
</dependency>
70+
<dependency>
71+
<groupId>org.apache.commons</groupId>
72+
<artifactId>commons-lang3</artifactId>
73+
</dependency>
7074

7175
<!-- Dependencies for Hadoop commons -->
7276

‎hadoop-submarine/hadoop-submarine-core/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/param/runjob/RunJobParameters.java‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@
4545
import org.apache.hadoop.yarn.submarine.common.ClientContext;
4646
import org.apache.hadoop.yarn.submarine.common.api.TensorFlowRole;
4747
import org.apache.hadoop.yarn.submarine.common.fs.RemoteDirectoryManager;
48-
import org.apache.hadoop.yarn.util.resource.ResourceUtils;
48+
import org.apache.hadoop.yarn.submarine.common.resource.ResourceUtils;
4949
import org.yaml.snakeyaml.introspector.Property;
5050
import org.yaml.snakeyaml.introspector.PropertyUtils;
5151

@@ -271,8 +271,7 @@ private Resource determineWorkerResource(ParametersHolder parametersHolder,
271271
throw new ParseException(
272272
"--" + CliConstants.WORKER_RES + " is absent.");
273273
}
274-
return ResourceUtils.createResourceFromString(workerResourceStr,
275-
clientContext.getOrCreateYarnClient().getResourceTypeInfo());
274+
return ResourceUtils.createResourceFromString(workerResourceStr);
276275
}
277276
return null;
278277
}

‎hadoop-submarine/hadoop-submarine-core/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/param/runjob/TensorFlowRunJobParameters.java‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@
2727
import org.apache.hadoop.yarn.submarine.client.cli.runjob.RoleParameters;
2828
import org.apache.hadoop.yarn.submarine.common.ClientContext;
2929
import org.apache.hadoop.yarn.submarine.common.api.TensorFlowRole;
30-
import org.apache.hadoop.yarn.util.resource.ResourceUtils;
30+
import org.apache.hadoop.yarn.submarine.common.resource.ResourceUtils;
3131

3232
import java.io.IOException;
3333
import java.util.List;
@@ -127,8 +127,7 @@ private Resource determinePSResource(ParametersHolder parametersHolder,
127127
if (psResourceStr == null) {
128128
throw new ParseException("--" + CliConstants.PS_RES + " is absent.");
129129
}
130-
return ResourceUtils.createResourceFromString(psResourceStr,
131-
clientContext.getOrCreateYarnClient().getResourceTypeInfo());
130+
return ResourceUtils.createResourceFromString(psResourceStr);
132131
}
133132
return null;
134133
}
@@ -151,9 +150,8 @@ private RoleParameters getTensorBoardParameters(
151150
if (tensorboardResourceStr == null || tensorboardResourceStr.isEmpty()) {
152151
tensorboardResourceStr = CliConstants.TENSORBOARD_DEFAULT_RESOURCES;
153152
}
154-
Resource tensorboardResource =
155-
ResourceUtils.createResourceFromString(tensorboardResourceStr,
156-
clientContext.getOrCreateYarnClient().getResourceTypeInfo());
153+
Resource tensorboardResource = ResourceUtils.createResourceFromString(
154+
tensorboardResourceStr);
157155
String tensorboardDockerImage =
158156
parametersHolder.getOptionValue(CliConstants.TENSORBOARD_DOCKER_IMAGE);
159157
return new RoleParameters(TensorFlowRole.TENSORBOARD, 1, null,

0 commit comments

Comments
 (0)