Skip to content

Commit 74e48e7

Browse files
author
tangzhankun
authored
Merge pull request apache#1 from tangzhankun/zhankun-HDL
merge TensorFlow-YARN from zhankun's branch
2 parents 4d5a247 + bcb5598 commit 74e48e7

File tree

82 files changed

+3844
-3
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

82 files changed

+3844
-3
lines changed

‎hadoop-deeplearning-project/README.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
Hadoop Deep Learning Project
2+
======================
3+
##[YARN-TensorFlow](YARN-TensorFlow/hadoop-yarn-applications-tensorflow/README.md)

‎hadoop-deeplearning-project/YARN-TensorFlow/hadoop-yarn-applications-tensorflow/README.md‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
TensorFlow on YARN
2+
======================
3+
TensorFlow on YARN is a YARN application to enable an easy way for end user to run TensorFlow scripts.
4+
5+
Note that current project is a prototype with limitation and is still under development
6+
7+
## Features
8+
- [x] Launch a TensorFlow cluster with specified number of worker and PS server
9+
- [x] Replace python layer with java bridge layer to start server
10+
- [x] Generate ClusterSpec dynamically
11+
- [x] RPC support for client to get ClusterSpec from AM
12+
- [x] Signal handling for graceful shutdown
13+
- [ ] Package TensorFlow runtime as a resource that can be distributed easily
14+
- [ ] Fault tolerance
15+
- [ ] Code refine and more tests
16+
17+
## Set up and run
18+
1. Git clone ..
19+
2. Compile [tensorflow-bridge](../tensorflow-bridge/README.md) and put libbridge.so to a place be aware to YARN application. For instance, JVM lib directory.
20+
3. Compile TensorFlow on YARN
21+
22+
```sh
23+
cd <path_to_hadoop-yarn-application-tensorflow>
24+
mvn clean package -DskipTests
25+
```
26+
4. Run your Tensorflow script. Let's assume a "job.py"
27+
28+
```sh
29+
./bin/yarn-tf -job job.py -numberworkers 4 -numberps 1 -jar <path_to_tensorflow-on-yarn-with-dependency_jar>
30+
```
31+
32+
Note that at present, the "job.py" should parse worker and PS server from parameters "ps" and "wk" populated by TensorFlow on YARN client in the form of comma seperated values.

‎hadoop-deeplearning-project/YARN-TensorFlow/hadoop-yarn-applications-tensorflow/bin/yarn-tf‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
#!/bin/bash
2+
3+
JOB=""
4+
WORKERS=0
5+
PSES=0
6+
JAR=""
7+
while true
8+
do
9+
case "$1" in
10+
-job)
11+
JOB="$2"
12+
echo "job script: $JOB"
13+
shift
14+
;;
15+
-numberworkers)
16+
WORKERS="$2"
17+
echo "worker num: $WORKERS"
18+
shift
19+
;;
20+
-numberps)
21+
PSES="$2"
22+
echo "ps num: $PSES"
23+
shift
24+
;;
25+
-jar)
26+
JAR="$2"
27+
echo "jar path: $JAR"
28+
shift
29+
;;
30+
*)
31+
shift
32+
break
33+
;;
34+
esac
35+
shift
36+
done
37+
38+
CLIENT_MAIN_CLASS="org.apache.hadoop.yarn.applications.tensorflow.Client"
39+
40+
yarn jar $JAR $CLIENT_MAIN_CLASS \
41+
--jar $JAR \
42+
--tf_client $JOB \
43+
--num_worker $WORKERS \
44+
--num_ps $PSES \
45+
--container_memory 4096
46+

‎hadoop-deeplearning-project/YARN-TensorFlow/hadoop-yarn-applications-tensorflow/pom.xml‎

Lines changed: 211 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,211 @@
1+
<?xml version="1.0"?>
2+
<!--
3+
Licensed under the Apache License, Version 2.0 (the "License");
4+
you may not use this file except in compliance with the License.
5+
You may obtain a copy of the License at
6+
7+
http://www.apache.org/licenses/LICENSE-2.0
8+
9+
Unless required by applicable law or agreed to in writing, software
10+
distributed under the License is distributed on an "AS IS" BASIS,
11+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
See the License for the specific language governing permissions and
13+
limitations under the License. See accompanying LICENSE file.
14+
-->
15+
<project xmlns="http://maven.apache.org/POM/4.0.0"
16+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
17+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
18+
http://maven.apache.org/xsd/maven-4.0.0.xsd">
19+
<parent>
20+
<artifactId>YARN-TensorFlow</artifactId>
21+
<groupId>org.apache.hadoop</groupId>
22+
<version>3.0.0-alpha2-SNAPSHOT</version>
23+
</parent>
24+
<modelVersion>4.0.0</modelVersion>
25+
<artifactId>hadoop-yarn-applications-tensorflow</artifactId>
26+
<version>3.0.0-alpha2-SNAPSHOT</version>
27+
<name>TensorFlow on YARN</name>
28+
29+
<repositories>
30+
<repository>
31+
<id>bintray</id>
32+
<name>Bintray Repository</name>
33+
<url>http://dl.bintray.com/fvunicorn/maven</url>
34+
</repository>
35+
</repositories>
36+
37+
<properties>
38+
<!-- jackson versions -->
39+
<jackson.version>1.9.13</jackson.version>
40+
</properties>
41+
42+
<dependencies>
43+
<!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
44+
<dependency>
45+
<groupId>org.apache.hadoop</groupId>
46+
<artifactId>hadoop-common</artifactId>
47+
<scope>provided</scope>
48+
</dependency>
49+
<!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
50+
<dependency>
51+
<groupId>org.apache.hadoop</groupId>
52+
<artifactId>hadoop-yarn-client</artifactId>
53+
<scope>provided</scope>
54+
</dependency>
55+
56+
<dependency>
57+
<groupId>junit</groupId>
58+
<artifactId>junit</artifactId>
59+
<scope>test</scope>
60+
</dependency>
61+
62+
<dependency>
63+
<groupId>org.tensorflow</groupId>
64+
<artifactId>java-bridge</artifactId>
65+
<version>0.1.0</version>
66+
</dependency>
67+
68+
<dependency>
69+
<groupId>org.apache.hadoop</groupId>
70+
<artifactId>hadoop-yarn-server-timelineservice</artifactId>
71+
<type>test-jar</type>
72+
<scope>test</scope>
73+
</dependency>
74+
<!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
75+
<dependency>
76+
<groupId>org.apache.hadoop</groupId>
77+
<artifactId>hadoop-common</artifactId>
78+
<type>test-jar</type>
79+
<scope>test</scope>
80+
</dependency>
81+
<!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
82+
<dependency>
83+
<groupId>org.apache.hadoop</groupId>
84+
<artifactId>hadoop-yarn-server-nodemanager</artifactId>
85+
<scope>test</scope>
86+
</dependency>
87+
<!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
88+
<dependency>
89+
<groupId>org.apache.hadoop</groupId>
90+
<artifactId>hadoop-yarn-server-resourcemanager</artifactId>
91+
<scope>test</scope>
92+
</dependency>
93+
<!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
94+
<dependency>
95+
<groupId>org.apache.hadoop</groupId>
96+
<artifactId>hadoop-yarn-server-tests</artifactId>
97+
<type>test-jar</type>
98+
<scope>test</scope>
99+
</dependency>
100+
<dependency>
101+
<groupId>org.mockito</groupId>
102+
<artifactId>mockito-all</artifactId>
103+
<scope>test</scope>
104+
</dependency>
105+
<dependency>
106+
<groupId>org.apache.hadoop</groupId>
107+
<artifactId>hadoop-yarn-server-timeline-pluginstorage</artifactId>
108+
<type>test-jar</type>
109+
<scope>test</scope>
110+
</dependency>
111+
<dependency>
112+
<groupId>org.apache.hadoop</groupId>
113+
<artifactId>hadoop-yarn-common</artifactId>
114+
<type>test-jar</type>
115+
<scope>test</scope>
116+
</dependency>
117+
<dependency>
118+
<groupId>org.apache.hadoop</groupId>
119+
<artifactId>hadoop-hdfs</artifactId>
120+
<scope>test</scope>
121+
</dependency>
122+
<dependency>
123+
<groupId>org.apache.hadoop</groupId>
124+
<artifactId>hadoop-hdfs</artifactId>
125+
<scope>test</scope>
126+
<type>test-jar</type>
127+
</dependency>
128+
</dependencies>
129+
130+
<build>
131+
<plugins>
132+
<plugin>
133+
<artifactId>maven-jar-plugin</artifactId>
134+
<executions>
135+
<execution>
136+
<goals>
137+
<goal>jar</goal>
138+
</goals>
139+
<!-- strictly speaking, the unit test is really a regression test. It
140+
needs the main jar to be available to be able to run. -->
141+
<phase>test-compile</phase>
142+
</execution>
143+
</executions>
144+
<configuration>
145+
<archive>
146+
<manifest>
147+
<mainClass>org.apache.hadoop.yarn.applications.tensorflow.Client</mainClass>
148+
</manifest>
149+
</archive>
150+
</configuration>
151+
</plugin>
152+
<plugin>
153+
<groupId>org.apache.maven.plugins</groupId>
154+
<artifactId>maven-surefire-plugin</artifactId>
155+
<configuration>
156+
<environmentVariables>
157+
<JAVA_HOME>${java.home}</JAVA_HOME>
158+
</environmentVariables>
159+
</configuration>
160+
</plugin>
161+
<plugin>
162+
<groupId>org.apache.hadoop</groupId>
163+
<artifactId>hadoop-maven-plugins</artifactId>
164+
<executions>
165+
<execution>
166+
<id>compile-protoc</id>
167+
<goals>
168+
<goal>protoc</goal>
169+
</goals>
170+
<configuration>
171+
<protocVersion>${protobuf.version}</protocVersion>
172+
<protocCommand>${protoc.path}</protocCommand>
173+
<imports>
174+
<param>${basedir}/../../../hadoop-common-project/hadoop-common/src/main/proto</param>
175+
<param>${basedir}/../../../hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto</param>
176+
<param>${basedir}/src/main/proto</param>
177+
</imports>
178+
<source>
179+
<directory>${basedir}/src/main/proto</directory>
180+
<includes>
181+
<include>yarn_tensorflow_cluster_protos.proto</include>
182+
<include>TensorflowCluster.proto</include>
183+
</includes>
184+
</source>
185+
</configuration>
186+
</execution>
187+
</executions>
188+
</plugin>
189+
<plugin>
190+
<groupId>org.apache.maven.plugins</groupId>
191+
<artifactId>maven-assembly-plugin</artifactId>
192+
<configuration>
193+
<descriptorRefs>
194+
<descriptorRef>jar-with-dependencies</descriptorRef>
195+
</descriptorRefs>
196+
</configuration>
197+
<executions>
198+
<execution>
199+
<id>package-yarn</id>
200+
<phase>package</phase>
201+
<goals>
202+
<goal>single</goal>
203+
</goals>
204+
</execution>
205+
</executions>
206+
</plugin>
207+
</plugins>
208+
</build>
209+
210+
211+
</project>

0 commit comments

Comments
 (0)