Skip to content

Commit ab0b180

Browse files
xkrogenxkrogen
committed
HDFS-12345 Add Dynamometer to hadoop-tools, a tool for scale testing the HDFS NameNode with real metadata and workloads. Contributed by Erik Krogen.
1 parent 48d7f00 commit ab0b180

File tree

71 files changed

+9202
-1
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

71 files changed

+9202
-1
lines changed

‎dev-support/bin/dist-layout-stitching‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -148,6 +148,7 @@ run cp -p "${ROOT}/hadoop-client-modules/hadoop-client-runtime/target/hadoop-cli
148148
run cp -p "${ROOT}/hadoop-client-modules/hadoop-client-minicluster/target/hadoop-client-minicluster-${VERSION}.jar" share/hadoop/client/
149149

150150
run copy "${ROOT}/hadoop-tools/hadoop-tools-dist/target/hadoop-tools-dist-${VERSION}" .
151+
run copy "${ROOT}/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-dist/target/hadoop-dynamometer-dist-${VERSION}" .
151152

152153

153154
echo

‎hadoop-assemblies/src/main/resources/assemblies/hadoop-dynamometer-blockgen.xml‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
<!--
2+
Licensed to the Apache Software Foundation (ASF) under one
3+
or more contributor license agreements. See the NOTICE file
4+
distributed with this work for additional information
5+
regarding copyright ownership. The ASF licenses this file
6+
to you under the Apache License, Version 2.0 (the
7+
"License"); you may not use this file except in compliance
8+
with the License. You may obtain a copy of the License at
9+
10+
http://www.apache.org/licenses/LICENSE-2.0
11+
12+
Unless required by applicable law or agreed to in writing, software
13+
distributed under the License is distributed on an "AS IS" BASIS,
14+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
See the License for the specific language governing permissions and
16+
limitations under the License.
17+
-->
18+
<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3"
19+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
20+
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3 http://maven.apache.org/xsd/assembly-1.1.3.xsd">
21+
<id>hadoop-dynamometer-blockgen</id>
22+
<formats>
23+
<format>dir</format>
24+
</formats>
25+
<includeBaseDirectory>false</includeBaseDirectory>
26+
27+
<fileSets>
28+
<fileSet>
29+
<directory>${basedir}/src/main/bash</directory>
30+
<outputDirectory>dynamometer-blockgen/bin</outputDirectory>
31+
<fileMode>0755</fileMode>
32+
</fileSet>
33+
</fileSets>
34+
35+
</assembly>

‎hadoop-assemblies/src/main/resources/assemblies/hadoop-dynamometer-infra.xml‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
<!--
2+
Licensed to the Apache Software Foundation (ASF) under one
3+
or more contributor license agreements. See the NOTICE file
4+
distributed with this work for additional information
5+
regarding copyright ownership. The ASF licenses this file
6+
to you under the Apache License, Version 2.0 (the
7+
"License"); you may not use this file except in compliance
8+
with the License. You may obtain a copy of the License at
9+
10+
http://www.apache.org/licenses/LICENSE-2.0
11+
12+
Unless required by applicable law or agreed to in writing, software
13+
distributed under the License is distributed on an "AS IS" BASIS,
14+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
See the License for the specific language governing permissions and
16+
limitations under the License.
17+
-->
18+
<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3"
19+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
20+
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3 http://maven.apache.org/xsd/assembly-1.1.3.xsd">
21+
<id>hadoop-dynamometer-infra</id>
22+
<formats>
23+
<format>dir</format>
24+
</formats>
25+
<includeBaseDirectory>false</includeBaseDirectory>
26+
27+
<fileSets>
28+
<fileSet>
29+
<directory>${basedir}/src/main/bash</directory>
30+
<outputDirectory>dynamometer-infra/bin</outputDirectory>
31+
<fileMode>0755</fileMode>
32+
</fileSet>
33+
</fileSets>
34+
35+
</assembly>

‎hadoop-assemblies/src/main/resources/assemblies/hadoop-dynamometer-workload.xml‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
<!--
2+
Licensed to the Apache Software Foundation (ASF) under one
3+
or more contributor license agreements. See the NOTICE file
4+
distributed with this work for additional information
5+
regarding copyright ownership. The ASF licenses this file
6+
to you under the Apache License, Version 2.0 (the
7+
"License"); you may not use this file except in compliance
8+
with the License. You may obtain a copy of the License at
9+
10+
http://www.apache.org/licenses/LICENSE-2.0
11+
12+
Unless required by applicable law or agreed to in writing, software
13+
distributed under the License is distributed on an "AS IS" BASIS,
14+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
See the License for the specific language governing permissions and
16+
limitations under the License.
17+
-->
18+
<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3"
19+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
20+
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3 http://maven.apache.org/xsd/assembly-1.1.3.xsd">
21+
<id>hadoop-dynamometer-workload</id>
22+
<formats>
23+
<format>dir</format>
24+
</formats>
25+
<includeBaseDirectory>false</includeBaseDirectory>
26+
27+
<fileSets>
28+
<fileSet>
29+
<directory>${basedir}/src/main/bash</directory>
30+
<outputDirectory>dynamometer-workload/bin</outputDirectory>
31+
<fileMode>0755</fileMode>
32+
</fileSet>
33+
</fileSets>
34+
35+
</assembly>

‎hadoop-assemblies/src/main/resources/assemblies/hadoop-dynamometer.xml‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
<!--
2+
Licensed to the Apache Software Foundation (ASF) under one or more
3+
contributor license agreements. See the NOTICE file distributed with
4+
this work for additional information regarding copyright ownership.
5+
The ASF licenses this file to You under the Apache License, Version 2.0
6+
(the "License"); you may not use this file except in compliance with
7+
the License. You may obtain a copy of the License at
8+
9+
http://www.apache.org/licenses/LICENSE-2.0
10+
11+
Unless required by applicable law or agreed to in writing, software
12+
distributed under the Li2cense is distributed on an "AS IS" BASIS,
13+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
See the License for the specific language governing permissions and
15+
limitations under the License.
16+
-->
17+
<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3"
18+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
19+
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3 http://maven.apache.org/xsd/assembly-1.1.3.xsd">
20+
<id>hadoop-dynamometer</id>
21+
<formats>
22+
<format>dir</format>
23+
</formats>
24+
<includeBaseDirectory>false</includeBaseDirectory>
25+
<fileSets>
26+
<fileSet>
27+
<directory>../hadoop-dynamometer-blockgen/target</directory>
28+
<outputDirectory>/share/hadoop/${hadoop.component}/sources</outputDirectory>
29+
<includes>
30+
<include>*-sources.jar</include>
31+
</includes>
32+
</fileSet>
33+
<fileSet>
34+
<directory>../hadoop-dynamometer-blockgen/target/hadoop-dynamometer-blockgen-${project.version}/dynamometer-blockgen</directory>
35+
<outputDirectory>/share/hadoop/${hadoop.component}/dynamometer/dynamometer-blockgen</outputDirectory>
36+
</fileSet>
37+
<fileSet>
38+
<directory>../hadoop-dynamometer-workload/target</directory>
39+
<outputDirectory>/share/hadoop/${hadoop.component}/sources</outputDirectory>
40+
<includes>
41+
<include>*-sources.jar</include>
42+
</includes>
43+
</fileSet>
44+
<fileSet>
45+
<directory>../hadoop-dynamometer-workload/target/hadoop-dynamometer-workload-${project.version}/dynamometer-workload</directory>
46+
<outputDirectory>/share/hadoop/${hadoop.component}/dynamometer/dynamometer-workload</outputDirectory>
47+
</fileSet>
48+
<fileSet>
49+
<directory>../hadoop-dynamometer-infra/target</directory>
50+
<outputDirectory>/share/hadoop/${hadoop.component}/sources</outputDirectory>
51+
<includes>
52+
<include>*-sources.jar</include>
53+
</includes>
54+
</fileSet>
55+
<fileSet>
56+
<directory>../hadoop-dynamometer-infra/target/hadoop-dynamometer-infra-${project.version}/dynamometer-infra</directory>
57+
<outputDirectory>/share/hadoop/${hadoop.component}/dynamometer/dynamometer-infra</outputDirectory>
58+
</fileSet>
59+
</fileSets>
60+
<dependencySets>
61+
<dependencySet>
62+
<outputDirectory>/share/hadoop/${hadoop.component}/lib</outputDirectory>
63+
<unpack>false</unpack>
64+
<scope>runtime</scope>
65+
<useProjectArtifact>false</useProjectArtifact>
66+
<excludes>
67+
<!-- use slf4j from common to avoid multiple binding warnings -->
68+
<exclude>org.slf4j:slf4j-api</exclude>
69+
<exclude>org.slf4j:slf4j-log4j12</exclude>
70+
</excludes>
71+
</dependencySet>
72+
</dependencySets>
73+
</assembly>

‎hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml‎

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -174,6 +174,28 @@
174174
<directory>../hadoop-sls/target/hadoop-sls-${project.version}/sls</directory>
175175
<outputDirectory>/share/hadoop/${hadoop.component}/sls</outputDirectory>
176176
</fileSet>
177+
<fileSet>
178+
<directory>../hadoop-dynamometer/hadoop-dynamometer-blockgen/target</directory>
179+
<outputDirectory>/share/hadoop/${hadoop.component}/sources</outputDirectory>
180+
<includes>
181+
<include>*-sources.jar</include>
182+
</includes>
183+
</fileSet>
184+
<fileSet>
185+
<directory>../hadoop-dynamometer/hadoop-dynamometer-blockgen/target/hadoop-dynamometer-blockgen-${project.version}/dynamometer-blockgen</directory>
186+
<outputDirectory>/share/hadoop/${hadoop.component}/dynamometer-blockgen</outputDirectory>
187+
</fileSet>
188+
<!--<fileSet>-->
189+
<!--<directory>../hadoop-dynamometer/hadoop-dynamometer-blockgen/target</directory>-->
190+
<!--<outputDirectory>/share/hadoop/${hadoop.component}/sources</outputDirectory>-->
191+
<!--<includes>-->
192+
<!--<include>*-sources.jar</include>-->
193+
<!--</includes>-->
194+
<!--</fileSet>-->
195+
<!--<fileSet>-->
196+
<!--<directory>../hadoop-dynamometer/hadoop-dynamometer-blockgen/target/hadoop-dynamometer-blockgen-${project.version}/dynamometer-blockgen</directory>-->
197+
<!--<outputDirectory>/share/hadoop/${hadoop.component}/dynamometer-blockgen</outputDirectory>-->
198+
<!--</fileSet>-->
177199
<fileSet>
178200
<directory>../hadoop-resourceestimator/target</directory>
179201
<outputDirectory>/share/hadoop/${hadoop.component}/sources</outputDirectory>
@@ -209,7 +231,7 @@
209231
<excludes>
210232
<exclude>org.apache.hadoop:hadoop-common</exclude>
211233
<exclude>org.apache.hadoop:hadoop-hdfs</exclude>
212-
<exclude>org.apache.hadoop:hadoop-mapreduce</exclude>
234+
<exclude>org.apache.hadoop:hadoop-client</exclude>
213235
<!-- pipes is native stuff, this just keeps pom from being package-->
214236
<exclude>org.apache.hadoop:hadoop-pipes</exclude>
215237
<!-- use slf4j from common to avoid multiple binding warnings -->

‎hadoop-dist/pom.xml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -196,6 +196,7 @@
196196
<argument>${project.version}</argument>
197197
<argument>${project.build.directory}</argument>
198198
<argument>${basedir}/../hadoop-tools</argument>
199+
<!--<argument>${basedir}/../hadoop-tools/hadoop-dynamometer</argument>-->
199200
</arguments>
200201
</configuration>
201202
</execution>

‎hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-blockgen/pom.xml‎

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
Licensed under the Apache License, Version 2.0 (the "License");
4+
you may not use this file except in compliance with the License.
5+
You may obtain a copy of the License at
6+
7+
http://www.apache.org/licenses/LICENSE-2.0
8+
9+
Unless required by applicable law or agreed to in writing, software
10+
distributed under the License is distributed on an "AS IS" BASIS,
11+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
See the License for the specific language governing permissions and
13+
limitations under the License. See accompanying LICENSE file.
14+
-->
15+
<project xmlns="http://maven.apache.org/POM/4.0.0"
16+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
17+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
18+
<modelVersion>4.0.0</modelVersion>
19+
<parent>
20+
<groupId>org.apache.hadoop</groupId>
21+
<artifactId>hadoop-project</artifactId>
22+
<version>3.3.0-SNAPSHOT</version>
23+
<relativePath>../../../hadoop-project</relativePath>
24+
</parent>
25+
<artifactId>hadoop-dynamometer-blockgen</artifactId>
26+
<version>3.3.0-SNAPSHOT</version>
27+
<description>Apache Hadoop Dynamometer Block Listing Generator</description>
28+
<name>Apache Hadoop Dynamometer Block Listing Generator</name>
29+
<packaging>jar</packaging>
30+
31+
<dependencies>
32+
<dependency>
33+
<groupId>org.apache.hadoop</groupId>
34+
<artifactId>hadoop-client</artifactId>
35+
<scope>compile</scope>
36+
</dependency>
37+
<dependency>
38+
<groupId>org.mockito</groupId>
39+
<artifactId>mockito-all</artifactId>
40+
<version>1.10.19</version>
41+
<scope>test</scope>
42+
</dependency>
43+
<dependency>
44+
<groupId>junit</groupId>
45+
<artifactId>junit</artifactId>
46+
<scope>test</scope>
47+
</dependency>
48+
<dependency>
49+
<groupId>org.apache.hadoop</groupId>
50+
<artifactId>hadoop-minicluster</artifactId>
51+
<scope>test</scope>
52+
</dependency>
53+
</dependencies>
54+
55+
<build>
56+
<plugins>
57+
<plugin>
58+
<groupId>org.apache.maven.plugins</groupId>
59+
<artifactId>maven-source-plugin</artifactId>
60+
<configuration>
61+
<attach>true</attach>
62+
</configuration>
63+
<executions>
64+
<execution>
65+
<goals>
66+
<goal>jar</goal>
67+
</goals>
68+
</execution>
69+
</executions>
70+
</plugin>
71+
</plugins>
72+
</build>
73+
74+
<profiles>
75+
<profile>
76+
<id>dist</id>
77+
<activation>
78+
<activeByDefault>false</activeByDefault>
79+
</activation>
80+
<build>
81+
<plugins>
82+
<plugin>
83+
<groupId>org.apache.maven.plugins</groupId>
84+
<artifactId>maven-assembly-plugin</artifactId>
85+
<dependencies>
86+
<dependency>
87+
<groupId>org.apache.hadoop</groupId>
88+
<artifactId>hadoop-assemblies</artifactId>
89+
<version>${project.version}</version>
90+
</dependency>
91+
</dependencies>
92+
<executions>
93+
<execution>
94+
<id>dist</id>
95+
<phase>prepare-package</phase>
96+
<goals>
97+
<goal>single</goal>
98+
</goals>
99+
<configuration>
100+
<appendAssemblyId>false</appendAssemblyId>
101+
<attach>false</attach>
102+
<finalName>${project.artifactId}-${project.version}</finalName>
103+
<descriptorRefs>
104+
<descriptorRef>hadoop-dynamometer-blockgen</descriptorRef>
105+
</descriptorRefs>
106+
</configuration>
107+
</execution>
108+
</executions>
109+
</plugin>
110+
<plugin>
111+
<groupId>org.apache.maven.plugins</groupId>
112+
<artifactId>maven-dependency-plugin</artifactId>
113+
<executions>
114+
<execution>
115+
<id>deplist</id>
116+
<phase>compile</phase>
117+
<goals>
118+
<goal>list</goal>
119+
</goals>
120+
<configuration>
121+
<!-- referenced by a built-in command -->
122+
<outputFile>${project.basedir}/target/hadoop-tools-deps/${project.artifactId}.tools-builtin.txt</outputFile>
123+
</configuration>
124+
</execution>
125+
</executions>
126+
</plugin>
127+
</plugins>
128+
</build>
129+
</profile>
130+
</profiles>
131+
</project>

0 commit comments

Comments
 (0)