'maven_hadoop_template'

.gitignore

@@ -0,0 +1,19 @@
+.classpath
+.project
+.classpath
+.settings
+target
+*.log
+data
+build
+bin
+assets
+runtime
+*.class
+*.jar
+*.war
+*.ear
+
+
+
+

pom.xml

@@ -0,0 +1,24 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <groupId>org.conan.myhadoop.mr</groupId>
+  <artifactId>myHadoop</artifactId>
+  <packaging>jar</packaging>
+  <version>1.0-SNAPSHOT</version>
+  <name>myHadoop</name>
+  <url>http://maven.apache.org</url>
+  <dependencies>
+  	<dependency>
+  		<groupId>org.apache.hadoop</groupId>
+  		<artifactId>hadoop-core</artifactId>
+  		<version>1.0.3</version>
+  	</dependency>
+
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <version>4.4</version>
+      <scope>test</scope>
+    </dependency>
+  </dependencies>
+</project>

src/main/java/org/conan/myhadoop/mr/WordCount.java

@@ -0,0 +1,82 @@
+package org.conan.myhadoop.mr;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.StringTokenizer;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.FileInputFormat;
+import org.apache.hadoop.mapred.FileOutputFormat;
+import org.apache.hadoop.mapred.JobClient;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.MapReduceBase;
+import org.apache.hadoop.mapred.Mapper;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.Reducer;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.mapred.TextInputFormat;
+import org.apache.hadoop.mapred.TextOutputFormat;
+
+
+public class WordCount {
+
+    public static class WordCountMapper extends MapReduceBase implements Mapper<Object, Text, Text, IntWritable> {
+        private final static IntWritable one = new IntWritable(1);
+        private Text word = new Text();
+
+        @Override
+        public void map(Object key, Text value, OutputCollector<Text, IntWritable> output, Reporter reporter) throws IOException {
+            StringTokenizer itr = new StringTokenizer(value.toString());
+            while (itr.hasMoreTokens()) {
+                word.set(itr.nextToken());
+                output.collect(word, one);
+            }
+
+        }
+    }
+
+    public static class WordCountReducer extends MapReduceBase implements Reducer<Text, IntWritable, Text, IntWritable> {
+        private IntWritable result = new IntWritable();
+
+        @Override
+        public void reduce(Text key, Iterator<IntWritable> values, OutputCollector<Text, IntWritable> output, Reporter reporter) throws IOException {
+            int sum = 0;
+            while (values.hasNext()) {
+                sum += values.next().get();
+            }
+            result.set(sum);
+            output.collect(key, result);
+        }
+    }
+
+
+    public static void main(String[] args) throws Exception {
+        String input = "hdfs://192.168.1.210:9000/user/hdfs/o_t_account";
+        String output = "hdfs://192.168.1.210:9000/user/hdfs/o_t_account/result";
+
+        JobConf conf = new JobConf(WordCount.class);
+        conf.setJobName("WordCount");
+        conf.addResource("classpath:/hadoop/core-site.xml");
+        conf.addResource("classpath:/hadoop/hdfs-site.xml");
+        conf.addResource("classpath:/hadoop/mapred-site.xml");
+
+        conf.setOutputKeyClass(Text.class);
+        conf.setOutputValueClass(IntWritable.class);
+
+        conf.setMapperClass(WordCountMapper.class);
+        conf.setCombinerClass(WordCountReducer.class);
+        conf.setReducerClass(WordCountReducer.class);
+
+        conf.setInputFormat(TextInputFormat.class);
+        conf.setOutputFormat(TextOutputFormat.class);
+
+        FileInputFormat.setInputPaths(conf, new Path(input));
+        FileOutputFormat.setOutputPath(conf, new Path(output));
+
+        JobClient.runJob(conf);
+        System.exit(0);
+    }
+
+}

src/main/resources/hadoop/core-site.xml

@@ -0,0 +1,17 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<configuration>
+  <property>
+    <name>fs.default.name</name>
+    <value>hdfs://master:9000</value>
+  </property>
+  <property>
+    <name>hadoop.tmp.dir</name>
+    <value>/home/conan/hadoop/tmp</value>
+  </property>
+  <property>
+    <name>io.sort.mb</name>
+    <value>256</value>
+  </property>
+</configuration>

src/main/resources/hadoop/hdfs-site.xml

@@ -0,0 +1,17 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<configuration>
+  <property>
+    <name>dfs.data.dir</name>
+    <value>/home/conan/hadoop/data</value>
+  </property>
+  <property>
+    <name>dfs.replication</name>
+    <value>1</value>
+  </property>
+  <property>
+    <name>dfs.permissions</name>
+    <value>false</value>
+  </property>
+</configuration>

src/main/resources/hadoop/mapred-site.xml

@@ -0,0 +1,9 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<configuration>
+  <property>
+    <name>mapred.job.tracker</name>
+    <value>hdfs://master:9001</value>
+  </property>
+</configuration>