开发Mapreduce

模拟hadoop中的wordcount示例

1.首先创建一个Maven项目

2.导入配置：

<dependencies>
		<dependency>
			<groupId>jdk.tools</groupId>
			<artifactId>jdk.tools</artifactId>
			<version>1.6</version>
			<scope>system</scope>
			<systemPath>C:\Program Files\Java\jdk1.8.0_131\lib\tools.jar</systemPath>
		</dependency>
		<!-- https://mvnrepository.com/artifact/junit/junit -->
		<dependency>
			<groupId>junit</groupId>
			<artifactId>junit</artifactId>
			<version>4.12</version>
			<scope>test</scope>
		</dependency>
		<!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-common -->
		<dependency>
			<groupId>org.apache.hadoop</groupId>
			<artifactId>hadoop-common</artifactId>
			<version>2.6.4</version>
		</dependency>
		<!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-hdfs -->
		<dependency>
			<groupId>org.apache.hadoop</groupId>
			<artifactId>hadoop-hdfs</artifactId>
			<version>2.6.4</version>
		</dependency>
		<!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-mapreduce-client-core -->
		<dependency>
			<groupId>org.apache.hadoop</groupId>
			<artifactId>hadoop-mapreduce-client-core</artifactId>
			<version>2.6.4</version>
		</dependency>

	</dependencies>

2.代码：

import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;


public class WordCountDemo {
	public static void main(String[] args) throws IOException, Exception, InterruptedException {
		Configuration configuration=new Configuration();
		Job job=Job.getInstance(configuration);
		job.setJarByClass(WordCountDemo.class);
		job.setMapperClass(WordCountMapper.class);
		job.setMapOutputKeyClass(Text.class);
		job.setMapOutputValueClass(LongWritable.class);
		job.setReducerClass(WordCountReducer.class);
		job.setOutputKeyClass(Text.class);
		job.setOutputValueClass(LongWritable.class);
		FileInputFormat.setInputPaths(job, new Path(args[0]));
		FileOutputFormat.setOutputPath(job, new Path(args[1]));;
		job.waitForCompletion(true);
	}
	public static class WordCountMapper extends Mapper<LongWritable, Text, Text, LongWritable>{

		@Override
		protected void map(LongWritable key, Text value, Mapper<LongWritable, Text, Text, LongWritable>.Context context)
				throws IOException, InterruptedException {
			String str=value.toString(); //转化为字符串
			String[] arr=str.split("\\s");  //拆分
			for (String s:arr) {     //遍历输出
				context.write(new Text(s), new LongWritable(1)); //将拆分好的字符串写出
			}
		}

	}
	public static class WordCountReducer extends Reducer<Text, LongWritable, Text, LongWritable>{

		@Override
		protected void reduce(Text arg0, Iterable<LongWritable> arg1,
				Reducer<Text, LongWritable, Text, LongWritable>.Context arg2) throws IOException, InterruptedException {
			long sum=0;
			for(LongWritable l:arg1 ) {
				sum+=l.get();
			}
			arg2.write(arg0, new LongWritable(sum ));
		}

	}
}

3.将其导成jar包，放入linux执行

 

选择主类

完成

执行：Hadoop jar wd.jar /word /out
查看：hdfs dfs -cat /out/*