MapReduce 单词统计 WordCount

1.添加MapReduce的相关jar包

2.实现map()方法

package com.mapreduce;

import java.io.IOException;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

public class WordCountMapper extends Mapper<LongWritable, Text, Text, IntWritable> {
    private Text outK=new Text();
    private IntWritable outV= new IntWritable(1);
	
	protected void map(LongWritable key, Text value,Context context)
			throws IOException, InterruptedException {
		// TODO Auto-generated method stub
		super.map(key, value, context);
		String data=value.toString();
		String[] words=data.split("[,\\.\\s]+");
	 for(String word : words){
		 
		 outK.set(word);
		context.write(outK,outV);
	}
	}

}

3.实现reduce()方法

package com.mapreduce;

import java.io.IOException;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

public class WordCountReducer extends Reducer<Text,IntWritable,Text,IntWritable> {
  private IntWritable outV=new IntWritable();
	@Override
	protected void reduce(Text key, Iterable<IntWritable> values,Context context) throws IOException, InterruptedException {
		super.reduce(key, values, context);
		int sum=0;
		for (IntWritable value : values) {
			sum += value.get();
		}
		
		outV.set(sum);
		context.write(key, outV);
	}

}

4.drive部分：

package com.mapreduce;

import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.IntWritable;

  
public class WordCountDriver {
  public static void main(String[] args) throws Exception {
	  Configuration conf = new Configuration();
	  Job job =Job.getInstance(conf);
	  
	  job.setJarByClass(WordCountDriver.class);
	  
	  job.setMapperClass(WordCountMapper.class);
	  job.setReducerClass(WordCountReducer.class);
	  
	  job.setMapOutputKeyClass(Text.class);
	  job.setMapOutputValueClass(IntWritable.class);
	  job.setOutputKeyClass(Text.class);
	  job.setOutputValueClass(IntWritable.class);
	  
	  FileInputFormat.setInputPaths(job, new Path(c:****)); //设置文件input的输入路径
	  FileOutputFormat.setOutputPath(job,new Path(c:****)); //设置输出路径output，注意输出路径文件必须是新建的，如果已经存在则无法运行
	  
	  boolean result=job.waitForCompletion(true);
	  System.exit(result ? 0 : 1);
}
}

将需要统计的文件放入 input文件夹，本地运行程序。output中即可查看结果

用命令行参数指定输入路径和输出路径 

FileInputFormat.setInputPaths(job, new Path(args[0]));

FileOutputFormat.setOutputPath(job,new Path(args[1]));

之后即可在集群中上传需要执行的文件，集群中运行程序