MapReduce练习二:ChainMapper和ChainReducer的使用

练习的题目如图：

解决这类问题的思路是：使用一个Mapper来过滤掉不需要的英文呢，然后就是常规的MapReduce程序，这里使用ChainMapper和ChainReducer，所用到的资源http://pan.baidu.com/s/1i502dIt

具体代码如下：

import java.io.IOException;
import java.util.Iterator;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.FileInputFormat;
import org.apache.hadoop.mapred.FileOutputFormat;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.KeyValueTextInputFormat;
import org.apache.hadoop.mapred.MapReduceBase;
import org.apache.hadoop.mapred.Mapper;
import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reducer;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapred.TextOutputFormat;
import org.apache.hadoop.mapred.lib.ChainMapper;
import org.apache.hadoop.mapred.lib.ChainReducer;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;


public class ChainMapReduce extends Configured implements Tool {
	
	public static class MapFilter extends MapReduceBase implements Mapper<Text,Text,Text,Text>
	{

		public void map(Text key, Text value, OutputCollector<Text, Text> collector,
				Reporter reporter) throws IOException {
			String s=key.toString();
			if(s.matches("^[0-9]*$"))
			{
				collector.collect(value, key);
			}
		}
		
	}

	public static class MapClass extends MapReduceBase implements Mapper<Text,Text,IntWritable,Text>
	{

		public void map(Text key, Text value, OutputCollector<IntWritable, Text> collect,
				Reporter report) throws IOException {
			collect.collect(new IntWritable(Integer.parseInt(key.toString())), key);
		}
		
	}
	
	
	public static class ReducerClass extends  MapReduceBase implements Reducer<IntWritable, Text, IntWritable, IntWritable>
	{

		public void reduce(IntWritable key, Iterator<Text> values,
				OutputCollector<IntWritable, IntWritable> collect, Reporter report)
				throws IOException {
			int count=0;
			while(values.hasNext())
			{
				values.next();
				count++;
			}
			collect.collect(key, new IntWritable(count));
		}

	}
	
	
	//核心
	public int run(String[] arg0) throws Exception {
		//创建一个作业
		Configuration conf=getConf();
		JobConf jobConf=new JobConf(conf,ChainMapReduce.class);
		//设置输入输出路径
		Path in=new Path(arg0[0]);
		Path out=new Path(arg0[1]);
		FileInputFormat.setInputPaths(jobConf,in);
		FileOutputFormat.setOutputPath(jobConf, out);
		//设置mapper、reducer对象
		jobConf.setJobName("ChainMapReduce");
		
		//指定K1、V1的数据格式
		jobConf.setInputFormat(KeyValueTextInputFormat.class);
		jobConf.setOutputFormat(TextOutputFormat.class);
		jobConf.set("key.value.separator.in.input.line", ",");
		
		JobConf map1=new JobConf(false);
		ChainMapper.addMapper(jobConf, MapFilter.class, Text.class, Text.class, Text.class, Text.class, true,map1);
		
		JobConf map2=new JobConf(false);
		ChainMapper.addMapper(jobConf, MapClass.class, Text.class, Text.class, IntWritable.class, Text.class, true, map2);
		
		JobConf reduce=new JobConf(false);
		ChainReducer.setReducer(jobConf, ReducerClass.class, IntWritable.class, Text.class, IntWritable.class, IntWritable.class, true, reduce);
		//启动MapReduce作业
		JobClient.runJob(jobConf);
		
		return 0;
	}
	
	public static void main(String[] args) throws Exception {
		int res=ToolRunner.run(new Configuration(), new ChainMapReduce(), args);
		System.exit(res);
	}

}