Hadoop 使用combiner提升性能

combiner的工作是在mapper结束之后，在本机上先进行的reducer操作，这样可以减轻数据网络传输的负担，提高性能。比如wordcount程序，mapper输出的是< key,1>这样的简直对，表示每个单词出现了一次，然后进行reduce，在数据量非常大的情况下，非常多的相同key的输出，在传送过程中会造成很大负担，所以我们可以在map节点上先进行combiner，将key相同的这些加起来，变成< key,N >，这样可以减少数据量。
combiner要实现Reducer接口：

import java.io.IOException;
import java.util.Iterator;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.util.EnumCounters.Map;
import org.apache.hadoop.io.DoubleWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.FileInputFormat;
import org.apache.hadoop.mapred.FileOutputFormat;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.MapReduceBase;
import org.apache.hadoop.mapred.Mapper;
import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reducer;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapred.TextInputFormat;
import org.apache.hadoop.mapred.TextOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;

import com.sun.org.apache.xerces.internal.util.SynchronizedSymbolTable;
import com.sun.security.auth.callback.TextCallbackHandler;


public class MyJob extends Configured implements Tool {

    public static class MapClass extends MapReduceBase implements Mapper<LongWritable, Text, Text, Text>{

        public void  map(LongWritable key,Text value,OutputCollector<Text, Text> output,Reporter reporter)throws IOException{

            String fields []=value.toString().split(",",-20);
            String country=fields[4];
            String numClaims=fields[8];
            if(numClaims.length()>0 && !numClaims.startsWith("\"")){
                output.collect(new Text(country), new Text(numClaims+",1"));
            }
        }
    }
    public static class Reduce extends MapReduceBase implements Reducer<Text, Text, Text, DoubleWritable>{

        public void reduce(Text key,Iterator<Text> values,OutputCollector<Text, DoubleWritable> output,Reporter reporter)throws IOException{

            double sum=0;
            int count=0;
            while(values.hasNext()){
                String fields []=values.next().toString().split(",");
                sum+=Double.parseDouble(fields[0]);
                count+=Integer.parseInt(fields[1]);
            }
            output.collect(key, new DoubleWritable(sum/count));
        }
    }
    public static class Combine extends MapReduceBase implements Reducer<Text, Text, Text, Text>{

        public void reduce(Text key,Iterator<Text> values,OutputCollector<Text, Text> output,Reporter reporter)throws IOException{

            double sum=0;
            int count=0;
            while(values.hasNext()){
                String fields []=values.next().toString().split(",");
                sum+=Double.parseDouble(fields[0]);
                count+=Integer.parseInt(fields[1]);
            }
            output.collect(key, new Text(sum+","+count));
        }
    }

    @Override
    public int run(String[] arg0) throws Exception {
        // TODO Auto-generated method stub

        Configuration configuration=getConf();
        JobConf job=new JobConf(configuration,MyJob.class);

        FileInputFormat.setInputPaths(job, new Path(arg0[0]));
        FileOutputFormat.setOutputPath(job, new Path(arg0[1]));

        job.setMapperClass(MapClass.class);
        job.setCombinerClass(Combine.class);
        job.setReducerClass(Reduce.class);

        job.setInputFormat(TextInputFormat.class);
        job.setOutputFormat(TextOutputFormat.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(DoubleWritable.class);
        job.setMapOutputValueClass(Text.class);

        JobClient.runJob(job);

        return 0;
    }

    public static void main(String[] args) throws Exception{
        // TODO Auto-generated method stub
        int res=ToolRunner.run(new Configuration(), new MyJob(),args);
        System.exit(res);
    }

}