MapReduce中Combiner方法使用

Combiner 会继承Reducer，它是一种mr的优化，用于减少服务器之间网络带宽的压力，它是在map结束后在每台服务器中都算出一个值，再传到shuffle中。适合于求和等每台服务器算出的值对最后结果没有影响的业务中，但是像求平均值等功能会带来误差所以不能使用。Combiner会在map结束后，shuffle开始前进行执行，

package com.it18zhang.day05.flow;

import java.io.IOException;

import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

public class FlowCountCombiner extends Reducer<Text, LongWritable, Text, LongWritable> {
  @Override
protected void reduce(Text key, Iterable<LongWritable> values,Context context)
        throws IOException, InterruptedException {
      long sum = 0;
      for(LongWritable value: values){
          sum += value.get();
      }

      context.write(key, new LongWritable(sum));

}
}

package com.it18zhang.day05.flow;

import java.io.IOException;
import java.net.URI;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

public class FlowCountDriver {

    public static void main(String[] args) throws Exception {

          String inPath = "";
          String outPath ="";

          if(args.length == 2){
              inPath = args[0];
              outPath = args[1];

          }





         Configuration conf = new Configuration();

         if(!outPath.equals("")){
             FileSystem fs = FileSystem.get(new URI("hdfs://192.168.81.201:8020/") , conf, "centos");
                Path f = new Path(outPath);
                boolean isTrue = fs.delete(f,true);
                if(isTrue){
                    System.out.println("删除成功");
                } 
         }
         Job job = Job.getInstance(conf);
         //设置主的环境
         job.setJarByClass(FlowCountDriver.class);

         //设置map和reduce
         job.setMapperClass(FlowCountMapper.class);
         job.setReducerClass(FlowCountReducer.class);

         //设置map的输出类型
         job.setMapOutputKeyClass(Text.class);
         job.setMapOutputValueClass(LongWritable.class);

         //设置reduce的输出类型
         job.setOutputKeyClass(Text.class);
         job.setOutputValueClass(LongWritable.class);


        //指定job的输入原始文件所在目录
         FileInputFormat.setInputPaths(job, new Path(inPath));
       //指定job的输出结果所在目录
         FileOutputFormat.setOutputPath(job, new Path(outPath));

       //将job中配置的相关参数，以及job所用的java类所在的jar包，提交给yarn去运行
        /*job.submit();*/
        boolean res = job.waitForCompletion(true);
        System.exit(res?0:1);
    }
}