二十四、 Combiner合并介绍

一、什么是Combiner合并

本质：其实Combiner就是reduce在每个maptask执行的数据汇总，特殊情况可以将reduce作为Combiner；

二、自定义Combiner实现步骤

1、需求？

统计过程中对每一个MapTask的输出进行局部汇总，以减小网络传输量即采用Combiner功能

2、实现方式有两种：Combiner与Reducer

3、Combiner解决

3.1、自定义一个Combiner继承Reducer，重写Reduce方法

public class WordcountCombiner extends Reducer<Text, IntWritable, Text, IntWritable>{

    IntWritable v = new IntWritable();

    @Override
    protected void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException {

        // 1 汇总
        int sum = 0;

        for(IntWritable value :values){
            sum += value.get();
        }

        v.set(sum);

        // 2 写出
        context.write(key, v);
    }
}

3.2、Mapper

public class WordcountMapper extends Mapper<LongWritable, Text,Text, IntWritable> {

    // 输出数据格式，也就是reduce接受的格式
    Text k = new Text();
    IntWritable v = new IntWritable(1);

    @Override
    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {

        //获取当前行文本
        String line = value.toString();

        //按照 \t 分割字符
        String[] words = line.split("\t");

        //循环将每个字符写出，reduce会按照相同key的数据聚集到一起
        for (String word : words) {
            k.set(word);
            //word  1
            //java  1
            context.write(k,v);
        }
    }
}

3.3、Reduce

public class WordcountReducer extends Reducer<Text,IntWritable,Text, IntWritable> {

    int sum ;
    IntWritable  v = new IntWritable();
    @Override
    protected void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException {
        sum = 0;
        for (IntWritable count : values) {
            sum += count.get();
        }
        v.set(sum);
        context.write(key,v);
    }
}

3.4、Driver

public class WordcountDriver {

    public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
        // 1 获取配置信息以及封装任务
        Configuration configuration = new Configuration();
        Job job = Job.getInstance(configuration);

        // 2 设置jar加载路径
        job.setJarByClass(WordcountDriver.class);

        // 3 设置map和reduce类
        job.setMapperClass(WordcountMapper.class);
        job.setReducerClass(WordcountReducer.class);

        // 4 设置map输出
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(IntWritable.class);

        // 5 设置最终输出kv类型
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(IntWritable.class);

//        // 6 设置输入和输出路径

        FileInputFormat.setInputPaths(job, new Path("/Users/chenjunying/Downloads/input/wd.txt"));
        FileOutputFormat.setOutputPath(job, new Path("/Users/chenjunying/Downloads/out1"));

        // 8.1 指定需要使用combiner，以及用哪个类作为combiner的逻辑
        job.setCombinerClass(WordcountCombiner.class);
        // 7 提交
        boolean result = job.waitForCompletion(true);

        System.exit(result ? 0 : 1);
    }
}

4、Reduce解决方式

  //将WordcountCombiner 替换为  WordcountReducer
        // 8.2 指定需要使用reducer，以及用哪个类作为combiner的逻辑
        job.setCombinerClass(WordcountReducer.class);