hadoop入门 使用MapReduce编写comibiner WordCount程序（五）

1、为什么要用combiner
常，一个map可能会产生大量的输出，combiner的作用就是在map端对输出先做一次合并，以减少传输到reducer的数据量。
2、mapper

package com.example.hadoop.combiner;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

import java.io.IOException;

public class WordCountMapper extends Mapper<LongWritable, Text,Text, IntWritable> {
    private Text outT=new Text();
    private IntWritable outV=new IntWritable(1);
    @Override
    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
        //1、获取一行
        String line=value.toString();
        //2、切割
        String[] words=line.split(" ");
        //3、循环写出
        for (String word : words) {
            //封装outK
            outT.set(word);
            //写出
            context.write(outT,outV);
        }
    }
}

3、reducer

package com.example.hadoop.combiner;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

import java.io.IOException;

public class WordCountReducer extends Reducer<Text, IntWritable,Text,IntWritable> {
    private IntWritable outV=new IntWritable();
    @Override
    protected void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException {
        int sum=0;
        //累计
        for (IntWritable value : values) {
            sum+=value.get();
        }
        outV.set(sum);
        //写出
        context.write(key,outV);
    }
}

4、combiner
其实代码逻辑和reducer是一样的，可以用同个类，但是这里为了清晰分开写，其实就是在map端先执行了以下reducer的逻辑

package com.example.hadoop.combiner;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

import java.io.IOException;

public class WorderCountCombiner extends Reducer<Text, IntWritable,Text,IntWritable>{
    private IntWritable outV=new IntWritable();
    @Override
    protected void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException {
        int sum=0;
        //累计
        for (IntWritable value : values) {
            sum+=value.get();
        }
        outV.set(sum);
        //写出
        context.write(key,outV);
    }
}

5、main方法
主要代码job.setCombinerClass(WorderCountCombiner.class);

package com.example.hadoop.combiner;


import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import java.io.IOException;

public class WordCountDriver {
    public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException {
        //1、获取job
        Configuration conf = new Configuration();
        Job job= Job.getInstance(conf);
        //2、设置jar包途径
        job.setJarByClass(WordCountDriver.class);
        //3、关联mapper和reducer
        job.setMapperClass(WordCountMapper.class);
        job.setReducerClass(WordCountReducer.class);
        //4、设置map输出的kv类型
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(IntWritable.class);
        //5、设置最终输出的kv类型
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(IntWritable.class);
        //设置combiner
        job.setCombinerClass(WorderCountCombiner.class);
        //6、设置输入路径和输出路径
        FileInputFormat.setInputPaths(job,new Path("D:\\cls.txt"));
        FileOutputFormat.setOutputPath(job,new Path("D:\\hadoop\\output"));
        //7、提交job
        boolean result=job.waitForCompletion(true);

        System.exit(result?0:1);
    }
}

6、测试数据

sprak sprak hive hadoop netty sprak sprak hive hadoop netty sprak hive hadoop netty hive hadoop netty
sprak sprak hive hadoop netty sprak sprak hive hadoop netty sprak hive hadoop netty hive hadoop netty
sprak sprak hive hadoop netty sprak sprak hive hadoop netty sprak hive hadoop netty hive hadoop netty

结果

7、debug过程
先执行combiner

再执行reducer