基于MapReduce计算TopN

一、项目需求

对上述用户流量表进行处理，输出总流量top10的数据

二、项目梳理

1、大致流程

项目设计分为如下类：
FlowBean类：封装用户数据，并定义数据比较方式
Mapper类：将从文件读入的每行数据按格式分隔后存入FlowBean输出
Comparator类：设置分组比较器，将所有数据输出到一个Reduce
Reducer类：将从Map端读取到的数据取top10，输出到目标文件

2、详细流程

1. 首先定义FlowBean，用来封装用户数据（手机号，上行流量，下行流量），还要实现WriteComparable接口，重写wirte()、readFields()用来实现数据的序列化，并重写compareTo()用来设置数据通过总流量进行降序排序。

2. 创建TopNMapper继承Mapper父类，重写map方法，在map方法中对每一行的数据通过spilt()进行切分，将切分好的数据存入定义的FlowBean变量，作为key输出，value没用到，就直接写NullWritable了。

3. 创建TopNComparator类，继承WriteComparator父类，重写compar()，使所有数据进入到同一个Reduce以便Reduce取出Top10。

4. 创建TopNReducer继承Reducer父类，重写reduce方法，在reduce方法中通过for循环的方式，将迭代器中的前10取出，输出到目标文件。

3、后序思考及优化

因为输入文件数据量小，考虑到在数据量很大的情况下，此种方法只开一个Reduce，必然会造成处理时间缓慢。于是采用了一个优化方法：通过Combiner
通过设置Combiner，将Map输出的数据进行局部求top10，再在reduce中，对combin的数据进行求top10操作，就会大大减少Map的输出量，减少了IO，提升了性能
注意：设置Combiner时应一并设置Comparator，使数据分到一组以便于求出top10

三、项目Demo

FlowBean类：

package com.wuyik1.topn;

import org.apache.hadoop.io.WritableComparable;

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;

public class FlowBean implements WritableComparable<FlowBean> {

    private String phone;
    private long upFlow;
    private long downFlow;
    private long sumFlow;

    public void set(String phone, long upFlow, long downFlow) {
        this.phone = phone;
        this.upFlow = upFlow;
        this.downFlow = downFlow;
        this.sumFlow = upFlow + downFlow;
    }

    @Override
    public String toString() {
        return phone + " " + upFlow + " " + downFlow + " " + sumFlow;
    }

    public String getPhone() {
        return phone;
    }

    public void setPhone(String phone) {
        this.phone = phone;
    }

    public long getUpFlow() {
        return upFlow;
    }

    public void setUpFlow(long upFlow) {
        this.upFlow = upFlow;
    }

    public long getDownFlow() {
        return downFlow;
    }

    public void setDownFlow(long downFlow) {
        this.downFlow = downFlow;
    }

    public long getSumFlow() {
        return sumFlow;
    }

    public void setSumFlow(long sumFlow) {
        this.sumFlow = sumFlow;
    }

    /**
     * 组内排序，按总流量升序
     * @param o
     * @return
     */
    @Override
    public int compareTo(FlowBean o) {
        //升序this.sumFlow放在前，降序this.sumFlow放在后
        return Long.compare(o.sumFlow,this.sumFlow);
    }

    @Override
    public void write(DataOutput dataOutput) throws IOException {

        dataOutput.writeUTF(phone);
        dataOutput.writeLong(upFlow);
        dataOutput.writeLong(downFlow);
        dataOutput.writeLong(sumFlow);

    }

    @Override
    public void readFields(DataInput dataInput) throws IOException {

        phone = dataInput.readUTF();
        upFlow = dataInput.readLong();
        downFlow = dataInput.readLong();
        sumFlow = dataInput.readLong();

    }
}

Mapper类：

package com.wuyik.topn;

import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

import java.io.IOException;

public class TopNMapper extends Mapper<LongWritable, Text, FlowBean, NullWritable> {

    private FlowBean flowBean = new FlowBean();

    @Override
    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
        String[] split = value.toString().split("\t");

        String phone = split[0];
        long upFlow = Long.parseLong(split[1]);
        long downFlow = Long.parseLong(split[2]);
        flowBean.set(phone, upFlow, downFlow);

        context.write(flowBean, NullWritable.get());

    }
}

Comparator类：

package com.wuyik.topn;

import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.io.WritableComparator;

public class TopNComparator extends WritableComparator {

    public TopNComparator() {
        super(FlowBean.class,true);
    }

    /**
     * 所有数据全部分到一组
     * @param a
     * @param b
     * @return
     */
    @Override
    public int compare(WritableComparable a, WritableComparable b) {
        return 0;
    }

}

Reducer类：

package com.wuyik.topn;

import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.mapreduce.Reducer;

import java.io.IOException;
import java.util.Iterator;

public class TopNReducer extends Reducer<FlowBean, NullWritable, FlowBean, NullWritable> {

    @Override
    protected void reduce(FlowBean key, Iterable<NullWritable> values, Context context) throws IOException, InterruptedException {

        Iterator<NullWritable> value = values.iterator();

        for (int i = 0; i < 10; i++) {

            if (value.hasNext()){
                context.write(key, NullWritable.get());
            }

        }

    }
}

Driver类：

package com.wuyik.topn;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import java.io.IOException;

public class TopNDriver {

    public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {

        Job job = Job.getInstance(new Configuration());

        job.setJarByClass(TopNDriver.class);

        job.setMapperClass(TopNMapper.class);
        job.setReducerClass(TopNReducer.class);

        job.setMapOutputKeyClass(FlowBean.class);
        job.setMapOutputValueClass(NullWritable.class);
        job.setOutputKeyClass(FlowBean.class);
        job.setOutputValueClass(NullWritable.class);

        //启用combiner求局部前10，优化Map输出量
        job.setCombinerClass(TopNReducer.class);
        job.setCombinerKeyGroupingComparatorClass(TopNComparator.class);

        job.setGroupingComparatorClass(TopNComparator.class);

        FileInputFormat.setInputPaths(job, new Path("E:\\hadoop_project\\input"));
        FileOutputFormat.setOutputPath(job, new Path("E:\\hadoop_project\\output"));

        boolean b = job.waitForCompletion(true);

        System.exit(b ? 0 : 1);

    }

}