MapReduce 开篇练习之 统计流量

pojo类

public class FlowBean implements WritableComparable<FlowBean>{
    private long upFlow;
    private long dFlow;
    private long sumFlow;

    //反序列化时，需要反射调用空参构造函数
    public FlowBean() {
    }

    public FlowBean(long upFlow, long dFlow) {
        this.upFlow = upFlow;
        this.dFlow = dFlow;
        this.sumFlow=upFlow+dFlow;
    }

    public void setBean(long upFlow, long dFlow) {
        this.upFlow = upFlow;
        this.dFlow = dFlow;
        this.sumFlow=upFlow+dFlow;
    }

    //序列化
    @Override
    public void write(DataOutput out) throws IOException {
        out.writeLong(upFlow);
        out.writeLong(dFlow);
        out.writeLong(sumFlow);
    }


/***
     * 反序列化方法
     *  注意：反序列化的顺序跟序列化的顺序完全一致
     * @param in
     * @throws IOException
     */

    @Override
    public void readFields(DataInput in) throws IOException {
        upFlow=in.readLong();
        dFlow=in.readLong();
        sumFlow=in.readLong();
    }

    @Override
    public String toString() {
        return upFlow + "\t" + dFlow + "\t" + sumFlow;
    }


    public long getUpFlow() {
        return upFlow;
    }

    public void setUpFlow(long upFlow) {
        this.upFlow = upFlow;
    }

    public long getdFlow() {
        return dFlow;
    }

    public void setdFlow(long dFlow) {
        this.dFlow = dFlow;
    }

    public long getSumFlow() {
        return sumFlow;
    }

    public void setSumFlow(long sumFlow) {
        this.sumFlow = sumFlow;
    }

    @Override
    public int compareTo(FlowBean o) {
        return this.sumFlow>o.getSumFlow()?-1:1;
    }
}

public class FlowCount {


    //Mapper
    static class FlowCountMapper extends Mapper<LongWritable,Text,Text,FlowBean>{

        @Override
        protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
            //将一行内容转成string
            String line=value.toString();

            System.out.println(line);
            //切分字段
            String[] split = line.split("\t");
            //取出手机号
            String phone=split[1];
            //取出上行流量下行流量
            long upFlow = Long.parseLong(split[split.length - 3]);
            long dFlow=Long.parseLong(split[split.length-2]);

            context.write(new Text(phone),new FlowBean(upFlow,dFlow));

        }
    }


    //Reducer
    static class FlowCountReducer extends Reducer<Text,FlowBean,Text,FlowBean> {

        @Override
        protected void reduce(Text key, Iterable<FlowBean> values, Context context) throws IOException, InterruptedException {
            long sum_upFlow = 0;
            long sum_dFlow = 0;

            //遍历所有bean，将其中的上行流量，下行流量分别累计
            for (FlowBean bean:values){
                sum_dFlow+=bean.getdFlow();
                sum_upFlow+=bean.getUpFlow();
            }

            FlowBean resultBean = new FlowBean(sum_upFlow,sum_dFlow);
            context.write(key,resultBean);

        }
    }


    public static void main(String [] args) throws IOException, ClassNotFoundException, InterruptedException {
        Configuration conf=new Configuration();
        Job job= Job.getInstance(conf);

        //指定本程序的jar包所在本地路径
        job.setJarByClass(FlowCount.class);

        //指定本业务job要使用的mapper/reducer业务类
        job.setMapperClass(FlowCountMapper.class);
        job.setReducerClass(FlowCountReducer.class);

        //指定mapper输出数据的kv类型
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(FlowBean.class);

        //指定最终输出数据的kv类型
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(FlowBean.class);

        //指定job的输入原始文件所在目录
        FileInputFormat.setInputPaths(job,new Path(args[0]));
        //指定job的输出结果所在目录
        FileOutputFormat.setOutputPath(job,new Path(args[1]));

        //将job中配置的相关参数，以及job所用的Java类所在的jar包，提交给yarn去运行
        boolean res = job.waitForCompletion(true);
        System.exit(res?0:1);

    }
}