Hadoop之MapReduce FlowBean 电话流量全排序

文件数据：

1	13736230513	192.196.100.1	www.hadoop.com	2481	24681	200
2	13846544121	192.196.100.2	264	0	200
3	13956435636	192.196.100.3	132	1512	200
4	13966251146	192.168.100.1	240	0	404
5	18271575951	192.168.100.2	www.hadoop.com	1527	2106	200
6	18418841312	192.168.100.3	www.hadoop.com	4116	1432	200
7	13590439668	192.168.100.4	1116	954	200
8	15910133277	192.168.100.5	www.hao123.com	3156	2936	200
9	13729199489	192.168.100.6	240	0	200
10	13630577991	192.168.100.7	www.shouhu.com	6960	690	200
11	15043685818	192.168.100.8	www.baidu.com	3659	3538	200
12	15959002129	192.168.100.9	www.hadoop.com	1938	180	500
13	13560439638	192.168.100.10	918	4938	200
14	13470253144	192.168.100.11	180	180	200
15	13682846555	192.168.100.12	www.qq.com	1938	2910	200
16	13992314666	192.168.100.13	www.gaga.com	3008	3720	200
17	13509468723	192.168.100.14	www.qinghua.com	7335	110349	404
18	18390173782	192.168.100.15	www.sogou.com	9531	2412	200
19	13975057813	192.168.100.16	www.baidu.com	11058	48243	200
20	13768778790	192.168.100.17	120	120	200
21	13568436656	192.168.100.18	www.alibaba.com	2481	24681	200
22	13568436656	192.168.100.19	1116	954	200

自定义FlowBean：

package com.mr.phone;

import org.apache.hadoop.io.WritableComparable;

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;

/**
 * @author kate
 */
public class FlowBean implements WritableComparable<FlowBean> {

    private String phone;
    private int upflow;
    private int downflow;
    private int sumflow;

    public FlowBean() {
    }

    public String getPhone() {
        return phone;
    }

    public void setPhone(String phone) {
        this.phone = phone;
    }

    public int getUpflow() {
        return upflow;
    }

    public void setUpflow(int upflow) {
        this.upflow = upflow;
    }

    public int getDownflow() {
        return downflow;
    }

    @Override
    public String toString() {
        return phone + "\t" + upflow + "\t" + downflow + "\t" + sumflow ;
    }

    public void setDownflow(int downflow) {
        this.downflow = downflow;
    }

    public int getSumflow() {
        return sumflow;
    }

    public void setSumflow(int sumflow) {
        this.sumflow = sumflow;
    }

    @Override
    public int compareTo(FlowBean o) {
        return o.getSumflow() - this.getSumflow();
    }

    @Override
    public void write(DataOutput out) throws IOException {
        out.writeUTF(this.phone);
        out.writeInt(upflow);
        out.writeInt(downflow);
        out.writeInt(sumflow);

    }

    @Override
    public void readFields(DataInput in) throws IOException {
        this.phone = in.readUTF();
        this.upflow = in.readInt();
        this.downflow = in.readInt();
        this.sumflow = in.readInt();
    }
}

自定义Mapper：

package com.mr.phone;

import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

import java.io.IOException;

/**
 * @author kate
 */
public class FlowBeanMapper  extends Mapper<LongWritable, Text,FlowBean, NullWritable> {
     FlowBean  flowBean= new  FlowBean();
    @Override
    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
        String[] split = value.toString().split("\t");
        String phone = split[1];
         //有的数据长度不对，所有需要从后取数据
        int upflow = Integer.parseInt(split[split.length - 3]);
        int downflow = Integer.parseInt(split[split.length - 2]);
        flowBean.setPhone(phone);
        flowBean.setUpflow(upflow);
        flowBean.setDownflow(downflow);
        flowBean.setSumflow(downflow+upflow);
        context.write(flowBean,NullWritable.get());
    }
}

自定义Reduce：

package com.mr.phone;

import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.mapreduce.Reducer;

import java.io.IOException;

/**
 * @author kate
 */
public class FlowBeanReduce extends Reducer<FlowBean, NullWritable,FlowBean,NullWritable> {
    FlowBean k=new FlowBean();
    @Override
    protected void reduce(FlowBean key, Iterable<NullWritable> values, Context context) throws IOException, InterruptedException {

        // 当电话没有重复 直接context.write(key,NullWritable.get());
        int upflow=0;
        int downflow=0;
        for (NullWritable value : values) {
         upflow+= key.getUpflow();
         downflow+=key.getDownflow();
        }

        k.setPhone(key.getPhone());
        k.setUpflow(upflow);
        k.setDownflow(downflow);
        k.setSumflow(upflow+downflow);
        context.write(k,NullWritable.get());
    }
}

自定义Driver：

package com.mr.phone;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import java.io.IOException;

/**
 * @author kate
 */
public class FlowDriver {
    public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
        // 1 获取job信息
        Configuration conf = new Configuration();
        Job job = Job.getInstance(conf);

        // 2 加载jar包
        job.setJarByClass(FlowDriver.class);

        // 3 关联map
        job.setMapperClass(FlowBeanMapper.class);
        job.setReducerClass(FlowBeanReduce.class);
        
        job.setMapOutputKeyClass(FlowBean.class);
        job.setMapOutputValueClass(NullWritable.class);
        // 4 设置最终输出类型
        job.setOutputKeyClass(FlowBean.class);
        job.setOutputValueClass(NullWritable.class);

        Path input = new Path("src/main/resources/input/phone/phone.txt");
        Path output = new Path("src/main/resources/output/phone");
        FileInputFormat.addInputPath(job, input);
        FileSystem system = FileSystem.get(new Configuration());
        if (system.exists(output)) {
            system.delete(output, true);
            System.out.println("存在的输出路径已经删除");
        }

        FileOutputFormat.setOutputPath(job, output);

        boolean b = job.waitForCompletion(true);
        System.exit(b ? 0 : 1);
    }
}

结果截图：