流量汇总mapreduce

FlowCountMapper.java

package os.os.flowcount;

import java.io.IOException;

import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;


/*
 * KEYIN:日子文件一行的起始偏移量
 * VALUE:日子文件的一行内容
 * 
 * KEYOUT:map阶段输出的key
 * VALUEOUT:流量信息,FlowBean
 */
public class FlowCountMapper extends Mapper<LongWritable, Text, Text, FlowBean>{

    @Override
    protected void map(LongWritable key, Text value,Context context)throws IOException, InterruptedException {

        //拿到一行日志内容 转换为string
        String line = value.toString();

        // 通过分隔符切换出各个字段
        String[] fields = StringUtils.split(line,"\t");

        //抽取需要的字段

        String phone = fields[1];
        long upFlow = Long.parseLong(fields[fields.length-3]);
        long downFlow = Long.parseLong(fields[fields.length -2 ]);

        //输出key,value对 <phone,FlowBean>
        FlowBean bean = new FlowBean(upFlow,downFlow);

        context.write(new Text(phone), bean);

    }
}

FlowCountReduce.java

package os.os.flowcount;

import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

public class FlowCountReduce extends Reducer<Text, FlowBean, Text, FlowBean>{

    /*
     * key是一个手机号
     * values是这个手机号对应的所有kv
     * @see org.apache.hadoop.mapreduce.Reducer#reduce(KEYIN, java.lang.Iterable, org.apache.hadoop.mapreduce.Reducer.Context)
     */
    protected void reduce(Text key, java.lang.Iterable<FlowBean> values, Context context) throws java.io.IOException ,InterruptedException {

        long upflowSum = 0;//上行流量的和
        long downflowSum = 0; //下行流量和

        for(FlowBean value: values) {
            upflowSum += value.getUpflow();
            downflowSum += value.getDownflow();
        }

        FlowBean bean = new FlowBean(upflowSum,downflowSum);

        context.write(key, bean);
    }

}

FlowBean.java

package os.os.flowcount;

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;

import org.apache.hadoop.io.Writable;

/*
 * 自定义的数据类型,要做hadoop集群中传递,需要实现hadoop序列化框架,就是去实现一个接口 Writable
 */
public class FlowBean implements Writable{

    private long upflow;    //上行流量
    private long downflow;  //下行流量
    private long sumflow; //总流量


    //因为反射机制需要,必须定义一个无参构造函数
    public FlowBean() {

    }

    public long getUpflow() {
        return upflow;
    }
    public void setUpflow(long upflow) {
        this.upflow = upflow;
    }
    public long getDownflow() {
        return downflow;
    }
    public void setDownflow(long downflow) {
        this.downflow = downflow;
    }


    public long getSumflow() {
        return sumflow;
    }

    public void setSumflow(long sumflow) {
        this.sumflow = sumflow;
    }

    public FlowBean(long upflow, long downflow) {
        this.upflow = upflow;
        this.downflow = downflow;
        this.sumflow = upflow + downflow;
    }
    /*
     * 序列化方法,将我们要传输的数据序列化字节流
     * @see org.apache.hadoop.io.Writable#write(java.io.DataOutput)
     */
    @Override
    public void write(DataOutput out) throws IOException {

        out.writeLong(upflow);
        out.writeLong(downflow);

    }

    /*
     * 反序列化的方法,从自己流中恢复出各个字段
     * @see org.apache.hadoop.io.Writable#readFields(java.io.DataInput)
     */
    @Override
    public void readFields(DataInput in) throws IOException {

        upflow = in.readLong();  //从网络字节序,变成主机字节序
        downflow = in.readLong();
    }

    @Override
    public String toString() {
        return "FlowBean [upflow=" + upflow + ", downflow=" + downflow + ", sumflow=" + sumflow + "]";
    }


}

FlowCountJob.java

package os.os.flowcount;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;



public class FlowCountJob {

    public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {

        Job job = Job.getInstance(new Configuration());


        job.setJarByClass(FlowCountJob.class);

        job.setMapperClass(FlowCountMapper.class);
        job.setReducerClass(FlowCountReduce.class);

        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(FlowBean.class);

        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(FlowBean.class);

        FileInputFormat.setInputPaths(job, new Path(args[0]));
        FileOutputFormat.setOutputPath(job, new Path(args[1]));

        job.waitForCompletion(true);
    }
}

日志格式
1363157985066 13726230503 00-FD-07-A4-72-B8:CMCC 120.196.100.82 i02.c.aliimg.com 24 27 2481 24681 200