MapReduce自定义writable统计文本中30W条信息中的电话号码的上下行及总流量

Driver类

public class flowSum {
static class mymapper extends Mapper<LongWritable, Text, Text, FlowBean>{
        @Override
        protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
            //切分字段          
            String[] fields = value.toString().split("\t"); 
            //取出手机号         
            String phoneNbr = fields[1];            
            //取出上行流量下行流量            
            long upFlow = Long.parseLong(fields[fields.length-3]);
            long dFlow = Long.parseLong(fields[fields.length-2]);   
//          FlowBean fb=new FlowBean(upFlow, dFlow);
//          context.write(new Text(phoneNbr), fb);
            context.write(new Text(phoneNbr), new FlowBean(upFlow, dFlow));
        }
    }
    
    static class myreducer extends Reducer< Text, FlowBean,Text, FlowBean>{
        @Override
        protected void reduce(Text key, Iterable<FlowBean> values, Context context) throws IOException, InterruptedException {
            long sum_upFlow = 0;    
            long sum_dFlow = 0;         
            //遍历所有bean，将其中的上行流量，下行流量分别累加    
            for(FlowBean bean: values){ 
                sum_upFlow += bean.getUpFlow();     
                sum_dFlow += bean.getdFlow();   
            }                       
            FlowBean fb = new FlowBean(sum_upFlow, sum_dFlow);      
            context.write(key, fb);
        }
    }
    
    public static void main(String[] args) throws Exception {
        Configuration conf = new Configuration();
        Job job=Job.getInstance(conf);
        //设置主函数的入口
        job.setJarByClass(flowSum.class);
        //设置job的mapper的类
        job.setMapperClass(mymapper.class);
        //设置job的reduce类
        job.setReducerClass(myreducer.class);
        //设置map输出key的类型
        job.setMapOutputKeyClass(Text.class);
        //设置map输出value的类型
        job.setMapOutputValueClass(FlowBean.class);     
        //设置reduce输出key的类型
        job.setOutputKeyClass(Text.class);
        //设置reduce输出value的类型
        job.setOutputValueClass(FlowBean.class);        
        
//      job.setCombinerClass(WordCountReduce.class);
        
        
        //文件输入类
        Path inpath=new Path("hdfs://master:9000/hadoop/input");
        FileInputFormat.addInputPath(job, inpath);
        //文件输出类
        Path outpath=new Path("hdfs://master:9000/hadoop/result2");
        FileOutputFormat.setOutputPath(job,outpath);
        //提交job
//      job.submit();//这个不打印日志
        job.waitForCompletion(true);
//      System.exit(job.waitForCompletion(true) ? 0 : 1);
    }
}

FlowBean

public class FlowBean implements Writable{
    private long upFlow;
    private long dFlow;
    private long sumFlow;
    // 反序列化时，需要反射调用空参构造函数，所以要显示定义一个
    public FlowBean() {
    }
    public FlowBean(long upFlow, long dFlow) {
        this.upFlow = upFlow;
        this.dFlow = dFlow;
        this.sumFlow = upFlow + dFlow;
    }
    public long getUpFlow() {
        return upFlow;
    }
    public void setUpFlow(long upFlow) {
        this.upFlow = upFlow;
    }
    public long getdFlow() {
        return dFlow;
    }
    public void setdFlow(long dFlow) {
        this.dFlow = dFlow;
    }
    public long getSumFlow() {
        return sumFlow;
    }
    public void setSumFlow(long sumFlow) {
        this.sumFlow = sumFlow;
    }
    /** * 序列化方法 */
    public void write(DataOutput out) throws IOException {
        out.writeLong(upFlow);
        out.writeLong(dFlow);
        out.writeLong(sumFlow);
    }
    /** * 反序列化方法 * 注意：反序列化的顺序跟序列化的顺序完全一致 */
    public void readFields(DataInput in) throws IOException {
        upFlow = in.readLong();
        dFlow = in.readLong();
        sumFlow = in.readLong();
    }
    @Override
    public String toString() {
        return upFlow + "\t" + dFlow + "\t" + sumFlow;
    }
}

要注意implements Writable接口否则会报错 Initialization of all the collectors failed. Error in last collector was :null