Mapreduce的排序初步

1． 需求

在得出统计每一个用户（手机号）所耗费的总上行流量、下行流量，总流量结果的基础之上再加一个需求：将统计结果按照总流量倒序排序。

2． 分析

基本思路：

实现自定义的bean来封装流量信息，并将bean作为map输出的key来传输

MR程序在处理数据的过程中会对数据排序(map输出的kv对传输到reduce之前，会排序)，排序的依据是map输出的key。所以，我们如果要实现自己需要的排序规则，则可以考虑将排序因素放到key中，让key实现接口：WritableComparable，然后重写key的compareTo方法。

3． 实现

自定义的bean

[AppleScript] 纯文本查看 复制代码 ? 001 002 003 004 005 006 007 008 009 010 011 012 013 014 015 016 017 018 019 020 021 022 023 024 025 026 027 028 029 030 031 032 033 034 035 036 037 038 039 040 041 042 043 044 045 046 047 048 049 050 051 052 053 054 055 056 057 058 059 060 061 062 063 064 065 066 067 068 069 070 071 072 073 074 075 076 077 078 079 080 081 082 083 084 085 086 087 088 089 090 091 092 093 094 095 096 097 098 099 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 public class FlowBean implements WritableComparable<FlowBean>{       private long upFlow;       private long downFlow;       private long sumFlow;           //这里反序列的时候会用到       public FlowBean() {       }            public FlowBean(long upFlow, long downFlow, long sumFlow) {           this.upFlow = upFlow;           this.downFlow = downFlow;           this.sumFlow = sumFlow;       }               public FlowBean(long upFlow, long downFlow) {           this.upFlow = upFlow;           this.downFlow = downFlow;           this.sumFlow = upFlow+downFlow;       }               public void set(long upFlow, long downFlow) {           this.upFlow = upFlow;           this.downFlow = downFlow;           this.sumFlow = upFlow+downFlow;       }            @Override       public String toString() {           return upFlow+"\t"+downFlow+"\t"+sumFlow;       }            /这里是序列化方法       @Override       public void write(DataOutput out) throws IOException {             out.writeLong(upFlow);             out.writeLong(downFlow);             out.writeLong(sumFlow);       }            /这里是反序列化方法              @Override       public void readFields(DataInput in) throws IOException {           //注意反序列化的顺序跟序列化的顺序一致          this.upFlow = in.readLong();          this.downFlow = in.readLong();          this.sumFlow = in.readLong();                   }            //这里进行bean的自定义比较大小       @Override       public int compareTo(FlowBean o) {           //实现按照 sumflow 的大小倒序排序           return this.sumFlow>o.getSumFlow()?-1:1;       }       }

 

[AppleScript] 纯文本查看 复制代码 ? 01 02 03 04 05 06 07 08 09 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 public class FlowSumMapper extends Mapper<LongWritable, Text, Text, FlowBean>{       Text k = new Text();       FlowBean v = new FlowBean();               @Override       protected void map(LongWritable key, Text value,Context context)               throws IOException, InterruptedException {                                   String line = value.toString();               String[] fields = line.split("\t");               String phoneNum = fields[1];               long upFlow = Long.parseLong(fields[fields.length-3]);               long downFlow = Long.parseLong(fields[fields.length-2]);               k.set(phoneNum);               v.set(upFlow, downFlow);                               context.write(k, v);       }   }        public class FlowSumReducer extends Reducer<Text, FlowBean, Text, FlowBean> {   FlowBean v = new FlowBean();            @Override       protected void reduce(Text key, Iterable<FlowBean> values, Context context)               throws IOException, InterruptedException {                long upFlowCount = 0;           long downFlowCount = 0;           for (FlowBean bean : values) {               upFlowCount += bean.getUpFlow();               downFlowCount += bean.getDownFlow();           }           v.set(upFlowCount, downFlowCount);           context.write(key, v);       }        }