需求:上次是从日志中将手机流量汇总汇总,现在需要按照总流量降序排列
下面是上次汇总的结果:
1348025**** 180 180 360
1350246**** 7335 110349 117684
1356043**** 3597 25635 29232
1356043**** 2034 5892 7926
1360284**** 1938 2910 4848
1366057**** 6960 690 7650
1371919**** 240 0 240
1372623**** 2481 24681 27162
1376077**** 120 120 240
1382654**** 264 0 264
1392231**** 3008 3720 6728
1392505**** 11058 48243 59301
......
1:定义FlowBean类
package PhoneDataComparable; import org.apache.hadoop.io.WritableComparable; import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; public class FlowBean implements WritableComparable<FlowBean> { private long upFlow; private long downFlow; private long sumFlow; public long getUpFlow() { return upFlow; } public void setUpFlow(long upFlow) { this.upFlow = upFlow; } public long getDownFlow() { return downFlow; } public void setDownFlow(long downFlow) { this.downFlow = downFlow; } public long getSumFlow() { return sumFlow; } public void setSumFlow(long sumFlow) { this.sumFlow = sumFlow; } //无参数的构造方法 public FlowBean(){} //构造方法赋值 public void set(long upFlow,long downFlow){ this.upFlow = upFlow; this.downFlow = downFlow; this.sumFlow = upFlow + downFlow; } //比较方法,降序是-1,升序是1 @Override public int compareTo(FlowBean o) { return this.sumFlow > o.getSumFlow() ? -1 : 1; } //序列化 @Override public void write(DataOutput out) throws IOException { out.writeLong(upFlow); out.writeLong(downFlow); out.writeLong(sumFlow); } //反序列化,反序列化顺序必须与序列化顺序一致 @Override public void readFields(DataInput in) throws IOException { this.upFlow = in.readLong(); this.downFlow = in.readLong(); this.sumFlow = in.readLong(); } @Override public String toString() { return upFlow + "\t" + downFlow + "\t" + sumFlow; } }
2:定义PhoneDataMap类
package PhoneDataComparable; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Mapper; import java.io.IOException; public class PhoneDataMap extends Mapper<LongWritable, Text, FlowBean, Text> { FlowBean k = new FlowBean(); Text v = new Text(); @Override protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException { //1:获取一行 String line = value.toString(); //2:切分字段 String[] fields = line.split("\t"); //3:封装对象 //取出数据 String phoneNum = fields[0]; //取出上行和下行流量 long upFlow = Long.parseLong(fields[1]); long downFlow = Long.parseLong(fields[2]); k.set(upFlow,downFlow); v.set(phoneNum); //4:输出 context.write(k,v); } }
3:定义PhoneDataReducer类
package PhoneDataComparable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Reducer; import java.io.IOException; public class PhoneDataReducer extends Reducer<FlowBean,Text,Text,FlowBean> { @Override protected void reduce(FlowBean key, Iterable<Text> values, Context context) throws IOException, InterruptedException { for(Text text : values){ context.write(text,key); } } }
4:定义PhoneDataDriver类
package PhoneDataComparable; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; import java.io.IOException; public class PhoneDataDriver { public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException { args = new String[]{"E:\\bigdata_code\\phone_data1","E:\\bigdata_code\\out1"}; //1.获取配置信息,job对象实例 Configuration conf = new Configuration(); Job job = Job.getInstance(); //2:指定jar包的本地路径 job.setJarByClass(PhoneDataDriver.class); //3.关联map和输入、输出数据类型 job.setMapperClass(PhoneDataMap.class); job.setMapOutputKeyClass(FlowBean.class); job.setMapOutputValueClass(Text.class); //4.关联reduce的输入、输出数据类型 job.setReducerClass(PhoneDataReducer.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(FlowBean.class); //5:定义输入和输出路径 FileInputFormat.setInputPaths(job,new Path(args[0])); FileOutputFormat.setOutputPath(job,new Path(args[1])); //6:提交 job.waitForCompletion(true); } } 在运行后,得到如下结果:
1350246**** 7335 110349 117684
1392505**** 11058 48243 59301
1356043**** 3597 25635 29232
1372623**** 2481 24681 27162
1832017**** 9531 2412 11943
1356043**** 2034 5892 7926
1366057**** 6960 690 7650
1501368**** 3659 3538 7197
1392231**** 3008 3720 6728
1592013**** 3156 2936 6092
8413**** 4116 1432 5548
1360284**** 1938 2910 4848
1821157**** 1527 2106 3633
1598900**** 1938 180 2118
1392643**** 132 1512 1644