0)需求
根据需求1产生的结果再次对总流量进行排序。
1)数据准备
2)分析
(1)把程序分两步走,第一步正常统计总流量,第二步再把结果进行排序
(2)context.write(总流量,手机号)
(3)FlowBean实现WritableComparable接口重写compareTo方法
@Override public int compareTo(FlowBean o) { // 倒序排列,从大到小 return this.sumFlow > o.getSumFlow() ? -1 : 1; } |
3)代码实现
(1)FlowBean对象在在需求1基础上增加了比较功能
package bigdata.b13.WritableComparable;
import lombok.Getter;
import lombok.Setter;
import org.apache.hadoop.io.WritableComparable;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
@Getter
@Setter
public class FlowBean implements WritableComparable<FlowBean> {
private long upflow;
private long downflow;
private long sumflow;
//空参构造器,反序列化需要
public FlowBean() {
}
public FlowBean(long upflow, long downflow, long sumflow) {
this.upflow = upflow;
this.downflow = downflow;
this.sumflow = sumflow;
}
@Override
// 自定义比较器,倒序排序
public int compareTo(FlowBean o) {
if (this.sumflow>o.sumflow){
return -1;
} else if (this.sumflow<o.sumflow) {
return 1;
}else {
return 0;
}
}
//序列化 注意:范徐丽阿华顺序和蓄力阿华顺序是对应
@Override
public void write(DataOutput dataOutput) throws IOException {
dataOutput.writeLong(upflow);
dataOutput.writeLong(downflow);
dataOutput.writeLong(sumflow);
}
//反序列化
@Override
public void readFields(DataInput dataInput) throws IOException {
this.upflow=dataInput.readLong();
this.downflow=dataInput.readLong();
this.sumflow=dataInput.readLong();
}
@Override
public String toString() {
return this.upflow+"\t"+this.downflow+"\t"+this.sumflow;
}
}
(2)编写mapper
package bigdata.b13.WritableComparable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
import java.io.IOException;
public class WritableComparableMapper extends Mapper<LongWritable, Text, FlowBean,Text> {
@Override
protected void map(LongWritable key, Text value, Mapper<LongWritable, Text, FlowBean, Text>.Context context) throws IOException, InterruptedException {
//获取数据
String string = value.toString();
//切分
String[] split = string.split("\t");
context.write(new FlowBean(Long.parseLong(split[1]),Long.parseLong(split[2]),Long.parseLong(split[3])),new Text(split[0]));
}
}
3)编写reducer
package bigdata.b13.WritableComparable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;
import java.io.IOException;
public class WritableComparableReducer extends Reducer<FlowBean, Text,Text,FlowBean> {
@Override
protected void reduce(FlowBean key, Iterable<Text> values, Reducer<FlowBean, Text, Text, FlowBean>.Context context) throws IOException, InterruptedException {
for (Text text:values){
context.write(text,key);
}
}
}
(4)编写driver
package bigdata.b13.WritableComparable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import java.io.IOException;
/**
* 需求3:将统计结果按照总流量倒序排序(全排序)
*/
public class WritableComparableDriver {
public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException {
args = new String[]{"D:\\test\\outwritable.txt","D:\\test\\writable2"};
//获取job信息
Configuration conf = new Configuration();
Job job = Job.getInstance(conf);
job.setJarByClass(WritableComparableDriver.class);
//关联map和reduce
job.setMapperClass(WritableComparableMapper.class);
job.setReducerClass(WritableComparableReducer.class);
//设置最终输出类型
job.setMapOutputKeyClass(FlowBean.class);
job.setMapOutputValueClass(Text.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(FlowBean.class);
//输入输出路径
FileInputFormat.setInputPaths(job,new Path(args[0]));
FileOutputFormat.setOutputPath(job,new Path(args[1]));
//提交任务
job.waitForCompletion(true);
}
}
5)输出结果