对案例二话费汇总案例汇总后的结果进行处理,按照总花费升序排序输出。
分析:
把程序分两步走,也就是要写两个mapreduce程序,第一个用于统计每个用户的总话费(已经在案例2.4实现),第二个mapreduce程序把结果排序后输出即可。
要想利用框架进行排序,要做两步:
1:把要排序的字段置于mapper的keyout,因为我们的总花费位于bean中,因此,要让bean位于mapper的keyout。
2:让bean实现WritableComparable接口,重写compareTo方法,通过该方法告知框架我们要按照bean的总花费进行排序,按照升序排序。
既然bean作为keyout,那么手机号就要当valueout。
a) Mapper端读取每行数据封装bean,context.write(bean<总话费>,手机号)
b) Phone类实现WritableComparable接口重写compareTo方法
c) Reduce无需做特殊处理,将kv原样写出即可。
程序设计:
1、Phone对象原来只实现Writable接口,现在要实现WritableComparable接口,比原来增加了比较功能
import org.apache.hadoop.io.WritableComparable;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
public class Phone implements WritableComparable<Phone> {
private int baseFee;
private int viceFee;
private int msgFee;
private int flowFee;
private int sumFree;
public Phone() {
}
public Phone(int baseFee, int viceFee, int msgFee, int flowFee, int sumFree) {
this.baseFee = baseFee;
this.viceFee = viceFee;
this.msgFee = msgFee;
this.flowFee = flowFee;
this.sumFree = sumFree;
}
public int getBaseFee() {
return baseFee;
}
public void setBaseFee(int baseFee) {
this.baseFee = baseFee;
}
public int getViceFee() {
return viceFee;
}
public void setViceFee(int viceFee) {
this.viceFee = viceFee;
}
public int getMsgFee() {
return msgFee;
}
public void setMsgFee(int msgFee) {
this.msgFee = msgFee;
}
public int getFlowFee() {
return flowFee;
}
public void setFlowFee(int flowFee) {
this.flowFee = flowFee;
}
public int getSumFree() {
return sumFree;
}
public void setSumFree(int sumFree) {
this.sumFree = sumFree;
}
public void readFields(DataInput in) throws IOException {
baseFee=in.readInt();
viceFee=in.readInt();
msgFee=in.readInt();
flowFee=in.readInt();
sumFree=in.readInt();
}
public void write(DataOutput out) throws IOException {
out.writeInt(baseFee);
out.writeInt(viceFee);
out.writeInt(msgFee);
out.writeInt(flowFee);
out.writeInt(sumFree);
}
public int compareTo(Phone o) {
return -(this.sumFree-o.sumFree);
}
@Override
public String toString() {
return baseFee + " "+ viceFee + " " + msgFee + " " + flowFee + " " + sumFree;
}
}
2、编写mapper
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
import java.io.IOException;
public class myMapper extends Mapper<LongWritable, Text,Phone,Text> {
@Override
protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
String line = value.toString();
String[] strings = line.split("\t");
Phone phone =new Phone();
phone.setBaseFee(Integer.parseInt(strings[1]));
phone.setViceFee(Integer.parseInt(strings[2]));
phone.setMsgFee(Integer.parseInt(strings[3]));
phone.setFlowFee(Integer.parseInt(strings[4]));
phone.setSumFree(Integer.parseInt(strings[5]));
String num=strings[0];
context.write(phone,new Text(num));
}
}
3、 编写reducer
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;
import java.io.IOException;
public class myReducer extends Reducer<Phone, Text,Text,Phone> {
@Override
protected void reduce(Phone key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
for (Text value : values) {
context.write(value,key);
}
}
}
4、编写driver
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
public class myDriver {
public static void main(String[] args) throws Exception{
Configuration conf = new Configuration();
Job job = Job.getInstance(conf);
job.setJarByClass(myDriver.class);
job.setMapperClass(myMapper.class);
job.setReducerClass(myReducer.class);
job.setMapOutputKeyClass(Phone.class);
job.setMapOutputValueClass(Text.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Phone.class);
FileInputFormat.setInputPaths(job,new Path("F:/out/free1/part-r-00000"));
FileSystem fs = FileSystem.get(conf);
Path outPath= new Path("F:/out/freeScor");
boolean b = fs.exists(outPath);
if(b) fs.delete(outPath,true);
FileOutputFormat.setOutputPath(job,outPath);
boolean b1 = job.waitForCompletion(true);
System.out.println(b1);
}
}
处理结果: