FlowBean对象
import org.apache.hadoop.io.Writable;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
// 1 实现writable接口
public class FlowBean implements Writable {
public long getUpFlow() {
return upFlow;
}
public void setUpFlow(long upFlow) {
this.upFlow = upFlow;
}
public long getDownFlow() {
return downFlow;
}
public void setDownFlow(long downFlow) {
this.downFlow = downFlow;
}
public long getSumFlow() {
return sumFlow;
}
public void setSumFlow(long sumFlow) {
this.sumFlow = sumFlow;
}
//上传流量
private long upFlow;
//下载流量
private long downFlow;
//流量总和
private long sumFlow;
//必须要有,反序列化要调用空参构造器
public FlowBean() {
}
public FlowBean(long upFlow, long downFlow) {
this.upFlow = upFlow;
this.downFlow = downFlow;
this.sumFlow = upFlow + downFlow;
}
public void set(long upFlow, long downFlow){
this.upFlow = upFlow;
this.downFlow = downFlow;
this.sumFlow = upFlow + downFlow;
}
/**
* 序列化
*
* @param out
* @throws IOException
*/
@Override
public void write(DataOutput out) throws IOException {
out.writeLong(upFlow);
out.writeLong(downFlow);
out.writeLong(sumFlow);
}
/**
* 反序列化
* 注:字段属性顺序必须一致
*
* @param in
* @throws IOException
*/
@Override
public void readFields(DataInput in) throws IOException {
this.upFlow = in.readLong();
this.downFlow = in.readLong();
this.sumFlow = in.readLong();
}
@Override
public String toString() {
return upFlow + "\t" + downFlow + "\t" + sumFlow;
}
}
实现Mapper
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
import java.io.IOException;
public class PhoneMapper extends Mapper<LongWritable, Text,Text,FlowBean> {
@Override
protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
String line = value.toString();
String[] num = line.split("\t");
long upflow = Long.parseLong(num[num.length - 3]);
long downflow = Long.parseLong(num[num.length - 2]);
context.write(new Text(num[1]),new FlowBean(upflow,downflow));
}
}
实现Reducer
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;
import java.io.IOException;
public class PhoneReduce extends Reducer<Text,FlowBean,Text,FlowBean>{
@Override
protected void reduce(Text key, Iterable<FlowBean> values, Context context) throws IOException, InterruptedException {
long sumUpFlow = 0;
long sumDownFlow = 0;
for(FlowBean value : values){
sumDownFlow += value.getDownFlow();
sumUpFlow += value.getUpFlow();
}
context.write(key,new FlowBean(sumUpFlow,sumDownFlow));
}
}
Main方法
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import java.io.IOException;
public class PhoneDriver {
public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException{
long startTime = System.currentTimeMillis();
// args = new String[]{"D:\\hiketime\\phone_data", "D:\\hiketime"};
//1.获取配置信息
Configuration conf = new Configuration();
Job job = Job.getInstance(conf);
//2.反射类
job.setJarByClass(PhoneDriver.class);
job.setMapperClass(PhoneMapper.class);
job.setReducerClass(PhoneReduce.class);
//3.Reduce输入、输出的K、V类型
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(FlowBean.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(FlowBean.class);
//4.数据的输入和输出的指定目录
FileInputFormat.setInputPaths(job,new Path(args[0]));
FileOutputFormat.setOutputPath(job,new Path(args[1]));
//5.提交job
job.waitForCompletion(true);
long endTime = System.currentTimeMillis();
System.out.println("程序运行的时间为:"+(endTime-startTime));
}
}
JKSJ