流量统计,第一列为用户手机号,第二列为上行流量,第三列为下行流量,ok 问题来了 根据用户统计用户上行和下行流量
1363157985066,2481,24681
1363157995052,264,0
1363157991076,132,1512
1363157995052,240,0
1363157993044,1527,2106
1363157995074,4116,1432
1363157993055,1116,954
1363157995033,3156,2936
1363157983019,240,0
1363157984041,6960,690
1363157973098,3659,3538
1363157986029,1938,180
1363157992093,918,4938
1363157986041,180,180
1363157984040,1938,2910
1363157995093,3008,3720
1363157982040,7335,110349
1363157986072,9531,2412
1363157990043,11058,48243
package com.demo.mapreduce;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.*;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.util.HashSet;
import java.util.Set;
/**
* Created by leslie on 17/6/25.
*/
public class MapreduceNet {
static class NetFlow implements WritableComparable<NetFlow>{
private String phone="";
private long upLoad=0l;
private long dLoad=0l;
private long sLoad=0l;
public NetFlow(String phone, long upLoad, long dLoad, long sLoad) {
this.phone = phone;
this.upLoad = upLoad;
this.dLoad = dLoad;
this.sLoad = sLoad;
}
public String getPhone() {
return phone;
}
public void setPhone(String phone) {
this.phone = phone;
}
public long getUpLoad() {
return upLoad;
}
public void setUpLoad(long upLoad) {
this.upLoad = upLoad;
}
public long getdLoad() {
return dLoad;
}
public void setdLoad(long dLoad) {
this.dLoad = dLoad;
}
public long getsLoad() {
return sLoad;
}
public void setsLoad(long sLoad) {
this.sLoad = sLoad;
}
@Override
public int compareTo(NetFlow o) {
this.setsLoad(this.getUpLoad()+this.getdLoad());
o.setsLoad(o.getdLoad()+o.getUpLoad());
return Long.compare(this.getsLoad(),this.getsLoad());
}
@Override
public void write(DataOutput dataOutput) throws IOException {
dataOutput.writeChars(phone);
dataOutput.writeLong(upLoad);
dataOutput.writeLong(dLoad);
dataOutput.writeLong(sLoad);
}
@Override
public void readFields(DataInput dataInput) throws IOException {
phone = dataInput.readUTF();
upLoad = dataInput.readLong();
dLoad = dataInput.readLong();
sLoad = dataInput.readLong();
}
}
static class MyMapper extends Mapper<LongWritable,Text,Text,NetFlow>{
@Override
protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
String[] strings = value.toString().split(",");
if(strings.length==3){
long uload = Long.parseLong(strings[1]);
long dLoad = Long.parseLong(strings[2]);
long sLoad = uload+dLoad;
NetFlow flow = new NetFlow(strings[0],uload,dLoad,sLoad);
context.write(new Text(strings[0]),flow);
}
}
}
static class MyComparator extends WritableComparator{
public MyComparator(){super(NetFlow.class,true);}
@Override
public int compare(WritableComparable a, WritableComparable b) {
NetFlow n1 = (NetFlow)a;
NetFlow n2 = (NetFlow)b;
return Long.compare(n1.getsLoad(),n2.getsLoad());
}
}
static class MyReducer extends Reducer<Text,NetFlow,Text,NullWritable>{
@Override
protected void reduce(Text key, Iterable<NetFlow> values, Context context) throws IOException, InterruptedException {
long uLoad = 0;long dLoad = 0;
for(NetFlow flow:values){
uLoad+=flow.getUpLoad();
dLoad+= flow.getdLoad();
}
long sLoad = uLoad+dLoad;
context.write(new Text(key.toString()+":"+uLoad+":"+dLoad+":"+sLoad),NullWritable.get());
}
}
public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
Configuration conf = new Configuration();
Job job = new Job(conf,"mapreduce");
Path mypath = new Path(args[1]);
FileSystem hdfs = mypath.getFileSystem(conf);
if (hdfs.isDirectory(mypath)) {
hdfs.delete(mypath, true);
}
job.setJarByClass(MapreduceNet.class);
// job.setGroupingComparatorClass(MyComparator.class);
job.setMapperClass(MyMapper.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(NetFlow.class);
job.setReducerClass(MyReducer.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(NullWritable.class);
job.setInputFormatClass( TextInputFormat.class );
job.setOutputFormatClass( TextOutputFormat.class );
FileInputFormat.addInputPath(job, new Path(args[0]));
FileOutputFormat.setOutputPath( job,new Path(args[1]));
System.exit(job.waitForCompletion(true)?0:1);
}
}
结果输出
1363157973098:3659:3538:7197
1363157982040:7335:110349:117684
1363157983019:240:0:240
1363157984040:1938:2910:4848
1363157984041:6960:690:7650
1363157985066:2481:24681:27162
1363157986029:1938:180:2118
1363157986041:180:180:360
1363157986072:9531:2412:11943
1363157990043:11058:48243:59301
1363157991076:132:1512:1644
1363157992093:918:4938:5856
1363157993044:1527:2106:3633
1363157993055:1116:954:2070
1363157995033:3156:2936:6092
1363157995052:504:0:504
1363157995074:4116:1432:5548
1363157995093:3008:3720:6728