hadoop 手机流统计

package com.Example;


import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;


import org.apache.hadoop.io.Writable;


public class DataWriter implements Writable {


// update
private int upPackNum;
private int upPayLoad;


// down
private int downPackNum;
private int downPayLoad;


public int getUpPackNum() {
return upPackNum;
}


public void setUpPackNum(int upPackNum) {
this.upPackNum = upPackNum;
}


public int getUpPayLoad() {
return upPayLoad;
}


public void setUpPayLoad(int upPayLoad) {
this.upPayLoad = upPayLoad;
}


public int getDownPackNum() {
return downPackNum;
}


public void setDownPackNum(int downPackNum) {
this.downPackNum = downPackNum;
}


public int getDownPayLoad() {
return downPayLoad;
}


public void setDownPayLoad(int downPayLoad) {
this.downPayLoad = downPayLoad;
}


public DataWriter() {
}


public void set(int upPackNum, int upPayLoad, int downPackNum,
int downPayLoad) {
this.upPackNum = upPackNum;
this.upPayLoad = upPayLoad;
this.downPackNum = downPackNum;
this.downPayLoad = downPayLoad;
}


@Override
public void readFields(DataInput out) throws IOException {
this.upPackNum = out.readInt();
this.upPayLoad = out.readInt();
this.downPackNum = out.readInt();
this.downPayLoad = out.readInt();
}


@Override
public void write(DataOutput out) throws IOException {
out.writeInt(upPackNum);
out.writeInt(upPayLoad);
out.writeInt(downPackNum);
out.writeInt(downPayLoad);
}


@Override
public String toString() {
return "Phone [upPackNum=" + upPackNum + ", upPayLoad=" + upPayLoad
+ ", downPackNum=" + downPackNum + ", downPayLoad="
+ downPayLoad + "]";
}


}



package com.Example;


import java.io.IOException;


import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;


public class PhoneValue {


public static class PhoneValueMap extends
Mapper<LongWritable, Text, Text, DataWriter> {
private DataWriter phone = new DataWriter();
private Text text = new Text();


@Override
protected void map(LongWritable key, Text value, Context context)
throws IOException, InterruptedException {
String valus = value.toString();
String[] str = valus.split("/t");
// 假如第二行的数据是手机号码
String phoneNum = str[1];
int upPackNum = Integer.valueOf(str[2]);
int upPayLoad = Integer.valueOf(str[3]);
int downPackNum = Integer.valueOf(str[4]);
int downPayLoad = Integer.valueOf(str[5]);
// set map ouput key / value
text.set(phoneNum); // 设置需要按照哪个键进行统计
phone.set(upPackNum, upPayLoad, downPackNum, downPayLoad);
// set map ouput
context.write(text, phone);
}
}


public static class PhoneValueReduce extends
Reducer<Text, DataWriter, Text, DataWriter> {
private DataWriter dataWriter = new DataWriter();


@Override
protected void reduce(Text arg0, Iterable<DataWriter> arg1, Context arg2) {
int upPackNum = 0;
int upPayLoad = 0;
int downPackNum = 0;
int downPayLoad = 0;
for (DataWriter dataWriter : arg1) {
upPackNum += dataWriter.getUpPackNum();
upPayLoad += dataWriter.getUpPayLoad();
downPackNum += dataWriter.getDownPackNum();
downPayLoad += dataWriter.getDownPayLoad();
}
// set datewritable
dataWriter.set(upPackNum, upPayLoad, downPackNum, downPayLoad);


try {
// set reduce ouput
arg2.write(arg0, dataWriter);
} catch (IOException e) {
e.printStackTrace();
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}


/**
* 
* @param args
* @return
*/
public int run(String args) {
// get config
Configuration configuration = new Configuration();
try {
// create job
Job job = Job.getInstance(configuration);
// set job
job.setJarByClass(PhoneValue.class);
// input
FileInputFormat.addInputPath(job, new Path(""));
// map
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(DataWriter.class);
// reduce
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(IntWritable.class);
// ouput
FileOutputFormat.setOutputPath(job, new Path(""));
// submit job
boolean start = job.waitForCompletion(true);
System.exit(start ? 0 : 1);
} catch (IOException e) {
e.printStackTrace();
} catch (ClassNotFoundException e) {
e.printStackTrace();
} catch (InterruptedException e) {
e.printStackTrace();
}
return 0;
}
}