*主要在 重写 readFields 时要定义values的长度,我在这卡了很久。
public static class MyMapper extends Mapper<LongWritable,Text,Text,IntArrayWritable>{
@Override
protected void map(LongWritable key, Text value, Mapper<LongWritable, Text, Text, IntArrayWritable>.Context context)
throws IOException, InterruptedException {
String[] splited = value.toString().split("\t");
String msisdn = splited[1];
Text k2 = new Text(msisdn);
String[] array=new String[4];
array[0]=splited[6];
array[1]=splited[7];
array[2]=splited[8];
array[3]=splited[9];
context.write(k2, new IntArrayWritable(array));
}
}
public static class MyReduer extends Reducer<Text,IntArrayWritable,Text,NullWritable>{
String k3=null;
@Override
protected void reduce(Text k2, Iterable<IntArrayWritable> v2s,
Reducer<Text, IntArrayWritable, Text, NullWritable>.Context context) throws IOException, InterruptedException {
long upPackNum = 0L;
long downPackNum = 0L;
long upPayLoad = 0L;
long downPayLoad = 0L;
for (IntArrayWritable kpiWritable : v2s) {
String[] writable= kpiWritable.toStrings();
upPackNum += Integer.parseInt(writable[0]);
downPackNum += Integer.parseInt(writable[1]);
upPayLoad += Integer.parseInt(writable[2]);
downPayLoad += Integer.parseInt(writable[3]);
}
k3=k2+"\t"+upPackNum+"\t"+downPackNum+"\t"+upPayLoad+"\t"+downPayLoad;
k2=new Text(k3);
context.write(k2, NullWritable.get());
}
}
public static void main(String[] args) throws Exception{
Configuration conf=new Configuration();
Job job=Job.getInstance(conf,TrafficStatistics2.class.getSimpleName());
job.setJarByClass(TrafficStatistics2.class);
job.setMapperClass(MyMapper.class);
job.setReducerClass(MyReduer.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(IntArrayWritable.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(NullWritable.class);
FileInputFormat.setInputPaths(job,new Path("hdfs://193.124.1.120:9000/HTTP_20130313143750.dat"));
FileOutputFormat.setOutputPath(job,new Path("hdfs://193.124.1.120:9000/TrafficStatistics_out1"));
job.waitForCompletion(true);
}
}
class IntArrayWritable extends ArrayWritable {
private Long[] values=null;
public IntArrayWritable() {
super(LongWritable.class);
}
public IntArrayWritable(String[] strings) {
super(LongWritable.class);
values=new Long[strings.length];
for (int i = 0; i < strings.length; i++) {
values[i] =Long.parseLong(strings[i]);
}
}
@Override
public String[] toStrings() {
String[] strings = new String[values.length];
for (int i = 0; i < values.length; i++) {
strings[i] = values[i]+"";
}
return strings;
}
@Override
public void readFields(DataInput in) throws IOException {
values = new Long[4];
for (int i = 0; i < values.length; i++) {
values[i] = in.readLong(); // store it in values
}
}
@Override
public void write(DataOutput out) throws IOException {
for (int i = 0; i < values.length; i++) {
out.writeLong(values[i]);
}
}