自定义实现Writable:
输入文件:
字段一 :手机号码 字段二:语音时长 字段三: 信息内容
统计相同手机号 所拨打的语音时长以及信息内容。
实现代码:(亲测有效)自定义实现了Writable数据类型(就是将字段二,三和为一起作为value)代码如下:
package KpiWritable;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.partition.HashPartitioner;
public class Phone {
/**
* @param args
*/
static final String INPUT_PATH = "hdfs://localhost:9000/input1/message";
static final String OUTPUT_PATH = "hdfs://localhost:9000/output1";
public static void main(String[] args) throws IOException, URISyntaxException, ClassNotFoundException, InterruptedException {
// TODO Auto-generated method stub
Configuration conf = new Configuration();
Job job = new Job(conf,Phone.class.getSimpleName());
final FileSystem filesystem = FileSystem.get(new URI(INPUT_PATH),conf);
final Path outPath = new Path(OUTPUT_PATH);
if(filesystem.exists(outPath)){
filesystem.delete(outPath, true);
}
//读取文件位置
FileInputFormat.setInputPaths(job,INPUT_PATH);
//设置map
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(PhoneWritable.class);
job.setMapperClass(Map.class);
//1.3分区
job.setPartitionerClass(HashPartitioner.class);
job.setNumReduceTasks(1);
//1.4 排序
//1.5 合并
//2.1 网络拷贝
//2.2 指定reducer类
job.setReducerClass(Reduce.class);
//2.3 输出类型
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(PhoneWritable.class);
//输出文件路径
FileOutputFormat.setOutputPath(job, outPath);
//执行
job.waitForCompletion(true);
}
//map
public static class Map extends Mapper<LongWritable, Text , Text, PhoneWritable>{
protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException{
final String[] splited = value.toString().split("\t");
final String msisdn = splited[0];
final Text k2 = new Text(msisdn);
System.out.println(splited[0]);
final PhoneWritable v2 = new PhoneWritable(splited[1],splited[2]);
context.write(k2, v2);
}
}
public static class Reduce extends Reducer<Text , PhoneWritable, Text , PhoneWritable>{
protected void reduce(Text k2,Iterable<PhoneWritable> v2,Context context) throws IOException, InterruptedException{
int time = 0;
//String content = "";
StringBuffer sb = new StringBuffer("");
for(PhoneWritable value : v2){
time += value.time;
sb.append(value.content);
sb.append("__");
}
final PhoneWritable v3 = new PhoneWritable(time+"",sb.toString());
context.write(k2, v3);
}
}
}
class PhoneWritable implements Writable{
int time;
String content;
PhoneWritable(){}
PhoneWritable(String time,String content){
this.time = Integer.parseInt(time);
this.content = content;
}
@Override
public void readFields(DataInput in) throws IOException {
// TODO Auto-generated method stub
this.time = in.readInt();
this.content = in.readLine();
}
@Override
public void write(DataOutput out) throws IOException {
// TODO Auto-generated method stub
out.writeInt(time);
out.writeBytes(content);
}
public String toString(){
return time+"\t"+content;
}
}
在终端进行查看结果,和予想的一致。