hadoop 自定义Writable

自定义实现Writable：

输入文件：

字段一 ：手机号码 字段二：语音时长  字段三： 信息内容

统计相同手机号 所拨打的语音时长以及信息内容。

实现代码：（亲测有效）自定义实现了Writable数据类型（就是将字段二，三和为一起作为value）代码如下：

package KpiWritable;

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;

import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.partition.HashPartitioner;

public class Phone {

	/**
	 * @param args
	 */
	static final String INPUT_PATH = "hdfs://localhost:9000/input1/message";
	static final String OUTPUT_PATH = "hdfs://localhost:9000/output1";
	public static void main(String[] args) throws IOException, URISyntaxException, ClassNotFoundException, InterruptedException {
		// TODO Auto-generated method stub
		Configuration conf = new Configuration();
		Job job = new Job(conf,Phone.class.getSimpleName());
		
		final FileSystem filesystem = FileSystem.get(new URI(INPUT_PATH),conf);
		final Path outPath = new Path(OUTPUT_PATH);
		if(filesystem.exists(outPath)){
			filesystem.delete(outPath, true);
		}
		
		//读取文件位置
		FileInputFormat.setInputPaths(job,INPUT_PATH);
		
		//设置map
		job.setMapOutputKeyClass(Text.class);
		job.setMapOutputValueClass(PhoneWritable.class);
		job.setMapperClass(Map.class);
		
		//1.3分区
		job.setPartitionerClass(HashPartitioner.class);
		job.setNumReduceTasks(1);
		
		//1.4 排序 
		 
		//1.5 合并
		
		//2.1 网络拷贝
		
		//2.2 指定reducer类
		job.setReducerClass(Reduce.class);
		
		//2.3 输出类型
		job.setOutputKeyClass(Text.class);
		job.setOutputValueClass(PhoneWritable.class);
		
		//输出文件路径
		FileOutputFormat.setOutputPath(job, outPath);
		
		//执行
		job.waitForCompletion(true);
		
		
	}
	
	//map
	public static class Map extends Mapper<LongWritable, Text , Text, PhoneWritable>{
		protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException{
			final String[] splited = value.toString().split("\t");
			final String msisdn = splited[0];
			final Text k2 = new Text(msisdn);
			System.out.println(splited[0]);
			final PhoneWritable v2 = new PhoneWritable(splited[1],splited[2]);
			context.write(k2, v2);
			}	
	}
	public static class Reduce extends Reducer<Text , PhoneWritable, Text , PhoneWritable>{
		
		protected void reduce(Text k2,Iterable<PhoneWritable> v2,Context context) throws IOException, InterruptedException{
			int time = 0;
			//String content = "";
			StringBuffer sb = new StringBuffer("");
			for(PhoneWritable value : v2){
				time += value.time;
				sb.append(value.content);
				sb.append("__");
			}
			final PhoneWritable v3 = new PhoneWritable(time+"",sb.toString());
			context.write(k2, v3);
		}
	}
}

class PhoneWritable implements Writable{

	int time;
	String content;
	
	PhoneWritable(){}
	
	PhoneWritable(String time,String content){
		this.time = Integer.parseInt(time);
		this.content = content;
	}
	@Override
	
	public void readFields(DataInput in) throws IOException {
		// TODO Auto-generated method stub
		this.time = in.readInt();
		this.content = in.readLine();
	}

	@Override
	public void write(DataOutput out) throws IOException {
		// TODO Auto-generated method stub
		out.writeInt(time);
		out.writeBytes(content);
	}
	public String toString(){
		return time+"\t"+content;
	}
}

在终端进行查看结果，和予想的一致。