hadoop系列七———mapreduce编程中自定义类型的序列化接口实现方式

自定义数据类型如何实现hadoop的序列化接口

自定义bean对象实现序列化接口

（1）必须实现Writable接口
（2）反序列化时，需要反射调用空参构造函数，所以必须有空参构造
（3）重写序列化方法在这里插入代码片
（4）重写反序列化方法
（5）注意反序列化的顺序和序列化的顺序完全一致
（6）要想把结果显示在文件中，需要重写 toString()，可用 \t 分开，方便后续用
（7）如果需要将自定义的 bean 放在 key 中传输，则还需要实现Comparable 接口，因为 MapReduce 框中的 Shuffle 过程要求对 key 必须能排序

实例：测试：完成手机号的总上行流量，总下行流量，总流量的统计

测试数据：

`1	13736230513	192.196.100.1	www.atguigu.com	2481	24681	200
2	13846544121	192.196.100.2			264	0	200
3 	13956435636	192.196.100.3			132	1512	200
4 	13966251146	192.168.100.1			240	0	404
5 	18271575951	192.168.100.2	www.atguigu.com	1527	2106	200
6 	13470253144	192.168.100.3	www.atguigu.com	4116	1432	200
7 	13590439668	192.168.100.4			1116	954	200
8 	15910133277	192.168.100.5	www.hao123.com	3156	2936	200
9 	13729199489	192.168.100.6			240	0	200
10 	13630577991	192.168.100.7	www.shouhu.com	6960	690	200`

本案例的功能：演示自定义数据类型如何实现hadoop的序列化接口

• 1、该类一定要保留空参构造函数
• 2、write方法中输出字段二进制数据的顺序 要与 readFields方法读取数据的顺序一致

代码：FlowBean

public class FlowBean implements Writable {

	private int upFlow;
	private int dFlow;
	private String phone;
	private int amountFlow;

	public FlowBean(){}
	
	public FlowBean(String phone, int upFlow, int dFlow) {
		this.phone = phone;
		this.upFlow = upFlow;
		this.dFlow = dFlow;
		this.amountFlow = upFlow + dFlow;
	}

	public String getPhone() {
		return phone;
	}

	public void setPhone(String phone) {
		this.phone = phone;
	}

	public int getUpFlow() {
		return upFlow;
	}

	public void setUpFlow(int upFlow) {
		this.upFlow = upFlow;
	}

	public int getdFlow() {
		return dFlow;
	}

	public void setdFlow(int dFlow) {
		this.dFlow = dFlow;
	}

	public int getAmountFlow() {
		return amountFlow;
	}

	public void setAmountFlow(int amountFlow) {
		this.amountFlow = amountFlow;
	}

	/**
	 * 注意反序列化的顺序和序列化的顺序完全一致
	 * hadoop系统在序列化该类的对象时要调用的方法
	 */
	@Override
	public void write(DataOutput out) throws IOException {

		out.writeInt(upFlow);
		out.writeUTF(phone);
		out.writeInt(dFlow);
		out.writeInt(amountFlow);

	}

	/**
	 * hadoop系统在反序列化该类的对象时要调用的方法
	 */
	@Override
	public void readFields(DataInput in) throws IOException {
		this.upFlow = in.readInt();
		this.phone = in.readUTF();
		this.dFlow = in.readInt();
		this.amountFlow = in.readInt();
	}

	@Override
	public String toString() {
		 
		return this.phone + ","+this.upFlow +","+ this.dFlow +"," + this.amountFlow;
	}
	
}

代码：Mapper

package cn.edu360.mr.flow;

import java.io.IOException;

import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

public class FlowCountMapper extends Mapper<LongWritable, Text, Text, FlowBean>{
	
	
	@Override
	protected void map(LongWritable key, Text value, Context context)
			throws IOException, InterruptedException {
		
		String line = value.toString();
		String[] fields = line.split("\t");
		
		String phone = fields[1];
		
		int upFlow = Integer.parseInt(fields[fields.length-3]);
		int dFlow = Integer.parseInt(fields[fields.length-2]);
		
		context.write(new Text(phone), new FlowBean(phone, upFlow, dFlow));
	}
	

}

代码：Reducer

	/**
	 *  key：是某个手机号
	 *  values：是这个手机号所产生的所有访问记录中的流量数据
	 *  
	 *  <135,flowBean1><135,flowBean2><135,flowBean3><135,flowBean4>
	 */
	@Override
	protected void reduce(Text key, Iterable<FlowBean> values, Reducer<Text, FlowBean, Text, FlowBean>.Context context)
			throws IOException, InterruptedException {

		int upSum = 0;
		int dSum = 0;
		
		for(FlowBean value:values){               //可用用增强for循环处理该迭代器
			upSum += value.getUpFlow();
			dSum += value.getdFlow();
		}
		
		
		context.write(key, new FlowBean(key.toString(), upSum, dSum));
		
	}