二、Hadoop 序列化
1、序列化概述
2、自定义 bean 对象实现序列化接口(Writable)
-
在企业开发中往往常用的基本序列化类型不能满足所有需求,比如在Hadoop框架内部传递一个bean对象,那么该对象就需要实现序列化接口。
-
具体实现bean对象序列化步骤如下7步。
(1)必须实现 Writable 接口
(2)反序列化时,需要反射调用空参构造函数,所以必须有空参构造
public FlowBean() {
super();
}
(3)重写序列化方法
@Override
public void write(DataOutput out) throws IOException {
out.writeLong(upFlow);
out.writeLong(downFlow);
out.writeLong(sumFlow);
}
(4)重写反序列化方法
@Override
public void readFields(DataInput in) throws IOException {
upFlow = in.readLong();
downFlow = in.readLong();
sumFlow = in.readLong();
}
(5)注意反序列化的顺序和序列化的顺序完全一致
(6)要想把结果显示在文件中,需要重写 toString(),可用”\t”分开,方便后续用。
(7)如果需要将自定义的 bean 放在 key 中传输,则还需要实现 Comparable 接口,因为MapReduce 框中的 Shuffle 过程要求对 key 必须能排序。详见后面排序案例。(hadoop 中只能对key 进行排序)
@Override
public int compareTo(FlowBean o) {
// 倒序排列,从大到小
return this.sumFlow > o.getSumFlow() ? -1 : 1;
}
3 、序列化案例实操
输入数据:
编写 MapReduce 程序
3.1 编写流量统计的Bean对象
// 1 实现writable接口
public class FlowBean implements Writable {
//设置属性变量
private long upFlow; //上行流量
private long downFlow; //下行流量
private long sumFlow; //总流量
//2 反序列化时,需要反射调用空参构造函数,所以必须有
public FlowBean() {
}
//提供三个属性的set和get方法
public long getUpFlow() {
return upFlow;
}
public void setUpFlow(long upFlow) {
this.upFlow = upFlow;
}
public long getDownFlow() {
return downFlow;
}
public void setDownFlow(long downFlow) {
this.downFlow = downFlow;
}
public long getSumFlow() {
return sumFlow;
}
public void setSumFlow(long sumFlow) {
this.sumFlow = sumFlow;
}
//重载方法
public void setSumFlow() {
this.sumFlow = this.upFlow + this.downFlow;
}
//实现 序列化 和 反序列化 方法 注意:序列化和反序列化的顺序必须一致
//3 写序列化方法
@Override
public void write(DataOutput dataOutput) throws IOException {
dataOutput.writeLong(upFlow);
dataOutput.writeLong(downFlow);
dataOutput.writeLong(sumFlow);
}
//4 反序列化方法
@Override
public void readFields(DataInput dataInput) throws IOException {
this.upFlow = dataInput.readLong();
this.downFlow = dataInput.readLong();
this.sumFlow = dataInput.readLong();
}
//5 重写 toString 方法,方便后续打印到文本
@Override
public String toString() {
return upFlow + "\t" + downFlow + "\t" + sumFlow;
}
}
3.2 编写Mapper类
public class FlowMapper extends Mapper<LongWritable, Text, Text, FlowBean> {
//创建输出的键值对对象
private Text outKey = new Text();
private FlowBean outValue = new FlowBean();
@Override
protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
//1、获取一行数据,转成字符串
String line = value.toString();
//2、切割一行数据,并按照制表符"\t"切割
String[] split = line.split("\t");
//3、抓取我们需要的数据
String phoneNumber = split[1];
String upFlow = split[split.length - 3];
String downFlow = split[split.length - 2];
//4、封装输出的Key,Value
outKey.set(phoneNumber);
outValue.setUpFlow(Long.parseLong(upFlow));
outValue.setDownFlow(Long.parseLong(downFlow));
//5、写出Key Value
context.write(outKey, outValue);
}
}
3.3 编写Reducer类
public class FlowReducer extends Reducer<Text, FlowBean, Text, FlowBean> {
//创建输出对象
private FlowBean outValue = new FlowBean();
//重写 reduce 方法
@Override
protected void reduce(Text key, Iterable<FlowBean> values, Context context) throws IOException, InterruptedException {
//设置计数器(防止同个key对应多个value时的情况)
long totalUpFlow = 0;
long totalDownFlow = 0;
//迭代同一个手机号的所有的flowbean,求出totalUpFlow totalDownFlow
for (FlowBean flowBean : values) {
totalUpFlow += flowBean.getUpFlow();
totalDownFlow += flowBean.getDownFlow();
}
//封装outValue
outValue.setUpFlow(totalUpFlow);
outValue.setDownFlow(totalDownFlow);
outValue.setSumFlow();//调用前面重载的方法
//写出Key Value
context.write(key, outValue);
}
}
3.4 编写Driver驱动类
public class FlowDriver {
public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
//1、获取job对象
Configuration conf = new Configuration();
Job job = Job.getInstance(conf);
//2、关联本地Driver类
job.setJarByClass(FlowDriver.class);
//3、关联 mapper 和 reducer
job.setMapperClass(FlowMapper.class);
job.setReducerClass(FlowReducer.class);
//4、设置 map 端的输出Key Value 的类型
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(FlowBean.class);
//5、设置 MapReduce 程序最终的输出 Key Value 类型
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(FlowBean.class);
//6、设置程序输入输出路径
FileInputFormat.setInputPaths(job, new Path("F:\\input\\inputflow"));
FileOutputFormat.setOutputPath(job, new Path("F:\\input\\MapReduce\\output4"));
//7、提交job
boolean b = job.waitForCompletion(true);
System.exit(b ? 0 : 1);
}
}