Reduce Join介绍及案例

《一线大厂Java面试题解析+核心总结学习笔记+最新讲解视频+实战项目源码》，点击传送门，即可获取！
}

//反序列化

@Override

public void readFields(DataInput in) throws IOException {

id = in.readUTF();

pid = in.readUTF();

amount = in.readInt();

pname = in.readUTF();

flag = in.readUTF();

}

public String getId() {

return id;

}

public void setId(String id) {

this.id = id;

}

public String getPid() {

return pid;

}

public void setPid(String pid) {

this.pid = pid;

}

public int getAmount() {

return amount;

}

public void setAmount(int amount) {

this.amount = amount;

}

public String getPname() {

return pname;

}

public void setPname(String pname) {

this.pname = pname;

}

public String getFlag() {

return flag;

}

public void setFlag(String flag) {

this.flag = flag;

}

@Override

public String toString() {

return id + “\t” + amount + “\t” + pname;

}

}

2.编写TableMapper类

package com.atguigu.mr.table;

import java.io.IOException;

import org.apache.hadoop.io.LongWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.Mapper;

import org.apache.hadoop.mapreduce.lib.input.FileSplit;

public class TableMapper extends Mapper<LongWritable, Text, Text, TableBean> {

String name;

@Override

protected void setup(Mapper<LongWritable, Text, Text, TableBean>.Context context)

throws IOException, InterruptedException {

//获取文件名称

FileSplit inputSplit = (FileSplit)context.getInputSplit();

name = inputSplit.getPath().getName();

}

TableBean tableBean = new TableBean();

Text k = new Text();

@Override

protected void map(LongWritable key, Text value, Mapper<LongWritable, Text, Text, TableBean>.Context context)

throws IOException, InterruptedException {

//获取一行

String line = value.toString();

if (name.startsWith(“order”)) {//订单表

String[] fields = line.split(“\t”);

//封装key和value

tableBean.setId(fields[0]);

tableBean.setPid(fields[1]);

tableBean.setAmount(Integer.parseInt(fields[2]));

tableBean.setPname(“”);

tableBean.setFlag(“order”);

k.set(fields[1]);

}else {//产品表

String[] fields = line.split(“\t”);

//封装key和value

tableBean.setId(“”);

tableBean.setPid(fields[0]);

tableBean.setAmount(0);

tableBean.setPname(fields[1]);

tableBean.setFlag(“pd”);

k.set(fields[0]);

}

//写出

context.write(k, tableBean);

}

}

3.编写TableReducer类

package com.atguigu.mr.table;

import java.io.IOException;

import java.lang.reflect.InvocationTargetException;

import java.util.ArrayList;

import org.apache.commons.beanutils.BeanUtils;

import org.apache.hadoop.hdfs.protocol.RollingUpgradeInfo.Bean;

import org.apache.hadoop.io.NullWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.Reducer;

public class TableReducer extends Reducer<Text, TableBean, TableBean, NullWritable> {

@Override

protected void reduce(Text key, Iterable values,

Context context) throws IOException, InterruptedException {

//存放所有订单集合

ArrayList orderBeans = new ArrayList<>();

//存放产品信息

TableBean pdBean = new TableBean();

for (TableBean tableBean : values) {

if (“order”.equals(tableBean.getFlag())) { //订单表

TableBean tmpBean = new TableBean();

try {

BeanUtils.copyProperties(tmpBean, tableBean);

orderBeans.add(tmpBean);

} catch (IllegalAccessException e) {

e.printStackTrace();

} catch (InvocationTargetException e) {

e.printStackTrace();

}

}else {

try {

BeanUtils.copyProperties(pdBean, tableBean);

} catch (IllegalAccessException e) {

e.printStackTrace();

} catch (InvocationTargetException e) {

e.printStackTrace();

}

}

}

for (TableBean tableBean : orderBeans) {

tableBean.setPname(pdBean.getPname());

context.write(tableBean, NullWritable.get());

}

}

}

4. 编写TableDriver类

package com.atguigu.mr.table;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.fs.Path;

import org.apache.hadoop.io.NullWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.Job;

import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;

import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

public class TableDriver {

public static void main(String[] args) throws IllegalArgumentException, IOException, ClassNotFoundException, InterruptedException {

// 0 根据自己电脑路径重新配置

args = new String[] { “S:\centos学习笔记\input\tableinput”, “S:\centos学习笔记\output\tableoutput” };

// 1 获取配置信息，或者job对象实例

Configuration configuration = new Configuration();

Job job = Job.getInstance(configuration);

// 2 指定本程序的jar包所在的本地路径

job.setJarByClass(TableDriver.class);

// 3 指定本业务job要使用的Mapper/Reducer业务类

job.setMapperClass(TableMapper.class);

job.setReducerClass(TableReducer.class);

// 4 指定Mapper输出数据的kv类型

job.setMapOutputKeyClass(Text.class);

job.setMapOutputValueClass(TableBean.class);

// 5 指定最终输出的数据的kv类型

job.setOutputKeyClass(TableBean.class);

job.setOutputValueClass(NullWritable.class);

// 6 指定job的输入原始文件所在目录

FileInputFormat.setInputPaths(job, new Path(args[0]));

FileOutputFormat.setOutputPath(job, new Path(args[1]));

// 7 将job中配置的相关参数，以及job所用的java类所在的jar包， 提交给yarn去运行

boolean result = job.waitForCompletion(true);

System.exit(result ? 0 : 1);

}

本次面试答案，以及收集到的大厂必问面试题分享：

《一线大厂Java面试题解析+核心总结学习笔记+最新讲解视频+实战项目源码》，点击传送门，即可获取！
n.class);

job.setOutputValueClass(NullWritable.class);

// 6 指定job的输入原始文件所在目录

FileInputFormat.setInputPaths(job, new Path(args[0]));

FileOutputFormat.setOutputPath(job, new Path(args[1]));

// 7 将job中配置的相关参数，以及job所用的java类所在的jar包， 提交给yarn去运行

boolean result = job.waitForCompletion(true);

System.exit(result ? 0 : 1);

}

本次面试答案，以及收集到的大厂必问面试题分享：

[外链图片转存中…(img-VrOVEarC-1714650949187)]

《一线大厂Java面试题解析+核心总结学习笔记+最新讲解视频+实战项目源码》，点击传送门，即可获取！