彷徨 | MapReduce实例七 | Join拼接

以用户订单数据为例 : 

JoinBean

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;

import org.apache.hadoop.io.Writable;

public class JoinBean implements Writable{
	private String oid;
	private String uid;
	private String name;
	private int age;
	private String gender;
	private String friend;
	
	public JoinBean() {
		// TODO Auto-generated constructor stub
	}
	
	public void set(String oid, String uid, String name, int age, String gender, String friend) {
		this.oid = oid;
		this.uid = uid;
		this.name = name;
		this.age = age;
		this.gender = gender;
		this.friend = friend;
	}

	public JoinBean(String oid, String uid, String name, int age, String gender, String friend) {
		super();
		this.oid = oid;
		this.uid = uid;
		this.name = name;
		this.age = age;
		this.gender = gender;
		this.friend = friend;
	}

	@Override
	public String toString() {
		return "[oid=" + oid + ", uid=" + uid + ", name=" + name + ", age=" + age + ", gender=" + gender
				+ ", friend=" + friend + "]";
	}

	public String getOid() {
		return oid;
	}

	public void setOid(String oid) {
		this.oid = oid;
	}

	public String getUid() {
		return uid;
	}

	public void setUid(String uid) {
		this.uid = uid;
	}

	public String getName() {
		return name;
	}

	public void setName(String name) {
		this.name = name;
	}

	public int getAge() {
		return age;
	}

	public void setAge(int age) {
		this.age = age;
	}

	public String getGender() {
		return gender;
	}

	public void setGender(String gender) {
		this.gender = gender;
	}

	public String getFriend() {
		return friend;
	}

	public void setFriend(String friend) {
		this.friend = friend;
	}

	@Override
	public void write(DataOutput out) throws IOException {
		out.writeUTF(oid);
		out.writeUTF(uid);
		out.writeUTF(name);
		out.writeInt(age);
		out.writeUTF(gender);
		out.writeUTF(friend);
	}
	
	@Override
	public void readFields(DataInput in) throws IOException {
		this.oid = in.readUTF();
		this.uid = in.readUTF();
		this.name = in.readUTF();
		this.age = in.readInt();
		this.gender = in.readUTF();
		this.friend = in.readUTF();
	}
	
}

Map,Reduce,main

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;

import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.FileSplit;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

/**
 * 用mr实现sql中的join模型
 * @author Administrator
 *
 */

public class join {
	
	public static class JoinMapper extends Mapper<LongWritable, Text, Text, JoinBean>{
		@Override
		protected void map(LongWritable key, Text value, Mapper<LongWritable, Text, Text, JoinBean>.Context context)
				throws IOException, InterruptedException {
			//获取任务切片信息
			FileSplit inputSplit = (FileSplit)context.getInputSplit();
			//从任务切片信息中获取文件路径
			String fileName = inputSplit.getPath().getName();
			
			String[] split = value.toString().split(",");
			JoinBean joinBean = new JoinBean();
			
			if (fileName.equals("user.txt")) {
				joinBean.set("NULL", split[0], split[1], Integer.parseInt(split[2]), split[3], split[4]);
			} else {
				joinBean.set(split[0], split[1], "NULL", 0, "NULL", "NULL");
			}
			context.write(new Text(joinBean.getUid()), joinBean);
		}
	}
	
	
	
	
	
	public static class JoinReduce extends Reducer<Text, JoinBean, JoinBean, NullWritable>{
		@SuppressWarnings("unused")
		@Override
		protected void reduce(Text key, Iterable<JoinBean> values,
				Reducer<Text, JoinBean, JoinBean, NullWritable>.Context context) throws IOException, InterruptedException {
			ArrayList<JoinBean> orderList = new ArrayList<>();
			JoinBean userBean = new JoinBean();
			
			for (JoinBean v : values) {
				if (v.getOid().equals("NULL")) {
					//如果这条数据是用户数据
					userBean.set(v.getOid(), v.getUid(), v.getName(), v.getAge(), v.getGender(),v.getFriend());
				} else {
					//如果这条数据是订单数据
					JoinBean newBean = new JoinBean();
					newBean.set(v.getOid(), v.getUid(), v.getName(), v.getAge(), v.getGender(), v.getFriend());
					orderList.add(newBean);
				}
			}
			// 拼接数据：
			for (JoinBean joinBean : orderList) {
				
				joinBean.setName(userBean.getName());
				joinBean.setAge(userBean.getAge());
				joinBean.setGender(userBean.getGender());
				joinBean.setFriend(userBean.getFriend());
				
				context.write(joinBean, NullWritable.get());
			}
		}
		
	}
	
	public static void main(String[] args) throws Exception {
		Configuration conf = new Configuration();
		Job job= Job.getInstance(conf);
		
		job.setJarByClass(join.class);
		
		job.setMapperClass(JoinMapper.class);
		job.setReducerClass(JoinReduce.class);
		
		job.setMapOutputKeyClass(Text.class);
		job.setMapOutputValueClass(JoinBean.class);
		
		job.setOutputKeyClass(JoinBean.class);
		job.setOutputValueClass(NullWritable.class);
		
		FileInputFormat.setInputPaths(job, new Path("E:/data/in/join/input"));
		FileOutputFormat.setOutputPath(job, new Path("E:/data/out/join/out"));
		
		File file = new File("E:/data/out/join/out");
		if (file.exists()) {
			FileUtils.deleteDirectory(file);
		}
		
		boolean completion = job.waitForCompletion(true);
		System.out.println(completion?"优秀":"失败");
	}
	
}

运行结果 :