mapreduce 实现内连接，左连接，右连接，全连接，反连接

测试数据
more user.txt (用户id,用户名)
1	用户1
2	用户2
3	用户3

more post.txt (用户id,帖子id,标题)
1	1	贴子1
1	2	贴子2
2	3	帖子3
4	4	贴子4
5	5	贴子5
5	6	贴子6
5	7	贴子7

 

 

查询结果

内连接
1	用户1	1	1	贴子1
1	用户1	1	2	贴子2
2	用户2	2	3	帖子3

左外连接
1	用户1	1	1	贴子1
1	用户1	1	2	贴子2
2	用户2	2	3	帖子3
3	用户3	 	 	 

右外连接
1	用户1	1	1	贴子1
1	用户1	1	2	贴子2
2	用户2	2	3	帖子3
 	 	4	4	贴子4
 	 	5	5	贴子5
 	 	5	6	贴子6
 	 	5	7	贴子7


全外连接
1	用户1	1	1	贴子1
1	用户1	1	2	贴子2
2	用户2	2	3	帖子3
3	用户3	 	 
 	 	4	4	贴子4
 	 	5	5	贴子5
 	 	5	6	贴子6
 	 	5	7	贴子7

反连接
3	用户3	 	 	 
 	 	4	4	贴子4
 	 	5	5	贴子5
 	 	5	6	贴子6
 	 	5	7	贴子7

 

代码如下：

package mapreduce.pattern.join;

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.net.URI;
import java.util.ArrayList;
import java.util.List;

import multiinput.post.PostJob;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.MultipleInputs;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;

/**
 * mapreduce 实现内连接，左连接，右连接，全连接，反连接
 * user.txt 用户表
 * post.txt 帖子表
 * 关联字段 userId
 * @author wxj
 *
 */
public class UserAndPostJoinJob
{

	static class UserAndPostWritable implements Writable
	{
		
		/**
		 * 类型 U表示用户,P表示帖子
		 */
		private String type;
		private String data;
		
		public UserAndPostWritable()
		{
			
		}
		
		public UserAndPostWritable(String type, String data)
		{
			super();
			this.type = type;
			this.data = data;
		}

		public String getType()
		{
			return type;
		}

		public void setType(String type)
		{
			this.type = type;
		}

		public String getData()
		{
			return data;
		}

		public void setData(String data)
		{
			this.data = data;
		}

		@Override
		public void readFields(DataInput input) throws IOException
		{
			type = input.readUTF();
			data = input.readUTF();
		}

		@Override
		public void write(DataOutput output) throws IOException
		{
			output.writeUTF(type);
			output.writeUTF(data);
		}
		
	}
	
	static class UserMapper extends Mapper<LongWritable, Text, Text, UserAndPostWritable>
	{
		@Override
		protected void map(LongWritable key, Text value,Context context) throws IOException, InterruptedException
		{
			String[] arr = value.toString().split("\t");
			Text userId = new Text(arr[0]);
			context.write(userId, new UserAndPostWritable("U",value.toString()));
		}

	}
	
	static class PostMapper extends Mapper<LongWritable, Text, Text, UserAndPostWritable>
	{
		@Override
		protected void map(LongWritable key, Text value,Context context) throws IOException, InterruptedException
		{
			String[] arr = value.toString().split("\t");
			Text userId = new Text(arr[0]);
			context.write(userId, new UserAndPostWritable("P",value.toString()));
			System.out.println(userId);
		}

	}
	
	static class PostReducer extends Reducer<Text, UserAndPostWritable, Text, Text>
	 {

		private List<Text> users = new ArrayList<Text>();
		private List<Text> posts = new ArrayList<Text>();
		
		private String joinType;

		
	 	@Override
		protected void setup(Context context) throws IOException,InterruptedException
		{
			super.setup(context);
			joinType = context.getConfiguration().get("joinType");
			//System.out.println("joinType: " + joinType);
		}

		protected void reduce(Text key, Iterable<UserAndPostWritable> iterable,Context context)throws IOException, InterruptedException
	 	{
			users.clear();
			posts.clear();
	 		for(UserAndPostWritable data : iterable)
	 		{
	 			//System.out.println(data.getType() + "," + data.getData());
	 			if(data.getType().equals("U"))
	 			{
	 				users.add(new Text(data.getData()));
	 			}
	 			else {
					posts.add(new Text(data.getData()));
				}
	 		}
	 		
	 		if(joinType.equals("innerJoin"))//内连接
	 		{
	 			if(users.size() > 0 && posts.size() > 0)
	 			{
	 				for(Text user : users)
		 			{
		 				for(Text post : posts)
		 				{
		 					context.write(new Text(user),new Text(post));
		 				}
		 			}
	 			}
	 		}
	 		else if(joinType.equals("leftOuter"))//左外连接
	 		{
	 			for(Text user : users)
	 			{
	 				if(posts.size() > 0)
	 				{
	 					for(Text post : posts)
		 				{
		 					context.write(new Text(user),new Text(post));
		 				}
	 				}
	 				else {
	 					context.write(new Text(user),createEmptyPost());
					}
	 			}
	 		}
	 		else if(joinType.equals("rightOuter"))//右外连接
	 		{
	 			for(Text post : posts)
 				{
	 				if(users.size() > 0)
	 				{
	 					for(Text user : users)
			 			{
		 					context.write(new Text(user),new Text(post));
			 			}
	 				}
	 				else {
						context.write(createEmptyUser(), post);
					}
 				}
	 		}
	 		else if(joinType.equals("allOuter"))//全外连接
	 		{
	 			if(users.size() > 0)
	 			{
	 				for(Text user : users)
	 				{
		 				if(posts.size() > 0)
		 				{
		 					for(Text post : posts)
			 				{
		 						context.write(new Text(user),new Text(post));
			 				}
		 				}
		 				else{
		 					context.write(new Text(user),createEmptyUser());
		 				}
	 				}
	 			}else {
	 				for(Text post : posts)
	 				{
		 				if(users.size() > 0)
		 				{
		 					for(Text user : users)
			 				{
		 						context.write(new Text(user),new Text(post));
			 				}
		 				}
		 				else{
		 					context.write(createEmptyUser(), post);
		 				}
	 				}
				}
	 		}
	 		else if(joinType.equals("anti"))//反连接
	 		{
	 			if(users.size() == 0 ^ posts.size() == 0)
	 			{
	 				for(Text user : users)
	 				{
 						context.write(new Text(user),createEmptyPost());
	 				}
	 				for(Text post : posts)
	 				{
 						context.write(createEmptyUser(),new Text(post));
	 				}
	 			}
	 		}
	 	}
		
		private Text createEmptyUser()
		{
			return new Text(" \t ");
		}
		
		private Text createEmptyPost()
		{
			return new Text(" \t \t ");
		}
	 	
	 }
	 
	 public static void main(String[] args)
	 {
			Configuration configuration = new Configuration();
			try
			{
				FileSystem fs = FileSystem.get(new URI("hdfs://master:9000"),configuration);
				Job job = new Job(configuration);
				job.setJarByClass(PostJob.class);
				//设置连接类型
				//innerJoin,leftOuter,rightOuter,allOuter,anti
				job.getConfiguration().set("joinType", "anti");
				//设置输出到part-r-00000时的分隔符
				job.getConfiguration().set("mapred.textoutputformat.separator", "\t");
				
				MultipleInputs.addInputPath(job, new Path("hdfs://master:9000/input/userandpost/user.txt"),TextInputFormat.class,UserMapper.class);
				MultipleInputs.addInputPath(job, new Path("hdfs://master:9000/input/userandpost/post.txt"), TextInputFormat.class, PostMapper.class);
				
				job.setMapOutputKeyClass(Text.class);
				job.setMapOutputValueClass(UserAndPostWritable.class);
				job.setReducerClass(PostReducer.class);
			    job.setOutputKeyClass(Text.class);
			    job.setOutputValueClass(Text.class);
				
				Path outPath = new Path("hdfs://master:9000/output/userandpost");
				if(fs.exists(outPath))
				{
					fs.delete(outPath,true);
				}
				TextOutputFormat.setOutputPath(job, outPath);
				job.setOutputFormatClass(TextOutputFormat.class);
				
				job.waitForCompletion(true);
				
			} catch (Exception e)
			{
				e.printStackTrace();
			}
	 }
	
}