两个MapReduce实现计算出用户间的共同好友

需求和思路

1.下面是用户的好友关系列表，每一行代表一个用户和他的好友列表。

需要求出哪些人两两之间有共同好友，及他俩的共同好友都有谁。
例如从前2天记录中可以看出，C、E是A、B的共同好友，最终的形式如下：

（2）实现思路
之前的示例中都是一个MapReduce计算出来的，这里我们使用2个MapReduce来实现。
1）第1个MapReduce
map
找出每个用户都是谁的好友，例如：
读一行A:B,C,D,F,E,O（A的好友有这些，反过来拆开，这些人中的每一个都是A的好友）
输出<B,A> <C,A> <D,A> <F,A> <E,A> <O,A>
再读一行B:A,C,E,K
输出<A,B> <C,B> <E,B> <K,B>
……
reduce
key相同的会分到一组，例如：
<C,A><C,B><C,E><C,F><C,G>…
Key:C
value: [ A, B, E, F, G ]
意义是：C是这些用户的好友。
遍历value就可以得到：
A B 有共同好友C
A E 有共同好友C
…
B E有共同好友 C
B F有共同好友 C
输出：
<A-B,C>
<A-E,C>
<A-F,C>
<A-G,C>
<B-E,C>
<B-F,C>
…
2）第2个MapReduce
对上一步的输出结果进行计算。
map
读出上一步的结果数据，组织成key value直接输出
例如：
读入一行<A-B,C>
直接输出<A-B,C>
reduce
读入数据，key相同的在一组
<A-B,C><A-B,F><A-B,G>…
输出：
A-B C,F,G,…
这样就得出了两个用户间的共同好友列表

第一个MapReduce类：StepFirst

package duo_MapReduce;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
public class StepFirst {
	static class FirstMapper extends Mapper<LongWritable,Text,Text,Text>{
		@Override
		protected void map(LongWritable key, Text value,Mapper<LongWritable, Text, Text, Text>.Context context)throws IOException, InterruptedException {
			String line = value.toString();
			String[] arr = line.split(":");
			String user = arr[0];
			String friends = arr[1];
			for (String friend : friends.split(",")) {
				context.write(new Text(friend), new Text(user));
			}
		}
	}
	static class FirstReducer extends Reducer<Text, Text, Text, Text>{
		@Override
		protected void reduce(Text friend, Iterable<Text> users,Context context)throws IOException, InterruptedException {
			StringBuffer buf = new StringBuffer();
			for (Text user : users) {
				buf.append(user).append(",");
			}
			context.write(new Text(friend), new Text(buf.toString()));
		}
	}
	public static void main(String[] args) throws Exception {
		Configuration conf = new Configuration();
		Job job = Job.getInstance(conf);
		job.setJarByClass(StepFirst.class);
		job.setOutputKeyClass(Text.class);
		job.setOutputValueClass(Text.class);
		job.setMapperClass(FirstMapper.class);
		job.setReducerClass(FirstReducer.class);
		FileInputFormat.setInputPaths(job, new Path(args[0]));
		FileOutputFormat.setOutputPath(job, new Path(args[1]));
		job.waitForCompletion(true);
	}
}

第二个MapReduce类：StepSecond

package duo_MapReduce;
import java.io.IOException;
import java.util.Arrays;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
public class StepSecond {
	static class SecondMapper extends Mapper<LongWritable,Text,Text,Text>{
		@Override
		protected void map(LongWritable key, Text value,Mapper<LongWritable, Text, Text, Text>.Context context)throws IOException, InterruptedException {
			String line = value.toString();
			String[] friend_users = line.split("\t");		
			String friend = friend_users[0];
			String[] users = friend_users[1].split(",");
			Arrays.sort(users);
			for (int i = 0; i < users.length-1; i++) {
				for (int j = i+1; j < users.length; j++) {
					context.write(new Text(users[i]+"-"+users[j]), new Text(friend));
				}
			}
		}
	}
	static class SecondReducer extends Reducer<Text, Text, Text, Text>{
		@Override
		protected void reduce(Text user_user, Iterable<Text> friends,Context context)throws IOException, InterruptedException {
			StringBuffer buf = new StringBuffer();
			for (Text friend : friends) {
				buf.append(friend).append(" ");
			}
			context.write(user_user, new Text(buf.toString()));
		}
	}
	public static void main(String[] args) throws Exception {
		Configuration conf = new Configuration();
		Job job = Job.getInstance(conf);
		job.setJarByClass(StepSecond.class);
		job.setOutputKeyClass(Text.class);
		job.setOutputValueClass(Text.class);
		job.setMapperClass(SecondMapper.class);
		job.setReducerClass(SecondReducer.class);
		FileInputFormat.setInputPaths(job, new Path(args[0]));
		FileOutputFormat.setOutputPath(job, new Path(args[1]));
		job.waitForCompletion(true);
	}	
}