MapReduce------找出共同好友

实现思路：

     某个人是哪些人的好友，将结果输出到文件。比如，A     B,C,D,E （A是BCDE共同的好友）第一个程序结束。

     跑第二次程序，将有A好友的人 两两组合形成 <B-C，A>格式的数据，最终跑出B-C  A,D,E.....的数据。

第一个mr程序

import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.MRBench.Reduce;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import combiner.WordCountDriver;
import combiner.WordCountMapper;
import combiner.WordCountReducer;

public class SharedFriendsStepOne {
	static class CommonFriendsStepOneMapper extends Mapper<LongWritable, Text, Text, Text> {
		// A:B,C,D,F,E,O
		// B:A,C,E
		protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
			String line = value.toString();
			String[] person_friends = line.split(":");
			// 输出<好友，人>
			for (String friend : person_friends[1].split(",")) {
				// 输出
				context.write(new Text(friend), new Text(person_friends[0]));
			}
			// 输出 b a , c a,   d a, f a, e a, o a
			// a b ,c b ,e b 
		};
	}

	static class CommonFriendsStepOneReducer extends Reducer<Text, Text, Text, Text> {
		@Override
		protected void reduce(Text friend, Iterable<Text> persons, Context context)
				throws IOException, InterruptedException {
			StringBuffer sf = new StringBuffer();
			for (Text person : persons) {
				sf.append(person).append(",");
			}
			context.write(friend, new Text(sf.toString()));
		}
	}

	public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
		Configuration conf = new Configuration();
 
		Job job = Job.getInstance(conf); 
 
		job.setJarByClass(SharedFriendsStepOne.class);
 
		job.setMapperClass(CommonFriendsStepOneMapper.class);
		job.setReducerClass(CommonFriendsStepOneReducer.class);
		
 
		job.setMapOutputKeyClass(Text.class);
		job.setMapOutputValueClass(Text.class);
		
		//指定最终输出的kv类型
		job.setOutputKeyClass(Text.class);
		job.setOutputValueClass(Text.class);
  
		//指定job 输出结果
		FileInputFormat.setInputPaths(job, new Path("g:/qq"));
		FileOutputFormat.setOutputPath(job, new Path("g:/qqout")); 
 
		boolean res = job.waitForCompletion(true);
		System.exit(res?0:1);
	}
}

第二个mr程序

import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.MRBench.Reduce;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import combiner.WordCountDriver;
import combiner.WordCountMapper;
import combiner.WordCountReducer;

public class SharedFriendsStepTwo {
	static class CommonFriendsStepTwoMapper extends Mapper<LongWritable, Text, Text, Text> {
		 /**
		  * A	I,K,C,B,G,F,H,O,D,
		  *	B	A,F,J,E,
		  * 朋友   人 人 人
		  */
		protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
			 String line = value.toString();
			 String[] friend_persons = line.split("\t");
			 
			 String friend = friend_persons[0];
			 String[] persons = friend_persons[1].split(",");
			 
			 //排序
			 Arrays.sort(persons);
			 for(int i=0;i<persons.length-1;i++) {
				 for(int j=i+1;j<persons.length;j++) {
					 context.write(new Text(persons[i]+"-"+persons[j]), new Text(friend));
				 }
			 }
			 //发出《人-人，好友》
			 
		};
	}

	static class CommonFriendsStepTwoReducer extends Reducer<Text, Text, Text, Text> {
		@Override
		protected void reduce(Text per, Iterable<Text> friends, Context context)
				throws IOException, InterruptedException {
			StringBuffer sf = new StringBuffer();
			 for(Text friend:friends) {
				 sf.append(friend).append(",");
			 }
			 
			 String[] px = sf.toString().split(",");
			 Arrays.sort(px);
			 sf.delete(0, sf.length());
			 for(String s : px) {
				 sf.append(s).append(",");
			 }
			 context.write(per, new Text(sf.toString()));
		}
	}

	public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
		Configuration conf = new Configuration();
 
		Job job = Job.getInstance(conf); 
 
		job.setJarByClass(SharedFriendsStepTwo.class);
 
		job.setMapperClass(CommonFriendsStepTwoMapper.class);
		job.setReducerClass(CommonFriendsStepTwoReducer.class);
		
 
		job.setMapOutputKeyClass(Text.class);
		job.setMapOutputValueClass(Text.class);
		
		//指定最终输出的kv类型
		job.setOutputKeyClass(Text.class);
		job.setOutputValueClass(Text.class);
  
		//指定job 输出结果
		FileInputFormat.setInputPaths(job, new Path("g:/qqout"));
		FileOutputFormat.setOutputPath(job, new Path("g:/qqout1")); 
 
		boolean res = job.waitForCompletion(true);
		System.exit(res?0:1);
	}
}

数据 

 

用户：好友列表
A:B,C,D,F,E,O,K,Z
B:A,C,E,K,Y,F
C:F,A,D,I,G,H
D:A,E,F,L,Q,P
E:B,C,D,M,L,P,V
F:A,B,C,D,E,O,M
G:A,C,D,E,F,Q
H:A,C,D,E,O,V,Q
I:A,O,C,D
J:B,O,I,J
K:A,C,D
L:D,E,F
M:E,F,G
O:A,H,I,J