mapReduce共同好友统计(twelve day)

最新推荐文章于 2024-01-11 14:44:59 发布

高辉

最新推荐文章于 2024-01-11 14:44:59 发布

阅读量327

点赞数 1

分类专栏：向大数据进军~每天记文章标签： mapReduce共同好友统计 mapReduce好友统计共同好友统计

本文链接：https://blog.csdn.net/ZJX103RLF/article/details/89075001

版权

向大数据进军~每天记专栏收录该内容

58 篇文章 0 订阅

订阅专栏

需求：统计那些人之间有共同好友，及好友是谁

public class CommonFriendsOne {
	
	public static class CommonFriendsOneMapper extends Mapper<LongWritable, Text, Text, Text>{
		Text k = new Text();
		Text v = new Text();
		
		//A:B,C,D,F,E,O
		// 输出：　B->A  C->A  D->A ...
		
		
		@Override
		protected void map(LongWritable key, Text value, Mapper<LongWritable, Text, Text, Text>.Context context)
				throws IOException, InterruptedException {
			String[] userAndFriends = value.toString().split(":");
			
			String user = userAndFriends[0];
			String[] friends = userAndFriends[1].split(",");
			v.set(user);
			for (String f : friends) {
				k.set(f);
				context.write(k, v);
			}
			
			
		}
		
		
		
	}
	
	
	
	public static class CommonFriendsOneReducer extends Reducer<Text, Text, Text, Text>{
		
		// 一组数据：  B -->  A  E  F  J .....
		// 一组数据：  C -->  B  F  E  J .....
		@Override
		protected void reduce(Text friend, Iterable<Text> users, Reducer<Text, Text, Text, Text>.Context context)
				throws IOException, InterruptedException {
			
			ArrayList<String> userList = new ArrayList<>();
			
			for(Text user: users){
				userList.add(user.toString());
			}
			
			Collections.sort(userList);
			
			for(int i=0;i<userList.size()-1;i++){
				for(int j=i+1;j<userList.size();j++){
					context.write(new Text(userList.get(i)+"-"+userList.get(j)), friend);
				}
			}
			
		}
		
	}
	
	
public static void main(String[] args) throws Exception {

		
		Configuration conf = new Configuration();  
		
		Job job = Job.getInstance(conf);

		job.setJarByClass(CommonFriendsOne.class);

		job.setMapperClass(CommonFriendsOneMapper.class);
		job.setReducerClass(CommonFriendsOneReducer.class);
		
		job.setMapOutputKeyClass(Text.class);
		job.setMapOutputValueClass(Text.class);
		
		job.setOutputKeyClass(Text.class);
		job.setOutputValueClass(Text.class);

		FileInputFormat.setInputPaths(job, new Path("F:\\mrdata\\friends\\input"));
		FileOutputFormat.setOutputPath(job, new Path("F:\\mrdata\\friends\\out"));

		job.waitForCompletion(true);
	}
}