MapReduce分析共同好友

两次使用mapreduce
输入文件:
A:B,C,D,F,E,O
B:A,C,E,K
C:F,A,D,I
D:A,E,F,L
E:B,C,D,M,L
F:A,B,C,D,E,O,M
G:A,C,D,E,F
H:A,C,D,E,O
I:A,O
J:B,O
K:A,C,D
L:D,E,F
M:E,F,G
O:A,H,I,J

最终需求结果如：A–B: C E
思路分析：从结论出发,最后一步的reduce 的key应为
A–B: c A–B: e
由此，改形式应该为第一次mapreduce的输出结果

所以进行两次mapreduce即可

FriendsPre.java:

public class FriendsPre {


    static public class FriendsMapper extends Mapper<LongWritable, Text, Text, Text> {
        Text friendText = new Text();
        Text usrText = new Text();
        @Override
        protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
            //获取每一行的好友内容
            String line = value.toString();
            String[] spil1 = line.split(":");
            //当前用户
            String usr = spil1[0];
            //用户的所有好友
            String[] friends = spil1[1].split(",");
            //以该用户为为value，好友为key,输出给map
            for (String friend:friends
                 ) {
                friendText.set(friend);
                usrText.set(usr);
                context.write(friendText, usrText);
            }
        }
    }


    static public class FriendsReducer extends Reducer<Text, Text, Text, NullWritable> {
       Text usrText2 = new Text();

        @Override
        protected void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
            //将iterator转化为list,因为iterator没有get方法，无法操作下标
            List<Text> usersList = new ArrayList<Text>();
            for (Text usr : values) {
                //Text usrText = new Text();
                //Text usrText = new Text();
                    Text usrText = new Text(usr);


                usersList.add(usrText);
            }

            for (int i = 0; i < usersList.size() - 1; i++) {
                for (int j = i + 1; j < usersList.size(); j++) {
                    //生成关系字符串如:a--b:c
                    usrText2.set(usersList.get(i).toString() + "--"+usersList.get(j).toString()+":"+key.toString());
                    context.write(usrText2, NullWritable.get());
                }
            }

        }
    }


    public static void main(String[] args) throws Exception {
        Configuration conf = new Configuration();

        Job job = Job.getInstance(conf);


        job.setJarByClass(FriendsPre.class);

        job.setJar("/Users/inequality/IdeaProjects/mapr/out/artifacts/mapr_jar/mapr.jar");
        //设置调用类
        job.setMapperClass(FriendsMapper.class);

        job.setReducerClass(FriendsReducer.class);

        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(Text.class);


        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(NullWritable.class);



        //指定输入输入输出路径

        FileInputFormat.setInputPaths(job, new Path(args[0]));
        FileOutputFormat.setOutputPath(job, new Path(args[1]));


        FileSystem fs = FileSystem.get(conf);
        if (fs.exists(new Path(args[1]))) {
            fs.delete(new Path(args[1]),true);
        }
        //将job中配置的相关参数，以及job所用的java类所用的jar包，提交给yarn去运行
        //job.submit();

        boolean res = job.waitForCompletion(true);

        System.exit(res ? 0 : 1);
    }
}

FirendsLast.java

    public class FriendsLast {


    static class FriendsLastMapper extends Mapper<LongWritable, Text, Text, Text> {
        Text relationText = new Text();
        Text friendText = new Text();
        @Override
        protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
            String line = value.toString();

            String[] words = line.split(":");
            //关系字符串
            String relation = words[0];
            //共同好友
            String friend = words[1];

            //输出
            relationText.set(relation);
            friendText.set(friend);

            context.write(relationText, friendText);
        }
    }


    static class FriendsLastReducer extends Reducer<Text, Text, Text, NullWritable> {

        Text re = new Text();
        @Override
        protected void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
            StringBuffer reSb = new StringBuffer(key.toString());
            reSb.append(": ");//a--b:
            for (Text friend : values) {
                reSb.append(friend.toString()).append(" ");
            }
            re.set(reSb.toString());
            context.write(re, NullWritable.get());
        }
    }


    public static void main(String[] args) throws Exception {

        Configuration conf = new Configuration();

        Job job = Job.getInstance(conf);


        job.setJarByClass(FriendsPre.class);

        job.setJar("/Users/inequality/IdeaProjects/mapr/out/artifacts/mapr_jar/mapr.jar");
        //设置调用类
        job.setMapperClass(FriendsLastMapper.class);

        job.setReducerClass(FriendsLastReducer.class);

        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(Text.class);


        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(NullWritable.class);



        //指定输入输入输出路径

        FileInputFormat.setInputPaths(job, new Path(args[0]));
        FileOutputFormat.setOutputPath(job, new Path(args[1]));


        FileSystem fs = FileSystem.get(conf);
        if (fs.exists(new Path(args[1]))) {
            fs.delete(new Path(args[1]),true);
        }
        //将job中配置的相关参数，以及job所用的java类所用的jar包，提交给yarn去运行
        //job.submit();

        boolean res = job.waitForCompletion(true);

        System.exit(res ? 0 : 1);
    }