mapreduce求共同好友

逻辑分析

以下是qq的好友列表数据，冒号前是一个用户，冒号后是该用户的所有好友（数据中的好友关系是单向的）

A:B,C,D,F,E,O
B:A,C,E,K
C:F,A,D,I
D:A,E,F,L
E:B,C,D,M,L
F:A,B,C,D,E,O,M
G:A,C,D,E,F
H:A,C,D,E,O
I:A,O
J:B,O
K:A,C,D
L:D,E,F
M:E,F,G
O:A,H,I,J

求出哪些人两两之间有共同好友，及他俩的共同好友都有谁？

解题思路：

第一步  
map
读一行   A:B,C,D,F,E,O
输出    <B,A><C,A><D,A><F,A><E,A><O,A>
在读一行   B:A,C,E,K
输出   <A,B><C,B><E,B><K,B>

REDUCE
拿到的数据比如<C,A><C,B><C,E><C,F><C,G>......

输出：  
<A-B,C>
<A-E,C>
<A-F,C>
<A-G,C>
<B-E,C>
<B-F,C>.....

第二步
map
读入一行<A-B,C>
直接输出<A-B,C>
reduce
读入数据  <A-B,C><A-B,F><A-B,G>.......
输出： A-B  C,F,G,.....

第一步：代码实现

public class ComonsFriendsStepOne  extends Configured implements Tool {
    @Override
    public int run(String[] args) throws Exception {
        Configuration conf = super.getConf();
        Job job = Job.getInstance(conf, ComonsFriendsStepOne.class.getSimpleName());
        job.setInputFormatClass(TextInputFormat.class);
        TextInputFormat.addInputPath(job,new Path("file:///F:\\\input"));
        job.setMapperClass(ComonsFriendsStepOneMapper.class);
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(Text.class);
        job.setReducerClass(ComonsFriendsStepOneReducer.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(Text.class);
        job.setOutputFormatClass(TextOutputFormat.class);
        TextOutputFormat.setOutputPath(job,new Path("file:///F:\\output"));
        boolean b = job.waitForCompletion(true);
        return b?0:1;
    }
    public  static class ComonsFriendsStepOneMapper  extends Mapper<LongWritable,Text,Text,Text>{
        @Override
        protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
            String[] split = value.toString().split(":");
            String person = split[0];
            String[] friends = split[1].split(",");
            for (String friend : friends) {
                context.write(new Text(friend),new Text(person));
            }
        }
    }
    public static class ComonsFriendsStepOneReducer extends Reducer<Text,Text,Text,Text>{
        @Override
        protected void reduce(Text friend, Iterable<Text> persons, Context context) throws IOException, InterruptedException {
            StringBuffer buffer = new StringBuffer();
            for (Text person : persons) {
                buffer.append(person).append("-");
            }
            context.write(friend,new Text(buffer.toString()));
        }
    }
    public static void main(String[] args) throws Exception {
        Configuration configuration = new Configuration();
        ToolRunner.run(configuration,new ComonsFriendsStepOne(),args);
    }
}

第二步：代码实现

public class ComonsFriendsStepTwo extends Configured implements Tool {
    @Override
    public int run(String[] args) throws Exception {

        Job job = Job.getInstance(super.getConf(), ComonsFriendsStepTwo.class.getSimpleName());
        job.setInputFormatClass(TextInputFormat.class);
        TextInputFormat.addInputPath(job,new Path("file:///F:\\output"));
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(Text.class);
        job.setMapperClass(ComonsFriendStepTwoMapper.class);
        job.setReducerClass(ComonsFriendStepTwoReducer.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(Text.class);

        job.setOutputFormatClass(TextOutputFormat.class);
        TextOutputFormat.setOutputPath(job,new Path("file:///F:\\outstep2"));
        boolean b = job.waitForCompletion(true);
        return b?0:1;
    }
    public  static  class ComonsFriendStepTwoMapper  extends Mapper<LongWritable,Text,Text,Text>{

        /**
         * A   F-D-O-I-H-B-K-G-C-
         * B   E-A-J-F-
         * C   K-A-B-E-F-G-H-
         */
        @Override
        protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
            String[] split = value.toString().split("\t");
            String friends = split[0];
            String[] persons = split[1].split("-");
            //排序，避免c-b  与b-c  这样的情况出现
            Arrays.sort(persons);
            for(int i =0;i< persons.length -1 ;i++){
                for(int j = i+1;j<persons.length;j++){
                    context.write(new Text(persons[i]+"-"+persons[j]),new Text(friends));
                }

            }
        }
    }
    public static class ComonsFriendStepTwoReducer extends Reducer<Text,Text,Text,Text>{
        @Override
        protected void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
            StringBuffer buffer = new StringBuffer();
            for (Text value : values) {
                buffer.append(value.toString()+"\t");
            }
            context.write(key,new Text(buffer.toString()));
        }
    }
    public static void main(String[] args) throws Exception {
        ToolRunner.run(new Configuration(),new ComonsFriendsStepTwo(),args);
    }
} 

扩展：求互粉的人。

转载于:https://www.cnblogs.com/alexzhang92/p/11415974.html