MapReduce实操案例

共同好友求取

要求
求出哪些人两两之间有共同好友，及他俩的共同好友都有谁？
解题思路

• 第一步
  map
  读一行 A:B,C,D,F,E,O
  输出 <B,A><C,A><D,A><F,A><E,A><O,A>
  在读一行 B:A,C,E,K
  输出 <A,B><C,B><E,B><K,B>
  reduce
  拿到的数据比如<C,A><C,B><C,E><C,F><C,G>…
  输出：
  <A-B,C>
  <A-E,C>
  <A-F,C>
  <A-G,C>
  <B-E,C>
  <B-F,C>…
• 第二步
  map
  读入一行<A-B,C>
  直接输出<A-B,C>
  reduce
  读入数据 <A-B,C><A-B,F><A-B,G>…
  输出： A-B C,F,G,…

代码实现

• 第一步

public class ComonsFriendsStepOne  extends Configured implements Tool {
   
    @Override
    public int run(String[] args) throws Exception {
   
        Configuration conf = super.getConf();
        Job job = Job.getInstance(conf, ComonsFriendsStepOne.class.getSimpleName());
        job.setInputFormatClass(TextInputFormat.class);
        TextInputFormat.addInputPath(job,new Path("file:///F:\\传智播客大数据离线阶段课程资料\\5、大数据离线第五天\\共同好友\\input"));
        job.setMapperClass(ComonsFriendsStepOneMapper.class);
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(Text.class);
        job.setReducerClass(ComonsFriendsStepOneReducer.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(Text.class);
        job.setOutputFormatClass(TextOutputFormat.class);
        TextOutputFormat.setOutputPath(job,new Path("file:///F:\\传智播客大数据离线阶段课程资料\\5、大数据离线第五天\\共同好友\\output"));
        boolean b = job.waitForCompletion(true);
        return b?0:1;
    }
    public  static class ComonsFriendsStepOneMapper  extends Mapper<LongWritable,Text,Text,Text>{
   
        @Override
        protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
   
            String[] split = value.toString().split(":");
            String person = split[0];
            String[] friends = split[1].split(",");
            for (String friend : friends) {
   
                context.write(new Text(friend),new Text(person));
            }
        }
    }
    public static class ComonsFriendsStepOneReducer extends Reducer<Text,Text,Text,Text>{
   
        @Override
        protected void reduce(Text friend, Iterable<Text> persons, Context context) throws IOException, InterruptedException {
   
            StringBuffer buffer = new StringBuffer();
            for (Text person : persons) {
   
                buffer.append(person).append("-");
            }
            context.write(friend,new Text(buffer.toString()));
        }
    }
    public static void main(String[] args) throws Exception {
   
        Configuration configuration = new Configuration();
        ToolRunner.run(configuration,new ComonsFriendsStepOne(),args);
    }
}

• 第二步

public class ComonsFriendsStepTwo extends Configured implements Tool {
   
    @Override
    public int run(String[] args) throws Exception {
   

        Job job = Job.getInstance(super.getConf(), ComonsFriendsStepTwo.class.getSimpleName());
        job.setInputFormatClass(TextInputFormat.class);
        TextInputFormat.addInputPath(job,new Path("file:///F:\\传智播客大数据离线阶段课程资料\\5、大数据离线第五天\\共同好友\\output"));
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(Text