找博客共同好友案例

1）需求：

以下是博客的好友列表数据，冒号前是一个用户，冒号后是该用户的所有好友（数据中的好友关系是单向的）

求出哪些人两两之间有共同好友，及他俩的共同好友都有谁？

2）需求分析：

先求出A、B、C、….等是谁的好友

3）代码实现

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.jobcontrol.ControlledJob;
import org.apache.hadoop.mapreduce.lib.jobcontrol.JobControl;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import java.io.IOException;
import java.util.Arrays;


/*输入
A:B,C,D,F,E,O
B:A,C,E,K
C:F,A,D,I
D:A,E,F,L
 */

/**
 * 输出
 * A-B	E C
 * A-C	D F
 * A-D	E F
 * A-E	D B C
 * A-F	O B C D E
 * A-G	F E C D
 */

/**
 * 2个job串联
 */
public class ShareFriends {
    public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException {
        args = new String[]{"D:\\test\\b16\\b163.txt", "D:\\test\\b16\\b1631", "D:\\test\\b16\\b1632"};
        Configuration conf = new Configuration();
        //第一次
        Job job1 = Job.getInstance(conf);
        job1.setMapperClass(OneShareFriendsMapper.class);
        job1.setReducerClass(OneShareFriendsReducer.class);
        job1.setMapOutputKeyClass(Text.class);
        job1.setMapOutputValueClass(Text.class);
        job1.setOutputKeyClass(Text.class);
        job1.setOutputValueClass(Text.class);

        FileSystem fs = FileSystem.get(conf);
        Path path = new Path(args[1]);
        if (fs.exists(path)){
            fs.delete(path,true);
        }

        FileInputFormat.setInputPaths(job1, new Path(args[0]));
        FileOutputFormat.setOutputPath(job1, new Path(args[1]));

        Job job2 = Job.getInstance(conf);

        job2.setMapperClass(TwoShareFriendsMapper.class);
        job2.setReducerClass(TwoShareFriendsReducer.class);
        job2.setMapOutputKeyClass(Text.class);
        job2.setMapOutputValueClass(Text.class);
        job2.setOutputKeyClass(Text.class);
        job2.setMapOutputValueClass(Text.class);

        Path path2 = new Path(args[2]);
        if (fs.exists(path2)){
            fs.delete(path2,true);
        }
        FileInputFormat.setInputPaths(job2,new Path(args[1]));
        FileOutputFormat.setOutputPath(job2,new Path(args[2]));

        JobControl sf = new JobControl("sf");
        ControlledJob ajob = new ControlledJob(job1.getConfiguration());
        ControlledJob bjob = new ControlledJob(job2.getConfiguration());
        bjob.addDependingJob(ajob);
        sf.addJob(ajob);
        sf.addJob(bjob);

        Thread t1 = new Thread(sf);
        t1.start();

        while (!sf.allFinished()){
            Thread.sleep(1000);
        }
        System.exit(0);

    }
}

class OneShareFriendsMapper extends Mapper<LongWritable, Text, Text, Text> {
    @Override
    protected void map(LongWritable key, Text value, Mapper<LongWritable, Text, Text, Text>.Context context) throws IOException, InterruptedException {
        String line = value.toString();
        String[] split = line.split(":");
        String person = split[0];
        String[] friends = split[1].split(",");

        for (String friend : friends) {
            //输出
            context.write(new Text(friend), new Text(person));
        }
    }
}

class OneShareFriendsReducer extends Reducer<Text, Text, Text, Text> {
    @Override
    protected void reduce(Text key, Iterable<Text> values, Reducer<Text, Text, Text, Text>.Context context) throws IOException, InterruptedException {
        StringBuffer stringBuffer = new StringBuffer();
        //拼接
        for (Text person : values) {
            stringBuffer.append(person).append(",");
        }
        context.write(key, new Text(stringBuffer.toString()));
    }

}

class TwoShareFriendsMapper extends Mapper<LongWritable, Text, Text, Text> {
    /*
    key     value
    好友     人，人，人，人。。。
    A	    I,K,C,B,G,F,H,O,D,
    B	    A,F,J,E,
    C	    A,E,B,H,F,G,K,
     */
    @Override
    protected void map(LongWritable key, Text value, Mapper<LongWritable, Text, Text, Text>.Context context) throws IOException, InterruptedException {
        String line = value.toString();
        String[] friend_persons = line.split("\t");

        String friend = friend_persons[0];
        String[] persons = friend_persons[1].split(",");

//        把persons装到arrays数列中
        Arrays.sort(persons);

        for (int i = 0; i < persons.length-1; i++) {
            for (int j = i+1; j < persons.length; j++) {

                context.write(new Text(persons[i]+"-"+persons[j]),new Text(friend));
                /*
                I-K A
                I-C A
                I-B A
                 */
            }
        }

    }
}
class TwoShareFriendsReducer extends Reducer<Text,Text,Text,Text>{
    @Override
    protected void reduce(Text key, Iterable<Text> values, Reducer<Text, Text, Text, Text>.Context context) throws IOException, InterruptedException {
        StringBuffer stringBuffer = new StringBuffer();

        //把friend串联起来
        /**
         * A-B	E C A与B的共同好友有E、C
         * A-C	D F
         */
        for (Text friend:values){
            stringBuffer.append(friend).append(" ");
        }
        context.write(key,new Text(stringBuffer.toString()));
    }
}