共同好友查找

需求：有如下文件，A的好友有B,C,D,F,E,O，查找出A和B的共同好友有哪些

A:B,C,D,F,E,O
B:A,C,E,K
C:F,A,D,I
D:A,E,F,L
E:B,C,D,M,L
F:A,B,C,D,E,O,M
G:A,C,D,E,F
H:A,C,D,E,O
I:A,O
J:B,O
K:A,C,D
L:D,E,F
M:E,F,G
O:A,H,I,J

1：定义Friends1Map类

package Friends;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
import java.io.IOException;

public class Friends1Map extends Mapper<LongWritable, Text, Text, Text> {
    @Override
    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
        String line = value.toString();
        String[] fields = line.split(":");
        //person在前，friends在后，拿到person   A:B,C,D,F,E,O
        String person = fields[0];
        //拿到friends
        String[] friends = fields[1].split(",");
        //重新拼接数据，friends在前，person在后
        for(String friend : friends){
            context.write(new Text(friend),new Text(person));
        }
    }
}

2：定义Friends1Reducer类

package Friends;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;
import java.io.IOException;

public class Friends1Reducer extends Reducer<Text, Text, Text, Text>{
    @Override
    protected void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
        StringBuffer sb = new StringBuffer();
        for (Text person : values){
             sb.append(person).append(",");
        }
        context.write(key, new Text(sb.toString()));
    }
}

3：定义了Friends1Driver类

package Friends;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import java.io.IOException;

public class Friends1Driver {
    public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
        args = new String[]{"E:\\bigdata_code\\friends","E:\\bigdata_code\\friends\\out1"};

        //1 获取job对象
        Configuration conf = new Configuration();
        Job job = Job.getInstance(conf);
        //2 指定jar包运行路径
        job.setJarByClass(Friends1Driver.class);
        //3 指定map类、map输出数据类型
        job.setMapperClass(Friends1Map.class);
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(Text.class);
        //4 指定Reduce类、reduce输出数据类型
        job.setReducerClass(Friends1Reducer.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(Text.class);
        //5 指定job的输入输出目录
        FileInputFormat.setInputPaths(job, new Path(args[0]));
        FileOutputFormat.setOutputPath(job, new Path(args[1]));
        //6 提交
        boolean result = job.waitForCompletion(true);
        System.exit(result ? 1:0);
    }
}

输出结果：

--------------------------------------

4：定义Friends2Map 类
package Friends;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
import java.io.IOException;
import java.util.Arrays;

public class Friends2Map extends Mapper<LongWritable, Text, Text, Text> {
    @Override
    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
        String line = value.toString();
        String[] friend_persons = line.split("\t");
        //拿到friend、person，A    I,K,C,B,G,F,H,O,D,
        String friend = friend_persons[0];
        String[] persons = friend_persons[1].split(",");
        Arrays.sort(persons);
        for (int x = 0; x < persons.length -1; x++){
            for(int y = x + 1; y < persons.length; y++){
                context.write(new Text(persons[x] + "-" + persons[y]), new Text(friend));
            }
        }
    }
}

5：定义了Friends2Reducer类

package Friends;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;
import java.io.IOException;

public class Friends2Reducer extends Reducer< Text, Text, Text, Text> {
    @Override
    protected void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
        StringBuilder sb = new StringBuilder();

        for (Text friend : values){
            sb.append(friend).append(" ");
        }
        context.write(key, new Text(sb.toString()));
    }
}

6：定义了Friends2Driver

package Friends;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import java.io.IOException;

public class Friends2Driver {
    public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
        args = new String[]{"E:\\bigdata_code\\friends\\out1","E:\\bigdata_code\\friends\\out2"};

        Configuration conf = new Configuration();
        Job job = Job.getInstance(conf);
        job.setJarByClass(Friends2Driver.class);

        job.setMapperClass(Friends2Map.class);
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(Text.class);

        job.setReducerClass(Friends2Reducer.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(Text.class);

        FileInputFormat.setInputPaths(job,new Path(args[0]));
        FileOutputFormat.setOutputPath(job,new Path(args[1]));

        job.waitForCompletion(true);
    }
}

输出结果：

package Friends;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.jobcontrol.ControlledJob;
import org.apache.hadoop.mapreduce.lib.jobcontrol.JobControl;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import java.io.IOException;

public class AllFriendsDriver {
    public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
        args = new String[]{"E:\\bigdata_code\\friends",
                            "E:\\bigdata_code\\friends\\out1",
                            "E:\\bigdata_code\\friends\\out2"};
                            
        Configuration conf = new Configuration();
        Job job1 = Job.getInstance(conf);
        job1.setJarByClass(AllFriendsDriver.class);

        job1.setMapperClass(Friends1Map.class);
        job1.setMapOutputKeyClass(Text.class);
        job1.setMapOutputValueClass(Text.class);

        job1.setReducerClass(Friends1Reducer.class);
        job1.setOutputKeyClass(Text.class);
        job1.setOutputValueClass(Text.class);

        FileInputFormat.setInputPaths(job1, new Path(args[0]));
        FileOutputFormat.setOutputPath(job1, new Path(args[1]));

        Job job2 = Job.getInstance(conf);

        job2.setMapperClass(Friends2Map.class);
        job2.setMapOutputKeyClass(Text.class);
        job2.setMapOutputValueClass(Text.class);

        job2.setReducerClass(Friends2Reducer.class);
        job2.setOutputKeyClass(Text.class);
        job2.setOutputValueClass(Text.class);

        FileInputFormat.setInputPaths(job2, new Path(args[1]));
        FileOutputFormat.setOutputPath(job2, new Path(args[2]));

        JobControl control = new JobControl("zhang");
        ControlledJob ajob = new ControlledJob(job1.getConfiguration());
        ControlledJob bjob = new ControlledJob(job2.getConfiguration());
        bjob.addDependingJob(ajob);
        control.addJob(ajob);
        control.addJob(bjob);
        Thread thread = new Thread();
        thread.start();
    }
}

在将job1、job2串联起来时候，遇到问题，报了个警告就不往下走了，初步查找，觉得是环境问题，但是换了饱之后还是同样问题，暂时不知如何解决。希望有大神能帮助看看
2019-06-24 17:14:30,117 WARN [org.apache.hadoop.util.NativeCodeLoader] - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable

Process finished with exit code 0