数据
思路
将所有的关系找出来,然后再把实质关系踢出,剩下的就是隐藏关系了
关系Mapper
找出所有关系,某个人认识哪些人,然后通过怎么一个关系,找出可能的隐藏好友
package friend;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
import java.io.IOException;
public class RelationMapper extends Mapper<LongWritable, Text,Text,Text> {
@Override
protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
String[] s = value.toString().split(" ");
context.write(new Text(s[0]),new Text(s[1]));
}
}
package friend;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
public class RelationReducer extends Reducer<Text,Text,Text, IntWritable> {
//收到的数据 tom: rose,smith,jim,lucy
//写出实质关系,名字大的放在前面
@Override
protected void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
String name = key.toString();//当前人名
List<String> list = new ArrayList<>();//装所有人信息,因为迭代器只能迭代一次
for (Text value : values) {
String s = value.toString();
list.add(s);
if(name.compareTo(s)<0)//将小的人名下写在前面
context.write(new Text(name + "-" + s),new IntWritable(1));
else
context.write(new Text(s + "-" + name),new IntWritable(1));
}
// 推测好友关系 - 根据好友列表,推测两两是否认识,把这种推测的关系用0表示,list中的都认识当前的name
for (int i = 0; i < list.size() - 1; i++) {
for (int j = i + 1; j < list.size(); j++) {
String f1 = list.get(i);
String f2 = list.get(j);
if (f1.compareTo(f2) < 0)
context.write(new Text(f1 + "-" + f2), new IntWritable(0));
else
context.write(new Text(f2 + "-" + f1), new IntWritable(0));
}
}
}
}
package friend;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import java.io.IOException;
public class RelationDriver {
public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
Configuration conf = new Configuration();
Job job = Job.getInstance(conf);
job.setJarByClass(RelationDriver.class);
job.setMapperClass(RelationMapper.class);
job.setReducerClass(RelationReducer.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(Text.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(IntWritable.class);
FileInputFormat.addInputPath(job,
new Path("hdfs://hadoop01:9000/txt/friend.txt"));
FileOutputFormat.setOutputPath(job,
new Path("hdfs://hadoop01:9000/result/relation"));
job.waitForCompletion(true);
}
}
package friend;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
import java.io.IOException;
public class FriendMapper extends Mapper<LongWritable, Text, Text, IntWritable> {
@Override
protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
String[] arr = value.toString().split("\t");
context.write(new Text(arr[0]), new IntWritable(Integer.parseInt(arr[1])));
}
}
package friend;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;
import java.io.IOException;
public class FriendReducer extends Reducer<Text, IntWritable, Text, NullWritable> {
@Override
protected void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException {
// values中放到的好友关系
for (IntWritable value : values) {
// 找的是隐藏好友
// 真实好友关系用的1表示,推测的好友关系用0表示
// 如果出现了1,那么说明他俩不是隐藏好友
// 如果全部都是0,那么说明隐藏好友
if(value.get()==1)
return;
}
context.write(key,NullWritable.get());
}
}
package friend;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import java.io.IOException;
public class FriendDriver {
public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
Configuration conf = new Configuration();
Job job = Job.getInstance(conf);
job.setJarByClass(FriendDriver.class);
job.setMapperClass(FriendMapper.class);
job.setReducerClass(FriendReducer.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(IntWritable.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(NullWritable.class);
FileInputFormat.addInputPath(job, new Path("hdfs://hadoop01:9000/result/relation"));
FileOutputFormat.setOutputPath(job,
new Path("hdfs://hadoop01:9000/result/friend"));
job.waitForCompletion(true);
}
}