逻辑分析
以下是qq的好友列表数据,冒号前是一个用户,冒号后是该用户的所有好友(数据中的好友关系是单向的)
A:B,C,D,F,E,O
B:A,C,E,K
C:F,A,D,I
D:A,E,F,L
E:B,C,D,M,L
F:A,B,C,D,E,O,M
G:A,C,D,E,F
H:A,C,D,E,O
I:A,O
J:B,O
K:A,C,D
L:D,E,F
M:E,F,G
O:A,H,I,J
求出哪些人两两之间有共同好友,及他俩的共同好友都有谁?
解题思路:
第一步
map
读一行 A:B,C,D,F,E,O
输出 <B,A><C,A><D,A><F,A><E,A><O,A>
在读一行 B:A,C,E,K
输出 <A,B><C,B><E,B><K,B>
REDUCE
拿到的数据比如<C,A><C,B><C,E><C,F><C,G>......
输出:
<A-B,C>
<A-E,C>
<A-F,C>
<A-G,C>
<B-E,C>
<B-F,C>.....
第二步
map
读入一行<A-B,C>
直接输出<A-B,C>
reduce
读入数据 <A-B,C><A-B,F><A-B,G>.......
输出: A-B C,F,G,.....
第一步:代码实现
public class ComonsFriendsStepOne extends Configured implements Tool {
@Override
public int run(String[] args) throws Exception {
Configuration conf = super.getConf();
Job job = Job.getInstance(conf, ComonsFriendsStepOne.class.getSimpleName());
job.setInputFormatClass(TextInputFormat.class);
TextInputFormat.addInputPath(job,new Path("file:///F:\\\input"));
job.setMapperClass(ComonsFriendsStepOneMapper.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(Text.class);
job.setReducerClass(ComonsFriendsStepOneReducer.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);
job.setOutputFormatClass(TextOutputFormat.class);
TextOutputFormat.setOutputPath(job,new Path("file:///F:\\output"));
boolean b = job.waitForCompletion(true);
return b?0:1;
}
public static class ComonsFriendsStepOneMapper extends Mapper<LongWritable,Text,Text,Text>{
@Override
protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
String[] split = value.toString().split(":");
String person = split[0];
String[] friends = split[1].split(",");
for (String friend : friends) {
context.write(new Text(friend),new Text(person));
}
}
}
public static class ComonsFriendsStepOneReducer extends Reducer<Text,Text,Text,Text>{
@Override
protected void reduce(Text friend, Iterable<Text> persons, Context context) throws IOException, InterruptedException {
StringBuffer buffer = new StringBuffer();
for (Text person : persons) {
buffer.append(person).append("-");
}
context.write(friend,new Text(buffer.toString()));
}
}
public static void main(String[] args) throws Exception {
Configuration configuration = new Configuration();
ToolRunner.run(configuration,new ComonsFriendsStepOne(),args);
}
}
第二步:代码实现
public class ComonsFriendsStepTwo extends Configured implements Tool {
@Override
public int run(String[] args) throws Exception {
Job job = Job.getInstance(super.getConf(), ComonsFriendsStepTwo.class.getSimpleName());
job.setInputFormatClass(TextInputFormat.class);
TextInputFormat.addInputPath(job,new Path("file:///F:\\output"));
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(Text.class);
job.setMapperClass(ComonsFriendStepTwoMapper.class);
job.setReducerClass(ComonsFriendStepTwoReducer.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);
job.setOutputFormatClass(TextOutputFormat.class);
TextOutputFormat.setOutputPath(job,new Path("file:///F:\\outstep2"));
boolean b = job.waitForCompletion(true);
return b?0:1;
}
public static class ComonsFriendStepTwoMapper extends Mapper<LongWritable,Text,Text,Text>{
/**
* A F-D-O-I-H-B-K-G-C-
* B E-A-J-F-
* C K-A-B-E-F-G-H-
*/
@Override
protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
String[] split = value.toString().split("\t");
String friends = split[0];
String[] persons = split[1].split("-");
//排序,避免c-b 与b-c 这样的情况出现
Arrays.sort(persons);
for(int i =0;i< persons.length -1 ;i++){
for(int j = i+1;j<persons.length;j++){
context.write(new Text(persons[i]+"-"+persons[j]),new Text(friends));
}
}
}
}
public static class ComonsFriendStepTwoReducer extends Reducer<Text,Text,Text,Text>{
@Override
protected void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
StringBuffer buffer = new StringBuffer();
for (Text value : values) {
buffer.append(value.toString()+"\t");
}
context.write(key,new Text(buffer.toString()));
}
}
public static void main(String[] args) throws Exception {
ToolRunner.run(new Configuration(),new ComonsFriendsStepTwo(),args);
}
}
扩展:求互粉的人。