推荐好友二度关系（Hadoop解决方案）

数据样本

hadoop  hello  
hadoop  cat
hdfs    world
tom cat
tom hdfs 
cat dog
hello   world
hello   dear
hello   lilei
lilei   hanmeimei

分析

推荐好友就是将可能认识的人推荐给他，简单的说，就是找到可能存在二度关系的人

思路

就是将所有关系都找出来

解决方案

不啰嗦，上代码

mapper阶段

import java.io.IOException;

import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

public class QQmapper extends Mapper<LongWritable, Text, Text, Text>{
    protected void map(LongWritable key, Text value,
            Context context)
            throws IOException, InterruptedException {
        String line = value.toString();
        String[] st = line.split(" ");
        context.write(new Text(st[0]), new Text(st[1]));
        context.write(new Text(st[1]), new Text(st[0]));
    }
}

reducer阶段

import java.io.IOException;
import java.util.HashSet;
import java.util.Iterator;

import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

public class QQreducer extends Reducer<Text, Text, Text, Text>{
    protected void reduce(Text key, Iterable<Text> value,Context context) 
            throws IOException,InterruptedException {
        HashSet<String> set = new HashSet<String>();
        for (Text tt : value) {
            set.add(tt.toString());
        }
        if(set.size() > 1){
            for (Iterator itr = set.iterator(); itr.hasNext();) {
                String name = (String) itr.next();
                for (Iterator itr1 = set.iterator(); itr1.hasNext();) {
                    String other = (String) itr1.next();
                    if (!name.equals(other)) {
                        context.write(new Text(name), new Text(other));
                    }
                }
            }
        }
    }
}

Eclipse集成hadoop插件测试

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import com.bj.qq.QQmapper;
import com.bj.qq.QQreducer;

public class JobsRun {
    public static void main(String[] args) {
        Configuration conf = new Configuration();
        conf.set("fs.default.name", "hdfs://node1:9000");
        conf.set("mapred.job.tracker", "node1:9001");
        conf.set("mapred.jar", "C:\\Users\\Administrator\\Desktop\\recomfriend.jar");
        try {
            Job job = new Job(conf);
            job.setJarByClass(JobsRun.class);
            job.setMapperClass(QQmapper.class);
            job.setReducerClass(QQreducer.class);
            job.setMapOutputKeyClass(Text.class);
            job.setMapOutputValueClass(Text.class);
            job.setNumReduceTasks(2);

            FileInputFormat.addInputPath(job, new Path("/user/root/input/"));
            FileOutputFormat.setOutputPath(job, new Path("/user/root/output/"));

            System.exit(job.waitForCompletion(true) ? 0 : 1);
        } catch (Exception e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }
    }
}

测试结果

world   tom
tom world
hanmeimei   hello
hello   hanmeimei
tom hadoop
tom dog
hadoop  tom
hadoop  dog
dog tom
dog hadoop
lilei   world
lilei   hadoop
lilei   dear
world   lilei
world   hadoop
world   dear
hadoop  lilei
hadoop  world
hadoop  dear
dear    lilei
dear    world
dear    hadoop
cat hdfs
hdfs    cat
hdfs    hello
hello   hdfs
cat hello
hello   cat