Hadoop入门之共同好友实现Demo

以下是qq的好友列表数据，冒号前是一个用，冒号后是该用户的所有好友（数据中的好友关系是单向的）

A:B,C,D,F,E,O

B:A,C,E,K

C:F,A,D,I

D:A,E,F,L

E:B,C,D,M,L

F:A,B,C,D,E,O,M

G:A,C,D,E,F

H:A,C,D,E,O

I:A,O

J:B,O

K:A,C,D

L:D,E,F

M:E,F,G

O:A,H,I,J

 

求出哪些人两两之间有共同好友，及他俩的共同好友都有谁？

package com.demo.friends;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;


/**
 * @Description: 
 *  1.以key为聚合点 将某个好友点先聚合出来 ,且对value集合进行排序
 *    B:A,E,F,J
 *    C:A,B,E,F,G,H,K
 *  2.以排序后的两个人为key,再次进行聚合操作
 *    A-E:B,C
 *    ....
 *    
 * @author: songqinghu
 * @date: 2017年8月30日 下午5:16:49
 * Version:1.0
 */
public class FriendOneStep {
    
    public static void main(String[] args) throws Exception {
        Configuration conf = new Configuration();
        conf.set("HADOOP_USER_NAME", "hadoop");
        Job job = Job.getInstance(conf);
        
        job.setJarByClass(FriendOneStep.class);
        
        job.setMapperClass(FriendOneStepMapper.class);
        job.setReducerClass(FriendOneStepReducer.class);
        
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(Text.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(Text.class);
        
        
        FileInputFormat.setInputPaths(job, new Path(args[0]));
        FileOutputFormat.setOutputPath(job, new Path(args[1]));
        
        
//        job.setNumReduceTasks(2);
        
        job.waitForCompletion(true);
    }
}

class FriendOneStepReducer extends Reducer<Text, Text, Text, Text>{
    
    @Override
    protected void reduce(Text text, Iterable<Text> iters, Context context)
            throws IOException, InterruptedException {
         //收集后进行排序组装
        List<String> list = new ArrayList<String>();
        for (Text t : iters) {
            list.add(t.toString());
        }
        Object[] array = list.toArray();
        Arrays.sort(array);
        
        StringBuffer sb = new StringBuffer();
        for (int i = 0; i < array.length; i++) {
            sb.append(array[i].toString());
            if(i<array.length-1){
                sb.append(",");
            }
        }
        
        context.write(text, new Text(sb.toString()));
        
    }
    
}

class FriendOneStepMapper extends Mapper<LongWritable, Text, Text, Text>{
    
    Text outKey = new Text();
    Text outValue = new Text();
    
    @Override
    protected void map(LongWritable lon, Text value, Context context)
            throws IOException, InterruptedException {
        
        String line = value.toString();
        String[] temp = line.split(":");
        outValue.set(temp[0]);
        
        String[] keys = temp[1].split(",");
        
        for (String key : keys) {
            outKey.set(key);
            context.write(outKey, outValue);
        }
    }
    
}

package com.demo.friends;

import java.io.IOException;
import java.util.Iterator;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

/**
 * 
 * @Description: 这一步以两个用户为Key和聚合
 * @author: songqinghu
 * @date: 2017年8月30日 下午5:52:50
 * Version:1.0
 */
public class FriendTwoStep {

    public static void main(String[] args) throws Exception {
        Configuration conf = new Configuration();
        conf.set("HADOOP_USER_NAME", "hadoop");
        Job job = Job.getInstance(conf);
        
        job.setJarByClass(FriendTwoStep.class);
        
        job.setMapperClass(FriendTwoStepMapper.class);
        job.setReducerClass(FriendTwoStepReducer.class);
        
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(Text.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(NullWritable.class);
        
        
        FileInputFormat.setInputPaths(job, new Path(args[0]));
        FileOutputFormat.setOutputPath(job, new Path(args[1]));
        
        
//        job.setNumReduceTasks(2);
        
        job.waitForCompletion(true);
    }
    
    
}

class FriendTwoStepReducer extends Reducer<Text, Text, Text,NullWritable>{
    
    @Override
    protected void reduce(Text text, Iterable<Text> iters, Context context)
            throws IOException, InterruptedException {
        //相同的键值对聚合在一起,将其共同好友组合在一起
        
        StringBuffer sb = new StringBuffer();
        sb.append(text.toString()).append(":");
        Iterator<Text> iterator = iters.iterator();
        while (iterator.hasNext()) {
            sb.append(iterator.next().toString());
            if(iterator.hasNext()){
                sb.append(",");
            }
            
        }
        context.write(new Text(sb.toString()), null);
    }
    
}


class FriendTwoStepMapper extends Mapper<LongWritable, Text, Text, Text>{
    
    Text outKey = new Text();
    Text outValue = new Text();
    @Override
    protected void map(LongWritable key, Text value, Context context)
            throws IOException, InterruptedException {
        String line = value.toString();
        String[] temp = line.split("\t");
        
        outValue.set(temp[0]);
        
        //切割出各个用户
        String[] friends = temp[1].split(",");
        //如果数量小于2那就跳过
        if(friends.length>1){
            //组装key
            for (int i = 0; i < friends.length-1; i++) {
                
                for (int j = i+1; j < friends.length; j++) {
                    outKey.set(friends[i]+"-"+friends[j]);
                    context.write(outKey,outValue);
                }
            }
            
        }
        
    }
    
}