11 map多级处理-找出隐藏好友-可能认识的人

数据

思路

将所有的关系找出来，然后再把实质关系踢出，剩下的就是隐藏关系了

关系Mapper

找出所有关系，某个人认识哪些人，然后通过怎么一个关系，找出可能的隐藏好友

package friend;

import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

import java.io.IOException;

public class RelationMapper extends Mapper<LongWritable, Text,Text,Text> {
    @Override
    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
        String[] s = value.toString().split(" ");
        context.write(new Text(s[0]),new Text(s[1]));
    }
}

package friend;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

public class RelationReducer extends Reducer<Text,Text,Text, IntWritable> {
    //收到的数据 tom: rose,smith,jim,lucy
    //写出实质关系，名字大的放在前面

    @Override
    protected void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
        String name = key.toString();//当前人名
        List<String> list = new ArrayList<>();//装所有人信息，因为迭代器只能迭代一次
        for (Text value : values) {
            String s = value.toString();
            list.add(s);
            if(name.compareTo(s)<0)//将小的人名下写在前面
                context.write(new Text(name + "-" + s),new IntWritable(1));
            else
                context.write(new Text(s + "-" + name),new IntWritable(1));
        }
        // 推测好友关系 - 根据好友列表，推测两两是否认识，把这种推测的关系用0表示,list中的都认识当前的name
        for (int i = 0; i < list.size() - 1; i++) {
            for (int j = i + 1; j < list.size(); j++) {
                String f1 = list.get(i);
                String f2 = list.get(j);
                if (f1.compareTo(f2) < 0)
                    context.write(new Text(f1 + "-" + f2), new IntWritable(0));
                else
                    context.write(new Text(f2 + "-" + f1), new IntWritable(0));
            }
        }
    }
}

package friend;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import java.io.IOException;

public class RelationDriver {
    public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
        Configuration conf = new Configuration();
        Job job = Job.getInstance(conf);

        job.setJarByClass(RelationDriver.class);
        job.setMapperClass(RelationMapper.class);
        job.setReducerClass(RelationReducer.class);

        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(Text.class);

        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(IntWritable.class);

        FileInputFormat.addInputPath(job,
                new Path("hdfs://hadoop01:9000/txt/friend.txt"));
        FileOutputFormat.setOutputPath(job,
                new Path("hdfs://hadoop01:9000/result/relation"));

        job.waitForCompletion(true);
    }
}

package friend;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

import java.io.IOException;

public class FriendMapper extends Mapper<LongWritable, Text, Text, IntWritable> {
    @Override
    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
        String[] arr = value.toString().split("\t");
        context.write(new Text(arr[0]), new IntWritable(Integer.parseInt(arr[1])));
    }
}

package friend;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

import java.io.IOException;

public class FriendReducer extends Reducer<Text, IntWritable, Text, NullWritable> {
    @Override
    protected void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException {
        // values中放到的好友关系
        for (IntWritable value : values) {
            // 找的是隐藏好友
            // 真实好友关系用的1表示，推测的好友关系用0表示
            // 如果出现了1，那么说明他俩不是隐藏好友
            // 如果全部都是0，那么说明隐藏好友
            if(value.get()==1)
                return;
        }
        context.write(key,NullWritable.get());
    }
}

package friend;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import java.io.IOException;

public class FriendDriver {
    public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {

        Configuration conf = new Configuration();
        Job job = Job.getInstance(conf);

        job.setJarByClass(FriendDriver.class);
        job.setMapperClass(FriendMapper.class);
        job.setReducerClass(FriendReducer.class);
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(IntWritable.class);

        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(NullWritable.class);

        FileInputFormat.addInputPath(job, new Path("hdfs://hadoop01:9000/result/relation"));
        FileOutputFormat.setOutputPath(job,
                new Path("hdfs://hadoop01:9000/result/friend"));

        job.waitForCompletion(true);
    }
}