大数据学习第11天：

大数据学习第11天：

好友推荐案例：

qq好友推荐 Hadoop好友world，hello好友是world…依次类推。 那么hadoop和hello有共同的好友world，所以hadoop和hello可能具有好友关系

代码如下

package com.sxt.hadoop.mr.fof;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;


import org.apache.hadoop.mapreduce.FailJob.FailReducer;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import com.sun.xml.internal.ws.policy.privateutil.PolicyUtils.Text;

public class MyFOF {
	public static void main(String[] args) throws Exception {
		Configuration conf =new Configuration();
		Job job =Job.getInstance(conf);
		job.setJarByClass(MyFOF.class);
		//配置
		
		
		//map
		job.setMapperClass(Fmapper.class);
		job.setMapOutputKeyClass(Text.class);
		job.setMapOutputValueClass(IntWritable.class);
		//reduce
		job.setReducerClass(FReducer.class);
		
		
		
		//input/output
		Path intput=new Path("/data/fof/intput");
		FileInputFormat.addInputPath(job,intput);
		
		Path output=new Path("/data/fof/output");
		if(output.getFileSystem(conf).exists(output)){
			output.getFileSystem(conf).delete(output,true);
			
		}
		FileOutputFormat.setOutputPath(job,output);
		//提交
		job.waitForCompletion(true);
		
		
		
	}

}

mapper端

package com.sxt.hadoop.mr.fof;

import java.io.IOException;
import java.lang.reflect.Array;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.util.StringUtils;

public class Fmapper extends Mapper<LongWritable,Text,Text,IntWritable>{
	
    Text mkey=new Text();
    IntWritable mval =new IntWritable();
    
   
    @Override
    protected void map(LongWritable key, Text value, Context context)
    		throws IOException, InterruptedException {
        //tom hello hadoop cat
    	String[] strs=StringUtils.split(value.toString(),' ');
    	   	
    	
    	for(int i=0;i<strs.length;i++){
    	    mkey.set(getFof(strs[0],strs[i]));
    		mval.set(0);
    		context.write(mkey, mval);

        	for(int j=i+1;j<strs.length;j++){
        	    mkey.set(getFof(strs[i],strs[j]));
        		mval.set(1);
        		context.write(mkey, mval);
        	}
    	}
    	
    	
    }
    public static String getFof(String s1,String s2){
		if(s1.compareTo(s2)<0){
			return s1+":"+s2;
		}
		return s2+":"+s1;
		
	}
	
	
	
}

reduce端

maping之后reduce之前，Shuffling进行洗牌，把相同key的整理在一起（默认的shuffling），结果如下： 洗牌结果(输入到reduce)：

package com.sxt.hadoop.mr.fof;

import java.io.IOException;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

public class FReducer extends Reducer<Text,IntWritable,Text,IntWritable>{
   
	IntWritable rval =new IntWritable();
	
	
	
	@Override
    protected void reduce(Text key, Iterable<IntWritable> values,
    	Context context) throws IOException, InterruptedException {
    	int flg =0;
    	int sum =0;
		for(IntWritable v : values){
			if(v.get()==0){
				flg=1;
			}
			sum+=v.get();
		}
		if(flg==0){
			rval.set(sum);
			context.write(key, rval);
		}
		
		
    	
    	
    	
    }