大数据学习第11天:
好友推荐案例:
qq好友推荐 Hadoop好友world,hello好友是world…依次类推。 那么hadoop和hello有共同的好友world,所以hadoop和hello可能具有好友关系
代码如下
package com.sxt.hadoop.mr.fof;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.mapreduce.FailJob.FailReducer;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import com.sun.xml.internal.ws.policy.privateutil.PolicyUtils.Text;
public class MyFOF {
public static void main(String[] args) throws Exception {
Configuration conf =new Configuration();
Job job =Job.getInstance(conf);
job.setJarByClass(MyFOF.class);
//配置
//map
job.setMapperClass(Fmapper.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(IntWritable.class);
//reduce
job.setReducerClass(FReducer.class);
//input/output
Path intput=new Path("/data/fof/intput");
FileInputFormat.addInputPath(job,intput);
Path output=new Path("/data/fof/output");
if(output.getFileSystem(conf).exists(output)){
output.getFileSystem(conf).delete(output,true);
}
FileOutputFormat.setOutputPath(job,output);
//提交
job.waitForCompletion(true);
}
}
mapper端
package com.sxt.hadoop.mr.fof;
import java.io.IOException;
import java.lang.reflect.Array;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.util.StringUtils;
public class Fmapper extends Mapper<LongWritable,Text,Text,IntWritable>{
Text mkey=new Text();
IntWritable mval =new IntWritable();
@Override
protected void map(LongWritable key, Text value, Context context)
throws IOException, InterruptedException {
//tom hello hadoop cat
String[] strs=StringUtils.split(value.toString(),' ');
for(int i=0;i<strs.length;i++){
mkey.set(getFof(strs[0],strs[i]));
mval.set(0);
context.write(mkey, mval);
for(int j=i+1;j<strs.length;j++){
mkey.set(getFof(strs[i],strs[j]));
mval.set(1);
context.write(mkey, mval);
}
}
}
public static String getFof(String s1,String s2){
if(s1.compareTo(s2)<0){
return s1+":"+s2;
}
return s2+":"+s1;
}
}
reduce端
maping之后reduce之前,Shuffling进行洗牌,把相同key的整理在一起(默认的shuffling),结果如下: 洗牌结果(输入到reduce):
package com.sxt.hadoop.mr.fof;
import java.io.IOException;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;
public class FReducer extends Reducer<Text,IntWritable,Text,IntWritable>{
IntWritable rval =new IntWritable();
@Override
protected void reduce(Text key, Iterable<IntWritable> values,
Context context) throws IOException, InterruptedException {
int flg =0;
int sum =0;
for(IntWritable v : values){
if(v.get()==0){
flg=1;
}
sum+=v.get();
}
if(flg==0){
rval.set(sum);
context.write(key, rval);
}
}