使用MapReduce实现推荐算法,进行迭代计算
Step 1:(去重)
第一次—Mapper
static class Step1_Mapper extends Mapper<LongWritable, Text, Text, NullWritable>{
protected void map(LongWritable key, Text value, Context context)
throws IOException, InterruptedException {
if(key.get()!=0){
context.write(value, NullWritable.get());
}
}
}
第一次——Reducer
static class Step1_Reducer extends Reducer<Text, IntWritable, Text, NullWritable>{
protected void reduce(Text key, Iterable<IntWritable> i, Context context)
throws IOException, InterruptedException {
context.write(key,NullWritable.get());
}
}
第一次——Main
public static boolean run(Configuration config,Map<String, String> paths){
try {
FileSystem fs =FileSystem.get(config);
Job job =Job.getInstance(config);
job.setJobName("step1");
job.setJarByClass(Step1.class);
job.setMapperClass(Step1_Mapper.class);
job.setReducerClass(Step1_Reducer.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(NullWritable.class);
FileInputFormat.addInputPath(job, new Path(paths.get("Step1Input")));
Path outpath=new Path(paths.get("Step1Output"));
if(fs.exists(outpath)){
fs.delete(outpath,true);
}
FileOutputFormat.setOutputPath(job, outpath);
boolean f= job.waitForCompletion(true);
return f;
} catch (Exception e) {
e.printStackTrace();
}
return false;
}
Step 2:
/**
* 按用户分组,计算所有物品出现的组合列表,得到用户对物品的喜爱度得分矩阵
u13 i160:1,
u14 i25:1,i223:1,
u16 i252:1,i276:1,i201:3,
u21 i266:1,
u24 i64:1,i218:1,i185:1,
u26 i276:1,i201:1,i348:1,i321:1,i136:1,
* @author root
*
*/
第二次——Mapper
static class Step2_Mapper extends Mapper<LongWritable, Text, Text, Text>{
//如果使用:用户+物品,同时作为输出key,更优
protected void map(LongWritable key, Text value,
Context context)
throws IOException, InterruptedException {
String[] tokens=value.toString().split(","); //定义一个token数组,并且使用,进行切割
String item=tokens[0]; //找出item的位置在token[0]
String user=tokens[1];//找出user的位置在token[1]
String action =tokens[2];//找出action的位置在token[2]
Integer rv =StartRun.R.get(action);
Text k= new Text(user);
// if(rv!=null){
Text v =new Text(item+":"+ rv.intValue());//讲rv的值定义为整型化
context.write(k, v);
}
}
第二次Reducer:
static class Step2_Reducer extends Reducer<Text, Text, Text, Text>{
protected void reduce(Text key, Iterable<Text> i,
Context context)
throws IOException, InterruptedException {
Map<String, Integer> r =new HashMap<String, Integer>();
for(Text value :i){
String[] vs =value.toString().split(":"); //定义一个中间数组,取名为vs
String item=vs[0];//找出item的位置是在vs[0]
Integer action=Integer.parseInt(vs[1]);//找出action的位置是在vs[1],并且对其进行整型化
action = ((Integer) (r.get(item)==null? 0:r.get(item))).intValue() + action;
r.put(item,action);//使用r把item和action合并,放到一起
}
StringBuffer sb =new StringBuffer(); //定义一个字符串缓冲区
for(Entry<String, Integer> entry :r.entrySet() ){ //设置entry入口,并且进行遍历
sb.append(entry.getKey()+":"+entry.getValue().intValue()+",");//追加key、value的值
}
context.write(key,new Text(sb.toString()));
}
}
第二次——Main
public static boolean run(Configuration config,Map<String, String> paths){
try {
// config.set("mapred.jar", "C:\\Users\\Administrator\\Desktop\\wc.jar");
FileSystem fs =FileSystem.get(config);
Job job =Job.getInstance(config);
job.setJobName("step2");
job.setJarByClass(StartRun.class);
job.setMapperClass(Step2_Mapper.class);
job.setReducerClass(Step2_Reducer.class);
//
job