1.Mapper端设计
package com.mycat.mapd_movie_mapjoin;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
public class RatingMapper extends Mapper<LongWritable, Text, IntWritable,Text> {
Map<Integer,String> map=new HashMap<>();
IntWritable mk=new IntWritable();
Text mv=new Text();
@Override
protected void setup(Context context) throws IOException, InterruptedException {
Path[] cacheFiles = context.getLocalCacheArchives();
String path=cacheFiles[0].toString();
BufferedReader bf=new BufferedReader(new FileReader(path));
String line=null;
while((line=bf.readLine())!=null){
String[] movies = line.split("::");
map.put(Integer.parseInt(movies[0].trim()),movies[1]+"\t"+movies[2]);
}
}
@Override
protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
String[] lines = value.toString().split("::");
Integer joinKey=Integer.parseInt(lines[0].trim() );
if(map.containsKey(joinKey)){
String res=map.get(joinKey)+lines[0]+"\t"+lines[2]+"\t"+lines[3];
mk.set(joinKey);
mv.set(res);
context.write(mk,mv);
}
}
}
2.Driver端设计
package com.mycat.mapd_movie_mapjoin;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
/**
* reduce端join的缺陷:-----适合大表和大表关联
* 1)数据倾斜---分区分布不均匀
* 2)因为reduce端采用的集合,数据量大的时候,可能会产生OOM
* 3)reducetask本身并行度不高,导致性能比较低----经验值是:DataNode数量*0.95
*/
public class RatingDriver {
public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException, URISyntaxException {
System.setProperty("HADOOP_USER_NAME","hadoop");
Configuration conf=new Configuration();
conf.set("fs.defaultFS","hdfs:/mkmg/");
Job job = Job.getInstance(conf);
job.setJarByClass(RatingDriver.class);
job.setMapperClass(RatingMapper.class);
job.setOutputKeyClass(IntWritable.class);
job.setOutputValueClass(Text.class);
job.setNumReduceTasks(0);
job.addCacheArchive(new URI("/movie/movies.dat"));// hdfs
FileInputFormat.addInputPath(job,new Path("/rate"));
FileSystem fs=FileSystem.get(conf);
Path out=new Path("/mapjoin_out");
if(fs.exists(out)){
fs.delete(out,true);
}
FileOutputFormat.setOutputPath(job,out);
job.waitForCompletion(true);
}
}