利用MapReduce分析sogou500数据(搜索过仙剑奇侠传的用户)

import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.HashMap;
import java.util.Map;


import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.filecache.DistributedCache;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;


public class Mapjoin {
public static class MapjoinMapper extends Mapper<Object, Text, Text, Text>{
Map<String, Integer> map = new HashMap<>();
Text uid = new Text();
protected void setup(Context context) throws IOException{
Path[] caches = DistributedCache.getLocalCacheFiles(context.getConfiguration());
System.out.println("-----"+caches);
for (Path cache : caches) {
String file = cache.toUri().toString();
if (file.indexOf("uuid") >= 0) {
BufferedReader br = new BufferedReader(new FileReader(file));
String line = null;
while(null != (line = br.readLine())){
map.put(line.trim(), 1);
}
}
}
System.out.println("===="+map);
}
@Override
protected void map(Object key, Text value, Mapper<Object, Text, Text, Text>.Context context)
throws IOException, InterruptedException {
// TODO Auto-generated method stub
String sogouLine = value.toString();
String[] arr = sogouLine.split("\t");
if (null != arr[1] && null !=map.get(arr[1]) && !arr[2].contains("仙剑奇侠传")) {
uid.set(arr[1]);
context.write(new Text(arr[1] + "-" +arr[2]), uid);
}
}
}
public static void main(String[] args) throws IllegalArgumentException, IOException, URISyntaxException, ClassNotFoundException, InterruptedException {

Job job = Job.getInstance(new Configuration());
job.setJarByClass(MapjoinMapper.class);
job.setMapperClass(MapjoinMapper.class);
job.setNumReduceTasks(0);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);
Path cache = new Path(args[0]);
String cacheLink = cache.toUri().toString()+"#uuid";
DistributedCache.addCacheFile(new URI(cacheLink), job.getConfiguration());
FileInputFormat.addInputPath(job, new Path(args[1]));
FileOutputFormat.setOutputPath(job, new Path(args[2]));
job.waitForCompletion(true);
}


}