思路1:
map阶段——切字段,抽取域名作为key,url作为value,返回即可
reduce阶段——用迭代器,将一个域名的一组url迭代出来,挨个放入一个hashmap中进行计数,最后从这个hashmap中挑出次数最多的3个url作为结果返回
思路2:
map阶段:解析数据,将域名作为key,1作为value
reduce阶段:
reduce方法中——对一个域名的一组1累加,然后将 <域名,总次数>放入一个成员变量Treemap中
cleanup方法中——从treemap中挑出次数最高的n个域名作为结果输出
PageTopnMapper:
import java.io.IOException;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
public class PageTopnMapper extends Mapper<LongWritable, Text, Text, IntWritable>{
@Override
protected void map(LongWritable key, Text value, Mapper<LongWritable, Text, Text, IntWritable>.Context context)
throws IOException, InterruptedException {
String line = value.toString();
String[] split = line.split(" ");
context.write(new Text(split[1]), new IntWritable(1));
}
}
PageTopnReducer:
import java.io.IOException;
import java.util.Map.Entry;
import java.util.Set;
import java.util.TreeMap;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;
public class PageTopnReducer extends Reducer<Text, IntWritable, Text, IntWritable>{
TreeMap<PageCount, Object> treeMap = new TreeMap<>();
@Override
protected void reduce(Text key, Iterable<IntWritable> values,
Reducer<Text, IntWritable, Text, IntWritable>.Context context) throws IOException, InterruptedException {
int count = 0;
for (IntWritable value : values) {
count += value.get();
}
PageCount pageCount = new PageCount();
pageCount.set(key.toString(), count);
treeMap.put(pageCount,null);
}
@Override
protected void cleanup(Context context)
throws IOException, InterruptedException {
Configuration conf = context.getConfiguration();
int topn = conf.getInt("top.n", 5);
Set<Entry<PageCount, Object>> entrySet = treeMap.entrySet();
int i= 0;
for (Entry<PageCount, Object> entry : entrySet) {
context.write(new Text(entry.getKey().getPage()), new IntWritable(entry.getKey().getCount()));
i++;
if(i==topn) return;
}
}
}
JobSubmitter:
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
public class JobSubmitter {
public static void main(String[] args) throws Exception {
/**
* 通过加载classpath下的*-site.xml文件解析参数
*/
Configuration conf = new Configuration();
conf.addResource("xx-oo.xml");
/**
* 通过代码设置参数
*/
//conf.setInt("top.n", 3);
//conf.setInt("top.n", Integer.parseInt(args[0]));
/**
* 通过属性配置文件获取参数
*/
/*Properties props = new Properties();
props.load(JobSubmitter.class.getClassLoader().getResourceAsStream("topn.properties"));
conf.setInt("top.n", Integer.parseInt(props.getProperty("top.n")));*/
Job job = Job.getInstance(conf);
job.setJarByClass(JobSubmitter.class);
job.setMapperClass(PageTopnMapper.class);
job.setReducerClass(PageTopnReducer.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(IntWritable.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(IntWritable.class);
FileInputFormat.setInputPaths(job, new Path("F:\\mrdata\\url\\input"));
FileOutputFormat.setOutputPath(job, new Path("F:\\mrdata\\url\\output"));
job.waitForCompletion(true);
}
}
ProvincePartitioner:
import java.util.HashMap;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Partitioner;
/**
* 本类是提供给MapTask用的
* MapTask通过这个类的getPartition方法,来计算它所产生的每一对kv数据该分发给哪一个reduce task
* @author ThinkPad
*
*/
public class ProvincePartitioner extends Partitioner<Text, FlowBean>{
static HashMap<String,Integer> codeMap = new HashMap<>();
static{
codeMap.put("135", 0);
codeMap.put("136", 1);
codeMap.put("137", 2);
codeMap.put("138", 3);
codeMap.put("139", 4);
}
@Override
public int getPartition(Text key, FlowBean value, int numPartitions) {
Integer code = codeMap.get(key.toString().substring(0, 3));
return code==null?5:code;
}
}
JobSubmitter增加代码:
// 设置参数:maptask在做数据分区时,用哪个分区逻辑类 (如果不指定,它会用默认的HashPartitioner)
job.setPartitionerClass(ProvincePartitioner.class);
// 由于我们的ProvincePartitioner可能会产生6种分区号,所以,需要有6个reduce task来接收
job.setNumReduceTasks(6);