public class Item implements Comparable<Item>{
private Long count;
private String content;
public Item() {
}
public Item(Long count, String content) {
this.count = count;
this.content = content;
}
public Long getCount() {
return count;
}
public void setCount(Long count) {
this.count = count;
}
public String getContent() {
return content;
}
public void setContent(String content) {
this.content = content;
}
@Override
public int compareTo(Item o) {
return Long.compare(count, o.getCount());
}
}
public class TopNMapper extends Mapper<Object, Text, NullWritable, Text> {
private int N;
private PriorityQueue<Item> priorityQueue = new PriorityQueue<>();
private static Logger LOGGER = Logger.getLogger(TopNMapper.class);
@Override
protected void setup(Context context) throws IOException, InterruptedException {
this.N = context.getConfiguration().getInt("N", 5);
}
@Override
public void map(Object key, Text value, Context context) throws IOException, InterruptedException {
String[] words = value.toString().split("\t");
if (words.length < 2) {
return;
}
LOGGER.info("Debug: Map get text = [" + value.toString() + "]");
Long count = Long.parseLong(words[1]);
Item item = new Item(count, value.toString());
if (priorityQueue.size() < N || count > priorityQueue.peek().getCount()) {
priorityQueue.offer(item);
}
if (priorityQueue.size() > N) {
priorityQueue.poll();
}
}
@Override
public void cleanup(Context context) throws IOException, InterruptedException {
for (Item item : priorityQueue) {
LOGGER.info("Debug: Map write to context = [" + item.getContent().toString() + "]");
context.write(NullWritable.get(), new Text(item.getContent()));
}
}
public static void main(String[] args) {
}
}
public class TopNReducer extends Reducer<NullWritable, Text, NullWritable, Text> {
private int N;
private PriorityQueue<Item> priorityQueue = new PriorityQueue();
private static Logger LOGGER = Logger.getLogger(TopNReducer.class);
@Override
protected void setup(Context context) throws IOException, InterruptedException {
this.N = context.getConfiguration().getInt("N", 5);
}
@Override
protected void reduce(NullWritable key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
for (Text value : values) {
LOGGER.info("Debug: text = [" + value.toString() + "]");
String[] words = value.toString().split("\t");
Long count = Long.parseLong(words[1]);
Item item = new Item(count, value.toString());
if (priorityQueue.size() < N || count > priorityQueue.peek().getCount()) {
priorityQueue.offer(item);
}
if (priorityQueue.size() > N) {
priorityQueue.poll();
}
}
for (Item topN : priorityQueue) {
context.write(NullWritable.get(), new Text(topN.getContent()));
}
}
}
public class TopNDriver extends Configured implements Tool {
private static Logger LOGGER = Logger.getLogger(TopNDriver.class);
public int run(String[] args) throws Exception {
Job job = new Job(getConf());
job.setJarByClass(TopNDriver.class);
int N = Integer.parseInt(args[0]); // top N
job.getConfiguration().setInt("N", N);
job.setJobName("TopNDriver");
job.setMapperClass(TopNMapper.class);
job.setReducerClass(TopNReducer.class);
job.setNumReduceTasks(1);
// map() output (K,V)
job.setMapOutputKeyClass(NullWritable.class);
job.setMapOutputValueClass(Text.class);
// reduce() output (K,V)
job.setOutputKeyClass(NullWritable.class);
job.setOutputValueClass(Text.class);
// args[1] = input directory
// args[2] = output directory
FileInputFormat.setInputPaths(job, new Path(args[1]));
FileOutputFormat.setOutputPath(job, new Path(args[2]));
boolean status = job.waitForCompletion(true);
LOGGER.info("run(): status="+status);
return status ? 0 : 1;
}
public static void main(String[] args) throws Exception {
if (args.length != 3) {
LOGGER.warn("usage TopNDriver <N> <input> <output>");
System.exit(1);
}
LOGGER.info("N=" + args[0]);
LOGGER.info("inputDir=" + args[1]);
LOGGER.info("outputDir=" + args[2]);
int returnStatus = ToolRunner.run(new TopNDriver(), args);
System.exit(returnStatus);
}
}