public class WordCount {
public static class WordCountMap extends Mapper<LongWritable, Text, Text, IntWritable> {
@Override
protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
String[] words = value.toString().split(" ");
for (String word : words) {
// 把这个单词转换成键值对
// 键:单词,值:1
// context.write方法,会把键值对包装好写出,给到reduce端
context.write(new Text(word), new IntWritable(1));
}
}
}
public static class WordCountReduce extends Reducer<Text, IntWritable, Text, IntWritable>{
PriorityQueue<Item> queue =new PriorityQueue<Item>();
@Override
protected void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException {
int sum = 0;
for (IntWritable value : values) {
sum += value.get();
}
Item item = new Item(key.toString(), (long) sum);
if (queue.size()<=10 || queue.peek().getCount()<item.getCount()){
queue.offer(item);
}
if (queue.size()>10){
queue.poll();
}
}
@Override
protected void cleanup(Context context) throws IOException, InterruptedException {
for (Item item :queue ) {
context.write(new Text(item.getWord()),new IntWritable(Math.toIntExact(item.getCount())));
}
}
}
public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
Configuration conf = new Configuration();
Job job = Job.getInstance(conf, "word count");
job.setJarByClass(WordCount.class);
job.setMapperClass(WordCountMap.class);
job.setReducerClass(WordCountReduce.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(IntWritable.class);
FileInputFormat.addInputPath(job, new Path("input"));
FileOutputFormat.setOutputPath(job, new Path("output"));
System.exit(job.waitForCompletion(true) ? 0 : 1);
}
}
public class Item implements Serializable,Comparable<Item> {
private String word;
private Long count;
public Item(String word, Long count) {
this.word = word;
this.count = count;
}
public String getWord() {
return word;
}
public void setWord(String word) {
this.word = word;
}
public Long getCount() {
return count;
}
public void setCount(Long count) {
this.count = count;
}
@Override
public int compareTo(Item o) {
return Long.compare(count,o.getCount());
}
}