MapReduce/Hadoop的TopN解决方案之键不唯一的情况

一、MapReduce/Hadoop的TopN解决方案之键唯一的情况（点击打开链接）

二、针对键不唯一的情况，即文件中可能出现多次关键字

解决办法：先讲不唯一键转换为唯一键，即使用MapReduce合并键相同的项，再使用（一）所述的唯一键TopN方案

即

package topN_hadoop1;

import java.io.IOException;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.mapreduce.Mapper;

public class AggregateByKeyMapper extends
         Mapper<Object, Text, Text, IntWritable> {
   private Text K2 = new Text();
   private IntWritable V2 = new IntWritable();

   @Override
   public void map(Object key, Text value, Context context)
         throws IOException, InterruptedException {
      String valueAsString = value.toString().trim();
      String[] tokens = valueAsString.split(",");
      if (tokens.length != 2) {
         return;
      }
      String url = tokens[0];
      int frequency =  Integer.parseInt(tokens[1]);
      K2.set(url);
      V2.set(frequency);
      context.write(K2, V2);
   }
}

package topN_hadoop1;

import java.io.IOException;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.mapreduce.Reducer;

public class AggregateByKeyReducer  extends
    Reducer<Text, IntWritable, Text, IntWritable> {
      @Override
      public void reduce(Text key, Iterable<IntWritable> values, Context context)  throws IOException, InterruptedException {
         int sum = 0;
         for (IntWritable value : values) {
               sum += value.get();
         }
         context.write(key, new IntWritable(sum));
      }
}

package topN_hadoop1;

import org.apache.log4j.Logger;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

public class AggregateByKeyDriver  extends Configured implements Tool {

   private static Logger THE_LOGGER = Logger.getLogger(AggregateByKeyDriver.class);

   public int run(String[] args) throws Exception {
      Job job = new Job(getConf());
      HadoopUtil.addJarsToDistributedCache(job, "/lib/");
      job.setJobName("AggregateByKeyDriver");

      job.setInputFormatClass(TextInputFormat.class);
      job.setOutputFormatClass(SequenceFileOutputFormat.class);

      job.setOutputKeyClass(Text.class);
      job.setOutputValueClass(IntWritable.class);

      job.setMapperClass(AggregateByKeyMapper.class);
      job.setReducerClass(AggregateByKeyReducer.class);
      job.setCombinerClass(AggregateByKeyReducer.class);

       // args[0] = input directory
       // args[1] = output directory
      FileInputFormat.setInputPaths(job, new Path(args[0]));
      FileOutputFormat.setOutputPath(job, new Path(args[1]));

      boolean status = job.waitForCompletion(true);
      THE_LOGGER.info("run(): status="+status);
      return status ? 0 : 1;
   }

   /**
   * The main driver for "Aggregate By Key" program.
   * Invoke this method to submit the map/reduce job.
   * @throws Exception When there is communication problems with the job tracker.
   */
   public static void main(String[] args) throws Exception {
      // Make sure there are exactly 2 parameters
      if (args.length != 2) {
         THE_LOGGER.warn("usage AggregateByKeyDriver <input> <output>");
         System.exit(1);
      }

      THE_LOGGER.info("inputDir="+args[0]);
      THE_LOGGER.info("outputDir="+args[1]);
      int returnStatus = ToolRunner.run(new AggregateByKeyDriver(), args);
      System.exit(returnStatus);
   }

}