wordcount例子程序

      hadoop-0.20.1里的wordcount源代码与0.19.2已经有较大的区别了。在新版本的hadoop中，org.apache.hadoop.mapred包被org.apache.hadoop.mapreduce所取代。不过为了保持兼容性，org.apache.hadoop.mapred还是存在于core里。

 

     ibm的技术文章（http://www.ibm.com/developerworks/cn/opensource/os-cn-hadoop2/index.htm）l写于2008年，使用的是hadoop-0.16.0，代码比较老了。实验室使用的是hadoop-0.19.2，所以我只好暂时放下hadoop-0.20.1，去研究hadoop-0.19.2的wordcount程序。

public class WordCount extends Configured implements Tool {
 
  /**
   * Counts the words in each line.
   * For each line of input, break the line into words and emit them as
   * (<b>word</b>, <b>1</b>).
   */
  public static class MapClass extends MapReduceBase
    implements Mapper<LongWritable, Text, Text, IntWritable> {
   
    private final static IntWritable one = new IntWritable(1);
    private Text word = new Text();
   
    public void map(LongWritable key, Text value,
                    OutputCollector<Text, IntWritable> output,
                    Reporter reporter) throws IOException {
      String line = value.toString();
      StringTokenizer itr = new StringTokenizer(line);
      while (itr.hasMoreTokens()) {
        word.set(itr.nextToken());
        output.collect(word, one);
      }
    }
  }
 
  /**
   * A reducer class that just emits the sum of the input values.
   */
  public static class Reduce extends MapReduceBase
    implements Reducer<Text, IntWritable, Text, IntWritable> {
   
    public void reduce(Text key, Iterator<IntWritable> values,
                       OutputCollector<Text, IntWritable> output,
                       Reporter reporter) throws IOException {
      int sum = 0;
      while (values.hasNext()) {
        sum += values.next().get();
      }
      output.collect(key, new IntWritable(sum));
    }
  }
 
  static int printUsage() {
    System.out.println("wordcount [-m <maps>] [-r <reduces>] <input> <output>");
    ToolRunner.printGenericCommandUsage(System.out);
    return -1;
  }
 
  /**
   * The main driver for word count map/reduce program.
   * Invoke this method to submit the map/reduce job.
   * @throws IOException When there is communication problems with the
   *                     job tracker.
   */
  public int run(String[] args) throws Exception {
    JobConf conf = new JobConf(getConf(), WordCount.class);
    conf.setJobName("wordcount");
 
    // the keys are words (strings)
    conf.setOutputKeyClass(Text.class);
    // the values are counts (ints)
    conf.setOutputValueClass(IntWritable.class);
   
    conf.setMapperClass(MapClass.class);       
    conf.setCombinerClass(Reduce.class);
    conf.setReducerClass(Reduce.class);
   

 

/******************这里是和hadoop-0.16.0里不同的部分***********************/

//提供了map task和reduce task个数的配置
    List<String> other_args = new ArrayList<String>();
    for(int i=0; i < args.length; ++i) {
      try {
        if ("-m".equals(args[i])) {
          conf.setNumMapTasks(Integer.parseInt(args[++i]));
        } else if ("-r".equals(args[i])) {
          conf.setNumReduceTasks(Integer.parseInt(args[++i]));
        } else {
          other_args.add(args[i]);
        }
      } catch (NumberFormatException except) {
        System.out.println("ERROR: Integer expected instead of " + args[i]);
        return printUsage();
      } catch (ArrayIndexOutOfBoundsException except) {
        System.out.println("ERROR: Required parameter missing from " +
                           args[i-1]);
        return printUsage();
      }
    }

    // Make sure there are exactly 2 parameters left.
    if (other_args.size() != 2) {
      System.out.println("ERROR: Wrong number of parameters: " +
                         other_args.size() + " instead of 2.");
      return printUsage();
    }
    FileInputFormat.setInputPaths(conf, other_args.get(0));
    FileOutputFormat.setOutputPath(conf, new Path(other_args.get(1)));
       
    JobClient.runJob(conf);
    return 0;
  }
 
 
  public static void main(String[] args) throws Exception {
    int res = ToolRunner.run(new Configuration(), new WordCount(), args);
    System.exit(res);
  }

}

 

 

     另外，如果要使用ibm mapreduce tools，不要再去ibm网站下载。因为ibm的plugin已经贡献给了hadoop,现在在$hadoop_home$/contrib/eclipse-plugin目录下。而且要确保hadoop eclipse-plugin的版本与hadoop server的版本一致，因为hadoop有自己的rpc远程调用框架，所以客户端的hadoop-core.jar必须与服务器一致.不然rpc协议有可能不兼容.