Hadoop MapReduce改编WordCount(以“+”符号计数)

import java.io.IOException;

import java.util.StringTokenizer;

import java.util.List;

import java.util.ArrayList;



import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.fs.Path;

import org.apache.hadoop.io.IntWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.Job;

import org.apache.hadoop.mapreduce.Mapper;

import org.apache.hadoop.mapreduce.Reducer;

import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;

import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;



public class WordCount {



  public static class TokenizerMapper

       extends Mapper<Object, Text, Text, Text>{



    private final static Text one = new Text("+");

    private Text word = new Text();



    public void map(Object key, Text value, Context context

                    ) throws IOException, InterruptedException {

      StringTokenizer itr = new StringTokenizer(value.toString().toLowerCase().replace("."," ").replace(","," ").replace("\""," ").replace(";"," ")); //将所有字母变小写并进行微处理

      while (itr.hasMoreTokens()) {

        word.set(itr.nextToken());

        context.write(word, one);

      }

    }

  }



  public static class IntSumReducer

       extends Reducer<Text,Text,Text,Text> {

    private Text result = new Text();

    List<Text> textList = new ArrayList<Text>();  //用于储存values

    String strAdd;



    public void reduce(Text key, Iterable<Text> values,

                       Context context

                       ) throws IOException, InterruptedException {

      textList.clear();  

      strAdd="";



      for (Text val : values) {

	textList.add(new Text(val));   //需要将values的值取出用新对象存储，否则会出现问题，文章下面有说明

      }



      for(Text text : textList){

	strAdd+=text.toString();

      }

	

      result.set(strAdd);

      if(strAdd.length()>=3){       //计数大于等于3时输出

	context.write(key, result);

      }

      

    }

  }



  public static void main(String[] args) throws Exception {

    Configuration conf = new Configuration();

    Job job = Job.getInstance(conf, "word count");

    job.setJarByClass(WordCount.class);

    job.setMapperClass(TokenizerMapper.class);

    job.setCombinerClass(IntSumReducer.class);

    job.setReducerClass(IntSumReducer.class);

    job.setOutputKeyClass(Text.class);

    job.setOutputValueClass(Text.class);

    FileInputFormat.addInputPath(job, new Path(args[0]));

    FileOutputFormat.setOutputPath(job, new Path(args[1]));

    System.exit(job.waitForCompletion(true) ? 0 : 1);

  }

}

------------------

运行结果如下

------------------

源代码中reduce部分采用ArrayList来存储输入的相同key值所对应的所有value（即“+”）

如果直接采用

for(Text val : values){
	strAdd+=val.toString();
}

--------------

输出的结果会产生如下错误：

计数明显出现错误，不同key值的“+”号的数量出现大量重复 

原因疑是产生在reduce阶段Iterable的遍历中：对象重用（ objects reuse ） 

----------------------------- 

引用开源中国xrzs博主的观点（详情：https://my.oschina.net/leejun2005/blog/131744） 

reduce方法的javadoc中已经说明了会出现的问题：  

The framework calls this method for each <key, (list of values)> pair in the grouped inputs. Output values must be of the same type as input values. Input keys must not be altered. The framework will reuse the key and value objects that are passed into the reduce, therefore the application should clone the objects they want to keep a copy of. 

也就是说虽然reduce方法会反复执行多次，但key和value相关的对象只有两个，reduce会反复重用这两个对象。所以如果要保存key或者value的结果，只能将其中的值取出另存或者重新clone一个对象（例如Text store = new Text(value) 或者 String a = value.toString()），而不能直接赋引用。因为引用从始至终都是指向同一个对象，你如果直接保存它们，那最后它们都指向最后一个输入记录。会影响最终计算结果而出错。