MapReduce源码解读之ReduceTask

写在前面

MapTask在执行context.write()之后，将结果溢写到磁盘，然后触发Reduce Shuffle，具体的shuffle过程可以参考MapReduce shuffle过程详解，这篇文章讲的通俗易懂，这样Map端的任务就算基本完成了。接下来我们看Reducer端做了哪些工作。跟源码时，可以从Reducer类的run方法开始分析。

1.入口：Reducer类的run()方法

/**
*根据Reducer的doc说明，Reducer的任务主要分为3个阶段
*1.Shuffle(相同的key被拉取到相同的分区中)
*2.Sort(map端已经做好了排序工作，reduce拉取的每个partition的数据都是有序的，这样很适合做归并排序)
*3.reduce
*/
public void run(Context context) throws IOException, InterruptedException {
    setup(context);
    try {
      //这里我们发现，map端的run()方法是context.nextKeyValue()，reduce端是context.nextKey()
      //可以看出它们的区别是，map按记录数来判断是否还有数据，而reduce是按key分组来判断是否还有数据
      //也就是说map端一条记录执行一次map方法，而ruduce端一组相同的key执行一次reduece方法
      //要了解如何去判断的，需要进入ReduceTask的run()去看源码
      while (context.nextKey()) {
        reduce(context.getCurrentKey(), context.getValues(), context);
        // If a back up store is used, reset it
        Iterator<VALUEIN> iter = context.getValues().iterator();
        if(iter instanceof ReduceContext.ValueIterator) {
          ((ReduceContext.ValueIterator<VALUEIN>)iter).resetBackupStore();        
        }
      }
    } finally {
      cleanup(context);
    }
  }

2. 进入Reduece Task类的run()方法

  public void run(JobConf job, final TaskUmbilicalProtocol umbilical)
    throws IOException, InterruptedException, ClassNotFoundException {
    job.setBoolean(JobContext.SKIP_RECORDS, isSkipping());

    if (isMapOrReduce()) {
	  //从Map的输出文件中拷贝属于自己分区的数据
      copyPhase = getProgress().addPhase("copy");
      //分组排序
      sortPhase  = getProgress().addPhase("sort");
      //reduce计算
      reducePhase = getProgress().addPhase("reduce");
    }
    // start thread that will handle communication with parent
    TaskReporter reporter = startReporter(umbilical);
    
    boolean useNewApi = job.getUseNewReducer();
    initialize(job, getJobID(), reporter, useNewApi);

    // check if it is a cleanupJobTask
    if (jobCleanup) {
      runJobCleanupTask(umbilical, reporter);
      return;
    }
    if (jobSetup) {
      runJobSetupTask(umbilical, reporter);
      return;
    }
    if (taskCleanup) {
      runTaskCleanupTask(umbilical, reporter);
      return;
    }
    
    // Initialize the codec
    codec = initCodec();
    RawKeyValueIterator rIter = null;
    ShuffleConsumerPlugin shuffleConsumerPlugin = null;
    
    Class combinerClass = conf.getCombinerClass();
    CombineOutputCollector combineCollector = 
      (null != combinerClass) ? 
     new CombineOutputCollector(reduceCombineOutputCounter, reporter, conf) : null;

    Class<? extends ShuffleConsumerPlugin> clazz =
          job.getClass(MRConfig.SHUFFLE_CONSUMER_PLUGIN, Shuffle.class, ShuffleConsumerPlugin.class);
					
    shuffleConsumerPlugin = ReflectionUtils.newInstance(clazz, job);
    LOG.info("Using ShuffleConsumerPlugin: " + shuffleConsumerPlugin);

    ShuffleConsumerPlugin.Context shuffleContext = 
      new ShuffleConsumerPlugin.Context(getTaskID(), job, FileSystem.getLocal(job), umbilical, 
                  super.lDirAlloc, reporter, codec, 
                  combinerClass, combineCollector, 
                  spilledRecordsCounter, reduceCombineInputCounter,
                  shuffledMapsCounter,
                  reduceShuffleBytes, failedShuffleCounter,
                  mergedMapOutputsCounter,
                  taskStatus, copyPhase, sortPhase, this,
                  mapOutputFile, localMapFiles);
    shuffleConsumerPlugin.init(shuffleContext);
		
	//创建了一个迭代器，将数据从磁盘中一条一条地拉取
    rIter = shuffleConsumerPlugin.run();

    // free up the data structures
    mapOutputFilesOnDisk.clear();
     // 排序过程结束
    sortPhase.complete();                        
    setPhase(TaskStatus.Phase.REDUCE); 
    statusUpdate(umbilical);
    Class keyClass = job.getMapOutputKeyClass();
    Class valueClass = job.getMapOutputValueClass();
	
	//创建一个分组比较器
	//如果用户没有自定义这个分组比较器，那么就去取用户自定义的排序比较器，如果用户没有设置排序比较器，那么默认为getOutputKeyComparator()
    RawComparator comparator = job.getOutputValueGroupingComparator();

    if (useNewApi) {
    //将迭代器，比较器等传入runNewReducer()方法，进入这个方法
      runNewReducer(job, umbilical, reporter, rIter, comparator, 
                    keyClass, valueClass);
    } else {
      runOldReducer(job, umbilical, reporter, rIter, comparator, 
                    keyClass, valueClass);
    }

    shuffleConsumerPlugin.close();
    done(umbilical, reporter);
  }

3. 进入runNewReducer()方法

  private <INKEY,INVALUE,OUTKEY,OUTVALUE>
  void runNewReducer(JobConf job,
                     final TaskUmbilicalProtocol umbilical,
                     final TaskReporter reporter,
                     RawKeyValueIterator rIter,
                     RawComparator<INKEY> comparator,
                     Class<INKEY> keyClass,
                     Class<INVALUE> valueClass
                     ) throws IOException,InterruptedException, 
                              ClassNotFoundException {
    // 将传进来的迭代器包装成一个新的迭代器
    final RawKeyValueIterator rawIter = rIter;
    rIter = new RawKeyValueIterator() {
      public void close() throws IOException {
        rawIter.close();
      }
      public DataInputBuffer getKey() throws IOException {
        return rawIter.getKey();
      }
      public Progress getProgress() {
        return rawIter.getProgress();
      }
      public DataInputBuffer getValue() throws IOException {
        return rawIter.getValue();
      }
      public boolean next() throws IOException {
        boolean ret = rawIter.next();
        reporter.setProgress(rawIter.getProgress().getProgress());
        return ret;
      }
    };
    // 创建一个taskContext用于反射类
    org.apache.hadoop.mapreduce.TaskAttemptContext taskContext =
      new org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl(job,
          getTaskID(), reporter);
    // 通过反射的方式，拿到用户实现的Reducer类
    org.apache.hadoop.mapreduce.Reducer<INKEY,INVALUE,OUTKEY,OUTVALUE> reducer =
      (org.apache.hadoop.mapreduce.Reducer<INKEY,INVALUE,OUTKEY,OUTVALUE>)
        ReflectionUtils.newInstance(taskContext.getReducerClass(), job);
    org.apache.hadoop.mapreduce.RecordWriter<OUTKEY,OUTVALUE> trackedRW = 
      new NewTrackingRecordWriter<OUTKEY, OUTVALUE>(this, taskContext);
    job.setBoolean("mapred.skip.on", isSkipping());
    job.setBoolean(JobContext.SKIP_RECORDS, isSkipping());
    org.apache.hadoop.mapreduce.Reducer.Context 

		//创建一个reducerContext，之后会将它传给reducer.run()方法
         reducerContext = createReduceContext(reducer, job, getTaskID(),
                                               rIter, reduceInputKeyCounter, 
                                               reduceInputValueCounter, 
                                               trackedRW,
                                               committer,
                                               reporter, comparator, keyClass,
                                               valueClass);
    try {
	
	  //这里由用户实现的reducer执行run()方法，进入run()方法
      reducer.run(reducerContext);
    } finally {
      trackedRW.close(reducerContext);
    }
  }

4.进入run()方法

public void run(Context context) throws IOException, InterruptedException {
    setup(context);
    try {
      //这里又回到了第一步，现在我们开始进入nextKey()方法去看
      while (context.nextKey()) {
        reduce(context.getCurrentKey(), context.getValues(), context);
        // If a back up store is used, reset it
        Iterator<VALUEIN> iter = context.getValues().iterator();
        if(iter instanceof ReduceContext.ValueIterator) {
          ((ReduceContext.ValueIterator<VALUEIN>)iter).resetBackupStore();        
        }
      }
    } finally {
      cleanup(context);
    }
  }

5. 进入nextKey()方法

 /** Start processing next unique key. */
  public boolean nextKey() throws IOException,InterruptedException {
  //这里判断了nextkey是否与当前key相等
    while (hasMore && nextKeyIsSame) {
      nextKeyValue();
    }
    if (hasMore) {
      if (inputKeyCounter != null) {
        inputKeyCounter.increment(1);
      }
      //这里返回是否还有记录和值，注意map阶段也有这个方法，进入这个方法
      return nextKeyValue();
    } else {
      return false;
    }
  }

6.进入nextKeyValue()方法

/**
*这个方法的逻辑与map端的nextKeyValue()方法很相似，但是它多做了一件事：多取一个key
*/
public boolean nextKeyValue() throws IOException, InterruptedException {
    if (!hasMore) {
      key = null;
      value = null;
      return false;
    }
    firstValue = !nextKeyIsSame;
    
    //获取key
    DataInputBuffer nextKey = input.getKey();
    currentRawKey.set(nextKey.getData(), nextKey.getPosition(), 
                      nextKey.getLength() - nextKey.getPosition());
    buffer.reset(currentRawKey.getBytes(), 0, currentRawKey.getLength());
    key = keyDeserializer.deserialize(key);

	//获取value
    DataInputBuffer nextVal = input.getValue();
    buffer.reset(nextVal.getData(), nextVal.getPosition(), nextVal.getLength()
        - nextVal.getPosition());
    value = valueDeserializer.deserialize(value);

    currentKeyLength = nextKey.getLength() - nextKey.getPosition();
    currentValueLength = nextVal.getLength() - nextVal.getPosition();

    if (isMarked) {
      backupStore.write(nextKey, nextVal);
    }

	//这里多取了一个key
    hasMore = input.next();
    if (hasMore) {
      nextKey = input.getKey();

	  //这里调用了我们传进来的key比较器，如果两个key相等，就会返回true
	  //回到run()方法
      nextKeyIsSame = comparator.compare(currentRawKey.getBytes(), 0, 
                                     currentRawKey.getLength(),
                                     nextKey.getData(),
                                     nextKey.getPosition(),
                                     nextKey.getLength() - nextKey.getPosition()
                                         ) == 0;
    } else {
      nextKeyIsSame = false;
    }
    inputValueCounter.increment(1);
    return true;
  }

7.回到run()方法

public void run(Context context) throws IOException, InterruptedException {
    setup(context);
    try {
      while (context.nextKey()) {
		
		//获取当前key(就是直接return key)，进入getValues()方法
        reduce(context.getCurrentKey(), context.getValues(), context);
        // If a back up store is used, reset it
        Iterator<VALUEIN> iter = context.getValues().iterator();
        if(iter instanceof ReduceContext.ValueIterator) {
          ((ReduceContext.ValueIterator<VALUEIN>)iter).resetBackupStore();        
        }
      }
    } finally {
      cleanup(context);
    }
  }

8. 进入getValues()方法

  public 
  Iterable<VALUEIN> getValues() throws IOException, InterruptedException {
  //这里返回了一个Iterable对象，里面封装了一个Iterator()方法
  //这个Iterator()方法会返回一个ValueIterator迭代器，进入ValueIterator类
    return iterable;
  }

9. 进入ValueIterator类

/**
*可以看到，这个ValueIterator实现了hasNext()和next()方法
*/
  protected class ValueIterator implements ReduceContext.ValueIterator<VALUEIN> {

    private boolean inReset = false;
    private boolean clearMarkFlag = false;

    @Override
    public boolean hasNext() {
      try {
        if (inReset && backupStore.hasNext()) {
          return true;
        } 
      } catch (Exception e) {
        e.printStackTrace();
        throw new RuntimeException("hasNext failed", e);
      }
      return firstValue || nextKeyIsSame;
    }

    @Override
    public VALUEIN next() {
      if (inReset) {
        try {
          if (backupStore.hasNext()) {
            backupStore.next();
            DataInputBuffer next = backupStore.nextValue();
            buffer.reset(next.getData(), next.getPosition(), next.getLength()
                - next.getPosition());
            value = valueDeserializer.deserialize(value);
            return value;
          } else {
            inReset = false;
            backupStore.exitResetMode();
            if (clearMarkFlag) {
              clearMarkFlag = false;
              isMarked = false;
            }
          }
        } catch (IOException e) {
          e.printStackTrace();
          throw new RuntimeException("next value iterator failed", e);
        }
      } 
      // if this is the first record, we don't need to advance
      if (firstValue) {
        firstValue = false;
        return value;
      }
      // if this isn't the first record and the next key is different, they
      // can't advance it here.
      if (!nextKeyIsSame) {
        throw new NoSuchElementException("iterate past last value");
      }
      // otherwise, go to the next key/value pair
      try {

		//最终会调用nextKeyValue()
		//到这里reduce的nextKey()就分析完了，接下来就是执行用户实现的reduce逻辑代码的环节了
        nextKeyValue();
        return value;
      } catch (IOException ie) {
        throw new RuntimeException("next value iterator failed", ie);
      } catch (InterruptedException ie) {
        // this is bad, but we can't modify the exception list of java.util
        throw new RuntimeException("next value iterator interrupted", ie);        
      }
    }

10. 总结

reduce阶段并没有直接将所有数据都一次拉取到内存，而是先进行shuffle和归并排序，将map的输出数据整理成分区有序且分组有序的状态，然后创建一个迭代器Iter，但是，如果将这个迭代器直接传给reduce，那么reduce会将所有分区的数据都读进来，所以又创建了一个新的迭代器iterable去包装Iter并调用Iter的nextKeyValue()方法，当调用context.getValues()方法时，hasNext()方法判断NextKeyIsSame，如果为真则调用next()方法，next()方法调用了Iter的nextKeyValue()方法，这个方法会更新NextKeyIsSame的值，每传一条数据，都会更新NextKeyIsSame，当两个key不相等时，NextKeyIsSame为false，reduce()方法结束，但是Iter还在，下一次调用reduce()方法时，nextKeyValue()方法会被新的iterable调用，这样就开始了新的一组数据的计算。reduce()方法充分应用了迭代器模式，一次I/O就完成了数据的计算过程，规避了OOM的问题，