mapreduce中combiner执行情况

mapreduce中combiner执行情况

• combiner在mapreduce过程中可能在溢出和规并时执行：源码见类【MapTask】
• 溢出时combiner执行情况源码：OutputCollector => spill

private void sortAndSpill() throws IOException, ClassNotFoundException, InterruptedException {
// ...
//如果未提供combiner，就不会执行merge过程
if (this.combinerRunner == null) {
    for(DataInputBuffer key = new DataInputBuffer(); spindex < mend && this.kvmeta.get(this.offsetFor(spindex % this.maxRec) + 2) == i; ++spindex) {
        int kvoff = this.offsetFor(spindex % this.maxRec);
        int keystart = this.kvmeta.get(kvoff + 1);
        int valstart = this.kvmeta.get(kvoff + 0);
        key.reset(this.kvbuffer, keystart, valstart - keystart);
        this.getVBytesForOffset(kvoff, value);
        writer.append(key, value);
    }
} else {
//计算当前分区的开始（spstart）和结束（spindex）的位置
    int spstart;
    for(spstart = spindex; spindex < mend && this.kvmeta.get(this.offsetFor(spindex % this.maxRec) + 2) == i; ++spindex) {
        ;
    }
// 如果（spstart == spindex），只有一条溢出记录也不用执行merge过程
    if (spstart != spindex) {
        this.combineCollector.setWriter(writer);
        RawKeyValueIterator kvIter = new MapTask.MapOutputBuffer.MRResultIterator(spstart, spindex);
        this.combinerRunner.combine(kvIter, this.combineCollector);
    }
}

writer.close();
}

• 规并时combiner执行情况源码：partitioners => merge

private void mergeParts() throws IOException, InterruptedException, ClassNotFoundException{
// ...
//this.minSpillForCombine = this.job.getInt("mepreduce.map.combine.minspills",3);
//如果提供合并器，且当前溢出文件数量超过最小合并引出数量“3”
if (this.combinerRunner != null && this.numSpills >= this.minSpillsForCombine) {
    this.combineCollector.setWriter(writer);
    this.combinerRunner.combine(kvIter, this.combineCollector);
} else {
    Merger.writeFile(kvIter, writer, this.reporter, this.job);
}
writer.close();
// ...
}