9、wordCount本地模式源码分析

最新推荐文章于 2024-04-19 23:59:11 发布

gdgylpc

最新推荐文章于 2024-04-19 23:59:11 发布

阅读量71

点赞数

分类专栏： Hadoop 文章标签： java hadoop 大数据 spark mapreduce

本文链接：https://blog.csdn.net/resilienter/article/details/103696533

版权

Hadoop 专栏收录该内容

25 篇文章 1 订阅

订阅专栏

wordcount本地运行模式源码分析

提交流程

    // 判断state状态，如果在运行wait之后进行conf的设置时会出现running的错误
  if (state == JobState.DEFINE) {
      submit();
    }

@SuppressWarnings("unchecked")
    public void init(MapOutputCollector.Context context
                    ) throws IOException, ClassNotFoundException {
// 总的分区数取决于reduceTask的数量
      partitions = job.getNumReduceTasks();
      rfs = ((LocalFileSystem)FileSystem.getLocal(job)).getRaw();

      //sanity checks
// 获取溢写的百分比，读取mapreduce.map.sort.spill.percent，如果没有配置，默认0.8
      final float spillper =
        job.getFloat(JobContext.MAP_SORT_SPILL_PERCENT, (float)0.8);
     // 缓冲区的初始值，读取mapreduce.task.io.sort.mb，默认100M
 final int sortmb = job.getInt(JobContext.IO_SORT_MB, 100);
      indexCacheMemoryLimit = job.getInt(JobContext.INDEX_CACHE_MEMORY_LIMIT,
                                         INDEX_CACHE_MEMORY_LIMIT_DEFAULT);
      if (spillper > (float)1.0 || spillper <= (float)0.0) {
        throw new IOException("Invalid \"" + JobContext.MAP_SORT_SPILL_PERCENT +
            "\": " + spillper);
      }
      if ((sortmb & 0x7FF) != sortmb) {
        throw new IOException(
            "Invalid \"" + JobContext.IO_SORT_MB + "\": " + sortmb);
      }
// 指定溢写前的排序方式，使用快速排序，只排索引
      sorter = ReflectionUtils.newInstance(job.getClass("map.sort.class",
            QuickSort.class, IndexedSorter.class), job);

     // 获取输出key的比较器，影响排序的结果
comparator = job.getOutputKeyComparator();

      // k/v serialization
          // 涉及Key-value序列化，Hadoop可以为实现了Wriable接口的Key-value提供序列化器
      keyClass = (Class<K>)job.getMapOutputKeyClass();
      valClass = (Class<V>)job.getMapOutputValueClass();
      serializationFactory = new SerializationFactory(job);
      keySerializer = serializationFactory.getSerializer(keyClass);
      keySerializer.open(bb);
      valSerializer = serializationFactory.getSerializer(valClass);
      valSerializer.open(bb);

      // output counters
      mapOutputByteCounter = reporter.getCounter(TaskCounter.MAP_OUTPUT_BYTES);
      mapOutputRecordCounter =
        reporter.getCounter(TaskCounter.MAP_OUTPUT_RECORDS);
      fileOutputByteCounter = reporter
          .getCounter(TaskCounter.MAP_OUTPUT_MATERIALIZED_BYTES);

      // compression
      // MapTask最终写文件的分区溢写数据时，可以采取压缩
      if (job.getCompressMapOutput()) {
        Class<? extends CompressionCodec> codecClass =
          job.getMapOutputCompressorClass(DefaultCodec.class);
        codec = ReflectionUtils.newInstance(codecClass, job);
      } else {
        codec = null;
      }

      // combiner
    // 合并器
      final Counters.Counter combineInputCounter =
        reporter.getCounter(TaskCounter.COMBINE_INPUT_RECORDS);
      combinerRunner = CombinerRunner.create(job, getTaskID(), 
                                             combineInputCounter,
                                             reporter, null);
      if (combinerRunner != null) {
        final Counters.Counter combineOutputCounter =
          reporter.getCounter(TaskCounter.COMBINE_OUTPUT_RECORDS);
        combineCollector= new CombineOutputCollector<K,V>(combineOutputCounter, reporter, job);
      } else {
        combineCollector = null;
      }
          }