hadoop等待提交过程的源码分析

步骤 程序层数 具体描述
1 1 job.waitForCompletion(true);
    字面意思等待执行完毕,其实在里面跑,要花时间
    先连接,再提交
    
2 2 job.waitForCompletion里
2 2 if (state == JobState.DEFINE) 确保状态默认
3 2 submit() 

3 3 submit里
    Submit的作用Submit the job to the cluster and return immediately.
    submit里创建了一个JobSubmitter类型的submitter来submitter.submitJobInternal(Job.this, cluster);
4 3 ensureState(JobState.DEFINE);
5 3 setUseNewAPI();
6 3 connect();

6 4 connect里
    获得了一个Cluster
    Cluseter里有属性:
    clientProtocolProvider = provider;
    client = clientProtocol;
    clientProtocol被赋值选择的Runner
7 4 cluster = return new Cluster(getConfiguration());
7 5 getConfiguration()后
    进入public Cluster(Configuration conf)
    
8 5 Cluster里
8 5 this(null, conf);调用本类另一个构造器
8 5 public Cluster(InetSocketAddress jobTrackAddr, Configuration conf) 
其中的一个属性有volatile 关键字,保证providerList存在内存而不cpu寄存器里
8 5 private volatile List<ClientProtocolProvider> providerList = null; 
8 5 赋值完conf和ugi后进入initialize(jobTrackAddr, conf);

9 6 initialize里
9 6 initProviderList()

10 7 initProviderList()里
     同步加锁的运行
     synchronized (frameworkLoader) 
     把框架默认提供的YarnClientProtocolProvider和LocalClientProtocolProvider全部拿出
     static Iterable<ClientProtocolProvider> frameworkLoader = ServiceLoader.load(ClientProtocolProvider.class);
10 7 for (ClientProtocolProvider provider : frameworkLoader) {  localProviderList.add(provider);} 
     providerList = localProviderList;
     
11 6 出到initialize里
11 6 for (ClientProtocolProvider provider : providerList)
     如果jobTrackAddr == null,就provider.create(conf)获取clientProtocol.
     下面判断是创建Yarn还是Local的ClientProtocol,即框架默认的两个"业务员"YARNRunner 和LocalJobRunner
11 6 if (jobTrackAddr == null) {  clientProtocol = provider.create(conf);
     根据不同的provider{LocalClientProtocolProvider或YarnClientProtocolProvider} ,
     create了不同的clientProtocol 如果不是Yarn就创建了Local.
     YARN的话,clientProtocol 获得一个YARNRunner(conf) ,
     Local的话,clientProtocol 获得一个LocalJobRunner(conf).
11 6 结束initialize

12 5 返回Cluster里
     最终配置了该cluster的conf,ugi,clientProtocol 
     this.conf = conf;
     this.ugi = UserGroupInformation.getCurrentUser();
     initialize(jobTrackAddr, conf);
12 5 结束Cluster

13 4 返回到connect,把获取的Cluster对象赋给Job的cluster属性
13 4 connect结束

14 3 返回到submit
14 3 创建JobSubmitter submitter 
     创建方式:final JobSubmitter submitter = getJobSubmitter(cluster.getFileSystem(), cluster.getClient());
     通过该job的属性cluster的方法cluster.getClient(),
     把原来在Cluser里获得的选择的Runner传给JobSubmitter
  
15 3 使用JobSubmitter
15 3 submitter.submitJobInternal(Job.this, cluster);

16 4 JobSubmitter内部的submitJobInternal方法
16 4 checkSpecs(job)用于校验输出路径,确认有路径

17 5 checkSpecs方法里
17 5 output.checkOutputSpecs(job);

18 6 checkOutputSpecs方法里
     在这里,提交job前,确认输出路径不存在
     // Ensure that the output directory is set and not already there
     Path outDir = getOutputPath(job);

19 5 返回checkSpecs方法

20 4 返回JobSubmitter内部的submitJobInternal方法
20 4 Path jobStagingArea = JobSubmissionFiles.getStagingDir(cluster, conf);
     //获取Job临时工作目录..../.staging

21 4 JobID jobId = submitClient.getNewJobID();
     //获取提交的job的jobId
     
22 4 Path submitJobDir = new Path(jobStagingArea, jobId.toString());
     //通过上面的工作目录和jobid生成Job提交路径

23 4 copyAndConfigureFiles(job, submitJobDir); 
     //拷贝Job相关的配置信息，并将job的提交路径在磁盘中创建出来

24 5 copyAndConfigureFiles里
24 5 rUploader.uploadResources(job, jobSubmitDir);//上传jar包,本地模式没有

25 6 uploadResources方法里
25 6 uploadResourcesInternal(job, submitJobDir);

26 7 uploadResourcesInternal方法里
     // Create the submission directory for the MapReduce job.
     submitJobDir = jtFs.makeQualified(submitJobDir);
     submitJobDir = new Path(submitJobDir.toUri().getPath());
     FsPermission mapredSysPerms = new FsPermission(JobSubmissionFiles.JOB_DIR_PERMISSION);
     mkdirs(jtFs, submitJobDir, mapredSysPerms);
     uploadFiles(job, files, submitJobDir, mapredSysPerms, replication,
    fileSCUploadPolicies, statCache);
     uploadLibJars(job, libjars, submitJobDir, mapredSysPerms, replication,
    fileSCUploadPolicies, statCache);
     uploadArchives(job, archives, submitJobDir, mapredSysPerms, replication,
    archiveSCUploadPolicies, statCache);
     uploadJobJar(job, jobJar, submitJobDir, replication, statCache);
     addLog4jToDistributedCache(job, submitJobDir);

27 6 返回uploadResources

28 5 返回copyAndConfigureFiles

29 4 返回submitJobInternal

30 4 Path submitJobFile = JobSubmissionFiles.getJobConfPath(submitJobDir);
     获取xml文件路径
     
31 4 int maps = writeSplits(job, submitJobDir);
     //生成切片信息

32 5 进入writeSplits
32 5 maps = writeNewSplits(job, jobSubmitDir);

33 6 进入writeNewSplits
33 6 InputFormat<?, ?> input = ReflectionUtils.newInstance(job.getInputFormatClass(), conf);
     用反射拿取InputFormat抽象类的实例,如果用户没有设置的,用默认的TextInputFormat
     InputFormat---->FileInputFormat---->TextInputFormat
     
34 6 List<InputSplit> splits = input.getSplits(job); 

35 7 进入getSplits方法
36 7 long minSize = Math.max(getFormatMinSplitSize(), getMinSplitSize(job));
     获取切片最小值,默认1或者自己的设定值,如果设定值大于block的大小,以设定值为准
37 7 long maxSize = getMaxSplitSize(job);
     获取切片最小值,默认Long.MAX_VALUE或者自己的设定值,如果设定值小于block的大小,以设定值为准
38 7 List<InputSplit> splits = new ArrayList<InputSplit>();
     创建切片列表   
39 7 for (FileStatus file: files) 
     按文件来选择,里面先判断可不可切

40 7 如果可切
    if (isSplitable(job, path)) {
      long blockSize = file.getBlockSize();//获取块大小
      long splitSize = computeSplitSize(blockSize, minSize, maxSize);//计算切片大小
      
41 8 进入computeSplitSize
    protected long computeSplitSize(long blockSize, long minSize,long maxSize) {
        return Math.max(minSize, Math.min(maxSize, blockSize));
    } 

42 7 返回if (isSplitable(job, path)) 
      long bytesRemaining = length;//文件剩余大小
      while (((double) bytesRemaining)/splitSize > SPLIT_SLOP) {//SPLIT_SLOP为1.1,避免最后一片太小.注:片和block不是一个概念
        int blkIndex = getBlockIndex(blkLocations, length-bytesRemaining);
        splits.add(makeSplit(path, 
        length-bytesRemaining, splitSize,
        //每次从length - bytesRemaining的位置开始,偏移量为上面计算的splitSize
                    blkLocations[blkIndex].getHosts(),//块所在的机器
                    blkLocations[blkIndex].getCachedHosts()//块的其他副本信息
                    ));
        bytesRemaining -= splitSize;
      }    
      if (bytesRemaining != 0) {//上面的循环跑完后,把不满足循环条件的剩余文件切成一片
        int blkIndex = getBlockIndex(blkLocations, length-bytesRemaining);
        splits.add(makeSplit(path, length-bytesRemaining, bytesRemaining,
                   blkLocations[blkIndex].getHosts(),
                   blkLocations[blkIndex].getCachedHosts()));
      }
    }

43 6 返回writeNewSplits
     获得splits列表,每个split记录读取的是哪个文件，从文件的一个位置开始读取到另一个位置
     如:file:/D:/programs/java/idealProjects/mr/input/sum/in.txt:0+107

44 6 JobSplitWriter.createSplitFiles(jobSubmitDir, conf, jobSubmitDir.getFileSystem(conf), array);
     在磁盘上创建job.split文件,job.splitmetainfo文件,以及他们的crc文件
     
45 5 返回writeSplits
     列表长度,return array.length;获取的值赋给maps
     
46 4 返回submitJobInternal
     return maps;获取的值赋给int maps

47 4 conf.setInt(MRJobConfig.NUM_MAPS, maps);//根据切片数量设置开启的MapTask数量
     
48 4 writeConf(conf, submitJobFile);// Write job file to submit dir job.xml
     //写配置信息,job.xml和它的crc文件

49 4 最后正式提交,如果是Local的submitClient,就是LocalJobRunner
     status = submitClient.submitJob(jobId, submitJobDir.toString(), job.getCredentials());
     JobSubmitter类定义私有属性:private ClientProtocol submitClient;
     getCredentials() Get the reported username for this job.
     
50 5 进入submitJob
     submitJob里new了一个有参Job
     job里的this.start();正式开启线程运行

51 2 monitorAndPrintJob();--运行,对当前的job进行监控，并打印job的信息,会删除上面创建的临时文件