【Hadoop篇】YARN源码分析（二）

0、回顾【Hadoop篇】YARN源码分析（一）

NodeManager启动了MRAppMaster（MRAppMaster就是ApplicationMaster的一种实现）

1、MRAppMaster统一管理这个Job的所有Task

1.MRAppMaster启动RMContainerAllocator服务
2.MRAppMaster注册、初始化、启动JobImpl
3.JobImpl创建MapTask和ReduceTask，并提交给TaskImpl
4.TaskImpl将Task提交给TaskAttemptImpl,TaskAttemptImpl提交给RMContainerAllocator处理

public class MRAppMaster extends CompositeService {
    public static void main(String[] args) {
        MRAppMaster appMaster = new MRAppMaster(applicationAttemptId, containerId, nodeHostString, Integer.parseInt(nodePortString), Integer.parseInt(nodeHttpPortString), appSubmitTime);
        initAndStartAppMaster(appMaster, conf, jobUserName);
        ->appMaster.init(conf);
        ->appMaster.start();
          ->serviceStart();
    }
    
    protected void serviceStart() throws Exception {
        job = createJob(getConfig(), forcedState, shutDownMessage);
        ->Job newJob = new JobImpl(jobId, appAttemptID, conf, dispatcher.getEventHandler(),taskAttemptListener, jobTokenSecretManager, jobCredentials, clock,completedTasksFromPreviousRun, metrics,committer, newApiCommitter,currentUser.getUserName(), appSubmitTime, amInfos, context, forcedState, diagnostic);
        ->dispatcher.register(JobFinishEvent.Type.class, createJobFinishEventHandler());   
        
        JobEvent initJobEvent = new JobEvent(job.getID(), JobEventType.JOB_INIT);
        jobEventDispatcher.handle(initJobEvent);
        ->TaskSplitMetaInfo[] taskSplitMetaInfo = createSplits(job, job.jobId);  --【JobImpl】
        ->job.numMapTasks = taskSplitMetaInfo.length;  --【JobImpl】
        ->job.numReduceTasks = job.conf.getInt(MRJobConfig.NUM_REDUCES, 0);  --【JobImpl】
        ->createMapTasks(job, inputLength, taskSplitMetaInfo); //创建MapTask
          ->for (int i=0; i < job.numMapTasks; ++i)
            ->TaskImpl task = new MapTaskImpl(job.jobId, i,...)  --【JobImpl】
            ->job.addTask(task);  --【JobImpl】
              ->tasks.put(task.getID(), task);      --【JobImpl】
        ->createReduceTasks(job);  //创建ReduceTask
        
        startJobs();
        ->JobEvent startJobEvent = new JobStartEvent(job.getID(), recoveredJobStartTime);
          ->super(jobID, JobEventType.JOB_START);
        ->dispatcher.getEventHandler().handle(startJobEvent);
          ->job.eventHandler.handle(new CommitterJobSetupEvent(job.jobId, job.jobContext));  --【JobImpl.StartTransition】
            ->super(CommitterEventType.JOB_SETUP);
            ->handleJobSetup((CommitterJobSetupEvent) event);  --【CommitterEventHandler.EventProcessor】
              ->context.getEventHandler().handle(new JobSetupCompletedEvent(event.getJobID()));  --【CommitterEventHandler.EventProcessor】
                ->super(jobID, JobEventType.JOB_SETUP_COMPLETED);
                //向TaskImpl提交MapTask
                ->job.scheduleTasks(job.mapTasks, job.numReduceTasks == 0);   --【JobImpl.SetupCompletedTransition】
                  ->for (TaskId taskID : taskIDs) 
                    ->eventHandler.handle(new TaskEvent(taskID, TaskEventType.T_SCHEDULE));
                //向TaskImpl提交ReduceTask    
                ->job.scheduleTasks(job.reduceTasks, true);    
    }
    
    private final class ContainerAllocatorRouter extends AbstractService implements ContainerAllocator, RMHeartbeatHandler {
        private final ClientService clientService;
        private final AppContext context;
        private ContainerAllocator containerAllocator;

        protected void serviceStart() throws Exception {
            this.containerAllocator = new RMContainerAllocator(this.clientService, this.context, preemptionPolicy);
            ((Service)this.containerAllocator).init(getConfig());
            ((Service)this.containerAllocator).start();
        }
        
        public void handle(ContainerAllocatorEvent event) {
             this.containerAllocator.handle(event);
        }
    }
}

public abstract class TaskImpl implements Task, EventHandler<TaskEvent> {
    .addTransition(TaskStateInternal.NEW, TaskStateInternal.SCHEDULED, TaskEventType.T_SCHEDULE, new InitialScheduleTransition())
    
    private static class InitialScheduleTransition implements SingleArcTransition<TaskImpl, TaskEvent> {
        public void transition(TaskImpl task, TaskEvent event) {
            task.addAndScheduleAttempt(Avataar.VIRGIN);
            ->addAndScheduleAttempt(avataar, false);
              ->eventHandler.handle(new TaskAttemptEvent(attempt.getID(), TaskAttemptEventType.TA_SCHEDULE));
        }
    }   
}

public abstract class TaskAttemptImpl implements org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt,EventHandler<TaskAttemptEvent> {
    .addTransition(TaskAttemptStateInternal.NEW, TaskAttemptStateInternal.UNASSIGNED,TaskAttemptEventType.TA_SCHEDULE, new RequestContainerTransition(false))
    
    static class RequestContainerTransition implements SingleArcTransition<TaskAttemptImpl, TaskAttemptEvent> {
        public void transition(TaskAttemptImpl taskAttempt, TaskAttemptEvent event) {
            taskAttempt.eventHandler.handle(new ContainerRequestEvent(taskAttempt.attemptId, taskAttempt.resourceCapability,taskAttempt.dataLocalHosts.toArray(new String[taskAttempt.dataLocalHosts.size()]),taskAttempt.dataLocalRacks.toArray(new String[taskAttempt.dataLocalRacks.size()])));
            ->super(attemptID, ContainerAllocator.EventType.CONTAINER_REQ);
            ->eventQueue.put(event);  --eventHandlingThread去取【RMContainerAllocator 】
        }
    }
}

2、RMContainerAllocator根据Task创建Container

1.RMContainerAllocator调用ApplicationMasterProtocol的registerApplicationMaster向ResourceManager注册ApplicationMaster
2.RMContainerAllocator不断heartbeat检查，通过ApplicationMasterProtocol的allocate向ResourceManager申请Container资源
3.如果Container资源申请成功，则运行处理的Task
4.如果MapTask全部处理完成，则开始执行pendingReduces中的ReduceTask

public class RMContainerAllocator extends RMContainerRequestor implements ContainerAllocator {
   protected void serviceStart() throws Exception {
       this.eventHandlingThread = new Thread() {
           public void run() {
               while (!stopped.get() && !Thread.currentThread().isInterrupted()) {
                   event = RMContainerAllocator.this.eventQueue.take();
                   handleEvent(event);
                   ->if (event.getType() == ContainerAllocator.EventType.CONTAINER_REQ) 
                     ->if (isMap) 
                      ->handleMapContainerRequest(reqEvent);
                        ->scheduledRequests.addMap(reqEvent); 
                    ->else
                      ->handleReduceContainerRequest(reqEvent);
                        ->pendingReduces.add(new ContainerRequest(reqEvent,PRIORITY_REDUCE, reduceNodeLabelExpression));  
               }
           }
       }
       this.eventHandlingThread.start();
       super.serviceStart();
       ->scheduler= createSchedulerProxy();    --【RMCommunicator】
       ->job = context.getJob(jobId);   --【RMCommunicator】
       ->register();   --【RMCommunicator】
         ->RegisterApplicationMasterResponse response = scheduler.registerApplicationMaster(request);
       ->startAllocatorThread();   --【RMCommunicator】
         ->allocatorThread = new Thread(new AllocatorRunnable());
         ->allocatorThread.start();
           ->public void run()  --【RMCommunicator.AllocatorRunnable】
             ->while (!stopped.get() && !Thread.currentThread().isInterrupted())
               ->heartbeat();
                 //获取到Container资源，则运行Containers
                 ->List<Container> allocatedContainers = getResources();   --【RMContainerAllocator】
                   ->response = makeRemoteRequest();
                     ->AllocateResponse allocateResponse = scheduler.allocate(allocateRequest);  --scheduler是ApplicationMasterProtocol
                   ->List<Container> newContainers = response.getAllocatedContainers(); 
                 ->if (allocatedContainers != null && allocatedContainers.size() > 0)
                   ->scheduledRequests.assign(allocatedContainers);  
                 ->scheduleReduces(getJob().getTotalMaps(), completedMaps,scheduledRequests.maps.size(), scheduledRequests.reduces.size(),assignedRequests.maps.size(), assignedRequests.reduces.size(),mapResourceRequest, reduceResourceRequest, pendingReduces.size(),maxReduceRampupLimit, reduceSlowStart);
                   //如果MapTask全部执行完毕，则执行ReduceTask
                   ->if (scheduledMaps == 0 && numPendingReduces > 0) 
                     ->scheduleAllReduces();
                       ->for (ContainerRequest req : pendingReduces)
                         ->scheduledRequests.addReduce(req);  
                           ->addContainerReq(req);
   }
}

3、创建脚本，将YarnChild设置为Container的启动类

1.通过ContainerManagementProtocol向NodeManager启动Container

public class RMContainerAllocator extends RMContainerRequestor implements ContainerAllocator {
   class ScheduledRequests {
       private void assign(List<Container> allocatedContainers) {
           assignContainers(allocatedContainers);
           ->Iterator<Container> it = allocatedContainers.iterator();
           ->while (it.hasNext()) 
             ->containerAssigned(allocated, assigned);
               ->eventHandler.handle(new TaskAttemptContainerAssignedEvent(assigned.attemptID, allocated, applicationACLs));
                  ->super(id, TaskAttemptEventType.TA_ASSIGNED); 
                 ->Container container = cEvent.getContainer();  --【TaskAttemptImpl.ContainerAssignedTransition】 
                 ->ContainerLaunchContext launchContext = createContainerLaunchContext(cEvent.getApplicationACLs(), taskAttempt.conf, taskAttempt.jobToken,taskAttempt.remoteTask, taskAttempt.oldJobId, taskAttempt.jvmID,taskAttempt.taskAttemptListener, taskAttempt.credentials);
                   ->List<String> commands = MapReduceChildJVM.getVMCommand(taskAttemptListener.getAddress(), remoteTask, jvmID);
                     ->vargs.add(MRApps.crossPlatformifyMREnv(task.conf, Environment.JAVA_HOME) + "/bin/java");
                     //设置YarnChild为启动类
                     ->vargs.add(YarnChild.class.getName());  // main of Child
                     ->StringBuilder mergedCommand = new StringBuilder();
                     ->for (CharSequence str : vargs)
                       ->mergedCommand.append(str).append(" ");
                     ->Vector<String> vargsFinal = new Vector<String>(1);
                     ->vargsFinal.add(mergedCommand.toString());
                     ->return vargsFinal;
                 ->taskAttempt.eventHandler.handle(new ContainerRemoteLaunchEvent(taskAttempt.attemptId, launchContext, container, taskAttempt.remoteTask));   --【TaskAttemptImpl.ContainerAssignedTransition】
                   ->Container c = getContainer(event);   --【ContainerLauncherImpl】
                   ->c.launch(launchEvent);    --【ContainerLauncherImpl】
                     ->StartContainersResponse response = proxy.getContainerManagementProtocol().startContainers(requestList);    --【ContainerLauncherImpl】
       }
   }
}

4、NodeManager通过ContainerManagementProtocol接收startContainers请求，执行脚本启动YarnChild，同【Hadoop篇】YARN源码分析（一）

public class ContainerManagerImpl extends CompositeService implements ContainerManager {
    public StartContainersResponse startContainers(StartContainersRequest requests) throws YarnException, IOException {
        ...
    }
}