Hadoop Map/Reduce执行全流程关键代码

1. Hadoop Map/Reduce 执行流程关键代码  
2.   
3. JobClient.runJob(conf) | 运行job  
4. |-->JobClient jc = new JobClient(job);  
5. |-->RunningJob rj = jc.submitJob(job);  
6.     |-->submitJobInternal(job);  
7.         |-->int reduces = job.getNumReduceTasks();  
8.         |-->JobContext context = new JobContext(job, jobId);  
9.         |-->maps = writeOldSplits(job, submitSplitFile);  
10.         |-->job.setNumMapTasks(maps);  
11.         |-->job.writeXml(out);  
12.         |-->JobStatus status = jobSubmitClient.submitJob(jobId); 
    
13. ----------------------------------------------------------------------------------------------
    

1.   
2. JobTracker.submitJob(JobId) |提交job  
3. |-->JobInProgress job = new JobInProgress(jobId, this, this.conf);  
4. |-->checkAccess(job, QueueManager.QueueOperation.SUBMIT_JOB);  |检查权限  
5. |-->checkMemoryRequirements(job);  |检查内存需求  
6. |-->addJob(jobId, job);  |添加至job队列  
7.     |-->jobs.put(job.getProfile().getJobID(), job);  
8.     |--> for (JobInProgressListener listener : jobInProgressListeners) ｜添加至监听器，供调度使用  
9.         |-->listener.jobAdded(job);  
10.  
    
11. ------------------------------------------------------------------------------------------------------
    
12. JobTracker.heartbeat()  |JobTracker启动后供TaskTracker以RPC方式来调用，返回Response集合  
13. |-->List<TaskTrackerAction> actions = new ArrayList<TaskTrackerAction>();  
14. |-->tasks = taskScheduler.assignTasks(taskTrackerStatus);  |通过调度器选择合适的tasks  
15. |-->for (Task task : tasks)  
16.     |-->expireLaunchingTasks.addNewTask(task.getTaskID());  
17.     |-->actions.add(new LaunchTaskAction(task));  |实际actions还会添加commmitTask等  
18. |-->response.setHeartbeatInterval(nextInterval);  
19. |-->response.setActions(actions.toArray(new TaskTrackerAction[actions.size()]));  
20. |-->return response;  
21.   
22.  
23. ---------------------------------------------------------------------------------------------------------
24.  
    
25. TaskTracker.offerService |TaskTracker启动后通过offerservice()不断发心跳至JobTracker中  
26. |-->transmitHeartBeat()  
27.     |-->HeartbeatResponse heartbeatResponse = jobClient.heartbeat(status, justStarted, justInited,askForNewTask, heartbeatResponseId);  
28. |-->TaskTrackerAction[] actions = heartbeatResponse.getActions();  
29. |-->for(TaskTrackerAction action: actions)  
30.     |-->if (action instanceof LaunchTaskAction)  
31.         |-->addToTaskQueue((LaunchTaskAction)action);  |添加至执行Queue，根据map/reduce task分别添加  
32.             |-->if (action.getTask().isMapTask()) {  
33.                 |-->mapLauncher.addToTaskQueue(action);  
34.                     |-->TaskInProgress tip = registerTask(action, this);  
35.                     |-->tasksToLaunch.add(tip);  
36.                     |-->tasksToLaunch.notifyAll();  |唤醒阻塞进程  
37.             |-->else   
38.                 |-->reduceLauncher.addToTaskQueue(action);  
39.  
40. --------------------------------------------------------------------------------------------------------
41.  
    
42. TaskLauncher.run()  
43. |--> while (tasksToLaunch.isEmpty())   
44.              |-->tasksToLaunch.wait();  
45. |-->tip = tasksToLaunch.remove(0);  
46. |-->startNewTask(tip);  
47.     |-->localizeJob(tip);  
48.         |-->launchTaskForJob(tip, new JobConf(rjob.jobConf));   
49.             |-->tip.setJobConf(jobConf);  
50.             |-->tip.launchTask();  |TaskInProgress.launchTask()  
51.                 |-->this.runner = task.createRunner(TaskTracker.this, this); |区分map/reduce  
52.                 |-->this.runner.start(); 
    
53. ----------------------------------------------------------------------------------------
    
54. MapTaskRunner.run()  |执行MapTask  
55. |-->File workDir = new File(lDirAlloc.getLocalPathToRead()  |准备执行路径  
56. |-->String jar = conf.getJar();  |准备jar包  
57. |-->File jvm = new File(new File(System.getProperty("java.home"), "bin"), "java");  |获取jvm  
58. |-->vargs.add(Child.class.getName());  ｜添加参数，Child类作为main主函数启动  
59. |-->tracker.addToMemoryManager(t.getTaskID(), t.isMapTask(), conf, pidFile);  |添加至内存管理  
60. |-->jvmManager.launchJvm(this, jvmManager.constructJvmEnv(setup,vargs,stdout,stderr,logSize,  |统一纳入jvm管理器当中并启动  
61.                 workDir, env, pidFile, conf));  
62.         |-->mapJvmManager.reapJvm(t, env);  |区分map/reduce操作  
63.  
    
64. ---------------------------------------------------------------------------------------------------------------
    
65. JvmManager.reapJvm()  |  
66. |--> while (jvmIter.hasNext())  
67.     |-->JvmRunner jvmRunner = jvmIter.next().getValue();  
68.     |-->JobID jId = jvmRunner.jvmId.getJobId();  
69.     |-->setRunningTaskForJvm(jvmRunner.jvmId, t);  
70. |-->spawnNewJvm(jobId, env, t);  
71.     |-->JvmRunner jvmRunner = new JvmRunner(env,jobId);  
72.         |-->jvmIdToRunner.put(jvmRunner.jvmId, jvmRunner);  
73.     |-->jvmRunner.start();   |执行JvmRunner的run()方法  
74.         |-->jvmRunner.run()  
75.             |-->runChild(env);  
76.                 |-->List<String> wrappedCommand =  TaskLog.captureOutAndError(env.setup, env.vargs, env.stdout, env.stderr,  
77.                          env.logSize, env.pidFile);  |选取main函数  
78.                 |-->shexec.execute();  |执行  
79.                 |-->int exitCode = shexec.getExitCode(); ｜获取执行状态值  
80.                 |--> updateOnJvmExit(jvmId, exitCode, killed); ｜更新Jvm状态  
81.  
    
82. ---------------------------------------------------------------------------------------------------------
    
83. Child.main() 执行Task(map/reduce)  
84. |-->JVMId jvmId = new JVMId(firstTaskid.getJobID(),firstTaskid.isMap(),jvmIdInt);  
85. |-->TaskUmbilicalProtocol umbilical = (TaskUmbilicalProtocol)RPC.getProxy(TaskUmbilicalProtocol.class,  
86.         TaskUmbilicalProtocol.versionID, address, defaultConf);  
87. |--> while (true)   
88.     |-->JvmTask myTask = umbilical.getTask(jvmId);  
89.     |-->task = myTask.getTask();  
90.     |-->taskid = task.getTaskID();  
91.     |-->TaskRunner.setupWorkDir(job);  
92.     |-->task.run(job, umbilical);   |以maptask为例  
93.         |-->TaskReporter reporter = new TaskReporter(getProgress(), umbilical);  
94.         |-->if (useNewApi)  
95.             |-->runNewMapper(job, split, umbilical, reporter);  
96.         |-->else  
97.             |-->runOldMapper(job, split, umbilical, reporter);  
98.                 |-->inputSplit = (InputSplit) ReflectionUtils.newInstance(job.getClassByName(splitClass), job);  
99.                 |-->MapRunnable<INKEY,INVALUE,OUTKEY,OUTVALUE> runner =  ReflectionUtils.newInstance(job.getMapRunnerClass(), job);  
100.                 |-->runner.run(in, new OldOutputCollector(collector, conf), reporter);  
101.  ------------------------------------------------------------------------------------------------------------------------
     
102. MapRunner.run()  
103. |--> K1 key = input.createKey();  
104. |-->V1 value = input.createValue();  
105. |-->while (input.next(key, value))   
106.     |-->mapper.map(key, value, output, reporter);  
107.     |--> if(incrProcCount)   
108.         |-->reporter.incrCounter(SkipBadRecords.COUNTER_GROUP,   
109.                 |-->SkipBadRecords.COUNTER_MAP_PROCESSED_RECORDS, 1);  
110. |-->mapper.close();  
111.   
112.