客户端在提交一个作业的时候，会通过RPC调用服务端接口提交该作业，此时运行做的配置文件可jar文件已经上传到了HDFS中，服务端作业提交的过程中会创建该作业的实例JobInProgress,然后把该对象加入作业监控和初始化两个Listener中，另外还包含一些校验操作，比如所在队列是否在运行，内存配置是否超出限制等，重头戏是JobInProgress的创建，提交过程如下：
public JobStatus submitJob(JobID jobId, String jobSubmitDir, Credentials ts)
throws IOException {
JobInfo jobInfo = null;
//获得提交用户信息
UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
//防止job重复提交
synchronized (this) {
if (jobs.containsKey(jobId)) {
// job already running, don't start twice
return jobs.get(jobId).getStatus();
}
//建立JobInfo，包含jobid、提交用户、和提交目录信息，该类同样实现了Writable接口
jobInfo = new JobInfo(jobId, new Text(ugi.getShortUserName()),
new Path(jobSubmitDir));
}
// Create the JobInProgress, do not lock the JobTracker since
// we are about to copy job.xml from HDFS
JobInProgress job = null;
try {
//创建JobInProgress，包含作业的统计信息，如map数量、reduce数量、正在运行
//的MR数量、失败的任务次数、成功多少、完成多少等等。。
job = new JobInProgress(this, this.conf, jobInfo, 0, ts);
} catch (Exception e) {
throw new IOException(e);
}
synchronized (this) {
// 检测队列是否在运行，默认队列名为default
String queue = job.getProfile().getQueueName();
if (!queueManager.isRunning(queue)) {
throw new IOException("Queue \"" + queue + "\" is not running");
}
try {
aclsManager.checkAccess(job, ugi, Operation.SUBMIT_JOB);
} catch (IOException ioe) {
LOG.warn("Access denied for user " + job.getJobConf().getUser()
+ ". Ignoring job " + jobId, ioe);
job.fail();
throw ioe;
}
// 检查作业的内存配置
try {
checkMemoryRequirements(job);
} catch (IOException ioe) {
throw ioe;
}
boolean recovered = true; // TODO: Once the Job recovery code is there,
// (MAPREDUCE-873) we
// must pass the "recovered" flag accurately.
// This is handled in the trunk/0.22
if (!recovered) {
// Store the information in a file so that the job can be recovered
// later (if at all)
Path jobDir = getSystemDirectoryForJob(jobId);
FileSystem.mkdirs(fs, jobDir, new FsPermission(SYSTEM_DIR_PERMISSION));
FSDataOutputStream out = fs.create(getSystemFileForJob(jobId));
jobInfo.write(out);
out.close();
}
// 提交作业，返回作业状态
JobStatus status;
try {
status = addJob(jobId, job);
} catch (IOException ioe) {
LOG.info("Job " + jobId + " submission failed!", ioe);
status = job.getStatus();
status.setFailureInfo(StringUtils.stringifyException(ioe));
failJob(job);
throw ioe;
}
return status;
}
}
JobInProgress(JobTracker jobtracker, final JobConf default_conf,
JobInfo jobInfo, int rCount, Credentials ts)
throws IOException, InterruptedException {
try {
this.restartCount = rCount;//重启次数
this.jobId = JobID.downgrade(jobInfo.getJobID());//获得jobid
//获得监控作业的URL
String url = "http://" + jobtracker.getJobTrackerMachine() + ":"
+ jobtracker.getInfoPort() + "/jobdetails.jsp?jobid=" + jobId;
this.jobtracker = jobtracker;
//创建新的作业状态，初始值为4，准备状态
this.status = new JobStatus(jobId, 0.0f, 0.0f, JobStatus.PREP);
this.status.setUsername(jobInfo.getUser().toString());
this.jobtracker.getInstrumentation().addPrepJob(conf, jobId);
// 设置启动时间
this.startTime = jobtracker.getClock().getTime();
status.setStartTime(startTime);
//获得本地文件系统
this.localFs = jobtracker.getLocalFileSystem();
this.tokenStorage = ts;
// 获得提交目录
jobSubmitDir = jobInfo.getJobSubmitDir();
user = jobInfo.getUser().toString();
userUGI = UserGroupInformation.createRemoteUser(user);
if (ts != null) {
for (Token<? extends TokenIdentifier> token : ts.getAllTokens()) {
userUGI.addToken(token);
}
}
fs = userUGI.doAs(new PrivilegedExceptionAction<FileSystem>() {
public FileSystem run() throws IOException {
return jobSubmitDir.getFileSystem(default_conf);
}});
//检测作业配置文件job.xml尺寸
Path submitJobFile = JobSubmissionFiles.getJobConfPath(jobSubmitDir);
FileStatus fstatus = fs.getFileStatus(submitJobFile);
if (fstatus.getLen() > jobtracker.MAX_JOBCONF_SIZE) {
throw new IOException("Exceeded max jobconf size: "
+ fstatus.getLen() + " limit: " + jobtracker.MAX_JOBCONF_SIZE);
}
//设置本地配置文件名称，并把HDFS中的配置文件拷贝的本地
this.localJobFile = default_conf.getLocalPath(JobTracker.SUBDIR
+"/"+jobId + ".xml");
Path jobFilePath = JobSubmissionFiles.getJobConfPath(jobSubmitDir);
jobFile = jobFilePath.toString();
fs.copyToLocalFile(jobFilePath, localJobFile);
conf = new JobConf(localJobFile);
if (conf.getUser() == null) {
this.conf.setUser(user);
}
//提交用户检测
if (!conf.getUser().equals(user)) {
String desc = "The username " + conf.getUser() + " obtained from the " +
"conf doesn't match the username " + user + " the user " +
"authenticated as";
AuditLogger.logFailure(user, Operation.SUBMIT_JOB.name(), conf.getUser(),
jobId.toString(), desc);
throw new IOException(desc);
}
//优先级设置，用于后面的resort
this.priority = conf.getJobPriority();
this.status.setJobPriority(this.priority);
String queueName = conf.getQueueName();
this.profile = new JobProfile(user, jobId,
jobFile, url, conf.getJobName(), queueName);
Queue queue = this.jobtracker.getQueueManager().getQueue(queueName);
if (queue == null) {
throw new IOException("Queue \"" + queueName + "\" does not exist");
}
this.queueMetrics = queue.getMetrics();
this.queueMetrics.addPrepJob(conf, jobId);
//通用属性设置：提交节点的主机名、IP、map数、reduce数
this.submitHostName = conf.getJobSubmitHostName();
this.submitHostAddress = conf.getJobSubmitHostAddress();
this.numMapTasks = conf.getNumMapTasks();
this.numReduceTasks = conf.getNumReduceTasks();
//内存设置
this.memoryPerMap = conf.getMemoryForMapTask();
this.memoryPerReduce = conf.getMemoryForReduceTask();
//任务事件列表，在客户端查询作业状态时会把该数据返回，可以反映任务状态
this.taskCompletionEvents = new ArrayList<TaskCompletionEvent>
(numMapTasks + numReduceTasks + 10);
// Construct the jobACLs
status.setJobACLs(jobtracker.getJobACLsManager().constructJobACLs(conf));
//MR允许失败比例的配置
this.mapFailuresPercent = conf.getMaxMapTaskFailuresPercent();
this.reduceFailuresPercent = conf.getMaxReduceTaskFailuresPercent();
//一个作业最大允许失败数
this.maxTaskFailuresPerTracker = conf.getMaxTaskFailuresPerTracker();
//是否允许推测执行
hasSpeculativeMaps = conf.getMapSpeculativeExecution();
hasSpeculativeReduces = conf.getReduceSpeculativeExecution();
// a limit on the input size of the reduce.
// we check to see if the estimated input size of
// of each reduce is less than this value. If not
// we fail the job. A value of -1 just means there is no
// limit set.
reduce_input_limit = -1L;
this.maxLevel = jobtracker.getNumTaskCacheLevels();
this.anyCacheLevel = this.maxLevel+1;
this.nonLocalMaps = new LinkedList<TaskInProgress>();
this.failedMaps = new TreeSet<TaskInProgress>(failComparator);
this.nonLocalRunningMaps = new LinkedHashSet<TaskInProgress>();
this.runningMapCache = new IdentityHashMap<Node, Set<TaskInProgress>>();
this.nonRunningReduces = new TreeSet<TaskInProgress>(failComparator);
this.runningReduces = new LinkedHashSet<TaskInProgress>();
this.resourceEstimator = new ResourceEstimator(this);
this.reduce_input_limit = conf.getLong("mapreduce.reduce.input.limit",
DEFAULT_REDUCE_INPUT_LIMIT);
// register job's tokens for renewal
DelegationTokenRenewal.registerDelegationTokensForRenewal(
jobInfo.getJobID(), ts, jobtracker.getConf());
// 最大任务数校验
checkTaskLimits();
} finally {
//close all FileSystems that was created above for the current user
//At this point, this constructor is called in the context of an RPC, and
//hence the "current user" is actually referring to the kerberos
//authenticated user (if security is ON).
FileSystem.closeAllForUGI(UserGroupInformation.getCurrentUser());
}
} private synchronized JobStatus addJob(JobID jobId, JobInProgress job)
throws IOException {
totalSubmissions++;
synchronized (jobs) {
synchronized (taskScheduler) {
//在JobTracker中记录该job
jobs.put(job.getProfile().getJobID(), job);
for (JobInProgressListener listener : jobInProgressListeners) {
listener.jobAdded(job);//向两个Listener中添加该job，JobQueueJobInProgressListener和EagerTaskInitializationListener
}
}
}
myInstrumentation.submitJob(job.getJobConf(), jobId);
job.getQueueMetrics().submitJob(job.getJobConf(), jobId);
LOG.info("Job " + jobId + " added successfully for user '"
+ job.getJobConf().getUser() + "' to queue '"
+ job.getJobConf().getQueueName() + "'");
//记录提交日志
AuditLogger.logSuccess(job.getUser(),
Operation.SUBMIT_JOB.name(), jobId.toString());
return job.getStatus();
}
扫一扫
444
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
