MapRdeuce
job提交流程和切片划分源码解析
在这个mapreduce程序中使用的是wordcout案例
// 提交job
boolean result = job.waitForCompletion(true);
源码:
public boolean waitForCompletion(boolean verbose) throws IOException, InterruptedException, ClassNotFoundException {
if (this.state == Job.JobState.DEFINE) {
//job的目前状态信息 DEFINE
this.submit();
}
return this.isSuccessful();
}
提交job时先建立连接,获取当前客户端的信息(当前是本地环境)
this.connect();
private synchronized void connect() throws IOException, InterruptedException, ClassNotFoundException {
if (this.cluster == null) {
this.cluster = (Cluster)this.ugi.doAs(new PrivilegedExceptionAction<Cluster>() {
public Cluster run() throws IOException, InterruptedException, ClassNotFoundException {
//创建提交job的代理
return new Cluster(Job.this.getConfiguration());
}
});
}
}
public Cluster(Configuration conf) throws IOException {
this((InetSocketAddress)null, conf);
}
private volatile List<ClientProtocolProvider> providerList;// null
public Cluster(InetSocketAddress jobTrackAddr, Configuration conf) throws IOException {
//判断是本地运行环境还是yarn集群运行环境
this.initialize(jobTrackAddr, conf);
}
private void initialize(InetSocketAddress jobTrackAddr, Configuration conf) throws IOException {
this.initProviderList();
Iterator var4 = this.providerList.iterator();
//遍历运行环境集合得到此时程序的运行环境
while(var4.hasNext()) {
ClientProtocolProvider provider = (ClientProtocolProvider)var4.next();
ClientProtocol clientProtocol = null;
try {
if (jobTrackAddr == null) {
clientProtocol = provider.create(conf);
} else {
clientProtocol = provider.create(jobTrackAddr, conf);
}
if (clientProtocol != null) {
this.clientProtocolProvider = provider;//LocalClien or YarnClient
this.client = clientProtocol;
break;
}
}
}
//初始化运行环境,localProviderList中存放两种运行环境 LocalClient,YarnClient
private void initProviderList() {
if (this.providerList