ApplicationMasterLauncher(基于hadoop 2.7.6)
ApplicationMasterLauncher主要处理AMLauncherEventType.LAUNCH和AMLauncherEventType.CLEANUP两类事件.这里主要讲LAUNCH事件.
ApplicationMasterLauncher主要是基于生产者消费者模式,根据AttemptStoredTransition生成的AMLauncherEventType.LAUNCH事件生成对应类型的AMLauncher放入masterEvents事件队列.
而ApplicationMasterLauncher初始化启动时,会启动launcherHandlingThread线程,该线程又维护了一个线程池ThreadPoolExecutor launcherPool,会异步执行AMLauncher对应的launch方法或cleanup方法.
/**
* ApplicationMasterLauncher即是一个服务,也是一个事件处理器,它处理AMLauncherEventType类型的事件,该类型事件有两种,分别是请求启动一个ApplicationMaster的LAUNCH事件和请求清理一个ApplicationMaster的CLEANUP事件.
*ApplicationMasterLauncher维护了一个线程池,从而能够尽快地处理事件.
* 1.如果ApplicationMasterLauncher收到LAUNCH事件,它会与对应的NodeManager通信,要求它启动ApplicationMaster.
* 2.如果ApplicationMasterLauncher收到CLEANUP事件,它与对应的NodeManager通信,要求它杀死ApplicationMaster.
*
* 主要采用了生产者消费者的设计模式,handle是一个入口,将事件放入队列,然后启动LauncherThread,由LauncherThread线程从队列里取出事件提交到线程池中.
*/
public class ApplicationMasterLauncher extends AbstractService implements
EventHandler<AMLauncherEvent> {
private static final Log LOG = LogFactory.getLog(
ApplicationMasterLauncher.class);
private ThreadPoolExecutor launcherPool;
private LauncherThread launcherHandlingThread;
/**
* 事件队列
*/
private final BlockingQueue<Runnable> masterEvents
= new LinkedBlockingQueue<Runnable>();
protected final RMContext context;
public ApplicationMasterLauncher(RMContext context) {
super(ApplicationMasterLauncher.class.getName());
this.context = context;
this.launcherHandlingThread = new LauncherThread();
}
@Override
protected void serviceInit(Configuration conf) throws Exception {
int threadCount = conf.getInt(
YarnConfiguration.RM_AMLAUNCHER_THREAD_COUNT,
YarnConfiguration.DEFAULT_RM_AMLAUNCHER_THREAD_COUNT);
ThreadFactory tf = new ThreadFactoryBuilder()
.setNameFormat("ApplicationMasterLauncher #%d")
.build();
launcherPool = new ThreadPoolExecutor(threadCount, threadCount, 1,
TimeUnit.HOURS, new LinkedBlockingQueue<Runnable>());
launcherPool.setThreadFactory(tf);
Configuration newConf = new YarnConfiguration(conf);
newConf.setInt(CommonConfigurationKeysPublic.
IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY,
conf.getInt(YarnConfiguration.RM_NODEMANAGER_CONNECT_RETIRES,
YarnConfiguration.DEFAULT_RM_NODEMANAGER_CONNECT_RETIRES));
setConfig(newConf);
super.serviceInit(newConf);
}
@Override
protected void serviceStart() throws Exception {
launcherHandlingThread.start();
super.serviceStart();
}
/**
* 创建AMLauncher,AMLauncher实现了处理LAUNCH和CLEANUP事件的具体逻辑
* @param application
* @param event
* @return
*/
protected Runnable createRunnableLauncher(RMAppAttempt application,
AMLauncherEventType event) {
Runnable launcher =
new AMLauncher(context, application, event, getConfig());
return launcher;
}
private void launch(RMAppAttempt application) {
Runnable launcher = createRunnableLauncher(application,
AMLauncherEventType.LAUNCH);
masterEvents.add(launcher);
}
@Override
protected void serviceStop() throws Exception {
launcherHandlingThread.interrupt();
try {
launcherHandlingThread.join();
} catch (InterruptedException ie) {
LOG.info(launcherHandlingThread.getName() + " interrupted during join ",
ie); }
launcherPool.shutdown();
}
private class LauncherThread extends Thread {
public LauncherThread() {
super("ApplicationMaster Launcher");
}
@Override
public void run() {
while (!this.isInterrupted()) {
Runnable toLaunch;
try {
toLaunch = masterEvents.take();
launcherPool.execute(toLaunch);
} catch (InterruptedException e) {
LOG.warn(this.getClass().getName() + " interrupted. Returning.");
return;
}
}
}
}
private void cleanup(RMAppAttempt application) {
Runnable launcher = createRunnableLauncher(application, AMLauncherEventType.CLEANUP);
masterEvents.add(launcher);
}
@Override
public synchronized void handle(AMLauncherEvent appEvent) {
AMLauncherEventType event = appEvent.getType();
RMAppAttempt application = appEvent.getAppAttempt();
switch (event) {
case LAUNCH:
launch(application);
break;
case CLEANUP:
cleanup(application);
break;
default:
break;
}
}
}