集群部署
Standalone模式
启动JobManager
standalone
方式部署,入口类org.apache.flink.runtime.entrypoint.StandaloneSessionClusterEntrypoint
main
函数
public static void main(String[] args) {
//打印有关环境的信息
EnvironmentInformation.logEnvironmentInfo(LOG, StandaloneSessionClusterEntrypoint.class.getSimpleName(), args);
//注册一些信号处理
SignalHandler.register(LOG);
//安装安全关闭的钩子
JvmShutdownSafeguard.installAsShutdownHook(LOG);
EntrypointClusterConfiguration entrypointClusterConfiguration = null;
CommandLineParser commandLineParser = new CommandLineParser(new EntrypointClusterConfigurationParserFactory());
try {
//对传入的参数进行解析
//内部通过EntrypointClusterConfigurationParserFactory解析配置文件,返回EntrypointClusterConfiguration为ClusterConfiguration的子类
entrypointClusterConfiguration = (EntrypointClusterConfiguration)commandLineParser.parse(args);
} catch (FlinkParseException var5) {
LOG.error("Could not parse command line arguments {}.", args, var5);
commandLineParser.printHelp(StandaloneSessionClusterEntrypoint.class.getSimpleName());
System.exit(1);
}
Configuration configuration = loadConfiguration(entrypointClusterConfiguration);
//创建了StandaloneSessionClusterEntrypoint
StandaloneSessionClusterEntrypoint entrypoint = new StandaloneSessionClusterEntrypoint(configuration);
//启动集群的entrypoint
//这个方法接受的是父类ClusterEntrypoint,可想而知其他几种启动方式也是通过这个方法
ClusterEntrypoint.runClusterEntrypoint(entrypoint);
}
main
函数中将StandaloneSessionClusterEntrypoint
实例化并作为参数调用ClusterEntrypoint
的静态方法ClusterEntrypoint
public static void runClusterEntrypoint(ClusterEntrypoint clusterEntrypoint) {
String clusterEntrypointName = clusterEntrypoint.getClass().getSimpleName();
try {
//启动JobManager
clusterEntrypoint.startCluster();
} catch (ClusterEntrypointException var3) {
...
}
...
}
在方法内部调用ClusterEntrypoint
的startCluster()
方法
public void startCluster() throws ClusterEntrypointException {
LOG.info("Starting {}.", getClass().getSimpleName());
try {
//安装默认的文件系统
configureFileSystems(configuration);
securityContext.runSecured((Callable<Void>) () -> {
//启动JobManager
runCluster(configuration);
return null;
});
} catch (Throwable t) {
...
}
}
在该方法内部调用runCluster
方法启动JobManager
private void runCluster(Configuration configuration) throws Exception {
synchronized (lock) {
//初始化服务(比如RPC、高可用、监控等服务)
initializeServices(configuration);
// write host information into configuration
configuration.setString(JobManagerOptions.ADDRESS, commonRpcService.getAddress());
configuration.setInteger(JobManagerOptions.PORT, commonRpcService.getPort());
final DispatcherResourceManagerComponentFactory dispatcherResourceManagerComponentFactory = createDispatcherResourceManagerComponentFactory(configuration);
//这段代码主要是启动Web监控服务、Dispatcher和ResourceManager
clusterComponent = dispatcherResourceManagerComponentFactory.create(
configuration,
ioExecutor,
commonRpcService,
haServices,
blobServer,
heartbeatServices,
metricRegistry,
archivedExecutionGraphStore,
new RpcMetricQueryServiceRetriever(metricRegistry.getMetricQueryServiceRpcService()),
this);
}
}
用于启动Dispathcer
、ResourceManager
等服务的create
的实现如下:
public DispatcherResourceManagerComponent create(
Configuration configuration,
Executor ioExecutor,
RpcService rpcService,
HighAvailabilityServices highAvailabilityServices,
BlobServer blobServer,
HeartbeatServices heartbeatServices,
MetricRegistry metricRegistry,
ArchivedExecutionGraphStore archivedExecutionGraphStore,
MetricQueryServiceRetriever metricQueryServiceRetriever,
FatalErrorHandler fatalErrorHandler) throws Exception {
LeaderRetrievalService dispatcherLeaderRetrievalService = null;
LeaderRetrievalService resourceManagerRetrievalService = null;
WebMonitorEndpoint<?> webMonitorEndpoint = null;
ResourceManager<?> resourceManager = null;
ResourceManagerMetricGroup resourceManagerMetricGroup = null;
DispatcherRunner dispatcherRunner = null;
try {
//获取Dispatcher选举服务
dispatcherLeaderRetrievalService = highAvailabilityServices.getDispatcherLeaderRetriever();
//获取RM选举服务
resourceManagerRetrievalService = highAvailabilityServices.getResourceManagerLeaderRetriever();
final LeaderGatewayRetriever<DispatcherGateway> dispatcherGatewayRetriever = new RpcGatewayRetriever<>(
rpcService,
DispatcherGateway.class,
DispatcherId::fromUuid,
10,
Time.milliseconds(50L));
final LeaderGatewayRetriever<ResourceManagerGateway> resourceManagerGatewayRetriever = new RpcGatewayRetriever<>(
rpcService,
ResourceManagerGateway.class,
ResourceManagerId::fromUuid,
10,
Time.milliseconds(50L));
final ExecutorService executor = WebMonitorEndpoint.createExecutorService(
configuration.getInteger(RestOptions.SERVER_NUM_THREADS),
configuration.getInteger(RestOptions.SERVER_THREAD_PRIORITY),
"DispatcherRestEndpoint");
final long updateInterval = configuration.getLong(MetricOptions.METRIC_FETCHER_UPDATE_INTERVAL);
final MetricFetcher metricFetcher = updateInterval == 0
? VoidMetricFetcher.INSTANCE
: MetricFetcherImpl.fromConfiguration(
configuration,
metricQueryServiceRetriever,
dispatcherGatewayRetriever,
executor);
//创建状态监控的RestEndpoint,用来处理Web UI上发送的请求
webMonitorEndpoint = restEndpointFactory.createRestEndpoint(
configuration,
dispatcherGatewayRetriever,
resourceManagerGatewayRetriever,
blobServer,
executor,
metricFetcher,
highAvailabilityServices.getClusterRestEndpointLeaderElectionService(),
fatalErrorHandler);
log.debug("Starting Dispatcher REST endpoint.");
//启动状态监控的RestEndpoint
webMonitorEndpoint.start();
final String hostname = RpcUtils.getHostname(rpcService);
resourceManagerMetricGroup = ResourceManagerMetricGroup.create(metricRegistry, hostname);
//创建ResourceManager,属性成员包含了SlotManager和JobLeaderIdService
//SlotManager用于Slot的管理
//JobLeaderIdService用于为每个Job分配JobMaster
resourceManager = resourceManagerFactory.createResourceManager(
configuration,
ResourceID.generate(),
rpcService,
highAvailabilityServices,
heartbeatServices,
fatalErrorHandler,
new ClusterInformation(hostname, blobServer.getPort()),
webMonitorEndpoint.getRestBaseUrl(),
resourceManagerMetricGroup);
final HistoryServerArchivist historyServerArchivist = HistoryServerArchivist.createHistoryServerArchivist(configuration, webMonitorEndpoint);
final PartialDispatcherServices partialDispatcherServices = new PartialDispatcherServices(
configuration,
highAvailabilityServices,
resourceManagerGatewayRetriever,
blobServer,
heartbeatServices,
() -> MetricUtils.instantiateJobManagerMetricGroup(metricRegistry, hostname),
archivedExecutionGraphStore,
fatalErrorHandler,
historyServerArchivist,
metricRegistry.getMetricQueryServiceGatewayRpcAddress());
log.debug("Starting Dispatcher.");
//创建并启动Dispatcher
dispatcherRunner = dispatcherRunnerFactory.createDispatcherRunner(
highAvailabilityServices.getDispatcherLeaderElectionService(),
fatalErrorHandler,
new HaServicesJobGraphStoreFactory(highAvailabilityServices),
ioExecutor,
rpcService,
partialDispatcherServices);
log.debug("Starting ResourceManager.");
resourceManager.start();
resourceManagerRetrievalService.start(resourceManagerGatewayRetriever);
dispatcherLeaderRetrievalService.start(dispatcherGatewayRetriever);
return new DispatcherResourceManagerComponent(
dispatcherRunner,
resourceManager,
dispatcherLeaderRetrievalService,
resourceManagerRetrievalService,
webMonitorEndpoint);
} catch (Exception exception) {
...
}
}
重点关注三个部分:
- 创建并启动
WebMonitorEndpoint
,webMonitorEndpoint
用于Flink
集群的状态监控,内部初始化了各种Handler
,用于处理Web UI
发出的请求 - 创建并启动
ResourceManager
,ResourceManager
用于Flink
集群的资源调度,其属性成员有SlotManager
和JobLeaderIdService
,其中SlotManager
用于管理Slot
,JobLeaderIdService
用于为每个Job
分配JobMaster
- 创建并启动
Dispatcher
,Dispatcher
用于处理调度提交的任务
接下来看Dispatcher
的创建和启动过程,通过调用dispatcherRunnerFactory.createDispatcherRunner()
方法实现,dispatcherRunnerFactory
为DispatcherRunnerFacotry
接口的实现类DefaultDispatcherRunnerFactory
对象
public DispatcherRunner createDispatcherRunner(
LeaderElectionService leaderElectionService,
FatalErrorHandler fatalErrorHandler,
JobGraphStoreFactory jobGraphStoreFactory,
Executor ioExecutor,
RpcService rpcService,
PartialDispatcherServices partialDispatcherServices) throws Exception {
final DispatcherLeaderProcessFactory dispatcherLeaderProcessFactory = dispatcherLeaderProcessFactoryFactory.createFactory(
jobGraphStoreFactory,
ioExecutor,
rpcService,
partialDispatcherServices,
fatalErrorHandler);
//创建并启动Dispatcher
return DefaultDispatcherRunner.create(
leaderElectionService,
fatalErrorHandler,
dispatcherLeaderProcessFactory);
}
通过调用DefaultDispatcherRunner
的静态方法create()
创建并启动Dispatcher
public static DispatcherRunner create(
LeaderElectionService leaderElectionService,
FatalErrorHandler fatalErrorHandler,
DispatcherLeaderProcessFactory dispatcherLeaderProcessFactory) throws Exception {
//实例化DefaultDispatcherRunner
final DefaultDispatcherRunner dispatcherRunner = new DefaultDispatcherRunner(
leaderElectionService,
fatalErrorHandler,
dispatcherLeaderProcessFactory);
//创建并启动Dispatcher
return DispatcherRunnerLeaderElectionLifecycleManager.createFor(dispatcherRunner, leaderElectionService);
}
将DefaultDispatcherRunner
对象和选举服务作为参数调用DispatcherRunnerLeaderElectionLifecycleManager
的createFor
方法,在该方法内部实例化DispatcherRunnerLeaderElectionLifecycleManager
,在其构造器内部将DefaultDispatcherRunner
对象作为参数调用选举服务的start
方法,该方法是个接口方法,实际上是调用实现类StandaloneLeaderElectionService
的start
方法
public void start(LeaderContender newContender) throws Exception {
if (contender != null) {
// Service was already started
throw new IllegalArgumentException("Leader election service cannot be started multiple times.");
}
contender = Preconditions.checkNotNull(newContender);
// directly grant leadership to the given contender
contender.grantLeadership(HighAvailabilityServices.DEFAULT_LEADER_ID);
}
contender
为DefaultDispatcherRunner
对象,称为竞选对象,继承了LeaderContender
类,如果竞选leader
成功调用granLeadership
方法,其内部会调用startNewDispatcherLeaderProcess
方法
private void startNewDispatcherLeaderProcess(UUID leaderSessionID) {
//停止正在运行的DispatcherLeaderProcess
stopDispatcherLeaderProcess();
//创建新的DispatcherLeaderProcess
dispatcherLeaderProcess = createNewDispatcherLeaderProcess(leaderSessionID);
final DispatcherLeaderProcess newDispatcherLeaderProcess = dispatcherLeaderProcess;
//启动DispatcherLeaderProcess
FutureUtils.assertNoException( previousDispatcherLeaderProcessTerminationFuture.thenRun(newDispatcherLeaderProcess::start));
}
调用DispatcherLeaderProcess
的start
方法
public final void start() {
runIfStateIs(
State.CREATED,
this::startInternal);
}
用一个线程来执行startInternal
,该方法内部调用了onStart
方法
protected void onStart() {
//启动一些服务(JobGraph)
startServices();
//创建并启动Dispatcher
onGoingRecoveryOperation = recoverJobsAsync()
.thenAccept(this::createDispatcherIfRunning)
.handle(this::onErrorIfRunning);
}
通过调用createDispatcherIfRunning
方法创建并启动Dispatcher
private void createDispatcherIfRunning(Collection<JobGraph> jobGraphs) {
runIfStateIs(State.RUNNING, () -> createDispatcher(jobGraphs));
}
实际上createDispatcherIfRunning
方法是用一个线程来执行createDispatcher
来创建启动Dispatcher
private void createDispatcher(Collection<JobGraph> jobGraphs) {
//真正的创建并启动Dispatcher
final DispatcherGatewayService dispatcherService = dispatcherGatewayServiceFactory.create(
DispatcherId.fromUuid(getLeaderSessionId()),
jobGraphs,
jobGraphStore);
completeDispatcherSetup(dispatcherService);
}
通过dispatcherGatewayServiceFactory.create()
调用真正的创建并启动Dispatcher
public AbstractDispatcherLeaderProcess.DispatcherGatewayService create(
DispatcherId fencingToken,
Collection<JobGraph> recoveredJobs,
JobGraphWriter jobGraphWriter) {
final Dispatcher dispatcher;
try {
//创建Dispatcher
dispatcher = dispatcherFactory.createDispatcher(
rpcService,
fencingToken,
recoveredJobs,
PartialDispatcherServicesWithJobGraphStore.from(partialDispatcherServices, jobGraphWriter));
} catch (Exception e) {
throw new FlinkRuntimeException("Could not create the Dispatcher rpc endpoint.", e);
}
//启动Dispatcher
dispatcher.start();
return DefaultDispatcherGatewayService.from(dispatcher);
}
启动TaskManager
入口类org.apache.flink.runtime.taskexecutor.TaskManagerRunner
public static void main(String[] args) throws Exception {
EnvironmentInformation.logEnvironmentInfo(LOG, "TaskManager", args);
...
//启动TaskManager
runTaskManagerSecurely(args, ResourceID.generate());
}
调用runTaskManagerSecurely
启动TaskManager
public static void runTaskManagerSecurely(String[] args, ResourceID resourceID) {
try {
//根据参数加载配置
final Configuration configuration = loadConfiguration(args);
//安装默认文件系统
FileSystem.initialize(configuration, PluginUtils.createPluginManagerFromRootFolder(configuration));
//安装安全认证的上下文
SecurityUtils.install(new SecurityConfiguration(configuration));
SecurityUtils.getInstalledContext().runSecured(() -> {
//运行TaskManger
runTaskManager(configuration, resourceID);
return null;
});
} catch (Throwable t) {
final Throwable strippedThrowable = ExceptionUtils.stripException(t, UndeclaredThrowableException.class);
LOG.error("TaskManager initialization failed.", strippedThrowable);
System.exit(STARTUP_FAILURE_RETURN_CODE);
}
}
该方法内部主要关注runTaskManager()
方法的调用
public static void runTaskManager(Configuration configuration, ResourceID resourceId)