Flink源码篇-集群部署与任务提交

集群部署

Standalone模式

启动JobManager

standalone方式部署，入口类org.apache.flink.runtime.entrypoint.StandaloneSessionClusterEntrypoint

main函数

public static void main(String[] args) {
   
	//打印有关环境的信息
    EnvironmentInformation.logEnvironmentInfo(LOG, StandaloneSessionClusterEntrypoint.class.getSimpleName(), args);
	//注册一些信号处理
    SignalHandler.register(LOG);
	//安装安全关闭的钩子
    JvmShutdownSafeguard.installAsShutdownHook(LOG);
    EntrypointClusterConfiguration entrypointClusterConfiguration = null;
    CommandLineParser commandLineParser = new CommandLineParser(new EntrypointClusterConfigurationParserFactory());

    try {
   
	//对传入的参数进行解析
	//内部通过EntrypointClusterConfigurationParserFactory解析配置文件，返回EntrypointClusterConfiguration为ClusterConfiguration的子类
        entrypointClusterConfiguration = (EntrypointClusterConfiguration)commandLineParser.parse(args);
    } catch (FlinkParseException var5) {
   
        LOG.error("Could not parse command line arguments {}.", args, var5);
        commandLineParser.printHelp(StandaloneSessionClusterEntrypoint.class.getSimpleName());
        System.exit(1);
    }

    Configuration configuration = loadConfiguration(entrypointClusterConfiguration);
	//创建了StandaloneSessionClusterEntrypoint
    StandaloneSessionClusterEntrypoint entrypoint = new StandaloneSessionClusterEntrypoint(configuration);
	//启动集群的entrypoint
	//这个方法接受的是父类ClusterEntrypoint，可想而知其他几种启动方式也是通过这个方法
    ClusterEntrypoint.runClusterEntrypoint(entrypoint);
}

main函数中将StandaloneSessionClusterEntrypoint实例化并作为参数调用ClusterEntrypoint的静态方法ClusterEntrypoint

public static void runClusterEntrypoint(ClusterEntrypoint clusterEntrypoint) {
   
    String clusterEntrypointName = clusterEntrypoint.getClass().getSimpleName();

    try {
   
		//启动JobManager
        clusterEntrypoint.startCluster();
    } catch (ClusterEntrypointException var3) {
   
        ...
    }

    ...
}

在方法内部调用ClusterEntrypoint的startCluster()方法

public void startCluster() throws ClusterEntrypointException {
   
   LOG.info("Starting {}.", getClass().getSimpleName());

   try {
   
      //安装默认的文件系统
      configureFileSystems(configuration);

      securityContext.runSecured((Callable<Void>) () -> {
   
		
          //启动JobManager
         runCluster(configuration);

         return null;
      });
   } catch (Throwable t) {
   
      ...
}
}

在该方法内部调用runCluster方法启动JobManager

private void runCluster(Configuration configuration) throws Exception {
   
   synchronized (lock) {
   

//初始化服务（比如RPC、高可用、监控等服务）
      initializeServices(configuration);

      // write host information into configuration
      configuration.setString(JobManagerOptions.ADDRESS, commonRpcService.getAddress());
      configuration.setInteger(JobManagerOptions.PORT, commonRpcService.getPort());

      final DispatcherResourceManagerComponentFactory dispatcherResourceManagerComponentFactory = createDispatcherResourceManagerComponentFactory(configuration);

//这段代码主要是启动Web监控服务、Dispatcher和ResourceManager
      clusterComponent = dispatcherResourceManagerComponentFactory.create(
         configuration,
         ioExecutor,
         commonRpcService,
         haServices,
         blobServer,
         heartbeatServices,
         metricRegistry,
         archivedExecutionGraphStore,
         new RpcMetricQueryServiceRetriever(metricRegistry.getMetricQueryServiceRpcService()),
         this);

  }
}

用于启动Dispathcer、ResourceManager等服务的create的实现如下：

public DispatcherResourceManagerComponent create(
			Configuration configuration,
			Executor ioExecutor,
			RpcService rpcService,
			HighAvailabilityServices highAvailabilityServices,
			BlobServer blobServer,
			HeartbeatServices heartbeatServices,
			MetricRegistry metricRegistry,
			ArchivedExecutionGraphStore archivedExecutionGraphStore,
			MetricQueryServiceRetriever metricQueryServiceRetriever,
			FatalErrorHandler fatalErrorHandler) throws Exception {
   

		LeaderRetrievalService dispatcherLeaderRetrievalService = null;
		LeaderRetrievalService resourceManagerRetrievalService = null;
		WebMonitorEndpoint<?> webMonitorEndpoint = null;
		ResourceManager<?> resourceManager = null;
		ResourceManagerMetricGroup resourceManagerMetricGroup = null;
		DispatcherRunner dispatcherRunner = null;

		try {
   
            
             //获取Dispatcher选举服务
			dispatcherLeaderRetrievalService = highAvailabilityServices.getDispatcherLeaderRetriever();

             //获取RM选举服务
			resourceManagerRetrievalService = highAvailabilityServices.getResourceManagerLeaderRetriever();

			final LeaderGatewayRetriever<DispatcherGateway> dispatcherGatewayRetriever = new RpcGatewayRetriever<>(
				rpcService,
				DispatcherGateway.class,
				DispatcherId::fromUuid,
				10,
				Time.milliseconds(50L));

			final LeaderGatewayRetriever<ResourceManagerGateway> resourceManagerGatewayRetriever = new RpcGatewayRetriever<>(
				rpcService,
				ResourceManagerGateway.class,
				ResourceManagerId::fromUuid,
				10,
				Time.milliseconds(50L));

			final ExecutorService executor = WebMonitorEndpoint.createExecutorService(
				configuration.getInteger(RestOptions.SERVER_NUM_THREADS),
				configuration.getInteger(RestOptions.SERVER_THREAD_PRIORITY),
				"DispatcherRestEndpoint");

			final long updateInterval = configuration.getLong(MetricOptions.METRIC_FETCHER_UPDATE_INTERVAL);
			final MetricFetcher metricFetcher = updateInterval == 0
				? VoidMetricFetcher.INSTANCE
				: MetricFetcherImpl.fromConfiguration(
					configuration,
					metricQueryServiceRetriever,
					dispatcherGatewayRetriever,
					executor);

             //创建状态监控的RestEndpoint，用来处理Web UI上发送的请求
			webMonitorEndpoint = restEndpointFactory.createRestEndpoint(
				configuration,
				dispatcherGatewayRetriever,
				resourceManagerGatewayRetriever,
				blobServer,
				executor,
				metricFetcher,
				highAvailabilityServices.getClusterRestEndpointLeaderElectionService(),
				fatalErrorHandler);

			log.debug("Starting Dispatcher REST endpoint.");
             //启动状态监控的RestEndpoint
			webMonitorEndpoint.start();

			final String hostname = RpcUtils.getHostname(rpcService);

			resourceManagerMetricGroup = ResourceManagerMetricGroup.create(metricRegistry, hostname);
             //创建ResourceManager，属性成员包含了SlotManager和JobLeaderIdService
             //SlotManager用于Slot的管理
             //JobLeaderIdService用于为每个Job分配JobMaster
			resourceManager = resourceManagerFactory.createResourceManager(
				configuration,
				ResourceID.generate(),
				rpcService,
				highAvailabilityServices,
				heartbeatServices,
				fatalErrorHandler,
				new ClusterInformation(hostname, blobServer.getPort()),
				webMonitorEndpoint.getRestBaseUrl(),
				resourceManagerMetricGroup);

			final HistoryServerArchivist historyServerArchivist = HistoryServerArchivist.createHistoryServerArchivist(configuration, webMonitorEndpoint);

			final PartialDispatcherServices partialDispatcherServices = new PartialDispatcherServices(
				configuration,
				highAvailabilityServices,
				resourceManagerGatewayRetriever,
				blobServer,
				heartbeatServices,
				() -> MetricUtils.instantiateJobManagerMetricGroup(metricRegistry, hostname),
				archivedExecutionGraphStore,
				fatalErrorHandler,
				historyServerArchivist,
				metricRegistry.getMetricQueryServiceGatewayRpcAddress());

			log.debug("Starting Dispatcher.");
             //创建并启动Dispatcher
			dispatcherRunner = dispatcherRunnerFactory.createDispatcherRunner(
				highAvailabilityServices.getDispatcherLeaderElectionService(),
				fatalErrorHandler,
				new HaServicesJobGraphStoreFactory(highAvailabilityServices),
				ioExecutor,
				rpcService,
				partialDispatcherServices);

			log.debug("Starting ResourceManager.");
			resourceManager.start();

			resourceManagerRetrievalService.start(resourceManagerGatewayRetriever);
			dispatcherLeaderRetrievalService.start(dispatcherGatewayRetriever);

			return new DispatcherResourceManagerComponent(
				dispatcherRunner,
				resourceManager,
				dispatcherLeaderRetrievalService,
				resourceManagerRetrievalService,
				webMonitorEndpoint);

		} catch (Exception exception) {
   
			...
		}
	}

重点关注三个部分：

• 创建并启动WebMonitorEndpoint，webMonitorEndpoint用于Flink集群的状态监控，内部初始化了各种Handler，用于处理Web UI发出的请求
• 创建并启动ResourceManager，ResourceManager用于Flink集群的资源调度，其属性成员有SlotManager和JobLeaderIdService，其中SlotManager用于管理Slot，JobLeaderIdService用于为每个Job分配JobMaster
• 创建并启动Dispatcher，Dispatcher用于处理调度提交的任务

接下来看Dispatcher的创建和启动过程，通过调用dispatcherRunnerFactory.createDispatcherRunner()方法实现，dispatcherRunnerFactory为DispatcherRunnerFacotry接口的实现类DefaultDispatcherRunnerFactory对象

public DispatcherRunner createDispatcherRunner(
			LeaderElectionService leaderElectionService,
			FatalErrorHandler fatalErrorHandler,
			JobGraphStoreFactory jobGraphStoreFactory,
			Executor ioExecutor,
			RpcService rpcService,
			PartialDispatcherServices partialDispatcherServices) throws Exception {
   

		final DispatcherLeaderProcessFactory dispatcherLeaderProcessFactory = dispatcherLeaderProcessFactoryFactory.createFactory(
			jobGraphStoreFactory,
			ioExecutor,
			rpcService,
			partialDispatcherServices,
			fatalErrorHandler);

    	//创建并启动Dispatcher
		return DefaultDispatcherRunner.create(
			leaderElectionService,
			fatalErrorHandler,
			dispatcherLeaderProcessFactory);
	}

通过调用DefaultDispatcherRunner的静态方法create()创建并启动Dispatcher

public static DispatcherRunner create(
			LeaderElectionService leaderElectionService,
			FatalErrorHandler fatalErrorHandler,
			DispatcherLeaderProcessFactory dispatcherLeaderProcessFactory) throws Exception {
   
    	 //实例化DefaultDispatcherRunner
		final DefaultDispatcherRunner dispatcherRunner = new DefaultDispatcherRunner(
			leaderElectionService,
			fatalErrorHandler,
			dispatcherLeaderProcessFactory);
    	 //创建并启动Dispatcher
		return DispatcherRunnerLeaderElectionLifecycleManager.createFor(dispatcherRunner, leaderElectionService);
	}

将DefaultDispatcherRunner对象和选举服务作为参数调用DispatcherRunnerLeaderElectionLifecycleManager的createFor方法，在该方法内部实例化DispatcherRunnerLeaderElectionLifecycleManager，在其构造器内部将DefaultDispatcherRunner对象作为参数调用选举服务的start方法，该方法是个接口方法，实际上是调用实现类StandaloneLeaderElectionService的start方法

public void start(LeaderContender newContender) throws Exception {
   
		if (contender != null) {
   
			// Service was already started
			throw new IllegalArgumentException("Leader election service cannot be started multiple times.");
		}

		contender = Preconditions.checkNotNull(newContender);

		// directly grant leadership to the given contender
		contender.grantLeadership(HighAvailabilityServices.DEFAULT_LEADER_ID);
	}

contender为DefaultDispatcherRunner对象，称为竞选对象，继承了LeaderContender类，如果竞选leader成功调用granLeadership方法，其内部会调用startNewDispatcherLeaderProcess方法

private void startNewDispatcherLeaderProcess(UUID leaderSessionID) {
   
    	//停止正在运行的DispatcherLeaderProcess
		stopDispatcherLeaderProcess();

    	//创建新的DispatcherLeaderProcess
		dispatcherLeaderProcess = createNewDispatcherLeaderProcess(leaderSessionID);

		final DispatcherLeaderProcess newDispatcherLeaderProcess = dispatcherLeaderProcess;
         //启动DispatcherLeaderProcess
		FutureUtils.assertNoException(	previousDispatcherLeaderProcessTerminationFuture.thenRun(newDispatcherLeaderProcess::start));
	}

调用DispatcherLeaderProcess的start方法

public final void start() {
   
		runIfStateIs(
			State.CREATED,
			this::startInternal);
	}

用一个线程来执行startInternal，该方法内部调用了onStart方法

protected void onStart() {
   
   //启动一些服务（JobGraph）
   startServices();

   //创建并启动Dispatcher
   onGoingRecoveryOperation = recoverJobsAsync()
      .thenAccept(this::createDispatcherIfRunning)
      .handle(this::onErrorIfRunning);
}

通过调用createDispatcherIfRunning方法创建并启动Dispatcher

private void createDispatcherIfRunning(Collection<JobGraph> jobGraphs) {
   
		runIfStateIs(State.RUNNING, () -> createDispatcher(jobGraphs));
	}

实际上createDispatcherIfRunning方法是用一个线程来执行createDispatcher来创建启动Dispatcher

private void createDispatcher(Collection<JobGraph> jobGraphs) {
   

    	//真正的创建并启动Dispatcher
		final DispatcherGatewayService dispatcherService = dispatcherGatewayServiceFactory.create(
			DispatcherId.fromUuid(getLeaderSessionId()),
			jobGraphs,
			jobGraphStore);

		completeDispatcherSetup(dispatcherService);
	}

通过dispatcherGatewayServiceFactory.create()调用真正的创建并启动Dispatcher

public AbstractDispatcherLeaderProcess.DispatcherGatewayService create(
			DispatcherId fencingToken,
			Collection<JobGraph> recoveredJobs,
			JobGraphWriter jobGraphWriter) {
   
		final Dispatcher dispatcher;
		try {
   
            //创建Dispatcher
			dispatcher = dispatcherFactory.createDispatcher(
				rpcService,
				fencingToken,
				recoveredJobs,
				PartialDispatcherServicesWithJobGraphStore.from(partialDispatcherServices, jobGraphWriter));
		} catch (Exception e) {
   
			throw new FlinkRuntimeException("Could not create the Dispatcher rpc endpoint.", e);
		}

    	//启动Dispatcher
		dispatcher.start();

		return DefaultDispatcherGatewayService.from(dispatcher);
	}

启动TaskManager

入口类org.apache.flink.runtime.taskexecutor.TaskManagerRunner

public static void main(String[] args) throws Exception {
   
        EnvironmentInformation.logEnvironmentInfo(LOG, "TaskManager", args);
        
    	...

    	//启动TaskManager
        runTaskManagerSecurely(args, ResourceID.generate());
    }

调用runTaskManagerSecurely启动TaskManager

public static void runTaskManagerSecurely(String[] args, ResourceID resourceID) {
   
   try {
   

	  //根据参数加载配置
      final Configuration configuration = loadConfiguration(args);

      //安装默认文件系统
      FileSystem.initialize(configuration, PluginUtils.createPluginManagerFromRootFolder(configuration));

      //安装安全认证的上下文
      SecurityUtils.install(new SecurityConfiguration(configuration));

      SecurityUtils.getInstalledContext().runSecured(() -> {
   

         //运行TaskManger
         runTaskManager(configuration, resourceID);
         return null;
      });
   } catch (Throwable t) {
   
      final Throwable strippedThrowable = ExceptionUtils.stripException(t, UndeclaredThrowableException.class);
      LOG.error("TaskManager initialization failed.", strippedThrowable);
      System.exit(STARTUP_FAILURE_RETURN_CODE);
   }
}

该方法内部主要关注runTaskManager()方法的调用

public static void runTaskManager(Configuration configuration, ResourceID resourceId)