flink源码分析1-2--yarnPerJob任务提交流程--执行用户代码

flink源码分析1-2–yarnPerJob任务提交流程–执行用户代码

1、run方法的核心逻辑—executeProgram(effectiveConfiguration, program);

//CliFrontend.java，executeProgram(effectiveConfiguration, program);点进去：
	protected void executeProgram(final Configuration configuration, final PackagedProgram program) throws ProgramInvocationException {
		ClientUtils.executeProgram(new DefaultExecutorServiceLoader(), configuration, program, false, false);
	}
//再点进去，到ClientUtils.executeProgram(new DefaultExecutorServiceLoader(), configuration, program, false, false);当前类：ClientUtils.java
	public static void executeProgram(
			PipelineExecutorServiceLoader executorServiceLoader,
			Configuration configuration,
			PackagedProgram program,
			boolean enforceSingleJobExecution,
			boolean suppressSysout) throws ProgramInvocationException {
		checkNotNull(executorServiceLoader);
		final ClassLoader userCodeClassLoader = program.getUserCodeClassLoader();
		final ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
		try {
			Thread.currentThread().setContextClassLoader(userCodeClassLoader);

			LOG.info("Starting program (detached: {})", !configuration.getBoolean(DeploymentOptions.ATTACHED));

			/*TODO 配置环境的上下文，用户代码里的 getExecutionEnvironment就会拿到这些环境信息*/
			ContextEnvironment.setAsContext(
				executorServiceLoader,
				configuration,
				userCodeClassLoader,
				enforceSingleJobExecution,
				suppressSysout);

			StreamContextEnvironment.setAsContext(
				executorServiceLoader,
				configuration,
				userCodeClassLoader,
				enforceSingleJobExecution,
				suppressSysout);

			try {
				program.invokeInteractiveModeForExecution();
			} finally {
				ContextEnvironment.unsetAsContext();
				StreamContextEnvironment.unsetAsContext();
			}
		} finally {
			Thread.currentThread().setContextClassLoader(contextClassLoader);
		}
	}
//这个动作是调用executeProgram()方法过程中包含的类加载器赋值和配置环境上下文的动作，完成这些动作之后，才可以加载用户代码的类，拿到上下文和ExecutionEnvironment。最后的核心逻辑是program.invokeInteractiveModeForExecution();它接下来调用PackagedProgram的callMainMethod(mainClass, args)方法，传入用户代码的main函数类

/*
 *当前类：org.apache.flink.client.program.PackagedProgram
 *Clifrontend.main(args)->cli.parseAndRun(args)->run->ClientUtils.executeProgram(effectiveConfiguration, program)->program.invokeInteractiveModeForExecution()->callMainMethod(mainClass, args)
 */
	private static void callMainMethod(Class<?> entryClass, String[] args) throws ProgramInvocationException {
		Method mainMethod;
		if (!Modifier.isPublic(entryClass.getModifiers())) {
			throw new ProgramInvocationException("The class " + entryClass.getName() + " must be public.");
		}

		try {
			mainMethod = entryClass.getMethod("main", String[].class);
		} catch (NoSuchMethodException e) {
			throw new ProgramInvocationException("The class " + entryClass.getName() + " has no main(String[]) method.");
		} catch (Throwable t) {
			throw new ProgramInvocationException("Could not look up the main(String[]) method from the class " +
				entryClass.getName() + ": " + t.getMessage(), t);
		}

		if (!Modifier.isStatic(mainMethod.getModifiers())) {
			throw new ProgramInvocationException("The class " + entryClass.getName() + " declares a non-static main method.");
		}
		if (!Modifier.isPublic(mainMethod.getModifiers())) {
			throw new ProgramInvocationException("The class " + entryClass.getName() + " declares a non-public main method.");
		}

		try {
			/*TODO 调用用户代码的main方法*/
			mainMethod.invoke(null, (Object) args);
		} catch (IllegalArgumentException e) {
			throw new ProgramInvocationException("Could not invoke the main method, arguments are not matching.", e);
		} catch (IllegalAccessException e) {
			throw new ProgramInvocationException("Access to the main method was denied: " + e.getMessage(), e);
		} catch (InvocationTargetException e) {
			Throwable exceptionInMethod = e.getTargetException();
			if (exceptionInMethod instanceof Error) {
				throw (Error) exceptionInMethod;
			} else if (exceptionInMethod instanceof ProgramParametrizationException) {
				throw (ProgramParametrizationException) exceptionInMethod;
			} else if (exceptionInMethod instanceof ProgramInvocationException) {
				throw (ProgramInvocationException) exceptionInMethod;
			} else {
				throw new ProgramInvocationException("The main method caused an error: " + exceptionInMethod.getMessage(), exceptionInMethod);
			}
		} catch (Throwable t) {
			throw new ProgramInvocationException("An error occurred while invoking the program's main method: " + t.getMessage(), t);
		}
	}
//到mainMethod.invoke(null, (Object) args);开始调用用户的main方法，从execute()方法开始，用户的代码和依赖以及flink的配置文件，各种依赖包将被读取，封装，然后启动yarn任务，部署flink集群的各种组件，提交用户代码，jar包依赖和配置文件……

2、执行用户的代码的main方法，触发execute()方法

/*
 *当前类：org.apache.flink.streaming.api.environment.StreamingExecutionEnvironment
 *StreamingExecutionEnvironment.execute(jobName)
 */
	public JobExecutionResult execute() throws Exception {
		return execute(getJobName());
	}


	public JobExecutionResult execute(String jobName) throws Exception {
		Preconditions.checkNotNull(jobName, "Streaming Job name should not be null.");

		/*TODO 获取StreamGraph，并接着执行*/
		return execute(getStreamGraph(jobName));
	}

	public JobExecutionResult execute(StreamGraph streamGraph) throws Exception {
		final JobClient jobClient = executeAsync(streamGraph);

		try {
			final JobExecutionResult jobExecutionResult;

			if (configuration.getBoolean(DeploymentOptions.ATTACHED)) {
				jobExecutionResult = jobClient.getJobExecutionResult().get();
			} else {
				jobExecutionResult = new DetachedJobExecutionResult(jobClient.getJobID());
			}

			jobListeners.forEach(jobListener -> jobListener.onJobExecuted(jobExecutionResult, null));

			return jobExecutionResult;
		} catch (Throwable t) {
			// get() on the JobExecutionResult Future will throw an ExecutionException. This
			// behaviour was largely not there in Flink versions before the PipelineExecutor
			// refactoring so we should strip that exception.
			Throwable strippedException = ExceptionUtils.stripExecutionException(t);

			jobListeners.forEach(jobListener -> {
				jobListener.onJobExecuted(null, strippedException);
			});
			ExceptionUtils.rethrowException(strippedException);

			// never reached, only make javac happy
			return null;
		}
	}

	public JobClient executeAsync(StreamGraph streamGraph) throws Exception {
		checkNotNull(streamGraph, "StreamGraph cannot be null.");
		checkNotNull(configuration.get(DeploymentOptions.TARGET), "No execution.target specified in your configuration file.");

		final PipelineExecutorFactory executorFactory =
			executorServiceLoader.getExecutorFactory(configuration);

		checkNotNull(
			executorFactory,
			"Cannot find compatible factory for specified execution.target (=%s)",
			configuration.get(DeploymentOptions.TARGET));

		CompletableFuture<JobClient> jobClientFuture = executorFactory
			.getExecutor(configuration)
			.execute(streamGraph, configuration, userClassloader);

		try {
			JobClient jobClient = jobClientFuture.get();
			jobListeners.forEach(jobListener -> jobListener.onJobSubmitted(jobClient, null));
			return jobClient;
		} catch (ExecutionException executionException) {
			final Throwable strippedException = ExceptionUtils.stripExecutionException(executionException);
			jobListeners.forEach(jobListener -> jobListener.onJobSubmitted(null, strippedException));

			throw new FlinkException(
				String.format("Failed to execute job '%s'.", streamGraph.getJobName()),
				strippedException);
		}
	}

/*
 *当前类：org.apache.flink.client.deployment.executors.AbstractJobClusterExecutor
 *StreamingExecutionEnvironment.execute(jobName)->AbstractJobClusterExecutor.execute(streamGraph, configuration, userClassloader)
 */
	@Override
	public CompletableFuture<JobClient> execute(@Nonnull final Pipeline pipeline, @Nonnull final Configuration configuration, @Nonnull final ClassLoader userCodeClassloader) throws Exception {
		/*TODO 将 流图（StreamGraph） 转换成 作业图（JobGraph）*/
		final JobGraph jobGraph = PipelineExecutorUtils.getJobGraph(pipeline, configuration);

		/*TODO 集群描述器：创建、启动了 YarnClient， 包含了一些yarn、flink的配置和环境信息*/
		try (final ClusterDescriptor<ClusterID> clusterDescriptor = clusterClientFactory.createClusterDescriptor(configuration)) {
			final ExecutionConfigAccessor configAccessor = ExecutionConfigAccessor.fromConfiguration(configuration);

			/*TODO 集群特有资源配置：JobManager内存、TaskManager内存、每个Tm的slot数*/
			final ClusterSpecification clusterSpecification = clusterClientFactory.getClusterSpecification(configuration);

			final ClusterClientProvider<ClusterID> clusterClientProvider = clusterDescriptor
					.deployJobCluster(clusterSpecification, jobGraph, configAccessor.getDetachedMode());
			LOG.info("Job has been submitted with JobID " + jobGraph.getJobID());

			return CompletableFuture.completedFuture(
					new ClusterClientJobClientAdapter<>(clusterClientProvider, jobGraph.getJobID(), userCodeClassloader));
		}
	}

/*
 *当前类：org.apache.flink.yarn.YarnClusterClientFactory
 *StreamingExecutionEnvironment.execute(jobName)->AbstractJobClusterExecutor.execute(streamGraph, configuration, userClassloader)->createClusterDescriptor(configuration)
 *TODO 集群描述器：创建、启动了 YarnClient， 包含了一些yarn、flink的配置和环境信息
 */
	@Override
	//传配置，之前cli那边封装的
	public YarnClusterDescriptor createClusterDescriptor(Configuration configuration) {
		checkNotNull(configuration);

		//拿配置的路径：flink-conf.yaml
		final String configurationDirectory =
				configuration.get(DeploymentOptionsInternal.CONF_DIR);
		//把配置路径设置到yarn日志配置
		YarnLogConfigUtil.setLogConfigFileInConfig(configuration, configurationDirectory);

		return getClusterDescriptor(configuration);
	}

	private YarnClusterDescriptor getClusterDescriptor(Configuration configuration) {
		/*TODO 创建了YarnClient*/
		final YarnClient yarnClient = YarnClient.createYarnClient();
		//创建初始化yarnConfiguration对象
		final YarnConfiguration yarnConfiguration = new YarnConfiguration();

		/*TODO 初始化、启动 YarnClient*/
		yarnClient.init(yarnConfiguration);
		yarnClient.start();

		//根据传进来的配置configuration，刚刚初始化的yarnConfiguration对象和yarnClient，创建一个yarn集群描述器
		return new YarnClusterDescriptor(
				configuration,
				yarnConfiguration,
				yarnClient,//这里的yarnClient已经把配置拿到了，见上几行yarnClient.init(yarnConfiguration);
				YarnClientYarnClusterInformationRetriever.create(yarnClient),
				false);
	}
//此时再次回到execute方法当中：AbstractJobClusterExecutor.execute(streamGraph, configuration, userClassloader)
//接下来开始获取yarn配置：final ClusterSpecification clusterSpecification = clusterClientFactory.getClusterSpecification(configuration);

/*
 *当前类：org.apache.flink.client.deployment.AbstractContainerizedClusterClientFactory
 *StreamingExecutionEnvironment.execute(jobName)->AbstractJobClusterExecutor.execute(streamGraph, configuration, userClassloader)->AbstractContainerizedClusterClientFactory.getClusterSpecification(Configuration configuration)
 */
	@Override
	public ClusterSpecification getClusterSpecification(Configuration configuration) {
		checkNotNull(configuration);

		final int jobManagerMemoryMB = JobManagerProcessUtils.processSpecFromConfigWithNewOptionToInterpretLegacyHeap(
				configuration,
				JobManagerOptions.TOTAL_PROCESS_MEMORY)
			.getTotalProcessMemorySize()
			.getMebiBytes();

		final int taskManagerMemoryMB = TaskExecutorProcessUtils
			.processSpecFromConfig(TaskExecutorProcessUtils.getConfigurationMapLegacyTaskManagerHeapSizeToConfigOption(
				configuration, TaskManagerOptions.TOTAL_PROCESS_MEMORY))
			.getTotalProcessMemorySize()
			.getMebiBytes();

		int slotsPerTaskManager = configuration.getInteger(TaskManagerOptions.NUM_TASK_SLOTS);

		return new ClusterSpecification.ClusterSpecificationBuilder()
			.setMasterMemoryMB(jobManagerMemoryMB)
			.setTaskManagerMemoryMB(taskManagerMemoryMB)
			.setSlotsPerTaskManager(slotsPerTaskManager)
			.createClusterSpecification();
	}
//这一步获取配置：JobManager内存、TaskManager内存、每个Tm的slot数，这些是flink集群特有的，后面要拿着这些配置到yarn上部署flink集群
//回到execute方法当中：AbstractJobClusterExecutor.execute(streamGraph, configuration, userClassloader)
//接下来要走到execute最后一步--准备部署集群：final ClusterClientProvider<ClusterID> clusterClientProvider = clusterDescriptor.deployJobCluster(clusterSpecification, jobGraph, configAccessor.getDetachedMode());