主流程
命令行程序入口:实际最终执行org.apache.flink.client.cli.CliFrontend#parseAndRun
/** Submits the job based on the arguments. */
public static void main(final String[] args) {
EnvironmentInformation.logEnvironmentInfo(LOG, "Command Line Client", args);
// 1. find the configuration directory
final String configurationDirectory = getConfigurationDirectoryFromEnv();
// 2. load the global configuration
final Configuration configuration =
GlobalConfiguration.loadConfiguration(configurationDirectory);
// 3. load the custom command lines
final List<CustomCommandLine> customCommandLines =
loadCustomCommandLines(configuration, configurationDirectory);
try {
final CliFrontend cli = new CliFrontend(configuration, customCommandLines);
SecurityUtils.install(new SecurityConfiguration(cli.configuration));
int retCode =
SecurityUtils.getInstalledContext().runSecured(() -> cli.parseAndRun(args));
System.exit(retCode);
} catch (Throwable t) {
final Throwable strippedThrowable =
ExceptionUtils.stripException(t, UndeclaredThrowableException.class);
LOG.error("Fatal error while running command line interface.", strippedThrowable);
strippedThrowable.printStackTrace();
System.exit(31);
}
}
针对参数为run的,执行org.apache.flink.client.cli.CliFrontend#run
protected void run(String[] args) throws Exception {
LOG.info("Running 'run' command.");
final Options commandOptions = CliFrontendParser.getRunCommandOptions();
final CommandLine commandLine = getCommandLine(commandOptions, args, true);
// evaluate help flag
if (commandLine.hasOption(HELP_OPTION.getOpt())) {
CliFrontendParser.printHelpForRun(customCommandLines);
return;
}
final CustomCommandLine activeCommandLine =
validateAndGetActiveCommandLine(checkNotNull(commandLine));
final ProgramOptions programOptions = ProgramOptions.create(commandLine);
final List<URL> jobJars = getJobJarAndDependencies(programOptions);
final Configuration effectiveConfiguration =
getEffectiveConfiguration(activeCommandLine, commandLine, programOptions, jobJars);
LOG.debug("Effective executor configuration: {}", effectiveConfiguration);
try (PackagedProgram program = getPackagedProgram(programOptions, effectiveConfiguration)) {
executeProgram(effectiveConfiguration, program);
}
}
发起执行分支
PackagedProgram构建
private PackagedProgram(
@Nullable File jarFile,
List<URL> classpaths,
@Nullable String entryPointClassName,
Configuration configuration,
SavepointRestoreSettings savepointRestoreSettings,
String... args)
throws ProgramInvocationException {
this.classpaths = checkNotNull(classpaths);
this.savepointSettings = checkNotNull(savepointRestoreSettings);
this.args = checkNotNull(args);
checkArgument(
jarFile != null || entryPointClassName != null,
"Either the jarFile or the entryPointClassName needs to be non-null.");
// whether the job is a Python job.
this.isPython = isPython(entryPointClassName);
// load the jar file if exists
this.jarFile = loadJarFile(jarFile);
assert this.jarFile != null || entryPointClassName != null;
// now that we have an entry point, we can extract the nested jar files (if any)
this.extractedTempLibraries =
this.jarFile == null
? Collections.emptyList()
: extractContainedLibraries(this.jarFile);
this.userCodeClassLoader =
ClientUtils.buildUserCodeClassLoader(
getJobJarAndDependencies(),
classpaths,
getClass().getClassLoader(),
configuration);
// load the entry point class
this.mainClass =
loadMainClass(
// if no entryPointClassName name was given, we try and look one up through
// the manifest
entryPointClassName != null
? entryPointClassName
: getEntryPointClassNameFromJar(this.jarFile),
userCodeClassLoader);
if (!hasMainMethod(mainClass)) {
throw new ProgramInvocationException(
"The given program class does not have a main(String[]) method.");
}
}
userCodeClassLoader构建:
this.userCodeClassLoader =
ClientUtils.buildUserCodeClassLoader(
getJobJarAndDependencies(),
classpaths,
getClass().getClassLoader(),
configuration);
public static URLClassLoader buildUserCodeClassLoader(
List<URL> jars, List<URL> classpaths, ClassLoader parent, Configuration configuration) {
URL[] urls = new URL[jars.size() + classpaths.size()];
for (int i = 0; i < jars.size(); i++) {
urls[i] = jars.get(i);
}
for (int i = 0; i < classpaths.size(); i++) {
urls[i + jars.size()] = classpaths.get(i);
}
final String[] alwaysParentFirstLoaderPatterns =
CoreOptions.getParentFirstLoaderPatterns(configuration);
final String classLoaderResolveOrder =
configuration.getString(CoreOptions.CLASSLOADER_RESOLVE_ORDER);
FlinkUserCodeClassLoaders.ResolveOrder resolveOrder =
FlinkUserCodeClassLoaders.ResolveOrder.fromString(classLoaderResolveOrder);
final boolean checkClassloaderLeak =
configuration.getBoolean(CoreOptions.CHECK_LEAKED_CLASSLOADER);
return FlinkUserCodeClassLoaders.create(
resolveOrder,
urls,
parent,
alwaysParentFirstLoaderPatterns,
NOOP_EXCEPTION_HANDLER,
checkClassloaderLeak);
}
// child-first模式下最终调用
// class ChildFirstClassLoader extends FlinkUserCodeClassLoader
public ChildFirstClassLoader(
URL[] urls,
ClassLoader parent,
String[] alwaysParentFirstPatterns,
Consumer<Throwable> classLoadingExceptionHandler) {
super(urls, parent, classLoadingExceptionHandler);
this.alwaysParentFirstPatterns = alwaysParentFirstPatterns;
}
// ChildFirstClassLoader 继承自URLClassLoader
public URLClassLoader(URL[] urls, ClassLoader parent) {
super(parent);
// this is to make the stack depth consistent with 1.1
SecurityManager security = System.getSecurityManager();
if (security != null) {
security.checkCreateClassLoader();
}
this.acc = AccessController.getContext();
ucp = new URLClassPath(urls, acc);
}
主入口main方法所在类加载:
private static Class<?> loadMainClass(String className, ClassLoader cl) throws ProgramInvocationException { ClassLoader contextCl = null; try { contextCl = Thread.currentThread().getContextClassLoader(); Thread.currentThread().setContextClassLoader(cl); return Class.forName(className, false, cl); } catch.... finally { if (contextCl != null) { Thread.currentThread().setContextClassLoader(contextCl); } }
1.替换当前线程的上下文classLoader为userCodeClassLoader
2.加载main函数所在类
3.恢复当前线程的上下文classLoader