Flink 动态加载 Jar 包，实现自定义算子加载执行

动态加载Jar

对应的参数：

• path：Jar 的存放路径。
• env：Flink 的环境实例。
• classPath：动态Jar的类路径。

import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.net.URL;
import java.net.URLClassLoader;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;

import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;

import com.ygsoft.dataprocess.exception.LoadFunctionException;

/**
 * 动态加载jar
 * @author yinlilan
 *
 */
public class LoadFunction {

	public static Class<?> loadJar(final String path, final StreamExecutionEnvironment env, final String classPath) throws LoadFunctionException {
		
		final String newPath = "file:///" + path;
		
		try {
			loadJar(new URL(newPath));

			Field configuration = StreamExecutionEnvironment.class.getDeclaredField("configuration");
			configuration.setAccessible(true);
			Configuration o = (Configuration) configuration.get(env);

			Field confData = Configuration.class.getDeclaredField("confData");
			confData.setAccessible(true);
			
			@SuppressWarnings("unchecked")
			Map<String, Object> temp = (Map<String, Object>) confData.get(o);
			List<String> jarList = new ArrayList<>();
			jarList.add(newPath);
			temp.put("pipeline.classpaths", jarList);
			
			Class<?> result = Class.forName(classPath);
			
			return result;
		} catch(Exception e) {
			throw new LoadFunctionException("加载动态jar包异常");
		}
	}

	// 动态加载Jar
	private static void loadJar(final URL jarUrl) {
		// 从URLClassLoader类加载器中获取类的addURL方法
		Method method = null;
		try {
			method = URLClassLoader.class.getDeclaredMethod("addURL", URL.class);
		} catch (NoSuchMethodException | SecurityException e1) {
			e1.printStackTrace();
		}
		// 获取方法的访问权限
		boolean accessible = method.isAccessible();
		try {
			// 修改访问权限为可写
			if (accessible == false) {
				method.setAccessible(true);
			}
			// 获取系统类加载器
			URLClassLoader classLoader = (URLClassLoader) ClassLoader.getSystemClassLoader();
			// jar路径加入到系统url路径里
			method.invoke(classLoader, jarUrl);
		} catch (Exception e) {
			e.printStackTrace();
		} finally {
			method.setAccessible(accessible);
		}
	}
}

动态Jar调用方式

import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.util.Map;

import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;

import com.ygsoft.dataprocess.exception.LoadFunctionException;
import com.ygsoft.dataprocess.load.LoadFunction;
import com.ygsoft.dataprocess.sdk.domain.NodeParam;
import com.ygsoft.dataprocess.sdk.flow.IFlow;
import com.ygsoft.dataprocess.vo.flow.CustomCondition;

/**
 * 自定义算子实现
 * @author yinlilan
 *
 */
public class CustomConditionFlow implements IFlow {

	private StreamExecutionEnvironment env;
	
	public CustomConditionFlow(StreamExecutionEnvironment env) {
		this.env = env;
	}
	
	@Override
	public DataStream<Map<String, String>> execute(final DataStream<Map<String, String>> input, final NodeParam param) {
		final Object nodeParams = param.getParams();
		final CustomCondition condition = new CustomCondition(nodeParams);
		
		final String jarPath = condition.getJarPath();
		final String classPath = condition.getClassPath();
		final String methodName = condition.getMethod();
		
		try {
			Class<?> c = LoadFunction.loadJar(jarPath, env, classPath);
			
			Object obj = c.newInstance();
			
			// 获取方法对象
			Method method = obj.getClass().getMethod(methodName, DataStream.class, param.getClass());
			
			// 执行方法
			@SuppressWarnings("unchecked")
			DataStream<Map<String, String>> result = (DataStream<Map<String, String>>) method.invoke(obj, input, param);
			
			return result;
		} catch (LoadFunctionException e) {
			e.printStackTrace();
			throw new RuntimeException(e.getMessage());
		} catch (InstantiationException e) {
			e.printStackTrace();
			throw new RuntimeException(e);
		} catch (IllegalAccessException e) {
			e.printStackTrace();
			throw new RuntimeException(e);
		} catch (NoSuchMethodException e) {
			e.printStackTrace();
			throw new RuntimeException(e);
		} catch (SecurityException e) {
			e.printStackTrace();
			throw new RuntimeException(e);
		} catch (IllegalArgumentException e) {
			e.printStackTrace();
			throw new RuntimeException(e);
		} catch (InvocationTargetException e) {
			e.printStackTrace();
			throw new RuntimeException(e);
		}
	}

}