实际项目中需要使用更新广播变量的方式来满足需求,其代码具体如下:
@NotProguard
public class WbBroadcastWrapper {
Broadcast<List<String>> broadcastBlackWhiteList = null;
private static WbBroadcastWrapper obj = new WbBroadcastWrapper();
private WbBroadcastWrapper(){}
public static WbBroadcastWrapper getInstance() {
return obj;
}
public JavaSparkContext getSparkContext(SparkContext sc) {
JavaSparkContext jsc = JavaSparkContext.fromSparkContext(sc);
return jsc;
}
public Broadcast<List<String>> updateAndGetWbBroadCast(SparkContext sparkContext,Map<String,String> params) {
if(broadcastBlackWhiteList == null || judgeBlackAndWhiteIsUpdate(params)) {
JRedisClusterHelper instance = JRedisClusterHelper.getInstance();
String path = params.get("redis.jedisPoolConfig.urls");
JedisCluster jedisCluster = instance.getJedisCluster(path);
String blackWhiteSql = jedisCluster.hget("black_white_list", "1");
DbFetcher dbFetcher = new DbFetcher();
List<String> fetchSqlList = dbFetcher.fetchSqlList(blackWhiteSql,params);
if(broadcastBlackWhiteList != null) {
broadcastBlackWhiteList.unpersist();
}
broadcastBlackWhiteList = getSparkContext(sparkContext).broadcast(fetchSqlList);
}
return broadcastBlackWhiteList;
}
public static boolean judgeBlackAndWhiteIsUpdate(Map<String,String> parmas) {
JRedisClusterHelper instance = JRedisClusterHelper.getInstance();
String path = parmas.get("redis.jedisPoolConfig.urls");
JedisCluster jedisCluster = instance.getJedisCluster(path);
String modified = jedisCluster.get(FinalParameterHelper.BLACK_AND_WHITE_MODIFIED);
if(modified != null && Integer.parseInt(modified.trim()) == 1) {
jedisCluster.set(FinalParameterHelper.BLACK_AND_WHITE_MODIFIED, "0");
return true;
}
return false;
}
}