在 foreach rdd中执行,重新广播。可以基于job 批次数量 或者是 时间 public class BroadcastStringPeriodicUpdater { private static final int PERIOD = 60 * 1000; private static volatile BroadcastStringPeriodicUpdater instance; private Broadcast<String> broadcast; private long lastUpdate = 0L; private BroadcastStringPeriodicUpdater() {} public static BroadcastStringPeriodicUpdater getInstance() { if (instance == null) { synchronized (BroadcastStringPeriodicUpdater.class) { if (instance == null) { instance = new BroadcastStringPeriodicUpdater(); } } } return instance; } public String updateAndGet(SparkContext sc) { long now = System.currentTimeMillis(); long offset = now - lastUpdate; if (offset > PERIOD || broadcast == null) { if (broadcast != null) { broadcast.unpersist(); } lastUpdate = now; String value = fetchBroadcastValu
spark streaming动态修改广播变量
最新推荐文章于 2023-03-02 23:43:49 发布