LiveListenerBus
LiveListenerBus继承了SparkListenerBus,而SparkListenerBus继承了ListenerBus。
LiveListenerBus会维护存放监听器的线程安全的CopyOnWriteArrayList,并对CopyOnWriteArrayList做增删改查的操作。
LiveListenerBus使用了消费者和生产者的思想,使用了信号量机制保证对特定资源操作数量的控制。
LiveListenerBus还使用了多线程,保证线程安全。
源码清单
创建一个SparkListenerEvent事件阻塞队列eventQueue,由config.scala源码看到默认LISTENER_BUS_EVENT_QUEUE_SIZE设为10000,也可以通过设置属性spark.scheduler.listenerbus.eventqueue.size来调解阻塞队列的大小
private lazy val EVENT_QUEUE_CAPACITY = validateAndGetQueueSize()
private lazy val eventQueue = new LinkedBlockingQueue[SparkListenerEvent](EVENT_QUEUE_CAPACITY)
private def validateAndGetQueueSize(): Int = {
val queueSize = sparkContext.conf.get(LISTENER_BUS_EVENT_QUEUE_SIZE)
if (queueSize <= 0) {
throw new SparkException("spark.scheduler.listenerbus.eventqueue.size must be > 0!")
}
queueSize
}
config.scala部分源码,
private[spark] val LISTENER_BUS_EVENT_QUEUE_SIZE =
ConfigBuilder("spark.scheduler.listenerbus.eventqueue.size")
.intConf
.createWithDefault(10000)
创建变量
/*started 用以标记LiveListenerBus是否启功的变量
*stopped 用以标记LiveListenerBus是否停止的变量
*started和stopped皆为AtomicBoolean类型,是为了在多线程的环境确保线程安全*/
private val started = new AtomicBoolean(false)
private val stopped = new AtomicBoolean(false)
//droppedEventsCounter 为AtomicLong类型,存放被丢弃的事件数量
private val droppedEventsCounter = new AtomicLong(0L)
//记录上一次记录日志的时间
@volatile private var lastReportTimestamp = 0L
// 标记事件总线中是否有事件正在被处理的变量
private var processingEvent = false
//因为事件超出队列容量而被丢弃的标记
private val logDroppedEvent = new AtomicBoolean(false)
// Semaphore信号量是对特定资源同时访问的控制方法
private val eventLock = new Semaphore(0)
创建接受事件的线程
private val listenerThread = new Thread(name) {
//设置线程守护线程
setDaemon(true)
//停止SparkContext
override def run(): Unit = Utils.tryOrStopSparkContext(sparkContext) {
LiveListenerBus.withinListenerThread.withValue(true) {
while (true) {
//接收新的事件
eventLock.acquire()
self.synchronized {
processingEvent = true
}
try {
//从队列中取出第一个事件,并从队列中删除
val event = eventQueue.poll
if (event == null) {
// 跳出WHILE循环,并停止守护线程会
if (!stopped.get) {
throw new IllegalStateException("Polling `null` from eventQueue means" +
" the listener bus has been stopped. So `stopped` must be true")
}
return
}
//把事件发送到所有的监听器,以找到与事件相匹配的监听器
postToAll(event)
} finally {
self.synchronized {
processingEvent = false
}
}
}
}
}
}
启动线程,开始接受事件并发送给监听器
def start(): Unit = {
//判断started是否为false
if (started.compareAndSet(false, true)) {
//启动线程
listenerThread.start()
} else {
throw new IllegalStateException(s"$name already started!")
}
}
在事件队列eventQueue中添加事件
def post(event: SparkListenerEvent): Unit = {
//判断事件总线是否停止
if (stopped.get) {
// Drop further events to make `listenerThread` exit ASAP
logError(s"$name has already stopped! Dropping event $event")
return
}
//在队列中添加事件,超出丢列容量是返回false
val eventAdded = eventQueue.offer(event)
if (eventAdded) {
//信号量机制,释放信号量
eventLock.release()
} else {
//超过队列容量,删除新的事件
onDropEvent(event)
droppedEventsCounter.incrementAndGet()
}
//在日志中记录事件被删除
val droppedEvents = droppedEventsCounter.get
if (droppedEvents > 0) {
// 若当前事件被删除事件与上一次事件被删除时间超过60秒才会在日志中记录,避免日志爆满
if (System.currentTimeMillis() - lastReportTimestamp >= 60 * 1000) {
// 使用compareAndSet方法可以确保线程安全,因为可能多个线程会同时减少droppedEventsCounter的数量
if (droppedEventsCounter.compareAndSet(droppedEvents, 0)) {
val prevLastReportTimestamp = lastReportTimestamp
lastReportTimestamp = System.currentTimeMillis()
//记录到日志中
logWarning(s"Dropped $droppedEvents SparkListenerEvents since " +
new java.util.Date(prevLastReportTimestamp))
}
}
}
}
停止监听总线,等待时间被处理完后,删除新的事件
def stop(): Unit = {
if (!started.get()) {
throw new IllegalStateException(s"Attempted to stop $name that has not yet started!")
}
if (stopped.compareAndSet(false, true)) {
eventLock.release()
listenerThread.join()
} else {
// Keep quiet
}
}