spark 2.0 SparkListener 继承关系

SparkListener的最上层接口是SparkListenerInterface，是一个Spark调度器的事件的处理接口，大多数程序不直接实现这个接口，而应该直接扩展SparkListener或者SparkFirehoseLIstener。

/**
 * Interface for listening to events from the Spark scheduler. Most applications should probably
 * extend SparkListener or SparkFirehoseListener directly, rather than implementing this class.
 *
 * Note that this is an internal interface which might change in different Spark releases.
 */
private[spark] trait SparkListenerInterface {

  /**
   * Called when a stage completes successfully or fails, with information on the completed stage.
   */
  def onStageCompleted(stageCompleted: SparkListenerStageCompleted): Unit

  /**
   * Called when a stage is submitted
   */
  def onStageSubmitted(stageSubmitted: SparkListenerStageSubmitted): Unit

  /**
   * Called when a task starts
   */
  def onTaskStart(taskStart: SparkListenerTaskStart): Unit

  /**
   * Called when a task begins remotely fetching its result (will not be called for tasks that do
   * not need to fetch the result remotely).
   */
  def onTaskGettingResult(taskGettingResult: SparkListenerTaskGettingResult): Unit

  /**
   * Called when a task ends
   */
  def onTaskEnd(taskEnd: SparkListenerTaskEnd): Unit

  /**
   * Called when a job starts
   */
  def onJobStart(jobStart: SparkListenerJobStart): Unit

  /**
   * Called when a job ends
   */
  def onJobEnd(jobEnd: SparkListenerJobEnd): Unit

  /**
   * Called when environment properties have been updated
   */
  def onEnvironmentUpdate(environmentUpdate: SparkListenerEnvironmentUpdate): Unit

  /**
   * Called when a new block manager has joined
   */
  def onBlockManagerAdded(blockManagerAdded: SparkListenerBlockManagerAdded): Unit

  /**
   * Called when an existing block manager has been removed
   */
  def onBlockManagerRemoved(blockManagerRemoved: SparkListenerBlockManagerRemoved): Unit

  /**
   * Called when an RDD is manually unpersisted by the application
   */
  def onUnpersistRDD(unpersistRDD: SparkListenerUnpersistRDD): Unit

  /**
   * Called when the application starts
   */
  def onApplicationStart(applicationStart: SparkListenerApplicationStart): Unit

  /**
   * Called when the application ends
   */
  def onApplicationEnd(applicationEnd: SparkListenerApplicationEnd): Unit

  /**
   * Called when the driver receives task metrics from an executor in a heartbeat.
   */
  def onExecutorMetricsUpdate(executorMetricsUpdate: SparkListenerExecutorMetricsUpdate): Unit

  /**
   * Called when the driver registers a new executor.
   */
  def onExecutorAdded(executorAdded: SparkListenerExecutorAdded): Unit

  /**
   * Called when the driver removes an executor.
   */
  def onExecutorRemoved(executorRemoved: SparkListenerExecutorRemoved): Unit

  /**
   * Called when the driver receives a block update info.
   */
  def onBlockUpdated(blockUpdated: SparkListenerBlockUpdated): Unit

  /**
   * Called when other events like SQL-specific events are posted.
   */
  def onOtherEvent(event: SparkListenerEvent): Unit
}

SparkListener是SparkListenerInterface的一个默认实现，这个实现对所有的回调都不做任何操作。

注意这个一个内部的接口，不同的spark版本中可能变动。

/**
 * :: DeveloperApi ::
 * A default implementation for [[SparkListenerInterface]] that has no-op implementations for
 * all callbacks.
 *
 * Note that this is an internal interface which might change in different Spark releases.
 */
@DeveloperApi
abstract class SparkListener extends SparkListenerInterface {
  override def onStageCompleted(stageCompleted: SparkListenerStageCompleted): Unit = { }

  override def onStageSubmitted(stageSubmitted: SparkListenerStageSubmitted): Unit = { }

  override def onTaskStart(taskStart: SparkListenerTaskStart): Unit = { }

  override def onTaskGettingResult(taskGettingResult: SparkListenerTaskGettingResult): Unit = { }

  override def onTaskEnd(taskEnd: SparkListenerTaskEnd): Unit = { }

  override def onJobStart(jobStart: SparkListenerJobStart): Unit = { }

  override def onJobEnd(jobEnd: SparkListenerJobEnd): Unit = { }

  override def onEnvironmentUpdate(environmentUpdate: SparkListenerEnvironmentUpdate): Unit = { }

  override def onBlockManagerAdded(blockManagerAdded: SparkListenerBlockManagerAdded): Unit = { }

  override def onBlockManagerRemoved(
      blockManagerRemoved: SparkListenerBlockManagerRemoved): Unit = { }

  override def onUnpersistRDD(unpersistRDD: SparkListenerUnpersistRDD): Unit = { }

  override def onApplicationStart(applicationStart: SparkListenerApplicationStart): Unit = { }

  override def onApplicationEnd(applicationEnd: SparkListenerApplicationEnd): Unit = { }

  override def onExecutorMetricsUpdate(
      executorMetricsUpdate: SparkListenerExecutorMetricsUpdate): Unit = { }

  override def onExecutorAdded(executorAdded: SparkListenerExecutorAdded): Unit = { }

  override def onExecutorRemoved(executorRemoved: SparkListenerExecutorRemoved): Unit = { }

  override def onBlockUpdated(blockUpdated: SparkListenerBlockUpdated): Unit = { }

  override def onOtherEvent(event: SparkListenerEvent): Unit = { }
}

SparkFirehoseListener允许用户接收所有的SparkListener事件，用户应该重写onEvent方法。

这个是一个具体的Java类，为了确保当在SparkListener中增加方法时我们不忘记更新此类，如果忘记则会导致编译错误。

/**
 * Class that allows users to receive all SparkListener events.
 * Users should override the onEvent method.
 *
 * This is a concrete Java class in order to ensure that we don't forget to update it when adding
 * new methods to SparkListener: forgetting to add a method will result in a compilation error (if
 * this was a concrete Scala class, default implementations of new event handlers would be inherited
 * from the SparkListener trait).
 */
public class SparkFirehoseListener implements SparkListenerInterface {

    public void onEvent(SparkListenerEvent event) { }

    @Override
    public final void onStageCompleted(SparkListenerStageCompleted stageCompleted) {
        onEvent(stageCompleted);
    }

    @Override
    public final void onStageSubmitted(SparkListenerStageSubmitted stageSubmitted) {
        onEvent(stageSubmitted);
    }

    @Override
    public final void onTaskStart(SparkListenerTaskStart taskStart) {
        onEvent(taskStart);
    }

    @Override
    public final void onTaskGettingResult(SparkListenerTaskGettingResult taskGettingResult) {
        onEvent(taskGettingResult);
    }

    @Override
    public final void onTaskEnd(SparkListenerTaskEnd taskEnd) {
        onEvent(taskEnd);
    }

    @Override
    public final void onJobStart(SparkListenerJobStart jobStart) {
        onEvent(jobStart);
    }

    @Override
    public final void onJobEnd(SparkListenerJobEnd jobEnd) {
        onEvent(jobEnd);
    }

    @Override
    public final void onEnvironmentUpdate(SparkListenerEnvironmentUpdate environmentUpdate) {
        onEvent(environmentUpdate);
    }

    @Override
    public final void onBlockManagerAdded(SparkListenerBlockManagerAdded blockManagerAdded) {
        onEvent(blockManagerAdded);
    }

    @Override
    public final void onBlockManagerRemoved(SparkListenerBlockManagerRemoved blockManagerRemoved) {
        onEvent(blockManagerRemoved);
    }

    @Override
    public final void onUnpersistRDD(SparkListenerUnpersistRDD unpersistRDD) {
        onEvent(unpersistRDD);
    }

    @Override
    public final void onApplicationStart(SparkListenerApplicationStart applicationStart) {
        onEvent(applicationStart);
    }

    @Override
    public final void onApplicationEnd(SparkListenerApplicationEnd applicationEnd) {
        onEvent(applicationEnd);
    }

    @Override
    public final void onExecutorMetricsUpdate(
            SparkListenerExecutorMetricsUpdate executorMetricsUpdate) {
        onEvent(executorMetricsUpdate);
    }

    @Override
    public final void onExecutorAdded(SparkListenerExecutorAdded executorAdded) {
        onEvent(executorAdded);
    }

    @Override
    public final void onExecutorRemoved(SparkListenerExecutorRemoved executorRemoved) {
        onEvent(executorRemoved);
    }

    @Override
    public void onBlockUpdated(SparkListenerBlockUpdated blockUpdated) {
        onEvent(blockUpdated);
    }

    @Override
    public void onOtherEvent(SparkListenerEvent event) {
        onEvent(event);
    }
}