spark core 2.0 ExecutorSource Metrics统计

Spark executor 使用ExecutorSource统计Executor的各性能的使用指标。

private[spark]
class ExecutorSource(threadPool: ThreadPoolExecutor, executorId: String) extends Source {

  private def fileStats(scheme: String) : Option[FileSystem.Statistics] =
    FileSystem.getAllStatistics.asScala.find(s => s.getScheme.equals(scheme))

  private def registerFileSystemStat[T](
        scheme: String, name: String, f: FileSystem.Statistics => T, defaultValue: T) = {
    metricRegistry.register(MetricRegistry.name("filesystem", scheme, name), new Gauge[T] {
      override def getValue: T = fileStats(scheme).map(f).getOrElse(defaultValue)
    })
  }

  override val metricRegistry = new MetricRegistry()

  override val sourceName = "executor"

  // Gauge for executor thread pool's actively executing task counts
  metricRegistry.register(MetricRegistry.name("threadpool", "activeTasks"), new Gauge[Int] {
    override def getValue: Int = threadPool.getActiveCount()
  })

  // Gauge for executor thread pool's approximate total number of tasks that have been completed
  metricRegistry.register(MetricRegistry.name("threadpool", "completeTasks"), new Gauge[Long] {
    override def getValue: Long = threadPool.getCompletedTaskCount()
  })

  // Gauge for executor thread pool's current number of threads
  metricRegistry.register(MetricRegistry.name("threadpool", "currentPool_size"), new Gauge[Int] {
    override def getValue: Int = threadPool.getPoolSize()
  })

  // Gauge got executor thread pool's largest number of threads that have ever simultaneously
  // been in th pool
  metricRegistry.register(MetricRegistry.name("threadpool", "maxPool_size"), new Gauge[Int] {
    override def getValue: Int = threadPool.getMaximumPoolSize()
  })

  // Gauge for file system stats of this executor
  for (scheme <- Array("hdfs", "file")) {
    registerFileSystemStat(scheme, "read_bytes", _.getBytesRead(), 0L)
    registerFileSystemStat(scheme, "write_bytes", _.getBytesWritten(), 0L)
    registerFileSystemStat(scheme, "read_ops", _.getReadOps(), 0)
    registerFileSystemStat(scheme, "largeRead_ops", _.getLargeReadOps(), 0)
    registerFileSystemStat(scheme, "write_ops", _.getWriteOps(), 0)
  }
}

最后的registerFileSystemStat使用的是FileSystem.Statistics统计。如下所示，以getByteRead()为例。

    /**
     * Get the total number of bytes read
     * @return the number of bytes
     */
    public long getBytesRead() {
      return visitAll(new StatisticsAggregator<Long>() {
        private long bytesRead = 0;

        @Override
        public void accept(StatisticsData data) {
          bytesRead += data.bytesRead;
        }

        public Long aggregate() {
          return bytesRead;
        }
      });
    }

StatisticsAggregator是一个接口，如下所示。

 private interface StatisticsAggregator<T> {
      void accept(StatisticsData data);
      T aggregate();
    }

visitAll方法把这个统计对象相关的所有StatisticsData进行聚合，调用accept方法，如上面的getBytesRead，只对object的bytesRead属性进行聚合。

    /**
     * Apply the given aggregator to all StatisticsData objects associated with
     * this Statistics object.
     *
     * For each StatisticsData object, we will call accept on the visitor.
     * Finally, at the end, we will call aggregate to get the final total. 
     *
     * @param         The visitor to use.
     * @return        The total.
     */
    private synchronized <T> T visitAll(StatisticsAggregator<T> visitor) {
      visitor.accept(rootData);
      if (allData != null) {
        for (Iterator<StatisticsData> iter = allData.iterator();
            iter.hasNext(); ) {
          StatisticsData data = iter.next();
          visitor.accept(data);
          if (data.owner.get() == null) {
            /*
             * If the thread that created this thread-local data no
             * longer exists, remove the StatisticsData from our list
             * and fold the values into rootData.
             */
            rootData.add(data);
            iter.remove();
          }
        }
      }
      return visitor.aggregate();
    }

/**
     * Statistics data.
     * 
     * There is only a single writer to thread-local StatisticsData objects.
     * Hence, volatile is adequate here-- we do not need AtomicLong or similar
     * to prevent lost updates.
     * The Java specification guarantees that updates to volatile longs will
     * be perceived as atomic with respect to other threads, which is all we
     * need.
     */
    public static class StatisticsData {
      volatile long bytesRead;
      volatile long bytesWritten;
      volatile int readOps;
      volatile int largeReadOps;
      volatile int writeOps;
      /**
       * Stores a weak reference to the thread owning this StatisticsData.
       * This allows us to remove StatisticsData objects that pertain to
       * threads that no longer exist.
       */
      final WeakReference<Thread> owner;

      StatisticsData(WeakReference<Thread> owner) {
        this.owner = owner;
      }

      /**
       * Add another StatisticsData object to this one.
       */
      void add(StatisticsData other) {
        this.bytesRead += other.bytesRead;
        this.bytesWritten += other.bytesWritten;
        this.readOps += other.readOps;
        this.largeReadOps += other.largeReadOps;
        this.writeOps += other.writeOps;
      }

      /**
       * Negate the values of all statistics.
       */
      void negate() {
        this.bytesRead = -this.bytesRead;
        this.bytesWritten = -this.bytesWritten;
        this.readOps = -this.readOps;
        this.largeReadOps = -this.largeReadOps;
        this.writeOps = -this.writeOps;
      }

      @Override
      public String toString() {
        return bytesRead + " bytes read, " + bytesWritten + " bytes written, "
            + readOps + " read ops, " + largeReadOps + " large read ops, "
            + writeOps + " write ops";
      }
      
      public long getBytesRead() {
        return bytesRead;
      }
      
      public long getBytesWritten() {
        return bytesWritten;
      }
      
      public int getReadOps() {
        return readOps;
      }
      
      public int getLargeReadOps() {
        return largeReadOps;
      }
      
      public int getWriteOps() {
        return writeOps;
      }
    }