Hadoop yarn源码分析（十二） ResourceLocalizationService源码解析 2021SC@SDUSC

2021SC@SDUSC

一、简介

ResourceLocalizationService主要负责Container所需资源的本地化。它能够按照描述从hdfs上下载Container所需的文件资源， 并分散到各个磁盘上以防止出现访问热点。 此外， 它会为下载的文件添加访问控制限制， 并为之施加合适的磁盘空间使用份额。
Yarn将资源分为public、 private和application三类， 不同级别的资源对不同用户和应用程序的访问权限不同， 这也直接导致资源的本地化方式不同。 尽管它们的本地化由ResourceLocalizationService服务完成， 但内部由不同线程负载下载， 即所有应用程序的PUBLIC资源由ResourceLocalizationService内部专门的线程PublicLocalizer下载完成， 该线程内部维护了一个线程池以加快资源下载速度， 而每个应用程序的每种private或application资源分别由一个专门的线程LocalizerRunner负责下载。 该线程将启动一个资源下载客户端ContainerLocalizer， 该客户端通过RPC协议LocalizationProtocol访问ResourceLocalizationService服务， 以获取待下载的资源。

二、属性

2.1 LocalizationProtocol 协议

heartbeat心跳方法

public interface LocalizationProtocol {
  public LocalizerHeartbeatResponse heartbeat(LocalizerStatus status)
      throws YarnException, IOException;
}

2.2 属性

  //私有目录
  public static final String NM_PRIVATE_DIR = "nmPrivate";
  //私有目录权限700
  public static final FsPermission NM_PRIVATE_PERM = new FsPermission((short) 0700);
  //共有目录权限755
  private static final FsPermission PUBLIC_FILECACHE_FOLDER_PERMS =
      new FsPermission((short) 0755);

  private Server server;
  private InetSocketAddress localizationServerAddress;
  //缓存大小
  @VisibleForTesting
  long cacheTargetSize;
  //处理器缓存清理程序
  private long cacheCleanupPeriod;

  private final ContainerExecutor exec;
  protected final Dispatcher dispatcher;
  private final DeletionService delService;
  private LocalizerTracker localizerTracker;
  private RecordFactory recordFactory;
  private final ScheduledExecutorService cacheCleanup;
  //权限相关
  private LocalizerTokenSecretManager secretManager;
  //持久化存储
  private NMStateStoreService stateStore;
  //度量数据
  @VisibleForTesting
  final NodeManagerMetrics metrics;

  @VisibleForTesting
  LocalResourcesTracker publicRsrc;

  private LocalDirsHandlerService dirsHandler;
  private DirsChangeListener localDirsChangeListener;
  private DirsChangeListener logDirsChangeListener;
  private Context nmContext;
  private DiskValidator diskValidator;

  //私有资源的由用户名键入的LocalResourceTracker映射
  @VisibleForTesting
  final ConcurrentMap<String, LocalResourcesTracker> privateRsrc =
    new ConcurrentHashMap<String,LocalResourcesTracker>();

  //应用程序资源的由appid键入的LocalResourceTracker映射。
  private final ConcurrentMap<String,LocalResourcesTracker> appRsrc =
    new ConcurrentHashMap<String,LocalResourcesTracker>();
  
  FileContext lfs;

三、主要方法

3.1 构造方法

ResourceLocalizationService的实例对象是由ContainerManagerImpl中的createResourceLocalizationService()方法构建。在该方法中，对一些重要属性进行了赋值。构建了一个HadoopScheduledThreadPoolExecutor类的cacheCleanup变量，专门用来清理无效缓存。

  public ResourceLocalizationService(Dispatcher dispatcher,
      ContainerExecutor exec, DeletionService delService,
      LocalDirsHandlerService dirsHandler, Context context,
      NodeManagerMetrics metrics) {

    super(ResourceLocalizationService.class.getName());
    this.exec = exec;
    this.dispatcher = dispatcher;
    this.delService = delService;
    this.dirsHandler = dirsHandler;
    //清理无效缓存
    this.cacheCleanup = new HadoopScheduledThreadPoolExecutor(1,
        new ThreadFactoryBuilder()
          .setNameFormat("ResourceLocalizationService Cache Cleanup")
          .build());
    this.stateStore = context.getNMStateStore();
    this.nmContext = context;
    this.metrics = metrics;
  }

3.2 初始化方法

serviceInit()方法，对整个文件进行初始化，设置缓存大小、处理周期等。

  public void serviceInit(Configuration conf) throws Exception {
    this.validateConf(conf);
    this.publicRsrc = new LocalResourcesTrackerImpl(null, null, dispatcher,
        true, conf, stateStore, dirsHandler);
    this.recordFactory = RecordFactoryProvider.getRecordFactory(conf);

    try {
      //构建本地文件系统
      lfs = getLocalFileContext(conf);
      //设置默认权限，UMask权限
      lfs.setUMask(new FsPermission((short) FsPermission.DEFAULT_UMASK));

      if (!stateStore.canRecover()|| stateStore.isNewlyCreated()) {
        cleanUpLocalDirs(lfs, delService);
        cleanupLogDirs(lfs, delService);
        initializeLocalDirs(lfs);
        initializeLogDirs(lfs);
      }
    } catch (Exception e) {
      throw new YarnRuntimeException(
        "Failed to initialize LocalizationService", e);
    }

    diskValidator = DiskValidatorFactory.getInstance(
        YarnConfiguration.DEFAULT_DISK_VALIDATOR);
    //缓存大小
    cacheTargetSize =
      conf.getLong(YarnConfiguration.NM_LOCALIZER_CACHE_TARGET_SIZE_MB, YarnConfiguration.DEFAULT_NM_LOCALIZER_CACHE_TARGET_SIZE_MB) << 20;
    //清理周期
    cacheCleanupPeriod =
      conf.getLong(YarnConfiguration.NM_LOCALIZER_CACHE_CLEANUP_INTERVAL_MS, YarnConfiguration.DEFAULT_NM_LOCALIZER_CACHE_CLEANUP_INTERVAL_MS);
    localizationServerAddress = conf.getSocketAddr(
        YarnConfiguration.NM_BIND_HOST,
        YarnConfiguration.NM_LOCALIZER_ADDRESS,
        YarnConfiguration.DEFAULT_NM_LOCALIZER_ADDRESS,
        YarnConfiguration.DEFAULT_NM_LOCALIZER_PORT);

    localizerTracker = createLocalizerTracker(conf);
    addService(localizerTracker);
    dispatcher.register(LocalizerEventType.class, localizerTracker);
    //本地文件修改监听器
    localDirsChangeListener = new DirsChangeListener() {
      @Override
      public void onDirsChanged() {
        checkAndInitializeLocalDirs();
      }
    };
    //日志文件修改监听器
    logDirsChangeListener = new DirsChangeListener() {
      @Override
      public void onDirsChanged() {
        initializeLogDirs(lfs);
      }
    };
    super.serviceInit(conf);
  }

3.3 开始方法

serviceStart()方法，定时清理，默认使用cacheCleanup.scheduleWithFixedDelay来清理

  public void serviceStart() throws Exception {
    //开启定时清理
    cacheCleanup.scheduleWithFixedDelay(new CacheCleanup(dispatcher),
        cacheCleanupPeriod, cacheCleanupPeriod, TimeUnit.MILLISECONDS);
    //构建RPC服务
    server = createServer();
    //启动ROC服务
    server.start();
    localizationServerAddress =
        getConfig().updateConnectAddr(YarnConfiguration.NM_BIND_HOST,
                                      YarnConfiguration.NM_LOCALIZER_ADDRESS,
                                      YarnConfiguration.DEFAULT_NM_LOCALIZER_ADDRESS,
                                      server.getListenerAddress());
    LOG.info("Localizer started on port " + server.getPort());
    super.serviceStart();
    //注册本地文件监听程序
    dirsHandler.registerLocalDirsChangeListener(localDirsChangeListener);
    //注册日志文件监听程序
    dirsHandler.registerLogDirsChangeListener(logDirsChangeListener);
  }

四、清理缓存

4.1 CacheCleanup类

4.1.1 基本属性

CacheCleanup是ResourceLocalizationService 的内部类,用于定时清理缓存，有两个非常重要的默认值：
缓存大小cacheTargetSize和清理周期cacheCleanupPeriod

//缓存大小为10G
cacheTargetSize = conf.getLong(YarnConfiguration.NM_LOCALIZER_CACHE_TARGET_SIZE_MB, YarnConfiguration.DEFAULT_NM_LOCALIZER_CACHE_TARGET_SIZE_MB) << 20;
//清理周期为10min
cacheCleanupPeriod =  conf.getLong(YarnConfiguration.NM_LOCALIZER_CACHE_CLEANUP_INTERVAL_MS, YarnConfiguration.DEFAULT_NM_LOCALIZER_CACHE_CLEANUP_INTERVAL_MS);

  static class CacheCleanup extends Thread {

    private final Dispatcher dispatcher;

    public CacheCleanup(Dispatcher dispatcher) {
      super("CacheCleanup");
      this.dispatcher = dispatcher;
    }

4.2.2 run()方法

run方法，调用ResourceLocalizationService中的handle方法中switch语句的CLEAN_UP分支

@Override
    @SuppressWarnings("unchecked") // dispatcher not typed
    public void run() {
      dispatcher.getEventHandler().handle(
          new LocalizationEvent(LocalizationEventType.CACHE_CLEANUP));
    }

handle()方法

@Override
  public void handle(LocalizationEvent event) {
    // TODO: create log dir as $logdir/$user/$appId
    switch (event.getType()) {
    case INIT_APPLICATION_RESOURCES:
      handleInitApplicationResources(
          ((ApplicationLocalizationEvent)event).getApplication());
      break;
    case LOCALIZE_CONTAINER_RESOURCES:
      handleInitContainerResources((ContainerLocalizationRequestEvent) event);
      break;
    case CONTAINER_RESOURCES_LOCALIZED:
      handleContainerResourcesLocalized((ContainerLocalizationEvent) event);
      break;
    case CACHE_CLEANUP:
      handleCacheCleanup();
      break;
    case CLEANUP_CONTAINER_RESOURCES:
      handleCleanupContainerResources((ContainerLocalizationCleanupEvent)event);
      break;
    case DESTROY_APPLICATION_RESOURCES:
      handleDestroyApplicationResources(
          ((ApplicationLocalizationEvent)event).getApplication());
      break;
    default:
      throw new YarnRuntimeException("Unknown localization event: " + event);
    }
  }

4.3 LocalCacheCleaner类

处理本地缓存

4.3.1 属性

  //当前服务大小
  private long currentSize;
  //目标服务大小
  private final long targetSize;
  //删除服务
  private final DeletionService delService;
  //资源映射 LocalizedResource --> LocalResourcesTracker
  private final SortedMap<LocalizedResource, LocalResourcesTracker> resourceMap;

4.3.2 构造方法

使用了LRU缓存，用SortedMap类, 重写了compare方法。

  LocalCacheCleaner(DeletionService delService, long targetSize) {
    //使用LRU缓存
    this(delService, targetSize, new LRUComparator());
  }

  LocalCacheCleaner(DeletionService delService, long targetSize,
      Comparator<? super LocalizedResource> cmp) {
    this(delService, targetSize,
        new TreeMap<LocalizedResource, LocalResourcesTracker>(cmp));
  }
  //映射和服务赋值
  LocalCacheCleaner(DeletionService delService, long targetSize,
      SortedMap<LocalizedResource, LocalResourcesTracker> resourceMap) {
    this.resourceMap = resourceMap;
    this.delService = delService;
    this.targetSize = targetSize;
  }

4.3.3 LRUComparator

LRU策略，首先对比文件的时间，然后再对比hashcode

private static class LRUComparator implements Comparator<LocalizedResource>,
      Serializable {

    private static final long serialVersionUID = 7034380228434701685L;

    public int compare(LocalizedResource r1, LocalizedResource r2) {
      long ret = r1.getTimestamp() - r2.getTimestamp();
      if (0 == ret) {
        return System.identityHashCode(r1) - System.identityHashCode(r2);
      }
      return ret > 0 ? 1 : -1;
    }
  }

4.3.4 添加资源监控

从传递的LocalResourcesTracker中添加要从缓存中删除的资源。

  public void addResources(LocalResourcesTracker newTracker) {
    //newTracker 将传递的LocalResourcesTracker跟踪的所有资源添加到LocalCacheCleaner
    for (LocalizedResource resource : newTracker) {
      //累加新增资源，获取当前资源大小
      currentSize += resource.getSize();
      if (resource.getRefCount() > 0) {
        //如果资源被引用，忽略
        continue;
      }
      //将资源添加到对应的缓存中
      resourceMap.put(resource, newTracker);
    }
  }

4.3.5 清理缓存

按照用于构造此类的比较器生成的排序顺序从缓存中删除资源

  public LocalCacheCleanerStats cleanCache() {
    //根据当前的资源大小,构建状态
    LocalCacheCleanerStats stats = new LocalCacheCleanerStats(currentSize);
    //for循环 持续的清理, 直到达到目标缓存的大小。
    for (Iterator<Map.Entry<LocalizedResource, LocalResourcesTracker>> i =
        resourceMap.entrySet().iterator();
        currentSize - stats.totalDelSize > targetSize && i.hasNext();) {
      Map.Entry<LocalizedResource, LocalResourcesTracker> rsrc = i.next();
      LocalizedResource resource = rsrc.getKey();
      LocalResourcesTracker tracker = rsrc.getValue();
      if (tracker.remove(resource, delService)) {
        //使用delService服务清理资源
        stats.incDelSize(tracker.getUser(), resource.getSize());
      }
    }
    this.resourceMap.clear();
    // 返回值 统计此调用cleanCache期间清理的内容
    return stats;
  }