Hadoop DataNode启动之DataBlockScanner

  存储在HDFS中的数据块每隔一段时间就会进行校验，因为由于硬件故障，系统BUG，异常断电等问题会导致数据块损坏，这时就需要HDFS采用其他块副本来修复。Hadoop为了感应块的损坏会启动一个后台线程周期性的对数据块进行检测，这个原理和客户端读取数据时的校验规则一样，以块为单位，每读取一个block，都要对其内的每一个chunk进行校验，默认情况下一个block大小为64M，每个chunk大小为512字节，下面我们从启动开始分析下这个过程
 

 DataBlockScanner在DN启动时创建，在startDataNode函数中

 void startDataNode(Configuration conf, 
                     AbstractList<File> dataDirs, SecureResources resources
                     ) throws IOException {
		......
    //initialize periodic block scanner
    String reason = null;
    //获得扫描周期，默认为三周，参数单位为小时
    if (conf.getInt("dfs.datanode.scan.period.hours", 0) < 0) {
      reason = "verification is turned off by configuration";
    } else if ( !(data instanceof FSDataset) ) {
      reason = "verifcation is supported only with FSDataset";
    } 
    if ( reason == null ) {
    	//开始创建块扫描器，但并为启动
      blockScanner = new DataBlockScanner(this, (FSDataset)data, conf);
    } else {
      LOG.info("Periodic Block Verification is disabled because " +
               reason + ".");
    }
    ......
 }

块扫描器的启动在DN主循环的offerService中

    public void offerService() throws Exception {
    LOG.info("using BLOCKREPORT_INTERVAL of " + blockReportInterval + "msec" + 
       " Initial delay: " + initialBlockReportDelay + "msec");
    //
    // Now loop for a long time....
    //
    while (shouldRun) {
      try {        
         ....
         ....
        // start block scanner
        if (blockScanner != null && blockScannerThread == null &&
            upgradeManager.isUpgradeCompleted()) {
          LOG.info("Starting Periodic block scanner.");
          //创建后台守护线程并启动
          blockScannerThread = new Daemon(blockScanner);
          blockScannerThread.start();
        }
         ....
         ....
  } // offerService

 块校验的具体流程包含在线程执行体中，大致分为三个部分 1、块集合初始化  2、更新block校验时间 3、循环校验负责规则的block。中间还有节流器控制

public void run() {
    try {
      //1、块集合初始化
      init();
      
      // 2、更新block校验时间
      if (!assignInitialVerificationTimes()) {
        return;
      }
      //节流器配置
      adjustThrottler();
      //3、循环校验负责规则的block
      while (datanode.shouldRun && !Thread.interrupted()) {
        long now = System.currentTimeMillis();
        synchronized (this) {
          if ( now >= (currentPeriodStart + scanPeriod)) {
          	//开启新的校验周期
            startNewPeriod();
          }
        }
        if ( (now - getEarliestScanTime()) >= scanPeriod ) {
        	//校验一个数据块
          verifyFirstBlock();
        } else {
          try {
          	//循环间隔1秒
            Thread.sleep(1000);
          } catch (InterruptedException ignored) {}
        }
      }
    } catch (RuntimeException e) {
      LOG.warn("RuntimeException during DataBlockScanner.run() : " +
               StringUtils.stringifyException(e));
      throw e;
    } finally {
      shutdown();
      LOG.info("Exiting DataBlockScanner thread.");
    }
  }

  先看初始化部分，他会获得当前块报告，构建扫描信息类，获得块校验日志文件

private void init() {
    // 获得块报告数组，并打乱顺序，以使块扫描不会总发生在同一个卷组上
    Block arr[] = dataset.getBlockReport();
    Collections.shuffle(Arrays.asList(arr));
    //创建存放扫描信息的结构
    blockInfoSet = new TreeSet<BlockScanInfo>();
    blockMap = new HashMap<Block, BlockScanInfo>();
    //填充扫描信息集合
    long scanTime = -1;
    for (Block block : arr) {
      BlockScanInfo info = new BlockScanInfo( block );
      info.lastScanTime = scanTime--; 
      //still keep 'info.lastScanType' to NONE.
      addBlockInfo(info);
    }


    //获得扫描日志
    File dir = null;
    FSDataset.FSVolume[] volumes = dataset.volumes.volumes;
    for(FSDataset.FSVolume vol : volumes) {
      if (LogFileHandler.isFilePresent(vol.getDir(), verificationLogFile)) {
        dir = vol.getDir();
        break;
      }
    }
    if (dir == null) {
      dir = volumes[0].getDir();
    }
    
    try {
      // 创建日志记录器
      verificationLog = new LogFileHandler(dir, verificationLogFile, 100);
    } catch (IOException e) {
      LOG.warn("Could not open verfication log. " +
               "Verification times are not stored.");
    }
    
    synchronized (this) {
    	//创建节流器
      throttler = new BlockTransferThrottler(200, MAX_SCAN_RATE);
    }
  }

  下面看更新block校验时间的部分assignInitialVerificationTimes

  private boolean assignInitialVerificationTimes() {
    int numBlocks = 1;
    synchronized (this) {
    	//获得块数量
      numBlocks = Math.max(blockMap.size(), 1);
    }
    
    //构建日志读取器
    LogFileHandler.Reader logReader = null;
    try {
      if (verificationLog != null) {
        logReader = verificationLog.new Reader(false);
      }
    } catch (IOException e) {
      LOG.warn("Could not read previous verification times : " +
               StringUtils.stringifyException(e));
    }
    //循环读取日志，并更新当前行数
    if (verificationLog != null) {
      verificationLog.updateCurNumLines();
    }
    
    try {
    // 根据校验日志更新块的校验时间
    while (logReader != null && logReader.hasNext()) {
      if (!datanode.shouldRun || Thread.interrupted()) {
        return false;
      }
      //获取日志条目并更新
      LogEntry entry = LogEntry.parseEntry(logReader.next());
      if (entry != null) {
        updateBlockInfo(entry);
      }
    }
    } finally {
    	//更新完毕后关闭日志流
      IOUtils.closeStream(logReader);
    }
    
    //两个块之间的校验间隔，避免频繁块扫描
    long verifyInterval = (long) (Math.min( scanPeriod/2.0/numBlocks,
                                            10*60*1000 ));
    //最后校验时间
    long lastScanTime = System.currentTimeMillis() - scanPeriod;
    
    /* Before this loop, entries in blockInfoSet that are not
     * updated above have lastScanTime of <= 0 . Loop until first entry has
     * lastModificationTime > 0.
     */    
    synchronized (this) {
      if (blockInfoSet.size() > 0 ) {
        BlockScanInfo info;
        //循环更新块校验时间和间隔
        while ((info =  blockInfoSet.first()).lastScanTime < 0) {
          delBlockInfo(info);        
          info.lastScanTime = lastScanTime;
          lastScanTime += verifyInterval;
          addBlockInfo(info);
        }
      }
    }   
    return true;
 }

下面看块校验部分，这也是blockscanner的主体部分，包含了校验逻辑，其实这个逻辑和上一篇dataXceiverServer中块读取的逻辑一样，只不过这里块数据读取后并不发送到客户端，而是发送到一个空流里，我们先看如何开始一个新的扫描周期

private synchronized void startNewPeriod() {
    LOG.info("Starting a new period : work left in prev period : " +
             String.format("%.2f%%", (bytesLeft * 100.0)/totalBytesToScan));
    // reset the byte counts :
    bytesLeft = totalBytesToScan; //记录需要扫描的字节数
    currentPeriodStart = System.currentTimeMillis();//记录本周期的启动时间
  }

块校验函数如下

  private void verifyFirstBlock() {
    Block block = null;
    synchronized (this) {
      if ( blockInfoSet.size() > 0 ) {
      	//获取一个block，所有blockinfo放在一个TreeSet中，所里内部是排序过的
        block = blockInfoSet.first().block;
      }
    }
    
    if ( block != null ) {
      verifyBlock(block);//开始校验
    }
  }  

在blockSender.sendBlock函数中会走上一篇dataXceiverServer中一样的块读取逻辑

  private void verifyBlock(Block block) {
    BlockSender blockSender = null;
    //安全起见，如果第一次校验失败则会再校验校验一次
    for (int i=0; i<2; i++) {
      boolean second = (i > 0);      
      try {
        adjustThrottler();
        //构建BlockSender
        blockSender = new BlockSender(block, 0, -1, false, 
                                               false, true, datanode);
				//构建空的数据输出流
        DataOutputStream out = 
                new DataOutputStream(new IOUtils.NullOutputStream());
        //发送一个块
        blockSender.sendBlock(out, null, throttler);
				//当块更新完毕后则记录日志
        LOG.info((second ? "Second " : "") +
                 "Verification succeeded for " + block);
        
        if ( second ) {
          totalTransientErrors++;
        }
        //更新扫描状态
        updateScanStatus(block, ScanType.VERIFICATION_SCAN, true);


        return;
      } catch (IOException e) {
				........
      } finally {
        IOUtils.closeStream(blockSender);
        datanode.getMetrics().incrBlocksVerified();
        totalScans++;
        totalVerifications++;
      }
    }
  }

一旦块扫描完毕，就会更新扫描状态，通过函数updateScanStatus实现，内部调用updateScanStatusInternal

 private synchronized void updateScanStatusInternal(Block block,
      ScanType type, boolean scanOk, boolean updateOnly) {
      	
    if (!isInitialized()) {
      return;
    }
    BlockScanInfo info = blockMap.get(block);
    
    if ( info != null ) {
      delBlockInfo(info);//删除旧的扫描信息
    } else {
      if (updateOnly) {
        return;
      }
      // It might already be removed. Thats ok, it will be caught next time.
      info = new BlockScanInfo(block);
    }
    //构建新的扫描信息
    long now = System.currentTimeMillis();
    info.lastScanType = type;
    info.lastScanTime = now;
    info.lastScanOk = scanOk;
    addBlockInfo(info);
    
    if (type == ScanType.REMOTE_READ) {
      totalVerifications++;
    }
        
    // Don't update meta data too often in case of REMOTE_READ
    // of if the verification failed.
    long diff = now - info.lastLogTime;
    if (!scanOk || (type == ScanType.REMOTE_READ &&
                    diff < scanPeriod/3 && diff < ONE_DAY)) {
      return;
    }
    
    info.lastLogTime = now;
    LogFileHandler log = verificationLog;
    if (log != null) {
    	//记录扫描日志
      log.appendLine(LogEntry.newEnry(block, now));
    }
  }

校验日志位于current目录下，文件名为dncp_block_verification.log.curr，校验日志格式如下：

date="2013-09-2316:49:22,824" 校验时间：日期时间格式，毫秒数

time="1379926162824"          与上面表示同一时间，此处为时间戳格式

genstamp="1002"               块标记

id="-8049298064173279059"     块ID

通过上面的信息就可以确定一个块精确到毫秒的校验时间，具体时间通过System.currentTimeMillis()来获得。