osd scrub的过程

在osd的init函数中有新建一个time，这个time的回调函数是C_Tick_WithoutOSDLock。这个回调函数中会对pg做scrub
int OSD::init()
{
tick_timer_without_osd_lock.add_event_after(cct->_conf->osd_heartbeat_interval, new C_Tick_WithoutOSDLock(this));
}
class OSD::C_Tick_WithoutOSDLock : public Context {
  OSD *osd;
  public:
  explicit C_Tick_WithoutOSDLock(OSD *o) : osd(o) {}
  void finish(int r) override {
  #这里又调用osd自身的tick_without_osd_lock
    osd->tick_without_osd_lock();
  }
};
void OSD::tick_without_osd_lock()
{
#可见这里只对actve的osd做scrub
  if (is_active()) {
    if (!scrub_random_backoff()) {
	#下面这个函数正式开启scrub的流程
      sched_scrub();
    }
	}
	}
void OSD::sched_scrub()
{
  // if not permitted, fail fast
  #如果不允许做scrub 直接返回
  if (!service.can_inc_scrubs_pending()) {
    return;
  }
  #
  if (!cct->_conf->osd_scrub_during_recovery && service.is_recovery_active()) {
    dout(20) << __func__ << " not scheduling scrubs due to active recovery" << dendl;
    return;
  }


  utime_t now = ceph_clock_now();
  bool time_permit = scrub_time_permit(now);
  bool load_is_low = scrub_load_below_threshold();
  dout(20) << "sched_scrub load_is_low=" << (int)load_is_low << dendl;

  OSDService::ScrubJob scrub;
  if (service.first_scrub_stamp(&scrub)) {
    do {
      dout(30) << "sched_scrub examine " << scrub.pgid << " at " << scrub.sched_time << dendl;

      if (scrub.sched_time > now) {
	// save ourselves some effort
	dout(10) << "sched_scrub " << scrub.pgid << " scheduled at " << scrub.sched_time
		 << " > " << now << dendl;
	break;
      }
	#scrub 是周期性的，时间不到的话，不会计算scrub，
	#如果当前cpu的loading 很高的话，也不做scrub。
      if ((scrub.deadline.is_zero() || scrub.deadline >= now) && !(time_permit && load_is_low)) {
        dout(10) << __func__ << " not scheduling scrub for " << scrub.pgid << " due to "
                 << (!time_permit ? "time not permit" : "high load") << dendl;
        continue;
      }
	#查找要做scrub的pg，从这里可以知道scrub是以pg为单位的
      PG *pg = _lookup_lock_pg(scrub.pgid);
      if (!pg)
	continue;
      dout(10) << "sched_scrub scrubbing " << scrub.pgid << " at " << scrub.sched_time
	       << (pg->get_must_scrub() ? ", explicitly requested" :
		   (load_is_low ? ", load_is_low" : " deadline < now"))
	       << dendl;
		   #调用pg的函数做scrub
      if (pg->sched_scrub()) {
	pg->unlock();
	break;
      }
      pg->unlock();
    } while (service.next_scrub_stamp(scrub, &scrub));
  }
  dout(20) << "sched_scrub done" << dendl;
}

bool PG::sched_scrub()
{
	#这个函数前面会有很多的判断，最终会调用来加入queue
      queue_scrub();
 
}

bool PG::queue_scrub()
{
  assert(is_locked());
  #如果正在做scrub 则退出
  if (is_scrubbing()) {
    return false;
  }
  #进入队列
   requeue_scrub();
  return true;
}

bool PG::requeue_scrub(bool high_priority)
{
  assert(is_locked());
  if (scrub_queued) {
    dout(10) << __func__ << ": already queued" << dendl;
    return false;
  } else {
    dout(10) << __func__ << ": queueing" << dendl;
    scrub_queued = true;
	#最终还是调用osd的函数入queue
    osd->queue_for_scrub(this, high_priority);
    return true;
  }
}
void OSDService::queue_for_scrub(PG *pg, bool with_high_priority)
{
  unsigned scrub_queue_priority = pg->scrubber.priority;
  if (with_high_priority && scrub_queue_priority < cct->_conf->osd_client_op_priority) {
    scrub_queue_priority = cct->_conf->osd_client_op_priority;
  }
  const auto epoch = pg->get_osdmap()->get_epoch();
  最终调用enqueue_back 来加入到op_shardedwq 的最后
  enqueue_back(
    OpQueueItem(
      unique_ptr<OpQueueItem::OpQueueable>(new PGScrub(pg->get_pgid(), epoch)),
      cct->_conf->osd_scrub_cost,
      scrub_queue_priority,
      ceph_clock_now(),
      0,
      epoch));
}
void OSDService::enqueue_back(OpQueueItem&& qi)
{
  osd->op_shardedwq.queue(std::move(qi));
}

最终会在PG的另外一个线程中从op_shardedwq 中来做scrub
void PG::scrub(epoch_t queued, ThreadPool::TPHandle &handle)
{
   chunky_scrub(handle);
}
在chunky_scrub 中会跑一个状态机其scrub的状态分别如下：
    enum State {
      INACTIVE,
      NEW_CHUNK,
      WAIT_PUSHES,
      WAIT_LAST_UPDATE,
      BUILD_MAP,
      WAIT_REPLICAS,
      COMPARE_MAPS,
      WAIT_DIGEST_UPDATES,
      FINISH,
    } state;