rgw multisite 协程源码分析

rgw multisite 协程通过RGWCoroutinesManager::run启动，然后通过call或者spawn来添加新的op到调度队列。

• call的作用
  通过call来执行的op，会将该op加入到stack的顶部，然后该op得到调用，只有最顶部的op执行完成，该op会被出栈，然后接着执行最顶部的op
• spawn的作用
  通过spawn来执行的op，会生成一个新的stack，并添加到当前的env->scheduled_stacks中，然后这些stack会被顺序调度。由于stack中的op可以是异步的，所以，通过op的异步来实现stack之间的并发执行。
  代码解析如下：代码细节为涉及

RGWRemoteDataLog::run_sync(int num_shards)  ---> run(new RGWDataSyncControlCR(&sc, num_shards, tn))

// class RGWRemoteDataLog : public RGWCoroutinesManager
RGWCoroutinesManager::run(RGWCoroutine *op)   // *op = RGWDataSyncControlCR
  list<RGWCoroutinesStack *> stacks;
  RGWCoroutinesStack *stack = allocate_stack();   // 调用如下
    new RGWCoroutinesStack(cct, this)   // ops_mgr=this=&RGWRemoteDataLog
      if (start) ops.push_back(start);
      pos = ops.begin();  //ops.size() = 0
  op->get();
  stack->call(op);  // RGWCoroutinesStack::call(RGWCoroutine *next_op) ，调用如下
    ops.push_back(next_op);   // ops 的内容, ops = [RGWDataSyncControlCR]
    if (pos != ops.end()) ++pos; else pos = ops.begin();    // *pos = RGWDataSyncControlCR
  stacks.push_back(stack);
  r = run(stacks);  //RGWCoroutinesManager::run(list<RGWCoroutinesStack *>& stacks)

RGWCoroutinesManager::run(list<RGWCoroutinesStack *>& stacks) {
  int blocked_count = 0;
  int interval_wait_count = 0;
  bool canceled = false;
  RGWCoroutinesEnv env;
  bool op_not_blocked;
  uint64_t run_context = ++run_context_count;
  
  lock.lock();
  set<RGWCoroutinesStack *>& context_stacks = run_contexts[run_context];  // RGWRemoteDataLog的run_contexts的赋值操作
  list<RGWCoroutinesStack *> scheduled_stacks;
  for (auto& st : stacks) {
    context_stacks.insert(st);
    scheduled_stacks.push_back(st);
    st->set_is_scheduled(true);   // 设置调度标记
  }
  
  env.run_context = run_context;
  env.manager = this;         // this=&RGWRemoteDataLog
  env.scheduled_stacks = &scheduled_stacks;   // 在_schedule() 中被修改

  for (list<RGWCoroutinesStack *>::iterator iter = scheduled_stacks.begin(); iter != scheduled_stacks.end() && !going_down;) {
    RGWCompletionManager::io_completion io;
    RGWCoroutinesStack *stack = *iter;  
    ++iter;
    scheduled_stacks.pop_front();
    env.stack = stack;
    lock.unlock();

    ret = stack->operate(&env);   // RGWCoroutinesStack::operate，调用如下
      RGWCoroutine *op = *pos;    // RGWDataSyncControlCR
      op->stack = this;           // 给 RGWDataSyncControlCR 的父类RGWCoroutine中的stack赋值 RGWCoroutinesStack(cct, this)   // this=&RGWRemoteDataLog
      r = op->operate_wrapper();  // 执行operate，此时为：RGWBackoffControlCR::operate() ---> call(new RGWDataSyncCR(sc, num_shards, tn, backoff_ptr()))
        stack->call(op);  // ops = ops 的内容, ops = [RGWDataSyncControlCR, RGWDataSyncCR],  *pos = RGWDataSyncCR
    lock.lock();
    
    stack->set_is_scheduled(false);
    op_not_blocked = false;

    if (stack->is_io_blocked()) {
      if (stack->is_interval_waiting()) { interval_wait_count++;}
      blocked_count++;} 
    else if (stack->is_blocked()) {} 
    else if (stack->is_done()) {
      RGWCoroutinesStack *s;
      while (stack->unblock_stack(&s)) {
        if (!s->is_blocked_by_stack() && !s->is_done()) {
          if (s->is_io_blocked()) {
              if (stack->is_interval_waiting()) {
                interval_wait_count++;
              }
            blocked_count++;
          } else {
            s->_schedule();
          }
        }
      }
      
      if (stack->parent && stack->parent->waiting_for_child()) {
        stack->parent->set_wait_for_child(false);
        stack->parent->_schedule();
      }
      context_stacks.erase(stack);
      stack->put();
      stack = NULL;
    } else {
      op_not_blocked = true;
      stack->run_count++;
      stack->_schedule();   // env->manager->_schedule(env, this); RGWCoroutinesManager::_schedule
        if (!stack->is_scheduled) { // 当前没在调度
          env->scheduled_stacks->push_back(stack);  // 再把当前stack重新添加到scheduled_stacks
          stack->set_is_scheduled(true);
        }
        set<RGWCoroutinesStack *>& context_stacks = run_contexts[env->run_context];
        context_stacks.insert(stack); // 2个stack在context_stacks
    }

    if (!op_not_blocked && stack) { stack->run_count = 0;}

    // completion_mgr = new RGWCompletionManager(cct);
    while (completion_mgr->try_get_next(&io)) {
      handle_unblocked_stack(context_stacks, scheduled_stacks, io, &blocked_count);
    }

    while (blocked_count - interval_wait_count >= ops_window) {
      lock.unlock();
      ret = completion_mgr->get_next(&io);
      lock.lock();
      handle_unblocked_stack(context_stacks, scheduled_stacks, io, &blocked_count);
    }

next:
    while (scheduled_stacks.empty() && blocked_count > 0) {
      lock.unlock();
      ret = completion_mgr->get_next(&io);
      lock.lock();
      if (ret < 0) {
        ldout(cct, 5) << "completion_mgr.get_next() returned ret=" << ret << dendl;
      }
      if (going_down) {
        ret = -ECANCELED;
        canceled = true;
        break;
      }
      handle_unblocked_stack(context_stacks, scheduled_stacks, io, &blocked_count);
      iter = scheduled_stacks.begin();
    }
    
    if (canceled) break;
    // 改变循环条件，直到scheduled_stacks为空
    if (iter == scheduled_stacks.end()) iter = scheduled_stacks.begin();
  }

  for (auto stack : context_stacks) {
    stack->cancel();
  }
  
  run_contexts.erase(run_context);
  lock.unlock();

  return ret;
}
  


RGWCoroutinesStack *RGWCoroutinesStack::spawn(RGWCoroutine *source_op, RGWCoroutine *op, bool wait) { // NULL, RGWSimpleRadosReadCR, true
  if (!op) return NULL;
  
  rgw_spawned_stacks *s = (source_op ? &source_op->spawned : &spawned);
  RGWCoroutinesStack *stack = env->manager->allocate_stack();   // 重新生成一个stack
  s->add_pending(stack);
  stack->parent = this;   // 新生成的stack作为当前stack的子
  stack->get();
  stack->call(op); 
    ops = [op]

  env->manager->schedule(env, stack); // 开始调度stack，此时新生成的stack和之前的stack是并行执行的

  if (wait) {
    set_blocked_by(stack);
      blocked_by_stack.insert(s);
      s->blocking_stacks.insert(this);
  }

  return stack;
}

call和spawn的执行时序