rgw multisite 协程通过RGWCoroutinesManager::run启动,然后通过call或者spawn来添加新的op到调度队列。
- call的作用
通过call来执行的op,会将该op加入到stack的顶部,然后该op得到调用,只有最顶部的op执行完成,该op会被出栈,然后接着执行最顶部的op - spawn的作用
通过spawn来执行的op,会生成一个新的stack,并添加到当前的env->scheduled_stacks中,然后这些stack会被顺序调度。由于stack中的op可以是异步的,所以,通过op的异步来实现stack之间的并发执行。
代码解析如下:代码细节为涉及
RGWRemoteDataLog::run_sync(int num_shards) ---> run(new RGWDataSyncControlCR(&sc, num_shards, tn))
// class RGWRemoteDataLog : public RGWCoroutinesManager
RGWCoroutinesManager::run(RGWCoroutine *op) // *op = RGWDataSyncControlCR
list<RGWCoroutinesStack *> stacks;
RGWCoroutinesStack *stack = allocate_stack(); // 调用如下
new RGWCoroutinesStack(cct, this) // ops_mgr=this=&RGWRemoteDataLog
if (start) ops.push_back(start);
pos = ops.begin(); //ops.size() = 0
op->get();
stack->call(op); // RGWCoroutinesStack::call(RGWCoroutine *next_op) ,调用如下
ops.push_back(next_op); // ops 的内容, ops = [RGWDataSyncControlCR]
if (pos != ops.end()) ++pos; else pos = ops.begin(); // *pos = RGWDataSyncControlCR
stacks.push_back(stack);
r = run(stacks); //RGWCoroutinesManager::run(list<RGWCoroutinesStack *>& stacks)
RGWCoroutinesManager::run(list<RGWCoroutinesStack *>& stacks) {
int blocked_count = 0;
int interval_wait_count = 0;
bool canceled = false;
RGWCoroutinesEnv env;
bool op_not_blocked;
uint64_t run_context = ++run_context_count;
lock.lock();
set<RGWCoroutinesStack *>& context_stacks = run_contexts[run_context]; // RGWRemoteDataLog的run_contexts的赋值操作
list<RGWCoroutinesStack *> scheduled_stacks;
for (auto& st : stacks) {
context_stacks.insert(st);
scheduled_stacks.push_back(st);
st->set_is_scheduled(true); // 设置调度标记
}
env.run_context = run_context;
env.manager = this; // this=&RGWRemoteDataLog
env.scheduled_stacks = &scheduled_stacks; // 在_schedule() 中被修改
for (list<RGWCoroutinesStack *>::iterator iter = scheduled_stacks.begin(); iter != scheduled_stacks.end() && !going_down;) {
RGWCompletionManager::io_completion io;
RGWCoroutinesStack *stack = *iter;
++iter;
scheduled_stacks.pop_front();
env.stack = stack;
lock.unlock();
ret = stack->operate(&env); // RGWCoroutinesStack::operate,调用如下
RGWCoroutine *op = *pos; // RGWDataSyncControlCR
op->stack = this; // 给 RGWDataSyncControlCR 的父类RGWCoroutine中的stack赋值 RGWCoroutinesStack(cct, this) // this=&RGWRemoteDataLog
r = op->operate_wrapper(); // 执行operate,此时为:RGWBackoffControlCR::operate() ---> call(new RGWDataSyncCR(sc, num_shards, tn, backoff_ptr()))
stack->call(op); // ops = ops 的内容, ops = [RGWDataSyncControlCR, RGWDataSyncCR], *pos = RGWDataSyncCR
lock.lock();
stack->set_is_scheduled(false);
op_not_blocked = false;
if (stack->is_io_blocked()) {
if (stack->is_interval_waiting()) { interval_wait_count++;}
blocked_count++;}
else if (stack->is_blocked()) {}
else if (stack->is_done()) {
RGWCoroutinesStack *s;
while (stack->unblock_stack(&s)) {
if (!s->is_blocked_by_stack() && !s->is_done()) {
if (s->is_io_blocked()) {
if (stack->is_interval_waiting()) {
interval_wait_count++;
}
blocked_count++;
} else {
s->_schedule();
}
}
}
if (stack->parent && stack->parent->waiting_for_child()) {
stack->parent->set_wait_for_child(false);
stack->parent->_schedule();
}
context_stacks.erase(stack);
stack->put();
stack = NULL;
} else {
op_not_blocked = true;
stack->run_count++;
stack->_schedule(); // env->manager->_schedule(env, this); RGWCoroutinesManager::_schedule
if (!stack->is_scheduled) { // 当前没在调度
env->scheduled_stacks->push_back(stack); // 再把当前stack重新添加到scheduled_stacks
stack->set_is_scheduled(true);
}
set<RGWCoroutinesStack *>& context_stacks = run_contexts[env->run_context];
context_stacks.insert(stack); // 2个stack在context_stacks
}
if (!op_not_blocked && stack) { stack->run_count = 0;}
// completion_mgr = new RGWCompletionManager(cct);
while (completion_mgr->try_get_next(&io)) {
handle_unblocked_stack(context_stacks, scheduled_stacks, io, &blocked_count);
}
while (blocked_count - interval_wait_count >= ops_window) {
lock.unlock();
ret = completion_mgr->get_next(&io);
lock.lock();
handle_unblocked_stack(context_stacks, scheduled_stacks, io, &blocked_count);
}
next:
while (scheduled_stacks.empty() && blocked_count > 0) {
lock.unlock();
ret = completion_mgr->get_next(&io);
lock.lock();
if (ret < 0) {
ldout(cct, 5) << "completion_mgr.get_next() returned ret=" << ret << dendl;
}
if (going_down) {
ret = -ECANCELED;
canceled = true;
break;
}
handle_unblocked_stack(context_stacks, scheduled_stacks, io, &blocked_count);
iter = scheduled_stacks.begin();
}
if (canceled) break;
// 改变循环条件,直到scheduled_stacks为空
if (iter == scheduled_stacks.end()) iter = scheduled_stacks.begin();
}
for (auto stack : context_stacks) {
stack->cancel();
}
run_contexts.erase(run_context);
lock.unlock();
return ret;
}
RGWCoroutinesStack *RGWCoroutinesStack::spawn(RGWCoroutine *source_op, RGWCoroutine *op, bool wait) { // NULL, RGWSimpleRadosReadCR, true
if (!op) return NULL;
rgw_spawned_stacks *s = (source_op ? &source_op->spawned : &spawned);
RGWCoroutinesStack *stack = env->manager->allocate_stack(); // 重新生成一个stack
s->add_pending(stack);
stack->parent = this; // 新生成的stack作为当前stack的子
stack->get();
stack->call(op);
ops = [op]
env->manager->schedule(env, stack); // 开始调度stack,此时新生成的stack和之前的stack是并行执行的
if (wait) {
set_blocked_by(stack);
blocked_by_stack.insert(s);
s->blocking_stacks.insert(this);
}
return stack;
}
call和spawn的执行时序