Ceph版本:14.2.22
PrimaryLogPG::do_request
文件路径:ceph/src/osd/PrimaryLogPG.cc
OSD在收到客户端发送的请求时,会调用do_request函数,该函数是PrimaryLogPG类的方法。在该函数中,OSD会针对客户端请求类型执行不同操作。在正常情况下,OSD会调用do_op函数来处理客户端的请求。
void PrimaryLogPG::do_request(OpRequestRef &op, ThreadPool::TPHandle &handle)
{
if (op->osd_trace)
{
op->pg_trace.init("pg op", &trace_endpoint, &op->osd_trace);
op->pg_trace.event("do request");
}
// make sure we have a new enough map
//确保请求队列中的请求op是最新的
auto p = waiting_for_map.find(op->get_source());
//如果找到op,需要更新请求队列,如果没有找到,说明当前op就是最新的op,不需要入队
if (p != waiting_for_map.end())
{
// preserve ordering
dout(20) << __func__ << " waiting_for_map " << p->first << " not empty, queueing" << dendl;
p->second.push_back(op);
op->mark_delayed("waiting_for_map not empty");
return;
}
if (!have_same_or_newer_map(op->min_epoch))
{
dout(20) << __func__ << " min " << op->min_epoch << ", queue on waiting_for_map " << op->get_source() << dendl;
waiting_for_map[op->get_source()].push_back(op);
op->mark_delayed("op must wait for map");
osd->request_osdmap_update(op->min_epoch);
return;
}
//判断当前op是否可以取消,如果是,直接返回
if (can_discard_request(op))
{
return;
}
// pg-wide backoffs
const Message *m = op->get_req();//构建Messaged对象指针
int msg_type = m->get_type();//获取消息类型
if (m->get_connection()->has_feature(CEPH_FEATURE_RADOS_BACKOFF))
{
SessionRef session{static_cast<Session *>(m->get_connection()->get_priv().get())};
if (!session)
return; // drop it.
if (msg_type == CEPH_MSG_OSD_OP)
{
if (session->check_backoff(cct, info.pgid, info.pgid.pgid.get_hobj_start(), m))
{
return;
}
bool backoff = is_down() || is_incomplete() || (!is_active() && is_peered());
if (g_conf()->osd_backoff_on_peering && !backoff)
{
if (is_peering())
{
backoff = true;
}
}
if (backoff)
{
add_pg_backoff(session);
return;
}
}
// pg backoff acks at pg-level
if (msg_type == CEPH_MSG_OSD_BACKOFF)
{
const MOSDBackoff *ba = static_cast<const MOSDBackoff *>(m);
if (ba->begin != ba->end)
{
handle_backoff(op);
return;
}
}
}
//检查PG的状态,如果当前PG处于peering,需要等待
if (!is_peered())
{
// Delay unless PGBackend says it's ok
//判断PG后端在inactive状态下是否可以处理op,如果可以,就调用pgbackend->handle_message处理
if (pgbackend->can_handle_while_inactive(op))
{
bool handled = pgbackend->handle_message(op);
ceph_assert(handled);
return;
}
else
{//如果不可以,需要将当前op挂到waiting_for_peered队列中
waiting_for_peered.push_back(op);
op->mark_delayed("waiting for peered");
return;
}
}
//如果正在flush,需要将当前op挂到waiting_for_flush队列中
if (flushes_in_progress > 0)
{
dout(20) << flushes_in_progress << " flushes_in_progress pending "
<< "waiting for flush on " << op << dendl;
waiting_for_flush.push_back(op);
op->mark_delayed("waiting for flush");
return;
}
ceph_assert(is_peered() && flushes_in_progress == 0);
//如果PG后端在处理当前op,直接返回
if (pgbackend->handle_message(op))
return;
//如果PG后端没有处理当前op,根据消息类型,执行相关操作
switch (msg_type)
{
case CEPH_MSG_OSD_OP:
case CEPH_MSG_OSD_BACKOFF:
if (!is_active())
{
//如果当前PG处于非active状态,需要挂起op到waiting_for_active队列中
dout(20) << " peered, not active, waiting for active on " << op << dendl;
waiting_for_active.push_back(op);
op->mark_delayed("waiting for active");
return;
}
//如果当前PG处于正常状态
switch (msg_type)
{
case CEPH_MSG_OSD_OP:
// verify client features
if ((pool.info.has_tiers() || pool.info.is_tier()) && !op->has_feature(CEPH_FEATURE_OSD_CACHEPOOL))
{
//如果是关于分层存储的请求,直接返回
osd->reply_op_error(op, -EOPNOTSUPP);
return;
}
do_op(op);//处理当前op
break;
case CEPH_MSG_OSD_BACKOFF:
// object-level backoff acks handled in osdop context
handle_backoff(op);
break;
}
break;
case MSG_OSD_PG_SCAN:
do_scan(op, handle);
break;
case MSG_OSD_PG_BACKFILL:
do_backfill(op);
break;
case MSG_OSD_PG_BACKFILL_REMOVE:
do_backfill_remove(op);
break;
case MSG_OSD_SCRUB_RESERVE: {
const MOSDScrubReserve *m = static_cast<const MOSDScrubReserve *>(op->get_req());
switch (m->type)
{
case MOSDScrubReserve::REQUEST:
handle_scrub_reserve_request(op);
break;
case MOSDScrubReserve::GRANT:
handle_scrub_reserve_grant(op, m->from);
break;
case MOSDScrubReserve::REJECT:
handle_scrub_reserve_reject(op, m->from);
break;
case MOSDScrubReserve::RELEASE:
handle_scrub_reserve_release(op);
break;
}
}
break;
case MSG_OSD_REP_SCRUB:
replica_scrub(op, handle);
break;
case MSG_OSD_REP_SCRUBMAP:
do_replica_scrub_map(op);
break;
case MSG_OSD_PG_UPDATE_LOG_MISSING:
do_update_log_missing(op);
break;
case MSG_OSD_PG_UPDATE_LOG_MISSING_REPLY:
do_update_log_missing_reply(op);
break;
default:
ceph_abort_msg("bad message type in do_request");
}
}
PrimaryLogPG::do_op
文件路径:ceph/src/osd/PrimaryLogPG.cc
do_op函数是PrimaryLogPG类的方法,该函数是OSD处理请求的通用函数。在处理cache tier时,主要通过调用agent_choose_mode和maybe_handle_cache 函数。其中agent_choose_mode是用来计算cache pool的flush mode和evict mode,并触发cache pool的flush和evict操作。maybe_handle_cache是用来处理代理读写以及是否要将读写的数据promote到cache pool中。
void PrimaryLogPG::do_op(OpRequestRef &op)
{
...
...
//hit set相关设置
bool in_hit_set = false;
if (hit_set)
{
if (obc.get())
{
if (obc->obs.oi.soid != hobject_t() && hit_set->contains(obc->obs.oi.soid))
in_hit_set = true;
}
else
{
if (missing_oid != hobject_t() && hit_set->contains(missing_oid))
in_hit_set = true;
}
if (!op->hitset_inserted)
{
hit_set->insert(oid);
op->hitset_inserted = true;
if (hit_set->is_full() || hit_set_start_stamp + pool.info.hit_set_period <= m->get_recv_stamp())
{
hit_set_persist();
}
}
}
//cache tier agent相关处理
if (agent_state)
{
if (agent_choose_mode(false, op))
return;
}
if (obc.get() && obc->obs.exists && obc->obs.oi.has_manifest())
{
if (maybe_handle_manifest(op, write_ordered, obc))
return;
}
//cache tier 处理,如果cache pool 命中 object,则无需处理
//如果 cache pool 没有命中,则需要根据 cache tier 策略读写数据
if (maybe_handle_cache(op, write_ordered, obc, r, missing_oid, false, in_hit_set))
return;
//cache tier 处理结果
if (r && (r != -ENOENT || !obc))
{
// copy the reqids for copy get on ENOENT
if (r == -ENOENT && (m->ops[0].op.op == CEPH_OSD_OP_COPY_GET))
{
fill_in_copy_get_noent(op, oid, m->ops[0]);
return;
}
dout(20) << __func__ << ": find_object_context got error " << r << dendl;
if (op->may_write() && get_osdmap()->require_osd_release >= CEPH_RELEASE_KRAKEN)
{
record_write_error(op, oid, nullptr, r);
}
else
{
osd->reply_op_error(op, r);
}
return;
}
...
...
}
PrimaryLogPG::maybe_handle_cache
文件路径:ceph/src/osd/PrimaryLogPG.cc
maybe_handle_cache是PrimaryLogPG类的方法,实际上该函数并不是处理缓存读写数据的真正函数,它只是一个接口。它通过调用maybe_handle_cache_detail函数来实现具体的操作。
bool maybe_handle_cache(OpRequestRef op, bool write_ordered, ObjectContextRef obc, int r,
const hobject_t &missing_oid, bool must_promote, bool in_hit_set = false)
{
// 如果返回值是cache_result_t::NOOP,说明对 cache tier 没有任何处理
return cache_result_t::NOOP !=
maybe_handle_cache_detail(op, write_ordered, obc, r, missing_oid, must_promote, in_hit_set, nullptr);
}
PrimaryLogPG::cache_result_t PrimaryLogPG::maybe_handle_cache_detail
文件路径:ceph/src/osd/PrimaryLogPG.cc
maybe_handle_cache_detail是PrimaryLogPG类的方法,在该函数是处理cache tier的真正入口函数。在该函数中,会提前检测要读写的object是否在当前OSD的本地中存在,如果有,就不需要对cache tier做任何操作。如果没有,则会根据cache tier策略执行代理读写。同时也会根据promote策略决定是否需要将读写的数据promote到cache pool中。
注:我在调试代码的时候,发现writeback模式下的新写流程和官网说的不一致。官网在描述writeback模式下,数据先写入到cache pool中,然后由cache pool中flush到base pool中。实际上调试和读源码发现,源码是先将数据写入到base pool中,然后在由base pool中promote到cache pool中,与官网说的截然相反。
PrimaryLogPG::cache_result_t PrimaryLogPG::maybe_handle_cache_detail(OpRequestRef op, bool write_ordered,
ObjectContextRef obc, int r, hobject_t missing_oid,
bool must_promote, bool in_hit_set,
ObjectContextRef *promote_obc)
{
//检测请求是否需要从 cache pool 中读取数据
if (op && op->get_req() && op->get_req()->get_type() == CEPH_MSG_OSD_OP &&
(static_cast<const MOSDOp *>(op->get_req())->get_flags() & CEPH_OSD_FLAG_IGNORE_CACHE))
{
dout(20) << __func__ << ": ignoring cache due to flag" << dendl;
return cache_result_t::NOOP;
}
//函数走到这里,说明请求中需要从 cache pool 中读写数据,下一步需要检查有没有设置 cache tier
//检测是否根据 cache tier 有没有正确设置
//这里的pool不分data pool还是cache pool,因为osd可能在data pool,也可能在cache pool
// return quickly if caching is not enabled
if (pool.info.cache_mode == pg_pool_t::CACHEMODE_NONE)
return cache_result_t::NOOP;
// promote标志
must_promote = must_promote || op->need_promote();
if (obc)
dout(25) << __func__ << " " << obc->obs.oi << " " << (obc->obs.exists ? "exists" : "DNE") << " missing_oid "
<< missing_oid << " must_promote " << (int)must_promote << " in_hit_set " << (int)in_hit_set << dendl;
else
dout(25) << __func__ << " (no obc)"
<< " missing_oid " << missing_oid << " must_promote " << (int)must_promote << " in_hit_set "
<< (int)in_hit_set << dendl;
// 函数执行到这里,说明当前 OSD 一定是 cache pool 中的 OSD,此时的 pool 一定是 cache pool
// 根据 object 上下文检测当前 object 是否处于 blocked 状态,如果是,则需要等待
// if it is write-ordered and blocked, stop now
if (obc.get() && obc->is_blocked() && write_ordered)
{
// we're already doing something with this object
dout(20) << __func__ << " blocked on " << obc->obs.oi.soid << dendl;
return cache_result_t::NOOP;
}
//检测当前object是否被标记为从集群中删除,如果是,就不能对当前object操作
//这里的删除指的是逻辑上的删除,真正的删除操作会由定时任务去完成
if (r == -ENOENT && missing_oid == hobject_t())
{
// we know this object is logically absent (e.g., an undefined clone)
return cache_result_t::NOOP;
}
//检测当前 object 是否在已经在当前 OSD 本地中存在,也就是是否在 cache pool 中
//如果已经命中,就不需要对 cache tier 做任何处理
if (obc.get() && obc->obs.exists)
{
osd->logger->inc(l_osd_op_cache_hit);
return cache_result_t::NOOP;
}
//如果当前 OSD 不是 primary OSD,则需要将请求转发给 primary OSD 处理
//从另外一方面可以证明此时的 object 很有可能是副本
if (!is_primary())
{
dout(20) << __func__ << " cache miss; ask the primary" << dendl;
osd->reply_op_error(op, -EAGAIN);//将错误码 `-EAGAIN` 作为响应返回给客户端
return cache_result_t::REPLIED_WITH_EAGAIN;
}
//如果当前 object 被标记为从集群中删除,但是当前 object 的上下文依然存在
//说明当前 object 实际上并未真正的从集群中删除掉
//对于这种状态的 object,依然是可以读的
if (missing_oid == hobject_t() && obc.get())
{
missing_oid = obc->obs.oi.soid;
}
//走到这里,说明当前 OSD(cache pool) 中确实不存在当前 object
const MOSDOp *m = static_cast<const MOSDOp *>(op->get_req());
//获取当前 object 在集群中的存储位置相关信息
const object_locator_t oloc = m->get_object_locator();
//检测请求是不是需要跳过处理缓存,如果需要跳过,就不需要处理缓存
if (op->need_skip_handle_cache())
{
return cache_result_t::NOOP;
}
OpRequestRef promote_op;
//下面是处理 cache tier 的逻辑,如果设置了 cache tier, 将根据 cache tier 策略读取数据
switch (pool.info.cache_mode)
{
//writeback模式
case pg_pool_t::CACHEMODE_WRITEBACK:
//如果 cache pool 已经处理 full 状态,此时的请求只能读
if (agent_state && agent_state->evict_mode == TierAgentState::EVICT_MODE_FULL)
{
//如果当前请求不是write,也不是将当前请求缓存起来,说明当前请求可能是read
//如果当前请求是read,则执行代理读
if (!op->may_write() && !op->may_cache() && !write_ordered && !must_promote)
{
dout(20) << __func__ << " cache pool full, proxying read" << dendl;
do_proxy_read(op);//代理读
return cache_result_t::HANDLED_PROXY;
}
//走到这里,说明当前请求不是read,可能是write,需要将当前请求加入请求等待队列中
dout(20) << __func__ << " cache pool full, waiting" << dendl;
block_write_on_full_cache(missing_oid, op);
return cache_result_t::BLOCKED_FULL;
}
//走到这里,说明cache pool不是处于full状态,可以执行write或者read
//如果当前object需要promote到当前 cache pool 中
//或者当前osd内存缓存中没有命中当前object
//需要将当前object promote到 cache pool,并阻塞当前请求,直到当前object被promote到 cache pool中
if (must_promote || (!hit_set && !op->need_skip_promote()))
{
promote_object(obc, missing_oid, oloc, op, promote_obc);
return cache_result_t::BLOCKED_PROMOTE;
}
//走到这里,说明没有执行promote操作
// 处理write请求
if (op->may_write() || op->may_cache())
{
// 执行代理写
do_proxy_write(op);
// Promote too?
//是否要将写入的数据 promote 到 cache pool 中
if (!op->need_skip_promote() &&
maybe_promote(obc, missing_oid, oloc, in_hit_set, pool.info.min_write_recency_for_promote,
OpRequestRef(), promote_obc))
{
return cache_result_t::BLOCKED_PROMOTE;
}
return cache_result_t::HANDLED_PROXY;
}
// 处理read请求
else
{
//执行代理读
do_proxy_read(op);
// Avoid duplicate promotion
if (obc.get() && obc->is_blocked())
{
if (promote_obc)
*promote_obc = obc;
return cache_result_t::BLOCKED_PROMOTE;
}
// Promote too?
if (!op->need_skip_promote())
{
(void)maybe_promote(obc, missing_oid, oloc, in_hit_set, pool.info.min_read_recency_for_promote,
promote_op, promote_obc);
}
return cache_result_t::HANDLED_PROXY;
}
//走到这里,说明没有处理任何请求
ceph_abort_msg("unreachable");
return cache_result_t::NOOP;
//forward模式
case pg_pool_t::CACHEMODE_FORWARD:
// FIXME: this mode allows requests to be reordered.
do_cache_redirect(op);
return cache_result_t::HANDLED_REDIRECT;
//readonly模式
case pg_pool_t::CACHEMODE_READONLY:
// TODO: clean this case up
if (!obc.get() && r == -ENOENT)
{
// we don't have the object and op's a read
promote_object(obc, missing_oid, oloc, op, promote_obc);
return cache_result_t::BLOCKED_PROMOTE;
}
if (!r)
{ // it must be a write
do_cache_redirect(op);
return cache_result_t::HANDLED_REDIRECT;
}
// crap, there was a failure of some kind
return cache_result_t::NOOP;
//readforward模式
case pg_pool_t::CACHEMODE_READFORWARD:
// Do writeback to the cache tier for writes
if (op->may_write() || write_ordered || must_promote)
{
if (agent_state && agent_state->evict_mode == TierAgentState::EVICT_MODE_FULL)
{
dout(20) << __func__ << " cache pool full, waiting" << dendl;
block_write_on_full_cache(missing_oid, op);
return cache_result_t::BLOCKED_FULL;
}
promote_object(obc, missing_oid, oloc, op, promote_obc);
return cache_result_t::BLOCKED_PROMOTE;
}
// If it is a read, we can read, we need to forward it
do_cache_redirect(op);
return cache_result_t::HANDLED_REDIRECT;
//proxy模式
case pg_pool_t::CACHEMODE_PROXY:
if (!must_promote)
{
if (op->may_write() || op->may_cache() || write_ordered)
{
do_proxy_write(op);
return cache_result_t::HANDLED_PROXY;
}
else
{
do_proxy_read(op);
return cache_result_t::HANDLED_PROXY;
}
}
// ugh, we're forced to promote.
if (agent_state && agent_state->evict_mode == TierAgentState::EVICT_MODE_FULL)
{
dout(20) << __func__ << " cache pool full, waiting" << dendl;
block_write_on_full_cache(missing_oid, op);
return cache_result_t::BLOCKED_FULL;
}
promote_object(obc, missing_oid, oloc, op, promote_obc);
return cache_result_t::BLOCKED_PROMOTE;
//readproxy模式
case pg_pool_t::CACHEMODE_READPROXY:
// Do writeback to the cache tier for writes
if (op->may_write() || write_ordered || must_promote)
{
if (agent_state && agent_state->evict_mode == TierAgentState::EVICT_MODE_FULL)
{
dout(20) << __func__ << " cache pool full, waiting" << dendl;
block_write_on_full_cache(missing_oid, op);
return cache_result_t::BLOCKED_FULL;
}
promote_object(obc, missing_oid, oloc, op, promote_obc);
return cache_result_t::BLOCKED_PROMOTE;
}
// If it is a read, we can read, we need to proxy it
do_proxy_read(op);
return cache_result_t::HANDLED_PROXY;
default:
ceph_abort_msg("unrecognized cache_mode");
}
// 走到这里, 说明cache tier模式没有正确设置,导致对cache tier没有任何操作处理
return cache_result_t::NOOP;
}
PrimaryLogPG::promote_object
文件路径: ceph/src/osd/PrimaryLogPG.cc
void PrimaryLogPG::promote_object(ObjectContextRef obc, const hobject_t &missing_oid, const object_locator_t &oloc,
OpRequestRef op, ObjectContextRef *promote_obc)
{
hobject_t hoid = obc ? obc->obs.oi.soid : missing_oid;
ceph_assert(hoid != hobject_t());
//Scrub是Ceph中的一项数据完整性检查工作,会检查存储池中所有对象的数据是否正确.
//如果某个对象正在进行Scrub操作,那么对该对象的读写请求就会被阻塞,直到Scrub完成为止.
//检查对象是否被Scrub阻塞,如果是,则将操作请求放入等待队列中,等待Scrub完成后再处理.
if (write_blocked_by_scrub(hoid))
{
dout(10) << __func__ << " " << hoid << " blocked by scrub" << dendl;
if (op)
{
waiting_for_scrub.push_back(op);//将操作请求放入等待队列中
op->mark_delayed("waiting for scrub");
dout(10) << __func__ << " " << hoid << " placing op in waiting_for_scrub" << dendl;
}
else
{
dout(10) << __func__ << " " << hoid << " no op, dropping on the floor" << dendl;
}
return;
}
//创建对象上下文,该操作是针对 cache pool 中不存在的对象
if (!obc)
{ // we need to create an ObjectContext
ceph_assert(missing_oid != hobject_t());
obc = get_object_context(missing_oid, true);
}
// 是否需要promote obc
if (promote_obc)
*promote_obc = obc;
/*
* Before promote complete, if there are proxy-reads for the object,
* for this case we don't use DONTNEED.
*/
//设置对象数据的读取策略
unsigned src_fadvise_flags = LIBRADOS_OP_FLAG_FADVISE_SEQUENTIAL;
map<hobject_t, list<OpRequestRef>>::iterator q = in_progress_proxy_ops.find(obc->obs.oi.soid);
if (q == in_progress_proxy_ops.end())
{
src_fadvise_flags |= LIBRADOS_OP_FLAG_FADVISE_DONTNEED;
}
CopyCallback *cb;
object_locator_t my_oloc;
hobject_t src_hoid;
//下面是创建promote结束后的回调函数
//回调函数在start_copy函数执行完成后被调用
//如果对象全部位于同一个OSD上
if (!obc->obs.oi.has_manifest())
{
my_oloc = oloc;
my_oloc.pool = pool.info.tier_of;//获取 base pool
src_hoid = obc->obs.oi.soid;//表示复制操作的源对象ID
cb = new PromoteCallback(obc, this);
}
else
{//如果对象被分解成多块,分散到其他OSD中
if (obc->obs.oi.manifest.is_chunked())
{
src_hoid = obc->obs.oi.soid;
cb = new PromoteManifestCallback(obc, this);
}
else if (obc->obs.oi.manifest.is_redirect())
{
object_locator_t src_oloc(obc->obs.oi.manifest.redirect_target);
my_oloc = src_oloc;
src_hoid = obc->obs.oi.manifest.redirect_target;
cb = new PromoteCallback(obc, this);
}
else
{
ceph_abort_msg("unrecognized manifest type");
}
}
unsigned flags = CEPH_OSD_COPY_FROM_FLAG_IGNORE_OVERLAY | CEPH_OSD_COPY_FROM_FLAG_IGNORE_CACHE |
CEPH_OSD_COPY_FROM_FLAG_MAP_SNAP_CLONE | CEPH_OSD_COPY_FROM_FLAG_RWORDERED;
//执行复制操作
start_copy(cb, obc, src_hoid, my_oloc, 0, flags, obc->obs.oi.soid.snap == CEPH_NOSNAP, src_fadvise_flags, 0);
//确保对象已经被blocked,防止被其他操作使用
ceph_assert(obc->is_blocked());
//在 promote_object() 函数中,需要对对象进行数据复制和版本更新等操作,这些操作可能会影响对象的状态.
//为了避免在操作期间出现未预期的错误,需要将 obc 对象标记为 blocked 状态,以防止其他操作同时修改对象
//如果对象已经被标记为 blocked 状态,则函数立即返回:否则,函数会一直等待
if (op)
wait_for_blocked_object(obc->obs.oi.soid, op);
info.stats.stats.sum.num_promote++;
}
由于时间问题,文档还在努力的完善中…