Reader是如何读取到Writer的数据（底层逻辑）
【Apollo 6.0】 cyber rt是如何使用Reader读取到Writer发送的数据（顶层逻辑）
【Apollo 6.0】cyber rt是如何进行节点间管理的
上一篇文章（上层逻辑）中讲了reader读取数据的流程，但是里面没有讲到底层的Tansport是这么进行进程间通信，这里我们来解析一下。
在ReceiverManager::GetReceiver调用了transport::Transport::Instance()->CreateReceiver进行Receiver的创建：
template <typename M>
auto Transport::CreateReceiver(
const RoleAttributes& attr,
const typename Receiver<M>::MessageListener& msg_listener,
const OptionalMode& mode) -> typename std::shared_ptr<Receiver<M>> {
if (is_shutdown_.load()) {
AINFO << "transport has been shut down.";
return nullptr;
}
std::shared_ptr<Receiver<M>> receiver = nullptr;
RoleAttributes modified_attr = attr;
if (!modified_attr.has_qos_profile()) {
modified_attr.mutable_qos_profile()->CopyFrom(
QosProfileConf::QOS_PROFILE_DEFAULT);
}
//根据通信模式来创建Receiver，apollo默认使用的是OptionalMode::HYBRID，也就是default分支
switch (mode) {
case OptionalMode::INTRA:
receiver =
std::make_shared<IntraReceiver<M>>(modified_attr, msg_listener);
break;
case OptionalMode::SHM:
receiver = std::make_shared<ShmReceiver<M>>(modified_attr, msg_listener);
break;
case OptionalMode::RTPS:
receiver = std::make_shared<RtpsReceiver<M>>(modified_attr, msg_listener);
break;
default:
receiver = std::make_shared<HybridReceiver<M>>(
modified_attr, msg_listener, participant());
break;
}
RETURN_VAL_IF_NULL(receiver, nullptr);
if (mode != OptionalMode::HYBRID) {
receiver->Enable();
}
return receiver;
}
OptionalMode::HYBRID这个通信模式其实就是包含了上面三个模式，OptionalMode::HYBRID模式会根据对方的网络状态来选择需要的模式。
因此我们这里分析HybridReceiver：
template <typename M>
HybridReceiver<M>::HybridReceiver(
const RoleAttributes& attr,
const typename Receiver<M>::MessageListener& msg_listener,
const ParticipantPtr& participant)
: Receiver<M>(attr, msg_listener),
history_(nullptr),
participant_(participant) {
InitMode();
ObtainConfig();
InitHistory();
InitReceivers();
InitTransmitters();
}
template <typename M>
void HybridReceiver<M>::InitMode() {
mode_ = std::make_shared<proto::CommunicationMode>();
mapping_table_[SAME_PROC] = mode_->same_proc();
mapping_table_[DIFF_PROC] = mode_->diff_proc();
mapping_table_[DIFF_HOST] = mode_->diff_host();
}
template <typename M>
void HybridReceiver<M>::ObtainConfig() {
auto& global_conf = common::GlobalData::Instance()->Config();
if (!global_conf.has_transport_conf()) {
return;
}
if (!global_conf.transport_conf().has_communication_mode()) {
return;
}
mode_->CopyFrom(global_conf.transport_conf().communication_mode());
mapping_table_[SAME_PROC] = mode_->same_proc();
mapping_table_[DIFF_PROC] = mode_->diff_proc();
mapping_table_[DIFF_HOST] = mode_->diff_host();
}
//根据配置里的qos，设置缓存history_长度，可见HybridReceiver学RTPS也设置了读侧的消息缓存。需要注意这个缓存只是为RTPS模式设置的，所以只有当这个消息有qos本地缓存要求时才真正有意义
template <typename M>
void HybridReceiver<M>::InitHistory() {
HistoryAttributes history_attr(this->attr_.qos_profile().history(),
this->attr_.qos_profile().depth());
history_ = std::make_shared<History<M>>(history_attr);
if (this->attr_.qos_profile().durability() ==
QosDurabilityPolicy::DURABILITY_TRANSIENT_LOCAL) {
history_->Enable();
}
}
template <typename M>
void HybridReceiver<M>::InitReceivers() {
std::set<OptionalMode> modes;
modes.insert(mode_->same_proc());
modes.insert(mode_->diff_proc());
modes.insert(mode_->diff_host());
auto listener = std::bind(&HybridReceiver<M>::OnNewMessage, this,
std::placeholders::_1, std::placeholders::_2);
for (auto& mode : modes) {
switch (mode) {
case OptionalMode::INTRA:
receivers_[mode] =
std::make_shared<IntraReceiver<M>>(this->attr_, listener);
break;
case OptionalMode::SHM:
receivers_[mode] =
std::make_shared<ShmReceiver<M>>(this->attr_, listener);
break;
default:
receivers_[mode] =
std::make_shared<RtpsReceiver<M>>(this->attr_, listener);
break;
}
}
}
template <typename M>
void HybridReceiver<M>::InitTransmitters() {
std::unordered_map<uint64_t, RoleAttributes> empty;
for (auto& item : receivers_) {
transmitters_[item.first] = empty;
}
}
HybridReceiver构造函数所调用构造函数就是进行初始化的作用，主要就是初始化mapping_table_ 、receivers_ 和transmitters_ 。对于 mapping_table_的key我们这里讲一下：
//SAME_PROC：同一进程，默认使用INTRA模式通信，这种情况发生在同一个module内的通信，比如同一个module里不同的component相互通信，或是同一个component里自我通信
//DIFF_PROC：不同进程但在同一host，默认使用SHM，这种情况基本上就是单机上不同module互相通信时会用到，也是用得最多的模式
//DIFF_HOST：不同host上的不同进程，默认使用RTPS，这只有在不同host上通信时才用到
//通过GetRelation函数来判断使用什么
template <typename M>
Relation HybridReceiver<M>::GetRelation(const RoleAttributes& opposite_attr) {
if (opposite_attr.channel_name() != this->attr_.channel_name()) {
return NO_RELATION;
}
if (opposite_attr.host_ip() != this->attr_.host_ip()) {
return DIFF_HOST;
}
if (opposite_attr.process_id() != this->attr_.process_id()) {
return DIFF_PROC;
}
return SAME_PROC;
}
在HybridReceiver::HybridReceiver里创建了三种通信模式的receiver，这里我们将下apollo里使用得最多的ShmReceiver:
template <typename M>
ShmReceiver<M>::ShmReceiver(
const RoleAttributes& attr,
const typename Receiver<M>::MessageListener& msg_listener)
: Receiver<M>(attr, msg_listener) {
dispatcher_ = ShmDispatcher::Instance();
}
ShmDispatcher作为ShmReceiver的数据分发器：
ShmDispatcher::ShmDispatcher() : host_id_(0) { Init(); }
bool ShmDispatcher::Init() {
host_id_ = common::Hash(GlobalData::Instance()->HostIp());
notifier_ = NotifierFactory::CreateNotifier();
thread_ = std::thread(&ShmDispatcher::ThreadFunc, this);
//线程属性配置
scheduler::Instance()->SetInnerThreadAttr("shm_disp", &thread_);
return true;
}
跟上层通信类似，底层的ShmDispatcher也有个notifier_：
auto NotifierFactory::CreateNotifier() -> NotifierPtr {
std::string notifier_type(ConditionNotifier::Type());
auto& g_conf = GlobalData::Instance()->Config();
if (g_conf.has_transport_conf() && g_conf.transport_conf().has_shm_conf() &&
g_conf.transport_conf().shm_conf().has_notifier_type()) {
notifier_type = g_conf.transport_conf().shm_conf().notifier_type();
}
ADEBUG << "notifier type: " << notifier_type;
if (notifier_type == MulticastNotifier::Type()) {
return CreateMulticastNotifier();
} else if (notifier_type == ConditionNotifier::Type()) {
return CreateConditionNotifier();
}
AINFO << "unknown notifier, we use default notifier: " << notifier_type;
return CreateConditionNotifier();
}
auto NotifierFactory::CreateConditionNotifier() -> NotifierPtr {
return ConditionNotifier::Instance();
}
auto NotifierFactory::CreateMulticastNotifier() -> NotifierPtr {
return MulticastNotifier::Instance();
}
这里会根据notifier_type来创建ConditionNotifier或者MulticastNotifier。对于ConditionNotifier，该notifier里面也是使用了一个共享内存操作，当write端发送数据的时候会调用其Notify函数，向共享内存中写入数据，然后read端可以调用Listent函数来读取数据，数据是描述read端怎么从数据通信的共享内存中获取发送的数据。而对于MulticastNotifier则是使用了socket进行通知，ip:239.255.0.100 port:8888。具体的代码就不分析了，继续分析ShmDispatcher::Init():
thread_ = std::thread(&ShmDispatcher::ThreadFunc, this)
scheduler::Instance()->SetInnerThreadAttr("shm_disp", &thread_);
这里创建了ShmDispatcher的处理线程，这个线程是数据读取的关键，因为还没有创建对应的数据读取工具对象，因此这里还不做分析，ShmDispatcher::Init()结束，我们继续分析Transport::CreateReceiver。
if (mode != OptionalMode::HYBRID) {
receiver->Enable();
}
当模式不为OptionalMode::HYBRID就启动receiver，但是我们就是这个模式啊，那HybridReceiver::Enable到底是在哪里被调用的呢？
其实就是在上一篇文章里面最后没讲到的Reader::JoinTheTopology()里
template <typename MessageT>
void Reader<MessageT>::JoinTheTopology() {
// add listener
change_conn_ = channel_manager_->AddChangeListener(std::bind(
&Reader<MessageT>::OnChannelChange, this, std::placeholders::_1));
// get peer writers
const std::string& channel_name = this->role_attr_.channel_name();
std::vector<proto::RoleAttributes> writers;
channel_manager_->GetWritersOfChannel(channel_name, &writers);
for (auto& writer : writers) {
receiver_->Enable(writer);
}
channel_manager_->Join(this->role_attr_, proto::RoleType::ROLE_READER,
message::HasSerializer<MessageT>::value);
}
template <typename MessageT>
void Reader<MessageT>::OnChannelChange(const proto::ChangeMsg& change_msg) {
if (change_msg.role_type() != proto::RoleType::ROLE_WRITER) {
return;
}
auto& writer_attr = change_msg.role_attr();
if (writer_attr.channel_name() != this->role_attr_.channel_name()) {
return;
}
auto operate_type = change_msg.operate_type();
if (operate_type == proto::OperateType::OPT_JOIN) {
receiver_->Enable(writer_attr);
} else {
receiver_->Disable(writer_attr);
}
}
Reader::JoinTheTopology()里具体功能我们这里还是不做分析，这里粗略的讲解一下，通过channel_manager_->AddChangeListener函数，当reader所对应的channel发生变化的时候，将会触发Reader::OnChannelChange函数，如果是有Writer加入operate_type == proto::OperateType::OPT_JOIN则调用Enable启动receiver。而如果已经保存了channel所对应的writers，则直接遍历启动所有的receiver，好了，这个JoinTheTopology是服务发现功能的相关函数，后面会专门写一篇文章来进行讲解，这里直接分析Enable函数。
template <typename M>
void HybridReceiver<M>::Enable(const RoleAttributes& opposite_attr) {
auto relation = GetRelation(opposite_attr);
RETURN_IF(relation == NO_RELATION);
uint64_t id = opposite_attr.id();
std::lock_guard<std::mutex> lock(mutex_);
if (transmitters_[mapping_table_[relation]].count(id) == 0) {
transmitters_[mapping_table_[relation]].insert(
std::make_pair(id, opposite_attr));
receivers_[mapping_table_[relation]]->Enable(opposite_attr);
ReceiveHistoryMsg(opposite_attr);
}
}
HybridReceiver::Enable就是根据节点之间的关系来启动对应的通信模式，这里还是分析ShmReceiver：
template <typename M>
void ShmReceiver<M>::Enable(const RoleAttributes& opposite_attr) {
dispatcher_->AddListener<M>(
this->attr_, opposite_attr,
std::bind(&ShmReceiver<M>::OnNewMessage, this, std::placeholders::_1,
std::placeholders::_2));
}
这里调用了dispatcher_->AddListener来添加回调函数：
template <typename MessageT>
void ShmDispatcher::AddListener(const RoleAttributes& self_attr,
const RoleAttributes& opposite_attr,
const MessageListener<MessageT>& listener) {
// FIXME: make it more clean
auto listener_adapter = [listener](const std::shared_ptr<ReadableBlock>& rb,
const MessageInfo& msg_info) {
auto msg = std::make_shared<MessageT>();
RETURN_IF(!message::ParseFromArray(
rb->buf, static_cast<int>(rb->block->msg_size()), msg.get()));
listener(msg, msg_info);
};
Dispatcher::AddListener<ReadableBlock>(self_attr, opposite_attr,
listener_adapter);
AddSegment(self_attr);
}
这里构建了一个新的回调，然后调用父类的AddListener:
template <typename MessageT>
void Dispatcher::AddListener(const RoleAttributes& self_attr,
const RoleAttributes& opposite_attr,
const MessageListener<MessageT>& listener) {
if (is_shutdown_.load()) {
return;
}
uint64_t channel_id = self_attr.channel_id();
std::shared_ptr<ListenerHandler<MessageT>> handler;
ListenerHandlerBasePtr* handler_base = nullptr;
if (msg_listeners_.Get(channel_id, &handler_base)) {
handler =
std::dynamic_pointer_cast<ListenerHandler<MessageT>>(*handler_base);
if (handler == nullptr) {
AERROR << "please ensure that readers with the same channel["
<< self_attr.channel_name()
<< "] in the same process have the same message type";
return;
}
} else {
ADEBUG << "new reader for channel:"
<< GlobalData::GetChannelById(channel_id);
handler.reset(new ListenerHandler<MessageT>());
msg_listeners_.Set(channel_id, handler);
}
handler->Connect(self_attr.id(), opposite_attr.id(), listener);
}
Dispatcher::AddListener主要干的活就是创建handler然后存入msg_listeners_，并且将回调传入handler->Connect :
template <typename MessageT>
void ListenerHandler<MessageT>::Connect(uint64_t self_id, uint64_t oppo_id,
const Listener& listener) {
WriteLockGuard<AtomicRWLock> lock(rw_lock_);
if (signals_.find(oppo_id) == signals_.end()) {
signals_[oppo_id] = std::make_shared<MessageSignal>();
}
auto connection = signals_[oppo_id]->Connect(listener);
if (!connection.IsConnected()) {
AWARN << oppo_id << " " << self_id << " connect failed!";
return;
}
if (signals_conns_.find(oppo_id) == signals_conns_.end()) {
signals_conns_[oppo_id] = ConnectionMap();
}
signals_conns_[oppo_id][self_id] = connection;
}
这里根据传入的self_id和oppo_id分别设置对应的signal map，这里的signal其实是模仿QT的信号与槽机制实现的，简单的来说呢，就是多个槽（回调函数）可以绑定一个信号，或者一个槽绑定多个信号（apollo这里应该就实现了前一种）。apollo里是这么实现的：当signal执行其重载的括号操作符的时候，将会调用其绑定的全部槽函数。槽绑定信号是通过Connect函数，因此这里listener绑定了signals_[oppo_id]。回到ShmDispatcher::AddListener继续分析。
void ShmDispatcher::AddSegment(const RoleAttributes& self_attr) {
uint64_t channel_id = self_attr.channel_id();
WriteLockGuard<AtomicRWLock> lock(segments_lock_);
if (segments_.count(channel_id) > 0) {
return;
}
auto segment = SegmentFactory::CreateSegment(channel_id);
segments_[channel_id] = segment;
previous_indexes_[channel_id] = UINT32_MAX;
}
auto SegmentFactory::CreateSegment(uint64_t channel_id) -> SegmentPtr {
std::string segment_type(XsiSegment::Type());
auto& shm_conf = GlobalData::Instance()->Config();
if (shm_conf.has_transport_conf() &&
shm_conf.transport_conf().has_shm_conf() &&
shm_conf.transport_conf().shm_conf().has_shm_type()) {
segment_type = shm_conf.transport_conf().shm_conf().shm_type();
}
ADEBUG << "segment type: " << segment_type;
if (segment_type == PosixSegment::Type()) {
return std::make_shared<PosixSegment>(channel_id);
}
return std::make_shared<XsiSegment>(channel_id);
}
AddSegment的作用就是添加共享内存的操作类Segment，这里有两个实现PosixSegment和XsiSegment。两个的差别这里就不做分析了，反正就是里面封装了对共享内存的操作，感兴趣的可以自己分析。将Segment的对象放入到segments_当中，其数据类型为std::unordered_map< uint64_t，std::shared_ptr >。
Enable分析完了，你应该会发出这样的疑问：Enable也只是做相关类的初始化和创建工作，数据的读取流程没有体现啊，到底是在哪里实现的？
这我们就需要回过头来分析刚才没有分析的ShmDispatcher处理线程ThreadFunc：
void ShmDispatcher::ThreadFunc() {
ReadableInfo readable_info;
while (!is_shutdown_.load()) {
if (!notifier_->Listen(100, &readable_info)) {
ADEBUG << "listen failed.";
continue;
}
if (readable_info.host_id() != host_id_) {
ADEBUG << "shm readable info from other host.";
continue;
}
uint64_t channel_id = readable_info.channel_id();
uint32_t block_index = readable_info.block_index();
{
ReadLockGuard<AtomicRWLock> lock(segments_lock_);
if (segments_.count(channel_id) == 0) {
continue;
}
// check block index
if (previous_indexes_.count(channel_id) == 0) {
previous_indexes_[channel_id] = UINT32_MAX;
}
uint32_t& previous_index = previous_indexes_[channel_id];
if (block_index != 0 && previous_index != UINT32_MAX) {
if (block_index == previous_index) {
ADEBUG << "Receive SAME index " << block_index << " of channel "
<< channel_id;
} else if (block_index < previous_index) {
ADEBUG << "Receive PREVIOUS message. last: " << previous_index
<< ", now: " << block_index;
} else if (block_index - previous_index > 1) {
ADEBUG << "Receive JUMP message. last: " << previous_index
<< ", now: " << block_index;
}
}
previous_index = block_index;
ReadMessage(channel_id, block_index);
}
}
}
ThreadFunc是ShmDispatcher的线程处理函数，里面最为关键的就是notifier_->Listen，上面讲到当write端发送数据的时候会调用其Notify函数，向共享内存中写入数据，然后read端可以调用Listent函数来读取数据。ThreadFunc这里就在轮询的调用Listent函数，当Listren读取到数据之后，将会分析从而得到如何从数据通信的共享内存拿出数据的方式，然后调用ReadMessage读取数据。
void ShmDispatcher::ReadMessage(uint64_t channel_id, uint32_t block_index) {
ADEBUG << "Reading sharedmem message: "
<< GlobalData::GetChannelById(channel_id)
<< " from block: " << block_index;
auto rb = std::make_shared<ReadableBlock>();
rb->index = block_index;
if (!segments_[channel_id]->AcquireBlockToRead(rb.get())) {
AWARN << "fail to acquire block, channel: "
<< GlobalData::GetChannelById(channel_id)
<< " index: " << block_index;
return;
}
MessageInfo msg_info;
const char* msg_info_addr =
reinterpret_cast<char*>(rb->buf) + rb->block->msg_size();
if (msg_info.DeserializeFrom(msg_info_addr, rb->block->msg_info_size())) {
OnMessage(channel_id, rb, msg_info);
} else {
AERROR << "error msg info of channel:"
<< GlobalData::GetChannelById(channel_id);
}
segments_[channel_id]->ReleaseReadBlock(*rb);
}
从segments_中拿出channel对应的std::shared_ptr来读取共享内存中的数据， ReadMessage总的来说就是从数据共享内存里面拿出数据，然后再通过OnMessage来进行数据传递：
void ShmDispatcher::OnMessage(uint64_t channel_id,
const std::shared_ptr<ReadableBlock>& rb,
const MessageInfo& msg_info) {
if (is_shutdown_.load()) {
return;
}
ListenerHandlerBasePtr* handler_base = nullptr;
if (msg_listeners_.Get(channel_id, &handler_base)) {
auto handler = std::dynamic_pointer_cast<ListenerHandler<ReadableBlock>>(
*handler_base);
handler->Run(rb, msg_info);
} else {
AERROR << "Cannot find " << GlobalData::GetChannelById(channel_id)
<< "'s handler.";
}
}
从msg_listeners_拿出channel对应的ListenerHandler，执行其Run函数：
template <typename MessageT>
void ListenerHandler<MessageT>::Run(const Message& msg,
const MessageInfo& msg_info) {
signal_(msg, msg_info);
uint64_t oppo_id = msg_info.sender_id().HashValue();
ReadLockGuard<AtomicRWLock> lock(rw_lock_);
if (signals_.find(oppo_id) == signals_.end()) {
return;
}
(*signals_[oppo_id])(msg, msg_info);
}
*(signals_[oppo_id])(msg, msg_info) 这调用就是上面讲的信号的发送，会执行其所有的槽函数。那么这个槽函数是什么呢，如果有从上一章中一直跟着的读者会发现，虽然经过了层层的传递和多重的封装，但是其最初始的函数就是我们在 ReceiverManager::GetReceiver中调用transport::Transport::Instance()->CreateReceiver传入的lamda。分析到这里应该是把数据从底层到上层的流向讲清楚了，总结下流程：
对于shm来说：
写端：
writer —> Segment::AcquireBlockToWrite —> notifier::Notify
读端：
ShmDispatcher::ThreadFunc —> notifier::Listen —> Segment::AcquireBlockToRead —> ListenerHandler::Run —> (*signals_[oppo_id])(msg, msg_info) —> DataDispatcher —> DataVisitor::buffer —> DataNotifier::Instance()::Notify —> callback —> Scheduler::NotifyProcessor —> 执行协程 —> 从DataVisitor::buffer拿出数据 —> 放入reader的blocker —>执行用户传入的回调(如果有的话)
参考链接：
621
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
