kbengine是同时支持linux与windows平台的,以前可能做windows平台比较多,本能的认为windows平台下使用的是IOCP完成端口,看了源码后,已经大体的能够看出windows下使用的是select,linux下使用的是epoll,接下来我们从下面步骤进行分析。
1.loginapp是组件中最简单的,先分析下启动流程
int KBENGINE_MAIN(int argc, char* argv[])
{
ENGINE_COMPONENT_INFO& info = g_kbeSrvConfig.getLoginApp();
return kbeMainT<Loginapp>(argc, argv, LOGINAPP_TYPE, info.externalTcpPorts_min,
info.externalTcpPorts_max, -1, -1, info.externalInterface, 0, 0, info.internalInterface);
}
2.在server中对networkinterface进行初始化
int kbeMainT(int argc, char * argv[], COMPONENT_TYPE componentType,
int32 extlisteningTcpPort_min = -1, int32 extlisteningTcpPort_max = -1,
int32 extlisteningUdpPort_min = -1, int32 extlisteningUdpPort_max = -1, const char * extlisteningInterface = "",
int32 intlisteningPort_min = 0, int32 intlisteningPort_max = 0, const char * intlisteningInterface = "")
{
//.....
//此处对networkinterface进行初始化
Network::NetworkInterface networkInterface(&dispatcher,
extlisteningTcpPort_min, extlisteningTcpPort_max, extlisteningUdpPort_min, extlisteningUdpPort_max, extlisteningInterface,
channelCommon.extReadBufferSize, channelCommon.extWriteBufferSize,
intlisteningPort_min, intlisteningPort_max, intlisteningInterface,
channelCommon.intReadBufferSize, channelCommon.intWriteBufferSize);
//.......
}
3.NetworkInterface构造函数调用
//根据配置区分了TCP与UDP
if(extlisteningTcpPort_min != -1)
{
pExtListenerReceiver_ = new ListenerTcpReceiver(extTcpEndpoint_, Channel::EXTERNAL, *this);
this->initialize("EXTERNAL-TCP", htons(extlisteningTcpPort_min), htons(extlisteningTcpPort_max),
extlisteningInterface, &extTcpEndpoint_, pExtListenerReceiver_, extrbuffer, extwbuffer);
// 如果配置了对外端口范围, 如果范围过小这里extEndpoint_可能没有端口可用了
if(extlisteningTcpPort_min != -1)
{
KBE_ASSERT(extTcpEndpoint_.good() && "Channel::EXTERNAL-TCP: no available port, "
"please check for kbengine[_defs].xml!\n");
}
}
if (extlisteningUdpPort_min != -1)
{
pExtUdpListenerReceiver_ = new ListenerUdpReceiver(extUdpEndpoint_, Channel::EXTERNAL, *this);
this->initialize("EXTERNAL-UDP", htons(extlisteningUdpPort_min), htons(extlisteningUdpPort_max),
extlisteningInterface, &extUdpEndpoint_, pExtUdpListenerReceiver_, extrbuffer, extwbuffer);
// 如果配置了对外端口范围, 如果范围过小这里extEndpoint_可能没有端口可用了
if (extlisteningUdpPort_min != -1)
{
KBE_ASSERT(extUdpEndpoint_.good() && "Channel::EXTERNAL-UDP: no available udp-port, "
"please check for kbengine[_defs].xml!\n");
}
}
if (intlisteningPort_min != -1)
{
pIntListenerReceiver_ = new ListenerTcpReceiver(intTcpEndpoint_, Channel::INTERNAL, *this);
this->initialize("INTERNAL-TCP", htons(intlisteningPort_min), htons(intlisteningPort_max),
intlisteningInterface, &intTcpEndpoint_, pIntListenerReceiver_, intrbuffer, intwbuffer);
}
KBE_ASSERT(good() && "NetworkInterface::NetworkInterface: no available port, "
"please check for kbengine[_defs].xml!\n");
pDelayedChannels_->init(this->dispatcher(), this);
此处对比一下xml文件中的配置信息
<loginapp>
<!-- 脚本入口模块, 相当于main函数
(Entry module, like the main-function)
-->
<entryScriptFile> kbemain </entryScriptFile>
<!-- 指定接口地址,可配置网卡名、MAC、IP
(Interface address specified, configurable NIC/MAC/IP)
-->
<internalInterface> </internalInterface>
<externalInterface> </externalInterface> <!-- Type: String -->
<!-- 强制指定外部IP地址或者域名,在某些网络环境下,可能会使用端口映射的方式来访问局域网内部的KBE服务器,那么KBE在当前
的机器上获得的外部地址是局域网地址,此时某些功能将会不正常。例如:账号激活邮件中给出的回调地址, 登录baseapp。
注意:服务端并不会检查这个地址的可用性,因为无法检查。
(Forced to specify an external IP-address or Domain-name, In some server environment, May use the port mapping to access KBE,
So KBE on current machines on the external IP address may be a LAN IP address, Then some functions will not normal.
For example: account activation email address given callback.
Note: the availability of server does not check the address, because cannot check)
-->
<externalAddress> </externalAddress> <!-- Type: String -->
<!-- 暴露给客户端的端口范围
(Exposed to the client port range)
-->
<externalTcpPorts_min> 20013 </externalTcpPorts_min> <!-- Type: Integer -->
<externalTcpPorts_max> 0 </externalTcpPorts_max> <!-- Type: Integer -->
<externalUdpPorts_min> -1 </externalUdpPorts_min> <!-- Type: Integer -->
<externalUdpPorts_max> -1 </externalUdpPorts_max> <!-- Type: Integer -->
<!-- 加密登录信息
(The encrypted user login information)
可选择的加密方式(Optional encryption):
0: 无加密(No Encryption)
1: Blowfish
2: RSA (res\key\kbengine_private.key)
-->
<encrypt_login> 2 </encrypt_login>
<!-- listen监听队列最大值
(listen: Maximum listen queue)
-->
<SOMAXCONN> 511 </SOMAXCONN>
<!-- 账号的类型 (Account types)
1: 普通账号 (Normal Account)
2: email账号(需要激活) (Email Account, Note: activation required.)
3: 智能账号(自动识别Email, 普通号码等) (Smart Account (Email or Normal, etc.))
-->
<account_type> 3 </account_type>
<!-- http回调接口,处理认证、密码重置等
(注意:http_cbhost一般会被引擎替换为externalInterface或者externalAddress,仅第一个loginapp才会开启这个服务)
(Http-callback interface, handling authentication, password reset, etc.)
-->
<http_cbhost> localhost </http_cbhost>
<http_cbport> 21103 </http_cbport>
<!-- Telnet服务, 如果端口被占用则向后尝试31001..
(Telnet service, if the port is occupied backwards to try 31001)
-->
<telnet_service>
<port> 31000 </port>
<password> pwd123456 </password>
<!-- 命令默认层
(layer of default the command)
-->
<default_layer> python </default_layer>
</telnet_service>
</loginapp>
4.分析一下初始化函数initialize
1>创建套接字
//创建套接字,TCP是数据流,UDP是报文
if(isTCP)
pEP->socket(SOCK_STREAM);
else
pEP->socket(SOCK_DGRAM);
2>绑定端口
// 尝试绑定到端口,如果被占用向后递增
bool foundport = false;
uint32 listeningPort = listeningPort_min;
if(listeningPort_min != listeningPort_max)
{
for(int lpIdx=ntohs(listeningPort_min); lpIdx<=ntohs(listeningPort_max); ++lpIdx)
{
listeningPort = htons(lpIdx);
if (pEP->bind(listeningPort, ifIPAddr) != 0)
{
continue;
}
else
{
foundport = true;
break;
}
}
}
else
{
if (pEP->bind(listeningPort, ifIPAddr) == 0)
{
foundport = true;
}
}
3>监听
if (isTCP)
{
if (pEP->listen(backlog) == -1)
{
ERROR_MSG(fmt::format("NetworkInterface::initialize({}): listen to {} ({})\n",
pEndPointName, address.c_str(), kbe_strerror()));
pEP->close();
return false;
}
}
4>注册读文件描述符
//注册读写文件描述符
this->dispatcher().registerReadFileDescriptor(*pEP, pLR);
5>设置非阻塞
//设置非阻塞
pEP->setnonblocking(true);
5.进入registerReadFileDescriptor函数中
bool EventDispatcher::registerReadFileDescriptor(int fd,
InputNotificationHandler * handler)
{
return pPoller_->registerForRead(fd, handler);
}
//
bool EventPoller::registerForRead(int fd,
InputNotificationHandler * handler)
{
if (!this->doRegisterForRead(fd))
{
return false;
}
fdReadHandlers_[ fd ] = handler;
return true;
}
//
//此处区分了读写
bool EpollPoller::doRegister(int fd, bool isRead, bool isRegister)
{
struct epoll_event ev;
memset(&ev, 0, sizeof(ev)); // stop valgrind warning
int op;
ev.data.fd = fd;
// Handle the case where the file is already registered for the opposite
// action.
// 此处采用的模式是LT模式
if (this->isRegistered(fd, !isRead))
{
op = EPOLL_CTL_MOD;
ev.events = isRegister ? EPOLLIN|EPOLLOUT :
isRead ? EPOLLOUT : EPOLLIN;
}
else
{
// TODO: Could be good to use EPOLLET (leave like select for now).
ev.events = isRead ? EPOLLIN : EPOLLOUT;
op = isRegister ? EPOLL_CTL_ADD : EPOLL_CTL_DEL;
}
if (epoll_ctl(epfd_, op, fd, &ev) < 0)
{
const char* MESSAGE = "EpollPoller::doRegister: Failed to {} {} file "
"descriptor {} ({})\n";
if (errno == EBADF)
{
WARNING_MSG(fmt::format(MESSAGE,
(isRegister ? "add" : "remove"),
(isRead ? "read" : "write"),
fd,
kbe_strerror()));
}
else
{
ERROR_MSG(fmt::format(MESSAGE,
(isRegister ? "add" : "remove"),
(isRead ? "read" : "write"),
fd,
kbe_strerror()));
}
return false;
}
return true;
}
doRegisterForRead做了平台兼容,windows下为select,linux下为epoll
知识点扩展:
1>IO模型:阻塞IO、非阻塞IO、多路复用IO 、信号驱动IO、异步IO
2>多路复用IO:select、poll、epoll
3>三者的区别:
-
select:
- 单个进程可监视的fd数量被限制,即能监听端口的大小有限
- 对socket进行扫描时是线性扫描,即采用轮询的方法,效率较低
- 需要维护一个用来存放大量fd的数据结构,这样会使得用户空间和内核空间在传递该结构时复制开销大
-
poll
- 大量的fd的数组被整体复制于用户态和内核地址空间之间,连接数大时,耗时
- 没有fd数量限制,fd通过链表串联
- poll还有一个特点是“水平触发”,如果报告了fd后,没有被处理,那么下次poll时会再次报告该fd【EPOLL LT的特点】
-
epoll
- 没有最大并发连接的限制,能打开的FD的上限远大于1024(1G的内存上能监听约10万个端口)
- 效率提升,不是轮询的方式,不会随着FD数目的增加效率下降。只有活跃可用的FD才会调用callback函数
- 内存拷贝,利用mmap()文件映射内存加速与内核空间的消息传递;即epoll使用mmap减少复制开销
4>EPOLL的水平触发(LT)与边缘触发(ET)
-
Level Triggered (LT) 水平触发
- accept一个连接,添加到epoll中监听EPOLLIN事件
- 当EPOLLIN事件到达时,read fd中的数据并处理
- 当需要写出数据时,把数据write到fd中,如果数据较大,无法一次性写入,那么在epoll中监听EPOLLOUT事件
- 当EPOLLOUT事件到达时,继续把数据write到fd中,如果数据写入完毕,那么在epoll中关闭EPOLLOUT事件
-
Edge Triggered (ET) 边沿触发
- accept一个连接,添加到epoll中监听EPOLLIN|EPOLLOUT事件
- 当EPOLLIN事件到达时,read fd中的数据并处理,read需要一直读,直到返回EAGAIN为止
- 当需要写出数据时,把数据write到fd中,直到数据全部写完,或者write返回EAGAIN
- 当EPOLLOUT事件到达时,继续把数据write到fd中,直到数据全部写完,或者write返回EAGAIN