产生背景
通过前面的学习我们知道了服务器在处理大量连接时可以使用epoll这样的IO多路复用技术来实现高并发。但是使用过程中发现,通过面向过程的编程方式使用IO多路复用技术并不是很方便。于是产生一种基于面向对象思想来封装网络IO层的模式,Reactor模式。
Reactor
Reactor翻译过来是反应堆,有点难以理解,其实是只对于事件有特定的响应。Reactor模式还有一个名字Dispatcher,相对来说更加贴切,分配器。使用epoll监听事件,收到事件之后根据类型进行分发。
Reactor负责事件监听与分发。事件一般有两种,一种是建立连接,一种是数据读写。
- 建立连接,我们一般交由一个Acceptor来处理专门负责奖励连接然后交换Reactor监听新的socekt
- 数据读写,一般交由一个Handler来负责,大概流程是 read->处理业务->write
Reactor模型的组织方式有多种,reactor和handler数量可以分为
- 单reactor单线程/进程
- 单reactor多线程/进程
- 多reactor多线程/进程
理论上来说还多reactor单进程,但是相对于单reactor单进程来说除了复杂之外处理效率并没有提升
单reactor单线程/进程
实现简单,不需要考虑线程之间通信竞争问题,适合业务处理较快的场景。
缺点:
- 事件监听,建立连接,业务处理都在一个线程,如果业务处理耗时较长会阻塞其他功能。
- 无法利用多核cpu性能。
单reactor多线程
监听事件,建立连接的流程与单线程一致,在handler处理时有差别。handler只负责读写数据,而业务处理交由其他工作线程处理。一种实现办法是handler在读到数据后发送到工作线程的消息队列,工作线程会不断的读取消息队列中的数据然后处理,处理结束结果返回handler,之后通过send发送。
- 因为使用多线程,可以充分利用多核cpu的资源,同时线程间通信需要在共享资源加锁,比如handler保存计算结果的缓存
- 单reactor模式下,一个reactor监听所有事件,可能存在性能瓶颈。
- 至于单reactor多进程实现起来麻烦很多,主要是进程间通信实现难度要高得多。
多reactor多线程
多reactor模式一般设置主从reactor, 主reactor只关注listenfd,监听建立连接的事件, 然后交给accpeter处理。连接建立后,读写事件的监控交给子reactor负责,子reactor和对应handler在同一线程,有新的读写事件发生交给对应handler处理来完成对应业务。
示例
EventSocket管理一个连接的数据对应一个客户端,实现了基本读和写功能。
class EventSocket
{
public:
EventSocket():
fd_(0),
status_(0),
length_(0)
{}
~EventSocket()
{}
int getFd() {return fd_;}
void setFd(int fd) {fd_ = fd;}
int getStatus() {return status_;}
void setStatus(int status) {status_ = status;}
bool readData();
bool writeData();
private:
int fd_;
int status_; // 0 none 1 listening 2 common
int length_;
int buffer_[BUFF_LENGTH];
};
EventSocketGroup用于管理所有的连接,负责连接的创建以及销毁。这里为了方便直接用map管理,更好的做法可以使用池化计数。
class EventSocketGroup
{
public:
EventSocketGroup():
reactor_(NULL)
{}
~EventSocketGroup()
{}
void setReactor(EpollReactor* reactor) {reactor_ = reactor;}
EventSocket* getSocket(int socket_fd);
bool onFDRead(EventSocket* socket_ptr);
bool onFDWrite(EventSocket* socket_ptr);
void closeSocket(EventSocket* socket_ptr);
private:
std::map<int, EventSocket> socket_map_;
EpollReactor* reactor_;
};
EpollHandler指定了epoll事件的回调接口。
EpollReactor管理epoll对象,实现了添加封装了epoll事件的增删改功能。其中execute函数实现了epoll_wait的主要逻辑,在事件发生时回调EpollHandler中注册的函数。
class EpollHandler
{
public:
typedef std::function<int()> AcceptCallback ;
typedef std::function<bool(EventSocket* socket_ptr)> ReadCallBack;
typedef std::function<bool(EventSocket* socket_ptr)> WriteCallBack;
typedef std::function<void(EventSocket* socket_ptr)> CloseCallBack;
void setAcceptCB(const AcceptCallback& cb) { accept_cb_ = cb;}
void setReadCB(const ReadCallBack& cb) {read_cb_ = cb;}
void setWriteCB(const WriteCallBack& cb) {write_cb_ = cb;}
void setCloseCB(const CloseCallBack& cb) {close_cb_ = cb;}
public:
AcceptCallback accept_cb_;
ReadCallBack read_cb_;
WriteCallBack write_cb_;
CloseCallBack close_cb_;
};
class EpollReactor
{
public:
EpollReactor(): epfd_(0)
{}
~EpollReactor()
{}
bool init();
bool addEvent(int events, EventSocket* socket_ptr);
bool modEvent(int events, EventSocket* socket_ptr);
void delEvent(EventSocket* socket_ptr);
void execute();
EpollHandler& getHandler() { return handler_;}
private:
int epfd_;
struct epoll_event events_list_[MAX_EPOLL_EVENTS + 1];
EpollHandler handler_;
};
EpollAcceptor负责维护监听socket,同时在有新的连接加入时,注册到Reactor监控。
class EpollAcceptor
{
public:
EpollAcceptor():
listenfd_(0),
reactor_(NULL),
group_(NULL)
{}
~EpollAcceptor() {}
void setReactor(EpollReactor* reactor) {reactor_ = reactor;}
void setSocketGroup(EventSocketGroup* group) {group_ = group;}
bool startListen(int port);
int onFDAccept();
private:
int listenfd_;
EpollReactor* reactor_;
EventSocketGroup* group_;
};
TcpServer负责所有模块的初始化,以及流程管理。
init初始化各个模块,注册各种事件的回调函数。
start开始监听端口等待连接。
run在一个循环中反复调用epoll_wait来监听处理事件
stop负责资源回收,示例为了演示reactor如何构建没有设置服务器停止方式,后续就省略了。
class TcpServer
{
public:
TcpServer():
port_(0)
{}
~TcpServer()
{}
public:
bool init(int port);
bool start();
void run();
void stop();
private:
int port_;
EpollReactor reactor_;
EpollAcceptor accepter_;
EventSocketGroup group_;
};
完整示例
#include <iostream>
#include <stdio.h>
#include <sys/socket.h>
#include <sys/types.h>
#include <netinet/in.h>
#include <fcntl.h>
#include <pthread.h>
#include <unistd.h>
#include <sys/epoll.h>
#include <malloc.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>
#include <arpa/inet.h>
#include <vector>
#include <map>
#include <functional>
#define BUFF_LENGTH 1024
#define MAX_EPOLL_EVENTS 1024
using namespace std;
class EventSocketGroup;
class EventSocket
{
public:
EventSocket():
fd_(0),
status_(0),
length_(0)
{}
~EventSocket()
{}
int getFd() {return fd_;}
void setFd(int fd) {fd_ = fd;}
int getStatus() {return status_;}
void setStatus(int status) {status_ = status;}
bool readData();
bool writeData();
private:
int fd_;
int status_; // 0 none 1 listening 2 common
int length_;
int buffer_[BUFF_LENGTH];
};
bool EventSocket::readData()
{
int len = recv(fd_, buffer_, BUFF_LENGTH, 0);
if (len > 0)
{
length_ = len;
buffer_[len] = '\0';
printf("recv [fd=%d] %d:%s\n", fd_, len, buffer_);
}
else if (len == 0)
{
printf("disconnected [%d]\n", fd_);
return false;
}
else
{
if (errno == EAGAIN)
{
// 读完了
return true;
}
printf("read [fd=%d] error. %s\n", fd_, strerror(errno));
return false;
}
return true;
}
bool EventSocket::writeData()
{
printf("write fd:%d \n", fd_);
int len = send(fd_, buffer_, length_, 0);
if (len > 0)
{
printf("send [fd=%d] len%d:%s\n", fd_, len, buffer_);
}
else
{
printf("send [fd=%d] error. %s\n", fd_, strerror(errno));
return false;
}
return true;
}
class EpollHandler
{
public:
typedef std::function<int()> AcceptCallback ;
typedef std::function<bool(EventSocket* socket_ptr)> ReadCallBack;
typedef std::function<bool(EventSocket* socket_ptr)> WriteCallBack;
typedef std::function<void(EventSocket* socket_ptr)> CloseCallBack;
void setAcceptCB(const AcceptCallback& cb) { accept_cb_ = cb;}
void setReadCB(const ReadCallBack& cb) {read_cb_ = cb;}
void setWriteCB(const WriteCallBack& cb) {write_cb_ = cb;}
void setCloseCB(const CloseCallBack& cb) {close_cb_ = cb;}
public:
AcceptCallback accept_cb_;
ReadCallBack read_cb_;
WriteCallBack write_cb_;
CloseCallBack close_cb_;
};
class EpollReactor
{
public:
EpollReactor(): epfd_(0)
{}
~EpollReactor()
{}
bool init();
bool addEvent(int events, EventSocket* socket_ptr);
bool modEvent(int events, EventSocket* socket_ptr);
void delEvent(EventSocket* socket_ptr);
void execute();
EpollHandler& getHandler() { return handler_;}
private:
int epfd_;
struct epoll_event events_list_[MAX_EPOLL_EVENTS + 1];
EpollHandler handler_;
};
bool EpollReactor::init()
{
epfd_ = epoll_create(1);
if (epfd_ <= 0)
{
printf("create epfd error: %s \n", strerror(errno));
return false;
}
printf("init succ: %d \n", epfd_);
return true;
}
void EpollReactor::delEvent(EventSocket* socket_ptr)
{
struct epoll_event ep_ev = {0, {0}};
ep_ev.data.ptr = (void*)socket_ptr;
socket_ptr->setStatus(0);
epoll_ctl(epfd_, EPOLL_CTL_DEL, socket_ptr->getFd(), &ep_ev) ;
}
bool EpollReactor::addEvent(int events, EventSocket* socket_ptr)
{
struct epoll_event ep_ev = {0, {0}};
ep_ev.data.ptr = (void*)socket_ptr;
ep_ev.events = events;
if (epoll_ctl(epfd_, EPOLL_CTL_ADD, socket_ptr->getFd(), &ep_ev) < 0)
{
printf("event add failed. fd=%d, events[%d] \n", socket_ptr->getFd(), events);
return false;
}
return true;
}
bool EpollReactor::modEvent(int events, EventSocket* socket_ptr)
{
struct epoll_event ep_ev = {0, {0}};
ep_ev.data.ptr = (void*)socket_ptr;
ep_ev.events = events;
if (epoll_ctl(epfd_, EPOLL_CTL_MOD, socket_ptr->getFd(), &ep_ev) < 0)
{
printf("event add failed. fd=%d, events[%d] \n", socket_ptr->getFd(), events);
return false;
}
return true;
}
void EpollReactor::execute()
{
int nready = epoll_wait(epfd_, events_list_, MAX_EPOLL_EVENTS, 1000);
if (nready < 0)
{
return;
}
for (int i = 0; i < nready; ++i)
{
EventSocket* socket_ptr = (struct EventSocket*)events_list_[i].data.ptr;
if (socket_ptr->getStatus() == 1)
{
//accepter
if (handler_.accept_cb_() > 0)
{
printf("new clientfd \n");
}
else
{
printf("accept new clientfd failed.\n");
}
}
else
{
// handler
if (events_list_[i].events & EPOLLIN)
{
if (false == handler_.read_cb_(socket_ptr))
{
handler_.close_cb_(socket_ptr);
}
}
if (events_list_[i].events & EPOLLOUT)
{
if (false == handler_.write_cb_(socket_ptr))
{
handler_.close_cb_(socket_ptr);
}
}
}
}
}
class EventSocketGroup
{
public:
EventSocketGroup():
reactor_(NULL)
{}
~EventSocketGroup()
{}
void setReactor(EpollReactor* reactor) {reactor_ = reactor;}
EventSocket* getSocket(int socket_fd);
bool onFDRead(EventSocket* socket_ptr);
bool onFDWrite(EventSocket* socket_ptr);
void closeSocket(EventSocket* socket_ptr);
private:
std::map<int, EventSocket> socket_map_;
EpollReactor* reactor_;
};
EventSocket* EventSocketGroup::getSocket(int fd)
{
auto iter = socket_map_.find(fd);
if (iter == socket_map_.end())
{
socket_map_[fd] = EventSocket();
socket_map_[fd].setFd(fd);
return &(socket_map_[fd]);
}
return &(iter->second);
}
void EventSocketGroup::closeSocket(EventSocket* socket_ptr)
{
reactor_->delEvent(socket_ptr);
close(socket_ptr->getFd());
socket_map_.erase(socket_ptr->getFd());
}
bool EventSocketGroup::onFDRead(EventSocket* socket_ptr)
{
if (socket_ptr->readData())
{
// 读成功监听写
if (false == reactor_->modEvent(EPOLLOUT, socket_ptr))
{
return false;
}
}
else
{
return false;
}
return true;
}
bool EventSocketGroup::onFDWrite(EventSocket* socket_ptr)
{
if (socket_ptr->writeData())
{
// 写成功监听读
if (false == reactor_->modEvent(EPOLLIN, socket_ptr))
{
return false;
}
}
else
{
return false;
}
return true;
}
/
class EpollAcceptor
{
public:
EpollAcceptor():
listenfd_(0),
reactor_(NULL),
group_(NULL)
{}
~EpollAcceptor() {}
void setReactor(EpollReactor* reactor) {reactor_ = reactor;}
void setSocketGroup(EventSocketGroup* group) {group_ = group;}
bool startListen(int port);
int onFDAccept();
private:
int listenfd_;
EpollReactor* reactor_;
EventSocketGroup* group_;
};
bool EpollAcceptor::startListen(int port)
{
int listenfd = socket(AF_INET, SOCK_STREAM, 0);
fcntl(listenfd, F_SETFL, O_NONBLOCK);
struct sockaddr_in server_addr;
memset(&server_addr, 0, sizeof(server_addr));
server_addr.sin_family = AF_INET;
server_addr.sin_addr.s_addr = htonl(INADDR_ANY);
server_addr.sin_port = htons(port);
// ::bind避免与std::bind冲突
if (-1 == ::bind(listenfd, (struct sockaddr*)&server_addr, sizeof(server_addr)))
{
printf("bind error %d \n", port);
return false;
}
if (listen(listenfd, 10) < 0)
{
printf("listen error %d \n", port);
return false;
}
printf("listen server port %d \n", port);
listenfd_ = listenfd;
EventSocket* socket_ptr = group_->getSocket(listenfd_);
if (NULL == socket_ptr)
{
printf("accept error \n");
return -1;
}
socket_ptr->setStatus(1);
reactor_->addEvent(EPOLLIN, socket_ptr);
return true;
}
int EpollAcceptor::onFDAccept()
{
struct sockaddr_in client_addr;
socklen_t len = sizeof(client_addr);
int clientfd;
if ((clientfd = accept(listenfd_, (struct sockaddr*) &client_addr, &len)) == -1)
{
printf("accept failed: %s \n", strerror(errno));
return -1;
}
// 设置为非阻塞
int flag = 0;
if ((flag = fcntl(clientfd, F_SETFL, O_NONBLOCK)) < 0)
{
printf("fcntl nonblock failed: %s \n", strerror(errno));
return -1;
}
EventSocket* socket_ptr = group_->getSocket(clientfd);
if (NULL == socket_ptr)
{
printf("accept error \n");
return -1;
}
socket_ptr->setStatus(2);
reactor_->addEvent(EPOLLIN, socket_ptr);
printf("new connect: [%s:%d], pos[%d] \n",
inet_ntoa(client_addr.sin_addr),
ntohs(client_addr.sin_port),
clientfd);
return clientfd;
}
///
class TcpServer
{
public:
TcpServer():
port_(0)
{}
~TcpServer()
{}
public:
bool init(int port);
bool start();
void run();
void stop();
private:
int port_;
EpollReactor reactor_;
EpollAcceptor accepter_;
EventSocketGroup group_;
};
bool TcpServer::init(int port)
{
port_ = port;
if (false == reactor_.init())
{
printf("socket group init failed.\n");
return false;
}
// 初始化 socket_group
group_.setReactor(&reactor_);
// 初始化 accepter
accepter_.setReactor(&reactor_);
accepter_.setSocketGroup(&group_);
// 设置reactor 回调
reactor_.getHandler().setAcceptCB(std::bind(&EpollAcceptor::onFDAccept, &accepter_));
reactor_.getHandler().setReadCB(std::bind(&EventSocketGroup::onFDRead, &group_, std::placeholders::_1));
reactor_.getHandler().setWriteCB(std::bind(&EventSocketGroup::onFDWrite, &group_, std::placeholders::_1));
reactor_.getHandler().setCloseCB(std::bind(&EventSocketGroup::closeSocket, &group_, std::placeholders::_1));
printf("init succ.\n");
return true;
}
bool TcpServer::start()
{
if (false == accepter_.startListen(port_))
{
printf("socket group start failed.\n");
return false;
}
printf("start succ.\n");
return true;
}
void TcpServer::run()
{
printf("Tcp server run.\n");
while (true)
{
reactor_.execute();
}
}
void TcpServer::stop()
{
printf("Tcp server stop.\n");
}
int main()
{
int port = 9999;
TcpServer server;
if (false == server.init(port))
{
printf("server init failed.\n");
return 1;
}
if (false == server.start())
{
printf("server start failed.\n");
return 1;
}
server.run();
server.stop();
return 0;
}
荐一个零声学院免费教程,个人觉得老师讲得不错,分享给大家:[Linux,Nginx,ZeroMQ,MySQL,Redis,
fastdfs,MongoDB,ZK,流媒体,CDN,P2P,K8S,Docker,
TCP/IP,协程,DPDK等技术内容,点击立即学习:
链接