高效的半同步/半异步模式的实现

先介绍一下半同步/半异步模式：

首先半同步/半异步模式中的同步和异步和前面的IO模型中的同步和异步是完全不用的概念。在IO模型中，同步和异步区分的是内核向应用程序通知的是何种IO事件(是就绪事件还是完成事件)，以及该由谁来完成IO读写（是应用程序还是内核）。在并发模式中，同步指的是程序完全按照代码序列的顺序执行，异步指的是程序的执行需要由系统事件来驱动。常见的系统事件包括中断 信号等。

比如8-8a描述了同步的读操作 ，图8-8b则描述了异步的读操作。

按照同步方式运行的线程称为同步线程，按照异步方式运行的线程成为异步线程。显然异步线程的执行效率高，实时性强，这是很多嵌入式程序采用的模型。但编写异步方式执行的程序相对复杂，难于调试和扩展，且不适合大量的并发。而同步线程则相反，它虽然效率比较低，实时性较差，但逻辑简单。因此，对于像服务器这种既要求较好的实时性，又要求同时处理多个客户请求的应用程序，我们就应该同时使用同步线程和异步线程来实现。即使用半同步/半异步模式来实现！

半同步/半异步模式中，同步线程用于处理客户逻辑，异步线程用于处理IO事件。异步线程监听到客户请求后，就将其封装成请求对象并插入请求队列中。请求队列将通知某个工作在同步模式的工作线程来读取并处理该对象。具体选择哪个工作线程来为新的客户请求服务，则取决于请求队列的设计。

下面图8-9总结了半同步/半异步的工作流程

在服务器程序中，如果结合考虑两种事件处理模式和几种IO模型，则半同步/半异步模式就存在多种变体。其中有一种变体成为半同步/半反应对模式，如下图8-10所示

图8-10中，主线程插入请求队列中的任务就是就绪的连接socket.这说明该图所示的半同步/半反应堆模式采用的事件处理模式是Reactor模式，它要求工作线程自己从socket上读取客户请求和往socket写入服务器应答。实际上，半同步/半反应对模式也可以使用模拟的Proactor事件处理模式，即由主线程来完成数据的读写。在这种情况下，主线程一般会将应用程序数据 任务类型等信息封装为一个任务对象，如何将其插入请求队列。工作线程从请求队列中取得任务对象之后，即可直接处理，而无须执行读写操作了。

可见 图8-11中，每个线程都维持着自己的事件循环，它们各自独立监听不同的事件，因此在这种搞笑的半同步/半异步模式中，每个线程都工作在异步模式。

下面吧大概的代码展示一下

#include "SocketServer.h"


SocketServer::SocketServer(void) :
  m_nport(5001)
 ,m_epollfd(-1)
 ,m_bindsocket(-1)
 ,m_bstop(false)
{
}

SocketServer::~SocketServer(void)
{

}

void SocketServer::S_WorkService(void* arg)
{
	int epollfd = *(int*)arg;
	SOCKETServer::Instance()->WorkService(epollfd);
}

void SocketServer::WorkService(int epollfd)
{
	epoll_event events[ MAX_EVENT_NUMBER ];	
	 while( !m_bstop )
    {
        int event_num = epoll_wait( epollfd, events, MAX_EVENT_NUMBER, -1 );
		if ( event_num < 0 && (errno != EINTR))
        {
			DC_ERROR("epoll_wait error ,errmsg = %s",strerror(errno));
            break;
        }
		
		for ( int i = 0; i < event_num; i++ )
        {
            int sock = events[i].data.fd;
			
			struct sockaddr_in client_address;
			socklen_t client_addrlength = sizeof( client_address );
			getpeername(sock, (struct sockaddr *)&client_address, &client_addrlength); 
			char remoteAddress[INET_ADDRSTRLEN ] = {0};
			inet_ntop( AF_INET, &client_address.sin_addr, remoteAddress, INET_ADDRSTRLEN );
			int remotePort = ntohs( client_address.sin_port );
			
            if ( events[i].events & EPOLLIN )
            {
				char* pRecvBuff = new  char[SOCKET_BUF_SIZE];
				int nRemainDataSize = 0;
				 while( true )
				 {
					int nBytesThisTime = recv( sock, pRecvBuff + nRemainDataSize, SOCKET_BUF_SIZE -1-nRemainDataSize, 0 );
					if( nBytesThisTime < 0 )
					{
						if( ( errno == EAGAIN ) || ( errno == EWOULDBLOCK ) )
						{
							 break;
						}
						DC_ERROR("socket errormsg = %s , %s:%d close",strerror(errno),remoteAddress ,remotePort);
						del_socket_epoll(epollfd,sock);
						break;
					}
					else if (nBytesThisTime == 0)
					{
						DC_ERROR("socket errormsg = %s , %s:%d close",strerror(errno),remoteAddress ,remotePort);
						nRemainDataSize = 0;
						del_socket_epoll(epollfd,sock);
						break;
					}
					nRemainDataSize += nBytesThisTime;
				 }
				 //如果读取失败的话就直接返回
				 if(nRemainDataSize == 0)
				 {
					 continue;
				 }
				
				DC_INFO("recv %s:%d data size = %d" ,remoteAddress ,remotePort,nRemainDataSize);
				delete(pRecvBuff);pRecvBuff = NULL;
				
				if(0 != reset_socket_epoll(epollfd, sock))
				{
					del_socket_epoll(epollfd,sock);
				}
            }
            else if( events[i].events & ( EPOLLRDHUP | EPOLLHUP | EPOLLERR ) )
            {
               DC_ERROR("socket errormsg = %s , %s:%d close",strerror(errno),remoteAddress ,remotePort);
			   del_socket_epoll(epollfd,sock);
            }
			else
			{
				DC_ERROR("socket errormsg = %s , %s:%d close",strerror(errno),remoteAddress ,remotePort);
			}
        }
		
	}
	
}
int SocketServer::StartServer(int port ,int threadNum )
{
	m_nport = port;
	m_nThreadNum = threadNum;
	
	m_bindsocket = socket( PF_INET, SOCK_STREAM, 0 );
	if(m_bindsocket < 0)
	{
		DC_ERROR("socket error ,errmsg = %s",strerror(errno));
		return SERVER_ERROR;
	}
	
	/*地址可复用 time_wait*/	
	if(0 != make_socket_reuseable(m_bindsocket))
	{
		return SERVER_ERROR;
	}
	
	/*设置超时时间*/
	if(0 != make_socket_timeout(m_bindsocket,SOCKET_TIMEOUT))
	{
		return SERVER_ERROR;
	}
	
	/*设置缓冲区大小*/
	if(0 != make_socket_buffsize(m_bindsocket,SOCKET_BUF_SIZE))
	{
		return SERVER_ERROR;
	}
	
	/*设置非阻塞*/
	if(0 != make_socket_nonblock(m_bindsocket))
	{
		return SERVER_ERROR;
	}
	
	/*绑定地址和端口*/
	struct sockaddr_in address;
    bzero( &address, sizeof( address ) );
    address.sin_family = AF_INET;
    inet_pton( AF_INET, "0.0.0.0", &address.sin_addr );
    address.sin_port = htons( m_nport );
	if(0 != bind( m_bindsocket, ( struct sockaddr* )&address, sizeof(address) ))
	{
		DC_ERROR("bind %d error ,errmsg = %s",m_nport,strerror(errno));
		return SERVER_ERROR;
	}
	
	/*监听端口*/
	if(0 != listen(m_bindsocket, 128))
	{
		DC_ERROR("bind %d error ,errmsg = %s",m_nport,strerror(errno));
		return SERVER_ERROR;
	}
	
	/*创建工作线程以及对应的epoll句柄*/
	for(int i = 0 ;i < m_nThreadNum ;i++)
	{
		int epollfd = epoll_create( 5 );
		swartz_thread_detached_create((void*)S_WorkService, (void*)&epollfd, 0, 0);	
		//休眠50ms ,保证工作线程里面的epollfd是正确的
		usleep(50*1000);
		m_EpollVec.push_back(epollfd);
	}
	
	/*epoll 监听*/
    epoll_event events[ MAX_EVENT_NUMBER ];
    m_epollfd = epoll_create( 5 );
	if(m_epollfd == -1)
	{
		DC_ERROR("epoll_create  error ,errmsg = %s",strerror(errno));
		return SERVER_ERROR;
	}
	if(0 != add_socket_epoll(m_epollfd, m_bindsocket,false))
	{
		return SERVER_ERROR;
	}
	
    while( 1 )
    {
		static int ClusterNum = 0;
        int event_num = epoll_wait( m_epollfd, events, MAX_EVENT_NUMBER, -1 );
        if ( event_num < 0 && (errno != EINTR))
        {
			DC_ERROR("epoll_wait error ,errmsg = %s",strerror(errno));
            break;
        }
    
        for ( int i = 0; i < event_num; i++ )
        {
            int sockfd = events[i].data.fd;
            if ( sockfd == m_bindsocket )
            {
				struct sockaddr_in client_address;
				socklen_t client_addrlength = sizeof( client_address );
				int clientfd = accept( m_bindsocket, ( struct sockaddr* )&client_address, &client_addrlength );
				if(clientfd < 0)
				{
					DC_ERROR("accept error ,errmsg = %s",strerror(errno));
					continue ;
				}
				
				char remoteAddress[INET_ADDRSTRLEN ] = {0};
				inet_ntop( AF_INET, &client_address.sin_addr, remoteAddress, INET_ADDRSTRLEN );
				int remotePort = ntohs( client_address.sin_port );
				DC_INFO("%s:%d connect" , remoteAddress, remotePort );
				
				/*设置非阻塞*/
				if(0 != make_socket_nonblock(clientfd))
				{
					continue;
				}
	
				if(0 != add_socket_epoll(m_EpollVec[ClusterNum%m_nThreadNum], clientfd,true))
				{
					continue;
				}
				ClusterNum++;
            }
            else
            {
                DC_INFO("other thing happened ,event = %d ",events[i].events);
            } 
        }
    }
	
	m_bstop = true;
	StopServer();
	
	return SERVER_OK;
}



int SocketServer::make_socket_nonblock(int sock)
{
	int flags;
	if ((flags = fcntl(sock, F_GETFL, NULL)) < 0) 
	{
		DC_ERROR("fcntl(%d, F_GETFL) ,ermsg = %s", sock,strerror(errno));
		return SERVER_ERROR;
	}
	if (fcntl(sock, F_SETFL, flags | O_NONBLOCK) == -1)
	{
		DC_ERROR("fcntl(%d, F_SETFL) O_NONBLOCK ,ermsg = %s", sock,strerror(errno));
		return SERVER_ERROR;
	}
	return SERVER_OK;
	
}

int SocketServer::make_socket_reuseable(int sock)
{
	int reuse = 1;
    if(0 != setsockopt( sock, SOL_SOCKET, SO_REUSEADDR, &reuse, sizeof( reuse )))
	{
		DC_ERROR("setsockopt SO_REUSEADDR error ,errmsg = %s",strerror(errno));
		return SERVER_ERROR;
	}
	struct linger tcpLinger;
	tcpLinger.l_onoff  = 1;
	tcpLinger.l_linger = 0;

	if (0 != setsockopt(sock, SOL_SOCKET, SO_LINGER, &tcpLinger, sizeof(tcpLinger)))
	{
		DC_ERROR("setsockopt SO_LINGER error ,errmsg = %s",strerror(errno));
		return SERVER_ERROR;
	}
	return SERVER_OK;	
}

int SocketServer::make_socket_timeout(int sock,int time)
{
	/*查询和设置发送超时时间*/
    struct timeval send_timeout;  
	send_timeout.tv_sec = time;  
    send_timeout.tv_usec = 0; 
	int len = sizeof( timeval );	
    if(0 != setsockopt( sock, SOL_SOCKET, SO_SNDTIMEO, &send_timeout, sizeof(send_timeout) ))
	{
		DC_ERROR("setsockopt SO_SNDTIMEO error ,errmsg = %s",strerror(errno));
		return SERVER_ERROR;
	}
	getsockopt( sock, SOL_SOCKET, SO_SNDTIMEO, &send_timeout, ( socklen_t* )&len);
    DC_INFO( "the send timeout after settting is %ds", send_timeout.tv_sec/1000 );
	
	/*查询和设置接收超时时间*/
	struct timeval recv_timeout;  
	recv_timeout.tv_sec = time;  
    recv_timeout.tv_usec = 0;  
    if(0 != setsockopt( sock, SOL_SOCKET, SO_RCVTIMEO, &recv_timeout, sizeof( recv_timeout) ))
	{
		DC_ERROR("setsockopt SO_RCVTIMEO error ,errmsg = %s",strerror(errno));
		return SERVER_ERROR;
	}
	getsockopt( sock, SOL_SOCKET, SO_RCVTIMEO, &recv_timeout, ( socklen_t* )&len);
    DC_INFO( "the recv timeout after setting is %ds", recv_timeout.tv_sec/1000 );
	return SERVER_OK;
}
int SocketServer::make_socket_buffsize(int sock,int size)
{
	/*查询和设置接收缓冲区*/
	int recvbuf = 0;
	int len = sizeof( recvbuf );
	getsockopt( sock, SOL_SOCKET, SO_RCVBUF, &recvbuf, ( socklen_t* )&len);
    DC_INFO( "the receive buffer size before settting is %d", recvbuf );
	
	recvbuf = size;
    if(0 != setsockopt( sock, SOL_SOCKET, SO_RCVBUF, &recvbuf, sizeof( recvbuf) ))
	{
		DC_ERROR("setsockopt SO_RCVBUF error ,errmsg = %s",strerror(errno));
		return SERVER_ERROR;
	}
	getsockopt( sock, SOL_SOCKET, SO_RCVBUF, &recvbuf, ( socklen_t* )&len);
    DC_INFO( "the receive buffer size after settting is %d", recvbuf );
	
	/*查询和设置发送缓冲区*/
	int sendbuf = 0;
	getsockopt( sock, SOL_SOCKET, SO_SNDBUF, &sendbuf, ( socklen_t* )&len);
    DC_INFO( "the tcp send buffer size before setting is %d", sendbuf );
	
	sendbuf = size;
    if(0 != setsockopt( sock, SOL_SOCKET, SO_SNDBUF, &sendbuf, sizeof( sendbuf) ))
	{
		DC_ERROR("setsockopt SO_SNDBUF error ,errmsg = %s",strerror(errno));
		return SERVER_ERROR;
	}
	getsockopt( sock, SOL_SOCKET, SO_SNDBUF, &sendbuf, ( socklen_t* )&len);
    DC_INFO( "the tcp send buffer size after setting is %d", sendbuf );
	return SERVER_OK;
}

int SocketServer::add_socket_epoll(int epollfd ,int socket,bool oneshot)
{
	epoll_event event;
    event.data.fd = socket;
    event.events = EPOLLIN | EPOLLET ;
	if( oneshot )
    {
        event.events |= EPOLLONESHOT;
    }
    if(0 != epoll_ctl( epollfd, EPOLL_CTL_ADD, socket, &event ))
	{
		DC_ERROR("epoll_ctl  error ,errmsg = %s",strerror(errno));
		return SERVER_ERROR;
	}
	return SERVER_OK;
}

int SocketServer::reset_socket_epoll(int epollfd ,int socket)
{
	epoll_event event;
    event.data.fd = socket;
    event.events = EPOLLIN | EPOLLET | EPOLLONESHOT;
    if(0 != epoll_ctl( epollfd, EPOLL_CTL_MOD, socket, &event ))
	{
		DC_ERROR("epoll_ctl  error ,errmsg = %s",strerror(errno));
		return SERVER_ERROR;
	}
	return SERVER_OK;
}

int SocketServer::del_socket_epoll(int epollfd ,int socket)
{
    if(0 != epoll_ctl( epollfd, EPOLL_CTL_DEL, socket,0))
	{
		DC_ERROR("epoll_ctl  error ,errmsg = %s",strerror(errno));
		return SERVER_ERROR;
	}
	close(socket);
	return SERVER_OK;
}

void SocketServer::StopServer()
{
	/*关闭所有的文件描述符*/
	for (int sockfd = 3; sockfd < getdtablesize(); sockfd++)
	{
		close(sockfd);
	}
}

#ifndef _SOCKER_SERVER_H
#define _SOCKER_SERVER_H

#include "common.h"

class SocketServer
{
public:
	SocketServer(void);
	~SocketServer(void);
	
public:
	int StartServer(int port = 5001,int threadNum = 10);
	void StopServer();	
	
	static void S_WorkService(void* arg);
	void WorkService(int epollfd);
private:
	int make_socket_nonblock(int sock);	
	int make_socket_reuseable(int sock);
	int make_socket_timeout(int sock,int time);	
	int make_socket_buffsize(int sock,int size);	
	int add_socket_epoll(int epollfd ,int socket,bool oneshot);
	int reset_socket_epoll(int epollfd ,int socket);
	int del_socket_epoll(int epollfd ,int socket);

private:
	bool             m_bstop;                 //是否关闭
	int            	 m_nport;				  //端口号
	int            	 m_epollfd;               //主线程监听epoll 句柄
	int           	 m_bindsocket;            //监听的socket
	int           	 m_nThreadNum;            //接收数据工作线程的数量
	std::vector<int> m_EpollVec ;             //工作线程的epoll句柄集
};

typedef singleton<SocketServer> SOCKETServer;

#endif