一个可以直接用的最基本的iocp服务器
在每个TcpConn对象中可以自己增加各种回调接口
至于心跳检测 可以用带外数据oob在客户端来实现;
1.这个服务器没有使用acceptex
2.再wsarecv的时候并没有直接存放缓冲区. 而是在 GetQueuedCompletionStatus 返回再去recv数据
这样做的目的是为了减少内存的使用, 让wsarecv仅仅产生通知, 类似epoll, 因此GetQueuedCompletionStatus返回0也是正常的
代码里并没有让socket nonblock. 如果你修改成nonblock了, 则只需要修改 while(1) recv 直到返回wsawouldblock 即可;
3.给别人写的, 所以直接单文件
4.里面锁的过程 都可以用原子操作来替换. 因为是一个通用服务器,所以也只能做最大限度的保证, 如果对于发送数据能确定只由单一线程来发送的话,完全可以删除所有锁
enum EnumIOState { OP_None = 0 , OP_Read = 1, OP_Write = 2};
class CriticalLock{
public:
CriticalLock() {
InitializeCriticalSectionAndSpinCount(&__cs,4000);
}
~CriticalLock() {
DeleteCriticalSection(&__cs);
}
inline void lock() {
EnterCriticalSection(&__cs);
}
inline void unlock() {
LeaveCriticalSection(&__cs);
}
private:
CRITICAL_SECTION __cs;
};
struct SocketUtils{
static int setSocketLinger(SOCKET sock,bool bOpen, int iTimeout){
LINGER linger;
linger.l_onoff = bOpen;
linger.l_linger = iTimeout;
return ::setsockopt(sock, SOL_SOCKET, SO_LINGER, (char*)&linger,sizeof(LINGER));
}
static int setKeepAlive(SOCKET sock , bool bOpen){
int opt = bOpen;
return ::setsockopt(sock, SOL_SOCKET, SO_KEEPALIVE, (char*)&opt,
sizeof(opt));
}
static void forceCloseSokcet(SOCKET s) {
if(s == INVALID_SOCKET)
return;
setSocketLinger(s,true,0);
closesocket(s) ;
}
};
struct PerIOInfo{
OVERLAPPED __overlapped;
WSABUF __wsabuf;
int __op;
PerIOInfo(const int & op) : __op(op){
__wsabuf.buf = NULL;
__wsabuf.len = 0;
resetOverlapped();
}
void resetOverlapped() {
memset(&__overlapped , 0 , sizeof(__overlapped));
}
};
struct PerIOReadInfo : public PerIOInfo{
PerIOReadInfo() : PerIOInfo(EnumIOState::OP_Read){
memset(&__wsabuf,0,0);
}
void reset() {
resetOverlapped();
memset(&__wsabuf,0,0);
}
};
struct PerIOWriteInfo : public PerIOInfo {
unsigned long __iWrittenBytes;
unsigned long __iTotalBytes;
PerIOWriteInfo() : PerIOInfo(EnumIOState::OP_Write),
__iWrittenBytes(0),__iTotalBytes(0)
{
}
void reset() {
resetOverlapped();
__iWrittenBytes = 0;
__iTotalBytes = 0;
}
};
struct PerSocketInfo{
SOCKET __clientSocket;
SOCKADDR_IN __clientSocketAddr;
public:
PerSocketInfo(): __clientSocket(INVALID_SOCKET) {}
PerSocketInfo(SOCKET sock) : __clientSocket(sock){}
~PerSocketInfo() {
close();
}
void setAddr(SOCKADDR * addr , const int & iLen) {
memcpy(&__clientSocketAddr, addr, iLen);
}
inline bool isValidSocket() const { return __clientSocket != INVALID_SOCKET;}
void close() {
if(__clientSocket != INVALID_SOCKET){
closesocket(__clientSocket);
__clientSocket = INVALID_SOCKET;
}
}
void reset(SOCKET s = INVALID_SOCKET) {
memset(&__clientSocketAddr,0,sizeof(__clientSocketAddr));
close();
if(s != INVALID_SOCKET){
__clientSocket = s;
}
}
private:
PerSocketInfo(const PerSocketInfo &);
PerSocketInfo & operator=(const PerSocketInfo &);
};
struct TcpConn{
enum ConnState{ConnStateAlive = 0 , ConnStateDelete = 1};
PerSocketInfo __socketInfo;
PerIOReadInfo __readInfo;
PerIOWriteInfo __writeInfo;
volatile long __connState;
HANDLE __hProcessSocketIOCP;
CriticalLock __lock;
std::deque<std::vector<char>> __sendBuffers;
bool __bSending;
bool __bRecving;
TcpConn(SOCKET s , HANDLE hProcessSocketIOCP):
__socketInfo(s),
__connState(ConnStateAlive),
__hProcessSocketIOCP(hProcessSocketIOCP),
__bSending(false),
__bRecving(true)
{}
~TcpConn() {
__sendBuffers.clear();
}
bool changeConnStateDelete() {
return ConnState::ConnStateAlive == InterlockedCompareExchange(&__connState,ConnState::ConnStateDelete,ConnState::ConnStateAlive);
}
SOCKET getSocket() { return __socketInfo.__clientSocket;}
inline bool isConnAlive() {
return isValidSocket() && (ConnState::ConnStateAlive == InterlockedCompareExchange(&__connState,ConnState::ConnStateAlive,ConnState::ConnStateAlive));
}
inline bool isValidSocket() { return __socketInfo.isValidSocket();}
inline bool isAdd() { return ConnState::ConnStateAlive == __connState;}
inline bool isDelete() { return ConnState::ConnStateDelete == __connState;}
inline void lock() { __lock.lock();}
inline void unlock() {__lock.unlock();}
inline void shutdownSocket() {
if(isValidSocket())
::shutdown(getSocket(),SD_BOTH );
}
void closeSocketAndPostDelete() {
__socketInfo.close();
changeConnStateDelete();
if(false == __bSending && false == __bRecving){
PostQueuedCompletionStatus(__hProcessSocketIOCP,0,(ULONG_PTR)this,0);
}
}
void handleRead(const DWORD & recvBytes) {
using std::cout;
using std::endl;
char szBuffer[16384] = {0};
lock();
if( false == isValidSocket()) {
__bRecving = false;
closeSocketAndPostDelete();
unlock();
return;
}
int iRet = ::recv(getSocket(),szBuffer,sizeof(szBuffer),0);
if(iRet <= 0 ){
__bRecving = false;
closeSocketAndPostDelete();
unlock();
return;
}
//处理数据
szBuffer[iRet] = 0;
cout << "data:" << szBuffer << endl;
recvData();
unlock();
}
void handleSend(const DWORD & sendBytes) {
lock();
if(0 == sendBytes || false == isValidSocket()) {
__bSending = false;
closeSocketAndPostDelete();
unlock();
return;
}
__writeInfo.__iWrittenBytes += sendBytes;
if(__writeInfo.__iWrittenBytes < __writeInfo.__iTotalBytes){
__writeInfo.__wsabuf.buf = __sendBuffers.front().data() + __writeInfo.__iWrittenBytes;
__writeInfo.__wsabuf.len = __writeInfo.__iTotalBytes - __writeInfo.__iWrittenBytes;
sendLeftData();
unlock();
return;
}
__sendBuffers.pop_front();
if(__sendBuffers.size() > 0){
__writeInfo.reset();
__writeInfo.__wsabuf.buf = __sendBuffers.front().data();
__writeInfo.__wsabuf.len = __sendBuffers.front().size();
__writeInfo.__iTotalBytes = __writeInfo.__wsabuf.len;
sendLeftData();
unlock();
}
else{
__bSending = false;
}
unlock();
}
void sendLeftData() {
int iLastError = 0;
int iRet = 0;
DWORD dwBytesSend = 0;
__writeInfo.resetOverlapped();
iRet = WSASend(getSocket(),&__writeInfo.__wsabuf,1,&dwBytesSend,0,&__writeInfo.__overlapped,NULL);
iLastError = WSAGetLastError();
if(SOCKET_ERROR == iRet && (WSA_IO_PENDING != iLastError)){
__bSending = false;
closeSocketAndPostDelete();
}
}
bool recvData() {
DWORD dwRecvBytes = 0;
DWORD dwFlag = 0;
int iLastError = 0;
__readInfo.resetOverlapped();
int iRet = WSARecv(__socketInfo.__clientSocket,&__readInfo.__wsabuf,1,&dwRecvBytes,&dwFlag,&__readInfo.__overlapped,NULL);
iLastError = WSAGetLastError();;
if(SOCKET_ERROR == iRet && WSA_IO_PENDING != iLastError){
__bRecving = false;
closeSocketAndPostDelete();
return false;
}
return true;
}
bool sendData(const char * pData, unsigned int & iLen) {
if( NULL == pData || iLen < 1) return false;
return sendData(std::vector<char>(pData, pData + iLen));
}
bool sendData(const std::string & data) {
if(data.size() < 1) return false;
return sendData(std::vector<char>(data.c_str(),data.c_str()+data.size()));
}
bool sendData(const std::vector<char> & buffer) {
DWORD dwBytesSend = 0;
int iRet = 0;
int iLastError = 0;
//用原子判断只是为了用户在其他线程疯狂调用. 不用每次lock
if( buffer.size() < 1 || !isConnAlive()) return false;
lock();
if(false == __socketInfo.isValidSocket() ) {
unlock();
return false;
}
__sendBuffers.push_back(std::move(buffer));
if(!__bSending){
__writeInfo.reset();
__writeInfo.__wsabuf.buf = __sendBuffers.front().data();
__writeInfo.__wsabuf.len = __sendBuffers.front().size();
__writeInfo.__iTotalBytes = __writeInfo.__wsabuf.len;
iRet = WSASend(getSocket(),&__writeInfo.__wsabuf,1,&dwBytesSend,0,&__writeInfo.__overlapped,NULL);
iLastError = WSAGetLastError();
if(iRet == SOCKET_ERROR && (iLastError != WSA_IO_PENDING)){
closeSocketAndPostDelete();
unlock();
return false;
}
__bSending = true;
}
unlock();
return 0 == iRet;
}
void reset(){
__socketInfo.reset();
__readInfo.reset();
__writeInfo.reset();
__sendBuffers.clear();
__connState = ConnState::ConnStateAlive;
}
};
class Acceptor{
public:
Acceptor():
__hListenSocket(INVALID_SOCKET),
__hRWIOCP(NULL),
__hProcessSocketIOCP(NULL),
__hAcceptThread(NULL),
__hProcessSocketThread(NULL)
{
}
void start(SOCKET sListenSocket , HANDLE hRWIOCP) {
__hListenSocket = sListenSocket;
__hRWIOCP = hRWIOCP;
__hAcceptThread = (HANDLE)_beginthreadex(0,0,startAccept,this,0,0);
assert(__hAcceptThread != NULL);
}
~Acceptor(){
stop();
}
void stop() {
if(__hListenSocket != INVALID_SOCKET)
closesocket(__hListenSocket);
WaitForSingleObject(__hAcceptThread,INFINITE);
CloseHandle(__hAcceptThread);
}
private:
static bool bindSocketAndIOCP(SOCKET s, HANDLE hIOCP , ULONG_PTR bindKey) {
return ( hIOCP == CreateIoCompletionPort((HANDLE)s,hIOCP,bindKey,0) );
}
void stopProcessSocketThread() {
PostQueuedCompletionStatus(__hProcessSocketIOCP,0,0,0);
WaitForSingleObject(__hProcessSocketThread,INFINITE);
CloseHandle(__hProcessSocketIOCP);
CloseHandle(__hProcessSocketThread);
}
bool initProcessSocketIOCPAndThread() {
__hProcessSocketIOCP = CreateIoCompletionPort(INVALID_HANDLE_VALUE,NULL,NULL,1);
return __hProcessSocketIOCP != NULL;
}
virtual void startAcceptThread() {
initProcessSocketIOCPAndThread();
__hProcessSocketThread = (HANDLE)_beginthreadex(0,0,startProcessSocket,this,0,0);
assert(__hProcessSocketThread != NULL);
using std::cout;
using std::endl;
SOCKET hClientSokcet;
SOCKADDR_IN sClientAddr;
int iAddrlen = 0;
int iError = 0 ;
bool bBindSuccess = false;
cout << "start accept tid:" << GetCurrentThreadId() << endl;
while(1) {
iAddrlen = sizeof(sClientAddr);
hClientSokcet = ::accept( __hListenSocket , (SOCKADDR* )&sClientAddr, &iAddrlen);
if(hClientSokcet == INVALID_SOCKET) {
iError = WSAGetLastError();
if(WSAEINTR == iError){
cout << "accept thread quit ,listen socket closed" << endl;
break;
}
else {
cout << "accept failed : " << iError << endl;
continue;
}
}
TcpConn * conn = new TcpConn(hClientSokcet,__hProcessSocketIOCP);
conn->__socketInfo.setAddr((SOCKADDR* )&sClientAddr,iAddrlen);
bBindSuccess = bindSocketAndIOCP(hClientSokcet,__hRWIOCP,(ULONG_PTR)conn);
if(!bBindSuccess) {
delete conn;
cout << "bindSocketAndIOCP failed" << endl;
continue;
}
CreateIoCompletionPort((HANDLE)hClientSokcet,__hRWIOCP,(ULONG_PTR)conn,0);
PostQueuedCompletionStatus(__hProcessSocketIOCP,0,(ULONG_PTR)conn, &conn->__readInfo.__overlapped);
}
stopProcessSocketThread();
}
virtual void processSocketAddAndDelete() {
using std::cout;
using std::endl;
DWORD dwBytes = 0;
TcpConn * conn = NULL;
OVERLAPPED * overlapped = NULL;
BOOL iRet = FALSE;
DWORD iLastError = 0;
DWORD tid = GetCurrentThreadId();
bool bQuit = false;
int iNeedQuitCount = 0 ;
while(1) {
iRet = GetQueuedCompletionStatus(__hProcessSocketIOCP,&dwBytes,(PULONG_PTR)&conn, &overlapped,INFINITE);
iLastError = GetLastError();
if(FALSE == iRet) {
if(ERROR_ABANDONED_WAIT_0 == iLastError)
cout << "iocp closed" << endl;
else
cout << "quit error:" << iLastError << endl;
break;
}
if(NULL == conn) {
//自己关闭, 通知退出线程前 关闭所有conn,等待所有conn正常退出
bQuit = true;
iNeedQuitCount = __conn.size();
if(0 == iNeedQuitCount)
break;
for( std::map<TcpConn *, TcpConn *>::iterator iter_begin = __conn.begin() ; iter_begin != __conn.end(); ++iter_begin) {
iter_begin->second->shutdownSocket();
}
continue;
}
if( conn->isAdd()){
__conn.insert(std::make_pair(conn,conn));
conn->recvData();
}
else if( conn->isDelete()) {
__conn.erase(conn);
delete conn;
}
else{
cout << __func__ << " conn error" << endl;
}
if(bQuit){
--iNeedQuitCount;
if(0 == iNeedQuitCount)
break;
}
}
if(!__conn.empty()){
std::map<TcpConn *, TcpConn *>::iterator iter_begin = __conn.begin();
for(;iter_begin != __conn.end() ;){
delete iter_begin->second;
}
__conn.clear();
}
}
static unsigned int startProcessSocket(void * arg){
Acceptor * obj = (Acceptor *)arg;
obj->processSocketAddAndDelete();
return 0;
}
static unsigned int startAccept(void * arg) {
Acceptor * obj = (Acceptor *)arg;
obj->startAcceptThread();
return 0;
}
private:
HANDLE __hProcessSocketThread;
HANDLE __hAcceptThread;
HANDLE __hRWIOCP;
HANDLE __hProcessSocketIOCP;
SOCKET __hListenSocket;
std::map<TcpConn*, TcpConn*> __conn;
private:
Acceptor(const Acceptor &);
Acceptor & operator=(const Acceptor &);
};
class IOCPServer {
public:
IOCPServer(const unsigned short & port = 19988 ):
__listenSocket( INVALID_SOCKET ),
__servPort(port),
__iServerThreads(0),
__hIOCP(NULL),
__arrIOCPThreads(NULL)
{
}
bool start() {
bool bSuccess = initListenSocket() && initServerThreads();
if(bSuccess)
__acceptor.start(__listenSocket,__hIOCP);
return bSuccess;
}
void stop() {
if(__listenSocket != INVALID_SOCKET){
closesocket(__listenSocket);
__listenSocket = INVALID_SOCKET;
__acceptor.stop();
}
closeServerThreads();
}
private:
void closeServerThreads() {
for(int i = 0; i < __iServerThreads; ++i) {
PostQueuedCompletionStatus(__hIOCP,0,0,0);
}
WaitForMultipleObjects(__iServerThreads,__arrIOCPThreads,TRUE,INFINITE);
}
static unsigned int __stdcall iocpThread(void * arg) {
IOCPServer * obj = (IOCPServer * ) arg;
HANDLE hIOCP = obj->__hIOCP;
DWORD dwBytes = 0;
TcpConn * conn = NULL;
OVERLAPPED * overlapped = NULL;
BOOL bSuccess = FALSE;
DWORD iLastError = 0;
DWORD tid = GetCurrentThreadId();
PerIOInfo * ioInfo = NULL;
using namespace std;
while(1) {
bSuccess = GetQueuedCompletionStatus(hIOCP,&dwBytes,(PULONG_PTR) &conn,&overlapped,INFINITE);
iLastError = GetLastError();
if(FALSE == bSuccess) {
if(ERROR_ABANDONED_WAIT_0 == iLastError) {
// close iocp;
cout << "close iocp" << endl;
break;
}
}
if(NULL == conn && NULL == overlapped ){
break;
}
ioInfo = (PerIOInfo *)overlapped;
if(OP_Read == ioInfo->__op){
conn->handleRead(dwBytes);
}
else if (OP_Write == ioInfo->__op){
conn->handleSend(dwBytes);
}
}
return 0;
}
bool initServerThreads(){
using namespace std;
SYSTEM_INFO sysInfo = {0};
GetSystemInfo(&sysInfo);
__iServerThreads = sysInfo.dwNumberOfProcessors + 2;
__hIOCP = CreateIoCompletionPort(INVALID_HANDLE_VALUE,0,0,sysInfo.dwNumberOfProcessors);
if(NULL == __hIOCP){
cout << "CreateIoCompletionPort failed : " << GetLastError() << endl;
return false;
}
__arrIOCPThreads = new HANDLE[__iServerThreads];
assert(__arrIOCPThreads != NULL);
for(int i = 0 ; i < __iServerThreads; ++i){
__arrIOCPThreads[i] = (HANDLE)_beginthreadex(0,0,iocpThread,this,0,0);
}
return true;
}
bool initListenSocket() {
using namespace std;
if(INVALID_SOCKET !=__listenSocket){
closesocket(__listenSocket);
}
SOCKET hListenSocket = WSASocketW(AF_INET,SOCK_STREAM,0,0,0,WSA_FLAG_OVERLAPPED);
if( INVALID_SOCKET == hListenSocket ){
cout << "WSASocketW failed:" << WSAGetLastError() << endl;
return false;
}
int openReuse = 1;
int openReuseLen = sizeof(openReuse);
setsockopt(hListenSocket,SOL_SOCKET,SO_REUSEADDR,(char*)&openReuse,openReuseLen);
memset(&__servAddr,0,sizeof(__servAddr));
__servAddr.sin_addr.S_un.S_addr = INADDR_ANY;
__servAddr.sin_port = htons(__servPort);
__servAddr.sin_family = AF_INET;
if( SOCKET_ERROR == ::bind(hListenSocket,(struct sockaddr *)&__servAddr,sizeof(__servAddr)) ) {
cout << "bind failed : " << WSAGetLastError() << endl;
closesocket(hListenSocket);
return false;
}
if( SOCKET_ERROR == ::listen(hListenSocket,SOMAXCONN) ) {
cout << "listen failed: " << WSAGetLastError() << endl;
closesocket(hListenSocket);
return false;
}
__listenSocket = hListenSocket;
return true;
}
private:
SOCKET __listenSocket;
unsigned short __servPort;
SOCKADDR_IN __servAddr;
int __iServerThreads;
HANDLE __hIOCP;
HANDLE * __arrIOCPThreads;
Acceptor __acceptor;
private:
IOCPServer(const IOCPServer &);
IOCPServer & operator=(const IOCPServer);
};
int main(int argc, char *argv[])
{
using namespace std;
WSADATA wsadata;
if( NO_ERROR != WSAStartup(MAKEWORD(2,2),&wsadata) ) {
cout << "wsastartup failed" << endl;
return 0;
}
IOCPServer serv;
serv.start();
//服务器启动完成, 可以做别的事
while(1)
Sleep(1000);
WSACleanup();
return 0;
}