完成端口服务器

一个可以直接用的最基本的iocp服务器

在每个TcpConn对象中可以自己增加各种回调接口

至于心跳检测 可以用带外数据oob在客户端来实现;

 

1.这个服务器没有使用acceptex

2.再wsarecv的时候并没有直接存放缓冲区. 而是在 GetQueuedCompletionStatus 返回再去recv数据

   这样做的目的是为了减少内存的使用, 让wsarecv仅仅产生通知, 类似epoll, 因此GetQueuedCompletionStatus返回0也是正常的

   代码里并没有让socket nonblock. 如果你修改成nonblock了, 则只需要修改 while(1)  recv  直到返回wsawouldblock 即可;

 

3.给别人写的, 所以直接单文件

4.里面锁的过程 都可以用原子操作来替换. 因为是一个通用服务器,所以也只能做最大限度的保证, 如果对于发送数据能确定只由单一线程来发送的话,完全可以删除所有锁

 

enum EnumIOState { OP_None = 0 , OP_Read = 1, OP_Write = 2};
class  CriticalLock{
public:
    CriticalLock() {
        InitializeCriticalSectionAndSpinCount(&__cs,4000);
    }
    ~CriticalLock() {
        DeleteCriticalSection(&__cs);
    }
    inline void lock() {
        EnterCriticalSection(&__cs);
    }
    inline void unlock() {
        LeaveCriticalSection(&__cs);
    }

private:
    CRITICAL_SECTION __cs;
};

struct SocketUtils{
    static int setSocketLinger(SOCKET sock,bool bOpen, int iTimeout){
        LINGER linger;
        linger.l_onoff = bOpen;
        linger.l_linger = iTimeout;
        return ::setsockopt(sock, SOL_SOCKET, SO_LINGER, (char*)&linger,sizeof(LINGER));
    }

    static int setKeepAlive(SOCKET sock , bool bOpen){
        int opt = bOpen;
        return ::setsockopt(sock, SOL_SOCKET, SO_KEEPALIVE, (char*)&opt,
            sizeof(opt));
    }

    static void forceCloseSokcet(SOCKET s) {
        if(s == INVALID_SOCKET)
            return;

        setSocketLinger(s,true,0);
        closesocket(s) ;
    }
};



struct PerIOInfo{
    OVERLAPPED __overlapped;
    WSABUF __wsabuf;
    int __op;

    PerIOInfo(const int & op) : __op(op){
        __wsabuf.buf = NULL;
        __wsabuf.len = 0;
        resetOverlapped();
    }


   void resetOverlapped() {
       memset(&__overlapped , 0 , sizeof(__overlapped));
    }
};

struct PerIOReadInfo : public PerIOInfo{
    PerIOReadInfo() : PerIOInfo(EnumIOState::OP_Read){
        memset(&__wsabuf,0,0);
    }

    void reset() {
        resetOverlapped();
        memset(&__wsabuf,0,0);
    }

};


struct PerIOWriteInfo : public PerIOInfo {
    unsigned long __iWrittenBytes;
    unsigned long __iTotalBytes;
    PerIOWriteInfo() : PerIOInfo(EnumIOState::OP_Write),
        __iWrittenBytes(0),__iTotalBytes(0)
    {
    }

    void reset() {
        resetOverlapped();
        __iWrittenBytes = 0;
        __iTotalBytes = 0;
    }


};


struct PerSocketInfo{
    SOCKET __clientSocket;
    SOCKADDR_IN __clientSocketAddr;


public:
    PerSocketInfo(): __clientSocket(INVALID_SOCKET) {}
    PerSocketInfo(SOCKET sock) : __clientSocket(sock){}
    ~PerSocketInfo() {
        close();
    }

    void setAddr(SOCKADDR * addr , const int & iLen) {
        memcpy(&__clientSocketAddr, addr, iLen);
    }
    inline bool isValidSocket() const { return __clientSocket != INVALID_SOCKET;}

    void close() {
        if(__clientSocket != INVALID_SOCKET){
            closesocket(__clientSocket);
             __clientSocket = INVALID_SOCKET;
        }
    }

    void reset(SOCKET s = INVALID_SOCKET) {
        memset(&__clientSocketAddr,0,sizeof(__clientSocketAddr));
        close();
        if(s != INVALID_SOCKET){
            __clientSocket = s;
        }
    }


private:
    PerSocketInfo(const PerSocketInfo &);
    PerSocketInfo & operator=(const PerSocketInfo &);


};

struct TcpConn{
    enum ConnState{ConnStateAlive = 0 , ConnStateDelete = 1};

    PerSocketInfo __socketInfo;
    PerIOReadInfo __readInfo;
    PerIOWriteInfo __writeInfo;
    volatile long __connState;
    HANDLE __hProcessSocketIOCP;
    CriticalLock __lock;
    std::deque<std::vector<char>> __sendBuffers;
    bool __bSending;
    bool __bRecving;

    TcpConn(SOCKET s , HANDLE hProcessSocketIOCP):
        __socketInfo(s),
        __connState(ConnStateAlive),
        __hProcessSocketIOCP(hProcessSocketIOCP),
        __bSending(false),
        __bRecving(true)
    {}
    ~TcpConn() {
        __sendBuffers.clear();
    }

    bool changeConnStateDelete() {
        return ConnState::ConnStateAlive == InterlockedCompareExchange(&__connState,ConnState::ConnStateDelete,ConnState::ConnStateAlive);
    }


    SOCKET getSocket() { return __socketInfo.__clientSocket;}

    inline bool isConnAlive() {
         return  isValidSocket() &&  (ConnState::ConnStateAlive == InterlockedCompareExchange(&__connState,ConnState::ConnStateAlive,ConnState::ConnStateAlive));
    }

    inline bool isValidSocket() { return __socketInfo.isValidSocket();}
    inline bool isAdd() { return ConnState::ConnStateAlive == __connState;}
    inline bool isDelete() { return ConnState::ConnStateDelete == __connState;}

    inline void lock() { __lock.lock();}
    inline void unlock() {__lock.unlock();}
    inline void shutdownSocket() {
        if(isValidSocket())
            ::shutdown(getSocket(),SD_BOTH );
    }

    void closeSocketAndPostDelete() {
        __socketInfo.close();
        changeConnStateDelete();
        if(false == __bSending && false == __bRecving){
             PostQueuedCompletionStatus(__hProcessSocketIOCP,0,(ULONG_PTR)this,0);
        }

    }

    void handleRead(const DWORD & recvBytes) {
        using std::cout;
        using std::endl;
        char szBuffer[16384] = {0};

        lock();
        if( false == isValidSocket()) {
            __bRecving = false;
            closeSocketAndPostDelete();
            unlock();
            return;
        }

        int iRet = ::recv(getSocket(),szBuffer,sizeof(szBuffer),0);
        if(iRet <= 0 ){
            __bRecving = false;
            closeSocketAndPostDelete();
            unlock();
            return;
        }

        //处理数据
        szBuffer[iRet] = 0;
        cout << "data:" << szBuffer << endl;

        recvData();
        unlock();
    }

    void handleSend(const DWORD & sendBytes) {

        lock();
        if(0 == sendBytes || false == isValidSocket()) {
            __bSending = false;
            closeSocketAndPostDelete();
            unlock();
            return;
        }

        __writeInfo.__iWrittenBytes += sendBytes;
        if(__writeInfo.__iWrittenBytes  < __writeInfo.__iTotalBytes){
            __writeInfo.__wsabuf.buf = __sendBuffers.front().data() + __writeInfo.__iWrittenBytes;
            __writeInfo.__wsabuf.len = __writeInfo.__iTotalBytes - __writeInfo.__iWrittenBytes;

            sendLeftData();
            unlock();
            return;
        }

        __sendBuffers.pop_front();
        if(__sendBuffers.size() > 0){
            __writeInfo.reset();
            __writeInfo.__wsabuf.buf = __sendBuffers.front().data();
            __writeInfo.__wsabuf.len = __sendBuffers.front().size();
            __writeInfo.__iTotalBytes = __writeInfo.__wsabuf.len;

            sendLeftData();
            unlock();
        }
        else{
            __bSending = false;
        }

        unlock();
    }

    void sendLeftData() {
        int iLastError = 0;
        int iRet = 0;
        DWORD dwBytesSend = 0;
        __writeInfo.resetOverlapped();

        iRet = WSASend(getSocket(),&__writeInfo.__wsabuf,1,&dwBytesSend,0,&__writeInfo.__overlapped,NULL);
        iLastError = WSAGetLastError();
        if(SOCKET_ERROR == iRet && (WSA_IO_PENDING != iLastError)){
            __bSending = false;
            closeSocketAndPostDelete();
        }
    }


    bool recvData() {
        DWORD dwRecvBytes = 0;
        DWORD dwFlag = 0;
        int iLastError = 0;
        __readInfo.resetOverlapped();

        int iRet = WSARecv(__socketInfo.__clientSocket,&__readInfo.__wsabuf,1,&dwRecvBytes,&dwFlag,&__readInfo.__overlapped,NULL);
        iLastError = WSAGetLastError();;
        if(SOCKET_ERROR == iRet && WSA_IO_PENDING != iLastError){
            __bRecving = false;
            closeSocketAndPostDelete();
            return false;
        }

        return true;
    }

    bool sendData(const char * pData, unsigned int & iLen) {
        if( NULL == pData || iLen < 1) return false;
        return sendData(std::vector<char>(pData, pData + iLen));
    }

    bool sendData(const std::string & data) {
        if(data.size() < 1) return false;
        return sendData(std::vector<char>(data.c_str(),data.c_str()+data.size()));
    }

    bool sendData(const std::vector<char> & buffer) {
        DWORD dwBytesSend = 0;
        int iRet = 0;
        int iLastError = 0;

        //用原子判断只是为了用户在其他线程疯狂调用. 不用每次lock
        if( buffer.size() < 1 || !isConnAlive()) return false;

        lock();
        if(false == __socketInfo.isValidSocket() ) {
            unlock();
            return false;
        }

        __sendBuffers.push_back(std::move(buffer));
        if(!__bSending){
            __writeInfo.reset();
            __writeInfo.__wsabuf.buf = __sendBuffers.front().data();
            __writeInfo.__wsabuf.len = __sendBuffers.front().size();
            __writeInfo.__iTotalBytes = __writeInfo.__wsabuf.len;
            iRet = WSASend(getSocket(),&__writeInfo.__wsabuf,1,&dwBytesSend,0,&__writeInfo.__overlapped,NULL);
            iLastError = WSAGetLastError();
            if(iRet == SOCKET_ERROR && (iLastError != WSA_IO_PENDING)){
                closeSocketAndPostDelete();
                unlock();
                return false;
            }

            __bSending = true;
        }
        unlock();
        return 0 ==  iRet;
    }

    void reset(){
        __socketInfo.reset();
        __readInfo.reset();
        __writeInfo.reset();
        __sendBuffers.clear();
        __connState = ConnState::ConnStateAlive;
    }
};


class Acceptor{
public:
    Acceptor():
        __hListenSocket(INVALID_SOCKET),
        __hRWIOCP(NULL),
        __hProcessSocketIOCP(NULL),
        __hAcceptThread(NULL),
        __hProcessSocketThread(NULL)
    {
    }



    void start(SOCKET sListenSocket , HANDLE hRWIOCP) {
        __hListenSocket = sListenSocket;
        __hRWIOCP = hRWIOCP;

        __hAcceptThread = (HANDLE)_beginthreadex(0,0,startAccept,this,0,0);
        assert(__hAcceptThread != NULL);
    }

    ~Acceptor(){
        stop();
    }

    void stop() {
        if(__hListenSocket != INVALID_SOCKET)
            closesocket(__hListenSocket);

        WaitForSingleObject(__hAcceptThread,INFINITE);
        CloseHandle(__hAcceptThread);

    }

private:
    static bool bindSocketAndIOCP(SOCKET s, HANDLE hIOCP , ULONG_PTR bindKey) {
         return ( hIOCP == CreateIoCompletionPort((HANDLE)s,hIOCP,bindKey,0) );
    }

    void stopProcessSocketThread() {
        PostQueuedCompletionStatus(__hProcessSocketIOCP,0,0,0);
        WaitForSingleObject(__hProcessSocketThread,INFINITE);
        CloseHandle(__hProcessSocketIOCP);
        CloseHandle(__hProcessSocketThread);
    }

    bool initProcessSocketIOCPAndThread() {
        __hProcessSocketIOCP = CreateIoCompletionPort(INVALID_HANDLE_VALUE,NULL,NULL,1);
        return __hProcessSocketIOCP != NULL;
    }

    virtual void startAcceptThread() {
        initProcessSocketIOCPAndThread();
        __hProcessSocketThread = (HANDLE)_beginthreadex(0,0,startProcessSocket,this,0,0);
        assert(__hProcessSocketThread != NULL);

        using std::cout;
        using std::endl;


        SOCKET hClientSokcet;
        SOCKADDR_IN sClientAddr;
        int iAddrlen = 0;
        int iError = 0 ;
        bool bBindSuccess = false;

        cout <<  "start accept tid:" << GetCurrentThreadId() << endl;

        while(1) {

            iAddrlen = sizeof(sClientAddr);
            hClientSokcet = ::accept( __hListenSocket , (SOCKADDR* )&sClientAddr, &iAddrlen);
            if(hClientSokcet ==  INVALID_SOCKET) {
                iError = WSAGetLastError();
                if(WSAEINTR == iError){
                    cout << "accept thread quit ,listen socket closed" << endl;
                    break;
                }
                else {
                    cout << "accept failed : " << iError << endl;
                    continue;
                }
            }



            TcpConn * conn = new TcpConn(hClientSokcet,__hProcessSocketIOCP);
            conn->__socketInfo.setAddr((SOCKADDR* )&sClientAddr,iAddrlen);
            bBindSuccess = bindSocketAndIOCP(hClientSokcet,__hRWIOCP,(ULONG_PTR)conn);
            if(!bBindSuccess) {
                delete conn;
                cout << "bindSocketAndIOCP failed" << endl;
                continue;
            }


            CreateIoCompletionPort((HANDLE)hClientSokcet,__hRWIOCP,(ULONG_PTR)conn,0);
            PostQueuedCompletionStatus(__hProcessSocketIOCP,0,(ULONG_PTR)conn, &conn->__readInfo.__overlapped);

        }

        stopProcessSocketThread();

    }



    virtual void processSocketAddAndDelete() {
        using std::cout;
        using std::endl;


        DWORD dwBytes = 0;
        TcpConn * conn  = NULL;
        OVERLAPPED  * overlapped = NULL;
        BOOL iRet = FALSE;
        DWORD iLastError = 0;
        DWORD tid = GetCurrentThreadId();

        bool bQuit = false;
        int iNeedQuitCount = 0 ;


        while(1) {
            iRet = GetQueuedCompletionStatus(__hProcessSocketIOCP,&dwBytes,(PULONG_PTR)&conn, &overlapped,INFINITE);
            iLastError = GetLastError();
            if(FALSE == iRet) {
                if(ERROR_ABANDONED_WAIT_0 == iLastError)
                    cout << "iocp closed" << endl;
                else
                    cout << "quit error:" << iLastError << endl;
                break;
            }


            if(NULL == conn) {
                //自己关闭, 通知退出线程前 关闭所有conn,等待所有conn正常退出
                bQuit = true;
                iNeedQuitCount = __conn.size();
                if(0 == iNeedQuitCount)
                    break;
                for( std::map<TcpConn *, TcpConn *>::iterator iter_begin = __conn.begin() ; iter_begin != __conn.end(); ++iter_begin) {
                    iter_begin->second->shutdownSocket();
                }
                continue;
            }

            if( conn->isAdd()){
                __conn.insert(std::make_pair(conn,conn));
                conn->recvData();
            }
            else if( conn->isDelete()) {
                __conn.erase(conn);
                delete conn;
            }
            else{
                cout << __func__ << " conn error" << endl;
            }

            if(bQuit){
                --iNeedQuitCount;
                if(0 == iNeedQuitCount)
                    break;
            }
        }


        if(!__conn.empty()){
            std::map<TcpConn *, TcpConn *>::iterator iter_begin = __conn.begin();
            for(;iter_begin != __conn.end() ;){
                delete iter_begin->second;
            }
           __conn.clear();
        }
    }

    static unsigned int startProcessSocket(void * arg){
         Acceptor * obj  = (Acceptor *)arg;
        obj->processSocketAddAndDelete();
        return 0;
    }

    static unsigned int startAccept(void * arg) {
          Acceptor * obj  = (Acceptor *)arg;
          obj->startAcceptThread();
          return 0;
    }
private:

    HANDLE __hProcessSocketThread;
    HANDLE __hAcceptThread;

    HANDLE __hRWIOCP;
    HANDLE __hProcessSocketIOCP;

    SOCKET __hListenSocket;


    std::map<TcpConn*, TcpConn*> __conn;
private:
    Acceptor(const Acceptor &);
    Acceptor & operator=(const Acceptor &);
};


class IOCPServer {
public:

    IOCPServer(const unsigned short  & port = 19988 ):
        __listenSocket( INVALID_SOCKET ),
        __servPort(port),
        __iServerThreads(0),
        __hIOCP(NULL),
        __arrIOCPThreads(NULL)
    {

    }



    bool start() {
        bool bSuccess = initListenSocket() && initServerThreads();
        if(bSuccess)
            __acceptor.start(__listenSocket,__hIOCP);
        return bSuccess;
    }

    void stop() {
        if(__listenSocket != INVALID_SOCKET){
            closesocket(__listenSocket);
            __listenSocket = INVALID_SOCKET;
            __acceptor.stop();
        }
        closeServerThreads();
    }

private:
    void closeServerThreads() {
        for(int i = 0; i < __iServerThreads; ++i) {
            PostQueuedCompletionStatus(__hIOCP,0,0,0);
        }
        WaitForMultipleObjects(__iServerThreads,__arrIOCPThreads,TRUE,INFINITE);
    }


    static unsigned int __stdcall  iocpThread(void *  arg) {

        IOCPServer * obj = (IOCPServer * ) arg;
        HANDLE hIOCP = obj->__hIOCP;
        DWORD dwBytes = 0;
        TcpConn * conn = NULL;
        OVERLAPPED * overlapped = NULL;
        BOOL bSuccess = FALSE;
        DWORD iLastError = 0;
        DWORD tid = GetCurrentThreadId();
        PerIOInfo * ioInfo = NULL;
        using namespace  std;



        while(1) {
            bSuccess = GetQueuedCompletionStatus(hIOCP,&dwBytes,(PULONG_PTR) &conn,&overlapped,INFINITE);
            iLastError = GetLastError();


            if(FALSE == bSuccess) {
                if(ERROR_ABANDONED_WAIT_0 == iLastError) {
                    // close iocp;
                    cout << "close iocp" << endl;
                    break;
                }
            }

            if(NULL == conn && NULL == overlapped ){
                break;
            }

            ioInfo = (PerIOInfo *)overlapped;
            if(OP_Read == ioInfo->__op){
                conn->handleRead(dwBytes);
            }
            else if (OP_Write == ioInfo->__op){
                conn->handleSend(dwBytes);
            }

        }




        return 0;
    }

    bool initServerThreads(){
        using namespace  std;
        SYSTEM_INFO sysInfo = {0};
        GetSystemInfo(&sysInfo);
        __iServerThreads = sysInfo.dwNumberOfProcessors + 2;
        __hIOCP = CreateIoCompletionPort(INVALID_HANDLE_VALUE,0,0,sysInfo.dwNumberOfProcessors);

        if(NULL == __hIOCP){
            cout <<  "CreateIoCompletionPort failed : " << GetLastError() << endl;
            return false;
        }

        __arrIOCPThreads = new HANDLE[__iServerThreads];
        assert(__arrIOCPThreads != NULL);

        for(int i = 0 ;  i < __iServerThreads; ++i){
            __arrIOCPThreads[i] = (HANDLE)_beginthreadex(0,0,iocpThread,this,0,0);
        }


        return true;
    }



    bool initListenSocket() {
        using namespace  std;
        if(INVALID_SOCKET !=__listenSocket){
            closesocket(__listenSocket);
        }

        SOCKET hListenSocket = WSASocketW(AF_INET,SOCK_STREAM,0,0,0,WSA_FLAG_OVERLAPPED);
        if( INVALID_SOCKET  == hListenSocket ){
            cout << "WSASocketW failed:" << WSAGetLastError() << endl;
            return false;
        }

        int openReuse = 1;
        int openReuseLen = sizeof(openReuse);
        setsockopt(hListenSocket,SOL_SOCKET,SO_REUSEADDR,(char*)&openReuse,openReuseLen);
        memset(&__servAddr,0,sizeof(__servAddr));
        __servAddr.sin_addr.S_un.S_addr = INADDR_ANY;
        __servAddr.sin_port = htons(__servPort);
        __servAddr.sin_family = AF_INET;

        if(  SOCKET_ERROR == ::bind(hListenSocket,(struct sockaddr *)&__servAddr,sizeof(__servAddr)) ) {
             cout << "bind failed : " << WSAGetLastError() << endl;
             closesocket(hListenSocket);
              return false;
        }

        if( SOCKET_ERROR == ::listen(hListenSocket,SOMAXCONN)  ) {
            cout << "listen failed: " << WSAGetLastError() << endl;
            closesocket(hListenSocket);
            return false;
        }


        __listenSocket = hListenSocket;
        return true;
    }



private:
    SOCKET __listenSocket;
    unsigned short __servPort;
    SOCKADDR_IN __servAddr;
    int __iServerThreads;
    HANDLE __hIOCP;
    HANDLE * __arrIOCPThreads;
    Acceptor __acceptor;




private:
    IOCPServer(const IOCPServer &);
    IOCPServer & operator=(const IOCPServer);
};

int main(int argc, char *argv[])
{
    using namespace std;
    WSADATA wsadata;
    if( NO_ERROR !=  WSAStartup(MAKEWORD(2,2),&wsadata) ) {
        cout << "wsastartup failed" << endl;
        return 0;
    }



    IOCPServer serv;
    serv.start();

    //服务器启动完成, 可以做别的事


    while(1)
    Sleep(1000);

    WSACleanup();
    return 0;
}