完成端口IOCP与AcceptEx， NO MFC， JUST win api

参考：《windows网络编程》、度娘、谷歌、还有一篇博客和它的源码http://blog.csdn.net/piggyxp/article/details/6922277，感谢PiggyXP，不过我的代码有更好的可读性，感谢broadcom的嵌入式C++系统带给我的编程经验，不过流程基本相似，可以关注一些细节

HttpServer\ targetver.h：           Windows platform header，系统生成的

HttpServer\ stdafx.h：                 系统头文件和一些通用define

HttpServer\stdafx.cpp :               预编译 stdafx.h使用的源文件，无内容

HttpServer\Start.cpp：                          console程序入口，目前不必实现stop，因为结束console主线程，该进程的其余线程也将退出

                                                TODO：做界面的时候再实现stop和restart功能，可能需要使用PostQueuedCompletionStatus函数来通知worker线程退出

HttpServer\IOCPServer.h：

HttpServer\IOCPServer.cpp：             完成端口模型的TCP server

HttpServer\ SocketHandle.h：

HttpServer\ SocketHandle.cpp：      对应于每一个socket的handle类定义，一个socket handle可持有多个IO数据组成的list

HttpServer\ PerIOData.h:

HttpServer\ PerIOData.cpp：            单个IO数据，包含 overlapped结构，而且该结构必须在类内存空间的最开始

流程图：

源码如下，其他的不细说了，看注释吧，不过是英文的，英文语法就别深究了：

HttpServer\stdafx.h：

// stdafx.h : include file for standard system include files,
// or project specific include files that are used frequently, but
// are changed infrequently
//

#pragma once

#include "targetver.h"

#include <cstdio>
#include <cassert>
#include <tchar.h>
#include <string.h>

// TODO: reference additional headers your program requires here
// win header and lib
#include <winsock2.h>
#include <MSWSock.h>
#pragma comment(lib,"ws2_32.lib")

#include <list>
using namespace std;
#define IOCP_DBUG(format, ...) printf("%s,%d: " format, __FUNCTION__, __LINE__, ##__VA_ARGS__);

HttpServer\ Start.cpp

#include "stdafx.h"
#include "IOCPServer.h"


int _tmain(int argc, _TCHAR* argv[])
{
	IOCP_DBUG("enter Start\n");
	IOCPServer *iocpserver = new IOCPServer();
	iocpserver->Start();

	while(TRUE) 
	{
		Sleep(5000);
	}

	delete iocpserver;
	return 0;
}

HttpServer\ IOCPServer.h：

#pragma once

#include "stdafx.h"
#include "PerIOData.h"
#include "SocketHandle.h"


// this is IOCPServer which use IOCP
// has no GUI, we provide two public function to call
// two public function: start() and stop()
// you can write new interface
class IOCPServer
{
public:
	IOCPServer(void);
	virtual ~IOCPServer(void);

public:
	bool Start(void);
	bool Stop(void);

// self member hold in whole life of the class object
//so we have no need to tranfer these in function parameters
private:
	HANDLE completeport;
	SocketHandle *listenhandle;

private:
	LPFN_ACCEPTEX lpfnacceptex;
	LPFN_GETACCEPTEXSOCKADDRS lpfngetacceptexsockaddrs;

//Initialize functions, check the cpp file 
//Return value for BOOL function:
//		true, if succeed;
//		false, if failed.
//
private:
	BOOL InitializeIOCP(void);
	void DeInitializeIOCP(void);
	BOOL InitializeSocket(void);
	void DeInitializeSocket(void);
	BOOL GetAcceptExFuction(void);


private:
	// CreateThread only can use static function in class
	// so must transfer the 'this' pointer to lpParam to use non-static function member
	static DWORD WINAPI WorkerThread(LPVOID lpParam);


// the six function member below has same parameter format and return value
//
//Parameter:
//		connecthandle - the client socket and addr storage struct for the operation
//		listeniodata/connectiodata - the io data for the operation
//			this struct PerIOData MUST begin with WSAOVERLAPPED/OVERLAPPED struct,
//			To convert the PerIOData pointer to OVERLAPPED pointer.
//			actually, the first address should be the same, so they are compatible
//
//Return value:
//		true, if succeed;
//		false, if failed.
//
private:
	// post function for AcceptEx
	// no need for do function, ACCEPT_POSTED signal corresponding to listen socket
	// listeniodata is hold in whole life of master thread
	// MUST not release in worker thread
	void PostAcceptEx(PerIOData *listeniodata);
	BOOL DoAcceptEx(PerIOData *listeniodata);

private:
	// post and do function for recv/send 
	// connectiodata is hold in worker thread
	void PostRecv(SocketHandle *connecthandle, PerIOData *connectiodata);
	BOOL DoRecv(SocketHandle *connecthandle, PerIOData *connectiodata);
	void PostSend(SocketHandle *connecthandle, PerIOData *connectiodata);
	BOOL DoSend(SocketHandle *connecthandle, PerIOData *connectiodata);
};

HttpServer\ IOCPServer.cpp：

#include "stdafx.h"
#include "IOCPServer.h"


// we only post accpetex after do acceptex, if there is no mistake
// so the MAX_POST_ACCEPT is the number we post accpetex first
// and it is max client number we can accpet in the same time
// but we almost can not see difference between 10 and 100 clients, because this program is simple and fast
// In test, 100 clients seems to accept immediately  in the same time, so this number have no need to be bigger
#define MAX_POST_ACCEPT 10

//the port
#define SERVERPORT 5863

//backlog, see Stevens' book, the nginx set this to 511
//we have no need to set it to be that big
#define BACKLOG 200


//constructor
IOCPServer::IOCPServer(void):
	completeport(NULL),
	listenhandle(NULL),
	lpfnacceptex(NULL),
	lpfngetacceptexsockaddrs(NULL)
{
	IOCP_DBUG("enter IOCPServer\n");

	//load the win socket lib
	WSADATA wsaData;
	if(NO_ERROR != WSAStartup(MAKEWORD(2,2), &wsaData))
	{
		IOCP_DBUG("WSAStartup WinSock 2.2 failed！\n");
		assert(0);
	}

	//establish listen socket
	listenhandle = new SocketHandle();
}

// destructor
IOCPServer::~IOCPServer(void)
{
	IOCP_DBUG("enter ~IOCPServer\n");
	
	delete listenhandle;
}

// start the server
// we use IOCP instead of select events
// of cource, you can register other method, not implement here
bool IOCPServer::Start(void)
{
	IOCP_DBUG("enter Start\n");

	//Initialize IOCP
	if(FALSE == InitializeIOCP())
	{
		IOCP_DBUG("InitializeIOCP() failed: %d\n", GetLastError());
		DeInitializeIOCP();
		return FALSE;
	}
	IOCP_DBUG("InitializeIOCP() succeed\n");

	//Initialize Socket, associate it with IOCP
	if(FALSE == InitializeSocket())
	{
		IOCP_DBUG("InitializeSocket() failed: %d\n", GetLastError());
		DeInitializeSocket();
		return FALSE;
	}
	IOCP_DBUG("InitializeSocket() succeed\n");

	// get AcceptEx func pointer
	if(FALSE == GetAcceptExFuction())
	{
		IOCP_DBUG("GetAcceptExFuction() failed: %d\n", GetLastError());
		return FALSE;
	}
	IOCP_DBUG("GetAcceptExFuction() succeed\n");

	// new listeniodata only here
	// post some ACCEPT_POSTED signal, let the server start
	for(int i=0; i<MAX_POST_ACCEPT; ++i)
	{
		PerIOData *listeniodata = listenhandle->GetNewIOData();
		PostAcceptEx(listeniodata);
	}

	return TRUE;
}

//Initialize IOCP
BOOL IOCPServer::InitializeIOCP(void)
{
	IOCP_DBUG("enter InitializeIOCP\n");

	// establish completeport
	completeport = CreateIoCompletionPort(INVALID_HANDLE_VALUE, NULL, 0, 0 );
	if ( NULL == completeport)
	{
		IOCP_DBUG("CreateIoCompletionPort failed: %d!\n", WSAGetLastError());
		return FALSE;
	}

	// run worker thread
	SYSTEM_INFO systeminfo;
	GetSystemInfo(&systeminfo);
	for(DWORD i=0; i < systeminfo.dwNumberOfProcessors; ++i)
	{
		HANDLE threadhandle;
		threadhandle = CreateThread(NULL, 0, WorkerThread, (LPVOID)this, 0, NULL);
		CloseHandle(threadhandle);
	}

	return TRUE;
}

//DeInitialize IOCP
void IOCPServer::DeInitializeIOCP(void)
{
	IOCP_DBUG("enter DeInitializeIOCP\n");

	CloseHandle(completeport);
}

//Initialize Socket
BOOL IOCPServer::InitializeSocket(void)
{
	IOCP_DBUG("enter InitializeSocket\n");
	
	listenhandle->ssocket = WSASocket(AF_INET, SOCK_STREAM, 0, NULL, 0, WSA_FLAG_OVERLAPPED);
	if(INVALID_SOCKET == listenhandle->ssocket)
	{
		IOCP_DBUG("WSASocket failed: %d.\n", WSAGetLastError());
		return FALSE;
	}

	//associate listen socket with IOCP, let IOCP get ACCEPT_POSTED signal
	if (NULL == CreateIoCompletionPort((HANDLE)listenhandle->ssocket, completeport, (ULONG_PTR)listenhandle, 0))
	{
		IOCP_DBUG("CreateIoCompletionPort() failed: %d.\n", WSAGetLastError());
		return FALSE;
	}

	// set reuse addr option
	int reuseaddr = 1;
	if (SOCKET_ERROR == 
		setsockopt(listenhandle->ssocket, SOL_SOCKET, SO_REUSEADDR, (const char*) &reuseaddr, sizeof(int)))
	{
		IOCP_DBUG("listenhandle->ssocket: setsockopt(SO_REUSEADDR) failed %d\n", WSAGetLastError());
		closesocket(listenhandle->ssocket);
		return FALSE;
	}

	//bind and listen
	SOCKADDR_IN ServerAddress;
	ZeroMemory((char *)&ServerAddress, sizeof(ServerAddress));
	ServerAddress.sin_family = AF_INET;
	ServerAddress.sin_addr.s_addr = htonl(INADDR_ANY);                              
	ServerAddress.sin_port = htons(SERVERPORT);     

	if (SOCKET_ERROR == 
		bind(listenhandle->ssocket, (PSOCKADDR) &ServerAddress, sizeof(SOCKADDR_IN))) 
	{
		IOCP_DBUG("listensocket: bind() failed, %d\n", WSAGetLastError());
		closesocket(listenhandle->ssocket);
		return FALSE;
	}

	if (SOCKET_ERROR == listen(listenhandle->ssocket, BACKLOG)) 
	{
		IOCP_DBUG("listensocket: listen() failed %d\n", WSAGetLastError());
		closesocket(listenhandle->ssocket);
		return FALSE;
	}

	return TRUE;
}

//DeInitialize Socket
void IOCPServer::DeInitializeSocket(void)
{
	IOCP_DBUG("enter DeInitializeSocket\n");

	closesocket(listenhandle->ssocket);
	GlobalFree(listenhandle);
}

//get two function pointer
//this has a lower occupancy rate than use win lib directlly
BOOL IOCPServer::GetAcceptExFuction(void)
{
	IOCP_DBUG("enter GetAcceptExFuction\n");

	SOCKET s = listenhandle->ssocket;
	if(INVALID_SOCKET == s)
	{
		IOCP_DBUG("WSASocket failed: %d.\n", WSAGetLastError());
		return FALSE;
	}

	// get the fuction pointer for AcceptEx
	DWORD dwBytes = 0; 
	GUID GuidAcceptEx = WSAID_ACCEPTEX;
	if (SOCKET_ERROR == 
		WSAIoctl(s, SIO_GET_EXTENSION_FUNCTION_POINTER, &GuidAcceptEx, sizeof(GUID),
		&lpfnacceptex, sizeof(LPFN_ACCEPTEX), &dwBytes, NULL, NULL))
	{
		IOCP_DBUG("WSAIoctl(SIO_GET_EXTENSION_FUNCTION_POINTER, WSAID_ACCEPTEX) failed"
			": %d.\n", WSAGetLastError());
		return FALSE;
	}

	// get the fuction pointer for GetAcceptExSockAddrs
	GUID GuidAcceptExSockAddrs = WSAID_GETACCEPTEXSOCKADDRS; 
	if (SOCKET_ERROR == 
		WSAIoctl(s, SIO_GET_EXTENSION_FUNCTION_POINTER, &GuidAcceptExSockAddrs, sizeof(GUID),
		&lpfngetacceptexsockaddrs, sizeof(LPFN_GETACCEPTEXSOCKADDRS), &dwBytes, NULL, NULL))
	{
		IOCP_DBUG("WSAIoctl(SIO_GET_EXTENSION_FUNCTION_POINTER, WSAID_ACCEPTEX) failed"
			": %d.\n", WSAGetLastError());
		return FALSE;
	}
	return TRUE;
}

//worker thread, usually not one,
//they are busy until exit.
//because these thread is asynchronous,
//so one socket can have more than one io.
DWORD WINAPI IOCPServer::WorkerThread(LPVOID lpParam)
{    
	IOCP_DBUG("enter WorkerThread\n");

	IOCPServer* temp = (IOCPServer*)lpParam;
	LPWSAOVERLAPPED lpoverlapped = NULL;
	SocketHandle *sockethandle = NULL;
	PerIOData *periodata = NULL;
	DWORD bytestransfered = 0;

	while (TRUE)
	{
		// get the status of comleteport
		BOOL ret = 
			GetQueuedCompletionStatus(temp->completeport, &bytestransfered,
			(PULONG_PTR)&sockethandle, &lpoverlapped, INFINITE);

		if(FALSE == ret)  
		{  
			int sockerror = WSAGetLastError();
			IOCP_DBUG("GetQueuedCompletionStatus failed: %d.\n", sockerror);
			if(64 == sockerror) //host is down, client close abnormally
			{
				if(temp->listenhandle != sockethandle)
				{
					IOCP_DBUG("client %s:%d is DOWN abnormally, we need delete the handle\n", 
						inet_ntoa(sockethandle->ssocketaddr.sin_addr), 
						ntohs(sockethandle->ssocketaddr.sin_port));

					// the destructor of sockethandle do recycle
					delete sockethandle;
				}else
				{
					IOCP_DBUG("something wrong in listenhandle, you need restart the process");
				}
			}
			continue;  
		}
		
		// read the data in completeport
		periodata = CONTAINING_RECORD(lpoverlapped, PerIOData, overlapped);
		if(NULL == periodata)  
		{  
			IOCP_DBUG("CONTAINING_RECORD failed: %d.\n", WSAGetLastError());
			continue;  
		}

		// free io and handle, if client close the socket
		// all normal handle and io free here, if there is no mistake
		if((0 == bytestransfered) &&
			( PerIOData::RECV_POSTED == periodata->operationtype
			|| PerIOData::SEND_POSTED == periodata->operationtype))  
		{  
			IOCP_DBUG("client %s:%d has close the socket, we need close the connection\n",
				inet_ntoa(sockethandle->ssocketaddr.sin_addr), 
				ntohs(sockethandle->ssocketaddr.sin_port));

			// the destructor of sockethandle do recycle
			delete sockethandle;
			continue;  
		}
		
		switch(periodata->operationtype)  
		{  
			// AcceptEx  
			case PerIOData::ACCEPT_POSTED:
			{
				IOCP_DBUG("recv the ACCEPT_POSTED signal\n");

				if(FALSE == temp->DoAcceptEx(periodata))
				{
					// here sockethandle is listenhandle
					// periodata is listeniodata
					// MUST NOT release them here, let master thread do the bad thing
					// JUST re post
					temp->PostAcceptEx(periodata);
				}
			}
			break;

			// RECV
			case PerIOData::RECV_POSTED:
			{
				IOCP_DBUG("recv the RECV_POSTED signal\n");

				// re post, if failed
				// MUST not close socket here
				// because one socket may have more io in use
				if(FALSE == temp->DoRecv(sockethandle, periodata))
				{
					//MUST not force to release the periodata which hold overlapped struct
					//re post
					temp->PostRecv(sockethandle, periodata);
				}
			}
			break;

			// SEND
			case PerIOData::SEND_POSTED:
			{
				IOCP_DBUG("recv the SEND_POSTED signal\n");

				// re post, if failed
				// no need to construct new io, send use the same one with recv
				if(FALSE == temp->DoSend(sockethandle, periodata))
				{
					// MUST not close socket here
					// because one socket may have more io in use
					//MUST not force to release the periodata which hold overlapped struct
					temp->PostSend(sockethandle, periodata);
				}
			}
			break;

			case PerIOData::NULL_POSTED:
			default:
				IOCP_DBUG("MUST exist a problem, the last error: %d.\n", WSAGetLastError());
			break;
		} //switch
	}//while
	return 0;
}

// post AcceptEx
void IOCPServer::PostAcceptEx(PerIOData* listeniodata)
{
	IOCP_DBUG("enter PostAcceptEx\n");

	if(NULL == listeniodata)
	{
		IOCP_DBUG("this is a fatal mistake, almost no way to reach here\n");
		return;
	}

	// need new socket before AcceptEx
	listeniodata->acceptsocket = WSASocket(AF_INET, SOCK_STREAM, 0, NULL, 0, WSA_FLAG_OVERLAPPED);
	if(INVALID_SOCKET == listeniodata->acceptsocket)
	{
		IOCP_DBUG("WSASocket failed: %d.\n", WSAGetLastError());
		return;
	}

	// set the connect delay for 3 sec
	// because AcceptEx will not return, if client just connect but send no bytes
	// win7 do not support SO_CONNECT_TIME to prevent DDOS
	// instead, dwReceiveDataLength should be ((sizeof(SOCKADDR_IN)+16)*2)
	// only get the local and remote addr
	/*
	INT timeout = 3;
	if (SOCKET_ERROR == 
		setsockopt(listeniodata->acceptsocket, SOL_SOCKET, SO_CONNECT_TIME,
			(const char*)&timeout, sizeof(timeout)))
	{
		IOCP_DBUG("setsockopt(SO_CONNECT_TIME) failed: %d.\n", WSAGetLastError());
		return;
	}
	*/

	// do some clean
	// make operationtype signal be ACCEPT_POSTED
	listeniodata->ResetIO();
	listeniodata->operationtype = PerIOData::ACCEPT_POSTED;  
	DWORD dwBytes = 0; 
	if (FALSE == 
		lpfnacceptex(listenhandle->ssocket, listeniodata->acceptsocket, listeniodata->databuf.buf, 
		0, sizeof(SOCKADDR_IN)+16, sizeof(SOCKADDR_IN)+16,
		&dwBytes, &listeniodata->overlapped))
	{
		if (WSA_IO_PENDING != WSAGetLastError()) 
		{
			IOCP_DBUG("AcceptEx() falied: %d.\n", WSAGetLastError());
			closesocket(listeniodata->acceptsocket);
			listeniodata->acceptsocket = -1;
			return;
		}
	}

	return;
}

BOOL IOCPServer::DoAcceptEx(PerIOData* listeniodata)
{
	/* SO_UPDATE_ACCEPT_CONTEXT is required for shutdown() to work */
	// win7 do not support SO_UPDATE_ACCEPT_CONTEXT
	/*
	if (SOCKET_ERROR == 
		setsockopt(listeniodata->acceptsocket, SOL_SOCKET, 
			SO_UPDATE_ACCEPT_CONTEXT, (char *)&listenhandle->ssocket, sizeof(SOCKET)))
	{
		IOCP_DBUG("setsockopt(SO_UPDATE_ACCEPT_CONTEXT) failed: %d.\n", WSAGetLastError());
		return FALSE;
	}
	*/

	//get the addr for client which connect me
	//for AcceptEx, Should use GetAcceptExSockAddrs
	SOCKADDR_IN *lplocalsockaddr = NULL;
	SOCKADDR_IN *lpremotesockaddr = NULL;
	int localsockaddrlen = sizeof(SOCKADDR_IN);
	int remotesockaddrlen = sizeof(SOCKADDR_IN);

	lpfngetacceptexsockaddrs(listeniodata->databuf.buf, 0,
		sizeof(SOCKADDR_IN)+16,	sizeof(SOCKADDR_IN)+16,
		(LPSOCKADDR*)&lplocalsockaddr, &localsockaddrlen,
		(LPSOCKADDR*)&lpremotesockaddr, &remotesockaddrlen);

	SocketHandle *connecthandle = new SocketHandle();
	connecthandle->ssocket = listeniodata->acceptsocket;
	memcpy(&connecthandle->ssocketaddr, lpremotesockaddr, sizeof(SOCKADDR_IN));
	if (NULL == CreateIoCompletionPort((HANDLE)connecthandle->ssocket, completeport, (ULONG_PTR)connecthandle, 0))
	{
		IOCP_DBUG("CreateIoCompletionPort() failed: %d.\n", WSAGetLastError());
		return FALSE;
	}

	// usually we need to recv something, so post recv 
	// we new connecthandle and connectiodata here first
	
	PerIOData *connectiodata = connecthandle->GetNewIOData();
	PostRecv(connecthandle, connectiodata);

	//post new one
	PerIOData *newlisteniodata = listenhandle->GetNewIOData();
	PostAcceptEx(newlisteniodata);

	return TRUE;
}

// post recv
void IOCPServer::PostRecv(SocketHandle* connecthandle, PerIOData* connectiodata)
{
	IOCP_DBUG("enter PostRecv\n");

	if(NULL == connecthandle || NULL == connectiodata)
	{
		IOCP_DBUG("this is a fatal mistake, almost no way to reach here\n");
		return;
	}

	// do some clean
	// make operationtype signal be RECV_POSTED
	DWORD flags = 0;
	DWORD recvbytes = 0;
	connectiodata->ResetIO();
	connectiodata->operationtype = PerIOData::RECV_POSTED;

	//recv the message
	int nbytes = WSARecv(connecthandle->ssocket, &connectiodata->databuf, 1, 
		&recvbytes, &flags, &connectiodata->overlapped, NULL );
	if ((SOCKET_ERROR == nbytes) && (WSA_IO_PENDING != WSAGetLastError()))
	{
		IOCP_DBUG("WSARecv() failed: %d.\n", WSAGetLastError());
		return;
	}

	return;
}

// do after recv
BOOL IOCPServer::DoRecv(SocketHandle* connecthandle, PerIOData* connectiodata)
{
	IOCP_DBUG("enter DoRecv\n");

	if(NULL == connecthandle || NULL == connectiodata)
	{
		IOCP_DBUG("this is a fatal mistake, almost no way to reach here\n");
		return FALSE;
	}

	IOCP_DBUG("receive from %s:%d, message: %s.\n", 
		inet_ntoa(connecthandle->ssocketaddr.sin_addr), 
		ntohs(connecthandle->ssocketaddr.sin_port),connectiodata->databuf.buf);

	//TODO
	//process the massage
	
	// usually we need send after recv
	PostSend(connecthandle, connectiodata);

	// handle recv signal complete, post new one
	//MUST not force to release the periodata which hold overlapped struct
	//new one periodata
	PerIOData *newconnectiodata = connecthandle->GetNewIOData();
	PostRecv(connecthandle, newconnectiodata);

	return TRUE;
}

//post send
void IOCPServer::PostSend(SocketHandle* connecthandle, PerIOData* connectiodata)
{
	IOCP_DBUG("enter PostSend\n");

	if(NULL == connecthandle || NULL == connectiodata)
	{
		IOCP_DBUG("this is a fatal mistake, almost no way to reach here\n");
		return;
	}

	// do some clean
	// make operationtype signal be SEND_POSTED
	DWORD flags = 0;
	DWORD sendbytes = 0;
	connectiodata->ResetIO();
	connectiodata->operationtype = PerIOData::SEND_POSTED;

	//TODO
	//build the message
	char *temp = "just for test";
	strncpy_s(connectiodata->databuf.buf, connectiodata->databuf.len, temp, strlen(temp));
	*(connectiodata->databuf.buf + strlen(temp)) = 0;

	//send the message
	int nbytes = WSASend(connecthandle->ssocket, &connectiodata->databuf, 1,
		&sendbytes, flags, &connectiodata->overlapped, NULL );
	if ((SOCKET_ERROR == nbytes) && (WSA_IO_PENDING != WSAGetLastError()))
	{
		IOCP_DBUG("WSASend() failed: %d.\n", WSAGetLastError());
		return;
	}

	return;
}

//DO after send
BOOL IOCPServer::DoSend(SocketHandle* connecthandle, PerIOData* connectiodata)
{
	IOCP_DBUG("enter DoSend\n");

	if(NULL == connecthandle || NULL == connectiodata)
	{
		IOCP_DBUG("this is a fatal mistake, almost no way to reach here\n");
		return FALSE;
	}

	IOCP_DBUG("send to %s:%d, message: %s.\n", 
		inet_ntoa(connecthandle->ssocketaddr.sin_addr), 
		ntohs(connecthandle->ssocketaddr.sin_port),connectiodata->databuf.buf );

        //just do it in the end of one io here
        //this is dangerous in somewhere else
        connecthandle->RemoveIOData(connectiodata);
	return TRUE;
}

HttpServer\ SocketHandle.h：

#pragma once

#include "stdafx.h"
#include "PerIOData.h"


class SocketHandle
{
public:
	SocketHandle(void);
	virtual ~SocketHandle(void);

public:
	//the socket and its addr the handle hold
	SOCKET ssocket;
	SOCKADDR_IN ssocketaddr;

	//the io list belong to me
	list<PerIOData*> sockiodata;

public:
	PerIOData* GetNewIOData(void);
	
	// check the cpp source file for the reason
	void RemoveIOData(PerIOData* speriodata);
};

HttpServer\ SocketHandle.cpp：

#include "stdafx.h"
#include "SocketHandle.h"

SocketHandle::SocketHandle(void)
{
	ssocket = INVALID_SOCKET;
	ZeroMemory(&ssocketaddr, sizeof(ssocketaddr)); 
}

SocketHandle::~SocketHandle(void)
{
	//close the socket
	if( ssocket!=INVALID_SOCKET )
	{
		closesocket(ssocket);
		ssocket = INVALID_SOCKET;
	}

	// release the list
	list <PerIOData*>::iterator myIterator = sockiodata.begin();
	for ( ; myIterator != sockiodata.end(); myIterator++)
	{
		delete *myIterator;
		*myIterator = NULL;
	}
	sockiodata.clear();
}

// new iodata and add it in list
//return new io which will be used
PerIOData* SocketHandle::GetNewIOData(void)
{
	PerIOData* speriodata = new PerIOData();
	sockiodata.push_back(speriodata);
	return speriodata;
}

//delete the io transfered and erase it form list
//MUST not force to release the periodata which hold overlapped struct
//let the deconstruct func do the recycle
//we need it just in the end of one io's life.
//just for case, some ugly client keep sending message to me with no interval

void SocketHandle::RemoveIOData(PerIOData* speriodata)
{
	if(NULL == speriodata)
	{
		IOCP_DBUG("you just transfer a NULL pointer\n");
		return;
	}

	list <PerIOData*>::iterator myIterator = sockiodata.begin();
	for ( ; myIterator != sockiodata.end(); myIterator++)
	{
		if(*myIterator == speriodata)
		{
			delete *myIterator;
			*myIterator = NULL;
			sockiodata.erase(myIterator);				
			break;
		}
	}
}

HttpServer\ PerIOData.h:

#pragma once

#include "stdafx.h"

// per io data for socket handle
// one socket can has more than one io
//WSAOVERLAPPED struct must be the first in memory of this class
// so this class and its base class MUST not have virtual func
class PerIOData
{
public:
	PerIOData(void);
	~PerIOData(void);

public:
	// overlapped struct, must be in the first place of class memory
	WSAOVERLAPPED overlapped;

	//just for ACCEPT_POSTED signal
	SOCKET acceptsocket;

	//the buffer of me
	WSABUF databuf;

	//the signal belong to me
	DWORD operationtype;

public:
	void ResetIO(void);

// this four signal is the public signal for other class, IOCPServer etc..
//		ACCEPT_POSTED - signal for accept
//		RECV_POSTED   - signal for recv
//		SEND_POSTED   - signal for send
//		NULL_POSTED   - terminal signal, no meaning
//
public:
	typedef enum  
	{  
		ACCEPT_POSTED,
		RECV_POSTED,
		SEND_POSTED,
		NULL_POSTED
	}OPERATION_TYPE;
};

HttpServer\ PerIOData.cpp：

#include "stdafx.h"
#include "PerIOData.h"

// default buffer size
#define BUFFERSIZE 1024

//new buffer and Initialize
PerIOData::PerIOData(void):
	operationtype(NULL_POSTED),
	acceptsocket(INVALID_SOCKET)
{
	ZeroMemory(&overlapped, sizeof(overlapped));  
	databuf.buf = new char[BUFFERSIZE];
	databuf.len = BUFFERSIZE;
	ZeroMemory(databuf.buf, databuf.len);
}

//delete the buffer
PerIOData::~PerIOData(void)
{
	if(NULL != databuf.buf)
	{
		delete [] databuf.buf;
		databuf.buf = NULL;
	}
}

//reset buffer and overlapped of io
void PerIOData::ResetIO(void)
{
	databuf.len = BUFFERSIZE;
	ZeroMemory(databuf.buf, databuf.len);
	ZeroMemory(&overlapped, sizeof(WSAOVERLAPPED));
}