完整的IOCP模型 Echo服务器及代码分析

首先，先感谢http://www.cnblogs.com/talenth/p/7068392.html 这篇博文，作者写的通俗易懂，语言幽默，偶然一次在公交车上见到这篇博文相见恨晚，一口气读下来很长一篇，有了整体的认知，又翻看代码，查看其它资料，反复研究每个细节，终于IOCP模型基本懂了，下面给出一些心得。建议先看一遍上面提及的博客，再看这篇文章，应该会很快就能理解。

IOCP模型也称完成端口，有人说称为完成队列可能更合适，我也觉得其实。因为IOCP是在IO完成后才告诉程序，并且操作是异步执行的，也就是CPU的一个核心执行到IO操作时不用停下来等待IO完成，再执行其他的操作，所以这会快很多。

流程：
首先程序需要创建唯一的一个完成端口句柄，用于将SOCKET绑定到它上面，然后这个SOCKET上投递的IO操作完成后，这个完成端口句柄就可以收到并进行处理。
投递请求时需要用到一个OVERLAPPED结构，此结构每个SOCKET的每个IO操作都要有唯一的一个，不要复用。不然就只能响应一个操作。
下面是给出详细的代码分析过程，代码部分来自上面的那个博客，修正了里面一些设计，并添加了部分操作。
首先头文件直接给出：

#pragma once
#include<WinSock2.h>
#include <MSWSock.h>//AcceptEx 和 GetAcceptExSockaddrs 的GUID，用于导出函数指针
#include<Windows.h>
#pragma comment(lib,"ws2_32.lib")
#include<iostream>
#include<list>
#include<vector>
#include<map>
#include<string>
#include<assert.h>
#include<algorithm>
using namespace std;
#define MAX_BUFFER_LEN        8192  
// 释放指针宏
#define RELEASE_PTR(x)                      {if(x ){delete x;x=NULL;}}
// 释放句柄宏
#define RELEASE_HANDLE(x)               {if(x != NULL && x!=INVALID_HANDLE_VALUE){ CloseHandle(x);x = NULL;}}
// 释放Socket宏
#define RELEASE_SOCKET(x)               {if(x) { closesocket(x);x=INVALID_SOCKET;}}

/* Mingw doesn't have these in its mswsock.h.  The values are copied from
wine.h.   Perhaps if we copy them exactly, the cargo will come again.
*/
#ifndef WSAID_ACCEPTEX
#define WSAID_ACCEPTEX \
{0xb5367df1, 0xcbac, 0x11cf, { 0x95, 0xca, 0x00, 0x80, 0x5f, 0x48, 0xa1, 0x92 }}
#endif
#ifndef WSAID_CONNECTEX
#define WSAID_CONNECTEX \
{0x25a207b9, 0xddf3, 0x4660, { 0x8e, 0xe9, 0x76, 0xe5, 0x8c, 0x74, 0x06, 0x3e }}
#endif
#ifndef WSAID_GETACCEPTEXSOCKADDRS
#define WSAID_GETACCEPTEXSOCKADDRS \
{0xb5367df2, 0xcbac, 0x11cf, { 0x95, 0xca, 0x00, 0x80, 0x5f, 0x48, 0xa1, 0x92 }}
#endif

#define IP "127.0.0.1"
#define PORT 8000
//
// 在完成端口上投递的I/O操作的类型
typedef enum _OPERATION_TYPE
{
	ACCEPT_POSTED = 0,                     // 标志投递的Accept操作
	SEND_POSTED,                       // 标志投递的是发送操作
	RECV_POSTED,                       // 标志投递的是接收操作
	NULL_POSTED                        // 用于初始化，无意义
}OPERATION_TYPE;

typedef struct _PER_IO_CONTEXT
{
// 每一个重叠网络操作的重叠结构(针对每一个Socket的每一个操作，都要有一个) 
// 因此下标干脆用上面的OPERATION_TYPE类型来确定
	OVERLAPPED     m_Overlapped;              
	SOCKET         m_socket;                               // 这个网络操作所使用的Socket
	WSABUF         m_wsaBuf;                                   // WSA类型的缓冲区，用于给重叠操作传参数的
	char           m_szBuffer[MAX_BUFFER_LEN];                 // 这个是WSABUF里具体存字符的缓冲区
	OPERATION_TYPE m_OpType;                                   // 标识网络操作的类型(对应上面的枚举)

	// 初始化
	_PER_IO_CONTEXT()
	{
		for(int i=0;i<3;i++)
			ZeroMemory(&m_Overlapped, sizeof(OVERLAPPED));
		ZeroMemory(m_szBuffer, MAX_BUFFER_LEN);
		m_socket = INVALID_SOCKET;
		m_wsaBuf.buf = m_szBuffer;
		m_wsaBuf.len = MAX_BUFFER_LEN;
		m_OpType = NULL_POSTED;
	}
	// 释放掉Socket
	~_PER_IO_CONTEXT()
	{
		if (m_socket != INVALID_SOCKET)
		{
			::closesocket(m_socket);
			m_socket = INVALID_SOCKET;
		}
	}
	// 重置缓冲区内容
	void ResetBuffer()
	{
		ZeroMemory(m_szBuffer, MAX_BUFFER_LEN);
	}
	/*_PER_IO_CONTEXT* GetNewIoContext()
	{
		_PER_IO_CONTEXT* p = new _PER_IO_CONTEXT;
		EnterCriticalSection(&cs);
		all_io.push_back(p);
		LeaveCriticalSection(&cs);
		return p;
	}*/
} PER_IO_CONTEXT, *PPER_IO_CONTEXT;

typedef struct _PER_SOCKET_CONTEXT
{
	~_PER_SOCKET_CONTEXT() {
		for (auto &p : m_arrayIoContext)
		{
			if (p)
				delete p;
		}
	}
	_PER_IO_CONTEXT* GetNewIoContext()
	{
		_PER_IO_CONTEXT* p = new _PER_IO_CONTEXT;

		m_arrayIoContext.push_back(p);

		return p;
	}
	void RemoveIoContext(PER_IO_CONTEXT *pIoContext,bool bDelete=true)
	{
		for (auto it = m_arrayIoContext.begin(); it != m_arrayIoContext.end(); it++)
		{
			if (pIoContext && *it == pIoContext)
			{
				if (bDelete)
				{
					delete pIoContext;
					pIoContext = NULL;
				}
				m_arrayIoContext.erase(it);
				break;
			}
		}
	}

	SOCKET      m_socket;                                  // 每一个客户端连接的Socket
	SOCKADDR_IN m_ClientAddr;                              // 客户端的地址
	vector<PER_IO_CONTEXT*> m_arrayIoContext;             // 客户端网络操作的上下文数据，

}PER_SOCKET_CONTEXT;

class CIOCPModel
{
public:
	CIOCPModel();
	~CIOCPModel();
	
	bool Init();
	void ServerLoop();
	void Stop();
	void OutPutErrorMsg(int, string s = "");
protected:
	bool InitIocp();//初始化iocp，创建work线程
	bool InitListenSocket(); 
	bool GetExtensionFunctions();
	bool PrePostAccept();
//	PER_SOCKET_CONTEXT * GetNewIoContext();
	bool PostAccept(PER_IO_CONTEXT *pAcceptIoContext);
	bool DoAccept(PER_IO_CONTEXT *pCurIoContext);
	bool PostRecv(PER_IO_CONTEXT *pCurIoContext);
	
	void RemoveContext(_PER_SOCKET_CONTEXT *&);
	static DWORD WINAPI WorkerThread(LPVOID);

	void Lock(){ EnterCriticalSection(&m_lock); }
	void UnLock(){ LeaveCriticalSection(&m_lock); }
	bool HandleError(PER_SOCKET_CONTEXT *pContext, const DWORD& dwErr);
	bool IsSocketAlive(SOCKET);

	void Echo(PER_SOCKET_CONTEXT *,PER_IO_CONTEXT *);
private:
	HANDLE m_hIocpHandle;// 完成端口句柄
	LPFN_ACCEPTEX m_lpfnAcceptEx; //AcceptEx函数指针
	LPFN_GETACCEPTEXSOCKADDRS m_lpfnGetAcceptSockAddr; //GetAcceptSockAddr函数指针
	_PER_SOCKET_CONTEXT *m_pListenContext; //监听IO
	HANDLE *m_pWorkerThreadHandle; //工作线程句柄
	int m_nThreadNum; //线程个数 
	CRITICAL_SECTION m_lock;//对 m_allIoContext 访问加锁
	list<_PER_SOCKET_CONTEXT *>m_allIoContext; //所有已经连接的socket

};

详细说一下上面的两个结构体，PER_SOCKET_CONTEXT与_PER_IO_CONTEXT，由于每个SOCKET上可能有不同的IO操作，所以IO操作单独定义一个结构体，用于管理IO操作，后面可以体会这个结构体的便捷处。里面有个m_szBuffer，但初始化的时候又m_wsaBuf.buf = m_szBuffer; 因为m_wsaBuf.buf 是一个指针，我估计这样设计是为了不用在堆上分配内存。

初始化IOCP，就是建立一个空的完成端口，然后创建双倍CPU核心的工作线程，如果看过上面的博客，应该知道为什么这么设计了
 

bool CIOCPModel::InitIocp()
{
	m_hIocpHandle = CreateIoCompletionPort(INVALID_HANDLE_VALUE, 0, NULL, 0);
	if (!m_hIocpHandle)
	{
		OutPutErrorMsg(__LINE__);
		WSACleanup();
		return false;
	}
	//获取cpu核心
	SYSTEM_INFO si;
	GetSystemInfo(&si);
	m_nThreadNum = 2 * si.dwNumberOfProcessors;
	m_pWorkerThreadHandle = new HANDLE[m_nThreadNum];
	DWORD nThreadID;
	for (int i = 0; i < m_nThreadNum; i++)
	{
		m_pWorkerThreadHandle[i] = ::CreateThread(0, 0, WorkerThread, this, 0, &nThreadID);
	}
	cout << "InitIOCP end" << endl;
	return true;
}

初始化监听的SOCKET_CONTEXT

bool CIOCPModel::InitListenSocket()
{
	WSAData wsa;
	WSAStartup(MAKEWORD(2, 2), &wsa);
	SOCKADDR_IN server_addr;
	int addrLen = sizeof(SOCKADDR_IN);
	ZeroMemory(&server_addr, 0, addrLen);
	server_addr.sin_family = AF_INET;
	server_addr.sin_addr.S_un.S_addr = inet_addr(IP);
	server_addr.sin_port = htons(PORT);
	m_pListenContext = new _PER_SOCKET_CONTEXT;
	m_pListenContext->m_socket = WSASocket(AF_INET, SOCK_STREAM, 0, NULL, 0, WSA_FLAG_OVERLAPPED);
	if (m_pListenContext->m_socket == INVALID_SOCKET)
	{
		WSACleanup();
		return false;
	}
	if (!CreateIoCompletionPort((HANDLE)m_pListenContext->m_socket, m_hIocpHandle, (DWORD)m_pListenContext, 0))
	{
		OutPutErrorMsg(__LINE__);
		WSACleanup();
		return false;
	}
	cout << "Listen Socket绑定完成端口 完成" << endl;
	if (bind(m_pListenContext->m_socket, (sockaddr*)&server_addr, addrLen) == SOCKET_ERROR)
	{
		OutPutErrorMsg(__LINE__);
		WSACleanup();
		return false;
	}
	if (listen(m_pListenContext->m_socket, SOMAXCONN) == SOCKET_ERROR)
	{
		OutPutErrorMsg(__LINE__);
		WSACleanup();
		return false;
	}
	cout << "Init ListenSocket end" << endl;
	return true;
}

此处将监听socket绑定到了完成端口中，用于等待客户端连接，因为有客户端连接的话这个socket会有一个完成包。

一开始先投递10个AcceptEx，这个可以自己设定

bool CIOCPModel::PrePostAccept()
{
	for (int i = 0; i<10; i++)
	{
		//对于ACCEPT_POSTED的投递在m_pListenContext 上
		PER_IO_CONTEXT* pAcceptIoContext = m_pListenContext->GetNewIoContext();

		if (false == PostAccept(pAcceptIoContext))
		{
			m_pListenContext->RemoveIoContext(pAcceptIoContext);
			return false;
		}
	}
    return true;
}
bool CIOCPModel::PostAccept(PER_IO_CONTEXT* pAcceptIoContext)//pAcceptIoContext是加在监听socket的IO列表中的
{
	assert(INVALID_SOCKET != m_pListenContext->m_socket);
	// 准备参数
	DWORD dwBytes = 0;
	pAcceptIoContext->m_OpType = ACCEPT_POSTED;
	WSABUF *p_wbuf = &pAcceptIoContext->m_wsaBuf;
	OVERLAPPED *p_ol = &pAcceptIoContext->m_Overlapped;

	// 为以后新连入的客户端先准备好Socket
	pAcceptIoContext->m_socket = WSASocket(AF_INET, SOCK_STREAM, IPPROTO_TCP, NULL, 0, WSA_FLAG_OVERLAPPED);

	if (INVALID_SOCKET == pAcceptIoContext->m_socket)
	{
		OutPutErrorMsg(__LINE__);
		return false;
		//END("创建用于Accept的Socket失败");
	}
	// 投递AcceptEx
	if (FALSE == m_lpfnAcceptEx(m_pListenContext->m_socket, pAcceptIoContext->m_socket, p_wbuf->buf, p_wbuf->len - ((sizeof(SOCKADDR_IN)+16) * 2),
		sizeof(SOCKADDR_IN)+16, sizeof(SOCKADDR_IN)+16, &dwBytes, p_ol))
	{
		if (WSA_IO_PENDING != WSAGetLastError())
		{
			OutPutErrorMsg(__LINE__, "投递 AcceptEx 请求失败");
			return false;
			//END("投递 AcceptEx 请求失败");
		}
	}
	return true;
}

注意，此处的IO_CONTEXT都是根据m_pListenContext来获取的，也就是说都加在了m_pListenContext的那个IO列表中。

再看下工作线程是怎么处理的
 

DWORD WINAPI CIOCPModel::WorkerThread(LPVOID param)
{
	CIOCPModel * pModel = (CIOCPModel *)param;
	OVERLAPPED * pOverLapped;
	DWORD  dwBytesTransfered = 0;
	PER_SOCKET_CONTEXT * pIoSocketContext = NULL;

	while (1)
	{
		BOOL bReturn = GetQueuedCompletionStatus(
			pModel->m_hIocpHandle,
			&dwBytesTransfered,
			(PULONG_PTR)&pIoSocketContext,//调用CreateIoCompletionPort把socket绑定到iocp时传入的第三个参数是什么, GetQueuedCompletionStatus第三个参数就会返回什么
			&pOverLapped,
			INFINITE);
		/*cout << pModel->m_pListenContext << endl;
		cout << pIoContext << endl;
		cout << pOverLapped << endl;
		cout << pModel->m_allIoContext.back() << endl;*/
		if (NULL == (DWORD)pIoSocketContext)
		{
			break;
		}
		if (!bReturn)
		{
			DWORD dwErr = GetLastError();

			// 显示一下提示信息
			if (!pModel->HandleError(pIoSocketContext, dwErr))
			{
				break;
			}

			continue;
		}
		else
		{
			// 读取传入的参数
			PER_IO_CONTEXT * pIoContext = CONTAINING_RECORD(pOverLapped, PER_IO_CONTEXT, m_Overlapped);
			// 判断是否有客户端断开了
			if ((0 == dwBytesTransfered) && (RECV_POSTED == pIoContext->m_OpType || SEND_POSTED == pIoContext->m_OpType))
			{
				pModel->RemoveContext(pIoSocketContext);
				pModel->Lock();
				printf("客户端断开连接,连接数：%d\n", pModel->m_allIoContext.size() - 10);
				pModel->UnLock();
				continue;
			}
			else
			{
				
				DWORD dwSendBytes, Flags;
				int nBytesSend;
				switch (pIoContext->m_OpType)
				{
				//TODO 第一组数据的处理
				case ACCEPT_POSTED:
					pModel->DoAccept(pIoContext);
					break;
				case RECV_POSTED:
					cout << "收到" << pIoContext->m_szBuffer << endl;
					pModel->Echo(pIoSocketContext,pIoContext);//发送给源客户端,由于下面就要清空缓冲区，所以先发送了。。

					pIoContext->ResetBuffer();//释放发送缓冲区
					pModel->PostRecv(pIoContext);

					
				/*	pIoContext->m_OpType = SEND_POSTED;
					nBytesSend = WSASend(pIoContext->m_socket, &pIoContext->m_wsaSendBuf, 1, &dwSendBytes, 0,
						&pIoContext->m_Overlapped[SEND_POSTED], NULL);
					if ((nBytesSend < 0) && (WSA_IO_PENDING != WSAGetLastError()))
					{
						pModel->OutPutErrorMsg(__LINE__);
						return 0;
					}	*/
					
					break;
				case SEND_POSTED:
					cout << "发送" << pIoContext->m_szBuffer << endl;
					pIoContext->ResetBuffer();
					break;
				default:
					break;
				}
			}
		}
	}
	return 0;
}

注：每次投递操作时都要提供一个OVERLAPPED结构，而像发送接收消息这种涉及缓冲区的，还需要提供一个WASBUF结构，缓冲区信息就存在这里面，而OVERLAPPED这个就像身份证一样，它完成了才会有其他东西。所以定义一个结构体来管理会方便很多，而且可以通过CONTAINING_RECORD宏，参见我的博客https://mp.csdn.net/postedit/84973937 可以取出整个结构体，这样简直太方便了，得到了所有想要的东西。

下面详细看一下接收连接
 

bool CIOCPModel::DoAccept(PER_IO_CONTEXT * pAcceptIoContext)
{
	SOCKADDR_IN* ClientAddr = NULL;
	SOCKADDR_IN* LocalAddr = NULL;
	int remoteLen = sizeof(SOCKADDR_IN), localLen = sizeof(SOCKADDR_IN);

	// GetAcceptSockAddr可以取得客户端和本地端的地址信息以及客户端发来的第一组数据
	m_lpfnGetAcceptSockAddr(pAcceptIoContext->m_wsaBuf.buf, pAcceptIoContext->m_wsaBuf.len - ((sizeof(SOCKADDR_IN)+16) * 2),
		sizeof(SOCKADDR_IN)+16, sizeof(SOCKADDR_IN)+16, (LPSOCKADDR*)&LocalAddr, &localLen, (LPSOCKADDR*)&ClientAddr, &remoteLen);
	cout << "第一组消息:" << pAcceptIoContext->m_wsaBuf.buf << endl;
	
	PER_SOCKET_CONTEXT* pNewSocketContext = new PER_SOCKET_CONTEXT;
	m_allIoContext.push_back(pNewSocketContext);
	pNewSocketContext->m_socket = pAcceptIoContext->m_socket;
//	memcpy(&(pNewSocketContext->m_ClientAddr), ClientAddr, sizeof(SOCKADDR_IN));
	接收连接后关联到完成端口
	if (CreateIoCompletionPort((HANDLE)pNewSocketContext->m_socket, m_hIocpHandle, (ULONG_PTR)pNewSocketContext, 0) == NULL)
	{
		OutPutErrorMsg(__LINE__);
		return false;
	}
	//此处新建一个socket后要把客户端的SOCKET拷贝过去
	PER_IO_CONTEXT* pNewIoContext = pNewSocketContext->GetNewIoContext();
	pNewIoContext->m_OpType = RECV_POSTED;
	pNewIoContext->m_socket = pNewSocketContext->m_socket;

	//原本设想：新的SOCKET_IO创建完后m_pListenContext里的那个用于连接的PER_IO_CONTEXT可以删掉了，因为它投递的AcceptEx请求
	//已经连接了，已经没用了。不然它会一直增大,注意，此处仅仅从m_pListenContext的那个vector中移除，不delete
	//但是仅移除的话，那这个IO_CONTEXT就内存泄露了。
	//m_pListenContext->RemoveIoContext(pAcceptIoContext,false);


	// 绑定完毕之后，就可以开始在这个新的客户端Socket上投递接收信息完成请求了
	if (false == PostRecv(pNewIoContext))
	{
		RemoveContext(pNewSocketContext);
		return false;
	}
	//有一个新连接后要投递新的请求
	PER_IO_CONTEXT * pNewAcceptIoContext = m_pListenContext->GetNewIoContext();
	if (false == PostAccept(pNewAcceptIoContext))
	{
		//因为GetNewIoContext会把pNewIoContent加入到IO列表中，所以要从IO列表中删除
		m_pListenContext->RemoveIoContext(pNewAcceptIoContext);
		return false;
	}
	return true;
}

此处有一个参数pAcceptIoContext，看下WorkThread，传递的这个参数是那个在投递AcceptEx时创建的，接收连接后再创建一个SOCKET_IO_CONTEXT，并在它上面投递接收请求。最后投递一个新的ACCEPT请求，保证下次有连接时已经创建好了。

最后附上整体的cpp文件，而main函数中只需要新建个对象，初始化，开始循环即可。。。
CIOCPModel obj;
obj.Init();
obj.ServerLoop();

#include "IOCPModel.h"


CIOCPModel::CIOCPModel()
{
	InitializeCriticalSection(&m_lock);
}


CIOCPModel::~CIOCPModel()
{
	DeleteCriticalSection(&m_lock);
	delete m_pListenContext;
	CloseHandle(m_hIocpHandle);
	for (int i = 0; i<m_nThreadNum; i++)
	{
		CloseHandle(m_pWorkerThreadHandle[i]);
	}
	delete m_pWorkerThreadHandle;

	for (auto p : m_allIoContext)
		delete p;
	m_allIoContext.clear();
}
void CIOCPModel::OutPutErrorMsg(int line,string strMsg)
{
	cout << strMsg << endl;
	char szBuf[128];
	LPVOID lpMsgBuf;
	DWORD dw = GetLastError();
	FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM, NULL,
		dw, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPTSTR)&lpMsgBuf, 0, NULL);
	sprintf(szBuf, "行:%d (出错码=%d): %s", line, dw, lpMsgBuf);
	LocalFree(lpMsgBuf);
	//输出提示。
	cout << szBuf << endl;
}
bool CIOCPModel::Init()
{
	if (!InitIocp())
		return false;
	if (!InitListenSocket())
		return false;

	if (!GetExtensionFunctions())
		return false;
	
	if (!PrePostAccept())
		return false;
	return true;
}
bool CIOCPModel::InitListenSocket()
{
	WSAData wsa;
	WSAStartup(MAKEWORD(2, 2), &wsa);
	SOCKADDR_IN server_addr;
	int addrLen = sizeof(SOCKADDR_IN);
	ZeroMemory(&server_addr, 0, addrLen);
	server_addr.sin_family = AF_INET;
	server_addr.sin_addr.S_un.S_addr = inet_addr(IP);
	server_addr.sin_port = htons(PORT);
	m_pListenContext = new _PER_SOCKET_CONTEXT;
	m_pListenContext->m_socket = WSASocket(AF_INET, SOCK_STREAM, 0, NULL, 0, WSA_FLAG_OVERLAPPED);
	if (m_pListenContext->m_socket == INVALID_SOCKET)
	{
		WSACleanup();
		return false;
	}
	if (!CreateIoCompletionPort((HANDLE)m_pListenContext->m_socket, m_hIocpHandle, (DWORD)m_pListenContext, 0))
	{
		OutPutErrorMsg(__LINE__);
		WSACleanup();
		return false;
	}
	cout << "Listen Socket绑定完成端口 完成" << endl;
	if (bind(m_pListenContext->m_socket, (sockaddr*)&server_addr, addrLen) == SOCKET_ERROR)
	{
		OutPutErrorMsg(__LINE__);
		WSACleanup();
		return false;
	}
	if (listen(m_pListenContext->m_socket, SOMAXCONN) == SOCKET_ERROR)
	{
		OutPutErrorMsg(__LINE__);
		WSACleanup();
		return false;
	}
	cout << "Init ListenSocket end" << endl;
	return true;
}
bool CIOCPModel::InitIocp()
{
	m_hIocpHandle = CreateIoCompletionPort(INVALID_HANDLE_VALUE, 0, NULL, 0);
	if (!m_hIocpHandle)
	{
		OutPutErrorMsg(__LINE__);
		WSACleanup();
		return false;
	}
	//获取cpu核心
	SYSTEM_INFO si;
	GetSystemInfo(&si);
	m_nThreadNum = 2 * si.dwNumberOfProcessors;
	m_pWorkerThreadHandle = new HANDLE[m_nThreadNum];
	DWORD nThreadID;
	for (int i = 0; i < m_nThreadNum; i++)
	{
		m_pWorkerThreadHandle[i] = ::CreateThread(0, 0, WorkerThread, this, 0, &nThreadID);
	}
	cout << "InitIOCP end" << endl;
	return true;
}
DWORD WINAPI CIOCPModel::WorkerThread(LPVOID param)
{
	CIOCPModel * pModel = (CIOCPModel *)param;
	OVERLAPPED * pOverLapped;
	DWORD  dwBytesTransfered = 0;
	PER_SOCKET_CONTEXT * pIoSocketContext = NULL;

	while (1)
	{
		BOOL bReturn = GetQueuedCompletionStatus(
			pModel->m_hIocpHandle,
			&dwBytesTransfered,
			(PULONG_PTR)&pIoSocketContext,//调用CreateIoCompletionPort把socket绑定到iocp时传入的第三个参数是什么, GetQueuedCompletionStatus第三个参数就会返回什么
			&pOverLapped,
			INFINITE);
		/*cout << pModel->m_pListenContext << endl;
		cout << pIoContext << endl;
		cout << pOverLapped << endl;
		cout << pModel->m_allIoContext.back() << endl;*/
		if (NULL == (DWORD)pIoSocketContext)
		{
			break;
		}
		if (!bReturn)
		{
			DWORD dwErr = GetLastError();

			// 显示一下提示信息
			if (!pModel->HandleError(pIoSocketContext, dwErr))
			{
				break;
			}

			continue;
		}
		else
		{
			// 读取传入的参数
			PER_IO_CONTEXT * pIoContext = CONTAINING_RECORD(pOverLapped, PER_IO_CONTEXT, m_Overlapped);
			// 判断是否有客户端断开了
			if ((0 == dwBytesTransfered) && (RECV_POSTED == pIoContext->m_OpType || SEND_POSTED == pIoContext->m_OpType))
			{
				pModel->RemoveContext(pIoSocketContext);
				pModel->Lock();
				printf("客户端断开连接,连接数：%d\n", pModel->m_allIoContext.size() - 10);
				pModel->UnLock();
				continue;
			}
			else
			{
				
				DWORD dwSendBytes, Flags;
				int nBytesSend;
				switch (pIoContext->m_OpType)
				{
				//TODO 第一组数据的处理
				case ACCEPT_POSTED:
					pModel->DoAccept(pIoContext);
					break;
				case RECV_POSTED:
					cout << "收到" << pIoContext->m_szBuffer << endl;
					pModel->Echo(pIoSocketContext,pIoContext);//发送给源客户端,由于下面就要清空缓冲区，所以先发送了。。

					pIoContext->ResetBuffer();//释放发送缓冲区
					pModel->PostRecv(pIoContext);

					
				/*	pIoContext->m_OpType = SEND_POSTED;
					nBytesSend = WSASend(pIoContext->m_socket, &pIoContext->m_wsaSendBuf, 1, &dwSendBytes, 0,
						&pIoContext->m_Overlapped[SEND_POSTED], NULL);
					if ((nBytesSend < 0) && (WSA_IO_PENDING != WSAGetLastError()))
					{
						pModel->OutPutErrorMsg(__LINE__);
						return 0;
					}	*/
					
					break;
				case SEND_POSTED:
					cout << "发送" << pIoContext->m_szBuffer << endl;
					pIoContext->ResetBuffer();
					break;
				default:
					break;
				}
			}
		}
	}
	return 0;
}
bool CIOCPModel::GetExtensionFunctions()
{
	GUID GuidAcceptEx = WSAID_ACCEPTEX;
	GUID GuidGetAcceptExSockAddrs = WSAID_GETACCEPTEXSOCKADDRS;
	// 使用AcceptEx函数，因为这个是属于WinSock2规范之外的微软另外提供的扩展函数
	// 所以需要额外获取一下函数的指针，
	// 获取AcceptEx函数指针
	SOCKET s = socket(AF_INET, SOCK_STREAM, 0);
	if (s == INVALID_SOCKET)
		return false;
	DWORD dwBytes = 0;
	if (SOCKET_ERROR == WSAIoctl(
		s,
		SIO_GET_EXTENSION_FUNCTION_POINTER,
		&GuidAcceptEx,
		sizeof(GuidAcceptEx),
		&m_lpfnAcceptEx,
		sizeof(m_lpfnAcceptEx),
		&dwBytes,
		NULL,
		NULL))
	{
		closesocket(s);
		
		OutPutErrorMsg(__LINE__,"WSAIoctl 未能获取AcceptEx函数指针"); 
		return false;
	}

	// 获取GetAcceptExSockAddrs函数指针，也是同理
	if (SOCKET_ERROR == WSAIoctl(
		s,
		SIO_GET_EXTENSION_FUNCTION_POINTER,
		&GuidGetAcceptExSockAddrs,
		sizeof(GuidGetAcceptExSockAddrs),
		&m_lpfnGetAcceptSockAddr,
		sizeof(m_lpfnGetAcceptSockAddr),
		&dwBytes,
		NULL,
		NULL))
	{
		closesocket(s);
		OutPutErrorMsg(__LINE__, "WSAIoctl 未能获取GetAcceptSockAddr函数指针");
		return false;
	}
	closesocket(s);
	return true;
}
//PER_SOCKET_CONTEXT * CIOCPModel::GetNewIoContext()
//{
//	Lock();
//	_PER_IO_CONTEXT* p = new _PER_IO_CONTEXT;
//	m_allIoContext.push_back(p);
//	UnLock();
//	return p;
//}
bool CIOCPModel::PrePostAccept()
{
	for (int i = 0; i<10; i++)
	{
		//对于ACCEPT_POSTED的投递在m_pListenContext 上
		PER_IO_CONTEXT* pAcceptIoContext = m_pListenContext->GetNewIoContext();

		if (false == PostAccept(pAcceptIoContext))
		{
			m_pListenContext->RemoveIoContext(pAcceptIoContext);
			return false;
		}
	}
	return true;
}
bool CIOCPModel::PostAccept(PER_IO_CONTEXT* pAcceptIoContext)//pAcceptIoContext是加在监听socket的IO列表中的
{
	assert(INVALID_SOCKET != m_pListenContext->m_socket);
	// 准备参数
	DWORD dwBytes = 0;
	pAcceptIoContext->m_OpType = ACCEPT_POSTED;
	WSABUF *p_wbuf = &pAcceptIoContext->m_wsaBuf;
	OVERLAPPED *p_ol = &pAcceptIoContext->m_Overlapped;

	// 为以后新连入的客户端先准备好Socket
	pAcceptIoContext->m_socket = WSASocket(AF_INET, SOCK_STREAM, IPPROTO_TCP, NULL, 0, WSA_FLAG_OVERLAPPED);

	if (INVALID_SOCKET == pAcceptIoContext->m_socket)
	{
		OutPutErrorMsg(__LINE__);
		return false;
		//END("创建用于Accept的Socket失败");
	}
	// 投递AcceptEx
	if (FALSE == m_lpfnAcceptEx(m_pListenContext->m_socket, pAcceptIoContext->m_socket, p_wbuf->buf, p_wbuf->len - ((sizeof(SOCKADDR_IN)+16) * 2),
		sizeof(SOCKADDR_IN)+16, sizeof(SOCKADDR_IN)+16, &dwBytes, p_ol))
	{
		if (WSA_IO_PENDING != WSAGetLastError())
		{
			OutPutErrorMsg(__LINE__, "投递 AcceptEx 请求失败");
			return false;
			//END("投递 AcceptEx 请求失败");
		}
	}
	return true;
}
//pListenIoContext是m_pListenContext里 vector<PER_IO_CONTEXT*> m_arrayIoContext; 某一项
//因为投递AcceptEx的时候是加在m_pListenContext的m_arrayIoContext里的
bool CIOCPModel::DoAccept(PER_IO_CONTEXT * pAcceptIoContext)
{
	SOCKADDR_IN* ClientAddr = NULL;
	SOCKADDR_IN* LocalAddr = NULL;
	int remoteLen = sizeof(SOCKADDR_IN), localLen = sizeof(SOCKADDR_IN);

	// GetAcceptSockAddr可以取得客户端和本地端的地址信息以及客户端发来的第一组数据
	m_lpfnGetAcceptSockAddr(pAcceptIoContext->m_wsaBuf.buf, pAcceptIoContext->m_wsaBuf.len - ((sizeof(SOCKADDR_IN)+16) * 2),
		sizeof(SOCKADDR_IN)+16, sizeof(SOCKADDR_IN)+16, (LPSOCKADDR*)&LocalAddr, &localLen, (LPSOCKADDR*)&ClientAddr, &remoteLen);
	cout << "第一组消息:" << pAcceptIoContext->m_wsaBuf.buf << endl;
	
	PER_SOCKET_CONTEXT* pNewSocketContext = new PER_SOCKET_CONTEXT;
	m_allIoContext.push_back(pNewSocketContext);
	pNewSocketContext->m_socket = pAcceptIoContext->m_socket;
//	memcpy(&(pNewSocketContext->m_ClientAddr), ClientAddr, sizeof(SOCKADDR_IN));
	接收连接后关联到完成端口
	if (CreateIoCompletionPort((HANDLE)pNewSocketContext->m_socket, m_hIocpHandle, (ULONG_PTR)pNewSocketContext, 0) == NULL)
	{
		OutPutErrorMsg(__LINE__);
		return false;
	}
	//此处新建一个socket后要把客户端的SOCKET拷贝过去
	PER_IO_CONTEXT* pNewIoContext = pNewSocketContext->GetNewIoContext();
	pNewIoContext->m_OpType = RECV_POSTED;
	pNewIoContext->m_socket = pNewSocketContext->m_socket;

	//新的SOCKET_IO创建完后m_pListenContext里的那个用于连接的PER_IO_CONTEXT可以删掉了，因为它投递的AcceptEx请求
	//已经连接了，已经没用了。不然它会一直增大,注意，此处仅仅从m_pListenContext的那个vector中移除，不delete
	//但是仅移除的话，那这个IO_CONTEXT就内存泄露了。
	//m_pListenContext->RemoveIoContext(pAcceptIoContext,false);


	// 绑定完毕之后，就可以开始在这个新的客户端Socket上投递接收信息完成请求了
	if (false == PostRecv(pNewIoContext))
	{
		RemoveContext(pNewSocketContext);
		return false;
	}
	//有一个新连接后要投递新的请求
	PER_IO_CONTEXT * pNewAcceptIoContext = m_pListenContext->GetNewIoContext();
	if (false == PostAccept(pNewAcceptIoContext))
	{
		//因为GetNewIoContext会把pNewIoContent加入到IO列表中，所以要从IO列表中删除
		m_pListenContext->RemoveIoContext(pNewAcceptIoContext);
		return false;
	}
	return true;
}
bool CIOCPModel::PostRecv(PER_IO_CONTEXT *pCurIoContext)
{	// 初始化变量
	DWORD dwFlags = 0;
	DWORD dwBytes = 0;
	WSABUF *p_wbuf = &pCurIoContext->m_wsaBuf;
	OVERLAPPED *p_ol = &pCurIoContext->m_Overlapped;

	pCurIoContext->ResetBuffer();
	pCurIoContext->m_OpType = RECV_POSTED;

	// 初始化完成后，，投递WSARecv请求
	int nBytesRecv = WSARecv(pCurIoContext->m_socket, p_wbuf, 1, &dwBytes, &dwFlags, p_ol, NULL);

	// 如果返回值错误，并且错误的代码并非是Pending的话，那就说明这个重叠请求失败了
	if ((SOCKET_ERROR == nBytesRecv) && (WSA_IO_PENDING != WSAGetLastError()))
	{
		
		OutPutErrorMsg(__LINE__);
		return false;
	}
	return true;

}
void CIOCPModel::RemoveContext(_PER_SOCKET_CONTEXT *&pIoContext)
{
	Lock();
	//auto pos = remove_if(m_allIoContext.begin(), m_allIoContext.end(), [&](PER_IO_CONTEXT* p){return p == pIoContext; });
	//RELEASE_PTR(pIoContext);
	//m_allIoContext.erase(pos, m_allIoContext.end());	
	for (auto it = m_allIoContext.begin(); it != m_allIoContext.end(); it++)
	{
		if (pIoContext && *it == pIoContext)
		{
			delete pIoContext;
			pIoContext = NULL;
			m_allIoContext.erase(it);
			break;
		}
	}
	/*if (pIoContext)
		delete pIoContext;*/
	UnLock();
}
void CIOCPModel::ServerLoop()
{
	WaitForMultipleObjects(m_nThreadNum, m_pWorkerThreadHandle, TRUE, INFINITE);
}
void CIOCPModel::Stop()
{
	if (m_pListenContext != NULL && m_pListenContext->m_socket != INVALID_SOCKET)
	{

		for (int i = 0; i < m_nThreadNum; i++)
		{
			// 通知所有的完成端口操作退出
			PostQueuedCompletionStatus(m_hIocpHandle, 0, (DWORD)NULL, NULL);
		}

		// 等待所有的客户端资源退出
		WaitForMultipleObjects(m_nThreadNum, m_pWorkerThreadHandle, TRUE, INFINITE);

		// 清除客户端列表信息
		for (auto p : m_allIoContext)
			delete p;
		m_allIoContext.clear();
		// 释放其他资源

		WSACleanup();

		printf("停止监听\n");
	}
}
bool CIOCPModel::HandleError(PER_SOCKET_CONTEXT *pContext, const DWORD& dwErr)
{
	// 如果是超时了，就再继续等吧  
	if (WAIT_TIMEOUT == dwErr)
	{
		// 确认客户端是否还活着...
		if (!IsSocketAlive(pContext->m_socket))
		{
			printf("检测到客户端异常退出！");
			RemoveContext(pContext);
			return true;
		}
		else
		{
			printf("网络操作超时！重试中...");
			return true;
		}
	}
	// 可能是客户端异常退出了
	else if (ERROR_NETNAME_DELETED == dwErr)
	{
		printf("检测到客户端异常退出！\n");
		RemoveContext(pContext);
		return true;
	}
	else
	{
		printf("完成端口操作出现错误，线程退出。错误代码：%d", dwErr);
		return false;
	}
}
bool CIOCPModel::IsSocketAlive(SOCKET s)
{
	int nByteSent = send(s, "", 0, 0);
	if (-1 == nByteSent) return false;
	return true;
}

void CIOCPModel::Echo(PER_SOCKET_CONTEXT * pSocketIO, PER_IO_CONTEXT * pIoContext)
{
	//如果size为1说明是第一次接收，还没有投递过Send请求
	if (pSocketIO->m_arrayIoContext.size() == 1)
	{
		auto pSendIO = pSocketIO->GetNewIoContext();
		pSendIO->m_socket = pIoContext->m_socket;
		pSendIO->m_OpType = SEND_POSTED;
		strcpy(pSendIO->m_szBuffer, pIoContext->m_szBuffer);

		DWORD dwSendBytes;
		int nBytesSend = WSASend(pIoContext->m_socket, &pSendIO->m_wsaBuf, 1, &dwSendBytes, 0,
			&pSendIO->m_Overlapped, NULL);
		if ((nBytesSend < 0) && (WSA_IO_PENDING != WSAGetLastError()))
		{
			OutPutErrorMsg(__LINE__);
			return;
		}
	}
	else
	{
		//如果投递过send，则直接找到那个IO_CONTEXT继续投递
		for (const auto &ele : pSocketIO->m_arrayIoContext)
		{
			if (ele->m_OpType == SEND_POSTED)
			{		
				strcpy(ele->m_szBuffer, pIoContext->m_szBuffer);
				DWORD dwSendBytes;
				int nBytesSend = WSASend(pIoContext->m_socket, &ele->m_wsaBuf, 1, &dwSendBytes, 0,
					&ele->m_Overlapped, NULL);
				if ((nBytesSend < 0) && (WSA_IO_PENDING != WSAGetLastError()))
				{
					OutPutErrorMsg(__LINE__);
					return;
				}
			}
		}
	}
	
}