Win-服务器端 IOCP模型

1、IOCP概念
输入输出完成端口（Input/()utputCompletionPort，IOCP），是支持多个同时发生的异步I/O操作的应用程序编程接口，在WindowsNT的3．5版本以后，或AIX5版以后或Solaris第十版以后，开始支持。IOCP特别适合c/s模式网络服务器端模型。因为，让每一个socket有一个线程负责同步（阻塞）数据处理，one-thread-per-client的缺点是：一是如果连入的客户多了，就需要同样多的线程；二是不同的socket的数据处理都要线程切换的代价。

2、IOCP原理
通常的办法是，线程池中的工作线程的数量与CPU内核数量相同，以此来最小化线程切换代价。一个IOCP对象，在操作系统中可关联着多个Socket和（或）文件控制端。IOCP对象内部有一个先进先出(FIFO)队列，用于存放IOCP所关联的输入输出端的服务请求完成消息。请求输入输出服务的进程不接收10服务完成通知，而是检查IOCP的消息队列以确定10请求的状态。（线程池中的）多个线程负责从IOCP消息队列中取走完成通知并执行数据处理；如果队列中没有消息，那么线程阻塞挂起在该队列。这些线程从而实现了负载均衡。

3、内部结构
Windows中利用CreateIoCompletionPort命令创建完成端口对象时，统内部为该对象自动创建了5个数据结构，分别是：
设备列表(DeviceList)
IO完成请求队列(I/OCompletionQueue-FIFO）
等待线程队列(WaitingThreadList-LIFO)
释放线程队列(ReleasedThreadList)
暂停线程队列(PausedThreadList)

4、完成端口实现流程
1、创建一个完成端口CreateIoCompletionPort()
2、根据OS中CPU核心数量建立对应Worker线程
3、创建一个用于监听socket，绑定到完成端口上，然后开始在指定的端口上监听客户端连接请求
4、必须在监听socket上投递AcceptEx请求（mswsock.dIl mswsock.lib等）
5、Worker线程干的事情：使用GetQueuedCompletionStatus()监控完成端口
6、当接收到AcceptEx通知时 _DoAccept0
7、当收到Recv通知时，_DoRecv0
8、关闭完成端口

5、

6、IOCPServer.cpp

#include <WinSock2.h>
#include <Windows.h>
#include <vector>
#include <iostream>
using namespace std;

// IOCP需要调用到动态链接库
#pragma comment(lib,"Kernel32.lib")
// Socket编程需要用的动态连接库
#pragma comment(lib,"Ws2_32.lib")

const int g_DataBufferSize = 2048;			// 服务器端口
typedef struct{
	OVERLAPPED overLapped;
	WSABUF dataBuffer;
	char buffer[g_DataBufferSize];
	int bufferLength;
	int operationType;
}PER_IO_OPERATEION_DATA,*LPPER_IO_OPERATION_DATA,*LPPER_IO_DATA,PER_IO_DATA;

typedef struct{
	SOCKET socket;
	SOCKADDR_STORAGE clientAddr;
	char *pszClientName;
}PER_HANDLE_DATA,*LPPER_HANDLE_DATA;

// 设置自定义全局变量
HANDLE ComletionPort = NULL;
HANDLE hMutex = CreateMutex(NULL, FALSE, NULL);

const int g_DefaultPort = 6688;
SOCKET serverSocket;			// 服务器socket
vector<PER_HANDLE_DATA*> g_clientGroup;		// 专用于记录客户端向量组

// 设置自定义函数声明
BOOL InitNetwork(WORD port);	// 初始化网络相关操作
BOOL InitWorkThread();		// 初始化工作者线程
DWORD WINAPI ServerWorkThread(LPVOID CompletionPortID);		// 工作者线程处理函数
DWORD WINAPI ServerSendThread(LPVOID lpParam);	// 发送消息的函数
DWORD WINAPI AddClient(LPVOID lpParam);		// 向服务器添加客户端
BOOL DeleteClient(vector < PER_HANDLE_DATA*>&g_clientGroup, SOCKET&clientSocket);		// 向服务器删除客户端（退出）
int myStringLen(const char *str);	// 字符串处理函数


int _tmain(int argc, _TCHAR* argv[])
{
	if (!InitNetwork(g_DefaultPort))
		return -1;

	if (!InitWorkThread())  // 错误点
		return -1;
	HANDLE sendThread = CreateThread(NULL, 0, ServerSendThread, 0, 0, NULL);
	HANDLE addThread = CreateThread(NULL, 0, AddClient, 0, 0, NULL);
	WaitForSingleObject(sendThread, INFINITE);
	WaitForSingleObject(addThread, INFINITE);

	system("pause");

	return 0;
}

int myStringLen(const char *str)
{
	int i = 0;
	while (*str != '\0')
	{
		i++;
		str++;
	}
	return i;
}

BOOL InitNetwork(WORD port)	// 初始化网络相关操作
{
	WORD wVersionRequested = MAKEWORD(2, 2);
	WSADATA wsaData;
	DWORD Error = WSAStartup(wVersionRequested, &wsaData);
	
	if (Error != 0)
	{
		printf("\n初始化请求动态连接库失败.\n\n");
		return FALSE;
	}

	if (LOBYTE(wsaData.wVersion) != 2 || HIBYTE(wsaData.wVersion) != 2)
	{
		WSACleanup();
		cerr << "\n请求的版本不是2.2版本.\n\n";
		return FALSE;
	}

	// 建立服务器socket
	serverSocket = socket(AF_INET, SOCK_STREAM, 0);	// TCP协议流式套接字
	SOCKADDR_IN serverAddress;
	serverAddress.sin_addr.S_un.S_addr = htonl(INADDR_ANY);
	serverAddress.sin_family = AF_INET;
	serverAddress.sin_port = htons(port);

	int bindResult = bind(serverSocket, (SOCKADDR*)&serverAddress, sizeof(SOCKADDR));
	if (bindResult == SOCKET_ERROR)
	{
		cerr << "\n绑定服务器失败:" << GetLastError() << endl;
		return FALSE;
	}

	// 将SOCKET套接字设置为监听模式
	int listenResult = listen(serverSocket, 100);
	if (SOCKET_ERROR == listenResult)
	{
		cerr << "\n服务器进入监听模式失败:" << GetLastError() << endl;
		return FALSE;
	}

	cout << "\n\n\tIOCP模型服务器端已准备就绪，正在等待客户端连接..........\n" << endl;

	return TRUE;
}

BOOL InitWorkThread()		// 初始化工作者线程
{
	// 首先创建IOCP内核对象
	HANDLE completionPort = CreateIoCompletionPort(INVALID_HANDLE_VALUE, NULL, 0, 0);
	if (NULL == completionPort)	// 判断创建IO内核对象是否成功，失败就执行如下语句块
	{
		cerr << "创建IO完成端口失败:" << GetLastError() << endl;
		return FALSE;
	}

	// 创建IOCP线程（线程里面创建线程池，并且将完成端口传递到内核该线程） 内核：单核 双核 微内核
	SYSTEM_INFO mySystemInfo;
	GetSystemInfo(&mySystemInfo); // 确定处理器的核心数量
	
	// 根据CPU的核心数量创建对应线程
	for (DWORD i = 0; i < (mySystemInfo.dwNumberOfProcessors * 2 + 2); i++)
	{
		HANDLE threadHandle = CreateThread(NULL, 0, ServerWorkThread, completionPort, 0, NULL);
		if (NULL == threadHandle)
		{
			cerr << "创建线程失败:" << GetLastError() << endl;
			return FALSE;
		}
		CloseHandle(threadHandle);
	}

	return TRUE;
}

DWORD WINAPI ServerWorkThread(LPVOID lpParam) //工作者线程处理函数
{
	ComletionPort = (HANDLE)lpParam;
	DWORD BytesTransferred;
	LPOVERLAPPED IpOverlapped;
	LPPER_HANDLE_DATA PerHanleData = NULL;
	LPPER_IO_DATA PerIoData = NULL;
	DWORD RecvBytes;
	DWORD Flags = 0;
	BOOL bRet = false;

	while (true)
	{
		bRet = GetQueuedCompletionStatus(ComletionPort, &BytesTransferred, (PULONG_PTR)&PerHanleData, (LPOVERLAPPED*)&IpOverlapped, INFINITE);
		if (bRet == 0)
		{
			cerr << "\n获取完成端口状态发生失败:" << GetLastError() << endl;
			DeleteClient(g_clientGroup, PerHanleData->socket);
			continue;
		}


		PerIoData = (LPPER_IO_DATA)CONTAINING_RECORD(IpOverlapped, PER_IO_DATA, overLapped);

		// 检查在套接字上是否有错误发生
		if (BytesTransferred == 0)
		{
			closesocket(PerHanleData->socket);
			GlobalFree(PerHanleData);
			GlobalFree(PerIoData);
			continue;
		}

		// 开始处理数据信息，接收来自客户端的数据
		WaitForSingleObject(hMutex, INFINITE);
		SOCKET clientsock = NULL;
		cout << "有客户端数据为:" << PerIoData->dataBuffer.buf << endl;
		ReleaseMutex(hMutex);

		// 为下一个重叠调用建立单个IO操作数据
		ZeroMemory(&(PerIoData->overLapped), sizeof(OVERLAPPED));
		PerIoData->dataBuffer.len = 1024;
		PerIoData->dataBuffer.buf = PerIoData->buffer;
		PerIoData->operationType = 0;		// 读操作
		WSARecv(PerHanleData->socket, &(PerIoData->dataBuffer), 1, &RecvBytes, &Flags, &(PerIoData->overLapped), NULL);
	}

	return TRUE;
}

BOOL DeleteClient(vector < PER_HANDLE_DATA*>&g_clientGroup, SOCKET&clientSocket)	// 向服务器删除客户端（退出）
{
	for (int i = 0; i < g_clientGroup.size(); i++)
	{
		if (g_clientGroup[i]->socket == clientSocket)
		{
			g_clientGroup.erase(g_clientGroup.begin() + i);
			cout << "已监测到客户端退出.\n\n";
			return TRUE;
		}
	}

	return FALSE;
}

DWORD WINAPI ServerSendThread(LPVOID lpParam)	// 发送消息的函数
{
	while (1)
	{
		char data[2048];
		gets_s(data);
		int len;
		for (len = 0; data[len] != '\0'; len++);
		
		data[len] = '\n';
		data[++len] = '\0';
		cout << "数据为:" << data << g_clientGroup.size() << endl;

		WaitForSingleObject(hMutex, INFINITE);
		for (int i = 0; i < g_clientGroup.size(); ++i)
		{
			//发送数据
			send(g_clientGroup[i]->socket, data, 2048, 0);
		}
		ReleaseMutex(hMutex);
	}

	return 0;
}

DWORD WINAPI AddClient(LPVOID lpParam)		// 向服务器添加客户端
{
	while (true)
	{
		PER_HANDLE_DATA *PerHandleData = NULL;
		SOCKADDR_IN saRemote;
		int RemoteLen;
		SOCKET acceptSocket;

		RemoteLen = sizeof(saRemote);
		acceptSocket = accept(serverSocket, (SOCKADDR*)&saRemote, &RemoteLen);
		if (acceptSocket == SOCKET_ERROR)
		{
			cerr << "接收客户端失败:" << GetLastError() << endl;
			return -1;
		}

		PerHandleData = (LPPER_HANDLE_DATA)GlobalAlloc(GPTR, sizeof(PER_HANDLE_DATA));
		PerHandleData->socket = acceptSocket;
		memcpy(&PerHandleData->clientAddr, &saRemote, RemoteLen);
		g_clientGroup.push_back(PerHandleData); // 将单个客户端数据指针放到客户端组当中

		CreateIoCompletionPort((HANDLE)(PerHandleData->socket), ComletionPort, (DWORD)PerHandleData, 0);

		LPPER_IO_OPERATION_DATA PerIoData = NULL;
		PerIoData = (LPPER_IO_OPERATION_DATA)GlobalAlloc(GPTR, sizeof(PER_IO_OPERATEION_DATA));
		PerIoData->dataBuffer.len = 2048;
		PerIoData->dataBuffer.buf = PerIoData->buffer;
		PerIoData->operationType = 0; // 读取操作

		DWORD RecvBytes;
		DWORD Flags = 0;
		WSARecv(PerHandleData->socket, &(PerIoData->dataBuffer), 1, &RecvBytes, &Flags, &(PerIoData->overLapped), NULL);
	}
}

7、IOCPClient.cpp

#include <cstdio>
#include <iostream>
#include <string>
#include <cstring>
#include <WinSock2.h>
#include <Windows.h>
using namespace std;

#pragma comment(lib,"Ws2_32.lib")		// socket编程需要使用到动态链接库

SOCKET sockClient;		// 连接成功后套接字
HANDLE bufferMutex;		// 将其互斥成功正常通信的信息量句柄
const int DefaultPort = 6688;	// 端口号

DWORD WINAPI SendMsgThreadFunc(LPVOID lpParameter);	// 发送
DWORD WINAPI RecvMsgThreadFunc(LPVOID lpParameter);	// 接收

int _tmain(int argc, _TCHAR* argv[])
{
	// 加载socket动态链接库dll
	WORD wVersionRequested;
	WSADATA wsaData;	// 这个结构体类型用于接收windows socket的结构信息
	wVersionRequested = MAKEWORD(2, 2);
	int iError = WSAStartup(wVersionRequested, &wsaData);
	if (iError != 0)	// 如果返回0，则表示成功申请WSAStartup
		return -1;

	// 检查版本号是否正确
	if (LOBYTE(wsaData.wVersion) != 2 || HIBYTE(wsaData.wVersion) != 2)
	{
		WSACleanup();
		return -1;
	}

	sockClient = socket(AF_INET, SOCK_STREAM, 0);
	if (sockClient == INVALID_SOCKET)
	{
		printf("Error socket():%ld\n",WSAGetLastError());
		return -1;
	}

	SOCKADDR_IN addrSrv;
	addrSrv.sin_addr.S_un.S_addr = inet_addr("127.0.0.1");
	addrSrv.sin_family = AF_INET;
	addrSrv.sin_port = htons(DefaultPort);

	while (SOCKET_ERROR == connect(sockClient, (SOCKADDR*)&addrSrv, sizeof(SOCKADDR)))
	{
		// 如果未连接上服务器，则要求重新连接
		cout << "服务器连接失败，是否重新连接......?(Y/N):";
		char ch;
		while (cin >> ch && (!((ch != 'Y' && ch == 'N') || (ch == 'Y' && ch != 'N'))))
		{
			cout << "你输入错误，请重新输入:";
			cin.sync();
			cin.clear();
		}

		if (ch == 'Y')
		{
			continue;
		}
		else
		{
			cout << "退出系统当中......\n";
			system("pause");
			return 0;
		}
	}

	cin.sync();
	cout << "客户端已准备完毕，可以直接输入数据向服务器发送请求:\n";
	send(sockClient, "\nA Client Has Enter......\,", 200, 0);

	bufferMutex = CreateSemaphore(NULL, 1, 1, NULL);

	HANDLE sendThread = CreateThread(NULL, 0, SendMsgThreadFunc, NULL, 0, NULL);
	HANDLE recvThread = CreateThread(NULL, 0, RecvMsgThreadFunc, NULL, 0, NULL);

	WaitForSingleObject(sendThread, INFINITE);
	closesocket(sockClient);
	CloseHandle(sendThread);
	CloseHandle(recvThread);
	CloseHandle(bufferMutex);
	WSACleanup();

	system("pause");

	return 0;
}

DWORD WINAPI SendMsgThreadFunc(LPVOID lpParameter)	// 发送
{
	while (1)
	{
		string  strtemp;
		getline(cin, strtemp);
		WaitForSingleObject(bufferMutex, INFINITE);	// p（资源未被占用）
		if ("exit" == strtemp || "EXIT" == strtemp){
			strtemp.push_back('\0');
			send(sockClient, strtemp.c_str(), 200, 0);
			break;
		}
		else
		{
			strtemp.append("\n");
		}

		printf("\n客户端数据消息(若输入exit 或 EXIT)退出程序:");
		cout<<strtemp;
		send(sockClient, strtemp.c_str(), 200, 0);
		ReleaseSemaphore(bufferMutex, 1, NULL);		// V(资源占用完毕)
	}

	return 0;
}

DWORD WINAPI RecvMsgThreadFunc(LPVOID lpParameter)	// 接收
{
	while (1)
	{
		char recvBuf[200];
		recv(sockClient, recvBuf, 200, 0);
		WaitForSingleObject(bufferMutex, INFINITE);	// P(资源未被占用)
		printf("%s Datas: %s","Server",recvBuf);	// 接收消息
		ReleaseSemaphore(bufferMutex, 1, NULL);		// V(资源占用完毕)
	}

	return 0;
}