IOCP完成端口

一、使用完成端口的基本流程

1. 调用 CreateIoCompletionPort 创建一个完成端口
2. 根据系统有多少个处理器, 就建立做少个工作线程
3. 接收接入的 socket 连接, 可以用 accept/AcceptEx
4. 每当客户端接入, 调用 CreateIoCompletonPort 把新的socket与完成端口绑定
5. 投递网络请求 WSARecv/WSASend
6. 工作线程调用 GetQueuedCompletionStatus 扫描完成端口是否有完成操作, 一旦有则将完成的请求从队列中取出处理, 再投递下一个网络请求

// 创建/绑定完成端口
HANDLE CreateIoCompletionPort ( 
  HANDLE FileHandle, //需要绑定的句柄socket, 传递INVALID_HANDLE_VALUE则创建完成端口
  HANDLE ExistingCompletionPort, // 需要绑定的完成端口句柄
  ULONG_PTR CompletionKey, // 可以看作是一个用户参数
  DWORD NumberOfConcurrentThreads //设置0
);
// 查询完成端口状态
BOOL GetQueuedCompletionStatus( 
  __in HANDLE CompletionPort,    // 完成端口句柄
  __in LPDWORD lpNumberOfBytes,  // 操作完成后返回的字节数
  __out PULONG_PTR lpCompletionKey, // 返回建立完成端口的时候绑定的那个用户参数
  __out LPOVERLAPPED *lpOverlapped, //返回投递I/O 操作时指定的OVERLAPPED 结构
  __in DWORD dwMilliseconds // 等待完成端口的超时时间
                          // 如果线程不需要做其他的事情，那就INFINITE就行了
);
// 关闭完成端口
BOOL PostQueuedCompletionStatus(
  HANDLE CompletionPort,            // handle to an I/O completion port
  DWORD dwNumberOfBytesTransferred, // 指定GetQueuedCompletionStatus 函数的
                                    //  lpNumberOfBytesTransferred 参数的返回值
  ULONG_PTR dwCompletionKey,  // 指定GetQueuedCompletionStatus 函数的
                              // lpCompletionKey 参数的返回值
  LPOVERLAPPED lpOverlapped  //指定GetQueuedCompletionStatus 函数的lpOverlapped 参数的返回值
);

二、标识完成通知

投递多个网络请求, 在完成端口收到完成通知之后, 怎么区分该通知是哪个套接字投递的哪个请求？
1. CreateIoCompletionPort 绑定的时候有一个用户参数 CompletionKey 可以使用, 标识是哪个套接字的请求
2. WSARecv/WSASend 投递请求的时候有一个lpOverlapped参数可以利用, 标识是哪一个请求.
但是这里使用需要稍加修改

struct MYWSAOVERLAPPED
{
    WSAOVERLAPPED overlapped;
    // 定义其他用户数据
};

使用的时候将MYWSAOVERLAPPED当作WSAOVERLAPPED来使用。
 

// 下面的例程是回显服务演示

#include <WINSOCK2.H>
#include <STDIO.H>
#pragma comment(lib, "Ws2_32.lib")

#define BUFFER_SIZE 1024

DWORD WINAPI CompletionProcess(LPVOID lpParam);

// 自定义 CompletionKey
typedef struct _MY_COMPLETION_KEY
{
	SOCKET sock;		// 套接字句柄
	SOCKADDR_IN addr;	// 对应的客户端地址
} MY_COMPLETION_KEY, *PMY_COMPLETION_KEY;

// 自定义 Overlapped
typedef struct _MY_OVERLAPPED
{
	OVERLAPPED overlapped;	// 重叠结构
	char buf[BUFFER_SIZE];	// 数据缓冲区
	int optType;			// 操作类型
} MY_OVERLAPPED, *PMY_OVERLAPPED;

#define OPT_READ	1
#define OPT_WRITE	2
#define OPT_ACCEPT	3

void main()
{
	WSADATA wsaData;
	WSAStartup(MAKEWORD(2,2), &wsaData);

	// 创建完成端口对象
	HANDLE hCompletion = CreateIoCompletionPort(INVALID_HANDLE_VALUE, 0, 0, 0);
	HANDLE hProcess = CreateThread(NULL, 0, CompletionProcess, (LPVOID)hCompletion, 0,0);
//	CloseHandle(hProcess);

	// 创建监听套接字
	SOCKET sListen = socket(AF_INET, SOCK_STREAM, 0);
	SOCKADDR_IN addr;
	addr.sin_family = AF_INET;
	addr.sin_port = htons(9000);
	addr.sin_addr.S_un.S_addr = INADDR_ANY;
	bind(sListen, (SOCKADDR*)&addr, sizeof(addr));
	listen(sListen, SOMAXCONN);
	printf("[sock %d] listenning ... \n", sListen);

	// 处理连接
	while (TRUE)
	{
		SOCKADDR_IN addrRomote;
		int nLen = sizeof(SOCKADDR);
		SOCKET newSock = accept(sListen, (SOCKADDR*)&addrRomote, &nLen);
		printf("[sock %d] accept client %d\n", sListen, newSock);

		// 将新的连接绑定到完成端口
		PMY_COMPLETION_KEY pCompletionKey = //new MY_COMPLETION_KEY;
								(PMY_COMPLETION_KEY)GlobalAlloc(GPTR, sizeof(MY_COMPLETION_KEY));
		memset(pCompletionKey, 0, sizeof(MY_COMPLETION_KEY));
		pCompletionKey->sock = newSock;
		memcpy(&pCompletionKey->addr, &addrRomote, nLen);
		if (NULL == CreateIoCompletionPort((HANDLE)pCompletionKey->sock, hCompletion, (DWORD)pCompletionKey, 0))
		{
			printf("CreateIoCompletionPort error\n");
			break;
		}

		// 投递一个接受请求
		PMY_OVERLAPPED pOverlapped = //new MY_OVERLAPPED;  必须使用 GlobalAlloc, 否则WSARecv将会失败
						(PMY_OVERLAPPED)GlobalAlloc(GPTR, sizeof(MY_OVERLAPPED));
		pOverlapped->optType = OPT_READ;
		WSABUF buf;
		buf.buf = pOverlapped->buf;
		buf.len = BUFFER_SIZE;
		DWORD dwRecv;
		DWORD dwFlags = 0;
		int iRet = WSARecv(pCompletionKey->sock, &buf, 1, &dwRecv, &dwFlags, &pOverlapped->overlapped, NULL);
		if (iRet == SOCKET_ERROR)
		{
			if (WSAGetLastError() != WSA_IO_PENDING)
			{
				printf("WSARecv error %d\n", WSAGetLastError());
			}
		}
	}
}


DWORD WINAPI CompletionProcess(LPVOID lpParam)
{
	HANDLE hCompletion = (HANDLE)lpParam;
	DWORD dwTrans;
	PMY_COMPLETION_KEY pCompletionKey;
	PMY_OVERLAPPED pOverlapped;

	printf("CompletionProcess .. \n");

	while (TRUE)
	{
		pCompletionKey = NULL;
		pOverlapped = NULL;
		// 在完成端口上等待请求完成
		BOOL bOk = GetQueuedCompletionStatus(hCompletion, &dwTrans, (LPDWORD)&pCompletionKey, 
			(LPOVERLAPPED*)&pOverlapped, WSA_INFINITE);

		if (!bOk)
		{
			if (pCompletionKey)
			{
				printf("close client, sock %d\n", pCompletionKey->sock);
				closesocket(pCompletionKey->sock);
				GlobalFree(pCompletionKey);
				pCompletionKey = NULL;
			}
			if (pOverlapped)
			{
				GlobalFree(pCompletionKey);
				pOverlapped = NULL;
			}
			continue;
		}

		if (0 == dwTrans && 
			(pOverlapped->optType == OPT_READ || 
			pOverlapped->optType == OPT_WRITE))
		{
			printf("transfer data is 0, close client, sock %d\n", pCompletionKey->sock);
			closesocket(pCompletionKey->sock);
			if (pCompletionKey)
				GlobalFree(pCompletionKey);
			if (pOverlapped)
				GlobalFree(pOverlapped);
			pCompletionKey = NULL;
			pOverlapped = NULL;
			continue;
		}

		switch (pOverlapped->optType)
		{
		case OPT_READ:
			{
				pOverlapped->buf[dwTrans] = '\0';
				printf("sock %d recv data: %s\n", pCompletionKey->sock,pOverlapped->buf);

				// 将数据回传
				WSABUF wBuf;
				wBuf.buf = pOverlapped->buf;
				wBuf.len = dwTrans;
				pOverlapped->optType = OPT_WRITE;
				WSASend(pCompletionKey->sock, &wBuf, 1, &dwTrans, 0, &pOverlapped->overlapped, NULL);
				break;
			}
		case OPT_WRITE:
			{
				printf("WSASend completion, Input WSARecv\n");
				// 再次投递接收请求
				WSABUF wBuf;
				wBuf.buf = pOverlapped->buf;
				wBuf.len = sizeof(pOverlapped->buf);
				pOverlapped->optType = OPT_READ;
				DWORD nFlags = 0;
				WSARecv(pCompletionKey->sock, &wBuf, 1, &dwTrans, &nFlags, &pOverlapped->overlapped, NULL);
				break;
			}
		case OPT_ACCEPT:
			{
			}
		default:
			break;
		}
	}
	return 0;
}