参考
- 《TCP/IP网络编程》 尹圣雨
IOCP
IOCP(Input Output Completion Port,输入输出完成端口)是性能最好的Windows平台I/O模型。Linux有epoll,BSD有kqueue。
从重叠I/O理解IOCP
实现非阻塞模式的套接字
在Windows中通过ioctlsocket函数将套接字属性改为非阻塞式,其调用的含义是,将hLisnSock句柄引用的套接字I/O模式(FIONBIO)改为变量mode中指定的形式
SOCKET hLisnSock;
int mode = 1;
......
hListSock = WSASocket(PF_INET, SOCK_STREAM, 0, NULL, 0, WSA_FLAG_OVERLAPPED);
ioctlsocket(hLisnSock, FIONBIO, &mode);
......
其中FIONBIO是用于更改套接字I/O模式的选项,该函数的第三个参数中传入的变量中若有0,则说明套接字是阻塞模式的;如果存有非0值,则说明已将套接字模式改为非阻塞模式。改为非阻塞模式后,除了以阻塞模式进行I/O外,还具有:
- 如果在没有客户端连接请求的状态下调用accept函数,将直接返回INVALID_SOCKET。调用WSAGetLastError函数时返回WSAEWOULDBLOCK
- 调用accept函数时创建的套接字同样具有非阻塞属性
因此,针对非阻塞套接字调用accept函数并返回INVALID_SOCKET时,应该通过WSAGetLastError函数确认返回INVALID_SOCKET的理由
以纯重叠I/O方式实现回声服务器端
#include <stdio.h>
#include <stdlib.h>
#include <WinSock2.h>
#define BUF_SIZE 1024
void CALLBACK ReadCompRoutine(DWORD, DWORD, LPWSAOVERLAPPED, DWORD);
void CALLBACK WriteCompRoutine(DWORD, DWORD, LPWSAOVERLAPPED, DWORD);
void ErrorHandling(char* message);
typedef struct // 该结构体中包含套接字句柄、缓冲及缓冲相关信息
{
SOCKET hClntSock;
char buf[BUF_SIZE];
WSABUF wsaBuf;
}PER_IO_DATA, * LPPER_IO_DATA;
int main(int argc, char* argv[])
{
WSADATA wsaData;
SOCKET hLisnSock, hRecvSock;
SOCKADDR_IN lisnAdr, recvAdr;
LPWSAOVERLAPPED lpOvLp;
DWORD recvBytes;
LPPER_IO_DATA hbInfo;
int mode = 1, recvAdrSz, flagInfo = 0;
if (argc != 2)
{
printf("Usage: %s <port>\n", argv[0]);
exit(1);
}
if (WSAStartup(MAKEWORD(2, 2), &wsaData) != 0)
{
ErrorHandling("WSAStartup() error!");
}
hLisnSock = WSASocket(PF_INET, SOCK_STREAM, 0, NULL, 0, WSA_FLAG_OVERLAPPED);
ioctlsocket(hLisnSock, FIONBIO, &mode); // 该为非阻塞模式
memset(&lisnAdr, 0, sizeof(lisnAdr));
lisnAdr.sin_family = AF_INET;
lisnAdr.sin_addr.s_addr = htonl(INADDR_ANY);
lisnAdr.sin_port = htons(atoi(argv[1]));
if (bind(hLisnSock, (SOCKADDR*)&lisnAdr, sizeof(lisnAdr)) == SOCKET_ERROR)
{
ErrorHandling("bind() error");
}
if (listen(hLisnSock, 5) == SOCKET_ERROR)
{
ErrorHandling("listen() error");
}
recvAdrSz = sizeof(recvAdr);
while (1)
{
SleepEx(100, TRUE); // alertable状态
hRecvSock = accept(hLisnSock, (SOCKADDR*)&recvAdr, &recvAdrSz);
if (hRecvSock == INVALID_SOCKET) // 特殊处理accept返回值
{
if (WSAGetLastError() == WSAEWOULDBLOCK)
{
continue;
}
else
{
ErrorHandling("accept() error");
}
}
puts("Client connected......");
lpOvLp = (LPWSAOVERLAPPED)malloc(sizeof(WSAOVERLAPPED)); // 每个客户端都需要独立的结构体变量
memset(lpOvLp, 0, sizeof(WSAOVERLAPPED));
hbInfo = (LPPER_IO_DATA)malloc(sizeof(PER_IO_DATA));
hbInfo->hClntSock = (DWORD)hRecvSock;
(hbInfo->wsaBuf).buf = hbInfo->buf;
(hbInfo->wsaBuf).len = BUF_SIZE;
lpOvLp->hEvent = (HANDLE)hbInfo; // 基于CR函数的重叠I/O模型中不需要事件对象,可以用来写入其他信息
WSARecv(hRecvSock, &(hbInfo->wsaBuf), 1, &recvBytes, &flagInfo, lpOvLp, ReadCompRoutine);
}
closesocket(hRecvSock);
closesocket(hLisnSock);
WSACleanup();
return 0;
}
void CALLBACK ReadCompRoutine(DWORD dwError, DWORD szRecvBytes, LPWSAOVERLAPPED lpOverlapped, DWORD flags)
{
LPPER_IO_DATA hbInfo = (LPPER_IO_DATA)(lpOverlapped->hEvent);
SOCKET hSock = hbInfo->hClntSock;
LPWSABUF bufInfo = &(hbInfo->wsaBuf);
DWORD sentBytes;
if (szRecvBytes == 0)
{
closesocket(hSock);
free(lpOverlapped->hEvent);
free(lpOverlapped);
puts("Client disconnected......");
}
else
{
bufInfo->len = szRecvBytes;
WSASend(hSock, bufInfo, 1, &sentBytes, 0, lpOverlapped, WriteCompRoutine);
}
}
void CALLBACK WriteCompRoutine(DWORD dwError, DWORD szSendBytes, LPWSAOVERLAPPED lpOverlapped, DWORD flags)
{
LPPER_IO_DATA hbInfo = (LPPER_IO_DATA)(lpOverlapped->hEvent);
SOCKET hSock = hbInfo->hClntSock;
LPWSABUF bufInfo = &(hbInfo->wsaBuf);
DWORD recvBytes;
int flagInfo = 0;
WSARecv(hSock, bufInfo, 1, &recvBytes, &flagInfo, lpOverlapped, ReadCompRoutine);
}
void ErrorHandling(char* message)
{
fputs(message, stderr);
fputc('\n', stderr);
exit(1);
}
客户端
#include <stdio.h>
#include <stdlib.h>
#include <WinSock2.h>
#include <WS2tcpip.h>
#define BUF_SIZE 1024
void ErrorHandling(char* message);
int main(int argc, char* argv[])
{
WSADATA wsaData;
SOCKET hSocket;
SOCKADDR_IN servAdr;
char message[BUF_SIZE];
int strLen, readLen;
if (argc != 3)
{
printf("Usage: %s <IP> <port>\n", argv[0]);
exit(1);
}
if (WSAStartup(MAKEWORD(2, 2), &wsaData) != 0)
{
ErrorHandling("WSAStartup() error!");
}
hSocket = socket(PF_INET, SOCK_STREAM, 0);
if (hSocket == INVALID_SOCKET)
{
ErrorHandling("socket() error");
}
memset(&servAdr, 0, sizeof(servAdr));
servAdr.sin_family = AF_INET;
inet_pton(AF_INET, argv[1], &servAdr.sin_addr);
servAdr.sin_port = htons(atoi(argv[2]));
if (connect(hSocket, (SOCKADDR*)&servAdr, sizeof(servAdr)) == SOCKET_ERROR)
{
ErrorHandling("connect() error!");
}
else
{
puts("Connected......");
}
while (1)
{
fputs("Input message(Q to quit): ", stdout);
fgets(message, BUF_SIZE, stdin);
if (!strcmp(message, "q\n") || !strcmp(message, "Q\n"))
{
break;
}
strLen = strlen(message);
send(hSocket, message, strLen, 0);
readLen = 0;
while (1)
{
readLen += recv(hSocket, &message[readLen], BUF_SIZE - 1 - readLen, 0);
if (readLen >= strLen)
{
break;
}
}
message[strLen] = 0;
printf("Message from server: %s", message);
}
closesocket(hSocket);
WSACleanup();
return 0;
}
void ErrorHandling(char* message)
{
fputs(message, stderr);
fputc('\n', stderr);
exit(1);
}
从重叠I/O模型到IOCP
上述重叠I/O模型回声服务器端,重复调用非阻塞模式的accept函数和以进入alertable wait状态为目的的SleepEx函数将影响性能。既不能为了处理连接请求而只调用accept函数,也不能为了Completion Routine而只调用SleepEx函数,因此轮流调用了非阻塞模式的accept函数和SleepEx函数
可以考虑,让main线程(main函数内部)调用accept函数,再单独创建1个线程负责客户端I/O。这就是IOCP的服务器端模型。IOCP将创建专用的I/O线程,该线程负责与所有客户端进行I/O
理解IOCP不要把焦点集中于线程,而是注意:
- I/O是否以非阻塞模式工作
- 如何确定非阻塞模式的I/O是否完成
实现IOCP
IOCP中已完成的I/O信息将注册到完成端口对象(Completion Port,CP对象)。首先需要经过套接字和CP对象之间的连接请求。为此,需要:
- 创建完成端口对象
- 建立完成端口对象和套接字之间的联系
创建完成端口
#include <windows.h>
HANDLE CreateIoCompletionPort(HANDLE FileHandle, HANDLE ExistingCompletionPort, ULONG_PTR CompletionKey, DWORD NumberOfConcurrentThreads);
成功时返回CP对象句柄,失败时返回NULL。其中:
- FileHandle:创建CP对象时传递INVALID_HANDLE_VALUE
- ExistingCompletionPort:创建CP对象时传递NULL
- CompletionKey:创建CP对象时传递0
- NumberOfConcurrentThreads:分配给CP对象的用于处理I/O的线程数。如果为0,系统中的CPU个数就是可同时运行的最大线程数
示例:
HANDLE hCpObject;
......
hCpObject = CreateIoCompletionPort(INVALID_HANDLE_VALUE, NULL, 0, 2);
连接完成端口对象和套接字
该功能同样由CreateIoCompletionPort函数实现
#include <windows.h>
HANDLE CreateIoCompletionPort(HANDLE FileHandle, HANDLE ExistingCompletionPort, ULONG_PTR CompletionKey, DWORD NumberOfConcurrentThreads);
成功时返回CP对象句柄,失败时返回NULL。其中:
- FileHandle:要连接到CP对象的套接字句柄
- ExistingCompletionPort:要连接套接字的CP对象句柄
- CompletionKey:传递已完成I/O相关信息
- NumberOfConcurrentThreads:无论传递何值,只要第二个参数非NULL就会忽略
示例:
HANDLE hCpObject;
SOCKET hSock;
......
CreateIoCompletionPort((HANDLE)hSock, hCpObject, (DWORD)ioInfo, 0);
调用CreateIoCompletionPort函数后,只要针对hSock的I/O完成,相关信息就将注册到hCpObject指向的CP对象
确认CP中注册的已完成的I/O
#include <windows.h>
BOOL GetQueuedCompletionStatus(HANDLE CompletionPort, LPDWORD lpNumberOfBytes, PULONG_PTR lpCompletionKey, LPOVERLAPPED* lpOverlapped, DWORD dwMilliseconds);
成功时返回TRUE,失败时返回FALSE。其中:
- CompletionPort:注册有已完成I/O信息的CP对象句柄
- lpNumberOfBytes:用于保存I/O过程中传输的数据大小的变量地址值
- lpCompletionKey:用于保存CreateIoCompletionPort函数的第三个参数值的变量地址值
- lpOverlapped:用于保存调用WSASend、WSARecv函数时传递的OVERLAPPED结构体地址的变量地址值
- dwMilliseconds:超时信息,超过该指定时间后将返回FALSE并跳出函数。传递INFINITE时,程序将阻塞,直到已完成I/O信息写入CP对象
GetQueuedCompletionStatus由处理IOCP中已完成I/O的线程调用,在I/O完成且已注册相关信息时返回(如果最后一个参数传递INFINITE)
实现基于IOCP的回声服务器端
#include <stdio.h>
#include <stdlib.h>
#include <process.h>
#include <WinSock2.h>
#include <Windows.h>
#define BUF_SIZE 100
#define READ 3
#define WRITE 5
typedef struct // 保存与客户端相连套接字
{
SOCKET hClntSock;
SOCKADDR_IN clntAdr;
}PER_HANDLE_DATA, * LPPER_HANDLE_DATA;
typedef struct // 将I/O中使用的缓冲和重叠I/O中需要的OVERLAPPED结构体变量封装到同一结构体
{
OVERLAPPED overlapped;
WSABUF wsaBuf;
char buffer[BUF_SIZE];
int rwMode;
}PER_IO_DATA, * LPPER_IO_DATA;
DWORD WINAPI EchoThreadMain(LPVOID CompletionPortIO);
void ErrorHandling(char* message);
int main(int argc, char* argv[])
{
WSADATA wsaData;
HANDLE hComPort;
SYSTEM_INFO sysInfo;
LPPER_IO_DATA ioInfo;
LPPER_HANDLE_DATA handleInfo;
SOCKET hServSock;
SOCKADDR_IN servAdr;
int recvBytes, i, flags = 0;
if (WSAStartup(MAKEWORD(2, 2), &wsaData) != 0)
{
ErrorHandling("WSAStartup() error!");
}
hComPort = CreateIoCompletionPort(INVALID_HANDLE_VALUE, NULL, 0, 0); // 创建CP对象
GetSystemInfo(&sysInfo);
for (i = 0; i < sysInfo.dwNumberOfProcessors; i++)
{
_beginthreadex(NULL, 0, EchoThreadMain, (LPVOID)hComPort, 0, NULL); // 创建与CPU个数相等的线程
}
hServSock = WSASocket(PF_INET, SOCK_STREAM, 0, NULL, 0, WSA_FLAG_OVERLAPPED);
memset(&servAdr, 0, sizeof(servAdr));
servAdr.sin_family = AF_INET;
servAdr.sin_addr.s_addr = htonl(INADDR_ANY);
servAdr.sin_port = htons(atoi(argv[1]));
bind(hServSock, (SOCKADDR*)&servAdr, sizeof(servAdr));
listen(hServSock, 5);
while (1)
{
SOCKET hClntSock;
SOCKADDR_IN clntAdr;
int addrLen = sizeof(clntAdr);
hClntSock = accept(hServSock, (SOCKADDR*)&clntAdr, &addrLen);
handleInfo = (LPPER_HANDLE_DATA)malloc(sizeof(PER_HANDLE_DATA));
handleInfo->hClntSock = hClntSock;
memcpy(&(handleInfo->clntAdr), &clntAdr, addrLen);
CreateIoCompletionPort((HANDLE)hClntSock, hComPort, (DWORD)handleInfo, 0); // 连接CP对象和套接字。针对该套接字的重叠I/O完成时,已完成的信息将写入连接的CP对象
ioInfo = (LPPER_IO_DATA)malloc(sizeof(PER_IO_DATA)); // 相当于同时准备了WSARecv函数中需要的OVERLAPPED结构体变量、WSABUF结构体变量和缓冲
memset(&(ioInfo->overlapped), 0, sizeof(OVERLAPPED));
ioInfo->wsaBuf.len = BUF_SIZE;
ioInfo->wsaBuf.buf = ioInfo->buffer;
ioInfo->rwMode = READ; // 需要额外的变量区分2种I/O
WSARecv(handleInfo->hClntSock, &(ioInfo->wsaBuf), 1, &recvBytes, &flags, &(ioInfo->overlapped), NULL); // 第六个参数相当于传入了PER_IO_DATA结构体变量地址值
}
return 0;
}
DWORD WINAPI EchoThreadMain(LPVOID pComPort)
{
HANDLE hComPort = (HANDLE)pComPort;
SOCKET sock;
DWORD bytesTrans;
LPPER_HANDLE_DATA handleInfo;
LPPER_IO_DATA ioInfo;
DWORD flags = 0;
while (1)
{
GetQueuedCompletionStatus(hComPort, &bytesTrans, (LPDWORD)&handleInfo, (LPOVERLAPPED*)&ioInfo, INFINITE); // 第3个和第4个参数分别得到CreateIoCompletionPort传入的第3个参数和WSARecv或WSASend传入的第2个参数
sock = handleInfo->hClntSock;
if (ioInfo->rwMode == READ)
{
puts("message received!");
if (bytesTrans == 0)
{
closesocket(sock);
free(handleInfo);
free(ioInfo);
continue;
}
memset(&(ioInfo->overlapped), 0, sizeof(OVERLAPPED));
ioInfo->wsaBuf.len = bytesTrans;
ioInfo->rwMode = WRITE;
WSASend(sock, &(ioInfo->wsaBuf), 1, NULL, 0, &(ioInfo->overlapped), NULL); // 将收到的消息发送给客户端
ioInfo = (LPPER_IO_DATA)malloc(sizeof(PER_IO_DATA));
memset(&(ioInfo->overlapped), 0, sizeof(OVERLAPPED));
ioInfo->wsaBuf.len = BUF_SIZE;
ioInfo->wsaBuf.buf = ioInfo->buffer;
ioInfo->rwMode = READ;
WSARecv(sock, &(ioInfo->wsaBuf), 1, NULL, &flags, &(ioInfo->overlapped), NULL); // 接收客户端消息
}
else
{
puts("message sent!");
free(ioInfo);
}
}
return 0;
}
void ErrorHandling(char* message)
{
fputs(message, stderr);
fputc('\n', stderr);
exit(1);
}