《TCP/IP网络编程》第23章 IOCP
通过重叠I/O理解IOCP
IOCP(Input Output Completion Port,输入输出完成端口)。
服务器端的响应时间和并发服务数是衡量服务器端好坏的重要因素。
硬件性能和分配带宽充足情况下,若响应时间和并发服务数出了问题,查看以下两点:
- 低效的I/O结构或低效的CPU使用
- 数据库设计和查询语句(Query)的结构
非阻塞模式的套接字
int mode=1;
SOCKET hLisnSock = WSASocket(PF_INET, SOCK_STREAM, 0, NULL, 0, WSA_FLAG_OVERLAPPED);
ioctlsocket(hLisnSock, FIONBIO, &mode);
非阻塞模式套接字特点:
- 无客户端连接请求状态下调用accpet函数,直接返回INVALID_SOCKET。调用WSAGetLastError函数返回WSAEWOULDBLOCK。
- accpet函数创建的套接字同样具有非阻塞属性。
纯重叠方式实现回声服务器端
23.CmplRouEchoServ_win.c
#include <stdio.h>
#include <stdlib.h>
#include <winsock2.h>
void CALLBACK ReadCompRoutine(DWORD, DWORD, LPWSAOVERLAPPED, DWORD);
void CALLBACK WriteCompRoutine(DWORD, DWORD, LPWSAOVERLAPPED, DWORD);
#define PORT 9999
#define BUF_SIZE 1024
typedef struct
{
SOCKET socket; // 套接字句柄
char buf[BUF_SIZE]; // 读写数据放置地址
WSABUF wsaBuf; // 指向放置读写的数据的地址和大小
} PER_IO_DATA, *LPPER_IO_DATA;
void ErrorHanding(const char *message)
{
fputs(message, stderr);
fputc('\n', stderr);
exit(1);
}
// 回调函数,输入缓冲区读取结束后调用
void CALLBACK ReadCompRoutine(DWORD dwError, DWORD szRecvBytes, LPWSAOVERLAPPED lpOverlapped, DWORD flags)
{
LPPER_IO_DATA hbInfo = (LPPER_IO_DATA)(lpOverlapped->hEvent);
SOCKET socket = hbInfo->socket;
LPWSABUF bufInfo = &(hbInfo->wsaBuf);
DWORD sentBytes;
if (szRecvBytes == 0) // 读取到EOF,客户端已断开连接
{
closesocket(socket);
// 释放malloc分配的内存
free(lpOverlapped->hEvent);
free(lpOverlapped);
puts("Client disconnected.....");
}
else
{
// 读取szRecvBytes字节成功,发送szRecvBytes字节
bufInfo->len = szRecvBytes;
// 发送结束后(如何确保刚好发送了szRecvBytes个字节数?)调用WriteCompRoutine回调函数
// 可以优化
WSASend(socket, bufInfo, 1, &szRecvBytes, 0, lpOverlapped, WriteCompRoutine);
}
}
// 回调函数,输出缓冲区发送结束后调用
void CALLBACK WriteCompRoutine(DWORD dwError, DWORD szSendBytes, LPWSAOVERLAPPED lpOverlapped, DWORD flags)
{
LPPER_IO_DATA hbInfo = (LPPER_IO_DATA)(lpOverlapped->hEvent);//获取数据
SOCKET socket = hbInfo->socket;
LPWSABUF bufInfo = &(hbInfo->wsaBuf);
DWORD recvBytes;
DWORD flagInfo = 0;
// 发送完成后,读取输入缓冲区,最多读取BUF_SIZE字节(无法保证读取的字节个数)
// 读取成功后,调用ReadCompRoutine函数
bufInfo->len = BUF_SIZE;
WSARecv(socket, bufInfo, 1, &recvBytes, &flagInfo, lpOverlapped, ReadCompRoutine);
}
int main(int argc, char *argv[])
{
WSADATA wsaData;
if (WSAStartup(MAKEWORD(2, 2), &wsaData) != 0)
ErrorHanding("WSAStartup() error!");
// WSA_FLAG_OVERLAPPED,重叠I/O套接字
SOCKET hServSock = WSASocket(PF_INET, SOCK_STREAM, 0, NULL, 0, WSA_FLAG_OVERLAPPED);
if (hServSock == INVALID_SOCKET)
ErrorHanding("WSAocket() error!");
// 非阻塞套接字
u_long mode = 1;
ioctlsocket(hServSock, FIONBIO, &mode);
int opt = 1;
if (setsockopt(hServSock, SOL_SOCKET, SO_REUSEADDR, (const char *)&opt, sizeof(opt)) < 0)
ErrorHanding("setsockopt() error!");
int szAddr = sizeof(SOCKADDR_IN);
SOCKADDR_IN servAddr;
memset(&servAddr, 0, szAddr);
servAddr.sin_family = AF_INET;
servAddr.sin_addr.s_addr = htonl(INADDR_ANY);
servAddr.sin_port = htons(PORT);
if (bind(hServSock, (SOCKADDR *)&servAddr, szAddr) == SOCKET_ERROR)
ErrorHanding("bind() error!");
if (listen(hServSock, 5) == SOCKET_ERROR)
ErrorHanding("listen() error!");
while (1)
{
// 激活CR函数(重叠I/O可读写时调用)
SleepEx(100, TRUE); // for alertable wait state,为了运行CR函数,影响效率
// 等待客户端连接
SOCKADDR_IN clntAddr;
SOCKET hClntSock = accept(hServSock, (SOCKADDR *)&clntAddr, &szAddr);
if (hClntSock == INVALID_SOCKET)
{
if (WSAGetLastError() == WSAEWOULDBLOCK) // 无客户端连接
continue;
else
ErrorHanding("accept() error!");
}
puts("Client connected.....");
// malloc创建的内存空间,随着hClntSock的关闭(收到客户端的EOF)而释放
LPPER_IO_DATA hbInfo = (LPPER_IO_DATA)malloc(sizeof(PER_IO_DATA));
hbInfo->socket = (DWORD)hClntSock;
(hbInfo->wsaBuf).len = BUF_SIZE;
(hbInfo->wsaBuf).buf = hbInfo->buf;
LPWSAOVERLAPPED lpOvLp = (LPWSAOVERLAPPED)malloc(sizeof(WSAOVERLAPPED));
memset(lpOvLp, 0, sizeof(WSAOVERLAPPED));
lpOvLp->hEvent = (HANDLE)hbInfo; // CR函数调用不需要事件对象,loPvLp->hEvent中可以写入其他信息
DWORD recvBytes;
DWORD flagInfo = 0;
// 读取输入缓冲区,最多读取BUF_SIZE字节(放入hbInfo->wsaBuf中), 读取成功后,调用ReadCompRoutine函数
// lpOvLp用于回调函数ReadCompRoutine访问
WSARecv(hClntSock, &(hbInfo->wsaBuf), 1, &recvBytes, &flagInfo, lpOvLp, ReadCompRoutine);
}
closesocket(hServSock);
WSACleanup();
return 0;
}
// gcc 23.CmplRouEchoServ_win.c -o 23.CmplRouEchoServ_win -lws2_32 && 23.CmplRouEchoServ_win
重新实现客户端
23.StableEchoClnt_win.c
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <winsock2.h>
#define IP "127.0.0.1"
#define PORT 9999
#define BUF_SIZE 1024
void ErrorHanding(const char *message)
{
fputs(message, stderr);
fputc('\n', stderr);
exit(1);
}
int main(int argc, char *argv[])
{
WSADATA wsaData;
if (WSAStartup(MAKEWORD(2, 2), &wsaData) != 0)
ErrorHanding("WSAStartup() error!");
SOCKET hSock = socket(PF_INET, SOCK_STREAM, 0);
if (hSock == INVALID_SOCKET)
ErrorHanding("socket() error!");
int szAddr = sizeof(SOCKADDR_IN);
SOCKADDR_IN servAddr;
memset(&servAddr, 0, szAddr);
servAddr.sin_family = AF_INET;
servAddr.sin_addr.s_addr = inet_addr(IP);
servAddr.sin_port = htons(PORT);
if (connect(hSock, (SOCKADDR *)&servAddr, szAddr) == SOCKET_ERROR)
ErrorHanding("connect() error!");
else
puts("Connected.......");
while (1)
{
fputs("Input message(Q to quit): ", stdout);
char message[BUF_SIZE] = {0};
fgets(message, BUF_SIZE - 1, stdin);
if (!strcmp(message, "q\n") || !strcmp(message, "Q\n"))
break;
int strLen = strlen(message);
send(hSock, message, strLen, 0);
int readLen = 0;
while (1)
{
readLen += recv(hSock, &message[readLen], BUF_SIZE - 1, 0);
if (readLen >= strLen)
break;
}
message[strLen] = 0;
printf("Message from server: %s\n", message);
}
closesocket(hSock);
WSACleanup();
return 0;
}
// gcc 23.StableEchoClnt_win.c -o 23.StableEchoClnt_win -lws2_32 && 23.StableEchoClnt_win
从重叠I/O模型到IOCP模型
accpet函数处理连接请求,SleepEx函数(以进入alertable wait状态为目的)用于Completion Routine。重复轮流调用非阻塞的accpet函数和SleepEx函数(较短超时时间),是重叠I/O结构的固有缺陷。
解决方法:
main线程调用accept函数,创建线程负责客户端I/O。(IOCP服务器端模型)
IOCP将创建专用的I/O线程,该线程负责与所有客户端进行I/O。
IOCP关注焦点:
- I/O是否以非阻塞模式工作?
- 如何确定非阻塞模式的I/O是否完成?
分阶段实现IOCP程序
创建“完成端口"
IOCP已完成的I/O信息将注册到完成端口对象(Completion Port,CP),即套接字和CP对象之间的连接请求(套接字I/O完成时,把状态信息注册到指定CP对象)。
套接字I/O完成时(可读或可写),CP对象中注册的状态信息改变(套接字与cp对象已连接)。
#include <windows.h>
//失败NULL
HANDLE CreateIoCompletionPort(
HANDLE FileHandle,//创建CP对象时传递INVALID_HANDLE_VALUE
HANDLE ExistingCompletionPort,//创建CP对象时传递NULL
ULONG_PTR CompletionKey,//创建CP对象时传递0
DWORD NumberOfConcurrentThreads //分配给CP对象的用于处理I/O的可同时运行的最大线程数,0表示CPU个数。
);
连接完成端口对象和套接字
#include <windows.h>
//失败NULL
HANDLE CreateIoCompletionPort(
HANDLE FileHandle,//连接到CP对象的套接字句柄
HANDLE ExistingCompletionPort,//连接到套接字的CP对象句柄
ULONG_PTR CompletionKey,//传递已完成I/O相关信息(绑定与套接字相关的信息)
DWORD NumberOfConcurrentThreads //ExistingCompletionPort非NULL时,此参数忽略
);
确认完成端口已完成的I/O和线程的I/O处理
#include <windows.h>
//可多个线程中调用,获取send或recv成功的数据
//WSASend、WSARecv完成后,系统会将可读写信息,写入CP队列中
BOOL GetQueuedCompletionStatus(
HANDLE CompletionPort,//注册有已完成I/O信息的CP对象句柄
LPDWORD lpNumberOfBytes,//保存I/O过程中传输数据大小
PULONG_PTR lpCompletionKey,//获取CreateIoCompletionPort函数中CompletionKey地址
LPOVERLAPPED *lpOverlapped, //获取WSASend、WSARecv函数中Overlapped(内部保存一次send、recv过程中的数据)
DWORD dwMilliseconds//超时后返回FALSE,INFINITE阻塞直到已完成I/O信息写入CP对象
);
GetQueuedCompletionStatus函数应该由处理IOCP中已完成I/O的线程调用。
IOCP将创建全职I/O线程,由该线程针对所有客户端进行I/O。应该由程序员自行创建调用WSASend、WSARecv等I/O函数的线程,只是该线程为了确认I/O的完成会调用GetQueuedCompletionStatus函数。
IOCP回声服务器端
23.IOCPEchoServ_win.c
#include <stdio.h>
#include <stdlib.h>
#include <process.h>
#include <winsock2.h>
#include <windows.h>
#define PORT 9999
#define BUF_SIZE 100
#define READ 3
#define WRITE 5
// socket info
typedef struct
{
SOCKET hClntSock; // 套接字句柄
SOCKADDR_IN clntAdr; // 客户端地址
} PER_HANDLE_DATA, *LPPER_HANDLE_DATA;
// buffer info
typedef struct
{
OVERLAPPED overlapped; // 用于异步I/O的结构体变量
WSABUF wsaBuf; // 指向缓冲区的指针,缓冲区的大小和缓冲区的首地址
char buffer[BUF_SIZE]; // 缓冲区
int rwMode; // READ or WRITE
} PER_IO_DATA, *LPPER_IO_DATA;
void ErrorHanding(const char *message)
{
fputs(message, stderr);
fputc('\n', stderr);
exit(1);
}
unsigned WINAPI EchoThreadMain(LPVOID pComPort)
{
while (1)
{
HANDLE hComPort = (HANDLE)pComPort;
DWORD bytesTrans;
LPPER_HANDLE_DATA handleInfo;
LPPER_IO_DATA ioInfo;
// 阻塞,直到I/O数据读写执行完后往下执行
// INFINITE,GetQueuedCompletionstatus函数在I/O完成且已注册相关信息时返回
// &ioInfo == ioInfo->overlapped,结构体变量地址 == 结构体首成员地址
GetQueuedCompletionStatus(hComPort, &bytesTrans, (PULONG_PTR)&handleInfo, (LPOVERLAPPED *)&ioInfo, INFINITE);
SOCKET socket = handleInfo->hClntSock;
if (ioInfo->rwMode == READ)
{
puts("message received!");
if (bytesTrans == 0)
{
puts("close connect!");
closesocket(socket);
free(handleInfo); // 客户端关闭时,释放空间
free(ioInfo);
continue;
}
// 将I/O读取的数据发送回客户端
// 修改发送长度和读写模型(利用了recv的变量地址)
memset(&(ioInfo->overlapped), 0, sizeof(OVERLAPPED));
ioInfo->wsaBuf.len = bytesTrans;
ioInfo->rwMode = WRITE;
WSASend(socket, &(ioInfo->wsaBuf), 1, NULL, 0, &(ioInfo->overlapped), NULL);
// 进行下一轮WSARecv和WSASend
// ioInfo必须malloc(区分send和recv),WSASend或WSARecv执行完后,都会调用GetQueuedCompletionStatus。
// 每一次send或recv后都会放入队列中,等待I/O完成后由GetQueuedCompletionStatus执行。
ioInfo = (LPPER_IO_DATA)malloc(sizeof(PER_IO_DATA));
memset(&(ioInfo->overlapped), 0, sizeof(OVERLAPPED));
ioInfo->wsaBuf.len = BUF_SIZE;
ioInfo->wsaBuf.buf = ioInfo->buffer;
ioInfo->rwMode = READ;
DWORD flags = 0;
WSARecv(socket, &(ioInfo->wsaBuf), 1, NULL, &flags, &(ioInfo->overlapped), NULL);
}
else
{
// WSASend发送成功后,释放内存
puts("message sent!");
free(ioInfo); // 释放
}
}
return 0;
}
int main(int argc, char *argv[])
{
WSADATA wsaData;
if (WSAStartup(MAKEWORD(2, 2), &wsaData) != 0)
ErrorHanding("WSAStartup() error!");
// 创建CP对象
HANDLE hComPort = CreateIoCompletionPort(INVALID_HANDLE_VALUE, NULL, 0, 0);
// 获得系统CPU核数
SYSTEM_INFO sysInfo;
GetSystemInfo(&sysInfo);
// 创建多个线程,且将CP对象句柄分配到线程
for (int i = 0; i < sysInfo.dwNumberOfProcessors; i++)
_beginthreadex(NULL, 0, EchoThreadMain, (LPVOID)hComPort, 0, NULL);
SOCKET hServSock = WSASocket(PF_INET, SOCK_STREAM, 0, NULL, 0, WSA_FLAG_OVERLAPPED);
if (hServSock == INVALID_SOCKET)
ErrorHanding("WSAocket() error!");
int opt = 1;
if (setsockopt(hServSock, SOL_SOCKET, SO_REUSEADDR, (const char *)&opt, sizeof(opt)) < 0)
ErrorHanding("setsockopt() error!");
int szAddr = sizeof(SOCKADDR_IN);
SOCKADDR_IN servAddr;
memset(&servAddr, 0, szAddr);
servAddr.sin_family = AF_INET;
servAddr.sin_addr.s_addr = htonl(INADDR_ANY);
servAddr.sin_port = htons(PORT);
if (bind(hServSock, (SOCKADDR *)&servAddr, szAddr) == SOCKET_ERROR)
ErrorHanding("bind() error!");
if (listen(hServSock, 5) == SOCKET_ERROR)
ErrorHanding("listen() error!");
while (1)
{
SOCKADDR_IN clntAddr;
SOCKET hClntSock = accept(hServSock, (SOCKADDR *)&clntAddr, &szAddr);
if (hClntSock == INVALID_SOCKET)
ErrorHanding("accept() error!");
puts("new connect!");
// handleInfo与套接字句柄绑定的指针
// 来新连接时创建,断开连接时释放
LPPER_HANDLE_DATA handleInfo = (LPPER_HANDLE_DATA)malloc(sizeof(PER_HANDLE_DATA));
handleInfo->hClntSock = hClntSock;
memcpy(&(handleInfo->clntAdr), &clntAddr, szAddr);
// 针对hClntSock套接字的重叠I/O完成时,已完成信息将写入连接的CP对象,这会引起GetQueue...函数的返回
// 关联客户端套接字,CP端口,句柄信息(区分不同客户端),I/O可读写时,GetQueue...会向下执行(类似条件变量)
CreateIoCompletionPort((HANDLE)hClntSock, hComPort, (ULONG_PTR)handleInfo, 0);
// 每次接收数据后,保存在ioInfo中
LPPER_IO_DATA ioInfo = (LPPER_IO_DATA)malloc(sizeof(PER_IO_DATA));
memset(&(ioInfo->overlapped), 0, sizeof(OVERLAPPED));
ioInfo->wsaBuf.len = BUF_SIZE;
ioInfo->wsaBuf.buf = ioInfo->buffer;
ioInfo->rwMode = READ; // IOCP不会区分输入完成和输出完成的状态(只通知完成I/O的状态),额外变量rwMode用于区分。
// 接收数据后,保存在ioInfo中
DWORD recvBytes;
DWORD flags = 0;
WSARecv(handleInfo->hClntSock, &(ioInfo->wsaBuf), 1, &recvBytes, &flags, &(ioInfo->overlapped), NULL);
}
closesocket(hServSock);
CloseHandle(hComPort);
WSACleanup();
return 0;
}
// gcc 23.IOCPEchoServ_win.c -o 23.IOCPEchoServ_win -lws2_32 && 23.IOCPEchoServ_win
IOCP特点
- 非阻塞模式的I/O,不会由I/O引发延迟。
- 查找已完成I/O时无需添加循环。
- 无需将作为I/O对象的套接字句柄保存到数组进行管理。
- 可以调整处理I/O的线程数,可实验后合理选用合适线程数。