高性能大容量SOCKET并发（三）：接收、发送、缓存

接收

完成端口是结合重叠端口一起使用的，在接收数据之前，我们需要投递一个接收请求，使用function WSARecv(s: TSocket; lpBuffers: LPWSABUF; dwBufferCount: DWORD;  var lpNumberOfBytesRecvd, lpFlags: DWORD; lpOverlapped: LPWSAOVERLAPPED;  lpCompletionRoutine: LPWSAOVERLAPPED_COMPLETION_ROUTINE): Integer函数，接收连接后，就需要投递一个请求，代码如下：

[delphi] view plain copy print ?

1. procedure TSocketHandle.PreRecv(AIocpRecord: PIocpRecord);  
2. var  
3.   iFlags, iTransfer: Cardinal;  
4.   iErrCode: Integer;  
5. begin  
6.   if not Assigned(AIocpRecord) then  
7.   begin  
8.     New(AIocpRecord);  
9.     AIocpRecord.WsaBuf.buf := @FIocpRecvBuf;  
10.     AIocpRecord.WsaBuf.len := MAX_IOCPBUFSIZE;  
11.     FIocpRecv := AIocpRecord;  
12.   end;  
13.   AIocpRecord.Overlapped.Internal := 0;  
14.   AIocpRecord.Overlapped.InternalHigh := 0;  
15.   AIocpRecord.Overlapped.Offset := 0;  
16.   AIocpRecord.Overlapped.OffsetHigh := 0;  
17.   AIocpRecord.Overlapped.hEvent := 0;  
18.   AIocpRecord.IocpOperate := ioRead;  
19.   iFlags := 0;  
20.   if WSARecv(FSocket, @AIocpRecord.WsaBuf, 1, iTransfer, iFlags, @AIocpRecord.Overlapped,  
21.     nil) = SOCKET_ERROR then  
22.   begin  
23.     iErrCode := WSAGetLastError;  
24.     if iErrCode = WSAECONNRESET then //客户端被关闭   
25.       FConnected := False;  
26.     if iErrCode <> ERROR_IO_PENDING then //不抛出异常，触发异常事件   
27.     begin  
28.       FIocpServer.DoError('WSARecv', GetLastWsaErrorStr);  
29.       ProcessNetError(iErrCode);  
30.     end;  
31.   end;  
32. end;  

procedure TSocketHandle.PreRecv(AIocpRecord: PIocpRecord);
var
  iFlags, iTransfer: Cardinal;
  iErrCode: Integer;
begin
  if not Assigned(AIocpRecord) then
  begin
    New(AIocpRecord);
    AIocpRecord.WsaBuf.buf := @FIocpRecvBuf;
    AIocpRecord.WsaBuf.len := MAX_IOCPBUFSIZE;
    FIocpRecv := AIocpRecord;
  end;
  AIocpRecord.Overlapped.Internal := 0;
  AIocpRecord.Overlapped.InternalHigh := 0;
  AIocpRecord.Overlapped.Offset := 0;
  AIocpRecord.Overlapped.OffsetHigh := 0;
  AIocpRecord.Overlapped.hEvent := 0;
  AIocpRecord.IocpOperate := ioRead;
  iFlags := 0;
  if WSARecv(FSocket, @AIocpRecord.WsaBuf, 1, iTransfer, iFlags, @AIocpRecord.Overlapped,
    nil) = SOCKET_ERROR then
  begin
    iErrCode := WSAGetLastError;
    if iErrCode = WSAECONNRESET then //客户端被关闭
      FConnected := False;
    if iErrCode <> ERROR_IO_PENDING then //不抛出异常，触发异常事件
    begin
      FIocpServer.DoError('WSARecv', GetLastWsaErrorStr);
      ProcessNetError(iErrCode);
    end;
  end;
end;

WSARecv会返回错误，可以帮助我们定位网络问题，如连接断了，具体函数代码如下：

[delphi] view plain copy print ?

1. procedure TSocketHandle.ProcessNetError(const AErrorCode: Integer);  
2. begin  
3.   if AErrorCode = WSAECONNRESET then //客户端断开连接   
4.     FConnected := False  
5.   else if AErrorCode = WSAEDISCON then //客户端断开连接   
6.     FConnected := False  
7.   else if AErrorCode = WSAENETDOWN then //网络异常   
8.     FConnected := False  
9.   else if AErrorCode = WSAENETRESET then //心跳包拦截到的异常   
10.     FConnected := False  
11.   else if AErrorCode = WSAESHUTDOWN then //连接被关闭   
12.     FConnected := False  
13.   else if AErrorCode = WSAETIMEDOUT then //连接断掉或者网络异常   
14.     FConnected := False;  
15. end;  

procedure TSocketHandle.ProcessNetError(const AErrorCode: Integer);
begin
  if AErrorCode = WSAECONNRESET then //客户端断开连接
    FConnected := False
  else if AErrorCode = WSAEDISCON then //客户端断开连接
    FConnected := False
  else if AErrorCode = WSAENETDOWN then //网络异常
    FConnected := False
  else if AErrorCode = WSAENETRESET then //心跳包拦截到的异常
    FConnected := False
  else if AErrorCode = WSAESHUTDOWN then //连接被关闭
    FConnected := False
  else if AErrorCode = WSAETIMEDOUT then //连接断掉或者网络异常
    FConnected := False;
end;

WSARecv函数还有另外一个作用，如果客户端断开连接了，WSARecv会收到一个0字节的返回，我们可以根据这个释放连接对象。如果客户端直接拔网线，WSARecv是检测不到，这是我们要根据超时来检测。

接收连接之后我们投递一次接收数据请求，然后收到数据处理后，再投递下一次请求，因而我们相当于每次只有一个接收请求，我们只需要定义一个接收结构体和一个固定大小的缓存就可以了，类似：

[delphi] view plain copy print ?

1. {* 保存投递请求的地址信息 *}  
2.     FIocpRecv: PIocpRecord;  
3.     FIocpRecvBuf: array[0..MAX_IOCPBUFSIZE-1] of Char;  

{* 保存投递请求的地址信息 *}
    FIocpRecv: PIocpRecord;
    FIocpRecvBuf: array[0..MAX_IOCPBUFSIZE-1] of Char;

接收的数据我们一起放在一个内存流中，主要代码如下：

[delphi] view plain copy print ?

1. procedure TSocketHandle.ProcessIOComplete(AIocpRecord: PIocpRecord;  
2.   const ACount: Cardinal);  
3. begin  
4.   case AIocpRecord.IocpOperate of  
5.     ioNone: Exit;  
6.     ioRead: //收到数据   
7.     begin  
8.       FActiveTime := Now;  
9.       ReceiveData(AIocpRecord.WsaBuf.buf, ACount);  
10.       if FConnected then  
11.         PreRecv(AIocpRecord); //投递请求   
12.     end;  
13.     ioWrite: //发送数据完成，需要释放AIocpRecord的指针   
14.     begin  
15.       FActiveTime := Now;  
16.       FSendOverlapped.Release(AIocpRecord);  
17.     end;  
18.     ioStream:  
19.     begin  
20.       FActiveTime := Now;  
21.       FSendOverlapped.Release(AIocpRecord);  
22.       WriteStream; //继续发送流   
23.     end;  
24.   end;  
25. end;  
26.   
27. procedure TSocketHandle.ReceiveData(AData: PAnsiChar; const ALen: Cardinal);  
28. begin  
29.   FInputBuf.Write(AData^, ALen);  
30.   Process;  
31. end;  

procedure TSocketHandle.ProcessIOComplete(AIocpRecord: PIocpRecord;
  const ACount: Cardinal);
begin
  case AIocpRecord.IocpOperate of
    ioNone: Exit;
    ioRead: //收到数据
    begin
      FActiveTime := Now;
      ReceiveData(AIocpRecord.WsaBuf.buf, ACount);
      if FConnected then
        PreRecv(AIocpRecord); //投递请求
    end;
    ioWrite: //发送数据完成，需要释放AIocpRecord的指针
    begin
      FActiveTime := Now;
      FSendOverlapped.Release(AIocpRecord);
    end;
    ioStream:
    begin
      FActiveTime := Now;
      FSendOverlapped.Release(AIocpRecord);
      WriteStream; //继续发送流
    end;
  end;
end;

procedure TSocketHandle.ReceiveData(AData: PAnsiChar; const ALen: Cardinal);
begin
  FInputBuf.Write(AData^, ALen);
  Process;
end;

有很多完成端口为了提高效率，会对接收队列使用内存池，以减少内存的分配和释放次数，使用内存流会有频繁的申请和释放操作，也容易造成碎片，我们这里没有使用内存池，而是使用FastMM来加快速度。FastMM在多线程和内存碎片方面做的比DELPHI自带的内存管理器要强，一般能提高2到3倍的性能。

发送

发送和接收是类似的，使用function WSASend(s: TSocket; lpBuffers: LPWSABUF; dwBufferCount: DWORD;  var lpNumberOfBytesSent: DWORD; dwFlags: DWORD; lpOverlapped: LPWSAOVERLAPPED;  lpCompletionRoutine: LPWSAOVERLAPPED_COMPLETION_ROUTINE): Integer函数，如发送一个Buffer的示例代码如下：

[delphi] view plain copy print ?

1. procedure TSocketHandle.WriteBuffer(const ABuffer; const ACount: Integer;  
2.   const AIocpOperate: TIocpOperate);  
3. var  
4.   IocpRec: PIocpRecord;  
5.   iErrCode: Integer;  
6.   dSend, dFlag: DWORD;  
7. begin  
8.   IocpRec := FSendOverlapped.Allocate(ACount);  
9.   FillChar(IocpRec.Overlapped, SizeOf(IocpRec.Overlapped), 0);  
10.   IocpRec.IocpOperate := AIocpOperate;  
11.   System.Move(ABuffer, IocpRec.WsaBuf.buf^, ACount);  
12.   dFlag := 0;  
13.   if WSASend(FSocket, @IocpRec.WsaBuf, 1, dSend, dFlag, @IocpRec.Overlapped, nil) = SOCKET_ERROR then  
14.   begin  
15.     iErrCode := WSAGetLastError;  
16.     if iErrCode <> ERROR_IO_PENDING then  
17.     begin  
18.       FIocpServer.DoError('WSASend', GetLastWsaErrorStr);  
19.       ProcessNetError(iErrCode);  
20.     end;  
21.   end;    
22. end;  

procedure TSocketHandle.WriteBuffer(const ABuffer; const ACount: Integer;
  const AIocpOperate: TIocpOperate);
var
  IocpRec: PIocpRecord;
  iErrCode: Integer;
  dSend, dFlag: DWORD;
begin
  IocpRec := FSendOverlapped.Allocate(ACount);
  FillChar(IocpRec.Overlapped, SizeOf(IocpRec.Overlapped), 0);
  IocpRec.IocpOperate := AIocpOperate;
  System.Move(ABuffer, IocpRec.WsaBuf.buf^, ACount);
  dFlag := 0;
  if WSASend(FSocket, @IocpRec.WsaBuf, 1, dSend, dFlag, @IocpRec.Overlapped, nil) = SOCKET_ERROR then
  begin
    iErrCode := WSAGetLastError;
    if iErrCode <> ERROR_IO_PENDING then
    begin
      FIocpServer.DoError('WSASend', GetLastWsaErrorStr);
      ProcessNetError(iErrCode);
    end;
  end;  
end;

检测网络连接类似接收逻辑。

发送时需要先申请一个缓存，发送完成后释放这个缓存，因而在WriteBuffer时需要先用FSendOverlapped申请一个缓存，然后用WSASend投递给完成端口，完成端口执行完成后，需要释放这块缓存，代码如下：

[delphi] view plain copy print ?

1. procedure TSocketHandle.ProcessIOComplete(AIocpRecord: PIocpRecord;  
2.   const ACount: Cardinal);  
3. begin  
4.   case AIocpRecord.IocpOperate of  
5.     ioNone: Exit;  
6.     ioRead: //收到数据   
7.     begin  
8.       FActiveTime := Now;  
9.       ReceiveData(AIocpRecord.WsaBuf.buf, ACount);  
10.       if FConnected then  
11.         PreRecv(AIocpRecord); //投递请求   
12.     end;  
13.     ioWrite: //发送数据完成，需要释放AIocpRecord的指针   
14.     begin  
15.       FActiveTime := Now;  
16.       FSendOverlapped.Release(AIocpRecord);  
17.     end;  
18.     ioStream:  
19.     begin  
20.       FActiveTime := Now;  
21.       FSendOverlapped.Release(AIocpRecord);  
22.       WriteStream; //继续发送流   
23.     end;  
24.   end;  
25. end;  

procedure TSocketHandle.ProcessIOComplete(AIocpRecord: PIocpRecord;
  const ACount: Cardinal);
begin
  case AIocpRecord.IocpOperate of
    ioNone: Exit;
    ioRead: //收到数据
    begin
      FActiveTime := Now;
      ReceiveData(AIocpRecord.WsaBuf.buf, ACount);
      if FConnected then
        PreRecv(AIocpRecord); //投递请求
    end;
    ioWrite: //发送数据完成，需要释放AIocpRecord的指针
    begin
      FActiveTime := Now;
      FSendOverlapped.Release(AIocpRecord);
    end;
    ioStream:
    begin
      FActiveTime := Now;
      FSendOverlapped.Release(AIocpRecord);
      WriteStream; //继续发送流
    end;
  end;
end;

在这儿如果要加快速度和减少内存碎片，也可以使用内存池等技术。

FSendOverlapped是一个列表管理，代码如下：

[delphi] view plain copy print ?

1. {* 发送所申请的重叠端口缓冲 *}  
2. TSendOverlapped = class  
3. private  
4.   FList: TList;  
5.   FLock: TCriticalSection;  
6.   procedure Clear;  
7. public  
8.   constructor Create;  
9.   destructor Destroy; override;  
10.   {* 申请一个 *}  
11.   function Allocate(ALen: Cardinal): PIocpRecord;  
12.   {* 释放一个 *}  
13.   procedure Release(AValue: PIocpRecord);  
14. end;  

  {* 发送所申请的重叠端口缓冲 *}
  TSendOverlapped = class
  private
    FList: TList;
    FLock: TCriticalSection;
    procedure Clear;
  public
    constructor Create;
    destructor Destroy; override;
    {* 申请一个 *}
    function Allocate(ALen: Cardinal): PIocpRecord;
    {* 释放一个 *}
    procedure Release(AValue: PIocpRecord);
  end;

[delphi] view plain copy print ?

1. { TSendOverlapped }  
2.   
3. procedure TSendOverlapped.Clear;  
4. var  
5.   i: Integer;  
6. begin  
7.   FLock.Enter;  
8.   try  
9.     for i := 0 to FList.Count - 1 do  
10.     begin  
11.       FreeMemory(PIocpRecord(FList.Items[i]).WsaBuf.buf);  
12.       PIocpRecord(FList.Items[i]).WsaBuf.buf := nil;  
13.       Dispose(PIocpRecord(FList.Items[i]));  
14.     end;  
15.     FList.Clear;  
16.   finally  
17.     FLock.Leave;  
18.   end;  
19. end;  
20.   
21. constructor TSendOverlapped.Create;  
22. begin  
23.   inherited Create;  
24.   FList := TList.Create;  
25.   FLock := TCriticalSection.Create;  
26. end;  
27.   
28. destructor TSendOverlapped.Destroy;  
29. begin  
30.   Clear;  
31.   FList.Free;  
32.   FLock.Free;  
33.   inherited;  
34. end;  
35.   
36. function TSendOverlapped.Allocate(ALen: Cardinal): PIocpRecord;  
37. begin  
38.   FLock.Enter;  
39.   try  
40.     New(Result);  
41.     Result.Overlapped.Internal := 0;  
42.     Result.Overlapped.InternalHigh := 0;  
43.     Result.Overlapped.Offset := 0;  
44.     Result.Overlapped.OffsetHigh := 0;  
45.     Result.Overlapped.hEvent := 0;  
46.     Result.IocpOperate := ioNone;  
47.     Result.WsaBuf.buf := GetMemory(ALen);  
48.     Result.WsaBuf.len := ALen;  
49.     FList.Add(Result);  
50.   finally  
51.     FLock.Leave;  
52.   end;  
53. end;  
54.   
55. procedure TSendOverlapped.Release(AValue: PIocpRecord);  
56. var  
57.   i: Integer;  
58. begin  
59.   FLock.Enter;  
60.   try  
61.     for i := 0 to FList.Count - 1 do  
62.     begin  
63.       if Cardinal(AValue) = Cardinal(FList[i]) then  
64.       begin  
65.         FreeMemory(PIocpRecord(FList.Items[i]).WsaBuf.buf);  
66.         PIocpRecord(FList.Items[i]).WsaBuf.buf := nil;  
67.         Dispose(PIocpRecord(FList.Items[i]));  
68.         FList.Delete(i);  
69.         Break;  
70.       end;  
71.     end;  
72.   finally  
73.     FLock.Leave;  
74.   end;  
75. end;  

{ TSendOverlapped }

procedure TSendOverlapped.Clear;
var
  i: Integer;
begin
  FLock.Enter;
  try
    for i := 0 to FList.Count - 1 do
    begin
      FreeMemory(PIocpRecord(FList.Items[i]).WsaBuf.buf);
      PIocpRecord(FList.Items[i]).WsaBuf.buf := nil;
      Dispose(PIocpRecord(FList.Items[i]));
    end;
    FList.Clear;
  finally
    FLock.Leave;
  end;
end;

constructor TSendOverlapped.Create;
begin
  inherited Create;
  FList := TList.Create;
  FLock := TCriticalSection.Create;
end;

destructor TSendOverlapped.Destroy;
begin
  Clear;
  FList.Free;
  FLock.Free;
  inherited;
end;

function TSendOverlapped.Allocate(ALen: Cardinal): PIocpRecord;
begin
  FLock.Enter;
  try
    New(Result);
    Result.Overlapped.Internal := 0;
    Result.Overlapped.InternalHigh := 0;
    Result.Overlapped.Offset := 0;
    Result.Overlapped.OffsetHigh := 0;
    Result.Overlapped.hEvent := 0;
    Result.IocpOperate := ioNone;
    Result.WsaBuf.buf := GetMemory(ALen);
    Result.WsaBuf.len := ALen;
    FList.Add(Result);
  finally
    FLock.Leave;
  end;
end;

procedure TSendOverlapped.Release(AValue: PIocpRecord);
var
  i: Integer;
begin
  FLock.Enter;
  try
    for i := 0 to FList.Count - 1 do
    begin
      if Cardinal(AValue) = Cardinal(FList[i]) then
      begin
        FreeMemory(PIocpRecord(FList.Items[i]).WsaBuf.buf);
        PIocpRecord(FList.Items[i]).WsaBuf.buf := nil;
        Dispose(PIocpRecord(FList.Items[i]));
        FList.Delete(i);
        Break;
      end;
    end;
  finally
    FLock.Leave;
  end;
end;

TSendOverlapped由于只给一个TSocketHandle服务，因而可以不加锁，因为对每个TSocketHandle来说，它是串行的，不会并行。

更详细代码见示例代码的IOCPSocket单元。

下载地址：http://download.csdn.net/detail/sqldebug_fan/4510076