本人不是专业的服务器程序员，所写的文章难免错漏，仅供参考，欢迎指正。
游戏中用到的网络编程多数是TCP连接，比如RPG类游戏都需要建立一个可靠的长连接来不停收发消息。有的游戏比如ACT类需要低延迟的游戏可能用的是P2P连接。还有一部分局域网对战类游戏会用到UDP广播来发送和同步局域网内的部分游戏主机信息。
1.封包处理
不管那种类型的网络连接首先要处理的都是消息的封包。
TCP消息的传输是可靠并且有序的，但是传输的过程中可能出现合并包和断包，因此消息的封包格式中必须包含一个字段用来保存包的长度。
一般windows上服务器可能是基于iocp，消息包设定了最大长度，大包是需要手动拆分的，比如发送商城物品列表时一次只发送一部分，并添加一个发送状态字段。
参见附录源码
PacketBase 是消息包的基类：
PacketBase::WriteHeader(); 发送时自动写入头部，消息包的前三个int字段分别是checksum&ID、size、debugID。
PacketBase::WriteValue(); 一系列函数用于向待发送缓冲写入数据
PacketBase::Read......(); 一系列函数用于读取包中的数据
PacketQueue 是带锁的消息包队列
发送消息的包分配在栈上，不会产生内存碎片无需内存池，接收消息的包需要使用内存池，这一点还未确定也没实现，留个坑不填了。
TcpStream 用于保存断包数据，等待接收到后续数据再拼接成完整包。关于断包处理类TcpStream一直有个疑惑，是一个接收线程对应一个TcpStream 还是一个socket对应一个TcpStream， 咨询了很多服务器开发人员，网上到处查资料也没有一个肯定的答案。纳闷有些小游戏服务器直接没有处理断包的代码也能玩。 个人倾向于一个socket对应一个TcpStream，但是并不用为每个连接的socket分配一个TcpStream，这样太浪费空间(因为断包可能出现在任意位置所以TcpStream要设置为最大封包长度)。 我们可以建立一个TcpStream的池子，某个socket需要的时候到池子中申请。暂时测试下来池子中TcpStream个数不会很大。
2.完成端口IOCP
做了一个带界面的服务器，很多服务器的程序都是不带界面的，甚至连黑框控制台都没有，可能是出于防止误操作考虑吧，效率上应该没啥影响。大约有几个要注意的地方
windows api向界面控制台打印输出时要注意必须在主线程打印，否则是无效的，所以添加了子线程打印函数LogSub。
玩家在登录完成之前发送的某些游戏内消息是直接丢弃的。
如果客户端卡死到达一段时间，则服务器不再给其发消息，否则会存在很多write overlapped，会占用很多发送缓冲，必要时要主动关闭该客户端，此时占用的write overlapped会被释放。
客户端关闭或断开连接时，服务器并不是总能收到通知，大约是四次挥手过程没有完成。有时客户端断开连接服务器会反复收到长度为0的消息，这时也需要主动关闭该客户端连接。可能心跳包检测是必须的。
测试服务器：凑合着先用，就一个进程，没做分布式 ，界面是win32api做的很简易。
封包代码：
//========================================================
// @Date: 2016.05
// @File: Include/Net/PacketList.h
// @Brief: PacketList
// @Author: LouLei
// @Email: twopointfive@163.com
// @Copyright (Crapell) - All Rights Reserved
//========================================================
#pragma once
#ifndef __PacketList__H__
#define __PacketList__H__
typedef void* PSocket;
#define Ptr2Socket(socket) (*((SOCKET*)socket))
#define Socket2Ptr(socket) (&socket)
void* CreateSocket();
void FreeSocket(void*& sokcet);
#include "General/Singleton.h"
#include "General/Thread.h"
//#include "General/ObjectPool.h"
//!减小缓冲使占用内存减少 1000player*2packet/per player = 2M级别
#define MaxPacketSize 1024*10
#define MaxValueLength 512
typedef short int Short;
enum PacketGrabRes
{
PR_ErrorStream, //
PR_ErrorChecksum, //
PR_Complete, //!包接收完成
PR_Incomplete, //!只收到一半的包流
PR_EmptyStream, //!空流
};
class OverlappedCustumRecv;
//!tcp是顺序接收数据，但可能出现合并包、分包的情况， 发送端发送A,B,C,D四个包,协议栈可能会发送A,BC,D,也就是把BC合成一个包发出去。
//!TCP协议栈在收到一个包的时候会同时计算下一个包的sequenceID,nextSequenceID = sequenceID + sizeof(currentPackage); 可以检测到包丢失
class PacketBase
{
friend class PacketQueue;
friend class NetClient;
friend class NetRobotsClient;
friend class NetRobot;
friend class NetServerIocp;
friend class NetPointUdp;
friend class PacketTemplateLib;
friend class InnerClient;
friend class InnerServer;
public:
PacketBase(int packetID);
virtual ~PacketBase();
virtual PacketBase* Alloc();
virtual void operator=(PacketBase* other){}
int GetPacketID();
void* GetParm();
int GetBufferSize();
char* GetBuffer();
virtual bool PreProcess();
virtual bool Process();
virtual void ToBuffer();
virtual void FromBuffer(void* syncGameInfo=NULL);
void SeekPos(int pos);
//!字段操作
void ReadHeader();
void ReadArray(void*value,int size);
template<class T>
void ReadValue(T& value);
void ReadString(char* value,int size);
template <int size>
void ReadString(char(&value)[size]);
void WriteHeader();
void WriteArray(const void* value,int size);
template<class T>
void WriteValue(const T& value);
void WriteString(const char* value);
//!准备发送：加校验 加密等
void PrepareSend();
//!直接复制转发
void CopyPacket(PacketBase* packet);
//!添加错误码m_res+复制转发
void CopyPacketRes(PacketBase* packet);
public:
//!buf前两个int分别是checksum&ID 和size，接收到包中会解除校验和
static PacketGrabRes GenPacketFromBuf(PacketBase*& packet, const char* buf, int &readSsize, int streamSize, PSocket fromSocket);
static void PushInTemplate(PacketBase* packet);
double GetAccumTime();
protected:
//!大包需要手动拆分
char m_buffer[MaxPacketSize];
int m_curPos;
//为了跨平台，不包含平台相关文件。
//如果使用外部socket指针，需要外部保证socket有效性。
//断开连接时，socket被先一步delete不好保证有效性（packet的处理要放在后面的主线程）。所以此处指向自己new出来的socket，自己负责delete。
PSocket m_fromSocket;
void* m_parm; //比如owner robot指针
//重叠字中可以保存 player指针，省去每个消息的map查找，提高效率
OverlappedCustumRecv* m_overlapedRecv;
public:
int m_packetID;
int m_streamSize;
//
int m_res; //错误码
static int HeadSize; //消息的前三个int分别是checksum&ID、size、debugID
#define DebugPacketID
//#ifdef DebugPacketID
unsigned int m_debugID;
static unsigned int DebugSendID;
static unsigned int DebugRecvID;
//#endif
static char PacketLogFlag[1024*10];
//static PacketBase* PacketTemplate[1024*10];
};
template<class T>
void PacketBase::WriteValue(const T& value)
{
int lenval=sizeof(T);
if(lenval>MaxValueLength)lenval=MaxValueLength;
memcpy(m_buffer+m_curPos,&value,lenval);
m_curPos+=lenval;
m_streamSize=m_curPos;
}
template<class T>
void PacketBase::ReadValue(T& value)
{
int lenval=sizeof(T);
if(lenval>MaxValueLength)lenval=MaxValueLength;
memcpy(&value,m_buffer+m_curPos,lenval);
m_curPos+=lenval;
}
template <int size>
void PacketBase::ReadString(char(&value)[size])
{
int lenstr=0;
ReadValue(lenstr);
if(lenstr>MaxValueLength)lenstr=MaxValueLength;
if(lenstr>=size)
{
memcpy(value, m_buffer+m_curPos,size-1);
value[size-1]='\0';
}
else
{
memcpy(value, m_buffer+m_curPos,lenstr);
value[lenstr]='\0';
}
m_curPos+=lenstr;
}
template<class T>
class PacketTemplateRegister
{
public:
PacketTemplateRegister()
{
//!确保全局变量构造顺序
PacketBase::PushInTemplate(new T);
}
void Space(){}
};
//
template<class T>
class PacketVisual:public PacketBase
{
public:
PacketVisual(int packetID):PacketBase(packetID){}
virtual PacketBase* Alloc(){return new T;}
virtual void operator=(PacketBase* other)
{
T* this_ = dynamic_cast<T*>(this);
T* other_ = dynamic_cast<T*>(other);
if (this_ && other_)
{
PSocket oldSocket = m_fromSocket;
//char* oldBuffer = m_buffer;
(*this_) = (*other_);
//深拷贝 恢复指针指向
Ptr2Socket(oldSocket) = Ptr2Socket(m_fromSocket);
m_fromSocket = oldSocket;
//memcpy(oldBuffer,m_buffer,m_bufferMax);
//m_buffer = oldBuffer;
}
}
//有的消息可能根据code写入不同的数据，或者list类消息数据不在结构体内，或者带有变长string、指针等，需要重写下面两个函数。
virtual void ToBuffer()
{
WriteHeader();
//copy packet T中的数据段，sizeof(T)-4-baseMember
WriteArray(this+sizeof(PacketBase),sizeof(T) - sizeof(PacketBase));
}
virtual void FromBuffer(void* syncGameInfo=NULL)
{
ReadHeader();
ReadArray(this+sizeof(PacketBase),sizeof(T) - sizeof(PacketBase));
}
protected:
//!利用虚函数必须被实现，使得静态成员被链接进来，
//!否则静态成员constructor不被链接,PacketTemplateRegister构造函数的断点都无法加
virtual void Space(){constructor.Space();}
static PacketTemplateRegister<T> constructor;
//todo
//send packet是栈数据 不会产生内存碎片 无需池子
//recv packet修改为 PacketT p; p.Parse(packetBase& b); 这样p为临时栈对象 b可以统一放在一个池子里 只需一把锁 （？p未知类型 无法定义临时变量？）
//!对象池, 一千消息就需要一千个池子，一千把锁??
//ObjectPool<T>
//PacketBase
//客户端发包限制
#ifdef CLIENT_APP
public:
static bool IsReady();
static bool m_waitingPacket;//某些消息在没收到反馈前不能多发
protected:
static double m_sendPeriod; //时间即时消息，一帧只能一次移动，捡取，技能等？ 其它消息0.1秒间隔。？ 列表1秒
static double m_nextSendTime;
#endif
};
template<class T>
PacketTemplateRegister<T> PacketVisual<T>::constructor;
#ifdef CLIENT_APP
template<class T> double PacketVisual<T>::m_waitingPacket = false;
template<class T> double PacketVisual<T>::m_sendPeriod = 0.033;
template<class T> double PacketVisual<T>::m_nextSendTime = 0;
template<class T> bool PacketVisual<T>::IsReady()
{
if (IsZero(m_sendPeriod)) return true;
if (m_sendPeriod < 0) return false;
double curTime = GetAccumTime();
if (curTime >= m_nextSendTime)
{
m_nextSendTime = curTime + m_sendPeriod;
return true;
}
}
#endif
class PacketQueue
{
friend class NetServerIocp;
public:
PacketQueue();
~PacketQueue();
int GetSize();
void DestroyList();
PacketBase* Pop();
bool Push(PacketBase *packet );
CriticalSection* GetCriticalSection();
static const char* classname(){return "PacketQueue";}
private:
CriticalSection m_criticalSection;
void* m_listPackets;
};
#if (defined SERVER_APP) || (defined INNERSERVER_APP)
#define ServerProcess virtual bool Process();
#else
#define ServerProcess
#endif
#ifdef CLIENT_APP
#define ClientProcess virtual bool Process();
#else
#define ClientProcess
#endif
//应用层实现
const char* ProtocolToString(int enumeration);
class TcpStream
{
public:
TcpStream();
~TcpStream();
int RecvStream( PSocket socket, char* buffer, int bufferSize, OverlappedCustumRecv* overlappedRecv);
PacketGrabRes GrabPacket(PacketBase*& packet);
//!（使用缓冲的时候也可能收到断包）
//!前面有MaxPacketSize-1大小的断包时加断点可能导致，又收到MaxPacketSize大小的流，所以缓冲大小MaxPacketSize*2。
char m_buffer[MaxPacketSize*2];
int m_incompleteStreamSize;
int streamSize;
char* curPos;
PSocket m_socket;
OverlappedCustumRecv* m_overlappedRecv;
static int ErrorPacketNum;
private:
};
#endif
//========================================================
// @Date: 2016.05
// @File: SourceLib/Net/PacketList.cpp
// @Brief: PacketList
// @Author: LouLei
// @Email: twopointfive@163.com
// @Copyright (Crapell) - All Rights Reserved
//========================================================
#include "General/Pch.h"
#ifdef WIN32APP
#include <winsock2.h>
#include <windows.h>
#include <windef.h>
typedef SOCKET HSocket;
#else
typedef unsigned int HSocket;
#endif
#include "General/AES.h"
#include "Net/PacketList.h"
#include <list>
#include <map>
#include "General/Timer.h"
#include "General/Pce.h"
void* CreateSocket()
{
HSocket* socket = new HSocket;
//Ptr2Socket(socket) = 7189;
return socket;
}
void FreeSocket(void*& socket_)
{
HSocket*socket = (HSocket*)socket_;
SafeDelete(socket);
socket_ = NULL;
}
//<PacketID,PacketBase*>
typedef std::map<int,PacketBase*> PACKETTEMPLATEMAP;
//全局对象m_mapPacketTemplate构造时间不能晚于第一次使用（PacketTemplateRegister构造调用PushInTemplate时使用）
//不定义为指针将难以控制两个全局变量的构造顺序
//static PACKETTEMPLATEMAP m_mapPacketTemplate = PACKETTEMPLATEMAP();
PACKETTEMPLATEMAP* m_mapPacketTemplate = NULL;//new PACKETTEMPLATEMAP();
//static PacketBase* PacketTemplate[1024*10];
char PacketBase::PacketLogFlag[1024*10];
class PacketTemplateGarbage
{
public:
~PacketTemplateGarbage()
{
for(PACKETTEMPLATEMAP::iterator it=m_mapPacketTemplate->begin();it!=m_mapPacketTemplate->end();++it)
{
delete (it->second);
}
m_mapPacketTemplate->clear();
delete(m_mapPacketTemplate);
m_mapPacketTemplate = NULL;
}
};
static PacketTemplateGarbage __PacketTemplateGarbage;
#ifdef DebugPacketID
int PacketBase::HeadSize = 12;
#else
int PacketBase::HeadSize = 8;
#endif
unsigned int PacketBase::DebugSendID = 0;
unsigned int PacketBase::DebugRecvID = 0;
PacketBase::PacketBase(int packetID)
:m_overlapedRecv(NULL)
{
m_packetID = packetID;
m_streamSize = 0;
m_curPos = 0;
m_res = 0;
m_parm = NULL;
m_debugID = 0;
m_buffer[0] = 0;
m_fromSocket = CreateSocket();
}
PacketBase::~PacketBase()
{
SafeDelete(m_fromSocket);
}
PacketGrabRes PacketBase::GenPacketFromBuf(PacketBase*& packet, const char* buf, int &readSsize, int streamSize, PSocket fromSocket)
{
//todo 缓冲池子
packet = NULL;
if (buf==NULL||streamSize<=0)
{
return PR_EmptyStream;
}
if (streamSize<HeadSize)
{
//不够头部 可能是正常断包
//return PR_ErrorStream;
return PR_Incomplete;
}
//消息的前三个int分别是checksum&ID、size、debugID
int *pIntBuffer = (int *)buf;
int size = pIntBuffer[1];
int packetID = (pIntBuffer[0]&0x0000ffff);
//
if ((*m_mapPacketTemplate).find(packetID)==(*m_mapPacketTemplate).end())
{
packet = NULL;
return PR_ErrorStream;
}
if (size <HeadSize/*= 0*/ || size > MaxPacketSize)
{
return PR_ErrorStream;
}
if (size > streamSize)
{
//等待流的重组
return PR_Incomplete;
}
//else if (size < streamSize)
//{可能是正常粘包
//}
//收到完整包后才能校验
short int checksum = (pIntBuffer[0]&0xffff0000)>>16;
pIntBuffer[0] &= 0x0000ffff; //校验前先把校验和清掉
bool bRight = false;
IsCheckSumRightFun(checksum,buf,size,bRight);
if (bRight==false)
{
packet = NULL;
return PR_ErrorChecksum;
}
{
packet = ((*m_mapPacketTemplate)[packetID])->Alloc();
//包含消息的前三个int checksum&ID、size、debugID
//packet->m_buffer = new char[size];
memcpy( packet->m_buffer, buf, size );
packet->m_streamSize = size;
packet->m_packetID = packetID;
//packet->m_fromSocket = fromSocket;
if(fromSocket)
{
Ptr2Socket(packet->m_fromSocket) = Ptr2Socket(fromSocket);
}
//若解析错误，则下一个packetID基本上是错误的
//挪到主线程,list线程不安全
//packet->FromBuffer(syncGameInfo);
readSsize = size;
}
return PR_Complete;
}
void PacketBase::PushInTemplate(PacketBase* packet)
{
if (!m_mapPacketTemplate)
{
m_mapPacketTemplate = new PACKETTEMPLATEMAP();
}
//map查找效率一万次1ms，10000人*10个消息=10ms消耗较大？ 用数组可以忽略？
PACKETTEMPLATEMAP& paks = (*m_mapPacketTemplate);
if (paks.find(packet->m_packetID) == paks.end())
{
paks[packet->m_packetID] = packet;//Clone();
}
else
{
MsgBoxFormat("There is another package that use the same ID！");
}
}
bool PacketBase::PreProcess()
{
return true;
}
bool PacketBase::Process()
{
return true;
}
void PacketBase::ToBuffer()
{
WriteHeader();
}
void PacketBase::FromBuffer(void* syncGameInfo)
{
ReadHeader();
}
char* PacketBase::GetBuffer()
{
return m_buffer;
}
int PacketBase::GetBufferSize()
{
return m_streamSize;
}
int PacketBase::GetPacketID()
{
return m_packetID;
}
PacketBase* PacketBase::Alloc()
{
return NULL;
}
void PacketBase::WriteString(const char* value)
{
int lenstr=strlen(value);
if(lenstr>MaxValueLength)
lenstr=MaxValueLength;
if(lenstr<0)
{
MsgBoxFormat("PacketWriteString len <0 string" );
return;
}
if (m_streamSize+sizeof(int)+lenstr>MaxPacketSize)
{
MsgBoxFormat("PacketWriteString buffsize overflow");
return;
}
WriteValue(lenstr);
WriteArray(value,lenstr);
m_streamSize=m_curPos;
}
void PacketBase::WriteArray(const void* value,int size)
{
int lenarr=size;
if(size<0)
{
MsgBoxFormat("PacketAddArray len <0 " );
return;
}
if (m_streamSize+sizeof(int)+lenarr>MaxPacketSize)
{
MsgBoxFormat("PacketAddArray buffsize overflow");
return;
}
/*if(lenarr>MaxValueLength)
{
MsgBoxFormat("PacketAddArray arr too big");
lenarr=MaxValueLength;
return;
}*/
memcpy(m_buffer+m_curPos,value, lenarr);
m_curPos+=lenarr;
m_streamSize=m_curPos;
}
//避免缓冲溢出
void PacketBase::ReadString(char*value,int size)
{
int lenstr=0;
ReadValue(lenstr);
if(lenstr>MaxValueLength || lenstr>MaxPacketSize-m_curPos)
{
MsgBoxFormat("PacketGetString too long string" );
return;
}
if(lenstr<0)
{
MsgBoxFormat("PacketGetString len <0 string" );
return;
}
if(lenstr>=size)
{
memcpy(value, m_buffer+m_curPos,size-1);
value[size-1]='\0';
}
else
{
memcpy(value, m_buffer+m_curPos,lenstr);
value[lenstr]='\0';
}
m_curPos+=lenstr;
}
void PacketBase::ReadArray(void*value,int size)
{
int lenarr=size;
if(lenarr<0)
{
MsgBoxFormat("PacketReadArray len <0 " );
return;
}
if(/*lenarr>MaxValueLength||*/ lenarr>MaxPacketSize-m_curPos)
{
MsgBoxFormat("PacketReadArray too long arr" );
lenarr=MaxValueLength;
return;
}
memcpy(value, m_buffer+m_curPos,lenarr);
m_curPos+=lenarr;
}
void PacketBase::WriteHeader()
{
//消息的前三个int分别是checksum&ID、size、debugID
int * head = (int *)this->m_buffer;
*head = m_packetID;
//*(int *)(m_buffer+sizeof(int)) = buffreSize;
m_curPos = HeadSize;
m_streamSize=m_curPos;
}
void PacketBase::ReadHeader()
{
//消息的前三个int分别是checksum&ID、size、debugID
int * head = (int *)this->m_buffer;
//*(int *)m_buffer = m_packetID;
//*(int *)(m_buffer+sizeof(int)) = buffreSize;
m_curPos = HeadSize;
#ifdef DebugPacketID
m_debugID = *(head+2);
#endif
}
void* PacketBase::GetParm()
{
return m_parm;
}
void PacketBase::CopyPacket(PacketBase* packet)
{
//sprintf(m_buffer,packet->m_buffer);
memcpy(m_buffer,packet->m_buffer,packet->m_streamSize);
//消息的前三个int分别是checksum&ID、size、debugID
*(int *)m_buffer = m_packetID;
m_streamSize = packet->m_streamSize;
m_curPos = m_streamSize;
}
void PacketBase::CopyPacketRes(PacketBase* packet)
{
int StreamSize = packet->m_streamSize+4;
//sprintf(m_buffer,packet->m_buffer);
memcpy(m_buffer+4,packet->m_buffer,packet->m_streamSize);
//消息的前三个int分别是checksum&ID、size、debugID
*(int *)m_buffer = m_packetID;
m_streamSize = StreamSize;
m_curPos = m_streamSize;
int* res = (int *)(m_buffer+HeadSize);//头部后第一个字段为m_res
*res = m_res;
}
void PacketBase::PrepareSend()
{
this->ToBuffer();
//消息的前三个int分别是checksum&ID、size、debugID
//统一填写长度
int * head = (int *)this->m_buffer;
//*head = m_packetID;
*(head+1) = this->m_streamSize;
m_debugID = DebugSendID++;
#ifdef DebugPacketID
*(head+2) = m_debugID;
#endif
short int checksum;
GetCheckSumFun(checksum,this->m_buffer,this->m_streamSize);
*(int *)(this->m_buffer) += (checksum<<16);
}
double PacketBase::GetAccumTime()
{
return G_Timer->GetAccumTime();
}
void PacketBase::SeekPos(int pos)
{
m_curPos = pos;
}
//==================^_^==================^_^==================^_^==================^_^
typedef std::list<PacketBase*> PacketsList;
PacketQueue::PacketQueue()
{
m_listPackets = new PacketsList;
}
PacketQueue::~PacketQueue()
{
DestroyList();
if (m_listPackets)
{
delete( (PacketsList*)m_listPackets );
m_listPackets = NULL;
}
}
void PacketQueue::DestroyList()
{
while (GetSize())
{
delete Pop();
}
}
PacketBase *PacketQueue::Pop()
{
if (((PacketsList*)m_listPackets)->empty())
return NULL;
PacketBase * temp = ((PacketsList*)m_listPackets)->back();
((PacketsList*)m_listPackets)->pop_back();
return temp;
}
bool PacketQueue::Push( PacketBase *packet )
{
((PacketsList*)m_listPackets)->push_front(packet);
return true;
}
int PacketQueue::GetSize()
{
return (int)((PacketsList*)m_listPackets)->size();
}
CriticalSection* PacketQueue::GetCriticalSection()
{
return &m_criticalSection;
}
int TcpStream::ErrorPacketNum = 0;
TcpStream::TcpStream()
:m_incompleteStreamSize(0)
,m_overlappedRecv(NULL)
{
}
TcpStream::~TcpStream()
{
}
int TcpStream::RecvStream(PSocket socket, char* buffer, int bufferSize, OverlappedCustumRecv* overlappedRecv)
{
if (buffer==NULL||bufferSize<=0)
{
return 0;
}
//默认没有缓存流
streamSize = bufferSize;
curPos = buffer;
//有缓存断包流
if(m_incompleteStreamSize > 0)
{
//这里基本走不到，除了大包或高并发？未经测试
//重定向
//新收到的拼接在缓存后面，已保证缓冲流前面不被取空
memcpy(m_buffer+m_incompleteStreamSize,curPos,streamSize);
streamSize += m_incompleteStreamSize;
curPos = m_buffer;
if (m_socket!=socket)
{
//assert(0);
}
}
else
{
//不拷贝，提高速度
}
m_socket = socket;
m_overlappedRecv = overlappedRecv;
return 1;
}
PacketGrabRes TcpStream::GrabPacket(PacketBase*& packet)
{
//网络层ip协议保证了包的完整性？如果收到某个socketA的半个包，中间不会插入其它socketB的包，直至该socketA剩下的半个包接收完毕，这样就阻塞了。
//网络层为了保证包的完整性，必然限制最大包（约1.5k），向下根据数据链路层的MTU（最大传输单元）还会分割（？传输层已经分割）。
//将大包（比如上传文件)在应用层拆成独立packet小包，每个包都带有packet包头。类似商城道具列表，收到一段即刻保存或转发一段。
//一般少量消息不拥堵缓冲时（即时发送且包少）不会分包，一个大消息超出缓冲直接发送失败？
if ( curPos==NULL || streamSize <= 0)
{
return PR_EmptyStream;
}
//粘包分包问题：可能合并了2.5个包，最后一个包被分拆
int readSize = 0;
PacketGrabRes res = PacketBase::GenPacketFromBuf(packet, curPos, readSize, streamSize,m_socket);
if (res==PR_Complete)
{
//拆出一个完整包
curPos += readSize;
streamSize -= readSize;
//仅最后一个包m_incompleteStreamSize=0有效，否则会被重置
m_incompleteStreamSize = 0;
//packet->m_overlapedRecv = m_overlapedRecv;//断开连接时，overlapped为野指针？
}
else if (res==PR_Incomplete)
{
//缓冲满 正常情况
//这里基本走不到，除了大包或高并发导致传输层缓存累积？因为网络层保证了一次包的完整性。
//当前流有剩余 ,将剩余流拷贝到buff，等待拼接
m_incompleteStreamSize = streamSize;
if(curPos!=m_buffer)
{
//1 m_buffer 头部被取空了一部分
//2 recvBuffer 头部被取空了一部分
memmove(m_buffer,curPos,streamSize);
}
else
{
//本来就是断包，且本次未取空头部
}
}
else if (res==PR_EmptyStream)
{
}
else if (res==PR_ErrorChecksum)
{
//校验错误
m_incompleteStreamSize = 0;
ErrorPacketNum++;
}
else if (res==PR_ErrorStream)
{
//包解析失败，可能导致后面所有包都解析出错。
//清空缓冲，等待到下一个正确包,直到下一段流没跨缓冲区，是正常包头，则接收恢复正常。
m_incompleteStreamSize = 0;
ErrorPacketNum++;
}
else
{
//assert(0);
}
return res;
}
IOCP代码：
//========================================================
// @Date: 2016.05
// @File: Include/Net/NetServerIocp.h
// @Brief: NetServerIocp
// @Author: LouLei
// @Email: twopointfive@163.com
// @Copyright (Crapell) - All Rights Reserved
//========================================================
#pragma once
#ifndef __NetServerIocp__H__
#define __NetServerIocp__H__
#include "General/singleton.h"
#include "Net/InnerServer.h"
//!服务器只做win32平台
#if (defined WIN32APP) && (defined SERVER_APP)
typedef void* PSocket;
#include "General/Thread.h"
class OverlappedCustum;
class OverlappedCustumRecv;
class PacketQueue;
class PacketBase;
class TcpStream;
#define G_Server NetServerIocp::GetSingleton()
typedef void (*OutPutFun)(const char *msg);
//!服务器放在单独的进程，同样可以把地址广播到局域网
//!即使放在主机客户端同一进程，收到消息也需主线程异步处理，已经延迟单帧时间，相比之下本机网络延迟可以忽略，或者可以通过进程间通信来解决？
//!高性能 在同一台计算机上，RakNet可以实现在两个程序之间每秒传输25，000条信息
//!联机方案
class NetServerIocp
{
public:
NetServerIocp();
~NetServerIocp();
bool InitServer(int serverPort);
bool Close();
//!子线程可调用，控制台可以立即输出，但ui无法立即刷新，需要主线程异步处理
void LogSub(const char* lpszFormat, ...);
void SetLogFun(OutPutFun fun);
virtual const char* GetPacketLogStr(PacketBase* packet);
void TerminateServer();
//!处理消息队列
void MainThreadProcessPacketPool();
int RecvOverlapped( OverlappedCustumRecv* overlapped );
int RecvStream( PSocket socket, char* buffer, int bufferSize ,OverlappedCustumRecv* overlapped);
int SendMsg( int packetID, PSocket socket, char* buffer, int bufferSize );
int SendPacketToSocket( PacketBase* packet,PSocket socket,bool prepareSend=true);
virtual void KickPlayer(PSocket socket);
int AcceptThread(void* threadParam );
int WorkerThread(void* threadParam);
void LogError( const char* msg );
void SetSpecialPacketID(int login,int disconnect);
static const char* classname(){return "NetServerIocp";}
static NetServerIocp* GetSingleton(){return m_this;};
static NetServerIocp* m_this;
protected:
bool m_bRunning;
HANDLE m_iocp;
PSocket m_listenSocket;
int m_port;
//消息队列
PacketQueue* m_packetQueue;
//线程
HANDLE m_acceptThread;
HANDLE m_workThread;
//登录断开协议id
int m_loginPacketID;
int m_disconnectPacketID;
//主线程可以直接调用logfun控制台输出，子线程不可以
OutPutFun m_logFun;
char m_logBuff[1024*10];
int m_logSize;
CriticalSection m_logCriticalSection;
//!不是每个工作线程配备一个tcpstream，而是每个连接的socket都可能要配备一个tcpstream
//有断包时保存在（overlapped）完成键中，共用TcpStream
//TcpStream* m_tcpStream;
};
#endif
#endif
//========================================================
// @Date: 2016.05
// @File: SourceLib/Net/NetServerIocp.cpp
// @Brief: NetServerIocp
// @Author: LouLei
// @Email: twopointfive@163.com
// @Copyright (Crapell) - All Rights Reserved
//========================================================
#include "General/Pch.h"
#include <list>
#include "General/AES.h"
//通过socket 获得player
//每帧 数据库批量插入 事务原子性
#if (defined WIN32APP) && (defined SERVER_APP)
#include <stdlib.h>
#include <winsock2.h> //<winsock2.h> 必须在<windows.h> 前面
#include <windows.h>
#include "Net/NetServerIocp.h"
#include "Net/PacketList.h"
#include "General/General.h"
#include "General/ObjectPool.h"
#include "General/Pce.h"
//#include "../SourceDemoSync/Packet/Protocol.h"
#pragma message(__FILE__": linking with Ws2_32.lib")
#pragma comment(lib, "Ws2_32.lib")
#define OP_READ 0
#define OP_WRITE 1
#define OP_ACCEPT 2
ObjectPool<TcpStream> G_TcpStreamParserPool(10);
// OVERLAPPED 扩展
class OverlappedCustum
{
public:
OVERLAPPED overlapped;
SOCKET socket;
int op;
char buf[MaxPacketSize];
DWORD buflen;
int zeroTrans;
};
class OverlappedCustumSend
{
public:
OVERLAPPED overlapped;
SOCKET socket;
int op;
char buf[MaxPacketSize];
DWORD buflen;
int zeroTrans;
int sendedLen;
};
class OverlappedCustumRecv
{
public:
OverlappedCustumRecv()
:tcpStream(NULL)
{}
~OverlappedCustumRecv()
{
if (tcpStream)
{
G_TcpStreamParserPool.DelObject(tcpStream);
}
}
OVERLAPPED overlapped;
SOCKET socket;
int op;
char buf[MaxPacketSize];
DWORD buflen;
int zeroTrans;
TcpStream* tcpStream;
//NetPlayer* player;
//char bLogin; //是否通过登录验证 可以通过player==NULL来判断? 未成功登录前只有有限的几个消息允许处理！！
};
ObjectPool<OverlappedCustumSend> G_OverlappedPool(10);
typedef std::list<PacketBase*> PacketsList;
DWORD WINAPI AcceptThread( void* threadParam )
{
return NetServerIocp::GetSingleton()->AcceptThread(threadParam);
}
DWORD WINAPI WorkerThread(void* threadParam)
{
return NetServerIocp::GetSingleton()->WorkerThread(threadParam);
}
NetServerIocp* NetServerIocp::m_this = NULL;
NetServerIocp::NetServerIocp()
:m_bRunning(false)
,m_packetQueue(NULL)
,m_logFun(NULL)
,m_loginPacketID(-1)
,m_disconnectPacketID(-1)
{
m_this = this;
m_logBuff[0]='\0';
m_logSize = 0;
//m_tcpStream = new TcpStream;
m_listenSocket = CreateSocket();
}
NetServerIocp::~NetServerIocp()
{
Close();
//SafeDelete(m_tcpStream);
FreeSocket(m_listenSocket);
}
bool NetServerIocp::Close()
{
if(m_bRunning == false)
return true;
m_bRunning = false;
closesocket(Ptr2Socket(m_listenSocket));//放在前面，否则accept阻塞acceptThread
WaitForSingleObject(m_acceptThread, INFINITE);
CloseHandle(m_acceptThread);
CloseHandle(m_iocp);//放在前面，否则GetQueuedCompletionStatus阻塞workThread
WaitForSingleObject(m_workThread, INFINITE);
CloseHandle(m_workThread);
SafeDelete(m_packetQueue);
return true;
}
bool NetServerIocp::InitServer(int serverPort)
{
UnitTestRegister::UnitTestAll();
m_port = serverPort;
WSADATA wsd;
// 初始化socket库， 保证ws2_32.dll已经加载
if (WSAStartup(MAKEWORD(2, 2), &wsd) != NO_ERROR)
return false;
//
SOCKET listenSocket;
listenSocket = socket(AF_INET, // IPv4
SOCK_STREAM, // 顺序的、可靠的、基于连接、双向的数据流通信
IPPROTO_IP // 使用TCP协议
);
if (listenSocket == INVALID_SOCKET)
{
WSACleanup();
return false;
}
Ptr2Socket(m_listenSocket) = listenSocket;
// bind 服务端的通信协议、IP地址、端口
SOCKADDR_IN local;
local.sin_addr.s_addr = htonl(INADDR_ANY);
local.sin_family = AF_INET;
local.sin_port = htons(m_port);
int ret = bind(listenSocket,(struct sockaddr *)&local,sizeof(local));
if(ret == SOCKET_ERROR)
{
TerminateServer();
return false;
}
// 创建完成端口
m_iocp = CreateIoCompletionPort(INVALID_HANDLE_VALUE,NULL,
(ULONG_PTR)0,//completion key 附加参数
0);
if ( !m_iocp )
{
TerminateServer();
return false;
}
//ZeroMemory( G_ServerPlayers, sizeof(GamePlayers) );
// 开始监听
if ( listen(listenSocket,SOMAXCONN) != 0 )
{
TerminateServer();
return false;
}
m_bRunning = true;
//监听线程
m_acceptThread = CreateThread( NULL, 0, ::AcceptThread, NULL, 0, NULL );
//工作线程的个数一般设置为processors *2+2
//工作线程
m_workThread = CreateThread( NULL, 0, ::WorkerThread, NULL, 0, NULL );
m_packetQueue = new PacketQueue;
LogSub("Server start success! address=%s;port=%d;","this",serverPort);
return true;
}
void NetServerIocp::LogSub(const char* lpszFormat, ...)
{
AutoLock lock(&m_logCriticalSection);
va_list args;
int nBuf;
char szBuffer[512];
va_start(args, lpszFormat);
nBuf = _vsnprintf_s(szBuffer, sizeof(szBuffer)*sizeof(TCHAR), lpszFormat, args);
va_end(args);
int size = strlen(szBuffer);
if (m_logSize+size<10200)
{
m_logSize += size;
m_logSize += 2;
strcat(m_logBuff,szBuffer);
strcat(m_logBuff,"\r\n");
}
}
void NetServerIocp::LogError(const char* msg )
{
int error = WSAGetLastError();
switch (error)
{
case WSA_IO_PENDING:
// LogSub("%s :WSA_IO_PENDING!",msg);
break;
case WSAENOBUFS:
LogSub("%s :No buffers!",msg);
break;
case WSANOTINITIALISED:
LogSub("%s :Need Startup()!",msg);
break;
case WSAEINVAL:
LogSub("%s :Need listen()!",msg);
break;
case WSAENOTSOCK:
LogSub("%s :Not a socket!",msg);
break;
default:;
LogSub("%s :unknow error!",msg);
}
}
void NetServerIocp::TerminateServer()
{
closesocket( Ptr2Socket(m_listenSocket) );
WSACleanup();
}
int NetServerIocp::RecvOverlapped( OverlappedCustumRecv* overlappedRecv )
{
DWORD flags = 0;
overlappedRecv->op = OP_READ;
overlappedRecv->buflen = 0;
//有些客户端在关闭时会连续不断触发GetQueuedCompletionStatus(返回true，BytesTransferred值为0，GetLastError()==WSA_IO_PENDING)。可能是采用了不正确的关闭方式
//如果不清除buf，则连续接收最后一条消息
ZeroMemory(&(overlappedRecv->buf),MaxPacketSize);
ZeroMemory(&(overlappedRecv->overlapped),sizeof(OVERLAPPED));
WSABUF wbuf;
wbuf.buf = overlappedRecv->buf;
wbuf.len = MaxPacketSize;
//传递一个OVERLAPPEDPLUS结构(WSASend、 WSARecv等函数)
//这个函数投递一个异步IO请求，非阻塞直接返回。系统完成时通过GetQueuedCompletionStatus通知，GetQueuedCompletionStatus是阻塞的。
//如果接收操作立即完成，overlappedRecv->bytes会返回函数调用所接收到的字节数? 否则为0
int ret = WSARecv((overlappedRecv->socket),
&wbuf,
1,
&(overlappedRecv->buflen),
&flags,
&(overlappedRecv->overlapped),
NULL);
if ( ret == SOCKET_ERROR )
{
int error = WSAGetLastError();
//注意nagle算法
//返回WSA_IO_PENDING，是因为TCP/IP层缓冲区中没有数据可取，系统将会锁定我们投递的WSARecv的buffer，直到TCP/IP层缓冲区中有新的数据到来。
LogError("recv error");
}
return ret;
}
int NetServerIocp::SendMsg( int packetID, PSocket socket, char* buffer, int bufferSize )
{
if (!m_bRunning)
{
return 0;
}
//LogSub("send %d" ,packetID);
if (buffer==NULL)
{
return 0;
}
int sendAccum = 0;
//非阻塞：发送缓冲满了 重复send是错误的? 异步非阻塞，导致高频发送，挪到发送成功腾出缓冲时发送剩余部分
//即使没发完 系统会自动后续发送，可以释放buf，buf会被系统锁定不会被重新new到，但是由于pool的原因如果释放到pool还是可能被改写，所以统一接到全部发送完成消息时再删除
//while (sendAccum<bufferSize)
{
//乱序问题?
//多线程使用异步通信方式向同一个接收端(socket)同时发送数据，会导致接收端接收的数据混乱
//线程1第一次发送：123456789，假设未发送完，只发送了123
//线程2第一次发送：abcdefgh，假设未发送完，只发送了abc
//线程1第二次发送：456789，发送完成
//线程2第二次发送：defgh，发送完成
//接收端最终接收的数据为：123abc456789defgh。
//主线程发消息，工作线程释放
OverlappedCustumSend* overlappedSend = G_OverlappedPool.NewObject();
//手动将packet包分为tcp包是安全的，因为客户端只连接了服务端一个socket
if (bufferSize>=MaxPacketSize)
{
LogError("send bufferSize>= MaxPacketSize");
int a = 0;
}
int toSend = min(MaxPacketSize,bufferSize/*-sendAccum*/);
overlappedSend->socket = Ptr2Socket(socket);
overlappedSend->op = OP_WRITE;
overlappedSend->sendedLen = 0;
DWORD bytes = 0;
ZeroMemory(&(overlappedSend->overlapped),sizeof(OVERLAPPED));
memcpy(overlappedSend->buf, buffer/*+sendAccum*/, toSend );
WSABUF wbuf;
wbuf.buf = overlappedSend->buf;
wbuf.len = toSend;
int ret = WSASend(Ptr2Socket(socket),
&wbuf,
1,
&bytes,
0,
&(overlappedSend->overlapped),
NULL );
overlappedSend->buflen = toSend;
//overlappedSend->buflen = bytes
if ( ret == SOCKET_ERROR )
{
int error = WSAGetLastError();
//返回WSA_IO_PENDING表示TCP/IP层缓冲区已满，
//系统将锁定用户的程序缓冲区到系统的非分页内存中。
//直到TCP/IP层缓冲区有空余的地方来接受拷贝我们的程序缓冲区数据才拷贝走，并将给IOCP一个完成消息。
if (error == WSAECONNABORTED)
{
//非pending错误时 是否会泄露？
//GetQueuedCompletionStatus能收到通知吗？
//closesocket(socket);
//delete overlappedSend;
//某客户端断开
LogError("send error:connect aborted");
return -999;
}
else
{
//
//若某客户端死循环或断点挂起，会运行到此，Overlapped不断创建且无法被释放。
//64次失败后断开客户端，允许客户端短暂调试？
//实测：断开连接后 pending的overlappedSend会被GetQueuedCompletionStatus接收到
//发送缓冲区占满时不会阻断其他socket也发送失败？每个tcp连接独立的发送缓冲区？ 只有在心跳包保活期间才发送消息 ？
//发送的内容比接收缓冲区大。系统完成通知会多次返回。?
//若干次发送失败后必须剔除
LogError("send error:io pending");
return -1;
}
}
else
{
sendAccum += bytes;
}
}
return sendAccum;
}
int NetServerIocp::RecvStream( PSocket socket, char* buffer, int bufferSize ,OverlappedCustumRecv* overlapped)
{
bool kick = false;
if (!m_bRunning)
{
return !kick;
}
if (buffer==NULL||bufferSize<=0)
{
//kick = true;
return !kick;
}
if (overlapped->tcpStream==NULL)
{
overlapped->tcpStream = G_TcpStreamParserPool.NewObject();
overlapped->tcpStream->m_incompleteStreamSize = 0;
}
TcpStream* tcpStream = overlapped->tcpStream;//m_tcpStream
tcpStream->RecvStream(socket,buffer,bufferSize,overlapped);
PacketBase* packet = NULL;
PacketGrabRes res = tcpStream->GrabPacket(packet);
while ( 1)
{
if (res==PR_Complete)
{
//拆出一个完整包 消息进栈前上锁
AutoLock lock(m_packetQueue->GetCriticalSection());
m_packetQueue->Push(packet);
//no break
res = tcpStream->GrabPacket(packet);
}
else if (res==PR_Incomplete)
{
//缓冲满 正常情况
LogSub("解析到正常断包!");
break;
}
else if (res==PR_EmptyStream)
{
//取完，没有断包，释放tcpStream
if (overlapped->tcpStream)
{
G_TcpStreamParserPool.DelObject(overlapped->tcpStream);
overlapped->tcpStream = NULL;
}
break;
}
else if (res==PR_ErrorChecksum)
{
//校验错误
//kick = true;
if (overlapped->tcpStream)
{
overlapped->tcpStream->m_incompleteStreamSize = 0;
G_TcpStreamParserPool.DelObject(overlapped->tcpStream);
overlapped->tcpStream = NULL;
}
LogSub("!!!!!!!!! 包校验错误! ");
break;
}
else if (res==PR_ErrorStream)
{
//流解析出错，可能导致后面所有包都解析出错。
//清空缓冲，等待到下一个正确包,直到下一段流没跨缓冲区，是正常包头，则接收恢复正常。
//kick = true;
if (overlapped->tcpStream)
{
overlapped->tcpStream->m_incompleteStreamSize = 0;
G_TcpStreamParserPool.DelObject(overlapped->tcpStream);
overlapped->tcpStream = NULL;
}
LogSub("!!!!!!!!! 包解析失败! ");
break;
}
}
return !kick;
}
int NetServerIocp::SendPacketToSocket( PacketBase* packet,PSocket socket ,bool prepareSend)
{
if(prepareSend)
{
packet->PrepareSend();
}
const char* txt = GetPacketLogStr(packet);
if(txt)
LogSub("send %s",txt);
return SendMsg( packet->m_packetID, socket,packet->m_buffer, packet->m_streamSize);
}
void NetServerIocp::KickPlayer( PSocket socket )
{
int size = sizeof(sockaddr_in);
sockaddr_in client;
getpeername(Ptr2Socket(socket),(sockaddr *)&client,&size);
LogSub("断开连接：[%s]， reason: kick player。\n",inet_ntoa(client.sin_addr));
//？
closesocket(Ptr2Socket(socket));
}
//侦听线程
int NetServerIocp::AcceptThread(void* threadParam )
{
sockaddr_in client;
int size;
SOCKET acceptSocket;
OverlappedCustumRecv *overlappedRecv;
while(m_bRunning)
{
size = sizeof(sockaddr_in);
//accept是阻塞函数，接收到连接时才返回，可以用select函数设置侦听的时间
acceptSocket = accept(Ptr2Socket(m_listenSocket),(sockaddr *)&client,&size);
if(acceptSocket != INVALID_SOCKET)
{
LogSub("连接建立： %s!", inet_ntoa(client.sin_addr));
//每一个玩家连接进来，则创建一个overlappedCustum
//注意以下设置acceptSocket 而非listenSocket
//阻塞方式的时候，会自动使用Nagle算法,Nagle虽然解决了小封包问题，提高了吞吐量，但导致了较高的不可预测的延迟
//1.发送端使用Nagle算法,TCP包头需要占用一定的字节数，所以发送这样的包时，大部分的传输开销花在包头上了；
//2.接收端使用clark算法或者delay ack。发送方Nagle算法的实质是拼接小数据包，如果当前数据不够一个MSS，则等待，直到有足够的数据能拼到一起或者新的ACK到来。接收方Clark算法是一有数据包就发送ACK，但通告窗口为0，直到有足够的接收缓存，再通告较大的窗口，delay ack是指不立即回复ACK，直到有多个数据包到来，或者有足够的缓冲区
//window上只有delay ack 没有clark？？ 接收端不会自动粘包？？
//禁止发送端粘包
const char chOpt=1;
int ret=setsockopt(acceptSocket,IPPROTO_TCP,TCP_NODELAY,&chOpt,sizeof(char));
if(ret==SOCKET_ERROR)
{
LogSub("setsockopt TCP_NODELAY error.");
}
//
//系统默认的状态发送和接收一次最大(约为8.5K)；在实际的过程中如果发送或是接收的数据量比较大，可以设置socket缓冲区，避免send(),recv()不断的循环收发：
// 接收缓冲区
int nRecvBuf = 64 * 1024; //设置为64K
setsockopt(acceptSocket, SOL_SOCKET, SO_RCVBUF, (const char*)&nRecvBuf, sizeof(int));
//发送缓冲区
int nSendBuf = 64*1024;
setsockopt(acceptSocket, SOL_SOCKET, SO_SNDBUF, (const char*)&nSendBuf, sizeof(int));
//设置后，若断开，则在使用该socket读写时立即失败，并返回ETIMEDOUT错误
int keepAlive = 1; // 开启keepalive属性
int keepIdle = 60; // 60秒内没有数据往来,则进行探测
int keepInterval = 5; // 探测时间间隔
int keepCount = 3; // 探测尝试的最大次数.
setsockopt(acceptSocket, SOL_SOCKET, SO_KEEPALIVE, (const char *)&keepAlive, sizeof(keepAlive));
//setsockopt(acceptSocket, SOL_TCP, TCP_KEEPIDLE, (void*)&keepIdle, sizeof(keepIdle));
//setsockopt(acceptSocket, SOL_TCP, TCP_KEEPINTVL, (void *)&keepInterval, sizeof(keepInterval));
//setsockopt(acceptSocket, SOL_TCP, TCP_KEEPCNT, (void *)&keepCount, sizeof(keepCount));
overlappedRecv = new OverlappedCustumRecv;
if ( overlappedRecv )
{
//套接字与完成端口的关联，在这个套接字上进行的任何重叠操作都将通过完成端口发出完成通知
CreateIoCompletionPort((HANDLE)acceptSocket, m_iocp, NULL, 0);
//保存套接字
overlappedRecv->socket = acceptSocket;
RecvOverlapped( overlappedRecv );
//{
// //验证客户端合法性
// S2CPacketVerificate packet;
// //atoi(packet.m_VerificateCode,m_fromSocket);
// packet.m_res = S2CPacketVerificate::Req_DoVerificate;
// G_Server->SendPacketToSocket(&packet,m_fromSocket);
//}
}
else
{
closesocket( acceptSocket );
LogSub("falied to new overlapped: %s!", inet_ntoa(client.sin_addr));
}
}
else
{ // accept error
LogError("accept error");
}
}
return 0;
}
/*
TCP/IP模型分为5层：应用层、传输层、网络层、数据链路层以及 物理层
应用层：http、ftp协议
传输层：TCP、UDP协议
tcp（Transmission Control Protocol）
面向连接（先要和对方确定连接、传输结束需要断开连接，类似打电话）、复杂可靠的、有很好的重传和查错机制。
TCP协议没有收到对方的确认包，会有超时重传，每个数据包是有序列号的，传输层根据序列号来保证A,B包的顺序，
即使B比A先到达了，TCP也会是等A到达之后，先把A提交给应用层
udp（user datagram protocol）
面向无连接（无需确认对方是否存在，类似寄包裹）、简单高效、没有重传机制。一般用于即时通讯、广播通信等
网络层：IP协议，负责对数据加上IP地址和其他的数据（后面会讲到）以确定传输的目标。网络层用来处理网络中流动的数据包，数据包为最小的传递单位.
保证包的完整性，若传输层发出的包过大，在网络层会被分包，同时在接收端的网络层会被组包，有一个完整的包才会交给传输层，若包不完整会丢弃
*/
//不断的查询 send receive 状态(完成端口发出的完成通知)
int NetServerIocp::WorkerThread(void* threadParam)
{
ULONG_PTR ckey;
OVERLAPPED *localOverlapped = NULL;
OverlappedCustum *overlappedCustum = NULL;
OverlappedCustumRecv *overlappedRecv = NULL;
OverlappedCustumSend *overlappedSend = NULL;
DWORD BytesTransferred;
//每个acceptsocket各需要一个用于接收的overlappedCustum
//所有acceptsocket同时只需要n个用于发送的overlappedCustum，n和可能发送的线程数有关
while(m_bRunning)
{
//阻塞函数
//ckey是关联完成端口与套接字时的完成键；
BOOL ret = GetQueuedCompletionStatus(m_iocp, &BytesTransferred,&ckey,&localOverlapped,INFINITE);
if (localOverlapped)
{
//从成员变量指针反推对象指针，此处因为overlapped放在开头转换后地址不变
overlappedCustum = CONTAINING_RECORD(localOverlapped, OverlappedCustum, overlapped);
switch (overlappedCustum->op)
{
case OP_READ :overlappedRecv = CONTAINING_RECORD(localOverlapped, OverlappedCustumRecv, overlapped);break;
case OP_WRITE:overlappedSend = CONTAINING_RECORD(localOverlapped, OverlappedCustumSend, overlapped);break;
}
}
else
{
overlappedCustum = NULL;
overlappedRecv = NULL;
overlappedSend = NULL;
}
//断开连接:返回ERROR_SUCCESS,并且lpNumberOfBytes等于0
if ( ret == FALSE)
{
//int error = WSAGetLastError();
//switch(error)
//{
// case ERROR_NETNAME_DELETED:
// break;
//}
if(overlappedCustum)
{
// 远端玩家断开连接
int size = sizeof(sockaddr_in);
sockaddr_in client;
getpeername(overlappedCustum->socket,(sockaddr *)&client,&size);
LogSub("断开连接：[%s]， reason: GetQueuedCompletionStatus()==false， op: %s", inet_ntoa(client.sin_addr),overlappedCustum->op==OP_READ?"read":"write");
//移除玩家
AutoLock lock(m_packetQueue->GetCriticalSection());
char buf[32];int *pIntBuffer=(int *)buf;*pIntBuffer++= m_disconnectPacketID;*pIntBuffer++ = PacketBase::HeadSize;*pIntBuffer++= PacketBase::DebugRecvID++;
int readSize = 0;
PacketBase* packet;
PacketGrabRes res = PacketBase::GenPacketFromBuf(packet, buf,readSize,PacketBase::HeadSize,&overlappedCustum->socket);
if (res==PR_Complete)
{
//packet->m_overlapedRecv = overlappedRecv;//一定是recv？，下面delete后成为野指针？
m_packetQueue->Push(packet);
}
//delete overlappedCustum;
switch (overlappedCustum->op)
{
case OP_READ :delete overlappedRecv;break;
case OP_WRITE:G_OverlappedPool.DelObject(overlappedSend);break;//m_sendFailNum>=3主动断开时，这里进来3次，保证了发送overlapped不会泄露
}
continue;
}
else
{
//服务器关闭
continue;
}
}
//有些客户端在关闭时会连续不断触发GetQueuedCompletionStatus(返回true，BytesTransferred值为0，GetLastError()==WSA_IO_PENDING)。可能是采用了不正确的关闭方式
//如果直接断开，有时候会出现误断,同一个IP如果连续3次，则视为断开
if (BytesTransferred<=0)
{
//发送或接收长度为0
//如果errno == EINTR 则是接收到信号后返回
overlappedCustum->zeroTrans++;
if (overlappedCustum->zeroTrans>3)
{
//远端玩家断开连接 ,移除玩家
// 消息进栈前上锁
AutoLock lock(m_packetQueue->GetCriticalSection());
char buf[32];int *pIntBuffer=(int *)buf;*pIntBuffer++= m_disconnectPacketID;*pIntBuffer++ = PacketBase::HeadSize;*pIntBuffer++= PacketBase::DebugRecvID++;
int readSize = 0;
PacketBase* packet;
PacketGrabRes res = PacketBase::GenPacketFromBuf(packet, buf,readSize,PacketBase::HeadSize,&overlappedCustum->socket);
if (res==PR_Complete)
{
//packet->m_overlapedRecv = overlappedRecv;//下面delete后成为野指针？
m_packetQueue->Push(packet);
}
int size = sizeof(sockaddr_in);
sockaddr_in client;
getpeername(overlappedCustum->socket,(sockaddr *)&client,&size);
LogSub("断开连接：[%s]， reason: zeroTrans>3, op: %s", inet_ntoa(client.sin_addr),overlappedCustum->op==OP_READ?"read":"write");
//删除重叠，不要再RecvMsg投递该端口的读取请求
//delete overlappedCustum;
switch (overlappedCustum->op)
{
case OP_READ :delete overlappedRecv;break;
case OP_WRITE:G_OverlappedPool.DelObject(overlappedSend);break;
}
continue;
}
}
else
{
overlappedCustum->zeroTrans = 0;
}
// 有重叠操作完成
switch (overlappedCustum->op)
{
//接收成功
case OP_READ:
{
//OverlappedCustumRecv* overlappedRecv = CONTAINING_RECORD(localOverlapped, OverlappedCustumRecv, overlapped);
//todo 完成键中保存 client 或 player 指针 传递到packet中，就无需再取一次 假如没秒1万消息 1万次map查找也要耗时数毫秒
//玩家没断开 overlapedMy不会调用delete，socket引用保持有效
int streamSize = BytesTransferred;//overlappedCustum->bytes;
int continueRecv = RecvStream(&overlappedRecv->socket,overlappedRecv->buf,streamSize,overlappedRecv);
//Log("RecvMsg: ", overlappedRecv->wbuf.buf+8);
if(continueRecv)
{
//继续接收下一条
RecvOverlapped( overlappedRecv );
}
else
{
//收到5个错误包，踢掉
//if(GetErrPacketNum(overlappedCustum->socket)>5)
//todo 放到主线程kick?
KickPlayer(&overlappedRecv->socket);
}
break;
}
//发送成功
case OP_WRITE:
{
//OverlappedCustumSend* overlappedSend = CONTAINING_RECORD(localOverlapped, OverlappedCustumSend, overlapped);
overlappedSend->sendedLen += BytesTransferred;
//若没发送完
if(overlappedSend->sendedLen < overlappedCustum->buflen)
{
//系统自动发送剩余部分
//如果连接异常，若干次发送长度为0后，也会删除，不会泄露？
int a = 0;
}
else
{
//每发送一条消息new一个,发完delete
G_OverlappedPool.DelObject(overlappedSend);
}
break;
}
default:
{
LogSub("error:recv overlappedCustum->op %d" ,overlappedCustum->op);
break;
}
}
}
return 0;
}
//!多个工作线程提高吞吐量，但是包都放到主线程池，主线程处理？
void NetServerIocp::MainThreadProcessPacketPool()
{
if (!m_bRunning || !m_packetQueue)
{
return;
}
if(m_logFun && m_logBuff[0])
{
AutoLock lock(&m_logCriticalSection);
m_logFun(m_logBuff);
m_logSize = 0;
m_logBuff[0]='\0';
}
PacketBase *packet;
int lastmsg = 0;
//上锁
int size = 0;
{
AutoLock lock(m_packetQueue->GetCriticalSection());
size = m_packetQueue->GetSize();
//其实无法完全避免，由于多线程的原因，包可能一次收不全，还在流里
//如果同时收到2个以上玩家断开连接，先标记，否则给断开的玩家发消息出错
PacketsList* packetList = (PacketsList*)m_packetQueue->m_listPackets;
for (PacketsList::iterator it=packetList->begin();it!=packetList->end();++it)
{
(*it)->PreProcess();
}
}
while (size>0)
{
{
AutoLock lock(m_packetQueue->GetCriticalSection());
packet = m_packetQueue->Pop();
size = m_packetQueue->GetSize();
}
//屏蔽大部分未登录发起的攻击
//if (m_loginPacketID==-1 && packet->m_packetID!=m_loginPacketID)
//{
// //因为应用层几乎每个消息都会GetPlayerBySocket，所以此处不做重复检查
// NetPlayer* player = m_netLobby->GetPlayerBySocket(packet->m_fromSocket);
// if (player==NULL)
// {
// kick(player);
// continue;
// }
//}
//buffer拷贝浪费的效率？
packet->FromBuffer();
packet->m_debugID = PacketBase::DebugRecvID++;
const char* txt = GetPacketLogStr(packet);
if(txt)
{
LogSub(" recv %s",txt);
}
packet->Process();
delete packet;
}
}
void NetServerIocp::SetLogFun(OutPutFun fun)
{
m_logFun = fun;
}
void NetServerIocp::SetSpecialPacketID(int login,int disconnect)
{
m_loginPacketID = login;
m_disconnectPacketID = disconnect;
}
const char* NetServerIocp::GetPacketLogStr(PacketBase* packet)
{
return "";
}
#endif
3726
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
