这个文档描述了一个本人在 15 年前的后端并发服务器的实现,采用 C 语言开发,以 epoll + 动态线程池的方式受理客户端的请求。现在的生态环境已经非常成熟,各种第三方和开源的组件都提供了完美的解决方案,不再需要开发人员自己造轮子了。
- 首先是头文件中的定义。
#define SUCCESS 0
#define FAILURE 1
#define UINT unsigned int
#define UCHAR unsigned char
#define ULONG unsigned long
#define USHORT unsigned short
#define ULONGLONG unsigned long long
#define LOW_WEIGHT 0.2 // 低于该值销毁部分线程
#define TOP_WEIGHT 0.6 // 高于该值创建部分线程
#define DELAY_TIME 50 // 创建线程延时
#define SLEEP_TIME 10 // 管理线程池线程休眠延时
#define PACKET_SIZE 1024 // 报文大小
#define LISTENQ 20 // 监听队列
#define EVENT_MAX 256 // epoll_wait返回活动套接字数组大小
#define CLIENT_MAX 10000 // 并发最大值
#define SERVER_PORT 12345 // 主套接字监听端口
#define IDLE_STATUS 0 // 线程空闲
#define BUSY_STATUS 1 // 线程忙碌
- 接下来是线程池和线程属性定义。
/* 工作线程属性 */
typedef struct tagWorkThread
{
pthread_t m_iWorkThreadID;
UINT m_uiStatus;
pthread_cond_t m_condWork;
pthread_mutex_t m_mutexWork;
int m_iClientSocket;
}WORKTHREAD_S;
#define NEW_LEVEL 10 // 每次新创建线程数
#define MIN_LEVEL 10 // 线程池内线程数最小值
#define MAX_LEVEL 100 // 线程池内线程数最大值
/* 线程池属性 */
typedef struct tagThreadPool
{
int m_iEpoll;
UINT m_uiMinimum;
UINT m_uiMaximal;
UINT m_uiCurrent;
pthread_t m_iManegeThreadID;
pthread_mutex_t m_mutexPool;
pthread_mutex_t m_mutexManage;
WORKTHREAD_S *m_pstWork;
}THREADPOOL_S;
/* 全局变量用于保存析构数据地址 */
typedef struct tagGeneralParameter
{
int *m_piSocket;
THREADPOOL_S *m_pstThreadPool;
}GENERALPARAMETER_S;
- 然后是一些必要的初始化
/********************************************************/
/* 日期:2009年10月15日 */
/* 函数:变更文件描述符的内核限制 */
/* 参数:无 */
/********************************************************/
int InitLimit()
{
struct rlimit stLimit;
stLimit.rlim_max = stLimit.rlim_cur = CLIENT_MAX;
if( setrlimit(RLIMIT_NOFILE,&stLimit) == -1 ) {
fprintf(stderr,"Failed to setrlimit,%s!\n",strerror(errno));
return FAILURE;
}
return SUCCESS;
}
/********************************************************/
/* 日期:2009年10月15日 */
/* 函数:初始化epoll相关资源 */
/* 参数:无 */
/********************************************************/
int InitEpoll()
{
int iReturn = -1;
iReturn = epoll_create(CLIENT_MAX);
if( iReturn < 0 ) {
fprintf(stderr,"Failed to epoll_create,%s!\n",strerror(errno));
return iReturn;
}
return iReturn;
}
/********************************************************/
/* 日期:2009年10月15日 */
/* 函数:初始化主监听套接字 */
/* 参数:piSocket主监听套接字;pstServer服务器信息 */
/* iEpoll套接字队列 */
/********************************************************/
int InitSocket(int *piSocket,struct sockaddr_in *pstServer,int iEpoll)
{
int iReturn = 0;
struct epoll_event stEpoll;
*piSocket = socket(AF_INET,SOCK_STREAM,0);
if( *piSocket < 0 ) {
fprintf(stderr,"Failed to socket,%s!\n",strerror(errno));
return FAILURE;
}
memset(pstServer,'\0',sizeof(struct sockaddr_in));
pstServer->sin_family = AF_INET;
pstServer->sin_addr.s_addr = htonl(INADDR_ANY);
pstServer->sin_port = htons(SERVER_PORT);
iReturn = bind(*piSocket,(struct sockaddr *)pstServer,sizeof(struct sockaddr_in));
if( iReturn < 0 ) {
fprintf(stderr,"Failed to bind,%s!\n",strerror(errno));
return FAILURE;
}
iReturn = listen(*piSocket,LISTENQ);
if( iReturn < 0 ) {
fprintf(stderr,"Failed to listen,%s!\n",strerror(errno));
return FAILURE;
}
/* 设置套接字为非阻塞模式 */
if( fcntl(*piSocket,F_SETFL,fcntl(*piSocket,F_GETFD,0) | O_NONBLOCK) == -1 ) {
fprintf(stderr,"Failed to fcntl,%s!\n",strerror(errno));
return FAILURE;
}
stEpoll.data.fd = *piSocket;
stEpoll.events = EPOLLIN | EPOLLET;
if( epoll_ctl(iEpoll,EPOLL_CTL_ADD,*piSocket,&stEpoll) < 0 ) {
fprintf(stderr,"Failed to epoll_ctl,%s\n",strerror(errno));
return FAILURE;
}
return SUCCESS;
}
- 再有一些必要的通用调用
/********************************************************/
/* 日期:2009年10月15日 */
/* 函数:信号处理函数 */
/* 参数:信号值 */
/********************************************************/
void SignalQuit(int iSignal)
{
ReleaseResource();
exit(SUCCESS);
}
/********************************************************/
/* 日期:2009年10月15日 */
/* 函数:析构函数 */
/* 参数:无 */
/********************************************************/
void ReleaseResource()
{
int iCount = 0;
int iCurrent = 0;
pthread_mutex_lock(&g_stGeneralParameter.m_pstThreadPool->m_mutexPool);
iCurrent = g_stGeneralParameter.m_pstThreadPool->m_uiCurrent;
pthread_mutex_unlock(&g_stGeneralParameter.m_pstThreadPool->m_mutexPool);
for( iCount = 0 ; iCount < iCurrent ; iCount ++ ) {
pthread_cancel((g_stGeneralParameter.m_pstThreadPool->m_pstWork + iCount)->m_iWorkThreadID);
pthread_cond_destroy(&(g_stGeneralParameter.m_pstThreadPool->m_pstWork + iCount)->m_condWork);
pthread_mutex_destroy(&(g_stGeneralParameter.m_pstThreadPool->m_pstWork + iCount)->m_mutexWork);
}
pthread_cancel(g_stGeneralParameter.m_pstThreadPool->m_iManegeThreadID);
pthread_mutex_destroy(&g_stGeneralParameter.m_pstThreadPool->m_mutexPool);
pthread_mutex_destroy(&g_stGeneralParameter.m_pstThreadPool->m_mutexManage);
//if( g_stGeneralParameter.m_pstThreadPool->m_pstWork ) free(g_stGeneralParameter.m_pstThreadPool->m_pstWork);
if( *g_stGeneralParameter.m_piSocket ) {
close(*g_stGeneralParameter.m_piSocket);
}
}
- 再然后是线程池的构造和析构
/********************************************************/
/* 日期:2009年10月15日 */
/* 函数:创建线程 */
/* 参数:pstThreadPool线程池信息 */
/********************************************************/
int CreateThreadGroup(THREADPOOL_S *pstThreadPool)
{
int iCount = 0;
int iNumber = 0;
int iCurrent = 0;
pthread_attr_t attrThread;
/* 创建和销毁二个线程的互斥锁,避免创建销毁同时发生 */
pthread_mutex_lock(&pstThreadPool->m_mutexManage);
/* 读取线程池中线程总数 */
pthread_mutex_lock(&pstThreadPool->m_mutexPool);
iCurrent = pstThreadPool->m_uiCurrent;
pthread_mutex_unlock(&pstThreadPool->m_mutexPool);
if( !iCurrent ) {
iNumber = MIN_LEVEL;
}
else if( iCurrent + NEW_LEVEL <= MAX_LEVEL ) {
iNumber = NEW_LEVEL;
}
else {
return SUCCESS;
}
/* 创建的新线程设置为分离态,由系统回收其资源 */
if( pthread_attr_init(&attrThread) ) {
fprintf(stderr,"Failed to pthread_attr_init,%s!\n",strerror(errno));
return FAILURE;
}
if( pthread_attr_setdetachstate(&attrThread,PTHREAD_CREATE_DETACHED) ) {
fprintf(stderr,"Failed to pthread_attr_setdetachstate,%s!\n",strerror(errno));
return FAILURE;
}
for( iCount = 0 ; iCount < iNumber ; iCount ++ ) {
pthread_cond_init(&(pstThreadPool->m_pstWork + iCurrent + iCount)->m_condWork,NULL);
pthread_mutex_init(&(pstThreadPool->m_pstWork + iCurrent + iCount)->m_mutexWork,NULL);
if( pthread_create(&(pstThreadPool->m_pstWork + iCurrent + iCount)->m_iWorkThreadID,&attrThread,(void *)WorkThreads,(void *)pstThreadPool) ) {
fprintf(stderr,"Failed to pthread_create,%s!\n",strerror(errno));
return FAILURE;
}
}
/* 创建完成后再更改线程池线程总数 */
pthread_mutex_lock(&pstThreadPool->m_mutexPool);
pstThreadPool->m_uiCurrent += iNumber;
pthread_mutex_unlock(&pstThreadPool->m_mutexPool);
pthread_attr_destroy(&attrThread);
/* 延时是为了避免线程信号在线程创建前到达,避免系统未处理用户数据而在epoll_wait上挂起,延时值为经验值 */
usleep(DELAY_TIME);
pthread_mutex_unlock(&pstThreadPool->m_mutexManage);
return SUCCESS;
}
/********************************************************/
/* 日期:2009年10月15日 */
/* 函数:销毁线程 */
/* 参数:pstThreadPool线程池信息 */
/********************************************************/
int DeleteThreadGroup(THREADPOOL_S *pstThreadPool)
{
int iCount = 0;
int iCurrent = 0;
/* 创建和销毁二个线程的互斥锁,避免创建销毁同时发生 */
pthread_mutex_lock(&pstThreadPool->m_mutexManage);
/* 读取线程池中线程总数 */
pthread_mutex_lock(&pstThreadPool->m_mutexPool);
iCurrent = pstThreadPool->m_uiCurrent - 1;
if( iCurrent > MIN_LEVEL ) {
iCount = (pstThreadPool->m_uiCurrent -= NEW_LEVEL);
}
else {
pthread_mutex_unlock(&pstThreadPool->m_mutexPool);
return SUCCESS;
}
pthread_mutex_unlock(&pstThreadPool->m_mutexPool);
while( iCurrent >= iCount ) {
pthread_cancel((pstThreadPool->m_pstWork + iCurrent)->m_iWorkThreadID);
pthread_cond_destroy(&(pstThreadPool->m_pstWork + iCurrent)->m_condWork);
pthread_mutex_destroy(&(pstThreadPool->m_pstWork + iCurrent)->m_mutexWork);
iCurrent --;
}
pthread_mutex_unlock(&pstThreadPool->m_mutexManage);
return SUCCESS;
}
/********************************************************/
/* 日期:2009年10月15日 */
/* 函数:工作线程 */
/* 参数:pstThreadPool线程池信息 */
/********************************************************/
void *WorkThreads(THREADPOOL_S *pstThreadPool)
{
int iArray = 0;
struct epoll_event stEpoll;
iArray = GetWorkThreadArray(pstThreadPool,pthread_self());
if( iArray < 0 ) pthread_exit(NULL);
while( 1 ) {
pthread_mutex_lock(&(pstThreadPool->m_pstWork + iArray)->m_mutexWork);
while( (pstThreadPool->m_pstWork + iArray)->m_uiStatus == IDLE_STATUS ) {
pthread_cond_wait(&(pstThreadPool->m_pstWork + iArray)->m_condWork,&(pstThreadPool->m_pstWork + iArray)->m_mutexWork);
}
pthread_mutex_unlock(&(pstThreadPool->m_pstWork + iArray)->m_mutexWork);
/* 用户数据处理函数,客户端终止EPOLLIN行为数据读取返回为0 */
if( !PrivateFunction(iArray,(pstThreadPool->m_pstWork + iArray)->m_iClientSocket) ) {
stEpoll.data.fd = (pstThreadPool->m_pstWork + iArray)->m_iClientSocket;
epoll_ctl(pstThreadPool->m_iEpoll,EPOLL_CTL_DEL,(pstThreadPool->m_pstWork + iArray)->m_iClientSocket,&stEpoll);
close((pstThreadPool->m_pstWork + iArray)->m_iClientSocket);
}
pthread_mutex_lock(&(pstThreadPool->m_pstWork + iArray)->m_mutexWork);
(pstThreadPool->m_pstWork + iArray)->m_uiStatus = IDLE_STATUS;
pthread_mutex_unlock(&(pstThreadPool->m_pstWork + iArray)->m_mutexWork);
}
pthread_exit(NULL);
}
- 接下来是初始化线程池管理线程和相关操作
/********************************************************/
/* 日期:2009年10月15日 */
/* 函数:初始化线程池管理线程 */
/* 参数:pstThreadPool线程池信息 */
/********************************************************/
int InitManageThread(THREADPOOL_S *pstThreadPool)
{
if( pthread_create(&pstThreadPool->m_iManegeThreadID,NULL,(void *)ManageThread,(void *)pstThreadPool) ) {
fprintf(stderr,"Failed to pthread_create,%s!\n",strerror(errno));
return FAILURE;
}
return SUCCESS;
}
/********************************************************/
/* 日期:2009年10月15日 */
/* 函数:线程池管理线程 */
/* 参数:pstThreadPool线程池信息 */
/********************************************************/
void *ManageThread(THREADPOOL_S *pstThreadPool)
{
int iCount = 0;
int iNumber = 0;
float fPercent = 0.0;
while( 1 ) {
/* 管理线程轮询休眠时间经验值 */
sleep(SLEEP_TIME);
for( iCount = 0 ; iCount < pstThreadPool->m_uiCurrent ; iCount ++ ) {
if( (pstThreadPool->m_pstWork + iCount)->m_uiStatus == BUSY_STATUS ) iNumber ++;
}
/* 线程忙碌百分比 */
fPercent = (float)(iNumber / pstThreadPool->m_uiCurrent);
if( fPercent < LOW_WEIGHT ) {
if( DeleteThreadGroup(pstThreadPool) ) {
fprintf(stderr,"Failed to delete thread group!\n");
continue;
}
}
else if( fPercent > TOP_WEIGHT ) {
if( CreateThreadGroup(pstThreadPool) ) {
fprintf(stderr,"Failed to create thread group!\n");
continue;
}
}
}
pthread_exit(NULL);
}
/********************************************************/
/* 日期:2009年10月15日 */
/* 函数:获取线程数据中的下标 */
/* 参数:pstThreadPool线程池信息;iWorkThreadID线程ID */
/********************************************************/
int GetWorkThreadArray(THREADPOOL_S *pstThreadPool,pthread_t iWorkThreadID)
{
int iCount = 0;
for( iCount = 0 ; iCount < pstThreadPool->m_uiMaximal ; iCount ++ ) {
if( (pstThreadPool->m_pstWork + iCount)->m_iWorkThreadID == iWorkThreadID ) return iCount;
}
return -1;
}
- 最后是主程入口
GENERALPARAMETER_S g_stGeneralParameter = { NULL };
/********************************************************/
/* 日期:2009年10月15日 */
/* 函数:主函数,程序入口。 */
/* 参数:h打印帮助信息;v现实版本信息 */
/********************************************************/
int main(int argc,char *argv[])
{
pid_t iPid;
int iEpoll = 0;
int iWorks = 0;
int iCount = 0;
int iNumber = 0;
int iSocket = 0;
int iCurrent = 0;
int iNewSocket = 0;
int iOptionIndex = 0;
int iOptionReturn = 0;
socklen_t iClientSize = 0;
THREADPOOL_S stThreadPool;
struct sockaddr_in stServer;
struct sockaddr_in stClient;
struct epoll_event stEpoll;
struct epoll_event stEvent[EVENT_MAX];
struct option stOptions[] = {
{"help",0,0,'h'},
{"version",0,0,'v'},
{0,0,0,0},
};
while( 1 ) {
iOptionReturn = getopt_long(argc,argv,"hv",stOptions,&iOptionIndex);
if( iOptionReturn < 0 ) {
break;
}
switch(iOptionReturn) {
case 'h': PrintHelp();
return SUCCESS;
case 'v': printf("Copyright (C) 2008. \n");
printf("Version 1.0.0 Build on %s %s.\n",__DATE__,__TIME__);
return SUCCESS;
default: return SUCCESS;
}
}
/* 创建子进程并让子进程成为守护进程 */
iPid = fork();
if( iPid < 0 ) {
fprintf(stderr,"Failed to fork,%s!\n",strerror(errno));
return FAILURE;
}
else if( iPid > 0 ) {
return SUCCESS;
}
if( setsid() < 0 ) {
fprintf(stderr,"Failed to setsid,%s!\n",strerror(errno));
return FAILURE;
}
if( chdir("/") < 0 ) {
fprintf(stderr,"Failed to chdir,%s!\n",strerror(errno));
return FAILURE;
}
umask(0);
/* 注册析构函数和信号处理函数 */
atexit(ReleaseResource);
signal(SIGINT,SignalQuit);
signal(SIGSEGV,SignalQuit);
signal(SIGTERM,SignalQuit);
/* 更改内核文件描述符限制 */
printf("Setting the max value of file handle in kernel ...\n");
if( InitLimit() ) {
fprintf(stderr,"Failed to initialize limit,%s!\n",strerror(errno));
return FAILURE;
}
/* 初始化epoll相关调用 */
printf("Initializing epoll handle....\n");
iEpoll = InitEpoll();
if( iEpoll < 0 ) {
fprintf(stderr,"Failed to initialize poll,%s!\n",strerror(errno));
return FAILURE;
}
/* 初始化主套接字 */
printf("Initializing socket...\n");
if( InitSocket(&iSocket,&stServer,iEpoll) ) {
fprintf(stderr,"Failed to initialize socket,%s!\n",strerror(errno));
return FAILURE;
}
/* 创建线程池 */
printf("Creating thread pool...\n");
if( InitThreadPool(&stThreadPool,iEpoll) ) {
fprintf(stderr,"Failed to initialize thread pool!\n");
return FAILURE;
}
/* 创建线程池管理线程 */
printf("Creating a manage thread to control thread's total...\n");
if( InitManageThread(&stThreadPool) ) {
fprintf(stderr,"Failed to create ControlThread,%s!\n",strerror(errno));
return FAILURE;
}
/* 将析构数据赋予全局变量并在析构函数中调用 */
g_stGeneralParameter.m_piSocket = &iSocket;
g_stGeneralParameter.m_pstThreadPool = &stThreadPool;
/* 关闭标准输入输出和错误 */
close(STDIN_FILENO);
close(STDOUT_FILENO);
close(STDERR_FILENO);
/* 主进程进入监听状态 */
iClientSize = (socklen_t)sizeof(struct sockaddr_in);
while( 1 ) {
memset(&stEvent,'\0',sizeof(struct epoll_event) * EVENT_MAX);
iNumber = epoll_wait(iEpoll,stEvent,EVENT_MAX,-1);
for( iWorks = 0 ; iWorks < iNumber ; iWorks ++ ) {
/* 新接入套接字 */
if( stEvent[iWorks].data.fd == iSocket ) {
while( (iNewSocket = accept(iSocket,(struct sockaddr *)&stClient,&iClientSize)) > 0 ) {
if( fcntl(iNewSocket,F_SETFL,fcntl(iNewSocket,F_GETFD,0) | O_NONBLOCK) == -1 ) {
fprintf(stderr,"Failed to fcntl,%s!\n",strerror(errno));
continue;
}
stEpoll.events = EPOLLIN | EPOLLET;
stEpoll.data.fd = iNewSocket;
if( epoll_ctl(iEpoll,EPOLL_CTL_ADD,iNewSocket,&stEpoll) < 0 ) {
fprintf(stderr,"Failed to epoll_ctl,%s\n",strerror(errno));
continue;
}
}
}
/* 已有套接字 */
else {
iCount = 0;
/* 线程池中线程总数 */
pthread_mutex_lock(&stThreadPool.m_mutexPool);
iCurrent = stThreadPool.m_uiCurrent;
pthread_mutex_unlock(&stThreadPool.m_mutexPool);
/* 最大线程数 */
if( iCurrent == MAX_LEVEL ) {
while( iCount < MAX_LEVEL ) {
pthread_mutex_lock(&(stThreadPool.m_pstWork + iCount)->m_mutexWork);
if( !(stThreadPool.m_pstWork + iCount)->m_uiStatus ) {
(stThreadPool.m_pstWork + iCount)->m_uiStatus = BUSY_STATUS;
(stThreadPool.m_pstWork + iCount)->m_iClientSocket = stEvent[iWorks].data.fd;
pthread_mutex_unlock(&(stThreadPool.m_pstWork + iCount)->m_mutexWork);
pthread_cond_signal(&(stThreadPool.m_pstWork + iCount)->m_condWork);
break;
}
else {
pthread_mutex_unlock(&(stThreadPool.m_pstWork + iCount)->m_mutexWork);
}
if( ++ iCount == MAX_LEVEL ) iCount = 0;
}
}
/* 非最大线程数 */
else {
for( ; iCount < iCurrent ; iCount ++ ) {
pthread_mutex_lock(&(stThreadPool.m_pstWork + iCount)->m_mutexWork);
if( (stThreadPool.m_pstWork + iCount)->m_uiStatus == IDLE_STATUS ) {
(stThreadPool.m_pstWork + iCount)->m_uiStatus = BUSY_STATUS;
(stThreadPool.m_pstWork + iCount)->m_iClientSocket = stEvent[iWorks].data.fd;
pthread_mutex_unlock(&(stThreadPool.m_pstWork + iCount)->m_mutexWork);
pthread_cond_signal(&(stThreadPool.m_pstWork + iCount)->m_condWork);
break;
}
else {
pthread_mutex_unlock(&(stThreadPool.m_pstWork + iCount)->m_mutexWork);
}
}
/* 所有线程忙碌并创建新线程 */
if( iCount == iCurrent ) {
if( !CreateThreadGroup(&stThreadPool) ) {
pthread_mutex_lock(&(stThreadPool.m_pstWork + iCount)->m_mutexWork);
if( !(stThreadPool.m_pstWork + iCount)->m_uiStatus ) {
(stThreadPool.m_pstWork + iCount)->m_uiStatus = BUSY_STATUS;
(stThreadPool.m_pstWork + iCount)->m_iClientSocket = stEvent[iWorks].data.fd;
pthread_mutex_unlock(&(stThreadPool.m_pstWork + iCount)->m_mutexWork);
pthread_cond_signal(&(stThreadPool.m_pstWork + iCount)->m_condWork);
}
else {
pthread_mutex_unlock(&(stThreadPool.m_pstWork + iCount)->m_mutexWork);
}
}
}
}
}
}
}
return SUCCESS;
}
代码中关键位置都附上了注释,熟悉 epoll 和底层线程实现的话看着应该不复杂,供大家参考。