Linux Socket 编程 --深入理解 epoll 与 Reactor epoll 实现百万并发

没有Reactor 模式的 epoll

#include <stdio.h>
#include <sys/socket.h>
#include <sys/types.h>
#include <netinet/in.h>
#include <fcntl.h>
#include <pthread.h>
#include <sys/epoll.h>
#include <time.h>
#include <string.h>
#include <stdlib.h>

#include <arpa/inet.h>

#define BACKLOG  10
#define BASE_PORT   9999
#define PORT_NUMBER  1
#define EVENTS_LENGTH  1000
#define BUFFER_LENGTH  1024
/*
struct sock_item{
    int fd;
    char * rbuffer;
    int rlength;
    char * wbuffer;
    int wlength;

    int event;

    void (*recv_cb)(int fd,char *buffer,int length);
    void (*send_cb)(int fd,char *buffer,int length);
    void (*accept_cb)(int fd,char *buffer,int length);
     
};

struct eventblock{
    struct sock_item *items;
    struct eventblock* next;

};

struct reactor{
     int epfd;
     int blockcnt;
     struct eventblock* evblk;
};

int reactor_resize(struct reactor * r)
{
    //传参判断
    if(r == NULL)  return -1;
    if(r->evblk == NULL)  return -1;

    struct eventblock *blk = r->evblk;
    while(blk != NULL && blk->next != NULL)
    {
       
    }
}
struct sock_item* reactor_lookup(struct reactor* r,int sockfd)
{
    
}

*/

int Init_Server(short port)
{
    int listenfd;
    struct sockaddr_in servaddr;
    listenfd = socket(AF_INET,SOCK_STREAM,0);
    if(listenfd == -1) 
    {
        printf("create socket error\n"); // 使用 perror最好
        return -1;
    }
    servaddr.sin_family = AF_INET;
    servaddr.sin_addr.s_addr = htonl(INADDR_ANY);
    servaddr.sin_port = htons(port);
    if(bind(listenfd,(struct sockaddr*)&servaddr,sizeof(servaddr))==-1) 
    {
        return -2;
    }
    listen(listenfd,BACKLOG);//监听 等待 client连接
    return listenfd;
}

int Is_Serverfd(int *fds,int connfd)
{
    int i = 0;
    for(i =0;i < PORT_NUMBER;i++)
    {
         if(fds[i]==connfd)
        {
            return 1;
        }
    }
    return 0;
}

int main()
{
    int epfd,clientfd;
    struct epoll_event  ev, events[EVENTS_LENGTH];
    int ret;
    char rbuffer[BUFFER_LENGTH] = {0};//所有的fd 公用一个
    char wbuffer[BUFFER_LENGTH] = {0};//所有的fd 公用一个
    int sockfds[PORT_NUMBER]={0};

    epfd = epoll_create(1);
    int i = 0;
    for(i=0;i<PORT_NUMBER;i++)
    {
        sockfds[i]= Init_Server(BASE_PORT + i);
        printf("listen fd:%d\n",sockfds[i]);
        ev.events = EPOLLIN;
        ev.data.fd = sockfds[i];
        if(epoll_ctl(epfd,EPOLL_CTL_ADD,sockfds[i],&ev)== -1)
        {
            printf("err epoll_ctl\n");
        }
    }
    
    while(1)
    {
        int nready = epoll_wait(epfd,events,EVENTS_LENGTH,1000);
        printf("--------nready=%d\n",nready);
        int i=0;
        for(i = 0; i < nready; i++)
        {
            int eventfd = events[i].data.fd;
            printf("No:%d---eventfd:%d\n",i,eventfd);
            // 判断 epoll 返回的活动的 fd 是不是 scokfd,是就 accept 获得 clientfd, 不是 就进入收发流程
            if(Is_Serverfd(sockfds,eventfd)) // 是 serverfd
            {
                struct sockaddr_in client;
                socklen_t len =sizeof(client);
                clientfd= accept(eventfd,(struct sockaddr*)& client,&len);
                char *ip_addr;
                ip_addr= inet_ntoa(*(struct in_addr *)&(client.sin_addr.s_addr));
                printf("Client <%d--%s:%d>\n",clientfd,ip_addr,ntohs(client.sin_port));
                 // 把获取到的Clientfd 也加入到兴趣列表里
                //struct epoll_event ev;
                ev.events= EPOLLIN;
                ev.data.fd = clientfd;
                epoll_ctl(epfd,EPOLL_CTL_ADD,clientfd,&ev);
                
            }else if (events[i].events & EPOLLIN)
            {     
                    memset(rbuffer,0,sizeof(rbuffer));
                    memset(rbuffer,0,sizeof(wbuffer));
                    ret = recv(eventfd,rbuffer,BUFFER_LENGTH,0);
                    if(ret > 0)
                    {
                        rbuffer[BUFFER_LENGTH]='\0';
                        printf("buffer: %s,ret:%d\n",rbuffer,ret);
                        memcpy(wbuffer,rbuffer,BUFFER_LENGTH);
                        memset(rbuffer,0,sizeof(rbuffer));
                        ev.events = EPOLLOUT;
                        ev.data.fd = eventfd;
                        epoll_ctl(epfd,EPOLL_CTL_MOD,eventfd,&ev);
                    }else if(ret == 0){
                        close(eventfd);
                    }

            }
            else if(events[i].events & EPOLLOUT)
            {
                int sent= send(eventfd,wbuffer,ret,0);
                printf("sent:%d\n",sent);
                memset(rbuffer,0,sizeof(rbuffer));
                memset(rbuffer,0,sizeof(wbuffer));
                ev.events = EPOLLIN;
                ev.data.fd = eventfd;
                epoll_ctl(epfd,EPOLL_CTL_MOD,eventfd,&ev);
            }
        }
    }
    return 0;
}

总结

  1. 所有 fd 共用 rbuffer 与 wbuffer ,当并发数量很多,同时读写的时候,如果不加锁就会出现 rbuffer 与 wbuffer 混乱的问题。如果加锁就会影响读写性能。

reactor 模式测试验证

看一下涉及到的 对象

struct reactor / struct eventblock / struct sock_items
在这里插入图片描述

看一下reactor 代码的运行 运行

struct sock_items{
    int fd;
    char *rbuffer;
    int  rlength;
    char *wbuffer;
    int wlength;

    int event;

    void (*recv_cb)(int fd,char *buffer,int length);
    void (*send_cb)(int fd,char *buffer,int length);
    void (*accept_cb)(int fd,char *buffer,int length);

};

struct eventblock{
     struct sock_items*  items;
     struct eventblock* next;
};

struct reactor{
    int epfd;
    int blkcnt; // block 块的数量
    struct eventblock*  evblk; // 指向第一个 eventblock
};

int reactor_resize(struct reactor * r)
{
    if(r == NULL) return -1;
    //if(r->evblk == NULL) return -1;

    struct eventblock * blk = r->evblk;
    struct sock_items* item = (struct sock_items*) calloc(ITEM_LENGTH , sizeof(struct sock_items));
    if(item == NULL) return  -4;

    struct eventblock *block =  (struct eventblock*)calloc(1,sizeof(struct eventblock));
    if(block == NULL)
    {
        free(item);
        return -5;
    }
    memset(block,0,sizeof(struct eventblock));
    block->items = item;
    block->next = NULL;
    if(blk == NULL)
    {
        r->evblk = block;
    }else{
          while(blk !=NULL && blk->next!=NULL)
         {
            blk = blk->next;
         }
         blk->next = block;
    }
    r->blkcnt ++;
    return 0;
}

struct sock_item* reactor_lookup(struct reactor *r,int sockfd)
{
    if ( r== NULL)  return NULL;
    if(sockfd <= 0)  return NULL;

    int blkidx = sockfd / ITEM_LENGTH;
    while(blkidx  >= r->blkcnt)
    {
        reactor_resize(r);
    }
    int i = 0;
    struct eventblock *blk = r->evblk;
    while(i++ < blkidx && blk !=NULL)
    {
        blk = blk->next;
    }
    return &blk->items[sockfd % ITEM_LENGTH];
 
}

// 测试 reactor 能不能正常分配 内存
int main()
	{
	  struct reactor *r = (struct reactor*) calloc(1,sizeof(struct reactor));
	  r->epfd = 1;
	  int i = 1;
	  while(i < 11)
	  {
	    struct sock_items *item = reactor_lookup(r,i);
	    item->fd = i;
	    item->rbuffer = calloc(1,10);
	    item->rlength = 0;
	    item->wbuffer = calloc(1,10);
	    item->wlength = 0;
	    i++;
	  }
	}

请添加图片描述

epoll + Reactor 代码

#include <stdio.h>
#include <sys/socket.h>
#include <sys/types.h>
#include <netinet/in.h>
#include <fcntl.h>
#include <pthread.h>
#include <sys/epoll.h>
#include <time.h>
#include <string.h>
#include <stdlib.h>

#include <arpa/inet.h>

#define BACKLOG  10
#define BASE_PORT   9999
#define PORT_NUMBER  1
#define EVENTS_LENGTH  1000
#define BUFFER_LENGTH  1024
#define ITEM_LENGTH  1000

struct sock_items{
    int fd;
    char *rbuffer;
    int  rlength;
    char *wbuffer;
    int wlength;

    int event;

    void (*recv_cb)(int fd,char *buffer,int length);
    void (*send_cb)(int fd,char *buffer,int length);
    void (*accept_cb)(int fd,char *buffer,int length);

};

struct eventblock{
     struct sock_items*  items;
     struct eventblock* next;
};

struct reactor{
    int epfd;
    int blkcnt; // block 块的数量
    struct eventblock*  evblk; // 指向第一个 eventblock
};

int reactor_resize(struct reactor * r)
{
    if(r == NULL) return -1;
    printf("reactor reszie\n");
    struct eventblock * blk = r->evblk;
    struct sock_items* item = (struct sock_items*) calloc(ITEM_LENGTH , sizeof(struct sock_items));
    if(item == NULL) return  -4;

    struct eventblock *block =  (struct eventblock*)calloc(1,sizeof(struct eventblock));
    if(block == NULL)
    {
        free(item);
        return -5;
    }
    memset(block,0,sizeof(struct eventblock));
    block->items = item;
    block->next = NULL;
    if(blk == NULL)  // 第一次 链表里面没有任何节点的时候
    {
        r->evblk = block;
    }else{             // 其他情况下,首先要找到队尾,然后把队尾重新指向 它
        while(blk !=NULL && blk->next !=NULL)
        {
           blk = blk->next;
        }
        blk->next = block;
    }

    r->blkcnt ++;
    return 0;
}

struct sock_item* reactor_lookup(struct reactor *r,int sockfd)
{
    if ( r== NULL)  return NULL;
    if(sockfd <= 0)  return NULL;
    printf("reactor lookup---> %d\n",sockfd);
    int blkidx = sockfd / ITEM_LENGTH;
    //printf("reactor lookup---> %d\n",blkidx);
    while(blkidx  >= r->blkcnt)
    {
        reactor_resize(r);
    }
    int i = 0;
    struct eventblock *blk = r->evblk;
    while(i++ < blkidx && blk !=NULL)
    {
        blk = blk->next;
    }
    return &blk->items[sockfd % ITEM_LENGTH];
 
}

int Init_Server(short port)
{
    int listenfd;
    struct sockaddr_in servaddr;
    listenfd = socket(AF_INET,SOCK_STREAM,0);
    if(listenfd == -1) 
    {
        printf("create socket error\n"); // 使用 perror最好
        return -1;
    }
    servaddr.sin_family = AF_INET;
    servaddr.sin_addr.s_addr = htonl(INADDR_ANY);
    servaddr.sin_port = htons(port);
    if(bind(listenfd,(struct sockaddr*)&servaddr,sizeof(servaddr))==-1) 
    {
        return -2;
    }
    listen(listenfd,BACKLOG);//监听 等待 client连接
    return listenfd;
}

int Is_Serverfd(int *fds,int connfd)
{
    for(int i =0;i < PORT_NUMBER;i++)
    {
         if(fds[i]==connfd)
        {
            return 1;
        }
    }
    return 0;
}


int main()
{
    int epfd,clientfd;
    struct epoll_event  ev, events[EVENTS_LENGTH];
    int ret;
    int sockfds[PORT_NUMBER]={0};
    //char rbuffer[BUFFER_LENGTH] = {0};//所有的fd 公用一个
    //char wbuffer[BUFFER_LENGTH] = {0};//所有的fd 公用一个
    struct reactor  *r = (struct reactor*) calloc(1,sizeof(struct reactor));
    if( r== NULL)
    {
        return -3;
    }
    r->blkcnt = 0;
    r->epfd = epoll_create(1);
    for(int i=0;i<PORT_NUMBER;i++)
    {
        sockfds[i]= Init_Server(BASE_PORT + i);
        printf("listen fd:%d\n",sockfds[i]);
        ev.events = EPOLLIN;
        ev.data.fd = sockfds[i];
        if(epoll_ctl(r->epfd,EPOLL_CTL_ADD,sockfds[i],&ev)== -1)
        {
            printf("err epoll_ctl\n");
        }
    }
    
    while(1)
    {
        int nready = epoll_wait(r->epfd,events,EVENTS_LENGTH,1000);
        //printf("--------nready=%d\n",nready);
        for(int i = 0; i < nready; i++)
        {
            int eventfd = events[i].data.fd;
            // 判断 epoll 返回的活动的 fd 是不是 scokfd,是就 accept 获得 clientfd, 不是 就进入收发流程
            if(Is_Serverfd(sockfds,eventfd)) // 是 serverfd
            {
                printf("is Server fd\n");
                struct sockaddr_in client;
                socklen_t len =sizeof(client);
                clientfd= accept(eventfd,(struct sockaddr*)& client,&len);
                if(clientfd == -1) break;
                // if(clientfd % 1000 == 999){
                //     printf("accept:%d\n",clientfd);
                // }
                char *ip_addr;
                ip_addr= inet_ntoa(*(struct in_addr *)&(client.sin_addr.s_addr));
                printf("Client <%s:%d>\n",ip_addr,ntohs(client.sin_port));
                 // 把获取到的Clientfd 也加入到兴趣列表里
                //struct epoll_event ev;
                ev.events= EPOLLIN;
                ev.data.fd = clientfd;
                epoll_ctl(r->epfd,EPOLL_CTL_ADD,clientfd,&ev);

                struct sock_items *item = reactor_lookup(r,clientfd);// 查找函数
                item->fd = clientfd;
                item->rbuffer = calloc(1,BUFFER_LENGTH);
                item->rlength = 0;
                item->wbuffer = calloc(1,BUFFER_LENGTH);
                item->wlength = 0;
                
            }else if (events[i].events & EPOLLIN)
            {     
                       printf("recv\n");
                       struct sock_items *item = reactor_lookup(r,eventfd);// 查找函数
                       char * rbuffer = item->rbuffer;
                       char * wbuffer = item->wbuffer;
                       ret = recv(item->fd,rbuffer,BUFFER_LENGTH,0);
                       if(ret > 0)
                       {
                         printf("recv %d> 0 \n",ret);
                         rbuffer[BUFFER_LENGTH]='\0';
                         printf("buffer: %s,ret:%d\n",rbuffer,ret);
                         memcpy(wbuffer,rbuffer,BUFFER_LENGTH);
                         ev.events = EPOLLOUT;
                         ev.data.fd = eventfd;
                         epoll_ctl(r->epfd,EPOLL_CTL_MOD,eventfd,&ev);
                       }else if(ret == 0){
                          printf("recv = 0 \n");
                          free(rbuffer);
                          free(wbuffer);
                          item->fd = 0;
                          close(eventfd);
                       }

            }
            else if(events[i].events & EPOLLOUT)
            {
                printf("send\n");
                struct sock_items *item = reactor_lookup(r,eventfd);// 查找函数
                char * rbuffer = item->rbuffer;
                char * wbuffer = item->wbuffer;
                int sent= send(item->fd,wbuffer,ret,0);
                printf("sent:%d\n",sent);
                ev.events = EPOLLIN;
                ev.data.fd = eventfd;
                epoll_ctl(r->epfd,EPOLL_CTL_MOD,eventfd,&ev);
            }
        }
    }
    return 0;
}

测试记录

三客户端简单测试

大链接建链接测试(承载测试)

每个业务的 QPS(吞吐量测试)

断开连接的时候

数据传输量(每秒传送的数据包)

文章参考与<零声教育>的C/C++linux服务期高级架构系统教程学习:
服务器高级架构体系:
https://ke.qq.com/course/417774?flowToken=1010783

评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值