Linux高并发学习---epoll单机百万并发实现

10 篇文章 0 订阅

一、前情提要

在上一篇文章中(Linux高并发学习—epoll的reactor实现),已经讲述了epoll的reactor实现方式,本篇文章将沿用代码实现单机百万并发的实现。

服务端代码增加了一个TCP连接的链表,用于支持大量TCP连接;客户端代码则是循环创建socket连接。话不多说,咱直接看代码!

二、代码细节

服务端之所以能支持上百万个TCP连接,离不开五元组的支持,其中源端口开放了100个,而客户端的作为目的端口也有上万个端口能够正常使用,排列组合下就有100万以上啦!

#include <stdio.h>
#include <ctype.h>
#include <unistd.h>
#include <fcntl.h>
#include <sys/types.h>
#include <arpa/inet.h>
#include <sys/socket.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>
#include <sys/select.h>
#include <sys/poll.h>
#include <sys/epoll.h>
#include <pthread.h>

#define SERVER_PORT     9000
#define BUFFER_SIZE     1024
#define MAX_EPOLL_SIZE  512
#define MAX_LISTEN_BAGS 512

#define INIT            0
#define ACCEPT          1
#define READ            2
#define WRITE           3

typedef int (*callBack)(int, int, void*);

typedef struct _event_item 
{
     int fd;
     int events;
     void *args;
     callBack rhandle;     // 读事件回调
     callBack whandle;     // 写事件回调
     
     unsigned char sendBuf[BUFFER_SIZE];
     int sendLen;
     unsigned char recvBuf[BUFFER_SIZE];
     int recvLen;
} event_item;

typedef struct _block
{
     event_item *events;
     struct _block *next;
}event_block;

typedef struct _reactor
{
     int epollfd;
     int blockCnt;
     event_block *blk;       // 链表,用于存储百万TCP连接
}reactor;

int acceptCB(int fd, int events, void* args);
int readCB(int fd, int events, void* args);
int writeCB(int fd, int events, void* args);
int init_reactor(reactor *r);

reactor *instance = NULL;
reactor *getInstance(){
     if(instance == NULL){
          instance = (reactor*)malloc(sizeof(reactor));
          if(instance == NULL) 
               return NULL;
          memset(instance, 0, sizeof(reactor));
          if(init_reactor(instance) < 0){
               free(instance);
               return NULL;
          }
     }

     return instance;
}

int init_socket(short port){
     int reuseAddr = 1;
     int reusePort = 1;
     int fd = socket(AF_INET, SOCK_STREAM, 0);
     if(fd < 0)
     {
          printf("errno: %s, socket() failed!\n", strerror(errno));
          return -1;
     }
     if(fcntl(fd, F_SETFL, O_NONBLOCK) < 0){
          printf("errno: %s, fcntl() failed!\n", strerror(errno));
          return -1;
     }

     struct sockaddr_in serverAddr;
     memset(&serverAddr, 0, sizeof(serverAddr));
     serverAddr.sin_family = AF_INET;
     serverAddr.sin_port = htons(port);
     serverAddr.sin_addr.s_addr = htonl(INADDR_ANY);

     setsockopt(fd, SOL_SOCKET, SO_REUSEPORT, (const void*)&reusePort, sizeof(reusePort));
     setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, (const void*)&reuseAddr, sizeof(reuseAddr));
     if(bind(fd, (struct sockaddr*)&serverAddr, sizeof(serverAddr)) < 0){
          printf("errno: %s, bind() failed!\n", strerror(errno));
          return -1;
     }

     if(listen(fd, MAX_LISTEN_BAGS) < 0){
          printf("errno: %s, listen() failed!\n", strerror(errno));
          return -1;
     }

     return fd; 
}

int block_alloc(){
     reactor *r = getInstance();
     if(r == NULL){
          printf("block_alloc(): getInstance() failed!\n");
          return -1;
     }
     if(r->blk == NULL){
          printf("r->blk == NULL!\n");
          return -1;
     }

     event_item *ev = (event_item*)malloc((MAX_EPOLL_SIZE) * sizeof(event_item));
     if(ev == NULL){
          printf("block_alloc(): malloc() failed!\n");
          return -1;
     }
     memset(ev, 0, (MAX_EPOLL_SIZE) * sizeof(event_item));

     event_block *blk = (event_block*)malloc(sizeof(event_block));
     if(blk == NULL){
          printf("block_alloc(): malloc() failed!\n");
          return -1;
     }

     blk->events = ev;
     blk->next = NULL;

     event_block *p = r->blk;
     while(p->next){
          p = p->next;
     }
     p->next = blk;
     r->blockCnt ++;

     return 0;
}

int init_reactor(reactor *r){
     if(r == NULL)  return -1;
     
     memset(r, 0, sizeof(reactor));
     r->epollfd = epoll_create(1);
     if(r->epollfd <= 0){
          printf("errno: %s, epoll_create() failed!\n", strerror(errno));
          return -1;
     }

     event_block *blk = (event_block*)malloc(sizeof(event_block));
     if(blk == NULL){
          printf("init_reactor() malloc failed!\n");
          return -1;
     }
     event_item *item = (event_item*)malloc((MAX_EPOLL_SIZE) * sizeof(event_item));
     if(item == NULL){
          printf("init_reactor() malloc failed!\n");
          return -1;
     }
     memset(item, 0, (MAX_EPOLL_SIZE) * sizeof(event_item));

     blk->events = item;
     blk->next = NULL;
     r->blk = blk;
     r->blockCnt = 1;

     return 0;
}

// 根据提供的fd从链表中查询对应的event_item结构
event_item *find_events(int fd){
     reactor *r = getInstance();
     if(r == NULL){
          printf("find_events(): getInstance() failed!\n");
          return NULL;
     }

     int blkidx = fd / MAX_EPOLL_SIZE;
     while(blkidx >= r->blockCnt){   // 要查询的fd大于目前的容量,需要扩容
          block_alloc();
     }

     event_block *cur = r->blk;
     int cnt = 0;
     while(cnt++ < blkidx && cur != NULL){
          cur = cur->next;
     }
     return &(cur->events[fd % MAX_EPOLL_SIZE]);
}

int destory_reactor(){
     reactor *r = getInstance();
     if(r == NULL){
          printf("destory_reactor(): getInstance() failed!\n");
          return -1;
     }

     close(r->epollfd);
     event_block *cur = r->blk;
     event_block *nxt = NULL;
     while(cur != NULL){
          nxt = cur->next;
          free(cur->events);
          free(cur);
          cur = nxt->next;
     }

     return 0;
}

int del_reactor_events(int fd){
     reactor *r = getInstance();
     if(r == NULL){
          printf("del_reactor_events(): getInstance() failed!\n");
          return -1;
     }

     struct epoll_event ev;
     ev.data.ptr = NULL;
     epoll_ctl(r->epollfd, EPOLL_CTL_DEL, fd, &ev);
     
     return 0;
}

int set_reactor_events(int fd, int event, void *args){
     struct epoll_event ev;
     event_item *item = find_events(fd);
     reactor *r = getInstance();
     if(r == NULL){
          printf("set_reactor_events(): getInstance() failed!\n");
          return -1;
     }

     if(event == ACCEPT){
          item->fd = fd;
          item->args = args;
          item->rhandle = acceptCB;
          ev.events = EPOLLIN;
     }else if(event == READ){
          item->fd = fd;
          item->args = args;
          item->rhandle = readCB;
          ev.events = EPOLLIN;
          // ev.events |= EPOLLET;
     }else if(event == WRITE){
          item->fd = fd;
          item->args = args;
          item->whandle = writeCB;
          ev.events = EPOLLOUT;
     }

     ev.data.ptr = item;
     if(item->events == INIT){
          epoll_ctl(r->epollfd, EPOLL_CTL_ADD, fd, &ev);
          item->events = event;
     }else if(item->events != event){
          epoll_ctl(r->epollfd, EPOLL_CTL_MOD, fd, &ev);
          item->events = event;
     }

     return 0;
}

int acceptCB(int fd, int events, void* args){
     struct sockaddr_in clientAdr;
     socklen_t len = sizeof(clientAdr);
     int connfd = accept(fd, (struct sockaddr*)&clientAdr, &len);
     if(fcntl(connfd, F_SETFL, O_NONBLOCK) < 0){
          printf("errno: %s, fcntl() failed!\n", strerror(errno));
          return -1;
     }
     printf("new connection: %d\n", connfd);
     set_reactor_events(connfd, READ, args);

     return 0;
}

int readCB(int fd, int events, void* args){
     reactor *r = getInstance();
     if(r == NULL){
          printf("readCB(): getInstance() failed!\n");
          return -1;
     }

     event_item *item = find_events(fd);
     unsigned char *rbuf = item->recvBuf;

     int n = recv(fd, rbuf, BUFFER_SIZE, 0);
     if(n == 0){
          printf("errno: %s, clientfd %d closed!\n", strerror(errno), fd);
          del_reactor_events(fd);
          close(fd);
          return -1;
     }else if(n < 0){
          if(errno != EINTR && errno != EWOULDBLOCK && errno != EAGAIN){
               printf("errno: %s, error!\n", strerror(errno));
               del_reactor_events(fd);
               close(fd);
               return -1;
          }
     }
     else{
          unsigned char *sbuf = item->sendBuf;
          memcpy(sbuf, rbuf, n);
          item->sendLen = n;
          printf("recv from fd = %d: %s\n", fd, sbuf);
          set_reactor_events(fd, WRITE, args);
     }

     return 0;
}

int writeCB(int fd, int events, void* args){
     reactor *r = getInstance();
     if(r == NULL){
          printf("reactor_loop(): getInstance() failed!\n");
          return -1;
     }

     event_item *item = find_events(fd);
     unsigned char *sbuf = item->sendBuf;
     int len = item->sendLen;

     int ret = send(fd, sbuf, len, 0);
     if(ret < len){
          if(ret == 0){
               del_reactor_events(fd);
               close(fd);
          }
          set_reactor_events(fd, WRITE, args);
     }else{
          set_reactor_events(fd, READ, args);
     }
     return 0;
}

int reactor_loop(){
     struct epoll_event events[MAX_EPOLL_SIZE] = {0};
     reactor *r = getInstance();
     if(r == NULL){
          printf("reactor_loop(): getInstance() failed!\n");
          return -1;
     }

     while(1){
          int nready = epoll_wait(r->epollfd, events, MAX_EPOLL_SIZE, -1);
          if(nready == -1)    
               continue;
          for(int i = 0; i < nready; ++i){
               event_item *item = (event_item *)events[i].data.ptr;
               if(events[i].events & EPOLLIN)
                    (*item->rhandle)(item->fd, 0, NULL);
               if(events[i].events & EPOLLOUT)
                    (*item->whandle)(item->fd, 0, NULL);
          }         
     }

     return 0;
}

int main(int argc, char *argv[])
{
     int sockfd = 0;
     int port = SERVER_PORT;

     for (int i = port; i < port + 100; i++)
     {
          sockfd = init_socket(i);
          if(sockfd < 0)  
               return -1;
          set_reactor_events(sockfd, ACCEPT, NULL);
     }
     // sockfd = init_socket(SERVER_PORT);
     // if(sockfd < 0)  
     //      return -1;
     // set_reactor_events(sockfd, ACCEPT, NULL);

     reactor_loop();

     return 0;
}

客户端的代码则很简单了,一直循环创建连接就OK了,如下:



#include <stdio.h>
#include <string.h>
#include <stdlib.h>

#include <sys/types.h>
#include <sys/socket.h>
#include <sys/epoll.h>
#include <errno.h>
#include <netinet/tcp.h>
#include <arpa/inet.h>
#include <netdb.h>
#include <fcntl.h>


#define MAX_BUFFER		128
#define MAX_EPOLLSIZE	(512*1024)
#define MAX_PORT		100

#define TIME_SUB_MS(tv1, tv2)  ((tv1.tv_sec - tv2.tv_sec) * 1000 + (tv1.tv_usec - tv2.tv_usec) / 1000)

int isContinue = 0;

static int ntySetNonblock(int fd) {
	int flags;

	flags = fcntl(fd, F_GETFL, 0);
	if (flags < 0) return flags;
	flags |= O_NONBLOCK;
	if (fcntl(fd, F_SETFL, flags) < 0) return -1;
	return 0;
}

static int ntySetReUseAddr(int fd) {
	int reuse = 1;
	return setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, (char *)&reuse, sizeof(reuse));
}



int main(int argc, char **argv) {
	if (argc <= 2) {
		printf("Usage: %s ip port\n", argv[0]);
		exit(0);
	}

	const char *ip = argv[1];
	int port = atoi(argv[2]);
	int connections = 0;
	char buffer[128] = {0};
	int i = 0, index = 0;

	struct epoll_event events[MAX_EPOLLSIZE];
	
	int epoll_fd = epoll_create(MAX_EPOLLSIZE);
	
	strcpy(buffer, " Data From MulClient\n");
		
	struct sockaddr_in addr;
	memset(&addr, 0, sizeof(struct sockaddr_in));
	
	addr.sin_family = AF_INET;
	addr.sin_addr.s_addr = inet_addr(ip);

	struct timeval tv_begin;
	gettimeofday(&tv_begin, NULL);

	while (1) {
		if (++index >= MAX_PORT) index = 0;
		
		struct epoll_event ev;
		int sockfd = 0;

		if (connections < 512000 && !isContinue) {
			sockfd = socket(AF_INET, SOCK_STREAM, 0);
			if (sockfd == -1) {
				perror("socket");
				goto err;
			}

			//ntySetReUseAddr(sockfd);
			addr.sin_port = htons(port+index);

			if (connect(sockfd, (struct sockaddr*)&addr, sizeof(struct sockaddr_in)) < 0) {
				perror("connect");
				goto err;
			}
			ntySetNonblock(sockfd);
			ntySetReUseAddr(sockfd);

			sprintf(buffer, "Hello Server: client --> %d\n", connections);
			send(sockfd, buffer, strlen(buffer), 0);

			ev.data.fd = sockfd;
			ev.events = EPOLLIN | EPOLLOUT;
			epoll_ctl(epoll_fd, EPOLL_CTL_ADD, sockfd, &ev);
		
			connections ++;
		}
		//connections ++;
		if (connections % 1000 == 999 || connections >= 512000) {
			struct timeval tv_cur;
			memcpy(&tv_cur, &tv_begin, sizeof(struct timeval));
			
			gettimeofday(&tv_begin, NULL);

			int time_used = TIME_SUB_MS(tv_begin, tv_cur);
			printf("connections: %d, sockfd:%d, time_used:%d\n", connections, sockfd, time_used);

			int nfds = epoll_wait(epoll_fd, events, connections, 100);
			for (i = 0;i < nfds;i ++) {
				int clientfd = events[i].data.fd;

				if (events[i].events & EPOLLOUT) {
					sprintf(buffer, "data from %d\n", clientfd);
					send(sockfd, buffer, strlen(buffer), 0);
				} else if (events[i].events & EPOLLIN) {
					char rBuffer[MAX_BUFFER] = {0};				
					ssize_t length = recv(sockfd, rBuffer, MAX_BUFFER, 0);
					if (length > 0) {
						printf(" RecvBuffer:%s\n", rBuffer);

						if (!strcmp(rBuffer, "quit")) {
							isContinue = 0;
						}
						
					} else if (length == 0) {
						printf(" Disconnect clientfd:%d\n", clientfd);
						connections --;
						close(clientfd);
					} else {
						if (errno == EINTR) continue;

						printf(" Error clientfd:%d, errno:%d\n", clientfd, errno);
						close(clientfd);
					}
				} else {
					printf(" clientfd:%d, errno:%d\n", clientfd, errno);
					close(clientfd);
				}
			}
		}

		usleep(500);
	}

	return 0;

err:
	printf("error : %s\n", strerror(errno));
	return 0;
	
}


三、系统配置

只写代码当然还不够,我们需要修改Linux内核参数才行哈!

1、首先我们需要单个进程支持的文件描述符fd的个数:
使用 ulimit -n 1048576,可临时更改,生效范围为当前会话
永久修改的方法:

vim /etc/security/limits.conf
添加:
soft nofile 1048576
hard nofile 1048576

再修改所有进程支持的文件描述符个数:

echo 1048576 > /proc/sys/fs/file-max

2、修改Linux内核协议栈的相关参数

echo 786432 2097152 3145728 > /proc/sys/net/ipv4/tcp_mem
echo 4096 4096 16777216 > /proc/sys/net/ipv4/tcp_rmem
echo 4096 4096 16777216 > /proc/sys/net/ipv4/tcp_wmem
echo 16384 > /proc/sys/net/ipv4/tcp_max_orphans
echo 1048576> /proc/sys/net/nf_conntrack_max
echo 1200> /proc/sys/net/netfilter/nf_conntrack_tcp_timeout_established

修改完之后,可以使用sysctl -a 查看内核参数是否已经改变了!
关于这些参数的含义我给出下面一些参考链接:
《关于高负载服务器Kernel的TCP参数优化》
《百万并发之 tcp_mem》

四、并发测试

设置完参数之后,我们准备三个虚拟机,一个作为服务端,其他作为客户端,客户端代码会一直创建socket连接到512000个连接,两个客户端一共创建1024000个连接。
其中服务端虚拟机内存是6个G,客户端虚拟机内存为2个G。
允许情况如下:
在这里插入图片描述
由于我是在一台笔记本上进行的测试,在连接达到90w+的时候,内存已经爆满(无奈摊手),只能强行kill掉进程了!
在这里插入图片描述

五、后续

本篇文章关于单线程百万并发的测试就告一段落了,代码中还有挺多不足的哈,欢迎各位同学提出哈!

评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包

打赏作者

阿杰的小鱼塘

你的鼓励将是我创作的最大动力

¥1 ¥2 ¥4 ¥6 ¥10 ¥20
扫码支付:¥1
获取中
扫码支付

您的余额不足,请更换扫码支付或充值

打赏作者

实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值