一、前情提要
在上一篇文章中（Linux高并发学习—epoll的reactor实现），已经讲述了epoll的reactor实现方式，本篇文章将沿用代码实现单机百万并发的实现。
服务端代码增加了一个TCP连接的链表，用于支持大量TCP连接；客户端代码则是循环创建socket连接。话不多说，咱直接看代码！
二、代码细节
服务端之所以能支持上百万个TCP连接，离不开五元组的支持，其中源端口开放了100个，而客户端的作为目的端口也有上万个端口能够正常使用，排列组合下就有100万以上啦！
#include <stdio.h>
#include <ctype.h>
#include <unistd.h>
#include <fcntl.h>
#include <sys/types.h>
#include <arpa/inet.h>
#include <sys/socket.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>
#include <sys/select.h>
#include <sys/poll.h>
#include <sys/epoll.h>
#include <pthread.h>
#define SERVER_PORT 9000
#define BUFFER_SIZE 1024
#define MAX_EPOLL_SIZE 512
#define MAX_LISTEN_BAGS 512
#define INIT 0
#define ACCEPT 1
#define READ 2
#define WRITE 3
typedef int (*callBack)(int, int, void*);
typedef struct _event_item
{
int fd;
int events;
void *args;
callBack rhandle; // 读事件回调
callBack whandle; // 写事件回调
unsigned char sendBuf[BUFFER_SIZE];
int sendLen;
unsigned char recvBuf[BUFFER_SIZE];
int recvLen;
} event_item;
typedef struct _block
{
event_item *events;
struct _block *next;
}event_block;
typedef struct _reactor
{
int epollfd;
int blockCnt;
event_block *blk; // 链表，用于存储百万TCP连接
}reactor;
int acceptCB(int fd, int events, void* args);
int readCB(int fd, int events, void* args);
int writeCB(int fd, int events, void* args);
int init_reactor(reactor *r);
reactor *instance = NULL;
reactor *getInstance(){
if(instance == NULL){
instance = (reactor*)malloc(sizeof(reactor));
if(instance == NULL)
return NULL;
memset(instance, 0, sizeof(reactor));
if(init_reactor(instance) < 0){
free(instance);
return NULL;
}
}
return instance;
}
int init_socket(short port){
int reuseAddr = 1;
int reusePort = 1;
int fd = socket(AF_INET, SOCK_STREAM, 0);
if(fd < 0)
{
printf("errno: %s, socket() failed!\n", strerror(errno));
return -1;
}
if(fcntl(fd, F_SETFL, O_NONBLOCK) < 0){
printf("errno: %s, fcntl() failed!\n", strerror(errno));
return -1;
}
struct sockaddr_in serverAddr;
memset(&serverAddr, 0, sizeof(serverAddr));
serverAddr.sin_family = AF_INET;
serverAddr.sin_port = htons(port);
serverAddr.sin_addr.s_addr = htonl(INADDR_ANY);
setsockopt(fd, SOL_SOCKET, SO_REUSEPORT, (const void*)&reusePort, sizeof(reusePort));
setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, (const void*)&reuseAddr, sizeof(reuseAddr));
if(bind(fd, (struct sockaddr*)&serverAddr, sizeof(serverAddr)) < 0){
printf("errno: %s, bind() failed!\n", strerror(errno));
return -1;
}
if(listen(fd, MAX_LISTEN_BAGS) < 0){
printf("errno: %s, listen() failed!\n", strerror(errno));
return -1;
}
return fd;
}
int block_alloc(){
reactor *r = getInstance();
if(r == NULL){
printf("block_alloc(): getInstance() failed!\n");
return -1;
}
if(r->blk == NULL){
printf("r->blk == NULL!\n");
return -1;
}
event_item *ev = (event_item*)malloc((MAX_EPOLL_SIZE) * sizeof(event_item));
if(ev == NULL){
printf("block_alloc(): malloc() failed!\n");
return -1;
}
memset(ev, 0, (MAX_EPOLL_SIZE) * sizeof(event_item));
event_block *blk = (event_block*)malloc(sizeof(event_block));
if(blk == NULL){
printf("block_alloc(): malloc() failed!\n");
return -1;
}
blk->events = ev;
blk->next = NULL;
event_block *p = r->blk;
while(p->next){
p = p->next;
}
p->next = blk;
r->blockCnt ++;
return 0;
}
int init_reactor(reactor *r){
if(r == NULL) return -1;
memset(r, 0, sizeof(reactor));
r->epollfd = epoll_create(1);
if(r->epollfd <= 0){
printf("errno: %s, epoll_create() failed!\n", strerror(errno));
return -1;
}
event_block *blk = (event_block*)malloc(sizeof(event_block));
if(blk == NULL){
printf("init_reactor() malloc failed!\n");
return -1;
}
event_item *item = (event_item*)malloc((MAX_EPOLL_SIZE) * sizeof(event_item));
if(item == NULL){
printf("init_reactor() malloc failed!\n");
return -1;
}
memset(item, 0, (MAX_EPOLL_SIZE) * sizeof(event_item));
blk->events = item;
blk->next = NULL;
r->blk = blk;
r->blockCnt = 1;
return 0;
}
// 根据提供的fd从链表中查询对应的event_item结构
event_item *find_events(int fd){
reactor *r = getInstance();
if(r == NULL){
printf("find_events(): getInstance() failed!\n");
return NULL;
}
int blkidx = fd / MAX_EPOLL_SIZE;
while(blkidx >= r->blockCnt){ // 要查询的fd大于目前的容量，需要扩容
block_alloc();
}
event_block *cur = r->blk;
int cnt = 0;
while(cnt++ < blkidx && cur != NULL){
cur = cur->next;
}
return &(cur->events[fd % MAX_EPOLL_SIZE]);
}
int destory_reactor(){
reactor *r = getInstance();
if(r == NULL){
printf("destory_reactor(): getInstance() failed!\n");
return -1;
}
close(r->epollfd);
event_block *cur = r->blk;
event_block *nxt = NULL;
while(cur != NULL){
nxt = cur->next;
free(cur->events);
free(cur);
cur = nxt->next;
}
return 0;
}
int del_reactor_events(int fd){
reactor *r = getInstance();
if(r == NULL){
printf("del_reactor_events(): getInstance() failed!\n");
return -1;
}
struct epoll_event ev;
ev.data.ptr = NULL;
epoll_ctl(r->epollfd, EPOLL_CTL_DEL, fd, &ev);
return 0;
}
int set_reactor_events(int fd, int event, void *args){
struct epoll_event ev;
event_item *item = find_events(fd);
reactor *r = getInstance();
if(r == NULL){
printf("set_reactor_events(): getInstance() failed!\n");
return -1;
}
if(event == ACCEPT){
item->fd = fd;
item->args = args;
item->rhandle = acceptCB;
ev.events = EPOLLIN;
}else if(event == READ){
item->fd = fd;
item->args = args;
item->rhandle = readCB;
ev.events = EPOLLIN;
// ev.events |= EPOLLET;
}else if(event == WRITE){
item->fd = fd;
item->args = args;
item->whandle = writeCB;
ev.events = EPOLLOUT;
}
ev.data.ptr = item;
if(item->events == INIT){
epoll_ctl(r->epollfd, EPOLL_CTL_ADD, fd, &ev);
item->events = event;
}else if(item->events != event){
epoll_ctl(r->epollfd, EPOLL_CTL_MOD, fd, &ev);
item->events = event;
}
return 0;
}
int acceptCB(int fd, int events, void* args){
struct sockaddr_in clientAdr;
socklen_t len = sizeof(clientAdr);
int connfd = accept(fd, (struct sockaddr*)&clientAdr, &len);
if(fcntl(connfd, F_SETFL, O_NONBLOCK) < 0){
printf("errno: %s, fcntl() failed!\n", strerror(errno));
return -1;
}
printf("new connection: %d\n", connfd);
set_reactor_events(connfd, READ, args);
return 0;
}
int readCB(int fd, int events, void* args){
reactor *r = getInstance();
if(r == NULL){
printf("readCB(): getInstance() failed!\n");
return -1;
}
event_item *item = find_events(fd);
unsigned char *rbuf = item->recvBuf;
int n = recv(fd, rbuf, BUFFER_SIZE, 0);
if(n == 0){
printf("errno: %s, clientfd %d closed!\n", strerror(errno), fd);
del_reactor_events(fd);
close(fd);
return -1;
}else if(n < 0){
if(errno != EINTR && errno != EWOULDBLOCK && errno != EAGAIN){
printf("errno: %s, error!\n", strerror(errno));
del_reactor_events(fd);
close(fd);
return -1;
}
}
else{
unsigned char *sbuf = item->sendBuf;
memcpy(sbuf, rbuf, n);
item->sendLen = n;
printf("recv from fd = %d: %s\n", fd, sbuf);
set_reactor_events(fd, WRITE, args);
}
return 0;
}
int writeCB(int fd, int events, void* args){
reactor *r = getInstance();
if(r == NULL){
printf("reactor_loop(): getInstance() failed!\n");
return -1;
}
event_item *item = find_events(fd);
unsigned char *sbuf = item->sendBuf;
int len = item->sendLen;
int ret = send(fd, sbuf, len, 0);
if(ret < len){
if(ret == 0){
del_reactor_events(fd);
close(fd);
}
set_reactor_events(fd, WRITE, args);
}else{
set_reactor_events(fd, READ, args);
}
return 0;
}
int reactor_loop(){
struct epoll_event events[MAX_EPOLL_SIZE] = {0};
reactor *r = getInstance();
if(r == NULL){
printf("reactor_loop(): getInstance() failed!\n");
return -1;
}
while(1){
int nready = epoll_wait(r->epollfd, events, MAX_EPOLL_SIZE, -1);
if(nready == -1)
continue;
for(int i = 0; i < nready; ++i){
event_item *item = (event_item *)events[i].data.ptr;
if(events[i].events & EPOLLIN)
(*item->rhandle)(item->fd, 0, NULL);
if(events[i].events & EPOLLOUT)
(*item->whandle)(item->fd, 0, NULL);
}
}
return 0;
}
int main(int argc, char *argv[])
{
int sockfd = 0;
int port = SERVER_PORT;
for (int i = port; i < port + 100; i++)
{
sockfd = init_socket(i);
if(sockfd < 0)
return -1;
set_reactor_events(sockfd, ACCEPT, NULL);
}
// sockfd = init_socket(SERVER_PORT);
// if(sockfd < 0)
// return -1;
// set_reactor_events(sockfd, ACCEPT, NULL);
reactor_loop();
return 0;
}
客户端的代码则很简单了，一直循环创建连接就OK了，如下：
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <sys/epoll.h>
#include <errno.h>
#include <netinet/tcp.h>
#include <arpa/inet.h>
#include <netdb.h>
#include <fcntl.h>
#define MAX_BUFFER 128
#define MAX_EPOLLSIZE (512*1024)
#define MAX_PORT 100
#define TIME_SUB_MS(tv1, tv2) ((tv1.tv_sec - tv2.tv_sec) * 1000 + (tv1.tv_usec - tv2.tv_usec) / 1000)
int isContinue = 0;
static int ntySetNonblock(int fd) {
int flags;
flags = fcntl(fd, F_GETFL, 0);
if (flags < 0) return flags;
flags |= O_NONBLOCK;
if (fcntl(fd, F_SETFL, flags) < 0) return -1;
return 0;
}
static int ntySetReUseAddr(int fd) {
int reuse = 1;
return setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, (char *)&reuse, sizeof(reuse));
}
int main(int argc, char **argv) {
if (argc <= 2) {
printf("Usage: %s ip port\n", argv[0]);
exit(0);
}
const char *ip = argv[1];
int port = atoi(argv[2]);
int connections = 0;
char buffer[128] = {0};
int i = 0, index = 0;
struct epoll_event events[MAX_EPOLLSIZE];
int epoll_fd = epoll_create(MAX_EPOLLSIZE);
strcpy(buffer, " Data From MulClient\n");
struct sockaddr_in addr;
memset(&addr, 0, sizeof(struct sockaddr_in));
addr.sin_family = AF_INET;
addr.sin_addr.s_addr = inet_addr(ip);
struct timeval tv_begin;
gettimeofday(&tv_begin, NULL);
while (1) {
if (++index >= MAX_PORT) index = 0;
struct epoll_event ev;
int sockfd = 0;
if (connections < 512000 && !isContinue) {
sockfd = socket(AF_INET, SOCK_STREAM, 0);
if (sockfd == -1) {
perror("socket");
goto err;
}
//ntySetReUseAddr(sockfd);
addr.sin_port = htons(port+index);
if (connect(sockfd, (struct sockaddr*)&addr, sizeof(struct sockaddr_in)) < 0) {
perror("connect");
goto err;
}
ntySetNonblock(sockfd);
ntySetReUseAddr(sockfd);
sprintf(buffer, "Hello Server: client --> %d\n", connections);
send(sockfd, buffer, strlen(buffer), 0);
ev.data.fd = sockfd;
ev.events = EPOLLIN | EPOLLOUT;
epoll_ctl(epoll_fd, EPOLL_CTL_ADD, sockfd, &ev);
connections ++;
}
//connections ++;
if (connections % 1000 == 999 || connections >= 512000) {
struct timeval tv_cur;
memcpy(&tv_cur, &tv_begin, sizeof(struct timeval));
gettimeofday(&tv_begin, NULL);
int time_used = TIME_SUB_MS(tv_begin, tv_cur);
printf("connections: %d, sockfd:%d, time_used:%d\n", connections, sockfd, time_used);
int nfds = epoll_wait(epoll_fd, events, connections, 100);
for (i = 0;i < nfds;i ++) {
int clientfd = events[i].data.fd;
if (events[i].events & EPOLLOUT) {
sprintf(buffer, "data from %d\n", clientfd);
send(sockfd, buffer, strlen(buffer), 0);
} else if (events[i].events & EPOLLIN) {
char rBuffer[MAX_BUFFER] = {0};
ssize_t length = recv(sockfd, rBuffer, MAX_BUFFER, 0);
if (length > 0) {
printf(" RecvBuffer:%s\n", rBuffer);
if (!strcmp(rBuffer, "quit")) {
isContinue = 0;
}
} else if (length == 0) {
printf(" Disconnect clientfd:%d\n", clientfd);
connections --;
close(clientfd);
} else {
if (errno == EINTR) continue;
printf(" Error clientfd:%d, errno:%d\n", clientfd, errno);
close(clientfd);
}
} else {
printf(" clientfd:%d, errno:%d\n", clientfd, errno);
close(clientfd);
}
}
}
usleep(500);
}
return 0;
err:
printf("error : %s\n", strerror(errno));
return 0;
}
三、系统配置
只写代码当然还不够，我们需要修改Linux内核参数才行哈！
1、首先我们需要单个进程支持的文件描述符fd的个数：
使用 ulimit -n 1048576，可临时更改，生效范围为当前会话
永久修改的方法：
vim /etc/security/limits.conf
添加：
soft nofile 1048576
hard nofile 1048576
再修改所有进程支持的文件描述符个数：
echo 1048576 > /proc/sys/fs/file-max
2、修改Linux内核协议栈的相关参数
echo 786432 2097152 3145728 > /proc/sys/net/ipv4/tcp_mem
echo 4096 4096 16777216 > /proc/sys/net/ipv4/tcp_rmem
echo 4096 4096 16777216 > /proc/sys/net/ipv4/tcp_wmem
echo 16384 > /proc/sys/net/ipv4/tcp_max_orphans
echo 1048576> /proc/sys/net/nf_conntrack_max
echo 1200> /proc/sys/net/netfilter/nf_conntrack_tcp_timeout_established
修改完之后，可以使用sysctl -a 查看内核参数是否已经改变了！
关于这些参数的含义我给出下面一些参考链接：
《关于高负载服务器Kernel的TCP参数优化》
《百万并发之 tcp_mem》
四、并发测试
设置完参数之后，我们准备三个虚拟机，一个作为服务端，其他作为客户端，客户端代码会一直创建socket连接到512000个连接，两个客户端一共创建1024000个连接。
其中服务端虚拟机内存是6个G，客户端虚拟机内存为2个G。
允许情况如下：
由于我是在一台笔记本上进行的测试，在连接达到90w+的时候，内存已经爆满（无奈摊手），只能强行kill掉进程了！
五、后续
本篇文章关于单线程百万并发的测试就告一段落了，代码中还有挺多不足的哈，欢迎各位同学提出哈！
3914
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
