reactor单反应堆实现百万并发
1.对并发概念的理解
服务器并发概念:服务器同时能承载的客户端数量。
服务器并发的基本要求:
(1)5w以上的响应请求
(2)cpu占用率在60%左右
(3)内存占用率在80%左右
对并发理解的误区:
(1)QPS:单位时间内能够处理请求的数量
(2)服务器处理客户端请求的数量 -----> 没有限定时间
2.五元组保证唯一的网络 IO
五元组定义 | 描述 |
---|---|
源IP | 客户端IP |
目的IP | 服务端IP |
源端口 | 客户端port:0~65535 |
目的端口 | 服务端port:0~65535 |
PROTOCOL | 协议类型:TCP/UDP |
每一个五元组就能确定一个网络IO
查看端口范围:cat /proc/sys/net/ipv4/ip_local_port_range
线上的环境:一般是服务器绑定一个端口,每个客户端IP都是不一样的。
局域网测试环境:由于PC机的性能限制,只能采用服务器监听多个端口,几个客户端IP连接多个服务器端口的方式。
3.测试环境的要求
(1)方案一:
三台ubuntu虚拟机、内存每台分配4GB
一台做服务器(监听100个端口),其中两台做客户端,每个客户端连接50万
(2)方案二:
四台ubuntu虚拟机,服务器内存4GB(监听100个端口)
三台客户端分配3GB内存,每台连接34万
4.限制网络IO的几大因素
(1)open files (进程文件最大打开个数)虚拟机默认1024
查看open files的方法:ulimit -a
修改open files的方法:
- 临时修改:ulimit -n 修改的值
- 永久修改:vim /etc/security/limits.conf
(2)file-max(系统文件最大个数)默认1024
修改file-max:vim /etc/sysctl.conf
生效指令:sudo sysctl -p
(3)nf_conntrack_max(三次握手最后一次最大连接数)默认不开启
-
不开启时执行sudo sysctl -p:
提示没有nf_conntrack_max
- 开启modprobe ip_conntrack模块使nf_conntrack_max生效:
开启命令:sudo modprobe ip_conntrack
5.如何将客户端每1000个连接的耗时变短
(1)多个accept放在一个线程处理——本质没有变化,耗时还是一样
(2)多个accept放在多个线程处理——通过每个线程listen多个port方式处理,可以缩短耗时
(3)多个accept放在多个进程处理——nginx的处理方式,没有fd限制,适合前后连接没有关联关系的场景
多线程和多进程处理的区别:
- 多进程不需要加锁,每个进程独享资源
- 多进程fd的上限比多线程多得多
单服务器模型:
6.C10M问题的突破
(1)早期C10k的突破得益于epoll的引入,epoll的剖析后期会有专门的专题
(2)目前服务器在epoll的reactor模型下,达到C1000k已经不是问题了,本文的测试就是针对C1000k的测试做的专题
(3)C10M的问题需要引入用户态协议栈,做到零拷贝的方式,直接从网卡映射出虚拟内存,绕过内核直达应用层,减少内核和用户态之间拷贝数据所耗的资源。
7.高性能服务器的五大维度
影响服务性能的五元组:
- CPU
- 内存
- 网络带宽(磁盘)
- 操作系统
- 应用程序
设计高性能服务器从四个方面考虑:
(1)CPU粘合:业务分离,多CPU多进程方式,每个CPU处理一种业务(sched_setaffinity可以绑定CPU和进程的亲和性)
(2)网络IO:用户态协议栈、mmap
(3)磁盘:文件系统的磁盘IO/网络IO,统一用协程调度
(4)内存:内存池管理内存(避免频繁的创建和销毁内存,内存创建大小不一,造成内存的碎片化)
8.百万并发测试服务器程序和客户端程序
服务器程序rector.c:
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/socket.h>
#include <sys/epoll.h>
#include <arpa/inet.h>
#include <fcntl.h>
#include <unistd.h>
#include <errno.h>
#define BUFFER_LENGTH 1024
#define MAX_EPOLL_EVENTS 1024*1024 //connection
#define MAX_EPOLL_ITEM 102400 //con
#define SERVER_PORT 8888
#define LISTEN_PORT_COUNT 100
typedef int NCALLBACK(int ,int, void*);
struct ntyevent {
int fd;
int events;
void *arg;
int (*callback)(int fd, int events, void *arg);
int status;
char buffer[BUFFER_LENGTH];
int length;
long last_active;
};
struct ntyreactor {
int epfd;
struct ntyevent *events; // 1024 * 1024
};
int recv_cb(int fd, int events, void *arg);
int send_cb(int fd, int events, void *arg);
void nty_event_set(struct ntyevent *ev, int fd, NCALLBACK callback, void *arg) {
ev->fd = fd;
ev->callback = callback;
ev->events = 0;
ev->arg = arg;
ev->last_active = time(NULL);
return ;
}
int nty_event_add(int epfd, int events, struct ntyevent *ev) {
struct epoll_event ep_ev = {0, {0}};
ep_ev.data.ptr = ev;
ep_ev.events = ev->events = events;
int op;
if (ev->status == 1) {
op = EPOLL_CTL_MOD;
} else {
op = EPOLL_CTL_ADD;
ev->status = 1;
}
if (epoll_ctl(epfd, op, ev->fd, &ep_ev) < 0) {
printf("event add failed [fd=%d], events[%d]\n", ev->fd, events);
return -1;
}
return 0;
}
int nty_event_del(int epfd, struct ntyevent *ev) {
struct epoll_event ep_ev = {0, {0}};
if (ev->status != 1) {
return -1;
}
ep_ev.data.ptr = ev;
ev->status = 0;
epoll_ctl(epfd, EPOLL_CTL_DEL, ev->fd, &ep_ev);
return 0;
}
int recv_cb(int fd, int events, void *arg) {
struct ntyreactor *reactor = (struct ntyreactor*)arg;
struct ntyevent *ev = reactor->events+fd;
int len = recv(fd, ev->buffer, BUFFER_LENGTH, 0);
nty_event_del(reactor->epfd, ev);
if (len > 0) {
ev->length = len;
ev->buffer[len] = '\0';
printf("C[%d]:%s\n", fd, ev->buffer);
nty_event_set(ev, fd, send_cb, reactor);
nty_event_add(reactor->epfd, EPOLLOUT, ev);
} else if (len == 0) {
close(ev->fd);
printf("[fd=%d] pos[%ld], closed\n", fd, ev-reactor->events);
} else {
close(ev->fd);
printf("recv[fd=%d] error[%d]:%s\n", fd, errno, strerror(errno));
}
return len;
}
int send_cb(int fd, int events, void *arg) {
struct ntyreactor *reactor = (struct ntyreactor*)arg;
struct ntyevent *ev = reactor->events+fd;
int len = send(fd, ev->buffer, ev->length, 0);
if (len > 0) {
printf("send[fd=%d], [%d]%s\n", fd, len, ev->buffer);
nty_event_del(reactor->epfd, ev);
nty_event_set(ev, fd, recv_cb, reactor);
nty_event_add(reactor->epfd, EPOLLIN, ev);
} else {
close(ev->fd);
nty_event_del(reactor->epfd, ev);
printf("send[fd=%d] error %s\n", fd, strerror(errno));
}
return len;
}
int accept_cb(int fd, int events, void *arg) {
struct ntyreactor *reactor = (struct ntyreactor*)arg;
if (reactor == NULL) return -1;
struct sockaddr_in client_addr;
socklen_t len = sizeof(client_addr);
int clientfd;
if ((clientfd = accept(fd, (struct sockaddr*)&client_addr, &len)) == -1) {
if (errno != EAGAIN && errno != EINTR) {
}
printf("accept: %s\n", strerror(errno));
return -1;
}
int i = 0;
do {
#if 0
for (i = 0;i < MAX_EPOLL_EVENTS;i ++) {
if (reactor->events[i].status == 0) {
break;
}
}
if (i == MAX_EPOLL_EVENTS) {
printf("%s: max connect limit[%d]\n", __func__, MAX_EPOLL_EVENTS);
break;
}
#endif
int flag = 0;
if ((flag = fcntl(clientfd, F_SETFL, O_NONBLOCK)) < 0) {
printf("%s: fcntl nonblocking failed, %d\n", __func__, MAX_EPOLL_EVENTS);
break;
}
nty_event_set(&reactor->events[clientfd], clientfd, recv_cb, reactor);
nty_event_add(reactor->epfd, EPOLLIN, &reactor->events[clientfd]);
} while (0);
printf("new connect [%s:%d][time:%ld], pos[%d]\n",
inet_ntoa(client_addr.sin_addr), ntohs(client_addr.sin_port), reactor->events[i].last_active, i);
return 0;
}
int init_sock(short port) {
int fd = socket(AF_INET, SOCK_STREAM, 0);
fcntl(fd, F_SETFL, O_NONBLOCK);
struct sockaddr_in server_addr;
memset(&server_addr, 0, sizeof(server_addr));
server_addr.sin_family = AF_INET;
server_addr.sin_addr.s_addr = htonl(INADDR_ANY);
server_addr.sin_port = htons(port);
bind(fd, (struct sockaddr*)&server_addr, sizeof(server_addr));
if (listen(fd, 20) < 0) {
printf("listen failed : %s\n", strerror(errno));
}
printf("listen port : %d\n", port);
return fd;
}
int ntyreactor_init(struct ntyreactor *reactor) {
if (reactor == NULL) return -1;
memset(reactor, 0, sizeof(struct ntyreactor));
reactor->epfd = epoll_create(1);
if (reactor->epfd <= 0) {
printf("create epfd in %s err %s\n", __func__, strerror(errno));
return -2;
}
reactor->events = (struct ntyevent*)malloc((MAX_EPOLL_EVENTS) * sizeof(struct ntyevent));
if (reactor->events == NULL) {
printf("create epfd in %s err %s\n", __func__, strerror(errno));
close(reactor->epfd);
return -3;
}
}
int ntyreactor_destory(struct ntyreactor *reactor) {
close(reactor->epfd);
free(reactor->events);
}
int ntyreactor_addlistener(struct ntyreactor *reactor, int sockfd, NCALLBACK *acceptor) {
if (reactor == NULL) return -1;
if (reactor->events == NULL) return -1;
nty_event_set(&reactor->events[sockfd], sockfd, acceptor, reactor);
nty_event_add(reactor->epfd, EPOLLIN, &reactor->events[sockfd]);
return 0;
}
int ntyreactor_run(struct ntyreactor *reactor) {
if (reactor == NULL) return -1;
if (reactor->epfd < 0) return -1;
if (reactor->events == NULL) return -1;
struct epoll_event events[MAX_EPOLL_ITEM];
int checkpos = 0, i;
while (1) {
#if 0
long now = time(NULL);
for (i = 0;i < 100;i ++, checkpos ++) {
if (checkpos == MAX_EPOLL_EVENTS) {
checkpos = 0;
}
if (reactor->events[checkpos].status != 1) {
continue;
}
long duration = now - reactor->events[checkpos].last_active;
if (duration >= 60) {
close(reactor->events[checkpos].fd);
printf("[fd=%d] timeout\n", reactor->events[checkpos].fd);
nty_event_del(reactor->epfd, &reactor->events[checkpos]);
}
}
#endif
int nready = epoll_wait(reactor->epfd, events, MAX_EPOLL_ITEM, 1000);
if (nready < 0) {
printf("epoll_wait error, exit\n");
continue;
}
for (i = 0;i < nready;i ++) {
struct ntyevent *ev = (struct ntyevent*)events[i].data.ptr;
if ((events[i].events & EPOLLIN) && (ev->events & EPOLLIN)) {
ev->callback(ev->fd, events[i].events, ev->arg);
}
if ((events[i].events & EPOLLOUT) && (ev->events & EPOLLOUT)) {
ev->callback(ev->fd, events[i].events, ev->arg);
}
}
}
}
int main(int argc, char *argv[]) {
unsigned short port = SERVER_PORT;
if (argc == 2) {
port = atoi(argv[1]);
}
struct ntyreactor *reactor = (struct ntyreactor*)malloc(sizeof(struct ntyreactor));
ntyreactor_init(reactor);
int listenfd[LISTEN_PORT_COUNT] = {0};
int i = 0;
for (i = 0;i < LISTEN_PORT_COUNT;i ++) {
listenfd[i] = init_sock(port+i);
ntyreactor_addlistener(reactor, listenfd[i], accept_cb);
}
ntyreactor_run(reactor);
ntyreactor_destory(reactor);
for (i = 0;i < LISTEN_PORT_COUNT;i ++) {
close(listenfd[i]);
}
return 0;
}
客户端程序mul_port_client_epoll.c:
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <sys/epoll.h>
#include <errno.h>
#include <netinet/tcp.h>
#include <arpa/inet.h>
#include <netdb.h>
#include <fcntl.h>
#define MAX_BUFFER 128
#define MAX_EPOLLSIZE (384*1024)
#define MAX_PORT 100
#define TIME_SUB_MS(tv1, tv2) ((tv1.tv_sec - tv2.tv_sec) * 1000 + (tv1.tv_usec - tv2.tv_usec) / 1000)
int isContinue = 0;
static int ntySetNonblock(int fd) {
int flags;
flags = fcntl(fd, F_GETFL, 0);
if (flags < 0) return flags;
flags |= O_NONBLOCK;
if (fcntl(fd, F_SETFL, flags) < 0) return -1;
return 0;
}
static int ntySetReUseAddr(int fd) {
int reuse = 1;
return setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, (char *)&reuse, sizeof(reuse));
}
int main(int argc, char **argv) {
if (argc <= 2) {
printf("Usage: %s ip port\n", argv[0]);
exit(0);
}
const char *ip = argv[1];
int port = atoi(argv[2]);
int connections = 0;
char buffer[128] = {0};
int i = 0, index = 0;
struct epoll_event events[MAX_EPOLLSIZE];
int epoll_fd = epoll_create(MAX_EPOLLSIZE);
strcpy(buffer, " Data From MulClient\n");
struct sockaddr_in addr;
memset(&addr, 0, sizeof(struct sockaddr_in));
addr.sin_family = AF_INET;
addr.sin_addr.s_addr = inet_addr(ip);
struct timeval tv_begin;
gettimeofday(&tv_begin, NULL);
while (1) {
if (++index >= MAX_PORT) index = 0;
struct epoll_event ev;
int sockfd = 0;
if (connections < 500000 && !isContinue) {
sockfd = socket(AF_INET, SOCK_STREAM, 0);
if (sockfd == -1) {
perror("socket");
goto err;
}
//ntySetReUseAddr(sockfd);
addr.sin_port = htons(port+index);
if (connect(sockfd, (struct sockaddr*)&addr, sizeof(struct sockaddr_in)) < 0) {
perror("connect");
goto err;
}
ntySetNonblock(sockfd);
ntySetReUseAddr(sockfd);
sprintf(buffer, "Hello Server: client --> %d\n", connections);
send(sockfd, buffer, strlen(buffer), 0);
ev.data.fd = sockfd;
ev.events = EPOLLIN | EPOLLOUT;
epoll_ctl(epoll_fd, EPOLL_CTL_ADD, sockfd, &ev);
connections ++;
}
//connections ++;
if (connections % 1000 == 999 || connections >= 500000) {
struct timeval tv_cur;
memcpy(&tv_cur, &tv_begin, sizeof(struct timeval));
gettimeofday(&tv_begin, NULL);
int time_used = TIME_SUB_MS(tv_begin, tv_cur);
printf("connections: %d, sockfd:%d, time_used:%d\n", connections, sockfd, time_used);
int nfds = epoll_wait(epoll_fd, events, connections, 100);
for (i = 0;i < nfds;i ++) {
int clientfd = events[i].data.fd;
if (events[i].events & EPOLLOUT) {
sprintf(buffer, "data from %d\n", clientfd);
send(sockfd, buffer, strlen(buffer), 0);
} else if (events[i].events & EPOLLIN) {
char rBuffer[MAX_BUFFER] = {0};
ssize_t length = recv(sockfd, rBuffer, MAX_BUFFER, 0);
if (length > 0) {
printf(" RecvBuffer:%s\n", rBuffer);
if (!strcmp(rBuffer, "quit")) {
isContinue = 0;
}
} else if (length == 0) {
printf(" Disconnect clientfd:%d\n", clientfd);
connections --;
close(clientfd);
} else {
if (errno == EINTR) continue;
printf(" Error clientfd:%d, errno:%d\n", clientfd, errno);
close(clientfd);
}
} else {
printf(" clientfd:%d, errno:%d\n", clientfd, errno);
close(clientfd);
}
}
}
usleep(1 * 1000);
}
return 0;
err:
printf("error : %s\n", strerror(errno));
return 0;
}