目录
什么是并发量？
同时承载客户端的数量。承载是在连接的基础上200ms内能不能对数据库、网络带宽、内存操作、日志等进行操作返回给客户端。
C10K, C1000K, C10M是什么意思
首字母 C 是 Client 的缩写，C10K是处理1万客户端, C1000K是处理100万客户端, C10M是处理1000万客户端
服务端怎么区分客户端的socket？
socket是由五元组：sip(源IP地址),dip(目标IP地址),sport(源端口号),dport(目标端口号),protocol(传输层协议）组成的，每个socket之间的至少有一个不一样
接下来我们来逐步实现百万并发量
开始调试
服务端和客户端都需要4g以上内存，不然比较勉强
客户端代码
在epoll的基础上修改
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <sys/epoll.h>
#include <errno.h>
#include <netinet/tcp.h>
#include <arpa/inet.h>
#include <netdb.h>
#include <fcntl.h>
#define MAX_BUFFER 128
#define MAX_EPOLLSIZE (384*1024)
#define MAX_PORT 0
#define TIME_SUB_MS(tv1, tv2) ((tv1.tv_sec - tv2.tv_sec) * 1000 + (tv1.tv_usec - tv2.tv_usec) / 1000)
int isContinue = 0;
//设置fd非阻塞
static int ntySetNonblock(int fd) {
int flags;
flags = fcntl(fd, F_GETFL, 0);
if (flags < 0) return flags;
flags |= O_NONBLOCK;
if (fcntl(fd, F_SETFL, flags) < 0) return -1;
return 0;
}
//设置端口释放后马上可以使用,不设置的话要等2分钟
static int ntySetReUseAddr(int fd) {
int reuse = 1;
return setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, (char *)&reuse, sizeof(reuse));
}
int main(int argc, char **argv) {
if (argc <= 2) {
printf("Usage: %s ip port\n", argv[0]);
exit(0);
}
const char *ip = argv[1];
int port = atoi(argv[2]);
int connections = 0;
char buffer[128] = {0};
int i = 0, index = 0;
struct epoll_event events[MAX_EPOLLSIZE];
int epoll_fd = epoll_create(MAX_EPOLLSIZE);
strcpy(buffer, " Data From MulClient\n");
struct sockaddr_in addr;
memset(&addr, 0, sizeof(struct sockaddr_in));
addr.sin_family = AF_INET;
addr.sin_addr.s_addr = inet_addr(ip);
struct timeval tv_begin;
gettimeofday(&tv_begin, NULL);
while (1) {
if (++index >= MAX_PORT) index = 0;
struct epoll_event ev;
int sockfd = 0;
if (connections <= 1000000 && !isContinue) {
sockfd = socket(AF_INET, SOCK_STREAM, 0);
if (sockfd == -1) {
perror("socket");
goto err;
}
addr.sin_port = htons(port+index);
if (connect(sockfd, (struct sockaddr*)&addr, sizeof(struct sockaddr_in)) < 0) {
perror("connect");
goto err;
}
//设置fd非阻塞
ntySetNonblock(sockfd);
//设置fd的端口马上可以使用
ntySetReUseAddr(sockfd);
sprintf(buffer, "Hello Server: client --> %d\n", connections);
send(sockfd, buffer, strlen(buffer), 0);
ev.data.fd = sockfd;
ev.events = EPOLLIN | EPOLLOUT;
epoll_ctl(epoll_fd, EPOLL_CTL_ADD, sockfd, &ev);
connections ++;
}
//connections ++;
if (connections % 1000 == 999 || connections >= 340000) {
struct timeval tv_cur;
memcpy(&tv_cur, &tv_begin, sizeof(struct timeval));
gettimeofday(&tv_begin, NULL);
//计算1000连接的时间
int time_used = TIME_SUB_MS(tv_begin, tv_cur);
printf("connections: %d, sockfd:%d, time_used:%d\n", connections, sockfd, time_used);
int nfds = epoll_wait(epoll_fd, events, connections, 100);
for (i = 0;i < nfds;i ++) {
int clientfd = events[i].data.fd;
//发消息
if (events[i].events & EPOLLOUT) {
sprintf(buffer, "data from %d\n", clientfd);
send(sockfd, buffer, strlen(buffer), 0);
} else if (events[i].events & EPOLLIN) {
char rBuffer[MAX_BUFFER] = {0};
ssize_t length = recv(sockfd, rBuffer, MAX_BUFFER, 0);
if (length > 0) {
printf(" RecvBuffer:%s\n", rBuffer);
if (!strcmp(rBuffer, "quit")) {
isContinue = 0;
}
} else if (length == 0) {
printf(" Disconnect clientfd:%d\n", clientfd);
connections --;
close(clientfd);
} else {
if (errno == EINTR) continue;
printf(" Error clientfd:%d, errno:%d\n", clientfd, errno);
close(clientfd);
}
} else {
printf(" clientfd:%d, errno:%d\n", clientfd, errno);
close(clientfd);
}
}
}
//1000微秒
usleep(1 * 1000);
}
return 0;
err:
printf("error : %s\n", strerror(errno));
return 0;
}
服务端代码
就用单线程的reator代码修改一下
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <netinet/tcp.h>
#include <arpa/inet.h>
#include <pthread.h>
#include <errno.h>
#include <sys/epoll.h>
#define BUFFER_LENGTH 1024
struct sockitem { //
int sockfd;
int (*callback)(int fd, int events, void *arg);
char recvbuffer[BUFFER_LENGTH]; //
char sendbuffer[BUFFER_LENGTH];
int rlength;
int slength;
};
struct reactor {
int epfd;
struct epoll_event events[512];
};
struct reactor *eventloop = NULL;
int recv_cb(int fd, int events, void *arg);
int send_cb(int fd, int events, void *arg) {
struct sockitem *si = (struct sockitem*)arg;
send(fd, si->sendbuffer, si->slength, 0); //
struct epoll_event ev;
ev.events = EPOLLIN | EPOLLET;
si->sockfd = fd;
si->callback = recv_cb;
ev.data.ptr = si;
epoll_ctl(eventloop->epfd, EPOLL_CTL_MOD, fd, &ev);
}
int recv_cb(int fd, int events, void *arg) {
struct sockitem *si = (struct sockitem*)arg;
struct epoll_event ev;
int ret = recv(fd, si->recvbuffer, BUFFER_LENGTH, 0);
if (ret < 0) {
if (errno == EAGAIN || errno == EWOULDBLOCK) { //
return -1;
} else {
}
ev.events = EPOLLIN;
epoll_ctl(eventloop->epfd, EPOLL_CTL_DEL, fd, &ev);
close(fd);
free(si);
} else if (ret == 0) {
printf("disconnect %d\n", fd);
ev.events = EPOLLIN;
epoll_ctl(eventloop->epfd, EPOLL_CTL_DEL, fd, &ev);
close(fd);
free(si);
} else {
printf("Recv: %s, %d Bytes\n", si->recvbuffer, ret);
si->rlength = ret;
memcpy(si->sendbuffer, si->recvbuffer, si->rlength);
si->slength = si->rlength;
struct epoll_event ev;
ev.events = EPOLLOUT | EPOLLET;
si->sockfd = fd;
si->callback = send_cb;
ev.data.ptr = si;
epoll_ctl(eventloop->epfd, EPOLL_CTL_MOD, fd, &ev);
}
}
int accept_cb(int fd, int events, void *arg) {
struct sockaddr_in client_addr;
memset(&client_addr, 0, sizeof(struct sockaddr_in));
socklen_t client_len = sizeof(client_addr);
int clientfd = accept(fd, (struct sockaddr*)&client_addr, &client_len);
if (clientfd <= 0) return -1;
char str[INET_ADDRSTRLEN] = {0};
printf("recv from %s at port %d\n", inet_ntop(AF_INET, &client_addr.sin_addr, str, sizeof(str)),
ntohs(client_addr.sin_port));
struct epoll_event ev;
ev.events = EPOLLIN | EPOLLET;
struct sockitem *si = (struct sockitem*)malloc(sizeof(struct sockitem));
si->sockfd = clientfd;
si->callback = recv_cb;
ev.data.ptr = si;
epoll_ctl(eventloop->epfd, EPOLL_CTL_ADD, clientfd, &ev);
return clientfd;
}
int main(int argc, char *argv[]) {
if (argc < 2) {
return -1;
}
int port = atoi(argv[1]);
int sockfd = socket(AF_INET, SOCK_STREAM, 0);
if (sockfd < 0) {
return -1;
}
struct sockaddr_in addr;
memset(&addr, 0, sizeof(struct sockaddr_in));
addr.sin_family = AF_INET;
addr.sin_port = htons(port);
addr.sin_addr.s_addr = INADDR_ANY;
if (bind(sockfd, (struct sockaddr*)&addr, sizeof(struct sockaddr_in)) < 0) {
return -2;
}
if (listen(sockfd, 5) < 0) {
return -3;
}
eventloop = (struct reactor*)malloc(sizeof(struct reactor));
eventloop->epfd = epoll_create(1);
struct epoll_event ev;
ev.events = EPOLLIN;
struct sockitem *si = (struct sockitem*)malloc(sizeof(struct sockitem));
si->sockfd = sockfd;
si->callback = accept_cb;
ev.data.ptr = si;
epoll_ctl(eventloop->epfd, EPOLL_CTL_ADD, sockfd, &ev);
while (1) {
int nready = epoll_wait(eventloop->epfd, eventloop->events, 512, -1);
if (nready < -1) {
break;
}
int i = 0;
for (i = 0;i < nready;i ++) {
if (eventloop->events[i].events & EPOLLIN) {
struct sockitem *si = (struct sockitem*)eventloop->events[i].data.ptr;
si->callback(si->sockfd, eventloop->events[i].events, si);
}
if (eventloop->events[i].events & EPOLLOUT) {
struct sockitem *si = (struct sockitem*)eventloop->events[i].data.ptr;
si->callback(si->sockfd, eventloop->events[i].events, si);
}
}
}
}
运行过程中可以在终端输入htop查看内存使用情况
问题1
客户端在连接数999和socket数1002就提示文件数超过
服务端提示的是socket创建到1023断开
在linux终端输入ulimit -a可以查看到各种限制信息，可以看到open files是1024个，那么就要修改服务端和客户端的open files数
把服务端的最大文件数改为100W，ulimit修改是临时的重启系统就会恢复，要永久修改就要在/etc/security/limits.conf
临时修改：
永久修改：
要注意soft limit必须小于hard limit，重启生效
现在服务端没显示断开，客户端显示断开，那么继续修改客户端的最大文件数为100W
问题2
在客户端在连接数28000左右断开了
提示Cannot assign requested address是客户端连接端口用完了
在终端输入sysctl -a |grep port_range可以看到可用端口数是32768~60999，约2.8W个端口
因为这里我是一个服务端和一个客户端并发连接，如果要实现百万并发的话服务端和客户端就需要100W个socket套接字，即源IP * 目标IP * 源端口 * 目标端口 >= 100W（因为我这里只用一个服务端和一个客户端，socket的五元组只有端口变化，IP地址没有变化）。这里端口大概2.8W个，那么服务端和客户端大概设置100个listen端口数进行连接(端口可以复用,socket不能共用)。
100端口客户端代码
原来的客户端代码的MAX_PORT定义改为100个
服务端设置100个listen端口
100端口服务端代码
进行了亿点的代码优化
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/socket.h>
#include <sys/epoll.h>
#include <arpa/inet.h>
#include <fcntl.h>
#include <unistd.h>
#include <errno.h>
#define BUFFER_LENGTH 1024
//处理100W连接事件
#define MAX_EPOLL_EVENTS 1024*1024 //connection
//处理10W收发事件
#define MAX_EPOLL_ITEM 102400 //con
#define SERVER_PORT 8888
#define LISTEN_PORT_COUNT 100 //源端口加上100个，因为端口数最多65535
typedef int NCALLBACK(int ,int, void*);
struct ntyevent {
int fd;
int events;
void *arg;
int (*callback)(int fd, int events, void *arg);
int status;
char buffer[BUFFER_LENGTH];
int length;
long last_active;
};
struct ntyreactor {
int epfd;
struct ntyevent *events; // 1024 * 1024
};
int recv_cb(int fd, int events, void *arg);
int send_cb(int fd, int events, void *arg);
//事件设置
void nty_event_set(struct ntyevent *ev, int fd, NCALLBACK callback, void *arg) {
ev->fd = fd;
ev->callback = callback;
ev->events = 0;
ev->arg = arg;
ev->last_active = time(NULL);
}
//添加事件
int nty_event_add(int epfd, int events, struct ntyevent *ev) {
struct epoll_event ep_ev = {0, {0}};
ep_ev.data.ptr = ev;
ep_ev.events = ev->events = events;
int op;
if (ev->status == 1) {
op = EPOLL_CTL_MOD;
} else {
op = EPOLL_CTL_ADD;
ev->status = 1;
}
if (epoll_ctl(epfd, op, ev->fd, &ep_ev) < 0) {
printf("event add failed [fd=%d], events[%d]\n", ev->fd, events);
return -1;
}
return 0;
}
//删除事件
int nty_event_del(int epfd, struct ntyevent *ev) {
struct epoll_event ep_ev = {0, {0}};
if (ev->status != 1) {
return -1;
}
ep_ev.data.ptr = ev;
ev->status = 0;
epoll_ctl(epfd, EPOLL_CTL_DEL, ev->fd, &ep_ev);
return 0;
}
int recv_cb(int fd, int events, void *arg) {
struct ntyreactor *reactor = (struct ntyreactor*)arg;
struct ntyevent *ev = reactor->events+fd;
int len = recv(fd, ev->buffer, BUFFER_LENGTH, 0);
nty_event_del(reactor->epfd, ev);
if (len > 0) {
ev->length = len;
ev->buffer[len] = '\0';
printf("C[%d]:%s\n", fd, ev->buffer);
nty_event_set(ev, fd, send_cb, reactor);
nty_event_add(reactor->epfd, EPOLLOUT, ev);
} else if (len == 0) {
close(ev->fd);
printf("[fd=%d] pos[%ld], closed\n", fd, ev-reactor->events);
} else {
close(ev->fd);
printf("recv[fd=%d] error[%d]:%s\n", fd, errno, strerror(errno));
}
return len;
}
int send_cb(int fd, int events, void *arg) {
struct ntyreactor *reactor = (struct ntyreactor*)arg;
struct ntyevent *ev = reactor->events+fd;
int len = send(fd, ev->buffer, ev->length, 0);
if (len > 0) {
printf("send[fd=%d], [%d]%s\n", fd, len, ev->buffer);
nty_event_del(reactor->epfd, ev);
nty_event_set(ev, fd, recv_cb, reactor);
nty_event_add(reactor->epfd, EPOLLIN, ev);
} else {
close(ev->fd);
nty_event_del(reactor->epfd, ev);
printf("send[fd=%d] error %s\n", fd, strerror(errno));
}
return len;
}
int accept_cb(int fd, int events, void *arg) {
struct ntyreactor *reactor = (struct ntyreactor*)arg;
if (reactor == NULL) return -1;
struct sockaddr_in client_addr;
socklen_t len = sizeof(client_addr);
int clientfd;
if ((clientfd = accept(fd, (struct sockaddr*)&client_addr, &len)) == -1) {
if (errno != EAGAIN && errno != EINTR) {
}
printf("accept: %s\n", strerror(errno));
return -1;
}
int i = 0;
do {
int flag = 0;
if ((flag = fcntl(clientfd, F_SETFL, O_NONBLOCK)) < 0) {
printf("%s: fcntl nonblocking failed, %d\n", __func__, MAX_EPOLL_EVENTS);
break;
}
nty_event_set(&reactor->events[clientfd], clientfd, recv_cb, reactor);
nty_event_add(reactor->epfd, EPOLLIN, &reactor->events[clientfd]);
} while (0);
printf("new connect [%s:%d][time:%ld], pos[%d]\n",
inet_ntoa(client_addr.sin_addr), ntohs(client_addr.sin_port), reactor->events[i].last_active, i);
return 0;
}
//初始化socket并监听
int init_sock(short port) {
int fd = socket(AF_INET, SOCK_STREAM, 0);
fcntl(fd, F_SETFL, O_NONBLOCK);
struct sockaddr_in server_addr;
memset(&server_addr, 0, sizeof(server_addr));
server_addr.sin_family = AF_INET;
server_addr.sin_addr.s_addr = htonl(INADDR_ANY);
server_addr.sin_port = htons(port);
bind(fd, (struct sockaddr*)&server_addr, sizeof(server_addr));
if (listen(fd, 20) < 0) {
printf("listen failed : %s\n", strerror(errno));
}
printf("listen port : %d\n", port);
return fd;
}
//初始化epoll
int ntyreactor_init(struct ntyreactor *reactor) {
if (reactor == NULL) return -1;
memset(reactor, 0, sizeof(struct ntyreactor));
reactor->epfd = epoll_create(1);
if (reactor->epfd <= 0) {
printf("create epfd in %s err %s\n", __func__, strerror(errno));
return -2;
}
reactor->events = (struct ntyevent*)malloc((MAX_EPOLL_EVENTS) * sizeof(struct ntyevent));
if (reactor->events == NULL) {
printf("create epfd in %s err %s\n", __func__, strerror(errno));
close(reactor->epfd);
return -3;
}
}
int ntyreactor_destory(struct ntyreactor *reactor) {
close(reactor->epfd);
free(reactor->events);
}
//注册socket到epoll事件集合中
int ntyreactor_addlistener(struct ntyreactor *reactor, int sockfd, NCALLBACK *acceptor) {
if (reactor == NULL) return -1;
if (reactor->events == NULL) return -1;
nty_event_set(&reactor->events[sockfd], sockfd, acceptor, reactor);
nty_event_add(reactor->epfd, EPOLLIN, &reactor->events[sockfd]);
return 0;
}
int ntyreactor_run(struct ntyreactor *reactor) {
if (reactor == NULL) return -1;
if (reactor->epfd < 0) return -1;
if (reactor->events == NULL) return -1;
struct epoll_event events[MAX_EPOLL_ITEM];
int checkpos = 0, i;
while (1) {
int nready = epoll_wait(reactor->epfd, events, MAX_EPOLL_ITEM, 1000);
if (nready < 0) {
printf("epoll_wait error, exit\n");
continue;
}
for (i = 0;i < nready;i ++) {
struct ntyevent *ev = (struct ntyevent*)events[i].data.ptr;
if ((events[i].events & EPOLLIN) && (ev->events & EPOLLIN)) {
ev->callback(ev->fd, events[i].events, ev->arg);
}
if ((events[i].events & EPOLLOUT) && (ev->events & EPOLLOUT)) {
ev->callback(ev->fd, events[i].events, ev->arg);
}
}
}
}
int main(int argc, char *argv[]) {
unsigned short port = SERVER_PORT;
if (argc == 2) {
port = atoi(argv[1]);
}
struct ntyreactor *reactor = (struct ntyreactor*)malloc(sizeof(struct ntyreactor));
ntyreactor_init(reactor);
int listenfd[LISTEN_PORT_COUNT] = {0};
int i = 0;
//创建100个socket，每个socket最多连接65535个
for (i = 0;i < LISTEN_PORT_COUNT;i ++) {
//服务端socket开始监听
listenfd[i] = init_sock(port+i);
//每个socket加到epoll监听队列
ntyreactor_addlistener(reactor, listenfd[i], accept_cb);
}
//处理监听事件
ntyreactor_run(reactor);
ntyreactor_destory(reactor);
for (i = 0;i < LISTEN_PORT_COUNT;i ++) {
close(listenfd[i]);
}
return 0;
}
问题3
服务端在18W连接的时候断开了
可以看到最大文件数是195142
在/etc/sysctl.conf加上fs.file-max = 1048576，修改系统设定的最大文件数
file-max和open files的区别
file-max是整个系统最大的文件句柄数，open files是最多可以打开多少个文件， open files是基于file-max的
重启生效
问题4
在大概4.9W的时候提示Connection timed out，这是connect超时
在终端下输入cat /proc/sys/net/netfilter/nf_conntrack_max查看允许连接数，如果显示没有这个文件
需要在/etc/modules加上nf_conntrack，重启加载模块
nf_conntrack_max默认是65536的
原因是iptables为了防止攻击，限制了连接数（也就是防火墙）
iptables是基于netfilter（linux内核协议栈的一套防火墙系统）框架的应用程序
在/etc/sysctl.conf加上
//最大跟踪连接数，默认是65536个
net.nf_conntrack_max = 1048576
//稳定的连接状态最多保留多久，单位是秒，默认是432000秒，就是5天
net.netfilter.nf_conntrack_tcp_timeout_established = 1200
重启或者sysctl -p生效
这次我开了5个终端运行客户端相当于5个进程进行连接（但是端口跟socket都是共用的）
问题5
在客户端连接数6.8W断开，5个客户端的话就是6.8W * 5 = 34W
在服务端终端输入dmesg
查看tcp的缓存设置
(3个INTEGER变量)：low, pressure, high
tcp_mem：
- low：当TCP使用了低于该值的内存页面数时，TCP不会考虑释放内存。
- pressure：当TCP使用了超过该值的内存页面数量时，TCP试图稳定其内存使用，进入压力模式，当内存消耗低于low值时则退出压力状态。
- high：允许所有tcp sockets用于排队缓冲数据报的页面量，当内存占用超过此值，系统拒绝分配socket，后台日志输出“TCP: too many orphaned sockets”或“Out of socket memory”
tcp_rmem：
- low：默认是4096，缓冲区的最小大小
- pressure：默认是87380，接收缓冲区的初始大小
- high：允许接收缓冲区的最大大小
tcp_wmem：
- low：默认是4096，缓冲区的最小大小
- pressure：默认是16384，发送缓冲区的初始大小
- high：允许发送缓冲区的最大大小
优化连接速度
在/etc/sysctl.conf修改
//扩大内存页面大小
net.ipv4.tcp_mem = 262144 524288 786432
//减少写缓冲区，加快速度
net.ipv4.tcp_wmem = 1024 1024 2048
//减少读缓冲区，加快速度
net.ipv4.tcp_rmem = 1024 1024 2048
在/etc/sysctl.conf修改
客户端
服务端
问题6
在客户端终端输入dmesg，out of memory是我电脑内存不足,在内存满的时候,linux内核有个OOM killer的机制会杀掉最占用内存的进程,一台电脑也不太现实（上班的时候再测）
这次用两台电脑测,客户端开6个进程
在终端输入cat /proc/net/sockstat可以看到创建的tcp_socket数和tcp_mem数
在终端输入htop,在连接数60W的时候CPU占用率是50%(两个CPU),内存用了2.35G
每个客户端连接数10W的时候,每1000连接数大概需要15秒
问题7
在93W并发的时候服务端内存满了(可能在80W就开始连接失败了)…
大概需要4G以上的内存才够用,换两个6G内存的电脑重新测
百万并发达成
达到规定的socket上限
大概用了2个小时,还是在内网的情况下。
如果想测试外网的话，租一台服务器或者用nginx代理
如果想加快速度就用上线程或线程池
可能会遇到的问题
在终端输入cat /proc/net/sockstat可以看到orphans数
orphans数量达到net.ipv4.tcp_max_orphans约一半时，就会报：Out of socket memory
在/etc/sysctl.conf加上net.ipv4.tcp_max_orphans = 16384，重启清理orphans数
888
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
