模拟10K问题

1、  几个指标

QPS（TPS）：每秒钟request/事务 数量

并发数： 系统同时处理的request/事务数

响应时间：  一般取平均响应时间

2、  编写一个服务器测试服务器器支持的最大连接数

采用epoll实现支持1万个链接，做一个简单回显。

服务端采用epoll库实现。

客户端采用多线程方式实现。

3、  遇到的问题如下

a. 如何创建1万个线程？

inux 系统中单个进程的最大线程数有其最大的限制 PTHREAD_THREADS_MAX。
这个限制可以在 /usr/include/bits/local_lim.h 中查看。对 linuxthreads 这个值一般是 1024，对于 nptl 则没有硬性的限制，仅仅受限于系统的资源这个系统的资源主要就是线程的 stack 所占用的内存，用 ulimit -s 可以查看默认的线程栈大小，一般情况下，这个值是 8M。

通过测试程序，验证只能创建 381 个线程。

 

可以通过ulimit –s 256设置栈大小，然后通过设置线程属性。

pthread_attr_setstacksize(&fthd_attr,stacksize)

 

b. 如何创建1万个socket文件描述符？

Linux系统支持的文件句柄数限制ulimit–n查看，显然无法满足1万个文件的情况，只能修改rlimit。

 

4、程序源码：

         服务端：

#include  <unistd.h>
#include  <sys/types.h>       /* basic system data types */
#include  <sys/socket.h>      /* basic socket definitions */
#include  <netinet/in.h>      /* sockaddr_in{} and other Internet defns */
#include  <arpa/inet.h>       /* inet(3) functions */
#include <sys/epoll.h> /* epoll function */
#include <fcntl.h>     /* nonblocking */
#include <sys/resource.h> /*setrlimit */

#include <stdlib.h>
#include <errno.h>
#include <stdio.h>
#include <string.h>



#define MAXEPOLLSIZE 10000
#define MAXLINE 10240
int handle(int connfd);
int setnonblocking(int sockfd)
{
	if (fcntl(sockfd, F_SETFL, fcntl(sockfd, F_GETFD, 0)|O_NONBLOCK) == -1) {
		return -1;
	}
	return 0;
}

int main(int argc, char **argv)
{
	int  servPort = 6888;
	int listenq = 1024;

	int listenfd, connfd, kdpfd, nfds, n, nread, curfds,acceptCount = 0;
	struct sockaddr_in servaddr, cliaddr;
	socklen_t socklen = sizeof(struct sockaddr_in);
	struct epoll_event ev;
	struct epoll_event events[MAXEPOLLSIZE];
	struct rlimit rt;
	char buf[MAXLINE];

	/* 设置每个进程允许打开的最大文件数 */
	rt.rlim_max = rt.rlim_cur = MAXEPOLLSIZE;
	if (setrlimit(RLIMIT_NOFILE, &rt) == -1) 
	{
		perror("setrlimit error");
		return -1;
	}


	bzero(&servaddr, sizeof(servaddr));
	servaddr.sin_family = AF_INET; 
	servaddr.sin_addr.s_addr = htonl (INADDR_ANY);
	servaddr.sin_port = htons (servPort);

	listenfd = socket(AF_INET, SOCK_STREAM, 0); 
	if (listenfd == -1) {
		perror("can't create socket file");
		return -1;
	}

	int opt = 1;
	setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, &opt, sizeof(opt));

	if (setnonblocking(listenfd) < 0) {
		perror("setnonblock error");
	}

	if (bind(listenfd, (struct sockaddr *) &servaddr, sizeof(struct sockaddr)) == -1) 
	{
		perror("bind error");
		return -1;
	} 
	if (listen(listenfd, listenq) == -1) 
	{
		perror("listen error");
		return -1;
	}
	/* 创建 epoll 句柄，把监听 socket 加入到 epoll 集合里 */
	kdpfd = epoll_create(MAXEPOLLSIZE);
	ev.events = EPOLLIN | EPOLLET;
	ev.data.fd = listenfd;
	if (epoll_ctl(kdpfd, EPOLL_CTL_ADD, listenfd, &ev) < 0) 
	{
		fprintf(stderr, "epoll set insertion error: fd=%d\n", listenfd);
		return -1;
	}
	curfds = 1;

	printf("epollserver startup,port %d, max connection is %d, backlog is %d\n", servPort, MAXEPOLLSIZE, listenq);

	for (;;) {
		/* 等待有事件发生 */
		nfds = epoll_wait(kdpfd, events, curfds, -1);
		if (nfds == -1)
		{
			perror("epoll_wait");
			continue;
		}
		/* 处理所有事件 */
		for (n = 0; n < nfds; ++n)
		{
			if (events[n].data.fd == listenfd) 
			{
				connfd = accept(listenfd, (struct sockaddr *)&cliaddr,&socklen);
				if (connfd < 0) 
				{
					perror("accept error");
					continue;
				}

				sprintf(buf, "accept form %s:%d\n", inet_ntoa(cliaddr.sin_addr), cliaddr.sin_port);
				printf("%d:%s", ++acceptCount, buf);

				if (curfds >= MAXEPOLLSIZE) {
					fprintf(stderr, "too many connection, more than %d\n", MAXEPOLLSIZE);
					close(connfd);
					continue;
				} 
				if (setnonblocking(connfd) < 0) {
					perror("setnonblocking error");
				}
				ev.events = EPOLLIN | EPOLLET;
				ev.data.fd = connfd;
				if (epoll_ctl(kdpfd, EPOLL_CTL_ADD, connfd, &ev) < 0)
				{
					fprintf(stderr, "add socket '%d' to epoll failed: %s\n", connfd, strerror(errno));
					return -1;
				}
				curfds++;
				continue;
			} 
			// 处理客户端请求
			if (handle(events[n].data.fd) < 0) {
				epoll_ctl(kdpfd, EPOLL_CTL_DEL, events[n].data.fd,&ev);
				curfds--;


			}
		}
	}
	close(listenfd);
	return 0;
}
int handle(int connfd) {
	int nread;
	char buf[MAXLINE];
	nread = read(connfd, buf, MAXLINE);//读取客户端socket流

	if (nread == 0) {
		printf("client close the connection\n");
		close(connfd);
		return -1;
	} 
	if (nread < 0) {
		perror("read error");
		close(connfd);
		return -1;
	}	
	write(connfd, buf, nread);//响应客户端  
	return 0;
}

         客户端：

#include <stdio.h>
#include <stdlib.h>
#include <pthread.h>
#include <sys/socket.h>
#include <sys/select.h>
#include <netinet/in.h>
#include <string.h>
#include <arpa/inet.h>
#include <unistd.h>
#include <sys/resource.h>

#define THREAD_MAX 10000
#define BUFFER_SIZE 1024

pthread_t pid[THREAD_MAX];
int thread_name[THREAD_MAX];

char server_ip[32];
int port;

void *fun(void *arg)
{
	int *no = (int *)arg;

	char recv_buf[BUFFER_SIZE];
	char send_buf[BUFFER_SIZE];

	printf("thread %d\n", *no);

	int client_socket;
	struct sockaddr_in server_addr;
	memset(&server_addr, '\0', sizeof(server_addr));

	server_addr.sin_family = AF_INET;
	server_addr.sin_port = htons(port);
	server_addr.sin_addr.s_addr = inet_addr(server_ip);

	client_socket = socket(AF_INET, SOCK_STREAM, 0);

	if(connect(client_socket, (struct sockaddr *)&server_addr, sizeof(server_addr)) < 0)
	{
		perror("connet error\n");
		return NULL;
	}
	memset(send_buf, '\0', BUFFER_SIZE);
	snprintf(send_buf, BUFFER_SIZE, "send socket %d\n", *no);
	send(client_socket, send_buf, strlen(send_buf), 0);

	while(1)
	{
		memset(send_buf, '\0', BUFFER_SIZE);
		snprintf(send_buf, BUFFER_SIZE, "send socket %d\n", *no);
		if(send(client_socket, send_buf, strlen(send_buf), 0) == -1)
		{
			perror("send error");
		}

		memset(recv_buf, '\0', BUFFER_SIZE);
		if (recv(client_socket, recv_buf, sizeof(recv_buf), 0) == -1)
		{
			perror("recv error");
		}
		printf("recv: %s \n", recv_buf);

		sleep(1000);
	}
	return NULL;
}


static void SetLimits(void)
{
    struct rlimit nlim;
    int ret = 0;
    nlim.rlim_max = 65535;
    nlim.rlim_cur = nlim.rlim_max;
    //nlim.rlim_max = RLIM_INFINITY;
    ret += setrlimit(RLIMIT_NOFILE, &nlim);
    if (ret != 0) {
        getrlimit(RLIMIT_NOFILE, &nlim);
        nlim.rlim_cur = nlim.rlim_max;
        setrlimit(RLIMIT_NOFILE, &nlim);
        getrlimit(RLIMIT_NOFILE, &nlim);
        printf("Files limits: %lld/%lld\n", (long long)nlim.rlim_cur, (long long)nlim.rlim_max);
    }

    nlim.rlim_cur = RLIM_INFINITY;
    nlim.rlim_max = RLIM_INFINITY;
    ret += setrlimit(RLIMIT_NPROC, &nlim);

    nlim.rlim_cur = RLIM_INFINITY;
    nlim.rlim_max = RLIM_INFINITY;
    ret += setrlimit(RLIMIT_CORE, &nlim);
    if (ret == 0) {
        printf("Limits changed\n");
    }
    else {
        printf("Limits not changed\n");
        sleep(2);
    }
}


int main(int argc, char *argv[])
{
	int thread_num;
	int i, len, ret;
	size_t stacksize;
	pthread_attr_t fthd_attr;

	if(argc < 2 || argc != 4)
	{
		printf("please thread num\n");
		return 0;
	}

	thread_num = atoi(argv[1]);


	len = strlen(argv[2]);
	memset(server_ip, '\0', 32);
	memcpy(server_ip, argv[2], len);
	port = atoi(argv[3]);

	printf("thread %d, ip %s, port %d\n", thread_num, server_ip, port);

	SetLimits();

	pthread_attr_init(&fthd_attr);

	stacksize = 145*1024;

	if (pthread_attr_setstacksize(&fthd_attr, stacksize) != 0) {
        perror("Unable to set thread stack size");
        exit(-1);
    }

	for(i = 0; i < thread_num; i++)
	{
		thread_name[i] = i;
	}

	for( i = 0; i < thread_num; i++ )
	{
		ret = pthread_create(&pid[i], NULL, (void *)fun, (void *)&thread_name[i]);
		if(ret != 0 )
		{
			perror("Create thread error\n");
			return -1;
		}
        printf("Create Thread %d\n", i);
        usleep(50000);
	}

	for( i = 0; i < thread_num; i++ )
	{
		pthread_join(pid[i], NULL);
	}

	return EXIT_SUCCESS;
}

5、测试效果

能够达到1万个socket连接的情况；

 

6、 tcp状态转换图

 

7、 高性能服务器考虑的因素

缓存的处理；

        线程的处理；

注:事必躬亲