Linux系统TCP连接性能

一、测试环境

1. debian11 64bit

2. 设置系统网络限制

   sudo vi /etc/sysctl.conf
   	fs.file-max=10485760
   	net.ipv4.ip_local_port_range=1024 65535
   	#net.ipv4.tcp_tw_recycle=1
   	net.ipv4.tcp_tw_reuse=1
   	net.ipv4.tcp_timestamps=1
   	net.core.rmem_default=209715200
   	net.core.wmem_default=209715200
   	net.core.rmem_max=209715200
   	net.core.wmem_max=209715200
   sudo /sbin/sysctl -p
   

3. 设置文件描述符

   sudo vi /etc/security/limits.conf 
   	root soft nofile 65535
   	root hard nofile 65535
   sudo vi /etc/profile
   	ulimit -SHn 65535
   
   sudo reboot
   

二、测试代码

1. 服务端代码: tcpserver.c

   #include <stdio.h>
   #include <stdlib.h>
   #include <string.h>
   #include <unistd.h>
   #include <sys/socket.h>
   #include <netinet/in.h>
   #include <arpa/inet.h>
   #include <time.h>
   #include <sys/time.h>
   #include <pthread.h>
   
   #define CONNECT_NUM 65535
   
   static int fd_count = 0;
   static int fd_list[CONNECT_NUM] = {0};
   pthread_mutex_t	connect_mutex;
   
   void* connect_thread(void* arg){
   	uint16_t port = (uint16_t)arg;
   	int sockfd = socket(AF_INET,SOCK_STREAM,0);
   	if(sockfd < 0){
   		printf("socket error!\n");
   		exit(-1);
   	}
   
   	struct sockaddr_in my_addr;
   	memset(&my_addr, 0, sizeof(my_addr));
   	my_addr.sin_family=AF_INET;
   	my_addr.sin_port=htons(port);
   	my_addr.sin_addr.s_addr=htonl(INADDR_ANY);
   	
   	printf("Binding server to port %d\n",port);
   	int err_log = bind(sockfd,(struct sockaddr *)&my_addr,sizeof(my_addr));
   	if(err_log!=0){
   		printf("bind error!\n");
   		close(sockfd);
   		exit(-1);
   	}
   	err_log = listen(sockfd, 1000);
   	if(err_log != 0){
   		perror("listen");
   		close(sockfd);		
   		exit(-1);
   	}
   	printf("listen client @port=%d...\n",port);
   	while(1){
   		int count=0;
   		char recv_buf[128] = "";
   		struct sockaddr_in client_addr;
   		char cli_ip[INET_ADDRSTRLEN] = "";
   		
   		
   		socklen_t cliaddr_len = sizeof(client_addr);
   		int connfd = accept(sockfd, (struct sockaddr*)&client_addr, &cliaddr_len);       
   		if(connfd < 0){
   			perror("accept");
   			continue;
   		}
   		inet_ntop(AF_INET, &client_addr.sin_addr, cli_ip, INET_ADDRSTRLEN);
   		recv(connfd, recv_buf, sizeof(recv_buf), 0);
   		if(strcmp(recv_buf, "ok") != 0){
   			printf("recv error by %s\n", recv_buf);
   			
   		}
   		pthread_mutex_lock(&connect_mutex);
   		fd_list[fd_count] = connfd;
   		fd_count++;
   		pthread_mutex_unlock(&connect_mutex);
   		printf("thread%d ip:%s, port:%d, connfd:%d, fd_count:%d\n",port, cli_ip, ntohs(client_addr.sin_port), connfd, fd_count);
   	}
   	close(sockfd);
   	return NULL;
   	}
   int main(int argc, char *argv[]){
   	pthread_t pid1, pid2;
   	uint16_t port1 = 10000;
   	uint16_t port2 = 10001;
   	pthread_mutex_init(&connect_mutex, NULL);
   	pthread_create(&pid1, NULL, connect_thread, (void*)port1);
   	pthread_create(&pid2, NULL, connect_thread, (void*)port2);
   	pthread_join(pid1, NULL);
   	pthread_join(pid2, NULL);
   	return 0;
   }
   
   

2. 客户端代码: tcpclient.c

   #include <stdio.h>
   #include <stdlib.h>
   #include <string.h>
   #include <unistd.h>
   #include <sys/socket.h>
   #include <netinet/in.h>
   #include <arpa/inet.h>
   #include <time.h>
   #include <sys/time.h>
   #include <pthread.h>
   
   static int connect_total = 1;
   typedef struct {
   	pthread_t pid;
   	uint16_t port;
   	uint32_t duration;
   }th_info_t;
   uint64_t clocktime_now() {
   	struct timespec ts;
   	clock_gettime(CLOCK_MONOTONIC,&ts);
   	return ts.tv_sec * 1000000 + ( ts.tv_nsec / 1000 );
   }
   
   void* connect_thread(void* arg){
   	char server_ip[16] = "192.168.1.108";
   	int connect_num = (connect_total+1)/2;
   	int connect_idx = 0;
   	th_info_t* th_info = (th_info_t*)arg;
   	printf("thread%d connect_num:%d\n", th_info->port, connect_num);
   	uint64_t start_time = clocktime_now();
   	while(connect_idx < connect_num){
   		connect_idx++;
   		uint64_t start = clocktime_now();
   		int sockfd = socket(AF_INET, SOCK_STREAM, 0);
   		if(sockfd < 0){
   			printf("sockfd error!\n");
   			exit(-1);
   		}
   
   		struct sockaddr_in dest_addr;
   		memset(&dest_addr, 0, sizeof(dest_addr));
   		dest_addr.sin_family = AF_INET;
   		dest_addr.sin_port = htons(th_info->port);
   		inet_pton(AF_INET, server_ip, &dest_addr.sin_addr);
   		
   		int err_log = connect(sockfd, (struct sockaddr*)&dest_addr, sizeof(dest_addr));      // 主动连接服务器
   		if(err_log != 0){
   			perror("connect");
   			close(sockfd);
   			exit(-1);
   		}
   		int ret = send(sockfd, "ok", strlen("ok"), 0);
   		printf("thread%d connect fd:%d count:%d, ts:%dus\n", th_info->port, sockfd, connect_idx, clocktime_now()-start);
   	}
   	th_info->duration = (clocktime_now()-start_time)/1000;
   	return NULL;
   }
   int main(int argc, char *argv[]){
   	th_info_t th_info1, th_info2;
   	th_info1.port = 10000;
   	th_info2.port = 10001;
   	if(argc >= 2){
   		connect_total = atoi(argv[1]);
   	}
   	pthread_create(&th_info1.pid, NULL, connect_thread, (void*)&th_info1);
   	pthread_create(&th_info2.pid, NULL, connect_thread, (void*)&th_info2);
   	pthread_join(th_info1.pid, NULL);
   	pthread_join(th_info2.pid, NULL);
   	printf("thread%d connect success duration:%dms\n", th_info1.port, th_info1.duration);
   	printf("thread%d connect success duration:%dms\n", th_info2.port, th_info2.duration);
   	sleep(10);
   	return 0;
   }
   
   

3. 编译脚本: make-linux.sh

   #!/bin/bash
   gcc -o server tcpserver.c -lpthread
   gcc -o client tcpclient.c -lpthread
   
   

三、编译测试

1. 将服务代码、客户端代码、编译脚本放入同一个目录
2. ./make-linux.sh
3. 测试
   • 第一次测试：客户端建立40000连接，平均到10000和10001端口线程
     结果：每次连接完成20us(微妙)左右
   • 第二次测试：客户端建立33000连接，只连接到10000端口线程
     结果：前32255个连接，每个连接20us(微妙)左右；后745个连接，每个连接完成3400us(微妙)左右
   • 第三次测试后：客户端建立33000连接，平均到2个10000端口线程（2个线程都访问10000端口）
     结果：和第一次一样，说明延迟是服务端造成的
4. 结论
   • 单个socket的连接数超过最大值32255后，每个连接的完成耗时变大
   • 为什么是32255？net.ipv4.ip_local_port_range设置端口范围 (65535-1024)/2=3225.5 ，至于为什么是超过范围量的二分之一变慢（改变范围，测出的结果），还没找到答案

四、解决方案

1. 服务端在单进程情况，想要提高连接数的效率，目前想到的方案：多线程监听多个端口