前言:流媒体服务发送udp包时,和tcp比较效率很低,在网上查的有sendmmsg(sendmsg的加强版)方法可以提高效率,特意做了个比较测试如下。
测试硬件:Xeon(R) CPU @ 1.80GHz 32核, 10G网卡 的两台服务器
测试记录:单线程死循环发是的流量情况(结收方iftop统计,发送方打开iftop流量会下降很多):
sendto : 2.55Gb, send:2.56Gb, sendmmsg:2.55Gb iftop统计的流量存在误差,可以看出3个函数的发送性能不相上下,当然,产生这个结果还有可能是网卡等其他方面的原因限制了函数发挥,最大就2.56Gb了。
发送100万个udp包,统计使用时间比较(运行10次,取平均值, times(0)精确到10ms):
sendto :4.739 s, send:4.333s, sendmmsg:4.305s
sendmmsg的效率更快些。
结论:实验结果失望,sendmmsg并没有多大的优化提升,死循环发送时,内核态cpu、软中断等都差不多,也许是实验存在漏洞或受到其它因素限制。
结论修正:上次的实验是在两台万兆网卡服务器上进行的,最近在一台千兆网卡服务器上发流到万兆网卡服务器,使用sendmmsg发送同样的数据cpu占用有所提升,sendmsg/sendmmsg也可以不建立连接直接发送数据,建立连接时效果更好。sendmsg在tcp中发送多块数据效率很好。
备注:sendmsg/sendmmsg无连接发送和sendto一样,需要将 struct msghdr结构体中的msg_name 指针指向目标地址,msg_namelen填地址长度,即sendto最后两个参数。
sendmsg和sendmmsg区别:两者都能发送多块数据,区别在于sendmsg会将所有数据整合成一个UDP包发出,sendmmsg这是每个mmsghdr一个UDP包,所有发送RTP这种事只能用sendmmsg了。
测试代码:
sendto方式:
#include <sys/types.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <stdlib.h>
#include <unistd.h>
#include <stdio.h>
#include <sys/times.h>
#define SEG_SIZE 1500
#define PKG_NUM 100*10000
int
main(void)
{
int sockfd = socket(AF_INET, SOCK_DGRAM, 0);
struct sockaddr_in server_addr;
bzero(&server_addr, sizeof(server_addr));
server_addr.sin_family = AF_INET;
server_addr.sin_port = htons(28000);
inet_pton(AF_INET, "192.168.21.173", &server_addr.sin_addr.s_addr);
char data[SEG_SIZE];
memset(data, 0xfe, sizeof(data));
time_t start = times(0);
unsigned int send_num = 0;
while (1)//send_num < PKG_NUM)
{
if (0 > sendto(sockfd, data, SEG_SIZE, 0, (struct sockaddr*)&server_addr, sizeof(server_addr)))
{
printf("sendmmsg error\n");
//break;
}
send_num++;
}
printf("use time: %u0ms\n", times(0)-start);
close(sockfd);
return 0;
}
send方式:
#include <string.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <stdlib.h>
#include <unistd.h>
#include <stdio.h>
#include <sys/times.h>
#define SEG_SIZE 1500
#define PKG_NUM 10000*100
int
main(void)
{
int sockfd = socket(AF_INET, SOCK_DGRAM, 0);
struct sockaddr_in server_addr;
bzero(&server_addr, sizeof(server_addr));
server_addr.sin_family = AF_INET;
server_addr.sin_port = htons(28000);
inet_pton(AF_INET, "192.168.21.173", &server_addr.sin_addr.s_addr);
if (connect(sockfd, (struct sockaddr*)&server_addr, sizeof(server_addr)) == -1)
{
printf("connect error!\n");
return -1;
}
char data[SEG_SIZE];
memset(data, 0xfe, sizeof(data));
time_t start = times(0);
unsigned int send_num = 0;
while(1)//send_num < PKG_NUM)
{
if (0 > send(sockfd, data, SEG_SIZE, 0))
{
printf("sendmmsg error\n");
//break;
}
send_num += 1;
}
printf("use time: %u0ms\n", times(0)- start);
close(sockfd);
return 0;
}
sendmmsg方式:
//#define _GNU_SOURCE
#include <string.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <stdlib.h>
#include <unistd.h>
#include <stdio.h>
#include <sys/times.h>
#include <iostream>
#define MAX_SEG 20
#define SEG_SIZE 1500
#define PKG_NUM 10000*100
/*
struct mmsghdr
{
struct msghdr msg_hdr;
unsigned int msg_len;
};
*/
int
main(void)
{
int sockfd = socket(AF_INET, SOCK_DGRAM, 0);
struct sockaddr_in server_addr;
bzero(&server_addr, sizeof(server_addr));
server_addr.sin_family = AF_INET;
server_addr.sin_port = htons(28000);
inet_pton(AF_INET, "192.168.21.173", &server_addr.sin_addr.s_addr);
if (connect(sockfd, (struct sockaddr*)&server_addr, sizeof(server_addr)) == -1)
{
printf("connect error!\n");
return -1;
}
struct iovec iovs[MAX_SEG];
struct mmsghdr mmsg[MAX_SEG];
bzero(&iovs, sizeof(iovs));
bzero(&mmsg, sizeof(mmsg));
char data[SEG_SIZE];
memset(data, 0xfe, sizeof(data));
int i;
for (i = 0; i < MAX_SEG; ++i)
{
iovs[i].iov_base = data;
iovs[i].iov_len = sizeof(data);
mmsg[i].msg_hdr.msg_iov = iovs + i;
mmsg[i].msg_hdr.msg_iovlen = 1;
}
time_t start = times(0);
unsigned int send_num = 0;
while (1)//send_num < PKG_NUM)
{
if(0 > sendmmsg(sockfd, mmsg, MAX_SEG, 0))
{
printf("sendmmsg error\n");
//break;
}
send_num += MAX_SEG;
}
printf("use time:%u0ms\n", times(0)-start);
close(sockfd);
return 0;
}
注意:sendmmsg需要用g++编译,如果用gcc编译,必须自定义struct mmsghdr或者打开后_GNU_SOURCE,sendmmsg和recvmmsg的官方示例:https://www.man7.org/linux/man-pages/man2/sendmmsg.2.html,https://www.man7.org/linux/man-pages/man2/recvmmsg.2.html
附带接收方代码:
#define _GNU_SOURCE
#include <netinet/ip.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/socket.h>
int
main(void)
{
#define VLEN 10
#define BUFSIZE 1600
#define TIMEOUT 1
int sockfd, retval, i;
struct sockaddr_in addr;
struct mmsghdr msgs[VLEN];
struct iovec iovecs[VLEN];
char bufs[VLEN][BUFSIZE + 1];
struct timespec timeout;
sockfd = socket(AF_INET, SOCK_DGRAM, 0);
if (sockfd == -1) {
perror("socket()");
exit(EXIT_FAILURE);
}
addr.sin_family = AF_INET;
addr.sin_addr.s_addr = htonl(INADDR_ANY);
addr.sin_port = htons(28000);
if (bind(sockfd, (struct sockaddr *) &addr, sizeof(addr)) == -1) {
perror("bind()");
exit(EXIT_FAILURE);
}
memset(msgs, 0, sizeof(msgs));
for (i = 0; i < VLEN; i++) {
iovecs[i].iov_base = bufs[i];
iovecs[i].iov_len = BUFSIZE;
msgs[i].msg_hdr.msg_iov = &iovecs[i];
msgs[i].msg_hdr.msg_iovlen = 1;
}
timeout.tv_sec = TIMEOUT;
timeout.tv_nsec = 0;
while (1){
retval = recvmmsg(sockfd, msgs, VLEN, 0, &timeout);
if (retval == -1) {
perror("recvmmsg()");
exit(EXIT_FAILURE);
}
}
return 0;
}
接收方代码根据官方示例修改的,但无论怎样都触发不了超时,retval返回一直都是1,或许是收到了什么硬件限制。
相关参考资料:https://blog.csdn.net/nice_wen/article/details/83902568