阻塞方式
/*
g++ sp_block.cpp -g -O0 -o sp_block
*/
#include <unistd.h>
#include <stdio.h>
#include <time.h>
// socketpair
#include <sys/types.h>
#include <sys/socket.h>
// fork
#include <sys/types.h>
#include <unistd.h>
// atoi
#include <stdlib.h>
// errno
#include <errno.h>
// strerror
#include <string.h>
#define srs_trace(msg, ...) \
printf("[%d][%d]", time(NULL), getpid()); \
printf(msg, ##__VA_ARGS__); \
if(errno) \
printf(", errno=%d(%s)\n", errno, strerror(errno)); \
else \
printf("\n")
int RecvFD(int fd, int* pfd){
*pfd = 0;
msghdr msg;
// msg_iov
iovec iov[1];
iov[0].iov_base = pfd;
iov[0].iov_len = sizeof(int);
msg.msg_iov = iov;
msg.msg_iovlen = 1;
// msg_name
msg.msg_name = NULL;
msg.msg_namelen = 0;
// msg_control
union { // union to create a 8B aligned memory.
struct cmsghdr cm; // 16B = 8+4+4
char space[CMSG_SPACE(sizeof(int))]; // 24B = 16+4+4
} cmsg;
memset(&cmsg, 0, sizeof(cmsg));
msg.msg_control = (cmsghdr*)&cmsg;
msg.msg_controllen = sizeof(cmsg);
if((recvmsg(fd, &msg, 0)) == -1){
srs_trace("st_recvmsg recv fd error.");
return -1;
}
*pfd = *(int *)CMSG_DATA(&cmsg.cm);
if(*pfd <= 0){
srs_trace("if transfer fd, the fd must be positive. actual is %d", *pfd);
return -1;
}
srs_trace("get a fd from msg, fd=%d", *pfd);
return 0;
}
int SendFD(int fd, int clientfd){
// transfer the fd.
msghdr msg;
union {
struct cmsghdr cm;
char space[CMSG_SPACE(sizeof(int))];
} cmsg;
memset(&cmsg, 0, sizeof(cmsg));
cmsg.cm.cmsg_level = SOL_SOCKET;
cmsg.cm.cmsg_type = SCM_RIGHTS; // we are sending fd.
cmsg.cm.cmsg_len = CMSG_LEN(sizeof(int));
msg.msg_control = (cmsghdr*)&cmsg;
msg.msg_controllen = sizeof(cmsg);
*(int *)CMSG_DATA(&cmsg.cm) = clientfd;
// init msg_iov
iovec iov[1];
iov[0].iov_base = &clientfd;
iov[0].iov_len = sizeof(int);
msg.msg_iov = iov;
msg.msg_iovlen = 1;
// init msg_name
msg.msg_name = NULL;
msg.msg_namelen = 0;
// send fd
if(sendmsg(fd, &msg, 0) == -1){
srs_trace("st_sendmsg failed");
return -1;
}
srs_trace("st_sendmsg send fd=%d", clientfd);
return 0;
}
int main(int argc, char** argv){
srs_trace("Error value trace:");
srs_trace(" EMSGSIZE=%d(%s)", EMSGSIZE, strerror(EMSGSIZE));
srs_trace(" EAGAIN=%d(%s)", EAGAIN, strerror(EAGAIN));
if(argc <= 7){
srs_trace("Usage: <%s> "
"<send_count> <send_size> <send_sleep> "
"<recv_count> <recv_size> <recv_sleep> "
"<transfer_fd>",
argv[0]);
return -1;
}
int send_count = atoi(argv[1]);
int send_size = atoi(argv[2]);
int send_sleep = atoi(argv[3]);
int recv_count = atoi(argv[4]);
int recv_size = atoi(argv[5]);
int recv_sleep = atoi(argv[6]);
int transfer_fd = atoi(argv[7]);
srs_trace("send_size=%dB send_count=%d send_sleep=%dms "
"recv_size=%dB recv_count=%d recv_sleep=%dms transfer_fd=%d",
send_size, send_count, send_sleep, recv_size, recv_count,
recv_sleep, transfer_fd);
char* packet_send = new char[send_size];
char* packet_recv = new char[recv_size];
int size = 0;
int fds[2];
if((socketpair(PF_UNIX, SOCK_STREAM, 0, fds)) == -1){
srs_trace("create socketpair failed");
return -1;
}
srs_trace("create socketpair success");
int pid = fork();
if(pid == -1){
srs_trace("create [sender] failed");
return -1;
}
if(pid == 0){
int client = socket(PF_INET, SOCK_STREAM, 0);
int fd = fds[1];
srs_trace("[sender] pid=%d, fd=%d, peer=%d, client=%d", getpid(), fd, fds[0], client);
int nb_sent = 0;
for(int i = 0; send_size > 0 && i < send_count; i++){
errno = 0;
srs_trace("[sender] start to send msg");
if((size = send(fd, packet_send, send_size, 0)) != send_size){
srs_trace("[sender] send packet error. i=%d, size=%d", i, size);
return -1;
}
nb_sent += size;
srs_trace("[sender] send packet. size=%d", size);
if(send_sleep > 0 && i < send_count - 1){
usleep(send_sleep * 1000);
}
}
srs_trace("[sender] send send_count=%d send_size=%dB sent=%dB, sleep=%dms success",
send_count, send_size, nb_sent, send_sleep);
if(transfer_fd && SendFD(fd, client) == -1){
srs_trace("[sender] send fd error");
return -1;
}
srs_trace("[sender] send fd ok, fd=%d", client);
}
else{
int fd = fds[0];
srs_trace("[receiver] pid=%d, fd=%d, peer=%d", getpid(), fd, fds[1]);
int nb_recv = 0;
for(int i = 0; recv_size > 0 && i < recv_count; i++){
errno = 0;
srs_trace("start to read msg");
if((size = recv(fd, packet_recv, recv_size, 0)) == -1){
srs_trace("[receiver] recv packet error. i=%d, size=%d", i, size);
return -1;
}
nb_recv += size;
srs_trace("[receiver] recv packet. size=%d", size);
if(recv_sleep > 0 && i < recv_count - 1){
usleep(recv_sleep * 1000);
}
}
srs_trace("[receiver] recv recv_count=%d recv_size=%dB recv=%dB, sleep=%dms success",
recv_count, recv_size, nb_recv, recv_sleep);
int client = 0;
if(transfer_fd && RecvFD(fd, &client) == -1){
srs_trace("[receiver] recv fd error");
return -1;
}
srs_trace("[receiver] recv fd ok, fd=%d", client);
}
srs_trace("test completed, sleep a while then exit");
sleep(30);
return 0;
}
./sp_block 2 2 3000 1 4 0 0
[1414330113][13832][receiver] pid=13832, fd=3, peer=4
[1414330113][13833][sender] pid=13833, fd=4, peer=3, client=5
[1414330113][13832][receiver] recv recv_count=1 recv_size=4B recv=2B, sleep=0ms success
[1414330116][13833][sender] send send_count=2 send_size=2B sent=4B, sleep=3000ms success
./sp_block 2 2 0 1 4 0 0
[1414330176][13849][receiver] pid=13849, fd=3, peer=4
[1414330176][13850][sender] pid=13850, fd=4, peer=3, client=5
[1414330176][13850][sender] send send_count=2 send_size=2B sent=4B, sleep=0ms success
[1414330176][13849][receiver] recv recv_count=1 recv_size=4B recv=4B, sleep=0ms success
./sp_block 2 2 3000 1 4 0 1
[1414330769][13986][receiver] pid=13986, fd=3, peer=4
[1414330769][13987][sender] pid=13987, fd=4, peer=3, client=5
[1414330769][13986][receiver] recv recv_count=1 recv_size=4B recv=2B, sleep=0ms success
[1414330772][13987][sender] send send_count=2 send_size=2B sent=4B, sleep=3000ms success
[1414330772][13986][receiver] recv fd error
[1414330772][13987][sender] send fd ok, fd=5
./sp_block 2 2 0 1 4 0 1
[1414330912][14018][receiver] pid=14018, fd=3, peer=4
[1414330912][14019][sender] pid=14019, fd=4, peer=3, client=5
[1414330912][14019][sender] send send_count=2 send_size=2B sent=4B, sleep=0ms success
[1414330912][14018][receiver] recv recv_count=1 recv_size=4B recv=4B, sleep=0ms success
[1414330912][14019][sender] send fd ok, fd=5
[1414330912][14018][receiver] recv fd ok, fd=5
非阻塞方式
/*
g++ sp_noblock.cpp -g -O0 -o sp_noblock
*/
#include <unistd.h>
#include <stdio.h>
#include <time.h>
// socketpair
#include <sys/types.h>
#include <sys/socket.h>
// fork
#include <sys/types.h>
#include <unistd.h>
// atoi
#include <stdlib.h>
// errno
#include <errno.h>
// strerror
#include <string.h>
#define srs_trace(msg, ...) \
printf("[%d][%d]", time(NULL), getpid()); \
printf(msg, ##__VA_ARGS__); \
if(errno) \
printf(", errno=%d(%s)\n", errno, strerror(errno)); \
else \
printf("\n")
int RecvFD(int fd, int* pfd){
*pfd = 0;
msghdr msg;
// msg_iov
iovec iov[1];
iov[0].iov_base = pfd;
iov[0].iov_len = sizeof(int);
msg.msg_iov = iov;
msg.msg_iovlen = 1;
// msg_name
msg.msg_name = NULL;
msg.msg_namelen = 0;
// msg_control
union { // union to create a 8B aligned memory.
struct cmsghdr cm; // 16B = 8+4+4
char space[CMSG_SPACE(sizeof(int))]; // 24B = 16+4+4
} cmsg;
memset(&cmsg, 0, sizeof(cmsg));
msg.msg_control = (cmsghdr*)&cmsg;
msg.msg_controllen = sizeof(cmsg);
if((recvmsg(fd, &msg, MSG_DONTWAIT)) == -1){
srs_trace("st_recvmsg recv fd error.");
return -1;
}
*pfd = *(int *)CMSG_DATA(&cmsg.cm);
if(*pfd <= 0){
srs_trace("if transfer fd, the fd must be positive. actual is %d", *pfd);
return -1;
}
srs_trace("get a fd from msg, fd=%d", *pfd);
return 0;
}
int SendFD(int fd, int clientfd){
// transfer the fd.
msghdr msg;
union {
struct cmsghdr cm;
char space[CMSG_SPACE(sizeof(int))];
} cmsg;
memset(&cmsg, 0, sizeof(cmsg));
cmsg.cm.cmsg_level = SOL_SOCKET;
cmsg.cm.cmsg_type = SCM_RIGHTS; // we are sending fd.
cmsg.cm.cmsg_len = CMSG_LEN(sizeof(int));
msg.msg_control = (cmsghdr*)&cmsg;
msg.msg_controllen = sizeof(cmsg);
*(int *)CMSG_DATA(&cmsg.cm) = clientfd;
// init msg_iov
iovec iov[1];
iov[0].iov_base = &clientfd;
iov[0].iov_len = sizeof(int);
msg.msg_iov = iov;
msg.msg_iovlen = 1;
// init msg_name
msg.msg_name = NULL;
msg.msg_namelen = 0;
// send fd
if(sendmsg(fd, &msg, MSG_DONTWAIT) == -1){
srs_trace("st_sendmsg failed");
return -1;
}
srs_trace("st_sendmsg send fd=%d", clientfd);
return 0;
}
int main(int argc, char** argv){
srs_trace("Error value trace:");
srs_trace(" EMSGSIZE=%d(%s)", EMSGSIZE, strerror(EMSGSIZE));
srs_trace(" EAGAIN=%d(%s)", EAGAIN, strerror(EAGAIN));
if(argc <= 7){
srs_trace("Usage: <%s> "
"<send_count> <send_size> <send_sleep> "
"<recv_count> <recv_size> <recv_sleep> "
"<transfer_fd>",
argv[0]);
return -1;
}
int send_count = atoi(argv[1]);
int send_size = atoi(argv[2]);
int send_sleep = atoi(argv[3]);
int recv_count = atoi(argv[4]);
int recv_size = atoi(argv[5]);
int recv_sleep = atoi(argv[6]);
int transfer_fd = atoi(argv[7]);
srs_trace("send_size=%dB send_count=%d send_sleep=%dms "
"recv_size=%dB recv_count=%d recv_sleep=%dms transfer_fd=%d",
send_size, send_count, send_sleep, recv_size, recv_count,
recv_sleep, transfer_fd);
char* packet_send = new char[send_size];
char* packet_recv = new char[recv_size];
int size = 0;
int fds[2];
if((socketpair(PF_UNIX, SOCK_STREAM, 0, fds)) == -1){
srs_trace("create socketpair failed");
return -1;
}
srs_trace("create socketpair success");
int pid = fork();
if(pid == -1){
srs_trace("create [sender] failed");
return -1;
}
if(pid == 0){
int client = socket(PF_INET, SOCK_STREAM, 0);
int fd = fds[1];
srs_trace("[sender] pid=%d, fd=%d, peer=%d, client=%d", getpid(), fd, fds[0], client);
int nb_sent = 0;
for(int i = 0; send_size > 0 && i < send_count; i++){
errno = 0;
srs_trace("[sender] start to send msg");
if(send_sleep > 0){
usleep(send_sleep * 1000);
}
if((size = send(fd, packet_send, send_size, MSG_DONTWAIT)) != send_size){
srs_trace("[sender] send packet error. i=%d, size=%d", i, size);
return -1;
}
nb_sent += size;
srs_trace("[sender] send packet. size=%d", size);
}
srs_trace("[sender] send send_count=%d send_size=%dB sent=%dB, sleep=%dms success",
send_count, send_size, nb_sent, send_sleep);
if(transfer_fd && SendFD(fd, client) == -1){
srs_trace("[sender] send fd error");
return -1;
}
srs_trace("[sender] send fd ok, fd=%d", client);
}
else{
int fd = fds[0];
srs_trace("[receiver] pid=%d, fd=%d, peer=%d", getpid(), fd, fds[1]);
int nb_recv = 0;
for(int i = 0; recv_size > 0 && i < recv_count; i++){
errno = 0;
srs_trace("start to read msg");
if(recv_sleep > 0){
usleep(recv_sleep * 1000);
}
if((size = recv(fd, packet_recv, recv_size, MSG_DONTWAIT)) == -1){
srs_trace("[receiver] recv packet error. i=%d, size=%d", i, size);
return -1;
}
nb_recv += size;
srs_trace("[receiver] recv packet. size=%d", size);
}
srs_trace("[receiver] recv recv_count=%d recv_size=%dB recv=%dB, sleep=%dms success",
recv_count, recv_size, nb_recv, recv_sleep);
int client = 0;
if(transfer_fd && RecvFD(fd, &client) == -1){
srs_trace("[receiver] recv fd error");
return -1;
}
srs_trace("[receiver] recv fd ok, fd=%d", client);
}
srs_trace("test completed, sleep a while then exit");
sleep(30);
return 0;
}
非阻塞读取socketpair,都得处理读取出来的字节少于期待的字节的情况。
ST的处理方式
附上st的处理方式:
/*
g++ sp_st_fd.cpp -g -O0 -Ist-1.9/obj st-1.9/obj/libst.a -o sp_st_fd
*/
#include <unistd.h>
#include <stdio.h>
// socketpair
#include <sys/types.h>
#include <sys/socket.h>
// fork
#include <sys/types.h>
#include <unistd.h>
// atoi
#include <stdlib.h>
// errno
#include <errno.h>
// strerror
#include <string.h>
#include <st.h>
int RecvFD(st_netfd_t stfd, int* pfd){
*pfd = 0;
msghdr msg;
// msg_iov
iovec iov[1];
iov[0].iov_base = pfd;
iov[0].iov_len = sizeof(int);
msg.msg_iov = iov;
msg.msg_iovlen = 1;
// msg_name
msg.msg_name = NULL;
msg.msg_namelen = 0;
// msg_control
union { // union to create a 8B aligned memory.
struct cmsghdr cm; // 16B = 8+4+4
char space[CMSG_SPACE(sizeof(int))]; // 24B = 16+4+4
} cmsg;
memset(&cmsg, 0, sizeof(cmsg));
msg.msg_control = (cmsghdr*)&cmsg;
msg.msg_controllen = sizeof(cmsg);
if((st_recvmsg(stfd, &msg, 0, ST_UTIME_NO_TIMEOUT)) == -1){
printf("st_recvmsg recv fd error. errno=%d(%#x)\n", errno, errno);
return -1;
}
*pfd = *(int *)CMSG_DATA(&cmsg.cm);
if(*pfd <= 0){
printf("if transfer fd, the fd must be positive. actual is %d\n", *pfd);
return -1;
}
printf("get a fd from msg, fd=%d\n", *pfd);
return 0;
}
int SendFD(st_netfd_t stfd, int clientfd){
// transfer the fd.
msghdr msg;
union {
struct cmsghdr cm;
char space[CMSG_SPACE(sizeof(int))];
} cmsg;
memset(&cmsg, 0, sizeof(cmsg));
cmsg.cm.cmsg_level = SOL_SOCKET;
cmsg.cm.cmsg_type = SCM_RIGHTS; // we are sending fd.
cmsg.cm.cmsg_len = CMSG_LEN(sizeof(int));
msg.msg_control = (cmsghdr*)&cmsg;
msg.msg_controllen = sizeof(cmsg);
*(int *)CMSG_DATA(&cmsg.cm) = clientfd;
// init msg_iov
iovec iov[1];
iov[0].iov_base = &clientfd;
iov[0].iov_len = sizeof(int);
msg.msg_iov = iov;
msg.msg_iovlen = 1;
// init msg_name
msg.msg_name = NULL;
msg.msg_namelen = 0;
// send fd
if(st_sendmsg(stfd, &msg, 0, ST_UTIME_NO_TIMEOUT) == -1){
printf("st_sendmsg failed, errno=%d(%#x)\n", errno, errno);
return -1;
}
printf("st_sendmsg send fd=%d\n", clientfd);
return 0;
}
void* thread(void* arg){
for(;;){
printf("in st thread.\n");
st_sleep(1);
}
return NULL;
}
int main(int argc, char** argv){
#if 1
printf("errno=EMSGSIZE(%d), %s\n", EMSGSIZE, strerror(EMSGSIZE));
if(argc <= 6){
printf("Usage: <%s> <send_count> <recv_count> <packet_send_size> <packet_recv_size> <parent_process_sleep> <sub_process_sleep>\n", argv[0]);
return -1;
}
int send_count = atoi(argv[1]);
int recv_count = atoi(argv[2]);
int packet_send_size = atoi(argv[3]);
int packet_recv_size = atoi(argv[4]);
int parent_process_sleep = atoi(argv[5]);
int sub_process_sleep = atoi(argv[6]);
printf("packet_send_size=%dB packet_recv_size=%dB send_count=%d recv_count=%d parent_process_sleep=%dms sub_process_sleep=%dms\n",
packet_send_size, packet_recv_size, send_count, recv_count, parent_process_sleep, sub_process_sleep);
char* packet_send = new char[packet_send_size];
char* packet_recv = new char[packet_recv_size];
int size = 0;
#endif
int fds[2];
if((socketpair(PF_UNIX, SOCK_STREAM, 0, fds)) == -1){
printf("create socketpair failed\n");
return -1;
}
printf("create socketpair success\n");
int pid = fork();
if(pid == -1){
printf("create [sub] failed\n");
return -1;
}
// init st.
if(st_init() == -1){
printf("st_init error\n");
return -1;
}
if(st_thread_create(thread, NULL, 0, 0) == NULL){
printf("create st thread error.\n");
return -1;
}
if(pid == 0){
int client = socket(PF_INET, SOCK_STREAM, 0);
int fd = fds[1];
printf("[sub] pid=%d, fd=%d, peer=%d, client=%d\n", getpid(), fd, fds[0], client);
st_netfd_t stfd = st_netfd_open(fd);
if(stfd == NULL){
printf("[sub] open stfd failed.\n");
return -1;
}
for(int i = 0; i < send_count; i++){
errno = 0;
if(sub_process_sleep > 0){
usleep(sub_process_sleep * 1000);
}
printf("[sub] start to send msg and fd.\n");
if(packet_send_size > 0){
if((size = st_write(stfd, packet_send, packet_send_size, ST_UTIME_NO_TIMEOUT)) != packet_send_size){
printf("[sub] send packet error. i=%d, size=%d, errno=%d, msg=%s\n", i, size, errno, strerror(errno));
return -1;
}
printf("[sub] send packet. size=%d, errno=%d(%s)\n", size, errno, strerror(errno));
}
}
if(SendFD(stfd, client) == -1){
printf("[sub] send fd error. errno=%d, msg=%s\n", errno, strerror(errno));
return -1;
}
printf("[sub] send send_count=%d packet_send_size=%dB, sleep=%dms success\n", send_count, packet_send_size, sub_process_sleep);
}
else{
int fd = fds[0];
printf("[parent] pid=%d, fd=%d, peer=%d\n", getpid(), fd, fds[1]);
st_netfd_t stfd = st_netfd_open(fd);
if(stfd == NULL){
printf("[parent] open stfd failed.\n");
return -1;
}
int client = 0;
for(int i = 0; i < recv_count; i++){
errno = 0;
if(parent_process_sleep > 0){
usleep(parent_process_sleep * 1000);
}
if(client > 0){
close(client);
}
printf("start to read msg and fd.\n");
if(packet_recv_size > 0){
if((size = st_read(stfd, packet_recv, packet_recv_size, ST_UTIME_NO_TIMEOUT)) == -1){
printf("[parent] recv packet error. i=%d, size=%d, errno=%d, msg=%s\n", i, size, errno, strerror(errno));
return -1;
}
printf("[parent] recv packet. size=%d, errno=%d(%s)\n", size, errno, strerror(errno));
}
}
if(RecvFD(stfd, &client) == -1){
printf("[sub] send fd error. errno=%d, msg=%s", errno, strerror(errno));
return -1;
}
printf("[parent] recv recv_count=%d packet_recv_size=%dB, sleep=%dms success\n", recv_count, packet_recv_size, parent_process_sleep);
}
sleep(30);
return 0;
}
./sp_st_fd 2 2 10 20 0 3000
发送2次,每次发送10字节。
收取时,每次收取20字节,就会有EAGAIN消息。
第1次收取,收取10字节。
第2次收取,收取10字节。
[parent] recv packet. size=10, errno=11(Resource temporarily unavailable)
[parent] recv packet. size=10, errno=11(Resource temporarily unavailable)
[parent] recv recv_count=2 packet_recv_size=20B, sleep=0ms success
然后正常的收到fd。
结论
结论是:
1. 发送端和接收端,同时操作的数据应该是一样的。譬如,发送端一次发送100个字节的头,接收端先接受100个字节的头。发送端再发送x字节的数据,接收端接受x字节的数据。
2.发送端对于每次来回不能发送多次,否则接收端就会出现EAGAIN或者其他的情况。譬如头是100字节,发送端就必须一次发送100字节,而不能做两次发送,一次发送20一次发送80.
3.可见每个消息不超过16000,超过后需要分多次发送了。
4.只要单次一次发送,不管是否是非阻塞,都能收到。譬如发送fd,一次发送一个msghdr,只要发送端一次发送,接收端一次接收,就没有问题。不会出现收到一半的情况。
5.只有当发送端因为某种原因发送了一部分,接收端一次接收的超过了发送的部分,就会出现非阻塞的EAGAIN。
再论结论
原因就是因为sendmsg和recvmsg是原子操作。
譬如有以下协议:
1. 发送4字节类型
2. 发送4字节长度
3. 发送长度指定的数据
4. 发送一个cmsg消息
读取时不能超过发送的组合单元,譬如假设是分四次发送的,那么就必须分四次接收,因为不能保证1和2是一起收到的。
假设1和2是一起发送,3单独发送,4单独发送;那么接收时可以一起接收1和2,但是不能一起接收1和2和3。也可以分开接收1,然后收2.
打个比方,这个就像包裹一样的,如果快递时1和2是一起打包的,到门口接收时肯定是1和2一起到的,进门接收时可以收1,然后收2,或者一起收。