[TCP] TCP_NODELAY?

可能很多人都像我一样，很早就听说过TCP_NODELAY这个选项，明白它跟nagle算法相关，但是就只是停留在表面上了。
最近，实验室在做一套低时延可靠传输协议。作为对比，我希望搞清楚TCP在实际场景中丢包重传到底要消耗多少时间。为此，通过netem在loopback上模拟丢包和延时，我只需要在send以及recv这两个时刻打时间戳就可以知道每个包大概的时延。
以上是背景。然后问题来了。

在MacOS 10.11上，直接进行测试，时延基本为0，符合预计，毕竟没有人为加丢包和延时；
在Linux 4.13上，进行测试，发现时延经常出现30ms、40ms这样的异常值，但是，当把单个数据包的长度降低到500B以下时，恢复正常；这个现象很有趣，也扰乱了思路。
当然，最后才想起可能是TCP_NODELAY，所以纸上得来终觉浅，设置TCP_NODELAY之后，Linux上恢复正常。

#include <stdio.h>
#include <string.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <sys/time.h>
#include <fcntl.h>
#include <unistd.h>
#include <assert.h>
#include <netinet/tcp.h>
#include <errno.h>

#define N 	(20000)
#define USETCP	(1)
#define LEN	(1024)

long getts() {
    struct timeval tv;
    gettimeofday(&tv, NULL);
    long ts = tv.tv_sec * 1000 + tv.tv_usec / 1000;
    return ts;
}

ssize_t nrecv(int sock, void *buf, size_t len, int flags) {
#ifdef USETCP
	size_t nr;
	size_t nleft = len;
	void *ptr = buf;

	while (nleft > 0) {
            if ((nr = recv(sock, ptr, nleft, flags)) < 0) {
                if (errno == EINTR)
		    nr = 0;
	        else
		    return -1;
	    } else if (nr == 0)
        	break;

	    nleft -= nr;
 	    ptr += nr;
	}

	return (len - nleft);
#else
	return recv(sock, buf, len, flags);
#endif
}

typedef struct {
	long ts;
	char buf[LEN - sizeof(long)];
} DataWrapper;

int main(int argc, char *argv[])
{
    if (argc != 2) {
        printf("Not Enough Parameter\n");
        return -1;
    } else if (memcmp(argv[1], "c", 1) == 0) {
#ifdef USETCP
        // Init Socket Begins
        int sock = socket(PF_INET, SOCK_STREAM, 0);
        struct sockaddr_in addr;
        memset(&addr, 0, sizeof(addr));
        inet_pton(PF_INET, "127.0.0.1", &addr.sin_addr);
        addr.sin_family = AF_INET;
        addr.sin_port = htons(7777);
        connect(sock, (struct sockaddr *) &addr, sizeof(addr));
        int on = 1;
	setsockopt(sock, IPPROTO_TCP, TCP_NODELAY, &on, sizeof(on));
	// Init Socket Ends
#else
	int sock = socket(PF_INET, SOCK_DGRAM, 0);
	struct sockaddr_in addr;
        memset(&addr, 0, sizeof(addr));
        inet_pton(PF_INET, "127.0.0.1", &addr.sin_addr);
        addr.sin_family = AF_INET;
        addr.sin_port = htons(7777);
        connect(sock, (struct sockaddr *) &addr, sizeof(addr));
#endif
        DataWrapper dw;

        for (int i = 0; i < N; i++) {
            dw.ts = getts();
            send(sock, &dw, sizeof(dw), 0);

            // 1000KBps => 1KB/ms => 1 packet / 1000us
            usleep(1000); // roughly
        }
    } else if (memcmp(argv[1], "s", 1) == 0) {
#ifdef USETCP
	// Init Socket Begins
        int listensock = socket(PF_INET, SOCK_STREAM, 0);
        int opt = 1;
        setsockopt(listensock, SOL_SOCKET, SO_REUSEADDR, &opt, sizeof(opt));
        struct sockaddr_in addr;
        memset(&addr, 0, sizeof(addr));
        addr.sin_family = AF_INET;
        addr.sin_addr.s_addr = htons(INADDR_ANY);
        addr.sin_port = htons(7777);
        if (bind(listensock, (struct sockaddr *)&addr, sizeof(addr)) < 0) {
            printf("tcpsock Bind Failed\n");
            return -1;
        }
        listen(listensock, 128);
        int sock = accept(listensock, NULL, NULL);
        int on = 1;
	setsockopt(sock, IPPROTO_TCP, TCP_NODELAY, &on, sizeof(on));
	// Init Socket Ends
#else
        int sock = socket(PF_INET, SOCK_DGRAM, 0);
        struct sockaddr_in addr;
        memset(&addr, 0, sizeof(addr));
        addr.sin_family = AF_INET;
        addr.sin_addr.s_addr = htons(INADDR_ANY);
        addr.sin_port = htons(7777);
        if (bind(sock, (struct sockaddr *)&addr, sizeof(addr)) < 0) {
            printf("tcpsock Bind Failed\n");
            return -1;
        }

        struct timeval timeout = {2, 0};
        setsockopt(sock, SOL_SOCKET, SO_RCVTIMEO, (void *)&timeout, sizeof(struct timeval));
#endif
        DataWrapper dw;

        for (int i = 0; i < N; i++) {
            int nrcv = nrecv(sock, &dw, sizeof(dw), 0);
            if (nrcv != sizeof(dw)) {
                fprintf(stderr, "nrcv: %d\n", nrcv);
                return -1;
            }
            printf("%ld\n", getts() - dw.ts);
        }
    } else {
        printf("Error Parameter\n");
        return -1;
    }
}