在github上找了一个用C++开启并测试Tcp Fast Open,看完之后来验证。
client:
#include <iostream>
#include <string>
#include <vector>
#include <memory>
#include <cstdlib>
#include <cstdio>
#include <cstring>
#include <random>
#include <functional>
#include <limits>
#include <algorithm>
#include <thread>
#include <atomic>
#include <unistd.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <netdb.h>
#include "common.h"
#define SERVER_PORT "32345"
int getsock(const std::string &server_addr, const std::vector<char> &msg_to_send)
{
struct addrinfo hint, *res;
std::memset(&hint, 0, sizeof(hint));
hint.ai_family = AF_UNSPEC;
hint.ai_socktype = SOCK_STREAM;
int e = getaddrinfo(server_addr.c_str(), SERVER_PORT, &hint, &res);
if( e == -1 )
{
cerror("getaddrinfo");
std::exit(EXIT_FAILURE);
}
int sock;
struct addrinfo *rp;
for( rp = res; rp != nullptr; rp = rp->ai_next )
{
// Need to create the socket again when EINTR is returned
redo:
sock = socket(rp->ai_family, rp->ai_socktype, rp->ai_protocol);
if( sock == -1 )
continue;
ssize_t r = sendto(sock, &msg_to_send[0], msg_to_send.size(),
MSG_FASTOPEN, rp->ai_addr, rp->ai_addrlen);
if( r == -1 && errno == EINTR )
{
close(sock);
goto redo;
}
// Exit the loop when sendto succeeded
if( r != -1 )
break;
// Try without TCP FastOpen
r = connect(sock, rp->ai_addr, rp->ai_addrlen);
if( r == -1 && errno == EINTR )
{
close(sock);
goto redo;
}
if( r != -1 )
{
ssize_t sent;
do
{
sent = send(sock, &msg_to_send[0], msg_to_send.size(), 0);
} while( sent == -1 && sent == EINTR );
// Success
if( r != -1 )
break;
}
close(sock);
}
freeaddrinfo(res);
if( rp == nullptr )
{
std::cerr << "Couldn't get a socket." << std::endl;
std::exit(EXIT_FAILURE);
}
return sock;
}
void send_recv(const std::string &server_addr, const int msg_size, std::atomic_int &count)
{
std::vector<char> msg_buf(msg_size), recv_buf(msg_size);
std::mt19937 rand_engine;
std::uniform_int_distribution<char> distribution(
std::numeric_limits<char>::min(), std::numeric_limits<char>::max());
auto generator = std::bind(distribution, rand_engine);
while( count-- > 0 )
{
std::generate(msg_buf.begin(), msg_buf.end(), generator);
FD sock(getsock(server_addr, msg_buf));
shutdown(sock, SHUT_WR);
ssize_t read_bytes = 0;
while( read_bytes != msg_size )
{
ssize_t r;
do
{
r = recv(sock, &recv_buf[read_bytes], recv_buf.size() - read_bytes, 0);
} while( r == -1 && errno == EINTR );
if( r == 0 ) break;
if( r == -1 )
{
cerror("recv");
std::exit(EXIT_FAILURE);
}
read_bytes += r;
}
if( msg_buf != recv_buf )
std::cerr << "Receive mismatch." << std::endl;
}
}
int main(int argc, char *argv[])
{
if( argc <= 4 )
{
std::cerr << "Usage: client [server addr] [msg size] [count] [thread]" << std::endl;
return EXIT_FAILURE;
}
std::string server_addr(argv[1]);
int msg_size = std::atoi(argv[2]);
std::atomic_int count(std::atoi(argv[3]));
int nthreads = std::atoi(argv[4]);
std::vector<std::thread> threads;
threads.reserve(nthreads);
for( int i = 0; i < nthreads; ++i )
threads.push_back(std::thread(send_recv, server_addr, msg_size, std::ref(count)));
std::for_each(threads.begin(), threads.end(), std::mem_fn(&std::thread::join));
}
server
#include <iostream>
#include <string>
#include <vector>
#include <memory>
#include <cstdlib>
#include <cstdio>
#include <cstring>
#include <unistd.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <sys/epoll.h>
#include <netinet/tcp.h>
#include <netdb.h>
#define DEFAULT_PORT "32345"
#define TFO_QLEN 128
#define LISTEN_QLEN 128
#define BUFFER_SIZE 4096
#define MAX_EVENTS 32
#include "common.h"
int open_socket(const std::string &port)
{
struct addrinfo hint, *res;
std::memset(&hint, 0, sizeof(hint));
hint.ai_family = AF_UNSPEC;
hint.ai_socktype = SOCK_STREAM;
hint.ai_flags = AI_PASSIVE | AI_NUMERICSERV;
int e = getaddrinfo(nullptr, port.c_str(), &hint, &res);
if( e != 0 )
{
std::cerr << "getaddrinfo: " << gai_strerror(e) << std::endl;
std::exit(EXIT_FAILURE);
}
int sock;
struct addrinfo *rp;
for( rp = res; rp != nullptr; rp = rp->ai_next )
{
sock = socket(rp->ai_family, rp->ai_socktype | SOCK_NONBLOCK, rp->ai_protocol);
if( sock == -1 ) continue;
if( bind(sock, rp->ai_addr, rp->ai_addrlen) == 0 ) break;
close(sock);
}
if( rp == nullptr )
{
std::cerr << "Couldn't bind." << std::endl;
std::exit(EXIT_FAILURE);
}
int protocol = rp->ai_protocol;
freeaddrinfo(res);
const int qlen = TFO_QLEN;
e = setsockopt(sock, protocol, TCP_FASTOPEN, &qlen, sizeof(qlen));
if( e == -1 )
{
close(sock);
cerror("setsockopt");
std::exit(EXIT_FAILURE);
}
if( listen(sock, LISTEN_QLEN) == -1 )
{
close(sock);
cerror("listen");
std::exit(EXIT_FAILURE);
}
return sock;
}
int accept_and_register(int sfd, int epollfd)
{
// Accept a new connection
struct sockaddr addr;
socklen_t addrlen = sizeof(addr);
int s;
do
{
s = accept4(sfd, &addr, &addrlen, SOCK_NONBLOCK);
} while( s == -1 && errno == EINTR );
if( s == -1 )
{
cerror("accept");
std::exit(EXIT_FAILURE);
}
struct epoll_event ev;
ev.events = EPOLLIN | EPOLLET;
ev.data.fd = s;
if( epoll_ctl(epollfd, EPOLL_CTL_ADD, s, &ev) == -1 )
{
perror("epoll_ctl: EPOLL_CTL_ADD");
std::exit(EXIT_FAILURE);
}
return s;
}
void process_client_socket(int s, int epollfd)
{
static std::vector<char> buf(BUFFER_SIZE);
ssize_t rs;
while( (rs = recv(s, &buf[0], buf.size(), 0)) != 0 )
{
if( rs == -1 )
{
if( errno == EINTR ) continue;
if( errno == EAGAIN || errno == EWOULDBLOCK ) return;
cerror("recv");
break;
}
ssize_t sent;
// EAGAIN or EWOULDBLOCK is retuned when the socket buffer is full
// We should try with another socket for maximum performance, however,
// we just wait until the buffer becomes available as we don't have
// enough buffer for that.
do {
sent = send(s, &buf[0], rs, 0);
} while( sent == -1 &&
(errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK) );
if( sent == -1 )
{
cerror("send");
break;
}
}
if( epoll_ctl(epollfd, EPOLL_CTL_DEL, s, nullptr) == -1 )
cerror("epoll_ctl: EPOLL_CTL_DEL");
close(s);
}
void server(const std::string &port)
{
FD sfd(open_socket(port));
struct epoll_event ev, events[MAX_EVENTS];
FD epollfd(epoll_create1(0));
if( epollfd == -1 )
{
cerror("epoll_create1");
std::exit(EXIT_FAILURE);
}
ev.events = EPOLLIN;
ev.data.fd = sfd;
if( epoll_ctl(epollfd, EPOLL_CTL_ADD, sfd, &ev) == -1 )
{
perror("epoll_ctl: EPOLL_CTL_ADD");
std::exit(EXIT_FAILURE);
}
while( 1 )
{
const int nfds = epoll_wait(epollfd, events, MAX_EVENTS, -1);
if( nfds == -1 )
{
if( errno == EINTR ) continue;
cerror("epoll_wait");
std::exit(EXIT_FAILURE);
}
for( int i = 0; i < nfds; ++i )
{
if( events[i].data.fd == sfd )
accept_and_register(sfd, epollfd);
else
process_client_socket(events[i].data.fd, epollfd);
}
}
}
int main(int argc, char *argv[])
{
std::string port;
if( argc <= 1 )
{
port = DEFAULT_PORT;
} else {
port = argv[1];
}
server(port);
}
common.h
#include <iostream>
#include <cstring>
static void cerror(const char *str)
{
std::cerr << str << ": " << std::strerror(errno) << std::endl;
}
class FD
{
private:
int fd_;
public:
FD(int fd): fd_(fd) {}
~FD()
{
this->close();
}
operator int() const
{
return fd_;
}
void close()
{
if( fd_ == -1 ) return;
::close(fd_);
fd_ = -1;
}
};
Makefile
CXX = g++
CXXFLAGS = -O -std=c++11 -Wall
TARGETS = client server
.PHONY: all clean
all: $(TARGETS)
client: client.cpp common.h
$(CXX) $(CXXFLAGS) -pthread -o $@ $<
server: server.cpp common.h
$(CXX) $(CXXFLAGS) -o $@ $<
clean:
rm -f $(TARGETS)
四个文件保存好然后进入文件夹make.
开启TFO:sudo sysctl -w net.ipv4.tcp_fastopen=3 (client和Server端都要开启)
server:
./server
client:
./client <server ip><message size><test count><threads>
eg:./client 192.168.1.112 1024 100 1 1
抓包分析:
第一个数据包
(上传的时候用的是Ctrl+v然后没有上传成功,再测试已经开启TFO了所以第一个SYN包已经带数据了)
第三个数据包
(同第一个数据包原因)
再次建立TCP连接时
第九个数据包
可以看到第一次SYN和第二次SYN+ACK两次握手,我们打开第一个数据包可以看到因为第一次是携带FOC请求报文没携带数据包,再然后打开第三个数据包可以看到ACK携带有我们测试的数据,第四个数据包因为客户端没有请求所以没有返回数据。第5、6、7、8是tcp的四次挥手不用看。我们再看第九个数据包,可以看到SYN请求时携带数据包的(Data 1024Bytes),由此可以看出开启TFO后在第一次握手的时候是可以携带数据包的,这样就能节省一个RTT。
也可以用命令来看是否成功开启TFO:grep ‘^TcpExt:’ /proc/net/netstat | cut -d ’ ’ -f 91-96 | column -
TCPFastOpenActiveFail TCPFastOpenPassive TCPFastOpenPassiveFail TCPFastOpenListenOverflow TCPFastOpenCookieReqd TCPSpuriousRtxHostQueues
0 99 0 0 1 0
TCPFastOpenPassive 数量增长代表成功开启TFO功能,TCPFastOpenCookieReqd代表缓存的地址数。
分析包的时候还发现一个问题:4个字节Fast Open选项在options里没有发现,暂时还不知道为何。