[TCP]TCP 文件传输速度优化 - Linux零拷贝Sendfile函数

这篇文章主要是记录一下，使用sendfile()和普通的文件传输方式对比，有多大的差异。
主要是启动两个桥接的VMWARE ubuntu虚拟机，一个发送端，一个接收端，进行文件传输。
参考文章在本文末尾列出。
完整代码在这里

使用普通文件传输方式

发送代码

int Sendctl::StartTransfer()
{   
    if(checkFileSize() == -1)
        return -1;
    int remainsize = m_filesize, sendsize = 0 , buffsize = 64 * 1024;
    char sendbuff[buffsize] = { 0x00 };

   
    memcpy(sendbuff, m_ptrFileName, strlen(m_ptrFileName));
     std::cout << DisPlayCurrentTime() << "send file name" << sendbuff << std::endl;
    int res = MySend(m_conn_fd, sendbuff, strlen(sendbuff));
    if(res == -1) return -1;
    else if(res == 0) return 0;

    sleep(1); //防止沾包
    memset(sendbuff, 0x00, buffsize);
    sprintf(sendbuff, "%d" , m_filesize);
    std::cout << DisPlayCurrentTime() << "send file size" << sendbuff << std::endl;
    res = MySend(m_conn_fd, sendbuff, strlen(sendbuff));
    if(res == -1) return -1;
    else if(res == 0) return 0;

    sleep(1);//防止沾包
    int old_sendsize = 0;
    std::cout << DisPlayCurrentTime() << "send file start" << std::endl;
    while(sendsize < m_filesize && m_fs.read(sendbuff,remainsize>buffsize?buffsize:remainsize)) { 
        res = MySend(m_conn_fd, sendbuff, remainsize>buffsize?buffsize:remainsize);
        if(res == -1) return -1;
        else if(res == 0) return 0;
        sendsize += res;
        remainsize -= res;
        if(sendsize - old_sendsize > 10000000) {
           std::cout << "send size:" << sendsize << std::endl;
           old_sendsize = sendsize;
        } 
    }
     std::cout << DisPlayCurrentTime() << m_ptrFileName << "send end, send size:" << sendsize << std::endl;
    return 0;
}

接受代码

int main()
{
    int listen_fd = socket(AF_INET, SOCK_STREAM, 0);
    if(listen_fd == -1) {
        perror("socket");
        return -1;
    }

    sockaddr_in local_addr;
    local_addr.sin_family = AF_INET;
    local_addr.sin_addr.s_addr = htonl(INADDR_ANY);
    local_addr.sin_port = htons(LISTEN_PORT);
    std::cout << "server addr:" << inet_ntoa(local_addr.sin_addr) << ",port:" << LISTEN_PORT << std::endl;
    socklen_t len = sizeof(local_addr);
    if(bind(listen_fd, (sockaddr*)&local_addr, len) == -1) {
        perror("bind");
        return -1;
    }

    if(listen(listen_fd,5) == -1) {
        perror("listen");
        return -1;
    }

    sockaddr_in conn_addr;
    int conn_fd = accept(listen_fd,(sockaddr*)&conn_addr,&len);
    if(conn_fd == -1) {
        perror("acccept");
        return -1;
    }

    int remainsize = 0, recvsize = 0 , buffsize = 64 * 1024;
    char buff[buffsize] = { 0x00 };

    std::cout << "recv file name" << std::endl;
    int res = MyRecv(conn_fd, fileinfo.filename, 256);
    if(res == 0 || res == -1) return res;

    std::cout << "recv file size" << std::endl;
    res = MyRecv(conn_fd, buff, buffsize);
    if(res == 0 || res == -1) return res;
    fileinfo.filesize = atoi(buff);

    std::cout << "recv file name:" << fileinfo.filename << ",size:" << fileinfo.filesize << std::endl;
    std::fstream fs;
    fs.open(fileinfo.filename, std::ios::out | std::ios::trunc);
    if(!fs.is_open()) {
        std::cout << fileinfo.filename << ", open failed" << std::endl;
        return -1;
    }
    fs.seekg(0);
    int old_recvsize = 0;
    while(recvsize < fileinfo.filesize)
    {
        int res = MyRecv(conn_fd, buff, buffsize);
        if (res == 0 || res == -1)
            return res;
        fs.write(buff, res);       
        recvsize += res; 
        if(recvsize - old_recvsize > 10000000) {
            std::cout << "recv size:" << recvsize << std::endl;
            old_recvsize = recvsize;
        }
    }
    fs.close();
    std::cout << fileinfo.filename << ", recv end, recv size:" << recvsize << std::endl;
    return 0;
}   

使用Sendfile传输方式

sendfile()不是不存在拷贝，还是需要从磁盘拷贝到内核缓冲区，再拷贝到协议栈的，只是不需要从内核缓冲区拷贝到用户缓冲区了。

发送代码

在执行./client的时候，传入参数zerocopy，激活全局变量is_zerocopy_enable为true。

int Sendctl::StartTransfer()
{   
    if(checkFileSize() == -1)
        return -1;
    int remainsize = m_filesize, sendsize = 0 , buffsize = 64 * 1024;
    char sendbuff[buffsize] = { 0x00 };

   
    memcpy(sendbuff, m_ptrFileName, strlen(m_ptrFileName));
     std::cout << DisPlayCurrentTime() << "send file name" << sendbuff << std::endl;
    int res = MySend(m_conn_fd, sendbuff, strlen(sendbuff));
    if(res == -1) return -1;
    else if(res == 0) return 0;

    sleep(1); //防止沾包
    memset(sendbuff, 0x00, buffsize);
    sprintf(sendbuff, "%d" , m_filesize);
    std::cout << DisPlayCurrentTime() << "send file size" << sendbuff << std::endl;
    res = MySend(m_conn_fd, sendbuff, strlen(sendbuff));
    if(res == -1) return -1;
    else if(res == 0) return 0;

    sleep(1);//防止沾包
     std::cout << "is_zerocopy_enable is " << is_zerocopy_enable << std::endl;
    if(is_zerocopy_enable) {
        std::cout << "sendfile branch" << std::endl;
        ssize_t sendsize = 0, max_sendsize = 0;
        std::cout << DisPlayCurrentTime() << m_ptrFileName << "send file start" << std::endl;
        while(max_sendsize < m_filesize) {
            int res = sendfile(m_conn_fd, m_file_fd, &sendsize, m_filesize);
            if (res < 0)
            {
                perror("sendfile");
                return -1;
            }
            max_sendsize += sendsize;
             std::cout << "send file send size:" << max_sendsize << std::endl;
        }
        std::cout << DisPlayCurrentTime() << m_ptrFileName << "send file end, send size:" << sendsize << std::endl;
    }
    else {
       //使用普通文件传输方式的代码
    }
    return 0;
}

接受代码

发送端可以使用sendfile()来发送，但是接收端还是需要使用recv()来socket接收。所以直接使用使用普通文件传输方式的接受代码。

文件传输时间

两种方式在传输文件中没有很大的差别，235m的文件，都传输了30多秒，不知道是不是和我用的WiFi，网络比较差有关。

参考文章：

sendfile | 传说中的零拷贝（主要用于网络中文件传输