epoll 与 io_uring 性能测试

1. 测试说明

学习过 epoll 和 io_uring 好奇那个性能更好一些，所以分别通过 2 种技术实现 tcpserver。通过相同的客户端来测试 100 个连接，访问 100 万次服务器，它们的 qps 分别是多少。

有想自己测试的小伙伴也可以使用下面代码，自己动手测试一下，对于 io_uring 原理不太清楚的可以看我另外一篇文章了解一下。

2. 测试结果

epoll 的测试结果：

io_uring 的测试结果：

3. 总结

通过上面测试结果可以看到，在相同的环境下，io_uring 与 epoll 的 qps 相差不大，epoll 要稍微高一些。当然这并不能说明 io_uring 比 epoll 的性能低，原因可能是本次没有涉及到大量的磁盘 IO 操作，有兴趣的小伙伴可以自己实现测试一下，有时间我也会在尝试一下。

4. 测试代码

#include <stdio.h>
#include <unistd.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <arpa/inet.h>
#include <stdlib.h>
#include <string.h>
#include <sys/epoll.h>

#define MAX_EVENTS 10
#define MAX_BUFF_SIZE 1024

int main() {
    int sockfd = socket(AF_INET, SOCK_STREAM, 0);
    if (sockfd < 0) {
        perror("socket");
        return -1;
    }

    struct sockaddr_in addr;
    addr.sin_family = AF_INET;
    addr.sin_port = htons(2048);
    addr.sin_addr.s_addr = INADDR_ANY;
    if (bind(sockfd, (struct sockaddr*)&addr, sizeof(addr)) < 0) {
        perror("bind");
        close(sockfd);
        return -1;
    }

    listen(sockfd, 10);

    // 创建 epoll 句柄
    int epollfd = epoll_create(MAX_EVENTS);
    if (epollfd == -1) {
        perror("epoll_create");
        close(sockfd);
        return -1;
    }

    // 添加监听套接字 sockfd 到 epoll
    struct epoll_event event;
    event.events = EPOLLIN;
    event.data.fd = sockfd;
    if (epoll_ctl(epollfd, EPOLL_CTL_ADD, sockfd, &event) == -1) {
        perror("epoll_ctl");
        close(epollfd);
        close(sockfd);
        return -1;
    }

    struct epoll_event events[MAX_EVENTS];

    char buff[MAX_BUFF_SIZE];

    while (1) {
        // 监听事件
        int nfds = epoll_wait(epollfd, events, MAX_EVENTS, -1);
        if (nfds == -1) {
            perror("epoll_wait");
            break;
        }

        for (int i = 0; i < nfds; ++i) {
            if (events[i].data.fd == sockfd) {
                // 有新连接
                int clientfd = accept(sockfd, NULL, NULL);
                if (clientfd == -1) {
                    perror("accept");
                    continue;
                }

                // 添加新连接到 epoll
                event.events = EPOLLIN;
                event.data.fd = clientfd;
                if (epoll_ctl(epollfd, EPOLL_CTL_ADD, clientfd, &event) == -1) {
                    perror("epoll_ctl");
                    close(clientfd);
                    continue;
                }
            } else {
                // 已连接套接字可读
                int n = recv(events[i].data.fd, buff, sizeof(buff), 0);
                if (n <= 0) {
                    if (n == 0) {
                        // 连接关闭
                        //printf("exit fd[%d]\n", events[i].data.fd);
                    } else {
                        // 出错
                        perror("recv");
                    }

                    // 从 epoll 中移除套接字
                    epoll_ctl(epollfd, EPOLL_CTL_DEL, events[i].data.fd, NULL);
                    close(events[i].data.fd);
                } else {
                    // 正常接收到数据
                    //printf("fd[%d] recv: %s\n", events[i].data.fd, buff);
                    send(events[i].data.fd, buff, n, 0);
                }
            }
        }
    }

    close(epollfd);
    close(sockfd);
    return 0;
}

#include <stdio.h>
#include <liburing.h>
#include <netinet/in.h>
#include <string.h>
#include <unistd.h>


#define EVENT_ACCEPT    0
#define EVENT_READ              1
#define EVENT_WRITE             2

struct conn_info {
        int fd;
        int event;
};


int init_server(unsigned short port) {

        int sockfd = socket(AF_INET, SOCK_STREAM, 0);
        struct sockaddr_in serveraddr;
        memset(&serveraddr, 0, sizeof(struct sockaddr_in));
        serveraddr.sin_family = AF_INET;
        serveraddr.sin_addr.s_addr = htonl(INADDR_ANY);
        serveraddr.sin_port = htons(port);

        if (-1 == bind(sockfd, (struct sockaddr*)&serveraddr, sizeof(struct sockaddr))) {
                perror("bind");
                return -1;
        }

        listen(sockfd, 10);

        return sockfd;
}

#define ENTRIES_LENGTH          1024
#define BUFFER_LENGTH           1024

int set_event_recv(struct io_uring *ring, int sockfd,
                                      void *buf, size_t len, int flags) {

        struct io_uring_sqe *sqe = io_uring_get_sqe(ring);

        struct conn_info accept_info = {
                .fd = sockfd,
                .event = EVENT_READ,
        };

        io_uring_prep_recv(sqe, sockfd, buf, len, flags);
        memcpy(&sqe->user_data, &accept_info, sizeof(struct conn_info));
}


int set_event_send(struct io_uring *ring, int sockfd,
                                      void *buf, size_t len, int flags) {

        struct io_uring_sqe *sqe = io_uring_get_sqe(ring);

        struct conn_info accept_info = {
                .fd = sockfd,
                .event = EVENT_WRITE,
        };

        io_uring_prep_send(sqe, sockfd, buf, len, flags);
        memcpy(&sqe->user_data, &accept_info, sizeof(struct conn_info));
}

int set_event_accept(struct io_uring *ring, int sockfd, struct sockaddr *addr,
                                        socklen_t *addrlen, int flags) {

        struct io_uring_sqe *sqe = io_uring_get_sqe(ring);

        struct conn_info accept_info = {
                .fd = sockfd,
                .event = EVENT_ACCEPT,
        };

        io_uring_prep_accept(sqe, sockfd, (struct sockaddr*)addr, addrlen, flags);
        memcpy(&sqe->user_data, &accept_info, sizeof(struct conn_info));

}

int main(int argc, char *argv[]) {

        unsigned short port = 9999;
        int sockfd = init_server(port);

        struct io_uring_params params;
        memset(&params, 0, sizeof(params));

        struct io_uring ring;
        io_uring_queue_init_params(ENTRIES_LENGTH, &ring, &params);


#if 0
        struct sockaddr_in clientaddr;
        socklen_t len = sizeof(clientaddr);
        accept(sockfd, (struct sockaddr*)&clientaddr, &len);
#else

        struct sockaddr_in clientaddr;
        socklen_t len = sizeof(clientaddr);
        set_event_accept(&ring, sockfd, (struct sockaddr*)&clientaddr, &len, 0);

#endif

        char buffer[BUFFER_LENGTH] = {0};

        while (1) {

                io_uring_submit(&ring);

                struct io_uring_cqe *cqe;
                io_uring_wait_cqe(&ring, &cqe);

                struct io_uring_cqe *cqes[128];
                int nready = io_uring_peek_batch_cqe(&ring, cqes, 128);  // epoll_wait

                int i = 0;
                for (i = 0;i < nready;i ++) {

                        struct io_uring_cqe *entries = cqes[i];
                        struct conn_info result;
                        memcpy(&result, &entries->user_data, sizeof(struct conn_info));

                        if (result.event == EVENT_ACCEPT) {

                                set_event_accept(&ring, sockfd, (struct sockaddr*)&clientaddr, &len, 0);

                                int connfd = entries->res;
                                set_event_recv(&ring, connfd, buffer, BUFFER_LENGTH, 0);

                        } else if (result.event == EVENT_READ) {  //

                                int ret = entries->res;

                                if (ret == 0) {
                                        close(result.fd);
                                } else if (ret > 0) {
                                        set_event_send(&ring, result.fd, buffer, ret, 0);
                                }
                        }  else if (result.event == EVENT_WRITE) {
                                int ret = entries->res;
                                set_event_recv(&ring, result.fd, buffer, BUFFER_LENGTH, 0);
                        }

                }

                io_uring_cq_advance(&ring, nready);
        }

}

#include <stdio.h>
#include <string.h>
#include <sys/socket.h>
#include <stdlib.h>
#include <unistd.h>

#include <sys/time.h>
#include <pthread.h>
#include <arpa/inet.h>


typedef struct test_context_s {

        char serverip[16];
        int port;
        int threadnum;
        int connection;
        int requestion;

#if 1
        int failed;
#endif

} test_context_t;

typedef struct test_context_s test_context_t;

int connect_tcpserver(const char *ip, unsigned short port) {

        int connfd = socket(AF_INET, SOCK_STREAM, 0);

        struct sockaddr_in tcpserver_addr;
        memset(&tcpserver_addr, 0, sizeof(struct sockaddr_in));

        tcpserver_addr.sin_family = AF_INET;
        tcpserver_addr.sin_addr.s_addr = inet_addr(ip);
        tcpserver_addr.sin_port = htons(port);

        int ret = connect(connfd, (struct sockaddr*)&tcpserver_addr, sizeof(struct sockaddr_in));
        if (ret) {
                perror("connect");
                return -1;
        }

        return connfd;
}

#define TIME_SUB_MS(tv1, tv2)  ((tv1.tv_sec - tv2.tv_sec) * 1000 + (tv1.tv_usec - tv2.tv_usec) / 1000)


#define TEST_MESSAGE   "ABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890abcdefghijklmnopqrstuvwxyz\r\n"
#define RBUFFER_LENGTH          2048

#define WBUFFER_LENGTH          2048

int send_recv_tcppkt(int fd) {


#if 0
        int res = send(fd, TEST_MESSAGE, strlen(TEST_MESSAGE), 0);
        if (res < 0) {
                exit(1);
        }

        char rbuffer[RBUFFER_LENGTH] = {0};
        res = recv(fd, rbuffer, RBUFFER_LENGTH, 0);
        if (res <= 0) {
                exit(1);
        }

        if (strcmp(rbuffer, TEST_MESSAGE) != 0) {
                printf("failed: '%s' != '%s'\n", rbuffer, TEST_MESSAGE);
                return -1;
        }
#else


        char wbuffer[WBUFFER_LENGTH] = {0};
        int i = 0;

        for (i = 0;i < 8;i ++) {
                strcpy(wbuffer + i * strlen(TEST_MESSAGE), TEST_MESSAGE);
        }

        int res = send(fd, wbuffer, strlen(wbuffer), 0);
        if (res < 0) {
                exit(1);
        }

        char rbuffer[RBUFFER_LENGTH] = {0};
        res = recv(fd, rbuffer, RBUFFER_LENGTH, 0);
        if (res <= 0) {
                exit(1);
        }

        if (strcmp(rbuffer, wbuffer) != 0) {
                printf("failed: '%s' != '%s'\n", rbuffer, wbuffer);
                return -1;
        }

#endif

        return 0;
}

static void *test_qps_entry(void *arg) {

        test_context_t *pctx = (test_context_t*)arg;


        int connfd = connect_tcpserver(pctx->serverip, pctx->port);
        if (connfd < 0) {
                printf("connect_tcpserver failed\n");
                return NULL;
        }

        int count = pctx->requestion / pctx->threadnum;
        int i = 0;

        int res;

        while (i++ < count) {
                res = send_recv_tcppkt(connfd);
                if (res != 0) {
                        printf("send_recv_tcppkt failed\n");
                        pctx->failed ++; // 
                        continue;
                }
        }

        return NULL;
}

// ./test_qps_tcpclient -s 127.0.0.1 -p 2048 -t 50 -c 100 -n 10000
int main(int argc, char *argv[]) {

        int ret = 0;
        test_context_t ctx = {0};


        int opt;
        while ((opt = getopt(argc, argv, "s:p:t:c:n:?")) != -1) {

                switch (opt) {

                        case 's':
                                printf("-s: %s\n", optarg);
                                strcpy(ctx.serverip, optarg);
                                break;

                        case 'p':
                                printf("-p: %s\n", optarg);

                                ctx.port = atoi(optarg);
                                break;

                        case 't':
                                printf("-t: %s\n", optarg);
                                ctx.threadnum = atoi(optarg);
                                break;

                        case 'c':
                                printf("-c: %s\n", optarg);
                                ctx.connection = atoi(optarg);
                                break;

                        case 'n':
                                printf("-n: %s\n", optarg);
                                ctx.requestion = atoi(optarg);
                                break;

                        default:
                                return -1;

                }

        }

        pthread_t *ptid = malloc(ctx.threadnum * sizeof(pthread_t));
        int i = 0;

        struct timeval tv_begin;
        gettimeofday(&tv_begin, NULL);
        for (i = 0;i < ctx.threadnum;i ++) {
                pthread_create(&ptid[i], NULL, test_qps_entry, &ctx);
        }

        for (i = 0;i < ctx.threadnum;i ++) {
                pthread_join(ptid[i], NULL);
        }

        struct timeval tv_end;
        gettimeofday(&tv_end, NULL);

        int time_used = TIME_SUB_MS(tv_end, tv_begin);


        printf("success: %d, failed: %d, time_used: %d, qps: %d\n", ctx.requestion-ctx.failed, 
                ctx.failed, time_used, ctx.requestion * 1000 / time_used);


clean: 
        free(ptid);

        return ret;
}