qemu侧 网络包收包调试记录

前言

void net_init_tap_one(...) {
    TAPState *s = net_tap_fd_init(peer, model, name, fd, vnet_hdr);
    // 创建tap 网络设备
        nc = qemu_new_net_client(&net_tap_info, peer, model, name);
            qemu_net_client_setup(nc, info, peer, model, name, qemu_net_client_destructor, true);
            	nc->info = info;
            	nc->incoming_queue = qemu_new_net_queue(qemu_deliver_packet_iov, nc);
    			nc->destructor = destructor;
    			nc->is_datapath = is_datapath;
                QTAILQ_INSERT_TAIL(&net_clients, nc, next);
                QTAILQ_INIT(&nc->filters);

        TAPState *s = DO_UPCAST(TAPState, nc, nc);
        tap_set_offload(&s->nc, 0, 0, 0, 0, 0);
        tap_read_poll(s, true);
            tap_update_fd_handler(s);
                // 找到iohandler_ctx中对应AioHander节点赋值io_read和io_write, 加入到source
                qemu_set_fd_handler(s->fd, tap_send ，tap_writable,s);
                    aio_set_fd_handler(iohandler_ctx, fd, false,
                       fd_read, fd_write, NULL, NULL, opaque);

        s->exit.notify = tap_exit_notify;
        qemu_add_exit_notifier(&s->exit);

断点调试net_init_tap_one

(gdb) b net_init_tap_one
(gdb) i args
tap = 0x55555689e750
peer = 0x0
model = 0x55555603d3ad "tap"
name = 0x555556ad0f10 "plr2"
ifname = 0x7fffffffde00 "tap0"
script = 0x555556ad0e10 "no"
downscript = 0x555556acf0f0 "no"
vhostfdname = 0x0
vnet_hdr = 1
fd = 13 // 这个对应是"/dev/net/tun" 设备
errp = 0x7fffffffddb8

(gdb) n
681         TAPState *s = net_tap_fd_init(peer, model, name, fd, vnet_hdr);
(gdb) n
684         tap_set_sndbuf(s->fd, tap, &err);
(gdb) p s->nc
$90 = {info = 0x5555566aab80 <net_tap_info>, ...
(gdb) p {s->read_poll ,s->write_poll,s->using_vnet_hdr,s->using_vnet_hdr,s->has_ufo,s->enabled}
$93 = {true, false, false, false, true, true}
(gdb) p s->host_vnet_hdr_len 
$95 = 10
(gdb) p s->exit 
$96 = {notify = 0x5555558f92f3 <tap_exit_notify>, node = {le_next = 
    0x5555567c1650 <qemu_unlink_pidfile_notifier>, le_prev = 0x5555567c13c0 <exit_notifiers>}}
(gdb) s
tap_set_sndbuf (fd=13, tap=0x55555689e750, errp=) at ../net/tap-linux.c:134
(gdb) n
696             snprintf(s->nc.info_str, sizeof(s->nc.info_str),
(gdb) n
700             if (strcmp(downscript, "no") != 0) {
// 后面没执行啥

调试 net_tap_fd_init

(gdb) n
410         	tap_fd_set_vnet_hdr_len(s->fd, s->host_vnet_hdr_len);
(gdb) n
412         tap_read_poll(s, true);

(gdb) s
tap_read_poll () at ../net/tap.c:82
(gdb) s
tap_update_fd_handler () at ../net/tap.c:74
(gdb) s
qemu_set_fd_handler (fd=13, fd_read=0x5555558f8d2d <tap_send>, fd_write=0x0 ,TAPState) at ../util/main-loop.c:582
(gdb) s
aio_set_fd_handler (ctx=iohandler_ctx, fd, is_external=false, 
    io_read=tap_send, io_write=0, 
    io_poll=0, io_poll_ready=0, opaque=TAPState)
   
120         node = find_aio_handler(ctx, fd);
(gdb) n
123         if (!io_read && !io_write && !io_poll) {
(gdb) p *node
Cannot access memory at address 0x0
(gdb) n
133             poll_disable_change = !io_poll - (node && !node->io_poll);
149                 new_node->pfd.fd = fd;
(gdb) n
153             g_source_add_poll(&ctx->source, &new_node->pfd);
(gdb) p ctx->source 
$98 = {callback_data = 0x0, callback_funcs = 0x0, source_funcs = 0x5555567bcac0 <aio_source_funcs>, ...  name = 0x5555568a26e0 "io-handler", }
(gdb) n
158             QLIST_INSERT_HEAD_RCU(&ctx->aio_handlers, new_node, node);
(gdb) p new_node->pfd.events 
$99 = 25

tap_send

(gdb) bt
#0  tap_send (opaque=0x555555f91ec6 <_nocheck__trace_lockcnt_fast_path_success+38>) at ../net/tap.c:186
#1  0x0000555555f69a0c in aio_dispatch_handler (ctx=0x5555568998f0, node=0x5555570fb570)
    at ../util/aio-posix.c:368
#2  0x0000555555f69bcd in aio_dispatch_handlers (ctx=0x5555568998f0) at ../util/aio-posix.c:411
#3  0x0000555555f69c27 in aio_dispatch (ctx=0x5555568998f0) at ../util/aio-posix.c:421
#4  0x0000555555f82020 in aio_ctx_dispatch (source=0x5555568998f0, callback=0x0, user_data=0x0)
    at ../util/async.c:312
#5  0x00007ffff7a7117d in g_main_context_dispatch () at /lib/x86_64-linux-gnu/libglib-2.0.so.0
#6  0x0000555555f92dab in glib_pollfds_poll () at ../util/main-loop.c:232
#7  0x0000555555f92e29 in os_host_main_loop_wait (timeout=147477372) at ../util/main-loop.c:255
#8  0x0000555555f92f3a in main_loop_wait (nonblocking=0) at ../util/main-loop.c:531
#9  0x0000555555897ca9 in qemu_main_loop () at ../softmmu/runstate.c:727
#10 0x000055555583fa5a in main (argc=16, argv=0x7fffffffe2d8, envp=0x7fffffffe360) at ../softmmu/main.c:52

(gdb) n
tap_send (opaque=0x555556af3730) at ../net/tap.c:196
196             size = tap_read_packet(s->fd, s->buf, sizeof(s->buf));
(gdb) n
197             if (size <= 0) {
(gdb) print s->buf 
value requires 69632 bytes, which is more than max-value-size
(gdb) print size 
$100 = 74
(gdb) p *(s->buf)@74
$101 = "\000\000\000\000\000\000\000\000\000\000\377\377\377\377\377\377\000\000\000\000\000\001\b\006\000\001\b\000\006\004\000\001\000\000\000\000\000\001\061{@\001\000\000\000\000\000\000\061{D\346", '\000' <repeats 21 times>

(gdb) x /74xb s->buf
0x555556af3d2c: 0x00    0x00    0x00    0x00    0x00    0x00    0x00    0x00
0x555556af3d34: 0x00    0x00    0xff    0xff    0xff    0xff    0xff    0xff
0x555556af3d3c: 0x00    0x00    0x00    0x00    0x00    0x01    0x08    0x06
0x555556af3d44: 0x00    0x01    0x08    0x00    0x06    0x04    0x00    0x01
0x555556af3d4c: 0x00    0x00    0x00    0x00    0x00    0x01    0x31    0x7b
0x555556af3d54: 0x40    0x01    0x00    0x00    0x00    0x00    0x00    0x00
0x555556af3d5c: 0x31    0x7b    0x44    0xe6    0x00    0x00    0x00    0x00
0x555556af3d64: 0x00    0x00    0x00    0x00    0x00    0x00    0x00    0x00
0x555556af3d6c: 0x00    0x00    0x00    0x00    0x00    0x00    0x00    0x00
0x555556af3d74: 0x00    0x00

(gdb) n
201             if (s->host_vnet_hdr_len && !s->using_vnet_hdr) {
(gdb) n
202                 buf  += s->host_vnet_hdr_len;
(gdb) n
203                 size -= s->host_vnet_hdr_len;
(gdb) p size
$102 = 64
(gdb) s
qemu_send_packet_async (sender=sender=0x555556af3730, buf=, size=64, sent_cb=tap_send_completed)
    at ../net/net.c:697
(gdb) s
qemu_send_packet_async_with_flags (sender=, flags=0, 
    buf= , size=64, sent_cb=) at ../net/net.c:663
(gdb) p *sender
$103 = {info = 0x5555566aab80 <net_tap_info>, ..
(gdb) p *(sender->peer)
$104 = {info = 0x5555566cb520 <net_e1000_info>, 

(gdb) s
qemu_net_queue_send (queue=, sender=, flags=0, 
    data=, size=64, sent_cb=tap_send_completed)
    at ../net/queue.c:213

(gdb) s
qemu_net_queue_deliver (queue=, sender=, flags=, 
    data=, size=) at ../net/queue.c:156
(gdb) n
164         ret = queue->deliver(sender, flags, &iov, 1, queue->opaque);
(gdb) s
qemu_deliver_packet_iov (sender=, flags=, iov=, iovcnt=,  opaque=) at ../net/net.c:768
(gdb)  p *nc
// 与网络发包过程nc 是相反的
$106 = {info = 0x5555566cb520 <net_e1000_info>, ...
(gdb) n
782             ret = nc->info->receive_iov(nc, iov, iovcnt);
(gdb) s
e1000_receive_iov (nc=, iov=, iovcnt=1) at ../hw/net/e1000.c:961

e1000_receive_iov

数据包从buf中通过pci_dma_write接口注入到了e1000的数据包接收内存中，当然这里的dma并不是真正的硬件DMA操作，而是虚拟化成普通内存的写，因为Guest OS的物理内存是Qemu的虚拟内存，因此Qemu可以直接访问，而Guest并不知道这一切。最后通过set_ics进行Guest收包中断的注入 (虚拟中断)

(gdb) n
988         if (size < sizeof(min_buf)) {
(gdb) n
995         } else if (iov->iov_len < MAXIMUM_ETHERNET_HDR_LEN) {
(gdb) n
1002        if (e1000x_is_oversized(s->mac_reg, size)) {

(gdb) n
1010        if (e1000x_vlan_enabled(s->mac_reg) &&
(gdb) n
1027        rdh_start = s->mac_reg[RDH];
(gdb) n
1028        desc_offset = 0;
(gdb) n
1029        total_size = size + e1000x_fcs_len(s->mac_reg);
(gdb) p rdh_start
$107 = 196
(gdb) p total_size 
$108 = 140737344377825

(gdb) n
1035            desc_size = total_size - desc_offset;
(gdb) n
1040            pci_dma_read(d, base, &desc, sizeof(desc));
(gdb) n
1046                    hwaddr ba = le64_to_cpu(desc.buffer_addr);
(gdb) n
1053                        pci_dma_write(d, ba, iov->iov_base + iov_ofs, iov_copy);

(gdb) p {iov_copy,iov_ofs,copy_size}
$113 = {64, 0, 64}

(gdb) x /64xb ba
0x3d3adc0:      Cannot access memory at address 0x3d3adc0

(gdb) n
1063                desc_offset += desc_size;
(gdb) n
1064                desc.length = cpu_to_le16(desc_size);
(gdb) n
1075            pci_dma_write(d, base, &desc, sizeof(desc))


(gdb) s
set_ics (s=0x5555575578d0, index=0, val=128) at ../hw/net/e1000.c:405
(gdb) s
408         set_interrupt_cause(s, 0, val | s->mac_reg[ICR]);
(gdb) s
set_interrupt_cause (args: s=, index=0, val=128
) at ../hw/net/e1000.c:327