dpdk18.11 收发包流程分析

pci probe

RTE_PMD_REGISTER_PCI(net_ixgbe, rte_ixgbe_pmd); 宏注册了net_ixgbe driver到pci bus

rte_ixgbe_pmd 的定义如下

static struct rte_pci_driver rte_ixgbe_pmd = {
    .id_table = pci_id_ixgbe_map,
    .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC,
    .probe = eth_ixgbe_pci_probe,
    .remove = eth_ixgbe_pci_remove,
};

当扫描到到的device 能和pci_id_ixgbe_map 里的id 匹配上时就表示这个device和net_ixgbe 这个driver相匹配，接着便会调用.probe = eth_ixgbe_pci_probe指定的probe函数,这里我们关注下probe 函数对收发报函数的设置

static int
->eth_ixgbe_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,struct rte_pci_device *)
	    ->retval = rte_eth_dev_create(&pci_dev->device, pci_dev->device.name,
        sizeof(struct ixgbe_adapter),
        eth_dev_pci_specific_init, pci_dev,
        eth_ixgbe_dev_init, NULL);
        	->eth_ixgbe_dev_init(struct rte_eth_dev *, void *init_params )

eth_ixgbe_dev_init 中会完成driver的初始化，其中就包括了调用 ixgbe_set_tx_function、ixgbe_set_rx_function根据情况设置对应的收发函数

static int
eth_ixgbe_dev_init(struct rte_eth_dev *eth_dev, void *init_params __rte_unused)
{
	 /*省略*/
    eth_dev->dev_ops = &ixgbe_eth_dev_ops;
    eth_dev->rx_pkt_burst = &ixgbe_recv_pkts;
    eth_dev->tx_pkt_burst = &ixgbe_xmit_pkts;
    eth_dev->tx_pkt_prepare = &ixgbe_prep_pkts;
     /*省略*/
    if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
    struct ixgbe_tx_queue *txq;
    /* TX queue function in primary, set by last queue initialized
     * Tx queue may not initialized by primary process
     */
    if (eth_dev->data->tx_queues) {
        txq = eth_dev->data->tx_queues[eth_dev->data->nb_tx_queues-1];
        ixgbe_set_tx_function(eth_dev, txq);
    } else {
        /* Use default TX function if we get here */
        PMD_INIT_LOG(NOTICE, "No TX queues configured yet. "
                 "Using default TX function.");
    }

    	ixgbe_set_rx_function(eth_dev);
        return 0;	//对于从核只会执行到这里
    }
    /*省略*/
    return 0;
}

设置收发函数

1. tx函数设置
   • 发送队列offloads为0时
     • 未启用向量 tx_pkt_burst = ixgbe_xmit_pkts_simple
     • 启用向量 dev->tx_pkt_burst = ixgbe_xmit_pkts_vec;
   • 发送队列offloads非0
     • dev->tx_pkt_burst = ixgbe_xmit_pkts
2. rx函数设置
   • if dev->data->lro设置时
     • adapter->rx_bulk_alloc_allowed 设置，允许bulk时dev->rx_pkt_burst = ixgbe_recv_pkts_lro_bulk_alloc;
     • dev->rx_pkt_burst = ixgbe_recv_pkts_lro_single_alloc;
   • else if dev->data->scattered_rx 设置时
     • dev->data->scattered_rx 设置，支持向量时 dev->rx_pkt_burst = ixgbe_recv_scattered_pkts_vec;
     • adapter->rx_bulk_alloc_allowed 设置支持bulk时 dev->rx_pkt_burst = ixgbe_recv_pkts_lro_bulk_alloc;
     • dev->rx_pkt_burst = ixgbe_recv_pkts_lro_single_alloc;
   • else ifadapter->rx_vec_allowed 设置时
     • dev->rx_pkt_burst = ixgbe_recv_pkts_vec;
   • else if adapter->rx_bulk_alloc_allowed 设置时
     • dev->rx_pkt_burst = ixgbe_recv_pkts_bulk_alloc;
   • else dev->rx_pkt_burst = ixgbe_recv_pkts;

启动网卡及配置收发队列

下面是以L2FWD为例，总结了下dpdk对网卡配置、收发队列配置的API。以下代码做了省略，删除了返回判断代码

int
main(int argc, char **argv)
{   
    struct lcore_queue_conf *qconf;
    int ret;
    uint16_t nb_ports;
    uint16_t nb_ports_available = 0;
    uint16_t portid, last_port;
    unsigned lcore_id, rx_lcore_id;
    unsigned nb_ports_in_mask = 0;
    unsigned int nb_lcores = 0;
    unsigned int nb_mbufs;
        
    /* init EAL */
    ret = rte_eal_init(argc, argv);
    if (ret < 0)
        rte_exit(EXIT_FAILURE, "Invalid EAL arguments\n");
    argc -= ret;
    argv += ret;
    
    force_quit = false;
    signal(SIGINT, signal_handler);
    signal(SIGTERM, signal_handler);
    
    /* 解析传入的参数 (after the EAL ones) */
    ret = l2fwd_parse_args(argc, argv);
    if (ret < 0)
        rte_exit(EXIT_FAILURE, "Invalid L2FWD arguments\n");
        
    printf("MAC updating %s\n", mac_updating ? "enabled" : "disabled");
              
    nb_ports = rte_eth_dev_count_avail();
    if (nb_ports == 0)
        rte_exit(EXIT_FAILURE, "No Ethernet ports - bye\n");

    /* check port mask to possible port mask */
    if (l2fwd_enabled_port_mask & ~((1 << nb_ports) - 1))
        rte_exit(EXIT_FAILURE, "Invalid portmask; possible (0x%x)\n",
            (1 << nb_ports) - 1);

	/*计算所需的mbuf个数*/
    nb_mbufs = RTE_MAX(nb_ports * (nb_rxd + nb_txd + MAX_PKT_BURST +
        nb_lcores * MEMPOOL_CACHE_SIZE), 8192U);

    /* 创建内存池 */
    l2fwd_pktmbuf_pool = rte_pktmbuf_pool_create("mbuf_pool", nb_mbufs,
        MEMPOOL_CACHE_SIZE, 0, RTE_MBUF_DEFAULT_BUF_SIZE,
        rte_socket_id());

    /* Initialise each port */
    RTE_ETH_FOREACH_DEV(portid) {
        struct rte_eth_rxconf rxq_conf;
        struct rte_eth_txconf txq_conf;
        struct rte_eth_conf local_port_conf = port_conf;
        struct rte_eth_dev_info dev_info;

        /* skip ports that are not enabled */
        if ((l2fwd_enabled_port_mask & (1 << portid)) == 0) {
            printf("Skipping disabled port %u\n", portid);
            continue;
        }
        nb_ports_available++;

        /* init port */
        printf("Initializing port %u... ", portid);
        fflush(stdout);
        rte_eth_dev_info_get(portid, &dev_info);
        if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_MBUF_FAST_FREE)
            local_port_conf.txmode.offloads |=
                DEV_TX_OFFLOAD_MBUF_FAST_FREE;
        ret = rte_eth_dev_configure(portid, 1, 1, &local_port_conf);

        ret = rte_eth_dev_adjust_nb_rx_tx_desc(portid, &nb_rxd,
                               &nb_txd);

        rte_eth_macaddr_get(portid,&l2fwd_ports_eth_addr[portid]);

        /* init one RX queue */
        fflush(stdout);
        rxq_conf = dev_info.default_rxconf;
        rxq_conf.offloads = local_port_conf.rxmode.offloads;
        ret = rte_eth_rx_queue_setup(portid, 0, nb_rxd,
                         rte_eth_dev_socket_id(portid),
                         &rxq_conf,
                         l2fwd_pktmbuf_pool);
        if (ret < 0)
            rte_exit(EXIT_FAILURE, "rte_eth_rx_queue_setup:err=%d, port=%u\n",
                  ret, portid);

        /* init one TX queue on each port */
        fflush(stdout);
        txq_conf = dev_info.default_txconf;
        txq_conf.offloads = local_port_conf.txmode.offloads;
        ret = rte_eth_tx_queue_setup(portid, 0, nb_txd,
                rte_eth_dev_socket_id(portid),
                &txq_conf);

        /* Initialize TX buffers */
        tx_buffer[portid] = rte_zmalloc_socket("tx_buffer",
                RTE_ETH_TX_BUFFER_SIZE(MAX_PKT_BURST), 0,
                rte_eth_dev_socket_id(portid));
        if (tx_buffer[portid] == NULL)
            rte_exit(EXIT_FAILURE, "Cannot allocate buffer for tx on port %u\n",
                    portid);

        rte_eth_tx_buffer_init(tx_buffer[portid], MAX_PKT_BURST);
		/*设置错误统计回调函数*/
        ret = rte_eth_tx_buffer_set_err_callback(tx_buffer[portid],
                rte_eth_tx_buffer_count_callback,
                &port_statistics[portid].dropped);

        /* Start device */
        ret = rte_eth_dev_start(portid);

        rte_eth_promiscuous_enable(portid);
    }

    if (!nb_ports_available) {
        rte_exit(EXIT_FAILURE,
            "All available ports are disabled. Please set portmask.\n");
    }

    check_all_ports_link_status(l2fwd_enabled_port_mask);

    ret = 0;
    /* launch per-lcore init on every lcore */
    rte_eal_mp_remote_launch(l2fwd_launch_one_lcore, NULL, CALL_MASTER);
    RTE_LCORE_FOREACH_SLAVE(lcore_id) {
        if (rte_eal_wait_lcore(lcore_id) < 0) {
            ret = -1;
            break;
        }
    }

    RTE_ETH_FOREACH_DEV(portid) {
        if ((l2fwd_enabled_port_mask & (1 << portid)) == 0)
            continue;
        printf("Closing port %d...", portid);
        rte_eth_dev_stop(portid);
        rte_eth_dev_close(portid);
        printf(" Done\n");
    }
    printf("Bye...\n");

    return ret;
}

rte_eth_dev_configure

rte_eth_dev_configure是EAL对外提供的api（lib/librte_ethdev/rte_ethdev.c），下面分析下dpdk中对网卡设备的配置

rte_eth_rx_queue_setup

rte_eth_rx_queue_setup是EAL对外提供的api（lib/librte_ethdev/rte_ethdev.c），下面分析下dpdk中对队列的配置

• 在rte_eth_dev_configure中设置的offload属性会被设置到每个队列
• 调用调用eth_dev_ops中的rx_queue_setup回调函数
  在ixgbe网卡中，对应的rx_queue_setup回调函数是ixgbe_dev_rx_queue_setup

rte_eth_rx_queue_setup就完成了

下面就是启动网卡设备了：