DPDK初始化Mellanox失败（mlx5）无法获取VF

记录一下，方便以后查阅。

错误如下

1、
EAL: VFIO support initialized
EAL: Probe PCI driver: mlx5_pci (15b3:101e) device: 0000:12:01.0 (socket 0)
mlx5_pci: no Verbs device matches PCI device 0000:12:01.0, are kernel drivers loaded?
common_mlx5: Failed to load driver = mlx5_pci.
EAL: Requested device 0000:b5:01.2 cannot be used
EAL: Bus (pci) probe failed.
EAL: No legacy callbacks, legacy socket not created

代码如下
code from：
https://github.com/zartbot/learn_dpdk

#include <stdint.h>
#include <unistd.h>
#include <inttypes.h>
#include <rte_eal.h>
#include <rte_ethdev.h>
#include <rte_cycles.h>
#include <rte_lcore.h>
#include <rte_mbuf.h>
#include <rte_ether.h>
#include <rte_ip.h>
#include <rte_udp.h>
#include <pthread.h>
#include <string.h>

#define MAX_PORTS 16
#define RX_RING_SIZE 1024
#define TX_RING_SIZE 1024

#define NUM_MBUFS 8191
#define MBUF_CACHE_SIZE 250
#define BURST_SIZE 32

static int hwts_dynfield_offset = -1;

static inline rte_mbuf_timestamp_t *
hwts_field(struct rte_mbuf *mbuf)
{
 return RTE_MBUF_DYNFIELD(mbuf,
   hwts_dynfield_offset, rte_mbuf_timestamp_t *);
}

typedef uint64_t tsc_t;
static int tsc_dynfield_offset = -1;

static inline tsc_t *
tsc_field(struct rte_mbuf *mbuf)
{
 return RTE_MBUF_DYNFIELD(mbuf, tsc_dynfield_offset, tsc_t *);
}

static uint16_t
add_timestamps(uint16_t port __rte_unused, uint16_t qidx __rte_unused,
  struct rte_mbuf **pkts, uint16_t nb_pkts,
  uint16_t max_pkts __rte_unused, void *_ __rte_unused)
{
 unsigned i;
 uint64_t now = rte_rdtsc();

 for (i = 0; i < nb_pkts; i++)
  printf("packet[%d] sendtime: %ld\n",i,now);
 return nb_pkts;
}

static const struct rte_eth_conf port_conf_default = {
    .rxmode = {
        .max_rx_pkt_len = RTE_ETHER_MAX_LEN,
    },
};

static inline int
port_init(uint16_t port, struct rte_mempool *mbuf_pool)
{
    struct rte_eth_conf port_conf = port_conf_default;
    const uint16_t rx_rings = 1, tx_rings = 1;
    uint16_t nb_rxd = RX_RING_SIZE;
    uint16_t nb_txd = TX_RING_SIZE;
    int retval;
    uint16_t q;
    struct rte_eth_dev_info dev_info;
    struct rte_eth_txconf txconf;

    if (!rte_eth_dev_is_valid_port(port))
        return -1;

    retval = rte_eth_dev_info_get(port, &dev_info);
    if (retval != 0)
    {
        printf("Error during getting device (port %u) info: %s\n",
               port, strerror(-retval));
        return retval;
    }
    printf("\n\ninitializing port %d...\n", port);


    if (dev_info.rx_offload_capa & DEV_RX_OFFLOAD_CHECKSUM)
    {
        printf("port[%u] support RX cheksum offload.\n", port);
        port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_CHECKSUM;
    }


 if (!(dev_info.rx_offload_capa & DEV_RX_OFFLOAD_TIMESTAMP)) {
   printf("\nERROR: Port %u does not support hardware timestamping\n"
     , port);
   return -1;
  }
  port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_TIMESTAMP;
  rte_mbuf_dyn_rx_timestamp_register(&hwts_dynfield_offset, NULL);
  if (hwts_dynfield_offset < 0) {
   printf("ERROR: Failed to register timestamp field\n");
   return -rte_errno;
 }

    if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_MBUF_FAST_FREE)
    {
        printf("port[%u] support TX mbuf fast free offload.\n", port);
        port_conf.txmode.offloads |= DEV_TX_OFFLOAD_MBUF_FAST_FREE;
    }

    if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_MT_LOCKFREE)
    {
        printf("port[%u] support TX MT lock free offload.\n", port);
        port_conf.txmode.offloads |= DEV_TX_OFFLOAD_MT_LOCKFREE;
    }

    if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_IPV4_CKSUM)
    {
        printf("port[%u] support TX IPv4 checksum offload.\n", port);
        port_conf.txmode.offloads |= DEV_TX_OFFLOAD_IPV4_CKSUM;
    }

    if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_UDP_CKSUM)
    {
        printf("port[%u] support TX UDP checksum offload.\n", port);
        port_conf.txmode.offloads |= DEV_TX_OFFLOAD_UDP_CKSUM;
    }

    if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_TCP_CKSUM)
    {
        printf("port[%u] support TX TCP checksum offload.\n", port);
        port_conf.txmode.offloads |= DEV_TX_OFFLOAD_TCP_CKSUM;
    }

    if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_SCTP_CKSUM)
    {
        printf("port[%u] support TX SCTP checksum offload.\n", port);
        port_conf.txmode.offloads |= DEV_TX_OFFLOAD_SCTP_CKSUM;
    }

    /* Configure the Ethernet device. */
    retval = rte_eth_dev_configure(port, rx_rings, tx_rings, &port_conf);
    if (retval != 0)
        return retval;

    retval = rte_eth_dev_adjust_nb_rx_tx_desc(port, &nb_rxd, &nb_txd);
    if (retval != 0)
        return retval;

    /* Allocate and set up 1 RX queue per Ethernet port. */
    for (q = 0; q < rx_rings; q++)
    {
        retval = rte_eth_rx_queue_setup(port, q, nb_rxd,
                                        rte_eth_dev_socket_id(port), NULL, mbuf_pool);
        if (retval < 0)
            return retval;
    }

    txconf = dev_info.default_txconf;
    txconf.offloads = port_conf.txmode.offloads;
    /* Allocate and set up 1 TX queue per Ethernet port. */
    for (q = 0; q < tx_rings; q++)
    {
        retval = rte_eth_tx_queue_setup(port, q, nb_txd,
                                        rte_eth_dev_socket_id(port), &txconf);
        if (retval < 0)
            return retval;
    }

    /* Start the Ethernet port. */
    retval = rte_eth_dev_start(port);
    if (retval < 0)
        return retval;

    struct rte_eth_link link;
    do
    {
        retval = rte_eth_link_get_nowait(port, &link);
        if (retval < 0)
        {
            printf("Failed link get (port %u): %s\n",
                   port, rte_strerror(-retval));
            return retval;
        }
        else if (link.link_status)
            break;

        printf("Waiting for Link up on port %" PRIu16 "\n", port);
        sleep(1);
    } while (!link.link_status);

    /* Display the port MAC address. */
    struct rte_ether_addr addr;
    retval = rte_eth_macaddr_get(port, &addr);
    if (retval != 0)
        return retval;

    printf("Port[%u] MAC: %02" PRIx8 ":%02" PRIx8 ":%02" PRIx8
           ":%02" PRIx8 ":%02" PRIx8 ":%02" PRIx8 "\n",
           port,
           addr.addr_bytes[0], addr.addr_bytes[1],
           addr.addr_bytes[2], addr.addr_bytes[3],
           addr.addr_bytes[4], addr.addr_bytes[5]);

    /* Enable RX in promiscuous mode for the Ethernet device. */
    retval = rte_eth_promiscuous_enable(port);
    if (retval != 0)
        return retval;

    //rte_eth_add_rx_callback(port, 0, add_timestamps, NULL);

    return 0;
}

rte_be32_t string_to_ip(char *s)
{
    unsigned char a[4];
    int rc = sscanf(s, "%hhd.%hhd.%hhd.%hhd", a + 0, a + 1, a + 2, a + 3);
    if (rc != 4)
    {
        fprintf(stderr, "bad source IP address format. Use like: 1.1.1.1\n");
        exit(1);
    }
    return (rte_be32_t)(a[3]) << 24 |
           (rte_be32_t)(a[2]) << 16 |
           (rte_be32_t)(a[1]) << 8 |
           (rte_be32_t)(a[0]);
}




static int
lcore_send(struct rte_mempool *mbuf_pool) {

    struct rte_ether_hdr *eth_hdr;
    struct rte_ipv4_hdr *ipv4_hdr;
    struct rte_udp_hdr *udp_hdr;

    //Defined header in UDP
    struct SRoU
    {
        uint8_t magic_num;
        uint8_t srou_length;
        uint8_t flags;
        uint8_t next_protcol;
        uint64_t pad;
    };

    //init mac
    struct rte_ether_addr s_addr = {{0x14, 0x02, 0xEC, 0x89, 0x8D, 0x24}};
    struct rte_ether_addr d_addr = {{0x3c, 0xfd, 0xfe, 0xa9, 0xa8, 0x89}};

    //init IP header
    rte_be32_t s_ip_addr = string_to_ip("1.0.0.253");
    rte_be32_t d_ip_addr = string_to_ip("1.0.0.1");
    uint16_t ether_type = rte_cpu_to_be_16(0x0800);

    //init udp payload
    struct SRoU obj = {
        .magic_num = 1,
        .srou_length = 4,
        .flags = 0xFF,
        .next_protcol = 0,
    };
    struct SRoU *msg;
    struct rte_mbuf *pkt[BURST_SIZE];

 

    for(;;) {
           for (int i = 0; i < BURST_SIZE; i++)
    {
        pkt[i] = rte_pktmbuf_alloc(mbuf_pool);
 }

 for (int i = 0; i < BURST_SIZE; i++)
    {
        eth_hdr = rte_pktmbuf_mtod(pkt[i], struct rte_ether_hdr *);
        eth_hdr->d_addr = d_addr;
         struct rte_ether_addr s_addr = {{0x14, 0x02, 0xEC, 0x89, 0x8D, i}};
        eth_hdr->s_addr = s_addr;
        eth_hdr->ether_type = ether_type;

        ipv4_hdr = rte_pktmbuf_mtod_offset(pkt[i], struct rte_ipv4_hdr *, sizeof(struct rte_ether_hdr));
        ipv4_hdr->version_ihl = 0x45;
        ipv4_hdr->next_proto_id = 0x11;
        ipv4_hdr->src_addr = s_ip_addr;
        ipv4_hdr->dst_addr = d_ip_addr;
        ipv4_hdr->time_to_live = 0x40;

        udp_hdr = rte_pktmbuf_mtod_offset(pkt[i], struct rte_udp_hdr *, sizeof(struct rte_ether_hdr) + sizeof(struct rte_ipv4_hdr));
        udp_hdr->dgram_len = rte_cpu_to_be_16(sizeof(struct SRoU) + sizeof(struct rte_udp_hdr));
        udp_hdr->src_port = rte_cpu_to_be_16(1234);
        udp_hdr->dst_port = rte_cpu_to_be_16(6666);
        ipv4_hdr->total_length = rte_cpu_to_be_16(sizeof(struct SRoU) + sizeof(struct rte_udp_hdr) + sizeof(struct rte_ipv4_hdr));

        msg = (struct SRoU *)(rte_pktmbuf_mtod(pkt[i], char *) + sizeof(struct rte_ether_hdr) + sizeof(struct rte_ipv4_hdr) + sizeof(struct rte_udp_hdr));
        *msg = obj;
        int pkt_size = sizeof(struct SRoU) + sizeof(struct rte_ether_hdr) + sizeof(struct rte_ipv4_hdr) + sizeof(struct rte_udp_hdr);

        pkt[i]->l2_len = sizeof(struct rte_ether_hdr);
        pkt[i]->l3_len = sizeof(struct rte_ipv4_hdr);
        pkt[i]->l4_len = sizeof(struct rte_udp_hdr);
        pkt[i]->ol_flags |= PKT_TX_IPV4 | PKT_TX_IP_CKSUM | PKT_TX_UDP_CKSUM;
        ipv4_hdr->hdr_checksum = 0;
        udp_hdr->dgram_cksum = rte_ipv4_phdr_cksum(ipv4_hdr, pkt[i]->ol_flags);
        pkt[i]->data_len = pkt_size;
        pkt[i]->pkt_len = pkt_size;
    }

    uint16_t nb_tx = rte_eth_tx_burst(0, 0, pkt, BURST_SIZE);
    for (int i = 0; i < BURST_SIZE; i++)
    {
        rte_pktmbuf_free(pkt[i]);
    }

   }

}

int main(int argc, char *argv[])
{

    struct rte_mempool *mbuf_pool;
    unsigned nb_ports;
    uint16_t portid;
 	uint64_t now = rte_rdtsc();
 	uint64_t pkt_cnt = 0;

    int ret = rte_eal_init(argc, argv);
    if (ret < 0)
        rte_exit(EXIT_FAILURE, "initlize fail!");

    printf("\n\n\n*****************************************\n");

    nb_ports = rte_eth_dev_count_avail();
    printf("number of available port: %d\n", nb_ports);

    /* Creates a new mempool in memory to hold the mbufs. */
    mbuf_pool = rte_pktmbuf_pool_create("MBUF_POOL", NUM_MBUFS * nb_ports,
                                        MBUF_CACHE_SIZE, 0, RTE_MBUF_DEFAULT_BUF_SIZE, rte_socket_id());

    if (mbuf_pool == NULL)
        rte_exit(EXIT_FAILURE, "Cannot create mbuf pool\n");

    /* Initialize all ports. */
    RTE_ETH_FOREACH_DEV(portid)
    if (port_init(portid, mbuf_pool) != 0)
        rte_exit(EXIT_FAILURE, "Cannot init port %" PRIu16 "\n",
                 portid);

    /* start packet send function on lcore-1 */
    rte_eal_remote_launch((lcore_function_t *)lcore_send,mbuf_pool,1);

    struct rte_mbuf *rx_pkt[BURST_SIZE];
    for (int i = 0; i < BURST_SIZE; i++)
    {
        rx_pkt[i] = rte_pktmbuf_alloc(mbuf_pool);
    }


    uint64_t freq = rte_get_tsc_hz() ;
 
    for (;;)
    {
        uint16_t nb_rx = rte_eth_rx_burst(1, 0, rx_pkt, BURST_SIZE);
        if (unlikely(nb_rx == 0))
        {
            continue;
        }
  pkt_cnt += nb_rx;
  
  if (unlikely(rte_rdtsc() - now > freq)) {
   printf("PPS: %ld\n",pkt_cnt);
   pkt_cnt= 0;
   now = rte_rdtsc() ;
  }
        /*struct rte_ether_hdr *eth_hdr;  
        for (int i = 0; i < nb_rx; i++)
        {
            eth_hdr = rte_pktmbuf_mtod(rx_pkt[i], struct rte_ether_hdr *);
            printf("Recv Pkt[%d] from MAC: %02" PRIx8 " %02" PRIx8 " %02" PRIx8
                   " %02" PRIx8 " %02" PRIx8 " %02" PRIx8 " \n",i,
                   eth_hdr->s_addr.addr_bytes[0], eth_hdr->s_addr.addr_bytes[1],
                   eth_hdr->s_addr.addr_bytes[2], eth_hdr->s_addr.addr_bytes[3],
                   eth_hdr->s_addr.addr_bytes[4], eth_hdr->s_addr.addr_bytes[5]);
            rte_pktmbuf_free(rx_pkt[i]);
        }*/
    }
    return 0;
}

使用的链接选项

-L/home/alen/dpdk-20.11.1/build/lib -L/home/alen/dpdk-20.11.1/build/drivers  -Wl,-Bdynamic -Wl,--whole-archive -Wl,-lrte_mempool_ring -Wl,-lrte_pci -Wl,-lrte_bus_pci -Wl,-lrte_bus_vdev -Wl,-lrte_net  -Wl,-lrte_distributor -Wl,-lrte_reorder -Wl,-lrte_kni -Wl,-lrte_pipeline -Wl,-lrte_table -Wl,-lrte_eventdev -Wl,-lrte_timer -Wl,-lrte_hash -Wl,-lrte_jobstats -Wl,-lrte_lpm -Wl,-lrte_power -Wl,-lrte_acl  -Wl,-lrte_meter -Wl,-lrte_sched -Wl,--start-group -Wl,-lrte_kvargs -Wl,-lrte_mbuf -Wl,-lrte_ip_frag -Wl,-lrte_ethdev -Wl,-lrte_cryptodev -Wl,-lrte_mempool -Wl,-lrte_ring -Wl,-lrte_eal -Wl,-lrte_cmdline  -Wl,-lrte_cfgfile  -Wl,-lrte_net_bond -Wl,-lrte_net_vmxnet3  -Wl,-lrte_net_virtio  -Wl,-lrte_net_cxgbe -Wl,-lrte_net_enic  -Wl,-lrte_net_i40e -Wl,-lrte_net_iavf -Wl,-lrte_net_ice -Wl,-lrte_net_fm10k -Wl,-lrte_net_ixgbe -Wl,-lrte_net_e1000  -Wl,-lrte_net_ring -Wl,-lrte_net_af_packet -Wl,-lrte_net_null -Wl,-lrte_telemetry -Wl,-lrte_security -Wl,-lrte_rcu -Wl,-lrte_common_iavf -Wl,-lrte_common_cpt -Wl,-lrte_common_dpaax -Wl,-lrte_common_octeontx -Wl,-lrte_common_octeontx2 -Wl,-lrte_common_sfc_efx -Wl,-lrte_bus_dpaa -Wl,-lrte_bus_fslmc -Wl,-lrte_bus_ifpga -Wl,-lrte_bus_vmbus -Wl,-lrte_mempool_bucket -Wl,-lrte_mempool_dpaa -Wl,-lrte_mempool_dpaa2 -Wl,-lrte_mempool_octeontx -Wl,-lrte_mempool_octeontx2 -Wl,-lrte_mempool_stack -Wl,-lrte_net_ark -Wl,-lrte_net_atlantic -Wl,-lrte_net_avp -Wl,-lrte_net_axgbe  -Wl,-lrte_net_bnx2x -Wl,-lrte_net_bnxt  -Wl,-lrte_net_dpaa -Wl,-lrte_net_dpaa2  -Wl,-lrte_net_ena -Wl,-lrte_net_enetc -Wl,-lrte_net_failsafe -Wl,-lrte_net_hinic -Wl,-lrte_net_hns3 -Wl,-lrte_net_igc  -Wl,-lrte_net_kni -Wl,-lrte_net_liquidio -Wl,-lrte_net_memif -Wl,-lrte_net_netvsc -Wl,-lrte_net_nfp -Wl,-lrte_net_octeontx -Wl,-lrte_net_octeontx2 -Wl,-lrte_net_pfe -Wl,-lrte_net_qede -Wl,-lrte_net_sfc -Wl,-lrte_net_tap -Wl,-lrte_net_thunderx -Wl,-lrte_net_txgbe -Wl,-lrte_net_vdev_netvsc -Wl,-lrte_net_vhost -Wl,-lrte_net_mlx5 -Wl,-lrte_common_mlx5 -Wl,-lrte_regex_mlx5 -Wl,-lrte_vdpa_mlx5 -Wl,-lrte_raw_dpaa2_cmdif -Wl,-lrte_raw_dpaa2_qdma -Wl,-lrte_raw_ioat -Wl,-lrte_raw_ntb -Wl,-lrte_raw_octeontx2_dma -Wl,-lrte_raw_octeontx2_ep -Wl,-lrte_raw_skeleton -Wl,-lrte_crypto_bcmfs -Wl,-lrte_crypto_caam_jr -Wl,-lrte_crypto_dpaa_sec -Wl,-lrte_crypto_dpaa2_sec -Wl,-lrte_crypto_nitrox -Wl,-lrte_crypto_null -Wl,-lrte_crypto_octeontx -Wl,-lrte_crypto_octeontx2 -Wl,-lrte_crypto_scheduler -Wl,-lrte_crypto_virtio -Wl,-lrte_compress_octeontx -Wl,-lrte_compress_zlib -Wl,-lrte_regex_octeontx2 -Wl,-lrte_vdpa_ifc -Wl,-lrte_event_dlb -Wl,-lrte_event_dlb2 -Wl,-lrte_event_dpaa -Wl,-lrte_event_dpaa2 -Wl,-lrte_event_octeontx2 -Wl,-lrte_event_opdl -Wl,-lrte_event_skeleton -Wl,-lrte_event_sw -Wl,-lrte_event_dsw -Wl,-lrte_event_octeontx -Wl,-lrte_node -Wl,-lrte_graph -Wl,-lrte_bpf -Wl,-lrte_flow_classify -Wl,-lrte_fib -Wl,-lrte_ipsec -Wl,-lrte_vhost -Wl,-lrte_stack -Wl,-lrte_rib -Wl,-lrte_regexdev -Wl,-lrte_rawdev -Wl,-lrte_pdump -Wl,-lrte_member -Wl,-lrte_latencystats -Wl,-lrte_gso -Wl,-lrte_gro -Wl,-lrte_efd -Wl,-lrte_compressdev -Wl,-lrte_bitratestats -Wl,-lrte_bbdev -Wl,-lrte_metrics -Wl,--end-group -Wl,--no-whole-archive -Wl,-Bdynamic -lhugetlbfs -lnuma -ldl -Wl,-lm -Wl,-lrt -Wl,-ldl -lz -lelf

解决方法
http://doc.dpdk.org/guides/nics/mlx5.html#how-to-configure-a-vf-as-trusted

1、 Create 2 VFs on the PF pf0 when in Legacy SR-IOV mode:
/sys/class/net/eth0/device/sriov_numvfs
用sriov_numvfs貌似不可以工作
移除VF

$ echo 0 > /sys/class/net/enp181s0f1/device/mlx5_num_vfs
$ echo 2 > /sys/class/net/enp181s0f1/device/mlx5_num_vfs

2、Verify the VFs are created:

$lspci |grep Mellanox
82:00.0 Ethernet controller: Mellanox Technologies MT27800 Family [ConnectX-5]
82:00.1 Ethernet controller: Mellanox Technologies MT27800 Family [ConnectX-5]
82:00.2 Ethernet controller: Mellanox Technologies MT27800 Family [ConnectX-5 Virtual Function]
82:00.3 Ethernet controller: Mellanox Technologies MT27800 Family [ConnectX-5 Virtual Function]

3、Unbind all VFs. For each VF PCIe, using the following command to unbind the driver:

$ echo "0000:b5:01.2" >> /sys/bus/pci/drivers/mlx5_core/unbind

4、Set the VFs to be trusted for the kernel by using one of the methods below:
• Using sysfs file:

$ echo ON | tee /sys/class/net/enp181s0f1/device/sriov/0/trust
$ echo ON | tee /sys/class/net/enp181s0f1/device/sriov/1/trust

5、Using “ip link” command:

$ ip link set enp181s0f1 vf 0 trust on
$ ip link set enp181s0f1 vf 1 trust on

6、Configure all VFs using mlxreg:
这一步我们可以忽略

$ mlxreg -d /dev/mst/mt4121_pciconf0 --reg_name VHCA_TRUST_LEVEL --yes --set "all_vhca=0x1,trust_level=0x1"

Note
Firmware version used must be >= xx.29.1016 and MFT >= 4.18
7. For each VF PCIe, using the following command to bind the driver:

$ echo "0000:b5:01.2" >> /sys/bus/pci/drivers/mlx5_core/bind

到这一步宿主机上是可以成功获取并初始化VF了，但是如果是docker里面运行，需要注意使用net --host参数，使得宿主机和容器用一个网络。同时，最为重要的是，如果是在宿主机中生成VF，则必须先生成VF，后启动docker，否则在docker中则不能访问的VF。如果已存在docker并启动了，那我们在echo 2生成VF时，就需要stop docker，然后echo 2生成VF，然后docker start。当然也有其他容器和主机网卡设置的技术，例如pipework等。
成功的结果如下

EAL: Detected 4 lcore(s)
EAL: Detected 1 NUMA nodes
EAL: Detected shared linkage of DPDK
EAL: Multi-process socket /var/run/dpdk/rte/mp_socket
EAL: Selected IOVA mode 'PA'
EAL: No available 1048576 kB hugepages reported
EAL: VFIO support initialized
EAL:   Invalid NUMA socket, default to 0
EAL: Probe PCI driver: net_virtio (1af4:1041) device: 0000:01:00.0 (socket 0)
eth_virtio_pci_init(): Failed to init PCI device

EAL: Requested device 0000:01:00.0 cannot be used
EAL:   Invalid NUMA socket, default to 0
EAL: Probe PCI driver: mlx5_pci (15b3:101a) device: 0000:06:00.0 (socket 0)
EAL:   Invalid NUMA socket, default to 0
EAL: Probe PCI driver: mlx5_pci (15b3:101a) device: 0000:07:00.0 (socket 0)
TELEMETRY: No legacy callbacks, legacy socket not created



*****************************************
number of available port: 2


initializing port 0...
port[0] support RX cheksum offload.
port[0] support TX mbuf fast free offload.
port[0] support TX IPv4 checksum offload.
port[0] support TX UDP checksum offload.
port[0] support TX TCP checksum offload.
Port[0] MAC: 7a:99:ed:5f:e3:a6


initializing port 1...
port[1] support RX cheksum offload.
port[1] support TX mbuf fast free offload.
port[1] support TX IPv4 checksum offload.
port[1] support TX UDP checksum offload.
port[1] support TX TCP checksum offload.
Port[1] MAC: 5a:d8:51:db:17:2d
PPS: 2081708
PPS: 23912180

2、
EAL: Failed to open group 58
EAL: 0001:01:08.0 not managed by VFIO driver, skipping
EAL: Failed to open group 60
EAL: 0001:01:0e.0 not managed by VFIO driver, skipping

如果是这个错误，一般发生在container中。
1、重新生成vf，先检查是否是在container exit的状态下生成的VF。需要保证生成vf时候，容器是退出的，这点很关键。
2、如果第一种方式不行。在启动docker时加–device参数
ETH1_VFIO_0 = /sys/class/net/eth1/device/virtfn0/iommu_group
ETH2_VFIO_0 = /sys/class/net/eth2/device/virtfn0/iommu_group

docker run -dit --device /dev/vfio/$ETH1_VFIO_0 dockerimage bash

3、容器中使用vf
a、在启动容器时通过 --net host参数和主机共用网卡
b、使用pipework工具将host上的eth，pipework到容器。

./pipework --direct-phys eth1 -i eth1 container_name 0/0