x710 hash 分片与非分片 tcp 报文异常问题

longyu_wlz

已于 2022-11-10 20:57:13 修改

阅读量1.9k

点赞数 1

分类专栏： dpdk 文章标签： x710 hash hash_filter 分片报文hash

于 2021-04-18 23:22:26 首次发布

本文链接：https://blog.csdn.net/Longyu_wlz/article/details/115841974

版权

dpdk 专栏收录该内容

94 篇文章 29 订阅

订阅专栏

问题描述

当 rss_hf 中配置了 ETH_RSS_FRAG_IPV4 与 ETH_RSS_NONFRAG_IPV4_TCP 参数后，一些连接的分片报文会被 hash 到其它队列中，问题的关键是这些分片的报文没有 L4 port number。

当不配置 ETH_RSS_NONFRAG_IPV4_TCP 时，ETH_RSS_FRAG_IPV4 哈希函数不会应用到非分片报文上，这些报文将会被投递到队列 0。

异常 hash 配置

配置内容如下：

       #define RSS_X710_KEY_SIZE 52

         static unsigned char tr_rss_key_x710[] = {
           0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a,
           0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a,
           0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a,
           0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a,
           0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a,
           0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a,
           0x6d, 0x5a, 0x6d, 0x5a,
       };
  
       port_conf->rxmode.mq_mode = ETH_MQ_RX_RSS;

       port_conf->rx_adv_conf.rss_conf.rss_key = tr_rss_key_x710;
       port_conf->rx_adv_conf.rss_conf.rss_hf = ETH_RSS_PROTO_MASK;
       port_conf->rx_adv_conf.rss_conf.rss_key_len = RSS_X710_KEY_SIZE;
}

设置上面的 rss_key，且 rss_hf 配置 ETH_RSS_PROTO_MASK，能 hash 到多队列，不分片报文正常，tcp 分片报文存在问题。

流量配比

tcp 非分片流，源 ip 与目的 ip 100:100 分布。

测试验证过程

下面的测试过程中，首先打 100:100 的 tcp 非分片流，能够 hash 到多队列后修改为有分片的 tcp 流，进一步验证。

1.使用网卡默认的 rss_key，且修改 rss_hf 为 ETH_RSS_IPV4 | ETH_RSS_IPV6

测试结果：不能 hash 到多队列。

2.使用默认的 rss_key 并配置网卡 hash_filter，代码来自 google

补丁代码如下：

struct rte_eth_conf new_port_conf = {
  .rxmode = {
    .mq_mode = ETH_MQ_RX_RSS,
  },
  .rx_adv_conf = {
    .rss_conf = {
        .rss_hf = ETH_RSS_IP |
              ETH_RSS_TCP |
              ETH_RSS_UDP |
              ETH_RSS_SCTP,
    }
   },
};


#define UINT32_BIT (CHAR_BIT * sizeof(uint32_t))
int sym_hash_enable(int port_id, uint32_t ftype, enum rte_eth_hash_function function)
{
    struct rte_eth_hash_filter_info info;
    int ret = 0;
    uint32_t idx = 0;
    uint32_t offset = 0;

    memset(&info, 0, sizeof(info));

    ret = rte_eth_dev_filter_supported(port_id, RTE_ETH_FILTER_HASH);
    if (ret < 0) {
        printf("RTE_ETH_FILTER_HASH not supported on port: %d",
                         port_id);
        return ret;
    }

    info.info_type = RTE_ETH_HASH_FILTER_GLOBAL_CONFIG;
    info.info.global_conf.hash_func = function;

    idx = ftype / UINT32_BIT;
    offset = ftype % UINT32_BIT;
    info.info.global_conf.valid_bit_mask[idx] |= (1ULL << offset);
    info.info.global_conf.sym_hash_enable_mask[idx] |=
                        (1ULL << offset);

    ret = rte_eth_dev_filter_ctrl(port_id, RTE_ETH_FILTER_HASH,
                                  RTE_ETH_FILTER_SET, &info);
    if (ret < 0)
    {
        printf("Cannot set global hash configurations"
                        "on port %u", port_id);
        return ret;
    }

    return 0;
}

int sym_hash_set(int port_id, int enable)
{
    int ret = 0;
    struct rte_eth_hash_filter_info info;

    memset(&info, 0, sizeof(info));

    ret = rte_eth_dev_filter_supported(port_id, RTE_ETH_FILTER_HASH);
    if (ret < 0) {
        printf("RTE_ETH_FILTER_HASH not supported on port: %d",
                         port_id);
        return ret;
    }

    info.info_type = RTE_ETH_HASH_FILTER_SYM_HASH_ENA_PER_PORT;
    info.info.enable = enable;
    ret = rte_eth_dev_filter_ctrl(port_id, RTE_ETH_FILTER_HASH,
                        RTE_ETH_FILTER_SET, &info);

    if (ret < 0)
    {
        printf("Cannot set symmetric hash enable per port "
                        "on port %u", port_id);
        return ret;
    }

    return 0;
}

dev_configure 使用 new_port_conf 配置，并在 dev_configure 前执行如下代码：

                sym_hash_enable(portid, RTE_ETH_FLOW_NONFRAG_IPV4_TCP, RTE_ETH_HASH_FUNCTION_TOEPLITZ);
                sym_hash_enable(portid, RTE_ETH_FLOW_NONFRAG_IPV4_UDP, RTE_ETH_HASH_FUNCTION_TOEPLITZ);
                sym_hash_enable(portid, RTE_ETH_FLOW_FRAG_IPV4, RTE_ETH_HASH_FUNCTION_TOEPLITZ);
                sym_hash_enable(portid, RTE_ETH_FLOW_NONFRAG_IPV4_SCTP, RTE_ETH_HASH_FUNCTION_TOEPLITZ);
                sym_hash_enable(portid, RTE_ETH_FLOW_NONFRAG_IPV4_OTHER, RTE_ETH_HASH_FUNCTION_TOEPLITZ);

                sym_hash_set(portid, 1);

测试结果：能够 hash 开，但是 tcp 分片报文仍旧有问题。

3.测试使用 0x6d、0x5a … 的 rss_key，修改 rss_hf flag 内容为 ETH_RSS_IPV4 | ETH_RSS_IPV6

测试结果：不能 hash 到多队列。

4.测试使用默认的 rss_key，并在执行 dev_configure 前添加 filter_ctrl 的补丁代码