dpdk-lcore-mask 对 handler线程cpu亲和性的影响

秋千无闻

已于 2022-08-24 14:56:39 修改

阅读量767

点赞数

分类专栏： ovs-dpdk 文章标签： ovs ovs-dpdk

于 2022-08-24 10:42:49 首次发布

本文链接：https://blog.csdn.net/qq_20679687/article/details/126499558

版权

ovs-dpdk 专栏收录该内容

1 篇文章 0 订阅

订阅专栏

ovs-dpdk中有一个配置参数dpdk-lcore-mask ，但使用这个配置参数的时候，主线程的cpu亲和性限制为master core，导致后续生成的进程都只在 master core上运行。

当不配置该参数时，则为自动配置模式，该模式下，最后会恢复主线程的cpu亲和性。

推荐配置：不配置 dpdk-lcore-mask 。这样的话，主线程的子线程可在更多的CPU核中调度。

代码分析

ovs-dpdk中的dpdk-lcore-mask的参数解析代码

dpdk_init__()
    construct_dpdk_args(ovs_other_config, &args);
        construct_dpdk_options(ovs_other_config, args);
            {"dpdk-lcore-mask",   "-c",             false, NULL},

    // if dpdk-lcore-mask is set
    if (args_contains(&args, "-c") || args_contains(&args, "-l")) {
        auto_determine = false;
    }
    /**
     * NOTE: This is an unsophisticated mechanism for determining the DPDK
     * lcore for the DPDK Master.
     */
    if (auto_determine) {
        const struct ovs_numa_info_core *core;
        int cpu = 0;

        /* Get the main thread affinity */
        affinity = ovs_numa_thread_getaffinity_dump();
        if (affinity) {
            cpu = INT_MAX;
            FOR_EACH_CORE_ON_DUMP (core, affinity) {
                if (cpu > core->core_id) {
                    cpu = core->core_id;
                }
            }
        } else {
            /* User did not set dpdk-lcore-mask and unable to get current
             * thread affintity - default to core #0 */
            VLOG_ERR("Thread getaffinity failed. Using core #0");
        }
        svec_add(&args, "-l");
        svec_add_nocopy(&args, xasprintf("%d", cpu));
    }
    // init rte env. dpdk库，会限制主线程的CPU亲和性为master core.
    result = rte_eal_init(args.n, argv);

    /* Set the main thread affinity back to pre rte_eal_init() value */
    // 自动配置模式下（即不配置dpdk-lcore-mask），会恢复CPU亲和性。
    if (affinity) {
        ovs_numa_thread_setaffinity_dump(affinity);
        ovs_numa_dump_destroy(affinity);
    }

dpdk库中的亲和性初始化

对 current thread 和 slave work thread的affinity 影响

rte_eal_init()
    // 设置当前线程的affinity为master core
    pthread_setaffinity_np(pthread_self(), sizeof(rte_cpuset_t),
            &lcore_config[config->main_lcore].cpuset) != 0 {
	// 设置work线程的affinity为lcore
	RTE_LCORE_FOREACH_WORKER(i) {
		eal_thread_create(&lcore_config[i].thread_id, i)
		pthread_setaffinity_np(lcore_config[i].thread_id,
			    sizeof(rte_cpuset_t), &lcore_config[i].cpuset);
	}

ovs-dpdk 系统调试

配置1 - 设置dpdk-lcore-mask

ovs-vsctl set Open_vSwitch . other_config:dpdk-lcore-mask="0x00000f0";
ovs-vsctl set Open_vSwitch . other_config:n-handler-threads=8
ovs-vsctl set Open_vSwitch . other_config:n-revalidator-threads=8

log1 - handler固定在core 4

[root@ct8test88 z]# pidstat -t -p $(cat /usr/local/var/run/openvswitch/ovs-vswitchd.pid)
Linux 4.18.0-348.7.1.el8_5.x86_64 (ct8test88)   08/23/2022      _x86_64_        (128 CPU)

07:26:43 PM   UID      TGID       TID    %usr %system  %guest   %wait    %CPU   CPU  Command
07:26:43 PM     0    840428         -    0.01    0.00    0.00    0.00    0.01     4  ovs-vswitchd
07:26:43 PM     0         -    840428    0.00    0.00    0.00    0.00    0.00     4  |__ovs-vswitchd
07:26:43 PM     0         -    840429    0.00    0.00    0.00    0.00    0.00   117  |__eal-intr-thread
07:26:43 PM     0         -    840430    0.00    0.00    0.00    0.00    0.00   119  |__rte_mp_handle
07:26:43 PM     0         -    840431    0.00    0.00    0.00    0.00    0.00     5  |__lcore-worker-5
07:26:43 PM     0         -    840432    0.00    0.00    0.00    0.00    0.00     6  |__lcore-worker-6
07:26:43 PM     0         -    840433    0.00    0.00    0.00    0.00    0.00     7  |__lcore-worker-7
07:26:43 PM     0         -    840438    0.00    0.00    0.00    0.00    0.00     8  |__ovs-vswitchd
07:26:43 PM     0         -    840439    0.00    0.00    0.00    0.00    0.00     4  |__dpdk_watchdog1
07:26:43 PM     0         -    840441    0.00    0.00    0.00    0.00    0.00     4  |__urcu2
07:26:43 PM     0         -    840449    0.00    0.00    0.00    0.00    0.00     4  |__ct_clean3
07:26:43 PM     0         -    840450    0.00    0.00    0.00    0.00    0.00     4  |__ipf_clean14
07:26:43 PM     0         -    840451    0.00    0.00    0.00    0.00    0.00     4  |__handler20
07:26:43 PM     0         -    840452    0.00    0.00    0.00    0.00    0.00     4  |__handler19
07:26:43 PM     0         -    840453    0.00    0.00    0.00    0.00    0.00     4  |__handler18
07:26:43 PM     0         -    840454    0.00    0.00    0.00    0.00    0.00     4  |__handler17
07:26:43 PM     0         -    840455    0.00    0.00    0.00    0.00    0.00     4  |__handler16
07:26:43 PM     0         -    840456    0.00    0.00    0.00    0.00    0.00     4  |__handler15
07:26:43 PM     0         -    840457    0.00    0.00    0.00    0.00    0.00     4  |__handler4
07:26:43 PM     0         -    840458    0.00    0.00    0.00    0.00    0.00     4  |__handler13
07:26:43 PM     0         -    840459    0.00    0.00    0.00    0.00    0.00     4  |__revalidator5
07:26:43 PM     0         -    840460    0.00    0.00    0.00    0.00    0.00     4  |__revalidator6
07:26:43 PM     0         -    840461    0.00    0.00    0.00    0.00    0.00     4  |__revalidator11
07:26:43 PM     0         -    840462    0.00    0.00    0.00    0.00    0.00     4  |__revalidator7
07:26:43 PM     0         -    840463    0.00    0.00    0.00    0.00    0.00     4  |__revalidator8
07:26:43 PM     0         -    840464    0.00    0.00    0.00    0.00    0.00     4  |__revalidator12
07:26:43 PM     0         -    840465    0.00    0.00    0.00    0.00    0.00     4  |__revalidator9
07:26:43 PM     0         -    840466    0.00    0.00    0.00    0.00    0.00     4  |__revalidator10
07:26:43 PM     0         -    840472    0.00    0.00    0.00    0.00    0.00     6  |__pmd-c06/id:24
07:26:43 PM     0         -    840473    0.00    0.00    0.00    0.00    0.00     7  |__pmd-c07/id:23
07:26:43 PM     0         -    840474    0.00    0.00    0.00    0.00    0.00     5  |__pmd-c05/id:22
0
[root@ct8test88 z]# taskset -cp 991962
pid 991962's current affinity list: 4

配置2 - 不设置dpdk-lcore-mask

ovs-vsctl --no-wait set Open_vSwitch . other_config:n-handler-threads=8
ovs-vsctl --no-wait set Open_vSwitch . other_config:n-revalidator-threads=8

log2 - hander 分散在多核中

[root@ct8test88 z]# pidstat -t -p $(cat /usr/local/var/run/openvswitch/ovs-vswitchd.pid)
Linux 4.18.0-348.7.1.el8_5.x86_64 (ct8test88)   08/23/2022      _x86_64_        (128 CPU)

07:34:03 PM   UID      TGID       TID    %usr %system  %guest   %wait    %CPU   CPU  Command
07:34:03 PM     0    841970         -    0.01    0.00    0.00    0.00    0.01    25  ovs-vswitchd
07:34:03 PM     0         -    841970    0.00    0.00    0.00    0.00    0.00    25  |__ovs-vswitchd
07:34:03 PM     0         -    841971    0.00    0.00    0.00    0.00    0.00   121  |__eal-intr-thread
07:34:03 PM     0         -    841972    0.00    0.00    0.00    0.00    0.00   119  |__rte_mp_handle
07:34:03 PM     0         -    841975    0.00    0.00    0.00    0.00    0.00    11  |__ovs-vswitchd
07:34:03 PM     0         -    841976    0.00    0.00    0.00    0.00    0.00     1  |__dpdk_watchdog1
07:34:03 PM     0         -    841978    0.00    0.00    0.00    0.00    0.00     9  |__urcu2
07:34:03 PM     0         -    841988    0.00    0.00    0.00    0.00    0.00   119  |__ct_clean3
07:34:03 PM     0         -    841989    0.00    0.00    0.00    0.00    0.00   122  |__ipf_clean4
07:34:03 PM     0         -    841990    0.00    0.00    0.00    0.00    0.00    60  |__handler5
07:34:03 PM     0         -    841991    0.00    0.00    0.00    0.00    0.00   125  |__handler6
07:34:03 PM     0         -    841992    0.00    0.00    0.00    0.00    0.00    97  |__handler7
07:34:03 PM     0         -    841993    0.00    0.00    0.00    0.00    0.00    98  |__handler8
07:34:03 PM     0         -    841994    0.00    0.00    0.00    0.00    0.00    99  |__handler9
07:34:03 PM     0         -    841995    0.00    0.00    0.00    0.00    0.00    59  |__handler10
07:34:03 PM     0         -    841996    0.00    0.00    0.00    0.00    0.00   100  |__handler11
07:34:03 PM     0         -    841997    0.00    0.00    0.00    0.00    0.00   103  |__handler12
07:34:03 PM     0         -    841998    0.00    0.00    0.00    0.00    0.00    20  |__revalidator13
07:34:03 PM     0         -    841999    0.00    0.00    0.00    0.00    0.00    17  |__revalidator14
07:34:03 PM     0         -    842000    0.00    0.00    0.00    0.00    0.00    14  |__revalidator15
07:34:03 PM     0         -    842001    0.00    0.00    0.00    0.00    0.00    18  |__revalidator16
07:34:03 PM     0         -    842002    0.00    0.00    0.00    0.00    0.00    21  |__revalidator17
07:34:03 PM     0         -    842003    0.00    0.00    0.00    0.00    0.00    24  |__revalidator18
07:34:03 PM     0         -    842004    0.00    0.00    0.00    0.00    0.00    23  |__revalidator19
07:34:03 PM     0         -    842006    0.00    0.00    0.00    0.00    0.00    22  |__revalidator20
07:34:03 PM     0         -    842015    0.00    0.00    0.00    0.00    0.00     6  |__pmd-c06/id:21
07:34:03 PM     0         -    842016    0.00    0.00    0.00    0.00    0.00     7  |__pmd-c07/id:22
07:34:03 PM     0         -    842017    0.00    0.00    0.00    0.00    0.00     5  |__pmd-c05/id:23
07:34:03 PM     0         -    842018    0.00    0.00    0.00    0.00    0.00     4  |__pmd-c04/id:24

[root@ct8test88 z]# taskset -cp 991962
pid 991962's current affinity list: 1-127