最近在centos7 x86做实时性性能测试，发现节点性能变化很大，尤其在进行cpu打压之后，性能就会变得相当不稳定，怀疑是有其他进程在争抢实时性CPU的资源，从而导致该节点的性能变化过大。
1：查看隔离的cpu，如下：
cat /etc/tuned/realtime-virtual-host-variables.conf
isolated_cores=2,3
2： 查看目前系统线程占用的cpu资源
tuna -CP
[root@kolla ~]# tuna -CP
thread ctxt_switches
pid SCHED_ rtpri affinity voluntary nonvoluntary cmd
1 OTHER 0 0xff 167294 2845 systemd
2 OTHER 0 0xff 2368 0 kthreadd
4 OTHER 0 0 10 0 kworker/0:0H
5 OTHER 0 0xff 13 0 kworker/u16:0
6 OTHER 0 0 16805 2 ksoftirqd/0
7 FIFO 99 0 18904 0 migration/0
8 OTHER 0 0xff 5 0 rcu_bh
9 OTHER 0 0xff 8277527 14 rcu_sched
10 OTHER 0 0xff 2 0 lru-add-drain
51 OTHER 0 0xff 144 0 kdevtmpfs
52 OTHER 0 0xff 2 0 netns
53 OTHER 0 0xff 6432 0 khungtaskd
54 OTHER 0 0xff 2 0 writeback
55 OTHER 0 0xff 2 0 kintegrityd
56 OTHER 0 0xff 2 0 bioset
57 OTHER 0 0xff 2 0 bioset
58 OTHER 0 0xff 2 0 bioset
59 OTHER 0 0xff 2 0 kblockd
60 OTHER 0 0xff 2 0 md
61 OTHER 0 0xff 2 0 edac-poller
62 OTHER 0 0xff 2 0 watchdogd
68 OTHER 0 0,1,2,3 3 1 kswapd0
69 OTHER 0 4,5,6,7 3 0 kswapd1
70 OTHER 0 0xff 2 0 ksmd
71 OTHER 0 0xff 77173 4 khugepaged
72 OTHER 0 0xff 2 0 crypto
80 OTHER 0 0xff 2 0 kthrotld
81 OTHER 0 0,1,2,3 27639 27 kworker/u17:1
82 OTHER 0 0xff 2 0 kmpath_rdacd
83 OTHER 0 0xff 2 0 kaluad
84 OTHER 0 0xff 2 0 kpsmoused
86 OTHER 0 0xff 2 0 ipv6_addrconf
100 OTHER 0 0xff 2 0 deferwq
101 OTHER 0 0xff 504 89 kworker/u16:1
161 OTHER 0 4,5,6,7 34208 22 kworker/u18:1
185 OTHER 0 0xff 837002 13 kauditd
260 OTHER 0 0xff 2 0 iscsi_eh
267 OTHER 0 0xff 2 0 rpciod
268 OTHER 0 0xff 2 0 xprtiod
368 OTHER 0 0xff 2 0 ata_sff
615 OTHER 0 0xff 9 3 scsi_eh_0
616 OTHER 0 0xff 2 0 scsi_tmf_0
617 OTHER 0 0xff 4 0 scsi_eh_1
618 OTHER 0 0xff 2 0 scsi_tmf_1
619 OTHER 0 0xff 2 0 virtscsi-scan
620 OTHER 0 0xff 2 0 scsi_eh_2
621 OTHER 0 0xff 2 0 scsi_tmf_2
622 OTHER 0 4,5,6,7 2119 0 kworker/u18:2
632 OTHER 0 0,1,2,3 23 0 kworker/u17:2
646 OTHER 0 0xff 2 0 bioset
647 OTHER 0 0xff 2 0 xfsalloc
648 OTHER 0 0xff 2 0 xfs_mru_cache
649 OTHER 0 0xff 2 0 xfs-buf/sda1
650 OTHER 0 0xff 2 0 xfs-data/sda1
651 OTHER 0 0xff 2 0 xfs-conv/sda1
652 OTHER 0 5 3 2 xfs-cil/sda1
653 OTHER 0 0xff 2 0 xfs-reclaim/sda
654 OTHER 0 0xff 2 0 xfs-log/sda1
655 OTHER 0 0xff 2 0 xfs-eofblocks/s
656 OTHER 0 0xff 4717155 120 xfsaild/sda1
657 OTHER 0 0 5535 0 kworker/0:1H
658 OTHER 0 6 128999 0 kworker/6:1H
764 OTHER 0 0xff 71903 414 systemd-journal
786 OTHER 0 0xff 3 2 lvmetad
806 OTHER 0 0xff 1228 435 systemd-udevd
820 OTHER 0 5 25602 0 kworker/5:1H
1067 OTHER 0 0xff 2 0 kvm-irqfd-clean
1154 OTHER 0 0xff 2 0 nfit
1171 OTHER 0 7 5941 0 kworker/7:1H
1176 OTHER 0 0xff 805640 35116 auditd
1177 OTHER 0 0xff 98588 3 auditd
1206 OTHER 0 0xff 72632 65 dbus-daemon
1207 OTHER 0 0xff 25711 0 rpcbind
1208 OTHER 0 2 3257 0 kworker/2:1H
1210 OTHER 0 0xff 44646 28 systemd-logind
1213 OTHER 0 0xff 77172 4 irqbalance
1215 OTHER 0 0xff 17 3 qemu-ga
1217 OTHER 0 0xff 15846 24 polkitd
1237 OTHER 0 0xff 3 0 gmain
1238 OTHER 0 0xff 15513 13 gdbus
1239 OTHER 0 0xff 1 0 polkitd
1240 OTHER 0 0xff 1 0 JS GC Helper
1241 OTHER 0 0xff 8 0 JS Sour~ Thread
1246 OTHER 0 0xff 3 0 polkitd
1221 OTHER 0 0xff 8 0 gssproxy
1223 OTHER 0 0xff 1 0 gssproxy
1224 OTHER 0 0xff 1 0 gssproxy
1225 OTHER 0 0xff 1 0 gssproxy
1226 OTHER 0 0xff 1 0 gssproxy
1227 OTHER 0 0xff 1 0 gssproxy
1310 OTHER 0 0xff 308442 26 ovsdb-server
1383 OTHER 0 0xff 789 77 ovs-vswitchd
1590 OTHER 0 0xff 93 0 dhclient
1722 OTHER 0 0xff 95 0 dhclient
1846 OTHER 0 0xff 137 1 dhclient
1898 OTHER 0 6 885 0 kworker/6:1
1914 OTHER 0 0xff 478 19 tuned
2339 OTHER 0 0xff 22 0 tuned
2321 OTHER 0 0xff 2 0 gmain
2331 OTHER 0 0xff 1 0 tuned
2322 OTHER 0 0xff 770803 23 tuned
1917 OTHER 0 0xff 466 4 containerd
2465 OTHER 0 0xff 135420 168 containerd
2442 OTHER 0 0xff 176570 236 containerd
2443 OTHER 0 0xff 102454 145 containerd
2444 OTHER 0 0xff 171532 231 containerd
2253 OTHER 0 0xff 874246 170 containerd
2254 OTHER 0 0xff 43 1 containerd
2255 OTHER 0 0xff 60538 71 containerd
2256 OTHER 0 0xff 1 1 containerd
6327 OTHER 0 0xff 81212 130 containerd
2456 OTHER 0 0xff 13 0 containerd
2457 OTHER 0 0xff 168999 223 containerd
2458 OTHER 0 0xff 128209 183 containerd
2459 OTHER 0 0xff 175338 253 containerd
2460 OTHER 0 0xff 104466 135 containerd
1957 OTHER 0 0xff 1316 2 rsyslogd
2088 OTHER 0 0xff 84759 21 rs:main Q:Reg
2083 OTHER 0 0xff 928888 121 in:imjournal
1996 OTHER 0 0xff 95 2 sshd
2055 OTHER 0 0xff 4 2 agetty
2061 OTHER 0 0xff 5 1 agetty
2087 OTHER 0 0xff 14377 4 crond
2441 OTHER 0 3 100396 0 kworker/3:1H
2466 OTHER 0 0xff 174 1 dockerd
8224 OTHER 0 0xff 33945 415 dockerd
2468 OTHER 0 0xff 334142 10 dockerd
2469 OTHER 0 0xff 85739 544 dockerd
2470 OTHER 0 0xff 79911 389 dockerd
2471 OTHER 0 0xff 67878 577 dockerd
2472 OTHER 0 0xff 1 0 dockerd
2473 OTHER 0 0xff 53 0 dockerd
2474 OTHER 0 0xff 76647 543 dockerd
2475 OTHER 0 0xff 40454 74 dockerd
2476 OTHER 0 0xff 74451 543 dockerd
2477 OTHER 0 0xff 41107 512 dockerd
2478 OTHER 0 0xff 65735 561 dockerd
2522 OTHER 0 0xff 61754 526 dockerd
2523 OTHER 0 0xff 33892 94 dockerd
8222 OTHER 0 0xff 2 0 dockerd
8223 OTHER 0 0xff 33391 273 dockerd
2479 OTHER 0 1 4126 0 kworker/1:1H
2488 OTHER 0 0xff 89 4 mysqld
2490 OTHER 0 0xff 4 0 mysqld
2491 OTHER 0 0xff 25725 1 mysqld
2492 OTHER 0 0xff 1542898 5 mysqld
2493 OTHER 0 0xff 1542908 6 mysqld
2494 OTHER 0 0xff 1542901 7 mysqld
2495 OTHER 0 0xff 1542902 9 mysqld
2496 OTHER 0 0xff 1542899 7 mysqld
2497 OTHER 0 0xff 1542902 7 mysqld
2498 OTHER 0 0xff 1542961 6 mysqld
2499 OTHER 0 0xff 1542965 5 mysqld
2500 OTHER 0 0xff 1542902 8 mysqld
2501 OTHER 0 0xff 1542904 14 mysqld
2502 OTHER 0 0xff 1048155 10 mysqld
2504 OTHER 0 0xff 771545 9 mysqld
2505 OTHER 0 0xff 771528 11 mysqld
2506 OTHER 0 0xff 154325 2 mysqld
2507 OTHER 0 0xff 77166 1 mysqld
2760 OTHER 0 4 10463 0 kworker/4:1H
2813 OTHER 0 0xff 777550 10 tmux
2814 OTHER 0 0xff 908 1 bash
2869 OTHER 0 0xff 848 2 bash
3268 OTHER 0 5 198 0 kworker/5:2
3540 OTHER 0 5 7 0 kworker/5:1
3584 OTHER 0 6 404 0 kworker/6:2
3696 OTHER 0 3 21059 0 kworker/3:1
3860 OTHER 0 5 6 0 kworker/5:0
4289 OTHER 0 0xff 50 1 sshd
4291 OTHER 0 0xff 31 1 bash
4328 OTHER 0 0xff 0 21 tuna
8175 OTHER 0 2 113435 5 kworker/2:0
10386 OTHER 0 7 33837 0 kworker/7:0
13035 OTHER 0 2 6216 2 kworker/2:1
14401 OTHER 0 4 3 0 kworker/4:0
15235 OTHER 0 0 538293 55 kworker/0:1
17369 OTHER 0 0xff 280 283 beam.smp
17669 OTHER 0 0xff 2 0 sys_sig_dispatc
17670 OTHER 0 0xff 2 0 sys_msg_dispatc
17671 OTHER 0 0xff 1 0 async_1
17672 OTHER 0 0xff 3 0 async_2
17673 OTHER 0 0xff 1 0 async_3
17674 OTHER 0 0xff 2 0 async_4
17675 OTHER 0 0xff 2 0 async_5
17676 OTHER 0 0xff 1 0 async_6
17677 OTHER 0 0xff 4 0 async_7
17678 OTHER 0 0xff 5 0 async_8
17679 OTHER 0 0xff 2 0 async_9
17680 OTHER 0 0xff 1 0 async_10
17681 OTHER 0 0xff 1 0 async_11
17682 OTHER 0 0xff 1 0 async_12
17808 OTHER 0 0xff 1 0 1_dirty_cpu_sch
17809 OTHER 0 0xff 1 0 2_dirty_cpu_sch
17810 OTHER 0 0xff 1 0 3_dirty_cpu_sch
17811 OTHER 0 0xff 1 85 4_dirty_cpu_sch
17812 OTHER 0 0xff 1 0 5_dirty_cpu_sch
17813 OTHER 0 0xff 1 0 6_dirty_cpu_sch
17814 OTHER 0 0xff 1 96 7_dirty_cpu_sch
17815 OTHER 0 0xff 5 0 8_dirty_cpu_sch
17816 OTHER 0 0xff 2 181 1_dirty_io_sche
17817 OTHER 0 0xff 187514 11023441 2_dirty_io_sche
17818 OTHER 0 0xff 4 171 3_dirty_io_sche
17819 OTHER 0 0xff 50818 2739045 4_dirty_io_sche
17820 OTHER 0 0xff 321090 17321945 5_dirty_io_sche
17821 OTHER 0 0xff 115097 6779083 6_dirty_io_sche
17822 OTHER 0 0xff 4 18 7_dirty_io_sche
17823 OTHER 0 0xff 103312 5919915 8_dirty_io_sche
17824 OTHER 0 0xff 3 113 9_dirty_io_sche
17825 OTHER 0 0xff 7 4 10_dirty_io_sch
17826 OTHER 0 0xff 1480243 216 aux
17827 OTHER 0 0xff 213272 1 0_poller
以上可知，像docker、mysql等进程也占用了隔离的cpu,所以需要进行手动设置
3：容器绑核
docker update --cpuset-cpus 0,1 容器id
查看容器的绑核，docker inspect 容器ID如下：
4：线程绑定cpu
taskset -cp cpu-list pid
# cpu-list可以是0,1这样的一个核，也可以是1-2这样的，表示绑定在1和2上面
# pid 表示进程
使用如下命令将其他的进程统一绑定到0,1cpu上
[root@kolla ~]# tuna -CP | grep 0xff | awk '{print $1}' | xargs -i taskset -cp 0,1 {}
pid 1's current affinity list: 0-7
pid 1's new affinity list: 0,1
pid 2's current affinity list: 0-7
pid 2's new affinity list: 0,1
pid 5's current affinity list: 0-7
taskset: failed to set pid 5's affinity: Invalid argument
pid 8's current affinity list: 0-7
pid 8's new affinity list: 0,1
pid 9's current affinity list: 0-7
pid 9's new affinity list: 0,1
pid 10's current affinity list: 0-7
taskset: failed to set pid 10's affinity: Invalid argument
pid 51's current affinity list: 0-7
pid 51's new affinity list: 0,1
pid 52's current affinity list: 0-7
taskset: failed to set pid 52's affinity: Invalid argument
pid 53's current affinity list: 0-7
pid 53's new affinity list: 0,1
pid 54's current affinity list: 0-7
taskset: failed to set pid 54's affinity: Invalid argument
pid 55's current affinity list: 0-7
taskset: failed to set pid 55's affinity: Invalid argument
pid 56's current affinity list: 0-7
taskset: failed to set pid 56's affinity: Invalid argument
pid 57's current affinity list: 0-7
taskset: failed to set pid 57's affinity: Invalid argument
pid 58's current affinity list: 0-7
taskset: failed to set pid 58's affinity: Invalid argument
pid 59's current affinity list: 0-7
taskset: failed to set pid 59's affinity: Invalid argument
或者也可以使用如下命令直接更新线程绑核（一般不这样用）
tuna -U -t '*' -c 0 -m
5：查看绑定结果
tuna -CP
[root@kolla ~]# tuna -CP| grep docker
2466 OTHER 0 0,1 174 1 dockerd
8224 OTHER 0 0,1 33987 416 dockerd
2468 OTHER 0 0,1 334220 10 dockerd
2469 OTHER 0 0,1 85749 545 dockerd
2470 OTHER 0 0,1 79911 389 dockerd
2471 OTHER 0 0,1 67878 577 dockerd
2472 OTHER 0 0,1 1 0 dockerd
2473 OTHER 0 0,1 53 0 dockerd
2474 OTHER 0 0,1 76647 543 dockerd
2475 OTHER 0 0,1 40454 74 dockerd
2476 OTHER 0 0,1 74463 544 dockerd
2477 OTHER 0 0,1 41108 512 dockerd
2478 OTHER 0 0,1 65737 561 dockerd
2522 OTHER 0 0,1 61759 526 dockerd
2523 OTHER 0 0,1 33892 94 dockerd
8222 OTHER 0 0,1 2 0 dockerd
8223 OTHER 0 0,1 33434 273 dockerd
3629
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
