linux 进程内存、CPU 利用率高排查方法

1.查看内存、CPU高的进程ID
top

PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
9120 root 20 0 17.3g 13g 9780 S 0 10.9 43927:56 kvm
12466 root 20 0 13.0g 9.3g 13m S 6 7.4 1293:03 kvm
23034 root 20 0 12.9g 7.6g 8936 S 4 6.1 10058:02 kvm
11733 root 20 0 9560m 6.0g 13m S 6 4.8 1902:07 kvm
23303 root 20 0 15.5g 4.6g 7604 S 9 3.7 6526:43 kvm
23201 root 20 0 13.0g 4.3g 8872 S 0 3.5 4968:50 kvm

2.查看进程下的线程
top -H -p 9120

PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
9123 root 20 0 17.3g 13g 9780 S 1 10.9 3467:10 kvm
4462 root 20 0 17.3g 13g 9780 S 1 10.9 5775:55 kvm
4461 root 20 0 17.3g 13g 9780 S 0 10.9 5741:15 kvm
4464 root 20 0 17.3g 13g 9780 S 0 10.9 5884:20 kvm
4467 root 20 0 17.3g 13g 9780 S 0 10.9 5811:54 kvm
9120 root 20 0 17.3g 13g 9780 S 0 10.9 39:12.99 kvm
9125 root 20 0 17.3g 13g 9780 S 0 10.9 0:00.20 kvm
4460 root 20 0 17.3g 13g 9780 S 0 10.9 5598:04 kvm
4466 root 20 0 17.3g 13g 9780 S 0 10.9 5802:40 kvm
4470 root 20 0 17.3g 13g 9780 S 0 10.9 5781:50 kvm
26601 root 20 0 17.3g 13g 9780 S 0 10.9 0:00.00 kvm

3.查看线程打开的文件系统
lsop -p 9120
COMMAND PID USER FD TYPE DEVICE SIZE/OFF NODE NAME
kvm 9120 root cwd DIR 8,2 4096 2 /
kvm 9120 root rtd DIR 8,2 4096 2 /
kvm 9120 root txt REG 8,2 18688689 4587571 /usr/bin/qemu-system-x86_64-hp
kvm 9120 root mem REG 8,2 1518928 2883812 /usr/lib/x86_64-linux-gnu/libdb-5.1.so
kvm 9120 root mem REG 8,2 22432 2892389 /usr/lib/x86_64-linux-gnu/sasl2/libsasldb.so.2.0.25
kvm 9120 root mem REG 8,2 18512 2892794 /usr/lib/x86_64-linux-gnu/sasl2/li

4.查看进程，RSS为内存使用量
pmap -x 9120
Address Kbytes RSS Dirty Mode Mapping
000055d72b71c000 0 4852 0 r-x-- qemu-system-x86_64-hp
000055d72bf20000 0 384 384 r---- qemu-system-x86_64-hp
000055d72bfea000 0 412 408 rw— qemu-system-x86_64-hp
000055d72c05e000 0 56 56 rw— [ anon ]
000055d72dcb8000 0 139880 139880 rw— [ anon ]

5.查看线程信息cat /proc/9120/task/9125/status
Name: kvm
State: S (sleeping)
Tgid: 9120
Ngid: 9125
Pid: 9125
PPid: 1
TracerPid: 0
Uid: 0 0 0 0
Gid: 0 0 0 0
FDSize: 128
Groups:
NStgid: 9120
NSpid: 9125
NSpgid: 9119
NSsid: 9119
VmPeak: 18264660 kB
VmSize: 18165312 kB
VmLck: 0 kB
VmPin: 0 kB
VmHWM: 16733364 kB
VmRSS: 14294168 kB
VmData: 17985008 kB
VmStk: 136 kB
VmExe: 6164 kB
VmLib: 20160 kB
VmPTE: 33356 kB
VmPMD: 84 kB
VmSwap: 2543680 kB
Threads: 10
SigQ: 3/513342
SigPnd: 0000000000000000
ShdPnd: 0000000000000000
SigBlk: fffffffe7ffbfeff
SigIgn: 0000000000001000
SigCgt: 0000000180004243
CapInh: 0000000000000000
CapPrm: 0000003fffffffff
CapEff: 0000003fffffffff
CapBnd: 0000003fffffffff
Seccomp: 0
Cpus_allowed: ffffffff
Cpus_allowed_list: 0-31
Mems_allowed: 00000000,00000003
Mems_allowed_list: 0-1
voluntary_ctxt_switches: 482
nonvoluntary_ctxt_switches: 11

6.stace -p 9120 可以发现使用ppoll在循环调用
ppoll([{fd=89, events=POLLIN|POLLERR|POLLHUP}, {fd=88, events=POLLIN|POLLERR|POLLHUP}, {fd=87, events=PO=84, events=POLLIN|POLLERR|POLLHUP}, {fd=83, events=POLLIN|POLLERR|POLLHUP}, {fd=82, events=POLLIN|POLLEs=POLLIN|POLLERR|POLLHUP}, {fd=78, events=POLLIN|POLLERR|POLLHUP}, {fd=77, events=POLLIN|POLLERR|POLLHUPOLLERR|POLLHUP}, {fd=73, events=POLLIN|POLLERR|POLLHUP}, {fd=72, events=POLLIN|POLLERR|POLLHUP}, {fd=71,LHUP}, {fd=68, events=POLLIN|POLLERR|POLLHUP}, {fd=67, events=POLLIN|POLLERR|POLLHUP}, {fd=66, events=PO=63, events=POLLIN|POLLERR|POLLHUP}, {fd=62, events=POLLIN|POLLERR|POLLHUP}, {fd=61, events=POLLIN|POLLEs=POLLIN|POLLERR|POLLHUP}, …], 75, {0, 0}, NULL, 8) = 1 ([…], left {0, 0})
read(5, “\4\0\0\0\0\0\0\0”, 16) = 8
write(5, “\1\0\0\0\0\0\0\0”, 8) = 8
write(5, “\1\0\0\0\0\0\0\0”, 8) = 8
ioctl(10, 0x4020aea5, 0x7fff7f64f1a0) = 1
write(5, “\1\0\0\0\0\0\0\0”, 8) = 8
write(5, “\1\0\0\0\0\0\0\0”, 8) = 8