文章目录
有几个日常使用非常频繁的工具不列入，比如top、iotop、dstat。
1. cpu
1.1 利用率
#看%idle列百分比
mpstat -P ALL
Linux 3.10.0-957.21.3.el7.x86_64 (xxxx) 09/21/2020 _x86_64_ (16 CPU)
09:17:29 AM CPU %usr %nice %sys %iowait %irq %soft %steal %guest %gnice %idle
09:17:29 AM all 1.52 0.00 0.24 0.05 0.00 0.04 0.00 0.00 0.00 98.16
09:17:29 AM 0 1.74 0.00 0.42 0.52 0.00 0.25 0.00 0.00 0.00 97.07
09:17:29 AM 1 1.17 0.00 0.14 0.00 0.00 0.01 0.00 0.00 0.00 98.68
09:17:29 AM 2 1.65 0.00 0.40 0.15 0.00 0.26 0.00 0.00 0.00 97.54
09:17:29 AM 3 1.39 0.00 0.16 0.00 0.00 0.01 0.00 0.00 0.00 98.44
09:17:29 AM 4 1.78 0.00 0.30 0.05 0.00 0.00 0.00 0.00 0.00 97.87
09:17:29 AM 5 1.29 0.00 0.17 0.00 0.00 0.00 0.00 0.00 0.00 98.53
09:17:29 AM 6 1.75 0.00 0.28 0.01 0.00 0.00 0.00 0.00 0.00 97.96
09:17:29 AM 7 1.30 0.00 0.16 0.00 0.00 0.00 0.00 0.00 0.00 98.53
09:17:29 AM 8 1.74 0.00 0.28 0.01 0.00 0.00 0.00 0.00 0.00 97.97
09:17:29 AM 9 1.30 0.00 0.16 0.00 0.00 0.00 0.00 0.00 0.00 98.53
09:17:29 AM 10 1.74 0.00 0.28 0.00 0.00 0.00 0.00 0.00 0.00 97.98
09:17:29 AM 11 1.31 0.00 0.16 0.00 0.00 0.00 0.00 0.00 0.00 98.52
09:17:29 AM 12 1.73 0.00 0.28 0.00 0.00 0.00 0.00 0.00 0.00 97.98
09:17:29 AM 13 1.32 0.00 0.16 0.00 0.00 0.00 0.00 0.00 0.00 98.52
09:17:29 AM 14 1.76 0.00 0.28 0.04 0.00 0.00 0.00 0.00 0.00 97.92
09:17:29 AM 15 1.35 0.00 0.17 0.00 0.00 0.00 0.00 0.00 0.00 98.48
#看%idle列百分比
sar -P ALL
Linux 3.10.0-957.21.3.el7.x86_64 (xxxx) 09/21/2020 _x86_64_ (16 CPU)
12:00:01 AM CPU %user %nice %system %iowait %steal %idle
12:10:01 AM all 1.34 0.00 0.38 0.13 0.00 98.16
12:10:01 AM 0 1.48 0.00 1.03 0.36 0.00 97.13
12:10:01 AM 1 0.94 0.00 0.18 0.01 0.00 98.88
12:10:01 AM 2 2.34 0.00 1.06 1.12 0.00 95.48
12:10:01 AM 3 1.32 0.00 0.24 0.00 0.00 98.44
12:10:01 AM 4 2.19 0.00 0.49 0.40 0.00 96.93
12:10:01 AM 5 0.74 0.00 0.27 0.00 0.00 98.98
12:10:01 AM 6 1.38 0.00 0.38 0.05 0.00 98.19
12:10:01 AM 7 0.79 0.00 0.24 0.00 0.00 98.97
12:10:01 AM 8 1.55 0.00 0.32 0.01 0.00 98.12
12:10:01 AM 9 1.00 0.00 0.18 0.00 0.00 98.83
12:10:01 AM 10 1.40 0.00 0.36 0.02 0.00 98.23
12:10:01 AM 11 1.29 0.00 0.22 0.00 0.00 98.49
12:10:01 AM 12 1.57 0.00 0.35 0.01 0.00 98.08
12:10:01 AM 13 0.95 0.00 0.18 0.00 0.00 98.86
12:10:01 AM 14 1.48 0.00 0.34 0.04 0.00 98.14
12:10:01 AM 15 1.00 0.00 0.19 0.00 0.00 98.81
...
09:00:01 AM CPU %user %nice %system %iowait %steal %idle
09:10:01 AM all 1.59 0.00 0.42 0.20 0.00 97.79
09:10:01 AM 0 2.34 0.00 1.05 2.89 0.00 93.71
09:10:01 AM 1 0.66 0.00 0.17 0.00 0.00 99.17
09:10:01 AM 2 2.20 0.00 0.79 0.05 0.00 96.96
09:10:01 AM 3 0.84 0.00 0.24 0.02 0.00 98.90
09:10:01 AM 4 2.03 0.00 0.48 0.10 0.00 97.39
09:10:01 AM 5 1.32 0.00 0.20 0.00 0.00 98.48
09:10:01 AM 6 1.83 0.00 0.35 0.04 0.00 97.78
09:10:01 AM 7 0.91 0.00 0.24 0.00 0.00 98.85
09:10:01 AM 8 1.96 0.00 0.48 0.00 0.00 97.56
09:10:01 AM 9 1.41 0.00 0.32 0.00 0.00 98.27
09:10:01 AM 10 1.82 0.00 0.49 0.00 0.00 97.68
09:10:01 AM 11 1.32 0.00 0.29 0.00 0.00 98.39
09:10:01 AM 12 2.22 0.00 0.44 0.00 0.00 97.34
09:10:01 AM 13 1.25 0.00 0.29 0.00 0.00 98.45
09:10:01 AM 14 1.98 0.00 0.51 0.05 0.00 97.46
09:10:01 AM 15 1.40 0.00 0.35 0.00 0.00 98.25
Average: CPU %user %nice %system %iowait %steal %idle
Average: all 1.39 0.00 0.29 0.08 0.00 98.24
Average: 0 1.75 0.00 0.68 0.91 0.00 96.66
Average: 1 0.82 0.00 0.14 0.00 0.00 99.04
Average: 2 1.90 0.00 0.60 0.23 0.00 97.27
Average: 3 0.97 0.00 0.19 0.00 0.00 98.83
Average: 4 2.15 0.00 0.33 0.09 0.00 97.42
Average: 5 0.95 0.00 0.18 0.00 0.00 98.87
Average: 6 1.83 0.00 0.32 0.02 0.00 97.83
Average: 7 0.94 0.00 0.17 0.00 0.00 98.89
Average: 8 1.75 0.00 0.32 0.01 0.00 97.93
Average: 9 0.97 0.00 0.18 0.00 0.00 98.86
Average: 10 1.75 0.00 0.32 0.01 0.00 97.93
Average: 11 1.01 0.00 0.19 0.00 0.00 98.80
Average: 12 1.71 0.00 0.32 0.00 0.00 97.96
Average: 13 0.98 0.00 0.17 0.00 0.00 98.85
Average: 14 1.72 0.00 0.33 0.03 0.00 97.92
Average: 15 1.01 0.00 0.18 0.00 0.00 98.81
#看id列值，百分比
vmstat 1
procs -----------memory---------- ---swap-- -----io---- -system-- ------cpu-----
r b swpd free buff cache si so bi bo in cs us sy id wa st
1 0 0 1531864 440008 23990836 0 0 2 48 0 0 2 0 98 0 0
0 0 0 1531748 440008 23990836 0 0 0 0 1321 2108 0 0 100 0 0
0 0 0 1531896 440008 23990868 0 0 0 64 1484 2107 0 0 100 0 0
0 0 0 1531772 440008 23990868 0 0 0 0 1201 1920 0 0 100 0 0
0 0 0 1531772 440008 23990868 0 0 0 0 1256 1975 0 0 100 0 0
0 0 0 1531524 440008 23990868 0 0 0 28 1857 2816 0 0 99 0 0
0 0 0 1531524 440008 23990868 0 0 0 0 1189 1892 0 0 100 0 0
0 0 0 1531672 440008 23990864 0 0 0 12 1187 1905 0 0 100 0 0
0 0 0 1531672 440008 23990864 0 0 0 0 1138 1857 0 0 100 0 0
0 0 0 1531672 440008 23990864 0 0 0 0 1246 1994 0 0 100 0 0
0 0 0 1531672 440008 23990864 0 0 0 24 1179 1890 0 0 100 0 0
0 0 0 1531548 440008 23990864 0 0 0 8 1284 1972 0 0 100 0 0
1 0 0 1531292 440012 23990860 0 0 0 12 1171 1881 0 0 100 0 0
0 0 0 1531548 440012 23990864 0 0 0 0 1202 1938 0 0 100 0 0
0 0 0 1531548 440012 23990864 0 0 0 0 1249 1953 0 0 100 0 0
0 0 0 1531548 440012 23990864 0 0 0 0 2847 3423 2 0 98 0 0
1 0 0 1531376 440012 23990864 0 0 0 72 1291 2137 0 0 100 0 0
0 0 0 1531376 440012 23990864 0 0 0 24 1164 1922 0 0 100 0 0
0 0 0 1531340 440012 23990792 0 0 0 0 1213 1899 0 0 100 0 0
0 0 0 1531464 440012 23990792 0 0 0 0 1304 1988 0 0 100 0 0
#看%CPU列值，对应cpu百分比
pidstat 1
Linux 3.10.0-957.21.3.el7.x86_64 (xxxx) 09/21/2020 _x86_64_ (16 CPU)
09:24:47 AM UID PID %usr %system %guest %CPU CPU Command
09:24:48 AM 0 28756 0.99 0.00 0.00 0.99 0 AliYunDun
09:24:48 AM 0 29346 0.00 0.99 0.00 0.99 6 pidstat
09:24:48 AM 0 31212 14.85 5.94 0.00 20.79 7 presto-server
09:24:48 AM UID PID %usr %system %guest %CPU CPU Command
09:24:49 AM 0 2769 4.00 0.00 0.00 4.00 0 exe
09:24:49 AM 0 21486 1.00 0.00 0.00 1.00 15 java
09:24:49 AM 0 29346 0.00 1.00 0.00 1.00 6 pidstat
09:24:49 AM 0 31212 4.00 4.00 0.00 8.00 7 presto-server
09:24:49 AM UID PID %usr %system %guest %CPU CPU Command
09:24:50 AM 0 21645 0.00 1.00 0.00 1.00 11 java
09:24:50 AM 0 29346 1.00 0.00 0.00 1.00 6 pidstat
09:24:50 AM 0 31212 54.00 5.00 0.00 59.00 7 presto-server
09:24:50 AM UID PID %usr %system %guest %CPU CPU Command
09:24:51 AM 0 21486 1.00 0.00 0.00 1.00 15 java
09:24:51 AM 0 29346 0.00 1.00 0.00 1.00 5 pidstat
09:24:51 AM 0 31212 26.00 0.00 0.00 26.00 7 presto-server
09:24:51 AM UID PID %usr %system %guest %CPU CPU Command
09:24:52 AM 0 29346 0.00 1.00 0.00 1.00 5 pidstat
09:24:52 AM 0 31212 11.00 6.00 0.00 17.00 7 presto-server
1.2 饱和度
#看r列对比cpu vcore
vmstat 1
procs -----------memory---------- ---swap-- -----io---- -system-- ------cpu-----
r b swpd free buff cache si so bi bo in cs us sy id wa st
1 0 0 1489264 440360 24005792 0 0 2 48 0 0 2 0 98 0 0
0 0 0 1488888 440360 24005832 0 0 0 228 2597 3103 2 0 98 0 0
2 0 0 1488996 440376 24005824 0 0 0 128 2871 3276 2 1 97 0 0
0 0 0 1489440 440376 24005840 0 0 0 0 1885 2514 2 0 98 0 0
0 0 0 1489440 440376 24005840 0 0 0 0 2587 3248 2 0 98 0 0
5 0 0 1482236 440376 24005840 0 0 0 0 24530 29426 27 3 70 0 0
1 0 0 1480936 440376 24005840 0 0 0 0 21104 24061 14 2 84 0 0
0 0 0 1483072 440376 24006076 0 0 0 24 2606 4608 0 0 99 0 0
0 0 0 1483072 440376 24006076 0 0 0 0 1691 2527 0 0 99 0 0
0 0 0 1483676 440376 24006068 0 0 0 0 1686 2211 1 0 99 0 0
0 0 0 1483676 440376 24006068 0 0 0 0 1268 1944 0 0 99 0 0
0 0 0 1483948 440384 24006068 0 0 0 20 1530 2435 1 0 99 0 0
0 0 0 1483964 440384 24006068 0 0 0 0 1296 2104 0 0 99 0 0
0 0 0 1483964 440384 24006068 0 0 0 16 1180 1885 0 0 100 0 0
#看runq-sz对比cpu vcore，可以找到超出时间点，再看日志
sar -q
Linux 3.10.0-957.21.3.el7.x86_64 (xxxx) 09/21/2020 _x86_64_ (16 CPU)
12:00:01 AM runq-sz plist-sz ldavg-1 ldavg-5 ldavg-15 blocked
12:10:01 AM 4 1134 0.21 0.21 0.17 0
12:20:01 AM 3 1135 0.15 0.30 0.23 0
12:30:01 AM 12 1136 0.16 0.20 0.21 0
12:40:01 AM 5 1137 0.29 0.18 0.19 0
12:50:01 AM 3 1124 0.21 0.14 0.15 1
01:00:01 AM 14 1127 0.25 0.16 0.14 0
01:10:01 AM 6 1131 0.00 0.05 0.11 1
01:20:01 AM 4 1138 0.09 0.10 0.12 0
01:30:01 AM 8 1133 0.05 0.14 0.14 0
01:40:01 AM 3 1120 0.06 0.18 0.20 0
01:50:01 AM 3 1132 0.11 0.16 0.20 0
02:00:01 AM 4 1126 0.27 0.22 0.23 0
02:10:01 AM 6 1120 0.01 0.17 0.25 0
02:20:01 AM 3 1129 0.22 0.34 0.32 0
02:30:01 AM 29 1141 0.18 0.22 0.27 0
02:40:01 AM 4 1130 0.08 0.16 0.22 0
02:50:01 AM 5 1147 0.04 0.19 0.23 0
03:00:01 AM 8 1140 0.08 0.18 0.21 0
03:10:01 AM 3 1138 0.01 0.10 0.18 0
03:20:01 AM 5 1134 0.36 0.20 0.17 0
03:30:01 AM 27 1142 0.12 0.21 0.21 0
03:40:01 AM 5 1130 0.15 0.28 0.27 0
03:50:01 AM 4 1140 0.16 0.24 0.25 0
04:00:01 AM 7 1138 1.25 0.41 0.29 0
04:10:01 AM 3 1124 0.26 0.26 0.26 0
04:20:01 AM 3 1140 0.20 0.43 0.38 0
04:30:01 AM 21 1149 0.26 0.24 0.30 0
04:40:01 AM 3 1130 0.20 0.27 0.31 0
04:50:01 AM 3 1127 0.10 0.22 0.27 0
05:00:01 AM 10 1127 0.24 0.23 0.24 0
05:10:01 AM 3 1124 0.17 0.26 0.28 0
05:20:01 AM 5 1136 0.18 0.21 0.23 0
05:30:01 AM 4 1144 0.16 0.16 0.21 0
05:40:01 AM 4 1138 0.07 0.09 0.16 0
05:50:01 AM 4 1165 0.10 0.15 0.16 0
06:00:01 AM 6 1149 0.09 0.12 0.13 0
06:10:01 AM 3 1139 0.05 0.12 0.13 0
06:20:01 AM 4 1132 0.65 0.37 0.23 0
06:30:01 AM 5 1149 0.16 0.18 0.20 0
06:30:01 AM runq-sz plist-sz ldavg-1 ldavg-5 ldavg-15 blocked
06:40:01 AM 4 1127 0.29 0.24 0.22 0
06:50:01 AM 6 1145 0.04 0.12 0.18 0
07:00:01 AM 4 1165 0.13 0.13 0.15 0
07:10:01 AM 3 1145 0.14 0.28 0.22 0
07:20:01 AM 3 1140 0.06 0.22 0.22 0
07:30:01 AM 5 1151 0.12 0.11 0.16 0
07:40:01 AM 3 1127 0.24 0.27 0.22 0
07:50:01 AM 4 1140 0.11 0.40 0.33 0
08:00:01 AM 4 1136 0.14 0.16 0.22 0
08:10:01 AM 4 1148 0.07 0.10 0.17 0
08:20:01 AM 4 1140 0.26 0.26 0.23 0
08:30:01 AM 4 1152 0.25 0.25 0.22 0
08:40:01 AM 4 1147 0.02 0.18 0.27 0
08:50:01 AM 3 1140 0.13 0.18 0.22 0
09:00:01 AM 4 1190 0.08 0.35 0.40 0
09:10:01 AM 5 1172 0.12 0.26 0.37 0
09:20:01 AM 3 1152 0.08 0.30 0.36 0
Average: 6 1139 0.17 0.21 0.23 0
#看run对比cpu vcore
dstat -p
---procs---
run blk new
0 0 4.5
0 0 0
0 0 0
0 1.0 4.0
0 0 1.0
0 0 0
0 0 3.0
0 0 0
0 0 0
0 0 3.0
0 0 0
1.3 错误
#通过Perf检查cpu错误，这个还在摸索
yum install -y perf
perf record -a -g -F 997 sleep 10
perf report --stdio
perf report -i perf.data
perf top
2. 内存容量
2.1 利用率
free -h
total used free shared buff/cache available
Mem: 62G 4.2G 34G 1.8M 24G 57G
Swap: 0B 0B 0B
#看free值
vmstat 1
procs -----------memory---------- ---swap-- -----io---- -system-- ------cpu-----
r b swpd free buff cache si so bi bo in cs us sy id wa st
1 0 0 30605208 455520 25129440 0 0 2 48 0 0 2 0 98 0 0
0 0 0 30605188 455520 25129252 0 0 0 1952 1276 2101 0 0 100 0 0
0 0 0 30605188 455520 25129252 0 0 0 0 1836 2752 0 0 100 0 0
0 0 0 30605304 455520 25129256 0 0 0 0 1521 2364 0 0 100 0 0
17 1 0 30594660 455520 25129256 0 0 56 56 13233 14928 13 1 86 0 0
2 0 0 30567776 455520 25129444 0 0 100 0 29714 33495 34 4 63 0 0
#看%memused列
sar -r
Linux 3.10.0-957.21.3.el7.x86_64 (xxxx) 09/21/2020 _x86_64_ (16 CPU)
12:00:01 AM kbmemfree kbmemused %memused kbbuffers kbcached kbcommit %commit kbactive kbinact kbdirty
12:10:01 AM 2152152 63654540 96.73 436956 22214992 40805708 62.01 51180624 10591448 25600
12:20:01 AM 2089212 63717480 96.83 437096 22280084 40804908 62.01 51185436 10652272 156
12:30:01 AM 940680 64866012 98.57 437248 23434256 40780900 61.97 51250152 11737500 64088
12:40:01 AM 408660 65398032 99.38 436684 23834016 41024404 62.34 51373820 12147108 212
12:50:01 AM 357952 65448740 99.46 436804 23848928 41005284 62.31 51389108 12179900 38156
01:00:01 AM 447984 65358708 99.32 436312 23775548 40939568 62.21 51373016 12112196 152
01:10:01 AM 1683176 64123516 97.44 436456 22554388 40925624 62.19 51227600 11022324 152
01:20:01 AM 1055152 64751540 98.40 436656 23186696 40944164 62.22 51255536 11617552 392468
01:30:01 AM 446472 65360220 99.32 436628 23816636 40915660 62.18 51285020 12201244 116
01:40:01 AM 347924 65458768 99.47 436736 23948356 40881580 62.12 51281072 12305968 232
01:50:01 AM 402948 65403744 99.39 436756 23898032 40880304 62.12 51319996 12211112 104
02:00:01 AM 424508 65382184 99.35 436924 23880652 40863688 62.10 51316396 12193432 92
02:10:01 AM 2599900 63206792 96.05 437168 21726848 40849996 62.08 51061176 10274572 124
02:20:01 AM 2513684 63293008 96.18 437344 21818064 40844408 62.07 51059236 10359812 140
02:30:01 AM 2514188 63292504 96.18 437472 21824324 40834280 62.05 51052344 10365216 104
02:40:01 AM 1177564 64629128 98.21 437612 23185904 40820612 62.03 51085744 11669512 80
02:50:01 AM 1172248 64634444 98.22 437732 23189524 40831296 62.05 51086252 11672612 112
03:00:01 AM 416988 65389704 99.37 437160 23956340 40811052 62.02 51145408 12371000 96
03:10:01 AM 2828024 62978668 95.70 437284 21548320 40815352 62.02 51008004 10097520 468
03:20:01 AM 2578188 63228504 96.08 437432 21796856 40810988 62.02 51024956 10330308 39244
03:30:01 AM 1698328 64108364 97.42 437576 22679448 40803236 62.00 51072156 11162136 104
03:40:01 AM 1621460 64185232 97.54 437820 22758080 40797956 62.00 51064876 11247220 124
03:50:01 AM 1226600 64580092 98.14 437992 23151076 40808972 62.01 51089080 11616160 181504
04:00:01 AM 425556 65381136 99.35 437540 23957400 40794252 61.99 51140032 12368276 120
04:10:01 AM 2172020 63634672 96.70 437772 22197756 40805756 62.01 51061504 10699176 29068
2.2 饱和度
#看pgscank+pgscand
sar -B
Linux 3.10.0-957.21.3.el7.x86_64 (xxxx) 09/21/2020 _x86_64_ (16 CPU)
12:00:01 AM pgpgin/s pgpgout/s fault/s majflt/s pgfree/s pgscank/s pgscand/s pgsteal/s %vmeff
12:10:01 AM 0.01 2711.22 887.63 0.00 4065.41 4.62 0.00 4.62 100.00
12:20:01 AM 0.00 193.39 789.68 0.00 613.08 0.00 0.00 0.00 0.00
12:30:01 AM 0.03 2088.99 804.65 0.00 2357.05 0.00 0.00 0.00 0.00
12:40:01 AM 16.91 1626.47 848.22 0.00 2083.60 189.15 0.00 189.15 100.00
...
08:30:01 AM 0.01 1707.76 796.28 0.00 1440.20 0.00 0.00 0.00 0.00
08:40:01 AM 2.65 305.28 822.13 0.00 801.81 0.00 0.00 0.00 0.00
08:50:01 AM 0.06 478.32 798.58 0.00 1379.61 0.00 0.00 0.00 0.00
09:00:01 AM 1.48 1997.65 811.94 0.00 2645.12 279.41 0.00 279.41 100.00
09:10:01 AM 0.03 1700.61 809.90 0.00 3534.76 8.63 0.00 8.63 100.00
09:20:01 AM 4.11 1594.87 836.14 0.01 1664.99 0.00 0.00 0.00 0.00
09:30:01 AM 105.78 53.43 868.93 0.02 1355.42 0.00 0.00 0.00 0.00
09:40:01 AM 246.19 2130.54 1164.77 0.18 4440.68 169.32 0.00 169.29 99.99
09:50:01 AM 79.36 60.07 1142.54 0.00 22955.35 118.79 0.30 119.09 100.00
10:00:01 AM 454.18 350.79 1220.49 0.00 24430.02 237.49 0.00 237.49 100.00
10:10:01 AM 234.92 2496.84 932.87 0.00 4843.63 0.00 0.00 0.00 0.00
10:20:01 AM 15.38 43.14 798.34 0.00 1114.98 0.00 0.00 0.00 0.00
10:30:01 AM 83.34 1926.77 814.10 0.00 4828.27 0.00 0.00 0.00 0.00
Average: 20.01 1137.13 828.88 0.00 2576.88 55.01 0.00 55.01 99.98
2.3 错误
dmesg
...
[15168266.919783] AliSecGuard: loading out-of-tree module taints kernel.
[15168266.919823] AliSecGuard: module verification failed: signature and/or required key missing - tainting kernel
3. 网络接口
3.1 利用率
#看吞吐量
ip -s link
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN mode DEFAULT group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
RX: bytes packets errors dropped overrun mcast
32085198771302 1881009187 0 0 0 0
TX: bytes packets errors dropped carrier collsns
32085198771302 1881009187 0 0 0 0
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP mode DEFAULT group default qlen 1000
link/ether 00:16:3e:0a:86:b6 brd ff:ff:ff:ff:ff:ff
RX: bytes packets errors dropped overrun mcast
62685220343242 46595998571 0 183 0 0
TX: bytes packets errors dropped carrier collsns
56316627053476 7886941584 0 0 0 0
#看rx、tx
sar -n DEV
Linux 3.10.0-957.21.3.el7.x86_64 (xxxx) 09/21/2020 _x86_64_ (16 CPU)
12:00:01 AM IFACE rxpck/s txpck/s rxkB/s txkB/s rxcmp/s txcmp/s rxmcst/s
12:10:01 AM eth0 1978.63 530.26 2412.43 1599.24 0.00 0.00 0.00
12:10:01 AM lo 79.36 79.36 1190.66 1190.66 0.00 0.00 0.00
12:20:01 AM eth0 196.71 113.17 149.46 127.43 0.00 0.00 0.00
12:20:01 AM lo 64.84 64.84 1090.28 1090.28 0.00 0.00 0.00
12:30:01 AM eth0 1576.03 321.85 2016.62 1311.62 0.00 0.00 0.00
12:30:01 AM lo 23.54 23.54 304.20 304.20 0.00 0.00 0.00
12:40:01 AM eth0 1183.93 203.85 1546.09 806.03 0.00 0.00 0.00
12:40:01 AM lo 68.70 68.70 1118.69 1118.69 0.00 0.00 0.00
12:50:01 AM eth0 385.66 168.21 379.37 280.20 0.00 0.00 0.00
12:50:01 AM lo 69.65 69.65 1141.32 1141.32 0.00 0.00 0.00
01:00:01 AM eth0 1350.82 321.44 1685.48 1095.89 0.00 0.00 0.00
01:00:01 AM lo 29.39 29.39 409.01 409.01 0.00 0.00 0.00
01:10:01 AM eth0 241.21 125.79 202.72 139.28 0.00 0.00 0.00
01:10:01 AM lo 61.11 61.11 994.01 994.01 0.00 0.00 0.00
01:20:01 AM eth0 946.90 256.17 1142.10 722.73 0.00 0.00 0.00
01:20:01 AM lo 67.03 67.03 1098.29 1098.29 0.00 0.00 0.00
01:30:01 AM eth0 1081.02 253.55 1348.13 917.74 0.00 0.00 0.00
01:30:01 AM lo 37.59 37.59 545.72 545.72 0.00 0.00 0.00
01:40:01 AM eth0 1616.00 367.82 2017.65 1293.44 0.00 0.00 0.00
01:40:01 AM lo 78.43 78.43 1294.59 1294.59 0.00 0.00 0.00
3.2 饱和度
#看overruns、dropped
ifconfig
eth0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
inet xx.xx.xx.xx netmask 255.255.255.0 broadcast xx.xx.xx.xx
ether 00:16:3e:0a:86:b6 txqueuelen 1000 (Ethernet)
RX packets 46596058589 bytes 62685265094302 (57.0 TiB)
RX errors 0 dropped 183 overruns 0 frame 0
TX packets 7886978197 bytes 56316657629732 (51.2 TiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
lo: flags=73<UP,LOOPBACK,RUNNING> mtu 65536
inet 127.0.0.1 netmask 255.0.0.0
loop txqueuelen 1000 (Local Loopback)
RX packets 1881024814 bytes 32085316557312 (29.1 TiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 1881024814 bytes 32085316557312 (29.1 TiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
netstat -s
Ip:
10552202666 total packets received
0 forwarded
0 incoming packets discarded
10552202662 incoming packets delivered
9767415646 requests sent out
258 dropped because of missing route
...
Tcp:
35146791 active connections openings
60695927 passive connection openings
82708 failed connection attempts
195541 connection resets received
136 connections established
10548203174 segments received
46201690722 segments send out
14592669 segments retransmited
321 bad segments received.
1661897 resets sent
Udp:
3998752 packets received
357 packets to unknown port received.
0 packet receive errors
4139226 packets sent
0 receive buffer errors
0 send buffer errors
...
3.3 错误
#看errors、dropped
ifconfig
eth0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
inet xx.xx.xx.xx netmask 255.255.255.0 broadcast xx.xx.xx.xx
ether 00:16:3e:0a:86:b6 txqueuelen 1000 (Ethernet)
RX packets 46596058589 bytes 62685265094302 (57.0 TiB)
RX errors 0 dropped 183 overruns 0 frame 0
TX packets 7886978197 bytes 56316657629732 (51.2 TiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
#看RX-ERR、TX-ERR值
netstat -i
Kernel Interface table
Iface MTU RX-OK RX-ERR RX-DRP RX-OVR TX-OK TX-ERR TX-DRP TX-OVR Flg
eth0 1500 46596218040 0 183 0 7887042825 0 0 0 BMRU
lo 65536 1881062515 0 0 0 1881062515 0 0 0 LRU
#看所有
sar -n EDEV
Linux 3.10.0-957.21.3.el7.x86_64 (xxxx) 09/21/2020 _x86_64_ (16 CPU)
12:00:01 AM IFACE rxerr/s txerr/s coll/s rxdrop/s txdrop/s txcarr/s rxfram/s rxfifo/s txfifo/s
12:10:01 AM eth0 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
12:10:01 AM lo 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
12:20:01 AM eth0 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
12:20:01 AM lo 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
12:30:01 AM eth0 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
12:30:01 AM lo 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
12:40:01 AM eth0 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
12:40:01 AM lo 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
12:50:01 AM eth0 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
12:50:01 AM lo 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
01:00:01 AM eth0 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
01:00:01 AM lo 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
01:10:01 AM eth0 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
01:10:01 AM lo 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
01:20:01 AM eth0 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
01:20:01 AM lo 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
01:30:01 AM eth0 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
01:30:01 AM lo 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
01:40:01 AM eth0 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
01:40:01 AM lo 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
4. 存储设备I/O
4.1 利用率
#看%util列
iostat -xz 1
Linux 3.10.0-957.21.3.el7.x86_64 (xxxx) 09/21/2020 _x86_64_ (16 CPU)
avg-cpu: %user %nice %system %iowait %steal %idle
1.52 0.00 0.27 0.05 0.00 98.16
Device: rrqm/s wrqm/s r/s w/s rkB/s wkB/s avgrq-sz avgqu-sz await r_await w_await svctm %util
vda 0.00 0.91 0.02 2.25 0.86 14.99 13.98 0.01 3.92 8.13 3.88 0.31 0.07
vdb 0.00 4.95 0.30 30.35 36.27 758.48 51.88 0.04 3.89 7.64 3.85 0.38 1.15
avg-cpu: %user %nice %system %iowait %steal %idle
0.50 0.00 0.19 0.19 0.00 99.12
Device: rrqm/s wrqm/s r/s w/s rkB/s wkB/s avgrq-sz avgqu-sz await r_await w_await svctm %util
vda 0.00 0.00 1.00 0.00 8.00 0.00 16.00 0.03 29.00 29.00 0.00 29.00 2.90
avg-cpu: %user %nice %system %iowait %steal %idle
1.31 0.00 0.56 0.00 0.00 98.13
Device: rrqm/s wrqm/s r/s w/s rkB/s wkB/s avgrq-sz avgqu-sz await r_await w_await svctm %util
avg-cpu: %user %nice %system %iowait %steal %idle
0.38 0.00 0.25 0.00 0.00 99.37
Device: rrqm/s wrqm/s r/s w/s rkB/s wkB/s avgrq-sz avgqu-sz await r_await w_await svctm %util
vda 0.00 15.00 0.00 4.00 0.00 80.00 40.00 0.00 1.00 0.00 1.00 0.25 0.10
4.2 饱和度
#看avgqu-sz大于1，或者await较高
iostat -xNz 1
Linux 3.10.0-957.21.3.el7.x86_64 (xxxx) 09/21/2020 _x86_64_ (16 CPU)
avg-cpu: %user %nice %system %iowait %steal %idle
1.52 0.00 0.27 0.05 0.00 98.16
Device: rrqm/s wrqm/s r/s w/s rkB/s wkB/s avgrq-sz avgqu-sz await r_await w_await svctm %util
vda 0.00 0.91 0.02 2.25 0.86 14.99 13.98 0.01 3.92 8.14 3.88 0.31 0.07
vdb 0.00 4.95 0.30 30.35 36.27 758.47 51.88 0.04 3.89 7.64 3.85 0.38 1.15
avg-cpu: %user %nice %system %iowait %steal %idle
0.44 0.00 0.19 0.00 0.00 99.38
Device: rrqm/s wrqm/s r/s w/s rkB/s wkB/s avgrq-sz avgqu-sz await r_await w_await svctm %util
vda 0.00 6.00 0.00 2.00 0.00 32.00 32.00 0.00 0.50 0.00 0.50 0.50 0.10
avg-cpu: %user %nice %system %iowait %steal %idle
0.44 0.00 0.25 0.00 0.00 99.31
Device: rrqm/s wrqm/s r/s w/s rkB/s wkB/s avgrq-sz avgqu-sz await r_await w_await svctm %util
vdb 0.00 4.00 0.00 2.00 0.00 24.00 24.00 0.00 0.50 0.00 0.50 0.50 0.10
4.3 错误
#dtrace还在摸索，暂时看dmesg
dmesg|grep -iw io
[ 0.000000] ACPI: PM-Timer IO Port: 0x608
[ 0.723977] pci_bus 0000:00: root bus resource [io 0x0000-0x0cf7 window]
[ 0.723979] pci_bus 0000:00: root bus resource [io 0x0d00-0xadff window]
[ 0.723980] pci_bus 0000:00: root bus resource [io 0xae0f-0xaeff window]
[ 0.723982] pci_bus 0000:00: root bus resource [io 0xaf20-0xafdf window]
[ 0.723983] pci_bus 0000:00: root bus resource [io 0xafe4-0xffff window]
[ 0.728667] pci 0000:00:01.1: reg 0x20: [io 0xc120-0xc12f]
[ 0.730248] pci 0000:00:01.1: legacy IDE quirk: reg 0x10: [io 0x01f0-0x01f7]
[ 0.730250] pci 0000:00:01.1: legacy IDE quirk: reg 0x14: [io 0x03f6]
[ 0.730251] pci 0000:00:01.1: legacy IDE quirk: reg 0x18: [io 0x0170-0x0177]
[ 0.730253] pci 0000:00:01.1: legacy IDE quirk: reg 0x1c: [io 0x0376]
[ 0.734296] pci 0000:00:01.2: reg 0x20: [io 0xc0c0-0xc0df]
[ 0.736228] pci 0000:00:01.3: quirk: [io 0x0600-0x063f] claimed by PIIX4 ACPI
[ 0.736235] pci 0000:00:01.3: quirk: [io 0x0700-0x070f] claimed by PIIX4 SMB
[ 0.750510] pci 0000:00:03.0: reg 0x10: [io 0xc000-0xc03f]
[ 0.761366] pci 0000:00:04.0: reg 0x10: [io 0xc0e0-0xc0ff]
[ 0.772269] pci 0000:00:05.0: reg 0x10: [io 0xc040-0xc07f]
[ 0.783097] pci 0000:00:06.0: reg 0x10: [io 0xc080-0xc0bf]
[ 0.793233] pci 0000:00:07.0: reg 0x10: [io 0xc100-0xc11f]
[ 0.799352] vgaarb: device added: PCI:0000:00:02.0,decodes=io+mem,owns=io+mem,locks=none
[ 0.811669] pci_bus 0000:00: resource 4 [io 0x0000-0x0cf7 window]
[ 0.811671] pci_bus 0000:00: resource 5 [io 0x0d00-0xadff window]
[ 0.811673] pci_bus 0000:00: resource 6 [io 0xae0f-0xaeff window]
[ 0.811674] pci_bus 0000:00: resource 7 [io 0xaf20-0xafdf window]
[ 0.811676] pci_bus 0000:00: resource 8 [io 0xafe4-0xffff window]
[ 1.037196] PCI-DMA: Using software bounce buffering for IO (SWIOTLB)
[ 1.037200] software IO TLB [mem 0xbbfdf000-0xbffdf000] (64MB) mapped at [ffff8bc3fbfdf000-ffff8bc3fffdefff]
[ 1.061383] io scheduler noop registered
[ 1.061386] io scheduler deadline registered (default)
[ 1.061408] io scheduler cfq registered
[ 1.061412] io scheduler mq-deadline registered
[ 1.061415] io scheduler kyber registered
[ 1.108678] uhci_hcd 0000:00:01.2: irq 11, io base 0x0000c0c0
815
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
