linux 5 + rac 10.2.0.4磁盘经常非常繁忙,导致经常出现数据库hang的情况,一节点无故重启。简要做如下记录:--get wwwn on Linux OS
[root@prod01 host5]# pwd
/sys/class/fc_host/host5
[root@test01 host5]# ls -ls port_name
0 -r--r--r-- 1 root root 4096 May 20 15:38 port_name
[root@test01 host5]# more port_name
0x10000000c986e9cd[root@prod01 host5]# ls -lrt
total 0
drwxr-xr-x 2 root root 0 May 20 15:31 statistics
--w------- 1 root root 4096 May 20 15:32 uevent
lrwxrwxrwx 1 root root 0 May 20 15:32 device -> ../../../devices/pci0000:80/0000:80:01.0/0000:86:00.0/host5
lrwxrwxrwx 1 root root 0 May 20 15:33 subsystem -> ../../../class/fc_host
-rw-r--r-- 1 root root 4096 May 20 15:38 tgtid_bind_type
-r--r--r-- 1 root root 4096 May 20 15:38 supported_speeds
-r--r--r-- 1 root root 4096 May 20 15:38 supported_fc4s
-r--r--r-- 1 root root 4096 May 20 15:38 supported_classes
-r--r--r-- 1 root root 4096 May 20 15:38 speed
-r--r--r-- 1 root root 4096 May 20 15:38 port_type
-r--r--r-- 1 root root 4096 May 20 15:38 port_state
-r--r--r-- 1 root root 4096 May 20 15:38 port_name
-r--r--r-- 1 root root 4096 May 20 15:38 port_id
-r--r--r-- 1 root root 4096 May 20 15:38 node_name
-r--r--r-- 1 root root 4096 May 20 15:38 maxframe_size
--w------- 1 root root 4096 May 20 15:38 issue_lip
-r--r--r-- 1 root root 4096 May 20 15:38 fabric_name
-r--r--r-- 1 root root 4096 May 20 15:38 active_fc4s/var/log/message:lpfc 0000:86:00.0: 0:1305 Link Down Event x14a received Data: x14a x20 x80110 x0 x0
rport-5:0-2: blocked FC remote port time out: saving binding
lpfc 0000:86:00.0: 0:(0):0203 Devloss timeout on WWPN 20:15:00:a0:b8:47:ab:78 NPort x020f00 Data: x0 x7 x0
94 [RAIDarray.mpp]Array_Module_0:1:0:2 Selection Retry count exhausted
7 [RAIDarray.mpp]Array_Module_0:1:0 Path Failed
496 [RAIDarray.mpp]Array_Module_0:1:0:2 No new path: fall to failover controller case. vcmnd SN 2713476 pdev H5:C0:T0:L2 0x00/0x00/0x00 0x00010000 mpp_status:6
497 [RAIDarray.mpp]Array_Module_0:1:0:2 Failed controller to 0. retry. vcmnd SN 2713476 pdev H5:C0:T0:L2 0x00/0x00/0x00 0x00010000 mpp_status:6
94 [RAIDarray.mpp]Array_Module_0:1:0:2 Selection Retry count exhausted--调整前IO状态信息[root@prod01 ~]# sar -d 3 10Average: DEV tps rd_sec/s wr_sec/s avgrq-sz avgqu-sz await svctm %util
Average: dev8-0 2.43 0.00 81.78 33.64 0.00 0.10 0.05 0.01
Average: dev8-1 2.43 0.00 81.78 33.64 0.00 0.10 0.05 0.01
Average: dev8-2 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
Average: dev8-16 0.43 0.27 3.20 8.00 0.00 0.38 0.38 0.02
Average: dev8-17 0.43 0.27 3.20 8.00 0.00 0.38 0.38 0.02
Average: dev8-32 0.43 0.27 3.20 8.00 0.00 1.00 1.00 0.04
Average: dev8-33 0.43 0.27 3.20 8.00 0.00 1.00 1.00 0.04
Average: dev8-48 0.80 0.53 0.27 1.00 1.44 1679.42 901.96 72.08
Average: dev8-49 0.80 0.53 0.27 1.00 1.44 1679.42 901.96 72.08
Average: dev8-64 0.80 0.53 0.27 1.00 1.44 1679.38 900.96 72.00
Average: dev8-65 0.80 0.53 0.27 1.00 1.44 1679.38 900.96 72.00
Average: dev8-80 0.80 0.53 0.27 1.00 1.43 1679.08 901.50 72.05
Average: dev8-81 0.80 0.53 0.27 1.00 1.43 1679.08 901.50 72.05
Average: dev8-96 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
Average: dev8-112 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
Average: dev8-128 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
Average: dev8-144 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
Average: dev8-160 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
Average: dev8-176 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
Average: dev8-192 0.10 0.00 0.80 8.00 0.62 5444.33 6175.67 61.69
Average: dev8-208 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
Average: dev8-224 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00其中dev8-48,dev8-49,dev8-64,dev8-65,dev8-80,dev8-81几块盘仅存放ocr与voting disk,仍然非常繁忙!结合操作系统日志,怀疑光纤线、光纤卡之一存在问题。查找过程中发现一条线路光纤信号很弱,更换之后。--调整后IO状态信息[root@prod01 ~]# sar -d 3 10
Linux 2.6.18-164.el5 (pcl02) 05/20/201106:51:22 PM DEV tps rd_sec/s wr_sec/s avgrq-sz avgqu-sz await svctm %util
06:51:25 PM dev8-0 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
06:51:25 PM dev8-1 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
06:51:25 PM dev8-2 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
06:51:25 PM dev8-16 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
06:51:25 PM dev8-17 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
06:51:25 PM dev8-32 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
06:51:25 PM dev8-33 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
06:51:25 PM dev8-48 3.00 2.00 1.00 1.00 0.00 0.11 0.11 0.03
06:51:25 PM dev8-49 3.00 2.00 1.00 1.00 0.00 0.11 0.11 0.03
06:51:25 PM dev8-64 3.00 2.00 1.00 1.00 0.00 0.22 0.22 0.07
06:51:25 PM dev8-65 3.00 2.00 1.00 1.00 0.00 0.22 0.22 0.07
06:51:25 PM dev8-80 3.00 2.00 1.00 1.00 0.00 0.11 0.11 0.03
06:51:25 PM dev8-81 3.00 2.00 1.00 1.00 0.00 0.11 0.11 0.03
06:51:25 PM dev8-96 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
06:51:25 PM dev8-112 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
06:51:25 PM dev8-128 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
06:51:25 PM dev8-144 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
06:51:25 PM dev8-160 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
06:51:25 PM dev8-176 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
06:51:25 PM dev8-192 0.33 0.00 2.67 8.00 0.00 1.00 1.00 0.03
06:51:25 PM dev8-208 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
06:51:25 PM dev8-224 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00