RAC下multipath多路径聚合
(2009-12-03 14:54:31)
标签:
分类: 工作
两个节点,分别连接两台SAN交换,接到同一个存储上.
生成48条通道,为了能让48条通道冗余,用了linux下的multipath
配置multipath.conf
cat /etc/multipath.conf
.........
# The defaults section
defaults {
udev_dir /dev
polling_interval 10
selector "round-robin 0"
path_grouping_policy failover
getuid_callout "/sbin/scsi_id -g -u -s /block/%n"
prio const
path_checker tur
rr_min_io 100
rr_weight uniform
failback immediate
no_path_retry 12
user_friendly_names yes
}
.........
blacklist {
# wwid 26353900f02796769
devnode "^(ram|raw|loop|fd|md|dm-|sr|scd|st)[0-9]*"
devnode "^hd[a-z][[0-9]*]"
devnode "^cciss!c[0-9]d[0-9]*"
.......
multipaths {
# based on its wwid
multipath {
wwid 36001438005de9b800000600000140000
alias PSIFile
}
multipath {
wwid 36001438005de9b800000600000280000 把光纤的WWID号起个别名
alias PSIDataRaw01
}
multipath {
wwid 36001438005de9b8000006000002e0000
alias PSIDataRaw02
}}
device {
vendor "HP"
product "HSV2[01]0|HSV300|HSV4[05]0"
getuid_callout "/sbin/scsi_id -g -u -s /block/%n"
prio alua
hardware_handler "0"
path_selector "round-robin 0"
path_grouping_policy group_by_prio
failback immediate
rr_weight uniform
no_path_retry 18
rr_min_io 100
path_checker tur
}
/etc/init.d/multipath start
/etc/init.d/boot.multipathd start
这样会在/dev下生成/dev/dm-*, /dev/disk/by-name和/dev/mapper的设备名称.
ls -l /dev/disk/
total 0
drwxr-xr-x 2 root root 820
Dec 4 2009
by-id
drwxr-xr-x 2 root root 240
Dec 4 2009
by-name
drwxr-xr-x 2 root root 3460 Dec 4 2009 by-path
drwxr-xr-x 2 root root 180
Dec 4 2009
by-uuid
ls -l /dev/mapper/
total 0
brw------- 1 root root 253, 2 Dec 4 2009
PSIDataRaw01
brw------- 1 root root 253, 3 Dec 4 2009
PSIDataRaw01-part1
brw------- 1 root root 253, 4 Dec 4 2009
PSIDataRaw01-part2
brw------- 1 root root 253, 7 Dec 4 2009
PSIDataRaw01-part3
brw------- 1 root root 253, 0 Dec 4 2009
PSIDataRaw02
brw------- 1 root root 253, 5 Dec 4 2009
PSIDataRaw02-part1
brw------- 1 root root 253, 6 Dec 4 2009
PSIDataRaw02-part2
brw-r----- 1 oracle oinstall 253, 8 Dec 3
22:18 PSIDataRaw02-part3
brw------- 1 root root 253, 1 Dec 4 2009
PSIFile
brw------- 1 root root 253, 9 Dec 4 2009
PSIFile-part1
lrwxrwxrwx 1 root root 16 Dec 4 2009
control -> ../device-mapper
ls -l /dev/dm*
brw-r----- 1 root disk 253, 0 Dec 4 2009 /dev/dm-0
brw-r----- 1 root disk 253, 1 Dec 4 2009 /dev/dm-1
brw-r----- 1 root disk 253, 2 Dec 4 2009 /dev/dm-2
brw-r----- 1 root disk 253, 3 Dec 4 2009
/dev/dm-3 -----做multipath时,重启后这些设备名称会重新识别.
brw-r----- 1 root disk 253, 4 Dec 4 2009 /dev/dm-4
brw-r----- 1 root disk 253, 5 Dec 4 2009 /dev/dm-5
brw-r----- 1 root disk 253, 6 Dec 4 2009 /dev/dm-6
brw-r----- 1 root disk 253, 7 Dec 4 2009 /dev/dm-7
brw-r----- 1 root disk 253, 8 Dec 4 2009 /dev/dm-8
brw-r----- 1 root disk 253, 9 Dec 4 2009 /dev/dm-9
ls -l /dev/disk/by-name/
total 0
lrwxrwxrwx 1 root root 10 Dec 4 2009 PSIDataRaw01 ->
../../dm-2
lrwxrwxrwx 1 root root 10 Dec 4 2009 PSIDataRaw01-part1
-> ../../dm-3
lrwxrwxrwx 1 root root 10 Dec 4 2009 PSIDataRaw01-part2
-> ../../dm-4
lrwxrwxrwx 1 root root 10 Dec 4 2009 PSIDataRaw01-part3 ->
../../dm-7 ----重启后,前面的alias name不会变化,但dm-*会重新识别.
lrwxrwxrwx 1 root root 10 Dec 4 2009 PSIDataRaw02 ->
../../dm-0
lrwxrwxrwx 1 root root 10 Dec 4 2009 PSIDataRaw02-part1
-> ../../dm-5
lrwxrwxrwx 1 root root 10 Dec 4 2009 PSIDataRaw02-part2
-> ../../dm-6
lrwxrwxrwx 1 root root 10 Dec 4 2009 PSIDataRaw02-part3
-> ../../dm-8
lrwxrwxrwx 1 root root 10 Dec 4 2009 PSIFile ->
../../dm-1
lrwxrwxrwx 1 root root 10 Dec 4 2009 PSIFile-part1 ->
../../dm-9
在RAC的RAW设备绑定中,不能用raw raw1 /dev/dm-*来实现.
cat /etc/raw
raw1:mapper/PSIDataRaw02-part1 --OCR
raw2:mapper/PSIDataRaw02-part2 --OCR
raw3:mapper/PSIDataRaw01-part1 --VOTEDISK
raw4:mapper/PSIDataRaw01-part2 --VOTEDISK
raw5:mapper/PSIDataRaw01-part3 --VOTEDISK
在SUSE
10中.RAW设备分区属主和权限不能写在/etc/rc.d/boot.local中.要用udev的rules来单独定义,单独在/etc/udev/rules.d下建个rules来定义这些.
PSIDB01:~ # cat /etc/udev/rules.d/99-raw.rules
KERNEL=="raw[1-2]*", OWNER="root", GROUP="oinstall",
MODE="640"
KERNEL=="raw[3-5]*", OWNER="oracle", GROUP="oinstall",
MODE="660"
# /etc/init.d/boot.udev
stop # /etc/init.d/boot.udev
start --重启udev
用作ASM磁盘的分区属主和权限在/etc/rc.d/boot.local下添加:
/bin/chown oracle:oinstall /dev/mapper/PSIDataRaw02-part3
/bin/chmod 640 /dev/mapper/PSIDataRaw02-part3
chkconfig raw on
chkconfig multipathd on
chkconfig boot.multipath boot.
重启节点后,要重新执行
/oracle/product/crs/root.sh
(今天早上ORACLE回复,可以运行/etc/init.d/init.cssd startcheck
去查看相关问题,root.sh可以fix一些配置问题.印象中我也运行了这个命令,好象是报对crsctl没有权限.问题虽然解决,错过一次学习的机会)
查看crs进程
ps -ef |grep init
root 1 0 0 17:02
? 00:00:09 init [5] root 6888 25389 0 22:38
pts/2 00:00:00 grep init
root 10113 1 0 17:03
? 00:00:00 /usr/sbin/sshd -o PidFile=/var/run/sshd.init.pid
root 31640 1 0 22:26
? 00:00:01 /bin/sh /etc/init.d/init.cssd fatal
root 31837 1 0 22:26
? 00:00:00 /bin/sh /etc/init.d/init.evmd run
root 31946 1 0 22:26
? 00:00:00 /bin/sh /etc/init.d/init.crsd run
root 32317 31640 0 22:27
? 00:00:00 /bin/sh /etc/init.d/init.cssd oprocd
root 32324 31640 0 22:27
? 00:00:00 /bin/sh /etc/init.d/init.cssd oclsomon
root 32340 31640 0 22:27
? 00:00:00 /bin/sh /etc/init.d/init.cssd daemon
ps -ef |grep crs
oracle 586 32340 0 22:27
? 00:00:00 /oracle/product/crs/bin/ocssd.bin
oracle 644 31837 0 22:27
? 00:00:00 /bin/su -l oracle -c sh -c 'ulimit -c unlimited; cd
/oracle/product/crs/log/psidb02/evmd; exec
/oracle/product/crs/bin/evmd '
oracle 645 644 0 22:27
? 00:00:00 /oracle/product/crs/bin/evmd.bin
root 736 31946 0 22:27
? 00:00:01 /oracle/product/crs/bin/crsd.bin reboot
oracle 841 645 0 22:27
? 00:00:00 /oracle/product/crs/bin/evmlogger.bin -o
/oracle/product/crs/evm/log/evmlogger.info -l
/oracle/product/crs/evm/log/evmlogger.log
oracle 1638 1 0 22:28
? 00:00:00 /oracle/product/crs/opmn/bin/ons -d
oracle 1639 1638 0 22:28
? 00:00:00 /oracle/product/crs/opmn/bin/ons -d
root 7161 25389 0 22:39
pts/2 00:00:00 grep crs
root 31946 1 0 22:26
?