rac linux 多路径 udev,RAC下multipath多路径聚合

两个节点,分别连接两台SAN交换,接到同一个存储上.

生成48条通道,为了能让48条通道冗余,用了linux下的multipath

配置multipath.conf

cat /etc/multipath.conf

.........

# The defaults section

defaults {

udev_dir /dev

polling_interval 10

selector "round-robin 0"

path_grouping_policy failover

getuid_callout "/sbin/scsi_id -g -u -s /block/%n"

prio const

path_checker tur

rr_min_io 100

rr_weight uniform

failback immediate

no_path_retry 12

user_friendly_names yes

}

.........

blacklist {

# wwid 26353900f02796769

devnode "^(ram|raw|loop|fd|md|dm-|sr|scd|st)[0-9]*"

devnode "^hd[a-z][[0-9]*]"

devnode "^cciss!c[0-9]d[0-9]*"

.......

multipaths {

# based on its wwid

multipath {

wwid 36001438005de9b800000600000140000

alias PSIFile

}

multipath

{

wwid 36001438005de9b800000600000280000 把光纤的WWID号起个别名

alias PSIDataRaw01

}

multipath

{

wwid 36001438005de9b8000006000002e0000

alias PSIDataRaw02

}}

device {

vendor "HP"

product "HSV2[01]0|HSV300|HSV4[05]0"

getuid_callout "/sbin/scsi_id -g -u -s /block/%n"

prio alua

hardware_handler "0"

path_selector "round-robin 0"

path_grouping_policy group_by_prio

failback immediate

rr_weight uniform

no_path_retry 18

rr_min_io 100

path_checker tur

}

/etc/init.d/multipath start

/etc/init.d/boot.multipathd start

这样会在/dev下生成/dev/dm-*, /dev/disk/by-name和/dev/mapper的设备名称.

ls -l /dev/disk/

total 0

drwxr-xr-x 2 root root 820 Dec 4 2009 by-id

drwxr-xr-x 2 root root 240 Dec 4 2009

by-name

drwxr-xr-x 2 root root 3460 Dec 4 2009 by-path

drwxr-xr-x 2 root root 180 Dec 4 2009

by-uuid

ls -l /dev/mapper/

total 0

brw------- 1 root root 253, 2

Dec 4 2009 PSIDataRaw01

brw------- 1 root root 253, 3

Dec 4 2009 PSIDataRaw01-part1

brw------- 1 root root 253, 4

Dec 4 2009 PSIDataRaw01-part2

brw------- 1 root root 253, 7

Dec 4 2009 PSIDataRaw01-part3

brw------- 1 root root 253, 0

Dec 4 2009 PSIDataRaw02

brw------- 1 root root 253, 5

Dec 4 2009 PSIDataRaw02-part1

brw------- 1 root root 253, 6

Dec 4 2009 PSIDataRaw02-part2

brw-r----- 1 oracle oinstall 253, 8 Dec 3

22:18 PSIDataRaw02-part3

brw------- 1 root root 253, 1

Dec 4 2009 PSIFile

brw------- 1 root root 253, 9

Dec 4 2009 PSIFile-part1

lrwxrwxrwx 1 root root 16 Dec 4 2009 control -> ../device-mapper

ls -l /dev/dm*

brw-r----- 1 root disk 253, 0 Dec 4 2009

/dev/dm-0

brw-r----- 1 root disk 253, 1 Dec 4 2009

/dev/dm-1

brw-r----- 1 root disk 253, 2 Dec 4 2009

/dev/dm-2

brw-r----- 1 root disk 253, 3 Dec 4 2009

/dev/dm-3 -----做multipath时,重启后这些设备名称会重新识别.

brw-r----- 1 root disk 253, 4 Dec 4 2009

/dev/dm-4

brw-r----- 1 root disk 253, 5 Dec 4 2009

/dev/dm-5

brw-r----- 1 root disk 253, 6 Dec 4 2009

/dev/dm-6

brw-r----- 1 root disk 253, 7 Dec 4 2009

/dev/dm-7

brw-r----- 1 root disk 253, 8 Dec 4 2009

/dev/dm-8

brw-r----- 1 root disk 253, 9 Dec 4 2009

/dev/dm-9

ls -l /dev/disk/by-name/

total 0

lrwxrwxrwx 1 root root 10 Dec 4 2009 PSIDataRaw01

-> ../../dm-2

lrwxrwxrwx 1 root root 10 Dec 4 2009 PSIDataRaw01-part1

-> ../../dm-3

lrwxrwxrwx 1 root root 10 Dec 4 2009 PSIDataRaw01-part2

-> ../../dm-4

lrwxrwxrwx 1 root root 10 Dec 4 2009 PSIDataRaw01-part3 ->

../../dm-7 ----重启后,前面的alias name不会变化,但dm-*会重新识别.

lrwxrwxrwx 1 root root 10 Dec 4 2009 PSIDataRaw02

-> ../../dm-0

lrwxrwxrwx 1 root root 10 Dec 4 2009 PSIDataRaw02-part1

-> ../../dm-5

lrwxrwxrwx 1 root root 10 Dec 4 2009 PSIDataRaw02-part2

-> ../../dm-6

lrwxrwxrwx 1 root root 10 Dec 4 2009 PSIDataRaw02-part3

-> ../../dm-8

lrwxrwxrwx 1 root root 10 Dec 4 2009 PSIFile

-> ../../dm-1

lrwxrwxrwx 1 root root 10 Dec 4 2009 PSIFile-part1

-> ../../dm-9

在RAC的RAW设备绑定中,不能用raw raw1 /dev/dm-*来实现.

cat /etc/raw

raw1:mapper/PSIDataRaw02-part1 --OCR

raw2:mapper/PSIDataRaw02-part2 --OCR

raw3:mapper/PSIDataRaw01-part1 --VOTEDISK

raw4:mapper/PSIDataRaw01-part2 --VOTEDISK

raw5:mapper/PSIDataRaw01-part3 --VOTEDISK

在SUSE

10中.RAW设备分区属主和权限不能写在/etc/rc.d/boot.local中.要用udev的rules来单独定义,单独在/etc/udev/rules.d下建个rules来定义这些.

PSIDB01:~ # cat

/etc/udev/rules.d/99-raw.rules

KERNEL=="raw[1-2]*", OWNER="root", GROUP="oinstall",

MODE="640"

KERNEL=="raw[3-5]*", OWNER="oracle", GROUP="oinstall",

MODE="660"

# /etc/init.d/boot.udev

stop # /etc/init.d/boot.udev start --重启udev

用作ASM磁盘的分区属主和权限在/etc/rc.d/boot.local下添加:

/bin/chown oracle:oinstall

/dev/mapper/PSIDataRaw02-part3

/bin/chmod 640 /dev/mapper/PSIDataRaw02-part3

chkconfig raw on

chkconfig multipathd on

chkconfig boot.multipath boot.

重启节点后,要重新执行

/oracle/product/crs/root.sh

(今天早上ORACLE回复,可以运行/etc/init.d/init.cssd startcheck

去查看相关问题,root.sh可以fix一些配置问题.印象中我也运行了这个命令,好象是报对crsctl没有权限.问题虽然解决,错过一次学习的机会)

查看crs进程

ps -ef |grep init

root 1 0 0 17:02

? 00:00:09 init [5] root 6888 25389 0

22:38 pts/2 00:00:00 grep init

root 10113 1 0 17:03 ? 00:00:00 /usr/sbin/sshd -o

PidFile=/var/run/sshd.init.pid

root 31640 1 0 22:26 ? 00:00:01 /bin/sh

/etc/init.d/init.cssd fatal

root 31837 1 0 22:26 ? 00:00:00 /bin/sh

/etc/init.d/init.evmd run

root 31946 1 0 22:26 ? 00:00:00 /bin/sh

/etc/init.d/init.crsd run

root 32317 31640 0

22:27 ? 00:00:00 /bin/sh

/etc/init.d/init.cssd oprocd

root 32324 31640 0

22:27 ? 00:00:00 /bin/sh

/etc/init.d/init.cssd oclsomon

root 32340 31640 0

22:27 ? 00:00:00 /bin/sh

/etc/init.d/init.cssd daemon

ps -ef |grep crs

oracle 586 32340 0

22:27 ? 00:00:00

/oracle/product/crs/bin/ocssd.bin

oracle 644 31837 0

22:27 ? 00:00:00 /bin/su -l oracle -c sh -c

'ulimit -c unlimited; cd /oracle/product/crs/log/psidb02/evmd; exec

/oracle/product/crs/bin/evmd '

oracle 645 644 0

22:27 ? 00:00:00

/oracle/product/crs/bin/evmd.bin

root 736 31946 0

22:27 ? 00:00:01

/oracle/product/crs/bin/crsd.bin reboot

oracle 841 645 0

22:27 ? 00:00:00

/oracle/product/crs/bin/evmlogger.bin -o

/oracle/product/crs/evm/log/evmlogger.info -l

/oracle/product/crs/evm/log/evmlogger.log

oracle 1638 1 0

22:28 ? 00:00:00

/oracle/product/crs/opmn/bin/ons -d

oracle 1639 1638 0

22:28 ? 00:00:00

/oracle/product/crs/opmn/bin/ons -d

root 7161 25389 0

22:39 pts/2 00:00:00 grep crs

root 31946 1 0 22:26 ?