linux下multipath实施rac的asm disk

点击打开链接

前篇blog中 http://www.dbaxiaoyu.com/archives/1130 小鱼说到了linux下多路径绑定单实例的asm，当然一般而言单实例下我们还是会尽可能选择文件系统，而如果考虑到rac环境由于需要共享存储，而共享存储更多的需要固化物理磁盘，此时我们可以选择的是asmlib、udev、多路径，一般而言linux上用的较多的是asmlib和udev绑定，多路径在unix上面较多。


下面小鱼介绍下如何用linux自带的多路径软件mulitpath绑定共享存储来创建asm disk：
1 获取共享存储的wwid唯一设备号。
[root@rac01 ~]# scsi_id -g -u -s /block/sdd
SATA_VBOX_HARDDISK_VB3e724211-d3c035ee_
[root@rac01 ~]# scsi_id -g -u -s /block/sde
SATA_VBOX_HARDDISK_VB4ba68a09-6456b28e_
[root@rac01 ~]# scsi_id -g -u -s /block/sdf
SATA_VBOX_HARDDISK_VB487de809-4febb72d_
[root@rac01 ~]# scsi_id -g -u -s /block/sdg
SATA_VBOX_HARDDISK_VB4659564f-3030047e_


blacklist {
devnode "sd[a-c]"
}


2 配置共享存储的多路径
[root@rac01 ~]# cat /etc/multipath.conf
blacklist {
devnode "sd[a-c]"
}


multipaths {
multipath {
wwid SATA_VBOX_HARDDISK_VB3e724211-d3c035ee_
alias data_mpath1
}
multipath {
wwid SATA_VBOX_HARDDISK_VB4ba68a09-6456b28e_
alias data_mpath2
}
multipath {
wwid SATA_VBOX_HARDDISK_VB487de809-4febb72d_
alias back_mpath1
}
multipath {
wwid SATA_VBOX_HARDDISK_VB4659564f-3030047e_
alias back_mpath2
}
}
拷贝多路径配置文件到节点2
[root@rac01 ~]# scp /etc/multipath.conf 192.168.56.103:/etc/multipath.conf


3 生成多路径磁盘
分别在两个节点执行下面的命令。
[root@rac01 ~]# multipath -F
[root@rac01 ~]# multipath -v2
create: data_mpath1 (SATA_VBOX_HARDDISK_VB3e724211-d3c035ee_) undef ATA,VBOX HARDDISK
size=4.0G features='0' hwhandler='0' wp=undef
`-+- policy='round-robin 0' prio=1 status=undef
`- 3:0:0:0 sdd 8:48 undef ready running
create: data_mpath2 (SATA_VBOX_HARDDISK_VB4ba68a09-6456b28e_) undef ATA,VBOX HARDDISK
size=4.0G features='0' hwhandler='0' wp=undef
`-+- policy='round-robin 0' prio=1 status=undef
`- 4:0:0:0 sde 8:64 undef ready running
create: back_mpath1 (SATA_VBOX_HARDDISK_VB487de809-4febb72d_) undef ATA,VBOX HARDDISK
size=4.0G features='0' hwhandler='0' wp=undef
`-+- policy='round-robin 0' prio=1 status=undef
`- 5:0:0:0 sdf 8:80 undef ready running
create: back_mpath2 (SATA_VBOX_HARDDISK_VB4659564f-3030047e_) undef ATA,VBOX HARDDISK
size=2.0G features='0' hwhandler='0' wp=undef
`-+- policy='round-robin 0' prio=1 status=undef
`- 6:0:0:0 sdg 8:96 undef ready running
[root@rac01 ~]# multipath -ll
back_mpath2 (SATA_VBOX_HARDDISK_VB4659564f-3030047e_) dm-3 ATA,VBOX HARDDISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='round-robin 0' prio=1 status=active
`- 6:0:0:0 sdg 8:96 active ready running
back_mpath1 (SATA_VBOX_HARDDISK_VB487de809-4febb72d_) dm-2 ATA,VBOX HARDDISK
size=4.0G features='0' hwhandler='0' wp=rw
`-+- policy='round-robin 0' prio=1 status=active
`- 5:0:0:0 sdf 8:80 active ready running
data_mpath2 (SATA_VBOX_HARDDISK_VB4ba68a09-6456b28e_) dm-1 ATA,VBOX HARDDISK
size=4.0G features='0' hwhandler='0' wp=rw
`-+- policy='round-robin 0' prio=1 status=active
`- 4:0:0:0 sde 8:64 active ready running
data_mpath1 (SATA_VBOX_HARDDISK_VB3e724211-d3c035ee_) dm-0 ATA,VBOX HARDDISK
size=4.0G features='0' hwhandler='0' wp=rw
`-+- policy='round-robin 0' prio=1 status=active
`- 3:0:0:0 sdd 8:48 active ready running


4对多路径磁盘进行规划（这里可以选择分区也可以选择不分区，注意如果分区最好对/dev/mapper 下的目录进行分区，例如fdisk /dev/mapper/back_mpath1）


分区完成后分别在两个节点重启下iscsi服务和udev规则检验
[root@rac01 ~]# service iscsi restart
[root@rac01 ~]# start_udev
[root@rac01 ~]# ls -l /dev/mapper/
total 0
brw-rw---- 1 root disk 253, 2 Jun 27 22:35 back_mpath1
brw-rw---- 1 root disk 253, 5 Jun 27 22:35 back_mpath1p1
brw-rw---- 1 root disk 253, 3 Jun 27 22:35 back_mpath2
brw-rw---- 1 root disk 253, 4 Jun 27 22:35 back_mpath2p1
crw------- 1 root root 10, 62 Jun 27 21:10 control
brw-rw---- 1 root disk 253, 0 Jun 27 22:54 data_mpath1
brw-rw---- 1 root disk 253, 6 Jun 27 22:55 data_mpath1p1
brw-rw---- 1 root disk 253, 1 Jun 27 22:54 data_mpath2
brw-rw---- 1 root disk 253, 7 Jun 27 22:55 data_mpath2p1


5 改变磁盘的宿主权限。
[root@rac01 oracle]# cat /etc/udev/rules.d/99-asm-multipath.rules
PROGRAM="/bin/chown oracle:oinstall /dev/mapper/back_mpath1p1"
PROGRAM="/bin/chown oracle:oinstall /dev/mapper/back_mpath2p1"
PROGRAM="/bin/chown oracle:oinstall /dev/mapper/data_mpath1p1"
PROGRAM="/bin/chown oracle:oinstall /dev/mapper/data_mpath2p1"
[root@rac01 oracle]# ls -l /dev/mapper/
total 0
brw-rw---- 1 root disk 253, 2 Jun 27 22:36 back_mpath1
brw-rw---- 1 oracle oinstall 253, 5 Jun 27 22:36 back_mpath1p1
brw-rw---- 1 root disk 253, 3 Jun 27 22:36 back_mpath2
brw-rw---- 1 oracle oinstall 253, 4 Jun 27 22:36 back_mpath2p1
crw------- 1 root root 10, 62 Jun 27 21:11 control
brw-rw---- 1 root disk 253, 0 Jun 27 22:36 data_mpath1
brw-rw---- 1 oracle oinstall 253, 6 Jun 27 23:09 data_mpath1p1
brw-rw---- 1 root disk 253, 1 Jun 27 22:36 data_mpath2
brw-rw---- 1 oracle oinstall 253, 7 Jun 27 23:09 data_mpath2p1


拷贝规则文件到另一个节点
[root@rac01 oracle]# scp /etc/udev/rules.d/99-asm-multipath.rules 192.168.56.103:/etc/udev/rules.d/99-asm-multipath.rules
[root@rac02 oracle]# start_udev
[root@rac02 oracle]# ls -l /dev/mapper/
total 0
brw-rw---- 1 root disk 253, 2 Jun 27 22:36 back_mpath1
brw-rw---- 1 oracle oinstall 253, 5 Jun 27 22:36 back_mpath1p1
brw-rw---- 1 root disk 253, 3 Jun 27 22:36 back_mpath2
brw-rw---- 1 oracle oinstall 253, 4 Jun 27 22:36 back_mpath2p1
crw------- 1 root root 10, 62 Jun 27 21:11 control
brw-rw---- 1 root disk 253, 0 Jun 27 22:36 data_mpath1
brw-rw---- 1 oracle oinstall 253, 6 Jun 27 23:09 data_mpath1p1
brw-rw---- 1 root disk 253, 1 Jun 27 22:36 data_mpath2
brw-rw---- 1 oracle oinstall 253, 7 Jun 27 23:09 data_mpath2p1


此时asm disk所需的盘符已经规定好了，接下来可以直接进行asm配置，需要注意的是asm_diskstring需要设置/dev/mapper/*设备，而不要设置/dev/dm-*形式的设备，/dev/dm-*设备是不固定的而且也是供内部使用的。


Asm实例的信息如下：
SQL> show parameter asm;


NAME TYPE
------------------------------------ ----------------------
VALUE
------------------------------
asm_diskgroups string
DATA, BACK
asm_diskstring string
/dev/mapper/*
asm_power_limit integer
1
SQL> select disk_number,path,name from v$asm_disk;

DISK_NUMBER PATH NAME
----------- ---------------------------------------- --------------------
1 /dev/mapper/data_mpath2p1 DATA_0001
0 /dev/mapper/data_mpath1p1 DATA_0000
0 /dev/mapper/back_mpath1p1 BACK_0000
1 /dev/mapper/back_mpath2p1 BACK_0001