As the root user, install the device-mapper-multipath package using the yum package manager
[root@vzwc1 ~]# cat /etc/redhat-release
Red Hat Enterprise Linux Server release 6.4 (Santiago)
[root@vzwc1 ~]# uname -p
x86_64
[root@vzwc1 ~]# yum install device-mapper-multipath
Copy the multipath.conf file found within /usr/share/doc/device-mapper-multipath-0.4.9/ to /etc/
[root@vzwc1 ~]# cp /usr/share/doc/device-mapper-multipath-0.4.9/multipath.conf /etc/multipath.conf
Capture the scsi id of the local disks on the system.
[root@vzwc2 ~]# for i in b c d e f g h i j
> do
> scsi_id –whitelisted –replace-whitespace –device=/dev/sd$i
> done
1ATA_VBOX_HARDDISK_VBda7290c1-228b4bfb
1ATA_VBOX_HARDDISK_VBcb76adf7-43d27c31
1ATA_VBOX_HARDDISK_VB2077c81c-7cfe5922
1ATA_VBOX_HARDDISK_VB7282bfc8-69601726
1ATA_VBOX_HARDDISK_VBa0072176-9e9ab0a0
1ATA_VBOX_HARDDISK_VB2a7b18e2-04053806
1ATA_VBOX_HARDDISK_VB40939671-89967d5e
1ATA_VBOX_HARDDISK_VB664cb850-2065525c
1ATA_VBOX_HARDDISK_VBa498ea69-53cdb110
[root@vzwc2 ~]#
Enter the above values into /etc/multipath and give friendly aliases to uuid
[root@vzwc1 ~]# cat /etc/multipath.conf
blacklist {
devnode "^(ram|raw|loop|fd|md|dm-|sr|scd|st)[0-9]*"
devnode "^hd[a-z]"
wwid 1ATA_VBOX_HARDDISK_VB82d3887f-a892e124 #exclude sda wwid
}
defaults {
user_friendly_names no
getuid_callout "/sbin/scsi_id –whitelisted –replace-whitespace –device=/dev/%n"
}
multipaths {
multipath {
wwid 1ATA_VBOX_HARDDISK_VBda7290c1-228b4bfb
alias ocrvotedisk1
path_grouping_policy multibus
path_selector "round-robin 0"
failback immediate
rr_weight priorities
no_path_retry 5
}
multipath {
wwid 1ATA_VBOX_HARDDISK_VBcb76adf7-43d27c31
alias ocrvotedisk2
path_grouping_policy multibus
path_selector "round-robin 0"
failback immediate
rr_weight priorities
no_path_retry 5
}
multipath {
wwid 1ATA_VBOX_HARDDISK_VB2077c81c-7cfe5922
alias ocrvotedisk3
path_grouping_policy multibus
path_selector "round-robin 0"
failback immediate
rr_weight priorities
no_path_retry 5
}
multipath {
wwid 1ATA_VBOX_HARDDISK_VB7282bfc8-69601726
alias ocrvotedisk4
path_grouping_policy multibus
path_selector "round-robin 0"
failback immediate
rr_weight priorities
no_path_retry 5
}
multipath {
wwid 1ATA_VBOX_HARDDISK_VBa0072176-9e9ab0a0
alias ocrvotedisk5
path_grouping_policy multibus
path_selector "round-robin 0"
failback immediate
rr_weight priorities
no_path_retry 5
}
multipath {
wwid 1ATA_VBOX_HARDDISK_VB2a7b18e2-04053806
alias data1
path_grouping_policy multibus
path_selector "round-robin 0"
failback immediate
rr_weight priorities
no_path_retry 5
}
multipath {
wwid 1ATA_VBOX_HARDDISK_VB40939671-89967d5e
alias data2
path_grouping_policy multibus
path_selector "round-robin 0"
failback immediate
rr_weight priorities
no_path_retry 5
}
multipath {
wwid 1ATA_VBOX_HARDDISK_VB664cb850-2065525c
alias fra1
path_grouping_policy multibus
path_selector "round-robin 0"
failback immediate
rr_weight priorities
no_path_retry 5
}
multipath {
wwid 1ATA_VBOX_HARDDISK_VBa498ea69-53cdb110
alias fra2
path_grouping_policy multibus
path_selector "round-robin 0"
failback immediate
rr_weight priorities
no_path_retry 5
}
}
Restart multipathd service
[root@vzwc1 ~]# service multipathd restart
ok
Stopping multipathd daemon: [ OK ]
Starting multipathd daemon: [ OK ]
[root@vzwc1 ~]# chkconfig multipathd on
Check if multipath is setup properly
[root@vzwc2 ~]# multipath -ll
fra2 (1ATA_VBOX_HARDDISK_VBa498ea69-53cdb110) dm-8 ATA,VBOX HARDDISK
size=12G features='1 queue_if_no_path' hwhandler='0' wp=rw
`-+- policy='round-robin 0' prio=1 status=active
`- 11:0:0:0 sdj 8:144 active ready running
ocrvotedisk5 (1ATA_VBOX_HARDDISK_VBa0072176-9e9ab0a0) dm-4 ATA,VBOX HARDDISK
size=2.0G features='1 queue_if_no_path' hwhandler='0' wp=rw
`-+- policy='round-robin 0' prio=1 status=active
`- 7:0:0:0 sdf 8:80 active ready running
fra1 (1ATA_VBOX_HARDDISK_VB664cb850-2065525c) dm-7 ATA,VBOX HARDDISK
size=12G features='1 queue_if_no_path' hwhandler='0' wp=rw
`-+- policy='round-robin 0' prio=1 status=active
`- 10:0:0:0 sdi 8:128 active ready running
ocrvotedisk4 (1ATA_VBOX_HARDDISK_VB7282bfc8-69601726) dm-3 ATA,VBOX HARDDISK
size=2.0G features='1 queue_if_no_path' hwhandler='0' wp=rw
`-+- policy='round-robin 0' prio=1 status=active
`- 6:0:0:0 sde 8:64 active ready running
ocrvotedisk3 (1ATA_VBOX_HARDDISK_VB2077c81c-7cfe5922) dm-2 ATA,VBOX HARDDISK
size=2.0G features='1 queue_if_no_path' hwhandler='0' wp=rw
`-+- policy='round-robin 0' prio=1 status=active
`- 5:0:0:0 sdd 8:48 active ready running
ocrvotedisk2 (1ATA_VBOX_HARDDISK_VBcb76adf7-43d27c31) dm-1 ATA,VBOX HARDDISK
size=2.0G features='1 queue_if_no_path' hwhandler='0' wp=rw
`-+- policy='round-robin 0' prio=1 status=active
`- 4:0:0:0 sdc 8:32 active ready running
ocrvotedisk1 (1ATA_VBOX_HARDDISK_VBda7290c1-228b4bfb) dm-0 ATA,VBOX HARDDISK
size=2.0G features='1 queue_if_no_path' hwhandler='0' wp=rw
`-+- policy='round-robin 0' prio=1 status=active
`- 3:0:0:0 sdb 8:16 active ready running
data2 (1ATA_VBOX_HARDDISK_VB40939671-89967d5e) dm-6 ATA,VBOX HARDDISK
size=20G features='1 queue_if_no_path' hwhandler='0' wp=rw
`-+- policy='round-robin 0' prio=1 status=active
`- 9:0:0:0 sdh 8:112 active ready running
data1 (1ATA_VBOX_HARDDISK_VB2a7b18e2-04053806) dm-5 ATA,VBOX HARDDISK
size=20G features='1 queue_if_no_path' hwhandler='0' wp=rw
`-+- policy='round-robin 0' prio=1 status=active
`- 8:0:0:0 sdg 8:96 active ready running
[root@vzwc1 ~]# ls -l /dev/mapper/
total 0
crw-rw—- 1 root root 10, 236 Sep 24 13:56 control
lrwxrwxrwx 1 root root 7 Sep 24 13:57 data1 -> ../dm-5
lrwxrwxrwx 1 root root 7 Sep 24 13:57 data2 -> ../dm-6
lrwxrwxrwx 1 root root 7 Sep 24 13:56 fra1 -> ../dm-7
lrwxrwxrwx 1 root root 7 Sep 24 13:56 fra2 -> ../dm-8
lrwxrwxrwx 1 root root 7 Sep 24 13:57 ocrvotedisk1 -> ../dm-0
lrwxrwxrwx 1 root root 7 Sep 24 13:56 ocrvotedisk2 -> ../dm-1
lrwxrwxrwx 1 root root 7 Sep 24 13:56 ocrvotedisk3 -> ../dm-2
lrwxrwxrwx 1 root root 7 Sep 24 13:56 ocrvotedisk4 -> ../dm-3
lrwxrwxrwx 1 root root 7 Sep 24 13:56 ocrvotedisk5 -> ../dm-4
[root@vzwc1 ~]# for i in ocrvotedisk1 ocrvotedisk2 ocrvotedisk3 ocrvotedisk4 ocrvotedisk5 data1 data2 fra1 fra2
> do
> printf "%s %s\n" "$i" "$(ls -ll /dev/mapper/$i)"
> done
ocrvotedisk1 lrwxrwxrwx 1 root root 7 Sep 24 13:57 /dev/mapper/ocrvotedisk1 -> ../dm-0
ocrvotedisk2 lrwxrwxrwx 1 root root 7 Sep 24 13:56 /dev/mapper/ocrvotedisk2 -> ../dm-1
ocrvotedisk3 lrwxrwxrwx 1 root root 7 Sep 24 13:56 /dev/mapper/ocrvotedisk3 -> ../dm-2
ocrvotedisk4 lrwxrwxrwx 1 root root 7 Sep 24 13:56 /dev/mapper/ocrvotedisk4 -> ../dm-3
ocrvotedisk5 lrwxrwxrwx 1 root root 7 Sep 24 13:56 /dev/mapper/ocrvotedisk5 -> ../dm-4
data1 lrwxrwxrwx 1 root root 7 Sep 24 13:57 /dev/mapper/data1 -> ../dm-5
data2 lrwxrwxrwx 1 root root 7 Sep 24 13:57 /dev/mapper/data2 -> ../dm-6
fra1 lrwxrwxrwx 1 root root 7 Sep 24 13:56 /dev/mapper/fra1 -> ../dm-7
fra2 lrwxrwxrwx 1 root root 7 Sep 24 13:56 /dev/mapper/fra2 -> ../dm-8
Determine partitioned alias for target device
[root@vzwc1 ~]# dmsetup ls|sort
data1 (252:5)
data2 (252:6)
fra1 (252:7)
fra2 (252:8)
ocrvotedisk1 (252:0)
ocrvotedisk2 (252:1)
ocrvotedisk3 (252:2)
ocrvotedisk4 (252:3)
ocrvotedisk5 (252:4)
Edit /etc/udev/rules.d/12-dm-permissions.rules file to set the above device ownership as grid:asmadmin
[root@vzwc1 ~]# cat /etc/udev/rules.d/12-dm-permissions.rules
ENV{DM_NAME}=="ocrvotedisk1", OWNER:="grid", GROUP:="asmadmin", MODE:="660", SYMLINK+="iscsi/oraasm-$env{DM_NAME}"
ENV{DM_NAME}=="ocrvotedisk2", OWNER:="grid", GROUP:="asmadmin", MODE:="660", SYMLINK+="iscsi/oraasm-$env{DM_NAME}"
ENV{DM_NAME}=="ocrvotedisk3", OWNER:="grid", GROUP:="asmadmin", MODE:="660", SYMLINK+="iscsi/oraasm-$env{DM_NAME}"
ENV{DM_NAME}=="ocrvotedisk4", OWNER:="grid", GROUP:="asmadmin", MODE:="660", SYMLINK+="iscsi/oraasm-$env{DM_NAME}"
ENV{DM_NAME}=="ocrvotedisk5", OWNER:="grid", GROUP:="asmadmin", MODE:="660", SYMLINK+="iscsi/oraasm-$env{DM_NAME}"
ENV{DM_NAME}=="data1", OWNER:="grid", GROUP:="asmadmin", MODE:="660", SYMLINK+="iscsi/oraasm-$env{DM_NAME}"
ENV{DM_NAME}=="data2", OWNER:="grid", GROUP:="asmadmin", MODE:="660", SYMLINK+="iscsi/oraasm-$env{DM_NAME}"
ENV{DM_NAME}=="fra1", OWNER:="grid", GROUP:="asmadmin", MODE:="660", SYMLINK+="iscsi/oraasm-$env{DM_NAME}"
ENV{DM_NAME}=="fra2", OWNER:="grid", GROUP:="asmadmin", MODE:="660", SYMLINK+="iscsi/oraasm-$env{DM_NAME}"
Restart multipathd service
[root@vzwc1 ~]# /etc/init.d/multipathd restart
ok
Stopping multipathd daemon: [ OK ]
Starting multipathd daemon: [ OK ]
Check if permissions are correct
[root@vzwc1 ~]# ls -l /dev/dm-*
brw-rw—- 1 grid asmadmin 252, 0 Sep 24 16:35 /dev/dm-0
brw-rw—- 1 grid asmadmin 252, 1 Sep 24 16:35 /dev/dm-1
brw-rw—- 1 grid asmadmin 252, 2 Sep 24 16:35 /dev/dm-2
brw-rw—- 1 grid asmadmin 252, 3 Sep 24 16:35 /dev/dm-3
brw-rw—- 1 grid asmadmin 252, 4 Sep 24 16:35 /dev/dm-4
brw-rw—- 1 grid asmadmin 252, 5 Sep 24 16:35 /dev/dm-5
brw-rw—- 1 grid asmadmin 252, 6 Sep 24 16:35 /dev/dm-6
brw-rw—- 1 grid asmadmin 252, 7 Sep 24 16:35 /dev/dm-7
brw-rw—- 1 grid asmadmin 252, 8 Sep 24 16:35 /dev/dm-8
Check if oracle ASM symbolic links exists for multipath devices
[root@vzwc2 ~]# ls -l /dev/iscsi/*
lrwxrwxrwx 1 root root 7 Sep 24 16:38 /dev/iscsi/oraasm-data1 -> ../dm-5
lrwxrwxrwx 1 root root 7 Sep 24 16:38 /dev/iscsi/oraasm-data2 -> ../dm-6
lrwxrwxrwx 1 root root 7 Sep 24 16:38 /dev/iscsi/oraasm-fra1 -> ../dm-7
lrwxrwxrwx 1 root root 7 Sep 24 16:38 /dev/iscsi/oraasm-fra2 -> ../dm-8
lrwxrwxrwx 1 root root 7 Sep 24 16:38 /dev/iscsi/oraasm-ocrvotedisk1 -> ../dm-0
lrwxrwxrwx 1 root root 7 Sep 24 16:38 /dev/iscsi/oraasm-ocrvotedisk2 -> ../dm-1
lrwxrwxrwx 1 root root 7 Sep 24 16:38 /dev/iscsi/oraasm-ocrvotedisk3 -> ../dm-2
lrwxrwxrwx 1 root root 7 Sep 24 16:38 /dev/iscsi/oraasm-ocrvotedisk4 -> ../dm-3
lrwxrwxrwx 1 root root 7 Sep 24 16:38 /dev/iscsi/oraasm-ocrvotedisk5 -> ../dm-4
Configure ASM disk change discovery path
[grid@vzwc2 ~]$ crsctl query css votedisk
## STATE File Universal Id File Name Disk group
– —– —————– ——— ———
1. ONLINE 1374f266ca874f15bfde7ca23d073e5f (/dev/mapper/ocrvotedisk1) [CRSDG]
2. ONLINE 4145a926ff0d4f6dbf63e2c3695c580e (/dev/mapper/ocrvotedisk2) [CRSDG]
3. ONLINE 6f3ab4bbe0c14f97bfb20c91820d898b (/dev/mapper/ocrvotedisk3) [CRSDG]
4. ONLINE 8c973dd3f1da4f0dbf2080e04fcf9bd7 (/dev/mapper/ocrvotedisk4) [CRSDG]
5. ONLINE 1bed4a3c6f624fabbfbe2564e81df250 (/dev/mapper/ocrvotedisk5) [CRSDG]
Located 5 voting disk(s).
[grid@vzwc2 ~]$ sqlplus / as sysasm
SQL*Plus: Release 11.2.0.4.0 Production on Wed Sep 24 17:17:22 2014
Copyright (c) 1982, 2013, Oracle. All rights reserved.
Connected to:
Oracle Database 11g Enterprise Edition Release 11.2.0.4.0 – 64bit Production
With the Real Application Clusters and Automatic Storage Management options
SQL> select * from v$active_instances;
INST_NUMBER INST_NAME
———– ——————–
1 vzwc1:+ASM1
2 vzwc2:+ASM2
SQL> select path from v$asm_disk;
PATH
——————————————————————————–
/dev/mapper/data2
/dev/mapper/fra2
/dev/mapper/fra1
/dev/mapper/data1
/dev/mapper/ocrvotedisk2
/dev/mapper/ocrvotedisk3
/dev/mapper/ocrvotedisk1
/dev/mapper/ocrvotedisk4
/dev/mapper/ocrvotedisk5
9 rows selected.
Oracle ASM on RHEL 5 with multipath
As the root user, install the device-mapper-multipath package using the yum package manager
[root@db11g ~]# cat /etc/redhat-release
Red Hat Enterprise Linux Server release 5.9 (Tikanga)
[root@db11g ~]# yum install device-mapper*
Loaded plugins: rhnplugin, security
This system is not registered with ULN.
You can use up2date –register to register.
ULN support will be disabled.
Setting up Install Process
Package device-mapper-multipath-0.4.9-56.0.3.el5.x86_64 already installed and latest version
Package device-mapper-multipath-libs-0.4.9-56.0.3.el5.x86_64 already installed and latest version
Package device-mapper-event-1.02.67-2.el5.x86_64 already installed and latest version
Package device-mapper-1.02.67-2.el5.x86_64 already installed and latest version
Package device-mapper-1.02.67-2.el5.i386 already installed and latest version
Nothing to do
[root@db11g ~]#
[root@db11g ~]#
[root@db11g ~]#
[root@db11g ~]#
[root@db11g ~]# rpm -qa|grep device-mapper
device-mapper-multipath-libs-0.4.9-56.0.3.el5
device-mapper-event-1.02.67-2.el5
device-mapper-1.02.67-2.el5
device-mapper-1.02.67-2.el5
device-mapper-multipath-0.4.9-56.0.3.el5
Capture the scsi id of the local disks on the system
[root@db11g ~]# /sbin/scsi_id -g -u -s /block/sdb
SATA_VBOX_HARDDISK_VB3c0bb909-10aab3a0_
[root@db11g ~]# /sbin/scsi_id -g -u -s /block/sdc
SATA_VBOX_HARDDISK_VB3edd7dc1-fd831bf4_
Get to the grid user's uid and gid, here the gid is asmadmin group
[root@db11g ~]# cat /etc/passwd|grep grid
grid:x:1100:1000:Grid Infrastructure Owner:/home/grid:/bin/bash
[root@db11g ~]# cat /etc/group|grep asmadmin
asmadmin:x:1200:grid,oracle
[root@db11g ~]# id -a grid
uid=1100(grid) gid=1000(oinstall) groups=1000(oinstall),1200(asmadmin),1201(asmdba),1202(asmoper),1300(dba)
Enter the above values into /etc/multipath.conf
[root@db11g ~]# cp /usr/share/doc/device-mapper-multipath-0.4.9/multipath.conf /etc/
[root@db11g ~]# cat /etc/multipath.conf
defaults {
user_friendly_names no
queue_without_daemon no
flush_on_last_del yes
max_fds max
}
blacklist {
devnode "^hd[a-z]"
devnode "^(ram|raw|loop|fd|md|dm-|sr|scd|st)[0-9]*"
devnode "^cciss.*"
}
devices {
device {
vendor "OPNFILER "
product "LUN"
path_grouping_policy group_by_prio
features "3 queue_if_no_path pg_init_retries 50"
getuid_callout "/sbin/scsi_id -g -u -s /block/%n"
path_checker tur
path_selector "round-robin 0"
hardware_handler "1 alua"
failback immediate
rr_weight uniform
rr_min_io 128
}
}
multipaths {
multipath {
wwid SATA_VBOX_HARDDISK_VB3c0bb909-10aab3a0_
alias datadisk
uid 1100
gid 1200
}
multipath {
wwid SATA_VBOX_HARDDISK_VB3edd7dc1-fd831bf4_
alias fradisk
uid 1100
gid 1200
}
}
Start multipathd service
[root@db11g ~]# modprobe dm-multipath
[root@db11g ~]# modprobe dm-round-robin
[root@db11g ~]# chkconfig multipathd start
[root@db11g ~]# chkconfig multipathd on
[root@db11g ~]# /etc/init.d/multipathd start
Starting multipathd daemon: [ OK ]
[root@db11g ~]# ps -ef|grep multipathd
root 3084 1 0 13:16 ? 00:00:00 /sbin/multipathd
root 3186 2877 0 13:16 pts/0 00:00:00 grep multipathd
[root@db11g ~]# multipath -F
[root@db11g ~]# multipath -v2
Jan 15 13:17:40 | SATA_VBOX_HARDDISK_VBa64c2685-cc2f3dd0_: ignoring map
create: datadisk (SATA_VBOX_HARDDISK_VB3c0bb909-10aab3a0_) undef ATA,VBOX HARDDISK
size=16G features='0' hwhandler='0' wp=undef
`-+- policy='round-robin 0' prio=1 status=undef
`- 1:0:0:0 sdb 8:16 undef ready running
create: fradisk (SATA_VBOX_HARDDISK_VB3edd7dc1-fd831bf4_) undef ATA,VBOX HARDDISK
size=10G features='0' hwhandler='0' wp=undef
`-+- policy='round-robin 0' prio=1 status=undef
`- 2:0:0:0 sdc 8:32 undef ready running
[root@db11g ~]#
[root@db11g ~]# multipath -ll
fradisk (SATA_VBOX_HARDDISK_VB3edd7dc1-fd831bf4_) dm-3 ATA,VBOX HARDDISK
size=10G features='0' hwhandler='0' wp=rw
`-+- policy='round-robin 0' prio=1 status=active
`- 2:0:0:0 sdc 8:32 active ready running
datadisk (SATA_VBOX_HARDDISK_VB3c0bb909-10aab3a0_) dm-2 ATA,VBOX HARDDISK
size=16G features='0' hwhandler='0' wp=rw
`-+- policy='round-robin 0' prio=1 status=active
`- 1:0:0:0 sdb 8:16 active ready running
[root@db11g ~]# ll /dev/mapper/*disk
brw-rw—- 1 grid asmadmin 252, 2 Jan 15 13:17 /dev/mapper/datadisk
brw-rw—- 1 grid asmadmin 252, 3 Jan 15 13:17 /dev/mapper/fradisk
7481
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
