首先在存储上划分LUN,本文以openfiler进行举例
划分两个8G的LUN 一个给DATA磁盘组 一个给FRA 磁盘组
map LUN
在RAC两个节点,分别扫描LUN设备
扫盘脚本如下,root用户执行
[root@rac01 ~]# cat scan-disk.sh
#!/bin/bash
#scan disk
for i in $(seq 0 100)
do
echo '- - -' > /sys/class/scsi_host/host${i}/scan
done &> /dev/null
扫描之后使用multipath -ll指令,发现多了两个新的LUN设备
[root@rac01 ~]# multipath -ll
asm-fra01 (14f504e46494c4500423356646a332d4565674d2d74515555) dm-6 OPNFILER,VIRTUAL-DISK
size=8.0G features='0' hwhandler='0' wp=rw
|-+- policy='service-time 0' prio=1 status=active
| `- 33:0:0:6 sdo 8:224 active ready running
`-+- policy='service-time 0' prio=1 status=enabled
`- 34:0:0:6 sdm 8:192 active ready running
asm-data03 (14f504e46494c45004a7a416363632d523475692d746f3068) dm-5 OPNFILER,VIRTUAL-DISK
size=8.0G features='0' hwhandler='0' wp=rw
|-+- policy='service-time 0' prio=1 status=active
| `- 34:0:0:5 sdk 8:160 active ready running
`-+- policy='service-time 0' prio=1 status=enabled
`- 33:0:0:5 sdn 8:208 active ready running
asm-ocr3 (14f504e46494c45004d37476d64492d516f326d2d55747862) dm-7 OPNFILER,VIRTUAL-DISK
size=3.0G features='0' hwhandler='0' wp=rw
|-+- policy='service-time 0' prio=1 status=active
| `- 34:0:0:2 sde 8:64 active ready running
`-+- policy='service-time 0' prio=1 status=enabled
`- 33:0:0:2 sdh 8:112 active ready running
asm-data02 (14f504e46494c4500763534334d612d363538792d32626e6e) dm-3 OPNFILER,VIRTUAL-DISK
size=8.0G features='0' hwhandler='0' wp=rw
|-+- policy='service-time 0' prio=1 status=active
| `- 34:0:0:4 sdi 8:128 active ready running
`-+- policy='service-time 0' prio=1 status=enabled
`- 33:0:0:4 sdl 8:176 active ready running
asm-ocr2 (14f504e46494c45004f6647784c6b2d634348482d4d537443) dm-8 OPNFILER,VIRTUAL-DISK
size=3.0G features='0' hwhandler='0' wp=rw
|-+- policy='service-time 0' prio=1 status=active
| `- 34:0:0:1 sdd 8:48 active ready running
`-+- policy='service-time 0' prio=1 status=enabled
`- 33:0:0:1 sdf 8:80 active ready running
asm-data01 (14f504e46494c4500707a4b7430482d633044672d6158706b) dm-9 OPNFILER,VIRTUAL-DISK
size=8.0G features='0' hwhandler='0' wp=rw
|-+- policy='service-time 0' prio=1 status=active
| `- 33:0:0:3 sdj 8:144 active ready running
`-+- policy='service-time 0' prio=1 status=enabled
`- 34:0:0:3 sdg 8:96 active ready running
asm-ocr1 (14f504e46494c45007831743376342d42645a362d38646763) dm-4 OPNFILER,VIRTUAL-DISK
size=3.0G features='0' hwhandler='0' wp=rw
|-+- policy='service-time 0' prio=1 status=active
| `- 34:0:0:0 sdb 8:16 active ready running
`-+- policy='service-time 0' prio=1 status=enabled
`- 33:0:0:0 sdc 8:32 active ready running
mpathi (14f504e46494c450068306d5534302d6e6a4b782d694a4741) dm-11 OPNFILER,VIRTUAL-DISK
size=8.0G features='0' hwhandler='0' wp=rw
|-+- policy='service-time 0' prio=1 status=active
| `- 34:0:0:8 sds 65:32 active ready running
`-+- policy='service-time 0' prio=1 status=enabled
`- 33:0:0:8 sdq 65:0 active ready running
mpathh (14f504e46494c4500336d586655422d77766e312d56507641) dm-10 OPNFILER,VIRTUAL-DISK
size=8.0G features='0' hwhandler='0' wp=rw
|-+- policy='service-time 0' prio=1 status=active
| `- 34:0:0:7 sdr 65:16 active ready running
`-+- policy='service-time 0' prio=1 status=enabled
`- 33:0:0:7 sdp 8:240 active ready running
asm实例下查询磁盘组信息
su - grid
[rac01:grid]:/home/grid>sqlplus / as sysasm
SQL> set linesize 200 pages 999
SQL> col path for a30
SQL> select mount_status, path, os_mb from v$asm_disk order by 2;
MOUNT_STATUS PATH OS_MB
--------------------- ------------------------------ ----------
CACHED /dev/asm-data01 8192
CACHED /dev/asm-data02 8192
CACHED /dev/asm-data03 8192
CACHED /dev/asm-fra01 8192
CACHED /dev/asm-ocr1 3072
CACHED /dev/asm-ocr2 3072
CACHED /dev/asm-ocr3 3072
7 rows selected.
配置multipath和udev,注意:两个节点都要做
查看新增磁盘信息
[root@rac01 ~]# multipath -ll|grep -A4 mpath*
mpathi (14f504e46494c450068306d5534302d6e6a4b782d694a4741) dm-11 OPNFILER,VIRTUAL-DISK
size=8.0G features='0' hwhandler='0' wp=rw
|-+- policy='service-time 0' prio=1 status=active
| `- 34:0:0:8 sds 65:32 active ready running
`-+- policy='service-time 0' prio=1 status=enabled
--
mpathh (14f504e46494c4500336d586655422d77766e312d56507641) dm-10 OPNFILER,VIRTUAL-DISK
size=8.0G features='0' hwhandler='0' wp=rw
|-+- policy='service-time 0' prio=1 status=active
| `- 34:0:0:7 sdr 65:16 active ready running
`-+- policy='service-time 0' prio=1 status=enabled
备份multipath配置文件
[root@rac01 ~]# cp /etc/multipath.conf /etc/multipath.conf.20210529.bak
[root@rac01 ~]# ll /etc/multipath.conf.20210529.bak
-rw------- 1 root root 3380 May 29 17:14 /etc/multipath.conf.20210529.bak
编辑multipath配置文件,追加新的LUN设备wwid和别名
[root@rac01 ~]# vim /etc/multipath.conf
multipaths {
multipath {
wwid 14f504e46494c45007831743376342d42645a362d38646763
alias asm-ocr1
}
multipath {
wwid 14f504e46494c45004f6647784c6b2d634348482d4d537443
alias asm-ocr2
}
multipath {
wwid 14f504e46494c45004d37476d64492d516f326d2d55747862
alias asm-ocr3
}
multipath {
wwid 14f504e46494c4500707a4b7430482d633044672d6158706b
alias asm-data01
}
multipath {
wwid 14f504e46494c4500763534334d612d363538792d32626e6e
alias asm-data02
}
multipath {
wwid 14f504e46494c45004a7a416363632d523475692d746f3068
alias asm-data03
}
multipath {
wwid 14f504e46494c4500423356646a332d4565674d2d74515555
alias asm-fra01
}
multipath {
wwid 14f504e46494c450068306d5534302d6e6a4b782d694a4741
alias asm-data04
}
multipath {
wwid 14f504e46494c4500336d586655422d77766e312d56507641
alias asm-fra02
}
}
重启multipathd服务
[root@rac01 ~]# systemctl restart multipathd
查看是否配置成功
[root@rac01 ~]# multipath -ll | grep -E "data04|fra02"
asm-fra02 (14f504e46494c4500336d586655422d77766e312d56507641) dm-10 OPNFILER,VIRTUAL-DISK
asm-data04 (14f504e46494c450068306d5534302d6e6a4b782d694a4741) dm-11 OPNFILER,VIRTUAL-DISK
修改udev绑定
[root@rac01 ~]# vim /etc/udev/rules.d/99-oracle-asmdevice.rules
追加2行新的磁盘信息
KERNEL=="dm-*",ENV{DM_UUID}=="mpath-14f504e46494c450068306d5534302d6e6a4b782d694a4741", ACTION=="add|change", OWNER:="grid", GROUP:="asmadmin", MODE:="660", SYMLINK+="asm-data04"
KERNEL=="dm-*",ENV{DM_UUID}=="mpath-14f504e46494c4500336d586655422d77766e312d56507641", ACTION=="add|change", OWNER:="grid", GROUP:="asmadmin", MODE:="660", SYMLINK+="asm-fra02"
重载udev
[root@rac01 ~]# /sbin/udevadm control --reload-rules
[root@rac01 ~]# /sbin/udevadm trigger --type=devices --action=change
验证磁盘属主和权限
[root@rac01 ~]# ls -l /dev/dm*
brw-rw---- 1 root disk 253, 0 May 29 17:33 /dev/dm-0
brw-rw---- 1 root disk 253, 1 May 29 17:33 /dev/dm-1
brw-rw---- 1 grid asmadmin 253, 10 May 29 17:33 /dev/dm-10
brw-rw---- 1 grid asmadmin 253, 11 May 29 17:33 /dev/dm-11
brw-rw---- 1 root disk 253, 2 May 29 17:33 /dev/dm-2
brw-rw---- 1 grid asmadmin 253, 3 May 29 17:33 /dev/dm-3
brw-rw---- 1 grid asmadmin 253, 4 May 29 17:33 /dev/dm-4
brw-rw---- 1 grid asmadmin 253, 5 May 29 17:33 /dev/dm-5
brw-rw---- 1 grid asmadmin 253, 6 May 29 17:33 /dev/dm-6
brw-rw---- 1 grid asmadmin 253, 7 May 29 17:33 /dev/dm-7
brw-rw---- 1 grid asmadmin 253, 8 May 29 17:33 /dev/dm-8
brw-rw---- 1 grid asmadmin 253, 9 May 29 17:33 /dev/dm-9
添加新磁盘,节点1操作即可
su - grid
[rac01:grid]:/home/grid>sqlplus / as sysasm
查看rebalance的最大权值
SQL> show parameter limit;
NAME TYPE VALUE
------------------------------------ --------------------------------- ------------------------------
asm_power_limit integer 1
最大权值取值范围0-11,0代表不做rebalance ,上限设置的越高消耗的CPU和IO资源越多,rebalance过程越快。一般为4,可以动态调整,调整指令如下
SQL> alter system set asm_power_limit=4 scope=memory sid='*';
System altered.
SQL> show parameter limit;
NAME TYPE VALUE
------------------------------------ --------------------------------- ------------------------------
asm_power_limit integer 4
注意:这个值在不同的RAC节点可以设置不同的值,例如1节点设置8,2节点设置4
查看ASM实例的SGA大小
SQL> show parameter memory_target
NAME TYPE VALUE
------------------------------------ --------------------------------- ------------------------------
memory_target big integer 1076M
SQL> show parameter sga_max_size;
NAME TYPE VALUE
------------------------------------ --------------------------------- ------------------------------
sga_max_size big integer 1088M
SQL> show parameter memory_max_target
NAME TYPE VALUE
------------------------------------ --------------------------------- ------------------------------
memory_max_target big integer 1076M
ASM实例的SGA大小影响rebalance的速度,建议生产环境至少设置为2G;修改SGA为静态参数,需要重启ASM实例后生效
alter system set memory_max_target = 2G scope = spfile sid = '*';
alter system set memory_target = 2G scope = spfile sid = '*';
alter system set sga_max_size = 2G scope = spfile sid = '*';
查询磁盘组挂载状态,closed为新加的盘
set line 200
set pagesize 200
col path for a50
SQL> select mount_status, path, os_mb from v$asm_disk order by 2;
MOUNT_STATUS PATH OS_MB
--------------------- -------------------------------------------------- ----------
CACHED /dev/asm-data01 8192
CACHED /dev/asm-data02 8192
CACHED /dev/asm-data03 8192
CLOSED /dev/asm-data04 8192
CACHED /dev/asm-fra01 8192
CLOSED /dev/asm-fra02 8192
CACHED /dev/asm-ocr1 3072
CACHED /dev/asm-ocr2 3072
CACHED /dev/asm-ocr3 3072
查询磁盘组总容量和剩余容量,单位GB
SQL> select name, round(total_mb/1024, 2) "TotalGB", round(free_mb/1024, 2) "FreeGB" from v$asm_diskgroup;
NAME TotalGB FreeGB
----- ---------- ----------
DATA 24 20
FRA 8 7.04
OCR 9 8.11
执行加盘操作,多块磁盘添加时用逗号间隔。示例:alter diskgroup DATA add disk '/dev/asm-data04','/dev/asm-data05';
alter diskgroup DATA add disk '/dev/asm-data04';
alter diskgroup FRA add disk '/dev/asm-fra02';
查询结果
SQL> select name, round(total_mb/1024, 2) "TotalGB", round(free_mb/1024, 2) "FreeGB" from v$asm_diskgroup;
NAME TotalGB FreeGB
---------- ---------- ----------
DATA 32 27.98
FRA 16 15.03
OCR 9 8.11