案例描述:
同事在rac上扩asm磁盘,由于没有修改hdisk的reserve_policy属性,导致另外一个节点看不到磁盘,而能看见磁盘的节点也无法使用节点。
节点1
Last unsuccessful login: Mon Dec 5 19:47:44 2011 on ssh from 172.20.xxx.xxx
Last login: Mon Dec 5 20:52:51 2011 on /dev/pts/3 from 172.20.xxx.xxx
*******************************************************************************
* *
* *
* Welcome to AIX Version 6.1! *
* *
* *
* Please see the README file in /usr/lpp/bos for information pertinent to *
* this release of the AIX Operating System. *
* *
* *
*******************************************************************************
# sqlplus / as sysdba
ksh: sqlplus: not found
# su - oracle
$ srvctl stop instance -d calldb -i calldb1
$ srvctl stop nodeapps -n callcenter01
$ ps -ef | grep +ASM1
oracle 5046336 1 0 Jul 01 - 1:38 asm_psp0_+ASM1
oracle 5701852 1 0 Jul 01 - 60:49 asm_diag_+ASM1
oracle 8519878 1 0 Jul 01 - 73:38 asm_lmon_+ASM1
oracle 8585458 1 0 Jul 01 - 1:34 asm_smon_+ASM1
oracle 9240712 1 0 Jul 01 - 135:13 asm_lmd0_+ASM1
oracle 9306286 1 0 Jul 01 - 13:19 asm_pmon_+ASM1
oracle 9371768 1 0 Jul 01 - 1:58 asm_dbw0_+ASM1
oracle 9437290 1 0 Jul 01 - 4:27 asm_mman_+ASM1
oracle 9634004 1 0 Jul 01 - 57:25 asm_lms0_+ASM1
oracle 9830494 1 0 Jul 01 - 1:52 asm_rbal_+ASM1
oracle 9896080 1 0 Jul 01 - 3:09 asm_ckpt_+ASM1
oracle 9961524 1 0 Jul 01 - 6:31 asm_gmon_+ASM1
oracle 10092722 1 0 Jul 01 - 2:23 asm_lgwr_+ASM1
oracle 10158138 1 0 Jul 01 - 5:56 asm_lck0_+ASM1
oracle 10682438 1 0 Jul 01 - 0:01 asm_o000_+ASM1
$ ps -ef | grep +ASM1
oracle 5046336 1 0 Jul 01 - 1:38 asm_psp0_+ASM1
oracle 5701852 1 0 Jul 01 - 60:49 asm_diag_+ASM1
oracle 8519878 1 0 Jul 01 - 73:38 asm_lmon_+ASM1
oracle 8585458 1 0 Jul 01 - 1:34 asm_smon_+ASM1
oracle 9240712 1 0 Jul 01 - 135:13 asm_lmd0_+ASM1
oracle 9306286 1 0 Jul 01 - 13:19 asm_pmon_+ASM1
oracle 9371768 1 0 Jul 01 - 1:58 asm_dbw0_+ASM1
oracle 9437290 1 0 Jul 01 - 4:27 asm_mman_+ASM1
oracle 9634004 1 0 Jul 01 - 57:25 asm_lms0_+ASM1
oracle 9830494 1 0 Jul 01 - 1:52 asm_rbal_+ASM1
oracle 9896080 1 0 Jul 01 - 3:09 asm_ckpt_+ASM1
oracle 9961524 1 0 Jul 01 - 6:31 asm_gmon_+ASM1
oracle 10092722 1 0 Jul 01 - 2:23 asm_lgwr_+ASM1
oracle 10158138 1 0 Jul 01 - 5:56 asm_lck0_+ASM1
oracle 10682438 1 0 Jul 01 - 0:01 asm_o000_+ASM1
oracle 11075592 25296954 0 21:00:22 pts/4 0:00 grep +ASM1
$ srvctl stop asm -n callcenter01
$ ps -ef | grep +ASM1
$ ps -ef | grep oracle
oracle 6881366 8257594 0 Jul 01 - 41:59 /oracle/product/10g/crs/bin/oclsomon.bin
root 6946900 5439548 0 Jul 01 - 3:29 /oracle/product/10g/crs/bin/oprocd.bin run -t 1000 -m 500 -f
root 7208986 8060934 0 Jul 01 - 1047:19 /oracle/product/10g/crs/bin/crsd.bin reboot
oracle 7602386 25296954 0 21:01:23 pts/4 0:00 grep oracle
oracle 7929876 7471110 0 Jul 01 - 12:29 /oracle/product/10g/crs/bin/evmd.bin
oracle 8126582 8978514 0 Jul 01 - 295:13 /oracle/product/10g/crs/bin/ocssd.bin
oracle 8257594 9044004 0 Jul 01 - 0:00 /bin/sh -c cd /oracle/product/10g/crs/log/callcenter01/cssd/oclsomon; ulimit -c unlimited; /oracle/product/10g/crs/bin/oclsomon || exit $?
oracle 8388688 7929876 0 Jul 01 - 0:57 /oracle/product/10g/crs/bin/evmlogger.bin -o /oracle/product/10g/crs/evm/log/evmlogger.info -l /oracle/product/10g/crs/evm/log/evmlogger.log
oracle 8978514 8323118 0 Jul 01 - 0:00 /bin/sh -c ulimit -c unlimited; cd /oracle/product/10g/crs/log/callcenter01/cssd; /oracle/product/10g/crs/bin/ocssd || exit $?
oracle 11075636 25296954 4 21:01:23 pts/4 0:00 ps -ef
oracle 15990976 26345650 0 20:59:56 pts/5 0:00 tail -f alert_CALLDB1.log
oracle 25296954 15728770 0 20:57:09 pts/4 0:00 -ksh
oracle 26345650 26214538 0 20:58:20 pts/5 0:00 -ksh
$ srvctl start nodeapps -n callcenter01
$ ps -ef | grep +ASM1
$ ps -ef | grep +ASM1
$ ps -ef | grep +ASM1
$ srvctl start asm -n callcenter01
$ ps -ef | grep +ASM1
oracle 17039514 1 0 21:03:10 - 0:00 asm_smon_+ASM1
oracle 17498148 1 0 21:03:09 - 0:00 asm_lgwr_+ASM1
oracle 17694968 1 0 21:03:10 - 0:00 asm_ckpt_+ASM1
oracle 17957018 1 0 21:03:09 - 0:00 asm_dbw0_+ASM1
oracle 18219066 1 0 21:03:09 - 0:00 asm_mman_+ASM1
oracle 18284646 1 0 21:03:10 - 0:00 asm_rbal_+ASM1
oracle 18546892 1 0 21:03:11 - 0:00 asm_lck0_+ASM1
oracle 18612394 1 0 21:03:09 - 0:00 asm_lmd0_+ASM1
oracle 18677934 1 0 21:03:09 - 0:00 asm_lmon_+ASM1
oracle 18809076 1 0 21:03:09 - 0:00 asm_lms0_+ASM1
oracle 18874418 1 0 21:03:10 - 0:00 asm_gmon_+ASM1
oracle 19071072 1 0 21:03:09 - 0:00 asm_diag_+ASM1
oracle 19595276 1 0 21:03:09 - 0:00 asm_psp0_+ASM1
oracle 20250844 1 0 21:03:09 - 0:00 asm_pmon_+ASM1
$ srvctl start instance -d calldb -i calldb1
$ export ORACLE_SID=+ASM1
$ sqlplus / as sysdba
SQL*Plus: Release 10.2.0.4.0 - Production on Mon Dec 5 21:04:46 2011
Copyright (c) 1982, 2007, Oracle. All Rights Reserved.
Connected to:
Oracle Database 10g Enterprise Edition Release 10.2.0.4.0 - 64bit Production
With the Partitioning, Real Application Clusters, OLAP, Data Mining
and Real Application Testing options
SQL> desc v$asm_disk;
Name Null? Type
----------------------------------------- -------- ----------------------------
GROUP_NUMBER NUMBER
DISK_NUMBER NUMBER
COMPOUND_INDEX NUMBER
INCARNATION NUMBER
MOUNT_STATUS VARCHAR2(7)
HEADER_STATUS VARCHAR2(12)
MODE_STATUS VARCHAR2(7)
STATE VARCHAR2(8)
REDUNDANCY VARCHAR2(7)
LIBRARY VARCHAR2(64)
TOTAL_MB NUMBER
FREE_MB NUMBER
NAME VARCHAR2(30)
FAILGROUP VARCHAR2(30)
LABEL VARCHAR2(31)
PATH VARCHAR2(256)
UDID VARCHAR2(64)
PRODUCT VARCHAR2(32)
CREATE_DATE DATE
MOUNT_DATE DATE
REPAIR_TIMER NUMBER
READS NUMBER
WRITES NUMBER
READ_ERRS NUMBER
WRITE_ERRS NUMBER
READ_TIME NUMBER
WRITE_TIME NUMBER
BYTES_READ NUMBER
BYTES_WRITTEN NUMBER
SQL> select PATH,NAME from v$asm_disk
2 /
PATH
--------------------------------------------------------------------------------
NAME
--------------------------------------------------------------------------------
/dev/rhdisk4
/dev/rhdisk5
/dev/rhdisk6
PATH
--------------------------------------------------------------------------------
NAME
--------------------------------------------------------------------------------
/dev/rhdisk8
/dev/rhdisk7
CALLDG_0000
SQL> alter diskgroup CALLDG add disk '/dev/rhdisk8';
alter diskgroup CALLDG add disk '/dev/rhdisk8'
*
ERROR at line 1:
ORA-15032: not all alterations performed
ORA-15033: disk '/dev/rhdisk8' belongs to diskgroup "CALLDG"
SQL> alter diskgroup CALLDG add disk '/dev/rhdisk8';
alter diskgroup CALLDG add disk '/dev/rhdisk8'
*
ERROR at line 1:
ORA-15032: not all alterations performed
ORA-15033: disk '/dev/rhdisk8' belongs to diskgroup "CALLDG"
SQL> SELECT name, header_status, path FROM V$ASM_DISK
2 ;
NAME
--------------------------------------------------------------------------------
HEADER_STATUS
------------------------------------
PATH
--------------------------------------------------------------------------------
FOREIGN
/dev/rhdisk4
FOREIGN
/dev/rhdisk5
NAME
--------------------------------------------------------------------------------
HEADER_STATUS
------------------------------------
PATH
--------------------------------------------------------------------------------
FOREIGN
/dev/rhdisk6
MEMBER
NAME
--------------------------------------------------------------------------------
HEADER_STATUS
------------------------------------
PATH
--------------------------------------------------------------------------------
/dev/rhdisk8
CALLDG_0000
MEMBER
/dev/rhdisk7
SQL> col name for a30
SQL> col PATH for a30
SQL> /
NAME HEADER_STATUS
------------------------------ ------------------------------------
PATH
------------------------------
FOREIGN
/dev/rhdisk4
FOREIGN
/dev/rhdisk5
FOREIGN
/dev/rhdisk6
NAME HEADER_STATUS
------------------------------ ------------------------------------
PATH
------------------------------
MEMBER
/dev/rhdisk8
CALLDG_0000 MEMBER
/dev/rhdisk7
#### 这里发现 /dev/rhdisk8加入MEMBER但是NAME是空的。。。。
SQL> set linesize 999
SQL> /
NAME HEADER_STATUS PATH
------------------------------ ------------------------------------ ------------------------------
FOREIGN /dev/rhdisk4
FOREIGN /dev/rhdisk5
FOREIGN /dev/rhdisk6
MEMBER /dev/rhdisk8
CALLDG_0000 MEMBER /dev/rhdisk7
节点2
Last unsuccessful login: Mon Dec 5 18:38:37 2011 on /dev/pts/5 from
Last login: Mon Dec 5 20:28:18 2011 on /dev/pts/14 from
*******************************************************************************
* *
* *
* Welcome to AIX Version 6.1! *
* *
* *
* Please see the README file in /usr/lpp/bos for information pertinent to *
* this release of the AIX Operating System. *
* *
* *
*******************************************************************************
# su - oracle
$ srvctl stop instance -d calldb -i calldb2
$ srvctl stop asm -n callcenter02
$ srvctl start asm -n callcenter02
$ srvctl start instance -d calldb -i calldb2
$ lsattr -El hdisk8
PCM PCM/friend/hsvpcm Path Control Module False
PR_key_value none Persistant Reserve Key Value True
algorithm fail_over Algorithm True
hcheck_cmd test_unit_rdy Health Check Command True
hcheck_interval 60 Health Check Interval True
hcheck_mode nonactive Health Check Mode True
location Location Label True
lun_id 0x7000000000000 Logical Unit Number ID False
lun_reset_spt yes SCSI LUN reset True
max_transfer 0x40000 Maximum TRANSFER Size True
node_name 0x50014380025d6e30 FC Node Name False
pvid none Physical volume identifier False
q_type simple Queuing TYPE True
queue_depth 8 Queue DEPTH True
reserve_policy no_reserve Reserve Policy True
rw_timeout 60 READ/WRITE time out value True
scsi_id 0x10000 SCSI ID False
unique_id 34213600508B4000B3E5F000090000045000006HSV30002HPfcp Unique device identifier False
ww_name 0x50014380025d6e38 FC World Wide Name False
这里reserve_policy已经修改成 no_reserve
$ export ORACLE_SID=+ASM2
$ sqlplus / as sysdba
SQL*Plus: Release 10.2.0.4.0 - Production on Mon Dec 5 21:19:13 2011
Copyright (c) 1982, 2007, Oracle. All Rights Reserved.
Connected to:
Oracle Database 10g Enterprise Edition Release 10.2.0.4.0 - 64bit Production
With the Partitioning, Real Application Clusters, OLAP, Data Mining
and Real Application Testing options
SQL> SELECT name, header_status, path FROM V$ASM_DISK ;
NAME
--------------------------------------------------------------------------------
HEADER_STATUS
------------------------------------
PATH
--------------------------------------------------------------------------------
FOREIGN
/dev/rhdisk4
FOREIGN
/dev/rhdisk5
NAME
--------------------------------------------------------------------------------
HEADER_STATUS
------------------------------------
PATH
--------------------------------------------------------------------------------
FOREIGN
/dev/rhdisk6
MEMBER
NAME
--------------------------------------------------------------------------------
HEADER_STATUS
------------------------------------
PATH
--------------------------------------------------------------------------------
/dev/rhdisk8
CALLDG_0000
MEMBER
/dev/rhdisk7
SQL> set linesize 999
SQL> /
NAME HEADER_STATUS PATH
------------------------------------------------------------------------------------------ ------------------------------------ ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
FOREIGN /dev/rhdisk4
FOREIGN /dev/rhdisk5
FOREIGN /dev/rhdisk6
MEMBER /dev/rhdisk8
CALLDG_0000 MEMBER /dev/rhdisk7
SQL> desc v$asm_diskgroup;
Name Null? Type
---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -------- ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
GROUP_NUMBER NUMBER
NAME VARCHAR2(30)
SECTOR_SIZE NUMBER
BLOCK_SIZE NUMBER
ALLOCATION_UNIT_SIZE NUMBER
STATE VARCHAR2(11)
TYPE VARCHAR2(6)
TOTAL_MB NUMBER
FREE_MB NUMBER
REQUIRED_MIRROR_FREE_MB NUMBER
USABLE_FILE_MB NUMBER
OFFLINE_DISKS NUMBER
UNBALANCED VARCHAR2(1)
COMPATIBILITY VARCHAR2(60)
DATABASE_COMPATIBILITY VARCHAR2(60)
SQL> seelct
SP2-0042: unknown command "seelct" - rest of line ignored.
SQL>
SQL> select TOTAL_MB from v$asm_diskgroup;
TOTAL_MB
----------
204800
SQL> alter diskgroup CALLDG add disk '/dev/rhdisk8';
alter diskgroup CALLDG add disk '/dev/rhdisk8'
*
ERROR at line 1:
ORA-15032: not all alterations performed
ORA-15033: disk '/dev/rhdisk8' belongs to diskgroup "CALLDG"
加不进去,disk '/dev/rhdisk8' belongs to diskgroup "CALLDG",但实际SELECT name, header_status, path FROM V$ASM_DISK ;看不到name。。。
这时使用force命令:
SQL> alter diskgroup CALLDG add disk '/dev/rhdisk8' force;
Diskgroup altered.
SQL> select TOTAL_MB from v$asm_diskgroup;
TOTAL_MB
----------
409600
成功了。。。
SQL>