11g rac启动失败一例

[root@rac1 ~]# /opt/app/grid/product/11.2.0/bin/crsctl start crs
CRS-4640: Oracle High Availability Services is already active
CRS-4000: Command Start failed, or completed with errors.

[grid@rac1 ~]$ srvctl start database -d rac
PRCD-1027 : Failed to retrieve database rac
PRCR-1070 : Failed to check if resource ora.rac.db is registered
Cannot communicate with crsd

[grid@rac1 ~]$ ps aux | grep ora_
grid     12801  0.0  0.0   3920   660 pts/0    R+   17:30   0:00 grep ora_

[grid@rac1 ~]$ crsctl check crs
CRS-4639: Could not contact Oracle High Availability Services

[grid@rac1 ~]$ crs_stat -t
CRS-0184: Cannot communicate with the CRS daemon.

This command is deprecated and has been replaced by 'crsctl status resource'
This command remains for backward compatibility only
Usage:  crs_stat [resource_name [...]] [-v] [-l] [-q] [-c cluster_member]
        crs_stat [resource_name [...]] -t [-v] [-q] [-c cluster_member]
        crs_stat -p [resource_name [...]] [-q]
        crs_stat [-a] application -g
        crs_stat [-a] application -r [-c cluster_member]
        crs_stat -f [resource_name [...]] [-q] [-c cluster_member]
        crs_stat -ls [resource_name [...]] [-q]

+++检查发现，原来是磁盘空间满了
ERROR:
ORA-09817: Write to audit file failed.
Linux Error: 28: No space left on device
Additional information: 12
ORA-09945: Unable to initialize the audit trail file
Linux Error: 28: No space left on device

+++清理磁盘后恢复正常
[grid@rac1 ~]$ crsctl check cluster
CRS-4537: Cluster Ready Services is online
CRS-4529: Cluster Synchronization Services is online
CRS-4533: Event Manager is online

[grid@rac1 ~]$ crsctl check crs
CRS-4638: Oracle High Availability Services is online
CRS-4537: Cluster Ready Services is online
CRS-4529: Cluster Synchronization Services is online
CRS-4533: Event Manager is online

+++节点rac1没有起来，手工启一下
[grid@rac1 ~]$ srvctl status database -d rac
Instance rac1 is not running on node rac1
Instance rac2 is running on node rac2
[grid@rac1 ~]$ srvctl start instance -d rac -i rac1
[grid@rac1 ~]$ 
[grid@rac1 ~]$ 
[grid@rac1 ~]$ srvctl status database -d rac       
Instance rac1 is running on node rac1
Instance rac2 is running on node rac2

[grid@rac1 ~]$ srvctl status listener
Listener LISTENER is enabled
Listener LISTENER is running on node(s): rac1,rac2
[grid@rac1 ~]$ srvctl status scan
SCAN VIP scan1 is enabled
SCAN VIP scan1 is running on node rac2
SCAN VIP scan2 is enabled
SCAN VIP scan2 is running on node rac1

+++检查crs资源
[grid@rac1 ~]$ crs_stat -t
Name           Type           Target    State     Host        
------------------------------------------------------------
ora.CRS.dg     ora....up.type ONLINE    ONLINE    rac1        
ora.DATA.dg    ora....up.type ONLINE    ONLINE    rac1        
ora....ER.lsnr ora....er.type ONLINE    ONLINE    rac1        
ora....N1.lsnr ora....er.type ONLINE    ONLINE    rac2        
ora....N2.lsnr ora....er.type ONLINE    ONLINE    rac1        
ora.asm        ora.asm.type   ONLINE    ONLINE    rac1        
ora.eons       ora.eons.type  ONLINE    ONLINE    rac1        
ora.gsd        ora.gsd.type   OFFLINE   OFFLINE               
ora....network ora....rk.type ONLINE    ONLINE    rac1        
ora.oc4j       ora.oc4j.type  OFFLINE   OFFLINE               
ora.ons        ora.ons.type   ONLINE    ONLINE    rac1        
ora.rac.db     ora....se.type ONLINE    ONLINE    rac1        
ora....cs1.svc ora....ce.type ONLINE    UNKNOWN   rac2        
ora....cs2.svc ora....ce.type ONLINE    ONLINE    rac2        
ora....SM1.asm application    ONLINE    ONLINE    rac1        
ora....C1.lsnr application    ONLINE    ONLINE    rac1        
ora.rac1.gsd   application    OFFLINE   OFFLINE               
ora.rac1.ons   application    ONLINE    ONLINE    rac1        
ora.rac1.vip   ora....t1.type ONLINE    ONLINE    rac1        
ora....SM2.asm application    ONLINE    ONLINE    rac2        
ora....C2.lsnr application    ONLINE    ONLINE    rac2        
ora.rac2.gsd   application    OFFLINE   OFFLINE               
ora.rac2.ons   application    ONLINE    ONLINE    rac2        
ora.rac2.vip   ora....t1.type ONLINE    ONLINE    rac2        
ora....ry.acfs ora....fs.type ONLINE    ONLINE    rac1        
ora.scan1.vip  ora....ip.type ONLINE    ONLINE    rac2        
ora.scan2.vip  ora....ip.type ONLINE    ONLINE    rac1        

[grid@rac1 ~]$ crs_stat
。。。
NAME=ora.rac.racs1.svc
TYPE=ora.service.type
TARGET=ONLINE
STATE=UNKNOWN on rac2
。。。

+++发现ora....cs1.svc处于UNKNOWN状态
手工起一下
[grid@rac1 ~]$ crs_stop ora.rac.racs1.svc
Attempting to stop `ora.rac.racs1.svc` on member `rac2`
ORA-44304: service racs1 does not exist
ORA-06512: at "SYS.DBMS_SYS_ERROR", line 86
ORA-06512: at "SYS.DBMS_SERVICE", line 443
ORA-06512: at "SYS.DBMS_SERVICE", line 391
ORA-06512: at line 1

Stop of `ora.rac.racs1.svc` on member `rac2` succeeded.
[grid@rac1 ~]$ crs_start ora.rac.racs1.svc
Attempting to start `ora.rac.racs1.svc` on member `rac1`
Start of `ora.rac.racs1.svc` on member `rac1` succeeded.
[grid@rac1 ~]$ crs_stat -t
Name           Type           Target    State     Host        
------------------------------------------------------------
ora.CRS.dg     ora....up.type ONLINE    ONLINE    rac1        
ora.DATA.dg    ora....up.type ONLINE    ONLINE    rac1        
ora....ER.lsnr ora....er.type ONLINE    ONLINE    rac1        
ora....N1.lsnr ora....er.type ONLINE    ONLINE    rac2        
ora....N2.lsnr ora....er.type ONLINE    ONLINE    rac1        
ora.asm        ora.asm.type   ONLINE    ONLINE    rac1        
ora.eons       ora.eons.type  ONLINE    ONLINE    rac1        
ora.gsd        ora.gsd.type   OFFLINE   OFFLINE               
ora....network ora....rk.type ONLINE    ONLINE    rac1        
ora.oc4j       ora.oc4j.type  OFFLINE   OFFLINE               
ora.ons        ora.ons.type   ONLINE    ONLINE    rac1        
ora.rac.db     ora....se.type ONLINE    ONLINE    rac1        
ora....cs1.svc ora....ce.type ONLINE    ONLINE    rac1        
ora....cs2.svc ora....ce.type ONLINE    ONLINE    rac2        
ora....SM1.asm application    ONLINE    ONLINE    rac1        
ora....C1.lsnr application    ONLINE    ONLINE    rac1        
ora.rac1.gsd   application    OFFLINE   OFFLINE               
ora.rac1.ons   application    ONLINE    ONLINE    rac1        
ora.rac1.vip   ora....t1.type ONLINE    ONLINE    rac1        
ora....SM2.asm application    ONLINE    ONLINE    rac2        
ora....C2.lsnr application    ONLINE    ONLINE    rac2        
ora.rac2.gsd   application    OFFLINE   OFFLINE               
ora.rac2.ons   application    ONLINE    ONLINE    rac2        
ora.rac2.vip   ora....t1.type ONLINE    ONLINE    rac2        
ora....ry.acfs ora....fs.type ONLINE    ONLINE    rac1        
ora.scan1.vip  ora....ip.type ONLINE    ONLINE    rac2        
ora.scan2.vip  ora....ip.type ONLINE    ONLINE    rac1        

+++已经个恢复正常