[root@rac2 ~]# crsctl stat res -t -init
--------------------------------------------------------------------------------
Name Target State Server State details
--------------------------------------------------------------------------------
Cluster Resources
--------------------------------------------------------------------------------
ora.asm
1 ONLINE ONLINE rac2 Started,STABLE
ora.cluster_interconnect.haip
1 ONLINE ONLINE rac2 STABLE
ora.crf
1 ONLINE ONLINE rac2 STABLE
ora.crsd
1 ONLINE OFFLINE STABLE
ora.cssd
1 ONLINE ONLINE rac2 STABLE
ora.cssdmonitor
1 ONLINE ONLINE rac2 STABLE
ora.ctssd
1 ONLINE ONLINE rac2 OBSERVER,STABLE
ora.diskmon
1 OFFLINE OFFLINE STABLE
ora.evmd
1 ONLINE ONLINE rac2 STABLE
ora.gipcd
1 ONLINE ONLINE rac2 STABLE
ora.gpnpd
1 ONLINE ONLINE rac2 STABLE
ora.mdnsd
1 ONLINE ONLINE rac2 STABLE
ora.storage
1 ONLINE OFFLINE rac2 STARTING
--------------------------------------------------------------------------------
[root@rac2 ~]# crsctl status res -t
CRS-4535: Cannot communicate with Cluster Ready Services
CRS-4000: Command Status failed, or completed with errors.
[root@rac2 ~]#
[root@rac2 trace]# tail -f al*log
2022-09-21 04:15:59.726 [OCTSSD(22788)]CRS-8500: Oracle Clusterware OCTSSD process is starting with operating system process ID 22788
2022-09-21 04:16:00.284 [OLOGGERD(22808)]CRS-8500: Oracle Clusterware OLOGGERD process is starting with operating system process ID 22808
2022-09-21 04:16:00.415 [OCTSSD(22788)]CRS-2403: The Cluster Time Synchronization Service on host rac2 is in observer mode.
2022-09-21 04:16:02.347 [OCTSSD(22788)]CRS-2401: The Cluster Time Synchronization Service started on host rac2.
2022-09-21 04:16:02.347 [OCTSSD(22788)]CRS-2407: The new Cluster Time Synchronization Service reference node is host rac2.
2022-09-21 04:16:17.939 [ORAROOTAGENT(22218)]CRS-5019: All OCR locations are on ASM disk groups [DATA], and none of these disk groups are mounted. Details are at "(:CLSN00140:)" in "/u01/app/grid/diag/crs/rac2/crs/trace/ohasd_orarootagent_root.trc".
2022-09-21 04:26:13.756 [ORAROOTAGENT(22218)]CRS-5818: Aborted command 'start' for resource 'ora.storage'. Details at (:CRSAGF00113:) {0:5:3} in /u01/app/grid/diag/crs/rac2/crs/trace/ohasd_orarootagent_root.trc.
2022-09-21 04:26:14.699 [ORAROOTAGENT(22218)]CRS-5017: The resource action "ora.storage start" encountered the following error:
2022-09-21 04:26:14.699+Storage::start action aborted. For details refer to "(:CLSN00107:)" in "/u01/app/grid/diag/crs/rac2/crs/trace/ohasd_orarootagent_root.trc".
2022-09-21 04:26:14.703 [OHASD(22062)]CRS-2757: Command 'Start' timed out waiting for response from the resource 'ora.storage'. Details at (:CRSPE00221:) {0:5:3} in /u01/app/grid/diag/crs/rac2/crs/trace/ohasd.trc.
q
^C
[root@rac2 trace]# pwd
/u01/app/grid/diag/crs/rac2/crs/trace
[root@rac2 trace]#
[root@rac2 trace]# vi /u01/app/grid/diag/crs/rac2/crs/trace/ohasd_orarootagent_root.trc
2022-09-21 04:34:21.173 :CLSDYNAM:1470088960: [ora.storage]{0:5:3} [check] (:CLSN00140:)StorageAgent::check 300 parsekgforretcodes OCR dgName DATA state 1
2022-09-21 04:34:21.174 :CLSDYNAM:1470088960: [ora.storage]{0:5:3} [check] (:CLSN00109:) Agent::commonCheck check failed action:0104 retval:1
2022-09-21 04:34:21.174 :CLSDYNAM:1470088960: [ora.storage]{0:5:3} [check] Agent::doStateDump Default Agent Dump
2022-09-21 04:34:21.174 :CLSDYNAM:1470088960: [ora.storage]{0:5:3} [check] Agent::doStateDump last call info:
2022-09-21 04:34:21.174 :CLSDYNAM:1470088960: [ora.storage]{0:5:3} [check] Time:09/21/2022 04:34:20.989 Tint:{0:5:3} action:104 resname:ora.storage lastCall:Agent::commonCheck call check
2022-09-21 04:34:21.174 :CLSDYNAM:1470088960: [ora.storage]{0:5:3} [check] Time:09/21/2022 04:34:20.989 Tint:{0:5:3} action:104 resname:ora.storage lastCall:StorageAgent::check 000 entry {
2022-09-21 04:34:21.174 :CLSDYNAM:1470088960: [ora.storage]{0:5:3} [check] Time:09/21/2022 04:34:20.989 Tint:{0:5:3} action:104 resname:ora.storage lastCall:StorageAgent::check 100 targetcheck
2022-09-21 04:34:21.174 :CLSDYNAM:1470088960: [ora.storage]{0:5:3} [check] Time:09/21/2022 04:34:21.132 Tint:{0:5:3} action:104 resname:ora.storage lastCall:Agent::getPrevAction prevAction:104
2022-09-21 04:34:21.174 :CLSDYNAM:1470088960: [ora.storage]{0:5:3} [check] Time:09/21/2022 04:34:21.132 Tint:{0:5:3} action:104 resname:ora.storage lastCall:Agent::getPrevAction prevAction:104
2022-09-21 04:34:21.174 :CLSDYNAM:1470088960: [ora.storage]{0:5:3} [check] Time:09/21/2022 04:34:21.159 Tint:{0:5:3} action:104 resname:ora.storage lastCall:StorageAgent::check 101 targetcheck target:0
2022-09-21 04:34:21.174 :CLSDYNAM:1470088960: [ora.storage]{0:5:3} [check] Time:09/21/2022 04:34:21.159 Tint:{0:5:3} action:104 resname:ora.storage lastCall:StorageAgent::check 160 NODEROLE_HUB getOCRdetails
2022-09-21 04:34:21.174 :CLSDYNAM:1470088960: [ora.storage]{0:5:3} [check] Time:09/21/2022 04:34:21.159 Tint:{0:5:3} action:104 resname:ora.storage lastCall:StorageAgent::check 180 storage type:0
2022-09-21 04:34:21.174 :CLSDYNAM:1470088960: [ora.storage]{0:5:3} [check] Time:09/21/2022 04:34:21.167 Tint:{0:5:3} action:104 resname:ora.storage lastCall:StorageAgent::parsekgforetcodes Calling kgfoCheckMount DATA
2022-09-21 04:34:21.174 :CLSDYNAM:1470088960: [ora.storage]{0:5:3} [check] Time:09/21/2022 04:34:21.173 Tint:{0:5:3} action:104 resname:ora.storage lastCall:StorageAgent::parsekgforetcodes retcode = 6, kgfoCheckMount(DATA), flag 4
2022-09-21 04:34:21.174 :CLSDYNAM:1470088960: [ora.storage]{0:5:3} [check] Time:09/21/2022 04:34:21.173 Tint:{0:5:3} action:104 resname:ora.storage lastCall:StorageAgent::check 260 kgfo returncode 1
2022-09-21 04:34:21.174 :CLSDYNAM:1470088960: [ora.storage]{0:5:3} [check] Time:09/21/2022 04:34:21.173 Tint:{0:5:3} action:104 resname:ora.storage lastCall:(:CLSN00140:)StorageAgent::check 300 parsekgforretcodes OCR dgName DATA state 1
2022-09-21 04:34:21.174 :CLSDYNAM:1470088960: [ora.storage]{0:5:3} [check] Time:09/21/2022 04:34:21.174 Tint:{0:5:3} action:104 resname:ora.storage lastCall:StorageAgent::check 999 exit clsagfw_res_status:1
2022-09-21 04:34:21.174 :CLSDYNAM:1470088960: [ora.storage]{0:5:3} [check] Time:09/21/2022 04:34:21.174 Tint:{0:5:3} action:104 resname:ora.storage lastCall:Agent::commonCheck check clsagfw_res_status:1
2022-09-21 04:34:21.174 :CLSDYNAM:1470088960: [ora.storage]{0:5:3} [check] Time:09/21/2022 04:34:21.174 Tint:{0:5:3} action:104 resname:ora.storage lastCall:Agent::getPrevAction prevAction:104
2022-09-21 04:34:21.174 :CLSDYNAM:1470088960: [ora.storage]{0:5:3} [check] Time:09/21/2022 04:34:21.174 Tint:{0:5:3} action:104 resname:ora.storage lastCall:Agent::getPrevAction prevAction:104
2022-09-21 04:34:21.174 :CLSDYNAM:1470088960: [ora.storage]{0:5:3} [check] Time:09/21/2022 04:34:21.174 Tint:{0:5:3} action:104 resname:ora.storage lastCall:Agent::getPrevAction prevAction:104
2022-09-21 04:34:21.174 :CLSDYNAM:1470088960: [ora.storage]{0:5:3} [check] Time:09/21/2022 04:34:21.174 Tint:{0:5:3} action:104 resname:ora.storage lastCall:Agent::getPrevAction prevAction:104
2022-09-21 04:34:21.174 :CLSDYNAM:1470088960: [ora.storage]{0:5:3} [check] Time:09/21/2022 04:34:21.174 Tint:{0:5:3} action:104 resname:ora.storage lastCall:Agent::getPrevAction prevAction:104
2022-09-21 04:34:21.174 :CLSDYNAM:1470088960: [ora.storage]{0:5:3} [check] Time:09/21/2022 04:34:21.174 Tint:{0:5:3} action:104 resname:ora.storage lastCall:(:CLSN00109:) Agent::commonCheck check failed action:0104 retval:1
2022-09-21 04:34:32.579 :CLSDYNAM:1470088960: [ora.ctssd]{0:5:3} [check] translateReturnCodes, return = 0, state detail = OBSERVERCheckcb data [0x7f4e500827e0]: mode[0xee] offset[0 ms].
2022-09-21 04:34:32.606 : AGFW:1467987712: [ INFO] {0:0:2} Agent received the message: AGENT_HB[Engine] ID 12293:4270
2022-09-21 04:34:37.621 : USRTHRD:1470088960: [ INFO] {0:5:3} Thread:[NetHAMain] Check: 0-1
2022-09-21 04:34:46.163 :CLSDYNAM:1470088960: [ ora.crf]{0:5:3} [check] Check return = 0, state detail = NULL
[root@rac2 trace]# vi /u01/app/grid/diag/crs/rac2/crs/trace/ohasd.trc
022-09-21 04:34:10.545 :UiServer:3149911808: [ INFO] {0:0:1812} Sending to PE. ctx= 0x7fd3d0204ab0, ClientPID=22808 set Properties (root,1564659)
2022-09-21 04:34:10.545 : CRSPE:3425507072: [ INFO] {0:0:1812} Processing PE command id=1836 origin:rac2. Description: [Stat Resource : 0x7fd3c82337c0]
2022-09-21 04:34:10.555 :UiServer:3149911808: [ INFO] {0:0:1812} Done for ctx=0x7fd3d0204ab0
2022-09-21 04:34:10.743 :UiServer:3149911808: [ INFO] {0:0:1813} Sending to PE. ctx= 0x7fd3d0205390, ClientPID=22808 set Properties (root,1565155)
2022-09-21 04:34:10.744 : CRSPE:3425507072: [ INFO] {0:0:1813} Processing PE command id=1837 origin:rac2. Description: [Stat Resource : 0x7fd3c815e1e0]
2022-09-21 04:34:10.747 :UiServer:3149911808: [ INFO] {0:0:1813} Done for ctx=0x7fd3d0205390
2022-09-21 04:34:14.287 :GIPCHTHR:3145709312: gipchaDaemonWork: DaemonThread heart beat, time interval since last heartBeat 31090loopCount 28
2022-09-21 08:08:21.735 [OCSSD(19422)]CRS-1720: Cluster Synchronization Services daemon (CSSD) is ready for operation.
2022-09-21 08:08:22.070 [OCTSSD(19677)]CRS-8500: Oracle Clusterware OCTSSD process is starting with operating system process ID 19677
2022-09-21 08:08:22.616 [OCTSSD(19677)]CRS-2403: The Cluster Time Synchronization Service on host rac2 is in observer mode.
2022-09-21 08:08:24.816 [OCTSSD(19677)]CRS-2401: The Cluster Time Synchronization Service started on host rac2.
2022-09-21 08:08:24.816 [OCTSSD(19677)]CRS-2407: The new Cluster Time Synchronization Service reference node is host rac2.
2022-09-21 08:08:40.250 [ORAROOTAGENT(19094)]CRS-5019: All OCR locations are on ASM disk groups [DATA], and none of these disk groups are mounted. Details are at "(:CLSN00140:)" in "/u01/app/grid/diag/crs/rac2/crs/trace/ohasd_orarootagent_root.trc".
vi /u01/app/grid/diag/crs/rac2/crs/trace/ohasd_orarootagent_root.trc
022-09-21 08:13:07.917 : USRTHRD:3035715328: [ INFO] {0:5:3} Thread:kgfoCheckMountExt isRunning is reset to false here
2022-09-21 08:13:07.917 : USRTHRD:3035715328: [ INFO] {0:5:3} Thread:kgfoCheckMountExt isFinished set to true
2022-09-21 08:13:07.917 : USRTHRD:3046221568: [ INFO] {0:5:3} Thread:kgfoCheckMountExt Tasklet::doTask m_cv.timewait returned
2022-09-21 08:13:07.917 : USRTHRD:3046221568: [ INFO] {0:5:3} Thread:kgfoCheckMountExt Tasklet::doTask(kgfoCheckMountExt) executed in 7 seconds
2022-09-21 08:13:07.917 :CLSDYNAM:3046221568: [ora.storage]{0:5:3} [start] StorageAgent::parsekgforetcodes retcode = 7, kgfoCheckMount(DATA), flag 2
2022-09-21 08:13:07.917 :CLSDYNAM:3046221568: [ora.storage]{0:5:3} [start] (null) category: 7, operation: kgfoAl06, loc: kgfokge, OS error: 15077, other: ORA-15077: could not locate ASM instance serving a required diskgroup
2022-09-21 08:13:07.917 :CLSDYNAM:3046221568: [ora.storage]{0:5:3} [start] StorageAgent::check 260 kgfo returncode 1
2022-09-21 08:13:07.917 :CLSDYNAM:3046221568: [ora.storage]{0:5:3} [start] (:CLSN00140:)StorageAgent::check 300 parsekgforretcodes OCR dgName DATA state 1
2022-09-21 08:13:07.918 :CLSDYNAM:3046221568: [ora.storage]{0:5:3} [start] Storage::start waiting for check to not return PARTIAL or UNPLANNED_OFFLINE 1
SQL> select *from dual;
ADDR INDX INST_ID CON_ID D
---------------- ---------- ---------- ---------- -
000000001350C49C 0 2 0 X
---sqlnet.ora 错了启动不了,启动后打开也没有用。要重启
SQL> exit
Disconnected from Oracle Database 19c Enterprise Edition Release 19.0.0.0.0 - Production
Version 19.3.0.0.0
[grid@rac2 trace]$ crsctl stat res -t
CRS-4535: Cannot communicate with Cluster Ready Services
CRS-4000: Command Status failed, or completed with errors.
[grid@rac2 trace]$ crsctl stat res -t
CRS-4535: Cannot communicate with Cluster Ready Services
CRS-4000: Command Status failed, or completed with errors.
[grid@rac2 trace]$ crsctl stat res -t -init
--------------------------------------------------------------------------------
Name Target State Server State details
--------------------------------------------------------------------------------
Cluster Resources
--------------------------------------------------------------------------------
ora.asm
1 ONLINE ONLINE rac2 Started,STABLE
ora.cluster_interconnect.haip
1 ONLINE ONLINE rac2 STABLE
ora.crf
1 ONLINE ONLINE rac2 STABLE
ora.crsd
1 ONLINE OFFLINE STABLE
ora.cssd
1 ONLINE ONLINE rac2 STABLE
ora.cssdmonitor
1 ONLINE ONLINE rac2 STABLE
ora.ctssd
1 ONLINE ONLINE rac2 OBSERVER,STABLE
ora.diskmon
1 OFFLINE OFFLINE STABLE
ora.evmd
1 ONLINE ONLINE rac2 STABLE
ora.gipcd
1 ONLINE ONLINE rac2 STABLE
ora.gpnpd
1 ONLINE ONLINE rac2 STABLE
ora.mdnsd
1 ONLINE ONLINE rac2 STABLE
ora.storage
1 ONLINE OFFLINE rac2 STARTING
--------------------------------------------------------------------------------
[grid@rac2 trace]$
--------------正常情况
[root@rac2 ~]# su - oracle
Last login: Wed Sep 21 20:05:32 CST 2022 on pts/2
[oracle@rac2 ~]$ sqlplus '/as sysdba'
SQL*Plus: Release 19.0.0.0.0 - Production on Wed Sep 21 20:23:43 2022
Version 19.3.0.0.0
Copyright (c) 1982, 2019, Oracle. All rights reserved.
Connected to:
Oracle Database 19c Enterprise Edition Release 19.0.0.0.0 - Production
Version 19.3.0.0.0
SQL> exit
Disconnected from Oracle Database 19c Enterprise Edition Release 19.0.0.0.0 - Production
Version 19.3.0.0.0
[oracle@rac2 ~]$ exit
logout
[root@rac2 ~]# su - grid
Last login: Wed Sep 21 20:22:05 CST 2022
[grid@rac2 ~]$ sqlplus '/as sysdba'
SQL*Plus: Release 19.0.0.0.0 - Production on Wed Sep 21 20:23:55 2022
Version 19.3.0.0.0
Copyright (c) 1982, 2019, Oracle. All rights reserved.
Connected to:
Oracle Database 19c Enterprise Edition Release 19.0.0.0.0 - Production
Version 19.3.0.0.0
SQL> select 8 from dual;
8
----------
8
SQL> select *from dual;
ADDR INDX INST_ID CON_ID D
---------------- ---------- ---------- ---------- -
000000001350C49C 0 2 0 X
SQL> exit
Disconnected from Oracle Database 19c Enterprise Edition Release 19.0.0.0.0 - Production
Version 19.3.0.0.0
[grid@rac2 ~]$ crsctl stat res -t
--------------------------------------------------------------------------------
Name Target State Server State details
--------------------------------------------------------------------------------
Local Resources
--------------------------------------------------------------------------------
ora.LISTENER.lsnr
ONLINE ONLINE rac2 STABLE
ora.chad
ONLINE ONLINE rac2 STABLE
ora.net1.network
ONLINE ONLINE rac2 STABLE
ora.ons
ONLINE ONLINE rac2 STABLE
--------------------------------------------------------------------------------
Cluster Resources
--------------------------------------------------------------------------------
ora.ASMNET1LSNR_ASM.lsnr(ora.asmgroup)
1 ONLINE OFFLINE STABLE
2 ONLINE ONLINE rac2 STABLE
3 ONLINE OFFLINE STABLE
ora.DATA.dg(ora.asmgroup)
1 OFFLINE OFFLINE STABLE
2 ONLINE ONLINE rac2 STABLE
3 OFFLINE OFFLINE STABLE
ora.LISTENER_SCAN1.lsnr
1 ONLINE ONLINE rac2 STABLE
ora.asm(ora.asmgroup)
1 ONLINE OFFLINE STABLE
2 ONLINE ONLINE rac2 Started,STABLE
3 OFFLINE OFFLINE STABLE
ora.asmnet1.asmnetwork(ora.asmgroup)
1 ONLINE OFFLINE STABLE
2 ONLINE ONLINE rac2 STABLE
3 OFFLINE OFFLINE STABLE
ora.cvu
1 ONLINE ONLINE rac2 STABLE
ora.orcl.db
1 ONLINE OFFLINE STABLE
2 ONLINE ONLINE rac2 Open,HOME=/u01/app/o
racle/product/19.0.0
/db_1,STABLE
ora.orcl.srv_single.svc
1 ONLINE ONLINE rac2 STABLE
ora.qosmserver
1 ONLINE OFFLINE STABLE
ora.rac1.vip
1 ONLINE INTERMEDIATE rac2 FAILED OVER,STABLE
ora.rac2.vip
1 ONLINE ONLINE rac2 STABLE
ora.scan1.vip
1 ONLINE ONLINE rac2 STABLE
--------------------------------------------------------------------------------
[grid@rac2 ~]$ crsctl stat res -t -init
--------------------------------------------------------------------------------
Name Target State Server State details
--------------------------------------------------------------------------------
Cluster Resources
--------------------------------------------------------------------------------
ora.asm
1 ONLINE ONLINE rac2 Started,STABLE
ora.cluster_interconnect.haip
1 ONLINE ONLINE rac2 STABLE
ora.crf
1 ONLINE ONLINE rac2 STABLE
ora.crsd
1 ONLINE ONLINE rac2 STABLE
ora.cssd
1 ONLINE ONLINE rac2 STABLE
ora.cssdmonitor
1 ONLINE ONLINE rac2 STABLE
ora.ctssd
1 ONLINE ONLINE rac2 OBSERVER,STABLE
ora.diskmon
1 OFFLINE OFFLINE STABLE
ora.evmd
1 ONLINE ONLINE rac2 STABLE
ora.gipcd
1 ONLINE ONLINE rac2 STABLE
ora.gpnpd
1 ONLINE ONLINE rac2 STABLE
ora.mdnsd
1 ONLINE ONLINE rac2 STABLE
ora.storage
1 ONLINE ONLINE rac2 STABLE
--------------------------------------------------------------------------------
[grid@rac2 ~]$ lsnrctl status
LSNRCTL for Linux: Version 19.0.0.0.0 - Production on 21-SEP-2022 20:24:38
Copyright (c) 1991, 2019, Oracle. All rights reserved.
Connecting to (DESCRIPTION=(ADDRESS=(PROTOCOL=IPC)(KEY=LISTENER)))
STATUS of the LISTENER
------------------------
Alias LISTENER
Version TNSLSNR for Linux: Version 19.0.0.0.0 - Production
Start Date 21-SEP-2022 08:21:05
Uptime 0 days 0 hr. 3 min. 33 sec
Trace Level off
Security ON: Local OS Authentication
SNMP OFF
Listener Parameter File /u01/app/19.0.0/grid/network/admin/listener.ora
Listener Log File /u01/app/grid/diag/tnslsnr/rac2/listener/alert/log.xml
Listening Endpoints Summary...
(DESCRIPTION=(ADDRESS=(PROTOCOL=ipc)(KEY=LISTENER)))
(DESCRIPTION=(ADDRESS=(PROTOCOL=tcp)(HOST=192.168.1.110)(PORT=1521)))
(DESCRIPTION=(ADDRESS=(PROTOCOL=tcp)(HOST=192.168.1.111)(PORT=1521)))
(DESCRIPTION=(ADDRESS=(PROTOCOL=tcps)(HOST=rac2)(PORT=5500))(Security=(my_wallet_directory=/u01/app/oracle/product/19.0.0/db_1/admin/orcl/xdb_wallet))(Presentation=HTTP)(Session=RAW))
Services Summary...
Service "+ASM" has 1 instance(s).
Instance "+ASM2", status READY, has 1 handler(s) for this service...
Service "+ASM_DATA" has 1 instance(s).
Instance "+ASM2", status READY, has 1 handler(s) for this service...
Service "orcl" has 1 instance(s).
Instance "orcl2", status READY, has 1 handler(s) for this service...
Service "orclXDB" has 1 instance(s).
Instance "orcl2", status READY, has 1 handler(s) for this service...
Service "srv_single" has 1 instance(s).
Instance "orcl2", status READY, has 1 handler(s) for this service...
The command completed successfully
[grid@rac2 ~]$