[grid@paycore01 ~]$ crsctl
status resource -t
--------------------------------------------------------------------------------
Name Target State Server State
details
--------------------------------------------------------------------------------
Local
Resources
--------------------------------------------------------------------------------
ora.ASMNET1LSNR_ASM.lsnr
ONLINE ONLINE paycore01 STABLE
ONLINE ONLINE paycore02 STABLE
ora.DATA01.V001_D01_1G.advm
ONLINE ONLINE paycore01 STABLE
ONLINE ONLINE paycore02 STABLE
ora.DATA01.dg
ONLINE ONLINE paycore01 STABLE
ONLINE ONLINE paycore02 STABLE
ora.LISTENER.lsnr
ONLINE ONLINE paycore01 STABLE
ONLINE ONLINE paycore02 STABLE
ora.MGMT.dg
ONLINE ONLINE paycore01 STABLE
ONLINE ONLINE paycore02 STABLE
ora.OCRVOTE.dg
ONLINE ONLINE paycore01 STABLE
ONLINE ONLINE paycore02 STABLE
ora.chad
ONLINE ONLINE paycore01 STABLE
ONLINE ONLINE paycore02 STABLE
ora.net1.network
ONLINE ONLINE paycore01 STABLE
ONLINE ONLINE paycore02 STABLE
ora.ons
ONLINE ONLINE paycore01 STABLE
ONLINE ONLINE paycore02 STABLE
ora.proxy_advm
ONLINE ONLINE paycore01 STABLE
ONLINE ONLINE paycore02 STABLE
--------------------------------------------------------------------------------
Cluster
Resources
--------------------------------------------------------------------------------
ora.LISTENER_SCAN1.lsnr
1 ONLINE ONLINE paycore02 STABLE
ora.LISTENER_SCAN2.lsnr
1 ONLINE ONLINE paycore01 STABLE
ora.LISTENER_SCAN3.lsnr
1 ONLINE ONLINE paycore01 STABLE
ora.MGMTLSNR
1 ONLINE ONLINE paycore01 169.254.76.211 172.1
6.6.11 172.16.6.111,
STABLE
ora.axxtrip.db
1 ONLINE ONLINE paycore01 Open,HOME=/oracle/ap
p/oracle/11.2.0/db_1
,STABLE
ora.axxtrip.s_axxtrip.svc
1 ONLINE ONLINE paycore01 STABLE
ora.asm
1 ONLINE ONLINE paycore01 Started,STABLE
2 ONLINE ONLINE paycore02 Started,STABLE
3 OFFLINE
OFFLINE STABLE
ora.cvu
1 ONLINE ONLINE paycore01 STABLE
ora.mgmtdb
1 ONLINE ONLINE paycore01 Open,STABLE
ora.nxxpay.db
1 ONLINE ONLINE paycore01 Open,HOME=/oracle/ap
p/oracle/11.2.0/db_1
,STABLE
2 ONLINE ONLINE paycore02 Open,HOME=/oracle/ap
p/oracle/11.2.0/db_1
,STABLE
ora.pxxdit.crm_fan.svc
1 ONLINE ONLINE paycore01 STABLE
ora.pxxdit.crm_taf.svc
1 ONLINE ONLINE paycore01 STABLE
ora.pxxdit.crm_taf2.svc
1 ONLINE ONLINE paycore01 STABLE
2 ONLINE ONLINE paycore02 STABLE
ora.pxxdit.db
1 ONLINE ONLINE paycore01 Open,HOME=/oracle/ap
p/oracle/11.2.0/db_1
,STABLE
2 ONLINE ONLINE paycore02 Open,HOME=/oracle/ap
p/oracle/11.2.0/db_1
,STABLE
ora.paycore01.vip
1 ONLINE ONLINE paycore01 STABLE
ora.paycore02.vip
1 ONLINE ONLINE paycore02 STABLE
ora.qosmserver
1 ONLINE ONLINE paycore01 STABLE
ora.scan1.vip
1 ONLINE ONLINE paycore02 STABLE
ora.scan2.vip
1 ONLINE ONLINE paycore01 STABLE
ora.scan3.vip
1 ONLINE ONLINE paycore01 STABLE
--------------------------------------------------------------------------------
[grid@paycore01 ~]$
adrci>
show incdir
ADR Home =
/oracle/app/grid/diag/tnslsnr/paycore02/listener_scan2:
*************************************************************************
ADR Home =
/oracle/app/grid/diag/tnslsnr/paycore02/listener:
*************************************************************************
ADR Home =
/oracle/app/grid/diag/tnslsnr/paycore02/mgmtlsnr:
*************************************************************************
ADR Home =
/oracle/app/grid/diag/tnslsnr/paycore02/asmnet1lsnr_asm:
*************************************************************************
ADR Home =
/oracle/app/grid/diag/tnslsnr/paycore02/listener_scan3:
*************************************************************************
ADR Home =
/oracle/app/grid/diag/tnslsnr/paycore02/listener_scan1:
*************************************************************************
ADR Home =
/oracle/app/grid/diag/rdbms/_mgmtdb/-MGMTDB:
*************************************************************************
ADR Home =
/oracle/app/grid/diag/asm/user_grid/host_579982181_107:
*************************************************************************
ADR Home =
/oracle/app/grid/diag/asm/+asm/+ASM2:
*************************************************************************
ADR Home =
/oracle/app/grid/diag/asm/user_root/host_579982181_107:
*************************************************************************
ADR Home =
/oracle/app/grid/diag/crs/paycore02/crs:
*************************************************************************
W
[grid@paycore02 -MGMTDB]$ pwd
/oracle/app/grid/diag/rdbms/_mgmtdb/-MGMTDB
[grid@paycore02 -MGMTDB]$ ls
-l
total
56
drwxr-xr-x
2 grid oinstall 4096 Apr 28 15:26 alert
drwxr-xr-x
2 grid oinstall 4096 Apr 28 15:26 cdump
drwxr-xr-x
2 grid oinstall 4096 Apr 28 15:26 hm
drwxr-xr-x
2 grid oinstall 4096 Apr 28 15:26 incident
drwxr-xr-x
2 grid oinstall 4096 Apr 28 15:26 incpkg
drwxr-xr-x
2 grid oinstall 4096 Apr 28 15:26 ir
drwxr-xr-x
2 grid oinstall 4096 Apr 28 15:27 lck
drwxr-xr-x
7 grid oinstall 4096 Apr 28 15:26 log
drwxr-xr-x
2 grid oinstall 4096 Apr 28 15:27 metadata
drwxr-xr-x
2 grid oinstall 4096 Apr 28 15:26 metadata_dgif
drwxr-xr-x
2 grid oinstall 4096 Apr 28 15:26 metadata_pv
drwxr-xr-x
2 grid oinstall 4096 Apr 28 15:26 stage
drwxr-xr-x
2 grid oinstall 4096 Apr 28 15:26 sweep
drwxr-xr-x
2 grid oinstall 4096 May 2 15:34 trace
[grid@paycore02 -MGMTDB]$ cd
alert
[grid@paycore02 alert]$ ls
log.xml
[grid@paycore02 alert]$ cd ..
metadata_pv stage sweep trace
[grid@paycore02 -MGMTDB]$ ls
-l
total
56
drwxr-xr-x
2 grid oinstall 4096 Apr 28 15:26 alert
drwxr-xr-x
2 grid oinstall 4096 Apr 28 15:26 cdump
drwxr-xr-x
2 grid oinstall 4096 Apr 28 15:26 hm
drwxr-xr-x
2 grid oinstall 4096 Apr 28 15:26 incident
drwxr-xr-x
2 grid oinstall 4096 Apr 28 15:26 incpkg
drwxr-xr-x
2 grid oinstall 4096 Apr 28 15:26 ir
drwxr-xr-x
2 grid oinstall 4096 Apr 28 15:27 lck
drwxr-xr-x
7 grid oinstall 4096 Apr 28 15:26 log
drwxr-xr-x
2 grid oinstall 4096 Apr 28 15:27 metadata
drwxr-xr-x
2 grid oinstall 4096 Apr 28 15:26 metadata_dgif
drwxr-xr-x
2 grid oinstall 4096 Apr 28 15:26 metadata_pv
drwxr-xr-x
2 grid oinstall 4096 Apr 28 15:26 stage
drwxr-xr-x
2 grid oinstall 4096 Apr 28 15:26 sweep
drwxr-xr-x
2 grid oinstall 4096 May 2 15:34 trace
[grid@paycore02 -MGMTDB]$ cd
trace
[grid@paycore02 trace]$ ls
alert_-MGMTDB.log -MGMTDB_dbw0_28126.trm -MGMTDB_gen0_9548.trm -MGMTDB_mark_9621.trm -MGMTDB_ora_9454.trm -MGMTDB_svcb_9571.trm
-MGMTDB_aqpc_29055.trc -MGMTDB_dbw0_9577.trc -MGMTDB_lg00_28148.trc -MGMTDB_mmon_28231.trc -MGMTDB_ora_9618.trc -MGMT02_28791.trc
-MGMTDB_aqpc_29055.trm -MGMTDB_dbw0_9577.trm -MGMTDB_lg00_28148.trm -MGMTDB_mmon_28231.trm -MGMTDB_ora_9618.trm -MGMT02_28791.trm
-MGMTDB_aqpc_9689.trc -MGMTDB_dia0_28124_base_1.trc -MGMTDB_lg00_9583.trc -MGMTDB_mmon_9605.trc -MGMTDB_p000_28634.trc -MGMT02_9681.trc
-MGMTDB_aqpc_9689.trm -MGMTDB_dia0_28124_base_1.trm -MGMTDB_lg00_9583.trm -MGMTDB_mmon_9605.trm -MGMTDB_p000_28634.trm -MGMT02_9681.trm
-MGMTDB_asmb_28217.trc -MGMTDB_dia0_28124.trc -MGMTDB_lg01_28153.trc -MGMTDB_ofsd_28100_28105.trc -MGMTDB_p000_9673.trc -MGMTDB_vkrm_28117.trc
-MGMTDB_asmb_28217.trm -MGMTDB_dia0_28124.trm -MGMTDB_lg01_28153.trm -MGMTDB_ofsd_28100_28105.trm -MGMTDB_p000_9673.trm -MGMTDB_vkrm_28117.trm
-MGMTDB_asmb_9601.trc -MGMTDB_dia0_9575_base_1.trc -MGMTDB_lg01_9587.trc -MGMTDB_ofsd_9563_9565.trc -MGMTDB_p001_28636.trc -MGMTDB_vkrm_9569.trc
-MGMTDB_asmb_9601.trm -MGMTDB_dia0_9575_base_1.trm -MGMTDB_lg01_9587.trm -MGMTDB_ofsd_9563_9565.trm -MGMTDB_p001_28636.trm -MGMTDB_vkrm_9569.trm
-MGMTDB_ckpt_28146.trc -MGMTDB_dia0_9575.trc -MGMTDB_lgwr_28143.trc -MGMTDB_ora_14326.trc -MGMTDB_p001_9675.trc -MGMTDB_vktm_28081.trc
-MGMTDB_ckpt_28146.trm -MGMTDB_dia0_9575.trm -MGMTDB_lgwr_28143.trm -MGMTDB_ora_14326.trm -MGMTDB_p001_9675.trm -MGMTDB_vktm_28081.trm
-MGMTDB_ckpt_9581.trc -MGMTDB_fenc_28220.trc -MGMTDB_lgwr_9579.trc -MGMTDB_ora_27648.trc -MGMTDB_rbal_28198.trc -MGMTDB_vktm_9544.trc
-MGMTDB_ckpt_9581.trm -MGMTDB_fenc_28220.trm -MGMTDB_lgwr_9579.trm -MGMTDB_ora_27648.trm -MGMTDB_rbal_28198.trm -MGMTDB_vktm_9544.trm
-MGMTDB_dbrm_28112.trc -MGMTDB_fenc_9603.trc -MGMTDB_m003_11066.trc -MGMTDB_ora_28272.trc -MGMTDB_rbal_9599.trc
-MGMTDB_dbrm_28112.trm -MGMTDB_fenc_9603.trm -MGMTDB_m003_11066.trm -MGMTDB_ora_28272.trm -MGMTDB_rbal_9599.trm
-MGMTDB_dbrm_9567.trc -MGMTDB_gen0_28085.trc -MGMTDB_mark_28301.trc -MGMTDB_ora_32204.trc -MGMTDB_svcb_28119.trc
-MGMTDB_dbrm_9567.trm -MGMTDB_gen0_28085.trm -MGMTDB_mark_28301.trm -MGMTDB_ora_32204.trm -MGMTDB_svcb_28119.trm
-MGMTDB_dbw0_28126.trc -MGMTDB_gen0_9548.trc -MGMTDB_mark_9621.trc -MGMTDB_ora_9454.trc -MGMTDB_svcb_9571.trc
[grid@paycore02 trace]$ tail -100
alert_-MGMTDB.log
Lost write
protection disabled
Using
STANDBY_ARCHIVE_DEST parameter default value as
/oracle/app/11.2.0/grid/dbs/arch
2017-05-02T15:33:15.983273+08:00
Completed:
ALTER DATABASE MOUNT
ALTER
DATABASE OPEN
2017-05-02T15:33:17.811285+08:00
Ping
without log force is disabled:
instance mounted in exclusive
mode.
2017-05-02T15:33:18.369109+08:00
Beginning
crash recovery of 1 threads
parallel recovery started with 2
processes
2017-05-02T15:33:19.044977+08:00
Started
redo scan
2017-05-02T15:33:20.035822+08:00
Completed
redo scan
read 10003 KB redo, 727 data blocks need
recovery
2017-05-02T15:33:21.227768+08:00
Started
redo application at
Thread 1: logseq 23, block 1211099,
offset 0
2017-05-02T15:33:21.286436+08:00
Recovery
of Online Redo Log: Thread 1 Group 2 Seq 23 Reading mem
0
Mem# 0:
+MGMT/_MGMTDB/ONLINELOG/group_2.262.942495437
2017-05-02T15:33:21.541681+08:00
Completed
redo application of 6.04MB
2017-05-02T15:33:21.955711+08:00
Completed
crash recovery at
Thread 1: RBA 23.1231106.16, nab 1231106,
scn 0x0000000000269f63
727 data blocks read, 727 data blocks
written, 10003 redo k-bytes read
2017-05-02T15:33:22.628030+08:00
Endian
type of dictionary set to little
2017-05-02T15:33:22.699784+08:00
TT00: Gap
Manager starting (PID:28785)
2017-05-02T15:33:23.240774+08:00
Thread 1
advanced to log sequence 24 (thread open)
Thread 1
opened at log sequence 24
Current log# 3 seq# 24 mem# 0:
+MGMT/_MGMTDB/ONLINELOG/group_3.263.942495437
Successful
open of redo thread 1
2017-05-02T15:33:23.782158+08:00
MTTR
advisory is disabled because FAST_START_MTTR_TARGET is not
set
2017-05-02T15:33:28.097794+08:00
[28272]
Successfully onlined Undo Tablespace 2.
Undo
initialization finished serial:0 start:361457536 end:361459431
diff:1895 ms (1.9 seconds)
Verifying
minimum file header compatibility for tablespace encryption for pdb
1..
Verifying
file header compatibility for tablespace encryption completed for
pdb 1
Database
Characterset is AL32UTF8
2017-05-02T15:33:31.065643+08:00
No
Resource Manager plan active
2017-05-02T15:33:32.684755+08:00
replication_dependency_tracking turned off (no async
multimaster replication found)
2017-05-02T15:33:33.747247+08:00
Starting
background process AQPC
2017-05-02T15:33:33.778204+08:00
AQPC
started with pid=48, OS id=29055
2017-05-02T15:33:39.724498+08:00
PDB$SEED(2):Autotune of undo retention is turned
on.
2017-05-02T15:33:40.097787+08:00
PDB$SEED(2):Endian type of dictionary set to
little
PDB$SEED(2):Undo initialization finished serial:0
start:361471837 end:361471837 diff:0 ms (0.0 seconds)
2017-05-02T15:33:40.792411+08:00
PDB$SEED(2):Database Characterset for PDB$SEED is
AL32UTF8
2017-05-02T15:33:42.126160+08:00
PDB$SEED(2):Opatch validation is skipped for PDB PDB$SEED
(con_id=0)
2017-05-02T15:33:43.536353+08:00
PDB$SEED(2):Opening pdb with no Resource Manager plan
active
2017-05-02T15:33:44.871281+08:00
Starting
background process CJQ0
Completed:
ALTER DATABASE OPEN
2017-05-02T15:33:44.896228+08:00
CJQ0
started with pid=54, OS id=29618
ALTER
PLUGGABLE DATABASE ALL OPEN
GIMR_DSCREP_10(3):Autotune of undo retention is turned
on.
2017-05-02T15:33:45.246361+08:00
GIMR_DSCREP_10(3):Endian type of dictionary set to
little
2017-05-02T15:33:47.307161+08:00
GIMR_DSCREP_10(3):[28272] Successfully onlined Undo
Tablespace 2.
GIMR_DSCREP_10(3):Undo initialization finished serial:0
start:361476967 end:361478656 diff:1689 ms (1.7 seconds)
GIMR_DSCREP_10(3):Database Characterset for GIMR_DSCREP_10
is AL32UTF8
2017-05-02T15:33:52.510889+08:00
GIMR_DSCREP_10(3):Opatch validation is skipped for PDB
GIMR_DSCREP_10 (con_id=0)
2017-05-02T15:33:59.200580+08:00
GIMR_DSCREP_10(3):Opening pdb with no Resource Manager plan
active
2017-05-02T15:34:03.300831+08:00
Pluggable
database GIMR_DSCREP_10 opened read write
Completed:
ALTER PLUGGABLE DATABASE ALL OPEN
2017-05-02T15:34:08.990346+08:00
Shared IO
Pool defaulting to 48MB. Trying to get it from Buffer Cache for
process 28231.
2017-05-02T15:34:10.719853+08:00
ALTER
SYSTEM SET listener_networks='(( NAME=private_network)
(LOCAL_LISTENER="(DESCRIPTION=(ADDRESS=(PROTOCOL=TCP)(HOST=169.254.27.77)(PORT=1525))(ADDRESS=(PROTOCOL=TCP)(HOST=172.16.6.12)(PORT=1525))(ADDRESS=(PROTOCOL=TCP)(HOST=172.16.6.121)(PORT=1525)))"))'
SCOPE=MEMORY SID='-MGMTDB';
2017-05-02T15:34:11.235636+08:00
===========================================================
Dumping
current patch information
===========================================================
No patches
have been applied
===========================================================
2017-05-02T15:34:11.928118+08:00
License
high water mark = 5
2017-05-02T15:34:11.936365+08:00
USER
(ospid: 32204): terminating the instance
2017-05-02T15:34:12.979910+08:00
Instance
terminated by USER, pid = 32204
[grid@paycore02 trace]$
adrci>
show alert
Choose the
home from which to view the alert log:
1:
diag/tnslsnr/paycore02/listener_scan2
2:
diag/tnslsnr/paycore02/listener
3:
diag/tnslsnr/paycore02/mgmtlsnr
4:
diag/tnslsnr/paycore02/asmnet1lsnr_asm
5:
diag/tnslsnr/paycore02/listener_scan3
6:
diag/tnslsnr/paycore02/listener_scan1
7: diag/rdbms/_mgmtdb/-MGMTDB
8:
diag/asm/user_grid/host_579982181_107
9:
diag/asm/+asm/+ASM2
10:
diag/asm/user_root/host_579982181_107
11:
diag/crs/paycore02/crs
12:
diag/asmtool/user_grid/host_579982181_107
13:
diag/asmtool/user_root/host_579982181_107
14:
diag/apx/+apx/+APX2
15:
diag/clients/user_grid/host_579982181_107
16:
diag/clients/user_oracle/host_579982181_107
17:
diag/clients/user_root/host_579982181_107
18:
diag/afdboot/user_root/host_579982181_107
Q: to
quit
Please
select option: 7
Output the
results to file: /tmp/alert_29036_1404_-MGMTDB_1.ado
2017-04-28
15:26:56.394000 +08:00
Starting
ORACLE instance (normal) (OS id: 9454)
CLI
notifier numLatches:3 maxDescs:849
**********************************************************************
Dump of
system resources acquired for SHARED GLOBAL AREA (SGA)
Per process system memlock (soft) limit =
UNLIMITED
Expected per process system memlock
(soft) limit to lock
SHARED GLOBAL AREA (SGA) into memory:
1026M
Available system pagesizes:
4K, 2048K
Supported system pagesize(s):
PAGESIZE AVAILABLE_PAGES EXPECTED_PAGES ALLOCATED_PAGES ERROR(s)
4K Configured 5 262149 NONE
2048K 0 513 0 NONE
RECOMMENDATION:
1. For optimal performance, configure
system with expected number
of pages for every supported system
pagesize prior to the next
instance restart operation.
**********************************************************************
LICENSE_MAX_SESSION = 0
LICENSE_SESSIONS_WARNING = 0
Initial
number of CPU is 2
Number of
processor cores in the system is 12
Number of
processor sockets in the system is 4
Shared
memory segment for instance monitoring created
Using
LOG_ARCHIVE_DEST_1 parameter default value as
/oracle/app/11.2.0/grid/dbs/arch
Autotune
of undo retention is turned on.
IMODE=BR
ILAT
=84
LICENSE_MAX_USERS = 0
SYS
auditing is enabled
NOTE:
remote asm mode is remote (mode 0x2; from cluster type)
2017-04-28
15:26:57.660000 +08:00
NOTE:
Cluster configuration type = CLUSTER [4]
NUMA
system with 4 nodes detected
Oracle
Database 12c Enterprise Edition Release 12.2.0.1.0 - 64bit
Production.
ORACLE_HOME: /oracle/app/11.2.0/grid
System
name: Linux
Node
name: paycore02
Release: 2.6.32-358.el6.x86_64
Version: #1 SMP Tue Jan 29 11:47:41 EST 2013
Machine: x86_64
Using
parameter settings in server-side spfile
+MGMT/_MGMTDB/PARAMETERFILE/spfile.269.942495797
System
parameters with non-default values:
processes = 500
cpu_count =
2
nls_language = "AMERICAN"
nls_territory = "AMERICA"
sga_target = 1G
control_files = "+MGMT/_MGMTDB/CONTROLFILE/current.260.942495435"
db_block_size = 8192
compatible =
"12.2.0.0.0"
cluster_database = FALSE
ILAT
=84
LICENSE_MAX_USERS = 0
SYS
auditing is enabled
NOTE:
remote asm mode is remote (mode 0x2; from cluster type)
2017-04-28
15:26:57.660000 +08:00
NOTE:
Cluster configuration type = CLUSTER [4]
NUMA
system with 4 nodes detected
Oracle
Database 12c Enterprise Edition Release 12.2.0.1.0 - 64bit
Production.
ORACLE_HOME: /oracle/app/11.2.0/grid
System
name: Linux
Node
name: paycore02
Release: 2.6.32-358.el6.x86_64
Version: #1 SMP Tue Jan 29 11:47:41 EST 2013
Machine: x86_64
Using
parameter settings in server-side spfile
+MGMT/_MGMTDB/PARAMETERFILE/spfile.269.942495797
System
parameters with non-default values:
processes = 500
cpu_count =
2
nls_language = "AMERICAN"
nls_territory = "AMERICA"
sga_target = 1G
control_files = "+MGMT/_MGMTDB/CONTROLFILE/current.260.942495435"
db_block_size = 8192
compatible = "12.2.0.0.0"
cluster_database = FALSE
db_create_file_dest = "+MGMT"
undo_tablespace = "UNDOTBS1"
_disable_txn_alert = 3
_partition_large_extents =
"false"
recyclebin = "off"
remote_login_passwordfile=
"EXCLUSIVE"
dispatchers = "(PROTOCOL=TCP) (SERVICE=-MGMTDBXDB)"
job_queue_processes = 1
_kolfuseslf = TRUE
parallel_min_servers = 0
audit_file_dest = "/oracle/app/grid/admin/_mgmtdb/adump"
audit_trail = "NONE"
db_name = "_mgmtdb"
open_cursors = 150
pga_aggregate_target = 500M
statistics_level = "TYPICAL"
max_string_size = "EXTENDED"
diagnostic_dest = "/oracle/app/grid"
enable_pluggable_database=
TRUE
NOTE:
remote asm mode is remote (mode 0x2; from cluster type)
Oracle
instance running with ODM: Oracle Direct NFS ODM Library Version
4.0
============================================================
NOTE:
PatchLevel of this instance 0
============================================================
2017-04-28
15:27:02.899000 +08:00
……