The generic steps to follow when adding the new node to the cluster are:
Install Operating System
Install required software
Add/modify users and groups required for the installation
Configure network
Configure kernel parameters
Configure services required such as NTP
Run the "root.sh" commands on the new node as directed
Add Instance to Clustered Database
A database instance will be established on the new node. Specifically, an instance named "zwc3" will be added to "zwc" – a pre-existing clustered database.
Satisfy Node Instance Dependencies
Satisfy all node instance dependencies, such as init.ora parameters, etc.
From the new node "zhongwc3", run the following commands to create the passwordfile, "init.ora" file, and "oratab" entry for the new instance
From a node with an existing instance of "zwc" issue the following commands to create the needed public log thread, undo tablespace, and "init.ora" entries for the new instance
Install Operating System
Install required software
Add/modify users and groups required for the installation
Configure network
Configure kernel parameters
Configure services required such as NTP
Configure storage (multipathing, zoning, storage discovery,UDEV, ASMLib)
zhongwc3 node
[root@zhongwc3 ~]# cat /home/oracle/.bash_profile
# .bash_profile
# Get the aliases and functions
if [ -f ~/.bashrc ]; then
. ~/.bashrc
fi
# User specific environment and startup programs
PATH=$PATH:$HOME/bin
export PATH
alias ls="ls -FA"
ORACLE_SID=zwc3; export ORACLE_SID
JAVA_HOME=/usr/local/java; export JAVA_HOME
ORACLE_BASE=/u01/app/oracle; export ORACLE_BASE
ORACLE_HOME=$ORACLE_BASE/product/12.1.0/db_1; export ORACLE_HOME
ORACLE_PATH=/u01/app/common/oracle/sql; export ORACLE_PATH
ORACLE_TERM=xterm; export ORACLE_TERM
TNS_ADMIN=$ORACLE_HOME/network/admin; export TNS_ADMIN
ORA_NLS11=$ORACLE_HOME/nls/data; export ORA_NLS11
NLS_LANG=AMERICAN_AMERICA.ZHS16GBK; export NLS_LANG
PATH=.:${JAVA_HOME}/bin:${PATH}:$HOME/bin:$ORACLE_HOME/bin:$ORACLE_HOME/OPatch
PATH=${PATH}:/usr/bin:/bin:/usr/bin/X11:/usr/local/bin
PATH=${PATH}:/u01/app/common/oracle/bin
export PATH
LD_LIBRARY_PATH=$ORACLE_HOME/lib
LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:$ORACLE_HOME/oracm/lib
LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/lib:/usr/lib:/usr/local/lib
export LD_LIBRARY_PATH
CLASSPATH=$ORACLE_HOME/JRE
CLASSPATH=${CLASSPATH}:$ORACLE_HOME/jlib
CLASSPATH=${CLASSPATH}:$ORACLE_HOME/rdbms/jlib
CLASSPATH=${CLASSPATH}:$ORACLE_HOME/network/jlib
export CLASSPATH
THREADS_FLAG=native; export THREADS_FLAG
export TEMP=/tmp
export TMPDIR=/tmp
umask 022
[root@zhongwc3 ~]# cat /home/grid/.bash_profile
# .bash_profile
# Get the aliases and functions
if [ -f ~/.bashrc ]; then
. ~/.bashrc
fi
# User specific environment and startup programs
PATH=$PATH:$HOME/bin
export PATH
alias ls="ls -FA"
ORACLE_SID=+ASM3; export ORACLE_SID
JAVA_HOME=/usr/local/java; export JAVA_HOME
ORACLE_BASE=/u01/app/grid; export ORACLE_BASE
ORACLE_HOME=/u01/app/12.1.0/grid; export ORACLE_HOME
ORACLE_PATH=/u01/app/oracle/common/oracle/sql; export ORACLE_PATH
ORACLE_TERM=xterm; export ORACLE_TERM
TNS_ADMIN=$ORACLE_HOME/network/admin; export TNS_ADMIN
ORA_NLS11=$ORACLE_HOME/nls/data; export ORA_NLS11
NLS_LANG=AMERICAN_AMERICA.ZHS16GBK; export NLS_LANG
PATH=.:${JAVA_HOME}/bin:${PATH}:$HOME/bin:$ORACLE_HOME/bin:$ORACLE_HOME/OPatch
PATH=${PATH}:/usr/bin:/bin:/usr/bin/X11:/usr/local/bin
PATH=${PATH}:/u01/app/common/oracle/bin
export PATH
LD_LIBRARY_PATH=$ORACLE_HOME/lib
LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:$ORACLE_HOME/oracm/lib
LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/lib:/usr/lib:/usr/local/lib
export LD_LIBRARY_PATH
CLASSPATH=$ORACLE_HOME/JRE
CLASSPATH=${CLASSPATH}:$ORACLE_HOME/jlib
CLASSPATH=${CLASSPATH}:$ORACLE_HOME/rdbms/jlib
CLASSPATH=${CLASSPATH}:$ORACLE_HOME/network/jlib
export CLASSPATH
THREADS_FLAG=native; export THREADS_FLAG
export TEMP=/tmp
export TMPDIR=/tmp
umask 022
[root@zhongwc3 ~]# cat .bash_profile
# .bash_profile
# Get the aliases and functions
if [ -f ~/.bashrc ]; then
. ~/.bashrc
fi
# User specific environment and startup programs
PATH=$PATH:$HOME/bin
export PATH
export ORACLE_HOME=/u01/app/12.1.0/grid
export PATH=$ORACLE_HOME/bin:$ORACLE_HOME/OPatch:$PATH
[root@zhongwc3 ~]#
[root@zhongwc3 ~]# cat /etc/hosts
127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4
::1 localhost localhost.localdomain localhost6 localhost6.localdomain6
#public IP
192.168.1.31 zhongwc1
192.168.1.32 zhongwc2
192.168.1.33 zhongwc3
#private IP
172.168.1.31 zhongwc1-priv
172.168.1.32 zhongwc2-priv
172.168.1.33 zhongwc3-priv
#VIP
192.168.1.131 zhongwc1-vip
192.168.1.132 zhongwc2-vip
192.168.1.133 zhongwc3-vip
#scanip
192.168.1.179 zhongwc-cluster zhongwc-cluster-scan
cluvfy stage -pre nodeadd -n zhongwc3 -fixup -fixupnoexec
[grid@zhongwc1 ~]$ cluvfy stage -pre nodeadd -n zhongwc3 -fixup -fixupnoexec
Performing pre-checks for node addition
Checking node reachability...
Node reachability check passed from node "zhongwc1"
Checking user equivalence...
User equivalence check passed for user "grid"
Package existence check passed for "cvuqdisk"
Checking CRS integrity...
CRS integrity check passed
Clusterware version consistency passed.
Checking shared resources...
Checking CRS home location...
"/u01/app/12.1.0/grid" is not shared
Shared resources check for node addition passed
Checking node connectivity...
Checking hosts config file...
Verification of the hosts config file successful
Check: Node connectivity using interfaces on subnet "172.168.1.0"
Node connectivity passed for subnet "172.168.1.0" with node(s) zhongwc1,zhongwc2,zhongwc3
TCP connectivity check passed for subnet "172.168.1.0"
Check: Node connectivity using interfaces on subnet "192.168.1.0"
Node connectivity passed for subnet "192.168.1.0" with node(s) zhongwc3,zhongwc2,zhongwc1
TCP connectivity check passed for subnet "192.168.1.0"
Checking subnet mask consistency...
Subnet mask consistency check passed for subnet "192.168.1.0".
Subnet mask consistency check passed for subnet "172.168.1.0".
Subnet mask consistency check passed.
Node connectivity check passed
Checking multicast communication...
Checking subnet "172.168.1.0" for multicast communication with multicast group "224.0.0.251"...
Check of subnet "172.168.1.0" for multicast communication with multicast group "224.0.0.251" passed.
Check of multicast communication passed.
Total memory check failed
Check failed on nodes:
zhongwc1
Available memory check passed
Swap space check passed
Free disk space check passed for "zhongwc3:/usr,zhongwc3:/var,zhongwc3:/etc,zhongwc3:/u01/app/12.1.0/grid,zhongwc3:/sbin,zhongwc3:/tmp"
Free disk space check passed for "zhongwc1:/usr,zhongwc1:/var,zhongwc1:/etc,zhongwc1:/u01/app/12.1.0/grid,zhongwc1:/sbin,zhongwc1:/tmp"
Check for multiple users with UID value 1100 passed
User existence check passed for "grid"
Run level check passed
Hard limits check passed for "maximum open file descriptors"
Soft limits check passed for "maximum open file descriptors"
Hard limits check passed for "maximum user processes"
Soft limits check passed for "maximum user processes"
System architecture check passed
Kernel version check passed
Kernel parameter check passed for "semmsl"
Kernel parameter check passed for "semmns"
Kernel parameter check passed for "semopm"
Kernel parameter check passed for "semmni"
Kernel parameter check passed for "shmmax"
Kernel parameter check passed for "shmmni"
Kernel parameter check passed for "shmall"
Kernel parameter check passed for "file-max"
Kernel parameter check passed for "ip_local_port_range"
Kernel parameter check passed for "rmem_default"
Kernel parameter check passed for "rmem_max"
Kernel parameter check passed for "wmem_default"
Kernel parameter check passed for "wmem_max"
Kernel parameter check passed for "aio-max-nr"
Package existence check passed for "binutils"
Package existence check passed for "compat-libcap1"
Package existence check passed for "compat-libstdc++-33(x86_64)"
Package existence check passed for "libgcc(x86_64)"
Package existence check passed for "libstdc++(x86_64)"
Package existence check passed for "libstdc++-devel(x86_64)"
Package existence check passed for "sysstat"
Package existence check passed for "gcc"
Package existence check passed for "gcc-c++"
Package existence check passed for "ksh"
Package existence check passed for "make"
Package existence check passed for "glibc(x86_64)"
Package existence check passed for "glibc-devel(x86_64)"
Package existence check passed for "libaio(x86_64)"
Package existence check passed for "libaio-devel(x86_64)"
Package existence check passed for "nfs-utils"
Check for multiple users with UID value 0 passed
Current group ID check passed
Starting check for consistency of primary group of root user
Check for consistency of root user's primary group passed
Group existence check passed for "asmadmin"
Group existence check passed for "asmoper"
Group existence check passed for "asmdba"
Checking ASMLib configuration.
Check for ASMLib configuration passed.
Checking OCR integrity...
OCR integrity check passed
Checking Oracle Cluster Voting Disk configuration...
Oracle Cluster Voting Disk configuration check passed
Time zone consistency check passed
Starting Clock synchronization checks using Network Time Protocol(NTP)...
NTP Configuration file check started...
No NTP Daemons or Services were found to be running
Clock synchronization check using Network Time Protocol(NTP) passed
User "grid" is not part of "root" group. Check passed
Checking integrity of file "/etc/resolv.conf" across nodes
"domain" and "search" entries do not coexist in any "/etc/resolv.conf" file
All nodes have same "search" order defined in file "/etc/resolv.conf"
The DNS response time for an unreachable node is within acceptable limit on all nodes
Check for integrity of file "/etc/resolv.conf" passed
Checking integrity of name service switch configuration file "/etc/nsswitch.conf" ...
Check for integrity of name service switch configuration file "/etc/nsswitch.conf" passed
NOTE:
No fixable verification failures to fix
Pre-check for node addition was successful. Extend Clusterware
The clusterware software will be extended to the new node.
Run "addNode.sh"
From an existing node, run "addNode.sh" to extend the clusterware to the new node "zhongwc3":
[grid@zhongwc1 ~]$ cd $ORACLE_HOME/addnode
[grid@zhongwc1 addnode]$ export IGNORE_PREADDNODE_CHECKS=Y
[grid@zhongwc1 addnode]$ ./addnode.sh -silent "CLUSTER_NEW_NODES={zhongwc3}" "CLUSTER_NEW_VIRTUAL_HOSTNAMES={zhongwc3-vip}" "CLUSTER_NEW_PRIVATE_NODE_NAMES={zhongwc3-priv}"Starting Oracle Universal Installer...
Checking Temp space: must be greater than 120 MB. Actual 45342 MB Passed
Checking swap space: must be greater than 150 MB. Actual 8178 MB Passed
[WARNING] [INS-13014] Target environment does not meet some optional requirements.
CAUSE: Some of the optional prerequisites are not met. See logs for details. /u01/app/oraInventory/logs/addNodeActions2013-10-21_09-13-10PM.log
ACTION: Identify the list of failed prerequisite checks from the log: /u01/app/oraInventory/logs/addNodeActions2013-10-21_09-13-10PM.log. Then either from the log file or from installation manual find the appropriate configuration to meet the prerequisites and fix it manually.
Prepare Configuration in progress.
Prepare Configuration successful.
.................................................. 9% Done.
You can find the log of this install session at:
/u01/app/oraInventory/logs/addNodeActions2013-10-21_09-13-10PM.log
Instantiate files in progress.
Instantiate files successful.
.................................................. 15% Done.
Copying files to node in progress.
Copying files to node successful.
.................................................. 79% Done.
Saving cluster inventory in progress.
.................................................. 87% Done.
Saving cluster inventory successful.
The Cluster Node Addition of /u01/app/12.1.0/grid was successful.
Please check '/tmp/silentInstall.log' for more details.
As a root user, execute the following script(s):
1. /u01/app/oraInventory/orainstRoot.sh
2. /u01/app/12.1.0/grid/root.sh
Execute /u01/app/oraInventory/orainstRoot.sh on the following nodes:
[zhongwc3]
Execute /u01/app/12.1.0/grid/root.sh on the following nodes:
[zhongwc3]
The scripts can be executed in parallel on all the nodes. If there are any policy managed databases managed by cluster, proceed with the addnode procedure without executing the root.sh script. Ensure that root.sh script is executed after all the policy managed databases managed by clusterware are extended to the new nodes.
..........
Update Inventory in progress.
.................................................. 100% Done.
Update Inventory successful.
Successfully Setup Software.Run the "root.sh" commands on the new node as directed
[root@zhongwc3 ~]# /u01/app/oraInventory/orainstRoot.sh
Changing permissions of /u01/app/oraInventory.
Adding read,write permissions for group.
Removing read,write,execute permissions for world.
Changing groupname of /u01/app/oraInventory to oinstall.
The execution of the script is complete.
[root@zhongwc3 ~]#
[root@zhongwc3 ~]#
[root@zhongwc3 ~]# /u01/app/12.1.0/grid/root.sh
Check /u01/app/12.1.0/grid/install/root_zhongwc3_2013-10-21_21-26-30.log for the output of root script[root@zhongwc3 ~]# cat /u01/app/12.1.0/grid/install/root_zhongwc3_2013-10-21_21-26-30.log
Performing root user operation for Oracle 12c
The following environment variables are set as:
ORACLE_OWNER= grid
ORACLE_HOME= /u01/app/12.1.0/grid
Copying dbhome to /usr/local/bin ...
Copying oraenv to /usr/local/bin ...
Copying coraenv to /usr/local/bin ...
Creating /etc/oratab file...
Entries will be added to the /etc/oratab file as needed by
Database Configuration Assistant when a database is created
Finished running generic part of root script.
Now product-specific root actions will be performed.
Relinking oracle with rac_on option
Using configuration parameter file: /u01/app/12.1.0/grid/crs/install/crsconfig_params
2013/10/21 21:26:39 CLSRSC-363: User ignored prerequisites during installation
OLR initialization - successful
2013/10/21 21:27:04 CLSRSC-330: Adding Clusterware entries to file 'oracle-ohasd.conf'
CRS-4133: Oracle High Availability Services has been stopped.
CRS-4123: Oracle High Availability Services has been started.
CRS-4133: Oracle High Availability Services has been stopped.
CRS-4123: Oracle High Availability Services has been started.
CRS-2791: Starting shutdown of Oracle High Availability Services-managed resources on 'zhongwc3'
CRS-2673: Attempting to stop 'ora.drivers.acfs' on 'zhongwc3'
CRS-2677: Stop of 'ora.drivers.acfs' on 'zhongwc3' succeeded
CRS-2793: Shutdown of Oracle High Availability Services-managed resources on 'zhongwc3' has completed
CRS-4133: Oracle High Availability Services has been stopped.
CRS-4123: Starting Oracle High Availability Services-managed resources
CRS-2672: Attempting to start 'ora.mdnsd' on 'zhongwc3'
CRS-2672: Attempting to start 'ora.evmd' on 'zhongwc3'
CRS-2676: Start of 'ora.mdnsd' on 'zhongwc3' succeeded
CRS-2676: Start of 'ora.evmd' on 'zhongwc3' succeeded
CRS-2672: Attempting to start 'ora.gpnpd' on 'zhongwc3'
CRS-2676: Start of 'ora.gpnpd' on 'zhongwc3' succeeded
CRS-2672: Attempting to start 'ora.gipcd' on 'zhongwc3'
CRS-2676: Start of 'ora.gipcd' on 'zhongwc3' succeeded
CRS-2672: Attempting to start 'ora.cssdmonitor' on 'zhongwc3'
CRS-2676: Start of 'ora.cssdmonitor' on 'zhongwc3' succeeded
CRS-2672: Attempting to start 'ora.cssd' on 'zhongwc3'
CRS-2672: Attempting to start 'ora.diskmon' on 'zhongwc3'
CRS-2676: Start of 'ora.diskmon' on 'zhongwc3' succeeded
CRS-2789: Cannot stop resource 'ora.diskmon' as it is not running on server 'zhongwc3'
CRS-2676: Start of 'ora.cssd' on 'zhongwc3' succeeded
CRS-2672: Attempting to start 'ora.cluster_interconnect.haip' on 'zhongwc3'
CRS-2672: Attempting to start 'ora.ctssd' on 'zhongwc3'
CRS-2676: Start of 'ora.ctssd' on 'zhongwc3' succeeded
CRS-2676: Start of 'ora.cluster_interconnect.haip' on 'zhongwc3' succeeded
CRS-2672: Attempting to start 'ora.asm' on 'zhongwc3'
CRS-2676: Start of 'ora.asm' on 'zhongwc3' succeeded
CRS-2672: Attempting to start 'ora.storage' on 'zhongwc3'
CRS-2676: Start of 'ora.storage' on 'zhongwc3' succeeded
CRS-2672: Attempting to start 'ora.crf' on 'zhongwc3'
CRS-2676: Start of 'ora.crf' on 'zhongwc3' succeeded
CRS-2672: Attempting to start 'ora.crsd' on 'zhongwc3'
CRS-2676: Start of 'ora.crsd' on 'zhongwc3' succeeded
CRS-6017: Processing resource auto-start for servers: zhongwc3
CRS-2672: Attempting to start 'ora.ons' on 'zhongwc3'
CRS-2672: Attempting to start 'ora.FRADG.dg' on 'zhongwc3'
CRS-2672: Attempting to start 'ora.DATADG.dg' on 'zhongwc3'
CRS-2676: Start of 'ora.FRADG.dg' on 'zhongwc3' succeeded
CRS-2676: Start of 'ora.ons' on 'zhongwc3' succeeded
CRS-2676: Start of 'ora.DATADG.dg' on 'zhongwc3' succeeded
CRS-6016: Resource auto-start has completed for server zhongwc3
CRS-6024: Completed start of Oracle Cluster Ready Services-managed resources
CRS-4123: Oracle High Availability Services has been started.
2013/10/21 21:31:25 CLSRSC-343: Successfully started Oracle clusterware stack
clscfg: EXISTING configuration version 5 detected.
clscfg: version 5 is 12c Release 1.
Successfully accumulated necessary OCR keys.
Creating OCR keys for user 'root', privgrp 'root'..
Operation successful.
2013/10/21 21:31:43 CLSRSC-325: Configure Oracle Grid Infrastructure for a Cluster ... succeeded
If successful, clusterware daemons, the listener, the ASM instance, etc.
[root@zhongwc3 ~]# crs_stat -t
Name Type Target State Host
------------------------------------------------------------
ora.DATADG.dg ora....up.type ONLINE ONLINE zhongwc1
ora.FRADG.dg ora....up.type ONLINE ONLINE zhongwc1
ora....ER.lsnr ora....er.type ONLINE ONLINE zhongwc1
ora....N1.lsnr ora....er.type ONLINE ONLINE zhongwc1
ora.MGMTLSNR ora....nr.type ONLINE ONLINE zhongwc1
ora.SYSDG.dg ora....up.type ONLINE ONLINE zhongwc1
ora.asm ora.asm.type ONLINE ONLINE zhongwc1
ora.cvu ora.cvu.type ONLINE ONLINE zhongwc1
ora.mgmtdb ora....db.type ONLINE ONLINE zhongwc1
ora....network ora....rk.type ONLINE ONLINE zhongwc1
ora.oc4j ora.oc4j.type ONLINE ONLINE zhongwc1
ora.ons ora.ons.type ONLINE ONLINE zhongwc1
ora.scan1.vip ora....ip.type ONLINE ONLINE zhongwc1
ora....SM1.asm application ONLINE ONLINE zhongwc1
ora....C1.lsnr application ONLINE ONLINE zhongwc1
ora....wc1.ons application ONLINE ONLINE zhongwc1
ora....wc1.vip ora....t1.type ONLINE ONLINE zhongwc1
ora....SM2.asm application ONLINE ONLINE zhongwc2
ora....C2.lsnr application ONLINE ONLINE zhongwc2
ora....wc2.ons application ONLINE ONLINE zhongwc2
ora....wc2.vip ora....t1.type ONLINE ONLINE zhongwc2
ora....SM3.asm application ONLINE ONLINE zhongwc3
ora....C3.lsnr application ONLINE ONLINE zhongwc3
ora....wc3.ons application ONLINE ONLINE zhongwc3
ora....wc3.vip ora....t1.type ONLINE ONLINE zhongwc3
ora....crm.svc ora....ce.type ONLINE ONLINE zhongwc1
ora.zwc.db ora....se.type ONLINE ONLINE zhongwc1
ora....erp.svc ora....ce.type ONLINE ONLINE zhongwc2
[root@zhongwc3 ~]# crs_stat -t -c zhongwc3
Name Type Target State Host
------------------------------------------------------------
ora....SM3.asm application ONLINE ONLINE zhongwc3
ora....C3.lsnr application ONLINE ONLINE zhongwc3
ora....wc3.ons application ONLINE ONLINE zhongwc3
ora....wc3.vip ora....t1.type ONLINE ONLINE zhongwc3
[root@zhongwc3 ~]#
[root@zhongwc3 ~]#
[root@zhongwc3 ~]# ps -ef|grep -i asm
root 26023 2 0 21:28 ? 00:00:00 [asmWorkerThread]
root 26024 2 0 21:28 ? 00:00:00 [asmWorkerThread]
root 26025 2 0 21:28 ? 00:00:00 [asmWorkerThread]
root 26026 2 0 21:28 ? 00:00:00 [asmWorkerThread]
root 26027 2 0 21:28 ? 00:00:00 [asmWorkerThread]
grid 26789 1 0 21:30 ? 00:00:00 asm_pmon_+ASM3
grid 26791 1 0 21:30 ? 00:00:00 asm_psp0_+ASM3
grid 26793 1 6 21:30 ? 00:00:13 asm_vktm_+ASM3
grid 26797 1 0 21:30 ? 00:00:00 asm_gen0_+ASM3
grid 26799 1 0 21:30 ? 00:00:00 asm_mman_+ASM3
grid 26803 1 0 21:30 ? 00:00:00 asm_diag_+ASM3
grid 26805 1 0 21:30 ? 00:00:00 asm_ping_+ASM3
grid 26807 1 0 21:30 ? 00:00:00 asm_dia0_+ASM3
grid 26809 1 0 21:30 ? 00:00:00 asm_lmon_+ASM3
grid 26811 1 0 21:30 ? 00:00:00 asm_lmd0_+ASM3
grid 26813 1 0 21:30 ? 00:00:01 asm_lms0_+ASM3
grid 26817 1 0 21:30 ? 00:00:00 asm_lmhb_+ASM3
grid 26819 1 0 21:30 ? 00:00:00 asm_lck1_+ASM3
grid 26821 1 0 21:30 ? 00:00:00 asm_dbw0_+ASM3
grid 26823 1 0 21:30 ? 00:00:00 asm_lgwr_+ASM3
grid 26825 1 0 21:30 ? 00:00:00 asm_ckpt_+ASM3
grid 26827 1 0 21:30 ? 00:00:00 asm_smon_+ASM3
grid 26829 1 0 21:30 ? 00:00:00 asm_lreg_+ASM3
grid 26831 1 0 21:30 ? 00:00:00 asm_rbal_+ASM3
grid 26833 1 0 21:30 ? 00:00:00 asm_gmon_+ASM3
grid 26835 1 0 21:30 ? 00:00:00 asm_mmon_+ASM3
grid 26837 1 0 21:30 ? 00:00:00 asm_mmnl_+ASM3
grid 26839 1 0 21:30 ? 00:00:00 asm_lck0_+ASM3
grid 26843 1 0 21:30 ? 00:00:00 asm_gcr0_+ASM3
grid 26856 1 0 21:30 ? 00:00:00 oracle+ASM3 (DESCRIPTION=(LOCAL=YES)(ADDRESS=(PROTOCOL=beq)))
grid 26886 1 0 21:30 ? 00:00:00 oracle+ASM3_ocr (DESCRIPTION=(LOCAL=YES)(ADDRESS=(PROTOCOL=beq)))
grid 26888 1 0 21:30 ? 00:00:00 asm_asmb_+ASM3
grid 26890 1 0 21:30 ? 00:00:00 oracle+ASM3_asmb_+asm3 (DESCRIPTION=(LOCAL=YES)(ADDRESS=(PROTOCOL=beq)))
grid 26892 1 0 21:30 ? 00:00:00 asm_o000_+ASM3
grid 26894 1 0 21:30 ? 00:00:00 oracle+ASM3_o000_+asm3 (DESCRIPTION=(LOCAL=YES)(ADDRESS=(PROTOCOL=beq)))
grid 26987 1 0 21:31 ? 00:00:00 oracle+ASM3 (DESCRIPTION=(LOCAL=YES)(ADDRESS=(PROTOCOL=beq)))
grid 26993 1 0 21:31 ? 00:00:00 oracle+ASM3 (DESCRIPTION=(LOCAL=YES)(ADDRESS=(PROTOCOL=beq)))
root 27417 14386 0 21:34 pts/3 00:00:00 grep -i asm
[root@zhongwc3 ~]#
[root@zhongwc3 ~]#
[root@zhongwc3 ~]# crsctl stat res -t
--------------------------------------------------------------------------------
Name Target State Server State details
--------------------------------------------------------------------------------
Local Resources
--------------------------------------------------------------------------------
ora.DATADG.dg
ONLINE ONLINE zhongwc1 STABLE
ONLINE ONLINE zhongwc2 STABLE
ONLINE ONLINE zhongwc3 STABLE
ora.FRADG.dg
ONLINE ONLINE zhongwc1 STABLE
ONLINE ONLINE zhongwc2 STABLE
ONLINE ONLINE zhongwc3 STABLE
ora.LISTENER.lsnr
ONLINE ONLINE zhongwc1 STABLE
ONLINE ONLINE zhongwc2 STABLE
ONLINE ONLINE zhongwc3 STABLE
ora.SYSDG.dg
ONLINE ONLINE zhongwc1 STABLE
ONLINE ONLINE zhongwc2 STABLE
ONLINE ONLINE zhongwc3 STABLE
ora.asm
ONLINE ONLINE zhongwc1 Started,STABLE
ONLINE ONLINE zhongwc2 Started,STABLE
ONLINE ONLINE zhongwc3 Started,STABLE
ora.net1.network
ONLINE ONLINE zhongwc1 STABLE
ONLINE ONLINE zhongwc2 STABLE
ONLINE ONLINE zhongwc3 STABLE
ora.ons
ONLINE ONLINE zhongwc1 STABLE
ONLINE ONLINE zhongwc2 STABLE
ONLINE ONLINE zhongwc3 STABLE
--------------------------------------------------------------------------------
Cluster Resources
--------------------------------------------------------------------------------
ora.LISTENER_SCAN1.lsnr
1 ONLINE ONLINE zhongwc1 STABLE
ora.MGMTLSNR
1 ONLINE ONLINE zhongwc1 169.254.198.87 172.1
68.1.31,STABLE
ora.cvu
1 ONLINE ONLINE zhongwc1 STABLE
ora.mgmtdb
1 ONLINE ONLINE zhongwc1 Open,STABLE
ora.oc4j
1 ONLINE ONLINE zhongwc1 STABLE
ora.scan1.vip
1 ONLINE ONLINE zhongwc1 STABLE
ora.zhongwc1.vip
1 ONLINE ONLINE zhongwc1 STABLE
ora.zhongwc2.vip
1 ONLINE ONLINE zhongwc2 STABLE
ora.zhongwc3.vip
1 ONLINE ONLINE zhongwc3 STABLE
ora.zwc.crm.svc
1 ONLINE ONLINE zhongwc1 STABLE
ora.zwc.db
1 ONLINE ONLINE zhongwc1 Open,STABLE
2 ONLINE ONLINE zhongwc2 Open,STABLE
ora.zwc.erp.svc
1 ONLINE ONLINE zhongwc2 STABLE
--------------------------------------------------------------------------------[root@zhongwc3 ~]# crsctl check cluster -all
**************************************************************
zhongwc1:
CRS-4537: Cluster Ready Services is online
CRS-4529: Cluster Synchronization Services is online
CRS-4533: Event Manager is online
**************************************************************
zhongwc2:
CRS-4537: Cluster Ready Services is online
CRS-4529: Cluster Synchronization Services is online
CRS-4533: Event Manager is online
**************************************************************
zhongwc3:
CRS-4537: Cluster Ready Services is online
CRS-4529: Cluster Synchronization Services is online
CRS-4533: Event Manager is online
**************************************************************Extend Oracle Database Software
From an existing node – as the database software owner – run the following command to extend the Oracle database software to the new node "zhongwc3"
[oracle@zhongwc1 addnode]$ ./addnode.sh -silent "CLUSTER_NEW_NODES={zhongwc3}"
Starting Oracle Universal Installer...
Checking Temp space: must be greater than 120 MB. Actual 45068 MB Passed
Checking swap space: must be greater than 150 MB. Actual 8176 MB Passed
Prepare Configuration in progress.
Prepare Configuration successful.
.................................................. 9% Done.
You can find the log of this install session at:
/u01/app/oraInventory/logs/addNodeActions2013-10-21_09-37-23PM.log
Instantiate files in progress.
Instantiate files successful.
.................................................. 15% Done.
Copying files to node in progress.
Copying files to node successful.
.................................................. 79% Done.
Saving cluster inventory in progress.
.................................................. 87% Done.
Saving cluster inventory successful.
The Cluster Node Addition of /u01/app/oracle/product/12.1.0/db_1 was successful.
Please check '/tmp/silentInstall.log' for more details.
As a root user, execute the following script(s):
1. /u01/app/oracle/product/12.1.0/db_1/root.sh
Execute /u01/app/oracle/product/12.1.0/db_1/root.sh on the following nodes:
[zhongwc3]
..........
Update Inventory in progress.
.................................................. 100% Done.
Update Inventory successful.
Successfully Setup Software.[root@zhongwc3 ~]# /u01/app/oracle/product/12.1.0/db_1/root.sh[root@zhongwc3 ~]# cat /u01/app/oracle/product/12.1.0/db_1/install/root_zhongwc3_2013-10-21_21-50-25.log
Performing root user operation for Oracle 12c
The following environment variables are set as:
ORACLE_OWNER= oracle
ORACLE_HOME= /u01/app/oracle/product/12.1.0/db_1
Copying dbhome to /usr/local/bin ...
Copying oraenv to /usr/local/bin ...
Copying coraenv to /usr/local/bin ...
Entries will be added to the /etc/oratab file as needed by
Database Configuration Assistant when a database is created
Finished running generic part of root script.
Now product-specific root actions will be performed.Add Instance to Clustered Database
A database instance will be established on the new node. Specifically, an instance named "zwc3" will be added to "zwc" – a pre-existing clustered database.
Satisfy Node Instance Dependencies
Satisfy all node instance dependencies, such as init.ora parameters, etc.
From the new node "zhongwc3", run the following commands to create the passwordfile, "init.ora" file, and "oratab" entry for the new instance
[oracle@zhongwc3 ~]$ cd $ORACLE_HOME/dbs
[oracle@zhongwc3 dbs]$ mv initzwc1.ora initzwc3.ora
[oracle@zhongwc3 dbs]$ echo "zwc:$ORACLE_HOME:N" >> /etc/oratab From a node with an existing instance of "zwc" issue the following commands to create the needed public log thread, undo tablespace, and "init.ora" entries for the new instance
[oracle@zhongwc1 ~]$ sqlplus / as sysdba
SQL*Plus: Release 12.1.0.1.0 Production on Mon Oct 21 22:03:06 2013
Copyright (c) 1982, 2013, Oracle. All rights reserved.
Connected to:
Oracle Database 12c Enterprise Edition Release 12.1.0.1.0 - 64bit Production
With the Partitioning, Real Application Clusters, Automatic Storage Management, OLAP,
Advanced Analytics and Real Application Testing options
SQL> alter database add logfile thread 3 group 5 ('+DATADG','+FRADG') size 50M, group 6 ('+DATADG','+FRADG') size 50M;
Database altered.
SQL> alter database enable public thread 3;
Database altered.
SQL> create undo tablespace undotbs3 datafile '+DATADG' size 200M autoextend on;
Tablespace created.
SQL> alter system set undo_tablespace='undotbs3' scope=spfile sid='zwc3';
System altered.
SQL> alter system set instance_number=3 scope=spfile sid='zwc3';
System altered.
SQL> alter system set cluster_database_instances=3 scope=spfile sid='*';
System altered.
Update Oracle Cluster Registry (OCR)
The OCR will be updated to account for a new instance – "zwc3" – being added to the "zwc" cluster database .
[oracle@zhongwc3 ~]$ srvctl add instance -d zwc -i zwc3 -n zhongwc3
[oracle@zhongwc3 ~]$ srvctl status database -d zwc -v
Instance zwc1 is running on node zhongwc1 with online services crm. Instance status: Open.
Instance zwc2 is running on node zhongwc2 with online services erp. Instance status: Open.
Instance zwc3 is not running on node zhongwc3
[oracle@zhongwc3 ~]$
[oracle@zhongwc3 ~]$ srvctl config database -d zwc
Database unique name: zwc
Database name: zwc
Oracle home: /u01/app/oracle/product/12.1.0/db_1
Oracle user: oracle
Spfile: +DATADG/zwc/spfilezwc.ora
Password file: +DATADG/zwc/orapwzwc
Domain:
Start options: open
Stop options: immediate
Database role: PRIMARY
Management policy: AUTOMATIC
Server pools: zwc
Database instances: zwc1,zwc2,zwc3
Disk Groups: FRADG,DATADG
Mount point paths:
Services: crm,erp
Type: RAC
Start concurrency:
Stop concurrency:
Database is administrator managedStart "zwc3" Instance
Now that all the prerequisites have been satisfied and OCR updated, the "zwc3" instance will be started.
Start the newly created instance – "zwc3" – and verify
[oracle@zhongwc3 ~]$ srvctl start instance -d zwc -i zwc3
[oracle@zhongwc3 ~]$
[oracle@zhongwc3 ~]$ srvctl status databse -d zwc -v
Usage: srvctl <command> <object> [<options>]
commands: enable|disable|export|import|start|stop|relocate|status|add|remove|modify|getenv|setenv|unsetenv|config|convert|upgrade|downgrade|predict
objects: database|instance|service|nodeapps|vip|network|asm|ioserver|diskgroup|listener|srvpool|server|scan|scan_listener|oc4j|home|filesystem|gns|cvu|havip|exportfs|gridhomeserver|gridhomeclient|mgmtdb|mgmtlsnr|volume
For detailed help on each command and object and its options use:
srvctl <command> -help [-compatible] or
srvctl <command> <object> -help [-compatible]
PRKO-2001 : Invalid command line syntax
[oracle@zhongwc3 ~]$
[oracle@zhongwc3 ~]$ srvctl status database -d zwc -v
Instance zwc1 is running on node zhongwc1 with online services crm. Instance status: Open.
Instance zwc2 is running on node zhongwc2 with online services erp. Instance status: Open.
Instance zwc3 is running on node zhongwc3. Instance status: Open.[oracle@zhongwc2 ~]$ scp $ORACLE_HOME/network/admin/tnsnames.ora zhongwc3:$ORACLE_HOME/network/admin
The authenticity of host 'zhongwc3 (192.168.1.33)' can't be established.
RSA key fingerprint is 24:78:f8:dc:52:f4:36:5a:e6:ae:77:2e:2d:40:6a:33.
Are you sure you want to continue connecting (yes/no)? yes
Warning: Permanently added 'zhongwc3,192.168.1.33' (RSA) to the list of known hosts.
tnsnames.ora 100% 333 0.3KB/s 00:00 [oracle@zhongwc3 admin]$ sqlplus sys@zwc as sysdba
SQL*Plus: Release 12.1.0.1.0 Production on Mon Oct 21 22:16:24 2013
Copyright (c) 1982, 2013, Oracle. All rights reserved.
Enter password:
Connected to:
Oracle Database 12c Enterprise Edition Release 12.1.0.1.0 - 64bit Production
With the Partitioning, Real Application Clusters, Automatic Storage Management, OLAP,
Advanced Analytics and Real Application Testing options
SQL> col host_name format a11
SQL> set line 300
SQL> alter session set nls_date_format='yyyy-mm-dd hh24:mi:ss';
Session altered.
SQL> select INSTANCE_NAME,HOST_NAME,VERSION,STARTUP_TIME,STATUS,ACTIVE_STATE,INSTANCE_ROLE,DATABASE_STATUS from gv$INSTANCE;
INSTANCE_NAME HOST_NAME VERSION STARTUP_TIME STATUS ACTIVE_ST INSTANCE_ROLE DATABASE_STATUS
---------------- ----------- ----------------- ------------------- ------------ --------- ------------------ -----------------
zwc3 zhongwc3 12.1.0.1.0 2013-10-21 22:12:58 OPEN NORMAL PRIMARY_INSTANCE ACTIVE
zwc2 zhongwc2 12.1.0.1.0 2013-10-21 17:34:04 OPEN NORMAL PRIMARY_INSTANCE ACTIVE
zwc1 zhongwc1 12.1.0.1.0 2013-10-21 17:29:16 OPEN NORMAL PRIMARY_INSTANCE ACTIVE
[root@zhongwc3 ~]# crsctl query crs activeversion
Oracle Clusterware active version on the cluster is [12.1.0.1.0]
[root@zhongwc3 ~]#
[root@zhongwc3 ~]# crsctl query css votedisk
## STATE File Universal Id File Name Disk group
-- ----- ----------------- --------- ---------
1. ONLINE 6cafa9cf7bcb4f25bf51469905ac8ef9 (/dev/asm-diskb) [SYSDG]
Located 1 voting disk(s).
[root@zhongwc3 ~]#
[root@zhongwc3 ~]# srvctl status nodeapps
VIP zhongwc1-vip is enabled
VIP zhongwc1-vip is running on node: zhongwc1
VIP zhongwc2-vip is enabled
VIP zhongwc2-vip is running on node: zhongwc2
VIP zhongwc3-vip is enabled
VIP zhongwc3-vip is running on node: zhongwc3
Network is enabled
Network is running on node: zhongwc1
Network is running on node: zhongwc2
Network is running on node: zhongwc3
ONS is enabled
ONS daemon is running on node: zhongwc1
ONS daemon is running on node: zhongwc2
ONS daemon is running on node: zhongwc3[root@zhongwc3 ~]# crsctl stat res -t
--------------------------------------------------------------------------------
Name Target State Server State details
--------------------------------------------------------------------------------
Local Resources
--------------------------------------------------------------------------------
ora.DATADG.dg
ONLINE ONLINE zhongwc1 STABLE
ONLINE ONLINE zhongwc2 STABLE
ONLINE ONLINE zhongwc3 STABLE
ora.FRADG.dg
ONLINE ONLINE zhongwc1 STABLE
ONLINE ONLINE zhongwc2 STABLE
ONLINE ONLINE zhongwc3 STABLE
ora.LISTENER.lsnr
ONLINE ONLINE zhongwc1 STABLE
ONLINE ONLINE zhongwc2 STABLE
ONLINE ONLINE zhongwc3 STABLE
ora.SYSDG.dg
ONLINE ONLINE zhongwc1 STABLE
ONLINE ONLINE zhongwc2 STABLE
ONLINE ONLINE zhongwc3 STABLE
ora.asm
ONLINE ONLINE zhongwc1 Started,STABLE
ONLINE ONLINE zhongwc2 Started,STABLE
ONLINE ONLINE zhongwc3 Started,STABLE
ora.net1.network
ONLINE ONLINE zhongwc1 STABLE
ONLINE ONLINE zhongwc2 STABLE
ONLINE ONLINE zhongwc3 STABLE
ora.ons
ONLINE ONLINE zhongwc1 STABLE
ONLINE ONLINE zhongwc2 STABLE
ONLINE ONLINE zhongwc3 STABLE
--------------------------------------------------------------------------------
Cluster Resources
--------------------------------------------------------------------------------
ora.LISTENER_SCAN1.lsnr
1 ONLINE ONLINE zhongwc1 STABLE
ora.MGMTLSNR
1 ONLINE ONLINE zhongwc1 169.254.198.87 172.1
68.1.31,STABLE
ora.cvu
1 ONLINE ONLINE zhongwc1 STABLE
ora.mgmtdb
1 ONLINE ONLINE zhongwc1 Open,STABLE
ora.oc4j
1 ONLINE ONLINE zhongwc1 STABLE
ora.scan1.vip
1 ONLINE ONLINE zhongwc1 STABLE
ora.zhongwc1.vip
1 ONLINE ONLINE zhongwc1 STABLE
ora.zhongwc2.vip
1 ONLINE ONLINE zhongwc2 STABLE
ora.zhongwc3.vip
1 ONLINE ONLINE zhongwc3 STABLE
ora.zwc.crm.svc
1 ONLINE ONLINE zhongwc1 STABLE
ora.zwc.db
1 ONLINE ONLINE zhongwc1 Open,STABLE
2 ONLINE ONLINE zhongwc2 Open,STABLE
3 ONLINE ONLINE zhongwc3 Open,STABLE
ora.zwc.erp.svc
1 ONLINE ONLINE zhongwc2 STABLE
--------------------------------------------------------------------------------
342
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
