Oracle11g_RAC_OELR5-U4_iSCSI（非域名解析）-添加第三个节点

ClusterWare软件(Grid)   本地文件系统        本地磁盘
    voting disk(Grid)       ASM                 共享磁盘
    OCR(Grid)               ASM                 共享磁盘
    数据库软件              本地文件系统        本地磁盘
    数据库                  ASM                 共享磁盘



#public
192.168.10.101 rac1.fedge.com rac1
192.168.10.102 rac2.fedge.com rac2
192.168.10.103 rac3.fedge.com rac3
#vip
192.168.10.11   rac1-vip
192.168.10.12   rac2-vip
192.168.10.13   rac3-vip
#private
100.100.0.101 rac1-priv
100.100.0.102 rac2-priv
100.100.0.103 rac3-priv
#scan
192.168.10.170 scan.fedge.com rac-scan
192.168.10.171 scan.fedge.com rac-scan
192.168.10.172 scan.fedge.com rac-scan
#dns
192.168.10.4  dns.fedge.com  dns12c

===========================================
#rac01,rac02上的/etc/resolv.conf，添加

search fedge.com
nameserver 192.168.10.4

cat >> /etc/sysconfig/network-scripts/ifcfg-eth0 <<EOF
DNS1=192.168.10.4
EOF

======================================
wget http://public-yum.oracle.com/public-yum-ol6.repo

yum install binutils-2.* compat-libstdc++-33* elfutils-libelf-0.* elfutils-libelf-devel-* gcc-4.* gcc-c++-4.* glibc-2.* glibc-common-2.* glibc-devel-2.* glibc-headers-2.* ksh-2* libaio-0.* libaio-devel-0.* libgcc-4.* libstdc++-4.* libstdc++-devel-4.* make-3.* sysstat-7.* unixODBC-2.* unixODBC-devel-2.* pdksh*



rpm -q binutils compat-libstdc++-33 elfutils-libelf elfutils-libelf-devel gcc gcc-c++
rpm -q glibc glibc-common glibc-devel glibc-headers ksh libaio libaio-devel
rpm -q libgcc libstdc++ libstdc++-devel make numactl-devel sysstat unixODBC unixODBC-devel

mount /dev/cdrom /mnt/hgfs/
rpm -ivh /mnt/hgfs/Server/Packages/numactl-devel-2.0.7-6.el6.x86_64.rpm
rpm -ivh /mnt/hgfs/Server/Packages/compat-db42-4.2.52-15.el6.x86_64.rpm
rpm -ivh /mnt/hgfs/Server/Packages/compat-db43-4.3.29-15.el6.x86_64.rpm
rpm -ivh /mnt/hgfs/Server/Packages/compat-db-4.6.21-15.el6.x86_64.rpm
rpm -ivh /mnt/hgfs/Server/Packages/compat-libcap1-1.10-1.x86_64.rpm
rpm -ivh /mnt/hgfs/Server/Packages/unixODBC-devel-2.2.14-12.el6_3.x86_64.rpm
rpm -ivh /mnt/hgfs/Server/Packages/libstdc++-devel-4.4.7-3.el6.x86_64.rpm
rpm -ivh /mnt/hgfs/Server/Packages/libaio-devel-0.3.107-10.el6.x86_64.rpm
rpm -ivh /mnt/hgfs/Server/Packages/kernel-headers-2.6.32-358.el6.x86_64.rpm
rpm -ivh /mnt/hgfs/Server/Packages/glibc-headers-2.12-1.107.el6.x86_64.rpm
rpm -ivh /mnt/hgfs/Server/Packages/glibc-devel-2.12-1.107.el6.x86_64.rpm
rpm -ivh /mnt/hgfs/Server/Packages/ksh-20100621-19.el6.x86_64.rpm

cat >> /etc/security/limits.conf <<EOF
oracle              soft    nproc   2047
oracle              hard    nproc   16384
oracle              soft    nofile  1024
oracle              hard    nofile  65536
oracle              soft    stack   10240
grid                soft    nproc   2047
grid                hard    nproc   16384
grid                soft    nofile  1024
grid                hard    nofile  65536
grid                soft    stack   10240
EOF

vi /etc/selinux/config
service iptables save
service iptables stop
/etc/rc.d/init.d/iptables status




#禁用 NTP  采用 CTSS
/sbin/service ntpd stop
chkconfig ntpd off
mv /etc/ntp.conf /etc/ntp.conf.original                       
rm /var/run/ntpd.pid       

rpm -ivh /mnt/hgfs/Server/Packages/iscsi-initiator-utils-*     

service iscsi start
service iscsid start
chkconfig iscsid on
chkconfig iscsi on
          
service iscsid start
chkconfig --list | grep iscsi
cat messages | grep iscsi


iscsiadm -m discovery -t sendtargets -p 100.100.0.43

100.100.0.43:3260,1 iqn.2006-01.com.openfiler:racdb.fra1
100.100.0.43:3260,1 iqn.2006-01.com.openfiler:racdb.data1
100.100.0.43:3260,1 iqn.2006-01.com.openfiler:racdb.crs1

iscsiadm -m node -T iqn.2006-01.com.openfiler:racdb.crs1 -p 100.100.0.43 -l
iscsiadm -m node -T iqn.2006-01.com.openfiler:racdb.data1 -p 100.100.0.43 -l
iscsiadm -m node -T iqn.2006-01.com.openfiler:racdb.fra1 -p 100.100.0.43 -l
iscsiadm -m node -T iqn.2006-01.com.openfiler:racdb.crs1 -p 100.100.0.43 --op update -n node.startup -v automatic
iscsiadm -m node -T iqn.2006-01.com.openfiler:racdb.data1 -p 100.100.0.43 --op update -n node.startup -v automatic
iscsiadm -m node -T iqn.2006-01.com.openfiler:racdb.fra1 -p 100.100.0.43 --op update -n node.startup -v automatic


(cd /dev/disk/by-path; ls -l *openfiler* | awk '{FS=" "; print $9 " " $10 " " $11}')

touch /etc/udev/rules.d/99-asm-iscsi.rules
cat >>/etc/udev/rules.d/99-asm-iscsi.rules <<EOF
SUBSYSTEM=="block",KERNEL=="sd*",ENV{ID_SERIAL}=="14f504e46494c45524f4452576c352d4d58386e2d4a7a6a53", BUS=="scsi",SYMLINK+="iscsi/crs1/part"
SUBSYSTEM=="block",KERNEL=="sd*",ENV{ID_SERIAL}=="14f504e46494c4552467248686c4c2d693570742d49314d6f", BUS=="scsi",SYMLINK+="iscsi/data1/part"
SUBSYSTEM=="block",KERNEL=="sd*",ENV{ID_SERIAL}=="14f504e46494c455255476461584a2d734a31382d63724845", BUS=="scsi",SYMLINK+="iscsi/fra1/part"
EOF


udevadm trigger
start_udev

ls -l /dev/iscsi/*


service iscsi stop
service iscsi start

**********************************************************************************************0
su - root
userdel grid
rm -rf /home/grid

userdel oracle
rm -rf /home/oracle

groupdel oinstall
groupdel dba
groupdel oper  
groupdel asmadmin
groupdel asmdba
groupdel asmoper

groupadd -g 1000 oinstall
groupadd -g 1200 asmadmin
groupadd -g 1201 asmdba
groupadd -g 1202 asmoper
groupadd -g 1300 dba
groupadd -g 1301 oper


useradd -m -u 1100 -g oinstall -G asmadmin,asmdba,asmoper -d /home/grid -s /bin/bash -c "Grid Infrastructure Owner" grid
passwd grid


useradd -m -u 1101 -g oinstall -G dba,oper,asmdba -d /home/oracle -s /bin/bash -c "Oracle Software Owner" oracle
passwd oracle


fdisk /dev/sdb
mkfs -t ext4 /dev/sdb1
cat >> /etc/fstab <<EOF
/dev/sdc1               /u01                    ext4    defaults        1 2
EOF


rm -fr /u01
mkdir /u01
mount /dev/sdc1 /u01

mkdir -p /u01/app/grid
mkdir -p /u01/app/oracle
mkdir -p /u01/app/11.2.0/grid
mkdir -p /u01/app/oraInventory

chown -R grid:oinstall /home/grid
chown -R grid:asmadmin /dev/iscsi/*
chown -R grid:oinstall /u01/app/grid
chown -R grid:oinstall /u01/app/11.2.0/grid
chown -R oracle:oinstall /u01/app/oracle
chown -R grid:oinstall  /u01/app/oraInventory

chmod -R 775 /u01/app/grid/
chmod -R 775 /u01/app/oracle/
chmod -R 775 /u01/app/11.2.0/grid
chmod -R 775 /u01/app/oraInventory



id grid
###########################
su - grid
cat /dev/null > .bash_profile
cat >> .bash_profile <<EOF
# ---------------------------------------------------
# .bash_profile
# ---------------------------------------------------
# OS User:      grid
# Application:  Oracle Grid Infrastructure
# Version:      Oracle 11g release 2
# ---------------------------------------------------

# Get the aliases and functions
if [ -f ~/.bashrc ]; then
    . ~/.bashrc
fi
alias ls="ls -FA"

ORACLE_SID=+ASM3; export ORACLE_SID

JAVA_HOME=/usr/local/java; export JAVA_HOME

ORACLE_BASE=/u01/app/grid; export ORACLE_BASE

ORACLE_HOME=/u01/app/11.2.0/grid; export ORACLE_HOME

ORACLE_PATH=/u01/app/oracle/common/oracle/sql; export ORACLE_PATH

ORACLE_TERM=xterm; export ORACLE_TERM

NLS_DATE_FORMAT="YYYY-MON-DD HH24:MI:SS"; export NLS_DATE_FORMAT

TNS_ADMIN=$ORACLE_HOME/network/admin; export TNS_ADMIN

ORA_NLS11=$ORACLE_HOME/nls/data; export ORA_NLS11

PATH=.:${JAVA_HOME}/bin:${PATH}:$HOME/bin:$ORACLE_HOME/bin
PATH=${PATH}:/usr/bin:/bin:/usr/bin/X11:/usr/local/bin
PATH=${PATH}:/u01/app/common/oracle/bin
export PATH

LD_LIBRARY_PATH=$ORACLE_HOME/lib
LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:$ORACLE_HOME/oracm/lib
LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/lib:/usr/lib:/usr/local/lib
export LD_LIBRARY_PATH

CLASSPATH=$ORACLE_HOME/JRE
CLASSPATH=${CLASSPATH}:$ORACLE_HOME/jlib
CLASSPATH=${CLASSPATH}:$ORACLE_HOME/rdbms/jlib
CLASSPATH=${CLASSPATH}:$ORACLE_HOME/network/jlib
export CLASSPATH

THREADS_FLAG=native; export THREADS_FLAG

export TEMP=/tmp
export TMPDIR=/tmp
export TMP=/tmp
export CVUQDISK_GRP=oinstall

export LANG=en_US.UTF-8

umask 022
EOF
-------------------

su - oracle

cat /dev/null > .bash_profile
cat >> .bash_profile <<EOF
# ---------------------------------------------------
# .bash_profile
# ---------------------------------------------------
# OS User:      oracle
# Application:  Oracle Database Software Owner
# Version:      Oracle 11g release 2
# ---------------------------------------------------

# Get the aliases and functions
if [ -f ~/.bashrc ]; then
      . ~/.bashrc
fi

alias ls="ls -FA"

ORACLE_SID=racdb3; export ORACLE_SID

ORACLE_UNQNAME=racdb; export ORACLE_UNQNAME

JAVA_HOME=/usr/local/java; export JAVA_HOME

ORACLE_BASE=/u01/app/oracle; export ORACLE_BASE

ORACLE_HOME=$ORACLE_BASE/product/11.2.0/dbhome_1; export ORACLE_HOME

ORACLE_PATH=/u01/app/common/oracle/sql; export ORACLE_PATH

ORACLE_TERM=xterm; export ORACLE_TERM

NLS_DATE_FORMAT="YYYY-MON-DD HH24:MI:SS"; export NLS_DATE_FORMAT

TNS_ADMIN=$ORACLE_HOME/network/admin; export TNS_ADMIN

ORA_NLS11=$ORACLE_HOME/nls/data; export ORA_NLS11

PATH=.:${JAVA_HOME}/bin:${PATH}:$HOME/bin:$ORACLE_HOME/bin
PATH=${PATH}:/usr/bin:/bin:/usr/bin/X11:/usr/local/bin
PATH=${PATH}:/u01/app/common/oracle/bin
export PATH

LD_LIBRARY_PATH=$ORACLE_HOME/lib
LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:$ORACLE_HOME/oracm/lib
LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/lib:/usr/lib:/usr/local/lib
export LD_LIBRARY_PATH

CLASSPATH=$ORACLE_HOME/JRE
CLASSPATH=${CLASSPATH}:$ORACLE_HOME/jlib
CLASSPATH=${CLASSPATH}:$ORACLE_HOME/rdbms/jlib
CLASSPATH=${CLASSPATH}:$ORACLE_HOME/network/jlib
export CLASSPATH

THREADS_FLAG=native; export THREADS_FLAG

export TEMP=/tmp
export TMPDIR=/tmp

export LANG=en_US.UTF-8

umask 022
EOF
----------------------
su - root

cat >>/etc/profile <<EOF
if [ \$USER = "oracle" ] || [ \$USER = "grid" ]; then
    if [ \$SHELL = "/bin/ksh" ]; then
        ulimit -p 16384
        ulimit -n 65536
    else
        ulimit -u 16384 -n 65536
    fi
    umask 022
fi
EOF

cat >> /etc/csh.login <<EOF
if ( \$USER == "oracle" || \$USER == "grid" ) then
    limit maxproc 16384
    limit descriptors 65536
endif
EOF

cat /proc/meminfo | grep MemTotal
cat /proc/meminfo | grep SwapTotal

dd if=/dev/zero of=tempswap bs=1k count=4000000
chmod 600 tempswap
mke2fs tempswap <<EOF
y
EOF

mkswap tempswap
swapon tempswap

cat /proc/meminfo | grep SwapTotal
*******************************************************************************************1

rpm -qa --queryformat "%{NAME}-%{VERSION}-%{RELEASE} (%{ARCH})\n"| grep ssh
#openssh-clients-5.3p1-84.1.el6 (x86_64)
#libssh2-1.4.2-1.el6 (x86_64)
#openssh-5.3p1-84.1.el6 (x86_64)
#openssh-server-5.3p1-84.1.el6 (x86_64)
#openssh-askpass-5.3p1-84.1.el6 (x86_64)

-------------------------------
#rac1节点 grid  oracle 各执行一次

#（替代方法）利用OUI 界面 cluster node -> setup 步骤安装, 或静默方式
(
su - grid
$ORACLE_HOME/oui/bin/runSSHSetup.sh -user grid -hosts "rac2 rac3" -advanced -exverify

su - oracle
$ORACLE_HOME/oui/bin/runSSHSetup.sh -user oracle -hosts "rac2 rac3" -advanced -exverify

)


touch ~/.ssh/authorized_keys
ls -l ~/.ssh
ssh rac1 cat ~/.ssh/id_dsa.pub >> ~/.ssh/authorized_keys

ssh rac3 cat ~/.ssh/id_dsa.pub >> ~/.ssh/authorized_keys

ls -l ~/.ssh

scp ~/.ssh/authorized_keys rac2:.ssh/authorized_keys

chmod 600 ~/.ssh/authorized_keys
-------------------------------------
ssh rac1 "date;hostname"

ssh rac3 "date;hostname"
-------------------------
ssh rac1 hostname
ssh rac3 hostname
------------------------
-------------------------
 cat /etc/sysctl.conf

cat >> /etc/sysctl.conf <<EOF

# Controls the maximum number of shared memory segments system wide
kernel.shmmni = 4096

# Sets the following semaphore values:
# SEMMSL_value  SEMMNS_value  SEMOPM_value  SEMMNI_value
kernel.sem = 250 32000 100 128

# Sets the maximum number of file-handles that the Linux kernel will allocate
fs.file-max = 6815744

# Defines the local port range that is used by TCP and UDP
# traffic to choose the local port
net.ipv4.ip_local_port_range = 9000 65500

# Default setting in bytes of the socket "receive" buffer which
# may be set by using the SO_RCVBUF socket option
net.core.rmem_default=262144

# Maximum setting in bytes of the socket "receive" buffer which
# may be set by using the SO_RCVBUF socket option
net.core.rmem_max=4194304

# Default setting in bytes of the socket "send" buffer which
# may be set by using the SO_SNDBUF socket option
net.core.wmem_default=262144

# Maximum setting in bytes of the socket "send" buffer which
# may be set by using the SO_SNDBUF socket option
net.core.wmem_max=1048576

# Maximum number of allowable concurrent asynchronous I/O requests requests
fs.aio-max-nr=1048576
EOF

sysctl -p

------------------------

scp /media/soft/oracleasmlib-2.0.4-1.el6.x86_64.rpm  rac3:/home/grid/oracleasmlib-2.0.4-1.el6.x86_64.rpm
scp /media/soft/oracleasm-support-2.1.8-1.el6.x86_64.rpm  rac3:/home/grid/oracleasm-support-2.1.8-1.el6.x86_64.rpm
ssh rac3 rpm -ivh /home/grid/oracleasmlib-2.0.4-1.el6.x86_64.rpm
ssh rac3 rpm -ivh /home/grid/oracleasm-support-2.1.8-1.el6.x86_64.rpm
----------------------
/usr/sbin/oracleasm configure -i <<EOF
grid
asmadmin
y
y
EOF
/usr/sbin/oracleasm init
------------------------
为了防止selinux阻止访问disk header
备份/etc/selinux/config
# cp /etc/selinux/config  /etc/selinux/config_bak

b)修改/etc/selinux/config
注释掉   SELINUX=enforcing
添加     SELINUX=disabled
vi /etc/selinux/config
# SELINUX=enforcing
  SELINUX=disabled
c)重启操作系统
# reboot

su - root

/usr/sbin/oracleasm scandisks
/usr/sbin/oracleasm listdisks

scp /media/soft/gridsoft/grid/rpm/cvuqdisk-1.0.9-1.rpm rac3:/home/grid/cvuqdisk-1.0.9-1.rpm
ssh root@rac3 'rpm -ivh /home/grid/cvuqdisk-1.0.9-1.rpm'

vi /etc/sysconfig/ntpd
# Drop root to id 'ntp:ntp' by default.
OPTIONS="-x -u ntp:ntp -p /var/run/ntpd.pid"
# Set to 'yes' to sync hw clock after successful ntpdate
SYNC_HWCLOCK=no
# Additional options for ntpdate
NTPDATE_OPTIONS=""

*********************************************************************
-----------------------------------
(注意：配置 DNS无需此步骤)
mv /usr/bin/nslookup /usr/bin/nslookup.original

cat >> /usr/bin/nslookup <<EOF
#!/bin/bash

HOSTNAME=${1}

if [[ $HOSTNAME = "rac-scan" ]]; then
    echo "Server:          24.154.1.34"
    echo "Address:         24.154.1.34#53"
    echo "Non-authoritative answer:"
    echo "Name:    rac-scan"
    echo "Address:  192.168.0.172"
else
    /usr/bin/nslookup.original $HOSTNAME
fi
EOF

chmod 755 /usr/bin/nslookup
------------------------------------20170407
su - grid
[grid@rac1 ~]$ $ORACLE_HOME/bin/cluvfy stage -post hwos -n rac3
[grid@rac1 ~]$ $ORACLE_HOME/bin/cluvfy comp peer -refnode rac1 -n rac3 -orainv oinstall -osdba dba -verbose
[grid@rac1 ~]$ $ORACLE_HOME/bin/cluvfy stage -pre nodeadd -n rac3 -fixup -verbose
[grid@rac1 ~]$ cd $ORACLE_HOME/oui/bin
[grid@rac1 ~]$ export IGNORE_PREADDNODE_CHECKS=Y
[grid@rac1 ~]$ ./addNode.sh -silent -ignoreSysPrereqs -force "CLUSTER_NEW_NODES={rac3}" "CLUSTER_NEW_VIRTUAL_HOSTNAMES={rac3-vip}"

/u01/app/oraInventory/orainstRoot.sh
/u01/app/11.2.0/grid/root.sh
-------------------------------------------
[grid@rac1 ~]$ $ORACLE_HOME/bin/cluvfy stage -post nodeadd -n rac3 -verbose

$ORACLE_HOME/bin/crsctl check crs

$ORACLE_HOME/bin/olsnodes -n

ps -ef | grep lsnr | grep -v 'grep' | grep -v 'ocfs' | awk '{print $9}'

$ORACLE_HOME/bin/srvctl status asm -a

$ORACLE_HOME/bin/ocrcheck

$ORACLE_HOME/bin/crsctl query css votedisk

su - oracle
[oracle@rac1 ~]$ cd $ORACLE_HOME/oui/bin
[oracle@rac1 ~]$ ./addNode.sh -silent "CLUSTER_NEW_NODES={rac3}"
/u01/app/oracle/product/11.2.0/dbhome_1/root.sh

----------------------------------------------------
[grid@rac1 ~]$ crsctl status resource -t
发现rac3的vip, listener都没有成功，手动添加

[root@rac1 ~]# /u01/app/11.2.0/grid/bin/srvctl remove vip -i rac3
[root@rac1 ~]# /u01/app/11.2.0/grid/bin/srvctl add vip -n rac3 -A rac3-vip/255.255.255.0 -k 1
[grid@rac1 ~]$ srvctl start vip -n rac3
[grid@rac1 ~]$ srvctl start listener -n rac3
[grid@rac1 ~]$ crsctl status resource -t
[oracle@rac1 ~]$ srvctl add service -s TAF -d racdb -r racdb3 -u
[oracle@rac1 ~]$ srvctl modify service -s TAF -d racdb -q true -e SELECT -m BASIC -w 5 -z 180
[oracle@rac1 ~]$ srvctl config service -s TAF -d racdb
[oracle@rac1 ~]$ srvctl start service -s TAF -i racdb3 -d racdb

[oracle@rac1 ~]$ emca -displayConfig dbcontrol -cluster