简单Dream-风格rac笔记

rac1   rac2
内存2560M

root          30G      scsi0       20                       
orainstall    40                                 
tempsoft      20                                
                                              
gird1         1        scsi1                        
grid2         1                                
system        21                                 
recovery      22                                
data01        23                                
data02        24                                
                                              
                                              

2 oracle 数据库安装环境配置

需要安装的软件列表：
先把光驱放到操作系统中挂载在/mnt目录，然后通过yum安装以下软件包。
[root@db01 /]# vi /etc/yum.conf
[EL]
name =Linux 6.5 DVD
baseurl=file:///mnt/Server/
gpgcheck=0
enabled=1

mv /etc/yum.repos.d/public-yum-ol6.repo  /etc/yum.repos.d/public-yum-ol6.repo123
yum clean all

yum install binutils -y
yum install compat-libcap1 -y
yum install compat-libstdc++-33 -y
yum install gcc -y
yum install gcc-c++ -y
yum install glibc -y
yum install glibc-devel -y
yum install ksh -y
yum install libgcc -y
yum install libstdc++ -y
yum install libstdc++-devel -y
yum install libaio -y
yum install libaio-devel -y
yum install libXext -y
yum install libXtst -y
yum install libX11 -y
yum install libXau -y
yum install libxcb -y
yum install libXi -y
yum install make -y
yum install sysstat -y
yum install unixODBC -y
yum install unixODBC-devel -y

 

2.3两台主机节点编辑/etc/hosts文件，添加如下内容（IP地址根据实际情况）：

public ip ,vip,scan ip必须在同一个网段，scan ip 一定要单独一个ip

vi /etc/hosts
如果hosts文件中没有下面这行需要添加上
27.0.0.1 localhost.localdomain localhost

#public ip ent0
10.131.6.201  itpux1
10.131.6.202  itpux2

#private ip ent1
10.10.10.201 itpux1prv
10.10.10.202 itpux2prv

#virtual ip
10.131.6.203 itpux1vip
10.131.6.204 itpux2vip

#scan ip
10.131.6.205 itpuxscan

注意主机名不要使用大写字母，且主机名不能有_下划线和-横线

2.4添加组与用户
在两个节点增加用户与组：
/usr/sbin/groupadd -g 501 oinstall
/usr/sbin/groupadd -g 502 dba
/usr/sbin/groupadd -g 503 oper
/usr/sbin/groupadd -g 504 asmadmin
/usr/sbin/groupadd -g 505 asmoper
/usr/sbin/groupadd -g 506 asmdba
/usr/sbin/useradd -g oinstall -G dba,asmdba,oper oracle
/usr/sbin/useradd -g oinstall -G asmadmin,asmdba,asmoper,oper,dba grid
 
 
[root@ora1 ~]# id oracle
uid=501(oracle) gid=501(oinstall) groups=501(oinstall),502(dba),503(oper),506(asmdba)
[root@ora1 ~]# id grid
uid=502(grid) gid=501(oinstall) groups=501(oinstall),502(dba),503(oper),504(asmadmin),505(asmoper),506(asmdba)
 
 
修改用户密码：
passwd grid
passwd oracle
登陆测试（必须做），分别用oracle,grid用户通过ssh登录，按提示修改密码
 
 
2.5 添加文件系统
 fdiks
 mkfs.ext4 /dev/sdb1

[root@itpux2 ~]# mkdir /oracle
[root@itpux2 ~]# mkdir /soft
 
mount /dev/sdb1 /oracle

vi /etc/fstab
 
[grid@db01:/home/grid]$df -h
Filesystem            Size  Used Avail Use% Mounted on
/dev/sda1              20G  4.1G   15G  22% /
tmpfs                 1.9G  643M  1.3G  34% /dev/shm
/dev/sdc1              38G  9.5G   27G  27% /oracle
/dev/sdb1              40G  9.0G   29G  24% /soft
 
 
mkdir -p /oracle/app/grid
mkdir -p /oracle/app/11.2.0/grid
chown -R grid:oinstall /oracle
 
mkdir -p /oracle/app/oraInventory
chown -R grid:oinstall /oracle/app/oraInventory
 
mkdir -p /oracle/app/oracle
chown -R oracle:oinstall /oracle/app/oracle
chmod -R 775 /oracle
 
 登录grid oracle测试一下
 
2.6 修改操作系统参数
 
1、修改/etc/security/limits.conf文件，增加内容如下：
vi /etc/security/limits.conf
 
#ORACLE SETTING
grid                 soft    nproc   2047 
grid                 hard    nproc   16384
grid                 soft    nofile  1024
grid                 hard    nofile  65536
grid       soft   stack    10240
grid       hard   stack    32768
oracle               soft    nproc   2047
oracle               hard    nproc   16384
oracle               soft    nofile  1024
oracle               hard    nofile  65536
oracle      soft   stack    10240
oracle     hard   stack    32768

 

2、修改/etc/security/limits.d/90-nproc.conf文件内容
Amend the "/etc/security/limits.d/90-nproc.conf" file as described below. See MOS Note [ID 1487773.1]
# Change this
*          soft    nproc    1024
# To this
* - nproc 16384

3、禁用SELINUX
 
Change the setting of SELinux to permissive by editing the "/etc/selinux/config" file, making sure the SELINUX flag is set as follows.
SELINUX=disabled
 
让SELINUX生效：
setenforce 0

4、关闭iptables防火墙
 
If you have the Linux firewall enabled, you will need to disable or configure it, as shown here or here. The following is an example of disabling the firewall.
# service iptables stop
# chkconfig iptables off
可以用setup关闭防火墙
 
5、修改/etc/pam.d/login文件
 
vi /etc/pam.d/login
 
#ORACLE SETTING
session    required     pam_limits.so
 
6、修改sysctl.conf配置文件
 
# vi /etc/sysctl.conf
#ORACLE SETTING
fs.aio-max-nr = 1048576
fs.file-max = 6815744
kernel.shmmax = 2147483648
kernel.shmall = 524288
kernel.shmmni = 4096
kernel.sem = 250 32000 100 128
net.ipv4.ip_local_port_range = 9000 65500
net.core.rmem_default = 262144
net.core.rmem_max = 4194304
net.core.wmem_default = 262144
net.core.wmem_max = 1048586
 
#kernel.shmmax = 2147483648  (大于SGA，小于物理内存)
#kernel.shmall = 524288   （物理内存/4K）
如果以上参数设置不正确，可能导致以下报错：
ORA-27102: out of memory Linux-X86_64 Error: 28: No space left on device .
 
让配置生效：
sysctl -p

7、shm的修改
 
/dev/shm 共享内存不足的处理
解决方法：
例如：为了将/dev/shm的大小增加到1GB，修改/etc/fstab的这行：默认的： 
none     /dev/shm       tmpfs   defaults        0 0
改成： 
none     /dev/shm       tmpfs   defaults,size=1025m        0 0
size参数也可以用G作单位：size＝2G。
重新mount /dev/shm使之生效：
# mount -o remount /dev/shm
或者：
# umount /dev/shm
# mount -a
马上可以用"df -h"命令检查变化。
 
8、禁止NTP
Either configure NTP, or make sure it is not configured so the Oracle Cluster Time Synchronization Service (ctssd) can synchronize the times of the RAC nodes. If you want to deconfigure NTP do the following, which is what I did for this installation.
# service ntpd stop
Shutting down ntpd:                                        [  OK  ]
# chkconfig ntpd off
# mv /etc/ntp.conf /etc/ntp.conf.orig
# rm /var/run/ntpd.pid

 

2.7 配置grid和oracle用户的环境变量
su - grid
grid用户的环境变量如下：
 
PS1="[`whoami`@`hostname`:"'$PWD]$'
export PS1
umask 022
alias sqlplus="rlwrap sqlplus"
export TMP=/tmp
export LANG=en_US
export TMPDIR=$TMP
export ORACLE_HOSTNAME=itpux1
ORACLE_SID=+ASM1; export ORACLE_SID
ORACLE_TERM=xterm; export ORACLE_TERM
ORACLE_BASE=/oracle/app/grid; export ORACLE_BASE
ORACLE_HOME=/oracle/app/11.2.0/grid; export ORACLE_HOME
NLS_DATE_FORMAT="yyyy-mm-dd HH24:MI:SS"; export NLS_DATE_FORMAT
PATH=.:$PATH:$HOME/bin:$ORACLE_HOME/bin; export PATH
THREADS_FLAG=native; export THREADS_FLAG
if [ $USER = "oracle" ] || [ $USER = "grid" ]; then
        if [ $SHELL = "/bin/ksh" ]; then
            ulimit -p 16384
              ulimit -n 65536
  else
   ulimit -u 16384 -n 65536
      fi
    umask 022
fi
 
su - oracle
oracle用户的环境变量如下： 
PS1="[`whoami`@`hostname`:"'$PWD]$'
alias sqlplus="rlwrap sqlplus"
alias rman="rlwrap rman"
export PS1
export TMP=/tmp
export LANG=en_US
export TMPDIR=$TMP
export ORACLE_HOSTNAME=itpux1
export ORACLE_UNQNAME=itpuxdb
ORACLE_BASE=/oracle/app/oracle; export ORACLE_BASE
ORACLE_HOME=$ORACLE_BASE/product/11.2.0/db_1; export ORACLE_HOME
ORACLE_SID=itpux1; export ORACLE_SID
ORACLE_TERM=xterm; export ORACLE_TERM
NLS_DATE_FORMAT="yyyy-mm-dd HH24:MI:SS"; export NLS_DATE_FORMAT
NLS_LANG=AMERICAN_AMERICA.ZHS16GBK;export NLS_LANG
PATH=.:$PATH:$HOME/bin:$ORACLE_BASE/product/11.2.0/db_1/bin:$ORACLE_HOME/bin; export PATH
THREADS_FLAG=native; export THREADS_FLAG
if [ $USER = "oracle" ] || [ $USER = "grid" ]; then
        if [ $SHELL = "/bin/ksh" ]; then
            ulimit -p 16384
              ulimit -n 65536
  else
   ulimit -u 16384 -n 65536
      fi
    umask 022
fi
 
 
节点二：
 
su - grid
grid的环境变量
 
PS1="[`whoami`@`hostname`:"'$PWD]$'
export PS1
umask 022
alias sqlplus="rlwrap sqlplus"
export TMP=/tmp
export LANG=en_US
export TMPDIR=$TMP
export ORACLE_HOSTNAME=itpux2
ORACLE_SID=+ASM2; export ORACLE_SID
ORACLE_TERM=xterm; export ORACLE_TERM
ORACLE_BASE=/oracle/app/grid; export ORACLE_BASE
ORACLE_HOME=/oracle/app/11.2.0/grid; export ORACLE_HOME
NLS_DATE_FORMAT="yyyy-mm-dd HH24:MI:SS"; export NLS_DATE_FORMAT
PATH=.:$PATH:$HOME/bin:$ORACLE_HOME/bin; export PATH
THREADS_FLAG=native; export THREADS_FLAG
if [ $USER = "oracle" ] || [ $USER = "grid" ]; then
        if [ $SHELL = "/bin/ksh" ]; then
            ulimit -p 16384
              ulimit -n 65536
  else
   ulimit -u 16384 -n 65536
      fi
    umask 022
fi
 
 
su - oracle
 
PS1="[`whoami`@`hostname`:"'$PWD]$'
alias sqlplus="rlwrap sqlplus"
alias rman="rlwrap rman"
export PS1
export TMP=/tmp
export LANG=en_US
export TMPDIR=$TMP
export ORACLE_HOSTNAME=itpux2
export ORACLE_UNQNAME=itpuxdb
ORACLE_BASE=/oracle/app/oracle; export ORACLE_BASE
ORACLE_HOME=$ORACLE_BASE/product/11.2.0/db_1; export ORACLE_HOME
ORACLE_SID=itpux2; export ORACLE_SID
ORACLE_TERM=xterm; export ORACLE_TERM
NLS_DATE_FORMAT="yyyy-mm-dd HH24:MI:SS"; export NLS_DATE_FORMAT
NLS_LANG=AMERICAN_AMERICA.ZHS16GBK;export NLS_LANG
PATH=.:$PATH:$HOME/bin:$ORACLE_BASE/product/11.2.0/db_1/bin:$ORACLE_HOME/bin; export PATH
THREADS_FLAG=native; export THREADS_FLAG
if [ $USER = "oracle" ] || [ $USER = "grid" ]; then
        if [ $SHELL = "/bin/ksh" ]; then
                    ulimit -p 16384
              ulimit -n 65536
  else
   ulimit -u 16384 -n 65536
      fi
    umask 022
fi

重新登录，使其生效
然后env |grep ORACLE --检查是否生效，生效会出现oracle开头的变量设置

 

2.8 3.9 配置SSH信任关系
配置信任关系
设置SSH，
1).在主节点RAC1上以grid,oracle用户身份生成用户的公匙和私匙
# su - oracle  （su - grid）
$ mkdir ~/.ssh
$ ssh-keygen -t rsa
$ ssh-keygen -t dsa
 
2).在副节点RAC2上执行相同的操作，确保通信无阻
# su - oracle  （su - grid）
$ mkdir ~/.ssh
$ ssh-keygen -t rsa
$ ssh-keygen -t dsa
 
3).在主节点RAC1上oracle(grid)用户执行以下操作
$ cat ~/.ssh/id_rsa.pub >> ./.ssh/authorized_keys
$ cat ~/.ssh/id_dsa.pub >> ./.ssh/authorized_keys
$ ssh itpux2 cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
$ ssh itpux2 cat ~/.ssh/id_dsa.pub >> ~/.ssh/authorized_keys
$ scp ~/.ssh/authorized_keys itpux2:~/.ssh/authorized_keys
 
4).主节点RAC1上执行检验操作
$ ssh itpux1 date
$ ssh itpux2 date
$ ssh itpux1prv date
$ ssh itpux2prv date
 
5).在副节点RAC2上执行检验操作
$ ssh itpux1 date
$ ssh itpux2 date
$ ssh itpux1prv date
$ ssh itpux2prv date

asm安装包（节点12同样的操作）
oracleasmlib-2.0.4-1el6.x86_64---百度oracle asmlib，在官网下载
oracleasm-support-2.1.8-1.el6.x86_64.rpm--安装介质里面有

asm配置(节点12同样的操作)
[root@itpux1 soft]# /etc/init.d/oracleasm configure
Configuring the Oracle ASM library driver.

This will configure the on-boot properties of the Oracle ASM library
driver.  The following questions will determine whether the driver is
loaded on boot and what permissions it will have.  The current values
will be shown in brackets ('[]').  Hitting <ENTER> without typing an
answer will keep that current value.  Ctrl-C will abort.

Default user to own the driver interface []: grid
Default group to own the driver interface []: asmadmin
Start Oracle ASM library driver on boot (y/n) [n]: y
Scan for Oracle ASM disks on boot (y/n) [y]: y
Writing Oracle ASM library driver configuration: done
Initializing the Oracle ASMLib driver:                     [  OK  ]
Scanning the system for Oracle ASMLib disks:               [  OK  ]

 

配置磁盘
 
关闭主机
 
1、磁盘使用规划
dggrid1 1个1g   sdd
dggrid2 1个1g  sde
dgsystem 21g   sdf
dgrecovery 22g   sdg
dgdata01 23g   sdh
dgdata02 24g   sdi
 
 
2、虚拟机配置
虚拟机磁盘配置
 
disk.locking = "FALSE"
diskLib.dataCacheMaxSize = "0"
diskLib.dataCacheMaxReadAheadSize = "0"
diskLib.dataCacheMinReadAheadSize = "0"
diskLib.dataCachePageSize = "4096"
diskLib.maxUnsyncedWrites = "0"
disk.EnableUUID = "TRUE"
scsi1.sharedBus = "virtual"
scsi1.virtualDev = "lsilogic"

 

把每个磁盘格式化成一个区
fdisk -l
fdiks /dev/sdd  n p 1  w
第一台主机分区之后，第二台主机就不用在分区了
创建asm盘
+DGGRID1
+DGGRID2 
+DGSYSTEM
+DGRECOVERY
+DGDATA01
+DGDATA02

/etc/init.d/oracleasm createdisk dggrid1 /dev/sdd1
/etc/init.d/oracleasm createdisk dggrid2 /dev/sde1
/etc/init.d/oracleasm createdisk dgsystem /dev/sdf1
/etc/init.d/oracleasm createdisk dgrecovery /dev/sdg1
/etc/init.d/oracleasm createdisk dgdata01 /dev/sdh1
/etc/init.d/oracleasm createdisk dgdata02 /dev/sdi1

扫描磁盘
/etc/init.d/oracleasm scandisks

/etc/init.d/oracleasm listdisks

主机1完成后，主机二直接扫描

安装grid软件需要vnc，
再root下安装vnc包，（安装介质里有）
rpm -ivh libvncserver-0.9.7-4.el6.x86_64.rpm
rpm -ivh tigervnc-1.1.0-5.el6_4.1.x86_64.rpm
rpm -ivh tigervnc-server-1.1.0-5.el6_4.1.x86_64.rpm
rpm -ivh gtk-vnc-0.3.10-3.el6.x86_64.rpm
rpm -ivh gtk-vnc-python-0.3.10-3.el6.x86_64.rpm

然后主机12进行下面操作
安装cvuqdisk包并验证

chown -R grid:oinstall /soft
到grid用户下解压grid软件
  
解压完成后到root用户下
设置环境变量CVUQDISK_GRP,使其指向作为cvuqdisk的所有者所在的组（本次为oinstall）
   export CVUQDISK_GRP=oinstall

在root下安装
[root@itpux1 ~]# cd /soft/grid/rpm/
[root@itpux1 rpm]# ls
cvuqdisk-1.0.9-1.rpm
[root@itpux1 rpm]# rpm -ivh cvuqdisk-1.0.9-1.rpm
在另一个节点执行
chown -R grid:oinstall /soft
节点1 scp  cvuqdisk-1.0.9-1.rpm itpux2:/soft
节点2root用户
export CVUQDISK_GRP=oinstall
rpm -ivh cvuqdisk-1.0.9-1.rpm
在两个OracleRAC节点上安装操作系统程序包cvuqdisk,如果没有，集群验证实用程序就无法发现共享磁盘。
使用CVU验证是否满足Oracle集群件要求
记住要作为grid用户在将要执行Oracle安装的节点（racnode1）上运行。
此外，必须为grid用户配置通过用户等效性实现的SSH连通性

 

su - grid
vncserver--启动vnc服务，然后输入2次密码之后就可以用远程桌面连接

2台主机安装完成后
在grid软件目录里运行一下命令，验证grid环境是否合适--必须要用grid用户
./runcluvfy.sh stage -pre crsinst -n itpux1,itpux2 -verbose
                   @@@@@@@@@@@@运行报错：[grid@node1 grid]$ ./runcluvfy.sh stage -pre crsinst -n node1,node2 -fixup -verbose
                                        /bin/rm: cannot remove directory `/tmp/bootstrap': Operation not permitted
                                        ./runcluvfy.sh: line 99: /tmp/bootstrap/ouibootstrap.log: Permission denied
                                        --无法进行校验，报错对bootstrap目录无权限                                                          
                   @@@@@@@@@@@@解决方案：grid用户下cd /tmp     ls -ll查看权限，然后用root用户]# chown -R grid:oinstall /tmp/bootstrap
                                                 # chmod -R 777 /tmp/bootstrap ,更改完权限，然后在用grid用户继续验证                                             
 
 --安装过程中缺少的包
rpm -ivh /mnt/Packages/ibaio-0.3.105 (i386)
rpm -ivh /mnt/Packages/ompat-libstdc++-33-3.2.3 (i386)
rpm -ivh /mnt/Packages/ibaio-devel-0.3.105 (i386)
rpm -ivh /mnt/Packages/ibgcc-3.4.6 (i386)
rpm -ivh /mnt/Packages/ibstdc++-3.4.6 (i386)
rpm -ivh /mnt/Packages/dksh-5.2.14
rpm -ivh /mnt/Packages/                                                                                                 
r用户CVU验证硬件和操作系统设置--虚拟机暂时不需要检查                                                                                                 
./runcluvfy.sh stage -post hwos -n itpux1,itpux2 -verbose

然后登录vnc，在grid文件夹下输入./runInstaller

scan配置
cluster scan：gridcluster  --集群名称
scanname: itpuxscan
scanport :1521

/oracle/app/oraInventory/orainstRoot.sh
oracle/app/11.2.0/grid/root.sh
CRS-4124: Oracle High Availability Services startup failed.
CRS-4000: Command Start failed, or completed with errors.
ohasd failed to start: Inappropriate ioctl for device
ohasd failed to start at /oracle/app/11.2.0/grid/crs/install/rootcrs.pl line 443.