基础环境
服务器品牌 | IBM |
服务器型号 | X3650M2 |
cpu类型 | Intel Xeon E5620@ 2.40GHz |
cpu个数 | 2 |
内存大小 | 24G |
磁盘阵列 | DS3200 |
硬盘大小 | 1.8TB |
软件准备:
--Oracle Database (includes Oracle Database and Oracle RAC)
p10404530_112030_
platform_1of7.zip
p10404530_112030_
platform_2of7.zip
--Oracle Grid Infrastructure (includes Oracle ASM, Oracle Clusterware, and Oracle Restart)
p10404530_112030_platform_3of7.zip
1配置yum源
vim /etc/yum.repos.d/rhel-debuginfo.repo
ORADB1:
[Server]
name=Server
baseurl=file:///mnt/Server
enabled=1
gpgcheck=0
ORADB2:
[Server]
name=Server
baseurl=ftp://oradb1/pub/Server
enabled=1
gpgcheck=0
2同时检查以下包是否被安装
rpm -q --queryformat "%{NAME}-%{VERSION}-%{RELEASE} (%{ARCH})\n" \
binutils \
compat-libstdc++-33 \
elfutils-libelf \
elfutils-libelf-devel \
gcc \
gcc-c++ \
glibc \
glibc-common \
glibc-devel \
glibc-headers \
ksh \
libaio \
libaio-devel \
libgcc \
libstdc++ \
libstdc++-devel \
make \
sysstat \
unixODBC \
unixODBC-devel
通过检查发现我的系统一下4个包没有安装:
package libaio-devel is not installed
package sysstat is not installed
package unixODBC is not installed
package unixODBC-devel is not installed
package sysstat is not installed
--挂载CD镜像到mnt目录下:
[root@localhost tmp]# mount /dev/cdrom /mnt/
mount: block device /dev/cdrom is write-protected, mounting read-only
--yum安装这几个包
[root@localhost tmp]# yum install libaio-devel package libaio-develunixODBC unixODBC-devel sysstat –y
--yum安装第二个节点的时候可以重新挂载
[root@localhost tmp]# mount /dev/cdrom /var/ftp/pub/
mount: block device /dev/cdrom is write-protected, mounting read-only
3.关掉大部分不用服务(可能因人因环境而异).
chkconfig bluetooth --level 2345 off
chkconfig ip6tables --level 2345 off
chkconfig portmap --level 2345 off
chkconfig sendmail --level 2345 off
chkconfig vsftpd --level 2345 off
chkconfig iptables off
chkconfig ntpd off
4在每一个节点上配置/etc/hosts文件,文件内容如下
# Public Network ? (eth0)
10.0.18.12oradb1.bdstar.com oradb1
10.0.18.13oradb2.bdstar.com oradb2
# Public Virtual IP (VIP) addresses ? (eth0:1)
10.0.18.120oradb1-vip.bdstar.com oradb1-vip
10.0.18.130oradb2-vip.bdstar.com oradb2-vip
# Private Interconnect ? (eth1)
10.10.10.120 oradb1-priv.bdstar.comoradb1-priv
10.10.10.130 oradb2-priv.bdstar.comoradb2-priv
#Compute node Client Access SCAN Interface details
10.0.18.36oradb-scan.bdstar.com oradb-scan
5修改主机名
[root@localhost ~]# vim /etc/sysconfig/network
Oradb1上添加如下:
NETWORKING=yes
NETWORKING_IPV6=no
HOSTNAME=oradb1.bdstar.com
Oradb2上添加如下:
NETWORKING=yes
NETWORKING_IPV6=no
HOSTNAME=oradb1.bdstar.com
6配置集群时间同步服务— (CTSS)
--不采用系统上的NTP服务,利用oracle集群的CTSS服务
[root@racnode1 ~]# /sbin/service ntpd stop
[root@racnode1 ~]# chkconfig ntpd off
[root@racnode1 ~]# mv /etc/ntp.conf /etc/ntp.conf.original
还要删除以下文件:
[root@racnode1 ~]# rm /var/run/ntpd.pid
7内核参数的配置
[root@rac1 ~]# cd /etc/
[root@rac1 etc]# sysctl -p
net.ipv4.conf.default.rp_filter = 1
net.ipv4.conf.default.accept_source_route = 0
kernel.sysrq = 0
kernel.core_uses_pid = 1
net.ipv4.tcp_syncookies = 1
kernel.msgmnb = 65536
kernel.msgmax = 65536
kernel.shmmax = 68719476736
kernel.shmall = 4294967296
net.ipv4.ip_local_port_range = 9000 65500
fs.file-max = 6815744
fs.suid_dumpable = 1
fs.aio-max-nr = 1048576
kernel.shmmni = 4096
kernel.sem = 250 32000 100 128
net.core.rmem_default = 262144
net.core.rmem_max = 4194304
net.core.wmem_default = 262144
net.core.wmem_max = 1048586
8 DNS的配置
只在节点1做一个DNS服务端即可:
[root@localhost etc]# yum install bind bind-chroot caching-nameserver
# vim /var/named/chroot/etc/named.conf
options {
directory "/var/named";
};
zone "localdomain" in {
type master;
file "localdomain.zone";
};
zone "bdstar.com" IN {
type master;
file "bdstar.com.zone";
};
[root@oradb1 ~]# cd /var/named/chroot/var/named/
[root@oradb1 named]# vim bdstar.com.zone
$TTL86400
@IN SOA @ root (
42 ; serial (d. adams)
3H ; refresh
15M ; retry
1W ; expiry
1D ) ; minimum
@ IN NSoradb1.bdstar.com.
oradb1IN A 10.0.18.12
oradb2IN A 10.0.18.13
oradb1-vipIN A 10.0.18.120
oradb2-vipIN A 10.0.18.130
oradb-scanIN A 10.0.18.36
[root@oradb1 named]# vim localdomain.zone
$TTL86400
@IN SOA localhost root (
42 ; serial (d. adams)
3H ; refresh
15M ; retry
1W ; expiry
1D ) ; minimum
IN NS localhost
oradb1IN A 10.0.18.12
oradb2IN A 10.0.18.13
oradb1-vipIN A 10.0.18.120
oradb2-vipIN A 10.0.18.130
oradb-scanIN A 10.0.18.36
[root@oradb1 named]# cat /etc/resolv.conf
nameserver 10.0.18.12
search localdomain bdstar.com
[root@oradb1 named]# service named restart
--测试下是否生效
[root@oradb1 named]# nslookup oradb1.bdstar.com
Server:10.0.18.12
Address:10.0.18.12#53
Name:oradb1.bdstar.com
Address: 10.0.18.12
--调整DNS解析
由于oracle强烈建议不要见scan放入到hosts中,这样做的结果是只能有一个SCAN的IP被使用。所以我们调整解析的顺序:
[root@oradb1 log]# vim /etc/nsswitch.conf
hosts:dns files
[root@oradb1 ~]# /sbin/service nscd restart
9创建grid和oracle用户,在每一个节点操作
groupadd -g 1000 oinstall
groupadd -g 1020 asmadmin
groupadd -g 1021 asmdba
groupadd -g 1022 asmoper
groupadd -g 1300 dba
useradd -u 1101 -g oinstall -G dba,asmdba -c "Oracle Software Owner" oracle
useradd -u 1100 -g oinstall -G asmadmin,asmdba,asmoper,dba -c "Grid Infrastructure Owner" grid
--修改oracle grid密码
passwd oracle
passwd grid
10创建oracle和grid的Oracle Base 目录
mkdir -p/u01/app/11.2.0/grid
mkdir -p /u01/app/grid
chown -R grid:oinstall /u01
mkdir -p /u01/app/oracle
chown oracle:oinstall /u01/app/oracle
chmod -R 775 /u01
-- Oracle Inventory Directory
mkdir -p /u01/app/oraInventory
chown -R grid:oinstall /u01/app/oraInventory
chmod -R 755 /u01/app/oraInventory
11配制grid,oracle用户环境变量
节点1grid用户:
~/.bash_profile增加以下内容
export ORACLE_BASE=/u01/app/grid
export ORACLE_HOME=/u01/app/11.2.0/grid
export PATH=$PATH:$ORACLE_HOME/bin
export NLS_LANG="SIMPLIFIED CHINESE_CHINA".AL32UTF8
export ORACLE_SID=+ASM1
umask 022
stty erase ^H
节点2grid用户:
~/.bash_profile增加以下内容
export ORACLE_BASE=/u01/app/grid
export ORACLE_HOME=/u01/app/11.2.0/grid
export PATH=$PATH:$ORACLE_HOME/bin
export NLS_LANG="SIMPLIFIED CHINESE_CHINA".AL32UTF8
export ORACLE_SID=+ASM2
umask 022
stty erase ^H
节点1 ORACLE用户:
~/.bash_profile增加以下内容
export ORACLE_BASE=/u01/app/oracle
export ORACLE_HOME=$ORACLE_BASE/product/11.2.0/dbhome_1
export PATH=$PATH:$ORACLE_HOME/bin
export NLS_LANG="SIMPLIFIED CHINESE_CHINA".AL32UTF8
export ORACLE_SID=bdspoc1
umask 022
stty erase ^H
节点2 ORACLE用户:
export ORACLE_BASE=/u01/app/oracle
export ORACLE_HOME=$ORACLE_BASE/product/11.2.0/dbhome_1
export PATH=$PATH:$ORACLE_HOME/bin
export NLS_LANG="SIMPLIFIED CHINESE_CHINA".AL32UTF8
export ORACLE_SID=bdspoc2
umask 022
stty erase ^H
12设置用户资源限制
1.在文件 /etc/security/limits.conf加以下参数:
#Install oracle setting
oraclesoft nproc 2047
oraclehard nproc 16384
oraclesoft nofile 1024
oraclehard nofile 65536
oraclesoft stack 10240
gridsoft nproc 2047
gridhard nproc 16384
gridsoft nofile1024
gridhard nofile 65536
gridsoft stack 10240
增加如下行在/etc/pam.d/login文件中
sessionrequired pam_limits.so
vim /etc/profile
if [ $USER = "oracle" ] || [ $USER = "grid" ]; then
if [ $SHELL = "/bin/ksh" ]; then
ulimit -p 16384
ulimit -n 65536
else
ulimit -u 16384 -n 65536
fi
umask 022
fi
For the C shell (csh or tcsh), add the following lines to the /etc/csh.login file:
if ( $USER = "oracle" || $USER = "grid" ) then
limit maxproc 16384
limit descriptors 65536
endif
13设置RDAC
由于用的是IBM的磁盘阵列,所以需要在系统端安装多路径软件。
下载地址:http://support.netapp.com/NOW/public/apbu/oemcp/
将下载的RDAC包解压生成linuxrdac-09.03.0C05.0638文件夹
进入文件夹后编译该包,
[root@oradb1 linuxrdac-09.03.0C05.0638]# make
make[1]: Entering directory `/usr/src/kernels/2.6.18-274.el5-x86_64'
….
gcc mppUtil.o mppUtilSysdep.o -o mppUtil
gcc -o genuniqueid genuniqueid.c
[root@oradb1 linuxrdac-09.03.0C05.0638]# make install
make[1]: Entering directory `/tmp/linuxrdac-09.03.0C05.0638'
….
MPP driver package has been successfully installed on your system.
--在/boot文件夹下生成一个mpp的文件
[root@oradb1 boot]# pwd
/boot
[root@oradb1 boot]# ls
config-2.6.18-274.el5initrd-2.6.18-274.el5kdump.img symvers-2.6.18-274.el5.gz
grublost+found System.map-2.6.18-274.el5
initrd-2.6.18-274.el5.imgmpp-2.6.18-274.el5.img vmlinuz-2.6.18-274.el5
--修改启动选项
[root@oradb1 tmp]# vim /boot/grub/menu.lst
default=1
timeout=5
splashimage=(hd0,0)/grub/splash.xpm.gz
hiddenmenu
title Red Hat Enterprise Linux Server (2.6.18-274.el5)
root (hd0,0)
kernel /vmlinuz-2.6.18-274.el5 ro root=LABEL=/ rhgb quiet
initrd /initrd-2.6.18-274.el5.img
title Red Hat Enterprise Linux Server with MPP surpport
root (hd0,0)
kernel /vmlinuz-2.6.18-274.el5 ro root=LABEL=/ rhgb quiet
initrd /mpp-2.6.18-274.el5.img
节点2做同样的事
14共享磁盘分区
[root@oradb1 ~]# fdisk /dev/sdc
The number of cylinders for this disk is set to 72815.
There is nothing wrong with that, but this is larger than 1024,
and could in certain setups cause problems with:
1) software that runs at boot time (e.g., old versions of LILO)
2) booting and partitioning software from other OSs
(e.g., DOS FDISK, OS/2 FDISK)
Command (m for help): n
Command action
e extended
p primary partition (1-4)
p
Partition number (1-4): 1
First cylinder (1-72815, default 1):
Using default value 1
Last cylinder or +size or +sizeM or +sizeK (1-72815, default 72815): +1024M
Command (m for help): n
Command action
e extended
p primary partition (1-4)
p
Partition number (1-4): 2
First cylinder (126-72815, default 126):
Using default value 126
Last cylinder or +size or +sizeM or +sizeK (126-72815, default 72815):
Using default value 72815
Command (m for help): w
The partition table has been altered!
Calling ioctl() to re-read partition table.
Syncing disks.
[root@oradb1 ~]# /sbin/partprobe
在节点2查看共享磁盘情况,OK!和1节点的一直。
15 asmlib包的安装
下载:http://www.oracle.com/technetwork/server-storage/linux/downloads/rhel5-084877.html
查看对应的版本后下载后安装:[root@oradb1 pub]# uname -a
Linux oradb1.bdstar.com 2.6.18-274.el5 #1 SMP Fri Jul 8 17:36:59 EDT 2011 x86_64 x86_64 x86_64 GNU/Linux
[root@oradb1 pub]# rpm -ivh oracleasm-support-2.1.8-1.el5.x86_64.rpm
warning: oracleasm-support-2.1.8-1.el5.x86_64.rpm: Header V3 DSA signature: NOKEY, key ID 1e5e0159
Preparing...########################################### [100%]
1:oracleasm-support ########################################### [100%]
[root@oradb1 pub]# rpm -ivh oracleasm-2.6.18-274.el5-2.0.5-1.el5.x86_64.rpm
warning: oracleasm-2.6.18-274.el5-2.0.5-1.el5.x86_64.rpm: Header V3 DSA signature: NOKEY, key ID 1e5e0159
Preparing...########################################### [100%]
1:oracleasm-2.6.18-274.el########################################### [100%]
[root@oradb1 pub]# rpm -ivh oracleasmlib-2.0.4-1.el5.x86_64.rpm
warning: oracleasmlib-2.0.4-1.el5.x86_64.rpm: Header V3 DSA signature: NOKEY, key ID 1e5e0159
Preparing...########################################### [100%]
1:oracleasmlib ########################################### [100%]
16使用root用户安装配置asmlib
--这里使用asm工具对设备进行持久化,当然也可以使用udev的方式对设备进行持久化!
--双节点都要configure
[root@oradb1 pub]# /etc/init.d/oracleasm configure
Configuring the Oracle ASM library driver.
This will configure the on-boot properties of the Oracle ASM library
driver.The following questions will determine whether the driver is
loaded on boot and what permissions it will have.The current values
will be shown in brackets ('[]').Hitting <ENTER> without typing an
answer will keep that current value.Ctrl-C will abort.
Default user to own the driver interface []: grid
Default group to own the driver interface []: asmadmin
Start Oracle ASM library driver on boot (y/n) [n]: y
Scan for Oracle ASM disks on boot (y/n) [y]: y
Writing Oracle ASM library driver configuration: done
Initializing the Oracle ASMLib driver:[ OK ]
Scanning the system for Oracle ASMLib disks:[ OK ]
--单节点进行创建配置
[root@oradb1 pub]# /etc/init.d/oracleasm createdisk crs01 /dev/sdb1
Marking disk "crs01" as an ASM disk:[ OK ]
[root@oradb1 pub]# /etc/init.d/oracleasm createdisk crs02 /dev/sdc1
Marking disk "crs02" as an ASM disk:[ OK ]
[root@oradb1 pub]# /etc/init.d/oracleasm createdisk crs02 /dev/sdc1
Marking disk "crs02" as an ASM disk:[ OK ]
[root@oradb1 pub]# /etc/init.d/oracleasm createdisk data01 /dev/sdb2
Marking disk "data01" as an ASM disk:[ OK ]
[root@oradb1 pub]# /etc/init.d/oracleasm createdisk data02 /dev/sdc2
Marking disk "data02" as an ASM disk:[ OK ]
[root@oradb1 pub]# /etc/init.d/oracleasm createdisk data03 /dev/sdd2
Marking disk "data03" as an ASM disk:[ OK ]
--每个节点都要查看下
--/usr/sbin/oracleasm为更高级的oracleasm命令
[root@oradb1 pub]# /usr/sbin/oracleasm listdisks
CRS01
CRS02
CRS03
DATA01
DATA02
DATA03
--每个节点都进行注册
[root@oradb1 pub]# /usr/sbin/oracleasm scandisks
Reloading disk partitions: done
Cleaning any stale ASM disks...
Scanning system for ASM disks...
17安装CVU软件
两节点上分别作
# export CVUQDISK_GRP=oinstall
# cd /mnt/stage/cvu/cv/remenv
# rpm -ivh cvuqdisk-1.0.9-1.rpm
18检查配置
./runcluvfy.sh stage -pre crsinst -n node1,node2 -verbose > /tmp/checkrac.txt
reference :
最常见的5个导致 RAC 实例崩溃的问题[文章 ID1549191.1]
最常见的5个CRS/Grid Infrastructure 安装问题[文章 ID1549192.1]
最常见的5 个导致节点重新启动、驱逐或 CRS 意外重启的问题[文章 ID1524455.1]