ORACLE RAC 11.2.0.4 for RHEL6.8安装遭遇PRVF 9992与DBCA遭遇ORA-19504&ORA-15001

   无论是物理实体机，还是vmware或virtualbox虚拟机，自己已经安装过N多套10g、11g的rac集群系统，
昨天算是第一次遭遇到PRVF-9992 : Group of device “/dev/mapper/datadg″ did not match the expected group
由于以前在湘潭安装过oracle rac 11.2.0.4 for rhel6.6，安装过程非常顺利，用的方法与这次衡阳的安装也是一模一样，自己感觉到非常纳闷。
    操作系统：ORACLE LINUX RHEL6.8
    数据库版本：ORACLE 11.2.0.4
    物理机器： Dell R730 Server
    出现的问题：PRVF 9992
本次安装，集群共享磁盘使用的是multipath映射，修改前的配置：
[root@rac1 soft]#cat /etc/multipath.conf
# multipath.conf written by anaconda
defaults {
        user_friendly_names yes
}
blacklist {
        devnode "^(ram|raw|loop|fd|md|dm-|sr|scd|st)[0-9]*"
        devnode "^hd[a-z]"
        devnode "^dcssblk[0-9]*"
        device {
                vendor "DGC"
                product "LUNZ"
        }
        device {
                vendor "IBM"
                product "S/390.*"
        }
        # don't count normal SATA devices as multipaths
        device {
                vendor  "ATA"
        }
        # don't count 3ware devices as multipaths
        device {
                vendor  "3ware"
        }
        device {
                vendor  "AMCC"
        }
        # nor highpoint devices
        device {
                vendor  "HPT"
        }
        wwid "361866da084d8d1002032f80e0b9458e7"
        wwid "*"
}
blacklist_exceptions {
        wwid "3600a098000abf3880000025d58a036e7"
        wwid "3600a098000abf3880000025658a0367d"
        wwid "3600a098000abf3880000026158a0371a"
        wwid "3600a098000abf3880000025f58a03705"
        wwid "3600a098000abf3880000025b58a036c4"
        wwid "3600a098000abf3880000026358a03730"
blacklist_exceptions {
        wwid "3600a098000abf3880000025d58a036e7"
        wwid "3600a098000abf3880000025658a0367d"
        wwid "3600a098000abf3880000026158a0371a"
        wwid "3600a098000abf3880000025f58a03705"
        wwid "3600a098000abf3880000025b58a036c4"
        wwid "3600a098000abf3880000026358a03730"
        wwid "3600a098000abf3880000025958a036a0"
}
multipaths {
        multipath {
                uid 0
                gid 0
                wwid "3600a098000abf3880000025d58a036e7"
                mode 0600
        }
        multipath {
                uid 0
                gid 0
                wwid "3600a098000abf3880000025658a0367d"
                mode 0600
        }
        multipath {
                uid 0
                gid 0
                wwid "3600a098000abf3880000026158a0371a"
                mode 0600
        }
        multipath {
                uid 0
                gid 0
                wwid "3600a098000abf3880000025f58a03705"
                mode 0600
        }
        multipath {
                uid 0
                gid 0
                wwid "3600a098000abf3880000025b58a036c4"
                mode 0600
        }
        multipath {
                uid 0
                gid 0
                wwid "3600a098000abf3880000026358a03730"
                mode 0600
        }
        multipath {
                uid 0
                gid 0
                wwid "3600a098000abf3880000025958a036a0"
                mode 0600
        }
}
修改后的配置：
# multipath.conf written by anaconda
defaults {
        user_friendly_names yes
}
blacklist {
        devnode "^(ram|raw|loop|fd|md|dm-|sr|scd|st)[0-9]*"
        devnode "^hd[a-z]"
        devnode "^dcssblk[0-9]*"
        device {
                vendor "DGC"
                product "LUNZ"
        }
        device {
                vendor "IBM"
                product "S/390.*"
        }
        # don't count normal SATA devices as multipaths
        device {
                vendor  "ATA"
        }
        # don't count 3ware devices as multipaths
        device {
                vendor  "3ware"
        }
        device {
                vendor  "AMCC"
        }
        # nor highpoint devices
        device {
                vendor  "HPT"
        }
        wwid "361866da084d8d1002032f80e0b9458e7"
        wwid "*"
}
blacklist_exceptions {
        wwid "3600a098000abf3880000025d58a036e7"
        wwid "3600a098000abf3880000025658a0367d"
        wwid "3600a098000abf3880000026158a0371a"
        wwid "3600a098000abf3880000025f58a03705"
        wwid "3600a098000abf3880000025b58a036c4"
        wwid "3600a098000abf3880000026358a03730"
blacklist_exceptions {
        wwid "3600a098000abf3880000025d58a036e7"
        wwid "3600a098000abf3880000025658a0367d"
        wwid "3600a098000abf3880000026158a0371a"
        wwid "3600a098000abf3880000025f58a03705"
        wwid "3600a098000abf3880000025b58a036c4"
        wwid "3600a098000abf3880000026358a03730"
        wwid "3600a098000abf3880000025958a036a0"
}
multipaths {
multipath {
uid 0
gid 0
wwid "3600a098000a1157d000002d25798865a"
mode 0600
}
multipath {
uid 600
gid 600
alias ocr1
wwid "3600a098000a1157d000002d457988679"
mode 0600
}
multipath {
uid 600
gid 600
alias arch
wwid "3600a098000a1157d000002cf57988639"
mode 0600
}
multipath {
uid 0
gid 0
wwid "3600a098000a1157d000002de579886d4"
mode 0600
}
multipath {
uid 600
gid 600
alias data3
wwid "3600a098000a1157d000002e0579886e9"
mode 0600
}
multipath {
uid 600
gid 600
alias ocr2
wwid "3600a098000a1157d000002d65798868a"
mode 0600
}
multipath {
uid 600
gid 600
alias ocr3
wwid "3600a098000a1157d000002d85798869a"
mode 0600
}
multipath {
uid 600
gid 600
alias data1
wwid "3600a098000a1157d000002da579886ac"
mode 0600
}
multipath {
uid 600
gid 600
alias data2
wwid "3600a098000a1157d000002dc579886bf"
mode 0600
}
}
外加udev修改读写权限：
[root@rac1 rules.d]# cat 12-mulitpath-privs.rules 
ENV{DM_NAME}=="arch*", OWNER:="grid", GROUP:="asmadmin", MODE:="660"
ENV{DM_NAME}=="ocr*", OWNER:="grid", GROUP:="asmadmin", MODE:="660"
ENV{DM_NAME}=="data*", OWNER:="grid", GROUP:="asmadmin", MODE:="660"
[root@rac1 rules.d]# 
主机经过重启，multipath重启，udev重启，再次尝试安装grid，还是有prvf-9992警告；想不到的是：忽略prvf-9992，grid竟然安装成功：
[root@rac1 Packages]# /u01/oracle/app/grid/oraInventory/orainstRoot.sh
Changing permissions of /u01/oracle/app/grid/oraInventory.
Adding read,write permissions for group.
Removing read,write,execute permissions for world.
Changing groupname of /u01/oracle/app/grid/oraInventory to oinstall.
The execution of the script is complete.
[root@rac1 Packages]# /u01/oracle/app/grid/home/root.sh
Performing root user operation for Oracle 11g 
The following environment variables are set as:
    ORACLE_OWNER= grid
    ORACLE_HOME=  /u01/oracle/app/grid/home
Enter the full pathname of the local bin directory: [/usr/local/bin]: 
   Copying dbhome to /usr/local/bin ...
   Copying oraenv to /usr/local/bin ...
   Copying coraenv to /usr/local/bin ...
Creating /etc/oratab file...
Entries will be added to the /etc/oratab file as needed by
Database Configuration Assistant when a database is created
Finished running generic part of root script.
Now product-specific root actions will be performed.
Using configuration parameter file: /u01/oracle/app/grid/home/crs/install/crsconfig_params
Creating trace directory
User ignored Prerequisites during installation
Installing Trace File Analyzer
OLR initialization - successful
  root wallet
  root wallet cert
  root cert export
  peer wallet
  profile reader wallet
  pa wallet
  peer wallet keys
  pa wallet keys
  peer cert request
  pa cert request
  peer cert
  pa cert
  peer root cert TP
  profile reader root cert TP
  pa root cert TP
  peer pa cert TP
  pa peer cert TP
  profile reader pa cert TP
  profile reader peer cert TP
  peer user cert
  pa user cert
Adding Clusterware entries to upstart
CRS-2672: Attempting to start 'ora.mdnsd' on 'rac1'
CRS-2676: Start of 'ora.mdnsd' on 'rac1' succeeded
CRS-2672: Attempting to start 'ora.gpnpd' on 'rac1'
CRS-2676: Start of 'ora.gpnpd' on 'rac1' succeeded
CRS-2672: Attempting to start 'ora.cssdmonitor' on 'rac1'
CRS-2672: Attempting to start 'ora.gipcd' on 'rac1'
CRS-2676: Start of 'ora.cssdmonitor' on 'rac1' succeeded
CRS-2676: Start of 'ora.gipcd' on 'rac1' succeeded
CRS-2672: Attempting to start 'ora.cssd' on 'rac1'
CRS-2672: Attempting to start 'ora.diskmon' on 'rac1'
CRS-2676: Start of 'ora.diskmon' on 'rac1' succeeded
CRS-2676: Start of 'ora.cssd' on 'rac1' succeeded
ASM created and started successfully.
Disk Group clsdg created successfully.
clscfg: -install mode specified
Successfully accumulated necessary OCR keys.
Creating OCR keys for user 'root', privgrp 'root'..
Operation successful.
CRS-4256: Updating the profile
Successful addition of voting disk 5edddd8fca634f07bfbf345f7c10f0c2.
Successful addition of voting disk 7c64e7be3a1d4f9fbf9f5983896a0adb.
Successful addition of voting disk 47924c4f2dda4f36bf3fca4111247e3f.
Successfully replaced voting disk group with +clsdg.
CRS-4256: Updating the profile
CRS-4266: Voting file(s) successfully replaced
##  STATE    File Universal Id                File Name Disk group
--  -----    -----------------                --------- ---------
 1. ONLINE   5edddd8fca634f07bfbf345f7c10f0c2 (/dev/mapper/crs1) [CLSDG]
 2. ONLINE   7c64e7be3a1d4f9fbf9f5983896a0adb (/dev/mapper/crs2) [CLSDG]
 3. ONLINE   47924c4f2dda4f36bf3fca4111247e3f (/dev/mapper/crs3) [CLSDG]
Located 3 voting disk(s).
CRS-2672: Attempting to start 'ora.asm' on 'rac1'
CRS-2676: Start of 'ora.asm' on 'rac1' succeeded
CRS-2672: Attempting to start 'ora.CLSDG.dg' on 'rac1'
CRS-2676: Start of 'ora.CLSDG.dg' on 'rac1' succeeded
Configure Oracle Grid Infrastructure for a Cluster ... succeeded
[root@rac1 Packages]#
[root@rac2 ~]# /u01/oracle/app/grid/oraInventory/orainstRoot.sh
Changing permissions of /u01/oracle/app/grid/oraInventory.
Adding read,write permissions for group.
Removing read,write,execute permissions for world.
Changing groupname of /u01/oracle/app/grid/oraInventory to oinstall.
The execution of the script is complete.
[root@rac2 ~]# /u01/oracle/app/grid/home/root.sh
Performing root user operation for Oracle 11g 
The following environment variables are set as:
    ORACLE_OWNER= grid
    ORACLE_HOME=  /u01/oracle/app/grid/home
Enter the full pathname of the local bin directory: [/usr/local/bin]: 
   Copying dbhome to /usr/local/bin ...
   Copying oraenv to /usr/local/bin ...
   Copying coraenv to /usr/local/bin ...
Creating /etc/oratab file...
Entries will be added to the /etc/oratab file as needed by
Database Configuration Assistant when a database is created
Finished running generic part of root script.
Now product-specific root actions will be performed.
Using configuration parameter file: /u01/oracle/app/grid/home/crs/install/crsconfig_params
Creating trace directory
User ignored Prerequisites during installation
Installing Trace File Analyzer
OLR initialization - successful
Adding Clusterware entries to upstart
CRS-4402: The CSS daemon was started in exclusive mode but found an active CSS daemon on node rac1, number 1, and is terminating
An active cluster was found during exclusive startup, restarting to join the cluster
Configure Oracle Grid Infrastructure for a Cluster ... succeeded
[root@rac2 ~]# 
[grid@rac1 ~]$ crs_stat -t
Name           Type           Target    State     Host        
------------------------------------------------------------
ora.CLSDG.dg   ora....up.type ONLINE    ONLINE    rac1        
ora....N1.lsnr ora....er.type ONLINE    ONLINE    rac1        
ora.asm        ora.asm.type   ONLINE    ONLINE    rac1        
ora.cvu        ora.cvu.type   ONLINE    ONLINE    rac1        
ora.gsd        ora.gsd.type   OFFLINE   OFFLINE               
ora....network ora....rk.type ONLINE    ONLINE    rac1        
ora.oc4j       ora.oc4j.type  ONLINE    ONLINE    rac1        
ora.ons        ora.ons.type   ONLINE    ONLINE    rac1        
ora....SM1.asm application    ONLINE    ONLINE    rac1        
ora.rac1.gsd   application    OFFLINE   OFFLINE               
ora.rac1.ons   application    ONLINE    ONLINE    rac1        
ora.rac1.vip   ora....t1.type ONLINE    ONLINE    rac1        
ora....SM2.asm application    ONLINE    ONLINE    rac2        
ora.rac2.gsd   application    OFFLINE   OFFLINE               
ora.rac2.ons   application    ONLINE    ONLINE    rac2        
ora.rac2.vip   ora....t1.type ONLINE    ONLINE    rac2        
ora.scan1.vip  ora....ip.type ONLINE    ONLINE    rac1        
[grid@rac1 ~]$ 
[grid@rac1 ~]$ crsctl stat res -t
--------------------------------------------------------------------------------
NAME           TARGET  STATE        SERVER                   STATE_DETAILS       
--------------------------------------------------------------------------------
Local Resources
--------------------------------------------------------------------------------
ora.CLSDG.dg
               ONLINE  ONLINE       rac1                                         
               ONLINE  ONLINE       rac2                                         
ora.asm
               ONLINE  ONLINE       rac1                     Started             
               ONLINE  ONLINE       rac2                     Started             
ora.gsd
               OFFLINE OFFLINE      rac1                                         
               OFFLINE OFFLINE      rac2                                         
ora.net1.network
               ONLINE  ONLINE       rac1                                         
               ONLINE  ONLINE       rac2                                         
ora.ons
               ONLINE  ONLINE       rac1                                         
               ONLINE  ONLINE       rac2                                         
--------------------------------------------------------------------------------
Cluster Resources
--------------------------------------------------------------------------------
ora.LISTENER_SCAN1.lsnr
      1        ONLINE  ONLINE       rac1                                         
ora.cvu
      1        ONLINE  ONLINE       rac1                                         
ora.oc4j
      1        ONLINE  ONLINE       rac1                                         
ora.rac1.vip
      1        ONLINE  ONLINE       rac1                                         
ora.rac2.vip
      1        ONLINE  ONLINE       rac2                                         
ora.scan1.vip
      1        ONLINE  ONLINE       rac1                                         
[grid@rac1 ~]$ 
[grid@rac1 ~]$ lsnrctl status racscan
LSNRCTL for Linux: Version 11.2.0.4.0 - Production on 13-FEB-2017 22:26:25
Copyright (c) 1991, 2013, Oracle.  All rights reserved.
Connecting to (DESCRIPTION=(CONNECT_DATA=(SERVICE_NAME=))(ADDRESS=(PROTOCOL=TCP)(HOST=10.141.101.144)(PORT=1521)))
STATUS of the LISTENER
------------------------
Alias                     LISTENER_SCAN1
Version                   TNSLSNR for Linux: Version 11.2.0.4.0 - Production
Start Date                13-FEB-2017 22:18:10
Uptime                    0 days 0 hr. 8 min. 15 sec
Trace Level               off
Security                  ON: Local OS Authentication
SNMP                      OFF
Listener Parameter File   /u01/oracle/app/grid/home/network/admin/listener.ora
Listener Log File         /u01/oracle/app/grid/home/log/diag/tnslsnr/rac1/listener_scan1/alert/log.xml
Listening Endpoints Summary...
  (DESCRIPTION=(ADDRESS=(PROTOCOL=ipc)(KEY=LISTENER_SCAN1)))
  (DESCRIPTION=(ADDRESS=(PROTOCOL=tcp)(HOST=10.141.101.144)(PORT=1521)))
The listener supports no services
The command completed successfully
[grid@rac1 ~]$ 
    另一个问题是：DBCA建库遭遇ORA-19504&ORA-15001
开始给DATADG一共4T的共享存储，但是grid通过asmca查看时显示totalsize是4TB，而freesize只有2TB；
然后，将datadg拆开成datadg1、datadg2,每个2T，再次通过asmca查看时显示totalsize和freesize一致了。
但是，重试dbca建库的时候还是到9%报错，依然是：ORA-19504&ORA-15001。怀疑是mutipath问题，将映射配置改为：
multipath {
uid 600
gid 600
alias arch
wwid "3600a098000a1157d000002cf57988639"
mode 0660
path_grouping_policy    failover
}
并且重新手工同步主机时钟（由于操作系统不是自己安装的，集群grid安装成功后，查看操作系统时钟，2台主机时间相差10多分钟，以后要更加细心），
然后， grid 关闭集群，重启操作系统后，oracle用户再次尝试dbca成功执行：
[grid@rac2 ~]$ crs_stat -t
Name           Type           Target    State     Host        
------------------------------------------------------------
ora.ARCHDG.dg  ora....up.type ONLINE    ONLINE    rac1        
ora.CLSDG.dg   ora....up.type ONLINE    ONLINE    rac1        
ora.DATADG1.dg ora....up.type ONLINE    ONLINE    rac1        
ora.DATADG2.dg ora....up.type ONLINE    ONLINE    rac1        
ora....ER.lsnr ora....er.type ONLINE    ONLINE    rac1        
ora....N1.lsnr ora....er.type ONLINE    ONLINE    rac2        
ora.asm        ora.asm.type   ONLINE    ONLINE    rac1        
ora.cvu        ora.cvu.type   ONLINE    ONLINE    rac1        
ora.gsd        ora.gsd.type   OFFLINE   OFFLINE               
ora....network ora....rk.type ONLINE    ONLINE    rac1        
ora.oc4j       ora.oc4j.type  ONLINE    ONLINE    rac1        
ora.ons        ora.ons.type   ONLINE    ONLINE    rac1        
ora.orcl.db    ora....se.type ONLINE    ONLINE    rac1        
ora....SM1.asm application    ONLINE    ONLINE    rac1        
ora....C1.lsnr application    ONLINE    ONLINE    rac1        
ora.rac1.gsd   application    OFFLINE   OFFLINE               
ora.rac1.ons   application    ONLINE    ONLINE    rac1        
ora.rac1.vip   ora....t1.type ONLINE    ONLINE    rac1        
ora....SM2.asm application    ONLINE    ONLINE    rac2        
ora....C2.lsnr application    ONLINE    ONLINE    rac2        
ora.rac2.gsd   application    OFFLINE   OFFLINE               
ora.rac2.ons   application    ONLINE    ONLINE    rac2        
ora.rac2.vip   ora....t1.type ONLINE    ONLINE    rac2        
ora.scan1.vip  ora....ip.type ONLINE    ONLINE    rac2        
[grid@rac2 ~]$ 


[oracle@rac1 ~]$ sqlplus / as sysdba


SQL*Plus: Release 11.2.0.4.0 Production on Mon Feb 13 18:34:39 2017


Copyright (c) 1982, 2013, Oracle.  All rights reserved.




Connected to:
Oracle Database 11g Enterprise Edition Release 11.2.0.4.0 - 64bit Production
With the Partitioning, Real Application Clusters, Automatic Storage Management, OLAP,
Data Mining and Real Application Testing options


SQL> select open_mode from v$database;


OPEN_MODE
--------------------
READ WRITE


SQL> 
SQL> select status from v$instance;


STATUS
------------
OPEN


SQL> 


[grid@rac1 ~]$ crsctl stat res -t
--------------------------------------------------------------------------------
NAME           TARGET  STATE        SERVER                   STATE_DETAILS       
--------------------------------------------------------------------------------
Local Resources
--------------------------------------------------------------------------------
ora.ARCHDG.dg
               ONLINE  ONLINE       rac1                                         
               ONLINE  ONLINE       rac2                                         
ora.CLSDG.dg
               ONLINE  ONLINE       rac1                                         
               ONLINE  ONLINE       rac2                                         
ora.DATADG1.dg
               ONLINE  ONLINE       rac1                                         
               ONLINE  ONLINE       rac2                                         
ora.DATADG2.dg
               ONLINE  ONLINE       rac1                                         
               ONLINE  ONLINE       rac2                                         
ora.LISTENER.lsnr
               ONLINE  ONLINE       rac1                                         
               ONLINE  ONLINE       rac2                                         
ora.asm
               ONLINE  ONLINE       rac1                     Started             
               ONLINE  ONLINE       rac2                     Started             
ora.gsd
               OFFLINE OFFLINE      rac1                                         
               OFFLINE OFFLINE      rac2                                         
ora.net1.network
               ONLINE  ONLINE       rac1                                         
               ONLINE  ONLINE       rac2                                         
ora.ons
               ONLINE  ONLINE       rac1                                         
               ONLINE  ONLINE       rac2                                         
--------------------------------------------------------------------------------
Cluster Resources
--------------------------------------------------------------------------------
ora.LISTENER_SCAN1.lsnr
      1        ONLINE  ONLINE       rac2                                         
ora.cvu
      1        ONLINE  ONLINE       rac1                                         
ora.oc4j
      1        ONLINE  ONLINE       rac1                                         
ora.orcl.db
      1        ONLINE  ONLINE       rac1                     Open                
      2        ONLINE  ONLINE       rac2                     Open                
ora.rac1.vip
      1        ONLINE  ONLINE       rac1                                         
ora.rac2.vip
      1        ONLINE  ONLINE       rac2                                         
ora.scan1.vip
      1        ONLINE  ONLINE       rac2                                         
[grid@rac1 ~]$