1)callout for rac
1@@@@create a shell for evmwatch.
@@@evmwatch monitor the your status of rac using log.
@@@$CRS_HOME=/u01/crs1020
[root@station3 usrco]# ls /u01/crs1020/racg/usrco
callout1.sh
####begin_1-1############################################
[root@station3 usrco]# cat callout1.sh
#!/bin/sh
NOTIFY_EVENTTYPE=$1
AWK=/usr/bin/awk
for ARGS in $*; do
   PROPERTY=`echo $ARGS | $AWK -F"=" '{print $1}'`
   VALUE=`echo $ARGS | $AWK -F"=" '{print $2}'`
   case $PROPERTY in
     VERSION|version)     NOTIFY_VERSION=$VALUE ;;
     SERVICE|service)     NOTIFY_SERVICE=$VALUE ;;
     DATABASE|database)   NOTIFY_DATABASE=$VALUE ;;
     INSTANCE|instance)   NOTIFY_INSTANCE=$VALUE ;;
     HOST|host)           NOTIFY_HOST=$VALUE ;;
     STATUS|status)       NOTIFY_STATUS=$VALUE ;;
     REASON|reason)       NOTIFY_REASON=$VALUE ;;
     CARD|card)           NOTIFY_CARDINALITY=$VALUE ;;
     TIMESTAMP|timestamp) NOTIFY_LOGDATE=$VALUE ;;    
     ??:??:??)            NOTIFY_LOGTIME=$PROPERTY ;; 
   esac
done

if ((( [ $NOTIFY_EVENTTYPE = "SERVICEMEMBER" ] ||
       [ $NOTIFY_EVENTTYPE = "SERVICE"       ] ||
       [ $NOTIFY_EVENTTYPE = "INSTANCE"      ]     \
     ) &&                                          \
     ( [ $NOTIFY_STATUS    = "up"            ] ||  \
       [ $NOTIFY_STATUS    = "down"          ]     \
     )) &&                                         \
     ( [ $NOTIFY_DATABASE  = "RDBB"          ]     \
     ))
then
     echo $* >> /u01/crs1020/racg/log/crsevtco.log    
fi
######end_1-1############################################


2@@@@create service
@@@
[root@station3 ~]# crs_stat -p | grep NAME=ora | awk -F"=" '{print $2}'
or
[root@station3 ~]# crs_stat -p | grep NAME=ora | cut -f2 -d '='
ora.RDBB.RDBB1.inst
ora.RDBB.RDBB2.inst
ora.RDBB.SCO1.RDBB1.srv   @@@<== create service "sco1" using dbca
ora.RDBB.SCO1.cs          @@@<== --prefered RDBB1  --availible RDBB2
ora.RDBB.SCO2.RDBB2.srv   @@@<==
ora.RDBB.SCO2.cs          @@@<== --prefered RDBB2  --availible RDBB1
ora.RDBB.db
ora.station3.ASM1.asm
ora.station3.LISTENER_STATION3.lsnr
ora.station3.gsd
ora.station3.ons
ora.station3.vip
ora.station4.ASM2.asm
ora.station4.LISTENER_STATION4.lsnr
ora.station4.gsd
ora.station4.ons
ora.station4.vip

@@@
@@@or create using text mode
/u01/crs1020/bin/srvctl add service -d RDBB -s SCO1 -r RDBB1 -a RDBB2
/u01/crs1020/bin/srvctl add service -d RDBB -s SCO2 -r RDBB2 -a RDBB1


3@@@@evmwatch
@@@
@@@station3(node1)
[root@station3 ~]#/u01/crs1020/bin/evmwatch -A -t "@timestamp @@"
....
"17-Jun-2012 12:39:32 CRS ora.RDBB.RDBB1.inst is transitioning from state OFFLINE to state ONLINE on member station3"
"17-Jun-2012 12:40:26 RAC: ora.RDBB.RDBB1.inst: up: "
"17-Jun-2012 12:40:26 RAC: ora.RDBB.RDBB1.inst: imstop: "
"17-Jun-2012 12:40:26 RAC: ora.RDBB.RDBB1.inst: imcheck: "
"17-Jun-2012 12:40:26 CRS ora.RDBB.SCO1.RDBB1.srv was modified
......

@@@
@@@station4(node2)
[root@station4 ~]#/u01/crs1020/bin/evmwatch -A -t "@timestamp @@"
....
"17-Jun-2012 12:39:32 CRS ora.RDBB.RDBB1.inst is transitioning from state OFFLINE to state ONLINE on member station3"
"17-Jun-2012 12:40:26 RAC: ora.RDBB.RDBB1.inst: up: "
"17-Jun-2012 12:40:26 RAC: ora.RDBB.RDBB1.inst: imstop: "
"17-Jun-2012 12:40:26 RAC: ora.RDBB.RDBB1.inst: imcheck: "
"17-Jun-2012 12:40:26 CRS ora.RDBB.SCO1.RDBB1.srv was modified
.......

@@@
@@@the log was generated by the shell.
[root@station3 log]# tail -f  /u01/crs1020/racg/log/crsevtco.log
..........
SERVICEMEMBER VERSION=1.0 service=SCO1 database=RDBB instance=RDBB1 host=station3 status=up reason=user card=1 timestamp=17-Jun-2012 11:02:08
SERVICE VERSION=1.0 service=SCO1 database=RDBB instance= host=station3 status=up reason=user timestamp=17-Jun-2012 11:02:08
INSTANCE VERSION=1.0 service=RDBB database=RDBB instance=RDBB1 host=station3 status=down reason=user timestamp=17-Jun-2012 11:14:23
SERVICEMEMBER VERSION=1.0 service=SCO1 database=RDBB instance=RDBB1 host=station3 status=down reason=failure timestamp=17-Jun-2012 11:14:24
INSTANCE VERSION=1.0 service=RDBB database=RDBB instance=RDBB1 host=station3 status=up reason=user timestamp=17-Jun-2012 12:40:26


4@@@@remove the service
/u01/crs1020/bin/srvctl stop service -d RDBB -s SCO1
/u01/crs1020/bin/srvctl stop service -d RDBB -s SCO2
/u01/crs1020/bin/srvctl remove service -d RDBB -s SCO1
/u01/crs1020/bin/srvctl remove service -d RDBB -s SCO2

export ORACLE_SID=RDBB
export ORACLE_HOME=/u01/app/oracle/product/10.2.0/db_1
$ORACLE_HOME/bin/sqlplus / as sysdba <<SQL
conn / as sysdba
exec DBMS_SERVICE.DELETE_SERVICE('SCO1');
exec DBMS_SERVICE.DELETE_SERVICE('SCO2');
SQL