autofailover

#!/bin/bash
export PATH=$PATH:$HOME/bin:/opt/pgsql/bin
export PGDATA=/data/pg5444
LOG_FILE="/data/pg5444/repmgr/auto_failover.log"
TIME=`date '+%Y-%m-%d %H:%M:%S'`
PSQL="psql -U repmgr repmgr"
HOSTNAME=`hostname`
SECONDARY_STANDBY=`$PSQL -t -c " select name from  repl_nodes where active='t' and type='standby' "|grep -v $HOSTNAME|sed /^$/d|sed 's/ //g' `
STANDBY_FOLLOW="/opt/pgsql/bin/repmgr -f /data/pg5444/repmgr/repmgr.conf --verbose -F standby follow"
VIP="192.168.100.10"
GATEWAY="192.168.100.254"
echo "Promoting Standby at `date '+%Y-%m-%d %H:%M:%S'`" >>$LOG_FILE  2>&1
 repmgr -f /data/pg5444/repmgr/repmgr.conf  --verbose standby promote >>$LOG_FILE  2>&1
if [ $? == 0 ];then
  IS_RECOVERY="t"
  while [ "$IS_RECOVERY" = "t" ];do
  IS_RECOVERY=`$PSQL -c "select pg_is_in_recovery()" | sed -n '3,3p'|sed 's/ //g'`
  if [ "$IS_RECOVERY" = "f" ];then
    echo "$TIME FAILOVER-INFO: Promote successfully!" >> $LOG_FILE    2>&1
        echo "$TIME FAILOVER-INFO: Switch To Async Replication..." >> $LOG_FILE 2>&1
          sed -i /synchronous_standby_names/d  $PGDATA/postgresql.conf && pg_ctl reload >>$LOG_FILE 2>&1
    if [ $? == 0 ];then
        echo "$TIME FAILOVER-INFO: Switch to Async mode successfully!" >> $LOG_FILE 2>&1
        echo "Standy begin to follow new master..." >> $LOG_FILE 2>&1
     ssh $SECONDARY_STANDBY " $STANDBY_FOLLOW ">/dev/null  2>&1 &
     commandpid=$!; sleep 30 ;kill -9 $commandpid >/dev/null 2>&1
         STANDBY_COUNT=`$PSQL -t -c  'select count(*) from pg_stat_replication'|sed /^$/d|sed 's/ //g' `
      if [ $STANDBY_COUNT == 1 ];then
      echo "$TIME FAILVOER-INFO: Standby have followed master successfully!" >> $LOG_FILE 2>&1      
         echo "$TIME FAILOVER-INFO: Database writable test..." >>$LOG_FILE 2>&1
         $PSQL -c "INSERT INTO failover_monitor(time,state) values(now(),'Ok')"
        if [ $? == 0 ];then 
          echo "$TIME FAILOVER-INFO: Database is writable! " >> $LOG_FILE 2>&1
          sudo ifconfig bond0:1 $VIP broadcast 192.168.100.255 netmask 255.255.255.0 && sudo arping -I bond0 -c 5 -s $VIP $GATEWAY >> $LOG_FILE 2>&1
          ip a|grep 'bond0:1' >>$LOG_FILE 2>&1
          if [ $? == 0 ];then 
             echo "$TIME FAILOVER-INFO: Binding VIP to new master successfully!" >> $LOG_FILE 2>&1
          else
             echo "$TIME FAILOVER-ERROR: Binding VIP to new master failed" >> $LOG_FILE 2>&1
          fi
        else
             echo "$TIME FAILOVER-ERROR: Database can't be writable!" >> $LOG_FILE 2>&1
        fi
      else
        echo "$TIME FAILOVER-ERROR: Standby have followed master failed!" >> $LOG_FILE 2>&1
      fi
    else
      echo "$TIME FAILOVER-ERROR:Switch to Async mode failed!" >> $LOG_FILE 2>&1
    fi
  else
    echo "$TIME FAILOVER-ERROR: the database is still in recovery! Sleep for 1s and Retry..." >> $LOG_FILE 2>&1
    sleep 1
  fi
  done
else
  echo "$TIME ERROR: Promote failed! Please check it!!!" >> $LOG_FILE 2>&1
fi