#!/bin/bash
export PATH=$PATH:$HOME/bin:/opt/pgsql/bin
export PGDATA=/data/pg5444
LOG_FILE="/data/pg5444/repmgr/auto_failover.log"
TIME=`date '+%Y-%m-%d %H:%M:%S'`
PSQL="psql -U repmgr repmgr"
HOSTNAME=`hostname`
SECONDARY_STANDBY=`$PSQL -t -c " select name from repl_nodes where active='t' and type='standby' "|grep -v $HOSTNAME|sed /^$/d|sed 's/ //g' `
STANDBY_FOLLOW="/opt/pgsql/bin/repmgr -f /data/pg5444/repmgr/repmgr.conf --verbose -F standby follow"
VIP="192.168.100.10"
GATEWAY="192.168.100.254"
echo "Promoting Standby at `date '+%Y-%m-%d %H:%M:%S'`" >>$LOG_FILE 2>&1
repmgr -f /data/pg5444/repmgr/repmgr.conf --verbose standby promote >>$LOG_FILE 2>&1
if [ $? == 0 ];then
IS_RECOVERY="t"
while [ "$IS_RECOVERY" = "t" ];do
IS_RECOVERY=`$PSQL -c "select pg_is_in_recovery()" | sed -n '3,3p'|sed 's/ //g'`
if [ "$IS_RECOVERY" = "f" ];then
echo "$TIME FAILOVER-INFO: Promote successfully!" >> $LOG_FILE 2>&1
echo "$TIME FAILOVER-INFO: Switch To Async Replication..." >> $LOG_FILE 2>&1
sed -i /synchronous_standby_names/d $PGDATA/postgresql.conf && pg_ctl reload >>$LOG_FILE 2>&1
if [ $? == 0 ];then
echo "$TIME FAILOVER-INFO: Switch to Async mode successfully!" >> $LOG_FILE 2>&1
echo "Standy begin to follow new master..." >> $LOG_FILE 2>&1
ssh $SECONDARY_STANDBY " $STANDBY_FOLLOW ">/dev/null 2>&1 &
commandpid=$!; sleep 30 ;kill -9 $commandpid >/dev/null 2>&1
STANDBY_COUNT=`$PSQL -t -c 'select count(*) from pg_stat_replication'|sed /^$/d|sed 's/ //g' `
if [ $STANDBY_COUNT == 1 ];then
echo "$TIME FAILVOER-INFO: Standby have followed master successfully!" >> $LOG_FILE 2>&1
echo "$TIME FAILOVER-INFO: Database writable test..." >>$LOG_FILE 2>&1
$PSQL -c "INSERT INTO failover_monitor(time,state) values(now(),'Ok')"
if [ $? == 0 ];then
echo "$TIME FAILOVER-INFO: Database is writable! " >> $LOG_FILE 2>&1
sudo ifconfig bond0:1 $VIP broadcast 192.168.100.255 netmask 255.255.255.0 && sudo arping -I bond0 -c 5 -s $VIP $GATEWAY >> $LOG_FILE 2>&1
ip a|grep 'bond0:1' >>$LOG_FILE 2>&1
if [ $? == 0 ];then
echo "$TIME FAILOVER-INFO: Binding VIP to new master successfully!" >> $LOG_FILE 2>&1
else
echo "$TIME FAILOVER-ERROR: Binding VIP to new master failed" >> $LOG_FILE 2>&1
fi
else
echo "$TIME FAILOVER-ERROR: Database can't be writable!" >> $LOG_FILE 2>&1
fi
else
echo "$TIME FAILOVER-ERROR: Standby have followed master failed!" >> $LOG_FILE 2>&1
fi
else
echo "$TIME FAILOVER-ERROR:Switch to Async mode failed!" >> $LOG_FILE 2>&1
fi
else
echo "$TIME FAILOVER-ERROR: the database is still in recovery! Sleep for 1s and Retry..." >> $LOG_FILE 2>&1
sleep 1
fi
done
else
echo "$TIME ERROR: Promote failed! Please check it!!!" >> $LOG_FILE 2>&1
fi
export PATH=$PATH:$HOME/bin:/opt/pgsql/bin
export PGDATA=/data/pg5444
LOG_FILE="/data/pg5444/repmgr/auto_failover.log"
TIME=`date '+%Y-%m-%d %H:%M:%S'`
PSQL="psql -U repmgr repmgr"
HOSTNAME=`hostname`
SECONDARY_STANDBY=`$PSQL -t -c " select name from repl_nodes where active='t' and type='standby' "|grep -v $HOSTNAME|sed /^$/d|sed 's/ //g' `
STANDBY_FOLLOW="/opt/pgsql/bin/repmgr -f /data/pg5444/repmgr/repmgr.conf --verbose -F standby follow"
VIP="192.168.100.10"
GATEWAY="192.168.100.254"
echo "Promoting Standby at `date '+%Y-%m-%d %H:%M:%S'`" >>$LOG_FILE 2>&1
repmgr -f /data/pg5444/repmgr/repmgr.conf --verbose standby promote >>$LOG_FILE 2>&1
if [ $? == 0 ];then
IS_RECOVERY="t"
while [ "$IS_RECOVERY" = "t" ];do
IS_RECOVERY=`$PSQL -c "select pg_is_in_recovery()" | sed -n '3,3p'|sed 's/ //g'`
if [ "$IS_RECOVERY" = "f" ];then
echo "$TIME FAILOVER-INFO: Promote successfully!" >> $LOG_FILE 2>&1
echo "$TIME FAILOVER-INFO: Switch To Async Replication..." >> $LOG_FILE 2>&1
sed -i /synchronous_standby_names/d $PGDATA/postgresql.conf && pg_ctl reload >>$LOG_FILE 2>&1
if [ $? == 0 ];then
echo "$TIME FAILOVER-INFO: Switch to Async mode successfully!" >> $LOG_FILE 2>&1
echo "Standy begin to follow new master..." >> $LOG_FILE 2>&1
ssh $SECONDARY_STANDBY " $STANDBY_FOLLOW ">/dev/null 2>&1 &
commandpid=$!; sleep 30 ;kill -9 $commandpid >/dev/null 2>&1
STANDBY_COUNT=`$PSQL -t -c 'select count(*) from pg_stat_replication'|sed /^$/d|sed 's/ //g' `
if [ $STANDBY_COUNT == 1 ];then
echo "$TIME FAILVOER-INFO: Standby have followed master successfully!" >> $LOG_FILE 2>&1
echo "$TIME FAILOVER-INFO: Database writable test..." >>$LOG_FILE 2>&1
$PSQL -c "INSERT INTO failover_monitor(time,state) values(now(),'Ok')"
if [ $? == 0 ];then
echo "$TIME FAILOVER-INFO: Database is writable! " >> $LOG_FILE 2>&1
sudo ifconfig bond0:1 $VIP broadcast 192.168.100.255 netmask 255.255.255.0 && sudo arping -I bond0 -c 5 -s $VIP $GATEWAY >> $LOG_FILE 2>&1
ip a|grep 'bond0:1' >>$LOG_FILE 2>&1
if [ $? == 0 ];then
echo "$TIME FAILOVER-INFO: Binding VIP to new master successfully!" >> $LOG_FILE 2>&1
else
echo "$TIME FAILOVER-ERROR: Binding VIP to new master failed" >> $LOG_FILE 2>&1
fi
else
echo "$TIME FAILOVER-ERROR: Database can't be writable!" >> $LOG_FILE 2>&1
fi
else
echo "$TIME FAILOVER-ERROR: Standby have followed master failed!" >> $LOG_FILE 2>&1
fi
else
echo "$TIME FAILOVER-ERROR:Switch to Async mode failed!" >> $LOG_FILE 2>&1
fi
else
echo "$TIME FAILOVER-ERROR: the database is still in recovery! Sleep for 1s and Retry..." >> $LOG_FILE 2>&1
sleep 1
fi
done
else
echo "$TIME ERROR: Promote failed! Please check it!!!" >> $LOG_FILE 2>&1
fi