keepalived+LVS，nginx

keepalived官网：http://www.keepalived.org/

拓扑：

两台keepalived服务器设置selinx、防火墙

set enforce 0
service firewalld stop
service iptables stop

两台keepalived服务器开启ip转发

echo 1 > /proc/sys/net/ipv4/ip_forward

两台keepalived服务器上安装keepalived

yum install keepalived -y

安装后编辑配置文件 /etc/keepalived/keepalived.conf

注意：修改配置文件前先备份配置文件

cp keepalived.conf keepalived.conf.bak

打开配置文件进行修改

vim /etc/keepalived/keepalived.conf

配置文件内容：

! Configuration File for keepalived  
  
global_defs {  
   notification_email {  
        root@localhost
   }  
   notification_email_from root@localhost
   smtp_connect_timeout 3  
   smtp_server 127.0.0.1  
   router_id LVS_DEVEL01  
}  

vrrp_script chk_schedown {
   script "[[ -f /etc/keepalived/down ]] && exit 1 || exit 0"
   interval 2
   weight -2
}

vrrp_instance VI_1 {  
    interface ens33  
    state MASTER  
    priority 101
    virtual_router_id 51 
    garp_master_delay 1 
 
    authentication {  
        auth_type PASS  
        auth_pass 1111
    }  

    track_interface {  
       ens33    
    }  

    virtual_ipaddress {  
        192.168.137.100/24 dev ens33 label ens33:0 broadcast 192.168.137.255
    }  

    track_script {  
        chk_schedown
    }    
} 


virtual_server 192.168.137.100 80 {
    delay_loop 6
    lb_algo rr 
    lb_kind DR
    persistence_timeout 50
    protocol TCP

#    sorry_server 192.168.200.200 1358

    real_server 192.168.137.130 80 {
        weight 1
        HTTP_GET {
            url { 
              path /
              status_code 200
            }
            connect_timeout 3
            nb_get_retry 3
            delay_before_retry 3
        }
    }

    real_server 192.168.137.20 80 {
        weight 1
        HTTP_GET {
            url { 
              path /
              status_code 200
            }
            connect_timeout 3
            nb_get_retry 3
            delay_before_retry 3
        }
    }
}

将配置文件复制到备用keepalived服务器上：

scp /etc/keepalived/keepalived.conf root@192.168.137.130:/etc/keepalived/keepalived.conf

修改：  state BACKUP

            priority 100

            router_id LVS_DEVEL02 

启动两台keepalived服务器上的keepalived服务

service keepalived start

通过service keepalived status和tail -20/var/log/messages信息查看状态

在主节点使用ipvsadm -L –n查看生成的ipvs规则

[root@node30 keepalived]# ipvsadm -L -n       
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  192.168.137.100:80 rr persistent 50
  -> 192.168.137.132:80           Route   1      0          2         
  -> 192.168.137.135:80           Route   1      0          0    

图示已经生成了ipvs规则

realserver的操作：

两台realserver新建脚本文件lvs_realserver.sh,在脚本文件lvs_realserver.sh中写入以下内容

#! /bin/bash
vip=192.168.137.100
ifconfig lo:0 $vip broadcast $vip netmask 255.255.255.255 up 
route add -host $vip lo:0
echo "1" >/proc/sys/net/ipv4/conf/lo/arp_ignore
echo "2" >/proc/sys/net/ipv4/conf/lo/arp_announce
echo "1" >/proc/sys/net/ipv4/conf/all/arp_ignore
echo "2" >/proc/sys/net/ipv4/conf/all/arp_announce
sysctl -p > /dev/null 2>&1

赋予执行权限：

chmod +x  lvs_realserver.sh

关闭selinux和防火墙

setenforce 0
service iptables stop
service firewalld stop

打开ip转发

安装httpd服务：

echo 1 > /proc/sys/net/ipv4/ip_forward

在四台主机上可以使用ip addr show查看VIP的情况，下图为keepalived的MASTER服务器的信息，keepalived的BACKUP服务器上没有生效，其他机器均生效VIP，发生keepalived的MASTER、BACKUP切换时VIP会转移，始终在MASTER服务器上生效。

安装httpd服务：

yum install httpd -y

编辑默认页面

vim /var/www/html/index.html

写入标识主机的内容,如

<h1>IP地址</h1>

启动httpd服务

service httpd start

先检查这两台服务器的httpd是否可以访问

curl IP

如：

curl 192.168.137.132
curl 192.168.137.135

可以在四台机上查看VIP情况，keepalived的BACKUP服务器上没有VIP，其他主机均有。

ip addr show

在其他主机访问VIP

再通过ipvsadm -L -n --stats查看统计信息

keepalived的状态监控

实例状态通知
    notify_master ：节点变为master时执行
    notify_backup ： 节点变为backup时执行
    notify_fault  ： 节点变为故障时执行
	虚拟服务器检测通知
	    notify_up   ： 虚拟服务器up时执行
            notify_down  ： 虚拟服务器down时执行

配置keepalived为实现nginx高可用的配置文件示例：

! Configuration File for keepalived  
  
global_defs {  
   notification_email {  
        root@localhost
   }  
   notification_email_from root@localhost 
   smtp_connect_timeout 3  
   smtp_server 127.0.0.1  
   router_id LVS_DEVEL01  
}  

vrrp_script chk_nginx {  
    script "killall -0 nginx"  
    interval 1  
    weight 2  
}  

vrrp_script chk_mantaince_down {
   script "[[ -f /etc/keepalived/down ]] && exit 1 || exit 0"
   interval 1
   weight -2
}

vrrp_instance VI_1 {  
    interface ens33  
    state MASTER  # BACKUP for slave routers
    priority 101  # 100 for BACKUP
    virtual_router_id 51 
    garp_master_delay 1 
  
    authentication {  
        auth_type PASS  
        auth_pass password  
    }  
    track_interface {  
       ens33  
    }  
    virtual_ipaddress {  
        192.168.137.100/24 dev ens33 label ens33:0 
    }  
    track_script {  
        chk_nginx  
        chk_mantaince_down
    }  
  
 
    notify_master "/etc/keepalived/notify.sh master"  
    notify_backup "/etc/keepalived/notify.sh backup"  
    notify_fault "/etc/keepalived/notify.sh fault"  
} 

下面是一个notify.sh脚本的简单示例：

#!/bin/bash
# 

vip=192.168.137.100
contact='root@localhost'

notify() {
    mailsubject="`hostname` to be $1: $vip floating"
    mailbody="`date '+%F %H:%M:%S'`: vrrp transition, `hostname` changed to be $1"
    echo $mailbody | mail -s "$mailsubject" $contact
}

case "$1" in
    master)
        notify master
        /etc/rc.d/init.d/nginx start
        exit 0
    ;;
    backup)
        notify backup
        /etc/rc.d/init.d/nginx stop
        exit 0
    ;;
    fault)
        notify fault
        /etc/rc.d/init.d/nginx stop
        exit 0
    ;;
    *)
        echo 'Usage: `basename $0` {master|backup|fault}'
        exit 1
    ;;
esac

注意：

    1、上面的state为当前节点的起始状态，通常在master/slave的双节点模型中，其一个默认为MASTER，而别一个默认为BACKUP。

    2、priority为当关节点在当前虚拟路由器中的优先级，master的优先级应该大于slave的；

配置keepalived为实现nginx高可用的双主模型配置文件示例：

说明：其基本实现思想为创建两个虚拟路由器，并以两个节点互为主从。

! Configuration File for keepalived  
  
global_defs {  
   notification_email {  
        root@localhost
   }  
   notification_email_from root@localhost
   smtp_connect_timeout 3  
   smtp_server 127.0.0.1  
   router_id LVS_DEVEL01  
}  

vrrp_script chk_nginx {  
    script "killall -0 nginx"  
    interval 1  
    weight 2  
}  

vrrp_script chk_mantaince_down {
   script "[[ -f /etc/keepalived/down ]] && exit 1 || exit 0"
   interval 1
   weight 2
}

vrrp_instance VI_1 {  
    interface ens33  
    state MASTER  # BACKUP for slave routers
    priority 101  # 100 for BACKUP
    virtual_router_id 51 
    garp_master_delay 1 
  
    authentication {  
        auth_type PASS  
        auth_pass password  
    }  
    track_interface {  
       ens33    
    }  
    virtual_ipaddress {  
        192.168.137.100/124 dev ens33 label ens33:0 
    }  
    track_script {  
        chk_nginx  
        chk_mantaince_down
    }  
  
 
    notify_master "/etc/keepalived/notify.sh master"  
    notify_backup "/etc/keepalived/notify.sh backup"  
    notify_fault "/etc/keepalived/notify.sh fault"  
} 

vrrp_instance VI_2 {  
    interface ens33  
    state BACKUP  # BACKUP for slave routers
    priority 100  # 100 for BACKUP
    virtual_router_id 52
    garp_master_delay 1 
  
    authentication {  
        auth_type PASS  
        auth_pass password  
    }  
    track_interface {  
       ens33    
    }  
    virtual_ipaddress {  
        192.168.137.100/24 dev ens33 label ens33:1
    }  
    track_script {  
        chk_nginx  
        chk_mantaince_down
    }    
}

说明：

    对于VI_1和VI_2来说，两个节点要互为主从关系；

keepalived通知脚本进阶示例：

-s, --service SERVICE,...：指定服务脚本名称，当状态切换时可自动启动、重启或关闭此服务；
-a, --address VIP: 指定相关虚拟路由器的VIP地址；
-m, --mode {mm|mb}：指定虚拟路由的模型，mm表示主主，mb表示主备；它们表示相对于同一种服务而方，其VIP的工作类型；
-n, --notify {master|backup|fault}：指定通知的类型，即vrrp角色切换的目标角色；
-h, --help：获取脚本的使用帮助；

#!/bin/bash
# Author: MageEdu <linuxedu@foxmail.com>
# description: An example of notify script
# Usage: notify.sh -m|--mode {mm|mb} -s|--service SERVICE1,... 
-a|--address VIP  -n|--notify {master|backup|falut} -h|--help 

#contact='root@localhost'
helpflag=0
serviceflag=0
modeflag=0
addressflag=0
notifyflag=0

contact='root@localhost'

Usage() {
  echo "Usage: notify.sh [-m|--mode {mm|mb}] [-s|--service SERVICE1,...] 
<-a|--address VIP>  <-n|--notify {master|backup|falut}>" 
  echo "Usage: notify.sh -h|--help"
}

ParseOptions() {
  local I=1;
  if [ $# -gt 0 ]; then
    while [ $I -le $# ]; do
      case $1 in
	  -s|--service)
		[ $# -lt 2 ] && return 3
 	    serviceflag=1
 		services=(`echo $2|awk -F"," '{for(i=1;i<=NF;i++) print $i}'`)
		shift 2 ;;
	  -h|--help)
 		helpflag=1
		return 0
        shift
		;;
	  -a|--address)
		[ $# -lt 2 ] && return 3
	    addressflag=1
		vip=$2
		shift 2
		;;
	  -m|--mode)
		[ $# -lt 2 ] && return 3
		mode=$2
		shift 2
		;;
	  -n|--notify)
		[ $# -lt 2 ] && return 3
		notifyflag=1
		notify=$2
		shift 2
		;;
	  *)
		echo "Wrong options..."
		Usage
		return 7
		;;
       esac
    done
    return 0
  fi
}

#workspace=$(dirname $0)

RestartService() {
  if [ ${#@} -gt 0 ]; then
    for I in $@; do
      if [ -x /etc/rc.d/init.d/$I ]; then
        /etc/rc.d/init.d/$I restart
      else
        echo "$I is not a valid service..."
      fi
    done
  fi
}

StopService() {
  if [ ${#@} -gt 0 ]; then
    for I in $@; do
      if [ -x /etc/rc.d/init.d/$I ]; then
        /etc/rc.d/init.d/$I stop
      else
        echo "$I is not a valid service..."
      fi
    done
  fi
}


Notify() {
    mailsubject="`hostname` to be $1: $vip floating"
    mailbody="`date '+%F %H:%M:%S'`, vrrp transition, `hostname` changed to be $1."
    echo $mailbody | mail -s "$mailsubject" $contact
}


# Main Function
ParseOptions $@
[ $? -ne 0 ] && Usage && exit 5

[ $helpflag -eq 1 ] && Usage && exit 0

if [ $addressflag -ne 1 -o $notifyflag -ne 1 ]; then
  Usage
  exit 2
fi

mode=${mode:-mb}

case $notify in
'master')
  if [ $serviceflag -eq 1 ]; then
      RestartService ${services[*]}
  fi
  Notify master
  ;;
'backup')
  if [ $serviceflag -eq 1 ]; then
    if [ "$mode" == 'mb' ]; then
      StopService ${services[*]}
    else
      RestartService ${services[*]}
    fi
  fi
  Notify backup
  ;;
'fault')
  Notify fault
  ;;
*)
  Usage
  exit 4
  ;;
esac

在keepalived.conf配置文件中，其调用方法如下所示：

 notify_master "/etc/keepalived/notify.sh -n master -a 192.168.137.100" 

 notify_backup "/etc/keepalived/notify.sh -n backup -a 192.168.137.100" 

 notify_fault"/etc/keepalived/notify.sh -n fault –a 192.168.137.100" 

参考博文：http://blog.51cto.com/lizhenliang/1653523