基于Keepalived+Haproxy搭建四层负载均衡器

一、前言
    Haproxy是稳定、高性能、高可用性的负载均衡解决方案，支持HTTP及TCP代理后端服务器池，因支持强大灵活的7层acl规则，广泛作为HTTP反向代理。本文则详细介绍如何利用它的四层交换与Keepalived实现一个负载均衡器，适用于Socket、ICE、Mail、Mysql、私有通讯等任意TCP服务。系统架构图如下：
=======================================================================================
  App Server1     App Server1     App Server1     App Server1
      |__________________________________________________________|
                                  |
       LB1-Master<-------VRRP------>LB2-Backup        Haproxy rr
                           |                           Health Check
                           |                           Mail Notify
                           |                           Failover
       |——————————————————————————————————————————————————————|
       |                                                     | TCP MODE                                                      
  Socket App1    Socket App1    Socket App1    Socket App1      
==========================================================================================  

二、平台环境
OS:Centos5.5
MASTER:172.16.34.235
BACKUP:172.16.34.236
VIP:172.16.34.225
Serivce Port:11231

三、平台安装配置
1、添加非本机IP邦定支持

#vi /etc/sysctl.conf
net.ipv4.ip_nonlocal_bind=1
#sysctl –p


2、配置平台日志支持

#vi /etc/syslog.conf
添加：
local3.*        /var/log/haproxy.log
local0.*        /var/log/haproxy.log

#vi /etc/sysconfig/syslog
修改：
SYSLOGD_OPTIONS="-r -m 0"
#/etc/init.d/syslog restart


3、关闭SELINUX
vi /etc/sysconfig/selinux
修改：
SELINUX=disabled
#setenforce 0

====================================================
#!/bin/sh
echo "# Add by hanxiaoqi" >>/etc/sysctl.conf
echo "net.ipv4.ip_nonlocal_bind=1" >>/etc/sysctl.conf
/sbin/sysctl -p
echo "# Set log path by hanxiaoqi" >>/etc/syslog.conf
echo "local3.*                                                /var/log/haproxy.log" >>/etc/syslog.conf
echo "local0.*                                                /var/log/haproxy.log" >>/etc/syslog.conf
sed -i 's#SYSLOGD_OPTIONS="-m 0"#SYSLOGD_OPTIONS="-r -m 0"#g' /etc/sysconfig/syslog
/etc/init.d/syslog restart
======================================================


4、配置iptables，添加VRRP通讯支持

iptables -A INPUT -d 224.0.0.18 -j ACCEPT


5、Keepalived的安装、配置

cd keepalived-1.1.19
./configure --prefix=/usr/local/keepalived
make && make install

cp /usr/local/keepalived/sbin/keepalived /usr/sbin
cp /usr/local/keepalived/etc/sysconfig/keepalived /etc/sysconfig/
cp /usr/local/keepalived/etc/rc.d/init.d/keepalived /etc/init.d/

#vi /etc/keepalived/keepalived.conf
---------------------------------------------
! Configuration File for keepalived
global_defs {
   notification_email {
   hanxiaoqi@xqhan.com
   }
   notification_email_from hanxiaoqi@xqhan.com
   smtp_server 127.0.0.1
   smtp_connect_timeout 3
   router_id LVS_DEVEL
}
vrrp_script chk_haproxy {  
   script "killall -0 haproxy" 
   interval 2  
   weight 2  
} 

vrrp_instance VI_1 {
    state MASTER #从为BACKUP
    interface eth0
    virtual_router_id 51 #路由ID，可通过#tcpdump vrrp查看
    priority 100 #从为80  
    advert_int 1 #VRRP Multicast广播周期秒数
    garp_master_delay 1 #主从切换时间，单位为秒
    authentication {
        auth_type PASS
        auth_pass pwadmin
    }
track_interface {  
       eth0  
       eth1 
    virtual_ipaddress {
        172.16.34.225
    }
track_script {  
       chk_haproxy  
    }
notify_master "/etc/keepalived/Mailnotify.py master" 
notify_backup "/etc/keepalived/Mailnotify.py backup" 
notify_fault "/etc/keepalived/Mailnotify.py fault" 
}  
---------------------------------------------

6、Haproxy的安装与配置

#cd /data/lvs
#wget http://haproxy.1wt.eu/download/1.4/src/haproxy-1.4.11.tar.gz
#tar -zxvf haproxy-1.4.11.tar.gz
#cd haproxy-1.4.11
#make install
#mkdir -p /usr/local/haproxy/etc
#mkdir -p /usr/local/haproxy/sbin
#cp examples/haproxy.cfg /usr/local/haproxy/etc
#ln -s /usr/local/sbin/haproxy /usr/local/haproxy/sbin/haproxy

#vi /usr/local/haproxy/etc/haproxy.cfg
--------------------------------------------------
# this config needs haproxy-1.1.28 or haproxy-1.2.1

global
        #log 127.0.0.1   local0
        log 127.0.0.1   local1 notice
        #log loghost    local0 info
        maxconn 4096
        #chroot /usr/share/haproxy
        uid 99
        gid 99
        daemon
        #debug
        #quiet
        pidfile /usr/local/haproxy/haproxy.pid
       
defaults
        log     global
        mode    http
        #option  httplog
        option  dontlognull
        retries 3
        redispatch
        maxconn 2000
        contimeout      5000
        clitimeout      50000
        srvtimeout      50000

listen  web     172.16.34.225:80
        mode    tcp
        maxconn 2000
        #cookie  SERVERID rewrite
        balance roundrobin
        server  web_1 172.16.34.237:80 cookie  check inter 5000 rise 2 fall 5
        server  web_2 172.16.34.238:80 cookie  check inter 5000 rise 2 fall 5
        srvtimeout      20000
       
listen  stats_auth 172.16.34.238:80
        stats enable
        stats uri /admin-status #管理地址
        stats auth admin:pwadmin #管理帐号:管理密码
        # listen stats_auth 172.16.34.237:80 # backup config
        stats admin if TRUE 
----------------------------------------------------------
7、邮件通知程序(python实现)
#vi /etc/keepalived/Mailnotify.py

#!/usr/local/bin/python  
#coding: utf-8  
from email.MIMEMultipart import MIMEMultipart  
from email.MIMEText import MIMEText  
from email.MIMEImage import MIMEImage  
from email.header import Header  
import sys 
import smtplib 
 
#---------------------------------------------------------------  
# Name:        Mailnotify.py  
# Purpose:     Mail notify to SA  
# Author:      hanxiaoqi 
# Email:       hanxiaoqi@xqhan.com 
# Created:     2012/11/28  
# Copyright:   (c) 2012  
#--------------------------------------------------------------  
strFrom = 'hanxiaoqi@xqhan.com'  
strTo = 'hanxiaoqi@xqhan.com'  
smtp_server='smtp.xqhan.com'
smtp_user='hanxiaoqi@xqhan.com'   
smtp_pass='123456'
 
if sys.argv[1]!="master" and sys.argv[1]!="backup"  and sys.argv[1]!="fault":  
    sys.exit()  
else:  
    notify_type=sys.argv[1]  
 
 
mail_title='[紧急]负载均衡器邮件通知'  
mail_body_plain=notify_type+'被激活，请做好应急处理。'  
mail_body_html='<b><font color=red>'+notify_type+'被激活，请做好应急处理。</font></b>'  
 
msgRoot = MIMEMultipart('related')  
msgRoot['Subject'] =Header(mail_title,'utf-8')  
msgRoot['From'] = strFrom  
msgRoot['To'] = strTo  
 
msgAlternative = MIMEMultipart('alternative')  
msgRoot.attach(msgAlternative)  
 
msgText = MIMEText(mail_body_plain, 'plain', 'utf-8')  
msgAlternative.attach(msgText)  
 
msgText = MIMEText(mail_body_html, 'html','utf-8')  
msgAlternative.attach(msgText)  
 
smtp = smtplib.SMTP()  
smtp.connect(smtp_server)  
smtp.login(smtp_user,smtp_pass)  
smtp.sendmail(strFrom, strTo, msgRoot.as_string())  
smtp.quit()
--------------------------------------------------
注：修改成系统python实际路径“#!/usr/local/bin/python”(第一行)
#chmod +x /etc/keepalived/Mailnotify.py
#/usr/local/haproxy/sbin/haproxy -f /usr/local/haproxy/etc/haproxy.cfg
#service keepalived start

8、查看VRRP通讯记录
#tcpdump vrrp


tcpdump: verbose output suppressed, use -v or -vv for full protocol decode
listening on eth0, link-type EN10MB (Ethernet), capture size 96 bytes
15:49:05.270017 IP 192.168.0.20 > VRRP.MCAST.NET: VRRPv2, Advertisement, vrid 50, prio 100, authtype simple, intvl 1s, length 20


四、Haproxy界面
访问http://172.16.34.235/admin-status，输入帐号admin密码pwadmin进入管理监控平台。

haproxy-1.4.9以后版本最大的亮点是添加了手工启用/禁用功能，对升级变更应用时非常有用。