haproxy keepalive实践

1 安装haproxy

wget http://download.openpkg.org/components/cache/haproxy/haproxy-2.6.6.tar.gz
tar -zxvf haproxy-2.6.6.tar.gz
cd haproxy-2.6.6
mkdir -p /app/haproxy
# 安装依赖,解决haproxy.c:80:31的问题
sudo yum -y install gcc openssl-devel pcre-devel systemd-devel
src/haproxy.c:80:31: fatal error: systemd/sd-daemon.h: No such file or directory
 #include <systemd/sd-daemon.h>

sudo make ARCH=x86_64 TARGET=linux3100 USE_PCRE=1 USE_OPENSSL=1 USE_ZLIB=1 USE_SYSTEMD=1 USE_CPU_AFFINITY=1 PREFIX=/app/haproxy

sudo make install PREFIX=/app/haproxy
sudo su - 
echo 'export PATH=$PATH:/app/haproxy/sbin' >> /etc/profile
tail -1 /etc/profile
source /etc/profile

 
cd /app/haproxy
mkdir  -p bin conf logs  var/chroot

配置转发

[root@server-10-160 haproxy]# cat /etc/sysctl.conf
# sysctl settings are defined through files in
# /usr/lib/sysctl.d/, /run/sysctl.d/, and /etc/sysctl.d/.
#
# Vendors settings live in /usr/lib/sysctl.d/.
# To override a whole file, create a new file with the same in
# /etc/sysctl.d/ and put new settings there. To override
# only specific settings, add a file with a lexically later
# name in /etc/sysctl.d/ and put new settings there.
#
# For more information, see sysctl.conf(5) and sysctl.d(5).

fs.file-max = 101365
vm.max_map_count=655360

net.ipv4.ip_forward = 1
net.ipv4.ip_nonlocal_bind = 1
net.ipv4.conf.all.rp_filter = 0
net.ipv4.conf.default.rp_filter = 0
#net.bridge.bridge-nf-call-iptables = 1
#net.bridge.bridge-nf-call-ip6tables = 1

用户

useradd haproxy -s  /sbin/nologin 

配置

vi /usr/lib/systemd/system/haproxy.service

[Unit]
Description=Haproxy
[Service]
Type=forking
ExecStart=/app/haproxy/sbin/haproxy -f /app/haproxy/conf/haproxy.cfg
ExecReload=/app/haproxy/sbin/haproxy -f /app/haproxy/conf/haproxy.cfg -sf 'cat /run/haproxy.pid'
[Install]
WantedBy=multi-user.target

# 加载生效
systemctl daemon-reload

在设置开机自启动的时候systemctl enable haproxy,提示Failed to execute operation: Invalid argument,执行systemctl list-unit-files,那是因为haproxy.service中配置错误

haproxy.cfg配置

global
    chroot /app/haproxy/var/chroot
    group  haproxy
    user haproxy
    daemon
    log 127.0.0.1:514 local0  notice
    #warning info
    pidfile /var/run/haproxy.pid
    #pidfile /drbd_data/haproxy/var/run/haproxy.pid
    maxconn 20000
    spread-checks 3
    # haproxy 2 不支持nbproc
    # nbproc 4

defaults 
    log global
    mode http 
    retries 3 
    option redispatch
    timeout connect 10000
    timeout client 50000
    timeout server 50000
    timeout client 50s
    timeout server 50s
    timeout connect  5s

listen admin
       bind *:8888
       mode  http
       stats enable
       stats hide-version
       stats uri /admin?status
       stats auth ha:ha
       stats   refresh 5s
       bind-process    1    #此行为上面加入到配置文件当中的
       #监控页面的刷新时间


#---------------------------------------------------------------------
#http协议转发 ACL规则 定义转发规则
#acl web-client path_beg -i /vsphere-client
#acl bbs hdr_reg(host) -i ^(bbs.test.com|shequ.test.com|forum) 
#acl monitor hdr_beg(host) -i monitor.test.com    #定义ACL名称,对应的请求的主机头是monitor.test.com  
#acl www hdr_beg(host) -i www.test.com 
#use_backend  cache.test.com if static    
#use_backend  monitor.test.com if bbs or monitor 
#use_backend  www.test.com if www 
#use_backend  vsphere-client if web-client 
#---------------------------------------------------------------------

frontend www
    bind *:80
    mode http
    option forwardfor
    option httpclose
    option httplog      #启用提前将HTTP请求记入日志，不能用于backend区段。
    option dontlognull  #保证HAProxy不记录上级负载均衡发送过来的用于检测状态没有数据的心跳包。   空连接
    option logasap
    balance roundrobin
    log global
    #cookie SERVERID insert indirect #haproxy基于cookie实现会话绑定
    timeout client 15s
    timeout server 15s
    option  allbackups


    #定义ACL名称,对应的请求的主机头是txy.580sc.net
	acl web1 hdr_reg(host) -i ^(bookinfo.580sc.net|cstom.580sc.net|txy2.580sc.net|metersphere-server.580sc.net) 
	acl web2 hdr_beg(host) -i bookinfo.580sc.net
	
	use_backend webporter if web1 or web2 

	default_backend openresty

backend openresty
	mode http
	balance roundrobin
	cookie SERVERID insert indirect nocache
    server server-10-162 10.101.10.162:80    maxconn 5000  check inter 4000  rise 3 fall 5  
    server server-10-163 10.101.10.163:80    maxconn 5000  check inter 4000  rise 3 fall 5
	
#---------------------------------------------------------------------
# round robin balancing between the kubesphere porterLB backends
#---------------------------------------------------------------------

backend webporter              #定义后端服务器群(web server/apache/nginx/iis..) 
    mode http 
    option  forwardfor    #后端服务器(apache/nginx/iis/*),从Http Header中获得客户端IP 
    balance leastconn     #负载均衡的方式,最小连接 
    cookie SERVERID       #插入serverid到cookie中,serverid后面可以定义 
    server eip199 10.101.10.199:80 cookie server1 check inter 2000 rise 3 fall 3 weight 3 


frontend https_frontend
	bind *:443
    mode tcp
    log global
    option tcplog
    timeout client 3600s
    backlog 4096
    maxconn 1000000
	default_backend https_back

backend https_back
	mode tcp
    option log-health-checks
    option redispatch
    option tcplog
    balance roundrobin
    timeout connect 1s
    timeout queue 5s
    timeout server 3600s
    balance roundrobin
	server server-10-162 10.101.10.162:443    maxconn 50000  check inter 4000  rise 3 fall 5
    server server-10-163 10.101.10.163:443    maxconn 50000  check inter 4000   rise 3 fall 5

查看端口统计信息

[root@server-10-161 ~]# ss -tnl
[root@server-10-161 conf]# ss -tnl
State      Recv-Q Send-Q                                          Local Address:Port                                                         Peer Address:Port              
LISTEN     0      128                                                         *:80                                                                      *:*                  
LISTEN     0      128                                                         *:22                                                                      *:*                  
LISTEN     0      20                                                          *:2007                                                                    *:*                  
LISTEN     0      128                                                         *:8888                                                                    *:*                  
LISTEN     0      100                                                 127.0.0.1:25                                                                      *:*                  
LISTEN     0      128                                                         *:443                                                                     *:*                  
LISTEN     0      128                                                      [::]:22                                                                   [::]:*                  
LISTEN     0      100                                                     [::1]:25                                                                   [::]:* 

查看端口对应的进程

[root@server-10-161 conf]# netstat -tunlp | grep 2007
tcp        0      0 0.0.0.0:2007            0.0.0.0:*               LISTEN      914/fileserver      
[root@server-10-161 conf]# netstat -tunlp | grep 8888
tcp        0      0 0.0.0.0:8888            0.0.0.0:*               LISTEN      31602/haproxy 

2 keepalived
Keepalived详解
keepalived官网

yum -y install gcc curl openssl-devel libnl3-devel net-snmp-devel
tar -zxvf keepalived-2.2.7.tar.gz
sudo cp -r keepalived-2.2.7 /app
sudo su -
cd /app/keepalived-2.2.7
./configure --prefix=/app/keepalived 
make && make install
# 设置快捷
ln -s /app/keepalived/sbin/keepalived /usr/sbin/

再centos7.9中，执行下面的命令后，重新make && make install

yum install automake -y
autoreconf -ivf

# 用于解决下面的问题
 cd . && /bin/sh /app/keepalived-2.2.7/build-aux/missing automake-1.16 --foreign
/app/keepalived-2.2.7/build-aux/missing: line 81: automake-1.16: command not found
WARNING: 'automake-1.16' is missing on your system.

设置启动

vi /usr/lib/systemd/system/keepalived.service
[Unit]
Description=LVS and VRRP High Availability Monitor
After=network-online.target syslog.target
Wants=network-online.target

[Service]
Type=forking
PIDFile=/run/keepalived.pid
KillMode=process
EnvironmentFile=-/app/keepalived/etc/sysconfig/keepalived
ExecStart=/app/keepalived/sbin/keepalived  $KEEPALIVED_OPTIONS
ExecReload=/bin/kill -HUP $MAINPID

[Install]
WantedBy=multi-user.target

# 使之生效
systemctl daemon-reload

cd /app/keepalived/etc/keepalived
cp keepalived.conf.sample keepalived.conf
mkdir -p /etc/keepalived
cp /app/keepalived/etc/keepalived/keepalived.conf /etc/keepalived/
# 设置开机启动
systemctl enable keepalived

haproxy+keepalived(主从模式)实现高可用环境的简单配置
参考haproxy+keepalived高可用搭建 实现vip漂移,照抄出现下面的问题，注意网卡的名称

Nov 28 17:07:01 server-10-160 Keepalived[16872]: Starting Keepalived v2.2.7 (01/16,2022)
Nov 28 17:07:01 server-10-160 Keepalived[16872]: Running on Linux 3.10.0-1160.el7.x86_64 #1 SMP Mon Oct 19 16:18:59 UTC 2020 (built for Linux 3.10.0)
Nov 28 17:07:01 server-10-160 Keepalived[16872]: Command line: '/app/keepalived/sbin/keepalived' '--dont-fork' '-D'
Nov 28 17:07:01 server-10-160 Keepalived[16872]: WARNING - using deprecated default config file '/etc/keepalived/keepalived.conf' - please move to '/usr/local/etc/keepalived/ke
Nov 28 17:07:01 server-10-160 Keepalived[16872]: Opening file '/etc/keepalived/keepalived.conf'.
Nov 28 17:07:01 server-10-160 Keepalived[16872]: Configuration file /etc/keepalived/keepalived.conf
Nov 28 17:07:01 server-10-160 Keepalived[16872]: NOTICE: setting config option max_auto_priority should result in better keepalived performance
Nov 28 17:07:01 server-10-160 Keepalived[16872]: Starting VRRP child process, pid=16873
Nov 28 17:07:01 server-10-160 Keepalived_vrrp[16873]: Registering Kernel netlink reflector
Nov 28 17:07:01 server-10-160 Keepalived_vrrp[16873]: Registering Kernel netlink command channel
Nov 28 17:07:01 server-10-160 Keepalived_vrrp[16873]: Script user 'keepalived_script' does not exist
Nov 28 17:07:01 server-10-160 Keepalived_vrrp[16873]: (/etc/keepalived/keepalived.conf: Line 20) WARNING - interface bond0 for vrrp_instance http1 doesn't exist
Nov 28 17:07:01 server-10-160 Keepalived_vrrp[16873]: Non-existent interface specified in configuration

cd /etc/keepalived
vi check_haproxy.sh
#!/bin/bash
if [ $(ps -C haproxy --no-header | wc -l) -eq 0 ]; then    ###判断haproxy是否已经启动
    systemctl start haproxy                                ###如果没有启动，则启动haproxy程序
fi
sleep 2                                                    ###睡眠两秒钟，等待haproxy完全启动
if [ $(ps -C haproxy --no-header | wc -l) -eq 0 ]; then    ###判断haproxy是否已经启动
    systemctl stop keepalived                              ###如果haproxy没有启动起来，则将keepalived停掉，则VIP自动漂移到另外一台haproxy机器,实现了对haproxy的高可用
fi
#
chmod +x check_haproxy.sh

主节点配置

! Copnfiguration File for keepalived
global_defs {
   notification_email {
    xxxx@qq.com
   }
   notification_email_from keepalived@localhost
   smtp_server 127.0.0.1
   smtp_connect_timeout 30
   # 开启SNMP陷阱
   enable_traps
   # 配置于host那么相同
   router_id server-10-160
}

vrrp_script chk_haproxy {
    script "/etc/keepalived/check_haproxy.sh"
    interval 2
    weight 2
}

vrrp_instance http1 {
    state MASTER
    interface eth0
    virtual_router_id 69
    priority 100
    advert_int 1
    nopreempt   
    authentication {
    auth_type PASS
    auth_pass 111111
    }
    virtual_ipaddress {
                #配置vip
        10.101.10.40
    }
    track_script {
        chk_haproxy
    }
}

从节点配置

! Configuration File for keepalived
global_defs {
   notification_email {
    xxx@qq.com
   }
   notification_email_from keepalived@localhost
   smtp_server 127.0.0.1
   smtp_connect_timeout 30
   enable_traps
   router_id server-10-161
}

vrrp_script chk_haproxy {
    script "/etc/keepalived/check_haproxy.sh"
    interval 2
    weight 2
}


vrrp_instance http1 {
    state BACKUP
    interface eth0
    virtual_router_id 69
    priority 70
    advert_int 1
    authentication {
    auth_type PASS
    auth_pass 111111
    }
    virtual_ipaddress {
        10.101.10.40
    }
    track_script {
        chk_haproxy
    }
}

使用下列指令查看IP是否绑定成功（会在eth0上显示出来）,执行ip addr
下面是在主节点

[root@server-10-160 keepalived]# ip add show
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 42:33:a3:8a:71:43 brd ff:ff:ff:ff:ff:ff
    inet 10.101.10.160/24 brd 10.101.10.255 scope global eth0
       valid_lft forever preferred_lft forever
    inet 10.101.10.12/32 scope global eth0
       valid_lft forever preferred_lft forever
    inet6 fe80::4033:a3ff:fe8a:7143/64 scope link 
       valid_lft forever preferred_lft forever

下面是在从节点执行

[root@server-10-161 keepalived]# ip addr
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 3e:44:17:6b:cc:2d brd ff:ff:ff:ff:ff:ff
    inet 10.101.10.161/24 brd 10.101.10.255 scope global eth0
       valid_lft forever preferred_lft forever
    inet6 fe80::3c44:17ff:fe6b:cc2d/64 scope link 
       valid_lft forever preferred_lft forever

将原主节点关机，很快切换在从节点看到vip切换过来了，配置成功。
当原主节点启动后，vip又漂移回去了
看这个脚本是否生效，将两个haproxy都停掉，haproxy没有自动起来
3 防火墙

# 开启防火墙
systemctl start firewalld

Keepalived时主备负载均衡器都有VIP的问题：VRRP协议问题
需要执行脚本

firewall-cmd --direct --permanent --add-rule ipv4 filter INPUT 0  --protocol vrrp -j ACCEPT