实现 KeepAlive + Haproxy + Nginx 七层负载均衡 + 动静分离

最新推荐文章于 2024-05-11 14:44:15 发布

笨小孩@GF 知行合一

最新推荐文章于 2024-05-11 14:44:15 发布

阅读量10w+

点赞数 6

分类专栏：学习笔记文章标签： LInux Nginx KeepAliaved Haproxy 读写分离

本文链接：https://blog.csdn.net/gaofei0428/article/details/118596110

版权

学习笔记专栏收录该内容

125 篇文章 69 订阅

订阅专栏

试验环境：
系统统一使用 CentOS 7.6
192.168.2.6 KeepAlived + Haproxy
192.168.2.10 KeepAlived + Haproxy
192.168.2.4 Nginx 静态服务器1
192.168.2.7 Nginx 静态服务器2
192.168.2.11 Nginx + php 动态服务器
192.168.2.12 Nginx + php 动态服务器2
Firewalld 、iptables 以及 SElinux 均为关闭状态
首先配置 Keepalived + Haproxy
在 192.168.2.6 和 2.10 上配置好互为主备
安装过程请参考：https://blog.csdn.net/gaofei0428/article/details/104454894?spm=1001.2014.3001.5501
添加（VIP）虚拟网卡
ifconfig ens33:0 192.168.2.168 netmask 255.255.255.255 up
下面只列出配置文件和运行状态
192.168.2.6
vim /etc/keepalived/keepalived.conf
! Configuration File for keepalived

global_defs {
#notification_email {
# acassen@firewall.loc
# failover@firewall.loc
# sysadmin@firewall.loc
#}
#notification_email_from Alexandre.Cassen@firewall.loc
#smtp_server 192.168.200.1
#smtp_connect_timeout 30
router_id LVS_DEVEL
vrrp_skip_check_adv_addr
#vrrp_strict
vrrp_garp_interval 0.001
vrrp_gna_interval 0
}

vrrp_instance DR_1 {
state MASTER
interface ens33
virtual_router_id 51
lvs_sync_daemon_inteface ens33
priority 100
advert_int 1
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
192.168.2.168
}
}

vrrp_instance DR_2 {
state BACKUP
interface ens33
virtual_router_id 53
lvs_sync_daemon_inteface ens33
priority 100
advert_int 1
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
192.168.2.169
}
}
192.168.2.10
cat /etc/keepalived/keepalived.conf
! Configuration File for keepalived

global_defs {
#notification_email {
# acassen@firewall.loc
# failover@firewall.loc
# sysadmin@firewall.loc
#}
#notification_email_from Alexandre.Cassen@firewall.loc
#smtp_server 192.168.200.1
#smtp_connect_timeout 30
router_id LVS_DEVEL
vrrp_skip_check_adv_addr
#vrrp_strict
vrrp_garp_interval 0.001
vrrp_gna_interval 0
}

vrrp_instance DR_1 {
state BACKUP
interface ens33
virtual_router_id 51
lvs_sync_daemon_inteface ens33
priority 90
advert_int 1
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
192.168.2.168
}
}

vrrp_instance DR_2 {
state MASTER
interface ens33
virtual_router_id 53
lvs_sync_daemon_inteface ens33
priority 110
advert_int 1
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
192.168.2.169
}
}

下面开始安装配置 Haproxy
官网： https://www.haproxy.com （访问比较慢）
github : https://github.com/haproxy/haproxy/releases/

首先要安装 lua5.4.3
wget https://www.lua.org/ftp/lua-5.4.3.tar.gz
tar xf lua-5.4.3.tar.gz
cd lua-5.4.3
make linux
make install INSTALL_TOP=/usr/local/lu

下面开始安装 haproxy
安装依赖
yum install vim iotop bc gcc gcc-c++ glibc glibc-devel pcre pcre-devel openssl openssl-devel zip unzip zlib-devel net-tools lrzsz tree ntpdate telnet lsof tcpdump wget libevent libevent-devel bc systemd-devel bash-completion traceroute -y
yum install libtermcap-devel ncurses-devel libevent-devel readline-devel -y
下载 haproxy ，这里做实验使用最新测试版本，生产环境请选择稳定版本（2.4）
wget https://github.com/haproxy/haproxy/archive/refs/tags/v2.5-dev1.tar.gz
tar xf v2.5-dev1.tar.gz
make -j $(nproc) TARGET=linux-glibc USE_OPENSSL=1 USE_ZLIB=1 USE_LUA=1 LUA_LIB=/usr/local/lua/lib/ LUA_INC=/usr/local/lua/include/ USE_PCRE=1 USE_SYSTEMD=1
make install PREFIX=/usr/local/haproxy

cp /usr/local/haproxy/sbin/haproxy /usr/sbin/ 复制 haproxy 运行脚本
cp ./examples/haproxy.init /etc/init.d/haproxy 复制 haproxy 启动脚本
修改如下
#!/bin/sh

# chkconfig: - 85 15

# description: HA-Proxy server

# processname: haproxy

# config: /usr/local/haproxy/etc/haproxy.cfg

# pidfile: /usr/local/haproxy/run/haproxy.pid

# Source function library.

if [ -f /etc/init.d/functions ]; then

. /etc/init.d/functions

elif [ -f /etc/rc.d/init.d/functions ] ; then

. /etc/rc.d/init.d/functions

else

exit 0

fi

# Source networking configuration.

#. /etc/sysconfig/network

# Check that networking is up.

#[ "$NETWORKING" = "no" ] && exit 0

# This is our service name

BASENAME=`haproxy`

BIN=/usr/sbin/haproxy

CFG=/usr/local/haproxy/etc/haproxy.cfg

[ -f $CFG ] || exit 1

PIDFILE=/usr/local/haproxy/run/haproxy.pid

LOCKFILE=/usr/local/haproxy/run/haproxy

RETVAL=0

start() {

quiet_check

if [ $? -ne 0 ]; then

echo "Errors found in configuration file, check it with '$BASENAME check'."

return 1

fi

echo -n "Starting $BASENAME: "

daemon $BIN -D -f $CFG -p $PIDFILE

RETVAL=$?

echo

[ $RETVAL -eq 0 ] && touch $LOCKFILE

return $RETVAL

}

stop() {

echo -n "Shutting down $BASENAME: "

killproc $BASENAME -USR1

RETVAL=$?

echo

[ $RETVAL -eq 0 ] && rm -f $LOCKFILE

[ $RETVAL -eq 0 ] && rm -f $PIDFILE

return $RETVAL

}

restart() {

quiet_check

if [ $? -ne 0 ]; then

echo "Errors found in configuration file, check it with '$BASENAME check'."

return 1

fi

stop

start

}

reload() {

if ! [ -s $PIDFILE ]; then

return 0

fi

quiet_check

if [ $? -ne 0 ]; then

echo "Errors found in configuration file, check it with '$BASENAME check'."

return 1

fi

$BIN -D -f $CFG -p $PIDFILE -sf $(cat $PIDFILE)

}

check() {

$BIN -c -q -V -f $CFG

}

quiet_check() {

$BIN -c -q -f $CFG

}

rhstatus() {

status $BASENAME

}

condrestart() {

[ -e $LOCKFILE ] && restart || :

}

# See how we were called.

case "$1" in

start)

start

;;

stop)

stop

;;

restart)

restart

;;

reload)

reload

;;

condrestart)

condrestart

;;

status)

rhstatus

;;

check)

check

;;

*)

echo $"Usage: $BASENAME {start|stop|restart|reload|condrestart|status|check}"

exit 1

esac
exit $?

chmod 755 /etc/init.d/haproxy

创建运行用户
useradd haproxy -Ms /sbin/nologin

mkdir /usr/local/haproxy/etc

vim /usr/local/haproxy/etc/haproxy.cfg #新创建该文件
global
   log 127.0.0.1 local0
   #log 127.0.0.1 local1 notice
   #log loghost local0 info
   maxconn 4096
   chroot /usr/local/haproxy
   uid 99
   gid 99
   daemon
   #nbproc 1
   pidfile /usr/local/haproxy/run/haproxy.pid
defaults
   log global
   log 127.0.0.1 local3
   mode http
   option httplog
   option httpclose
   option dontlognull
   option forwardfor
   option redispatch
   retries 2
   maxconn 2000
   balance roundrobin
   stats uri /haproxy-stats
   timeout connect 5000
   timeout client 50000
   timeout server 50000
   mode http
   option httpchk GET /index.php
frontend http
   bind *:80
   default_backend http_back
backend http_back
   server Ngxin_html1 192.168.2.4:80 weight 3 check
   server Nginx_html2 192.168.2.7:80 weight 3 check
   server Nginx_php1 192.168.2.11:80 weight 3 check
   server Nginx_php2 192.168.2.12:80 weight 3 check
#server node1 192.168.179.131:8081 check inter 2000 rise 3 fall 3 weight 30
# inter 2000 健康检查时间间隔 2 秒
# rise 3 检测多少次才认为是正常的
# fall 3 失败多少次才认为是不可用的
# weight 30 权重

关于负载均衡算法
#source 根据请求源 IP
#static-rr 根据权重
#leastconn 最少连接者先处理
#uri 根据请求的 uri
#url_param 根据请求的 url 参数
#rdp-cookie 据据 cookie(name)来锁定并哈希每一次请求
#hdr(name) 根据 HTTP 请求头来锁定每一次 HTTP 请求
#roundrobin 轮询方式
mkdir -p /usr/local/haproxy/run 创建 PID 文件存放目录及赋予权限
chown -R nobody. /usr/local/haproxy/
配置日志收集
vim /etc/rsyslog.conf
$ModLoad imuxsock # provides support for local system logging (e.g. via logger command)
$ModLoad imjournal # provides access to the systemd journal
$ModLoad imudp
$UDPServerRun 514
$WorkDirectory /var/lib/rsyslog
$ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat
$IncludeConfig /etc/rsyslog.d/*.conf
$OmitLocalLogging on
$IMJournalStateFile imjournal.state
*.info;mail.none;authpriv.none;cron.none /var/log/messages
authpriv.* /var/log/secure
mail.* -/var/log/maillog
cron.* /var/log/cron
*.emerg :omusrmsg:*
uucp,news.crit /var/log/spooler
local7.* /var/log/boot.log
local3.* /var/log/haproxy.log
local0.* /var/log/haproxy.log

修改完配置文件一定要重启服务或者 reload

依次启动 keepalive 和 haproxy
systemctl restart keepalived.service
/usr/local/haproxy/sbin/haproxy -f /usr/local/haproxy/etc/haproxy.cfg
停止 haproxy
killall haproxy 或者 pkill haproxy
没有 killall 命令？安装 yum -y install psmisc
设置开机启动
chkconfig --add haproxy
chkconfig haproxy on
chkconfig --list haproxy

然后在另外一台 keepalive 上做同样配置

打卡浏览器访问 VIP

rr 论循成功

测试漂移
首先停止 192.168.2.6 的 keepalive

观察另外一台 192.168.2.10 的运行情况

成功漂移，测试浏览网页

停止 192.168.2.10 后依次打开 192.168.2.6 和 2.10

查看 HAproxy 的监控页面

下面配置 nginx 动静分离
修改配置文件
vim /usr/local/haproxy/etc/haproxy.cfg
global
log 127.0.0.1 local0
#log 127.0.0.1 local1 notice
#log loghost local0 info
maxconn 4096
chroot /usr/local/haproxy
uid 99
gid 99
daemon
#nbproc 1
pidfile /usr/local/haproxy/run/haproxy.pid
defaults
log global
log 127.0.0.1 local3
mode http
option httplog
option httpclose
option dontlognull
option forwardfor
option redispatch
retries 2
maxconn 2000
balance roundrobin
stats uri /haproxy-stats
timeout connect 5000
timeout client 50000
timeout server 50000
mode http
frontend http
bind *:80
acl url_static path_beg -i /upload/
acl url_static path_end -i .jpg .jpeg .gif .png .css .js
acl url_php path_end -i .php
use_backend imgserver if url_static
use_backend php if url_php
default_backend static

backend static
#cookie srv insert nocache
balance roundrobin
option httpchk GET /index.html
server web1 192.168.2.4:80 cookie web1 weight 3 check inter 2000 rise 2 fall 3
server web2 192.168.2.7:80 cookie web1 weight 3 check inter 2000 rise 2 fall 3
backend php
balance roundrobin
option httpchk GET /index.php
server php1 192.168.2.11:80 check inter 2000 fall 3
backend imgserver
balance roundrobin
server img1 192.168.2.12:80 check inter 2000 fall 3
重启服务
pkill haproxy
/usr/local/haproxy/sbin/haproxy -f /usr/local/haproxy/etc/haproxy.cfg

访问静态页面 html 就分配到 2.4 和 2.7
访问动态页面 php 就分配到 2.11
访问图片 jpg png等就分配到 2.12

测试 keepalive

Zabbix 监控 Haproxy
根据官方文档提示，在 /usr/local/haproxy/etc/haproxy.cfg 末尾添加

frontend stats
bind *:8404
stats enable
stats uri /stats
stats refresh 10s

重启 haproxy 服务，然后在 zabbix web 管理界面添加
选择 haproxy 主机，左键-------配置-----模板-----
搜索 HAProxy by HTTP -----更新

Zabbix 监控 KeepAlived
编辑 shell 脚本
mkdir /etc/zabbix/scripts
vim /etc/zabbix/scripts/check_keepalived.sh
#!/bin/bash

if [[ `ip address show dev ens33 |grep -E "192.168.2.168/32|192.168.2.169/32"|wc -l` == "0" ]]; then
echo "0";
else
echo "1";
fi
chmod +x /etc/zabbix/scripts/check_keepalived.sh

在配置文件末尾添加
tail -n2 /etc/zabbix/zabbix_agentd.conf
UnsafeUserParameters=1
UserParameter=check_keepalived,/etc/zabbix/scripts/check_keepalived.sh

重启 zabbix-agent 服务
然后在 web 界面添加监控项
配置 ----主机----监控项----创建监控项

创建触发器

这里的问题和恢复表达式的值，就是基于 shell 脚本的判断值
添加

测试依次停止 2台服务器的 keepalive

查看报表-----动作日志

停止另外一台

依次恢复

静态页面缓存
在配置文件中添加以下内容，开启缓存
cache section
cache
声明cache区域，分配一个共享缓存名称。

total-max-size
定义使用内存大小，这个块将会被分为1kb的小块，它的最大值为4095

max-object-size
定义缓存对象的大小，不能大于total-max-size的一般。如果没有定义，将会等于256个cache size。当objects的大小大于该值后将不会被缓存

max-age
定义缓存过期时间，默认60s，
proxy section
http-request cache-use [{if | unless} ]
试着去发送中缓存的对象。
http-response cache-store [{if | unless} ]
将http-response缓存起来，
http-request cache-use foobar
http-response cache-store foobar
cache foobar
total-max-size 4
max-age 240