简介介绍:
Nginx大家众所周知,可以做缓存服务器,负载均衡器,那么当Nginx做负载均衡器时,必然要抗受多个客户端的访问。一台负载均衡器必然会受到压力瓶颈,那么我们就要多搞几台Nginx负载均衡器,那么多台Nginx就面临这ip地址的问题。当一台Nginx挂掉后,我们还需要将另一台上线,这整的很麻烦,这时我们就需要用keepalived进行主备负载均衡器的切换。
二:keepalived实现主备切换原理:基于虚拟路由协议vrrp,实现地址游离。我们使用keepalived在两台负载均衡主机上配置虚拟ip地址,进行游离,主节点定期向备用节点发送心跳包,当主节点岩机之后,备节点能够抢占主节点的资源,当主节点恢复正常后,再抢回资源,实现高可用。
三:keepalived+Nginx的配置:
后端服务器 | 负载均衡器+keepalived |
---|---|
node3:10.5.100.183 | node2:10.5.100.208 |
node1:10.5.100.207 | node4:10.5.100.146 |
Node3节点与Node1节点为后端处理web请求的服务器。
Node2节点与Node4节点做负载均衡器高可用。
一:配置后端web服务器:
Node1节点:
[root@node1 ~]# yum install httpd -y
[root@node1 ~]# cd /var/www/html/
[root@node1 html]# echo "<>this is node1<>" > /var/www/html/index.html
[root@node1 ~]# systemctl restart httpd
[root@node1 ~]# curl http://10.5.100.207
<>this is node1<>
Node3节点:
[root@node1 ~]# yum install httpd -y
[root@node1 ~]# cd /var/www/html/
[root@node1 html]# echo "<>this is node3<>" > /var/www/html/index.html
[root@node1 ~]# systemctl restart httpd
[root@node1 ~]# curl http://10.5.100.183
<>this is node3<>
二:配置负载均衡主机
Node2节点:
[root@node2 ~]# yum install nginx -y
[root@node2 ~]# vim /etc/nginx/nginx.conf 编辑Nginx配置文件,配置upstream后端负载均衡主机
#user nobody;
worker_processes 2;
#error_log logs/error.log;
#error_log logs/error.log notice;
#error_log logs/error.log info;
#pid logs/nginx.pid;
events {
worker_connections 10240;
}
http {
include mime.types;
default_type application/octet-stream;
#log_format main '$remote_addr - $remote_user [$time_local] "$request" '
# '$status $body_bytes_sent "$http_referer" '
# '"$http_user_agent" "$http_x_forwarded_for"';
#access_log logs/access.log main;
sendfile on;
#tcp_nopush on;
#keepalive_timeout 0;
keepalive_timeout 65;
#gzip on;
upstream webserver { 配置负载均衡主机。
server 10.5.100.183:80 weight=1;
server 10.5.100.207:80 weight=1;
}
server {
listen 80;
server_name localhost;
#charset koi8-r;
#access_log logs/host.access.log main;
location / {
root html;
index index.html index.htm;
proxy_pass http://webserver;
}
}
[root@node2 ~]# curl http://10.5.100.208 用本地地址测试负载均衡情况
<h1>this is node1</h1>
[root@node2 ~]# curl http://10.5.100.208
<h1>this is node3</h1>
[root@node2 ~]#
Node4节点与Node2是负载均衡器的高可用模式,所以提供相同的服务。
Node4节点:
[root@node4 ~]# yum install nginx -y
[root@node4 ~]# vim /etc/nginx/nginx.conf
#user nobody;
worker_processes 2;
#error_log logs/error.log;
#error_log logs/error.log notice;
#error_log logs/error.log info;
#pid logs/nginx.pid;
events {
worker_connections 10240;
}
http {
include mime.types;
default_type application/octet-stream;
#log_format main '$remote_addr - $remote_user [$time_local] "$request" '
# '$status $body_bytes_sent "$http_referer" '
# '"$http_user_agent" "$http_x_forwarded_for"';
#access_log logs/access.log main;
sendfile on;
#tcp_nopush on;
#keepalive_timeout 0;
keepalive_timeout 65;
#gzip on;
upstream webserver { 配置负载均衡主机。
server 10.5.100.183:80 weight=1;
server 10.5.100.207:80 weight=1;
}
server {
listen 80;
server_name localhost;
#charset koi8-r;
#access_log logs/host.access.log main;
location / {
root html;
index index.html index.htm;
proxy_pass http://webserver;
}
[root@node4 ~]# curl http://10.5.100.146 用本地地址测试负载均衡情况。
<h1>this is node3</h1>
[root@node4 ~]# curl http://10.5.100.146
<h1>this is node1</h1>
[root@node4 ~]# curl http://10.5.100.146
<h1>this is node3</h1>
[root@node4 ~]# curl http://10.5.100.146
<h1>this is node1</h1>
[root@node4 ~]#
三:两台主机装载keepalived实现当主节点宕机时高可用方案。Nginx+keepalived实现高可用时,利用脚本测试Nginx是否运行正常来检测。
Node2节点。
[root@node2 ~]# yum install keepalived -y
[root@node2 ~]# yum install psmisc -y 这是killall所需软件包。
[root@node2 ~]# vim /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
notification_email {
root@localhost
}
notification_email_from admin@localhost
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id node2.yan.com
vrrp_mcast_group 224.18.0.100
}
vrrp_script chk_nginx {
script "/usr/bin/killall -0 nginx &> /dev/null" 这里是脚本监测nginx的运行状态。
interval 1
weight -10
}
global_defs { 这几项可有可无啊
router_id k8s
script_user root
enable_script_security
}
vrrp_instance VI_1 {
state BACKUP
interface ens33
virtual_router_id 61
priority 99
advert_int 1
authentication {
auth_type PASS
auth_pass 59a76f0a
}
virtual_ipaddress {
10.5.100.89/24 设置的虚拟ip地址。
}
track_script {
chk_nginx 在虚拟实例中调用上面脚本。
}
}
[root@node2 ~]# vim /etc/sysconfig/keepalived 编辑keepalived日志配置,开启keepalived配置文件
# Options for keepalived. See `keepalived --help' output and keepalived(8) and
# keepalived.conf(5) man pages for a list of all options. Here are the most
# common ones :
#
# --vrrp -P Only run with VRRP subsystem.
# --check -C Only run with Health-checker subsystem.
# --dont-release-vrrp -V Dont remove VRRP VIPs & VROUTEs on daemon stop.
# --dont-release-ipvs -I Dont remove IPVS topology on daemon stop.
# --dump-conf -d Dump the configuration data.
# --log-detail -D Detailed log messages.
# --log-facility -S 0-7 Set local syslog facility (default=LOG_DAEMON)
#
KEEPALIVED_OPTIONS="-D -S 3" 添加-S指明3级别
[root@node2 ~]# vim /etc/rsyslog.conf 编辑日志服务配置文件,添加keepalived日志文件
# Save boot messages also to boot.log
local7.* /var/log/boot.log
local3.* /var/log/keepalived/keepalived.log
[root@node2 ~]# systemctl restart rsyslog
[root@node2 ~]# systemctl restart keepalived
[root@node2 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
link/ether 00:0c:29:ad:af:e0 brd ff:ff:ff:ff:ff:ff
inet 10.5.100.208/24 brd 10.5.100.255 scope global noprefixroute dynamic ens33
valid_lft 637264sec preferred_lft 637264sec
Node4主节点:
[root@node4 ~]# yum install keepalived -y
[root@node4 ~]# vim /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
notification_email {
root@localhost
}
notification_email_from admin@localhost
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id node4.yan.com
#vrrp_mcast_group 224.18.0.100
}
vrrp_script chk_nginx {
script "/usr/bin/killall -0 nginx &> /dev/null"
interval 1
weight -10
}
vrrp_instance VI_1 {
state MASTER
interface enp2s0
virtual_router_id 61
priority 100
advert_int 1
authentication {
auth_type PASS
auth_pass 59a76f0a
}
virtual_ipaddress {
10.5.100.89/24
}
track_script {
chk_nginx
}
}
[root@node4 ~]# vim /etc/sysconfig/keepalived 编辑keepalived日志配置,开启keepalived配置文件
# Options for keepalived. See `keepalived --help' output and keepalived(8) and
# keepalived.conf(5) man pages for a list of all options. Here are the most
# common ones :
#
# --vrrp -P Only run with VRRP subsystem.
# --check -C Only run with Health-checker subsystem.
# --dont-release-vrrp -V Dont remove VRRP VIPs & VROUTEs on daemon stop.
# --dont-release-ipvs -I Dont remove IPVS topology on daemon stop.
# --dump-conf -d Dump the configuration data.
# --log-detail -D Detailed log messages.
# --log-facility -S 0-7 Set local syslog facility (default=LOG_DAEMON)
#
KEEPALIVED_OPTIONS="-D -S 3" 添加-S指明3级别
[root@node4 ~]# vim /etc/rsyslog.conf 编辑日志服务配置文件,添加keepalived日志文件
# Save boot messages also to boot.log
local7.* /var/log/boot.log
local3.* /var/log/keepalived/keepalived.log
[root@node2 ~]# systemctl restart rsyslog
[root@node4 ~]# systemctl restart keepalived
[root@node4 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: enp2s0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
link/ether 00:23:24:5f:f9:eb brd ff:ff:ff:ff:ff:ff
inet 10.5.100.146/24 brd 10.5.100.255 scope global noprefixroute dynamic enp2s0
valid_lft 674267sec preferred_lft 674267sec
inet 10.5.100.89/24 scope global secondary enp2s0
valid_lft forever preferred_lft forever
inet6 fe80::6b73:a081:3ef2:9683/64 scope link noprefixroute
valid_lft forever preferred_lft forever
当keepalived配置完成后,关闭防火墙,默认虚拟ip地址在Node4节点上。说明Node4节点为主负载均衡器。
现在我们在Node4主节点访问虚拟ip地址,看会不会负载均衡。
[root@node4 ~]# curl http://10.5.100.89
<h1>this is node3</h1>
[root@node4 ~]# curl http://10.5.100.89
<h1>this is node1</h1>
[root@node4 ~]#
现在我们来验证对Nginx负载均衡器做高可用,测试当我们在Node4节点关闭Nginx进程,查看虚拟IP的转换,并且客户端再次请求,看会不会接受到后端服务器处理。
[root@node4 ~]# systemctl stop nginx
[root@node4 ~]# ss -tnl
State Recv-Q Send-Q Local Address:Port Peer Address:Port
LISTEN 0 128 *:22 *:*
LISTEN 0 100 127.0.0.1:25 *:*
LISTEN 0 128 :::22 :::*
LISTEN 0 100 ::1:25 :::*
[root@node2 ~]# ip a 当主节点nginx进程down掉之后,在Node2备节点看到了虚拟ip地址
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
link/ether 00:0c:29:ad:af:e0 brd ff:ff:ff:ff:ff:ff
inet 10.5.100.208/24 brd 10.5.100.255 scope global noprefixroute dynamic ens33
valid_lft 636887sec preferred_lft 636887sec
inet 10.5.100.89/24 scope global secondary ens33
valid_lft forever preferred_lft forever
通过浏览器再次访问10.5.100.89,已经没有任何问题,实现了高可用。
三:在Node2与Node4中配置状态变化脚本,利用脚本配置主节点Nginx进程永远不关闭,就算关闭也要自动重启。
Node2节点:
[root@node2 ~]# vim /etc/keepalived/notify.sh
#!/bin/bash
vip=10.5.100.89
contact='root@localhost'
notify() {
mailsubject="`hostname` to be $1: $vip floating"
mailbody="`date '+%F %H:%M:%S'`: vrrp transition, `hostname` change to be $1"
echo $mailbody | mail -s "$mailsubject" $contact
}
case "$1" in
master)
notify master
exit 0
;;
backup)
notify backup
exit 0
;;
fault)
notify fault
exit 0
;;
*)
echo 'usage:`basename $0` {master|backup|fault}'
exit 1
;;
esac
Node4节点:
[root@node4 ~]# vim /etc/keepalived/notify.sh
#!/bin/bash
vip=10.5.100.89
contact='root@localhost'
notify() {
mailsubject="`hostname` to be $1: $vip floating"
mailbody="`date '+%F %H:%M:%S'`: vrrp transition, `hostname` change to be $1"
echo $mailbody | mail -s "$mailsubject" $contact
}
case "$1" in
master)
notify master
systmectl restart nginx 这里表示当主节点的Nginx进程关闭时,成为主重启。
exit 0
;;
backup)
notify backup
systmectl restart nginx 这里表示当主节点的Nginx进程关闭时,马上成为备节点,立即重启NGINX
在成为主机点。
exit 0
;;
fault)
notify fault
exit 0
;;
*)
echo 'usage:`basename $0` {master|backup|fault}'
exit 1
;;
esac
测试:当关闭主节点的Nginx进程,立马Nginx进程又起来了
[root@node4 ~]# systemctl stop nginx
[root@node4 ~]# ss -tnl
State Recv-Q Send-Q Local Address:Port Peer Address:Port
LISTEN 0 128 *:8080 *:*
LISTEN 0 128 *:80 *:*
LISTEN 0 128 *:22 *:*
LISTEN 0 100 127.0.0.1:25 *:*
LISTEN 0 128 :::22 :::*
LISTEN 0 100 ::1:25 :::*
[root@node4 ~]#
You have new mail in /var/spool/mail/root
[root@node4 ~]#
[root@node4 ~]#
[root@node4 ~]# mail
Heirloom Mail version 12.5 7/5/10. Type ? for help.
"/var/spool/mail/root": 6 messages 4 new 5 unread
1 root Wed Jan 8 11:34 19/699 "node4 to be master: 10.5.100.89 floating"
U 2 root Wed Jan 8 11:35 19/698 "node4 to be backup: 10.5.100.89 floating"
>N 3 root Wed Jan 8 11:38 18/688 "node4 to be master: 10.5.100.89 floating"
N 4 root Wed Jan 8 13:43 18/688 "node4 to be master: 10.5.100.89 floating"
N 5 root Wed Jan 8 13:45 18/688 "node4 to be backup: 10.5.100.89 floating"
N 6 root Wed Jan 8 13:45 18/688 "node4 to be master: 10.5.100.89 floating"
[root@node4 ~]# ip a 查看虚拟IP地址仍没有转移。
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: enp2s0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
link/ether 00:23:24:5f:f9:eb brd ff:ff:ff:ff:ff:ff
inet 10.5.100.146/24 brd 10.5.100.255 scope global noprefixroute dynamic enp2s0
valid_lft 616026sec preferred_lft 616026sec
inet 10.5.100.89/24 scope global secondary enp2s0
valid_lft forever preferred_lft forever
inet6 fe80::6b73:a081:3ef2:9683/64 scope link noprefixroute
valid_lft forever preferred_lft forever
像上述所表示的就是将主节点一直是主机点进程一关闭就重启,这势必会影响一段过程中用户的访问。
总结:Nginx+keepalived实现高可用时,基于keepalived的虚拟路由vrrp,提供相同功能的主机,具有相同的访问入口,根据脚本来判断Nginx的运行状态来做高可用方案。