1.简介
keepalived是一个通用的高可用
vrrp virtual router redundancy Protocol
虚拟路由
2.安装和配置
2.2 安装
官网:http://keepalived.org/
192.168.1.103
192.168.1.105
启动keepalived
root@ubuntu20:~# journalctl -u keepalived.service
-- Logs begin at Mon 2021-08-30 21:40:06 CST, end at Fri 2023-07-14 21:23:59 CST. --
Jul 14 21:22:44 ubuntu20.04.3.example.com systemd[1]: Condition check resulted in Keepalive Daemon (LVS and VRRP) being skipped.
ubuntu默认没有配置文件,拷贝即可,dpkg -S keepalived 查看文件
cp /usr/share/doc/keepalived/samples/keepalived.conf.sample /etc/keepalived/keepalived.conf
源码安装
[root@openvpn-server apps]# tar -xf keepalived-2.2.8.tar.gz
[root@openvpn-server apps]# cd keepalived-2.2.8/
./configure --prefix=/usr/local/keepalived/ --disable-fwmark
安装后警告
*** WARNING - this build will not support IPVS with IPv6. Please install libnl/libnl-3 dev libraries to support IPv6 with IPVS.
执行
yum install libnl3-devel
make && make install
启动提示Jul 14 21:49:20 openvpn-server Keepalived[182187]: Config files missing '/usr/local/keepalived/etc/keepalived/keepalived.conf
拷贝配置文件
cp /usr/local/keepalived/etc/keepalived/keepalived.conf.sample /usr/local/keepalived/etc/keepalived/keepalived.conf
修改service指定配置文件路径,可以放在/usr/local/keepalived/etc/下
ExecStart=/usr/local/keepalived/sbin/keepalived -f /usr/local/keepalived/etc/keepalived.conf --dont-fork $KEEPALIVED_OPTIONS
添加主机路由
105上面添加到103主机的路由
route add -net 192.168.200.0/24 gw 192.168.1.103 dev eth0
route add -host 192.168.200.11 gw 192.168.1.103 #单独ip添加
router_id ka1.example.com #每个keepalived主机唯一标识,建议使用当前主机名,如果多节点重
名可能会影响切换脚本执行
vrrp_mcast_group4 224.0.0.18 通过组播地址通信,确定是否同一个集群
配置日志
Keepalived 日志默认输出到系统日志 /var/log/messages 文件中,可以修改配置使其输出到一个独立的 log 文件中,配置rsyslog
源码编译安装的位置在安装目录下的etc/sysconfig
vim /usr/local/keepalived/etc/sysconfig/keepalived
KEEPALIVED_OPTIONS="-D -S 6
ubuntu yum安装的文件位置在/etc/default/keepalived,当不知道文件路径时使用dpkg -L 查看
root@ubuntu20:/var/log# cat /etc/default/keepalived
# Options to pass to keepalived
# DAEMON_ARGS are appended to the keepalived command-line
DAEMON_ARGS="-D -S 6"
centons 的位置,当不知道路径时使用rpm -ql
cat /etc/sysconfig/keepalived
vi /etc/rsyslog.d/6-keepalived.conf
local6.* /var/log/keepalived.log
重启服务
systemctl restart keepalived.service rsyslog.service
查看日志
tail /var/log/keepalived.log
keepalived使用子配置文件,在keepalived文件中添加
include conf.d/*.conf
mkdir conf.d
tcpdump -i eth0 -nn host 224.0.0.18
抢占延迟模式 preempt_delay,抢占延迟模式,即优先级高的主机恢复后,不会立即抢回VIP,而是延迟一段时间(默认300s)再抢回VIP
cat www.luo.conf
vrrp_instance VI_1 {
state BACKUP
interface eth0
virtual_router_id 222
priority 100
advert_int 1
preempt_delay 30
virtual_ipaddress {
192.168.1.188
}
}
使用单播
二个机器用单播
#在所有节点vrrp_instance语句块中设置对方主机的IP,建议设置为专用于对应心跳线网络的地址,而非使用业务网络
unicast_src_ip #指定发送单播的源IP
unicast_peer {
#指定接收单播的对方目标主机IP
…
}
master
root@ubuntu20:/etc/keepalived# cat keepalived.conf
! Configuration File for keepalived
global_defs {
notification_email {
2368756722@qq.com
}
notification_email_from Alexandre.Cassen@firewall.loc
smtp_server 192.168.200.1
smtp_connect_timeout 30
router_id kv1
vrrp_skip_check_adv_addr
vrrp_garp_interval 0
}
include /etc/keepalived/conf.d/*.conf
root@ubuntu20:/etc/keepalived# cat conf.d/www.luo.conf
vrrp_instance VI_1 {
state MASTER
interface eth0
virtual_router_id 222
priority 100
advert_int 1
#preempt_delay 30
authentication {
auth_type PASS
auth_pass 123456
}
virtual_ipaddress {
192.168.1.188 dev eth0 label eth0:1
}
unicast_src_ip 192.168.1.103
unicast_peer {
192.168.1.109
}
}
slave
root@ubuntu20:/etc/keepalived# cat keepalived.conf
! Configuration File for keepalived
global_defs {
notification_email {
2368756722@qq.com
}
notification_email_from Alexandre.Cassen@firewall.loc
smtp_server 192.168.200.1
smtp_connect_timeout 30
router_id kv2
vrrp_skip_check_adv_addr
vrrp_garp_interval 0
}
include /etc/keepalived/conf.d/*.conf
root@ubuntu20:/etc/keepalived# cat conf.d/ww.luo.conf
vrrp_instance VI_1 {
state BACKUP
interface eth0
virtual_router_id 222
priority 80
advert_int 1
#preempt_delay 30
authentication {
auth_type PASS
auth_pass 123456
}
virtual_ipaddress {
192.168.1.188 dev eth0 label eth0:1
}
unicast_src_ip 192.168.1.109
unicast_peer {
192.168.1.103
}
}
查看
tcpdump -i eth0 -nn src host 192.168.1.103 and dst host 192.168.1.109
配置邮件
报错1:
Reading state information… Done
E: Unable to locate package libiosocket-ssl-perl
notify.sh: line 54: sendemail: command not found
邮件发送失败! [FAILED]
没有安装sendemail
apt install sendemail
报错2
root@ubuntu20:/etc/keepalived/conf.d# bash 100.sh master
Jul 15 17:18:50 ubuntu20 sendemail[166166]: ERROR => No TLS support! SendEmail can't load required libraries. (try installing Net::SSLeay and IO::Socket::SSL)
邮件发送失败! [FAILED]
安装软件
apt-get install libnet-ssleay-perl libio-socket-ssl-perl
添加httpd检测
HTTP_GET {
url {
path /
status_code 200
}
connect_timeout 3
nb_ger_retry 3
delay_before_retry 3
connect_port 80
}
}
while true ;do curl 192.168.1.188 ; sleep 1; done
实现双主的 LVS-DR 模式
后端web配置文件一样:192.168.1.80 192.168.1.120 ,通过nginx虚拟主机添加二个网站,并添加vip2到lo:2
root@ubuntu20:/etc/nginx/conf.d# cat www.luo.edu.conf
server {
listen 80;
server_name www.luo.edu;
root /data/site2;
}
root@ubuntu20:/etc/nginx/conf.d# cat www.luo.org.conf
server {
listen 80;
server_name www.luo.org;
root /data/site1;
}
创建默认默认页面,80为例子
mkdir /data/site{1,2}
root@server:/data# cat /data/site2/index.html
www.luo.edu 192.168.1.80
root@server:/data# cat /data/site1/index.html
www.luo.org 192.168.1.80
web服务器添加vip2
ip a a 192.168.1.200/32 dev lo label lo:2
keepalived配置
拷贝原来的文件,修改vrrp_instance、 virtual_route_id 、priority 、virtual_ipaddress 的ip和接口、virtual_server IP
cp www.luo.org.conf www.luo.edu.conf
virtual_route_id 每个虚拟路由器唯一标识,范围:0-255
root@ubuntu20:/etc/keepalived/conf.d# cat www.luo.edu.conf
vrrp_instance VI_2 {
state BACKUP
interface eth0
virtual_router_id 88
priority 80
advert_int 1
#preempt_delay 30
authentication {
auth_type PASS
auth_pass 123456
}
virtual_ipaddress {
192.168.1.200 dev eth0 label eth0:2
}
notify_master "/etc/keepalived/conf.d/notify.sh master"
notify_backup "/etc/keepalived/conf.d/notify.sh backup"
notify_fault "/etc/keepalived/conf.d/notify.sh fault"
}
virtual_server 192.168.1.200 80 {
delay_loop 6
lb_algo rr
lb_kind DR
protocol TCP
real_server 192.168.1.80 80{
weight 1
HTTP_GET {
url {
path /
status_code 200
}
connect_timeout 3
nb_ger_retry 3
delay_before_retry 3
connect_port 80
}
}
real_server 192.168.1.120 80{
weight 1
TCP_CHECK {
connect_timeout 3
nb_ger_retry 3
delay_before_retry 3
connect_port 80
}
}
测试
curl -Hwww.luo.edu 192.168.1.188
实现单主的 LVS-DR 模式,利用FWM绑定成多个服 务为一个集群服务
apache快速添加证书
ubuntu安装OpenSSL开发包和Apache的SSL模块,使用以下命令
sudo apt install libssl-dev
sudo a2enmod ssl
mod_ssl已经成功安装在你的Ubuntu系统上,你可以在Apache配置文件中启用和配置SSL。可以使用以下命令打开Apache的默认SSL配置文件:vim /etc/apache2/sites-available/default-ssl.conf
最后,启用默认的SSL站点,使用以下命令:
systemctl reload apache2
sudo a2ensite default-ssl.conf
重启 sudo service apache2 restart
curl -k 忽略检测证书
root@server:/etc/apache2/sites-available# curl -k https://192.168.1.80
apache
centos安装: yum install mod_ssl
vim /etc/httpd/conf.d/ssl.conf
netstat -antp|grep 443
iptables -t mangle -A PREROUTING -d 192.168.1.188 -p tcp -m multiport --dports 80,443 -j MARK --set-mark 6
查看规则
root@ubuntu20:/etc/keepalived/conf.d# iptables -t mangle -nvL
Chain PREROUTING (policy ACCEPT 1218 packets, 101K bytes)
pkts bytes target prot opt in out source destination
42 2779 MARK tcp -- * * 0.0.0.0/0 192.168.1.188 multiport dports 80,443 MARK set 0x6
备份并修改配置文件
cp www.luo.org.conf{,.bak} 修改为virtual_server fwmark 6 {
virtual_server fwmark 6 {
delay_loop 6
lb_algo rr
lb_kind DR
protocol TCP
sorry_server 127.0.0.1 80 #当后端服务都挂掉后,自己提供服务,需要有服务
real_server 192.168.1.80 80{
weight 1
HTTP_GET {
url {
path /
status_code 200
}
connect_timeout 3
nb_ger_retry 3
delay_before_retry 3
connect_port 80
}
}
real_server 192.168.1.120 80{ #需要添加端口,用与健康检测
weight 1
TCP_CHECK {
connect_timeout 3
nb_ger_retry 3
delay_before_retry 3
connect_port 80
}
}
}
root@ubuntu20:~# curl -k https://192.168.1.188
apache 192.168.1.80
root@ubuntu20:~# curl -k https://192.168.1.188
120
通过脚本,运行失败,降低优先级
happroxy的高可用
keepalived01 192.168.1.103
keepalived02 192.168.1.109
haproxy01 192.168.1.103
haproxy02 192.168.1.109
后端网站 192.168.1.80 192.168.1.120
安装配置haproxy配置文件一样
haproxy 均衡后端web
apt install haproxy -y
vi cat haproxy.cfg
listen stats
stats enable
bind 0.0.0.0:9992
stats uri /haproxy_status
listen www.luo.org
bind 192.168.1.188:80
server 192.168.1.80 192.168.1.80:80 check
server 192.168.1.120 192.168.1.120:80 check
添加内核参数,
vim /etc/sysctl.conf
net.ipv4.ip_nonlocal_bind = 1 #当没有vip是,haproxy也能运行,解决keepalived从节点
拷贝到109节点,重启
scp haproxy.cfg 192.168.1.109:/etc/haproxy/
root@ubuntu20:/etc/haproxy# systemctl restart haproxy
健康检测
添加检测脚本,检测到haproxy不存在就减优先级,实现keepalived的vip切换。以实现高可用
root@ubuntu20:~# cat /etc/keepalived/conf.d/check_haproxy.sh
#!/bin/bash
killall -0 haproxy
chmod +x check_haproxy.sh
103 keepalived配置
cat ../keepalived.conf
! Configuration File for keepalived
global_defs {
notification_email {
2368756722@qq.com
}
notification_email_from Alexandre.Cassen@firewall.loc
smtp_server 192.168.200.1
smtp_connect_timeout 30
router_id kv1
vrrp_skip_check_adv_addr
vrrp_garp_interval 0
vrrp_mcast_group4 224.0.0.18
}
include /etc/keepalived/conf.d/*.conf
cat /etc/keepalived/conf.d/www.luo.org.conf
vrrp_script check_haproxy { #定义haproxy检测脚本
script "/etc/keepalived/conf.d/check_haproxy.sh"
interval 1
weight -30
fall 3
rise 2
timeout 2
}
vrrp_instance VI_1 {
state BACKUP
interface eth0
virtual_router_id 222
priority 80
advert_int 1
#preempt_delay 30
authentication {
auth_type PASS
auth_pass 123456
}
virtual_ipaddress {
192.168.1.188 dev eth0 label eth0:1
}
# unicast_src_ip 192.168.1.109
# unicast_peer {
# 192.168.1.103
# }
notify_master "/etc/keepalived/conf.d/notify.sh master"
notify_backup "/etc/keepalived/conf.d/notify.sh backup"
notify_fault "/etc/keepalived/conf.d/notify.sh fault"
track_script {
check_haproxy #调用前面定义的脚本
}
}
109 keepalived配置
cat /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
notification_email {
2368756722@qq.com
}
notification_email_from Alexandre.Cassen@firewall.loc
smtp_server 192.168.200.1
smtp_connect_timeout 30
router_id kv2
vrrp_skip_check_adv_addr
vrrp_garp_interval 0
vrrp_mcast_group4 224.0.0.18
}
include /etc/keepalived/conf.d/*.conf
root@ubuntu20:/etc/haproxy# cat /etc/keepalived/conf.d/www.luo.org.conf
vrrp_script check_haproxy { #定义脚本
script "/etc/keepalived/conf.d/check_haproxy.sh"
interval 1
weight -30
fall 3
rise 2
timeout 2
}
vrrp_instance VI_1 {
state BACKUP
interface eth0
virtual_router_id 222
priority 80
advert_int 1
#preempt_delay 30
authentication {
auth_type PASS
auth_pass 123456
}
virtual_ipaddress {
192.168.1.188 dev eth0 label eth0:1
}
# unicast_src_ip 192.168.1.109
# unicast_peer {
# 192.168.1.103
# }
notify_master "/etc/keepalived/conf.d/notify.sh master"
notify_backup "/etc/keepalived/conf.d/notify.sh backup"
notify_fault "/etc/keepalived/conf.d/notify.sh fault"
track_script {
check_haproxy #调用前面定义的脚本
}
}
haproxy添加配置
listen web_http
bind 192.168.1.188:80
server web1 192.168.1.80:80 check
server web2 192.168.1.120:80 check
目前节点haproxy和keepalived运行正常,把103haproxy停掉,keepalived检测到脚本返回非0,自动降低优先级,切换vip到109上
tcpdump -i eth0 -nn host 224.0.0.18。可以看到103keepalived的优先级降到70,比192.168.1.109的优先级低,就进行切换
systemctl stop haproxy.service
kilall不严谨,通过定义curl 页面 返回状态页是否正常
非抢占模式
各 Keepalived 服务器 state 配置为 BACKUP
nopreempt
生产中一个节点设置为抢占,一个设置为不抢占,主节点设置为非抢占式,备用节点设置为抢占式
keepalived:
VRRP 解决VIP高可用
LVS 主从 主主
script haproxy nginx高可用