k1.Keepalived 架构和安装

vrrp 协议的软件实现，原生设计目的为了高可用 ipvs服务

keepalived 是高可用集群的通用无状态应用解决方案

官网：http://keepalived.org/

功能：

• 基于vrrp协议完成地址流动
• 为vip地址所在的节点生成ipvs规则(在配置文件中预先定义)
• 为ipvs集群的各RS做健康状态检测
• 基于脚本调用接口完成脚本中定义的功能，进而影响集群事务，以此支持nginx、haproxy等服务

Keepalived 相关文件

• 软件包名：keepalived
• 主程序文件：/usr/sbin/keepalived
• 主配置文件：/etc/keepalived/keepalived.conf
• 配置文件示例：/usr/share/doc/keepalived/
• Unit File：/lib/systemd/system/keepalived.service
• Unit File的环境配置文件：
  • /etc/sysconfig/keepalived CentOS
  • /etc/default/keepalived Ubuntu

1.Keepalived 安装

1.1 包安装

[root@Ubuntu2204 ~]#apt update;apt -y install keepalived

# 默认缺少配置，服务无法启动，提示/etc/keepalived/keepalived.conf 不存在
[root@Ubuntu2204 ~]#systemctl status keepalived.service 
○ keepalived.service - Keepalive Daemon (LVS and VRRP)
     Loaded: loaded (/lib/systemd/system/keepalived.service; enabled; vendor preset: enabled)
     Active: inactive (dead)
  Condition: start condition failed at Mon 2024-07-22 20:56:40 CST; 1min 12s ago
             └─ ConditionFileNotEmpty=/etc/keepalived/keepalived.conf was not met

Jul 22 20:56:40 Ubuntu2204 systemd[1]: Condition check resulted in Keepalive Daemon (LVS and VRRP) being skipped.

# 利用范例生成配置文件
[root@Ubuntu2204 ~]#cp /usr/share/doc/keepalived/samples/keepalived.conf.sample /etc/keepalived/keepalived.conf
[root@Ubuntu2204 ~]#systemctl start keepalived.service 
[root@Ubuntu2204 ~]#systemctl status keepalived.service 
● keepalived.service - Keepalive Daemon (LVS and VRRP)
     Loaded: loaded (/lib/systemd/system/keepalived.service; enabled; vendor preset: enabled)
     Active: active (running) since Mon 2024-07-22 20:59:54 CST; 3s ago
   Main PID: 4013 (keepalived)
      Tasks: 3 (limit: 2176)
     Memory: 4.4M
        CPU: 81ms
     CGroup: /system.slice/keepalived.service
             ├─4013 /usr/sbin/keepalived --dont-fork
             ├─4014 /usr/sbin/keepalived --dont-fork
             └─4015 /usr/sbin/keepalived --dont-fork
             
[root@Ubuntu2204 ~]#pstree -p | grep keepalived
           |-keepalived(4013)-+-keepalived(4014)
           |                  `-keepalived(4015)

1.2 编译安装

# Ubuntu20.04安装相关包
[root@Ubuntu2204 ~]#apt update;apt -y install make gcc ipvsadm build-essential pkg-config automake autoconf libipset-dev libnl-3-dev libnl-genl-3-dev libssl-dev libxtables-dev libip4tc-dev libip6tc-dev libmagic-dev libsnmp-dev libglib2.0-dev libpcre2-dev libnftnl-dev libmnl-dev libsystemd-dev

[root@Ubuntu2204 ~]#wget https://keepalived.org/software/keepalived-2.2.7.tar.gz
[root@Ubuntu2204 ~]#tar xf keepalived-2.3.1.tar.gz -C /usr/local/src/
[root@Ubuntu2204 ~]#cd /usr/local/src/keepalived-2.2.7/

# 选项--disable-fwmark 可用于禁用iptables规则,可访止VIP无法访问,无此选项默认会启用iptables规则
[root@Ubuntu2204 keepalived-2.2.7]#./configure --prefix=/usr/local/keepalived --disable-fwmark
[root@Ubuntu2204 keepalived-2.2.7]#make -j 2 && make install
[root@Ubuntu2204 keepalived-2.2.7]#mkdir /etc/keepalived
[root@Ubuntu2204 keepalived-2.2.7]#cp /usr/local/keepalived/etc/keepalived/keepalived.conf.sample /etc/keepalived/keepalived.conf
[root@Ubuntu2204 keepalived-2.2.7]#cp ./keepalived/keepalived.service /lib/systemd/system/
[root@Ubuntu2204 keepalived-2.2.7]#systemctl daemon-reload 
[root@Ubuntu2204 keepalived-2.2.7]#systemctl enable --now keepalived.service

2. KeepAlived 配置说明

配置文件：/etc/keepalived/keepalived.conf

配置文件组成

• GLOBAL CONFIGURATION Global definitions：
  • 定义邮件配置，route_id，vrrp配置，多播地址等
• VRRP CONFIGURATION VRRP instance(s)：
  • 定义每个vrrp虚拟路由器
• LVS CONFIGURATION
  • Virtual server group(s)
  • Virtual server(s)：LVS集群的VS和RS

2.1 全局配置

#/etc/keepalived/keepalived.conf 
global_defs {
 notification_email {
 root@localhost #keepalived 发生故障切换时邮件发送的目标邮箱，可以按行区分写多个
 root@wangxiaochun.com 
  29308620@qq.com 
 }
 notification_email_from keepalived@localhost  #发邮件的地址
 smtp_server 127.0.0.1     #邮件服务器地址
 smtp_connect_timeout 30   #邮件服务器连接timeout
 router_id ka1.example.com #每个keepalived主机唯一标识，建议使用当前主机名，如果多节点重名可能会影响切换脚本执行
vrrp_skip_check_adv_addr  #对所有通告报文都检查，会比较消耗性能，启用此配置后，如果收到的通告报文和上一个报文是同一个路由器，则跳过检查，默认值为全检查
 vrrp_strict #严格遵守VRRP协议,启用此项后以下状况将无法启动服务:1.无VIP地址 2.配置了单播邻居 3.在VRRP版本2中有IPv6地址，开启动此项并且没有配置vrrp_iptables时会自动开启iptables防火墙规则，默认导致VIP无法访问,建议不加此项配置
 vrrp_garp_interval 0 #gratuitous ARP messages 报文发送延迟，0表示不延迟
 vrrp_gna_interval 0  #unsolicited NA messages （不请自来）消息发送延迟
 vrrp_mcast_group4 224.0.0.18 #指定组播IP地址范围：224.0.0.0到239.255.255.255,默认值：224.0.0.18 
 vrrp_iptables        #此项和vrrp_strict同时开启时，则不会添加防火墙规则,如果无配置vrrp_strict项,则无需启用此项配置
}

2.2 配置虚拟路由器

vrrp_instance <STRING> { #<String>为vrrp的实例名,一般为业务名称
 配置参数
 ......
 }
#配置参数：
state MASTER|BACKUP#当前节点在此虚拟路由器上的初始状态，状态为MASTER或者BACKUP
interface IFACE_NAME #绑定为当前虚拟路由器使用的物理接口，如：eth0,bond0,br0,可以和VIP不在一个网卡
virtual_router_id VRID #每个虚拟路由器唯一标识，范围：0-255，每个虚拟路由器此值必须唯一，否则服务无法启动，同属一个虚拟路由器的多个keepalived节点必须相同,务必要确认在同一网络中此值必须唯一
priority 100 #当前物理节点在此虚拟路由器的优先级，范围：1-254，每个keepalived主机节点此值不同
advert_int 1 #vrrp通告的时间间隔，默认1s
authentication { #认证机制
 auth_type AH|PASS   #AH为IPSEC认证(不推荐),PASS为简单密码(建议使用)
 auth_pass <PASSWORD> #预共享密钥，仅前8位有效，同一个虚拟路由器的多个keepalived节点必须一样
}
virtual_ipaddress { #虚拟IP,生产环境可能指定上百个IP地址
 <IPADDR>/<MASK> brd <IPADDR> dev <STRING> scope <SCOPE> label <LABEL>
 192.168.200.100 #指定VIP，不指定网卡，默认为eth0,注意：不指定/prefix,默认
为/32
 192.168.200.101/24 dev eth1   #指定VIP的网卡，建议和interface指令指定的网卡不在一个
网卡
 192.168.200.102/24 dev eth2 label eth2:1 #指定VIP的网卡label 
}
track_interface { #配置监控网络接口，一旦出现故障，则转为FAULT状态实现地址转移
 eth0
 eth1
 …
} 

2.3启用 Keepalived 日志功能

# 包安装
[root@Ubuntu2204 ~]#vim /etc/default/keepalived

DAEMON_ARGS="-D -S 6 "

# 编译安装
[root@Ubuntu2204 ~]#vim /apps/keepalived/etc/sysconfig/keepalived

KEEPALIVED_OPTIONS="-D -S 6"

[root@Ubuntu2204 ~]#vim /etc/rsyslog.conf

local6.*    /var/log/keepalived.log

[root@Ubuntu2204 ~]#systemctl restart keepalived.service rsyslog.service
[root@Ubuntu2204 ~]#tail -f /var/log/keepalived.log
Jul 23 10:31:12 ym Keepalived_vrrp[2585]: Sending gratuitous ARP on eth0 for 192.168.200.13
Jul 23 10:31:12 ym Keepalived_vrrp[2585]: Sending gratuitous ARP on eth0 for 192.168.200.11
Jul 23 10:31:12 ym Keepalived_vrrp[2585]: Sending gratuitous ARP on eth0 for 192.168.200.12
Jul 23 10:31:12 ym Keepalived_vrrp[2585]: Sending gratuitous ARP on eth0 for 192.168.200.13
Jul 23 10:31:12 ym Keepalived_vrrp[2585]: Sending gratuitous ARP on eth0 for 192.168.200.11
Jul 23 10:31:12 ym Keepalived_vrrp[2585]: Sending gratuitous ARP on eth0 for 192.168.200.12
Jul 23 10:31:12 ym Keepalived_vrrp[2585]: Sending gratuitous ARP on eth0 for 192.168.200.13

2.4 实现 Keepalived 独立子配置文件

当生产环境复杂时， /etc/keepalived/keepalived.conf 文件中内容过多，不易管理，可以将不同集 群的配置，比如：不同集群的VIP配置放在独立的子配置文件中

利用include 指令可以实现包含子配置文件

[root@Ubuntu2204 ~]#mkdir /etc/keepalived/conf.d/
[root@Ubuntu2204 ~]#cat /etc/keepalived/keepalived.conf
! Configuration File for keepalived

global_defs {
   router_id ka1.anan.com
   vrrp_mcast_group4 224.1.1.1
}

include /etc/keepalived/conf.d/*.conf

[root@Ubuntu2204 ~]#vim /etc/keepalived/conf.d/cluster1.conf

3. Keepalived 实现 VRRP

3.1 实现master/slave的 Keepalived 单主架构

# MASTER配置
[root@Ubuntu2204 ~]#vim /etc/keepalived/keepalived.conf 
! Configuration File for keepalived

global_defs {
   router_id ka1.anan.com
   vrrp_mcast_group4 224.1.1.1
}

include /etc/keepalived/conf.d/*.conf


[root@Ubuntu2204 ~]#vim /etc/keepalived/conf.d/cluster1.conf
vrrp_instance VI_1 {
    state MASTER
    interface eth0
    virtual_router_id 66
    priority 100
    advert_int 1

    authentication {
        auth_type PASS
        auth_pass 123456
    }

    virtual_ipaddress {
        10.0.0.200/24 dev eth0 label eth0:0
    }
}
[root@Ubuntu2204 ~]#systemctl restart keepalived.service

# BACKUP配置
[root@Ubuntu2204 ~]#vim /etc/keepalived/keepalived.conf 
! Configuration File for keepalived

global_defs {
   router_id ka2.anan.com
   vrrp_mcast_group4 224.1.1.1
}

include /etc/keepalived/conf.d/*.conf

[root@Ubuntu2204 ~]#vim /etc/keepalived/conf.d/cluster1.conf
vrrp_instance VI_1 {
    state BACKUP		# 修改
    interface eth0
    virtual_router_id 66
    priority 80			# 修改
    advert_int 1

    authentication {
        auth_type PASS
        auth_pass 123456
    }

    virtual_ipaddress {
        10.0.0.200/24 dev eth0 label eth0:0
    }
}
[root@Ubuntu2204 ~]#systemctl restart keepalived.service

# 抓包观察
[root@Ubuntu2204 ~]#tcpdump -i eth0 -nn host 224.1.1.1
tcpdump: verbose output suppressed, use -v[v]... for full protocol decode
listening on eth0, link-type EN10MB (Ethernet), snapshot length 262144 bytes
11:38:00.626377 IP 10.0.0.100 > 224.1.1.1: VRRPv2, Advertisement, vrid 66, prio 100, authtype simple, intvl 1s, length 20
11:38:01.627249 IP 10.0.0.100 > 224.1.1.1: VRRPv2, Advertisement, vrid 66, prio 100, authtype simple, intvl 1s, length 20

4. 抢占模式和非抢占模式

默认为抢占模式 preempt，即当高优先级的主机恢复在线后，会抢占低先级的主机的master角色，造成 网络抖动，建议设置为非抢占模式 nopreempt ，即高优先级主机恢复后，并不会抢占低优先级主机的 master 角色

注意: 非抢占模式下,如果原主机down机, VIP迁移至的新主机, 后续新主机也发生down时,仍会将VIP迁移 回原主机

非抢占模式 nopreempt

注意：要关闭 VIP抢占，必须将各 Keepalived 服务器 state 配置为 BACKUP

# ka1
[root@Ubuntu2204 ~]#cat /etc/keepalived/conf.d/cluster1.conf
vrrp_instance VI_1 {
    state BACKUP
    interface eth0
    virtual_router_id 66
    priority 100
    advert_int 1
    nopreempt			# 修改为非抢占式

# ka2
[root@Ubuntu2204 ~]#cat /etc/keepalived/conf.d/cluster1.conf 
vrrp_instance VI_1 {
    state BACKUP
    interface eth0
    virtual_router_id 66
    priority 80
    advert_int 1

抢占延迟模式 preempt_delay

抢占延迟模式，即优先级高的主机恢复后，不会立即抢回VIP，而是延迟一段时间（默认300s）再抢回 VIP

需要各keepalived服务器state为BACKUP,并且不要启用 vrrp_strict

# ka1
[root@Ubuntu2204 ~]#cat /etc/keepalived/conf.d/cluster1.conf
vrrp_instance VI_1 {
    state BACKUP
    interface eth0
    virtual_router_id 66
    priority 100
    advert_int 1
    preempt_delay 60    #抢占延迟模式，默认延迟300s

# ka2
[root@Ubuntu2204 ~]#cat /etc/keepalived/conf.d/cluster1.conf 
vrrp_instance VI_1 {
    state BACKUP
    interface eth0
    virtual_router_id 66
    priority 80
    advert_int 1

5. VIP 单播配置

默认keepalived主机之间利用多播相互通告消息，会造成网络拥塞，可以替换成单播，减少网络流量

注意：启用 vrrp_strict 时，不能启用单播

#在所有节点vrrp_instance语句块中设置对方主机的IP，建议设置为专用于对应心跳线网络的地址，而非使用业务网络
unicast_src_ip <IPADDR>  #指定发送单播的源IP
unicast_peer {
   <IPADDR>     #指定接收单播的对方目标主机IP
   ......
}

# 每台主机上添加一块仅主机网卡  绑定为当前虚拟路由器使用的物理接口
eth1: ka1 :192.168.10.100
eth1: ka2 :192.168.10.110
# ka1
[root@Ubuntu2204 ~]#vim /etc/keepalived/keepalived.conf 
! Configuration File for keepalived

global_defs {
   router_id ka1.anan.com
   #vrrp_mcast_group4 224.1.1.1
}

include /etc/keepalived/conf.d/*.conf

[root@Ubuntu2204 ~]#cat /etc/keepalived/conf.d/cluster1.conf 
vrrp_instance VI_1 {
    state MASTER
    interface eth1		# 修改为eth1
    virtual_router_id 66
    priority 100
    advert_int 1
    preempt_delay 60    
    
    authentication {
        auth_type PASS
        auth_pass 123456
    }

    virtual_ipaddress {
        10.0.0.200/24 dev eth0 label eth0:0
    }
    
    unicast_src_ip 192.168.10.100
    unicast_peer{
        192.168.10.110
    }
}

# ka2
[root@Ubuntu2204 ~]#vim /etc/keepalived/keepalived.conf 
! Configuration File for keepalived

global_defs {
   router_id ka2.anan.com
   #vrrp_mcast_group4 224.1.1.1
}

include /etc/keepalived/conf.d/*.conf

[root@Ubuntu2204 ~]#cat /etc/keepalived/conf.d/cluster1.conf 
vrrp_instance VI_1 {
    state BACKUP
    interface eth1		# 修改为eth1
    virtual_router_id 66
    priority 80
    advert_int 1

    authentication {
        auth_type PASS
        auth_pass 123456
    }

    virtual_ipaddress {
        10.0.0.200/24 dev eth0 label eth0:0
    }
    
    unicast_src_ip 192.168.10.110
    unicast_peer{
        192.168.10.100
    }
}

# 抓包观察
[root@Ubuntu2204 ~]#tcpdump -i eth1 -nn src host 192.168.10.100 and dst host 192.168.10.110
tcpdump: verbose output suppressed, use -v[v]... for full protocol decode
listening on eth1, link-type EN10MB (Ethernet), snapshot length 262144 bytes
12:58:00.850213 IP 192.168.10.100 > 192.168.10.110: VRRPv2, Advertisement, vrid 66, prio 100, authtype simple, intvl 1s, length 20
12:58:01.851947 IP 192.168.10.100 > 192.168.10.110: VRRPv2, Advertisement, vrid 66, prio 100, authtype simple, intvl 1s, length 20
12:58:02.853426 IP 192.168.10.100 > 192.168.10.110: VRRPv2, Advertisement, vrid 66, prio 100, authtype simple, intvl 1s, length 20