#
keepalived–高可用集群
在生产环境中,有些服务是不能中断的,这时候我们就需要用到集群环境,而调度器lvs和nginx的反向代理也需要高可用性,所以这里我们需要用到keepalived
###**keepalived **
使用的是虚拟路由冗余协议 ,实现虚拟的路由冗余。
过多的理论不多讲,直接上示例。
###示例
####一.使用keepalived进行lvs主备的调度
- 首选我们从最底层的RS开始配置 (开始前关闭iptables 及 selinux)
[root@localhost ~]# cat lvs_dr_rs.sh
#!/bin/bash
vip=192.168.3.100 --> 标明DR的VIP地址
mask='255.255.255.255'
dev=lo:1
case $1 in
start)
echo 1 > /proc/sys/net/ipv4/conf/all/arp_ignore
echo 1 > /proc/sys/net/ipv4/conf/lo/arp_ignore
echo 2 > /proc/sys/net/ipv4/conf/all/arp_announce
echo 2 > /proc/sys/net/ipv4/conf/lo/arp_announce
ifconfig $dev $vip netmask $mask broadcast $vip up
echo "The RS Server is Ready!"
;;
stop)
ifconfig $dev down
echo 0 > /proc/sys/net/ipv4/conf/all/arp_ignore
echo 0 > /proc/sys/net/ipv4/conf/lo/arp_ignore
echo 0 > /proc/sys/net/ipv4/conf/all/arp_announce
echo 0 > /proc/sys/net/ipv4/conf/lo/arp_announce
echo "The RS Server is Canceled!"
;;
*)
echo "Usage: $(basename $0) start|stop"
exit 1
;;
esac
2.keepalived 主配置 192.168.3.11
[root@localhost ~]# yum install keepalived ---> 直接yum 安装 keepalived 包
[root@localhost ~]#vim /etc/keepalived/keepalived.conf
1 ! Configuration File for keepalived
2
3 global_defs {
4 notification_email {
5 root@localhost ---> 定义收件邮件
6 }
7 notification_email_from root@localhost --->定义发件邮箱
8 smtp_server 127.0.0.1 --->邮箱IP
9 smtp_connect_timeout 30 --->邮箱超时是间
10 router_id LVS1 --->配置全局的路由名称
11 vrrp_mcast_group 224.100.100.100 --->多播地址 如果不设备默认地址是224.0.0.18
12 }
13
14 vrrp_instance VI_1 { --->这里给vrrp起个名字
15 state MASTER --->这里设置vrrp的主备关系
16 interface eth0 --->对应在哪个接口上
17 virtual_router_id 88 --->vrrp的id号
18 priority 100 --->些设备的vrrp优先级
19 advert_int 1 --->vrrp的通告时间间隔
20 authentication { --->vrrp的验证
21 auth_type PASS --->密码验证类型
22 auth_pass 12121212 --->设置的密码
23 }
24 virtual_ipaddress { --->设置lvs的VIP
25 192.168.3.100/24 dev eth0 label eth0:0 --->我们直接把VIP设置在eth0上
26 }
27 }
28
29 virtual_server 192.168.3.100 80 { --->绑定的vrrpIP端口信息应用到lvs上
30 delay_loop 3 --->检查后端服务器时间间隔3秒
31 lb_algo rr --->设置调度算法
32 lb_kind DR --->设置集群类型
33 protocol TCP
34
35 real_server 192.168.3.1 80 { --->这里我们设置RS的地址,有几台RS添几台
36 weight 1 --->添加权重
37 HTTP_GET { --->使用http进行服务器的状态验证
38 url { --->定义默认的usr访问
39 path /
40 status_code 200 --->验证返回的状态值是否为200 如果不是就T掉些RS
41 }
42 connect_timeout 3 --->设置连接超时时间 3秒
43 nb_get_retry 3
44 delay_before_retry 3
45 }
46 }
47
48 real_server 192.168.3.2 80 {
49 weight 1
50 HTTP_GET {
51 url {
52 path /
53 status_code 200
54 }
55 connect_timeout 3
56 nb_get_retry 3
57 delay_before_retry 3
58 }
59 }
60 }
3.keepalived 备配置 192.168.3.12
1 ! Configuration File for keepalived
2
3 global_defs {
4 notification_email {
5 root@localhost
6 }
7 notification_email_from root@localhost
8 smtp_server 127.0.0.1
9 smtp_connect_timeout 30
10 router_id LVS1
11 vrrp_mcast_group 224.100.100.100
12 }
13
14 vrrp_instance VI_1 {
15 state BACKUP --->这里我们配为BACKUP
16 interface eth0
17 virtual_router_id 88
18 priority 80 --->备的优选级调低
19 advert_int 1
20 authentication {
21 auth_type PASS
22 auth_pass 12121212
23 }
24 virtual_ipaddress {
25 192.168.3.100/24 dev eth0 label eth0:0
26 }
27 }
28
29 virtual_server 192.168.3.100 80 {
30 delay_loop 3
31 lb_algo rr
32 lb_kind DR
33 protocol TCP
34
35 real_server 192.168.3.1 80 {
36 weight 1
37 HTTP_GET {
38 url {
39 path /
40 status_code 200
41 }
42 connect_timeout 3
43 nb_get_retry 3
44 delay_before_retry 3
45 }
46 }
47
48 real_server 192.168.3.2 80 {
49 weight 1
50 HTTP_GET {
51 url {
52 path /
53 status_code 200
54 }
55 connect_timeout 3
56 nb_get_retry 3
57 delay_before_retry 3
58 }
59 }
60 }
4.测试:
[root@station1 ~]# curl 192.168.3.100
this is 3.2 page
[root@station1 ~]# curl 192.168.3.100
this is 3.1 page
[root@station1 ~]# curl 192.168.3.100
this is 3.2 page
[root@station1 ~]# curl 192.168.3.100
this is 3.1 page
[root@station1 ~]# curl 192.168.3.100
this is 3.2 page
[root@station1 ~]# curl 192.168.3.100
this is 3.1 page
[root@localhost ~]# tcpdump -i eth0 -nn host 224.0.0.18
tcpdump: verbose output suppressed, use -v or -vv for full protocol decode
listening on eth0, link-type EN10MB (Ethernet), capture size 65535 bytes
17:03:03.588389 IP 192.168.3.11 > 224.0.0.18: VRRPv2, Advertisement, vrid 88, prio 100, authtype simple, intvl 1s, length 20
17:03:04.589769 IP 192.168.3.11 > 224.0.0.18: VRRPv2, Advertisement, vrid 88, prio 100, authtype simple, intvl 1s, length 20
17:03:05.265369 IP 192.168.3.11 > 224.0.0.18: VRRPv2, Advertisement, vrid 88, prio 0, authtype simple, intvl 1s, length 20 --->当我们把主的那台keepalived服务关闭后,主的优选级降为0
17:03:05.954910 IP 192.168.3.12 > 224.0.0.18: VRRPv2, Advertisement, vrid 88, prio 80, authtype simple, intvl 1s, length 20 --->这时候备的接收到主的优选级为0后开始接管
17:03:06.956057 IP 192.168.3.12 > 224.0.0.18: VRRPv2, Advertisement, vrid 88, prio 80, authtype simple, intvl 1s, length 20
17:03:07.957407 IP 192.168.3.12 > 224.0.0.18: VRRPv2, Advertisement, vrid 88, prio 80, authtype simple, intvl 1s, length 20
17:03:23.971653 IP 192.168.3.12 > 224.0.0.18: VRRPv2, Advertisement, vrid 88, prio 80, authtype simple, intvl 1s, length 20
17:03:24.973304 IP 192.168.3.12 > 224.0.0.18: VRRPv2, Advertisement, vrid 88, prio 80, authtype simple, intvl 1s, length 20
17:03:25.974527 IP 192.168.3.12 > 224.0.0.18: VRRPv2, Advertisement, vrid 88, prio 80, authtype simple, intvl 1s, length 20
17:03:25.974686 IP 192.168.3.11 > 224.0.0.18: VRRPv2, Advertisement, vrid 88, prio 100, authtype simple, intvl 1s, length 20 --->当我们开启了主的服务后,主又开始发通告信息了
17:03:25.975042 IP 192.168.3.12 > 224.0.0.18: VRRPv2, Advertisement, vrid 88, prio 80, authtype simple, intvl 1s, length 20 --->当主发通告信息后备的看到自己的优选级比不过主就不发通告了
17:03:25.975177 IP 192.168.3.11 > 224.0.0.18: VRRPv2, Advertisement, vrid 88, prio 100, authtype simple, intvl 1s, length 20 --->这里候主抢占成功
17:03:26.977253 IP 192.168.3.11 > 224.0.0.18: VRRPv2, Advertisement, vrid 88, prio 100, authtype simple, intvl 1s, length 20
17:03:27.977924 IP 192.168.3.11 > 224.0.0.18: VRRPv2, Advertisement, vrid 88, prio 100, authtype simple, intvl 1s, length 20
####二.使用keepalived进行lvs主主的调度
1.RS的配置 (由于电脑不能带起过多的虚拟机,所以只能做到之前100的RS里)
[root@localhost ~]# vim lvs_dr_rs.sh
mask='255.255.255.255'
#!/bin/bash
vip=192.168.3.100
vip2=192.168.3.200
mask='255.255.255.255'
dev=lo:1
dev2=lo:2
case $1 in
start)
echo 1 > /proc/sys/net/ipv4/conf/all/arp_ignore
echo 1 > /proc/sys/net/ipv4/conf/lo/arp_ignore
echo 2 > /proc/sys/net/ipv4/conf/all/arp_announce
echo 2 > /proc/sys/net/ipv4/conf/lo/arp_announce
ifconfig $dev $vip netmask $mask broadcast $vip up
ifconfig $dev2 $vip2 netmask $mask broadcast $vip2 up
echo "The RS Server is Ready!"
;;
stop)
ifconfig $dev down
ifconfig $dev2 down
echo 0 > /proc/sys/net/ipv4/conf/all/arp_ignore
echo 0 > /proc/sys/net/ipv4/conf/lo/arp_ignore
echo 0 > /proc/sys/net/ipv4/conf/all/arp_announce
echo 0 > /proc/sys/net/ipv4/conf/lo/arp_announce
echo "The RS Server is Canceled!"
;;
*)
echo "Usage: $(basename $0) start|stop"
exit 1
;;
esac
2.192.168.3.11 的配置
[root@localhost keepalived]# cat keepalived.conf
! Configuration File for keepalived
global_defs {
notification_email {
root@localhost
}
notification_email_from root@localhost
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id LVS1
vrrp_mcast_group 224.100.100.100
}
vrrp_instance VI_1 {
state MASTER
interface eth0
virtual_router_id 88
priority 100
advert_int 1
authentication {
auth_type PASS
auth_pass 12121212
}
virtual_ipaddress {
192.168.3.100/24 dev eth0 label eth0:0
}
}
virtual_server 192.168.3.100 80 {
delay_loop 3
lb_algo rr
lb_kind DR
protocol TCP
real_server 192.168.3.1 80 {
weight 1
HTTP_GET {
url {
path /
status_code 200
}
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
}
}
real_server 192.168.3.2 80 {
weight 1
HTTP_GET {
url {
path /
status_code 200
}
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
}
}
}
vrrp_instance VI_2 { -->重新定义一个VRRP的配置
state BACKUP -->这里我们配置的是备用
interface eth0
virtual_router_id 98 -->通告的ID不能和之前的相同
priority 80 -->备用的通告优先级不能高于主
advert_int 1
authentication {
auth_type PASS
auth_pass 21212121
}
virtual_ipaddress {
192.168.3.200/24 dev eth0 label eth0:1 -->这里的网卡绑在eth0:1上
}
}
virtual_server 192.168.3.200 80 { -->这里我们定义的VIP的地址
delay_loop 3
lb_algo rr
lb_kind DR
protocol TCP
real_server 192.168.3.1 80 {
weight 1
HTTP_GET {
url {
path /
status_code 200
}
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
}
}
real_server 192.168.3.2 80 {
weight 1
HTTP_GET {
url {
path /
status_code 200
}
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
}
}
}
3.192.168.3.12的配置
[root@CentOS6 keepalived]#cat keepalived.conf
! Configuration File for keepalived
global_defs {
notification_email {
root@localhost
}
notification_email_from root@localhost
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id LVS1
vrrp_mcast_group 224.100.100.100
}
vrrp_instance VI_1 {
state BACKUP
interface eth0
virtual_router_id 88
priority 80
advert_int 1
authentication {
auth_type PASS
auth_pass 12121212
}
virtual_ipaddress {
192.168.3.100/24 dev eth0 label eth0:0
}
}
virtual_server 192.168.3.100 80 {
delay_loop 3
lb_algo rr
lb_kind DR
protocol TCP
real_server 192.168.3.1 80 {
weight 1
HTTP_GET {
url {
path /
status_code 200
}
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
}
}
real_server 192.168.3.2 80 {
weight 1
HTTP_GET {
url {
path /
status_code 200
}
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
}
}
}
vrrp_instance VI_2 {
state MASTER
interface eth0
virtual_router_id 98
priority 100
advert_int 1
authentication {
auth_type PASS
auth_pass 21212121
}
virtual_ipaddress {
192.168.3.200/24 dev eth0 label eth0:1
}
}
virtual_server 192.168.3.200 80 {
delay_loop 3
lb_algo rr
lb_kind DR
protocol TCP
real_server 192.168.3.1 80 {
weight 1
HTTP_GET {
url {
path /
status_code 200
}
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
}
}
real_server 192.168.3.2 80 {
weight 1
HTTP_GET {
url {
path /
status_code 200
}
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
}
}
}
4.测试及查看抓包信息
[root@station1 mail]# curl 192.168.3.100
this is 3.1 page
[root@station1 mail]# curl 192.168.3.100
this is 3.2 page
[root@station1 mail]# curl 192.168.3.100
this is 3.1 page
[root@station1 mail]# curl 192.168.3.100
this is 3.2 page
[root@station1 mail]# curl 192.168.3.100
this is 3.1 page
[root@station1 mail]# curl 192.168.3.100
this is 3.2 page
[root@station1 mail]# curl 192.168.3.200
this is 3.2 page
[root@station1 mail]# curl 192.168.3.200
this is 3.1 page
[root@station1 mail]# curl 192.168.3.200
this is 3.2 page
[root@station1 mail]# curl 192.168.3.200
this is 3.1 page
[root@localhost ~]# tcpdump -i eth0 -nn host 224.0.0.18
tcpdump: verbose output suppressed, use -v or -vv for full protocol decode
listening on eth0, link-type EN10MB (Ethernet), capture size 65535 bytes
17:28:05.436459 IP 192.168.3.11 > 224.0.0.18: VRRPv2, Advertisement, vrid 88, prio 100, authtype simple, intvl 1s, length 20
17:28:05.501952 IP 192.168.3.12 > 224.0.0.18: VRRPv2, Advertisement, vrid 98, prio 100, authtype simple, intvl 1s, length 20
17:28:06.436944 IP 192.168.3.11 > 224.0.0.18: VRRPv2, Advertisement, vrid 88, prio 100, authtype simple, intvl 1s, length 20
17:28:06.502715 IP 192.168.3.12 > 224.0.0.18: VRRPv2, Advertisement, vrid 98, prio 100, authtype simple, intvl 1s, length 20
17:28:07.437864 IP 192.168.3.11 > 224.0.0.18: VRRPv2, Advertisement, vrid 88, prio 100, authtype simple, intvl 1s, length 20
17:28:07.503495 IP 192.168.3.12 > 224.0.0.18: VRRPv2, Advertisement, vrid 98, prio 100, authtype simple, intvl 1s, length 20
17:28:08.438457 IP 192.168.3.11 > 224.0.0.18: VRRPv2, Advertisement, vrid 88, prio 100, authtype simple, intvl 1s, length 20
17:28:08.505272 IP 192.168.3.12 > 224.0.0.18: VRRPv2, Advertisement, vrid 98, prio 100, authtype simple, intvl 1s, length 20 --> 这里可以看到11就VRID 88的主 12为VRID 98的主
####三.使用keepalived进行nginx主备的调度
1.主的192.168.3.11的配置
-----------------------------------------------------------------------Nginx 的配置
[root@localhost ~]# cat /etc/nginx/nginx.conf
# For more information on configuration, see:
# * Official English Documentation: http://nginx.org/en/docs/
# * Official Russian Documentation: http://nginx.org/ru/docs/
user nginx;
worker_processes auto;
error_log /var/log/nginx/error.log;
pid /var/run/nginx.pid;
# Load dynamic modules. See /usr/share/nginx/README.dynamic.
include /usr/share/nginx/modules/*.conf;
events {
worker_connections 1024;
}
http {
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';
access_log /var/log/nginx/access.log main;
upstream websrvs {
server 192.168.3.1:80 weight=2;
server 192.168.3.2:80 weight=1;
}
server {
listen 192.168.3.100:80;
root /app/site1;
location / {
proxy_pass http://websrvs;
}
}
sendfile on;
tcp_nopush on;
tcp_nodelay on;
keepalive_timeout 65;
types_hash_max_size 2048;
include /etc/nginx/mime.types;
default_type application/octet-stream;
# Load modular configuration files from the /etc/nginx/conf.d directory.
# See http://nginx.org/en/docs/ngx_core_module.html#include
# for more information.
include /etc/nginx/conf.d/*.conf;
}
---------------------------------------------------------------------------keepalived配置
[root@localhost ~]# cat /etc/keepalived/keepalived.conf
! Configuration:
global_defs {
notification_email {
root@localhost
}
notification_email_from root@localhost
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id LVS1
vrrp_mcast_group 224.100.100.100
}
vrrp_script chk_down {
script "[[ -f /etc/keepalived/down ]] && exit 1 || exit 0"
interval 1
weight -20
}
vrrp_script chk_nginx{
script "kiallall -0 nginx && exit 0 || exit 1"
interval 1
weight -20
fall 2
rise 1
}
vrrp_instance VI_1 {
state MASTER
interface eth0
virtual_router_id 88
priority 100
advert_int 1
authentication {
auth_type PASS
auth_pass 12121212
}
virtual_ipaddress {
192.168.3.100/24 dev eth0 label eth0:0
}
track_script{
chk_down
chk_nginx
}
notify_master "/etc/keepalived/notify.sh master"
notify_backup "/etc/keepalived/notify.sh backup"
notify_fault "/etc/keepalived/notify.sh fault"
}
2.备的192.168.3.12配置
nginx的配置与主的一样
-------------------------------------------------------------------keepalived配置
[root@CentOS6 keepalived]#cat /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
notification_email {
root@localhost
}
notification_email_from root@localhost
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id LVS1
vrrp_mcast_group 224.100.100.100
}
vrrp_script chk_down {
script "[[ -f /etc/keepalived/down ]] && exit 1 || exit 0"
interval 1
weight -20
}
vrrp_script chk_nginx{
script "kiallall -0 nginx && exit 0 || exit 1"
interval 1
weight -20
fall 2
rise 1
}
vrrp_instance VI_1 {
state BACKUP
interface eth0
virtual_router_id 88
priority 80
advert_int 1
authentication {
auth_type PASS
auth_pass 12121212
}
virtual_ipaddress {
192.168.3.100/24 dev eth0 label eth0:0
}
track_script{
chk_down
chk_nginx
}
notify_master "/etc/keepalived/notify.sh master"
notify_backup "/etc/keepalived/notify.sh backup"
notify_fault "/etc/keepalived/notify.sh fault"
}
3.后台的两台RS就简单的配置下nginx的web service
4.测试
[root@station1 ~]# curl 192.168.3.100
this is 3.2 page
[root@station1 ~]# curl 192.168.3.100
this is 3.1 page
[root@station1 ~]# curl 192.168.3.100
this is 3.1 page
[root@station1 ~]# curl 192.168.3.100
this is 3.2 page
[root@station1 ~]# curl 192.168.3.100
this is 3.1 page
[root@localhost keepalived]# tcpdump -i eth0 -nn host 224.0.0.18
15:15:44.475689 IP 192.168.3.11 > 224.0.0.18: VRRPv2, Advertisement, vrid 88, prio 100, authtype simple, intvl 1s, length 20
15:15:45.476960 IP 192.168.3.11 > 224.0.0.18: VRRPv2, Advertisement, vrid 88, prio 100, authtype simple, intvl 1s, length 20
15:15:46.478534 IP 192.168.3.11 > 224.0.0.18: VRRPv2, Advertisement, vrid 88, prio 100, authtype simple, intvl 1s, length 20
15:15:46.809101 IP 192.168.3.11 > 224.0.0.18: VRRPv2, Advertisement, vrid 88, prio 0, authtype simple, intvl 1s, length 20
15:15:47.498791 IP 192.168.3.12 > 224.0.0.18: VRRPv2, Advertisement, vrid 88, prio 80, authtype simple, intvl 1s, length 20
15:15:48.500509 IP 192.168.3.12 > 224.0.0.18: VRRPv2, Advertisement, vrid 88, prio 80, authtype simple, intvl 1s, length 20
15:15:49.501727 IP 192.168.3.12 > 224.0.0.18: VRRPv2, Advertisement, vrid 88, prio 80, authtype simple, intvl 1s, length 2