【centos6.5】
高可用负载均衡群集Keepalived+LVS+LVM+DR
一、高可用负载均衡群集Keepalived+LVS+LVM+DR
1.0实验环境
[root@BLM ~]# vi /etc/sysconfig/network-scripts/ifcfg-eth0
DEVICE=eth0
HWADDR=00:0c:29:ac:ff:92
TYPE=Ethernet
UUID=b7ea0783-e15f-4f78-84ff-017f1c97781d
ONBOOT=yes
NM_CONTROLLED=yes
BOOTPROTO=none
IPADDR=192.168.23.10
NETMASK=255.255.255.0
GATEWAY=192.168.23.1
[root@BLS ~]# vi /etc/sysconfig/network-scripts/ifcfg-eth0
DEVICE=eth0
HWADDR=00:0C:29:99:0E:7A
TYPE=Ethernet
UUID=b7ea0783-e15f-4f78-84ff-017f1c97781d
ONBOOT=yes
NM_CONTROLLED=yes
BOOTPROTO=none
IPADDR=192.168.23.130
GATEWAY=192.168.23.10
NETMASK=255.255.255.0
[root@web ~]# vi /etc/sysconfig/network-scripts/ifcfg-eth2
DEVICE=eth2
HWADDR=00:0c:29:b1:a6:35
TYPE=Ethernet
#UUID=b7ea0783-e15f-4f78-84ff-017f1c97781d
ONBOOT=yes
NM_CONTROLLED=yes
BOOTPROTO=none
IPADDR=192.168.23.131
GATEWAY=192.168.23.10
NETMASK=255.255.255.0
[root@web2 ~]# vi /etc/sysconfig/network-scripts/ifcfg-eth2
DEVICE=eth2HWADDR=00:0c:29:b2:da:2b
TYPE=Ethernet
ONBOOT=yes
NM_CONTROLLED=yes
BOOTPROTO=none
IPADDR=192.168.23.129
GATEWAY=192.168.23.10
NETMASK=255.255.255.0
主负载均衡器 192.168.23.10 BLM
从负载均衡器 192.168.23.130 BLS
Web 192.168.23. 131 web
Web2 192.168.23.129 web2
Vip 192.168.23.66
1.1 web 、web2配置
【web、web2已经安装过apache,】
[root@web ~]# netstat -anpt|grep httpd
tcp 0 0 :::80 :::* LISTEN 24700/httpd
[root@web ~]# chkconfig httpd on
[root@web ~]# echo "web" >> /usr/local/apache/htdocs/index.html【源安装的路径是这个】
【用yum安装的路径是etc/www/html/index.html】
[root@web ~]# cat /opt/lvs-dr 【手动添加】
#!/bin/bash
#lvs-dr
VIP="192.168.23.66" 【虚拟的】
/sbin/ifconfig lo:vip $VIP broadcast $VIP netmask 255.255.255.255
/sbin/route add -host $VIP dev lo:vip
echo 1 > /proc/sys/net/ipv4/conf/lo/arp_ignore【解决ARP的办法】
echo 2 > /proc/sys/net/ipv4/conf/lo/arp_announce
echo 1 > /proc/sys/net/ipv4/conf/all/arp_ignore
echo 2 > /proc/sys/net/ipv4/conf/all/arp_announce
[root@web ~]# chmod +x /opt/lvs-dr
[root@web ~]# /opt/lvs-dr
[root@web ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 16436 qdisc noqueue state UNKNOWN
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
inet 192.168.23.66/32 brd 192.168.23.66 scope global lo:vip
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: eth2: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
link/ether 00:0c:29:b1:a6:35 brd ff:ff:ff:ff:ff:ff
inet 192.168.23.131/24 brd 192.168.23.255 scope global eth2
inet6 fe80::20c:29ff:feb1:a635/64 scope link
valid_lft forever preferred_lft forever
[root@web ~]# echo "/opt/lvs-dr" >>/etc/rc.local 【添加路径到该文件中,/etc/rc.local是个配置文件,这个配置文件会在用户登录之前读取,每次系统启动时会执行一次】
【web2】
[root@web 2~]# netstat -anpt|grep httpd
tcp 0 0 :::80 :::* LISTEN 24700/httpd
[root@web2 ~]# chkconfig httpd on
[root@web 2~]# echo "web2" >> /usr/local/apache/htdocs/index.html【源安装的路径是这个】
【用yum安装的路径是etc/www/html/index.html】
[root@web2 ~]# cat /opt/lvs-dr 【手动添加】
#!/bin/bash
#lvs-dr
VIP="192.168.23.66" 【虚拟的】
/sbin/ifconfig lo:vip $VIP broadcast $VIP netmask 255.255.255.255
/sbin/route add -host $VIP dev lo:vip
echo 1 > /proc/sys/net/ipv4/conf/lo/arp_ignore
echo 2 > /proc/sys/net/ipv4/conf/lo/arp_announce
echo 1 > /proc/sys/net/ipv4/conf/all/arp_ignore
echo 2 > /proc/sys/net/ipv4/conf/all/arp_announce
[root@web 2~]# chmod +x /opt/lvs-dr
[root@web 2~]# /opt/lvs-dr
[root@web 2~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 16436 qdisc noqueue state UNKNOWN
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
inet 192.168.23.66/32 brd 192.168.23.66 scope global lo:vip
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: eth2: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
link/ether 00:0c:29:b2:da:2b brd ff:ff:ff:ff:ff:ff
inet 192.168.23.129/24 brd 192.168.23.255 scope global eth2
inet6 fe80::20c:29ff:feb2:da2b/64 scope link
valid_lft forever preferred_lft forever
[root@web2 ~]# echo "/opt/lvs-dr" >>/etc/rc.local
【scp -p 192.168.23.129://opt /opt -p选项代表保持原来的权限,包括atime、ctime、mtime都保持不变】
【atime, access time 访问时间】
【mtime ,modify time 文件数据最新的修改时间,编辑,修改】
【ctime ,change time 文件状态改变时间,权限,组,用户】
【find -atime -mtime -ctime】
【[root@nginx ~]# find /etc/ -ctime -20 -ctime +1寻找/etc/目录下所有一天前,20天以后,1天前文件状态修改过的文件】
1.2 主负载均衡器(BLM)配置
[root@BLM ~]# modprobe ip_vs
[root@BLM ~]# cat /proc/net/ip_vs
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
[root@BLM ~]# rpm -q ipvsadm keepalived
ipvsadm-1.26-2.el6.x86_64
package keepalived is not installed
[root@BLM ~]# yum -y install keepalived
[root@BLM ~]# cd /etc/keepalived/
[root@BLM keepalived]# cp keepalived.conf keepalived.conf.origin
[root@BLM keepalived]# vim keepalived.conf
! Configuration File for keepalived
global_defs {
notification_email {
haolilong@126.com 【报警邮件地址,每行一个】
}
notification_email_from root@www.kgc.com【设置邮件的发送地址】
smtp_server 172.0.0.1 【设置smtp server 地址】
smtp_connect_timeout 30【设置连接smtp服务器超时时间,30秒】
router_id LVS_DEVEL_BLM【运行Keepalived服务器标识,发邮件时显示在邮件标题中的信息,Backup(Slave)服务器将此项改为LVS_DEVEL_BLS】
}
【SMTP邮件服务器,既简单邮件传输协议,它是一组用于有源地址到目的地址传送邮件的规则】
【vrrp虚拟路由冗余协议,是由IETF提出的解决局域网中配置静态网关出现单点失效现象的路由协议】
vrrp_instance VI_1 {【vrrp实例定义部分】
state MASTER 【指定Keepalived的角色,MASTER表示主服务器,BACKUP或SLAVE表示备用服务器】
interface eth0 【设备名,指定HA检测网络的接口】
virtual_router_id 51【虚拟路由标识,这个标识是一个数字,并且同一个vrrp实例使用唯一的标识,即同一个vrrp_instance下,MASTER和BACKUP必须是一致的】【vrrp_instance, vrrp实例】
priority 100【优先级1-254,数字越大优先级越高,主服务器一定要高过备份服务器,且两者之间的数值差越小越好。MASTER优先级为100,BACKUP可设为99】
advert_int 1【设定MASTER与BACKUP负载均衡器之间同步检查的间隔时间2秒】
authentication {【设置验证类型和密码】
auth_type PASS【设置验证类型,主要有PASS和AH两种】
auth_pass 1111【设置验证密码,在一个vrrp_instance下,MASTER与BACKUP必须使用相同的密码才能正常通信】
}
virtual_ipaddress {【设置虚拟ip地址,可以设置多个虚拟ip地址,每行一个】
192.168.23.66
}
}
virtual_server 192.168.23.66 80 {【设置虚拟服务器,需要指定虚拟ip地址和服务端口,ip与端口之间用空格隔开】
delay_loop 2 【设置健康检查时间,2秒】
lb_algo rr【设置负载调度算法,这里设置为rr,即轮询算法】
lb_kind DR【设置LVS实现负载均衡的机制,可以有NAT、TUM、DR三个模式】
!nat_mask 255.255.255.0【若非使用NAT模式,此行需要注解掉】
!persistence_timeout 50【存留超时时间,50秒,即客户机连接成功后,50秒后才会切换服务器】
protocol TCP【指定转发协议,TCP或UDP】
real_server 192.168.23.131 80 {【设置真实服务器,需要指定真实ip地址和服务端口 ,ip与端口之间用空格隔开】
weight 1【配置服务节点的权值,权值大小用数字表示,数字越大权值越高,设置权值的大小可以为不同性能的服务器分配不同的负载,可以对性能高的服务器设置较高的权值,而对性能较低的服务器设置相对较低的权值,这样就合理的利用和分配了系统资源】
TCP_CHECK {【realserve的状态检测设置部位,单位是秒】
connect_timeout 10【10秒无响应超时】
nb_get_retry 3【重试次数】
delay_before_retry 3【俩次重试的间隔为3秒】
connect_port 80【测试连接端口】
}
}
real_server 192.168.23.129 80 {
weight 1
TCP_CHECK {
connect_timeout 10
nb_get_retry 3
delay_before_retry 3
connect_port 80
}
}
}
[root@BLM keepalived]# /etc/init.d/keepalived start
正在启动 keepalived: [确定]
[root@BLM keepalived]# ipvsadm -ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP 192.168.23.66:80 rr
-> 192.168.23.129:80 Route 1 0 0
-> 192.168.23.131:80 Route 1 0 0
[root@BLM keepalived]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 16436 qdisc noqueue state UNKNOWN
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
link/ether 00:0c:29:ac:ff:92 brd ff:ff:ff:ff:ff:ff
inet 192.168.23.10/24 brd 192.168.23.255 scope global eth0
inet 192.168.23.66/32 scope global eth0:vip
inet6 fe80::20c:29ff:feac:ff92/64 scope link
valid_lft forever preferred_lft forever
【健康检查时间默认6秒】
【模式默认NAT】
1.3 从负载均衡器(BLS)配置
[root@BLS ~]# modprobe ip_vs
[root@BLS ~]# cat /proc/net/ip_vs
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
[root@BLS ~]# rpm -q ipvsadm keepalived
package ipvsadm is not installed
package keepalived is not installed
[root@BLS ~]# yum -y install ipvsadm keepalived
[root@BLS ~]# rpm -q ipvsadm keepalived
ipvsadm-1.26-2.el6.x86_64
keepalived-1.2.7-3.el6.x86_64
[root@BLS ~]# mv /etc/keepalived/keepalived.conf /etc/keepalived/keepalived.conf.origin
[root@BLS ~]# scp 192.168.23.10:/etc/keepalived/keepalived.conf /etc/keepalived/
The authenticity of host '192.168.23.10 (192.168.23.10)' can't be established.
RSA key fingerprint is 0a:27:71:e8:77:61:ca:38:a7:06:49:34:5b:89:4f:22.
Are you sure you want to continue connecting (yes/no)? yes
Warning: Permanently added '192.168.23.10' (RSA) to the list of known hosts.
root@192.168.23.10's password: 123123
keepalived.conf 100% 1023 1.0KB/s 00:00
[root@BLS ~]# vim /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
notification_email {
haolilong@126.com
}
notification_email_from root@www.kgc.com
smtp_server 172.0.0.1
smtp_connect_timeout 30
router_id LVS_BLM
}
vrrp_instance VI_1 {
state BACKUP 【或者SLAVE】备份
interface eth2
virtual_router_id 51
priority 99
advert_int 1
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
192.168.23.66
}
}
virtual_server 192.168.23.66 80 {
delay_loop 2
lb_algo rr
lb_kind DR
!nat_mask 255.255.255.0
!persistence_timeout 50
protocol TCP
real_server 192.168.23.131 80 {
weight 1
TCP_CHECK {
connect_timeout 10
nb_get_retry 3
delay_before_retry 3
connect_port 80
}
}
real_server 192.168.23.129 80 {
weight 1
TCP_CHECK {
connect_timeout 10
nb_get_retry 3
delay_before_retry 3
connect_port 80
}
}
}
[root@BLS ~]# /etc/init.d/keepalived start
正在启动 keepalived: [确定]
[root@BLS ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 16436 qdisc noqueue state UNKNOWN
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: eth2: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
link/ether 00:0c:29:e9:2a:2d brd ff:ff:ff:ff:ff:ff
inet 192.168.23.130/24 brd 192.168.23.255 scope global eth2【此处没有出现vip是对的】
inet6 fe80::20c:29ff:fee9:2a2d/64 scope link
valid_lft forever preferred_lft forever
[root@BLS ~]# ipvsadm -ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP 192.168.23.66:80 rr
-> 192.168.23.129:80 Route 1 0 0
-> 192.168.23.131:80 Route 1 0 0
1.4 客户机测试
[root@BLS ~]# curl 192.168.23.66
<html><body><h1>web1</h1></body></html>
[root@BLS ~]# curl 192.168.23.66
<html><body><h1>web2</h1></body></html>
[root@BLS ~]# curl 192.168.23.66
<html><body><h1>web1</h1></body></html>
[root@BLS ~]# curl 192.168.23.66
<html><body><h1>web2</h1></body></html>
[root@web ~]# curl 192.168.23.66
<html><body><h1>web1</h1></body></html>
[root@web2 ~]# curl 192.168.23.66
<html><body><h1>web2</h1></body></html>
【主负载均衡器启用时 curl vip 不显示】
1.4.1查看主负载均衡器记录
[root@BLM ~]# ipvsadm -Lnc
IPVS connection entries
pro expire state source virtual destination
TCP 01:53 FIN_WAIT 192.168.23.130:39914 192.168.23.66:80 192.168.23.129:80
TCP 01:16 FIN_WAIT 192.168.23.1:11036 192.168.23.66:80 192.168.23.129:80
TCP 01:22 FIN_WAIT 192.168.23.1:11035 192.168.23.66:80 192.168.23.131:80
TCP 01:29 FIN_WAIT 192.168.23.1:11037 192.168.23.66:80 192.168.23.129:80
TCP 01:47 FIN_WAIT 192.168.23.1:11040 192.168.23.66:80 192.168.23.131:80
1.4.2查看负载均衡器记录
[root@BLS ~]# ipvsadm -Lnc
IPVS connection entries
pro expire state source virtual destination
1.4.3健康检测测试
[root@BLM ~]# ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP 192.168.23.66:80 rr
-> 192.168.23.129:80 Route 1 1 0
-> 192.168.23.131:80 Route 1 1 0
1.4.4关闭web2的httpd服务
[root@web2 ~]# /usr/local/apache/bin/apachectl stop 【用源代码就的用这个命令】
1.4.5发现web2的记录不见了,再开启web2的httpd服务,然后web2的记录就会回来
[root@BLM ~]# ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP 192.168.23.66:80 rr
-> 192.168.23.131:80 Route 1 1 0
1.4.6高可用测试:暂停主负载均衡的网络服务
1.4.7客户机可以正常访问,恢复主负载均衡器的网络服务,VIP将会回到BLM上
[root@BLS ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 16436 qdisc noqueue state UNKNOWN
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: eth2: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
link/ether 00:0c:29:e9:2a:2d brd ff:ff:ff:ff:ff:ff
inet 192.168.23.130/24 brd 192.168.23.255 scope global eth2
inet 192.168.23.66/32 scope global eth2
inet6 fe80::20c:29ff:fee9:2a2d/64 scope link
valid_lft forever preferred_lft forever
实验补充:
1、配置完文件后,需要重启服务,在测试时不通,可以重启服务,换浏览器,多刷新,各台机子互相ping。
2.出现以下情况
原因,此目录里有eth1会找错文件,建议删除
[root@BLS ~]# cd /etc/sysconfig/network-scripts/
[root@BLS network-scripts]# ls
ifcfg-eth0 ifcfg-eth1