1 缘起
在一次实际项目中,lvs 只能承载量很低,需要对lvs和Linux内核参数进行优化。
2 为什么使用lvs+keepalived架构
(1)LVS可以实现负载均衡,但是不能够进行健康检查。比如一个RS出现故障,LVS 仍然会把请求转发给故障的RS服务器,这样就会导致请求的无效性;keepalived 软件可以进行健康检查。
(2)使用keepalived能同时实现 LVS 的高可用性,解决 LVS 单点故障的问题。
3 lvs+keepalived部署
3.1 部署图
注意:
(1)lvs+keepalived至少需要2台服务器。
(2)需要一个VIP。
(3)RS服务器不能和LVS以及Keepalived复用。
4.2 lvs+keepalived部署(2台DS服务器)
安装ipvs和keepalived
yum install ipvsadm keepalived -y
keepalived配置
注意:
(1)keepalived是否需要争抢主IP,如果不需要,需要把state都修改为BACKUP,并配置nopreempt。
(2)persistence_timeout的作用是:在一定时间内使来自于同一个Client的所有TCP请求被负载到同一个RealServer上,查看ipvsadm -S -n
主节点
#主节点( MASTER )配置文件
cat > /etc/keepalived/keepalived.conf <
! Configuration File forkeepalived
global_defs {
router_id LVS_DEVEL
}
vrrp_instance VI_1 {
state BACKUP ! 主为master,不争抢模式改为BACKUP
nopreempt !不争抢模式添加
interfaceeth0
virtual_router_id 51
priority 100
advert_int 1
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
192.168.112.10
}
}
virtual_server 192.168.112.1080{
delay_loop 6
lb_algo rr
lb_kind DR
! persistence_timeout 0
protocol TCP
real_server 192.168.112.1380{
weight 1
TCP_CHECK {
connect_timeout 10
retry 3
delay_before_retry 3
connect_port 80
}
}
real_server 192.168.112.1480{
weight 1
TCP_CHECK {
connect_timeout 10
retry 3
delay_before_retry 3
connect_port 80
}
}
real_server 192.168.112.1580{
weight 1
TCP_CHECK {
connect_timeout 10
retry 3
delay_before_retry 3
connect_port 80
}
}
}
EOF
从节点
cat > /etc/keepalived/keepalived.conf <
! Configuration File forkeepalived
global_defs {
router_id LVS_DEVEL
}
vrrp_instance VI_1 {
state BACKUP
nopreempt ! 不争抢模式添加
interfaceeth0
virtual_router_id 51
priority 90
advert_int 1
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
192.168.112.10
}
}
virtual_server 192.168.112.1080{
delay_loop 6
lb_algo rr
lb_kind DR
! persistence_timeout 0
protocol TCP
real_server 192.168.112.1380{
weight 1
TCP_CHECK {
connect_timeout 10
retry 3
delay_before_retry 3
connect_port 80
}
}
real_server 192.168.112.1480{
weight 1
TCP_CHECK {
connect_timeout 10
retry 3
delay_before_retry 3
connect_port 80
}
}
real_server 192.168.112.1580{
weight 1
TCP_CHECK {
connect_timeout 10
retry 3
delay_before_retry 3
connect_port 80
}
}
}
EOF
内核参数
echo1> /proc/sys/net/ipv4/ip_forward
sysctl -w net.ipv4.ip_forward=1
启动
systemctl enable keepalived
systemctl start keepalived
4.3 真实服务器配置
配置脚本
不需要在lvs+keepalived的服务器上配置,需要在所有的真实服务器上配置。注意vip必须与前面keepalived上的vip相同。
# vim lvs_dr_rs.sh
SNS_VIP=192.168.112.10
/etc/rc.d/init.d/functions
case"$1"in
start)
ifconfig lo:0$SNS_VIP netmask255.255.255.255broadcast $SNS_VIP
/sbin/route add -host $SNS_VIP dev lo:0
echo "1">/proc/sys/net/ipv4/conf/lo/arp_ignore
echo "2">/proc/sys/net/ipv4/conf/lo/arp_announce
echo "1">/proc/sys/net/ipv4/conf/all/arp_ignore
echo "2">/proc/sys/net/ipv4/conf/all/arp_announce
sysctl -p >/dev/null2>&1
echo "RealServer Start OK"
;;
stop)
ifconfig lo:0down
route del $SNS_VIP >/dev/null2>&1
echo "0">/proc/sys/net/ipv4/conf/lo/arp_ignore
echo "0">/proc/sys/net/ipv4/conf/lo/arp_announce
echo "0">/proc/sys/net/ipv4/conf/all/arp_ignore
echo "0">/proc/sys/net/ipv4/conf/all/arp_announce
echo "RealServer Stoped"
;;
*)
echo "Usage: $0 {start|stop}"
exit 1
esac
exit 0
执行生效
# 所有RS节点上分别执行脚本:
chmod +x lvs_dr_rs.sh
./lvs_dr_rs.sh start
5 参数优化
5.1 LVS参数
增大ipvs模块hash table的大小
ipvs模块hash table默认值为2^12=4096,改为2^20=1048576。可以用ipvsadm -l命令查询当前hash table的大小。
IP Virtual Server version1.2.1(size=4096)
修改方法:
在/etc/modprobe.d/目录下添加文件ip_vs.conf,内容为:
options ip_vs conn_tab_bits=20
重新加载ipvs模块。
IP Virtual Server version1.2.1(size=1048576)
修改 LVS 表中的 timeout
ipvsadm --set90060300
ipvsadm -ln --timeout
Timeout (tcp tcpfin udp): 90060300
5.2 文件句柄及进程数
* soft nofile1024000
* hard nofile 1024000
* soft nproc 1024000
* hard nproc 1024000
5.3 内核参数
fs.file-max =1048576
net.ipv4.ip_forward = 1
net.core.wmem_default = 8388608
net.core.wmem_max = 16777216
net.core.rmem_default = 8388608
net.core.rmem_max = 16777216
net.core.somaxconn = 65535
net.core.optmem_max = 81920
net.core.netdev_max_backlog = 262144
net.ipv4.route.gc_timeout = 20
net.ipv4.tcp_syncookies = 1
net.ipv4.tcp_abort_on_overflow = 1
net.ipv4.tcp_max_tw_buckets = 6000
net.ipv4.tcp_sack = 1
net.ipv4.tcp_window_scaling = 1
net.ipv4.tcp_no_metrics_save = 1
net.ipv4.tcp_rmem = 3276813107216777216
net.ipv4.tcp_wmem = 819213107216777216
net.ipv4.tcp_mem = 94500000915000000927000000
net.ipv4.tcp_max_syn_backlog = 262144
net.ipv4.tcp_max_orphans = 3276800
net.ipv4.tcp_timestamps = 0
net.ipv4.tcp_synack_retries = 1
net.ipv4.tcp_syn_retries = 1
net.ipv4.tcp_tw_recycle = 1
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_fin_timeout = 10
net.ipv4.tcp_keepalive_time = 120
net.ipv4.tcp_keepalive_probes = 3
net.ipv4.tcp_keepalive_intvl = 15
net.ipv4.tcp_retries2 = 5
net.ipv4.ip_local_port_range = 102465000
net.ipv4.conf.default.rp_filter =1
net.ipv4.conf.default.accept_source_route =0
net.ipv4.conf.all.arp_ignore = 1
net.ipv4.conf.all.arp_announce = 2
#modprobe ip_conntrack
net.netfilter.nf_conntrack_tcp_timeout_established = 180
net.netfilter.nf_conntrack_max = 1048576
net.nf_conntrack_max = 1048576
kernel.sysrq = 0
kernel.core_uses_pid = 1
kernel.msgmnb = 65536
kernel.msgmax = 65536
kernel.shmmax = 68719476736
kernel.shmall = 4294967296
注意:
net.ipv4.tcp_tw_recycle = 1有坑,在nat环境下慎用。
越是大并发,越要注意net.ipv4.tcp_max_tw_buckets的值不能太大。
【编辑推荐】
【责任编辑:张燕妮 TEL:(010)68476606】
点赞 0