lvs介绍与实现--3种模式

lvs介绍与实现–3种模式

linux virtual server
vs–virtual server 。负责调度–director server(DS)
rs–real server 负责服务
L4–四层路由器或交换机
阿里的四层LSB（server load balance）基于lvs+keepalived实现
LVS：ipvsadm/ipvs
ipvsadm:用户空间的命令行工具，规则管理器
ipvs:工作于netfilter的INPUT钩子上的框架
lvs四种工作模型
nat，dr， tun，fullnat
lvs没有健康性检查，对于后台挂掉的服务也会进行调度

实验1：基于nat的LVS实现

环境规划：
client：桥接
LVS：双网卡（桥接，vmnet8），ip_forward=1
rsrs2：vmnet8，gw：lvsDIP

[root@lvs ~]# ipvsadm -A -t 192.168.1.223:80 -s rr			#	-s代表调度策略--轮询，对外发布的ip地址192.168.1.223，端口号80
[root@lvs ~]# ipvsadm -a -t 192.168.1.223:80 -r 192.168.18.128:80 -m   # -r--real server ；-m Masq ， 伪装模式，就是dnat
[root@lvs ~]# ipvsadm -a -t 192.168.1.223:80 -r 192.168.18.129:80 -m
[root@lvs ~]# ipvsadm -Ln	
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  192.168.1.223:80 rr
  -> 192.168.18.128:80            Masq    1      0          0         
  -> 192.168.18.129:80            Masq    1      0          0 

##客户端访问检验，确实是rr轮询模式
[root@client ~]# while true;do curl 192.168.1.223;sleep 1 ;done
rs1
rs2
rs1
rs2

##查看httpd的access_log，发现源ip确实是客户端的真实ip地址	
[root@httpd2 httpd]# tail -f access_log 
192.168.1.222 - - [04/May/2021:12:00:24 -0400] "GET / HTTP/1.1" 200 4 "-" "curl/7.29.0"
192.168.1.222 - - [04/May/2021:12:00:26 -0400] "GET / HTTP/1.1" 200 4 "-" "curl/7.29.0"
192.168.1.222 - - [04/May/2021:14:04:32 -0400] "GET / HTTP/1.1" 200 4 "-" "curl/7.29.0"
192.168.1.222 - - [04/May/2021:14:04:33 -0400] "GET / HTTP/1.1" 200 4 "-" "curl/7.29.0"

lvs-nat总结：

1. RIP和DIP应在同一个网络中，且应使用私网地址，RS的网关要指向DIP
2. 请求报文与响应报文必须经过director转发，director容易成为系统瓶颈
3. 支持端口映射，可以修改请求报文的目标PORT
4. VS必须是linux系统，RS可以是任意OS

实验2：基于DR的LVS实现

环境规划：
client：桥接，gw：router.eth0
router：双网卡（eth0桥接，eth1 vmnet8），ip_forward=1
rs1，rs2，lvs：vmnet8，gw：router.eth1

数据报文流向
ps：LVS对外发布的port，并没有实际在本机监听，只是作为是否需要进行LVS规则转换的一个标记

实现

#lvs配置对外发布的vip：192.168.18.132，以及dr的lvs规则.
#vip子网掩码四个255表示不和任意ip在同网段，vip不作为源地址给别人发报文，vip就等于一个标志
[root@lvs ~]# ifconfig lo:1 192.168.18.132 netmask 255.255.255.255 #broadcast 192.168.18.132 up
[root@lvs ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet 192.168.18.132/32 scope global lo:1
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 00:0c:29:d4:a0:83 brd ff:ff:ff:ff:ff:ff
    inet 192.168.18.131/24 brd 192.168.18.255 scope global noprefixroute ens33
       valid_lft forever preferred_lft forever
    inet6 fe80::20c:29ff:fed4:a083/64 scope link 
       valid_lft forever preferred_lft forever

[root@lvs ~]# ipvsadm -A -t 192.168.18.132:80 -s rr
[root@lvs ~]# ipvsadm -a -t 192.168.18.132:80 -r 192.168.18.128 -g
[root@lvs ~]# ipvsadm -a -t 192.168.18.132:80 -r 192.168.18.129 -g
[root@lvs ~]# ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  192.168.18.132:80 rr
  -> 192.168.18.128:80            Route   1      0          0         
  -> 192.168.18.129:80            Route   1      0          0

#httpd1配置vip，并且禁止基于arp协议的vip宣告以及响应
[root@httpd1 ~]# echo 1 > /proc/sys/net/ipv4/conf/all/arp_ignore
[root@httpd1 ~]# echo 1 > /proc/sys/net/ipv4/conf/lo/arp_ignore
[root@httpd1 ~]# echo 2 > /proc/sys/net/ipv4/conf/all/arp_announce
[root@httpd1 ~]# echo 2 > /proc/sys/net/ipv4/conf/lo/arp_announce
[root@httpd1 ~]# ifconfig lo:1 192.168.18.132 netmask 255.255.255.255 #broadcast 192.168.18.132 up
[root@httpd1 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet 192.168.18.132/32 scope global lo:1
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 00:0c:29:99:f9:5c brd ff:ff:ff:ff:ff:ff
    inet 192.168.18.128/24 brd 192.168.18.255 scope global noprefixroute ens33
       valid_lft forever preferred_lft forever
    inet6 fe80::20c:29ff:fe99:f95c/64 scope link 
       valid_lft forever preferred_lft forever

#httpd2配置vip，并且禁止基于arp协议的vip宣告以及响应
[root@httpd2 ~]# echo 1 > /proc/sys/net/ipv4/conf/all/arp_ignore
[root@httpd2 ~]# echo 1 > /proc/sys/net/ipv4/conf/lo/arp_ignore
[root@httpd2 ~]# echo 2 > /proc/sys/net/ipv4/conf/all/arp_announce
[root@httpd2 ~]# echo 2 > /proc/sys/net/ipv4/conf/lo/arp_announce
[root@httpd2 ~]# ifconfig lo:1 192.168.18.132 netmask 255.255.255.255 #broadcast 192.168.18.132 up
[root@httpd2 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet 192.168.18.132/32 scope global lo:1
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 00:0c:29:4e:94:91 brd ff:ff:ff:ff:ff:ff
    inet 192.168.18.129/24 brd 192.168.18.255 scope global noprefixroute ens33
       valid_lft forever preferred_lft forever
    inet6 fe80::20c:29ff:fe4e:9491/64 scope link 
       valid_lft forever preferred_lft forever

[root@client ~]# while true;do curl  192.168.18.132;sleep 1;done
rs1
rs2
rs1
rs2

[root@router ~]# arp -n ##因为其余vip的自宣告以及arp响应取消了，vip的mac有且仅有一个
Address HWtype HWaddress Flags Mask Iface
192.168.1.105 ether 70:85:c2:f9:a5:a9 C br0
192.168.18.132 ether 00:0c:29:d4:a0:83 C ens38
192.168.18.128 ether 00:0c:29:99:f9:5c C ens38
192.168.1.1 ether 80:8f:1d:f9:2e:5a C br0
192.168.1.222 ether 00:0c:29:2b:88:60 C br0
192.168.18.131 ether 00:0c:29:d4:a0:83 C ens38
192.168.18.129 ether 00:0c:29:4e:94:91 C ens38

lvs-dr总结：

DR模式的特性

1. 保证前端路由将目标地址为VIP报文统统发给Director Server，而不是RS
2. RS可以使用私有地址；也可以是公网地址，如果使用公网地址，此时可以通过互联网对RIP进行直接访问
3. RS跟Director Server必须在同一个物理网络中（没有路由器），因为Director Server需要通过arp协议去获取RS的物理mac。而非同一物理网络中，路由器会阻止arp传播
4. 所有的请求报文经由Director Server，但响应报文必须不能进过Director Server ，RS的网关绝不允许指向DIP
5. 不支持端口映射
6. RS可以是大多数常见的操作系统

实验3：基于DR的LVS跨网段实现

## LVS对外发布10.35.78.50:80,client需要访问10.35.78.50。因此在路由器上配置一条10.35.78.40，其作用是增加一条10.35.78.0/24的路由。注意不能配置在lo网卡上，因为lo回环网卡只在本机内部通信，默认不增加路由
[root@router ~]# cat /etc/sysconfig/network-scripts/ifcfg-eth1
TYPE=Ethernet
BOOTPROTO=static
NAME=eth1
ONBOOT=yes
IPADDR=192.168.18.130
NETMASK=255.255.255.0
IPADDR1=10.35.78.40
NETMASK1=255.255.255.0
[root@router ~]# route -n
Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
0.0.0.0         192.168.1.1     0.0.0.0         UG    425    0        0 eth0    ##如果单单是路由器的话，路由器是不需要网关的
10.35.78.0      0.0.0.0         255.255.255.0   U     101    0        0 eth1
192.168.1.0     0.0.0.0         255.255.255.0   U     425    0        0 eth0
192.168.18.0    0.0.0.0         255.255.255.0   U     101    0        0 eth1

[root@lvs ~] ifconfig lo:1 10.35.78.50 netmask 255.255.255.255 #broadcast 10.35.78.50  up
[root@lvs ~] ipvsadm -A -t 10.35.78.50:80 -s rr
[root@lvs ~] ipvsadm -a -t 10.35.78.50:80 -r 192.168.18.128 -g
[root@lvs ~] ipvsadm -a -t 10.35.78.50:80 -r 192.168.18.129 -g
[root@lvs ~]# ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  10.35.78.50:80 rr
  -> 192.168.18.128:80            Route   1      0          0         
  -> 192.168.18.129:80            Route   1      0          0 
[root@lvs ~]#  ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo	
       valid_lft forever preferred_lft forever
    inet 10.35.78.50/32 scope global lo:1
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 00:0c:29:d4:a0:83 brd ff:ff:ff:ff:ff:ff
    inet 192.168.18.131/24 brd 192.168.18.255 scope global noprefixroute ens33
       valid_lft forever preferred_lft forever
    inet6 fe80::20c:29ff:fed4:a083/64 scope link 
       valid_lft forever preferred_lft forever

[root@httpd1 ~]# ifconfig lo:1 10.35.78.50 netmask 255.255.255.255 #broadcast 10.35.78.50  up
[root@httpd1 ~]# echo 1 > /proc/sys/net/ipv4/conf/all/arp_ignore
[root@httpd1 ~]# echo 1 > /proc/sys/net/ipv4/conf/lo/arp_ignore
[root@httpd1 ~]# echo 2 > /proc/sys/net/ipv4/conf/all/arp_announce
[root@httpd1 ~]# echo 2 > /proc/sys/net/ipv4/conf/lo/arp_announce
[root@httpd1 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet 10.35.78.50/32 scope global lo:1
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 00:0c:29:99:f9:5c brd ff:ff:ff:ff:ff:ff
    inet 192.168.18.128/24 brd 192.168.18.255 scope global noprefixroute ens33
       valid_lft forever preferred_lft forever
    inet6 fe80::20c:29ff:fe99:f95c/64 scope link 
       valid_lft forever preferred_lft forever

[root@httpd2 ~]# ifconfig lo:1 10.35.78.50 netmask 255.255.255.255 #broadcast 10.35.78.50  up
[root@httpd2 ~]# echo 1 > /proc/sys/net/ipv4/conf/all/arp_ignore
[root@httpd2 ~]# echo 1 > /proc/sys/net/ipv4/conf/lo/arp_ignore
[root@httpd2 ~]# echo 2 > /proc/sys/net/ipv4/conf/all/arp_announce
[root@httpd2 ~]# echo 2 > /proc/sys/net/ipv4/conf/lo/arp_announce
[root@httpd2 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet 10.35.78.50/32 scope global lo:1
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 00:0c:29:4e:94:91 brd ff:ff:ff:ff:ff:ff
    inet 192.168.18.129/24 brd 192.168.18.255 scope global noprefixroute ens33
       valid_lft forever preferred_lft forever
    inet6 fe80::20c:29ff:fe4e:9491/64 scope link 
       valid_lft forever preferred_lft forever