DVR介绍

dvr是distributed virtual router的缩写，目的是为了解决openstack neutron部署的扩展问题，分发网络节点的流量负载到计算节点上。如果实例绑定了floating ip，外网流量直接从计算节点出去，不经过网络节点。对于那些没绑定floating ip的实例外网流量还是走snat，要经过网络节点。


DVR配置

这里说明一下，我的controller node和network node在同一台机器上

controller节点
[root@controller-162 ~(keystone_admin)]# vim /etc/neutron/neutron.conf  # 添加如下配置选项
router_distributed = True

[root@controller-162 ~(keystone_admin)]# vim /etc/neutron/plugins/ml2/ml2_conf.ini   # 网络采用ml2+vxlan,配置如下
[ml2]
type_drivers = flat,vxlan,vlan,gre
tenant_network_types = vxlan
mechanism_drivers = openvswitch,l2population
[ml2_type_flat]
[ml2_type_vlan]
[ml2_type_gre]
[ml2_type_vxlan]
vni_ranges = 1:1000
vxlan_group = 239.1.1.1
[agent]
l2_population = True
tunnel_types = vxlan
enable_distributed_routing = True
[ovs]
enable_tunneling = True
tunnel_type = vxlan
integration_bridge = br-int
local_ip = 10.0.0.162
[securitygroup]
firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver
enable_security_group = True
enable_ipset = True

[root@controller-162 ~(keystone_admin)]# vim /etc/neutron/l3_agent.ini 
[DEFAULT]
interface_driver = neutron.agent.linux.interface.OVSInterfaceDriver
ovs_use_veth = True
use_namespaces = True
external_network_bridge = br-ex
router_delete_namespaces = True
agent_mode = dvr_snat             # 虚拟机snat上外网走network node 的L3


compute节点
[root@compute-2 ~]# vim /etc/neutron/plugins/ml2/ml2_conf.ini   # 网络采用ml2+vxlan,配置如下
[ml2]
type_drivers = flat,vxlan,vlan,gre
tenant_network_types = vxlan
mechanism_drivers = openvswitch,l2population
[ml2_type_flat]
[ml2_type_vlan]
[ml2_type_gre]
[ml2_type_vxlan]
vni_ranges = 1:1000
vxlan_group = 239.1.1.1
[agent]
l2_population = True
tunnel_types = vxlan
enable_distributed_routing = True
[ovs]
enable_tunneling = True
tunnel_type = vxlan
integration_bridge = br-int
local_ip=10.0.0.2
[securitygroup]
firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver
enable_security_group = True
enable_ipset = True

[root@compute-2 ~]# vim /etc/neutron/l3_agent.ini  # compute node也要起l3-agent,记住还要add-br br-ex,add-port br-ex eth2这些
[DEFAULT]
interface_driver = neutron.agent.linux.interface.OVSInterfaceDriver
ovs_use_veth = True
use_namespaces = True
external_network_bridge = br-ex
router_delete_namespaces = True
agent_mode = dvr                     # floating ip直接走compute node的l3 

[root@compute-2 ~]# vim /etc/neutron/metadata_agent.ini  # compute node也起了l3，所以会有找metadata的问题
[DEFAULT]
auth_url = http://controller-162:35357/v2.0
auth_region = regionOne
admin_tenant_name = service
admin_user = neutron
admin_password = neutron
nova_metadata_ip = controller-162                #  controller node的metadata地址(nova-api接管metadata服务)
metadata_proxy_shared_secret = meta_pass


DVR功能

先来看下controller node的namespace

[root@controller-162 ~(keystone_admin)]# ip netns
qdhcp-f8876645-352e-48d2-b96c-304cb8de805f
snat-ac4a4d9b-27c7-492b-824a-ae384710ab2a        # 虚拟机上外网专门的snat，单独出来的namespace
qrouter-ac4a4d9b-27c7-492b-824a-ae384710ab2a
[root@compute-3 ~]# ip netns
qrouter-ac4a4d9b-27c7-492b-824a-ae384710ab2a    #  qrouter-xxxxxxx  跟上面一样的哦，floating ip的dnat规则在这里

# 我在dashboard上面绑定floating ip的时候找不到port，后台命令行可以绑定，这个bug openstack官方已经修复。
# 详情见这里 https://review.openstack.org/#/c/132383/3/openstack_dashboard/api/neutron.py
#           https://ask.openstack.org/en/question/51634/juno-dvr-associate-floating-ip-reported-no-ports-available/

[root@controller-162 ~(keystone_admin)]# nova floating-ip-associate test-5 172.16.101.2   # 绑定floating ip，出现了下面这个东东
[root@compute-3 ~]# ip netns
fip-53f6877e-2f46-43e3-93b7-7e22786cacb2    # 绑定floating ip后，多出来这个namespace
qrouter-ac4a4d9b-27c7-492b-824a-ae384710ab2a中rfp接口和fip-53f6877e-2f46-43e3-93b7-7e22786cacb2中的fpr接口是一对veth，打通两个namespace

[root@compute-3 ~]# route -n   
Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
0.0.0.0         172.16.101.254  0.0.0.0         UG    0      0        0 fg-d7d8b46c-14       #  外网的网关在这里
169.254.30.174  0.0.0.0         255.255.255.254 U     0      0        0 fpr-02fa05dc-3
169.254.31.184  0.0.0.0         255.255.255.254 U     0      0        0 fpr-30e06cb3-7
169.254.31.224  0.0.0.0         255.255.255.254 U     0      0        0 fpr-1eaac5a6-2
172.16.101.0    0.0.0.0         255.255.255.0   U     0      0        0 fg-f57ba4c0-e1
172.16.101.0    0.0.0.0         255.255.255.0   U     0      0        0 fg-d7d8b46c-14
172.16.101.24   169.254.31.184  255.255.255.255 UGH   0      0        0 fpr-30e06cb3-7
172.16.101.70   169.254.31.224  255.255.255.255 UGH   0      0        0 fpr-1eaac5a6-2
172.16.101.71   169.254.31.224  255.255.255.255 UGH   0      0        0 fpr-1eaac5a6-2
172.16.101.72   169.254.31.224  255.255.255.255 UGH   0      0        0 fpr-1eaac5a6-2

Trouble Shooting

问题：
ERROR neutron.agent.linux.utils [req-adef3886-1fdd-4904-a442-0f5d43d70d1b None]
Command: ['sudo', 'neutron-rootwrap', '/etc/neutron/rootwrap.conf', 'ip', 'netns', 'exec', 'qrouter-02fa05dc-355f-42fb-8608-6b5d920ece6a', 'ip', '-4', 'neigh', 'del', '10.0.0.38', 'lladdr', 'fa:16:3e:b8:6d:15', 'dev', 'qr-55a5ec64-31']
Exit code: 2
Stdout: ''
Stderr: 'RTNETLINK answers: No such file or directory\n'
# neutron/agent/l3_agent.py中没有判断这个device是否存在，这个bug官方已修。

解决方法：
vim /usr/lib/python2.7/site-packages/neutron/agent/l3_agent.py # 编辑l3_agent.py文件
 
             int_dev = ip_wrapper.add_veth(rtr_2_fip_name,             # 查找这些代码所在位置
                                      fip_2_rtr_name, fip_ns_name)
             self.internal_ns_interface_added(str(rtr_2_fip),
                                         rtr_2_fip_name, ri.ns_name)
             self.internal_ns_interface_added(str(fip_2_rtr),
                                         fip_2_rtr_name, fip_ns_name)
             int_dev[0].link.set_up()
             int_dev[1].link.set_up()
 
# 在这些代码前面，加个判断 
 
             if not ip_lib.device_exists(rtr_2_fip_name,self.root_helper,
                                      namespace=ri.ns_name):
                 int_dev = ip_wrapper.add_veth(rtr_2_fip_name,
                                      fip_2_rtr_name, fip_ns_name)
                 self.internal_ns_interface_added(str(rtr_2_fip),
                                         rtr_2_fip_name, ri.ns_name)
                 self.internal_ns_interface_added(str(fip_2_rtr),
                                         fip_2_rtr_name, fip_ns_name)
                 int_dev[0].link.set_up()
                 int_dev[1].link.set_up()

参考链接

https://wiki.openstack.org/wiki/Neutron/DVR/HowTo

https://wiki.openstack.org/wiki/Neutron/DVR#Juno_and_Distributed_Routing

http://www.openstack.cn/p2510.html

DistributedVirtualRouter-East-WestNorthSouthwithServices.pdf