Play with Neutron IPv6 (by quqi99)

作者：张华  发表于：2016-06-13
版权声明：可以任意转载，转载时请务必以超链接形式标明文章原始出处和作者信息及本版权声明

( http://blog.csdn.net/quqi99 )

实验环境准备

因为家中是中国移动宽带，没有公网IP，所以ipv6 6to4 tunnel是用不了的。为此将openwrt路由器中的WAN6口删除，然后点击"Network -> Interfaces"菜单配置IPv6 ULA-Prefix=2001:2:3:4500::/56（它会修改配置/etc/config/radvd），并在WAN口中配置IPv6 Setting(Router Advertisement-Service=server mode, DHCPv6-Service
=server mode, NDP-Proxy=disabled [5], DHCPv6-Mode=stateless+stateful)。这样家中的台式机会获取到IPv6地址2001:2:3:4500:c12c:6b76:fa2e:74ee/64， 同时openwrt的br-lan口会有网关地址2001:2:3:4500::1/60。

台式机的网络配置如下：

vi /etc/network/interfaces

allow-ovs br-phy
iface br-phy inet dhcp
pre-up /usr/bin/ovs-vsctl -- --may-exist add-br br-phy
pre-up /usr/bin/ovs-vsctl -- --may-exist add-port br-phy eth0
ovs_type OVSBridge
ovs_ports eth0

iface br-phy inet6 static
pre-up modprobe ipv6
address 2001:2:3:4500:fa32:e4ff:febe:87cd
netmask 64
gateway 2001:2:3:4500::1

allow-br-phy eth0
iface eth0 inet manual
ovs_bridge br-phy
ovs_type 

验证环境

sudo ip addr del 2001:2:3:4500::1/64 dev br-phy
sudo ip addr add 2001:2:3:4500:fa32:e4ff:febe:87cd dev br-phy
sudo ip -6 route add 2001:2:3:4500::/64 dev br-phy
route -6 -n |grep 2001
ping6 2001:2:3:4500::1

Devstack

我使用的devstack localrc配置文件如下，其中和IPv6相关的部分见黑体：

#OFFLINE=True
unset http_proxy
unset https_proxy
rm -rf /bak/openstack/devstack/accrc/admin/*
sudo rm -rf /opt/stack/data/nova/instances/*
sudo rm -rf /usr/lib/python2.7/dist-packages/*six*
sudo chown -R hua:root /opt/stack/
sudo ovs-vsctl -- --may-exist add-br br-phy
sudo ovs-vsctl -- --may-exist add-port br-phy eth0
sudo ip -6 addr add 2001:2:3:4500:fa32:e4ff:febe:87cd/64 dev br-phy

# MULTI-HOST
# ENABLED_SERVICES=n-cpu,qpid,neutron,q-agt
ENABLED_SERVICES=rabbit,mysql,key,g-api,g-reg
ENABLED_SERVICES+=,n-api,n-crt,n-obj,n-cpu,n-cond,n-sch
ENABLED_SERVICES+=,cinder,c-api,c-vol,c-sch
#ENABLED_SERVICES+=,q-svc,q-agt,q-dhcp,q-l3,q-meta,neutron,q-lbaas,q-fwaas,q-vpn
ENABLED_SERVICES+=,q-svc,q-agt,q-dhcp,q-l3,q-meta,neutron
#enable_plugin networking-ovs-dpdk https://github.com/openstack/networking-ovs-dpdk master
#enable_plugin networking-sfc /bak/openstack/networking-sfc

#ENABLED_SERVICES+=,horizon
KEYSTONE_TOKEN_FORMAT=UUID
OS_IDENTITY_API_VERSION=2
IP_VERSION=4
RABBIT_HEARTBEAT_TIMEOUT_THRESHOLD=7200
Q_USE_ROOTWRAP=False

#IPv6 tenant network
IP_VERSION=6
IPV6_RA_MODE=slaac
IPV6_ADDRESS_MODE=slaac
HOST_IPV6=2001:2:3:4500:fa32:e4ff:febe:87cd

#IPv6 management network
SERVICE_IP_VERSION=6
SERVICE_HOST=[$HOST_IPV6]
MYSQL_HOST=$HOST_IPV6

#Provider Network & Public Network
Q_USE_PROVIDERNET_FOR_PUBLIC=True
OVS_PHYSICAL_BRIDGE=br-phy
PUBLIC_BRIDGE=br-phy
PUBLIC_INTERFACE=eth0
PUBLIC_PHYSICAL_NETWORK=physnet1
PHYSICAL_NETWORK=physnet1
OVS_BRIDGE_MAPPINGS=physnet1:br-phy
IPV6_PUBLIC_RANGE=2001:2:3:4500::/64
IPV6_PUBLIC_NETWORK_GATEWAY=2001:2:3:4500::1

FIXED_RANGE_V6=db:4127:804a::/64
IPV6_PRIVATE_NETWORK_GATEWAY=db:4127:804a::1

#Q_USE_PROVIDER_NETWORKING=True
#PROVIDER_NETWORK_TYPE=flat
#PROVIDER_SUBNET_NAME=provider_net
#IPV6_PROVIDER_FIXED_RANGE=2001:2:3:4500::/64
#IPV6_PROVIDER_NETWORK_GATEWAY=2001:2:3:4500::1


#HOST_IP=172.16.1.1
#SERVICE_HOST=$HOST_IP
#MYSQL_HOST=$SERVICE_HOST
RABBIT_HOST=$SERVICE_HOST
GLANCE_HOSTPORT=$SERVICE_HOST:9292
Q_HOST=$SERVICE_HOST

DEST=/bak/openstack
GIT_BASE="https://git.openstack.org"
RABBIT_HEARTBEAT_TIMEOUT_THRESHOLD=120   #with kombu_reconnect_delay=3.0
FIXED_RANGE=10.0.1.0/24
#113 -- 118, http://jodies.de/ipcalc
FLOATING_RANGE=192.168.101.0/24
Q_FLOATING_ALLOCATION_POOL=start=192.168.101.3,end=192.168.101.100
PUBLIC_NETWORK_GATEWAY=192.168.101.1
NETWORK_GATEWAY=10.0.1.1

DATABASE_USER=root
DATABASE_PASSWORD=password
ADMIN_PASSWORD=password
SERVICE_PASSWORD=password
RABBIT_PASSWORD=password
SERVICE_TOKEN=ADMIN
LOGFILE=$DEST/logs/stack.log
ENABLE_DEBUG_LOG_LEVEL=False
SYSLOG=False
SCREEN_LOGDIR=$DEST/logs
LOG_COLOR=False
Q_USE_DEBUG_COMMAND=False
APACHE_ENABLED_SERVICES+=keystone
USE_SSL=False
Q_USE_ROOTWRAP_DAEMON=False
disable_service tls-proxy

#ENABLED_SERVICES+=,heat,h-api,h-api-cfn,h-api-cw,h-eng
CIRROS_VERSION=0.3.4
#DOWNLOAD_DEFAULT_IMAGES=False
#IPSEC_PACKAGE=strongswan

# Neutron Agent configuration
Q_USE_NAMESPACE=True
Q_ALLOW_OVERLAPPING_IP=True
# Neutron ML2 configuration
Q_PLUGIN=ml2
Q_ML2_TENANT_NETWORK_TYPE=flat,vlan,gre,vxlan
Q_ML2_PLUGIN_TYPE_DRIVERS=flat,vlan,gre,vxlan
#Q_DVR_MODE=dvr_snat
Q_DVR_MODE=legacy
# VLAN
ENABLE_TENANT_VLANS=True
ML2_VLAN_RANGES=physnet1:1000:2999,physnet2:1000:2999
# GRE
ENABLE_TENANT_TUNNELS=True
TENANT_TUNNEL_RANGE=1000:2000
Q_ML2_PLUGIN_VXLAN_TYPE_OPTIONS=(vni_ranges=1000:2000)

# Avanced ML2 configuration
Q_AGENT=openvswitch
Q_ML2_PLUGIN_MECHANISM_DRIVERS=openvswitch,linuxbridge,l2population,sriovnicswitch

台式机只有一块网卡eth0，并被加到了br-phy，不想devstack改动这块的网络配置。另外，memcached client使用ipv6时有一个bug必须以inet6为前缀（inet6:${SERVICE_HOST}:11211）。所以我修改了devstack脚本，如下：

diff --git a/lib/keystone b/lib/keystone
index 5695004..55b581f 100644
--- a/lib/keystone
+++ b/lib/keystone
@@ -490,7 +490,7 @@ function configure_auth_token_middleware {
     iniset $conf_file $section auth_uri $KEYSTONE_SERVICE_URI
     iniset $conf_file $section cafile $SSL_BUNDLE_FILE
     iniset $conf_file $section signing_dir $signing_dir
-    iniset $conf_file $section memcached_servers $SERVICE_HOST:11211
+    iniset $conf_file $section memcached_servers inet6:${SERVICE_HOST}:11211
 }
 
 # init_keystone() - Initialize databases, etc.
diff --git a/lib/neutron b/lib/neutron
index c0db20d..dc0d677 100644
--- a/lib/neutron
+++ b/lib/neutron
@@ -478,7 +478,8 @@ function _set_config_files {
 function cleanup_neutron {
     if is_neutron_legacy_enabled; then
         # Call back to old function
-        cleanup_mutnauq "$@"
+        #cleanup_mutnauq "$@"
+        echo 'skip to move route from eth0 to br-phy'
     else
         cleanup_neutron_new "$@"
     fi
diff --git a/lib/neutron_plugins/openvswitch_agent b/lib/neutron_plugins/openvswitch_agent
index 69e38f4..f9a4510 100644
--- a/lib/neutron_plugins/openvswitch_agent
+++ b/lib/neutron_plugins/openvswitch_agent
@@ -20,7 +20,8 @@ function neutron_plugin_create_nova_conf {
 }
 
 function neutron_plugin_install_agent_packages {
-    _neutron_ovs_base_install_agent_packages
+    #_neutron_ovs_base_install_agent_packages
+    echo 'skip install ovs package'
 }
 
 function neutron_plugin_configure_debug_command {
diff --git a/lib/neutron_plugins/services/l3 b/lib/neutron_plugins/services/l3
index 350aed8..dab8432 100644
--- a/lib/neutron_plugins/services/l3
+++ b/lib/neutron_plugins/services/l3
@@ -101,10 +101,11 @@ function _configure_neutron_l3_agent {
 
     neutron_plugin_configure_l3_agent
 
-    _move_neutron_addresses_route "$PUBLIC_INTERFACE" "$OVS_PHYSICAL_BRIDGE" True False "inet"
+    #_move_neutron_addresses_route "$PUBLIC_INTERFACE" "$OVS_PHYSICAL_BRIDGE" True False "inet"
 
     if [[ $(ip -f inet6 a s dev "$PUBLIC_INTERFACE" | grep -c 'global') != 0 ]]; then
-        _move_neutron_addresses_route "$PUBLIC_INTERFACE" "$OVS_PHYSICAL_BRIDGE" False False "inet6"
+       # _move_neutron_addresses_route "$PUBLIC_INTERFACE" "$OVS_PHYSICAL_BRIDGE" False False "inet6"
+       echo 'avoid to move ip'
     fi
 }
 
@@ -348,8 +349,8 @@ function _neutron_configure_router_v6 {
             local ipv6_cidr_len=${IPV6_PUBLIC_RANGE#*/}
 
             # Configure interface for public bridge
-            sudo ip -6 addr add $ipv6_ext_gw_ip/$ipv6_cidr_len dev $ext_gw_interface
-            sudo ip -6 route replace $FIXED_RANGE_V6 via $IPV6_ROUTER_GW_IP dev $ext_gw_interface
+            #sudo ip -6 addr add $ipv6_ext_gw_ip/$ipv6_cidr_len dev $ext_gw_interface
+            #sudo ip -6 route replace $FIXED_RANGE_V6 via $IPV6_ROUTER_GW_IP dev $ext_gw_interface
         fi
         _neutron_set_router_id
     fi
diff --git a/tools/install_pip.sh b/tools/install_pip.sh
index 1267699..eca967d 100755
--- a/tools/install_pip.sh
+++ b/tools/install_pip.sh
@@ -76,7 +76,7 @@ function install_get_pip {
             die $LINENO "Download of get-pip.py failed"
         touch $LOCAL_PIP.downloaded
     fi
-    sudo -H -E python $LOCAL_PIP -c $TOOLS_DIR/cap-pip.txt
+    #sudo -H -E python $LOCAL_PIP -c $TOOLS_DIR/cap-pip.txt
     if python3_enabled; then
         sudo -H -E python${PYTHON3_VERSION} $LOCAL_PIP -c $TOOLS_DIR/cap-pip.txt
     fi

创建虚机

export OS_PROJECT_NAME=demo
export OS_PASSWORD=password
export OS_AUTH_STRATEGY=keystone
export OS_AUTH_URL=http://[2001:2:3:4500:fa32:e4ff:febe:87cd]:5000/v2.0
export OS_USERNAME=admin
export OS_TENANT_NAME=demo

nova secgroup-add-rule default icmp -1 -1 ::/0
nova secgroup-add-rule default tcp 22 22 ::/0

结果验证

hua@node1:~$ sudo ip netns exec qrouter-e8860b21-560f-4701-9e02-e91ce26da781 ping6 -c 1 db:4127:804a:0:f816:3eff:fef6:f779
PING db:4127:804a:0:f816:3eff:fef6:f779(db:4127:804a:0:f816:3eff:fef6:f779) 56 data bytes
64 bytes from db:4127:804a:0:f816:3eff:fef6:f779: icmp_seq=1 ttl=64 time=0.616 ms

hua@node1:~$ sudo ip netns exec qrouter-e8860b21-560f-4701-9e02-e91ce26da781 ssh cirros@db:4127:804a:0:f816:3eff:fef6:f779
cirros@db:4127:804a:0:f816:3eff:fef6:f779's password:
$ route -n -A inet6
Kernel IPv6 routing table
Destination                                 Next Hop                                Flags Metric Ref    Use Iface
db:4127:804a::1/128                         db:4127:804a::1                         UAC   0      4        1 eth0    
db:4127:804a::/64                           ::                                      UA    256    0        0 eth0    
fe80::/64                                   ::                                      U     256    0        0 eth0    
::/0                                        fe80::f816:3eff:fe35:348e               UGDA  1024   0        0 eth0    
::1/128                                     ::                                      U     0      0        1 lo      
db:4127:804a:0:f816:3eff:fef6:f779/128      ::                                      U     0      145       1 lo      
fe80::f816:3eff:fef6:f779/128               ::                                      U     0      10       1 lo      
ff02::1/128                                 ff02::1                                 UC    0      1        0 eth0    
ff00::/8                                    ::                                      U     256    0        0 eth0    

hua@node1:~$ sudo ip netns exec qdhcp-40bc6c81-f8ca-4516-b00d-2f370a329a86 ps -ef|grep radvd
root     28255  4756  0 16:18 pts/35   00:00:00 radvd -C /opt/stack/data/neutron/ra/e8860b21-560f-4701-9e02-e91ce26da781.radvd.conf -p /opt/stack/data/neutron/external/pids/e8860b21-560f-4701-9e02-e91ce26da781.pid.radvd -m syslog

hua@node1:~$ cat /opt/stack/data/neutron/ra/e8860b21-560f-4701-9e02-e91ce26da781.radvd.conf
interface qr-6cb2a0a1-5b
{
   AdvSendAdvert on;
   MinRtrAdvInterval 30;
   MaxRtrAdvInterval 100;
   AdvLinkMTU 1500;
   prefix db:4127:804a::/64
   {
        AdvOnLink on;
        AdvAutonomous on;
   };
};

用Juju部署

若没有IPv6环境的话，也可以为测试模拟一个IPv6环境：

sudo sysctl -w net.ipv6.conf.all.forwarding=1
sudo ip addr add 2001:db8:0:1::1/64 dev eth0
sudo apt-get install radvd
$ cat /etc/radvd.conf 
    interface eth0
    {
       AdvSendAdvert on;
       prefix 2001:db8:0:1::/64
       {
            AdvOnLink on;
            AdvAutonomous on;
       };
    };
sudo /etc/init.d/radvd restart
sudo ip6tables -F

之后使用下列命令安装即可：

bzr branch lp:~openstack-charm-testers/+junk/ipv6
juju bootstrap
cd ipv6
juju-deployer -c bundles/ipv6/next-ipv6-ha.yaml -d xenial-mitaka

#注意，此命令由于是往neutron-gateway虚机节点上新增一网卡并加到br-ex网卡， 如果执行多次会造成br-ex有多块网卡引发IPv6路由广播风暴加重性能造成ssh连接断开
./configure   

neutron subnet-create --ip-version=6 --name=ext-v6-subnet --gateway 2001:db8:0:1::1 --allocation-pool start=2001:db8:0:1::5,end=2001:db8:0:1:ffff:ffff:ffff:fffe --disable-dhcp ext_net 2001:db8:0:1::/64
neutron net-create private
neutron subnet-create --ip-version=6 --name=private_v6_subnet --ipv6-address-mode=slaac --ipv6-ra-mode=slaac private 2001:db8:0:2::/64
neutron router-interface-add provider-router private_v6_subnet

nova keypair-add --pub-key ~/.ssh/id_rsa.pub mykey
NET_ID=$(neutron net-list |grep 'private' |awk '{print $2}')
nova boot --poll --key-name mykey --image trusty --flavor 2 --nic net-id=$NET_ID i1

nova secgroup-add-rule default icmp -1 -1 0.0.0.0/0
nova secgroup-add-rule default tcp 22 22 0.0.0.0/0
nova secgroup-add-rule default icmp -1 -1 ::/0
nova secgroup-add-rule default tcp 22 22 ::/0
ssh -i mykey ubuntu@10.5.150.1 -v

主要要注意2个问题：

1, Bug (https://bugs.launchpad.net/charms/+source/keystone/+bug/1581598 ), 所以将keystone-hacluster, openstack-dashboard-hacluster, nova-hacluster, neutron-hacluster, cinder-hacluster, glance-hacluster, ceph-radosgw-hacluster, swift-hacluster, mysql-hacluster的branch改为下列值：

      #branch:  lp:~openstack-charmers/charms/trusty/hacluster/next
      branch: https://github.com/openstack/charm-hacluster

2, 当虚机里的网卡不是eth0时，而比如是ens3时，需要将keystone, openstack-dashboard, nova-cloud-controller, neutron-api, cinder, glance, ceph-radosgw, swift-proxy, mysql的ha-bindiface参数改为ens3. 另外是否修改***-hacluster等一系列charm的corosync_bindiface: 'ens3'待确认。

    ha-bindiface: 'ens3'

参考

[1] https://review.openstack.org/#/c/87987/
[2] https://review.openstack.org/#/c/97823/
[3] http://docs.openstack.org/mitaka/networking-guide/adv-config-ipv6.html
[4] https://www.sixxs.net/wiki/Setting_Up_an_IPv6_home_network_with_Ubuntu
[5] http://bbs.swdyz.com/thread270sw1dyz1.shtml
[6] http://blogs.rdoproject.org/7199/journey-of-ipv6-in-openstack
[7] https://cloudbau.github.io/openstack/neutron/networking/2016/05/17/neutron-ipv6.html
[8] http://docs.openstack.org/mitaka/networking-guide/adv-config-ipv6.html