ovn:经典的租户网络创建Demo

#https:#www.jianshu.com/p/dc565d6aaebd
###创建两个交换机和一个路由器
#创建两个虚拟交换机 ls1 和 ls2
ovn-nbctl ls-add ls1
ovn-nbctl ls-add ls2
#创建一个虚拟路由器 lr1
ovn-nbctl lr-add lr1

#在虚拟路由器 lr1 上添加端口，用来连接虚拟交换机 ls1
ovn-nbctl lrp-add lr1 lr1-ls1 00:00:00:00:00:01 10.10.10.1/24

#在虚拟交换机 ls1 上添加端口，用来连接虚拟路由器 lr1
ovn-nbctl lsp-add ls1 ls1-lr1
#端口类型必须为 router
ovn-nbctl lsp-set-type ls1-lr1 router
#设置地址，必须和 lr1-ls1 的一致
ovn-nbctl lsp-set-addresses ls1-lr1 00:00:00:00:00:01
#指定 router-port
ovn-nbctl lsp-set-options ls1-lr1 router-port=lr1-ls1

#在虚拟路由器 lr1 上添加端口，用来连接虚拟交换机 ls2
ovn-nbctl lrp-add lr1 lr1-ls2 00:00:00:00:00:02 10.10.20.1/24

#在虚拟交换机 ls2 上添加端口，用来连接虚拟路由器 lr1
ovn-nbctl lsp-add ls2 ls2-lr1
#端口类型必须为 router
ovn-nbctl lsp-set-type ls2-lr1 router
#设置地址，必须和 lr1-ls2 的一致
#指定 router-port
ovn-nbctl lsp-set-addresses ls2-lr1 00:00:00:00:00:02
ovn-nbctl lsp-set-options ls2-lr1 router-port=lr1-ls2

###在交换机上添加vm接口
#在虚拟交换机 ls1 上添加两个端口，指定 mac 和 ip(10.10.10.0/24网段)，用来连接vm
ovn-nbctl lsp-add ls1 ls1-vm1
ovn-nbctl lsp-set-addresses ls1-vm1 "00:00:00:00:00:03 10.10.10.2"
ovn-nbctl lsp-set-port-security ls1-vm1 "00:00:00:00:00:03 10.10.10.2"

ovn-nbctl lsp-add ls1 ls1-vm2
ovn-nbctl lsp-set-addresses ls1-vm2 "00:00:00:00:00:04 10.10.10.3"
ovn-nbctl lsp-set-port-security ls1-vm2 "00:00:00:00:00:04 10.10.10.3"

ovn-nbctl lsp-add ls1 ls1-vm3
ovn-nbctl lsp-set-addresses ls1-vm3 "00:00:00:00:00:06 10.10.10.4"
ovn-nbctl lsp-set-port-security ls1-vm3 "00:00:00:00:00:06 10.10.10.4"

#在虚拟交换机 ls2 上添加两个端口，指定 mac 和 ip(10.10.20.0/24网段)，用来连接vm
ovn-nbctl lsp-add ls2 ls2-vm1
ovn-nbctl lsp-set-addresses ls2-vm1 "00:00:00:00:00:03 10.10.20.2"
ovn-nbctl lsp-set-port-security ls2-vm1 "00:00:00:00:00:03 10.10.20.2"

ovn-nbctl lsp-add ls2 ls2-vm2
ovn-nbctl lsp-set-addresses ls2-vm2 "00:00:00:00:00:04 10.10.20.3"
ovn-nbctl lsp-set-port-security ls2-vm2 "00:00:00:00:00:04 10.10.20.3"

ovn-nbctl lsp-add ls2 ls2-vm3
ovn-nbctl lsp-set-addresses ls2-vm3 "00:00:00:00:00:06 10.10.20.4"
ovn-nbctl lsp-set-port-security ls2-vm3 "00:00:00:00:00:06 10.10.20.4"

###创建四个vm
#在 node-1 节点上，创建两个namespace，用来模拟两个vm，使用 "iface-id" 指定
#这两个vm属于 ls1
ip netns add vm1
ovs-vsctl add-port br-int vm1 -- set interface vm1 type=internal
ip link set vm1 netns vm1
ip netns exec vm1 ip link set vm1 address 00:00:00:00:00:03
ip netns exec vm1 ip addr add 10.10.10.2/24 dev vm1
ip netns exec vm1 ip link set vm1 up
ip netns exec vm1 ip route add default via 10.10.10.1 dev vm1
ovs-vsctl set Interface vm1 external_ids:iface-id=ls1-vm1


ip netns add vm2
ovs-vsctl add-port br-int vm2 -- set interface vm2 type=internal
ip link set vm2 netns vm2
ip netns exec vm2 ip link set vm2 address 00:00:00:00:00:04
ip netns exec vm2 ip addr add 10.10.10.3/24 dev vm2
ip netns exec vm2 ip link set vm2 up
ip netns exec vm2 ip route add default via 10.10.10.1 dev vm2
ovs-vsctl set Interface vm2 external_ids:iface-id=ls1-vm2

ip netns add vm3
ovs-vsctl add-port br-int vm3 -- set interface vm3 type=internal
ip link set vm3 netns vm3
ip netns exec vm3 ip link set vm3 address 00:00:00:00:00:06
ip netns exec vm3 ip addr add 10.10.10.4/24 dev vm3
ip netns exec vm3 ip link set vm3 up
ip netns exec vm3 ip route add default via 10.10.10.1 dev vm3
ovs-vsctl set Interface vm3 external_ids:iface-id=ls1-vm3


#在 node2 节点上，创建两个namespace，用来模拟两个vm，使用 "iface-id" 指定这两个vm属于 ls2
ip netns add vm1
ovs-vsctl add-port br-int vm1 -- set interface vm1 type=internal
ip link set vm1 netns vm1
ip netns exec vm1 ip link set vm1 address 00:00:00:00:00:03
ip netns exec vm1 ip addr add 10.10.20.2/24 dev vm1
ip netns exec vm1 ip link set vm1 up
ip netns exec vm1 ip route add default via 10.10.20.1 dev vm1
ovs-vsctl set Interface vm1 external_ids:iface-id=ls2-vm1

ip netns add vm2
ovs-vsctl add-port br-int vm2 -- set interface vm2 type=internal
ip link set vm2 netns vm2
ip netns exec vm2 ip link set vm2 address 00:00:00:00:00:04
ip netns exec vm2 ip addr add 10.10.20.3/24 dev vm2
ip netns exec vm2 ip link set vm2 up
ip netns exec vm2 ip route add default via 10.10.20.1 dev vm2
ovs-vsctl set Interface vm2 external_ids:iface-id=ls2-vm2

ip netns add vm3
ovs-vsctl add-port br-int vm3 -- set interface vm3 type=internal
ip link set vm3 netns vm3
ip netns exec vm3 ip link set vm3 address 00:00:00:00:00:06
ip netns exec vm3 ip addr add 10.10.20.4/24 dev vm3
ip netns exec vm3 ip link set vm3 up
ip netns exec vm3 ip route add default via 10.10.20.1 dev vm3
ovs-vsctl set Interface vm3 external_ids:iface-id=ls2-vm3

#==创建ovs端口=========================
TYPE="OVSPort"
OVS_BRIDGE=br-phy
DEVICE_TYPE=ovs
BOOTPROTO="static"
DEFROUTE=yes
NAME="enp0s8"
DEVICE="enp0s8"
ONBOOT="yes"
#==创建外部网络网桥================
TYPE="OVSBridge"
DEVICETYPE=ovs
OVSBOOTPROTO="static"
DEFROUTE="yes"
NAME="br-phy"
DEVICE="br-phy"
ONBOOT="yes"
IPADDR="172.18.0.11"
PREFIX="24"
GATEWAY="172.18.0.1"
DNS1="114.114.114.114"

#解决默认路由优先级问题
#[root@node-1 network-scripts]# ip route
#default via 172.18.0.1 dev br-phy 
#default via 192.168.3.1 dev enp0s3 proto static metric 109 
#169.254.0.0/16 dev br-phy scope link metric 1011 
#172.18.0.0/24 dev br-phy proto kernel scope link src 172.18.0.11 
#192.168.3.0/24 dev enp0s3 proto kernel scope link src 192.168.3.111 metric 109

ip route del default via 172.18.0.1 dev br-phy
ip route add default via 172.18.0.1 dev br-phy metric 110
#

#解决ovs  system-id 由于是克隆的ovs导致总是一致的问题
#1.在每个节点安装ovs必须是在干净的环境
#2.临时修改system-id的值
#3.修改/etc/openvswitch/system-id.conf的值

#编辑租户网络网卡
TYPE="Ethernet"
PROXY_METHOD="none"
BROWSER_ONLY="no"


BOOTPROTO="static"
DEFROUTE="yes"
IPV4_FAILURE_FATAL="no"
IPV6INIT="yes"
IPV6_AUTOCONF="yes"
IPV6_DEFROUTE="yes"
IPV6_FAILURE_FATAL="no"
IPV6_ADDR_GEN_MODE="stable-privacy"
NAME="em3"
DEVICE="em3"
ONBOOT="yes"
IPADDR="172.168.2.11"
PREFIX="24"
GATEWAY="172.168.2.254"
DNS1="114.114.114.114"
IPV6_PRIVACY="no"

#关闭selinux
setenforce 0

echo 1 > /proc/sys/net/ipv4/ip_forward
vi /etc/sysctl.conf
net.ipv4.ip_forward = 1
sysctl -p

#10.10.20.2 ping  10.10.20.1 网关不通
#经排查是未设置ls2交换机上的路由器端口的mac地址

######开始创建分布式网关端口相关组件

#设置分布式路由端口
#ovn-nbctl lrp-add lr1 lr1-lslocal 00:00:00:00:00:05 10.10.40.1/24
#设置 Distributed Gateway Ports 的方式之一：
ovn-nbctl ha-chassis-group-add ha1
ovn-nbctl ha-chassis-group-add-chassis ha1 master 1
ovn-nbctl ha-chassis-group-add-chassis ha1 node1 2
#465efd10-c0e0-4966-be32-a20b213a2dbc 为 ha1 的uuid，可通过 ovn-nbctl ha-chassis-group-list 查看
ovn-nbctl set Logical_Router_Port  lr1-lslocal ha_chassis_group=465efd10-c0e0-4966-be32-a20b213a2dbc

#设置 Distributed Gateway Ports 的方式之二：
ovn-nbctl lrp-set-gateway-chassis lr1-lslocal master 1
ovn-nbctl lrp-set-gateway-chassis lr1-lslocal node1 2


# 开始
#添加逻辑交换机，用于连接外部网络
ovn-nbctl lrp-add lr1 lr1-lslocal 00:00:00:00:00:05 172.18.0.100/24
ovn-nbctl ls-add lslocal
ovn-nbctl lsp-add lslocal lslocal-lr1
ovn-nbctl lsp-set-type lslocal-lr1 router
ovn-nbctl lsp-set-addresses lslocal-lr1 00:00:00:00:00:05
ovn-nbctl lsp-set-options lslocal-lr1 router-port=lr1-lslocal

#问题网关节点设置的ip不通，是因为路由器的网关接口mac与外部逻辑交换机的路由器端口mac地址不符

ovn-nbctl lsp-add lslocal lslocal-localnet
ovn-nbctl lsp-set-addresses lslocal-localnet unknown
ovn-nbctl lsp-set-type lslocal-localnet localnet
ovn-nbctl lsp-set-options lslocal-localnet network_name=externalnet

#只有指定的gateway-chassis上需要执行下面命令，本实验中master和node1都被指定成gateway-chassis
#在master节点上执行
#ovs-vsctl add-br br-phy
#ovs-vsctl add-port br-phy ens8
ovs-vsctl set Open_vSwitch . external-ids:ovn-bridge-mappings=externalnet:br-phy
#ip link set dev br-phy up
#ip addr add 10.10.40.2/24 dev br-phy

#在node1节点上执行
#ovs-vsctl add-br br-phy
#ovs-vsctl add-port br-phy ens8
ovs-vsctl set Open_vSwitch . external-ids:ovn-bridge-mappings=externalnet:br-phy
#ip link set dev br-phy up
#ip addr add 10.10.40.3/24 dev br-phy

ovn-nbctl lrp-set-gateway-chassis lr1-lslocal 4e172a94-e755-4f3d-96bc-0e991eec4ec7 1
ovn-nbctl lrp-set-gateway-chassis lr1-lslocal 5e172a94-e755-4f3d-96bc-0e991eec4ec8 2

#在lr1上添加nat表项
ovn-nbctl -- --id=@nat create nat type="snat" logical_ip=10.10.10.0/24 \
external_ip=10.10.40.1 -- add logical_router lr1 nat @nat