这篇笔记写的时间很久，记录了Docker 的网络知识，概念从浅入深，对docker网络有一个大体介绍，对部分细节做了详细解析，但涉及的知识点太多，未能在一篇中全部罗列，后续逐步完善吧。
目录
参考
- 链接: docker官方文档
- 链接: Docker 入门到实战教程(三)镜像和容器
- 链接: docker入门(利用docker部署web应用)
- 链接: docker网络配置
- 链接: 对已有的docker容器增加新的端口映射
- 链接: 教你如何修改运行中的docker容器的端口映射的三种方式
- 链接: Docker 容器添加新端口映射的方法与步骤
- 链接: docker修改，新增，删减容器端口
- 链接: Docker6种网络配置详解，网络模式应该这么选
- 链接: 设置网卡为混杂模式
- 链接: Docker 网络模型之 macvlan 详解，图解，实验完整
- 链接: Docker Macvlan网络创建及通信配置
- 链接: Docker- 7.1、跨主机网络-macvlan
- 链接: Docker网络教程（四）macvlan 网络教程
- 链接: 图解几个与Linux网络虚拟化相关的虚拟网卡-VETH/MACVLAN/MACVTAP/IPVLAN
- 链接: IPVlan 详解
- 链接: IPVLAN Driver HOWTO
- 链接: IPVlan 源码探秘
- 链接: ipvlan内核代码流程
- 链接: 从 VLAN 到 IPVLAN: 聊聊虚拟网络设备及其在云原生中的应用
- 链接: Linux网络协议栈6–ipvlan
- 链接: ipvlan-l3s模式
环境
Centos 7.9
Ubuntu22.04
网络命令
1. 默认网络
[root@centos7-18 ~]# docker network ls
NETWORK ID NAME DRIVER SCOPE
2aff696bbb5b bridge bridge local
c96e896f937a host host local
e22393a5d569 none null local
2. 查看信息
2.1 查看bridge网络信息
// “bridge”是网络NAME,也可以使用NETWORK ID
[root@centos7-18 ~]# docker network inspect bridge
[
{
"Name": "bridge",
"Id": "1fd2800b01713fe51854bb3200198e24827a5c20bd27f7445ad444ea11aeaf42",
"Created": "2023-11-29T11:01:38.400193108+08:00",
"Scope": "local",
"Driver": "bridge",
"EnableIPv6": false,
"IPAM": {
"Driver": "default",
"Options": null,
"Config": [
{
"Subnet": "172.17.0.0/16",
"Gateway": "172.17.0.1"
}
]
},
"Internal": false,
"Attachable": false,
"Ingress": false,
"ConfigFrom": {
"Network": ""
},
"ConfigOnly": false,
"Containers": {
"a323d4e86a48f4e8336e010fb8c88ee4459b7adcebac374239476379a80702f5": {
"Name": "nginx3",
"EndpointID": "ba607fa2edeceb00ece9cf9249e3132f725e7ad31dc3e8a3584ca7979d5320e8",
"MacAddress": "02:42:ac:11:00:04",
"IPv4Address": "172.17.0.4/16",
"IPv6Address": ""
},
"ae8525b5a94e936911df8a1100b9988ddb9999c32979ab0b57100f4b1bdaf7e3": {
"Name": "nginx2",
"EndpointID": "01e34b1978e7947da271be80d5a9ceca8bf51e94e0cdd77c83db64bf09174b3e",
"MacAddress": "02:42:ac:11:00:03",
"IPv4Address": "172.17.0.3/16",
"IPv6Address": ""
},
"e93d68b0d6c484ec43fcbb1b04987eedcf9af6a3d5e269b153f72debac4b3f9f": {
"Name": "nginx1",
"EndpointID": "2c52f3dd5e15da7f0de7f61a3b103c390e378ffd58599cb22e12cd9d48a17bab",
"MacAddress": "02:42:ac:11:00:02",
"IPv4Address": "172.17.0.2/16",
"IPv6Address": ""
}
},
"Options": {
"com.docker.network.bridge.default_bridge": "true",
"com.docker.network.bridge.enable_icc": "true",
"com.docker.network.bridge.enable_ip_masquerade": "true",
"com.docker.network.bridge.host_binding_ipv4": "0.0.0.0",
"com.docker.network.bridge.name": "docker0",
"com.docker.network.driver.mtu": "1500"
},
"Labels": {}
}
]
2.2 查看host网络信息
// 查看host网络，网络ID是c96e896f937a
[root@centos7-18 ~]# docker network inspect c96e896f937a
[
{
"Name": "host",
"Id": "c96e896f937a0cf6c1d5e3d5b92bc92383736c347127c104700a2ee8016105f4",
"Created": "2023-11-15T13:43:42.187656452+08:00",
"Scope": "local",
"Driver": "host",
"EnableIPv6": false,
"IPAM": {
"Driver": "default",
"Options": null,
"Config": []
},
"Internal": false,
"Attachable": false,
"Ingress": false,
"ConfigFrom": {
"Network": ""
},
"ConfigOnly": false,
"Containers": {},
"Options": {},
"Labels": {}
}
]
2.3 查看none网络信息
// 查看none网络，网络ID是e22393a5d569
[root@centos7-18 ~]# docker network inspect e22393a5d569
[
{
"Name": "none",
"Id": "e22393a5d56907104e17faf6605b9ab5da2c8d7e149ff866340d55b4f46c3b4e",
"Created": "2023-11-15T13:43:42.175652527+08:00",
"Scope": "local",
"Driver": "null",
"EnableIPv6": false,
"IPAM": {
"Driver": "default",
"Options": null,
"Config": []
},
"Internal": false,
"Attachable": false,
"Ingress": false,
"ConfigFrom": {
"Network": ""
},
"ConfigOnly": false,
"Containers": {},
"Options": {},
"Labels": {}
}
]
2.4 查看bridge IPv6网络
官方文档：Enable IPv6 support
2.4.1 Centos 7.9启用IPv6
- 创建daemon.json配置文件
[root@centos7-18 ~]# vim /etc/docker/daemon.json
- 输入参数并保存
{
"ipv6": true,
"fixed-cidr-v6": "2001:db8:1::/64",
"experimental": true,
"ip6tables": true
}
~
~
~
~
:wq
- 重启docker服务
[root@centos7-18 ~]# systemctl restart docker
2.4.2 Ubuntu 22.04启用IPv6
- 查看docker配置文件路径
root@ubuntu22-25:~# ps -ef | grep daemon.json| grep -v grep
root 4450 1 0 16:05 ? 00:00:00 dockerd --group docker --exec-root=/run/snap.docker --data-root=/var/snap/docker/common/var-lib-docker --pidfile=/run/snap.docker/docker.pid --config-file=/var/snap/docker/2915/config/daemon.json
- 编辑daemon.json
root@ubuntu22-25:~# vim /var/snap/docker/2915/config/daemon.json
- 输入参数并保存
{
"log-level": "error", // 注意补全“，”
"ipv6": true,
"fixed-cidr-v6": "2001:db8:1::/64",
"experimental": true,
"ip6tables": true
}
~
~
~
~
:wq
- 重启docker服务
root@ubuntu22-25:~# systemctl restart snap.docker.dockerd.service
2.4.3 查看IPv6网络信息
- 查看bridge网络，参数EnableIPv6=true
root@ubuntu22-25:~# docker network inspect bridge
[
{
"Name": "bridge",
"Id": "6e0c422c8ae561ac8d90b825d653458f3cf26a1916d8b774032fdf0bbd2ceee2",
"Created": "2024-02-20T16:22:00.922887429+08:00",
"Scope": "local",
"Driver": "bridge",
"EnableIPv6": true,
"IPAM": {
"Driver": "default",
"Options": null,
"Config": [
{
"Subnet": "172.17.0.0/16",
"Gateway": "172.17.0.1"
},
{
"Subnet": "2001:db8:1::/64"
}
]
},
"Internal": false,
"Attachable": false,
"Ingress": false,
"ConfigFrom": {
"Network": ""
},
"ConfigOnly": false,
"Containers": {},
"Options": {
"com.docker.network.bridge.default_bridge": "true",
"com.docker.network.bridge.enable_icc": "true",
"com.docker.network.bridge.enable_ip_masquerade": "true",
"com.docker.network.bridge.host_binding_ipv4": "0.0.0.0",
"com.docker.network.bridge.name": "docker0",
"com.docker.network.driver.mtu": "1500"
},
"Labels": {}
}
]
- 查看docker0，已配置IPv6地址：2001:db8:1::1
root@ubuntu22-25:~# ip a show docker0
3: docker0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc noqueue state DOWN group default
link/ether 02:42:54:0d:28:91 brd ff:ff:ff:ff:ff:ff
inet 172.17.0.1/16 brd 172.17.255.255 scope global docker0
valid_lft forever preferred_lft forever
inet6 2001:db8:1::1/64 scope global
valid_lft forever preferred_lft forever
inet6 fe80::1/64 scope link
valid_lft forever preferred_lft forever
inet6 fe80::42:54ff:fe0d:2891/64 scope link
valid_lft forever preferred_lft forever
3. 创建网络
可以自定义网络，然后将容器添加到网络当中。默认是bridge网络。
注意bridge网络只能用于单宿主机
3.1 创建bridge网络
[root@centos7-18 ~]# docker network create -d bridge mynetwork
dc01376fa302f4f0f70a93dc12b1ea9079af427bca84f0f5202a81e3d96785c4
// 查看bridge类型网络
[root@centos7-18 ~]# docker network ls -f driver=bridge
NETWORK ID NAME DRIVER SCOPE
1fd2800b0171 bridge bridge local
dc01376fa302 mynetwork bridge local
[root@centos7-18 ~]#
[root@centos7-18 ~]# docker network inspect mynetwork
[
{
"Name": "mynetwork",
"Id": "dc01376fa302f4f0f70a93dc12b1ea9079af427bca84f0f5202a81e3d96785c4",
"Created": "2023-11-30T11:18:36.505915514+08:00",
"Scope": "local",
"Driver": "bridge",
"EnableIPv6": false,
"IPAM": {
"Driver": "default",
"Options": {},
"Config": [
{
"Subnet": "172.18.0.0/16",
"Gateway": "172.18.0.1"
}
]
},
"Internal": false,
"Attachable": false,
"Ingress": false,
"ConfigFrom": {
"Network": ""
},
"ConfigOnly": false,
"Containers": {},
"Options": {},
"Labels": {}
}
]
3.2 创建bridge网络(网络参数)
// C段网络
[root@centos7-18 ~]# docker network create -d bridge \
> --subnet 192.168.55.0/24 \
> --gateway 192.168.55.1 \
> mytestnetwork
42260d2f0250771e3b697d9d8e70c80474383be93f806a0af679109129271811
[root@centos7-18 ~]# docker network ls
NETWORK ID NAME DRIVER SCOPE
1fd2800b0171 bridge bridge local
c96e896f937a host host local
dc01376fa302 mynetwork bridge local
42260d2f0250 mytestnetwork bridge local
e22393a5d569 none null local
// 查看网络详情
[root@centos7-18 ~]# docker network inspect 42260d2f0250
[
{
"Name": "mytestnetwork",
"Id": "42260d2f0250771e3b697d9d8e70c80474383be93f806a0af679109129271811",
"Created": "2023-11-30T13:53:43.471517601+08:00",
"Scope": "local",
"Driver": "bridge",
"EnableIPv6": false,
"IPAM": {
"Driver": "default",
"Options": {},
"Config": [
{
"Subnet": "192.168.55.0/24",
"Gateway": "192.168.55.1"
}
]
},
"Internal": false,
"Attachable": false,
"Ingress": false,
"ConfigFrom": {
"Network": ""
},
"ConfigOnly": false,
"Containers": {},
"Options": {},
"Labels": {}
}
]
3.3 查看宿主机网络
// 添加了两个bridge网络，多出了两块虚拟网卡
[root@centos7-18 ~]# ifconfig
br-42260d2f0250: flags=4099<UP,BROADCAST,MULTICAST> mtu 1500
inet 192.168.55.1 netmask 255.255.255.0 broadcast 192.168.55.255
ether 02:42:f8:56:43:17 txqueuelen 0 (Ethernet)
RX packets 0 bytes 0 (0.0 B)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 8 bytes 656 (656.0 B)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
br-dc01376fa302: flags=4099<UP,BROADCAST,MULTICAST> mtu 1500
inet 172.18.0.1 netmask 255.255.0.0 broadcast 172.18.255.255
ether 02:42:f7:ce:1e:fe txqueuelen 0 (Ethernet)
RX packets 73 bytes 6688 (6.5 KiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 91 bytes 7703 (7.5 KiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
docker0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
inet 172.17.0.1 netmask 255.255.0.0 broadcast 172.17.255.255
inet6 fe80::42:fbff:fe6c:e068 prefixlen 64 scopeid 0x20<link>
ether 02:42:fb:6c:e0:68 txqueuelen 0 (Ethernet)
RX packets 73 bytes 6688 (6.5 KiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 91 bytes 7703 (7.5 KiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
4. 接入网络
4.1 运行容器加入网络
// 运行容器nginx4加入mytestnetwork网络
[root@centos7-18 ~]# docker run -itd --name nginx4 --network mytestnetwork nginx-sj /bin/bash
2e7b62e1f6fed35011eedbd0d88979b801ac80ab73842f17ae77e239b2336d83
[root@centos7-18 ~]# docker ps
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
2e7b62e1f6fe nginx-sj "/bin/bash" 5 seconds ago Up 4 seconds
// 进入nginx4容器,测试网络 nginx4
[root@centos7-18 ~]# docker exec -it nginx4 /bin/bash
root@2e7b62e1f6fe:/# ifconfig
eth0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
inet 192.168.55.2 netmask 255.255.255.0 broadcast 192.168.55.255
ether 02:42:c0:a8:37:02 txqueuelen 0 (Ethernet)
RX packets 8 bytes 656 (656.0 B)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 0 bytes 0 (0.0 B)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
lo: flags=73<UP,LOOPBACK,RUNNING> mtu 65536
inet 127.0.0.1 netmask 255.0.0.0
loop txqueuelen 1000 (Local Loopback)
RX packets 0 bytes 0 (0.0 B)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 0 bytes 0 (0.0 B)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
// 访问网关
root@2e7b62e1f6fe:/# ping 192.168.55.1
PING 192.168.55.1 (192.168.55.1): 56 data bytes
64 bytes from 192.168.55.1: icmp_seq=0 ttl=64 time=0.141 ms
64 bytes from 192.168.55.1: icmp_seq=1 ttl=64 time=0.089 ms
^C--- 192.168.55.1 ping statistics ---
2 packets transmitted, 2 packets received, 0% packet loss
round-trip min/avg/max/stddev = 0.089/0.115/0.141/0.026 ms
// 访问百度
root@2e7b62e1f6fe:/# ping www.baidu.com
PING www.a.shifen.com (110.242.68.3): 56 data bytes
64 bytes from 110.242.68.3: icmp_seq=0 ttl=127 time=14.147 ms
64 bytes from 110.242.68.3: icmp_seq=1 ttl=127 time=14.258 ms
^C--- www.a.shifen.com ping statistics ---
2 packets transmitted, 2 packets received, 0% packet loss
round-trip min/avg/max/stddev = 14.147/14.203/14.258/0.056 ms
root@2e7b62e1f6fe:/#
4.2 已创建的容器接入网络
// 查看当前运行容器
[root@centos7-18 ~]# docker ps
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
2e7b62e1f6fe nginx-sj "/bin/bash" 9 minutes ago Up 9 minutes nginx4
a323d4e86a48 nginx:latest "/docker-entrypoint.…" 24 hours ago Up 4 hours 0.0.0.0:1025->80/tcp, :::1025->80/tcp nginx3
ae8525b5a94e nginx-sj:latest "/bin/bash" 24 hours ago Up 4 hours 0.0.0.0:82->80/tcp, :::82->80/tcp nginx2
e93d68b0d6c4 nginx-sj:latest "/bin/bash" 24 hours ago Up 4 hours 0.0.0.0:81->80/tcp, :::81->80/tcp nginx1
// 将已运行的nginx2容器，接入mytestnetwork网络，指定IP为192.168.55.129
[root@centos7-18 ~]# docker network connect --ip 192.168.55.129 mytestnetwork nginx2
// 查看mytestnetwork网络
[root@centos7-18 ~]# docker network inspect mytestnetwork
// 两个容器均在网络中
[
{
"Name": "mytestnetwork",
"Id": "42260d2f0250771e3b697d9d8e70c80474383be93f806a0af679109129271811",
"Created": "2023-11-30T13:53:43.471517601+08:00",
"Scope": "local",
"Driver": "bridge",
"EnableIPv6": false,
"IPAM": {
"Driver": "default",
"Options": {},
"Config": [
{
"Subnet": "192.168.55.0/24",
"Gateway": "192.168.55.1"
}
]
},
"Internal": false,
"Attachable": false,
"Ingress": false,
"ConfigFrom": {
"Network": ""
},
"ConfigOnly": false,
"Containers": {
"2e7b62e1f6fed35011eedbd0d88979b801ac80ab73842f17ae77e239b2336d83": {
"Name": "nginx4",
"EndpointID": "c54320d7d31dbf1e130e7671e54b1bdb3be4b6338ed9e954d152a35bf3a5238e",
"MacAddress": "02:42:c0:a8:37:02",
"IPv4Address": "192.168.55.2/24",
"IPv6Address": ""
},
"ae8525b5a94e936911df8a1100b9988ddb9999c32979ab0b57100f4b1bdaf7e3": {
"Name": "nginx2",
"EndpointID": "0d116a6bc6ad3240741ecd48832ad7740c071d669467e55d1a9d61ad1fbe301c",
"MacAddress": "02:42:c0:a8:37:81",
"IPv4Address": "192.168.55.129/24",
"IPv6Address": ""
}
},
"Options": {},
"Labels": {}
}
]
// 进入nginx2容器，测试网络
[root@centos7-18 ~]# docker exec -it nginx2 /bin/bash
// 容器包含两块网卡
root@ae8525b5a94e:/# ifconfig
eth0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
inet 172.17.0.3 netmask 255.255.0.0 broadcast 172.17.255.255
ether 02:42:ac:11:00:03 txqueuelen 0 (Ethernet)
RX packets 8 bytes 656 (656.0 B)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 0 bytes 0 (0.0 B)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
eth1: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
inet 192.168.55.129 netmask 255.255.255.0 broadcast 192.168.55.255
ether 02:42:c0:a8:37:81 txqueuelen 0 (Ethernet)
RX packets 8 bytes 656 (656.0 B)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 0 bytes 0 (0.0 B)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
lo: flags=73<UP,LOOPBACK,RUNNING> mtu 65536
inet 127.0.0.1 netmask 255.0.0.0
loop txqueuelen 1000 (Local Loopback)
RX packets 0 bytes 0 (0.0 B)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 0 bytes 0 (0.0 B)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
root@ae8525b5a94e:/#
// 测试容器网络通信
root@ae8525b5a94e:/# ping 192.168.55.2
PING 192.168.55.2 (192.168.55.2): 56 data bytes
64 bytes from 192.168.55.2: icmp_seq=0 ttl=64 time=0.119 ms
64 bytes from 192.168.55.2: icmp_seq=1 ttl=64 time=0.189 ms
^C--- 192.168.55.2 ping statistics ---
2 packets transmitted, 2 packets received, 0% packet loss
round-trip min/avg/max/stddev = 0.119/0.154/0.189/0.035 ms
root@ae8525b5a94e:/# ping 172.17.0.2
PING 172.17.0.2 (172.17.0.2): 56 data bytes
64 bytes from 172.17.0.2: icmp_seq=0 ttl=64 time=0.202 ms
64 bytes from 172.17.0.2: icmp_seq=1 ttl=64 time=0.095 ms
^C--- 172.17.0.2 ping statistics ---
2 packets transmitted, 2 packets received, 0% packet loss
round-trip min/avg/max/stddev = 0.095/0.149/0.202/0.054 ms
root@ae8525b5a94e:/# ping www.baidu.com
PING www.a.shifen.com (110.242.68.3): 56 data bytes
64 bytes from 110.242.68.3: icmp_seq=0 ttl=127 time=13.927 ms
64 bytes from 110.242.68.3: icmp_seq=1 ttl=127 time=18.609 ms
^C--- www.a.shifen.com ping statistics ---
2 packets transmitted, 2 packets received, 0% packet loss
round-trip min/avg/max/stddev = 13.927/16.268/18.609/2.341 ms
root@ae8525b5a94e:/#
5. 断开网络
// 断开nginx2容器连接网络mytestnetwork
[root@centos7-18 ~]# docker network disconnect mytestnetwork nginx2
// 查看nginx2容器的IP，已没有192.168.55.129地址
[root@centos7-18 ~]# docker exec -it nginx2 ifconfig
eth0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
inet 172.17.0.3 netmask 255.255.0.0 broadcast 172.17.255.255
ether 02:42:ac:11:00:03 txqueuelen 0 (Ethernet)
RX packets 17 bytes 1348 (1.3 KiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 9 bytes 633 (633.0 B)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
lo: flags=73<UP,LOOPBACK,RUNNING> mtu 65536
inet 127.0.0.1 netmask 255.0.0.0
loop txqueuelen 1000 (Local Loopback)
RX packets 2 bytes 177 (177.0 B)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 2 bytes 177 (177.0 B)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
// 查看mytestnetwork网络，已没有容器nginx2的IP
[root@centos7-18 ~]# docker network inspect mytestnetwork
[
{
"Name": "mytestnetwork",
"Id": "42260d2f0250771e3b697d9d8e70c80474383be93f806a0af679109129271811",
"Created": "2023-11-30T13:53:43.471517601+08:00",
"Scope": "local",
"Driver": "bridge",
"EnableIPv6": false,
"IPAM": {
"Driver": "default",
"Options": {},
"Config": [
{
"Subnet": "192.168.55.0/24",
"Gateway": "192.168.55.1"
}
]
},
"Internal": false,
"Attachable": false,
"Ingress": false,
"ConfigFrom": {
"Network": ""
},
"ConfigOnly": false,
"Containers": {
"2e7b62e1f6fed35011eedbd0d88979b801ac80ab73842f17ae77e239b2336d83": {
"Name": "nginx4",
"EndpointID": "c54320d7d31dbf1e130e7671e54b1bdb3be4b6338ed9e954d152a35bf3a5238e",
"MacAddress": "02:42:c0:a8:37:02",
"IPv4Address": "192.168.55.2/24",
"IPv6Address": ""
}
},
"Options": {},
"Labels": {}
}
]
6. 删除网络
6.1 删除所有未使用的网络
// 查看当前网络
[root@centos7-18 ~]# docker network ls
NETWORK ID NAME DRIVER SCOPE
1fd2800b0171 bridge bridge local
c96e896f937a host host local
dc01376fa302 mynetwork bridge local
42260d2f0250 mytestnetwork bridge local
e22393a5d569 none null local
// 清除了未使用的mynetwork网络
[root@centos7-18 ~]# docker network prune
WARNING! This will remove all custom networks not used by at least one container.
Are you sure you want to continue? [y/N] y
Deleted Networks:
mynetwork
6.2 删除指定网络
// 删除mytestnetwork网络时，提示错误信息，网络有接入的容器
[root@centos7-18 ~]# docker network rm mytestnetwork
Error response from daemon: error while removing network: network mytestnetwork id 42260d2f0250771e3b697d9d8e70c80474383be93f806a0af679109129271811 has active endpoints
[root@centos7-18 ~]#
// 断开容器nginx4的网络连接，删除成功
[root@centos7-18 ~]# docker network disconnect mytestnetwork nginx4
[root@centos7-18 ~]# docker network rm mytestnetwork
mytestnetwork
// mytestnetwork网络已删除
[root@centos7-18 ~]# docker network ls
NETWORK ID NAME DRIVER SCOPE
1fd2800b0171 bridge bridge local
c96e896f937a host host local
e22393a5d569 none null local
[root@centos7-18 ~]#
网络类型
1.网络类型
docker网络类型分6种：
- none
- host
- bridge
- Overlay
- Macvlan
- Ipvlan
2. none
在None网络模式下，容器不会拥有任何网络接口，也没有IP地址，因此无法与外部网络通信。这种模式通常用于特殊场景，例如：需要禁用网络功能的容器。（摘自: MCNU云原生文章）
- 参数 --network=none
[root@centos7-18 ~]# docker run -itd --name=redis-sj --network=none redis
f6f06e0f2f740b62e778a246266f9f6c1612590ccb4c054d68bbe6c8c6f40df8
- ports 为空
[root@centos7-18 ~]# docker ps
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
f6f06e0f2f74 redis "docker-entrypoint.s…" 7 seconds ago Up 6 seconds redis-sj
- 容器详细信息网络为none
[root@centos7-18 ~]# docker inspect f6f06e0f2f74
3. host
在Host网络模式下，容器将直接使用宿主机的网络接口，这样容器就可以像宿主机一样，使用主机的IP地址和端口。因此，这种模式下的容器与主机之间没有网络隔离，但它们与其他容器之间则是隔离的。
与其他Docker网络模式相比，Host网络模式的主要特点包括：
- 简单易用：不需要对容器进行网络配置，与宿主机直接共享网络。
- 性能高：因为容器与宿主机共享网络，容器之间的通信无需经过网络层，可以获得更高的网络性能。
- 安全性较低：容器与宿主机共享网络，容器之间的网络隔离会降低，容器中的应用程序可以直接访问宿主机上的网络接口。
Host网络模式适用于一些特定场景，例如需要访问宿主机上的网络接口、需要最大化网络性能、需要简单易用的网络设置等。（摘自: MCNU云原生文章）
3.1 查看宿主机服务端口
[root@centos7-18 ~]# netstat -ntlp
Active Internet connections (only servers)
Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name
tcp 0 0 0.0.0.0:111 0.0.0.0:* LISTEN 570/rpcbind
tcp 0 0 192.168.122.1:53 0.0.0.0:* LISTEN 1380/dnsmasq
tcp 0 0 0.0.0.0:22 0.0.0.0:* LISTEN 891/sshd
tcp 0 0 127.0.0.1:631 0.0.0.0:* LISTEN 893/cupsd
tcp6 0 0 :::33060 :::* LISTEN 1337/mysqld
tcp6 0 0 :::3306 :::* LISTEN 1337/mysqld
tcp6 0 0 :::111 :::* LISTEN 570/rpcbind
tcp6 0 0 :::22 :::* LISTEN 891/sshd
tcp6 0 0 ::1:631 :::* LISTEN 893/cupsd
3.2启动host容器
[root@centos7-18 ~]# docker run -itd --name nginx1 --network host nginx:latest
435c79b1cdcc4d915eb4927298fdb294bbb1635c9fe2f3888aee6d431575a319
[root@centos7-18 ~]# docker ps
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
435c79b1cdcc nginx:latest "/docker-entrypoint.…" 13 seconds ago Up 12 seconds nginx1
[root@centos7-18 ~]#
注意，启动后ports为空
3.3查看host网络
存在启动的容器，IP为空
[root@centos7-18 ~]# docker network ls
NETWORK ID NAME DRIVER SCOPE
198e731b1647 bridge bridge local
c96e896f937a host host local
e22393a5d569 none null local
[root@centos7-18 ~]# docker network inspect c96e896f937a
[
{
"Name": "host",
"Id": "c96e896f937a0cf6c1d5e3d5b92bc92383736c347127c104700a2ee8016105f4",
"Created": "2023-11-15T13:43:42.187656452+08:00",
"Scope": "local",
"Driver": "host",
"EnableIPv6": false,
"IPAM": {
"Driver": "default",
"Options": null,
"Config": []
},
"Internal": false,
"Attachable": false,
"Ingress": false,
"ConfigFrom": {
"Network": ""
},
"ConfigOnly": false,
"Containers": {
"435c79b1cdcc4d915eb4927298fdb294bbb1635c9fe2f3888aee6d431575a319": {
"Name": "nginx1",
"EndpointID": "e2f99a5e4c17e89d6627c10d2dab18c6329b0b302bed5ca8c8f1210630839547",
"MacAddress": "",
"IPv4Address": "",
"IPv6Address": ""
}
},
"Options": {},
"Labels": {}
}
]
3.4再次查看宿主机服务端口
nginx进程监听80端口
[root@centos7-18 ~]# netstat -nltp
Active Internet connections (only servers)
Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name
tcp 0 0 0.0.0.0:111 0.0.0.0:* LISTEN 570/rpcbind
tcp 0 0 0.0.0.0:80 0.0.0.0:* LISTEN 6520/nginx: master
tcp 0 0 192.168.122.1:53 0.0.0.0:* LISTEN 1380/dnsmasq
tcp 0 0 0.0.0.0:22 0.0.0.0:* LISTEN 891/sshd
tcp 0 0 127.0.0.1:631 0.0.0.0:* LISTEN 893/cupsd
tcp6 0 0 :::33060 :::* LISTEN 1337/mysqld
tcp6 0 0 :::3306 :::* LISTEN 1337/mysqld
tcp6 0 0 :::111 :::* LISTEN 570/rpcbind
tcp6 0 0 :::80 :::* LISTEN 6520/nginx: master
tcp6 0 0 :::22 :::* LISTEN 891/sshd
tcp6 0 0 ::1:631 :::* LISTEN 893/cupsd
[root@centos7-18 ~]#
访问宿主服务80端口
3.5查看容器IP
- 查看容器IP，类型为host，IP为空
[root@centos7-18 ~]# docker inspect 435c79b1cdcc
- 进入容器，发现容器内IP与宿主服务器显示一致
[root@centos7-18 ~]# docker exec -it 435c79b1cdcc
# ifconfig
docker0: flags=4099<UP,BROADCAST,MULTICAST> mtu 1500
inet 172.17.0.1 netmask 255.255.0.0 broadcast 172.17.255.255
ether 02:42:7d:9b:4c:08 txqueuelen 0 (Ethernet)
RX packets 0 bytes 0 (0.0 B)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 0 bytes 0 (0.0 B)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
enp0s5: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
inet 10.211.55.18 netmask 255.255.255.0 broadcast 10.211.55.255
inet6 fe80::21c:42ff:fe60:87b2 prefixlen 64 scopeid 0x20<link>
inet6 fdb2:2c26:f4e4:0:21c:42ff:fe60:87b2 prefixlen 64 scopeid 0x0<global>
ether 00:1c:42:60:87:b2 txqueuelen 1000 (Ethernet)
RX packets 2956 bytes 343345 (335.2 KiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 1590 bytes 311215 (303.9 KiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
lo: flags=73<UP,LOOPBACK,RUNNING> mtu 65536
inet 127.0.0.1 netmask 255.0.0.0
inet6 ::1 prefixlen 128 scopeid 0x10<host>
loop txqueuelen 1000 (Local Loopback)
RX packets 4 bytes 416 (416.0 B)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 4 bytes 416 (416.0 B)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
virbr0: flags=4099<UP,BROADCAST,MULTICAST> mtu 1500
inet 192.168.122.1 netmask 255.255.255.0 broadcast 192.168.122.255
ether 52:54:00:9f:13:77 txqueuelen 1000 (Ethernet)
RX packets 0 bytes 0 (0.0 B)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 0 bytes 0 (0.0 B)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
3.6 测试两个host网络容器端口冲突
- 启动两个host容器
[root@centos7-18 ~]# docker images
REPOSITORY TAG IMAGE ID CREATED SIZE
nginx latest 605c77e624dd 23 months ago 141MB
[root@centos7-18 ~]# docker run -itd --name nginx1 --network host nginx:latest /bin/bash
[root@centos7-18 ~]# docker run -itd --name nginx2 --network host nginx:latest /bin/bash
[root@centos7-18 ~]# docker ps
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
f6ed2fa31616 nginx:latest "/docker-entrypoint.…" 25 minutes ago Up 25 minutes nginx2
92176206c4d1 nginx:latest "/docker-entrypoint.…" About an hour ago Up 25 minutes nginx1
[root@centos7-18 ~]#
- 修改容器nginx1的/usr/share/nginx/html/index.html文件并启动nginx
root@centos7-18:/# /etc/init.d/nginx start
Restarting nginx: nginx2023/11/28 04:24:28 [notice] 38#38: signal 15 (SIGTERM) received from 116, exiting
2023/11/28 04:24:28 [notice] 90#90: exiting
2023/11/28 04:24:28 [notice] 90#90: exit
2023/11/28 04:24:28 [notice] 91#91: exiting
2023/11/28 04:24:28 [notice] 91#91: exit
2023/11/28 04:24:28 [notice] 38#38: signal 17 (SIGCHLD) received from 91
2023/11/28 04:24:28 [notice] 38#38: worker process 91 exited with code 0
2023/11/28 04:24:28 [notice] 38#38: worker process 90 exited with code 0
2023/11/28 04:24:28 [notice] 38#38: exit
2023/11/28 04:24:28 [notice] 118#118: using the "epoll" event method
2023/11/28 04:24:28 [notice] 118#118: nginx/1.21.5
2023/11/28 04:24:28 [notice] 118#118: built by gcc 10.2.1 20210110 (Debian 10.2.1-6)
2023/11/28 04:24:28 [notice] 118#118: OS: Linux 3.10.0-1160.6.1.el7.x86_64
2023/11/28 04:24:28 [notice] 118#118: getrlimit(RLIMIT_NOFILE): 1048576:1048576
2023/11/28 04:24:28 [notice] 119#119: start worker processes
2023/11/28 04:24:28 [notice] 119#119: start worker process 120
2023/11/28 04:24:28 [notice] 119#119: start worker process 121
root@centos7-18:/#
- 查看宿主机端口并访问
[root@centos7-18 ~]# netstat -ntlp
Active Internet connections (only servers)
Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name
tcp 0 0 0.0.0.0:111 0.0.0.0:* LISTEN 570/rpcbind
tcp 0 0 0.0.0.0:80 0.0.0.0:* LISTEN 19697/nginx: master
tcp 0 0 192.168.122.1:53 0.0.0.0:* LISTEN 1380/dnsmasq
tcp 0 0 0.0.0.0:22 0.0.0.0:* LISTEN 891/sshd
tcp 0 0 127.0.0.1:631 0.0.0.0:* LISTEN 893/cupsd
tcp6 0 0 :::22 :::* LISTEN 891/sshd
tcp6 0 0 ::1:631 :::* LISTEN 893/cupsd
[root@centos7-18 ~]#
- 修改容器nginx2的/usr/share/nginx/html/index.html文件并启动nginx
root@centos7-18:/etc/init.d# /etc/init.d/nginx start
2023/11/28 04:38:06 [emerg] 22#22: bind() to 0.0.0.0:80 failed (98: Address already in use)
nginx: [emerg] bind() to 0.0.0.0:80 failed (98: Address already in use)
2023/11/28 04:38:06 [notice] 22#22: try again to bind() after 500ms
2023/11/28 04:38:06 [emerg] 22#22: bind() to 0.0.0.0:80 failed (98: Address already in use)
nginx: [emerg] bind() to 0.0.0.0:80 failed (98: Address already in use)
2023/11/28 04:38:06 [notice] 22#22: try again to bind() after 500ms
2023/11/28 04:38:06 [emerg] 22#22: bind() to 0.0.0.0:80 failed (98: Address already in use)
nginx: [emerg] bind() to 0.0.0.0:80 failed (98: Address already in use)
2023/11/28 04:38:06 [notice] 22#22: try again to bind() after 500ms
2023/11/28 04:38:06 [emerg] 22#22: bind() to 0.0.0.0:80 failed (98: Address already in use)
nginx: [emerg] bind() to 0.0.0.0:80 failed (98: Address already in use)
2023/11/28 04:38:06 [notice] 22#22: try again to bind() after 500ms
2023/11/28 04:38:06 [emerg] 22#22: bind() to 0.0.0.0:80 failed (98: Address already in use)
nginx: [emerg] bind() to 0.0.0.0:80 failed (98: Address already in use)
2023/11/28 04:38:06 [notice] 22#22: try again to bind() after 500ms
2023/11/28 04:38:06 [emerg] 22#22: still could not bind()
nginx: [emerg] still could not bind()
root@centos7-18:/etc/init.d#
提示宿主机端口被占用
3.7 补充
如果启动host容器时添加参数P或p，会提示警告信息，并且参数无效
[root@centos7-18 ~]# docker run -itd --name nginx3 --network host -p 8080:80 nginx:latest /bin/bash
WARNING: Published ports are discarded when using host network mode
496fd3671501c028127e604f16c5fae736ddf23d0182fb031c4067d526e41e45
[root@centos7-18 ~]#
[root@centos7-18 ~]# docker inspect nginx3
4. bridge
Docker Bridge网络模式是Docker默认的网络模式，它将容器连接到Docker宿主机的虚拟网络中，并为每个容器分配一个IP地址。容器之间可以通过IP地址直接通信，也可以通过容器名称进行通信。
在Bridge网络模式下，Docker会为每个容器创建一个虚拟网桥，容器之间的通信都是通过这个虚拟网桥进行的。Docker宿主机上的所有容器都连接到这个虚拟网桥上，而Docker宿主机本身也是通过一个特殊的容器（名为docker0）连接到这个虚拟网桥上的。
当Docker创建容器时，它会自动为容器分配一个IP地址，并将该地址添加到虚拟网桥的配置中。这样，在容器之间通信时，就可以使用这个IP地址进行通信了。
需要注意的是，Bridge网络模式下的容器只能和同一台主机上的容器通信，无法和其他主机上的容器通信。如果需要实现跨主机通信，可以使用Docker的Overlay网络模式。（摘自: MCNU云原生文章）
4.1 查看宿主机网络
- 启动docker服务后，自动创建docker0
[root@centos7-18 ~]# systemctl start docker
[root@centos7-18 ~]# ifconfig
docker0: flags=4099<UP,BROADCAST,MULTICAST> mtu 1500
inet 172.17.0.1 netmask 255.255.0.0 broadcast 172.17.255.255
inet6 fe80::42:7dff:fe9b:4c08 prefixlen 64 scopeid 0x20<link>
ether 02:42:7d:9b:4c:08 txqueuelen 0 (Ethernet)
RX packets 0 bytes 0 (0.0 B)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 3 bytes 266 (266.0 B)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
enp0s5: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
inet 10.211.55.18 netmask 255.255.255.0 broadcast 10.211.55.255
inet6 fe80::21c:42ff:fe60:87b2 prefixlen 64 scopeid 0x20<link>
inet6 fdb2:2c26:f4e4:0:21c:42ff:fe60:87b2 prefixlen 64 scopeid 0x0<global>
ether 00:1c:42:60:87:b2 txqueuelen 1000 (Ethernet)
RX packets 18880 bytes 2256560 (2.1 MiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 9898 bytes 1907225 (1.8 MiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
lo: flags=73<UP,LOOPBACK,RUNNING> mtu 65536
inet 127.0.0.1 netmask 255.0.0.0
inet6 ::1 prefixlen 128 scopeid 0x10<host>
loop txqueuelen 1000 (Local Loopback)
RX packets 25 bytes 1840 (1.7 KiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 25 bytes 1840 (1.7 KiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
4.2 运行两个容器，查看网络
- 参数-p： 指定容器的映射端口
[root@centos7-18 ~]# docker ps -a
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
[root@centos7-18 ~]# docker run -itd --name nginx1 -p 81:80 nginx-sj:latest /bin/bash
e93d68b0d6c484ec43fcbb1b04987eedcf9af6a3d5e269b153f72debac4b3f9f
[root@centos7-18 ~]# docker run -itd --name nginx2 -p 82:80 nginx-sj:latest /bin/bash
ae8525b5a94e936911df8a1100b9988ddb9999c32979ab0b57100f4b1bdaf7e3
[root@centos7-18 ~]# docker ps
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
ae8525b5a94e nginx-sj:latest "/bin/bash" 7 seconds ago Up 6 seconds 0.0.0.0:82->80/tcp, :::82->80/tcp nginx2
e93d68b0d6c4 nginx-sj:latest "/bin/bash" 40 seconds ago Up 39 seconds 0.0.0.0:81->80/tcp, :::81->80/tcp nginx1
- 参数-P： 系统自动创建映射端口
[root@centos7-18 ~]# docker run -itd --name nginx3 -P nginx:latest /bin/bash
a323d4e86a48f4e8336e010fb8c88ee4459b7adcebac374239476379a80702f5
- 容器服务端口映射结果
[root@centos7-18 ~]# docker ps
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
a323d4e86a48 nginx:latest "/docker-entrypoint.…" 5 seconds ago Up 4 seconds 0.0.0.0:1024->80/tcp, :::1024->80/tcp nginx3
ae8525b5a94e nginx-sj:latest "/bin/bash" 5 minutes ago Up 5 minutes 0.0.0.0:82->80/tcp, :::82->80/tcp nginx2
e93d68b0d6c4 nginx-sj:latest "/bin/bash" 5 minutes ago Up 5 minutes 0.0.0.0:81->80/tcp, :::81->80/tcp nginx1
[root@centos7-18 ~]#
4.3 查看容器网络
- 容器 nginx3 IP：172.17.0.4
[root@centos7-18 ~]# docker inspect a323d4e86a48 | grep Networks -C16
"HostIp": "::",
"HostPort": "1024"
}
]
},
"SandboxKey": "/var/run/docker/netns/287f51467c5a",
"SecondaryIPAddresses": null,
"SecondaryIPv6Addresses": null,
"EndpointID": "89aa513a8ca93effce0aaf8a3bf7299c9835294545995f54537d0b78be0b6fdb",
"Gateway": "172.17.0.1",
"GlobalIPv6Address": "",
"GlobalIPv6PrefixLen": 0,
"IPAddress": "172.17.0.4",
"IPPrefixLen": 16,
"IPv6Gateway": "",
"MacAddress": "02:42:ac:11:00:04",
"Networks": {
"bridge": {
"IPAMConfig": null,
"Links": null,
"Aliases": null,
"NetworkID": "1fd2800b01713fe51854bb3200198e24827a5c20bd27f7445ad444ea11aeaf42",
"EndpointID": "89aa513a8ca93effce0aaf8a3bf7299c9835294545995f54537d0b78be0b6fdb",
"Gateway": "172.17.0.1",
"IPAddress": "172.17.0.4",
"IPPrefixLen": 16,
"IPv6Gateway": "",
"GlobalIPv6Address": "",
"GlobalIPv6PrefixLen": 0,
"MacAddress": "02:42:ac:11:00:04",
"DriverOpts": null
}
}
[root@centos7-18 ~]#
- 容器 nginx2 IP：172.17.0.3
- 容器 nginx1 IP：172.17.0.2
[root@centos7-18 ~]# docker exec -it nginx2 ifconfig
eth0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
inet 172.17.0.3 netmask 255.255.0.0 broadcast 172.17.255.255
ether 02:42:ac:11:00:03 txqueuelen 0 (Ethernet)
RX packets 8 bytes 656 (656.0 B)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 0 bytes 0 (0.0 B)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
lo: flags=73<UP,LOOPBACK,RUNNING> mtu 65536
inet 127.0.0.1 netmask 255.0.0.0
loop txqueuelen 1000 (Local Loopback)
RX packets 0 bytes 0 (0.0 B)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 0 bytes 0 (0.0 B)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
[root@centos7-18 ~]# docker exec -it nginx1 ifconfig
eth0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
inet 172.17.0.2 netmask 255.255.0.0 broadcast 172.17.255.255
ether 02:42:ac:11:00:02 txqueuelen 0 (Ethernet)
RX packets 21 bytes 2464 (2.4 KiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 0 bytes 0 (0.0 B)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
lo: flags=73<UP,LOOPBACK,RUNNING> mtu 65536
inet 127.0.0.1 netmask 255.0.0.0
loop txqueuelen 1000 (Local Loopback)
RX packets 0 bytes 0 (0.0 B)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 0 bytes 0 (0.0 B)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
- 测试容器间网络连接
[root@centos7-18 ~]# docker exec -it nginx1 /bin/bash
//测试网关docker0
root@e93d68b0d6c4:/# ping 172.17.0.1
PING 172.17.0.1 (172.17.0.1): 56 data bytes
64 bytes from 172.17.0.1: icmp_seq=0 ttl=64 time=0.098 ms
64 bytes from 172.17.0.1: icmp_seq=1 ttl=64 time=0.091 ms
64 bytes from 172.17.0.1: icmp_seq=2 ttl=64 time=0.091 ms
^C--- 172.17.0.1 ping statistics ---
3 packets transmitted, 3 packets received, 0% packet loss
round-trip min/avg/max/stddev = 0.091/0.093/0.098/0.000 ms
//测试容器nginx2
root@e93d68b0d6c4:/# ping 172.17.0.3
PING 172.17.0.3 (172.17.0.3): 56 data bytes
64 bytes from 172.17.0.3: icmp_seq=0 ttl=64 time=0.160 ms
64 bytes from 172.17.0.3: icmp_seq=1 ttl=64 time=0.165 ms
64 bytes from 172.17.0.3: icmp_seq=2 ttl=64 time=0.097 ms
^C--- 172.17.0.3 ping statistics ---
3 packets transmitted, 3 packets received, 0% packet loss
round-trip min/avg/max/stddev = 0.097/0.141/0.165/0.031 ms
//测试容器nginx3
root@e93d68b0d6c4:/# ping 172.17.0.4
PING 172.17.0.4 (172.17.0.4): 56 data bytes
64 bytes from 172.17.0.4: icmp_seq=0 ttl=64 time=0.168 ms
64 bytes from 172.17.0.4: icmp_seq=1 ttl=64 time=0.100 ms
^C--- 172.17.0.4 ping statistics ---
2 packets transmitted, 2 packets received, 0% packet loss
round-trip min/avg/max/stddev = 0.100/0.134/0.168/0.034 ms
root@e93d68b0d6c4:/#
- 测试外部访问服务
// 进入容器nginx1
[root@centos7-18 ~]# docker exec -it nginx1 /bin/bash
// 启动nginx服务
root@e93d68b0d6c4:/# /etc/init.d/nginx start
2023/11/29 07:05:50 [notice] 29#29: using the "epoll" event method
2023/11/29 07:05:50 [notice] 29#29: nginx/1.21.5
2023/11/29 07:05:50 [notice] 29#29: built by gcc 10.2.1 20210110 (Debian 10.2.1-6)
2023/11/29 07:05:50 [notice] 29#29: OS: Linux 3.10.0-1160.6.1.el7.x86_64
2023/11/29 07:05:50 [notice] 29#29: getrlimit(RLIMIT_NOFILE): 1048576:1048576
2023/11/29 07:05:50 [notice] 30#30: start worker processes
2023/11/29 07:05:50 [notice] 30#30: start worker process 31
2023/11/29 07:05:50 [notice] 30#30: start worker process 32
root@e93d68b0d6c4:/# exit
exit
// 查看容器nginx1内的服务进程
[root@centos7-18 ~]# docker top nginx1
UID PID PPID C STIME TTY TIME CMD
root 4424 4404 0 14:34 pts/0 00:00:00 /bin/bash
root 6967 4404 0 15:05 ? 00:00:00 nginx: master process /usr/sbin/nginx -c /etc/nginx/nginx.conf
101 6968 6967 0 15:05 ? 00:00:00 nginx: worker process
101 6969 6967 0 15:05 ? 00:00:00 nginx: worker process
[root@centos7-18 ~]#
// 显示容器映射端口
[root@centos7-18 ~]# docker ps
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
a323d4e86a48 nginx:latest "/docker-entrypoint.…" 35 minutes ago Up 35 minutes 0.0.0.0:1024->80/tcp, :::1024->80/tcp nginx3
ae8525b5a94e nginx-sj:latest "/bin/bash" 40 minutes ago Up 40 minutes 0.0.0.0:82->80/tcp, :::82->80/tcp nginx2
e93d68b0d6c4 nginx-sj:latest "/bin/bash" 40 minutes ago Up 40 minutes 0.0.0.0:81->80/tcp, :::81->80/tcp nginx1
// 查看宿主机监听端口
[root@centos7-18 ~]# netstat -ntlp
Active Internet connections (only servers)
Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name
tcp 0 0 0.0.0.0:111 0.0.0.0:* LISTEN 578/rpcbind
tcp 0 0 0.0.0.0:81 0.0.0.0:* LISTEN 4382/docker-proxy
tcp 0 0 0.0.0.0:82 0.0.0.0:* LISTEN 4504/docker-proxy
tcp 0 0 192.168.122.1:53 0.0.0.0:* LISTEN 1434/dnsmasq
tcp 0 0 0.0.0.0:22 0.0.0.0:* LISTEN 889/sshd
tcp 0 0 127.0.0.1:631 0.0.0.0:* LISTEN 888/cupsd
tcp 0 0 0.0.0.0:1024 0.0.0.0:* LISTEN 5145/docker-proxy
tcp6 0 0 :::111 :::* LISTEN 578/rpcbind
tcp6 0 0 :::81 :::* LISTEN 4388/docker-proxy
tcp6 0 0 :::82 :::* LISTEN 4509/docker-proxy
tcp6 0 0 :::22 :::* LISTEN 889/sshd
tcp6 0 0 ::1:631 :::* LISTEN 888/cupsd
tcp6 0 0 :::1024 :::* LISTEN 5150/docker-proxy
tcp6 0 0 :::33060 :::* LISTEN 1356/mysqld
tcp6 0 0 :::3306 :::* LISTEN 1356/mysqld
[root@centos7-18 ~]#
// 测试容器nginx1的nginx服务（监听端口81），测试成功
[root@centos7-18 ~]# curl http://10.211.55.18:81
<!DOCTYPE html>
<html>
<head>
<title>Welcome to nginx1!</title>
<style>
html { color-scheme: light dark; }
body { width: 35em; margin: 0 auto;
font-family: Tahoma, Verdana, Arial, sans-serif; }
</style>
</head>
<body>
<h1>Welcome to nginx1!</h1>
<p>If you see this page, the nginx web server is successfully installed and
working. Further configuration is required.</p>
<p>For online documentation and support please refer to
<a href="http://nginx.org/">nginx.org</a>.<br/>
Commercial support is available at
<a href="http://nginx.com/">nginx.com</a>.</p>
<p><em>Thank you for using nginx.</em></p>
</body>
</html>
// 测试容器nginx2和nginx3的监听端口，测试失败，原因nginx服务未启动
[root@centos7-18 ~]# curl http://10.211.55.18:82
curl: (7) Failed connect to 10.211.55.18:82; 拒绝连接
[root@centos7-18 ~]# curl http://10.211.55.18:1024
curl: (7) Failed connect to 10.211.55.18:1024; 拒绝连接
[root@centos7-18 ~]#
5. Overlay
Overlay网络模式用于连接跨主机的容器。在这种模式下，Docker会创建一个虚拟网络，所有连接到这个虚拟网络的容器都可以相互通信，无论这些容器运行在哪个主机上。Overlay网络模式需要使用Docker Swarm集群管理工具来管理多个Docker主机之间的通信。（摘自: MCNU云原生文章）
5.1 初始化Swarm集群
- 创建Overlay网络需要先创建Swarm。
[root@centos7-18 ~]# docker network create --driver overlay mynetwork
Error response from daemon: This node is not a swarm manager. Use "docker swarm init" or "docker swarm join" to connect this node to swarm and try again.
-
初始化Swarm需要开发的端口
- TCP 端口 2377 ，集群管理通讯
- TCP 与 UDP 端口 7946 ，节点间通讯
- UDP 端口 4789，overlay 网络通讯
-
宿主机centos7-10创建Swarm
[root@centos7-10 ~]# docker swarm init
Swarm initialized: current node (w92t66jii4jflz1vb0kotr0df) is now a manager.
To add a worker to this swarm, run the following command:
docker swarm join --token SWMTKN-1-4lkmzrtm2qo824v8zpndp8jd861gnahpimtvdty14wlzbj7rnb-3fqp3ituzhzltpytadx90vcea 10.211.55.10:2377
To add a manager to this swarm, run 'docker swarm join-token manager' and follow the instructions.
[root@centos7-10 ~]#
- 宿主机centos7-10 docker网络中自动创建 ingress 和 docker_gwbridge
[root@centos7-10 ~]# docker network ls
NETWORK ID NAME DRIVER SCOPE
b11c2d3b254b bridge bridge local
333348eb8ff4 docker_gwbridge bridge local
65a57293726b host host local
9j40lm9s5dte ingress overlay swarm
2db17c1087bf none null local
[root@centos7-10 ~]#
- 宿主机centos7-18和centos7-22 加入Swarm
[root@centos7-18 ~]# docker swarm join --token SWMTKN-1-4lkmzrtm2qo824v8zpndp8jd861gnahpimtvdty14wlzbj7rnb-3fqp3ituzhzltpytadx90vcea 10.211.55.10:2377
This node joined a swarm as a worker.
[root@centos7-22 ~]# docker swarm join --token SWMTKN-1-4lkmzrtm2qo824v8zpndp8jd861gnahpimtvdty14wlzbj7rnb-3fqp3ituzhzltpytadx90vcea 10.211.55.10:2377
This node joined a swarm as a worker.
- 在Swarm manager节点（宿主机centos7-10）上验证节点情况
[root@centos7-10 ~]# docker node ls
ID HOSTNAME STATUS AVAILABILITY MANAGER STATUS ENGINE VERSION
w92t66jii4jflz1vb0kotr0df * centos7-10.localdomain Ready Active Leader 24.0.7
9p6kone8vdwkdne55svrj6k5n centos7-18.localdomain Ready Active 24.0.7
x5y16qjbxuxv322to4hbiymik centos7-22 Ready Active 24.0.7
5.2 创建自定义Overlay网络
5.2.1 创建Swarm集群使用的Overlay网络
- 创建overlay网络ol_network
[root@centos7-10 ~]# docker network create -d overlay --subnet 172.16.0.0/16 --gateway 172.16.0.1 ol_network
9u6c1vlfy4kvli7ihka8f5mjf
[root@centos7-10 ~]# docker network ls
NETWORK ID NAME DRIVER SCOPE
b11c2d3b254b bridge bridge local
333348eb8ff4 docker_gwbridge bridge local
65a57293726b host host local
9j40lm9s5dte ingress overlay swarm
2db17c1087bf none null local
9u6c1vlfy4kv ol_network overlay swarm
- 创建本地volume：my-volume
- 挂载点：/var/lib/docker/volumes/my-volume/_data
[root@centos7-10 ~]# docker volume inspect my-volume
[
{
"CreatedAt": "2024-02-26T17:54:28+08:00",
"Driver": "local",
"Labels": null,
"Mountpoint": "/var/lib/docker/volumes/my-volume/_data",
"Name": "my-volume",
"Options": null,
"Scope": "local"
}
]
- 使用ol_network和my-volume存储卷，创建swarm服务my-nginx
- 网络使用ol_network
- my-volume映射到容器的/usr/share/nginx/html/目录（nginx的服务目录）
- 发布端口是9090
- 服务副本数是3
- 镜像是3台宿主机本地的nginx-sj:2024022601
[root@centos7-10 ~]# docker service create --name my-nginx --network ol_network --mount type=volume,src=my-volume,dst=/usr/share/nginx/html/ -p 9090:80 --replicas=3 nginx-sj:2024022601
image nginx-sj:2024022601 could not be accessed on a registry to record
its digest. Each node will access nginx-sj:2024022601 independently,
possibly leading to different nodes running different
versions of the image.
7rfsp2zazavrnckuw074dv8dl
overall progress: 3 out of 3 tasks
1/3: running [==================================================>]
2/3: running [==================================================>]
3/3: running [==================================================>]
verify: Service converged
- 分别在三台宿主机上创建三个文件：index1.html
[root@centos7-10 ~]# echo "Host IP is 10.211.55.10" >> /var/lib/docker/volumes/my-volume/_data/index1.html
[root@centos7-22 ~]# echo "Host IP is 10.211.55.22" >> /var/lib/docker/volumes/my-volume/_data/index1.html
[root@centos7-18 ~]# echo "Host IP is 10.211.55.18" >> /var/lib/docker/volumes/my-volume/_data/index1.html
- 查看服务启动情况
[root@centos7-10 ~]# docker service ls
ID NAME MODE REPLICAS IMAGE PORTS
7rfsp2zazavr my-nginx replicated 3/3 nginx-sj:2024022601 *:9090->80/tcp
[root@centos7-10 ~]# docker service ps 7rfsp2zazavr
ID NAME IMAGE NODE DESIRED STATE CURRENT STATE ERROR PORTS
elw6yuwr0yid my-nginx.1 nginx-sj:2024022601 centos7-10.localdomain Running Running 16 minutes ago
aqvtll4euet2 my-nginx.2 nginx-sj:2024022601 centos7-22 Running Running 16 minutes ago
ymjwzn1aykd0 my-nginx.3 nginx-sj:2024022601 centos7-18.localdomain Running Running 16 minutes ago
- 查看服务详情
- “Name”: “my-nginx”
- “Image”: “nginx-sj:2024022601”
- “Mounts”: [{
“Type”: “volume”,
“Source”: “my-volume”,
“Target”: “/usr/share/nginx/html/”
}] - “EndpointSpec”: {
“Mode”: “vip”,
“Ports”: [{
“Protocol”: “tcp”,
“TargetPort”: 80,
“PublishedPort”: 9090,
“PublishMode”: “ingress”
}]} - “VirtualIPs”: [{
“NetworkID”: “9j40lm9s5dtedg04r6jywqr1k”,
“Addr”: “10.0.0.15/24”
},{
“NetworkID”: “9u6c1vlfy4kvli7ihka8f5mjf”,
“Addr”: “172.16.0.27/16”
}]
10.0.0.15/24是 Ingress网络的VIP，172.16.0.27/16是 自定义Overlay网络（ol_network）的VIP。关于VIP的使用，后面再说。
[root@centos7-10 ~]# docker service inspect my-nginx
[
{
"ID": "7rfsp2zazavrnckuw074dv8dl",
"Version": {
"Index": 1661
},
"CreatedAt": "2024-02-26T09:59:24.571367943Z",
"UpdatedAt": "2024-03-05T15:03:31.69323247Z",
"Spec": {
"Name": "my-nginx",
"Labels": {},
"TaskTemplate": {
"ContainerSpec": {
"Image": "nginx-sj:2024022601",
"Init": false,
"Mounts": [
{
"Type": "volume",
"Source": "my-volume",
"Target": "/usr/share/nginx/html/"
}
],
"StopGracePeriod": 10000000000,
"DNSConfig": {},
"Isolation": "default"
},
"Resources": {
"Limits": {},
"Reservations": {}
},
"RestartPolicy": {
"Condition": "any",
"Delay": 5000000000,
"MaxAttempts": 0
},
"Placement": {},
"Networks": [
{
"Target": "9u6c1vlfy4kvli7ihka8f5mjf"
}
],
"ForceUpdate": 0,
"Runtime": "container"
},
"Mode": {
"Replicated": {
"Replicas": 3
}
},
"UpdateConfig": {
"Parallelism": 1,
"FailureAction": "pause",
"Monitor": 5000000000,
"MaxFailureRatio": 0,
"Order": "stop-first"
},
"RollbackConfig": {
"Parallelism": 1,
"FailureAction": "pause",
"Monitor": 5000000000,
"MaxFailureRatio": 0,
"Order": "stop-first"
},
"EndpointSpec": {
"Mode": "vip",
"Ports": [
{
"Protocol": "tcp",
"TargetPort": 80,
"PublishedPort": 9090,
"PublishMode": "ingress"
}
]
}
},
"PreviousSpec": {
"Name": "my-nginx",
"Labels": {},
"TaskTemplate": {
"ContainerSpec": {
"Image": "nginx-sj:2024022601",
"Init": false,
"Mounts": [
{
"Type": "volume",
"Source": "my-volume",
"Target": "/usr/share/nginx/html/"
}
],
"DNSConfig": {},
"Isolation": "default"
},
"Resources": {
"Limits": {},
"Reservations": {}
},
"Placement": {},
"Networks": [
{
"Target": "9u6c1vlfy4kvli7ihka8f5mjf"
}
],
"ForceUpdate": 0,
"Runtime": "container"
},
"Mode": {
"Replicated": {
"Replicas": 2
}
},
"EndpointSpec": {
"Mode": "vip",
"Ports": [
{
"Protocol": "tcp",
"TargetPort": 80,
"PublishedPort": 9090,
"PublishMode": "ingress"
}
]
}
},
"Endpoint": {
"Spec": {
"Mode": "vip",
"Ports": [
{
"Protocol": "tcp",
"TargetPort": 80,
"PublishedPort": 9090,
"PublishMode": "ingress"
}
]
},
"Ports": [
{
"Protocol": "tcp",
"TargetPort": 80,
"PublishedPort": 9090,
"PublishMode": "ingress"
}
],
"VirtualIPs": [
{
"NetworkID": "9j40lm9s5dtedg04r6jywqr1k",
"Addr": "10.0.0.15/24"
},
{
"NetworkID": "9u6c1vlfy4kvli7ihka8f5mjf",
"Addr": "172.16.0.27/16"
}
]
}
}
]
5.2.2 创建Swarm和独立容器使用的Overlay网络
上述网络中是不允许独立容器加入的，如加入会提示错误信息
[root@centos7-10 ~]# docker run -itd --network ol_network ubuntu22:2024022601 /bin/bash
39413e7563efdf5e240115546586a2bf9b5d04c6bcfa47c5af2c3203f4706c06
docker: Error response from daemon: Could not attach to network ol_network: rpc error: code = PermissionDenied desc = network ol_network not manually attachable.
[root@centos7-10 ~]#
- 创建独立容器使用的Overlay网络ol_standalone_network
- 创建命令增加–attachable，允许独立容器和Swarm容器均可以连接网络
// 创建独立容器使用的Overlay网络ol_standalone_network
[root@centos7-10 ~]# docker network create -d overlay --attachable --subnet 172.19.0.0/16 --gateway 172.19.0.1 ol_standalone_network
mwtyi4j8x0ycnzmzmqsjt11w7
- 创建独立容器加入ol_standalone_network
[root@centos7-10 ~]# docker run -itd --network ol_standalone_network --ip 172.19.0.2 ubuntu22:2024022601 /bin/bash
aebb47779d089f6c84c7b2453932b945e565827fbb611c81dd33580c7732e089
- 查看网络ol_standalone_network
- “Attachable”: true,
- “Driver”: “overlay”,
- “com.docker.network.driver.overlay.vxlanid_list”: “4099”
- 独立容器已经加入网络
// 查看ol_standalone_network信息
[root@centos7-10 ~]# docker network inspect ol_standalone_network
[
{
"Name": "ol_standalone_network",
"Id": "mwtyi4j8x0ycnzmzmqsjt11w7",
"Created": "2024-03-12T13:21:52.528855154+08:00",
"Scope": "swarm",
"Driver": "overlay",
"EnableIPv6": false,
"IPAM": {
"Driver": "default",
"Options": null,
"Config": [
{
"Subnet": "172.19.0.0/16",
"Gateway": "172.19.0.1"
}
]
},
"Internal": false,
"Attachable": true,
"Ingress": false,
"ConfigFrom": {
"Network": ""
},
"ConfigOnly": false,
"Containers": {
"aebb47779d089f6c84c7b2453932b945e565827fbb611c81dd33580c7732e089": {
"Name": "stupefied_gould",
"EndpointID": "738d98a9b5b3fa7b1bd04aba065e4d87f1022244fdbac7436e5b1fa051ed139d",
"MacAddress": "02:42:ac:13:00:02",
"IPv4Address": "172.19.0.2/16",
"IPv6Address": ""
},
"lb-ol_standalone_network": {
"Name": "ol_standalone_network-endpoint",
"EndpointID": "e937cdb4e51d863b869be895246074cb78e58cf19cdf9f9770f5bb2bdd59bd14",
"MacAddress": "02:42:ac:13:00:03",
"IPv4Address": "172.19.0.3/16",
"IPv6Address": ""
}
},
"Options": {
"com.docker.network.driver.overlay.vxlanid_list": "4099"
},
"Labels": {},
"Peers": [
{
"Name": "ec5d0b20deba",
"IP": "10.211.55.10"
}
]
}
]
5.3 测试自定义Overlay网络
5.3.1 外部通过Overlay网络访问服务
- 请求轮询访问服务
jinshideMacBook-Pro-2:~ shijin$ curl http://10.211.55.10:9090/index1.html
Host IP is 10.211.55.22
jinshideMacBook-Pro-2:~ shijin$ curl http://10.211.55.10:9090/index1.html
Host IP is 10.211.55.18
jinshideMacBook-Pro-2:~ shijin$ curl http://10.211.55.10:9090/index1.html
Host IP is 10.211.55.10
jinshideMacBook-Pro-2:~ shijin$ curl http://10.211.55.10:9090/index1.html
Host IP is 10.211.55.22
jinshideMacBook-Pro-2:~ shijin$ curl http://10.211.55.10:9090/index1.html
Host IP is 10.211.55.18
jinshideMacBook-Pro-2:~ shijin$ curl http://10.211.55.10:9090/index1.html
Host IP is 10.211.55.10
5.3.2 Overlay网络内容器互相访问
- 宿主机Centos7-10上容器（172.16.0.28） Ping 172.16.0.43（Centos7-22），通
- 宿主机Centos7-10上容器（172.16.0.28） Ping 172.16.0.4（Centos7-18），通
root@3f75b0e3f124:/# ping 172.16.0.43 -c3
PING 172.16.0.43 (172.16.0.43): 56 data bytes
64 bytes from 172.16.0.43: icmp_seq=0 ttl=64 time=0.605 ms
64 bytes from 172.16.0.43: icmp_seq=1 ttl=64 time=0.762 ms
64 bytes from 172.16.0.43: icmp_seq=2 ttl=64 time=0.655 ms
--- 172.16.0.43 ping statistics ---
3 packets transmitted, 3 packets received, 0% packet loss
round-trip min/avg/max/stddev = 0.605/0.674/0.762/0.065 ms
root@3f75b0e3f124:/# ping 172.16.0.4 -c3
PING 172.16.0.4 (172.16.0.4): 56 data bytes
64 bytes from 172.16.0.4: icmp_seq=0 ttl=64 time=1.016 ms
64 bytes from 172.16.0.4: icmp_seq=1 ttl=64 time=0.935 ms
64 bytes from 172.16.0.4: icmp_seq=2 ttl=64 time=0.857 ms
--- 172.16.0.4 ping statistics ---
3 packets transmitted, 3 packets received, 0% packet loss
round-trip min/avg/max/stddev = 0.857/0.936/1.016/0.065 ms
root@3f75b0e3f124:/#
5.4 Overlay网络连接说明
5.4.1 Overlay网络容器网卡情况
- 进入宿主机centos7-10上的容器查看IP
- eth0 10.0.0.16 属于ingress网络
- eth1 172.16.0.28 属于自定义Overlay网络（ol_network）
- eth2 172.18.0.3 属于docker_gwbridge网络
// 进入宿主机10.211.55.10上的容器3f75b0e3f124
[root@centos7-10 ~]# docker ps -a
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
3f75b0e3f124 nginx-sj:2024022601 "nginx -g 'daemon of…" 56 minutes ago Up 56 minutes my-nginx.1.elw6yuwr0yid85477cqysbx5c
[root@centos7-10 ~]# docker exec -it 3f /bin/bash
// 查看容器IP
root@3f75b0e3f124:/# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
664: eth0@if665: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc noqueue state UP group default
link/ether 02:42:0a:00:00:10 brd ff:ff:ff:ff:ff:ff link-netnsid 0
inet 10.0.0.16/24 brd 10.0.0.255 scope global eth0
valid_lft forever preferred_lft forever
666: eth2@if667: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default
link/ether 02:42:ac:12:00:03 brd ff:ff:ff:ff:ff:ff link-netnsid 2
inet 172.18.0.3/16 brd 172.18.255.255 scope global eth2
valid_lft forever preferred_lft forever
668: eth1@if669: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc noqueue state UP group default
link/ether 02:42:ac:10:00:1c brd ff:ff:ff:ff:ff:ff link-netnsid 1
inet 172.16.0.28/16 brd 172.16.255.255 scope global eth1
valid_lft forever preferred_lft forever
- 查看宿主机centos7-10上ingress网络
- 容器3f75b0e3f124 IP 10.0.0.16/24（每个容器的IP不同）
- ingress-sbox IP 10.0.0.2/24（每个集群节点的IP不同，例如宿主机centos7-18上ingress-sbox “IPv4Address”: “10.0.0.3/24”,）
- VNI 4096
[root@centos7-10 ~]# docker network inspect ingress
[
{
"Name": "ingress",
"Id": "9j40lm9s5dtedg04r6jywqr1k",
"Created": "2024-02-26T10:54:37.535807972+08:00",
"Scope": "swarm",
"Driver": "overlay",
"EnableIPv6": false,
"IPAM": {
"Driver": "default",
"Options": null,
"Config": [
{
"Subnet": "10.0.0.0/24",
"Gateway": "10.0.0.1"
}
]
},
"Internal": false,
"Attachable": false,
"Ingress": true,
"ConfigFrom": {
"Network": ""
},
"ConfigOnly": false,
"Containers": {
"3f75b0e3f12428bac0099ccd9c5a40e7e69ac50e6a6368c13f677e52a1c8557a": {
"Name": "my-nginx.1.elw6yuwr0yid85477cqysbx5c",
"EndpointID": "845252b69360249d22212303b7be06df51c92b67416c7155c8d9cfe7ee488172",
"MacAddress": "02:42:0a:00:00:10",
"IPv4Address": "10.0.0.16/24",
"IPv6Address": ""
},
"ingress-sbox": {
"Name": "ingress-endpoint",
"EndpointID": "a84011091b58eb59bcf716fce6dff34759ca8a34af4c5ad753221508cdd2304b",
"MacAddress": "02:42:0a:00:00:02",
"IPv4Address": "10.0.0.2/24",
"IPv6Address": ""
}
},
"Options": {
"com.docker.network.driver.overlay.vxlanid_list": "4096"
},
"Labels": {},
"Peers": [
{
"Name": "36c12810b62d",
"IP": "10.211.55.10"
},
{
"Name": "8c9ea1baadab",
"IP": "10.211.55.22"
},
{
"Name": "d1c0ea4bd8fd",
"IP": "10.211.55.18"
}
]
}
]
- 查看宿主机centos7-10上Overlay网络(ol_network)
- 容器3f75b0e3f124 IP 172.16.0.28/16
- lb-ol_network IP 172.16.0.31/16
- VNI 4098
[root@centos7-10 ~]# docker network inspect ol_network
[
{
"Name": "ol_network",
"Id": "9u6c1vlfy4kvli7ihka8f5mjf",
"Created": "2024-02-26T17:59:24.77121473+08:00",
"Scope": "swarm",
"Driver": "overlay",
"EnableIPv6": false,
"IPAM": {
"Driver": "default",
"Options": null,
"Config": [
{
"Subnet": "172.16.0.0/16",
"Gateway": "172.16.0.1"
}
]
},
"Internal": false,
"Attachable": false,
"Ingress": false,
"ConfigFrom": {
"Network": ""
},
"ConfigOnly": false,
"Containers": {
"3f75b0e3f12428bac0099ccd9c5a40e7e69ac50e6a6368c13f677e52a1c8557a": {
"Name": "my-nginx.1.elw6yuwr0yid85477cqysbx5c",
"EndpointID": "9f2afa4733fa92595372ec4a97345186d22b9569aaf3c392b9d14e550e33dd6e",
"MacAddress": "02:42:ac:10:00:1c",
"IPv4Address": "172.16.0.28/16",
"IPv6Address": ""
},
"lb-ol_network": {
"Name": "ol_network-endpoint",
"EndpointID": "b87c0028a6b83ed61ba3bfcbf2bf17370ed4af39631d23b6e972f8ee63ef452a",
"MacAddress": "02:42:ac:10:00:1f",
"IPv4Address": "172.16.0.31/16",
"IPv6Address": ""
}
},
"Options": {
"com.docker.network.driver.overlay.vxlanid_list": "4098"
},
"Labels": {},
"Peers": [
{
"Name": "36c12810b62d",
"IP": "10.211.55.10"
},
{
"Name": "8c9ea1baadab",
"IP": "10.211.55.22"
},
{
"Name": "d1c0ea4bd8fd",
"IP": "10.211.55.18"
}
]
}
]
- 查看宿主机centos7-10上docker_gwbridge网络
- 容器3f75b0e3f124 IP 172.18.0.3/16
- ingress-sbox IP 172.18.0.2/16
[root@centos7-10 ~]# docker network inspect docker_gwbridge
[
{
"Name": "docker_gwbridge",
"Id": "333348eb8ff434a7be478f93741588da9624bbab0b0787e5c68f2696816d3909",
"Created": "2024-02-22T13:42:37.480107344+08:00",
"Scope": "local",
"Driver": "bridge",
"EnableIPv6": false,
"IPAM": {
"Driver": "default",
"Options": null,
"Config": [
{
"Subnet": "172.18.0.0/16",
"Gateway": "172.18.0.1"
}
]
},
"Internal": false,
"Attachable": false,
"Ingress": false,
"ConfigFrom": {
"Network": ""
},
"ConfigOnly": false,
"Containers": {
"3f75b0e3f12428bac0099ccd9c5a40e7e69ac50e6a6368c13f677e52a1c8557a": {
"Name": "gateway_95f56fdac3fd",
"EndpointID": "583a5ddc952ec4357d506b2330258c7abe10b5be250d98f0212d6ff30ed15c1c",
"MacAddress": "02:42:ac:12:00:03",
"IPv4Address": "172.18.0.3/16",
"IPv6Address": ""
},
"ingress-sbox": {
"Name": "gateway_ingress-sbox",
"EndpointID": "d87bca0ffde2ce4ed88b9551ffae5ec59f4a71beabfa4cdfc88871fc229d1f85",
"MacAddress": "02:42:ac:12:00:02",
"IPv4Address": "172.18.0.2/16",
"IPv6Address": ""
}
},
"Options": {
"com.docker.network.bridge.enable_icc": "false",
"com.docker.network.bridge.enable_ip_masquerade": "true",
"com.docker.network.bridge.name": "docker_gwbridge"
},
"Labels": {}
}
]
5.4.2 Overlay网络容器网卡连接
上图中以10.211.55.10与10.211.55.22两个节点为例，显示了10.211.55.10节点上的网络命名空间和网卡连接关系，同时显示了ingress网络和自定义overlay网络（ol_network）的Vxlan隧道情况。
- 查看容器所在的命名空间
[root@centos7-10 netns]# docker inspect 3f75b0e3f124 | grep -i sand
"SandboxID": "95f56fdac3fde1b04f6d274df2486868292cce38eae802b4438f336c68ac4b2a",
"SandboxKey": "/var/run/docker/netns/95f56fdac3fd",
- 将容器的netns目录链接到/var/run/netns，查看命名空间
- 95f56fdac3fd (id: 4) 容器3f75b0e3f124的命名空间
- 1-9u6c1vlfy4 (id: 2) 自定义overlay网络（ol_network）的VTEP（VXLAN Tunnel Endpoints）
- 1-9j40lm9s5d (id: 0) ingress网络的VTEP（VXLAN Tunnel Endpoints）
[root@centos7-10 netns]# ln -s /var/run/docker/netns/ /var/run/netns
[root@centos7-10 ~]# ip netns show
95f56fdac3fd (id: 4) // 容器3f75b0e3f124的命名空间
1-9u6c1vlfy4 (id: 2) // 自定义overlay网络（ol_network）的VTEP（VXLAN Tunnel Endpoints）
lb_9u6c1vlfy (id: 3)
1-9j40lm9s5d (id: 0) // ingress网络的VTEP（VXLAN Tunnel Endpoints）
ingress_sbox (id: 1)
[root@centos7-10 ~]#
- 自定义overlay网络（ol_network）的VTEP，vxlan id = 4098
节点10.211.55.22与10.211.55.10的命名空间名称一致（1-9u6c1vlfy4），也可以查看到
[root@centos7-10 ~]# ip netns exec 1-9u6c1vlfy4 ip -d link show type vxlan
661: vxlan0@if661: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc noqueue master br0 state UNKNOWN mode DEFAULT group default
link/ether 1a:6c:eb:be:ad:8c brd ff:ff:ff:ff:ff:ff link-netnsid 0 promiscuity 1
vxlan id 4098 srcport 0 0 dstport 4789 proxy l2miss l3miss ageing 300 noudpcsum noudp6zerocsumtx noudp6zerocsumrx
bridge_slave state forwarding priority 32 cost 100 hairpin off guard off root_block off fastleave off learning on flood on port_id 0x8001 port_no 0x1 designated_port 32769 designated_cost 0 designated_bridge 8000.1a:6c:eb:be:ad:8c designated_root 8000.1a:6c:eb:be:ad:8c hold_timer 0.00 message_age_timer 0.00 forward_delay_timer 0.00 topology_change_ack 0 config_pending 0 proxy_arp off proxy_arp_wifi off mcast_router 1 mcast_fast_leave off mcast_flood on addrgenmode eui64 numtxqueues 1 numrxqueues 1 gso_max_size 65536 gso_max_segs 65535
- 自定义overlay网络（ol_network）的vxlan隧道
[root@centos7-10 ~]# ip netns exec 1-9u6c1vlfy4 bridge -d fdb show dev vxlan0
1a:6c:eb:be:ad:8c master br0 permanent
// 10.211.55.22节点容器4e9f5b291297的网卡 245: eth1@if246,IP 172.16.0.43
02:42:ac:10:00:2b dst 10.211.55.22 link-netnsid 0 self permanent
02:42:ac:10:00:2d dst 10.211.55.22 link-netnsid 0 self permanent
02:42:ac:10:00:2a dst 10.211.55.18 link-netnsid 0 self permanent
02:42:ac:10:00:2c dst 10.211.55.18 link-netnsid 0 self permanent
- ingress网络的VTEP，vxlan id = 4096
节点10.211.55.22与10.211.55.10的命名空间名称一致（1-9j40lm9s5d），也可以查看到
[root@centos7-10 ~]# ip netns exec 1-9j40lm9s5d ip -d link show type vxlan
599: vxlan0@if599: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc noqueue master br0 state UNKNOWN mode DEFAULT group default
link/ether 3a:3b:2f:96:62:f1 brd ff:ff:ff:ff:ff:ff link-netnsid 0 promiscuity 1
vxlan id 4096 srcport 0 0 dstport 4789 proxy l2miss l3miss ageing 300 noudpcsum noudp6zerocsumtx noudp6zerocsumrx
bridge_slave state forwarding priority 32 cost 100 hairpin off guard off root_block off fastleave off learning on flood on port_id 0x8001 port_no 0x1 designated_port 32769 designated_cost 0 designated_bridge 8000.1e:48:56:bd:ab:df designated_root 8000.1e:48:56:bd:ab:df hold_timer 0.00 message_age_timer 0.00 forward_delay_timer 0.00 topology_change_ack 0 config_pending 0 proxy_arp off proxy_arp_wifi off mcast_router 1 mcast_fast_leave off mcast_flood on addrgenmode eui64 numtxqueues 1 numrxqueues 1 gso_max_size 65536 gso_max_segs 65535
- ingress网络的vxlan隧道
[root@centos7-10 ~]# ip netns exec 1-9j40lm9s5d bridge -d fdb show dev vxlan0
3a:3b:2f:96:62:f1 master br0 permanent
02:42:0a:00:00:04 dst 10.211.55.22 link-netnsid 0 self permanent
// 10.211.55.22节点容器4e9f5b291297的网卡 241: eth0@if242,IP 10.0.0.26/24
02:42:0a:00:00:1a dst 10.211.55.22 link-netnsid 0 self permanent
02:42:0a:00:00:19 dst 10.211.55.18 link-netnsid 0 self permanent
02:42:0a:00:00:03 dst 10.211.55.18 link-netnsid 0 self permanent
5.4.3 Overlay网络通讯协议发送
1. 容器访问外部服务
- 容器3f75b0e3f124 Ping www.sohu.com
容器3f75b0e3f124网卡eth2 --> 宿主机网卡veth86582c0 -->网桥docker_gwbridge --> 宿主机enp0s5 --> www.sohu.com
root@3f75b0e3f124:/# ping www.sohu.com -c1
PING best.51-65.cjt.sdytuntx.diansu-cdn.net (123.125.244.107): 56 data bytes
64 bytes from 123.125.244.107: icmp_seq=0 ttl=127 time=4.954 ms
--- best.51-65.cjt.sdytuntx.diansu-cdn.net ping statistics ---
1 packets transmitted, 1 packets received, 0% packet loss
round-trip min/avg/max/stddev = 4.954/4.954/4.954/0.000 ms
- 抓667: veth86582c0@if666（容器内666: eth2@if667的Veth pair）
[root@centos7-10 ~]# tcpdump -nv -e -i veth86582c0
tcpdump: verbose output suppressed, use -v or -vv for full protocol decode
listening on veth86582c0, link-type EN10MB (Ethernet), capture size 262144 bytes
16:40:12.541580 02:42:ac:12:00:03 > 02:42:2d:85:61:de, ethertype IPv4 (0x0800), length 72: 172.18.0.3.45737 > 10.211.55.1.53: 18127+ A? www.sohu.com. (30)
16:40:12.558467 02:42:2d:85:61:de > 02:42:ac:12:00:03, ethertype IPv4 (0x0800), length 218: 10.211.55.1.53 > 172.18.0.3.45737: 18127 4/0/0 CNAME www.sohu.com.dsa.dnsv1.com., CNAME best.sched.d0-dk.tdnsdp1.cn., CNAME best.51-65.cjt.sdytuntx.diansu-cdn.net., A 123.125.244.107 (176)
16:40:12.559584 02:42:ac:12:00:03 > 02:42:2d:85:61:de, ethertype IPv4 (0x0800), length 98: 172.18.0.3 > 123.125.244.107: ICMP echo request, id 302, seq 0, length 64
16:40:12.564452 02:42:2d:85:61:de > 02:42:ac:12:00:03, ethertype IPv4 (0x0800), length 98: 123.125.244.107 > 172.18.0.3: ICMP echo reply, id 302, seq 0, length 64
- 抓docker_gwbridge
[root@centos7-10 ~]# tcpdump -nv -e -i docker_gwbridge
tcpdump: verbose output suppressed, use -v or -vv for full protocol decode
listening on docker_gwbridge, link-type EN10MB (Ethernet), capture size 262144 bytes
16:40:12.541580 02:42:ac:12:00:03 > 02:42:2d:85:61:de, ethertype IPv4 (0x0800), length 72: 172.18.0.3.45737 > 10.211.55.1.53: 18127+ A? www.sohu.com. (30)
16:40:12.558450 02:42:2d:85:61:de > 02:42:ac:12:00:03, ethertype IPv4 (0x0800), length 218: 10.211.55.1.53 > 172.18.0.3.45737: 18127 4/0/0 CNAME www.sohu.com.dsa.dnsv1.com., CNAME best.sched.d0-dk.tdnsdp1.cn., CNAME best.51-65.cjt.sdytuntx.diansu-cdn.net., A 123.125.244.107 (176)
16:40:12.559584 02:42:ac:12:00:03 > 02:42:2d:85:61:de, ethertype IPv4 (0x0800), length 98: 172.18.0.3 > 123.125.244.107: ICMP echo request, id 302, seq 0, length 64
16:40:12.564446 02:42:2d:85:61:de > 02:42:ac:12:00:03, ethertype IPv4 (0x0800), length 98: 123.125.244.107 > 172.18.0.3: ICMP echo reply, id 302, seq 0, length 64
- 抓宿主机centos7-10的enp0s5网卡
[root@centos7-10 ~]# tcpdump -nv -e -i enp0s5 icmp
tcpdump: verbose output suppressed, use -v or -vv for full protocol decode
listening on enp0s5, link-type EN10MB (Ethernet), capture size 262144 bytes
16:40:12.559657 00:1c:42:ae:b6:41 > 00:1c:42:00:00:18, ethertype IPv4 (0x0800), length 98: 10.211.55.10 > 123.125.244.107: ICMP echo request, id 302, seq 0, length 64
16:40:12.564392 00:1c:42:00:00:18 > 00:1c:42:ae:b6:41, ethertype IPv4 (0x0800), length 98: 123.125.244.107 > 10.211.55.10: ICMP echo reply, id 302, seq 0, length 64
2. 跨主机之间容器通讯
- 容器（172.16.0.28）通过Overlay网络 Ping 其他节点（10.211.55.22）上的容器IP 172.16.0.43
容器3f75b0e3f124网卡eth1 --> 命名空间1-9u6c1vlfy4下br0 -->命名空间1-9u6c1vlfy4下vxlan0（Vxlan VTEP VNI=4098） --> 宿主机（10.211.55.10）的enp0s5 --> 10.211.55.22 --> vxlan0（Vxlan VTEP VNI=4098）–> 172.16.0.43
root@3f75b0e3f124:/# ping 172.16.0.43 -c2
PING 172.16.0.43 (172.16.0.43): 56 data bytes
64 bytes from 172.16.0.43: icmp_seq=0 ttl=64 time=0.739 ms
64 bytes from 172.16.0.43: icmp_seq=1 ttl=64 time=1.073 ms
--- 172.16.0.43 ping statistics ---
2 packets transmitted, 2 packets received, 0% packet loss
round-trip min/avg/max/stddev = 0.739/0.906/1.073/0.167 ms
- 抓命名空间1-9u6c1vlfy4下br0（连接了Vxlan VTEP和容器网卡的veth pair）
[root@centos7-10 ~]# ip netns exec 1-9u6c1vlfy4 tcpdump -nv -e -i br0
tcpdump: listening on br0, link-type EN10MB (Ethernet), capture size 262144 bytes
17:19:37.731541 02:42:ac:10:00:1c > 02:42:ac:10:00:2b, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 64, id 35057, offset 0, flags [DF], proto ICMP (1), length 84)
172.16.0.28 > 172.16.0.43: ICMP echo request, id 317, seq 0, length 64
17:19:37.732078 02:42:ac:10:00:2b > 02:42:ac:10:00:1c, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 64, id 21573, offset 0, flags [none], proto ICMP (1), length 84)
172.16.0.43 > 172.16.0.28: ICMP echo reply, id 317, seq 0, length 64
抓命名空间1-9u6c1vlfy4下vxlan0（Vxlan VTEP VNI=4098）
[root@centos7-10 ~]# ip netns exec 1-9u6c1vlfy4 tcpdump -nv -e -i vxlan0
tcpdump: listening on vxlan0, link-type EN10MB (Ethernet), capture size 262144 bytes
17:19:37.731574 02:42:ac:10:00:1c > 02:42:ac:10:00:2b, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 64, id 35057, offset 0, flags [DF], proto ICMP (1), length 84)
172.16.0.28 > 172.16.0.43: ICMP echo request, id 317, seq 0, length 64
17:19:37.732078 02:42:ac:10:00:2b > 02:42:ac:10:00:1c, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 64, id 21573, offset 0, flags [none], proto ICMP (1), length 84)
172.16.0.43 > 172.16.0.28: ICMP echo reply, id 317, seq 0, length 64
- 抓宿主机centos7-10的网卡enp0s5（网络连接）
- tcpdump ‘udp[39]=1’ -nv -e -i enp0s5 含义是抓协议报文中UDP header 启始位置0偏移至39字节处（inner IP header中protocol）的值等于1（协议号1=icmp）。
- 结果显示数据包外层Vxlan走UDP，内层封装ICMP。
命令详细解析，请见我另一篇笔记：tcpdump使用pcap-filter抓Vxlan包内数据
[root@centos7-10 ~]# tcpdump 'udp[39]=1' -nv -e -i enp0s5
tcpdump: listening on enp0s5, link-type EN10MB (Ethernet), capture size 262144 bytes
17:19:37.731613 00:1c:42:ae:b6:41 > 00:1c:42:60:87:b2, ethertype IPv4 (0x0800), length 148: (tos 0x0, ttl 64, id 5915, offset 0, flags [none], proto UDP (17), length 134)
10.211.55.10.37241 > 10.211.55.18.4789: VXLAN, flags [I] (0x08), vni 4098
02:42:ac:10:00:1c > 02:42:ac:10:00:2b, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 64, id 35057, offset 0, flags [DF], proto ICMP (1), length 84)
172.16.0.28 > 172.16.0.43: ICMP echo request, id 317, seq 0, length 64
17:19:37.731958 00:1c:42:60:87:b2 > 00:1c:42:ae:b6:41, ethertype IPv4 (0x0800), length 148: (tos 0x0, ttl 64, id 29936, offset 0, flags [none], proto UDP (17), length 134)
10.211.55.18.36132 > 10.211.55.10.4789: VXLAN, flags [I] (0x08), vni 4098
02:42:ac:10:00:2b > 02:42:ac:10:00:1c, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 64, id 21573, offset 0, flags [none], proto ICMP (1), length 84)
172.16.0.43 > 172.16.0.28: ICMP echo reply, id 317, seq 0, length 64
3. 外部请求访问服务
以下简述外部请求访问服务，如需了解详细流程可以参见我另一篇笔记：Swarm集群负载均衡的实现方式
- 外部访问Swarm节点1：centos7-10的9090端口
外部请求Swarm节点1的9090端口 --> 宿主机centos7-10命名空间内DNAT目的IP至172.18.0.2 -->命名空间ingress_sbox下启用ipvs，SNAT修改源IP为10.0.0.2 --> 经命名空间1-9j40lm9s5d下br0桥接 -->同一节点的容器 或 经ingress网络转发至其他节点的容器–> 容器内 9090 REDIRECT 80 --> 容器内nginx服务
// 外部客户的请求集群节点centos7-10 两次
jinshideMacBook-Pro-2:~ shijin$ curl http://10.211.55.10:9090/index1.html
Host IP is 10.211.55.10
jinshideMacBook-Pro-2:~ shijin$ curl http://10.211.55.10:9090/index1.html
Host IP is 10.211.55.18
- 宿主机命名空间iptables nat表 PREROUTING链
- Chain PREROUTING --> Chain DOCKER-INGRESS
- DNAT规则 tcp dpt:9090 to:172.18.0.2:9090
- 172.18.0.2网卡在命名空间ingress_sbox下
[root@centos7-10 ~]# iptables -L -nv -t nat
Chain PREROUTING (policy ACCEPT 571 packets, 54891 bytes)
pkts bytes target prot opt in out source destination
438 26280 DOCKER-INGRESS all -- * * 0.0.0.0/0 0.0.0.0/0 ADDRTYPE match dst-type LOCAL
444 26640 DOCKER all -- * * 0.0.0.0/0 0.0.0.0/0 ADDRTYPE match dst-type LOCAL
......
Chain DOCKER-INGRESS (2 references)
pkts bytes target prot opt in out source destination
0 0 DNAT tcp -- * * 0.0.0.0/0 0.0.0.0/0 tcp dpt:9090 to:172.18.0.2:9090
438 26280 RETURN all -- * * 0.0.0.0/0 0.0.0.0/0
- 命名空间ingress_sbox中iptables在mangle表打标记
- PREROUTING 链中，将所有目标端口是9090的，mark 0x102 (258)
- INPUT 链中，将所有目的地址是10.0.0.15的，mark 0x102 (258)
说明：后续ipvs会根据标记（258）定义集群
[root@centos7-10 ~]# ip netns exec ingress_sbox iptables -L -nv -t mangle
Chain PREROUTING (policy ACCEPT 912 packets, 75745 bytes)
pkts bytes target prot opt in out source destination
525 35100 MARK tcp -- * * 0.0.0.0/0 0.0.0.0/0 tcp dpt:9090 MARK set 0x102
Chain INPUT (policy ACCEPT 537 packets, 36145 bytes)
pkts bytes target prot opt in out source destination
0 0 MARK all -- * * 0.0.0.0/0 10.0.0.15 MARK set 0x102
Chain FORWARD (policy ACCEPT 375 packets, 39600 bytes)
pkts bytes target prot opt in out source destination
Chain OUTPUT (policy ACCEPT 537 packets, 35982 bytes)
pkts bytes target prot opt in out source destination
Chain POSTROUTING (policy ACCEPT 912 packets, 75582 bytes)
pkts bytes target prot opt in out source destination
- 命名空间ingress_sbox中启用了IPVS
上述查看Swarm服务my-nginx的时候，看到ingress的VIP是10.0.0.15。在ingress_sbox中，看到eth0配置了两个IP 10.0.0.15/32 和 10.0.0.2/24，我理解：- “访问请求”先被DNAT 目标地址到172.18.0.2上。进入172.18.0.2时，如果“访问请求”符合服务转发规则或是发往VIP的，那么就在mangle表打上firewall-mark标记，准备IPVS转发。其中每个集群节点都有VIP 10.0.0.15/32，每个节点收到的“访问请求”都是集群本地节点的10.0.0.15/32（32掩码在ingress网络中隐藏了VIP）。
- “访问请求”进入ingress_sbox命名空间时，被SNAT源地址10.0.0.2/24（ingress网络每个节点的IP不同，例如节点2centos7-18的IP是10.0.0.3/24，这保证了IPVS向其他节点上容器分配时的网络通讯）。
- 如果“访问请求”被ipvs分配到本地容器，那么直接转发进入容器。
- 如果“访问请求”被ipvs分配到其他节点上的容器，那么经过ingress网络转发进入其他节点的容器。
// ingress_sbox命名空间下的IP，其中eth0配置了10.0.0.15/32（VIP） 和 10.0.0.2/24
[root@centos7-10 ~]# ip netns exec ingress_sbox ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
6: eth0@if7: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc noqueue state UP group default
link/ether 02:42:0a:00:00:02 brd ff:ff:ff:ff:ff:ff link-netnsid 0
inet 10.0.0.2/24 brd 10.0.0.255 scope global eth0
valid_lft forever preferred_lft forever
inet 10.0.0.15/32 scope global eth0
valid_lft forever preferred_lft forever
8: eth1@if9: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default
link/ether 02:42:ac:12:00:02 brd ff:ff:ff:ff:ff:ff link-netnsid 1
inet 172.18.0.2/16 brd 172.18.255.255 scope global eth1
valid_lft forever preferred_lft forever
// 查看ipvs配置，集群启动了2个real server（2副本）
[root@centos7-10 ~]# ip netns exec ingress_sbox ipvsadm -L -n
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
FWM 258 rr
-> 10.0.0.16:0 Masq 1 0 0
-> 10.0.0.6:0 Masq 1 0 0
// 命名空间ingress_sbox下查看ipvs 连接表，看到有两个连接
[root@centos7-10 ~]# ip netns exec ingress_sbox ipvsadm -L -n -c
IPVS connection entries
pro expire state source virtual destination
TCP 01:57 TIME_WAIT 10.211.55.2:60342 172.18.0.2:9090 10.0.0.16:9090
TCP 01:54 TIME_WAIT 10.211.55.2:60346 172.18.0.2:9090 10.0.0.6:9090
- 命名空间ingress_sbox中iptables nat表 POSTROUTING链
- SNAT规则 10.0.0.2
[root@centos7-10 ~]# ip netns exec ingress_sbox iptables -L -nv -t nat
......
Chain POSTROUTING (policy ACCEPT 0 packets, 0 bytes)
pkts bytes target prot opt in out source destination
0 0 DOCKER_POSTROUTING all -- * * 0.0.0.0/0 127.0.0.11
0 0 SNAT all -- * * 0.0.0.0/0 10.0.0.0/24 ipvs to:10.0.0.2
- 容器3f75b0e3f124的命名空间95f56fdac3fd中iptables nat表 PREROUTING链
- REDIRECT规则 9090 REDIRECT 80
[root@centos7-10 ~]# ip netns exec 3f75b0e3f124 iptables -L -n -v -t nat
Chain PREROUTING (policy ACCEPT 0 packets, 0 bytes)
pkts bytes target prot opt in out source destination
4 256 REDIRECT tcp -- * * 0.0.0.0/0 10.0.0.16 tcp dpt:9090 redir ports 80
- 抓包命名空间ingress_sbox下eth1（172.18.0.2）
- 宿主机iptables 将收到9090的请求 DNAT至 172.18.0.2:9090
[root@centos7-10 ~]# ip netns exec ingress_sbox tcpdump -nnnv -i eth1
tcpdump: listening on eth1, link-type EN10MB (Ethernet), capture size 262144 bytes
20:55:36.234565 IP (tos 0x0, ttl 63, id 0, offset 0, flags [DF], proto TCP (6), length 64)
10.211.55.2.59719 > 172.18.0.2.9090: Flags [SEW], cksum 0xd63e (correct), seq 100232582, win 65535, options [mss 1460,nop,wscale 6,nop,nop,TS val 1243391918 ecr 0,sackOK,eol], length 0
20:55:36.235163 IP (tos 0x0, ttl 63, id 0, offset 0, flags [DF], proto TCP (6), length 60)
172.18.0.2.9090 > 10.211.55.2.59719: Flags [S.E], cksum 0xf3db (correct), seq 3269045449, ack 100232583, win 27960, options [mss 1410,sackOK,TS val 23729602 ecr 1243391918,nop,wscale 7], length 0
- 抓包节点1centos7-10下命名空间ingress_sbox下eth0（10.0.0.2）
- 客户端请求两次，curl http://10.211.55.10:9090/index1.html
- ipvs 轮询发送Real IP 10.0.0.16:9090（节点1centos7-10） 和 10.0.0.6:9090（节点2centos7-18 ）各一次
// 命名空间ingress_sbox下查看ipvs 连接表，看到有两个连接
[root@centos7-10 ~]# ip netns exec ingress_sbox ipvsadm -L -n -c
IPVS connection entries
pro expire state source virtual destination
TCP 01:57 TIME_WAIT 10.211.55.2:60342 172.18.0.2:9090 10.0.0.16:9090
TCP 01:54 TIME_WAIT 10.211.55.2:60346 172.18.0.2:9090 10.0.0.6:9090
// 命名空间ingress_sbox下查看conntrack，可以看到建立了两条连接记录
// origin(10.211.55.2-->172.18.0.2)，reply(10.0.0.16-->10.0.0.2)
[root@centos7-10 ~]# ip netns exec ingress_sbox conntrack -L
tcp 6 116 TIME_WAIT src=10.211.55.2 dst=172.18.0.2 sport=60342 dport=9090 src=10.0.0.16 dst=10.0.0.2 sport=9090 dport=60342 [ASSURED] mark=0 secctx=system_u:object_r:unlabeled_t:s0 use=1
tcp 6 115 TIME_WAIT src=10.211.55.2 dst=172.18.0.2 sport=60346 dport=9090 src=10.0.0.6 dst=10.0.0.2 sport=9090 dport=60346 [ASSURED] mark=0 secctx=system_u:object_r:unlabeled_t:s0 use=1
conntrack v1.4.4 (conntrack-tools): 2 flow entries have been shown.
// 抓包ingress_sbox 下eth0网卡（10.0.0.2）
[root@centos7-10 ~]# ip netns exec ingress_sbox tcpdump -nnnv -i eth0
tcpdump: listening on eth0, link-type EN10MB (Ethernet), capture size 262144 bytes
21:00:08.845984 IP (tos 0x0, ttl 63, id 0, offset 0, flags [DF], proto TCP (6), length 64)
10.0.0.2.60342 > 10.0.0.16.9090: Flags [SEW], cksum 0xa4ad (correct), seq 1985229939, win 65535, options [mss 1460,nop,wscale 6,nop,nop,TS val 1243642214 ecr 0,sackOK,eol], length 0
21:00:08.846541 IP (tos 0x0, ttl 64, id 0, offset 0, flags [DF], proto TCP (6), length 60)
10.0.0.16.9090 > 10.0.0.2.60342: Flags [S.E], cksum 0x0605 (correct), seq 2692833920, ack 1985229940, win 27960, options [mss 1410,sackOK,TS val 24002213 ecr 1243642214,nop,wscale 7], length 0
......
21:00:09.941852 IP (tos 0x0, ttl 63, id 0, offset 0, flags [DF], proto TCP (6), length 64)
10.0.0.2.60346 > 10.0.0.6.9090: Flags [SEW], cksum 0xfde0 (correct), seq 3605690040, win 65535, options [mss 1460,nop,wscale 6,nop,nop,TS val 1243643216 ecr 0,sackOK,eol], length 0
21:00:09.941940 IP (tos 0x0, ttl 64, id 0, offset 0, flags [DF], proto TCP (6), length 60)
10.0.0.6.9090 > 10.0.0.2.60346: Flags [S.E], cksum 0x1439 (incorrect -> 0x4d4a), seq 2619077628, ack 3605690041, win 27960, options [mss 1410,sackOK,TS val 24036220 ecr 1243643216,nop,wscale 7], length 0
......
- 抓节点2宿主机centos7-18上命名空间 1-9j40lm9s5d下的vxlan0（ingress网络VTEP VNI 4096）
- 当请求节点1centos7-10的9090端口时，能看到节点1上ipvs通过ingress网络转发到节点2centos7-18上访问10.0.0.6:9090的请求
// 注意10.0.0.2是节点1 ingress_sbox下eth0（10.0.0.2）。节点2centos7-18上是10.0.0.3
[root@centos7-18 ~]# ip netns exec 1-9j40lm9s5d tcpdump -nnnv -i vxlan0
tcpdump: listening on vxlan0, link-type EN10MB (Ethernet), capture size 262144 bytes
21:42:15.609559 IP (tos 0x0, ttl 63, id 0, offset 0, flags [DF], proto TCP (6), length 64)
10.0.0.2.49781 > 10.0.0.6.9090: Flags [SEW], cksum 0x924e (correct), seq 1977814095, win 65535, options [mss 1460,nop,wscale 6,nop,nop,TS val 1245961592 ecr 0,sackOK,eol], length 0
21:42:15.609664 IP (tos 0x0, ttl 64, id 0, offset 0, flags [DF], proto TCP (6), length 60)
10.0.0.6.9090 > 10.0.0.2.49781: Flags [S.E], cksum 0x1436 (incorrect -> 0xe870), seq 3404315318, ack 1977814096, win 27960, options [mss 1410,sackOK,TS val 26527509 ecr 1245961592,nop,wscale 7], length 0
- 抓节点2宿主机centos7-18上容器命名空间的数据包
// 节点2宿主机centos7-18上容器的eth0 IP 10.0.0.6/24
[root@centos7-18 ~]# ip netns exec 9a1c7776082b ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
16: eth1@if17: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc noqueue state UP group default
link/ether 02:42:ac:10:00:04 brd ff:ff:ff:ff:ff:ff link-netnsid 1
inet 172.16.0.4/16 brd 172.16.255.255 scope global eth1
valid_lft forever preferred_lft forever
18: eth2@if19: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default
link/ether 02:42:ac:12:00:03 brd ff:ff:ff:ff:ff:ff link-netnsid 2
inet 172.18.0.3/16 brd 172.18.255.255 scope global eth2
valid_lft forever preferred_lft forever
20: eth0@if21: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc noqueue state UP group default
link/ether 02:42:0a:00:00:06 brd ff:ff:ff:ff:ff:ff link-netnsid 0
inet 10.0.0.6/24 brd 10.0.0.255 scope global eth0
valid_lft forever preferred_lft forever
// 节点2的容器内能够抓到 10.0.0.2.49781 > 10.0.0.6.9090 的包
[root@centos7-18 ~]# ip netns exec 9a1c7776082b tcpdump -nnnv
tcpdump: listening on eth0, link-type EN10MB (Ethernet), capture size 262144 bytes
21:42:15.609609 IP (tos 0x0, ttl 63, id 0, offset 0, flags [DF], proto TCP (6), length 64)
10.0.0.2.49781 > 10.0.0.6.9090: Flags [SEW], cksum 0x924e (correct), seq 1977814095, win 65535, options [mss 1460,nop,wscale 6,nop,nop,TS val 1245961592 ecr 0,sackOK,eol], length 0
21:42:15.609658 IP (tos 0x0, ttl 64, id 0, offset 0, flags [DF], proto TCP (6), length 60)
10.0.0.6.9090 > 10.0.0.2.49781: Flags [S.E], cksum 0x1436 (incorrect -> 0xe870), seq 3404315318, ack 1977814096, win 27960, options [mss 1410,sackOK,TS val 26527509 ecr 1245961592,nop,wscale 7], length 0
- conntrack连接记录会显示reply的源端口是80
[root@centos7-18 ~]# ip netns exec 9a1c7776082b conntrack -L
tcp 6 114 TIME_WAIT src=10.0.0.2 dst=10.0.0.6 sport=55964 dport=9090 src=10.0.0.6 dst=10.0.0.2 sport=80 dport=55964 [ASSURED] mark=0 use=1
conntrack v1.4.4 (conntrack-tools): 1 flow entries have been shown.
6. Macvlan
Macvlan是一种容器网络模式，它允许Docker容器直接连接到主机的物理网络接口。这使得容器能够直接获得一个IP地址，并且在网络上可以像主机一样运行。Macvlan网络模式的优点是容器的网络性能非常好，而且可以实现容器与主机在同一局域网内相互通信。
在使用Macvlan网络模式时，需要指定一个父网卡（parent interface），也就是一个物理网卡，容器将使用该网卡进行通信。容器可以使用静态IP地址或从DHCP服务器获取IP地址。与其他网络模式相比，Macvlan网络模式的缺点是较为复杂，配置也相对困难。（引自: MCNU云原生文章）
Docker提供Macvlan网络的两种形式：桥接模式 和 802.1q 中继桥接模式。
6.1 网络准备-设置网卡混杂模式
- 以宿主机centos7-18为例，设置它的网卡（enp0s5）混杂模式（PROMISC）。
- PROMISC模式用于支持Macvlan网络
- PROMISC模式：网卡接收所有经过的数据包，包括不是发给本机的包，即不验证MAC地址。
- 注意！大多数云服务商禁止设置网卡的混杂模式（PROMISC）。
// 宿主机centos7-18的网卡情况
[root@centos7-18 ~]# ifconfig
docker0: flags=4099<UP,BROADCAST,MULTICAST> mtu 1500
inet 172.17.0.1 netmask 255.255.0.0 broadcast 172.17.255.255
inet6 fe80::42:fbff:fe6c:e068 prefixlen 64 scopeid 0x20<link>
ether 02:42:fb:6c:e0:68 txqueuelen 0 (Ethernet)
RX packets 79 bytes 6999 (6.8 KiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 96 bytes 8115 (7.9 KiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
enp0s5: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
inet 10.211.55.18 netmask 255.255.255.0 broadcast 10.211.55.255
inet6 fe80::21c:42ff:fe60:87b2 prefixlen 64 scopeid 0x20<link>
inet6 fdb2:2c26:f4e4:0:21c:42ff:fe60:87b2 prefixlen 64 scopeid 0x0<global>
ether 00:1c:42:60:87:b2 txqueuelen 1000 (Ethernet)
RX packets 29842 bytes 3073757 (2.9 MiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 168551 bytes 364116152 (347.2 MiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
- 增加网卡 promisc模式：ifconfig enp0s5 promisc,取消使用 ifconfig enp0s5 -promisc
- 也可以使用ip link set enp0s5 promisc on, 取消使用ip link set enp0s5 promisc off
[root@centos7-18 ~]# ifconfig | grep enp0s5
enp0s5: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
// 增加网卡 promisc模式,取消使用 ifconfig enp0s5 -promisc
// 也可以使用ip link set enp0s5 promisc on, 取消使用ip link set enp0s5 promisc off
[root@centos7-18 ~]# ifconfig enp0s5 promisc
[root@centos7-18 ~]# ifconfig | grep enp0s5
enp0s5: flags=4419<UP,BROADCAST,RUNNING,PROMISC,MULTICAST> mtu 1500
[root@centos7-18 ~]#
6.2 创建网络-Macvlan桥接模式
在桥接中，网络流量在宿主机网卡（eth0或enp0s5 ）流通，Docker 会用它的 MAC 地址，把流量路由进容器里。对于网络上的网络设备来说，容器似乎是物理上连接到网络上的。（引自: Docker网络教程（四）macvlan 网络教程
Docker官方文档 Network drivers / Macvlan
Docker官方文档 Macvlan network tutorial）
Macvlan桥接网络与宿主机网络是隔离的。
测试方式：创建一个macvlan桥接网络（macvlan-bridge），使用macvlan模式bridge，网络地址段与宿主机网络地址段相同(10.211.55.0/24)。然后创建两个容器加入到网络（macvlan-bridge）并运行。测试容器的IP是否与宿主机网络联通。
- 1. 查看宿主机centos7-10当前网络
[root@centos7-10 ~]# ifconfig
docker0: flags=4099<UP,BROADCAST,MULTICAST> mtu 1500
inet 172.17.0.1 netmask 255.255.0.0 broadcast 172.17.255.255
ether 02:42:63:e3:47:b1 txqueuelen 0 (Ethernet)
RX packets 0 bytes 0 (0.0 B)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 0 bytes 0 (0.0 B)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
// 网卡已开启混杂模式 PROMISC
enp0s5: flags=4419<UP,BROADCAST,RUNNING,PROMISC,MULTICAST> mtu 1500
inet 10.211.55.10 netmask 255.255.255.0 broadcast 10.211.55.255
inet6 fe80::7e0c:1902:e1ca:4324 prefixlen 64 scopeid 0x20<link>
inet6 fdb2:2c26:f4e4:0:cd1f:12f3:4076:6d89 prefixlen 64 scopeid 0x0<global>
ether 00:1c:42:ae:b6:41 txqueuelen 1000 (Ethernet)
RX packets 176348 bytes 209839378 (200.1 MiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 37243 bytes 5077466 (4.8 MiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
lo: flags=73<UP,LOOPBACK,RUNNING> mtu 65536
inet 127.0.0.1 netmask 255.0.0.0
inet6 ::1 prefixlen 128 scopeid 0x10<host>
loop txqueuelen 1000 (Local Loopback)
RX packets 85 bytes 6979 (6.8 KiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 85 bytes 6979 (6.8 KiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
[root@centos7-10 ~]#
[root@centos7-10 ~]# docker network ls
NETWORK ID NAME DRIVER SCOPE
28187d179df7 bridge bridge local
65a57293726b host host local
2db17c1087bf none null local
-
2. 在宿主机centos7-10上创建Macvlan桥接网络
- 桥接Macvlan网络使用宿主机网卡（enp0s5），网卡已设置为PROMISC。
- 桥接Macvlan网络默认使用Macvlan bridg模式，可选private、vepa、passthru，详见Macvlan概念（引自:图解几个与Linux网络虚拟化相关的虚拟网卡、（译文）Bridge vs Macvlan、 （译文）Macvlan vs Ipvlan 、 Docker官方文档：Macvlan ）
- 宿主机centos7-10的IP 10.211.55.10/24，网关10.211.55.1。创建一个同网段的Macvlan网络，命名macvlan-bridge
[root@centos7-10 ~]# docker network create -d macvlan \
> --subnet 10.211.55.0/24 \
> --gateway 10.211.55.1 \
> -o parent=enp0s5 \
> -o macvlan_mode=bridge \
> macvlan-bridge
f0573b7d7ecefea65e66dc1b8ff1037bc55538d61a2a38ddff0d0abf9e4a9707
[root@centos7-10 ~]#
- 3.查看宿主机centos7-10上创建的Macvlan桥接网络macvlan-bridge
- macvlan-bridge 的macvlan模式是bridge
- 父网卡是宿主机centos7-10的enp0s5
- 子网IP地址10.211.55.0/24，网关10.211.55.1（与宿主机网段相同）
[root@centos7-10 ~]# docker network ls
NETWORK ID NAME DRIVER SCOPE
28187d179df7 bridge bridge local
65a57293726b host host local
f0573b7d7ece macvlan-bridge macvlan local
2db17c1087bf none null local
[root@centos7-10 ~]#
[root@centos7-10 ~]# docker network inspect macvlan-bridge
[
{
"Name": "macvlan-bridge",
"Id": "f0573b7d7ecefea65e66dc1b8ff1037bc55538d61a2a38ddff0d0abf9e4a9707",
"Created": "2023-12-14T22:29:08.194169588+08:00",
"Scope": "local",
"Driver": "macvlan", // macvlan 网络
"EnableIPv6": false,
"IPAM": {
"Driver": "default",
"Options": {},
"Config": [
{
"Subnet": "10.211.55.0/24", // 网段
"Gateway": "10.211.55.1" // 网关
}
]
},
"Internal": false,
"Attachable": false,
"Ingress": false,
"ConfigFrom": {
"Network": ""
},
"ConfigOnly": false,
"Containers": {},
"Options": {
"macvlan_mode": "bridge", // 模式bridge， 默认模式
"parent": "enp0s5" // 父网卡
},
"Labels": {}
}
]
- 4. 在宿主机centos7-10上创建两个容器接入macvlan-bridge网络
- 容器ubuntu-55 IP 10.211.55.55
- 容器ubuntu-56 IP 10.211.55.56
// 创建容器ubuntu-55 加入macvlan-bridge
[root@centos7-10 ~]# docker run -itd --rm --name ubuntu-55 \
--ip 10.211.55.55 \
--network macvlan-bridge \
ubuntu22:latest /bin/bash
8c6e8c7be9ab516a690a7d7fe626815642b2474c8248e602450701583ba701d7
// 创建容器ubuntu-56 加入macvlan-bridge
[root@centos7-10 ~]# docker run -itd --rm --name ubuntu-56 \
--ip 10.211.55.56 \
--network macvlan-bridge \
ubuntu22:latest /bin/bash
36b7db4c7c3bc3f39871417f969b2468c3351a6e695b659bf3ab143379f09110
[root@centos7-10 ~]#
[root@centos7-10 ~]# docker ps
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
36b7db4c7c3b ubuntu22:latest "/bin/bash" 5 seconds ago Up 4 seconds ubuntu-56
8c6e8c7be9ab ubuntu22:latest "/bin/bash" 17 seconds ago Up 16 seconds ubuntu-55
[root@centos7-10 ~]#
- 5. 查看容器网卡与macvlan-bridge网络
- 容器ubuntu-55 网卡eth0@if2，IP 10.211.55.55，独立MAC地址（见下图）
- 容器ubuntu-56 网卡eth0@if2，IP 10.211.55.56，独立MAC地址
- 容器网卡中的if2表明容器网卡是宿主机全局ID为2网卡（enp0s5）的子接口
// 容器ubuntu-56的eth0@if2同样也是宿主机全局ID为2网卡（enp0s5）的子接口
[root@centos7-10 ~]# docker exec -it ubuntu-56 ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
9: eth0@if2: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default
link/ether 02:42:0a:d3:37:38 brd ff:ff:ff:ff:ff:ff link-netnsid 0
inet 10.211.55.56/24 brd 10.211.55.255 scope global eth0
valid_lft forever preferred_lft forever
[root@centos7-10 ~]#
// 查看macvlan-bridge网络
// 增加了两个容器ubuntu-55和ubuntu-56，独立MAC地址
[root@centos7-10 ~]# docker network inspect macvlan-bridge
[
{
"Name": "macvlan-bridge",
"Id": "f0573b7d7ecefea65e66dc1b8ff1037bc55538d61a2a38ddff0d0abf9e4a9707",
"Created": "2023-12-14T22:29:08.194169588+08:00",
"Scope": "local",
"Driver": "macvlan",
"EnableIPv6": false,
"IPAM": {
"Driver": "default",
"Options": {},
"Config": [
{
"Subnet": "10.211.55.0/24",
"Gateway": "10.211.55.1"
}
]
},
"Internal": false,
"Attachable": false,
"Ingress": false,
"ConfigFrom": {
"Network": ""
},
"ConfigOnly": false,
"Containers": {
"36b7db4c7c3bc3f39871417f969b2468c3351a6e695b659bf3ab143379f09110": {
"Name": "ubuntu-56",
"EndpointID": "d8569f58d9411fd7e8116fc9464c1db8e6d4c7982f447f713ab9826e38301939",
"MacAddress": "02:42:0a:d3:37:38",
"IPv4Address": "10.211.55.56/24",
"IPv6Address": ""
},
"8c6e8c7be9ab516a690a7d7fe626815642b2474c8248e602450701583ba701d7": {
"Name": "ubuntu-55",
"EndpointID": "6f9cc97f1735023ec70827ed21523acbc68b022aac709bea1bf88fa870b843e4",
"MacAddress": "02:42:0a:d3:37:37",
"IPv4Address": "10.211.55.55/24",
"IPv6Address": ""
}
},
"Options": {
"macvlan_mode": "bridge",
"parent": "enp0s5"
},
"Labels": {}
}
]
[root@centos7-10 ~]#
- 6. 测试容器网络与宿主机centos7-10网络通讯
容器网络地址段虽然与宿主机网络地址段相同，但macvlan网络与宿主机网络隔离，与父网卡不通。- 容器ubuntu-55 Ping自己的IP（10.211.55.55），网络通
- 容器ubuntu-55 Ping 同macvlan网络下其它容器的IP（10.211.55.56），网络通。注意当前是macvlan的模式是bridge
- 容器ubuntu-55 Ping 宿主机父网卡IP（10.211.55.10），网络不通
- 容器ubuntu-55 Ping 宿主机网关IP（10.211.55.1），网络通
- 容器ubuntu-55 Ping 同网络其它主机IP（10.211.55.18），网络通
- 容器ubuntu-55 Ping 外网（www.baidu.com），网络通
// 容器ubuntu-55 Ping自己的IP（10.211.55.55），网络通
[root@centos7-10 ~]# docker exec -it ubuntu-55 ping -c3 10.211.55.55
PING 10.211.55.55 (10.211.55.55): 56 data bytes
64 bytes from 10.211.55.55: icmp_seq=0 ttl=64 time=0.066 ms
64 bytes from 10.211.55.55: icmp_seq=1 ttl=64 time=0.096 ms
64 bytes from 10.211.55.55: icmp_seq=2 ttl=64 time=0.097 ms
--- 10.211.55.55 ping statistics ---
3 packets transmitted, 3 packets received, 0% packet loss
round-trip min/avg/max/stddev = 0.066/0.086/0.097/0.000 ms
[root@centos7-10 ~]#
// 容器ubuntu-55 Ping 同macvlan网络下其它容器的IP（10.211.55.56），网络通
[root@centos7-10 ~]# docker exec -it ubuntu-55 ping -c3 10.211.55.56
PING 10.211.55.56 (10.211.55.56): 56 data bytes
64 bytes from 10.211.55.56: icmp_seq=0 ttl=64 time=0.074 ms
64 bytes from 10.211.55.56: icmp_seq=1 ttl=64 time=0.097 ms
64 bytes from 10.211.55.56: icmp_seq=2 ttl=64 time=0.096 ms
--- 10.211.55.56 ping statistics ---
3 packets transmitted, 3 packets received, 0% packet loss
round-trip min/avg/max/stddev = 0.074/0.089/0.097/0.000 ms
[root@centos7-10 ~]#
// 容器ubuntu-55 Ping 宿主机父网卡IP（10.211.55.10），网络不通
[root@centos7-10 ~]# docker exec -it ubuntu-55 ping -c3 10.211.55.10
PING 10.211.55.10 (10.211.55.10): 56 data bytes
92 bytes from 8c6e8c7be9ab (10.211.55.55): Destination Host Unreachable
92 bytes from 8c6e8c7be9ab (10.211.55.55): Destination Host Unreachable
92 bytes from 8c6e8c7be9ab (10.211.55.55): Destination Host Unreachable
--- 10.211.55.10 ping statistics ---
3 packets transmitted, 0 packets received, 100% packet loss
[root@centos7-10 ~]#
// 容器ubuntu-55 Ping 宿主机网关IP（10.211.55.1），**网络通**
[root@centos7-10 ~]# docker exec -it ubuntu-55 ping -c3 10.211.55.1
PING 10.211.55.1 (10.211.55.1): 56 data bytes
64 bytes from 10.211.55.1: icmp_seq=0 ttl=128 time=0.225 ms
64 bytes from 10.211.55.1: icmp_seq=1 ttl=128 time=0.350 ms
64 bytes from 10.211.55.1: icmp_seq=2 ttl=128 time=0.375 ms
--- 10.211.55.1 ping statistics ---
3 packets transmitted, 3 packets received, 0% packet loss
round-trip min/avg/max/stddev = 0.225/0.317/0.375/0.066 ms
[root@centos7-10 ~]#
// 容器ubuntu-55 Ping 同网络其它主机IP（10.211.55.18），网络通
[root@centos7-10 ~]# docker exec -it ubuntu-55 ping -c3 10.211.55.18
PING 10.211.55.18 (10.211.55.18): 56 data bytes
64 bytes from 10.211.55.18: icmp_seq=0 ttl=64 time=0.347 ms
64 bytes from 10.211.55.18: icmp_seq=1 ttl=64 time=0.526 ms
64 bytes from 10.211.55.18: icmp_seq=2 ttl=64 time=0.564 ms
--- 10.211.55.18 ping statistics ---
3 packets transmitted, 3 packets received, 0% packet loss
round-trip min/avg/max/stddev = 0.347/0.479/0.564/0.095 ms
[root@centos7-10 ~]#
// 容器ubuntu-55 Ping 外网（www.baidu.com），网络通
[root@centos7-10 ~]# docker exec -it ubuntu-55 ping -c3 www.baidu.com
PING www.a.shifen.com (110.242.68.3): 56 data bytes
64 bytes from 110.242.68.3: icmp_seq=0 ttl=128 time=12.555 ms
64 bytes from 110.242.68.3: icmp_seq=1 ttl=128 time=15.255 ms
64 bytes from 110.242.68.3: icmp_seq=2 ttl=128 time=14.517 ms
--- www.a.shifen.com ping statistics ---
3 packets transmitted, 3 packets received, 0% packet loss
round-trip min/avg/max/stddev = 12.555/14.109/15.255/1.139 ms
[root@centos7-10 ~]#
6.3 创建网络-一个物理接口只能创建一个Macvlan网络
一个物理接口只能创建一个Macvlan网络，如果重复创建会提示接口已被占用。 如果需要创建多个Macvlan网络可以使用 802.1q 中继桥接模式，创建多个子接口，从而实现创建多个Macvlan网络。
测试方式：在宿主机centos7-10的enp0s5接口上再创建一个macvlan桥接网络，测试是否可以创建成功。
验证结果：错误提示，已经存在网络在用。
// 查看当前docker网络
[root@centos7-10 ~]# docker network ls
NETWORK ID NAME DRIVER SCOPE
28187d179df7 bridge bridge local
65a57293726b host host local
f0573b7d7ece macvlan-bridge macvlan local
2db17c1087bf none
// 创建macvlan网络失败
[root@centos7-10 ~]# docker network create -d macvlan \
> --subnet=192.168.100.0/24 \
> --gateway=192.168.100.1 \
> -o parent=enp0s5 \
> macvlan2-test
Error response from daemon: network dm-f0573b7d7ece is already using parent interface enp0s5
[root@centos7-10 ~]#
6.4 创建网络-Macvlan 802.1q 中继桥接模式网络
在802.1q中继桥接中，流量流经宿主机网卡（ eth0或enp0s5 ） 的一个子接口（例如称为eth0.100或enp0s5.100），Docker使用其(子接口)MAC地址将流量路由到容器里。对于网络上的网络设备来说，容器似乎是物理上连接到网络上的。（引自: Docker网络教程（四）macvlan 网络教程
Docker官方文档 Network drivers / Macvlan
Docker官方文档 Macvlan network tutorial）
测试方式：在宿主机centos7-10的enp0s5接口上创建macvlan子接口桥接网络。
验证结果：验证802.1q 中继桥接模式网络。
因测试过程复杂，我单独写了一份笔记，详见： Linux单主机模拟测试docker macvlan网络
6.5 创建网络-验证Macvlan的bridge、vepa、private、passthru模式
Macvlan网络分为四种模式：bridge、vepa、private、passthru
验证方式：创建四种模式的macvlan网络，加入容器，验证网络通讯。
6.5.1 Macvlan bridge模式
Macvlan bridge模式下，所有子接口都连接到父接口上，子接口之间流量可以直接交换，无需流出父接口（见下图）。（引自: （译文）Bridge vs Macvlan）
- 创建一个Macvlan网络（macvaln-bridge），macvlan模式为bridge
// 创建一个Macvlan网络，模式为bridge
[root@centos7-10 ~]# docker network create -d macvlan \
> --subnet=192.168.100.0/24 \
> --gateway=192.168.100.1 \
> -o parent=enp0s5 \
> -o macvlan_mode=bridge \
> macvlan-bridge
6289e989ac3fe49bf10489d02a4e8fa128510e35ff4bd758a678bade270f4619
- 创建两个容器（ubuntu-bridge2: 192.168.100.2 和 ubuntu-bridge3: 192.168.100.3）加入网络（macvaln-bridge）并运行
// 创建两个容器加入macvlan-bridge网络，并运行
[root@centos7-10 ~]# docker run -itd --rm \
> --name ubuntu-bridge2 \
> --ip 192.168.100.2 \
> --network macvlan-bridge \
> ubuntu22:latest \
> /bin/bash
625fbcc3a81cdc0c4a6cd602d7db635ad10398ae0f3262be5ba6af9f4a299f0c
[root@centos7-10 ~]#
[root@centos7-10 ~]# docker run -itd --rm \
> --name ubuntu-bridge3 \
> --ip 192.168.100.3 \
> --network macvlan-bridge \
> ubuntu22:latest \
> /bin/bash
2c264e4f940203ba14051a1ce8df53181b68b6c4813d40245e45e9426cab4bc1
[root@centos7-10 ~]# docker ps
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
2c264e4f9402 ubuntu22:latest "/bin/bash" 4 seconds ago Up 3 seconds ubuntu-bridge3
625fbcc3a81c ubuntu22:latest "/bin/bash" 22 seconds ago Up 21 seconds ubuntu-bridge2
[root@centos7-10 ~]#
// 查看macvlan-bridge，macvlan_mode:bridge
// 容器ubuntu-bridge2 IP 192.168.100.2
// 容器ubuntu-bridge3 IP 192.168.100.3
[root@centos7-10 ~]# docker network inspect macvlan-bridge
[
{
"Name": "macvlan-bridge",
"Id": "6289e989ac3fe49bf10489d02a4e8fa128510e35ff4bd758a678bade270f4619",
"Created": "2023-12-08T14:58:32.524122386+08:00",
"Scope": "local",
"Driver": "macvlan",
"EnableIPv6": false,
"IPAM": {
"Driver": "default",
"Options": {},
"Config": [
{
"Subnet": "192.168.100.0/24",
"Gateway": "192.168.100.1"
}
]
},
"Internal": false,
"Attachable": false,
"Ingress": false,
"ConfigFrom": {
"Network": ""
},
"ConfigOnly": false,
"Containers": {
"2c264e4f940203ba14051a1ce8df53181b68b6c4813d40245e45e9426cab4bc1": {
"Name": "ubuntu-bridge3",
"EndpointID": "8a04a741f7dbfa4bda5906ecf800fd306b71e73205abec9741b3036a8316b32e",
"MacAddress": "02:42:c0:a8:64:03",
"IPv4Address": "192.168.100.3/24",
"IPv6Address": ""
},
"625fbcc3a81cdc0c4a6cd602d7db635ad10398ae0f3262be5ba6af9f4a299f0c": {
"Name": "ubuntu-bridge2",
"EndpointID": "452d19d228e4accfa1d2318a13f9bc9610c208153de14a4c8070a18d0ca07dd2",
"MacAddress": "02:42:c0:a8:64:02",
"IPv4Address": "192.168.100.2/24",
"IPv6Address": ""
}
},
"Options": {
"macvlan_mode": "bridge",
"parent": "enp0s5"
},
"Labels": {}
}
]
[root@centos7-10 ~]#
- 测试两个容器IP（192.168.100.2和192.168.100.3）之间可以联通
// 验证容器之间IP可以连通
[root@centos7-10 ~]# docker exec -it ubuntu-bridge2 ip addr show
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
6: eth0@if2: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default
link/ether 02:42:c0:a8:64:02 brd ff:ff:ff:ff:ff:ff link-netnsid 0
inet 192.168.100.2/24 brd 192.168.100.255 scope global eth0
valid_lft forever preferred_lft forever
[root@centos7-10 ~]# docker exec -it ubuntu-bridge2 ping 192.168.100.3
PING 192.168.100.3 (192.168.100.3): 56 data bytes
64 bytes from 192.168.100.3: icmp_seq=0 ttl=64 time=0.228 ms
64 bytes from 192.168.100.3: icmp_seq=1 ttl=64 time=0.107 ms
64 bytes from 192.168.100.3: icmp_seq=2 ttl=64 time=0.106 ms
^C--- 192.168.100.3 ping statistics ---
3 packets transmitted, 3 packets received, 0% packet loss
round-trip min/avg/max/stddev = 0.106/0.147/0.228/0.057 ms
[root@centos7-10 ~]#
6.5.2 Macvlan vepa模式
Macvlan vepa( Virtual Ethernet Port Aggregator )模式下，所有子接口都连接到父接口上，子接口之间的流量将流出父接口后再进行交换，VEPA模式需要物理交换机支持IEEE 802.1Qbg（见下图）。（引自: （译文）Bridge vs Macvlan）
为了验证macvlan vepa，配合使用Linux bridge开启hairpin方式模拟外部交换机支持802.1q的情况
- 创建一对veth pair：veth1和veth1_br
// 创建一对veth pair：veth1和veth1_br
[root@centos7-10 ~]# ip link add name veth1 type veth peer name veth1_br
[root@centos7-10 ~]#
[root@centos7-10 ~]# ip link set veth1 up
[root@centos7-10 ~]# ip link set veth1_br up
- 创建linux bridge br0，将veth1_br加入br0
// 创建linux bridge br0
[root@centos7-10 ~]# brctl addbr br0
[root@centos7-10 ~]# ip link set br0 up
// 创建veth1_br加入br0
[root@centos7-10 ~]# brctl addif br0 veth1_br
[root@centos7-10 ~]# brctl show
bridge name bridge id STP enabled interfaces
br0 8000.226662726bd6 no veth1_br
- 创建macvlan网络macvlan_vepa，模式vepa
// 创建一个Macvlan网络，模式为vepa
[root@centos7-10 ~]# docker network create -d macvlan --subnet 192.168.20.0/24 --gateway 192.168.20.1 -o parent=veth1 -o macvlan_mode=vepa macvlan_vepa
7a5f745fbb05055a3b034e0fae8e6d8ba8adad69b30619d4f822b9af27515b7a
[root@centos7-10 ~]#
[root@centos7-10 ~]# docker network ls
NETWORK ID NAME DRIVER SCOPE
5d2354770162 bridge bridge local
65a57293726b host host local
7a5f745fbb05 macvlan_vepa macvlan local
2db17c1087bf none null local
- 创建两个容器：ubuntu-111和ubuntu-112，加入macvlan_vepa
// 创建一个Macvlan网络，模式为vepa
[root@centos7-10 ~]# docker run -itd --network macvlan_vepa --ip 192.168.20.111 --name ubuntu-111 --rm ubuntu22:latest /bin/bash
f85cd895d142186ce00ba5f1fc468696bd4918bd17eba442e67aa26c8152137a
[root@centos7-10 ~]# docker run -itd --network macvlan_vepa --ip 192.168.20.112 --name ubuntu-112 --rm ubuntu22:latest /bin/bash
af92463bf2979c28ecb976c64f0c032e7c919d76badecc164296767ba36d09a1
// 查看容器
[root@centos7-10 ~]# docker ps -a
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
af92463bf297 ubuntu22:latest "/bin/bash" 11 seconds ago Up 11 seconds ubuntu-112
f85cd895d142 ubuntu22:latest "/bin/bash" 26 seconds ago Up 25 seconds ubuntu-111
[root@centos7-10 ~]#
- 在外部交换br0不支持802.1q情况下，macvlan vepa网络的容器间无法通讯
// 查看ubuntu-111 ip
[root@centos7-10 ~]# docker exec -it ubuntu-111 ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
8: eth0@if5: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default
link/ether 02:42:c0:a8:14:6f brd ff:ff:ff:ff:ff:ff link-netnsid 0
inet 192.168.20.111/24 brd 192.168.20.255 scope global eth0
valid_lft forever preferred_lft forever
// ubuntu-111(192.168.20.111) ping ubuntu-112(192.168.20.112),不通
[root@centos7-10 ~]# docker exec -it ubuntu-111 ping 192.168.20.112 -c3
PING 192.168.20.112 (192.168.20.112): 56 data bytes
92 bytes from f85cd895d142 (192.168.20.111): Destination Host Unreachable
92 bytes from f85cd895d142 (192.168.20.111): Destination Host Unreachable
92 bytes from f85cd895d142 (192.168.20.111): Destination Host Unreachable
--- 192.168.20.112 ping statistics ---
3 packets transmitted, 0 packets received, 100% packet loss
- 开启外部交换br0的hairpin功能，支持802.1qbg
// 开启外部交换hairpin模式
[root@centos7-10 ~]# brctl hairpin br0 veth1_br on
// 查看开启结果
[root@centos7-10 ~]# bridge -d link | grep -A5 veth1
4: veth1_br state UP @veth1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 master br0 state forwarding priority 32 cost 2
hairpin on guard off root_block off fastleave off learning on flood on mcast_flood on
[root@centos7-10 ~]#
- 在外部交换br0支持802.1qbg情况下， macvlan网络(模式vepa)的容器间可以通讯
// ubuntu-111(192.168.20.111) ping ubuntu-112(192.168.20.112),通
[root@centos7-10 ~]# docker exec -it ubuntu-111 ping 192.168.20.112 -c3
PING 192.168.20.112 (192.168.20.112): 56 data bytes
64 bytes from 192.168.20.112: icmp_seq=0 ttl=64 time=0.089 ms
64 bytes from 192.168.20.112: icmp_seq=1 ttl=64 time=0.112 ms
64 bytes from 192.168.20.112: icmp_seq=2 ttl=64 time=0.119 ms
--- 192.168.20.112 ping statistics ---
3 packets transmitted, 3 packets received, 0% packet loss
round-trip min/avg/max/stddev = 0.089/0.107/0.119/0.000 ms
6.5.3 Macvlan private模式
Macvlan private模式下，同一个父接口上的子接口不能互相通信，子接口上的所有报文都必须流出父接口，即使从外部交换机上将子接口的报文送回同一个父接口上的其他子接口，该报文也将被丢弃。（见下图）。（引自: （译文）Bridge vs Macvlan）
为了验证macvlan private，配合使用Linux bridge开启hairpin方式模拟外部交换机支持802.1q的情况-
- 创建一个Macvlan网络，模式为private
// 创建一个Macvlan网络，模式为private
[root@centos7-10 ~]# docker network create -d macvlan --subnet 192.168.30.0/24 --gateway=192.168.30.1 -o parent=veth1 -o macvlan_mode=private macvlan_private
2c0016018bee2e63375843e2cb892eff0d2dfb457ebd9ea1c42a306fecfeecf3
[root@centos7-10 ~]# docker network ls
NETWORK ID NAME DRIVER SCOPE
5d2354770162 bridge bridge local
65a57293726b host host local
2c0016018bee macvlan_private macvlan local
2db17c1087bf none null local
[root@centos7-10 ~]#
- 创建两个容器加入macvlan-private网络，并运行
- ubuntu-121 192.168.30.121
- ubuntu-122 192.168.30.122
// 创建两个容器加入macvlan-private网络，并运行
[root@centos7-10 ~]# docker run -itd --rm --network macvlan_private --ip 192.168.30.121 --name ubuntu-121 ubuntu22:latest /bin/bash
52ab125ab578b11a9825243bfd8ae33372a879b497fa1906ffcbd38fbb19fec5
[root@centos7-10 ~]# docker run -itd --rm --network macvlan_private --ip 192.168.30.122 --name ubuntu-122 ubuntu22:latest /bin/bash
0256581620e3f42a85cde7dd5d713690c621168b463a868c98e4b581efe33d8e
[root@centos7-10 ~]#
[root@centos7-10 ~]# docker ps -a
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
0256581620e3 ubuntu22:latest "/bin/bash" 17 seconds ago Up 17 seconds ubuntu-122
52ab125ab578 ubuntu22:latest "/bin/bash" 27 seconds ago Up 27 seconds ubuntu-121
[root@centos7-10 ~]#
- 在外部Linux bridge开启hairpin的情况下，测试容器间网络通讯
- ubuntu-121（192.168.30.121）Ping ubuntu-122（192.168.30.122），不通
// 外部bridge端口开启hairpin
[root@centos7-10 ~]# bridge -d link
4: veth1_br state UP @veth1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 master br0 state forwarding priority 32 cost 2
hairpin on guard off root_block off fastleave off learning on flood on mcast_flood on
[root@centos7-10 ~]#
// 验证容器之间IP连通
// 容器ubuntu-121 IP 192.168.30.121
[root@centos7-10 ~]# docker exec -it ubuntu-121 ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
10: eth0@if5: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default
link/ether 02:42:c0:a8:1e:79 brd ff:ff:ff:ff:ff:ff link-netnsid 0
inet 192.168.30.121/24 brd 192.168.30.255 scope global eth0
valid_lft forever preferred_lft forever
// ping 192.168.30.122，不通
[root@centos7-10 ~]# docker exec -it ubuntu-121 ping 192.168.30.122 -c3
PING 192.168.30.122 (192.168.30.122): 56 data bytes
92 bytes from 52ab125ab578 (192.168.30.121): Destination Host Unreachable
92 bytes from 52ab125ab578 (192.168.30.121): Destination Host Unreachable
92 bytes from 52ab125ab578 (192.168.30.121): Destination Host Unreachable
--- 192.168.30.122 ping statistics ---
3 packets transmitted, 0 packets received, 100% packet loss
[root@centos7-10 ~]#
6.5.4 Macvlan Passthru模式
Macvlan Passthru模式下，使虚拟机可以直接连接到物理接口上（见下图）。（引自: （译文）Bridge vs Macvlan）
- 创建macvlan passthru网络
// 创建macvlan passthru网络
[root@centos7-22 ~]# docker network create -d macvlan --subnet 10.211.55.0/24 --gateway 10.211.55.1 --aux-address used_ip1=10.211.55.22 -o parent=enp0s5 -o macvlan_mode=passthru macvlan_passthru
3d217a153d2ba70fe238f60567d16db0309eacb8786b5d2cd892a374530c6e84
[root@centos7-22 ~]# docker network ls
NETWORK ID NAME DRIVER SCOPE
6407a11e8d1a bridge bridge local
65a57293726b host host local
3d217a153d2b macvlan_passthru macvlan local
2db17c1087bf none null local
[root@centos7-22 ~]#
- 创建容器加入macvlan passthru网络
请务必保证可以通过终端登录，创建容器时会断开当前网络连接！
// An highlighted block
[root@centos7-22 ~]# docker run -itd --rm --network macvlan_passthru --ip 10.211.55.131 --name ubuntu-131 ubuntu22:latest /bin/bash
da50b093bcc41ccf926318acd760f86d0588e9a5c6ef1e4b09d7c0203e3374e2
[root@centos7-22 ~]#
- 终端登录容器测试网络
- 查看容器IP 10.211.55.131
- ping 网关 10.211.55.1，通
- ping 外网，通
7. Ipvlan
Ipvlan网络模式是另一种容器网络模式，它允许Docker容器使用主机的IP地址来访问网络。不同于Macvlan模式，Ipvlan模式中的容器不需要使用主机的物理网卡，而是通过虚拟的MAC地址来访问网络。Ipvlan模式的优点是配置简单，而且性能也不错。
Ipvlan网络模式支持三种模式：L2模式、L3模式、L3S模式。L2模式使用与Macvlan模式相似的技术，允许容器与主机在同一子网内。L3模式则允许容器与主机在不同的子网内，需要通过路由器进行通信。（引自: MCNU云原生文章）。L3S模式启用了Iptables conntrack 连接跟踪。
Ipvlan网络模式支持三种flag：bridge、private、vepa。与macvlan网络类似，默认是bridge。
关于 VEPA/passthrough/bridge/private
IPVlan 和 MACVlan 为什么会有各种模式和 flag，比如 VEPA，private，passthrough 等等？它们的区别在哪里？其实在内核的分析中，我们已经大致了解了这几种模式的表现，假如主设备是一个聊天群，所有的群成员都可以向外发消息，那么几种模式表现形式如下：
1.private 模式，群成员之间禁言的，既不能在群内聊天，也不能在群外聊天。
2.bridge 模式，任何群成员都可以在群内聊天。
3.VEPA 模式，群内禁止群成员聊天，但是群成员可以在群外私聊。
4.passthrough模式 ，只有一人是群主了，只有群主能发言。
那么为什么会有这几种模式呢？我们从内核的表现来看，无论是 port，还是 bridge，其实都是网络的概念，也就是说，从一开始，Linux 就在努力将自己表现成一个合格的网络设备，对于主设备，Linux 努力将它做成一个交换机，对于子设备，那就是一个个网线背后的设备，这样看起来就很合理了。实际上正是这样，无论是 VEPA 还是 private，它们最初都是网络概念。其实不止是 Linux，我们见到很多致力于把自己伪装成物理网络的项目，都沿袭了这些行为模式。（引自: 从 VLAN 到 IPVLAN: 聊聊虚拟网络设备及其在云原生中的应用）
Ipvlan 要求 linux 内核版本 v4.2+
Linux下测试可参见我另一篇笔记：Linux ipvlan详解（l2、l3、l3s和bridge、private和vepa模式）
7.1 创建网络-Ipvlan
7.1.1 创建Ipvlan网络
root@ubuntu22-25:~# docker network create -d ipvlan \
> --subnet 10.211.55.0/24 \
> --gateway 10.211.55.1 \ // l2不指定默认网络使用第1个IP，l3忽略gateway参数
> -o parent=enp0s5 \ // 不指定等同于--internal
> -o ipvlan_mode=l2 \ // 默认l2，共l2、l3、l3s三种
> -o ipvlan_flag=bridge \ // 默认bridge，共bridge、private、vepa三种
> ipvlan_net
da7d273d2b4e51b75f659efd4fa96e3a9e9c1614c26befe5e9e1be81ef265898
- 说明
- 参数 --gateway
- l2不指定时默认使用网络第1个IP做网关，
- l3忽略gateway参数
- 参数 --ipvlan_mode 默认l2，共l2、l3、l3s三种
- 参数 --ipvlan_flag 默认bridge，共bridge、private、vepa三种
- 参数 -o parent 与–internal互斥，不使用-o parent 与 --internal 结果相同
- 参数 --internal 与-o parent互斥，创建一个 netlink 类型的假的（dummy）父接口，并将其作为父接口使用，有效地隔离了网络
- 参数 --gateway
7.1.2 一个物理接口只能创建一个Ipvlan网络
- 一个网卡（父接口）只能创建一个ipvlan网络，如果重复创建，会报错！
可以通过创建多个vlan子网卡的方式，创建多个ipvlan网络
// 一块网卡上只能创建一个ipvlan网络
root@ubuntu22-25:~# docker network create -d ipvlan --subnet 192.168.10.0/24 -o parent=enp0s5 -o ipvlan_mode=l2 ipvl2_bridge
Error response from daemon: network di-db618466a31c is already using parent interface enp0s5
7.1.3 创建IPvlan 802.1q 中继网络
root@ubuntu22-25:~# docker network create -d ipvlan \
> --subnet 10.211.55.0/24 \
> --gateway 10.211.55.1 \
> -o parent=enp0s5.1 \ // 使用“网卡.vlanid”的方式定义中继网络父接口
> -o ipvlan_mode=l2 \
> -o ipvlan_flag=bridge \
> ipvlan_net
da7d273d2b4e51b75f659efd4fa96e3a9e9c1614c26befe5e9e1be81ef265898
- 说明
- 参数 -o parent： 使用enp0s5网卡，创建vlanid=1的子网卡（enp0s5.1），作为ipvlan网络（ipvlan_net）的父接口。
// 宿主机多出了enp0s5.1网卡，查看enp0s5和enp0s5.1网卡
root@ubuntu22-25:~# ip a show | grep -A5 -E 'enp0s5|enp0s5.1'
2: enp0s5: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
link/ether 00:1c:42:1c:d1:0f brd ff:ff:ff:ff:ff:ff
inet 10.211.55.25/24 brd 10.211.55.255 scope global enp0s5
valid_lft forever preferred_lft forever
inet6 fdb2:2c26:f4e4:0:21c:42ff:fe1c:d10f/64 scope global dynamic mngtmpaddr noprefixroute
valid_lft 2591800sec preferred_lft 604600sec
inet6 fe80::21c:42ff:fe1c:d10f/64 scope link
valid_lft forever preferred_lft forever
--
9: enp0s5.1@enp0s5: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default
link/ether 00:1c:42:1c:d1:0f brd ff:ff:ff:ff:ff:ff
inet6 fe80::21c:42ff:fe1c:d10f/64 scope link
valid_lft forever preferred_lft forever
root@ubuntu22-25:~#
// 查看/proc/net/vlan/，多了一个enp0s5.1文件
root@ubuntu22-25:~# cat /proc/net/vlan/
config enp0s5.1
// 查看enp0s5.1文件，VID：1
root@ubuntu22-25:~# cat /proc/net/vlan/enp0s5.1
enp0s5.1 VID: 1 REORDER_HDR: 1 dev->priv_flags: 1021
total frames received 0
total bytes received 0
Broadcast/Multicast Rcvd 0
total frames transmitted 9
total bytes transmitted 726
Device: enp0s5
INGRESS priority mappings: 0:0 1:0 2:0 3:0 4:0 5:0 6:0 7:0
EGRESS priority mappings:
// 查看config文件，VLAN ID 是1
root@ubuntu22-25:~# cat /proc/net/vlan/config
VLAN Dev name | VLAN ID
Name-Type: VLAN_NAME_TYPE_RAW_PLUS_VID_NO_PAD
enp0s5.1 | 1 | enp0s5
- 也可以手动创建vlan子网卡（enp0s5_vlan2 ），vlanid=2，创建docker ipvlan网络时使用
注意：手动创建的vlan子网卡，需要手动删除
root@ubuntu22-25:~# ip link add link enp0s5 name enp0s5_vlan2 type vlan id 2
root@ubuntu22-25:~# cat /proc/net/vlan/
config enp0s5_vlan2
root@ubuntu22-25:~# cat /proc/net/vlan/config
VLAN Dev name | VLAN ID
Name-Type: VLAN_NAME_TYPE_RAW_PLUS_VID_NO_PAD
enp0s5_vlan2 | 2 | enp0s5
root@ubuntu22-25:~#
7.2 创建网络-Ipvlan L2模式
Ipvlan L2 模式与macvlan的bridge模式类似。父接口充当父接口与子接口之间的交换机。同一父接口下相同子网的虚拟机和容器能够通过父接口直接进行通信。去往不同子网的流量将流出父接口发送至默认网关（物理路由器）。工作在L2模式下的Ipvlan会将广播包/组播包发送至所有子接口上。（引自: （译文）Macvlan vs Ipvlan）
7.2.1 Ipvlan l2 bridge模式
Docker官方文档 IPvlan network driver
- 创建ipvlan l2 bridge网络：ipvlan_l2_bridge
root@ubuntu22-25:~# docker network create -d ipvlan --subnet 10.211.55.0/24 --gateway 10.211.55.1 -o parent=enp0s5 -o ipvlan_mode=l2 -o ipvlan_flag=bridge ipvlan_l2_bridge
e39e9d00f99e7059ad47a6a80a9a7443e3857b8eb09a94f71c43443fee02df7a
root@ubuntu22-25:~# docker network ls
NETWORK ID NAME DRIVER SCOPE
31e02b7dfeec bridge bridge local
52518594eab6 host host local
e39e9d00f99e ipvlan_l2_bridge ipvlan local
14101777743d none null local
root@ubuntu22-25:~#
- 运行两个容器：ubuntu-101和ubuntu-102，加入ipvlan_l2_bridge
- ubuntu-101 10.211.55.101
- ubuntu-102 10.211.55.102
root@ubuntu22-25:~# docker run -itd --rm --network ipvlan_l2_bridge --ip 10.211.55.101 --name ubuntu-101 ubuntu22:latest /bin/bash
12559b2e0da9d16f5b12c7b9f2bc2daf9aaf48fcd056df9c0a22d98d1cdbf6ca
root@ubuntu22-25:~#
root@ubuntu22-25:~# docker run -itd --rm --network ipvlan_l2_bridge --ip 10.211.55.102 --name ubuntu-102 ubuntu22:latest /bin/bash
d1584f4ec5a7b1fd74095d16056e102aaf515427c80dafb9d84667cf4935b97a
root@ubuntu22-25:~#
- 进入容器ubuntu-101
root@ubuntu22-25:~# docker ps -a
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
d1584f4ec5a7 ubuntu22:latest "/bin/bash" 27 seconds ago Up 27 seconds ubuntu-102
12559b2e0da9 ubuntu22:latest "/bin/bash" 3 hours ago Up 3 hours ubuntu-101
root@ubuntu22-25:~# docker exec -it ubuntu-101 /bin/bash
root@12559b2e0da9:/# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
4: eth0@if2: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default
link/ether 00:1c:42:1c:d1:0f brd ff:ff:ff:ff:ff:ff link-netnsid 0
inet 10.211.55.101/24 brd 10.211.55.255 scope global eth0
valid_lft forever preferred_lft forever
- ipvlan l2 bridge模式下，容器间可以直接通讯
- 容器ubuntu-101（10.211.55.101）Ping 10.211.55.102，通
root@12559b2e0da9:/# ping 10.211.55.102
PING 10.211.55.102 (10.211.55.102): 56 data bytes
64 bytes from 10.211.55.102: icmp_seq=0 ttl=64 time=0.166 ms
64 bytes from 10.211.55.102: icmp_seq=1 ttl=64 time=0.088 ms
^C--- 10.211.55.102 ping statistics ---
2 packets transmitted, 2 packets received, 0% packet loss
round-trip min/avg/max/stddev = 0.088/0.127/0.166/0.039 ms
- 容器ubuntu-101（10.211.55.101）Ping 10.211.55.1（外部网关），通
root@12559b2e0da9:/# ping 10.211.55.1
PING 10.211.55.1 (10.211.55.1): 56 data bytes
64 bytes from 10.211.55.1: icmp_seq=0 ttl=128 time=0.304 ms
64 bytes from 10.211.55.1: icmp_seq=1 ttl=128 time=0.450 ms
^C--- 10.211.55.1 ping statistics ---
2 packets transmitted, 2 packets received, 0% packet loss
round-trip min/avg/max/stddev = 0.304/0.377/0.450/0.073 ms
- 容器ubuntu-101（10.211.55.101）Ping 10.211.55.25（父网卡），不通
root@12559b2e0da9:/# ping 10.211.55.25 -c3
PING 10.211.55.25 (10.211.55.25): 56 data bytes
92 bytes from 12559b2e0da9 (10.211.55.101): Destination Host Unreachable
92 bytes from 12559b2e0da9 (10.211.55.101): Destination Host Unreachable
92 bytes from 12559b2e0da9 (10.211.55.101): Destination Host Unreachable
--- 192.168.10.1 ping statistics ---
3 packets transmitted, 0 packets received, 100% packet loss
7.2.2 Ipvlan l2 private模式
- 创建Ipvlan l2 private网络：ipvlan_l2_private
root@ubuntu22-25:~# docker network create -d ipvlan --subnet 10.211.55.0/24 --gateway 10.211.55.1 -o parent=enp0s5 -o ipvlan_mode=l2 -o ipvlan_flag=private ipvlan_l2_private
b68d389196b14bd0f6bd8d552f72e1d6050e180aebdcfc0f5da6628a6853dc2b
root@ubuntu22-25:~# docker network ls
NETWORK ID NAME DRIVER SCOPE
31e02b7dfeec bridge bridge local
52518594eab6 host host local
b68d389196b1 ipvlan_l2_private ipvlan local
14101777743d none null local
root@ubuntu22-25:~#
- 运行两个容器：ubuntu-111和ubuntu-112，加入ipvlan_l2_private
root@ubuntu22-25:~# docker run -itd --rm --network ipvlan_l2_private --ip 10.211.55.111 --name ubuntu-111 ubuntu22:latest /bin/bash
390f52af387fd665d5bfd74054e62bad0ce34178466235f2df1b9c98403b127b
root@ubuntu22-25:~# docker run -itd --rm --network ipvlan_l2_private --ip 10.211.55.112 --name ubuntu-112 ubuntu22:latest /bin/bash
09754f9749ac18b1c61ba6384c5c9c9f4cdbd9f4bb1b792104aeb0ffb3a0bcfc
root@ubuntu22-25:~#
- 进入容器ubuntu-111
- ubuntu-111（10.211.55.111） Ping 10.211.55.1（外部网关），通
- ubuntu-111（10.211.55.111） Ping 10.211.55.112（ubuntu-112），不通
// An highlighted block
root@ubuntu22-25:~# docker exec -it ubuntu-111 /bin/bash
root@390f52af387f:/# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
6: eth0@if2: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UNKNOWN group default
link/ether 00:1c:42:1c:d1:0f brd ff:ff:ff:ff:ff:ff link-netnsid 0
inet 10.211.55.111/24 brd 10.211.55.255 scope global eth0
valid_lft forever preferred_lft forever
// ubuntu-111 Ping 10.211.55.1（外部网关），通
root@390f52af387f:/# ping 10.211.55.1
PING 10.211.55.1 (10.211.55.1): 56 data bytes
64 bytes from 10.211.55.1: icmp_seq=0 ttl=128 time=0.519 ms
64 bytes from 10.211.55.1: icmp_seq=1 ttl=128 time=0.401 ms
^C--- 10.211.55.1 ping statistics ---
2 packets transmitted, 2 packets received, 0% packet loss
round-trip min/avg/max/stddev = 0.401/0.460/0.519/0.059 ms
// ubuntu-111 Ping 10.211.55.112（ubuntu-112），不通
root@390f52af387f:/# ping 10.211.55.112
PING 10.211.55.112 (10.211.55.112): 56 data bytes
92 bytes from 390f52af387f (10.211.55.111): Destination Host Unreachable
92 bytes from 390f52af387f (10.211.55.111): Destination Host Unreachable
92 bytes from 390f52af387f (10.211.55.111): Destination Host Unreachable
92 bytes from 390f52af387f (10.211.55.111): Destination Host Unreachable
^C--- 10.211.55.112 ping statistics ---
5 packets transmitted, 0 packets received, 100% packet loss
7.2.3 Ipvlan l2 vepa模式
验证步骤简述如下：
- 创建两对veth pair：veth1和veth1_br，veth2和veth2_br。其中，veth1_br和veth2_br加入Linux bridge br0中连通。
- 创建namespace ns1，将veth1加入其中，同时创建veth1的vlan子网卡：veth1.1，vlanid=1。veth1.1模拟隔离网络空间的外部网络。
- 使用veth2，创建docker ipvlan l2 vepa网络iplv2_vepa，父网卡为veth2.1（vlanid=1）。创建两个容器ubuntu-100和ubuntu-200加入网络ipvl2_vepa。
- 测试ubuntu-100和ubuntu-200访问外部网络中veth1.1（同属vlan 1），访问本地veth2.1（子网卡访问父网卡）
- 测试ubuntu-100访问ubuntu-200
具体如下图：
- 创建网络命名空间（namespace）ns1
root@ubuntu22-25:~# ip netns add ns1
- 创建两对veth pair：veth1和veth1_br，veth2和veth2_br
root@ubuntu22-25:~# ip link add veth1 type veth peer veth1_br
root@ubuntu22-25:~# ip link add veth2 type veth peer veth2_br
- 创建Linux bridge：br0，将veth1_br和veth2_br加入br0
root@ubuntu22-25:~# brctl addbr br0
root@ubuntu22-25:~# brctl addif br0 veth1_br
root@ubuntu22-25:~# brctl addif br0 veth2_br
- 启用veth1、veth2、veth1_br、veth2_br和br0
root@ubuntu22-25:~# ip link set veth1 up
root@ubuntu22-25:~# ip link set veth1_br up
root@ubuntu22-25:~# ip link set veth2 up
root@ubuntu22-25:~# ip link set veth2_br up
root@ubuntu22-25:~# ip link set br0 up
- veth1加入ns1，创建veth1的vlan子网卡veth1.1 vlanid=1，配置IP 192.168.10.254
- 启用veth1、veth1.1、lo网卡
root@ubuntu22-25:~# ip link set veth1 netns ns1
root@ubuntu22-25:~# ip netns exec ns1 ip link add link veth1 name veth1.1 type vlan id 1
root@ubuntu22-25:~# ip netns exec ns1 ip addr add 192.168.10.254/24 dev veth1.1
root@ubuntu22-25:~# ip netns exec ns1 ip link set lo up
root@ubuntu22-25:~# ip netns exec ns1 ip link set veth1 up
root@ubuntu22-25:~# ip netns exec ns1 ip link set veth1.1 up
root@ubuntu22-25:~# ip netns exec ns1 ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: veth1.1@veth1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default qlen 1000
link/ether 9a:72:89:8d:28:29 brd ff:ff:ff:ff:ff:ff
inet 192.168.10.254/24 scope global veth1.1
valid_lft forever preferred_lft forever
inet6 fe80::9872:89ff:fe8d:2829/64 scope link
valid_lft forever preferred_lft forever
4: veth1@if3: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default qlen 1000
link/ether 9a:72:89:8d:28:29 brd ff:ff:ff:ff:ff:ff link-netnsid 0
inet6 fe80::9872:89ff:fe8d:2829/64 scope link
valid_lft forever preferred_lft forever
- 创建docker ipvlan网络ipvl2_vepa，mode l2、flag vepa
- 父网卡veth2.1（vlanid=1）
- 子网192.168.10.0/24，网关192.168.10.1
// 创建docker ipvlan网络ipvl2_vepa
root@ubuntu22-25:~# docker network create -d ipvlan --subnet 192.168.10.0/24 --gateway 192.168.10.1 -o parent=veth2.1 -o ipvlan_mode=l2 -o ipvlan_flag=vepa ipvl2_vepa
6a8a389eade975bc7e8b341dcd2c258fefcf8beba7430d9446bcc5316794ce79
// 可以查看veth2.1的vlanid是1
root@ubuntu22-25:~# cat /proc/net/vlan/config
VLAN Dev name | VLAN ID
Name-Type: VLAN_NAME_TYPE_RAW_PLUS_VID_NO_PAD
veth2.1 | 1 | veth2
root@ubuntu22-25:~#
root@ubuntu22-25:~# docker run -itd --rm --network ipvl2_vepa --ip 192.168.10.100 --name ubuntu-100 ubuntu22:latest /bin/bash
58d69ea743d7167225d98a88d4d0ff6f2afff6ca7fb49a15d3c1baea6d7c568e
root@ubuntu22-25:~# docker run -itd --rm --network ipvl2_vepa --ip 192.168.10.200 --name ubuntu-200 ubuntu22:latest /bin/bash
fc4c80b1bf8cb0804041a11a9344e88a77669834c40a9dc5f43fafed918eb7f1
- 查看docker ipvlan网络ipvl2_vepa
// An highlighted block
root@ubuntu22-25:~# docker network inspect ipvl2_vepa
[
{
"Name": "ipvl2_vepa",
"Id": "6a8a389eade975bc7e8b341dcd2c258fefcf8beba7430d9446bcc5316794ce79",
"Created": "2024-02-17T12:20:57.017116076+08:00",
"Scope": "local",
"Driver": "ipvlan",
"EnableIPv6": false,
"IPAM": {
"Driver": "default",
"Options": {},
"Config": [
{
"Subnet": "192.168.10.0/24",
"Gateway": "192.168.10.1"
}
]
},
"Internal": false,
"Attachable": false,
"Ingress": false,
"ConfigFrom": {
"Network": ""
},
"ConfigOnly": false,
"Containers": {
"58d69ea743d7167225d98a88d4d0ff6f2afff6ca7fb49a15d3c1baea6d7c568e": {
"Name": "ubuntu-100",
"EndpointID": "2b08d53503cb33ce8aebefe50d18d58523373f2bb857162487fba22d390c85d8",
"MacAddress": "",
"IPv4Address": "192.168.10.100/24",
"IPv6Address": ""
},
"fc4c80b1bf8cb0804041a11a9344e88a77669834c40a9dc5f43fafed918eb7f1": {
"Name": "ubuntu-200",
"EndpointID": "9dc87b33438d4048261d67cbaee713ebe9c28b991f0fb069d774cf12b986ac43",
"MacAddress": "",
"IPv4Address": "192.168.10.200/24",
"IPv6Address": ""
}
},
"Options": {
"ipvlan_flag": "vepa",
"ipvlan_mode": "l2",
"parent": "veth2.1"
},
"Labels": {}
}
]
- 配置docker ipvlan网络父网卡veth2.1的IP 192.168.10.1
// An highlighted block
root@ubuntu22-25:~# ip addr add 192.168.10.1/24 dev veth2.1
root@ubuntu22-25:~# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: enp0s5: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
link/ether 00:1c:42:1c:d1:0f brd ff:ff:ff:ff:ff:ff
inet 10.211.55.25/24 brd 10.211.55.255 scope global enp0s5
valid_lft forever preferred_lft forever
inet6 fdb2:2c26:f4e4:0:21c:42ff:fe1c:d10f/64 scope global dynamic mngtmpaddr noprefixroute
valid_lft 2591573sec preferred_lft 604373sec
inet6 fe80::21c:42ff:fe1c:d10f/64 scope link
valid_lft forever preferred_lft forever
3: veth1_br@if4: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue master br0 state UP group default qlen 1000
link/ether de:76:19:60:a5:38 brd ff:ff:ff:ff:ff:ff link-netns ns1
inet6 fe80::dc76:19ff:fe60:a538/64 scope link
valid_lft forever preferred_lft forever
5: veth2_br@veth2: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue master br0 state UP group default qlen 1000
link/ether 32:7a:42:e3:b0:ea brd ff:ff:ff:ff:ff:ff
inet6 fe80::307a:42ff:fee3:b0ea/64 scope link
valid_lft forever preferred_lft forever
6: veth2@veth2_br: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default qlen 1000
link/ether 3a:bf:7f:75:16:89 brd ff:ff:ff:ff:ff:ff
inet6 fe80::38bf:7fff:fe75:1689/64 scope link
valid_lft forever preferred_lft forever
7: br0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default qlen 1000
link/ether ca:19:42:6b:60:e5 brd ff:ff:ff:ff:ff:ff
inet6 fe80::c819:42ff:fe6b:60e5/64 scope link
valid_lft forever preferred_lft forever
8: docker0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc noqueue state DOWN group default
link/ether 02:42:6a:e9:a6:11 brd ff:ff:ff:ff:ff:ff
inet 172.17.0.1/16 brd 172.17.255.255 scope global docker0
valid_lft forever preferred_lft forever
10: veth2.1@veth2: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default
link/ether 3a:bf:7f:75:16:89 brd ff:ff:ff:ff:ff:ff
inet 192.168.10.1/24 scope global veth2.1
valid_lft forever preferred_lft forever
inet6 fe80::38bf:7fff:fe75:1689/64 scope link
valid_lft forever preferred_lft forever
- 默认br0不开启hairpin功能下测试
- ubuntu-100（192.168.10.100）Ping 192.168.10.254（veth1.1），通
- ubuntu-100（192.168.10.100）Ping 192.168.10.1（veth2.1），不通
- ubuntu-100（192.168.10.100）Ping 192.168.10.200（ubuntu-200），不通
// 进入ubuntu-100
root@ubuntu22-25:~# docker exec -it ubuntu-100 /bin/bash
root@58d69ea743d7:/# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
11: eth0@if10: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UNKNOWN group default
link/ether 3a:bf:7f:75:16:89 brd ff:ff:ff:ff:ff:ff link-netnsid 0
inet 192.168.10.100/24 brd 192.168.10.255 scope global eth0
valid_lft forever preferred_lft forever
// ubuntu-100（192.168.10.100）Ping 192.168.10.254（veth1.1），通
root@58d69ea743d7:/# ping 192.168.10.254
PING 192.168.10.254 (192.168.10.254): 56 data bytes
64 bytes from 192.168.10.254: icmp_seq=0 ttl=64 time=0.081 ms
64 bytes from 192.168.10.254: icmp_seq=1 ttl=64 time=0.116 ms
^C--- 192.168.10.254 ping statistics ---
2 packets transmitted, 2 packets received, 0% packet loss
round-trip min/avg/max/stddev = 0.081/0.099/0.116/0.000 ms
// ubuntu-100（192.168.10.100）Ping 192.168.10.1（veth2.1），不通
root@58d69ea743d7:/# ping 192.168.10.1 -c3
PING 192.168.10.1 (192.168.10.1): 56 data bytes
92 bytes from 58d69ea743d7 (192.168.10.100): Destination Host Unreachable
92 bytes from 58d69ea743d7 (192.168.10.100): Destination Host Unreachable
92 bytes from 58d69ea743d7 (192.168.10.100): Destination Host Unreachable
--- 192.168.10.1 ping statistics ---
3 packets transmitted, 0 packets received, 100% packet loss
// ubuntu-100（192.168.10.100）Ping 192.168.10.200（ubuntu-200），不通
root@58d69ea743d7:/# ping 192.168.10.200 -c3
PING 192.168.10.200 (192.168.10.200): 56 data bytes
92 bytes from 58d69ea743d7 (192.168.10.100): Destination Host Unreachable
92 bytes from 58d69ea743d7 (192.168.10.100): Destination Host Unreachable
92 bytes from 58d69ea743d7 (192.168.10.100): Destination Host Unreachable
--- 192.168.10.200 ping statistics ---
3 packets transmitted, 0 packets received, 100% packet loss
- 开启br0的hairpin功能再测试
- ubuntu-100（192.168.10.100）Ping 192.168.10.200（ubuntu-200），通
// 查看br0
root@ubuntu22-25:~# brctl show
bridge name bridge id STP enabled interfaces
br0 8000.ca19426b60e5 no veth1_br
veth2_br
docker0 8000.02426ae9a611 no
// 开启 br0下veth2_br的hairpin功能
root@ubuntu22-25:~# brctl hairpin br0 veth2_br on
// 查看开启结果
root@ubuntu22-25:~# bridge -d link
3: veth1_br@if4: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 master br0 state forwarding priority 32 cost 2
hairpin off guard off root_block off fastleave off learning on flood on mcast_flood on mcast_to_unicast off neigh_suppress off vlan_tunnel off isolated off
5: veth2_br@veth2: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 master br0 state forwarding priority 32 cost 2
hairpin on guard off root_block off fastleave off learning on flood on mcast_flood on mcast_to_unicast off neigh_suppress off vlan_tunnel off isolated off
7: br0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 master br0
8: docker0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 master docker0
// ubuntu-100（192.168.10.100）Ping 192.168.10.200（ubuntu-200），通
root@58d69ea743d7:/# ping 192.168.10.200 -c3
PING 192.168.10.200 (192.168.10.200): 56 data bytes
64 bytes from 192.168.10.200: icmp_seq=0 ttl=64 time=0.341 ms
64 bytes from 192.168.10.200: icmp_seq=1 ttl=64 time=0.108 ms
64 bytes from 192.168.10.200: icmp_seq=2 ttl=64 time=0.220 ms
--- 192.168.10.200 ping statistics ---
3 packets transmitted, 3 packets received, 0% packet loss
round-trip min/avg/max/stddev = 0.108/0.223/0.341/0.095 ms
7.3 创建网络-Ipvlan L3模式
工作在L3模式下的Ipvlan充当父接口和子接口之间的三层设备（路由器）。L3模式下的Ipvlan在子接口之间路由报文，提供3层网络之间的全连接。每个子接口都必须配置不同的子网网段。例如不可以在多个子接口上配置相同的10.10.40.0/24网段。因为广播报文是二层域的，所以他们不能在子接口上相互传递。Ipvlan也不支持多播。
Ipvlan L3不支持路由协议，因此也无法将连接的子网网段通知物理网络上的路由器。需要人为在物理路由器上为子接口上的子网网段配置指向父接口IP的静态路由。（引自: （译文）Macvlan vs Ipvlan）
7.3.1 Ipvlan L3 bridge模式
- 宿主机ubuntu22-25创建Ipvlan l3 bridge网络 ipvl_l3_bridge
- 父接口 enp0s5（10.211.55.25）
- 子网 192.168.10.0/24
- 子网 192.168.20.0/24
- 子网 192.168.30.0/24
- mode l3，flag bridge
// 查看宿主机ubuntu22-25的网卡enp0s5
root@ubuntu22-25:~# ip a show enp0s5
2: enp0s5: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
link/ether 00:1c:42:1c:d1:0f brd ff:ff:ff:ff:ff:ff
inet 10.211.55.25/24 brd 10.211.55.255 scope global enp0s5
valid_lft forever preferred_lft forever
inet6 fdb2:2c26:f4e4:0:21c:42ff:fe1c:d10f/64 scope global dynamic mngtmpaddr noprefixroute
valid_lft 2591662sec preferred_lft 604462sec
inet6 fe80::21c:42ff:fe1c:d10f/64 scope link
valid_lft forever preferred_lft forever
// 创建Ipvlan l3 bridge网络 ipvl_l3_bridge
root@ubuntu22-25:~# docker network create -d ipvlan \
> --subnet 192.168.10.0/24 \
> --subnet 192.168.20.0/24 \
> --subnet 192.168.30.0/24 \
> -o parent=enp0s5 \
> -o ipvlan_mode=l3 \
> -o ipvlan_flag=bridge \
> ipvl_l3_bridge
a693c97cf2cf050b45c782bebb3ae2a7138f93e948675fdae2a6448bbd62f33f
- 运行3个容器ubuntu-10、ubuntu-20和ubuntu-30，加入ipvl_l3_bridge
- ubuntu-10 IP 192.168.10.10
- ubuntu-20 IP 192.168.20.20
- ubuntu-30 IP 192.168.30.30
root@ubuntu22-25:~# docker run -itd --rm --network ipvl_l3_bridge --ip 192.168.10.10 --name ubuntu-10 ubuntu22:latest /bin/bash
246dcd2ba25da1b88afe8afbac7079ed5e0f0307a99b6dc19f041cc31d5d6523
root@ubuntu22-25:~# docker run -itd --rm --network ipvl_l3_bridge --ip 192.168.20.20 --name ubuntu-20 ubuntu22:latest /bin/bash
799f4c9b8d052827736c3b843cc31956c783155b0a1313ffb87743d7d25a384f
root@ubuntu22-25:~# docker run -itd --rm --network ipvl_l3_bridge --ip 192.168.30.30 --name ubuntu-30 ubuntu22:latest /bin/bash
8382edfe30e3b69d173778e2283b91c6d994191d62b402e4266123005b6aefe3
- Ipvlan l3 bridge模式下，不同子网的容器间可以通讯
// 进入容器ubuntu-10，192.168.10.10
root@ubuntu22-25:~# docker exec -it ubuntu-10 /bin/bash
root@246dcd2ba25d:/# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
9: eth0@if2: <BROADCAST,MULTICAST,NOARP,UP,LOWER_UP> mtu 1500 qdisc noqueue state UNKNOWN group default
link/ether 00:1c:42:1c:d1:0f brd ff:ff:ff:ff:ff:ff link-netnsid 0
inet 192.168.10.10/24 brd 192.168.10.255 scope global eth0
valid_lft forever preferred_lft forever
// Ping 192.168.20.20（ubuntu-20）,通
root@246dcd2ba25d:/# ping 192.168.20.20 -c2
PING 192.168.20.20 (192.168.20.20): 56 data bytes
64 bytes from 192.168.20.20: icmp_seq=0 ttl=64 time=0.081 ms
64 bytes from 192.168.20.20: icmp_seq=1 ttl=64 time=0.087 ms
--- 192.168.20.20 ping statistics ---
2 packets transmitted, 2 packets received, 0% packet loss
round-trip min/avg/max/stddev = 0.081/0.084/0.087/0.000 ms
// Ping 192.168.30.30（ubuntu-30）,通
root@246dcd2ba25d:/# ping 192.168.30.30 -c2
PING 192.168.30.30 (192.168.30.30): 56 data bytes
64 bytes from 192.168.30.30: icmp_seq=0 ttl=64 time=0.089 ms
64 bytes from 192.168.30.30: icmp_seq=1 ttl=64 time=0.093 ms
--- 192.168.30.30 ping statistics ---
2 packets transmitted, 2 packets received, 0% packet loss
round-trip min/avg/max/stddev = 0.089/0.091/0.093/0.000 ms
- Ipvlan l3 bridge模式下，容器不能直接访问ipvlan网络父接口
// Ping 10.211.55.25（ipvlan网络父接口enp0s5）,不通
root@799f4c9b8d05:/# ping 10.211.55.25 -c3
PING 10.211.55.25 (10.211.55.25): 56 data bytes
--- 10.211.55.25 ping statistics ---
3 packets transmitted, 0 packets received, 100% packet loss
root@799f4c9b8d05:/#
- 外部网络中宿主机 ubuntu22-24 增加静态路由访问容器
- 访问192.168.10.0/24，下一跳网关：10.211.55.25，接口设备：ubuntu22-24上网卡enp0s5
- 访问192.168.20.0/24，下一跳网关：10.211.55.25，接口设备：ubuntu22-24上网卡enp0s5
- 访问192.168.30.0/24，下一跳网关：10.211.55.25，接口设备：ubuntu22-24上网卡enp0s5
root@ubuntu22-24:~# ip route add 192.168.10.0/24 via 10.211.55.25 dev enp0s5
root@ubuntu22-24:~# ip route add 192.168.20.0/24 via 10.211.55.25 dev enp0s5
root@ubuntu22-24:~# ip route add 192.168.30.0/24 via 10.211.55.25 dev enp0s5
// 查看添加结果
root@ubuntu22-24:~# ip route
default via 10.211.55.1 dev enp0s5 proto static
10.211.55.0/24 dev enp0s5 proto kernel scope link src 10.211.55.24
192.168.10.0/24 via 10.211.55.25 dev enp0s5
192.168.20.0/24 via 10.211.55.25 dev enp0s5
192.168.30.0/24 via 10.211.55.25 dev enp0s5
- 宿主机ubuntu22-24可以访问宿主机ubuntu22-24上的容器
// Ping 容器ubuntu-10（192.168.10.10），通
root@ubuntu22-24:~# ping 192.168.10.10 -c2
PING 192.168.10.10 (192.168.10.10): 56 data bytes
64 bytes from 192.168.10.10: icmp_seq=0 ttl=64 time=0.509 ms
64 bytes from 192.168.10.10: icmp_seq=1 ttl=64 time=0.542 ms
--- 192.168.10.10 ping statistics ---
2 packets transmitted, 2 packets received, 0% packet loss
round-trip min/avg/max/stddev = 0.509/0.526/0.542/0.000 ms
// Ping 容器ubuntu-20（192.168.20.20），通
root@ubuntu22-24:~# ping 192.168.20.20 -c2
PING 192.168.20.20 (192.168.20.20): 56 data bytes
64 bytes from 192.168.20.20: icmp_seq=0 ttl=64 time=0.463 ms
64 bytes from 192.168.20.20: icmp_seq=1 ttl=64 time=0.544 ms
--- 192.168.20.20 ping statistics ---
2 packets transmitted, 2 packets received, 0% packet loss
round-trip min/avg/max/stddev = 0.463/0.504/0.544/0.041 ms
// Ping 容器ubuntu-30（192.168.30.30），通
root@ubuntu22-24:~# ping 192.168.30.30 -c2
PING 192.168.30.30 (192.168.30.30): 56 data bytes
64 bytes from 192.168.30.30: icmp_seq=0 ttl=64 time=0.425 ms
64 bytes from 192.168.30.30: icmp_seq=1 ttl=64 time=0.582 ms
--- 192.168.30.30 ping statistics ---
2 packets transmitted, 2 packets received, 0% packet loss
round-trip min/avg/max/stddev = 0.425/0.503/0.582/0.079 ms
- 容器访问外部网络宿主机ubuntu22-24（10.211.55.24），经父接口（宿主机ubuntu22-25的enp0s5）转发通讯
// 进入容器ubuntu-20
root@ubuntu22-25:~# docker exec -it ubuntu-20 /bin/bash
root@799f4c9b8d05:/# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
10: eth0@if2: <BROADCAST,MULTICAST,NOARP,UP,LOWER_UP> mtu 1500 qdisc noqueue state UNKNOWN group default
link/ether 00:1c:42:1c:d1:0f brd ff:ff:ff:ff:ff:ff link-netnsid 0
inet 192.168.20.20/24 brd 192.168.20.255 scope global eth0
valid_lft forever preferred_lft forever
// 容器Ping 10.211.55.24（宿主机ubuntu22-24网卡enp0s5），通
root@799f4c9b8d05:/# ping 10.211.55.24
PING 10.211.55.24 (10.211.55.24): 56 data bytes
64 bytes from 10.211.55.24: icmp_seq=0 ttl=64 time=0.664 ms
64 bytes from 10.211.55.24: icmp_seq=1 ttl=64 time=0.543 ms
^C--- 10.211.55.24 ping statistics ---
2 packets transmitted, 2 packets received, 0% packet loss
round-trip min/avg/max/stddev = 0.543/0.604/0.664/0.061 ms
7.3.2 Ipvlan L3 private模式
- 宿主机ubuntu22-25创建Ipvlan L3 private网络ipvl_l3_private
- 子网 192.168.10.0/24
- 子网 192.168.20.0/24
- mode l3，flag private
root@ubuntu22-25:~# docker network create -d ipvlan --subnet 192.168.10.0/24 --subnet 192.168.20.0/24 -o parent=enp0s5 -o ipvlan_mode=l3 -o ipvlan_flag=private ipvl_l3_private
424a2dcf1bb0f547281ccaa447a9885d23ec33bc2fa6e462cbdb9bcf59a1e59b
- 运行3个容器ubuntu-10、ubuntu-20和ubuntu-22，加入ipvl_l3_private
- ubuntu-10 192.168.10.10
- ubuntu-20 192.168.20.10
- ubuntu-21 192.168.20.21
// 运行容器ubuntu-10、ubuntu-20和ubuntu-22，加入ipvl_l3_private
root@ubuntu22-25:~# docker run -itd --rm --network ipvl_l3_private --ip 192.168.10.10 --name ubuntu-10 ubuntu22:latest /bin/bash
248e9cbe1d81444ee4ac97312d927ccbffc2023b4a3fd43dc55954ff5e656cfd
root@ubuntu22-25:~# docker run -itd --rm --network ipvl_l3_private --ip 192.168.20.20 --name ubuntu-20 ubuntu22:latest /bin/bash
32bcaec46aa63d89da60452fa66a51709de092216397ceefc99a8a69ef348051
root@ubuntu22-25:~# docker run -itd --rm --network ipvl_l3_private --ip 192.168.20.21 --name ubuntu-21 ubuntu22:latest /bin/bash
63f36b6d28bded8e003d6a9a7cc4e8d1ef392139cd5a22381624b6460df36f10
// 查看运行结果
root@ubuntu22-25:~# docker ps -a
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
63f36b6d28bd ubuntu22:latest "/bin/bash" 5 seconds ago Up 4 seconds ubuntu-21
32bcaec46aa6 ubuntu22:latest "/bin/bash" 16 seconds ago Up 15 seconds ubuntu-20
248e9cbe1d81 ubuntu22:latest "/bin/bash" 42 seconds ago Up 41 seconds ubuntu-10
- ipvlan l3 private模式下，容器间不能通讯
- 容器ubuntu-20（192.168.20.20）Ping 192.168.20.21（容器ubuntu-21），不通
- 容器ubuntu-20（192.168.20.20）Ping 192.168.10.10（容器ubuntu-10），不通
说明：ipvlan l3模式下会丢弃所有广播和多播流量。如果ipvlan 的flag是prviate，那么其它包也被丢弃。 具体可参见我另一篇笔记：Linux ipvlan详解（l2、l3、l3s和bridge、private和vepa模式）
root@ubuntu22-25:~# docker exec -it ubuntu-20 /bin/bash
root@32bcaec46aa6:/# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
6: eth0@if2: <BROADCAST,MULTICAST,NOARP,UP,LOWER_UP> mtu 1500 qdisc noqueue state UNKNOWN group default
link/ether 00:1c:42:1c:d1:0f brd ff:ff:ff:ff:ff:ff link-netnsid 0
inet 192.168.20.20/24 brd 192.168.20.255 scope global eth0
valid_lft forever preferred_lft forever
root@32bcaec46aa6:/# ping 129.168.20.21
PING 129.168.20.21 (129.168.20.21): 56 data bytes
^C--- 129.168.20.21 ping statistics ---
2 packets transmitted, 0 packets received, 100% packet loss
root@32bcaec46aa6:/# ping 129.168.10.10
PING 129.168.10.10 (129.168.10.10): 56 data bytes
^C--- 129.168.10.10 ping statistics ---
5 packets transmitted, 0 packets received, 100% packet loss
- ipvlan l3 private模式下，容器不能直接访问宿主机父接口
// Ping 10.211.55.25（宿主机ubuntu22-25上父接口enp0s5），不通
root@32bcaec46aa6:/# ping 10.211.55.25
PING 10.211.55.25 (10.211.55.25): 56 data bytes
^C--- 10.211.55.25 ping statistics ---
3 packets transmitted, 0 packets received, 100% packet loss
- 外部网络中宿主机ubuntu22-24增加静态路由访问容器
- 访问192.168.10.0/24，下一跳网关：10.211.55.25，接口设备：ubuntu22-24上网卡enp0s5
- 访问192.168.20.0/24，下一跳网关：10.211.55.25，接口设备：ubuntu22-24上网卡enp0s5
// 配置静态路由
root@ubuntu22-24:~# ip route add 192.168.10.0/24 via 10.211.55.25 dev enp0s5
root@ubuntu22-24:~# ip route add 192.168.20.0/24 via 10.211.55.25 dev enp0s5
// 查看配置结果
root@ubuntu22-24:~# ip route
default via 10.211.55.1 dev enp0s5 proto static
10.211.55.0/24 dev enp0s5 proto kernel scope link src 10.211.55.24
192.168.10.0/24 via 10.211.55.25 dev enp0s5
192.168.20.0/24 via 10.211.55.25 dev enp0s5
- 容器访问外部网络宿主机ubuntu22-24（10.211.55.24），经父接口（宿主机ubuntu22-25的enp0s5）转发通讯
// 进入容器ubuntu-20
root@ubuntu22-25:~# docker exec -it ubuntu-20 /bin/bash
// Ping 10.211.55.24(外部宿主机ubuntu22-24),通
root@32bcaec46aa6:/# ping 10.211.55.24
PING 10.211.55.24 (10.211.55.24): 56 data bytes
64 bytes from 10.211.55.24: icmp_seq=0 ttl=64 time=0.498 ms
64 bytes from 10.211.55.24: icmp_seq=1 ttl=64 time=0.530 ms
^C--- 10.211.55.24 ping statistics ---
2 packets transmitted, 2 packets received, 0% packet loss
round-trip min/avg/max/stddev = 0.498/0.514/0.530/0.000 ms
7.3.3 Ipvlan l3 vepa模式
验证步骤简述如下：
- 在宿主机ubuntu22-25上，创建docker ipvlan l3 vepa网络ipvl_l3_vepa，父接口使用enp0s5，子网包括：172.16.1.0/24和172.16.2.0/24。创建3个容器ubuntu-10（172.16.1.10）、ubuntu-20（172.16.2.20）和ubuntu-21（172.16.2.21）加入网络ipvl_l3_vepa。
- 在宿主机ubuntu22-24上，增加访问ipvl_l3_vepa的静态路由
- 测试3个容器可以访问外部宿主机ubuntu22-24，彼此之间不能访问，不能直接访问父接口enp0s5
- 在宿主机ubuntu22-25上增加静态路由，将172.16.1.0/24和172.16.2.0/24指向宿主机ubuntu22-24。宿主机ubuntu22-24开启三层转发功能
- 测试3个容器经过父接口enp0s5外发，经宿主机ubuntu22-24三层转发后，可以互相通讯
- 在宿主机ubuntu22-25上创建ipvlan l3 vepa网络ipvl_l3_vepa
* 父接口enp0s5
* mode l3、flag vepa
* 子网172.16.1.0/24 和 172.16.2.0/24
root@ubuntu22-25:~# docker network create -d ipvlan --subnet 172.16.1.0/24 --subnet 172.16.2.0/24 -o parent=enp0s5 -o ipvlan_mode=l3 -o ipvlan_flag=vepa ipvl_l3_vepa
ea405e95d92f1b20539b560dd531a31f5133d4ffc43ae2372be870cb84b7b79e
- 创建3个容器ubuntu-10、ubuntu-20和ubuntu-21，加入ipvl_l3_vepa
root@ubuntu22-25:~# docker run -itd --rm --network ipvl_l3_vepa --ip 172.16.1.10 --name ubuntu-10 ubuntu22:latest /bin/bash
af46625f7dfb56d053a4fa46cf7811d25358c370d9c5f0e1cbdf06e5690ed0f1
root@ubuntu22-25:~# docker run -itd --rm --network ipvl_l3_vepa --ip 172.16.2.20 --name ubuntu-20 ubuntu22:latest /bin/bash
2bcc44b7066581ad5187ce58fe4debb47bcf326f2250c59e21171c2d877bc06f
root@ubuntu22-25:~# docker run -itd --rm --network ipvl_l3_vepa --ip 172.16.2.21 --name ubuntu-21 ubuntu22:latest /bin/bash
b3d617bf413cc3187b1ec3b80677b16bd5fbea99a35a920ef97ddded40cc660d
// 查看创建结果
root@ubuntu22-25:~# docker ps -a
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
b3d617bf413c ubuntu22:latest "/bin/bash" 10 seconds ago Up 9 seconds ubuntu-21
2bcc44b70665 ubuntu22:latest "/bin/bash" 27 seconds ago Up 26 seconds ubuntu-20
af46625f7dfb ubuntu22:latest "/bin/bash" 5 minutes ago Up 5 minutes ubuntu-10
- 进入容器ubuntu-20（172.16.2.20），测试ipvlan l3 vepa网络默认通讯情况
- 容器间不能互相通讯，Ping ubuntu-21（172.16.2.21），不通
- 容器间不能互相通讯，Ping ubuntu-10（172.16.1.10），不通
- 容器不能直接与父接口通讯，Ping enp0s5（10.211.55.25），不通
// 进入容器ubuntu-20
root@ubuntu22-25:~# docker exec -it ubuntu-20 /bin/bash
root@2bcc44b70665:/# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
10: eth0@if2: <BROADCAST,MULTICAST,NOARP,UP,LOWER_UP> mtu 1500 qdisc noqueue state UNKNOWN group default
link/ether 00:1c:42:1c:d1:0f brd ff:ff:ff:ff:ff:ff link-netnsid 0
inet 172.16.2.20/24 brd 172.16.2.255 scope global eth0
valid_lft forever preferred_lft forever
// 容器间不能互相通讯
root@2bcc44b70665:/# ping 172.16.2.21 -c2
PING 172.16.2.21 (172.16.2.21): 56 data bytes
--- 172.16.2.21 ping statistics ---
2 packets transmitted, 0 packets received, 100% packet loss
// 容器间不能互相通讯
root@2bcc44b70665:/# ping 172.16.1.10 -c2
PING 172.16.1.10 (172.16.1.10): 56 data bytes
--- 172.16.1.10 ping statistics ---
2 packets transmitted, 0 packets received, 100% packet loss
// 容器不能直接与父接口通讯
root@2bcc44b70665:/# ping 10.211.55.25 -c2
PING 10.211.55.25 (10.211.55.25): 56 data bytes
--- 10.211.55.25 ping statistics ---
2 packets transmitted, 0 packets received, 100% packet loss
- 在宿主机ubuntu22-24上，增加访问ipvl_l3_vepa的静态路由
// 增加2条静态路由
root@ubuntu22-24:~# ip route add 172.16.1.0/24 via 10.211.55.25 dev enp0s5
root@ubuntu22-24:~# ip route add 172.16.2.0/24 via 10.211.55.25 dev enp0s5
// 查看路由表
root@ubuntu22-24:~# ip route
default via 10.211.55.1 dev enp0s5 proto static
10.211.55.0/24 dev enp0s5 proto kernel scope link src 10.211.55.24
172.16.1.0/24 via 10.211.55.25 dev enp0s5
172.16.2.0/24 via 10.211.55.25 dev enp0s5
- 容器ubuntu-20测试访问外部宿主机ubuntu22-24，通
root@ubuntu22-25:~# docker exec -it ubuntu-20 ping 10.211.55.24 -c3
PING 10.211.55.24 (10.211.55.24): 56 data bytes
64 bytes from 10.211.55.24: icmp_seq=0 ttl=64 time=0.605 ms
64 bytes from 10.211.55.24: icmp_seq=1 ttl=64 time=0.566 ms
64 bytes from 10.211.55.24: icmp_seq=2 ttl=64 time=0.531 ms
--- 10.211.55.24 ping statistics ---
3 packets transmitted, 3 packets received, 0% packet loss
round-trip min/avg/max/stddev = 0.531/0.567/0.605/0.030 ms
- 宿主机ubuntu22-25上增加静态路由，将172.16.1.0/24和172.16.2.0/24指向ubuntu22-24
- ubuntu22-24开启三层转发，测试容器经过外部路由转发通讯
// 添加静态路由
root@ubuntu22-25:~# ip route add 172.16.1.0/24 via 10.211.55.24 dev enp0s5
root@ubuntu22-25:~# ip route add 172.16.2.0/24 via 10.211.55.24 dev enp0s5
root@ubuntu22-25:~# ip route
default via 10.211.55.1 dev enp0s5 proto static
10.211.55.0/24 dev enp0s5 proto kernel scope link src 10.211.55.25
172.16.1.0/24 via 10.211.55.24 dev enp0s5
172.16.2.0/24 via 10.211.55.24 dev enp0s5
172.17.0.0/16 dev docker0 proto kernel scope link src 172.17.0.1 linkdown
// ubuntu22-24开启三层转发
root@ubuntu22-24:~# sysctl -w net.ipv4.ip_forward=1
net.ipv4.ip_forward = 1
root@ubuntu22-24:~#
- 容器ubuntu-20经父接口enp0s5外发、ubuntu22-24三层转发，与ipvl_l3_vepa下其它容器通讯
- 容器ubuntu-20–>父接口enp0s5–>ubuntu22-24三层转发–>父接口enp0s5–>ubuntu-10。Ping 172.16.1.10，通
- 容器ubuntu-20–>父接口enp0s5–>ubuntu22-24三层转发–>父接口enp0s5–>ubuntu-21。Ping 172.16.2.21，通
// ping ubuntu-10（172.16.1.10），通
root@ubuntu22-25:~# docker exec -it ubuntu-20 ping 172.16.1.10
PING 172.16.1.10 (172.16.1.10): 56 data bytes
92 bytes from 10.211.55.24: Redirect Host
64 bytes from 172.16.1.10: icmp_seq=0 ttl=63 time=89.767 ms
64 bytes from 172.16.1.10: icmp_seq=1 ttl=63 time=0.783 ms
64 bytes from 172.16.1.10: icmp_seq=2 ttl=63 time=0.807 ms
^C--- 172.16.1.10 ping statistics ---
3 packets transmitted, 3 packets received, 0% packet loss
round-trip min/avg/max/stddev = 0.783/30.452/89.767/41.942 ms
// ping ubuntu-10（172.16.2.21），通
root@ubuntu22-25:~# docker exec -it ubuntu-20 ping 172.16.2.21
PING 172.16.2.21 (172.16.2.21): 56 data bytes
64 bytes from 172.16.2.21: icmp_seq=0 ttl=63 time=1.648 ms
64 bytes from 172.16.2.21: icmp_seq=1 ttl=63 time=0.508 ms
64 bytes from 172.16.2.21: icmp_seq=2 ttl=63 time=0.781 ms
92 bytes from 10.211.55.24: Redirect Host
64 bytes from 172.16.2.21: icmp_seq=3 ttl=63 time=104.310 ms
64 bytes from 172.16.2.21: icmp_seq=4 ttl=63 time=0.775 ms
^C--- 172.16.2.21 ping statistics ---
5 packets transmitted, 5 packets received, 0% packet loss
round-trip min/avg/max/stddev = 0.508/21.604/104.310/41.355 ms
// ping enp0s5（10.211.55.25），通
root@ubuntu22-25:~# docker exec -it ubuntu-20 ping 10.211.55.25
PING 10.211.55.25 (10.211.55.25): 56 data bytes
64 bytes from 10.211.55.25: icmp_seq=0 ttl=63 time=0.406 ms
64 bytes from 10.211.55.25: icmp_seq=1 ttl=63 time=0.553 ms
64 bytes from 10.211.55.25: icmp_seq=2 ttl=63 time=0.393 ms
^C--- 10.211.55.25 ping statistics ---
3 packets transmitted, 3 packets received, 0% packet loss
round-trip min/avg/max/stddev = 0.393/0.451/0.553/0.073 ms
root@ubuntu22-25:~#
7.4 创建网络-Ipvlan L3S模式
IPVLAN L3S 与 IPVLAN L3 功能类似，两者区别：IPVLAN L3S启用了iptables (conn-tracking)。
7.4.1 Ipvlan L3S模式下，iptables管控conntrack状态
- 创建ipvlan l3s bridge网络ipvl_l3s_bridge
- 父接口 enp0s5
- 子网 192.168.10.0/24 和 172.16.1.0/24
// An highlighted block
root@ubuntu22-25:~# docker network create -d ipvlan --subnet 192.168.10.0/24 --subnet 172.16.1.0/24 -o parent=enp0s5 -o ipvlan_mode=l3s -o ipvlan_flag=bridge ipvl_l3s_bridge
e250d2ea23ca0dd915d43f0c55c9587ee84cfab1d20c6f329708e3191baca142
root@ubuntu22-25:~#
- 运行两个容器ubuntu-100 和 ubuntu-200，加入ipvl_l3s_bridge
// An highlighted block
root@ubuntu22-25:~# docker run -itd --rm --network ipvl_l3s_bridge --ip 192.168.10.100 --name ubuntu-100 ubuntu22:latest /bin/bash
dd4c21e8ad69cb42d3f3847d5e73fc0c6dd43f9ff43201d5125a07337c9bc128
root@ubuntu22-25:~# docker run -itd --rm --network ipvl_l3s_bridge --ip 172.16.1.200 --name ubuntu-200 ubuntu22:latest /bin/bash
aee487bf4d21511fb448148725fbc3b92300bfcd969284df633beffca3894025
root@ubuntu22-25:~#
root@ubuntu22-25:~# docker ps -a
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
aee487bf4d21 ubuntu22:latest "/bin/bash" 47 seconds ago Up 46 seconds ubuntu-200
dd4c21e8ad69 ubuntu22:latest "/bin/bash" 4 minutes ago Up 4 minutes ubuntu-100
- 宿主机ubuntu22-24增加访问容器网络ipvl_l3s_bridge的静态路由
// An highlighted block
root@ubuntu22-24:~# ip route
default via 10.211.55.1 dev enp0s5 proto static
10.211.55.0/24 dev enp0s5 proto kernel scope link src 10.211.55.24
root@ubuntu22-24:~# ip route add 172.16.1.0/24 via 10.211.55.25 dev enp0s5
root@ubuntu22-24:~# ip route add 192.168.10.0/24 via 10.211.55.25 dev enp0s5
- 宿主机ubuntu22-24 Ping 192.168.10.100（ubuntu-100），通
- 宿主机ubuntu22-24 Ping 172.16.1.200（ubuntu-200），通
root@ubuntu22-24:~# ping 192.168.10.100 -c2
PING 192.168.10.100 (192.168.10.100): 56 data bytes
64 bytes from 192.168.10.100: icmp_seq=0 ttl=64 time=0.439 ms
64 bytes from 192.168.10.100: icmp_seq=1 ttl=64 time=0.552 ms
--- 192.168.10.100 ping statistics ---
2 packets transmitted, 2 packets received, 0% packet loss
round-trip min/avg/max/stddev = 0.439/0.496/0.552/0.057 ms
root@ubuntu22-24:~# ping 172.16.1.200 -c2
PING 172.16.1.200 (172.16.1.200): 56 data bytes
64 bytes from 172.16.1.200: icmp_seq=0 ttl=64 time=0.443 ms
64 bytes from 172.16.1.200: icmp_seq=1 ttl=64 time=0.328 ms
--- 172.16.1.200 ping statistics ---
2 packets transmitted, 2 packets received, 0% packet loss
round-trip min/avg/max/stddev = 0.328/0.386/0.443/0.058 ms
- 容器ubuntu-100 Ping 10.211.55.24，通
- 容器ubuntu-200 Ping 10.211.55.24，通
root@ubuntu22-25:~# docker exec -it ubuntu-100 ping 10.211.55.24 -c2
PING 10.211.55.24 (10.211.55.24): 56 data bytes
64 bytes from 10.211.55.24: icmp_seq=0 ttl=64 time=0.794 ms
64 bytes from 10.211.55.24: icmp_seq=1 ttl=64 time=0.575 ms
--- 10.211.55.24 ping statistics ---
2 packets transmitted, 2 packets received, 0% packet loss
round-trip min/avg/max/stddev = 0.575/0.684/0.794/0.110 ms
root@ubuntu22-25:~# docker exec -it ubuntu-200 ping 10.211.55.24 -c2
PING 10.211.55.24 (10.211.55.24): 56 data bytes
64 bytes from 10.211.55.24: icmp_seq=0 ttl=64 time=0.369 ms
64 bytes from 10.211.55.24: icmp_seq=1 ttl=64 time=0.588 ms
--- 10.211.55.24 ping statistics ---
2 packets transmitted, 2 packets received, 0% packet loss
round-trip min/avg/max/stddev = 0.369/0.478/0.588/0.110 ms
- iptables查看当前 filter 表
- INPUT 和 OUTPUT链默认都是ACCEPT
root@ubuntu22-25:~# iptables -L -nv -t filter
Chain INPUT (policy ACCEPT 0 packets, 0 bytes)
pkts bytes target prot opt in out source destination
Chain FORWARD (policy DROP 0 packets, 0 bytes)
pkts bytes target prot opt in out source destination
0 0 DOCKER-USER all -- * * 0.0.0.0/0 0.0.0.0/0
0 0 DOCKER-ISOLATION-STAGE-1 all -- * * 0.0.0.0/0 0.0.0.0/0
0 0 ACCEPT all -- * docker0 0.0.0.0/0 0.0.0.0/0 ctstate RELATED,ESTABLISHED
0 0 DOCKER all -- * docker0 0.0.0.0/0 0.0.0.0/0
0 0 ACCEPT all -- docker0 !docker0 0.0.0.0/0 0.0.0.0/0
0 0 ACCEPT all -- docker0 docker0 0.0.0.0/0 0.0.0.0/0
Chain OUTPUT (policy ACCEPT 0 packets, 0 bytes)
pkts bytes target prot opt in out source destination
Chain DOCKER (1 references)
pkts bytes target prot opt in out source destination
Chain DOCKER-ISOLATION-STAGE-1 (1 references)
pkts bytes target prot opt in out source destination
0 0 DOCKER-ISOLATION-STAGE-2 all -- docker0 !docker0 0.0.0.0/0 0.0.0.0/0
0 0 RETURN all -- * * 0.0.0.0/0 0.0.0.0/0
Chain DOCKER-ISOLATION-STAGE-2 (1 references)
pkts bytes target prot opt in out source destination
0 0 DROP all -- * docker0 0.0.0.0/0 0.0.0.0/0
0 0 RETURN all -- * * 0.0.0.0/0 0.0.0.0/0
Chain DOCKER-USER (1 references)
pkts bytes target prot opt in out source destination
0 0 RETURN all -- * * 0.0.0.0/0 0.0.0.0/0
- iptables 增加filter表 INPUT链一条DROP记录
- 禁止10.211.55.24 新建 与192.168.10.100 的连接
- 测试ubuntu22-24 Ping 192.168.10.100（容器ubuntu-100），不通
// 增加一条DROP，禁止10.211.55.24新建与192.168.10.100的连接
root@ubuntu22-25:~# iptables -A INPUT -s 10.211.55.24 -d 192.168.10.100 -m conntrack --ctstate NEW -j DROP
// 查看结果
root@ubuntu22-25:~# iptables -L -nv -t filter
Chain INPUT (policy ACCEPT 0 packets, 0 bytes)
pkts bytes target prot opt in out source destination
0 0 DROP all -- * * 10.211.55.24 192.168.10.100 ctstate NEW
......
// 宿主机ubuntu22-24 Ping 192.168.10.100，不通（添加DROP记录之前可以通）
root@ubuntu22-24:~# ping 192.168.10.100 -c5
PING 192.168.10.100 (192.168.10.100): 56 data bytes
--- 192.168.10.100 ping statistics ---
5 packets transmitted, 0 packets received, 100% packet loss
root@ubuntu22-24:~#
- 对比测试ubuntu22-24 Ping 172.16.1.200（容器ubuntu-100），通
- 同时宿主机ubuntu22-25可以看到10.211.55.24与172.16.1.200连接的conntrack记录
conntrack概念详见：Ubuntu conntrack命令
// 10.211.55.24 Ping 172.16.1.200，iptables策略放行
root@ubuntu22-24:~# ping 172.16.1.200 -c5
PING 172.16.1.200 (172.16.1.200): 56 data bytes
64 bytes from 172.16.1.200: icmp_seq=0 ttl=64 time=0.462 ms
64 bytes from 172.16.1.200: icmp_seq=1 ttl=64 time=0.604 ms
64 bytes from 172.16.1.200: icmp_seq=2 ttl=64 time=0.536 ms
64 bytes from 172.16.1.200: icmp_seq=3 ttl=64 time=0.570 ms
64 bytes from 172.16.1.200: icmp_seq=4 ttl=64 time=0.522 ms
--- 172.16.1.200 ping statistics ---
5 packets transmitted, 5 packets received, 0% packet loss
round-trip min/avg/max/stddev = 0.462/0.539/0.604/0.048 ms
// 查看conntrack记录，可以看到10.211.55.24与172.16.1.200建立的连接
root@ubuntu22-25:~# conntrack -L -o extend
ipv4 2 tcp 6 431999 ESTABLISHED src=10.211.55.25 dst=10.211.55.2 sport=22 dport=50120 src=10.211.55.2 dst=10.211.55.25 sport=50120 dport=22 [ASSURED] mark=0 use=1
conntrack v1.4.6 (conntrack-tools): 1 flow entries have been shown.
root@ubuntu22-25:~# conntrack -L -o extend
ipv4 2 tcp 6 431999 ESTABLISHED src=10.211.55.25 dst=10.211.55.2 sport=22 dport=50120 src=10.211.55.2 dst=10.211.55.25 sport=50120 dport=22 [ASSURED] mark=0 use=1
ipv4 2 icmp 1 29 src=10.211.55.24 dst=172.16.1.200 type=8 code=0 id=2325 src=172.16.1.200 dst=10.211.55.24 type=0 code=0 id=2325 mark=0 use=1
conntrack v1.4.6 (conntrack-tools): 2 flow entries have been shown.
8 总结
以下摘自：Docker6种网络配置详解，网络模式应该这么选）
基于以上介绍做一下Docker网络模式适用场景的总结：
bridge（默认）：这种模式适合在单台主机上运行多个容器的场景，且多个容器之间需要互相访问。
host：容器和主机共享网络命名空间和网络接口。这种模式可以实现最大的网络性能和最小的网络延迟，适合需要高网络性能的场景。
none：容器没有网络连接。这种模式适用于特殊的情况，如仅用于生成数据卷的容器。
overlay：在 Docker Swarm 模式下使用的网络模式，允许容器在不同主机上的 Docker 引擎中通信。这种模式适合分布式应用程序的场景。
macvlan：将容器连接到物理网络，每个容器分配一个 MAC 地址。这种模式适合需要容器使用主机网络拥有独立 IP 地址的场景，如在企业内部网络中运行容器，暴露相关的服务供内部人员使用。
ipvlan：类似于 macvlan，但容器共享物理接口，可以更有效地使用网络带宽。这种模式适用于需要在容器之间共享网络命名空间的场景。
扫一扫
830
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
