跨主机容器之间的通讯

跨主机容器之间的通讯

Flannel

overlay覆盖性网络，不支持路由转发，通过数据etcd数据库保存子网信息以及网络分配信息

给每台主机分配一个网段

通过utp传输数据包

主机名	IP	功能	软件
Node1	192.168.2.2	主控主机	etcd flannel docker
Node2	192.168.2.3	被控主机	flannel docker

Node1

安装etcd

yum - install etcd

安装flannel

yum -y install flannel

配置etcd文件      //修改以下两行

[root@node1 ~]# vim /etc/etcd/etcd.conf

ETCD_LISTEN_CLIENT_URLS="http://0.0.0.0:2379,httpd://0.0.0.0:4001"

ETCD_ADVERTISE_CLIENT_URLS="http://192.168.2.2:2379,http://192.168.2.2:4001"    //自己的IP地址

systemctl start etcd.service

systemctl enable etcd.service

etcdctl set testdir/testkey0 1000

etcdctl get testdir/testkey0

[root@node1 ~]# etcdctl set testdir/testkey0 1000

1000

[root@node1 ~]# etcdctl get testdir/testkey0

1000

测试集群健康

[root@node1 ~]# etcdctl -C http://192.168.2.2:4001 cluster-health

member 8e9e05c52164694d is healthy: got healthy result from http://192.168.2.2:2379

cluster is healthy

配置flannel要访问的etcd数据库所在的位置

[root@node1 ~]# vim /etc/sysconfig/flanneld

FLANNEL_ETCD_ENDPOINTS="http://192.168.2.2:2379"      //数据库的位置

[root@node1 ~]# etcdctl set b 123

123

[root@node1 ~]# etcdctl get b

123

[root@node1 ~]# etcdctl mk /atomic.io/network/config '{ "Network" : "172.20.0.0/16" }'   //指定之后容器ip地址的分配

{ "Network" : "172.20.0.0/16" }

启动flanneld

[root@node1 ~]# systemctl start flanneld.service

[root@node1 ~]# systemctl enable flanneld.service

查看ip地址

[root@node1 ~]# ip a s

inet 172.20.92.0/16 scope global flannel0

下载docker

编写一个脚本docker.sh

Vim docker.sh

br_netfilter

EOF

modprobe overlay

modprobe br_netfilter

cat << EOF | tee /etc/sysctl.d/k8s.conf

net.bridge.bridge-nf-call-iptables = 1

net.bridge.bridge-nf-call-ip6tables = 1

net.ipv4.ip_forward = 1

EOF

sysctl --system

yum install -y yum-utils device-mapper-persistent-data lvm2

yum-config-manager --add-repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo

yum install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin -y

[root@node1 ~]# bash docker.sh

[root@node1 ~]# systemctl start docker.service

查看dockerip

ip a s

5: docker0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc noqueue state DOWN group default

    link/ether 02:42:2c:dd:93:f8 brd ff:ff:ff:ff:ff:ff

    inet 172.17.0.1/16 brd 172.17.255.255 scope global docker0

       valid_lft forever preferred_lft forever

查看flannel子网ip

[root@node1 ~]# cat /run/flannel/subnet.env

FLANNEL_NETWORK=172.20.0.0/16

FLANNEL_SUBNET=172.20.92.1/24

FLANNEL_MTU=1472

FLANNEL_IPMASQ=false

配置daemon.json文件

从其他主机复制一份daemon.json

scp root@192.168.2.13:/etc/docker/daemon.json /etc/docker/

[root@node1 ~]# vim /etc/docker/daemon.json

{

        "registry-mirrors": [

                "https://do.nark.eu.org",

                "https://dc.j8.work",

                "https://docker.m.daocloud.io",

                "https://dockerproxy.com",

                "https://docker.mirrors.ustc.edu.cn",

                "https://docker.nju.edu.cn"

        ]

,

        "hosts": [

               "tcp://0.0.0.0:2375",

               "unix:///var/run/docker.sock"

        ],

        "insecure-registries":[

                "http://192.168.2.2:5050"

        ]

}

[root@node1 ~]# vim /usr/lib/systemd/system/docker.service

ExecStart=/usr/bin/dockerd

ExecReload=/bin/kill -s HUP $MAINPID

[root@node1 ~]# systemctl daemon-reload

[root@node1 ~]# systemctl restart docker.service

[root@node1 ~]# cat /run/flannel/subnet.env

FLANNEL_NETWORK=172.20.0.0/16

FLANNEL_SUBNET=172.20.92.1/24

FLANNEL_MTU=1472

FLANNEL_IPMASQ=false

[root@node1 ~]# vim /etc/docker/daemon.json

{

        "registry-mirrors": [

                "https://do.nark.eu.org",

                "https://dc.j8.work",

                "https://docker.m.daocloud.io",

                "https://dockerproxy.com",

                "https://docker.mirrors.ustc.edu.cn",

                "https://docker.nju.edu.cn"

        ]

,

        "hosts": [

               "tcp://0.0.0.0:2375",

               "unix:///var/run/docker.sock"

        ],

        "insecure-registries":[

                "http://192.168.2.2:5050"

        ],

        "bip" : "172.20.92.1/24",

        "mtu" :  1472

}

[root@node1 ~]# systemctl restart docker.service

iP a s        //结果是flannel子网与docker子网保持一致

4: flannel0: <POINTOPOINT,MULTICAST,NOARP,UP,LOWER_UP> mtu 1472 qdisc pfifo_fast state UNKNOWN group default qlen 500

    link/none

    inet 172.20.92.0/16 scope global flannel0

       valid_lft forever preferred_lft forever

    inet6 fe80::96c7:b4dc:5cc3:f913/64 scope link flags 800

       valid_lft forever preferred_lft forever

5: docker0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1472 qdisc noqueue state DOWN group default

    link/ether 02:42:2c:dd:93:f8 brd ff:ff:ff:ff:ff:ff

    inet 172.20.92.1/24 brd 172.20.92.255 scope global docker0

       valid_lft forever preferred_lft forever

[root@node1 ~]# docker pull centos

[root@node1 ~]# docker run -it centos:latest /bin/bash

[root@4a37c9ff2dce /]# [root@node1 ~]#

[root@node1 ~]# docker inspect 4a37|grep IPAdd

            "SecondaryIPAddresses": null,

            "IPAddress": "172.20.92.2",

                    "IPAddress": "172.20.92.2",

Node2

安装flannel

Yum -y install flannel

配置flannel要访问的etcd数据库所在的位置

[root@node2 ~]#  vim /etc/sysconfig/flanneld

FLANNEL_ETCD_ENDPOINTS="http://192.168.2.2:2379"   //node1的IP地址

启动flannel

[root@node2 ~]# systemctl start flanneld.service

查看flannel分配的ip网段

[root@node2 ~]# ip a s

inet 172.20.80.0/16 scope global flannel0

[root@node2 ~]# cat /run/flannel/subnet.env

FLANNEL_NETWORK=172.20.0.0/16

FLANNEL_SUBNET=172.20.80.1/24

FLANNEL_MTU=1472

FLANNEL_IPMASQ=false

将docker.sh文件拉取到node2机器上

scp  docker.sh root@192.168.2.3:~

运行这个脚本下载安装docker

source docker.sh

[root@node2 ~]# systemctl start docker.service

将flannel分配网段写入到daemon.json

[root@node2 ~]# scp root@192.168.2.2:/etc/docker/daemon.json /etc/docker/

[root@node2 ~]# vim /etc/docker/daemon.json //只改这两个

 "bip" : "172.20.80.1/24",

 "mtu" : 1472

[root@node2 ~]# cat /run/flannel/subnet.env

FLANNEL_NETWORK=172.20.0.0/16

FLANNEL_SUBNET=172.20.80.1/24

FLANNEL_MTU=1472

FLANNEL_IPMASQ=false

[root@node2 ~]# systemctl restart docker.service

重启docker如果不能重启，就修改远程管理

[root@node2 ~]# vim /usr/lib/systemd/system/docker.service

ExecStart=/usr/bin/dockerd

ExecReload=/bin/kill -s HUP $MAINPID

重启docker

[root@node2 ~]# systemctl daemon-reload

[root@node2 ~]# systemctl restart docker.service

[root@node2 ~]# ip a s

4: flannel0: <POINTOPOINT,MULTICAST,NOARP,UP,LOWER_UP> mtu 1472 qdisc pfifo_fast state UNKNOWN group default qlen 500

    link/none

    inet 172.20.80.0/16 scope global flannel0

       valid_lft forever preferred_lft forever

    inet6 fe80::538a:3c3e:6b72:66ad/64 scope link flags 800

       valid_lft forever preferred_lft forever

5: docker0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1472 qdisc noqueue state DOWN group default

    link/ether 02:42:2f:fa:fe:ce brd ff:ff:ff:ff:ff:ff

    inet 172.20.80.1/24 brd 172.20.80.255 scope global docker0

       valid_lft forever preferred_lft forever

[root@node2 ~]# docker pull centos

拉取一个centos镜像

Using default tag: latest

latest: Pulling from library/centos

a1d0c7532777: Pull complete

Digest: sha256:a27fd8080b517143cbbbab9dfb7c8571c40d67d534bbdee55bd6c473f432b177

Status: Downloaded newer image for centos:latest

docker.io/library/centos:latest

[root@node2 ~]# docker run -it centos:latest /bin/bash

Ping node1中容器的IP地址

[root@6d5944f42024 /]# ping 172.20.92.1

PING 172.20.92.1 (172.20.92.1) 56(84) bytes of data.

64 bytes from 172.20.92.1: icmp_seq=1 ttl=61 time=2.29 ms

64 bytes from 172.20.92.1: icmp_seq=2 ttl=61 time=1.12 ms

在node1里面ping node2里面的容器的IP地址

[root@node1 ~]# docker attach 4a

[root@4a37c9ff2dce /]# ping 172.20.80.1

PING 172.20.80.1 (172.20.80.1) 56(84) bytes of data.

64 bytes from 172.20.80.1: icmp_seq=1 ttl=61 time=1.26 ms

总结

工作原理：

1. 使用flannel为docker主机（宿主机）分配网段
2. 网段的信息以及ip的信息保存在etcd数据库中
3. 当flannel开始运行的时候，会从etcd数据库中读取{“Network”:”172.20.0.0/16”,随机为当前的主机添加一个flannel0网卡
4. 配置docker下的daemon.json文件，让docker0网卡变成和flannel的网段一致，之后docker下创建的容器的ip就在flannel的网段控制之内