八，Kubernetes_v1.14.2部署 flannel 网络

最新推荐文章于 2021-04-13 17:04:41 发布

LanVv

最新推荐文章于 2021-04-13 17:04:41 发布

阅读量849

点赞数 1

分类专栏： kubernetes 文章标签： kubernetes

本文链接：https://blog.csdn.net/qq_40460909/article/details/90549095

版权

kubernetes 专栏收录该内容

20 篇文章 5 订阅

订阅专栏

一，下载和分发 flanneld 二进制文件

#下载
wget https://github.com/coreos/flannel/releases/download/v0.11.0/flannel-v0.11.0-linux-amd64.tar.gz

#解压
mkdir /opt/kubernetes/package/flannel_v0.11
tar -zxvf flannel-v0.11.0-linux-amd64.tar.gz -C /opt/kubernetes/package/flannel_v0.11

#验证是否解压成功
ls /opt/kubernetes/package/flannel_v0.11

二，分发 flanneld 二进制文件到集群所有节点

source /root/env.sh
for master_ip in ${MASTER_IPS[@]}
do
echo -e "\033[31m>>> ${master_ip} \033[0m"
scp /opt/kubernetes/package/flannel_v0.11/{flanneld,mk-docker-opts.sh} root@${master_ip}:/opt/kubernetes/bin
ssh root@${master_ip} "chmod +x /opt/kubernetes/bin/*"
done

#验证是否分发成功
source /root/env.sh
for master_ip in ${MASTER_IPS[@]}
do
echo -e "\033[31m>>> ${master_ip} \033[0m"
ssh root@${master_ip} "ls -ld /opt/kubernetes/bin/{flanneld,mk-docker-opts.sh}"
done

三，创建 flannel 证书和私钥

cd /opt/kubernetes/ssl

cat > flanneld-csr.json << EOF
{
  "CN": "flanneld",
  "hosts": [],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "BeiJing",
      "L": "BeiJing",
      "O": "k8s",
      "OU": "System"
    }
  ]
}
EOF

该证书只会被 kubectl 当做 client 证书使用，所以 hosts 字段为空；

四，生成证书和私钥

cd /opt/kubernetes/ssl

cfssl gencert -ca=/opt/kubernetes/ssl/ca.pem \
-ca-key=/opt/kubernetes/ssl/ca-key.pem \
-config=/opt/kubernetes/ssl/ca-config.json \
-profile=kubernetes flanneld-csr.json | cfssljson -bare flanneld

#验证是否生成成功
ls -ld flanneld.pem flanneld-key.pem

五，将生成的证书和私钥分发到所有节点（master 和 worker）

cd /opt/kubernetes/ssl

source /root/env.sh
for master_ip in ${MASTER_IPS[@]}
do
echo -e "\033[31m>>> ${master_ip} \033[0m"
scp flanneld.pem flanneld-key.pem root@${master_ip}:/opt/kubernetes/ssl
done

#验证是否分发成功
source /root/env.sh
for master_ip in ${MASTER_IPS[@]}
do
echo -e "\033[31m>>> ${master_ip} \033[0m"
ssh root@${master_ip} "ls -ld /opt/kubernetes/ssl/{flanneld.pem,flanneld-key.pem}"
done

六，创建remove-docker0.sh

cd /opt/kubernetes/bin

cat > remove-docker0.sh << "EOF"
#!/bin/bash
# Delete default docker bridge, so that docker can start with flannel network.
# exit on any erro
set -e

rc=0
ip link show docker0 > /dev/null 2>&1 || rc="$?"
if [[ "$rc" -eq "0" ]];then
ip link set dev docker0 down
ip link delete docker0
fi
EOF

#验证是否创建成功
ls -ld remove-docker0.sh

七，分发remove-docker0.sh到各个node节点

cd /opt/kubernetes/bin

source /root/env.sh
for master_ip in ${MASTER_IPS[@]}
do
echo -e "\033[31m>>> ${master_ip} \033[0m"
scp remove-docker0.sh root@${master_ip}:/opt/kubernetes/bin
ssh root@${master_ip} "chmod +x /opt/kubernetes/bin/*"
done

#验证是否分发成功
source /root/env.sh
for master_ip in ${MASTER_IPS[@]}
do
echo -e "\033[31m>>> ${master_ip} \033[0m"
ssh root@${master_ip} "ls -ld /opt/kubernetes/bin/remove-docker0.sh"
done

八，配置flannel

cat > /opt/kubernetes/cfg/flannel << EOF
FLANNEL_ETCD="-etcd-endpoints=${ETCD_ENDPOINTS}"
FLANNEL_ETCD_KEY="-etcd-prefix=/kubernetes/network"
FLANNEL_ETCD_CAFILE="--etcd-cafile=/opt/kubernetes/ssl/ca.pem"
FLANNEL_ETCD_CERTFILE="--etcd-certfile=/opt/kubernetes/ssl/flanneld.pem"
FLANNEL_ETCD_KEYFILE="--etcd-keyfile=/opt/kubernetes/ssl/flanneld-key.pem"
EOF

九，分发flannel

source /root/env.sh
for master_ip in ${MASTER_IPS[@]}
do
echo -e "\033[31m>>> ${master_ip} \033[0m"
scp /opt/kubernetes/cfg/flannel root@${master_ip}:/opt/kubernetes/cfg/flannel
done

#验证是否分发成功
source /root/env.sh
for master_ip in ${MASTER_IPS[@]}
do
echo -e "\033[31m>>> ${master_ip} \033[0m"
ssh root@${master_ip} "ls -ld /opt/kubernetes/cfg/flannel"
done

十，创建flannel服务的service文件

cat > /usr/lib/systemd/system/flannel.service << "EOF"
[Unit]
Description=Flanneld overlay address etcd agent
After=network.target
Before=docker.service

[Service]
EnvironmentFile=-/opt/kubernetes/cfg/flannel
ExecStartPre=/opt/kubernetes/bin/remove-docker0.sh
ExecStart=/opt/kubernetes/bin/flanneld ${FLANNEL_ETCD} ${FLANNEL_ETCD_KEY} ${FLANNEL_ETCD_CAFILE} ${FLANNEL_ETCD_CERTFILE} ${FLANNEL_ETCD_KEYFILE}
ExecStartPost=/opt/kubernetes/bin/mk-docker-opts.sh -d /run/flannel/docker

Type=notify

[Install]
WantedBy=multi-user.target
RequiredBy=docker.service
EOF

#验证是否创建成功
ls -ld /usr/lib/systemd/system/flannel.service

mk-docker-opts.sh 脚本将分配给 flanneld 的 Pod 子网网段信息写入 /run/flannel/docker 文件，后续 docker 启动时使用这个文件中的环境变量配置 docker0 网桥；

十一，分发 flanneld systemd文件到所有节点

source /root/env.sh
for master_ip in ${MASTER_IPS[@]}
do
echo -e "\033[31m>>> ${master_ip} \033[0m"
scp /usr/lib/systemd/system/flannel.service root@${master_ip}:/usr/lib/systemd/system/flannel.service
done

#验证是否分发成功
source /root/env.sh
for master_ip in ${MASTER_IPS[@]}
do
echo -e "\033[31m>>> ${master_ip} \033[0m"
ssh root@${master_ip} "ls -ld /usr/lib/systemd/system/flannel.service"
done

十二，下载Flannel CNI集成

#下载
wget https://github.com/containernetworking/plugins/releases/download/v0.7.5/cni-plugins-amd64-v0.7.5.tgz

#解压
mkdir /opt/kubernetes/package/cni_v0.7.5
tar -zxvf cni-plugins-amd64-v0.7.5.tgz -C /opt/kubernetes/package/cni_v0.7.5

#验证是否解压成功
ls /opt/kubernetes/package/cni_v0.7.5

十三，分发cni到各个节点

source /root/env.sh
for master_ip in ${MASTER_IPS[@]}
do
echo -e "\033[31m>>> ${master_ip} \033[0m"
ssh root@${master_ip} "mkdir -p /opt/kubernetes/bin/cni"
scp /opt/kubernetes/package/cni_v0.7.5/* root@${master_ip}:/opt/kubernetes/bin/cni
ssh root@${master_ip} "chmod +x /opt/kubernetes/bin/cni/*"
done

#验证是否分发成功
source /root/env.sh
for master_ip in ${MASTER_IPS[@]}
do
echo -e "\033[31m>>> ${master_ip} \033[0m"
ssh root@${master_ip} "ls /opt/kubernetes/bin/cni"
done

十四，在etcd中创建key

/opt/kubernetes/bin/etcdctl \
--ca-file /opt/kubernetes/ssl/ca.pem \
--cert-file /opt/kubernetes/ssl/flanneld.pem \
--key-file /opt/kubernetes/ssl/flanneld-key.pem \
--no-sync -C ${ETCD_ENDPOINTS} \
mk /kubernetes/network/config '{ "Network": "10.2.0.0/16", "Backend": { "Type": "vxlan", "VNI": 1 }}'

十五，启动flannel服务

source /root/env.sh
for master_ip in ${MASTER_IPS[@]}
do
echo -e "\033[31m>>> ${master_ip} \033[0m"
ssh root@${master_ip} "systemctl daemon-reload && systemctl enable flannel && systemctl restart flannel"
done

十六，检查flannel服务启动结果

source /root/env.sh
for master_ip in ${MASTER_IPS[@]}
do
echo -e "\033[31m>>> ${master_ip} \033[0m"
ssh root@${master_ip} "systemctl status flannel | grep Active"
done

确保状态为 active (running)，否则查看日志，确认原因

journalctl -u flannel

十七，检查分配给各 flanneld 的 Pod 网段信息

etcdctl \
--endpoints=${ETCD_ENDPOINTS} \
--ca-file=/opt/kubernetes/ssl/ca.pem \
--cert-file=/opt/kubernetes/ssl/flanneld.pem \
--key-file=/opt/kubernetes/ssl/flanneld-key.pem \
get ${FLANNEL_ETCD_PREFIX}/config

输出：
{ "Network": "10.2.0.0/16", "Backend": { "Type": "vxlan", "VNI": 1 }}

十八，查看已分配的 Pod 子网段列表(/24)

etcdctl \
--endpoints=${ETCD_ENDPOINTS} \
--ca-file=/opt/kubernetes/ssl/ca.pem \
--cert-file=/opt/kubernetes/ssl/flanneld.pem \
--key-file=/opt/kubernetes/ssl/flanneld-key.pem \
ls ${FLANNEL_ETCD_PREFIX}/subnets

输出: (结果是部署情况而定，网段可能与下面不一样)
/kubernetes/network/subnets/10.2.63.0-24
/kubernetes/network/subnets/10.2.55.0-24
/kubernetes/network/subnets/10.2.67.0-24

十九，查看某一 Pod 网段对应的节点 IP 和 flannel 接口地址

etcdctl \
--endpoints=${ETCD_ENDPOINTS} \
--ca-file=/opt/kubernetes/ssl/ca.pem \
--cert-file=/opt/kubernetes/ssl/flanneld.pem \
--key-file=/opt/kubernetes/ssl/flanneld-key.pem \
get ${FLANNEL_ETCD_PREFIX}/subnets/10.2.63.0-24

输出: (结果是部署情况而定，网段可能与下面不一样)
{"PublicIP":"172.27.128.11","BackendType":"vxlan","BackendData":{"VtepMAC":"da:e9:aa:41:a0:9e"}}

二十，验证各节点能通过 Pod 网段互通

source /root/env.sh
for master_ip in ${MASTER_IPS[@]}
do
echo -e "\033[31m>>> ${master_ip} \033[0m"
ssh root@${master_ip} "/usr/sbin/ip addr show flannel.1|grep -w inet"
done

输出: (结果是部署情况而定，网段可能与下面不一样)
>>> 172.27.128.11
inet 10.2.63.0/32 scope global flannel.1
>>> 172.27.128.12
inet 10.2.55.0/32 scope global flannel.1
>>> 172.27.128.13
inet 10.2.67.0/32 scope global flannel.1

二十一，在各节点上 ping 所有 flannel 接口 IP，确保能通

source /root/env.sh
for master_ip in ${MASTER_IPS[@]}
do
echo -e "\033[31m>>> ${master_ip} \033[0m"
ssh root@${master_ip} "ping -c 1 10.2.63.0"
ssh root@${master_ip} "ping -c 1 10.2.55.0"
ssh root@${master_ip} "ping -c 1 10.2.67.0"
done

输出: (结果是部署情况而定，网段可能与下面不一样)
>>> 172.27.128.11
PING 10.2.63.0 (10.2.63.0) 56(84) bytes of data.
64 bytes from 10.2.63.0: icmp_seq=1 ttl=64 time=0.015 ms

--- 10.2.63.0 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.015/0.015/0.015/0.000 ms
PING 10.2.55.0 (10.2.55.0) 56(84) bytes of data.
64 bytes from 10.2.55.0: icmp_seq=1 ttl=64 time=0.358 ms

--- 10.2.55.0 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.358/0.358/0.358/0.000 ms
PING 10.2.67.0 (10.2.67.0) 56(84) bytes of data.
64 bytes from 10.2.67.0: icmp_seq=1 ttl=64 time=0.384 ms

--- 10.2.67.0 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.384/0.384/0.384/0.000 ms
>>> 172.27.128.12
PING 10.2.63.0 (10.2.63.0) 56(84) bytes of data.
64 bytes from 10.2.63.0: icmp_seq=1 ttl=64 time=0.270 ms

--- 10.2.63.0 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.270/0.270/0.270/0.000 ms
PING 10.2.55.0 (10.2.55.0) 56(84) bytes of data.
64 bytes from 10.2.55.0: icmp_seq=1 ttl=64 time=0.016 ms

--- 10.2.55.0 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.016/0.016/0.016/0.000 ms
PING 10.2.67.0 (10.2.67.0) 56(84) bytes of data.
64 bytes from 10.2.67.0: icmp_seq=1 ttl=64 time=0.316 ms

--- 10.2.67.0 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.316/0.316/0.316/0.000 ms
>>> 172.27.128.13
PING 10.2.63.0 (10.2.63.0) 56(84) bytes of data.
64 bytes from 10.2.63.0: icmp_seq=1 ttl=64 time=0.293 ms

--- 10.2.63.0 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.293/0.293/0.293/0.000 ms
PING 10.2.55.0 (10.2.55.0) 56(84) bytes of data.
64 bytes from 10.2.55.0: icmp_seq=1 ttl=64 time=0.226 ms

--- 10.2.55.0 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.226/0.226/0.226/0.000 ms
PING 10.2.67.0 (10.2.67.0) 56(84) bytes of data.
64 bytes from 10.2.67.0: icmp_seq=1 ttl=64 time=0.013 ms

--- 10.2.67.0 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.013/0.013/0.013/0.000 ms