4. K8S之Flannel

安装并配置Flannel

需要在所有 node 节点安装 Flannel ，使 pod 可以跨界点间通信
目前集群 pod 之间不能互相通信，从pod 172.7.13.2 中无法 ping 通 172.7.14.3

[root@192-168-199-13 ~]# kubectl get pod -o wide
NAME             READY   STATUS    RESTARTS   AGE   IP           NODE                      NOMINATED NODE   READINESS GATES
nginx-ds-6z2cl   1/1     Running   2          12d   172.7.14.3   192-168-199-14.host.com   <none>           <none>
nginx-ds-ln87w   1/1     Running   2          12d   172.7.13.2   192-168-199-13.host.com   <none>           <none>

[root@192-168-199-13 ~]# kubectl exec -it nginx-ds-ln87w /bin/bash

root@nginx-ds-ln87w:/# ping 172.7.14.3
PING 172.7.14.3 (172.7.14.3): 48 data bytes

从 172.7.14.3 中也无法 ping 通 172.7.13.2

[root@192-168-199-14 ~]# kubectl get pod -o wide
NAME             READY   STATUS    RESTARTS   AGE   IP           NODE                      NOMINATED NODE   READINESS GATES
nginx-ds-6z2cl   1/1     Running   2          12d   172.7.14.3   192-168-199-14.host.com   <none>           <none>
nginx-ds-ln87w   1/1     Running   2          12d   172.7.13.2   192-168-199-13.host.com   <none>           <none>
[root@192-168-199-14 ~]# kubectl exec -it nginx-ds-6z2cl /bin/bash
root@nginx-ds-6z2cl:/# ping 172.7.13.2
PING 172.7.13.2 (172.7.13.2): 48 data bytes

从 flannel官网下载软件

[root@192-168-199-13 ~]# wget https://github.com/coreos/flannel/releases/download/v0.11.0/flannel-v0.11.0-linux-amd64.tar.gz

解压软件包，并创建软连接

[root@192-168-199-13 ~]# mkdir /opt/flannel-v0.11.0
[root@192-168-199-13 ~]# tar xf flannel-v0.11.0-linux-amd64.tar.gz -C /opt/flannel-v0.11.0
[root@192-168-199-13 ~]# ln -s /opt/flannel-v0.11.0/ /opt/flannel

拷贝证书，使 flannel 可以连接 etcd

[root@192-168-199-13 ~]# cd /opt/flannel
[root@192-168-199-13 flannel]# mkdir cert
[root@192-168-199-13 flannel]# cd cert/
[root@192-168-199-13 cert]# scp root@192.168.199.15:/opt/certs/ca.pem ./
[root@192-168-199-13 cert]# scp root@192.168.199.15:/opt/certs/client.pem ./
[root@192-168-199-13 cert]# scp root@192.168.199.15:/opt/certs/client-key.pem ./


[root@192-168-199-13 cert]# ll
total 12
-rw-r--r-- 1 root root 1346 Jun 15 19:49 ca.pem
-rw------- 1 root root 1675 Jun 15 19:50 client-key.pem
-rw-r--r-- 1 root root 1363 Jun 15 19:50 client.pem

定义网络相关变量，启动 falnnel 时会引用此文件

[root@192-168-199-13 flannel]# cat subnet.env
# 整个 pod 集群使用的网段
FLANNEL_NETWORK=172.7.0.0/16
# 本机 pod 使用的网段
FLANNEL_SUBNET=172.7.13.1/24
FLANNEL_MTU=1500
FLANNEL_IPMASQ=false

创建 flanneld 的启动脚本

[root@192-168-199-13 flannel]# cat flanneld.sh
#!/bin/sh
./flanneld \
  # 自己本身的 node ip
  --public-ip=192.168.199.13 \
  # etcd集群的地址
  --etcd-endpoints=https://192.168.199.12:2379,https://192.168.199.13:2379,https://192.168.199.14:2379 \
  --etcd-keyfile=./cert/client-key.pem \
  --etcd-certfile=./cert/client.pem \
  --etcd-cafile=./cert/ca.pem \
  --iface=eth0 \
  --subnet-file=./subnet.env \
  --healthz-port=2401

对脚本进行授权

[root@192-168-199-13 flannel]# chmod a+x flanneld.sh

在 etcd 集群中任意一台主机上配置 flannel 使用 host-gw 的网络模型，可以通过 ./etcdctl member list 命令查看集群状态

[root@192-168-199-13 ~]# cd /opt/etcd
# 配置为 host-gw 模式
[root@192-168-199-13 etcd]# ./etcdctl set /coreos.com/network/config '{"Network": "172.7.0.0/16", "Backend": {"Type": "host-gw"}}'
# 验证配置
[root@192-168-199-13 etcd]# ./etcdctl get /coreos.com/network/config
{"Network": "172.7.0.0/16", "Backend": {"Type": "host-gw"}}

除此之外还有 Vxlan 网络模型 和 直接路由模型

配置 supervisord

[root@192-168-199-13 etcd]# cat /etc/supervisord.d/flannel.ini
[program:flanneld-199-13]
command=/opt/flannel/flanneld.sh                             ; the program (relative uses PATH, can take args)
numprocs=1                                                   ; number of processes copies to start (def 1)
directory=/opt/flannel                                       ; directory to cwd to before exec (def no cwd)
autostart=true                                               ; start at supervisord start (default: true)
autorestart=true                                             ; retstart at unexpected quit (default: true)
startsecs=30                                                 ; number of secs prog must stay running (def. 1)
startretries=3                                               ; max # of serial start failures (default 3)
exitcodes=0,2                                                ; 'expected' exit codes for process (default 0,2)
stopsignal=QUIT                                              ; signal used to kill process (default TERM)
stopwaitsecs=10                                              ; max num secs to wait b4 SIGKILL (default 10)
user=root                                                    ; setuid to this UNIX account to run the program
redirect_stderr=true                                         ; redirect proc stderr to stdout (default false)
stdout_logfile=/data/logs/flanneld/flanneld.stdout.log       ; stderr log path, NONE for none; default AUTO
stdout_logfile_maxbytes=64MB                                 ; max # logfile bytes b4 rotation (default 50MB)
stdout_logfile_backups=4                                     ; # of stdout logfile backups (default 10)
stdout_capture_maxbytes=1MB                                  ; number of bytes in 'capturemode' (default 0)
stdout_events_enabled=false                                  ; emit events on stdout writes (default false)

注意 [ ] 中的名字

创建 中说用到的目录

[root@192-168-199-13 ~]# mkdir /data/logs/flanneld/

查看 supervisord 的状态

[root@192-168-199-13 etcd]# supervisorctl update

[root@192-168-199-13 etcd]# supervisorctl status
etcd-server-199-13               RUNNING   pid 721, uptime 16:23:15
flanneld-199-13                  RUNNING   pid 57257, uptime 0:00:46
kube-apiserver-199-13            RUNNING   pid 717, uptime 16:23:15
kube-controller-manager-199-13   RUNNING   pid 722, uptime 16:23:15
kube-kubelet-199-13              RUNNING   pid 714, uptime 16:23:15
kube-proxy-199-13                RUNNING   pid 713, uptime 16:23:15
kube-scheduler-199-13            RUNNING   pid 716, uptime 16:23:15

192.168.199.14 节点配置内容，同上

验证Flannel

配置完成后，两个节点之间的 pod 可以进行正常的通信

root@nginx-ds-ln87w:/# ping 172.7.14.3
PING 172.7.14.3 (172.7.14.3): 48 data bytes
56 bytes from 172.7.14.3: icmp_seq=498 ttl=62 time=0.561 ms
56 bytes from 172.7.14.3: icmp_seq=499 ttl=62 time=0.428 ms
56 bytes from 172.7.14.3: icmp_seq=500 ttl=62 time=0.499 ms

root@nginx-ds-6z2cl:/# ping 172.7.13.2
PING 172.7.13.2 (172.7.13.2): 48 data bytes
56 bytes from 172.7.13.2: icmp_seq=470 ttl=62 time=0.498 ms
56 bytes from 172.7.13.2: icmp_seq=471 ttl=62 time=0.503 ms
56 bytes from 172.7.13.2: icmp_seq=472 ttl=62 time=0.566 ms

优化 iptables 规则

查看 iptables 的现有规则

[root@192-168-199-13 ~]# iptables-save | grep -i postrouting
:POSTROUTING ACCEPT [53:3216]
:KUBE-POSTROUTING - [0:0]
-A POSTROUTING -m comment --comment "kubernetes postrouting rules" -j KUBE-POSTROUTING
# 来源地址是 172.7.13.0/24 且 不是docker0 网卡出网的，做SNAT
-A POSTROUTING -s 172.7.13.0/24 ! -o docker0 -j MASQUERADE
-A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -m mark --mark 0x4000/0x4000 -j MASQUERADE
-A KUBE-POSTROUTING -m comment --comment "Kubernetes endpoints dst ip:port, source ip for solving hairpin purpose" -m set --match-set KUBE-LOOP-BACK dst,dst,src -j MASQUERADE

现在的规则是通过 docker0 出网的，不做SNAT。需要加上出网地址是 172.7.0.0/16 网段也不做SNAT
安装并启动 iptables

[root@192-168-199-13 ~]# yum install iptables-services -y
[root@192-168-199-13 ~]# systemctl start iptables
[root@192-168-199-13 ~]# systemctl enable iptables

修改 iptables 规则

# 删除现有规则
[root@192-168-199-13 ~]# iptables -t nat -D POSTROUTING -s 172.7.13.0/24 ! -o docker0 -j MASQUERADE
# 优化规则
[root@192-168-199-13 ~]# iptables -t nat -I POSTROUTING -s 172.7.13.0/24 ! -d 172.7.0.0/16 ! -o docker0 -j MASQUERADE
# 保存
[root@192-168-199-13 ~]# iptables-save > /etc/sysconfig/iptables

若出现pod节点不互通情况，请执行以下操作

[root@192-168-199-13 ~]# iptables-save | grep -i reject
-A INPUT -j REJECT --reject-with icmp-host-prohibited
-A FORWARD -j REJECT --reject-with icmp-host-prohibited
[root@192-168-199-13 ~]# iptables -t filter -D INPUT -j REJECT --reject-with icmp-host-prohibited
[root@192-168-199-13 ~]# iptables -t filter -D FORWARD -j REJECT --reject-with icmp-host-prohibited

其他 node 节点也需要操作以上内容