【K8S运维知识汇总】第3天3：flanneld安装部署详解

安装flanneld

[root@hdss7-22 ~]# cd /opt/src/
[root@hdss7-22 src]# rz

[root@hdss7-21 src]# ll
总用量 452336
-rw-r--r-- 1 root root   9850227 6月  22 21:16 etcd-v3.1.20-linux-amd64.tar.gz
-rw-r--r-- 1 root root   9565743 6月  22 20:24 flannel-v0.11.0-linux-amd64.tar.gz
-rw-r--r-- 1 root root 443770238 6月  22 21:11 kubernetes-server-linux-amd64-v1.15.2.tar.gz
[root@hdss7-21 src]# mkdir /opt/flannel-v0.11.0
[root@hdss7-21 src]# tar -xzvf flannel-v0.11.0-linux-amd64.tar.gz -C /opt/flannel-v0.11.0/
 
[root@hdss7-21 src]# cd !$
[root@hdss7-21 flannel-v0.11.0]# ll
总用量 34436
-rwxr-xr-x 1 root root 35249016 1月  29 2019 flanneld
-rwxr-xr-x 1 root root     2139 10月 23 2018 mk-docker-opts.sh
-rw-r--r-- 1 root root     4300 10月 23 2018 README.md

[root@hdss7-21 flannel-v0.11.0]# ln -s /opt/flannel-v0.11.0/ /opt/flannel
[root@hdss7-22 opt]# cd flannel
[root@hdss7-22 flannel]# ll
总用量 34436
-rwxr-xr-x 1 root root 35249016 1月  29 2019 flanneld
-rwxr-xr-x 1 root root     2139 10月 23 2018 mk-docker-opts.sh
-rw-r--r-- 1 root root     4300 10月 23 2018 README.md

复制证书（完成与etcd的通信）

[root@hdss7-21 flannel]# mkdir cert
[root@hdss7-21 flannel]# cd cert
 
[root@hdss7-21 cert]# scp 10.4.7.200:/opt/certs/ca.pem .
root@10.4.7.200's password: 
ca.pem                                                                                                                                         100% 1346     2.0MB/s   00:00    
[root@hdss7-21 cert]# scp 10.4.7.200:/opt/certs/client.pem .
root@10.4.7.200's password: 
client.pem                                                                                                                                     100% 1371     1.7MB/s   00:00    
[root@hdss7-21 cert]# scp 10.4.7.200:/opt/certs/client-key.pem .
root@10.4.7.200's password: 
client-key.pem                                                                                                                                 100% 1675     1.7MB/s   00:00    
[root@hdss7-21 cert]# ll
总用量 12
-rw-r--r-- 1 root root 1346 7月   2 11:26 ca.pem
-rw------- 1 root root 1675 7月   2 11:27 client-key.pem
-rw-r--r-- 1 root root 1371 7月   2 11:26 client.pem

创建配置文件

[root@hdss7-22 cert]# cd ..
[root@hdss7-21 flannel]# vi subnet.env
# 根据实际主机修改 FLANNEL_SUBNET=172.7.21.1/24

[root@hdss7-21 flannel]# cat subnet.env 
FLANNEL_NETWORK=172.7.0.0/16
FLANNEL_SUBNET=172.7.21.1/24
FLANNEL_MTU=1500
FLANNEL_IPMASQ=false

[root@hdss7-21 flannel]# vi flanneld.sh
[root@hdss7-21 flannel]# cat flanneld.sh 
#!/bin/sh
./flanneld \
  --public-ip=10.4.7.21 \
  --etcd-endpoints=https://10.4.7.12:2379,https://10.4.7.21:2379,https://10.4.7.22:2379 \
  --etcd-keyfile=./cert/client-key.pem \
  --etcd-certfile=./cert/client.pem \
  --etcd-cafile=./cert/ca.pem \
  --iface=ens33 \
  --subnet-file=./subnet.env \
  --healthz-port=2401
  
[root@hdss7-21 flannel]# chmod +x /opt/flannel/flanneld.sh 
[root@hdss7-21 flannel]# mkdir  -p /data/logs/flanneld

在etcd中增加网络配置信息

[root@hdss7-21 flannel]# cd /opt/etcd
# 查看集群中的主服务器，在主服务器上去配置参数
[root@hdss7-21 etcd]# ./etcdctl member list
988139385f78284: name=etcd-server-7-22 peerURLs=https://10.4.7.22:2380 clientURLs=http://127.0.0.1:2379,https://10.4.7.22:2379 isLeader=false
5a0ef2a004fc4349: name=etcd-server-7-21 peerURLs=https://10.4.7.21:2380 clientURLs=http://127.0.0.1:2379,https://10.4.7.21:2379 isLeader=false
f4a0cb0a765574a8: name=etcd-server-7-12 peerURLs=https://10.4.7.12:2380 clientURLs=http://127.0.0.1:2379,https://10.4.7.12:2379 isLeader=true

# 使用 host-gw模型 
# 寻找集群中的任意一台etcd服务，且只运行一次写入etcd中即可
[root@hdss7-21 etcd]# ./etcdctl set /coreos.com/network/config '{"Network": "172.7.0.0/16", "Backend": {"Type": "host-gw"}}'
{"Network": "172.7.0.0/16", "Backend": {"Type": "host-gw"}}


# 查看网络模型配置
[root@hdss7-21 etcd]# ./etcdctl get /coreos.com/network/config
{"Network": "172.7.0.0/16", "Backend": {"Type": "host-gw"}}

创建启动脚本

编辑supervisor启动脚本：

[program:flanneld-7-21] 部分记得修改

[root@hdss7-22 etcd]# vi /etc/supervisord.d/flannel.ini
[program:flanneld-7-21]
command=/opt/flannel/flanneld.sh                             ; the program (relative uses PATH, can take args)
numprocs=1                                                   ; number of processes copies to start (def 1)
directory=/opt/flannel                                       ; directory to cwd to before exec (def no cwd)
autostart=true                                               ; start at supervisord start (default: true)
autorestart=true                                             ; retstart at unexpected quit (default: true)
startsecs=30                                                 ; number of secs prog must stay running (def. 1)
startretries=3                                               ; max # of serial start failures (default 3)
exitcodes=0,2                                                ; 'expected' exit codes for process (default 0,2)
stopsignal=QUIT                                              ; signal used to kill process (default TERM)
stopwaitsecs=10                                              ; max num secs to wait b4 SIGKILL (default 10)
user=root                                                    ; setuid to this UNIX account to run the program
redirect_stderr=true                                         ; redirect proc stderr to stdout (default false)
stdout_logfile=/data/logs/flanneld/flanneld.stdout.log       ; stderr log path, NONE for none; default AUTO
stdout_logfile_maxbytes=64MB                                 ; max # logfile bytes b4 rotation (default 50MB)
stdout_logfile_backups=4                                     ; # of stdout logfile backups (default 10)
stdout_capture_maxbytes=1MB                                  ; number of bytes in 'capturemode' (default 0)
stdout_events_enabled=false                                  ; emit events on stdout writes (default false)

[root@hdss7-21 etcd]# mkdir -p /data/logs/flanneld/

[root@hdss7-21 etcd]# supervisorctl update

[root@hdss7-21 etcd]# supervisorctl status
etcd-server-7-21                 RUNNING   pid 12272, uptime 5 days, 23:36:17
flanneld-7-21                    STARTING  
kube-apiserver-7-21              RUNNING   pid 13277, uptime 5 days, 0:09:58
kube-controller-manager-7-21     RUNNING   pid 64267, uptime 1 day, 1:32:05
kube-kubelet-7-21                RUNNING   pid 17290, uptime 3 days, 17:30:48
kube-proxy-7-21                  RUNNING   pid 55520, uptime 3 days, 14:16:13
kube-scheduler-7-21              RUNNING   pid 64261, uptime 1 day, 1:32:05

# 也可以使用该命令启动对应的脚本
[root@hdss7-22 flannel]# supervisorctl start flanneld-7-22

验证不同宿主机中的容器是否能够通信

[root@hdss7-21 etcd]# ping  172.7.21.2 
PING 172.7.21.2 (172.7.21.2) 56(84) bytes of data.
64 bytes from 172.7.21.2: icmp_seq=1 ttl=64 time=0.092 ms
64 bytes from 172.7.21.2: icmp_seq=2 ttl=64 time=0.098 ms
64 bytes from 172.7.21.2: icmp_seq=3 ttl=64 time=0.043 ms
64 bytes from 172.7.21.2: icmp_seq=4 ttl=64 time=0.094 ms

[root@hdss7-21 etcd]# curl 172.7.21.2
<!DOCTYPE html>
<html>
<head>
<title>Welcome to nginx!</title>
<style>
    body {
        width: 35em;
        margin: 0 auto;
        font-family: Tahoma, Verdana, Arial, sans-serif;
    }
</style>
</head>
<body>
<h1>Welcome to nginx!</h1>
<p>If you see this page, the nginx web server is successfully installed and
working. Further configuration is required.</p>

<p>For online documentation and support please refer to
<a href="http://nginx.org/">nginx.org</a>.<br/>
Commercial support is available at
<a href="http://nginx.com/">nginx.com</a>.</p>

<p><em>Thank you for using nginx.</em></p>
</body>
</html>



[root@hdss7-22 flannel]# ping 172.7.21.1
PING 172.7.21.1 (172.7.21.1) 56(84) bytes of data.
64 bytes from 172.7.21.1: icmp_seq=1 ttl=64 time=0.200 ms
64 bytes from 172.7.21.1: icmp_seq=2 ttl=64 time=0.624 ms
64 bytes from 172.7.21.1: icmp_seq=3 ttl=64 time=0.483 ms
64 bytes from 172.7.21.1: icmp_seq=4 ttl=64 time=0.669 ms

网络连接ping不通的问题处理

# 没有开启内核ipv4转发功能
[root@hdss7-22 flannel]#  cat /proc/sys/net/ipv4/ip_forward
0
[root@hdss7-22 flannel]# vi /etc/sysctl.conf 

# sysctl settings are defined through files in
# /usr/lib/sysctl.d/, /run/sysctl.d/, and /etc/sysctl.d/.
#
# Vendors settings live in /usr/lib/sysctl.d/.
# To override a whole file, create a new file with the same in
# /etc/sysctl.d/ and put new settings there. To override
# only specific settings, add a file with a lexically later
# name in /etc/sysctl.d/ and put new settings there.
#
# For more information, see sysctl.conf(5) and sysctl.d(5).
net.ipv4.ip_forward = 1
                            
[root@hdss7-22 flannel]# sysctl -p