二进制部署K8s集群(一) etcd集群

二进制部署K8s集群(一) etcd集群

前置准备

主机名	IP	说明
master	192.168.1.39	控制节点
proxy	192.168.1.252	跳板机
node-0001	192.168.1.40	工作节点1
node-0002	192.168.1.41	工作节点2
node-0003	192.168.1.42	工作节点3

操作系统初始化

#master node1~3
#关闭防火墙
systemctl stop firewalld
systemctl disable firewalld

#关闭selinux
sed -i 's/enforcing/disabled/' /etc/selinux/config
setenforce 0

#关闭swap
swapoff -a 
sed -ri 's/.*swap.*/#&/' /etc/fstab

#将桥接的IPv4流量传递到Iptables的链
cat > /etc/sysctl.d/k8s.conf << EOF
net.bridge.bridge-nf-call-ip6tables=1
net.bridge.bridge-nf-call-iptables=1
EOF

sysctl --system  #生效

#单单master中添加主机名  
cat >> /etc/hosts << EOF
192.168.1.40 node1-0001
192.168.1.41 node1-0002
192.168.1.42 node1-0003
192.168.1.39 master1
EOF


#时间同步
yum install ntpdate -y
ntpdate time.windows.com

部署Etcd集群

主机名	IP	说明
node1-0001	192.168.1.40	etcd-1
node1-0002	192.168.1.41	etcd-2
node1-0003	192.168.1.42	etcd-3

使用cfssl生成证书

#master 上完成 任选
#下载cfssl的程序   #json格式生成     还有openssl  较难
wget https://pkg.cfssl.org/R1.2/cfssl_linux-amd64 
wget https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64 
wget https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64

#添加执行权限
chmod +x cfssl_linux-amd64 cfssljson_linux-amd64 cfssl-certinfo_linux-amd64 

#移动到相应目录
mv cfssl_linux-amd64 /usr/local/bin/cfssl 
mv cfssljson_linux-amd64 /usr/local/bin/cfssljson 
mv cfssl-certinfo_linux-amd64 /usr/bin/cfssl-certinfo

自签证书颁发机构(CA)

#master
mkdir -p ~/TLS/{etcd,k8s}
cd ~/TLS/etcd

cat > ca-config.json << EOF
{
  "signing": {
    "default": {
      "expiry": "87600h"    
    },
    "profiles": {
      "www": {
        "expiry": "87600h",
        "usages": [
          "signing",
          "key encipherment",
          "server auth",
          "client auth"
         ]
       }
     }
  }
}
EOF

cat > ca-csr.json << EOF
{
        "CN": "etcd CA",
        "key": {
                "algo": "rsa",
                "size": 2048
        },
        "names": [
                {
                        "C": "CN",
                        "L": "Beijing",
                        "ST": "Beijing"
                }
        ]
}
EOF

#生成证书
cfssl gencert -initca ca-csr.json | cfssljson -bare ca -
ls  #查看生成
ca-key.pem ca.pem

使用自签的CA签Etcd Https证书

#创建证书申请文件
cat > server-csr.json << EOF
{
        "CN": "etcd",
        "hosts": [         #内部通信用IP 可以多设置预留
        "192.168.1.40",
        "192.168.1.41",
        "192.168.1.42"
        ],
        "key": {
                "algo": "rsa",
                "size": 2048
        },
        "names": [
                {
                        "C": "CN",
                        "L": "BeiJing",
                        "ST": "BeiJing"
                }
        ]
}
EOF

#生成证书
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=www server-csr.json | cfssljson -bare server

ls   #查看

server-key.pem  server.pem


#到这  就有了 ca-key.pem ca.pem   server-key.pem server.pem

#将生成的证书发送到节点上
scp ~/TLS/etcd/ca*pem ~/TLS/etcd/server*pem   节点:/opt/etcd/ssl/

下载etcd二进制文件并配置

https://github.com/etcd-io/etcd/releases/download/v3.4.9/etcd-v3.4.9-linux-amd64.tar.gz

#node1-0001 上下载
mkdir -p /opt/etcd/{bin,cfg,ssl}
tar -zxvf etcd-v3.4.9-linux-amd64.tar.gz
#复制二进制程序到目录下
mv etcd-v3.4.9-linux-amd64/{etcd,etcdctl} /opt/etcd/bin/

#创建etcd配置文件
vim /opt/etcd/cfg/etcd.conf
======================
#[Member]
#节点名称，集群中唯一
ETCD_NAME="etcd-1" 
#数据目录
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
#集群通信监听地址
ETCD_LISTEN_PEER_URLS="https://192.168.1.40:2380"
#客户端访问监听地址
ETCD_LISTEN_CLIENT_URLS="https://192.168.1.40:2379"

#[Clustering]
#集群通告地址
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.1.40:2380"
#客户端通告地址
ETCD_ADVERTISE_CLIENT_URLS="https://192.168.1.40:2379"
#集群节点地址
ETCD_INITIAL_CLUSTER="etcd-1=https://192.168.1.40:2380,etcd-2=https://192.168.1.41:2380,etcd-3=https://192.168.1.42:2380"
#集群Token
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
#加入集群的当前状态，new是新集群，existing表示加入已有集群
ETCD_INITIAL_CLUSTER_STATE="new"
================================

#配置Unit文件    使得可以用systemctl 管理
vim /usr/lib/systemd/system/etcd.service
========================================
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target
[Service]
Type=notify
EnvironmentFile=/opt/etcd/cfg/etcd.conf    #配置文件
ExecStart=/opt/etcd/bin/etcd \              #启动命令
--cert-file=/opt/etcd/ssl/server.pem \      #参数  证书
--key-file=/opt/etcd/ssl/server-key.pem \
--peer-cert-file=/opt/etcd/ssl/server.pem \
--peer-key-file=/opt/etcd/ssl/server-key.pem \
--trusted-ca-file=/opt/etcd/ssl/ca.pem \
--peer-trusted-ca-file=/opt/etcd/ssl/ca.pem \
--logger=zap
Restart=on-failure
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
============================================

#复制配置文件和Unit文件到另外两个节点 41和42
scp -r /opt/etcd root@192.168.1.41:/opt/
scp /usr/lib/systemd/system/etcd.service root@192.168.1.41:/usr/lib/systemd/system/

#修改另两台etcd节点的etcd配置文件信息
vim /opt/etcd/cfg/etcd.conf
=======================
#[Member]
#节点名称，集群中唯一
ETCD_NAME="etcd-2"         #41为2 
#数据目录
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
#集群通信监听地址
ETCD_LISTEN_PEER_URLS="https://192.168.1.41:2380"  #修改
#客户端访问监听地址
ETCD_LISTEN_CLIENT_URLS="https://192.168.1.41:2379"  #修改

#[Clustering]
#集群通告地址
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.1.41:2380" #修改
#客户端通告地址
ETCD_ADVERTISE_CLIENT_URLS="https://192.168.1.41:2379" #修改
#集群节点地址
ETCD_INITIAL_CLUSTER="etcd-1=https://192.168.1.40:2380,etcd-2=https://192.168.1.41:2380,etcd-3=https://192.168.1.42:2380"
#集群Token
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
#加入集群的当前状态，new是新集群，existing表示加入已有集群
ETCD_INITIAL_CLUSTER_STATE="new"
=========================================


#配置三个节点的etcd.conf后  启动服务
systemctl daemon-reload   #重载服务  新加了etcd.service
systemctl enable --now etcd   #开机自启  启动

验证

跳板机ansible查看

#proxy
ansible docker2 -m shell -a "systemctl status etcd"

#42的状态OK
192.168.1.42 | CHANGED | rc=0 >>
● etcd.service - Etcd Server
   Loaded: loaded (/usr/lib/systemd/system/etcd.service; enabled; vendor preset: disabled)
   Active: active (running) since Sat 2021-03-06 14:21:39 CST; 2h 19min ago     #状态为ac (run)
 Main PID: 714 (etcd)
    Tasks: 10
   Memory: 31.8M
   CGroup: /system.slice/etcd.service
           └─714 /opt/etcd/bin/etcd --cert-file=/opt/etcd/ssl/server.pem --key-file=/opt/etcd/ssl/server-key.pem --peer-cert-file=/opt/etcd/ssl/server.pem --peer-key-file=/opt/etcd/ssl/server-key.pem --trusted-ca-file=/opt/etcd/ssl/ca.pem --peer-trusted-ca-file=/opt/etcd/ssl/ca.pem --logger=zap

#41的状态OK
192.168.1.41 | CHANGED | rc=0 >>
● etcd.service - Etcd Server
   Loaded: loaded (/usr/lib/systemd/system/etcd.service; enabled; vendor preset: disabled)
   Active: active (running) since Sat 2021-03-06 14:21:49 CST; 2h 18min ago
 Main PID: 757 (etcd)
    Tasks: 11
   Memory: 29.5M
   CGroup: /system.slice/etcd.service
           └─757 /opt/etcd/bin/etcd --cert-file=/opt/etcd/ssl/server.pem --key-file=/opt/etcd/ssl/server-key.pem --peer-cert-file=/opt/etcd/ssl/server.pem --peer-key-file=/opt/etcd/ssl/server-key.pem --trusted-ca-file=/opt/etcd/ssl/ca.pem --peer-trusted-ca-file=/opt/etcd/ssl/ca.pem --logger=zap

#40的状态OK
192.168.1.40 | CHANGED | rc=0 >>
● etcd.service - Etcd Server
   Loaded: loaded (/usr/lib/systemd/system/etcd.service; enabled; vendor preset: disabled)
   Active: active (running) since Sat 2021-03-06 14:21:37 CST; 2h 19min ago
 Main PID: 705 (etcd)
    Tasks: 11
   Memory: 31.1M
   CGroup: /system.slice/etcd.service
           └─705 /opt/etcd/bin/etcd --cert-file=/opt/etcd/ssl/server.pem --key-file=/opt/etcd/ssl/server-key.pem --peer-cert-file=/opt/etcd/ssl/server.pem --peer-key-file=/opt/etcd/ssl/server-key.pem --trusted-ca-file=/opt/etcd/ssl/ca.pem --peer-trusted-ca-file=/opt/etcd/ssl/ca.pem --logger=zap




#查看集群   之后改的    又加了一个master的节点作为etcd节点   证书配置加个地址  节点地址加一个
/opt/etcd/bin/etcdctl --cacert=/opt/etcd/ssl/ca.pem --cert=/opt/etcd/ssl/server.pem --key=/opt/etcd/ssl/server-key.pem \
> --endpoints="https://192.168.1.39:2379,https://192.168.1.40:2379,https://192.168.1.41:2379,https://192.168.1.42:2379" \
> endpoint health

https://192.168.1.41:2379 is healthy: successfully committed proposal: took = 15.244131ms
https://192.168.1.40:2379 is healthy: successfully committed proposal: took = 15.171511ms
https://192.168.1.39:2379 is healthy: successfully committed proposal: took = 17.114115ms
https://192.168.1.42:2379 is healthy: successfully committed proposal: took = 17.165271ms