kubernetes kubeadm 部署etcd集群（内部方法一）

一，前置准备，这里不做解释

安装：docker kubectl kubelet kubeadm

修改：docker仓库修改为国内

二、关闭swap和防火墙

swapoff -a
sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab

systemctl disable firewalld && systemctl stop firewalld

三、如果需要用ipvs就开启模块

以下模块需要开启
ip_vs
ip_vs_rr
ip_vs_wrr
ip_vs_sh
nf_conntrack_ipv4

检查有没有开启
cut -f1 -d " "  /proc/modules | grep -e ip_vs -e nf_conntrack_ipv4

没有的话，使用以下命令加载
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack_ipv4

四、禁用selinux

setenforce 0
#修改/etc/sysconfig/selinux
SELINUX=disabled

五、ssh免密自行修改，添加全部主机到hosts

六、内核修改

cat <<EOF >  /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward=1
EOF

sysctl --system

七、这里做测试不再安装haproxy和keepalived

八、修改init-defaults

kubeadm config print init-defaults > init-k8s-m001.yaml

#k8s-m001 init-k8s-m001.yaml
apiVersion: kubeadm.k8s.io/v1beta2
bootstrapTokens:
- groups:
  - system:bootstrappers:kubeadm:default-node-token
  token: abcdef.0123456789abcdef
  ttl: 24h0m0s
  usages:
  - signing
  - authentication
kind: InitConfiguration
localAPIEndpoint:
  advertiseAddress: 0.0.0.0
  bindPort: 6443
nodeRegistration:
  criSocket: /var/run/dockershim.sock
  name: k8s-m001
  taints:
  - effect: NoSchedule
    key: node-role.kubernetes.io/master
---
apiServer:
  timeoutForControlPlane: 1m0s
  certSANs:
  - "k8s-m001"
  - "k8s-m003"
  - "k8s-m002"
  - "192.168.50.121"
  - "192.168.50.122"
  - "192.168.50.123"
  - "127.0.0.1"
apiVersion: kubeadm.k8s.io/v1beta2
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controllerManager: {}
dns:
  type: CoreDNS
  
controlPlaneEndpoint: "192.168.50.121:6443"   #正式环境这里为vip地址，这个地址必须要加入到主机certSAN中

etcd:
  local:
    extraArgs:
      initial-cluster: "k8s-m001=https://192.168.50.121:2380"
      initial-cluster-state: new
      name: k8s-m001
      listen-peer-urls: "https://192.168.50.121:2380"
      listen-client-urls: "https://127.0.0.1:2379,https://192.168.50.121:2379"
      advertise-client-urls: "https://192.168.50.121:2379"
      initial-advertise-peer-urls: "https://192.168.50.121:2380"
    serverCertSANs:
      - "k8s-m001"
      - "k8s-m003"
      - "k8s-m002"
      - "192.168.50.121"
      - "192.168.50.122"
      - "192.168.50.123"
      - "127.0.0.1"
    peerCertSANs:
      - "k8s-m001"
      - "k8s-m003"
      - "k8s-m002"
      - "192.168.50.121"
      - "192.168.50.122"
      - "192.168.50.123"
      - "127.0.0.1"

imageRepository: registry.aliyuncs.com/google_containers
kind: ClusterConfiguration
kubernetesVersion: v1.20.0
networking:
  dnsDomain: cluster.local
  serviceSubnet: 10.96.0.0/12
  podSubnet: 10.244.0.0/16
scheduler: {}

九、下载镜像

kubeadm config images list --config=init-k8s-m001.yaml
kubeadm config images pull --config=init-k8s-m001.yaml

十、初始化k8s-m001

kubeadm init --config=init-k8s-m001.yaml

十一、kubectl管理添加

  mkdir -p $HOME/.kube
  sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
  sudo chown $(id -u):$(id -g) $HOME/.kube/config

十二、复制证书到其它服务器

rsync -a /etc/kubernetes/pki k8s-m002:/etc/kubernetes/
rsync -a /etc/kubernetes/pki k8s-m003:/etc/kubernetes/

十三、在master-m001上将另外master-m002 master-m003加入到etcd集群

#可以使用这个方法进行etcd集群添加
kubectl exec -n kube-system [etcd-k8s-m001] -- etcdctl \
--cacert /etc/kubernetes/pki/etcd/ca.crt \
--cert /etc/kubernetes/pki/etcd/peer.crt \
--key /etc/kubernetes/pki/etcd/peer.key \
--endpoints=https://192.168.50.121:2379 member add [master-m002] \
--peer-urls https://192.168.50.122:2380

#也可以这样操作
docker run --rm -it \
--net host \
-v /etc/kubernetes:/etc/kubernetes registry.aliyuncs.com/google_containers/etcd:3.4.13-0 etcdctl \
--cert /etc/kubernetes/pki/etcd/peer.crt \
--key /etc/kubernetes/pki/etcd/peer.key \
--cacert /etc/kubernetes/pki/etcd/ca.crt \
--endpoints https://192.168.50.121:2379  member add master-m002 --peer-urls="https://192.168.50.122:2380"

十四、初始化master-m002 master-m003，两个相同，修改一下ip

#master-m002 master-m003  init-k8s-m002.yaml init-k8s-m003.yaml
apiVersion: kubeadm.k8s.io/v1beta2
bootstrapTokens:
- groups:
  - system:bootstrappers:kubeadm:default-node-token
  token: abcdef.0123456789abcdef
  ttl: 24h0m0s
  usages:
  - signing
  - authentication
kind: InitConfiguration
localAPIEndpoint:
  advertiseAddress: 0.0.0.0
  bindPort: 6443
nodeRegistration:
  criSocket: /var/run/dockershim.sock
  name: k8s-m002
  taints:
  - effect: NoSchedule
    key: node-role.kubernetes.io/master
---
apiServer:
  timeoutForControlPlane: 1m0s
  certSANs:
  - "k8s-m001"
  - "k8s-m003"
  - "k8s-m002"
  - "192.168.50.121"
  - "192.168.50.122"
  - "192.168.50.123"
  - "127.0.0.1"
apiVersion: kubeadm.k8s.io/v1beta2
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controllerManager: {}
dns:
  type: CoreDNS
controlPlaneEndpoint: "192.168.50.121:6443"

etcd:
  local:
    extraArgs:
      initial-cluster: "k8s-m001=https://192.168.50.121:2380,k8s-m002=https://192.168.50.122:2380"
      initial-cluster-state: existing
      name: "k8s-m002"
      listen-peer-urls: "https://192.168.50.122:2380"
      listen-client-urls: "https://127.0.0.1:2379,https://192.168.50.122:2379"
      advertise-client-urls: "https://192.168.50.122:2379"
      initial-advertise-peer-urls: "https://192.168.50.122:2380"
    serverCertSANs:
      - "k8s-m001"
      - "k8s-m003"
      - "k8s-m002"
      - "192.168.50.121"
      - "192.168.50.122"
      - "192.168.50.123"
      - "127.0.0.1"
    peerCertSANs:
      - "k8s-m001"
      - "k8s-m003"
      - "k8s-m002"
      - "192.168.50.121"
      - "192.168.50.122"
      - "192.168.50.123"
      - "127.0.0.1"
imageRepository: registry.aliyuncs.com/google_containers
kind: ClusterConfiguration
kubernetesVersion: v1.20.0
networking:
  dnsDomain: cluster.local
  serviceSubnet: 10.96.0.0/12
  podSubnet: 10.244.0.0/16
scheduler: {}

kubeadm init --config=init-k8s-m002.yaml

十五、查看etcd集群节点是否正常

查看 etcd 集群已有的节点
kubectl exec -n kube-system master-m001 -- etcdctl \
--cacert /etc/kubernetes/pki/etcd/ca.crt \
--cert /etc/kubernetes/pki/etcd/peer.crt \
--key /etc/kubernetes/pki/etcd/peer.key \
--endpoints=https://192.168.50.121:2379 member list

#也可以这样
docker run --rm -it \
--net host \
-v /etc/kubernetes:/etc/kubernetes registry.aliyuncs.com/google_containers/etcd:3.4.13-0 etcdctl \
--cert /etc/kubernetes/pki/etcd/peer.crt \
--key /etc/kubernetes/pki/etcd/peer.key \
--cacert /etc/kubernetes/pki/etcd/ca.crt \
--endpoints https://192.168.50.121:2379 endpoint health --cluster

十六、查看集群状态

kubectl get nodes
kubectl get pods -n kube-system

十七、网络插件安装Flannel

#下载不下来就先网页下载上传到服务器
wget https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml
#修改 k8s.gcr.io 为 registry.aliyuncs.com/google_containers
kubectl apply -f kube-flannel.yml

#flannel 默认会使用主机的第一张物理网卡，如果你有多张网卡，需要通过配置单独指定。修改 kube-flannel.yml 中的以下部分。如果你有一张物理网卡，可以不用修改。

vim kube-flannel.yml 

 containers:
      - name: kube-flannel
        image: registry.aliyuncs.com/google_containers/flannel:v0.10.0-amd64
        command:
        - /opt/bin/flanneld
        args:
        - --ip-masq
        - --kube-subnet-mgr
        - --iface=ens33              #添加网卡名称

#不用vip就要修改，我这里测试没有用vip
#最后修改一下
vim /etc/kubernetes/manifests/etcd.yaml
#修改 .kube/config ip地址也要改  #

#允许master节点部署pod，使用命令如下:

kubectl taint nodes --all node-role.kubernetes.io/master-

#禁止master部署pod

kubectl taint nodes k8s node-role.kubernetes.io/master=true:NoSchedule

其它命令

#生成一条永久有效的token
kubeadm token create --ttl 0
#查询token
kubeadm token list
#获取ca证书sha256编码hash值
openssl x509 -pubkey -in /etc/kubernetes/pki/ca.crt | openssl rsa -pubin -outform der 2>/dev/null | openssl dgst -sha256 -hex | sed 's/^.* //'

#node节点加入
kubeadm join [10.167.11.153:6443] --token [o4avtg.65ji6b778nyacw68] --discovery-token-ca-cert-hash [sha256:2cc3029123db737f234186636330e87b5510c173c669f513a9c0e0da395515b0]