CC00010.CloudKubernetes——|Kubernetes&二进制部署.V06|——|部署Worker Node|

一、部署worker Node节点：下面还是在Master Node上操作，即同时作为worker Node

### --- 创建工作目录并拷贝二进制文件
~~~     在所有worker node创建工作目录：

[root@k8s-master ~]# mkdir -p /opt/kubernetes/{bin,cfg,ssl,logs}

~~~     从master节点拷贝：
[root@k8s-master ~]# cd kubernetes/server/bin
 
~~~     本地拷贝
[root@k8s-master bin]# cp kubelet kube-proxy /opt/kubernetes/bin

二、部署kubelet

### --- 创建配置文件
~~~		--hostname-override:显示名称，集群中唯一
~~~		--network-plugin:启用CNI
~~~		--kubeconfig：空路径,会自动生成，后面用于连接apiserver
~~~		--bootstrap-kubeconfig：首次启动向apiserver申请证书
~~~		--config：配置参数文件
~~~		--cert-dir：kubelet证书生成目录
~~~		pod-infra-container-image：管理Pod网络容器的镜像

[root@k8s-master ~]# cat > /opt/kubernetes/cfg/kubelet.conf << EOF
> KUBELET_OPTS="--logtostderr=false \\
> --v=2 \\
> --log-dir=/opt/kubernetes/logs \\
> --hostname-override=k8s-master \\
> --network-plugin=cni \\
> --kubeconfig=/opt/kubernetes/cfg/kubelet.kubeconfig \\
> --bootstrap-kubeconfig=/opt/kubernetes/cfg/bootstrap.kubeconfig \\
> --config=/opt/kubernetes/cfg/kubelet-config.yml \\
> --cert-dir=/opt/kubernetes/ssl \\
> --pod-infra-container-image=lizhenliang/pause-amd64:3.0"
> EOF

### --- 配置参数文件

[root@k8s-master ~]# cat > /opt/kubernetes/cfg/kubelet-config.yml << EOF
> kind: KubeletConfiguration
> apiVersion: kubelet.config.k8s.io/v1beta1
> address: 0.0.0.0
> port: 10250
> readOnlyPort: 10255
> cgroupDriver: cgroupfs
> clusterDNS:
> - 10.0.0.2
> clusterDomain: cluster.local
> failSwapOn: false
> authentication:
> anonymous:
> enabled: false
> webhook:
> cacheTTL: 2m0s
> enabled: true
> x509:
> clientCAFile: /opt/kubernetes/ssl/ca.pem
> authorization:
> mode: Webhook
> webhook:
> cacheAuthorizedTTL: 5m0s
> cacheUnauthorizedTTL: 30s
> evictionHard:
> imagefs.available: 15%
> memory.available: 100Mi
> nodefs.available: 10%
> nodefs.inodesFree: 5%
> maxOpenFiles: 1000000
> maxPods: 110
> EOF

### --- 生成bootstrap.kubeconfig文件
~~~		生成kubelet bootstrap kubeconfig配置文件
KUBE_APISERVER="https://10.10.10.11:6443 #apiserver IP：PORT 
TOKEN="c47ffb939f5ca36231d9e312a252940"                     # 与token.csv里保持一致

[root@k8s-master ~]# kubectl config set-cluster kubernetes \
> --certificate-authority=/opt/kubernetes/ssl/ca.pem \
> --embed-certs=true \
> --server=${https://10.10.10.11:6443} \
> --kubeconfig=bootstrap.kubeconfig 
Cluster "kubernetes" set.

[root@k8s-master ~]# kubectl config set-credentials "kubelet-bootstrap" \
> --token=${c47ffb939f5ca36231d9e3121a252940} \
> --kubeconfig=bootstrap.kubeconfig 
User "kubelet-bootstrap" set.

[root@k8s-master ~]# kubectl config set-context default \
> --cluster=kubernetes \
> --user="kubelet-bootstrap" \
> --kubeconfig=bootstrap.kubeconfig
Context "default" created.
 
[root@k8s-master ~]# kubectl config use-context default \
> --kubeconfig=bootstrap.kubeconfig
Switched to context "default".

~~~		拷贝到配置文件路径：

[root@k8s-master ~]# cp bootstrap.kubeconfig /opt/kubernetes/cfg

### --- systemd管理kubelet

[root@k8s-master ~]# cat > /usr/lib/systemd/system/kubelet.service << EOF
> [Unit]
> Description=Kubernetes Kubelet
> After=docker.service
> [Service]
> EnvironmentFile=/opt/kubernetes/cfg/kubelet.conf
> ExecStart=/opt/kubernetes/bin/kubelet \$KUBELET_OPTS
> Restart=on-failure
> LimitNOFILE=65536
> [Install]
> WantedBy=multi-user.target
> EOF

### --- 启动设置开机启动

[root@k8s-master ~]# systemctl daemon-reload
[root@k8s-master ~]# systemctl start kubelet
[root@k8s-master ~]# systemctl enable kubelet

三、批准kubelet证书申请并加入集群

### --- 查看kubelet证书请求

[root@k8s-master ~]# kubectl get csr
NAME            AGE     SIGNERNAME      REQUESTOR       CONDITION
node-csr-uCEFPOIiDdlLODKts8J658HrFq9cz-K6M4G7bjhk8A     6m3s
kubernetes.io/kubr-spidrtbrt-client-kubelet  kubelet-bootstrap Pending

### --- 批准申请

[root@k8s-master ~]# kubectl certificate approve node-csr-uCEGPOIiDdlLODKts8J658HRF区9CZ--K6M4G7bjhk8A

### --- 查看节点
~~~		由于网络插件还没有部署，节点会没有准备就绪NotReady

[root@k8s-master ~]# kubectl get node

四、部署kube-proxy

### --- 创建配置文件

[root@k8s-master ~]# cat > /opt/kubernetes/cfg/kube-proxy.conf << EOF
> KUBE_PROXY_OPTS="--logtostderr=false \\
> --v=2 \\
> --log-dir=/opt/kubernetes/logs \\
> --config=/opt/kubernetes/cfg/kube-proxy-config.yml"
> EOF

### --- 配置参数文件

[root@k8s-master ~]# cat > /opt/kubernetes/cfg/kube-proxy-config.yml << EOF
> kind: KubeProxyConfiguration
> apiVersion: kubeproxy.config.k8s.io/v1alpha1
> bindAddress: 0.0.0.0
> metricsBindAddress: 0.0.0.0:10249
> clientConnection:
> kubeconfig: /opt/kubernetes/cfg/kube-proxy.kubeconfig
> hostnameOverride: k8s-master
> clusterCIDR: 10.0.0.0/24
> EOF

### --- 生成kube-proxy.kubeconfig文件
~~~		生成kube-proxy证书：
~~~		创建证书请求文件

[root@k8s-master k8s]# cd TLS/k8s
[root@k8s-master k8s]# cat > kube-proxy-csr.json<< EOF
> {
> "CN": "system:kube-proxy",
> "hosts": [],
> "key": {
> "algo": "rsa",
> "size": 2048
> },
> "names": [
> {
> "C": "CN",
> "L": "BeiJing",
> "ST": "BeiJing",
> "O": "k8s",
> "OU": "System"
> }
> ]
> }
> EOF

~~~		生成证书

[root@k8s-master k8s]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes kube-proxy-csr.json | cfssljson -bare kube-proxy
2021/02/22 00:29:22 [INFO] generate received request
2021/02/22 00:29:22 [INFO] received CSR
2021/02/22 00:29:22 [INFO] generating key: rsa-2048
2021/02/22 00:29:23 [INFO] encoded CSR
2021/02/22 00:29:23 [INFO] signed certificate with serial number 210829839218231482690292346511240448736967423544
2021/02/22 00:29:23 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2.3 ("Information Requirements").

[root@k8s-master k8s]# ls kube-proxy*pem
kube-proxy-key.pem  kube-proxy.pem

~~~		生成kubeconfig文件：

[root@k8s-master k8s]# kubectl config set-cluster kubernetes \
> --certificate-authority=/opt/kubernetes/ssl/ca.pem \
> --embed-certs=true \
> --server=${https://10.10.10.11:6443} \
> --kubeconfig=kube-proxy.kubeconfig
Cluster "kubernetes" set.
[root@k8s-master k8s]# kubectl config set-credentials kube-proxy \
> --client-certificate=./kube-proxy.pem \
> --client-key=./kube-proxy-key.pem \
> --embed-certs=true \
> --kubeconfig=kube-proxy.kubeconfig
User "kube-proxy" set.
[root@k8s-master k8s]# kubectl config set-context default \
> --cluster=kubernetes \
> --user=kube-proxy \
> --kubeconfig=kube-proxy.kubeconfig
Context "default" created.
[root@k8s-master k8s]# kubectl config use-context default --kubeconfig=kube-proxy.kubeconfig
Switched to context "default".

~~~		拷贝到配置文件指定路径：

[root@k8s-master k8s]# cp kube-proxy.kubeconfig /opt/kubernetes/cfg/

### --- systemd管理kube-proxy

[root@k8s-master ~]# cat > /usr/lib/systemd/system/kube-proxy.service << EOF
> [Unit]
> Description=Kubernetes Proxy
> After=network.target
> [Service]
> EnvironmentFile=/opt/kubernetes/cfg/kube-proxy.conf
> ExecStart=/opt/kubernetes/bin/kube-proxy \$KUBE_PROXY_OPTS
> Restart=on-failure
> LimitNOFILE=65536
> [Install]
> WantedBy=multi-user.target
> EOF

### --- 启动并设置开机启动

[root@k8s-master ~]# systemctl daemon-reload
[root@k8s-master ~]# systemctl start kube-proxy
[root@k8s-master ~]# systemctl enable kube-proxy

五、部署CNT网络

### --- 先准备好CNI二进制文件：
~~~		下载地址：
https://github.com/containernetworking/plugins/releases/download/v0.8.6/cni-plugins-linux-amd64-v0.8.6.tgz
[root@k8s-master ~]# wget -c https://github.com/containernetworking/plugins/releases/download/v0.9.1/cni-plugins-linux-amd64-v0.9.1.tgz

 ~~~		解压二进制包并移动到默认工作目录
[root@k8s-master ~]# mkdir -p  /opt/cni/bin
[root@k8s-master ~]# tar -zxvf cni-plugins-linux-amd64-v0.9.1.tgz  -C /opt/cni/bin/

### --- 部署CNI网络：

[root@k8s-master ~]# wget -c https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kubeflannel.yml
[root@k8s-master ~]# sed -i -r "s#quay.io/coreos/flannel:.*-amd64#lizhenliang/flannel:v0.12.0-amd64#g" kube-flannel.yml

### --- 默认镜像地址无法访问，修改为docker hub镜像仓库
~~~		部署好网络插件，Node准备就绪

[root@k8s-master ~]# kubectl apply -f kube-flannel.yml
[root@k8s-master ~]# kubectl get pods -n kube-system
[root@k8s-master ~]# kubectl get node

六、授权apiserver访问kubelet

### --- 授权apiserver访问kubelet

[root@k8s-master ~]# cat > apiserver-to-kubelet-rbac.yaml<< EOF
> apiVersion: rbac.authorization.k8s.io/v1
> kind: ClusterRole
> metadata:
> annotations:
> rbac.authorization.kubernetes.io/autoupdate: "true"
> labels:
> kubernetes.io/bootstrapping: rbac-defaults
> name: system:kube-apiserver-to-kubelet
> rules:
> - apiGroups:
> - ""
> resources:
> - nodes/proxy
> - nodes/stats
> - nodes/log
> - nodes/spec
> - nodes/metrics
> - pods/log
> verbs:
> - "*"
> ---
> apiVersion: rbac.authorization.k8s.io/v1
> kind: ClusterRoleBinding
> metadata:
> name: system:kube-apiserver
> namespace: ""
> roleRef:
> apiGroup: rbac.authorization.k8s.io
> kind: ClusterRole
> name: system:kube-apiserver-to-kubelet
> subjects:
> - apiGroup: rbac.authorization.k8s.io
> kind: User
> name: kubernetes
> EOF

[root@k8s-master ~]# kubectl apply -f apiserver-to-kubelet-rbac.yaml