kubeadm部署k8s直接证书100年

cloud_engineer

已于 2022-07-21 15:07:30 修改

阅读量4.6k

点赞数 6

分类专栏： kubernetes 文章标签： kubernetes docker 容器

于 2022-07-20 23:28:18 首次发布

本文链接：https://blog.csdn.net/cloud_engineer/article/details/125902903

版权

kubernetes 专栏收录该内容

16 篇文章 5 订阅

订阅专栏

1 准备环境

1.1 主机列表

cat >> /etc/hosts <<EOF
10.0.12.16 k8s-master
EOF

1.2 关闭防火墙

systemctl disable firewalld --now

1.3 关闭selinux

setenforce 0
sed -i 's/^SELINUX=.*/SELINUX=disabled/' /etc/selinux/config

1.4 关闭swap分区

swapoff -a
sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab

1.5 更新Yum源

更改CentOS-Base.repo

wget -O /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo

新增docker.repo

wget https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo -O /etc/yum.repos.d/docker-ce.repo

新增k8s.repo

cat <<EOF > /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64/
enabled=1
gpgcheck=0
repo_gpgcheck=0
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF
yum repolist

1.6 安装必要工具

yum install wget jg yum-utils psmisc vim net-tools telnet yum-utils device-mapper-persistent-data lvm2 git -y

1.7 时间同步配置

yum install -y ntpdate
crontab -e
0 */1 * * * /usr/sbin/ntpdate ntp1.aliyun.com

1.8 将桥接的IPv4流量传递到iptables的链

cat > /etc/sysctl.d/kubernetes.conf <<EOF
net.bridge.bridge-nf-call-iptables=1
net.bridge.bridge-nf-call-ip6tables=1
net.ipv4.ip_forward=1
vm.swappiness=0
EOF
sysctl --system

2 基本组件安装

2.1 docker安装

安装docker19.03版本

yum install docker-ce-19.03.* -y

启动服务

 systemctl enable docker --now

2.2 docker镜像加速及修改cgroup方式

cat > /etc/docker/daemon.json <<EOF
 {
 "registry-mirrors": ["https://v16stybc.mirror.aliyuncs.com"],
 "exec-opts":["native.cgroupdriver=systemd"],
 "log-driver":"json-file",
 "log-opts": {
  "max-size": "100m"
  },
 "storage-driver":"overlay2",
 "storage-opts": [
  "overlay2.override_kernel_check=true"
  ]
}
EOF
systemctl restart docker

2.3 安装kubernetes1.18.6安装包

yum install -y kubelet-1.18.6 kubeadm-1.18.6 kubectl-1.18.6

3 更新证书为100年

3.1 安装go编译器

（1）下载二进制包

wget https://dl.google.com/go/go1.13.4.linux-amd64.tar.gz

（2）解压

tar -zxvf go1.13.4.linux-amd64.tar.gz -C /usr/local

（3）导入环境变量

mkdir -p /home/gopath
cat >> /etc/profile <<EOF
export GOROOT=/usr/local/go
export GOPATH=/home/gopath
export PATH=\$PATH:\$GOROOT/bin
EOF
source /etc/profile

（4）查看版本

# go version
go version go1.13.4 linux/amd64

3.2 编译kubeadm并替换

（1）下载源码

git clone https://github.com/kubernetes/kubernetes.git

（2）切换到自己的版本，修改源码，比如我的是v1.18.6版本

cd /root/kubernetes
git checkout v1.18.6

vi cmd/kubeadm/app/constants/constants.go，找到CertificateValidity，修改如下

....
const (
        // KubernetesDir is the directory Kubernetes owns for storing various configuration files
        KubernetesDir = "/etc/kubernetes"
        // ManifestsSubDirName defines directory name to store manifests
        ManifestsSubDirName = "manifests"
        // TempDirForKubeadm defines temporary directory for kubeadm
        // should be joined with KubernetesDir.
        TempDirForKubeadm = "tmp"

        // CertificateValidity defines the validity for all the signed certificates generated by kubeadm
        CertificateValidity = time.Hour * 24 * 365 * 100
....

（3）编译kubeadm

make WHAT=cmd/kubeadm

编译完生成如下目录和二进制文件

# ll _output/bin/
总用量 76708
-rwxr-xr-x 1 root root  6819840 12月 11 00:24 conversion-gen
-rwxr-xr-x 1 root root  6795264 12月 11 00:24 deepcopy-gen
-rwxr-xr-x 1 root root  6766592 12月 11 00:24 defaulter-gen
-rwxr-xr-x 1 root root  4887622 12月 11 00:24 go2make
-rwxr-xr-x 1 root root  2109440 12月 11 00:25 go-bindata
-rwxr-xr-x 1 root root 39731200 12月 11 00:26 kubeadm
-rwxr-xr-x 1 root root 11436032 12月 11 00:24 openapi-gen
[root@k8s-master01 kubernetes]# pwd
/root/kubernetes

（4）将新生成的kubeadm进行替换

cd /root/kubernetes
cp _output/bin/kubeadm /usr/bin/kubeadm

4 master节点部署

开机自启kubelet:

systemctl enable kubelet

单master节点部署直接下面

kubeadm init \
  --apiserver-advertise-address=10.0.12.16 \
  --image-repository registry.aliyuncs.com/google_containers \
  --kubernetes-version v1.18.6 \
  --service-cidr=172.16.0.0/16 \
  --pod-network-cidr=172.18.0.0/16 \
  --ignore-preflight-errors=all

输入log如下：

kubeadm init \
>   --apiserver-advertise-address=10.0.12.16 \
>   --image-repository registry.aliyuncs.com/google_containers \
>   --kubernetes-version v1.18.6 \
>   --service-cidr=172.16.0.0/16 \
>   --pod-network-cidr=172.18.0.0/16 \
>   --ignore-preflight-errors=all
W0720 22:54:33.428938   24592 configset.go:202] WARNING: kubeadm cannot validate component configs for API groups [kubelet.config.k8s.io kubeproxy.config.k8s.io]
[init] Using Kubernetes version: v1.18.6
[preflight] Running pre-flight checks
[preflight] Pulling images required for setting up a Kubernetes cluster
[preflight] This might take a minute or two, depending on the speed of your internet connection
[preflight] You can also perform this action in beforehand using 'kubeadm config images pull'
[kubelet-start] Writing kubelet environment file with flags to file "/var/lib/kubelet/kubeadm-flags.env"
[kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml"
[kubelet-start] Starting the kubelet
[certs] Using certificateDir folder "/etc/kubernetes/pki"
[certs] Generating "ca" certificate and key
[certs] Generating "apiserver" certificate and key
[certs] apiserver serving cert is signed for DNS names [k8s-master kubernetes kubernetes.default kubernetes.default.svc kubernetes.default.svc.cluster.local] and IPs [172.16.0.1 10.0.12.16]
[certs] Generating "apiserver-kubelet-client" certificate and key
[certs] Generating "front-proxy-ca" certificate and key
[certs] Generating "front-proxy-client" certificate and key
[certs] Generating "etcd/ca" certificate and key
[certs] Generating "etcd/server" certificate and key
[certs] etcd/server serving cert is signed for DNS names [k8s-master localhost] and IPs [10.0.12.16 127.0.0.1 ::1]
[certs] Generating "etcd/peer" certificate and key
[certs] etcd/peer serving cert is signed for DNS names [k8s-master localhost] and IPs [10.0.12.16 127.0.0.1 ::1]
[certs] Generating "etcd/healthcheck-client" certificate and key
[certs] Generating "apiserver-etcd-client" certificate and key
[certs] Generating "sa" key and public key
[kubeconfig] Using kubeconfig folder "/etc/kubernetes"
[kubeconfig] Writing "admin.conf" kubeconfig file
[kubeconfig] Writing "kubelet.conf" kubeconfig file
[kubeconfig] Writing "controller-manager.conf" kubeconfig file
[kubeconfig] Writing "scheduler.conf" kubeconfig file
[control-plane] Using manifest folder "/etc/kubernetes/manifests"
[control-plane] Creating static Pod manifest for "kube-apiserver"
[control-plane] Creating static Pod manifest for "kube-controller-manager"
W0720 22:55:21.759819   24592 manifests.go:225] the default kube-apiserver authorization-mode is "Node,RBAC"; using "Node,RBAC"
[control-plane] Creating static Pod manifest for "kube-scheduler"
W0720 22:55:21.761225   24592 manifests.go:225] the default kube-apiserver authorization-mode is "Node,RBAC"; using "Node,RBAC"
[etcd] Creating static Pod manifest for local etcd in "/etc/kubernetes/manifests"
[wait-control-plane] Waiting for the kubelet to boot up the control plane as static Pods from directory "/etc/kubernetes/manifests". This can take up to 4m0s
[apiclient] All control plane components are healthy after 22.002118 seconds
[upload-config] Storing the configuration used in ConfigMap "kubeadm-config" in the "kube-system" Namespace
[kubelet] Creating a ConfigMap "kubelet-config-1.18" in namespace kube-system with the configuration for the kubelets in the cluster
[upload-certs] Skipping phase. Please see --upload-certs
[mark-control-plane] Marking the node k8s-master as control-plane by adding the label "node-role.kubernetes.io/master=''"
[mark-control-plane] Marking the node k8s-master as control-plane by adding the taints [node-role.kubernetes.io/master:NoSchedule]
[bootstrap-token] Using token: vei9xh.le13jkptvpl6iru7
[bootstrap-token] Configuring bootstrap tokens, cluster-info ConfigMap, RBAC Roles
[bootstrap-token] configured RBAC rules to allow Node Bootstrap tokens to get nodes
[bootstrap-token] configured RBAC rules to allow Node Bootstrap tokens to post CSRs in order for nodes to get long term certificate credentials
[bootstrap-token] configured RBAC rules to allow the csrapprover controller automatically approve CSRs from a Node Bootstrap Token
[bootstrap-token] configured RBAC rules to allow certificate rotation for all node client certificates in the cluster
[bootstrap-token] Creating the "cluster-info" ConfigMap in the "kube-public" namespace
[kubelet-finalize] Updating "/etc/kubernetes/kubelet.conf" to point to a rotatable kubelet client certificate and key
[addons] Applied essential addon: CoreDNS
[addons] Applied essential addon: kube-proxy

Your Kubernetes control-plane has initialized successfully!

To start using your cluster, you need to run the following as a regular user:

  mkdir -p $HOME/.kube
  sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
  sudo chown $(id -u):$(id -g) $HOME/.kube/config

You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
  https://kubernetes.io/docs/concepts/cluster-administration/addons/

Then you can join any number of worker nodes by running the following on each as root:

kubeadm join 10.0.12.16:6443 --token vei9xh.le13jkptvpl6iru7 \
    --discovery-token-ca-cert-hash sha256:d3c15f377a45bec44b0f2bb6be839004230d38f3052eccee5afedcc69d089d29

这样查出的证书就是100年了，以后无需要更新了：

[root@k8s-master ~]# kubeadm alpha certs check-expiration
[check-expiration] Reading configuration from the cluster...
[check-expiration] FYI: You can look at this config file with 'kubectl -n kube-system get cm kubeadm-config -oyaml'

CERTIFICATE                EXPIRES                  RESIDUAL TIME   CERTIFICATE AUTHORITY   EXTERNALLY MANAGED
admin.conf                 Jun 26, 2122 14:55 UTC   99y                                     no      
apiserver                  Jun 26, 2122 14:55 UTC   99y             ca                      no      
apiserver-etcd-client      Jun 26, 2122 14:55 UTC   99y             etcd-ca                 no      
apiserver-kubelet-client   Jun 26, 2122 14:55 UTC   99y             ca                      no      
controller-manager.conf    Jun 26, 2122 14:55 UTC   99y                                     no      
etcd-healthcheck-client    Jun 26, 2122 14:55 UTC   99y             etcd-ca                 no      
etcd-peer                  Jun 26, 2122 14:55 UTC   99y             etcd-ca                 no      
etcd-server                Jun 26, 2122 14:55 UTC   99y             etcd-ca                 no      
front-proxy-client         Jun 26, 2122 14:55 UTC   99y             front-proxy-ca          no      
scheduler.conf             Jun 26, 2122 14:55 UTC   99y                                     no      

CERTIFICATE AUTHORITY   EXPIRES                  RESIDUAL TIME   EXTERNALLY MANAGED
ca                      Jul 17, 2032 14:55 UTC   9y              no      
etcd-ca                 Jul 17, 2032 14:55 UTC   9y              no      
front-proxy-ca          Jul 17, 2032 14:55 UTC   9y              no

5 安装网络组件calico

calico下载地址：https://docs.projectcalico.org/v3.15/manifests/calico.yaml

kubectl apply -f calico.yaml

观察集群情况：

[root@k8s-master ~]# kubectl apply -f calico.yaml
configmap/calico-config created
customresourcedefinition.apiextensions.k8s.io/bgpconfigurations.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/bgppeers.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/blockaffinities.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/clusterinformations.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/felixconfigurations.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/globalnetworkpolicies.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/globalnetworksets.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/hostendpoints.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/ipamblocks.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/ipamconfigs.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/ipamhandles.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/ippools.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/kubecontrollersconfigurations.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/networkpolicies.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/networksets.crd.projectcalico.org created
clusterrole.rbac.authorization.k8s.io/calico-kube-controllers created
clusterrolebinding.rbac.authorization.k8s.io/calico-kube-controllers created
clusterrole.rbac.authorization.k8s.io/calico-node created
clusterrolebinding.rbac.authorization.k8s.io/calico-node created
daemonset.apps/calico-node created
serviceaccount/calico-node created
deployment.apps/calico-kube-controllers created
serviceaccount/calico-kube-controllers created
[root@k8s-master ~]# kubectl get nodes
NAME         STATUS     ROLES    AGE   VERSION
k8s-master   NotReady   master   12m   v1.18.6
[root@k8s-master ~]# kubectl get pod -A
NAMESPACE     NAME                                     READY   STATUS     RESTARTS   AGE
kube-system   calico-kube-controllers-79b5f89d-d8t9v   0/1     Pending    0          7s
kube-system   calico-node-7t9v7                        0/1     Init:0/3   0          7s
kube-system   coredns-7ff77c879f-97dkq                 0/1     Pending    0          11m
kube-system   coredns-7ff77c879f-qq6kh                 0/1     Pending    0          11m
kube-system   etcd-k8s-master                          1/1     Running    0          12m
kube-system   kube-apiserver-k8s-master                1/1     Running    0          12m
kube-system   kube-controller-manager-k8s-master       1/1     Running    0          12m
kube-system   kube-proxy-z4gs8                         1/1     Running    0          11m
kube-system   kube-scheduler-k8s-master                1/1     Running    0          12m

安装calico后，集群正常使用了：

[root@k8s-master ~]# kubectl get pod -A
NAMESPACE     NAME                                     READY   STATUS    RESTARTS   AGE
kube-system   calico-kube-controllers-79b5f89d-d8t9v   1/1     Running   0          15m
kube-system   calico-node-7t9v7                        1/1     Running   0          15m
kube-system   coredns-7ff77c879f-97dkq                 1/1     Running   0          27m
kube-system   coredns-7ff77c879f-qq6kh                 1/1     Running   0          27m
kube-system   etcd-k8s-master                          1/1     Running   0          27m
kube-system   kube-apiserver-k8s-master                1/1     Running   0          27m
kube-system   kube-controller-manager-k8s-master       1/1     Running   0          27m
kube-system   kube-proxy-z4gs8                         1/1     Running   0          27m
kube-system   kube-scheduler-k8s-master                1/1     Running   0          27m
[root@k8s-master ~]# kubectl get nodes
NAME         STATUS   ROLES    AGE   VERSION
k8s-master   Ready    master   27m   v1.18.6