k8s高可用集群

胖胖胖胖胖虎

已于 2023-02-17 17:32:18 修改

阅读量511

点赞数

分类专栏： K8S 文章标签： kubernetes

于 2022-01-17 20:27:54 首次发布

本文链接：https://blog.csdn.net/qq_15138049/article/details/122547419

版权

K8S 专栏收录该内容

102 篇文章 1 订阅

订阅专栏

haproxy

[root@hadoop03 pki]# cat /etc/haproxy/haproxy.cfg
global
  maxconn  2000
  ulimit-n  16384
  log  127.0.0.1 local0 err
  stats timeout 30s

defaults
  log global
  mode  http
  option  httplog
  timeout connect 5000
  timeout client  50000
  timeout server  50000
  timeout http-request 15s
  timeout http-keep-alive 15s

frontend k8s-master
  bind 0.0.0.0:8443
  bind 127.0.0.1:8443
  mode tcp
  option tcplog
  tcp-request inspect-delay 5s
  default_backend k8s-master

backend k8s-master
  mode tcp
  option tcplog
  option tcp-check
  balance roundrobin
  #default-server inter 10s downinter 5s rise 2 fall 2 slowstart 60s maxconn 250 maxqueue 256 weight 100
  server hadoop03 192.168.153.103:6443  check inter 2000 fall 2 rise 2 weight 100
  server hadoop04 192.168.153.104:6443  check inter 2000 fall 2 rise 2 weight 100

keepalived

### hadoop03 ###
[root@hadoop03 pki]# cat /etc/keepalived/keepalived.conf
!Configuration: command not found

global_defs {
    router_id LVS_DEVEL
}
vrrp_script chk_kubernetes {
    script "/etc/keepalived/check_kubernetes.sh"
    interval 2
    weight -5
    fall 3
    rise 2
}
vrrp_instance VI_1 {
    state MASTER
    interface ens33
    #mcast_src_ip 192.168.153.103
    virtual_router_id 51
    priority 100
    advert_int 2
    authentication {
        auth_type PASS
        auth_pass K8SHA_KA_AUTH
    }
    virtual_ipaddress {
        192.168.153.100 ### vip
    }
    #track_script {
    #   chk_kubernetes
    #}
}



### hadoop04 ###
[root@hadoop04 pki]# cat /etc/keepalived/keepalived.conf
!Configuration: command not found

global_defs {
    router_id LVS_DEVEL
}
vrrp_script chk_kubernetes {
    script "/etc/keepalived/check_kubernetes.sh"
    interval 2
    weight -5
    fall 3
    rise 2
}
vrrp_instance VI_1 {
    state BACKUP
    interface ens33
    #mcast_src_ip 192.168.153.104
    virtual_router_id 51
    priority 100 
    advert_int 2
    authentication {
        auth_type PASS
        auth_pass K8SHA_KA_AUTH
    }
    virtual_ipaddress {
        192.168.153.100 ### vip
    }
    #track_script {
    #   chk_kubernetes
    #}
}

/etc/keepalived/chk_kubernetes.sh

#!/bin/bash
netstat -ntlp | grep 6443 || exit 1

kubeadm-config.yaml

[root@hadoop03 module]# cat kubeadm-config.yaml
apiVersion: kubeadm.k8s.io/v1beta2
kind: ClusterConfiguration
kubernetesVersion: v1.22.3
apiServer:
  certSANs:    #填写所有kube-apiserver节点的hostname、IP、VIP(好像也可以不用写，只写vip就行)
  - 192.168.153.100
  - 192.168.153.103
  - 192.168.153.104
controlPlaneEndpoint: "192.168.153.100:6443" #vip
imageRepository: hadoop03:5000

networking:
  podSubnet: "10.244.0.0/16"
  serviceSubnet: 10.1.0.0/16

---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
featureGates:
  SupportIPVSProxyMode: true
mode: ipvs

kubeadm init

[root@hadoop03 module]# kubeadm init --config /opt/module/kubeadm-config.yaml
[init] Using Kubernetes version: v1.22.3
[preflight] Running pre-flight checks
[preflight] Pulling images required for setting up a Kubernetes cluster
[preflight] This might take a minute or two, depending on the speed of your internet connection
[preflight] You can also perform this action in beforehand using 'kubeadm config images pull'
[certs] Using certificateDir folder "/etc/kubernetes/pki"
[certs] Generating "ca" certificate and key
[certs] Generating "apiserver" certificate and key
[certs] apiserver serving cert is signed for DNS names [hadoop03 kubernetes kubernetes.default kubernetes.default.svc kubernetes.default.svc.cluste  r.local] and IPs [10.96.0.1 192.168.153.103 192.168.153.100 192.168.153.101 192.168.153.102]
[certs] Generating "apiserver-kubelet-client" certificate and key
[certs] Generating "front-proxy-ca" certificate and key
[certs] Generating "front-proxy-client" certificate and key
[certs] Generating "etcd/ca" certificate and key
[certs] Generating "etcd/server" certificate and key
[certs] etcd/server serving cert is signed for DNS names [hadoop03 localhost] and IPs [192.168.153.103 127.0.0.1 ::1]
[certs] Generating "etcd/peer" certificate and key
[certs] etcd/peer serving cert is signed for DNS names [hadoop03 localhost] and IPs [192.168.153.103 127.0.0.1 ::1]
[certs] Generating "etcd/healthcheck-client" certificate and key
[certs] Generating "apiserver-etcd-client" certificate and key
[certs] Generating "sa" key and public key
[kubeconfig] Using kubeconfig folder "/etc/kubernetes"
[kubeconfig] Writing "admin.conf" kubeconfig file
[kubeconfig] Writing "kubelet.conf" kubeconfig file
[kubeconfig] Writing "controller-manager.conf" kubeconfig file
[kubeconfig] Writing "scheduler.conf" kubeconfig file
[kubelet-start] Writing kubelet environment file with flags to file "/var/lib/kubelet/kubeadm-flags.env"
[kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml"
[kubelet-start] Starting the kubelet
[control-plane] Using manifest folder "/etc/kubernetes/manifests"
[control-plane] Creating static Pod manifest for "kube-apiserver"
[control-plane] Creating static Pod manifest for "kube-controller-manager"
[control-plane] Creating static Pod manifest for "kube-scheduler"
[etcd] Creating static Pod manifest for local etcd in "/etc/kubernetes/manifests"
[wait-control-plane] Waiting for the kubelet to boot up the control plane as static Pods from directory "/etc/kubernetes/manifests". This can take   up to 4m0s
[apiclient] All control plane components are healthy after 14.004020 seconds
[upload-config] Storing the configuration used in ConfigMap "kubeadm-config" in the "kube-system" Namespace
[kubelet] Creating a ConfigMap "kubelet-config-1.22" in namespace kube-system with the configuration for the kubelets in the cluster
[upload-certs] Skipping phase. Please see --upload-certs
[mark-control-plane] Marking the node hadoop03 as control-plane by adding the labels: [node-role.kubernetes.io/master(deprecated) node-role.kuberne  tes.io/control-plane node.kubernetes.io/exclude-from-external-load-balancers]
[mark-control-plane] Marking the node hadoop03 as control-plane by adding the taints [node-role.kubernetes.io/master:NoSchedule]
[bootstrap-token] Using token: k4gu2q.zv8wqjyfuphyrjuw
[bootstrap-token] Configuring bootstrap tokens, cluster-info ConfigMap, RBAC Roles
[bootstrap-token] configured RBAC rules to allow Node Bootstrap tokens to get nodes
[bootstrap-token] configured RBAC rules to allow Node Bootstrap tokens to post CSRs in order for nodes to get long term certificate credentials
[bootstrap-token] configured RBAC rules to allow the csrapprover controller automatically approve CSRs from a Node Bootstrap Token
[bootstrap-token] configured RBAC rules to allow certificate rotation for all node client certificates in the cluster
[bootstrap-token] Creating the "cluster-info" ConfigMap in the "kube-public" namespace
[kubelet-finalize] Updating "/etc/kubernetes/kubelet.conf" to point to a rotatable kubelet client certificate and key
[addons] Applied essential addon: CoreDNS
[addons] Applied essential addon: kube-proxy

Your Kubernetes control-plane has initialized successfully!

To start using your cluster, you need to run the following as a regular user:

  mkdir -p $HOME/.kube
  sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
  sudo chown $(id -u):$(id -g) $HOME/.kube/config

Alternatively, if you are the root user, you can run:

  export KUBECONFIG=/etc/kubernetes/admin.conf

You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
  https://kubernetes.io/docs/concepts/cluster-administration/addons/

You can now join any number of control-plane nodes by copying certificate authorities
and service account keys on each node and then running the following as root:

  kubeadm join 192.168.153.100:6443 --token k4gu2q.zv8wqjyfuphyrjuw \
        --discovery-token-ca-cert-hash sha256:35f55b6af45ea5c5224c7fb412bc70d54ad8416e2156c720039bcb68efbe678f \
        --control-plane

Then you can join any number of worker nodes by running the following on each as root:

kubeadm join 192.168.153.100:6443 --token k4gu2q.zv8wqjyfuphyrjuw \
        --discovery-token-ca-cert-hash sha256:35f55b6af45ea5c5224c7fb412bc70d54ad8416e2156c720039bcb68efbe678f

在这里插入图片描述

[root@hadoop03 module]# cat cert-master.sh
USER=root # customizable
CONTROL_PLANE_IPS="192.168.153.104"
for host in ${CONTROL_PLANE_IPS}; do
    scp /etc/kubernetes/pki/ca.crt "${USER}"@$host:
    scp /etc/kubernetes/pki/ca.key "${USER}"@$host:
    scp /etc/kubernetes/pki/sa.key "${USER}"@$host:
    scp /etc/kubernetes/pki/sa.pub "${USER}"@$host:
    scp /etc/kubernetes/pki/front-proxy-ca.crt "${USER}"@$host:
    scp /etc/kubernetes/pki/front-proxy-ca.key "${USER}"@$host:
    scp /etc/kubernetes/pki/etcd/ca.crt "${USER}"@$host:etcd-ca.crt
    # Quote this line if you are using external etcd
    scp /etc/kubernetes/pki/etcd/ca.key "${USER}"@$host:etcd-ca.key
done

[root@hadoop04 ~]# cat mv-cert.sh
USER=root # customizable
mkdir -p /etc/kubernetes/pki/etcd
mv /${USER}/ca.crt /etc/kubernetes/pki/
mv /${USER}/ca.key /etc/kubernetes/pki/
mv /${USER}/sa.pub /etc/kubernetes/pki/
mv /${USER}/sa.key /etc/kubernetes/pki/
mv /${USER}/front-proxy-ca.crt /etc/kubernetes/pki/
mv /${USER}/front-proxy-ca.key /etc/kubernetes/pki/
mv /${USER}/etcd-ca.crt /etc/kubernetes/pki/etcd/ca.crt
# Quote this line if you are using external etcd
mv /${USER}/etcd-ca.key /etc/kubernetes/pki/etcd/ca.key

[root@hadoop04 pki]#  kubeadm join 192.168.153.100:6443 --token k4gu2q.zv8wqjyfuphyrjuw \
>         --discovery-token-ca-cert-hash sha256:35f55b6af45ea5c5224c7fb412bc70d54ad8416e2156c720039bcb68efbe678f \
>         --control-plane
[preflight] Running pre-flight checks
[preflight] Reading configuration from the cluster...
[preflight] FYI: You can look at this config file with 'kubectl -n kube-system get cm kubeadm-config -o yaml'
[preflight] Running pre-flight checks before initializing the new control plane instance
[preflight] Pulling images required for setting up a Kubernetes cluster
[preflight] This might take a minute or two, depending on the speed of your internet connection
[preflight] You can also perform this action in beforehand using 'kubeadm config images pull'
[certs] Using certificateDir folder "/etc/kubernetes/pki"
[certs] Generating "apiserver" certificate and key
[certs] apiserver serving cert is signed for DNS names [hadoop04 kubernetes kubernetes.default kubernetes.default.s                                  vc kubernetes.default.svc.cluster.local] and IPs [10.96.0.1 192.168.153.104 192.168.153.100 192.168.153.101 192.168                                  .153.102 192.168.153.103]
[certs] Generating "apiserver-kubelet-client" certificate and key
[certs] Generating "etcd/server" certificate and key
[certs] etcd/server serving cert is signed for DNS names [hadoop04 localhost] and IPs [192.168.153.104 127.0.0.1 ::                                  1]
[certs] Generating "etcd/healthcheck-client" certificate and key
[certs] Generating "apiserver-etcd-client" certificate and key
[certs] Generating "etcd/peer" certificate and key
[certs] etcd/peer serving cert is signed for DNS names [hadoop04 localhost] and IPs [192.168.153.104 127.0.0.1 ::1]
[certs] Generating "front-proxy-client" certificate and key
[certs] Valid certificates and keys now exist in "/etc/kubernetes/pki"
[certs] Using the existing "sa" key
[kubeconfig] Generating kubeconfig files
[kubeconfig] Using kubeconfig folder "/etc/kubernetes"
[kubeconfig] Writing "admin.conf" kubeconfig file
[kubeconfig] Writing "controller-manager.conf" kubeconfig file
[kubeconfig] Writing "scheduler.conf" kubeconfig file
[control-plane] Using manifest folder "/etc/kubernetes/manifests"
[control-plane] Creating static Pod manifest for "kube-apiserver"
[control-plane] Creating static Pod manifest for "kube-controller-manager"
[control-plane] Creating static Pod manifest for "kube-scheduler"
[check-etcd] Checking that the etcd cluster is healthy
[kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml"
[kubelet-start] Writing kubelet environment file with flags to file "/var/lib/kubelet/kubeadm-flags.env"
[kubelet-start] Starting the kubelet
[kubelet-start] Waiting for the kubelet to perform the TLS Bootstrap...
[etcd] Announced new etcd member joining to the existing etcd cluster
[etcd] Creating static Pod manifest for "etcd"
[etcd] Waiting for the new etcd member to join the cluster. This can take up to 40s
The 'update-status' phase is deprecated and will be removed in a future release. Currently it performs no operation
[mark-control-plane] Marking the node hadoop04 as control-plane by adding the labels: [node-role.kubernetes.io/mast                                  er(deprecated) node-role.kubernetes.io/control-plane node.kubernetes.io/exclude-from-external-load-balancers]
[mark-control-plane] Marking the node hadoop04 as control-plane by adding the taints [node-role.kubernetes.io/maste                                  r:NoSchedule]

This node has joined the cluster and a new control plane instance was created:

* Certificate signing request was sent to apiserver and approval was received.
* The Kubelet was informed of the new secure connection details.
* Control plane (master) label and taint were applied to the new node.
* The Kubernetes control plane instances scaled up.
* A new etcd member was added to the local/stacked etcd cluster.

To start administering your cluster from this node, you need to run the following as a regular user:

        mkdir -p $HOME/.kube
        sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
        sudo chown $(id -u):$(id -g) $HOME/.kube/config

Run 'kubectl get nodes' to see this node join the cluster.

[root@hadoop03 module]# kubectl get nodes
NAME       STATUS     ROLES                  AGE   VERSION
hadoop03   Ready      control-plane,master   40m   v1.22.3
hadoop04   Ready      control-plane,master   13m   v1.22.3

[root@hadoop03 k8s]# kubectl  get pod
NAME                     READY   STATUS    RESTARTS   AGE
nginx-controller-w9p2h   1/1     Running   0          16s
[root@hadoop03 k8s]# kubectl  get svc
NAME                     TYPE        CLUSTER-IP    EXTERNAL-IP   PORT(S)          AGE
kubernetes               ClusterIP   10.1.0.1      <none>        443/TCP          29h
nginx-service-nodeport   NodePort    10.1.116.80   <none>        8000:31583/TCP   2s
[root@hadoop03 k8s]# curl localhost:31583
^C
[root@hadoop03 k8s]# curl hadoop01:31583
<!DOCTYPE html>
<html>
<head>
<title>Welcome to nginx!</title>
<style>
html { color-scheme: light dark; }
body { width: 35em; margin: 0 auto;
font-family: Tahoma, Verdana, Arial, sans-serif; }
</style>
</head>
<body>
<h1>Welcome to nginx!</h1>
<p>If you see this page, the nginx web server is successfully installed and
working. Further configuration is required.</p>

<p>For online documentation and support please refer to
<a href="http://nginx.org/">nginx.org</a>.<br/>
Commercial support is available at
<a href="http://nginx.com/">nginx.com</a>.</p>

<p><em>Thank you for using nginx.</em></p>
</body>
</html>
[root@hadoop03 k8s]# kubectl  get pod
NAME                     READY   STATUS    RESTARTS   AGE
nginx-controller-w9p2h   1/1     Running   0          77s
[root@hadoop03 k8s]# kubectl  get nodes
NAME       STATUS   ROLES                  AGE     VERSION
hadoop01   Ready    <none>                 5m27s   v1.22.3
hadoop03   Ready    control-plane,master   29h     v1.22.3
hadoop04   Ready    control-plane,master   29h     v1.22.3
[root@hadoop03 k8s]# service kubelet stop
Redirecting to /bin/systemctl stop  kubelet.service
[root@hadoop03 k8s]# ps aux | grep kubelet
root       3833 12.8  7.8 1103712 303764 ?      Ssl  09:46   6:49 kube-apiserver --advertise-address=192.168.15                                  3.103 --allow-privileged=true --authorization-mode=Node,RBAC --client-ca-file=/etc/kubernetes/pki/ca.crt --enab                                  le-admission-plugins=NodeRestriction --enable-bootstrap-token-auth=true --etcd-cafile=/etc/kubernetes/pki/etcd/                                  ca.crt --etcd-certfile=/etc/kubernetes/pki/apiserver-etcd-client.crt --etcd-keyfile=/etc/kubernetes/pki/apiserv                                  er-etcd-client.key --etcd-servers=https://127.0.0.1:2379 --kubelet-client-certificate=/etc/kubernetes/pki/apise                                  rver-kubelet-client.crt --kubelet-client-key=/etc/kubernetes/pki/apiserver-kubelet-client.key --kubelet-preferr                                  ed-address-types=InternalIP,ExternalIP,Hostname --proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client                                  .crt --proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client.key --requestheader-allowed-names=front-pro                                  xy-client --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.crt --requestheader-extra-headers-p                                  refix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-Us                                  er --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-k                                  ey-file=/etc/kubernetes/pki/sa.pub --service-account-signing-key-file=/etc/kubernetes/pki/sa.key --service-clus                                  ter-ip-range=10.1.0.0/16 --tls-cert-file=/etc/kubernetes/pki/apiserver.crt --tls-private-key-file=/etc/kubernet                                  es/pki/apiserver.key
root     130090  0.0  0.0 112656   956 pts/0    S+   10:39   0:00 grep --color=auto kubelet
[root@hadoop03 k8s]# kill -9 3833
[root@hadoop03 k8s]# curl hadoop01:31583
<!DOCTYPE html>
<html>
<head>
<title>Welcome to nginx!</title>
<style>
html { color-scheme: light dark; }
body { width: 35em; margin: 0 auto;
font-family: Tahoma, Verdana, Arial, sans-serif; }
</style>
</head>
<body>
<h1>Welcome to nginx!</h1>
<p>If you see this page, the nginx web server is successfully installed and
working. Further configuration is required.</p>

<p>For online documentation and support please refer to
<a href="http://nginx.org/">nginx.org</a>.<br/>
Commercial support is available at
<a href="http://nginx.com/">nginx.com</a>.</p>

<p><em>Thank you for using nginx.</em></p>
</body>
</html>
[root@hadoop03 k8s]# curl hadoop02:31583
curl: (7) Failed connect to hadoop02:31583; Connection refused
[root@hadoop03 k8s]# curl hadoop01:31583
<!DOCTYPE html>
<html>
<head>
<title>Welcome to nginx!</title>
<style>
html { color-scheme: light dark; }
body { width: 35em; margin: 0 auto;
font-family: Tahoma, Verdana, Arial, sans-serif; }
</style>
</head>
<body>
<h1>Welcome to nginx!</h1>
<p>If you see this page, the nginx web server is successfully installed and
working. Further configuration is required.</p>

<p>For online documentation and support please refer to
<a href="http://nginx.org/">nginx.org</a>.<br/>
Commercial support is available at
<a href="http://nginx.com/">nginx.com</a>.</p>

<p><em>Thank you for using nginx.</em></p>
</body>
</html>
[root@hadoop03 k8s]#
[root@hadoop03 k8s]# kubectl  get nodes
NAME       STATUS     ROLES                  AGE     VERSION
hadoop01   Ready      <none>                 9m10s   v1.22.3
hadoop03   NotReady   control-plane,master   29h     v1.22.3
hadoop04   Ready      control-plane,master   29h     v1.22.3