k8s-集群高可用安装

云深不知处～

于 2024-06-19 23:52:05 发布

阅读量416

点赞数 4

文章标签： kubernetes 容器云原生

本文链接：https://blog.csdn.net/qq_35798667/article/details/139815744

版权

环境准备

k8s版本 1.26.8
docker 20.10.22-3.el7

k8s11 192.168.0.11
k8s12 192.168.0.12
k8s13 192.168.0.13
k8s14 192.168.0.14
k8s15 192.168.0.15

192.168.0.200 浮动IP

yum install docker-ce-20.10.22-3.el7  -y

yum install -y kubeadm-1.26.8 

# 指定版本安装 
yum install kubelet-1.26.8  kubeadm-1.26.8  kubectl-1.26.8 -y

Docker准备

所有node节点安装

Docker 安装 yum源

yum-config-manager --add-repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo

yum makecache fast

Docker 安装

yum install docker-ce -y

启动Docker服务开机自启动

systemctl enable docker && systemctl start docker

# 或者

systemctl enable --now docker

修改cgroup方式及配置docker镜像加速器

cat <<EOF> /etc/docker/daemon.json
{ 	
	"exec-opts": ["native.cgroupdriver=systemd"], 	
	"registry-mirrors": ["https://pawd5kdt.mirror.aliyuncs.com"]
 } 
EOF


# 重启docker
systemctl daemon-reload
systemctl restart docker

cri-dockerd 安装

下载/上传/解压

https://github.com/Mirantis/cri-dockerd/tags
在这里插入图片描述

tar -zxvf cri-dockerd-0.3.0.amd64.tgz -C /usr/local/software/

赋权/拷贝

chmod +x cri-dockerd

# 在 k8s11 节点执行
cp cri-dockerd /usr/local/bin/
scp cri-dockerd k8s12:/usr/local/bin/
scp cri-dockerd k8s13:/usr/local/bin/
scp cri-dockerd k8s14:/usr/local/bin/
scp cri-dockerd k8s15:/usr/local/bin/

配置 cri-docker.service

cat <<"EOF" > /etc/systemd/system/cri-docker.service
[Unit]
Description=CRI Interface for Docker Application Container Engine
Documentation=https://docs.mirantis.com
After=network-online.target firewalld.service docker.service
Wants=network-online.target
Requires=cri-docker.socket

[Service]
Type=notify

ExecStart=/usr/local/bin/cri-dockerd  --pod-infra-container-image=registry.aliyuncs.com/google_containers/pause:3.9

ExecReload=/bin/kill -s HUP $MAINPID
TimeoutSec=0
RestartSec=2
Restart=always

StartLimitBurst=3

StartLimitInterval=60s

LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity

TasksMax=infinity
Delegate=yes
KillMode=process

[Install]
WantedBy=multi-user.target

EOF

配置 cri-docker.socket

cat <<"EOF" > /etc/systemd/system/cri-docker.socket
[Unit]
Description=CRI Docker Socket for the API
PartOf=cri-docker.service

[Socket]
ListenStream=%t/cri-dockerd.sock
SocketMode=0660
SocketUser=root
SocketGroup=docker

[Install]
WantedBy=sockets.target

EOF

启动cri-docker并设置开机自动启动

systemctl daemon-reload
systemctl enable cri-docker --now

systemctl is-active cri-docker

高可用组件安装

所有master节点安装


yum -y install keepalived haproxy

ha 配置文件
三台master节点均配置一样 k8s11 k8s12 k8s13
vim /etc/haproxy/haproxy.cfg


global
    log         127.0.0.1 local2
    chroot      /var/lib/haproxy
    pidfile     /var/run/haproxy.pid
    maxconn     4000
    user        haproxy
    group       haproxy
    daemon
    # turn on stats unix socket
    stats socket /var/lib/haproxy/stats
defaults
    mode                    http
    log                     global
    option                  httplog
    option                  dontlognull
    option                  http-server-close
    option                  forwardfor       except 127.0.0.0/8
    option                  redispatch
    retries                 3
    timeout http-request    10s
    timeout queue           1m
    timeout connect         10s
    timeout client          1m
    timeout server          1m
    timeout http-keep-alive 10s
    timeout check           10s
    maxconn                 3000
#---------------------------------------------------------------------
# main frontend which proxys to the backends
#---------------------------------------------------------------------
frontend apiserver
    bind *:9443
    mode tcp
    option tcplog
    default_backend apiserver
#---------------------------------------------------------------------
# static backend for serving up images, stylesheets and such
#---------------------------------------------------------------------
backend apiserver
    mode        tcp
    balance     roundrobin
    server k8s11 192.168.0.11:6443 check
    server k8s12 192.168.0.12:6443 check
    server k8s13 192.168.0.13:6443 check
#---------------------------------------------------------------------
# round robin balancing between the various backends
#---------------------------------------------------------------------
listen admin_stats
    bind *:8888  #登录页面所绑定的地址加端口
    mode http #监控的模式
    log 127.0.0.1 local0 err #错误日志等级
    stats refresh 30s
    stats uri /stats   #登录页面的网址，IP:8888/stats 即为登录网址
    stats realm welcome login\ Haproxy
    stats auth admin:admin123       #web页面的用户名和密码
    stats hide-version
    stats admin if TRUE

ha 启动

# 设置开机启动
systemctl enable haproxy

# 开启haproxy
systemctl start haproxy

# 查看启动状态
systemctl status haproxy

访问

192.168.0.11:8888/stats 指定ip访问
192.168.0.200:8888/stats vip访问
在这里插入图片描述

keepalived配置

k8s11

cat > /etc/keepalived/keepalived.conf <<EOF 

global_defs {
    router_id k8s11
}
vrrp_script check_haproxy {
    script "killall -0 haproxy"
    interval 3
    weight -2
    fall 10
    rise 2
}
vrrp_instance VI_1 {
    state MASTER
    interface ens33
    virtual_router_id 42
    priority 250
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass ceb1b3ec013d66163d6ab
    }
    virtual_ipaddress {
        192.168.0.200
    }
    track_script {
        check_haproxy
    }
}


EOF

k8s12


cat > /etc/keepalived/keepalived.conf <<EOF

global_defs {
    router_id k8s12
}
vrrp_script check_haproxy {
    script "killall -0 haproxy"
    interval 3
    weight -2
    fall 10
    rise 2
}
vrrp_instance VI_1 {
    state MASTER
    interface ens33
    virtual_router_id 42
    priority 200
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass ceb1b3ec013d66163d6ab
    }
    virtual_ipaddress {
        192.168.0.200
    }
    track_script {
        check_haproxy
    }
}


EOF

k8s13


cat > /etc/keepalived/keepalived.conf <<EOF

global_defs {
    router_id k8s13
}
vrrp_script check_haproxy {
    script "killall -0 haproxy"
    interval 3
    weight -2
    fall 10
    rise 2
}
vrrp_instance VI_1 {
    state MASTER
    interface ens33
    virtual_router_id 42
    priority 150
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass ceb1b3ec013d66163d6ab
    }
    virtual_ipaddress {
        192.168.0.200
    }
    track_script {
        check_haproxy
    }
}


EOF

keepalived 启动

# 启动keepalived
systemctl start keepalived.service
# 设置开机启动
systemctl enable keepalived.service
# 查看启动状态
systemctl status keepalived.service

k8s安装

# 所有节点安装
yum install kubelet-1.26.8  kubeadm-1.26.8  kubectl-1.26.8 -y

配置kubelet

为了实现docker使用的cgroupdriver与kubelet使用的cgroup的一致性，建议修改如下文件内容

cat <<EOF > /etc/sysconfig/kubelet
KUBELET_EXTRA_ARGS="--cgroup-driver=systemd"
EOF

# 配置开机自启动
# 设置kubelet为开机自启动即可，由于没有生成配置文件，集群初始化后自动启动
systemctl enable kubelet

# 注意，此时不需要启动，它的配置文件是在k8s集群配置之后才有效
systemctl start kubelet

查看下载镜像

查看镜像

kubeadm config images list

下载镜像

kubeadm config images pull

在这里插入图片描述

kubeadm config images pull --cri-socket unix:///var/run/cri-dockerd.sock 
--image-repository registry.aliyuncs.com/google_containers

生成配置文件


kubeadm config print init-defaults > kubeadm-config.yaml


kubeadm config print init-defaults --component-configs KubeletConfiguration

kubeadm config print init-defaults --component-configs KubeProxyConfiguration

vim kubeadm-config.yaml



apiVersion: kubeadm.k8s.io/v1beta3
bootstrapTokens:
- groups:
  - system:bootstrappers:kubeadm:default-node-token
  token: abcdef.0123456789abcdef
  ttl: 24h0m0s
  usages:
  - signing
  - authentication
kind: InitConfiguration
localAPIEndpoint:
  advertiseAddress: 192.168.0.11
  bindPort: 6443
nodeRegistration:
  criSocket: unix:///var/run/cri-dockerd.sock
  imagePullPolicy: IfNotPresent
---
apiServer:
  timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta3
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controllerManager: {}
dns: {}
etcd:
  local:
    dataDir: /var/lib/etcd
imageRepository: registry.aliyuncs.com/google_containers
kind: ClusterConfiguration
kubernetesVersion: 1.26.8
networking:
  dnsDomain: cluster.local
  podSubnet: 10.244.0.0/16
  serviceSubnet: 10.96.0.0/12
scheduler: {}
apiServerCertsSANs:
- 192.168.0.200
controlPlaneEndpoint: "192.168.0.200:9443"
---
apiVersion: kubelet.config.k8s.io/v1beta1
authentication:
  anonymous:
    enabled: false
  webhook:
    cacheTTL: 0s
    enabled: true
  x509:
    clientCAFile: /etc/kubernetes/pki/ca.crt
authorization:
  mode: Webhook
  webhook:
    cacheAuthorizedTTL: 0s
    cacheUnauthorizedTTL: 0s
cgroupDriver: systemd
clusterDNS:
- 10.96.0.10
clusterDomain: cluster.local
cpuManagerReconcilePeriod: 0s
evictionPressureTransitionPeriod: 0s
fileCheckFrequency: 0s
healthzBindAddress: 127.0.0.1
healthzPort: 10248
httpCheckFrequency: 0s
imageMinimumGCAge: 0s
kind: KubeletConfiguration
logging:
  flushFrequency: 0
  options:
    json:
      infoBufferSize: "0"
  verbosity: 0
memorySwap: {}
nodeStatusReportFrequency: 0s
nodeStatusUpdateFrequency: 0s
rotateCertificates: true
runtimeRequestTimeout: 0s
shutdownGracePeriod: 0s
shutdownGracePeriodCriticalPods: 0s
staticPodPath: /etc/kubernetes/manifests
streamingConnectionIdleTimeout: 0s
syncFrequency: 0s
volumeStatsAggPeriod: 0s

集群初始化


kubeadm init --config=kubeadm-config.yaml --upload-certs --v=9

在这里插入图片描述

加入其他master节点

复制k8s证书到其他master节点

# 复制文件
/etc/kube
rnetes/pki/ca.crt
/etc/kubernetes/pki/ca.key
/etc/kubernetes/pki/sa.key
/etc/kubernetes/pki/sa.pub
/etc/kubernetes/pki/front-proxy-ca.crt
/etc/kubernetes/pki/front-proxy-ca.key
/etc/kubernetes/pki/etcd/ca.key
/etc/kubernetes/pki/etcd/ca.crt


vim aa.txt
/etc/kubernetes/pki/ca.crt
/etc/kubernetes/pki/ca.key
/etc/kubernetes/pki/sa.key
/etc/kubernetes/pki/sa.pub
/etc/kubernetes/pki/front-proxy-ca.crt
/etc/kubernetes/pki/front-proxy-ca.key
/etc/kubernetes/pki/etcd/ca.key
/etc/kubernetes/pki/etcd/ca.crt

# 压缩
tar czf cert.tar.gz -T aa.txt

# 查看
tar tf cert.tar.gz

# 传到其他master
scp cert.tar.gz k8s12:$PWD
scp cert.tar.gz k8s13:$PWD

# 解压到指定目录
tar zxf cert.tar.gz -C /



#加入master节点
kubeadm join 192.168.0.200:9443 --token abcdef.0123456789abcdef --discovery-token-ca-cert-hash sha256:7656cff0d7180b1a126503e494c470a6d611af46876170eb1cc3cbc601a03573 --control-plane --certificate-key 78959298f66f35ec2f0b9c1c57ad894e7f26fc8705022cf6c2de3b835ed9e8e8 --cri-socket unix:///var/run/cri-dockerd.sock


# 加入工作节点
kubeadm join 192.168.0.200:9443 --token abcdef.0123456789abcdef --discovery-token-ca-cert-hash sha256:7656cff0d7180b1a126503e494c470a6d611af46876170eb1cc3cbc601a03573 --cri-socket unix:///var/run/cri-dockerd.sock

在这里插入图片描述

集群重置

kubeadm reset  --cri-socket /var/run/cri-dockerd.sock
rm -rf $HOME/.kube/config

部署网络插件 calico

# maste k8s11 节点执行
# 下载 calico 插件：
wget https://docs.projectcalico.org/manifests/calico.yaml --no-check-certificate


# 下载完后还需要修改里面定义Pod网络（CALICO_IPV4POOL_CIDR），与前面kubeadm init的–pod-network-cidr指定的一样
- name: CALICO_IPV4POOL_CIDR
    value: "10.244.0.0/16"
    
    
# 下载 calico 相关镜像 
kubectl apply -f calico.yaml

卸载

yum -y remove kubelet kubeadm kubectl
sudo kubeadm reset -f
sudo rm -rvf ~/.kube/
sudo rm -rvf /etc/kubernetes/
sudo rm -rvf /etc/systemd/system/kubelet.service.d
sudo rm -rvf /etc/systemd/system/kubelet.service
sudo rm -rvf /usr/bin/kube*
sudo rm -rvf /etc/cni
sudo rm -rvf /opt/cni
sudo rm -rvf /var/lib/etcd
sudo rm -rvf /var/etcd

云深不知处～

关注

4
点赞
踩
3

收藏

觉得还不错? 一键收藏
0
评论
k8s-集群高可用安装

为了实现docker使用的cgroupdriver与kubelet使用的cgroup的一致性，建议修改如下文件内容。192.168.0.200:8888/stats vip访问。192.168.0.11:8888/stats 指定ip访问。192.168.0.200 浮动IP。k8s版本 1.26.8。所有node节点安装。
复制链接

扫一扫