k8s二进制安装

已于 2022-04-07 13:59:36 修改
阅读量1k 收藏 4
点赞数
分类专栏： linux 文章标签： linux docker
于 2022-03-25 08:14:01 首次发布
版权声明：本文为博主原创文章，遵循 CC 4.0 BY-SA 版权协议，转载请附上原文出处链接和本声明。
本文链接：https://blog.csdn.net/m0_67954492/article/details/123426077
版权

基础环境准备

1.配置本地hosts解析

192.168.0.71 k8s-master01
192.168.0.72 k8s-master02
192.168.0.73 k8s-master03
192.168.0.236 k8s-master-lb
192.168.0.74 k8s-node01
192.168.0.75 k8s-node02

2.配置yum源

curl -o /etc/yum.repos.d/CentOS-Base.repo https://mirrors.aliyun.com/repo/Centos-7.repo
yum install -y yum-utils device-mapper-persistent-data lvm2
yum-config-manager --add-repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
sed -i -e '/mirrors.cloud.aliyuncs.com/d' -e '/mirrors.aliyuncs.com/d' /etc/yum.repos.d/CentOS-Base.repo

3.安装必备软件

yum install wget jq psmisc vim net-tools telnet yum-utils device-mapper-persistent-data lvm2 git -y

4.关闭冲突软件

systemctl disable --now firewalld 
systemctl disable --now dnsmasq
systemctl disable --now NetworkManager​
setenforce 0
sed -i 's#SELINUX=enforcing#SELINUX=disabled#g' /etc/sysconfig/selinux
sed -i 's#SELINUX=enforcing#SELINUX=disabled#g' /etc/selinux/config

5.所有节点停掉swap

所有节点关闭swap分区，fstab注释swap
swapoff -a && sysctl -w vm.swappiness=0
sed -ri '/^[^#]*swap/s@^@#@' /etc/fstab

6.配置时间同步

安装ntpdate
rpm -ivh http://mirrors.wlnmp.com/centos/wlnmp-release-centos.noarch.rpm
yum install ntpdate -y​
所有节点同步时间。时间同步配置如下：
ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
echo 'Asia/Shanghai' >/etc/timezone
ntpdate time2.aliyun.com
# 加入到crontab
*/5 * * * * /usr/sbin/ntpdate time2.aliyun.com

cd /root/ ; git clone https://github.com.cnpmjs.org/dotbalo/k8s-ha-install.git

7.系统内核优化

vim /etc/security/limits.conf
i# 末尾添加如下内容
* soft nofile 655360
* hard nofile 131072
* soft nproc 655350
* hard nproc 655350
* soft memlock unlimited
* hard memlock unlimited　

master01配置所有节点免密登陆

ssh-keygen
for i in k8s-master01 k8s-master02 k8s-master03 k8s-node01 k8s-node02;do sshpass -p1 ssh-copy-id -i /root/.ssh/id_rsa.pub -o StrictHostKeyChecking=no $i;done

为了避免k8s版本太新导致与内核版本不兼容需要升级系统及内核版本

升级系统

yum update -y --exclude=kernel* && reboot

升级系统内核

# 下载内核升级包
cd /root
wget http://193.49.22.109/elrepo/kernel/el7/x86_64/RPMS/kernel-ml-devel-4.19.12-1.el7.elrepo.x86_64.rpm
wget http://193.49.22.109/elrepo/kernel/el7/x86_64/RPMS/kernel-ml-4.19.12-1.el7.elrepo.x86_64.rpm
# 所有节点安装内核
cd /root && yum localinstall -y kernel-ml*
# 更改内核启动顺序
grub2-set-default  0 && grub2-mkconfig -o /etc/grub2.cfg
grubby --args="user_namespace.enable=1" --update-kernel="$(grubby --default-kernel)"
# 检查内核版本情况
[root@k8s-master02 ~]# grubby --default-kernel
/boot/vmlinuz-4.19.12-1.el7.elrepo.x86_64

8.所有节点安装lvs

# 安装软件包
yum install ipvsadm ipset sysstat conntrack libseccomp -y
# 所有节点配置ipvs模块，在内核4.19+版本nf_conntrack_ipv4已经改为nf_conntrack， 4.18以下使用nf_conntrack_ipv4即可：
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack
vim /etc/modules-load.d/ipvs.conf    

 # 加入以下内容
ip_vs
ip_vs_lc
ip_vs_wlc
ip_vs_rr
ip_vs_wrr
ip_vs_lblc
ip_vs_lblcr
ip_vs_dh
ip_vs_sh
ip_vs_fo
ip_vs_nq
ip_vs_sed
ip_vs_ftp
ip_vs_sh
nf_conntrack
ip_tables
ip_set
xt_set
ipt_set
ipt_rpfilter
ipt_REJECT
ipip

    # 然后执行systemctl enable --now systemd-modules-load.service即可

# 检查是否加载：
[root@k8s-master01 ~]# lsmod | grep -e ip_vs -e nf_conntrack
nf_conntrack_ipv4      16384  23 
nf_defrag_ipv4         16384  1 nf_conntrack_ipv4
nf_conntrack          135168  10 xt_conntrack,nf_conntrack_ipv6,nf_conntrack_ipv4,nf_nat,nf_nat_ipv6,ipt_MASQUERADE,nf_nat_ipv4,xt_nat,nf_conntrack_netlink,ip_vs

# 开启一些k8s集群中必须的内核参数，所有节点配置k8s内核：
cat <<EOF > /etc/sysctl.d/k8s.conf
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
fs.may_detach_mounts = 1
vm.overcommit_memory=1
vm.panic_on_oom=0
fs.inotify.max_user_watches=89100
fs.file-max=52706963
fs.nr_open=52706963
net.netfilter.nf_conntrack_max=2310720

net.ipv4.tcp_keepalive_time = 600
net.ipv4.tcp_keepalive_probes = 3
net.ipv4.tcp_keepalive_intvl =15
net.ipv4.tcp_max_tw_buckets = 36000
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_max_orphans = 327680
net.ipv4.tcp_orphan_retries = 3
net.ipv4.tcp_syncookies = 1
net.ipv4.tcp_max_syn_backlog = 16384
net.ipv4.ip_conntrack_max = 65536
net.ipv4.tcp_max_syn_backlog = 16384
net.ipv4.tcp_timestamps = 0
net.core.somaxconn = 16384
EOF
sysctl --system

# 所有节点配置完内核后，重启服务器，保证重启后内核依旧加载
reboot
lsmod | grep --color=auto -e ip_vs -e nf_conntrack

安装部署docker及k8s组件

安装docker

yum install docker-ce-19.03.* -y

配置docker

mkdir /etc/docker
cat > /etc/docker/daemon.json <<EOF
{
  "exec-opts": ["native.cgroupdriver=systemd"],
  "insecure-registries": ["harbor.michaelxia.cn"]
}
EOF

启动docker

systemctl daemon-reload && systemctl enable --now docker

安装部署k8s组件及etcd

下载k8s安装包

wget https://dl.k8s.io/v1.20.0/kubernetes-server-linux-amd64.tar.gz

下载etcd安装包

官方地址：v3.5 docs | etcd

ETCD_VER=v3.5.0

# choose either URL
GOOGLE_URL=https://storage.googleapis.com/etcd
GITHUB_URL=https://github.com/etcd-io/etcd/releases/download
DOWNLOAD_URL=${GOOGLE_URL}

rm -f /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz
rm -rf /tmp/etcd-download-test && mkdir -p /tmp/etcd-download-test

curl -L ${DOWNLOAD_URL}/${ETCD_VER}/etcd-${ETCD_VER}-linux-amd64.tar.gz -o etcd-${ETCD_VER}-linux-amd64.tar.gz

# curl -L https://storage.googleapis.com/etcd/v3.5.0/etcd-v3.5.0-linux-amd64.tar.gz -o etcd-v3.5.0-linux-amd64.tar.gz

解压

tar -xf kubernetes-server-linux-amd64.tar.gz  --strip-components=3 -C /usr/local/bin kubernetes/server/bin/kube{let,ctl,-apiserver,-controller-manager,-scheduler,-proxy}
tar -zxvf etcd-v3.5.0-linux-amd64.tar.gz --strip-components=1 -C /usr/local/bin etcd-v3.5.0-linux-amd64/etcd{,ctl}

分发组件到各节点

MasterNodes='k8s-master02 k8s-master03'
WorkNodes='k8s-node01 k8s-node02'
for NODE in $MasterNodes; do echo $NODE; scp /usr/local/bin/kube{let,ctl,-apiserver,-controller-manager,-scheduler,-proxy} $NODE:/usr/local/bin/; scp /usr/local/bin/etcd* $NODE:/usr/local/bin/; done
for NODE in $WorkNodes; do     scp /usr/local/bin/kube{let,-proxy} $NODE:/usr/local/bin/ ; done

Master01上在软件包项目中切换分支到1.20

cd k8s-ha-install && git checkout manual-installation-v1.20.x

生成证书

在k8s内部各组件交互过程中，k8s是通过双向认证来保障数据传输安全性的，因此各组件在通信时需要通过证书加密传输，master和node节点都需要配置相应的证书才可以实现通信。

这里使用cfssl来生成证书工具

在master01上准备cfssl工具

wget "https://pkg.cfssl.org/R1.2/cfssl_linux-amd64" -O /usr/local/bin/cfssl
wget "https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64" -O /usr/local/bin/cfssljson
chmod +x /usr/local/bin/cfssl /usr/local/bin/cfssljson

etcd证书

在所有master节点创建etcd证书目录

mkdir /etc/etcd/ssl -p

创建kubernetes相关证书目录

mkdir -p /etc/kubernetes/pki

生成etcd证书

 cd /root/k8s-ha-install/pki

# 生成etcd CA证书和CA证书的key
cfssl gencert -initca etcd-ca-csr.json | cfssljson -bare /etc/etcd/ssl/etcd-ca​
cfssl gencert \
   --ca=/etc/etcd/ssl/etcd-ca.pem \
   --ca-key=/etc/etcd/ssl/etcd-ca-key.pem \
   --config=ca-config.json \
   --hostname=127.0.0.1,192.168.0.71,192.168.0.72,192.168.0.73,192.168.0.76,192.168.0.77,192.168.0.78 \
   --profile=kubernetes \
   etcd-csr.json | cfssljson -bare /etc/etcd/ssl/etcd

　将证书复制到其他节点

MasterNodes='k8s-master02 k8s-master03'

WorkNodes='k8s-node01 k8s-node02'

for NODE in $MasterNodes; do
   ssh $NODE "mkdir -p /etc/etcd/ssl"
for FILE in etcd-ca-key.pem  etcd-ca.pem  etcd-key.pem  etcd.pem; do 
scp /etc/etcd/ssl/${FILE} $NODE:/etc/etcd/ssl/${FILE}
   done
 done

K8S组件证书

Master01生成kubernetes CA证书

[root@k8s-master01 pki]# cd /root/k8s-ha-install/pki
cfssl gencert -initca ca-csr.json | cfssljson -bare /etc/kubernetes/pki/ca

# 查询证书有效期
openssl x509 -in /etc/kubernetes/pki/ca.pem -noout -dates

# 10.96.0.是k8s service的网段，如果说需要更改k8s service网段，那就需要更改10.96.0.1

# 如果不是高可用集群，192.168.0.236为Master01的IP

cfssl gencert   -ca=/etc/kubernetes/pki/ca.pem   -ca-key=/etc/kubernetes/pki/ca-key.pem   -config=ca-config.json -hostname=10.96.0.1,192.168.0.236,127.0.0.1,kubernetes,kubernetes.default,kubernetes.default.svc,kubernetes.default.svc.cluster,kubernetes.default.svc.cluster.local,192.168.0.71,192.168.0.72,192.168.0.73   -profile=kubernetes   apiserver-csr.json | cfssljson -bare /etc/kubernetes/pki/apiserver

生成apiserver的聚合证书。Requestheader-client-xxx requestheader-allowwd-xxx:aggerator

cfssl gencert   -initca front-proxy-ca-csr.json | cfssljson -bare /etc/kubernetes/pki/front-proxy-ca 

cfssl gencert   -ca=/etc/kubernetes/pki/front-proxy-ca.pem   -ca-key=/etc/kubernetes/pki/front-proxy-ca-key.pem   -config=ca-config.json   -profile=kubernetes   front-proxy-client-csr.json | cfssljson -bare /etc/kubernetes/pki/front-proxy-client

返回结果（忽略警告）

生成controller-manage的证书

cfssl gencert \
   -ca=/etc/kubernetes/pki/ca.pem \
   -ca-key=/etc/kubernetes/pki/ca-key.pem \
   -config=ca-config.json \
   -profile=kubernetes \
   manager-csr.json | cfssljson -bare /etc/kubernetes/pki/controller-manager

# 注意，如果不是高可用集群，192.168.0.236:8443改为master01的地址，8443改为apiserver的端口，默认是6443

# set-cluster：设置一个集群项，
kubectl config set-cluster kubernetes \
     --certificate-authority=/etc/kubernetes/pki/ca.pem \
     --embed-certs=true \
     --server=https://192.168.0.236:8443 \
     --kubeconfig=/etc/kubernetes/controller-manager.kubeconfig

# 设置一个环境项，一个上下文
kubectl config set-context system:kube-controller-manager@kubernetes \
    --cluster=kubernetes \
    --user=system:kube-controller-manager \
    --kubeconfig=/etc/kubernetes/controller-manager.kubeconfig​
# set-credentials 设置一个用户项

kubectl config set-credentials system:kube-controller-manager \
     --client-certificate=/etc/kubernetes/pki/controller-manager.pem \
     --client-key=/etc/kubernetes/pki/controller-manager-key.pem \
     --embed-certs=true \
     --kubeconfig=/etc/kubernetes/controller-manager.kubeconfig

# 使用某个环境当做默认环境

kubectl config use-context system:kube-controller-manager@kubernetes \
     --kubeconfig=/etc/kubernetes/controller-manager.kubeconfig

# 设置scheduler证书
cfssl gencert \
   --ca=/etc/kubernetes/pki/ca.pem \
   --ca-key=/etc/kubernetes/pki/ca-key.pem \
   --config=ca-config.json \
   --profile=kubernetes \
   scheduler-csr.json | cfssljson -bare /etc/kubernetes/pki/scheduler

# 注意，如果不是高可用集群，192.168.0.236:8443改为master01的地址，8443改为apiserver的端口，默认是6443

kubectl config set-cluster kubernetes \
     --certificate-authority=/etc/kubernetes/pki/ca.pem \
     --embed-certs=true \
     --server=https://192.168.0.236:8443 \
     --kubeconfig=/etc/kubernetes/scheduler.kubeconfig

kubectl config set-credentials system:kube-scheduler \
     --client-certificate=/etc/kubernetes/pki/scheduler.pem \
     --client-key=/etc/kubernetes/pki/scheduler-key.pem \
     --embed-certs=true \
     --kubeconfig=/etc/kubernetes/scheduler.kubeconfig

kubectl config set-context system:kube-scheduler@kubernetes \
     --cluster=kubernetes \
     --user=system:kube-scheduler \
     --kubeconfig=/etc/kubernetes/scheduler.kubeconfig

kubectl config use-context system:kube-scheduler@kubernetes \
     --kubeconfig=/etc/kubernetes/scheduler.kubeconfig

cfssl gencert \
   --ca=/etc/kubernetes/pki/ca.pem \
   --ca-key=/etc/kubernetes/pki/ca-key.pem \
   --config=ca-config.json \
   --profile=kubernetes \
   admin-csr.json | cfssljson -bare /etc/kubernetes/pki/admin
​
# 注意，如果不是高可用集群，192.168.0.236:8443改为master01的地址，8443改为apiserver的端口，默认是6443​
kubectl config set-cluster kubernetes     --certificate-authority=/etc/kubernetes/pki/ca.pem     --embed-certs=true     --server=https://192.168.0.236:8443     --kubeconfig=/etc/kubernetes/admin.kubeconfig
kubectl config set-credentials kubernetes-admin     --client-certificate=/etc/kubernetes/pki/admin.pem     --client-key=/etc/kubernetes/pki/admin-key.pem     --embed-certs=true     --kubeconfig=/etc/kubernetes/admin.kubeconfig

kubectl config set-context kubernetes-admin@kubernetes                  --cluster=kubernetes                                                    --user=kubernetes-admin --kubeconfig=/etc/kubernetes/admin.kubeconfig

kubectl config use-context kubernetes-admin@kubernetes     --kubeconfig=/etc/kubernetes/admin.kubeconfig

创建ServiceAccount Key--secret

openssl genrsa -out /etc/kubernetes/pki/sa.key 2048
openssl rsa -in /etc/kubernetes/pki/sa.key -pubout -out /etc/kubernetes/pki/sa.pub

发送证书到各master节点

#!/bin/bash
for NODE in k8s-master02 k8s-master03
do
    for FILE in $(ls /etc/kubernetes/pki | grep -v etcd)
    do
        scp /etc/kubernetes/pki/${FILE} $NODE:/etc/kubernetes/pki/${FILE}
    done
    for FILE in admin.kubeconfig controller-manager.kubeconfig scheduler.kubeconfig
    do
        scp /etc/kubernetes/${FILE} $NODE:/etc/kubernetes/${FILE}
    done
done

配置etcd及高可用架构

etcd配置

三节点分别编写配置文件

master01

vim /etc/etcd/etcd.config.yml
iname: 'k8s-master01'
data-dir: /var/lib/etcd
wal-dir: /var/lib/etcd/wal
snapshot-count: 5000
heartbeat-interval: 100
election-timeout: 1000
quota-backend-bytes: 0
listen-peer-urls: 'https://192.168.0.71:2380'
listen-client-urls: 'https://192.168.0.71:2379,http://127.0.0.1:2379'
max-snapshots: 3
max-wals: 5
cors:
initial-advertise-peer-urls: 'https://192.168.0.71:2380'
advertise-client-urls: 'https://192.168.0.71:2379'
discovery:
discovery-fallback: 'proxy'
discovery-proxy:
discovery-srv:
initial-cluster: 'k8s-master01=https://192.168.0.71:2380,k8s-master02=https://192.168.0.72:2380,k8s-master03=https://192.168.0.73:2380'
initial-cluster-token: 'etcd-k8s-cluster'
initial-cluster-state: 'new'
strict-reconfig-check: false
enable-v2: true
enable-pprof: true
proxy: 'off'
proxy-failure-wait: 5000
proxy-refresh-interval: 30000
proxy-dial-timeout: 1000
proxy-write-timeout: 5000
proxy-read-timeout: 0
client-transport-security:
  cert-file: '/etc/kubernetes/pki/etcd/etcd.pem'
  key-file: '/etc/kubernetes/pki/etcd/etcd-key.pem'
  client-cert-auth: true
  trusted-ca-file: '/etc/kubernetes/pki/etcd/etcd-ca.pem'
  auto-tls: true
peer-transport-security:
  cert-file: '/etc/kubernetes/pki/etcd/etcd.pem'
  key-file: '/etc/kubernetes/pki/etcd/etcd-key.pem'
  peer-client-cert-auth: true
  trusted-ca-file: '/etc/kubernetes/pki/etcd/etcd-ca.pem'
  auto-tls: true
debug: false
log-package-levels:
log-outputs: [default]
force-new-cluster: false

master02

vim /etc/etcd/etcd.config.yml
iname: 'k8s-master02'
data-dir: /var/lib/etcd
wal-dir: /var/lib/etcd/wal
snapshot-count: 5000
heartbeat-interval: 100
election-timeout: 1000
quota-backend-bytes: 0
listen-peer-urls: 'https://192.168.0.72:2380'
listen-client-urls: 'https://192.168.0.72:2379,http://127.0.0.1:2379'
max-snapshots: 3
max-wals: 5
cors:
initial-advertise-peer-urls: 'https://192.168.0.72:2380'
advertise-client-urls: 'https://192.168.0.72:2379'
discovery:
discovery-fallback: 'proxy'
discovery-proxy:
discovery-srv:
initial-cluster: 'k8s-master01=https://192.168.0.71:2380,k8s-master02=https://192.168.0.72:2380,k8s-master03=https://192.168.0.73:2380'
initial-cluster-token: 'etcd-k8s-cluster'
initial-cluster-state: 'new'
strict-reconfig-check: false
enable-v2: true
enable-pprof: true
proxy: 'off'
proxy-failure-wait: 5000
proxy-refresh-interval: 30000
proxy-dial-timeout: 1000
proxy-write-timeout: 5000
proxy-read-timeout: 0
client-transport-security:
  cert-file: '/etc/kubernetes/pki/etcd/etcd.pem'
  key-file: '/etc/kubernetes/pki/etcd/etcd-key.pem'
  client-cert-auth: true
  trusted-ca-file: '/etc/kubernetes/pki/etcd/etcd-ca.pem'
  auto-tls: true
peer-transport-security:
  cert-file: '/etc/kubernetes/pki/etcd/etcd.pem'
  key-file: '/etc/kubernetes/pki/etcd/etcd-key.pem'
  peer-client-cert-auth: true
  trusted-ca-file: '/etc/kubernetes/pki/etcd/etcd-ca.pem'
  auto-tls: true
debug: false
log-package-levels:
log-outputs: [default]
force-new-cluster: false

Master03

vim /etc/etcd/etcd.config.yml
iname: 'k8s-master03'
data-dir: /var/lib/etcd
wal-dir: /var/lib/etcd/wal
snapshot-count: 5000
heartbeat-interval: 100
election-timeout: 1000
quota-backend-bytes: 0
listen-peer-urls: 'https://192.168.0.73:2380'
listen-client-urls: 'https://192.168.0.73:2379,http://127.0.0.1:2379'
max-snapshots: 3
max-wals: 5
cors:
initial-advertise-peer-urls: 'https://192.168.0.73:2380'
advertise-client-urls: 'https://192.168.0.73:2379'
discovery:
discovery-fallback: 'proxy'
discovery-proxy:
discovery-srv:
initial-cluster: 'k8s-master01=https://192.168.0.71:2380,k8s-master02=https://192.168.0.72:2380,k8s-master03=https://192.168.0.73:2380'
initial-cluster-token: 'etcd-k8s-cluster'
initial-cluster-state: 'new'
strict-reconfig-check: false
enable-v2: true
enable-pprof: true
proxy: 'off'
proxy-failure-wait: 5000
proxy-refresh-interval: 30000
proxy-dial-timeout: 1000
proxy-write-timeout: 5000
proxy-read-timeout: 0
client-transport-security:
  cert-file: '/etc/kubernetes/pki/etcd/etcd.pem'
  key-file: '/etc/kubernetes/pki/etcd/etcd-key.pem'
  client-cert-auth: true
  trusted-ca-file: '/etc/kubernetes/pki/etcd/etcd-ca.pem'
  auto-tls: true
peer-transport-security:
  cert-file: '/etc/kubernetes/pki/etcd/etcd.pem'
  key-file: '/etc/kubernetes/pki/etcd/etcd-key.pem'
  peer-client-cert-auth: true
  trusted-ca-file: '/etc/kubernetes/pki/etcd/etcd-ca.pem'
  auto-tls: true
debug: false
log-package-levels:
log-outputs: [default]
force-new-cluster: false

配置启动管理

vim /usr/lib/systemd/system/etcd.service
i[Unit]
Description=Etcd Service
Documentation=https://coreos.com/etcd/docs/latest/
After=network.target

[Service]
Type=notify
ExecStart=/usr/local/bin/etcd --config-file=/etc/etcd/etcd.config.yml
Restart=on-failure
RestartSec=10
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
Alias=etcd3.service

配置etcd证书路径

mkdir /etc/kubernetes/pki/etcd
ln -s /etc/etcd/ssl/* /etc/kubernetes/pki/etcd/
systemctl daemon-reload
systemctl enable --now etcd

启动成功后查看状态

export ETCDCTL_API=3
etcdctl --endpoints="192.168.0.73:2379,192.168.0.72:2379,192.168.0.71:2379" --cacert=/etc/kubernetes/pki/etcd/etcd-ca.pem --cert=/etc/kubernetes/pki/etcd/etcd.pem --key=/etc/kubernetes/pki/etcd/etcd-key.pem  endpoint status --write-out=table

配置高可用架构

高可用配置（注意：如果不是高可用集群，haproxy和keepalived无需安装）

如果在云上安装也无需执行此章节的步骤，可以直接使用云上的lb，比如阿里云slb，腾讯云elb等

公有云要用公有云自带的负载均衡，比如阿里云的SLB，腾讯云的ELB，用来替代haproxy和keepalived，因为公有云大部分都是不支持keepalived的，另外如果用阿里云的话，kubectl控制端不能放在master节点，推荐使用腾讯云，因为阿里云的slb有回环的问题，也就是slb代理的服务器不能反向访问SLB，但是腾讯云修复了这个问题。

安装keepalive与haproxy

yum install keepalived haproxy nginx-all-modules.noarch -y

通过nginx实现高可用调度

vim /etc/nginx/nginx.conf
# 在event模块后面添加四层代理
stream {
    server {
      listen 8443;
      proxy_pass k8s_api;
    }
    upstream k8s_api {
        server 192.168.0.71:6443;
        server 192.168.0.72:6443;
        server 192.168.0.73:6443;
    }

}

所有节点ha配置相同

global
  maxconn  2000
  ulimit-n  16384
  log  127.0.0.1 local0 err
  stats timeout 30s

defaults
  log global
  mode  http
  option  httplog
  timeout connect 5000
  timeout client  50000
  timeout server  50000
  timeout http-request 15s
  timeout http-keep-alive 15s

frontend k8s-master
  bind 0.0.0.0:8443
  bind 127.0.0.1:8443
  mode tcp
  option tcplog
  tcp-request inspect-delay 5s
  default_backend k8s-master

backend k8s-master
  mode tcp
  option tcplog
  option tcp-check
  balance roundrobin
  default-server inter 10s downinter 5s rise 2 fall 2 slowstart 60s maxconn 250 maxqueue 256 weight 100
  server k8s-master01    192.168.0.71:6443  check
  server k8s-master02    192.168.0.72:6443  check
  server k8s-master03    192.168.0.73:6443  check

master01keepalive

! Configuration File for keepalived
global_defs {
    router_id LVS_DEVEL
}
vrrp_script chk_apiserver {
    script "/etc/keepalived/check_apiserver.sh"
    interval 5 
    weight -5
    fall 2
    rise 1
}
vrrp_instance VI_1 {
    state MASTER
    interface eth0
    mcast_src_ip 192.168.0.71
    virtual_router_id 51
    priority 101
    nopreempt
    advert_int 2
    authentication {
        auth_type PASS
        auth_pass K8SHA_KA_AUTH
    }
    virtual_ipaddress {
        192.168.0.236
    }
    track_script {
      chk_apiserver 
} }
​
# 7.2 Master02 keepalived
! Configuration File for keepalived
global_defs {
    router_id LVS_DEVEL
}
vrrp_script chk_apiserver {
    script "/etc/keepalived/check_apiserver.sh"
    interval 5 
    weight -5
    fall 2
    rise 1

}
vrrp_instance VI_1 {
    state BACKUP
    interface eth0
    mcast_src_ip 192.168.0.72
    virtual_router_id 51
    priority 100
    nopreempt
    advert_int 2
    authentication {
        auth_type PASS
        auth_pass K8SHA_KA_AUTH
    }
    virtual_ipaddress {
        192.168.0.236
    }
    track_script {
      chk_apiserver 
} }
​
# 7.3 Master03 keepalived
! Configuration File for keepalived
global_defs {
    router_id LVS_DEVEL
}
vrrp_script chk_apiserver {
    script "/etc/keepalived/check_apiserver.sh"
    interval 5
    weight -5
    fall 2  
    rise 1
}
vrrp_instance VI_1 {
    state BACKUP
    interface eth0
    mcast_src_ip 192.168.0.73
    virtual_router_id 51
    priority 100
    nopreempt
    advert_int 2
    authentication {
        auth_type PASS
        auth_pass K8SHA_KA_AUTH
    }
    virtual_ipaddress {
        192.168.0.236
    }
    track_script {
      chk_apiserver 
} }

防脑裂脚本

vim /etc/keepalived/check_apiserver.sh
i#!/bin/bash

err=0
for k in $(seq 1 3)
do
    check_code=$(pgrep nginx)
    if [[ $check_code == "" ]]; then
        err=$(expr $err + 1)
        sleep 1
        continue
    else
        err=0
        break
    fi
done

if [[ $err != "0" ]]; then
    echo "systemctl stop keepalived"
    /usr/bin/systemctl stop keepalived
    exit 1
else
    exit 0
fi

chmod +x /etc/keepalived/check_apiserver.sh
systemctl enable --now haproxy.service keepalived.service

配置各组件并启动

kube-apiserver

配置参数地址：kube-apiserver | Kubernetes

vim /usr/lib/systemd/system/kube-apiserver.service
i[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/kubernetes/kubernetes
After=network.target​
[Service]
ExecStart=/usr/local/bin/kube-apiserver \
      --v=2  \
      --logtostderr=true  \
      --allow-privileged=true  \
      --bind-address=0.0.0.0  \
      --secure-port=6443  \
      --insecure-port=0  \
      --advertise-address=192.168.0.71 \
      --service-cluster-ip-range=10.96.0.0/12  \
      --service-node-port-range=30000-32767  \
      --etcd-servers=https://192.168.0.71:2379,https://192.168.0.72:2379,https://192.168.0.73:2379 \
      --etcd-cafile=/etc/etcd/ssl/etcd-ca.pem  \
      --etcd-certfile=/etc/etcd/ssl/etcd.pem  \
      --etcd-keyfile=/etc/etcd/ssl/etcd-key.pem  \
      --client-ca-file=/etc/kubernetes/pki/ca.pem  \
      --tls-cert-file=/etc/kubernetes/pki/apiserver.pem  \
      --tls-private-key-file=/etc/kubernetes/pki/apiserver-key.pem  \
      --kubelet-client-certificate=/etc/kubernetes/pki/apiserver.pem  \
      --kubelet-client-key=/etc/kubernetes/pki/apiserver-key.pem  \
      --service-account-key-file=/etc/kubernetes/pki/sa.pub  \
      --service-account-signing-key-file=/etc/kubernetes/pki/sa.key  \
      --service-account-issuer=https://kubernetes.default.svc.cluster.local \
      --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname  \
      --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,ResourceQuota  \
      --authorization-mode=Node,RBAC  \
      --enable-bootstrap-token-auth=true  \
      --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.pem  \
      --proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.pem  \
      --proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client-key.pem  \
      --requestheader-allowed-names=aggregator  \
      --requestheader-group-headers=X-Remote-Group  \
      --requestheader-extra-headers-prefix=X-Remote-Extra-  \
      --requestheader-username-headers=X-Remote-User
      # --token-auth-file=/etc/kubernetes/token.csv
​
Restart=on-failure
RestartSec=10s
LimitNOFILE=65535
​
[Install]
WantedBy=multi-user.target

controller-manager

配置详解：kube-controller-manager | Kubernetes

vim /usr/lib/systemd/system/controller-manager.service
i[Unit]
Description=Kubernetes Controller Manager
Documentation=https://github.com/kubernetes/kubernetes
After=network.target

[Service]
ExecStart=/usr/local/bin/kube-controller-manager \
      --v=2 \
      --logtostderr=true \
      --address=127.0.0.1 \
      --root-ca-file=/etc/kubernetes/pki/ca.pem \
      --cluster-signing-cert-file=/etc/kubernetes/pki/ca.pem \
      --cluster-signing-key-file=/etc/kubernetes/pki/ca-key.pem \
      --service-account-private-key-file=/etc/kubernetes/pki/sa.key \
      --kubeconfig=/etc/kubernetes/controller-manager.kubeconfig \
      --leader-elect=true \
      --use-service-account-credentials=true \
      --node-monitor-grace-period=40s \
      --node-monitor-period=5s \
      --pod-eviction-timeout=2m0s \
      --controllers=*,bootstrapsigner,tokencleaner \
      --allocate-node-cidrs=true \
      --cluster-cidr=172.16.0.0/12 \
      --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.pem \
      --node-cidr-mask-size=24   
Restart=always
RestartSec=10s

[Install]
WantedBy=multi-user.target

scheduler

vim /usr/lib/systemd/system/kube-scheduler.service
i[Unit]
Description=Kubernetes Scheduler
Documentation=https://github.com/kubernetes/kubernetes
After=network.target

[Service]
ExecStart=/usr/local/bin/kube-scheduler \
      --v=2 \
      --logtostderr=true \
      --address=127.0.0.1 \
      --leader-elect=true \
      --kubeconfig=/etc/kubernetes/scheduler.kubeconfig

Restart=always
RestartSec=10s

[Install]
WantedBy=multi-user.target

systemctl daemon-reload
systemctl enable --now kube-apiserver.service controller-manager.service kube-scheduler.service

通过Bootstrapping签发node节点证书

cd /root/k8s-ha-install/bootstrap
kubectl config set-cluster kubernetes     --certificate-authority=/etc/kubernetes/pki/ca.pem     --embed-certs=true     --server=https://192.168.0.236:8443     --kubeconfig=/etc/kubernetes/bootstrap-kubelet.kubeconfig
kubectl config set-credentials tls-bootstrap-token-user     --token=c8ad9c.2e4d610cf3e7426e --kubeconfig=/etc/kubernetes/bootstrap-kubelet.kubeconfig
kubectl config set-context tls-bootstrap-token-user@kubernetes     --cluster=kubernetes     --user=tls-bootstrap-token-user     --kubeconfig=/etc/kubernetes/bootstrap-kubelet.kubeconfig
kubectl config use-context tls-bootstrap-token-user@kubernetes     --kubeconfig=/etc/kubernetes/bootstrap-kubelet.kubeconfig

mkdir -p /root/.kube ; cp /etc/kubernetes/admin.kubeconfig /root/.kube/config
kubectl create -f bootstrap.secret.yaml

向node分发证书

cd /etc/kubernetes/

for NODE in k8s-master02 k8s-master03 k8s-node01 k8s-node02; do
     ssh $NODE mkdir -p /etc/kubernetes/pki /etc/etcd/ssl /etc/etcd/ssl
     for FILE in etcd-ca.pem etcd.pem etcd-key.pem; do
       scp /etc/etcd/ssl/$FILE $NODE:/etc/etcd/ssl/
     done
     for FILE in pki/ca.pem pki/ca-key.pem pki/front-proxy-ca.pem bootstrap-kubelet.kubeconfig; do
       scp /etc/kubernetes/$FILE $NODE:/etc/kubernetes/${FILE}
 done
 done

部署kubelet

　所有节点创建相关目录

mkdir -p /var/lib/kubelet /var/log/kubernetes /etc/systemd/system/kubelet.service.d /etc/kubernetes/manifests/

所有节点配置kubelet service

 

[root@k8s-master01 bootstrap]# vim  /usr/lib/systemd/system/kubelet.service

 

i[Unit]
 

Description=Kubernetes Kubelet
 

Documentation=https://github.com/kubernetes/kubernetes
 

After=docker.service
 

Requires=docker.service

 

[Service]
 

ExecStart=/usr/local/bin/kubelet

 

Restart=always
 

StartLimitInterval=0
 

RestartSec=10

 

[Install]
 

WantedBy=multi-user.target
 

所有节点配置kubelet service的配置文件
 

vim /etc/systemd/system/kubelet.service.d/10-kubelet.conf

 

i[Service]
 

Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.kubeconfig --kubeconfig=/etc/kubernetes/kubelet.kubeconfig"
 

Environment="KUBELET_SYSTEM_ARGS=--network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin"
 

Environment="KUBELET_CONFIG_ARGS=--config=/etc/kubernetes/kubelet-conf.yml --pod-infra-container-image=registry.cn-hangzhou.aliyuncs.com/google_containers/pause-amd64:3.2"
 

Environment="KUBELET_EXTRA_ARGS=--node-labels=node.kubernetes.io/node=''"
 

ExecStart=
 

ExecStart=/usr/local/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_SYSTEM_ARGS $KUBELET_EXTRA_ARGS
 

准备kubelet配置文件
 

[root@k8s-master01 bootstrap]# vim /etc/kubernetes/kubelet-conf.yml

 

iapiVersion: kubelet.config.k8s.io/v1beta1
 

kind: KubeletConfiguration
 

address: 0.0.0.0
 

port: 10250
 

readOnlyPort: 10255
 

authentication:
 

  anonymous:
 

    enabled: false
 

  webhook:
 

    cacheTTL: 2m0s
 

    enabled: true
 

  x509:
 

    clientCAFile: /etc/kubernetes/pki/ca.pem
 

authorization:
 

  mode: Webhook
 

  webhook:
 

    cacheAuthorizedTTL: 5m0s
 

    cacheUnauthorizedTTL: 30s
 

cgroupDriver: systemd
 

cgroupsPerQOS: true
 

clusterDNS:
 

- 10.96.0.10
 

clusterDomain: cluster.local
 

containerLogMaxFiles: 5
 

containerLogMaxSize: 10Mi
 

contentType: application/vnd.kubernetes.protobuf
 

cpuCFSQuota: true
 

cpuManagerPolicy: none
 

cpuManagerReconcilePeriod: 10s
 

enableControllerAttachDetach: true
 

enableDebuggingHandlers: true
 

enforceNodeAllocatable:
 

- pods
 

eventBurst: 10
 

eventRecordQPS: 5
 

evictionHard:
 

  imagefs.available: 15%
 

  memory.available: 100Mi
 

  nodefs.available: 10%
 

  nodefs.inodesFree: 5%
 

evictionPressureTransitionPeriod: 5m0s
 

failSwapOn: true
 

fileCheckFrequency: 20s
 

hairpinMode: promiscuous-bridge
 

healthzBindAddress: 127.0.0.1
 

healthzPort: 10248
 

httpCheckFrequency: 20s
 

imageGCHighThresholdPercent: 85
 

imageGCLowThresholdPercent: 80
 

imageMinimumGCAge: 2m0s
 

iptablesDropBit: 15
 

iptablesMasqueradeBit: 14
 

kubeAPIBurst: 10
 

kubeAPIQPS: 5
 

makeIPTablesUtilChains: true
 

maxOpenFiles: 1000000
 

maxPods: 110
 

nodeStatusUpdateFrequency: 10s
 

oomScoreAdj: -999
 

podPidsLimit: -1
 

registryBurst: 10
 

registryPullQPS: 5
 

resolvConf: /etc/resolv.conf
 

rotateCertificates: true
 

runtimeRequestTimeout: 2m0s
 

serializeImagePulls: true
 

staticPodPath: /etc/kubernetes/manifests
 

streamingConnectionIdleTimeout: 4h0m0s
 

syncFrequency: 1m0s
 

volumeStatsAggPeriod: 1m0s
 

启动kubelet
 

systemctl daemon-reload
 

systemctl enable --now kubelet
 

 

部署kube-proxy
 

以下操作在Master01执行
 

cd /root/k8s-ha-install
 

kubectl -n kube-system create serviceaccount kube-proxy
 

kubectl create clusterrolebinding system:kube-proxy         --clusterrole system:node-proxier         --serviceaccount kube-system:kube-proxy
 

SECRET=$(kubectl -n kube-system get sa/kube-proxy \
 

    --output=jsonpath='{.secrets[0].name}')
 

JWT_TOKEN=$(kubectl -n kube-system get secret/$SECRET \
 

--output=jsonpath='{.data.token}' | base64 -d)
 

PKI_DIR=/etc/kubernetes/pki
 

K8S_DIR=/etc/kubernetes
 

kubectl config set-cluster kubernetes     --certificate-authority=/etc/kubernetes/pki/ca.pem     --embed-certs=true     --server=https://192.168.0.236:8443     --kubeconfig=${K8S_DIR}/kube-proxy.kubeconfig
 

kubectl config set-credentials kubernetes     --token=${JWT_TOKEN}     --kubeconfig=/etc/kubernetes/kube-proxy.kubeconfig
 

kubectl config set-context kubernetes     --cluster=kubernetes     --user=kubernetes     --kubeconfig=/etc/kubernetes/kube-proxy.kubeconfig
 

kubectl config use-context kubernetes     --kubeconfig=/etc/kubernetes/kube-proxy.kubeconfig
 

准备proxy service文件
 

vim /usr/lib/systemd/system/kube-proxy.service
 

i[Unit]
 

Description=Kubernetes Kube Proxy
 

Documentation=https://github.com/kubernetes/kubernetes
 

After=network.target

 

[Service]
 

ExecStart=/usr/local/bin/kube-proxy \
 

  --config=/etc/kubernetes/kube-proxy.conf \
 

  --v=2

 

Restart=always
 

RestartSec=10s

 

[Install]
 

WantedBy=multi-user.target
 

准备kube-proxy配置文件
 

apiVersion: kubeproxy.config.k8s.io/v1alpha1
 

bindAddress: 0.0.0.0
 

clientConnection:
 

  acceptContentTypes: ""
 

  burst: 10
 

  contentType: application/vnd.kubernetes.protobuf
 

  kubeconfig: /etc/kubernetes/kube-proxy.kubeconfig
 

  qps: 5
 

clusterCIDR: 172.16.0.0/12
 

configSyncPeriod: 15m0s
 

conntrack:
 

  max: null
 

  maxPerCore: 32768
 

  min: 131072
 

  tcpCloseWaitTimeout: 1h0m0s
 

  tcpEstablishedTimeout: 24h0m0s
 

enableProfiling: false
 

healthzBindAddress: 0.0.0.0:10256
 

hostnameOverride: ""
 

iptables:
 

  masqueradeAll: false
 

  masqueradeBit: 14
 

  minSyncPeriod: 0s
 

  syncPeriod: 30s 
 

[root@k8s-master01 kube-proxy]# cat kube-proxy.conf 
 

apiVersion: kubeproxy.config.k8s.io/v1alpha1
 

bindAddress: 0.0.0.0
 

clientConnection:
 

  acceptContentTypes: ""
 

  burst: 10
 

  contentType: application/vnd.kubernetes.protobuf
 

  kubeconfig: /etc/kubernetes/kube-proxy.kubeconfig
 

  qps: 5
 

clusterCIDR: 172.16.0.0/12
 

configSyncPeriod: 15m0s
 

conntrack:
 

  max: null
 

  maxPerCore: 32768
 

  min: 131072
 

  tcpCloseWaitTimeout: 1h0m0s
 

  tcpEstablishedTimeout: 24h0m0s
 

enableProfiling: false
 

healthzBindAddress: 0.0.0.0:10256
 

hostnameOverride: ""
 

iptables:
 

  masqueradeAll: false
 

  masqueradeBit: 14
 

  minSyncPeriod: 0s
 

  syncPeriod: 30s
 

ipvs:
 

  masqueradeAll: true
 

  minSyncPeriod: 5s
 

  scheduler: "rr"
 

  syncPeriod: 30s
 

kind: KubeProxyConfiguration
 

metricsBindAddress: 127.0.0.1:10249
 

mode: "ipvs"
 

nodePortAddresses: null
 

oomScoreAdj: -999
 

portRange: ""
 

udpIdleTimeout: 250ms

 

for NODE in k8s-master01 k8s-master02 k8s-master03 k8s-node01 k8s-node02; do
 

     scp ${K8S_DIR}/kube-proxy.kubeconfig $NODE:/etc/kubernetes/kube-proxy.kubeconfig
 

     scp kube-proxy/kube-proxy.conf $NODE:/etc/kubernetes/kube-proxy.conf
 

     scp kube-proxy/kube-proxy.service $NODE:/usr/lib/systemd/system/kube-proxy.service
 

 done
 

for NODE in; do
 

     scp /etc/kubernetes/kube-proxy.kubeconfig $NODE:/etc/kubernetes/kube-proxy.kubeconfig
 

     scp kube-proxy/kube-proxy.conf $NODE:/etc/kubernetes/kube-proxy.conf
 

     scp kube-proxy/kube-proxy.service $NODE:/usr/lib/systemd/system/kube-proxy.service
 

 done

 

systemctl daemon-reload
 

systemctl enable --now kube-proxy
 

 

安装cni网络插件
 

 

calico官方文档地址：Configuring calico/node
 

sed -i 's#etcd_endpoints: "http://<ETCD_IP>:<ETCD_PORT>"#etcd_endpoints: "https://192.168.0.71:2379,https://192.168.0.72:2379,https://192.168.0.73:2379"#g' calico-etcd.yaml

 

ETCD_CA=`cat /etc/kubernetes/pki/etcd/etcd-ca.pem | base64 | tr -d '\n'`
 

ETCD_CERT=`cat /etc/kubernetes/pki/etcd/etcd.pem | base64 | tr -d '\n'`
 

ETCD_KEY=`cat /etc/kubernetes/pki/etcd/etcd-key.pem | base64 | tr -d '\n'`

 

sed -i "s@# etcd-key: null@etcd-key: ${ETCD_KEY}@g; s@# etcd-cert: null@etcd-cert: ${ETCD_CERT}@g; s@# etcd-ca: null@etcd-ca: ${ETCD_CA}@g" calico-etcd.yaml

 

sed -i 's#etcd_ca: ""#etcd_ca: "/calico-secrets/etcd-ca"#g; s#etcd_cert: ""#etcd_cert: "/calico-secrets/etcd-cert"#g; s#etcd_key: "" #etcd_key: "/calico-secrets/etcd-key" #g' calico-etcd.yaml

 

# 更改此处为自己的pod网段
 

POD_SUBNET="172.16.0.0/12"

 

sed -i 's@# - name: CALICO_IPV4POOL_CIDR@- name: CALICO_IPV4POOL_CIDR@g; s@#   value: "192.168.0.0/16"@  value: '"${POD_SUBNET}"'@g' calico-etcd.yaml
 

遇到的坑：
 

如果添加了环境变量在calico-controller和calico-node中添加env环境指向apiserver
 

- name: KUBERNETES_SERVICE_HOST
 

  value: "192.168.0.236"
 

- name: KUBERNETES_SERVICE_PORT
 

  value: "8443"
 

- name: KUBERNETES_SERVICE_PORT_HTTPS
 

  value: "8443"
 

安装calico
 

kubectl apply -f calico-etcd.yaml

 

docker pull registry.cn-beijing.aliyuncs.com/dotbalo/cni:v3.15.3
 

docker pull registry.cn-beijing.aliyuncs.com/dotbalo/pod2daemon-flexvol:v3.15.3
 

docker pull registry.cn-beijing.aliyuncs.com/dotbalo/node:v3.15.3
 

docker pull registry.cn-beijing.aliyuncs.com/dotbalo/kube-controllers:v3.15.3

 

docker tag registry.cn-beijing.aliyuncs.com/dotbalo/cni:v3.15.3 harbor.michaelxia.cn/calico/cni:v3.15.3
 

docker tag registry.cn-beijing.aliyuncs.com/dotbalo/pod2daemon-flexvol:v3.15.3 harbor.michaelxia.cn/calico/pod2daemon-flexvol:v3.15.3
 

docker tag registry.cn-beijing.aliyuncs.com/dotbalo/node:v3.15.3 harbor.michaelxia.cn/calico/node:v3.15.3
 

docker tag registry.cn-beijing.aliyuncs.com/dotbalo/kube-controllers:v3.15.3  harbor.michaelxia.cn/calico/kube-controllers:v3.15.3

 

docker push harbor.michaelxia.cn/calico/cni:v3.15.3
 

docker push harbor.michaelxia.cn/calico/pod2daemon-flexvol:v3.15.3
 

docker push harbor.michaelxia.cn/calico/node:v3.15.3
 

docker push harbor.michaelxia.cn/calico/kube-controllers:v3.15.3
 

安装coredns
 

cd /root/k8s-ha-install/
 

# 设定coredns的解析地址为clusterip中的第十个ip
 

sed -i "s#10.96.0.10#10.96.0.10#g" CoreDNS/coredns.yaml
 

kubectl  create -f CoreDNS/coredns.yaml 
 

安装metric
 

cd /root/k8s-ha-install/metrics-server-0.4.x/

 

kubectl  create -f . 

 

集群验证
 

  1. Pod必须能解析Service
  2. Pod必须能解析跨namespace的Service
  3. 每个节点都必须要能访问Kubernetes的kubernetes svc 443和kube-dns的service 53
  4. Pod和Pod之前要能通
  5. 同namespace能通信
  6. 跨namespace能通信
 

 

创建dashboard
 

cd /root/k8s-ha-install/dashboard/
 

kubectl create -f .
 

企业级k8s配置
 

docker配置
 

vim /etc/docker/daemon.json
 

i{
 

 "registry-mirrors": [
 

    "https://registry.docker-cn.com",
 

    "http://hub-mirror.c.163.com",
 

    "https://docker.mirrors.ustc.edu.cn"
 

  ],
 

 "exec-opts": ["native.cgroupdriver=systemd"],
 

 "max-concurrent-downloads": 10,
 

 "max-concurrent-uploads": 5,
 

 "log-opts": {
 

   "max-size": "300m",
 

   "max-file": "2"
 

 },
 

 "live-restore": true
 

}
 

controllermanager配置
 

--cluster-signing-duration=876000h0m0s  #配置当kubelet证书过期后，bootsrap签发证书的有效期为100年
 

kubelet配置
 

vim /etc/systemd/system/kubelet.service.d/10-kubelet.conf 
 

# 修改加密方式保证kubelet安全性检测
 

--tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384  

 

vim /etc/kubernetes/kubelet-conf.yml
 

添加如下配置
 

rotateServerCertificates: true
 

allowedUnsafeSysctls:
 

 - "net.core*"
 

 - "net.ipv4.*"
 

kubeReserved:
 

  cpu: "1"
 

  memory: 1Gi
 

  ephemeral-storage: 10Gi
 

systemReserved:
 

  cpu: "1"
 

  memory: 1Gi
 

  ephemeral-storage: 10Gi
 

kubectl连接多个k8s集群
 

假如已经准备好2个集群的配置文件，分别为 $HOME/.kube/config1 和 ``$HOME/.kube/config2
 

config1 信息如下
 

[root@node-01 .kube]# cat $HOME/.kube/config1
 

apiVersion: v1
 

kind: Config
 

clusters:
 

- cluster:
 

    api-version: v1
 

    certificate-authority-data: xxxxxxx
 

    server: "https://172.20.8.113:6443"
 

  name: "cn-k8s"
 

contexts:
 

- context:
 

    cluster: "cn-k8s"
 

    user: "kube-admin-local"
 

  name: "cn-k8s"
 

current-context: "cn-k8s"
 

users:
 

- name: "kube-admin-local"
 

  user:
 

    client-certificate-data: xxxxxx
 

    client-key-data: xxxxxx
 

config2 信息如下
 

[root@node-01 .kube]# cat $HOME/.kube/config2
 

apiVersion: v1
 

kind: Config
 

clusters:
 

- cluster:
 

    api-version: v1
 

    certificate-authority-data: xxxxxx
 

    server: "https://172.19.8.113:6443"
 

  name: "jp-k8s"
 

contexts:
 

- context:
 

    cluster: "jp-k8s"
 

    user: "kube-admin-local"
 

  name: "jp-k8s"
 

current-context: "jp-k8s"
 

users:
 

- name: "kube-admin-local"
 

  user:
 

    client-certificate-data: xxxxxx
 

    client-key-data: xxxxxx
 

通过config信息，可以看到两个集群的cluster name，context name，以及用户信息。
 

配置文件已准备好，下面开始变身了。文件合成：
 

cd $HOME/.kube/config
 

KUBECONFIG=config1:config2 kubectl config view --flatten > $HOME/.kube/config

 

那么如何使用呢？
 

1、查看cluster name以及context name
 

[root@node-01 ~]# kubectl config view
 

apiVersion: v1
 

clusters:
 

- cluster:
 

    certificate-authority-data: DATA+OMITTED
 

    server: https://172.19.8.113:6443
 

  name: cnlocal
 

- cluster:
 

    certificate-authority-data: DATA+OMITTED
 

    server: https://172.19.8.113:6443
 

  name: jplocal
 

contexts:
 

- context:
 

    cluster: cnlocal
 

    user: kube-admin-local
 

  name: cnlocal
 

- context:
 

    cluster: jplocal
 

    user: kube-admin-local
 

  name: local
 

current-context: jplocal
 

kind: Config
 

preferences: {}
 

users:
 

- name: kube-admin-local
 

  user:
 

    client-certificate-data: REDACTED
 

    client-key-data: REDACTED
 

2、查看当前使用的集群
 

[root@node-01 ~]# kubectl config current-context
 

jplocal
 

3、修改当前使用的集群
 

[root@node-01 ~]# kubectl config use-context cnlocal
 

Switched to context "cnlocal".

 

安装过程中需要注意的问题：
 

  1.  
    kubeadm
     
  2.  
    二进制
     
  3.  
    自动化安装
     
  
    1. Ansible
    2. Master节点安装不需要写自动化。
    3. 添加Node节点，playbook。
  4.  
    安装需要注意的细节
     
  
    1. 上面的细节配置
    2. 生产环境中etcd一定要和系统盘分开，一定要用ssd硬盘。
    3. Docker数据盘也要和系统盘分开，有条件的话可以使用ssd硬盘

                

        

        

    




    

      

        

            

              
                
                  熬夜波比a
                
              
            

            

                关注
              
            

        

        

          
    
            
  • 
              
                
                
                
                
                      0
                
              
              
    点赞
    
            
    • 
            
  • 
              
                
                
                
              
              
    踩
    
            
    • 
            
  • 
              
                
                
                
                
                    4
                
              
              
    
                
    
                  
                    收藏
                  
                
    
              
    
              
    
                
    
                  觉得还不错?
                  
                    一键收藏
                  
                 
                
    
              
    
            
    • 
          
  • 
            
    
              
              
            
    
              
              
              
                    0
              
            
            
    评论
    
          
    • 
          
  • 
          
    • 
          
  • 
            
              
            
              
    
            
              
                
            
    
          
    • 
        

      

      

            

                专栏目录
          










                



	

		

			

				
					
k8s 二进制安装 详细安装步骤

				
			

			

				

					wyq2070857323的博客
				

				

					05-15
					
					930
					
				

			

		

		

			
				
etcd是CoreOS团队于2013年6月发起的开源项目，它的目标是构建一个高可用的分布式键值（key-value）数据库。etcd内部采用raft协议作为一致性算法，etcd是go语言编写的。关闭防火墙 关闭selinux 关闭swap 根据规划设置主机名做域名映射 调整内核参数 时间同步。

			
		

	



                

              
                



	

		

			

				
					
k8s二进制安装部署(详细)

					
热门推荐

				
			

			

				

					qq_44078641的博客
				

				

					09-15
					
					2万+
					
				

			

		

		

			
				
一、生产环境部署k8s常见的几种方式
1.1 kubeadm
Kubeadm是一个K8s部署工具，提供kubeadm init和kubeadm join，用于快速部署Kubernetes集群。

1.2 二进制
从github下载发行版的二进制包，手动部署每个组件，组成Kubernetes集群。
小结：  
Kubeadm降低部署门槛，但屏蔽了很多细节，遇到问题很难排查。如果想更容易可控，推荐使用二进制包部署Kubernetes集群，虽然手动部署
麻烦点，期间可以学习很多工作原理，也利于后期维护。

1.3

			
		

	



                


  
              

                


	

		

			

				
					
k8s二进制安装（迅速部署）

				
			

			

				

					m0_63190689的博客
				

				

					08-12
					
					1538
					
				

			

		

		

			
				
k8s 二进制部署

			
		

	





	

		

			

				
					
k8s二进制安装篇5-flannel网络组件安装

				
			

			

				

					qq_45271350的博客
				

				

					09-07
					
					1224
					
				

			

		

		

			
				
一，验证已安装的集群
~]# kubectl get nodes
NAME            STATUS   ROLES    AGE   VERSION
192.168.1.246   Ready    <none>   27m   v1.19.14
192.168.1.247   Ready    <none>   25m   v1.19.14
192.168.1.248   Ready    <none>   17m   v1.19.14

~]# kubec

			
		

	



		

			
		



	

		

			

				
					
k8s二进制安装方式

				
			

			

				

					FOREVER_DO
				

				

					06-30
					
					3431
					
				

			

		

		

			
				
k8s二进制安装方式环境准备
环境准备

centos7，三台机子，内存2G4核
master需要安装
etcd
kube-apiserver
kube-controller-manager
kube-scheduler
node需要安装
Docker
kubelet
kube-proxy
下载官方的软件包https://github.com/kubernetes/kubernetes/
这里我下...

			
		

	





	

		

			

				
					
k8s二进制安装部署超超详细指南

				
			

			

				

					qq_41009846的博客
				

				

					07-09
					
					4114
					
				

			

		

		

			
				
k8s安装部署详细指南概念什么是k8sK8s的功能k8s集群架构K8s部署步骤集群结构实验环境规划部署单master集群集群规划：初始化服务器关闭防火墙（所有节点都执行）关闭selinux（所有节点都执行）配置主机名称（所有节点都执行）配置名称解析 host文件（所有节点都执行）配置时间同步配置k8s-master1配置客户端 node1 node2关闭交换分区（会导致服务起不来）部署集群给etcd颁发证书部署etcd
概念
什么是k8s
K8s是一组服务器集群 
K8s所管理的集群节点上的容器

K8s的

			
		

	





	

		

			

				
					
k8s二进制安装部署(详细)（3主2从）

				
			

			

				

					faoids的博客
				

				

					05-04
					
					1591
					
				

			

		

		

			
				
k8s二进制安装部署(详细)

			
		

	





	

		

			

				
					
k8s二进制安装篇7-coredns组件安装

				
			

			

				

					qq_45271350的博客
				

				

					11-29
					
					1489
					
				

			

		

		

			
				
在kubernetes中部署coredns
1、下载coredns需要的资源清单
https://github.com/kubernetes/kubernetes/tree/master/cluster/addons/dns/coredns

2，创建目录
mkdir  -p  /opt/k8s-yaml/coredns
cd /opt/k8s-yaml/coredns
3，将配置文件拷贝到服务器
cat  coredns.yaml
apiVersion: v1
kind: ServiceAccount
m

			
		

	





	

		

			

				
					
二进制安装k8s

				
			

			

				

					zyxfx的博客
				

				

					11-27
					
					1183
					
				

			

		

		

			
				
Kubernetes是一个开源的，用于管理云平台中多个主机上的容器化的应用，Kubernetes的目标是让部署容器化的应用简单并且高效（powerful）,Kubernetes提供了应用部署，规划，更新，维护的一种机制。

			
		

	





	

		

			

				
					
k8s二进制安装.pdf

				
			

			

				

					05-17
				

			

		

		

			
				
较详细的Kubernetes-v1.12二进制部署集群文档，含基础插件

			
		

	





	

		

			

				
					
k8s二进制安装资源包

				
			

			

				

					02-18
				

			

		

		

			
				
本文将详细解析"K8s二进制安装资源包"中的核心概念，以及如何进行二进制方式安装Kubernetes。  在二进制安装Kubernetes的过程中，我们需要了解以下几个关键组件：  1. **kubelet**：它是Kubernetes节点的主要代理，...

			
		

	





	

		

			

				
					
k8s  二进制安装 etcd 3.5.1

				
			

			

				

					03-21
				

			

		

		

			
				
### 二、二进制安装etcd 3.5.1  1. **下载二进制文件**：首先，从官方仓库下载etcd 3.5.1的Linux AMD64版本。你可以访问[etcd releases页面](https://github.com/etcd-io/etcd/releases)下载，或者通过wget命令：   ...

			
		

	





	

		

			

				
					
K8S二进制安装文档--kubectl安装及kubeconfig文件创建.txt

				
			

			

				

					07-27
				

			

		

		

			
				
K8S二进制安装文档--kubectl安装及kubeconfig文件创建.txt

			
		

	





	

		

			

				
					
Linux：基础命令学习

				
			

			

				

					qq_55038440的博客
				

				

					07-20
					
					1503
					
				

			

		

		

			
				
实例：-F根据文件类型在列出的文件名称后加一符号。实例： -R 递归显示目录中的所有文件和子目录。. 开头的隐藏文件也会列出。可执行文件则加 "*"，用于显示目录文件信息。

			
		

	





	

		

			

				
					
LINUX之MMC子系统分析

				
			

			

				

					gy794627991的博客
				

				

					07-23
					
					1093
					
				

			

		

		

			
				
MMC子系统从协议到源码分析

			
		

	





	

		

			

				
					
RockyLinux 9 PXE Server bios+uefi 自动化部署 RockLinux 8 9

					
最新发布

				
			

			

				

					极致，细节
				

				

					07-24
					
					99
					
				

			

		

		

			
				
PXE（Preboot eXecution Environment，预启动执行环境）是一种网络启动协议，允许计算机通过网络启动而不是使用本地硬盘。PXE服务器是实现这一功能的服务器，它提供了启动镜像和引导加载程序，使得客户端计算机可以通过网络启动并安装操作系统或运行其他软件。

			
		

	





	

		

			

				
					
k8s二进制安装啥意思

				
			

			

				

					09-02
				

			

		

		

			
				
Kubernetes (k8s) 是一个开源的容器编排平台，用于自动化部署、扩展和管理容器化应用程序。k8s的二进制安装指的是通过下载和安装预编译的可执行文件来安装和配置Kubernetes。这意味着你需要手动下载所需的二进制文件，并将其放置在正确的位置，然后配置环境变量和其他必要的设置，以便能够运行和管理Kubernetes集群。这种安装方法相对灵活，适用于具有特定需求或对自定义配置更感兴趣的人。

			
		

	



              




          




        



    





    



      

      

        
      

      





	

		评论	
	

	
	




  

    

      添加红包
      
    

    

      

        
        

          
          
        

        
请填写红包祝福语或标题

      

      

        
        

          
          个
        

        
红包个数最小为10个

      

      

        
        

          
          元
        

        
红包金额最低5元

      

      

        
        

          当前余额3.43元
          前往充值 >
        

      

      

        

          需支付：10.00元
        

        
        
      

    

  





  

    
    
  

  

    
    
  








  

    

      

        

          

            
            
成就一亿技术人!

          

          

        

        

          

          

            领取后你会自动成为博主和红包主的粉丝
            规则
          

        

      

      

        

          

            
              
            
            

              
hope_wisdom
 发出的红包
            

          

        

        

          

          

          

        

      

    

    

  



      
      

      
实付元

      
使用余额支付

      

        

          

            
            点击重新获取
          

        

        
扫码支付

      

      

        
          
            
          
          
        
      

      

        
        钱包余额
          0
          

            
            

              

                
抵扣说明：

                
 1.余额是钱包充值的虚拟货币，按照1:1的比例进行支付金额的抵扣。
 2.余额无法直接购买下载，可以购买VIP、付费专栏及课程。

              

            

          

      

      余额充值