利用Kubeadm离线安装kubernets1.15.0

最新推荐文章于 2024-07-11 16:29:44 发布

Jiang_James_L

最新推荐文章于 2024-07-11 16:29:44 发布

阅读量2.4k

点赞数 1

分类专栏： kubernetes

本文链接：https://blog.csdn.net/u014305062/article/details/100933754

版权

kubernetes 专栏收录该内容

0 篇文章 0 订阅

订阅专栏

1 主机列表

本次实验选择4台主机，3台作为Master节点，1台作为Node节点

节点ip	OS版本	kernel版本	hostname	docker版本	节点类型
192.168.137.129	CentOS7.4	3.10.0-693	centos	18.09.2	Master
192.168.137.130	CentOS7.4	3.10.0-693	centos1	18.09.2	Master
192.168.137.131	CentOS7.4	3.10.0-693	centos2	18.09.2	Master
192.168.137.132	CentOS7.4	3.10.0-693	centos3	18.09.2	Node

注：
master节点需要做互信

2 下载离线包

2.1 获取kubernetes所需容器镜像

下载kubernetes-server-linux-amd64.tar.gz(包含kubernetes所需的容器镜像))
kubernetes-server-linux-amd64.tar.gz
解压kubernetes-server-linux-amd64.tar.gz
获取容器镜像

docker load -i cloud-controller-manager.tar
docker load -i kube-controller-manager.tar
docker load -i kube-apiserver.tar
docker load -i kube-proxy.tar
docker load -i kube-scheduler.tar

上传到本地镜像仓库

docker tag k8s.gcr.io/kube-apiserver:v1.15.0 192.168.137.129/k8s.gcr.io/kube-apiserver:v1.15.0
docker push 192.168.137.129/k8s.gcr.io/kube-apiserver:v1.15.0
docker tag k8s.gcr.io/cloud-controller-manager:v1.15.0 192.168.137.129/k8s.gcr.io/cloud-controller-manager:v1.15.0
docker push 192.168.137.129/k8s.gcr.io/cloud-controller-manager:v1.15.0
docker tag k8s.gcr.io/kube-controller-manager:v1.15.0 192.168.137.129/k8s.gcr.io/kube-controller-manager:v1.15.0
docker push 192.168.137.129/k8s.gcr.io/kube-controller-manager:v1.15.0
docker tag k8s.gcr.io/kube-proxy:v1.15.0 192.168.137.129/k8s.gcr.io/kube-proxy:v1.15.0
docker push 192.168.137.129/k8s.gcr.io/kube-proxy:v1.15.0
docker tag k8s.gcr.io/kube-scheduler:v1.15.0 192.168.137.129/k8s.gcr.io/kube-scheduler:v1.15.0
docker push 192.168.137.129/k8s.gcr.io/kube-scheduler:v1.15.0

2.2 获取kubernetes rpm安装包

配置kubernetes国内yum源并下载rpm包

mkdir /var/www/html/kubernetes1.15.0
cat <<EOF > /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=1
repo_gpgcheck=1
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF
yum install kubeadm-1.15.0 kubelet-1.15.0 kubectl-1.15.0 --downloadonly --downloaddir=/var/www/html/kubernetes1.15.0

生成repodata

cd /var/www/html/kubernetes1.15.0
createrepo ./

配置本地kubernetes yum源
需要在所有主机上配置。

cat <<EOF > /etc/yum.repos.d/kubernetes1.15.0.repo
[kubernetes1.15.0]
name=Kubernetes1.15.0
baseurl=http://192.168.137.129:18080/kubernetes1.15.0/
enabled=1
gpgcheck=0
EOF

3 主机配置修改

需要在所有主机上配置。

关闭防火墙

systemctl stop firewalld.service
systemctl disable firewalld.service
sed -i '/^SELINUX=/c SELINUX=disabled' /etc/selinux/config
setenforce 0

关闭swap分区

swapoff -a
yes | cp /etc/fstab /etc/fstab_bak
cat /etc/fstab_bak |grep -v swap > /etc/fstab

配置sysctl.conf

cat << EOF > /etc/sysctl.conf
net.ipv4.ip_forward=1
net.ipv4.tcp_syncookies = 1
net.ipv4.tcp_timestamps = 0
net.ipv4.tcp_tw_reuse = 0
net.ipv4.tcp_tw_recycle = 0
net.ipv4.tcp_fin_timeout = 15
net.ipv4.tcp_max_tw_buckets = 262144
net.ipv4.conf.all.rp_filter = 1
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.core.somaxconn = 65535
net.core.netdev_max_backlog = 262144
net.ipv4.tcp_sack = 0
net.ipv4.tcp_window_scaling = 0
net.core.rmem_default=4194304
net.core.rmem_max=4194304
net.core.wmem_default=262144
net.core.wmem_max=1048586
fs.suid_dumpable = 1
fs.aio-max-nr = 1048576
fs.file-max = 6815744
kernel.shmmni = 4096
kernel.sem = 250 32000 100 128
vm.swappiness = 0
net.ipv4.ip_local_port_range = 9500 61000
EOF
sysctl -p

开启ipvs

cat << EOF > /etc/sysconfig/modules/ipvs.modules
#!/bin/bash
ipvs_modules="ip_vs ip_vs_lc ip_vs_wlc ip_vs_rr ip_vs_wrr ip_vs_lblc ip_vs_lblcr ip_vs_dh ip_vs_sh ip_vs_fo ip_vs_nq ip_vs_sed ip_vs_ftp nf_conntrack"
for kernel_module in \${ipvs_modules}; do
  modinfo -F filename \${kernel_module} > /dev/null 2>&1
  if [ $? -eq 0 ]; then
    modprobe -- \${kernel_module}
  fi
done
EOF
chmod 755 /etc/sysconfig/modules/ipvs.modules && bash /etc/sysconfig/modules/ipvs.modules && lsmod | grep ip_vs

修改docker cgroupdriver为systemd

mkdir -p /etc/docker && tee /etc/docker/daemon.json <<- EOF
{
  "exec-opts": ["native.cgroupdriver=systemd"],
  "log-driver": "json-file",
  "log-opts": {
    "max-size": "100m"
  },
  "insecure-registries": ["192.168.137.129"]
}
EOF
systemctl daemon-reload
systemctl restart docker

4 安装配置kubernetes

安装kubernetes

yum install -y kubeadm-1.15.0 kubelet-1.15.0 kubectl-1.15.0
systemctl enable --now kubelet

配置kubernetes

cat << EOF > /etc/default/kubelet
KUBELET_EXTRA_ARGS=--cgroup-driver=systemd   ### 与docker cgroupdriver 对应
EOF
systemctl daemon-reload
systemctl restart kubelet
echo "export KUBECONFIG=/etc/kubernetes/admin.conf" >> /etc/profile
source /etc/profile

5 配置controlPlaneEndpoint

编辑haproxy.cfg

cat << EOF > /data/haproxy.cfg
global
  daemon
  log 127.0.0.1 local0
  spread-checks 5
  max-spread-checks 15000
  maxconn 50000
  stats timeout 30s

defaults
  log                  global
  retries                   3
  backlog               10000
  maxconn               10000
  timeout connect          3s
  timeout client          30s
  timeout server          30s
  timeout tunnel        3600s
  timeout http-keep-alive  1s
  timeout http-request    15s
  timeout queue           30s
  timeout tarpit          60s
  option            dontlognull
  option            http-server-close
  option            redispatch
  option            forwardfor
  
listen stats
  bind 0.0.0.0:1080
  mode http
  stats enable
  option httplog
  log 127.0.0.1 local0 err
  stats refresh 30s
  maxconn 10
  stats uri /
  stats realm Haproxy\ Statistics
  stats auth admin:123456
  stats hide-version
  stats admin if TRUE
  monitor-uri /_haproxy_health_check


frontend kube-api
  bind *:8443
  mode tcp
  use_backend kube-master

backend kube-master
  balance source
  mode tcp
  server   192_168_137_129_6443    192.168.137.129:6443    check inter 2000 fall 3
  server   192_168_137_130_6443    192.168.137.130:6443    check inter 2000 fall 3
  server   192_168_137_131_6443    192.168.137.131:6443    check inter 2000 fall 3
EOF

编写haproxy容器启动脚本

cat << EOF > /data/start_haproxy.sh
#!/bin/bash
if netstat -anp | grep 1080; then
  echo "****************** port 1080 is already used ******************"
else
  /usr/bin/docker run -d --net=host -p 1080:1080 -p 8443:8443 -v /data/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg 192.168.137.129/public/haproxy:1.9.7
fi
EOF

编写haproxy容器自启动服务

cat << EOF > /usr/lib/systemd/system/docker-haproxy.service
[Unit]
Description=haproxy manager
After=docker.service

[Service]
WorkingDirectory=/data
Type=simple
ExecStart=/bin/bash /data/start_haproxy.sh

[Install]
WantedBy=multi-user.target
EOF
systemctl start docker-haproxy
systemctl enable docker-haproxy

注：
可以使用haproxy和keepalive实现HA

6 配置 kubeadm init 参数

获取模板：

kubeadm config print init-defaults --component-configs KubeProxyConfiguration

output：

apiVersion: kubeadm.k8s.io/v1beta2
bootstrapTokens:
- groups:
  - system:bootstrappers:kubeadm:default-node-token
  token: abcdef.0123456789abcdef
  ttl: 24h0m0s
  usages:
  - signing
  - authentication
kind: InitConfiguration
localAPIEndpoint:
  advertiseAddress: 1.2.3.4
  bindPort: 6443
nodeRegistration:
  criSocket: /var/run/dockershim.sock
  name: centos1
  taints:
  - effect: NoSchedule
    key: node-role.kubernetes.io/master
---
apiServer:
  timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta2
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controllerManager: {}
dns:
  type: CoreDNS
etcd:
  local:
    dataDir: /var/lib/etcd
imageRepository: k8s.gcr.io
kind: ClusterConfiguration
kubernetesVersion: v1.14.0
networking:
  dnsDomain: cluster.local
  serviceSubnet: 10.96.0.0/12
scheduler: {}
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
bindAddress: 0.0.0.0
clientConnection:
  acceptContentTypes: ""
  burst: 10
  contentType: application/vnd.kubernetes.protobuf
  kubeconfig: /var/lib/kube-proxy/kubeconfig.conf
  qps: 5
clusterCIDR: ""
configSyncPeriod: 15m0s
conntrack:
  maxPerCore: 32768
  min: 131072
  tcpCloseWaitTimeout: 1h0m0s
  tcpEstablishedTimeout: 24h0m0s
enableProfiling: false
healthzBindAddress: 0.0.0.0:10256
hostnameOverride: ""
iptables:
  masqueradeAll: false
  masqueradeBit: 14
  minSyncPeriod: 0s
  syncPeriod: 30s
ipvs:
  excludeCIDRs: null
  minSyncPeriod: 0s
  scheduler: ""
  strictARP: false
  syncPeriod: 30s
kind: KubeProxyConfiguration
metricsBindAddress: 127.0.0.1:10249
mode: ""
nodePortAddresses: null
oomScoreAdj: -999
portRange: ""
resourceContainer: /kube-proxy
udpIdleTimeout: 250ms
winkernel:
  enableDSR: false
  networkName: ""
  sourceVip: ""

编辑kubeadm init config.yaml

cat << EOF > /etc/kubernetes/kubeadm-config.yaml
apiVersion: kubeadm.k8s.io/v1beta2
kind: ClusterConfiguration
clusterName: k8s-test-local
kubernetesVersion: v1.15.0
controlPlaneEndpoint: "192.168.137.129:8443"
imageRepository: 192.168.137.129/k8s.gcr.io
dns:
  type: CoreDNS
etcd:
  local:
    dataDir: /var/lib/etcd
    serverCertSANs:
      - 192.168.137.129
      - 192.168.137.130
      - 192.168.137.131    
      - 127.0.0.1
      - centos
      - centos1
      - centos2
      - localhost
    peerCertSANs:
      - 192.168.137.129
      - 192.168.137.130
      - 192.168.137.131   
      - 127.0.0.1
      - centos
      - centos1
      - centos2
      - localhost
apiServer:
  certSANs:
      - 192.168.137.129
      - 192.168.137.130
      - 192.168.137.131
      - 192.168.137.132     
      - 127.0.0.1
      - centos
      - centos1
      - centos2
      - centos3
      - localhost
  extraArgs:
    service-node-port-range: 1025-65535
networking:
  dnsDomain: k8s.test.local
  serviceSubnet: 10.96.0.0/12
  podSubnet: 10.244.0.0/16
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
mode: ipvs
EOF

7 初始化集群

kubeadm init --config=/etc/kubernetes/kubeadm-config.yaml --upload-certs

output：

....

Your Kubernetes control-plane has initialized successfully!

To start using your cluster, you need to run the following as a regular user:

  mkdir -p $HOME/.kube
  sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
  sudo chown $(id -u):$(id -g) $HOME/.kube/config

You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
  https://kubernetes.io/docs/concepts/cluster-administration/addons/

You can now join any number of the control-plane node running the following command on each as root:

  kubeadm join 192.168.137.129:8443 --token t977ih.3xjsm4f5vm3xewtk \
    --discovery-token-ca-cert-hash sha256:01a3b35eaca17e2ba36f3e309205d2a16bdd0a9c008a50499e8420382d488f8d \
    --experimental-control-plane --certificate-key c19e7914f1aab84d1dd05ea35a500a8dc85cc3c10d84cd022aebc7ad7084d553

Please note that the certificate-key gives access to cluster sensitive data, keep it secret!
As a safeguard, uploaded-certs will be deleted in two hours; If necessary, you can use 
"kubeadm init phase upload-certs --upload-certs" to reload certs afterward.

Then you can join any number of worker nodes by running the following on each as root:

kubeadm join 192.168.137.129:8443 --token t977ih.3xjsm4f5vm3xewtk \
    --discovery-token-ca-cert-hash sha256:01a3b35eaca17e2ba36f3e309205d2a16bdd0a9c008a50499e8420382d488f8d

8 安装网络插件

kubectl apply -f kube-flannel.yml

9 安装配置dashboard

安装dashboard

kubectl apply -f kubernetes-dashboard.yaml

配置dashboard访问
通过 API Server 访问 Dashboard

创建ServiceAccount
> kubectl create serviceaccount dashboard-admin -n kube-system

绑定相关role
> kubectl create clusterrolebinding dashboard-cluster-admin --clusterrole=cluster-admin --serviceaccount=kube-system:dashboard-admin
–clusterrole:指定使用的集群角色,系统默认自带很多，
–serviceaccount：指定认证使用的ServiceAccount,由第一步创建

获取ServiceAccount使用的Secret
> kubectl describe sa dashboard-admin -n kube-system
Name:                dashboard-admin
Namespace:           kube-system
Labels:              <none>
Annotations:         <none>
Image pull secrets:  <none>
Mountable secrets:   dashboard-admin-token-477hc
Tokens:              dashboard-admin-token-477hc
Events:              <none>

> kubectl describe secret dashboard-admin-token-2l7k5 -n kube-system
Name:         dashboard-admin-token-477hc
Namespace:    kube-system
Labels:       <none>
Annotations:  kubernetes.io/service-account.name: dashboard-admin
              kubernetes.io/service-account.uid: b2fd5e6e-41e6-481a-9e33-5b4130383b8f

Type:  kubernetes.io/service-account-token

Data
====
ca.crt:     1025 bytes
namespace:  11 bytes
token:      eyJhbGciOiJSUzI1NiIsImtpZCI6IiJ9.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJrdWJlLXN5c3RlbSIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VjcmV0Lm5hbWUiOiJkYXNoYm9hcmQtYWRtaW4tdG9rZW4tNDc3aGMiLCJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlcnZpY2UtYWNjb3VudC5uYW1lIjoiZGFzaGJvYXJkLWFkbWluIiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9zZXJ2aWNlLWFjY291bnQudWlkIjoiYjJmZDVlNmUtNDFlNi00ODFhLTllMzMtNWI0MTMwMzgzYjhmIiwic3ViIjoic3lzdGVtOnNlcnZpY2VhY2NvdW50Omt1YmUtc3lzdGVtOmRhc2hib2FyZC1hZG1pbiJ9.h6cd6H_GXiTpd-4M50yHU5zW2wFd98GOHvw5ZaLDfGj-eoLuiRIF80xw1XvrSx_lQ0eRsdiHAxRhW-pcIIYn800FlG8WhEQx7g_wRMg6QxaRzubk-DjafCQhem5h6bUS67VnPyrj-D_g3mbBZvaj9Vwf4BL6QA2jJXWr27mfaduLSBYKUDSxRFOi-PcQAJKc3y8UaB3Wy14QfKtihb2e1LB73ON8UXxV5V6ugyLxNXM71UnjJD9ZvtKMmuFTJU0q1MMNFAsCCfjaYO7THa8JAxU82pvWBByavs1HghwkNYeC_unhEwWLvaJrfJ209Vs-WOWZhTsESR0W4nntSasJXg

Dashboard

10 加入 master 节点

kubeadm join 192.168.137.129:8443 --token t977ih.3xjsm4f5vm3xewtk \
    --discovery-token-ca-cert-hash sha256:01a3b35eaca17e2ba36f3e309205d2a16bdd0a9c008a50499e8420382d488f8d \
    --experimental-control-plane --certificate-key c19e7914f1aab84d1dd05ea35a500a8dc85cc3c10d84cd022aebc7ad7084d553

注：
生成加入命令
kubeadm token create --print-join-command
生成新的 certificate-key
kubeadm init phase upload-certs

11 加入 node 节点

kubeadm join 192.168.137.129:8443 --token t977ih.3xjsm4f5vm3xewtk \
    --discovery-token-ca-cert-hash sha256:01a3b35eaca17e2ba36f3e309205d2a16bdd0a9c008a50499e8420382d488f8d

12 查看集群状态

kubectl get nodes,pods -A

output：

NAME           STATUS   ROLES    AGE   VERSION
node/centos1   Ready    master   12m   v1.15.0
node/centos2   Ready    master   13m   v1.15.0

NAMESPACE     NAME                                        READY   STATUS    RESTARTS   AGE
kube-system   pod/coredns-6c45f5bc48-ml9xb                1/1     Running   0          13m
kube-system   pod/coredns-6c45f5bc48-sx2cz                1/1     Running   0          13m
kube-system   pod/etcd-centos1                            1/1     Running   0          12m
kube-system   pod/etcd-centos2                            1/1     Running   0          12m
kube-system   pod/kube-apiserver-centos1                  1/1     Running   0          12m
kube-system   pod/kube-apiserver-centos2                  1/1     Running   0          12m
kube-system   pod/kube-controller-manager-centos1         1/1     Running   0          12m
kube-system   pod/kube-controller-manager-centos2         1/1     Running   1          12m
kube-system   pod/kube-flannel-ds-amd64-g2x6m             1/1     Running   0          11m
kube-system   pod/kube-flannel-ds-amd64-rskbf             1/1     Running   0          11m
kube-system   pod/kube-proxy-5d7zw                        1/1     Running   0          13m
kube-system   pod/kube-proxy-f7l55                        1/1     Running   0          12m
kube-system   pod/kube-scheduler-centos1                  1/1     Running   0          12m
kube-system   pod/kube-scheduler-centos2                  1/1     Running   1          12m
kube-system   pod/kubernetes-dashboard-777fdff474-6sjb7   1/1     Running   0          3m22s

13 查看证书有效期

openssl x509 -in /etc/kubernetes/pki/apiserver.crt -noout -text |grep ' Not '

14 检查ETCD集群健康

docker run --rm -it \
--net host \
-v /etc/kubernetes:/etc/kubernetes 192.168.137.129/k8s.gcr.io/etcd:3.3.10 etcdctl \
--cert-file /etc/kubernetes/pki/etcd/peer.crt \
--key-file /etc/kubernetes/pki/etcd/peer.key \
--ca-file /etc/kubernetes/pki/etcd/ca.crt \
--endpoints https://192.168.137.129:2379 cluster-health

Jiang_James_L

关注

1
点赞
踩
3

收藏

觉得还不错? 一键收藏
2
评论
利用Kubeadm离线安装kubernets1.15.0

1 主机列表本次实验选择4台主机，3台作为Master节点，1台作为Node节点节点ipOS版本kernel版本hostnamedocker版本节点类型192.168.137.129CentOS7.43.10.0-693centos18.09.2Master192.168.137.130CentOS7.43.10.0-693centos118...
复制链接

扫一扫

专栏目录