一. 使用 kubeadm 部署一个分布式的 Kubernetes 集群。
1、环境准备
1-1、主机清单
192.168.6.100 k8s-master01.yong.com k8s-master01 kubeapi.yong.com kubeapi
192.168.6.101 k8s-node01.yong.com k8s-node01
192.168.6.102 k8s-node02.yong.com k8s-node02
192.168.6.103 k8s-node03.yong.com k8s-node03
1-2、软件清单
docker-ce 20.10.23
cri-dockerd 0.3.1.3
kubeadm 1.26.1
flannel 0.20.1
1-3、系统基础环境
1-3-1、关闭防火墙
#所有节点执行:
[root@k8s-master01 ~]# ufw disable
[root@k8s-master01 ~]# ufw status
1-3-2、时间同步
#所有节点执行:
[root@k8s-master01 ~]# apt install -y ntpdate
root@k8s-master01:~# crontab -l #添加crontab 同步time1.aliyun.com
*/5 * * * * /usr/sbin/ntpdate time1.aliyun.com &> /dev/null && hwclock -w &> /dev/null
1-3-3、主机名互相解析
#所有节点执行:
[root@k8s-master01 ~]# cat /etc/hosts
192.168.6.100 k8s-master01.yong.com k8s-master01 kubeapi.yong.com kubeapi
192.168.6.101 k8s-node01.yong.com k8s-node01
192.168.6.102 k8s-node02.yong.com k8s-node02
192.168.6.103 k8s-node03.yong.com k8s-node03
1-3-4、禁用swap
#所有节点执行:
[root@k8s-master01 ~]# swapoff -a
[root@k8s-master01 ~]# systemctl --type swap
#若不禁用Swap设备,需要在后续编辑kubelet的配置文件/etc/default/kubelet,设置其忽略Swap启用的状态错误,内容:KUBELET_EXTRA_ARGS="--fail-swap-on=false"
2、安装docker-ce
#所有节点执行:
#安装必要的一些系统工具
[root@k8s-master01 ~]# apt update
[root@k8s-master01 ~]# apt -y install apt-transport-https ca-certificates curl software-properties-common
#安装GPG证书
[root@k8s-master01 ~]# curl -fsSL http://mirrors.aliyun.com/docker-ce/linux/ubuntu/gpg | apt-key add -
OK
#写入软件源信息
[root@k8s-master01 ~]# add-apt-repository "deb [arch=amd64] http://mirrors.aliyun.com/docker-ce/linux/ubuntu $(lsb_release -cs) stable"
#更新并安装Docker-CE
[root@k8s-master01 ~]# apt update
[root@k8s-master01 ~]# apt install -y docker-ce
#所有节点执行:
kubelet需要让docker容器引擎使用systemd作为CGroup的驱动,其默认值为cgroupfs,因而,我们还需要编辑docker的配置文件/etc/docker/daemon.json,添加如下内容,其中的registry-mirrors用于指明使用的镜像加速服务。
[root@k8s-master01 ~]# mkdir /etc/docker
[root@k8s-master01 ~]# vim /etc/docker/daemon.json
{
"registry-mirrors": [
"https://registry.docker-cn.com"
],
"exec-opts": ["native.cgroupdriver=systemd"],
"log-driver": "json-file",
"log-opts": {
"max-size": "200m"
},
"storage-driver": "overlay2"
}
[root@k8s-master01 ~]# systemctl daemon-reload
[root@k8s-master01 ~]# systemctl start docker
[root@k8s-master01 ~]# systemctl enable docker
3、安装cri-dockerd
#所有节点执行:
#下载地址:https://github.com/Mirantis/cri-dockerd
[root@k8s-master01 ~]# apt install ./cri-dockerd_0.3.1.3-0.ubuntu-focal_amd64.deb -y
#完成安装后,相应的服务cri-dockerd.service便会自动启动
[root@k8s-master01 ~]#systemctl status cri-docker.service
4、安装kubeadm、kubelet和kubectl
#所有节点执行:
#在各主机上生成kubelet和kubeadm等相关程序包的仓库,可参考阿里云官网
[root@k8s-master01 ~]# apt update
[root@k8s-master01 ~]# apt install -y apt-transport-https curl
[root@k8s-master01 ~]# curl https://mirrors.aliyun.com/kubernetes/apt/doc/apt-key.gpg | apt-key add -
[root@k8s-master01 ~]#cat <<EOF >/etc/apt/sources.list.d/kubernetes.list
> deb https://mirrors.aliyun.com/kubernetes/apt/ kubernetes-xenial main
> EOF
#更新仓库并安装
[root@k8s-master01 ~]# apt update
[root@k8s-master01 ~]# apt install -y kubelet kubeadm kubectl
#注意:先不要启动,只是设置开机自启动
[root@k8s-master01 ~]# systemctl enable kubelet
#确定kubeadm等程序文件的版本
[root@k8s-master01 ~]# kubeadm version
kubeadm version: &version.Info{Major:"1", Minor:"26", GitVersion:"v1.26.1", GitCommit:"8f94681cd294aa8cfd3407b8191f6c70214973a4", GitTreeState:"clean", BuildDate:"2023-01-18T15:56:50Z", GoVersion:"go1.19.5", Compiler:"gc", Platform:"linux/amd64"}
5、整合kubelet和cri-dockerd
5-1、配置cri-dockerd
#所有节点执行:
[root@k8s-master01 ~]# vim /usr/lib/systemd/system/cri-docker.service
#ExecStart=/usr/bin/cri-dockerd --container-runtime-endpoint fd://
ExecStart=/usr/bin/cri-dockerd --pod-infra-container-image=registry.aliyuncs.com/google_containers/pause:3.8 --container-runtime-endpoint fd:// --network-plugin=cni --cni-bin-dir=/opt/cni/bin --cni-cache-dir=/var/lib/cni/cache --cni-conf-dir=/etc/cni/net.d
#说明:
需要添加的各配置参数(各参数的值要与系统部署的CNI插件的实际路径相对应):
--network-plugin:指定网络插件规范的类型,这里要使用CNI;
--cni-bin-dir:指定CNI插件二进制程序文件的搜索目录;
--cni-cache-dir:CNI插件使用的缓存目录;
--cni-conf-dir:CNI插件加载配置文件的目录;
配置完成后,重载并重启cri-docker.service服务。
[root@k8s-master01 ~]# systemctl daemon-reload && systemctl restart cri-docker.service
[root@k8s-master01 ~]# systemctl status cri-docker
5-2、配置kubelet
#所有节点执行:
#配置kubelet,为其指定cri-dockerd在本地打开的Unix Sock文件的路径,该路径一般默认为“/run/cri-dockerd.sock“
[root@k8s-master01 ~]# mkdir /etc/sysconfig
[root@k8s-master01 ~]# vim /etc/sysconfig/kubelet
KUBELET_KUBEADM_ARGS="--container-runtime=remote --container-runtime-endpoint=/run/cri-dockerd.sock"
[root@k8s-master01 ~]# cat /etc/sysconfig/kubelet
KUBELET_KUBEADM_ARGS="--container-runtime=remote --container-runtime-endpoint=/run/cri-dockerd.sock"
#说明:该配置也可不进行,而是直接在后面的各kubeadm命令上使用“--cri-socket unix:///run/cri-dockerd.sock”选项
6、初始化第一个主节点
6-1、#第一个主节点执行:
#列出k8s所需要的镜像
[root@k8s-master01 ~]# kubeadm config images list
registry.k8s.io/kube-apiserver:v1.25.3
registry.k8s.io/kube-controller-manager:v1.25.3
registry.k8s.io/kube-scheduler:v1.25.3
registry.k8s.io/kube-proxy:v1.25.3
registry.k8s.io/pause:3.8
registry.k8s.io/etcd:3.5.4-0
registry.k8s.io/coredns/coredns:v1.9.3
6-2、#使用阿里云拉取所需镜像
[root@k8s-master01 ~]# kubeadm config images pull --image-repository=registry.aliyuncs.com/google_containers --cri-socket unix:///run/cri-dockerd.sock
[root@k8s-master01 ~]# docker images
REPOSITORY TAG IMAGE ID CREATED
registry.aliyuncs.com/google_containers/kube-apiserver v1.25.3 0346dbd74bcb 3 weeks ago
registry.aliyuncs.com/google_containers/kube-scheduler v1.25.3 6d23ec0e8b87 3 weeks ago
registry.aliyuncs.com/google_containers/kube-controller-manager v1.25.3 603999231275 3 weeks ago
registry.aliyuncs.com/google_containers/kube-proxy v1.25.3 beaaf00edd38 3 weeks ago
registry.aliyuncs.com/google_containers/pause 3.8 4873874c08ef 4 months ago
registry.aliyuncs.com/google_containers/etcd 3.5.4-0 a8a176a5d5d6 5 months ago
registry.aliyuncs.com/google_containers/coredns v1.9.3 5185b96f0bec 5 months ago
6-3、集群初始化命令
root@k8s-master01:~# kubeadm init --control-plane-endpoint="kubeapi.magedu.com" --kubernetes-version=v1.26.0 --pod-network-cidr=10.244.0.0/16 --service-cidr=10.96.0.0/12 --token-ttl=0 --cri-socket unix:///run/cri-dockerd.sock --upload-certs --image-repository=registry.aliyuncs.com/google_containers
[init] Using Kubernetes version: v1.26.0
[preflight] Running pre-flight checks
[preflight] Pulling images required for setting up a Kubernetes cluster
[preflight] This might take a minute or two, depending on the speed of your internet connection
[preflight] You can also perform this action in beforehand using 'kubeadm config images pull'
[certs] Using certificateDir folder "/etc/kubernetes/pki"
[certs] Generating "ca" certificate and key
[certs] Generating "apiserver" certificate and key
[certs] apiserver serving cert is signed for DNS names [k8s-master01 kubeapi.magedu.com kubernetes kubernetes.default kubernetes.default.svc kubernetes.default.svc.cluster.local] and IPs [10.96.0.1 172.31.6.100]
[certs] Generating "apiserver-kubelet-client" certificate and key
[certs] Generating "front-proxy-ca" certificate and key
[certs] Generating "front-proxy-client" certificate and key
[certs] Generating "etcd/ca" certificate and key
[certs] Generating "etcd/server" certificate and key
[certs] etcd/server serving cert is signed for DNS names [k8s-master01 localhost] and IPs [172.31.6.100 127.0.0.1 ::1]
[certs] Generating "etcd/peer" certificate and key
[certs] etcd/peer serving cert is signed for DNS names [k8s-master01 localhost] and IPs [172.31.6.100 127.0.0.1 ::1]
[certs] Generating "etcd/healthcheck-client" certificate and key
[certs] Generating "apiserver-etcd-client" certificate and key
[certs] Generating "sa" key and public key
[kubeconfig] Using kubeconfig folder "/etc/kubernetes"
[kubeconfig] Writing "admin.conf" kubeconfig file
[kubeconfig] Writing "kubelet.conf" kubeconfig file
[kubeconfig] Writing "controller-manager.conf" kubeconfig file
[kubeconfig] Writing "scheduler.conf" kubeconfig file
[kubelet-start] Writing kubelet environment file with flags to file "/var/lib/kubelet/kubeadm-flags.env"
[kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml"
[kubelet-start] Starting the kubelet
[control-plane] Using manifest folder "/etc/kubernetes/manifests"
[control-plane] Creating static Pod manifest for "kube-apiserver"
[control-plane] Creating static Pod manifest for "kube-controller-manager"
[control-plane] Creating static Pod manifest for "kube-scheduler"
[etcd] Creating static Pod manifest for local etcd in "/etc/kubernetes/manifests"
[wait-control-plane] Waiting for the kubelet to boot up the control plane as static Pods from directory "/etc/kubernetes/manifests". This can take up to 4m0s
[apiclient] All control plane components are healthy after 6.502147 seconds
[upload-config] Storing the configuration used in ConfigMap "kubeadm-config" in the "kube-system" Namespace
[kubelet] Creating a ConfigMap "kubelet-config" in namespace kube-system with the configuration for the kubelets in the cluster
[upload-certs] Storing the certificates in Secret "kubeadm-certs" in the "kube-system" Namespace
[upload-certs] Using certificate key:
e55f9016eb00aa4a020ce346b3c2b260c5a4895b983b3985ab66f1a837dee6aa
[mark-control-plane] Marking the node k8s-master01 as control-plane by adding the labels: [node-role.kubernetes.io/control-plane node.kubernetes.io/exclude-from-external-load-balancers]
[mark-control-plane] Marking the node k8s-master01 as control-plane by adding the taints [node-role.kubernetes.io/control-plane:NoSchedule]
[bootstrap-token] Using token: dczgke.vkldo06vg73gju55
[bootstrap-token] Configuring bootstrap tokens, cluster-info ConfigMap, RBAC Roles
[bootstrap-token] Configured RBAC rules to allow Node Bootstrap tokens to get nodes
[bootstrap-token] Configured RBAC rules to allow Node Bootstrap tokens to post CSRs in order for nodes to get long term certificate credentials
[bootstrap-token] Configured RBAC rules to allow the csrapprover controller automatically approve CSRs from a Node Bootstrap Token
[bootstrap-token] Configured RBAC rules to allow certificate rotation for all node client certificates in the cluster
[bootstrap-token] Creating the "cluster-info" ConfigMap in the "kube-public" namespace
[kubelet-finalize] Updating "/etc/kubernetes/kubelet.conf" to point to a rotatable kubelet client certificate and key
[addons] Applied essential addon: CoreDNS
[addons] Applied essential addon: kube-proxy
Your Kubernetes control-plane has initialized successfully!
To start using your cluster, you need to run the following as a regular user:
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
Alternatively, if you are the root user, you can run:
export KUBECONFIG=/etc/kubernetes/admin.conf
You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
https://kubernetes.io/docs/concepts/cluster-administration/addons/
You can now join any number of the control-plane node running the following command on each as root:
kubeadm join kubeapi.magedu.com:6443 --token dczgke.vkldo06vg73gju55 \
--discovery-token-ca-cert-hash sha256:86cb7fe18297e27060d078b95af5c2fc7120e6ccd3f850093e62e666d3c09ff1 \
--control-plane --certificate-key e55f9016eb00aa4a020ce346b3c2b260c5a4895b983b3985ab66f1a837dee6aa
Please note that the certificate-key gives access to cluster sensitive data, keep it secret!
As a safeguard, uploaded-certs will be deleted in two hours; If necessary, you can use
"kubeadm init phase upload-certs --upload-certs" to reload certs afterward.
Then you can join any number of worker nodes by running the following on each as root:
kubeadm join kubeapi.magedu.com:6443 --token dczgke.vkldo06vg73gju55 \
--discovery-token-ca-cert-hash sha256:86cb7fe18297e27060d078b95af5c2fc7120e6ccd3f850093e62e666d3c09ff1
7、部署网络插件
#所有节点执行:
#下载链接:
https://github.com/flannel-io/flannel/releases
[root@k8s-master01 ~]# mkdir /opt/bin
[root@k8s-master01 ~]# cp flanneld-amd64 /opt/bin/flanneld
[root@k8s-master01 ~]# chmod +x /opt/bin/flanneld
#第一个主节点执行:
#部署kube-flannel
[root@k8s-master01 ~]# kubectl apply -f https://raw.githubusercontent.com/flannel-io/flannel/master/Documentation/kube-flannel.yml
namespace/kube-flannel created
clusterrole.rbac.authorization.k8s.io/flannel created
clusterrolebinding.rbac.authorization.k8s.io/flannel created
serviceaccount/flannel created
configmap/kube-flannel-cfg created
daemonset.apps/kube-flannel-ds created
#确认Pod的状态为“Running”
[root@k8s-master01 ~]# kubectl get pods -n kube-flannel
NAME READY STATUS RESTARTS AGE
kube-flannel-ds-9bkgl 1/1 Running 0 50s
#此时,k8s-master01已经就绪
[root@k8s-master01 ~]# kubectl get nodes
NAME STATUS ROLES AGE VERSION
k8s-master01 Ready control-plane 20m v1.26.1
8、添加其他节点到集群中
#k8s-node01---k8s-node03执行:
#k8s-node01---k8s-node03加入集群
kubeadm join kubeapi.magedu.com:6443 --token dczgke.vkldo06vg73gju55 --discovery-token-ca-cert-hash sha256:86cb7fe18297e27060d078b95af5c2fc7120e6ccd3f850093e62e666d3c09ff1 --cri-socket unix:///run/cri-dockerd.sock
#注意,命令需要加上--cri-socket unix:///run/cri-dockerd.sock
#第一节点验证节点添加结果
[root@k8s-master01 ~]#kubectl get nodes
NAME STATUS ROLES AGE VERSION
k8s-master01 Ready control-plane 22h v1.26.1
k8s-node01 Ready <none> 15m v1.26.1
k8s-node02 Ready <none> 22h v1.26.1
k8s-node03 Ready <none> 22h v1.26.1
二. 在集群上编排运行 demoapp,并使用 Service 完成 Pod 发现和服务发布。
#创建deployment资源
root@k8s-master01:~# kubectl create deployment demoapp --image=ikubernetes/demoapp:v1.0 --replicas=3
deployment.apps/demoapp created
#查看pods信息,三个
root@k8s-master01:~# kubectl get pods
NAME READY STATUS RESTARTS AGE
demoapp-75f59c894-7gnkn 1/1 Running 0 14m
demoapp-75f59c894-kt4h2 1/1 Running 0 14m
demoapp-75f59c894-txxtp 1/1 Running 0 14m
#创建services资源
root@k8s-master01:~# kubectl create service nodeport demoapp --tcp=80:80
service/demoapp created
#查看Service对象demoapp使用的NodePort,以便于在集群外部进行访问:
root@k8s-master01:~# kubectl get svc -l app=demoapp
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
demoapp NodePort 10.109.40.66 <none> 80:31784/TCP 32s
#demoapp是一个web应用,因此,用户可以于集群外部通过“http://NodeIP:31784”这个URL访问demoapp上的应用,例如于集群外通过浏览器访问“http://192.168.6.101:31784”。
#对demoapp.default.svc服务发起访问请求,验证其负载均衡的效果。
root@k8s-master01:~# curl 10.109.40.66
iKubernetes demoapp v1.0 !! ClientIP: 10.244.0.0, ServerName: demoapp-75f59c894-txxtp, ServerIP: 10.244.2.2!
root@k8s-master01:~# curl 10.109.40.66
iKubernetes demoapp v1.0 !! ClientIP: 10.244.0.0, ServerName: demoapp-75f59c894-7gnkn, ServerIP: 10.244.3.2!
root@k8s-master01:~# curl 10.109.40.66
iKubernetes demoapp v1.0 !! ClientIP: 10.244.0.0, ServerName: demoapp-75f59c894-7gnkn, ServerIP: 10.244.3.2!
root@k8s-master01:~# curl 10.109.40.66
iKubernetes demoapp v1.0 !! ClientIP: 10.244.0.0, ServerName: demoapp-75f59c894-kt4h2, ServerIP: 10.244.1.2!
三. 要求以配置文件的方式,在集群上编排运行 nginx,并使用 Service 完成 Pod 发现和服务发布。
#使用–dry-run -o yaml命令导出配置文件,为模板
root@k8s-master01:~# kubectl create deployment demoapp --image=ikubernetes/demoapp:v1.0 --replicas=3 --dry-run=client -o yaml>nginx-config.yaml
cat nginx-config.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
creationTimestamp: null
labels:
app: demoapp
name: demoapp
spec:
replicas: 3
selector:
matchLabels:
app: demoapp
strategy: {}
template:
metadata:
creationTimestamp: null
labels:
app: demoapp
spec:
containers:
- image: ikubernetes/demoapp:v1.0
name: demoapp
resources: {}
status: {}
#修改nginx-config.yaml配置文件
root@k8s-master01:~# cat nginx-config.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app: nginx
name: nginx
spec:
replicas: 2
selector:
matchLabels:
app: nginx
template:
metadata:
labels:
app: nginx
spec:
containers:
- image: nginx:alpine
name: nginx
#使用配置文件生成depolyment资源
root@k8s-master01:~# kubectl create -f nginx-config.yaml
deployment.apps/nginx created
#查看创建的pods信息
root@k8s-master01:~# kubectl get pods
NAME READY STATUS RESTARTS AGE
demoapp-75f59c894-7gnkn 1/1 Running 0 90m
demoapp-75f59c894-kt4h2 1/1 Running 0 90m
demoapp-75f59c894-txxtp 1/1 Running 0 90m
nginx-6c557cc74d-pkkv7 1/1 Running 0 111s
nginx-6c557cc74d-wr8zt 1/1 Running 0 111s
四. 扩展作业:要求以配置文件的方式,在集群上编排运行 wordpress 和 mysql,并使用 Service 完成 Pod 发现和服务发布。
#mysql pod yaml配置文件
root@k8s-master01:~# cat mysql_pod.yaml
apiVersion: v1
kind: Pod
metadata:
name: mysql
namespace: default
labels:
app: mysql
spec:
containers:
- name: mysql
image: mysql:8.0
ports:
- containerPort: 3306
env:
- name: MYSQL_RANDOM_ROOT_PASSWORD
value: '1'
- name: MYSQL_DATABASE
value: wpdb
- name: MYSQL_USER
value: wpuser
- name: MYSQL_PASSWORD
value: magedu.com
#创建mysql pods
root@k8s-master01:~# kubectl create -f mysql_pod.yaml
pod/mysql created
#查看创建的mysql pods信息
root@k8s-master01:~# kubectl get pods --show-labels
NAME READY STATUS RESTARTS AGE LABELS
mysql 1/1 Running 0 7m38s app=mysql
``
#mysql svc yaml配置
```bash
root@k8s-master01:~# cat mysql_pod_svc.yaml
apiVersion: v1
kind: Service
metadata:
name: mysql
spec:
selector:
app: mysql
ports:
- port: 3306
targetPort: 3306
#创建mysql svc服务
root@k8s-master01:~# kubectl create -f mysql_pod_svc.yaml
service/mysql created
#查看创建mysql svc信息
root@k8s-master01:~# kubectl get svc
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
kubernetes ClusterIP 10.96.0.1 <none> 443/TCP 3d22h
mysql ClusterIP 10.99.136.215 <none> 3306/TCP 16m
#wordpress yaml配置信息
root@k8s-master01:~# cat wordpress.yaml
apiVersion: v1
kind: Pod
metadata:
name: wordpress
namespace: default
labels:
app: wordpress
spec:
containers:
- name: wordpress
image: wordpress
ports:
- containerPort: 80
env:
- name: WORDPRESS_DB_HOST
value: mysql
- name: WORDPRESS_DB_NAME
value: wpdb
- name: WORDPRESS_DB_USER
value: wpuser
- name: WORDPRESS_DB_PASSWORD
value: magedu.com
#创建wordpress pods
root@k8s-master01:~# kubectl create -f wordpress.yaml
pod/wordpress created
#查看wordpress svc yaml配置文件
root@k8s-master01:~# cat wordpress_svc.yaml
apiVersion: v1
kind: Service
metadata:
name: wordpress
spec:
selector:
app: wordpress
type: NodePort
ports:
- port: 80
targetPort: 80
nodePort: 30008
#创建wordpress service服务
root@k8s-master01:~# kubectl create -f wordpress_svc.yaml
service/wordpress created
#创建查看wordpress svc信息
root@k8s-master01:~# kubectl get svc
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
kubernetes ClusterIP 10.96.0.1 <none> 443/TCP 3d22h
mysql ClusterIP 10.99.136.215 <none> 3306/TCP 16m
wordpress NodePort 10.106.175.103 <none> 80:30008/TCP 2m18s
#登录wordpress服务,node节点,端口:30008