查看磁盘大小
[root@localhost ~]# free -m
total used free shared buff/cache available
Mem: 1819 761 244 13 814 786
Swap: 2047 0 2047
[root@localhost ~]# swapoff -a
修改fstab文件
[root@localhost ~]# vim /etc/fstab
将net.bridge.bridge-nf-call-ip6tables = 1、net.bridge.bridge-nf-call-iptables = 1
放进新建的k8s.conf文件中
[root@localhost sysctl.d]# vim k8s.conf
允许iptables检查桥接流量
[root@localhost sysctl.d]# sysctl --system
* Applying /usr/lib/sysctl.d/00-system.conf ...
net.bridge.bridge-nf-call-ip6tables = 0
net.bridge.bridge-nf-call-iptables = 0
net.bridge.bridge-nf-call-arptables = 0
* Applying /usr/lib/sysctl.d/10-default-yama-scope.conf ...
kernel.yama.ptrace_scope = 0
* Applying /usr/lib/sysctl.d/50-default.conf ...
kernel.sysrq = 16
kernel.core_uses_pid = 1
net.ipv4.conf.default.rp_filter = 1
net.ipv4.conf.all.rp_filter = 1
net.ipv4.conf.default.accept_source_route = 0
net.ipv4.conf.all.accept_source_route = 0
net.ipv4.conf.default.promote_secondaries = 1
net.ipv4.conf.all.promote_secondaries = 1
fs.protected_hardlinks = 1
fs.protected_symlinks = 1
* Applying /usr/lib/sysctl.d/60-libvirtd.conf ...
fs.aio-max-nr = 1048576
* Applying /etc/sysctl.d/99-sysctl.conf ...
* Applying /etc/sysctl.d/docker.conf ...
* Applying /etc/sysctl.d/k8s.conf ...
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
* Applying /etc/sysctl.conf ...
[root@localhost sysctl.d]#
[root@localhost sysctl.d]# ssh-keygen
Generating public/private rsa key pair.
Enter file in which to save the key (/root/.ssh/id_rsa):
Enter passphrase (empty for no passphrase):
Enter same passphrase again:
Your identification has been saved in /root/.ssh/id_rsa.
Your public key has been saved in /root/.ssh/id_rsa.pub.
The key fingerprint is:
SHA256:tec63TfTIXRCgyYZY4g6UkmFqLZQmn38YSTqzFVTR9E root@localhost.localdomain
The key's randomart image is:
+---[RSA 2048]----+
| o.+...o=*o. |
| ..+..+..+.oEo |
|.=.o.+ . + . . |
|=ooo+ o . . o . |
|o+oo.o .S . o o |
| .+ . o . . |
| .... o|
| ... .oo|
| .. .o|
+----[SHA256]-----+
禁用selinux和iptables
[root@localhost ~]# getenforce
Permissive
发现没有变化,需要重启
[root@localhost ~]# reboot
[root@localhost ~]# getenforce
Disabled
看到已经关闭
[root@localhost ~]# iptables -L
Chain INPUT (policy ACCEPT)
target prot opt source destination
Chain FORWARD (policy ACCEPT)
target prot opt source destination
Chain OUTPUT (policy ACCEPT)
target prot opt source destination
看到iptables无规则
配置 Docker 守护程序,使用 systemd 来管理容器的 cgroup
重启docker
[root@localhost docker]# systemctl restart docker
查看
是否配置成功
[root@localhost docker]# docker info
看到配置成功
安装 kubeadm、kubelet 和 kubectl
官方文档中安装的是Google的,我选择安装阿里云的速度较快
配置repo文件
将keepache设为1
安装kubelet kubeadm kubectl
此时,基本环境搭建成功
使用kubeadm创建集群
启动并设置开机自启动
[root@localhost yum.repos.d]# systemctl enable --now kubelet
Created symlink from /etc/systemd/system/multi-user.target.wants/kubelet.service to /usr/lib/systemd/sysem/kubelet.service.
[root@localhost ~]# kubeadm config print init-defaults
apiVersion: kubeadm.k8s.io/v1beta3
bootstrapTokens:
- groups:
- system:bootstrappers:kubeadm:default-node-token
token: abcdef.0123456789abcdef
ttl: 24h0m0s
usages:
- signing
- authentication
kind: InitConfiguration
localAPIEndpoint:
advertiseAddress: 1.2.3.4
bindPort: 6443
nodeRegistration:
criSocket: /var/run/dockershim.sock
imagePullPolicy: IfNotPresent
name: node
taints: null
---
apiServer:
timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta3
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controllerManager: {}
dns: {}
etcd:
local:
dataDir: /var/lib/etcd
imageRepository: k8s.gcr.io
kind: ClusterConfiguration
kubernetesVersion: 1.23.0
networking:
dnsDomain: cluster.local
serviceSubnet: 10.96.0.0/12
scheduler: {}
[root@localhost ~]# kubeadm config images list --image-repository registry.aliyuncs.com/google_containers
registry.aliyuncs.com/google_containers/kube-apiserver:v1.23.1
registry.aliyuncs.com/google_containers/kube-controller-manager:v1.23.1
registry.aliyuncs.com/google_containers/kube-scheduler:v1.23.1
registry.aliyuncs.com/google_containers/kube-proxy:v1.23.1
registry.aliyuncs.com/google_containers/pause:3.6
registry.aliyuncs.com/google_containers/etcd:3.5.1-0
registry.aliyuncs.com/google_containers/coredns:v1.8.6
[root@localhost ~]# kubeadm config images pull --image-repository registry.aliyuncs.com/google_containers
[config/images] Pulled registry.aliyuncs.com/google_containers/kube-apiserver:v1.23.1
[config/images] Pulled registry.aliyuncs.com/google_containers/kube-controller-manager:v1.23.1
[config/images] Pulled registry.aliyuncs.com/google_containers/kube-scheduler:v1.23.1
[config/images] Pulled registry.aliyuncs.com/google_containers/kube-proxy:v1.23.1
[config/images] Pulled registry.aliyuncs.com/google_containers/pause:3.6
[config/images] Pulled registry.aliyuncs.com/google_containers/etcd:3.5.1-0
[config/images] Pulled registry.aliyuncs.com/google_containers/coredns:v1.8.6
配置hosts文件,做域名映射
[root@localhost ~]# vim /etc/hosts
登录harbor
[root@localhost harbor]# docker login reg.smy.org
Username: admin
Password:
WARNING! Your password will be stored unencrypted in /root/.docker/config.json.
Configure a credential helper to remove this warning. See
https://docs.docker.com/engine/reference/commandline/login/#credentials-store
Login Succeeded
登陆成功
查看
[root@localhost harbor]# docker images | grep ^registry
registry.aliyuncs.com/google_containers/kube-apiserver v1.23.1 b6d7abedde39 2 weeks ago 135MB
registry.aliyuncs.com/google_containers/kube-proxy v1.23.1 b46c42588d51 2 weeks ago 112MB
registry.aliyuncs.com/google_containers/kube-scheduler v1.23.1 71d575efe628 2 weeks ago 53.5MB
registry.aliyuncs.com/google_containers/kube-controller-manager v1.23.1 f51846a4fd28 2 weeks ago 125MB
registry.aliyuncs.com/google_containers/etcd 3.5.1-0 25f8c7f3da61 2 months ago 293MB
registry.aliyuncs.com/google_containers/coredns v1.8.6 a4ca41631cc7 2 months ago 46.8MB
registry.aliyuncs.com/google_containers/pause 3.6 6270bb605e12 4 months ago 683kB
将以上信息以:为分隔符显示1,2列
[root@localhost harbor]# docker images | grep ^registry | awk '{print $1":"$2 }'
registry.aliyuncs.com/google_containers/kube-apiserver:v1.23.1
registry.aliyuncs.com/google_containers/kube-proxy:v1.23.1
registry.aliyuncs.com/google_containers/kube-controller-manager:v1.23.1
registry.aliyuncs.com/google_containers/kube-scheduler:v1.23.1
registry.aliyuncs.com/google_containers/etcd:3.5.1-0
registry.aliyuncs.com/google_containers/coredns:v1.8.6
registry.aliyuncs.com/google_containers/pause:3.6
设置将含有registry的标签为k8s
[root@localhost harbor]# docker images | grep ^registry | awk '{print $1":"$2 }' | awk -F/ '{system("docker tag "$0" reg.smy.org/k8s/"$3"")}'
查看
[root@localhost harbor]# docker images
REPOSITORY TAG IMAGE ID CREATED SIZE
reg.smy.org/library/nginx latest f6987c8d6ed5 2 weeks ago 141MB
nginx latest f6987c8d6ed5 2 weeks ago 141MB
registry.aliyuncs.com/google_containers/kube-apiserver v1.23.1 b6d7abedde39 2 weeks ago 135MB
reg.smy.org/k8s/kube-apiserver v1.23.1 b6d7abedde39 2 weeks ago 135MB
reg.smy.org/k8s/kube-proxy v1.23.1 b46c42588d51 2 weeks ago 112MB
registry.aliyuncs.com/google_containers/kube-proxy v1.23.1 b46c42588d51 2 weeks ago 112MB
reg.smy.org/k8s/kube-controller-manager v1.23.1 f51846a4fd28 2 weeks ago 125MB
registry.aliyuncs.com/google_containers/kube-controller-manager v1.23.1 f51846a4fd28 2 weeks ago 125MB
reg.smy.org/k8s/kube-scheduler v1.23.1 71d575efe628 2 weeks ago 53.5MB
registry.aliyuncs.com/google_containers/kube-scheduler v1.23.1 71d575efe628 2 weeks ago 53.5MB
goharbor/harbor-exporter v2.4.1 0ee361b5454a 2 weeks ago 82.7MB
goharbor/chartmuseum-photon v2.4.1 337a9d03b197 2 weeks ago 174MB
goharbor/redis-photon v2.4.1 2bbe19b8cc5d 2 weeks ago 156MB
goharbor/trivy-adapter-photon v2.4.1 d39fd42c93f0 2 weeks ago 151MB
goharbor/notary-server-photon v2.4.1 e75c68b6638b 2 weeks ago 111MB
goharbor/notary-signer-photon v2.4.1 17a627caf6dd 2 weeks ago 108MB
goharbor/harbor-registryctl v2.4.1 bff3d2665456 2 weeks ago 137MB
goharbor/registry-photon v2.4.1 b46e45c49d6f 2 weeks ago 79.2MB
goharbor/nginx-photon v2.4.1 78aad8c8ef41 2 weeks ago 45.7MB
goharbor/harbor-log v2.4.1 c86562daee64 2 weeks ago 160MB
goharbor/harbor-jobservice v2.4.1 d8baceb5f1f6 2 weeks ago 220MB
goharbor/harbor-core v2.4.1 80a4e3e60403 2 weeks ago 198MB
goharbor/harbor-portal v2.4.1 1c9ed1cc3cb9 2 weeks ago 54.7MB
goharbor/harbor-db v2.4.1 417d0a840247 2 weeks ago 222MB
goharbor/prepare v2.4.1 e9158d34970b 2 weeks ago 257MB
busybox latest ffe9d497c324 3 weeks ago 1.24MB
reg.smy.org/k8s/etcd 3.5.1-0 25f8c7f3da61 2 months ago 293MB
registry.aliyuncs.com/google_containers/etcd 3.5.1-0 25f8c7f3da61 2 months ago 293MB
reg.smy.org/k8s/coredns v1.8.6 a4ca41631cc7 2 months ago 46.8MB
registry.aliyuncs.com/google_containers/coredns v1.8.6 a4ca41631cc7 2 months ago 46.8MB
reg.smy.org/k8s/pause 3.6 6270bb605e12 4 months ago 683kB
registry.aliyuncs.com/google_containers/pause 3.6 6270bb605e12 4 months ago 683kB
reg.smy.org/library/game2048 latest 19299002fdbe 5 years ago 55.5MB
yakexi007/game2048 latest 19299002fdbe 5 years ago 55.5MB
yakexi007/mario latest 9a35a9e43e8c 6 years ago 198MB
查看标签为k8s的项目
[root@localhost harbor]# docker images | grep k8s
reg.smy.org/k8s/kube-apiserver v1.23.1 b6d7abedde39 2 weeks ago 135MB
reg.smy.org/k8s/kube-proxy v1.23.1 b46c42588d51 2 weeks ago 112MB
reg.smy.org/k8s/kube-controller-manager v1.23.1 f51846a4fd28 2 weeks ago 125MB
reg.smy.org/k8s/kube-scheduler v1.23.1 71d575efe628 2 weeks ago 53.5MB
reg.smy.org/k8s/etcd 3.5.1-0 25f8c7f3da61 2 months ago 293MB
reg.smy.org/k8s/coredns v1.8.6 a4ca41631cc7 2 months ago 46.8MB
reg.smy.org/k8s/pause 3.6 6270bb605e12 4 months ago 683kB
上传
[root@localhost harbor]# docker images | grep k8s | awk '{system("docker push "$1":"$2"")}'
The push refers to repository [reg.smy.org/k8s/kube-apiserver]
79f770f7d109: Pushed
333ac6342b5d: Pushed
6d75f23be3dd: Pushed
v1.23.1:digest:sha256:6c191ee12d8cf9ca0f69d4078b9ea4ce0e8c99e887ad039d541ffd93b432f2b0size:949
The push refers to repository [reg.smy.org/k8s/kube-proxy]
6c041f55d0e4: Pushed
2b8347a02bc5: Pushed
194a408e97d8: Pushed
unauthorized: project k8s not found: project k8s not found
The push refers to repository [reg.smy.org/k8s/kube-scheduler]
31830be0192c: Pushed
333ac6342b5d: Pushed
6d75f23be3dd: Pushed
v1.23.1:digest:sha256:cf6b3ef2b319ab4af0be81c0cc344a21663fd7aaa59c27915ab3bfc46ad6f384size:950
The push refers to repository [reg.smy.org/k8s/kube-controller-manager]
24667a6627a7: Pushed
333ac6342b5d: Pushed
6d75f23be3dd: Pushed
v1.23.1:digest:sha256:55231f7b93248c28e8bdc1790a22d05ae11e6bc640c91701819c0f2e52ec83c2size:949
The push refers to repository [reg.smy.org/k8s/etcd]
62ae031121b1: Pushed
664dd6f2834b: Pushed
d80003ff5706: Pushed
b6e8c573c18d: Pushed
6d75f23be3dd: Mounted
fromk8s/kube-scheduler3.5.10:digest:sha256:05c1a3be66823dcaca55ebe17c3c9a60de7ceb948047da3e95308348325ddd5asize:1372
The push refers to repository [reg.smy.org/k8s/coredns]
80e4a2390030: Pushed
256bc5c338a6: Pushed
v1.8.6:digest:sha256:8916c89e1538ea3941b58847e448a2c6d940c01b8e716b20423d2d8b189d3972size:739
The push refers to repository [reg.smy.org/k8s/pause]
1021ef88c797: Pushed
3.6:digest:sha256:74bf6fc6be13c4ec53a86a5acf9fdbc6787b176db0693659ad6ac89f115e182csize:526
更改仓库指向
[root@localhost ~]# kubeadm config images list -image-repository reg.smy.org/k8s
registry.aliyuncs.com/google_containers/kube-apiserver:v1.23.1
registry.aliyuncs.com/google_containers/kube-controller-manager:v1.23.1
registry.aliyuncs.com/google_containers/kube-scheduler:v1.23.1
registry.aliyuncs.com/google_containers/kube-proxy:v1.23.1
registry.aliyuncs.com/google_containers/pause:3.6
registry.aliyuncs.com/google_containers/etcd:3.5.1-0
registry.aliyuncs.com/google_containers/coredns:v1.8.6
初始化集群
[root@localhost ~]# kubeadm init --pod-network-cidr=10.224.0.0/16 --image-trpository reg.smy.org/k8s
[root@localhost ~]# kubeadm init --pod-network-cidr=10.244.0.0/16 --image-repository reg.westos.org/k8s
[init] Using Kubernetes version: v1.23.1
[preflight] Running pre-flight checks
操控集群
[root@localhost ~]# export KUBECONFIG=/etc/kubernetes/admin.conf
[root@localhost ~]# pwd
/root
[root@localhost ~]# vim .bash_profile
下载kube-flannel.yml
[root@localhost ~]# wget https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml
[root@localhost ~]# docker pull rancher/mirrored-flannelcni-flannel-cni-plugin:v1.0.0
v1.0.0: Pulling from rancher/mirrored-flannelcni-flannel-cni-plugin
4e9f2cdf4387: Pull complete
6ff8050baee9: Pull complete
Digest: sha256:bfe8f30c74bc6f31eba0cc6659e396dbdd5ab171314ed542cc238ae046660ede
Status: Downloaded newer image for rancher/mirrored-flannelcni-flannel-cni-plugin:v1.0.0
docker.io/rancher/mirrored-flannelcni-flannel-cni-plugin:v1.0.0
[root@localhost ~]# docker tag rancher/mirrored-flannelcni-flannel-cni-plugin:v1.0.0 reg.smy.org/k8s/mirrored-flannelcni-flannel-cni-plugin:v1.0.0
[root@localhost ~]# docker push reg.smy.org/k8s/mirrored-flannelcni-flannel-cni-plugin:v1.0.0
The push refers to repository [reg.smy.org/k8s/mirrored-flannelcni-flannel-cni-plugin]
df1287c3829d: Pushed
f1dd685eb59e: Pushed
v1.0.0: digest: sha256:1511c5f15ca57b7c5256944f407113bb493e829befc4ba0488245e465b1365b9 size: 738
[root@server1 ~]# docker pull quay.io/coreos/flannel:v0.15.1
v0.15.1: Pulling from coreos/flannel
5758d4e389a3: Pull complete
5570bac27614: Pull complete
7d1eea13ca76: Pull complete
a8108e817db1: Pull complete
741353eab5bb: Pull complete
f427d97d2e68: Pull complete
fd61dae2c96c: Pull complete
Digest: sha256:9a296fbb67790659adc3701e287adde3c59803b7fcefe354f1fc482840cdb3d9
Status: Downloaded newer image for quay.io/coreos/flannel:v0.15.1
quay.io/coreos/flannel:v0.15.1
[root@server1 ~]# docker tag quay.io/coreos/flannel:v0.15.1 reg.westos.org/k8s/flannel:v0.15.1
[root@server1 ~]# docker push reg.westos.org/k8s/flannel:v0.15.1
The push refers to repository [reg.westos.org/k8s/flannel]
5c32c759eea2: Pushed
8c5368be4bdf: Pushed
54089bc26b6b: Pushed
2ad3602f224f: Pushed
ab9ef8fb7abb: Pushed
5feaed8da748: Pushed
7fcb75871b21: Pushed
v0.15.1: digest: sha256:a3ebdc7e5e44d1ba3ba8ccd8399e81444102bd35f5f480997a637a42d1e1da6b size: 1785
修改配置文件kube-flannel.yml
[root@localhost ~]# vim kube-flannel.yml
修改其中的image
image: rancher/mirrored-flannelcni-flannel-cni-plugin:v1.0.0
command:
- cp
args:
- -f
- /flannel
- /opt/cni/bin/flannel
volumeMounts:
- name: cni-plugin
mountPath: /opt/cni/bin
- name: install-cni
image: quay.io/coreos/flannel:v0.15.1
command:
- cp
args:
- -f
- /etc/kube-flannel/cni-conf.json
- /etc/cni/net.d/10-flannel.conflist
volumeMounts:
- name: cni
mountPath: /etc/cni/net.d
- name: flannel-cfg
mountPath: /etc/kube-flannel/
containers:
- name: kube-flannel
image: quay.io/coreos/flannel:v0.15.1
[root@localhost ~]# vim .bashrc
[root@localhost ~]# cat .bashrc
[root@localhost ~]# source .bashrc
[root@localhost ~]# kubectl apply -f kube-flannel.yml
Warning: policy/v1beta1 PodSecurityPolicy is deprecated in v1.21+, unavailable in v1.25+
podsecuritypolicy.policy/psp.flannel.unprivileged created
clusterrole.rbac.authorization.k8s.io/flannel created
clusterrolebinding.rbac.authorization.k8s.io/flannel created
serviceaccount/flannel created
configmap/kube-flannel-cfg created
daemonset.apps/kube-flannel-ds created
[root@localhost ~]# kubectl get pod -n kube-system
NAME READY STATUS RESTARTS AGE
coredns-7b56f6bc55-24dr7 1/1 Running 0 3h31m
coredns-7b56f6bc55-j5r9t 1/1 Running 0 3h31m
etcd-server1 1/1 Running 0 3h31m
kube-apiserver-server1 1/1 Running 0 3h31m
kube-controller-manager-server1 1/1 Running 1 (72m ago) 3h31m
kube-flannel-ds-7cj2m 1/1 Running 0 6m12s
kube-proxy-qbkkz 1/1 Running 0 3h31m
kube-scheduler-server1 1/1 Running 0 3h31m
[root@localhost ~]# kubectl get node
NAME STATUS ROLES AGE VERSION
localhost Ready control-plane,master 3h32m v1.23.1
kubeadm join 192.168.1.140:6443 --token blhsdl.ouxywyokbn21sk7i --discovery-token-ca-cert-hash sha256:9ede37800agbcf80e2a6cc23f441e9177624a94cfl3e8e227563baff8e9ef320
第二台主机配置
[root@localhost ~]# kubeadm join 192.168.1.140:6443 --token b1hsd1.ouxywyokbn21sk7i \
> --discovery-token-ca-cert-hash sha256:9ede37800a9bcf80e2a6cc23f441e9177624a94cf13e8e227563baff8e9ef320
[preflight] Running pre-flight checks
[WARNING FileExisting-tc]: tc not found in system path
[WARNING Hostname]: hostname "localhost" could not be reached
[WARNING Hostname]: hostname "localhost": lookup server2 on 114.114.114.114:53: no such host
[preflight] Reading configuration from the cluster...
[preflight] FYI: You can look at this config file with 'kubectl -n kube-system get cm kubeadm-config -o yaml'
[kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml"
[kubelet-start] Writing kubelet environment file with flags to file "/var/lib/kubelet/kubeadm-flags.env"
[kubelet-start] Starting the kubelet
[kubelet-start] Waiting for the kubelet to perform the TLS Bootstrap...
This node has joined the cluster:
* Certificate signing request was sent to apiserver and a response was received.
* The Kubelet was informed of the new secure connection details.
Run 'kubectl get nodes' on the control-plane to see this node join the cluster.
[root@localhost ~]# docker images
REPOSITORY TAG IMAGE ID CREATED SIZE
reg.smy.org/k8s/pause 3.6 6270bb605e12 4 months ago 683kB
[root@localhost ~]# docker images
REPOSITORY TAG IMAGE ID CREATED SIZE
reg.westos.org/k8s/kube-proxy v1.23.1 b46c42588d51 2 weeks ago 112MB
reg.westos.org/k8s/flannel v0.15.1 e6ea68648f0c 6 weeks ago 69.5MB
reg.westos.org/k8s/mirrored-flannelcni-flannel-cni-plugin v1.0.0 cd5235cd7dc2 2 months ago 9.03MB
reg.westos.org/k8s/pause 3.6 6270bb605e12 4 months ago 683kB
登上第一台主机
[root@localhost ~]# kubectl get node
NAME STATUS ROLES AGE VERSION
server1 Ready control-plane,master 4h5m v1.23.1
server2 Ready <none> 2m47s v1.23.1
[root@localhost ~]# kubeadm token list
TOKEN TTL EXPIRES USAGES DESCRIPTION EXTRA GROUPS
b1hsd1.ouxywyokbn21sk7i 19h 2022-01-01T10:04:55Z authentication,signing The default bootstrap token generated by 'kubeadm init'. system:bootstrappers:kubeadm:default-node-token
[root@localhost ~]# kubectl get pod -n kube-system
NAME READY STATUS RESTARTS AGE
coredns-7b56f6bc55-24dr7 1/1 Running 0 4h15m
coredns-7b56f6bc55-j5r9t 1/1 Running 0 4h15m
etcd-server1 1/1 Running 0 4h15m
kube-apiserver-server1 1/1 Running 0 4h15m
kube-controller-manager-server1 1/1 Running 1 (115m ago) 4h15m
kube-flannel-ds-7cj2m 1/1 Running 0 49m
kube-flannel-ds-jxb6j 1/1 Running 0 13m
kube-proxy-qbkkz 1/1 Running 0 4h15m
kube-proxy-wxxvg 1/1 Running 0 13m
kube-scheduler-server1 1/1 Running 0 4h15m
反正步骤大差不差都是这样