目录
一、多mater加入之前的单节点
systemctl stop firewalld
setenforce 0
#部署master2
将master01上的kuberetes目录拷贝至master02上
scp -r /opt/kubernetes/ root@192.168.241.5:/opt
将master01上的三个组件启动脚本拷贝至master02上
scp /usr/lib/systemd/system/{kube-apiserver,kube-controller-manager,kube-scheduler}.service root@192.168.241.5:/usr/lib/systemd/system/
修改master02配置文件 kube-apiserver中的IP地址
cd /opt/kubernetes/cfg
vim kube-apiserver
KUBE_APISERVER_OPTS="--logtostderr=true \
--v=4 \
--etcd-servers=https://192.168.241.20:2379,https://192.168.241.3:2379,https://192.168.241.4:2379 \
--bind-address=192.168.241.5 \ #修改master2的IP地址
--secure-port=6443 \
--advertise-address=192.168.241.5 \ #修改master2的IP地址
拷贝master01上已有的etcd证书给master02使用
注意：master02一定要有etcd证书（不装etcd也需要etcd证书，因为master02也是要与etcd交互的）
scp -r /opt/etcd/ root@192.168.241.5:/opt
#启动master02的三个组件服务
systemctl start kube-apiserver
systemctl enable kube-apiserver
systemctl status kube-apiserver
systemctl start kube-controller-manager
systemctl enable kube-controller-manager
systemctl status kube-controller-manager
systemctl start kube-scheduler
systemctl enable kube-scheduler
systemctl status kube-scheduler
#增加环境变量
vim /etc/profile
#末尾添加
export PATH=$path:/opt/kubernetes/bin
source /etc/profile
# 查看master02是否可以检测到node节点
kubectl get node
此时的master2无法控制node节点，只能访问etcd
二、lvs（keepalived+nginx）
所有nginx节点都需要操作
#负载均衡部署
systemctl stop firewalld
setenforce 0
#部署nginx
vim /etc/yum.repos.d/nginx.repo
[nginx]
name=nginx repo
baseurl=http://nginx.org/packages/centos/7/$basearch/
gpgcheck=0
yum list
yum -y install nginx
#添加四层转发(添加stream模块)
#在events模块和http模块中间添加一个独立的stream模块
vim /etc/nginx/nginx.conf
…………省略内容
stream {
log_format main '$remote_addr $upstream_addr - [$time_local] $status $upstream_bytes_sent';
access_log /var/log/nginx/k8s-access.log main; #指定日志存放目录
upstream k8s-apiserver {
#master01的ip地址和端口
server 192.168.241.20:6443; #6443是apiserver的端口号
#master02的ip地址和端口
server 192.168.241.5:6443;
}
server {
listen 6443;
proxy_pass k8s-apiserver;
}
}
…………省略内容
# 开启nginx服务
nginx -t #检查配置文件是否有语法错误
systemctl start nginx #开启nginx服务
netstat -ntap | grep nginx #查看nginx状态及监听端口6443
#部署keepalived高可用 #在2台nginx服务器上配置
yum -y install keepalived
#修改配置文件
删除原有配置文件，重新定义添加
vim /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
# 接收邮件地址
notification_email {
acassen@firewall.loc
failover@firewall.loc
sysadmin@firewall.loc
}
# 邮件发送地址
notification_email_from Alexandre.Cassen@firewall.loc
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id NGINX_MASTER
}
vrrp_script check_nginx {
script "/etc/nginx/check_nginx.sh" #监控nginx脚本的路径，稍后会创建
}
vrrp_instance VI_1 {
state MASTER #lb01该节点为MASTER,lb02设为BACKUP
interface ens33
virtual_router_id 51
priority 100 #优先级，lb01为master，优先级100。lb02为backup，优先级设90
advert_int 1
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
192.168.241.100/24 #VIP地址（虚拟IP）
}
track_script {
check_nginx
}
}
#创建nginx监控脚本
vim /etc/nginx/check_nginx.sh
count=$(ps -ef |grep nginx |egrep -cv "grep|$$")
if [ "$count" -eq 0 ];then
systemctl stop keepalived
fi
chmod +x /etc/nginx/check_nginx.sh
#开启服务
systemctl start keepalived.service
systemctl status keepalived.service
#查看漂移地址
ip a
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
link/ether 00:0c:29:91:1c:7f brd ff:ff:ff:ff:ff:ff
inet 192.168.241.6/24 brd 192.168.241.255 scope global ens33
valid_lft forever preferred_lft forever
inet 192.168.241.100/24 scope global secondary ens33 ##vip已经起来了
valid_lft forever preferred_lft forever
inet6 fe80::38d2:d1fa:bd9c:3f26/64 scope link
valid_lft forever preferred_lft forever
#验证漂移地址
lb01中使用pkill nginx，再在lb02中使用ip a查看vip地址
#结束lb01上的nginx
pkill nginx
#keepalived也关闭了
systemctl status keepalived
#查看lb01地址
[root@localhost nginx]# ip a
......
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
link/ether 00:0c:29:91:1c:7f brd ff:ff:ff:ff:ff:ff
inet 192.168.241.6/24 brd 192.168.241.255 scope global ens33
valid_lft forever preferred_lft forever
inet6 fe80::38d2:d1fa:bd9c:3f26/64 scope link
valid_lft forever preferred_lft forever
......
这时lb01上的漂移地址就没有了
#查看lb02地址
[root@localhost nginx]# ip a
......
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
link/ether 00:0c:29:2a:02:b2 brd ff:ff:ff:ff:ff:ff
inet 192.168.241.7/24 brd 192.168.241.255 scope global ens33
valid_lft forever preferred_lft forever
inet 192.168.241.100/24 scope global secondary ens33
valid_lft forever preferred_lft forever
inet6 fe80::fda:8925:c9d0:1438/64 scope link
valid_lft forever preferred_lft forever
......
# 恢复操作（在lb01中先启动给nginx服务，在启动keepalived服务）
因为有nginx监控，如果先启动keepalived是启不了的，
systemctl start nginx
systemctl start keepalived
#再次使用ip a查看lb01地址
漂移地址就回到了lb01上，因为lb01是主节点，优先级高
[root@localhost nginx]# ip a
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
link/ether 00:0c:29:91:1c:7f brd ff:ff:ff:ff:ff:ff
inet 192.168.241.6/24 brd 192.168.241.255 scope global ens33
valid_lft forever preferred_lft forever
inet 192.168.241.100/24 scope global secondary ens33
valid_lft forever preferred_lft forever
inet6 fe80::38d2:d1fa:bd9c:3f26/64 scope link
valid_lft forever preferred_lft forever
#node节点指向VIP漂移地址
两个node均需要操作
#修改两个node节点配置文件（bootstrap.kubeconfig 、kubelet.kubeconfig、kube-proxy.kubeconfig），server ip统一VIP地址
vim /opt/kubernetes/cfg/bootstrap.kubeconfig
server: https://192.168.241.100:6443
vim /opt/kubernetes/cfg/kubelet.kubeconfig
server: https://192.168.241.100:6443
vim /opt/kubernetes/cfg/kube-proxy.kubeconfig
server: https://192.168.241.100:6443
#重启服务
systemctl restart kubelet.service
systemctl restart kube-proxy.service
#替换完成后自检
grep 100 *
bootstrap.kubeconfig: server: https://192.168.241.100:6443
kubelet.kubeconfig: server: https://192.168.241.100:6443
kube-proxy.kubeconfig: server: https://192.168.241.100:6443
#在lb01上查看nginx的k8s日志
#检查日志是否完成了访问，建立了负载均衡
cat /var/log/nginx/k8s-access.log
192.168.241.3 192.168.241.20:6443 - [14/Apr/2021:17:33:07 +0800] 200 1119
192.168.241.3 192.168.241.5:6443 - [14/Apr/2021:17:33:07 +0800] 200 1120
192.168.241.4 192.168.241.5:6443 - [14/Apr/2021:17:33:11 +0800] 200 1120
192.168.241.4 192.168.241.5:6443 - [14/Apr/2021:17:33:11 +0800] 200 1118
##k8s集群测试
master01操作
# 测试创建pod
kubectl run nginx --image=nginx
#查看pod状态
kubectl get pods
#该指令可以查看到资源具体信息，IP及所在节点
kubectl get pods -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE
nginx-dbddb74b8-8qrm4 1/1 Running 0 7m18s 172.17.69.3 192.168.241.3 <none>
#查看日志问题
#查看pod资源的日志
[root@localhost ~]# kubectl logs nginx-dbddb74b8-8qrm4
Error from server (Forbidden): Forbidden (user=system:anonymous, verb=get, resource=nodes, subresource=proxy) ( pods/log nginx-dbddb74b8-8qrm4)
原因：出现 error 是由于权限不足，需要提权解决办法（添加匿名用户授予权限）：
kubectl create clusterrolebinding cluster-system-anonymous --clusterrole=cluster-admin --user=system:anonymous
提权后查看日志记录是空的，因为现在这个容器并没有被访问
[root@localhost ~]# kubectl logs nginx-dbddb74b8-8qrm4
/docker-entrypoint.sh: /docker-entrypoint.d/ is not empty, will attempt to perform configuration
/docker-entrypoint.sh: Looking for shell scripts in /docker-entrypoint.d/
/docker-entrypoint.sh: Launching /docker-entrypoint.d/10-listen-on-ipv6-by-default.sh
10-listen-on-ipv6-by-default.sh: info: Getting the checksum of /etc/nginx/conf.d/default.conf
10-listen-on-ipv6-by-default.sh: info: Enabled listen on IPv6 in /etc/nginx/conf.d/default.conf
/docker-entrypoint.sh: Launching /docker-entrypoint.d/20-envsubst-on-templates.sh
/docker-entrypoint.sh: Launching /docker-entrypoint.d/30-tune-worker-processes.sh
/docker-entrypoint.sh: Configuration complete; ready for start up
#访问测试（创建的资源是在192.168.241.3节点上的，所以在该节点进行访问）
[root@localhost ~]# kubectl get pods -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE
nginx-dbddb74b8-8qrm4 1/1 Running 0 7m18s 172.17.69.3 192.168.241.3 <none>
去对应的node节点上使用curl 172.17.69.3
[root@hzh ~]# curl 172.17.69.3
#访问后就会产生日志记录（再次回到master01查看日志）
[root@localhost ~]# kubectl logs nginx-dbddb74b8-8qrm4
/docker-entrypoint.sh: /docker-entrypoint.d/ is not empty, will attempt to perform configuration
/docker-entrypoint.sh: Looking for shell scripts in /docker-entrypoint.d/
/docker-entrypoint.sh: Launching /docker-entrypoint.d/10-listen-on-ipv6-by-default.sh
10-listen-on-ipv6-by-default.sh: info: Getting the checksum of /etc/nginx/conf.d/default.conf
10-listen-on-ipv6-by-default.sh: info: Enabled listen on IPv6 in /etc/nginx/conf.d/default.conf
/docker-entrypoint.sh: Launching /docker-entrypoint.d/20-envsubst-on-templates.sh
/docker-entrypoint.sh: Launching /docker-entrypoint.d/30-tune-worker-processes.sh
/docker-entrypoint.sh: Configuration complete; ready for start up
172.17.69.1 - - [14/Apr/2021:09:42:34 +0000] "GET / HTTP/1.1" 200 612 "-" "curl/7.29.0" "-"
172.17.79.0 - - [14/Apr/2021:09:43:31 +0000] "GET / HTTP/1.1" 200 612 "-" "curl/7.29.0" "-"
三、dashboard
# 在master01上操作
#创建dashboard工作目录
[root@master01 k8s]# mkdir dashboard
[root@master01 k8s]# cd dashboard
#拷贝官方的yaml文件（此处已经提前下载好，直接拷贝至dashboard工作目录）
官网下载地址：https://github.com/kubernetes/tree/master/cluster/addons/dashboard
1、 创建rdac控制管理资源（kind：Role）
1）创建
[root@master01 dashboard]# kubectl create -f dashboard-rbac.yaml #-f：以文件（yaml文件）的形式创建资源
role.rbac.authorization.k8s.io/kubernetes-dashboard-minimal created
rolebinding.rbac.authorization.k8s.io/kubernetes-dashboard-minimal created
2）查看yaml文件，查看name名称及namespace命名空间
[root@master01 dashboard]# vim dashboard-rbac.yaml
里面创建的资源kind是Role角色
3）查看Role角色资源
[root@master01 dashboard]# kubectl get Role -n kube-system #-n：指向命名空间
NAME AGE
extension-apiserver-authentication-reader 27h
kubernetes-dashboard-minimal 91s
system::leader-locking-kube-controller-manager 27h
system::leader-locking-kube-scheduler 27h
system:controller:bootstrap-signer 27h
system:controller:cloud-provider 27h
system:controller:token-cleaner 27h
2、 创建secret安全资源（kind：Secret）
1）创建
[root@master01 dashboard]# kubectl create -f dashboard-secret.yaml
secret/kubernetes-dashboard-certs created
secret/kubernetes-dashboard-key-holder created
2）查看yaml文件，查看namespace命名空间及name名称
[root@master01 dashboard]# vim dashboard-secret.yaml
3） 查看创建的资源
[root@master01 dashboard]# kubectl get Secret -n kube-system
NAME TYPE DATA AGE
default-token-4vhn6 kubernetes.io/service-account-token 3 27h
kubernetes-dashboard-certs Opaque 0 23s
kubernetes-dashboard-key-holder Opaque 0 23s
3、 创建configmap配置管理资源（kind：ConfigMap）
1）创建
[root@master01 dashboard]# kubectl create -f dashboard-configmap.yaml
configmap/kubernetes-dashboard-settings created
2）查看yaml文件，查看namespace命名空间及name名称
[root@master01 dashboard]# vim dashboard-configmap.yaml
3）查看创建的资源
[root@master01 dashboard]# kubectl get ConfigMap -n kube-system
NAME DATA AGE
extension-apiserver-authentication 1 27h
kubernetes-dashboard-settings 0 22s
4、创建控制资源（kind：ServiceAccount、Deployment）
1）创建
[root@master01 dashboard]# kubectl create -f dashboard-controller.yaml
serviceaccount/kubernetes-dashboard created
deployment.apps/kubernetes-dashboard created
创建了两个资源（ServiceAccount服务访问、Deployment控制器资源）
2） 查看yaml文件，查看namespace命名空间及name名称
[root@master01 dashboard]# vim dashboard-controller.yaml
3） 查看创建的资源
[root@master01 dashboard]# kubectl get ServiceAccount -n kube-system
NAME SECRETS AGE
default 1 27h
kubernetes-dashboard 1 27s
[root@master01 dashboard]# kubectl get Deployment -n kube-system
NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE
kubernetes-dashboard 1 1 1 1 59s
5、 创建service资源（kind：Service）
1） 创建
[root@master01 dashboard]# kubectl create -f dashboard-service.yaml
service/kubernetes-dashboard created
2） 查看yaml文件，查看namespace命名空间及name名称
service资源一旦使用，则说明服务就已经提供出去并且被访问，所以必定会提供端口
[root@master01 dashboard]# vim dashboard-service.yaml
pod提供的端口，用户不能直接访问
node节点端口，用户可以访问node节点端口，映射到pod提供的端口
3） 查看创建的资源
[root@master01 dashboard]# kubectl get Service -n kube-system
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
kubernetes-dashboard NodePort 10.0.0.60 <none> 443:30001/TCP 18s
6、查看资源
1） 创建完成后查看指定的命名空间kube-system下的pod资源
这时资源创建完成，可以查看整个pods资源
[root@master01 dashboard]# kubectl get pods -n kube-system
NAME READY STATUS RESTARTS AGE
kubernetes-dashboard-65f974f565-2xl7d 1/1 Running 0 2m15s
2） 查看多个资源（可以用逗号" ，" 隔开）
[root@master01 dashboard]# kubectl get pods,service -n kube-system
NAME READY STATUS RESTARTS AGE
pod/kubernetes-dashboard-65f974f565-2xl7d 1/1 Running 0 2m39s
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
service/kubernetes-dashboard NodePort 10.0.0.60 <none> 443:30001/TCP 80s
3） 查看pods资源具体创建在哪个节点
[root@master01 dashboard]# kubectl get pods -n kube-system -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE
kubernetes-dashboard-65f974f565-2xl7d 1/1 Running 0 3m17s 172.17.69.4 192.168.241.3 <none>
7、访问测试
访问nodeIP加端口号进行测试：https://192.168.241.3:30001，如出现以下问题，属于浏览器证书不被信任（谷歌浏览器的问题），一些老版本的浏览器可以进行访问
下面我们就来解决证书不被信任的问题
8、解决浏览器无法访问的问题
1、创建证书
[root@master01 dashboard]# vim dashboard-cert.sh
cat > dashboard-csr.json <<EOF
{
"CN": "Dashboard",
"hosts": [],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"L": "BeiJing",
"ST": "BeiJing"
}
]
}
EOF
#产生ca证书
K8S_CA=$1
cfssl gencert -ca=$K8S_CA/ca.pem -ca-key=$K8S_CA/ca-key.pem -config=$K8S_CA/ca-config.json -profile=kubernetes dashboard-csr.json | cfssljson -bare dashboard
#删除原本的证书并重新创建证书
kubectl delete secret kubernetes-dashboard-certs -n kube-system
kubectl create secret generic kubernetes-dashboard-certs --from-file=./ -n kube-system
2、生成证书到指定目录
[root@master01 dashboard]# bash dashboard-cert.sh /root/k8s/k8s-cert/
3、更改yaml文件指向证书位置
[root@master01 dashboard]# vim dashboard-controller.yaml
……省略内容
args:
# PLATFORM-SPECIFIC ARGS HERE
- --auto-generate-certificates
- --tls-key-file=dashboard-key.pem #添加
- --tls-cert-file=dashboard.pem #添加
……省略内容
4、apply重新部署
变更过的yaml文件需要重新部署资源
[root@master01 dashboard]# kubectl apply -f dashboard-controller.yaml
注意：重新部署后，资源所在的节点有可能会变动，建议重新检查下资源信息及所在节点
[root@master01 dashboard]# kubectl get pods -n kube-system -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE
kubernetes-dashboard-7dffbccd68-pt2st 1/1 Running 0 35s 172.17.79.3 192.168.241.4 <none>
5、访问验证
浏览器访问：https://192.168.241.4:30001，这时就可以正常访问了
9、访问web网站页面
输入nodeIP及端口访问后，会进入登录方式的选择，正常生产环境会选择令牌的方式区登录
这里我们也选择令牌的方式登录
1、生成令牌，创建资源
1） 创建admin账户角色资源，相当于管理员
[root@master01 dashboard]# kubectl create -f k8s-admin.yaml
serviceaccount/dashboard-admin created
clusterrolebinding.rbac.authorization.k8s.io/dashboard-admin created
2） 查看secret安全资源
[root@master01 dashboard]# kubectl get secret -n kube-system
NAME TYPE DATA AGE
dashboard-admin-token-8xp9n kubernetes.io/service-account-token 3 15s #生成了admin-token资源
default-token-4vhn6 kubernetes.io/service-account-token 3 27h
kubernetes-dashboard-certs Opaque 11 3m42s
kubernetes-dashboard-key-holder Opaque 2 12m
kubernetes-dashboard-token-mxlp9 kubernetes.io/service-account-token 3 9m50s
3） 查看admin-token
[root@master01 dashboard]# kubectl describe secret dashboard-admin-token-8xp9n -n kube-system
Name: dashboard-admin-token-8xp9n
Namespace: kube-system
Labels: <none>
Annotations: kubernetes.io/service-account.name: dashboard-admin
kubernetes.io/service-account.uid: a502f1dd-9d0e-11eb-9757-000c297eb227
Type: kubernetes.io/service-account-token
Data
====
ca.crt: 1359 bytes
namespace: 11 bytes
token: eyJhbGciOiJSUzI1NiIsImtpZCI6IiJ9.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJrdWJlLXN5c3RlbSIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VjcmV0Lm5hbWUiOiJkYXNoYm9hcmQtYWRtaW4tdG9rZW4tOHhwOW4iLCJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlcnZpY2UtYWNjb3VudC5uYW1lIjoiZGFzaGJvYXJkLWFkbWluIiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9zZXJ2aWNlLWFjY291bnQudWlkIjoiYTUwMmYxZGQtOWQwZS0xMWViLTk3NTctMDAwYzI5N2ViMjI3Iiwic3ViIjoic3lzdGVtOnNlcnZpY2VhY2NvdW50Omt1YmUtc3lzdGVtOmRhc2hib2FyZC1hZG1pbiJ9.gH8HMFx1lLsXEP9Lh1wq4hhG-Kq6MyCUNsUo30hNVAduVomgHYFIKNeFxC82oBrSbZX2keM2D2qfIQJk-LSImehDuHrqje67btaQxGGb0bk3RAN4-GDF4JdeFjGYQdIgXfrajbYqICYg1EsvQVWTjQEP5cJ3VJUKXOg4_8Yee3b8h6J5EsX-r7R4I68nghQeh9hiMb5FS_iVPrc2CHHGNbavekI671NwnrFJ_IkwFguHHJ8yNx3pve3UYRPRWAyhcSP16EJfoHFgUK4m7JdzLl1oEhjxjf5hE8N3LnFFmphahGrM_cjLBctLx-hkoL-gPxv5mVi-OCyW60xzJHUjPw
末尾token：后面的内容就是令牌码，记录下来用于web网站的登录
2、复制令牌码，进入web登录页面，登录
现在我们就进入到了k8s的web网站页面，可以在里面进行相应的查看及操作，这里就不一一演示了。
至此，我们整个k8s集群的二进制部署就全部完成了，包括单master节点，多master节点以及web网站页面的显示
2226
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
