搭建K8S-web-MySQL-keepalived 负载均衡集群项目
部署目录
总体架构图
架构图说明：
keepalivd部分：k8s内部的ip无法实现外部访问，这样用户是无法使用的，所以用k8s节点的ip段做VIP让外网可以用唯一的一个ip访问集群。双节点保证了可以用性。
web部分：django做的web集群，又绑定了svc的NodePORT模式，此时访问k8s集群所有的节点都可以访问到的，但是用户不能不断的变化访问IP，所以又交给keepalived去实现单一IP访问。web架构也用deployment模式管理快速升级，变更版本。（注意svc的rr轮训机制，并不保留web用户的seesion需要配合开发将seesion保存到数据库或文件中，实现会话统一）。
Mysql部分：同样使用svc+deployment的方式管理mysql镜像，方便版本变更，单点故障。但是只用了1个副本管理的方式，因为多副本会造成数据不一致。同时使用pv+pvc的方式持久化存储。
安装MySQL
下载MySQL镜像（网速不好，提前下载，不容易报错）
docker pull mysql:5.7.38
docker tag mysql:5.7.38 c104.cheng.com/cheng/mysql:5.7.38
#推送到自己的私有仓库，因为默认k8s集群，不会所有 节点都有本地镜像的，如果不放自己的私有仓库，都是走公网仓库，肯能拉取太慢而失败！
docker login https://c104.cheng.com -u admin -p Harbor12345
#如果没有私有仓库参考我的搭建Harbor文章
docker push c104.cheng.com/cheng/mysql:5.7.38
创建一个Mysql的deployment模板
创建一个Mysql的deployment模板，然后去修改它，不可能手写那么多基础标签，而且容易出错。
kubectl create deployment mysql5738 --image=c104.cheng.com/cheng/mysql:5.7.38
#此时启动会有问题，因为默认mysql启动是要参数的！所以导出yaml文件，增加标签：ports、env。
#导出json、yaml皆可以。（现在只有一个deploy，可以不写名字）
kubectl get deployment mysql5738 -o json > mysql5738.json
#修改完成后，删除重新创建。（deployment的名字不要有 点 不然 dns解析会有报错）
kubectl delete deployment mysql5738
kubectl apply deployment mysql5738 -f mysql5738.json
#查看创建的deploy、rs、pod，只有deploy是自己定义的名字，其他名字都是随机加了后缀！我们创建deploy的时候，本质创建了三种资源，切记，也是创建pod的常用方式。
kubectl get deploy
kubectl get rs
kubectl get pod
#注意：
#pod漂移：此特性会造成单个pod的业务会中断。切记！！！！！
#mysql镜像的数据，没有映射到本地，就会缓存在 镜像的缓存层！此时删除pod，自动创建，但是数据丢失！！！
#节点漂移默认5分钟，期间get 查看都是running的，因为kubelet还没上报到api
#mysql pod docker 内存少于1G会出现，Back-off restarting failed container ，重启异常
#如果节点中有正常启动pod后，那些Back-off 的会变成 Evicted 状态 #这种僵尸pod，本质已经死了。因为资源不够导致的。可以直接删除
为mysql的pod绑定svc
此时svc的名字和deployment 是一样的。注意理解。
kubectl expose deployment mysql5738 --port=3306 --target-port=3306 --protocol=TCP --type=ClusterIP
#查看ClusterIP，为了以后连接mysql，如：
kubectl get svc
搭建nfs部署pv
yum install -y nfs-utils rpcbind
mkdir /root/data && echo "/root/data/ *(insecure,rw,sync,no_root_squash)" > /etc/exports
chown -R nfsnobody.nfsnobody /root/data
systemctl start rpcbind && systemctl start nfs-server && systemctl enable rpcbind && systemctl enable nfs-server
#验证
showmount -e localhost
rpcinfo -p localhost
cd /root/data
mkdir mysqletc mysqldata
chown -R nfsnobody.nfsnobody ./*
#所有k8s节点
yum install -y nfs-utils rpcbind
systemctl start rpcbind
systemctl enable rpcbind
systemctl status rpcbind
#直接使用nfs，挂载到pod中：略 （注意挂载的容器目录，不要是有内容的，会清空的。）
#或者创建pv的方式：优点，可以控制文件大小，传输速度等！
#pvc会根据pv的大小，精准匹配绑定；如何没合适的，找稍微大的pv
如果绑定后单独删除一方，另一方不会释放，等待另一方的创建，后再连接。如果更换双方，需要删除自己的信息（yaml文件中）
#还有自动分配pv的类型StorageClass；pvc找不到合适的pv，就会自动创建合适的pv
#先创建pv 再创建pvc有合适的自动绑定
#kubelet 异常报错，因为kubelet开机会统计cpu等资源信息。
编辑 /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf 文件，并添加下面配置：
CPUAccounting=true
MemoryAccounting=true
附：涉及的yaml文件清单
apiVersion: v1
items:
- apiVersion: extensions/v1beta1
kind: Deployment
metadata:
annotations:
deployment.kubernetes.io/revision: "1"
kubectl.kubernetes.io/last-applied-configuration: |
{"apiVersion":"extensions/v1beta1","kind":"Deployment","metadata":{"annotations":{"deployment.kubernetes.io/revision":"1"},"creationTimestamp":"2022-06-15T21:52:06Z","generation":1,"labels":{"app":"mysql5738"},"name":"mysql5738","namespace":"default","resourceVersion":"224507","selfLink":"/apis/extensions/v1beta1/namespaces/default/deployments/mysql5738","uid":"8e0a6f8e-6e7d-4735-bcb7-13ff5f7d2ded"},"spec":{"progressDeadlineSeconds":600,"replicas":1,"revisionHistoryLimit":10,"selector":{"matchLabels":{"app":"mysql5738"}},"strategy":{"rollingUpdate":{"maxSurge":"25%","maxUnavailable":"25%"},"type":"RollingUpdate"},"template":{"metadata":{"creationTimestamp":null,"labels":{"app":"mysql5738"}},"spec":{"containers":[{"env":[{"name":"MYSQL_ROOT_PASSWORD","value":"123456"}],"image":"c104.cheng.com/cheng/mysql:5.7.38","imagePullPolicy":"IfNotPresent","name":"mysql","ports":[{"containerPort":3306,"protocol":"TCP"}],"resources":{},"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","volumeMounts":[{"mountPath":"/var/lib/mysql","name":"mysqldatapvc"}]}],"dnsPolicy":"ClusterFirst","restartPolicy":"Always","schedulerName":"default-scheduler","securityContext":{},"terminationGracePeriodSeconds":30,"volumes":[{"name":"mysqldatapvc","persistentVolumeClaim":{"claimName":"mysql5738pvc"}}]}}},"status":{"conditions":[{"lastTransitionTime":"2022-06-15T21:52:06Z","lastUpdateTime":"2022-06-15T21:52:06Z","message":"Deployment does not have minimum availability.","reason":"MinimumReplicasUnavailable","status":"False","type":"Available"},{"lastTransitionTime":"2022-06-15T21:52:06Z","lastUpdateTime":"2022-06-15T21:52:06Z","message":"ReplicaSet \"mysql5738-59cd758d7b\" is progressing.","reason":"ReplicaSetUpdated","status":"True","type":"Progressing"}],"observedGeneration":1,"replicas":1,"unavailableReplicas":1,"updatedReplicas":1}}
creationTimestamp: "2022-06-16T11:31:44Z"
generation: 1
labels:
app: mysql5738
name: mysql5738
namespace: default
resourceVersion: "297465"
selfLink: /apis/extensions/v1beta1/namespaces/default/deployments/mysql5738
uid: 1df44519-4ecf-4bcb-8428-42d386b35ccf
spec:
progressDeadlineSeconds: 600
replicas: 1
revisionHistoryLimit: 10
selector:
matchLabels:
app: mysql5738
strategy:
rollingUpdate:
maxSurge: 25%
maxUnavailable: 25%
type: RollingUpdate
template:
metadata:
creationTimestamp: null
labels:
app: mysql5738
spec:
containers:
- env:
- name: MYSQL_ROOT_PASSWORD
value: "123456"
image: c104.cheng.com/cheng/mysql:5.7.38
imagePullPolicy: IfNotPresent
name: mysql
ports:
- containerPort: 3306
protocol: TCP
resources: {}
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /var/lib/mysql
name: mysqldatapvc
dnsPolicy: ClusterFirst
restartPolicy: Always
schedulerName: default-scheduler
securityContext: {}
terminationGracePeriodSeconds: 30
volumes:
- name: mysqldatapvc
persistentVolumeClaim:
claimName: mysql5738pvc
status:
availableReplicas: 1
conditions:
- lastTransitionTime: "2022-06-16T11:31:45Z"
lastUpdateTime: "2022-06-16T11:31:45Z"
message: Deployment has minimum availability.
reason: MinimumReplicasAvailable
status: "True"
type: Available
- lastTransitionTime: "2022-06-16T11:31:44Z"
lastUpdateTime: "2022-06-16T11:31:45Z"
message: ReplicaSet "mysql5738-f64f7f77f" has successfully progressed.
reason: NewReplicaSetAvailable
status: "True"
type: Progressing
observedGeneration: 1
readyReplicas: 1
replicas: 1
updatedReplicas: 1
- apiVersion: v1
kind: PersistentVolume
metadata:
annotations:
kubectl.kubernetes.io/last-applied-configuration: |
{"apiVersion":"v1","kind":"PersistentVolume","metadata":{"annotations":{},"name":"mysql5738pv"},"spec":{"accessModes":["ReadWriteMany"],"capacity":{"storage":"10Gi"},"nfs":{"path":"/root/data/mysqldatapv","server":"192.168.18.104"}}}
pv.kubernetes.io/bound-by-controller: "yes"
creationTimestamp: "2022-06-16T11:17:09Z"
finalizers:
- kubernetes.io/pv-protection
name: mysql5738pv
resourceVersion: "296682"
selfLink: /api/v1/persistentvolumes/mysql5738pv
uid: efb652c2-d353-424c-87e3-73ea3b87af70
spec:
accessModes:
- ReadWriteMany
capacity:
storage: 10Gi
claimRef:
apiVersion: v1
kind: PersistentVolumeClaim
name: mysql5738pvc
namespace: default
resourceVersion: "296337"
uid: 3df9df5b-4f0b-4aeb-ad61-5d495f3ec451
nfs:
path: /root/data/mysqldatapv
server: 192.168.18.104
persistentVolumeReclaimPolicy: Retain
volumeMode: Filesystem
status:
phase: Bound
- apiVersion: v1
kind: PersistentVolumeClaim
metadata:
annotations:
kubectl.kubernetes.io/last-applied-configuration: |
{"apiVersion":"v1","kind":"PersistentVolumeClaim","metadata":{"annotations":{},"name":"mysql5738pvc","namespace":"default"},"spec":{"accessModes":["ReadWriteMany"],"resources":{"requests":{"storage":"200Mi"}}}}
pv.kubernetes.io/bind-completed: "yes"
pv.kubernetes.io/bound-by-controller: "yes"
creationTimestamp: "2022-06-16T11:19:21Z"
finalizers:
- kubernetes.io/pvc-protection
name: mysql5738pvc
namespace: default
resourceVersion: "296684"
selfLink: /api/v1/namespaces/default/persistentvolumeclaims/mysql5738pvc
uid: 3df9df5b-4f0b-4aeb-ad61-5d495f3ec451
spec:
accessModes:
- ReadWriteMany
resources:
requests:
storage: 200Mi
volumeMode: Filesystem
volumeName: mysql5738pv
status:
accessModes:
- ReadWriteMany
capacity:
storage: 10Gi
phase: Bound
- apiVersion: v1
kind: Service
metadata:
creationTimestamp: "2022-06-13T23:52:28Z"
labels:
component: apiserver
provider: kubernetes
name: kubernetes
namespace: default
resourceVersion: "151"
selfLink: /api/v1/namespaces/default/services/kubernetes
uid: 4106c941-92eb-4ee6-9b71-2dd06ca8b473
spec:
clusterIP: 10.96.0.1
ports:
- name: https
port: 443
protocol: TCP
targetPort: 6443
sessionAffinity: None
type: ClusterIP
status:
loadBalancer: {}
- apiVersion: v1
kind: Service
metadata:
creationTimestamp: "2022-06-15T22:26:52Z"
labels:
app: mysql5738
name: mysql5738
namespace: default
resourceVersion: "227501"
selfLink: /api/v1/namespaces/default/services/mysql5738
uid: 9d63ff56-31cb-449c-9954-120ea93181e8
spec:
clusterIP: 10.102.125.157
ports:
- port: 3306
protocol: TCP
targetPort: 3306
selector:
app: mysql5738
sessionAffinity: None
type: ClusterIP
status:
loadBalancer: {}
kind: List
metadata:
resourceVersion: ""
selfLink: ""
WEB程序安装
docker安装web程序
测试系统正常：创建数据库，拉取镜像，启动
CREATE DATABASE `djangoblog` /*!40100 DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci */;
docker pull liangliangyy/djangoblog:latest
docker run -d -p 80:8000 -e DJANGO_MYSQL_HOST=10.102.125.157 -e DJANGO_MYSQL_PASSWORD=123456 -e DJANGO_MYSQL_USER=root -e DJANGO_MYSQL_DATABASE=djangoblog --name djangoblog liangliangyy/djangoblog:latest
#创建web网站用户
docker exec -it djangoblog python /code/djangoblog/manage.py createsuperuser
#验证
#http://192.168.18.101/
改web的docker为k8s部署
#测试可以正常访问，改用k8s部署。
docker stop acb44d26573e
docker rm acb44d26573e
#首先创建deployment模板，然后删除它，修改yaml文件，增加启动参数;这样可以快速得到yaml文件的参数！！！
kubectl create deployment djangoblog --image=c104.cheng.com/cheng/djangoblog:v1.1.1
kubectl get deploy djangoblog -o yaml >djangoblog111.yaml
kubectl delete -f djangoblog111.yaml
#修改env和ports参数后，重新创建deployment
kubectl apply -f djangoblog111.yaml
#为pod绑定，此时svc的名字和deployment
kubectl expose deployment djangoblog --target-port=8000 --protocol=TCP --type=NodePort
##查看绑定情况：
kubectl get svc -o wide
##扩容到3个pod，创建负载均衡的节点
kubectl scale deployment djangoblog --replicas=3
附：涉及的yaml文件
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
annotations:
deployment.kubernetes.io/revision: "1"
creationTimestamp: "2022-06-16T21:09:32Z"
generation: 1
labels:
app: djangoblog
name: djangoblog
namespace: default
resourceVersion: "347847"
selfLink: /apis/extensions/v1beta1/namespaces/default/deployments/djangoblog
uid: bcdbd9ef-2bc5-4828-9f21-fae8e80ca503
spec:
progressDeadlineSeconds: 600
replicas: 1
revisionHistoryLimit: 10
selector:
matchLabels:
app: djangoblog
strategy:
rollingUpdate:
maxSurge: 25%
maxUnavailable: 25%
type: RollingUpdate
template:
metadata:
creationTimestamp: null
labels:
app: djangoblog
spec:
containers:
- env:
- name: DJANGO_MYSQL_HOST
value: "10.102.125.157"
- name: DJANGO_MYSQL_PASSWORD
value: "123456"
- name: DJANGO_MYSQL_USER
value: root
- name: DJANGO_MYSQL_DATABASE
value: djangoblog
image: c104.cheng.com/cheng/djangoblog:v1.1.1
imagePullPolicy: IfNotPresent
name: djangoblog
ports:
- containerPort: 8000
protocol: TCP
resources: {}
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
dnsPolicy: ClusterFirst
restartPolicy: Always
schedulerName: default-scheduler
securityContext: {}
terminationGracePeriodSeconds: 30
status:
conditions:
- lastTransitionTime: "2022-06-16T21:09:32Z"
lastUpdateTime: "2022-06-16T21:10:12Z"
message: ReplicaSet "djangoblog-76bb6b66f4" has successfully progressed.
reason: NewReplicaSetAvailable
status: "True"
type: Progressing
- lastTransitionTime: "2022-06-16T21:10:35Z"
lastUpdateTime: "2022-06-16T21:10:35Z"
message: Deployment does not have minimum availability.
reason: MinimumReplicasUnavailable
status: "False"
type: Available
observedGeneration: 1
replicas: 1
unavailableReplicas: 1
updatedReplicas: 1
安装keepalive
实现 用户访问高可用，安装过程比较简单：略。
主节点配置文件
! Configuration File for keepalived
global_defs {
notification_email {
acassen@firewall.loc
failover@firewall.loc
sysadmin@firewall.loc
}
notification_email_from Alexandre.Cassen@firewall.loc
smtp_server 192.168.200.1
smtp_connect_timeout 30
router_id LVS_DEVEL01
}
vrrp_instance VI_1 {
state MASTER
interface ens33
virtual_router_id 51
priority 100
advert_int 1
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
192.168.18.10/24
}
}
从节点配置文件
! Configuration File for keepalived
global_defs {
notification_email {
acassen@firewall.loc
failover@firewall.loc
sysadmin@firewall.loc
}
notification_email_from Alexandre.Cassen@firewall.loc
smtp_server 192.168.200.1
smtp_connect_timeout 30
router_id LVS_DEVEL02
}
vrrp_instance VI_1 {
state BACKUP
interface ens33
virtual_router_id 51
priority 99
advert_int 1
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
192.168.18.10/24
}
}
拓展：
安装nginx做端口转发，用户访问80端口即可；增加ssl访问，加密数据。
安装证书：
1014 openssl genrsa -des3 -out ca.key 2048
1015 openssl req -new -key ca.key -out server.csr
1017 mv ca.key ca.key.org
1018 openssl rsa -in ca.key.org -out ca.key
1019 openssl x509 -req -days 36500 -in server.csr -signkey ca.key -out server.crt
1020 ll
1021 chmod -R 777 ./*
nginx配置文件：
server {
listen 443 ssl http2;
listen [::]:443 ssl http2;
server_name _;
root /usr/share/nginx/html;
location / {
proxy_pass http://10.100.207.173:8000; # 转发规则
proxy_set_header Host $proxy_host; # 修改转发请求头，让8080端口的应用可以受到真实的请求
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
ssl on;
ssl_certificate "/root/ssl/server.crt";
ssl_certificate_key "/root/ssl/ca.key";
ssl_session_cache shared:SSL:1m;
ssl_session_timeout 10m;
ssl_ciphers HIGH:!aNULL:!MD5;
ssl_prefer_server_ciphers on;
include /etc/nginx/default.d/*.conf;
error_page 404 /404.html;
location = /40x.html {
}
error_page 500 502 503 504 /50x.html;
location = /50x.html {
}
}
登陆web操作页面
通过keepalived的vip访问web节点，查看日志输出三个都有日志输出，并且无页面报错，集群搭建完成。
4906
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
