简介
K8S中容器的数据的持久化对于有状态的应用是很重要的,考虑到横向扩展,所以一般都会使用网络存储来作为持久化的介质。NFS(有的叫NAS)就是经常使用的一种。本文介绍两种使用NFS作为后端存储,动态创建PVC的方法。两种方法我在产线环境都验证过。
第一种: nfs-client-provisioner (较老)
第二种: csi-nfs-driver (较新)
准备
Kubernetes : 1.17.6
实施
nfs-client-provisioner
1. 部署nfs-client-provisioner , 文件里面的nfs的服务器地址和路径换成自己的
kubectl apply -f nfs-storageclass-allInOne.yaml
kind: ServiceAccount
apiVersion: v1
metadata:
name: nfs-client-provisioner
namespace: kube-system
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: nfs-client-provisioner-runner
rules:
- apiGroups: [""]
resources: ["persistentvolumes"]
verbs: ["get", "list", "watch", "create", "delete"]
- apiGroups: [""]
resources: ["persistentvolumeclaims"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: ["storage.k8s.io"]
resources: ["storageclasses"]
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources: ["events"]
verbs: ["create", "update", "patch"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: run-nfs-client-provisioner
subjects:
- kind: ServiceAccount
name: nfs-client-provisioner
namespace: kube-system
roleRef:
kind: ClusterRole
name: nfs-client-provisioner-runner
apiGroup: rbac.authorization.k8s.io
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
namespace: kube-system
name: leader-locking-nfs-client-provisioner
rules:
- apiGroups: [""]
resources: ["endpoints"]
verbs: ["get", "list", "watch", "create", "update", "patch"]
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
namespace: kube-system
name: leader-locking-nfs-client-provisioner
subjects:
- kind: ServiceAccount
name: nfs-client-provisioner
# replace with namespace where provisioner is deployed
namespace: kube-system
roleRef:
kind: Role
name: leader-locking-nfs-client-provisioner
apiGroup: rbac.authorization.k8s.io
---
kind: Deployment
apiVersion: apps/v1
metadata:
name: nfs-client-provisioner
namespace: kube-system
spec:
replicas: 2
strategy:
type: Recreate
selector:
matchLabels:
app: nfs-client-provisioner
template:
metadata:
labels:
app: nfs-client-provisioner
spec:
serviceAccountName: nfs-client-provisioner
affinity:
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
podAffinityTerm:
topologyKey: kubernetes.io/hostname
labelSelector:
matchLabels:
app: nfs-client-provisioner
# nodeAffinity:
# requiredDuringSchedulingIgnoredDuringExecution:
# nodeSelectorTerms:
# - matchExpressions:
# - key: dedicated
# operator: In
# values:
# - "cmp"
containers:
- name: nfs-client-provisioner
image: quay.io/external_storage/nfs-client-provisioner:v3.1.0-k8s1.11
volumeMounts:
- name: nfs-client-root
mountPath: /persistentvolumes
env:
- name: PROVISIONER_NAME
value: nfs-client-provisioner
- name: NFS_SERVER
value: 10.159.1.105 #nfs服务器的地址
- name: NFS_PATH
value: /data/nfs_basedir/ #nfs的export 的目录
volumes:
- name: nfs-client-root
nfs:
server: 10.159.1.105 #nfs服务器的地址
path: /data/nfs_basedir/ #nfs的export 的目录
---
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: nfs
annotations:
storageclass.kubernetes.io/is-default-class: true #是否是默认的SC
provisioner: nfs-client-provisioner
reclaimPolicy: Delete #有两种选择, Delete是PV删除以后,文件也从NFS里面删除;Retain是保留文件
mountOptions:
- noresvport
- vers=3
parameters:
archiveOnDelete: "false" #删除的时候是否归档,如果是true就是软删除,只是会重命名pv的文件目录,这样会造成很多垃圾数据
2. 使用新创建的StorageClass,动态创建PVC
kubectl apply -f test-nfs-sc.yaml
apiVersion: v1
kind: Pod
metadata:
name: nginx
spec:
containers:
- name: nginx
image: nginx:latest
ports:
- containerPort: 80
volumeMounts:
- name: www
mountPath: /usr/share/nginx/html
volumes:
- name: www
persistentVolumeClaim:
claimName: nginx
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: nginx
spec:
storageClassName: "nfs"
accessModes:
- ReadWriteMany
resources:
requests:
storage: 5Gi
3. 验证PVC是否创建成功。进入到容器中,创建文件,然后把nfs挂载到本地,看看文件是否存在
kubectl get pvc
[root@cmp-k8s-prd-10-128-148-230 nfs-ksyun]# kubectl get pvc
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE
nginx Bound pvc-b58182db-a9ca-434b-88ae-6d28c9795242 5Gi RWX nfs 2s
csi-nfs-driver
1. 部署csi-nfs-driver
kubectl apply -f rbac-csi-nfs-controller.yaml
rbac-csi-nfs-controller.yaml
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: csi-nfs-controller-sa
namespace: kube-system
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: nfs-external-provisioner-role
rules:
- apiGroups: [""]
resources: ["persistentvolumes"]
verbs: ["get", "list", "watch", "create", "delete"]
- apiGroups: [""]
resources: ["persistentvolumeclaims"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: ["storage.k8s.io"]
resources: ["storageclasses"]
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources: ["events"]
verbs: ["get", "list", "watch", "create", "update", "patch"]
- apiGroups: ["storage.k8s.io"]
resources: ["csinodes"]
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources: ["nodes"]
verbs: ["get", "list", "watch"]
- apiGroups: ["coordination.k8s.io"]
resources: ["leases"]
verbs: ["get", "list", "watch", "create", "update", "patch"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: nfs-csi-provisioner-binding
subjects:
- kind: ServiceAccount
name: csi-nfs-controller-sa
namespace: kube-system
roleRef:
kind: ClusterRole
name: nfs-external-provisioner-role
apiGroup: rbac.authorization.k8s.io
kubectl apply -f csi-nfs-driverinfo.yaml
kubectl apply -f csi-nfs-node.yaml
kubectl apply -f csi-nfs-controller.yaml
kubectl apply -f storageclass-nfs.yaml
csi-nfs-driverinfo.yaml
---
apiVersion: storage.k8s.io/v1beta1
kind: CSIDriver
metadata:
name: nfs.csi.k8s.io
spec:
attachRequired: false
volumeLifecycleModes:
- Persistent
podInfoOnMount: true
csi-nfs-node.yaml
---
# This YAML file contains driver-registrar & csi driver nodeplugin API objects
# that are necessary to run CSI nodeplugin for nfs
kind: DaemonSet
apiVersion: apps/v1
metadata:
name: csi-nfs-node
namespace: kube-system
spec:
selector:
matchLabels:
app: csi-nfs-node
template:
metadata:
labels:
app: csi-nfs-node
spec:
hostNetwork: true # original nfs connection would be broken without hostNetwork setting
dnsPolicy: ClusterFirstWithHostNet
containers:
- name: liveness-probe
image: registry.cn-hangzhou.aliyuncs.com/imagesfromgoogle/livenessprobe:v2.1.0
args:
- --csi-address=/csi/csi.sock
- --probe-timeout=3s
- --health-port=29653
- --v=5
volumeMounts:
- name: socket-dir
mountPath: /csi
resources:
limits:
cpu: 100m
memory: 100Mi
requests:
cpu: 10m
memory: 20Mi
- name: node-driver-registrar
image: registry.cn-hangzhou.aliyuncs.com/imagesfromgoogle/csi-node-driver-registrar:v2.0.1
lifecycle:
preStop:
exec:
command: ["/bin/sh", "-c", "rm -rf /registration/csi-nfsplugin /registration/csi-nfsplugin-reg.sock"]
args:
- --v=5
- --csi-address=/csi/csi.sock
- --kubelet-registration-path=/var/lib/kubelet/plugins/csi-nfsplugin/csi.sock
env:
- name: KUBE_NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
volumeMounts:
- name: socket-dir
mountPath: /csi
- name: registration-dir
mountPath: /registration
- name: nfs
securityContext:
privileged: true
capabilities:
add: ["SYS_ADMIN"]
allowPrivilegeEscalation: true
image: registry.cn-hangzhou.aliyuncs.com/imagesfromgoogle/nfsplugin:amd64-linux-canary
args:
- "-v=5"
- "--nodeid=$(NODE_ID)"
- "--endpoint=$(CSI_ENDPOINT)"
env:
- name: NODE_ID
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: CSI_ENDPOINT
value: unix:///csi/csi.sock
imagePullPolicy: "IfNotPresent"
volumeMounts:
- name: socket-dir
mountPath: /csi
- name: pods-mount-dir
mountPath: /var/lib/kubelet/pods
mountPropagation: "Bidirectional"
volumes:
- name: socket-dir
hostPath:
path: /var/lib/kubelet/plugins/csi-nfsplugin
type: DirectoryOrCreate
- name: pods-mount-dir
hostPath:
path: /var/lib/kubelet/pods
type: Directory
- hostPath:
path: /var/lib/kubelet/plugins_registry
type: Directory
name: registration-dir
csi-nfs-controller.yaml
---
kind: Deployment
apiVersion: apps/v1
metadata:
name: csi-nfs-controller
namespace: kube-system
spec:
replicas: 2
selector:
matchLabels:
app: csi-nfs-controller
template:
metadata:
labels:
app: csi-nfs-controller
spec:
serviceAccountName: csi-nfs-controller-sa
nodeSelector:
kubernetes.io/os: linux
priorityClassName: system-cluster-critical
tolerations:
- key: "node-role.kubernetes.io/master"
operator: "Equal"
value: "true"
effect: "NoSchedule"
containers:
- name: csi-provisioner
image: registry.cn-hangzhou.aliyuncs.com/imagesfromgoogle/csi-provisioner:v2.0.4
args:
- "-v=5"
- "--csi-address=$(ADDRESS)"
- "--leader-election"
env:
- name: ADDRESS
value: /csi/csi.sock
volumeMounts:
- mountPath: /csi
name: socket-dir
resources:
limits:
cpu: 100m
memory: 100Mi
requests:
cpu: 10m
memory: 20Mi
- name: liveness-probe
image: registry.cn-hangzhou.aliyuncs.com/imagesfromgoogle/livenessprobe:v2.1.0
args:
- --csi-address=/csi/csi.sock
- --probe-timeout=3s
- --health-port=29652
- --v=5
volumeMounts:
- name: socket-dir
mountPath: /csi
resources:
limits:
cpu: 100m
memory: 100Mi
requests:
cpu: 10m
memory: 20Mi
- name: nfs
image: registry.cn-hangzhou.aliyuncs.com/imagesfromgoogle/nfsplugin:amd64-linux-canary
securityContext:
privileged: true
capabilities:
add: ["SYS_ADMIN"]
allowPrivilegeEscalation: true
imagePullPolicy: IfNotPresent
args:
- "-v=5"
- "--nodeid=$(NODE_ID)"
- "--endpoint=$(CSI_ENDPOINT)"
env:
- name: NODE_ID
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: CSI_ENDPOINT
value: unix:///csi/csi.sock
volumeMounts:
- name: pods-mount-dir
mountPath: /var/lib/kubelet/pods
mountPropagation: "Bidirectional"
- mountPath: /csi
name: socket-dir
resources:
limits:
cpu: 200m
memory: 200Mi
requests:
cpu: 10m
memory: 20Mi
volumes:
- name: pods-mount-dir
hostPath:
path: /var/lib/kubelet/pods
type: Directory
- name: socket-dir
emptyDir: {}
storageclass-nfs.yaml ,修改以下的nfs的地址和目录
---
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: nfs-csi
provisioner: nfs.csi.k8s.io
parameters:
server: 10.159.1.105 #修改为自己的nfs的服务器地址
share: /data/nfs_basedir/ #修改为nfs的目录
reclaimPolicy: Retain # only retain is supported,目前这个回收策略只支持Retain
volumeBindingMode: Immediate
mountOptions:
- vers=3
- noresvport
2. 查看部署的pod是否都启动了
kubectl get po -n kube-system|grep csi-nfs
root@workstation:/data/nfs-csi# kubectl get po -n kube-system|grep csi-nfs
csi-nfs-controller-59697d8b6-lm5v8 3/3 Running 0 5h35m
csi-nfs-controller-59697d8b6-pbv5k 3/3 Running 0 5h35m
csi-nfs-node-5zvfh 3/3 Running 0 5h35m
csi-nfs-node-8qcw6 3/3 Running 0 5h35m
csi-nfs-node-sjf8m 3/3 Running 0 5h35m
csi-nfs-node-tkq4w 3/3 Running 0 5h35m
csi-nfs-node-xls9l 3/3 Running 3 5h35m
3. 验证新的SC已经创建成功
kubectl get sc
root@workstation:/data/rke/mercury_k8s/nfs-storage-class/nfs-csi# kubectl get sc
NAME PROVISIONER RECLAIMPOLICY VOLUMEBINDINGMODE ALLOWVOLUMEEXPANSION AGE
nfs-csi nfs.csi.k8s.io Retain Immediate false 154m
4. 使用创建的SC,动态创建PVC
kubectl apply -f test-csi-nfs-driver.yaml
---
apiVersion: v1
kind: Service
metadata:
name: nginx
labels:
app: nginx
spec:
ports:
- port: 80
name: web
clusterIP: None
selector:
app: nginx
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: web
spec:
selector:
matchLabels:
app: nginx
serviceName: "nginx"
replicas: 1
template:
metadata:
labels:
app: nginx
spec:
terminationGracePeriodSeconds: 10
containers:
- name: nginx
image: nginx
ports:
- containerPort: 80
name: web
volumeMounts:
- name: www
mountPath: /usr/share/nginx/html
volumeClaimTemplates:
- metadata:
name: www
spec:
accessModes: [ "ReadWriteOnce" ]
storageClassName: "nfs-csi"
resources:
requests:
storage: 1Gi
5.查看新创建的PVC,可以登录容器创建文件,然后把nfs挂在本地查看文件。这个操作就不演示了。
root@workstation:/data/nfs-csi# kubectl get pvc
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE
www-web-0 Bound pvc-9c6e135e-8df4-4d50-aad2-ed8e054f5c12 1Gi RWO nfs-csi 2m52s