k8s之nfs挂载,使用一个节点进行存储

环境：
master1 192.168.0.145
node1 192.168.0.146
node2 192.168.0.147
现要求所有不同节点的pod,文件统一上传至node1节点上的文件里，日志也同样放在node1上

步骤如下：

配置NFS服务器在node1上

# 安装 
yum -y install nfs-utils
# 创建或使用用已有的文件夹作为nfs文件存储点
vi /etc/exports
(写入如下内容)
/mnt *(rw,no_root_squash,sync)
# 配置生效并查看生效
exportfs -r
exportfs
(显示如下：)
/mnt <world>
# 启动rpcbind、nfs服务
systemctl restart rpcbind && systemctl enable rpcbind
systemctl restart nfs-server && systemctl enable nfs-server

查看 RPC 服务的注册状况

rpcinfo -p localhost

# 显示如下 
program vers proto   port  service
100000    4   tcp    111  portmapper
100000    3   tcp    111  portmapper
.....

showmount测试

showmount -e 192.168.0.146

# 显示如下
Export list for 192.168.0.146:
/mnt *

问题排查，如果遇到showmount -e 192.168.0.146失败，使用如下命令

# 先停用
systemctl stop rpcbind
systemctl stop nfs
# 再重启
systemctl start rpcbind
systemctl start nfs

在master和其它节点都安装nfs-utils,命令如下

yum -y install nfs-utils

在 master 节点服务器上新建deerchain-ci-cd-pv.yaml 用于创建 pv 和 pvc 的 yaml 文件,内容如下,多个文件挂载我声明多个pvc,如下：

apiVersion: v1
kind: PersistentVolume
metadata:
  name: deerchain-log-pv
  namespace: ms
  labels:
    pv: deerchain-log-pv
spec:
  capacity:
    storage: 10000Gi
  accessModes:
    - ReadWriteMany
  nfs:  
    server: 192.168.0.147
    path: "/mnt/log/"   #NFS目录，需要该目录在NFS上存在，不存在需要先去创建目录
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: deerchain-log-pvc
  namespace: ms
spec:
  accessModes:
    - ReadWriteMany
  resources:
    requests:
      storage: 10000Gi  #容量

---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: deerchain-upload-pv
  namespace: ms
  labels:
    pv: deerchain-upload-pv
spec:
  capacity:
    storage: 10000Gi
  accessModes:
    - ReadWriteMany
  nfs:  
    server: 192.168.0.147
    path: "/mnt/lulianFiles/"   #NFS目录，需要该目录在NFS上存在，不存在需要先去创建目录
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: deerchain-upload-pvc
  namespace: ms
spec:
  accessModes:
    - ReadWriteMany
  resources:
    requests:
      storage: 10000Gi  #容量

启动命令

kubectl apply -f deerchain-ci-cd-pv.yaml -n pro

# 查看服务
kubectl get pv -n pro
kubectl get pvc -n pro

helm values.yaml文件

# Default values for discovery.
# This is a YAML-formatted file.
# Declare variables to be passed into your templates.

replicaCount: 1

namespace: ms

imagePullSecrets: harbor-pull-secret-188

envFrom: jvm-conf-map

volumes:
  log:
    name: nfs-log-file
    pvcName: deerchain-log-pvc
    path: /mnt/log
  upload:
    name: nfs-upload-file
    pvcName: deerchain-upload-pvc
    path: /mnt/lulianFiles

# IfNotPresent Always
image:
  tag: latest
  env: deerchain-develop
  harborPath: harbor.wuliuhub.com:8443
  pullPolicy: Always
  gateway:
    name: deerchain-gateway-9527
    port: 9527
  platform:
    name: deerchain-platform-23000
    port: 23000
  allinpay:
    name: deerchain-allinpay-api-2001
    port: 21000
  appMember:
    name: deerchain-app-member-21000
    port: 21000
  attachment:
    name: deerchain-attachment-3001
    port: 3001
  contract:
    name: deerchain-contract-10001
    port: 10001
  enterprise:
    name: deerchain-enterprise-6001
    port: 6001
  webEnterprise:
    name: deerchain-enterprise-24000
    port: 24000
  goods:
    name: deerchain-goods-9001
    port: 9001
  invoice:
    name: deerchain-invoice-7001
    port: 7001
  manager:
    name: deerchain-manager-13001
    port: 13001
  member:
    name: deerchain-member-5001
    port: 5001
  orders:
    name: deerchain-orders-11001
    port: 11001
  panel:
    name: deerchain-panel-28000
    port: 28000
  settlement:
    name: deerchain-settlement-8001
    port: 8001
  system:
    name: deerchain-system-4001
    port: 4001
  taxation:
    name: deerchain-taxation-25000
    port: 25000
  timeTask:
    name: deerchain-time-task-27000
    port: 27000
  toOutApi:
    name: deerchain-to-out-api-12001
    port: 12001
  changeData:
    name: deerchain-change-data-22000
    port: 22000
  messagePush:
    name: deerchain-message-push-26000
    port: 26000


service:
  type: NodePort
  port: 1111
  nodePort: 30222

ingress:
  enabled: false
  annotations:
    kubernetes.io/ingress.class: nginx
  path: /
  hosts: gateway.k8s.com
  tls: []
  #  - secretName: chart-example-tls
  #    hosts:
  #      - chart-example.local

resources: {}
  # We usually recommend not to specify default resources and to leave this as a conscious
  # choice for the user. This also increases chances charts run on environments with little
  # resources, such as Minikube. If you do want to specify resources, uncomment the following
  # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
  # limits:
  #  cpu: 100m
  #  memory: 128Mi
  # requests:
  #  cpu: 100m
#  memory: 128Mi

nodeSelector: {}

tolerations: []

affinity: {}

使用方法，在k8s yaml文件里配置如下内容：

apiVersion: apps/v1
kind: Deployment
metadata:
  name: {{ .Values.image.system.name }}
  namespace: {{ .Values.namespace }}
spec:
  replicas: {{ .Values.replicaCount }}
  selector:
    matchLabels:
      project: {{ .Values.namespace }}
      app: {{ .Values.image.system.name }}

  template:
    metadata:
      labels:
        project: {{ .Values.namespace }}
        app: {{ .Values.image.system.name }}
    spec:
      imagePullSecrets:
        - name: {{ .Values.imagePullSecrets }}
      containers:
        - name: {{ .Values.image.system.name }}
          image: "{{ .Values.image.harborPath }}/{{ .Values.image.env }}/{{ .Values.image.system.name }}:{{ .Values.image.tag }}"
          imagePullPolicy: {{ .Values.image.pullPolicy }}
          ports:
            - containerPort: {{ .Values.image.system.port }}
              protocol: TCP
          envFrom:
            - configMapRef:
                name: {{ .Values.envFrom }}
          volumeMounts:
            - name: nfs-log-file
              mountPath: /mnt/log
            - name: nfs-upload-file
              mountPath: /mnt/lulianFiles
      volumes:
        - name: {{ .Values.volumes.log.name }}
          persistentVolumeClaim:
            claimName: {{ .Values.volumes.log.pvcName }}
        - name: {{ .Values.volumes.upload.name }}
          persistentVolumeClaim:
            claimName: {{ .Values.volumes.upload.pvcName }}