K8S---外部NFS存储动态配置及使用

nfs存储服务器配置

1、创建共享目录和权限设置

[root@k8s-node1 data]#  mkdir -p /data/k8sdata
[root@k8s-node1 data]#  chown -R 755 /data/k8s/
[root@k8s-node1 data]#  ll /data/
总用量 206744
drwxr-xr-x. 5 root root        49 5月  10 16:16 bao
-rw-r--r--  1 root root 211696640 5月  24 15:23 centos.tar
drwxr-xr-x  2 root root         6 6月   6 10:36 k8sdata
-rw-r--r--. 1 root root      5750 5月  10 17:46 kube-flannel.yml

2、通过yum进行nfs安装

[root@k8s-master ~]#yum -y install nfs-utils rpcbind

3、配置 nfs，nfs 的默认配置文件在 /etc/exports

[root@k8s-master ~]# vim /etc/exports 
/data/k8sdata  *(rw,sync,no_root_squash)

4、启动服务，启动顺序，先启动rpc，在启动nfs

[root@k8s-node1 data]# systemctl enable rpcbind
[root@k8s-node1 data]# systemctl start rpcbind
[root@k8s-node1 data]# systemctl status rpcbind

[root@k8s-node1 data]# systemctl enable nfs
[root@k8s-node1 data]# systemctl start  nfs
[root@k8s-node1 data]# systemctl status  nfs

5、查看相关信息

查看进程信息
[root@k8s-node1 k8sdata]# rpcinfo -p|grep nfs
    100003    3   tcp   2049  nfs
    100003    4   tcp   2049  nfs
    100227    3   tcp   2049  nfs_acl
    100003    3   udp   2049  nfs
    100003    4   udp   2049  nfs
    100227    3   udp   2049  nfs_acl

查看挂载信息
[root@k8s-node1 k8sdata]# cat /var/lib/nfs/etab 
/data/k8sdata	*(rw,sync,wdelay,hide,nocrossmnt,secure,no_root_squash,no_all_squash,no_subtree_check,secure_locks,acl,no_pnfs,anonuid=65534,anongid=65534,sec=sys,secure,no_root_squash,no_all_squash)

查看可以登录的nfs地址
[root@k8s-node1 k8sdata]# showmount -e 192.168.73.133
Export list for 192.168.73.133:
/data/k8sdata *

nfs存储动态配置及使用

官方地址
https://github.com/kubernetes-sigs/nfs-subdir-external-provisioner

授权 RBAC

# rbac.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
  name: nfs-client-provisioner
  # replace with namespace where provisioner is deployed
  namespace: default
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: nfs-client-provisioner-runner
rules:
  - apiGroups: [""]
    resources: ["nodes"]
    verbs: ["get", "list", "watch"]
  - apiGroups: [""]
    resources: ["persistentvolumes"]
    verbs: ["get", "list", "watch", "create", "delete"]
  - apiGroups: [""]
    resources: ["persistentvolumeclaims"]
    verbs: ["get", "list", "watch", "update"]
  - apiGroups: ["storage.k8s.io"]
    resources: ["storageclasses"]
    verbs: ["get", "list", "watch"]
  - apiGroups: [""]
    resources: ["events"]
    verbs: ["create", "update", "patch"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: run-nfs-client-provisioner
subjects:
  - kind: ServiceAccount
    name: nfs-client-provisioner
    # replace with namespace where provisioner is deployed
    namespace: default
roleRef:
  kind: ClusterRole
  name: nfs-client-provisioner-runner
  apiGroup: rbac.authorization.k8s.io
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: leader-locking-nfs-client-provisioner
  # replace with namespace where provisioner is deployed
  namespace: default
rules:
  - apiGroups: [""]
    resources: ["endpoints"]
    verbs: ["get", "list", "watch", "create", "update", "patch"]
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: leader-locking-nfs-client-provisioner
  # replace with namespace where provisioner is deployed
  namespace: default
subjects:
  - kind: ServiceAccount
    name: nfs-client-provisioner
    # replace with namespace where provisioner is deployed
    namespace: default
roleRef:
  kind: Role
  name: leader-locking-nfs-client-provisioner
  apiGroup: rbac.authorization.k8s.io

# 创建
# kubectl create -f rbac.yaml

创建部署

# deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: nfs-client-provisioner
  labels:
    app: nfs-client-provisioner
  # replace with namespace where provisioner is deployed
  namespace: default
spec:
  replicas: 2
  strategy:
    type: Recreate
  selector:
    matchLabels:
      app: nfs-client-provisioner
  template:
    metadata:
      labels:
        app: nfs-client-provisioner
    spec:
      serviceAccountName: nfs-client-provisioner
      containers:
        - name: nfs-client-provisioner
          image: k8s.gcr.io/sig-storage/nfs-subdir-external-provisioner:v4.0.2
          volumeMounts:
            - name: nfs-client-root
              mountPath: /persistentvolumes
          env:
            - name: PROVISIONER_NAME
              value: k8s-sigs.io/nfs-subdir-external-provisioner
            - name: NFS_SERVER
              value: 192.168.73.133
            - name: NFS_PATH
              value: /data/k8sdata
      volumes:
        - name: nfs-client-root
          nfs:
            server: 192.168.73.133
            path: /data/k8sdata

# 创建
# kubectl create -f deployment.yaml

创建 StorageClass

# class.yaml
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: nfs-client
provisioner: k8s-sigs.io/nfs-subdir-external-provisioner # or choose another name, must match deployment's env PROVISIONER_NAME'
parameters:
  archiveOnDelete: "false"

# 创建
# kubectl create -f class.yaml

创建 PVC

# test-claim.yaml
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
  name: test-claim
spec:
  storageClassName: nfs-client
  accessModes:
    - ReadWriteMany
  resources:
    requests:
      storage: 10Mi

# 创建
# kubectl create -f test-claim.yaml

使用 PVC

# test-pod.yaml
kind: Pod
apiVersion: v1
metadata:
  name: test-pod
spec:
  containers:
  - name: test-pod
    image: busybox:latest
    imagePullPolicy: IfNotPresent
    command:
      - "/bin/sh"
    args:
      - "-c"
      - "touch /mnt/SUCCESS && exit 0 || exit 1"  # 在存储中创建文件
    volumeMounts:
      - name: nfs-pvc
        mountPath: "/mnt"
  restartPolicy: "Never"
  volumes:
    - name: nfs-pvc
      persistentVolumeClaim:
        claimName: test-claim    # 使用 pvc 

# 创建
# kubectl create -f test-pod.yaml

查看

# 查看pod
[root@k8s-master1 nfs-subdir-external]# kubectl get pods 
NAME                                      READY   STATUS                   RESTARTS          AGE
nfs-client-provisioner-6958d4656b-8cqfg   1/1     Running                  0                 10m
nfs-client-provisioner-6958d4656b-vdn9t   1/1     Running                  0                 10m
test-pod                                  0/1     Completed                0                 65s

# 查看pv
[root@k8s-master1 nfs-subdir-external]# kubectl get pv
NAME                                       CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS   CLAIM                STORAGECLASS   REASON   AGE
pvc-04ae6acf-9666-4a7d-9d8d-0d1f8c3bdd4d   10Mi       RWX            Delete           Bound    default/test-claim   nfs-client              7m32s

# 查看pvc
[root@k8s-master1 nfs-subdir-external]# kubectl get pvc
NAME         STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS   AGE
test-claim   Bound    pvc-04ae6acf-9666-4a7d-9d8d-0d1f8c3bdd4d   10Mi       RWX            nfs-client     7m41s

# 查看创建的文件 SUCCESS
[root@k8s-node1 ~]# ll /data/k8sdata/
总用量 0
drwxrwxrwx 2 root root 21 6月   8 16:52 default-test-claim-pvc-04ae6acf-9666-4a7d-9d8d-0d1f8c3bdd4d
[root@k8s-node1 ~]# 
[root@k8s-node1 ~]# ll /data/k8sdata/default-test-claim-pvc-04ae6acf-9666-4a7d-9d8d-0d1f8c3bdd4d/
总用量 0
-rw-r--r-- 1 root root 0 6月   8 16:52 SUCCESS
[root@k8s-node1 ~]#