k8s nfs-csi实践

本文实践k8s csi中的nfs插件安装及使用流程，k8s环境如下

root@master:~# kubectl get node -o wide
NAME     STATUS     ROLES           AGE    VERSION   INTERNAL-IP    EXTERNAL-IP   OS-IMAGE       KERNEL-VERSION      CONTAINER-RUNTIME
master   Ready      control-plane   250d   v1.24.1   192.168.56.2   <none>        Ubuntu 21.04   5.11.0-49-generic   containerd://1.5.11
node1    Ready      <none>          99d    v1.24.1   192.168.56.3   <none>        Ubuntu 21.04   5.11.0-49-generic   containerd://1.5.11

安装nfs-server

在master上安装nfs

apt-get install nfs-kernel-server

设置nfs目录

mkdir -p /srv/nfs
chown nobody:nogroup /srv/nfs
chmod 0777 /srv/nfs

添加管理ip到配置文件/etc/exports中

/srv/nfs 192.168.56.0/24(rw,sync,no_subtree_check)

重启nfs

systemctl restart nfs-kernel-server

node1上挂载nfs文件系统，使node1和master可访问相同的nfs目录

mkdir -p /srv/nfs
mount -t nfs 192.168.56.2:/srv/nfs /srv/nfs

安装csi nfs driver

下载csi-nfs代码，执行安装脚本

git clone https://github.com/kubernetes-csi/csi-driver-nfs.git
cd csi-driver-nfs
root@master:~/csi-driver-nfs# ./deploy/install-driver.sh
Installing NFS CSI driver, version: master ...
serviceaccount/csi-nfs-controller-sa created
serviceaccount/csi-nfs-node-sa created
clusterrole.rbac.authorization.k8s.io/nfs-external-provisioner-role created
clusterrolebinding.rbac.authorization.k8s.io/nfs-csi-provisioner-binding created
csidriver.storage.k8s.io/nfs.csi.k8s.io created
deployment.apps/csi-nfs-controller created
daemonset.apps/csi-nfs-node created
NFS CSI driver installed successfully.

安装成功后，可看到多了deployment和daemonsets

root@master:~/csi-driver-nfs# kubectl get deployment -A
NAMESPACE     NAME                  READY   UP-TO-DATE   AVAILABLE   AGE
kube-system   csi-nfs-controller    1/1     1            1           36s
...

root@master:~/csi-driver-nfs# kubectl get daemonsets -A
NAMESPACE     NAME           DESIRED   CURRENT   READY   UP-TO-DATE   AVAILABLE   NODE SELECTOR            AGE
kube-system   csi-nfs-node   2         2         2       2            2           kubernetes.io/os=linux   50s
...

root@master:~/csi-driver-nfs# kubectl get pod -A
NAMESPACE     NAME                                  READY   STATUS    RESTARTS          AGE
...
kube-system   csi-nfs-controller-7f547655c8-b8b75   3/3     Running   0                 66s
kube-system   csi-nfs-node-726p4                    3/3     Running   0                 64s
kube-system   csi-nfs-node-v7dw5                    3/3     Running   0                 64s
...

使用csi nfs

创建storageclass_test.yaml，provisioner为nfs.csi.k8s.io，parameters为nfs server参数

root@master:~# cat storageclass_test.yaml
allowVolumeExpansion: true
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: csi-nfs-sc
  resourceVersion: "862171"
provisioner: nfs.csi.k8s.io
parameters:
  server: 192.168.56.2
  share: /srv/nfs

reclaimPolicy: Delete
volumeBindingMode: Immediate

创建pvc-test.yaml，指定storageClassName

root@master:~# cat pvc-test.yaml
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: pvc-test
  namespace: default
spec:
  accessModes:
  - ReadWriteOnce
  resources:
    requests:
      storage: 1Gi
  storageClassName: csi-nfs-sc

创建pod配置文件pod-test.yaml，指定pvc

root@master:~# cat pod-test.yaml
apiVersion: v1
kind: Pod
metadata:
  name: my-csi-app1
  namespace: default
spec:
  containers:
  - command:
    - sleep
    - "1000000"
    image: nginx:1.14
    imagePullPolicy: IfNotPresent
    name: my-frontend
    volumeMounts:
    - mountPath: /data
      name: my-csi-volume
  volumes:
  - name: my-csi-volume
    persistentVolumeClaim:
      claimName: pvc-test

应用上面三个yaml文件，创建出pv，pvc和pod

root@master:~# kubectl apply -f storageclass_test.yaml
storageclass.storage.k8s.io/csi-nfs-sc created
root@master:~# kubectl apply -f pvc-test.yaml
persistentvolumeclaim/pvc-test created
root@master:~# kubectl apply -f pod-test.yaml
pod/my-csi-app1 created
root@master:~# kubectl get pv
NAME                                       CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS   CLAIM              STORAGECLASS      REASON   AGE
pvc-afaba19e-edf1-40f8-8983-61dfabf180a6   1Gi        RWO            Delete           Bound    default/pvc-test   csi-nfs-sc                 8s
root@master:~# kubectl get pvc
NAME       STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS      AGE
pvc-test   Bound    pvc-afaba19e-edf1-40f8-8983-61dfabf180a6   1Gi        RWO            csi-nfs-sc        11s
root@master:~# kubectl get pod
NAME          READY   STATUS    RESTARTS   AGE
my-csi-app1   1/1     Running   0          10s

登录pod内部，在nfs存储目录创建测试文件 /data/test

root@master:~# kubectl exec -it my-csi-app1 bash
root@my-csi-app1:/# echo hello > /data/test

退出pod，可在node上的nfs目录看到pod中创建的文件，证明csi nfs可正常工作

root@my-csi-app1:/# exit
root@master:~# cat /srv/nfs/pvc-afaba19e-edf1-40f8-8983-61dfabf180a6/test
hello