k8s部署minio+StorageClass+NFS

搭建minio+StorageClass+NFS,大致有以下几个步骤:

1. 创建一个可用的NFS Server。
2. 创建Service Account，这是用来管控NFS provisioner在k8s集群中运行的权限。
3. 创建NFS provisioner，有两个功能，一个是在NFS共享目录下创建挂载点(volume)，另一个则是建了PV并将PV与NFS的挂载点建立关联。
4. 创建StorageClass，负责建立PVC并调用NFS provisioner进行预定的工作，并让PV与PVC建立管理。
5. 创建minio服务，连接StorageClass

1、安装nfs（这里用docker-compose安装的）

docker-compose安装nfs
centos安装nfs

2、在需要安装minio的服务器上安装rpcbind nfs-utils（必须安装）

#nfs端口号2049
#RPC端口号111
rpm -q rpcbind nfs-utils     #查询是否安装
yum install -y nfs-utils rpcbind   #安装nfs和rpc的软件包

systemctl start rpcbind   #开启rpcbind服务
systemctl enable rpcbind   #开机自启rpcbind服务

3、使用以下文档配置account及相关权限

rbac.yaml: #唯一需要修改的地方只有namespace,根据实际情况定义

apiVersion: v1
kind: ServiceAccount
metadata:
  name: nfs-client-provisioner
  # replace with namespace where provisioner is deployed
  namespace: paas-basic        #根据实际环境设定namespace,下面类同
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: nfs-client-provisioner-runner
rules:
  - apiGroups: [""]
    resources: ["persistentvolumes"]
    verbs: ["get", "list", "watch", "create", "delete"]
  - apiGroups: [""]
    resources: ["persistentvolumeclaims"]
    verbs: ["get", "list", "watch", "update"]
  - apiGroups: ["storage.k8s.io"]
    resources: ["storageclasses"]
    verbs: ["get", "list", "watch"]
  - apiGroups: [""]
    resources: ["events"]
    verbs: ["create", "update", "patch"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: run-nfs-client-provisioner
subjects:
  - kind: ServiceAccount
    name: nfs-client-provisioner
    # replace with namespace where provisioner is deployed
    namespace: paas-basic
roleRef:
  kind: ClusterRole
  name: nfs-client-provisioner-runner
  apiGroup: rbac.authorization.k8s.io
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: leader-locking-nfs-client-provisioner
    # replace with namespace where provisioner is deployed
  namespace: paas-basic
rules:
  - apiGroups: [""]
    resources: ["endpoints"]
    verbs: ["get", "list", "watch", "create", "update", "patch"]
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: leader-locking-nfs-client-provisioner
  namespace: paas-basic
subjects:
  - kind: ServiceAccount
    name: nfs-client-provisioner
    # replace with namespace where provisioner is deployed
    namespace: paas-basic
roleRef:
  kind: Role
  name: leader-locking-nfs-client-provisioner
  apiGroup: rbac.authorization.k8s.io

4、创建NFS provisioner

nfs-provisioner.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: nfs-client-provisioner
  labels:
    app: nfs-client-provisioner
  # replace with namespace where provisioner is deployed
  namespace: paas-basic  #与RBAC文件中的namespace保持一致
spec:
  replicas: 1
  selector:
    matchLabels:
      app: nfs-client-provisioner
  strategy:
    type: Recreate
  selector:
    matchLabels:
      app: nfs-client-provisioner
  template:
    metadata:
      labels:
        app: nfs-client-provisioner
    spec:
      serviceAccountName: nfs-client-provisioner
      containers:
        - name: nfs-client-provisioner
          image: quay.io/external_storage/nfs-client-provisioner:latest
          volumeMounts:
            - name: nfs-client-root
              mountPath: /persistentvolumes
          env:
            - name: PROVISIONER_NAME
              value: qgg-nfs-storage  #provisioner名称,请确保该名称与 nfs-StorageClass.yaml文件中的provisioner名称保持一致
            - name: NFS_SERVER
              value: 192.168.180.48   #NFS Server IP地址
            - name: NFS_PATH
              value: /minio-data/     #NFS服务器根目录上需要建好该文件夹
      volumes:
        - name: nfs-client-root
          nfs:
            server: 192.168.180.48  #NFS Server IP地址
            path: /minio-data/      #NFS服务器根目录上需要建好该文件夹

5、创建NFS资源的StorageClass

nfs-StorageClass.yaml

apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: managed-nfs-storage
provisioner: qgg-nfs-storage #这里的名称要和provisioner配置文件中的环境变量PROVISIONER_NAME保持一致parameters:  archiveOnDelete: "false"

6、直接页面创建（此步可以替换 3、4、5步，不过没有限定名称空间，创建在kube-system里）

创建完后去k8s上查询创建的内容

#创建了对象
[root@app01 minio]# kubectl get pv,pvc,pod --all-namespaces
NAME                                         CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS   CLAIM                                    STORAGECLASS                   REASON   AGE
persistentvolume/nfs-pv-minio-storageclass   10Gi       RWX            Retain           Bound    kube-system/nfs-pvc-minio-storageclass   nfs-storageclass-provisioner            113s

NAMESPACE     NAME                                               STATUS   VOLUME                      CAPACITY   ACCESS MODES   STORAGECLASS                   AGE
kube-system   persistentvolumeclaim/nfs-pvc-minio-storageclass   Bound    nfs-pv-minio-storageclass   10Gi       RWX            nfs-storageclass-provisioner   113s

NAMESPACE     NAME                                             READY   STATUS    RESTARTS   AGE
kube-system   pod/eip-nfs-minio-storageclass-98cc5498d-dnkhm   1/1     Running   0          113s

#创建了权限
[root@app01 minio]# kubectl get ClusterRole,ClusterRoleBinding,Role,RoleBinding --all-namespaces

6、创建minio

minio-distributed-statefulset.yaml

apiVersion: v1
kind: Service
metadata:
  name: minio
  namespace: paas-basic
  labels:
    app: minio
spec:
  selector:
    app: minio
  type: NodePort
  ports:
    - name: minio-api
      port: 9000
      targetPort: 9000
      nodePort: 39000
      protocol: TCP
    - name: minio-ui
      port: 9001
      targetPort: 9001
      nodePort: 39001
      protocol: TCP
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: minio
  namespace: paas-basic
spec:
  serviceName: minio
  podManagementPolicy: Parallel
  replicas: 4
  selector:
    matchLabels:
      app: minio
  template:
    metadata:
      labels:
        app: minio
    spec:
      containers:
      - name: minio
        env:
        - name: MINIO_ACCESS_KEY
          value: "admin"
        - name: MINIO_SECRET_KEY
          value: "Qwe@1245"
        image: minio/minio:RELEASE.2022-03-22T02-05-10Z
        args:
        - server
        - http://minio-{0...3}.minio.paas-basic.svc.cluster.local/data
        - '--console-address'
        - ':9001'
        ports:
        - containerPort: 9000
        - containerPort: 9001
        volumeMounts:
        - name: data
          mountPath: /data
  volumeClaimTemplates:
  - metadata:
      name: data
    spec:
      accessModes:
        - ReadWriteOnce
      resources:
        requests:
          storage: 20Gi
      storageClassName: minio-storageclass

访问地址：http://ip:39001/