k8s & cephfs(动态pvc)

已于 2024-05-28 10:51:05 修改
阅读量460 收藏 2
点赞数 1
文章标签： kubernetes 容器 云原生
于 2024-05-28 10:50:50 首次发布
版权声明：本文为博主原创文章，遵循 CC 4.0 BY-SA 版权协议，转载请附上原文出处链接和本声明。
本文链接：https://blog.csdn.net/tty2020/article/details/139260378
版权

官方参考文档：
GitHub - ceph/ceph-csi at v3.9.0

测试版本

Ceph Version

Ceph CSI Version

Container Orchestrator Name

Version Tested

v17.2.7

v3.9.0

Kubernetes

v1.25.6

安装Ceph-csi

Step 1 Download GitHub - ceph/ceph-csi at v3.9.0

root@sd-k8s-master-1:~# wget https://github.com/ceph/ceph-csi/archive/refs/tags/v3.9.0.zip
root@sd-k8s-master-1:~# unzip v3.9.0.zip

Step 2 创建csidriver

root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# cat csidriver.yaml 
#
# /!\ DO NOT MODIFY THIS FILE
#
# This file has been automatically generated by Ceph-CSI yamlgen.
# The source for the contents can be found in the api/deploy directory, make
# your modifications there.
#
---
apiVersion: storage.k8s.io/v1
kind: CSIDriver
metadata:
  name: "cephfs.csi.ceph.com"
  namespace: cephfs
spec:
  attachRequired: false
  podInfoOnMount: false
  fsGroupPolicy: File
  seLinuxMount: true
root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# kubectl create -f csidriver.yaml

Step 3 为sidecar容器和节点插件部署RBAC：

root@sd-k8s-master-1:~# cd /root/ceph-csi-3.9.0/deploy/cephfs/kubernetes
root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# cat csi-provisioner-rbac.yaml
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: cephfs-csi-provisioner
  namespace: cephfs

---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: cephfs-external-provisioner-runner
rules:
  - apiGroups: [""]
    resources: ["nodes"]
    verbs: ["get", "list", "watch"]
  - apiGroups: [""]
    resources: ["secrets"]
    verbs: ["get", "list", "watch"]
  - apiGroups: [""]
    resources: ["events"]
    verbs: ["list", "watch", "create", "update", "patch"]
  - apiGroups: [""]
    resources: ["persistentvolumes"]
    verbs: ["get", "list", "watch", "create", "update", "delete", "patch"]
  - apiGroups: [""]
    resources: ["persistentvolumeclaims"]
    verbs: ["get", "list", "watch", "update"]
  - apiGroups: [""]
    resources: ["persistentvolumeclaims/status"]
    verbs: ["update", "patch"]
  - apiGroups: ["storage.k8s.io"]
    resources: ["storageclasses"]
    verbs: ["get", "list", "watch"]
  - apiGroups: ["snapshot.storage.k8s.io"]
    resources: ["volumesnapshots"]
    verbs: ["get", "list"]
  - apiGroups: ["snapshot.storage.k8s.io"]
    resources: ["volumesnapshots/status"]
    verbs: ["get", "list", "patch"]
  - apiGroups: ["snapshot.storage.k8s.io"]
    resources: ["volumesnapshotcontents"]
    verbs: ["get", "list", "watch", "update", "patch"]
  - apiGroups: ["snapshot.storage.k8s.io"]
    resources: ["volumesnapshotclasses"]
    verbs: ["get", "list", "watch"]
  - apiGroups: ["storage.k8s.io"]
    resources: ["csinodes"]
    verbs: ["get", "list", "watch"]
  - apiGroups: ["snapshot.storage.k8s.io"]
    resources: ["volumesnapshotcontents/status"]
    verbs: ["update", "patch"]
  - apiGroups: [""]
    resources: ["configmaps"]
    verbs: ["get"]
  - apiGroups: [""]
    resources: ["serviceaccounts"]
    verbs: ["get"]
  - apiGroups: [""]
    resources: ["serviceaccounts/token"]
    verbs: ["create"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: cephfs-csi-provisioner-role
subjects:
  - kind: ServiceAccount
    name: cephfs-csi-provisioner
    namespace: cephfs
roleRef:
  kind: ClusterRole
  name: cephfs-external-provisioner-runner
  apiGroup: rbac.authorization.k8s.io

---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  # replace with non-cephfs namespace name
  namespace: cephfs
  name: cephfs-external-provisioner-cfg
rules:
  - apiGroups: [""]
    resources: ["configmaps"]
    verbs: ["get", "list", "watch"]
  - apiGroups: ["coordination.k8s.io"]
    resources: ["leases"]
    verbs: ["get", "watch", "list", "delete", "update", "create"]

---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: cephfs-csi-provisioner-role-cfg
  # replace with non-cephfs namespace name
  namespace: cephfs
subjects:
  - kind: ServiceAccount
    name: cephfs-csi-provisioner
    # replace with non-cephfs namespace name
    namespace: cephfs
roleRef:
  kind: Role
  name: cephfs-external-provisioner-cfg
  apiGroup: rbac.authorization.k8s.io
root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# cat csi-nodeplugin-rbac.yaml 
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: cephfs-csi-nodeplugin
  namespace: cephfs
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: cephfs-csi-nodeplugin
rules:
  - apiGroups: [""]
    resources: ["secrets"]
    verbs: ["get"]
  - apiGroups: [""]
    resources: ["configmaps"]
    verbs: ["get"]
  - apiGroups: [""]
    resources: ["serviceaccounts"]
    verbs: ["get"]
  - apiGroups: [""]
    resources: ["serviceaccounts/token"]
    verbs: ["create"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: cephfs-csi-nodeplugin
subjects:
  - kind: ServiceAccount
    name: cephfs-csi-nodeplugin
    # replace with non-cephfs namespace name
    namespace: cephfs
roleRef:
  kind: ClusterRole
  name: cephfs-csi-nodeplugin
  apiGroup: rbac.authorization.k8s.io
root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# kubectl create -f csi-provisioner-rbac.yaml
root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# kubectl create -f csi-nodeplugin-rbac.yaml

Step 4 编辑configmap文件

root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# cat csi-config-map.yaml
apiVersion: v1
kind: ConfigMap
data:
  config.json: |-
    [
          {
        "clusterID": "92ab6c78-7edc-11ee-aec4-5e807f521aec",
        "monitors": [
          "10.220.9.13:6789",
          "10.220.9.14:6789",
          "10.220.9.15:6789"
         ]
       }
    ]
metadata:
  name: ceph-csi-config
  namespace: cephfs
root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# cat ceph-config.yaml 
apiVersion: v1
kind: ConfigMap
data:
  ceph.conf: |
    [global]
    auth_cluster_required = cephx
    auth_service_required = cephx
    auth_client_required = cephx

    # Workaround for http://tracker.ceph.com/issues/23446
    fuse_set_user_groups = false

    # ceph-fuse which uses libfuse2 by default has write buffer size of 2KiB
    # adding 'fuse_big_writes = true' option by default to override this limit
    # see https://github.com/ceph/ceph-csi/issues/1928
    fuse_big_writes = true
  # keyring is a required key and its value should be empty
  keyring: |
metadata:
  name: ceph-config
  namespace: cephfs
root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# kubectl create -f ceph-csi-config.yaml
root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# kubectl create -f ceph-config.yaml

Step 5 部署 CSI Sidecar 容器：
#注释ceph-csi-encryption-kms-config volume

root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# sed -i 's/registry.k8s.io/k8s.dockerproxy.com/g' csi-cephfsplugin-provisioner.yaml
root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# sed -i 's/registry.k8s.io/k8s.dockerproxy.com/g' csi-cephfsplugin.yaml
root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# cat csi-cephfsplugin-provisioner.yaml
---
kind: Service
apiVersion: v1
metadata:
  name: csi-cephfsplugin-provisioner
  namespace: cephfs
  labels:
    app: csi-metrics
spec:
  selector:
    app: csi-cephfsplugin-provisioner
  ports:
    - name: http-metrics
      port: 8080
      protocol: TCP
      targetPort: 8681

---
kind: Deployment
apiVersion: apps/v1
metadata:
  name: csi-cephfsplugin-provisioner
  namespace: cephfs
spec:
  selector:
    matchLabels:
      app: csi-cephfsplugin-provisioner
  replicas: 3
  template:
    metadata:
      labels:
        app: csi-cephfsplugin-provisioner
    spec:
      #      affinity:
      #        podAntiAffinity:
      #          requiredDuringSchedulingIgnoredDuringExecution:
      #            - labelSelector:
      #                matchExpressions:
      #                  - key: app
      #                    operator: In
      #                    values:
      #                      - csi-cephfsplugin-provisioner
      #              topologyKey: "kubernetes.io/hostname"
      serviceAccountName: cephfs-csi-provisioner
      priorityClassName: system-cluster-critical
      containers:
        - name: csi-provisioner
          image: k8s.dockerproxy.com/sig-storage/csi-provisioner:v3.5.0
          args:
            - "--csi-address=$(ADDRESS)"
            - "--v=1"
            - "--timeout=150s"
            - "--leader-election=true"
            - "--retry-interval-start=500ms"
            - "--feature-gates=Topology=false"
            - "--feature-gates=HonorPVReclaimPolicy=true"
            - "--prevent-volume-mode-conversion=true"
            - "--extra-create-metadata=true"
          env:
            - name: ADDRESS
              value: unix:///csi/csi-provisioner.sock
          imagePullPolicy: "IfNotPresent"
          volumeMounts:
            - name: socket-dir
              mountPath: /csi
        - name: csi-resizer
          image: k8s.dockerproxy.com/sig-storage/csi-resizer:v1.8.0
          args:
            - "--csi-address=$(ADDRESS)"
            - "--v=1"
            - "--timeout=150s"
            - "--leader-election"
            - "--retry-interval-start=500ms"
            - "--handle-volume-inuse-error=false"
            - "--feature-gates=RecoverVolumeExpansionFailure=true"
          env:
            - name: ADDRESS
              value: unix:///csi/csi-provisioner.sock
          imagePullPolicy: "IfNotPresent"
          volumeMounts:
            - name: socket-dir
              mountPath: /csi
        - name: csi-snapshotter
          image: k8s.dockerproxy.com/sig-storage/csi-snapshotter:v6.2.2
          args:
            - "--csi-address=$(ADDRESS)"
            - "--v=1"
            - "--timeout=150s"
            - "--leader-election=true"
            - "--extra-create-metadata=true"
          env:
            - name: ADDRESS
              value: unix:///csi/csi-provisioner.sock
          imagePullPolicy: "IfNotPresent"
          volumeMounts:
            - name: socket-dir
              mountPath: /csi
        - name: csi-cephfsplugin
          image: quay.io/cephcsi/cephcsi:v3.9.0
          args:
            - "--nodeid=$(NODE_ID)"
            - "--type=cephfs"
            - "--controllerserver=true"
            - "--endpoint=$(CSI_ENDPOINT)"
            - "--v=5"
            - "--drivername=cephfs.csi.ceph.com"
            - "--pidlimit=-1"
            - "--enableprofiling=false"
            - "--setmetadata=true"
          env:
            - name: POD_IP
              valueFrom:
                fieldRef:
                  fieldPath: status.podIP
            - name: NODE_ID
              valueFrom:
                fieldRef:
                  fieldPath: spec.nodeName
            - name: CSI_ENDPOINT
              value: unix:///csi/csi-provisioner.sock
            - name: POD_NAMESPACE
              valueFrom:
                fieldRef:
                  fieldPath: metadata.namespace
          # - name: KMS_CONFIGMAP_NAME
          #   value: encryptionConfig
          imagePullPolicy: "IfNotPresent"
          volumeMounts:
            - name: socket-dir
              mountPath: /csi
            - name: host-sys
              mountPath: /sys
            - name: lib-modules
              mountPath: /lib/modules
              readOnly: true
            - name: host-dev
              mountPath: /dev
            - name: ceph-config
              mountPath: /etc/ceph/
            - name: ceph-csi-config
              mountPath: /etc/ceph-csi-config/
            - name: keys-tmp-dir
              mountPath: /tmp/csi/keys
                # - name: ceph-csi-encryption-kms-config
##                #   mountPath: /etc/ceph-csi-encryption-kms-config/
        - name: liveness-prometheus
          image: quay.io/cephcsi/cephcsi:v3.9.0
          args:
            - "--type=liveness"
            - "--endpoint=$(CSI_ENDPOINT)"
            - "--metricsport=8681"
            - "--metricspath=/metrics"
            - "--polltime=60s"
            - "--timeout=3s"
          env:
            - name: CSI_ENDPOINT
              value: unix:///csi/csi-provisioner.sock
            - name: POD_IP
              valueFrom:
                fieldRef:
                  fieldPath: status.podIP
          volumeMounts:
            - name: socket-dir
              mountPath: /csi
          imagePullPolicy: "IfNotPresent"
      volumes:
        - name: socket-dir
          emptyDir: {
            medium: "Memory"
          }
        - name: host-sys
          hostPath:
            path: /sys
        - name: lib-modules
          hostPath:
            path: /lib/modules
        - name: host-dev
          hostPath:
            path: /dev
        - name: ceph-config
          configMap:
            name: ceph-config
        - name: ceph-csi-config
          configMap:
            name: ceph-csi-config
        - name: keys-tmp-dir
          emptyDir: {
            medium: "Memory"
          }
      # - name: ceph-csi-encryption-kms-config
          # configMap:
            # name: ceph-csi-encryption-kms-config
root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# cat csi-cephfsplugin.yaml
---
kind: DaemonSet
apiVersion: apps/v1
metadata:
  name: csi-cephfsplugin
  namespace: cephfs
spec:
  selector:
    matchLabels:
      app: csi-cephfsplugin
  template:
    metadata:
      labels:
        app: csi-cephfsplugin
    spec:
      serviceAccountName: cephfs-csi-nodeplugin
      priorityClassName: system-node-critical
      hostNetwork: true
      hostPID: true
      # to use e.g. Rook orchestrated cluster, and mons' FQDN is
      # resolved through k8s service, set dns policy to cluster first
      dnsPolicy: ClusterFirstWithHostNet
      containers:
        - name: driver-registrar
          # This is necessary only for systems with SELinux, where
          # non-privileged sidecar containers cannot access unix domain socket
          # created by privileged CSI driver container.
          securityContext:
            privileged: true
            allowPrivilegeEscalation: true
          image: k8s.dockerproxy.com/sig-storage/csi-node-driver-registrar:v2.8.0
          args:
            - "--v=1"
            - "--csi-address=/csi/csi.sock"
            - "--kubelet-registration-path=/var/lib/kubelet/plugins/cephfs.csi.ceph.com/csi.sock"
          env:
            - name: KUBE_NODE_NAME
              valueFrom:
                fieldRef:
                  fieldPath: spec.nodeName
          volumeMounts:
            - name: socket-dir
              mountPath: /csi
            - name: registration-dir
              mountPath: /registration
        - name: csi-cephfsplugin
          securityContext:
            privileged: true
            capabilities:
              add: ["SYS_ADMIN"]
            allowPrivilegeEscalation: true
          image: quay.io/cephcsi/cephcsi:v3.9.0
          args:
            - "--nodeid=$(NODE_ID)"
            - "--type=cephfs"
            - "--nodeserver=true"
            - "--endpoint=$(CSI_ENDPOINT)"
            - "--v=5"
            - "--drivername=cephfs.csi.ceph.com"
            - "--enableprofiling=false"
            # If topology based provisioning is desired, configure required
            # node labels representing the nodes topology domain
            # and pass the label names below, for CSI to consume and advertise
            # its equivalent topology domain
            # - "--domainlabels=failure-domain/region,failure-domain/zone"
          env:
            - name: POD_IP
              valueFrom:
                fieldRef:
                  fieldPath: status.podIP
            - name: NODE_ID
              valueFrom:
                fieldRef:
                  fieldPath: spec.nodeName
            - name: CSI_ENDPOINT
              value: unix:///csi/csi.sock
            - name: POD_NAMESPACE
              valueFrom:
                fieldRef:
                  fieldPath: metadata.namespace
          # - name: KMS_CONFIGMAP_NAME
          #   value: encryptionConfig
          imagePullPolicy: "IfNotPresent"
          volumeMounts:
            - name: socket-dir
              mountPath: /csi
            - name: mountpoint-dir
              mountPath: /var/lib/kubelet/pods
              mountPropagation: Bidirectional
            - name: plugin-dir
              mountPath: /var/lib/kubelet/plugins
              mountPropagation: "Bidirectional"
            - name: host-sys
              mountPath: /sys
            - name: etc-selinux
              mountPath: /etc/selinux
              readOnly: true
            - name: lib-modules
              mountPath: /lib/modules
              readOnly: true
            - name: host-dev
              mountPath: /dev
            - name: host-mount
              mountPath: /run/mount
            - name: ceph-config
              mountPath: /etc/ceph/
            - name: ceph-csi-config
              mountPath: /etc/ceph-csi-config/
            - name: keys-tmp-dir
              mountPath: /tmp/csi/keys
            - name: ceph-csi-mountinfo
              mountPath: /csi/mountinfo
          # - name: ceph-csi-encryption-kms-config
          #   mountPath: /etc/ceph-csi-encryption-kms-config/
        - name: liveness-prometheus
          securityContext:
            privileged: true
            allowPrivilegeEscalation: true
          image: quay.io/cephcsi/cephcsi:v3.9.0
          args:
            - "--type=liveness"
            - "--endpoint=$(CSI_ENDPOINT)"
            - "--metricsport=8681"
            - "--metricspath=/metrics"
            - "--polltime=60s"
            - "--timeout=3s"
          env:
            - name: CSI_ENDPOINT
              value: unix:///csi/csi.sock
            - name: POD_IP
              valueFrom:
                fieldRef:
                  fieldPath: status.podIP
          volumeMounts:
            - name: socket-dir
              mountPath: /csi
          imagePullPolicy: "IfNotPresent"
      volumes:
        - name: socket-dir
          hostPath:
            path: /var/lib/kubelet/plugins/cephfs.csi.ceph.com/
            type: DirectoryOrCreate
        - name: registration-dir
          hostPath:
            path: /var/lib/kubelet/plugins_registry/
            type: Directory
        - name: mountpoint-dir
          hostPath:
            path: /var/lib/kubelet/pods
            type: DirectoryOrCreate
        - name: plugin-dir
          hostPath:
            path: /var/lib/kubelet/plugins
            type: Directory
        - name: host-sys
          hostPath:
            path: /sys
        - name: etc-selinux
          hostPath:
            path: /etc/selinux
        - name: lib-modules
          hostPath:
            path: /lib/modules
        - name: host-dev
          hostPath:
            path: /dev
        - name: host-mount
          hostPath:
            path: /run/mount
        - name: ceph-config
          configMap:
            name: ceph-config
        - name: ceph-csi-config
          configMap:
            name: ceph-csi-config
        - name: keys-tmp-dir
          emptyDir: {
            medium: "Memory"
          }
        - name: ceph-csi-mountinfo
          hostPath:
            path: /var/lib/kubelet/plugins/cephfs.csi.ceph.com/mountinfo
            type: DirectoryOrCreate
              #- name: ceph-csi-encryption-kms-config
              #  configMap:
              #    name: ceph-csi-encryption-kms-config
---
# This is a service to expose the liveness metrics
apiVersion: v1
kind: Service
metadata:
  name: csi-metrics-cephfsplugin
  namespace: cephfs
  labels:
    app: csi-metrics
spec:
  ports:
    - name: http-metrics
      port: 8080
      protocol: TCP
      targetPort: 8681
  selector:
    app: csi-cephfsplugin
root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# kubectl create -f csi-cephfsplugin-provisioner.yaml
root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# kubectl create -f csi-cephfsplugin.yaml

Step 6 Ceph创建cephfs

[root@ceph01 ~]# ceph osd pool cephfs
[root@ceph01 ~]# ceph auth get-key client.admin
xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx

Step 7 创建ceph secret

root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# cat ceph-sc.yaml 
apiVersion: v1
kind: Secret
metadata:
  name: csi-cephfs-secret
  namespace: cephfs
stringData:
  userID: admin
  userKey: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
  adminID: admin
  adminKey: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# kubectl create -f ceph-sc.yaml
root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# kubectl get secrets -n cephfs
NAME                TYPE     DATA   AGE
csi-cephfs-secret   Opaque   4      49m

Step 8 创建storageclass

root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# cat ceph-storage-class.yaml 
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: csi-cephfs-sc
provisioner: cephfs.csi.ceph.com
parameters:
  clusterID: 92ab6c78-7edc-11ee-aec4-5e807f521aec 
  fsName: cephfs          #cephfs名称，上面创建
  pool: cephfs_data           #cephfs pool名称
  #  mounter: fuse       #挂载方式
  csi.storage.k8s.io/provisioner-secret-name: csi-cephfs-secret
  csi.storage.k8s.io/provisioner-secret-namespace: cephfs
  csi.storage.k8s.io/controller-expand-secret-name: csi-cephfs-secret
  csi.storage.k8s.io/controller-expand-secret-namespace: cephfs
  csi.storage.k8s.io/node-stage-secret-name: csi-cephfs-secret
  csi.storage.k8s.io/node-stage-secret-namespace: cephfs
reclaimPolicy: Delete
allowVolumeExpansion: true
#mountOptions:
 # - discard
root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# kubectl create -f ceph-storage-class.yaml
root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# kubectl get storageclass
NAME                   PROVISIONER                                     RECLAIMPOLICY   VOLUMEBINDINGMODE   ALLOWVOLUMEEXPANSION   AGE
csi-cephfs-sc          cephfs.csi.ceph.com                             Delete          Immediate           true                   138m

Step 9 创建pvc测试

root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# cat test-pvc.yaml 
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: cephfs-test-pvc
  namespace: cephfs
spec:
  accessModes:
    - ReadWriteMany
  resources:
    requests:
      storage: 1Gi
  storageClassName: csi-cephfs-sc
root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# kubectl create -f test-pvc.yaml
root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# kubectl get pvc -n cephfs
NAME              STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS    AGE
cephfs-test-pvc   Bound    pvc-94236c61-a5b7-494d-89ad-e1e18eaad175   1Gi        RWX            csi-cephfs-sc   49m

Step 10 创建pod测试

root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# cat test-pod.yaml
---
apiVersion: v1
kind: Pod
metadata:
  name: test-pd
spec:
  terminationGracePeriodSeconds: 0
  containers:
  - image: harbor.zetyun.cn/gcp/nginx:1.25.3
    name: test-container
    volumeMounts:
    - mountPath: /cache
      name: cache-volume
  volumes:
  - name: cache-volume
    persistentVolumeClaim:
      claimName: cephfs-test-pvc1  #填写pvc名称
root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# kubectl create -f test-pod.yaml
root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# kubectl get po test-pd 
NAME      READY   STATUS    RESTARTS   AGE
test-pd   1/1     Running   0          6m49s

医者运维
关注
k8s使用cephfs动态存储
weixin_42715413的博客
04-25 2494
参考https://github.com/kubernetes-incubator/external-storage/tree/master/ceph/cephfs # 官方没有cephfs动态卷支持 # 使用社区提供的cephfs-provisioner kubectl create ns cephfs #部署Provisioner # yaml文件 https://github.co...
使用 RBD 作为 Kubernetes 存储解决方案的最佳实践指南！
民工哥的博客
06-28 131
戳下方名片，关注并星标！回复“1024”获取2TB学习资源！????体系化学习：运维工程师打怪升级进阶之路 4.0— 特色专栏—MySQL/PostgreSQL/MongoDBElasticSearch/Hadoop/RedisKubernetes/Docker/DevOpsKafka/RabbitMQ/Zookeeper监控平台/应用与服务/集群管理N...
用 k8s 1.20以上版本解决动StorageClass态创建PVC一直处于 Pending 状态
delimar的博客
07-20 359
原因：在 k8s 1.20 之后，出于对性能和统一 apiserver 调用方式的初衷，k8s 移除了对 SelfLink 的支持，而默认上面指定的 provisioner 版本需要 SelfLink 功能，因此 PVC 无法进行自动制备。用 k8s 1.20以上版本，通过 kubectl get pvc，发现 PVC一直处于 Pending 状态。1.由于现在镜像源都被ban了网上随便找个nfs-subdir-external-provisioner镜像源。但是很坑，这两种都不行！4.启动deploy。
K8S PVC一直处于pending状态，无法创建
build_zzz的博客
07-13 543
kubernetes 创建pvc异常，一直处于pending状态，修改kube-apiserver配置解决
k8s 对接ceph并使用StorageClass动态创建pvc
隔壁老姚的博客
08-27 5092
k8s 对接ceph并使用StorageClass创建动态pvc 前提条件，已经安装ceph和k8s，具体环境信息如下： 一. 服务器环境 操作系统：centos 7.6 ceph版本：14.2.10（nautilus） k8s版本：v1.18.0 arch: ARM64 GO版本：go1.14.6 docker版本：19.03.12 在k8s中使用volume主要有两种使用方式，一种的静态方法，一种是动态方法，本文主要介绍动态方法。k8s集群默认不支持动态创建pvc，官方推出了external-sto
k8s & ceph(静态pvc)
简云的博客
05-28 759
在上面的示例中，monitors 应该填写 Ceph Monitor 的 IP 地址和端口号，user 应该填写连接到 Ceph 集群所使用的 Ceph 客户端用户名。#在上面的示例中，keyring 和 ceph.conf 的值应该是经过 Base64 编码的字符串。
k8s & ceph rbd(动态pvc)
简云的博客
05-28 305
注释ceph-csi-encryption-kms-config volume & oidc token。Step 6 Ceph创建ceph rbd pool。Step 8 创建storageclass。Step 4 编辑configmap文件。Step 7 创建ceph secret。Step 2 创建csidriver。Step 10 创建pod测试。Step 9 创建pvc测试。
K8s 使用 CephFS 作为后端存储（静态供给、动态供给）
小毕超博客
04-27 1115
CephFS是 Ceph 中基于RADOS（可扩展分布式对象存储）构建，通过将文件数据划分为对象并分布到集群中的多个存储节点上来实现高可用性和可扩展性。
K8S结合CephFS使用
李半仙
01-10 2402
K8S结合CephFS使用 在ceph查看密钥 ceph auth get-key client.admin |base64 在k8s-master上面创建secret #vim ceph-secret.yaml apiVersion: v1 kind: Secret metadata: name: ceph-secret data: key: QVFDa2R0cGhiZlZXRWhBQVd3VDAxWnl5OWlaU3BWN3NlcTk3bWc9PQ== 创建PV #vim pv
K8S使用ceph-csi持久化存储之cephfs部署验证快照
xiaxia2022的博客
09-13 4076
一、集群和组件版本 K8S集群：1.19.4+ Ceph集群：ceph version 15.2.13 (c44bc49e7a57a87d84dfff2a077a2058aa2172e2) octopus (stable) Ceph-CSI：release-v3.3 (csi版本要对应不然创建pvc要报错) Linue kernel：5.13.12-1.el7.elrepo.x86_64 （内核版本需要升级新版） 镜像版本： k8s.gcr.io/sig-storage/csi-provi
⑤ K8s使用Ceph作为存储、监控、日常运维管理_ceph运维管理平台
2401_83974660的博客
04-18 764
1、创建pod时，kubelet需要使用rbd命令去检测和挂载pv对应的ceph image，所以要在所有的worker节点安装ceph客户端ceph-common。将ceph的ceph.client.admin.keyring和ceph.conf文件拷贝到master的/etc/ceph目录下。#3、创建k8s访问ceph的用户 在ceph的mon或者admin节点。#4、查看key 在ceph的mon或者admin节点和kube用户。#5、创建 admin secret(使用admin的key)
K8s使用Ceph作为后端存储
xinhuixu博客
07-27 1797
RGW（RADOS Gateway））：对象存储网关，接口与S3和Swift兼容。Ceph是一个开源的分布式存储系统，具有高扩展性、高性能、高可靠性等特点，提。是ceph的核心概念， 服务端数据均衡和恢复的最小粒度；Ceph目前最新版本16（P版），市面上应用最广泛的是12（L版）支持对象存储、块存储和文件系统。对象存储集群，Ceph核心，实现用户数据分配、故障转移等集群操作。ceph-deploy：ceph集群部署节点，负责集群整体部署，是存储对象的逻辑分区，它规定了数据冗余的类型和对应的副本。
k8s挂载目录_kubernetes多节点的pod挂载同一个cephfs目录
weixin_39902107的博客
12-21 1989
一、安装cephfs方法一：直接进入deploy目录，执行：ceph-deploy --overwrite-conf mds create ceph01:mds-daemon-1上面的ceph01是机器的hostname方法二：1、在需要安装的目标机器上创建mds目录mkdir -p /var/lib/ceph/mds/ceph-02、生成mds的keyring，并将其写入/var/lib/cep...
k8s备份恢复工具--velero
professorman的博客
10-17 173
k8s的备份和恢复工具主要包括服务端和客户端两部分，服务端运行在k8s集群中，客户端运行在本地的命令行，本地需要配置好k8s的kubeconfig和kubectl客户端工具需要对象存储当做中间备份或存储的工具（S3，minio，OSS等）工作原理即是将k8s资源备份到对象存储当中，并可以从对象存储中拉取备份数据来恢复集群资源对象velero地址https://github.com/vmware-tanzu/velero下面操作的机器是配置好k8s的kubectl和kubeconfig的。
【K8S系列】Kubernetes pod节点NotReady问题及解决方案详解【已解决】
颜淡慕潇
10-17 1069
Kubernetes 集群中的每个节点都是运行容器化应用的基础。当节点状态显示为 `NotReady` 时，意味着该节点无法正常工作，这可能会导致 Pod 无法调度，从而影响整个应用的可用性。本文将深入分析节点不健康的各种原因、详细的排查步骤以及有效的解决方案。
rancher安装并快速部署k8s 管理集群工具
最新发布
咸鱼很闲
10-17 294
安装的时候可以去github上检索rancher看看最新版本适配那个版本的docker，这里安装23.0.1版本。创建master节点（复制url到master节点安装）创建node节点（复制url到master节点安装）3台用于k8s集群 ，1台用于rancher。输入集群的名字其他的默认，完成之后点击创建。点击这，可以直接控制集群。每台服务器新增配置文件。
为什么k8s不支持docker-kubernetes
zonghengzhikong的博客
10-17 707
Kubernetes不再支持Docker作为容器运行时，是为了简化架构、提升性能，并适应现代化容器管理需求的结果。这一变化并不意味着Docker将退出容器生态，而是Docker从直接的运行时角色转向了容器开发和构建工具的角色。
K8S技术深度解析与实践案例
运维人生
10-12 1159
K8s作为当前云原生和微服务架构的首选平台，凭借其强大的容器编排和管理能力，为企业提供了高效、可靠的应用部署和管理解决方案。本文深入探讨了K8s的基本概念、架构设计、核心组件、应用场景以及实战案例，展示了K8s在运维中的强大功能和便利性。未来，随着K8s生态的不断发展和完善，它将在企业运维中发挥更加重要的作用。
在K8S+CEPHFS的应用环境中，K8S PV卷访问模式可以有哪几种?
06-03
在K8S+CEPHFS的应用环境中，K8S PV卷访问模式可以有以下几种： 1. `ReadOnlyMany`：只读多人访问模式，多个Pod可以以只读方式挂载同一个CEPHFS的PV卷，但是不能写入数据。 2. `ReadWriteOnce`：单节点读写访问模式...
评论
添加红包

请填写红包祝福语或标题

个

红包个数最小为10个

元

红包金额最低5元

当前余额3.43元 前往充值 >
需支付：10.00元
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包

打赏作者

医者运维

你的鼓励将是我创作的最大动力

¥1 ¥2 ¥4 ¥6 ¥10 ¥20
扫码支付：¥1
获取中
扫码支付

您的余额不足，请更换扫码支付或充值

打赏作者

实付元
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明：

1.余额是钱包充值的虚拟货币，按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载，可以购买VIP、付费专栏及课程。

余额充值