ceph-csi搭建踩坑过程(尽量和官档一致)

官档: https://docs.ceph.com/en/pacific/rbd/rbd-kubernetes/?highlight=csi#configure-ceph-csi

1 创建池
[root@ceph1 ~]# ceph osd pool create kubernetes 8
pool 'kubernetes' created

[root@ceph1 ~]# ceph osd pool ls
.rgw.root
.........
kubernetes

2 初始化
rbd pool init kubernetes

3 创建用户 生成密钥
[root@ceph1 ~]# ceph auth get-or-create client.kubernetes mon 'profile rbd' osd 'profile rbd pool=kubernetes' mgr 'profile rbd pool=kubernetes'
[client.kubernetes]
        key = AQBMN21hzeVzGBAA2B1nSkpWtE8LPxy/Osburg==
        
4 查看clusterId(fsid) 和 mon地址
[root@ceph1 ~]# ceph mon dump
dumped monmap epoch 1
epoch 1
fsid c60a3959-9e56-45e4-9a87-6c186cf28922
last_changed 2021-05-12 18:59:24.593941
created 2021-05-12 18:59:24.593941
min_mon_release 14 (nautilus)
0: [v2:192.168.11.121:3300/0,v1:192.168.11.121:6789/0] mon.ceph1
1: [v2:192.168.11.122:3300/0,v1:192.168.11.122:6789/0] mon.ceph2
2: [v2:192.168.11.123:3300/0,v1:192.168.11.123:6789/0] mon.ceph3

5 生成config/secret   clusterID mon地址得替换成自己ceph集群的
cat <<EOF > csi-config-map.yaml
---
apiVersion: v1
kind: ConfigMap
data:
  config.json: |-
    [
      {
        "clusterID": "c60a3959-9e56-45e4-9a87-6c186cf28922",
        "monitors": [
          "192.168.11.121:6789",
          "192.168.11.122:6789",
          "192.168.11.123:6789"
        ]
      }
    ]
metadata:
  name: ceph-csi-config
EOF

5 生成config/secret  userKey得替换成自己集群的
cat <<EOF > csi-rbd-secret.yaml
---
apiVersion: v1
kind: Secret
metadata:
  name: csi-rbd-secret
  namespace: default
stringData:
  userID: kubernetes
  userKey: AQBMN21hzeVzGBAA2B1nSkpWtE8LPxy/Osburg==
EOF

5 生成config/secret(以下步骤是官方文档没有写的配置,科学上网下载下来 apply下,源码的pr别人有提这个问题)

wget https://raw.githubusercontent.com/ceph/ceph-csi/devel/examples/ceph-conf.yaml
wget https://github.com/ceph/ceph-csi/blob/devel/examples/kms/vault/kms-config.yaml

6  创建rbac授权
$ kubectl apply -f https://raw.githubusercontent.com/ceph/ceph-csi/master/deploy/rbd/kubernetes/csi-provisioner-rbac.yaml   
$ kubectl apply -f https://raw.githubusercontent.com/ceph/ceph-csi/master/deploy/rbd/kubernetes/csi-nodeplugin-rbac.yaml

7 创建deployment/daemonset 其中的镜像得改为国内和科学上网docker save/load
注意csi-rbdplugin-provisioner.yaml   配置了pod互斥,replicas为3,如果机器少.可以去掉affinity,或者改为软互斥
$ wget https://raw.githubusercontent.com/ceph/ceph-csi/master/deploy/rbd/kubernetes/csi-rbdplugin-provisioner.yaml   
$ kubectl apply -f csi-rbdplugin-provisioner.yaml
$ wget https://raw.githubusercontent.com/ceph/ceph-csi/master/deploy/rbd/kubernetes/csi-rbdplugin.yaml
$ kubectl apply -f csi-rbdplugin.yaml

8 测试
注意:这一步加上 imageFeatures: layering    否则csi-rbdplugin-provisioner的pod日志会报:  missing required parameter imageFeatures

clusterID得替换成自己的

cat <<EOF > csi-rbd-sc.yaml
---
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
   name: csi-rbd-sc
provisioner: rbd.csi.ceph.com
parameters:
   clusterID: c60a3959-9e56-45e4-9a87-6c186cf28922
    imageFeatures: layering
   pool: kubernetes
   csi.storage.k8s.io/provisioner-secret-name: csi-rbd-secret
   csi.storage.k8s.io/provisioner-secret-namespace: default
   csi.storage.k8s.io/node-stage-secret-name: csi-rbd-secret
   csi.storage.k8s.io/node-stage-secret-namespace: default
reclaimPolicy: Delete
mountOptions:
   - discard
EOF
kubectl apply -f csi-rbd-sc.yaml

cat <<EOF > raw-block-pvc.yaml
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: raw-block-pvc
spec:
  accessModes:
    - ReadWriteOnce
  volumeMode: Block
  resources:
    requests:
      storage: 1Gi
  storageClassName: csi-rbd-sc
EOF
kubectl apply -f raw-block-pvc.yaml

cat <<EOF > raw-block-pod.yaml
---
apiVersion: v1
kind: Pod
metadata:
  name: pod-with-raw-block-volume
spec:
  containers:
    - name: fc-container
      image: fedora:26
      command: ["/bin/sh", "-c"]
      args: ["tail -f /dev/null"]
      volumeDevices:
        - name: data
          devicePath: /dev/xvda
  volumes:
    - name: data
      persistentVolumeClaim:
        claimName: raw-block-pvc
EOF
kubectl apply -f raw-block-pod.yaml

cat <<EOF > pvc.yaml
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: rbd-pvc
spec:
  accessModes:
    - ReadWriteOnce
  volumeMode: Filesystem
  resources:
    requests:
      storage: 1Gi
  storageClassName: csi-rbd-sc
EOFkubectl apply -f pvc.yaml

cat <<EOF > pod.yaml
---
apiVersion: v1
kind: Pod
metadata:
  name: csi-rbd-demo-pod
spec:
  containers:
    - name: web-server
      image: nginx
      volumeMounts:
        - name: mypvc
          mountPath: /var/lib/www/html
  volumes:
    - name: mypvc
      persistentVolumeClaim:
        claimName: rbd-pvc
        readOnly: false
EOF
kubectl apply -f pod.yaml

上图:

storageclass

pvc 

 pv