kubernetes v1.23.3 安装ceph(rook-release-1.10)

节点信息

[root@cloud-1 ~]# kubectl get node
NAME      STATUS   ROLES                  AGE   VERSION
cloud-1   Ready    control-plane,master   43h   v1.23.3
[root@cloud-1 ~]# kubectl get node -o wide
NAME      STATUS   ROLES                  AGE   VERSION   INTERNAL-IP   EXTERNAL-IP   OS-IMAGE                           KERNEL-VERSION                 CONTAINER-RUNTIME
cloud-1   Ready    control-plane,master   43h   v1.23.3   172.17.4.40   <none>        Rocky Linux 8.7 (Green Obsidian)   4.18.0-425.10.1.el8_7.x86_64   containerd://1.6.19

rook-release-1.10 下载

https://github.com/rook/rook

安装过程

条件：需要有未使用的空磁盘 如： 以下vdb vdc vdd是给ceph osd 使用

[root@cloud-1 ~]# lsblk
NAME   MAJ:MIN RM  SIZE RO TYPE MOUNTPOINT
sr0     11:0    1  492K  0 rom
vda    252:0    0  100G  0 disk
├─vda1 252:1    0  100M  0 part /boot/efi
├─vda2 252:2    0 1000M  0 part /boot
├─vda3 252:3    0    1M  0 part
└─vda4 252:4    0 98.9G  0 part /
vdb    252:16   0  100G  0 disk
vdc    252:32   0  100G  0 disk
vdd    252:48   0  100G  0 disk

cd rook-release-1.10/deploy/examples/

因为这次环境是间节点  所以 ceph mon mgr 数量需要设置为1（默认3）
将 cluster.yaml修改
  mon:
    count: 1
  mgr:
    count: 1


kubectl create -f crds.yaml -f common.yaml -f operator.yaml
kubectl create -f cluster.yaml
kubectl apply -f toolbox.yaml

部署过各中发再csi一些镜像无法下载  本地导入处理
 ctr -n=k8s.io image import /root/csi-snapshotterv6.2.1
 ctr -n=k8s.io image import /root/csi-resizerv1.7.0
 ctr -n=k8s.io image import /root/csi-provisionerv3.4.0
 ctr -n=k8s.io image import /root/csi-node-driver-registrarv2.7.0
 ctr -n=k8s.io image import /root/csi-attacherv4.1.0
镜像下载地址：
链接：https://pan.baidu.com/s/1BwvCjtr81hR7AvFE-VzEmw 
提取码：rp5s 

[root@cloud-1 examples]# kubectl  get pod -n rook-ceph
NAME                                                READY   STATUS      RESTARTS   AGE
csi-cephfsplugin-provisioner-5b6959d889-bsn9m       5/5     Running     0          122m
csi-cephfsplugin-z58dt                              2/2     Running     0          122m
csi-rbdplugin-gv49b                                 2/2     Running     0          122m
csi-rbdplugin-provisioner-d6df8c994-qzx9j           5/5     Running     0          122m
rook-ceph-crashcollector-cloud-1-5c7cbd7f79-782zg   1/1     Running     0          120m
rook-ceph-mgr-a-85bc59646f-jn49l                    2/2     Running     0          121m
rook-ceph-mon-a-cff85f57d-jp74j                     2/2     Running     0          121m
rook-ceph-operator-76584b7c88-dxphv                 1/1     Running     0          135m
rook-ceph-osd-0-64c75cfbff-mvxsf                    2/2     Running     0          120m
rook-ceph-osd-1-57dddc6f96-v9btn                    2/2     Running     0          120m
rook-ceph-osd-2-c6f9cd587-rcjpl                     2/2     Running     0          120m
rook-ceph-osd-prepare-cloud-1-lhf8s                 0/1     Completed   0          121m
rook-ceph-tools-6765648554-zwvdn                    1/1     Running     0          97m

[root@cloud-1 examples]# kubectl -n rook-ceph  exec -it rook-ceph-tools-6765648554-zwvdn bash
kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.
bash-4.4$ ceph -s
  cluster:
    id:     8825d050-f111-481d-8acd-f9682977dae5
    health: HEALTH_WARN
            Degraded data redundancy: 4/6 objects degraded (66.667%), 1 pg degraded, 1 pg undersized
            1 mgr modules have recently crashed

  services:
    mon: 1 daemons, quorum a (age 2h)
    mgr: a(active, since 2h)
    osd: 3 osds: 3 up (since 2h), 3 in (since 2h)

  data:
    pools:   1 pools, 1 pgs
    objects: 2 objects, 449 KiB
    usage:   59 MiB used, 300 GiB / 300 GiB avail
    pgs:     4/6 objects degraded (66.667%)
             1 active+undersized+degraded

bash-4.4$ ceph osd tree
ID  CLASS  WEIGHT   TYPE NAME         STATUS  REWEIGHT  PRI-AFF
-1         0.29306  root default
-3         0.29306      host cloud-1
 0    hdd  0.09769          osd.0         up   1.00000  1.00000
 1    hdd  0.09769          osd.1         up   1.00000  1.00000
 2    hdd  0.09769          osd.2         up   1.00000  1.00000
bash-4.4$ ceph osd df
ID  CLASS  WEIGHT   REWEIGHT  SIZE     RAW USE  DATA     OMAP  META    AVAIL    %USE  VAR   PGS  STATUS
 0    hdd  0.09769   1.00000  100 GiB   19 MiB  144 KiB   0 B  19 MiB  100 GiB  0.02  0.98    0      up
 1    hdd  0.09769   1.00000  100 GiB   21 MiB  596 KiB   0 B  20 MiB  100 GiB  0.02  1.04    1      up
 2    hdd  0.09769   1.00000  100 GiB   19 MiB  144 KiB   0 B  19 MiB  100 GiB  0.02  0.98    0      up
                       TOTAL  300 GiB   59 MiB  884 KiB   0 B  58 MiB  300 GiB  0.02
MIN/MAX VAR: 0.98/1.04  STDDEV: 0

对接ceph

[root@cloud-1 rook-release-1.10]# find / -name storageclass.yaml
/root/rook-release-1.10/deploy/examples/csi/cephfs/storageclass.yaml
[root@cloud-1 cephfs]# kubectl  apply -f storageclass.yaml
storageclass.storage.k8s.io/rook-cephfs created
[root@cloud-1 cephfs]# kubectl get sc -A
NAME          PROVISIONER                     RECLAIMPOLICY   VOLUMEBINDINGMODE   ALLOWVOLUMEEXPANSION   AGE
rook-cephfs   rook-ceph.cephfs.csi.ceph.com   Delete          Immediate           true                   5s