ceph (luminous 版) crushmap 与 pool结合用于物理划分 IO 使用域

目的

利用 crushmap ， 把不同的 osd 存放到不同的分组中
不同的 ceph pool 独立使用到不同的 crushmap 分组 （即使用不同的 osd 磁盘）
从物理上隔离了不同的 pool 数据读写 IO

ceph 结构参考

hostA	hostB	hostC	备注
osd.0	osd.4	osd.8	把 osd0 ~ osd.11 划分到 root=default 根中 只供 volumes pool 调用
osd.1	osd.5	osd.9
osd.2	osd.6	osd.10
osd.3	osd.7	osd.11
osd.12	osd.14	osd.16	把 osd14 ~ osd.17 划分到 root=noah 根中 只供 noah pool 调用
osd.13	osd.15	osd.17

crushmap 管理

添加 bucket

ceph osd crush add-bucket hostA host
ceph osd crush add-bucket hostB host
ceph osd crush add-bucket hostC host

把主机添加到对应 crushmap 的 root 中

ceph osd crush move  hostA  root=noah
ceph osd crush move  hostB  root=noah
ceph osd crush move  hostC  root=noah

把对应 OSD 添加到对应主机中

ceph osd crush add osd.12 4.0 host=hostA root=noah
ceph osd crush add osd.13 4.0 host=hostA root=noah
ceph osd crush add osd.14 4.0 host=hostB root=noah
ceph osd crush add osd.15 4.0 host=hostB root=noah
ceph osd crush add osd.16 4.0 host=hostC root=noah
ceph osd crush add osd.17 4.0 host=hostC root=noah

参考 osd tree

[root@ns-storage-020100 ~]# ceph osd tree
ID  CLASS WEIGHT   TYPE NAME                              STATUS REWEIGHT PRI-AFF
-12       24.00000 root noah                                                        ################# 这里是另外一个独立的区间
 -9        8.00000     host hostA
 12   hdd  4.00000         osd.12                             up  1.00000 1.00000
 13   hdd  4.00000         osd.13                             up  1.00000 1.00000
-10        8.00000     host hostB
 14   hdd  4.00000         osd.14                             up  1.00000 1.00000
 15   hdd  4.00000         osd.15                             up  1.00000 1.00000
-11        8.00000     host hostC
 16   hdd  4.00000         osd.16                             up  1.00000 1.00000
 17   hdd  4.00000         osd.17                             up  1.00000 1.00000
 -1       48.00000 root default                                                      ################# 这里是另外一个独立的区间
 -2       16.00000     host hostA
  0   hdd  4.00000         osd.0                              up  1.00000 1.00000
  1   hdd  4.00000         osd.1                              up  1.00000 1.00000
  2   hdd  4.00000         osd.2                              up  1.00000 1.00000
  3   hdd  4.00000         osd.3                              up  1.00000 1.00000
 -3       16.00000     host hostB
  4   hdd  4.00000         osd.4                              up  1.00000 1.00000
  5   hdd  4.00000         osd.5                              up  1.00000 1.00000
  6   hdd  4.00000         osd.6                              up  1.00000 1.00000
  7   hdd  4.00000         osd.7                              up  1.00000 1.00000
 -4       16.00000     host hostC
  8   hdd  4.00000         osd.8                              up  1.00000 1.00000
  9   hdd  4.00000         osd.9                              up  1.00000 1.00000
 10   hdd  4.00000         osd.10                             up  1.00000 1.00000
 11   hdd  4.00000         osd.11                             up  1.00000 1.00000

crushmap 管理

查询 crushmap

ceph osd getcrushmap -o crushmap.dump
crushtool -d crushmap.dump -o crushmap.txt


tail -n 25 crushmap.txt

root noah {
        id -12                          # do not change unnecessarily
        id -14 class hdd                # do not change unnecessarily
        # weight 24.000
        alg straw2
        hash 0  # rjenkins1
        item hostA weight 8.000
        item hostB weight 8.000
        item hostC weight 8.000
}
# rules
rule replicated_rule {                  <- 默认下只有这么一个 rules, 需要手动添加一个 rules 对应 noah (root)
        id 0
        type replicated
        min_size 1
        max_size 10
        step take default
        step chooseleaf firstn 0 type host
        step emit
}
# end crush map

添加 rule 规则

rule noah_rule {
        id 1
        type replicated
        min_size 1
        max_size 3
        step take noah                                    <- 这里定义了使用 root=noah 的机器
        step chooseleaf firstn 0 type host
        step emit
}

应用 rules

crushtool -c crushmap.txt -o crushmap.done
ceph osd setcrushmap -i crushmap.done

创建 pool

# ceph osd pool create noahpool 128 128
pool 'noahpool' created
# ceph osd pool set noahpool size 3
set pool 3 size to 3
# ceph osd pool set noahpool crush_rule noah_rule                    <--- 这里定义
set pool 3 crush_rule to noah_rule
# ceph osd pool application enable noahpool rbd
enabled application 'rbd' on pool 'noahpool'

检测 pool 信息

#  ceph osd pool get noahpool all
size: 3
min_size: 1
crash_replay_interval: 0
pg_num: 128
pgp_num: 128
crush_rule: noah_rule                             <--- 这里应用了 noah 机器群定义
hashpspool: true
nodelete: false
nopgchange: false
nosizechange: false
write_fadvise_dontneed: false
noscrub: false
nodeep-scrub: false
use_gmt_hitset: 1
auid: 0
fast_read: 0