ceph集群安装（块设备）3PC（3mon+3osd+3mgr）+1ceph-deploy机器

ceph节点操作：
创建sudo用户，加入公钥，允许部署节点访问。
disable /etc/selinux/config
setenforce 0
yum install yum-plugin-priorities -y
时间同步：
chronyc makestep
部署节点操作：
yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
cat << EOM > /etc/yum.repos.d/ceph.repo
[ceph-noarch]
name=Ceph noarch packages
baseurl=https://download.ceph.com/rpm-mimic/el7/noarch
enabled=1
gpgcheck=1
type=rpm-md
gpgkey=https://download.ceph.com/keys/release.asc
EOM

md5-3efbe24a1057a87acb1e0a39134b9779

cat << EOM >>/etc/hosts
192.168.200.237 ceph-1
192.168.200.238 ceph-2
192.168.200.239 ceph-3
EOM

md5-3efbe24a1057a87acb1e0a39134b9779

创建部署配置文件等使用的目录，所有操作在该目录下执行：
mkdir my-cluster
cd my-cluster

md5-3efbe24a1057a87acb1e0a39134b9779

清理旧的ceph：
ceph-deploy purge {ceph-node} [{ceph-node}]
ceph-deploy purgedata {ceph-node} [{ceph-node}]
ceph-deploy forgetkeys
rm ceph.*

md5-08e97920284d1ea0729ce10639a57f02

ceph-deploy --username fungaming new ceph-1 ceph-2 ceph-3

md5-3efbe24a1057a87acb1e0a39134b9779

vi ceph.conf
public network = 192.168.200.0/24
osd pool default pg num = 128
osd pool default pgp num = 128
[mon]
mon allow pool delete = true

md5-3efbe24a1057a87acb1e0a39134b9779

export CEPH_DEPLOY_REPO_URL=http://mirrors.163.com/ceph/rpm-mimic/el7
export CEPH_DEPLOY_GPG_URL=http://mirrors.163.com/ceph/keys/release.asc
安装ceph
ceph-deploy --username fungaming install ceph-1 ceph-2 ceph-3
安装mon
ceph-deploy --username fungaming mon create ceph-1 ceph-2 ceph-3
ceph-deploy --username fungaming mon create-initial
同步配置
ceph-deploy --username fungaming admin ceph-1 ceph-2 ceph-3
安装mgr
ceph-deploy --username fungaming mgr ceph-1 ceph-2 ceph-3
检查ceph osd使用的磁盘
ceph-deploy disk list ceph-1 ceph-2 ceph-3
擦净磁盘
ceph-deploy disk zap ceph-1 /dev/sdb
安装osd
ceph-deploy --username fungaming osd create --data /dev/sdb ceph-1
ceph-deploy --username fungaming osd create --data /dev/sdb ceph-2
ceph-deploy --username fungaming osd create --data /dev/sdb ceph-3
ceph-deploy --username fungaming osd create --data /dev/sdc ceph-1
ceph-deploy --username fungaming osd create --data /dev/sdc ceph-2
ceph-deploy --username fungaming osd create --data /dev/sdc ceph-3

#安装完成。

md5-45eceb139399beebae44dfa61d03e2e9

根据官网计算方法http://ceph.com/pgcalc/
Total PGs = ((Total_number_of_OSD * 100) / max_replication_count) / pool_count
osd:6 relications: 3 pool: 2
pg=100, 接近128 , 所以pg_num 设为128

md5-3efbe24a1057a87acb1e0a39134b9779

重新推送配置：
ceph-deploy --username fungaming --overwrite-conf config push ceph-1 ceph-2 ceph-3
推送后ceph节点重启操作：
systemctl restart ceph-mon.target

md5-bb56810970953244af44fbe6aad74720

ceph health detail
ceph -s
检查mon法定人数状态
ceph quorum_status --format json-pretty

md5-4d1e61805c7501e885237cabe2353724

创建root，host
ceph osd crush add-bucket root-nvme root
ceph osd crush add-bucket root-ssd root
ceph osd crush add-bucket host1-nvme host
ceph osd crush add-bucket host2-nvme host
ceph osd crush add-bucket host3-nvme host
ceph osd crush add-bucket host1-ssd host
ceph osd crush add-bucket host2-ssd host
ceph osd crush add-bucket host3-ssd host
把host加入root
ceph osd crush move host1-ssd root=root-ssd
ceph osd crush move host2-ssd root=root-ssd
ceph osd crush move host3-ssd root=root-ssd
ceph osd crush move host3-nvme root=root-nvme
ceph osd crush move host2-nvme root=root-nvme
ceph osd crush move host1-nvme root=root-nvme
把osd加入host
ceph osd crush move osd.0 host=host1-nvme
ceph osd crush move osd.1 host=host2-nvme
ceph osd crush move osd.2 host=host3-nvme
ceph osd crush move osd.3 host=host1-ssd
ceph osd crush move osd.4 host=host2-ssd
ceph osd crush move osd.5 host=host3-ssd

md5-3efbe24a1057a87acb1e0a39134b9779

导出CRUSH map：
ceph osd getcrushmap -o crushmap.txt
crushtool -d crushmap.txt -o crushmap-decompile

md5-3efbe24a1057a87acb1e0a39134b9779

修改规则vi crushmap-decompile：

rules

rule nvme {
id 1
type replicated
min_size 1
max_size 10
step take root-nvme
step chooseleaf firstn 0 type host
step emit
}

rule ssd {
id 2
type replicated
min_size 1
max_size 10
step take root-ssd
step chooseleaf firstn 0 type host
step emit
}

md5-3efbe24a1057a87acb1e0a39134b9779

导入CRUSH map：
crushtool -c crushmap-decompile -o crushmap-compiled
ceph osd setcrushmap -i crushmap-compiled

md5-3efbe24a1057a87acb1e0a39134b9779

配置ceph.conf，让OSD启动不更新crushmap
[osd]
osd crush update on start = false

md5-9d8e4c7d8f5ece0532aaeed81ad586d1

###手动修改class标签
#查看当前集群布局
ceph osd tree
#查看crush class
ceph osd crush class ls
#删除osd.0，osd.1，osd.2的class
for i in 0 1 2;do ceph osd crush rm-device-class osd.KaTeX parse error: Expected 'EOF', got '#' at position 8: i;done #̲设置osd.0，osd.1，o…i;done
#创建一个优先使用nvme设备的crush规则
ceph osd crush rule create-replicated rule-auto-nvme default host nvme
#创建一个优先使用ssd设备的crush规则
ceph osd crush rule create-replicated rule-auto-ssd default host ssd
#查看集群的rule
ceph osd crush rule ls

md5-29b4d7ae93fad73933d7a981ab7ac830

#创建
ceph osd pool create pool-ssd 128
#创建并设置规则：
ceph osd pool create pool-ssd 128 128 rule-auto-ssd

设置pool的类型

ceph osd pool application enable pool-ssd rbd

md5-2c3e5569b6b7a81b60cf6bb3445c63f5

#查看与设置pg num
ceph osd pool get hdd_pool pg_num
ceph osd pool get hdd_pool pgp_num
ceph osd pool set rbd pg_num 256
ceph osd pool set rbd pgp_num 256
#ods与pool状态
ceph osd tree
rados df
ceph df
ceph osd lspools

#查看crush_rule及pool的rule
ceph osd crush rule ls
ceph osd pool get pool-ssd crush_rule
#查看pool详情
ceph osd pool ls detail

md5-9eccebae6357321ac792dddc7f01bfd0

rados -p pool-ssd ls
echo “hahah” >test.txt
rados -p pool-ssd put test test.txt
rados -p pool-ssd ls
#查看该对象的osd组
ceph osd map pool-ssd test
#删除
rados rm -p pool-ssd test

md5-900ff76fea595ea98fe5abdd9d11fc00

#删除pool
ceph osd pool delete pool-ssd pool-ssd --yes-i-really-really-mean-it
#设置 pool的crush_rule
ceph osd pool set pool-ssd crush_rule rule-ssd

md5-de58c22f7109a8dd4ee9a5bd7635f55b

export CEPH_DEPLOY_REPO_URL=http://mirrors.163.com/ceph/rpm-mimic/el7
export CEPH_DEPLOY_GPG_URL=http://mirrors.163.com/ceph/keys/release.asc
ceph-deploy install ceph-client
ceph-deploy admin ceph-client

md5-4e8112f57074799244f94ce6c464bd83

#创建一个块设备image
rbd create pool-ssd/foo --size 1024 --image-feature layering
#把 image 映射为块设备
rbd map pool-ssd/foo --name client.admin

mkfs.xfs /dev/rbd0
mount /dev/rbd0 /mnt/rbdtest/

md5-32dae1cfad34885200ab4c2dc63c824b

#ceph客户端操作
镜像信息
rbd ls pool-ssd
rbd info pool-ssd/foo
查看已映射块设备
rbd showmapped
取消块设备映射
rbd unmap /dev/rbd0
删除块设备映像
rbd rm pool-ssd/foo