上面左边是我的个人微信，如需进一步沟通，请加微信。 右边是我的公众号“Openstack私有云”，如有兴趣，请关注。
在实际生产使用过程中，难免会有ceph的osd硬盘损坏的时候，本文针对这种场景描述更换故障磁盘的操作，本文的操作环境是通过kolla部署的openstack，3个HA节点上各有3个osd，我们将osd.6 、osd.7 、osd.8 剔除并更换。
1、首先查看和收集相关的信息：
查看ceph集群空间使用：
#docker exec -it ceph_mon ceph df
GLOBAL:
SIZE AVAIL RAW USED %RAW USED
584G 548G 37548M 6.27
POOLS:
NAME ID USED %USED MAX AVAIL OBJECTS
rbd 0 0 0 1017M 0
.rgw.root 1 1588 0 1017M 4
default.rgw.control 2 0 0 1017M 8
default.rgw.data.root 3 670 0 1017M 2
default.rgw.gc 4 0 0 1017M 32
default.rgw.log 5 0 0 1017M 127
images 6 13286M 92.89 1017M 6674
volumes 7 230 0 1017M 7
backups 8 0 0 1017M 0
vms 9 5519M 84.44 1017M 1495
default.rgw.users.uid 10 209 0 1017M 2
default.rgw.buckets.index 11 0 0 1017M 1
default.rgw.buckets.data 12 0 0 1017M 1
查看OSD的分布权重：
# docker exec -it ceph_mon ceph osd tree
ID WEIGHT TYPE NAME UP/DOWN REWEIGHT PRIMARY-AFFINITY
-1 9.00000 root default
-2 3.00000 host 192.168.1.132
0 1.00000 osd.0 up 1.00000 1.00000
3 1.00000 osd.3 up 1.00000 1.00000
6 1.00000 osd.6 up 1.00000 1.00000
-3 3.00000 host 192.168.1.130
1 1.00000 osd.1 up 1.00000 1.00000
5 1.00000 osd.5 up 1.00000 1.00000
8 1.00000 osd.8 up 1.00000 1.00000
-4 3.00000 host 192.168.1.131
2 1.00000 osd.2 up 1.00000 1.00000
4 1.00000 osd.4 up 1.00000 1.00000
7 1.00000 osd.7 up 1.00000 1.00000
查看crushmap：
# docker exec -it ceph_mon ceph osd getcrushmap -o /var/log/kolla/ceph/crushmap.bin
got crush map from osdmap epoch 247
# docker exec -it ceph_mon crushtool -d /var/log/kolla/ceph/crushmap.bin -o /var/log/kolla/ceph/crushmap
# cd /var/lib/docker/volumes/kolla_logs/_data/ceph
# more crushmap
# begin crush map
tunable choose_local_tries 0
tunable choose_local_fallback_tries 0
tunable choose_total_tries 50
tunable chooseleaf_descend_once 1
tunable chooseleaf_vary_r 1
tunable straw_calc_version 1
# devices
device 0 osd.0
device 1 osd.1
device 2 osd.2
device 3 osd.3
device 4 osd.4
device 5 osd.5
device 6 osd.6
device 7 osd.7
device 8 osd.8
# types
type 0 osd
type 1 host
type 2 chassis
type 3 rack
type 4 row
type 5 pdu
type 6 pod
type 7 room
type 8 datacenter
type 9 region
type 10 root
# buckets
host 192.168.1.132 {
id -2# do not change unnecessarily
# weight 3.000
alg straw
hash 0# rjenkins1
item osd.0 weight 1.000
item osd.3 weight 1.000
item osd.6 weight 1.000
}
host 192.168.1.130 {
id -3# do not change unnecessarily
# weight 3.000
alg straw
hash 0# rjenkins1
item osd.1 weight 1.000
item osd.5 weight 1.000
item osd.8 weight 1.000
}
host 192.168.1.131 {
id -4# do not change unnecessarily
# weight 3.000
alg straw
hash 0# rjenkins1
item osd.2 weight 1.000
item osd.4 weight 1.000
item osd.7 weight 1.000
}
root default {
id -1# do not change unnecessarily
# weight 9.000
alg straw
hash 0# rjenkins1
item 192.168.1.132 weight 3.000
item 192.168.1.130 weight 3.000
item 192.168.1.131 weight 3.000
}
# rules
rule replicated_ruleset {
ruleset 0
type replicated
min_size 1
max_size 10
step take default
step chooseleaf firstn 0 type host
step emit
}
rule disks {
ruleset 1
type replicated
min_size 1
max_size 10
step take default
step chooseleaf firstn 0 type host
step emit
}
# end crush map
查看磁盘挂载情况：
[root@control01 ceph]# df -h
Filesystem Size Used Avail Use% Mounted on
/dev/mapper/centos-root 88G 14G 74G 16% /
devtmpfs 9.8G 0 9.8G 0% /dev
tmpfs 9.8G 0 9.8G 0% /dev/shm
tmpfs 9.8G 20M 9.8G 1% /run
tmpfs 9.8G 0 9.8G 0% /sys/fs/cgroup
/dev/sdd1 5.0G 4.8G 227M 96% /var/lib/ceph/osd/a007c495-e2e9-4a08-a565-83dfef1df30d
/dev/sdb1 95G 4.1G 91G 5% /var/lib/ceph/osd/b7267089-cae5-4e28-b9ff-a37c373c0d34
/dev/sdc1 95G 4.6G 91G 5% /var/lib/ceph/osd/c35eea02-f07d-4557-acdb-7280a571aaf9
检查健康状态：
# docker exec -it ceph_mon ceph health detail
HEALTH_ERR 1 full osd(s); 2 near full osd(s); full flag(s) set
osd.8 is full at 95%
osd.6 is near full at 90%
osd.7 is near full at 85%
full flag(s) set
# docker exec -it ceph_mon ceph -s
cluster 33932e16-1909-4d68-b085-3c01d0432adc
health HEALTH_ERR
1 full osd(s)
2 near full osd(s)
full flag(s) set
monmap e2: 3 mons at {192.168.1.130=192.168.1.130:6789/0,192.168.1.131=192.168.1.131:6789/0,192.168.1.132=192.168.1.132:6789/0}
election epoch 60, quorum 0,1,2 192.168.1.130,192.168.1.131,192.168.1.132
osdmap e247: 9 osds: 9 up, 9 in
flags nearfull,full,sortbitwise,require_jewel_osds
pgmap v179945: 640 pgs, 13 pools, 18806 MB data, 8353 objects
37548 MB used, 548 GB / 584 GB avail
640 active+clean
查看文件系统配置文件fstab：
# cat /etc/fstab
#
# /etc/fstab
# Created by anaconda on Mon Jan 22 13:32:42 2018
#
# Accessible filesystems, by reference, are maintained under '/dev/disk'
# See man pages fstab(5), findfs(8), mount(8) and/or blkid(8) for more info
#
/dev/mapper/centos-root / xfs defaults 0 0
UUID=c2c088f8-a530-4099-a9ef-0fd41508d304 /boot xfs defaults 0 0
UUID=7FA3-61ED /boot/efi vfat defaults,uid=0,gid=0,umask=0077,shortname=winnt 0 0
UUID=b7267089-cae5-4e28-b9ff-a37c373c0d34 /var/lib/ceph/osd/b7267089-cae5-4e28-b9ff-a37c373c0d34 xfs defaults,noatime 0 0
UUID=c35eea02-f07d-4557-acdb-7280a571aaf9 /var/lib/ceph/osd/c35eea02-f07d-4557-acdb-7280a571aaf9 xfs defaults,noatime 0 0
UUID=a007c495-e2e9-4a08-a565-83dfef1df30d /var/lib/ceph/osd/a007c495-e2e9-4a08-a565-83dfef1df30d xfs defaults,noatime 0 0
查看磁盘分区与挂在文件系统对应关系：
# lsblk|grep osd
├─sdb1 8:17 0 95G 0 part /var/lib/ceph/osd/b7267089-cae5-4e28-b9ff-a37c373c0d34
├─sdc1 8:33 0 95G 0 part /var/lib/ceph/osd/c35eea02-f07d-4557-acdb-7280a571aaf9
├─sdd1 8:49 0 5G 0 part /var/lib/ceph/osd/a007c495-e2e9-4a08-a565-83dfef1df30d
2.执行reweight，让这个osd上的数据迁移到其他osd，同步可以使用ceph -w查看实时数据迁移情况 ：
# docker exec -it ceph_mon ceph osd crush reweight osd.8 0.0 //同样的命令重平衡osd.7、osd.6 权重为0
3.剔除osd出集群：
# docker exec -it ceph_mon ceph osd out osd.8 //同样的命令out掉osd.7、osd.6 # docker exec -it ceph_mon ceph osd crush remove osd.8 //同样的命令crush remove掉osd.7、osd.6 # docker exec -it ceph_mon ceph auth del osd.8 //同样的命令auth del掉osd.7、osd.6 # docker stop ceph_osd_8 //在另外2台服务器，同样的命令stop掉ceph_osd_7、ceph_osd_6 # docker exec -it ceph_mon ceph osd rm osd.8 //同样的命令rm掉osd.7、osd.6
4.整个操作过程，可以使用 # docker exec -it ceph_mon ceph -w 查看实时的ceph集群数据变化，或者使用# docker exec -it ceph_mon ceph -s 查看总体情况。
# docker exec -it ceph_mon ceph -s
cluster 33932e16-1909-4d68-b085-3c01d0432adc
health HEALTH_OK
monmap e2: 3 mons at {192.168.1.130=192.168.1.130:6789/0,192.168.1.131=192.168.1.131:6789/0,192.168.1.132=192.168.1.132:6789/0}
election epoch 60, quorum 0,1,2 192.168.1.130,192.168.1.131,192.168.1.132
osdmap e297: 6 osds: 6 up, 6 in
flags sortbitwise,require_jewel_osds
pgmap v181612: 640 pgs, 13 pools, 14884 MB data, 7296 objects
30025 MB used, 540 GB / 569 GB avail
640 active+clean
如上所示，说明3个磁盘剔除成功，整个ceph集群数据pg分布到osd重新获得平衡。
5.替换磁盘后，重新使用kolla部署工具添加osd磁盘，可以参考我的另一篇博客“Openstack 之 kolla 部署ceph”中的后半部分，扩容磁盘，简单描述如下：
直接在宿主机上增加硬盘，然后对每一块硬盘打标签，然后重新执行deploy即可，步骤如下：
1）..OSD盘打标：
|
1
2
|
parted
/dev/sdd
-s -- mklabel gpt mkpart KOLLA_CEPH_OSD_BOOTSTRAP 1 -1
|
注意，整块盘作为一个OSD只能用标签KOLLA_CEPH_OSD_BOOTSTRAP ，如果日志分区单独使用SSD固态硬盘的分区，则使用不同标签，比如/dev/sdb 标签为KOLLA_CEPH_OSD_BOOTSTRAP_SDC ，日志分区/dev/sdh1 标签为：KOLLA_CEPH_OSD_BOOTSTRAP_SDC_J
2）.部署前的检查：
|
1
|
tools
/kolla_ansible
prechecks -i 3node
|
注意：3node是inventory文件，根据实际环境替换。
检查报错6780端口占用，查询这个端口是ceph_rgw容器占用，临时将这个容器关闭： docker stop ceph_rgw
3）.部署：
|
1
|
tools
/kolla_ansible
deploy -i 3node
|
注意：3node是inventory文件，根据实际环境替换。
部署完成后，将上面关闭的容器ceph_rgw启动：
|
1
|
docker start ceph_rgw
|
附：执行deploy的时候，同时通过ceph -s 监控ceph的情况如下：
[root@control02 mariadb]# docker exec -it ceph_mon ceph -s
cluster 33932e16-1909-4d68-b085-3c01d0432adc
health HEALTH_ERR
2 pgs are stuck inactive for more than 300 seconds
13 pgs degraded
83 pgs peering
2 pgs recovering
11 pgs recovery_wait
2 pgs stuck inactive
75 pgs stuck unclean
recovery 424/14592 objects degraded (2.906%)
recovery 1826/14592 objects misplaced (12.514%)
3/9 in osds are down
monmap e2: 3 mons at {192.168.1.130=192.168.1.130:6789/0,192.168.1.131=192.168.1.131:6789/0,192.168.1.132=192.168.1.132:6789/0}
election epoch 66, quorum 0,1,2 192.168.1.130,192.168.1.131,192.168.1.132
osdmap e354: 9 osds: 6 up, 9 in; 382 remapped pgs
flags sortbitwise,require_jewel_osds
pgmap v340179: 640 pgs, 13 pools, 14885 MB data, 7296 objects
30253 MB used, 540 GB / 569 GB avail
424/14592 objects degraded (2.906%)
1826/14592 objects misplaced (12.514%)
423 active+clean
113 active+remapped
81 remapped+peering
11 active+recovery_wait+degraded
8 active
2 active+recovering+degraded
2 peering
recovery io 118 MB/s, 102 objects/s
[root@control02 mariadb]# docker exec -it ceph_mon ceph -s
cluster 33932e16-1909-4d68-b085-3c01d0432adc
health HEALTH_WARN
11 pgs backfill_wait
85 pgs degraded
4 pgs recovering
81 pgs recovery_wait
31 pgs stuck unclean
recovery 4087/14835 objects degraded (27.550%)
recovery 486/14835 objects misplaced (3.276%)
monmap e2: 3 mons at {192.168.1.130=192.168.1.130:6789/0,192.168.1.131=192.168.1.131:6789/0,192.168.1.132=192.168.1.132:6789/0}
election epoch 66, quorum 0,1,2 192.168.1.130,192.168.1.131,192.168.1.132
osdmap e366: 9 osds: 9 up, 9 in; 11 remapped pgs
flags sortbitwise,require_jewel_osds
pgmap v340248: 640 pgs, 13 pools, 14885 MB data, 7296 objects
30572 MB used, 824 GB / 854 GB avail
4087/14835 objects degraded (27.550%)
486/14835 objects misplaced (3.276%)
544 active+clean
81 active+recovery_wait+degraded
11 active+remapped+wait_backfill
4 active+recovering+degraded
recovery io 54446 kB/s, 67 objects/s
[root@control02 mariadb]# docker exec -it ceph_mon ceph -s
cluster 33932e16-1909-4d68-b085-3c01d0432adc
health HEALTH_WARN
1 pgs backfill_wait
23 pgs degraded
3 pgs recovering
20 pgs recovery_wait
10 pgs stuck unclean
recovery 1373/14594 objects degraded (9.408%)
recovery 4/14594 objects misplaced (0.027%)
monmap e2: 3 mons at {192.168.1.130=192.168.1.130:6789/0,192.168.1.131=192.168.1.131:6789/0,192.168.1.132=192.168.1.132:6789/0}
election epoch 66, quorum 0,1,2 192.168.1.130,192.168.1.131,192.168.1.132
osdmap e380: 9 osds: 9 up, 9 in; 1 remapped pgs
flags sortbitwise,require_jewel_osds
pgmap v340305: 640 pgs, 13 pools, 14885 MB data, 7296 objects
30234 MB used, 825 GB / 854 GB avail
1373/14594 objects degraded (9.408%)
4/14594 objects misplaced (0.027%)
616 active+clean
20 active+recovery_wait+degraded
3 active+recovering+degraded
1 active+remapped+wait_backfill
recovery io 99837 kB/s, 35 objects/s
[root@control02 mariadb]#
[root@control02 mariadb]# docker exec -it ceph_mon ceph -s
cluster 33932e16-1909-4d68-b085-3c01d0432adc
health HEALTH_OK
monmap e2: 3 mons at {192.168.1.130=192.168.1.130:6789/0,192.168.1.131=192.168.1.131:6789/0,192.168.1.132=192.168.1.132:6789/0}
election epoch 66, quorum 0,1,2 192.168.1.130,192.168.1.131,192.168.1.132
osdmap e382: 9 osds: 9 up, 9 in
flags sortbitwise,require_jewel_osds
pgmap v340363: 640 pgs, 13 pools, 14885 MB data, 7296 objects
30204 MB used, 825 GB / 854 GB avail
640 active+clean
从上面的监控记录可以看到，添加osd之后整个ceph集群有一个重新平衡数据的过程。从开始的HEALTH_ERR状态，到HEALTH_WARN，最后到HEALTH_OK 。
2018年3月29日注：
服务器系统重启后发现centos操作系统进入紧急模式，提示如下：
welcome to emergency mode！after logging in ，type “journalctl -xb” to view system logs，“systemctl reboot” to reboot ，“systemctl default” to try again to boot into default mode。
give root password for maintenance
最后发现是替换了osd硬盘之后， 文件/etc/fstab 里面关于原来的磁盘配置项还在，所以在系统启动时找不到相应的设备文件而出错。解决办法就是将这个文件里面对应被替换的osd硬盘的记录删除：
[root@control01 etc]# more /etc/fstab # # /etc/fstab # Created by anaconda on Mon Jan 22 13:32:42 2018 # # Accessible filesystems, by reference, are maintained under '/dev/disk' # See man pages fstab(5), findfs(8), mount(8) and/or blkid(8) for more info # /dev/mapper/centos-root / xfs defaults 0 0 UUID=c2c088f8-a530-4099-a9ef-0fd41508d304 /boot xfs defaults 0 0 UUID=7FA3-61ED /boot/efi vfat defaults,uid=0,gid=0,umask=0077,shortname=winnt 0 0 UUID=b7267089-cae5-4e28-b9ff-a37c373c0d34 /var/lib/ceph/osd/b7267089-cae5-4e28-b9ff-a37c373c0d34 xfs defaults,noatime 0 0 UUID=c35eea02-f07d-4557-acdb-7280a571aaf9 /var/lib/ceph/osd/c35eea02-f07d-4557-acdb-7280a571aaf9 xfs defaults,noatime 0 0 #UUID=a007c495-e2e9-4a08-a565-83dfef1df30d /var/lib/ceph/osd/a007c495-e2e9-4a08-a565-83dfef1df30d xfs defaults,noatime 0 0 UUID=e758ebfe-e4fc-41e2-a493-b66ad5aecd0f /var/lib/ceph/osd/e758ebfe-e4fc-41e2-a493-b66ad5aecd0f xfs defaults,noatime 0 0
如上面所示，倒数第二行对应的就是原来被替换的OSD硬盘。如何找出需要删除的记录，可以用这条命令：
[root@control01 etc]# lsblk |grep osd ├─sdb1 8:17 0 95G 0 part /var/lib/ceph/osd/b7267089-cae5-4e28-b9ff-a37c373c0d34 ├─sdc1 8:33 0 95G 0 part /var/lib/ceph/osd/c35eea02-f07d-4557-acdb-7280a571aaf9 ├─sdd1 8:49 0 95G 0 part /var/lib/ceph/osd/e758ebfe-e4fc-41e2-a493-b66ad5aecd0f
转载于:https://blog.51cto.com/yuweibing/2086375
691
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
