今天作业:
场景:3mon集群
准备工作:停掉两个mon,并手动移除掉两个mon工作目录的/var/lib/ceph/mon
故障场景:3mon集群,故障2个mon的集群恢复
预期:恢复mon集群
#1.导出正常节点上的mon
[root@node01 ~]# monmaptool --create --clobber --fsid 3ce4bc3c-6cd6-488b-b803-ffd260f15131 --add node01 172.25.80.1:6789 --add node02 172.25.80.2:6789 --add node03 172.25.80.3:6789 monmap1.txt
#结果
monmaptool: monmap file monmap1.txt
monmaptool: set fsid to 3ce4bc3c-6cd6-488b-b803-ffd260f15131
monmaptool: writing epoch 0 to monmap1.txt (3 monitors)
#查看
[root@node01 ~]# ls |grep mon
monmap1.txt
monmaptool --print ./monmap1.txt
#手动移除异常节点
[root@node01 ~]# systemctl stop ceph-mon.target #所有mon节点操作
[root@node01 ~]# monmaptool ./monmap1.txt --rm node02
monmaptool: monmap file ./monmap1.txt
monmaptool: removing node02
monmaptool: writing epoch 0 to ./monmap1.txt (2 monitors)
[root@node01 ~]# monmaptool ./monmap1.txt --rm node03
monmaptool: monmap file ./monmap1.txt
monmaptool: removing node03
monmaptool: writing epoch 0 to ./monmap1.txt (1 monitors)
[root@node01 tmp]# monmaptool --print ./monmap1.txt
monmaptool: monmap file ./monmap1.txt
epoch 0
fsid 62ff4eca-a3ef-48ac-b81a-477f57711e35
last_changed 2024-07-15 21:07:29.172721
created 2024-07-15 21:07:29.172721
0: 172.25.80.1:6789/0 mon.node01 #只剩node01
#正常节点重写进去
[root@node01 tmp]# ceph-mon -i node01 --inject-monmap ./monmap1.txt
#重启正常节点服务
[root@node01 ~]# systemctl start ceph-mon.target
#此刻变成单节点,需要多等待一会儿
[root@node01 tmp]# ceph -s
cluster:
id: 62ff4eca-a3ef-48ac-b81a-477f57711e35
health: HEALTH_OK
services:
mon: 1 daemons, quorum node01
mgr: node01(active), standbys: node02, node03
osd: 9 osds: 9 up, 9 in
data:
pools: 0 pools, 0 pgs
objects: 0 objects, 0 B
usage: 9.0 GiB used, 81 GiB / 90 GiB avail
pgs:
#注意下面的格式,可以看到这个是部署时候存下来的keyring,值都一样
[root@node01 ceph-node01]# cat /root/ceph-deploy/ceph.mon.keyring
[mon.]
key = AQAYMJFmAAAAABAAGOezbiWey0bJhT0ldHN4mA==
caps mon = allow *
#注意下面的格式,可以看到这个是部署时候存下来的keyring,值都一样
[root@node01 ceph-node01]# cat /var/lib/ceph/mon/ceph-node01/keyring
[mon.]
key = AQAYMJFmAAAAABAAGOezbiWey0bJhT0ldHN4mA==
caps mon = "allow *"
#将mon的keyring拷贝到其他节点
[root@node01 ceph-node01]# scp /var/lib/ceph/mon/ceph-node01/keyring node02:/tmp/
keyring 100% 77 279.3KB/s 00:00
[root@node01 ceph-node01]# scp /var/lib/ceph/mon/ceph-node01/keyring node03:/tmp/
keyring 100% 77 211.1KB/s 00:00
#将monmap文件也拷贝过去
[root@node01 ~]# scp monmap1.txt node02:/tmp
monmap1.txt 100% 402 1.4MB/s 00:00
[root@node01 ~]# scp monmap1.txt node03:/tmp
monmap1.txt
#创建目录和授权
[root@node02 lib]# mkdir -p /var/lib/ceph/mon/ceph-node02
[root@node02 lib]# chown -R ceph:ceph /var/lib/ceph/mon
#重做,确认没报错
[root@node02 lib]# ceph-mon --cluster ceph -i node02 --mkfs --monmap /tmp/monmap1.txt --keyring /tmp/keyring -c /etc/ceph/ceph.conf
#确认文件生成
[root@node02 ceph-node02]# cat /var/lib/ceph/mon/ceph-node02/keyring
[mon.]
key = AQAYMJFmAAAAABAAGOezbiWey0bJhT0ldHN4mA==
caps mon = "allow *"
#重启这个节点上的mon服务
[root@node02 ceph-node02]# systemctl restart ceph-mon.target
[root@node02 ceph-node02]# systemctl status ceph-mon #ceph-mon@node02.service出现,之前应该没了
ceph-mon@ ceph-mon@node02.service ceph-mon.target
node03也一样
测试只是恢复成了单节点的mon,另外两个mon没加进去,后面测试新增mon节点来做高可用,新增mon已成功