ceph作业03

今天作业:
场景:3mon集群
准备工作:停掉两个mon，并手动移除掉两个mon工作目录的/var/lib/ceph/mon
故障场景:3mon集群，故障2个mon的集群恢复
预期:恢复mon集群

#1.导出正常节点上的mon
[root@node01 ~]# monmaptool  --create  --clobber  --fsid  3ce4bc3c-6cd6-488b-b803-ffd260f15131 --add  node01  172.25.80.1:6789  --add  node02  172.25.80.2:6789   --add  node03  172.25.80.3:6789   monmap1.txt

#结果
monmaptool: monmap file monmap1.txt
monmaptool: set fsid to 3ce4bc3c-6cd6-488b-b803-ffd260f15131
monmaptool: writing epoch 0 to monmap1.txt (3 monitors)


#查看
[root@node01 ~]# ls |grep mon
monmap1.txt

monmaptool --print ./monmap1.txt


#手动移除异常节点
[root@node01 ~]# systemctl stop ceph-mon.target   #所有mon节点操作

[root@node01 ~]# monmaptool  ./monmap1.txt --rm   node02
monmaptool: monmap file ./monmap1.txt
monmaptool: removing node02
monmaptool: writing epoch 0 to ./monmap1.txt (2 monitors)

[root@node01 ~]# monmaptool  ./monmap1.txt --rm   node03
monmaptool: monmap file ./monmap1.txt
monmaptool: removing node03
monmaptool: writing epoch 0 to ./monmap1.txt (1 monitors)

[root@node01 tmp]# monmaptool --print ./monmap1.txt
monmaptool: monmap file ./monmap1.txt
epoch 0
fsid 62ff4eca-a3ef-48ac-b81a-477f57711e35
last_changed 2024-07-15 21:07:29.172721
created 2024-07-15 21:07:29.172721
0: 172.25.80.1:6789/0 mon.node01   #只剩node01


#正常节点重写进去
[root@node01 tmp]# ceph-mon -i node01   --inject-monmap ./monmap1.txt

#重启正常节点服务
[root@node01 ~]# systemctl start ceph-mon.target

#此刻变成单节点，需要多等待一会儿
[root@node01 tmp]# ceph -s
  cluster:
    id:     62ff4eca-a3ef-48ac-b81a-477f57711e35
    health: HEALTH_OK

  services:
    mon: 1 daemons, quorum node01
    mgr: node01(active), standbys: node02, node03
    osd: 9 osds: 9 up, 9 in

  data:
    pools:   0 pools, 0 pgs
    objects: 0  objects, 0 B
    usage:   9.0 GiB used, 81 GiB / 90 GiB avail
    pgs:




#注意下面的格式，可以看到这个是部署时候存下来的keyring,值都一样
[root@node01 ceph-node01]# cat /root/ceph-deploy/ceph.mon.keyring
[mon.]
key = AQAYMJFmAAAAABAAGOezbiWey0bJhT0ldHN4mA==
caps mon = allow *

#注意下面的格式，可以看到这个是部署时候存下来的keyring,值都一样
[root@node01 ceph-node01]# cat /var/lib/ceph/mon/ceph-node01/keyring
[mon.]
	key = AQAYMJFmAAAAABAAGOezbiWey0bJhT0ldHN4mA==
	caps mon = "allow *"


#将mon的keyring拷贝到其他节点
[root@node01 ceph-node01]# scp /var/lib/ceph/mon/ceph-node01/keyring  node02:/tmp/
keyring                                                                                               100%   77   279.3KB/s   00:00
[root@node01 ceph-node01]# scp /var/lib/ceph/mon/ceph-node01/keyring  node03:/tmp/
keyring                                                                                               100%   77   211.1KB/s   00:00


#将monmap文件也拷贝过去
[root@node01 ~]# scp   monmap1.txt node02:/tmp
monmap1.txt                                                                                           100%  402     1.4MB/s   00:00

[root@node01 ~]# scp   monmap1.txt node03:/tmp
monmap1.txt

#创建目录和授权
[root@node02 lib]# mkdir  -p  /var/lib/ceph/mon/ceph-node02
[root@node02 lib]# chown -R ceph:ceph   /var/lib/ceph/mon


#重做，确认没报错
[root@node02 lib]# ceph-mon  --cluster    ceph  -i  node02 --mkfs  --monmap  /tmp/monmap1.txt  --keyring  /tmp/keyring  -c  /etc/ceph/ceph.conf

#确认文件生成
[root@node02 ceph-node02]# cat  /var/lib/ceph/mon/ceph-node02/keyring
[mon.]
	key = AQAYMJFmAAAAABAAGOezbiWey0bJhT0ldHN4mA==
	caps mon = "allow *"


#重启这个节点上的mon服务
[root@node02 ceph-node02]# systemctl   restart   ceph-mon.target

[root@node02 ceph-node02]# systemctl   status   ceph-mon    #ceph-mon@node02.service出现，之前应该没了
ceph-mon@                ceph-mon@node02.service  ceph-mon.target


node03也一样

测试只是恢复成了单节点的mon，另外两个mon没加进去，后面测试新增mon节点来做高可用，新增mon已成功