本站以分享各种运维经验和运维所需要的技能为主
《python零基础入门》:python零基础入门学习
《python运维脚本》: python运维脚本实践
《shell》:shell学习
《terraform》持续更新中:terraform_Aws学习零基础入门到最佳实战
《k8》暂未更新
《docker学习》暂未更新
《ceph学习》ceph日常问题解决分享
《日志收集》ELK+各种中间件
《运维日常》运维日常
《linux》运维面试100问
ceph集群在执行ceph-volume命令卡住了
背景:
在创建好集群,mon起来后,创建了几个lvm之后,准备开始创建osd,结果命令卡住了
# sudo /usr/sbin/ceph-volume --cluster ceph lvm prepare --bluestore --data ceph-osd-168-vg/osd-168-data --osd-id 168 --osd-fsid `uuidgen`
^CTraceback (most recent call last):
File "/usr/sbin/ceph-volume", line 9, in <module>
load_entry_point('ceph-volume==1.0.0', 'console_scripts', 'ceph-volume')()
File "/usr/lib/python2.7/site-packages/ceph_volume/main.py", line 38, in __init__
self.main(self.argv)
File "/usr/lib/python2.7/site-packages/ceph_volume/decorators.py", line 59, in newfunc
return f(*a, **kw)
File "/usr/lib/python2.7/site-packages/ceph_volume/main.py", line 149, in main
terminal.dispatch(self.mapper, subcommand_args)
File "/usr/lib/python2.7/site-packages/ceph_volume/terminal.py", line 194, in dispatch
instance.main()
File "/usr/lib/python2.7/site-packages/ceph_volume/devices/lvm/main.py", line 40, in main
terminal.dispatch(self.mapper, self.argv)
File "/usr/lib/python2.7/site-packages/ceph_volume/terminal.py", line 194, in dispatch
instance.main()
File "/usr/lib/python2.7/site-packages/ceph_volume/devices/lvm/prepare.py", line 355, in main
self.args = parser.parse_args(self.argv)
File "/usr/lib64/python2.7/argparse.py", line 1688, in parse_args
args, argv = self.parse_known_args(args, namespace)
File "/usr/lib64/python2.7/argparse.py", line 1720, in parse_known_args
namespace, args = self._parse_known_args(args, namespace)
File "/usr/lib64/python2.7/argparse.py", line 1926, in _parse_known_args
start_index = consume_optional(start_index)
File "/usr/lib64/python2.7/argparse.py", line 1866, in consume_optional
take_action(action, args, option_string)
File "/usr/lib64/python2.7/argparse.py", line 1778, in take_action
argument_values = self._get_values(action, argument_strings)
File "/usr/lib64/python2.7/argparse.py", line 2218, in _get_values
value = self._get_value(action, arg_string)
File "/usr/lib64/python2.7/argparse.py", line 2247, in _get_value
result = type_func(arg_string)
File "/usr/lib/python2.7/site-packages/ceph_volume/util/arg_validators.py", line 16, in __call__
device = Device(string)
File "/usr/lib/python2.7/site-packages/ceph_volume/util/device.py", line 91, in __init__
self._parse()
File "/usr/lib/python2.7/site-packages/ceph_volume/util/device.py", line 112, in _parse
sys_info.devices = disk.get_devices()
File "/usr/lib/python2.7/site-packages/ceph_volume/util/disk.py", line 779, in get_devices
if lvm.is_lv(diskname):
File "/usr/lib/python2.7/site-packages/ceph_volume/api/lvm.py", line 277, in is_lv
lvs = lvs if lvs is not None else Volumes()
File "/usr/lib/python2.7/site-packages/ceph_volume/api/lvm.py", line 780, in __init__
self._populate()
File "/usr/lib/python2.7/site-packages/ceph_volume/api/lvm.py", line 784, in _populate
for lv_item in get_api_lvs():
File "/usr/lib/python2.7/site-packages/ceph_volume/api/lvm.py", line 322, in get_api_lvs
verbose_on_failure=False
File "/usr/lib/python2.7/site-packages/ceph_volume/process.py", line 201, in call
**kw
File "/usr/lib64/python2.7/subprocess.py", line 711, in __init__
errread, errwrite)
File "/usr/lib64/python2.7/subprocess.py", line 1308, in _execute_child
data = _eintr_retry_call(os.read, errpipe_read, 1048576)
File "/usr/lib64/python2.7/subprocess.py", line 478, in _eintr_retry_call
return func(*args)
KeyboardInterrupt
折腾了一下午,第二天早上灵机一动,简直茅塞顿开,在分析和排错过程中,我在其他的节点也尝试创建osd,是正常的,部署过程完全一致,这时就排除环境问题了,接着就开始去看日志
# tail -f /var/log/ceph/ceph-volume.log
发现有很多lvm报错:
[2023-11-17 09:56:28,867][ceph_volume.process][INFO ] stdout ";"/dev/ceph-osd-39-48-vg/osd-39-dbs";"osd-39-dbs";"ceph-osd-39-48-vg";"TH2NbO-FkpP-OwWg-Takg-dA3S-HHiG-rjnMSS";"145.00g
[2023-11-17 09:56:28,868][ceph_volume.process][INFO ] stdout ";"/dev/ceph-osd-39-48-vg/osd-39-wal";"osd-39-wal";"ceph-osd-39-48-vg";"4LeOJZ-aO4A-lcGc-9E2T-dgGg-4slY-Z67f02";"4.00g
[2023-11-17 09:56:28,868][ceph_volume.process][INFO ] stdout ";"/dev/ceph-osd-39-48-vg/osd-40-dbs";"osd-40-dbs";"ceph-osd-39-48-vg";"deiH09-zMRi-IxPd-2hrx-fcNy-x7xn-C6HMns";"145.00g
[2023-11-17 09:56:28,868][ceph_volume.process][INFO ] stdout ";"/dev/ceph-osd-39-48-vg/osd-40-wal";"osd-40-wal";"ceph-osd-39-48-vg";"PTSVas-ELiR-2Obg-98XM-UyyA-hNsS-Iw9qdk";"4.00g
[2023-11-17 09:56:28,868][ceph_volume.process][INFO ] stdout ";"/dev/ceph-osd-39-48-vg/osd-41-dbs";"osd-41-dbs";"ceph-osd-39-48-vg";"hBeM5X-UMDG-5DZU-m7ph-t4tz-ynop-d3pR4v";"145.00g
[2023-11-17 09:56:28,868][ceph_volume.process][INFO ] stdout ";"/dev/ceph-osd-39-48-vg/osd-41-wal";"osd-41-wal";"ceph-osd-39-48-vg";"Vi2tq1-q2pj-nl0C-2NT3-TPGk-74Nb-YmXDxq";"4.00g
[2023-11-17 09:56:28,868][ceph_volume.process][INFO ] stdout ";"/dev/ceph-osd-41-vg/osd-41-data";"osd-41-data";"ceph-osd-41-vg";"PXOBjY-RRsG-od59-bQC7-eSQB-8z1p-1iteTT";"<9.10t
[2023-11-17 09:56:28,868][ceph_volume.process][INFO ] stdout ";"/dev/ceph-osd-6-15-vg/osd-6-dbs";"osd-6-dbs";"ceph-osd-6-15-vg";"ZFlD3R-mrpZ-IWDE-TGPu-E9Iv-K2Oz-jQw0aO";"145.00g
[2023-11-17 09:56:28,868][ceph_volume.process][INFO ] stdout ";"/dev/ceph-osd-6-15-vg/osd-6-wal";"osd-6-wal";"ceph-osd-6-15-vg";"Dr92zd-fx2w-9dI4-6nXD-Xeiw-WkwZ-lnjUPl";"4.00g
[2023-11-17 09:56:28,868][ceph_volume.process][INFO ] stdout ";"/dev/ceph-osd-6-15-vg/osd-7-dbs";"osd-7-dbs";"ceph-osd-6-15-vg";"uf4XgZ-GpZX-A7Ra-ycGe-LFjr-Ljk3-CCQjj0";"145.00g
[2023-11-17 09:56:28,868][ceph_volume.process][INFO ] stdout ";"/dev/ceph-osd-6-15-vg/osd-7-wal";"osd-7-wal";"ceph-osd-6-15-vg";"chzf6p-7BGU-RuXK-T3ST-XeRv-ZyzP-0pfa0k";"4.00g
[2023-11-17 09:56:28,868][ceph_volume.process][INFO ] stdout ";"/dev/ceph-osd-6-15-vg/osd-8-dbs";"osd-8-dbs";"ceph-osd-6-15-vg";"LARUIQ-Nrms-1EIE-SWsj-X6is-R3N2-CUfr3U";"145.00g
[2023-11-17 09:56:28,868][ceph_volume.process][INFO ] stdout ";"/dev/ceph-osd-6-15-vg/osd-8-wal";"osd-8-wal";"ceph-osd-6-15-vg";"IdtQeV-hiJ2-AO4s-VkA7-CA5p-PF7L-rt5uWs";"4.00g
[2023-11-17 09:56:28,868][ceph_volume.process][INFO ] stdout ";"/dev/ceph-osd-9-18-vg/osd-10-dbs";"osd-10-dbs";"ceph-osd-9-18-vg";"8kURuP-f7i3-60Uz-DfBG-46XW-2w1G-fIcfqF";"145.00g
[2023-11-17 09:56:28,868][ceph_volume.process][INFO ] stdout ";"/dev/ceph-osd-9-18-vg/osd-10-wal";"osd-10-wal";"ceph-osd-9-18-vg";"tD81bi-CR8V-Xwxk-hB0B-2dhQ-dgX3-KauAaZ";"4.00g
[2023-11-17 09:56:28,868][ceph_volume.process][INFO ] stdout ";"/dev/ceph-osd-9-18-vg/osd-11-dbs";"osd-11-dbs";"ceph-osd-9-18-vg";"h625h3-ZcFv-8Jz5-1N1E-0Yhw-y4Zi-cxIEia";"145.00g
[2023-11-17 09:56:28,868][ceph_volume.process][INFO ] stdout ";"/dev/ceph-osd-9-18-vg/osd-11-wal";"osd-11-wal";"ceph-osd-9-18-vg";"crZrX3-b7GC-PnZ9-62Oa-xBDB-FPT4-W3iWUi";"4.00g
[2023-11-17 09:56:28,868][ceph_volume.process][INFO ] stdout ";"/dev/ceph-osd-9-18-vg/osd-9-dbs";"osd-9-dbs";"ceph-osd-9-18-vg";"THZUaH-s8Mi-On3C-rDre-6Ayi-rqzO-u47te4";"145.00g
[2023-11-17 09:56:28,868][ceph_volume.process][INFO ] stdout ";"/dev/ceph-osd-9-18-vg/osd-9-wal";"osd-9-wal";"ceph-osd-9-18-vg";"GCylED-R8IW-epti-Guo4-ETs0-zpEM-Of3vOX";"4.00g
[2023-11-17 09:56:28,868][ceph_volume.process][INFO ] stderr WARNING: Not using device /dev/sdbj for PV BIDWbR-Nqsb-XLx9-qjxV-XptS-fGUv-WlJ0Gn.
[2023-11-17 09:56:28,868][ceph_volume.process][INFO ] stderr WARNING: PV BIDWbR-Nqsb-XLx9-qjxV-XptS-fGUv-WlJ0Gn prefers device /dev/sdcz because device is used by LV.
[2023-11-17 09:56:28,871][ceph_volume.process][INFO ] Running command: /usr/sbin/dmsetup splitname --noheadings --separator=';' --nameprefixes /dev/mapper/ceph--osd--39--48--vg-osd--41--wal
[2023-11-17 09:56:29,854][ceph_volume.process][INFO ] stdout DM_VG_NAME='/dev/mapper/ceph-osd-39-48-vg'';'DM_LV_NAME='osd-41-wal'';'DM_LV_LAYER=''
[2023-11-17 09:56:29,854][ceph_volume.process][INFO ] Running command: /usr/sbin/lvs --noheadings --readonly --separator=";" -o lv_tags,lv_path,lv_name,vg_name,lv_uuid,lv_size
[2023-11-17 09:56:31,098][ceph_volume.process][INFO ] stdout ";"/dev/ceph-osd-0-9-vg/osd-0-dbs";"osd-0-dbs";"ceph-osd-0-9-vg";"I0fBjd-e8zD-2flG-lYpf-RmTS-JxU6-LKaOEw";"145.00g
[2023-11-17 09:56:31,098][ceph_volume.process][INFO ] stdout ";"/dev/ceph-osd-0-9-vg/osd-0-wal";"osd-0-wal";"ceph-osd-0-9-vg";"lv5xke-qHVy-rc40-GqKJ-8p8A-J8En-n0bGFy";"4.00g
[2023-11-17 09:56:31,098][ceph_volume.process][INFO ] stdout ";"/dev/ceph-osd-0-9-vg/osd-1-dbs";"osd-1-dbs";"ceph-osd-0-9-vg";"x4nJYK-ou71-Uv13-pmPQ-V0c6-IR5P-ctwt0A";"145.00g
[2023-11-17 09:56:31,098][ceph_volume.process][INFO ] stdout ";"/dev/ceph-osd-0-9-vg/osd-1-wal";"osd-1-wal";"ceph-osd-0-9-vg";"TU7pF2-PUps-mrZ8-JOu1-MO20-c6PC-f5k0fx";"4.00g
[2023-11-17 09:56:31,098][ceph_volume.process][INFO ] stdout ";"/dev/ceph-osd-0-9-vg/osd-2-dbs";"osd-2-dbs";"ceph-osd-0-9-vg";"RK45OK-XRs1-ENJf-CdTz-rDlh-xIyR-coZSag";"145.00g
[2023-11-17 09:56:31,098][ceph_volume.process][INFO ] stdout ";"/dev/ceph-osd-0-9-vg/osd-2-wal";"osd-2-wal";"ceph-osd-0-9-vg";"0KVc8s-0j7b-sfw1-DU6o-x5Pi-Ia5g-DGOJMe";"4.00g
[2023-11-17 09:56:31,098][ceph_volume.process][INFO ] stdout ";"/dev/ceph-osd-12-21-vg/osd-12-dbs";"osd-12-dbs";"ceph-osd-12-21-vg";"ZeEi2z-5NcX-SegB-QOOr-cKEl-TP0E-2TP9CA";"145.00g
[2023-11-17 09:56:31,098][ceph_volume.process][INFO ] stdout ";"/dev/ceph-osd-12-21-vg/osd-12-wal";"osd-12-wal";"ceph-osd-12-21-vg";"nB1Bfm-3ieg-yhut-fI2y-kI6K-D1np-kf3Yrf";"4.00g
[2023-11-17 09:56:31,098][ceph_volume.process][INFO ] stdout ";"/dev/ceph-osd-12-21-vg/osd-13-dbs";"osd-13-dbs";"ceph-osd-12-21-vg";"JGh7Vn-M8y7-EXSw-2yrG-v03y-EWTp-9NGz1E";"145.00g
[2023-11-17 09:56:31,098][ceph_volume.process][INFO ] stdout ";"/dev/ceph-osd-12-21-vg/osd-13-wal";"osd-13-wal";"ceph-osd-12-21-vg";"GIeJJL-qfvn-WpZH-qilN-YaIp-hBwb-HNZGD4";"4.00g
[2023-11-17 09:56:31,098][ceph_volume.process][INFO ] stdout ";"/dev/ceph-osd-12-21-vg/osd-14-dbs";"osd-14-dbs";"ceph-osd-12-21-vg";"6BioYO-UODr-CTAh-o7t4-1e7j-QmuZ-qkPsrd";"145.00g
[2023-11-17 09:56:31,098][ceph_volume.process][INFO ] stdout ";"/dev/ceph-osd-12-21-vg/osd-14-wal";"osd-14-wal";"ceph-osd-12-21-vg";"BJHpXs-4IWs-Z62T-aeFA-bdZt-AT8w-eh51Re";"4.00g
[2023-11-17 09:56:31,098][ceph_volume.process][INFO ] stdout ";"/dev/ceph-osd-15-24-vg/osd-15-dbs";"osd-15-dbs";"ceph-osd-15-24-vg";"2eEaOT-iKDj-m3hM-YUWH-09VS-OGGb-aGrmi2";"145.00g
[2023-11-17 09:56:31,098][ceph_volume.process][INFO ] stdout ";"/dev/ceph-osd-15-24-vg/osd-15-wal";"osd-15-wal";"ceph-osd-15-24-vg";"abl70F-pOho-lkzh-993c-x78P-n1Er-igz1wT";"4.00g
这时侯就意识到应该是lvm有问题,就把前面创建的lvm全部清除,重来就可: