monitor处理:
(gdb) bt
#0 SafeTimer::add_event_after (this=0x7efcfb7f23c0, seconds=0.9989011287689209, callback=callback@entry=0x7efc5c014be0) at /home/ceph/ceph/src/common/Timer.cc:118
#1 0x00007efcf0df330f in PaxosService::dispatch (this=0x7efcfb7fc950, op=...) at /home/ceph/ceph/src/mon/PaxosService.cc:132
#2 0x00007efcf0ca5926 in Monitor::dispatch_op (this=this@entry=0x7efcfb7f2330, op=...) at /home/ceph/ceph/src/mon/Monitor.cc:4320
#3 0x00007efcf0ca7829 in Monitor::_ms_dispatch (this=this@entry=0x7efcfb7f2330, m=m@entry=0x7efcb40bebe0) at /home/ceph/ceph/src/mon/Monitor.cc:4213
#4 0x00007efcf0cd3703 in Monitor::ms_dispatch (this=0x7efcfb7f2330, m=0x7efcb40bebe0) at /home/ceph/ceph/src/mon/Monitor.h:899
#5 0x00007efcf12006fa in ms_deliver_dispatch (m=0x7efcb40bebe0, this=0x7efcfb72fdf0) at /home/ceph/ceph/src/msg/Messenger.h:668
#6 DispatchQueue::entry (this=0x7efcfb72ff48) at /home/ceph/ceph/src/msg/DispatchQueue.cc:197
#7 0x00007efcf0fc0aed in DispatchQueue::DispatchThread::entry (this=<optimized out>) at /home/ceph/ceph/src/msg/DispatchQueue.h:101
#8 0x00007efcefa17dc5 in start_thread (arg=0x7efc57fff700) at pthread_create.c:308
#9 0x00007efced02473d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
(gdb) bt
#0 Paxos::commit_start (this=this@entry=0x7f29a9afc820) at /home/ceph/ceph/src/mon/Paxos.cc:866
#1 0x00007f299fe1a1e9 in Paxos::begin (this=this@entry=0x7f29a9afc820, v=...) at /home/ceph/ceph/src/mon/Paxos.cc:674
#2 0x00007f299fe1bc7a in Paxos::propose_pending (this=this@entry=0x7f29a9afc820) at /home/ceph/ceph/src/mon/Paxos.cc:1551
#3 0x00007f299fe1c6d6 in Paxos::finish_round (this=this@entry=0x7f29a9afc820) at /home/ceph/ceph/src/mon/Paxos.cc:1092
#4 0x00007f299fe1ebb2 in Paxos::commit_finish (this=0x7f29a9afc820) at /home/ceph/ceph/src/mon/Paxos.cc:943
#5 0x00007f299fe23241 in C_Committed::finish (this=0x7f2883c99aa0, r=<optimized out>) at /home/ceph/ceph/src/mon/Paxos.cc:828
#6 0x00007f299fce19f9 in Context::complete (this=0x7f2883c99aa0, r=<optimized out>) at /home/ceph/ceph/src/include/Context.h:70
#7 0x00007f299fe20960 in MonitorDBStore::C_DoTransaction::finish (this=0x7f28884132d0, r=<optimized out>) at /home/ceph/ceph/src/mon/MonitorDBStore.h:343
#8 0x00007f299fce19f9 in Context::complete (this=0x7f28884132d0, r=<optimized out>) at /home/ceph/ceph/src/include/Context.h:70
#9 0x00007f299ff46288 in Finisher::finisher_thread_entry (this=0x7f29a9a1a7c0) at /home/ceph/ceph/src/common/Finisher.cc:72
#10 0x00007f299ea4cdc5 in start_thread (arg=0x7f297bfff700) at pthread_create.c:308
#11 0x00007f299c05973d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
(gdb) bt
#0 AsyncConnection::send_message (this=0x7f295c04dc90, m=0x7f2970c53010) at /home/ceph/ceph/src/msg/async/AsyncConnection.cc:1939
#1 0x00007f299fe4d907 in OSDMonitor::share_map_with_random_osd (this=this@entry=0x7f29a9afe070) at /home/ceph/ceph/src/mon/OSDMonitor.cc:1643
#2 0x00007f299fe5c435 in OSDMonitor::update_from_paxos (this=0x7f29a9afe070, need_bootstrap=<optimized out>) at /home/ceph/ceph/src/mon/OSDMonitor.cc:579
#3 0x00007f299fe2456b in PaxosService::refresh (this=0x7f29a9afe070,need_bootstrap=need_bootstrap@entry=0x7f297bffd00f) at /home
/ceph/ceph/src/mon/PaxosService.cc:154
#4 0x00007f299fcbf7c3 in Monitor::refresh_from_paxos (this=0x7f29a9af88d0, need_bootstrap=need_bootstrap@entry=0x7f297bffd00f) at /home/ceph/ceph/src/mon/Monitor.cc:846
#5 0x00007f299fe0ee57 in Paxos::do_refresh (this=this@entry=0x7f29a9afc820) at /home/ceph/ceph/src/mon/Paxos.cc:1046
#6 0x00007f299fe1ea3d in Paxos::commit_finish (this=0x7f29a9afc820) at /home/ceph/ceph/src/mon/Paxos.cc:933
#7 0x00007f299fe23241 in C_Committed::finish (this=0x7f28938f1d30, r=<optimized out>) at /home/ceph/ceph/src/mon/Paxos.cc:828
#8 0x00007f299fce19f9 in Context::complete (this=0x7f28938f1d30, r=<optimized out>) at /home/ceph/ceph/src/include/Context.h:70
#9 0x00007f299fe20960 in MonitorDBStore::C_DoTransaction::finish (this=0x7f28946d09f0, r=<optimized out>) at /home/ceph/ceph/src/mon/MonitorDBStore.h:343
#10 0x00007f299fce19f9 in Context::complete (this=0x7f28946d09f0, r=<optimized out>) at /home/ceph/ceph/src/include/Context.h:70
#11 0x00007f299ff46288 in Finisher::finisher_thread_entry (this=0x7f29a9a1a7c0) at /home/ceph/ceph/src/common/Finisher.cc:72
#12 0x00007f299ea4cdc5 in start_thread (arg=0x7f297bfff700) at pthread_create.c:308
#13 0x00007f299c05973d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
osd 处理:
AsyncConnection::process -> DispatchQueue::enqueue --> DispatchQueue::entry
...
...
-> FileStore::queue_transactions
(gdb) bt
#0 FileStore::queue_transactions (this=0x55555eebd190, posr=0x55555f0094b0, tls=std::vector of length 1, capacity 1 = {...}, osd_
op=..., handle=0x0) at /home/ceph/ceph/src/os/filestore/FileStore.cc:2156
#1 0x0000555555aa32a1 in queue_transactions (handle=0x0, op=..., onreadable_sync=0x0, ondisk=<optimized out>, onreadable=0x55555e
ebd190, tls=std::vector of length 1, capacity 1 = {...}, osr=0x55555f0094b0, this=0x55555eebd190) at /home/ceph/ceph/src/os/Object
Store.h:1492
#2 ObjectStore::queue_transaction(ObjectStore::Sequencer*, ObjectStore::Transaction&&, Context*, Context*, Context*, boost::intru
sive_ptr<TrackedOp>, ThreadPool::TPHandle*) (this=0x55555eebd190, osr=0x55555f0094b0, t=t@entry=<unknown type in /home/ceph/ceph/b
uild/bin/ceph-osd, CU 0x1eae33, DIE 0x70b7cb>, onreadable=onreadable@entry=0x7ffeac029110, ondisk=<optimized out>, onreadable_sync
=onreadable_sync@entry=0x0, op=..., handle=0x0) at /home/ceph/ceph/src/os/ObjectStore.h:1480
#3 0x0000555555a52559 in OSD::handle_osd_map (this=this@entry=0x55555eff6d90, m=m@entry=0x7ffd2401a210) at /home/ceph/ceph/src/osd/OSD.cc:8117
#4 0x0000555555a65105 in OSD::_dispatch (this=this@entry=0x55555eff6d90, m=m@entry=0x7ffd2401a210) at /home/ceph/ceph/src/osd/OSD.cc:7491
#5 0x0000555555a65817 in OSD::ms_dispatch (this=0x55555eff6d90, m=0x7ffd2401a210) at /home/ceph/ceph/src/osd/OSD.cc:7182
#6 0x000055555630939a in ms_deliver_dispatch (m=0x7ffd2401a210, this=0x55555eeca2b0) at /home/ceph/ceph/src/msg/Messenger.h:668
#7 DispatchQueue::entry (this=0x55555eeca408) at /home/ceph/ceph/src/msg/DispatchQueue.cc:197
#8 0x00005555560e46fd in DispatchQueue::DispatchThread::entry (this=<optimized out>) at /home/ceph/ceph/src/msg/DispatchQueue.h:101
#9 0x00007ffff56a1dc5 in start_thread (arg=0x7ffebffff700) at pthread_create.c:308
#10 0x00007ffff479573d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
...
...
...
-> C_OnMapCommit::finish
-> PG::queue_peering_event
-> OSDService::queue_for_peering
-> peering_wq.queue
(gdb) bt
#0 PG::queue_peering_event (this=0x55555f084f70, evt=std::shared_ptr (count 1, weak 0) 0x7ffed40010a0) at /home/ceph/ceph/src/osd/PG.cc:6329
##1 0x0000555555aece6a in PG::queue_null (this=this@entry=0x55555f084f70, msg_epoch=37, query_epoch=<optimized out>) at /home/ceph/ceph/src/osd/PG.cc:6341
##2 0x0000555555a68379 in OSD::consume_map (this=this@entry=0x55555eff6cb0) at /home/ceph/ceph/src/osd/OSD.cc:8619
##3 0x0000555555a693e6 in OSD::_committed_osd_maps (this=0x55555eff6cb0, first=<optimized out>, last=<optimized out>, m=<optimized
# out>) at /home/ceph/ceph/src/osd/OSD.cc:8358
# #4 0x0000555555ac47e7 in C_OnMapCommit::finish (this=0x7ffeb4028fc0, r=<optimized out>) at /home/ceph/ceph/src/osd/OSD.cc:7840
# #5 0x0000555555a816c9 in Context::complete (this=0x7ffeb4028fc0, r=<optimized out>) at /home/ceph/ceph/src/include/Context.h:70
# #6 0x000055555604c508 in Finisher::finisher_thread_entry (this=0x55555eec6590) at /home/ceph/ceph/src/common/Finisher.cc:72
# #7 0x00007ffff56a1dc5 in start_thread (arg=0x7ffecffff700) at pthread_create.c:308
# #8 0x00007ffff479573d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
...
...
...
-> PeeringWQ:: _dequeue
-> PeeringWQ::_process
-> OSD::process_peering_events
…
-> PrimaryLogPG::kick_snap_trim
...
-> WaitReservation::WaitReservation()
(gdb) bt
#0 WaitReservation (ctx=..., this=0x7ffde0008850) at /home/ceph/ceph/src/osd/PrimaryLogPG.h:1683
#1 shallow_construct (outermostContextBase=..., pContext=<synthetic pointer>) at /home/ceph/ceph/build/boost/include/boost/statechart/state.hpp:89
#2 deep_construct (outermostContextBase=..., pContext=<synthetic pointer>) at /home/ceph/ceph/build/boost/include/boost/statechart/state.hpp:79
#3 deep_construct_inner_impl<boost::mpl::list<PrimaryLogPG::WaitReservation, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mp
l_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mp
l_::na> > (outermostContextBase=..., pInnerContext=<synthetic pointer>) at /home/ceph/ceph/build/boost/include/boost/statechart/simple_state.hpp:893
#4 deep_construct_inner<boost::mpl::list<PrimaryLogPG::WaitReservation, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na> > (outermostContextBase=..., pInnerContext=<synthetic pointer>) at /home/ceph/ceph/build/boost/include/boost/statechart/simple_state.hpp:625
#5 deep_construct (pContext=<synthetic pointer>, outermostContextBase=...) at /home/ceph/ceph/build/boost/include/boost/statechart/state.hpp:80
#6 construct (pContext=<synthetic pointer>, outermostContextBase=...) at /home/ceph/ceph/build/boost/include/boost/statechart/detail/constructor.hpp:93
#7 transit_impl<PrimaryLogPG::Trimming, PrimaryLogPG::SnapTrimmer, boost::statechart::detail::no_transition_function> (transitionAction=..., this=0x7ffde0005520) at /home/ceph/ceph/build/boost/include/boost/statechart/simple_state.hpp:798
#8 transit<PrimaryLogPG::Trimming> (this=0x7ffde0005520) at /home/ceph/ceph/build/boost/include/boost/statechart/simple_state.hpp:314
#9 PrimaryLogPG::NotTrimming::react (this=this@entry=0x7ffd94005220) at /home/ceph/ceph/src/osd/PrimaryLogPG.cc:14612
#10 0x0000555555c7104c in react<PrimaryLogPG::NotTrimming, boost::statechart::event_base, void const*> (eventType=<synthetic pointer>, evt=..., stt=...) at /home/ceph/ceph/build/boost/include/boost/statechart/custom_reaction.hpp:42
#11 local_react_impl<boost::mpl::list<boost::statechart::custom_reaction<PrimaryLogPG::KickTrim>, boost::statechart::transition<PrimaryLogPG::Reset, PrimaryLogPG::NotTrimming> >, boost::statechart::simple_state<PrimaryLogPG::NotTrimming, PrimaryLogPG::SnapTrimmer, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0> > (eventType=0x555556ab49f0 <boost::statechart::detail::id_holder<PrimaryLogPG::KickTrim>::idProvider_>, evt=..., stt=...) at /home(eventType=0x555556ab49f0 <boost::statechart::detail::id_holder<PrimaryLogPG::KickTrim>::idProvider_>, evt=..., stt=...) at /home/ceph/ceph/build/boost/include/boost/statechart/simple_state.hpp:816
#12 local_react<boost::mpl::list<boost::statechart::custom_reaction<PrimaryLogPG::KickTrim>, boost::statechart::transition<PrimaryLogPG::Reset, PrimaryLogPG::NotTrimming> > > (eventType=0x555556ab49f0 <boost::statechart::detail::id_holder<PrimaryLogPG::KickTrim>::idProvider_>, evt=..., this=0x7ffd94005220) at /home/ceph/ceph/build/boost/include/boost/statechart/simple_state.hpp:851
#13 boost::statechart::simple_state<PrimaryLogPG::NotTrimming, PrimaryLogPG::SnapTrimmer, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::react_impl (this=0x7ffd94005220, evt=..., eventType=0x555556ab49f0 <boost::statechart::detail::id_holder<PrimaryLogPG::KickTrim>::idProvider_>) at /home/ceph/ceph/build/boost/include/boost/statechart/simple_state.hpp:489
#14 0x0000555555bb6b72 in operator() (this=<synthetic pointer>) at /home/ceph/ceph/build/boost/include/boost/statechart/state_machine.hpp:87
#15 operator()<boost::statechart::detail::send_function<boost::statechart::detail::state_base<std::allocator<void>, boost::statechart::detail::rtti_policy>, boost::statechart::event_base, void const*>, boost::statechart::state_machine<PrimaryLogPG::SnapTrimmer, PrimaryLogPG::NotTrimming>::exception_event_handler> (this=0x55555f08f168, action=...) at /home/ceph/ceph/build/boost/include/boost/statechart/null_exception_translator.hpp:33
#16 send_event (evt=..., this=0x55555f08f110) at /home/ceph/ceph/build/boost/include/boost/statechart/state_machine.hpp:889
#17 process_event (evt=..., this=0x55555f08f110) at /home/ceph/ceph/build/boost/include/boost/statechart/state_machine.hpp:275
#18 PrimaryLogPG::kick_snap_trim (this=0x55555f08c530) at /home/ceph/ceph/src/osd/PrimaryLogPG.cc:4055
#19 0x0000555555b21047 in PG::RecoveryState::Active::react (this=this@entry=0x7ffde80070f0) at /home/ceph/ceph/src/osd/PG.cc:7772
#20 0x0000555555b89f56 in react<PG::RecoveryState::Active, boost::statechart::event_base, void const*> (eventType=<synthetic pointer>, evt=..., stt=...) at /home/ceph/ceph/build/boost/include/boost/statechart/custom_reaction.hpp:42
#21 local_react_impl<boost::mpl::list12<boost::statechart::custom_reaction<PG::ActMap>, boost::statechart::custom_reaction<PG::AdvMap>, boost::statechart::custom_reaction<PG::MInfoRec>, boost::statechart::custom_reaction<PG::MNotifyRec>, boost::statechart::custom_reaction<PG::MLogRec>, boost::statechart::custom_reaction<PG::Backfilled>, boost::statechart::custom_reaction<PG::AllReplicasActivated>, boost::statechart::custom_reaction<PG::DeferRecovery>, boost::statechart::custom_reaction<PG::DeferBackfill>, boost::statechart::custom_reaction<PG::UnfoundRecovery>, boost::statechart::custom_reaction<PG::UnfoundBackfill>, boost::statechart::custom_reaction<PG::DoRecovery> >, boost::statechart::simple_state<PG::RecoveryState::Active, PG::RecoveryState::Primary, PG::RecoveryState::Activating, (boost::statechart::history_mode)0> > (eventType=0x555556ab4870 <boost::statechart::detail::id_holder<PG::ActMap>::idProvider_>, evt=..., stt=...) at /home/ceph/ceph/build/boost/include/boost/statechart/simple_state.hpp:816
#22 local_react<boost::mpl::list12<boost::statechart::custom_reaction<PG::ActMap>, boost::statechart::custom_reaction<PG::AdvMap>,boost::statechart::custom_reaction<PG::MInfoRec>, boost::statechart::custom_reaction<PG::MNotifyRec>, boost::statechart::custom_reaction<PG::MLogRec>, boost::statechart::custom_reaction<PG::Backfilled>, boost::statechart::custom_reaction<PG::AllReplicasActivated>, boost::statechart::custom_reaction<PG::DeferRecovery>, boost::statechart::custom_reaction<PG::DeferBackfill>, boost::statechart::custom_reaction<PG::UnfoundRecovery>, boost::statechart::custom_reaction<PG::UnfoundBackfill>, boost::statechart::custom_reaction<PG::DoRecovery> > > (eventType=0x555556ab4870 <boost::statechart::detail::id_holder<PG::ActMap>::idProvider_>, evt=..., this=0x7ffde80070f0) at /home/ceph/ceph/build/boost/include/boost/statechart/simple_state.hpp:851
...
…
#27 0x0000555555b6212b in operator() (this=<synthetic pointer>) at /home/ceph/ceph/build/boost/include/boost/statechart/state_machine.hpp:87
#28 operator()<boost::statechart::detail::send_function<boost::statechart::detail::state_base<std::allocator<void>, boost::statechart::detail::rtti_policy>, boost::statechart::event_base, void const*>, boost::statechart::state_machine<PG::RecoveryState::RecoveryMachine, PG::RecoveryState::Initial>::exception_event_handler> (this=0x55555f08e858, action=...) at /home/ceph/ceph/build/boost/include/boost/statechart/null_exception_translator.hpp:33
#29 boost::statechart::state_machine<PG::RecoveryState::RecoveryMachine, PG::RecoveryState::Initial, std::allocator<void>, boost::statechart::null_exception_translator>::send_event (this=0x55555f08e800, evt=...) at /home/ceph/ceph/build/boost/include/boost/statechart/state_machine.hpp:889
#30 0x0000555555b298d0 in process_event (evt=..., this=0x55555f08e800) at /home/ceph/ceph/build/boost/include/boost/statechart/state_machine.hpp:275
#31 handle_event (rctx=0x0, evt=..., this=0x55555f08e800) at /home/ceph/ceph/src/osd/PG.h:2477
#32 PG::handle_activate_map (this=this@entry=0x55555f08c530, rctx=rctx@entry=0x7ffdf5bfc230) at /home/ceph/ceph/src/osd/PG.cc:6404
#33 0x0000555555a4fe12 in OSD::advance_pg (this=this@entry=0x55555eff6da0, osd_epoch=<optimized out>, pg=pg@entry=0x55555f08c530,handle=..., rctx=rctx@entry=0x7ffdf5bfc230, new_pgs=new_pgs@entry=0x7ffdf5bfc200) at /home/ceph/ceph/src/osd/OSD.cc:8521
#34 0x0000555555a502c3 in OSD::process_peering_events (this=0x55555eff6da0, pgs=Python Exception <type 'exceptions.ValueError'> Cannot find type const std::list<PG*, std::allocator<PG*> >::_Node:std::list, handle=...) at /home/ceph/ceph/src/osd/OSD.cc:9979
#35 0x0000555555ac00d7 in _process (handle=..., pgs=Python Exception <type 'exceptions.ValueError'> Cannot find type const std::list<PG*, std::allocator<PG*> >::_Node:std::list, this=<optimized out>) at /home/ceph/ceph/src/osd/OSD.h:1874
#36 ThreadPool::BatchWorkQueue<PG>::_void_process (this=<optimized out>, p=0x7ffde00033a0, handle=...) at /home/ceph/ceph/src/common/WorkQueue.h:120
#37 0x00005555560558bf in ThreadPool::worker (this=0x55555eff76c8, wt=0x55555f0af360) at /home/ceph/ceph/src/common/WorkQueue.cc:120
#38 0x00005555560567c0 in ThreadPool::WorkThread::entry (this=<optimized out>) at /home/ceph/ceph/src/common/WorkQueue.h:448
#39 0x00007ffff56a1dc5 in start_thread (arg=0x7ffdf5bfe700) at pthread_create.c:308
#40 0x00007ffff479573d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
...
...
-> WaitReservation::ReservationCB::finish
…
-> PrimaryLogPG::AwaitAsyncWork::AwaitAsyncWork()
-> OSDService::queue_for_snap_trim
-> op_shardedwq.queue
(gdb) bt
#0 PrimaryLogPG::AwaitAsyncWork::AwaitAsyncWork (this=0x7fdd4c0009d0, ctx=...) at /home/ceph/ceph/src/osd/PrimaryLogPG.cc:14640
##1 0x00007fdf66fcfd8f in shallow_construct (outermostContextBase=..., pContext=<synthetic pointer>) at /home/ceph/ceph/build/boos
#t/include/boost/statechart/state.hpp:89
##2 deep_construct (outermostContextBase=..., pContext=<synthetic pointer>) at /home/ceph/ceph/build/boost/include/boost/statechart/state.hpp:79
##3 construct (outermostContextBase=..., pContext=<synthetic pointer>) at /home/ceph/ceph/build/boost/include/boost/statechart/detail/constructor.hpp:93
##4 transit_impl<PrimaryLogPG::AwaitAsyncWork, PrimaryLogPG::SnapTrimmer, boost::statechart::detail::no_transition_function> (transitionAction=..., this=0x7fdd4c0009d0) at /home/ceph/ceph/build/boost/include/boost/statechart/simple_state.hpp:798
##5 transit<PrimaryLogPG::AwaitAsyncWork> (this=0x7fdd4c0009d0) at /home/ceph/ceph/build/boost/include/boost/statechart/simple_state.hpp:314
##6 PrimaryLogPG::WaitReservation::react (this=this@entry=0x7fdd48001320) at /home/ceph/ceph/src/osd/PrimaryLogPG.cc:14631
##7 0x00007fdf67060474 in react<PrimaryLogPG::WaitReservation, boost::statechart::event_base, void const*> (eventType=<synthetic p
#ointer>, evt=..., stt=...) at /home/ceph/ceph/build/boost/include/boost/statechart/custom_reaction.hpp:42
##8 local_react_impl<boost::mpl::list<boost::statechart::custom_reaction<PrimaryLogPG::SnapTrimReserved> >, boost::statechart::sim
#ple_state<PrimaryLogPG::WaitReservation, PrimaryLogPG::Trimming, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0> > (eventType=0x7fdf67eb39a0 <boost::statechart::detail::id_holder<PrimaryLogPG::SnapTrimReserved>::idProvider_>, evt=..., stt=...) at /home/ceph/ceph/build/boost/include/boost/statechart/simple_state.hpp:816
##9 local_react<boost::mpl::list<boost::statechart::custom_reaction<PrimaryLogPG::SnapTrimReserved> > > (eventType=0x7fdf67eb39a0<boost::statechart::detail::id_holder<PrimaryLogPG::SnapTrimReserved>::idProvider_>, evt=..., this=0x7fdd48001320) at /home/ceph/ceph/build/boost/include/boost/statechart/simple_state.hpp:851##10 boost::statechart::simple_state<PrimaryLogPG::WaitReservation, PrimaryLogPG::Trimming, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::react_impl (this=0x7fdd48001320, evt=..., eventType=0x7fdf67eb39a0 <boost::statechart::detail::id_holder<PrimaryLogPG::SnapTrimReserved>::idProvider_>) at /home/ceph/ceph/build/boost/include/boost/statechart/simple_state.hpp:489
##11 0x00007fdf6704aae6 in operator() (this=<synthetic pointer>) at /home/ceph/ceph/build/boost/include/boost/statechart/state_machine.hpp:8712 operator()<boost::statechart::detail::send_function<boost::statechart::detail::state_base<std::allocator<void>, boost::statechart::detail::rtti_policy>, boost::statechart::event_base, void const*>, boost::statechart::state_machine<PrimaryLogPG::SnapTrimmer
#, PrimaryLogPG::NotTrimming>::exception_event_handler> (this=0x7fde1c0138a8, action=...) at /home/ceph/ceph/build/boost/include/boost/statechart/null_exception_translator.hpp:33
##13 send_event (evt=..., this=0x7fde1c013850) at /home/ceph/ceph/build/boost/include/boost/statechart/state_machine.hpp:889
##14 process_event (evt=..., this=0x7fde1c013850) at /home/ceph/ceph/build/boost/include/boost/statechart/state_machine.hpp:275
##15 PrimaryLogPG::WaitReservation::ReservationCB::finish (this=0x7fdd48001300) at /home/ceph/ceph/src/osd/PrimaryLogPG.h:1667
##16 0x00007fdf66e806c9 in Context::complete (this=0x7fdd48001300, r=<optimized out>) at /home/ceph/ceph/src/include/Context.h:70
##17 0x00007fdf6744b508 in Finisher::finisher_thread_entry (this=0x7fdf7155b408) at /home/ceph/ceph/src/common/Finisher.cc:72
##18 0x00007fdf63ff5dc5 in start_thread (arg=0x7fdd47fff700) at pthread_create.c:308
##19 0x00007fdf630e973d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
-> OSD::ShardedOpWQ::_process
…
…
-> PrimaryLogPG::snap_trimmer
-> PrimaryLogPG::AwaitAsyncWork::react
-> PrimaryLogPG::trim_object
(gdb) bt
#0 PrimaryLogPG::snap_trimmer (this=0x55555f08c300, queued=100) at /home/ceph/ceph/src/osd/PrimaryLogPG.cc:4068
#1 0x0000555555a6f976 in internal_visit<PGSnapTrim&> (operand=..., this=<synthetic pointer>) at /home/ceph/ceph/build/boost/inclu
de/boost/variant/variant.hpp:1058
#2 visitation_impl_invoke_impl<boost::detail::variant::invoke_visitor<PGQueueable::RunVis, false>, void*, PGSnapTrim> (storage=0x
7ffd87ffcdc0, visitor=<synthetic pointer>) at /home/ceph/ceph/build/boost/include/boost/variant/detail/visitation_impl.hpp:114
#3 visitation_impl_invoke<boost::detail::variant::invoke_visitor<PGQueueable::RunVis, false>, void*, PGSnapTrim, boost::variant<b
oost::intrusive_ptr<OpRequest>, PGSnapTrim, PGScrub, PGRecovery>::has_fallback_type_> (t=0x0, storage=0x7ffd87ffcdc0, visitor=<syn
thetic pointer>, internal_which=<optimized out>) at /home/ceph/ceph/build/boost/include/boost/variant/detail/visitation_impl.hpp:1
57
#4 visitation_impl<mpl_::int_<0>, boost::detail::variant::visitation_impl_step<boost::mpl::l_iter<boost::mpl::l_item<mpl_::long_<
4l>, boost::intrusive_ptr<OpRequest>, boost::mpl::l_item<mpl_::long_<3l>, PGSnapTrim, boost::mpl::l_item<mpl_::long_<2l>, PGScrub,
boost::mpl::l_item<mpl_::long_<1l>, PGRecovery, boost::mpl::l_end> > > > >, boost::mpl::l_iter<boost::mpl::l_end> >, boost::detai
l::variant::invoke_visitor<PGQueueable::RunVis, false>, void*, boost::variant<boost::intrusive_ptr<OpRequest>, PGSnapTrim, PGScrub
, PGRecovery>::has_fallback_type_> (no_backup_flag=..., storage=0x7ffd87ffcdc0, visitor=<synthetic pointer>, logical_which=<optimi
zed out>, internal_which=<optimized out>) at /home/ceph/ceph/build/boost/include/boost/variant/detail/visitation_impl.hpp:238
#5 internal_apply_visitor_impl<boost::detail::variant::invoke_visitor<PGQueueable::RunVis, false>, void*> (storage=0x7ffd87ffcdc0
, visitor=<synthetic pointer>, logical_which=<optimized out>, internal_which=<optimized out>) at /home/ceph/ceph/build/boost/inclu
de/boost/variant/variant.hpp:2457
#6 internal_apply_visitor<boost::detail::variant::invoke_visitor<PGQueueable::RunVis, false> > (visitor=<synthetic pointer>, this
=0x7ffd87ffcdb8) at /home/ceph/ceph/build/boost/include/boost/variant/variant.hpp:2468
#7 apply_visitor<PGQueueable::RunVis> (visitor=..., this=0x7ffd87ffcdb8) at /home/ceph/ceph/build/boost/include/boost/variant/var
iant.hpp:2518
#8 apply_visitor<PGQueueable::RunVis, boost::variant<boost::intrusive_ptr<OpRequest>, PGSnapTrim, PGScrub, PGRecovery>&> (visitab
le=..., visitor=...) at /home/ceph/ceph/build/boost/include/boost/variant/detail/apply_visitor_unary.hpp:73
#9 run (handle=..., pg=..., osd=<optimized out>, this=0x7ffd87ffcdb8) at /home/ceph/ceph/src/osd/PGQueueable.h:140
#10 OSD::ShardedOpWQ::_process (this=<optimized out>, thread_index=<optimized out>, hb=<optimized out>) at /home/ceph/ceph/src/osd
/OSD.cc:10753
#11 0x000055555605471d in ShardedThreadPool::shardedthreadpool_worker (this=0x55555eff7850, thread_index=<optimized out>) at /home
/ceph/ceph/src/common/WorkQueue.cc:339
#12 0x00005555560566b0 in ShardedThreadPool::WorkThreadSharded::entry (this=<optimized out>) at /home/ceph/ceph/src/common/WorkQue
ue.h:689
#13 0x00007ffff56a1dc5 in start_thread (arg=0x7ffd87fff700) at pthread_create.c:308
#14 0x00007ffff479573d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113