stack
Thread 6 (Thread 0x7f2991b3b700 (LWP 26533)):
#0 0x00007f29928e16ac in __lll_lock_wait_private () from /lib64/libc.so.6
#1 0x00007f299285f952 in _L_lock_35 () from /lib64/libc.so.6
#2 0x00007f2992857670 in ptmalloc_lock_all () from /lib64/libc.so.6
#3 0x00007f299289af82 in fork () from /lib64/libc.so.6
#4 0x000000000043b8c7 in ProcessPool::Spawn (this=this@entry=0x7f29840009f0, ctx=...) at build/debug/src/module/kprocesspool/kprocesspool.cpp:406
#5 0x000000000043c06f in ProcessPool::Restart (this=this@entry=0x7f29840009f0, pid=26551) at build/debug/src/module/kprocesspool/kprocesspool.cpp:384
#6 0x000000000043c61a in SignalHandler (signo=<optimized out>) at build/debug/src/module/kprocesspool/kprocesspool.cpp:103
#7 <signal handler called>
#8 0x00007f29925cca82 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#9 0x00007f299354d445 in apsara::PTCond::wait(apsara::PTMutex const&, long) [clone .constprop.116] () from /data/wei.xiew/myworks/zark_worker/extralibs/apsara_common-72/lib64/libapsara_common.so.72
#10 0x00007f299354e29b in apsara::logging::AsyncFileSink::WriteToFile() () from /data/wei.xiew/myworks/zark_worker/extralibs/apsara_common-72/lib64/libapsara_common.so.72
#11 0x00007f299354f753 in std::_Function_handler<apsara::Any (), std::_Bind<std::_Mem_fn<int (apsara::logging::AsyncFileSink::*)()> (apsara::logging::AsyncFileSink*)> >::_M_invoke(std::_Any_data const&) () from /data/wei.xiew/myworks/zark_worker/extralibs/apsara_common-72/lib64/libapsara_common.so.72
#12 0x00007f299359bf37 in apsara::ThreadWrapperFunction(void*) () from /data/wei.xiew/myworks/zark_worker/extralibs/apsara_common-72/lib64/libapsara_common.so.72
#13 0x00007f29925c8dc5 in start_thread () from /lib64/libpthread.so.0
#14 0x00007f29928d3ced in clone () from /lib64/libc.so.6
Thread 5 (Thread 0x7f299133a700 (LWP 26534)):
#0 0x00007f29928e16ac in __lll_lock_wait_private () from /lib64/libc.so.6
#1 0x00007f299285f952 in _L_lock_35 () from /lib64/libc.so.6
#2 0x00007f2992857670 in ptmalloc_lock_all () from /lib64/libc.so.6
#3 0x00007f299289af82 in fork () from /lib64/libc.so.6
#4 0x000000000043b8c7 in ProcessPool::Spawn (this=this@entry=0x7f29840009f0, ctx=...) at build/debug/src/module/kprocesspool/kprocesspool.cpp:406
#5 0x000000000043c06f in ProcessPool::Restart (this=this@entry=0x7f29840009f0, pid=26550) at build/debug/src/module/kprocesspool/kprocesspool.cpp:384
#6 0x000000000043c61a in SignalHandler (signo=<optimized out>) at build/debug/src/module/kprocesspool/kprocesspool.cpp:103
#7 <signal handler called>
#8 0x00007f29925cca82 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#9 0x00007f299354d445 in apsara::PTCond::wait(apsara::PTMutex const&, long) [clone .constprop.116] () from /data/wei.xiew/myworks/zark_worker/extralibs/apsara_common-72/lib64/libapsara_common.so.72
#10 0x00007f299354e29b in apsara::logging::AsyncFileSink::WriteToFile() () from /data/wei.xiew/myworks/zark_worker/extralibs/apsara_common-72/lib64/libapsara_common.so.72
#11 0x00007f299354f753 in std::_Function_handler<apsara::Any (), std::_Bind<std::_Mem_fn<int (apsara::logging::AsyncFileSink::*)()> (apsara::logging::AsyncFileSink*)> >::_M_invoke(std::_Any_data const&) () from /data/wei.xiew/myworks/zark_worker/extralibs/apsara_common-72/lib64/libapsara_common.so.72
#12 0x00007f299359bf37 in apsara::ThreadWrapperFunction(void*) () from /data/wei.xiew/myworks/zark_worker/extralibs/apsara_common-72/lib64/libapsara_common.so.72
#13 0x00007f29925c8dc5 in start_thread () from /lib64/libpthread.so.0
#14 0x00007f29928d3ced in clone () from /lib64/libc.so.6
Thread 4 (Thread 0x7f2990b39700 (LWP 26535)):
#0 0x00007f29925cca82 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007f299354d445 in apsara::PTCond::wait(apsara::PTMutex const&, long) [clone .constprop.116] () from /data/wei.xiew/myworks/zark_worker/extralibs/apsara_common-72/lib64/libapsara_common.so.72
#2 0x00007f299354e29b in apsara::logging::AsyncFileSink::WriteToFile() () from /data/wei.xiew/myworks/zark_worker/extralibs/apsara_common-72/lib64/libapsara_common.so.72
#3 0x00007f299354f753 in std::_Function_handler<apsara::Any (), std::_Bind<std::_Mem_fn<int (apsara::logging::AsyncFileSink::*)()> (apsara::logging::AsyncFileSink*)> >::_M_invoke(std::_Any_data const&) () from /data/wei.xiew/myworks/zark_worker/extralibs/apsara_common-72/lib64/libapsara_common.so.72
#4 0x00007f299359bf37 in apsara::ThreadWrapperFunction(void*) () from /data/wei.xiew/myworks/zark_worker/extralibs/apsara_common-72/lib64/libapsara_common.so.72
#5 0x00007f29925c8dc5 in start_thread () from /lib64/libpthread.so.0
#6 0x00007f29928d3ced in clone () from /lib64/libc.so.6
Thread 3 (Thread 0x7f2989e11700 (LWP 26545)):
why
malloc() is not a function that can be safely called from a signal handler. It’s not a async-signal-safe function. So, you should never call malloc() from a signal handler.
You are only allowed to call a limited set of functons from a signal handler. See the man signal-safety for the list of functions you can safely call from a signal handler.
more examples
this thread
_int_malloc()
at frame 8,
<signal handler called>
at frame 7,
try to malloc at frame 2 ptmalloc_lock_all()
Thread 3 (Thread 0x7fd3c7114700 (LWP 12875)):
#0 0x00007fd3cfbe46ac in __lll_lock_wait_private () from /lib64/libc.so.6
#1 0x00007fd3cfb6296d in _L_lock_59 () from /lib64/libc.so.6
#2 0x00007fd3cfb5a6ac in ptmalloc_lock_all () from /lib64/libc.so.6
#3 0x00007fd3cfb9df82 in fork () from /lib64/libc.so.6
#4 0x00000000004325ce in ProcessPool::Spawn (this=0x7fd3c00009f0, ctx=...) at build/debug/src/module/kprocesspool/kprocesspool.cpp:429
#5 0x000000000043265b in ProcessPool::Restart (this=this@entry=0x7fd3c00009f0, pid=<optimized out>) at build/debug/src/module/kprocesspool/kprocesspool.cpp:410
#6 0x00000000004397ea in SignalHandler (signo=<optimized out>) at build/debug/src/module/kprocesspool/kprocesspool.cpp:102
#7 <signal handler called>
#8 0x00007fd3cfb5e33e in _int_malloc () from /lib64/libc.so.6
#9 0x00007fd3cfb6026c in malloc () from /lib64/libc.so.6
#10 0x00007fd3d0418e48 in operator new (sz=35) at ../../.././libstdc++-v3/libsupc++/new_op.cc:49
#11 0x00007fd3d04766d9 in allocate (this=<optimized out>, __n=<optimized out>) at /tmp/gcc-4.9.2/x86_64-unknown-linux-gnu/libstdc++-v3/include/ext/new_allocator.h:104
#12 std::string::_Rep::_S_create (__capacity=__capacity@entry=10, __old_capacity=__old_capacity@entry=0, __alloc=...) at /tmp/gcc-4.9.2/x86_64-unknown-linux-gnu/libstdc++-v3/include/bits/basic_string.tcc:607
#13 0x00007fd3d0901b65 in std::string::_S_construct<char const*> (__beg=0x43f76b "__LOGGER__", __end=<optimized out>, __a=...) at /usr/include/c++/4.9.2/bits/basic_string.tcc:138
#14 0x00007fd3d0478486 in _S_construct_aux<char const*> (__a=..., __end=<optimized out>, __beg=0x43f76b "__LOGGER__") at /tmp/gcc-4.9.2/x86_64-unknown-linux-gnu/libstdc++-v3/include/bits/basic_string.h:1743
#15 _S_construct<char const*> (__a=..., __end=<optimized out>, __beg=0x43f76b "__LOGGER__") at /tmp/gcc-4.9.2/x86_64-unknown-linux-gnu/libstdc++-v3/include/bits/basic_string.h:1764
#16 std::basic_string<char, std::char_traits<char>, std::allocator<char> >::basic_string (this=0x7fd3c7113160, __s=0x43f76b "__LOGGER__", __a=...) at /tmp/gcc-4.9.2/x86_64-unknown-linux-gnu/libstdc++-v3/include/bits/basic_string.tcc:215
#17 0x00007fd3d08a86b0 in apsara::logging::LogBuilder::operator() (this=this@entry=0x7fd3c71133c0, key=key@entry=0x43f76b "__LOGGER__", value=...) at build/release64/common/logging/logging.cpp:863
#18 0x0000000000434cfa in ProcessPool::WaitServiceReadiness (this=this@entry=0x7fd3c00009f0, ctx=..., defaultReadiness=...) at build/debug/src/module/kprocesspool/kprocesspool.cpp:559
#19 0x000000000043b146 in ProcessPool::RestartAll (this=this@entry=0x7fd3c00009f0, group=..., newFirst=newFirst@entry=true) at build/debug/src/module/kprocesspool/kprocesspool.cpp:517
#20 0x000000000041a769 in KProcessPoolUnitTest::RestartAllTest (this=0x1f4e9f0) at build/debug/src/module/unittest/kprocesspool_unittest.cpp:277
#21 0x0000000000424ddb in operator() (this=<optimized out>) at /usr/include/c++/4.9.2/functional:2439
#22 operator() (this=<optimized out>) at build/debug/src/test/unittest.h:1017
#23 std::_Function_handler<apsara::Any (), apsara::DropReturnFunctionWrapper>::_M_invoke(std::_Any_data const&) (__functor=...) at /usr/include/c++/4.9.2/functional:2025
#24 0x00007fd3d09088a7 in operator() (this=<optimized out>) at /usr/include/c++/4.9.2/functional:2439
#25 apsara::ThreadWrapperFunction (arg=<optimized out>) at build/release64/common/threading/threading.cpp:46
#26 0x00007fd3cf8cbdc5 in start_thread () from /lib64/libpthread.so.0
#27 0x00007fd3cfbd6ced in clone () from /lib64/libc.so.6
refer:
- https://github.com/ArcticaProject/nx-libs/issues/854
- https://stackoverflow.com/questions/40049751/malloc-inside-linux-signal-handler-cause-deadlock
- https://man7.org/linux/man-pages/man7/signal-safety.7.html
- https://blog.csdn.net/breaksoftware/article/details/100567271
- https://blog.csdn.net/dinghuiyang/article/details/132490765
- https://blog.csdn.net/DaliATOM/article/details/78293024
- https://web.stanford.edu/class/archive/cs/cs110/cs110.1196/static/lectures/08-race-conditions-deadlock-data-integrity/lecture-08-race-conditions-deadlock-data-integrity.pdf