linux 挂死问题定位分析

1.生成coredump 文件

根据生成的coredump文件，下载到编译环境中，结合gdb查找符号位，定位挂死位置

/opt/toolchains/gcc-4.7.3-cmc-201308-20140423/bin/mips-linux-uclibc-gdb ./linux_cmts_sym coredump
GNU gdb (GDB) 7.5.1
Copyright (C) 2012 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law.  Type "show copying"
and "show warranty" for details.
This GDB was configured as "--host=x86_64-unknown-linux-gnu --target=mips-linux-uclibc".
For bug reporting instructions, please see:
<http://www.gnu.org/software/gdb/bugs/>...
Reading symbols from /home/heshuanglai/project/CMC3219/linux_cmts_sym...done.

warning: core file may not match specified executable file.
[New LWP 231]

warning: Could not load shared library symbols for 9 libraries, e.g. /lib/libdl.so.0.
Use the "info sharedlibrary" command to see the complete listing.
Do you need "set solib-search-path" or "set sysroot"?
Core was generated by `./linux_cmts -cmd -app'.
Program terminated with signal 6, Aborted.
#0  0x2ac81dd0 in ?? ()
(gdb) info sharedlibrary  查看lib连接库
From        To          Syms Read   Shared Object Library
                        No          /lib/libdl.so.0
                        No          /lib/librt.so.0
                        No          /lib/libpthread.so.0
                        No          /lib/libstdc++.so.6
                        No          /lib/libm.so.0
                        No          /lib/libc.so.0
                        No          /lib/ld-uClibc.so.0
                        No          /lib/libubacktrace.so.0
                        No          /lib/libgcc_s.so.1
(gdb) bt 查找挂死位置
#0  0x2ac81dd0 in ?? ()
#1  0x2ac7a1c0 in ?? ()
Backtrace stopped: previous frame identical to this frame (corrupt stack?)
(gdb)
(gdb) set solib-absolute-prefix /opt/toolchains/gcc-4.7.3-cmc-201308-20140423/mips-linux-uclibc/sysroot/ 加载运行环境
Reading symbols from /opt/toolchains/gcc-4.7.3-cmc-201308-20140423/mips-linux-uclibc/sysroot/lib/libdl.so.0...done.
Loaded symbols for /opt/toolchains/gcc-4.7.3-cmc-201308-20140423/mips-linux-uclibc/sysroot/lib/libdl.so.0
Reading symbols from /opt/toolchains/gcc-4.7.3-cmc-201308-20140423/mips-linux-uclibc/sysroot/lib/librt.so.0...done.
Loaded symbols for /opt/toolchains/gcc-4.7.3-cmc-201308-20140423/mips-linux-uclibc/sysroot/lib/librt.so.0
Reading symbols from /opt/toolchains/gcc-4.7.3-cmc-201308-20140423/mips-linux-uclibc/sysroot/lib/libpthread.so.0...done.
Loaded symbols for /opt/toolchains/gcc-4.7.3-cmc-201308-20140423/mips-linux-uclibc/sysroot/lib/libpthread.so.0
Reading symbols from /opt/toolchains/gcc-4.7.3-cmc-201308-20140423/mips-linux-uclibc/sysroot/lib/libm.so.0...done.
Loaded symbols for /opt/toolchains/gcc-4.7.3-cmc-201308-20140423/mips-linux-uclibc/sysroot/lib/libm.so.0
Reading symbols from /opt/toolchains/gcc-4.7.3-cmc-201308-20140423/mips-linux-uclibc/sysroot/lib/libc.so.0...done.
Loaded symbols for /opt/toolchains/gcc-4.7.3-cmc-201308-20140423/mips-linux-uclibc/sysroot/lib/libc.so.0
Reading symbols from /opt/toolchains/gcc-4.7.3-cmc-201308-20140423/mips-linux-uclibc/sysroot/lib/ld-uClibc.so.0...done.
Loaded symbols for /opt/toolchains/gcc-4.7.3-cmc-201308-20140423/mips-linux-uclibc/sysroot/lib/ld-uClibc.so.0
Reading symbols from /opt/toolchains/gcc-4.7.3-cmc-201308-20140423/mips-linux-uclibc/sysroot/lib/libubacktrace.so.0...done.
Loaded symbols for /opt/toolchains/gcc-4.7.3-cmc-201308-20140423/mips-linux-uclibc/sysroot/lib/libubacktrace.so.0
Reading symbols from /opt/toolchains/gcc-4.7.3-cmc-201308-20140423/mips-linux-uclibc/sysroot/lib/libgcc_s.so.1...done.
Loaded symbols for /opt/toolchains/gcc-4.7.3-cmc-201308-20140423/mips-linux-uclibc/sysroot/lib/libgcc_s.so.1
(gdb) 
(gdb) where 显示挂死详细地方
#0  0x2ac81dd0 in __GI_raise (sig=6) at libpthread/nptl/sysdeps/unix/sysv/linux/raise.c:67
#1  0x2ac7a1c0 in __GI_abort () at libc/stdlib/abort.c:89
#2  0x2ac2f0b0 in __GI___assert (assertion=<optimized out>, filename=<optimized out>, linenumber=<optimized out>, function=<optimized out>)
    at libc/misc/assert/__assert.c:64
#3  0x00c5197c in WatchdogThread::WatchdogThread (this=0x1a118c8) at ../../WatchdogThread.cpp:115
#4  0x00c51d98 in WatchdogThread::Instance () at ../../WatchdogThread.cpp:624
#5  0x00802530 in CrashReport_ExitApp () at ../../CrashReport.cpp:415
#6  0x00804388 in CrashReport_Handler (n=<optimized out>, si=0x7fc29638, context=0x7fc296b8) at ../../CrashReport.cpp:330
#7  <signal handler called>
#8  0x2ac81dd0 in __GI_raise (sig=6) at libpthread/nptl/sysdeps/unix/sysv/linux/raise.c:67
#9  0x2ac7a1c0 in __GI_abort () at libc/stdlib/abort.c:89
#10 0x2ac2f0b0 in __GI___assert (assertion=<optimized out>, filename=<optimized out>, linenumber=<optimized out>, function=<optimized out>)
    at libc/misc/assert/__assert.c:64
#11 0x0065c34c in BDCPThread::BDCPThread (this=0x1a115a8, startSuspended=<optimized out>, initialPriority=<optimized out>) at ../../BDCP/BDCP.cpp:228
#12 0x0065c3d8 in theBDCPThread () at ../../BDCP/BDCP.cpp:176
#13 0x00ca89e0 in HalStartup () at ../../startup.cpp:369
#14 0x0061e2d8 in main (argc=<optimized out>, args=0x7fc29c64) at ../../startup.cpp:1208
(gdb) 

根据符号查看具体挂死到哪里
stack 0x2b915000-0x2b918000                                                                                                         
entry 2ac81d6c    called from 2ac7a1b8                                                                                              
entry 2ac7a13c    called from 2ac2f0a8                                                                                              
entry 2ac2f040    called from 2aaf6088                                                                                              
entry 2aaf5b10    called from 0097db78  BcmLnxMutexSemaphore::Lock(BcmMutexSemaphore::LockMode, unsigned long)                                                                                            
entry 0097d9f0  BcmLnxMutexSemaphore::BcmLnxMutexSemaphore(char const*)    called from 00b674a0     SchedulerLock(char const*, int)                                                                                         
entry 00b67438    called from 00b75848  SchedulerInterface::upstreamChan(unsigned char)                                                                                             
entry 00b7581c    called from 00b04c38   Onu::SetMoveCnuUpstream(unsigned char*, unsigned long, unsigned char*, unsigned long*)       00b04a1c                                                                                        
entry 00b04a20    called from 00b050b8                                                                                              
entry 00b05060 00b0505c T setCmcMoveCnuUpstream    called from 00cfe684    00cfe440 T ProcessBrcmOamMsg                                                                                           
entry 00cfe440    called from 00cf1cfc         00cf1ac0 T IeeeOamVendorHandler                                                                                      
entry 00cf1ac0    called from 00cf23c0                                                                                              
entry 00cf229c    called from 00d0baec                                                                                              
entry 00d0ba14 00d0ba0c  TkOltRxThread    called from 2aafc7ac                                                                                              
entry 2aafc6f4    called from 2aaef428                                                                                              
entry 2aaef39c    called from fffffff8                                                                                              
entry 0060e890    called from fffffff8 
wupeng@topvision:~/coredump$ addr2line -e linux_cmts_sym 0097db78  
/home/wupeng/b_dol_v226x_cmc/cmts/CmtsApp/build/Linux/../../../Common/OperatingSystem/LnxMessageQueue.cpp:67
wupeng@topvision:~/coredump$ 
wupeng@topvision:~/coredump$