linux进程异常跑飞定位总结

最近在维护C++的一个进程。
发现进程异常退出后没有生成core文件。
修改配置：
[root@localhost ~]# ulimit -a
core file size          (blocks, -c) 0
data seg size           (kbytes, -d) unlimited
scheduling priority             (-e) 0
file size               (blocks, -f) unlimited
pending signals                 (-i) 62256
max locked memory       (kbytes, -l) 64
max memory size         (kbytes, -m) unlimited
open files                      (-n) 65535
pipe size            (512 bytes, -p) 8
POSIX message queues     (bytes, -q) 819200
real-time priority              (-r) 0
stack size              (kbytes, -s) 10240     /*进程栈空间是10M*/
cpu time               (seconds, -t) unlimited
max user processes              (-u) 62256
virtual memory          (kbytes, -v) unlimited
file locks                      (-x) unlimited

修改core文件大小：ulimit -c unlimited

[root@localhost ~]# ulimit -a
core file size          (blocks, -c) unlimited
data seg size           (kbytes, -d) unlimited
scheduling priority             (-e) 0
file size               (blocks, -f) unlimited
pending signals                 (-i) 62256
max locked memory       (kbytes, -l) 64
max memory size         (kbytes, -m) unlimited
open files                      (-n) 65535
pipe size            (512 bytes, -p) 8
POSIX message queues     (bytes, -q) 819200
real-time priority              (-r) 0
stack size              (kbytes, -s) 10240
cpu time               (seconds, -t) unlimited
max user processes              (-u) 62256
virtual memory          (kbytes, -v) unlimited
file locks                      (-x) unlimited

然后再跑性能，发现进程异常退出后产生core文件。
打开core文件：gdb core.17639
GNU gdb (GDB) Red Hat Enterprise Linux 7.6.1-80.el7
Copyright (C) 2013 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law.  Type "show copying"
and "show warranty" for details.
This GDB was configured as "x86_64-redhat-linux-gnu".
For bug reporting instructions, please see:
<http://www.gnu.org/software/gdb/bugs/>...
[New LWP 17651]
[New LWP 17639]
[New LWP 17652]
[New LWP 17656]
[New LWP 17647]
[New LWP 17657]
[New LWP 17643]
[New LWP 17654]
[New LWP 17644]
[New LWP 17646]
[New LWP 17649]
[New LWP 17650]
[New LWP 17655]
[New LWP 17653]
Core was generated by `/zte/hss/zxhssdbio'.
Program terminated with signal 6, Aborted.
#0  0x00007f13df8e15f7 in ?? ()
"/zte/hss/core.17639" is a core file.
Please specify an executable to debug.
file zxhssdbio /*加载产生core文件的进程名符号表*/
(gdb) bt
#0  0x00007f13df8e15f7 in ?? ()
#1  0x00007f13df8e2ce8 in ?? ()
#2  0x0000000000000020 in ?? ()
#3  0x0000000000411183 in TrapHandler (signo=11, sip=0x7f13bdb5b470, ptContext=0x7f13bdb5b340) at dbio.cpp:580 /*信号11为访问了非法内存产生的信号。这里完全可以通过ptContext的内容打印出当前pc指针、ip指针需要在信号回调函数中添加打印*/
#4  0x00007f13e06a4100 in ?? ()
#5  0x0000000000000001 in ?? ()
#6  0x0000000000000000 in ?? ()

(gdb) info registers
rax            0x0      0
rbx            0x7f13a164e010   139722288848912
rcx            0xffffffffffffffff       -1
rdx            0x6      6
rsi            0x44f3   17651
rdi            0x44e7   17639
rbp            0x7f13bdb5b2f0   0x7f13bdb5b2f0
rsp            0x7f13bdb5b1b8   0x7f13bdb5b1b8
r8             0x2e50580        48563584
r9             0x10     16
r10            0x8      8
r11            0x206    518
r12            0x0      0
r13            0x7f13be0889c0   139722769336768
r14            0x7f13be088700   139722769336064
r15            0x7ffc60be2920   140721931561248
rip            0x7f13df8e15f7   0x7f13df8e15f7   /*发现此处RIP的值在objdump中无法找到该指令地址*/
eflags         0x206    [ PF IF ]
cs             0x33     51
ss             0x2b     43
ds             0x0      0
es             0x0      0
fs             0x0      0
gs             0x0      0

/*进入bt每个调用栈后查看下各个调用栈中的rip，看看是否有可循的蛛丝马迹.下面所有的rip都在bojdump中无法查找到指令。
  再看相关rip指令是0x7f开头的，所以可能是动态加载的动态库地址，因为在动态加载的所以dump文件中无法体现，初步判断可能是提供的lib库使用问题。*/
(gdb) f 4
#4  0x00007f13e06a4100 in ?? ()
(gdb) info frame
Stack level 4, frame at 0x7f13bdb5b348:
 rip = 0x7f13e06a4100; saved rip 0x1
 called by frame at 0x7f13bdb5b350, caller of frame at 0x7f13bdb5b340
 Arglist at 0x7f13bdb5b338, args:
 Locals at 0x7f13bdb5b338, Previous frame's sp is 0x7f13bdb5b348
 Saved registers:
  rip at 0x7f13bdb5b340
(gdb) f 5
#5  0x0000000000000001 in ?? ()
(gdb) info frame
Stack level 5, frame at 0x7f13bdb5b350:
 rip = 0x1; saved rip 0x0       /*发现此处的rip竟然为0x1和0x0，明显异常*/
 called by frame at 0x7f13bdb5b358, caller of frame at 0x7f13bdb5b348
 Arglist at 0x7f13bdb5b340, args:
 Locals at 0x7f13bdb5b340, Previous frame's sp is 0x7f13bdb5b350
 Saved registers:
  rip at 0x7f13bdb5b348
(gdb) f 6
#6  0x0000000000000000 in ?? ()
(gdb) info frame
Stack level 6, frame at 0x7f13bdb5b358:
 rip = 0x0; saved rip 0x0  /*此处异常，不应该两个地址都为0x00*/
 caller of frame at 0x7f13bdb5b350
 Arglist at 0x7f13bdb5b348, args:
 Locals at 0x7f13bdb5b348, Previous frame's sp is 0x7f13bdb5b358
 Saved registers:
  rip at 0x7f13bdb5b350

/*综上可以断定是因为栈被破坏或栈大量溢出，目前该linux系统上的进程栈空间是10M，通过查看代码无相当大的局部变量和过深的函数调用所以进程栈空间根本无法溢出；
那只有局部变量拷贝越界覆盖了栈中存放的ip指令地址，导致函数返回时跑到记录异常的ip地址上（有可能是指令地址、栈地址、变量地址等等），但操作系统认为是执行指令，因为约定是ip指令，所以不知道跑执行到哪里去，其实执行的是乱指令。*/

/*到这一步定位陷入艰巨，幸运的是可以低频率复现，所以借用强大的GDB功能*/

[root@localhost log]# ps -aux |grep zxhssdbio
root     19241  1.6  3.8 1694728 613252 ?      Sl   09:42   5:01 /zte/hss/zxhssdbio
root     29501  0.0  0.0 110236   916 pts/8    S+   14:50   0:00 grep --color=auto zxhssdbio
[root@localhost log]# gdb
GNU gdb (GDB) Red Hat Enterprise Linux 7.6.1-80.el7
Copyright (C) 2013 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law.  Type "show copying"
and "show warranty" for details.
This GDB was configured as "x86_64-redhat-linux-gnu".
For bug reporting instructions, please see:
<http://www.gnu.org/software/gdb/bugs/>.
(gdb) attach 19241
Attaching to process 19241
Reading symbols from /zte/hss/zxhssdbio...done.
Reading symbols from /zte/ztecn/libs/libACE.so.5.2.7...done.
Loaded symbols for /zte/ztecn/libs/libACE.so.5.2.7
Reading symbols from /zte/ztecn/libs/libmysqlclient.so.18...done.
Loaded symbols for /zte/ztecn/libs/libmysqlclient.so.18
Reading symbols from /zte/ztecn/libs/libprotobuf.so.8...done.
Loaded symbols for /zte/ztecn/libs/libprotobuf.so.8
Reading symbols from /lib64/libdl.so.2...(no debugging symbols found)...done.
Loaded symbols for /lib64/libdl.so.2
Reading symbols from /lib64/libpthread.so.0...(no debugging symbols found)...done.
[New LWP 19259]
[New LWP 19258]
[New LWP 19257]
[New LWP 19256]
[New LWP 19255]
[New LWP 19254]
[New LWP 19253]
[New LWP 19252]
[New LWP 19251]
[New LWP 19249]
[New LWP 19248]
[New LWP 19246]
[New LWP 19245]
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib64/libthread_db.so.1".
Loaded symbols for /lib64/libpthread.so.0
Reading symbols from /lib64/librt.so.1...(no debugging symbols found)...done.
Loaded symbols for /lib64/librt.so.1
Reading symbols from /lib64/libstdc++.so.6...(no debugging symbols found)...done.
Loaded symbols for /lib64/libstdc++.so.6
Reading symbols from /lib64/libm.so.6...(no debugging symbols found)...done.
Loaded symbols for /lib64/libm.so.6
Reading symbols from /lib64/libgcc_s.so.1...(no debugging symbols found)...done.
Loaded symbols for /lib64/libgcc_s.so.1
Reading symbols from /lib64/libc.so.6...(no debugging symbols found)...done.
Loaded symbols for /lib64/libc.so.6
Reading symbols from /lib64/ld-linux-x86-64.so.2...(no debugging symbols found)...done.
Loaded symbols for /lib64/ld-linux-x86-64.so.2
Reading symbols from /lib64/libnss_files.so.2...(no debugging symbols found)...done.
Loaded symbols for /lib64/libnss_files.so.2
0x00007fe60da3e413 in select () from /lib64/libc.so.6
Missing separate debuginfos, use: debuginfo-install glibc-2.17-105.el7.x86_64 glibc-2.17-106.el7_2.8.x86_64 libgcc-4.8.5-28.el7_5.1.x86_64 libstdc++-4.8.5-28.el7_5.1.x86_64
(gdb) set pr pr
(gdb) c
Continuing.

/*下面跑相关性能复现故障，复现一段时间后突然发现故障出现了，由于进程异常退出，GDB过程停留在当前上下文中，如下：*/

Program received signal SIGSEGV, Segmentation fault.
[Switching to Thread 0x7fe600dfa700 (LWP 19253)]
0x0000000000430046 in ACE_OS_String::memcpy (t=0x7fe600ad8c06, s=0x7fe5a3ffef98, len=257)
    at /root/pdssver/ACE_wrappers/ace/OS_String.inl:63
63      /root/pdssver/ACE_wrappers/ace/OS_String.inl: 没有那个文件或目录.

(gdb) bt
#0  0x0000000000430046 in ACE_OS_String::memcpy (t=0x7fe600ad8c06, s=0x7fe5a3ffef98, len=257)
    at /root/pdssver/ACE_wrappers/ace/OS_String.inl:63
#1  0x000000000054b893 in CSubscriberOper::mod_subscriber (this=0x7fe600accc70, tSubscribercond=..., tSubscriberGoal=...)
    at subscriberOper.cpp:650
#2  0x0000000000542d78 in CSrvConnection::mod_Subscriber (this=0x7fe5e748b010, input=0x7fe5b7213260, output=0x7fe600dd24e0,
    pDataService=0x7fe5c204f010) at subscriber_handle_omc.cpp:67
#3  0x00000000004e9f0a in dbio::Service::CfgDbio (this=0x7fe600dd25d0, input=0x7fe5b7213260, output=0x7fe600dd24e0)
    at ./soap/soapServer.cpp:583
#4  0x00000000004cc3fc in dbio::serve_dbio__CfgDbio (soap=0x7fe600dd25d0) at ./soap/dbioService.cpp:234
#5  0x00000000004cc5ee in dbio::Service::dispatch (this=0x7fe600dd25d0) at ./soap/dbioService.cpp:218
#6  0x00000000004cc6c4 in dbio::Service::serve (this=0x7fe600dd25d0) at ./soap/dbioService.cpp:194
#7  0x00000000004cc889 in dbio::Service::Analysis (this=0x7fe600dd25d0) at ./soap/dbioService.cpp:151
#8  0x00000000004a8053 in CSoapTask::svc (this=0x1d99320) at MsgHandleTask.cpp:214
#9  0x00007fe60f60462b in ACE_Task_Base::svc_run (args=<optimized out>) at Task.cpp:203
#10 0x00007fe60f58f597 in ACE_Thread_Adapter::invoke (this=0x1d99590) at Thread_Adapter.cpp:93
#11 0x00007fe60e740dc5 in start_thread () from /lib64/libpthread.so.0
#12 0x00007fe60da46ced in clone () from /lib64/libc.so.6
(gdb) f 1
#1  0x000000000054b893 in CSubscriberOper::mod_subscriber (this=0x7fe600accc70, tSubscribercond=..., tSubscriberGoal=...)
    at subscriberOper.cpp:650
650     subscriberOper.cpp: 没有那个文件或目录.
(gdb) p nCount
$9 = 0
(gdb) p m_tSubscriber[nCount]
$10 = {
  subscriber = {
    nAgencyID = 21,
    pcAgencyName = 0x0,
    pcMDN = 0x0,
    pcFUN = 0x0,
    nFID = 25,
    pcFleetName = 0x0,
    pcUserName = 0x0,
    pcPwd = 0x0,
    nStatus = 0,
    pcDetail = 0x0,
    pcActiveDate = 0x0,
    pcBeginDate = 0x0,
    pcEndDate = 0x0,
    pcDate = 0x0,
    nDisType = 0,
    nPower = 0,
    nGPSFlag = 1,
    nBindLicNum = 0,
    nGPSInterval = 0,
    nMonitor = 0,
    nOrgID = 0,
    pcOrgName = 0x0,
    pcAreaCode = 0x0
  },
  cAgencyName = '\000' <repeats 129 times>,
  cMDN = "19690005884\000ing\000:30",
  cFUN = '\000' <repeats 11 times>,
  cFleetName = '\000' <repeats 513 times>,
  cUserName = "19690005884\000\065:58:30\000\262+GP@\000\000\000\000\000\000\000\065\277", '\000' <repeats 95 times>,
  cPwd = '\000' <repeats 64 times>,
  cDetail = "<年卡><语音对讲+GPS>\000\000\000\000\000\000\065\000\000\000\000\000\000\000`\356\377\243\345\177\000\000\020\330ҹ\345\177\000\000\001\000\000\000\000\000\000\000@\035 \224\345\177\000\000\065\060\066\000\000\000\000\000\021\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000\001", '\000' <repeats 159 times>,
  cBeginDate = '\000' <repeats 32 times>,
  cEndDate = '\000' <repeats 32 times>,
  cDate = '\000' <repeats 64 times>,
  cAreaCode = '\000' <repeats 32 times>,
  cOrgName = '\000' <repeats 64 times>

/*原来是拷贝时，memcpy访问越界了，memcpy的第二个参数并没有257的长度，在gdb中手动敲入call memcpy的带参数函数，提示某个地址Cannot access memory at address。

 堆地址中，怎么会有受保护的内存呢。唯一的可能性是linux内核态下申请的堆内存，因为用户态访问内核态指令或数据地址，需要进行特权切换*/

/*查看下进程动态运行时maps内存范围*/

[root@localhost hss]# cat /proc/1392/maps
00400000-00685000 r-xp 00000000 fd:00 254403644                          /zte/hss/zxhssdbio
00884000-00888000 rw-p 00284000 fd:00 254403644                          /zte/hss/zxhssdbio
00888000-01a1f000 rw-p 00000000 00:00 0
031e1000-03202000 rw-p 00000000 00:00 0                                  [heap]
03202000-032ab000 rw-p 00000000 00:00 0                                  [heap]
7ff6dc000000-7ff6dc023000 rw-p 00000000 00:00 0
7ff6dc023000-7ff6e0000000 ---p 00000000 00:00 0             /*p表示私有数据，s表示共享，但再GDB权限下也可以x查看仅仅标识为p的数据的地址*/
7ff6e4000000-7ff6e4023000 rw-p 00000000 00:00 0
7ff6e4023000-7ff6e8000000 ---p 00000000 00:00 0
7ff6e8000000-7ff6e8023000 rw-p 00000000 00:00 0
7ff6e8023000-7ff6ec000000 ---p 00000000 00:00 0
7ff6ec000000-7ff6ec087000 rw-p 00000000 00:00 0
7ff6ec087000-7ff6f0000000 ---p 00000000 00:00 0
7ff6f204f000-7ff6f8000000 rw-p 00000000 00:00 0
7ff6f8000000-7ff6f8051000 rw-p 00000000 00:00 0
7ff6f8051000-7ff6fc000000 ---p 00000000 00:00 0
7ff6fc000000-7ff6fc023000 rw-p 00000000 00:00 0
7ff6fc023000-7ff700000000 ---p 00000000 00:00 0
7ff700000000-7ff700023000 rw-p 00000000 00:00 0
7ff700023000-7ff704000000 ---p 00000000 00:00 0
7ff704000000-7ff707ff5000 rw-p 00000000 00:00 0
7ff707ff5000-7ff708000000 ---p 00000000 00:00 0
7ff708000000-7ff708023000 rw-p 00000000 00:00 0
7ff708023000-7ff70c000000 ---p 00000000 00:00 0
7ff70f48b000-7ff714000000 rw-p 00000000 00:00 0
7ff714000000-7ff714023000 rw-p 00000000 00:00 0
7ff714023000-7ff718000000 ---p 00000000 00:00 0
7ff718000000-7ff71809a000 rw-p 00000000 00:00 0
7ff71809a000-7ff71c000000 ---p 00000000 00:00 0
7ff71c000000-7ff71c716000 rw-p 00000000 00:00 0
7ff71c716000-7ff720000000 ---p 00000000 00:00 0
7ff72036f000-7ff7203fa000 rw-p 00000000 00:00 0
7ff7203fa000-7ff7203fb000 ---p 00000000 00:00 0
7ff7203fb000-7ff720dfb000 rw-p 00000000 00:00 0                          [stack:1411]
7ff720dfb000-7ff720dfc000 ---p 00000000 00:00 0
7ff720dfc000-7ff7217fc000 rw-p 00000000 00:00 0                          [stack:1410]
7ff7217fc000-7ff7217fd000 ---p 00000000 00:00 0
7ff7217fd000-7ff7221fd000 rw-p 00000000 00:00 0                          [stack:1409]
7ff7221fd000-7ff7221fe000 ---p 00000000 00:00 0
7ff7221fe000-7ff722bfe000 rw-p 00000000 00:00 0                          [stack:1408]
7ff722bfe000-7ff722bff000 ---p 00000000 00:00 0
7ff722bff000-7ff7235ff000 rw-p 00000000 00:00 0                          [stack:1399]
7ff7235ff000-7ff723600000 ---p 00000000 00:00 0
7ff723600000-7ff724000000 rw-p 00000000 00:00 0                          [stack:1407]
7ff724000000-7ff724023000 rw-p 00000000 00:00 0
7ff724023000-7ff728000000 ---p 00000000 00:00 0
7ff728030000-7ff7283fa000 rw-p 00000000 00:00 0
7ff7283fa000-7ff7283fb000 ---p 00000000 00:00 0
7ff7283fb000-7ff728dfb000 rw-p 00000000 00:00 0                          [stack:1406]
7ff728dfb000-7ff728dfc000 ---p 00000000 00:00 0
7ff728dfc000-7ff7297fc000 rw-p 00000000 00:00 0                          [stack:1405]
7ff7297fc000-7ff7297fd000 ---p 00000000 00:00 0
7ff7297fd000-7ff72a1fd000 rw-p 00000000 00:00 0                          [stack:1404]
7ff72a1fd000-7ff72a1fe000 ---p 00000000 00:00 0
7ff72a1fe000-7ff72abfe000 rw-p 00000000 00:00 0                          [stack:1403]
7ff72abfe000-7ff72abff000 ---p 00000000 00:00 0
7ff72abff000-7ff72b5ff000 rw-p 00000000 00:00 0                          [stack:1400]
7ff72b5ff000-7ff72b600000 ---p 00000000 00:00 0
7ff72b600000-7ff72c000000 rw-p 00000000 00:00 0
7ff72c000000-7ff72c023000 rw-p 00000000 00:00 0
7ff72c023000-7ff730000000 ---p 00000000 00:00 0
7ff73001d000-7ff73084b000 rw-p 00000000 00:00 0
7ff73084b000-7ff73084c000 ---p 00000000 00:00 0
7ff73084c000-7ff73124c000 rw-p 00000000 00:00 0                          [stack:1397]
7ff73124c000-7ff73124d000 ---p 00000000 00:00 0
7ff73124d000-7ff731c4d000 rw-p 00000000 00:00 0                          [stack:1396]
7ff731c4d000-7ff731c59000 r-xp 00000000 fd:00 253519141                  /usr/lib64/libnss_files-2.17.so
7ff731c59000-7ff731e58000 ---p 0000c000 fd:00 253519141                  /usr/lib64/libnss_files-2.17.so
7ff731e58000-7ff731e59000 r--p 0000b000 fd:00 253519141                  /usr/lib64/libnss_files-2.17.so
7ff731e59000-7ff731e5a000 rw-p 0000c000 fd:00 253519141                  /usr/lib64/libnss_files-2.17.so
7ff731e5a000-7ff7358f9000 rw-p 00000000 00:00 0
7ff7358f9000-7ff735ab0000 r-xp 00000000 fd:00 252618805                  /usr/lib64/libc-2.17.so
7ff735ab0000-7ff735cb0000 ---p 001b7000 fd:00 252618805                  /usr/lib64/libc-2.17.so
7ff735cb0000-7ff735cb4000 r--p 001b7000 fd:00 252618805                  /usr/lib64/libc-2.17.so
7ff735cb4000-7ff735cb6000 rw-p 001bb000 fd:00 252618805                  /usr/lib64/libc-2.17.so
7ff735cb6000-7ff735cbb000 rw-p 00000000 00:00 0
7ff735cbb000-7ff735cd0000 r-xp 00000000 fd:00 253826158                  /usr/lib64/libgcc_s-4.8.5-20150702.so.1
7ff735cd0000-7ff735ecf000 ---p 00015000 fd:00 253826158                  /usr/lib64/libgcc_s-4.8.5-20150702.so.1
7ff735ecf000-7ff735ed0000 r--p 00014000 fd:00 253826158                  /usr/lib64/libgcc_s-4.8.5-20150702.so.1
7ff735ed0000-7ff735ed1000 rw-p 00015000 fd:00 253826158                  /usr/lib64/libgcc_s-4.8.5-20150702.so.1
7ff735ed1000-7ff735fd2000 r-xp 00000000 fd:00 252618813                  /usr/lib64/libm-2.17.so
7ff735fd2000-7ff7361d1000 ---p 00101000 fd:00 252618813                  /usr/lib64/libm-2.17.so
7ff7361d1000-7ff7361d2000 r--p 00100000 fd:00 252618813                  /usr/lib64/libm-2.17.so
7ff7361d2000-7ff7361d3000 rw-p 00101000 fd:00 252618813                  /usr/lib64/libm-2.17.so
7ff7361d3000-7ff7362bc000 r-xp 00000000 fd:00 253655069                  /usr/lib64/libstdc++.so.6.0.19
7ff7362bc000-7ff7364bb000 ---p 000e9000 fd:00 253655069                  /usr/lib64/libstdc++.so.6.0.19
7ff7364bb000-7ff7364c3000 r--p 000e8000 fd:00 253655069                  /usr/lib64/libstdc++.so.6.0.19
7ff7364c3000-7ff7364c5000 rw-p 000f0000 fd:00 253655069                  /usr/lib64/libstdc++.so.6.0.19
7ff7364c5000-7ff7364da000 rw-p 00000000 00:00 0
7ff7364da000-7ff7364e1000 r-xp 00000000 fd:00 253519144                  /usr/lib64/librt-2.17.so
7ff7364e1000-7ff7366e0000 ---p 00007000 fd:00 253519144                  /usr/lib64/librt-2.17.so
7ff7366e0000-7ff7366e1000 r--p 00006000 fd:00 253519144                  /usr/lib64/librt-2.17.so
7ff7366e1000-7ff7366e2000 rw-p 00007000 fd:00 253519144                  /usr/lib64/librt-2.17.so
7ff7366e2000-7ff7366f8000 r-xp 00000000 fd:00 253101108                  /usr/lib64/libpthread-2.17.so
7ff7366f8000-7ff7368f8000 ---p 00016000 fd:00 253101108                  /usr/lib64/libpthread-2.17.so
7ff7368f8000-7ff7368f9000 r--p 00016000 fd:00 253101108                  /usr/lib64/libpthread-2.17.so
7ff7368f9000-7ff7368fa000 rw-p 00017000 fd:00 253101108                  /usr/lib64/libpthread-2.17.so
7ff7368fa000-7ff7368fe000 rw-p 00000000 00:00 0
7ff7368fe000-7ff736901000 r-xp 00000000 fd:00 252618811                  /usr/lib64/libdl-2.17.so
7ff736901000-7ff736b00000 ---p 00003000 fd:00 252618811                  /usr/lib64/libdl-2.17.so
7ff736b00000-7ff736b01000 r--p 00002000 fd:00 252618811                  /usr/lib64/libdl-2.17.so
7ff736b01000-7ff736b02000 rw-p 00003000 fd:00 252618811                  /usr/lib64/libdl-2.17.so
7ff736b02000-7ff736c0d000 r-xp 00000000 fd:00 134774855                  /zte/ztecn/libs/libprotobuf.so.8
7ff736c0d000-7ff736e0c000 ---p 0010b000 fd:00 134774855                  /zte/ztecn/libs/libprotobuf.so.8
7ff736e0c000-7ff736e13000 rw-p 0010a000 fd:00 134774855                  /zte/ztecn/libs/libprotobuf.so.8
7ff736e13000-7ff73710b000 r-xp 00000000 fd:00 134774856                  /zte/ztecn/libs/libmysqlclient.so.18
7ff73710b000-7ff73730a000 ---p 002f8000 fd:00 134774856                  /zte/ztecn/libs/libmysqlclient.so.18
7ff73730a000-7ff7373dd000 rw-p 002f7000 fd:00 134774856                  /zte/ztecn/libs/libmysqlclient.so.18
7ff7373dd000-7ff7373e2000 rw-p 00000000 00:00 0
7ff7373e2000-7ff737635000 r-xp 00000000 fd:00 134830110                  /zte/ztecn/libs/libACE.so.5.2.7
7ff737635000-7ff737835000 ---p 00253000 fd:00 134830110                  /zte/ztecn/libs/libACE.so.5.2.7
7ff737835000-7ff73784c000 rw-p 00253000 fd:00 134830110                  /zte/ztecn/libs/libACE.so.5.2.7
7ff73784c000-7ff737851000 rw-p 00000000 00:00 0
7ff737851000-7ff737872000 r-xp 00000000 fd:00 253519134                  /usr/lib64/ld-2.17.so
7ff7378f5000-7ff737a33000 rw-p 00000000 00:00 0
7ff737a33000-7ff737a54000 rw-s 00000000 00:04 32769                      /SYSV940b67b5 (deleted)
7ff737a54000-7ff737a5b000 rw-p 00000000 00:00 0
7ff737a70000-7ff737a72000 rw-p 00000000 00:00 0
7ff737a72000-7ff737a73000 r--p 00021000 fd:00 253519134                  /usr/lib64/ld-2.17.so
7ff737a73000-7ff737a74000 rw-p 00022000 fd:00 253519134                  /usr/lib64/ld-2.17.so
7ff737a74000-7ff737a75000 rw-p 00000000 00:00 0
7ffcb987e000-7ffcb989f000 rw-p 00000000 00:00 0                          [stack]
7ffcb99ae000-7ffcb99b0000 r-xp 00000000 00:00 0                          [vdso]
ffffffffff600000-ffffffffff601000 r-xp 00000000 00:00 0                  [vsyscall]
[root@localhost hss]#
/*通过以上信息还是找不到core文件中的那几个rip指令地址范围，那就是跑到未使用的堆空间去了*/