crash使用方法

1.Crash 调试vmcore：
# pwd /var/crash/127.0.0.1-2014-06-23-12:37:00
# crash /usr/lib/debug/lib/module/`uname -r`/vmlinux vmcore
crash> bt /* 打印函数栈 */
crash> ps /* 查看系统崩溃时在运行的所有进程 */
crash> set 3016 /* 连接系统崩溃时正在运行的进程，进行debug */
crash> bt -f /* 打印函数栈数据 */
crash> dis pblaze_disk_make_request /* 反汇编指定函数 */
crash> struct bio ffff880188b50840 /* 将特定地址按照制定数据类型显示 */
crash> struct bio_vec 0xffff880188b508b0
crash> bt
crash> dmesg /* 查看系统崩溃时dmesg的信息 */
crash> bt -f
crash> dis attempt_merge
crash> dis attempt_back_merge
crash> bt -f
crash> dis attempt_merge
crash> struct request ffff880188b4db60
crash> struct bio ffff880188b50840
crash> struct bio_vec[4] 0xffff880188b508b0
crash> struct bio_vec 0xffff880188b508b0
crash> struct bio_vec 0xffff880188b508b8
crash> struct bio_vec 0xffff880188b508c0
crash> struct bio_vec 0xffff880188b508c8




2.Crash 调试卡死的进程：
crash -S System.map vmlinux /*导入内核调试信息以及符号表*/
crash> set 进程id /* 连接需要调试的进程 */
crash> mod -s memdisk /* 导入模块memdisk的符号表 */
crash> mod -s memcon /* 导入模块memcon的符号表 */
crash> bt /* 打印函数栈 */


crash> bt -f /* 打印函数栈详细数据 */
...
 #9 [ffff8801a8c17bf0] attempt_back_merge at ffffffff8126e256
    ffff8801a8c17bf8: ffff8801a7fe0338 0000000000000000 
    ffff8801a8c17c08: ffff8801a8c17c88 ffffffff81267fbb 
/* 函数栈内自底向上，自右向左存储数据。
1.右下角为第一个数据：返回到blk_queue_bio 函数的继续执行地址。 
2.左下角为第二个数据，blk_queue_bio 函数的栈底地址，返回时使用。
*/
#10 [ffff8801a8c17c10] blk_queue_bio at ffffffff81267fbb
    ffff8801a8c17c18: ffff88000001bd80 0000000000000000 
    ffff8801a8c17c28: ffff8801a8c17c78 ffff880188b50840 
    ffff8801a8c17c38: ffff88000001bd80 ffff8801a82c85c0 
    ffff8801a8c17c48: ffff8801a8c17c68 ffff880188b3c4c0 
    ffff8801a8c17c58: ffff880188b50840 ffff880188b50840 
    ffff8801a8c17c68: ffff880188b50840 ffff8801a96c9000 
    ffff8801a8c17c78: ffff8801a7fe0338 0000000000000007 
    ffff8801a8c17c88: ffff8801a8c17cf8 ffffffffa004e7b1 
...
/*根据函数栈中查看到的数据的地址，制定类型将数据打印出来，供debug*/
crash> struct request ffff880188b4db60 
struct request {
  queuelist = {
    next = 0xffff8801a6c0ab40, 
    prev = 0xffff880188b3c4c0
  }, 
  csd = {
    list = {
      next = 0x0, 
      prev = 0x0
    }, 
    func = 0, 
    info = 0x0, 
    flags = 0, 
    priv = 0
  }, 
  q = 0xffff8801a7fe0338, 
  cmd_flags = 17309728, 
  cmd_type = REQ_TYPE_FS, 
  atomic_flags = 0, 
  cpu = 9, 
  __data_len = 0, 
  __sector = 8, 
  bio = 0xffff880188b50f00, 
  biotail = 0xffff880188b50f00, 
  hash = {
    next = 0xffff880188b3c690, 
    pprev = 0xffff8801890dd230
  }, 
  {
    rb_node = {
      rb_parent_color = 18446612138902739936, 
      rb_right = 0x0, 
      rb_left = 0x0
    }, 
    completion_data = 0xffff880188b4dbe0
  }, 
  {
    elevator_private = {0x0, 0x0, 0x0}, 
    flush = {
      seq = 0, 
      list = {
        next = 0x0, 
        prev = 0x0
      }
    }
  }, 
  rq_disk = 0xffff8801a7e18800, 
  start_time = 4295322338, 
  start_time_ns = 655879450331, 
  io_start_time_ns = 0, 
  nr_phys_segments = 0, 
  ioprio = 0, 
  ref_count = 1, 
  special = 0x0, 
  buffer = 0x0, 
  tag = -1, 
  errors = 0, 
  __cmd = "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000", 
  cmd = 0xffff880188b4dc50 "", 
  cmd_len = 16, 
  extra_len = 0, 
  sense_len = 0, 
  resid_len = 0, 
  sense = 0x0, 
  deadline = 0, 
  timeout_list = {
    next = 0xffff880188b4dc88, 
    prev = 0xffff880188b4dc88
  }, 


crash> dis attempt_merge /* 反汇编制定的函数或制定的地址 */
0xffffffff8126de20 <attempt_merge>:     push   %rbp /*压入调用函数的栈底指针*/
0xffffffff8126de21 <attempt_merge+1>:   mov    %rsp,%rbp
0xffffffff8126de24 <attempt_merge+4>:   sub    $0x40,%rsp
0xffffffff8126de28 <attempt_merge+8>:   mov    %rbx,-0x28(%rbp) 
0xffffffff8126de2c <attempt_merge+12>:  mov    %r12,-0x20(%rbp)
0xffffffff8126de30 <attempt_merge+16>:  mov    %r13,-0x18(%rbp)
0xffffffff8126de34 <attempt_merge+20>:  mov    %r14,-0x10(%rbp)
0xffffffff8126de38 <attempt_merge+24>:  mov    %r15,-0x8(%rbp)
0xffffffff8126de3c <attempt_merge+28>:  nopl   0x0(%rax,%rax,1)
0xffffffff8126de41 <attempt_merge+33>:  cmpl   $0x1,0x44(%rsi)
0xffffffff8126de45 <attempt_merge+37>:  mov    %rdi,%r13
0xffffffff8126de48 <attempt_merge+40>:  mov    %rsi,%rbx
0xffffffff8126de4b <attempt_merge+43>:  mov    %rdx,%r12
0xffffffff8126de4e <attempt_merge+46>:  je     0xffffffff8126de70 <attempt_merge+80>






调试vmcore例子：
BUG: unable to handle kernel NULL pointer dereference at (null)
IP: [<ffffffff8126dfee>] attempt_merge+0x1ce/0x3b0
PGD 1a9ebe067 PUD 1aad69067 PMD 0 
Oops: 0000 [#1] SMP 
last sysfs file: /sys/devices/system/cpu/online
CPU 9 
Modules linked in: bio_submiter(U) ip6table_filter ip6_tables ebtable_nat ebtables ipt_MASQUERADE iptable_nat nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack ipt_REJECT xt_CHECKSUM iptable_mangle iptable_filter ip_tables bridge autofs4 8021q garp stp llc ipv6 vhost_net macvtap macvlan tun kvm_intel kvm uinput power_meter microcode iTCO_wdt iTCO_vendor_support dcdbas sg ses enclosure serio_raw lpc_ich mfd_core i7core_edac edac_core bnx2 memdisk(U) memcon(U) ext4 jbd2 mbcache sr_mod cdrom sd_mod crc_t10dif pata_acpi ata_generic ata_piix megaraid_sas dm_mirror dm_region_hash dm_log dm_mod [last unloaded: bio_submiter]


Pid: 3016, comm: user_submit Not tainted 2.6.32-431.el6.x86_64 #1 Dell Inc. PowerEdge R710/0XDX06
RIP: 0010:[<ffffffff8126dfee>]  [<ffffffff8126dfee>] attempt_merge+0x1ce/0x3b0
RSP: 0018:ffff8801a8c17ba8  EFLAGS: 00010046
RAX: 0000000000000000 RBX: ffff880188b3c4c0 RCX: 000000000000000d
RDX: 0000160000000000 RSI: ffff880188b385c0 RDI: 0000000000000200
RBP: ffff8801a8c17be8 R08: 0000000000000000 R09: 000000000000000d
R10: 0000000188b59400 R11: 0000000188b59200 R12: ffff880188b4db60
R13: ffff8801a7fe0338 R14: 6db6db6db6db6db7 R15: 0000160000000000
FS:  00007f5429dd3700(0000) GS:ffff88002dd20000(0000) knlGS:0000000000000000
Read scucceed, Return value: 0. Scucceed Case Number: 73. LBA addr: 3
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000000000000000 CR3: 00000001aa091000 CR4: 00000000000007e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Process user_submit (pid: 3016, threadinfo ffff8801a8c16000, task ffff8801a89d8080)
Stack:
 0000000000000000 0000000000000000 ffff8801a8c17bd8 ffff8801a7fe0338
<d> ffff880188b3c4c0 ffff880188b3c4c0 0000000000000000 0000000000000000
<d> ffff8801a8c17c08 ffffffff8126e256 ffff8801a7fe0338 0000000000000000
Call Trace:
 [<ffffffff8126e256>] attempt_back_merge+0x36/0x50
Read scucceed, Return value: 0. Scucceed Case Number: 73. LBA addr: 4
 [<ffffffff81267fbb>] blk_queue_bio+0x48b/0x620
 [<ffffffffa004e7b1>] pblaze_disk_make_request+0x2