linux环境查看当前内核栈已使用过的最大值及对应内核栈调用链信息

目录

1、挂载debugfs

2、启用内核栈追溯

3、查看内核栈已用最大值及对应调用链信息

4、使用watch动态查看

5、使用function tracer动态追踪内核函数堆栈

---

实验环境：

[root@yglocal ~]# cat /proc/version 
Linux version 3.8.13-44.1.1.el6uek.x86_64 (mockbuild@ca-build44.us.oracle.com) (gcc version 4.4.7 20120313 (Red Hat 4.4.7-3) (GCC) ) #2 SMP Wed Sep 10 06:10:25 PDT 2014
[root@yglocal ~]# 
[root@yglocal ~]# cat /etc/oracle-release 
Oracle Linux Server release 6.6
[root@yglocal ~]# cat /etc/redhat-release 
Red Hat Enterprise Linux Server release 6.6 (Santiago)

1、挂载debugfs

在centos/rhel6.x及之前版本系统，默认debugfs没有挂载，需要手动挂载下debugfs文件系统：

[root@yglocal ~]# ll /sys/kernel/debug/
total 0

如果/sys/kernel/debug目录下没有文件，则需要挂载下debugfs到该目录下：

[root@yglocal ~]# mount -t debugfs nodev /sys/kernel/debug
[root@yglocal ~]# cd /sys/kernel/debug/
[root@yglocal debug]# 
[root@yglocal debug]# ls
acpi  bdi  boot_params  cleancache  dri  dynamic_debug  extfrag  frontswap  gpio  hid  jbd2  kprobes  mce  regmap  regulator  sched_features  suspend_stats  tracing  usb  wakeup_sources  x86
[root@yglocal debug]# 

2、启用内核栈追溯

[root@yglocal debug]# cat /proc/sys/kernel/stack_tracer_enabled
0
[root@yglocal debug]# 
[root@yglocal debug]# echo 1 > /proc/sys/kernel/stack_tracer_enabled
[root@yglocal debug]# cat /proc/sys/kernel/stack_tracer_enabled
1

3、查看内核栈已用最大值及对应调用链信息

查看目前为止内核栈使用到的最大值及对应的内核栈信息：

[root@yglocal debug]# cat /sys/kernel/debug/tracing/stack_max_size
4072
[root@yglocal debug]# cat /sys/kernel/debug/tracing/stack_trace
        Depth    Size   Location    (39 entries)
        -----    ----   --------
  0)     4072     136   kmem_cache_alloc+0x21/0x2f0
  1)     3936      16   mempool_alloc_slab+0x15/0x20
  2)     3920     144   mempool_alloc+0x60/0x170
  3)     3776      16   scsi_sg_alloc+0x49/0x50
  4)     3760     112   __sg_alloc_table+0x61/0x130
  5)     3648      48   scsi_alloc_sgtable+0x28/0x60
  6)     3600      32   scsi_init_sgtable+0x21/0x70
  7)     3568      80   scsi_init_io+0x3d/0x170
  8)     3488      32   scsi_setup_fs_cmnd+0x79/0xe0
  9)     3456     128   sd_prep_fn+0xef/0xbd0 [sd_mod]
 10)     3328      64   blk_peek_request+0xce/0x220
 11)     3264      96   scsi_request_fn+0x60/0x570
 12)     3168      32   __blk_run_queue+0x37/0x50
 13)     3136      64   queue_unplugged+0x4e/0xc0
 14)     3072     112   blk_flush_plug_list+0x1cb/0x230
 15)     2960      32   io_schedule+0x75/0xd0
 16)     2928      16   sleep_on_buffer+0xe/0x20
 17)     2912      80   __wait_on_bit+0x5f/0x90
 18)     2832     112   out_of_line_wait_on_bit+0x78/0x90
 19)     2720      32   __wait_on_buffer+0x36/0x50
 20)     2688      48   ext4_wait_block_bitmap+0xd8/0xe0 [ext4]
 21)     2640     208   ext4_mb_init_cache+0x1d2/0x790 [ext4]
 22)     2432      96   ext4_mb_init_group+0x9e/0x100 [ext4]
 23)     2336      64   ext4_mb_good_group+0x12d/0x140 [ext4]
 24)     2272     176   ext4_mb_regular_allocator+0x1b9/0x430 [ext4]
 25)     2096     128   ext4_mb_new_blocks+0x38a/0x540 [ext4]
 26)     1968     272   ext4_ext_map_blocks+0x9bf/0xcc0 [ext4]
 27)     1696      96   ext4_map_blocks+0x1a1/0x2d0 [ext4]
 28)     1600     144   mpage_da_map_and_submit+0xb9/0x2e0 [ext4]
 29)     1456     320   ext4_da_writepages+0x2d2/0x590 [ext4]
 30)     1136      16   do_writepages+0x20/0x40
 31)     1120      96   __writeback_single_inode+0x46/0x190
 32)     1024     192   writeback_sb_inodes+0x21f/0x330
 33)      832      80   __writeback_inodes_wb+0x9e/0xd0
 34)      752     144   wb_writeback+0x1fb/0x2d0
 35)      608     160   wb_do_writeback+0x1f8/0x240
 36)      448     128   bdi_writeback_thread+0xaa/0x240
 37)      320     144   kthread+0xce/0xe0
 38)      176     176   ret_from_fork+0x7c/0xb0

4、使用watch动态查看

可以用watch每秒输出下目前内核栈使用过的最大值及当时内核栈调用链：

watch -n 1 'free -m; echo && echo "stack_max_size:"; cat /sys/kernel/debug/tracing/stack_max_size; echo && echo "stack_trace:"; cat /sys/kernel/debug/tracing/stack_trace'

Every 1.0s: free -m; echo && echo "stack_max_size:"; cat /sys/kernel/debug/tracing/stack_max_size; echo && echo "stack_trace:"; cat /sys/kernel/debug/tracing/stack_trace                               Fri Jan 15 15:06:42 2021

             total       used       free     shared    buffers     cached
Mem:          5712        611       5101          0         64        272
-/+ buffers/cache:        273       5439
Swap:         3831          0       3831

stack_max_size:
4072

stack_trace:
        Depth    Size   Location    (39 entries)
        -----    ----   --------
  0)     4072     136   kmem_cache_alloc+0x21/0x2f0
  1)     3936      16   mempool_alloc_slab+0x15/0x20
  2)     3920     144   mempool_alloc+0x60/0x170
  3)     3776      16   scsi_sg_alloc+0x49/0x50
  4)     3760     112   __sg_alloc_table+0x61/0x130
  5)     3648      48   scsi_alloc_sgtable+0x28/0x60
  6)     3600      32   scsi_init_sgtable+0x21/0x70
  7)     3568      80   scsi_init_io+0x3d/0x170
  8)     3488      32   scsi_setup_fs_cmnd+0x79/0xe0
  9)     3456     128   sd_prep_fn+0xef/0xbd0 [sd_mod]
 10)     3328      64   blk_peek_request+0xce/0x220
 11)     3264      96   scsi_request_fn+0x60/0x570
 12)     3168      32   __blk_run_queue+0x37/0x50
 13)     3136      64   queue_unplugged+0x4e/0xc0
 14)     3072     112   blk_flush_plug_list+0x1cb/0x230
 15)     2960      32   io_schedule+0x75/0xd0
 16)     2928      16   sleep_on_buffer+0xe/0x20
 17)     2912      80   __wait_on_bit+0x5f/0x90
 18)     2832     112   out_of_line_wait_on_bit+0x78/0x90
 19)     2720      32   __wait_on_buffer+0x36/0x50
 20)     2688      48   ext4_wait_block_bitmap+0xd8/0xe0 [ext4]
 21)     2640     208   ext4_mb_init_cache+0x1d2/0x790 [ext4]
 22)     2432      96   ext4_mb_init_group+0x9e/0x100 [ext4]
 23)     2336      64   ext4_mb_good_group+0x12d/0x140 [ext4]
 24)     2272     176   ext4_mb_regular_allocator+0x1b9/0x430 [ext4]
 25)     2096     128   ext4_mb_new_blocks+0x38a/0x540 [ext4]
 26)     1968     272   ext4_ext_map_blocks+0x9bf/0xcc0 [ext4]
 27)     1696      96   ext4_map_blocks+0x1a1/0x2d0 [ext4]
 28)     1600     144   mpage_da_map_and_submit+0xb9/0x2e0 [ext4]
 29)     1456     320   ext4_da_writepages+0x2d2/0x590 [ext4]
 30)     1136      16   do_writepages+0x20/0x40
 31)     1120      96   __writeback_single_inode+0x46/0x190
 32)     1024     192   writeback_sb_inodes+0x21f/0x330
 33)      832      80   __writeback_inodes_wb+0x9e/0xd0
 34)      752     144   wb_writeback+0x1fb/0x2d0
 35)      608     160   wb_do_writeback+0x1f8/0x240
 36)      448     128   bdi_writeback_thread+0xaa/0x240
 37)      320     144   kthread+0xce/0xe0
 38)      176     176   ret_from_fork+0x7c/0xb0

5、使用function tracer动态追踪内核函数堆栈

①查看当前内核支持的可追踪的函数

less /sys/kernel/debug/tracing/available_filter_functions

可以grep模糊查找要追踪的函数，比如openat：

[root@yglocal ~]# cat /sys/kernel/debug/tracing/available_filter_functions | grep openat
SyS_openat
compat_SyS_openat
path_openat

②将函数名设置到set_ftrace_filter中

echo SyS_openat /sys/kernel/debug/tracing/set_ftrace_filter

③设置当前追踪为函数追踪

echo function > /sys/kernel/debug/tracing/current_tracer 

④开启函数栈追踪

echo 1 > /sys/kernel/debug/tracing/options/stacktrace

⑤查看追踪结果

cat /sys/kernel/debug/tracing/trace

注：上述设置步骤可以一条命令搞定：

echo 'SyS_openat:stacktrace:10' > /sys/kernel/debug/tracing/set_ftrace_filter