示例
[root@localhost ~]# crash /usr/lib/debug/lib/modules/3.10.0-693.el7.x86_64/vmlinux /tmp/vmcore
Crash 简介
当Linux系统内核发生崩溃的时候,可以通过 KEXEC+KDUMP 等方式收集内核崩溃之前的内存,生成一个转储文件vmcore。内核开发者通过分析该vmcore文件就可以诊断出内核崩溃的原因,从而进行操作系统的代码改进。那么Crash就是一个被广泛使用的内核崩溃转储文件分析工具.
对调试来讲,gdb是非常适合的,但gdb始终是调试native的工具,不支持kernel信息显示,比如task信息之类的。crash补足了这个短板,由Dave Anderson开发和维护的一个内存转储分析工具,是基于GDB开发的 (GDB适用于用户进程的coredump,而Crash扩展了 GDB,使其适用于 linux kernel coredump)。
CentOS 7 配置 kdump 和安装 crash 工具
1)关于 kdump 和 crash
kdump是一种kernel crash dump的机制,它可以在内核crash时保存系统的内存信息用于后续的分析。kdump是基于kexec的。 crash是一个用于交互式地分析正在运行的Linux系统或者kernel crash后的core dump数据的工具。
2)在 CentOS 7 中配置 kdump
CentOS 7 默认已经启动 kdump 服务,无需额外配置:
[root@localhost ~]# systemctl status kdump.service
● kdump.service - Crash recovery kernel arming
Loaded: loaded (/usr/lib/systemd/system/kdump.service; enabled; vendor preset: enabled)
Active: active (exited) since Tue 2020-04-14 22:54:24 CST; 5 days ago
Process: 2123 ExecStart=/usr/bin/kdumpctl start (code=exited, status=0/SUCCESS)
Main PID: 2123 (code=exited, status=0/SUCCESS)
Memory: 0B
CGroup: /system.slice/kdump.service
Apr 14 22:54:23 localhost dracut[3654]: drwxr-xr-x 2 root root 0 Apr 14 22:54 usr/share/zoneinfo/Asia
Apr 14 22:54:23 localhost dracut[3654]: -rw-r--r-- 1 root root 388 Mar 28 2017 usr/share/zoneinfo/Asia/Shanghai
Apr 14 22:54:23 localhost dracut[3654]: drwxr-xr-x 2 root root 0 Apr 14 22:54 var
Apr 14 22:54:23 localhost dracut[3654]: lrwxrwxrwx 1 root root 11 Apr 14 22:54 var/lock -> ../run/lock
Apr 14 22:54:23 localhost dracut[3654]: lrwxrwxrwx 1 root root 6 Apr 14 22:54 var/run -> ../run
Apr 14 22:54:23 localhost dracut[3654]: ========================================================================
Apr 14 22:54:23 localhost dracut[3654]: *** Creating initramfs image file '/boot/initramfs-3.10.0-693.el7.x86_64kdump.img' done ***
Apr 14 22:54:24 localhost kdumpctl[2123]: kexec: loaded kdump kernel
Apr 14 22:54:24 localhost kdumpctl[2123]: Starting kdump: [OK]
Apr 14 22:54:24 localhost systemd[1]: Started Crash recovery kernel arming.
3)安装 crash 工具 首先需要安装对应的 kernel-debuginfo 软件包,比如:
[root@localhost ~]# uname -r
3.10.0-693.el7.x86_64
[root@localhost ~]# sed -i 's/enabled=0/enabled=1/g' /etc/yum.repos.d/CentOS-Debuginfo.repo
[root@localhost ~]# yum install kernel-debuginfo-3.10.0-693.el7.x86_64 -y
[root@localhost kernel-debuginfo]# rpm -ql kernel-debuginfo|grep vmlinux
/usr/lib/debug/lib/modules/3.10.0-693.el7.x86_64/vmlinux
安装好 kernel-debuginfo 包后,执行crash 命令即可进入交互分析:
[root@localhost ~]# crash /usr/lib/debug/lib/modules/3.10.0-693.el7.x86_64/vmlinux /var/crash/127.0.0.1-2020-04-14-22:51:43/vmcore
使用 crash 工具
[root@localhost ~]# crash /usr/lib/debug/lib/modules/3.10.0-693.el7.x86_64/vmlinux /var/crash/127.0.0.1-2020-04-14-22:51:43/vmcore
crash 7.2.3-10.el7
Copyright (C) 2002-2017 Red Hat, Inc.
Copyright (C) 2004, 2005, 2006, 2010 IBM Corporation
Copyright (C) 1999-2006 Hewlett-Packard Co
Copyright (C) 2005, 2006, 2011, 2012 Fujitsu Limited
Copyright (C) 2006, 2007 VA Linux Systems Japan K.K.
Copyright (C) 2005, 2011 NEC Corporation
Copyright (C) 1999, 2002, 2007 Silicon Graphics, Inc.
Copyright (C) 1999, 2000, 2001, 2002 Mission Critical Linux, Inc.
This program is free software, covered by the GNU General Public License,
and you are welcome to change it and/or distribute copies of it under
certain conditions. Enter "help copying" to see the conditions.
This program has absolutely no warranty. Enter "help warranty" for details.
GNU gdb (GDB) 7.6
Copyright (C) 2013 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law. Type "show copying"
and "show warranty" for details.
This GDB was configured as "x86_64-unknown-linux-gnu"...
KERNEL: /usr/lib/debug/lib/modules/3.10.0-693.el7.x86_64/vmlinux
DUMPFILE: /var/crash/127.0.0.1-2020-04-14-22:51:43/vmcore [PARTIAL DUMP]
CPUS: 56
DATE: Tue Apr 14 22:51:35 2020
UPTIME: 89 days, 01:59:36
LOAD AVERAGE: 3.88, 3.00, 3.06
TASKS: 2662
NODENAME: localhost
RELEASE: 3.10.0-693.el7.x86_64
VERSION: #1 SMP Tue Aug 22 21:09:27 UTC 2017
MACHINE: x86_64 (2599 Mhz)
MEMORY: 127.9 GB
PANIC: "general protection fault: 0000 [#1] SMP "
PID: 304
COMMAND: "kswapd1"
TASK: ffff88203e3b5ee0 [THREAD_INFO: ffff88103c110000]
CPU: 21
STATE: TASK_RUNNING (PANIC)
crash>
- help查看帮助文档
crash> help
* extend log rd task
alias files mach repeat timer
ascii foreach mod runq tree
bpf fuser mount search union
bt gdb net set vm
btop help p sig vtop
dev ipcs ps struct waitq
dis irq pte swap whatis
eval kmem ptob sym wr
exit list ptov sys q
crash version: 7.2.3-10.el7 gdb version: 7.6
For help on any command above, enter "help <command>".
For help on input options, enter "help input".
For help on output options, enter "help output".
- sys显示特殊系统信息,mach显示机器信息
crash> sys
KERNEL: /usr/lib/debug/lib/modules/3.10.0-693.el7.x86_64/vmlinux
DUMPFILE: /var/crash/127.0.0.1-2020-04-14-22:51:43/vmcore [PARTIAL DUMP]
CPUS: 56
DATE: Tue Apr 14 22:51:35 2020
UPTIME: 89 days, 01:59:36
LOAD AVERAGE: 3.88, 3.00, 3.06
TASKS: 2662
NODENAME: localhost
RELEASE: 3.10.0-693.el7.x86_64
VERSION: #1 SMP Tue Aug 22 21:09:27 UTC 2017
MACHINE: x86_64 (2599 Mhz)
MEMORY: 127.9 GB
PANIC: "general protection fault: 0000 [#1] SMP "
crash> mach
MACHINE TYPE: x86_64
MEMORY SIZE: 127.9 GB
CPUS: 56
PROCESSOR SPEED: 2599 Mhz
HZ: 1000
PAGE SIZE: 4096
KERNEL VIRTUAL BASE: ffff880000000000
KERNEL VMALLOC BASE: ffffc90000000000
KERNEL VMEMMAP BASE: ffffea0000000000
KERNEL START MAP: ffffffff80000000
KERNEL MODULES BASE: ffffffffc0000000
KERNEL STACK SIZE: 16384
IRQ STACK SIZE: 16384
IRQ STACKS:
CPU 0: ffff88103e600000
...
- log显示内核消息缓冲区
crash> log
[ 0.000000] Initializing cgroup subsys cpuset
[ 0.000000] Initializing cgroup subsys cpu
[ 0.000000] Initializing cgroup subsys cpuacct
[ 0.000000] Linux version 3.10.0-693.el7.x86_64 (builder@kbuilder.dev.centos.org) (gcc version 4.8.5 20150623 (Red Hat 4.8.5-16) (GCC) ) #1 SMP Tue Aug 22 21:09:27 UTC 2017
[ 0.000000] Command line: BOOT_IMAGE=/vmlinuz-3.10.0-693.el7.x86_64 root=/dev/mapper/centos-root ro crashkernel=auto rd.lvm.lv=centos/root rd.lvm.lv=centos/swap rhgb quiet ixgbe.allow_unsupported_sfp=1
...
[7696893.120810] cni0: port 2(veth7726f90c) entered disabled state
[7696893.134380] device veth7726f90c left promiscuous mode
[7696893.134391] cni0: port 2(veth7726f90c) entered disabled state
[7697028.796302] general protection fault: 0000 [#1] SMP
[7697028.796362] Modules linked in: binfmt_misc tcp_diag inet_diag veth vxlan ip6_udp_tunnel udp_tunnel xt_statistic xt_nat xt_recent ipt_REJECT nf_reject_ipv4 xt_comment xt_mark ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 xt_addrtype iptable_filter xt_conntrack nf_nat nf_conntrack libcrc32c br_netfilter bridge stp llc overlay(T) nvidia_uvm(POE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache bonding sunrpc sb_edac edac_core intel_powerclamp nvidia_drm(POE) coretemp intel_rapl iosf_mbi snd_hda_codec_hdmi kvm_intel nvidia_modeset(POE) kvm nvidia(POE) irqbypass ipmi_ssif crc32_pclmul ghash_clmulni_intel iTCO_wdt aesni_intel iTCO_vendor_support lrw gf128mul glue_helper ablk_helper cryptd dcdbas snd_hda_intel snd_hda_codec snd_hda_core
[7697028.796863] snd_hwdep snd_seq snd_seq_device snd_pcm snd_timer snd sg pcspkr mei_me soundcore mei lpc_ich shpchp ipmi_si ipmi_devintf ipmi_msghandler acpi_power_meter ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mgag200 drm_kms_helper mxm_wmi syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ixgbe crct10dif_pclmul crct10dif_common crc32c_intel igb ahci libahci megaraid_sas libata mdio i2c_algo_bit ptp i2c_core pps_core dca wmi dm_mirror dm_region_hash dm_log dm_mod
[7697028.797186] CPU: 21 PID: 304 Comm: kswapd1 Tainted: P W OE ------------ T 3.10.0-693.el7.x86_64 #1
[7697028.797242] Hardware name: Dell Inc. PowerEdge R730/072T6D, BIOS 2.8.0 005/17/2018
[7697028.797285] task: ffff88203e3b5ee0 ti: ffff88103c110000 task.ti: ffff88103c110000
[7697028.797334] RIP: 0010:[<ffffffff810b5819>] [<ffffffff810b5819>] down_read_trylock+0x9/0x50
[7697028.797389] RSP: 0018:ffff88103c113bd8 EFLAGS: 00010286
[7697028.797421] RAX: 0000000000000000 RBX: ffff8808cefd24b0 RCX: 0000000000000000
[7697028.797461] RDX: 0000000000000000 RSI: 0000000000000000 RDI: 80000004116db86f
[7697028.797502] RBP: ffff88103c113bd8 R08: ffffea004fbafaa0 R09: ffff880771f0d538
[7697028.797543] R10: 0000000000000000 R11: ffffffffffffffff R12: ffff8808cefd24b1
[7697028.797583] R13: ffffea004fbafa80 R14: 80000004116db86f R15: ffffea004fbafa80
[7697028.797624] FS: 0000000000000000(0000) GS:ffff88203e080000(0000) knlGS:0000000000000000
[7697028.797669] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[7697028.797702] CR2: 000000c421e1deb8 CR3: 0000000d68ba7000 CR4: 00000000003407e0
[7697028.797743] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[7697028.797783] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
[7697028.797824] Stack:
[7697028.799647] ffff88103c113c08 ffffffff811bdca5 ffffea004fbafa80 0000000000000000
[7697028.801325] 0000000000000002 0000000000000000 ffff88103c113c88 ffffffff811bdf4a
[7697028.803006] ffff88096a737b90 00000000811957c3 00000007ef794fd0 ffff88103c113cd8
[7697028.804731] Call Trace:
[7697028.806379] [<ffffffff811bdca5>] page_lock_anon_vma_read+0x55/0x110
[7697028.808041] [<ffffffff811bdf4a>] page_referenced+0x1ea/0x3a0
[7697028.809671] [<ffffffff81197474>] shrink_active_list+0x1f4/0x370
[7697028.811310] [<ffffffff81198d98>] balance_pgdat+0x1c8/0x5e0
[7697028.812947] [<ffffffff81199323>] kswapd+0x173/0x440
[7697028.814563] [<ffffffff810b1910>] ? wake_up_atomic_t+0x30/0x30
[7697028.816167] [<ffffffff811991b0>] ? balance_pgdat+0x5e0/0x5e0
[7697028.817748] [<ffffffff810b098f>] kthread+0xcf/0xe0
[7697028.819336] [<ffffffff810b08c0>] ? insert_kthread_work+0x40/0x40
[7697028.820934] [<ffffffff816b4f18>] ret_from_fork+0x58/0x90
[7697028.822499] [<ffffffff810b08c0>] ? insert_kthread_work+0x40/0x40
[7697028.824056] Code: 01 00 00 00 48 89 f8 f0 48 01 10 79 05 e8 50 c3 27 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 <48> 8b 07 48 89 c2 48 83 c2 01 7e 07 f0 48 0f b1 17 75 f0 48 f7
[7697028.827340] RIP [<ffffffff810b5819>] down_read_trylock+0x9/0x50
[7697028.828934] RSP <ffff88103c113bd8>
- bt显示内核堆栈跟踪
crash> bt
PID: 304 TASK: ffff88203e3b5ee0 CPU: 21 COMMAND: "kswapd1"
#0 [ffff88103c113950] machine_kexec at ffffffff8105c4cb
#1 [ffff88103c1139b0] __crash_kexec at ffffffff81104a32
#2 [ffff88103c113a80] crash_kexec at ffffffff81104b20
#3 [ffff88103c113a98] oops_end at ffffffff816ad278
#4 [ffff88103c113ac0] die at ffffffff8102e97b
#5 [ffff88103c113af0] do_general_protection at ffffffff816acbfe
#6 [ffff88103c113b20] general_protection at ffffffff816ac4a8
[exception RIP: down_read_trylock+9]
RIP: ffffffff810b5819 RSP: ffff88103c113bd8 RFLAGS: 00010286
RAX: 0000000000000000 RBX: ffff8808cefd24b0 RCX: 0000000000000000
RDX: 0000000000000000 RSI: 0000000000000000 RDI: 80000004116db86f
RBP: ffff88103c113bd8 R8: ffffea004fbafaa0 R9: ffff880771f0d538
R10: 0000000000000000 R11: ffffffffffffffff R12: ffff8808cefd24b1
R13: ffffea004fbafa80 R14: 80000004116db86f R15: ffffea004fbafa80
ORIG_RAX: ffffffffffffffff CS: 0010 SS: 0018
#7 [ffff88103c113be0] page_lock_anon_vma_read at ffffffff811bdca5
#8 [ffff88103c113c10] page_referenced at ffffffff811bdf4a
#9 [ffff88103c113c90] shrink_active_list at ffffffff81197474
#10 [ffff88103c113d48] balance_pgdat at ffffffff81198d98
#11 [ffff88103c113e20] kswapd at ffffffff81199323
#12 [ffff88103c113ec8] kthread at ffffffff810b098f
#13 [ffff88103c113f50] ret_from_fork at ffffffff816b4f18
- ps显示系统中进程的状态
crash> ps -l
[7697028835422305] [IN] PID: 63750 TASK: ffff88102d7deeb0 CPU: 52 COMMAND: "tensorflow_mode"
[7697028835296150] [IN] PID: 63801 TASK: ffff880da0edeeb0 CPU: 50 COMMAND: "tensorflow_mode"
[7697028835149553] [IN] PID: 63805 TASK: ffff880a69a33f40 CPU: 52 COMMAND: "tensorflow_mode"
[7697028835146379] [IN] PID: 63745 TASK: ffff881f37f0dee0 CPU: 52 COMMAND: "tensorflow_mode"
[7697028835422305] [IN] PID: 63750 TASK: ffff88102d7deeb0 CPU: 52 COMMAND: "tensorflow_mode"
[7697028835296150] [IN] PID: 63801 TASK: ffff880da0edeeb0 CPU: 50 COMMAND: "tensorflow_mode"
[7697028835149553] [IN] PID: 63805 TASK: ffff880a69a33f40 CPU: 52 COMMAND: "tensorflow_mode"
[7697028835146379] [IN] PID: 63745 TASK: ffff881f37f0dee0 CPU: 52 COMMAND: "tensorflow_mode"
[7697028835127079] [IN] PID: 63963 TASK: ffff880fc4775ee0 CPU: 50 COMMAND: "tensorflow_mode"
[7697028835109861] [IN] PID: 63467 TASK: ffff881e46f86eb0 CPU: 54 COMMAND: "tensorflow_mode"
[7697028835107882] [IN] PID: 64864 TASK: ffff882015fb9fa0 CPU: 54 COMMAND: "tensorflow_mode"
[7697028835071696] [IN] PID: 63897 TASK: ffff881027a46eb0 CPU: 48 COMMAND: "tensorflow_mode"
[7697028835067381] [IN] PID: 64740 TASK: ffff880f82f09fa0 CPU: 48 COMMAND: "tensorflow_mode"
[7697028835055845] [IN] PID: 64777 TASK: ffff881039b55ee0 CPU: 50 COMMAND: "tensorflow_mode"
[7697028835051451] [IN] PID: 63522 TASK: ffff882014f40fd0 CPU: 54 COMMAND: "tensorflow_mode"
[7697028835048803] [IN] PID: 63477 TASK: ffff880a69a30fd0 CPU: 52 COMMAND: "tensorflow_mode"
[7697028834875997] [IN] PID: 64718 TASK: ffff881039b52f70 CPU: 55 COMMAND: "tensorflow_mode"
[7697028834302994] [IN] PID: 64174 TASK: ffff88203d29af70 CPU: 40 COMMAND: "tensorflow_mode"
[7697028834225861] [IN] PID: 63490 TASK: ffff881c31678000 CPU: 48 COMMAND: "tensorflow_mode"
[7697028834065588] [IN] PID: 63638 TASK: ffff882024e19fa0 CPU: 46 COMMAND: "tensorflow_mode"
[7697028834061715] [IN] PID: 63555 TASK: ffff881c3167eeb0 CPU: 40 COMMAND: "tensorflow_mode"
[7697028833999797] [IN] PID: 63481 TASK: ffff881bd8f95ee0 CPU: 38 COMMAND: "tensorflow_mode"
[7697028833998512] [IN] PID: 63636 TASK: ffff881f37b48000 CPU: 46 COMMAND: "tensorflow_mode"
[7697028833967808] [IN] PID: 63793 TASK: ffff880f82f08000 CPU: 42 COMMAND: "tensorflow_mode"
[7697028833850746] [IN] PID: 63529 TASK: ffff882005f53f40 CPU: 44 COMMAND: "tensorflow_mode"
[7697028833650931] [IN] PID: 63424 TASK: ffff881bd9a71fa0 CPU: 39 COMMAND: "tensorflow_mode"
[7697028833611278] [IN] PID: 63526 TASK: ffff882014f46eb0 CPU: 35 COMMAND: "tensorflow_mode"
[7697028833319209] [IN] PID: 63631 TASK: ffff882005f56eb0 CPU: 37 COMMAND: "tensorflow_mode"
[7697028833276481] [UN] PID: 172051 TASK: ffff880eb9a60fd0 CPU: 41 COMMAND: "sha1sum"
...
- runq显示所有runqueue里的task
crash> runq
CPU 0 RUNQUEUE: ffff88103e616cc0
CURRENT: PID: 0 TASK: ffffffff819f9480 COMMAND: "swapper/0"
RT PRIO_ARRAY: ffff88103e616e50
[no tasks queued]
CFS RB_ROOT: ffff88103e616d68
[120] PID: 63779 TASK: ffff882013f1bf40 COMMAND: "tensorflow_mode"
CPU 1 RUNQUEUE: ffff88203de16cc0
CURRENT: PID: 0 TASK: ffff8810e9f86eb0 COMMAND: "swapper/1"
RT PRIO_ARRAY: ffff88203de16e50
[no tasks queued]
CFS RB_ROOT: ffff88203de16d68
[no tasks queued]
CPU 2 RUNQUEUE: ffff88103e656cc0
CURRENT: PID: 0 TASK: ffff88016976eeb0 COMMAND: "swapper/2"
RT PRIO_ARRAY: ffff88103e656e50
[no tasks queued]
CFS RB_ROOT: ffff88103e656d68
[no tasks queued]
...
- mount显示已挂载的文件系统信息
crash> mount
MOUNT SUPERBLK TYPE DEVNAME DIRNAME
ffff880169af8180 ffff880169b08800 rootfs rootfs /
ffff88103c354000 ffff880168b70800 sysfs sysfs /sys
ffff88103c354180 ffff880169b0b800 proc proc /proc
ffff88103c354300 ffff8810e88b8800 devtmpfs devtmpfs /dev
ffff88103c354480 ffff8810e8e20800 securityfs securityfs /sys/kernel/security
ffff88103c354600 ffff880168b71000 tmpfs tmpfs /dev/shm
ffff88103c354780 ffff88103c0e1800 devpts devpts /dev/pts
ffff88103c354900 ffff880168b71800 tmpfs tmpfs /run
ffff88103c354a80 ffff880168b72000 tmpfs tmpfs /sys/fs/cgroup
ffff88103c354c00 ffff880168b72800 cgroup cgroup /sys/fs/cgroup/systemd
...
- vm显示当前上下文的虚拟内存信息,swap显示交换设备信息
crash> vm
PID: 304 TASK: ffff88203e3b5ee0 CPU: 21 COMMAND: "kswapd1"
MM PGD RSS TOTAL_VM
0 0 0k 0k
crash> swap
SWAP_INFO_STRUCT TYPE SIZE USED PCT PRI FILENAME
ffff881038605000 PARTITION 4194300k 977544k 23% -1 /dev/dm-1
- net显示网络设备列表
crash> net
NET_DEVICE NAME IP ADDRESS(ES)
ffff8801687a0000 lo 127.0.0.1
ffff88103a3d4000 em3
ffff88103bf7c000 em4
ffff881039980000 em1
ffff881038820000 em2
ffff88203a3fc000 bond0 10.24.10.14
ffff88103dcfe000 docker0 172.17.0.1
ffff8810002e2000 flannel.1 10.244.60.0
ffff88102deaa000 cni0 10.244.60.1
ffff882038a7c000 veth16c139fd
ffff88203c62d000 vethcc9f5933
ffff8820317fc000 veth3e01a449
- file显示有关当前上下文的打开文件的信息
crash> file
struct file {
union {
struct list_head fu_list;
struct callback_head fu_rcuhead;
} f_u;
struct path f_path;
struct inode *f_inode;
const struct file_operations *f_op;
spinlock_t f_lock;
int f_sb_list_cpu_deprecated;
atomic_long_t f_count;
unsigned int f_flags;
fmode_t f_mode;
loff_t f_pos;
struct fown_struct f_owner;
const struct cred *f_cred;
struct file_ra_state f_ra;
u64 f_version;
void *f_security;
void *private_data;
struct list_head f_ep_links;
struct list_head f_tfile_llink;
struct address_space *f_mapping;
struct mutex f_pos_lock;
}
SIZE: 256