Confirm which is NULL
1.
crash> bt
PID: 6165 TASK: ffff883ed37b4600 CPU: 5 COMMAND: "rds-stress"
#0 [ffff883ed2fdf7f0] machine_kexec at ffffffff81060320
#1 [ffff883ed2fdf860] crash_kexec at ffffffff81115ed8
#2 [ffff883ed2fdf930] oops_end at ffffffff8101a8d8
#3 [ffff883ed2fdf960] no_context at ffffffff8106fbd1
#4 [ffff883ed2fdf9b0] __bad_area_nosemaphore at ffffffff8106fdcd
#5 [ffff883ed2fdfa00] bad_area_nosemaphore at ffffffff8106fee3
#6 [ffff883ed2fdfa10] __do_page_fault at ffffffff81070468
#7 [ffff883ed2fdfa80] do_page_fault at ffffffff81070787
#8 [ffff883ed2fdfac0] page_fault at ffffffff8170329f
[exception RIP: rds_ib_recv_cache_put+48]
RIP: ffffffffa04a6a30 RSP: ffff883ed2fdfb78 RFLAGS: 00010006
RAX: 0000000000000005 RBX: ffff883ed4c8ec00 RCX: dead000000000200
RDX: 0000000000004000 RSI: 00000000000009c0 RDI: ffff883ee1f00f10
RBP: ffff883ed2fdfb98 R8: ffff883f6248d450 R9: 00000000000007b3
R10: ffff883ed2fdfc78 R11: ffff883f5d80b140 R12: ffff883ee1f00f10
R13: 00000000000009c0 R14: 0000000000000206 R15: ffff883ed4c8ec20
ORIG_RAX: ffffffffffffffff CS: 0010 SS: 0018
#9 [ffff883ed2fdfba0] rds_ib_inc_free at ffffffffa04a6f5c [rds_rdma]
#10 [ffff883ed2fdfbf0] rds_inc_put at ffffffffa05e330c [rds]
#11 [ffff883ed2fdfc10] rds_recvmsg at ffffffffa05e35f7 [rds]
#12 [ffff883ed2fdfd00] sock_recvmsg at ffffffff81603680
#13 [ffff883ed2fdfd30] ___sys_recvmsg at ffffffff81600ebe
#14 [ffff883ed2fdfeb0] __sys_recvmsg at ffffffff81602e79
#15 [ffff883ed2fdff40] sys_recvmsg at ffffffff81602ed9
#16 [ffff883ed2fdff50] system_call_fastpath at ffffffff816fdb1a
RIP: 0000003786ae9ac0 RSP: 00007ffe5cf1d9d8 RFLAGS: 00000246
RAX: ffffffffffffffda RBX: 00007ffe5cf1fe60 RCX: 0000003786ae9ac0
RDX: 0000000000000040 RSI: 00007ffe5cf1fd80 RDI: 0000000000000003
RBP: 00007ffe5cf1fe50 R8: 00007f583fa43000 R9: 0000000000000000
R10: 00000000000c4e8a R11: 0000000000000246 R12: 00007ffe5cf1fe60
R13: 00007ffe5cf21500 R14: 00007f583fa46800 R15: 00007ffe5cf21500
ORIG_RAX: 000000000000002f CS: 0033 SS: 002b
2.
crash> dis -lr rds_ib_recv_cache_put+48
/home/yanjzhu/rpmbuild/BUILD/kernel-4.1.12/linux-4.1.12-1.el6uek/net/rds/ib_recv.c: 715
0xffffffffa04a6a00 <rds_ib_recv_cache_put>: push %rbp
0xffffffffa04a6a01 <rds_ib_recv_cache_put+1>: mov %rsp,%rbp
0xffffffffa04a6a04 <rds_ib_recv_cache_put+4>: push %r14
0xffffffffa04a6a06 <rds_ib_recv_cache_put+6>: push %r13
0xffffffffa04a6a08 <rds_ib_recv_cache_put+8>: push %r12
0xffffffffa04a6a0a <rds_ib_recv_cache_put+10>: push %rbx
0xffffffffa04a6a0b <rds_ib_recv_cache_put+11>: nopl 0x0(%rax,%rax,1)
0xffffffffa04a6a10 <rds_ib_recv_cache_put+16>: mov %rdi,%r12
0xffffffffa04a6a13 <rds_ib_recv_cache_put+19>: mov %rsi,%r13
/home/yanjzhu/rpmbuild/BUILD/kernel-4.1.12/linux-4.1.12-1.el6uek/./arch/x86/include/asm/paravirt.h: 810
0xffffffffa04a6a16 <rds_ib_recv_cache_put+22>: pushfq
0xffffffffa04a6a17 <rds_ib_recv_cache_put+23>: pop %rax
0xffffffffa04a6a18 <rds_ib_recv_cache_put+24>: nopl 0x0(%rax,%rax,1)
0xffffffffa04a6a1d <rds_ib_recv_cache_put+29>: mov %rax,%r14
/home/yanjzhu/rpmbuild/BUILD/kernel-4.1.12/linux-4.1.12-1.el6uek/./arch/x86/include/asm/paravirt.h: 820
0xffffffffa04a6a20 <rds_ib_recv_cache_put+32>: cli
0xffffffffa04a6a21 <rds_ib_recv_cache_put+33>: nopw 0x0(%rax,%rax,1)
/home/yanjzhu/rpmbuild/BUILD/kernel-4.1.12/linux-4.1.12-1.el6uek/net/rds/ib_recv.c: 721
0xffffffffa04a6a27 <rds_ib_recv_cache_put+39>: mov %gs:0x5fb66702(%rip),%eax # 0xd130
0xffffffffa04a6a2e <rds_ib_recv_cache_put+46>: cltq
0xffffffffa04a6a30 <rds_ib_recv_cache_put+48>: mov (%rsi),%rbx
From the above, the register rsi address is to point to NULL.
3.
crash> whatis rds_ib_recv_cache_put
void rds_ib_recv_cache_put(struct list_head *, struct rds_ib_refill_cache *);
rdi is struct list_head *
rsi is struct rds_ib_refill_cache *
And from the above rsi is not changed from the beginning.
So struct rds_ib_refill_cache is NULL.
4.
...
cache = ic->rds_ibdev->i_cache_frags + ic->i_frag_cache_inx;
rds_ib_recv_cache_put(&frag->f_cache_entry, cache);
...
...
rdsdebug("freeing ibinc %p inc %p\n", ibinc, inc);
rds_ib_recv_cache_put(&ibinc->ii_cache_entry,
&ic->rds_ibdev->i_cache_incs);
...
There are 2 parameters:i_cache_incs and i_cache_frags.
5.
[exception RIP: rds_ib_recv_cache_put+48]
RIP: ffffffffa04a6a30 RSP: ffff883ed2fdfb78 RFLAGS: 00010006
RAX: 0000000000000005 RBX: ffff883ed4c8ec00 RCX: dead000000000200
RDX: 0000000000004000 RSI: 00000000000009c0 RDI: ffff883ee1f00f10
RBP: ffff883ed2fdfb98 R8: ffff883f6248d450 R9: 00000000000007b3
R10: ffff883ed2fdfc78 R11: ffff883f5d80b140 R12: ffff883ee1f00f10
R13: 00000000000009c0 R14: 0000000000000206 R15: ffff883ed4c8ec20
ORIG_RAX: ffffffffffffffff CS: 0010 SS: 0018
RSI is 00000000000009c0
crash> eval 00000000000009c0
hexadecimal: 9c0
decimal: 2496 <---this
octal: 4700
binary: 0000000000000000000000000000000000000000000000000000100111000000
6.
crash> struct rds_ib_device -o
struct rds_ib_device {
[0] struct list_head list;
[16] struct list_head ipaddr_list;
[32] struct list_head conn_list;
[48] struct ib_device *dev;
[56] struct ib_pd *pd;
[64] bool use_fastreg;
[68] int fastreg_cq_vector;
[72] struct ib_cq *fastreg_cq;
[80] struct ib_wc fastreg_wc[32];
[2128] struct ib_qp *fastreg_qp;
[2136] struct tasklet_struct fastreg_tasklet;
[2176] atomic_t fastreg_wrs;
[2184] struct rw_semaphore fastreg_lock;
[2224] struct work_struct fastreg_reset_w;
[2256] struct ib_mr *mr;
[2264] struct rds_ib_mr_pool *mr_1m_pool;
[2272] struct rds_ib_mr_pool *mr_8k_pool;
[2280] unsigned int fmr_max_remaps;
[2284] unsigned int max_8k_fmrs;
[2288] unsigned int max_1m_fmrs;
[2292] int max_sge;
[2296] unsigned int max_wrs;
[2300] unsigned int max_initiator_depth;
[2304] unsigned int max_responder_resources;
[2308] spinlock_t spinlock;
[2312] atomic_t refcount;
[2320] struct work_struct free_work;
[2352] struct rds_ib_srq *srq;
[2360] struct rds_ib_port *ports;
[2368] struct ib_event_handler event_handler;
[2400] int *vector_load;
[2408] struct mutex vector_load_lock;
[2448] atomic_t free_dev;
[2456] struct mutex free_dev_lock;
[2496] struct rds_ib_refill_cache i_cache_incs; <-----2496 is here.
[2528] struct rds_ib_refill_cache i_cache_frags[3];
}
SIZE: 2624
i_cache_incs is NULL.
And RSI is 00000000000009c0.
From the source code,
...
rdsdebug("freeing ibinc %p inc %p\n", ibinc, inc);
rds_ib_recv_cache_put(&ibinc->ii_cache_entry,
&ic->rds_ibdev->i_cache_incs);
...
ic->rds_ibdev->i_cache_incs is 00000000000009c0, so probably ic->rds_ibdev is NULL.
To now, this is a use-after-free problem.