在写多线程的过程中,难免会遇到锁,一般使用的是mutex或者rwlock。
调试的过程中也发现,使用mutex的情况下,spinlock在perf会占有一定的比例。那么
1 mutex会在多大程度上影响程序的系能,什么情况下是值得使用的?
2 rwlock和mutex在实现上有什么不同?
3 spinlock和mutex有什么关系?
4 如何看待流行的非锁程序?性能会有提升吗?
从源码入手:mutex的接口使用,是pthread库中的pthread_mutex_lock,使用glibc-2.12.1,如下
/* Data structures for mutex handling. The structure of the attribute
type is not exposed on purpose. */
typedef union
{
struct <span style="background-color: rgb(255, 102, 102);">__pthread_mutex_s</span>
{
int <span style="color:#ff6666;">__lock</span>;
unsigned int __count;
int __owner;
#if __WORDSIZE == 64
unsigned int __nusers;
#endif
/* KIND must stay at this position in the structure to maintain
binary compatibility. */
int <span style="color:#ff6666;">__kind</span>;
#if __WORDSIZE == 64
int __spins;
__pthread_list_t __list;
# define __PTHREAD_MUTEX_HAVE_PREV 1
#else
unsigned int __nusers;
__extension__ union
{
int __spins;
__pthread_slist_t __list;
};
#endif
} __data;
char __size[__SIZEOF_PTHREAD_MUTEX_T];
long int __align;
} pthread_mutex_t;
int
<span style="background-color: rgb(255, 102, 102);">__pthread_mutex_lock </span>(mutex)
pthread_mutex_t *mutex;
{
assert (sizeof (mutex->__size) >= sizeof (mutex->__data));
unsigned int type = PTHREAD_MUTEX_TYPE (mutex);
if (__builtin_expect (type & ~PTHREAD_MUTEX_KIND_MASK_NP, 0))
return __pthread_mutex_lock_full (mutex);
pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
if (__builtin_expect (type, PTHREAD_MUTEX_TIMED_NP)
== PTHREAD_MUTEX_TIMED_NP)
{
simple:
/* Normal mutex. */
<span style="background-color: rgb(255, 102, 102);">LLL_MUTEX_LOCK </span>(mutex);
assert (mutex->__data.__owner == 0);
}
else if (__builtin_expect (type == PTHREAD_MUTEX_RECURSIVE_NP, 1))
{
/* Recursive mutex. */
/* Check whether we already hold the mutex. */
if (mutex->__data.__owner == id)
{
/* Just bump the counter. */
if (__builtin_expect (mutex->__data.__count + 1 == 0, 0))
/* Overflow of the counter. */
return EAGAIN;
++mutex->__data.__count;
return 0;
}
/* We have to get the mutex. */
LLL_MUTEX_LOCK (mutex);
assert (mutex->__data.__owner == 0);
mutex->__data.__count = 1;
}
else if (__builtin_expect (type == PTHREAD_MUTEX_ADAPTIVE_NP, 1))
{
if (! __is_smp)
goto simple;
if (LLL_MUTEX_TRYLOCK (mutex) != 0)
{
int cnt = 0;
int max_cnt = MIN (MAX_ADAPTIVE_COUNT,
mutex->__data.__spins * 2 + 10);
do
{
if (cnt++ >= max_cnt)
{
LLL_MUTEX_LOCK (mutex);
break;
}
#ifdef BUSY_WAIT_NOP
BUSY_WAIT_NOP;
#endif
}
while (LLL_MUTEX_TRYLOCK (mutex) != 0);
mutex->__data.__spins += (cnt - mutex->__data.__spins) / 8;
}
assert (mutex->__data.__owner == 0);
}
else
{
assert (type == PTHREAD_MUTEX_ERRORCHECK_NP);
/* Check whether we already hold the mutex. */
if (__builtin_expect (mutex->__data.__owner == id, 0))
return EDEADLK;
goto simple;
}
/* Record the ownership. */
mutex->__data.__owner = id;
#ifndef NO_INCR
++mutex->__data.__nusers;
#endif
return 0;
}
#define LLL_MUTEX_LOCK(mutex) \
lll_cond_lock ((mutex)->__data.__lock, PTHREAD_MUTEX_PSHARED (mutex))
#define lll_cond_lock(futex, private) \
(void) \
({ int ignore1, ignore2, ignore3; \
__asm __volatile (LOCK_INSTR "cmpxchgl %4, %2\n\t" \
"jnz 1f\n\t" \
".subsection 1\n\t" \
".type _L_cond_lock_%=, @function\n" \
"_L_cond_lock_%=:\n" \
"1:\tleaq %2, %%rdi\n" \
"2:\tsubq $128, %%rsp\n" \
"3:\tcallq <span style="background-color: rgb(255, 102, 102);">__lll_lock_wait</span>\n" \
"4:\taddq $128, %%rsp\n" \
"5:\tjmp 24f\n" \
"6:\t.size _L_cond_lock_%=, 6b-1b\n\t" \
".previous\n" \
LLL_STUB_UNWIND_INFO_5 \
"24:" \
: "=S" (ignore1), "=D" (ignore2), "=m" (futex), \
"=a" (ignore3) \
: "1" (2), "m" (futex), "3" (0), "0" (private) \
: "cx", "r11", "cc", "memory"); \
})
void
__lll_lock_wait (int *futex, int private)
{
if (*futex == 2)
lll_futex_wait (futex, 2, private);
while (atomic_exchange_acq (futex, 2) != 0)
lll_futex_wait (futex, 2, private);
}
#define lll_futex_wait(futex, val, private) \
lll_futex_timed_wait(futex, val, NULL, private)
#define lll_futex_timed_wait(futex, val, timeout, private) \
({ \
register const struct timespec *__to __asm ("r10") = timeout; \
int __status; \
register __typeof (val) _val __asm ("edx") = (val); \
__asm __volatile ("syscall" \
: "=a" (__status) \
: "0" (SYS_futex), "D" (futex), \
"S" (__lll_private_flag (FUTEX_WAIT, private)), \
"d" (_val), "r" (__to) \
: "memory", "cc", "r11", "cx"); \
__status; \
})
glibc最终会调用系统调用futex,系统调用futex的源码在linux-2.6.38中如下:
SYSCALL_DEFINE6(futex, u32 __user *, uaddr, int, op, u32, val,
struct timespec __user *, utime, u32 __user *, uaddr2,
u32, val3)
{
struct timespec ts;
ktime_t t, *tp = NULL;
u32 val2 = 0;
int cmd = op & FUTEX_CMD_MASK;
if (utime && (cmd == FUTEX_WAIT || cmd == FUTEX_LOCK_PI ||
cmd == FUTEX_WAIT_BITSET ||
cmd == FUTEX_WAIT_REQUEUE_PI)) {
if (copy_from_user(&ts, utime, sizeof(ts)) != 0)
return -EFAULT;
if (!timespec_valid(&ts))
return -EINVAL;
t = timespec_to_ktime(ts);
if (cmd == FUTEX_WAIT)
t = ktime_add_safe(ktime_get(), t);
tp = &t;
}
/*
* requeue parameter in 'utime' if cmd == FUTEX_*_REQUEUE_*.
* number of waiters to wake in 'utime' if cmd == FUTEX_WAKE_OP.
*/
if (cmd == FUTEX_REQUEUE || cmd == FUTEX_CMP_REQUEUE ||
cmd == FUTEX_CMP_REQUEUE_PI || cmd == FUTEX_WAKE_OP)
val2 = (u32) (unsigned long) utime;
return do_futex(uaddr, op, val, tp, uaddr2, val2, val3);
}
其中调用了do_futex
long do_futex(u32 __user *uaddr, int op, u32 val, ktime_t *timeout,
u32 __user *uaddr2, u32 val2, u32 val3)
{
int ret = -ENOSYS, cmd = op & FUTEX_CMD_MASK;
unsigned int flags = 0;
if (!(op & FUTEX_PRIVATE_FLAG))
flags |= FLAGS_SHARED;
if (op & FUTEX_CLOCK_REALTIME) {
flags |= FLAGS_CLOCKRT;
if (cmd != FUTEX_WAIT_BITSET && cmd != FUTEX_WAIT_REQUEUE_PI)
return -ENOSYS;
}
switch (cmd) {
case FUTEX_WAIT:
val3 = FUTEX_BITSET_MATCH_ANY;
case FUTEX_WAIT_BITSET:
ret = futex_wait(uaddr, flags, val, timeout, val3);
break;
case FUTEX_WAKE:
val3 = FUTEX_BITSET_MATCH_ANY;
case FUTEX_WAKE_BITSET:
ret = futex_wake(uaddr, flags, val, val3);
break;
case FUTEX_REQUEUE:
ret = futex_requeue(uaddr, flags, uaddr2, val, val2, NULL, 0);
break;
case FUTEX_CMP_REQUEUE:
ret = futex_requeue(uaddr, flags, uaddr2, val, val2, &val3, 0);
break;
case FUTEX_WAKE_OP:
ret = futex_wake_op(uaddr, flags, uaddr2, val, val2, val3);
break;
case FUTEX_LOCK_PI:
if (futex_cmpxchg_enabled)
ret = futex_lock_pi(uaddr, flags, val, timeout, 0);
break;
case FUTEX_UNLOCK_PI:
if (futex_cmpxchg_enabled)
ret = futex_unlock_pi(uaddr, flags);
break;
case FUTEX_TRYLOCK_PI:
if (futex_cmpxchg_enabled)
ret = futex_lock_pi(uaddr, flags, 0, timeout, 1);
break;
case FUTEX_WAIT_REQUEUE_PI:
val3 = FUTEX_BITSET_MATCH_ANY;
ret = futex_wait_requeue_pi(uaddr, flags, val, timeout, val3,
uaddr2);
break;
case FUTEX_CMP_REQUEUE_PI:
ret = futex_requeue(uaddr, flags, uaddr2, val, val2, &val3, 1);
break;
default:
ret = -ENOSYS;
}
return ret;
}
使用strace捕获系统调用时的一些参数如下:
futex(0x7fbf0e41e0ac, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x7fbf0e41e0a8, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1} <unfinished ...>
12312 write(8, "\1\0\0\0\0\0\0\0", 8 <unfinished ...>
12324 <... futex resumed> ) = 0
12307 <... futex resumed> ) = 1
12312 <... write resumed> ) = 8
12324 futex(0x7fbf0e41e080, FUTEX_WAIT_PRIVATE, 2, NULL <unfinished ...>
12312 futex(0x7fbf1a41e0ac, FUTEX_WAIT_PRIVATE, 1, NULL <unfinished ...>
12307 futex(0x7fbf0e41e080, FUTEX_WAKE_PRIVATE, 1 <unfinished ...>
12324 <... futex resumed> ) = -1 EAGAIN (Resource temporarily unavailable)
12307 <... futex resumed> ) = 0
12324 futex(0x7fbf0e41e080, FUTEX_WAKE_PRIVATE, 1 <unfinished ...>
12307 futex(0x7fbf1341e16c, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x7fbf1341e168, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1} <unfinished ...>
12324 <... futex resumed> ) = 0
12319 <... futex resumed> ) = 0
12307 <... futex resumed> ) = 1
12324 write(8, "\1\0\0\0\0\0\0\0", 8 <unfinished ...>
12319 futex(0x7fbf1341e140, FUTEX_WAIT_PRIVATE, 2, NULL <unfinished ...>
12307 futex(0x7fbf1341e140, FUTEX_WAKE_PRIVATE, 1 <unfinished ...>
12324 <... write resumed> ) = 8
12319 <... futex resumed> ) = -1 EAGAIN (Resource temporarily unavailable)
12307 <... futex resumed> ) = 0
12324 futex(0x7fbf0e41e0ac, FUTEX_WAIT_PRIVATE, 1, NULL <unfinished ...>
12319 futex(0x7fbf1341e140, FUTEX_WAKE_PRIVATE, 1 <unfinished ...>
12307 futex(0x7fbf1e42128c, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x7fbf1e421288, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1} <unfinished ...>
12319 <... futex resumed> ) = 0
12308 <... futex resumed> ) = 0
12307 <... futex resumed> ) = 1
12319 write(8, "\1\0\0\0\0\0\0\0", 8 <unfinished ...>
12308 futex(0x7fbf1e421260, FUTEX_WAIT_PRIVATE, 2, NULL <unfinished ...>
12307 futex(0x7fbf1e421260, FUTEX_WAKE_PRIVATE, 1 <unfinished ...>
grep其中一个线程的
12324 futex(0x7fbf0e41e080, FUTEX_WAKE_PRIVATE, 1 <unfinished ...>
12324 <... futex resumed> ) = 0
12324 write(8, "\1\0\0\0\0\0\0\0", 8 <unfinished ...>
12324 <... write resumed> ) = 8
12324 futex(0x7fbf0e41e0ac, FUTEX_WAIT_PRIVATE, 1, NULL <unfinished ...>
12324 <... futex resumed> ) = 0
12324 futex(0x7fbf0e41e080, FUTEX_WAIT_PRIVATE, 2, NULL <unfinished ...>
12324 <... futex resumed> ) = -1 EAGAIN (Resource temporarily unavailable)
12324 futex(0x7fbf0e41e080, FUTEX_WAKE_PRIVATE, 1 <unfinished ...>
12324 <... futex resumed> ) = 0
12324 write(8, "\1\0\0\0\0\0\0\0", 8) = 8
12324 futex(0x7fbf0e41e0ac, FUTEX_WAIT_PRIVATE, 1, NULL <unfinished ...>
12324 <... futex resumed> ) = 0
12324 futex(0x7fbf0e41e080, FUTEX_WAIT_PRIVATE, 2, NULL <unfinished ...>
12324 <... futex resumed> ) = -1 EAGAIN (Resource temporarily unavailable)
12324 futex(0x7fbf0e41e080, FUTEX_WAKE_PRIVATE, 1 <unfinished ...>
12324 <... futex resumed> ) = 0
12324 write(8, "\1\0\0\0\0\0\0\0", 8 <unfinished ...>
12324 <... write resumed> ) = 8
12324 futex(0x7fbf0e41e0ac, FUTEX_WAIT_PRIVATE, 1, NULL <unfinished ...>
12324 <... futex resumed> ) = 0
其中futex的第三个参数是futex的具体cmd,定义如下:
#define FUTEX_WAIT 0
#define FUTEX_WAKE 1
#define FUTEX_FD 2