在前方中提到过,经常使用的几个线程(进程)同步机制,其实都是调用futex来实现的,下面就从glibc2.8中把源码找出来对一下,看看到底是不是确实是如此,同时,对代码进行一些简单的分析。
每个细节的实现其实都可以展开来讲成独立的一篇,这里只是介绍同步机制的来龙去脉和实现,就不再对具体的细节进行展开,回头如果有时间,再对具体的每个技术进行专门的分析。
一、信号灯的实现
int __new_sem_wait (sem_t *sem)
{
/* We need to check whether we need to act upon a cancellation request here
because POSIX specifies that cancellation points "shall occur" in
sem_wait and sem_timedwait, which also means that they need to check
this regardless whether they block or not (unlike "may occur"
functions). See the POSIX Rationale for this requirement: Section
"Thread Cancellation Overview" [1] and austin group issue #1076 [2]
for thoughs on why this may be a suboptimal design.
[1] http://pubs.opengroup.org/onlinepubs/9699919799/xrat/V4_xsh_chap02.html
[2] http://austingroupbugs.net/view.php?id=1076 for thoughts on why this
\*/
__pthread_testcancel ();
//处理sem_wait
if (__new_sem_wait_fast ((struct new_sem *) sem, 0) == 0)
return 0;
else
return __new_sem_wait_slow((struct new_sem *) sem, NULL);
}
versioned_symbol (libpthread, \__new_sem_wait, sem_wait, GLIBC_2_1);
//处理新老两种库的sem的生成
#if SHLIB_COMPAT (libpthread, GLIBC_2_0, GLIBC_2_1)
int
attribute_compat_text_section
\__old_sem_wait (sem_t \*sem)
{
int \*futex = (int \*) sem;
int err;
//循环判断状态
do
{
if (atomic_decrement_if_positive (futex) > 0)
return 0;
/* Enable asynchronous cancellation. Required by the standard. \*/
int oldtype = \__pthread_enable_asynccancel ();
/* Always assume the semaphore is shared. \*/
err = lll_futex_wait (futex, 0, LLL_SHARED);
/* Disable asynchronous cancellation. \*/
\__pthread_disable_asynccancel (oldtype);
}
while (err == 0 || err == -EWOULDBLOCK);
\__set_errno (-err);
return -1;
}
compat_symbol (libpthread, \__old_sem_wait, sem_wait, GLIBC_2_0);
#endif
在新的2.8及以上版本中,兼容了老的sem和futex,又重新生成了一个新的sem,这个数据结构体比原来一个单纯的整型多了不少,看一下代码:
/* Semaphore variable structure. */
struct new_sem
{
#if \__HAVE_64B_ATOMICS
/* The data field holds both value (in the least-significant 32 bits) and
nwaiters. \*/
# if \__BYTE_ORDER == \__LITTLE_ENDIAN
# define SEM_VALUE_OFFSET 0
# elif \__BYTE_ORDER == \__BIG_ENDIAN
# define SEM_VALUE_OFFSET 1
# else
# error Unsupported byte order.
# endif
# define SEM_NWAITERS_SHIFT 32
# define SEM_VALUE_MASK (~(unsigned int)0)
uint64_t data;
int private;
int pad;
#else
# define SEM_VALUE_SHIFT 1
# define SEM_NWAITERS_MASK ((unsigned int)1)
unsigned int value;
int private;
int pad;
unsigned int nwaiters;
#endif
};
struct old_sem
{
unsigned int value;
};
不过怎么说呢,原理仍然是一样的。不过需要注意的是,网上很多的资料仍然是对老的进行的分析,所以在看代码时要与时俱进。
二、mutex实现
再看一下mutex的LOCK实现源码:
static int
__pthread_mutex_lock_full (pthread_mutex_t *mutex)
{
/* We cannot acquire the mutex nor has its owner died. Thus, try
to block using futexes. Set FUTEX_WAITERS if necessary so that
other threads are aware that there are potentially threads
blocked on the futex. Restart if oldval changed in the
meantime. \*/
if ((oldval & FUTEX_WAITERS) == 0)
{
//前文提到过的CAS
if (atomic_compare_and_exchange_bool_acq (&mutex->\__data.\__lock,
oldval | FUTEX_WAITERS,
oldval)
!= 0)
{
oldval = mutex->\__data.\__lock;
continue;
}
oldval |= FUTEX_WAITERS;
}
/* It is now possible that we share the FUTEX_WAITERS flag with
another thread; therefore, update assume_other_futex_waiters so
that we do not forget about this when handling other cases
above and thus do not cause lost wake-ups. \*/
assume_other_futex_waiters |= FUTEX_WAITERS;
/* Block using the futex and reload current lock value. \*/
lll_futex_wait (&mutex->\__data.\__lock, oldval,
PTHREAD_ROBUST_MUTEX_PSHARED (mutex));
oldval = mutex->\__data.\__lock;//前方提到参变量
}
/* We have acquired the mutex; check if it is still consistent. */
//此处即为一致性检查
if (\__builtin_expect (mutex->\__data.\__owner
== PTHREAD_MUTEX_NOTRECOVERABLE, 0))
{
/* This mutex is now not recoverable. \*/
mutex->__data.__count = 0;
int private = PTHREAD_ROBUST_MUTEX_PSHARED (mutex);
lll_unlock (mutex->__data.__lock, private);
/* FIXME This violates the mutex destruction requirements. See
\__pthread_mutex_unlock_full. \*/
THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
return ENOTRECOVERABLE;
}
}
三、条件变量的实现
glibc中条件变量的实现如下:
#define __lll_cond_lock(futex, private) \
((void) \
({ \
int \*\__futex = (futex); \
if (__glibc_unlikely (atomic_exchange_acq (\__futex, 2) != 0)) \
__lll_lock_wait (\__futex, private); \
}))
#define lll_cond_lock(futex, private) __lll_cond_lock (&(futex), private)
static void __attribute__ ((unused))
__condvar_acquire_lock (pthread_cond_t *cond, int private)
{
unsigned int s = atomic_load_relaxed (&cond->\__data.\__g1_orig_size);
while ((s & 3) == 0)
{
if (atomic_compare_exchange_weak_acquire (&cond->\__data.\__g1_orig_size,
&s, s | 1))
return;
/* TODO Spinning and back-off. \*/
}
/* We can't change from not acquired to acquired, so try to change to
acquired-with-futex-wake-request and do a futex wait if we cannot change
from not acquired. \*/
while (1)
{
while ((s & 3) != 2)
{
if (atomic_compare_exchange_weak_acquire
(&cond->__data.__g1_orig_size, &s, (s & ~(unsigned int) 3) | 2))
{
if ((s & 3) == 0)
return;
break;
}
/* TODO Back off. \*/
}
futex_wait_simple (&cond->__data.__g1_orig_size,
(s & ~(unsigned int) 3) | 2, private);
/* Reload so we see a recent value. \*/
s = atomic_load_relaxed (&cond->__data.__g1_orig_size);
}
}
/* See __condvar_acquire_lock. */
static void __attribute__ ((unused))
__condvar_release_lock (pthread_cond_t *cond, int private)
{
if ((atomic_fetch_and_release (&cond->__data.__g1_orig_size,
~(unsigned int) 3) & 3)
== 2)
futex_wake (&cond->__data.__g1_orig_size, 1, private);
}
四、系统调用
其实所有的调用,最终都要落到系统调用中来:
#define lll_futex_syscall(nargs, futexp, op, ...) \
({ \
INTERNAL_SYSCALL_DECL (\__err); \
long int \__ret = INTERNAL_SYSCALL (futex, \__err, nargs, futexp, op, \
\__VA_ARGS__); \
(__glibc_unlikely (INTERNAL_SYSCALL_ERROR_P (\__ret, \__err)) \
? -INTERNAL_SYSCALL_ERRNO (\__ret, \__err) : 0); \
})
#define lll_futex_wait(futexp, val, private) \
lll_futex_timed_wait (futexp, val, NULL, private)
#define lll_futex_timed_wait(futexp, val, timeout, private) \
lll_futex_syscall (4, futexp, \
__lll_private_flag (FUTEX_WAIT, private), \
val, timeout)
#define lll_futex_timed_wait_bitset(futexp, val, timeout, clockbit, private) \
lll_futex_syscall (6, futexp, \
__lll_private_flag (FUTEX_WAIT_BITSET | (clockbit), \
private), \
val, timeout, NULL /* Unused. */, \
FUTEX_BITSET_MATCH_ANY)
#define lll_futex_wake(futexp, nr, private) \
lll_futex_syscall (4, futexp, \
__lll_private_flag (FUTEX_WAKE, private), nr, 0)
/* Returns non-zero if error happened, zero if success. */
#define lll_futex_requeue(futexp, nr_wake, nr_move, mutex, val, private) \
lll_futex_syscall (6, futexp, \
__lll_private_flag (FUTEX_CMP_REQUEUE, private), \
nr_wake, nr_move, mutex, val)
/* Returns non-zero if error happened, zero if success. */
#define lll_futex_wake_unlock(futexp, nr_wake, nr_wake2, futexp2, private) \
lll_futex_syscall (6, futexp, \
__lll_private_flag (FUTEX_WAKE_OP, private), \
nr_wake, nr_wake2, futexp2, \
FUTEX_OP_CLEAR_WAKE_IF_GT_ONE)
/* Priority Inheritance support. */
#define lll_futex_wait_requeue_pi(futexp, val, mutex, private) \
lll_futex_timed_wait_requeue_pi (futexp, val, NULL, 0, mutex, private)
#define lll_futex_timed_wait_requeue_pi(futexp, val, timeout, clockbit, \
mutex, private) \
lll_futex_syscall (5, futexp, \
__lll_private_flag (FUTEX_WAIT_REQUEUE_PI \
| (clockbit), private), \
val, timeout, mutex)
#define lll_futex_cmp_requeue_pi(futexp, nr_wake, nr_move, mutex, \
val, private) \
lll_futex_syscall (6, futexp, \
__lll_private_flag (FUTEX_CMP_REQUEUE_PI, \
private), \
nr_wake, nr_move, mutex, val)
futex又可以分类成以下几部分:
normal futex:通用的futex.
pi-futex:pi协议的futex.
requeue-pi:使用队列的pi协议futex.
robust-futex:robust规则的futex.
lowlevellock提供三种锁的机制,一是基于0或1的互斥的锁规则,二是基于robust futex定义的锁规则,三是用于condition重新对临界区上锁的操作。在前文的参数介绍中也有类似的介绍。
这些代码最终又会调用下一篇分析中的内核部分代码。这样,就可以从应用到Glibc库再到内核把futex看得清清楚楚。