pthread-w32 之 pthread_cond_wait 问题

pthread-w32 之 pthread_cond_wait 问题

前言

• 之前在别的项目中，发现 pthread_cond_wait 只触发一次，一直是“绕行”解决，Linux和Win32代码在这里不同步。最近，又一次需要 pthread_cond_wait 方案，不爽之余，勉力着手解决。

问题

• 第二次 pthread_cond_wait 时函数“死锁”

• 测试代码

  测试代码来自 POSIX Threads Programming；该例子只触发一次 pthread_cond_wait 就退出循环，本文为了测试多次触发 pthread_cond_wait，有小改动。

    /**!
    * \Copyright All copyright reserved 2010 - 2021
    * \Brief     
    * \Source	https://hpc.llnl.gov/training/tutorials
    * \Source   https://hpc-tutorials.llnl.gov/posix/example_using_cond_vars/
    **/
    #include <stdio.h>
    #include <stdlib.h>
    #ifdef   _MSC_VER
    #include <winsock2.h>
    #include <windows.h>
    #pragma comment(lib, "ws2_32.lib")
    #endif
  
    #include <pthread.h>
  
    #define NUM_THREADS  3
    #define TCOUNT 10
    #define COUNT_LIMIT 12
  
    int     count = 0;
    pthread_mutex_t count_mutex;
    pthread_cond_t count_threshold_cv;
  
    void *inc_count(void *t)
    {
        int i;
        long my_id = (long)t;
  
        for (i = 0; /*i < TCOUNT*/(count < COUNT_LIMIT); i++)
        {
            pthread_mutex_lock(&count_mutex);
            count++;
            /*
            Check the value of count and signal waiting thread when condition is
            reached.  Note that this occurs while mutex is locked.
            */
            printf("inc_count(): thread %ld, count = %d  Threshold reached. ", my_id, count);
            pthread_cond_signal(&count_threshold_cv);
            printf("Just sent signal.\n");
            //printf("inc_count(): thread %ld, count = %d, unlocking mutex\n", my_id, count);
            pthread_mutex_unlock(&count_mutex);
  
            /* Do some work so threads can alternate on mutex lock */
            usleep(500 * 1000);
        }
        pthread_exit(NULL);
        printf("inc_count(): thread %ld, count = %d done ", my_id, count);
  
        return NULL;
    }
  
    void *watch_count(void *t)
    {
        long my_id = (long)t;
  
        printf("Starting watch_count(): thread %ld\n", my_id);
  
        /*
        Lock mutex and wait for signal.  Note that the pthread_cond_wait routine
        will automatically and atomically unlock mutex while it waits.
        Also, note that if COUNT_LIMIT is reached before this routine is run by
        the waiting thread, the loop will be skipped to prevent pthread_cond_wait
        from never returning.
        */
        pthread_mutex_lock(&count_mutex);
        count = 0;
        while (count < COUNT_LIMIT) 
        {
            printf("watch_count(): thread %ld Count= %d. Going into wait...\n", my_id, count);
            pthread_cond_wait(&count_threshold_cv, &count_mutex);
  
            printf("watch_count(): thread %ld Condition signal received. Count= %d\n", my_id, count);
            //printf("watch_count(): thread %ld Updating the value of count...\n", my_id, count);
            //printf("watch_count(): thread %ld count now = %d.\n", my_id, count);
        }
        printf("watch_count(): thread %ld Unlocking mutex.\n", my_id);
        pthread_mutex_unlock(&count_mutex);
        pthread_exit(NULL);
  
        return NULL;
    }
  
    int main(int argc, char *argv[])
    {
        int i, rc;
        long t1 = 1, t2 = 2, t3 = 3;
        pthread_t threads[3];
        pthread_attr_t attr;
  
        /* Initialize mutex and condition variable objects */
        pthread_mutex_init(&count_mutex, NULL);
        pthread_cond_init(&count_threshold_cv, NULL);
  
        /* For portability, explicitly create threads in a joinable state */
        pthread_attr_init(&attr);
        pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE);
        pthread_create(&threads[0], &attr, watch_count, (void *)t1);
        pthread_create(&threads[1], &attr, inc_count, (void *)t2);
        pthread_create(&threads[2], &attr, inc_count, (void *)t3);
  
        /* Wait for all threads to complete */
        for (i = 0; i < NUM_THREADS; i++) 
        {
            pthread_join(threads[i], NULL);
        }
        printf("Main(): Waited and joined with %d threads. Final value of count = %d. Done.\n",
            NUM_THREADS, count);
  
        /* Clean up and exit */
        rc = pthread_attr_destroy(&attr);
        rc = pthread_mutex_destroy(&count_mutex);
        rc = pthread_cond_destroy(&count_threshold_cv);
  
        pthread_exit(NULL);
        return 0;
    }
  

跟踪分析

• 单步执行 pthread_cond_wait，有如下发现

  	/* Thread can be cancelled in sem_wait() but this is OK */
    if (sem_wait(&(cv->semBlockLock)) != 0)     // sem->value 1-->0
    {
    	return errno;
    }
  
    ++(cv->nWaitersBlocked);
  
    if (sem_post(&(cv->semBlockLock)) != 0)     // sem->value 0-->1
    {
    	return errno;
    }
  
    /*
     * Setup this waiter cleanup handler
     */
    cleanup_args.mutexPtr = mutex;
    cleanup_args.cv = cv;
    cleanup_args.resultPtr = &result;
  
    #if defined(_MSC_VER) && _MSC_VER < 1400
    #pragma inline_depth(0)
    #endif
    pthread_cleanup_push(ptw32_cond_wait_cleanup, (void *)&cleanup_args);
  
    /*
    * Now we can release 'mutex' and...
    */
    if ((result = pthread_mutex_unlock(mutex)) == 0)
    {
        if (sem_timedwait(&(cv->semBlockQueue), abstime) != 0)     // sem->value 1-->0
        {
            result = errno;
        }
    }
  
    /*
     * Always cleanup
     */
    pthread_cleanup_pop(1);                 // 敲黑板、说重点：sem->value == 0，没有恢复！
  

• 继续看cleanup.c

    ptw32_cleanup_t * ptw32_pop_cleanup(int execute)
    {
        ptw32_cleanup_t *cleanup;
  
        cleanup = (ptw32_cleanup_t *)pthread_getspecific(ptw32_cleanupKey);
        
        if (cleanup != NULL)  // 这儿，cleanup == NULL， ptw32_cleanupKey == 0
        {
            if (execute && (cleanup->routine != NULL))
            {
                (*cleanup->routine) (cleanup->arg);
            }
            pthread_setspecific(ptw32_cleanupKey, (void *)cleanup->prev);
        }
        return (cleanup);
    }				/* ptw32_pop_cleanup */
  

• 跟踪 pthread_getspecific(ptw32_cleanupKey), 若 ptw32_cleanupKey 为 0 直接返回！

  到这里，基本上可以确定是使用 pthread-w32 的问题!

• 那么，为什么 ptw32_cleanupKey 是 0 呢？怎样初始化这个变量呢？

  搜索 pthread-w32 代码，在这一行找到答案

  ./ptw32_processInitialize.c:84:      (pthread_key_create (&ptw32_cleanupKey, NULL) != 0))
  

  然后，打开源代码，发现是在ptw32_processInitialize (void)函数中。

解决

• 在 main 函数中，添加如下调用

  #ifdef _MSC_VER
    ptw32_processInitialize();
  #endif
  ... // do somethings
  #ifdef _MSC_VER
    ptw32_processTerminate();
  #endif
  

  问题解决, enjoy it !

后记

• 百度ptw32_processInitialize，在 fychit 的博客 跨平台多线程编程 --by- fychit, 2010/5/6 (http://blog.chinaunix.net/uid-20776117-id-1847029.html) 中有讲到这个初始化函数，摘录如下：

  // 在程序起始处对 libpthread 作初始化：
  #if defined(PTW32_STATIC_LIB)
      ptw32_processInitialize();
  #endif
  

• 查阅文档 README.NONPORTABLE ，有如下说明

  BOOL pthread_win32_process_attach_np (void);
  BOOL pthread_win32_process_detach_np (void);
  BOOL pthread_win32_thread_attach_np (void);
  BOOL pthread_win32_thread_detach_np (void);
  
  These functions contain the code normally run via dllMain
  when the library is used as a dll but which need to be
  called explicitly by an application when the library
  is statically linked. As of version 2.9.0 of the library, static
  builds using either MSC or GCC will call pthread_win32_process_*
  automatically at application startup and exit respectively.
  
  Otherwise, you will need to call pthread_win32_process_attach_np()
  before you can call any pthread routines when statically linking.
  You should call pthread_win32_process_detach_np() before
  exiting your application to clean up. 
  
  pthread_win32_thread_attach_np() is currently a no-op, but 
  pthread_win32_thread_detach_np() is needed to clean up
  the implicit pthread handle that is allocated to a Win32 thread if
  it calls any pthreads routines. Call this routine when the 
  Win32 thread exits.
  
  Threads created through pthread_create() do not need to call
  pthread_win32_thread_detach_np().
  
  These functions invariably return TRUE except for 
  pthread_win32_process_attach_np() which will return FALSE
  if pthreads-win32 initialisation fails.
  
  

  如上描述，在 application 入口和出口，需要有如下的 non-portable 调用

  #ifdef PTW32_STATIC_LIB
      pthread_win32_process_attach_np();
  #endif
  /* do something with pthread library */
  #ifdef PTW32_STATIC_LIB
  pthread_win32_process_detach_np();
  #endif  
  

参考

• http://www.sourceware.org/pthreads-win32/
• ftp://sourceware.org/pub/pthreads-win32
• http://mirrors.kernel.org/sourceware/pthreads-win32/
• https://hpc.llnl.gov/training/tutorials