nginx线程池浅析

nginx线程池

源代码在ngx_thread_pool.c中

nginx线程池简化流程

A线程准备任务—>post任务—>线程池唤醒一个线程处理–>通过回调函数通知A线程处理完毕

数据结构分析

任务

struct ngx_thread_task_s {
    ngx_thread_task_t   *next;		//指向下一个任务
    ngx_uint_t           id;		//任务ID
    void                *ctx;		//任务的上下文
    void               (*handler)(void *data, ngx_log_t *log);		//处理任务的函数句柄
    ngx_event_t          event;			//跟任务关联的事件对象（当线程池处理成任务之后将会由主线程调用event对象的handler回调函数）
};

队列

typedef struct {
    ngx_thread_task_t        *first;
    ngx_thread_task_t       **last;
} ngx_thread_pool_queue_t;

线程池

struct ngx_thread_pool_s {
    ngx_thread_mutex_t        mtx;		//互斥锁，用于锁定任务队列，避免竞争状态。
    ngx_thread_pool_queue_t   queue;	//任务队列
    ngx_int_t                 waiting;	//有多少个任务正在等待处理
    ngx_thread_cond_t         cond;		//用于通知线程池有任务需要处理

    ngx_log_t                *log;		

    ngx_str_t                 name;		//线程池名称
    ngx_uint_t                threads;	//线程池由多少个线程组成（线程数）
    ngx_int_t                 max_queue;	//线程池最大能处理的任务数

    u_char                   *file;		//配置文件名
    ngx_uint_t                line;		//thread_pool配置在配置文件中的行号
};

代码分析

入口函数ngx_thread_pool_init_worker

此函数会调用线程池初始化函数

static ngx_int_t
ngx_thread_pool_init_worker(ngx_cycle_t *cycle)
{
    ngx_uint_t                i;
    ngx_thread_pool_t       **tpp;
    ngx_thread_pool_conf_t   *tcf;

	//如果不是worker或者只有一个worker就不起用线程池
    if (ngx_process != NGX_PROCESS_WORKER
        && ngx_process != NGX_PROCESS_SINGLE)
    {
        return NGX_OK;
    }

    tcf = (ngx_thread_pool_conf_t *) ngx_get_conf(cycle->conf_ctx,
                                                  ngx_thread_pool_module);

    if (tcf == NULL) {
        return NGX_OK;
    }

    ngx_thread_pool_queue_init(&ngx_thread_pool_done);		//初始化任务完成队列

    tpp = tcf->pools.elts;

    for (i = 0; i < tcf->pools.nelts; i++) {
        if (ngx_thread_pool_init(tpp[i], cycle->log, cycle->pool) != NGX_OK) {
            return NGX_ERROR;
        }
    }

    return NGX_OK;
}

线程池初始化ngx_thread_pool_init

static ngx_int_t
ngx_thread_pool_init(ngx_thread_pool_t *tp, ngx_log_t *log, ngx_pool_t *pool)
{
    int             err;
    pthread_t       tid;
    ngx_uint_t      n;
    pthread_attr_t  attr;

    if (ngx_notify == NULL) {
        ngx_log_error(NGX_LOG_ALERT, log, 0,
               "the configured event method cannot be used with thread pools");
        return NGX_ERROR;
    }
	
	/*
		#define ngx_thread_pool_queue_init(q)                                         \
		    (q)->first = NULL;                                                        \
		    (q)->last = &(q)->first
		
		first指针赋值NULL，**last执行first的地址
	*/
    ngx_thread_pool_queue_init(&tp->queue);		//初始化线程池队列

    if (ngx_thread_mutex_create(&tp->mtx, log) != NGX_OK) {
        return NGX_ERROR;
    }

    if (ngx_thread_cond_create(&tp->cond, log) != NGX_OK) {
        (void) ngx_thread_mutex_destroy(&tp->mtx, log);
        return NGX_ERROR;
    }

    tp->log = log;

    err = pthread_attr_init(&attr);
    if (err) {
        ngx_log_error(NGX_LOG_ALERT, log, err,
                      "pthread_attr_init() failed");
        return NGX_ERROR;
    }

    err = pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);		//是指线程的属性为分离
    if (err) {
        ngx_log_error(NGX_LOG_ALERT, log, err,
                      "pthread_attr_setdetachstate() failed");
        return NGX_ERROR;
    }

#if 0
    err = pthread_attr_setstacksize(&attr, PTHREAD_STACK_MIN);
    if (err) {
        ngx_log_error(NGX_LOG_ALERT, log, err,
                      "pthread_attr_setstacksize() failed");
        return NGX_ERROR;
    }
#endif

    for (n = 0; n < tp->threads; n++) {
        err = pthread_create(&tid, &attr, ngx_thread_pool_cycle, tp);
        if (err) {
            ngx_log_error(NGX_LOG_ALERT, log, err,
                          "pthread_create() failed");
            return NGX_ERROR;
        }
    }

    (void) pthread_attr_destroy(&attr);

    return NGX_OK;
}

线程运行函数

static void *
ngx_thread_pool_cycle(void *data)
{
    ngx_thread_pool_t *tp = data;

    int                 err;
    sigset_t            set;
    ngx_thread_task_t  *task;

#if 0
    ngx_time_update();
#endif

    ngx_log_debug1(NGX_LOG_DEBUG_CORE, tp->log, 0,
                   "thread in pool \"%V\" started", &tp->name);

    sigfillset(&set);

    sigdelset(&set, SIGILL);
    sigdelset(&set, SIGFPE);
    sigdelset(&set, SIGSEGV);
    sigdelset(&set, SIGBUS);

    err = pthread_sigmask(SIG_BLOCK, &set, NULL);
    if (err) {
        ngx_log_error(NGX_LOG_ALERT, tp->log, err, "pthread_sigmask() failed");
        return NULL;
    }

    for ( ;; ) {
    	//获取任务前先锁定
        if (ngx_thread_mutex_lock(&tp->mtx, tp->log) != NGX_OK) {
            return NULL;
        }

        /* the number may become negative */
        tp->waiting--;
		
		//判断任务队列是否为空，假如为空则条件等待
        while (tp->queue.first == NULL) {
            if (ngx_thread_cond_wait(&tp->cond, &tp->mtx, tp->log)
                != NGX_OK)
            {
                (void) ngx_thread_mutex_unlock(&tp->mtx, tp->log);
                return NULL;
            }
        }
	
		//获取任务
        task = tp->queue.first;
        tp->queue.first = task->next;

        if (tp->queue.first == NULL) {
            tp->queue.last = &tp->queue.first;
        }

        if (ngx_thread_mutex_unlock(&tp->mtx, tp->log) != NGX_OK) {
            return NULL;
        }

#if 0
        ngx_time_update();
#endif

        ngx_log_debug2(NGX_LOG_DEBUG_CORE, tp->log, 0,
                       "run task #%ui in thread pool \"%V\"",
                       task->id, &tp->name);

		//处理任务
        task->handler(task->ctx, tp->log);

        ngx_log_debug2(NGX_LOG_DEBUG_CORE, tp->log, 0,
                       "complete task #%ui in thread pool \"%V\"",
                       task->id, &tp->name);

        task->next = NULL;

		//通过自旋锁锁定，自旋锁会一直循环，不会让出cpu，适用于执行时间消耗小，比切换线程消耗的时间少
        ngx_spinlock(&ngx_thread_pool_done_lock, 1, 2048);

		//把处理完的任务放入完成队列
        *ngx_thread_pool_done.last = task;
        ngx_thread_pool_done.last = &task->next;

        ngx_memory_barrier();		//防止编译器优化，保证解锁操作是在上述语句执行完毕后再去执行的

        ngx_unlock(&ngx_thread_pool_done_lock);

        (void) ngx_notify(ngx_thread_pool_handler);		//通知主线程
    }
}

主线程post任务

ngx_int_t
ngx_thread_task_post(ngx_thread_pool_t *tp, ngx_thread_task_t *task)
{
	//判断任务是否已经添加过
    if (task->event.active) {
        ngx_log_error(NGX_LOG_ALERT, tp->log, 0,
                      "task #%ui already active", task->id);
        return NGX_ERROR;
    }

    if (ngx_thread_mutex_lock(&tp->mtx, tp->log) != NGX_OK) {
        return NGX_ERROR;
    }

	//待处理任务数达到线程池所能处理的最大数量，则报错
    if (tp->waiting >= tp->max_queue) {
        (void) ngx_thread_mutex_unlock(&tp->mtx, tp->log);

        ngx_log_error(NGX_LOG_ERR, tp->log, 0,
                      "thread pool \"%V\" queue overflow: %i tasks waiting",
                      &tp->name, tp->waiting);
        return NGX_ERROR;
    }

	//置位任务是否已经添加到任务队列
    task->event.active = 1;
	//ngx_thread_pool_task_id是一个全局变量，每添加一个任务则自增
    task->id = ngx_thread_pool_task_id++;
    task->next = NULL;

    if (ngx_thread_cond_signal(&tp->cond, tp->log) != NGX_OK) {
        (void) ngx_thread_mutex_unlock(&tp->mtx, tp->log);
        return NGX_ERROR;
    }

	//插入队列的一个小技巧，*last指向task，task赋值为task->next的地址，待下次添加时，把newtask赋值给*last，即task->next=newtask
    *tp->queue.last = task;
    tp->queue.last = &task->next;
	//待处理任务数自增
    tp->waiting++;

    (void) ngx_thread_mutex_unlock(&tp->mtx, tp->log);

    ngx_log_debug2(NGX_LOG_DEBUG_CORE, tp->log, 0,
                   "task #%ui added to thread pool \"%V\"",
                   task->id, &tp->name);

    return NGX_OK;
}

通知主线程函数ngx_thread_pool_handler

static void
ngx_thread_pool_handler(ngx_event_t *ev)
{
    ngx_event_t        *event;
    ngx_thread_task_t  *task;

    ngx_log_debug0(NGX_LOG_DEBUG_CORE, ev->log, 0, "thread pool handler");

	//函数：基于原子操作的自旋锁方法ngx_spinlock的实现
	//参数解释：
	//lock：原子变量表达的锁
	//value：标志位，锁是否被某一进程占用
	//spin：在多处理器系统内，当ngx_spinlock方法没有拿到锁时，当前进程在内核的一次调度中该方法等待其他处理器释放锁的时间
    ngx_spinlock(&ngx_thread_pool_done_lock, 1, 2048);

	//取出完成队列中所有的任务
    task = ngx_thread_pool_done.first;
    ngx_thread_pool_done.first = NULL;
    ngx_thread_pool_done.last = &ngx_thread_pool_done.first;

    ngx_memory_barrier();

    ngx_unlock(&ngx_thread_pool_done_lock);

    while (task) {
        ngx_log_debug1(NGX_LOG_DEBUG_CORE, ev->log, 0,
                       "run completion handler for task #%ui", task->id);

		//获取任务中的完成后事件
        event = &task->event;
        task = task->next;
		//标志事件已完成，并且未处于任务队列中标志
        event->complete = 1;
        event->active = 0;
		//执行事件
        event->handler(event);
    }
}

线程池结束函数

static void
ngx_thread_pool_exit_worker(ngx_cycle_t *cycle)
{
    ngx_uint_t                i;
    ngx_thread_pool_t       **tpp;
    ngx_thread_pool_conf_t   *tcf;

    if (ngx_process != NGX_PROCESS_WORKER
        && ngx_process != NGX_PROCESS_SINGLE)
    {
        return;
    }

    tcf = (ngx_thread_pool_conf_t *) ngx_get_conf(cycle->conf_ctx,
                                                  ngx_thread_pool_module);

    if (tcf == NULL) {
        return;
    }

    tpp = tcf->pools.elts;		//获取线程池的数组

    for (i = 0; i < tcf->pools.nelts; i++) {
        ngx_thread_pool_destroy(tpp[i]);	//销毁各线程池
    }
}

销毁线程池

static void
ngx_thread_pool_destroy(ngx_thread_pool_t *tp)
{
    ngx_uint_t           n;
    ngx_thread_task_t    task;
    volatile ngx_uint_t  lock;

    ngx_memzero(&task, sizeof(ngx_thread_task_t));

	//新建销毁线程池任务
    task.handler = ngx_thread_pool_exit_handler;
    task.ctx = (void *) &lock;

	//线程池中有多少个线程就发起多少个销毁消除池任务
    for (n = 0; n < tp->threads; n++) {
        lock = 1;
		//添加销毁任务至任务队列
        if (ngx_thread_task_post(tp, &task) != NGX_OK) {
            return;
        }
		//等到任务被执行完后在继续执行，否则让出cpu
        while (lock) {
            ngx_sched_yield();
        }

        task.event.active = 0;
    }

	//释放条件变量
    (void) ngx_thread_cond_destroy(&tp->cond, tp->log);
	//释放互斥锁
    (void) ngx_thread_mutex_destroy(&tp->mtx, tp->log);
}

thread_pool线程池的使用示例

nginx中的线程池主要是用于操作文件的IO操作。所以，在nginx中自带的模块ngx_http_file_cache.c文件中看到了线程池的使用。

/*********************** nginx/src/os/unix/ngx_files.c  **********************/
//file_cache模块的处理函数（涉及到了线程池）
static ssize_t  ngx_http_file_cache_aio_read(ngx_http_request_t *r, ngx_http_cache_t *c)
{
    .......
#if (NGX_THREADS)
  
    if (clcf->aio == NGX_HTTP_AIO_THREADS) {
        c->file.thread_task = c->thread_task;
        //这里注册的函数在下面语句中的ngx_thread_read函数中被调用
        c->file.thread_handler = ngx_http_cache_thread_handler;
        c->file.thread_ctx = r;
        //根据任务的属性，选择正确的线程池，并初始化task结构体中的各个成员       
        n = ngx_thread_read(&c->file, c->buf->pos, c->body_start, 0, r->pool);
  
        c->thread_task = c->file.thread_task;
        c->reading = (n == NGX_AGAIN);
  
        return n;
    }
#endif
  
    return ngx_read_file(&c->file, c->buf->pos, c->body_start, 0);
}
  
  
//task任务的处理函数
static ngx_int_t  ngx_http_cache_thread_handler(ngx_thread_task_t *task, ngx_file_t *file)
{
    .......
    tp = clcf->thread_pool;
    .......
     
    task->event.data = r;
    //注册thread_event_handler函数，该函数在处理pool_done队列中event事件时被调用
    task->event.handler = ngx_http_cache_thread_event_handler;
  
    //将任务放到线程池的任务队列中
    if (ngx_thread_task_post(tp, task) != NGX_OK) {
        return NGX_ERROR;
    }
    ......
}
  
/*********************** nginx/src/core/ngx_thread_pool.c  **********************/
//添加任务到队列中
ngx_int_t  ngx_thread_task_post(ngx_thread_pool_t *tp, ngx_thread_task_t *task)
{
    //如果当前的任务正在处理就退出
    if (task->event.active) {
        ngx_log_error(NGX_LOG_ALERT, tp->log, 0,
                      "task #%ui already active", task->id);
        return NGX_ERROR;
    }
  
    if (ngx_thread_mutex_lock(&tp->mtx, tp->log) != NGX_OK) {
        return NGX_ERROR;
    }
     
    //判断当前线程池等待的任务数量与最大队列长度的关系
    if (tp->waiting >= tp->max_queue) {
        (void) ngx_thread_mutex_unlock(&tp->mtx, tp->log);
  
        ngx_log_error(NGX_LOG_ERR, tp->log, 0,
                      "thread pool \"%V\" queue overflow: %i tasks waiting",
                      &tp->name, tp->waiting);
        return NGX_ERROR;
    }
    //激活任务
    task->event.active = 1;
  
    task->id = ngx_thread_pool_task_id++;
    task->next = NULL;
      
    //通知阻塞的线程有新事件加入，可以解除阻塞
    if (ngx_thread_cond_signal(&tp->cond, tp->log) != NGX_OK) {
        (void) ngx_thread_mutex_unlock(&tp->mtx, tp->log);
        return NGX_ERROR;
    }
  
    *tp->queue.last = task;
    tp->queue.last = &task->next;
  
    tp->waiting++;
  
    (void) ngx_thread_mutex_unlock(&tp->mtx, tp->log);
  
    ngx_log_debug2(NGX_LOG_DEBUG_CORE, tp->log, 0,
                   "task #%ui added to thread pool \"%V\"",
                   task->id, &tp->name);
  
    return NGX_OK;
}