7. nginx线程池

最新推荐文章于 2024-05-05 09:28:28 发布

yanghuo11

最新推荐文章于 2024-05-05 09:28:28 发布

阅读量365

点赞数

分类专栏： nginx 文章标签： nginx 多线程 linux

本文链接：https://blog.csdn.net/qq_41252394/article/details/110674290

版权

nginx 专栏收录该内容

10 篇文章 0 订阅

订阅专栏

文章目录

nginx模块
nginx线程池

nginx模块

模块定义

struct ngx_module_s {
    ngx_uint_t            ctx_index;							//同类的数组序号(类似二次索引)
    ngx_uint_t            index;								//模块数组序号
    char                 *name;									//模块名
    ngx_uint_t            spare0;								//保留
    ngx_uint_t            spare1;

    ngx_uint_t            version;								//nginx版本
    const char           *signature;							//模块的签名

    void                 *ctx;									//特定的模块
    ngx_command_t        *commands;								//配置指令数组
    ngx_uint_t            type;									//模块类型
	
    //进程,线程初始化和退出的回调,
    ngx_int_t           (*init_master)(ngx_log_t *log);	
    ngx_int_t           (*init_module)(ngx_cycle_t *cycle);
    ngx_int_t           (*init_process)(ngx_cycle_t *cycle);
    ngx_int_t           (*init_thread)(ngx_cycle_t *cycle);
    void                (*exit_thread)(ngx_cycle_t *cycle);
    void                (*exit_process)(ngx_cycle_t *cycle);
    void                (*exit_master)(ngx_cycle_t *cycle);

    uintptr_t             spare_hook0;							//保留
	...
    uintptr_t             spare_hook7;
};

配置指令结构

struct ngx_command_s {
    ngx_str_t             name;		//配置的指令名
    ngx_uint_t            type;		//参数类型
    char               *(*set)(ngx_conf_t *cf, ngx_command_t *cmd, void *conf);//解析函数
    ngx_uint_t            conf;		//配置结构的存储位置
    ngx_uint_t            offset;	//配置结构的偏移
    void                 *post;
};

模块的组织

文件是obj/ngx_modules.c

//别的文件声明的全局变量(每个模块都有ngx_module_t的全局变量)
extern ngx_module_t  ngx_core_module;
extern ngx_module_t  ngx_errlog_module;
extern ngx_module_t  ngx_conf_module;
extern ngx_module_t  ngx_regex_module;
...
//以组织所有的模块
ngx_module_t *ngx_modules[] = {
    &ngx_core_module,
    &ngx_errlog_module,
    &ngx_conf_module,
    &ngx_regex_module,
    &ngx_events_module,
	...,
    NULL
}
//所有模块的名字,下标和ngx_modules是对应的
char *ngx_module_names[] = {
    "ngx_core_module",
    "ngx_errlog_module",
    "ngx_conf_module",
    "ngx_regex_module",
    "ngx_events_module",
    ...,
    NULL
}

这个文件有两个全局变量,ngx_modules组织所有模块,ngx_module_names组织所有模块名。每个数组都是以’NULL’(哨兵)结尾。

core模块

类型:ngx_module_t->type = NGX_CORE_MODULE;

core模块的ctx

typedef struct {
    ngx_str_t             name;											//模块名
    void               *(*create_conf)(ngx_cycle_t *cycle);				//创建配置
    char               *(*init_conf)(ngx_cycle_t *cycle, void *conf);	//初始化配置
} ngx_core_module_t;

ngx_module_t->ctx = ngx_core_module_t;

core模块的执行流程

nginx模块以ngx_modules组织，可以直接下内存读写断点

在这里插入图片描述

给ngx_modules下内存读写断点，然后一直 ‘c’ 可以看到两个执行函数

main() -> ngx_preinit_modules()

ngx_int_t ngx_preinit_modules(void)
{
    ngx_uint_t  i;
	//初始化模块编号和名字
    for (i = 0; ngx_modules[i]; i++) {
        ngx_modules[i]->index = i;
        ngx_modules[i]->name = ngx_module_names[i];
    }
	//初始化静态模块数，和最大的模块数
    ngx_modules_n = i;
    ngx_max_module = ngx_modules_n + NGX_MAX_DYNAMIC_MODULES;//+动态模块的数量

    return NGX_OK;
}

这个函数初始化了所有模块的index和name

main() -> ngx_init_cycle() -> ngx_cycle_modules()

ngx_int_t ngx_cycle_modules(ngx_cycle_t *cycle)
{
    cycle->modules = ngx_pcalloc(cycle->pool, (ngx_max_module + 1)
                                              * sizeof(ngx_module_t *));
    if (cycle->modules == NULL) {
        return NGX_ERROR;
    }
	//指针直接拷贝
    ngx_memcpy(cycle->modules, ngx_modules,
               ngx_modules_n * sizeof(ngx_module_t *));
	//模块数量
    cycle->modules_n = ngx_modules_n;

    return NGX_OK;
}

这个函数把模块直接拷贝到cycle->modules中。

如果在’c’下去会发现nginx直接启动了,这意味这nginx后续并没有使用ngx_modules。所以我们要换一个内存断点。换成cycle->modules

给cycle->modules下内存读写断点

在这里插入图片描述

下完之后继续’c’

main() -> ngx_init_cycle()

//执行NGX_CORE_MODULE模块的create_conf函数
for (i = 0; cycle->modules[i]; i++) {
    if (cycle->modules[i]->type != NGX_CORE_MODULE) {
        continue;
    }

    module = cycle->modules[i]->ctx;

    if (module->create_conf) {
        rv = module->create_conf(cycle);
        if (rv == NULL) {
            ngx_destroy_pool(pool);
            return NULL;
        }
        //create_conf 返回值存放的地方
        cycle->conf_ctx[cycle->modules[i]->index] = rv;
    }
}

执行NGX_CORE_MODULE模块的create_conf函数

main() -> ngx_init_cycle() -> ngx_conf_parse() -> ngx_conf_handler()

for (i = 0; cf->cycle->modules[i]; i++) {
    ...
    rv = cmd->set(cf, cmd, conf);
}

调用module->commands->set解析配置参数

如果再’c’下去就是各个模块的组织配置了，这里只跟core模块，所以把断点delete掉,然后finish到ngx_init_cycle函数,最后继续下cycle->modules断点’c’下去

main() -> ngx_init_cycle()

//调用NGX_CORE_MODULE模块的init_conf
for (i = 0; cycle->modules[i]; i++) {
    if (cycle->modules[i]->type != NGX_CORE_MODULE) {
        continue;
    }

    module = cycle->modules[i]->ctx;

    if (module->init_conf) {
        if (module->init_conf(cycle,
                              cycle->conf_ctx[cycle->modules[i]->index])
            == NGX_CONF_ERROR)
        {
            environ = senv;
            ngx_destroy_cycle_pools(&conf);
            return NULL;
        }
    }
}

调用NGX_CORE_MODULE模块的init_conf

main() -> ngx_init_cycle() -> ngx_init_modules()

ngx_int_t ngx_init_modules(ngx_cycle_t *cycle)
{
    ngx_uint_t  i;

    for (i = 0; cycle->modules[i]; i++) {
        if (cycle->modules[i]->init_module) {
            if (cycle->modules[i]->init_module(cycle) != NGX_OK) {
                return NGX_ERROR;
            }
        }
    }

    return NGX_OK;
}

调用所有模块的init_module()

main() ->ngx_single_process_cycle()

for (i = 0; cycle->modules[i]; i++) {
    if (cycle->modules[i]->init_process) {
        if (cycle->modules[i]->init_process(cycle) == NGX_ERROR) {
            /* fatal */
            exit(2);
        }
    }
}

调用所有模块的init_process()。由于我使用的单进程模式所以是ngx_single_process_cycle这个函数

总结一下core调用流程:

使用ngx_preinit_modules()函数初始化模块的index和name
使用ngx_cycle_modules()函数完成转移
调用ngx_init_cycle
调用配置命令的set
调用ngx_init_cycle
调用init_module
调用init_process

nginx线程池

线程池模块定义

static ngx_command_t  ngx_thread_pool_commands[] = {

    { ngx_string("thread_pool"),	//配置的名字
      NGX_MAIN_CONF|NGX_DIRECT_CONF|NGX_CONF_TAKE23,	//再main域配置,两个或者三个参数
      ngx_thread_pool,				//配置解析函数
      0,
      0,
      NULL },
      ngx_null_command				//哨兵
};

static ngx_core_module_t  ngx_thread_pool_module_ctx = {
    ngx_string("thread_pool"),
    ngx_thread_pool_create_conf,
    ngx_thread_pool_init_conf
};

ngx_module_t  ngx_thread_pool_module = {
    NGX_MODULE_V1,
    &ngx_thread_pool_module_ctx,           /* module context */
    ngx_thread_pool_commands,              /* module directives */
    NGX_CORE_MODULE,                       /* module type */
    NULL,                                  /* init master */
    NULL,                                  /* init module */
    ngx_thread_pool_init_worker,           /* init process */
    NULL,                                  /* init thread */
    NULL,                                  /* exit thread */
    ngx_thread_pool_exit_worker,           /* exit process */
    NULL,                                  /* exit master */
    NGX_MODULE_V1_PADDING
};

可以看到nginx线程池是一个core模块,那么初始化流程可以看到模块定义得出

ngx_thread_pool_create_conf
ngx_thread_pool
ngx_thread_pool_init_conf
ngx_thread_pool_init_worker

线程池的其他结构

//线程池的ngx_thread_pool_s 指针数组
typedef struct {
    ngx_array_t               pools;
} ngx_thread_pool_conf_t;

//任务
struct ngx_thread_task_s {
    ngx_thread_task_t   *next;		//下个任务
    ngx_uint_t           id;		//任务id
    void                *ctx;		//回调参数
    void               (*handler)(void *data, ngx_log_t *log); //回调函数
    ngx_event_t          event;		//事件
};

//队列
typedef struct {
    ngx_thread_task_t        *first;  	//链表头节点
    ngx_thread_task_t       **last;		//指向最后一个节点
} ngx_thread_pool_queue_t;

#define ngx_thread_pool_queue_init(q)                                         \
    (q)->first = NULL;                                                        \
    (q)->last = &(q)->first

//线程池的配置
struct ngx_thread_pool_s {
    ngx_thread_mutex_t        mtx;				//锁
    ngx_thread_pool_queue_t   queue;			//队列
    ngx_int_t                 waiting;			//当前任务总数量
    ngx_thread_cond_t         cond;				//条件变量

    ngx_log_t                *log;

    ngx_str_t                 name;				//线程的名字
    ngx_uint_t                threads;			//线程数量
    ngx_int_t                 max_queue;		//最大的任务数量

    u_char                   *file;
    ngx_uint_t                line;
};

static ngx_str_t  ngx_thread_pool_default = ngx_string("default"); 	//默认线程池名
static ngx_uint_t               ngx_thread_pool_task_id;			//任务id
static ngx_atomic_t             ngx_thread_pool_done_lock;			//原子变量
static ngx_thread_pool_queue_t  ngx_thread_pool_done;				//完成任务队列

线程池初始化流程

创建和初始化ngx_thread_pool_conf_t

static void *ngx_thread_pool_create_conf(ngx_cycle_t *cycle)
{
    ngx_thread_pool_conf_t  *tcf;
	//创建数组
    tcf = ngx_pcalloc(cycle->pool, sizeof(ngx_thread_pool_conf_t));
    if (tcf == NULL) {
        return NULL;
    }

    if (ngx_array_init(&tcf->pools, cycle->pool, 4,
                       sizeof(ngx_thread_pool_t *))
        != NGX_OK)
    {
        return NULL;
    }

    return tcf;
}

需要注意的是返回值保存在cycle->conf_ctx[ngx_thread_pool_module.index]

在这里插入图片描述

解析配置信息

//配置方法是 thread_pool name threads=nums [max_queue=nums]
static char *ngx_thread_pool(ngx_conf_t *cf, ngx_command_t *cmd, void *conf)
{
    ngx_str_t          *value;
    ngx_uint_t          i;
    ngx_thread_pool_t  *tp;

    value = cf->args->elts;
	//value[1] 是名字,创建线程池结构加入线程池数组
    tp = ngx_thread_pool_add(cf, &value[1]);

    if (tp == NULL) {
        return NGX_CONF_ERROR;
    }

    if (tp->threads) {
        ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
                           "duplicate thread pool \"%V\"", &tp->name);
        return NGX_CONF_ERROR;
    }
	//最大的任务数量
    tp->max_queue = 65536;

    for (i = 2; i < cf->args->nelts; i++) {
		//threads=xxx 把'xxx'转成int
        if (ngx_strncmp(value[i].data, "threads=", 8) == 0) {
			//线程的数量
            tp->threads = ngx_atoi(value[i].data + 8, value[i].len - 8);

            if (tp->threads == (ngx_uint_t) NGX_ERROR || tp->threads == 0) {
                ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
                                   "invalid threads value \"%V\"", &value[i]);
                return NGX_CONF_ERROR;
            }

            continue;
        }

        if (ngx_strncmp(value[i].data, "max_queue=", 10) == 0) {
			//最大的任务数量
            tp->max_queue = ngx_atoi(value[i].data + 10, value[i].len - 10);

            if (tp->max_queue == NGX_ERROR) {
                ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
                                   "invalid max_queue value \"%V\"", &value[i]);
                return NGX_CONF_ERROR;
            }

            continue;
        }
    }

    if (tp->threads == 0) {
        ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
                           "\"%V\" must have \"threads\" parameter",
                           &cmd->name);
        return NGX_CONF_ERROR;
    }

    return NGX_CONF_OK;
}

在这里插入图片描述

初始化默认的线程池的配置结构

static char *
ngx_thread_pool_init_conf(ngx_cycle_t *cycle, void *conf)
{
    ngx_thread_pool_conf_t *tcf = conf;	//create里创建的数组

    ngx_uint_t           i;
    ngx_thread_pool_t  **tpp;	//线程池结构

    tpp = tcf->pools.elts;
	//遍历所有线程池配置数组,配置默认的线程池
    for (i = 0; i < tcf->pools.nelts; i++) {

        if (tpp[i]->threads) {
            continue;
        }
		//默认的线程池
        if (tpp[i]->name.len == ngx_thread_pool_default.len
            && ngx_strncmp(tpp[i]->name.data, ngx_thread_pool_default.data,
                           ngx_thread_pool_default.len)
               == 0)
        {
            tpp[i]->threads = 32;
            tpp[i]->max_queue = 65536;
            continue;
        }

        ngx_log_error(NGX_LOG_EMERG, cycle->log, 0,
                      "unknown thread pool \"%V\" in %s:%ui",
                      &tpp[i]->name, tpp[i]->file, tpp[i]->line);

        return NGX_CONF_ERROR;
    }

    return NGX_CONF_OK;
}

启动线程池

static ngx_int_t ngx_thread_pool_init_worker(ngx_cycle_t *cycle)
{
    ngx_uint_t                i;
    ngx_thread_pool_t       **tpp;
    ngx_thread_pool_conf_t   *tcf;
	//不是单进程或者WORKER进程
    if (ngx_process != NGX_PROCESS_WORKER
        && ngx_process != NGX_PROCESS_SINGLE)
    {
        return NGX_OK;
    }
	//线程池配置数组
    tcf = (ngx_thread_pool_conf_t *) ngx_get_conf(cycle->conf_ctx,
                                                  ngx_thread_pool_module);

    if (tcf == NULL) {
        return NGX_OK;
    }
	//队列初始化
    ngx_thread_pool_queue_init(&ngx_thread_pool_done);

    tpp = tcf->pools.elts;
	//启动所有线程池
    for (i = 0; i < tcf->pools.nelts; i++) {
        if (ngx_thread_pool_init(tpp[i], cycle->log, cycle->pool) != NGX_OK) {
            return NGX_ERROR;
        }
    }

    return NGX_OK;
}

static ngx_int_t ngx_thread_pool_init(ngx_thread_pool_t *tp, ngx_log_t *log, ngx_pool_t *pool)
{
    int             err;
    pthread_t       tid;
    ngx_uint_t      n;
    pthread_attr_t  attr;
	//必须要有事件通知机制
    if (ngx_notify == NULL) {
        ngx_log_error(NGX_LOG_ALERT, log, 0,
               "the configured event method cannot be used with thread pools");
        return NGX_ERROR;
    }
	//线程池队列初始化
    ngx_thread_pool_queue_init(&tp->queue);
	//创建互斥锁
    if (ngx_thread_mutex_create(&tp->mtx, log) != NGX_OK) {
        return NGX_ERROR;
    }
	//创建条件变量
    if (ngx_thread_cond_create(&tp->cond, log) != NGX_OK) {
        (void) ngx_thread_mutex_destroy(&tp->mtx, log);
        return NGX_ERROR;
    }

    tp->log = log;
	//设置线程属性DETACHED 分离
    err = pthread_attr_init(&attr);
    if (err) {
        ngx_log_error(NGX_LOG_ALERT, log, err,
                      "pthread_attr_init() failed");
        return NGX_ERROR;
    }

    err = pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
    if (err) {
        ngx_log_error(NGX_LOG_ALERT, log, err,
                      "pthread_attr_setdetachstate() failed");
        return NGX_ERROR;
    }

    for (n = 0; n < tp->threads; n++) {
		//创建线程	ngx_thread_pool_cycle线程处理函数
        err = pthread_create(&tid, &attr, ngx_thread_pool_cycle, tp);
        if (err) {
            ngx_log_error(NGX_LOG_ALERT, log, err,
                          "pthread_create() failed");
            return NGX_ERROR;
        }
    }
	//销毁属性
    (void) pthread_attr_destroy(&attr);

    return NGX_OK;
}

遍历每个线程池结构，然后根据threads创建线程数量

线程的回调函数

static void *ngx_thread_pool_cycle(void *data)
{
    ngx_thread_pool_t *tp = data;
    int                 err;
    sigset_t            set;
    ngx_thread_task_t  *task;

    ngx_log_debug1(NGX_LOG_DEBUG_CORE, tp->log, 0,
                   "thread in pool \"%V\" started", &tp->name);
	//填充线程信号
    sigfillset(&set);
	//不屏蔽的信号
    sigdelset(&set, SIGILL);
    sigdelset(&set, SIGFPE);
    sigdelset(&set, SIGSEGV);
    sigdelset(&set, SIGBUS);
	//设置信号屏蔽
    err = pthread_sigmask(SIG_BLOCK, &set, NULL);
    if (err) {
        ngx_log_error(NGX_LOG_ALERT, tp->log, err, "pthread_sigmask() failed");
        return NULL;
    }

    for ( ;; ) {
		//加锁
        if (ngx_thread_mutex_lock(&tp->mtx, tp->log) != NGX_OK) {
            return NULL;
        }

        /* the number may become negative */
        tp->waiting--;
		//如果队列没有任务则睡眠等待唤醒
        while (tp->queue.first == NULL) {
            if (ngx_thread_cond_wait(&tp->cond, &tp->mtx, tp->log)
                != NGX_OK)
            {
                (void) ngx_thread_mutex_unlock(&tp->mtx, tp->log);
                return NULL;
            }
        }
		//取队列中的一个任务
        task = tp->queue.first;
        tp->queue.first = task->next;
		//如果取万没有任务
        if (tp->queue.first == NULL) {
            tp->queue.last = &tp->queue.first;
        }
		//解锁
        if (ngx_thread_mutex_unlock(&tp->mtx, tp->log) != NGX_OK) {
            return NULL;
        }

        ngx_log_debug2(NGX_LOG_DEBUG_CORE, tp->log, 0,
                       "run task #%ui in thread pool \"%V\"",
                       task->id, &tp->name);
		//调用任务中的回调函数
        task->handler(task->ctx, tp->log);

        ngx_log_debug2(NGX_LOG_DEBUG_CORE, tp->log, 0,
                       "complete task #%ui in thread pool \"%V\"",
                       task->id, &tp->name);

        task->next = NULL;

        ngx_spinlock(&ngx_thread_pool_done_lock, 1, 2048);
		//加入完成完成队列队列
        *ngx_thread_pool_done.last = task;
        ngx_thread_pool_done.last = &task->next;

        ngx_memory_barrier();

        ngx_unlock(&ngx_thread_pool_done_lock);
		//通知完成任务回调
        (void) ngx_notify(ngx_thread_pool_handler);
    }
}

完成任务回调

static void ngx_thread_pool_handler(ngx_event_t *ev)
{
    ngx_event_t        *event;
    ngx_thread_task_t  *task;

    ngx_log_debug0(NGX_LOG_DEBUG_CORE, ev->log, 0, "thread pool handler");

    ngx_spinlock(&ngx_thread_pool_done_lock, 1, 2048);
	//得到链表头节点
    task = ngx_thread_pool_done.first;
    ngx_thread_pool_done.first = NULL;
    ngx_thread_pool_done.last = &ngx_thread_pool_done.first;

    ngx_memory_barrier();

    ngx_unlock(&ngx_thread_pool_done_lock);
	//遍历链表执行handler
    while (task) {
        ngx_log_debug1(NGX_LOG_DEBUG_CORE, ev->log, 0,
                       "run completion handler for task #%ui", task->id);

        event = &task->event;
        task = task->next;

        event->complete = 1;
        event->active = 0;

        event->handler(event);
    }
}

投递任务

ngx_int_t
ngx_thread_task_post(ngx_thread_pool_t *tp, ngx_thread_task_t *task)
{
	//任务在已经投递
    if (task->event.active) {
        ngx_log_error(NGX_LOG_ALERT, tp->log, 0,
                      "task #%ui already active", task->id);
        return NGX_ERROR;
    }

    if (ngx_thread_mutex_lock(&tp->mtx, tp->log) != NGX_OK) {
        return NGX_ERROR;
    }
	//任务达到上限
    if (tp->waiting >= tp->max_queue) {
        (void) ngx_thread_mutex_unlock(&tp->mtx, tp->log);

        ngx_log_error(NGX_LOG_ERR, tp->log, 0,
                      "thread pool \"%V\" queue overflow: %i tasks waiting",
                      &tp->name, tp->waiting);
        return NGX_ERROR;
    }
	//设置投递标志
    task->event.active = 1;
    task->id = ngx_thread_pool_task_id++;
    task->next = NULL;
	//唤醒线程
    if (ngx_thread_cond_signal(&tp->cond, tp->log) != NGX_OK) {
        (void) ngx_thread_mutex_unlock(&tp->mtx, tp->log);
        return NGX_ERROR;
    }
	//加入队列
    *tp->queue.last = task;
    tp->queue.last = &task->next;
    tp->waiting++;

    (void) ngx_thread_mutex_unlock(&tp->mtx, tp->log);

    ngx_log_debug2(NGX_LOG_DEBUG_CORE, tp->log, 0,
                   "task #%ui added to thread pool \"%V\"",
                   task->id, &tp->name);

    return NGX_OK;
}