nginx upstream模块详解（处理流程篇一 upstream处理）

upstream 是nginx作为代理及缓存的核心结构 并且请求上游  发送至下游都能由相关联的模块进行干预处理

upstream 模块流程处理如下

创建upstream

ngx_int_t
ngx_http_upstream_create(ngx_http_request_t *r)
{
    ngx_http_upstream_t  *u;

    u = r->upstream;

    if (u && u->cleanup) {    //upstream变量存在 并且配置有cleanup 进行清除
        r->main->count++;
        ngx_http_upstream_cleanup(r);
    }
    ...  //创建upstream 必要初始化
    return NGX_OK;
}

upstream初始化

void
ngx_http_upstream_init(ngx_http_request_t *r)
{
    ngx_connection_t     *c;

    c = r->connection;

    ngx_log_debug1(NGX_LOG_DEBUG_HTTP, c->log, 0,
                   "http init upstream, client timer: %d", c->read->timer_set);

#if (NGX_HTTP_V2)   //http 2.0初始化处理
    if (r->stream) { 
        ngx_http_upstream_init_request(r);
        return;
    }
#endif

    if (c->read->timer_set) {      //删除超时定时器
        ngx_del_timer(c->read);
    }
    //upstream 初始化
    ngx_http_upstream_init_request(r);
}

ngx_http_upstream_init_request具体实现

static void
ngx_http_upstream_init_request(ngx_http_request_t *r)
{
    //处于aio状态 则直接返回
    if (r->aio) {
        return;
    }

    u = r->upstream;

#if (NGX_HTTP_CACHE)

    if (u->conf->cache) {
        ngx_int_t  rc;

        rc = ngx_http_upstream_cache(r, u);

        if (rc == NGX_BUSY) {      //来不及处理 返回等待事件触发 NGX_BUSY用于并发请求miss 在获取数据之前 除第一个请求以外的其他请求均被阻塞
            return;
        }

        r->write_event_handler = ngx_http_request_empty_handler;  //写事件触发不做处理

        if (rc == NGX_ERROR) {  //缓存信息获取出错 直接将错误发送给请求端
            ngx_http_finalize_request(r, NGX_HTTP_INTERNAL_SERVER_ERROR);
            return;
        }

        if (rc == NGX_OK) {       //缓存命中  直接发送
            rc = ngx_http_upstream_cache_send(r, u);

            if (rc == NGX_DONE) {  //头部处理出错
                return;
            }

            if (rc == NGX_HTTP_UPSTREAM_INVALID_HEADER) {  //无效头部判定 会回源
                rc = NGX_DECLINED;
                r->cached = 0;
                u->buffer.start = NULL;
                u->cache_status = NGX_HTTP_CACHE_MISS;
                u->request_sent = 1;
            }

            if (ngx_http_upstream_cache_background_update(r, u) != NGX_OK) {
                rc = NGX_ERROR;
            }
        }
    }

#endif

    u->store = u->conf->store;

    if (!u->store && !r->post_action && !u->conf->ignore_client_abort) { //开启上下游连接检测  
        r->read_event_handler = ngx_http_upstream_rd_check_broken_connection;
        r->write_event_handler = ngx_http_upstream_wr_check_broken_connection;
    }

    if (r->request_body) {      //存在http请求包体 包体信息存入request_bufs中
        u->request_bufs = r->request_body->bufs;
    }

    if (u->create_request(r) != NGX_OK) {       //创建请求干预过程出错 直接响应服务器内部错误到请求端
        ngx_http_finalize_request(r, NGX_HTTP_INTERNAL_SERVER_ERROR);
        return;
    }

    if (ngx_http_upstream_set_local(r, u, u->conf->local) != NGX_OK) {//本地upstream地址配置
        ngx_http_finalize_request(r, NGX_HTTP_INTERNAL_SERVER_ERROR);
        return;
    }

    clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module);

    u->output.alignment = clcf->directio_alignment;
    u->output.pool = r->pool;
    u->output.bufs.num = 1;
    u->output.bufs.size = clcf->client_body_buffer_size;

    if (u->output.output_filter == NULL) {
        u->output.output_filter = ngx_chain_writer;
        u->output.filter_ctx = &u->writer;
    }

    u->writer.pool = r->pool;
    //upstream状态信息初始化
    if (r->upstream_states == NULL) {
          ...
    } else {

       ...
    }

   ... //生命周期结束清理处理设置

    if (u->resolved == NULL) {  //还未开始解析

        uscf = u->conf->upstream;

    } else {

#if (NGX_HTTP_SSL)
        u->ssl_name = u->resolved->host;
#endif

        host = &u->resolved->host;

        umcf = ngx_http_get_module_main_conf(r, ngx_http_upstream_module);

        uscfp = umcf->upstreams.elts;

        for (i = 0; i < umcf->upstreams.nelts; i++) {  //从配置的upstream地址信息中搜索（优先处理）

            uscf = uscfp[i];

            if (uscf->host.len == host->len
                && ((uscf->port == 0 && u->resolved->no_port)
                     || uscf->port == u->resolved->port)
                && ngx_strncasecmp(uscf->host.data, host->data, host->len) == 0)
            {
                goto found;
            }
        }

        if (u->resolved->sockaddr) {
                   ...  //已经解析地址信息检查
            if (ngx_http_upstream_create_round_robin_peer(r, u->resolved) //检查通过 创建轮询peer(根据返回的主机数量) 具有负载效果
                != NGX_OK)
            {
                ngx_http_upstream_finalize_request(r, u,
                                               NGX_HTTP_INTERNAL_SERVER_ERROR);
                return;
            }

            ngx_http_upstream_connect(r, u);     //连接至上游服务器

            return;
        }

        if (u->resolved->port == 0) {  //无效port返回服务器错误
            ...
            return;
        }

        temp.name = *host;

        ctx = ngx_resolve_start(clcf->resolver, &temp);  //开始域名解析  域名解析器由core_module配置 未配置对于非IP host处理会失败
        if (ctx == NULL) {
            ngx_http_upstream_finalize_request(r, u,
                                               NGX_HTTP_INTERNAL_SERVER_ERROR);
            return;
        }

        if (ctx == NGX_NO_RESOLVER) {  //未配置域名解析器  响应错误
            ngx_log_error(NGX_LOG_ERR, r->connection->log, 0,
                          "no resolver defined to resolve %V", host);

            ngx_http_upstream_finalize_request(r, u, NGX_HTTP_BAD_GATEWAY);
            return;
        }

       ... //upstream域名解析器信息设置  解析完成会在ngx_http_upstream_resolve_handler中触发

        if (ngx_resolve_name(ctx) != NGX_OK) {   //域名解析出错 响应错误请求（注意 域名解析过程是异步的）
            u->resolved->ctx = NULL;
            ngx_http_upstream_finalize_request(r, u,
                                               NGX_HTTP_INTERNAL_SERVER_ERROR);
            return;
        }

        return;
    }

found:

    if (uscf == NULL) {
        ngx_log_error(NGX_LOG_ALERT, r->connection->log, 0,
                      "no upstream configuration");
        ngx_http_upstream_finalize_request(r, u,
                                           NGX_HTTP_INTERNAL_SERVER_ERROR);
        return;
    }

    u->upstream = uscf;

#if (NGX_HTTP_SSL)
    u->ssl_name = uscf->host;
#endif

    if (uscf->peer.init(r, uscf) != NGX_OK) {     //连接信息初始化失败 响应服务器错误
        ngx_http_upstream_finalize_request(r, u,
                                           NGX_HTTP_INTERNAL_SERVER_ERROR);
        return;
    }

    u->peer.start_time = ngx_current_msec;       //设置连接时间

    if (u->conf->next_upstream_tries
        && u->peer.tries > u->conf->next_upstream_tries)
    {
        u->peer.tries = u->conf->next_upstream_tries; //重试次数
    }

    ngx_http_upstream_connect(r, u);     //连接到上游服务器
}

上游服务器连接ngx_http_upstream_connect处理

static void
ngx_http_upstream_connect(ngx_http_request_t *r, ngx_http_upstream_t *u)
{
    ngx_int_t          rc;
    ngx_connection_t  *c;

    r->connection->log->action = "connecting to upstream";

    if (u->state && u->state->response_time) {
        u->state->response_time = ngx_current_msec - u->state->response_time; //更新uptream state响应时间
    }
    ...  //upstream state信息初始化
    rc = ngx_event_connect_peer(&u->peer);         //发起到上游服务器请求

    ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
                   "http upstream connect: %i", rc);

    if (rc == NGX_ERROR) {         //连接错误 将错误响应到请求端
        ngx_http_upstream_finalize_request(r, u,
                                           NGX_HTTP_INTERNAL_SERVER_ERROR);
        return;
    }

    u->state->peer = u->peer.name;

    if (rc == NGX_BUSY) {              //来不及处理 尝试连接到另一个server （如果有）
        ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, "no live upstreams");
        ngx_http_upstream_next(r, u, NGX_HTTP_UPSTREAM_FT_NOLIVE);
        return;
    }

    if (rc == NGX_DECLINED) {           //连接被拒绝 尝试连接到另一个server (如果有)
        ngx_http_upstream_next(r, u, NGX_HTTP_UPSTREAM_FT_ERROR);
        return;
    }

    /* rc == NGX_OK || rc == NGX_AGAIN || rc == NGX_DONE */

    c = u->peer.connection;

    c->data = r;
    //读写事件处理设置
    c->write->handler = ngx_http_upstream_handler;       
    c->read->handler = ngx_http_upstream_handler;

    u->write_event_handler = ngx_http_upstream_send_request_handler;
    u->read_event_handler = ngx_http_upstream_process_header;

    c->sendfile &= r->connection->sendfile;
    u->output.sendfile = c->sendfile;

    if (c->pool == NULL) {

        /* we need separate pool here to be able to cache SSL connections */

        c->pool = ngx_create_pool(128, r->connection->log);
        ...  //创建连接内存池
    }

    c->log = r->connection->log;
    c->pool->log = c->log;
    c->read->log = c->log;
    c->write->log = c->log;

    /* init or reinit the ngx_output_chain() and ngx_chain_writer() contexts */

    u->writer.out = NULL;
    u->writer.last = &u->writer.out;
    u->writer.connection = c;
    u->writer.limit = 0;

    if (u->request_sent) {
        if (ngx_http_upstream_reinit(r, u) != NGX_OK) {   //重新初始化upstream 
            ngx_http_upstream_finalize_request(r, u,
                                               NGX_HTTP_INTERNAL_SERVER_ERROR);
            return;
        }
    }

    if (r->request_body
        && r->request_body->buf
        && r->request_body->temp_file
        && r == r->main)
    {   
        /*
         * the r->request_body->buf can be reused for one request only,
         * the subrequests should allocate their own temporary bufs
         */
        //请求包体buf循环利用 挂载到upstream 输出空闲buf中（对于将请求包体放入临时文件的请求）
        u->output.free = ngx_