civetweb作为radosgw的agent的处理机制

1、radosgw在启动的时候会启动civeweb，创建一个主线程后，在创建若干个子线程。主线程负责监听连接请求，分发任务。子线程负责干活

主线程跑的函数是如下

static void master_thread_run(void *thread_func_param)

这个函数主要是监听连接请求，采用的时poll机制，如果来了新请求，调用蓝色函数

                if (ctx->stop_flag == 0 && (pfd[i].revents & POLLIN)) {

                    accept_new_connection(&ctx->listening_sockets[i], ctx);

                }

accept_new_connection调用了produce_socket(ctx, &so)，生成socket消息，给线程池发信号，获取到信号的线程处理该消息

   /* If the queue is full, wait */

    while (ctx->stop_flag == 0 &&

           ctx->sq_head - ctx->sq_tail >= (int) ARRAY_SIZE(ctx->queue)) {

        (void) pthread_cond_wait(&ctx->sq_empty, &ctx->thread_mutex);

    }

 

    if (ctx->sq_head - ctx->sq_tail < (int) ARRAY_SIZE(ctx->queue)) {

        /* Copy socket to the queue and increment head */

        ctx->queue[ctx->sq_head % ARRAY_SIZE(ctx->queue)] = *sp;

        ctx->sq_head++;

        DEBUG_TRACE("queued socket %d", sp->sock);

    }

 

    (void) pthread_cond_signal(&ctx->sq_full);

    (void) pthread_mutex_unlock(&ctx->thread_mutex);

线程池中的每个线程都跑的如下函数

static void *worker_thread_run(void *thread_func_param)

......

 

        while (consume_socket(ctx, &conn->client)) {--------------------------这里面是获取锁

            conn->birth_time = time(NULL);

 

            /* Fill in IP, port info early so even if SSL setup below fails,

               error handler would have the corresponding info.

               Thanks to Johannes Winkelmann for the patch.

               TODO(lsm): Fix IPv6 case */

            conn->request_info.remote_port = ntohs(conn->client.rsa.sin.sin_port);

            memcpy(&conn->request_info.remote_ip,

                   &conn->client.rsa.sin.sin_addr.s_addr, 4);

            conn->request_info.remote_ip = ntohl(conn->request_info.remote_ip);

            conn->request_info.is_ssl = conn->client.is_ssl;

 

            if (!conn->client.is_ssl

#ifndef NO_SSL

                || sslize(conn, conn->ctx->ssl_ctx, SSL_accept)

#endif

               ) {

                process_new_connection(conn);--------------------处理消息

            }

 

            close_connection(conn);

       

static void process_new_connection(struct mg_connection *conn)

......

这里面先解析消息，在回调函数处理消息，在调回调函数做后续处理

if (ebuf[0] == '\0') {

            handle_request(conn);-------------处理消息

            if (conn->ctx->callbacks.end_request != NULL) {

                conn->ctx->callbacks.end_request(conn, conn->status_code);

            }

            log_access(conn);

        }

 

static void handle_request(struct mg_connection *conn)

......

    if (!conn->client.is_ssl && conn->client.ssl_redir &&

        (ssl_index = get_first_ssl_listener_index(conn->ctx)) > -1) {

        redirect_to_https_port(conn, ssl_index);

    } else if (!is_script_resource && !is_put_or_delete_request(conn) &&

               !check_authorization(conn, path)) {

        send_authorization_request(conn);

    } else if (conn->ctx->callbacks.begin_request != NULL &&

               conn->ctx->callbacks.begin_request(conn)) {-----------处理消息的函数

        /* Do nothing, callback has served the request */

    fast_forward_request(conn);

#if defined(USE_WEBSOCKET)

    }

......

 

//下面是处理消息的回调函数，获取消息体，调用红色部分处理消息，process_request里面获取对应的handler，获取对应的op，在调用相应op对应类的方法

static int civetweb_callback(struct mg_connection* conn) {

  struct mg_request_info* req_info = mg_get_request_info(conn);

  RGWMongooseEnv* pe = static_cast<RGWMongooseEnv *>(req_info->user_data);

  RGWRados* store = pe->store;

  RGWREST* rest = pe->rest;

  OpsLogSocket* olog = pe->olog;

 

  RGWRequest req(store->get_new_req_id());

  RGWMongoose client_io(conn, pe->port);

 

  {

    // hold a read lock over access to pe->store for reconfiguration

    RWLock::RLocker lock(pe->mutex);

 

    int ret = process_request(pe->store, rest, &req, &client_io, olog);

    if (ret < 0) {

      /* we don't really care about return code */

      dout(20) << "process_request() returned " << ret << dendl;

    }

  }

 

// Mark as processed

  return 1;

 

int process_request(RGWRados* store, RGWREST* rest, RGWRequest* req,

            RGWStreamIO* client_io, OpsLogSocket* olog)

{

......

  req->log(s, "pre-executing");

  op->pre_exec();

  req->log(s, "executing");

  op->execute();

  req->log(s, "completing");

  op->complete();

 

......

}