(3) s3cmd get object 代码流程

ceph version 12.2.5 (cad919881333ac92274171586c827e01f554a70a) luminous (stable)
s3cmd get 时的代码流程

get 代码调用流程

civetweb.c worker_thread:

static void *

worker_thread(void *thread_func_param)

{

 worker_thread_run(thread_func_param);

 return NULL;

}

civetweb.c worker_thread_run:

static void *

worker_thread_run(void *thread_func_param)

{

 mg_set_thread_name("worker");



  while (consume_socket(ctx, &conn->client)) {

    process_new_connection(conn);

   close_connection(conn);

  }



 ctx->running_worker_threads--;

 return NULL;

}

civetweb.c process_new_connection:

static void

process_new_connection(struct mg_connection *conn)

{

   keep_alive_enabled =

      !strcmp(conn->ctx->config[ENABLE_KEEP_ALIVE], "yes");



  do {



   if (ebuf[0] == '\0') {

     handle_request(conn);

    } 

   keep_alive = conn->ctx->stop_flag == 0 && keep_alive_enabled

                && conn->content_len >= 0 && should_keep_alive(conn);



   if ((conn->data_len < 0) || (conn->data_len > conn->buf_size)) {

    break;

   }

  } while (keep_alive);



}

civetweb.c handle_request:

/* This is the heart of the Civetweb's logic.

 * This function is called when the request is read, parsed and validated,

 * and Civetweb must decide what action to take: serve a file, or

 * a directory, or call embedded function, etcetera. */

static void

handle_request(struct mg_connection *conn)

{

  /* 1. get the request url */

  /* 1.1. split into url and query string */

  /* 1.2. decode url (if config says so) */

  /* 1.3. clean URIs, so a path like allowed_dir/../forbidden_file is

   * not possible (if config says so) */



  /* 2. do a https redirect, if required */



  /* 3. if this ip has limited speed, set it for this connection */



/* 4. call a "handle everything" callback, if registered */

  if (conn->ctx->callbacks.begin_request != NULL) {

     i = conn->ctx->callbacks.begin_request(conn);

   }



  /* 5. interpret the url to find out how the request must be handled

   */

}

rgw_civetweb_frontend.cc RGWCivetWebFrontend::process:

int RGWCivetWebFrontend::process(struct mg_connection* const conn)

{

 

  RGWRequest req(env.store->get_new_req_id());

  int http_ret = 0;

  int ret = process_request(env.store, env.rest, &req, env.uri_prefix,

                            *env.auth_registry, &client_io, env.olog, &http_ret);



  return http_ret;

}

rgw_process.cc process_request:

int process_request(RGWRados* const store,

                    RGWREST* const rest,

                    RGWRequest* const req,

                    const std::string& frontend_prefix,

                    const rgw_auth_registry_t& auth_registry,

                    RGWRestfulIO* const client_io,

                    OpsLogSocket* const olog,

                    int* http_ret)

{

RGWHandler_REST *handler = rest->get_handler(store, s,

                                               auth_registry,

                                               frontend_prefix,

                                               client_io, &mgr, &init_error);

  op = handler->get_op(store);

  ret = rgw_process_authenticated(handler, op, req, s);



}

rgw_process.cc rgw_process_authenticated:

int rgw_process_authenticated(RGWHandler_REST * const handler,

                              RGWOp *& op,

                              RGWRequest * const req,

                              req_state * const s,

                              const bool skip_retarget)

{

  req->log(s, "init permissions");

  int ret = handler->init_permissions(op);



 

  /* If necessary extract object ACL and put them into req_state. */

  req->log(s, "reading permissions");

  ret = handler->read_permissions(op);

 

  req->log(s, "init op");

  ret = op->init_processing();

 

  req->log(s, "verifying op mask");

  ret = op->verify_op_mask();

  

 req->log(s, "verifying op permissions");

  ret = op->verify_permission();

  

  req->log(s, "verifying op params");

  ret = op->verify_params();

 

  req->log(s, "pre-executing");

  op->pre_exec();



  req->log(s, "executing");

  op->execute();



  req->log(s, "completing");

  op->complete();



}

rgw_op.cc RGWGetObj::execute:

void RGWGetObj::execute()

{

      op_ret = read_op.iterate(ofs_x, end_x, filter);

}

rgw_rados.cc RGWRados::Object::Read::iterate:

int RGWRados::Object::Read::iterate(int64_t ofs, int64_t end, RGWGetDataCB *cb)

{

      int r = store->iterate_obj(obj_ctx, source->get_bucket_info(), state.obj, ofs, end, cct->_conf->rgw_get_obj_max_req_size, _get_obj_iterate_cb, (void *)data);

}

rgw_rados.cc RGWRados::iterate_obj:

int RGWRados::iterate_obj(RGWObjectCtx& obj_ctx,

                          const RGWBucketInfo& bucket_info, const rgw_obj& obj,

                          off_t ofs, off_t end,

     uint64_t max_chunk_size,

     int (*iterate_obj_cb)(const RGWBucketInfo&, const rgw_obj& obj,

                                                const rgw_raw_obj&, off_t, off_t, off_t, bool,

                                                RGWObjState *, void *),

                   void *arg)

{

  /* now get the relevant object stripe */

    RGWObjManifest::obj_iterator iter = astate->manifest.obj_find(ofs);



    RGWObjManifest::obj_iterator obj_end = astate->manifest.obj_end();



    for (; iter != obj_end && ofs <= end; ++iter) {

 

      while (ofs < next_stripe_ofs && ofs <= end) {



        r = iterate_obj_cb(bucket_info, obj, read_obj, ofs, read_ofs, read_len, reading_from_head, astate, arg);

 

        }

      }

    }

}

rgw_rados.cc _get_obj_iterate_cb:

static int _get_obj_iterate_cb(const RGWBucketInfo& bucket_info, const rgw_obj& obj, const rgw_raw_obj& read_obj, off_t obj_ofs, off_t read_ofs, off_t len, bool is_head_obj, RGWObjState *astate, void *arg)

{

  struct get_obj_data *d = (struct get_obj_data *)arg;



  return d->rados->get_obj_iterate_cb(d->ctx, astate, bucket_info, obj, read_obj, obj_ofs, read_ofs, len, is_head_obj, arg);

}

rgw_rados.cc RGWRados::get_obj_iterate_cb:

int RGWRados::get_obj_iterate_cb(RGWObjectCtx *ctx, RGWObjState *astate,

                         const RGWBucketInfo& bucket_info,

                         const rgw_obj& obj,

           const rgw_raw_obj& read_obj,

    off_t obj_ofs,

                         off_t read_ofs, off_t len,

                         bool is_head_obj, void *arg)

{

  // Flush data to client if there is any

  r = flush_read_list(d);



}

rgw_rados.cc RGWRados::flush_read_list:

int RGWRados::flush_read_list(struct get_obj_data *d)

{

  d->data_lock.Lock();

  list<bufferlist> l;

  l.swap(d->read_list);

  d->get();

  d->read_list.clear();



  d->data_lock.Unlock();



  int r = 0;



  list<bufferlist>::iterator iter;

  for (iter = l.begin(); iter != l.end(); ++iter) {

    bufferlist& bl = *iter;

    r = d->client_cb->handle_data(bl, 0, bl.length());

  }

}

rgw_op.cc RGWGetObj::get_data_cb:

int RGWGetObj::get_data_cb(bufferlist& bl, off_t bl_ofs, off_t bl_len)

{

  return send_response_data(bl, bl_ofs, bl_len);

}

rgw_rest_s3.cc RGWGetObj_ObjStore_S3::send_response_data

int RGWGetObj_ObjStore_S3::send_response_data(bufferlist& bl, off_t bl_ofs,

           off_t bl_len)

{

    int r = dump_body(s, bl.c_str() + bl_ofs, bl_len);

}

rgw_rest.cc dump_body:

int dump_body(struct req_state* const s,

              const char* const buf,

              const size_t len)

{

     return RESTFUL_IO(s)->send_body(buf, len);

}

rgw_client_io_filters.h rgw::io::AccountingFilterrgw::io::RestfulClient*::send_body

size_t send_body(const char* const buf,

                   const size_t len) override {

    const auto sent = DecoratedRestfulClient<T>::send_body(buf, len);

  }

rgw_client_io.h DecoratedRestfulClient:send_body

  size_t send_body(const char* const buf,

                   const size_t len) override {

    return get_decoratee().send_body(buf, len);

  }

rgw_client_io_filters.h rgw::io::BufferingFilter<rgw::io::ChunkingFilter<rgw::io::ConLenControllingFilter<RGWCivetWeb*> > >::send_body

template <typename T>

size_t BufferingFilter<T>::send_body(const char* const buf,

                                     const size_t len)

{

  return DecoratedRestfulClient<T>::send_body(buf, len);

}

rgw_client_io_filters.h rgw::io::ChunkingFilter<rgw::io::ConLenControllingFilter<RGWCivetWeb*> >::send_body

  size_t send_body(const char* buf,

                   const size_t len) override {

    if (! chunking_enabled) {

      return DecoratedRestfulClient<T>::send_body(buf, len);

    } else {

      static constexpr char HEADER_END[] = "\r\n";

      char sizebuf[32];

      const auto slen = snprintf(sizebuf, sizeof(buf), "%" PRIx64 "\r\n", len);

      size_t sent = 0;



      sent += DecoratedRestfulClient<T>::send_body(sizebuf, slen);

      sent += DecoratedRestfulClient<T>::send_body(buf, len);

      sent += DecoratedRestfulClient<T>::send_body(HEADER_END,

                                                   sizeof(HEADER_END) - 1);

      return sent;

    }

  }

rgw_client_io.h DecoratedRestfulClient:send_body

size_t send_body(const char* const buf,

                   const size_t len) override {

    return get_decoratee().send_body(buf, len);

  }

rgw_civetweb.cc RGWCivetWeb::write_data

size_t RGWCivetWeb::write_data(const char *buf, const size_t len)

{

  while (to_sent) {

    const int ret = mg_write(conn, buf, len);

  }

}

civetweb.c mg_write

int

mg_write(struct mg_connection *conn, const void *buf, size_t len)

{

 push_all(conn->ctx,

                        NULL,

                        conn->client.sock,

                        conn->ssl,

                        (const char *)buf,

                        (int64_t)allowed)) == allowed) 	

}

civetweb.c push_all

static int64_t

push_all(struct mg_context *ctx,

         FILE *fp,

         SOCKET sock,

         SSL *ssl,

         const char *buf,

         int64_t len)

{

 while (len > 0 && ctx->stop_flag == 0) {

  n = push(ctx, fp, sock, ssl, buf + nwritten, (int)len, timeout);

  }

}

civetweb.c push

/* Write data to the IO channel - opened file descriptor, socket or SSL

 * descriptor. Return number of bytes written. */

static int

push(struct mg_context *ctx,

     FILE *fp,

     SOCKET sock,

     SSL *ssl,

     const char *buf,

     int len,

     double timeout)

{

   n = (int)send(sock, buf, (len_t)len, MSG_NOSIGNAL);

}

调用堆栈

#0 mg_write (conn=0x555556d0a000, buf=0x55555b9e2000, len=4194304) at /root/wuyan/ceph/src/civetweb/src/civetweb.c:4260

#1 0x0000555555742bcf in RGWCivetWeb::write_data (this=0x7fff948bd400, buf=0x55555b9e2000 "data_30G", len=4194304) at /root/wuyan/ceph/src/rgw/rgw_civetweb.cc:19

#2 0x00005555557057f3 in send_body (len=4194304, buf=0x55555b9e2000 "data_30G", this=0x7fff948bd230) at /root/wuyan/ceph/src/rgw/rgw_client_io.h:241

#3 rgw::io::ChunkingFilter<rgw::io::ConLenControllingFilter<RGWCivetWeb*> >::send_body (this=0x7fff948bd230, buf=0x55555b9e2000 "data_30G", len=4194304)

    at /root/wuyan/ceph/src/rgw/rgw_client_io_filters.h:286

#4 0x0000555555705c2b in send_body (len=4194304, buf=<optimized out>, this=0x7fff948bd228) at /root/wuyan/ceph/src/rgw/rgw_client_io.h:241

#5 rgw::io::BufferingFilter<rgw::io::ChunkingFilter<rgw::io::ConLenControllingFilter<RGWCivetWeb*> > >::send_body (this=0x7fff948bd228, buf=<optimized out>, len=4194304)

    at /root/wuyan/ceph/src/rgw/rgw_client_io_filters.h:191

#6 0x0000555555703231 in send_body (len=<optimized out>, buf=<optimized out>, this=<optimized out>) at /root/wuyan/ceph/src/rgw/rgw_client_io.h:241

#7 rgw::io::AccountingFilter<rgw::io::RestfulClient*>::send_body (this=0x7fff948bd140, buf=<optimized out>, len=<optimized out>) at /root/wuyan/ceph/src/rgw/rgw_client_io_filters.h:115

#8 0x00005555558d0413 in dump_body (s=<optimized out>, buf=0x55555b9e2000 "data_30G", len=len@entry=4194304) at /root/wuyan/ceph/src/rgw/rgw_rest.cc:844

#9 0x000055555591c000 in RGWGetObj_ObjStore_S3::send_response_data (this=<optimized out>, bl=..., bl_ofs=0, bl_len=4194304) at /root/wuyan/ceph/src/rgw/rgw_rest_s3.cc:336

#10 0x00005555557f4a3b in RGWGetObj::get_data_cb (this=0x555556d1f400, bl=..., bl_ofs=0, bl_len=4194304) at /root/wuyan/ceph/src/rgw/rgw_op.cc:1580

#11 0x0000555555849796 in RGWRados::flush_read_list (this=this@entry=0x5555567e8000, d=d@entry=0x5555568d6780) at /root/wuyan/ceph/src/rgw/rgw_rados.cc:10653

#12 0x000055555588d019 in RGWRados::get_obj_iterate_cb (this=0x5555567e8000, ctx=<optimized out>, astate=astate@entry=0x5555567f63a0, bucket_info=..., obj=..., read_obj=...,

    obj_ofs=obj_ofs@entry=16777216, read_ofs=read_ofs@entry=0, len=len@entry=4194304, is_head_obj=is_head_obj@entry=false, arg=arg@entry=0x5555568d6780)

    at /root/wuyan/ceph/src/rgw/rgw_rados.cc:10737

#13 0x000055555588d476 in _get_obj_iterate_cb (bucket_info=..., obj=..., read_obj=..., obj_ofs=obj_ofs@entry=16777216, read_ofs=read_ofs@entry=0, len=len@entry=4194304, is_head_obj=false,

    astate=0x5555567f63a0, arg=arg@entry=0x5555568d6780) at /root/wuyan/ceph/src/rgw/rgw_rados.cc:10585

#14 0x00005555558905e6 in RGWRados::iterate_obj (this=this@entry=0x5555567e8000, obj_ctx=..., bucket_info=..., obj=..., ofs=16777216, ofs@entry=0, end=end@entry=32212264959,

    max_chunk_size=4194304,

    iterate_obj_cb=iterate_obj_cb@entry=0x55555588d430 <_get_obj_iterate_cb(RGWBucketInfo const&, rgw_obj const&, rgw_raw_obj const&, off_t, off_t, off_t, bool, RGWObjState*, void*)>,

    arg=arg@entry=0x5555568d6780) at /root/wuyan/ceph/src/rgw/rgw_rados.cc:10839

#15 0x0000555555890a5c in RGWRados::Object::Read::iterate (this=this@entry=0x7fff948bbf00, ofs=0, end=32212264959, cb=cb@entry=0x7fff948bbdd0) at /root/wuyan/ceph/src/rgw/rgw_rados.cc:10765

#16 0x000055555581c970 in RGWGetObj::execute (this=0x555556d1f400) at /root/wuyan/ceph/src/rgw/rgw_op.cc:1792

#17 0x0000555555835502 in rgw_process_authenticated (handler=handler@entry=0x555557ebba40, op=@0x7fff948bc638: 0x555556d1f400, req=req@entry=0x7fff948bd110, s=s@entry=0x7fff948bca20,

    skip_retarget=skip_retarget@entry=false) at /root/wuyan/ceph/src/rgw/rgw_process.cc:104

#18 0x00005555558362c8 in process_request (store=0x5555567e8000, rest=0x7fffffffe630, req=req@entry=0x7fff948bd110, frontend_prefix="", auth_registry=...,

    client_io=client_io@entry=0x7fff948bd140, olog=0x0, http_ret=http_ret@entry=0x7fff948bd10c) at /root/wuyan/ceph/src/rgw/rgw_process.cc:207

#19 0x0000555555703cb2 in RGWCivetWebFrontend::process (this=0x5555566074a0, conn=<optimized out>) at /root/wuyan/ceph/src/rgw/rgw_civetweb_frontend.cc:36

#20 0x000055555573e82f in handle_request (conn=conn@entry=0x555556d0a000) at /root/wuyan/ceph/src/civetweb/src/civetweb.c:9890

#21 0x00005555557401bb in process_new_connection (conn=<optimized out>) at /root/wuyan/ceph/src/civetweb/src/civetweb.c:12328

#22 worker_thread_run (thread_func_param=0x555556b06000) at /root/wuyan/ceph/src/civetweb/src/civetweb.c:12505

#23 worker_thread (thread_func_param=0x555556b06000) at /root/wuyan/ceph/src/civetweb/src/civetweb.c:12542

#24 0x00007ffff6b82e25 in start_thread () from /lib64/libpthread.so.0

#25 0x00007fffeb6fdbad in clone () from /lib64/libc.so.6