rgw && message

原链接

http://blog.csdn.net/litianze99/article/details/51489550

librados::IoCtxImpl::operate_read()                    //do read operation
    Objecter：md_config_obs_t：Dispatcher
        prepare_read_op()                              //new a Op instance is used by Objecter
        op_submit()                                    //submit operation
            _op_submit_with_budget()
                _op_submit()
                    _cal_target()                       //1.cacluate th up/active OSD sets 2.select primary osd as the target
                    _get_session()                      //1.Objecter keeps an OSDSession with each osd 2.new MOSDOp which will be used by OSD
                    _prepare_osd_osd_op()
                    _session_op_assign()                
                    _send_op()
                        op->session-con->send_message()
                            SimpleMessager->send_message()
                                _send_message()
                                    submit_message()
                                        Pip->_send(m)
                                            Pip->cond.signal()
                                                |
                                                |
                                            Pipe::write()
                                                _get_next_outgoing()
                                                write_message()
                                                    do_sendmsg()
                                                        sendmsg()


OSDSession *s = new OSDSession(cct, osd);
  osd_sessions[osd] = s;
  s->con = messenger->get_connection(osdmap->get_inst(osd));
  logger->inc(l_osdc_osd_session_open);
  logger->inc(l_osdc_osd_sessions, osd_sessions.size());
  s->get();
  *session = s;

using SimpleMessager
SimpleMessenger::get_connection()
    return local_connection  //if my_instd.addr equal dest.addr
    _lookup_pipe(dest.addr)  //if don't found exsit pipe we will new a pipe instance


 // create pipe
  Pipe *pipe = new Pipe(this, Pipe::STATE_CONNECTING,
            static_cast<PipeConnection*>(con));
  pipe->pipe_lock.Lock();
  pipe->set_peer_type(type);
  pipe->set_peer_addr(addr);
  pipe->policy = get_policy(type);
  pipe->start_writer();
  if (first)
    pipe->_send(first);
  pipe->pipe_lock.Unlock();
  pipe->register_pipe();                         //add pip instance to SimpleMessager->rack_pipe map
  pipes.insert(pipe);                            //add pip instance to SimpleMessager->pipes set

  return pipe;

 client side commponents Messager
 Responsibility
    a network entity in ceph
    handles transmission and reception osd messages
    based on socket
 Implementation
    SimpleMessenger
    AsyncMessenger
    Xio
 Role
    cluster_messenger:responsible for communicationg with other OSDs/Monitors
    client_messenger: responsible for communicating with client(qemu/kvm、ceph-fuse，etc)
    hbclient_messenger、hb_front_server_messenger hb_back_server_messenger



 SimpleMessager:

/*
 * This class handles transmission and reception of messages. Generally
 * speaking, there are several major components:
 *
 * - Connection(PipConnection)
 *    Each logical session is associated with a Connection.
 * - Pipe
 *    Each network connection is handled through a pipe, which handles
 *    the input and output of each message.  There is normally a 1:1
 *    relationship between Pipe and Connection, but logical sessions may
 *    get handed off between Pipes when sockets reconnect or during
 *    connection races.
 * - IncomingQueue
 *    Incoming messages are associated with an IncomingQueue, and there
 *    is one such queue associated with each Pipe.
 * - DispatchQueue
 *    IncomingQueues get queued in the DIspatchQueue, which is responsible
 *    for doing a round-robin sweep and processing them via a worker thread.
 * - SimpleMessenger
 *    It's the exterior class passed to the external message handler and
 *    most of the API details.
 *
 * Lock ordering:
 *
 *   SimpleMessenger::lock
 *       Pipe::pipe_lock
 *           DispatchQueue::lock
 *               IncomingQueue::lock
 */

 Accepter:
 /**
 * If the SimpleMessenger binds to a specific address, the Accepter runs
 * and listens for incoming connections.
 */


 DispatchQueue:
 /**
 * The DispatchQueue contains all the Pipes which have Messages
 * they want to be dispatched, carefully organized by Message priority
 * and permitted to deliver in a round-robin fashion.
 * See SimpleMessenger::dispatch_entry for details.
 */

 DispatchThread (in DispatchQueue)
  /**
   * The DispatchThread runs dispatch_entry to empty out the dispatch_queue.
   */


 Pip
 /**
   * The Pipe is the most complex SimpleMessenger component. It gets
   * two threads, one each for reading and writing on a socket it's handed
   * at creation time, and is responsible for everything that happens on
   * that socket. Besides message transmission, it's responsible for
   * propagating socket errors to the SimpleMessenger and then sticking
   * around in a state where it can provide enough data for the SimpleMessenger
   * to provide reliable Message delivery when it manages to reconnect.
   */
   Reader(in Pip)
   /**
     * The Reader thread handles all reads off the socket -- not just
     * Messages, but also acks and other protocol bits (excepting startup,
     * when the Writer does a couple of reads).
     * All the work is implemented in Pipe itself, of course.
     */
     Writer(in Pip)
    /**
     * The Writer thread handles all writes to the socket (after startup).
     * All the work is implemented in Pipe itself, of course.
     */