CLIENT数据流向OSD

CLIENT数据流向OSD

ceph-osd.cc

ceph_osd.cc中main函数是ceph-osd守护进程的入口，目前OSD会创建注册7个Messenger实例

Messenger实例名称	作用
ms_public	用来处理OSD和Client之间的消息
ms_cluster	用来处理OSD和集群其他OSD及MON之间的消息
ms_hb_back_client	用来处理client接收心跳消息
ms_hb_front_client	用来处理client发送心跳消息
ms_hb_back_server	用来处理OSD接收心跳消息
ms_hb_front_server	用来处理OSD发送心跳消息
ms_objecter	用来处理OSD和Objecter之间的消息

ceph-osd.cc int main(int argc, const char **argv)
{
  Messenger *ms_public = Messenger::create(g_ceph_context, public_msg_type,
					   entity_name_t::OSD(whoami), "client", nonce);
  Messenger *ms_cluster = Messenger::create(g_ceph_context, cluster_msg_type,
					    entity_name_t::OSD(whoami), "cluster", nonce);
  Messenger *ms_hb_back_client = Messenger::create(g_ceph_context, cluster_msg_type,
					     entity_name_t::OSD(whoami), "hb_back_client", nonce);
  Messenger *ms_hb_front_client = Messenger::create(g_ceph_context, public_msg_type,
					     entity_name_t::OSD(whoami), "hb_front_client", nonce);
  Messenger *ms_hb_back_server = Messenger::create(g_ceph_context, cluster_msg_type,
						   entity_name_t::OSD(whoami), "hb_back_server", nonce);
  Messenger *ms_hb_front_server = Messenger::create(g_ceph_context, public_msg_type,
						    entity_name_t::OSD(whoami), "hb_front_server", nonce);
  Messenger *ms_objecter = Messenger::create(g_ceph_context, public_msg_type,
					     entity_name_t::OSD(whoami), "ms_objecter", nonce);

osdptr = new OSD(g_ceph_context,
		   std::move(store),
		   whoami,
		   ms_cluster,
		   ms_public,
		   ms_hb_front_client,
		   ms_hb_back_client,
		   ms_hb_front_server,
		   ms_hb_back_server,
		   ms_objecter,
		   &mc,
		   data_path,
		   journal_path,
		   poolctx);
ms_public->start();

err = osdptr->init();

ms_public->wait();
}

OSD.cc

ms_public 传到init中的是client_messenger，add_dispatcher_tail会将OSD dispatch添加到client_messenger，后期会调用OSD 类中的ms_fast_dispatch函数

int OSD::init(){
  // i'm ready!
  client_messenger->add_dispatcher_tail(&mgrc);
  client_messenger->add_dispatcher_tail(this);
  cluster_messenger->add_dispatcher_head(this);

  hb_front_client_messenger->add_dispatcher_head(&heartbeat_dispatcher);
  hb_back_client_messenger->add_dispatcher_head(&heartbeat_dispatcher);
  hb_front_server_messenger->add_dispatcher_head(&heartbeat_dispatcher);
  hb_back_server_messenger->add_dispatcher_head(&heartbeat_dispatcher);

  objecter_messenger->add_dispatcher_head(service.objecter.get());
}

Messenger.h

Messenger是一个基类，AsyncMessenger继承该类。add_dispatcher_head/add_dispatcher_tail都会调用 ready();Messenger中的ready()函数为虚函数，会执行子类（AsyncMessenger）中的ready()函数。

 /**
   * Add a new Dispatcher to the front of the list. If you add
   * a Dispatcher which is already included, it will get a duplicate
   * entry. This will reduce efficiency but not break anything.
   *
   * @param d The Dispatcher to insert into the list.
   */
  void add_dispatcher_head(Dispatcher *d) {
    bool first = dispatchers.empty();
    dispatchers.push_front(d);
    if (d->ms_can_fast_dispatch_any())
      fast_dispatchers.push_front(d);
    if (first)
      ready();
  }
  /**
   * Add a new Dispatcher to the end of the list. If you add
   * a Dispatcher which is already included, it will get a duplicate
   * entry. This will reduce efficiency but not break anything.
   *
   * @param d The Dispatcher to insert into the list.
   */
  void add_dispatcher_tail(Dispatcher *d) {
    bool first = dispatchers.empty();
    dispatchers.push_back(d);
    if (d->ms_can_fast_dispatch_any())
      fast_dispatchers.push_back(d);
    if (first)
      ready();
  }

AsyncMessenger.c

AsyncMessenger ready函数执行的是dispatch_queue类中的start函数。

void AsyncMessenger::ready()
{
  ldout(cct,10) << __func__ << " " << get_myaddrs() << dendl;

  stack->ready();
  if (pending_bind) {
    int err = bindv(pending_bind_addrs);
    if (err) {
      lderr(cct) << __func__ << " postponed bind failed" << dendl;
      ceph_abort();
    }
  }

  std::lock_guard l{lock};
  for (auto &&p : processors)
    p->start();
  dispatch_queue.start();
}

DispatchQueue.cc

DispatchQueue中的start函数会执行local_delivery_thread的create函数

void DispatchQueue::start()
{
  ceph_assert(!stop);
  ceph_assert(!dispatch_thread.is_started());
  dispatch_thread.create("ms_dispatch");
  local_delivery_thread.create("ms_local");
}

Thread.cc

local_delivery_thread.create会执行Thread::try_create函数，该函数是创建一个线程

int Thread::try_create(size_t stacksize)
{
  pthread_attr_t *thread_attr = NULL;
  pthread_attr_t thread_attr_loc;
  
  stacksize &= CEPH_PAGE_MASK;  // must be multiple of page
  if (stacksize) {
    thread_attr = &thread_attr_loc;
    pthread_attr_init(thread_attr);
    pthread_attr_setstacksize(thread_attr, stacksize);
  }

  int r;

  // The child thread will inherit our signal mask.  Set our signal mask to
  // the set of signals we want to block.  (It's ok to block signals more
  // signals than usual for a little while-- they will just be delivered to
  // another thread or delieverd to this thread later.)

  #ifndef _WIN32
  sigset_t old_sigset;
  if (g_code_env == CODE_ENVIRONMENT_LIBRARY) {
    block_signals(NULL, &old_sigset);
  }
  else {
    int to_block[] = { SIGPIPE , 0 };
    block_signals(to_block, &old_sigset);
  }
  r = pthread_create(&thread_id, thread_attr, _entry_func, (void*)this);
  restore_sigset(&old_sigset);
  #else
  r = pthread_create(&thread_id, thread_attr, _entry_func, (void*)this);
  #endif

  if (thread_attr) {
    pthread_attr_destroy(thread_attr);	
  }

  return r;
}

DispatchQueue.h

线程函数为local_delivery_thread中的entry函数，该函数会继续调用DispatchQueue中的run_local_delivery函数

  class LocalDeliveryThread : public Thread {
    DispatchQueue *dq;
  public:
    explicit LocalDeliveryThread(DispatchQueue *dq) : dq(dq) {}
    void *entry() override {
      dq->run_local_delivery();
      return 0;
    }
  } local_delivery_thread;

DispatchQueue.cc

run_local_delivery函数会执行fast_dispatch函数，进行分发消息

  void DispatchQueue::run_local_delivery()
{
  std::unique_lock l{local_delivery_lock};
  while (true) {
    if (stop_local_delivery)
      break;
    if (local_messages.empty()) {
      local_delivery_cond.wait(l);
      continue;
    }
    auto p = std::move(local_messages.front());
    local_messages.pop();
    l.unlock();
    const ref_t<Message>& m = p.first;
    int priority = p.second;
    fast_preprocess(m);
    if (can_fast_dispatch(m)) {
      fast_dispatch(m);
    } else {
      enqueue(m, priority, 0);
    }
    l.lock();
  }
}

DispatchQueue.h

fast_dispatch为重载函数，最终执行Messenger中的ms_fast_dispatch

void fast_dispatch(Message* m) {
    return fast_dispatch(ceph::ref_t<Message>(m, false)); /* consume ref */
  }
  
void DispatchQueue::fast_dispatch(const ref_t<Message>& m)
{
  uint64_t msize = pre_dispatch(m);
  msgr->ms_fast_dispatch(m);
  post_dispatch(m, msize);
}

Messenger.h

ms_fast_dispatch会调用dispatcher的ms_fast_dispatch2

  /**
   * Deliver a single Message via "fast dispatch".
   *
   * @param m The Message we are fast dispatching.
   * If none of our Dispatchers can handle it, ceph_abort().
   */
  void ms_fast_dispatch(const ceph::ref_t<Message> &m) {
    m->set_dispatch_stamp(ceph_clock_now());
    for (const auto &dispatcher : fast_dispatchers) {
      if (dispatcher->ms_can_fast_dispatch2(m)) {
	dispatcher->ms_fast_dispatch2(m);
	return;
      }
    }
    ceph_abort();
  }

Dispatcher.h

dispatcher的ms_fast_dispatch2会执行ms_fast_dispatch，dispatcher类中的ms_fast_dispatch为虚函数，会调用子类的ms_fast_dispatch函数，最终执行了OSD::ms_fast_dispatch

   virtual void ms_fast_dispatch2(const MessageRef &m) {
    /* allow old style dispatch handling that expects a Message * with a floating ref */
    return ms_fast_dispatch(MessageRef(m).detach()); /* XXX N.B. always consumes ref */
  }

流程图概览

Ceph-osd::main
	Messenger::create
	OSD::OSD
	OSD::init
		Messenger::add_dispatcher_tail
		Messenger::add_dispatcher_head
			AsyncMessenger::ready
				DispatchQueue::start
					local_delivery_thread::create
						local_delivery_thread::entry
							DispatchQueue::run_local_delivery
								DispatchQueue::fast_dispatch
									DispatchQueue::fast_dispatch(const ref_t<Message>& m)
										Messenger::ms_fast_dispatch
											Dispatcher::ms_fast_dispatch2
												OSD::ms_fast_dispatch
											
注：class OSD : public Dispatcher