muduo库的EpollPoller剖析

EpollPoller和PollPoller一样，都是muduo库对I/O复用机制的封装，不过默认使用的是EpollPoller。在EventLoop中初始化构造poller_,调用newDefaultPoller(this)，构造默认的poller。

EventLoop::EventLoop()
  : looping_(false),  //表示还未循环
    quit_(false),
    eventHandling_(false),
    callingPendingFunctors_(false),
    iteration_(0),
    threadId_(CurrentThread::tid()),   //赋值真实id
    poller_(Poller::newDefaultPoller(this)),   //构造了一个实际的poller对象
    timerQueue_(new TimerQueue(this)),
    wakeupFd_(createEventfd()),
    wakeupChannel_(new Channel(this, wakeupFd_)),
    currentActiveChannel_(NULL)

newDefaultPoller内部是这样的：

Poller* Poller::newDefaultPoller(EventLoop* loop)
{
  if (::getenv("MUDUO_USE_POLL"))   //看环境变量，如果没有MUDUO_UER_POLL，默认epoll
  {
    return new PollPoller(loop);
  }
  else
  {
    return new EPollPoller(loop);
  }
}

也就是说，如果我们没有主动设置环境变量为MUDUO_USE_POLL，我们默认选择为构造EpollPoller。

EpollPoller重写了父类的三个函数：

virtual Timestamp poll(int timeoutMs, ChannelList* activeChannels);
virtual void updateChannel(Channel* channel);
virtual void removeChannel(Channel* channel);

EpollPoller的私有成员有：

private:
  static const int kInitEventListSize = 16;   //默认事件数组大小，是用来装epoll_wait()返回的可读或可写事件的
  									

  static const char* operationToString(int op);

  void fillActiveChannels(int numEvents,
                          ChannelList* activeChannels) const;
  void update(int operation, Channel* channel);

  typedef std::vector<struct epoll_event> EventList;

  int epollfd_;
  EventList events_;  //用来存发生事件的数组

其中kInitEventListSize是epoll模式中用来盛装返回的发生事件的数组大小，就是events_，它在构造函数中会被初始化，看下面：

EPollPoller::EPollPoller(EventLoop* loop)
  : Poller(loop),
    epollfd_(::epoll_create1(EPOLL_CLOEXEC)),   //创建epollfd，使用带1的版本
    events_(kInitEventListSize)    //vector这样用时初始化kInitEventListSize个大小空间
{
  if (epollfd_ < 0)   //在构造函数中判断，<0就abort()
  {
    LOG_SYSFATAL << "EPollPoller::EPollPoller";
  }
}

有一点需要注意，就是vector的初始化方式，这样可以预分配大小。至于我们以后不断向epoll中注册事件，然后导致返回的发生事件可能越来越多，events_装不下了，muduo也有自己的处理，是在EpollPoller的核心函数poll()函数中处理的：

  if (implicit_cast<size_t>(numEvents) == events_.size())  //如果返回的事件数目等于当前事件数组大小，就分配2倍空间
    {																
      events_.resize(events_.size()*2);   
    }														       

所以不必担心vector的大小问题了，后续会以乘以2倍的方式分配，这也是内存分配的常见做法。

我们再来回忆一下事件处理流程，先看一下事件注册图：

当某个客端，比如TCPConnection，或者我们自己写的代码调用Channel的enableReading()函数，就会引发Channel的Update()函数的调用，然后会调用EventLoop的UpdateChannel()函数，实际上会调用到Poller的updateChannel()函数。比如采用EpollPoller，那就会将事件注册到epoll的内核事件表中去，然后开启epoll_wait()，就是我们正常的e'poll流程了。

然后看一下事件发生后处理的时序，先看图：

如图，EventLoop调用poll()是上面事件注册的过程。目前我们处于Poller位置，当Poller中事件发生后，会调用fillActiveChannels()函数，把发生的事件填充给活跃事件通道表中，具体实现是下面的Poller的成员函数fillActiveChannels函数：

void EPollPoller::fillActiveChannels(int numEvents,
                                     ChannelList* activeChannels) const
{
  assert(implicit_cast<size_t>(numEvents) <= events_.size());//确定它的大小小于events_的大小，因为events_是预留的事件vector                                                                                    
  
  for (int i = 0; i < numEvents; ++i)   //挨个处理发生的numEvents个事件，epoll模式返回的events_数组中都是已经发生额事件，这有别于select和poll
  {
    Channel* channel = static_cast<Channel*>(events_[i].data.ptr);
/*
这是epoll模式epoll_event事件的数据结构，其中data不仅可以保存fd，也可以保存一个void*类型的指针。
typedef union epoll_data {
               void    *ptr;
               int      fd;
               uint32_t u32;
               uint64_t u64;
           } epoll_data_t;

           struct epoll_event {
               uint32_t     events;    // Epoll events 
               epoll_data_t data;      //User data variable 
           };
*/
    channel->set_revents(events_[i].events);   //把已发生的事件传给channel,写到通道当中
    activeChannels->push_back(channel);    //并且push_back进activeChannels
  }
}

最后一句push_back不言自明。我们再看EventLoop中核心函数也就是一直循环的loop()函数的处理：

void EventLoop::loop()
{
  assert(!looping_);
  assertInLoopThread();  //断言处于创建该对象的线程中
  looping_ = true;
  quit_ = false;  // FIXME: what if someone calls quit() before loop() ?
  LOG_TRACE << "EventLoop " << this << " start looping";

  while (!quit_)
  {
    activeChannels_.clear();  //首先清零
    pollReturnTime_ = poller_->poll(kPollTimeMs, &activeChannels_);  //调用poll返回活动的通道
    ++iteration_; 
    if (Logger::logLevel() <= Logger::TRACE)
    {
      printActiveChannels();  //日志登记，日志打印
    }
    // TODO sort channel by priority
    eventHandling_ = true;  //true
    for (ChannelList::iterator it = activeChannels_.begin();
        it != activeChannels_.end(); ++it)  //遍历通道来进行处理
    {
      currentActiveChannel_ = *it;
      currentActiveChannel_->handleEvent(pollReturnTime_);
    }
    currentActiveChannel_ = NULL;   //处理完了赋空
    eventHandling_ = false;  //false
    doPendingFunctors();   //
  }

  LOG_TRACE << "EventLoop " << this << " stop looping";
  looping_ = false;
}

当poll()返回时，loop()函数中会遍历活跃通道表activeChannels，并且执行它们每一个提前注册大的handleEvent()函数，处理完了会继续循环。

整个EpollPoller的作用就是这个样子了，接下来看完全的源码分析：

.h:

#ifndef MUDUO_NET_EVENTLOOP_H
#define MUDUO_NET_EVENTLOOP_H

#include <vector>

#include <boost/any.hpp>
#include <boost/function.hpp>
#include <boost/noncopyable.hpp>
#include <boost/scoped_ptr.hpp>

#include <muduo/base/Mutex.h>
#include <muduo/base/CurrentThread.h>
#include <muduo/base/Timestamp.h>
#include <muduo/net/Callbacks.h>
#include <muduo/net/TimerId.h>

namespace muduo
{
namespace net
{

class Channel;
class Poller;
class TimerQueue;

///
/// Reactor, at most one per thread.
///
/// This is an interface class, so don't expose too much details.
class EventLoop : boost::noncopyable
{
 public:
  typedef boost::function<void()> Functor;

  EventLoop();
  ~EventLoop();  // force out-line dtor, for scoped_ptr members.

  ///
  /// Loops forever.
  ///
  /// Must be called in the same thread as creation of the object.
  ///
  void loop();

  /// Quits loop.
  ///
  /// This is not 100% thread safe, if you call through a raw pointer,
  /// better to call through shared_ptr<EventLoop> for 100% safety.
  void quit();

  ///
  /// Time when poll returns, usually means data arrival.
  ///
  Timestamp pollReturnTime() const { return pollReturnTime_; }

  int64_t iteration() const { return iteration_; }

  /// Runs callback immediately in the loop thread.
  /// It wakes up the loop, and run the cb.
  /// If in the same loop thread, cb is run within the function.
  /// Safe to call from other threads.
  void runInLoop(const Functor& cb);
  /// Queues callback in the loop thread.
  /// Runs after finish pooling.
  /// Safe to call from other threads.
  void queueInLoop(const Functor& cb);

#ifdef __GXX_EXPERIMENTAL_CXX0X__
  void runInLoop(Functor&& cb);
  void queueInLoop(Functor&& cb);
#endif

  // timers

  ///
  /// Runs callback at 'time'.
  /// Safe to call from other threads.
  ///
  TimerId runAt(const Timestamp& time, const TimerCallback& cb);
  ///
  /// Runs callback after @c delay seconds.
  /// Safe to call from other threads.
  ///
  TimerId runAfter(double delay, const TimerCallback& cb);
  ///
  /// Runs callback every @c interval seconds.
  /// Safe to call from other threads.
  ///
  TimerId runEvery(double interval, const TimerCallback& cb);
  ///
  /// Cancels the timer.
  /// Safe to call from other threads.
  ///
  void cancel(TimerId timerId);

#ifdef __GXX_EXPERIMENTAL_CXX0X__
  TimerId runAt(const Timestamp& time, TimerCallback&& cb);
  TimerId runAfter(double delay, TimerCallback&& cb);
  TimerId runEvery(double interval, TimerCallback&& cb);
#endif

  // internal usage
  void wakeup();
  void updateChannel(Channel* channel);  //在poller中注册或者更新通道
  void removeChannel(Channel* channel);  //从poller中移除通道
  bool hasChannel(Channel* channel);

  // pid_t threadId() const { return threadId_; }
  void assertInLoopThread()
  {
    if (!isInLoopThread())  //如果不是，终止程序
    {
      abortNotInLoopThread();
    }
  }
  bool isInLoopThread() const { return threadId_ == CurrentThread::tid(); }  //比较线程id即可。
  // bool callingPendingFunctors() const { return callingPendingFunctors_; }
  bool eventHandling() const { return eventHandling_; }

  void setContext(const boost::any& context)
  { context_ = context; }

  const boost::any& getContext() const
  { return context_; }

  boost::any* getMutableContext()
  { return &context_; }

  static EventLoop* getEventLoopOfCurrentThread();

 private:
  void abortNotInLoopThread();
  void handleRead();  // waked up
  void doPendingFunctors();

  void printActiveChannels() const; // DEBUG

  typedef std::vector<Channel*> ChannelList;

  bool looping_; /* atomic */
  bool quit_; /* atomic and shared between threads, okay on x86, I guess. */
  bool eventHandling_; /* atomic */
  bool callingPendingFunctors_; /* atomic */
  int64_t iteration_;  
  const pid_t threadId_;   //当前所属对象线程id
  Timestamp pollReturnTime_;    //时间戳，poll返回的时间戳
  boost::scoped_ptr<Poller> poller_;  //poller对象
  boost::scoped_ptr<TimerQueue> timerQueue_;
  int wakeupFd_;
  // unlike in TimerQueue, which is an internal class,
  // we don't expose Channel to client.
  boost::scoped_ptr<Channel> wakeupChannel_;
  boost::any context_;

  // scratch variables
  ChannelList activeChannels_;   //Poller返回的活动通道，vector<channel*>类型
  Channel* currentActiveChannel_;   //当前正在处理的活动通道

  MutexLock mutex_;
  std::vector<Functor> pendingFunctors_; // @GuardedBy mutex_
};

}
}
#endif  // MUDUO_NET_EVENTLOOP_H

.cpp:

namespace
{
const int kNew = -1;
const int kAdded = 1;
const int kDeleted = 2;
}

EPollPoller::EPollPoller(EventLoop* loop)
  : Poller(loop),
    epollfd_(::epoll_create1(EPOLL_CLOEXEC)),   //创建epollfd，使用带1的版本
    events_(kInitEventListSize)    //vector这样用时初始化kInitEventListSize个大小空间
{
  if (epollfd_ < 0)   //在构造函数中判断，<0就abort()
  {
    LOG_SYSFATAL << "EPollPoller::EPollPoller";
  }
}

EPollPoller::~EPollPoller()
{
  ::close(epollfd_);   //关闭
}

Timestamp EPollPoller::poll(int timeoutMs, ChannelList* activeChannels)
{
  LOG_TRACE << "fd total count " << channels_.size();
  int numEvents = ::epoll_wait(epollfd_,
                               &*events_.begin(),
                               static_cast<int>(events_.size()),    //使用epoll_wait()，等待事件返回,返回发生的事件数目
                               timeoutMs);  //epoll
  int savedErrno = errno;    //错误号
  Timestamp now(Timestamp::now());  //得到时间戳
  if (numEvents > 0)
  {
    LOG_TRACE << numEvents << " events happended";
    fillActiveChannels(numEvents, activeChannels);  //调用fillActiveChannels，传入numEvents也就是发生的事件数目
    if (implicit_cast<size_t>(numEvents) == events_.size())  //如果返回的事件数目等于当前事件数组大小，就分配2倍空间
    {																//!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
      events_.resize(events_.size()*2);     //!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
    }														       //!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
  }
  else if (numEvents == 0)  //返回=0
  {
    LOG_TRACE << "nothing happended";
  }
  else
  {
    // error happens, log uncommon ones
    if (savedErrno != EINTR)   //如果不是EINTR信号，就把错误号保存下来，并且输入到日志中
    {
      errno = savedErrno;
      LOG_SYSERR << "EPollPoller::poll()";
    }
  }
  return now;  //返回时间戳
}

//把返回到的这么多个事件添加到activeChannels
/*protected:
  typedef std::map<int, Channel*> ChannelMap;
  ChannelMap channels_;
*/
void EPollPoller::fillActiveChannels(int numEvents,
                                     ChannelList* activeChannels) const
{
  assert(implicit_cast<size_t>(numEvents) <= events_.size());//确定它的大小小于events_的大小，因为events_是预留的事件vector
                                                                                       
/*typedef std::vector<struct epoll_event> EventList;   //modify by WY.Huang

  int epollfd_;
  EventList events_;
*/
  
  for (int i = 0; i < numEvents; ++i)   //挨个处理发生的numEvents个事件，epoll模式返回的events_数组中都是已经发生额事件，这有别于select和poll
  {
    Channel* channel = static_cast<Channel*>(events_[i].data.ptr);
/*
这是epoll模式epoll_event事件的数据结构，其中data不仅可以保存fd，也可以保存一个void*类型的指针。
typedef union epoll_data {
               void    *ptr;
               int      fd;
               uint32_t u32;
               uint64_t u64;
           } epoll_data_t;

           struct epoll_event {
               uint32_t     events;    // Epoll events 
               epoll_data_t data;      //User data variable 
           };
*/
#ifndef NDEBUG
    int fd = channel->fd();  //debug时做一下检测
    ChannelMap::const_iterator it = channels_.find(fd);
    assert(it != channels_.end());
    assert(it->second == channel);
#endif
    channel->set_revents(events_[i].events);   //把已发生的事件传给channel,写到通道当中
    activeChannels->push_back(channel);    //并且push_back进activeChannels
  }
}

//这个函数被调用是因为channel->enablereading()被调用，再调用channel->update()，再event_loop->updateChannel()，再->epoll或poll的updateChannel被调用
void EPollPoller::updateChannel(Channel* channel)   //更新通道
{
  Poller::assertInLoopThread();
  const int index = channel->index();   //channel是本函数参数，获得channel的index，初始状态index是-1
  LOG_TRACE << "fd = " << channel->fd()
    << " events = " << channel->events() << " index = " << index;
  if (index == kNew || index == kDeleted)  //index是在poll中是下标，在epoll中是三种状态，上面有三个常量
  {
    // a new one, add with EPOLL_CTL_ADD
    int fd = channel->fd();
    if (index == kNew)
    {
      assert(channels_.find(fd) == channels_.end());
      channels_[fd] = channel;
    }
    else // index == kDeleted
    {
      assert(channels_.find(fd) != channels_.end());
      assert(channels_[fd] == channel);
    }

    channel->set_index(kAdded);
    update(EPOLL_CTL_ADD, channel);
  }
  else
  {
    // update existing one with EPOLL_CTL_MOD/DEL
    int fd = channel->fd();
    (void)fd;
    assert(channels_.find(fd) != channels_.end());
    assert(channels_[fd] == channel);
    assert(index == kAdded);  //确保目前它存在
   //
   //poll是把文件描述符变为相反数减1，这里是使用EPOLL_CTL_DEL从内核事件表中删除
   //
    if (channel->isNoneEvent())  //如果什么也没关注，就直接干掉
    {
      update(EPOLL_CTL_DEL, channel);
      channel->set_index(kDeleted);     //删除之后设为deleted，表示已经删除，只是从内核事件表中删除，在channels_这个通道数组中并没有删除
    }
    else
    {
      update(EPOLL_CTL_MOD, channel);   //有关注，那就只是更新。更新成什么样子channel中会决定。
    }
  }
}

void EPollPoller::removeChannel(Channel* channel)
{
  Poller::assertInLoopThread();
  int fd = channel->fd();
  LOG_TRACE << "fd = " << fd;
  assert(channels_.find(fd) != channels_.end());
  assert(channels_[fd] == channel);
  assert(channel->isNoneEvent());
  int index = channel->index();
  assert(index == kAdded || index == kDeleted);
  size_t n = channels_.erase(fd);
  (void)n;
  assert(n == 1);

  if (index == kAdded)
  {
    update(EPOLL_CTL_DEL, channel);
  }
  channel->set_index(kNew);
}

//epoll_ctl函数的封装
void EPollPoller::update(int operation, Channel* channel)
{
  struct epoll_event event;
  bzero(&event, sizeof event);
  event.events = channel->events();
  event.data.ptr = channel;   //传入data.ptr
  int fd = channel->fd();
  LOG_TRACE << "epoll_ctl op = " << operationToString(operation)
    << " fd = " << fd << " event = { " << channel->eventsToString() << " }";
  if (::epoll_ctl(epollfd_, operation, fd, &event) < 0)  //使用epoll_ctl
  {
    if (operation == EPOLL_CTL_DEL)  //如果delete失败，不退出
    {
      LOG_SYSERR << "epoll_ctl op =" << operationToString(operation) << " fd =" << fd;
    }
    else
    {
      LOG_SYSFATAL << "epoll_ctl op =" << operationToString(operation) << " fd =" << fd;
    }
  }
}

const char* EPollPoller::operationToString(int op)  //调试用的函数
{
  switch (op)
  {
    case EPOLL_CTL_ADD:
      return "ADD";
    case EPOLL_CTL_DEL:
      return "DEL";
    case EPOLL_CTL_MOD:
      return "MOD";
    default:
      assert(false && "ERROR op");
      return "Unknown Operation";
  }
}