thrift 框架源码：线程池管理类 ThreadManager

thrift/lib/cpp/src/thrift/concurrency 这个路径下基本是thrift用到的多线程相关的代码

ThreadManager

原注释：

• 这是一个线程池管理类，用线程工厂创建线程。
• 初始化后，不会再创建线程或销毁线程，而是保存空闲线程数变量、活跃线程数变量
• 维护一个任务（task）队列，并平衡服务的响应与等待时间，通知线程池策略对象（PoolPolicy object ）相关的信息，由PoolPolicy object 决定池大小调整，这样的好处是可以将策略解耦出来。

class ThreadManager {

protected:
  ThreadManager() = default;

public:
  typedef std::function<void(std::shared_ptr<Runnable>)> ExpireCallback;

  virtual ~ThreadManager() = default;

  // 启动管理类，验证各项初始化，并分配资源
  virtual void start() = 0;

  /**
   * 终止管理类，丢弃所有未处理任务。
   * 关闭所有worker线程，释放所有资源.
   * 会阻塞知道所有线程返回，所以有可能会一直阻塞.
   * worker线程是否join 取决于线程工厂*/
  virtual void stop() = 0;

  // 运行状态枚举类
  enum STATE { UNINITIALIZED, STARTING, STARTED, JOINING, STOPPING, STOPPED };

  virtual STATE state() const = 0;

  // 返回当前线程工厂
  virtual std::shared_ptr<ThreadFactory> threadFactory() const = 0;

  // 设置新工厂，如果与原有工厂的销毁策略不同则抛异常
  virtual void threadFactory(std::shared_ptr<ThreadFactory> value) = 0;

  // 添加 worker线程
  virtual void addWorker(size_t value = 1) = 0;

  /**
   * 移除 worker线程
   * 如果工厂的销毁策略运行线程执行完则join 
   * 阻塞直到线程数达标.
   * \param[in]  value  移除线程数
   * \throws InvalidArgumentException 若value大于已有线程数
   */
  virtual void removeWorker(size_t value = 1) = 0;

  // 返回空闲线程数
  virtual size_t idleWorkerCount() const = 0;

  // 返回总线程数
  virtual size_t workerCount() const = 0;

  // 返回等待 task 数
  virtual size_t pendingTaskCount() const = 0;

  // 返回所有task数（执行中与等待中）
  virtual size_t totalTaskCount() const = 0;

  // 返回允许最大等待task数
  virtual size_t pendingTaskCountMax() const = 0;

  // 过期task 数
  virtual size_t expiredTaskCount() const = 0;

  /**
   * 添加一个task，会被 worker线程执行.
   * 会阻塞 若  pendingTaskCountMax()非零，且pendingTaskCount() 大于等于pendingTaskCountMax().  If this method is called in the
   * context of a ThreadManager worker thread it will throw a
   * TooManyPendingTasksException
   *
   * @param task  任务
   * @param timeout 最多等待微妙数. Specific cases:
   * timeout = 0  : 永远等待.
   * timeout = -1 : 若阻塞立即返回
   * @param expiration 任务过期微妙数，超过则被丢弃无法执行.
   * @throws TooManyPendingTasksException Pending task count exceeds max pending task count
   */
  virtual void add(std::shared_ptr<Runnable> task,
                   int64_t timeout = 0LL,
                   int64_t expiration = 0LL) = 0;

  // 移除等待的task
  virtual void remove(std::shared_ptr<Runnable> task) = 0;

  /**
   * Remove the next pending task which would be run.
   *
   * @return the task removed.
   */
  virtual std::shared_ptr<Runnable> removeNextPending() = 0;

  /**
   * Remove tasks from front of task queue that have expired.
   */
  virtual void removeExpiredTasks() = 0;

  /**
   * task 过期回调函数 
   * @param expireCallback a function called with the shared_ptr<Runnable> for
   * the expired task.
   */
  virtual void setExpireCallback(ExpireCallback expireCallback) = 0;

  static std::shared_ptr<ThreadManager> newThreadManager();

  /**
   * Creates a simple thread manager the uses count number of worker threads and has
   * a pendingTaskCountMax maximum pending tasks. The default, 0, specified no limit
   * on pending tasks
   */
  static std::shared_ptr<ThreadManager> newSimpleThreadManager(size_t count = 4,
                                                                 size_t pendingTaskCountMax = 0);

  class Task; // 内部类，标识一个task
 
  class Worker;  // 内部类，标识一个worker线程
 
  class Impl; // 内部类，标识一个 管理类的具体实现
};

 

内部类 Task

class ThreadManager::Task : public Runnable {

public:
  enum STATE { WAITING, EXECUTING, TIMEDOUT, COMPLETE };

  Task(shared_ptr<Runnable> runnable, uint64_t expiration = 0ULL)
    : runnable_(runnable),
      state_(WAITING) {
        if (expiration != 0ULL) {
          expireTime_.reset(new std::chrono::steady_clock::time_point(std::chrono::steady_clock::now() + std::chrono::milliseconds(expiration)));
        }
    }

  ~Task() override = default;

  void run() override {
    if (state_ == EXECUTING) {
      runnable_->run();
      state_ = COMPLETE;
    }
  }

  shared_ptr<Runnable> getRunnable() { return runnable_; }

  const unique_ptr<std::chrono::steady_clock::time_point> & getExpireTime() const { return expireTime_; }

private:
  shared_ptr<Runnable> runnable_;
  friend class ThreadManager::Worker;
  STATE state_;
  unique_ptr<std::chrono::steady_clock::time_point> expireTime_;
};

task 类继承了 Runnable 类，重写了run 方法，worker类是友元类，即worker 线程可以随意访问一个task 的内部成员。

有四种状态：等待、执行中、超时、完成。内部有一个 Runnable 成员。

 

内部类 worker

class ThreadManager::Worker : public Runnable {
  enum STATE { UNINITIALIZED, STARTING, STARTED, STOPPING, STOPPED };

public:
  Worker(ThreadManager::Impl* manager) : manager_(manager), state_(UNINITIALIZED) {}

  ~Worker() override = default;

private:
  bool isActive() const {
    return (manager_->workerCount_ <= manager_->workerMaxCount_)
           || (manager_->state_ == JOINING && !manager_->tasks_.empty());
  }

public:
  /**
   * 线程入口
   * 从task 队列中取出一个task 执行.
   */
  void run() override {
    Guard g(manager_->mutex_);

    /**
     * This method has three parts; one is to check for and account for
     * admitting a task which happens under a lock.  Then the lock is released
     * and the task itself is executed.  Finally we do some accounting
     * under lock again when the task completes.
     */

    /**
     * Admitting
     */

    /**
     * 增加worker数量，若达到最大值则通知 manager
     */
    bool active = manager_->workerCount_ < manager_->workerMaxCount_;
    if (active) {
      if (++manager_->workerCount_ == manager_->workerMaxCount_) {
        manager_->workerMonitor_.notify();
      }
    }

    while (active) {
      /**
        * While holding manager monitor block for non-empty task queue (Also
        * check that the thread hasn't been requested to stop). Once the queue
        * is non-empty, dequeue a task, release monitor, and execute. If the
        * worker max count has been decremented such that we exceed it, mark
        * ourself inactive, decrement the worker count and notify the manager
        * (technically we're notifying the next blocked thread but eventually
        * the manager will see it.
        */
      active = isActive(); // 检测manager是否要求停止

      while (active && manager_->tasks_.empty()) { // task队列为空，则wait 并释放锁，直到有task 如队列被唤醒
        manager_->idleCount_++; // 赋闲中
        manager_->monitor_.wait();
        active = isActive(); // 重新判断是否停止
        manager_->idleCount_--;
      }

      shared_ptr<ThreadManager::Task> task;

      if (active) {
        if (!manager_->tasks_.empty()) { // task队列非空
          task = manager_->tasks_.front(); // 弹出一个task
          manager_->tasks_.pop_front();
          if (task->state_ == ThreadManager::Task::WAITING) {
            task->state_ =
                (task->getExpireTime() && *(task->getExpireTime()) < std::chrono::steady_clock::now()) ?
                    ThreadManager::Task::TIMEDOUT :
                    ThreadManager::Task::EXECUTING; // 判断task是否超时，否则改为执行中
          }
        }

        /* 若设置了最大task数，而这里正好取了一个task，则唤醒被addtask 阻塞的线程 */
        if (manager_->pendingTaskCountMax_ != 0
            && manager_->tasks_.size() <= manager_->pendingTaskCountMax_ - 1) {
          manager_->maxMonitor_.notify();
        }
      }

      // 执行task
      if (task) {
        if (task->state_ == ThreadManager::Task::EXECUTING) {

          // 为不阻塞manager，这里释放锁，前面 guard 加锁了
          manager_->mutex_.unlock();

          try {
            task->run(); // 执行
          } catch (const std::exception& e) {
            GlobalOutput.printf("[ERROR] task->run() raised an exception: %s", e.what());
          } catch (...) {
            GlobalOutput.printf("[ERROR] task->run() raised an unknown exception");
          }

          // 重新获取manager的锁以继续（获取task）
          manager_->mutex_.lock();

        } else if (manager_->expireCallback_) {
          // 这里只能是超时了，需要回调
          manager_->expireCallback_(task->getRunnable());
          manager_->expiredCount_++;
        }
      }
    }

    // 结束了，停止当前任务
    manager_->deadWorkers_.insert(this->thread());
    if (--manager_->workerCount_ == manager_->workerMaxCount_) {
      manager_->workerMonitor_.notify();
    }
  }

private:
  ThreadManager::Impl* manager_;
  friend class ThreadManager::Impl;
  STATE state_;
};

worker类 继承了 runnable 类，主要方法是run，检测是否终止，否则从task 队列中取出一个task来执行。

 

manager 内部实现类 ThreadManager::Impl 

/**
 * ThreadManager class
 *
 * This class manages a pool of threads. It uses a ThreadFactory to create
 * threads.  It never actually creates or destroys worker threads, rather
 * it maintains statistics on number of idle threads, number of active threads,
 * task backlog, and average wait and service times.
 *
 * There are three different monitors used for signaling different conditions
 * however they all share the same mutex_.
 *
 * @version $Id:$
 */
class ThreadManager::Impl : public ThreadManager {

public:
  Impl()
    : workerCount_(0),
      workerMaxCount_(0),
      idleCount_(0),
      pendingTaskCountMax_(0),
      expiredCount_(0),
      state_(ThreadManager::UNINITIALIZED),
      monitor_(&mutex_),
      maxMonitor_(&mutex_),
      workerMonitor_(&mutex_) {}

  ~Impl() override { stop(); }

  void start() override;
  void stop() override;

  ThreadManager::STATE state() const override { return state_; }

  shared_ptr<ThreadFactory> threadFactory() const override {
    Guard g(mutex_);
    return threadFactory_;
  }

  void threadFactory(shared_ptr<ThreadFactory> value) override {
    Guard g(mutex_);
    if (threadFactory_ && threadFactory_->isDetached() != value->isDetached()) {
      throw InvalidArgumentException();
    }
    threadFactory_ = value;
  }

  void addWorker(size_t value) override;

  void removeWorker(size_t value) override;

  size_t idleWorkerCount() const override { return idleCount_; }

  size_t workerCount() const override {
    Guard g(mutex_);
    return workerCount_;
  }

  size_t pendingTaskCount() const override {
    Guard g(mutex_);
    return tasks_.size();
  }

  size_t totalTaskCount() const override {
    Guard g(mutex_);
    return tasks_.size() + workerCount_ - idleCount_;
  }

  size_t pendingTaskCountMax() const override {
    Guard g(mutex_);
    return pendingTaskCountMax_;
  }

  size_t expiredTaskCount() const override {
    Guard g(mutex_);
    return expiredCount_;
  }

  void pendingTaskCountMax(const size_t value) {
    Guard g(mutex_);
    pendingTaskCountMax_ = value;
  }

  void add(shared_ptr<Runnable> value, int64_t timeout, int64_t expiration) override;

  void remove(shared_ptr<Runnable> task) override;

  shared_ptr<Runnable> removeNextPending() override;

  void removeExpiredTasks() override {
    removeExpired(false);
  }

  void setExpireCallback(ExpireCallback expireCallback) override;

private:
  /**
   * Remove one or more expired tasks.
   * \param[in]  justOne  if true, try to remove just one task and return
   */
  void removeExpired(bool justOne);

  /**
   * \returns whether it is acceptable to block, depending on the current thread id
   */
  bool canSleep() const;

  /**
   * Lowers the maximum worker count and blocks until enough worker threads complete
   * to get to the new maximum worker limit.  The caller is responsible for acquiring
   * a lock on the class mutex_.
   */
  void removeWorkersUnderLock(size_t value);

  size_t workerCount_;
  size_t workerMaxCount_;
  size_t idleCount_;
  size_t pendingTaskCountMax_;
  size_t expiredCount_;
  ExpireCallback expireCallback_;

  ThreadManager::STATE state_;
  shared_ptr<ThreadFactory> threadFactory_;

  friend class ThreadManager::Task;
  typedef std::deque<shared_ptr<Task> > TaskQueue;
  TaskQueue tasks_;
  Mutex mutex_;
  Monitor monitor_;
  Monitor maxMonitor_;
  Monitor workerMonitor_;       // used to synchronize changes in worker count

  friend class ThreadManager::Worker;
  std::set<shared_ptr<Thread> > workers_;
  std::set<shared_ptr<Thread> > deadWorkers_;
  std::map<const Thread::id_t, shared_ptr<Thread> > idMap_;
};

这个类就是管理类，有一个用于创建线程的线程工厂成员 threadFactory_，友元类是ThreadManager::Task 和 ThreadManager::Worker，有一个task队列成员 tasks_，一个监控task队列是否超出最大限制的条件maxMonitor_，一个监控worker线程数是否超最大限制的条件workerMonitor_，worker线程集合workers_，要终止的worker线程集合 deadWorkers_。

 

线程池启动

void ThreadManager::Impl::start() {
  Guard g(mutex_);
  if (state_ == ThreadManager::STOPPED) { // 已经终止
    return;
  }

  if (state_ == ThreadManager::UNINITIALIZED) { // 未初始化
    if (!threadFactory_) { // 线程工厂必须已初始化以创建线程
      throw InvalidArgumentException();
    }
    state_ = ThreadManager::STARTED;
    monitor_.notifyAll(); // 通知相关线程
  }

  while (state_ == STARTING) {
    monitor_.wait();
  }
}

SimpleThreadManager::start

  void start() override {
    ThreadManager::Impl::pendingTaskCountMax(pendingTaskCountMax_);
    ThreadManager::Impl::start();
    addWorker(workerCount_);
  }

addworker 添加了worker线程

void ThreadManager::Impl::addWorker(size_t value) {
  std::set<shared_ptr<Thread> > newThreads; //线程集合
  for (size_t ix = 0; ix < value; ix++) { // value 由manager 构造函数传入
    shared_ptr<ThreadManager::Worker> worker
        = std::make_shared<ThreadManager::Worker>(this); //worker 构造函数需要一个manager作为入参
    newThreads.insert(threadFactory_->newThread(worker)); // 工厂类创建线程
  }

  Guard g(mutex_);
  workerMaxCount_ += value;
  workers_.insert(newThreads.begin(), newThreads.end());

  for (const auto & newThread : newThreads) {
    shared_ptr<ThreadManager::Worker> worker
        = dynamic_pointer_cast<ThreadManager::Worker, Runnable>(newThread->runnable());
    worker->state_ = ThreadManager::Worker::STARTING;
    newThread->start(); // 启动线程
    idMap_.insert(std::pair<const Thread::id_t, shared_ptr<Thread> >(newThread->getId(), newThread));
  }

  while (workerCount_ != workerMaxCount_) { // 保证同一时刻不超出最大worker线程数，才可以开始跑
    workerMonitor_.wait();
  }
}

线程启动start

  /**
   * 启动线程，触发绑定到此线程的runnable对象的run方法
   */
  void start() {
    if (getState() != uninitialized) {
      return;
    }

    std::shared_ptr<Thread> selfRef = shared_from_this();
    setState(starting);

    Synchronized sync(monitor_);
    thread_ = std::unique_ptr<std::thread>(new std::thread(threadMain, selfRef)); // 

    if (detached_)
      thread_->detach();
    
    // Wait for the thread to start and get far enough to grab everything
    // that it needs from the calling context, thus absolving the caller
    // from being required to hold on to runnable indefinitely.
    monitor_.wait();
  }

这里 线程绑定的是 threadMain 函数

void Thread::threadMain(std::shared_ptr<Thread> thread) {
  thread->setState(started);
  thread->runnable()->run();

  if (thread->getState() != stopping && thread->getState() != stopped) {
    thread->setState(stopping);
  }
}

调用了runnable 的run 函数，也就是开头worker对象的run，从task队列取task执行了。

以上就是thrift 的线程池管理类的启动流程，可以大致画个图：

其实线程池的实现基本是一个task 队列 + 多个worker线程，还有一些控制并发的对象例如锁或者条件变量等等。