brpc：RemoteTaskQueue

存储非worker线程所创建的 bthreads 的队列（非worker指的是没有建立TaskGroup的线程），非worker线程会随机选择一个 TaskGroup 来推送任务，即将该Task推送至每个TaskGroup的RemoteTaskQueue队列中，由于这里的竞争较小，所以采用的是使用锁来保证线程的安全性。

butil::BoundedQueue

首先来看一下brpc封装的有界队列，这是一个定长数组

首先看私有域中的属性和方法

private

private:
    // Since the space is possibly not owned, we disable copying.
    DISALLOW_COPY_AND_ASSIGN(BoundedQueue); 
    
    // 该场景中绝大多数情况下offset都比 cap容量要小，所以不使用 % 操作符
    static uint32_t _mod(uint32_t off, uint32_t cap) {
        while (off >= cap) {
            off -= cap;
        }
        return off;
    }
    
    uint32_t _count; // 当前已经存储的task数量
    uint32_t _cap; // 容量
    uint32_t _start; // 
    StorageOwnership _ownership; // 枚举类型，判断队列所有权
    void* _items; // 指向队列指针

对外表现为一块void*的内存，并且不可拷贝。

构造函数、析构函数

    // You have to pass the memory for storing items at creation.
    // The queue contains at most memsize/sizeof(T) items.
    BoundedQueue(void* mem, size_t memsize, StorageOwnership ownership)
        : _count(0)
        , _cap(memsize / sizeof(T))
        , _start(0)
        , _ownership(ownership)
        , _items(mem) {
        DCHECK(_items);
    };
    
    // Construct a queue with the given capacity.
    // The malloc() may fail silently, call initialized() to test validity
    // of the queue.
    explicit BoundedQueue(size_t capacity)
        : _count(0)
        , _cap(capacity)
        , _start(0)
        , _ownership(OWNS_STORAGE)
        , _items(malloc(capacity * sizeof(T))) {
        DCHECK(_items);
    };
    
    BoundedQueue()
        : _count(0)
        , _cap(0)
        , _start(0)
        , _ownership(NOT_OWN_STORAGE)
        , _items(NULL) {
    };

    ~BoundedQueue() {
        clear();
        if (_ownership == OWNS_STORAGE) {
            free(_items);
            _items = NULL;
        }
    }

构造函数有三种重载，OWNS_STORAGE 标志着该内存交由BoundedQueue管理，即内存是堆上分配，意味着析构时要手动回收。

Push

/ Push |item| into bottom side of this queue.
    // Returns true on success, false if queue is full.
    bool push(const T& item) {
        if (_count < _cap) {
            new ((T*)_items + _mod(_start + _count, _cap)) T(item); // placement new 
            ++_count;
            return true;
        }
        return false;
    }

new (pointer) T(item); 使用placement new 在指定的内存位置(T*)_items + _mod(…) 上计算得到的地址，然后动态构造一个 T 类型的新对象，并将其初始化为 item 的副本。这是一种特殊形式的 new 运算符，允许在已分配的内存空间上直接构造对象，而不是分配新的内存。

    void elim_push(const T& item) {
        if (_count < _cap) {
            new ((T*)_items + _mod(_start + _count, _cap)) T(item);
            ++_count;
        } else {
            ((T*)_items)[_start] = item;
            _start = _mod(_start + 1, _cap);
        }
    }

elim_push遵从FIFO特性，如果队列已满，则pop掉最早push进来的那个对象（也就是_start指向的对象），并将item放进队尾。

// Push a default-constructed item into bottom side of this queue
    // Returns address of the item inside this queue
    T* push() {
        if (_count < _cap) {
            return new ((T*)_items + _mod(_start + _count++, _cap)) T();
        }
        return NULL;
    }

T* push() push一个T类型的默认构造的对象，并返回其地址，如果空间不够，返回的是空指针

    // Push |item| into top side of this queue
    // Returns true on success, false if queue is full.
    bool push_top(const T& item) {
        if (_count < _cap) {
            _start = _start ? (_start - 1) : (_cap - 1);
            ++_count;
            new ((T*)_items + _start) T(item);
            return true;
        }
        return false;
    }    
// Push a default-constructed item into top side of this queue
    // Returns address of the item inside this queue
    T* push_top() {
        if (_count < _cap) {
            _start = _start ? (_start - 1) : (_cap - 1);
            ++_count;
            return new ((T*)_items + _start) T();
        }
        return NULL;
    }

push_top，即在队列头插入元素。

// Pop top-most item from this queue
    // Returns true on success, false if queue is empty
    bool pop() {
        if (_count) {
            --_count;
            ((T*)_items + _start)->~T();
            _start = _mod(_start + 1, _cap);
            return true;
        }
        return false;
    }
 // Pop top-most item from this queue and copy into |item|.
    // Returns true on success, false if queue is empty
    bool pop(T* item) {
        if (_count) {
            --_count;
            T* const p = (T*)_items + _start;
            *item = *p;
            p->~T();
            _start = _mod(_start + 1, _cap);
            return true;
        }
        return false;
    }

pop，是从队列头部pop一个对象。

// Pop bottom-most item from this queue
    // Returns true on success, false if queue is empty
    bool pop_bottom() {
        if (_count) {
            --_count;
            ((T*)_items + _mod(_start + _count, _cap))->~T();
            return true;
        }
        return false;
    }

    // Pop bottom-most item from this queue and copy into |item|.
    // Returns true on success, false if queue is empty
    bool pop_bottom(T* item) {
        if (_count) {
            --_count;
            T* const p = (T*)_items + _mod(_start + _count, _cap);
            *item = *p;
            p->~T();
            return true;
        }
        return false;
    }

pop_bottom从队尾pop一个对象

完整代码

enum StorageOwnership { OWNS_STORAGE, NOT_OWN_STORAGE };
template <typename T>
class BoundedQueue {
public:
    // You have to pass the memory for storing items at creation.
    // The queue contains at most memsize/sizeof(T) items.
    BoundedQueue(void* mem, size_t memsize, StorageOwnership ownership)
        : _count(0)
        , _cap(memsize / sizeof(T))
        , _start(0)
        , _ownership(ownership)
        , _items(mem) {
        DCHECK(_items);
    };
    
    // Construct a queue with the given capacity.
    // The malloc() may fail silently, call initialized() to test validity
    // of the queue.
    explicit BoundedQueue(size_t capacity)
        : _count(0)
        , _cap(capacity)
        , _start(0)
        , _ownership(OWNS_STORAGE)
        , _items(malloc(capacity * sizeof(T))) {
        DCHECK(_items);
    };
    
    BoundedQueue()
        : _count(0)
        , _cap(0)
        , _start(0)
        , _ownership(NOT_OWN_STORAGE)
        , _items(NULL) {
    };

    ~BoundedQueue() {
        clear();
        if (_ownership == OWNS_STORAGE) {
            free(_items);
            _items = NULL;
        }
    }

    // Push |item| into bottom side of this queue.
    // Returns true on success, false if queue is full.
    bool push(const T& item) {
        if (_count < _cap) {
            new ((T*)_items + _mod(_start + _count, _cap)) T(item);
            ++_count;
            return true;
        }
        return false;
    }

    // Push |item| into bottom side of this queue. If the queue is full,
    // pop topmost item first.
    void elim_push(const T& item) {
        if (_count < _cap) {
            new ((T*)_items + _mod(_start + _count, _cap)) T(item);
            ++_count;
        } else {
            ((T*)_items)[_start] = item;
            _start = _mod(_start + 1, _cap);
        }
    }
    
    // Push a default-constructed item into bottom side of this queue
    // Returns address of the item inside this queue
    T* push() {
        if (_count < _cap) {
            return new ((T*)_items + _mod(_start + _count++, _cap)) T();
        }
        return NULL;
    }

    // Push |item| into top side of this queue
    // Returns true on success, false if queue is full.
    bool push_top(const T& item) {
        if (_count < _cap) {
            _start = _start ? (_start - 1) : (_cap - 1);
            ++_count;
            new ((T*)_items + _start) T(item);
            return true;
        }
        return false;
    }    
    
    // Push a default-constructed item into top side of this queue
    // Returns address of the item inside this queue
    T* push_top() {
        if (_count < _cap) {
            _start = _start ? (_start - 1) : (_cap - 1);
            ++_count;
            return new ((T*)_items + _start) T();
        }
        return NULL;
    }
    
    // Pop top-most item from this queue
    // Returns true on success, false if queue is empty
    bool pop() {
        if (_count) {
            --_count;
            ((T*)_items + _start)->~T();
            _start = _mod(_start + 1, _cap);
            return true;
        }
        return false;
    }

    // Pop top-most item from this queue and copy into |item|.
    // Returns true on success, false if queue is empty
    bool pop(T* item) {
        if (_count) {
            --_count;
            T* const p = (T*)_items + _start;
            *item = *p;
            p->~T();
            _start = _mod(_start + 1, _cap);
            return true;
        }
        return false;
    }

    // Pop bottom-most item from this queue
    // Returns true on success, false if queue is empty
    bool pop_bottom() {
        if (_count) {
            --_count;
            ((T*)_items + _mod(_start + _count, _cap))->~T();
            return true;
        }
        return false;
    }

    // Pop bottom-most item from this queue and copy into |item|.
    // Returns true on success, false if queue is empty
    bool pop_bottom(T* item) {
        if (_count) {
            --_count;
            T* const p = (T*)_items + _mod(_start + _count, _cap);
            *item = *p;
            p->~T();
            return true;
        }
        return false;
    }

    // Pop all items
    void clear() {
        for (uint32_t i = 0; i < _count; ++i) {
            ((T*)_items + _mod(_start + i, _cap))->~T();
        }
        _count = 0;
        _start = 0;
    }

    // Get address of top-most item, NULL if queue is empty
    T* top() { 
        return _count ? ((T*)_items + _start) : NULL; 
    }
    const T* top() const { 
        return _count ? ((const T*)_items + _start) : NULL; 
    }

    // Randomly access item from top side.
    // top(0) == top(), top(size()-1) == bottom()
    // Returns NULL if |index| is out of range.
    T* top(size_t index) {
        if (index < _count) {
            return (T*)_items + _mod(_start + index, _cap);
        }
        return NULL;   // including _count == 0
    }
    const T* top(size_t index) const {
        if (index < _count) {
            return (const T*)_items + _mod(_start + index, _cap);
        }
        return NULL;   // including _count == 0
    }

    // Get address of bottom-most item, NULL if queue is empty
    T* bottom() { 
        return _count ? ((T*)_items + _mod(_start + _count - 1, _cap)) : NULL; 
    }
    const T* bottom() const {
        return _count ? ((const T*)_items + _mod(_start + _count - 1, _cap)) : NULL; 
    }
    
    // Randomly access item from bottom side.
    // bottom(0) == bottom(), bottom(size()-1) == top()
    // Returns NULL if |index| is out of range.
    T* bottom(size_t index) {
        if (index < _count) {
            return (T*)_items + _mod(_start + _count - index - 1, _cap);
        }
        return NULL;  // including _count == 0
    }
    const T* bottom(size_t index) const {
        if (index < _count) {
            return (const T*)_items + _mod(_start + _count - index - 1, _cap);
        }
        return NULL;  // including _count == 0
    }

    bool empty() const { return !_count; }
    bool full() const { return _cap == _count; }

    // Number of items
    size_t size() const { return _count; }

    // Maximum number of items that can be in this queue
    size_t capacity() const { return _cap; }

    // Maximum value of capacity()
    size_t max_capacity() const { return (1UL << (sizeof(_cap) * 8)) - 1; }

    // True if the queue was constructed successfully.
    bool initialized() const { return _items != NULL; }

    // Swap internal fields with another queue.
    void swap(BoundedQueue& rhs) {
        std::swap(_count, rhs._count);
        std::swap(_cap, rhs._cap);
        std::swap(_start, rhs._start);
        std::swap(_ownership, rhs._ownership);
        std::swap(_items, rhs._items);
    }

private:
    // Since the space is possibly not owned, we disable copying.
    DISALLOW_COPY_AND_ASSIGN(BoundedQueue);
    
    // This is faster than % in this queue because most |off| are smaller
    // than |cap|. This is probably not true in other place, be careful
    // before you use this trick.
    static uint32_t _mod(uint32_t off, uint32_t cap) {
        while (off >= cap) {
            off -= cap;
        }
        return off;
    }
    
    uint32_t _count;
    uint32_t _cap;
    uint32_t _start;
    StorageOwnership _ownership;
    void* _items;
};

简单使用

#include "butil/containers/bounded_queue.h"
int main() {
    // 创建一个栈上队列
    char storage[64];
    butil::BoundedQueue<int> q(storage, sizeof(storage), butil::NOT_OWN_STORAGE);
    q.push(1);
    q.push(2);
    LOG(INFO) << "bottom = " << *q.bottom()<< " top = " << *q.top();
    q.push_top(5);
    LOG(INFO) << "bottom = " << *q.bottom()<< " top = " << *q.top();
    LOG(INFO) << "top(1) = " << *q.top(1) << " bottom(0) = " << *q.bottom(0)<< "\n";


    butil::BoundedQueue<int> q2(10); // 堆上队列
    q2.push(3);
    q2.push(4);
    LOG(INFO) << "bottom = " << *q2.bottom()<< " top = " << *q2.top();
    q2.push_top(5);
    LOG(INFO) << "bottom = " << *q2.bottom()<< " top = " << *q2.top();
    LOG(INFO) << "top(1) = " << *q2.top(1) << " bottom(2) = " << *q2.bottom(0);


}

RemoteTaskQueue

RemoteTaskQueue 实际上就是上面BoundedQueue的封装，并增加了线程安全保证。

friend class TaskGroup;
DISALLOW_COPY_AND_ASSIGN(RemoteTaskQueue);
butil::BoundedQueue<bthread_t> _tasks;
butil::Mutex _mutex;

其中包含两个类属性，存放bthread_t类型的队列_tasks（bthread_t是uint64_t的别名），另一个就是一个互斥锁（对系统级mutex的封装）

int init(size_t cap) {
        const size_t memsize = sizeof(bthread_t) * cap;
        void* q_mem = malloc(memsize);
        if (q_mem == NULL) {
            return -1;
        }
        butil::BoundedQueue<bthread_t> q(q_mem, memsize, butil::OWNS_STORAGE);
        _tasks.swap(q);
        return 0;
    }

int init(size_t cap) 从堆上创建该BoundedQueue队列，意味着需要类自己管理其生命周期（BoundedQueue的析构时回收内存）

    bool pop(bthread_t* task) {
        if (_tasks.empty()) {
            return false;
        }
        _mutex.lock();
        const bool result = _tasks.pop(task);
        _mutex.unlock();
        return result;
    }

    bool push(bthread_t task) {
        _mutex.lock();
        const bool res = push_locked(task);
        _mutex.unlock();
        return res;
    }

    bool push_locked(bthread_t task) {
        return _tasks.push(task);
    }

    size_t capacity() const { return _tasks.capacity(); }

然后就是一些基础的push pop操作，并使用互斥锁保证插入删除的线程安全性。

RemoteTaskQueue完整代码

#ifndef BTHREAD_REMOTE_TASK_QUEUE_H
#define BTHREAD_REMOTE_TASK_QUEUE_H

#include "butil/containers/bounded_queue.h"
#include "butil/macros.h"
#include <bthread/types.h>

namespace bthread {

class TaskGroup;

// A queue for storing bthreads created by non-workers. Since non-workers
// randomly choose a TaskGroup to push which distributes the contentions,
// this queue is simply implemented as a queue protected with a lock.
// The function names should be self-explanatory.
class RemoteTaskQueue {
public:
    RemoteTaskQueue() {}

    int init(size_t cap) {
        const size_t memsize = sizeof(bthread_t) * cap;
        void* q_mem = malloc(memsize);
        if (q_mem == NULL) {
            return -1;
        }
        butil::BoundedQueue<bthread_t> q(q_mem, memsize, butil::OWNS_STORAGE);
        _tasks.swap(q);
        return 0;
    }

    bool pop(bthread_t* task) {
        if (_tasks.empty()) {
            return false;
        }
        _mutex.lock();
        const bool result = _tasks.pop(task);
        _mutex.unlock();
        return result;
    }

    bool push(bthread_t task) {
        _mutex.lock();
        const bool res = push_locked(task);
        _mutex.unlock();
        return res;
    }

    bool push_locked(bthread_t task) {
        return _tasks.push(task);
    }

    size_t capacity() const { return _tasks.capacity(); }
    
private:
friend class TaskGroup;
    DISALLOW_COPY_AND_ASSIGN(RemoteTaskQueue);
    butil::BoundedQueue<bthread_t> _tasks;
    butil::Mutex _mutex;
};

}  // namespace bthread

#endif  // BTHREAD_REMOTE_TASK_QUEUE_H