阅读本文之前,推荐先阅读上一篇对topic的分析。
什么是channel
结合上一篇对topic分析,channel的作用就是将topic的数据进行分流,topic会将发布的消息分配给所有连接的channel,而channel只会将消息发送给一个consumer,这样有利于下游的均衡负载。
源码分析
数据结构
// Channel represents the concrete type for a NSQ channel (and also
// implements the Queue interface)
//
// There can be multiple channels per topic, each with there own unique set
// of subscribers (clients).
//
// Channels maintain all client and message metadata, orchestrating in-flight
// messages, timeouts, requeuing, etc.
type Channel struct {
// 64bit atomic vars need to be first for proper alignment on 32bit platforms
// 队列和消息计数
requeueCount uint64
messageCount uint64
timeoutCount uint64
// 实例启用读写锁
sync.RWMutex
// 所属topic 名称 所属nsqd
topicName string
name string
nsqd *NSQD
// 消息存储队列
backend BackendQueue
// 消息缓存队列通道
memoryMsgChan chan *Message
exitFlag int32
exitMutex sync.RWMutex
// state tracking
// 订阅的消费者
clients map[int64]Consumer
// 状态标志
paused int32
ephemeral bool
// 删除回调函数
deleteCallback func(*Channel)
deleter sync.Once
// Stats tracking
e2eProcessingLatencyStream *quantile.Quantile
// TODO: these can be DRYd up
deferredMessages map[MessageID]*pqueue.Item
deferredPQ pqueue.PriorityQueue
deferredMutex sync.Mutex
inFlightMessages map[MessageID]*Message
inFlightPQ inFlightPqueue
inFlightMutex sync.Mutex
}
实例化
// NewChannel creates a new instance of the Channel type and returns a pointer
func NewChannel(topicName string, channelName string, nsqd *NSQD,
deleteCallback func(*Channel)) *Channel {
// 构造channel实例
c := &Channel{
topicName: topicName,
name: channelName,
memoryMsgChan: nil,
clients: make(map[int64]Consumer),
deleteCallback: deleteCallback,
nsqd: nsqd,
}
// create mem-queue only if size > 0 (do not use unbuffered chan)
// 创建一个基于内存的消息队列通道,用于外部topic将消息存到这个缓存通道
if nsqd.getOpts().MemQueueSize > 0 {
c.memoryMsgChan = make(chan *Message, nsqd.getOpts().MemQueueSize)
}
// 创建一个流存储端到端的时延
if len(nsqd.getOpts().E2EProcessingLatencyPercentiles) > 0 {
c.e2eProcessingLatencyStream = quantile.New(
nsqd.getOpts().E2EProcessingLatencyWindowTime,
nsqd.getOpts().E2EProcessingLatencyPercentiles,
)
}
// 初始化inFlightPQ和deferredPQ两个队列
c.initPQ()
if strings.HasSuffix(channelName, "#ephemeral") {
// 如果channel名称以"#ephemeral"结尾,就将其配置为临时channel
c.ephemeral = true
// 将消息存储在内存中
c.backend = newDummyBackendQueue()
} else {
// 普通channel,将消息存储在硬盘文件中
dqLogf := func(level diskqueue.LogLevel, f string, args ...interface{}) {
opts := nsqd.getOpts()
lg.Logf(opts.Logger, opts.LogLevel, lg.LogLevel(level), f, args...)
}
// backend names, for uniqueness, automatically include the topic...
backendName := getBackendName(topicName, channelName)
c.backend = diskqueue.New(
backendName,
nsqd.getOpts().DataPath,
nsqd.getOpts().MaxBytesPerFile,
int32(minValidMsgLength),
int32(nsqd.getOpts().MaxMsgSize)+minValidMsgLength,
nsqd.getOpts().SyncEvery,
nsqd.getOpts().SyncTimeout,
dqLogf,
)
}
c.nsqd.Notify(c, !c.ephemeral)
return c
}
消息处理
channel接收消息的操作函数如下。
// PutMessage writes a Message to the queue
// 向channel的缓存里存入一条消息
func (c *Channel) PutMessage(m *Message) error {
// 写操作线程安全
c.exitMutex.RLock()
defer c.exitMutex.RUnlock()
if c.Exiting() {
return errors.New("exiting")
}
err := c.put(m)
if err != nil {
return err
}
// 增加当前缓存中消息计数
atomic.AddUint64(&c.messageCount, 1)
return nil
}
func (c *Channel) put(m *Message) error {
select {
// 优先把消息存入内存缓存通道中
case c.memoryMsgChan <- m:
default:
// 内存通道满了,才写入硬盘中
err := writeMessageToBackend(m, c.backend)
c.nsqd.SetHealth(err)
if err != nil {
c.nsqd.logf(LOG_ERROR, "CHANNEL(%s): failed to write message to backend - %s",
c.name, err)
return err
}
}
return nil
}
// 存入时延消息
func (c *Channel) PutMessageDeferred(msg *Message, timeout time.Duration) {
atomic.AddUint64(&c.messageCount, 1)
c.StartDeferredTimeout(msg, timeout)
}
// TouchMessage resets the timeout for an in-flight message
// 重置那些在发送队列中得消息,拿出来再放回去
func (c *Channel) TouchMessage(clientID int64, id MessageID, clientMsgTimeout time.Duration) error {
// 取出发送给某个消费者的消息
msg, err := c.popInFlightMessage(clientID, id)
if err != nil {
return err
}
// 从发送队列中删除该消息
c.removeFromInFlightPQ(msg)
// 重置超时时间
newTimeout := time.Now().Add(clientMsgTimeout)
if newTimeout.Sub(msg.deliveryTS) >=
c.nsqd.getOpts().MaxMsgTimeout {
// we would have gone over, set to the max
newTimeout = msg.deliveryTS.Add(c.nsqd.getOpts().MaxMsgTimeout)
}
// 将消息又放了回去
msg.pri = newTimeout.UnixNano()
err = c.pushInFlightMessage(msg)
if err != nil {
return err
}
c.addToInFlightPQ(msg)
return nil
}
// FinishMessage successfully discards an in-flight message
// 当消息发送成功之后,删除发送缓存和队列中的消息
func (c *Channel) FinishMessage(clientID int64, id MessageID) error {
msg, err := c.popInFlightMessage(clientID, id)
if err != nil {
return err
}
c.removeFromInFlightPQ(msg)
if c.e2eProcessingLatencyStream != nil {
c.e2eProcessingLatencyStream.Insert(msg.Timestamp)
}
return nil
}
// RequeueMessage requeues a message based on `time.Duration`, ie:
//
// `timeoutMs` == 0 - requeue a message immediately
// `timeoutMs` > 0 - asynchronously wait for the specified timeout
// and requeue a message (aka "deferred requeue")
// 从发送缓存队列中拿取一条消息,然后更具超时时间进行压入发送队列
func (c *Channel) RequeueMessage(clientID int64, id MessageID, timeout time.Duration) error {
// remove from inflight first
msg, err := c.popInFlightMessage(clientID, id)
if err != nil {
return err
}
c.removeFromInFlightPQ(msg)
atomic.AddUint64(&c.requeueCount, 1)
// 如果超时时间为0 那么立刻压入当前缓存队列,不然放入延迟发送队列
if timeout == 0 {
c.exitMutex.RLock()
if c.Exiting() {
c.exitMutex.RUnlock()
return errors.New("exiting")
}
err := c.put(msg)
c.exitMutex.RUnlock()
return err
}
// deferred requeue
return c.StartDeferredTimeout(msg, timeout)
// pushInFlightMessage atomically adds a message to the in-flight dictionary
// 将一条信息放入消息发送字典当中
func (c *Channel) pushInFlightMessage(msg *Message) error {
c.inFlightMutex.Lock()
_, ok := c.inFlightMessages[msg.ID]
if ok {
c.inFlightMutex.Unlock()
return errors.New("ID already in flight")
}
c.inFlightMessages[msg.ID] = msg
c.inFlightMutex.Unlock()
return nil
}
// popInFlightMessage atomically removes a message from the in-flight dictionary
// 从消息发送字典中取出一条数据,并删除
func (c *Channel) popInFlightMessage(clientID int64, id MessageID) (*Message, error) {
c.inFlightMutex.Lock()
msg, ok := c.inFlightMessages[id]
if !ok {
c.inFlightMutex.Unlock()
return nil, errors.New("ID not in flight")
}
if msg.clientID != clientID {
c.inFlightMutex.Unlock()
return nil, errors.New("client does not own message")
}
delete(c.inFlightMessages, id)
c.inFlightMutex.Unlock()
return msg, nil
}
// 压入一条消息到发送队列中
func (c *Channel) addToInFlightPQ(msg *Message) {
c.inFlightMutex.Lock()
c.inFlightPQ.Push(msg)
c.inFlightMutex.Unlock()
}
// 从消息发送队列中删除一条消息
func (c *Channel) removeFromInFlightPQ(msg *Message) {
c.inFlightMutex.Lock()
if msg.index == -1 {
// this item has already been popped off the pqueue
c.inFlightMutex.Unlock()
return
}
c.inFlightPQ.Remove(msg.index)
c.inFlightMutex.Unlock()
}
// 压入一条消息到时延消息发送字典中
func (c *Channel) pushDeferredMessage(item *pqueue.Item) error {
c.deferredMutex.Lock()
// TODO: these map lookups are costly
id := item.Value.(*Message).ID
_, ok := c.deferredMessages[id]
if ok {
c.deferredMutex.Unlock()
return errors.New("ID already deferred")
}
c.deferredMessages[id] = item
c.deferredMutex.Unlock()
return nil
}
// 从时延消息发送字典中取出一条消息
func (c *Channel) popDeferredMessage(id MessageID) (*pqueue.Item, error) {
c.deferredMutex.Lock()
// TODO: these map lookups are costly
item, ok := c.deferredMessages[id]
if !ok {
c.deferredMutex.Unlock()
return nil, errors.New("ID not deferred")
}
delete(c.deferredMessages, id)
c.deferredMutex.Unlock()
return item, nil
}
// 根据传入时间,将正在发送队列中,所有发送超时失败的消息进行重新发送
func (c *Channel) processInFlightQueue(t int64) bool {
c.exitMutex.RLock()
defer c.exitMutex.RUnlock()
if c.Exiting() {
return false
}
dirty := false
for {
c.inFlightMutex.Lock()
// 取出一条超时的消息
msg, _ := c.inFlightPQ.PeekAndShift(t)
c.inFlightMutex.Unlock()
if msg == nil {
goto exit
}
// 用dirty表示是否存在超时失败的消息
dirty = true
_, err := c.popInFlightMessage(msg.clientID, msg.ID)
if err != nil {
goto exit
}
atomic.AddUint64(&c.timeoutCount, 1)
c.RLock()
// 获取该条超时消息的消费者
client, ok := c.clients[msg.clientID]
c.RUnlock()
if ok {
client.TimedOutMessage()
}
// 放入发送队列中,重新发送
c.put(msg)
}
exit:
return dirty
}
消费者操作
// AddClient adds a client to the Channel's client list
// 添加一个消费者到channel得订阅字典中
func (c *Channel) AddClient(clientID int64, client Consumer) error {
c.exitMutex.RLock()
defer c.exitMutex.RUnlock()
if c.Exiting() {
return errors.New("exiting")
}
c.RLock()
_, ok := c.clients[clientID]
numClients := len(c.clients)
c.RUnlock()
if ok {
return nil
}
// 最大消费者数量限制
maxChannelConsumers := c.nsqd.getOpts().MaxChannelConsumers
if maxChannelConsumers != 0 && numClients >= maxChannelConsumers {
return fmt.Errorf("consumers for %s:%s exceeds limit of %d",
c.topicName, c.name, maxChannelConsumers)
}
c.Lock()
c.clients[clientID] = client
c.Unlock()
return nil
}
// RemoveClient removes a client from the Channel's client list
// 删除一条消费者信息
func (c *Channel) RemoveClient(clientID int64) {
c.exitMutex.RLock()
defer c.exitMutex.RUnlock()
if c.Exiting() {
return
}
c.RLock()
_, ok := c.clients[clientID]
c.RUnlock()
if !ok {
return
}
c.Lock()
delete(c.clients, clientID)
c.Unlock()
if len(c.clients) == 0 && c.ephemeral == true {
go c.deleter.Do(func() { c.deleteCallback(c) })
}
}
总结
- channel对于需要发送的消息缓存设计和topic保持一致,做了两层缓存,上层是基于内存的缓存memoryMsgChan,利用golang的有缓冲通道实现,外部发布数据给topic时优先将数据放到memoryMsgChan,但是当消息积压太多,内存会耗尽,因此,还设计了基于硬盘的缓存backendChan,当memoryMsgChan满了的时候,就会将数据存储到backendChan,将消息转发channel的时候,优先从memoryMsgChan读取消息,然后从这样,backendChan,这样提高了数据读写效率的同时,又节约内存;如果为了进一步提高读写性能,并不考虑内存的使用情况,可以将Topic的名称以“#ephemeral”结尾,将Topic设置为临时Topic,那么backendChan同样是使用计算机内存,临时topic的通道数为0的时候,将会被删除。
- channel还存在两个消息缓存队列,分别为inFlightMessages和deferredMessages。其中,inFlightMessages存放的是已经发送出去但不知道有没有发送成功的消息,当确认发送成功之后,调用FinishMessage删除inFlightMessages中发送成功消息,不然,当出现消息发送超时的情况,又会调用processInFlightQueue函数将inFlightMessages中的超时消息重新添加到memoryMsgChan发送队列中,再次发送;其中,deferredMessages用以消息的延迟发送,当消息的配置信息中存在deferred不为0的情况,topic就会调用PutMessageDeferred函数将消息放入时延发送队列中,当时延满足条件,channel就会像消息放到memoryMsgChan中等待发送。
- 真正的消息投递给消费者的操作,是由外部的protocol模块完成的,它调用channel,从memoryMsgChan或者backendChan取出消息,根据协议要求发送给消费者。