NSQ源码分析之channel

阅读本文之前，推荐先阅读上一篇对topic的分析。

什么是channel

结合上一篇对topic分析，channel的作用就是将topic的数据进行分流，topic会将发布的消息分配给所有连接的channel，而channel只会将消息发送给一个consumer，这样有利于下游的均衡负载。

源码分析

数据结构

// Channel represents the concrete type for a NSQ channel (and also
// implements the Queue interface)
//
// There can be multiple channels per topic, each with there own unique set
// of subscribers (clients).
//
// Channels maintain all client and message metadata, orchestrating in-flight
// messages, timeouts, requeuing, etc.
type Channel struct {
	// 64bit atomic vars need to be first for proper alignment on 32bit platforms
	// 队列和消息计数
	requeueCount uint64
	messageCount uint64
	timeoutCount uint64
	// 实例启用读写锁
	sync.RWMutex
    // 所属topic 名称 所属nsqd
	topicName string
	name      string
	nsqd      *NSQD
    // 消息存储队列
	backend BackendQueue
    // 消息缓存队列通道
	memoryMsgChan chan *Message
	exitFlag      int32
	exitMutex     sync.RWMutex

	// state tracking
	// 订阅的消费者
	clients        map[int64]Consumer
	// 状态标志
	paused         int32
	ephemeral      bool
	// 删除回调函数
	deleteCallback func(*Channel)
	deleter        sync.Once

	// Stats tracking
	e2eProcessingLatencyStream *quantile.Quantile

	// TODO: these can be DRYd up
	deferredMessages map[MessageID]*pqueue.Item
	deferredPQ       pqueue.PriorityQueue
	deferredMutex    sync.Mutex
	inFlightMessages map[MessageID]*Message
	inFlightPQ       inFlightPqueue
	inFlightMutex    sync.Mutex
}

实例化

// NewChannel creates a new instance of the Channel type and returns a pointer
func NewChannel(topicName string, channelName string, nsqd *NSQD,
	deleteCallback func(*Channel)) *Channel {
    // 构造channel实例
	c := &Channel{
		topicName:      topicName,
		name:           channelName,
		memoryMsgChan:  nil,
		clients:        make(map[int64]Consumer),
		deleteCallback: deleteCallback,
		nsqd:           nsqd,
	}
	// create mem-queue only if size > 0 (do not use unbuffered chan)
	// 创建一个基于内存的消息队列通道，用于外部topic将消息存到这个缓存通道
	if nsqd.getOpts().MemQueueSize > 0 {
		c.memoryMsgChan = make(chan *Message, nsqd.getOpts().MemQueueSize)
	}
	// 创建一个流存储端到端的时延
	if len(nsqd.getOpts().E2EProcessingLatencyPercentiles) > 0 {
		c.e2eProcessingLatencyStream = quantile.New(
			nsqd.getOpts().E2EProcessingLatencyWindowTime,
			nsqd.getOpts().E2EProcessingLatencyPercentiles,
		)
	}
    // 初始化inFlightPQ和deferredPQ两个队列
	c.initPQ()
    
	if strings.HasSuffix(channelName, "#ephemeral") {
	    // 如果channel名称以"#ephemeral"结尾，就将其配置为临时channel
		c.ephemeral = true
		// 将消息存储在内存中
		c.backend = newDummyBackendQueue()
	} else {
	    // 普通channel，将消息存储在硬盘文件中
		dqLogf := func(level diskqueue.LogLevel, f string, args ...interface{}) {
			opts := nsqd.getOpts()
			lg.Logf(opts.Logger, opts.LogLevel, lg.LogLevel(level), f, args...)
		}
		// backend names, for uniqueness, automatically include the topic...
		backendName := getBackendName(topicName, channelName)
		c.backend = diskqueue.New(
			backendName,
			nsqd.getOpts().DataPath,
			nsqd.getOpts().MaxBytesPerFile,
			int32(minValidMsgLength),
			int32(nsqd.getOpts().MaxMsgSize)+minValidMsgLength,
			nsqd.getOpts().SyncEvery,
			nsqd.getOpts().SyncTimeout,
			dqLogf,
		)
	}

	c.nsqd.Notify(c, !c.ephemeral)

	return c
}

消息处理

channel接收消息的操作函数如下。

// PutMessage writes a Message to the queue
// 向channel的缓存里存入一条消息
func (c *Channel) PutMessage(m *Message) error {
    // 写操作线程安全
	c.exitMutex.RLock()
	defer c.exitMutex.RUnlock()
	if c.Exiting() {
		return errors.New("exiting")
	}
	err := c.put(m)
	if err != nil {
		return err
	}
	// 增加当前缓存中消息计数
	atomic.AddUint64(&c.messageCount, 1)
	return nil
}

func (c *Channel) put(m *Message) error {
	select {
	// 优先把消息存入内存缓存通道中
	case c.memoryMsgChan <- m:
	default:
	    // 内存通道满了，才写入硬盘中
		err := writeMessageToBackend(m, c.backend)
		c.nsqd.SetHealth(err)
		if err != nil {
			c.nsqd.logf(LOG_ERROR, "CHANNEL(%s): failed to write message to backend - %s",
				c.name, err)
			return err
		}
	}
	return nil
}

// 存入时延消息
func (c *Channel) PutMessageDeferred(msg *Message, timeout time.Duration) {
	atomic.AddUint64(&c.messageCount, 1)
	c.StartDeferredTimeout(msg, timeout)
}

// TouchMessage resets the timeout for an in-flight message
// 重置那些在发送队列中得消息，拿出来再放回去
func (c *Channel) TouchMessage(clientID int64, id MessageID, clientMsgTimeout time.Duration) error {
    // 取出发送给某个消费者的消息
	msg, err := c.popInFlightMessage(clientID, id)
	if err != nil {
		return err
	}
	// 从发送队列中删除该消息
	c.removeFromInFlightPQ(msg)
	// 重置超时时间
	newTimeout := time.Now().Add(clientMsgTimeout)
	if newTimeout.Sub(msg.deliveryTS) >=
		c.nsqd.getOpts().MaxMsgTimeout {
		// we would have gone over, set to the max
		newTimeout = msg.deliveryTS.Add(c.nsqd.getOpts().MaxMsgTimeout)
	}
	// 将消息又放了回去
	msg.pri = newTimeout.UnixNano()
	err = c.pushInFlightMessage(msg)
	if err != nil {
		return err
	}
	c.addToInFlightPQ(msg)
	return nil
}

// FinishMessage successfully discards an in-flight message
// 当消息发送成功之后，删除发送缓存和队列中的消息
func (c *Channel) FinishMessage(clientID int64, id MessageID) error {
	msg, err := c.popInFlightMessage(clientID, id)
	if err != nil {
		return err
	}
	c.removeFromInFlightPQ(msg)
	if c.e2eProcessingLatencyStream != nil {
		c.e2eProcessingLatencyStream.Insert(msg.Timestamp)
	}
	return nil
}

// RequeueMessage requeues a message based on `time.Duration`, ie:
//
// `timeoutMs` == 0 - requeue a message immediately
// `timeoutMs`  > 0 - asynchronously wait for the specified timeout
//     and requeue a message (aka "deferred requeue")
// 从发送缓存队列中拿取一条消息，然后更具超时时间进行压入发送队列
func (c *Channel) RequeueMessage(clientID int64, id MessageID, timeout time.Duration) error {
	// remove from inflight first
	msg, err := c.popInFlightMessage(clientID, id)
	if err != nil {
		return err
	}
	c.removeFromInFlightPQ(msg)
	atomic.AddUint64(&c.requeueCount, 1)

	// 如果超时时间为0 那么立刻压入当前缓存队列，不然放入延迟发送队列
	if timeout == 0 {
		c.exitMutex.RLock()
		if c.Exiting() {
			c.exitMutex.RUnlock()
			return errors.New("exiting")
		}
		err := c.put(msg)
		c.exitMutex.RUnlock()
		return err
	}

	// deferred requeue
	return c.StartDeferredTimeout(msg, timeout)

// pushInFlightMessage atomically adds a message to the in-flight dictionary
// 将一条信息放入消息发送字典当中
func (c *Channel) pushInFlightMessage(msg *Message) error {
	c.inFlightMutex.Lock()
	_, ok := c.inFlightMessages[msg.ID]
	if ok {
		c.inFlightMutex.Unlock()
		return errors.New("ID already in flight")
	}
	c.inFlightMessages[msg.ID] = msg
	c.inFlightMutex.Unlock()
	return nil
}

// popInFlightMessage atomically removes a message from the in-flight dictionary
// 从消息发送字典中取出一条数据，并删除
func (c *Channel) popInFlightMessage(clientID int64, id MessageID) (*Message, error) {
	c.inFlightMutex.Lock()
	msg, ok := c.inFlightMessages[id]
	if !ok {
		c.inFlightMutex.Unlock()
		return nil, errors.New("ID not in flight")
	}
	if msg.clientID != clientID {
		c.inFlightMutex.Unlock()
		return nil, errors.New("client does not own message")
	}
	delete(c.inFlightMessages, id)
	c.inFlightMutex.Unlock()
	return msg, nil
}

// 压入一条消息到发送队列中
func (c *Channel) addToInFlightPQ(msg *Message) {
	c.inFlightMutex.Lock()
	c.inFlightPQ.Push(msg)
	c.inFlightMutex.Unlock()
}

// 从消息发送队列中删除一条消息
func (c *Channel) removeFromInFlightPQ(msg *Message) {
	c.inFlightMutex.Lock()
	if msg.index == -1 {
		// this item has already been popped off the pqueue
		c.inFlightMutex.Unlock()
		return
	}
	c.inFlightPQ.Remove(msg.index)
	c.inFlightMutex.Unlock()
}
// 压入一条消息到时延消息发送字典中
func (c *Channel) pushDeferredMessage(item *pqueue.Item) error {
	c.deferredMutex.Lock()
	// TODO: these map lookups are costly
	id := item.Value.(*Message).ID
	_, ok := c.deferredMessages[id]
	if ok {
		c.deferredMutex.Unlock()
		return errors.New("ID already deferred")
	}
	c.deferredMessages[id] = item
	c.deferredMutex.Unlock()
	return nil
}

// 从时延消息发送字典中取出一条消息
func (c *Channel) popDeferredMessage(id MessageID) (*pqueue.Item, error) {
	c.deferredMutex.Lock()
	// TODO: these map lookups are costly
	item, ok := c.deferredMessages[id]
	if !ok {
		c.deferredMutex.Unlock()
		return nil, errors.New("ID not deferred")
	}
	delete(c.deferredMessages, id)
	c.deferredMutex.Unlock()
	return item, nil
}

// 根据传入时间，将正在发送队列中，所有发送超时失败的消息进行重新发送
func (c *Channel) processInFlightQueue(t int64) bool {
	c.exitMutex.RLock()
	defer c.exitMutex.RUnlock()

	if c.Exiting() {
		return false
	}

	dirty := false
	for {
		c.inFlightMutex.Lock()
		// 取出一条超时的消息
		msg, _ := c.inFlightPQ.PeekAndShift(t)
		c.inFlightMutex.Unlock()

		if msg == nil {
			goto exit
		}
		// 用dirty表示是否存在超时失败的消息
		dirty = true

		_, err := c.popInFlightMessage(msg.clientID, msg.ID)
		if err != nil {
			goto exit
		}
		atomic.AddUint64(&c.timeoutCount, 1)
		c.RLock()
		// 获取该条超时消息的消费者
		client, ok := c.clients[msg.clientID]
		c.RUnlock()
		if ok {
			client.TimedOutMessage()
		}
		// 放入发送队列中，重新发送
		c.put(msg)
	}

exit:
	return dirty
}

消费者操作

// AddClient adds a client to the Channel's client list
// 添加一个消费者到channel得订阅字典中
func (c *Channel) AddClient(clientID int64, client Consumer) error {
	c.exitMutex.RLock()
	defer c.exitMutex.RUnlock()

	if c.Exiting() {
		return errors.New("exiting")
	}

	c.RLock()
	_, ok := c.clients[clientID]
	numClients := len(c.clients)
	c.RUnlock()
	if ok {
		return nil
	}
    // 最大消费者数量限制
	maxChannelConsumers := c.nsqd.getOpts().MaxChannelConsumers
	if maxChannelConsumers != 0 && numClients >= maxChannelConsumers {
		return fmt.Errorf("consumers for %s:%s exceeds limit of %d",
			c.topicName, c.name, maxChannelConsumers)
	}

	c.Lock()
	c.clients[clientID] = client
	c.Unlock()
	return nil
}

// RemoveClient removes a client from the Channel's client list
// 删除一条消费者信息
func (c *Channel) RemoveClient(clientID int64) {
	c.exitMutex.RLock()
	defer c.exitMutex.RUnlock()

	if c.Exiting() {
		return
	}

	c.RLock()
	_, ok := c.clients[clientID]
	c.RUnlock()
	if !ok {
		return
	}

	c.Lock()
	delete(c.clients, clientID)
	c.Unlock()

	if len(c.clients) == 0 && c.ephemeral == true {
		go c.deleter.Do(func() { c.deleteCallback(c) })
	}
}

总结

1. channel对于需要发送的消息缓存设计和topic保持一致，做了两层缓存，上层是基于内存的缓存memoryMsgChan，利用golang的有缓冲通道实现，外部发布数据给topic时优先将数据放到memoryMsgChan，但是当消息积压太多，内存会耗尽，因此，还设计了基于硬盘的缓存backendChan，当memoryMsgChan满了的时候，就会将数据存储到backendChan，将消息转发channel的时候，优先从memoryMsgChan读取消息，然后从这样，backendChan，这样提高了数据读写效率的同时，又节约内存；如果为了进一步提高读写性能，并不考虑内存的使用情况，可以将Topic的名称以“#ephemeral”结尾，将Topic设置为临时Topic，那么backendChan同样是使用计算机内存，临时topic的通道数为0的时候，将会被删除。
2. channel还存在两个消息缓存队列，分别为inFlightMessages和deferredMessages。其中，inFlightMessages存放的是已经发送出去但不知道有没有发送成功的消息，当确认发送成功之后，调用FinishMessage删除inFlightMessages中发送成功消息，不然，当出现消息发送超时的情况，又会调用processInFlightQueue函数将inFlightMessages中的超时消息重新添加到memoryMsgChan发送队列中，再次发送；其中，deferredMessages用以消息的延迟发送，当消息的配置信息中存在deferred不为0的情况，topic就会调用PutMessageDeferred函数将消息放入时延发送队列中，当时延满足条件，channel就会像消息放到memoryMsgChan中等待发送。
3. 真正的消息投递给消费者的操作，是由外部的protocol模块完成的，它调用channel，从memoryMsgChan或者backendChan取出消息，根据协议要求发送给消费者。