NSQ 源码分析之NSQD 的启动

1.NSQD的配置(nsqd/options.go)包括如下参数：

type Options struct {
	/*******************基本信息配置*********************/
	//节点ID
	ID        int64       `flag:"node-id" cfg:"id"`
	//日志等级(可参考nsqd/logger.go)
	LogLevel  lg.LogLevel `flag:"log-level"`
	//日志前缀
	LogPrefix string      `flag:"log-prefix"`
	Logger    Logger

	/*******************地址配置，包括tcp,http,https等*********************/
	//....
	
	/*******************持久化配置*********************/
	//元数据存储地址（topic/channel的信息）
	DataPath        string        `flag:"data-path"`
	//每个channel最大消息个数，如果超过就需要写入磁盘
	MemQueueSize    int64         `flag:"mem-queue-size"`
	//写入磁盘的文件名格式是 xx-000000001, 该配置表示文件名的最大bit位
	//如果是2，文件格式是 xx-00
	MaxBytesPerFile int64         `flag:"max-bytes-per-file"`
	//缓冲区内容同步到磁盘的时间间隔
	SyncEvery       int64         `flag:"sync-every"`
	//同步超时时间
	SyncTimeout     time.Duration `flag:"sync-timeout"`


	/*******************nsqd.queueScanLoop 函数中使用如下参数*********************/
	//nsqd 扫描延时队列或者未ack队列的时间间隔
	QueueScanInterval        time.Duration
	//nsqd 调整工作线程和刷新topic.channel的时间间隔
	QueueScanRefreshInterval time.Duration
	//nsqd 扫描channel的数量
	QueueScanSelectionCount  int `flag:"queue-scan-selection-count"`
	//nsqd 扫描各种队列的最大工作线程
	QueueScanWorkerPoolMax   int `flag:"queue-scan-worker-pool-max"`
	//nsqd 允许扫描失败的比例。
	QueueScanDirtyPercent    float64

	/*******************MQ消息和命令相关配置*********************/
	//消息最大为被ack的时间，超过这个时间会被重新推入队列，重新消费
	MsgTimeout    time.Duration `flag:"msg-timeout"`
	MaxMsgTimeout time.Duration `flag:"max-msg-timeout"`
	//单个消息的最大字节数
	MaxMsgSize    int64         `flag:"max-msg-size"`
	//批量消息的，最大字节数
	MaxBodySize   int64         `flag:"max-body-size"`
	MaxReqTimeout time.Duration `flag:"max-req-timeout"`
	ClientTimeout time.Duration

	/*******************客户端相关配置*********************/
	//心跳间隔时间
	MaxHeartbeatInterval   time.Duration `flag:"max-heartbeat-interval"`
	//????
	MaxRdyCount            int64         `flag:"max-rdy-count"`
	MaxOutputBufferSize    int64         `flag:"max-output-buffer-size"`
	//根据最大最小缓存输出时间，生成一个desiredTimeout
	MaxOutputBufferTimeout time.Duration `flag:"max-output-buffer-timeout"`
	MinOutputBufferTimeout time.Duration `flag:"min-output-buffer-timeout"`
	OutputBufferTimeout    time.Duration `flag:"output-buffer-timeout"`
	//channel的最多消费者数量
	MaxChannelConsumers    int           `flag:"max-channel-consumers"`

	// statsd integration
	StatsdAddress       string        `flag:"statsd-address"`
	StatsdPrefix        string        `flag:"statsd-prefix"`
	StatsdInterval      time.Duration `flag:"statsd-interval"`
	StatsdMemStats      bool          `flag:"statsd-mem-stats"`
	StatsdUDPPacketSize int           `flag:"statsd-udp-packet-size"`

	// e2e message latency
	E2EProcessingLatencyWindowTime  time.Duration `flag:"e2e-processing-latency-window-time"`
	E2EProcessingLatencyPercentiles []float64     `flag:"e2e-processing-latency-percentile" cfg:"e2e_processing_latency_percentiles"`

	// TLS config
	TLSCert             string `flag:"tls-cert"`
	TLSKey              string `flag:"tls-key"`
	TLSClientAuthPolicy string `flag:"tls-client-auth-policy"`
	TLSRootCAFile       string `flag:"tls-root-ca-file"`
	TLSRequired         int    `flag:"tls-required"`
	TLSMinVersion       uint16 `flag:"tls-min-version"`

	// compression
	DeflateEnabled  bool `flag:"deflate"`
	MaxDeflateLevel int  `flag:"max-deflate-level"`
	SnappyEnabled   bool `flag:"snappy"`
}

2.nsqd.New 实例化流程

func New(opts *Options) (*NSQD, error) {
	var err error
	//元数据存储地址，元数据包括 topic 和 channel,具体可参考 meta 结构体
	dataPath := opts.DataPath
	if opts.DataPath == "" {
		cwd, _ := os.Getwd()
		dataPath = cwd
	}
	//日志
	if opts.Logger == nil {
		opts.Logger = log.New(os.Stderr, opts.LogPrefix, log.Ldate|log.Ltime|log.Lmicroseconds)
	}

	n := &NSQD{
		startTime:            time.Now(),
		topicMap:             make(map[string]*Topic),
		clients:              make(map[int64]Client),
		exitChan:             make(chan int),
		notifyChan:           make(chan interface{}),
		optsNotificationChan: make(chan struct{}, 1),
		dl:                   dirlock.New(dataPath),
	}
	//http 客户端，专门用于访问lookupd，上报或获取client,topic,channel的信息
	httpcli := http_api.NewClient(nil, opts.HTTPClientConnectTimeout, opts.HTTPClientRequestTimeout)
	n.ci = clusterinfo.New(n.logf, httpcli)

	//lookupd 集群 设置为空，后面会通过lookup 线程补充
	n.lookupPeers.Store([]*lookupPeer{})
	//存储配置
	n.swapOpts(opts)
	n.errValue.Store(errStore{})

	//.... 省略中间过程

	//TCP 连接处理（tcp.go）
	n.tcpServer = &tcpServer{}
	//监听
	n.tcpListener, err = net.Listen("tcp", opts.TCPAddress)
	if err != nil {
		return nil, fmt.Errorf("listen (%s) failed - %s", opts.TCPAddress, err)
	}
	//httP 服务监听
	n.httpListener, err = net.Listen("tcp", opts.HTTPAddress)
	if err != nil {
		return nil, fmt.Errorf("listen (%s) failed - %s", opts.HTTPAddress, err)
	}
	if n.tlsConfig != nil && opts.HTTPSAddress != "" {
		//https 监听
		n.httpsListener, err = tls.Listen("tcp", opts.HTTPSAddress, n.tlsConfig)
		if err != nil {
			return nil, fmt.Errorf("listen (%s) failed - %s", opts.HTTPSAddress, err)
		}
	}

	return n, nil
}

3.nsqd.Main 启动过程

func (n *NSQD) Main() error {
	ctx := &context{n}

	exitCh := make(chan error)
	var once sync.Once
	exitFunc := func(err error) {
		once.Do(func() {
			if err != nil {
				n.logf(LOG_FATAL, "%s", err)
			}
			exitCh <- err
		})
	}

	n.tcpServer.ctx = ctx
	/*
		util.WaitGroupWrapper(internal/util/wait_group_wrapper.go)
		这个方法只是，包装了 Add,和Done的操作
	*/
	//启动TCP 服务监听(tcp.go Handle)
	n.waitGroup.Wrap(func() {
		exitFunc(protocol.TCPServer(n.tcpListener, n.tcpServer, n.logf))
	})

	httpServer := newHTTPServer(ctx, false, n.getOpts().TLSRequired == TLSRequired)
	//启动Http 服务监听 (http.go)
	n.waitGroup.Wrap(func() {
		exitFunc(http_api.Serve(n.httpListener, httpServer, "HTTP", n.logf))
	})

	if n.tlsConfig != nil && n.getOpts().HTTPSAddress != "" {
		httpsServer := newHTTPServer(ctx, true, true)
		n.waitGroup.Wrap(func() {
			exitFunc(http_api.Serve(n.httpsListener, httpsServer, "HTTPS", n.logf))
		})
	}

	//启动延迟队列和超时队列等处理线程
	n.waitGroup.Wrap(n.queueScanLoop)
	//启动集群上报 client,topic,channel等处理线程
	n.waitGroup.Wrap(n.lookupLoop)
	if n.getOpts().StatsdAddress != "" {
		n.waitGroup.Wrap(n.statsdLoop)
	}

	err := <-exitCh
	return err
}

总结
nsqd的启动包括：
1.配置NSQD参数(option.go)
2.实例化nsqd(New)
3.启动tcp服务监听、http/https监听、queueScanLoop线程及lookupLoop线程

下次分享：nsqd中tcp 服务监听部分。