nsq 源码阅读(三) nsqlookupd tcp handler 处理函数

3. nsqlookupd 源码阅读（2） tcp handler 处理函数

每个nsqd 生产者的状态发生变化时，都会通知nsqlookup将自己从nsqlookup的DB中删除。

func (p *tcpServer) Handle(clientConn net.Conn) {
    p.ctx.nsqlookupd.logf(LOG_INFO, "TCP: new client(%s)", clientConn.RemoteAddr())

    // The client should initialize itself by sending a 4 byte sequence indicating
    // the version of the protocol that it intends to communicate, this will allow us
    // to gracefully upgrade the protocol away from text/line oriented to whatever...
    buf := make([]byte, 4)
    _, err := io.ReadFull(clientConn, buf)
    if err != nil {
        p.ctx.nsqlookupd.logf(LOG_ERROR, "failed to read protocol version - %s", err)
        return
    }
    // protocolMagic 默认是  "  V1"
    protocolMagic := string(buf)
    p.ctx.nsqlookupd.logf(LOG_INFO, "CLIENT(%s): desired protocol magic '%s'",
        clientConn.RemoteAddr(), protocolMagic)

    var prot protocol.Protocol
    switch protocolMagic {
    case "  V1":
        prot = &LookupProtocolV1{ctx: p.ctx}
    default:
        protocol.SendResponse(clientConn, []byte("E_BAD_PROTOCOL"))
        clientConn.Close()
        p.ctx.nsqlookupd.logf(LOG_ERROR, "client(%s) bad protocol magic '%s'",
            clientConn.RemoteAddr(), protocolMagic)
        return
    }
    // 循环读取消息
    err = prot.IOLoop(clientConn)
    if err != nil {
        p.ctx.nsqlookupd.logf(LOG_ERROR, "client(%s) - %s", clientConn.RemoteAddr(), err)
        return
    }
}

目前的 protocolMagic 只支持 V1 ,所以客户端在连接的时候，服务端首先要验证conn发过来的protocolMagic 是不是v1 是的话，new一个Protocol ,调用其IOLoop 循环读取方法。

func (p *LookupProtocolV1) IOLoop(conn net.Conn) error {
    var err error
    var line string

    client := NewClientV1(conn)
    reader := bufio.NewReader(client)
    for {
        line, err = reader.ReadString('\n')
        if err != nil {
            break
        }
        line = strings.TrimSpace(line)
        params := strings.Split(line, " ")
        fmt.Println("params", params)
        var response []byte
        response, err = p.Exec(client, reader, params)
        if err != nil {
            ctx := ""
            if parentErr := err.(protocol.ChildErr).Parent(); parentErr != nil {
                ctx = " - " + parentErr.Error()
            }
            p.ctx.nsqlookupd.logf(LOG_ERROR, "[%s] - %s%s", client, err, ctx)

            _, sendErr := protocol.SendResponse(client, []byte(err.Error()))
            if sendErr != nil {
                p.ctx.nsqlookupd.logf(LOG_ERROR, "[%s] - %s%s", client, sendErr, ctx)
                break
            }

            // errors of type FatalClientErr should forceably close the connection
            if _, ok := err.(*protocol.FatalClientErr); ok {
                break
            }
            continue
        }

        if response != nil {
            _, err = protocol.SendResponse(client, response)
            if err != nil {
                break
            }
        }
    }

    conn.Close()
    p.ctx.nsqlookupd.logf(LOG_INFO, "CLIENT(%s): closing", client)
    if client.peerInfo != nil {
        registrations := p.ctx.nsqlookupd.DB.LookupRegistrations(client.peerInfo.id)
        for _, r := range registrations {
            if removed, _ := p.ctx.nsqlookupd.DB.RemoveProducer(r, client.peerInfo.id); removed {
                p.ctx.nsqlookupd.logf(LOG_INFO, "DB: client(%s) UNREGISTER category:%s key:%s subkey:%s",
                    client, r.Category, r.Key, r.SubKey)
            }
        }
    }
    return err
}

IOLoop 方法会不停的读取客户端的数据信息，并且默认有以下四种类型。

类型	功能	备注
PING	nsqd 客户端通知nsqlookup 自己还活着
IDENTIFY	nsqd客户端首次连接nsqlookupd的时候，首先进行鉴定，以及记录这个生产者
REGISTER	新的topic 或者channel创建的时候，通知nsqlookupd 进行注册
UNREGISTER	删除时，通知注销

客户端nsqd链接的时候首先发送一个IDENTIFY 消息类型，进行鉴定，鉴定通过后会将nsqd存在topic信息等进行REGISTER注册，同时每隔15s发送一条PING 命令告诉nsqlookupd自己还活着。

对于不同的类型调用不同的处理方法:

func (p *LookupProtocolV1) Exec(client *ClientV1, reader *bufio.Reader, params []string) ([]byte, error) {
    switch params[0] {
    case "PING":
        return p.PING(client, params)
    case "IDENTIFY":
        return p.IDENTIFY(client, reader, params[1:])
    case "REGISTER":
        return p.REGISTER(client, reader, params[1:])
    case "UNREGISTER":
        return p.UNREGISTER(client, reader, params[1:])
    }
    return nil, protocol.NewFatalClientErr(nil, "E_INVALID", fmt.Sprintf("invalid command %s", params[0]))
}
s

(1) IDENTIFY // 鉴定 并记录下这个生产者

func (p *LookupProtocolV1) IDENTIFY(client *ClientV1, reader *bufio.Reader, params []string) ([]byte, error) {
    var err error
    // 只需要 IDENTIFY 鉴定一次，多次鉴定中断连接 
    if client.peerInfo != nil {
        return nil, protocol.NewFatalClientErr(err, "E_INVALID", "cannot IDENTIFY again")
    }

    var bodyLen int32
    err = binary.Read(reader, binary.BigEndian, &bodyLen)
    if err != nil {
        return nil, protocol.NewFatalClientErr(err, "E_BAD_BODY", "IDENTIFY failed to read body size")
    }

    body := make([]byte, bodyLen)
    _, err = io.ReadFull(reader, body)
    if err != nil {
        return nil, protocol.NewFatalClientErr(err, "E_BAD_BODY", "IDENTIFY failed to read body")
    }
    // body is a json structure with producer information
    // 新建一个 nsqd的info 结构
    peerInfo := PeerInfo{id: client.RemoteAddr().String()}
    // body = {"broadcast_address":"127.0.0.1","hostname":"caigaoxing","http_port":4151,"tcp_port":4150,"version":"1.0.0-compat"}

    err = json.Unmarshal(body, &peerInfo)
    if err != nil {
        return nil, protocol.NewFatalClientErr(err, "E_BAD_BODY", "IDENTIFY failed to decode JSON body")
    }
    peerInfo.RemoteAddress = client.RemoteAddr().String()

    // require all fields
    if peerInfo.BroadcastAddress == "" || peerInfo.TCPPort == 0 || peerInfo.HTTPPort == 0 || peerInfo.Version == "" {
        return nil, protocol.NewFatalClientErr(nil, "E_BAD_BODY", "IDENTIFY missing fields")
    }
    // 原子操作 该生产者的最后一次登录时间 
    atomic.StoreInt64(&peerInfo.lastUpdate, time.Now().UnixNano())

    p.ctx.nsqlookupd.logf(LOG_INFO, "CLIENT(%s): IDENTIFY Address:%s TCP:%d HTTP:%d Version:%s",
        client, peerInfo.BroadcastAddress, peerInfo.TCPPort, peerInfo.HTTPPort, peerInfo.Version)

    client.peerInfo = &peerInfo
    // 将该生产者保存到 RegistrationDB 中，同时key的 Category 为 client。注意保存的client的指针，所以PING的时候，直接更改client的值，将可以将RegistrationDB中的值进行更改。
    if p.ctx.nsqlookupd.DB.AddProducer(Registration{"client", "", ""}, &Producer{peerInfo: client.peerInfo}) {
        p.ctx.nsqlookupd.logf(LOG_INFO, "DB: client(%s) REGISTER category:%s key:%s subkey:%s", client, "client", "", "")
    }

    // build a response
    data := make(map[string]interface{})
    data["tcp_port"] = p.ctx.nsqlookupd.RealTCPAddr().Port
    data["http_port"] = p.ctx.nsqlookupd.RealHTTPAddr().Port
    data["version"] = version.Binary
    hostname, err := os.Hostname()
    if err != nil {
        log.Fatalf("ERROR: unable to get hostname %s", err)
    }
    data["broadcast_address"] = p.ctx.nsqlookupd.opts.BroadcastAddress
    data["hostname"] = hostname

    response, err := json.Marshal(data)
    if err != nil {
        p.ctx.nsqlookupd.logf(LOG_ERROR, "marshaling %v", data)
        return []byte("OK"), nil
    }
    return response, nil
}

(2) PING // 通知nsqlookup自己还活着

func (p *LookupProtocolV1) PING(client *ClientV1, params []string) ([]byte, error) {
    if client.peerInfo != nil {
        // we could get a PING before other commands on the same client connection
        cur := time.Unix(0, atomic.LoadInt64(&client.peerInfo.lastUpdate))
        now := time.Now()
        p.ctx.nsqlookupd.logf(LOG_INFO, "CLIENT(%s): pinged (last ping %s)", client.peerInfo.id,
            now.Sub(cur))
        // 其实就是原子的 更改了自己的最后一个更新的时间
        atomic.StoreInt64(&client.peerInfo.lastUpdate, now.UnixNano())
    }
    return []byte("OK"), nil
}

（3） REGISTER 注册新的topic和channel

func (p *LookupProtocolV1) REGISTER(client *ClientV1, reader *bufio.Reader, params []string) ([]byte, error) {
    if client.peerInfo == nil {
        return nil, protocol.NewFatalClientErr(nil, "E_INVALID", "client must IDENTIFY")
    }

    // 验证一下topic和channel的命名是否，同时返回topic 和channel
    topic, channel, err := getTopicChan("REGISTER", params)
    if err != nil {
        return nil, err
    }

    // 添加channel的信息到RegistrationDB中
    if channel != "" {
        key := Registration{"channel", topic, channel}
        if p.ctx.nsqlookupd.DB.AddProducer(key, &Producer{peerInfo: client.peerInfo}) {
            p.ctx.nsqlookupd.logf(LOG_INFO, "DB: client(%s) REGISTER category:%s key:%s subkey:%s",
                client, "channel", topic, channel)
        }
    }
     // 添加topic的信息到RegistrationDB中
    key := Registration{"topic", topic, ""}
    if p.ctx.nsqlookupd.DB.AddProducer(key, &Producer{peerInfo: client.peerInfo}) {
        p.ctx.nsqlookupd.logf(LOG_INFO, "DB: client(%s) REGISTER category:%s key:%s subkey:%s",
            client, "topic", topic, "")
    }

    return []byte("OK"), nil
}

（4） UNREGISTER 注销 删除的 的topic和channel

func (p *LookupProtocolV1) UNREGISTER(client *ClientV1, reader *bufio.Reader, params []string) ([]byte, error) {
    if client.peerInfo == nil {
        return nil, protocol.NewFatalClientErr(nil, "E_INVALID", "client must IDENTIFY")
    }

    topic, channel, err := getTopicChan("UNREGISTER", params)
    if err != nil {
        return nil, err
    }
    // 删除 channel 
    if channel != "" {
        key := Registration{"channel", topic, channel}
        removed, left := p.ctx.nsqlookupd.DB.RemoveProducer(key, client.peerInfo.id)
        if removed {
            p.ctx.nsqlookupd.logf(LOG_INFO, "DB: client(%s) UNREGISTER category:%s key:%s subkey:%s",
                client, "channel", topic, channel)
        }
        // for ephemeral channels, remove the channel as well if it has no producers
        //  // 如果这个channel 是个临时的channel 并且他没有生产者，也可以将该channel 删除
        if left == 0 && strings.HasSuffix(channel, "#ephemeral") {
            p.ctx.nsqlookupd.DB.RemoveRegistration(key)
        }
    } else {
        // 因为没有指定channel 所以就是删除topic 并且将所有的跟该topic有关的channel删除
        // no channel was specified so this is a topic unregistration
        // remove all of the channel registrations...
        // normally this shouldn't happen which is why we print a warning message
        // if anything is actually removed
        // 找到所有的符合条件的Registration
        registrations := p.ctx.nsqlookupd.DB.FindRegistrations("channel", topic, "*")
        // 将  该用户下所有的符合topic的channel都删除
        for _, r := range registrations {
            if removed, _ := p.ctx.nsqlookupd.DB.RemoveProducer(r, client.peerInfo.id); removed {
                p.ctx.nsqlookupd.logf(LOG_WARN, "client(%s) unexpected UNREGISTER category:%s key:%s subkey:%s",
                    client, "channel", topic, r.SubKey)
            }
        }
        // 最后将该用户 符合topic的删除
        key := Registration{"topic", topic, ""}
        if removed, _ := p.ctx.nsqlookupd.DB.RemoveProducer(key, client.peerInfo.id); removed {
            p.ctx.nsqlookupd.logf(LOG_INFO, "DB: client(%s) UNREGISTER category:%s key:%s subkey:%s",
                client, "topic", topic, "")
        }
    }

    return []byte("OK"), nil
}

handler 函数涉及对 RegistrationDB的操作，所以一定要先将RegistrationDB存储的内容看明白

完整的过程以及存储方式如下：
1.当有一个拥有topic和channel的生产者连接nsqlookup的时候,首先将该生产者添加到key为Registration{“client”, “”, “”} 的生产者数组中去.
2. 然后生产者将自己拥有的topic和channel通知nsqlookupd,nsqlookupd得到通知后，将该生产者添加到自己的key 为Registration{”channel”,topicinstance,channelinstance}的数组中去，其中的topicinstance,channelinstance 生产者传递上来的。
3. 最后nsqlookup将该生产者添加到自己的key为Registration{”topic”, topicinstance, “”}中去。该topicinstance是生产者传上来的。

举例：

实际数据 1个生产者
Produce1:
    topicAA:
            ChannelAAA
            ChannelAAB
    topicAB:
            ChannelAAA
            ChannelABB

RegistrationDB 的map存储的数据如下

map[
// [0xc042132240]只有一 个 客户端（生产者）
{client  }:[0xc042132240]
// topic为 topicAB 的客户端为1个
{topic topicAB }:[0xc042132a20]
// topic为 topicAA 的客户端为1个
{topic topicAA }:[0xc0421323f0]

{channel topicAA ChannelAAB}:[0xc0421324e0]
{channel topicAA ChannelAAA}:[0xc042132600] 
{channel topicAB ChannelAAA}:[0xc0421329f0] 
{channel topicAB ChannelABB}:[0xc042132ba0] 
]

以上的数据也可以调用nsqlookupd的/debug接口获取

{
  "channel:topicAA:ChannelAAA": [
    {
      "broadcast_address": "127.0.0.1",
      "hostname": "caigaoxing",
      "http_port": 4151,
      "id": "127.0.0.1:53587",
      "last_update": 1525948504411771900,
      "tcp_port": 4150,
      "tombstoned": false,
      "tombstoned_at": -6795364578871345152,
      "version": "1.0.0-compat"
    }
  ],
  "channel:topicAA:ChannelAAB": [
    {
      "broadcast_address": "127.0.0.1",
      "hostname": "caigaoxing",
      "http_port": 4151,
      "id": "127.0.0.1:53587",
      "last_update": 1525948504411771900,
      "tcp_port": 4150,
      "tombstoned": false,
      "tombstoned_at": -6795364578871345152,
      "version": "1.0.0-compat"
    }
  ],
  "channel:topicAB:ChannelAAA": [
    {
      "broadcast_address": "127.0.0.1",
      "hostname": "caigaoxing",
      "http_port": 4151,
      "id": "127.0.0.1:53587",
      "last_update": 1525948504411771900,
      "tcp_port": 4150,
      "tombstoned": false,
      "tombstoned_at": -6795364578871345152,
      "version": "1.0.0-compat"
    }
  ],
  "channel:topicAB:ChannelABB": [
    {
      "broadcast_address": "127.0.0.1",
      "hostname": "caigaoxing",
      "http_port": 4151,
      "id": "127.0.0.1:53587",
      "last_update": 1525948504411771900,
      "tcp_port": 4150,
      "tombstoned": false,
      "tombstoned_at": -6795364578871345152,
      "version": "1.0.0-compat"
    }
  ],
  "client::": [
    {
      "broadcast_address": "127.0.0.1",
      "hostname": "caigaoxing",
      "http_port": 4151,
      "id": "127.0.0.1:53587",
      "last_update": 1525948504411771900,
      "tcp_port": 4150,
      "tombstoned": false,
      "tombstoned_at": -6795364578871345152,
      "version": "1.0.0-compat"
    }
  ],
  "topic:topicAA:": [
    {
      "broadcast_address": "127.0.0.1",
      "hostname": "caigaoxing",
      "http_port": 4151,
      "id": "127.0.0.1:53587",
      "last_update": 1525948504411771900,
      "tcp_port": 4150,
      "tombstoned": false,
      "tombstoned_at": -6795364578871345152,
      "version": "1.0.0-compat"
    }
  ],
  "topic:topicAB:": [
    {
      "broadcast_address": "127.0.0.1",
      "hostname": "caigaoxing",
      "http_port": 4151,
      "id": "127.0.0.1:53587",
      "last_update": 1525948504411771900,
      "tcp_port": 4150,
      "tombstoned": false,
      "tombstoned_at": -6795364578871345152,
      "version": "1.0.0-compat"
    }
  ]
}