从bitswap学习使用libp2p

代码版本：go-bitswap@v0.1.11

前言

bitswapNetwork是bitswap的网络接口，负责bitswap的网络通信和dht调用，从bitswapNetwork可以知道bitswap是如何使用libp2p的。

阅读本文，最好看过bitswap的代码以及了解libp2p中host、swarm、conn和stream的基础知识。如果没看过代码，那么看看主干，也能学到如何使用libp2p。

bitswapNetwork

// NewFromIpfsHost returns a BitSwapNetwork supported by underlying IPFS host.
func NewFromIpfsHost(host host.Host, r routing.ContentRouting, opts ...NetOpt) BitSwapNetwork {
	s := Settings{}
	for _, opt := range opts {
		opt(&s)
	}

	bitswapNetwork := impl{
		host:    host,
		routing: r,

		protocolBitswap:       s.ProtocolPrefix + ProtocolBitswap,
		protocolBitswapOne:    s.ProtocolPrefix + ProtocolBitswapOne,
		protocolBitswapNoVers: s.ProtocolPrefix + ProtocolBitswapNoVers,
	}
	return &bitswapNetwork
}

// impl transforms the ipfs network interface, which sends and receives
// NetMessage objects, into the bitswap network interface.
type impl struct {
	host    host.Host
	routing routing.ContentRouting

	protocolBitswap       protocol.ID
	protocolBitswapOne    protocol.ID
	protocolBitswapNoVers protocol.ID

	// inbound messages from the network are forwarded to the receiver
	receiver Receiver

	stats Stats
}

bitswap的网络模块主要包括host和routing，host用于网络通信，主要是连接的创建和流的创建、收发和关闭；routing用于调用dht。

receiver就是bitswap，主要用于调用bitswap的ReceiveMessage()/PeerConnected()/PeerDisconnected()

此外，由于几个版本的bitswap通信消息结构体发生了变化，网络模块还包含3个bitswap的协议号，用于区别处理序列化和反序列化(protobuf)。

host

host这一块主要包含conn和stream。

conn

func (bsnet *impl) ConnectTo(ctx context.Context, p peer.ID) error {
	return bsnet.host.Connect(ctx, peer.AddrInfo{ID: p})
}

当session从dht网络找到provider后，providerQueryManager负责ConnectTo these provider;

// providermanager.go
func (pqm *ProviderQueryManager) findProviderWorker() {
	// findProviderWorker just cycles through incoming provider queries one
	// at a time. We have six of these workers running at once
	// to let requests go in parallel but keep them rate limited
	for {
		select {
		case fpr, ok := <-pqm.providerRequestsProcessing:
			if !ok {
				return
			}
			k := fpr.k
			log.Debugf("Beginning Find Provider Request for cid: %s", k.String())
			pqm.timeoutMutex.RLock()
			findProviderCtx, cancel := context.WithTimeout(fpr.ctx, pqm.findProviderTimeout)
			pqm.timeoutMutex.RUnlock()
			providers := pqm.network.FindProvidersAsync(findProviderCtx, k, maxProviders)
			wg := &sync.WaitGroup{}
			for p := range providers {
				wg.Add(1)
				go func(p peer.ID) {
					defer wg.Done()
					err := pqm.network.ConnectTo(findProviderCtx, p)
					if err != nil {
						log.Debugf("failed to connect to provider %s: %s", p, err)
						return
					}
					select {
					case pqm.providerQueryMessages <- &receivedProviderMessage{
						k: k,
						p: p,
					}:
					case <-pqm.ctx.Done():
						return
					}
				}(p)
			}
			wg.Wait()
			cancel()
			select {
			case pqm.providerQueryMessages <- &finishedProviderQueryMessage{
				k: k,
			}:
			case <-pqm.ctx.Done():
			}
		case <-pqm.ctx.Done():
			return
		}
	}
}

wantmanager通过peerHandler(peermanager对象)为每个连接的peer维护一个messagequeue，如果messagequeue.sender为空，则调用openSender，创建连接和流。

// messagequeue.go
func openSender(ctx context.Context, network MessageNetwork, p peer.ID) (bsnet.MessageSender, error) {
	// allow ten minutes for connections this includes looking them up in the
	// dht dialing them, and handshaking
	conctx, cancel := context.WithTimeout(ctx, time.Minute*10)
	defer cancel()

	err := network.ConnectTo(conctx, p)
	if err != nil {
		return nil, err
	}

	nsender, err := network.NewMessageSender(ctx, p)
	if err != nil {
		return nil, err
	}

	return nsender, nil
}

stream

new

前面说的openSender()就是调用NewMessageSender创建流，并且新创建的流保存在streamMessageSender.s中，后面使用就不需要再创建流了;

下面即将介绍的SendMessage()直接调用newStreamToPeer创建流。

func (bsnet *impl) NewMessageSender(ctx context.Context, p peer.ID) (MessageSender, error) {
	s, err := bsnet.newStreamToPeer(ctx, p)
	if err != nil {
		return nil, err
	}

	return &streamMessageSender{s: s, bsnet: bsnet}, nil
}

func (bsnet *impl) newStreamToPeer(ctx context.Context, p peer.ID) (network.Stream, error) {
	return bsnet.host.NewStream(ctx, p, bsnet.protocolBitswap, bsnet.protocolBitswapOne, bsnet.protocolBitswapNoVers)
}

Inbound

首先SetStreamHandler，当收到其它peer的stream write时，调用handleNewStream()函数。3个版本都是一样的handler函数，区别在于message proto结构体的解析。

func (bsnet *impl) SetDelegate(r Receiver) {
	bsnet.receiver = r
	bsnet.host.SetStreamHandler(bsnet.protocolBitswap, bsnet.handleNewStream)
	bsnet.host.SetStreamHandler(bsnet.protocolBitswapOne, bsnet.handleNewStream)
	bsnet.host.SetStreamHandler(bsnet.protocolBitswapNoVers, bsnet.handleNewStream)
	bsnet.host.Network().Notify((*netNotifiee)(bsnet))
	// TODO: StopNotify.

}

// handleNewStream receives a new stream from the network.
func (bsnet *impl) handleNewStream(s network.Stream) {
	defer s.Close()

	if bsnet.receiver == nil {
		_ = s.Reset()
		return
	}

	reader := msgio.NewVarintReaderSize(s, network.MessageSizeMax)
	for {
		received, err := bsmsg.FromMsgReader(reader)
		if err != nil {
			if err != io.EOF {
				_ = s.Reset()
				go bsnet.receiver.ReceiveError(err)
				log.Debugf("bitswap net handleNewStream from %s error: %s", s.Conn().RemotePeer(), err)
			}
			return
		}

		p := s.Conn().RemotePeer()
		ctx := context.Background()
		log.Debugf("bitswap net handleNewStream from %s", s.Conn().RemotePeer())
		bsnet.receiver.ReceiveMessage(ctx, p, received)
		atomic.AddUint64(&bsnet.stats.MessagesRecvd, 1)
	}
}

Outbound

MessageSender对象调用SendMsg()发送wantlist，SendMsg()调用msgToStream()，msgToStream将消息转化为proto格式后stream write。

type streamMessageSender struct {
	s     network.Stream
	bsnet *impl
}

func (s *streamMessageSender) Close() error {
	return helpers.FullClose(s.s)
}

func (s *streamMessageSender) Reset() error {
	return s.s.Reset()
}

func (s *streamMessageSender) SendMsg(ctx context.Context, msg bsmsg.BitSwapMessage) error {
	return s.bsnet.msgToStream(ctx, s.s, msg)
}

var sendMessageTimeout = time.Minute * 10

func (bsnet *impl) msgToStream(ctx context.Context, s network.Stream, msg bsmsg.BitSwapMessage) error {
	deadline := time.Now().Add(sendMessageTimeout)
	if dl, ok := ctx.Deadline(); ok {
		deadline = dl
	}

	if err := s.SetWriteDeadline(deadline); err != nil {
		log.Warningf("error setting deadline: %s", err)
	}

	switch s.Protocol() {
	case bsnet.protocolBitswap:
		if err := msg.ToNetV1(s); err != nil {
			log.Debugf("error: %s", err)
			return err
		}
	case bsnet.protocolBitswapOne, bsnet.protocolBitswapNoVers:
		if err := msg.ToNetV0(s); err != nil {
			log.Debugf("error: %s", err)
			return err
		}
	default:
		return fmt.Errorf("unrecognized protocol on remote: %s", s.Protocol())
	}

	if err := s.SetWriteDeadline(time.Time{}); err != nil {
		log.Warningf("error resetting deadline: %s", err)
	}
	return nil
}

SendMessage()首先new stream，再write，最后close，下次调用还需要重新创建流。主要用于发送block。

func (bsnet *impl) SendMessage(
	ctx context.Context,
	p peer.ID,
	outgoing bsmsg.BitSwapMessage) error {

	s, err := bsnet.newStreamToPeer(ctx, p)
	if err != nil {
		return err
	}

	if err = bsnet.msgToStream(ctx, s, outgoing); err != nil {
		_ = s.Reset()
		return err
	}
	atomic.AddUint64(&bsnet.stats.MessagesSent, 1)

	// TODO(https://github.com/libp2p/go-libp2p-net/issues/28): Avoid this goroutine.
	//nolint
	go helpers.AwaitEOF(s)
	return s.Close()

}

routing

routing通过dht, 向网络provide和findprovider。

当bitswap收到想要的块时，如果provideEnabled, 就会调用Provide()，向网络发布provider。

当session通过wantmanager发送wantlist请求，在一段时间内得不到想要的block时，session会通过providerQueryManager，向网络findprovider， 当providerQueryManager得到providers时，providerQueryManager负责和它们创建连接。

// FindProvidersAsync returns a channel of providers for the given key.
func (bsnet *impl) FindProvidersAsync(ctx context.Context, k cid.Cid, max int) <-chan peer.ID {
	out := make(chan peer.ID, max)
	go func() {
		defer close(out)
		providers := bsnet.routing.FindProvidersAsync(ctx, k, max)
		for info := range providers {
			if info.ID == bsnet.host.ID() {
				continue // ignore self as provider
			}
			bsnet.host.Peerstore().AddAddrs(info.ID, info.Addrs, peerstore.TempAddrTTL)
			select {
			case <-ctx.Done():
				return
			case out <- info.ID:
			}
		}
	}()
	return out
}

// Provide provides the key to the network
func (bsnet *impl) Provide(ctx context.Context, k cid.Cid) error {
	return bsnet.routing.Provide(ctx, k, true)
}

Notifiee

Notifiee一共有6个接口，调用的时机注释写得很清楚，这里就不再赘述。

// Notifiee is an interface for an object wishing to receive
// notifications from a Network.
type Notifiee interface {
	Listen(Network, ma.Multiaddr)      // called when network starts listening on an addr
	ListenClose(Network, ma.Multiaddr) // called when network stops listening on an addr
	Connected(Network, Conn)           // called when a connection opened
	Disconnected(Network, Conn)        // called when a connection closed
	OpenedStream(Network, Stream)      // called when a stream opened
	ClosedStream(Network, Stream)      // called when a stream closed

举一个case: 本节点有新连接（不管是listen，还是dial）时，swarm(libp2p swarm addConn)会调用notifyAll，通知所有Notify的对象(实现了Notifiee接口)有新的connection, BitSwapNetwork实现了Notifiee接口接口，并通过SetDelegate()函数Notify，因此Connected消息可以收到通知。

func (bsnet *impl) SetDelegate(r Receiver) {
	bsnet.receiver = r
	bsnet.host.SetStreamHandler(bsnet.protocolBitswap, bsnet.handleNewStream)
	bsnet.host.SetStreamHandler(bsnet.protocolBitswapOne, bsnet.handleNewStream)
	bsnet.host.SetStreamHandler(bsnet.protocolBitswapNoVers, bsnet.handleNewStream)
	bsnet.host.Network().Notify((*netNotifiee)(bsnet))
	// TODO: StopNotify.

}

type netNotifiee impl

func (nn *netNotifiee) Connected(n network.Network, v network.Conn) {
	nn.impl().receiver.PeerConnected(v.RemotePeer())
}

func (nn *netNotifiee) Disconnected(n network.Network, v network.Conn) {
	nn.impl().receiver.PeerDisconnected(v.RemotePeer())
}

func (nn *netNotifiee) OpenedStream(n network.Network, v network.Stream) {}
func (nn *netNotifiee) ClosedStream(n network.Network, v network.Stream) {}
func (nn *netNotifiee) Listen(n network.Network, a ma.Multiaddr)         {}
func (nn *netNotifiee) ListenClose(n network.Network, a ma.Multiaddr)    {}

Connected时，bitswap会通知egine和wantmanager，egine收到通知后，给新的节点创建了账本；wantmanager收到把新节点加入到自己的peerqueue，向新的peer询问是否有正在brocast的wantlists（存放在bs.wantmanager.bcwl）。

// PeerConnected is called by the network interface
// when a peer initiates a new connection to bitswap.
func (bs *Bitswap) PeerConnected(p peer.ID) {
	bs.wm.Connected(p)
	bs.engine.PeerConnected(p)
}

// PeerDisconnected is called by the network interface when a peer
// closes a connection
func (bs *Bitswap) PeerDisconnected(p peer.ID) {
	bs.wm.Disconnected(p)
	bs.engine.PeerDisconnected(p)
}