go ipfs provider体系

go-ipfs-provider@v0.4.3

system

system封装了Provider和Reprovider，对外提供provide和reprovide功能。
在system中，provide函数在CoreAPI中的Unixfs add和Pin add中被调用，reprovide函数会被ipfs bitswap命令调用。

tips: ipfs dht provide命令直接调用dht，不通过system。

// system.go

// System defines the interface for interacting with the value
// provider system
type System interface {
	Run()
	Close() error
	Provide(cid.Cid) error
	Reprovide(context.Context) error
}

type system struct {
	provider   Provider
	reprovider Reprovider
}

// NewSystem constructs a new provider system from a provider and reprovider
func NewSystem(provider Provider, reprovider Reprovider) System {
	return &system{provider, reprovider}
}

// Run the provider system by running the provider and reprovider
func (s *system) Run() {
	go s.provider.Run()
	go s.reprovider.Run()
}

// Provide a value
func (s *system) Provide(cid cid.Cid) error {
	return s.provider.Provide(cid)
}

// Reprovide all the previously provided values
func (s *system) Reprovide(ctx context.Context) error {
	return s.reprovider.Trigger(ctx)
}

interface

// Provider announces blocks to the network
type Provider interface {
	// Run is used to begin processing the provider work
	Run()
	// Provide takes a cid and makes an attempt to announce it to the network
	Provide(cid.Cid) error
	// Close stops the provider
	Close() error
}

// Reprovider reannounces blocks to the network
type Reprovider interface {
	// Run is used to begin processing the reprovider work and waiting for reprovide triggers
	Run()
	// Trigger a reprovide
	Trigger(context.Context) error
	// Close stops the reprovider
	Close() error
}

provider

provider维持了一个worker pool（默认8个worker），支持并行provide。

//simple/provider.go

// Provider announces blocks to the network
type Provider struct {
	ctx context.Context
	// the CIDs for which provide announcements should be made
	queue *q.Queue
	// used to announce providing to the network
	contentRouting routing.ContentRouting
	// how long to wait for announce to complete before giving up
	timeout time.Duration
	// how many workers concurrently work through thhe queue
	workerLimit int
}

// Run workers to handle provide requests.
func (p *Provider) Run() {
	p.handleAnnouncements()
}

// Provide the given cid using specified strategy.
func (p *Provider) Provide(root cid.Cid) error {
	return p.queue.Enqueue(root)
}

// Handle all outgoing cids by providing (announcing) them
func (p *Provider) handleAnnouncements() {
	for workers := 0; workers < p.workerLimit; workers++ {
		go func() {
			for p.ctx.Err() == nil {
				select {
				case <-p.ctx.Done():
					return
				case c, ok := <-p.queue.Dequeue():
					if !ok {
						// queue closed.
						return
					}

					p.doProvide(c)
				}
			}
		}()
	}
}

func (p *Provider) doProvide(c cid.Cid) {
	ctx := p.ctx
	if p.timeout > 0 {
		var cancel context.CancelFunc
		ctx, cancel = context.WithTimeout(ctx, p.timeout)
		defer cancel()
	} else {
		ctx = p.ctx
	}

	logP.Info("announce - start - ", c)
	if err := p.contentRouting.Provide(ctx, c, true); err != nil {
		logP.Warningf("Unable to provide entry: %s, %s", c, err)
	}
	logP.Info("announce - end - ", c)
}

reprovider

reprovidre定期启动，也可以手动触发，手动触发调用trigger函数，而不是直接调用reprovide函数的原因：避免定期启动和手动触发同时调用reprovide函数。
simple/provider.go

// Reprovider reannounces blocks to the network
type Reprovider struct {
	// Reprovider context. Cancel to stop, then wait on closedCh.
	ctx      context.Context
	cancel   context.CancelFunc
	closedCh chan struct{}

	// Trigger triggers a reprovide.
	trigger chan chan<- error

	// The routing system to provide values through
	rsys routing.ContentRouting

	keyProvider KeyChanFunc

	tick time.Duration
}

// Run re-provides keys with 'tick' interval or when triggered
func (rp *Reprovider) Run() {
	defer close(rp.closedCh)

	var initialReprovideCh, reprovideCh <-chan time.Time

	// If reproviding is enabled (non-zero)
	if rp.tick > 0 {
		reprovideTicker := time.NewTicker(rp.tick)
		defer reprovideTicker.Stop()
		reprovideCh = reprovideTicker.C

		// If the reprovide ticker is larger than a minute (likely),
		// provide once after we've been up a minute.
		//
		// Don't provide _immediately_ as we might be just about to stop.
		if rp.tick > time.Minute {
			initialReprovideTimer := time.NewTimer(time.Minute)
			defer initialReprovideTimer.Stop()

			initialReprovideCh = initialReprovideTimer.C
		}
	}

	var done chan<- error
	for rp.ctx.Err() == nil {
		select {
		case <-initialReprovideCh:
		case <-reprovideCh:
		case done = <-rp.trigger:
		case <-rp.ctx.Done():
			return
		}

		err := rp.Reprovide()

		// only log if we've hit an actual error, otherwise just tell the client we're shutting down
		if rp.ctx.Err() != nil {
			err = ErrClosed
		} else if err != nil {
			logR.Errorf("failed to reprovide: %s", err)
		}

		if done != nil {
			if err != nil {
				done <- err
			}
			close(done)
		}
	}
}

// Reprovide registers all keys given by rp.keyProvider to libp2p content routing
func (rp *Reprovider) Reprovide() error {
	keychan, err := rp.keyProvider(rp.ctx)
	if err != nil {
		return fmt.Errorf("failed to get key chan: %s", err)
	}
	for c := range keychan {
		// hash security
		if err := verifcid.ValidateCid(c); err != nil {
			logR.Errorf("insecure hash in reprovider, %s (%s)", c, err)
			continue
		}
		op := func() error {
			err := rp.rsys.Provide(rp.ctx, c, true)
			if err != nil {
				logR.Debugf("Failed to provide key: %s", err)
			}
			return err
		}

		err := backoff.Retry(op, backoff.WithContext(backoff.NewExponentialBackOff(), rp.ctx))
		if err != nil {
			logR.Debugf("Providing failed after number of retries: %s", err)
			return err
		}
	}
	return nil
}

// Trigger starts the reprovision process in rp.Run and waits for it to finish.
//
// Returns an error if a reprovide is already in progress.
func (rp *Reprovider) Trigger(ctx context.Context) error {
	resultCh := make(chan error, 1)
	select {
	case rp.trigger <- resultCh:
	default:
		return fmt.Errorf("reprovider is already running")
	}

	select {
	case err := <-resultCh:
		return err
	case <-rp.ctx.Done():
		return ErrClosed
	case <-ctx.Done():
		return ctx.Err()
	}
}

reprovide函数首先调用keyProvider，获取到cid，再调用dht的provide。
聪明的你很快就能想到，keyProvider决定了reprovide哪些块。

reprovide策略

// KeyChanFunc is function streaming CIDs to pass to content routing
type KeyChanFunc func(context.Context) (<-chan cid.Cid, error)

// NewBlockstoreProvider returns key provider using bstore.AllKeysChan
func NewBlockstoreProvider(bstore blocks.Blockstore) KeyChanFunc {
	return func(ctx context.Context) (<-chan cid.Cid, error) {
		return bstore.AllKeysChan(ctx)
	}
}

// Pinner interface defines how the simple.Reprovider wants to interact
// with a Pinning service
type Pinner interface {
	DirectKeys(ctx context.Context) ([]cid.Cid, error)
	RecursiveKeys(ctx context.Context) ([]cid.Cid, error)
}

// NewPinnedProvider returns provider supplying pinned keys
func NewPinnedProvider(onlyRoots bool, pinning Pinner, dag ipld.DAGService) KeyChanFunc {
	return func(ctx context.Context) (<-chan cid.Cid, error) {
		set, err := pinSet(ctx, pinning, dag, onlyRoots)
		if err != nil {
			return nil, err
		}

		outCh := make(chan cid.Cid)
		go func() {
			defer close(outCh)
			for c := range set.New {
				select {
				case <-ctx.Done():
					return
				case outCh <- c:
				}
			}

		}()

		return outCh, nil
	}
}

当前，keyProvider有两种可能：blockstore和pinner。至于选择哪种，是由配置文件决定的

  "Reprovider": {
    "Interval": "12h",
    "Strategy": "all"
  }

对应的解析代码:

// core/node/provider.go
// SimpleProviders creates the simple provider/reprovider dependencies
func SimpleProviders(reprovideStrategy string, reprovideInterval string) fx.Option {
	reproviderInterval := kReprovideFrequency
	if reprovideInterval != "" {
		dur, err := time.ParseDuration(reprovideInterval)
		if err != nil {
			return fx.Error(err)
		}

		reproviderInterval = dur
	}

	var keyProvider fx.Option
	switch reprovideStrategy {
	case "all":
		fallthrough
	case "":
		keyProvider = fx.Provide(simple.NewBlockstoreProvider)
	case "roots":
		keyProvider = fx.Provide(pinnedProviderStrategy(true))
	case "pinned":
		keyProvider = fx.Provide(pinnedProviderStrategy(false))
	default:
		return fx.Error(fmt.Errorf("unknown reprovider strategy '%s'", reprovideStrategy))
	}

	return fx.Options(
		fx.Provide(ProviderQueue),
		fx.Provide(SimpleProvider),
		keyProvider,
		fx.Provide(SimpleReprovider(reproviderInterval)),
	)
}

func pinnedProviderStrategy(onlyRoots bool) interface{} {
	return func(pinner pin.Pinner, dag ipld.DAGService) simple.KeyChanFunc {
		return simple.NewPinnedProvider(onlyRoots, pinner, dag)
	}
}

刷新策略有3种：all/roots/pinned，默认是all。具体含义如下：

• all代表reprovide blockstore中所有的block；
• roots代表reprovide pinned中标注着"direct"和"recursive"的block，不包括"indirect";
• pinned代表reprovide除了"Internal"外被pinned的所有的block。

Recursive 状态 文件块树被递归添加到pinner中，根文件块的状态是Recursive，非根文件块的状态是Indirect
Direct 状态 只有目标文件块添加到pinner中, 子孙块不做处理，目标文件块的状态就是Direct
Indirect 状态 文件块树被递归添加到pinner中，根文件块的状态是Recursive，非根文件块的状态是Indirect
Internal 状态 ipfs 使用文件块来保存pin状态，这些文件块的状态就是Internal
NotPinned 状态 文件块没有被pin，在GC时会被删除

对应的是go-ipfs-pinner中的mode

// Pin Modes
const (
	// Recursive pins pin the target cids along with any reachable children.
	Recursive Mode = iota

	// Direct pins pin just the target cid.
	Direct

	// Indirect pins are cids who have some ancestor pinned recursively.
	Indirect

	// Internal pins are cids used to keep the internal state of the pinner.
	Internal

	// NotPinned
	NotPinned

	// Any refers to any pinned cid
	Any
)