tendermint, evidence

Initialization

In NewNode(),

evidenceReactor, evidencePool, err := createEvidenceReactor(config, dbProvider, stateDB, logger)

createEvidenceReactor will create the evidenceDB, evidencePool and the reactor:

func createEvidenceReactor(config *cfg.Config, dbProvider DBProvider,
	stateDB dbm.DB, logger log.Logger) (*evidence.Reactor, *evidence.Pool, error) {

	evidenceDB, err := dbProvider(&DBContext{"evidence", config})
	if err != nil {
		return nil, nil, err
	}
	evidenceLogger := logger.With("module", "evidence")
	evidencePool := evidence.NewPool(stateDB, evidenceDB)
	evidencePool.SetLogger(evidenceLogger)
	evidenceReactor := evidence.NewReactor(evidencePool)
	evidenceReactor.SetLogger(evidenceLogger)
	return evidenceReactor, evidencePool, nil
}

evidenceDB and store

Similar with blockstore/state DB, this evidenceDB will be created with context ‘evidence’.

evidenceDB, err := dbProvider(&DBContext{"evidence", config})

The store provide persistent support for evidences.

/*
Requirements:
- Valid new evidence must be persisted immediately and never forgotten
- Uncommitted evidence must be continuously broadcast
- Uncommitted evidence has a partial order, the evidence’s priority

Impl:
- First commit atomically in outqueue, pending, lookup.
- Once broadcast, remove from outqueue. No need to sync
- Once committed, atomically remove from pending and update lookup.

Schema for indexing evidence (note you need both height and hash to find a piece of evidence):

“evidence-lookup”// -> Info
“evidence-outqueue”/// -> Info
“evidence-pending”// -> Info
*/

evPool

It offers Add/Update/Query support to ConsensusState. evPool will be passed to BlockExec.

// EvidencePool defines the EvidencePool interface used by the ConsensusState.
// Get/Set/Commit
type EvidencePool interface {
	PendingEvidence(int64) []types.Evidence
	AddEvidence(types.Evidence) error
	Update(*types.Block, State)
	// IsCommitted indicates if this evidence was already marked committed in another block.
	IsCommitted(types.Evidence) bool
}

// Pool maintains a pool of valid evidence
// in an Store.
type Pool struct {
	logger log.Logger

	store        *Store
	evidenceList *clist.CList // concurrent linked-list of evidence

	// needed to load validators to verify evidence
	stateDB dbm.DB

	// latest state
	mtx   sync.Mutex
	state sm.State
}

NewNode() Initialization of BlockExec and evPool:

	...
	// make block executor for consensus and blockchain reactors to execute blocks
	blockExec := sm.NewBlockExecutor(
		stateDB,
		logger.With("module", "state"),
		proxyApp.Consensus(),
		mempool,
		evidencePool,
		sm.BlockExecutorWithMetrics(smMetrics),
	)

	// Make BlockchainReactor
	bcReactor, err := createBlockchainReactor(config, state, blockExec, blockStore, fastSync, logger)
	if err != nil {
		return nil, errors.Wrap(err, "could not create blockchain reactor")
	}

	// Make ConsensusReactor
	consensusReactor, consensusState := createConsensusReactor(
		config, state, blockExec, blockStore, mempool, evidencePool,
		privValidator, csMetrics, fastSync, eventBus, consensusLogger,
	)
	...

evPool.PendingEvidence

// PendingEvidence returns up to maxNum uncommitted evidence.
// If maxNum is -1, all evidence is returned.
func (evpool *Pool) PendingEvidence(maxNum int64) []types.Evidence {
	return evpool.store.PendingEvidence(maxNum)
}

Call stack

github.com/tendermint/tendermint/state.(*BlockExecutor).CreateProposalBlock at execution.go:105
github.com/tendermint/tendermint/consensus.(*State).createProposalBlock at state.go:1029
github.com/tendermint/tendermint/consensus.(*State).defaultDecideProposal at state.go:963
github.com/tendermint/tendermint/consensus.(*State).defaultDecideProposal-fm at state.go:953
github.com/tendermint/tendermint/consensus.(*State).enterPropose at state.go:939
github.com/tendermint/tendermint/consensus.(*State).handleTxsAvailable at state.go:797
github.com/tendermint/tendermint/consensus.(*State).receiveRoutine at state.go:639
runtime.goexit at asm_amd64.s:1357
 - Async stack trace
github.com/tendermint/tendermint/consensus.(*State).OnStart at state.go:335

evPool.Update

// Update loads the latest
func (evpool *Pool) Update(block *types.Block, state sm.State) {

	// sanity check
	if state.LastBlockHeight != block.Height {
		panic(
			fmt.Sprintf("Failed EvidencePool.Update sanity check: got state.Height=%d with block.Height=%d",
				state.LastBlockHeight,
				block.Height,
			),
		)
	}

	// update the state
	evpool.mtx.Lock()
	evpool.state = state
	evpool.mtx.Unlock()

	// remove evidence from pending and mark committed
	evpool.MarkEvidenceAsCommitted(block.Height, block.Time, block.Evidence.Evidence)
}

evPool has to be updated when a new block is committed.
Call stack

github.com/tendermint/tendermint/evidence.(*Pool).Update at pool.go:79
github.com/tendermint/tendermint/state.(*BlockExecutor).ApplyBlock at execution.go:173
github.com/tendermint/tendermint/consensus.(*State).finalizeCommit at state.go:1431
github.com/tendermint/tendermint/consensus.(*State).tryFinalizeCommit at state.go:1350
github.com/tendermint/tendermint/consensus.(*State).enterCommit.func1 at state.go:1285
github.com/tendermint/tendermint/consensus.(*State).enterCommit at state.go:1322
github.com/tendermint/tendermint/consensus.(*State).addVote at state.go:1819
github.com/tendermint/tendermint/consensus.(*State).tryAddVote at state.go:1642
github.com/tendermint/tendermint/consensus.(*State).handleMsg at state.go:709
github.com/tendermint/tendermint/consensus.(*State).receiveRoutine at state.go:660
runtime.goexit at asm_amd64.s:1357
 - Async stack trace
github.com/tendermint/tendermint/consensus.(*State).OnStart at state.go:335

evPool.AddEvidence

When calling tryAddVote, it might add a DuplicateVoteEvidence to evPool, or from BroadcastEvidence RPC, evidence can be added to evPool, and persisted in evidence DB.

// AddEvidence checks the evidence is valid and adds it to the pool.
func (evpool *Pool) AddEvidence(evidence types.Evidence) (err error) {

	// TODO: check if we already have evidence for this
	// validator at this height so we dont get spammed

	if err := sm.VerifyEvidence(evpool.stateDB, evpool.State(), evidence); err != nil {
		return err
	}

	// fetch the validator and return its voting power as its priority
	// TODO: something better ?
	valset, _ := sm.LoadValidators(evpool.stateDB, evidence.Height())
	_, val := valset.GetByAddress(evidence.Address())
	priority := val.VotingPower

	added := evpool.store.AddNewEvidence(evidence, priority)
	if !added {
		// evidence already known, just ignore
		return
	}

	evpool.logger.Info("Verified new evidence of byzantine behaviour", "evidence", evidence)

	// add evidence to clist
	evpool.evidenceList.PushBack(evidence)

	return nil
}

evidence Reactor

Channel:

EvidenceChannel = byte(0x38)

Message:

// ListMessage contains a list of evidence.
type ListMessage struct {
	Evidence []types.Evidence
}

broadcastEvidenceRoutine

// AddPeer implements Reactor.
func (evR *Reactor) AddPeer(peer p2p.Peer) {
	go evR.broadcastEvidenceRoutine(peer)
}

evidences in evPool will be broadcasted to network.

Receive

It will receive ListMessage from Switch.

// Receive implements Reactor.
// It adds any received evidence to the evpool.
func (evR *Reactor) Receive(chID byte, src p2p.Peer, msgBytes []byte) {
	msg, err := decodeMsg(msgBytes)
	if err != nil {
		evR.Logger.Error("Error decoding message", "src", src, "chId", chID, "msg", msg, "err", err, "bytes", msgBytes)
		evR.Switch.StopPeerForError(src, err)
		return
	}

	if err = msg.ValidateBasic(); err != nil {
		evR.Logger.Error("Peer sent us invalid msg", "peer", src, "msg", msg, "err", err)
		evR.Switch.StopPeerForError(src, err)
		return
	}

	evR.Logger.Debug("Receive", "src", src, "chId", chID, "msg", msg)

	switch msg := msg.(type) {
	case *ListMessage:
		for _, ev := range msg.Evidence {
			err := evR.evpool.AddEvidence(ev)
			if err != nil {
				evR.Logger.Info("Evidence is not valid", "evidence", msg.Evidence, "err", err)
				// punish peer
				evR.Switch.StopPeerForError(src, err)
			}
		}
	default:
		evR.Logger.Error(fmt.Sprintf("Unknown message type %v", reflect.TypeOf(msg)))
	}
}