Gossip Service
Peers leverage gossip to broadcast ledger and channel data in a scalable fashion. Gossip messaging is continuous, and each peer on a channel is constantly receiving current and consistent ledger data from multiple peers.
Gossiped message is digitally signed.
Main functions
- Manages peer discovery and channel membership, by continually identifying available member peers, and eventually detecting peers that have gone offline.
- Disseminates ledger data across all peers on a channel. Any peer with data that is out of sync with the rest of the channel identifies the missing blocks and syncs itself by copying the correct data.
- Bring newly connected peers up to speed by allowing peer-to-peer state transfer update of ledger data.
Gossip-based broadcasting operates by peers receiving messages from other peers on the channel, and then forwarding these messages to a number of randomly selected peers on the channel, where this number is a configurable constant. Peers can also exercise a pull mechanism rather than waiting for delivery of a message. This cycle repeats, with the result of channel membership, ledger and state information continually being kept current and in sync. For dissemination of new blocks, the leader peer on the channel pulls the data from the ordering service and initiates gossip dissemination to peers in its own organization.
Basic
gossipSvc is the interface, implemeted by gossip node.
type GossipService struct {
gossipSvc
privateHandlers map[string]privateHandler
chains map[string]state.GossipStateProvider
leaderElection map[string]election.LeaderElectionService
deliveryService map[string]deliverservice.DeliverService
deliveryFactory DeliveryServiceFactory
lock sync.RWMutex
mcs api.MessageCryptoService
peerIdentity []byte
secAdv api.SecurityAdvisor
metrics *gossipmetrics.GossipMetrics
serviceConfig *ServiceConfig
privdataConfig *gossipprivdata.PrivdataConfig
}
// Node is a member of a gossip network
type Node struct {
selfIdentity api.PeerIdentityType
includeIdentityPeriod time.Time
certStore *certStore
idMapper identity.Mapper
presumedDead chan common.PKIidType
disc discovery.Discovery
comm comm.Comm
selfOrg api.OrgIdentityType
*comm.ChannelDeMultiplexer
logger util.Logger
stopSignal *sync.WaitGroup
conf *Config
toDieChan chan struct{}
stopFlag int32
emitter batchingEmitter
discAdapter *discoveryAdapter
secAdvisor api.SecurityAdvisor
chanState *channelState
disSecAdap *discoverySecurityAdapter
mcs api.MessageCryptoService
stateInfoMsgStore msgstore.MessageStore
certPuller pull.Mediator
gossipMetrics *metrics.GossipMetrics
}
// New creates the gossip service.
func New(
peerIdentity identity.SignerSerializer,
gossipMetrics *gossipmetrics.GossipMetrics,
endpoint string,
s *grpc.Server,
mcs api.MessageCryptoService,
secAdv api.SecurityAdvisor,
secureDialOpts api.PeerSecureDialOpts,
credSupport *corecomm.CredentialSupport,
deliverGRPCClient *corecomm.GRPCClient,
gossipConfig *gossip.Config,
serviceConfig *ServiceConfig,
privdataConfig *gossipprivdata.PrivdataConfig,
deliverServiceConfig *deliverservice.DeliverServiceConfig,
) (*GossipService, error) {
serializedIdentity, err := peerIdentity.Serialize()
if err != nil {
return nil, err
}
logger.Infof("Initialize gossip with endpoint %s", endpoint)
gossipComponent := gossip.New(
gossipConfig,
s,
secAdv,
mcs,
serializedIdentity,
secureDialOpts,
gossipMetrics,
)
return &GossipService{
gossipSvc: gossipComponent,
mcs: mcs,
privateHandlers: make(map[string]privateHandler),
chains: make(map[string]state.GossipStateProvider),
leaderElection: make(map[string]election.LeaderElectionService),
deliveryService: make(map[string]deliverservice.DeliverService),
deliveryFactory: &deliveryFactoryImpl{
signer: peerIdentity,
credentialSupport: credSupport,
deliverGRPCClient: deliverGRPCClient,
deliverServiceConfig: deliverServiceConfig,
},
peerIdentity: serializedIdentity,
secAdv: secAdv,
metrics: gossipMetrics,
serviceConfig: serviceConfig,
privdataConfig: privdataConfig,
}, nil
}
Gossip Message
Message Type
// *GossipMessage_AliveMsg
// *GossipMessage_MemReq Membership Request, handled by Discovery
// *GossipMessage_MemRes Membership Response, handled by Discovery
// *GossipMessage_DataMsg Data Block
// *GossipMessage_Hello BlockPuller Hello
// *GossipMessage_DataDig BlockPuller Digest
// *GossipMessage_DataReq BlockPuller DataReq
// *GossipMessage_DataUpdate BlockPuller DataUpdate
// *GossipMessage_Empty
// *GossipMessage_Conn
// *GossipMessage_StateInfo Published by Channel periodically
// *GossipMessage_StateSnapshot Response to StateInfoPullReq, handled by Channel
// *GossipMessage_StateInfoPullReq Sent out by Channel, periodically
// *GossipMessage_StateRequest StateProvider antiEntropy Request
// *GossipMessage_StateResponse StateProvider antiEntropy Response
// *GossipMessage_LeadershipMsg Handled by Election
// *GossipMessage_PeerIdentity
// *GossipMessage_Ack ACK, only by StateProvider
// *GossipMessage_PrivateReq
// *GossipMessage_PrivateRes
// *GossipMessage_PrivateData
Message Tag
Message Tag is way to group messages. It indicates who should be interested in the messages.
const (
GossipMessage_UNDEFINED GossipMessage_Tag = 0
GossipMessage_EMPTY GossipMessage_Tag = 1
GossipMessage_ORG_ONLY GossipMessage_Tag = 2 // Org message
GossipMessage_CHAN_ONLY GossipMessage_Tag = 3 // Channel message
GossipMessage_CHAN_AND_ORG GossipMessage_Tag = 4
GossipMessage_CHAN_OR_ORG GossipMessage_Tag = 5
)
idMapper
The idMapper holds mappings between pkiID and certificates(identities) of peers.
// Mapper holds mappings between pkiID
// to certificates(identities) of peers
type Mapper interface {
// Put associates an identity to its given pkiID, and returns an error
// in case the given pkiID doesn't match the identity
Put(pkiID common.PKIidType, identity api.PeerIdentityType) error
// Get returns the identity of a given pkiID, or error if such an identity
// isn't found
Get(pkiID common.PKIidType) (api.PeerIdentityType, error)
// Sign signs a message, returns a signed message on success
// or an error on failure
Sign(msg []byte) ([]byte, error)
// Verify verifies a signed message
Verify(vkID, signature, message []byte) error
// GetPKIidOfCert returns the PKI-ID of a certificate
GetPKIidOfCert(api.PeerIdentityType) common.PKIidType
// SuspectPeers re-validates all peers that match the given predicate
SuspectPeers(isSuspected api.PeerSuspector)
// IdentityInfo returns information known peer identities
IdentityInfo() api.PeerIdentitySet
// Stop stops all background computations of the Mapper
Stop()
}
MessageStoreExpirable
It is a MessageStore with the message replacing policy and invalidation trigger passed. It supports old message expiration after msgTTL, during expiration first external lock taken, expiration callback invoked and external lock released. Callback and external lock can be nil.
Basically, it is a tool providing validation and transient store.
This mechanism is used by:
- Gossip : add StateInfo msg to store
- Gossip Discovery : add Alive msg to store
- Gossip Channel : add Block data msg
- Gossip Channel : add Leadership msg
func NewMessageStoreExpirable(pol common.MessageReplacingPolicy, trigger invalidationTrigger, msgTTL time.Duration, externalLock func(), externalUnlock func(), externalExpire func(interface{})) MessageStore {
store := newMsgStore(pol, trigger)
store.msgTTL = msgTTL
if externalLock != nil {
store.externalLock = externalLock
}
if externalUnlock != nil {
store.externalUnlock = externalUnlock
}
if externalExpire != nil {
store.expireMsgCallback = externalExpire
}
go store.expirationRoutine()
return store
}
PubSub
Internally, Gossip uses a simple PubSub mechanism for coordinating peer connection asynchronously. This PusSub is used internally by some sub moudles of Gossip Service.
// Publish publishes an item to all subscribers on the topic
func (ps *PubSub) Publish(topic string, item interface{}) error {
ps.RLock()
defer ps.RUnlock()
s, subscribed := ps.subscriptions[topic]
if !subscribed {
return errors.New("no subscribers")
}
for _, sub := range s.ToArray() {
c := sub.(*subscription).c
select {
case c <- item:
default: // Not enough room in buffer, continue in order to not block publisher
}
}
return nil
}
// Subscribe returns a subscription to a topic that expires when given TTL passes
func (ps *PubSub) Subscribe(topic string, ttl time.Duration) Subscription {
sub := &subscription{
top: topic,
ttl: ttl,
c: make(chan interface{}, subscriptionBuffSize),
}
ps.Lock()
// Add subscription to subscriptions map
s, exists := ps.subscriptions[topic]
// If no subscription set for the topic exists, create one
if !exists {
s = NewSet()
ps.subscriptions[topic] = s
}
ps.Unlock()
// Add the subscription
s.Add(sub)
// When the timeout expires, remove the subscription
time.AfterFunc(ttl, func() {
ps.unSubscribe(sub)
})
return sub
}
Puller Mediator
Puller will do pull sync with the other endpoints. It can be initialized with the configuration to pull
the specified information.
// Mediator is a component wrap a PullEngine and provides the methods
// it needs to perform pull synchronization.
// The specialization of a pull mediator to a certain type of message is
// done by the configuration, a IdentifierExtractor, IdentifierExtractor
// given at construction, and also hooks that can be registered for each
// type of pullMsgType (hello, digest, req, res).
type Mediator interface {
// Stop stop the Mediator
Stop()
// RegisterMsgHook registers a message hook to a specific type of pull message
RegisterMsgHook(MsgType, MessageHook)
// Add adds a GossipMessage to the Mediator
Add(*protoext.SignedGossipMessage)
// Remove removes a GossipMessage from the Mediator with a matching digest,
// if such a message exits
Remove(digest string)
// HandleMessage handles a message from some remote peer
HandleMessage(msg protoext.ReceivedMessage)
}
go func() {
for !engine.toDie() {
time.Sleep(sleepTime)
if engine.toDie() {
return
}
engine.initiatePull()
}
}()
func (engine *PullEngine) initiatePull() {
engine.lock.Lock()
defer engine.lock.Unlock()
engine.acceptDigests()
for _, peer := range engine.SelectPeers() {
nonce := engine.newNONCE()
engine.outgoingNONCES.Add(nonce)
engine.nonces2peers[nonce] = peer
engine.peers2nonces[peer] = nonce
engine.Hello(peer, nonce)
}
time.AfterFunc(engine.digestWaitTime, func() {
engine.processIncomingDigests()
})
}
- When GossipService receives a pull message, it will do some verifications and pass the message to Puller for further processing.
- Puller will send a hello message to initiate the protocol and returns an NONCE that is expected to be returned in the digest message. This is actually asking what do you have
- From the degiest messages, Puller send Request to slected random destiation. This is to request the missing data
- Puller will handle the response, and the add the information to local store. Ok, now I have the data.
CertStore and Puller
Puller for certStore is configured as below:
PullMsgType_IDENTITY_MSG
GossipMessage_EMPTY
conf := pull.Config{
MsgType: pg.PullMsgType_IDENTITY_MSG,
Channel: []byte(""),
ID: g.conf.InternalEndpoint,
PeerCountToSelect: g.conf.PullPeerNum,
PullInterval: g.conf.PullInterval,
Tag: pg.GossipMessage_EMPTY,
PullEngineConfig: algo.PullEngineConfig{
DigestWaitTime: g.conf.DigestWaitTime,
RequestWaitTime: g.conf.RequestWaitTime,
ResponseWaitTime: g.conf.ResponseWaitTime,
},
}
...
selfIDMsg, err := certStore.createIdentityMessage()
if err != nil {
certStore.logger.Panicf("Failed creating self identity message: %+v", errors.WithStack(err))
}
puller.Add(selfIDMsg)
puller.RegisterMsgHook(pull.RequestMsgType, func(_ []string, msgs []*protoext.SignedGossipMessage, _ protoext.ReceivedMessage)
CertStore will add selfIDMsg to local store for the others to pull from. It also call RegisterMsgHook to intecept the Request message to update the idMapper of Gossip Node.
Gossip Node Start
Gossip will start a few background goroutines.
- syncDiscovery(), trigger Discover to sync up
- handlePresumedDead(), to process Comm dead
- Make a Gossip Msg selector and go acceptMessages(). Selector = !(Conn || Empty || PrivateData)
func (g *Node) start() {
go g.syncDiscovery()
go g.handlePresumedDead()
msgSelector := func(msg interface{}) bool {
gMsg, isGossipMsg := msg.(protoext.ReceivedMessage)
if !isGossipMsg {
return false
}
isConn := gMsg.GetGossipMessage().GetConn() != nil
isEmpty := gMsg.GetGossipMessage().GetEmpty() != nil
isPrivateData := protoext.IsPrivateDataMsg(gMsg.GetGossipMessage().GossipMessage)
return !(isConn || isEmpty || isPrivateData)
}
incMsgs := g.comm.Accept(msgSelector)
go g.acceptMessages(incMsgs)
g.logger.Info("Gossip instance", g.conf.ID, "started")
}
Gossip HandleMessage
Channel Message -> Channel.HandleMessage
DiscoveryMessage -> forwardDiscoveryMsg(m)
PullMsg -> g.certStore.handleMessage(m)
func (g *Node) handleMessage(m protoext.ReceivedMessage) {
...
if protoext.IsChannelRestricted(msg.GossipMessage) {
if gc := g.chanState.lookupChannelForMsg(m); gc == nil {
// If we're not in the channel, we should still forward to peers of our org
// in case it's a StateInfo message
if g.IsInMyOrg(discovery.NetworkMember{PKIid: m.GetConnectionInfo().ID}) && protoext.IsStateInfoMsg(msg.GossipMessage) {
if g.stateInfoMsgStore.Add(msg) {
g.emitter.Add(&emittedGossipMessage{
SignedGossipMessage: msg,
filter: m.GetConnectionInfo().ID.IsNotSameFilter,
})
}
}
if !g.toDie() {
g.logger.Debug("No such channel", msg.Channel, "discarding message", msg)
}
} else {
if protoext.IsLeadershipMsg(m.GetGossipMessage().GossipMessage) {
if err := g.validateLeadershipMessage(m.GetGossipMessage()); err != nil {
g.logger.Warningf("Failed validating LeaderElection message: %+v", errors.WithStack(err))
return
}
}
gc.HandleMessage(m)
}
return
}
if selectOnlyDiscoveryMessages(m) {
// It's a membership request, check its self information
// matches the sender
if m.GetGossipMessage().GetMemReq() != nil {
sMsg, err := protoext.EnvelopeToGossipMessage(m.GetGossipMessage().GetMemReq().SelfInformation)
if err != nil {
g.logger.Warningf("Got membership request with invalid selfInfo: %+v", errors.WithStack(err))
return
}
if !protoext.IsAliveMsg(sMsg.GossipMessage) {
g.logger.Warning("Got membership request with selfInfo that isn't an AliveMessage")
return
}
if !bytes.Equal(sMsg.GetAliveMsg().Membership.PkiId, m.GetConnectionInfo().ID) {
g.logger.Warning("Got membership request with selfInfo that doesn't match the handshake")
return
}
}
g.forwardDiscoveryMsg(m)
}
if protoext.IsPullMsg(msg.GossipMessage) && protoext.GetPullMsgType(msg.GossipMessage) == pg.PullMsgType_IDENTITY_MSG {
g.certStore.handleMessage(m)
}
}
Discovery
Discovery starts some background goroutines to probe the network for peers.
// Discovery is the interface that represents a discovery module
type Discovery interface {
// Lookup returns a network member, or nil if not found
Lookup(PKIID common.PKIidType) *NetworkMember
// Self returns this instance's membership information
Self() NetworkMember
// UpdateMetadata updates this instance's metadata
UpdateMetadata([]byte)
// UpdateEndpoint updates this instance's endpoint
UpdateEndpoint(string)
// Stops this instance
Stop()
// GetMembership returns the alive members in the view
GetMembership() []NetworkMember
// InitiateSync makes the instance ask a given number of peers
// for their membership information
InitiateSync(peerNum int)
// Connect makes this instance to connect to a remote instance
// The identifier param is a function that can be used to identify
// the peer, and to assert its PKI-ID, whether its in the peer's org or not,
// and whether the action was successful or not
Connect(member NetworkMember, id identifier)
}
...
// NewDiscoveryService returns a new discovery service with the comm module passed and the crypto service passed
func NewDiscoveryService(self NetworkMember, comm CommService, crypt CryptoService, disPol DisclosurePolicy,
config DiscoveryConfig) Discovery {
d := &gossipDiscoveryImpl{
self: self,
incTime: uint64(time.Now().UnixNano()),
seqNum: uint64(0),
deadLastTS: make(map[string]*timestamp),
aliveLastTS: make(map[string]*timestamp),
id2Member: make(map[string]*NetworkMember),
aliveMembership: util.NewMembershipStore(),
deadMembership: util.NewMembershipStore(),
crypt: crypt,
comm: comm,
lock: &sync.RWMutex{},
toDieChan: make(chan struct{}),
logger: util.GetLogger(util.DiscoveryLogger, self.InternalEndpoint),
disclosurePolicy: disPol,
pubsub: util.NewPubSub(),
aliveTimeInterval: config.AliveTimeInterval,
aliveExpirationTimeout: config.AliveExpirationTimeout,
aliveExpirationCheckInterval: config.AliveExpirationCheckInterval,
reconnectInterval: config.ReconnectInterval,
bootstrapPeers: config.BootstrapPeers,
}
...
d.msgStore = newAliveMsgStore(d)
go d.periodicalSendAlive()
go d.periodicalCheckAlive()
go d.handleMessages()
go d.periodicalReconnectToDead()
go d.handleEvents()
return d
}
Discovery will get the Gossip Msg from Gossip Service.
- periodicalSendAlive: Gossip Alive message which contains my external endpoint address
- periodicalCheckAlive: check memebership, move dead member out of alive member list
- handleMessages: process gossipped alive/memebership message from other peers
- periodicalReconnectToDead: try to connect dead peers and send membership request
- handleEvents: handle peer Disconnect and mark as dead; handle PKIID change and remove peer
batchingEmitter
It is the message batching mechanism, with a period flush. batchingEmitter is used for the gossip push/forwarding phase. Messages are added into the batchingEmitter, and they are forwarded periodically T times in batches and then discarded. If the batchingEmitter’s stored message count reaches a certain capacity, that also triggers a message dispatch.
type batchingEmitter interface {
// Add adds a message to be batched
Add(interface{})
// Stop stops the component
Stop()
// Size returns the amount of pending messages to be emitted
Size() int
}
comm.Comm
It is the communication manager for underlying gRPC, managing RPC connections with other peers.
RPC method and stream:
var _Gossip_serviceDesc = grpc.ServiceDesc{
ServiceName: "gossip.Gossip",
HandlerType: (*GossipServer)(nil),
Methods: []grpc.MethodDesc{
{
MethodName: "Ping",
Handler: _Gossip_Ping_Handler,
},
},
Streams: []grpc.StreamDesc{
{
StreamName: "GossipStream",
Handler: _Gossip_GossipStream_Handler,
ServerStreams: true,
ClientStreams: true,
},
},
Metadata: "gossip/message.proto",
}
Interface:
// Comm is an object that enables to communicate with other peers
// that also embed a CommModule.
type Comm interface {
// GetPKIid returns this instance's PKI id
GetPKIid() common.PKIidType
// Send sends a message to remote peers asynchronously
Send(msg *protoext.SignedGossipMessage, peers ...*RemotePeer)
// SendWithAck sends a message to remote peers, waiting for acknowledgement from minAck of them, or until a certain timeout expires
SendWithAck(msg *protoext.SignedGossipMessage, timeout time.Duration, minAck int, peers ...*RemotePeer) AggregatedSendResult
// Probe probes a remote node and returns nil if its responsive,
// and an error if it's not.
Probe(peer *RemotePeer) error
// Handshake authenticates a remote peer and returns
// (its identity, nil) on success and (nil, error)
Handshake(peer *RemotePeer) (api.PeerIdentityType, error)
// Accept returns a dedicated read-only channel for messages sent by other nodes that match a certain predicate.
// Each message from the channel can be used to send a reply back to the sender
Accept(common.MessageAcceptor) <-chan protoext.ReceivedMessage
// PresumedDead returns a read-only channel for node endpoints that are suspected to be offline
PresumedDead() <-chan common.PKIidType
// IdentitySwitch returns a read-only channel about identity change events
IdentitySwitch() <-chan common.PKIidType
// CloseConn closes a connection to a certain endpoint
CloseConn(peer *RemotePeer)
// Stop stops the module
Stop()
}
chainState
Maintains the Join/Leave state of a channel.
A GossipChannel will be created, initialized and put into chainState map when a channel created.
Call stack
github.com/hyperledger/fabric/gossip/gossip/channel.NewGossipChannel at channel.go:297
github.com/hyperledger/fabric/gossip/gossip.(*channelState).joinChannel at chanstate.go:108
github.com/hyperledger/fabric/gossip/gossip.(*Node).JoinChan at gossip_impl.go:188
github.com/hyperledger/fabric/gossip/service.(*GossipService).updateAnchors at gossip_service.go:425
github.com/hyperledger/fabric/gossip/service.(*configEventer).ProcessConfigUpdate at eventer.go:81
github.com/hyperledger/fabric/core/peer.(*Peer).createChannel.func1 at peer.go:267
github.com/hyperledger/fabric/common/channelconfig.(*BundleSource).Update at bundlesource.go:46
github.com/hyperledger/fabric/common/channelconfig.NewBundleSource at bundlesource.go:38
github.com/hyperledger/fabric/core/peer.(*Peer).createChannel at peer.go:323
type channelState struct {
stopping int32
sync.RWMutex
channels map[string]channel.GossipChannel
g *Node
}
// GossipChannel defines an object that deals with all channel-related messages
type GossipChannel interface {
// Self returns a StateInfoMessage about the peer
Self() *protoext.SignedGossipMessage
// GetPeers returns a list of peers with metadata as published by them
GetPeers() []discovery.NetworkMember
// PeerFilter receives a SubChannelSelectionCriteria and returns a RoutingFilter that selects
// only peer identities that match the given criteria
PeerFilter(api.SubChannelSelectionCriteria) filter.RoutingFilter
// IsMemberInChan checks whether the given member is eligible to be in the channel
IsMemberInChan(member discovery.NetworkMember) bool
// UpdateLedgerHeight updates the ledger height the peer
// publishes to other peers in the channel
UpdateLedgerHeight(height uint64)
// UpdateChaincodes updates the chaincodes the peer publishes
// to other peers in the channel
UpdateChaincodes(chaincode []*proto.Chaincode)
// IsOrgInChannel returns whether the given organization is in the channel
IsOrgInChannel(membersOrg api.OrgIdentityType) bool
// EligibleForChannel returns whether the given member should get blocks
// for this channel
EligibleForChannel(member discovery.NetworkMember) bool
// HandleMessage processes a message sent by a remote peer
HandleMessage(protoext.ReceivedMessage)
// AddToMsgStore adds a given GossipMessage to the message store
AddToMsgStore(msg *protoext.SignedGossipMessage)
// ConfigureChannel (re)configures the list of organizations
// that are eligible to be in the channel
ConfigureChannel(joinMsg api.JoinChannelMessage)
// LeaveChannel makes the peer leave the channel
LeaveChannel()
// Stop stops the channel's activity
Stop()
}
GossipChannel
It refers to every joined channel.
Background Goroutines
// Periodically publish state info
go gc.periodicalInvocation(gc.publishStateInfo, gc.stateInfoPublishScheduler.C)
// Periodically request state info
go gc.periodicalInvocation(gc.requestStateInfo, gc.stateInfoRequestScheduler.C)
ticker := time.NewTicker(gc.GetConf().TimeForMembershipTracker)
gc.membershipTracker = &membershipTracker{
getPeersToTrack: gc.GetPeers,
report: gc.reportMembershipChanges,
stopChan: make(chan struct{}, 1),
tickerChannel: ticker.C,
metrics: metrics,
chainID: channelID,
}
go gc.membershipTracker.trackMembershipChanges()
- publishStateInfo: Gossip my state info periodically
- requestStateInfo: Request state info from some random peers periodically
- trackMembershipChanges, checks which peers are offline and which are online for channel periodically. Make a report in case of any changes
Stop Channel
// Stop stop the channel operations
func (gc *gossipChannel) Stop() {
close(gc.stopChan)
close(gc.membershipTracker.stopChan)
gc.blocksPuller.Stop()
gc.stateInfoPublishScheduler.Stop()
gc.stateInfoRequestScheduler.Stop()
gc.leaderMsgStore.Stop()
gc.stateInfoMsgStore.Stop()
gc.blockMsgStore.Stop()
}
BlockPuller
Don’t really understand what Block Puller can do for us
GossipChannel will create and start the block puller. Bolck puller will try to pull Gossip DataMsg from other peers.
Gossip DataMsg is added to puller local store when a new data block is sent/received to/from peer. This local store will be managed by expiry mechanism of GossipChannel.
GossipChannel.HandleMessage will call gc.blocksPuller.HandleMessage(msg) for the Pull messages.
Call stack of creation:
github.com/hyperledger/fabric/gossip/gossip/algo.NewPullEngineWithFilter at pull.go:120
github.com/hyperledger/fabric/gossip/gossip/pull.NewPullMediator at pullstore.go:153
github.com/hyperledger/fabric/gossip/gossip/channel.(*gossipChannel).createBlockPuller at channel.go:464
github.com/hyperledger/fabric/gossip/gossip/channel.NewGossipChannel at channel.go:214
github.com/hyperledger/fabric/gossip/gossip.(*channelState).joinChannel at chanstate.go:108
github.com/hyperledger/fabric/gossip/gossip.(*Node).JoinChan at gossip_impl.go:188
github.com/hyperledger/fabric/gossip/service.(*GossipService).updateAnchors at gossip_service.go:425
github.com/hyperledger/fabric/gossip/service.(*configEventer).ProcessConfigUpdate at eventer.go:81
github.com/hyperledger/fabric/core/peer.(*Peer).createChannel.func1 at peer.go:267
github.com/hyperledger/fabric/common/channelconfig.(*BundleSource).Update at bundlesource.go:46
github.com/hyperledger/fabric/common/channelconfig.NewBundleSource at bundlesource.go:38
github.com/hyperledger/fabric/core/peer.(*Peer).createChannel at peer.go:323
github.com/hyperledger/fabric/core/peer.(*Peer).Initialize at peer.go:513
Puller is initialized with:
PullMsgType_BLOCK_MSG
GossipMessage_CHAN_AND_ORG
conf := pull.Config{
MsgType: proto.PullMsgType_BLOCK_MSG,
Channel: []byte(gc.chainID),
ID: gc.GetConf().ID,
PeerCountToSelect: gc.GetConf().PullPeerNum,
PullInterval: gc.GetConf().PullInterval,
Tag: proto.GossipMessage_CHAN_AND_ORG,
PullEngineConfig: algo.PullEngineConfig{
DigestWaitTime: gc.GetConf().DigestWaitTime,
RequestWaitTime: gc.GetConf().RequestWaitTime,
ResponseWaitTime: gc.GetConf().ResponseWaitTime,
},
}
Per Channel Service
Every joined channel needs to be initialized with Gossip service.
privateHandlers map[string]privateHandler
chains map[string]state.GossipStateProvider
leaderElection map[string]election.LeaderElectionService
deliveryService map[string]deliverservice.DeliverService
// InitializeChannel allocates the state provider and should be invoked once per channel per execution
func (g *GossipService) InitializeChannel(channelID string, ordererSource *orderers.ConnectionSource, store *transientstore.Store, support Support) {
g.lock.Lock()
defer g.lock.Unlock()
// Initialize new state provider for given committer
logger.Debug("Creating state provider for channelID", channelID)
servicesAdapter := &state.ServicesMediator{GossipAdapter: g, MCSAdapter: g.mcs}
// Initialize private data fetcher
dataRetriever := gossipprivdata.NewDataRetriever(store, support.Committer)
collectionAccessFactory := gossipprivdata.NewCollectionAccessFactory(support.IdDeserializeFactory)
fetcher := gossipprivdata.NewPuller(g.metrics.PrivdataMetrics, support.CollectionStore, g.gossipSvc, dataRetriever,
collectionAccessFactory, channelID, g.serviceConfig.BtlPullMargin)
coordinatorConfig := gossipprivdata.CoordinatorConfig{
TransientBlockRetention: g.serviceConfig.TransientstoreMaxBlockRetention,
PullRetryThreshold: g.serviceConfig.PvtDataPullRetryThreshold,
SkipPullingInvalidTransactions: g.serviceConfig.SkipPullingInvalidTransactionsDuringCommit,
}
selfSignedData := g.createSelfSignedData()
mspID := string(g.secAdv.OrgByPeerIdentity(selfSignedData.Identity))
coordinator := gossipprivdata.NewCoordinator(mspID, gossipprivdata.Support{
ChainID: channelID,
CollectionStore: support.CollectionStore,
Validator: support.Validator,
Committer: support.Committer,
Fetcher: fetcher,
CapabilityProvider: support.CapabilityProvider,
}, store, selfSignedData, g.metrics.PrivdataMetrics, coordinatorConfig,
support.IdDeserializeFactory)
var reconciler gossipprivdata.PvtDataReconciler
if g.privdataConfig.ReconciliationEnabled {
reconciler = gossipprivdata.NewReconciler(channelID, g.metrics.PrivdataMetrics,
support.Committer, fetcher, g.privdataConfig)
} else {
reconciler = &gossipprivdata.NoOpReconciler{}
}
pushAckTimeout := g.serviceConfig.PvtDataPushAckTimeout
g.privateHandlers[channelID] = privateHandler{
support: support,
coordinator: coordinator,
distributor: gossipprivdata.NewDistributor(channelID, g, collectionAccessFactory, g.metrics.PrivdataMetrics, pushAckTimeout),
reconciler: reconciler,
}
g.privateHandlers[channelID].reconciler.Start()
blockingMode := !g.serviceConfig.NonBlockingCommitMode
stateConfig := state.GlobalConfig()
g.chains[channelID] = state.NewGossipStateProvider(
channelID,
servicesAdapter,
coordinator,
g.metrics.StateMetrics,
blockingMode,
stateConfig)
if g.deliveryService[channelID] == nil {
g.deliveryService[channelID] = g.deliveryFactory.Service(g, ordererSource, g.mcs, g.serviceConfig.OrgLeader)
}
// Delivery service might be nil only if it was not able to get connected
// to the ordering service
if g.deliveryService[channelID] != nil {
// Parameters:
// - peer.gossip.useLeaderElection
// - peer.gossip.orgLeader
//
// are mutual exclusive, setting both to true is not defined, hence
// peer will panic and terminate
leaderElection := g.serviceConfig.UseLeaderElection
isStaticOrgLeader := g.serviceConfig.OrgLeader
if leaderElection && isStaticOrgLeader {
logger.Panic("Setting both orgLeader and useLeaderElection to true isn't supported, aborting execution")
}
if leaderElection {
logger.Debug("Delivery uses dynamic leader election mechanism, channel", channelID)
g.leaderElection[channelID] = g.newLeaderElectionComponent(channelID, g.onStatusChangeFactory(channelID,
support.Committer), g.metrics.ElectionMetrics)
} else if isStaticOrgLeader {
logger.Debug("This peer is configured to connect to ordering service for blocks delivery, channel", channelID)
g.deliveryService[channelID].StartDeliverForChannel(channelID, support.Committer, func() {})
} else {
logger.Debug("This peer is not configured to connect to ordering service for blocks delivery, channel", channelID)
}
} else {
logger.Warning("Delivery client is down won't be able to pull blocks for chain", channelID)
}
}
Private Handler
Private Data allows a defined subset of organizations on a channel the ability to endorse, commit, or query private data without having to create a separate channel. As below:
Initialization
Reconciler and Coordinator might be created to support PrivateData handler.
var reconciler gossipprivdata.PvtDataReconciler
if g.privdataConfig.ReconciliationEnabled {
reconciler = gossipprivdata.NewReconciler(channelID, g.metrics.PrivdataMetrics,
support.Committer, fetcher, g.privdataConfig)
} else {
reconciler = &gossipprivdata.NoOpReconciler{}
}
pushAckTimeout := g.serviceConfig.PvtDataPushAckTimeout
g.privateHandlers[channelID] = privateHandler{
support: support,
coordinator: coordinator,
distributor: gossipprivdata.NewDistributor(channelID, g, collectionAccessFactory, g.metrics.PrivdataMetrics, pushAckTimeout),
reconciler: reconciler,
}
g.privateHandlers[channelID].reconciler.Start()
Coordinator
It can help with ledger operations. It is used to persist private data into transient store by StorePvtData(), when endorder starts to simulate proposal.
// Coordinator orchestrates the flow of the new
// blocks arrival and in flight transient data, responsible
// to complete missing parts of transient data for given block.
type Coordinator interface {
// StoreBlock deliver new block with underlined private data
// returns missing transaction ids
StoreBlock(block *common.Block, data util.PvtDataCollections) error
// StorePvtData used to persist private data into transient store
StorePvtData(txid string, privData *protostransientstore.TxPvtReadWriteSetWithConfigInfo, blckHeight uint64) error
// GetPvtDataAndBlockByNum gets block by number and also returns all related private data
// that requesting peer is eligible for.
// The order of private data in slice of PvtDataCollections doesn't imply the order of
// transactions in the block related to these private data, to get the correct placement
// need to read TxPvtData.SeqInBlock field
GetPvtDataAndBlockByNum(seqNum uint64, peerAuth protoutil.SignedData) (*common.Block, util.PvtDataCollections, error)
// Get recent block sequence number
LedgerHeight() (uint64, error)
// Close coordinator, shuts down coordinator service
Close()
}
Private Data Puller
A puller is created to listen on PrivateDataMsg from Gossip network.
- handleRequest: make response mesage and send back
- handleResponse: publish the hashes of the elements in the received message
PvtDataReconciler
Reconciler starts pulling the missing private data preiodically. It sends the request to the Gossip network, meanwhile subscribes the HASH of the reuqests to PubSub service, and then waiting for the response from Puller. In the end, the private data wil be committed by CommitPvtDataOfOldBlocks()
type Reconciler struct {
channel string
metrics *metrics.PrivdataMetrics
ReconcileSleepInterval time.Duration
ReconcileBatchSize int
stopChan chan struct{}
startOnce sync.Once
stopOnce sync.Once
ReconciliationFetcher
committer.Committer
}
func (r *Reconciler) run() {
for {
select {
case <-r.stopChan:
return
case <-time.After(r.ReconcileSleepInterval):
logger.Debug("Start reconcile missing private info")
if err := r.reconcile(); err != nil {
logger.Error("Failed to reconcile missing private info, error: ", err.Error())
break
}
}
}
}
PvtDataDistributor
It is ued to distibute private data when endorser starts to simulate proposal.
// PvtDataDistributor interface to defines API of distributing private data
type PvtDataDistributor interface {
// Distribute broadcast reliably private data read write set based on policies
Distribute(txID string, privData *transientstore.TxPvtReadWriteSetWithConfigInfo, blkHt uint64) error
}
There is an ACK mechnism in PvtDataDistributor. That is, any private data messages distributed require ACK responding fromt the StateProvider when it processes Gossip message with private data.
State Provider
GossipStateProvider is the interface to acquire sequences of the ledger blocks. It is capable to full fill missing blocks by running state replication and sending request to get missing block to other nodes.
StateProvider will set up filter to Gossip Comm to only accept message belonged to the specific channel ID:
- Data GossipMessage
- State Change
- PrivateData GossipMessage
StateProvider starts some goroutines in the end:
// Listen for incoming communication
go s.receiveAndQueueGossipMessages(gossipChan)
go s.receiveAndDispatchDirectMessages(commChan)
// Deliver in order messages into the incoming channel
go s.deliverPayloads()
if s.config.StateEnabled {
// Execute anti entropy to fill missing gaps
go s.antiEntropy()
}
// Taking care of state request messages
go s.processStateRequests()
- receiveAndQueueGossipMessages: receive payload message and push to local buffer
- receiveAndDispatchDirectMessages: receive state change req/rsp message and do further processing; receive private data, and store to transient store
- deliverPayloads: Receiver block and private data, commit to ledger; update the other peers with the new block height via Gossip network
- antiEntropy: check the if our height is lower than the known height of all peers. Otherwise, start to fix the gap by sending State Request message via Gossip network. The State Response will be handled and directed by receiveAndDispatchDirectMessages In the end, blocks in the Response will be added. This is to allow the lagging peer to catch up with the network in speed.
- processStateRequests: handle State Request and send back the Response with the requested blocks
In summary, StateProvider will try to sync up the ledger with the network, by processing Gossip Message with block payload, or by requesting the missing block gap with the State Message. Internally StateProvider is using a buffer mechanism to handle the block payloads
Deliver Client
The delivery service instance, is used to establish connection to the specified in the configuration ordering service. It tries to set up a gRPC connection to Orderer’s Deliver service.
// StartDeliverForChannel starts blocks delivery for channel
// initializes the grpc stream for given chainID, creates blocks provider instance
// that spawns in go routine to read new blocks starting from the position provided by ledger
// info instance.
func (d *deliverServiceImpl) StartDeliverForChannel(chainID string, ledgerInfo blocksprovider.LedgerInfo, finalizer func()) error {
d.lock.Lock()
defer d.lock.Unlock()
if d.stopping {
errMsg := fmt.Sprintf("Delivery service is stopping cannot join a new channel %s", chainID)
logger.Errorf(errMsg)
return errors.New(errMsg)
}
if _, exist := d.blockProviders[chainID]; exist {
errMsg := fmt.Sprintf("Delivery service - block provider already exists for %s found, can't start delivery", chainID)
logger.Errorf(errMsg)
return errors.New(errMsg)
}
logger.Info("This peer will retrieve blocks from ordering service and disseminate to other peers in the organization for channel", chainID)
dc := &blocksprovider.Deliverer{
ChannelID: chainID,
Gossip: d.conf.Gossip,
Ledger: ledgerInfo,
BlockVerifier: d.conf.CryptoSvc,
Dialer: DialerAdapter{
Client: d.conf.DeliverGRPCClient,
},
Orderers: d.conf.OrdererSource,
DoneC: make(chan struct{}),
Signer: d.conf.Signer,
DeliverStreamer: DeliverAdapter{},
Logger: flogging.MustGetLogger("peer.blocksprovider").With("channel", chainID),
MaxRetryDelay: time.Duration(d.conf.DeliverServiceConfig.ReConnectBackoffThreshold),
MaxRetryDuration: d.conf.DeliverServiceConfig.ReconnectTotalTimeThreshold,
InitialRetryDelay: 100 * time.Millisecond,
YieldLeadership: !d.conf.IsStaticLeader,
}
if d.conf.DeliverGRPCClient.MutualTLSRequired() {
dc.TLSCertHash = util.ComputeSHA256(d.conf.DeliverGRPCClient.Certificate().Certificate[0])
}
d.blockProviders[chainID] = dc
go func() {
dc.DeliverBlocks()
finalizer()
}()
return nil
}
func (d *Deliverer) DeliverBlocks() {
failureCounter := 0
totalDuration := time.Duration(0)
// InitialRetryDelay * backoffExponentBase^n > MaxRetryDelay
// backoffExponentBase^n > MaxRetryDelay / InitialRetryDelay
// n * log(backoffExponentBase) > log(MaxRetryDelay / InitialRetryDelay)
// n > log(MaxRetryDelay / InitialRetryDelay) / log(backoffExponentBase)
maxFailures := int(math.Log(float64(d.MaxRetryDelay)/float64(d.InitialRetryDelay)) / math.Log(backoffExponentBase))
for {
select {
case <-d.DoneC:
return
default:
}
if failureCounter > 0 {
var sleepDuration time.Duration
if failureCounter-1 > maxFailures {
sleepDuration = d.MaxRetryDelay
} else {
sleepDuration = time.Duration(math.Pow(1.2, float64(failureCounter-1))*100) * time.Millisecond
}
totalDuration += sleepDuration
if totalDuration > d.MaxRetryDuration {
if d.YieldLeadership {
d.Logger.Warningf("attempted to retry block delivery for more than %v, giving up", d.MaxRetryDuration)
return
}
d.Logger.Warningf("peer is a static leader, ignoring peer.deliveryclient.reconnectTotalTimeThreshold")
}
d.sleeper.Sleep(sleepDuration, d.DoneC)
}
ledgerHeight, err := d.Ledger.LedgerHeight()
if err != nil {
d.Logger.Error("Did not return ledger height, something is critically wrong", err)
return
}
seekInfoEnv, err := d.createSeekInfo(ledgerHeight)
if err != nil {
d.Logger.Error("Could not create a signed Deliver SeekInfo message, something is critically wrong", err)
return
}
deliverClient, endpoint, cancel, err := d.connect(seekInfoEnv)
if err != nil {
d.Logger.Warningf("Could not connect to ordering service: %s", err)
failureCounter++
continue
}
connLogger := d.Logger.With("orderer-address", endpoint.Address)
recv := make(chan *orderer.DeliverResponse)
go func() {
for {
resp, err := deliverClient.Recv()
if err != nil {
connLogger.Warningf("Encountered an error reading from deliver stream: %s", err)
close(recv)
return
}
select {
case recv <- resp:
case <-d.DoneC:
close(recv)
return
}
}
}()
RecvLoop: // Loop until the endpoint is refreshed, or there is an error on the connection
for {
select {
case <-endpoint.Refreshed:
connLogger.Infof("Ordering endpoints have been refreshed, disconnecting from deliver to reconnect using updated endpoints")
break RecvLoop
case response, ok := <-recv:
if !ok {
connLogger.Warningf("Orderer hung up without sending status")
failureCounter++
break RecvLoop
}
err = d.processMsg(response)
if err != nil {
connLogger.Warningf("Got error while attempting to receive blocks: %v", err)
failureCounter++
break RecvLoop
}
failureCounter = 0
case <-d.DoneC:
break RecvLoop
}
}
// cancel and wait for our spawned go routine to exit
cancel()
<-recv
}
}
DeliverBlocks() is used to pull out blocks from the ordering service to distributed them across peers. It will set up the gRPC client, dial onto Orderer Deliver service, seek for the block with the specific height. Any valid block response will be handled by GossipStateProvider.AddBlock(), and then gossipped to the network.
...
gossipMsg := &gossip.GossipMessage{
Nonce: 0,
Tag: gossip.GossipMessage_CHAN_AND_ORG,
Channel: []byte(d.ChannelID),
Content: &gossip.GossipMessage_DataMsg{
DataMsg: &gossip.DataMessage{
Payload: payload,
},
},
}
// Add payload to local state payloads buffer
if err := d.Gossip.AddPayload(d.ChannelID, payload); err != nil {
d.Logger.Warningf("Block [%d] received from ordering service wasn't added to payload buffer: %v", blockNum, err)
return errors.WithMessage(err, "could not add block as payload")
}
// Gossip messages with other nodes
d.Logger.Debugf("Gossiping block [%d]", blockNum)
d.Gossip.Gossip(gossipMsg)
Leader Election
It is used to elect one peer per organization which will maintain connection with the ordering service and initiate distribution of newly arrived blocks across the peers of its own organization.
- Static — a system administrator manually configures a peer in an organization to be the leader.
- Dynamic — peers execute a leader election procedure to select one peer in an organization to become leader.
When a channel is initialized, newLeaderElectionComponent() will be invoked if Dynamic election is configured. Othwersize, Gossip will start the Deliver Client to Orderer if Static is configured.
ElectionAdpater
Details of how election works:
// Gossip leader election module
// Algorithm properties:
// - Peers break symmetry by comparing IDs
// - Each peer is either a leader or a follower,
// and the aim is to have exactly 1 leader if the membership view
// is the same for all peers
// - If the network is partitioned into 2 or more sets, the number of leaders
// is the number of network partitions, but when the partition heals,
// only 1 leader should be left eventually
// - Peers communicate by gossiping leadership proposal or declaration messages
// The Algorithm, in pseudo code:
//
//
// variables:
// leaderKnown = false
//
// Invariant:
// Peer listens for messages from remote peers
// and whenever it receives a leadership declaration,
// leaderKnown is set to true
//
// Startup():
// wait for membership view to stabilize, or for a leadership declaration is received
// or the startup timeout expires.
// goto SteadyState()
//
// SteadyState():
// while true:
// If leaderKnown is false:
// LeaderElection()
// If you are the leader:
// Broadcast leadership declaration
// If a leadership declaration was received from
// a peer with a lower ID,
// become a follower
// Else, you’re a follower:
// If haven’t received a leadership declaration within
// a time threshold:
// set leaderKnown to false
//
// LeaderElection():
// Gossip leadership proposal message
// Collect messages from other peers sent within a time period
// If received a leadership declaration:
// return
// Iterate over all proposal messages collected.
// If a proposal message from a peer with an ID lower
// than yourself was received, return.
// Else, declare yourself a leader
// LeaderElectionAdapter is used by the leader election module
// to send and receive messages and to get membership information
type LeaderElectionAdapter interface {
// Gossip gossips a message to other peers
Gossip(Msg)
// Accept returns a channel that emits messages
Accept() <-chan Msg
// CreateProposalMessage
CreateMessage(isDeclaration bool) Msg
// Peers returns a list of peers considered alive
Peers() []Peer
// ReportMetrics sends a report to the metrics server about a leadership status
ReportMetrics(isLeader bool)
}
func (ai *adapterImpl) Accept() <-chan Msg {
adapterCh, _ := ai.gossip.Accept(func(message interface{}) bool {
// Get only leadership org and channel messages
return message.(*proto.GossipMessage).Tag == proto.GossipMessage_CHAN_AND_ORG &&
protoext.IsLeadershipMsg(message.(*proto.GossipMessage)) &&
bytes.Equal(message.(*proto.GossipMessage).Channel, ai.channel)
}, false)
msgCh := make(chan Msg)
...
Election will use the Gossip Service to archieve the election purpose. The adapter itself is created with the Gossip service, but only handle leadership org and channel messages
adapter := election.NewAdapter(g, PKIid, gossipcommon.ChannelID(channelID), electionMetrics)
When adapter wants to propose/declare leadership, it asks Gossip Service to send the message to other gossip nodes.
// NewLeaderElectionService returns a new LeaderElectionService
func NewLeaderElectionService(adapter LeaderElectionAdapter, id string, callback leadershipCallback, config ElectionConfig) LeaderElectionService {
if len(id) == 0 {
panic("Empty id")
}
le := &leaderElectionSvcImpl{
id: peerID(id),
proposals: util.NewSet(),
adapter: adapter,
stopChan: make(chan struct{}),
interruptChan: make(chan struct{}, 1),
logger: util.GetLogger(util.ElectionLogger, ""),
callback: noopCallback,
config: config,
}
if callback != nil {
le.callback = callback
}
go le.start()
return le
}
func (le *leaderElectionSvcImpl) start() {
le.stopWG.Add(2)
go le.handleMessages()
le.waitForMembershipStabilization(le.config.StartupGracePeriod)
go le.run()
}
beLeader
Once becoming the leader, it will invoke onStatusChangeFactory() to start the deliver client.
func (g *GossipService) onStatusChangeFactory(channelID string, committer blocksprovider.LedgerInfo) func(bool) {
return func(isLeader bool) {
if isLeader {
yield := func() {
g.lock.RLock()
le := g.leaderElection[channelID]
g.lock.RUnlock()
le.Yield()
}
logger.Info("Elected as a leader, starting delivery service for channel", channelID)
if err := g.deliveryService[channelID].StartDeliverForChannel(channelID, committer, yield); err != nil {
logger.Errorf("Delivery service is not able to start blocks delivery for chain, due to %+v", err)
}
} else {
logger.Info("Renounced leadership, stopping delivery service for channel", channelID)
if err := g.deliveryService[channelID].StopDeliverForChannel(channelID); err != nil {
logger.Errorf("Delivery service is not able to stop blocks delivery for chain, due to %+v", err)
}
}
}
}
Be Leader call stack
github.com/hyperledger/fabric/core/deliverservice.(*deliverServiceImpl).StartDeliverForChannel at deliveryclient.go:109
github.com/hyperledger/fabric/gossip/service.(*GossipService).onStatusChangeFactory.func1 at gossip_service.go:488
github.com/hyperledger/fabric/gossip/election.(*leaderElectionSvcImpl).beLeader at election.go:400
github.com/hyperledger/fabric/gossip/election.(*leaderElectionSvcImpl).leaderElection at election.go:319
github.com/hyperledger/fabric/gossip/election.(*leaderElectionSvcImpl).run at election.go:268
runtime.goexit at asm_amd64.s:1357
- Async stack trace
github.com/hyperledger/fabric/gossip/election.(*leaderElectionSvcImpl).start at election.go:193