From a51bf486c8ba1a718764ea17d9e7794b53058962 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Thu, 6 Apr 2023 20:13:22 +0300 Subject: [PATCH 01/38] Extracted basic protocol violations into a separate interface. Updated structuring of consensus related consumers. Fixed tests and dependant code --- .../node_builder/access_node_builder.go | 2 +- cmd/verification_builder.go | 2 +- consensus/hotstuff/consumer.go | 32 ++++++++++++---- .../mocks/base_protocol_violation_consumer.go | 38 +++++++++++++++++++ consensus/hotstuff/mocks/consumer.go | 5 +++ .../hotstuff/mocks/finalization_consumer.go | 5 +++ consensus/hotstuff/model/errors.go | 14 ++++--- .../hotstuff/notifications/log_consumer.go | 11 ++++++ .../hotstuff/notifications/noop_consumer.go | 16 ++++++-- .../notifications/pubsub/distributor.go | 8 ++++ consensus/hotstuff/validator/validator.go | 23 ++++++----- consensus/hotstuff/votecollector/factory.go | 5 +-- consensus/recovery/recover_test.go | 16 ++++---- engine/access/ingestion/engine.go | 8 ---- .../common/follower/compliance_core_test.go | 8 +++- .../assigner/blockconsumer/consumer.go | 6 --- 16 files changed, 143 insertions(+), 56 deletions(-) create mode 100644 consensus/hotstuff/mocks/base_protocol_violation_consumer.go diff --git a/cmd/access/node_builder/access_node_builder.go b/cmd/access/node_builder/access_node_builder.go index a1dea36958d..19be7a8268d 100644 --- a/cmd/access/node_builder/access_node_builder.go +++ b/cmd/access/node_builder/access_node_builder.go @@ -962,7 +962,7 @@ func (builder *FlowAccessNodeBuilder) Build() (cmd.Node, error) { return nil, err } builder.RequestEng.WithHandle(builder.IngestEng.OnCollection) - builder.FinalizationDistributor.AddConsumer(builder.IngestEng) + builder.FinalizationDistributor.AddOnBlockFinalizedConsumer(builder.IngestEng.OnFinalizedBlock) return builder.IngestEng, nil }). diff --git a/cmd/verification_builder.go b/cmd/verification_builder.go index 52e0438d8b5..cf10a8b01a5 100644 --- a/cmd/verification_builder.go +++ b/cmd/verification_builder.go @@ -346,7 +346,7 @@ func (v *VerificationNodeBuilder) LoadComponentsAndModules() { return nil, fmt.Errorf("could not find latest finalized block and pending blocks to recover consensus follower: %w", err) } - finalizationDistributor.AddConsumer(blockConsumer) + finalizationDistributor.AddOnBlockFinalizedConsumer(blockConsumer.OnFinalizedBlock) // creates a consensus follower with ingestEngine as the notifier // so that it gets notified upon each new finalized block diff --git a/consensus/hotstuff/consumer.go b/consensus/hotstuff/consumer.go index 5eb592b9912..39087594ce0 100644 --- a/consensus/hotstuff/consumer.go +++ b/consensus/hotstuff/consumer.go @@ -7,6 +7,30 @@ import ( "github.com/onflow/flow-go/model/flow" ) +// BaseProtocolViolationConsumer consumes outbound notifications produced by compliance. +// Notifications can be produced by consensus participants and followers. +// Notifications are meant to report protocol violations that can be observed by executing compliance checks. +// +// Implementations must: +// - be concurrency safe +// - be non-blocking +// - handle repetition of the same events (with some processing overhead). +type BaseProtocolViolationConsumer interface { + // OnInvalidBlockDetected notifications are produced by components that have detected + // that a block proposal is invalid and need to report it. + // Most of the time such block can be detected by calling Validator.ValidateProposal. + // Prerequisites: + // Implementation must be concurrency safe; Non-blocking; + // and must handle repetition of the same events (with some processing overhead). + OnInvalidBlockDetected(err model.InvalidBlockError) + // OnDoubleProposeDetected notifications are produced by the Finalization Logic + // whenever a double block proposal (equivocation) was detected. + // Prerequisites: + // Implementation must be concurrency safe; Non-blocking; + // and must handle repetition of the same events (with some processing overhead). + OnDoubleProposeDetected(*model.Block, *model.Block) +} + // FinalizationConsumer consumes outbound notifications produced by the finalization logic. // Notifications represent finalization-specific state changes which are potentially relevant // to the larger node. The notifications are emitted in the order in which the @@ -17,6 +41,7 @@ import ( // - be non-blocking // - handle repetition of the same events (with some processing overhead). type FinalizationConsumer interface { + BaseProtocolViolationConsumer // OnBlockIncorporated notifications are produced by the Finalization Logic // whenever a block is incorporated into the consensus state. @@ -31,13 +56,6 @@ type FinalizationConsumer interface { // Implementation must be concurrency safe; Non-blocking; // and must handle repetition of the same events (with some processing overhead). OnFinalizedBlock(*model.Block) - - // OnDoubleProposeDetected notifications are produced by the Finalization Logic - // whenever a double block proposal (equivocation) was detected. - // Prerequisites: - // Implementation must be concurrency safe; Non-blocking; - // and must handle repetition of the same events (with some processing overhead). - OnDoubleProposeDetected(*model.Block, *model.Block) } // Consumer consumes outbound notifications produced by HotStuff and its components. diff --git a/consensus/hotstuff/mocks/base_protocol_violation_consumer.go b/consensus/hotstuff/mocks/base_protocol_violation_consumer.go new file mode 100644 index 00000000000..4971e14f6e8 --- /dev/null +++ b/consensus/hotstuff/mocks/base_protocol_violation_consumer.go @@ -0,0 +1,38 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mocks + +import ( + model "github.com/onflow/flow-go/consensus/hotstuff/model" + mock "github.com/stretchr/testify/mock" +) + +// BaseProtocolViolationConsumer is an autogenerated mock type for the BaseProtocolViolationConsumer type +type BaseProtocolViolationConsumer struct { + mock.Mock +} + +// OnDoubleProposeDetected provides a mock function with given fields: _a0, _a1 +func (_m *BaseProtocolViolationConsumer) OnDoubleProposeDetected(_a0 *model.Block, _a1 *model.Block) { + _m.Called(_a0, _a1) +} + +// OnInvalidBlockDetected provides a mock function with given fields: err +func (_m *BaseProtocolViolationConsumer) OnInvalidBlockDetected(err model.InvalidBlockError) { + _m.Called(err) +} + +type mockConstructorTestingTNewBaseProtocolViolationConsumer interface { + mock.TestingT + Cleanup(func()) +} + +// NewBaseProtocolViolationConsumer creates a new instance of BaseProtocolViolationConsumer. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewBaseProtocolViolationConsumer(t mockConstructorTestingTNewBaseProtocolViolationConsumer) *BaseProtocolViolationConsumer { + mock := &BaseProtocolViolationConsumer{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/consensus/hotstuff/mocks/consumer.go b/consensus/hotstuff/mocks/consumer.go index ee991cee08e..919d333384d 100644 --- a/consensus/hotstuff/mocks/consumer.go +++ b/consensus/hotstuff/mocks/consumer.go @@ -53,6 +53,11 @@ func (_m *Consumer) OnFinalizedBlock(_a0 *model.Block) { _m.Called(_a0) } +// OnInvalidBlockDetected provides a mock function with given fields: err +func (_m *Consumer) OnInvalidBlockDetected(err model.InvalidBlockError) { + _m.Called(err) +} + // OnInvalidTimeoutDetected provides a mock function with given fields: err func (_m *Consumer) OnInvalidTimeoutDetected(err model.InvalidTimeoutError) { _m.Called(err) diff --git a/consensus/hotstuff/mocks/finalization_consumer.go b/consensus/hotstuff/mocks/finalization_consumer.go index 5c5a5f4b922..bba788508a0 100644 --- a/consensus/hotstuff/mocks/finalization_consumer.go +++ b/consensus/hotstuff/mocks/finalization_consumer.go @@ -27,6 +27,11 @@ func (_m *FinalizationConsumer) OnFinalizedBlock(_a0 *model.Block) { _m.Called(_a0) } +// OnInvalidBlockDetected provides a mock function with given fields: err +func (_m *FinalizationConsumer) OnInvalidBlockDetected(err model.InvalidBlockError) { + _m.Called(err) +} + type mockConstructorTestingTNewFinalizationConsumer interface { mock.TestingT Cleanup(func()) diff --git a/consensus/hotstuff/model/errors.go b/consensus/hotstuff/model/errors.go index 85a05338d35..c296d9f8f9a 100644 --- a/consensus/hotstuff/model/errors.go +++ b/consensus/hotstuff/model/errors.go @@ -163,15 +163,19 @@ func (e InvalidTCError) Unwrap() error { return e.Err } -// InvalidBlockError indicates that the block with identifier `BlockID` is invalid +// InvalidBlockError indicates that the block is invalid type InvalidBlockError struct { - BlockID flow.Identifier - View uint64 - Err error + InvalidBlock *Proposal + Err error } func (e InvalidBlockError) Error() string { - return fmt.Sprintf("invalid block %x at view %d: %s", e.BlockID, e.View, e.Err.Error()) + return fmt.Sprintf( + "invalid block %x at view %d: %s", + e.InvalidBlock.Block.BlockID, + e.InvalidBlock.Block.View, + e.Err.Error(), + ) } // IsInvalidBlockError returns whether an error is InvalidBlockError diff --git a/consensus/hotstuff/notifications/log_consumer.go b/consensus/hotstuff/notifications/log_consumer.go index 0f3329c356d..64e76e3d34e 100644 --- a/consensus/hotstuff/notifications/log_consumer.go +++ b/consensus/hotstuff/notifications/log_consumer.go @@ -45,6 +45,17 @@ func (lc *LogConsumer) OnFinalizedBlock(block *model.Block) { Msg("block finalized") } +func (lc *LogConsumer) OnInvalidBlockDetected(err model.InvalidBlockError) { + invalidBlock := err.InvalidBlock.Block + lc.log.Warn(). + Uint64("block_view", invalidBlock.View). + Hex("proposer_id", invalidBlock.ProposerID[:]). + Hex("block_id", invalidBlock.BlockID[:]). + Uint64("qc_block_view", invalidBlock.QC.View). + Hex("qc_block_id", invalidBlock.QC.BlockID[:]). + Msgf("invalid block detected: %s", err.Error()) +} + func (lc *LogConsumer) OnDoubleProposeDetected(block *model.Block, alt *model.Block) { lc.log.Warn(). Uint64("block_view", block.View). diff --git a/consensus/hotstuff/notifications/noop_consumer.go b/consensus/hotstuff/notifications/noop_consumer.go index b5d980acdd3..7e25d026d4e 100644 --- a/consensus/hotstuff/notifications/noop_consumer.go +++ b/consensus/hotstuff/notifications/noop_consumer.go @@ -67,7 +67,9 @@ func (*NoopPartialConsumer) OnInvalidTimeoutDetected(model.InvalidTimeoutError) // no-op implementation of hotstuff.FinalizationConsumer -type NoopFinalizationConsumer struct{} +type NoopFinalizationConsumer struct { + NoopBaseProtocolViolationConsumer +} var _ hotstuff.FinalizationConsumer = (*NoopFinalizationConsumer)(nil) @@ -75,8 +77,6 @@ func (*NoopFinalizationConsumer) OnBlockIncorporated(*model.Block) {} func (*NoopFinalizationConsumer) OnFinalizedBlock(*model.Block) {} -func (*NoopFinalizationConsumer) OnDoubleProposeDetected(*model.Block, *model.Block) {} - // no-op implementation of hotstuff.TimeoutCollectorConsumer type NoopTimeoutCollectorConsumer struct{} @@ -111,3 +111,13 @@ type NoopQCCreatedConsumer struct{} var _ hotstuff.QCCreatedConsumer = (*NoopQCCreatedConsumer)(nil) func (*NoopQCCreatedConsumer) OnQcConstructedFromVotes(*flow.QuorumCertificate) {} + +// no-op implementation of hotstuff.BaseProtocolViolationConsumer + +type NoopBaseProtocolViolationConsumer struct{} + +var _ hotstuff.BaseProtocolViolationConsumer = (*NoopBaseProtocolViolationConsumer)(nil) + +func (n NoopBaseProtocolViolationConsumer) OnInvalidBlockDetected(model.InvalidBlockError) {} + +func (n NoopBaseProtocolViolationConsumer) OnDoubleProposeDetected(*model.Block, *model.Block) {} diff --git a/consensus/hotstuff/notifications/pubsub/distributor.go b/consensus/hotstuff/notifications/pubsub/distributor.go index d122ad8cde3..151c9671ba5 100644 --- a/consensus/hotstuff/notifications/pubsub/distributor.go +++ b/consensus/hotstuff/notifications/pubsub/distributor.go @@ -158,6 +158,14 @@ func (p *Distributor) OnFinalizedBlock(block *model.Block) { } } +func (p *Distributor) OnInvalidBlockDetected(err model.InvalidBlockError) { + p.lock.RLock() + defer p.lock.RUnlock() + for _, subscriber := range p.subscribers { + subscriber.OnInvalidBlockDetected(err) + } +} + func (p *Distributor) OnDoubleProposeDetected(block1, block2 *model.Block) { p.lock.RLock() defer p.lock.RUnlock() diff --git a/consensus/hotstuff/validator/validator.go b/consensus/hotstuff/validator/validator.go index f52366ad540..d882d834571 100644 --- a/consensus/hotstuff/validator/validator.go +++ b/consensus/hotstuff/validator/validator.go @@ -208,7 +208,7 @@ func (v *Validator) ValidateProposal(proposal *model.Proposal) error { // validate the proposer's vote and get his identity _, err := v.ValidateVote(proposal.ProposerVote()) if model.IsInvalidVoteError(err) { - return newInvalidBlockError(block, fmt.Errorf("invalid proposer signature: %w", err)) + return newInvalidBlockError(proposal, fmt.Errorf("invalid proposer signature: %w", err)) } if err != nil { return fmt.Errorf("error verifying leader signature for block %x: %w", block.BlockID, err) @@ -220,7 +220,7 @@ func (v *Validator) ValidateProposal(proposal *model.Proposal) error { return fmt.Errorf("error determining leader for block %x: %w", block.BlockID, err) } if leader != block.ProposerID { - return newInvalidBlockError(block, fmt.Errorf("proposer %s is not leader (%s) for view %d", block.ProposerID, leader, block.View)) + return newInvalidBlockError(proposal, fmt.Errorf("proposer %s is not leader (%s) for view %d", block.ProposerID, leader, block.View)) } // The Block must contain a proof that the primary legitimately entered the respective view. @@ -231,23 +231,23 @@ func (v *Validator) ValidateProposal(proposal *model.Proposal) error { if !lastViewSuccessful { // check if proposal is correctly structured if proposal.LastViewTC == nil { - return newInvalidBlockError(block, fmt.Errorf("QC in block is not for previous view, so expecting a TC but none is included in block")) + return newInvalidBlockError(proposal, fmt.Errorf("QC in block is not for previous view, so expecting a TC but none is included in block")) } // check if included TC is for previous view if proposal.Block.View != proposal.LastViewTC.View+1 { - return newInvalidBlockError(block, fmt.Errorf("QC in block is not for previous view, so expecting a TC for view %d but got TC for view %d", proposal.Block.View-1, proposal.LastViewTC.View)) + return newInvalidBlockError(proposal, fmt.Errorf("QC in block is not for previous view, so expecting a TC for view %d but got TC for view %d", proposal.Block.View-1, proposal.LastViewTC.View)) } // Check if proposal extends either the newest QC specified in the TC, or a newer QC // in edge cases a leader may construct a TC and QC concurrently such that TC contains // an older QC - in these case we still want to build on the newest QC, so this case is allowed. if proposal.Block.QC.View < proposal.LastViewTC.NewestQC.View { - return newInvalidBlockError(block, fmt.Errorf("TC in block contains a newer QC than the block itself, which is a protocol violation")) + return newInvalidBlockError(proposal, fmt.Errorf("TC in block contains a newer QC than the block itself, which is a protocol violation")) } } else if proposal.LastViewTC != nil { // last view ended with QC, including TC is a protocol violation - return newInvalidBlockError(block, fmt.Errorf("last view has ended with QC but proposal includes LastViewTC")) + return newInvalidBlockError(proposal, fmt.Errorf("last view has ended with QC but proposal includes LastViewTC")) } // Check signatures, keep the most expensive the last to check @@ -256,7 +256,7 @@ func (v *Validator) ValidateProposal(proposal *model.Proposal) error { err = v.ValidateQC(qc) if err != nil { if model.IsInvalidQCError(err) { - return newInvalidBlockError(block, fmt.Errorf("invalid qc included: %w", err)) + return newInvalidBlockError(proposal, fmt.Errorf("invalid qc included: %w", err)) } if errors.Is(err, model.ErrViewForUnknownEpoch) { // We require each replica to be bootstrapped with a QC pointing to a finalized block. Therefore, we should know the @@ -272,7 +272,7 @@ func (v *Validator) ValidateProposal(proposal *model.Proposal) error { err = v.ValidateTC(proposal.LastViewTC) if err != nil { if model.IsInvalidTCError(err) { - return newInvalidBlockError(block, fmt.Errorf("proposals TC's is not valid: %w", err)) + return newInvalidBlockError(proposal, fmt.Errorf("proposals TC's is not valid: %w", err)) } if errors.Is(err, model.ErrViewForUnknownEpoch) { // We require each replica to be bootstrapped with a QC pointing to a finalized block. Therefore, we should know the @@ -323,11 +323,10 @@ func (v *Validator) ValidateVote(vote *model.Vote) (*flow.Identity, error) { return voter, nil } -func newInvalidBlockError(block *model.Block, err error) error { +func newInvalidBlockError(proposal *model.Proposal, err error) error { return model.InvalidBlockError{ - BlockID: block.BlockID, - View: block.View, - Err: err, + InvalidBlock: proposal, + Err: err, } } diff --git a/consensus/hotstuff/votecollector/factory.go b/consensus/hotstuff/votecollector/factory.go index 31d36119978..1b5dda84103 100644 --- a/consensus/hotstuff/votecollector/factory.go +++ b/consensus/hotstuff/votecollector/factory.go @@ -50,9 +50,8 @@ func (f *VoteProcessorFactory) Create(log zerolog.Logger, proposal *model.Propos if err != nil { if model.IsInvalidVoteError(err) { return nil, model.InvalidBlockError{ - BlockID: proposal.Block.BlockID, - View: proposal.Block.View, - Err: fmt.Errorf("invalid proposer vote: %w", err), + InvalidBlock: proposal, + Err: fmt.Errorf("invalid proposer vote: %w", err), } } return nil, fmt.Errorf("processing proposer's vote for block %v failed: %w", proposal.Block.BlockID, err) diff --git a/consensus/recovery/recover_test.go b/consensus/recovery/recover_test.go index 3f337fb6da0..c8b12c65c5e 100644 --- a/consensus/recovery/recover_test.go +++ b/consensus/recovery/recover_test.go @@ -1,6 +1,7 @@ package recovery import ( + "fmt" "testing" "github.com/stretchr/testify/mock" @@ -27,21 +28,20 @@ func TestRecover(t *testing.T) { } // make 3 invalid blocks extend from the last valid block - invalidblocks := unittest.ChainFixtureFrom(3, pending[len(pending)-1]) + invalidBlocks := unittest.ChainFixtureFrom(3, pending[len(pending)-1]) invalid := make(map[flow.Identifier]struct{}) - for _, b := range invalidblocks { + for _, b := range invalidBlocks { invalid[b.ID()] = struct{}{} pending = append(pending, b.Header) } validator := &mocks.Validator{} validator.On("ValidateProposal", mock.Anything).Return(func(proposal *model.Proposal) error { - header := model.ProposalToFlow(proposal) - _, isInvalid := invalid[header.ID()] + _, isInvalid := invalid[proposal.Block.BlockID] if isInvalid { - return &model.InvalidBlockError{ - BlockID: header.ID(), - View: header.View, + return model.InvalidBlockError{ + InvalidBlock: proposal, + Err: fmt.Errorf(""), } } return nil @@ -51,5 +51,5 @@ func TestRecover(t *testing.T) { require.NoError(t, err) // only pending blocks are valid - require.Len(t, recovered, len(pending)) + require.Len(t, recovered, len(pending)-len(invalidBlocks)) } diff --git a/engine/access/ingestion/engine.go b/engine/access/ingestion/engine.go index 58b0617a2bd..a14ffd45034 100644 --- a/engine/access/ingestion/engine.go +++ b/engine/access/ingestion/engine.go @@ -575,14 +575,6 @@ func (e *Engine) OnCollection(originID flow.Identifier, entity flow.Entity) { } } -// OnBlockIncorporated is a noop for this engine since access node is only dealing with finalized blocks -func (e *Engine) OnBlockIncorporated(*model.Block) { -} - -// OnDoubleProposeDetected is a noop for this engine since access node is only dealing with finalized blocks -func (e *Engine) OnDoubleProposeDetected(*model.Block, *model.Block) { -} - // requestMissingCollections requests missing collections for all blocks in the local db storage once at startup func (e *Engine) requestMissingCollections(ctx context.Context) error { diff --git a/engine/common/follower/compliance_core_test.go b/engine/common/follower/compliance_core_test.go index 38c857d8974..bea663f9b69 100644 --- a/engine/common/follower/compliance_core_test.go +++ b/engine/common/follower/compliance_core_test.go @@ -165,12 +165,16 @@ func (s *CoreSuite) TestProcessingNotOrderedBatch() { func (s *CoreSuite) TestProcessingInvalidBlock() { blocks := unittest.ChainFixtureFrom(10, s.finalizedBlock) - s.validator.On("ValidateProposal", model.ProposalFromFlow(blocks[len(blocks)-1].Header)).Return(model.InvalidBlockError{Err: fmt.Errorf("")}).Once() + invalidProposal := model.ProposalFromFlow(blocks[len(blocks)-1].Header) + s.validator.On("ValidateProposal", invalidProposal).Return(model.InvalidBlockError{ + InvalidBlock: invalidProposal, + Err: fmt.Errorf(""), + }).Once() err := s.core.OnBlockRange(s.originID, blocks) require.NoError(s.T(), err, "sentinel error has to be handled internally") exception := errors.New("validate-proposal-exception") - s.validator.On("ValidateProposal", model.ProposalFromFlow(blocks[len(blocks)-1].Header)).Return(exception).Once() + s.validator.On("ValidateProposal", invalidProposal).Return(exception).Once() err = s.core.OnBlockRange(s.originID, blocks) require.ErrorIs(s.T(), err, exception, "exception has to be propagated") } diff --git a/engine/verification/assigner/blockconsumer/consumer.go b/engine/verification/assigner/blockconsumer/consumer.go index e0913a45fa6..982fe418688 100644 --- a/engine/verification/assigner/blockconsumer/consumer.go +++ b/engine/verification/assigner/blockconsumer/consumer.go @@ -98,12 +98,6 @@ func (c *BlockConsumer) OnFinalizedBlock(*model.Block) { c.unit.Launch(c.consumer.Check) } -// OnBlockIncorporated is to implement FinalizationConsumer -func (c *BlockConsumer) OnBlockIncorporated(*model.Block) {} - -// OnDoubleProposeDetected is to implement FinalizationConsumer -func (c *BlockConsumer) OnDoubleProposeDetected(*model.Block, *model.Block) {} - func (c *BlockConsumer) Ready() <-chan struct{} { err := c.consumer.Start(c.defaultIndex) if err != nil { From 99007dd5179dc38d166a259b1241e015ff5b164f Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Fri, 7 Apr 2023 16:28:43 +0300 Subject: [PATCH 02/38] Refactored consumer interfaces to be nested by event type. Updated usages. Integrated BaseProtocolViolationConsumer in compliance engine of consensus follower --- .../node_builder/access_node_builder.go | 4 +- cmd/collection/main.go | 4 +- cmd/consensus/main.go | 6 +- cmd/execution_builder.go | 4 +- cmd/observer/node_builder/observer_builder.go | 4 +- cmd/verification_builder.go | 4 +- consensus/aggregators.go | 2 +- consensus/config.go | 12 +-- consensus/follower.go | 2 +- consensus/hotstuff/consumer.go | 14 ++- consensus/hotstuff/forks/forks.go | 4 +- consensus/hotstuff/model/errors.go | 11 +++ .../hotstuff/notifications/noop_consumer.go | 9 +- .../notifications/pubsub/distributor.go | 19 +++++ .../pubsub/finalization_distributor.go | 79 ----------------- .../pubsub/follower_distributor.go | 85 +++++++++++++++++++ consensus/integration/nodes_test.go | 2 +- consensus/participant.go | 2 +- .../collection/epochmgr/factories/hotstuff.go | 8 +- engine/common/follower/compliance_core.go | 56 ++++++------ engine/common/follower/integration_test.go | 2 +- engine/common/synchronization/engine_test.go | 2 +- .../synchronization/finalized_snapshot.go | 2 +- engine/testutil/nodes.go | 2 +- follower/follower_builder.go | 4 +- .../execution_data_requester_test.go | 12 +-- 26 files changed, 200 insertions(+), 155 deletions(-) delete mode 100644 consensus/hotstuff/notifications/pubsub/finalization_distributor.go create mode 100644 consensus/hotstuff/notifications/pubsub/follower_distributor.go diff --git a/cmd/access/node_builder/access_node_builder.go b/cmd/access/node_builder/access_node_builder.go index 19be7a8268d..91b3859b35c 100644 --- a/cmd/access/node_builder/access_node_builder.go +++ b/cmd/access/node_builder/access_node_builder.go @@ -196,7 +196,7 @@ type FlowAccessNodeBuilder struct { FollowerState protocol.FollowerState SyncCore *chainsync.Core RpcEng *rpc.Engine - FinalizationDistributor *consensuspubsub.FinalizationDistributor + FinalizationDistributor *consensuspubsub.FollowerDistributor FinalizedHeader *synceng.FinalizedHeaderCache CollectionRPC access.AccessAPIClient TransactionTimings *stdmap.TransactionTimings @@ -576,7 +576,7 @@ func (builder *FlowAccessNodeBuilder) BuildExecutionDataRequester() *FlowAccessN } func FlowAccessNode(nodeBuilder *cmd.FlowNodeBuilder) *FlowAccessNodeBuilder { - dist := consensuspubsub.NewFinalizationDistributor() + dist := consensuspubsub.NewFollowerDistributor() dist.AddConsumer(notifications.NewSlashingViolationsConsumer(nodeBuilder.Logger)) return &FlowAccessNodeBuilder{ AccessNodeConfig: DefaultAccessNodeConfig(), diff --git a/cmd/collection/main.go b/cmd/collection/main.go index da7e946a98c..b070f362e3f 100644 --- a/cmd/collection/main.go +++ b/cmd/collection/main.go @@ -79,7 +79,7 @@ func main() { clusterComplianceConfig modulecompliance.Config pools *epochpool.TransactionPools // epoch-scoped transaction pools - finalizationDistributor *pubsub.FinalizationDistributor + finalizationDistributor *pubsub.FollowerDistributor finalizedHeader *consync.FinalizedHeaderCache push *pusher.Engine @@ -172,7 +172,7 @@ func main() { nodeBuilder. PreInit(cmd.DynamicStartPreInit). Module("finalization distributor", func(node *cmd.NodeConfig) error { - finalizationDistributor = pubsub.NewFinalizationDistributor() + finalizationDistributor = pubsub.NewFollowerDistributor() finalizationDistributor.AddConsumer(notifications.NewSlashingViolationsConsumer(node.Logger)) return nil }). diff --git a/cmd/consensus/main.go b/cmd/consensus/main.go index 077215a5235..059b8c89801 100644 --- a/cmd/consensus/main.go +++ b/cmd/consensus/main.go @@ -115,7 +115,7 @@ func main() { mainMetrics module.HotstuffMetrics receiptValidator module.ReceiptValidator chunkAssigner *chmodule.ChunkAssigner - finalizationDistributor *pubsub.FinalizationDistributor + finalizationDistributor *pubsub.FollowerDistributor dkgBrokerTunnel *dkgmodule.BrokerTunnel blockTimer protocol.BlockTimer finalizedHeader *synceng.FinalizedHeaderCache @@ -365,7 +365,7 @@ func main() { return err }). Module("finalization distributor", func(node *cmd.NodeConfig) error { - finalizationDistributor = pubsub.NewFinalizationDistributor() + finalizationDistributor = pubsub.NewFollowerDistributor() finalizationDistributor.AddConsumer(notifications.NewSlashingViolationsConsumer(nodeBuilder.Logger)) return nil }). @@ -560,7 +560,7 @@ func main() { mainMetrics, ) - notifier.AddConsumer(finalizationDistributor) + notifier.AddFollowerConsumer(finalizationDistributor) // initialize the persister persist := persister.New(node.DB, node.RootChainID) diff --git a/cmd/execution_builder.go b/cmd/execution_builder.go index 4499a2de684..cddd39cc95f 100644 --- a/cmd/execution_builder.go +++ b/cmd/execution_builder.go @@ -128,7 +128,7 @@ type ExecutionNode struct { computationManager *computation.Manager collectionRequester *requester.Engine ingestionEng *ingestion.Engine - finalizationDistributor *pubsub.FinalizationDistributor + finalizationDistributor *pubsub.FollowerDistributor finalizedHeader *synchronization.FinalizedHeaderCache checkAuthorizedAtBlock func(blockID flow.Identifier) (bool, error) diskWAL *wal.DiskWAL @@ -273,7 +273,7 @@ func (exeNode *ExecutionNode) LoadExecutionReceiptsStorage( } func (exeNode *ExecutionNode) LoadFinalizationDistributor(node *NodeConfig) error { - exeNode.finalizationDistributor = pubsub.NewFinalizationDistributor() + exeNode.finalizationDistributor = pubsub.NewFollowerDistributor() exeNode.finalizationDistributor.AddConsumer(notifications.NewSlashingViolationsConsumer(node.Logger)) return nil } diff --git a/cmd/observer/node_builder/observer_builder.go b/cmd/observer/node_builder/observer_builder.go index fc866e18df2..77948fd6d8f 100644 --- a/cmd/observer/node_builder/observer_builder.go +++ b/cmd/observer/node_builder/observer_builder.go @@ -170,7 +170,7 @@ type ObserverServiceBuilder struct { FollowerState stateprotocol.FollowerState SyncCore *chainsync.Core RpcEng *rpc.Engine - FinalizationDistributor *pubsub.FinalizationDistributor + FinalizationDistributor *pubsub.FollowerDistributor FinalizedHeader *synceng.FinalizedHeaderCache Committee hotstuff.DynamicCommittee Finalized *flow.Header @@ -567,7 +567,7 @@ func NewFlowObserverServiceBuilder(opts ...Option) *ObserverServiceBuilder { anb := &ObserverServiceBuilder{ ObserverServiceConfig: config, FlowNodeBuilder: cmd.FlowNode(flow.RoleAccess.String()), - FinalizationDistributor: pubsub.NewFinalizationDistributor(), + FinalizationDistributor: pubsub.NewFollowerDistributor(), } anb.FinalizationDistributor.AddConsumer(notifications.NewSlashingViolationsConsumer(anb.Logger)) // the observer gets a version of the root snapshot file that does not contain any node addresses diff --git a/cmd/verification_builder.go b/cmd/verification_builder.go index cf10a8b01a5..69e31f643b1 100644 --- a/cmd/verification_builder.go +++ b/cmd/verification_builder.go @@ -101,7 +101,7 @@ func (v *VerificationNodeBuilder) LoadComponentsAndModules() { verifierEng *verifier.Engine // the verifier engine chunkConsumer *chunkconsumer.ChunkConsumer blockConsumer *blockconsumer.BlockConsumer - finalizationDistributor *pubsub.FinalizationDistributor + finalizationDistributor *pubsub.FollowerDistributor finalizedHeader *commonsync.FinalizedHeaderCache committee *committees.Consensus @@ -178,7 +178,7 @@ func (v *VerificationNodeBuilder) LoadComponentsAndModules() { return nil }). Module("finalization distributor", func(node *NodeConfig) error { - finalizationDistributor = pubsub.NewFinalizationDistributor() + finalizationDistributor = pubsub.NewFollowerDistributor() finalizationDistributor.AddConsumer(notifications.NewSlashingViolationsConsumer(node.Logger)) return nil }). diff --git a/consensus/aggregators.go b/consensus/aggregators.go index 10bf86083c8..b5915b98697 100644 --- a/consensus/aggregators.go +++ b/consensus/aggregators.go @@ -25,7 +25,7 @@ func NewVoteAggregator( lowestRetainedView uint64, notifier hotstuff.Consumer, voteProcessorFactory hotstuff.VoteProcessorFactory, - distributor *pubsub.FinalizationDistributor, + distributor *pubsub.FollowerDistributor, ) (hotstuff.VoteAggregator, error) { createCollectorFactoryMethod := votecollector.NewStateMachineFactory(log, notifier, voteProcessorFactory.Create) diff --git a/consensus/config.go b/consensus/config.go index 8862ffd366e..cc99fd7991d 100644 --- a/consensus/config.go +++ b/consensus/config.go @@ -12,12 +12,12 @@ import ( // HotstuffModules is a helper structure to encapsulate dependencies to create // a hotStuff participant. type HotstuffModules struct { - Committee hotstuff.DynamicCommittee // consensus committee - Signer hotstuff.Signer // signer of proposal & votes - Persist hotstuff.Persister // last state of consensus participant - Notifier *pubsub.Distributor // observer for hotstuff events - FinalizationDistributor *pubsub.FinalizationDistributor // observer for finalization events, used by compliance engine - QCCreatedDistributor *pubsub.QCCreatedDistributor // observer for qc created event, used by leader + Committee hotstuff.DynamicCommittee // consensus committee + Signer hotstuff.Signer // signer of proposal & votes + Persist hotstuff.Persister // last state of consensus participant + Notifier *pubsub.Distributor // observer for hotstuff events + FinalizationDistributor *pubsub.FollowerDistributor // observer for finalization events, used by compliance engine + QCCreatedDistributor *pubsub.QCCreatedDistributor // observer for qc created event, used by leader TimeoutCollectorDistributor *pubsub.TimeoutCollectorDistributor Forks hotstuff.Forks // information about multiple forks Validator hotstuff.Validator // validator of proposals & votes diff --git a/consensus/follower.go b/consensus/follower.go index c366d2d8881..77693b81f1b 100644 --- a/consensus/follower.go +++ b/consensus/follower.go @@ -17,7 +17,7 @@ import ( // TODO: this needs to be integrated with proper configuration and bootstrapping. func NewFollower(log zerolog.Logger, committee hotstuff.DynamicCommittee, headers storage.Headers, updater module.Finalizer, - verifier hotstuff.Verifier, notifier hotstuff.FinalizationConsumer, rootHeader *flow.Header, + verifier hotstuff.Verifier, notifier hotstuff.ConsensusFollowerConsumer, rootHeader *flow.Header, rootQC *flow.QuorumCertificate, finalized *flow.Header, pending []*flow.Header, ) (*hotstuff.FollowerLoop, error) { diff --git a/consensus/hotstuff/consumer.go b/consensus/hotstuff/consumer.go index 39087594ce0..27c3ea20428 100644 --- a/consensus/hotstuff/consumer.go +++ b/consensus/hotstuff/consumer.go @@ -41,8 +41,6 @@ type BaseProtocolViolationConsumer interface { // - be non-blocking // - handle repetition of the same events (with some processing overhead). type FinalizationConsumer interface { - BaseProtocolViolationConsumer - // OnBlockIncorporated notifications are produced by the Finalization Logic // whenever a block is incorporated into the consensus state. // Prerequisites: @@ -58,6 +56,16 @@ type FinalizationConsumer interface { OnFinalizedBlock(*model.Block) } +// ConsensusFollowerConsumer consumes outbound notifications produced by consensus followers(not participants). +// Implementations must: +// - be concurrency safe +// - be non-blocking +// - handle repetition of the same events (with some processing overhead). +type ConsensusFollowerConsumer interface { + BaseProtocolViolationConsumer + FinalizationConsumer +} + // Consumer consumes outbound notifications produced by HotStuff and its components. // Notifications are consensus-internal state changes which are potentially relevant to // the larger node in which HotStuff is running. The notifications are emitted @@ -68,7 +76,7 @@ type FinalizationConsumer interface { // - be non-blocking // - handle repetition of the same events (with some processing overhead). type Consumer interface { - FinalizationConsumer + ConsensusFollowerConsumer CommunicatorConsumer // OnEventProcessed notifications are produced by the EventHandler when it is done processing diff --git a/consensus/hotstuff/forks/forks.go b/consensus/hotstuff/forks/forks.go index 82ce3161271..d7b67333f0a 100644 --- a/consensus/hotstuff/forks/forks.go +++ b/consensus/hotstuff/forks/forks.go @@ -36,7 +36,7 @@ type ancestryChain struct { // https://developers.diem.com/papers/diem-consensus-state-machine-replication-in-the-diem-blockchain/2021-08-17.pdf // Forks is NOT safe for concurrent use by multiple goroutines. type Forks struct { - notifier hotstuff.FinalizationConsumer + notifier hotstuff.ConsensusFollowerConsumer forest forest.LevelledForest finalizationCallback module.Finalizer @@ -46,7 +46,7 @@ type Forks struct { var _ hotstuff.Forks = (*Forks)(nil) -func New(trustedRoot *BlockQC, finalizationCallback module.Finalizer, notifier hotstuff.FinalizationConsumer) (*Forks, error) { +func New(trustedRoot *BlockQC, finalizationCallback module.Finalizer, notifier hotstuff.ConsensusFollowerConsumer) (*Forks, error) { if (trustedRoot.Block.BlockID != trustedRoot.QC.BlockID) || (trustedRoot.Block.View != trustedRoot.QC.View) { return nil, model.NewConfigurationErrorf("invalid root: root QC is not pointing to root block") } diff --git a/consensus/hotstuff/model/errors.go b/consensus/hotstuff/model/errors.go index c296d9f8f9a..4d482b1278d 100644 --- a/consensus/hotstuff/model/errors.go +++ b/consensus/hotstuff/model/errors.go @@ -184,6 +184,17 @@ func IsInvalidBlockError(err error) bool { return errors.As(err, &e) } +// AsInvalidBlockError determines whether the given error is a InvalidBlockError +// (potentially wrapped). It follows the same semantics as a checked type cast. +func AsInvalidBlockError(err error) (*InvalidBlockError, bool) { + var e InvalidBlockError + ok := errors.As(err, &e) + if ok { + return &e, true + } + return nil, false +} + func (e InvalidBlockError) Unwrap() error { return e.Err } diff --git a/consensus/hotstuff/notifications/noop_consumer.go b/consensus/hotstuff/notifications/noop_consumer.go index 7e25d026d4e..bd104151d56 100644 --- a/consensus/hotstuff/notifications/noop_consumer.go +++ b/consensus/hotstuff/notifications/noop_consumer.go @@ -11,6 +11,7 @@ import ( // NoopConsumer is an implementation of the notifications consumer that // doesn't do anything. type NoopConsumer struct { + NoopBaseProtocolViolationConsumer NoopFinalizationConsumer NoopPartialConsumer NoopCommunicatorConsumer @@ -67,9 +68,7 @@ func (*NoopPartialConsumer) OnInvalidTimeoutDetected(model.InvalidTimeoutError) // no-op implementation of hotstuff.FinalizationConsumer -type NoopFinalizationConsumer struct { - NoopBaseProtocolViolationConsumer -} +type NoopFinalizationConsumer struct{} var _ hotstuff.FinalizationConsumer = (*NoopFinalizationConsumer)(nil) @@ -118,6 +117,6 @@ type NoopBaseProtocolViolationConsumer struct{} var _ hotstuff.BaseProtocolViolationConsumer = (*NoopBaseProtocolViolationConsumer)(nil) -func (n NoopBaseProtocolViolationConsumer) OnInvalidBlockDetected(model.InvalidBlockError) {} +func (*NoopBaseProtocolViolationConsumer) OnInvalidBlockDetected(model.InvalidBlockError) {} -func (n NoopBaseProtocolViolationConsumer) OnDoubleProposeDetected(*model.Block, *model.Block) {} +func (*NoopBaseProtocolViolationConsumer) OnDoubleProposeDetected(*model.Block, *model.Block) {} diff --git a/consensus/hotstuff/notifications/pubsub/distributor.go b/consensus/hotstuff/notifications/pubsub/distributor.go index 151c9671ba5..31cf1147e30 100644 --- a/consensus/hotstuff/notifications/pubsub/distributor.go +++ b/consensus/hotstuff/notifications/pubsub/distributor.go @@ -6,6 +6,7 @@ import ( "github.com/onflow/flow-go/consensus/hotstuff" "github.com/onflow/flow-go/consensus/hotstuff/model" + "github.com/onflow/flow-go/consensus/hotstuff/notifications" "github.com/onflow/flow-go/model/flow" ) @@ -38,6 +39,24 @@ func (p *Distributor) AddConsumer(consumer hotstuff.Consumer) { p.subscribers = append(p.subscribers, consumer) } +// AddFollowerConsumer wraps +func (p *Distributor) AddFollowerConsumer(consumer hotstuff.ConsensusFollowerConsumer) { + p.lock.Lock() + defer p.lock.Unlock() + + var wrappedConsumer hotstuff.Consumer = &struct { + notifications.NoopCommunicatorConsumer + notifications.NoopPartialConsumer + hotstuff.ConsensusFollowerConsumer + }{ + notifications.NoopCommunicatorConsumer{}, + notifications.NoopPartialConsumer{}, + consumer, + } + + p.subscribers = append(p.subscribers, wrappedConsumer) +} + func (p *Distributor) OnStart(currentView uint64) { p.lock.RLock() defer p.lock.RUnlock() diff --git a/consensus/hotstuff/notifications/pubsub/finalization_distributor.go b/consensus/hotstuff/notifications/pubsub/finalization_distributor.go deleted file mode 100644 index 6d1c72ef8e6..00000000000 --- a/consensus/hotstuff/notifications/pubsub/finalization_distributor.go +++ /dev/null @@ -1,79 +0,0 @@ -package pubsub - -import ( - "sync" - - "github.com/onflow/flow-go/consensus/hotstuff" - "github.com/onflow/flow-go/consensus/hotstuff/model" - "github.com/onflow/flow-go/consensus/hotstuff/notifications" -) - -type OnBlockFinalizedConsumer = func(block *model.Block) -type OnBlockIncorporatedConsumer = func(block *model.Block) - -// FinalizationDistributor ingests finalization events from hotstuff and distributes it to subscribers. -type FinalizationDistributor struct { - notifications.NoopConsumer - blockFinalizedConsumers []OnBlockFinalizedConsumer - blockIncorporatedConsumers []OnBlockIncorporatedConsumer - hotStuffFinalizationConsumers []hotstuff.FinalizationConsumer - lock sync.RWMutex -} - -var _ hotstuff.Consumer = (*FinalizationDistributor)(nil) - -func NewFinalizationDistributor() *FinalizationDistributor { - return &FinalizationDistributor{ - blockFinalizedConsumers: make([]OnBlockFinalizedConsumer, 0), - blockIncorporatedConsumers: make([]OnBlockIncorporatedConsumer, 0), - lock: sync.RWMutex{}, - } -} - -func (p *FinalizationDistributor) AddOnBlockFinalizedConsumer(consumer OnBlockFinalizedConsumer) { - p.lock.Lock() - defer p.lock.Unlock() - p.blockFinalizedConsumers = append(p.blockFinalizedConsumers, consumer) -} - -func (p *FinalizationDistributor) AddOnBlockIncorporatedConsumer(consumer OnBlockIncorporatedConsumer) { - p.lock.Lock() - defer p.lock.Unlock() - p.blockIncorporatedConsumers = append(p.blockIncorporatedConsumers, consumer) -} - -func (p *FinalizationDistributor) AddConsumer(consumer hotstuff.FinalizationConsumer) { - p.lock.Lock() - defer p.lock.Unlock() - p.hotStuffFinalizationConsumers = append(p.hotStuffFinalizationConsumers, consumer) -} - -func (p *FinalizationDistributor) OnBlockIncorporated(block *model.Block) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, consumer := range p.blockIncorporatedConsumers { - consumer(block) - } - for _, consumer := range p.hotStuffFinalizationConsumers { - consumer.OnBlockIncorporated(block) - } -} - -func (p *FinalizationDistributor) OnFinalizedBlock(block *model.Block) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, consumer := range p.blockFinalizedConsumers { - consumer(block) - } - for _, consumer := range p.hotStuffFinalizationConsumers { - consumer.OnFinalizedBlock(block) - } -} - -func (p *FinalizationDistributor) OnDoubleProposeDetected(block1, block2 *model.Block) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, consumer := range p.hotStuffFinalizationConsumers { - consumer.OnDoubleProposeDetected(block1, block2) - } -} diff --git a/consensus/hotstuff/notifications/pubsub/follower_distributor.go b/consensus/hotstuff/notifications/pubsub/follower_distributor.go new file mode 100644 index 00000000000..54ad77ac925 --- /dev/null +++ b/consensus/hotstuff/notifications/pubsub/follower_distributor.go @@ -0,0 +1,85 @@ +package pubsub + +import ( + "sync" + + "github.com/onflow/flow-go/consensus/hotstuff" + "github.com/onflow/flow-go/consensus/hotstuff/model" +) + +type OnBlockFinalizedConsumer = func(block *model.Block) +type OnBlockIncorporatedConsumer = func(block *model.Block) + +// FollowerDistributor ingests consensus follower events and distributes it to subscribers. +type FollowerDistributor struct { + blockFinalizedConsumers []OnBlockFinalizedConsumer + blockIncorporatedConsumers []OnBlockIncorporatedConsumer + followerConsumers []hotstuff.ConsensusFollowerConsumer + lock sync.RWMutex +} + +var _ hotstuff.ConsensusFollowerConsumer = (*FollowerDistributor)(nil) + +func NewFollowerDistributor() *FollowerDistributor { + return &FollowerDistributor{ + blockFinalizedConsumers: make([]OnBlockFinalizedConsumer, 0), + blockIncorporatedConsumers: make([]OnBlockIncorporatedConsumer, 0), + lock: sync.RWMutex{}, + } +} + +func (p *FollowerDistributor) AddOnBlockFinalizedConsumer(consumer OnBlockFinalizedConsumer) { + p.lock.Lock() + defer p.lock.Unlock() + p.blockFinalizedConsumers = append(p.blockFinalizedConsumers, consumer) +} + +func (p *FollowerDistributor) AddOnBlockIncorporatedConsumer(consumer OnBlockIncorporatedConsumer) { + p.lock.Lock() + defer p.lock.Unlock() + p.blockIncorporatedConsumers = append(p.blockIncorporatedConsumers, consumer) +} + +func (p *FollowerDistributor) AddConsumer(consumer hotstuff.ConsensusFollowerConsumer) { + p.lock.Lock() + defer p.lock.Unlock() + p.followerConsumers = append(p.followerConsumers, consumer) +} + +func (p *FollowerDistributor) OnBlockIncorporated(block *model.Block) { + p.lock.RLock() + defer p.lock.RUnlock() + for _, consumer := range p.blockIncorporatedConsumers { + consumer(block) + } + for _, consumer := range p.followerConsumers { + consumer.OnBlockIncorporated(block) + } +} + +func (p *FollowerDistributor) OnFinalizedBlock(block *model.Block) { + p.lock.RLock() + defer p.lock.RUnlock() + for _, consumer := range p.blockFinalizedConsumers { + consumer(block) + } + for _, consumer := range p.followerConsumers { + consumer.OnFinalizedBlock(block) + } +} + +func (p *FollowerDistributor) OnDoubleProposeDetected(block1, block2 *model.Block) { + p.lock.RLock() + defer p.lock.RUnlock() + for _, consumer := range p.followerConsumers { + consumer.OnDoubleProposeDetected(block1, block2) + } +} + +func (p *FollowerDistributor) OnInvalidBlockDetected(err model.InvalidBlockError) { + p.lock.RLock() + defer p.lock.RUnlock() + for _, consumer := range p.followerConsumers { + consumer.OnInvalidBlockDetected(err) + } +} diff --git a/consensus/integration/nodes_test.go b/consensus/integration/nodes_test.go index b24b5b16ee4..2dd716b53f4 100644 --- a/consensus/integration/nodes_test.go +++ b/consensus/integration/nodes_test.go @@ -610,7 +610,7 @@ func createNode( comp, err := compliance.NewEngine(log, me, compCore) require.NoError(t, err) - finalizedHeader, err := synceng.NewFinalizedHeaderCache(log, state, pubsub.NewFinalizationDistributor()) + finalizedHeader, err := synceng.NewFinalizedHeaderCache(log, state, pubsub.NewFollowerDistributor()) require.NoError(t, err) identities, err := state.Final().Identities(filter.And( diff --git a/consensus/participant.go b/consensus/participant.go index 1f054e1594b..ae00594d0be 100644 --- a/consensus/participant.go +++ b/consensus/participant.go @@ -131,7 +131,7 @@ func NewValidator(metrics module.HotstuffMetrics, committee hotstuff.DynamicComm } // NewForks recovers trusted root and creates new forks manager -func NewForks(final *flow.Header, headers storage.Headers, updater module.Finalizer, notifier hotstuff.FinalizationConsumer, rootHeader *flow.Header, rootQC *flow.QuorumCertificate) (*forks.Forks, error) { +func NewForks(final *flow.Header, headers storage.Headers, updater module.Finalizer, notifier hotstuff.ConsensusFollowerConsumer, rootHeader *flow.Header, rootQC *flow.QuorumCertificate) (*forks.Forks, error) { // recover the trusted root trustedRoot, err := recoverTrustedRoot(final, headers, rootHeader, rootQC) if err != nil { diff --git a/engine/collection/epochmgr/factories/hotstuff.go b/engine/collection/epochmgr/factories/hotstuff.go index 9b27bfc7201..7d56fd72f3d 100644 --- a/engine/collection/epochmgr/factories/hotstuff.go +++ b/engine/collection/epochmgr/factories/hotstuff.go @@ -76,8 +76,8 @@ func (f *HotStuffFactory) CreateModules( log := f.createLogger(cluster) metrics := f.createMetrics(cluster.ChainID()) notifier := pubsub.NewDistributor() - finalizationDistributor := pubsub.NewFinalizationDistributor() - notifier.AddConsumer(finalizationDistributor) + followerDistributor := pubsub.NewFollowerDistributor() + notifier.AddFollowerConsumer(followerDistributor) notifier.AddConsumer(notifications.NewLogConsumer(log)) notifier.AddConsumer(hotmetrics.NewMetricsConsumer(metrics)) notifier.AddConsumer(notifications.NewTelemetryConsumer(log)) @@ -129,7 +129,7 @@ func (f *HotStuffFactory) CreateModules( finalizedBlock.View+1, notifier, voteProcessorFactory, - finalizationDistributor, + followerDistributor, ) if err != nil { return nil, nil, err @@ -163,7 +163,7 @@ func (f *HotStuffFactory) CreateModules( TimeoutAggregator: timeoutAggregator, QCCreatedDistributor: qcDistributor, TimeoutCollectorDistributor: timeoutCollectorDistributor, - FinalizationDistributor: finalizationDistributor, + FinalizationDistributor: followerDistributor, }, metrics, nil } diff --git a/engine/common/follower/compliance_core.go b/engine/common/follower/compliance_core.go index 2cae31059cb..c8f4e249d9e 100644 --- a/engine/common/follower/compliance_core.go +++ b/engine/common/follower/compliance_core.go @@ -38,17 +38,18 @@ const defaultPendingBlocksCacheCapacity = 1000 // Generally is NOT concurrency safe but some functions can be used in concurrent setup. type ComplianceCore struct { *component.ComponentManager - log zerolog.Logger - mempoolMetrics module.MempoolMetrics - tracer module.Tracer - pendingCache *cache.Cache - pendingTree *pending_tree.PendingTree - state protocol.FollowerState - follower module.HotStuffFollower - validator hotstuff.Validator - sync module.BlockRequester - certifiedRangesChan chan CertifiedBlocks // delivers ranges of certified blocks to main core worker - finalizedBlocksChan chan *flow.Header // delivers finalized blocks to main core worker. + log zerolog.Logger + mempoolMetrics module.MempoolMetrics + tracer module.Tracer + protocolViolationNotifier hotstuff.BaseProtocolViolationConsumer + pendingCache *cache.Cache + pendingTree *pending_tree.PendingTree + state protocol.FollowerState + follower module.HotStuffFollower + validator hotstuff.Validator + sync module.BlockRequester + certifiedRangesChan chan CertifiedBlocks // delivers ranges of certified blocks to main core worker + finalizedBlocksChan chan *flow.Header // delivers finalized blocks to main core worker. } var _ complianceCore = (*ComplianceCore)(nil) @@ -58,7 +59,7 @@ var _ complianceCore = (*ComplianceCore)(nil) func NewComplianceCore(log zerolog.Logger, mempoolMetrics module.MempoolMetrics, heroCacheCollector module.HeroCacheMetrics, - finalizationConsumer hotstuff.FinalizationConsumer, + followerConsumer hotstuff.ConsensusFollowerConsumer, state protocol.FollowerState, follower module.HotStuffFollower, validator hotstuff.Validator, @@ -66,7 +67,7 @@ func NewComplianceCore(log zerolog.Logger, tracer module.Tracer, ) (*ComplianceCore, error) { onEquivocation := func(block, otherBlock *flow.Block) { - finalizationConsumer.OnDoubleProposeDetected(model.BlockFromFlow(block.Header), model.BlockFromFlow(otherBlock.Header)) + followerConsumer.OnDoubleProposeDetected(model.BlockFromFlow(block.Header), model.BlockFromFlow(otherBlock.Header)) } finalizedBlock, err := state.Final().Head() @@ -75,17 +76,18 @@ func NewComplianceCore(log zerolog.Logger, } c := &ComplianceCore{ - log: log.With().Str("engine", "follower_core").Logger(), - mempoolMetrics: mempoolMetrics, - state: state, - pendingCache: cache.NewCache(log, defaultPendingBlocksCacheCapacity, heroCacheCollector, onEquivocation), - pendingTree: pending_tree.NewPendingTree(finalizedBlock), - follower: follower, - validator: validator, - sync: sync, - tracer: tracer, - certifiedRangesChan: make(chan CertifiedBlocks, defaultCertifiedRangeChannelCapacity), - finalizedBlocksChan: make(chan *flow.Header, defaultFinalizedBlocksChannelCapacity), + log: log.With().Str("engine", "follower_core").Logger(), + mempoolMetrics: mempoolMetrics, + state: state, + protocolViolationNotifier: followerConsumer, + pendingCache: cache.NewCache(log, defaultPendingBlocksCacheCapacity, heroCacheCollector, onEquivocation), + pendingTree: pending_tree.NewPendingTree(finalizedBlock), + follower: follower, + validator: validator, + sync: sync, + tracer: tracer, + certifiedRangesChan: make(chan CertifiedBlocks, defaultCertifiedRangeChannelCapacity), + finalizedBlocksChan: make(chan *flow.Header, defaultFinalizedBlocksChannelCapacity), } // prune cache to latest finalized view @@ -141,9 +143,9 @@ func (c *ComplianceCore) OnBlockRange(originID flow.Identifier, batch []*flow.Bl // 2. The QC within the block is valid. A valid QC proves validity of all ancestors. err := c.validator.ValidateProposal(hotstuffProposal) if err != nil { - if model.IsInvalidBlockError(err) { - // TODO potential slashing - log.Err(err).Msgf("received invalid block proposal (potential slashing evidence)") + if invalidBlockError, ok := model.AsInvalidBlockError(err); ok { + // TODO: potential slashing + c.protocolViolationNotifier.OnInvalidBlockDetected(*invalidBlockError) return nil } if errors.Is(err, model.ErrViewForUnknownEpoch) { diff --git a/engine/common/follower/integration_test.go b/engine/common/follower/integration_test.go index 17b7171f4e7..1a3b2a74f86 100644 --- a/engine/common/follower/integration_test.go +++ b/engine/common/follower/integration_test.go @@ -83,7 +83,7 @@ func TestFollowerHappyPath(t *testing.T) { }) require.NoError(t, err) - consensusConsumer := pubsub.NewFinalizationDistributor() + consensusConsumer := pubsub.NewFollowerDistributor() // use real consensus modules forks, err := consensus.NewForks(rootHeader, all.Headers, finalizer, consensusConsumer, rootHeader, rootQC) require.NoError(t, err) diff --git a/engine/common/synchronization/engine_test.go b/engine/common/synchronization/engine_test.go index ba83046a0e3..aa43f2a5800 100644 --- a/engine/common/synchronization/engine_test.go +++ b/engine/common/synchronization/engine_test.go @@ -168,7 +168,7 @@ func (ss *SyncSuite) SetupTest() { log := zerolog.New(io.Discard) metrics := metrics.NewNoopCollector() - finalizedHeader, err := NewFinalizedHeaderCache(log, ss.state, pubsub.NewFinalizationDistributor()) + finalizedHeader, err := NewFinalizedHeaderCache(log, ss.state, pubsub.NewFollowerDistributor()) require.NoError(ss.T(), err, "could not create finalized snapshot cache") idCache, err := cache.NewProtocolStateIDCache(log, ss.state, protocolEvents.NewDistributor()) diff --git a/engine/common/synchronization/finalized_snapshot.go b/engine/common/synchronization/finalized_snapshot.go index a98b9fe6758..a866cd7b9d8 100644 --- a/engine/common/synchronization/finalized_snapshot.go +++ b/engine/common/synchronization/finalized_snapshot.go @@ -29,7 +29,7 @@ type FinalizedHeaderCache struct { } // NewFinalizedHeaderCache creates a new finalized header cache. -func NewFinalizedHeaderCache(log zerolog.Logger, state protocol.State, finalizationDistributor *pubsub.FinalizationDistributor) (*FinalizedHeaderCache, error) { +func NewFinalizedHeaderCache(log zerolog.Logger, state protocol.State, finalizationDistributor *pubsub.FollowerDistributor) (*FinalizedHeaderCache, error) { cache := &FinalizedHeaderCache{ state: state, lm: lifecycle.NewLifecycleManager(), diff --git a/engine/testutil/nodes.go b/engine/testutil/nodes.go index 74eccf28b22..287febb3bc6 100644 --- a/engine/testutil/nodes.go +++ b/engine/testutil/nodes.go @@ -664,7 +664,7 @@ func ExecutionNode(t *testing.T, hub *stub.Hub, identity *flow.Identity, identit syncCore, err := chainsync.New(node.Log, chainsync.DefaultConfig(), metrics.NewChainSyncCollector(genesisHead.ChainID), genesisHead.ChainID) require.NoError(t, err) - finalizationDistributor := pubsub.NewFinalizationDistributor() + finalizationDistributor := pubsub.NewFollowerDistributor() latestExecutedHeight, _, err := execState.GetHighestExecutedBlockID(context.TODO()) require.NoError(t, err) diff --git a/follower/follower_builder.go b/follower/follower_builder.go index dad5247c820..cda75731365 100644 --- a/follower/follower_builder.go +++ b/follower/follower_builder.go @@ -108,7 +108,7 @@ type FollowerServiceBuilder struct { LibP2PNode p2p.LibP2PNode FollowerState protocol.FollowerState SyncCore *synchronization.Core - FinalizationDistributor *pubsub.FinalizationDistributor + FinalizationDistributor *pubsub.FollowerDistributor FinalizedHeader *synceng.FinalizedHeaderCache Committee hotstuff.DynamicCommittee Finalized *flow.Header @@ -357,7 +357,7 @@ func FlowConsensusFollowerService(opts ...FollowerOption) *FollowerServiceBuilde FollowerServiceConfig: config, // TODO: using RoleAccess here for now. This should be refactored eventually to have its own role type FlowNodeBuilder: cmd.FlowNode(flow.RoleAccess.String(), config.baseOptions...), - FinalizationDistributor: pubsub.NewFinalizationDistributor(), + FinalizationDistributor: pubsub.NewFollowerDistributor(), } ret.FinalizationDistributor.AddConsumer(notifications.NewSlashingViolationsConsumer(ret.Logger)) // the observer gets a version of the root snapshot file that does not contain any node addresses diff --git a/module/state_synchronization/requester/execution_data_requester_test.go b/module/state_synchronization/requester/execution_data_requester_test.go index e2e01cb7929..a490863a9cc 100644 --- a/module/state_synchronization/requester/execution_data_requester_test.go +++ b/module/state_synchronization/requester/execution_data_requester_test.go @@ -385,7 +385,7 @@ func generatePauseResume(pauseHeight uint64) (specialBlockGenerator, func()) { return generate, resume } -func (suite *ExecutionDataRequesterSuite) prepareRequesterTest(cfg *fetchTestRun) (state_synchronization.ExecutionDataRequester, *pubsub.FinalizationDistributor) { +func (suite *ExecutionDataRequesterSuite) prepareRequesterTest(cfg *fetchTestRun) (state_synchronization.ExecutionDataRequester, *pubsub.FollowerDistributor) { headers := synctest.MockBlockHeaderStorage( synctest.WithByID(cfg.blocksByID), synctest.WithByHeight(cfg.blocksByHeight), @@ -400,7 +400,7 @@ func (suite *ExecutionDataRequesterSuite) prepareRequesterTest(cfg *fetchTestRun suite.downloader = mockDownloader(cfg.executionDataEntries) - finalizationDistributor := pubsub.NewFinalizationDistributor() + finalizationDistributor := pubsub.NewFollowerDistributor() processedHeight := bstorage.NewConsumerProgress(suite.db, module.ConsumeProgressExecutionDataRequesterBlockHeight) processedNotification := bstorage.NewConsumerProgress(suite.db, module.ConsumeProgressExecutionDataRequesterNotification) @@ -428,7 +428,7 @@ func (suite *ExecutionDataRequesterSuite) prepareRequesterTest(cfg *fetchTestRun return edr, finalizationDistributor } -func (suite *ExecutionDataRequesterSuite) runRequesterTestHalts(edr state_synchronization.ExecutionDataRequester, finalizationDistributor *pubsub.FinalizationDistributor, cfg *fetchTestRun) receivedExecutionData { +func (suite *ExecutionDataRequesterSuite) runRequesterTestHalts(edr state_synchronization.ExecutionDataRequester, finalizationDistributor *pubsub.FollowerDistributor, cfg *fetchTestRun) receivedExecutionData { // make sure test helper goroutines are cleaned up ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() @@ -457,7 +457,7 @@ func (suite *ExecutionDataRequesterSuite) runRequesterTestHalts(edr state_synchr return fetchedExecutionData } -func (suite *ExecutionDataRequesterSuite) runRequesterTestPauseResume(edr state_synchronization.ExecutionDataRequester, finalizationDistributor *pubsub.FinalizationDistributor, cfg *fetchTestRun, expectedDownloads int, resume func()) receivedExecutionData { +func (suite *ExecutionDataRequesterSuite) runRequesterTestPauseResume(edr state_synchronization.ExecutionDataRequester, finalizationDistributor *pubsub.FollowerDistributor, cfg *fetchTestRun, expectedDownloads int, resume func()) receivedExecutionData { // make sure test helper goroutines are cleaned up ctx, cancel := context.WithCancel(context.Background()) signalerCtx := irrecoverable.NewMockSignalerContext(suite.T(), ctx) @@ -493,7 +493,7 @@ func (suite *ExecutionDataRequesterSuite) runRequesterTestPauseResume(edr state_ return fetchedExecutionData } -func (suite *ExecutionDataRequesterSuite) runRequesterTest(edr state_synchronization.ExecutionDataRequester, finalizationDistributor *pubsub.FinalizationDistributor, cfg *fetchTestRun) receivedExecutionData { +func (suite *ExecutionDataRequesterSuite) runRequesterTest(edr state_synchronization.ExecutionDataRequester, finalizationDistributor *pubsub.FollowerDistributor, cfg *fetchTestRun) receivedExecutionData { // make sure test helper goroutines are cleaned up ctx, cancel := context.WithCancel(context.Background()) signalerCtx := irrecoverable.NewMockSignalerContext(suite.T(), ctx) @@ -538,7 +538,7 @@ func (suite *ExecutionDataRequesterSuite) consumeExecutionDataNotifications(cfg } } -func (suite *ExecutionDataRequesterSuite) finalizeBlocks(cfg *fetchTestRun, finalizationDistributor *pubsub.FinalizationDistributor) { +func (suite *ExecutionDataRequesterSuite) finalizeBlocks(cfg *fetchTestRun, finalizationDistributor *pubsub.FollowerDistributor) { for i := cfg.StartHeight(); i <= cfg.endHeight; i++ { b := cfg.blocksByHeight[i] From 7f8c8ff835d421f45584a7a8496e7c419d801bd3 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Mon, 10 Apr 2023 12:49:48 +0300 Subject: [PATCH 03/38] Updated mocks. Fixed tests --- consensus/follower_test.go | 4 +- .../hotstuff/integration/instance_test.go | 1 + .../mocks/consensus_follower_consumer.go | 48 +++++++++++++++++++ .../hotstuff/mocks/finalization_consumer.go | 10 ---- .../common/follower/compliance_core_test.go | 26 +++++----- engine/testutil/nodes.go | 10 ++-- 6 files changed, 70 insertions(+), 29 deletions(-) create mode 100644 consensus/hotstuff/mocks/consensus_follower_consumer.go diff --git a/consensus/follower_test.go b/consensus/follower_test.go index 26a61c88ae5..cd76f7b7f5a 100644 --- a/consensus/follower_test.go +++ b/consensus/follower_test.go @@ -56,7 +56,7 @@ type HotStuffFollowerSuite struct { headers *mockstorage.Headers finalizer *mockmodule.Finalizer verifier *mockhotstuff.Verifier - notifier *mockhotstuff.FinalizationConsumer + notifier *mockhotstuff.ConsensusFollowerConsumer rootHeader *flow.Header rootQC *flow.QuorumCertificate finalized *flow.Header @@ -106,7 +106,7 @@ func (s *HotStuffFollowerSuite) SetupTest() { s.verifier.On("VerifyTC", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil).Maybe() // mock consumer for finalization notifications - s.notifier = mockhotstuff.NewFinalizationConsumer(s.T()) + s.notifier = mockhotstuff.NewConsensusFollowerConsumer(s.T()) // root block and QC parentID, err := flow.HexStringToIdentifier("aa7693d498e9a087b1cadf5bfe9a1ff07829badc1915c210e482f369f9a00a70") diff --git a/consensus/hotstuff/integration/instance_test.go b/consensus/hotstuff/integration/instance_test.go index 68aa714d1ba..325a5f9b40a 100644 --- a/consensus/hotstuff/integration/instance_test.go +++ b/consensus/hotstuff/integration/instance_test.go @@ -84,6 +84,7 @@ type Instance struct { } type MockedCommunicatorConsumer struct { + notifications.NoopBaseProtocolViolationConsumer notifications.NoopPartialConsumer notifications.NoopFinalizationConsumer *mocks.CommunicatorConsumer diff --git a/consensus/hotstuff/mocks/consensus_follower_consumer.go b/consensus/hotstuff/mocks/consensus_follower_consumer.go new file mode 100644 index 00000000000..f5a7de1259c --- /dev/null +++ b/consensus/hotstuff/mocks/consensus_follower_consumer.go @@ -0,0 +1,48 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mocks + +import ( + model "github.com/onflow/flow-go/consensus/hotstuff/model" + mock "github.com/stretchr/testify/mock" +) + +// ConsensusFollowerConsumer is an autogenerated mock type for the ConsensusFollowerConsumer type +type ConsensusFollowerConsumer struct { + mock.Mock +} + +// OnBlockIncorporated provides a mock function with given fields: _a0 +func (_m *ConsensusFollowerConsumer) OnBlockIncorporated(_a0 *model.Block) { + _m.Called(_a0) +} + +// OnDoubleProposeDetected provides a mock function with given fields: _a0, _a1 +func (_m *ConsensusFollowerConsumer) OnDoubleProposeDetected(_a0 *model.Block, _a1 *model.Block) { + _m.Called(_a0, _a1) +} + +// OnFinalizedBlock provides a mock function with given fields: _a0 +func (_m *ConsensusFollowerConsumer) OnFinalizedBlock(_a0 *model.Block) { + _m.Called(_a0) +} + +// OnInvalidBlockDetected provides a mock function with given fields: err +func (_m *ConsensusFollowerConsumer) OnInvalidBlockDetected(err model.InvalidBlockError) { + _m.Called(err) +} + +type mockConstructorTestingTNewConsensusFollowerConsumer interface { + mock.TestingT + Cleanup(func()) +} + +// NewConsensusFollowerConsumer creates a new instance of ConsensusFollowerConsumer. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewConsensusFollowerConsumer(t mockConstructorTestingTNewConsensusFollowerConsumer) *ConsensusFollowerConsumer { + mock := &ConsensusFollowerConsumer{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/consensus/hotstuff/mocks/finalization_consumer.go b/consensus/hotstuff/mocks/finalization_consumer.go index bba788508a0..7780a5e1c79 100644 --- a/consensus/hotstuff/mocks/finalization_consumer.go +++ b/consensus/hotstuff/mocks/finalization_consumer.go @@ -17,21 +17,11 @@ func (_m *FinalizationConsumer) OnBlockIncorporated(_a0 *model.Block) { _m.Called(_a0) } -// OnDoubleProposeDetected provides a mock function with given fields: _a0, _a1 -func (_m *FinalizationConsumer) OnDoubleProposeDetected(_a0 *model.Block, _a1 *model.Block) { - _m.Called(_a0, _a1) -} - // OnFinalizedBlock provides a mock function with given fields: _a0 func (_m *FinalizationConsumer) OnFinalizedBlock(_a0 *model.Block) { _m.Called(_a0) } -// OnInvalidBlockDetected provides a mock function with given fields: err -func (_m *FinalizationConsumer) OnInvalidBlockDetected(err model.InvalidBlockError) { - _m.Called(err) -} - type mockConstructorTestingTNewFinalizationConsumer interface { mock.TestingT Cleanup(func()) diff --git a/engine/common/follower/compliance_core_test.go b/engine/common/follower/compliance_core_test.go index bea663f9b69..be1989e8acb 100644 --- a/engine/common/follower/compliance_core_test.go +++ b/engine/common/follower/compliance_core_test.go @@ -34,13 +34,13 @@ func TestFollowerCore(t *testing.T) { type CoreSuite struct { suite.Suite - originID flow.Identifier - finalizedBlock *flow.Header - state *protocol.FollowerState - follower *module.HotStuffFollower - sync *module.BlockRequester - validator *hotstuff.Validator - finalizationConsumer *hotstuff.FinalizationConsumer + originID flow.Identifier + finalizedBlock *flow.Header + state *protocol.FollowerState + follower *module.HotStuffFollower + sync *module.BlockRequester + validator *hotstuff.Validator + followerConsumer *hotstuff.ConsensusFollowerConsumer ctx irrecoverable.SignalerContext cancel context.CancelFunc @@ -53,7 +53,7 @@ func (s *CoreSuite) SetupTest() { s.follower = module.NewHotStuffFollower(s.T()) s.validator = hotstuff.NewValidator(s.T()) s.sync = module.NewBlockRequester(s.T()) - s.finalizationConsumer = hotstuff.NewFinalizationConsumer(s.T()) + s.followerConsumer = hotstuff.NewConsensusFollowerConsumer(s.T()) s.originID = unittest.IdentifierFixture() s.finalizedBlock = unittest.BlockHeaderFixture() @@ -67,7 +67,7 @@ func (s *CoreSuite) SetupTest() { unittest.Logger(), metrics, metrics, - s.finalizationConsumer, + s.followerConsumer, s.state, s.follower, s.validator, @@ -166,10 +166,12 @@ func (s *CoreSuite) TestProcessingInvalidBlock() { blocks := unittest.ChainFixtureFrom(10, s.finalizedBlock) invalidProposal := model.ProposalFromFlow(blocks[len(blocks)-1].Header) - s.validator.On("ValidateProposal", invalidProposal).Return(model.InvalidBlockError{ + sentinelError := model.InvalidBlockError{ InvalidBlock: invalidProposal, Err: fmt.Errorf(""), - }).Once() + } + s.validator.On("ValidateProposal", invalidProposal).Return(sentinelError).Once() + s.followerConsumer.On("OnInvalidBlockDetected", sentinelError).Return().Once() err := s.core.OnBlockRange(s.originID, blocks) require.NoError(s.T(), err, "sentinel error has to be handled internally") @@ -238,7 +240,7 @@ func (s *CoreSuite) TestDetectingProposalEquivocation() { otherBlock.Header.View = block.Header.View s.validator.On("ValidateProposal", mock.Anything).Return(nil).Times(2) - s.finalizationConsumer.On("OnDoubleProposeDetected", mock.Anything, mock.Anything).Return().Once() + s.followerConsumer.On("OnDoubleProposeDetected", mock.Anything, mock.Anything).Return().Once() err := s.core.OnBlockRange(s.originID, []*flow.Block{block}) require.NoError(s.T(), err) diff --git a/engine/testutil/nodes.go b/engine/testutil/nodes.go index 287febb3bc6..a4f27d21836 100644 --- a/engine/testutil/nodes.go +++ b/engine/testutil/nodes.go @@ -664,7 +664,7 @@ func ExecutionNode(t *testing.T, hub *stub.Hub, identity *flow.Identity, identit syncCore, err := chainsync.New(node.Log, chainsync.DefaultConfig(), metrics.NewChainSyncCollector(genesisHead.ChainID), genesisHead.ChainID) require.NoError(t, err) - finalizationDistributor := pubsub.NewFollowerDistributor() + followerDistributor := pubsub.NewFollowerDistributor() latestExecutedHeight, _, err := execState.GetHighestExecutedBlockID(context.TODO()) require.NoError(t, err) @@ -700,19 +700,19 @@ func ExecutionNode(t *testing.T, hub *stub.Hub, identity *flow.Identity, identit node.ProtocolEvents.AddConsumer(ingestionEngine) - followerCore, finalizer := createFollowerCore(t, &node, followerState, finalizationDistributor, rootHead, rootQC) + followerCore, finalizer := createFollowerCore(t, &node, followerState, followerDistributor, rootHead, rootQC) // mock out hotstuff validator validator := new(mockhotstuff.Validator) validator.On("ValidateProposal", mock.Anything).Return(nil) - finalizedHeader, err := synchronization.NewFinalizedHeaderCache(node.Log, node.State, finalizationDistributor) + finalizedHeader, err := synchronization.NewFinalizedHeaderCache(node.Log, node.State, followerDistributor) require.NoError(t, err) core, err := follower.NewComplianceCore( node.Log, node.Metrics, node.Metrics, - finalizationDistributor, + followerDistributor, followerState, followerCore, validator, @@ -849,7 +849,7 @@ func (s *RoundRobinLeaderSelection) DKG(_ uint64) (hotstuff.DKG, error) { return nil, fmt.Errorf("error") } -func createFollowerCore(t *testing.T, node *testmock.GenericNode, followerState *badgerstate.FollowerState, notifier hotstuff.FinalizationConsumer, +func createFollowerCore(t *testing.T, node *testmock.GenericNode, followerState *badgerstate.FollowerState, notifier hotstuff.ConsensusFollowerConsumer, rootHead *flow.Header, rootQC *flow.QuorumCertificate) (module.HotStuffFollower, *confinalizer.Finalizer) { identities, err := node.State.AtHeight(0).Identities(filter.HasRole(flow.RoleConsensus)) From 0e897eda5a3a91c5134fe53d60a705637d24276a Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Mon, 10 Apr 2023 13:35:40 +0300 Subject: [PATCH 04/38] Refactored how InvalidBlockError is created --- consensus/hotstuff/forks/forks.go | 2 +- consensus/hotstuff/model/errors.go | 7 ++++++ consensus/hotstuff/validator/validator.go | 23 +++++++------------ consensus/hotstuff/votecollector/factory.go | 5 +--- consensus/recovery/recover_test.go | 6 +---- .../common/follower/compliance_core_test.go | 6 +---- 6 files changed, 19 insertions(+), 30 deletions(-) diff --git a/consensus/hotstuff/forks/forks.go b/consensus/hotstuff/forks/forks.go index fea219a8769..681e855a1f8 100644 --- a/consensus/hotstuff/forks/forks.go +++ b/consensus/hotstuff/forks/forks.go @@ -46,7 +46,7 @@ type Forks struct { var _ hotstuff.Forks = (*Forks)(nil) -func New(trustedRoot *model.CertifiedBlock, finalizationCallback module.Finalizer, notifier hotstuff.FinalizationConsumer) (*Forks, error) { +func New(trustedRoot *model.CertifiedBlock, finalizationCallback module.Finalizer, notifier hotstuff.ConsensusFollowerConsumer) (*Forks, error) { if (trustedRoot.Block.BlockID != trustedRoot.QC.BlockID) || (trustedRoot.Block.View != trustedRoot.QC.View) { return nil, model.NewConfigurationErrorf("invalid root: root QC is not pointing to root block") } diff --git a/consensus/hotstuff/model/errors.go b/consensus/hotstuff/model/errors.go index 4d482b1278d..db8fe1bf784 100644 --- a/consensus/hotstuff/model/errors.go +++ b/consensus/hotstuff/model/errors.go @@ -169,6 +169,13 @@ type InvalidBlockError struct { Err error } +func NewInvalidBlockErrorf(proposal *Proposal, msg string, args ...interface{}) error { + return InvalidBlockError{ + InvalidBlock: proposal, + Err: fmt.Errorf(msg, args...), + } +} + func (e InvalidBlockError) Error() string { return fmt.Sprintf( "invalid block %x at view %d: %s", diff --git a/consensus/hotstuff/validator/validator.go b/consensus/hotstuff/validator/validator.go index d882d834571..cde767c07e6 100644 --- a/consensus/hotstuff/validator/validator.go +++ b/consensus/hotstuff/validator/validator.go @@ -208,7 +208,7 @@ func (v *Validator) ValidateProposal(proposal *model.Proposal) error { // validate the proposer's vote and get his identity _, err := v.ValidateVote(proposal.ProposerVote()) if model.IsInvalidVoteError(err) { - return newInvalidBlockError(proposal, fmt.Errorf("invalid proposer signature: %w", err)) + return model.NewInvalidBlockErrorf(proposal, "invalid proposer signature: %w", err) } if err != nil { return fmt.Errorf("error verifying leader signature for block %x: %w", block.BlockID, err) @@ -220,7 +220,7 @@ func (v *Validator) ValidateProposal(proposal *model.Proposal) error { return fmt.Errorf("error determining leader for block %x: %w", block.BlockID, err) } if leader != block.ProposerID { - return newInvalidBlockError(proposal, fmt.Errorf("proposer %s is not leader (%s) for view %d", block.ProposerID, leader, block.View)) + return model.NewInvalidBlockErrorf(proposal, "proposer %s is not leader (%s) for view %d", block.ProposerID, leader, block.View) } // The Block must contain a proof that the primary legitimately entered the respective view. @@ -231,23 +231,23 @@ func (v *Validator) ValidateProposal(proposal *model.Proposal) error { if !lastViewSuccessful { // check if proposal is correctly structured if proposal.LastViewTC == nil { - return newInvalidBlockError(proposal, fmt.Errorf("QC in block is not for previous view, so expecting a TC but none is included in block")) + return model.NewInvalidBlockErrorf(proposal, "QC in block is not for previous view, so expecting a TC but none is included in block") } // check if included TC is for previous view if proposal.Block.View != proposal.LastViewTC.View+1 { - return newInvalidBlockError(proposal, fmt.Errorf("QC in block is not for previous view, so expecting a TC for view %d but got TC for view %d", proposal.Block.View-1, proposal.LastViewTC.View)) + return model.NewInvalidBlockErrorf(proposal, "QC in block is not for previous view, so expecting a TC for view %d but got TC for view %d", proposal.Block.View-1, proposal.LastViewTC.View) } // Check if proposal extends either the newest QC specified in the TC, or a newer QC // in edge cases a leader may construct a TC and QC concurrently such that TC contains // an older QC - in these case we still want to build on the newest QC, so this case is allowed. if proposal.Block.QC.View < proposal.LastViewTC.NewestQC.View { - return newInvalidBlockError(proposal, fmt.Errorf("TC in block contains a newer QC than the block itself, which is a protocol violation")) + return model.NewInvalidBlockErrorf(proposal, "TC in block contains a newer QC than the block itself, which is a protocol violation") } } else if proposal.LastViewTC != nil { // last view ended with QC, including TC is a protocol violation - return newInvalidBlockError(proposal, fmt.Errorf("last view has ended with QC but proposal includes LastViewTC")) + return model.NewInvalidBlockErrorf(proposal, "last view has ended with QC but proposal includes LastViewTC") } // Check signatures, keep the most expensive the last to check @@ -256,7 +256,7 @@ func (v *Validator) ValidateProposal(proposal *model.Proposal) error { err = v.ValidateQC(qc) if err != nil { if model.IsInvalidQCError(err) { - return newInvalidBlockError(proposal, fmt.Errorf("invalid qc included: %w", err)) + return model.NewInvalidBlockErrorf(proposal, "invalid qc included: %w", err) } if errors.Is(err, model.ErrViewForUnknownEpoch) { // We require each replica to be bootstrapped with a QC pointing to a finalized block. Therefore, we should know the @@ -272,7 +272,7 @@ func (v *Validator) ValidateProposal(proposal *model.Proposal) error { err = v.ValidateTC(proposal.LastViewTC) if err != nil { if model.IsInvalidTCError(err) { - return newInvalidBlockError(proposal, fmt.Errorf("proposals TC's is not valid: %w", err)) + return model.NewInvalidBlockErrorf(proposal, "proposals TC's is not valid: %w", err) } if errors.Is(err, model.ErrViewForUnknownEpoch) { // We require each replica to be bootstrapped with a QC pointing to a finalized block. Therefore, we should know the @@ -323,13 +323,6 @@ func (v *Validator) ValidateVote(vote *model.Vote) (*flow.Identity, error) { return voter, nil } -func newInvalidBlockError(proposal *model.Proposal, err error) error { - return model.InvalidBlockError{ - InvalidBlock: proposal, - Err: err, - } -} - func newInvalidQCError(qc *flow.QuorumCertificate, err error) error { return model.InvalidQCError{ BlockID: qc.BlockID, diff --git a/consensus/hotstuff/votecollector/factory.go b/consensus/hotstuff/votecollector/factory.go index 1b5dda84103..554c7675479 100644 --- a/consensus/hotstuff/votecollector/factory.go +++ b/consensus/hotstuff/votecollector/factory.go @@ -49,10 +49,7 @@ func (f *VoteProcessorFactory) Create(log zerolog.Logger, proposal *model.Propos err = processor.Process(proposal.ProposerVote()) if err != nil { if model.IsInvalidVoteError(err) { - return nil, model.InvalidBlockError{ - InvalidBlock: proposal, - Err: fmt.Errorf("invalid proposer vote: %w", err), - } + return nil, model.NewInvalidBlockErrorf(proposal, "invalid proposer vote: %w", err) } return nil, fmt.Errorf("processing proposer's vote for block %v failed: %w", proposal.Block.BlockID, err) } diff --git a/consensus/recovery/recover_test.go b/consensus/recovery/recover_test.go index c8b12c65c5e..94d7b76b6fe 100644 --- a/consensus/recovery/recover_test.go +++ b/consensus/recovery/recover_test.go @@ -1,7 +1,6 @@ package recovery import ( - "fmt" "testing" "github.com/stretchr/testify/mock" @@ -39,10 +38,7 @@ func TestRecover(t *testing.T) { validator.On("ValidateProposal", mock.Anything).Return(func(proposal *model.Proposal) error { _, isInvalid := invalid[proposal.Block.BlockID] if isInvalid { - return model.InvalidBlockError{ - InvalidBlock: proposal, - Err: fmt.Errorf(""), - } + return model.NewInvalidBlockErrorf(proposal, "") } return nil }) diff --git a/engine/common/follower/compliance_core_test.go b/engine/common/follower/compliance_core_test.go index be1989e8acb..41a98f63ec7 100644 --- a/engine/common/follower/compliance_core_test.go +++ b/engine/common/follower/compliance_core_test.go @@ -3,7 +3,6 @@ package follower import ( "context" "errors" - "fmt" "sync" "testing" "time" @@ -166,10 +165,7 @@ func (s *CoreSuite) TestProcessingInvalidBlock() { blocks := unittest.ChainFixtureFrom(10, s.finalizedBlock) invalidProposal := model.ProposalFromFlow(blocks[len(blocks)-1].Header) - sentinelError := model.InvalidBlockError{ - InvalidBlock: invalidProposal, - Err: fmt.Errorf(""), - } + sentinelError := model.NewInvalidBlockErrorf(invalidProposal, "") s.validator.On("ValidateProposal", invalidProposal).Return(sentinelError).Once() s.followerConsumer.On("OnInvalidBlockDetected", sentinelError).Return().Once() err := s.core.OnBlockRange(s.originID, blocks) From 6af3accedf66d0262ac326744e1e4ddec88e6a40 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Mon, 10 Apr 2023 15:32:59 +0300 Subject: [PATCH 05/38] Updated compliance engine to report protocol violations --- cmd/consensus/main.go | 72 +++++++++---------- consensus/config.go | 2 +- engine/collection/compliance/core.go | 50 +++++++------ engine/collection/compliance/core_test.go | 29 +++++--- .../epochmgr/factories/compliance.go | 2 + engine/collection/epochmgr/factories/epoch.go | 3 +- .../collection/epochmgr/factories/hotstuff.go | 2 +- .../execution_data_requester.go | 2 +- .../requester/execution_data_requester.go | 2 +- 9 files changed, 89 insertions(+), 75 deletions(-) diff --git a/cmd/consensus/main.go b/cmd/consensus/main.go index 059b8c89801..5fc02c23592 100644 --- a/cmd/consensus/main.go +++ b/cmd/consensus/main.go @@ -100,31 +100,31 @@ func main() { insecureAccessAPI bool accessNodeIDS []string - err error - mutableState protocol.ParticipantState - beaconPrivateKey *encodable.RandomBeaconPrivKey - guarantees mempool.Guarantees - receipts mempool.ExecutionTree - seals mempool.IncorporatedResultSeals - pendingReceipts mempool.PendingReceipts - receiptRequester *requester.Engine - syncCore *chainsync.Core - comp *compliance.Engine - hot module.HotStuff - conMetrics module.ConsensusMetrics - mainMetrics module.HotstuffMetrics - receiptValidator module.ReceiptValidator - chunkAssigner *chmodule.ChunkAssigner - finalizationDistributor *pubsub.FollowerDistributor - dkgBrokerTunnel *dkgmodule.BrokerTunnel - blockTimer protocol.BlockTimer - finalizedHeader *synceng.FinalizedHeaderCache - committee *committees.Consensus - epochLookup *epochs.EpochLookup - hotstuffModules *consensus.HotstuffModules - dkgState *bstorage.DKGState - safeBeaconKeys *bstorage.SafeBeaconPrivateKeys - getSealingConfigs module.SealingConfigsGetter + err error + mutableState protocol.ParticipantState + beaconPrivateKey *encodable.RandomBeaconPrivKey + guarantees mempool.Guarantees + receipts mempool.ExecutionTree + seals mempool.IncorporatedResultSeals + pendingReceipts mempool.PendingReceipts + receiptRequester *requester.Engine + syncCore *chainsync.Core + comp *compliance.Engine + hot module.HotStuff + conMetrics module.ConsensusMetrics + mainMetrics module.HotstuffMetrics + receiptValidator module.ReceiptValidator + chunkAssigner *chmodule.ChunkAssigner + followerDistributor *pubsub.FollowerDistributor + dkgBrokerTunnel *dkgmodule.BrokerTunnel + blockTimer protocol.BlockTimer + finalizedHeader *synceng.FinalizedHeaderCache + committee *committees.Consensus + epochLookup *epochs.EpochLookup + hotstuffModules *consensus.HotstuffModules + dkgState *bstorage.DKGState + safeBeaconKeys *bstorage.SafeBeaconPrivateKeys + getSealingConfigs module.SealingConfigsGetter ) nodeBuilder := cmd.FlowNode(flow.RoleConsensus.String()) @@ -365,8 +365,8 @@ func main() { return err }). Module("finalization distributor", func(node *cmd.NodeConfig) error { - finalizationDistributor = pubsub.NewFollowerDistributor() - finalizationDistributor.AddConsumer(notifications.NewSlashingViolationsConsumer(nodeBuilder.Logger)) + followerDistributor = pubsub.NewFollowerDistributor() + followerDistributor.AddConsumer(notifications.NewSlashingViolationsConsumer(nodeBuilder.Logger)) return nil }). Module("machine account config", func(node *cmd.NodeConfig) error { @@ -432,8 +432,8 @@ func main() { ) // subscribe for finalization events from hotstuff - finalizationDistributor.AddOnBlockFinalizedConsumer(e.OnFinalizedBlock) - finalizationDistributor.AddOnBlockIncorporatedConsumer(e.OnBlockIncorporated) + followerDistributor.AddOnBlockFinalizedConsumer(e.OnFinalizedBlock) + followerDistributor.AddOnBlockIncorporatedConsumer(e.OnBlockIncorporated) return e, err }). @@ -487,8 +487,8 @@ func main() { // subscribe engine to inputs from other node-internal components receiptRequester.WithHandle(e.HandleReceipt) - finalizationDistributor.AddOnBlockFinalizedConsumer(e.OnFinalizedBlock) - finalizationDistributor.AddOnBlockIncorporatedConsumer(e.OnBlockIncorporated) + followerDistributor.AddOnBlockFinalizedConsumer(e.OnFinalizedBlock) + followerDistributor.AddOnBlockIncorporatedConsumer(e.OnBlockIncorporated) return e, err }). @@ -560,7 +560,7 @@ func main() { mainMetrics, ) - notifier.AddFollowerConsumer(finalizationDistributor) + notifier.AddFollowerConsumer(followerDistributor) // initialize the persister persist := persister.New(node.DB, node.RootChainID) @@ -594,7 +594,7 @@ func main() { lowestViewForVoteProcessing, notifier, voteProcessorFactory, - finalizationDistributor) + followerDistributor) if err != nil { return nil, fmt.Errorf("could not initialize vote aggregator: %w", err) } @@ -627,7 +627,7 @@ func main() { Signer: signer, Persist: persist, QCCreatedDistributor: qcDistributor, - FinalizationDistributor: finalizationDistributor, + FollowerDistributor: followerDistributor, TimeoutCollectorDistributor: timeoutCollectorDistributor, Forks: forks, Validator: validator, @@ -731,7 +731,7 @@ func main() { return nil, fmt.Errorf("could not initialize compliance engine: %w", err) } - finalizationDistributor.AddOnBlockFinalizedConsumer(comp.OnFinalizedBlock) + followerDistributor.AddOnBlockFinalizedConsumer(comp.OnFinalizedBlock) return comp, nil }). @@ -755,7 +755,7 @@ func main() { return messageHub, nil }). Component("finalized snapshot", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { - finalizedHeader, err = synceng.NewFinalizedHeaderCache(node.Logger, node.State, finalizationDistributor) + finalizedHeader, err = synceng.NewFinalizedHeaderCache(node.Logger, node.State, followerDistributor) if err != nil { return nil, fmt.Errorf("could not create finalized snapshot cache: %w", err) } diff --git a/consensus/config.go b/consensus/config.go index cc99fd7991d..6e8497ff2cc 100644 --- a/consensus/config.go +++ b/consensus/config.go @@ -16,7 +16,7 @@ type HotstuffModules struct { Signer hotstuff.Signer // signer of proposal & votes Persist hotstuff.Persister // last state of consensus participant Notifier *pubsub.Distributor // observer for hotstuff events - FinalizationDistributor *pubsub.FollowerDistributor // observer for finalization events, used by compliance engine + FollowerDistributor *pubsub.FollowerDistributor // observer for finalization events, used by compliance engine QCCreatedDistributor *pubsub.QCCreatedDistributor // observer for qc created event, used by leader TimeoutCollectorDistributor *pubsub.TimeoutCollectorDistributor Forks hotstuff.Forks // information about multiple forks diff --git a/engine/collection/compliance/core.go b/engine/collection/compliance/core.go index 568ab3fce17..9139717951b 100644 --- a/engine/collection/compliance/core.go +++ b/engine/collection/compliance/core.go @@ -34,14 +34,15 @@ import ( // - The only exception is calls to `ProcessFinalizedView`, which is the only concurrency-safe // method of compliance.Core type Core struct { - log zerolog.Logger // used to log relevant actions with context - config compliance.Config - engineMetrics module.EngineMetrics - mempoolMetrics module.MempoolMetrics - hotstuffMetrics module.HotstuffMetrics - collectionMetrics module.CollectionMetrics - headers storage.Headers - state clusterkv.MutableState + log zerolog.Logger // used to log relevant actions with context + config compliance.Config + engineMetrics module.EngineMetrics + mempoolMetrics module.MempoolMetrics + hotstuffMetrics module.HotstuffMetrics + collectionMetrics module.CollectionMetrics + protocolViolationNotifier hotstuff.BaseProtocolViolationConsumer + headers storage.Headers + state clusterkv.MutableState // track latest finalized view/height - used to efficiently drop outdated or too-far-ahead blocks finalizedView counters.StrictMonotonousCounter finalizedHeight counters.StrictMonotonousCounter @@ -60,6 +61,7 @@ func NewCore( mempool module.MempoolMetrics, hotstuffMetrics module.HotstuffMetrics, collectionMetrics module.CollectionMetrics, + protocolViolationNotifier hotstuff.BaseProtocolViolationConsumer, headers storage.Headers, state clusterkv.MutableState, pending module.PendingClusterBlockBuffer, @@ -77,20 +79,21 @@ func NewCore( } c := &Core{ - log: log.With().Str("cluster_compliance", "core").Logger(), - config: config, - engineMetrics: collector, - mempoolMetrics: mempool, - hotstuffMetrics: hotstuffMetrics, - collectionMetrics: collectionMetrics, - headers: headers, - state: state, - pending: pending, - sync: sync, - hotstuff: hotstuff, - validator: validator, - voteAggregator: voteAggregator, - timeoutAggregator: timeoutAggregator, + log: log.With().Str("cluster_compliance", "core").Logger(), + config: config, + engineMetrics: collector, + mempoolMetrics: mempool, + hotstuffMetrics: hotstuffMetrics, + collectionMetrics: collectionMetrics, + protocolViolationNotifier: protocolViolationNotifier, + headers: headers, + state: state, + pending: pending, + sync: sync, + hotstuff: hotstuff, + validator: validator, + voteAggregator: voteAggregator, + timeoutAggregator: timeoutAggregator, } // initialize finalized boundary cache @@ -312,7 +315,8 @@ func (c *Core) processBlockProposal(proposal *cluster.Block, parent *flow.Header hotstuffProposal := model.ProposalFromFlow(header) err := c.validator.ValidateProposal(hotstuffProposal) if err != nil { - if model.IsInvalidBlockError(err) { + if invalidBlockErr, ok := model.AsInvalidBlockError(err); ok { + c.protocolViolationNotifier.OnInvalidBlockDetected(*invalidBlockErr) return engine.NewInvalidInputErrorf("invalid block proposal: %w", err) } if errors.Is(err, model.ErrViewForUnknownEpoch) { diff --git a/engine/collection/compliance/core_test.go b/engine/collection/compliance/core_test.go index ffa490fb31e..ecf914af8f6 100644 --- a/engine/collection/compliance/core_test.go +++ b/engine/collection/compliance/core_test.go @@ -49,16 +49,17 @@ type CommonSuite struct { childrenDB map[flow.Identifier][]flow.Slashable[*cluster.Block] // mocked dependencies - state *clusterstate.MutableState - snapshot *clusterstate.Snapshot - metrics *metrics.NoopCollector - headers *storage.Headers - pending *module.PendingClusterBlockBuffer - hotstuff *module.HotStuff - sync *module.BlockRequester - validator *hotstuff.Validator - voteAggregator *hotstuff.VoteAggregator - timeoutAggregator *hotstuff.TimeoutAggregator + state *clusterstate.MutableState + snapshot *clusterstate.Snapshot + metrics *metrics.NoopCollector + protocolViolationNotifier *hotstuff.BaseProtocolViolationConsumer + headers *storage.Headers + pending *module.PendingClusterBlockBuffer + hotstuff *module.HotStuff + sync *module.BlockRequester + validator *hotstuff.Validator + voteAggregator *hotstuff.VoteAggregator + timeoutAggregator *hotstuff.TimeoutAggregator // engine under test core *Core @@ -166,6 +167,9 @@ func (cs *CommonSuite) SetupTest() { // set up no-op metrics mock cs.metrics = metrics.NewNoopCollector() + // set up notifier for reporting protocol violations + cs.protocolViolationNotifier = hotstuff.NewBaseProtocolViolationConsumer(cs.T()) + // initialize the engine core, err := NewCore( unittest.Logger(), @@ -173,6 +177,7 @@ func (cs *CommonSuite) SetupTest() { cs.metrics, cs.metrics, cs.metrics, + cs.protocolViolationNotifier, cs.headers, cs.state, cs.pending, @@ -272,7 +277,9 @@ func (cs *CoreSuite) TestOnBlockProposal_FailsHotStuffValidation() { cs.Run("invalid block error", func() { // the block fails HotStuff validation *cs.validator = *hotstuff.NewValidator(cs.T()) - cs.validator.On("ValidateProposal", hotstuffProposal).Return(model.InvalidBlockError{}) + sentinelError := model.NewInvalidBlockErrorf(hotstuffProposal, "") + cs.validator.On("ValidateProposal", hotstuffProposal).Return(sentinelError) + cs.protocolViolationNotifier.On("OnInvalidBlockDetected", sentinelError).Return().Once() // we should notify VoteAggregator about the invalid block cs.voteAggregator.On("InvalidBlock", hotstuffProposal).Return(nil) diff --git a/engine/collection/epochmgr/factories/compliance.go b/engine/collection/epochmgr/factories/compliance.go index 777a5db03b6..1beed415634 100644 --- a/engine/collection/epochmgr/factories/compliance.go +++ b/engine/collection/epochmgr/factories/compliance.go @@ -58,6 +58,7 @@ func NewComplianceEngineFactory( func (f *ComplianceEngineFactory) Create( hotstuffMetrics module.HotstuffMetrics, + notifier hotstuff.BaseProtocolViolationConsumer, clusterState cluster.MutableState, headers storage.Headers, payloads storage.ClusterPayloads, @@ -75,6 +76,7 @@ func (f *ComplianceEngineFactory) Create( f.mempoolMetrics, hotstuffMetrics, f.colMetrics, + notifier, headers, clusterState, cache, diff --git a/engine/collection/epochmgr/factories/epoch.go b/engine/collection/epochmgr/factories/epoch.go index ca5bb9b03e4..4ea2757b592 100644 --- a/engine/collection/epochmgr/factories/epoch.go +++ b/engine/collection/epochmgr/factories/epoch.go @@ -161,6 +161,7 @@ func (factory *EpochComponentsFactory) Create( complianceEng, err := factory.compliance.Create( metrics, + hotstuffModules.FollowerDistributor, mutableState, headers, payloads, @@ -175,7 +176,7 @@ func (factory *EpochComponentsFactory) Create( return } compliance = complianceEng - hotstuffModules.FinalizationDistributor.AddOnBlockFinalizedConsumer(complianceEng.OnFinalizedBlock) + hotstuffModules.FollowerDistributor.AddOnBlockFinalizedConsumer(complianceEng.OnFinalizedBlock) sync, err = factory.sync.Create(cluster.Members(), state, blocks, syncCore, complianceEng) if err != nil { diff --git a/engine/collection/epochmgr/factories/hotstuff.go b/engine/collection/epochmgr/factories/hotstuff.go index 7d56fd72f3d..5eafd066ca7 100644 --- a/engine/collection/epochmgr/factories/hotstuff.go +++ b/engine/collection/epochmgr/factories/hotstuff.go @@ -163,7 +163,7 @@ func (f *HotStuffFactory) CreateModules( TimeoutAggregator: timeoutAggregator, QCCreatedDistributor: qcDistributor, TimeoutCollectorDistributor: timeoutCollectorDistributor, - FinalizationDistributor: followerDistributor, + FollowerDistributor: followerDistributor, }, metrics, nil } diff --git a/module/state_synchronization/execution_data_requester.go b/module/state_synchronization/execution_data_requester.go index e1671d89f87..18875950c31 100644 --- a/module/state_synchronization/execution_data_requester.go +++ b/module/state_synchronization/execution_data_requester.go @@ -14,7 +14,7 @@ type ExecutionDataReceivedCallback func(*execution_data.BlockExecutionData) type ExecutionDataRequester interface { component.Component - // OnBlockFinalized accepts block finalization notifications from the FinalizationDistributor + // OnBlockFinalized accepts block finalization notifications from the FollowerDistributor OnBlockFinalized(*model.Block) // AddOnExecutionDataFetchedConsumer adds a callback to be called when a new ExecutionData is received diff --git a/module/state_synchronization/requester/execution_data_requester.go b/module/state_synchronization/requester/execution_data_requester.go index 23667ab6e48..9dec1718942 100644 --- a/module/state_synchronization/requester/execution_data_requester.go +++ b/module/state_synchronization/requester/execution_data_requester.go @@ -247,7 +247,7 @@ func New( return e } -// OnBlockFinalized accepts block finalization notifications from the FinalizationDistributor +// OnBlockFinalized accepts block finalization notifications from the FollowerDistributor func (e *executionDataRequester) OnBlockFinalized(*model.Block) { e.finalizationNotifier.Notify() } From 412497517d0f890ab7288e9bb090be934019cdec Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Mon, 10 Apr 2023 15:44:50 +0300 Subject: [PATCH 06/38] Updated main consensus compliance engine to report protocol violations --- cmd/consensus/main.go | 1 + engine/consensus/compliance/core.go | 58 +++++++++++++----------- engine/consensus/compliance/core_test.go | 41 ++++++++++------- 3 files changed, 56 insertions(+), 44 deletions(-) diff --git a/cmd/consensus/main.go b/cmd/consensus/main.go index 5fc02c23592..1b25780fb80 100644 --- a/cmd/consensus/main.go +++ b/cmd/consensus/main.go @@ -705,6 +705,7 @@ func main() { node.Metrics.Mempool, mainMetrics, node.Metrics.Compliance, + followerDistributor, node.Tracer, node.Storage.Headers, node.Storage.Payloads, diff --git a/engine/consensus/compliance/core.go b/engine/consensus/compliance/core.go index d38e2b78dd4..c739d660a2e 100644 --- a/engine/consensus/compliance/core.go +++ b/engine/consensus/compliance/core.go @@ -36,16 +36,17 @@ import ( // - The only exception is calls to `ProcessFinalizedView`, which is the only concurrency-safe // method of compliance.Core type Core struct { - log zerolog.Logger // used to log relevant actions with context - config compliance.Config - engineMetrics module.EngineMetrics - mempoolMetrics module.MempoolMetrics - hotstuffMetrics module.HotstuffMetrics - complianceMetrics module.ComplianceMetrics - tracer module.Tracer - headers storage.Headers - payloads storage.Payloads - state protocol.ParticipantState + log zerolog.Logger // used to log relevant actions with context + config compliance.Config + engineMetrics module.EngineMetrics + mempoolMetrics module.MempoolMetrics + hotstuffMetrics module.HotstuffMetrics + complianceMetrics module.ComplianceMetrics + protocolViolationNotifier hotstuff.BaseProtocolViolationConsumer + tracer module.Tracer + headers storage.Headers + payloads storage.Payloads + state protocol.ParticipantState // track latest finalized view/height - used to efficiently drop outdated or too-far-ahead blocks finalizedView counters.StrictMonotonousCounter finalizedHeight counters.StrictMonotonousCounter @@ -64,6 +65,7 @@ func NewCore( mempool module.MempoolMetrics, hotstuffMetrics module.HotstuffMetrics, complianceMetrics module.ComplianceMetrics, + protocolViolationNotifier hotstuff.BaseProtocolViolationConsumer, tracer module.Tracer, headers storage.Headers, payloads storage.Payloads, @@ -83,22 +85,23 @@ func NewCore( } c := &Core{ - log: log.With().Str("compliance", "core").Logger(), - config: config, - engineMetrics: collector, - tracer: tracer, - mempoolMetrics: mempool, - hotstuffMetrics: hotstuffMetrics, - complianceMetrics: complianceMetrics, - headers: headers, - payloads: payloads, - state: state, - pending: pending, - sync: sync, - hotstuff: hotstuff, - validator: validator, - voteAggregator: voteAggregator, - timeoutAggregator: timeoutAggregator, + log: log.With().Str("compliance", "core").Logger(), + config: config, + engineMetrics: collector, + tracer: tracer, + mempoolMetrics: mempool, + hotstuffMetrics: hotstuffMetrics, + complianceMetrics: complianceMetrics, + protocolViolationNotifier: protocolViolationNotifier, + headers: headers, + payloads: payloads, + state: state, + pending: pending, + sync: sync, + hotstuff: hotstuff, + validator: validator, + voteAggregator: voteAggregator, + timeoutAggregator: timeoutAggregator, } // initialize finalized boundary cache @@ -320,7 +323,8 @@ func (c *Core) processBlockProposal(proposal *flow.Block, parent *flow.Header) e hotstuffProposal := model.ProposalFromFlow(header) err := c.validator.ValidateProposal(hotstuffProposal) if err != nil { - if model.IsInvalidBlockError(err) { + if invalidBlockErr, ok := model.AsInvalidBlockError(err); ok { + c.protocolViolationNotifier.OnInvalidBlockDetected(*invalidBlockErr) return engine.NewInvalidInputErrorf("invalid block proposal: %w", err) } if errors.Is(err, model.ErrViewForUnknownEpoch) { diff --git a/engine/consensus/compliance/core_test.go b/engine/consensus/compliance/core_test.go index 34bc9e3570c..829d7af77c2 100644 --- a/engine/consensus/compliance/core_test.go +++ b/engine/consensus/compliance/core_test.go @@ -58,22 +58,23 @@ type CommonSuite struct { childrenDB map[flow.Identifier][]flow.Slashable[*flow.Block] // mocked dependencies - me *module.Local - metrics *metrics.NoopCollector - tracer realModule.Tracer - headers *storage.Headers - payloads *storage.Payloads - state *protocol.ParticipantState - snapshot *protocol.Snapshot - con *mocknetwork.Conduit - net *mocknetwork.Network - prov *consensus.ProposalProvider - pending *module.PendingBlockBuffer - hotstuff *module.HotStuff - sync *module.BlockRequester - validator *hotstuff.Validator - voteAggregator *hotstuff.VoteAggregator - timeoutAggregator *hotstuff.TimeoutAggregator + me *module.Local + metrics *metrics.NoopCollector + tracer realModule.Tracer + headers *storage.Headers + payloads *storage.Payloads + state *protocol.ParticipantState + snapshot *protocol.Snapshot + con *mocknetwork.Conduit + net *mocknetwork.Network + prov *consensus.ProposalProvider + pending *module.PendingBlockBuffer + hotstuff *module.HotStuff + sync *module.BlockRequester + protocolViolationNotifier *hotstuff.BaseProtocolViolationConsumer + validator *hotstuff.Validator + voteAggregator *hotstuff.VoteAggregator + timeoutAggregator *hotstuff.TimeoutAggregator // engine under test core *Core @@ -244,6 +245,9 @@ func (cs *CommonSuite) SetupTest() { // set up no-op tracer cs.tracer = trace.NewNoopTracer() + // set up notifier for reporting protocol violations + cs.protocolViolationNotifier = hotstuff.NewBaseProtocolViolationConsumer(cs.T()) + // initialize the engine e, err := NewCore( unittest.Logger(), @@ -251,6 +255,7 @@ func (cs *CommonSuite) SetupTest() { cs.metrics, cs.metrics, cs.metrics, + cs.protocolViolationNotifier, cs.tracer, cs.headers, cs.payloads, @@ -355,7 +360,9 @@ func (cs *CoreSuite) TestOnBlockProposal_FailsHotStuffValidation() { cs.Run("invalid block error", func() { // the block fails HotStuff validation *cs.validator = *hotstuff.NewValidator(cs.T()) - cs.validator.On("ValidateProposal", hotstuffProposal).Return(model.InvalidBlockError{}) + sentinelError := model.NewInvalidBlockErrorf(hotstuffProposal, "") + cs.validator.On("ValidateProposal", hotstuffProposal).Return(sentinelError) + cs.protocolViolationNotifier.On("OnInvalidBlockDetected", sentinelError).Return().Once() // we should notify VoteAggregator about the invalid block cs.voteAggregator.On("InvalidBlock", hotstuffProposal).Return(nil) From e46d74e4aad526ec7002fba3682197528ba897e2 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Mon, 10 Apr 2023 18:24:52 +0300 Subject: [PATCH 07/38] Fixed tests --- consensus/hotstuff/follower/follower.go | 1 + consensus/integration/nodes_test.go | 1 + 2 files changed, 2 insertions(+) diff --git a/consensus/hotstuff/follower/follower.go b/consensus/hotstuff/follower/follower.go index cef8b3d0c1b..833f805a13b 100644 --- a/consensus/hotstuff/follower/follower.go +++ b/consensus/hotstuff/follower/follower.go @@ -44,6 +44,7 @@ func (f *FollowerLogic) FinalizedBlock() *model.Block { // AddBlock processes the given block proposal func (f *FollowerLogic) AddBlock(blockProposal *model.Proposal) error { // validate the block. skip if the proposal is invalid + // TODO: this block was already validated by follower engine, to be refactored err := f.validator.ValidateProposal(blockProposal) if err != nil { if model.IsInvalidBlockError(err) { diff --git a/consensus/integration/nodes_test.go b/consensus/integration/nodes_test.go index 2dd716b53f4..817fc46fcba 100644 --- a/consensus/integration/nodes_test.go +++ b/consensus/integration/nodes_test.go @@ -594,6 +594,7 @@ func createNode( metricsCollector, metricsCollector, metricsCollector, + notifier, tracer, headersDB, payloadsDB, From a4cde5ec1cd7ab1117121ac8dfa462d427394432 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Mon, 10 Apr 2023 23:13:44 +0300 Subject: [PATCH 08/38] Updated naming of notification distributors --- .../node_builder/access_node_builder.go | 18 ++-- cmd/collection/main.go | 18 ++-- cmd/consensus/main.go | 2 +- cmd/execution_builder.go | 82 +++++++++---------- cmd/observer/node_builder/observer_builder.go | 18 ++-- cmd/verification_builder.go | 32 ++++---- .../synchronization/finalized_snapshot.go | 4 +- follower/consensus_follower.go | 2 +- follower/follower_builder.go | 32 ++++---- .../requester/execution_data_requester.go | 2 +- .../execution_data_requester_test.go | 32 ++++---- 11 files changed, 121 insertions(+), 121 deletions(-) diff --git a/cmd/access/node_builder/access_node_builder.go b/cmd/access/node_builder/access_node_builder.go index ef211c178fc..9707db39762 100644 --- a/cmd/access/node_builder/access_node_builder.go +++ b/cmd/access/node_builder/access_node_builder.go @@ -196,7 +196,7 @@ type FlowAccessNodeBuilder struct { FollowerState protocol.FollowerState SyncCore *chainsync.Core RpcEng *rpc.Engine - FinalizationDistributor *consensuspubsub.FollowerDistributor + FollowerDistributor *consensuspubsub.FollowerDistributor FinalizedHeader *synceng.FinalizedHeaderCache CollectionRPC access.AccessAPIClient TransactionTimings *stdmap.TransactionTimings @@ -308,7 +308,7 @@ func (builder *FlowAccessNodeBuilder) buildFollowerCore() *FlowAccessNodeBuilder node.Storage.Headers, final, verifier, - builder.FinalizationDistributor, + builder.FollowerDistributor, node.RootBlock.Header, node.RootQC, builder.Finalized, @@ -336,7 +336,7 @@ func (builder *FlowAccessNodeBuilder) buildFollowerEngine() *FlowAccessNodeBuild node.Logger, node.Metrics.Mempool, heroCacheCollector, - builder.FinalizationDistributor, + builder.FollowerDistributor, builder.FollowerState, builder.FollowerCore, builder.Validator, @@ -369,7 +369,7 @@ func (builder *FlowAccessNodeBuilder) buildFollowerEngine() *FlowAccessNodeBuild func (builder *FlowAccessNodeBuilder) buildFinalizedHeader() *FlowAccessNodeBuilder { builder.Component("finalized snapshot", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { - finalizedHeader, err := synceng.NewFinalizedHeaderCache(node.Logger, node.State, builder.FinalizationDistributor) + finalizedHeader, err := synceng.NewFinalizedHeaderCache(node.Logger, node.State, builder.FollowerDistributor) if err != nil { return nil, fmt.Errorf("could not create finalized snapshot cache: %w", err) } @@ -544,7 +544,7 @@ func (builder *FlowAccessNodeBuilder) BuildExecutionDataRequester() *FlowAccessN builder.executionDataConfig, ) - builder.FinalizationDistributor.AddOnBlockFinalizedConsumer(builder.ExecutionDataRequester.OnBlockFinalized) + builder.FollowerDistributor.AddOnBlockFinalizedConsumer(builder.ExecutionDataRequester.OnBlockFinalized) return builder.ExecutionDataRequester, nil }) @@ -579,9 +579,9 @@ func FlowAccessNode(nodeBuilder *cmd.FlowNodeBuilder) *FlowAccessNodeBuilder { dist := consensuspubsub.NewFollowerDistributor() dist.AddConsumer(notifications.NewSlashingViolationsConsumer(nodeBuilder.Logger)) return &FlowAccessNodeBuilder{ - AccessNodeConfig: DefaultAccessNodeConfig(), - FlowNodeBuilder: nodeBuilder, - FinalizationDistributor: dist, + AccessNodeConfig: DefaultAccessNodeConfig(), + FlowNodeBuilder: nodeBuilder, + FollowerDistributor: dist, } } @@ -962,7 +962,7 @@ func (builder *FlowAccessNodeBuilder) Build() (cmd.Node, error) { return nil, err } builder.RequestEng.WithHandle(builder.IngestEng.OnCollection) - builder.FinalizationDistributor.AddOnBlockFinalizedConsumer(builder.IngestEng.OnFinalizedBlock) + builder.FollowerDistributor.AddOnBlockFinalizedConsumer(builder.IngestEng.OnFinalizedBlock) return builder.IngestEng, nil }). diff --git a/cmd/collection/main.go b/cmd/collection/main.go index b070f362e3f..d55b58bc8ab 100644 --- a/cmd/collection/main.go +++ b/cmd/collection/main.go @@ -78,9 +78,9 @@ func main() { rpcConf rpc.Config clusterComplianceConfig modulecompliance.Config - pools *epochpool.TransactionPools // epoch-scoped transaction pools - finalizationDistributor *pubsub.FollowerDistributor - finalizedHeader *consync.FinalizedHeaderCache + pools *epochpool.TransactionPools // epoch-scoped transaction pools + followerDistributor *pubsub.FollowerDistributor + finalizedHeader *consync.FinalizedHeaderCache push *pusher.Engine ing *ingest.Engine @@ -171,9 +171,9 @@ func main() { nodeBuilder. PreInit(cmd.DynamicStartPreInit). - Module("finalization distributor", func(node *cmd.NodeConfig) error { - finalizationDistributor = pubsub.NewFollowerDistributor() - finalizationDistributor.AddConsumer(notifications.NewSlashingViolationsConsumer(node.Logger)) + Module("follower distributor", func(node *cmd.NodeConfig) error { + followerDistributor = pubsub.NewFollowerDistributor() + followerDistributor.AddConsumer(notifications.NewSlashingViolationsConsumer(node.Logger)) return nil }). Module("mutable follower state", func(node *cmd.NodeConfig) error { @@ -259,7 +259,7 @@ func main() { return validator, err }). Component("finalized snapshot", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { - finalizedHeader, err = consync.NewFinalizedHeaderCache(node.Logger, node.State, finalizationDistributor) + finalizedHeader, err = consync.NewFinalizedHeaderCache(node.Logger, node.State, followerDistributor) if err != nil { return nil, fmt.Errorf("could not create finalized snapshot cache: %w", err) } @@ -292,7 +292,7 @@ func main() { node.Storage.Headers, finalizer, verifier, - finalizationDistributor, + followerDistributor, node.RootBlock.Header, node.RootQC, finalized, @@ -319,7 +319,7 @@ func main() { node.Logger, node.Metrics.Mempool, heroCacheCollector, - finalizationDistributor, + followerDistributor, followerState, followerCore, validator, diff --git a/cmd/consensus/main.go b/cmd/consensus/main.go index 1b25780fb80..280ecd8be48 100644 --- a/cmd/consensus/main.go +++ b/cmd/consensus/main.go @@ -364,7 +364,7 @@ func main() { syncCore, err = chainsync.New(node.Logger, node.SyncCoreConfig, metrics.NewChainSyncCollector(node.RootChainID), node.RootChainID) return err }). - Module("finalization distributor", func(node *cmd.NodeConfig) error { + Module("follower distributor", func(node *cmd.NodeConfig) error { followerDistributor = pubsub.NewFollowerDistributor() followerDistributor.AddConsumer(notifications.NewSlashingViolationsConsumer(nodeBuilder.Logger)) return nil diff --git a/cmd/execution_builder.go b/cmd/execution_builder.go index cddd39cc95f..28ed7b742e5 100644 --- a/cmd/execution_builder.go +++ b/cmd/execution_builder.go @@ -109,39 +109,39 @@ type ExecutionNode struct { builder *FlowNodeBuilder // This is needed for accessing the ShutdownFunc exeConf *ExecutionConfig - collector module.ExecutionMetrics - executionState state.ExecutionState - followerState protocol.FollowerState - committee hotstuff.DynamicCommittee - ledgerStorage *ledger.Ledger - events *storage.Events - serviceEvents *storage.ServiceEvents - txResults *storage.TransactionResults - results *storage.ExecutionResults - myReceipts *storage.MyExecutionReceipts - providerEngine *exeprovider.Engine - checkerEng *checker.Engine - syncCore *chainsync.Core - syncEngine *synchronization.Engine - followerCore *hotstuff.FollowerLoop // follower hotstuff logic - followerEng *followereng.ComplianceEngine // to sync blocks from consensus nodes - computationManager *computation.Manager - collectionRequester *requester.Engine - ingestionEng *ingestion.Engine - finalizationDistributor *pubsub.FollowerDistributor - finalizedHeader *synchronization.FinalizedHeaderCache - checkAuthorizedAtBlock func(blockID flow.Identifier) (bool, error) - diskWAL *wal.DiskWAL - blockDataUploader *uploader.Manager - executionDataStore execution_data.ExecutionDataStore - toTriggerCheckpoint *atomic.Bool // create the checkpoint trigger to be controlled by admin tool, and listened by the compactor - stopControl *ingestion.StopControl // stop the node at given block height - executionDataDatastore *badger.Datastore - executionDataPruner *pruner.Pruner - executionDataBlobstore blobs.Blobstore - executionDataTracker tracker.Storage - blobService network.BlobService - blobserviceDependable *module.ProxiedReadyDoneAware + collector module.ExecutionMetrics + executionState state.ExecutionState + followerState protocol.FollowerState + committee hotstuff.DynamicCommittee + ledgerStorage *ledger.Ledger + events *storage.Events + serviceEvents *storage.ServiceEvents + txResults *storage.TransactionResults + results *storage.ExecutionResults + myReceipts *storage.MyExecutionReceipts + providerEngine *exeprovider.Engine + checkerEng *checker.Engine + syncCore *chainsync.Core + syncEngine *synchronization.Engine + followerCore *hotstuff.FollowerLoop // follower hotstuff logic + followerEng *followereng.ComplianceEngine // to sync blocks from consensus nodes + computationManager *computation.Manager + collectionRequester *requester.Engine + ingestionEng *ingestion.Engine + followerDistributor *pubsub.FollowerDistributor + finalizedHeader *synchronization.FinalizedHeaderCache + checkAuthorizedAtBlock func(blockID flow.Identifier) (bool, error) + diskWAL *wal.DiskWAL + blockDataUploader *uploader.Manager + executionDataStore execution_data.ExecutionDataStore + toTriggerCheckpoint *atomic.Bool // create the checkpoint trigger to be controlled by admin tool, and listened by the compactor + stopControl *ingestion.StopControl // stop the node at given block height + executionDataDatastore *badger.Datastore + executionDataPruner *pruner.Pruner + executionDataBlobstore blobs.Blobstore + executionDataTracker tracker.Storage + blobService network.BlobService + blobserviceDependable *module.ProxiedReadyDoneAware } func (builder *ExecutionNodeBuilder) LoadComponentsAndModules() { @@ -173,7 +173,7 @@ func (builder *ExecutionNodeBuilder) LoadComponentsAndModules() { Module("execution metrics", exeNode.LoadExecutionMetrics). Module("sync core", exeNode.LoadSyncCore). Module("execution receipts storage", exeNode.LoadExecutionReceiptsStorage). - Module("finalization distributor", exeNode.LoadFinalizationDistributor). + Module("follower distributor", exeNode.LoadFollowerDistributor). Module("authorization checking function", exeNode.LoadAuthorizationCheckingFunction). Module("execution data datastore", exeNode.LoadExecutionDataDatastore). Module("execution data getter", exeNode.LoadExecutionDataGetter). @@ -272,9 +272,9 @@ func (exeNode *ExecutionNode) LoadExecutionReceiptsStorage( return nil } -func (exeNode *ExecutionNode) LoadFinalizationDistributor(node *NodeConfig) error { - exeNode.finalizationDistributor = pubsub.NewFollowerDistributor() - exeNode.finalizationDistributor.AddConsumer(notifications.NewSlashingViolationsConsumer(node.Logger)) +func (exeNode *ExecutionNode) LoadFollowerDistributor(node *NodeConfig) error { + exeNode.followerDistributor = pubsub.NewFollowerDistributor() + exeNode.followerDistributor.AddConsumer(notifications.NewSlashingViolationsConsumer(node.Logger)) return nil } @@ -858,7 +858,7 @@ func (exeNode *ExecutionNode) LoadFollowerCore( return nil, fmt.Errorf("could not find latest finalized block and pending blocks to recover consensus follower: %w", err) } - exeNode.finalizationDistributor.AddConsumer(exeNode.checkerEng) + exeNode.followerDistributor.AddConsumer(exeNode.checkerEng) // creates a consensus follower with ingestEngine as the notifier // so that it gets notified upon each new finalized block @@ -868,7 +868,7 @@ func (exeNode *ExecutionNode) LoadFollowerCore( node.Storage.Headers, final, verifier, - exeNode.finalizationDistributor, + exeNode.followerDistributor, node.RootBlock.Header, node.RootQC, finalized, @@ -901,7 +901,7 @@ func (exeNode *ExecutionNode) LoadFollowerEngine( node.Logger, node.Metrics.Mempool, heroCacheCollector, - exeNode.finalizationDistributor, + exeNode.followerDistributor, exeNode.followerState, exeNode.followerCore, validator, @@ -982,7 +982,7 @@ func (exeNode *ExecutionNode) LoadFinalizedSnapshot( error, ) { var err error - exeNode.finalizedHeader, err = synchronization.NewFinalizedHeaderCache(node.Logger, node.State, exeNode.finalizationDistributor) + exeNode.finalizedHeader, err = synchronization.NewFinalizedHeaderCache(node.Logger, node.State, exeNode.followerDistributor) if err != nil { return nil, fmt.Errorf("could not create finalized snapshot cache: %w", err) } diff --git a/cmd/observer/node_builder/observer_builder.go b/cmd/observer/node_builder/observer_builder.go index 77948fd6d8f..4994f26172f 100644 --- a/cmd/observer/node_builder/observer_builder.go +++ b/cmd/observer/node_builder/observer_builder.go @@ -170,7 +170,7 @@ type ObserverServiceBuilder struct { FollowerState stateprotocol.FollowerState SyncCore *chainsync.Core RpcEng *rpc.Engine - FinalizationDistributor *pubsub.FollowerDistributor + FollowerDistributor *pubsub.FollowerDistributor FinalizedHeader *synceng.FinalizedHeaderCache Committee hotstuff.DynamicCommittee Finalized *flow.Header @@ -340,7 +340,7 @@ func (builder *ObserverServiceBuilder) buildFollowerCore() *ObserverServiceBuild node.Storage.Headers, final, verifier, - builder.FinalizationDistributor, + builder.FollowerDistributor, node.RootBlock.Header, node.RootQC, builder.Finalized, @@ -367,7 +367,7 @@ func (builder *ObserverServiceBuilder) buildFollowerEngine() *ObserverServiceBui node.Logger, node.Metrics.Mempool, heroCacheCollector, - builder.FinalizationDistributor, + builder.FollowerDistributor, builder.FollowerState, builder.FollowerCore, builder.Validator, @@ -401,7 +401,7 @@ func (builder *ObserverServiceBuilder) buildFollowerEngine() *ObserverServiceBui func (builder *ObserverServiceBuilder) buildFinalizedHeader() *ObserverServiceBuilder { builder.Component("finalized snapshot", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { - finalizedHeader, err := synceng.NewFinalizedHeaderCache(node.Logger, node.State, builder.FinalizationDistributor) + finalizedHeader, err := synceng.NewFinalizedHeaderCache(node.Logger, node.State, builder.FollowerDistributor) if err != nil { return nil, fmt.Errorf("could not create finalized snapshot cache: %w", err) } @@ -549,7 +549,7 @@ func (builder *ObserverServiceBuilder) BuildExecutionDataRequester() *ObserverSe builder.executionDataConfig, ) - builder.FinalizationDistributor.AddOnBlockFinalizedConsumer(builder.ExecutionDataRequester.OnBlockFinalized) + builder.FollowerDistributor.AddOnBlockFinalizedConsumer(builder.ExecutionDataRequester.OnBlockFinalized) return builder.ExecutionDataRequester, nil }) @@ -565,11 +565,11 @@ func NewFlowObserverServiceBuilder(opts ...Option) *ObserverServiceBuilder { opt(config) } anb := &ObserverServiceBuilder{ - ObserverServiceConfig: config, - FlowNodeBuilder: cmd.FlowNode(flow.RoleAccess.String()), - FinalizationDistributor: pubsub.NewFollowerDistributor(), + ObserverServiceConfig: config, + FlowNodeBuilder: cmd.FlowNode(flow.RoleAccess.String()), + FollowerDistributor: pubsub.NewFollowerDistributor(), } - anb.FinalizationDistributor.AddConsumer(notifications.NewSlashingViolationsConsumer(anb.Logger)) + anb.FollowerDistributor.AddConsumer(notifications.NewSlashingViolationsConsumer(anb.Logger)) // the observer gets a version of the root snapshot file that does not contain any node addresses // hence skip all the root snapshot validations that involved an identity address anb.FlowNodeBuilder.SkipNwAddressBasedValidations = true diff --git a/cmd/verification_builder.go b/cmd/verification_builder.go index 69e31f643b1..a00f834f4f5 100644 --- a/cmd/verification_builder.go +++ b/cmd/verification_builder.go @@ -94,15 +94,15 @@ func (v *VerificationNodeBuilder) LoadComponentsAndModules() { processedBlockHeight *badger.ConsumerProgress // used in block consumer chunkQueue *badger.ChunksQueue // used in chunk consumer - syncCore *chainsync.Core // used in follower engine - assignerEngine *assigner.Engine // the assigner engine - fetcherEngine *fetcher.Engine // the fetcher engine - requesterEngine *requester.Engine // the requester engine - verifierEng *verifier.Engine // the verifier engine - chunkConsumer *chunkconsumer.ChunkConsumer - blockConsumer *blockconsumer.BlockConsumer - finalizationDistributor *pubsub.FollowerDistributor - finalizedHeader *commonsync.FinalizedHeaderCache + syncCore *chainsync.Core // used in follower engine + assignerEngine *assigner.Engine // the assigner engine + fetcherEngine *fetcher.Engine // the fetcher engine + requesterEngine *requester.Engine // the requester engine + verifierEng *verifier.Engine // the verifier engine + chunkConsumer *chunkconsumer.ChunkConsumer + blockConsumer *blockconsumer.BlockConsumer + followerDistributor *pubsub.FollowerDistributor + finalizedHeader *commonsync.FinalizedHeaderCache committee *committees.Consensus followerCore *hotstuff.FollowerLoop // follower hotstuff logic @@ -177,9 +177,9 @@ func (v *VerificationNodeBuilder) LoadComponentsAndModules() { return nil }). - Module("finalization distributor", func(node *NodeConfig) error { - finalizationDistributor = pubsub.NewFollowerDistributor() - finalizationDistributor.AddConsumer(notifications.NewSlashingViolationsConsumer(node.Logger)) + Module("follower distributor", func(node *NodeConfig) error { + followerDistributor = pubsub.NewFollowerDistributor() + followerDistributor.AddConsumer(notifications.NewSlashingViolationsConsumer(node.Logger)) return nil }). Module("sync core", func(node *NodeConfig) error { @@ -315,7 +315,7 @@ func (v *VerificationNodeBuilder) LoadComponentsAndModules() { }). Component("finalized snapshot", func(node *NodeConfig) (module.ReadyDoneAware, error) { var err error - finalizedHeader, err = commonsync.NewFinalizedHeaderCache(node.Logger, node.State, finalizationDistributor) + finalizedHeader, err = commonsync.NewFinalizedHeaderCache(node.Logger, node.State, followerDistributor) if err != nil { return nil, fmt.Errorf("could not create finalized snapshot cache: %w", err) } @@ -346,7 +346,7 @@ func (v *VerificationNodeBuilder) LoadComponentsAndModules() { return nil, fmt.Errorf("could not find latest finalized block and pending blocks to recover consensus follower: %w", err) } - finalizationDistributor.AddOnBlockFinalizedConsumer(blockConsumer.OnFinalizedBlock) + followerDistributor.AddOnBlockFinalizedConsumer(blockConsumer.OnFinalizedBlock) // creates a consensus follower with ingestEngine as the notifier // so that it gets notified upon each new finalized block @@ -356,7 +356,7 @@ func (v *VerificationNodeBuilder) LoadComponentsAndModules() { node.Storage.Headers, final, verifier, - finalizationDistributor, + followerDistributor, node.RootBlock.Header, node.RootQC, finalized, @@ -383,7 +383,7 @@ func (v *VerificationNodeBuilder) LoadComponentsAndModules() { node.Logger, node.Metrics.Mempool, heroCacheCollector, - finalizationDistributor, + followerDistributor, followerState, followerCore, validator, diff --git a/engine/common/synchronization/finalized_snapshot.go b/engine/common/synchronization/finalized_snapshot.go index a866cd7b9d8..2bab94f8385 100644 --- a/engine/common/synchronization/finalized_snapshot.go +++ b/engine/common/synchronization/finalized_snapshot.go @@ -29,7 +29,7 @@ type FinalizedHeaderCache struct { } // NewFinalizedHeaderCache creates a new finalized header cache. -func NewFinalizedHeaderCache(log zerolog.Logger, state protocol.State, finalizationDistributor *pubsub.FollowerDistributor) (*FinalizedHeaderCache, error) { +func NewFinalizedHeaderCache(log zerolog.Logger, state protocol.State, followerDistributor *pubsub.FollowerDistributor) (*FinalizedHeaderCache, error) { cache := &FinalizedHeaderCache{ state: state, lm: lifecycle.NewLifecycleManager(), @@ -45,7 +45,7 @@ func NewFinalizedHeaderCache(log zerolog.Logger, state protocol.State, finalizat cache.lastFinalizedHeader = snapshot - finalizationDistributor.AddOnBlockFinalizedConsumer(cache.onFinalizedBlock) + followerDistributor.AddOnBlockFinalizedConsumer(cache.onFinalizedBlock) return cache, nil } diff --git a/follower/consensus_follower.go b/follower/consensus_follower.go index 97dd227480e..56863bcf530 100644 --- a/follower/consensus_follower.go +++ b/follower/consensus_follower.go @@ -203,7 +203,7 @@ func NewConsensusFollower( cf := &ConsensusFollowerImpl{logger: anb.Logger} anb.BaseConfig.NodeRole = "consensus_follower" - anb.FinalizationDistributor.AddOnBlockFinalizedConsumer(cf.onBlockFinalized) + anb.FollowerDistributor.AddOnBlockFinalizedConsumer(cf.onBlockFinalized) cf.NodeConfig = anb.NodeConfig cf.Component, err = anb.Build() diff --git a/follower/follower_builder.go b/follower/follower_builder.go index cda75731365..200beff2333 100644 --- a/follower/follower_builder.go +++ b/follower/follower_builder.go @@ -105,16 +105,16 @@ type FollowerServiceBuilder struct { *FollowerServiceConfig // components - LibP2PNode p2p.LibP2PNode - FollowerState protocol.FollowerState - SyncCore *synchronization.Core - FinalizationDistributor *pubsub.FollowerDistributor - FinalizedHeader *synceng.FinalizedHeaderCache - Committee hotstuff.DynamicCommittee - Finalized *flow.Header - Pending []*flow.Header - FollowerCore module.HotStuffFollower - Validator hotstuff.Validator + LibP2PNode p2p.LibP2PNode + FollowerState protocol.FollowerState + SyncCore *synchronization.Core + FollowerDistributor *pubsub.FollowerDistributor + FinalizedHeader *synceng.FinalizedHeaderCache + Committee hotstuff.DynamicCommittee + Finalized *flow.Header + Pending []*flow.Header + FollowerCore module.HotStuffFollower + Validator hotstuff.Validator // for the observer, the sync engine participants provider is the libp2p peer store which is not // available until after the network has started. Hence, a factory function that needs to be called just before // creating the sync engine @@ -221,7 +221,7 @@ func (builder *FollowerServiceBuilder) buildFollowerCore() *FollowerServiceBuild builder.Validator = hotstuffvalidator.New(builder.Committee, verifier) followerCore, err := consensus.NewFollower(node.Logger, builder.Committee, node.Storage.Headers, final, verifier, - builder.FinalizationDistributor, node.RootBlock.Header, node.RootQC, builder.Finalized, builder.Pending) + builder.FollowerDistributor, node.RootBlock.Header, node.RootQC, builder.Finalized, builder.Pending) if err != nil { return nil, fmt.Errorf("could not initialize follower core: %w", err) } @@ -244,7 +244,7 @@ func (builder *FollowerServiceBuilder) buildFollowerEngine() *FollowerServiceBui node.Logger, node.Metrics.Mempool, heroCacheCollector, - builder.FinalizationDistributor, + builder.FollowerDistributor, builder.FollowerState, builder.FollowerCore, builder.Validator, @@ -278,7 +278,7 @@ func (builder *FollowerServiceBuilder) buildFollowerEngine() *FollowerServiceBui func (builder *FollowerServiceBuilder) buildFinalizedHeader() *FollowerServiceBuilder { builder.Component("finalized snapshot", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { - finalizedHeader, err := synceng.NewFinalizedHeaderCache(node.Logger, node.State, builder.FinalizationDistributor) + finalizedHeader, err := synceng.NewFinalizedHeaderCache(node.Logger, node.State, builder.FollowerDistributor) if err != nil { return nil, fmt.Errorf("could not create finalized snapshot cache: %w", err) } @@ -356,10 +356,10 @@ func FlowConsensusFollowerService(opts ...FollowerOption) *FollowerServiceBuilde ret := &FollowerServiceBuilder{ FollowerServiceConfig: config, // TODO: using RoleAccess here for now. This should be refactored eventually to have its own role type - FlowNodeBuilder: cmd.FlowNode(flow.RoleAccess.String(), config.baseOptions...), - FinalizationDistributor: pubsub.NewFollowerDistributor(), + FlowNodeBuilder: cmd.FlowNode(flow.RoleAccess.String(), config.baseOptions...), + FollowerDistributor: pubsub.NewFollowerDistributor(), } - ret.FinalizationDistributor.AddConsumer(notifications.NewSlashingViolationsConsumer(ret.Logger)) + ret.FollowerDistributor.AddConsumer(notifications.NewSlashingViolationsConsumer(ret.Logger)) // the observer gets a version of the root snapshot file that does not contain any node addresses // hence skip all the root snapshot validations that involved an identity address ret.FlowNodeBuilder.SkipNwAddressBasedValidations = true diff --git a/module/state_synchronization/requester/execution_data_requester.go b/module/state_synchronization/requester/execution_data_requester.go index 9dec1718942..bbe95c4f84e 100644 --- a/module/state_synchronization/requester/execution_data_requester.go +++ b/module/state_synchronization/requester/execution_data_requester.go @@ -43,7 +43,7 @@ import ( // // The requester is made up of 3 subcomponents: // -// * OnBlockFinalized: receives block finalized events from the finalization distributor and +// * OnBlockFinalized: receives block finalized events from the follower distributor and // forwards them to the blockConsumer. // // * blockConsumer: is a jobqueue that receives block finalization events. On each event, diff --git a/module/state_synchronization/requester/execution_data_requester_test.go b/module/state_synchronization/requester/execution_data_requester_test.go index a490863a9cc..36c3859cb98 100644 --- a/module/state_synchronization/requester/execution_data_requester_test.go +++ b/module/state_synchronization/requester/execution_data_requester_test.go @@ -302,10 +302,10 @@ func (suite *ExecutionDataRequesterSuite) TestRequesterHalts() { testData := suite.generateTestData(suite.run.blockCount, generate(suite.run.blockCount)) // start processing with all seals available - edr, finalizationDistributor := suite.prepareRequesterTest(testData) + edr, followerDistributor := suite.prepareRequesterTest(testData) testData.resumeHeight = testData.endHeight testData.expectedIrrecoverable = expectedErr - fetchedExecutionData := suite.runRequesterTestHalts(edr, finalizationDistributor, testData) + fetchedExecutionData := suite.runRequesterTestHalts(edr, followerDistributor, testData) assert.Less(suite.T(), len(fetchedExecutionData), testData.sealedCount) suite.T().Log("Shutting down test") @@ -400,7 +400,7 @@ func (suite *ExecutionDataRequesterSuite) prepareRequesterTest(cfg *fetchTestRun suite.downloader = mockDownloader(cfg.executionDataEntries) - finalizationDistributor := pubsub.NewFollowerDistributor() + followerDistributor := pubsub.NewFollowerDistributor() processedHeight := bstorage.NewConsumerProgress(suite.db, module.ConsumeProgressExecutionDataRequesterBlockHeight) processedNotification := bstorage.NewConsumerProgress(suite.db, module.ConsumeProgressExecutionDataRequesterNotification) @@ -423,12 +423,12 @@ func (suite *ExecutionDataRequesterSuite) prepareRequesterTest(cfg *fetchTestRun }, ) - finalizationDistributor.AddOnBlockFinalizedConsumer(edr.OnBlockFinalized) + followerDistributor.AddOnBlockFinalizedConsumer(edr.OnBlockFinalized) - return edr, finalizationDistributor + return edr, followerDistributor } -func (suite *ExecutionDataRequesterSuite) runRequesterTestHalts(edr state_synchronization.ExecutionDataRequester, finalizationDistributor *pubsub.FollowerDistributor, cfg *fetchTestRun) receivedExecutionData { +func (suite *ExecutionDataRequesterSuite) runRequesterTestHalts(edr state_synchronization.ExecutionDataRequester, followerDistributor *pubsub.FollowerDistributor, cfg *fetchTestRun) receivedExecutionData { // make sure test helper goroutines are cleaned up ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() @@ -444,8 +444,8 @@ func (suite *ExecutionDataRequesterSuite) runRequesterTestHalts(edr state_synchr edr.Start(signalerCtx) unittest.RequireCloseBefore(suite.T(), edr.Ready(), cfg.waitTimeout, "timed out waiting for requester to be ready") - // Send blocks through finalizationDistributor - suite.finalizeBlocks(cfg, finalizationDistributor) + // Send blocks through followerDistributor + suite.finalizeBlocks(cfg, followerDistributor) // testDone should never close because the requester paused unittest.RequireNeverClosedWithin(suite.T(), testDone, 100*time.Millisecond, "finished sending notifications unexpectedly") @@ -457,7 +457,7 @@ func (suite *ExecutionDataRequesterSuite) runRequesterTestHalts(edr state_synchr return fetchedExecutionData } -func (suite *ExecutionDataRequesterSuite) runRequesterTestPauseResume(edr state_synchronization.ExecutionDataRequester, finalizationDistributor *pubsub.FollowerDistributor, cfg *fetchTestRun, expectedDownloads int, resume func()) receivedExecutionData { +func (suite *ExecutionDataRequesterSuite) runRequesterTestPauseResume(edr state_synchronization.ExecutionDataRequester, followerDistributor *pubsub.FollowerDistributor, cfg *fetchTestRun, expectedDownloads int, resume func()) receivedExecutionData { // make sure test helper goroutines are cleaned up ctx, cancel := context.WithCancel(context.Background()) signalerCtx := irrecoverable.NewMockSignalerContext(suite.T(), ctx) @@ -471,8 +471,8 @@ func (suite *ExecutionDataRequesterSuite) runRequesterTestPauseResume(edr state_ edr.Start(signalerCtx) unittest.RequireCloseBefore(suite.T(), edr.Ready(), cfg.waitTimeout, "timed out waiting for requester to be ready") - // Send all blocks through finalizationDistributor - suite.finalizeBlocks(cfg, finalizationDistributor) + // Send all blocks through followerDistributor + suite.finalizeBlocks(cfg, followerDistributor) // requester should pause downloads until resume is called, so testDone should not be closed unittest.RequireNeverClosedWithin(suite.T(), testDone, 500*time.Millisecond, "finished unexpectedly") @@ -493,7 +493,7 @@ func (suite *ExecutionDataRequesterSuite) runRequesterTestPauseResume(edr state_ return fetchedExecutionData } -func (suite *ExecutionDataRequesterSuite) runRequesterTest(edr state_synchronization.ExecutionDataRequester, finalizationDistributor *pubsub.FollowerDistributor, cfg *fetchTestRun) receivedExecutionData { +func (suite *ExecutionDataRequesterSuite) runRequesterTest(edr state_synchronization.ExecutionDataRequester, followerDistributor *pubsub.FollowerDistributor, cfg *fetchTestRun) receivedExecutionData { // make sure test helper goroutines are cleaned up ctx, cancel := context.WithCancel(context.Background()) signalerCtx := irrecoverable.NewMockSignalerContext(suite.T(), ctx) @@ -509,8 +509,8 @@ func (suite *ExecutionDataRequesterSuite) runRequesterTest(edr state_synchroniza edr.Start(signalerCtx) unittest.RequireCloseBefore(suite.T(), edr.Ready(), cfg.waitTimeout, "timed out waiting for requester to be ready") - // Send blocks through finalizationDistributor - suite.finalizeBlocks(cfg, finalizationDistributor) + // Send blocks through followerDistributor + suite.finalizeBlocks(cfg, followerDistributor) // Pause until we've received all of the expected notifications unittest.RequireCloseBefore(suite.T(), testDone, cfg.waitTimeout, "timed out waiting for notifications") @@ -538,7 +538,7 @@ func (suite *ExecutionDataRequesterSuite) consumeExecutionDataNotifications(cfg } } -func (suite *ExecutionDataRequesterSuite) finalizeBlocks(cfg *fetchTestRun, finalizationDistributor *pubsub.FollowerDistributor) { +func (suite *ExecutionDataRequesterSuite) finalizeBlocks(cfg *fetchTestRun, followerDistributor *pubsub.FollowerDistributor) { for i := cfg.StartHeight(); i <= cfg.endHeight; i++ { b := cfg.blocksByHeight[i] @@ -552,7 +552,7 @@ func (suite *ExecutionDataRequesterSuite) finalizeBlocks(cfg *fetchTestRun, fina suite.T().Log(">>>> Sealing block", sealedHeader.ID(), sealedHeader.Height) } - finalizationDistributor.OnFinalizedBlock(&model.Block{}) // actual block is unused + followerDistributor.OnFinalizedBlock(&model.Block{}) // actual block is unused if cfg.stopHeight == i { break From 51c7fed878c1a4142c586cf4dc83f7dada767224 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Thu, 13 Apr 2023 14:59:18 +0300 Subject: [PATCH 09/38] Renamed BaseProtocolViolationConsumer -> ProtocolViolationConsumer --- consensus/hotstuff/consumer.go | 6 +-- .../mocks/base_protocol_violation_consumer.go | 38 ------------------- .../mocks/protocol_violation_consumer.go | 38 +++++++++++++++++++ .../hotstuff/notifications/noop_consumer.go | 4 +- engine/collection/compliance/core.go | 4 +- .../epochmgr/factories/compliance.go | 2 +- engine/common/follower/compliance_core.go | 2 +- engine/consensus/compliance/core.go | 4 +- 8 files changed, 49 insertions(+), 49 deletions(-) delete mode 100644 consensus/hotstuff/mocks/base_protocol_violation_consumer.go create mode 100644 consensus/hotstuff/mocks/protocol_violation_consumer.go diff --git a/consensus/hotstuff/consumer.go b/consensus/hotstuff/consumer.go index 27c3ea20428..0dbb99d0908 100644 --- a/consensus/hotstuff/consumer.go +++ b/consensus/hotstuff/consumer.go @@ -7,7 +7,7 @@ import ( "github.com/onflow/flow-go/model/flow" ) -// BaseProtocolViolationConsumer consumes outbound notifications produced by compliance. +// ProtocolViolationConsumer consumes outbound notifications produced by compliance. // Notifications can be produced by consensus participants and followers. // Notifications are meant to report protocol violations that can be observed by executing compliance checks. // @@ -15,7 +15,7 @@ import ( // - be concurrency safe // - be non-blocking // - handle repetition of the same events (with some processing overhead). -type BaseProtocolViolationConsumer interface { +type ProtocolViolationConsumer interface { // OnInvalidBlockDetected notifications are produced by components that have detected // that a block proposal is invalid and need to report it. // Most of the time such block can be detected by calling Validator.ValidateProposal. @@ -62,7 +62,7 @@ type FinalizationConsumer interface { // - be non-blocking // - handle repetition of the same events (with some processing overhead). type ConsensusFollowerConsumer interface { - BaseProtocolViolationConsumer + ProtocolViolationConsumer FinalizationConsumer } diff --git a/consensus/hotstuff/mocks/base_protocol_violation_consumer.go b/consensus/hotstuff/mocks/base_protocol_violation_consumer.go deleted file mode 100644 index 4971e14f6e8..00000000000 --- a/consensus/hotstuff/mocks/base_protocol_violation_consumer.go +++ /dev/null @@ -1,38 +0,0 @@ -// Code generated by mockery v2.21.4. DO NOT EDIT. - -package mocks - -import ( - model "github.com/onflow/flow-go/consensus/hotstuff/model" - mock "github.com/stretchr/testify/mock" -) - -// BaseProtocolViolationConsumer is an autogenerated mock type for the BaseProtocolViolationConsumer type -type BaseProtocolViolationConsumer struct { - mock.Mock -} - -// OnDoubleProposeDetected provides a mock function with given fields: _a0, _a1 -func (_m *BaseProtocolViolationConsumer) OnDoubleProposeDetected(_a0 *model.Block, _a1 *model.Block) { - _m.Called(_a0, _a1) -} - -// OnInvalidBlockDetected provides a mock function with given fields: err -func (_m *BaseProtocolViolationConsumer) OnInvalidBlockDetected(err model.InvalidBlockError) { - _m.Called(err) -} - -type mockConstructorTestingTNewBaseProtocolViolationConsumer interface { - mock.TestingT - Cleanup(func()) -} - -// NewBaseProtocolViolationConsumer creates a new instance of BaseProtocolViolationConsumer. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewBaseProtocolViolationConsumer(t mockConstructorTestingTNewBaseProtocolViolationConsumer) *BaseProtocolViolationConsumer { - mock := &BaseProtocolViolationConsumer{} - mock.Mock.Test(t) - - t.Cleanup(func() { mock.AssertExpectations(t) }) - - return mock -} diff --git a/consensus/hotstuff/mocks/protocol_violation_consumer.go b/consensus/hotstuff/mocks/protocol_violation_consumer.go new file mode 100644 index 00000000000..923c4c3cae7 --- /dev/null +++ b/consensus/hotstuff/mocks/protocol_violation_consumer.go @@ -0,0 +1,38 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mocks + +import ( + model "github.com/onflow/flow-go/consensus/hotstuff/model" + mock "github.com/stretchr/testify/mock" +) + +// ProtocolViolationConsumer is an autogenerated mock type for the ProtocolViolationConsumer type +type ProtocolViolationConsumer struct { + mock.Mock +} + +// OnDoubleProposeDetected provides a mock function with given fields: _a0, _a1 +func (_m *ProtocolViolationConsumer) OnDoubleProposeDetected(_a0 *model.Block, _a1 *model.Block) { + _m.Called(_a0, _a1) +} + +// OnInvalidBlockDetected provides a mock function with given fields: err +func (_m *ProtocolViolationConsumer) OnInvalidBlockDetected(err model.InvalidBlockError) { + _m.Called(err) +} + +type mockConstructorTestingTNewProtocolViolationConsumer interface { + mock.TestingT + Cleanup(func()) +} + +// NewProtocolViolationConsumer creates a new instance of ProtocolViolationConsumer. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewProtocolViolationConsumer(t mockConstructorTestingTNewProtocolViolationConsumer) *ProtocolViolationConsumer { + mock := &ProtocolViolationConsumer{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/consensus/hotstuff/notifications/noop_consumer.go b/consensus/hotstuff/notifications/noop_consumer.go index bd104151d56..f779844cc2b 100644 --- a/consensus/hotstuff/notifications/noop_consumer.go +++ b/consensus/hotstuff/notifications/noop_consumer.go @@ -111,11 +111,11 @@ var _ hotstuff.QCCreatedConsumer = (*NoopQCCreatedConsumer)(nil) func (*NoopQCCreatedConsumer) OnQcConstructedFromVotes(*flow.QuorumCertificate) {} -// no-op implementation of hotstuff.BaseProtocolViolationConsumer +// no-op implementation of hotstuff.ProtocolViolationConsumer type NoopBaseProtocolViolationConsumer struct{} -var _ hotstuff.BaseProtocolViolationConsumer = (*NoopBaseProtocolViolationConsumer)(nil) +var _ hotstuff.ProtocolViolationConsumer = (*NoopBaseProtocolViolationConsumer)(nil) func (*NoopBaseProtocolViolationConsumer) OnInvalidBlockDetected(model.InvalidBlockError) {} diff --git a/engine/collection/compliance/core.go b/engine/collection/compliance/core.go index 9139717951b..7608d96bf7f 100644 --- a/engine/collection/compliance/core.go +++ b/engine/collection/compliance/core.go @@ -40,7 +40,7 @@ type Core struct { mempoolMetrics module.MempoolMetrics hotstuffMetrics module.HotstuffMetrics collectionMetrics module.CollectionMetrics - protocolViolationNotifier hotstuff.BaseProtocolViolationConsumer + protocolViolationNotifier hotstuff.ProtocolViolationConsumer headers storage.Headers state clusterkv.MutableState // track latest finalized view/height - used to efficiently drop outdated or too-far-ahead blocks @@ -61,7 +61,7 @@ func NewCore( mempool module.MempoolMetrics, hotstuffMetrics module.HotstuffMetrics, collectionMetrics module.CollectionMetrics, - protocolViolationNotifier hotstuff.BaseProtocolViolationConsumer, + protocolViolationNotifier hotstuff.ProtocolViolationConsumer, headers storage.Headers, state clusterkv.MutableState, pending module.PendingClusterBlockBuffer, diff --git a/engine/collection/epochmgr/factories/compliance.go b/engine/collection/epochmgr/factories/compliance.go index 1beed415634..bfa412cc021 100644 --- a/engine/collection/epochmgr/factories/compliance.go +++ b/engine/collection/epochmgr/factories/compliance.go @@ -58,7 +58,7 @@ func NewComplianceEngineFactory( func (f *ComplianceEngineFactory) Create( hotstuffMetrics module.HotstuffMetrics, - notifier hotstuff.BaseProtocolViolationConsumer, + notifier hotstuff.ProtocolViolationConsumer, clusterState cluster.MutableState, headers storage.Headers, payloads storage.ClusterPayloads, diff --git a/engine/common/follower/compliance_core.go b/engine/common/follower/compliance_core.go index c92d93f4fe3..9abfaf5a7c0 100644 --- a/engine/common/follower/compliance_core.go +++ b/engine/common/follower/compliance_core.go @@ -41,7 +41,7 @@ type ComplianceCore struct { log zerolog.Logger mempoolMetrics module.MempoolMetrics tracer module.Tracer - protocolViolationNotifier hotstuff.BaseProtocolViolationConsumer + protocolViolationNotifier hotstuff.ProtocolViolationConsumer pendingCache *cache.Cache pendingTree *pending_tree.PendingTree state protocol.FollowerState diff --git a/engine/consensus/compliance/core.go b/engine/consensus/compliance/core.go index c739d660a2e..b4004979abe 100644 --- a/engine/consensus/compliance/core.go +++ b/engine/consensus/compliance/core.go @@ -42,7 +42,7 @@ type Core struct { mempoolMetrics module.MempoolMetrics hotstuffMetrics module.HotstuffMetrics complianceMetrics module.ComplianceMetrics - protocolViolationNotifier hotstuff.BaseProtocolViolationConsumer + protocolViolationNotifier hotstuff.ProtocolViolationConsumer tracer module.Tracer headers storage.Headers payloads storage.Payloads @@ -65,7 +65,7 @@ func NewCore( mempool module.MempoolMetrics, hotstuffMetrics module.HotstuffMetrics, complianceMetrics module.ComplianceMetrics, - protocolViolationNotifier hotstuff.BaseProtocolViolationConsumer, + protocolViolationNotifier hotstuff.ProtocolViolationConsumer, tracer module.Tracer, headers storage.Headers, payloads storage.Payloads, From e9201e44c3a131f8af07b25e1da1e9b158041db6 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Thu, 13 Apr 2023 14:59:42 +0300 Subject: [PATCH 10/38] Updated last commit --- consensus/hotstuff/integration/instance_test.go | 2 +- consensus/hotstuff/notifications/noop_consumer.go | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/consensus/hotstuff/integration/instance_test.go b/consensus/hotstuff/integration/instance_test.go index fc47ecf78e0..e981f335329 100644 --- a/consensus/hotstuff/integration/instance_test.go +++ b/consensus/hotstuff/integration/instance_test.go @@ -84,7 +84,7 @@ type Instance struct { } type MockedCommunicatorConsumer struct { - notifications.NoopBaseProtocolViolationConsumer + notifications.NoopProtocolViolationConsumer notifications.NoopPartialConsumer notifications.NoopFinalizationConsumer *mocks.CommunicatorConsumer diff --git a/consensus/hotstuff/notifications/noop_consumer.go b/consensus/hotstuff/notifications/noop_consumer.go index f779844cc2b..f3babd8f81a 100644 --- a/consensus/hotstuff/notifications/noop_consumer.go +++ b/consensus/hotstuff/notifications/noop_consumer.go @@ -11,7 +11,7 @@ import ( // NoopConsumer is an implementation of the notifications consumer that // doesn't do anything. type NoopConsumer struct { - NoopBaseProtocolViolationConsumer + NoopProtocolViolationConsumer NoopFinalizationConsumer NoopPartialConsumer NoopCommunicatorConsumer @@ -113,10 +113,10 @@ func (*NoopQCCreatedConsumer) OnQcConstructedFromVotes(*flow.QuorumCertificate) // no-op implementation of hotstuff.ProtocolViolationConsumer -type NoopBaseProtocolViolationConsumer struct{} +type NoopProtocolViolationConsumer struct{} -var _ hotstuff.ProtocolViolationConsumer = (*NoopBaseProtocolViolationConsumer)(nil) +var _ hotstuff.ProtocolViolationConsumer = (*NoopProtocolViolationConsumer)(nil) -func (*NoopBaseProtocolViolationConsumer) OnInvalidBlockDetected(model.InvalidBlockError) {} +func (*NoopProtocolViolationConsumer) OnInvalidBlockDetected(model.InvalidBlockError) {} -func (*NoopBaseProtocolViolationConsumer) OnDoubleProposeDetected(*model.Block, *model.Block) {} +func (*NoopProtocolViolationConsumer) OnDoubleProposeDetected(*model.Block, *model.Block) {} From 822df6046254535b9c6513a73a46895fb3069f54 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Thu, 13 Apr 2023 15:08:23 +0300 Subject: [PATCH 11/38] Removed extra validation in follower. Fixed tests --- consensus/hotstuff/follower/follower.go | 35 +------------------ .../hotstuff/notifications/log_consumer.go | 2 ++ engine/collection/compliance/core_test.go | 4 +-- engine/consensus/compliance/core_test.go | 4 +-- 4 files changed, 7 insertions(+), 38 deletions(-) diff --git a/consensus/hotstuff/follower/follower.go b/consensus/hotstuff/follower/follower.go index 833f805a13b..79f40635229 100644 --- a/consensus/hotstuff/follower/follower.go +++ b/consensus/hotstuff/follower/follower.go @@ -1,14 +1,12 @@ package follower import ( - "errors" "fmt" "github.com/rs/zerolog" "github.com/onflow/flow-go/consensus/hotstuff" "github.com/onflow/flow-go/consensus/hotstuff/model" - "github.com/onflow/flow-go/utils/logging" ) // FollowerLogic runs in non-consensus nodes. It informs other components within the node @@ -43,38 +41,7 @@ func (f *FollowerLogic) FinalizedBlock() *model.Block { // AddBlock processes the given block proposal func (f *FollowerLogic) AddBlock(blockProposal *model.Proposal) error { - // validate the block. skip if the proposal is invalid - // TODO: this block was already validated by follower engine, to be refactored - err := f.validator.ValidateProposal(blockProposal) - if err != nil { - if model.IsInvalidBlockError(err) { - f.log.Warn().Err(err). - Hex("block_id", logging.ID(blockProposal.Block.BlockID)). - Msg("invalid proposal") - return nil - } else if errors.Is(err, model.ErrViewForUnknownEpoch) { - f.log.Warn().Err(err). - Hex("block_id", logging.ID(blockProposal.Block.BlockID)). - Hex("qc_block_id", logging.ID(blockProposal.Block.QC.BlockID)). - Uint64("block_view", blockProposal.Block.View). - Msg("proposal for unknown epoch") - return nil - } else if errors.Is(err, model.ErrUnverifiableBlock) { - f.log.Warn().Err(err). - Hex("block_id", logging.ID(blockProposal.Block.BlockID)). - Hex("qc_block_id", logging.ID(blockProposal.Block.QC.BlockID)). - Uint64("block_view", blockProposal.Block.View). - Msg("unverifiable proposal") - // even if the block is unverifiable because the QC has been - // pruned, it still needs to be added to the forks, otherwise, - // a new block with a QC to this block will fail to be added - // to forks and crash the event loop. - } else if err != nil { - return fmt.Errorf("cannot validate block proposal %x: %w", blockProposal.Block.BlockID, err) - } - } - - err = f.finalizationLogic.AddProposal(blockProposal) + err := f.finalizationLogic.AddProposal(blockProposal) if err != nil { return fmt.Errorf("finalization logic cannot process block proposal %x: %w", blockProposal.Block.BlockID, err) } diff --git a/consensus/hotstuff/notifications/log_consumer.go b/consensus/hotstuff/notifications/log_consumer.go index 64e76e3d34e..65ed347bed3 100644 --- a/consensus/hotstuff/notifications/log_consumer.go +++ b/consensus/hotstuff/notifications/log_consumer.go @@ -48,6 +48,7 @@ func (lc *LogConsumer) OnFinalizedBlock(block *model.Block) { func (lc *LogConsumer) OnInvalidBlockDetected(err model.InvalidBlockError) { invalidBlock := err.InvalidBlock.Block lc.log.Warn(). + Str(logging.KeySuspicious, "true"). Uint64("block_view", invalidBlock.View). Hex("proposer_id", invalidBlock.ProposerID[:]). Hex("block_id", invalidBlock.BlockID[:]). @@ -58,6 +59,7 @@ func (lc *LogConsumer) OnInvalidBlockDetected(err model.InvalidBlockError) { func (lc *LogConsumer) OnDoubleProposeDetected(block *model.Block, alt *model.Block) { lc.log.Warn(). + Str(logging.KeySuspicious, "true"). Uint64("block_view", block.View). Hex("block_id", block.BlockID[:]). Hex("alt_id", alt.BlockID[:]). diff --git a/engine/collection/compliance/core_test.go b/engine/collection/compliance/core_test.go index ecf914af8f6..49d9e2474d1 100644 --- a/engine/collection/compliance/core_test.go +++ b/engine/collection/compliance/core_test.go @@ -52,7 +52,7 @@ type CommonSuite struct { state *clusterstate.MutableState snapshot *clusterstate.Snapshot metrics *metrics.NoopCollector - protocolViolationNotifier *hotstuff.BaseProtocolViolationConsumer + protocolViolationNotifier *hotstuff.ProtocolViolationConsumer headers *storage.Headers pending *module.PendingClusterBlockBuffer hotstuff *module.HotStuff @@ -168,7 +168,7 @@ func (cs *CommonSuite) SetupTest() { cs.metrics = metrics.NewNoopCollector() // set up notifier for reporting protocol violations - cs.protocolViolationNotifier = hotstuff.NewBaseProtocolViolationConsumer(cs.T()) + cs.protocolViolationNotifier = hotstuff.NewProtocolViolationConsumer(cs.T()) // initialize the engine core, err := NewCore( diff --git a/engine/consensus/compliance/core_test.go b/engine/consensus/compliance/core_test.go index 829d7af77c2..5f2a7ae6c33 100644 --- a/engine/consensus/compliance/core_test.go +++ b/engine/consensus/compliance/core_test.go @@ -71,7 +71,7 @@ type CommonSuite struct { pending *module.PendingBlockBuffer hotstuff *module.HotStuff sync *module.BlockRequester - protocolViolationNotifier *hotstuff.BaseProtocolViolationConsumer + protocolViolationNotifier *hotstuff.ProtocolViolationConsumer validator *hotstuff.Validator voteAggregator *hotstuff.VoteAggregator timeoutAggregator *hotstuff.TimeoutAggregator @@ -246,7 +246,7 @@ func (cs *CommonSuite) SetupTest() { cs.tracer = trace.NewNoopTracer() // set up notifier for reporting protocol violations - cs.protocolViolationNotifier = hotstuff.NewBaseProtocolViolationConsumer(cs.T()) + cs.protocolViolationNotifier = hotstuff.NewProtocolViolationConsumer(cs.T()) // initialize the engine e, err := NewCore( From 8c773bb21ac3c6dff720cd71e58d8b5e04005101 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Thu, 13 Apr 2023 15:09:42 +0300 Subject: [PATCH 12/38] Apply suggestions from code review Co-authored-by: Jordan Schalm --- consensus/hotstuff/consumer.go | 6 ++++-- consensus/hotstuff/notifications/pubsub/distributor.go | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/consensus/hotstuff/consumer.go b/consensus/hotstuff/consumer.go index 0dbb99d0908..2c144fe103e 100644 --- a/consensus/hotstuff/consumer.go +++ b/consensus/hotstuff/consumer.go @@ -25,6 +25,7 @@ type ProtocolViolationConsumer interface { OnInvalidBlockDetected(err model.InvalidBlockError) // OnDoubleProposeDetected notifications are produced by the Finalization Logic // whenever a double block proposal (equivocation) was detected. + // Equivocation occurs when the same leader proposes two different blocks for the same view. // Prerequisites: // Implementation must be concurrency safe; Non-blocking; // and must handle repetition of the same events (with some processing overhead). @@ -56,7 +57,8 @@ type FinalizationConsumer interface { OnFinalizedBlock(*model.Block) } -// ConsensusFollowerConsumer consumes outbound notifications produced by consensus followers(not participants). +// ConsensusFollowerConsumer consumes outbound notifications produced by consensus followers. +// It is a subset of the notifications produced by consensus participants. // Implementations must: // - be concurrency safe // - be non-blocking @@ -66,7 +68,7 @@ type ConsensusFollowerConsumer interface { FinalizationConsumer } -// Consumer consumes outbound notifications produced by HotStuff and its components. +// Consumer consumes outbound notifications produced by consensus participants. // Notifications are consensus-internal state changes which are potentially relevant to // the larger node in which HotStuff is running. The notifications are emitted // in the order in which the HotStuff algorithm makes the respective steps. diff --git a/consensus/hotstuff/notifications/pubsub/distributor.go b/consensus/hotstuff/notifications/pubsub/distributor.go index 31cf1147e30..74674ee8547 100644 --- a/consensus/hotstuff/notifications/pubsub/distributor.go +++ b/consensus/hotstuff/notifications/pubsub/distributor.go @@ -39,7 +39,7 @@ func (p *Distributor) AddConsumer(consumer hotstuff.Consumer) { p.subscribers = append(p.subscribers, consumer) } -// AddFollowerConsumer wraps +// AddFollowerConsumer registers the input `consumer` to be notified on ConsensusFollowerConsumer events. func (p *Distributor) AddFollowerConsumer(consumer hotstuff.ConsensusFollowerConsumer) { p.lock.Lock() defer p.lock.Unlock() From d73e7e79730117fcdbbab8f82262da3d541910cf Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Thu, 13 Apr 2023 15:23:46 +0300 Subject: [PATCH 13/38] resolved conflicts after merge --- consensus/hotstuff/forks/forks.go | 2 +- consensus/hotstuff/forks/forks2.go | 6 +++--- consensus/hotstuff/model/errors.go | 9 ++------- 3 files changed, 6 insertions(+), 11 deletions(-) diff --git a/consensus/hotstuff/forks/forks.go b/consensus/hotstuff/forks/forks.go index bb2518e93ef..31e46f8495e 100644 --- a/consensus/hotstuff/forks/forks.go +++ b/consensus/hotstuff/forks/forks.go @@ -46,7 +46,7 @@ type Forks struct { var _ hotstuff.Forks = (*Forks)(nil) -func New(trustedRoot *model.CertifiedBlock, finalizationCallback module.Finalizer, notifier hotstuff.FinalizationConsumer) (*Forks, error) { +func New(trustedRoot *model.CertifiedBlock, finalizationCallback module.Finalizer, notifier hotstuff.ConsensusFollowerConsumer) (*Forks, error) { if (trustedRoot.Block.BlockID != trustedRoot.CertifyingQC.BlockID) || (trustedRoot.Block.View != trustedRoot.CertifyingQC.View) { return nil, model.NewConfigurationErrorf("invalid root: root QC is not pointing to root block") } diff --git a/consensus/hotstuff/forks/forks2.go b/consensus/hotstuff/forks/forks2.go index 7cf71ae297a..d76ebfbc06b 100644 --- a/consensus/hotstuff/forks/forks2.go +++ b/consensus/hotstuff/forks/forks2.go @@ -26,7 +26,7 @@ type FinalityProof struct { // Forks is NOT safe for concurrent use by multiple goroutines. type Forks2 struct { finalizationCallback module.Finalizer - notifier hotstuff.FinalizationConsumer + notifier hotstuff.ConsensusFollowerConsumer forest forest.LevelledForest trustedRoot *model.CertifiedBlock @@ -41,7 +41,7 @@ type Forks2 struct { // As the result, the following should apply again // var _ hotstuff.Forks = (*Forks2)(nil) -func NewForks2(trustedRoot *model.CertifiedBlock, finalizationCallback module.Finalizer, notifier hotstuff.FinalizationConsumer) (*Forks2, error) { +func NewForks2(trustedRoot *model.CertifiedBlock, finalizationCallback module.Finalizer, notifier hotstuff.ConsensusFollowerConsumer) (*Forks2, error) { if (trustedRoot.Block.BlockID != trustedRoot.CertifyingQC.BlockID) || (trustedRoot.Block.View != trustedRoot.CertifyingQC.View) { return nil, model.NewConfigurationErrorf("invalid root: root QC is not pointing to root block") } @@ -165,7 +165,7 @@ func (f *Forks2) EnsureBlockIsValidExtension(block *model.Block) error { err := f.forest.VerifyVertex(blockContainer) if err != nil { if forest.IsInvalidVertexError(err) { - return model.NewInvalidBlockError(block.BlockID, block.View, fmt.Errorf("not a valid vertex for block tree: %w", err)) + return fmt.Errorf("not a valid vertex for block tree: %w", err) } return fmt.Errorf("block tree generated unexpected error validating vertex: %w", err) } diff --git a/consensus/hotstuff/model/errors.go b/consensus/hotstuff/model/errors.go index 4a0fb480663..a8b5e1b2366 100644 --- a/consensus/hotstuff/model/errors.go +++ b/consensus/hotstuff/model/errors.go @@ -169,18 +169,13 @@ type InvalidBlockError struct { Err error } -func NewInvalidBlockErrorf(proposal *Proposal, msg string, args ...interface{}) error { +func NewInvalidBlockErrorf(block *Proposal, msg string, args ...interface{}) error { return InvalidBlockError{ - InvalidBlock: proposal, + InvalidBlock: block, Err: fmt.Errorf(msg, args...), } } -// NewInvalidBlockError instantiates an `InvalidBlockError`. Input `err` cannot be nil. -func NewInvalidBlockError(blockID flow.Identifier, view uint64, err error) error { - return InvalidBlockError{BlockID: blockID, View: view, Err: err} -} - func (e InvalidBlockError) Error() string { return fmt.Sprintf( "invalid block %x at view %d: %s", From 82d7d35c2111acf15695a23898591dc131467271 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Mon, 1 May 2023 23:32:23 +0300 Subject: [PATCH 14/38] Fixed tests --- engine/access/access_test.go | 48 +++++++++++++------------- engine/access/ingestion/engine_test.go | 4 +-- engine/access/rest_api_test.go | 4 +-- engine/access/rpc/rate_limit_test.go | 4 +-- engine/access/secure_grpcr_test.go | 4 +-- 5 files changed, 32 insertions(+), 32 deletions(-) diff --git a/engine/access/access_test.go b/engine/access/access_test.go index c60a1af1e5e..d6aecc74fd9 100644 --- a/engine/access/access_test.go +++ b/engine/access/access_test.go @@ -52,27 +52,27 @@ import ( type Suite struct { suite.Suite - state *protocol.State - sealedSnapshot *protocol.Snapshot - finalSnapshot *protocol.Snapshot - epochQuery *protocol.EpochQuery - params *protocol.Params - signerIndicesDecoder *hsmock.BlockSignerDecoder - signerIds flow.IdentifierList - log zerolog.Logger - net *mocknetwork.Network - request *module.Requester - collClient *accessmock.AccessAPIClient - execClient *accessmock.ExecutionAPIClient - me *module.Local - rootBlock *flow.Header - sealedBlock *flow.Header - finalizedBlock *flow.Header - chainID flow.ChainID - metrics *metrics.NoopCollector - backend *backend.Backend - finalizationDistributor *pubsub.FinalizationDistributor - finalizedHeaderCache *synceng.FinalizedHeaderCache + state *protocol.State + sealedSnapshot *protocol.Snapshot + finalSnapshot *protocol.Snapshot + epochQuery *protocol.EpochQuery + params *protocol.Params + signerIndicesDecoder *hsmock.BlockSignerDecoder + signerIds flow.IdentifierList + log zerolog.Logger + net *mocknetwork.Network + request *module.Requester + collClient *accessmock.AccessAPIClient + execClient *accessmock.ExecutionAPIClient + me *module.Local + rootBlock *flow.Header + sealedBlock *flow.Header + finalizedBlock *flow.Header + chainID flow.ChainID + metrics *metrics.NoopCollector + backend *backend.Backend + followerDistributor *pubsub.FollowerDistributor + finalizedHeaderCache *synceng.FinalizedHeaderCache } // TestAccess tests scenarios which exercise multiple API calls using both the RPC handler and the ingest engine @@ -133,10 +133,10 @@ func (suite *Suite) SetupTest() { suite.chainID = flow.Testnet suite.metrics = metrics.NewNoopCollector() - suite.finalizationDistributor = pubsub.NewFinalizationDistributor() + suite.followerDistributor = pubsub.NewFollowerDistributor() var err error - suite.finalizedHeaderCache, err = synceng.NewFinalizedHeaderCache(suite.log, suite.state, suite.finalizationDistributor) + suite.finalizedHeaderCache, err = synceng.NewFinalizedHeaderCache(suite.log, suite.state, suite.followerDistributor) require.NoError(suite.T(), err) unittest.RequireCloseBefore(suite.T(), suite.finalizedHeaderCache.Ready(), time.Second, "expect to start before timeout") @@ -1256,7 +1256,7 @@ func (suite *Suite) TestLastFinalizedBlockHeightResult() { suite.finalizedBlock = newFinalizedBlock.Header // report new finalized block to finalized blocks cache - suite.finalizationDistributor.OnFinalizedBlock(model.BlockFromFlow(suite.finalizedBlock)) + suite.followerDistributor.OnFinalizedBlock(model.BlockFromFlow(suite.finalizedBlock)) time.Sleep(time.Millisecond * 100) // give enough time to process async event resp, err = handler.GetBlockHeaderByID(context.Background(), req) diff --git a/engine/access/ingestion/engine_test.go b/engine/access/ingestion/engine_test.go index db32e51b0ad..97f67a5f850 100644 --- a/engine/access/ingestion/engine_test.go +++ b/engine/access/ingestion/engine_test.go @@ -116,9 +116,9 @@ func (suite *Suite) SetupTest() { blocksToMarkExecuted, err := stdmap.NewTimes(100) require.NoError(suite.T(), err) - finalizationDistributor := pubsub.NewFinalizationDistributor() + followerDistributor := pubsub.NewFollowerDistributor() - finalizedHeaderCache, err := synceng.NewFinalizedHeaderCache(log, suite.proto.state, finalizationDistributor) + finalizedHeaderCache, err := synceng.NewFinalizedHeaderCache(log, suite.proto.state, followerDistributor) require.NoError(suite.T(), err) rpcEngBuilder, err := rpc.NewBuilder(log, suite.proto.state, rpc.Config{}, nil, nil, suite.blocks, suite.headers, suite.collections, diff --git a/engine/access/rest_api_test.go b/engine/access/rest_api_test.go index 34e0fa584f8..aae9cdba8f2 100644 --- a/engine/access/rest_api_test.go +++ b/engine/access/rest_api_test.go @@ -118,10 +118,10 @@ func (suite *RestAPITestSuite) SetupTest() { RESTListenAddr: unittest.DefaultAddress, } - finalizationDistributor := pubsub.NewFinalizationDistributor() + followerDistributor := pubsub.NewFollowerDistributor() var err error - finalizedHeaderCache, err := synceng.NewFinalizedHeaderCache(suite.log, suite.state, finalizationDistributor) + finalizedHeaderCache, err := synceng.NewFinalizedHeaderCache(suite.log, suite.state, followerDistributor) require.NoError(suite.T(), err) rpcEngBuilder, err := rpc.NewBuilder(suite.log, suite.state, config, suite.collClient, nil, suite.blocks, suite.headers, suite.collections, suite.transactions, diff --git a/engine/access/rpc/rate_limit_test.go b/engine/access/rpc/rate_limit_test.go index 0c7c1500b6f..36d45fb9721 100644 --- a/engine/access/rpc/rate_limit_test.go +++ b/engine/access/rpc/rate_limit_test.go @@ -117,10 +117,10 @@ func (suite *RateLimitTestSuite) SetupTest() { block := unittest.BlockHeaderFixture() suite.snapshot.On("Head").Return(block, nil) - finalizationDistributor := pubsub.NewFinalizationDistributor() + followerDistributor := pubsub.NewFollowerDistributor() var err error - finalizedHeaderCache, err := synceng.NewFinalizedHeaderCache(suite.log, suite.state, finalizationDistributor) + finalizedHeaderCache, err := synceng.NewFinalizedHeaderCache(suite.log, suite.state, followerDistributor) require.NoError(suite.T(), err) rpcEngBuilder, err := NewBuilder(suite.log, suite.state, config, suite.collClient, nil, suite.blocks, suite.headers, suite.collections, suite.transactions, nil, diff --git a/engine/access/secure_grpcr_test.go b/engine/access/secure_grpcr_test.go index 056702d527c..68d5b946438 100644 --- a/engine/access/secure_grpcr_test.go +++ b/engine/access/secure_grpcr_test.go @@ -109,9 +109,9 @@ func (suite *SecureGRPCTestSuite) SetupTest() { block := unittest.BlockHeaderFixture() suite.snapshot.On("Head").Return(block, nil) - finalizationDistributor := pubsub.NewFinalizationDistributor() + followerDistributor := pubsub.NewFollowerDistributor() - finalizedHeaderCache, err := synceng.NewFinalizedHeaderCache(suite.log, suite.state, finalizationDistributor) + finalizedHeaderCache, err := synceng.NewFinalizedHeaderCache(suite.log, suite.state, followerDistributor) require.NoError(suite.T(), err) rpcEngBuilder, err := rpc.NewBuilder(suite.log, suite.state, config, suite.collClient, nil, suite.blocks, suite.headers, suite.collections, suite.transactions, nil, From 6c926bcfb98aac85bd0b30b423254fd35bdc1357 Mon Sep 17 00:00:00 2001 From: Alexander Hentschel Date: Sat, 6 May 2023 14:43:50 -0700 Subject: [PATCH 15/38] =?UTF-8?q?=E2=80=A2=20modularized=20consumer=20inte?= =?UTF-8?q?rfaces=20for=20hotstuff=20notifications=20=E2=80=A2=20refactore?= =?UTF-8?q?d=20distributors=20to=20be=20easily=20composable,=20still=20exp?= =?UTF-8?q?osing=20functionality=20to=20only=20subscribe=20to=20a=20subset?= =?UTF-8?q?=20of=20events=20corresponding=20to=20a=20atomic=20interface=20?= =?UTF-8?q?for=20hotstuff=20notifications?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../node_builder/access_node_builder.go | 2 +- cmd/collection/main.go | 2 +- cmd/consensus/main.go | 2 +- cmd/execution_builder.go | 4 +- cmd/observer/node_builder/observer_builder.go | 2 +- cmd/verification_builder.go | 2 +- consensus/follower.go | 2 +- consensus/hotstuff/consumer.go | 101 +++---- consensus/hotstuff/forks/forks2.go | 4 +- .../hotstuff/integration/instance_test.go | 4 +- .../hotstuff/notifications/noop_consumer.go | 21 +- .../pubsub/communicator_distributor.go | 56 ++++ .../notifications/pubsub/distributor.go | 246 ++---------------- .../pubsub/finalization_distributor.go | 68 +++++ .../pubsub/follower_distributor.go | 85 ------ .../pubsub/participant_distributor.go | 143 ++++++++++ .../pubsub/protocol_violation_distributor.go | 85 ++++++ .../pubsub/qc_created_distributor.go | 2 +- .../pubsub/timeout_collector_distributor.go | 10 +- .../slashing_violation_consumer.go | 25 +- consensus/integration/nodes_test.go | 2 +- consensus/participant.go | 6 +- engine/common/follower/compliance_core.go | 2 +- engine/testutil/nodes.go | 2 +- follower/follower_builder.go | 2 +- 25 files changed, 483 insertions(+), 397 deletions(-) create mode 100644 consensus/hotstuff/notifications/pubsub/communicator_distributor.go create mode 100644 consensus/hotstuff/notifications/pubsub/finalization_distributor.go delete mode 100644 consensus/hotstuff/notifications/pubsub/follower_distributor.go create mode 100644 consensus/hotstuff/notifications/pubsub/participant_distributor.go create mode 100644 consensus/hotstuff/notifications/pubsub/protocol_violation_distributor.go diff --git a/cmd/access/node_builder/access_node_builder.go b/cmd/access/node_builder/access_node_builder.go index 22f133dc630..fffdde44a4b 100644 --- a/cmd/access/node_builder/access_node_builder.go +++ b/cmd/access/node_builder/access_node_builder.go @@ -609,7 +609,7 @@ func (builder *FlowAccessNodeBuilder) BuildExecutionDataRequester() *FlowAccessN func FlowAccessNode(nodeBuilder *cmd.FlowNodeBuilder) *FlowAccessNodeBuilder { dist := consensuspubsub.NewFollowerDistributor() - dist.AddConsumer(notifications.NewSlashingViolationsConsumer(nodeBuilder.Logger)) + dist.AddFollowerConsumer(notifications.NewSlashingViolationsConsumer(nodeBuilder.Logger)) return &FlowAccessNodeBuilder{ AccessNodeConfig: DefaultAccessNodeConfig(), FlowNodeBuilder: nodeBuilder, diff --git a/cmd/collection/main.go b/cmd/collection/main.go index 98921e0ec5b..0d87a13141f 100644 --- a/cmd/collection/main.go +++ b/cmd/collection/main.go @@ -173,7 +173,7 @@ func main() { PreInit(cmd.DynamicStartPreInit). Module("follower distributor", func(node *cmd.NodeConfig) error { followerDistributor = pubsub.NewFollowerDistributor() - followerDistributor.AddConsumer(notifications.NewSlashingViolationsConsumer(node.Logger)) + followerDistributor.AddFollowerConsumer(notifications.NewSlashingViolationsConsumer(node.Logger)) return nil }). Module("mutable follower state", func(node *cmd.NodeConfig) error { diff --git a/cmd/consensus/main.go b/cmd/consensus/main.go index 280ecd8be48..fae7ba475a6 100644 --- a/cmd/consensus/main.go +++ b/cmd/consensus/main.go @@ -366,7 +366,7 @@ func main() { }). Module("follower distributor", func(node *cmd.NodeConfig) error { followerDistributor = pubsub.NewFollowerDistributor() - followerDistributor.AddConsumer(notifications.NewSlashingViolationsConsumer(nodeBuilder.Logger)) + followerDistributor.AddProtocolViolationConsumer(notifications.NewSlashingViolationsConsumer(nodeBuilder.Logger)) return nil }). Module("machine account config", func(node *cmd.NodeConfig) error { diff --git a/cmd/execution_builder.go b/cmd/execution_builder.go index a2660ccc353..0bfd0ad10a0 100644 --- a/cmd/execution_builder.go +++ b/cmd/execution_builder.go @@ -274,7 +274,7 @@ func (exeNode *ExecutionNode) LoadExecutionReceiptsStorage( func (exeNode *ExecutionNode) LoadFollowerDistributor(node *NodeConfig) error { exeNode.followerDistributor = pubsub.NewFollowerDistributor() - exeNode.followerDistributor.AddConsumer(notifications.NewSlashingViolationsConsumer(node.Logger)) + exeNode.followerDistributor.AddProtocolViolationConsumer(notifications.NewSlashingViolationsConsumer(node.Logger)) return nil } @@ -854,7 +854,7 @@ func (exeNode *ExecutionNode) LoadFollowerCore( return nil, fmt.Errorf("could not find latest finalized block and pending blocks to recover consensus follower: %w", err) } - exeNode.followerDistributor.AddConsumer(exeNode.checkerEng) + exeNode.followerDistributor.AddFollowerConsumer(exeNode.checkerEng) // creates a consensus follower with ingestEngine as the notifier // so that it gets notified upon each new finalized block diff --git a/cmd/observer/node_builder/observer_builder.go b/cmd/observer/node_builder/observer_builder.go index 4074c9f244a..508af73d311 100644 --- a/cmd/observer/node_builder/observer_builder.go +++ b/cmd/observer/node_builder/observer_builder.go @@ -566,7 +566,7 @@ func NewFlowObserverServiceBuilder(opts ...Option) *ObserverServiceBuilder { FlowNodeBuilder: cmd.FlowNode("observer"), FollowerDistributor: pubsub.NewFollowerDistributor(), } - anb.FollowerDistributor.AddConsumer(notifications.NewSlashingViolationsConsumer(anb.Logger)) + anb.FollowerDistributor.AddFollowerConsumer(notifications.NewSlashingViolationsConsumer(anb.Logger)) // the observer gets a version of the root snapshot file that does not contain any node addresses // hence skip all the root snapshot validations that involved an identity address anb.FlowNodeBuilder.SkipNwAddressBasedValidations = true diff --git a/cmd/verification_builder.go b/cmd/verification_builder.go index 9924c4884c7..d7f2f196d16 100644 --- a/cmd/verification_builder.go +++ b/cmd/verification_builder.go @@ -179,7 +179,7 @@ func (v *VerificationNodeBuilder) LoadComponentsAndModules() { }). Module("follower distributor", func(node *NodeConfig) error { followerDistributor = pubsub.NewFollowerDistributor() - followerDistributor.AddConsumer(notifications.NewSlashingViolationsConsumer(node.Logger)) + followerDistributor.AddFollowerConsumer(notifications.NewSlashingViolationsConsumer(node.Logger)) return nil }). Module("sync core", func(node *NodeConfig) error { diff --git a/consensus/follower.go b/consensus/follower.go index 1fa432ef3d9..d155948833b 100644 --- a/consensus/follower.go +++ b/consensus/follower.go @@ -25,7 +25,7 @@ import ( func NewFollower(log zerolog.Logger, headers storage.Headers, updater module.Finalizer, - notifier hotstuff.ConsensusFollowerConsumer, + notifier hotstuff.FollowerConsumer, rootHeader *flow.Header, rootQC *flow.QuorumCertificate, finalized *flow.Header, diff --git a/consensus/hotstuff/consumer.go b/consensus/hotstuff/consumer.go index 2c144fe103e..43ab4dc012e 100644 --- a/consensus/hotstuff/consumer.go +++ b/consensus/hotstuff/consumer.go @@ -7,9 +7,9 @@ import ( "github.com/onflow/flow-go/model/flow" ) -// ProtocolViolationConsumer consumes outbound notifications produced by compliance. -// Notifications can be produced by consensus participants and followers. -// Notifications are meant to report protocol violations that can be observed by executing compliance checks. +// ProtocolViolationConsumer consumes outbound notifications about HotStuff-protocol violations. +// Such notifications are produced by the active consensus participants and to a lesser +// degree also the consensus follower. // // Implementations must: // - be concurrency safe @@ -23,6 +23,7 @@ type ProtocolViolationConsumer interface { // Implementation must be concurrency safe; Non-blocking; // and must handle repetition of the same events (with some processing overhead). OnInvalidBlockDetected(err model.InvalidBlockError) + // OnDoubleProposeDetected notifications are produced by the Finalization Logic // whenever a double block proposal (equivocation) was detected. // Equivocation occurs when the same leader proposes two different blocks for the same view. @@ -30,12 +31,47 @@ type ProtocolViolationConsumer interface { // Implementation must be concurrency safe; Non-blocking; // and must handle repetition of the same events (with some processing overhead). OnDoubleProposeDetected(*model.Block, *model.Block) + + // OnDoubleVotingDetected notifications are produced by the Vote Aggregation logic + // whenever a double voting (same voter voting for different blocks at the same view) was detected. + // Prerequisites: + // Implementation must be concurrency safe; Non-blocking; + // and must handle repetition of the same events (with some processing overhead). + OnDoubleVotingDetected(*model.Vote, *model.Vote) + + // OnInvalidVoteDetected notifications are produced by the Vote Aggregation logic + // whenever an invalid vote was detected. + // Prerequisites: + // Implementation must be concurrency safe; Non-blocking; + // and must handle repetition of the same events (with some processing overhead). + OnInvalidVoteDetected(err model.InvalidVoteError) + + // OnVoteForInvalidBlockDetected notifications are produced by the Vote Aggregation logic + // whenever vote for invalid proposal was detected. + // Prerequisites: + // Implementation must be concurrency safe; Non-blocking; + // and must handle repetition of the same events (with some processing overhead). + OnVoteForInvalidBlockDetected(vote *model.Vote, invalidProposal *model.Proposal) + + // OnDoubleTimeoutDetected notifications are produced by the Timeout Aggregation logic + // whenever a double timeout (same replica producing two different timeouts at the same view) was detected. + // Prerequisites: + // Implementation must be concurrency safe; Non-blocking; + // and must handle repetition of the same events (with some processing overhead). + OnDoubleTimeoutDetected(*model.TimeoutObject, *model.TimeoutObject) + + // OnInvalidTimeoutDetected notifications are produced by the Timeout Aggregation logic + // whenever an invalid timeout was detected. + // Prerequisites: + // Implementation must be concurrency safe; Non-blocking; + // and must handle repetition of the same events (with some processing overhead). + OnInvalidTimeoutDetected(err model.InvalidTimeoutError) } -// FinalizationConsumer consumes outbound notifications produced by the finalization logic. -// Notifications represent finalization-specific state changes which are potentially relevant -// to the larger node. The notifications are emitted in the order in which the -// finalization algorithm makes the respective steps. +// FinalizationConsumer consumes outbound notifications produced by the logic tracking +// forks and finalization. Such notifications are produced by the active consensus +// participants, and generally potentially relevant to the larger node. The notifications +// are emitted in the order in which the finalization algorithm makes the respective steps. // // Implementations must: // - be concurrency safe @@ -57,13 +93,13 @@ type FinalizationConsumer interface { OnFinalizedBlock(*model.Block) } -// ConsensusFollowerConsumer consumes outbound notifications produced by consensus followers. +// FollowerConsumer consumes outbound notifications produced by consensus followers. // It is a subset of the notifications produced by consensus participants. // Implementations must: // - be concurrency safe // - be non-blocking // - handle repetition of the same events (with some processing overhead). -type ConsensusFollowerConsumer interface { +type FollowerConsumer interface { ProtocolViolationConsumer FinalizationConsumer } @@ -78,9 +114,19 @@ type ConsensusFollowerConsumer interface { // - be non-blocking // - handle repetition of the same events (with some processing overhead). type Consumer interface { - ConsensusFollowerConsumer + FollowerConsumer CommunicatorConsumer + ParticipantConsumer +} +// ParticipantConsumer consumes outbound notifications produced by consensus participants +// actively proposing blocks, voting, collecting & aggregating votes to QCs, and participating in +// the pacemaker (sending timeouts, collecting & aggregating timeouts to TCs). +// Implementations must: +// - be concurrency safe +// - be non-blocking +// - handle repetition of the same events (with some processing overhead). +type ParticipantConsumer interface { // OnEventProcessed notifications are produced by the EventHandler when it is done processing // and hands control back to the EventLoop to wait for the next event. // Prerequisites: @@ -190,41 +236,6 @@ type Consumer interface { // Implementation must be concurrency safe; Non-blocking; // and must handle repetition of the same events (with some processing overhead). OnCurrentViewDetails(currentView, finalizedView uint64, currentLeader flow.Identifier) - - // OnDoubleVotingDetected notifications are produced by the Vote Aggregation logic - // whenever a double voting (same voter voting for different blocks at the same view) was detected. - // Prerequisites: - // Implementation must be concurrency safe; Non-blocking; - // and must handle repetition of the same events (with some processing overhead). - OnDoubleVotingDetected(*model.Vote, *model.Vote) - - // OnInvalidVoteDetected notifications are produced by the Vote Aggregation logic - // whenever an invalid vote was detected. - // Prerequisites: - // Implementation must be concurrency safe; Non-blocking; - // and must handle repetition of the same events (with some processing overhead). - OnInvalidVoteDetected(err model.InvalidVoteError) - - // OnVoteForInvalidBlockDetected notifications are produced by the Vote Aggregation logic - // whenever vote for invalid proposal was detected. - // Prerequisites: - // Implementation must be concurrency safe; Non-blocking; - // and must handle repetition of the same events (with some processing overhead). - OnVoteForInvalidBlockDetected(vote *model.Vote, invalidProposal *model.Proposal) - - // OnDoubleTimeoutDetected notifications are produced by the Timeout Aggregation logic - // whenever a double timeout (same replica producing two different timeouts at the same view) was detected. - // Prerequisites: - // Implementation must be concurrency safe; Non-blocking; - // and must handle repetition of the same events (with some processing overhead). - OnDoubleTimeoutDetected(*model.TimeoutObject, *model.TimeoutObject) - - // OnInvalidTimeoutDetected notifications are produced by the Timeout Aggregation logic - // whenever an invalid timeout was detected. - // Prerequisites: - // Implementation must be concurrency safe; Non-blocking; - // and must handle repetition of the same events (with some processing overhead). - OnInvalidTimeoutDetected(err model.InvalidTimeoutError) } // QCCreatedConsumer consumes outbound notifications produced by HotStuff and its components. diff --git a/consensus/hotstuff/forks/forks2.go b/consensus/hotstuff/forks/forks2.go index e9aaacca4a9..e4ebc59112b 100644 --- a/consensus/hotstuff/forks/forks2.go +++ b/consensus/hotstuff/forks/forks2.go @@ -19,7 +19,7 @@ import ( // Forks is NOT safe for concurrent use by multiple goroutines. type Forks struct { finalizationCallback module.Finalizer - notifier hotstuff.ConsensusFollowerConsumer + notifier hotstuff.FollowerConsumer forest forest.LevelledForest trustedRoot *model.CertifiedBlock @@ -30,7 +30,7 @@ type Forks struct { var _ hotstuff.Forks = (*Forks)(nil) -func New(trustedRoot *model.CertifiedBlock, finalizationCallback module.Finalizer, notifier hotstuff.ConsensusFollowerConsumer) (*Forks, error) { +func New(trustedRoot *model.CertifiedBlock, finalizationCallback module.Finalizer, notifier hotstuff.FollowerConsumer) (*Forks, error) { if (trustedRoot.Block.BlockID != trustedRoot.CertifyingQC.BlockID) || (trustedRoot.Block.View != trustedRoot.CertifyingQC.View) { return nil, model.NewConfigurationErrorf("invalid root: root QC is not pointing to root block") } diff --git a/consensus/hotstuff/integration/instance_test.go b/consensus/hotstuff/integration/instance_test.go index e981f335329..a082e2becbb 100644 --- a/consensus/hotstuff/integration/instance_test.go +++ b/consensus/hotstuff/integration/instance_test.go @@ -539,8 +539,8 @@ func NewInstance(t *testing.T, options ...Option) *Instance { ) require.NoError(t, err) - collectorDistributor.AddConsumer(logConsumer) - collectorDistributor.AddConsumer(&in) + collectorDistributor.AddTimeoutCollectorConsumer(logConsumer) + collectorDistributor.AddTimeoutCollectorConsumer(&in) return &in } diff --git a/consensus/hotstuff/notifications/noop_consumer.go b/consensus/hotstuff/notifications/noop_consumer.go index f3babd8f81a..c2f02acbcae 100644 --- a/consensus/hotstuff/notifications/noop_consumer.go +++ b/consensus/hotstuff/notifications/noop_consumer.go @@ -56,16 +56,6 @@ func (*NoopPartialConsumer) OnTimeoutProcessed(*model.TimeoutObject) {} func (*NoopPartialConsumer) OnCurrentViewDetails(uint64, uint64, flow.Identifier) {} -func (*NoopPartialConsumer) OnDoubleVotingDetected(*model.Vote, *model.Vote) {} - -func (*NoopPartialConsumer) OnInvalidVoteDetected(model.InvalidVoteError) {} - -func (*NoopPartialConsumer) OnVoteForInvalidBlockDetected(*model.Vote, *model.Proposal) {} - -func (*NoopPartialConsumer) OnDoubleTimeoutDetected(*model.TimeoutObject, *model.TimeoutObject) {} - -func (*NoopPartialConsumer) OnInvalidTimeoutDetected(model.InvalidTimeoutError) {} - // no-op implementation of hotstuff.FinalizationConsumer type NoopFinalizationConsumer struct{} @@ -120,3 +110,14 @@ var _ hotstuff.ProtocolViolationConsumer = (*NoopProtocolViolationConsumer)(nil) func (*NoopProtocolViolationConsumer) OnInvalidBlockDetected(model.InvalidBlockError) {} func (*NoopProtocolViolationConsumer) OnDoubleProposeDetected(*model.Block, *model.Block) {} + +func (*NoopProtocolViolationConsumer) OnDoubleVotingDetected(*model.Vote, *model.Vote) {} + +func (*NoopProtocolViolationConsumer) OnInvalidVoteDetected(model.InvalidVoteError) {} + +func (*NoopProtocolViolationConsumer) OnVoteForInvalidBlockDetected(*model.Vote, *model.Proposal) {} + +func (*NoopProtocolViolationConsumer) OnDoubleTimeoutDetected(*model.TimeoutObject, *model.TimeoutObject) { +} + +func (*NoopProtocolViolationConsumer) OnInvalidTimeoutDetected(model.InvalidTimeoutError) {} diff --git a/consensus/hotstuff/notifications/pubsub/communicator_distributor.go b/consensus/hotstuff/notifications/pubsub/communicator_distributor.go new file mode 100644 index 00000000000..521e06ee50b --- /dev/null +++ b/consensus/hotstuff/notifications/pubsub/communicator_distributor.go @@ -0,0 +1,56 @@ +package pubsub + +import ( + "sync" + "time" + + "github.com/onflow/flow-go/consensus/hotstuff" + "github.com/onflow/flow-go/consensus/hotstuff/model" + "github.com/onflow/flow-go/model/flow" +) + +// CommunicatorDistributor ingests outbound consensus messages from HotStuff's core logic and +// distributes them to subscribers. This logic only runs inside active consensus participants proposing +// blocks, voting, collecting + aggregating votes to QCs, and participating in the pacemaker (sending +// timeouts, collecting + aggregating timeouts to TCs). +// Concurrently safe. +type CommunicatorDistributor struct { + subscribers []hotstuff.CommunicatorConsumer + lock sync.RWMutex +} + +var _ hotstuff.CommunicatorConsumer = (*CommunicatorDistributor)(nil) + +func NewCommunicatorConsumerDistributor() *CommunicatorDistributor { + return &CommunicatorDistributor{} +} + +func (d *CommunicatorDistributor) AddCommunicatorConsumer(consumer hotstuff.CommunicatorConsumer) { + d.lock.Lock() + defer d.lock.Unlock() + d.subscribers = append(d.subscribers, consumer) +} + +func (d *CommunicatorDistributor) OnOwnVote(blockID flow.Identifier, view uint64, sigData []byte, recipientID flow.Identifier) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, s := range d.subscribers { + s.OnOwnVote(blockID, view, sigData, recipientID) + } +} + +func (d *CommunicatorDistributor) OnOwnTimeout(timeout *model.TimeoutObject) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, s := range d.subscribers { + s.OnOwnTimeout(timeout) + } +} + +func (d *CommunicatorDistributor) OnOwnProposal(proposal *flow.Header, targetPublicationTime time.Time) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, s := range d.subscribers { + s.OnOwnProposal(proposal, targetPublicationTime) + } +} diff --git a/consensus/hotstuff/notifications/pubsub/distributor.go b/consensus/hotstuff/notifications/pubsub/distributor.go index 74674ee8547..7c600a71e88 100644 --- a/consensus/hotstuff/notifications/pubsub/distributor.go +++ b/consensus/hotstuff/notifications/pubsub/distributor.go @@ -1,258 +1,44 @@ package pubsub import ( - "sync" - "time" - "github.com/onflow/flow-go/consensus/hotstuff" - "github.com/onflow/flow-go/consensus/hotstuff/model" - "github.com/onflow/flow-go/consensus/hotstuff/notifications" - "github.com/onflow/flow-go/model/flow" ) // Distributor distributes notifications to a list of subscribers (event consumers). // // It allows thread-safe subscription of multiple consumers to events. type Distributor struct { - subscribers []hotstuff.Consumer - lock sync.RWMutex + FollowerDistributor + CommunicatorDistributor + ParticipantDistributor } var _ hotstuff.Consumer = (*Distributor)(nil) -func (p *Distributor) OnEventProcessed() { - p.lock.RLock() - defer p.lock.RUnlock() - for _, subscriber := range p.subscribers { - subscriber.OnEventProcessed() - } -} - func NewDistributor() *Distributor { return &Distributor{} } // AddConsumer adds an event consumer to the Distributor func (p *Distributor) AddConsumer(consumer hotstuff.Consumer) { - p.lock.Lock() - defer p.lock.Unlock() - p.subscribers = append(p.subscribers, consumer) -} - -// AddFollowerConsumer registers the input `consumer` to be notified on ConsensusFollowerConsumer events. -func (p *Distributor) AddFollowerConsumer(consumer hotstuff.ConsensusFollowerConsumer) { - p.lock.Lock() - defer p.lock.Unlock() - - var wrappedConsumer hotstuff.Consumer = &struct { - notifications.NoopCommunicatorConsumer - notifications.NoopPartialConsumer - hotstuff.ConsensusFollowerConsumer - }{ - notifications.NoopCommunicatorConsumer{}, - notifications.NoopPartialConsumer{}, - consumer, - } - - p.subscribers = append(p.subscribers, wrappedConsumer) -} - -func (p *Distributor) OnStart(currentView uint64) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, subscriber := range p.subscribers { - subscriber.OnStart(currentView) - } -} - -func (p *Distributor) OnReceiveProposal(currentView uint64, proposal *model.Proposal) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, subscriber := range p.subscribers { - subscriber.OnReceiveProposal(currentView, proposal) - } -} - -func (p *Distributor) OnReceiveQc(currentView uint64, qc *flow.QuorumCertificate) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, subscriber := range p.subscribers { - subscriber.OnReceiveQc(currentView, qc) - } -} - -func (p *Distributor) OnReceiveTc(currentView uint64, tc *flow.TimeoutCertificate) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, subscriber := range p.subscribers { - subscriber.OnReceiveTc(currentView, tc) - } -} - -func (p *Distributor) OnPartialTc(currentView uint64, partialTc *hotstuff.PartialTcCreated) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, subscriber := range p.subscribers { - subscriber.OnPartialTc(currentView, partialTc) - } -} - -func (p *Distributor) OnLocalTimeout(currentView uint64) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, subscriber := range p.subscribers { - subscriber.OnLocalTimeout(currentView) - } -} - -func (p *Distributor) OnViewChange(oldView, newView uint64) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, subscriber := range p.subscribers { - subscriber.OnViewChange(oldView, newView) - } -} - -func (p *Distributor) OnQcTriggeredViewChange(oldView uint64, newView uint64, qc *flow.QuorumCertificate) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, subscriber := range p.subscribers { - subscriber.OnQcTriggeredViewChange(oldView, newView, qc) - } -} - -func (p *Distributor) OnTcTriggeredViewChange(oldView uint64, newView uint64, tc *flow.TimeoutCertificate) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, subscriber := range p.subscribers { - subscriber.OnTcTriggeredViewChange(oldView, newView, tc) - } + p.FollowerDistributor.AddFollowerConsumer(consumer) + p.CommunicatorDistributor.AddCommunicatorConsumer(consumer) + p.ParticipantDistributor.AddParticipantConsumer(consumer) } -func (p *Distributor) OnStartingTimeout(timerInfo model.TimerInfo) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, subscriber := range p.subscribers { - subscriber.OnStartingTimeout(timerInfo) - } +// FollowerDistributor ingests consensus follower events and distributes it to subscribers. +type FollowerDistributor struct { + ProtocolViolationDistributor + FinalizationDistributor } -func (p *Distributor) OnVoteProcessed(vote *model.Vote) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, subscriber := range p.subscribers { - subscriber.OnVoteProcessed(vote) - } -} - -func (p *Distributor) OnTimeoutProcessed(timeout *model.TimeoutObject) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, subscriber := range p.subscribers { - subscriber.OnTimeoutProcessed(timeout) - } -} - -func (p *Distributor) OnCurrentViewDetails(currentView, finalizedView uint64, currentLeader flow.Identifier) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, subscriber := range p.subscribers { - subscriber.OnCurrentViewDetails(currentView, finalizedView, currentLeader) - } -} - -func (p *Distributor) OnBlockIncorporated(block *model.Block) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, subscriber := range p.subscribers { - subscriber.OnBlockIncorporated(block) - } -} - -func (p *Distributor) OnFinalizedBlock(block *model.Block) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, subscriber := range p.subscribers { - subscriber.OnFinalizedBlock(block) - } -} - -func (p *Distributor) OnInvalidBlockDetected(err model.InvalidBlockError) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, subscriber := range p.subscribers { - subscriber.OnInvalidBlockDetected(err) - } -} - -func (p *Distributor) OnDoubleProposeDetected(block1, block2 *model.Block) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, subscriber := range p.subscribers { - subscriber.OnDoubleProposeDetected(block1, block2) - } -} - -func (p *Distributor) OnDoubleVotingDetected(vote1, vote2 *model.Vote) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, subscriber := range p.subscribers { - subscriber.OnDoubleVotingDetected(vote1, vote2) - } -} - -func (p *Distributor) OnInvalidVoteDetected(err model.InvalidVoteError) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, subscriber := range p.subscribers { - subscriber.OnInvalidVoteDetected(err) - } -} - -func (p *Distributor) OnVoteForInvalidBlockDetected(vote *model.Vote, invalidProposal *model.Proposal) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, subscriber := range p.subscribers { - subscriber.OnVoteForInvalidBlockDetected(vote, invalidProposal) - } -} - -func (p *Distributor) OnDoubleTimeoutDetected(timeout *model.TimeoutObject, altTimeout *model.TimeoutObject) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, subscriber := range p.subscribers { - subscriber.OnDoubleTimeoutDetected(timeout, altTimeout) - } -} - -func (p *Distributor) OnInvalidTimeoutDetected(err model.InvalidTimeoutError) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, subscriber := range p.subscribers { - subscriber.OnInvalidTimeoutDetected(err) - } -} - -func (p *Distributor) OnOwnVote(blockID flow.Identifier, view uint64, sigData []byte, recipientID flow.Identifier) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, s := range p.subscribers { - s.OnOwnVote(blockID, view, sigData, recipientID) - } -} +var _ hotstuff.FollowerConsumer = (*FollowerDistributor)(nil) -func (p *Distributor) OnOwnTimeout(timeout *model.TimeoutObject) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, s := range p.subscribers { - s.OnOwnTimeout(timeout) - } +func NewFollowerDistributor() *FollowerDistributor { + return &FollowerDistributor{} } -func (p *Distributor) OnOwnProposal(proposal *flow.Header, targetPublicationTime time.Time) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, s := range p.subscribers { - s.OnOwnProposal(proposal, targetPublicationTime) - } +func (d *FollowerDistributor) AddFollowerConsumer(consumer hotstuff.FollowerConsumer) { + d.FinalizationDistributor.AddFinalizationConsumer(consumer) + d.ProtocolViolationDistributor.AddProtocolViolationConsumer(consumer) } diff --git a/consensus/hotstuff/notifications/pubsub/finalization_distributor.go b/consensus/hotstuff/notifications/pubsub/finalization_distributor.go new file mode 100644 index 00000000000..a78dd88d13b --- /dev/null +++ b/consensus/hotstuff/notifications/pubsub/finalization_distributor.go @@ -0,0 +1,68 @@ +package pubsub + +import ( + "sync" + + "github.com/onflow/flow-go/consensus/hotstuff" + "github.com/onflow/flow-go/consensus/hotstuff/model" +) + +type OnBlockFinalizedConsumer = func(block *model.Block) +type OnBlockIncorporatedConsumer = func(block *model.Block) + +// FinalizationDistributor ingests events from HotStuff's logic for tracking forks + finalization +// and distributes them to subscribers. This logic generally runs inside all nodes (irrespectively whether +// they are active consensus participants or or only consensus followers). +// Concurrently safe. +type FinalizationDistributor struct { + blockFinalizedConsumers []OnBlockFinalizedConsumer + blockIncorporatedConsumers []OnBlockIncorporatedConsumer + consumers []hotstuff.FinalizationConsumer + lock sync.RWMutex +} + +var _ hotstuff.FinalizationConsumer = (*FinalizationDistributor)(nil) + +func NewFinalizationDistributor() *FinalizationDistributor { + return &FinalizationDistributor{} +} + +func (d *FinalizationDistributor) AddOnBlockFinalizedConsumer(consumer OnBlockFinalizedConsumer) { + d.lock.Lock() + defer d.lock.Unlock() + d.blockFinalizedConsumers = append(d.blockFinalizedConsumers, consumer) +} + +func (d *FinalizationDistributor) AddOnBlockIncorporatedConsumer(consumer OnBlockIncorporatedConsumer) { + d.lock.Lock() + defer d.lock.Unlock() + d.blockIncorporatedConsumers = append(d.blockIncorporatedConsumers, consumer) +} + +func (d *FinalizationDistributor) AddFinalizationConsumer(consumer hotstuff.FinalizationConsumer) { + d.lock.Lock() + defer d.lock.Unlock() + d.consumers = append(d.consumers, consumer) +} + +func (d *FinalizationDistributor) OnBlockIncorporated(block *model.Block) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, consumer := range d.blockIncorporatedConsumers { + consumer(block) + } + for _, consumer := range d.consumers { + consumer.OnBlockIncorporated(block) + } +} + +func (d *FinalizationDistributor) OnFinalizedBlock(block *model.Block) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, consumer := range d.blockFinalizedConsumers { + consumer(block) + } + for _, consumer := range d.consumers { + consumer.OnFinalizedBlock(block) + } +} diff --git a/consensus/hotstuff/notifications/pubsub/follower_distributor.go b/consensus/hotstuff/notifications/pubsub/follower_distributor.go deleted file mode 100644 index 54ad77ac925..00000000000 --- a/consensus/hotstuff/notifications/pubsub/follower_distributor.go +++ /dev/null @@ -1,85 +0,0 @@ -package pubsub - -import ( - "sync" - - "github.com/onflow/flow-go/consensus/hotstuff" - "github.com/onflow/flow-go/consensus/hotstuff/model" -) - -type OnBlockFinalizedConsumer = func(block *model.Block) -type OnBlockIncorporatedConsumer = func(block *model.Block) - -// FollowerDistributor ingests consensus follower events and distributes it to subscribers. -type FollowerDistributor struct { - blockFinalizedConsumers []OnBlockFinalizedConsumer - blockIncorporatedConsumers []OnBlockIncorporatedConsumer - followerConsumers []hotstuff.ConsensusFollowerConsumer - lock sync.RWMutex -} - -var _ hotstuff.ConsensusFollowerConsumer = (*FollowerDistributor)(nil) - -func NewFollowerDistributor() *FollowerDistributor { - return &FollowerDistributor{ - blockFinalizedConsumers: make([]OnBlockFinalizedConsumer, 0), - blockIncorporatedConsumers: make([]OnBlockIncorporatedConsumer, 0), - lock: sync.RWMutex{}, - } -} - -func (p *FollowerDistributor) AddOnBlockFinalizedConsumer(consumer OnBlockFinalizedConsumer) { - p.lock.Lock() - defer p.lock.Unlock() - p.blockFinalizedConsumers = append(p.blockFinalizedConsumers, consumer) -} - -func (p *FollowerDistributor) AddOnBlockIncorporatedConsumer(consumer OnBlockIncorporatedConsumer) { - p.lock.Lock() - defer p.lock.Unlock() - p.blockIncorporatedConsumers = append(p.blockIncorporatedConsumers, consumer) -} - -func (p *FollowerDistributor) AddConsumer(consumer hotstuff.ConsensusFollowerConsumer) { - p.lock.Lock() - defer p.lock.Unlock() - p.followerConsumers = append(p.followerConsumers, consumer) -} - -func (p *FollowerDistributor) OnBlockIncorporated(block *model.Block) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, consumer := range p.blockIncorporatedConsumers { - consumer(block) - } - for _, consumer := range p.followerConsumers { - consumer.OnBlockIncorporated(block) - } -} - -func (p *FollowerDistributor) OnFinalizedBlock(block *model.Block) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, consumer := range p.blockFinalizedConsumers { - consumer(block) - } - for _, consumer := range p.followerConsumers { - consumer.OnFinalizedBlock(block) - } -} - -func (p *FollowerDistributor) OnDoubleProposeDetected(block1, block2 *model.Block) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, consumer := range p.followerConsumers { - consumer.OnDoubleProposeDetected(block1, block2) - } -} - -func (p *FollowerDistributor) OnInvalidBlockDetected(err model.InvalidBlockError) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, consumer := range p.followerConsumers { - consumer.OnInvalidBlockDetected(err) - } -} diff --git a/consensus/hotstuff/notifications/pubsub/participant_distributor.go b/consensus/hotstuff/notifications/pubsub/participant_distributor.go new file mode 100644 index 00000000000..f0fae001a41 --- /dev/null +++ b/consensus/hotstuff/notifications/pubsub/participant_distributor.go @@ -0,0 +1,143 @@ +package pubsub + +import ( + "sync" + + "github.com/onflow/flow-go/consensus/hotstuff" + "github.com/onflow/flow-go/consensus/hotstuff/model" + "github.com/onflow/flow-go/model/flow" +) + +// ParticipantDistributor ingests events from HotStuff's core logic and distributes them to +// subscribers. This logic only runs inside active consensus participants proposing blocks, voting, +// collecting + aggregating votes to QCs, and participating in the pacemaker (sending timeouts, +// collecting + aggregating timeouts to TCs). +// Concurrently safe. +type ParticipantDistributor struct { + subscribers []hotstuff.ParticipantConsumer + lock sync.RWMutex +} + +var _ hotstuff.ParticipantConsumer = (*ParticipantDistributor)(nil) + +func NewConsensusParticipantDistributor() *ParticipantDistributor { + return &ParticipantDistributor{} +} + +func (d *ParticipantDistributor) AddParticipantConsumer(consumer hotstuff.ParticipantConsumer) { + d.lock.Lock() + defer d.lock.Unlock() + d.subscribers = append(d.subscribers, consumer) +} + +func (d *ParticipantDistributor) OnEventProcessed() { + d.lock.RLock() + defer d.lock.RUnlock() + for _, subscriber := range d.subscribers { + subscriber.OnEventProcessed() + } +} + +func (d *ParticipantDistributor) OnStart(currentView uint64) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, subscriber := range d.subscribers { + subscriber.OnStart(currentView) + } +} + +func (d *ParticipantDistributor) OnReceiveProposal(currentView uint64, proposal *model.Proposal) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, subscriber := range d.subscribers { + subscriber.OnReceiveProposal(currentView, proposal) + } +} + +func (d *ParticipantDistributor) OnReceiveQc(currentView uint64, qc *flow.QuorumCertificate) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, subscriber := range d.subscribers { + subscriber.OnReceiveQc(currentView, qc) + } +} + +func (d *ParticipantDistributor) OnReceiveTc(currentView uint64, tc *flow.TimeoutCertificate) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, subscriber := range d.subscribers { + subscriber.OnReceiveTc(currentView, tc) + } +} + +func (d *ParticipantDistributor) OnPartialTc(currentView uint64, partialTc *hotstuff.PartialTcCreated) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, subscriber := range d.subscribers { + subscriber.OnPartialTc(currentView, partialTc) + } +} + +func (d *ParticipantDistributor) OnLocalTimeout(currentView uint64) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, subscriber := range d.subscribers { + subscriber.OnLocalTimeout(currentView) + } +} + +func (d *ParticipantDistributor) OnViewChange(oldView, newView uint64) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, subscriber := range d.subscribers { + subscriber.OnViewChange(oldView, newView) + } +} + +func (d *ParticipantDistributor) OnQcTriggeredViewChange(oldView uint64, newView uint64, qc *flow.QuorumCertificate) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, subscriber := range d.subscribers { + subscriber.OnQcTriggeredViewChange(oldView, newView, qc) + } +} + +func (d *ParticipantDistributor) OnTcTriggeredViewChange(oldView uint64, newView uint64, tc *flow.TimeoutCertificate) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, subscriber := range d.subscribers { + subscriber.OnTcTriggeredViewChange(oldView, newView, tc) + } +} + +func (d *ParticipantDistributor) OnStartingTimeout(timerInfo model.TimerInfo) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, subscriber := range d.subscribers { + subscriber.OnStartingTimeout(timerInfo) + } +} + +func (d *ParticipantDistributor) OnVoteProcessed(vote *model.Vote) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, subscriber := range d.subscribers { + subscriber.OnVoteProcessed(vote) + } +} + +func (d *ParticipantDistributor) OnTimeoutProcessed(timeout *model.TimeoutObject) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, subscriber := range d.subscribers { + subscriber.OnTimeoutProcessed(timeout) + } +} + +func (d *ParticipantDistributor) OnCurrentViewDetails(currentView, finalizedView uint64, currentLeader flow.Identifier) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, subscriber := range d.subscribers { + subscriber.OnCurrentViewDetails(currentView, finalizedView, currentLeader) + } +} diff --git a/consensus/hotstuff/notifications/pubsub/protocol_violation_distributor.go b/consensus/hotstuff/notifications/pubsub/protocol_violation_distributor.go new file mode 100644 index 00000000000..4c08dacddac --- /dev/null +++ b/consensus/hotstuff/notifications/pubsub/protocol_violation_distributor.go @@ -0,0 +1,85 @@ +package pubsub + +import ( + "sync" + + "github.com/onflow/flow-go/consensus/hotstuff" + "github.com/onflow/flow-go/consensus/hotstuff/model" +) + +// ProtocolViolationDistributor ingests notifications about HotStuff-protocol violations and +// distributes them to subscribers. Such notifications are produced by the active consensus +// participants and to a lesser degree also the consensus follower. +// Concurrently safe. +type ProtocolViolationDistributor struct { + subscribers []hotstuff.ProtocolViolationConsumer + lock sync.RWMutex +} + +var _ hotstuff.ProtocolViolationConsumer = (*ProtocolViolationDistributor)(nil) + +func NewProtocolViolationDistributor() *ProtocolViolationDistributor { + return &ProtocolViolationDistributor{} +} + +func (d *ProtocolViolationDistributor) AddProtocolViolationConsumer(consumer hotstuff.ProtocolViolationConsumer) { + d.lock.Lock() + defer d.lock.Unlock() + d.subscribers = append(d.subscribers, consumer) +} + +func (d *ProtocolViolationDistributor) OnInvalidBlockDetected(err model.InvalidBlockError) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, subscriber := range d.subscribers { + subscriber.OnInvalidBlockDetected(err) + } +} + +func (d *ProtocolViolationDistributor) OnDoubleProposeDetected(block1, block2 *model.Block) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, subscriber := range d.subscribers { + subscriber.OnDoubleProposeDetected(block1, block2) + } +} + +func (d *ProtocolViolationDistributor) OnDoubleVotingDetected(vote1, vote2 *model.Vote) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, subscriber := range d.subscribers { + subscriber.OnDoubleVotingDetected(vote1, vote2) + } +} + +func (d *ProtocolViolationDistributor) OnInvalidVoteDetected(err model.InvalidVoteError) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, subscriber := range d.subscribers { + subscriber.OnInvalidVoteDetected(err) + } +} + +func (d *ProtocolViolationDistributor) OnVoteForInvalidBlockDetected(vote *model.Vote, invalidProposal *model.Proposal) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, subscriber := range d.subscribers { + subscriber.OnVoteForInvalidBlockDetected(vote, invalidProposal) + } +} + +func (d *ProtocolViolationDistributor) OnDoubleTimeoutDetected(timeout *model.TimeoutObject, altTimeout *model.TimeoutObject) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, subscriber := range d.subscribers { + subscriber.OnDoubleTimeoutDetected(timeout, altTimeout) + } +} + +func (d *ProtocolViolationDistributor) OnInvalidTimeoutDetected(err model.InvalidTimeoutError) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, subscriber := range d.subscribers { + subscriber.OnInvalidTimeoutDetected(err) + } +} diff --git a/consensus/hotstuff/notifications/pubsub/qc_created_distributor.go b/consensus/hotstuff/notifications/pubsub/qc_created_distributor.go index 166fa9cf757..b9adafc2567 100644 --- a/consensus/hotstuff/notifications/pubsub/qc_created_distributor.go +++ b/consensus/hotstuff/notifications/pubsub/qc_created_distributor.go @@ -24,7 +24,7 @@ func NewQCCreatedDistributor() *QCCreatedDistributor { } } -func (d *QCCreatedDistributor) AddConsumer(consumer hotstuff.QCCreatedConsumer) { +func (d *QCCreatedDistributor) AddQCCreatedConsumer(consumer hotstuff.QCCreatedConsumer) { d.lock.Lock() defer d.lock.Unlock() d.qcCreatedConsumers = append(d.qcCreatedConsumers, consumer) diff --git a/consensus/hotstuff/notifications/pubsub/timeout_collector_distributor.go b/consensus/hotstuff/notifications/pubsub/timeout_collector_distributor.go index 8387fb81663..aa9c0bd9397 100644 --- a/consensus/hotstuff/notifications/pubsub/timeout_collector_distributor.go +++ b/consensus/hotstuff/notifications/pubsub/timeout_collector_distributor.go @@ -7,8 +7,8 @@ import ( "github.com/onflow/flow-go/model/flow" ) -// TimeoutCollectorDistributor ingests events from hotstuff and distributes them to subscribers. -// Concurrently safe +// TimeoutCollectorDistributor ingests events from hotstuff's VoteCollector +// logic and distributes them to subscribers. Concurrently safe. // TODO: investigate if this can be updated using atomics to prevent locking on mutex since we always add all consumers // before delivering events. type TimeoutCollectorDistributor struct { @@ -19,12 +19,10 @@ type TimeoutCollectorDistributor struct { var _ hotstuff.TimeoutCollectorConsumer = (*TimeoutCollectorDistributor)(nil) func NewTimeoutCollectorDistributor() *TimeoutCollectorDistributor { - return &TimeoutCollectorDistributor{ - consumers: make([]hotstuff.TimeoutCollectorConsumer, 0), - } + return &TimeoutCollectorDistributor{} } -func (d *TimeoutCollectorDistributor) AddConsumer(consumer hotstuff.TimeoutCollectorConsumer) { +func (d *TimeoutCollectorDistributor) AddTimeoutCollectorConsumer(consumer hotstuff.TimeoutCollectorConsumer) { d.lock.Lock() defer d.lock.Unlock() d.consumers = append(d.consumers, consumer) diff --git a/consensus/hotstuff/notifications/slashing_violation_consumer.go b/consensus/hotstuff/notifications/slashing_violation_consumer.go index fb80e15e522..e67d87e3d48 100644 --- a/consensus/hotstuff/notifications/slashing_violation_consumer.go +++ b/consensus/hotstuff/notifications/slashing_violation_consumer.go @@ -3,6 +3,7 @@ package notifications import ( "github.com/rs/zerolog" + "github.com/onflow/flow-go/consensus/hotstuff" "github.com/onflow/flow-go/consensus/hotstuff/model" "github.com/onflow/flow-go/utils/logging" ) @@ -10,15 +11,27 @@ import ( // SlashingViolationsConsumer is an implementation of the notifications consumer that logs a // message for any slashable offenses. type SlashingViolationsConsumer struct { - NoopConsumer log zerolog.Logger } +var _ hotstuff.ProtocolViolationConsumer = (*SlashingViolationsConsumer)(nil) + func NewSlashingViolationsConsumer(log zerolog.Logger) *SlashingViolationsConsumer { return &SlashingViolationsConsumer{ log: log, } } +func (c *SlashingViolationsConsumer) OnInvalidBlockDetected(err model.InvalidBlockError) { + block := err.InvalidBlock.Block + c.log.Warn(). + Bool(logging.KeySuspicious, true). + Hex("proposer_id", block.ProposerID[:]). + Uint64("block_view", block.View). + Hex("block_id", block.BlockID[:]). + Hex("block_payloadhash", block.PayloadHash[:]). + Time("block_timestamp", block.Timestamp). + Msg("OnInvalidBlockDetected") +} func (c *SlashingViolationsConsumer) OnDoubleVotingDetected(vote1 *model.Vote, vote2 *model.Vote) { c.log.Warn(). @@ -41,6 +54,16 @@ func (c *SlashingViolationsConsumer) OnInvalidVoteDetected(err model.InvalidVote Msg("OnInvalidVoteDetected") } +func (c *SlashingViolationsConsumer) OnDoubleTimeoutDetected(timeout *model.TimeoutObject, altTimeout *model.TimeoutObject) { + c.log.Warn(). + Bool(logging.KeySuspicious, true). + Hex("timeout_creator", timeout.SignerID[:]). + Uint64("timeout_view", timeout.View). + Hex("timeout_id1", logging.ID(timeout.ID())). + Hex("timeout_id2", logging.ID(altTimeout.ID())). + Msg("OnDoubleTimeoutDetected") +} + func (c *SlashingViolationsConsumer) OnInvalidTimeoutDetected(err model.InvalidTimeoutError) { timeout := err.Timeout c.log.Warn(). diff --git a/consensus/integration/nodes_test.go b/consensus/integration/nodes_test.go index 817fc46fcba..6911118e877 100644 --- a/consensus/integration/nodes_test.go +++ b/consensus/integration/nodes_test.go @@ -531,7 +531,7 @@ func createNode( require.NoError(t, err) timeoutCollectorDistributor := pubsub.NewTimeoutCollectorDistributor() - timeoutCollectorDistributor.AddConsumer(logConsumer) + timeoutCollectorDistributor.AddTimeoutCollectorConsumer(logConsumer) timeoutProcessorFactory := timeoutcollector.NewTimeoutProcessorFactory( log, diff --git a/consensus/participant.go b/consensus/participant.go index 50e17b14b47..e6435c70b97 100644 --- a/consensus/participant.go +++ b/consensus/participant.go @@ -122,8 +122,8 @@ func NewParticipant( } // add observer, event loop needs to receive events from distributor - modules.QCCreatedDistributor.AddConsumer(loop) - modules.TimeoutCollectorDistributor.AddConsumer(loop) + modules.QCCreatedDistributor.AddQCCreatedConsumer(loop) + modules.TimeoutCollectorDistributor.AddTimeoutCollectorConsumer(loop) return loop, nil } @@ -139,7 +139,7 @@ func NewValidator(metrics module.HotstuffMetrics, committee hotstuff.DynamicComm } // NewForks recovers trusted root and creates new forks manager -func NewForks(final *flow.Header, headers storage.Headers, updater module.Finalizer, notifier hotstuff.ConsensusFollowerConsumer, rootHeader *flow.Header, rootQC *flow.QuorumCertificate) (*forks.Forks, error) { +func NewForks(final *flow.Header, headers storage.Headers, updater module.Finalizer, notifier hotstuff.FollowerConsumer, rootHeader *flow.Header, rootQC *flow.QuorumCertificate) (*forks.Forks, error) { // recover the trusted root trustedRoot, err := recoverTrustedRoot(final, headers, rootHeader, rootQC) if err != nil { diff --git a/engine/common/follower/compliance_core.go b/engine/common/follower/compliance_core.go index 92eec6d43ff..ed0f43abd12 100644 --- a/engine/common/follower/compliance_core.go +++ b/engine/common/follower/compliance_core.go @@ -59,7 +59,7 @@ var _ complianceCore = (*ComplianceCore)(nil) func NewComplianceCore(log zerolog.Logger, mempoolMetrics module.MempoolMetrics, heroCacheCollector module.HeroCacheMetrics, - followerConsumer hotstuff.ConsensusFollowerConsumer, + followerConsumer hotstuff.FollowerConsumer, state protocol.FollowerState, follower module.HotStuffFollower, validator hotstuff.Validator, diff --git a/engine/testutil/nodes.go b/engine/testutil/nodes.go index 52df37cac97..eb85762c0e4 100644 --- a/engine/testutil/nodes.go +++ b/engine/testutil/nodes.go @@ -853,7 +853,7 @@ func createFollowerCore( t *testing.T, node *testmock.GenericNode, followerState *badgerstate.FollowerState, - notifier hotstuff.ConsensusFollowerConsumer, + notifier hotstuff.FollowerConsumer, rootHead *flow.Header, rootQC *flow.QuorumCertificate, ) (module.HotStuffFollower, *confinalizer.Finalizer) { diff --git a/follower/follower_builder.go b/follower/follower_builder.go index 53b5adf804c..0b00a1c9e91 100644 --- a/follower/follower_builder.go +++ b/follower/follower_builder.go @@ -357,7 +357,7 @@ func FlowConsensusFollowerService(opts ...FollowerOption) *FollowerServiceBuilde FlowNodeBuilder: cmd.FlowNode(flow.RoleAccess.String(), config.baseOptions...), FollowerDistributor: pubsub.NewFollowerDistributor(), } - ret.FollowerDistributor.AddConsumer(notifications.NewSlashingViolationsConsumer(ret.Logger)) + ret.FollowerDistributor.AddFollowerConsumer(notifications.NewSlashingViolationsConsumer(ret.Logger)) // the observer gets a version of the root snapshot file that does not contain any node addresses // hence skip all the root snapshot validations that involved an identity address ret.FlowNodeBuilder.SkipNwAddressBasedValidations = true From e302f58c0197781c4cf08334809be6544ef61488 Mon Sep 17 00:00:00 2001 From: Alexander Hentschel Date: Sat, 6 May 2023 14:47:39 -0700 Subject: [PATCH 16/38] =?UTF-8?q?Revert=20"=E2=80=A2=20modularized=20consu?= =?UTF-8?q?mer=20interfaces=20for=20hotstuff=20notifications"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit 6c926bcfb98aac85bd0b30b423254fd35bdc1357. --- .../node_builder/access_node_builder.go | 2 +- cmd/collection/main.go | 2 +- cmd/consensus/main.go | 2 +- cmd/execution_builder.go | 4 +- cmd/observer/node_builder/observer_builder.go | 2 +- cmd/verification_builder.go | 2 +- consensus/follower.go | 2 +- consensus/hotstuff/consumer.go | 101 ++++--- consensus/hotstuff/forks/forks2.go | 4 +- .../hotstuff/integration/instance_test.go | 4 +- .../hotstuff/notifications/noop_consumer.go | 21 +- .../pubsub/communicator_distributor.go | 56 ---- .../notifications/pubsub/distributor.go | 246 ++++++++++++++++-- .../pubsub/finalization_distributor.go | 68 ----- .../pubsub/follower_distributor.go | 85 ++++++ .../pubsub/participant_distributor.go | 143 ---------- .../pubsub/protocol_violation_distributor.go | 85 ------ .../pubsub/qc_created_distributor.go | 2 +- .../pubsub/timeout_collector_distributor.go | 10 +- .../slashing_violation_consumer.go | 25 +- consensus/integration/nodes_test.go | 2 +- consensus/participant.go | 6 +- engine/common/follower/compliance_core.go | 2 +- engine/testutil/nodes.go | 2 +- follower/follower_builder.go | 2 +- 25 files changed, 397 insertions(+), 483 deletions(-) delete mode 100644 consensus/hotstuff/notifications/pubsub/communicator_distributor.go delete mode 100644 consensus/hotstuff/notifications/pubsub/finalization_distributor.go create mode 100644 consensus/hotstuff/notifications/pubsub/follower_distributor.go delete mode 100644 consensus/hotstuff/notifications/pubsub/participant_distributor.go delete mode 100644 consensus/hotstuff/notifications/pubsub/protocol_violation_distributor.go diff --git a/cmd/access/node_builder/access_node_builder.go b/cmd/access/node_builder/access_node_builder.go index fffdde44a4b..22f133dc630 100644 --- a/cmd/access/node_builder/access_node_builder.go +++ b/cmd/access/node_builder/access_node_builder.go @@ -609,7 +609,7 @@ func (builder *FlowAccessNodeBuilder) BuildExecutionDataRequester() *FlowAccessN func FlowAccessNode(nodeBuilder *cmd.FlowNodeBuilder) *FlowAccessNodeBuilder { dist := consensuspubsub.NewFollowerDistributor() - dist.AddFollowerConsumer(notifications.NewSlashingViolationsConsumer(nodeBuilder.Logger)) + dist.AddConsumer(notifications.NewSlashingViolationsConsumer(nodeBuilder.Logger)) return &FlowAccessNodeBuilder{ AccessNodeConfig: DefaultAccessNodeConfig(), FlowNodeBuilder: nodeBuilder, diff --git a/cmd/collection/main.go b/cmd/collection/main.go index 0d87a13141f..98921e0ec5b 100644 --- a/cmd/collection/main.go +++ b/cmd/collection/main.go @@ -173,7 +173,7 @@ func main() { PreInit(cmd.DynamicStartPreInit). Module("follower distributor", func(node *cmd.NodeConfig) error { followerDistributor = pubsub.NewFollowerDistributor() - followerDistributor.AddFollowerConsumer(notifications.NewSlashingViolationsConsumer(node.Logger)) + followerDistributor.AddConsumer(notifications.NewSlashingViolationsConsumer(node.Logger)) return nil }). Module("mutable follower state", func(node *cmd.NodeConfig) error { diff --git a/cmd/consensus/main.go b/cmd/consensus/main.go index fae7ba475a6..280ecd8be48 100644 --- a/cmd/consensus/main.go +++ b/cmd/consensus/main.go @@ -366,7 +366,7 @@ func main() { }). Module("follower distributor", func(node *cmd.NodeConfig) error { followerDistributor = pubsub.NewFollowerDistributor() - followerDistributor.AddProtocolViolationConsumer(notifications.NewSlashingViolationsConsumer(nodeBuilder.Logger)) + followerDistributor.AddConsumer(notifications.NewSlashingViolationsConsumer(nodeBuilder.Logger)) return nil }). Module("machine account config", func(node *cmd.NodeConfig) error { diff --git a/cmd/execution_builder.go b/cmd/execution_builder.go index 0bfd0ad10a0..a2660ccc353 100644 --- a/cmd/execution_builder.go +++ b/cmd/execution_builder.go @@ -274,7 +274,7 @@ func (exeNode *ExecutionNode) LoadExecutionReceiptsStorage( func (exeNode *ExecutionNode) LoadFollowerDistributor(node *NodeConfig) error { exeNode.followerDistributor = pubsub.NewFollowerDistributor() - exeNode.followerDistributor.AddProtocolViolationConsumer(notifications.NewSlashingViolationsConsumer(node.Logger)) + exeNode.followerDistributor.AddConsumer(notifications.NewSlashingViolationsConsumer(node.Logger)) return nil } @@ -854,7 +854,7 @@ func (exeNode *ExecutionNode) LoadFollowerCore( return nil, fmt.Errorf("could not find latest finalized block and pending blocks to recover consensus follower: %w", err) } - exeNode.followerDistributor.AddFollowerConsumer(exeNode.checkerEng) + exeNode.followerDistributor.AddConsumer(exeNode.checkerEng) // creates a consensus follower with ingestEngine as the notifier // so that it gets notified upon each new finalized block diff --git a/cmd/observer/node_builder/observer_builder.go b/cmd/observer/node_builder/observer_builder.go index 508af73d311..4074c9f244a 100644 --- a/cmd/observer/node_builder/observer_builder.go +++ b/cmd/observer/node_builder/observer_builder.go @@ -566,7 +566,7 @@ func NewFlowObserverServiceBuilder(opts ...Option) *ObserverServiceBuilder { FlowNodeBuilder: cmd.FlowNode("observer"), FollowerDistributor: pubsub.NewFollowerDistributor(), } - anb.FollowerDistributor.AddFollowerConsumer(notifications.NewSlashingViolationsConsumer(anb.Logger)) + anb.FollowerDistributor.AddConsumer(notifications.NewSlashingViolationsConsumer(anb.Logger)) // the observer gets a version of the root snapshot file that does not contain any node addresses // hence skip all the root snapshot validations that involved an identity address anb.FlowNodeBuilder.SkipNwAddressBasedValidations = true diff --git a/cmd/verification_builder.go b/cmd/verification_builder.go index d7f2f196d16..9924c4884c7 100644 --- a/cmd/verification_builder.go +++ b/cmd/verification_builder.go @@ -179,7 +179,7 @@ func (v *VerificationNodeBuilder) LoadComponentsAndModules() { }). Module("follower distributor", func(node *NodeConfig) error { followerDistributor = pubsub.NewFollowerDistributor() - followerDistributor.AddFollowerConsumer(notifications.NewSlashingViolationsConsumer(node.Logger)) + followerDistributor.AddConsumer(notifications.NewSlashingViolationsConsumer(node.Logger)) return nil }). Module("sync core", func(node *NodeConfig) error { diff --git a/consensus/follower.go b/consensus/follower.go index d155948833b..1fa432ef3d9 100644 --- a/consensus/follower.go +++ b/consensus/follower.go @@ -25,7 +25,7 @@ import ( func NewFollower(log zerolog.Logger, headers storage.Headers, updater module.Finalizer, - notifier hotstuff.FollowerConsumer, + notifier hotstuff.ConsensusFollowerConsumer, rootHeader *flow.Header, rootQC *flow.QuorumCertificate, finalized *flow.Header, diff --git a/consensus/hotstuff/consumer.go b/consensus/hotstuff/consumer.go index 43ab4dc012e..2c144fe103e 100644 --- a/consensus/hotstuff/consumer.go +++ b/consensus/hotstuff/consumer.go @@ -7,9 +7,9 @@ import ( "github.com/onflow/flow-go/model/flow" ) -// ProtocolViolationConsumer consumes outbound notifications about HotStuff-protocol violations. -// Such notifications are produced by the active consensus participants and to a lesser -// degree also the consensus follower. +// ProtocolViolationConsumer consumes outbound notifications produced by compliance. +// Notifications can be produced by consensus participants and followers. +// Notifications are meant to report protocol violations that can be observed by executing compliance checks. // // Implementations must: // - be concurrency safe @@ -23,7 +23,6 @@ type ProtocolViolationConsumer interface { // Implementation must be concurrency safe; Non-blocking; // and must handle repetition of the same events (with some processing overhead). OnInvalidBlockDetected(err model.InvalidBlockError) - // OnDoubleProposeDetected notifications are produced by the Finalization Logic // whenever a double block proposal (equivocation) was detected. // Equivocation occurs when the same leader proposes two different blocks for the same view. @@ -31,47 +30,12 @@ type ProtocolViolationConsumer interface { // Implementation must be concurrency safe; Non-blocking; // and must handle repetition of the same events (with some processing overhead). OnDoubleProposeDetected(*model.Block, *model.Block) - - // OnDoubleVotingDetected notifications are produced by the Vote Aggregation logic - // whenever a double voting (same voter voting for different blocks at the same view) was detected. - // Prerequisites: - // Implementation must be concurrency safe; Non-blocking; - // and must handle repetition of the same events (with some processing overhead). - OnDoubleVotingDetected(*model.Vote, *model.Vote) - - // OnInvalidVoteDetected notifications are produced by the Vote Aggregation logic - // whenever an invalid vote was detected. - // Prerequisites: - // Implementation must be concurrency safe; Non-blocking; - // and must handle repetition of the same events (with some processing overhead). - OnInvalidVoteDetected(err model.InvalidVoteError) - - // OnVoteForInvalidBlockDetected notifications are produced by the Vote Aggregation logic - // whenever vote for invalid proposal was detected. - // Prerequisites: - // Implementation must be concurrency safe; Non-blocking; - // and must handle repetition of the same events (with some processing overhead). - OnVoteForInvalidBlockDetected(vote *model.Vote, invalidProposal *model.Proposal) - - // OnDoubleTimeoutDetected notifications are produced by the Timeout Aggregation logic - // whenever a double timeout (same replica producing two different timeouts at the same view) was detected. - // Prerequisites: - // Implementation must be concurrency safe; Non-blocking; - // and must handle repetition of the same events (with some processing overhead). - OnDoubleTimeoutDetected(*model.TimeoutObject, *model.TimeoutObject) - - // OnInvalidTimeoutDetected notifications are produced by the Timeout Aggregation logic - // whenever an invalid timeout was detected. - // Prerequisites: - // Implementation must be concurrency safe; Non-blocking; - // and must handle repetition of the same events (with some processing overhead). - OnInvalidTimeoutDetected(err model.InvalidTimeoutError) } -// FinalizationConsumer consumes outbound notifications produced by the logic tracking -// forks and finalization. Such notifications are produced by the active consensus -// participants, and generally potentially relevant to the larger node. The notifications -// are emitted in the order in which the finalization algorithm makes the respective steps. +// FinalizationConsumer consumes outbound notifications produced by the finalization logic. +// Notifications represent finalization-specific state changes which are potentially relevant +// to the larger node. The notifications are emitted in the order in which the +// finalization algorithm makes the respective steps. // // Implementations must: // - be concurrency safe @@ -93,13 +57,13 @@ type FinalizationConsumer interface { OnFinalizedBlock(*model.Block) } -// FollowerConsumer consumes outbound notifications produced by consensus followers. +// ConsensusFollowerConsumer consumes outbound notifications produced by consensus followers. // It is a subset of the notifications produced by consensus participants. // Implementations must: // - be concurrency safe // - be non-blocking // - handle repetition of the same events (with some processing overhead). -type FollowerConsumer interface { +type ConsensusFollowerConsumer interface { ProtocolViolationConsumer FinalizationConsumer } @@ -114,19 +78,9 @@ type FollowerConsumer interface { // - be non-blocking // - handle repetition of the same events (with some processing overhead). type Consumer interface { - FollowerConsumer + ConsensusFollowerConsumer CommunicatorConsumer - ParticipantConsumer -} -// ParticipantConsumer consumes outbound notifications produced by consensus participants -// actively proposing blocks, voting, collecting & aggregating votes to QCs, and participating in -// the pacemaker (sending timeouts, collecting & aggregating timeouts to TCs). -// Implementations must: -// - be concurrency safe -// - be non-blocking -// - handle repetition of the same events (with some processing overhead). -type ParticipantConsumer interface { // OnEventProcessed notifications are produced by the EventHandler when it is done processing // and hands control back to the EventLoop to wait for the next event. // Prerequisites: @@ -236,6 +190,41 @@ type ParticipantConsumer interface { // Implementation must be concurrency safe; Non-blocking; // and must handle repetition of the same events (with some processing overhead). OnCurrentViewDetails(currentView, finalizedView uint64, currentLeader flow.Identifier) + + // OnDoubleVotingDetected notifications are produced by the Vote Aggregation logic + // whenever a double voting (same voter voting for different blocks at the same view) was detected. + // Prerequisites: + // Implementation must be concurrency safe; Non-blocking; + // and must handle repetition of the same events (with some processing overhead). + OnDoubleVotingDetected(*model.Vote, *model.Vote) + + // OnInvalidVoteDetected notifications are produced by the Vote Aggregation logic + // whenever an invalid vote was detected. + // Prerequisites: + // Implementation must be concurrency safe; Non-blocking; + // and must handle repetition of the same events (with some processing overhead). + OnInvalidVoteDetected(err model.InvalidVoteError) + + // OnVoteForInvalidBlockDetected notifications are produced by the Vote Aggregation logic + // whenever vote for invalid proposal was detected. + // Prerequisites: + // Implementation must be concurrency safe; Non-blocking; + // and must handle repetition of the same events (with some processing overhead). + OnVoteForInvalidBlockDetected(vote *model.Vote, invalidProposal *model.Proposal) + + // OnDoubleTimeoutDetected notifications are produced by the Timeout Aggregation logic + // whenever a double timeout (same replica producing two different timeouts at the same view) was detected. + // Prerequisites: + // Implementation must be concurrency safe; Non-blocking; + // and must handle repetition of the same events (with some processing overhead). + OnDoubleTimeoutDetected(*model.TimeoutObject, *model.TimeoutObject) + + // OnInvalidTimeoutDetected notifications are produced by the Timeout Aggregation logic + // whenever an invalid timeout was detected. + // Prerequisites: + // Implementation must be concurrency safe; Non-blocking; + // and must handle repetition of the same events (with some processing overhead). + OnInvalidTimeoutDetected(err model.InvalidTimeoutError) } // QCCreatedConsumer consumes outbound notifications produced by HotStuff and its components. diff --git a/consensus/hotstuff/forks/forks2.go b/consensus/hotstuff/forks/forks2.go index e4ebc59112b..e9aaacca4a9 100644 --- a/consensus/hotstuff/forks/forks2.go +++ b/consensus/hotstuff/forks/forks2.go @@ -19,7 +19,7 @@ import ( // Forks is NOT safe for concurrent use by multiple goroutines. type Forks struct { finalizationCallback module.Finalizer - notifier hotstuff.FollowerConsumer + notifier hotstuff.ConsensusFollowerConsumer forest forest.LevelledForest trustedRoot *model.CertifiedBlock @@ -30,7 +30,7 @@ type Forks struct { var _ hotstuff.Forks = (*Forks)(nil) -func New(trustedRoot *model.CertifiedBlock, finalizationCallback module.Finalizer, notifier hotstuff.FollowerConsumer) (*Forks, error) { +func New(trustedRoot *model.CertifiedBlock, finalizationCallback module.Finalizer, notifier hotstuff.ConsensusFollowerConsumer) (*Forks, error) { if (trustedRoot.Block.BlockID != trustedRoot.CertifyingQC.BlockID) || (trustedRoot.Block.View != trustedRoot.CertifyingQC.View) { return nil, model.NewConfigurationErrorf("invalid root: root QC is not pointing to root block") } diff --git a/consensus/hotstuff/integration/instance_test.go b/consensus/hotstuff/integration/instance_test.go index a082e2becbb..e981f335329 100644 --- a/consensus/hotstuff/integration/instance_test.go +++ b/consensus/hotstuff/integration/instance_test.go @@ -539,8 +539,8 @@ func NewInstance(t *testing.T, options ...Option) *Instance { ) require.NoError(t, err) - collectorDistributor.AddTimeoutCollectorConsumer(logConsumer) - collectorDistributor.AddTimeoutCollectorConsumer(&in) + collectorDistributor.AddConsumer(logConsumer) + collectorDistributor.AddConsumer(&in) return &in } diff --git a/consensus/hotstuff/notifications/noop_consumer.go b/consensus/hotstuff/notifications/noop_consumer.go index c2f02acbcae..f3babd8f81a 100644 --- a/consensus/hotstuff/notifications/noop_consumer.go +++ b/consensus/hotstuff/notifications/noop_consumer.go @@ -56,6 +56,16 @@ func (*NoopPartialConsumer) OnTimeoutProcessed(*model.TimeoutObject) {} func (*NoopPartialConsumer) OnCurrentViewDetails(uint64, uint64, flow.Identifier) {} +func (*NoopPartialConsumer) OnDoubleVotingDetected(*model.Vote, *model.Vote) {} + +func (*NoopPartialConsumer) OnInvalidVoteDetected(model.InvalidVoteError) {} + +func (*NoopPartialConsumer) OnVoteForInvalidBlockDetected(*model.Vote, *model.Proposal) {} + +func (*NoopPartialConsumer) OnDoubleTimeoutDetected(*model.TimeoutObject, *model.TimeoutObject) {} + +func (*NoopPartialConsumer) OnInvalidTimeoutDetected(model.InvalidTimeoutError) {} + // no-op implementation of hotstuff.FinalizationConsumer type NoopFinalizationConsumer struct{} @@ -110,14 +120,3 @@ var _ hotstuff.ProtocolViolationConsumer = (*NoopProtocolViolationConsumer)(nil) func (*NoopProtocolViolationConsumer) OnInvalidBlockDetected(model.InvalidBlockError) {} func (*NoopProtocolViolationConsumer) OnDoubleProposeDetected(*model.Block, *model.Block) {} - -func (*NoopProtocolViolationConsumer) OnDoubleVotingDetected(*model.Vote, *model.Vote) {} - -func (*NoopProtocolViolationConsumer) OnInvalidVoteDetected(model.InvalidVoteError) {} - -func (*NoopProtocolViolationConsumer) OnVoteForInvalidBlockDetected(*model.Vote, *model.Proposal) {} - -func (*NoopProtocolViolationConsumer) OnDoubleTimeoutDetected(*model.TimeoutObject, *model.TimeoutObject) { -} - -func (*NoopProtocolViolationConsumer) OnInvalidTimeoutDetected(model.InvalidTimeoutError) {} diff --git a/consensus/hotstuff/notifications/pubsub/communicator_distributor.go b/consensus/hotstuff/notifications/pubsub/communicator_distributor.go deleted file mode 100644 index 521e06ee50b..00000000000 --- a/consensus/hotstuff/notifications/pubsub/communicator_distributor.go +++ /dev/null @@ -1,56 +0,0 @@ -package pubsub - -import ( - "sync" - "time" - - "github.com/onflow/flow-go/consensus/hotstuff" - "github.com/onflow/flow-go/consensus/hotstuff/model" - "github.com/onflow/flow-go/model/flow" -) - -// CommunicatorDistributor ingests outbound consensus messages from HotStuff's core logic and -// distributes them to subscribers. This logic only runs inside active consensus participants proposing -// blocks, voting, collecting + aggregating votes to QCs, and participating in the pacemaker (sending -// timeouts, collecting + aggregating timeouts to TCs). -// Concurrently safe. -type CommunicatorDistributor struct { - subscribers []hotstuff.CommunicatorConsumer - lock sync.RWMutex -} - -var _ hotstuff.CommunicatorConsumer = (*CommunicatorDistributor)(nil) - -func NewCommunicatorConsumerDistributor() *CommunicatorDistributor { - return &CommunicatorDistributor{} -} - -func (d *CommunicatorDistributor) AddCommunicatorConsumer(consumer hotstuff.CommunicatorConsumer) { - d.lock.Lock() - defer d.lock.Unlock() - d.subscribers = append(d.subscribers, consumer) -} - -func (d *CommunicatorDistributor) OnOwnVote(blockID flow.Identifier, view uint64, sigData []byte, recipientID flow.Identifier) { - d.lock.RLock() - defer d.lock.RUnlock() - for _, s := range d.subscribers { - s.OnOwnVote(blockID, view, sigData, recipientID) - } -} - -func (d *CommunicatorDistributor) OnOwnTimeout(timeout *model.TimeoutObject) { - d.lock.RLock() - defer d.lock.RUnlock() - for _, s := range d.subscribers { - s.OnOwnTimeout(timeout) - } -} - -func (d *CommunicatorDistributor) OnOwnProposal(proposal *flow.Header, targetPublicationTime time.Time) { - d.lock.RLock() - defer d.lock.RUnlock() - for _, s := range d.subscribers { - s.OnOwnProposal(proposal, targetPublicationTime) - } -} diff --git a/consensus/hotstuff/notifications/pubsub/distributor.go b/consensus/hotstuff/notifications/pubsub/distributor.go index 7c600a71e88..74674ee8547 100644 --- a/consensus/hotstuff/notifications/pubsub/distributor.go +++ b/consensus/hotstuff/notifications/pubsub/distributor.go @@ -1,44 +1,258 @@ package pubsub import ( + "sync" + "time" + "github.com/onflow/flow-go/consensus/hotstuff" + "github.com/onflow/flow-go/consensus/hotstuff/model" + "github.com/onflow/flow-go/consensus/hotstuff/notifications" + "github.com/onflow/flow-go/model/flow" ) // Distributor distributes notifications to a list of subscribers (event consumers). // // It allows thread-safe subscription of multiple consumers to events. type Distributor struct { - FollowerDistributor - CommunicatorDistributor - ParticipantDistributor + subscribers []hotstuff.Consumer + lock sync.RWMutex } var _ hotstuff.Consumer = (*Distributor)(nil) +func (p *Distributor) OnEventProcessed() { + p.lock.RLock() + defer p.lock.RUnlock() + for _, subscriber := range p.subscribers { + subscriber.OnEventProcessed() + } +} + func NewDistributor() *Distributor { return &Distributor{} } // AddConsumer adds an event consumer to the Distributor func (p *Distributor) AddConsumer(consumer hotstuff.Consumer) { - p.FollowerDistributor.AddFollowerConsumer(consumer) - p.CommunicatorDistributor.AddCommunicatorConsumer(consumer) - p.ParticipantDistributor.AddParticipantConsumer(consumer) + p.lock.Lock() + defer p.lock.Unlock() + p.subscribers = append(p.subscribers, consumer) +} + +// AddFollowerConsumer registers the input `consumer` to be notified on ConsensusFollowerConsumer events. +func (p *Distributor) AddFollowerConsumer(consumer hotstuff.ConsensusFollowerConsumer) { + p.lock.Lock() + defer p.lock.Unlock() + + var wrappedConsumer hotstuff.Consumer = &struct { + notifications.NoopCommunicatorConsumer + notifications.NoopPartialConsumer + hotstuff.ConsensusFollowerConsumer + }{ + notifications.NoopCommunicatorConsumer{}, + notifications.NoopPartialConsumer{}, + consumer, + } + + p.subscribers = append(p.subscribers, wrappedConsumer) +} + +func (p *Distributor) OnStart(currentView uint64) { + p.lock.RLock() + defer p.lock.RUnlock() + for _, subscriber := range p.subscribers { + subscriber.OnStart(currentView) + } +} + +func (p *Distributor) OnReceiveProposal(currentView uint64, proposal *model.Proposal) { + p.lock.RLock() + defer p.lock.RUnlock() + for _, subscriber := range p.subscribers { + subscriber.OnReceiveProposal(currentView, proposal) + } +} + +func (p *Distributor) OnReceiveQc(currentView uint64, qc *flow.QuorumCertificate) { + p.lock.RLock() + defer p.lock.RUnlock() + for _, subscriber := range p.subscribers { + subscriber.OnReceiveQc(currentView, qc) + } +} + +func (p *Distributor) OnReceiveTc(currentView uint64, tc *flow.TimeoutCertificate) { + p.lock.RLock() + defer p.lock.RUnlock() + for _, subscriber := range p.subscribers { + subscriber.OnReceiveTc(currentView, tc) + } +} + +func (p *Distributor) OnPartialTc(currentView uint64, partialTc *hotstuff.PartialTcCreated) { + p.lock.RLock() + defer p.lock.RUnlock() + for _, subscriber := range p.subscribers { + subscriber.OnPartialTc(currentView, partialTc) + } +} + +func (p *Distributor) OnLocalTimeout(currentView uint64) { + p.lock.RLock() + defer p.lock.RUnlock() + for _, subscriber := range p.subscribers { + subscriber.OnLocalTimeout(currentView) + } +} + +func (p *Distributor) OnViewChange(oldView, newView uint64) { + p.lock.RLock() + defer p.lock.RUnlock() + for _, subscriber := range p.subscribers { + subscriber.OnViewChange(oldView, newView) + } +} + +func (p *Distributor) OnQcTriggeredViewChange(oldView uint64, newView uint64, qc *flow.QuorumCertificate) { + p.lock.RLock() + defer p.lock.RUnlock() + for _, subscriber := range p.subscribers { + subscriber.OnQcTriggeredViewChange(oldView, newView, qc) + } +} + +func (p *Distributor) OnTcTriggeredViewChange(oldView uint64, newView uint64, tc *flow.TimeoutCertificate) { + p.lock.RLock() + defer p.lock.RUnlock() + for _, subscriber := range p.subscribers { + subscriber.OnTcTriggeredViewChange(oldView, newView, tc) + } } -// FollowerDistributor ingests consensus follower events and distributes it to subscribers. -type FollowerDistributor struct { - ProtocolViolationDistributor - FinalizationDistributor +func (p *Distributor) OnStartingTimeout(timerInfo model.TimerInfo) { + p.lock.RLock() + defer p.lock.RUnlock() + for _, subscriber := range p.subscribers { + subscriber.OnStartingTimeout(timerInfo) + } } -var _ hotstuff.FollowerConsumer = (*FollowerDistributor)(nil) +func (p *Distributor) OnVoteProcessed(vote *model.Vote) { + p.lock.RLock() + defer p.lock.RUnlock() + for _, subscriber := range p.subscribers { + subscriber.OnVoteProcessed(vote) + } +} + +func (p *Distributor) OnTimeoutProcessed(timeout *model.TimeoutObject) { + p.lock.RLock() + defer p.lock.RUnlock() + for _, subscriber := range p.subscribers { + subscriber.OnTimeoutProcessed(timeout) + } +} + +func (p *Distributor) OnCurrentViewDetails(currentView, finalizedView uint64, currentLeader flow.Identifier) { + p.lock.RLock() + defer p.lock.RUnlock() + for _, subscriber := range p.subscribers { + subscriber.OnCurrentViewDetails(currentView, finalizedView, currentLeader) + } +} + +func (p *Distributor) OnBlockIncorporated(block *model.Block) { + p.lock.RLock() + defer p.lock.RUnlock() + for _, subscriber := range p.subscribers { + subscriber.OnBlockIncorporated(block) + } +} + +func (p *Distributor) OnFinalizedBlock(block *model.Block) { + p.lock.RLock() + defer p.lock.RUnlock() + for _, subscriber := range p.subscribers { + subscriber.OnFinalizedBlock(block) + } +} + +func (p *Distributor) OnInvalidBlockDetected(err model.InvalidBlockError) { + p.lock.RLock() + defer p.lock.RUnlock() + for _, subscriber := range p.subscribers { + subscriber.OnInvalidBlockDetected(err) + } +} + +func (p *Distributor) OnDoubleProposeDetected(block1, block2 *model.Block) { + p.lock.RLock() + defer p.lock.RUnlock() + for _, subscriber := range p.subscribers { + subscriber.OnDoubleProposeDetected(block1, block2) + } +} + +func (p *Distributor) OnDoubleVotingDetected(vote1, vote2 *model.Vote) { + p.lock.RLock() + defer p.lock.RUnlock() + for _, subscriber := range p.subscribers { + subscriber.OnDoubleVotingDetected(vote1, vote2) + } +} + +func (p *Distributor) OnInvalidVoteDetected(err model.InvalidVoteError) { + p.lock.RLock() + defer p.lock.RUnlock() + for _, subscriber := range p.subscribers { + subscriber.OnInvalidVoteDetected(err) + } +} + +func (p *Distributor) OnVoteForInvalidBlockDetected(vote *model.Vote, invalidProposal *model.Proposal) { + p.lock.RLock() + defer p.lock.RUnlock() + for _, subscriber := range p.subscribers { + subscriber.OnVoteForInvalidBlockDetected(vote, invalidProposal) + } +} + +func (p *Distributor) OnDoubleTimeoutDetected(timeout *model.TimeoutObject, altTimeout *model.TimeoutObject) { + p.lock.RLock() + defer p.lock.RUnlock() + for _, subscriber := range p.subscribers { + subscriber.OnDoubleTimeoutDetected(timeout, altTimeout) + } +} + +func (p *Distributor) OnInvalidTimeoutDetected(err model.InvalidTimeoutError) { + p.lock.RLock() + defer p.lock.RUnlock() + for _, subscriber := range p.subscribers { + subscriber.OnInvalidTimeoutDetected(err) + } +} + +func (p *Distributor) OnOwnVote(blockID flow.Identifier, view uint64, sigData []byte, recipientID flow.Identifier) { + p.lock.RLock() + defer p.lock.RUnlock() + for _, s := range p.subscribers { + s.OnOwnVote(blockID, view, sigData, recipientID) + } +} -func NewFollowerDistributor() *FollowerDistributor { - return &FollowerDistributor{} +func (p *Distributor) OnOwnTimeout(timeout *model.TimeoutObject) { + p.lock.RLock() + defer p.lock.RUnlock() + for _, s := range p.subscribers { + s.OnOwnTimeout(timeout) + } } -func (d *FollowerDistributor) AddFollowerConsumer(consumer hotstuff.FollowerConsumer) { - d.FinalizationDistributor.AddFinalizationConsumer(consumer) - d.ProtocolViolationDistributor.AddProtocolViolationConsumer(consumer) +func (p *Distributor) OnOwnProposal(proposal *flow.Header, targetPublicationTime time.Time) { + p.lock.RLock() + defer p.lock.RUnlock() + for _, s := range p.subscribers { + s.OnOwnProposal(proposal, targetPublicationTime) + } } diff --git a/consensus/hotstuff/notifications/pubsub/finalization_distributor.go b/consensus/hotstuff/notifications/pubsub/finalization_distributor.go deleted file mode 100644 index a78dd88d13b..00000000000 --- a/consensus/hotstuff/notifications/pubsub/finalization_distributor.go +++ /dev/null @@ -1,68 +0,0 @@ -package pubsub - -import ( - "sync" - - "github.com/onflow/flow-go/consensus/hotstuff" - "github.com/onflow/flow-go/consensus/hotstuff/model" -) - -type OnBlockFinalizedConsumer = func(block *model.Block) -type OnBlockIncorporatedConsumer = func(block *model.Block) - -// FinalizationDistributor ingests events from HotStuff's logic for tracking forks + finalization -// and distributes them to subscribers. This logic generally runs inside all nodes (irrespectively whether -// they are active consensus participants or or only consensus followers). -// Concurrently safe. -type FinalizationDistributor struct { - blockFinalizedConsumers []OnBlockFinalizedConsumer - blockIncorporatedConsumers []OnBlockIncorporatedConsumer - consumers []hotstuff.FinalizationConsumer - lock sync.RWMutex -} - -var _ hotstuff.FinalizationConsumer = (*FinalizationDistributor)(nil) - -func NewFinalizationDistributor() *FinalizationDistributor { - return &FinalizationDistributor{} -} - -func (d *FinalizationDistributor) AddOnBlockFinalizedConsumer(consumer OnBlockFinalizedConsumer) { - d.lock.Lock() - defer d.lock.Unlock() - d.blockFinalizedConsumers = append(d.blockFinalizedConsumers, consumer) -} - -func (d *FinalizationDistributor) AddOnBlockIncorporatedConsumer(consumer OnBlockIncorporatedConsumer) { - d.lock.Lock() - defer d.lock.Unlock() - d.blockIncorporatedConsumers = append(d.blockIncorporatedConsumers, consumer) -} - -func (d *FinalizationDistributor) AddFinalizationConsumer(consumer hotstuff.FinalizationConsumer) { - d.lock.Lock() - defer d.lock.Unlock() - d.consumers = append(d.consumers, consumer) -} - -func (d *FinalizationDistributor) OnBlockIncorporated(block *model.Block) { - d.lock.RLock() - defer d.lock.RUnlock() - for _, consumer := range d.blockIncorporatedConsumers { - consumer(block) - } - for _, consumer := range d.consumers { - consumer.OnBlockIncorporated(block) - } -} - -func (d *FinalizationDistributor) OnFinalizedBlock(block *model.Block) { - d.lock.RLock() - defer d.lock.RUnlock() - for _, consumer := range d.blockFinalizedConsumers { - consumer(block) - } - for _, consumer := range d.consumers { - consumer.OnFinalizedBlock(block) - } -} diff --git a/consensus/hotstuff/notifications/pubsub/follower_distributor.go b/consensus/hotstuff/notifications/pubsub/follower_distributor.go new file mode 100644 index 00000000000..54ad77ac925 --- /dev/null +++ b/consensus/hotstuff/notifications/pubsub/follower_distributor.go @@ -0,0 +1,85 @@ +package pubsub + +import ( + "sync" + + "github.com/onflow/flow-go/consensus/hotstuff" + "github.com/onflow/flow-go/consensus/hotstuff/model" +) + +type OnBlockFinalizedConsumer = func(block *model.Block) +type OnBlockIncorporatedConsumer = func(block *model.Block) + +// FollowerDistributor ingests consensus follower events and distributes it to subscribers. +type FollowerDistributor struct { + blockFinalizedConsumers []OnBlockFinalizedConsumer + blockIncorporatedConsumers []OnBlockIncorporatedConsumer + followerConsumers []hotstuff.ConsensusFollowerConsumer + lock sync.RWMutex +} + +var _ hotstuff.ConsensusFollowerConsumer = (*FollowerDistributor)(nil) + +func NewFollowerDistributor() *FollowerDistributor { + return &FollowerDistributor{ + blockFinalizedConsumers: make([]OnBlockFinalizedConsumer, 0), + blockIncorporatedConsumers: make([]OnBlockIncorporatedConsumer, 0), + lock: sync.RWMutex{}, + } +} + +func (p *FollowerDistributor) AddOnBlockFinalizedConsumer(consumer OnBlockFinalizedConsumer) { + p.lock.Lock() + defer p.lock.Unlock() + p.blockFinalizedConsumers = append(p.blockFinalizedConsumers, consumer) +} + +func (p *FollowerDistributor) AddOnBlockIncorporatedConsumer(consumer OnBlockIncorporatedConsumer) { + p.lock.Lock() + defer p.lock.Unlock() + p.blockIncorporatedConsumers = append(p.blockIncorporatedConsumers, consumer) +} + +func (p *FollowerDistributor) AddConsumer(consumer hotstuff.ConsensusFollowerConsumer) { + p.lock.Lock() + defer p.lock.Unlock() + p.followerConsumers = append(p.followerConsumers, consumer) +} + +func (p *FollowerDistributor) OnBlockIncorporated(block *model.Block) { + p.lock.RLock() + defer p.lock.RUnlock() + for _, consumer := range p.blockIncorporatedConsumers { + consumer(block) + } + for _, consumer := range p.followerConsumers { + consumer.OnBlockIncorporated(block) + } +} + +func (p *FollowerDistributor) OnFinalizedBlock(block *model.Block) { + p.lock.RLock() + defer p.lock.RUnlock() + for _, consumer := range p.blockFinalizedConsumers { + consumer(block) + } + for _, consumer := range p.followerConsumers { + consumer.OnFinalizedBlock(block) + } +} + +func (p *FollowerDistributor) OnDoubleProposeDetected(block1, block2 *model.Block) { + p.lock.RLock() + defer p.lock.RUnlock() + for _, consumer := range p.followerConsumers { + consumer.OnDoubleProposeDetected(block1, block2) + } +} + +func (p *FollowerDistributor) OnInvalidBlockDetected(err model.InvalidBlockError) { + p.lock.RLock() + defer p.lock.RUnlock() + for _, consumer := range p.followerConsumers { + consumer.OnInvalidBlockDetected(err) + } +} diff --git a/consensus/hotstuff/notifications/pubsub/participant_distributor.go b/consensus/hotstuff/notifications/pubsub/participant_distributor.go deleted file mode 100644 index f0fae001a41..00000000000 --- a/consensus/hotstuff/notifications/pubsub/participant_distributor.go +++ /dev/null @@ -1,143 +0,0 @@ -package pubsub - -import ( - "sync" - - "github.com/onflow/flow-go/consensus/hotstuff" - "github.com/onflow/flow-go/consensus/hotstuff/model" - "github.com/onflow/flow-go/model/flow" -) - -// ParticipantDistributor ingests events from HotStuff's core logic and distributes them to -// subscribers. This logic only runs inside active consensus participants proposing blocks, voting, -// collecting + aggregating votes to QCs, and participating in the pacemaker (sending timeouts, -// collecting + aggregating timeouts to TCs). -// Concurrently safe. -type ParticipantDistributor struct { - subscribers []hotstuff.ParticipantConsumer - lock sync.RWMutex -} - -var _ hotstuff.ParticipantConsumer = (*ParticipantDistributor)(nil) - -func NewConsensusParticipantDistributor() *ParticipantDistributor { - return &ParticipantDistributor{} -} - -func (d *ParticipantDistributor) AddParticipantConsumer(consumer hotstuff.ParticipantConsumer) { - d.lock.Lock() - defer d.lock.Unlock() - d.subscribers = append(d.subscribers, consumer) -} - -func (d *ParticipantDistributor) OnEventProcessed() { - d.lock.RLock() - defer d.lock.RUnlock() - for _, subscriber := range d.subscribers { - subscriber.OnEventProcessed() - } -} - -func (d *ParticipantDistributor) OnStart(currentView uint64) { - d.lock.RLock() - defer d.lock.RUnlock() - for _, subscriber := range d.subscribers { - subscriber.OnStart(currentView) - } -} - -func (d *ParticipantDistributor) OnReceiveProposal(currentView uint64, proposal *model.Proposal) { - d.lock.RLock() - defer d.lock.RUnlock() - for _, subscriber := range d.subscribers { - subscriber.OnReceiveProposal(currentView, proposal) - } -} - -func (d *ParticipantDistributor) OnReceiveQc(currentView uint64, qc *flow.QuorumCertificate) { - d.lock.RLock() - defer d.lock.RUnlock() - for _, subscriber := range d.subscribers { - subscriber.OnReceiveQc(currentView, qc) - } -} - -func (d *ParticipantDistributor) OnReceiveTc(currentView uint64, tc *flow.TimeoutCertificate) { - d.lock.RLock() - defer d.lock.RUnlock() - for _, subscriber := range d.subscribers { - subscriber.OnReceiveTc(currentView, tc) - } -} - -func (d *ParticipantDistributor) OnPartialTc(currentView uint64, partialTc *hotstuff.PartialTcCreated) { - d.lock.RLock() - defer d.lock.RUnlock() - for _, subscriber := range d.subscribers { - subscriber.OnPartialTc(currentView, partialTc) - } -} - -func (d *ParticipantDistributor) OnLocalTimeout(currentView uint64) { - d.lock.RLock() - defer d.lock.RUnlock() - for _, subscriber := range d.subscribers { - subscriber.OnLocalTimeout(currentView) - } -} - -func (d *ParticipantDistributor) OnViewChange(oldView, newView uint64) { - d.lock.RLock() - defer d.lock.RUnlock() - for _, subscriber := range d.subscribers { - subscriber.OnViewChange(oldView, newView) - } -} - -func (d *ParticipantDistributor) OnQcTriggeredViewChange(oldView uint64, newView uint64, qc *flow.QuorumCertificate) { - d.lock.RLock() - defer d.lock.RUnlock() - for _, subscriber := range d.subscribers { - subscriber.OnQcTriggeredViewChange(oldView, newView, qc) - } -} - -func (d *ParticipantDistributor) OnTcTriggeredViewChange(oldView uint64, newView uint64, tc *flow.TimeoutCertificate) { - d.lock.RLock() - defer d.lock.RUnlock() - for _, subscriber := range d.subscribers { - subscriber.OnTcTriggeredViewChange(oldView, newView, tc) - } -} - -func (d *ParticipantDistributor) OnStartingTimeout(timerInfo model.TimerInfo) { - d.lock.RLock() - defer d.lock.RUnlock() - for _, subscriber := range d.subscribers { - subscriber.OnStartingTimeout(timerInfo) - } -} - -func (d *ParticipantDistributor) OnVoteProcessed(vote *model.Vote) { - d.lock.RLock() - defer d.lock.RUnlock() - for _, subscriber := range d.subscribers { - subscriber.OnVoteProcessed(vote) - } -} - -func (d *ParticipantDistributor) OnTimeoutProcessed(timeout *model.TimeoutObject) { - d.lock.RLock() - defer d.lock.RUnlock() - for _, subscriber := range d.subscribers { - subscriber.OnTimeoutProcessed(timeout) - } -} - -func (d *ParticipantDistributor) OnCurrentViewDetails(currentView, finalizedView uint64, currentLeader flow.Identifier) { - d.lock.RLock() - defer d.lock.RUnlock() - for _, subscriber := range d.subscribers { - subscriber.OnCurrentViewDetails(currentView, finalizedView, currentLeader) - } -} diff --git a/consensus/hotstuff/notifications/pubsub/protocol_violation_distributor.go b/consensus/hotstuff/notifications/pubsub/protocol_violation_distributor.go deleted file mode 100644 index 4c08dacddac..00000000000 --- a/consensus/hotstuff/notifications/pubsub/protocol_violation_distributor.go +++ /dev/null @@ -1,85 +0,0 @@ -package pubsub - -import ( - "sync" - - "github.com/onflow/flow-go/consensus/hotstuff" - "github.com/onflow/flow-go/consensus/hotstuff/model" -) - -// ProtocolViolationDistributor ingests notifications about HotStuff-protocol violations and -// distributes them to subscribers. Such notifications are produced by the active consensus -// participants and to a lesser degree also the consensus follower. -// Concurrently safe. -type ProtocolViolationDistributor struct { - subscribers []hotstuff.ProtocolViolationConsumer - lock sync.RWMutex -} - -var _ hotstuff.ProtocolViolationConsumer = (*ProtocolViolationDistributor)(nil) - -func NewProtocolViolationDistributor() *ProtocolViolationDistributor { - return &ProtocolViolationDistributor{} -} - -func (d *ProtocolViolationDistributor) AddProtocolViolationConsumer(consumer hotstuff.ProtocolViolationConsumer) { - d.lock.Lock() - defer d.lock.Unlock() - d.subscribers = append(d.subscribers, consumer) -} - -func (d *ProtocolViolationDistributor) OnInvalidBlockDetected(err model.InvalidBlockError) { - d.lock.RLock() - defer d.lock.RUnlock() - for _, subscriber := range d.subscribers { - subscriber.OnInvalidBlockDetected(err) - } -} - -func (d *ProtocolViolationDistributor) OnDoubleProposeDetected(block1, block2 *model.Block) { - d.lock.RLock() - defer d.lock.RUnlock() - for _, subscriber := range d.subscribers { - subscriber.OnDoubleProposeDetected(block1, block2) - } -} - -func (d *ProtocolViolationDistributor) OnDoubleVotingDetected(vote1, vote2 *model.Vote) { - d.lock.RLock() - defer d.lock.RUnlock() - for _, subscriber := range d.subscribers { - subscriber.OnDoubleVotingDetected(vote1, vote2) - } -} - -func (d *ProtocolViolationDistributor) OnInvalidVoteDetected(err model.InvalidVoteError) { - d.lock.RLock() - defer d.lock.RUnlock() - for _, subscriber := range d.subscribers { - subscriber.OnInvalidVoteDetected(err) - } -} - -func (d *ProtocolViolationDistributor) OnVoteForInvalidBlockDetected(vote *model.Vote, invalidProposal *model.Proposal) { - d.lock.RLock() - defer d.lock.RUnlock() - for _, subscriber := range d.subscribers { - subscriber.OnVoteForInvalidBlockDetected(vote, invalidProposal) - } -} - -func (d *ProtocolViolationDistributor) OnDoubleTimeoutDetected(timeout *model.TimeoutObject, altTimeout *model.TimeoutObject) { - d.lock.RLock() - defer d.lock.RUnlock() - for _, subscriber := range d.subscribers { - subscriber.OnDoubleTimeoutDetected(timeout, altTimeout) - } -} - -func (d *ProtocolViolationDistributor) OnInvalidTimeoutDetected(err model.InvalidTimeoutError) { - d.lock.RLock() - defer d.lock.RUnlock() - for _, subscriber := range d.subscribers { - subscriber.OnInvalidTimeoutDetected(err) - } -} diff --git a/consensus/hotstuff/notifications/pubsub/qc_created_distributor.go b/consensus/hotstuff/notifications/pubsub/qc_created_distributor.go index b9adafc2567..166fa9cf757 100644 --- a/consensus/hotstuff/notifications/pubsub/qc_created_distributor.go +++ b/consensus/hotstuff/notifications/pubsub/qc_created_distributor.go @@ -24,7 +24,7 @@ func NewQCCreatedDistributor() *QCCreatedDistributor { } } -func (d *QCCreatedDistributor) AddQCCreatedConsumer(consumer hotstuff.QCCreatedConsumer) { +func (d *QCCreatedDistributor) AddConsumer(consumer hotstuff.QCCreatedConsumer) { d.lock.Lock() defer d.lock.Unlock() d.qcCreatedConsumers = append(d.qcCreatedConsumers, consumer) diff --git a/consensus/hotstuff/notifications/pubsub/timeout_collector_distributor.go b/consensus/hotstuff/notifications/pubsub/timeout_collector_distributor.go index aa9c0bd9397..8387fb81663 100644 --- a/consensus/hotstuff/notifications/pubsub/timeout_collector_distributor.go +++ b/consensus/hotstuff/notifications/pubsub/timeout_collector_distributor.go @@ -7,8 +7,8 @@ import ( "github.com/onflow/flow-go/model/flow" ) -// TimeoutCollectorDistributor ingests events from hotstuff's VoteCollector -// logic and distributes them to subscribers. Concurrently safe. +// TimeoutCollectorDistributor ingests events from hotstuff and distributes them to subscribers. +// Concurrently safe // TODO: investigate if this can be updated using atomics to prevent locking on mutex since we always add all consumers // before delivering events. type TimeoutCollectorDistributor struct { @@ -19,10 +19,12 @@ type TimeoutCollectorDistributor struct { var _ hotstuff.TimeoutCollectorConsumer = (*TimeoutCollectorDistributor)(nil) func NewTimeoutCollectorDistributor() *TimeoutCollectorDistributor { - return &TimeoutCollectorDistributor{} + return &TimeoutCollectorDistributor{ + consumers: make([]hotstuff.TimeoutCollectorConsumer, 0), + } } -func (d *TimeoutCollectorDistributor) AddTimeoutCollectorConsumer(consumer hotstuff.TimeoutCollectorConsumer) { +func (d *TimeoutCollectorDistributor) AddConsumer(consumer hotstuff.TimeoutCollectorConsumer) { d.lock.Lock() defer d.lock.Unlock() d.consumers = append(d.consumers, consumer) diff --git a/consensus/hotstuff/notifications/slashing_violation_consumer.go b/consensus/hotstuff/notifications/slashing_violation_consumer.go index e67d87e3d48..fb80e15e522 100644 --- a/consensus/hotstuff/notifications/slashing_violation_consumer.go +++ b/consensus/hotstuff/notifications/slashing_violation_consumer.go @@ -3,7 +3,6 @@ package notifications import ( "github.com/rs/zerolog" - "github.com/onflow/flow-go/consensus/hotstuff" "github.com/onflow/flow-go/consensus/hotstuff/model" "github.com/onflow/flow-go/utils/logging" ) @@ -11,27 +10,15 @@ import ( // SlashingViolationsConsumer is an implementation of the notifications consumer that logs a // message for any slashable offenses. type SlashingViolationsConsumer struct { + NoopConsumer log zerolog.Logger } -var _ hotstuff.ProtocolViolationConsumer = (*SlashingViolationsConsumer)(nil) - func NewSlashingViolationsConsumer(log zerolog.Logger) *SlashingViolationsConsumer { return &SlashingViolationsConsumer{ log: log, } } -func (c *SlashingViolationsConsumer) OnInvalidBlockDetected(err model.InvalidBlockError) { - block := err.InvalidBlock.Block - c.log.Warn(). - Bool(logging.KeySuspicious, true). - Hex("proposer_id", block.ProposerID[:]). - Uint64("block_view", block.View). - Hex("block_id", block.BlockID[:]). - Hex("block_payloadhash", block.PayloadHash[:]). - Time("block_timestamp", block.Timestamp). - Msg("OnInvalidBlockDetected") -} func (c *SlashingViolationsConsumer) OnDoubleVotingDetected(vote1 *model.Vote, vote2 *model.Vote) { c.log.Warn(). @@ -54,16 +41,6 @@ func (c *SlashingViolationsConsumer) OnInvalidVoteDetected(err model.InvalidVote Msg("OnInvalidVoteDetected") } -func (c *SlashingViolationsConsumer) OnDoubleTimeoutDetected(timeout *model.TimeoutObject, altTimeout *model.TimeoutObject) { - c.log.Warn(). - Bool(logging.KeySuspicious, true). - Hex("timeout_creator", timeout.SignerID[:]). - Uint64("timeout_view", timeout.View). - Hex("timeout_id1", logging.ID(timeout.ID())). - Hex("timeout_id2", logging.ID(altTimeout.ID())). - Msg("OnDoubleTimeoutDetected") -} - func (c *SlashingViolationsConsumer) OnInvalidTimeoutDetected(err model.InvalidTimeoutError) { timeout := err.Timeout c.log.Warn(). diff --git a/consensus/integration/nodes_test.go b/consensus/integration/nodes_test.go index 6911118e877..817fc46fcba 100644 --- a/consensus/integration/nodes_test.go +++ b/consensus/integration/nodes_test.go @@ -531,7 +531,7 @@ func createNode( require.NoError(t, err) timeoutCollectorDistributor := pubsub.NewTimeoutCollectorDistributor() - timeoutCollectorDistributor.AddTimeoutCollectorConsumer(logConsumer) + timeoutCollectorDistributor.AddConsumer(logConsumer) timeoutProcessorFactory := timeoutcollector.NewTimeoutProcessorFactory( log, diff --git a/consensus/participant.go b/consensus/participant.go index e6435c70b97..50e17b14b47 100644 --- a/consensus/participant.go +++ b/consensus/participant.go @@ -122,8 +122,8 @@ func NewParticipant( } // add observer, event loop needs to receive events from distributor - modules.QCCreatedDistributor.AddQCCreatedConsumer(loop) - modules.TimeoutCollectorDistributor.AddTimeoutCollectorConsumer(loop) + modules.QCCreatedDistributor.AddConsumer(loop) + modules.TimeoutCollectorDistributor.AddConsumer(loop) return loop, nil } @@ -139,7 +139,7 @@ func NewValidator(metrics module.HotstuffMetrics, committee hotstuff.DynamicComm } // NewForks recovers trusted root and creates new forks manager -func NewForks(final *flow.Header, headers storage.Headers, updater module.Finalizer, notifier hotstuff.FollowerConsumer, rootHeader *flow.Header, rootQC *flow.QuorumCertificate) (*forks.Forks, error) { +func NewForks(final *flow.Header, headers storage.Headers, updater module.Finalizer, notifier hotstuff.ConsensusFollowerConsumer, rootHeader *flow.Header, rootQC *flow.QuorumCertificate) (*forks.Forks, error) { // recover the trusted root trustedRoot, err := recoverTrustedRoot(final, headers, rootHeader, rootQC) if err != nil { diff --git a/engine/common/follower/compliance_core.go b/engine/common/follower/compliance_core.go index ed0f43abd12..92eec6d43ff 100644 --- a/engine/common/follower/compliance_core.go +++ b/engine/common/follower/compliance_core.go @@ -59,7 +59,7 @@ var _ complianceCore = (*ComplianceCore)(nil) func NewComplianceCore(log zerolog.Logger, mempoolMetrics module.MempoolMetrics, heroCacheCollector module.HeroCacheMetrics, - followerConsumer hotstuff.FollowerConsumer, + followerConsumer hotstuff.ConsensusFollowerConsumer, state protocol.FollowerState, follower module.HotStuffFollower, validator hotstuff.Validator, diff --git a/engine/testutil/nodes.go b/engine/testutil/nodes.go index eb85762c0e4..52df37cac97 100644 --- a/engine/testutil/nodes.go +++ b/engine/testutil/nodes.go @@ -853,7 +853,7 @@ func createFollowerCore( t *testing.T, node *testmock.GenericNode, followerState *badgerstate.FollowerState, - notifier hotstuff.FollowerConsumer, + notifier hotstuff.ConsensusFollowerConsumer, rootHead *flow.Header, rootQC *flow.QuorumCertificate, ) (module.HotStuffFollower, *confinalizer.Finalizer) { diff --git a/follower/follower_builder.go b/follower/follower_builder.go index 0b00a1c9e91..53b5adf804c 100644 --- a/follower/follower_builder.go +++ b/follower/follower_builder.go @@ -357,7 +357,7 @@ func FlowConsensusFollowerService(opts ...FollowerOption) *FollowerServiceBuilde FlowNodeBuilder: cmd.FlowNode(flow.RoleAccess.String(), config.baseOptions...), FollowerDistributor: pubsub.NewFollowerDistributor(), } - ret.FollowerDistributor.AddFollowerConsumer(notifications.NewSlashingViolationsConsumer(ret.Logger)) + ret.FollowerDistributor.AddConsumer(notifications.NewSlashingViolationsConsumer(ret.Logger)) // the observer gets a version of the root snapshot file that does not contain any node addresses // hence skip all the root snapshot validations that involved an identity address ret.FlowNodeBuilder.SkipNwAddressBasedValidations = true From 35692497a6436fd140e70b6c7f859031760c5791 Mon Sep 17 00:00:00 2001 From: Alexander Hentschel Date: Sat, 6 May 2023 14:43:50 -0700 Subject: [PATCH 17/38] =?UTF-8?q?=E2=80=A2=20modularized=20consumer=20inte?= =?UTF-8?q?rfaces=20for=20hotstuff=20notifications=20=E2=80=A2=20refactore?= =?UTF-8?q?d=20distributors=20to=20be=20easily=20composable,=20still=20exp?= =?UTF-8?q?osing=20functionality=20to=20only=20subscribe=20to=20a=20subset?= =?UTF-8?q?=20of=20events=20corresponding=20to=20a=20atomic=20interface=20?= =?UTF-8?q?for=20hotstuff=20notifications?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../node_builder/access_node_builder.go | 2 +- cmd/collection/main.go | 2 +- cmd/consensus/main.go | 2 +- cmd/execution_builder.go | 4 +- cmd/observer/node_builder/observer_builder.go | 2 +- cmd/verification_builder.go | 2 +- consensus/follower.go | 2 +- consensus/hotstuff/consumer.go | 101 +++---- consensus/hotstuff/forks/forks2.go | 4 +- .../hotstuff/integration/instance_test.go | 4 +- .../hotstuff/notifications/noop_consumer.go | 21 +- .../pubsub/communicator_distributor.go | 56 ++++ .../notifications/pubsub/distributor.go | 246 ++---------------- .../pubsub/finalization_distributor.go | 68 +++++ .../pubsub/follower_distributor.go | 85 ------ .../pubsub/participant_distributor.go | 143 ++++++++++ .../pubsub/protocol_violation_distributor.go | 85 ++++++ .../pubsub/qc_created_distributor.go | 2 +- .../pubsub/timeout_collector_distributor.go | 10 +- .../slashing_violation_consumer.go | 25 +- consensus/integration/nodes_test.go | 2 +- consensus/participant.go | 6 +- engine/common/follower/compliance_core.go | 2 +- engine/testutil/nodes.go | 2 +- follower/follower_builder.go | 2 +- 25 files changed, 483 insertions(+), 397 deletions(-) create mode 100644 consensus/hotstuff/notifications/pubsub/communicator_distributor.go create mode 100644 consensus/hotstuff/notifications/pubsub/finalization_distributor.go delete mode 100644 consensus/hotstuff/notifications/pubsub/follower_distributor.go create mode 100644 consensus/hotstuff/notifications/pubsub/participant_distributor.go create mode 100644 consensus/hotstuff/notifications/pubsub/protocol_violation_distributor.go diff --git a/cmd/access/node_builder/access_node_builder.go b/cmd/access/node_builder/access_node_builder.go index 22f133dc630..fffdde44a4b 100644 --- a/cmd/access/node_builder/access_node_builder.go +++ b/cmd/access/node_builder/access_node_builder.go @@ -609,7 +609,7 @@ func (builder *FlowAccessNodeBuilder) BuildExecutionDataRequester() *FlowAccessN func FlowAccessNode(nodeBuilder *cmd.FlowNodeBuilder) *FlowAccessNodeBuilder { dist := consensuspubsub.NewFollowerDistributor() - dist.AddConsumer(notifications.NewSlashingViolationsConsumer(nodeBuilder.Logger)) + dist.AddFollowerConsumer(notifications.NewSlashingViolationsConsumer(nodeBuilder.Logger)) return &FlowAccessNodeBuilder{ AccessNodeConfig: DefaultAccessNodeConfig(), FlowNodeBuilder: nodeBuilder, diff --git a/cmd/collection/main.go b/cmd/collection/main.go index 98921e0ec5b..0d87a13141f 100644 --- a/cmd/collection/main.go +++ b/cmd/collection/main.go @@ -173,7 +173,7 @@ func main() { PreInit(cmd.DynamicStartPreInit). Module("follower distributor", func(node *cmd.NodeConfig) error { followerDistributor = pubsub.NewFollowerDistributor() - followerDistributor.AddConsumer(notifications.NewSlashingViolationsConsumer(node.Logger)) + followerDistributor.AddFollowerConsumer(notifications.NewSlashingViolationsConsumer(node.Logger)) return nil }). Module("mutable follower state", func(node *cmd.NodeConfig) error { diff --git a/cmd/consensus/main.go b/cmd/consensus/main.go index 280ecd8be48..fae7ba475a6 100644 --- a/cmd/consensus/main.go +++ b/cmd/consensus/main.go @@ -366,7 +366,7 @@ func main() { }). Module("follower distributor", func(node *cmd.NodeConfig) error { followerDistributor = pubsub.NewFollowerDistributor() - followerDistributor.AddConsumer(notifications.NewSlashingViolationsConsumer(nodeBuilder.Logger)) + followerDistributor.AddProtocolViolationConsumer(notifications.NewSlashingViolationsConsumer(nodeBuilder.Logger)) return nil }). Module("machine account config", func(node *cmd.NodeConfig) error { diff --git a/cmd/execution_builder.go b/cmd/execution_builder.go index a2660ccc353..0bfd0ad10a0 100644 --- a/cmd/execution_builder.go +++ b/cmd/execution_builder.go @@ -274,7 +274,7 @@ func (exeNode *ExecutionNode) LoadExecutionReceiptsStorage( func (exeNode *ExecutionNode) LoadFollowerDistributor(node *NodeConfig) error { exeNode.followerDistributor = pubsub.NewFollowerDistributor() - exeNode.followerDistributor.AddConsumer(notifications.NewSlashingViolationsConsumer(node.Logger)) + exeNode.followerDistributor.AddProtocolViolationConsumer(notifications.NewSlashingViolationsConsumer(node.Logger)) return nil } @@ -854,7 +854,7 @@ func (exeNode *ExecutionNode) LoadFollowerCore( return nil, fmt.Errorf("could not find latest finalized block and pending blocks to recover consensus follower: %w", err) } - exeNode.followerDistributor.AddConsumer(exeNode.checkerEng) + exeNode.followerDistributor.AddFollowerConsumer(exeNode.checkerEng) // creates a consensus follower with ingestEngine as the notifier // so that it gets notified upon each new finalized block diff --git a/cmd/observer/node_builder/observer_builder.go b/cmd/observer/node_builder/observer_builder.go index 4074c9f244a..508af73d311 100644 --- a/cmd/observer/node_builder/observer_builder.go +++ b/cmd/observer/node_builder/observer_builder.go @@ -566,7 +566,7 @@ func NewFlowObserverServiceBuilder(opts ...Option) *ObserverServiceBuilder { FlowNodeBuilder: cmd.FlowNode("observer"), FollowerDistributor: pubsub.NewFollowerDistributor(), } - anb.FollowerDistributor.AddConsumer(notifications.NewSlashingViolationsConsumer(anb.Logger)) + anb.FollowerDistributor.AddFollowerConsumer(notifications.NewSlashingViolationsConsumer(anb.Logger)) // the observer gets a version of the root snapshot file that does not contain any node addresses // hence skip all the root snapshot validations that involved an identity address anb.FlowNodeBuilder.SkipNwAddressBasedValidations = true diff --git a/cmd/verification_builder.go b/cmd/verification_builder.go index 9924c4884c7..d7f2f196d16 100644 --- a/cmd/verification_builder.go +++ b/cmd/verification_builder.go @@ -179,7 +179,7 @@ func (v *VerificationNodeBuilder) LoadComponentsAndModules() { }). Module("follower distributor", func(node *NodeConfig) error { followerDistributor = pubsub.NewFollowerDistributor() - followerDistributor.AddConsumer(notifications.NewSlashingViolationsConsumer(node.Logger)) + followerDistributor.AddFollowerConsumer(notifications.NewSlashingViolationsConsumer(node.Logger)) return nil }). Module("sync core", func(node *NodeConfig) error { diff --git a/consensus/follower.go b/consensus/follower.go index 1fa432ef3d9..d155948833b 100644 --- a/consensus/follower.go +++ b/consensus/follower.go @@ -25,7 +25,7 @@ import ( func NewFollower(log zerolog.Logger, headers storage.Headers, updater module.Finalizer, - notifier hotstuff.ConsensusFollowerConsumer, + notifier hotstuff.FollowerConsumer, rootHeader *flow.Header, rootQC *flow.QuorumCertificate, finalized *flow.Header, diff --git a/consensus/hotstuff/consumer.go b/consensus/hotstuff/consumer.go index 2c144fe103e..43ab4dc012e 100644 --- a/consensus/hotstuff/consumer.go +++ b/consensus/hotstuff/consumer.go @@ -7,9 +7,9 @@ import ( "github.com/onflow/flow-go/model/flow" ) -// ProtocolViolationConsumer consumes outbound notifications produced by compliance. -// Notifications can be produced by consensus participants and followers. -// Notifications are meant to report protocol violations that can be observed by executing compliance checks. +// ProtocolViolationConsumer consumes outbound notifications about HotStuff-protocol violations. +// Such notifications are produced by the active consensus participants and to a lesser +// degree also the consensus follower. // // Implementations must: // - be concurrency safe @@ -23,6 +23,7 @@ type ProtocolViolationConsumer interface { // Implementation must be concurrency safe; Non-blocking; // and must handle repetition of the same events (with some processing overhead). OnInvalidBlockDetected(err model.InvalidBlockError) + // OnDoubleProposeDetected notifications are produced by the Finalization Logic // whenever a double block proposal (equivocation) was detected. // Equivocation occurs when the same leader proposes two different blocks for the same view. @@ -30,12 +31,47 @@ type ProtocolViolationConsumer interface { // Implementation must be concurrency safe; Non-blocking; // and must handle repetition of the same events (with some processing overhead). OnDoubleProposeDetected(*model.Block, *model.Block) + + // OnDoubleVotingDetected notifications are produced by the Vote Aggregation logic + // whenever a double voting (same voter voting for different blocks at the same view) was detected. + // Prerequisites: + // Implementation must be concurrency safe; Non-blocking; + // and must handle repetition of the same events (with some processing overhead). + OnDoubleVotingDetected(*model.Vote, *model.Vote) + + // OnInvalidVoteDetected notifications are produced by the Vote Aggregation logic + // whenever an invalid vote was detected. + // Prerequisites: + // Implementation must be concurrency safe; Non-blocking; + // and must handle repetition of the same events (with some processing overhead). + OnInvalidVoteDetected(err model.InvalidVoteError) + + // OnVoteForInvalidBlockDetected notifications are produced by the Vote Aggregation logic + // whenever vote for invalid proposal was detected. + // Prerequisites: + // Implementation must be concurrency safe; Non-blocking; + // and must handle repetition of the same events (with some processing overhead). + OnVoteForInvalidBlockDetected(vote *model.Vote, invalidProposal *model.Proposal) + + // OnDoubleTimeoutDetected notifications are produced by the Timeout Aggregation logic + // whenever a double timeout (same replica producing two different timeouts at the same view) was detected. + // Prerequisites: + // Implementation must be concurrency safe; Non-blocking; + // and must handle repetition of the same events (with some processing overhead). + OnDoubleTimeoutDetected(*model.TimeoutObject, *model.TimeoutObject) + + // OnInvalidTimeoutDetected notifications are produced by the Timeout Aggregation logic + // whenever an invalid timeout was detected. + // Prerequisites: + // Implementation must be concurrency safe; Non-blocking; + // and must handle repetition of the same events (with some processing overhead). + OnInvalidTimeoutDetected(err model.InvalidTimeoutError) } -// FinalizationConsumer consumes outbound notifications produced by the finalization logic. -// Notifications represent finalization-specific state changes which are potentially relevant -// to the larger node. The notifications are emitted in the order in which the -// finalization algorithm makes the respective steps. +// FinalizationConsumer consumes outbound notifications produced by the logic tracking +// forks and finalization. Such notifications are produced by the active consensus +// participants, and generally potentially relevant to the larger node. The notifications +// are emitted in the order in which the finalization algorithm makes the respective steps. // // Implementations must: // - be concurrency safe @@ -57,13 +93,13 @@ type FinalizationConsumer interface { OnFinalizedBlock(*model.Block) } -// ConsensusFollowerConsumer consumes outbound notifications produced by consensus followers. +// FollowerConsumer consumes outbound notifications produced by consensus followers. // It is a subset of the notifications produced by consensus participants. // Implementations must: // - be concurrency safe // - be non-blocking // - handle repetition of the same events (with some processing overhead). -type ConsensusFollowerConsumer interface { +type FollowerConsumer interface { ProtocolViolationConsumer FinalizationConsumer } @@ -78,9 +114,19 @@ type ConsensusFollowerConsumer interface { // - be non-blocking // - handle repetition of the same events (with some processing overhead). type Consumer interface { - ConsensusFollowerConsumer + FollowerConsumer CommunicatorConsumer + ParticipantConsumer +} +// ParticipantConsumer consumes outbound notifications produced by consensus participants +// actively proposing blocks, voting, collecting & aggregating votes to QCs, and participating in +// the pacemaker (sending timeouts, collecting & aggregating timeouts to TCs). +// Implementations must: +// - be concurrency safe +// - be non-blocking +// - handle repetition of the same events (with some processing overhead). +type ParticipantConsumer interface { // OnEventProcessed notifications are produced by the EventHandler when it is done processing // and hands control back to the EventLoop to wait for the next event. // Prerequisites: @@ -190,41 +236,6 @@ type Consumer interface { // Implementation must be concurrency safe; Non-blocking; // and must handle repetition of the same events (with some processing overhead). OnCurrentViewDetails(currentView, finalizedView uint64, currentLeader flow.Identifier) - - // OnDoubleVotingDetected notifications are produced by the Vote Aggregation logic - // whenever a double voting (same voter voting for different blocks at the same view) was detected. - // Prerequisites: - // Implementation must be concurrency safe; Non-blocking; - // and must handle repetition of the same events (with some processing overhead). - OnDoubleVotingDetected(*model.Vote, *model.Vote) - - // OnInvalidVoteDetected notifications are produced by the Vote Aggregation logic - // whenever an invalid vote was detected. - // Prerequisites: - // Implementation must be concurrency safe; Non-blocking; - // and must handle repetition of the same events (with some processing overhead). - OnInvalidVoteDetected(err model.InvalidVoteError) - - // OnVoteForInvalidBlockDetected notifications are produced by the Vote Aggregation logic - // whenever vote for invalid proposal was detected. - // Prerequisites: - // Implementation must be concurrency safe; Non-blocking; - // and must handle repetition of the same events (with some processing overhead). - OnVoteForInvalidBlockDetected(vote *model.Vote, invalidProposal *model.Proposal) - - // OnDoubleTimeoutDetected notifications are produced by the Timeout Aggregation logic - // whenever a double timeout (same replica producing two different timeouts at the same view) was detected. - // Prerequisites: - // Implementation must be concurrency safe; Non-blocking; - // and must handle repetition of the same events (with some processing overhead). - OnDoubleTimeoutDetected(*model.TimeoutObject, *model.TimeoutObject) - - // OnInvalidTimeoutDetected notifications are produced by the Timeout Aggregation logic - // whenever an invalid timeout was detected. - // Prerequisites: - // Implementation must be concurrency safe; Non-blocking; - // and must handle repetition of the same events (with some processing overhead). - OnInvalidTimeoutDetected(err model.InvalidTimeoutError) } // QCCreatedConsumer consumes outbound notifications produced by HotStuff and its components. diff --git a/consensus/hotstuff/forks/forks2.go b/consensus/hotstuff/forks/forks2.go index e9aaacca4a9..e4ebc59112b 100644 --- a/consensus/hotstuff/forks/forks2.go +++ b/consensus/hotstuff/forks/forks2.go @@ -19,7 +19,7 @@ import ( // Forks is NOT safe for concurrent use by multiple goroutines. type Forks struct { finalizationCallback module.Finalizer - notifier hotstuff.ConsensusFollowerConsumer + notifier hotstuff.FollowerConsumer forest forest.LevelledForest trustedRoot *model.CertifiedBlock @@ -30,7 +30,7 @@ type Forks struct { var _ hotstuff.Forks = (*Forks)(nil) -func New(trustedRoot *model.CertifiedBlock, finalizationCallback module.Finalizer, notifier hotstuff.ConsensusFollowerConsumer) (*Forks, error) { +func New(trustedRoot *model.CertifiedBlock, finalizationCallback module.Finalizer, notifier hotstuff.FollowerConsumer) (*Forks, error) { if (trustedRoot.Block.BlockID != trustedRoot.CertifyingQC.BlockID) || (trustedRoot.Block.View != trustedRoot.CertifyingQC.View) { return nil, model.NewConfigurationErrorf("invalid root: root QC is not pointing to root block") } diff --git a/consensus/hotstuff/integration/instance_test.go b/consensus/hotstuff/integration/instance_test.go index e981f335329..a082e2becbb 100644 --- a/consensus/hotstuff/integration/instance_test.go +++ b/consensus/hotstuff/integration/instance_test.go @@ -539,8 +539,8 @@ func NewInstance(t *testing.T, options ...Option) *Instance { ) require.NoError(t, err) - collectorDistributor.AddConsumer(logConsumer) - collectorDistributor.AddConsumer(&in) + collectorDistributor.AddTimeoutCollectorConsumer(logConsumer) + collectorDistributor.AddTimeoutCollectorConsumer(&in) return &in } diff --git a/consensus/hotstuff/notifications/noop_consumer.go b/consensus/hotstuff/notifications/noop_consumer.go index f3babd8f81a..c2f02acbcae 100644 --- a/consensus/hotstuff/notifications/noop_consumer.go +++ b/consensus/hotstuff/notifications/noop_consumer.go @@ -56,16 +56,6 @@ func (*NoopPartialConsumer) OnTimeoutProcessed(*model.TimeoutObject) {} func (*NoopPartialConsumer) OnCurrentViewDetails(uint64, uint64, flow.Identifier) {} -func (*NoopPartialConsumer) OnDoubleVotingDetected(*model.Vote, *model.Vote) {} - -func (*NoopPartialConsumer) OnInvalidVoteDetected(model.InvalidVoteError) {} - -func (*NoopPartialConsumer) OnVoteForInvalidBlockDetected(*model.Vote, *model.Proposal) {} - -func (*NoopPartialConsumer) OnDoubleTimeoutDetected(*model.TimeoutObject, *model.TimeoutObject) {} - -func (*NoopPartialConsumer) OnInvalidTimeoutDetected(model.InvalidTimeoutError) {} - // no-op implementation of hotstuff.FinalizationConsumer type NoopFinalizationConsumer struct{} @@ -120,3 +110,14 @@ var _ hotstuff.ProtocolViolationConsumer = (*NoopProtocolViolationConsumer)(nil) func (*NoopProtocolViolationConsumer) OnInvalidBlockDetected(model.InvalidBlockError) {} func (*NoopProtocolViolationConsumer) OnDoubleProposeDetected(*model.Block, *model.Block) {} + +func (*NoopProtocolViolationConsumer) OnDoubleVotingDetected(*model.Vote, *model.Vote) {} + +func (*NoopProtocolViolationConsumer) OnInvalidVoteDetected(model.InvalidVoteError) {} + +func (*NoopProtocolViolationConsumer) OnVoteForInvalidBlockDetected(*model.Vote, *model.Proposal) {} + +func (*NoopProtocolViolationConsumer) OnDoubleTimeoutDetected(*model.TimeoutObject, *model.TimeoutObject) { +} + +func (*NoopProtocolViolationConsumer) OnInvalidTimeoutDetected(model.InvalidTimeoutError) {} diff --git a/consensus/hotstuff/notifications/pubsub/communicator_distributor.go b/consensus/hotstuff/notifications/pubsub/communicator_distributor.go new file mode 100644 index 00000000000..521e06ee50b --- /dev/null +++ b/consensus/hotstuff/notifications/pubsub/communicator_distributor.go @@ -0,0 +1,56 @@ +package pubsub + +import ( + "sync" + "time" + + "github.com/onflow/flow-go/consensus/hotstuff" + "github.com/onflow/flow-go/consensus/hotstuff/model" + "github.com/onflow/flow-go/model/flow" +) + +// CommunicatorDistributor ingests outbound consensus messages from HotStuff's core logic and +// distributes them to subscribers. This logic only runs inside active consensus participants proposing +// blocks, voting, collecting + aggregating votes to QCs, and participating in the pacemaker (sending +// timeouts, collecting + aggregating timeouts to TCs). +// Concurrently safe. +type CommunicatorDistributor struct { + subscribers []hotstuff.CommunicatorConsumer + lock sync.RWMutex +} + +var _ hotstuff.CommunicatorConsumer = (*CommunicatorDistributor)(nil) + +func NewCommunicatorConsumerDistributor() *CommunicatorDistributor { + return &CommunicatorDistributor{} +} + +func (d *CommunicatorDistributor) AddCommunicatorConsumer(consumer hotstuff.CommunicatorConsumer) { + d.lock.Lock() + defer d.lock.Unlock() + d.subscribers = append(d.subscribers, consumer) +} + +func (d *CommunicatorDistributor) OnOwnVote(blockID flow.Identifier, view uint64, sigData []byte, recipientID flow.Identifier) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, s := range d.subscribers { + s.OnOwnVote(blockID, view, sigData, recipientID) + } +} + +func (d *CommunicatorDistributor) OnOwnTimeout(timeout *model.TimeoutObject) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, s := range d.subscribers { + s.OnOwnTimeout(timeout) + } +} + +func (d *CommunicatorDistributor) OnOwnProposal(proposal *flow.Header, targetPublicationTime time.Time) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, s := range d.subscribers { + s.OnOwnProposal(proposal, targetPublicationTime) + } +} diff --git a/consensus/hotstuff/notifications/pubsub/distributor.go b/consensus/hotstuff/notifications/pubsub/distributor.go index 74674ee8547..7c600a71e88 100644 --- a/consensus/hotstuff/notifications/pubsub/distributor.go +++ b/consensus/hotstuff/notifications/pubsub/distributor.go @@ -1,258 +1,44 @@ package pubsub import ( - "sync" - "time" - "github.com/onflow/flow-go/consensus/hotstuff" - "github.com/onflow/flow-go/consensus/hotstuff/model" - "github.com/onflow/flow-go/consensus/hotstuff/notifications" - "github.com/onflow/flow-go/model/flow" ) // Distributor distributes notifications to a list of subscribers (event consumers). // // It allows thread-safe subscription of multiple consumers to events. type Distributor struct { - subscribers []hotstuff.Consumer - lock sync.RWMutex + FollowerDistributor + CommunicatorDistributor + ParticipantDistributor } var _ hotstuff.Consumer = (*Distributor)(nil) -func (p *Distributor) OnEventProcessed() { - p.lock.RLock() - defer p.lock.RUnlock() - for _, subscriber := range p.subscribers { - subscriber.OnEventProcessed() - } -} - func NewDistributor() *Distributor { return &Distributor{} } // AddConsumer adds an event consumer to the Distributor func (p *Distributor) AddConsumer(consumer hotstuff.Consumer) { - p.lock.Lock() - defer p.lock.Unlock() - p.subscribers = append(p.subscribers, consumer) -} - -// AddFollowerConsumer registers the input `consumer` to be notified on ConsensusFollowerConsumer events. -func (p *Distributor) AddFollowerConsumer(consumer hotstuff.ConsensusFollowerConsumer) { - p.lock.Lock() - defer p.lock.Unlock() - - var wrappedConsumer hotstuff.Consumer = &struct { - notifications.NoopCommunicatorConsumer - notifications.NoopPartialConsumer - hotstuff.ConsensusFollowerConsumer - }{ - notifications.NoopCommunicatorConsumer{}, - notifications.NoopPartialConsumer{}, - consumer, - } - - p.subscribers = append(p.subscribers, wrappedConsumer) -} - -func (p *Distributor) OnStart(currentView uint64) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, subscriber := range p.subscribers { - subscriber.OnStart(currentView) - } -} - -func (p *Distributor) OnReceiveProposal(currentView uint64, proposal *model.Proposal) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, subscriber := range p.subscribers { - subscriber.OnReceiveProposal(currentView, proposal) - } -} - -func (p *Distributor) OnReceiveQc(currentView uint64, qc *flow.QuorumCertificate) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, subscriber := range p.subscribers { - subscriber.OnReceiveQc(currentView, qc) - } -} - -func (p *Distributor) OnReceiveTc(currentView uint64, tc *flow.TimeoutCertificate) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, subscriber := range p.subscribers { - subscriber.OnReceiveTc(currentView, tc) - } -} - -func (p *Distributor) OnPartialTc(currentView uint64, partialTc *hotstuff.PartialTcCreated) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, subscriber := range p.subscribers { - subscriber.OnPartialTc(currentView, partialTc) - } -} - -func (p *Distributor) OnLocalTimeout(currentView uint64) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, subscriber := range p.subscribers { - subscriber.OnLocalTimeout(currentView) - } -} - -func (p *Distributor) OnViewChange(oldView, newView uint64) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, subscriber := range p.subscribers { - subscriber.OnViewChange(oldView, newView) - } -} - -func (p *Distributor) OnQcTriggeredViewChange(oldView uint64, newView uint64, qc *flow.QuorumCertificate) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, subscriber := range p.subscribers { - subscriber.OnQcTriggeredViewChange(oldView, newView, qc) - } -} - -func (p *Distributor) OnTcTriggeredViewChange(oldView uint64, newView uint64, tc *flow.TimeoutCertificate) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, subscriber := range p.subscribers { - subscriber.OnTcTriggeredViewChange(oldView, newView, tc) - } + p.FollowerDistributor.AddFollowerConsumer(consumer) + p.CommunicatorDistributor.AddCommunicatorConsumer(consumer) + p.ParticipantDistributor.AddParticipantConsumer(consumer) } -func (p *Distributor) OnStartingTimeout(timerInfo model.TimerInfo) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, subscriber := range p.subscribers { - subscriber.OnStartingTimeout(timerInfo) - } +// FollowerDistributor ingests consensus follower events and distributes it to subscribers. +type FollowerDistributor struct { + ProtocolViolationDistributor + FinalizationDistributor } -func (p *Distributor) OnVoteProcessed(vote *model.Vote) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, subscriber := range p.subscribers { - subscriber.OnVoteProcessed(vote) - } -} - -func (p *Distributor) OnTimeoutProcessed(timeout *model.TimeoutObject) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, subscriber := range p.subscribers { - subscriber.OnTimeoutProcessed(timeout) - } -} - -func (p *Distributor) OnCurrentViewDetails(currentView, finalizedView uint64, currentLeader flow.Identifier) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, subscriber := range p.subscribers { - subscriber.OnCurrentViewDetails(currentView, finalizedView, currentLeader) - } -} - -func (p *Distributor) OnBlockIncorporated(block *model.Block) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, subscriber := range p.subscribers { - subscriber.OnBlockIncorporated(block) - } -} - -func (p *Distributor) OnFinalizedBlock(block *model.Block) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, subscriber := range p.subscribers { - subscriber.OnFinalizedBlock(block) - } -} - -func (p *Distributor) OnInvalidBlockDetected(err model.InvalidBlockError) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, subscriber := range p.subscribers { - subscriber.OnInvalidBlockDetected(err) - } -} - -func (p *Distributor) OnDoubleProposeDetected(block1, block2 *model.Block) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, subscriber := range p.subscribers { - subscriber.OnDoubleProposeDetected(block1, block2) - } -} - -func (p *Distributor) OnDoubleVotingDetected(vote1, vote2 *model.Vote) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, subscriber := range p.subscribers { - subscriber.OnDoubleVotingDetected(vote1, vote2) - } -} - -func (p *Distributor) OnInvalidVoteDetected(err model.InvalidVoteError) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, subscriber := range p.subscribers { - subscriber.OnInvalidVoteDetected(err) - } -} - -func (p *Distributor) OnVoteForInvalidBlockDetected(vote *model.Vote, invalidProposal *model.Proposal) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, subscriber := range p.subscribers { - subscriber.OnVoteForInvalidBlockDetected(vote, invalidProposal) - } -} - -func (p *Distributor) OnDoubleTimeoutDetected(timeout *model.TimeoutObject, altTimeout *model.TimeoutObject) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, subscriber := range p.subscribers { - subscriber.OnDoubleTimeoutDetected(timeout, altTimeout) - } -} - -func (p *Distributor) OnInvalidTimeoutDetected(err model.InvalidTimeoutError) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, subscriber := range p.subscribers { - subscriber.OnInvalidTimeoutDetected(err) - } -} - -func (p *Distributor) OnOwnVote(blockID flow.Identifier, view uint64, sigData []byte, recipientID flow.Identifier) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, s := range p.subscribers { - s.OnOwnVote(blockID, view, sigData, recipientID) - } -} +var _ hotstuff.FollowerConsumer = (*FollowerDistributor)(nil) -func (p *Distributor) OnOwnTimeout(timeout *model.TimeoutObject) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, s := range p.subscribers { - s.OnOwnTimeout(timeout) - } +func NewFollowerDistributor() *FollowerDistributor { + return &FollowerDistributor{} } -func (p *Distributor) OnOwnProposal(proposal *flow.Header, targetPublicationTime time.Time) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, s := range p.subscribers { - s.OnOwnProposal(proposal, targetPublicationTime) - } +func (d *FollowerDistributor) AddFollowerConsumer(consumer hotstuff.FollowerConsumer) { + d.FinalizationDistributor.AddFinalizationConsumer(consumer) + d.ProtocolViolationDistributor.AddProtocolViolationConsumer(consumer) } diff --git a/consensus/hotstuff/notifications/pubsub/finalization_distributor.go b/consensus/hotstuff/notifications/pubsub/finalization_distributor.go new file mode 100644 index 00000000000..a78dd88d13b --- /dev/null +++ b/consensus/hotstuff/notifications/pubsub/finalization_distributor.go @@ -0,0 +1,68 @@ +package pubsub + +import ( + "sync" + + "github.com/onflow/flow-go/consensus/hotstuff" + "github.com/onflow/flow-go/consensus/hotstuff/model" +) + +type OnBlockFinalizedConsumer = func(block *model.Block) +type OnBlockIncorporatedConsumer = func(block *model.Block) + +// FinalizationDistributor ingests events from HotStuff's logic for tracking forks + finalization +// and distributes them to subscribers. This logic generally runs inside all nodes (irrespectively whether +// they are active consensus participants or or only consensus followers). +// Concurrently safe. +type FinalizationDistributor struct { + blockFinalizedConsumers []OnBlockFinalizedConsumer + blockIncorporatedConsumers []OnBlockIncorporatedConsumer + consumers []hotstuff.FinalizationConsumer + lock sync.RWMutex +} + +var _ hotstuff.FinalizationConsumer = (*FinalizationDistributor)(nil) + +func NewFinalizationDistributor() *FinalizationDistributor { + return &FinalizationDistributor{} +} + +func (d *FinalizationDistributor) AddOnBlockFinalizedConsumer(consumer OnBlockFinalizedConsumer) { + d.lock.Lock() + defer d.lock.Unlock() + d.blockFinalizedConsumers = append(d.blockFinalizedConsumers, consumer) +} + +func (d *FinalizationDistributor) AddOnBlockIncorporatedConsumer(consumer OnBlockIncorporatedConsumer) { + d.lock.Lock() + defer d.lock.Unlock() + d.blockIncorporatedConsumers = append(d.blockIncorporatedConsumers, consumer) +} + +func (d *FinalizationDistributor) AddFinalizationConsumer(consumer hotstuff.FinalizationConsumer) { + d.lock.Lock() + defer d.lock.Unlock() + d.consumers = append(d.consumers, consumer) +} + +func (d *FinalizationDistributor) OnBlockIncorporated(block *model.Block) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, consumer := range d.blockIncorporatedConsumers { + consumer(block) + } + for _, consumer := range d.consumers { + consumer.OnBlockIncorporated(block) + } +} + +func (d *FinalizationDistributor) OnFinalizedBlock(block *model.Block) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, consumer := range d.blockFinalizedConsumers { + consumer(block) + } + for _, consumer := range d.consumers { + consumer.OnFinalizedBlock(block) + } +} diff --git a/consensus/hotstuff/notifications/pubsub/follower_distributor.go b/consensus/hotstuff/notifications/pubsub/follower_distributor.go deleted file mode 100644 index 54ad77ac925..00000000000 --- a/consensus/hotstuff/notifications/pubsub/follower_distributor.go +++ /dev/null @@ -1,85 +0,0 @@ -package pubsub - -import ( - "sync" - - "github.com/onflow/flow-go/consensus/hotstuff" - "github.com/onflow/flow-go/consensus/hotstuff/model" -) - -type OnBlockFinalizedConsumer = func(block *model.Block) -type OnBlockIncorporatedConsumer = func(block *model.Block) - -// FollowerDistributor ingests consensus follower events and distributes it to subscribers. -type FollowerDistributor struct { - blockFinalizedConsumers []OnBlockFinalizedConsumer - blockIncorporatedConsumers []OnBlockIncorporatedConsumer - followerConsumers []hotstuff.ConsensusFollowerConsumer - lock sync.RWMutex -} - -var _ hotstuff.ConsensusFollowerConsumer = (*FollowerDistributor)(nil) - -func NewFollowerDistributor() *FollowerDistributor { - return &FollowerDistributor{ - blockFinalizedConsumers: make([]OnBlockFinalizedConsumer, 0), - blockIncorporatedConsumers: make([]OnBlockIncorporatedConsumer, 0), - lock: sync.RWMutex{}, - } -} - -func (p *FollowerDistributor) AddOnBlockFinalizedConsumer(consumer OnBlockFinalizedConsumer) { - p.lock.Lock() - defer p.lock.Unlock() - p.blockFinalizedConsumers = append(p.blockFinalizedConsumers, consumer) -} - -func (p *FollowerDistributor) AddOnBlockIncorporatedConsumer(consumer OnBlockIncorporatedConsumer) { - p.lock.Lock() - defer p.lock.Unlock() - p.blockIncorporatedConsumers = append(p.blockIncorporatedConsumers, consumer) -} - -func (p *FollowerDistributor) AddConsumer(consumer hotstuff.ConsensusFollowerConsumer) { - p.lock.Lock() - defer p.lock.Unlock() - p.followerConsumers = append(p.followerConsumers, consumer) -} - -func (p *FollowerDistributor) OnBlockIncorporated(block *model.Block) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, consumer := range p.blockIncorporatedConsumers { - consumer(block) - } - for _, consumer := range p.followerConsumers { - consumer.OnBlockIncorporated(block) - } -} - -func (p *FollowerDistributor) OnFinalizedBlock(block *model.Block) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, consumer := range p.blockFinalizedConsumers { - consumer(block) - } - for _, consumer := range p.followerConsumers { - consumer.OnFinalizedBlock(block) - } -} - -func (p *FollowerDistributor) OnDoubleProposeDetected(block1, block2 *model.Block) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, consumer := range p.followerConsumers { - consumer.OnDoubleProposeDetected(block1, block2) - } -} - -func (p *FollowerDistributor) OnInvalidBlockDetected(err model.InvalidBlockError) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, consumer := range p.followerConsumers { - consumer.OnInvalidBlockDetected(err) - } -} diff --git a/consensus/hotstuff/notifications/pubsub/participant_distributor.go b/consensus/hotstuff/notifications/pubsub/participant_distributor.go new file mode 100644 index 00000000000..f0fae001a41 --- /dev/null +++ b/consensus/hotstuff/notifications/pubsub/participant_distributor.go @@ -0,0 +1,143 @@ +package pubsub + +import ( + "sync" + + "github.com/onflow/flow-go/consensus/hotstuff" + "github.com/onflow/flow-go/consensus/hotstuff/model" + "github.com/onflow/flow-go/model/flow" +) + +// ParticipantDistributor ingests events from HotStuff's core logic and distributes them to +// subscribers. This logic only runs inside active consensus participants proposing blocks, voting, +// collecting + aggregating votes to QCs, and participating in the pacemaker (sending timeouts, +// collecting + aggregating timeouts to TCs). +// Concurrently safe. +type ParticipantDistributor struct { + subscribers []hotstuff.ParticipantConsumer + lock sync.RWMutex +} + +var _ hotstuff.ParticipantConsumer = (*ParticipantDistributor)(nil) + +func NewConsensusParticipantDistributor() *ParticipantDistributor { + return &ParticipantDistributor{} +} + +func (d *ParticipantDistributor) AddParticipantConsumer(consumer hotstuff.ParticipantConsumer) { + d.lock.Lock() + defer d.lock.Unlock() + d.subscribers = append(d.subscribers, consumer) +} + +func (d *ParticipantDistributor) OnEventProcessed() { + d.lock.RLock() + defer d.lock.RUnlock() + for _, subscriber := range d.subscribers { + subscriber.OnEventProcessed() + } +} + +func (d *ParticipantDistributor) OnStart(currentView uint64) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, subscriber := range d.subscribers { + subscriber.OnStart(currentView) + } +} + +func (d *ParticipantDistributor) OnReceiveProposal(currentView uint64, proposal *model.Proposal) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, subscriber := range d.subscribers { + subscriber.OnReceiveProposal(currentView, proposal) + } +} + +func (d *ParticipantDistributor) OnReceiveQc(currentView uint64, qc *flow.QuorumCertificate) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, subscriber := range d.subscribers { + subscriber.OnReceiveQc(currentView, qc) + } +} + +func (d *ParticipantDistributor) OnReceiveTc(currentView uint64, tc *flow.TimeoutCertificate) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, subscriber := range d.subscribers { + subscriber.OnReceiveTc(currentView, tc) + } +} + +func (d *ParticipantDistributor) OnPartialTc(currentView uint64, partialTc *hotstuff.PartialTcCreated) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, subscriber := range d.subscribers { + subscriber.OnPartialTc(currentView, partialTc) + } +} + +func (d *ParticipantDistributor) OnLocalTimeout(currentView uint64) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, subscriber := range d.subscribers { + subscriber.OnLocalTimeout(currentView) + } +} + +func (d *ParticipantDistributor) OnViewChange(oldView, newView uint64) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, subscriber := range d.subscribers { + subscriber.OnViewChange(oldView, newView) + } +} + +func (d *ParticipantDistributor) OnQcTriggeredViewChange(oldView uint64, newView uint64, qc *flow.QuorumCertificate) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, subscriber := range d.subscribers { + subscriber.OnQcTriggeredViewChange(oldView, newView, qc) + } +} + +func (d *ParticipantDistributor) OnTcTriggeredViewChange(oldView uint64, newView uint64, tc *flow.TimeoutCertificate) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, subscriber := range d.subscribers { + subscriber.OnTcTriggeredViewChange(oldView, newView, tc) + } +} + +func (d *ParticipantDistributor) OnStartingTimeout(timerInfo model.TimerInfo) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, subscriber := range d.subscribers { + subscriber.OnStartingTimeout(timerInfo) + } +} + +func (d *ParticipantDistributor) OnVoteProcessed(vote *model.Vote) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, subscriber := range d.subscribers { + subscriber.OnVoteProcessed(vote) + } +} + +func (d *ParticipantDistributor) OnTimeoutProcessed(timeout *model.TimeoutObject) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, subscriber := range d.subscribers { + subscriber.OnTimeoutProcessed(timeout) + } +} + +func (d *ParticipantDistributor) OnCurrentViewDetails(currentView, finalizedView uint64, currentLeader flow.Identifier) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, subscriber := range d.subscribers { + subscriber.OnCurrentViewDetails(currentView, finalizedView, currentLeader) + } +} diff --git a/consensus/hotstuff/notifications/pubsub/protocol_violation_distributor.go b/consensus/hotstuff/notifications/pubsub/protocol_violation_distributor.go new file mode 100644 index 00000000000..4c08dacddac --- /dev/null +++ b/consensus/hotstuff/notifications/pubsub/protocol_violation_distributor.go @@ -0,0 +1,85 @@ +package pubsub + +import ( + "sync" + + "github.com/onflow/flow-go/consensus/hotstuff" + "github.com/onflow/flow-go/consensus/hotstuff/model" +) + +// ProtocolViolationDistributor ingests notifications about HotStuff-protocol violations and +// distributes them to subscribers. Such notifications are produced by the active consensus +// participants and to a lesser degree also the consensus follower. +// Concurrently safe. +type ProtocolViolationDistributor struct { + subscribers []hotstuff.ProtocolViolationConsumer + lock sync.RWMutex +} + +var _ hotstuff.ProtocolViolationConsumer = (*ProtocolViolationDistributor)(nil) + +func NewProtocolViolationDistributor() *ProtocolViolationDistributor { + return &ProtocolViolationDistributor{} +} + +func (d *ProtocolViolationDistributor) AddProtocolViolationConsumer(consumer hotstuff.ProtocolViolationConsumer) { + d.lock.Lock() + defer d.lock.Unlock() + d.subscribers = append(d.subscribers, consumer) +} + +func (d *ProtocolViolationDistributor) OnInvalidBlockDetected(err model.InvalidBlockError) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, subscriber := range d.subscribers { + subscriber.OnInvalidBlockDetected(err) + } +} + +func (d *ProtocolViolationDistributor) OnDoubleProposeDetected(block1, block2 *model.Block) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, subscriber := range d.subscribers { + subscriber.OnDoubleProposeDetected(block1, block2) + } +} + +func (d *ProtocolViolationDistributor) OnDoubleVotingDetected(vote1, vote2 *model.Vote) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, subscriber := range d.subscribers { + subscriber.OnDoubleVotingDetected(vote1, vote2) + } +} + +func (d *ProtocolViolationDistributor) OnInvalidVoteDetected(err model.InvalidVoteError) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, subscriber := range d.subscribers { + subscriber.OnInvalidVoteDetected(err) + } +} + +func (d *ProtocolViolationDistributor) OnVoteForInvalidBlockDetected(vote *model.Vote, invalidProposal *model.Proposal) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, subscriber := range d.subscribers { + subscriber.OnVoteForInvalidBlockDetected(vote, invalidProposal) + } +} + +func (d *ProtocolViolationDistributor) OnDoubleTimeoutDetected(timeout *model.TimeoutObject, altTimeout *model.TimeoutObject) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, subscriber := range d.subscribers { + subscriber.OnDoubleTimeoutDetected(timeout, altTimeout) + } +} + +func (d *ProtocolViolationDistributor) OnInvalidTimeoutDetected(err model.InvalidTimeoutError) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, subscriber := range d.subscribers { + subscriber.OnInvalidTimeoutDetected(err) + } +} diff --git a/consensus/hotstuff/notifications/pubsub/qc_created_distributor.go b/consensus/hotstuff/notifications/pubsub/qc_created_distributor.go index 166fa9cf757..b9adafc2567 100644 --- a/consensus/hotstuff/notifications/pubsub/qc_created_distributor.go +++ b/consensus/hotstuff/notifications/pubsub/qc_created_distributor.go @@ -24,7 +24,7 @@ func NewQCCreatedDistributor() *QCCreatedDistributor { } } -func (d *QCCreatedDistributor) AddConsumer(consumer hotstuff.QCCreatedConsumer) { +func (d *QCCreatedDistributor) AddQCCreatedConsumer(consumer hotstuff.QCCreatedConsumer) { d.lock.Lock() defer d.lock.Unlock() d.qcCreatedConsumers = append(d.qcCreatedConsumers, consumer) diff --git a/consensus/hotstuff/notifications/pubsub/timeout_collector_distributor.go b/consensus/hotstuff/notifications/pubsub/timeout_collector_distributor.go index 8387fb81663..aa9c0bd9397 100644 --- a/consensus/hotstuff/notifications/pubsub/timeout_collector_distributor.go +++ b/consensus/hotstuff/notifications/pubsub/timeout_collector_distributor.go @@ -7,8 +7,8 @@ import ( "github.com/onflow/flow-go/model/flow" ) -// TimeoutCollectorDistributor ingests events from hotstuff and distributes them to subscribers. -// Concurrently safe +// TimeoutCollectorDistributor ingests events from hotstuff's VoteCollector +// logic and distributes them to subscribers. Concurrently safe. // TODO: investigate if this can be updated using atomics to prevent locking on mutex since we always add all consumers // before delivering events. type TimeoutCollectorDistributor struct { @@ -19,12 +19,10 @@ type TimeoutCollectorDistributor struct { var _ hotstuff.TimeoutCollectorConsumer = (*TimeoutCollectorDistributor)(nil) func NewTimeoutCollectorDistributor() *TimeoutCollectorDistributor { - return &TimeoutCollectorDistributor{ - consumers: make([]hotstuff.TimeoutCollectorConsumer, 0), - } + return &TimeoutCollectorDistributor{} } -func (d *TimeoutCollectorDistributor) AddConsumer(consumer hotstuff.TimeoutCollectorConsumer) { +func (d *TimeoutCollectorDistributor) AddTimeoutCollectorConsumer(consumer hotstuff.TimeoutCollectorConsumer) { d.lock.Lock() defer d.lock.Unlock() d.consumers = append(d.consumers, consumer) diff --git a/consensus/hotstuff/notifications/slashing_violation_consumer.go b/consensus/hotstuff/notifications/slashing_violation_consumer.go index fb80e15e522..e67d87e3d48 100644 --- a/consensus/hotstuff/notifications/slashing_violation_consumer.go +++ b/consensus/hotstuff/notifications/slashing_violation_consumer.go @@ -3,6 +3,7 @@ package notifications import ( "github.com/rs/zerolog" + "github.com/onflow/flow-go/consensus/hotstuff" "github.com/onflow/flow-go/consensus/hotstuff/model" "github.com/onflow/flow-go/utils/logging" ) @@ -10,15 +11,27 @@ import ( // SlashingViolationsConsumer is an implementation of the notifications consumer that logs a // message for any slashable offenses. type SlashingViolationsConsumer struct { - NoopConsumer log zerolog.Logger } +var _ hotstuff.ProtocolViolationConsumer = (*SlashingViolationsConsumer)(nil) + func NewSlashingViolationsConsumer(log zerolog.Logger) *SlashingViolationsConsumer { return &SlashingViolationsConsumer{ log: log, } } +func (c *SlashingViolationsConsumer) OnInvalidBlockDetected(err model.InvalidBlockError) { + block := err.InvalidBlock.Block + c.log.Warn(). + Bool(logging.KeySuspicious, true). + Hex("proposer_id", block.ProposerID[:]). + Uint64("block_view", block.View). + Hex("block_id", block.BlockID[:]). + Hex("block_payloadhash", block.PayloadHash[:]). + Time("block_timestamp", block.Timestamp). + Msg("OnInvalidBlockDetected") +} func (c *SlashingViolationsConsumer) OnDoubleVotingDetected(vote1 *model.Vote, vote2 *model.Vote) { c.log.Warn(). @@ -41,6 +54,16 @@ func (c *SlashingViolationsConsumer) OnInvalidVoteDetected(err model.InvalidVote Msg("OnInvalidVoteDetected") } +func (c *SlashingViolationsConsumer) OnDoubleTimeoutDetected(timeout *model.TimeoutObject, altTimeout *model.TimeoutObject) { + c.log.Warn(). + Bool(logging.KeySuspicious, true). + Hex("timeout_creator", timeout.SignerID[:]). + Uint64("timeout_view", timeout.View). + Hex("timeout_id1", logging.ID(timeout.ID())). + Hex("timeout_id2", logging.ID(altTimeout.ID())). + Msg("OnDoubleTimeoutDetected") +} + func (c *SlashingViolationsConsumer) OnInvalidTimeoutDetected(err model.InvalidTimeoutError) { timeout := err.Timeout c.log.Warn(). diff --git a/consensus/integration/nodes_test.go b/consensus/integration/nodes_test.go index 817fc46fcba..6911118e877 100644 --- a/consensus/integration/nodes_test.go +++ b/consensus/integration/nodes_test.go @@ -531,7 +531,7 @@ func createNode( require.NoError(t, err) timeoutCollectorDistributor := pubsub.NewTimeoutCollectorDistributor() - timeoutCollectorDistributor.AddConsumer(logConsumer) + timeoutCollectorDistributor.AddTimeoutCollectorConsumer(logConsumer) timeoutProcessorFactory := timeoutcollector.NewTimeoutProcessorFactory( log, diff --git a/consensus/participant.go b/consensus/participant.go index 50e17b14b47..e6435c70b97 100644 --- a/consensus/participant.go +++ b/consensus/participant.go @@ -122,8 +122,8 @@ func NewParticipant( } // add observer, event loop needs to receive events from distributor - modules.QCCreatedDistributor.AddConsumer(loop) - modules.TimeoutCollectorDistributor.AddConsumer(loop) + modules.QCCreatedDistributor.AddQCCreatedConsumer(loop) + modules.TimeoutCollectorDistributor.AddTimeoutCollectorConsumer(loop) return loop, nil } @@ -139,7 +139,7 @@ func NewValidator(metrics module.HotstuffMetrics, committee hotstuff.DynamicComm } // NewForks recovers trusted root and creates new forks manager -func NewForks(final *flow.Header, headers storage.Headers, updater module.Finalizer, notifier hotstuff.ConsensusFollowerConsumer, rootHeader *flow.Header, rootQC *flow.QuorumCertificate) (*forks.Forks, error) { +func NewForks(final *flow.Header, headers storage.Headers, updater module.Finalizer, notifier hotstuff.FollowerConsumer, rootHeader *flow.Header, rootQC *flow.QuorumCertificate) (*forks.Forks, error) { // recover the trusted root trustedRoot, err := recoverTrustedRoot(final, headers, rootHeader, rootQC) if err != nil { diff --git a/engine/common/follower/compliance_core.go b/engine/common/follower/compliance_core.go index 92eec6d43ff..ed0f43abd12 100644 --- a/engine/common/follower/compliance_core.go +++ b/engine/common/follower/compliance_core.go @@ -59,7 +59,7 @@ var _ complianceCore = (*ComplianceCore)(nil) func NewComplianceCore(log zerolog.Logger, mempoolMetrics module.MempoolMetrics, heroCacheCollector module.HeroCacheMetrics, - followerConsumer hotstuff.ConsensusFollowerConsumer, + followerConsumer hotstuff.FollowerConsumer, state protocol.FollowerState, follower module.HotStuffFollower, validator hotstuff.Validator, diff --git a/engine/testutil/nodes.go b/engine/testutil/nodes.go index 52df37cac97..eb85762c0e4 100644 --- a/engine/testutil/nodes.go +++ b/engine/testutil/nodes.go @@ -853,7 +853,7 @@ func createFollowerCore( t *testing.T, node *testmock.GenericNode, followerState *badgerstate.FollowerState, - notifier hotstuff.ConsensusFollowerConsumer, + notifier hotstuff.FollowerConsumer, rootHead *flow.Header, rootQC *flow.QuorumCertificate, ) (module.HotStuffFollower, *confinalizer.Finalizer) { diff --git a/follower/follower_builder.go b/follower/follower_builder.go index 53b5adf804c..0b00a1c9e91 100644 --- a/follower/follower_builder.go +++ b/follower/follower_builder.go @@ -357,7 +357,7 @@ func FlowConsensusFollowerService(opts ...FollowerOption) *FollowerServiceBuilde FlowNodeBuilder: cmd.FlowNode(flow.RoleAccess.String(), config.baseOptions...), FollowerDistributor: pubsub.NewFollowerDistributor(), } - ret.FollowerDistributor.AddConsumer(notifications.NewSlashingViolationsConsumer(ret.Logger)) + ret.FollowerDistributor.AddFollowerConsumer(notifications.NewSlashingViolationsConsumer(ret.Logger)) // the observer gets a version of the root snapshot file that does not contain any node addresses // hence skip all the root snapshot validations that involved an identity address ret.FlowNodeBuilder.SkipNwAddressBasedValidations = true From a8fede0a74c4388c02662d1e82e241f1d3efaca1 Mon Sep 17 00:00:00 2001 From: Alexander Hentschel Date: Sat, 6 May 2023 20:46:02 -0700 Subject: [PATCH 18/38] fixed compile errors --- .../node_builder/access_node_builder.go | 2 +- cmd/collection/main.go | 2 +- cmd/execution_builder.go | 2 +- cmd/observer/node_builder/observer_builder.go | 2 +- cmd/verification_builder.go | 2 +- .../pubsub/communicator_distributor.go | 2 +- .../notifications/pubsub/distributor.go | 21 ++++++++++++------- .../pubsub/participant_distributor.go | 2 +- .../pubsub/qc_created_distributor.go | 4 +--- .../collection/epochmgr/factories/hotstuff.go | 8 +++---- follower/follower_builder.go | 2 +- 11 files changed, 26 insertions(+), 23 deletions(-) diff --git a/cmd/access/node_builder/access_node_builder.go b/cmd/access/node_builder/access_node_builder.go index fffdde44a4b..4ac6fb815b2 100644 --- a/cmd/access/node_builder/access_node_builder.go +++ b/cmd/access/node_builder/access_node_builder.go @@ -609,7 +609,7 @@ func (builder *FlowAccessNodeBuilder) BuildExecutionDataRequester() *FlowAccessN func FlowAccessNode(nodeBuilder *cmd.FlowNodeBuilder) *FlowAccessNodeBuilder { dist := consensuspubsub.NewFollowerDistributor() - dist.AddFollowerConsumer(notifications.NewSlashingViolationsConsumer(nodeBuilder.Logger)) + dist.AddProtocolViolationConsumer(notifications.NewSlashingViolationsConsumer(nodeBuilder.Logger)) return &FlowAccessNodeBuilder{ AccessNodeConfig: DefaultAccessNodeConfig(), FlowNodeBuilder: nodeBuilder, diff --git a/cmd/collection/main.go b/cmd/collection/main.go index 0d87a13141f..f4af64334ad 100644 --- a/cmd/collection/main.go +++ b/cmd/collection/main.go @@ -173,7 +173,7 @@ func main() { PreInit(cmd.DynamicStartPreInit). Module("follower distributor", func(node *cmd.NodeConfig) error { followerDistributor = pubsub.NewFollowerDistributor() - followerDistributor.AddFollowerConsumer(notifications.NewSlashingViolationsConsumer(node.Logger)) + followerDistributor.AddProtocolViolationConsumer(notifications.NewSlashingViolationsConsumer(node.Logger)) return nil }). Module("mutable follower state", func(node *cmd.NodeConfig) error { diff --git a/cmd/execution_builder.go b/cmd/execution_builder.go index 0bfd0ad10a0..48baf39f9b7 100644 --- a/cmd/execution_builder.go +++ b/cmd/execution_builder.go @@ -854,7 +854,7 @@ func (exeNode *ExecutionNode) LoadFollowerCore( return nil, fmt.Errorf("could not find latest finalized block and pending blocks to recover consensus follower: %w", err) } - exeNode.followerDistributor.AddFollowerConsumer(exeNode.checkerEng) + exeNode.followerDistributor.AddFinalizationConsumer(exeNode.checkerEng) // creates a consensus follower with ingestEngine as the notifier // so that it gets notified upon each new finalized block diff --git a/cmd/observer/node_builder/observer_builder.go b/cmd/observer/node_builder/observer_builder.go index 508af73d311..4bb052ef55a 100644 --- a/cmd/observer/node_builder/observer_builder.go +++ b/cmd/observer/node_builder/observer_builder.go @@ -566,7 +566,7 @@ func NewFlowObserverServiceBuilder(opts ...Option) *ObserverServiceBuilder { FlowNodeBuilder: cmd.FlowNode("observer"), FollowerDistributor: pubsub.NewFollowerDistributor(), } - anb.FollowerDistributor.AddFollowerConsumer(notifications.NewSlashingViolationsConsumer(anb.Logger)) + anb.FollowerDistributor.AddProtocolViolationConsumer(notifications.NewSlashingViolationsConsumer(anb.Logger)) // the observer gets a version of the root snapshot file that does not contain any node addresses // hence skip all the root snapshot validations that involved an identity address anb.FlowNodeBuilder.SkipNwAddressBasedValidations = true diff --git a/cmd/verification_builder.go b/cmd/verification_builder.go index d7f2f196d16..a4f0e11d6c8 100644 --- a/cmd/verification_builder.go +++ b/cmd/verification_builder.go @@ -179,7 +179,7 @@ func (v *VerificationNodeBuilder) LoadComponentsAndModules() { }). Module("follower distributor", func(node *NodeConfig) error { followerDistributor = pubsub.NewFollowerDistributor() - followerDistributor.AddFollowerConsumer(notifications.NewSlashingViolationsConsumer(node.Logger)) + followerDistributor.AddProtocolViolationConsumer(notifications.NewSlashingViolationsConsumer(node.Logger)) return nil }). Module("sync core", func(node *NodeConfig) error { diff --git a/consensus/hotstuff/notifications/pubsub/communicator_distributor.go b/consensus/hotstuff/notifications/pubsub/communicator_distributor.go index 521e06ee50b..094c4b9a440 100644 --- a/consensus/hotstuff/notifications/pubsub/communicator_distributor.go +++ b/consensus/hotstuff/notifications/pubsub/communicator_distributor.go @@ -21,7 +21,7 @@ type CommunicatorDistributor struct { var _ hotstuff.CommunicatorConsumer = (*CommunicatorDistributor)(nil) -func NewCommunicatorConsumerDistributor() *CommunicatorDistributor { +func NewCommunicatorDistributor() *CommunicatorDistributor { return &CommunicatorDistributor{} } diff --git a/consensus/hotstuff/notifications/pubsub/distributor.go b/consensus/hotstuff/notifications/pubsub/distributor.go index 7c600a71e88..0f1112c78d7 100644 --- a/consensus/hotstuff/notifications/pubsub/distributor.go +++ b/consensus/hotstuff/notifications/pubsub/distributor.go @@ -8,15 +8,19 @@ import ( // // It allows thread-safe subscription of multiple consumers to events. type Distributor struct { - FollowerDistributor - CommunicatorDistributor - ParticipantDistributor + *FollowerDistributor + *CommunicatorDistributor + *ParticipantDistributor } var _ hotstuff.Consumer = (*Distributor)(nil) func NewDistributor() *Distributor { - return &Distributor{} + return &Distributor{ + FollowerDistributor: NewFollowerDistributor(), + CommunicatorDistributor: NewCommunicatorDistributor(), + ParticipantDistributor: NewParticipantDistributor(), + } } // AddConsumer adds an event consumer to the Distributor @@ -28,14 +32,17 @@ func (p *Distributor) AddConsumer(consumer hotstuff.Consumer) { // FollowerDistributor ingests consensus follower events and distributes it to subscribers. type FollowerDistributor struct { - ProtocolViolationDistributor - FinalizationDistributor + *ProtocolViolationDistributor + *FinalizationDistributor } var _ hotstuff.FollowerConsumer = (*FollowerDistributor)(nil) func NewFollowerDistributor() *FollowerDistributor { - return &FollowerDistributor{} + return &FollowerDistributor{ + ProtocolViolationDistributor: NewProtocolViolationDistributor(), + FinalizationDistributor: NewFinalizationDistributor(), + } } func (d *FollowerDistributor) AddFollowerConsumer(consumer hotstuff.FollowerConsumer) { diff --git a/consensus/hotstuff/notifications/pubsub/participant_distributor.go b/consensus/hotstuff/notifications/pubsub/participant_distributor.go index f0fae001a41..47ca41846df 100644 --- a/consensus/hotstuff/notifications/pubsub/participant_distributor.go +++ b/consensus/hotstuff/notifications/pubsub/participant_distributor.go @@ -20,7 +20,7 @@ type ParticipantDistributor struct { var _ hotstuff.ParticipantConsumer = (*ParticipantDistributor)(nil) -func NewConsensusParticipantDistributor() *ParticipantDistributor { +func NewParticipantDistributor() *ParticipantDistributor { return &ParticipantDistributor{} } diff --git a/consensus/hotstuff/notifications/pubsub/qc_created_distributor.go b/consensus/hotstuff/notifications/pubsub/qc_created_distributor.go index b9adafc2567..481c3a6acf3 100644 --- a/consensus/hotstuff/notifications/pubsub/qc_created_distributor.go +++ b/consensus/hotstuff/notifications/pubsub/qc_created_distributor.go @@ -19,9 +19,7 @@ type QCCreatedDistributor struct { var _ hotstuff.QCCreatedConsumer = (*QCCreatedDistributor)(nil) func NewQCCreatedDistributor() *QCCreatedDistributor { - return &QCCreatedDistributor{ - qcCreatedConsumers: make([]hotstuff.QCCreatedConsumer, 0), - } + return &QCCreatedDistributor{} } func (d *QCCreatedDistributor) AddQCCreatedConsumer(consumer hotstuff.QCCreatedConsumer) { diff --git a/engine/collection/epochmgr/factories/hotstuff.go b/engine/collection/epochmgr/factories/hotstuff.go index 5eafd066ca7..f4c3524a553 100644 --- a/engine/collection/epochmgr/factories/hotstuff.go +++ b/engine/collection/epochmgr/factories/hotstuff.go @@ -76,12 +76,10 @@ func (f *HotStuffFactory) CreateModules( log := f.createLogger(cluster) metrics := f.createMetrics(cluster.ChainID()) notifier := pubsub.NewDistributor() - followerDistributor := pubsub.NewFollowerDistributor() - notifier.AddFollowerConsumer(followerDistributor) notifier.AddConsumer(notifications.NewLogConsumer(log)) notifier.AddConsumer(hotmetrics.NewMetricsConsumer(metrics)) notifier.AddConsumer(notifications.NewTelemetryConsumer(log)) - notifier.AddConsumer(notifications.NewSlashingViolationsConsumer(log)) + notifier.AddProtocolViolationConsumer(notifications.NewSlashingViolationsConsumer(log)) var ( err error @@ -129,7 +127,7 @@ func (f *HotStuffFactory) CreateModules( finalizedBlock.View+1, notifier, voteProcessorFactory, - followerDistributor, + notifier.FollowerDistributor, ) if err != nil { return nil, nil, err @@ -163,7 +161,7 @@ func (f *HotStuffFactory) CreateModules( TimeoutAggregator: timeoutAggregator, QCCreatedDistributor: qcDistributor, TimeoutCollectorDistributor: timeoutCollectorDistributor, - FollowerDistributor: followerDistributor, + FollowerDistributor: notifier.FollowerDistributor, }, metrics, nil } diff --git a/follower/follower_builder.go b/follower/follower_builder.go index 0b00a1c9e91..caa0773e546 100644 --- a/follower/follower_builder.go +++ b/follower/follower_builder.go @@ -357,7 +357,7 @@ func FlowConsensusFollowerService(opts ...FollowerOption) *FollowerServiceBuilde FlowNodeBuilder: cmd.FlowNode(flow.RoleAccess.String(), config.baseOptions...), FollowerDistributor: pubsub.NewFollowerDistributor(), } - ret.FollowerDistributor.AddFollowerConsumer(notifications.NewSlashingViolationsConsumer(ret.Logger)) + ret.FollowerDistributor.AddProtocolViolationConsumer(notifications.NewSlashingViolationsConsumer(ret.Logger)) // the observer gets a version of the root snapshot file that does not contain any node addresses // hence skip all the root snapshot validations that involved an identity address ret.FlowNodeBuilder.SkipNwAddressBasedValidations = true From 92c7f97831c294c4371be3afdbde9c82f3c008e0 Mon Sep 17 00:00:00 2001 From: Alexander Hentschel Date: Sun, 7 May 2023 20:52:54 -0700 Subject: [PATCH 19/38] updated mocks --- .../mocks/consensus_follower_consumer.go | 48 --------- consensus/hotstuff/mocks/follower_consumer.go | 73 +++++++++++++ .../hotstuff/mocks/participant_consumer.go | 102 ++++++++++++++++++ .../mocks/protocol_violation_consumer.go | 25 +++++ 4 files changed, 200 insertions(+), 48 deletions(-) delete mode 100644 consensus/hotstuff/mocks/consensus_follower_consumer.go create mode 100644 consensus/hotstuff/mocks/follower_consumer.go create mode 100644 consensus/hotstuff/mocks/participant_consumer.go diff --git a/consensus/hotstuff/mocks/consensus_follower_consumer.go b/consensus/hotstuff/mocks/consensus_follower_consumer.go deleted file mode 100644 index f5a7de1259c..00000000000 --- a/consensus/hotstuff/mocks/consensus_follower_consumer.go +++ /dev/null @@ -1,48 +0,0 @@ -// Code generated by mockery v2.21.4. DO NOT EDIT. - -package mocks - -import ( - model "github.com/onflow/flow-go/consensus/hotstuff/model" - mock "github.com/stretchr/testify/mock" -) - -// ConsensusFollowerConsumer is an autogenerated mock type for the ConsensusFollowerConsumer type -type ConsensusFollowerConsumer struct { - mock.Mock -} - -// OnBlockIncorporated provides a mock function with given fields: _a0 -func (_m *ConsensusFollowerConsumer) OnBlockIncorporated(_a0 *model.Block) { - _m.Called(_a0) -} - -// OnDoubleProposeDetected provides a mock function with given fields: _a0, _a1 -func (_m *ConsensusFollowerConsumer) OnDoubleProposeDetected(_a0 *model.Block, _a1 *model.Block) { - _m.Called(_a0, _a1) -} - -// OnFinalizedBlock provides a mock function with given fields: _a0 -func (_m *ConsensusFollowerConsumer) OnFinalizedBlock(_a0 *model.Block) { - _m.Called(_a0) -} - -// OnInvalidBlockDetected provides a mock function with given fields: err -func (_m *ConsensusFollowerConsumer) OnInvalidBlockDetected(err model.InvalidBlockError) { - _m.Called(err) -} - -type mockConstructorTestingTNewConsensusFollowerConsumer interface { - mock.TestingT - Cleanup(func()) -} - -// NewConsensusFollowerConsumer creates a new instance of ConsensusFollowerConsumer. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewConsensusFollowerConsumer(t mockConstructorTestingTNewConsensusFollowerConsumer) *ConsensusFollowerConsumer { - mock := &ConsensusFollowerConsumer{} - mock.Mock.Test(t) - - t.Cleanup(func() { mock.AssertExpectations(t) }) - - return mock -} diff --git a/consensus/hotstuff/mocks/follower_consumer.go b/consensus/hotstuff/mocks/follower_consumer.go new file mode 100644 index 00000000000..a8dcd9c9681 --- /dev/null +++ b/consensus/hotstuff/mocks/follower_consumer.go @@ -0,0 +1,73 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mocks + +import ( + model "github.com/onflow/flow-go/consensus/hotstuff/model" + mock "github.com/stretchr/testify/mock" +) + +// FollowerConsumer is an autogenerated mock type for the FollowerConsumer type +type FollowerConsumer struct { + mock.Mock +} + +// OnBlockIncorporated provides a mock function with given fields: _a0 +func (_m *FollowerConsumer) OnBlockIncorporated(_a0 *model.Block) { + _m.Called(_a0) +} + +// OnDoubleProposeDetected provides a mock function with given fields: _a0, _a1 +func (_m *FollowerConsumer) OnDoubleProposeDetected(_a0 *model.Block, _a1 *model.Block) { + _m.Called(_a0, _a1) +} + +// OnDoubleTimeoutDetected provides a mock function with given fields: _a0, _a1 +func (_m *FollowerConsumer) OnDoubleTimeoutDetected(_a0 *model.TimeoutObject, _a1 *model.TimeoutObject) { + _m.Called(_a0, _a1) +} + +// OnDoubleVotingDetected provides a mock function with given fields: _a0, _a1 +func (_m *FollowerConsumer) OnDoubleVotingDetected(_a0 *model.Vote, _a1 *model.Vote) { + _m.Called(_a0, _a1) +} + +// OnFinalizedBlock provides a mock function with given fields: _a0 +func (_m *FollowerConsumer) OnFinalizedBlock(_a0 *model.Block) { + _m.Called(_a0) +} + +// OnInvalidBlockDetected provides a mock function with given fields: err +func (_m *FollowerConsumer) OnInvalidBlockDetected(err model.InvalidBlockError) { + _m.Called(err) +} + +// OnInvalidTimeoutDetected provides a mock function with given fields: err +func (_m *FollowerConsumer) OnInvalidTimeoutDetected(err model.InvalidTimeoutError) { + _m.Called(err) +} + +// OnInvalidVoteDetected provides a mock function with given fields: err +func (_m *FollowerConsumer) OnInvalidVoteDetected(err model.InvalidVoteError) { + _m.Called(err) +} + +// OnVoteForInvalidBlockDetected provides a mock function with given fields: vote, invalidProposal +func (_m *FollowerConsumer) OnVoteForInvalidBlockDetected(vote *model.Vote, invalidProposal *model.Proposal) { + _m.Called(vote, invalidProposal) +} + +type mockConstructorTestingTNewFollowerConsumer interface { + mock.TestingT + Cleanup(func()) +} + +// NewFollowerConsumer creates a new instance of FollowerConsumer. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewFollowerConsumer(t mockConstructorTestingTNewFollowerConsumer) *FollowerConsumer { + mock := &FollowerConsumer{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/consensus/hotstuff/mocks/participant_consumer.go b/consensus/hotstuff/mocks/participant_consumer.go new file mode 100644 index 00000000000..739ced2b00d --- /dev/null +++ b/consensus/hotstuff/mocks/participant_consumer.go @@ -0,0 +1,102 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mocks + +import ( + hotstuff "github.com/onflow/flow-go/consensus/hotstuff" + flow "github.com/onflow/flow-go/model/flow" + + mock "github.com/stretchr/testify/mock" + + model "github.com/onflow/flow-go/consensus/hotstuff/model" +) + +// ParticipantConsumer is an autogenerated mock type for the ParticipantConsumer type +type ParticipantConsumer struct { + mock.Mock +} + +// OnCurrentViewDetails provides a mock function with given fields: currentView, finalizedView, currentLeader +func (_m *ParticipantConsumer) OnCurrentViewDetails(currentView uint64, finalizedView uint64, currentLeader flow.Identifier) { + _m.Called(currentView, finalizedView, currentLeader) +} + +// OnEventProcessed provides a mock function with given fields: +func (_m *ParticipantConsumer) OnEventProcessed() { + _m.Called() +} + +// OnLocalTimeout provides a mock function with given fields: currentView +func (_m *ParticipantConsumer) OnLocalTimeout(currentView uint64) { + _m.Called(currentView) +} + +// OnPartialTc provides a mock function with given fields: currentView, partialTc +func (_m *ParticipantConsumer) OnPartialTc(currentView uint64, partialTc *hotstuff.PartialTcCreated) { + _m.Called(currentView, partialTc) +} + +// OnQcTriggeredViewChange provides a mock function with given fields: oldView, newView, qc +func (_m *ParticipantConsumer) OnQcTriggeredViewChange(oldView uint64, newView uint64, qc *flow.QuorumCertificate) { + _m.Called(oldView, newView, qc) +} + +// OnReceiveProposal provides a mock function with given fields: currentView, proposal +func (_m *ParticipantConsumer) OnReceiveProposal(currentView uint64, proposal *model.Proposal) { + _m.Called(currentView, proposal) +} + +// OnReceiveQc provides a mock function with given fields: currentView, qc +func (_m *ParticipantConsumer) OnReceiveQc(currentView uint64, qc *flow.QuorumCertificate) { + _m.Called(currentView, qc) +} + +// OnReceiveTc provides a mock function with given fields: currentView, tc +func (_m *ParticipantConsumer) OnReceiveTc(currentView uint64, tc *flow.TimeoutCertificate) { + _m.Called(currentView, tc) +} + +// OnStart provides a mock function with given fields: currentView +func (_m *ParticipantConsumer) OnStart(currentView uint64) { + _m.Called(currentView) +} + +// OnStartingTimeout provides a mock function with given fields: _a0 +func (_m *ParticipantConsumer) OnStartingTimeout(_a0 model.TimerInfo) { + _m.Called(_a0) +} + +// OnTcTriggeredViewChange provides a mock function with given fields: oldView, newView, tc +func (_m *ParticipantConsumer) OnTcTriggeredViewChange(oldView uint64, newView uint64, tc *flow.TimeoutCertificate) { + _m.Called(oldView, newView, tc) +} + +// OnTimeoutProcessed provides a mock function with given fields: timeout +func (_m *ParticipantConsumer) OnTimeoutProcessed(timeout *model.TimeoutObject) { + _m.Called(timeout) +} + +// OnViewChange provides a mock function with given fields: oldView, newView +func (_m *ParticipantConsumer) OnViewChange(oldView uint64, newView uint64) { + _m.Called(oldView, newView) +} + +// OnVoteProcessed provides a mock function with given fields: vote +func (_m *ParticipantConsumer) OnVoteProcessed(vote *model.Vote) { + _m.Called(vote) +} + +type mockConstructorTestingTNewParticipantConsumer interface { + mock.TestingT + Cleanup(func()) +} + +// NewParticipantConsumer creates a new instance of ParticipantConsumer. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewParticipantConsumer(t mockConstructorTestingTNewParticipantConsumer) *ParticipantConsumer { + mock := &ParticipantConsumer{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/consensus/hotstuff/mocks/protocol_violation_consumer.go b/consensus/hotstuff/mocks/protocol_violation_consumer.go index 923c4c3cae7..b8785f0881f 100644 --- a/consensus/hotstuff/mocks/protocol_violation_consumer.go +++ b/consensus/hotstuff/mocks/protocol_violation_consumer.go @@ -17,11 +17,36 @@ func (_m *ProtocolViolationConsumer) OnDoubleProposeDetected(_a0 *model.Block, _ _m.Called(_a0, _a1) } +// OnDoubleTimeoutDetected provides a mock function with given fields: _a0, _a1 +func (_m *ProtocolViolationConsumer) OnDoubleTimeoutDetected(_a0 *model.TimeoutObject, _a1 *model.TimeoutObject) { + _m.Called(_a0, _a1) +} + +// OnDoubleVotingDetected provides a mock function with given fields: _a0, _a1 +func (_m *ProtocolViolationConsumer) OnDoubleVotingDetected(_a0 *model.Vote, _a1 *model.Vote) { + _m.Called(_a0, _a1) +} + // OnInvalidBlockDetected provides a mock function with given fields: err func (_m *ProtocolViolationConsumer) OnInvalidBlockDetected(err model.InvalidBlockError) { _m.Called(err) } +// OnInvalidTimeoutDetected provides a mock function with given fields: err +func (_m *ProtocolViolationConsumer) OnInvalidTimeoutDetected(err model.InvalidTimeoutError) { + _m.Called(err) +} + +// OnInvalidVoteDetected provides a mock function with given fields: err +func (_m *ProtocolViolationConsumer) OnInvalidVoteDetected(err model.InvalidVoteError) { + _m.Called(err) +} + +// OnVoteForInvalidBlockDetected provides a mock function with given fields: vote, invalidProposal +func (_m *ProtocolViolationConsumer) OnVoteForInvalidBlockDetected(vote *model.Vote, invalidProposal *model.Proposal) { + _m.Called(vote, invalidProposal) +} + type mockConstructorTestingTNewProtocolViolationConsumer interface { mock.TestingT Cleanup(func()) From 8a43970d090685f0f630fb3c00ce77bc6c5fb3d5 Mon Sep 17 00:00:00 2001 From: Alexander Hentschel Date: Sun, 7 May 2023 20:56:24 -0700 Subject: [PATCH 20/38] fixed compile error --- consensus/follower_test.go | 4 ++-- engine/common/follower/compliance_core_test.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/consensus/follower_test.go b/consensus/follower_test.go index edbf919becf..7496c103658 100644 --- a/consensus/follower_test.go +++ b/consensus/follower_test.go @@ -58,7 +58,7 @@ type HotStuffFollowerSuite struct { headers *mockstorage.Headers finalizer *mockmodule.Finalizer - notifier *mockhotstuff.ConsensusFollowerConsumer + notifier *mockhotstuff.FollowerConsumer rootHeader *flow.Header rootQC *flow.QuorumCertificate finalized *flow.Header @@ -84,7 +84,7 @@ func (s *HotStuffFollowerSuite) SetupTest() { s.finalizer = mockmodule.NewFinalizer(s.T()) // mock consumer for finalization notifications - s.notifier = mockhotstuff.NewConsensusFollowerConsumer(s.T()) + s.notifier = mockhotstuff.NewFollowerConsumer(s.T()) // root block and QC parentID, err := flow.HexStringToIdentifier("aa7693d498e9a087b1cadf5bfe9a1ff07829badc1915c210e482f369f9a00a70") diff --git a/engine/common/follower/compliance_core_test.go b/engine/common/follower/compliance_core_test.go index 67544a083fe..ff5213c3478 100644 --- a/engine/common/follower/compliance_core_test.go +++ b/engine/common/follower/compliance_core_test.go @@ -39,7 +39,7 @@ type CoreSuite struct { follower *module.HotStuffFollower sync *module.BlockRequester validator *hotstuff.Validator - followerConsumer *hotstuff.ConsensusFollowerConsumer + followerConsumer *hotstuff.FollowerConsumer ctx irrecoverable.SignalerContext cancel context.CancelFunc @@ -52,7 +52,7 @@ func (s *CoreSuite) SetupTest() { s.follower = module.NewHotStuffFollower(s.T()) s.validator = hotstuff.NewValidator(s.T()) s.sync = module.NewBlockRequester(s.T()) - s.followerConsumer = hotstuff.NewConsensusFollowerConsumer(s.T()) + s.followerConsumer = hotstuff.NewFollowerConsumer(s.T()) s.originID = unittest.IdentifierFixture() s.finalizedBlock = unittest.BlockHeaderFixture() From 29a88aa57b77ba3ed74ec3be4753f72a27a02448 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Mon, 8 May 2023 21:18:31 +0300 Subject: [PATCH 21/38] Attempt to modularize protocol violations by source. --- cmd/consensus/main.go | 12 +++-- consensus/aggregators.go | 7 +-- consensus/hotstuff/consumer.go | 9 ++++ .../notifications/pubsub/distributor.go | 19 +++++++ .../pubsub/protocol_violation_distributor.go | 40 -------------- .../timeout_aggregation_violation_consumer.go | 44 ++++++++++++++++ .../vote_aggregation_violation_consumer.go | 52 +++++++++++++++++++ .../hotstuff/timeoutcollector/factory.go | 4 +- .../timeoutcollector/timeout_collector.go | 9 ++-- .../voteaggregator/vote_aggregator.go | 4 +- .../hotstuff/votecollector/statemachine.go | 10 ++-- 11 files changed, 152 insertions(+), 58 deletions(-) create mode 100644 consensus/hotstuff/notifications/pubsub/timeout_aggregation_violation_consumer.go create mode 100644 consensus/hotstuff/notifications/pubsub/vote_aggregation_violation_consumer.go diff --git a/cmd/consensus/main.go b/cmd/consensus/main.go index fae7ba475a6..5fc0f25b867 100644 --- a/cmd/consensus/main.go +++ b/cmd/consensus/main.go @@ -583,6 +583,8 @@ func main() { } qcDistributor := pubsub.NewQCCreatedDistributor() + // TODO: connect to slashing violation consumer + voteAggregationViolationDistributor := pubsub.NewVoteAggregationViolationDistributor() validator := consensus.NewValidator(mainMetrics, wrappedCommittee) voteProcessorFactory := votecollector.NewCombinedVoteProcessorFactory(wrappedCommittee, qcDistributor.OnQcConstructedFromVotes) lowestViewForVoteProcessing := finalizedBlock.View + 1 @@ -593,16 +595,18 @@ func main() { node.Metrics.Mempool, lowestViewForVoteProcessing, notifier, + voteAggregationViolationDistributor, voteProcessorFactory, followerDistributor) if err != nil { return nil, fmt.Errorf("could not initialize vote aggregator: %w", err) } - timeoutCollectorDistributor := pubsub.NewTimeoutCollectorDistributor() + // TODO: connect to slashing violation consumer + timeoutAggregationDistributor := pubsub.NewTimeoutAggregationDistributor() timeoutProcessorFactory := timeoutcollector.NewTimeoutProcessorFactory( logger, - timeoutCollectorDistributor, + timeoutAggregationDistributor, committee, validator, msig.ConsensusTimeoutTag, @@ -614,7 +618,7 @@ func main() { node.Metrics.Mempool, notifier, timeoutProcessorFactory, - timeoutCollectorDistributor, + timeoutAggregationDistributor, lowestViewForVoteProcessing, ) if err != nil { @@ -628,7 +632,7 @@ func main() { Persist: persist, QCCreatedDistributor: qcDistributor, FollowerDistributor: followerDistributor, - TimeoutCollectorDistributor: timeoutCollectorDistributor, + TimeoutCollectorDistributor: timeoutAggregationDistributor.TimeoutCollectorDistributor, Forks: forks, Validator: validator, VoteAggregator: voteAggregator, diff --git a/consensus/aggregators.go b/consensus/aggregators.go index b5915b98697..863ee568faf 100644 --- a/consensus/aggregators.go +++ b/consensus/aggregators.go @@ -24,11 +24,12 @@ func NewVoteAggregator( mempoolMetrics module.MempoolMetrics, lowestRetainedView uint64, notifier hotstuff.Consumer, + violationConsumer hotstuff.VoteAggregationViolationConsumer, voteProcessorFactory hotstuff.VoteProcessorFactory, distributor *pubsub.FollowerDistributor, ) (hotstuff.VoteAggregator, error) { - createCollectorFactoryMethod := votecollector.NewStateMachineFactory(log, notifier, voteProcessorFactory.Create) + createCollectorFactoryMethod := votecollector.NewStateMachineFactory(log, notifier, violationConsumer, voteProcessorFactory.Create) voteCollectors := voteaggregator.NewVoteCollectors(log, lowestRetainedView, workerpool.New(4), createCollectorFactoryMethod) // initialize the vote aggregator @@ -37,7 +38,7 @@ func NewVoteAggregator( hotstuffMetrics, engineMetrics, mempoolMetrics, - notifier, + violationConsumer, lowestRetainedView, voteCollectors, ) @@ -57,7 +58,7 @@ func NewTimeoutAggregator(log zerolog.Logger, mempoolMetrics module.MempoolMetrics, notifier *pubsub.Distributor, timeoutProcessorFactory hotstuff.TimeoutProcessorFactory, - distributor *pubsub.TimeoutCollectorDistributor, + distributor *pubsub.TimeoutAggregationDistributor, lowestRetainedView uint64, ) (hotstuff.TimeoutAggregator, error) { diff --git a/consensus/hotstuff/consumer.go b/consensus/hotstuff/consumer.go index 43ab4dc012e..1604bef69f3 100644 --- a/consensus/hotstuff/consumer.go +++ b/consensus/hotstuff/consumer.go @@ -31,7 +31,9 @@ type ProtocolViolationConsumer interface { // Implementation must be concurrency safe; Non-blocking; // and must handle repetition of the same events (with some processing overhead). OnDoubleProposeDetected(*model.Block, *model.Block) +} +type VoteAggregationViolationConsumer interface { // OnDoubleVotingDetected notifications are produced by the Vote Aggregation logic // whenever a double voting (same voter voting for different blocks at the same view) was detected. // Prerequisites: @@ -52,7 +54,9 @@ type ProtocolViolationConsumer interface { // Implementation must be concurrency safe; Non-blocking; // and must handle repetition of the same events (with some processing overhead). OnVoteForInvalidBlockDetected(vote *model.Vote, invalidProposal *model.Proposal) +} +type TimeoutAggregationViolationConsumer interface { // OnDoubleTimeoutDetected notifications are produced by the Timeout Aggregation logic // whenever a double timeout (same replica producing two different timeouts at the same view) was detected. // Prerequisites: @@ -304,6 +308,11 @@ type TimeoutCollectorConsumer interface { OnNewTcDiscovered(certificate *flow.TimeoutCertificate) } +type TimeoutAggregationConsumer interface { + TimeoutAggregationViolationConsumer + TimeoutCollectorConsumer +} + // CommunicatorConsumer consumes outbound notifications produced by HotStuff and it's components. // Notifications allow the HotStuff core algorithm to communicate with the other actors of the consensus process. // Implementations must: diff --git a/consensus/hotstuff/notifications/pubsub/distributor.go b/consensus/hotstuff/notifications/pubsub/distributor.go index 0f1112c78d7..0459584885c 100644 --- a/consensus/hotstuff/notifications/pubsub/distributor.go +++ b/consensus/hotstuff/notifications/pubsub/distributor.go @@ -49,3 +49,22 @@ func (d *FollowerDistributor) AddFollowerConsumer(consumer hotstuff.FollowerCons d.FinalizationDistributor.AddFinalizationConsumer(consumer) d.ProtocolViolationDistributor.AddProtocolViolationConsumer(consumer) } + +type TimeoutAggregationDistributor struct { + *TimeoutAggregationViolationDistributor + *TimeoutCollectorDistributor +} + +var _ hotstuff.TimeoutAggregationConsumer = (*TimeoutAggregationDistributor)(nil) + +func NewTimeoutAggregationDistributor() *TimeoutAggregationDistributor { + return &TimeoutAggregationDistributor{ + TimeoutAggregationViolationDistributor: NewTimeoutAggregationViolationDistributor(), + TimeoutCollectorDistributor: NewTimeoutCollectorDistributor(), + } +} + +func (d *TimeoutAggregationDistributor) AddTimeoutAggregationConsumer(consumer hotstuff.TimeoutAggregationConsumer) { + d.TimeoutAggregationViolationDistributor.AddTimeoutAggregationViolationConsumer(consumer) + d.TimeoutCollectorDistributor.AddTimeoutCollectorConsumer(consumer) +} diff --git a/consensus/hotstuff/notifications/pubsub/protocol_violation_distributor.go b/consensus/hotstuff/notifications/pubsub/protocol_violation_distributor.go index 4c08dacddac..b03393263fe 100644 --- a/consensus/hotstuff/notifications/pubsub/protocol_violation_distributor.go +++ b/consensus/hotstuff/notifications/pubsub/protocol_violation_distributor.go @@ -43,43 +43,3 @@ func (d *ProtocolViolationDistributor) OnDoubleProposeDetected(block1, block2 *m subscriber.OnDoubleProposeDetected(block1, block2) } } - -func (d *ProtocolViolationDistributor) OnDoubleVotingDetected(vote1, vote2 *model.Vote) { - d.lock.RLock() - defer d.lock.RUnlock() - for _, subscriber := range d.subscribers { - subscriber.OnDoubleVotingDetected(vote1, vote2) - } -} - -func (d *ProtocolViolationDistributor) OnInvalidVoteDetected(err model.InvalidVoteError) { - d.lock.RLock() - defer d.lock.RUnlock() - for _, subscriber := range d.subscribers { - subscriber.OnInvalidVoteDetected(err) - } -} - -func (d *ProtocolViolationDistributor) OnVoteForInvalidBlockDetected(vote *model.Vote, invalidProposal *model.Proposal) { - d.lock.RLock() - defer d.lock.RUnlock() - for _, subscriber := range d.subscribers { - subscriber.OnVoteForInvalidBlockDetected(vote, invalidProposal) - } -} - -func (d *ProtocolViolationDistributor) OnDoubleTimeoutDetected(timeout *model.TimeoutObject, altTimeout *model.TimeoutObject) { - d.lock.RLock() - defer d.lock.RUnlock() - for _, subscriber := range d.subscribers { - subscriber.OnDoubleTimeoutDetected(timeout, altTimeout) - } -} - -func (d *ProtocolViolationDistributor) OnInvalidTimeoutDetected(err model.InvalidTimeoutError) { - d.lock.RLock() - defer d.lock.RUnlock() - for _, subscriber := range d.subscribers { - subscriber.OnInvalidTimeoutDetected(err) - } -} diff --git a/consensus/hotstuff/notifications/pubsub/timeout_aggregation_violation_consumer.go b/consensus/hotstuff/notifications/pubsub/timeout_aggregation_violation_consumer.go new file mode 100644 index 00000000000..db07ac9a82a --- /dev/null +++ b/consensus/hotstuff/notifications/pubsub/timeout_aggregation_violation_consumer.go @@ -0,0 +1,44 @@ +package pubsub + +import ( + "github.com/onflow/flow-go/consensus/hotstuff" + "github.com/onflow/flow-go/consensus/hotstuff/model" + "sync" +) + +// TimeoutAggregationViolationDistributor ingests notifications about HotStuff-protocol violations and +// distributes them to subscribers. Such notifications are produced by the active consensus +// participants and to a lesser degree also the consensus follower. +// Concurrently safe. +type TimeoutAggregationViolationDistributor struct { + subscribers []hotstuff.TimeoutAggregationViolationConsumer + lock sync.RWMutex +} + +var _ hotstuff.TimeoutAggregationViolationConsumer = (*TimeoutAggregationViolationDistributor)(nil) + +func NewTimeoutAggregationViolationDistributor() *TimeoutAggregationViolationDistributor { + return &TimeoutAggregationViolationDistributor{} +} + +func (d *TimeoutAggregationViolationDistributor) AddTimeoutAggregationViolationConsumer(consumer hotstuff.TimeoutAggregationViolationConsumer) { + d.lock.Lock() + defer d.lock.Unlock() + d.subscribers = append(d.subscribers, consumer) +} + +func (d *TimeoutAggregationViolationDistributor) OnDoubleTimeoutDetected(timeout *model.TimeoutObject, altTimeout *model.TimeoutObject) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, subscriber := range d.subscribers { + subscriber.OnDoubleTimeoutDetected(timeout, altTimeout) + } +} + +func (d *TimeoutAggregationViolationDistributor) OnInvalidTimeoutDetected(err model.InvalidTimeoutError) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, subscriber := range d.subscribers { + subscriber.OnInvalidTimeoutDetected(err) + } +} diff --git a/consensus/hotstuff/notifications/pubsub/vote_aggregation_violation_consumer.go b/consensus/hotstuff/notifications/pubsub/vote_aggregation_violation_consumer.go new file mode 100644 index 00000000000..33cee62ddf2 --- /dev/null +++ b/consensus/hotstuff/notifications/pubsub/vote_aggregation_violation_consumer.go @@ -0,0 +1,52 @@ +package pubsub + +import ( + "github.com/onflow/flow-go/consensus/hotstuff" + "github.com/onflow/flow-go/consensus/hotstuff/model" + "sync" +) + +// VoteAggregationViolationDistributor ingests notifications about HotStuff-protocol violations and +// distributes them to subscribers. Such notifications are produced by the active consensus +// participants and to a lesser degree also the consensus follower. +// Concurrently safe. +type VoteAggregationViolationDistributor struct { + subscribers []hotstuff.VoteAggregationViolationConsumer + lock sync.RWMutex +} + +var _ hotstuff.VoteAggregationViolationConsumer = (*VoteAggregationViolationDistributor)(nil) + +func NewVoteAggregationViolationDistributor() *VoteAggregationViolationDistributor { + return &VoteAggregationViolationDistributor{} +} + +func (d *VoteAggregationViolationDistributor) AddVoteAggregationViolationConsumer(consumer hotstuff.VoteAggregationViolationConsumer) { + d.lock.Lock() + defer d.lock.Unlock() + d.subscribers = append(d.subscribers, consumer) +} + +func (d *VoteAggregationViolationDistributor) OnDoubleVotingDetected(vote1, vote2 *model.Vote) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, subscriber := range d.subscribers { + subscriber.OnDoubleVotingDetected(vote1, vote2) + } +} + +func (d *VoteAggregationViolationDistributor) OnInvalidVoteDetected(err model.InvalidVoteError) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, subscriber := range d.subscribers { + subscriber.OnInvalidVoteDetected(err) + } +} + +func (d *VoteAggregationViolationDistributor) OnVoteForInvalidBlockDetected(vote *model.Vote, invalidProposal *model.Proposal) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, subscriber := range d.subscribers { + subscriber.OnVoteForInvalidBlockDetected(vote, invalidProposal) + } +} diff --git a/consensus/hotstuff/timeoutcollector/factory.go b/consensus/hotstuff/timeoutcollector/factory.go index e76c441d1ec..0f18b20b748 100644 --- a/consensus/hotstuff/timeoutcollector/factory.go +++ b/consensus/hotstuff/timeoutcollector/factory.go @@ -13,7 +13,7 @@ import ( type TimeoutCollectorFactory struct { log zerolog.Logger notifier hotstuff.Consumer - collectorNotifier hotstuff.TimeoutCollectorConsumer + collectorNotifier hotstuff.TimeoutAggregationConsumer processorFactory hotstuff.TimeoutProcessorFactory } @@ -23,7 +23,7 @@ var _ hotstuff.TimeoutCollectorFactory = (*TimeoutCollectorFactory)(nil) // No error returns are expected during normal operations. func NewTimeoutCollectorFactory(log zerolog.Logger, notifier hotstuff.Consumer, - collectorNotifier hotstuff.TimeoutCollectorConsumer, + collectorNotifier hotstuff.TimeoutAggregationConsumer, createProcessor hotstuff.TimeoutProcessorFactory, ) *TimeoutCollectorFactory { return &TimeoutCollectorFactory{ diff --git a/consensus/hotstuff/timeoutcollector/timeout_collector.go b/consensus/hotstuff/timeoutcollector/timeout_collector.go index 28a9dc6f2d6..d00a22b30e7 100644 --- a/consensus/hotstuff/timeoutcollector/timeout_collector.go +++ b/consensus/hotstuff/timeoutcollector/timeout_collector.go @@ -19,7 +19,7 @@ type TimeoutCollector struct { log zerolog.Logger notifier hotstuff.Consumer timeoutsCache *TimeoutObjectsCache // cache for tracking double timeout and timeout equivocation - collectorNotifier hotstuff.TimeoutCollectorConsumer + collectorNotifier hotstuff.TimeoutAggregationConsumer processor hotstuff.TimeoutProcessor newestReportedQC counters.StrictMonotonousCounter // view of newest QC that was reported newestReportedTC counters.StrictMonotonousCounter // view of newest TC that was reported @@ -31,7 +31,7 @@ var _ hotstuff.TimeoutCollector = (*TimeoutCollector)(nil) func NewTimeoutCollector(log zerolog.Logger, view uint64, notifier hotstuff.Consumer, - collectorNotifier hotstuff.TimeoutCollectorConsumer, + collectorNotifier hotstuff.TimeoutAggregationConsumer, processor hotstuff.TimeoutProcessor, ) *TimeoutCollector { return &TimeoutCollector{ @@ -64,7 +64,7 @@ func (c *TimeoutCollector) AddTimeout(timeout *model.TimeoutObject) error { return nil } if doubleTimeoutErr, isDoubleTimeoutErr := model.AsDoubleTimeoutError(err); isDoubleTimeoutErr { - c.notifier.OnDoubleTimeoutDetected(doubleTimeoutErr.FirstTimeout, doubleTimeoutErr.ConflictingTimeout) + c.collectorNotifier.OnDoubleTimeoutDetected(doubleTimeoutErr.FirstTimeout, doubleTimeoutErr.ConflictingTimeout) return nil } return fmt.Errorf("internal error adding timeout %v to cache for view: %d: %w", timeout.ID(), timeout.View, err) @@ -85,12 +85,13 @@ func (c *TimeoutCollector) processTimeout(timeout *model.TimeoutObject) error { err := c.processor.Process(timeout) if err != nil { if invalidTimeoutErr, ok := model.AsInvalidTimeoutError(err); ok { - c.notifier.OnInvalidTimeoutDetected(*invalidTimeoutErr) + c.collectorNotifier.OnInvalidTimeoutDetected(*invalidTimeoutErr) return nil } return fmt.Errorf("internal error while processing timeout: %w", err) } + // TODO: consider moving OnTimeoutProcessed to TimeoutAggregationConsumer, need to fix telemetry for this. c.notifier.OnTimeoutProcessed(timeout) // In the following, we emit notifications about new QCs, if their view is newer than any QC previously diff --git a/consensus/hotstuff/voteaggregator/vote_aggregator.go b/consensus/hotstuff/voteaggregator/vote_aggregator.go index 6f0063f0037..be0ef5981dc 100644 --- a/consensus/hotstuff/voteaggregator/vote_aggregator.go +++ b/consensus/hotstuff/voteaggregator/vote_aggregator.go @@ -37,7 +37,7 @@ type VoteAggregator struct { log zerolog.Logger hotstuffMetrics module.HotstuffMetrics engineMetrics module.EngineMetrics - notifier hotstuff.Consumer + notifier hotstuff.VoteAggregationViolationConsumer lowestRetainedView counters.StrictMonotonousCounter // lowest view, for which we still process votes collectors hotstuff.VoteCollectors queuedMessagesNotifier engine.Notifier @@ -58,7 +58,7 @@ func NewVoteAggregator( hotstuffMetrics module.HotstuffMetrics, engineMetrics module.EngineMetrics, mempoolMetrics module.MempoolMetrics, - notifier hotstuff.Consumer, + notifier hotstuff.VoteAggregationViolationConsumer, lowestRetainedView uint64, collectors hotstuff.VoteCollectors, ) (*VoteAggregator, error) { diff --git a/consensus/hotstuff/votecollector/statemachine.go b/consensus/hotstuff/votecollector/statemachine.go index 6b7173196ab..4b207f819fc 100644 --- a/consensus/hotstuff/votecollector/statemachine.go +++ b/consensus/hotstuff/votecollector/statemachine.go @@ -26,6 +26,7 @@ type VoteCollector struct { log zerolog.Logger workers hotstuff.Workers notifier hotstuff.Consumer + violationConsumer hotstuff.VoteAggregationViolationConsumer createVerifyingProcessor VerifyingVoteProcessorFactory votesCache VotesCache @@ -48,10 +49,11 @@ type atomicValueWrapper struct { func NewStateMachineFactory( log zerolog.Logger, notifier hotstuff.Consumer, + violationConsumer hotstuff.VoteAggregationViolationConsumer, verifyingVoteProcessorFactory VerifyingVoteProcessorFactory, ) voteaggregator.NewCollectorFactoryMethod { return func(view uint64, workers hotstuff.Workers) (hotstuff.VoteCollector, error) { - return NewStateMachine(view, log, workers, notifier, verifyingVoteProcessorFactory), nil + return NewStateMachine(view, log, workers, notifier, violationConsumer, verifyingVoteProcessorFactory), nil } } @@ -60,6 +62,7 @@ func NewStateMachine( log zerolog.Logger, workers hotstuff.Workers, notifier hotstuff.Consumer, + violationConsumer hotstuff.VoteAggregationViolationConsumer, verifyingVoteProcessorFactory VerifyingVoteProcessorFactory, ) *VoteCollector { log = log.With(). @@ -70,6 +73,7 @@ func NewStateMachine( log: log, workers: workers, notifier: notifier, + violationConsumer: violationConsumer, createVerifyingProcessor: verifyingVoteProcessorFactory, votesCache: *NewVotesCache(view), } @@ -92,7 +96,7 @@ func (m *VoteCollector) AddVote(vote *model.Vote) error { return nil } if doubleVoteErr, isDoubleVoteErr := model.AsDoubleVoteError(err); isDoubleVoteErr { - m.notifier.OnDoubleVotingDetected(doubleVoteErr.FirstVote, doubleVoteErr.ConflictingVote) + m.violationConsumer.OnDoubleVotingDetected(doubleVoteErr.FirstVote, doubleVoteErr.ConflictingVote) return nil } return fmt.Errorf("internal error adding vote %v to cache for block %v: %w", @@ -131,7 +135,7 @@ func (m *VoteCollector) processVote(vote *model.Vote) error { err := processor.Process(vote) if err != nil { if invalidVoteErr, ok := model.AsInvalidVoteError(err); ok { - m.notifier.OnInvalidVoteDetected(*invalidVoteErr) + m.violationConsumer.OnInvalidVoteDetected(*invalidVoteErr) return nil } // ATTENTION: due to how our logic is designed this situation is only possible From bed19c44b9990698a588f04fc6242bf42b2c9bc3 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Tue, 9 May 2023 11:50:27 +0300 Subject: [PATCH 22/38] Renamed consumers to better reflect events emitted. Added VoteAggregationConsumer --- cmd/consensus/main.go | 9 ++- consensus/hotstuff/consumer.go | 71 ++++++++++--------- .../hotstuff/notifications/noop_consumer.go | 4 +- .../notifications/pubsub/distributor.go | 19 +++++ .../pubsub/protocol_violation_distributor.go | 6 +- .../slashing_violation_consumer.go | 2 +- engine/collection/compliance/core.go | 4 +- .../epochmgr/factories/compliance.go | 2 +- engine/common/follower/compliance_core.go | 2 +- engine/consensus/compliance/core.go | 4 +- 10 files changed, 73 insertions(+), 50 deletions(-) diff --git a/cmd/consensus/main.go b/cmd/consensus/main.go index 5fc0f25b867..ed64de61154 100644 --- a/cmd/consensus/main.go +++ b/cmd/consensus/main.go @@ -582,11 +582,10 @@ func main() { return nil, err } - qcDistributor := pubsub.NewQCCreatedDistributor() // TODO: connect to slashing violation consumer - voteAggregationViolationDistributor := pubsub.NewVoteAggregationViolationDistributor() + voteAggregationDistributor := pubsub.NewVoteAggregationDistributor() validator := consensus.NewValidator(mainMetrics, wrappedCommittee) - voteProcessorFactory := votecollector.NewCombinedVoteProcessorFactory(wrappedCommittee, qcDistributor.OnQcConstructedFromVotes) + voteProcessorFactory := votecollector.NewCombinedVoteProcessorFactory(wrappedCommittee, voteAggregationDistributor.OnQcConstructedFromVotes) lowestViewForVoteProcessing := finalizedBlock.View + 1 voteAggregator, err := consensus.NewVoteAggregator( logger, @@ -595,7 +594,7 @@ func main() { node.Metrics.Mempool, lowestViewForVoteProcessing, notifier, - voteAggregationViolationDistributor, + voteAggregationDistributor, voteProcessorFactory, followerDistributor) if err != nil { @@ -630,7 +629,7 @@ func main() { Committee: wrappedCommittee, Signer: signer, Persist: persist, - QCCreatedDistributor: qcDistributor, + QCCreatedDistributor: voteAggregationDistributor.QCCreatedDistributor, FollowerDistributor: followerDistributor, TimeoutCollectorDistributor: timeoutAggregationDistributor.TimeoutCollectorDistributor, Forks: forks, diff --git a/consensus/hotstuff/consumer.go b/consensus/hotstuff/consumer.go index 1604bef69f3..e8149622087 100644 --- a/consensus/hotstuff/consumer.go +++ b/consensus/hotstuff/consumer.go @@ -7,7 +7,7 @@ import ( "github.com/onflow/flow-go/model/flow" ) -// ProtocolViolationConsumer consumes outbound notifications about HotStuff-protocol violations. +// ProposalViolationConsumer consumes outbound notifications about HotStuff-protocol violations. // Such notifications are produced by the active consensus participants and to a lesser // degree also the consensus follower. // @@ -15,7 +15,7 @@ import ( // - be concurrency safe // - be non-blocking // - handle repetition of the same events (with some processing overhead). -type ProtocolViolationConsumer interface { +type ProposalViolationConsumer interface { // OnInvalidBlockDetected notifications are produced by components that have detected // that a block proposal is invalid and need to report it. // Most of the time such block can be detected by calling Validator.ValidateProposal. @@ -97,32 +97,6 @@ type FinalizationConsumer interface { OnFinalizedBlock(*model.Block) } -// FollowerConsumer consumes outbound notifications produced by consensus followers. -// It is a subset of the notifications produced by consensus participants. -// Implementations must: -// - be concurrency safe -// - be non-blocking -// - handle repetition of the same events (with some processing overhead). -type FollowerConsumer interface { - ProtocolViolationConsumer - FinalizationConsumer -} - -// Consumer consumes outbound notifications produced by consensus participants. -// Notifications are consensus-internal state changes which are potentially relevant to -// the larger node in which HotStuff is running. The notifications are emitted -// in the order in which the HotStuff algorithm makes the respective steps. -// -// Implementations must: -// - be concurrency safe -// - be non-blocking -// - handle repetition of the same events (with some processing overhead). -type Consumer interface { - FollowerConsumer - CommunicatorConsumer - ParticipantConsumer -} - // ParticipantConsumer consumes outbound notifications produced by consensus participants // actively proposing blocks, voting, collecting & aggregating votes to QCs, and participating in // the pacemaker (sending timeouts, collecting & aggregating timeouts to TCs). @@ -308,11 +282,6 @@ type TimeoutCollectorConsumer interface { OnNewTcDiscovered(certificate *flow.TimeoutCertificate) } -type TimeoutAggregationConsumer interface { - TimeoutAggregationViolationConsumer - TimeoutCollectorConsumer -} - // CommunicatorConsumer consumes outbound notifications produced by HotStuff and it's components. // Notifications allow the HotStuff core algorithm to communicate with the other actors of the consensus process. // Implementations must: @@ -340,3 +309,39 @@ type CommunicatorConsumer interface { // and must handle repetition of the same events (with some processing overhead). OnOwnProposal(proposal *flow.Header, targetPublicationTime time.Time) } + +// FollowerConsumer consumes outbound notifications produced by consensus followers. +// It is a subset of the notifications produced by consensus participants. +// Implementations must: +// - be concurrency safe +// - be non-blocking +// - handle repetition of the same events (with some processing overhead). +type FollowerConsumer interface { + ProposalViolationConsumer + FinalizationConsumer +} + +// Consumer consumes outbound notifications produced by consensus participants. +// Notifications are consensus-internal state changes which are potentially relevant to +// the larger node in which HotStuff is running. The notifications are emitted +// in the order in which the HotStuff algorithm makes the respective steps. +// +// Implementations must: +// - be concurrency safe +// - be non-blocking +// - handle repetition of the same events (with some processing overhead). +type Consumer interface { + FollowerConsumer + CommunicatorConsumer + ParticipantConsumer +} + +type VoteAggregationConsumer interface { + VoteAggregationViolationConsumer + QCCreatedConsumer +} + +type TimeoutAggregationConsumer interface { + TimeoutAggregationViolationConsumer + TimeoutCollectorConsumer +} diff --git a/consensus/hotstuff/notifications/noop_consumer.go b/consensus/hotstuff/notifications/noop_consumer.go index c2f02acbcae..feb676e5d89 100644 --- a/consensus/hotstuff/notifications/noop_consumer.go +++ b/consensus/hotstuff/notifications/noop_consumer.go @@ -101,11 +101,11 @@ var _ hotstuff.QCCreatedConsumer = (*NoopQCCreatedConsumer)(nil) func (*NoopQCCreatedConsumer) OnQcConstructedFromVotes(*flow.QuorumCertificate) {} -// no-op implementation of hotstuff.ProtocolViolationConsumer +// no-op implementation of hotstuff.ProposalViolationConsumer type NoopProtocolViolationConsumer struct{} -var _ hotstuff.ProtocolViolationConsumer = (*NoopProtocolViolationConsumer)(nil) +var _ hotstuff.ProposalViolationConsumer = (*NoopProtocolViolationConsumer)(nil) func (*NoopProtocolViolationConsumer) OnInvalidBlockDetected(model.InvalidBlockError) {} diff --git a/consensus/hotstuff/notifications/pubsub/distributor.go b/consensus/hotstuff/notifications/pubsub/distributor.go index 0459584885c..16a882f8741 100644 --- a/consensus/hotstuff/notifications/pubsub/distributor.go +++ b/consensus/hotstuff/notifications/pubsub/distributor.go @@ -68,3 +68,22 @@ func (d *TimeoutAggregationDistributor) AddTimeoutAggregationConsumer(consumer h d.TimeoutAggregationViolationDistributor.AddTimeoutAggregationViolationConsumer(consumer) d.TimeoutCollectorDistributor.AddTimeoutCollectorConsumer(consumer) } + +type VoteAggregationDistributor struct { + *VoteAggregationViolationDistributor + *QCCreatedDistributor +} + +var _ hotstuff.VoteAggregationConsumer = (*VoteAggregationDistributor)(nil) + +func NewVoteAggregationDistributor() *VoteAggregationDistributor { + return &VoteAggregationDistributor{ + VoteAggregationViolationDistributor: NewVoteAggregationViolationDistributor(), + QCCreatedDistributor: NewQCCreatedDistributor(), + } +} + +func (d *VoteAggregationDistributor) AddVoteAggregationConsumer(consumer hotstuff.VoteAggregationConsumer) { + d.VoteAggregationViolationDistributor.AddVoteAggregationViolationConsumer(consumer) + d.QCCreatedDistributor.AddQCCreatedConsumer(consumer) +} diff --git a/consensus/hotstuff/notifications/pubsub/protocol_violation_distributor.go b/consensus/hotstuff/notifications/pubsub/protocol_violation_distributor.go index b03393263fe..60975a5120b 100644 --- a/consensus/hotstuff/notifications/pubsub/protocol_violation_distributor.go +++ b/consensus/hotstuff/notifications/pubsub/protocol_violation_distributor.go @@ -12,17 +12,17 @@ import ( // participants and to a lesser degree also the consensus follower. // Concurrently safe. type ProtocolViolationDistributor struct { - subscribers []hotstuff.ProtocolViolationConsumer + subscribers []hotstuff.ProposalViolationConsumer lock sync.RWMutex } -var _ hotstuff.ProtocolViolationConsumer = (*ProtocolViolationDistributor)(nil) +var _ hotstuff.ProposalViolationConsumer = (*ProtocolViolationDistributor)(nil) func NewProtocolViolationDistributor() *ProtocolViolationDistributor { return &ProtocolViolationDistributor{} } -func (d *ProtocolViolationDistributor) AddProtocolViolationConsumer(consumer hotstuff.ProtocolViolationConsumer) { +func (d *ProtocolViolationDistributor) AddProtocolViolationConsumer(consumer hotstuff.ProposalViolationConsumer) { d.lock.Lock() defer d.lock.Unlock() d.subscribers = append(d.subscribers, consumer) diff --git a/consensus/hotstuff/notifications/slashing_violation_consumer.go b/consensus/hotstuff/notifications/slashing_violation_consumer.go index e67d87e3d48..be518841675 100644 --- a/consensus/hotstuff/notifications/slashing_violation_consumer.go +++ b/consensus/hotstuff/notifications/slashing_violation_consumer.go @@ -14,7 +14,7 @@ type SlashingViolationsConsumer struct { log zerolog.Logger } -var _ hotstuff.ProtocolViolationConsumer = (*SlashingViolationsConsumer)(nil) +var _ hotstuff.ProposalViolationConsumer = (*SlashingViolationsConsumer)(nil) func NewSlashingViolationsConsumer(log zerolog.Logger) *SlashingViolationsConsumer { return &SlashingViolationsConsumer{ diff --git a/engine/collection/compliance/core.go b/engine/collection/compliance/core.go index 089590742ba..879c02bc34d 100644 --- a/engine/collection/compliance/core.go +++ b/engine/collection/compliance/core.go @@ -40,7 +40,7 @@ type Core struct { mempoolMetrics module.MempoolMetrics hotstuffMetrics module.HotstuffMetrics collectionMetrics module.CollectionMetrics - protocolViolationNotifier hotstuff.ProtocolViolationConsumer + protocolViolationNotifier hotstuff.ProposalViolationConsumer headers storage.Headers state clusterkv.MutableState // track latest finalized view/height - used to efficiently drop outdated or too-far-ahead blocks @@ -61,7 +61,7 @@ func NewCore( mempool module.MempoolMetrics, hotstuffMetrics module.HotstuffMetrics, collectionMetrics module.CollectionMetrics, - protocolViolationNotifier hotstuff.ProtocolViolationConsumer, + protocolViolationNotifier hotstuff.ProposalViolationConsumer, headers storage.Headers, state clusterkv.MutableState, pending module.PendingClusterBlockBuffer, diff --git a/engine/collection/epochmgr/factories/compliance.go b/engine/collection/epochmgr/factories/compliance.go index bfa412cc021..5db39834045 100644 --- a/engine/collection/epochmgr/factories/compliance.go +++ b/engine/collection/epochmgr/factories/compliance.go @@ -58,7 +58,7 @@ func NewComplianceEngineFactory( func (f *ComplianceEngineFactory) Create( hotstuffMetrics module.HotstuffMetrics, - notifier hotstuff.ProtocolViolationConsumer, + notifier hotstuff.ProposalViolationConsumer, clusterState cluster.MutableState, headers storage.Headers, payloads storage.ClusterPayloads, diff --git a/engine/common/follower/compliance_core.go b/engine/common/follower/compliance_core.go index ed0f43abd12..89924f1d129 100644 --- a/engine/common/follower/compliance_core.go +++ b/engine/common/follower/compliance_core.go @@ -41,7 +41,7 @@ type ComplianceCore struct { log zerolog.Logger mempoolMetrics module.MempoolMetrics tracer module.Tracer - protocolViolationNotifier hotstuff.ProtocolViolationConsumer + protocolViolationNotifier hotstuff.ProposalViolationConsumer pendingCache *cache.Cache pendingTree *pending_tree.PendingTree state protocol.FollowerState diff --git a/engine/consensus/compliance/core.go b/engine/consensus/compliance/core.go index b1c61799788..28d6d87561b 100644 --- a/engine/consensus/compliance/core.go +++ b/engine/consensus/compliance/core.go @@ -42,7 +42,7 @@ type Core struct { mempoolMetrics module.MempoolMetrics hotstuffMetrics module.HotstuffMetrics complianceMetrics module.ComplianceMetrics - protocolViolationNotifier hotstuff.ProtocolViolationConsumer + protocolViolationNotifier hotstuff.ProposalViolationConsumer tracer module.Tracer headers storage.Headers payloads storage.Payloads @@ -65,7 +65,7 @@ func NewCore( mempool module.MempoolMetrics, hotstuffMetrics module.HotstuffMetrics, complianceMetrics module.ComplianceMetrics, - protocolViolationNotifier hotstuff.ProtocolViolationConsumer, + protocolViolationNotifier hotstuff.ProposalViolationConsumer, tracer module.Tracer, headers storage.Headers, payloads storage.Payloads, From fcb48c39fa86e54ab48f26871c1e3f217836db78 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Tue, 9 May 2023 12:22:45 +0300 Subject: [PATCH 23/38] Restructured vote and timeout aggregator consumers to have all source specific events. Updated relevant code --- cmd/consensus/main.go | 3 +- consensus/aggregators.go | 10 ++-- consensus/config.go | 12 ++--- consensus/hotstuff/consumer.go | 34 +++++++------- consensus/hotstuff/event_loop.go | 2 +- consensus/hotstuff/eventloop/event_loop.go | 8 +++- .../hotstuff/notifications/noop_consumer.go | 16 +++---- .../notifications/pubsub/distributor.go | 6 +-- .../pubsub/participant_distributor.go | 16 ------- .../pubsub/qc_created_distributor.go | 37 --------------- .../pubsub/timeout_collector_distributor.go | 9 ++++ .../pubsub/vote_collector_distributor.go | 46 +++++++++++++++++++ .../timeoutaggregator/timeout_aggregator.go | 3 -- .../hotstuff/timeoutcollector/factory.go | 19 ++++---- .../timeoutcollector/timeout_collector.go | 35 +++++++------- .../hotstuff/votecollector/statemachine.go | 16 +++---- 16 files changed, 132 insertions(+), 140 deletions(-) delete mode 100644 consensus/hotstuff/notifications/pubsub/qc_created_distributor.go create mode 100644 consensus/hotstuff/notifications/pubsub/vote_collector_distributor.go diff --git a/cmd/consensus/main.go b/cmd/consensus/main.go index ed64de61154..209fd314289 100644 --- a/cmd/consensus/main.go +++ b/cmd/consensus/main.go @@ -593,7 +593,6 @@ func main() { node.Metrics.Engine, node.Metrics.Mempool, lowestViewForVoteProcessing, - notifier, voteAggregationDistributor, voteProcessorFactory, followerDistributor) @@ -629,7 +628,7 @@ func main() { Committee: wrappedCommittee, Signer: signer, Persist: persist, - QCCreatedDistributor: voteAggregationDistributor.QCCreatedDistributor, + QCCreatedDistributor: voteAggregationDistributor.VoteCollectorDistributor, FollowerDistributor: followerDistributor, TimeoutCollectorDistributor: timeoutAggregationDistributor.TimeoutCollectorDistributor, Forks: forks, diff --git a/consensus/aggregators.go b/consensus/aggregators.go index 863ee568faf..ccdab038989 100644 --- a/consensus/aggregators.go +++ b/consensus/aggregators.go @@ -23,13 +23,12 @@ func NewVoteAggregator( engineMetrics module.EngineMetrics, mempoolMetrics module.MempoolMetrics, lowestRetainedView uint64, - notifier hotstuff.Consumer, - violationConsumer hotstuff.VoteAggregationViolationConsumer, + notifier hotstuff.VoteAggregationConsumer, voteProcessorFactory hotstuff.VoteProcessorFactory, distributor *pubsub.FollowerDistributor, ) (hotstuff.VoteAggregator, error) { - createCollectorFactoryMethod := votecollector.NewStateMachineFactory(log, notifier, violationConsumer, voteProcessorFactory.Create) + createCollectorFactoryMethod := votecollector.NewStateMachineFactory(log, notifier, voteProcessorFactory.Create) voteCollectors := voteaggregator.NewVoteCollectors(log, lowestRetainedView, workerpool.New(4), createCollectorFactoryMethod) // initialize the vote aggregator @@ -38,7 +37,7 @@ func NewVoteAggregator( hotstuffMetrics, engineMetrics, mempoolMetrics, - violationConsumer, + notifier, lowestRetainedView, voteCollectors, ) @@ -62,7 +61,7 @@ func NewTimeoutAggregator(log zerolog.Logger, lowestRetainedView uint64, ) (hotstuff.TimeoutAggregator, error) { - timeoutCollectorFactory := timeoutcollector.NewTimeoutCollectorFactory(log, notifier, distributor, timeoutProcessorFactory) + timeoutCollectorFactory := timeoutcollector.NewTimeoutCollectorFactory(log, distributor, timeoutProcessorFactory) collectors := timeoutaggregator.NewTimeoutCollectors(log, lowestRetainedView, timeoutCollectorFactory) // initialize the timeout aggregator @@ -71,7 +70,6 @@ func NewTimeoutAggregator(log zerolog.Logger, hotstuffMetrics, engineMetrics, mempoolMetrics, - notifier, lowestRetainedView, collectors, ) diff --git a/consensus/config.go b/consensus/config.go index 6e8497ff2cc..d91c05697fb 100644 --- a/consensus/config.go +++ b/consensus/config.go @@ -12,12 +12,12 @@ import ( // HotstuffModules is a helper structure to encapsulate dependencies to create // a hotStuff participant. type HotstuffModules struct { - Committee hotstuff.DynamicCommittee // consensus committee - Signer hotstuff.Signer // signer of proposal & votes - Persist hotstuff.Persister // last state of consensus participant - Notifier *pubsub.Distributor // observer for hotstuff events - FollowerDistributor *pubsub.FollowerDistributor // observer for finalization events, used by compliance engine - QCCreatedDistributor *pubsub.QCCreatedDistributor // observer for qc created event, used by leader + Committee hotstuff.DynamicCommittee // consensus committee + Signer hotstuff.Signer // signer of proposal & votes + Persist hotstuff.Persister // last state of consensus participant + Notifier *pubsub.Distributor // observer for hotstuff events + FollowerDistributor *pubsub.FollowerDistributor // observer for finalization events, used by compliance engine + QCCreatedDistributor *pubsub.VoteCollectorDistributor // observer for qc created event, used by leader TimeoutCollectorDistributor *pubsub.TimeoutCollectorDistributor Forks hotstuff.Forks // information about multiple forks Validator hotstuff.Validator // validator of proposals & votes diff --git a/consensus/hotstuff/consumer.go b/consensus/hotstuff/consumer.go index e8149622087..57633000610 100644 --- a/consensus/hotstuff/consumer.go +++ b/consensus/hotstuff/consumer.go @@ -185,20 +185,6 @@ type ParticipantConsumer interface { // and must handle repetition of the same events (with some processing overhead). OnStartingTimeout(model.TimerInfo) - // OnVoteProcessed notifications are produced by the Vote Aggregation logic, each time - // we successfully ingest a valid vote. - // Prerequisites: - // Implementation must be concurrency safe; Non-blocking; - // and must handle repetition of the same events (with some processing overhead). - OnVoteProcessed(vote *model.Vote) - - // OnTimeoutProcessed notifications are produced by the Timeout Aggregation logic, - // each time we successfully ingest a valid timeout. - // Prerequisites: - // Implementation must be concurrency safe; Non-blocking; - // and must handle repetition of the same events (with some processing overhead). - OnTimeoutProcessed(timeout *model.TimeoutObject) - // OnCurrentViewDetails notifications are produced by the EventHandler during the course of a view with auxiliary information. // These notifications are generally not produced for all views (for example skipped views). // These notifications are guaranteed to be produced for all views we enter after fully processing a message. @@ -216,7 +202,7 @@ type ParticipantConsumer interface { OnCurrentViewDetails(currentView, finalizedView uint64, currentLeader flow.Identifier) } -// QCCreatedConsumer consumes outbound notifications produced by HotStuff and its components. +// VoteCollectorConsumer consumes outbound notifications produced by HotStuff and its components. // Notifications are consensus-internal state changes which are potentially relevant to // the larger node in which HotStuff is running. The notifications are emitted // in the order in which the HotStuff algorithm makes the respective steps. @@ -225,13 +211,20 @@ type ParticipantConsumer interface { // - be concurrency safe // - be non-blocking // - handle repetition of the same events (with some processing overhead). -type QCCreatedConsumer interface { +type VoteCollectorConsumer interface { // OnQcConstructedFromVotes notifications are produced by the VoteAggregator // component, whenever it constructs a QC from votes. // Prerequisites: // Implementation must be concurrency safe; Non-blocking; // and must handle repetition of the same events (with some processing overhead). OnQcConstructedFromVotes(*flow.QuorumCertificate) + + // OnVoteProcessed notifications are produced by the Vote Aggregation logic, each time + // we successfully ingest a valid vote. + // Prerequisites: + // Implementation must be concurrency safe; Non-blocking; + // and must handle repetition of the same events (with some processing overhead). + OnVoteProcessed(vote *model.Vote) } // TimeoutCollectorConsumer consumes outbound notifications produced by HotStuff's timeout aggregation @@ -280,6 +273,13 @@ type TimeoutCollectorConsumer interface { // Implementation must be concurrency safe; Non-blocking; // and must handle repetition of the same events (with some processing overhead). OnNewTcDiscovered(certificate *flow.TimeoutCertificate) + + // OnTimeoutProcessed notifications are produced by the Timeout Aggregation logic, + // each time we successfully ingest a valid timeout. + // Prerequisites: + // Implementation must be concurrency safe; Non-blocking; + // and must handle repetition of the same events (with some processing overhead). + OnTimeoutProcessed(timeout *model.TimeoutObject) } // CommunicatorConsumer consumes outbound notifications produced by HotStuff and it's components. @@ -338,7 +338,7 @@ type Consumer interface { type VoteAggregationConsumer interface { VoteAggregationViolationConsumer - QCCreatedConsumer + VoteCollectorConsumer } type TimeoutAggregationConsumer interface { diff --git a/consensus/hotstuff/event_loop.go b/consensus/hotstuff/event_loop.go index f107449c457..cadc1cc61e3 100644 --- a/consensus/hotstuff/event_loop.go +++ b/consensus/hotstuff/event_loop.go @@ -8,5 +8,5 @@ import ( type EventLoop interface { module.HotStuff TimeoutCollectorConsumer - QCCreatedConsumer + VoteCollectorConsumer } diff --git a/consensus/hotstuff/eventloop/event_loop.go b/consensus/hotstuff/eventloop/event_loop.go index ac231fa7d02..627a48e5e4a 100644 --- a/consensus/hotstuff/eventloop/event_loop.go +++ b/consensus/hotstuff/eventloop/event_loop.go @@ -313,7 +313,13 @@ func (el *EventLoop) OnNewTcDiscovered(tc *flow.TimeoutCertificate) { el.onTrustedTC(tc) } -// OnQcConstructedFromVotes implements hotstuff.QCCreatedConsumer and pushes received qc into processing pipeline. +// OnQcConstructedFromVotes implements hotstuff.VoteCollectorConsumer and pushes received qc into processing pipeline. func (el *EventLoop) OnQcConstructedFromVotes(qc *flow.QuorumCertificate) { el.onTrustedQC(qc) } + +// OnTimeoutProcessed implements hotstuff.TimeoutCollectorConsumer and is no-op +func (el *EventLoop) OnTimeoutProcessed(timeout *model.TimeoutObject) {} + +// OnVoteProcessed implements hotstuff.VoteCollectorConsumer and is no-op +func (el *EventLoop) OnVoteProcessed(vote *model.Vote) {} diff --git a/consensus/hotstuff/notifications/noop_consumer.go b/consensus/hotstuff/notifications/noop_consumer.go index feb676e5d89..ab3124bca0e 100644 --- a/consensus/hotstuff/notifications/noop_consumer.go +++ b/consensus/hotstuff/notifications/noop_consumer.go @@ -50,10 +50,6 @@ func (*NoopPartialConsumer) OnTcTriggeredViewChange(uint64, uint64, *flow.Timeou func (*NoopPartialConsumer) OnStartingTimeout(model.TimerInfo) {} -func (*NoopPartialConsumer) OnVoteProcessed(*model.Vote) {} - -func (*NoopPartialConsumer) OnTimeoutProcessed(*model.TimeoutObject) {} - func (*NoopPartialConsumer) OnCurrentViewDetails(uint64, uint64, flow.Identifier) {} // no-op implementation of hotstuff.FinalizationConsumer @@ -81,6 +77,8 @@ func (*NoopTimeoutCollectorConsumer) OnNewQcDiscovered(*flow.QuorumCertificate) func (*NoopTimeoutCollectorConsumer) OnNewTcDiscovered(*flow.TimeoutCertificate) {} +func (*NoopTimeoutCollectorConsumer) OnTimeoutProcessed(*model.TimeoutObject) {} + // no-op implementation of hotstuff.CommunicatorConsumer type NoopCommunicatorConsumer struct{} @@ -93,13 +91,15 @@ func (*NoopCommunicatorConsumer) OnOwnTimeout(*model.TimeoutObject) {} func (*NoopCommunicatorConsumer) OnOwnProposal(*flow.Header, time.Time) {} -// no-op implementation of hotstuff.QCCreatedConsumer +// no-op implementation of hotstuff.VoteCollectorConsumer + +type NoopVoteCollectorConsumer struct{} -type NoopQCCreatedConsumer struct{} +var _ hotstuff.VoteCollectorConsumer = (*NoopVoteCollectorConsumer)(nil) -var _ hotstuff.QCCreatedConsumer = (*NoopQCCreatedConsumer)(nil) +func (*NoopVoteCollectorConsumer) OnQcConstructedFromVotes(*flow.QuorumCertificate) {} -func (*NoopQCCreatedConsumer) OnQcConstructedFromVotes(*flow.QuorumCertificate) {} +func (*NoopVoteCollectorConsumer) OnVoteProcessed(*model.Vote) {} // no-op implementation of hotstuff.ProposalViolationConsumer diff --git a/consensus/hotstuff/notifications/pubsub/distributor.go b/consensus/hotstuff/notifications/pubsub/distributor.go index 16a882f8741..7d2ac5ed489 100644 --- a/consensus/hotstuff/notifications/pubsub/distributor.go +++ b/consensus/hotstuff/notifications/pubsub/distributor.go @@ -71,7 +71,7 @@ func (d *TimeoutAggregationDistributor) AddTimeoutAggregationConsumer(consumer h type VoteAggregationDistributor struct { *VoteAggregationViolationDistributor - *QCCreatedDistributor + *VoteCollectorDistributor } var _ hotstuff.VoteAggregationConsumer = (*VoteAggregationDistributor)(nil) @@ -79,11 +79,11 @@ var _ hotstuff.VoteAggregationConsumer = (*VoteAggregationDistributor)(nil) func NewVoteAggregationDistributor() *VoteAggregationDistributor { return &VoteAggregationDistributor{ VoteAggregationViolationDistributor: NewVoteAggregationViolationDistributor(), - QCCreatedDistributor: NewQCCreatedDistributor(), + VoteCollectorDistributor: NewQCCreatedDistributor(), } } func (d *VoteAggregationDistributor) AddVoteAggregationConsumer(consumer hotstuff.VoteAggregationConsumer) { d.VoteAggregationViolationDistributor.AddVoteAggregationViolationConsumer(consumer) - d.QCCreatedDistributor.AddQCCreatedConsumer(consumer) + d.VoteCollectorDistributor.AddQCCreatedConsumer(consumer) } diff --git a/consensus/hotstuff/notifications/pubsub/participant_distributor.go b/consensus/hotstuff/notifications/pubsub/participant_distributor.go index 47ca41846df..46149da7f32 100644 --- a/consensus/hotstuff/notifications/pubsub/participant_distributor.go +++ b/consensus/hotstuff/notifications/pubsub/participant_distributor.go @@ -118,22 +118,6 @@ func (d *ParticipantDistributor) OnStartingTimeout(timerInfo model.TimerInfo) { } } -func (d *ParticipantDistributor) OnVoteProcessed(vote *model.Vote) { - d.lock.RLock() - defer d.lock.RUnlock() - for _, subscriber := range d.subscribers { - subscriber.OnVoteProcessed(vote) - } -} - -func (d *ParticipantDistributor) OnTimeoutProcessed(timeout *model.TimeoutObject) { - d.lock.RLock() - defer d.lock.RUnlock() - for _, subscriber := range d.subscribers { - subscriber.OnTimeoutProcessed(timeout) - } -} - func (d *ParticipantDistributor) OnCurrentViewDetails(currentView, finalizedView uint64, currentLeader flow.Identifier) { d.lock.RLock() defer d.lock.RUnlock() diff --git a/consensus/hotstuff/notifications/pubsub/qc_created_distributor.go b/consensus/hotstuff/notifications/pubsub/qc_created_distributor.go deleted file mode 100644 index 481c3a6acf3..00000000000 --- a/consensus/hotstuff/notifications/pubsub/qc_created_distributor.go +++ /dev/null @@ -1,37 +0,0 @@ -package pubsub - -import ( - "sync" - - "github.com/onflow/flow-go/consensus/hotstuff" - "github.com/onflow/flow-go/model/flow" -) - -// QCCreatedDistributor ingests events about QC creation from hotstuff and distributes them to subscribers. -// Objects are concurrency safe. -// NOTE: it can be refactored to work without lock since usually we never subscribe after startup. Mostly -// list of observers is static. -type QCCreatedDistributor struct { - qcCreatedConsumers []hotstuff.QCCreatedConsumer - lock sync.RWMutex -} - -var _ hotstuff.QCCreatedConsumer = (*QCCreatedDistributor)(nil) - -func NewQCCreatedDistributor() *QCCreatedDistributor { - return &QCCreatedDistributor{} -} - -func (d *QCCreatedDistributor) AddQCCreatedConsumer(consumer hotstuff.QCCreatedConsumer) { - d.lock.Lock() - defer d.lock.Unlock() - d.qcCreatedConsumers = append(d.qcCreatedConsumers, consumer) -} - -func (d *QCCreatedDistributor) OnQcConstructedFromVotes(qc *flow.QuorumCertificate) { - d.lock.RLock() - defer d.lock.RUnlock() - for _, consumer := range d.qcCreatedConsumers { - consumer.OnQcConstructedFromVotes(qc) - } -} diff --git a/consensus/hotstuff/notifications/pubsub/timeout_collector_distributor.go b/consensus/hotstuff/notifications/pubsub/timeout_collector_distributor.go index aa9c0bd9397..82705328f74 100644 --- a/consensus/hotstuff/notifications/pubsub/timeout_collector_distributor.go +++ b/consensus/hotstuff/notifications/pubsub/timeout_collector_distributor.go @@ -1,6 +1,7 @@ package pubsub import ( + "github.com/onflow/flow-go/consensus/hotstuff/model" "sync" "github.com/onflow/flow-go/consensus/hotstuff" @@ -59,3 +60,11 @@ func (d *TimeoutCollectorDistributor) OnNewTcDiscovered(tc *flow.TimeoutCertific consumer.OnNewTcDiscovered(tc) } } + +func (d *TimeoutCollectorDistributor) OnTimeoutProcessed(timeout *model.TimeoutObject) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, subscriber := range d.consumers { + subscriber.OnTimeoutProcessed(timeout) + } +} diff --git a/consensus/hotstuff/notifications/pubsub/vote_collector_distributor.go b/consensus/hotstuff/notifications/pubsub/vote_collector_distributor.go new file mode 100644 index 00000000000..7754de13e30 --- /dev/null +++ b/consensus/hotstuff/notifications/pubsub/vote_collector_distributor.go @@ -0,0 +1,46 @@ +package pubsub + +import ( + "github.com/onflow/flow-go/consensus/hotstuff/model" + "sync" + + "github.com/onflow/flow-go/consensus/hotstuff" + "github.com/onflow/flow-go/model/flow" +) + +// VoteCollectorDistributor ingests events about QC creation from hotstuff and distributes them to subscribers. +// Objects are concurrency safe. +// NOTE: it can be refactored to work without lock since usually we never subscribe after startup. Mostly +// list of observers is static. +type VoteCollectorDistributor struct { + consumers []hotstuff.VoteCollectorConsumer + lock sync.RWMutex +} + +var _ hotstuff.VoteCollectorConsumer = (*VoteCollectorDistributor)(nil) + +func NewQCCreatedDistributor() *VoteCollectorDistributor { + return &VoteCollectorDistributor{} +} + +func (d *VoteCollectorDistributor) AddQCCreatedConsumer(consumer hotstuff.VoteCollectorConsumer) { + d.lock.Lock() + defer d.lock.Unlock() + d.consumers = append(d.consumers, consumer) +} + +func (d *VoteCollectorDistributor) OnQcConstructedFromVotes(qc *flow.QuorumCertificate) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, consumer := range d.consumers { + consumer.OnQcConstructedFromVotes(qc) + } +} + +func (d *VoteCollectorDistributor) OnVoteProcessed(vote *model.Vote) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, subscriber := range d.consumers { + subscriber.OnVoteProcessed(vote) + } +} diff --git a/consensus/hotstuff/timeoutaggregator/timeout_aggregator.go b/consensus/hotstuff/timeoutaggregator/timeout_aggregator.go index ae308c42048..7d359257176 100644 --- a/consensus/hotstuff/timeoutaggregator/timeout_aggregator.go +++ b/consensus/hotstuff/timeoutaggregator/timeout_aggregator.go @@ -35,7 +35,6 @@ type TimeoutAggregator struct { log zerolog.Logger hotstuffMetrics module.HotstuffMetrics engineMetrics module.EngineMetrics - notifier hotstuff.Consumer lowestRetainedView counters.StrictMonotonousCounter // lowest view, for which we still process timeouts collectors hotstuff.TimeoutCollectors queuedTimeoutsNotifier engine.Notifier @@ -52,7 +51,6 @@ func NewTimeoutAggregator(log zerolog.Logger, hotstuffMetrics module.HotstuffMetrics, engineMetrics module.EngineMetrics, mempoolMetrics module.MempoolMetrics, - notifier hotstuff.Consumer, lowestRetainedView uint64, collectors hotstuff.TimeoutCollectors, ) (*TimeoutAggregator, error) { @@ -66,7 +64,6 @@ func NewTimeoutAggregator(log zerolog.Logger, log: log.With().Str("component", "hotstuff.timeout_aggregator").Logger(), hotstuffMetrics: hotstuffMetrics, engineMetrics: engineMetrics, - notifier: notifier, lowestRetainedView: counters.NewMonotonousCounter(lowestRetainedView), collectors: collectors, queuedTimeoutsNotifier: engine.NewNotifier(), diff --git a/consensus/hotstuff/timeoutcollector/factory.go b/consensus/hotstuff/timeoutcollector/factory.go index 0f18b20b748..ba6c3fbc29f 100644 --- a/consensus/hotstuff/timeoutcollector/factory.go +++ b/consensus/hotstuff/timeoutcollector/factory.go @@ -11,10 +11,9 @@ import ( // TimeoutCollectorFactory implements hotstuff.TimeoutCollectorFactory, it is responsible for creating timeout collector // for given view. type TimeoutCollectorFactory struct { - log zerolog.Logger - notifier hotstuff.Consumer - collectorNotifier hotstuff.TimeoutAggregationConsumer - processorFactory hotstuff.TimeoutProcessorFactory + log zerolog.Logger + notifier hotstuff.TimeoutAggregationConsumer + processorFactory hotstuff.TimeoutProcessorFactory } var _ hotstuff.TimeoutCollectorFactory = (*TimeoutCollectorFactory)(nil) @@ -22,15 +21,13 @@ var _ hotstuff.TimeoutCollectorFactory = (*TimeoutCollectorFactory)(nil) // NewTimeoutCollectorFactory creates new instance of TimeoutCollectorFactory. // No error returns are expected during normal operations. func NewTimeoutCollectorFactory(log zerolog.Logger, - notifier hotstuff.Consumer, - collectorNotifier hotstuff.TimeoutAggregationConsumer, + notifier hotstuff.TimeoutAggregationConsumer, createProcessor hotstuff.TimeoutProcessorFactory, ) *TimeoutCollectorFactory { return &TimeoutCollectorFactory{ - log: log, - notifier: notifier, - collectorNotifier: collectorNotifier, - processorFactory: createProcessor, + log: log, + notifier: notifier, + processorFactory: createProcessor, } } @@ -44,7 +41,7 @@ func (f *TimeoutCollectorFactory) Create(view uint64) (hotstuff.TimeoutCollector if err != nil { return nil, fmt.Errorf("could not create TimeoutProcessor at view %d: %w", view, err) } - return NewTimeoutCollector(f.log, view, f.notifier, f.collectorNotifier, processor), nil + return NewTimeoutCollector(f.log, view, f.notifier, processor), nil } // TimeoutProcessorFactory implements hotstuff.TimeoutProcessorFactory, it is responsible for creating timeout processor diff --git a/consensus/hotstuff/timeoutcollector/timeout_collector.go b/consensus/hotstuff/timeoutcollector/timeout_collector.go index d00a22b30e7..90541a1a0c1 100644 --- a/consensus/hotstuff/timeoutcollector/timeout_collector.go +++ b/consensus/hotstuff/timeoutcollector/timeout_collector.go @@ -16,13 +16,12 @@ import ( // their view is newer than any QC or TC previously known to the TimeoutCollector. // This module is safe to use in concurrent environment. type TimeoutCollector struct { - log zerolog.Logger - notifier hotstuff.Consumer - timeoutsCache *TimeoutObjectsCache // cache for tracking double timeout and timeout equivocation - collectorNotifier hotstuff.TimeoutAggregationConsumer - processor hotstuff.TimeoutProcessor - newestReportedQC counters.StrictMonotonousCounter // view of newest QC that was reported - newestReportedTC counters.StrictMonotonousCounter // view of newest TC that was reported + log zerolog.Logger + timeoutsCache *TimeoutObjectsCache // cache for tracking double timeout and timeout equivocation + notifier hotstuff.TimeoutAggregationConsumer + processor hotstuff.TimeoutProcessor + newestReportedQC counters.StrictMonotonousCounter // view of newest QC that was reported + newestReportedTC counters.StrictMonotonousCounter // view of newest TC that was reported } var _ hotstuff.TimeoutCollector = (*TimeoutCollector)(nil) @@ -30,8 +29,7 @@ var _ hotstuff.TimeoutCollector = (*TimeoutCollector)(nil) // NewTimeoutCollector creates new instance of TimeoutCollector func NewTimeoutCollector(log zerolog.Logger, view uint64, - notifier hotstuff.Consumer, - collectorNotifier hotstuff.TimeoutAggregationConsumer, + notifier hotstuff.TimeoutAggregationConsumer, processor hotstuff.TimeoutProcessor, ) *TimeoutCollector { return &TimeoutCollector{ @@ -39,12 +37,11 @@ func NewTimeoutCollector(log zerolog.Logger, Str("component", "hotstuff.timeout_collector"). Uint64("view", view). Logger(), - notifier: notifier, - timeoutsCache: NewTimeoutObjectsCache(view), - processor: processor, - collectorNotifier: collectorNotifier, - newestReportedQC: counters.NewMonotonousCounter(0), - newestReportedTC: counters.NewMonotonousCounter(0), + notifier: notifier, + timeoutsCache: NewTimeoutObjectsCache(view), + processor: processor, + newestReportedQC: counters.NewMonotonousCounter(0), + newestReportedTC: counters.NewMonotonousCounter(0), } } @@ -64,7 +61,7 @@ func (c *TimeoutCollector) AddTimeout(timeout *model.TimeoutObject) error { return nil } if doubleTimeoutErr, isDoubleTimeoutErr := model.AsDoubleTimeoutError(err); isDoubleTimeoutErr { - c.collectorNotifier.OnDoubleTimeoutDetected(doubleTimeoutErr.FirstTimeout, doubleTimeoutErr.ConflictingTimeout) + c.notifier.OnDoubleTimeoutDetected(doubleTimeoutErr.FirstTimeout, doubleTimeoutErr.ConflictingTimeout) return nil } return fmt.Errorf("internal error adding timeout %v to cache for view: %d: %w", timeout.ID(), timeout.View, err) @@ -85,7 +82,7 @@ func (c *TimeoutCollector) processTimeout(timeout *model.TimeoutObject) error { err := c.processor.Process(timeout) if err != nil { if invalidTimeoutErr, ok := model.AsInvalidTimeoutError(err); ok { - c.collectorNotifier.OnInvalidTimeoutDetected(*invalidTimeoutErr) + c.notifier.OnInvalidTimeoutDetected(*invalidTimeoutErr) return nil } return fmt.Errorf("internal error while processing timeout: %w", err) @@ -113,12 +110,12 @@ func (c *TimeoutCollector) processTimeout(timeout *model.TimeoutObject) error { // system can only arrive earlier in our weakly ordered implementation. Hence, if anything, the recipient // receives the desired information _earlier_ but not later. if c.newestReportedQC.Set(timeout.NewestQC.View) { - c.collectorNotifier.OnNewQcDiscovered(timeout.NewestQC) + c.notifier.OnNewQcDiscovered(timeout.NewestQC) } // Same explanation for weak ordering of QCs also applies to TCs. if timeout.LastViewTC != nil { if c.newestReportedTC.Set(timeout.LastViewTC.View) { - c.collectorNotifier.OnNewTcDiscovered(timeout.LastViewTC) + c.notifier.OnNewTcDiscovered(timeout.LastViewTC) } } diff --git a/consensus/hotstuff/votecollector/statemachine.go b/consensus/hotstuff/votecollector/statemachine.go index 4b207f819fc..d62159ea9ef 100644 --- a/consensus/hotstuff/votecollector/statemachine.go +++ b/consensus/hotstuff/votecollector/statemachine.go @@ -25,8 +25,7 @@ type VoteCollector struct { sync.Mutex log zerolog.Logger workers hotstuff.Workers - notifier hotstuff.Consumer - violationConsumer hotstuff.VoteAggregationViolationConsumer + notifier hotstuff.VoteAggregationConsumer createVerifyingProcessor VerifyingVoteProcessorFactory votesCache VotesCache @@ -48,12 +47,11 @@ type atomicValueWrapper struct { func NewStateMachineFactory( log zerolog.Logger, - notifier hotstuff.Consumer, - violationConsumer hotstuff.VoteAggregationViolationConsumer, + notifier hotstuff.VoteAggregationConsumer, verifyingVoteProcessorFactory VerifyingVoteProcessorFactory, ) voteaggregator.NewCollectorFactoryMethod { return func(view uint64, workers hotstuff.Workers) (hotstuff.VoteCollector, error) { - return NewStateMachine(view, log, workers, notifier, violationConsumer, verifyingVoteProcessorFactory), nil + return NewStateMachine(view, log, workers, notifier, verifyingVoteProcessorFactory), nil } } @@ -61,8 +59,7 @@ func NewStateMachine( view uint64, log zerolog.Logger, workers hotstuff.Workers, - notifier hotstuff.Consumer, - violationConsumer hotstuff.VoteAggregationViolationConsumer, + notifier hotstuff.VoteAggregationConsumer, verifyingVoteProcessorFactory VerifyingVoteProcessorFactory, ) *VoteCollector { log = log.With(). @@ -73,7 +70,6 @@ func NewStateMachine( log: log, workers: workers, notifier: notifier, - violationConsumer: violationConsumer, createVerifyingProcessor: verifyingVoteProcessorFactory, votesCache: *NewVotesCache(view), } @@ -96,7 +92,7 @@ func (m *VoteCollector) AddVote(vote *model.Vote) error { return nil } if doubleVoteErr, isDoubleVoteErr := model.AsDoubleVoteError(err); isDoubleVoteErr { - m.violationConsumer.OnDoubleVotingDetected(doubleVoteErr.FirstVote, doubleVoteErr.ConflictingVote) + m.notifier.OnDoubleVotingDetected(doubleVoteErr.FirstVote, doubleVoteErr.ConflictingVote) return nil } return fmt.Errorf("internal error adding vote %v to cache for block %v: %w", @@ -135,7 +131,7 @@ func (m *VoteCollector) processVote(vote *model.Vote) error { err := processor.Process(vote) if err != nil { if invalidVoteErr, ok := model.AsInvalidVoteError(err); ok { - m.violationConsumer.OnInvalidVoteDetected(*invalidVoteErr) + m.notifier.OnInvalidVoteDetected(*invalidVoteErr) return nil } // ATTENTION: due to how our logic is designed this situation is only possible From d11f51726d2f5822713b3e92e8331204f3fd2889 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Tue, 9 May 2023 13:44:09 +0300 Subject: [PATCH 24/38] Connected telemetry and slashing consumers to consensus distributors. --- .../node_builder/access_node_builder.go | 2 +- cmd/collection/main.go | 2 +- cmd/consensus/main.go | 18 +++++++++++---- cmd/consensus/notifier.go | 2 -- cmd/execution_builder.go | 2 +- cmd/observer/node_builder/observer_builder.go | 2 +- cmd/verification_builder.go | 2 +- consensus/config.go | 22 +++++++++---------- .../notifications/pubsub/distributor.go | 4 ++-- .../pubsub/protocol_violation_distributor.go | 2 +- .../pubsub/vote_collector_distributor.go | 2 +- consensus/hotstuff/notifications/telemetry.go | 14 ++++++++++-- consensus/integration/nodes_test.go | 2 +- consensus/participant.go | 2 +- .../collection/epochmgr/factories/hotstuff.go | 4 ++-- follower/follower_builder.go | 2 +- 16 files changed, 51 insertions(+), 33 deletions(-) diff --git a/cmd/access/node_builder/access_node_builder.go b/cmd/access/node_builder/access_node_builder.go index 4ac6fb815b2..a08cf7d2221 100644 --- a/cmd/access/node_builder/access_node_builder.go +++ b/cmd/access/node_builder/access_node_builder.go @@ -609,7 +609,7 @@ func (builder *FlowAccessNodeBuilder) BuildExecutionDataRequester() *FlowAccessN func FlowAccessNode(nodeBuilder *cmd.FlowNodeBuilder) *FlowAccessNodeBuilder { dist := consensuspubsub.NewFollowerDistributor() - dist.AddProtocolViolationConsumer(notifications.NewSlashingViolationsConsumer(nodeBuilder.Logger)) + dist.AddProposalViolationConsumer(notifications.NewSlashingViolationsConsumer(nodeBuilder.Logger)) return &FlowAccessNodeBuilder{ AccessNodeConfig: DefaultAccessNodeConfig(), FlowNodeBuilder: nodeBuilder, diff --git a/cmd/collection/main.go b/cmd/collection/main.go index f4af64334ad..b5c4d9be8ed 100644 --- a/cmd/collection/main.go +++ b/cmd/collection/main.go @@ -173,7 +173,7 @@ func main() { PreInit(cmd.DynamicStartPreInit). Module("follower distributor", func(node *cmd.NodeConfig) error { followerDistributor = pubsub.NewFollowerDistributor() - followerDistributor.AddProtocolViolationConsumer(notifications.NewSlashingViolationsConsumer(node.Logger)) + followerDistributor.AddProposalViolationConsumer(notifications.NewSlashingViolationsConsumer(node.Logger)) return nil }). Module("mutable follower state", func(node *cmd.NodeConfig) error { diff --git a/cmd/consensus/main.go b/cmd/consensus/main.go index 209fd314289..d729d5999ce 100644 --- a/cmd/consensus/main.go +++ b/cmd/consensus/main.go @@ -366,7 +366,6 @@ func main() { }). Module("follower distributor", func(node *cmd.NodeConfig) error { followerDistributor = pubsub.NewFollowerDistributor() - followerDistributor.AddProtocolViolationConsumer(notifications.NewSlashingViolationsConsumer(nodeBuilder.Logger)) return nil }). Module("machine account config", func(node *cmd.NodeConfig) error { @@ -554,12 +553,17 @@ func main() { // create consensus logger logger := createLogger(node.Logger, node.RootChainID) + telemetryConsumer := notifications.NewTelemetryConsumer(logger) + slashingViolationConsumer := notifications.NewSlashingViolationsConsumer(nodeBuilder.Logger) + followerDistributor.AddProposalViolationConsumer(slashingViolationConsumer) + // initialize a logging notifier for hotstuff notifier := createNotifier( logger, mainMetrics, ) + notifier.AddParticipantConsumer(telemetryConsumer) notifier.AddFollowerConsumer(followerDistributor) // initialize the persister @@ -582,8 +586,11 @@ func main() { return nil, err } - // TODO: connect to slashing violation consumer + // create producer and connect it to consumers voteAggregationDistributor := pubsub.NewVoteAggregationDistributor() + voteAggregationDistributor.AddVoteCollectorConsumer(telemetryConsumer) + voteAggregationDistributor.AddVoteAggregationViolationConsumer(slashingViolationConsumer) + validator := consensus.NewValidator(mainMetrics, wrappedCommittee) voteProcessorFactory := votecollector.NewCombinedVoteProcessorFactory(wrappedCommittee, voteAggregationDistributor.OnQcConstructedFromVotes) lowestViewForVoteProcessing := finalizedBlock.View + 1 @@ -600,8 +607,11 @@ func main() { return nil, fmt.Errorf("could not initialize vote aggregator: %w", err) } - // TODO: connect to slashing violation consumer + // create producer and connect it to consumers timeoutAggregationDistributor := pubsub.NewTimeoutAggregationDistributor() + timeoutAggregationDistributor.AddTimeoutCollectorConsumer(telemetryConsumer) + timeoutAggregationDistributor.AddTimeoutAggregationViolationConsumer(slashingViolationConsumer) + timeoutProcessorFactory := timeoutcollector.NewTimeoutProcessorFactory( logger, timeoutAggregationDistributor, @@ -628,7 +638,7 @@ func main() { Committee: wrappedCommittee, Signer: signer, Persist: persist, - QCCreatedDistributor: voteAggregationDistributor.VoteCollectorDistributor, + VoteCollectorDistributor: voteAggregationDistributor.VoteCollectorDistributor, FollowerDistributor: followerDistributor, TimeoutCollectorDistributor: timeoutAggregationDistributor.TimeoutCollectorDistributor, Forks: forks, diff --git a/cmd/consensus/notifier.go b/cmd/consensus/notifier.go index 94fc57782e6..3826060cf63 100644 --- a/cmd/consensus/notifier.go +++ b/cmd/consensus/notifier.go @@ -17,11 +17,9 @@ func createLogger(log zerolog.Logger, chainID flow.ChainID) zerolog.Logger { // createNotifier creates a pubsub distributor and connects it to consensus consumers. func createNotifier(log zerolog.Logger, metrics module.HotstuffMetrics) *pubsub.Distributor { - telemetryConsumer := notifications.NewTelemetryConsumer(log) metricsConsumer := metricsconsumer.NewMetricsConsumer(metrics) logsConsumer := notifications.NewLogConsumer(log) dis := pubsub.NewDistributor() - dis.AddConsumer(telemetryConsumer) dis.AddConsumer(metricsConsumer) dis.AddConsumer(logsConsumer) return dis diff --git a/cmd/execution_builder.go b/cmd/execution_builder.go index 48baf39f9b7..8521f365100 100644 --- a/cmd/execution_builder.go +++ b/cmd/execution_builder.go @@ -274,7 +274,7 @@ func (exeNode *ExecutionNode) LoadExecutionReceiptsStorage( func (exeNode *ExecutionNode) LoadFollowerDistributor(node *NodeConfig) error { exeNode.followerDistributor = pubsub.NewFollowerDistributor() - exeNode.followerDistributor.AddProtocolViolationConsumer(notifications.NewSlashingViolationsConsumer(node.Logger)) + exeNode.followerDistributor.AddProposalViolationConsumer(notifications.NewSlashingViolationsConsumer(node.Logger)) return nil } diff --git a/cmd/observer/node_builder/observer_builder.go b/cmd/observer/node_builder/observer_builder.go index 4bb052ef55a..f86bd2117d3 100644 --- a/cmd/observer/node_builder/observer_builder.go +++ b/cmd/observer/node_builder/observer_builder.go @@ -566,7 +566,7 @@ func NewFlowObserverServiceBuilder(opts ...Option) *ObserverServiceBuilder { FlowNodeBuilder: cmd.FlowNode("observer"), FollowerDistributor: pubsub.NewFollowerDistributor(), } - anb.FollowerDistributor.AddProtocolViolationConsumer(notifications.NewSlashingViolationsConsumer(anb.Logger)) + anb.FollowerDistributor.AddProposalViolationConsumer(notifications.NewSlashingViolationsConsumer(anb.Logger)) // the observer gets a version of the root snapshot file that does not contain any node addresses // hence skip all the root snapshot validations that involved an identity address anb.FlowNodeBuilder.SkipNwAddressBasedValidations = true diff --git a/cmd/verification_builder.go b/cmd/verification_builder.go index a4f0e11d6c8..dba427a0a1a 100644 --- a/cmd/verification_builder.go +++ b/cmd/verification_builder.go @@ -179,7 +179,7 @@ func (v *VerificationNodeBuilder) LoadComponentsAndModules() { }). Module("follower distributor", func(node *NodeConfig) error { followerDistributor = pubsub.NewFollowerDistributor() - followerDistributor.AddProtocolViolationConsumer(notifications.NewSlashingViolationsConsumer(node.Logger)) + followerDistributor.AddProposalViolationConsumer(notifications.NewSlashingViolationsConsumer(node.Logger)) return nil }). Module("sync core", func(node *NodeConfig) error { diff --git a/consensus/config.go b/consensus/config.go index d91c05697fb..30e22a7fa3d 100644 --- a/consensus/config.go +++ b/consensus/config.go @@ -12,17 +12,17 @@ import ( // HotstuffModules is a helper structure to encapsulate dependencies to create // a hotStuff participant. type HotstuffModules struct { - Committee hotstuff.DynamicCommittee // consensus committee - Signer hotstuff.Signer // signer of proposal & votes - Persist hotstuff.Persister // last state of consensus participant - Notifier *pubsub.Distributor // observer for hotstuff events - FollowerDistributor *pubsub.FollowerDistributor // observer for finalization events, used by compliance engine - QCCreatedDistributor *pubsub.VoteCollectorDistributor // observer for qc created event, used by leader - TimeoutCollectorDistributor *pubsub.TimeoutCollectorDistributor - Forks hotstuff.Forks // information about multiple forks - Validator hotstuff.Validator // validator of proposals & votes - VoteAggregator hotstuff.VoteAggregator // aggregator of votes, used by leader - TimeoutAggregator hotstuff.TimeoutAggregator // aggregator of `TimeoutObject`s, used by every replica + Committee hotstuff.DynamicCommittee // consensus committee + Signer hotstuff.Signer // signer of proposal & votes + Persist hotstuff.Persister // last state of consensus participant + Notifier *pubsub.Distributor // observer for hotstuff events + FollowerDistributor *pubsub.FollowerDistributor // observer for finalization events, used by compliance engine + VoteCollectorDistributor *pubsub.VoteCollectorDistributor // observer for vote aggregation events, used by leader + TimeoutCollectorDistributor *pubsub.TimeoutCollectorDistributor // observer for timeout aggregation events + Forks hotstuff.Forks // information about multiple forks + Validator hotstuff.Validator // validator of proposals & votes + VoteAggregator hotstuff.VoteAggregator // aggregator of votes, used by leader + TimeoutAggregator hotstuff.TimeoutAggregator // aggregator of `TimeoutObject`s, used by every replica } type ParticipantConfig struct { diff --git a/consensus/hotstuff/notifications/pubsub/distributor.go b/consensus/hotstuff/notifications/pubsub/distributor.go index 7d2ac5ed489..fddf3c03c23 100644 --- a/consensus/hotstuff/notifications/pubsub/distributor.go +++ b/consensus/hotstuff/notifications/pubsub/distributor.go @@ -47,7 +47,7 @@ func NewFollowerDistributor() *FollowerDistributor { func (d *FollowerDistributor) AddFollowerConsumer(consumer hotstuff.FollowerConsumer) { d.FinalizationDistributor.AddFinalizationConsumer(consumer) - d.ProtocolViolationDistributor.AddProtocolViolationConsumer(consumer) + d.ProtocolViolationDistributor.AddProposalViolationConsumer(consumer) } type TimeoutAggregationDistributor struct { @@ -85,5 +85,5 @@ func NewVoteAggregationDistributor() *VoteAggregationDistributor { func (d *VoteAggregationDistributor) AddVoteAggregationConsumer(consumer hotstuff.VoteAggregationConsumer) { d.VoteAggregationViolationDistributor.AddVoteAggregationViolationConsumer(consumer) - d.VoteCollectorDistributor.AddQCCreatedConsumer(consumer) + d.VoteCollectorDistributor.AddVoteCollectorConsumer(consumer) } diff --git a/consensus/hotstuff/notifications/pubsub/protocol_violation_distributor.go b/consensus/hotstuff/notifications/pubsub/protocol_violation_distributor.go index 60975a5120b..054e1d6bdce 100644 --- a/consensus/hotstuff/notifications/pubsub/protocol_violation_distributor.go +++ b/consensus/hotstuff/notifications/pubsub/protocol_violation_distributor.go @@ -22,7 +22,7 @@ func NewProtocolViolationDistributor() *ProtocolViolationDistributor { return &ProtocolViolationDistributor{} } -func (d *ProtocolViolationDistributor) AddProtocolViolationConsumer(consumer hotstuff.ProposalViolationConsumer) { +func (d *ProtocolViolationDistributor) AddProposalViolationConsumer(consumer hotstuff.ProposalViolationConsumer) { d.lock.Lock() defer d.lock.Unlock() d.subscribers = append(d.subscribers, consumer) diff --git a/consensus/hotstuff/notifications/pubsub/vote_collector_distributor.go b/consensus/hotstuff/notifications/pubsub/vote_collector_distributor.go index 7754de13e30..5c2f07f0ed7 100644 --- a/consensus/hotstuff/notifications/pubsub/vote_collector_distributor.go +++ b/consensus/hotstuff/notifications/pubsub/vote_collector_distributor.go @@ -23,7 +23,7 @@ func NewQCCreatedDistributor() *VoteCollectorDistributor { return &VoteCollectorDistributor{} } -func (d *VoteCollectorDistributor) AddQCCreatedConsumer(consumer hotstuff.VoteCollectorConsumer) { +func (d *VoteCollectorDistributor) AddVoteCollectorConsumer(consumer hotstuff.VoteCollectorConsumer) { d.lock.Lock() defer d.lock.Unlock() d.consumers = append(d.consumers, consumer) diff --git a/consensus/hotstuff/notifications/telemetry.go b/consensus/hotstuff/notifications/telemetry.go index 67f0ca1339a..7bbf57f79de 100644 --- a/consensus/hotstuff/notifications/telemetry.go +++ b/consensus/hotstuff/notifications/telemetry.go @@ -32,12 +32,15 @@ import ( // // Telemetry does NOT capture slashing notifications type TelemetryConsumer struct { - NoopConsumer + NoopTimeoutCollectorConsumer + NoopVoteCollectorConsumer pathHandler *PathHandler noPathLogger zerolog.Logger } -var _ hotstuff.Consumer = (*TelemetryConsumer)(nil) +var _ hotstuff.ParticipantConsumer = (*TelemetryConsumer)(nil) +var _ hotstuff.VoteCollectorConsumer = (*TelemetryConsumer)(nil) +var _ hotstuff.TimeoutCollectorConsumer = (*TelemetryConsumer)(nil) // NewTelemetryConsumer creates consumer that reports telemetry events using logger backend. // Logger MUST include `chain` parameter as part of log context with corresponding chain ID to correctly map telemetry events to chain. @@ -240,6 +243,13 @@ func (t *TelemetryConsumer) OnCurrentViewDetails(currentView, finalizedView uint Msg("OnCurrentViewDetails") } +func (t *TelemetryConsumer) OnViewChange(oldView, newView uint64) { + t.pathHandler.NextStep(). + Uint64("old_view", oldView). + Uint64("new_view", newView). + Msg("OnViewChange") +} + // PathHandler maintains a notion of the current path through the state machine. // It allows to close a path and open new path. Each path is identified by a unique // (randomly generated) uuid. Along each path, we can capture information about relevant diff --git a/consensus/integration/nodes_test.go b/consensus/integration/nodes_test.go index 6911118e877..b6ce1f10c2d 100644 --- a/consensus/integration/nodes_test.go +++ b/consensus/integration/nodes_test.go @@ -566,7 +566,7 @@ func createNode( Committee: committee, Signer: signer, Persist: persist, - QCCreatedDistributor: qcDistributor, + VoteCollectorDistributor: qcDistributor, TimeoutCollectorDistributor: timeoutCollectorDistributor, VoteAggregator: voteAggregator, TimeoutAggregator: timeoutAggregator, diff --git a/consensus/participant.go b/consensus/participant.go index e6435c70b97..663da42ea16 100644 --- a/consensus/participant.go +++ b/consensus/participant.go @@ -122,7 +122,7 @@ func NewParticipant( } // add observer, event loop needs to receive events from distributor - modules.QCCreatedDistributor.AddQCCreatedConsumer(loop) + modules.VoteCollectorDistributor.AddVoteCollectorConsumer(loop) modules.TimeoutCollectorDistributor.AddTimeoutCollectorConsumer(loop) return loop, nil diff --git a/engine/collection/epochmgr/factories/hotstuff.go b/engine/collection/epochmgr/factories/hotstuff.go index f4c3524a553..5c98803f48e 100644 --- a/engine/collection/epochmgr/factories/hotstuff.go +++ b/engine/collection/epochmgr/factories/hotstuff.go @@ -79,7 +79,7 @@ func (f *HotStuffFactory) CreateModules( notifier.AddConsumer(notifications.NewLogConsumer(log)) notifier.AddConsumer(hotmetrics.NewMetricsConsumer(metrics)) notifier.AddConsumer(notifications.NewTelemetryConsumer(log)) - notifier.AddProtocolViolationConsumer(notifications.NewSlashingViolationsConsumer(log)) + notifier.AddProposalViolationConsumer(notifications.NewSlashingViolationsConsumer(log)) var ( err error @@ -159,7 +159,7 @@ func (f *HotStuffFactory) CreateModules( Persist: persister.New(f.db, cluster.ChainID()), VoteAggregator: voteAggregator, TimeoutAggregator: timeoutAggregator, - QCCreatedDistributor: qcDistributor, + VoteCollectorDistributor: qcDistributor, TimeoutCollectorDistributor: timeoutCollectorDistributor, FollowerDistributor: notifier.FollowerDistributor, }, metrics, nil diff --git a/follower/follower_builder.go b/follower/follower_builder.go index caa0773e546..d9b4941eaea 100644 --- a/follower/follower_builder.go +++ b/follower/follower_builder.go @@ -357,7 +357,7 @@ func FlowConsensusFollowerService(opts ...FollowerOption) *FollowerServiceBuilde FlowNodeBuilder: cmd.FlowNode(flow.RoleAccess.String(), config.baseOptions...), FollowerDistributor: pubsub.NewFollowerDistributor(), } - ret.FollowerDistributor.AddProtocolViolationConsumer(notifications.NewSlashingViolationsConsumer(ret.Logger)) + ret.FollowerDistributor.AddProposalViolationConsumer(notifications.NewSlashingViolationsConsumer(ret.Logger)) // the observer gets a version of the root snapshot file that does not contain any node addresses // hence skip all the root snapshot validations that involved an identity address ret.FlowNodeBuilder.SkipNwAddressBasedValidations = true From 041234f874c26a2b4a79e0d086d4e83e0421c49e Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Tue, 9 May 2023 13:50:50 +0300 Subject: [PATCH 25/38] Fixed initialization for collection nodes --- .../collection/epochmgr/factories/hotstuff.go | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/engine/collection/epochmgr/factories/hotstuff.go b/engine/collection/epochmgr/factories/hotstuff.go index 5c98803f48e..fada8e24194 100644 --- a/engine/collection/epochmgr/factories/hotstuff.go +++ b/engine/collection/epochmgr/factories/hotstuff.go @@ -75,11 +75,13 @@ func (f *HotStuffFactory) CreateModules( // setup metrics/logging with the new chain ID log := f.createLogger(cluster) metrics := f.createMetrics(cluster.ChainID()) + telemetryConsumer := notifications.NewTelemetryConsumer(log) + slashingConsumer := notifications.NewSlashingViolationsConsumer(log) notifier := pubsub.NewDistributor() notifier.AddConsumer(notifications.NewLogConsumer(log)) notifier.AddConsumer(hotmetrics.NewMetricsConsumer(metrics)) - notifier.AddConsumer(notifications.NewTelemetryConsumer(log)) - notifier.AddProposalViolationConsumer(notifications.NewSlashingViolationsConsumer(log)) + notifier.AddParticipantConsumer(telemetryConsumer) + notifier.AddProposalViolationConsumer(slashingConsumer) var ( err error @@ -112,11 +114,13 @@ func (f *HotStuffFactory) CreateModules( return nil, nil, err } - qcDistributor := pubsub.NewQCCreatedDistributor() + voteAggregationDistributor := pubsub.NewVoteAggregationDistributor() + voteAggregationDistributor.AddVoteCollectorConsumer(telemetryConsumer) + voteAggregationDistributor.AddVoteAggregationViolationConsumer(slashingConsumer) verifier := verification.NewStakingVerifier() validator := validatorImpl.NewMetricsWrapper(validatorImpl.New(committee, verifier), metrics) - voteProcessorFactory := votecollector.NewStakingVoteProcessorFactory(committee, qcDistributor.OnQcConstructedFromVotes) + voteProcessorFactory := votecollector.NewStakingVoteProcessorFactory(committee, voteAggregationDistributor.OnQcConstructedFromVotes) voteAggregator, err := consensus.NewVoteAggregator( log, metrics, @@ -125,7 +129,7 @@ func (f *HotStuffFactory) CreateModules( // since we don't want to aggregate votes for finalized view, // the lowest retained view starts with the next view of the last finalized view. finalizedBlock.View+1, - notifier, + voteAggregationDistributor, voteProcessorFactory, notifier.FollowerDistributor, ) @@ -133,9 +137,11 @@ func (f *HotStuffFactory) CreateModules( return nil, nil, err } - timeoutCollectorDistributor := pubsub.NewTimeoutCollectorDistributor() - timeoutProcessorFactory := timeoutcollector.NewTimeoutProcessorFactory(log, timeoutCollectorDistributor, committee, validator, msig.CollectorTimeoutTag) + timeoutCollectorDistributor := pubsub.NewTimeoutAggregationDistributor() + timeoutCollectorDistributor.AddTimeoutCollectorConsumer(telemetryConsumer) + timeoutCollectorDistributor.AddTimeoutAggregationViolationConsumer(slashingConsumer) + timeoutProcessorFactory := timeoutcollector.NewTimeoutProcessorFactory(log, timeoutCollectorDistributor, committee, validator, msig.CollectorTimeoutTag) timeoutAggregator, err := consensus.NewTimeoutAggregator( log, metrics, @@ -159,8 +165,8 @@ func (f *HotStuffFactory) CreateModules( Persist: persister.New(f.db, cluster.ChainID()), VoteAggregator: voteAggregator, TimeoutAggregator: timeoutAggregator, - VoteCollectorDistributor: qcDistributor, - TimeoutCollectorDistributor: timeoutCollectorDistributor, + VoteCollectorDistributor: voteAggregationDistributor.VoteCollectorDistributor, + TimeoutCollectorDistributor: timeoutCollectorDistributor.TimeoutCollectorDistributor, FollowerDistributor: notifier.FollowerDistributor, }, metrics, nil } From f40e60a8ee0e87648f2429c5649d40d6c5b52dff Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Tue, 9 May 2023 13:52:24 +0300 Subject: [PATCH 26/38] Updated mocks --- consensus/hotstuff/mocks/consumer.go | 35 ---------- consensus/hotstuff/mocks/event_loop.go | 10 +++ consensus/hotstuff/mocks/follower_consumer.go | 25 ------- .../hotstuff/mocks/participant_consumer.go | 10 --- .../mocks/proposal_violation_consumer.go | 38 +++++++++++ .../mocks/protocol_violation_consumer.go | 63 ------------------ .../hotstuff/mocks/qc_created_consumer.go | 34 ---------- .../mocks/timeout_aggregation_consumer.go | 66 +++++++++++++++++++ .../timeout_aggregation_violation_consumer.go | 38 +++++++++++ .../mocks/timeout_collector_consumer.go | 7 ++ .../mocks/vote_aggregation_consumer.go | 56 ++++++++++++++++ .../vote_aggregation_violation_consumer.go | 43 ++++++++++++ .../hotstuff/mocks/vote_collector_consumer.go | 41 ++++++++++++ 13 files changed, 299 insertions(+), 167 deletions(-) create mode 100644 consensus/hotstuff/mocks/proposal_violation_consumer.go delete mode 100644 consensus/hotstuff/mocks/protocol_violation_consumer.go delete mode 100644 consensus/hotstuff/mocks/qc_created_consumer.go create mode 100644 consensus/hotstuff/mocks/timeout_aggregation_consumer.go create mode 100644 consensus/hotstuff/mocks/timeout_aggregation_violation_consumer.go create mode 100644 consensus/hotstuff/mocks/vote_aggregation_consumer.go create mode 100644 consensus/hotstuff/mocks/vote_aggregation_violation_consumer.go create mode 100644 consensus/hotstuff/mocks/vote_collector_consumer.go diff --git a/consensus/hotstuff/mocks/consumer.go b/consensus/hotstuff/mocks/consumer.go index 919d333384d..aca84864891 100644 --- a/consensus/hotstuff/mocks/consumer.go +++ b/consensus/hotstuff/mocks/consumer.go @@ -33,16 +33,6 @@ func (_m *Consumer) OnDoubleProposeDetected(_a0 *model.Block, _a1 *model.Block) _m.Called(_a0, _a1) } -// OnDoubleTimeoutDetected provides a mock function with given fields: _a0, _a1 -func (_m *Consumer) OnDoubleTimeoutDetected(_a0 *model.TimeoutObject, _a1 *model.TimeoutObject) { - _m.Called(_a0, _a1) -} - -// OnDoubleVotingDetected provides a mock function with given fields: _a0, _a1 -func (_m *Consumer) OnDoubleVotingDetected(_a0 *model.Vote, _a1 *model.Vote) { - _m.Called(_a0, _a1) -} - // OnEventProcessed provides a mock function with given fields: func (_m *Consumer) OnEventProcessed() { _m.Called() @@ -58,16 +48,6 @@ func (_m *Consumer) OnInvalidBlockDetected(err model.InvalidBlockError) { _m.Called(err) } -// OnInvalidTimeoutDetected provides a mock function with given fields: err -func (_m *Consumer) OnInvalidTimeoutDetected(err model.InvalidTimeoutError) { - _m.Called(err) -} - -// OnInvalidVoteDetected provides a mock function with given fields: err -func (_m *Consumer) OnInvalidVoteDetected(err model.InvalidVoteError) { - _m.Called(err) -} - // OnLocalTimeout provides a mock function with given fields: currentView func (_m *Consumer) OnLocalTimeout(currentView uint64) { _m.Called(currentView) @@ -128,26 +108,11 @@ func (_m *Consumer) OnTcTriggeredViewChange(oldView uint64, newView uint64, tc * _m.Called(oldView, newView, tc) } -// OnTimeoutProcessed provides a mock function with given fields: timeout -func (_m *Consumer) OnTimeoutProcessed(timeout *model.TimeoutObject) { - _m.Called(timeout) -} - // OnViewChange provides a mock function with given fields: oldView, newView func (_m *Consumer) OnViewChange(oldView uint64, newView uint64) { _m.Called(oldView, newView) } -// OnVoteForInvalidBlockDetected provides a mock function with given fields: vote, invalidProposal -func (_m *Consumer) OnVoteForInvalidBlockDetected(vote *model.Vote, invalidProposal *model.Proposal) { - _m.Called(vote, invalidProposal) -} - -// OnVoteProcessed provides a mock function with given fields: vote -func (_m *Consumer) OnVoteProcessed(vote *model.Vote) { - _m.Called(vote) -} - type mockConstructorTestingTNewConsumer interface { mock.TestingT Cleanup(func()) diff --git a/consensus/hotstuff/mocks/event_loop.go b/consensus/hotstuff/mocks/event_loop.go index 3a15f4a4331..a1425da0629 100644 --- a/consensus/hotstuff/mocks/event_loop.go +++ b/consensus/hotstuff/mocks/event_loop.go @@ -58,6 +58,16 @@ func (_m *EventLoop) OnTcConstructedFromTimeouts(certificate *flow.TimeoutCertif _m.Called(certificate) } +// OnTimeoutProcessed provides a mock function with given fields: timeout +func (_m *EventLoop) OnTimeoutProcessed(timeout *model.TimeoutObject) { + _m.Called(timeout) +} + +// OnVoteProcessed provides a mock function with given fields: vote +func (_m *EventLoop) OnVoteProcessed(vote *model.Vote) { + _m.Called(vote) +} + // Ready provides a mock function with given fields: func (_m *EventLoop) Ready() <-chan struct{} { ret := _m.Called() diff --git a/consensus/hotstuff/mocks/follower_consumer.go b/consensus/hotstuff/mocks/follower_consumer.go index a8dcd9c9681..f94f43bf534 100644 --- a/consensus/hotstuff/mocks/follower_consumer.go +++ b/consensus/hotstuff/mocks/follower_consumer.go @@ -22,16 +22,6 @@ func (_m *FollowerConsumer) OnDoubleProposeDetected(_a0 *model.Block, _a1 *model _m.Called(_a0, _a1) } -// OnDoubleTimeoutDetected provides a mock function with given fields: _a0, _a1 -func (_m *FollowerConsumer) OnDoubleTimeoutDetected(_a0 *model.TimeoutObject, _a1 *model.TimeoutObject) { - _m.Called(_a0, _a1) -} - -// OnDoubleVotingDetected provides a mock function with given fields: _a0, _a1 -func (_m *FollowerConsumer) OnDoubleVotingDetected(_a0 *model.Vote, _a1 *model.Vote) { - _m.Called(_a0, _a1) -} - // OnFinalizedBlock provides a mock function with given fields: _a0 func (_m *FollowerConsumer) OnFinalizedBlock(_a0 *model.Block) { _m.Called(_a0) @@ -42,21 +32,6 @@ func (_m *FollowerConsumer) OnInvalidBlockDetected(err model.InvalidBlockError) _m.Called(err) } -// OnInvalidTimeoutDetected provides a mock function with given fields: err -func (_m *FollowerConsumer) OnInvalidTimeoutDetected(err model.InvalidTimeoutError) { - _m.Called(err) -} - -// OnInvalidVoteDetected provides a mock function with given fields: err -func (_m *FollowerConsumer) OnInvalidVoteDetected(err model.InvalidVoteError) { - _m.Called(err) -} - -// OnVoteForInvalidBlockDetected provides a mock function with given fields: vote, invalidProposal -func (_m *FollowerConsumer) OnVoteForInvalidBlockDetected(vote *model.Vote, invalidProposal *model.Proposal) { - _m.Called(vote, invalidProposal) -} - type mockConstructorTestingTNewFollowerConsumer interface { mock.TestingT Cleanup(func()) diff --git a/consensus/hotstuff/mocks/participant_consumer.go b/consensus/hotstuff/mocks/participant_consumer.go index 739ced2b00d..2d2b4141093 100644 --- a/consensus/hotstuff/mocks/participant_consumer.go +++ b/consensus/hotstuff/mocks/participant_consumer.go @@ -71,21 +71,11 @@ func (_m *ParticipantConsumer) OnTcTriggeredViewChange(oldView uint64, newView u _m.Called(oldView, newView, tc) } -// OnTimeoutProcessed provides a mock function with given fields: timeout -func (_m *ParticipantConsumer) OnTimeoutProcessed(timeout *model.TimeoutObject) { - _m.Called(timeout) -} - // OnViewChange provides a mock function with given fields: oldView, newView func (_m *ParticipantConsumer) OnViewChange(oldView uint64, newView uint64) { _m.Called(oldView, newView) } -// OnVoteProcessed provides a mock function with given fields: vote -func (_m *ParticipantConsumer) OnVoteProcessed(vote *model.Vote) { - _m.Called(vote) -} - type mockConstructorTestingTNewParticipantConsumer interface { mock.TestingT Cleanup(func()) diff --git a/consensus/hotstuff/mocks/proposal_violation_consumer.go b/consensus/hotstuff/mocks/proposal_violation_consumer.go new file mode 100644 index 00000000000..0a68f913038 --- /dev/null +++ b/consensus/hotstuff/mocks/proposal_violation_consumer.go @@ -0,0 +1,38 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mocks + +import ( + model "github.com/onflow/flow-go/consensus/hotstuff/model" + mock "github.com/stretchr/testify/mock" +) + +// ProposalViolationConsumer is an autogenerated mock type for the ProposalViolationConsumer type +type ProposalViolationConsumer struct { + mock.Mock +} + +// OnDoubleProposeDetected provides a mock function with given fields: _a0, _a1 +func (_m *ProposalViolationConsumer) OnDoubleProposeDetected(_a0 *model.Block, _a1 *model.Block) { + _m.Called(_a0, _a1) +} + +// OnInvalidBlockDetected provides a mock function with given fields: err +func (_m *ProposalViolationConsumer) OnInvalidBlockDetected(err model.InvalidBlockError) { + _m.Called(err) +} + +type mockConstructorTestingTNewProposalViolationConsumer interface { + mock.TestingT + Cleanup(func()) +} + +// NewProposalViolationConsumer creates a new instance of ProposalViolationConsumer. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewProposalViolationConsumer(t mockConstructorTestingTNewProposalViolationConsumer) *ProposalViolationConsumer { + mock := &ProposalViolationConsumer{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/consensus/hotstuff/mocks/protocol_violation_consumer.go b/consensus/hotstuff/mocks/protocol_violation_consumer.go deleted file mode 100644 index b8785f0881f..00000000000 --- a/consensus/hotstuff/mocks/protocol_violation_consumer.go +++ /dev/null @@ -1,63 +0,0 @@ -// Code generated by mockery v2.21.4. DO NOT EDIT. - -package mocks - -import ( - model "github.com/onflow/flow-go/consensus/hotstuff/model" - mock "github.com/stretchr/testify/mock" -) - -// ProtocolViolationConsumer is an autogenerated mock type for the ProtocolViolationConsumer type -type ProtocolViolationConsumer struct { - mock.Mock -} - -// OnDoubleProposeDetected provides a mock function with given fields: _a0, _a1 -func (_m *ProtocolViolationConsumer) OnDoubleProposeDetected(_a0 *model.Block, _a1 *model.Block) { - _m.Called(_a0, _a1) -} - -// OnDoubleTimeoutDetected provides a mock function with given fields: _a0, _a1 -func (_m *ProtocolViolationConsumer) OnDoubleTimeoutDetected(_a0 *model.TimeoutObject, _a1 *model.TimeoutObject) { - _m.Called(_a0, _a1) -} - -// OnDoubleVotingDetected provides a mock function with given fields: _a0, _a1 -func (_m *ProtocolViolationConsumer) OnDoubleVotingDetected(_a0 *model.Vote, _a1 *model.Vote) { - _m.Called(_a0, _a1) -} - -// OnInvalidBlockDetected provides a mock function with given fields: err -func (_m *ProtocolViolationConsumer) OnInvalidBlockDetected(err model.InvalidBlockError) { - _m.Called(err) -} - -// OnInvalidTimeoutDetected provides a mock function with given fields: err -func (_m *ProtocolViolationConsumer) OnInvalidTimeoutDetected(err model.InvalidTimeoutError) { - _m.Called(err) -} - -// OnInvalidVoteDetected provides a mock function with given fields: err -func (_m *ProtocolViolationConsumer) OnInvalidVoteDetected(err model.InvalidVoteError) { - _m.Called(err) -} - -// OnVoteForInvalidBlockDetected provides a mock function with given fields: vote, invalidProposal -func (_m *ProtocolViolationConsumer) OnVoteForInvalidBlockDetected(vote *model.Vote, invalidProposal *model.Proposal) { - _m.Called(vote, invalidProposal) -} - -type mockConstructorTestingTNewProtocolViolationConsumer interface { - mock.TestingT - Cleanup(func()) -} - -// NewProtocolViolationConsumer creates a new instance of ProtocolViolationConsumer. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewProtocolViolationConsumer(t mockConstructorTestingTNewProtocolViolationConsumer) *ProtocolViolationConsumer { - mock := &ProtocolViolationConsumer{} - mock.Mock.Test(t) - - t.Cleanup(func() { mock.AssertExpectations(t) }) - - return mock -} diff --git a/consensus/hotstuff/mocks/qc_created_consumer.go b/consensus/hotstuff/mocks/qc_created_consumer.go deleted file mode 100644 index e20bd948fb5..00000000000 --- a/consensus/hotstuff/mocks/qc_created_consumer.go +++ /dev/null @@ -1,34 +0,0 @@ -// Code generated by mockery v2.21.4. DO NOT EDIT. - -package mocks - -import ( - flow "github.com/onflow/flow-go/model/flow" - - mock "github.com/stretchr/testify/mock" -) - -// QCCreatedConsumer is an autogenerated mock type for the QCCreatedConsumer type -type QCCreatedConsumer struct { - mock.Mock -} - -// OnQcConstructedFromVotes provides a mock function with given fields: _a0 -func (_m *QCCreatedConsumer) OnQcConstructedFromVotes(_a0 *flow.QuorumCertificate) { - _m.Called(_a0) -} - -type mockConstructorTestingTNewQCCreatedConsumer interface { - mock.TestingT - Cleanup(func()) -} - -// NewQCCreatedConsumer creates a new instance of QCCreatedConsumer. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewQCCreatedConsumer(t mockConstructorTestingTNewQCCreatedConsumer) *QCCreatedConsumer { - mock := &QCCreatedConsumer{} - mock.Mock.Test(t) - - t.Cleanup(func() { mock.AssertExpectations(t) }) - - return mock -} diff --git a/consensus/hotstuff/mocks/timeout_aggregation_consumer.go b/consensus/hotstuff/mocks/timeout_aggregation_consumer.go new file mode 100644 index 00000000000..c123201f956 --- /dev/null +++ b/consensus/hotstuff/mocks/timeout_aggregation_consumer.go @@ -0,0 +1,66 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mocks + +import ( + flow "github.com/onflow/flow-go/model/flow" + + mock "github.com/stretchr/testify/mock" + + model "github.com/onflow/flow-go/consensus/hotstuff/model" +) + +// TimeoutAggregationConsumer is an autogenerated mock type for the TimeoutAggregationConsumer type +type TimeoutAggregationConsumer struct { + mock.Mock +} + +// OnDoubleTimeoutDetected provides a mock function with given fields: _a0, _a1 +func (_m *TimeoutAggregationConsumer) OnDoubleTimeoutDetected(_a0 *model.TimeoutObject, _a1 *model.TimeoutObject) { + _m.Called(_a0, _a1) +} + +// OnInvalidTimeoutDetected provides a mock function with given fields: err +func (_m *TimeoutAggregationConsumer) OnInvalidTimeoutDetected(err model.InvalidTimeoutError) { + _m.Called(err) +} + +// OnNewQcDiscovered provides a mock function with given fields: certificate +func (_m *TimeoutAggregationConsumer) OnNewQcDiscovered(certificate *flow.QuorumCertificate) { + _m.Called(certificate) +} + +// OnNewTcDiscovered provides a mock function with given fields: certificate +func (_m *TimeoutAggregationConsumer) OnNewTcDiscovered(certificate *flow.TimeoutCertificate) { + _m.Called(certificate) +} + +// OnPartialTcCreated provides a mock function with given fields: view, newestQC, lastViewTC +func (_m *TimeoutAggregationConsumer) OnPartialTcCreated(view uint64, newestQC *flow.QuorumCertificate, lastViewTC *flow.TimeoutCertificate) { + _m.Called(view, newestQC, lastViewTC) +} + +// OnTcConstructedFromTimeouts provides a mock function with given fields: certificate +func (_m *TimeoutAggregationConsumer) OnTcConstructedFromTimeouts(certificate *flow.TimeoutCertificate) { + _m.Called(certificate) +} + +// OnTimeoutProcessed provides a mock function with given fields: timeout +func (_m *TimeoutAggregationConsumer) OnTimeoutProcessed(timeout *model.TimeoutObject) { + _m.Called(timeout) +} + +type mockConstructorTestingTNewTimeoutAggregationConsumer interface { + mock.TestingT + Cleanup(func()) +} + +// NewTimeoutAggregationConsumer creates a new instance of TimeoutAggregationConsumer. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewTimeoutAggregationConsumer(t mockConstructorTestingTNewTimeoutAggregationConsumer) *TimeoutAggregationConsumer { + mock := &TimeoutAggregationConsumer{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/consensus/hotstuff/mocks/timeout_aggregation_violation_consumer.go b/consensus/hotstuff/mocks/timeout_aggregation_violation_consumer.go new file mode 100644 index 00000000000..552f8650f9f --- /dev/null +++ b/consensus/hotstuff/mocks/timeout_aggregation_violation_consumer.go @@ -0,0 +1,38 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mocks + +import ( + model "github.com/onflow/flow-go/consensus/hotstuff/model" + mock "github.com/stretchr/testify/mock" +) + +// TimeoutAggregationViolationConsumer is an autogenerated mock type for the TimeoutAggregationViolationConsumer type +type TimeoutAggregationViolationConsumer struct { + mock.Mock +} + +// OnDoubleTimeoutDetected provides a mock function with given fields: _a0, _a1 +func (_m *TimeoutAggregationViolationConsumer) OnDoubleTimeoutDetected(_a0 *model.TimeoutObject, _a1 *model.TimeoutObject) { + _m.Called(_a0, _a1) +} + +// OnInvalidTimeoutDetected provides a mock function with given fields: err +func (_m *TimeoutAggregationViolationConsumer) OnInvalidTimeoutDetected(err model.InvalidTimeoutError) { + _m.Called(err) +} + +type mockConstructorTestingTNewTimeoutAggregationViolationConsumer interface { + mock.TestingT + Cleanup(func()) +} + +// NewTimeoutAggregationViolationConsumer creates a new instance of TimeoutAggregationViolationConsumer. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewTimeoutAggregationViolationConsumer(t mockConstructorTestingTNewTimeoutAggregationViolationConsumer) *TimeoutAggregationViolationConsumer { + mock := &TimeoutAggregationViolationConsumer{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/consensus/hotstuff/mocks/timeout_collector_consumer.go b/consensus/hotstuff/mocks/timeout_collector_consumer.go index 459cfb8dd14..629f33f9a14 100644 --- a/consensus/hotstuff/mocks/timeout_collector_consumer.go +++ b/consensus/hotstuff/mocks/timeout_collector_consumer.go @@ -6,6 +6,8 @@ import ( flow "github.com/onflow/flow-go/model/flow" mock "github.com/stretchr/testify/mock" + + model "github.com/onflow/flow-go/consensus/hotstuff/model" ) // TimeoutCollectorConsumer is an autogenerated mock type for the TimeoutCollectorConsumer type @@ -33,6 +35,11 @@ func (_m *TimeoutCollectorConsumer) OnTcConstructedFromTimeouts(certificate *flo _m.Called(certificate) } +// OnTimeoutProcessed provides a mock function with given fields: timeout +func (_m *TimeoutCollectorConsumer) OnTimeoutProcessed(timeout *model.TimeoutObject) { + _m.Called(timeout) +} + type mockConstructorTestingTNewTimeoutCollectorConsumer interface { mock.TestingT Cleanup(func()) diff --git a/consensus/hotstuff/mocks/vote_aggregation_consumer.go b/consensus/hotstuff/mocks/vote_aggregation_consumer.go new file mode 100644 index 00000000000..0ab7b7f53aa --- /dev/null +++ b/consensus/hotstuff/mocks/vote_aggregation_consumer.go @@ -0,0 +1,56 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mocks + +import ( + flow "github.com/onflow/flow-go/model/flow" + + mock "github.com/stretchr/testify/mock" + + model "github.com/onflow/flow-go/consensus/hotstuff/model" +) + +// VoteAggregationConsumer is an autogenerated mock type for the VoteAggregationConsumer type +type VoteAggregationConsumer struct { + mock.Mock +} + +// OnDoubleVotingDetected provides a mock function with given fields: _a0, _a1 +func (_m *VoteAggregationConsumer) OnDoubleVotingDetected(_a0 *model.Vote, _a1 *model.Vote) { + _m.Called(_a0, _a1) +} + +// OnInvalidVoteDetected provides a mock function with given fields: err +func (_m *VoteAggregationConsumer) OnInvalidVoteDetected(err model.InvalidVoteError) { + _m.Called(err) +} + +// OnQcConstructedFromVotes provides a mock function with given fields: _a0 +func (_m *VoteAggregationConsumer) OnQcConstructedFromVotes(_a0 *flow.QuorumCertificate) { + _m.Called(_a0) +} + +// OnVoteForInvalidBlockDetected provides a mock function with given fields: vote, invalidProposal +func (_m *VoteAggregationConsumer) OnVoteForInvalidBlockDetected(vote *model.Vote, invalidProposal *model.Proposal) { + _m.Called(vote, invalidProposal) +} + +// OnVoteProcessed provides a mock function with given fields: vote +func (_m *VoteAggregationConsumer) OnVoteProcessed(vote *model.Vote) { + _m.Called(vote) +} + +type mockConstructorTestingTNewVoteAggregationConsumer interface { + mock.TestingT + Cleanup(func()) +} + +// NewVoteAggregationConsumer creates a new instance of VoteAggregationConsumer. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewVoteAggregationConsumer(t mockConstructorTestingTNewVoteAggregationConsumer) *VoteAggregationConsumer { + mock := &VoteAggregationConsumer{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/consensus/hotstuff/mocks/vote_aggregation_violation_consumer.go b/consensus/hotstuff/mocks/vote_aggregation_violation_consumer.go new file mode 100644 index 00000000000..c27e40c1513 --- /dev/null +++ b/consensus/hotstuff/mocks/vote_aggregation_violation_consumer.go @@ -0,0 +1,43 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mocks + +import ( + model "github.com/onflow/flow-go/consensus/hotstuff/model" + mock "github.com/stretchr/testify/mock" +) + +// VoteAggregationViolationConsumer is an autogenerated mock type for the VoteAggregationViolationConsumer type +type VoteAggregationViolationConsumer struct { + mock.Mock +} + +// OnDoubleVotingDetected provides a mock function with given fields: _a0, _a1 +func (_m *VoteAggregationViolationConsumer) OnDoubleVotingDetected(_a0 *model.Vote, _a1 *model.Vote) { + _m.Called(_a0, _a1) +} + +// OnInvalidVoteDetected provides a mock function with given fields: err +func (_m *VoteAggregationViolationConsumer) OnInvalidVoteDetected(err model.InvalidVoteError) { + _m.Called(err) +} + +// OnVoteForInvalidBlockDetected provides a mock function with given fields: vote, invalidProposal +func (_m *VoteAggregationViolationConsumer) OnVoteForInvalidBlockDetected(vote *model.Vote, invalidProposal *model.Proposal) { + _m.Called(vote, invalidProposal) +} + +type mockConstructorTestingTNewVoteAggregationViolationConsumer interface { + mock.TestingT + Cleanup(func()) +} + +// NewVoteAggregationViolationConsumer creates a new instance of VoteAggregationViolationConsumer. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewVoteAggregationViolationConsumer(t mockConstructorTestingTNewVoteAggregationViolationConsumer) *VoteAggregationViolationConsumer { + mock := &VoteAggregationViolationConsumer{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/consensus/hotstuff/mocks/vote_collector_consumer.go b/consensus/hotstuff/mocks/vote_collector_consumer.go new file mode 100644 index 00000000000..5c5b064e975 --- /dev/null +++ b/consensus/hotstuff/mocks/vote_collector_consumer.go @@ -0,0 +1,41 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mocks + +import ( + flow "github.com/onflow/flow-go/model/flow" + + mock "github.com/stretchr/testify/mock" + + model "github.com/onflow/flow-go/consensus/hotstuff/model" +) + +// VoteCollectorConsumer is an autogenerated mock type for the VoteCollectorConsumer type +type VoteCollectorConsumer struct { + mock.Mock +} + +// OnQcConstructedFromVotes provides a mock function with given fields: _a0 +func (_m *VoteCollectorConsumer) OnQcConstructedFromVotes(_a0 *flow.QuorumCertificate) { + _m.Called(_a0) +} + +// OnVoteProcessed provides a mock function with given fields: vote +func (_m *VoteCollectorConsumer) OnVoteProcessed(vote *model.Vote) { + _m.Called(vote) +} + +type mockConstructorTestingTNewVoteCollectorConsumer interface { + mock.TestingT + Cleanup(func()) +} + +// NewVoteCollectorConsumer creates a new instance of VoteCollectorConsumer. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewVoteCollectorConsumer(t mockConstructorTestingTNewVoteCollectorConsumer) *VoteCollectorConsumer { + mock := &VoteCollectorConsumer{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} From c22f9a881dd04e05896ade32d78f9bc8b40b673c Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Tue, 9 May 2023 14:14:09 +0300 Subject: [PATCH 27/38] Fixed consensus tests --- .../hotstuff/integration/instance_test.go | 21 +++++++++------- .../hotstuff/notifications/log_consumer.go | 10 +++++++- .../timeout_aggregator_test.go | 3 --- .../timeout_collector_test.go | 24 +++++++++---------- .../voteaggregator/vote_aggregator_test.go | 4 ++-- .../votecollector/statemachine_test.go | 5 ++-- consensus/integration/nodes_test.go | 23 +++++++++--------- 7 files changed, 47 insertions(+), 43 deletions(-) diff --git a/consensus/hotstuff/integration/instance_test.go b/consensus/hotstuff/integration/instance_test.go index a082e2becbb..ce742143567 100644 --- a/consensus/hotstuff/integration/instance_test.go +++ b/consensus/hotstuff/integration/instance_test.go @@ -432,7 +432,8 @@ func NewInstance(t *testing.T, options ...Option) *Instance { ) }, nil).Maybe() - createCollectorFactoryMethod := votecollector.NewStateMachineFactory(log, notifier, voteProcessorFactory.Create) + voteAggregationDistributor := pubsub.NewVoteAggregationDistributor() + createCollectorFactoryMethod := votecollector.NewStateMachineFactory(log, voteAggregationDistributor, voteProcessorFactory.Create) voteCollectors := voteaggregator.NewVoteCollectors(log, livenessData.CurrentView, workerpool.New(2), createCollectorFactoryMethod) metricsCollector := metrics.NewNoopCollector() @@ -443,14 +444,14 @@ func NewInstance(t *testing.T, options ...Option) *Instance { metricsCollector, metricsCollector, metricsCollector, - notifier, + voteAggregationDistributor, livenessData.CurrentView, voteCollectors, ) require.NoError(t, err) // initialize factories for timeout collector and timeout processor - collectorDistributor := pubsub.NewTimeoutCollectorDistributor() + timeoutAggregationDistributor := pubsub.NewTimeoutAggregationDistributor() timeoutProcessorFactory := mocks.NewTimeoutProcessorFactory(t) timeoutProcessorFactory.On("Create", mock.Anything).Return( func(view uint64) hotstuff.TimeoutProcessor { @@ -491,15 +492,14 @@ func NewInstance(t *testing.T, options ...Option) *Instance { in.committee, in.validator, aggregator, - collectorDistributor, + timeoutAggregationDistributor, ) require.NoError(t, err) return p }, nil).Maybe() timeoutCollectorFactory := timeoutcollector.NewTimeoutCollectorFactory( unittest.Logger(), - notifier, - collectorDistributor, + timeoutAggregationDistributor, timeoutProcessorFactory, ) timeoutCollectors := timeoutaggregator.NewTimeoutCollectors(log, livenessData.CurrentView, timeoutCollectorFactory) @@ -510,7 +510,6 @@ func NewInstance(t *testing.T, options ...Option) *Instance { metricsCollector, metricsCollector, metricsCollector, - notifier, livenessData.CurrentView, timeoutCollectors, ) @@ -539,8 +538,10 @@ func NewInstance(t *testing.T, options ...Option) *Instance { ) require.NoError(t, err) - collectorDistributor.AddTimeoutCollectorConsumer(logConsumer) - collectorDistributor.AddTimeoutCollectorConsumer(&in) + timeoutAggregationDistributor.AddTimeoutCollectorConsumer(logConsumer) + timeoutAggregationDistributor.AddTimeoutCollectorConsumer(&in) + + voteAggregationDistributor.AddVoteCollectorConsumer(logConsumer) return &in } @@ -665,3 +666,5 @@ func (in *Instance) OnNewQcDiscovered(qc *flow.QuorumCertificate) { func (in *Instance) OnNewTcDiscovered(tc *flow.TimeoutCertificate) { in.queue <- tc } + +func (in *Instance) OnTimeoutProcessed(*model.TimeoutObject) {} diff --git a/consensus/hotstuff/notifications/log_consumer.go b/consensus/hotstuff/notifications/log_consumer.go index 65ed347bed3..eba4d97cacf 100644 --- a/consensus/hotstuff/notifications/log_consumer.go +++ b/consensus/hotstuff/notifications/log_consumer.go @@ -18,7 +18,8 @@ type LogConsumer struct { } var _ hotstuff.Consumer = (*LogConsumer)(nil) -var _ hotstuff.TimeoutCollectorConsumer = (*LogConsumer)(nil) +var _ hotstuff.TimeoutAggregationConsumer = (*LogConsumer)(nil) +var _ hotstuff.VoteAggregationConsumer = (*LogConsumer)(nil) func NewLogConsumer(log zerolog.Logger) *LogConsumer { lc := &LogConsumer{ @@ -286,3 +287,10 @@ func (lc *LogConsumer) OnOwnProposal(header *flow.Header, targetPublicationTime Time("target_publication_time", targetPublicationTime). Msg("publishing HotStuff block proposal") } + +func (lc *LogConsumer) OnQcConstructedFromVotes(qc *flow.QuorumCertificate) { + lc.log.Info(). + Uint64("view", qc.View). + Hex("block_id", qc.BlockID[:]). + Msg("QC constructed from votes") +} diff --git a/consensus/hotstuff/timeoutaggregator/timeout_aggregator_test.go b/consensus/hotstuff/timeoutaggregator/timeout_aggregator_test.go index fddce6bd717..e8fd19b1bb8 100644 --- a/consensus/hotstuff/timeoutaggregator/timeout_aggregator_test.go +++ b/consensus/hotstuff/timeoutaggregator/timeout_aggregator_test.go @@ -33,14 +33,12 @@ type TimeoutAggregatorTestSuite struct { highestKnownView uint64 aggregator *TimeoutAggregator collectors *mocks.TimeoutCollectors - consumer *mocks.Consumer stopAggregator context.CancelFunc } func (s *TimeoutAggregatorTestSuite) SetupTest() { var err error s.collectors = mocks.NewTimeoutCollectors(s.T()) - s.consumer = mocks.NewConsumer(s.T()) s.lowestRetainedView = 100 @@ -51,7 +49,6 @@ func (s *TimeoutAggregatorTestSuite) SetupTest() { metricsCollector, metricsCollector, metricsCollector, - s.consumer, s.lowestRetainedView, s.collectors, ) diff --git a/consensus/hotstuff/timeoutcollector/timeout_collector_test.go b/consensus/hotstuff/timeoutcollector/timeout_collector_test.go index 691209cb179..3c83801cf72 100644 --- a/consensus/hotstuff/timeoutcollector/timeout_collector_test.go +++ b/consensus/hotstuff/timeoutcollector/timeout_collector_test.go @@ -27,23 +27,21 @@ func TestTimeoutCollector(t *testing.T) { type TimeoutCollectorTestSuite struct { suite.Suite - view uint64 - notifier *mocks.Consumer - collectorNotifier *mocks.TimeoutCollectorConsumer - processor *mocks.TimeoutProcessor - collector *TimeoutCollector + view uint64 + notifier *mocks.TimeoutAggregationConsumer + processor *mocks.TimeoutProcessor + collector *TimeoutCollector } func (s *TimeoutCollectorTestSuite) SetupTest() { s.view = 1000 - s.notifier = mocks.NewConsumer(s.T()) - s.collectorNotifier = mocks.NewTimeoutCollectorConsumer(s.T()) + s.notifier = mocks.NewTimeoutAggregationConsumer(s.T()) s.processor = mocks.NewTimeoutProcessor(s.T()) - s.collectorNotifier.On("OnNewQcDiscovered", mock.Anything).Maybe() - s.collectorNotifier.On("OnNewTcDiscovered", mock.Anything).Maybe() + s.notifier.On("OnNewQcDiscovered", mock.Anything).Maybe() + s.notifier.On("OnNewTcDiscovered", mock.Anything).Maybe() - s.collector = NewTimeoutCollector(unittest.Logger(), s.view, s.notifier, s.collectorNotifier, s.processor) + s.collector = NewTimeoutCollector(unittest.Logger(), s.view, s.notifier, s.processor) } // TestView tests that `View` returns the same value that was passed in constructor @@ -145,10 +143,10 @@ func (s *TimeoutCollectorTestSuite) TestAddTimeout_TONotifications() { s.T().Fatal("invalid test configuration") } - *s.collectorNotifier = *mocks.NewTimeoutCollectorConsumer(s.T()) + *s.notifier = *mocks.NewTimeoutAggregationConsumer(s.T()) var highestReportedQC *flow.QuorumCertificate - s.collectorNotifier.On("OnNewQcDiscovered", mock.Anything).Run(func(args mock.Arguments) { + s.notifier.On("OnNewQcDiscovered", mock.Anything).Run(func(args mock.Arguments) { qc := args.Get(0).(*flow.QuorumCertificate) if highestReportedQC == nil || highestReportedQC.View < qc.View { highestReportedQC = qc @@ -156,7 +154,7 @@ func (s *TimeoutCollectorTestSuite) TestAddTimeout_TONotifications() { }) lastViewTC := helper.MakeTC(helper.WithTCView(s.view - 1)) - s.collectorNotifier.On("OnNewTcDiscovered", lastViewTC).Once() + s.notifier.On("OnNewTcDiscovered", lastViewTC).Once() timeouts := make([]*model.TimeoutObject, 0, qcCount) for i := 0; i < qcCount; i++ { diff --git a/consensus/hotstuff/voteaggregator/vote_aggregator_test.go b/consensus/hotstuff/voteaggregator/vote_aggregator_test.go index 792c42cbca5..5e753689177 100644 --- a/consensus/hotstuff/voteaggregator/vote_aggregator_test.go +++ b/consensus/hotstuff/voteaggregator/vote_aggregator_test.go @@ -29,7 +29,7 @@ type VoteAggregatorTestSuite struct { aggregator *VoteAggregator collectors *mocks.VoteCollectors - consumer *mocks.Consumer + consumer *mocks.VoteAggregationConsumer stopAggregator context.CancelFunc errs <-chan error } @@ -37,7 +37,7 @@ type VoteAggregatorTestSuite struct { func (s *VoteAggregatorTestSuite) SetupTest() { var err error s.collectors = mocks.NewVoteCollectors(s.T()) - s.consumer = mocks.NewConsumer(s.T()) + s.consumer = mocks.NewVoteAggregationConsumer(s.T()) s.collectors.On("Start", mock.Anything).Once() unittest.ReadyDoneify(s.collectors) diff --git a/consensus/hotstuff/votecollector/statemachine_test.go b/consensus/hotstuff/votecollector/statemachine_test.go index 8ad19e98903..007dcce1fe2 100644 --- a/consensus/hotstuff/votecollector/statemachine_test.go +++ b/consensus/hotstuff/votecollector/statemachine_test.go @@ -32,7 +32,7 @@ type StateMachineTestSuite struct { suite.Suite view uint64 - notifier *mocks.Consumer + notifier *mocks.VoteAggregationConsumer workerPool *workerpool.WorkerPool factoryMethod VerifyingVoteProcessorFactory mockedProcessors map[flow.Identifier]*mocks.VerifyingVoteProcessor @@ -49,7 +49,7 @@ func (s *StateMachineTestSuite) TearDownTest() { func (s *StateMachineTestSuite) SetupTest() { s.view = 1000 s.mockedProcessors = make(map[flow.Identifier]*mocks.VerifyingVoteProcessor) - s.notifier = &mocks.Consumer{} + s.notifier = mocks.NewVoteAggregationConsumer(s.T()) s.factoryMethod = func(log zerolog.Logger, block *model.Proposal) (hotstuff.VerifyingVoteProcessor, error) { if processor, found := s.mockedProcessors[block.Block.BlockID]; found { @@ -152,7 +152,6 @@ func (s *StateMachineTestSuite) TestAddVote_VerifyingState() { s.T().Run("add-invalid-vote", func(t *testing.T) { vote := unittest.VoteForBlockFixture(block, unittest.WithVoteView(s.view)) processor.On("Process", vote).Return(model.NewInvalidVoteErrorf(vote, "")).Once() - s.notifier.On("OnVoteProcessed", vote).Once() s.notifier.On("OnInvalidVoteDetected", mock.Anything).Run(func(args mock.Arguments) { invalidVoteErr := args.Get(0).(model.InvalidVoteError) require.Equal(s.T(), vote, invalidVoteErr.Vote) diff --git a/consensus/integration/nodes_test.go b/consensus/integration/nodes_test.go index b6ce1f10c2d..dcfd925c9f9 100644 --- a/consensus/integration/nodes_test.go +++ b/consensus/integration/nodes_test.go @@ -480,7 +480,8 @@ func createNode( syncCore, err := synccore.New(log, synccore.DefaultConfig(), metricsCollector, rootHeader.ChainID) require.NoError(t, err) - qcDistributor := pubsub.NewQCCreatedDistributor() + voteAggregationDistributor := pubsub.NewVoteAggregationDistributor() + voteAggregationDistributor.AddVoteAggregationConsumer(logConsumer) forks, err := consensus.NewForks(rootHeader, headersDB, final, notifier, rootHeader, rootQC) require.NoError(t, err) @@ -514,9 +515,9 @@ func createNode( livenessData, err := persist.GetLivenessData() require.NoError(t, err) - voteProcessorFactory := votecollector.NewCombinedVoteProcessorFactory(committee, qcDistributor.OnQcConstructedFromVotes) + voteProcessorFactory := votecollector.NewCombinedVoteProcessorFactory(committee, voteAggregationDistributor.OnQcConstructedFromVotes) - createCollectorFactoryMethod := votecollector.NewStateMachineFactory(log, notifier, voteProcessorFactory.Create) + createCollectorFactoryMethod := votecollector.NewStateMachineFactory(log, voteAggregationDistributor, voteProcessorFactory.Create) voteCollectors := voteaggregator.NewVoteCollectors(log, livenessData.CurrentView, workerpool.New(2), createCollectorFactoryMethod) voteAggregator, err := voteaggregator.NewVoteAggregator( @@ -524,26 +525,25 @@ func createNode( metricsCollector, metricsCollector, metricsCollector, - notifier, + voteAggregationDistributor, livenessData.CurrentView, voteCollectors, ) require.NoError(t, err) - timeoutCollectorDistributor := pubsub.NewTimeoutCollectorDistributor() - timeoutCollectorDistributor.AddTimeoutCollectorConsumer(logConsumer) + timeoutAggregationDistributor := pubsub.NewTimeoutAggregationDistributor() + timeoutAggregationDistributor.AddTimeoutCollectorConsumer(logConsumer) timeoutProcessorFactory := timeoutcollector.NewTimeoutProcessorFactory( log, - timeoutCollectorDistributor, + timeoutAggregationDistributor, committee, validator, msig.ConsensusTimeoutTag, ) timeoutCollectorsFactory := timeoutcollector.NewTimeoutCollectorFactory( log, - notifier, - timeoutCollectorDistributor, + timeoutAggregationDistributor, timeoutProcessorFactory, ) timeoutCollectors := timeoutaggregator.NewTimeoutCollectors(log, livenessData.CurrentView, timeoutCollectorsFactory) @@ -553,7 +553,6 @@ func createNode( metricsCollector, metricsCollector, metricsCollector, - notifier, livenessData.CurrentView, timeoutCollectors, ) @@ -566,8 +565,8 @@ func createNode( Committee: committee, Signer: signer, Persist: persist, - VoteCollectorDistributor: qcDistributor, - TimeoutCollectorDistributor: timeoutCollectorDistributor, + VoteCollectorDistributor: voteAggregationDistributor.VoteCollectorDistributor, + TimeoutCollectorDistributor: timeoutAggregationDistributor.TimeoutCollectorDistributor, VoteAggregator: voteAggregator, TimeoutAggregator: timeoutAggregator, } From d0a71ab4df55f0e745a639dcf88e3d8ed8f0d73b Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Tue, 9 May 2023 14:25:33 +0300 Subject: [PATCH 28/38] Fixed compliance tests --- engine/collection/compliance/core.go | 8 ++++---- engine/collection/compliance/core_test.go | 8 ++++---- engine/common/follower/compliance_core.go | 6 +++--- engine/consensus/compliance/core.go | 8 ++++---- engine/consensus/compliance/core_test.go | 8 ++++---- 5 files changed, 19 insertions(+), 19 deletions(-) diff --git a/engine/collection/compliance/core.go b/engine/collection/compliance/core.go index 879c02bc34d..39cb9e4663c 100644 --- a/engine/collection/compliance/core.go +++ b/engine/collection/compliance/core.go @@ -40,7 +40,7 @@ type Core struct { mempoolMetrics module.MempoolMetrics hotstuffMetrics module.HotstuffMetrics collectionMetrics module.CollectionMetrics - protocolViolationNotifier hotstuff.ProposalViolationConsumer + proposalViolationNotifier hotstuff.ProposalViolationConsumer headers storage.Headers state clusterkv.MutableState // track latest finalized view/height - used to efficiently drop outdated or too-far-ahead blocks @@ -61,7 +61,7 @@ func NewCore( mempool module.MempoolMetrics, hotstuffMetrics module.HotstuffMetrics, collectionMetrics module.CollectionMetrics, - protocolViolationNotifier hotstuff.ProposalViolationConsumer, + proposalViolationNotifier hotstuff.ProposalViolationConsumer, headers storage.Headers, state clusterkv.MutableState, pending module.PendingClusterBlockBuffer, @@ -85,7 +85,7 @@ func NewCore( mempoolMetrics: mempool, hotstuffMetrics: hotstuffMetrics, collectionMetrics: collectionMetrics, - protocolViolationNotifier: protocolViolationNotifier, + proposalViolationNotifier: proposalViolationNotifier, headers: headers, state: state, pending: pending, @@ -314,7 +314,7 @@ func (c *Core) processBlockProposal(proposal *cluster.Block) error { err := c.validator.ValidateProposal(hotstuffProposal) if err != nil { if invalidBlockErr, ok := model.AsInvalidBlockError(err); ok { - c.protocolViolationNotifier.OnInvalidBlockDetected(*invalidBlockErr) + c.proposalViolationNotifier.OnInvalidBlockDetected(*invalidBlockErr) return engine.NewInvalidInputErrorf("invalid block proposal: %w", err) } if errors.Is(err, model.ErrViewForUnknownEpoch) { diff --git a/engine/collection/compliance/core_test.go b/engine/collection/compliance/core_test.go index 767e3bdcf96..c9fc0101784 100644 --- a/engine/collection/compliance/core_test.go +++ b/engine/collection/compliance/core_test.go @@ -52,7 +52,7 @@ type CommonSuite struct { state *clusterstate.MutableState snapshot *clusterstate.Snapshot metrics *metrics.NoopCollector - protocolViolationNotifier *hotstuff.ProtocolViolationConsumer + proposalViolationNotifier *hotstuff.ProposalViolationConsumer headers *storage.Headers pending *module.PendingClusterBlockBuffer hotstuff *module.HotStuff @@ -175,7 +175,7 @@ func (cs *CommonSuite) SetupTest() { cs.metrics = metrics.NewNoopCollector() // set up notifier for reporting protocol violations - cs.protocolViolationNotifier = hotstuff.NewProtocolViolationConsumer(cs.T()) + cs.proposalViolationNotifier = hotstuff.NewProposalViolationConsumer(cs.T()) // initialize the engine core, err := NewCore( @@ -184,7 +184,7 @@ func (cs *CommonSuite) SetupTest() { cs.metrics, cs.metrics, cs.metrics, - cs.protocolViolationNotifier, + cs.proposalViolationNotifier, cs.headers, cs.state, cs.pending, @@ -286,7 +286,7 @@ func (cs *CoreSuite) TestOnBlockProposal_FailsHotStuffValidation() { *cs.validator = *hotstuff.NewValidator(cs.T()) sentinelError := model.NewInvalidBlockErrorf(hotstuffProposal, "") cs.validator.On("ValidateProposal", hotstuffProposal).Return(sentinelError) - cs.protocolViolationNotifier.On("OnInvalidBlockDetected", sentinelError).Return().Once() + cs.proposalViolationNotifier.On("OnInvalidBlockDetected", sentinelError).Return().Once() // we should notify VoteAggregator about the invalid block cs.voteAggregator.On("InvalidBlock", hotstuffProposal).Return(nil) diff --git a/engine/common/follower/compliance_core.go b/engine/common/follower/compliance_core.go index 89924f1d129..98cb3b2fad9 100644 --- a/engine/common/follower/compliance_core.go +++ b/engine/common/follower/compliance_core.go @@ -41,7 +41,7 @@ type ComplianceCore struct { log zerolog.Logger mempoolMetrics module.MempoolMetrics tracer module.Tracer - protocolViolationNotifier hotstuff.ProposalViolationConsumer + proposalViolationNotifier hotstuff.ProposalViolationConsumer pendingCache *cache.Cache pendingTree *pending_tree.PendingTree state protocol.FollowerState @@ -79,7 +79,7 @@ func NewComplianceCore(log zerolog.Logger, log: log.With().Str("engine", "follower_core").Logger(), mempoolMetrics: mempoolMetrics, state: state, - protocolViolationNotifier: followerConsumer, + proposalViolationNotifier: followerConsumer, pendingCache: cache.NewCache(log, defaultPendingBlocksCacheCapacity, heroCacheCollector, onEquivocation), pendingTree: pending_tree.NewPendingTree(finalizedBlock), follower: follower, @@ -145,7 +145,7 @@ func (c *ComplianceCore) OnBlockRange(originID flow.Identifier, batch []*flow.Bl if err != nil { if invalidBlockError, ok := model.AsInvalidBlockError(err); ok { // TODO: potential slashing - c.protocolViolationNotifier.OnInvalidBlockDetected(*invalidBlockError) + c.proposalViolationNotifier.OnInvalidBlockDetected(*invalidBlockError) return nil } if errors.Is(err, model.ErrViewForUnknownEpoch) { diff --git a/engine/consensus/compliance/core.go b/engine/consensus/compliance/core.go index 28d6d87561b..8939bb68a5a 100644 --- a/engine/consensus/compliance/core.go +++ b/engine/consensus/compliance/core.go @@ -42,7 +42,7 @@ type Core struct { mempoolMetrics module.MempoolMetrics hotstuffMetrics module.HotstuffMetrics complianceMetrics module.ComplianceMetrics - protocolViolationNotifier hotstuff.ProposalViolationConsumer + proposalViolationNotifier hotstuff.ProposalViolationConsumer tracer module.Tracer headers storage.Headers payloads storage.Payloads @@ -65,7 +65,7 @@ func NewCore( mempool module.MempoolMetrics, hotstuffMetrics module.HotstuffMetrics, complianceMetrics module.ComplianceMetrics, - protocolViolationNotifier hotstuff.ProposalViolationConsumer, + proposalViolationNotifier hotstuff.ProposalViolationConsumer, tracer module.Tracer, headers storage.Headers, payloads storage.Payloads, @@ -92,7 +92,7 @@ func NewCore( mempoolMetrics: mempool, hotstuffMetrics: hotstuffMetrics, complianceMetrics: complianceMetrics, - protocolViolationNotifier: protocolViolationNotifier, + proposalViolationNotifier: proposalViolationNotifier, headers: headers, payloads: payloads, state: state, @@ -324,7 +324,7 @@ func (c *Core) processBlockProposal(proposal *flow.Block) error { err := c.validator.ValidateProposal(hotstuffProposal) if err != nil { if invalidBlockErr, ok := model.AsInvalidBlockError(err); ok { - c.protocolViolationNotifier.OnInvalidBlockDetected(*invalidBlockErr) + c.proposalViolationNotifier.OnInvalidBlockDetected(*invalidBlockErr) return engine.NewInvalidInputErrorf("invalid block proposal: %w", err) } if errors.Is(err, model.ErrViewForUnknownEpoch) { diff --git a/engine/consensus/compliance/core_test.go b/engine/consensus/compliance/core_test.go index 0c951115885..fac752367ec 100644 --- a/engine/consensus/compliance/core_test.go +++ b/engine/consensus/compliance/core_test.go @@ -71,7 +71,7 @@ type CommonSuite struct { pending *module.PendingBlockBuffer hotstuff *module.HotStuff sync *module.BlockRequester - protocolViolationNotifier *hotstuff.ProtocolViolationConsumer + proposalViolationNotifier *hotstuff.ProposalViolationConsumer validator *hotstuff.Validator voteAggregator *hotstuff.VoteAggregator timeoutAggregator *hotstuff.TimeoutAggregator @@ -253,7 +253,7 @@ func (cs *CommonSuite) SetupTest() { cs.tracer = trace.NewNoopTracer() // set up notifier for reporting protocol violations - cs.protocolViolationNotifier = hotstuff.NewProtocolViolationConsumer(cs.T()) + cs.proposalViolationNotifier = hotstuff.NewProposalViolationConsumer(cs.T()) // initialize the engine e, err := NewCore( @@ -262,7 +262,7 @@ func (cs *CommonSuite) SetupTest() { cs.metrics, cs.metrics, cs.metrics, - cs.protocolViolationNotifier, + cs.proposalViolationNotifier, cs.tracer, cs.headers, cs.payloads, @@ -369,7 +369,7 @@ func (cs *CoreSuite) TestOnBlockProposal_FailsHotStuffValidation() { *cs.validator = *hotstuff.NewValidator(cs.T()) sentinelError := model.NewInvalidBlockErrorf(hotstuffProposal, "") cs.validator.On("ValidateProposal", hotstuffProposal).Return(sentinelError) - cs.protocolViolationNotifier.On("OnInvalidBlockDetected", sentinelError).Return().Once() + cs.proposalViolationNotifier.On("OnInvalidBlockDetected", sentinelError).Return().Once() // we should notify VoteAggregator about the invalid block cs.voteAggregator.On("InvalidBlock", hotstuffProposal).Return(nil) From d159fecb70c23f94265031d57826eb396cab720c Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Tue, 9 May 2023 14:28:28 +0300 Subject: [PATCH 29/38] Linted --- .../pubsub/timeout_aggregation_violation_consumer.go | 3 ++- .../notifications/pubsub/timeout_collector_distributor.go | 2 +- .../pubsub/vote_aggregation_violation_consumer.go | 3 ++- .../notifications/pubsub/vote_collector_distributor.go | 2 +- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/consensus/hotstuff/notifications/pubsub/timeout_aggregation_violation_consumer.go b/consensus/hotstuff/notifications/pubsub/timeout_aggregation_violation_consumer.go index db07ac9a82a..8194c4c11b8 100644 --- a/consensus/hotstuff/notifications/pubsub/timeout_aggregation_violation_consumer.go +++ b/consensus/hotstuff/notifications/pubsub/timeout_aggregation_violation_consumer.go @@ -1,9 +1,10 @@ package pubsub import ( + "sync" + "github.com/onflow/flow-go/consensus/hotstuff" "github.com/onflow/flow-go/consensus/hotstuff/model" - "sync" ) // TimeoutAggregationViolationDistributor ingests notifications about HotStuff-protocol violations and diff --git a/consensus/hotstuff/notifications/pubsub/timeout_collector_distributor.go b/consensus/hotstuff/notifications/pubsub/timeout_collector_distributor.go index 82705328f74..bb13e683412 100644 --- a/consensus/hotstuff/notifications/pubsub/timeout_collector_distributor.go +++ b/consensus/hotstuff/notifications/pubsub/timeout_collector_distributor.go @@ -1,10 +1,10 @@ package pubsub import ( - "github.com/onflow/flow-go/consensus/hotstuff/model" "sync" "github.com/onflow/flow-go/consensus/hotstuff" + "github.com/onflow/flow-go/consensus/hotstuff/model" "github.com/onflow/flow-go/model/flow" ) diff --git a/consensus/hotstuff/notifications/pubsub/vote_aggregation_violation_consumer.go b/consensus/hotstuff/notifications/pubsub/vote_aggregation_violation_consumer.go index 33cee62ddf2..9ecc9192dad 100644 --- a/consensus/hotstuff/notifications/pubsub/vote_aggregation_violation_consumer.go +++ b/consensus/hotstuff/notifications/pubsub/vote_aggregation_violation_consumer.go @@ -1,9 +1,10 @@ package pubsub import ( + "sync" + "github.com/onflow/flow-go/consensus/hotstuff" "github.com/onflow/flow-go/consensus/hotstuff/model" - "sync" ) // VoteAggregationViolationDistributor ingests notifications about HotStuff-protocol violations and diff --git a/consensus/hotstuff/notifications/pubsub/vote_collector_distributor.go b/consensus/hotstuff/notifications/pubsub/vote_collector_distributor.go index 5c2f07f0ed7..9a4543e2ffb 100644 --- a/consensus/hotstuff/notifications/pubsub/vote_collector_distributor.go +++ b/consensus/hotstuff/notifications/pubsub/vote_collector_distributor.go @@ -1,10 +1,10 @@ package pubsub import ( - "github.com/onflow/flow-go/consensus/hotstuff/model" "sync" "github.com/onflow/flow-go/consensus/hotstuff" + "github.com/onflow/flow-go/consensus/hotstuff/model" "github.com/onflow/flow-go/model/flow" ) From bc7dc4774e61cdd9ebd00799e6122f6560ddcac4 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Tue, 9 May 2023 14:28:28 +0300 Subject: [PATCH 30/38] Linted. --- .../pubsub/timeout_aggregation_violation_consumer.go | 3 ++- .../notifications/pubsub/timeout_collector_distributor.go | 2 +- .../pubsub/vote_aggregation_violation_consumer.go | 3 ++- .../notifications/pubsub/vote_collector_distributor.go | 2 +- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/consensus/hotstuff/notifications/pubsub/timeout_aggregation_violation_consumer.go b/consensus/hotstuff/notifications/pubsub/timeout_aggregation_violation_consumer.go index db07ac9a82a..8194c4c11b8 100644 --- a/consensus/hotstuff/notifications/pubsub/timeout_aggregation_violation_consumer.go +++ b/consensus/hotstuff/notifications/pubsub/timeout_aggregation_violation_consumer.go @@ -1,9 +1,10 @@ package pubsub import ( + "sync" + "github.com/onflow/flow-go/consensus/hotstuff" "github.com/onflow/flow-go/consensus/hotstuff/model" - "sync" ) // TimeoutAggregationViolationDistributor ingests notifications about HotStuff-protocol violations and diff --git a/consensus/hotstuff/notifications/pubsub/timeout_collector_distributor.go b/consensus/hotstuff/notifications/pubsub/timeout_collector_distributor.go index 82705328f74..bb13e683412 100644 --- a/consensus/hotstuff/notifications/pubsub/timeout_collector_distributor.go +++ b/consensus/hotstuff/notifications/pubsub/timeout_collector_distributor.go @@ -1,10 +1,10 @@ package pubsub import ( - "github.com/onflow/flow-go/consensus/hotstuff/model" "sync" "github.com/onflow/flow-go/consensus/hotstuff" + "github.com/onflow/flow-go/consensus/hotstuff/model" "github.com/onflow/flow-go/model/flow" ) diff --git a/consensus/hotstuff/notifications/pubsub/vote_aggregation_violation_consumer.go b/consensus/hotstuff/notifications/pubsub/vote_aggregation_violation_consumer.go index 33cee62ddf2..9ecc9192dad 100644 --- a/consensus/hotstuff/notifications/pubsub/vote_aggregation_violation_consumer.go +++ b/consensus/hotstuff/notifications/pubsub/vote_aggregation_violation_consumer.go @@ -1,9 +1,10 @@ package pubsub import ( + "sync" + "github.com/onflow/flow-go/consensus/hotstuff" "github.com/onflow/flow-go/consensus/hotstuff/model" - "sync" ) // VoteAggregationViolationDistributor ingests notifications about HotStuff-protocol violations and diff --git a/consensus/hotstuff/notifications/pubsub/vote_collector_distributor.go b/consensus/hotstuff/notifications/pubsub/vote_collector_distributor.go index 5c2f07f0ed7..9a4543e2ffb 100644 --- a/consensus/hotstuff/notifications/pubsub/vote_collector_distributor.go +++ b/consensus/hotstuff/notifications/pubsub/vote_collector_distributor.go @@ -1,10 +1,10 @@ package pubsub import ( - "github.com/onflow/flow-go/consensus/hotstuff/model" "sync" "github.com/onflow/flow-go/consensus/hotstuff" + "github.com/onflow/flow-go/consensus/hotstuff/model" "github.com/onflow/flow-go/model/flow" ) From 12c7aae5630e7a84678faf1243fa024b5a558b9d Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Wed, 10 May 2023 13:15:43 +0300 Subject: [PATCH 31/38] Updated godoc and cleaned up distributors --- consensus/hotstuff/consumer.go | 38 +++++++++++++--- .../pubsub/communicator_distributor.go | 14 +++--- .../notifications/pubsub/distributor.go | 15 ++++--- .../pubsub/finalization_distributor.go | 2 +- .../pubsub/participant_distributor.go | 32 ++++++------- .../pubsub/proposal_violation_distributor.go | 45 +++++++++++++++++++ .../pubsub/protocol_violation_distributor.go | 45 ------------------- .../timeout_aggregation_violation_consumer.go | 15 +++---- .../pubsub/timeout_collector_distributor.go | 7 ++- .../vote_aggregation_violation_consumer.go | 17 ++++--- .../pubsub/vote_collector_distributor.go | 7 ++- 11 files changed, 132 insertions(+), 105 deletions(-) create mode 100644 consensus/hotstuff/notifications/pubsub/proposal_violation_distributor.go delete mode 100644 consensus/hotstuff/notifications/pubsub/protocol_violation_distributor.go diff --git a/consensus/hotstuff/consumer.go b/consensus/hotstuff/consumer.go index 57633000610..a0312734fcb 100644 --- a/consensus/hotstuff/consumer.go +++ b/consensus/hotstuff/consumer.go @@ -8,8 +8,7 @@ import ( ) // ProposalViolationConsumer consumes outbound notifications about HotStuff-protocol violations. -// Such notifications are produced by the active consensus participants and to a lesser -// degree also the consensus follower. +// Such notifications are produced by the active consensus participants and consensus follower. // // Implementations must: // - be concurrency safe @@ -33,6 +32,14 @@ type ProposalViolationConsumer interface { OnDoubleProposeDetected(*model.Block, *model.Block) } +// VoteAggregationViolationConsumer consumes outbound notifications about HotStuff-protocol violations specifically +// invalid votes during processing. +// Such notifications are produced by the Vote Aggregation logic. +// +// Implementations must: +// - be concurrency safe +// - be non-blocking +// - handle repetition of the same events (with some processing overhead). type VoteAggregationViolationConsumer interface { // OnDoubleVotingDetected notifications are produced by the Vote Aggregation logic // whenever a double voting (same voter voting for different blocks at the same view) was detected. @@ -56,6 +63,14 @@ type VoteAggregationViolationConsumer interface { OnVoteForInvalidBlockDetected(vote *model.Vote, invalidProposal *model.Proposal) } +// TimeoutAggregationViolationConsumer consumes outbound notifications about Active Pacemaker violations specifically +// invalid timeouts during processing. +// Such notifications are produced by the Timeout Aggregation logic. +// +// Implementations must: +// - be concurrency safe +// - be non-blocking +// - handle repetition of the same events (with some processing overhead). type TimeoutAggregationViolationConsumer interface { // OnDoubleTimeoutDetected notifications are produced by the Timeout Aggregation logic // whenever a double timeout (same replica producing two different timeouts at the same view) was detected. @@ -202,10 +217,9 @@ type ParticipantConsumer interface { OnCurrentViewDetails(currentView, finalizedView uint64, currentLeader flow.Identifier) } -// VoteCollectorConsumer consumes outbound notifications produced by HotStuff and its components. -// Notifications are consensus-internal state changes which are potentially relevant to -// the larger node in which HotStuff is running. The notifications are emitted -// in the order in which the HotStuff algorithm makes the respective steps. +// VoteCollectorConsumer consumes outbound notifications produced by HotStuff's vote aggregation +// component. These events are primarily intended for the HotStuff-internal state machine (EventHandler), +// but might also be relevant to the larger node in which HotStuff is running. // // Implementations must: // - be concurrency safe @@ -336,11 +350,23 @@ type Consumer interface { ParticipantConsumer } +// VoteAggregationConsumer consumes outbound notifications produced by Vote Aggregation logic. +// It is a subset of the notifications produced by consensus participants. +// Implementations must: +// - be concurrency safe +// - be non-blocking +// - handle repetition of the same events (with some processing overhead). type VoteAggregationConsumer interface { VoteAggregationViolationConsumer VoteCollectorConsumer } +// TimeoutAggregationConsumer consumes outbound notifications produced by Vote Aggregation logic. +// It is a subset of the notifications produced by consensus participants. +// Implementations must: +// - be concurrency safe +// - be non-blocking +// - handle repetition of the same events (with some processing overhead). type TimeoutAggregationConsumer interface { TimeoutAggregationViolationConsumer TimeoutCollectorConsumer diff --git a/consensus/hotstuff/notifications/pubsub/communicator_distributor.go b/consensus/hotstuff/notifications/pubsub/communicator_distributor.go index 094c4b9a440..5e0604fa83c 100644 --- a/consensus/hotstuff/notifications/pubsub/communicator_distributor.go +++ b/consensus/hotstuff/notifications/pubsub/communicator_distributor.go @@ -10,13 +10,13 @@ import ( ) // CommunicatorDistributor ingests outbound consensus messages from HotStuff's core logic and -// distributes them to subscribers. This logic only runs inside active consensus participants proposing +// distributes them to consumers. This logic only runs inside active consensus participants proposing // blocks, voting, collecting + aggregating votes to QCs, and participating in the pacemaker (sending // timeouts, collecting + aggregating timeouts to TCs). // Concurrently safe. type CommunicatorDistributor struct { - subscribers []hotstuff.CommunicatorConsumer - lock sync.RWMutex + consumers []hotstuff.CommunicatorConsumer + lock sync.RWMutex } var _ hotstuff.CommunicatorConsumer = (*CommunicatorDistributor)(nil) @@ -28,13 +28,13 @@ func NewCommunicatorDistributor() *CommunicatorDistributor { func (d *CommunicatorDistributor) AddCommunicatorConsumer(consumer hotstuff.CommunicatorConsumer) { d.lock.Lock() defer d.lock.Unlock() - d.subscribers = append(d.subscribers, consumer) + d.consumers = append(d.consumers, consumer) } func (d *CommunicatorDistributor) OnOwnVote(blockID flow.Identifier, view uint64, sigData []byte, recipientID flow.Identifier) { d.lock.RLock() defer d.lock.RUnlock() - for _, s := range d.subscribers { + for _, s := range d.consumers { s.OnOwnVote(blockID, view, sigData, recipientID) } } @@ -42,7 +42,7 @@ func (d *CommunicatorDistributor) OnOwnVote(blockID flow.Identifier, view uint64 func (d *CommunicatorDistributor) OnOwnTimeout(timeout *model.TimeoutObject) { d.lock.RLock() defer d.lock.RUnlock() - for _, s := range d.subscribers { + for _, s := range d.consumers { s.OnOwnTimeout(timeout) } } @@ -50,7 +50,7 @@ func (d *CommunicatorDistributor) OnOwnTimeout(timeout *model.TimeoutObject) { func (d *CommunicatorDistributor) OnOwnProposal(proposal *flow.Header, targetPublicationTime time.Time) { d.lock.RLock() defer d.lock.RUnlock() - for _, s := range d.subscribers { + for _, s := range d.consumers { s.OnOwnProposal(proposal, targetPublicationTime) } } diff --git a/consensus/hotstuff/notifications/pubsub/distributor.go b/consensus/hotstuff/notifications/pubsub/distributor.go index fddf3c03c23..5db5f602cad 100644 --- a/consensus/hotstuff/notifications/pubsub/distributor.go +++ b/consensus/hotstuff/notifications/pubsub/distributor.go @@ -4,7 +4,7 @@ import ( "github.com/onflow/flow-go/consensus/hotstuff" ) -// Distributor distributes notifications to a list of subscribers (event consumers). +// Distributor distributes notifications to a list of consumers (event consumers). // // It allows thread-safe subscription of multiple consumers to events. type Distributor struct { @@ -30,9 +30,10 @@ func (p *Distributor) AddConsumer(consumer hotstuff.Consumer) { p.ParticipantDistributor.AddParticipantConsumer(consumer) } -// FollowerDistributor ingests consensus follower events and distributes it to subscribers. +// FollowerDistributor ingests consensus follower events and distributes it to consumers. +// It allows thread-safe subscription of multiple consumers to events. type FollowerDistributor struct { - *ProtocolViolationDistributor + *ProposalViolationDistributor *FinalizationDistributor } @@ -40,16 +41,18 @@ var _ hotstuff.FollowerConsumer = (*FollowerDistributor)(nil) func NewFollowerDistributor() *FollowerDistributor { return &FollowerDistributor{ - ProtocolViolationDistributor: NewProtocolViolationDistributor(), + ProposalViolationDistributor: NewProtocolViolationDistributor(), FinalizationDistributor: NewFinalizationDistributor(), } } func (d *FollowerDistributor) AddFollowerConsumer(consumer hotstuff.FollowerConsumer) { d.FinalizationDistributor.AddFinalizationConsumer(consumer) - d.ProtocolViolationDistributor.AddProposalViolationConsumer(consumer) + d.ProposalViolationDistributor.AddProposalViolationConsumer(consumer) } +// TimeoutAggregationDistributor ingests timeout aggregation events and distributes it to consumers. +// It allows thread-safe subscription of multiple consumers to events. type TimeoutAggregationDistributor struct { *TimeoutAggregationViolationDistributor *TimeoutCollectorDistributor @@ -69,6 +72,8 @@ func (d *TimeoutAggregationDistributor) AddTimeoutAggregationConsumer(consumer h d.TimeoutCollectorDistributor.AddTimeoutCollectorConsumer(consumer) } +// VoteAggregationDistributor ingests vote aggregation events and distributes it to consumers. +// It allows thread-safe subscription of multiple consumers to events. type VoteAggregationDistributor struct { *VoteAggregationViolationDistributor *VoteCollectorDistributor diff --git a/consensus/hotstuff/notifications/pubsub/finalization_distributor.go b/consensus/hotstuff/notifications/pubsub/finalization_distributor.go index a78dd88d13b..e351575c122 100644 --- a/consensus/hotstuff/notifications/pubsub/finalization_distributor.go +++ b/consensus/hotstuff/notifications/pubsub/finalization_distributor.go @@ -11,7 +11,7 @@ type OnBlockFinalizedConsumer = func(block *model.Block) type OnBlockIncorporatedConsumer = func(block *model.Block) // FinalizationDistributor ingests events from HotStuff's logic for tracking forks + finalization -// and distributes them to subscribers. This logic generally runs inside all nodes (irrespectively whether +// and distributes them to consumers. This logic generally runs inside all nodes (irrespectively whether // they are active consensus participants or or only consensus followers). // Concurrently safe. type FinalizationDistributor struct { diff --git a/consensus/hotstuff/notifications/pubsub/participant_distributor.go b/consensus/hotstuff/notifications/pubsub/participant_distributor.go index 46149da7f32..f5047cd7a53 100644 --- a/consensus/hotstuff/notifications/pubsub/participant_distributor.go +++ b/consensus/hotstuff/notifications/pubsub/participant_distributor.go @@ -9,13 +9,13 @@ import ( ) // ParticipantDistributor ingests events from HotStuff's core logic and distributes them to -// subscribers. This logic only runs inside active consensus participants proposing blocks, voting, +// consumers. This logic only runs inside active consensus participants proposing blocks, voting, // collecting + aggregating votes to QCs, and participating in the pacemaker (sending timeouts, // collecting + aggregating timeouts to TCs). // Concurrently safe. type ParticipantDistributor struct { - subscribers []hotstuff.ParticipantConsumer - lock sync.RWMutex + consumers []hotstuff.ParticipantConsumer + lock sync.RWMutex } var _ hotstuff.ParticipantConsumer = (*ParticipantDistributor)(nil) @@ -27,13 +27,13 @@ func NewParticipantDistributor() *ParticipantDistributor { func (d *ParticipantDistributor) AddParticipantConsumer(consumer hotstuff.ParticipantConsumer) { d.lock.Lock() defer d.lock.Unlock() - d.subscribers = append(d.subscribers, consumer) + d.consumers = append(d.consumers, consumer) } func (d *ParticipantDistributor) OnEventProcessed() { d.lock.RLock() defer d.lock.RUnlock() - for _, subscriber := range d.subscribers { + for _, subscriber := range d.consumers { subscriber.OnEventProcessed() } } @@ -41,7 +41,7 @@ func (d *ParticipantDistributor) OnEventProcessed() { func (d *ParticipantDistributor) OnStart(currentView uint64) { d.lock.RLock() defer d.lock.RUnlock() - for _, subscriber := range d.subscribers { + for _, subscriber := range d.consumers { subscriber.OnStart(currentView) } } @@ -49,7 +49,7 @@ func (d *ParticipantDistributor) OnStart(currentView uint64) { func (d *ParticipantDistributor) OnReceiveProposal(currentView uint64, proposal *model.Proposal) { d.lock.RLock() defer d.lock.RUnlock() - for _, subscriber := range d.subscribers { + for _, subscriber := range d.consumers { subscriber.OnReceiveProposal(currentView, proposal) } } @@ -57,7 +57,7 @@ func (d *ParticipantDistributor) OnReceiveProposal(currentView uint64, proposal func (d *ParticipantDistributor) OnReceiveQc(currentView uint64, qc *flow.QuorumCertificate) { d.lock.RLock() defer d.lock.RUnlock() - for _, subscriber := range d.subscribers { + for _, subscriber := range d.consumers { subscriber.OnReceiveQc(currentView, qc) } } @@ -65,7 +65,7 @@ func (d *ParticipantDistributor) OnReceiveQc(currentView uint64, qc *flow.Quorum func (d *ParticipantDistributor) OnReceiveTc(currentView uint64, tc *flow.TimeoutCertificate) { d.lock.RLock() defer d.lock.RUnlock() - for _, subscriber := range d.subscribers { + for _, subscriber := range d.consumers { subscriber.OnReceiveTc(currentView, tc) } } @@ -73,7 +73,7 @@ func (d *ParticipantDistributor) OnReceiveTc(currentView uint64, tc *flow.Timeou func (d *ParticipantDistributor) OnPartialTc(currentView uint64, partialTc *hotstuff.PartialTcCreated) { d.lock.RLock() defer d.lock.RUnlock() - for _, subscriber := range d.subscribers { + for _, subscriber := range d.consumers { subscriber.OnPartialTc(currentView, partialTc) } } @@ -81,7 +81,7 @@ func (d *ParticipantDistributor) OnPartialTc(currentView uint64, partialTc *hots func (d *ParticipantDistributor) OnLocalTimeout(currentView uint64) { d.lock.RLock() defer d.lock.RUnlock() - for _, subscriber := range d.subscribers { + for _, subscriber := range d.consumers { subscriber.OnLocalTimeout(currentView) } } @@ -89,7 +89,7 @@ func (d *ParticipantDistributor) OnLocalTimeout(currentView uint64) { func (d *ParticipantDistributor) OnViewChange(oldView, newView uint64) { d.lock.RLock() defer d.lock.RUnlock() - for _, subscriber := range d.subscribers { + for _, subscriber := range d.consumers { subscriber.OnViewChange(oldView, newView) } } @@ -97,7 +97,7 @@ func (d *ParticipantDistributor) OnViewChange(oldView, newView uint64) { func (d *ParticipantDistributor) OnQcTriggeredViewChange(oldView uint64, newView uint64, qc *flow.QuorumCertificate) { d.lock.RLock() defer d.lock.RUnlock() - for _, subscriber := range d.subscribers { + for _, subscriber := range d.consumers { subscriber.OnQcTriggeredViewChange(oldView, newView, qc) } } @@ -105,7 +105,7 @@ func (d *ParticipantDistributor) OnQcTriggeredViewChange(oldView uint64, newView func (d *ParticipantDistributor) OnTcTriggeredViewChange(oldView uint64, newView uint64, tc *flow.TimeoutCertificate) { d.lock.RLock() defer d.lock.RUnlock() - for _, subscriber := range d.subscribers { + for _, subscriber := range d.consumers { subscriber.OnTcTriggeredViewChange(oldView, newView, tc) } } @@ -113,7 +113,7 @@ func (d *ParticipantDistributor) OnTcTriggeredViewChange(oldView uint64, newView func (d *ParticipantDistributor) OnStartingTimeout(timerInfo model.TimerInfo) { d.lock.RLock() defer d.lock.RUnlock() - for _, subscriber := range d.subscribers { + for _, subscriber := range d.consumers { subscriber.OnStartingTimeout(timerInfo) } } @@ -121,7 +121,7 @@ func (d *ParticipantDistributor) OnStartingTimeout(timerInfo model.TimerInfo) { func (d *ParticipantDistributor) OnCurrentViewDetails(currentView, finalizedView uint64, currentLeader flow.Identifier) { d.lock.RLock() defer d.lock.RUnlock() - for _, subscriber := range d.subscribers { + for _, subscriber := range d.consumers { subscriber.OnCurrentViewDetails(currentView, finalizedView, currentLeader) } } diff --git a/consensus/hotstuff/notifications/pubsub/proposal_violation_distributor.go b/consensus/hotstuff/notifications/pubsub/proposal_violation_distributor.go new file mode 100644 index 00000000000..e86614d9531 --- /dev/null +++ b/consensus/hotstuff/notifications/pubsub/proposal_violation_distributor.go @@ -0,0 +1,45 @@ +package pubsub + +import ( + "sync" + + "github.com/onflow/flow-go/consensus/hotstuff" + "github.com/onflow/flow-go/consensus/hotstuff/model" +) + +// ProposalViolationDistributor ingests notifications about HotStuff-protocol violations and +// distributes them to consumers. Such notifications are produced by the active consensus +// participants and the consensus follower. +// Concurrently safe. +type ProposalViolationDistributor struct { + consumers []hotstuff.ProposalViolationConsumer + lock sync.RWMutex +} + +var _ hotstuff.ProposalViolationConsumer = (*ProposalViolationDistributor)(nil) + +func NewProtocolViolationDistributor() *ProposalViolationDistributor { + return &ProposalViolationDistributor{} +} + +func (d *ProposalViolationDistributor) AddProposalViolationConsumer(consumer hotstuff.ProposalViolationConsumer) { + d.lock.Lock() + defer d.lock.Unlock() + d.consumers = append(d.consumers, consumer) +} + +func (d *ProposalViolationDistributor) OnInvalidBlockDetected(err model.InvalidBlockError) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, subscriber := range d.consumers { + subscriber.OnInvalidBlockDetected(err) + } +} + +func (d *ProposalViolationDistributor) OnDoubleProposeDetected(block1, block2 *model.Block) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, subscriber := range d.consumers { + subscriber.OnDoubleProposeDetected(block1, block2) + } +} diff --git a/consensus/hotstuff/notifications/pubsub/protocol_violation_distributor.go b/consensus/hotstuff/notifications/pubsub/protocol_violation_distributor.go deleted file mode 100644 index 054e1d6bdce..00000000000 --- a/consensus/hotstuff/notifications/pubsub/protocol_violation_distributor.go +++ /dev/null @@ -1,45 +0,0 @@ -package pubsub - -import ( - "sync" - - "github.com/onflow/flow-go/consensus/hotstuff" - "github.com/onflow/flow-go/consensus/hotstuff/model" -) - -// ProtocolViolationDistributor ingests notifications about HotStuff-protocol violations and -// distributes them to subscribers. Such notifications are produced by the active consensus -// participants and to a lesser degree also the consensus follower. -// Concurrently safe. -type ProtocolViolationDistributor struct { - subscribers []hotstuff.ProposalViolationConsumer - lock sync.RWMutex -} - -var _ hotstuff.ProposalViolationConsumer = (*ProtocolViolationDistributor)(nil) - -func NewProtocolViolationDistributor() *ProtocolViolationDistributor { - return &ProtocolViolationDistributor{} -} - -func (d *ProtocolViolationDistributor) AddProposalViolationConsumer(consumer hotstuff.ProposalViolationConsumer) { - d.lock.Lock() - defer d.lock.Unlock() - d.subscribers = append(d.subscribers, consumer) -} - -func (d *ProtocolViolationDistributor) OnInvalidBlockDetected(err model.InvalidBlockError) { - d.lock.RLock() - defer d.lock.RUnlock() - for _, subscriber := range d.subscribers { - subscriber.OnInvalidBlockDetected(err) - } -} - -func (d *ProtocolViolationDistributor) OnDoubleProposeDetected(block1, block2 *model.Block) { - d.lock.RLock() - defer d.lock.RUnlock() - for _, subscriber := range d.subscribers { - subscriber.OnDoubleProposeDetected(block1, block2) - } -} diff --git a/consensus/hotstuff/notifications/pubsub/timeout_aggregation_violation_consumer.go b/consensus/hotstuff/notifications/pubsub/timeout_aggregation_violation_consumer.go index 8194c4c11b8..25458088f87 100644 --- a/consensus/hotstuff/notifications/pubsub/timeout_aggregation_violation_consumer.go +++ b/consensus/hotstuff/notifications/pubsub/timeout_aggregation_violation_consumer.go @@ -7,13 +7,12 @@ import ( "github.com/onflow/flow-go/consensus/hotstuff/model" ) -// TimeoutAggregationViolationDistributor ingests notifications about HotStuff-protocol violations and -// distributes them to subscribers. Such notifications are produced by the active consensus -// participants and to a lesser degree also the consensus follower. +// TimeoutAggregationViolationDistributor ingests notifications about timeout aggregation violations and +// distributes them to consumers. Such notifications are produced by the timeout aggregation logic. // Concurrently safe. type TimeoutAggregationViolationDistributor struct { - subscribers []hotstuff.TimeoutAggregationViolationConsumer - lock sync.RWMutex + consumers []hotstuff.TimeoutAggregationViolationConsumer + lock sync.RWMutex } var _ hotstuff.TimeoutAggregationViolationConsumer = (*TimeoutAggregationViolationDistributor)(nil) @@ -25,13 +24,13 @@ func NewTimeoutAggregationViolationDistributor() *TimeoutAggregationViolationDis func (d *TimeoutAggregationViolationDistributor) AddTimeoutAggregationViolationConsumer(consumer hotstuff.TimeoutAggregationViolationConsumer) { d.lock.Lock() defer d.lock.Unlock() - d.subscribers = append(d.subscribers, consumer) + d.consumers = append(d.consumers, consumer) } func (d *TimeoutAggregationViolationDistributor) OnDoubleTimeoutDetected(timeout *model.TimeoutObject, altTimeout *model.TimeoutObject) { d.lock.RLock() defer d.lock.RUnlock() - for _, subscriber := range d.subscribers { + for _, subscriber := range d.consumers { subscriber.OnDoubleTimeoutDetected(timeout, altTimeout) } } @@ -39,7 +38,7 @@ func (d *TimeoutAggregationViolationDistributor) OnDoubleTimeoutDetected(timeout func (d *TimeoutAggregationViolationDistributor) OnInvalidTimeoutDetected(err model.InvalidTimeoutError) { d.lock.RLock() defer d.lock.RUnlock() - for _, subscriber := range d.subscribers { + for _, subscriber := range d.consumers { subscriber.OnInvalidTimeoutDetected(err) } } diff --git a/consensus/hotstuff/notifications/pubsub/timeout_collector_distributor.go b/consensus/hotstuff/notifications/pubsub/timeout_collector_distributor.go index bb13e683412..b2bfd6b235e 100644 --- a/consensus/hotstuff/notifications/pubsub/timeout_collector_distributor.go +++ b/consensus/hotstuff/notifications/pubsub/timeout_collector_distributor.go @@ -8,10 +8,9 @@ import ( "github.com/onflow/flow-go/model/flow" ) -// TimeoutCollectorDistributor ingests events from hotstuff's VoteCollector -// logic and distributes them to subscribers. Concurrently safe. -// TODO: investigate if this can be updated using atomics to prevent locking on mutex since we always add all consumers -// before delivering events. +// TimeoutCollectorDistributor ingests notifications about timeout aggregation and +// distributes them to consumers. Such notifications are produced by the timeout aggregation logic. +// Concurrently safe. type TimeoutCollectorDistributor struct { lock sync.RWMutex consumers []hotstuff.TimeoutCollectorConsumer diff --git a/consensus/hotstuff/notifications/pubsub/vote_aggregation_violation_consumer.go b/consensus/hotstuff/notifications/pubsub/vote_aggregation_violation_consumer.go index 9ecc9192dad..d9d1e9baa26 100644 --- a/consensus/hotstuff/notifications/pubsub/vote_aggregation_violation_consumer.go +++ b/consensus/hotstuff/notifications/pubsub/vote_aggregation_violation_consumer.go @@ -7,13 +7,12 @@ import ( "github.com/onflow/flow-go/consensus/hotstuff/model" ) -// VoteAggregationViolationDistributor ingests notifications about HotStuff-protocol violations and -// distributes them to subscribers. Such notifications are produced by the active consensus -// participants and to a lesser degree also the consensus follower. +// VoteAggregationViolationDistributor ingests notifications about vote aggregation violations and +// distributes them to consumers. Such notifications are produced by the vote aggregation logic. // Concurrently safe. type VoteAggregationViolationDistributor struct { - subscribers []hotstuff.VoteAggregationViolationConsumer - lock sync.RWMutex + consumers []hotstuff.VoteAggregationViolationConsumer + lock sync.RWMutex } var _ hotstuff.VoteAggregationViolationConsumer = (*VoteAggregationViolationDistributor)(nil) @@ -25,13 +24,13 @@ func NewVoteAggregationViolationDistributor() *VoteAggregationViolationDistribut func (d *VoteAggregationViolationDistributor) AddVoteAggregationViolationConsumer(consumer hotstuff.VoteAggregationViolationConsumer) { d.lock.Lock() defer d.lock.Unlock() - d.subscribers = append(d.subscribers, consumer) + d.consumers = append(d.consumers, consumer) } func (d *VoteAggregationViolationDistributor) OnDoubleVotingDetected(vote1, vote2 *model.Vote) { d.lock.RLock() defer d.lock.RUnlock() - for _, subscriber := range d.subscribers { + for _, subscriber := range d.consumers { subscriber.OnDoubleVotingDetected(vote1, vote2) } } @@ -39,7 +38,7 @@ func (d *VoteAggregationViolationDistributor) OnDoubleVotingDetected(vote1, vote func (d *VoteAggregationViolationDistributor) OnInvalidVoteDetected(err model.InvalidVoteError) { d.lock.RLock() defer d.lock.RUnlock() - for _, subscriber := range d.subscribers { + for _, subscriber := range d.consumers { subscriber.OnInvalidVoteDetected(err) } } @@ -47,7 +46,7 @@ func (d *VoteAggregationViolationDistributor) OnInvalidVoteDetected(err model.In func (d *VoteAggregationViolationDistributor) OnVoteForInvalidBlockDetected(vote *model.Vote, invalidProposal *model.Proposal) { d.lock.RLock() defer d.lock.RUnlock() - for _, subscriber := range d.subscribers { + for _, subscriber := range d.consumers { subscriber.OnVoteForInvalidBlockDetected(vote, invalidProposal) } } diff --git a/consensus/hotstuff/notifications/pubsub/vote_collector_distributor.go b/consensus/hotstuff/notifications/pubsub/vote_collector_distributor.go index 9a4543e2ffb..c96631aed78 100644 --- a/consensus/hotstuff/notifications/pubsub/vote_collector_distributor.go +++ b/consensus/hotstuff/notifications/pubsub/vote_collector_distributor.go @@ -8,10 +8,9 @@ import ( "github.com/onflow/flow-go/model/flow" ) -// VoteCollectorDistributor ingests events about QC creation from hotstuff and distributes them to subscribers. -// Objects are concurrency safe. -// NOTE: it can be refactored to work without lock since usually we never subscribe after startup. Mostly -// list of observers is static. +// VoteCollectorDistributor ingests notifications about vote aggregation and +// distributes them to consumers. Such notifications are produced by the vote aggregation logic. +// Concurrently safe. type VoteCollectorDistributor struct { consumers []hotstuff.VoteCollectorConsumer lock sync.RWMutex From 160ce7b5f5f5901e4e9ce9a8421f6ffa98b56f0e Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Wed, 10 May 2023 14:36:29 +0300 Subject: [PATCH 32/38] Updated usage of HotstuffModules --- cmd/consensus/main.go | 1 - consensus/config.go | 1 - engine/collection/epochmgr/factories/epoch.go | 4 ++-- engine/collection/epochmgr/factories/hotstuff.go | 1 - 4 files changed, 2 insertions(+), 5 deletions(-) diff --git a/cmd/consensus/main.go b/cmd/consensus/main.go index d729d5999ce..bc8adf78804 100644 --- a/cmd/consensus/main.go +++ b/cmd/consensus/main.go @@ -639,7 +639,6 @@ func main() { Signer: signer, Persist: persist, VoteCollectorDistributor: voteAggregationDistributor.VoteCollectorDistributor, - FollowerDistributor: followerDistributor, TimeoutCollectorDistributor: timeoutAggregationDistributor.TimeoutCollectorDistributor, Forks: forks, Validator: validator, diff --git a/consensus/config.go b/consensus/config.go index 30e22a7fa3d..6c6716b142d 100644 --- a/consensus/config.go +++ b/consensus/config.go @@ -16,7 +16,6 @@ type HotstuffModules struct { Signer hotstuff.Signer // signer of proposal & votes Persist hotstuff.Persister // last state of consensus participant Notifier *pubsub.Distributor // observer for hotstuff events - FollowerDistributor *pubsub.FollowerDistributor // observer for finalization events, used by compliance engine VoteCollectorDistributor *pubsub.VoteCollectorDistributor // observer for vote aggregation events, used by leader TimeoutCollectorDistributor *pubsub.TimeoutCollectorDistributor // observer for timeout aggregation events Forks hotstuff.Forks // information about multiple forks diff --git a/engine/collection/epochmgr/factories/epoch.go b/engine/collection/epochmgr/factories/epoch.go index 4ea2757b592..231bc0c3cc8 100644 --- a/engine/collection/epochmgr/factories/epoch.go +++ b/engine/collection/epochmgr/factories/epoch.go @@ -161,7 +161,7 @@ func (factory *EpochComponentsFactory) Create( complianceEng, err := factory.compliance.Create( metrics, - hotstuffModules.FollowerDistributor, + hotstuffModules.Notifier, mutableState, headers, payloads, @@ -176,7 +176,7 @@ func (factory *EpochComponentsFactory) Create( return } compliance = complianceEng - hotstuffModules.FollowerDistributor.AddOnBlockFinalizedConsumer(complianceEng.OnFinalizedBlock) + hotstuffModules.Notifier.AddOnBlockFinalizedConsumer(complianceEng.OnFinalizedBlock) sync, err = factory.sync.Create(cluster.Members(), state, blocks, syncCore, complianceEng) if err != nil { diff --git a/engine/collection/epochmgr/factories/hotstuff.go b/engine/collection/epochmgr/factories/hotstuff.go index fada8e24194..c6d521bc851 100644 --- a/engine/collection/epochmgr/factories/hotstuff.go +++ b/engine/collection/epochmgr/factories/hotstuff.go @@ -167,7 +167,6 @@ func (f *HotStuffFactory) CreateModules( TimeoutAggregator: timeoutAggregator, VoteCollectorDistributor: voteAggregationDistributor.VoteCollectorDistributor, TimeoutCollectorDistributor: timeoutCollectorDistributor.TimeoutCollectorDistributor, - FollowerDistributor: notifier.FollowerDistributor, }, metrics, nil } From 3d255efc0bea065401eb12be525931f4793cef23 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Wed, 10 May 2023 14:50:29 +0300 Subject: [PATCH 33/38] Renamed ProtocolViolationConsumer to ProposalViolationConsumer --- .../hotstuff/integration/instance_test.go | 2 +- .../hotstuff/notifications/noop_consumer.go | 20 +++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/consensus/hotstuff/integration/instance_test.go b/consensus/hotstuff/integration/instance_test.go index ce742143567..0f971248c70 100644 --- a/consensus/hotstuff/integration/instance_test.go +++ b/consensus/hotstuff/integration/instance_test.go @@ -84,7 +84,7 @@ type Instance struct { } type MockedCommunicatorConsumer struct { - notifications.NoopProtocolViolationConsumer + notifications.NoopProposalViolationConsumer notifications.NoopPartialConsumer notifications.NoopFinalizationConsumer *mocks.CommunicatorConsumer diff --git a/consensus/hotstuff/notifications/noop_consumer.go b/consensus/hotstuff/notifications/noop_consumer.go index ab3124bca0e..b32509676cf 100644 --- a/consensus/hotstuff/notifications/noop_consumer.go +++ b/consensus/hotstuff/notifications/noop_consumer.go @@ -11,7 +11,7 @@ import ( // NoopConsumer is an implementation of the notifications consumer that // doesn't do anything. type NoopConsumer struct { - NoopProtocolViolationConsumer + NoopProposalViolationConsumer NoopFinalizationConsumer NoopPartialConsumer NoopCommunicatorConsumer @@ -103,21 +103,21 @@ func (*NoopVoteCollectorConsumer) OnVoteProcessed(*model.Vote) {} // no-op implementation of hotstuff.ProposalViolationConsumer -type NoopProtocolViolationConsumer struct{} +type NoopProposalViolationConsumer struct{} -var _ hotstuff.ProposalViolationConsumer = (*NoopProtocolViolationConsumer)(nil) +var _ hotstuff.ProposalViolationConsumer = (*NoopProposalViolationConsumer)(nil) -func (*NoopProtocolViolationConsumer) OnInvalidBlockDetected(model.InvalidBlockError) {} +func (*NoopProposalViolationConsumer) OnInvalidBlockDetected(model.InvalidBlockError) {} -func (*NoopProtocolViolationConsumer) OnDoubleProposeDetected(*model.Block, *model.Block) {} +func (*NoopProposalViolationConsumer) OnDoubleProposeDetected(*model.Block, *model.Block) {} -func (*NoopProtocolViolationConsumer) OnDoubleVotingDetected(*model.Vote, *model.Vote) {} +func (*NoopProposalViolationConsumer) OnDoubleVotingDetected(*model.Vote, *model.Vote) {} -func (*NoopProtocolViolationConsumer) OnInvalidVoteDetected(model.InvalidVoteError) {} +func (*NoopProposalViolationConsumer) OnInvalidVoteDetected(model.InvalidVoteError) {} -func (*NoopProtocolViolationConsumer) OnVoteForInvalidBlockDetected(*model.Vote, *model.Proposal) {} +func (*NoopProposalViolationConsumer) OnVoteForInvalidBlockDetected(*model.Vote, *model.Proposal) {} -func (*NoopProtocolViolationConsumer) OnDoubleTimeoutDetected(*model.TimeoutObject, *model.TimeoutObject) { +func (*NoopProposalViolationConsumer) OnDoubleTimeoutDetected(*model.TimeoutObject, *model.TimeoutObject) { } -func (*NoopProtocolViolationConsumer) OnInvalidTimeoutDetected(model.InvalidTimeoutError) {} +func (*NoopProposalViolationConsumer) OnInvalidTimeoutDetected(model.InvalidTimeoutError) {} From 2afe79e0d7360acffc30d4bfb1301cbba7a86e44 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Thu, 11 May 2023 12:11:03 +0300 Subject: [PATCH 34/38] Apply suggestions from PR review --- consensus/hotstuff/notifications/log_consumer.go | 8 +++++++- consensus/hotstuff/notifications/pubsub/distributor.go | 1 + engine/common/follower/compliance_core.go | 1 - 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/consensus/hotstuff/notifications/log_consumer.go b/consensus/hotstuff/notifications/log_consumer.go index eba4d97cacf..c16c85ec824 100644 --- a/consensus/hotstuff/notifications/log_consumer.go +++ b/consensus/hotstuff/notifications/log_consumer.go @@ -179,6 +179,7 @@ func (lc *LogConsumer) OnCurrentViewDetails(currentView, finalizedView uint64, c func (lc *LogConsumer) OnDoubleVotingDetected(vote *model.Vote, alt *model.Vote) { lc.log.Warn(). + Str(logging.KeySuspicious, "true"). Uint64("vote_view", vote.View). Hex("voted_block_id", vote.BlockID[:]). Hex("alt_id", alt.BlockID[:]). @@ -188,6 +189,7 @@ func (lc *LogConsumer) OnDoubleVotingDetected(vote *model.Vote, alt *model.Vote) func (lc *LogConsumer) OnInvalidVoteDetected(err model.InvalidVoteError) { lc.log.Warn(). + Str(logging.KeySuspicious, "true"). Uint64("vote_view", err.Vote.View). Hex("voted_block_id", err.Vote.BlockID[:]). Hex("voter_id", err.Vote.SignerID[:]). @@ -196,6 +198,7 @@ func (lc *LogConsumer) OnInvalidVoteDetected(err model.InvalidVoteError) { func (lc *LogConsumer) OnVoteForInvalidBlockDetected(vote *model.Vote, proposal *model.Proposal) { lc.log.Warn(). + Str(logging.KeySuspicious, "true"). Uint64("vote_view", vote.View). Hex("voted_block_id", vote.BlockID[:]). Hex("voter_id", vote.SignerID[:]). @@ -205,6 +208,7 @@ func (lc *LogConsumer) OnVoteForInvalidBlockDetected(vote *model.Vote, proposal func (lc *LogConsumer) OnDoubleTimeoutDetected(timeout *model.TimeoutObject, alt *model.TimeoutObject) { lc.log.Warn(). + Str(logging.KeySuspicious, "true"). Uint64("timeout_view", timeout.View). Hex("signer_id", logging.ID(timeout.SignerID)). Hex("timeout_id", logging.ID(timeout.ID())). @@ -214,7 +218,9 @@ func (lc *LogConsumer) OnDoubleTimeoutDetected(timeout *model.TimeoutObject, alt func (lc *LogConsumer) OnInvalidTimeoutDetected(err model.InvalidTimeoutError) { log := err.Timeout.LogContext(lc.log).Logger() - log.Warn().Msgf("invalid timeout detected: %s", err.Error()) + log.Warn(). + Str(logging.KeySuspicious, "true"). + Msgf("invalid timeout detected: %s", err.Error()) } func (lc *LogConsumer) logBasicBlockData(loggerEvent *zerolog.Event, block *model.Block) *zerolog.Event { diff --git a/consensus/hotstuff/notifications/pubsub/distributor.go b/consensus/hotstuff/notifications/pubsub/distributor.go index 5db5f602cad..ea461a23742 100644 --- a/consensus/hotstuff/notifications/pubsub/distributor.go +++ b/consensus/hotstuff/notifications/pubsub/distributor.go @@ -46,6 +46,7 @@ func NewFollowerDistributor() *FollowerDistributor { } } +// AddFollowerConsumer registers the input `consumer` to be notified on `hotstuff.ConsensusFollowerConsumer` events. func (d *FollowerDistributor) AddFollowerConsumer(consumer hotstuff.FollowerConsumer) { d.FinalizationDistributor.AddFinalizationConsumer(consumer) d.ProposalViolationDistributor.AddProposalViolationConsumer(consumer) diff --git a/engine/common/follower/compliance_core.go b/engine/common/follower/compliance_core.go index 98cb3b2fad9..c7e569d1e81 100644 --- a/engine/common/follower/compliance_core.go +++ b/engine/common/follower/compliance_core.go @@ -144,7 +144,6 @@ func (c *ComplianceCore) OnBlockRange(originID flow.Identifier, batch []*flow.Bl err := c.validator.ValidateProposal(hotstuffProposal) if err != nil { if invalidBlockError, ok := model.AsInvalidBlockError(err); ok { - // TODO: potential slashing c.proposalViolationNotifier.OnInvalidBlockDetected(*invalidBlockError) return nil } From 4626ec625c62710e249abce775d58657030d3a40 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Thu, 11 May 2023 12:19:27 +0300 Subject: [PATCH 35/38] Introduced InvalidBlockError, added usage in forks. Slight refactoring of InvalidProposalError --- consensus/hotstuff/consumer.go | 2 +- consensus/hotstuff/forks/forks2.go | 6 +-- consensus/hotstuff/mocks/consumer.go | 2 +- consensus/hotstuff/mocks/follower_consumer.go | 2 +- .../mocks/proposal_violation_consumer.go | 2 +- consensus/hotstuff/model/errors.go | 49 ++++++++++++++++-- .../hotstuff/notifications/log_consumer.go | 4 +- .../hotstuff/notifications/noop_consumer.go | 2 +- .../pubsub/proposal_violation_distributor.go | 2 +- .../slashing_violation_consumer.go | 4 +- consensus/hotstuff/validator.go | 2 +- consensus/hotstuff/validator/validator.go | 18 +++---- .../hotstuff/validator/validator_test.go | 50 +++++++++---------- consensus/hotstuff/vote_collector.go | 4 +- .../voteaggregator/vote_aggregator.go | 2 +- .../voteaggregator/vote_aggregator_test.go | 4 +- consensus/hotstuff/votecollector/factory.go | 6 +-- .../hotstuff/votecollector/factory_test.go | 6 +-- engine/collection/compliance/core.go | 2 +- engine/collection/compliance/core_test.go | 6 +-- engine/common/follower/compliance_core.go | 4 +- .../common/follower/compliance_core_test.go | 2 +- engine/consensus/compliance/core.go | 2 +- engine/consensus/compliance/core_test.go | 6 +-- 24 files changed, 112 insertions(+), 77 deletions(-) diff --git a/consensus/hotstuff/consumer.go b/consensus/hotstuff/consumer.go index a0312734fcb..0b76027b146 100644 --- a/consensus/hotstuff/consumer.go +++ b/consensus/hotstuff/consumer.go @@ -21,7 +21,7 @@ type ProposalViolationConsumer interface { // Prerequisites: // Implementation must be concurrency safe; Non-blocking; // and must handle repetition of the same events (with some processing overhead). - OnInvalidBlockDetected(err model.InvalidBlockError) + OnInvalidBlockDetected(err model.InvalidProposalError) // OnDoubleProposeDetected notifications are produced by the Finalization Logic // whenever a double block proposal (equivocation) was detected. diff --git a/consensus/hotstuff/forks/forks2.go b/consensus/hotstuff/forks/forks2.go index e4ebc59112b..1544855f089 100644 --- a/consensus/hotstuff/forks/forks2.go +++ b/consensus/hotstuff/forks/forks2.go @@ -155,11 +155,7 @@ func (f *Forks) EnsureBlockIsValidExtension(block *model.Block) error { err := f.forest.VerifyVertex(blockContainer) if err != nil { if forest.IsInvalidVertexError(err) { - return model.NewInvalidBlockErrorf(&model.Proposal{ - Block: block, - SigData: nil, - LastViewTC: nil, - }, "not a valid vertex for block tree: %w", err) + return model.NewInvalidBlockErrorf(block, "not a valid vertex for block tree: %w", err) } return fmt.Errorf("block tree generated unexpected error validating vertex: %w", err) } diff --git a/consensus/hotstuff/mocks/consumer.go b/consensus/hotstuff/mocks/consumer.go index aca84864891..23776596a43 100644 --- a/consensus/hotstuff/mocks/consumer.go +++ b/consensus/hotstuff/mocks/consumer.go @@ -44,7 +44,7 @@ func (_m *Consumer) OnFinalizedBlock(_a0 *model.Block) { } // OnInvalidBlockDetected provides a mock function with given fields: err -func (_m *Consumer) OnInvalidBlockDetected(err model.InvalidBlockError) { +func (_m *Consumer) OnInvalidBlockDetected(err model.InvalidProposalError) { _m.Called(err) } diff --git a/consensus/hotstuff/mocks/follower_consumer.go b/consensus/hotstuff/mocks/follower_consumer.go index f94f43bf534..225459ffe15 100644 --- a/consensus/hotstuff/mocks/follower_consumer.go +++ b/consensus/hotstuff/mocks/follower_consumer.go @@ -28,7 +28,7 @@ func (_m *FollowerConsumer) OnFinalizedBlock(_a0 *model.Block) { } // OnInvalidBlockDetected provides a mock function with given fields: err -func (_m *FollowerConsumer) OnInvalidBlockDetected(err model.InvalidBlockError) { +func (_m *FollowerConsumer) OnInvalidBlockDetected(err model.InvalidProposalError) { _m.Called(err) } diff --git a/consensus/hotstuff/mocks/proposal_violation_consumer.go b/consensus/hotstuff/mocks/proposal_violation_consumer.go index 0a68f913038..d775b3e923d 100644 --- a/consensus/hotstuff/mocks/proposal_violation_consumer.go +++ b/consensus/hotstuff/mocks/proposal_violation_consumer.go @@ -18,7 +18,7 @@ func (_m *ProposalViolationConsumer) OnDoubleProposeDetected(_a0 *model.Block, _ } // OnInvalidBlockDetected provides a mock function with given fields: err -func (_m *ProposalViolationConsumer) OnInvalidBlockDetected(err model.InvalidBlockError) { +func (_m *ProposalViolationConsumer) OnInvalidBlockDetected(err model.InvalidProposalError) { _m.Called(err) } diff --git a/consensus/hotstuff/model/errors.go b/consensus/hotstuff/model/errors.go index a8b5e1b2366..3ee63351b06 100644 --- a/consensus/hotstuff/model/errors.go +++ b/consensus/hotstuff/model/errors.go @@ -163,13 +163,52 @@ func (e InvalidTCError) Unwrap() error { return e.Err } +// InvalidProposalError indicates that the proposal is invalid +type InvalidProposalError struct { + InvalidProposal *Proposal + Err error +} + +func NewInvalidProposalErrorf(proposal *Proposal, msg string, args ...interface{}) error { + return InvalidProposalError{ + InvalidProposal: proposal, + Err: fmt.Errorf(msg, args...), + } +} + +func (e InvalidProposalError) Error() string { + return fmt.Sprintf( + "invalid proposal %x at view %d: %s", + e.InvalidProposal.Block.BlockID, + e.InvalidProposal.Block.View, + e.Err.Error(), + ) +} + +// IsInvalidProposalError returns whether an error is InvalidProposalError +func IsInvalidProposalError(err error) bool { + var e InvalidProposalError + return errors.As(err, &e) +} + +// AsInvalidProposalError determines whether the given error is a InvalidProposalError +// (potentially wrapped). It follows the same semantics as a checked type cast. +func AsInvalidProposalError(err error) (*InvalidProposalError, bool) { + var e InvalidProposalError + ok := errors.As(err, &e) + if ok { + return &e, true + } + return nil, false +} + // InvalidBlockError indicates that the block is invalid type InvalidBlockError struct { - InvalidBlock *Proposal + InvalidBlock *Block Err error } -func NewInvalidBlockErrorf(block *Proposal, msg string, args ...interface{}) error { +func NewInvalidBlockErrorf(block *Block, msg string, args ...interface{}) error { return InvalidBlockError{ InvalidBlock: block, Err: fmt.Errorf(msg, args...), @@ -179,8 +218,8 @@ func NewInvalidBlockErrorf(block *Proposal, msg string, args ...interface{}) err func (e InvalidBlockError) Error() string { return fmt.Sprintf( "invalid block %x at view %d: %s", - e.InvalidBlock.Block.BlockID, - e.InvalidBlock.Block.View, + e.InvalidBlock.BlockID, + e.InvalidBlock.View, e.Err.Error(), ) } @@ -191,7 +230,7 @@ func IsInvalidBlockError(err error) bool { return errors.As(err, &e) } -// AsInvalidBlockError determines whether the given error is a InvalidBlockError +// AsInvalidBlockError determines whether the given error is a InvalidProposalError // (potentially wrapped). It follows the same semantics as a checked type cast. func AsInvalidBlockError(err error) (*InvalidBlockError, bool) { var e InvalidBlockError diff --git a/consensus/hotstuff/notifications/log_consumer.go b/consensus/hotstuff/notifications/log_consumer.go index c16c85ec824..4f97fb53343 100644 --- a/consensus/hotstuff/notifications/log_consumer.go +++ b/consensus/hotstuff/notifications/log_consumer.go @@ -46,8 +46,8 @@ func (lc *LogConsumer) OnFinalizedBlock(block *model.Block) { Msg("block finalized") } -func (lc *LogConsumer) OnInvalidBlockDetected(err model.InvalidBlockError) { - invalidBlock := err.InvalidBlock.Block +func (lc *LogConsumer) OnInvalidBlockDetected(err model.InvalidProposalError) { + invalidBlock := err.InvalidProposal.Block lc.log.Warn(). Str(logging.KeySuspicious, "true"). Uint64("block_view", invalidBlock.View). diff --git a/consensus/hotstuff/notifications/noop_consumer.go b/consensus/hotstuff/notifications/noop_consumer.go index b32509676cf..568ff20a012 100644 --- a/consensus/hotstuff/notifications/noop_consumer.go +++ b/consensus/hotstuff/notifications/noop_consumer.go @@ -107,7 +107,7 @@ type NoopProposalViolationConsumer struct{} var _ hotstuff.ProposalViolationConsumer = (*NoopProposalViolationConsumer)(nil) -func (*NoopProposalViolationConsumer) OnInvalidBlockDetected(model.InvalidBlockError) {} +func (*NoopProposalViolationConsumer) OnInvalidBlockDetected(model.InvalidProposalError) {} func (*NoopProposalViolationConsumer) OnDoubleProposeDetected(*model.Block, *model.Block) {} diff --git a/consensus/hotstuff/notifications/pubsub/proposal_violation_distributor.go b/consensus/hotstuff/notifications/pubsub/proposal_violation_distributor.go index e86614d9531..b2ed5f533af 100644 --- a/consensus/hotstuff/notifications/pubsub/proposal_violation_distributor.go +++ b/consensus/hotstuff/notifications/pubsub/proposal_violation_distributor.go @@ -28,7 +28,7 @@ func (d *ProposalViolationDistributor) AddProposalViolationConsumer(consumer hot d.consumers = append(d.consumers, consumer) } -func (d *ProposalViolationDistributor) OnInvalidBlockDetected(err model.InvalidBlockError) { +func (d *ProposalViolationDistributor) OnInvalidBlockDetected(err model.InvalidProposalError) { d.lock.RLock() defer d.lock.RUnlock() for _, subscriber := range d.consumers { diff --git a/consensus/hotstuff/notifications/slashing_violation_consumer.go b/consensus/hotstuff/notifications/slashing_violation_consumer.go index be518841675..af4fc385178 100644 --- a/consensus/hotstuff/notifications/slashing_violation_consumer.go +++ b/consensus/hotstuff/notifications/slashing_violation_consumer.go @@ -21,8 +21,8 @@ func NewSlashingViolationsConsumer(log zerolog.Logger) *SlashingViolationsConsum log: log, } } -func (c *SlashingViolationsConsumer) OnInvalidBlockDetected(err model.InvalidBlockError) { - block := err.InvalidBlock.Block +func (c *SlashingViolationsConsumer) OnInvalidBlockDetected(err model.InvalidProposalError) { + block := err.InvalidProposal.Block c.log.Warn(). Bool(logging.KeySuspicious, true). Hex("proposer_id", block.ProposerID[:]). diff --git a/consensus/hotstuff/validator.go b/consensus/hotstuff/validator.go index 17a14ea0603..5bcc77f1810 100644 --- a/consensus/hotstuff/validator.go +++ b/consensus/hotstuff/validator.go @@ -22,7 +22,7 @@ type Validator interface { // ValidateProposal checks the validity of a proposal. // During normal operations, the following error returns are expected: - // * model.InvalidBlockError if the block is invalid + // * model.InvalidProposalError if the block is invalid // * model.ErrViewForUnknownEpoch if the proposal refers unknown epoch ValidateProposal(proposal *model.Proposal) error diff --git a/consensus/hotstuff/validator/validator.go b/consensus/hotstuff/validator/validator.go index cde767c07e6..b9cafdc5d89 100644 --- a/consensus/hotstuff/validator/validator.go +++ b/consensus/hotstuff/validator/validator.go @@ -197,7 +197,7 @@ func (v *Validator) ValidateQC(qc *flow.QuorumCertificate) error { // A block is considered as valid if it's a valid extension of existing forks. // Note it doesn't check if it's conflicting with finalized block // During normal operations, the following error returns are expected: -// - model.InvalidBlockError if the block is invalid +// - model.InvalidProposalError if the block is invalid // - model.ErrViewForUnknownEpoch if the proposal refers unknown epoch // // Any other error should be treated as exception @@ -208,7 +208,7 @@ func (v *Validator) ValidateProposal(proposal *model.Proposal) error { // validate the proposer's vote and get his identity _, err := v.ValidateVote(proposal.ProposerVote()) if model.IsInvalidVoteError(err) { - return model.NewInvalidBlockErrorf(proposal, "invalid proposer signature: %w", err) + return model.NewInvalidProposalErrorf(proposal, "invalid proposer signature: %w", err) } if err != nil { return fmt.Errorf("error verifying leader signature for block %x: %w", block.BlockID, err) @@ -220,7 +220,7 @@ func (v *Validator) ValidateProposal(proposal *model.Proposal) error { return fmt.Errorf("error determining leader for block %x: %w", block.BlockID, err) } if leader != block.ProposerID { - return model.NewInvalidBlockErrorf(proposal, "proposer %s is not leader (%s) for view %d", block.ProposerID, leader, block.View) + return model.NewInvalidProposalErrorf(proposal, "proposer %s is not leader (%s) for view %d", block.ProposerID, leader, block.View) } // The Block must contain a proof that the primary legitimately entered the respective view. @@ -231,23 +231,23 @@ func (v *Validator) ValidateProposal(proposal *model.Proposal) error { if !lastViewSuccessful { // check if proposal is correctly structured if proposal.LastViewTC == nil { - return model.NewInvalidBlockErrorf(proposal, "QC in block is not for previous view, so expecting a TC but none is included in block") + return model.NewInvalidProposalErrorf(proposal, "QC in block is not for previous view, so expecting a TC but none is included in block") } // check if included TC is for previous view if proposal.Block.View != proposal.LastViewTC.View+1 { - return model.NewInvalidBlockErrorf(proposal, "QC in block is not for previous view, so expecting a TC for view %d but got TC for view %d", proposal.Block.View-1, proposal.LastViewTC.View) + return model.NewInvalidProposalErrorf(proposal, "QC in block is not for previous view, so expecting a TC for view %d but got TC for view %d", proposal.Block.View-1, proposal.LastViewTC.View) } // Check if proposal extends either the newest QC specified in the TC, or a newer QC // in edge cases a leader may construct a TC and QC concurrently such that TC contains // an older QC - in these case we still want to build on the newest QC, so this case is allowed. if proposal.Block.QC.View < proposal.LastViewTC.NewestQC.View { - return model.NewInvalidBlockErrorf(proposal, "TC in block contains a newer QC than the block itself, which is a protocol violation") + return model.NewInvalidProposalErrorf(proposal, "TC in block contains a newer QC than the block itself, which is a protocol violation") } } else if proposal.LastViewTC != nil { // last view ended with QC, including TC is a protocol violation - return model.NewInvalidBlockErrorf(proposal, "last view has ended with QC but proposal includes LastViewTC") + return model.NewInvalidProposalErrorf(proposal, "last view has ended with QC but proposal includes LastViewTC") } // Check signatures, keep the most expensive the last to check @@ -256,7 +256,7 @@ func (v *Validator) ValidateProposal(proposal *model.Proposal) error { err = v.ValidateQC(qc) if err != nil { if model.IsInvalidQCError(err) { - return model.NewInvalidBlockErrorf(proposal, "invalid qc included: %w", err) + return model.NewInvalidProposalErrorf(proposal, "invalid qc included: %w", err) } if errors.Is(err, model.ErrViewForUnknownEpoch) { // We require each replica to be bootstrapped with a QC pointing to a finalized block. Therefore, we should know the @@ -272,7 +272,7 @@ func (v *Validator) ValidateProposal(proposal *model.Proposal) error { err = v.ValidateTC(proposal.LastViewTC) if err != nil { if model.IsInvalidTCError(err) { - return model.NewInvalidBlockErrorf(proposal, "proposals TC's is not valid: %w", err) + return model.NewInvalidProposalErrorf(proposal, "proposals TC's is not valid: %w", err) } if errors.Is(err, model.ErrViewForUnknownEpoch) { // We require each replica to be bootstrapped with a QC pointing to a finalized block. Therefore, we should know the diff --git a/consensus/hotstuff/validator/validator_test.go b/consensus/hotstuff/validator/validator_test.go index 8dbf03736d1..ea41778c259 100644 --- a/consensus/hotstuff/validator/validator_test.go +++ b/consensus/hotstuff/validator/validator_test.go @@ -116,7 +116,7 @@ func (ps *ProposalSuite) TestProposalSignatureError() { assert.Error(ps.T(), err, "a proposal should be rejected if signature check fails") // check that the error is not one that leads to invalid - assert.False(ps.T(), model.IsInvalidBlockError(err), "if signature check fails, we should not receive an ErrorInvalidBlock") + assert.False(ps.T(), model.IsInvalidProposalError(err), "if signature check fails, we should not receive an ErrorInvalidBlock") } func (ps *ProposalSuite) TestProposalSignatureInvalidFormat() { @@ -131,7 +131,7 @@ func (ps *ProposalSuite) TestProposalSignatureInvalidFormat() { assert.Error(ps.T(), err, "a proposal with an invalid signature should be rejected") // check that the error is an invalid proposal error to allow creating slashing challenge - assert.True(ps.T(), model.IsInvalidBlockError(err), "if signature is invalid, we should generate an invalid error") + assert.True(ps.T(), model.IsInvalidProposalError(err), "if signature is invalid, we should generate an invalid error") } func (ps *ProposalSuite) TestProposalSignatureInvalid() { @@ -146,7 +146,7 @@ func (ps *ProposalSuite) TestProposalSignatureInvalid() { assert.Error(ps.T(), err, "a proposal with an invalid signature should be rejected") // check that the error is an invalid proposal error to allow creating slashing challenge - assert.True(ps.T(), model.IsInvalidBlockError(err), "if signature is invalid, we should generate an invalid error") + assert.True(ps.T(), model.IsInvalidProposalError(err), "if signature is invalid, we should generate an invalid error") } func (ps *ProposalSuite) TestProposalWrongLeader() { @@ -163,12 +163,12 @@ func (ps *ProposalSuite) TestProposalWrongLeader() { assert.Error(ps.T(), err, "a proposal from the wrong proposer should be rejected") // check that the error is an invalid proposal error to allow creating slashing challenge - assert.True(ps.T(), model.IsInvalidBlockError(err), "if the proposal has wrong proposer, we should generate a invalid error") + assert.True(ps.T(), model.IsInvalidProposalError(err), "if the proposal has wrong proposer, we should generate a invalid error") } // TestProposalQCInvalid checks that Validator handles the verifier's error returns correctly. // In case of `model.InvalidFormatError` and model.ErrInvalidSignature`, we expect the Validator -// to recognize those as an invalid QC, i.e. returns an `model.InvalidBlockError`. +// to recognize those as an invalid QC, i.e. returns an `model.InvalidProposalError`. // In contrast, unexpected exceptions and `model.InvalidSignerError` should _not_ be // interpreted as a sign of an invalid QC. func (ps *ProposalSuite) TestProposalQCInvalid() { @@ -180,7 +180,7 @@ func (ps *ProposalSuite) TestProposalQCInvalid() { // check that validation fails and the failure case is recognized as an invalid block err := ps.validator.ValidateProposal(ps.proposal) - assert.True(ps.T(), model.IsInvalidBlockError(err), "if the block's QC signature is invalid, an ErrorInvalidBlock error should be raised") + assert.True(ps.T(), model.IsInvalidProposalError(err), "if the block's QC signature is invalid, an ErrorInvalidBlock error should be raised") }) ps.Run("invalid-format", func() { @@ -190,7 +190,7 @@ func (ps *ProposalSuite) TestProposalQCInvalid() { // check that validation fails and the failure case is recognized as an invalid block err := ps.validator.ValidateProposal(ps.proposal) - assert.True(ps.T(), model.IsInvalidBlockError(err), "if the block's QC has an invalid format, an ErrorInvalidBlock error should be raised") + assert.True(ps.T(), model.IsInvalidProposalError(err), "if the block's QC has an invalid format, an ErrorInvalidBlock error should be raised") }) // Theoretically, `VerifyQC` could also return a `model.InvalidSignerError`. However, @@ -207,7 +207,7 @@ func (ps *ProposalSuite) TestProposalQCInvalid() { // check that validation fails and the failure case is recognized as an invalid block err := ps.validator.ValidateProposal(ps.proposal) assert.Error(ps.T(), err) - assert.False(ps.T(), model.IsInvalidBlockError(err)) + assert.False(ps.T(), model.IsInvalidProposalError(err)) }) ps.Run("unknown-exception", func() { @@ -219,7 +219,7 @@ func (ps *ProposalSuite) TestProposalQCInvalid() { // check that validation fails and the failure case is recognized as an invalid block err := ps.validator.ValidateProposal(ps.proposal) assert.ErrorIs(ps.T(), err, exception) - assert.False(ps.T(), model.IsInvalidBlockError(err)) + assert.False(ps.T(), model.IsInvalidProposalError(err)) }) ps.Run("verify-qc-err-view-for-unknown-epoch", func() { @@ -227,11 +227,11 @@ func (ps *ProposalSuite) TestProposalQCInvalid() { ps.verifier.On("VerifyQC", ps.voters, ps.block.QC.SigData, ps.parent.View, ps.parent.BlockID).Return(model.ErrViewForUnknownEpoch) ps.verifier.On("VerifyVote", ps.voter, ps.vote.SigData, ps.block.View, ps.block.BlockID).Return(nil) - // check that validation fails and the failure is considered internal exception and NOT an InvalidBlock error + // check that validation fails and the failure is considered internal exception and NOT an InvalidProposal error err := ps.validator.ValidateProposal(ps.proposal) assert.Error(ps.T(), err) assert.NotErrorIs(ps.T(), err, model.ErrViewForUnknownEpoch) - assert.False(ps.T(), model.IsInvalidBlockError(err)) + assert.False(ps.T(), model.IsInvalidProposalError(err)) }) } @@ -247,7 +247,7 @@ func (ps *ProposalSuite) TestProposalQCError() { assert.Error(ps.T(), err, "a proposal with an invalid QC should be rejected") // check that the error is an invalid proposal error to allow creating slashing challenge - assert.False(ps.T(), model.IsInvalidBlockError(err), "if we can't verify the QC, we should not generate a invalid error") + assert.False(ps.T(), model.IsInvalidProposalError(err), "if we can't verify the QC, we should not generate a invalid error") } // TestProposalWithLastViewTC tests different scenarios where last view has ended with TC @@ -286,7 +286,7 @@ func (ps *ProposalSuite) TestProposalWithLastViewTC() { // in this case proposal without LastViewTC is considered invalid ) err := ps.validator.ValidateProposal(proposal) - require.True(ps.T(), model.IsInvalidBlockError(err)) + require.True(ps.T(), model.IsInvalidProposalError(err)) ps.verifier.AssertNotCalled(ps.T(), "VerifyQC") ps.verifier.AssertNotCalled(ps.T(), "VerifyTC") }) @@ -304,7 +304,7 @@ func (ps *ProposalSuite) TestProposalWithLastViewTC() { helper.WithTCNewestQC(ps.block.QC))), ) err := ps.validator.ValidateProposal(proposal) - require.True(ps.T(), model.IsInvalidBlockError(err)) + require.True(ps.T(), model.IsInvalidProposalError(err)) ps.verifier.AssertNotCalled(ps.T(), "VerifyQC") ps.verifier.AssertNotCalled(ps.T(), "VerifyTC") }) @@ -323,7 +323,7 @@ func (ps *ProposalSuite) TestProposalWithLastViewTC() { helper.WithTCNewestQC(helper.MakeQC(helper.WithQCView(ps.block.View+1))))), ) err := ps.validator.ValidateProposal(proposal) - require.True(ps.T(), model.IsInvalidBlockError(err)) + require.True(ps.T(), model.IsInvalidProposalError(err)) ps.verifier.AssertNotCalled(ps.T(), "VerifyQC") ps.verifier.AssertNotCalled(ps.T(), "VerifyTC") }) @@ -347,7 +347,7 @@ func (ps *ProposalSuite) TestProposalWithLastViewTC() { // this is considered an invalid TC, because highest QC's view is not equal to max{NewestQCViews} proposal.LastViewTC.NewestQCViews[0] = proposal.LastViewTC.NewestQC.View + 1 err := ps.validator.ValidateProposal(proposal) - require.True(ps.T(), model.IsInvalidBlockError(err) && model.IsInvalidTCError(err)) + require.True(ps.T(), model.IsInvalidProposalError(err) && model.IsInvalidTCError(err)) ps.verifier.AssertNotCalled(ps.T(), "VerifyTC") }) ps.Run("included-tc-threshold-not-reached", func() { @@ -368,7 +368,7 @@ func (ps *ProposalSuite) TestProposalWithLastViewTC() { )), ) err = ps.validator.ValidateProposal(proposal) - require.True(ps.T(), model.IsInvalidBlockError(err) && model.IsInvalidTCError(err)) + require.True(ps.T(), model.IsInvalidProposalError(err) && model.IsInvalidTCError(err)) ps.verifier.AssertNotCalled(ps.T(), "VerifyTC") }) ps.Run("included-tc-highest-qc-invalid", func() { @@ -394,7 +394,7 @@ func (ps *ProposalSuite) TestProposalWithLastViewTC() { ps.verifier.On("VerifyQC", ps.voters, qc.SigData, qc.View, qc.BlockID).Return(model.ErrInvalidSignature).Once() err := ps.validator.ValidateProposal(proposal) - require.True(ps.T(), model.IsInvalidBlockError(err) && model.IsInvalidTCError(err)) + require.True(ps.T(), model.IsInvalidProposalError(err) && model.IsInvalidTCError(err)) }) ps.Run("verify-qc-err-view-for-unknown-epoch", func() { newestQC := helper.MakeQC( @@ -420,7 +420,7 @@ func (ps *ProposalSuite) TestProposalWithLastViewTC() { newestQC.View, newestQC.BlockID).Return(model.ErrViewForUnknownEpoch).Once() err := ps.validator.ValidateProposal(proposal) require.Error(ps.T(), err) - require.False(ps.T(), model.IsInvalidBlockError(err)) + require.False(ps.T(), model.IsInvalidProposalError(err)) require.False(ps.T(), model.IsInvalidTCError(err)) require.NotErrorIs(ps.T(), err, model.ErrViewForUnknownEpoch) }) @@ -440,7 +440,7 @@ func (ps *ProposalSuite) TestProposalWithLastViewTC() { ps.verifier.On("VerifyTC", ps.voters, []byte(proposal.LastViewTC.SigData), proposal.LastViewTC.View, proposal.LastViewTC.NewestQCViews).Return(model.ErrInvalidSignature).Once() err := ps.validator.ValidateProposal(proposal) - require.True(ps.T(), model.IsInvalidBlockError(err) && model.IsInvalidTCError(err)) + require.True(ps.T(), model.IsInvalidProposalError(err) && model.IsInvalidTCError(err)) ps.verifier.AssertCalled(ps.T(), "VerifyTC", ps.voters, []byte(proposal.LastViewTC.SigData), proposal.LastViewTC.View, proposal.LastViewTC.NewestQCViews) }) @@ -455,7 +455,7 @@ func (ps *ProposalSuite) TestProposalWithLastViewTC() { helper.WithLastViewTC(helper.MakeTC()), ) err := ps.validator.ValidateProposal(proposal) - require.True(ps.T(), model.IsInvalidBlockError(err)) + require.True(ps.T(), model.IsInvalidProposalError(err)) ps.verifier.AssertNotCalled(ps.T(), "VerifyTC") }) ps.verifier.AssertExpectations(ps.T()) @@ -671,7 +671,7 @@ func (qs *QCSuite) TestQCSignatureError() { // TestQCSignatureInvalid verifies that the Validator correctly handles the model.ErrInvalidSignature. // This error return from `Verifier.VerifyQC` is an expected failure case in case of a byzantine input, where -// one of the signatures in the QC is broken. Hence, the Validator should wrap it as InvalidBlockError. +// one of the signatures in the QC is broken. Hence, the Validator should wrap it as InvalidProposalError. func (qs *QCSuite) TestQCSignatureInvalid() { // change the verifier to fail the QC signature *qs.verifier = mocks.Verifier{} @@ -695,7 +695,7 @@ func (qs *QCSuite) TestQCVerifyQC_ErrViewForUnknownEpoch() { // TestQCSignatureInvalidFormat verifies that the Validator correctly handles the model.InvalidFormatError. // This error return from `Verifier.VerifyQC` is an expected failure case in case of a byzantine input, where -// some binary vector (e.g. `sigData`) is broken. Hence, the Validator should wrap it as InvalidBlockError. +// some binary vector (e.g. `sigData`) is broken. Hence, the Validator should wrap it as InvalidProposalError. func (qs *QCSuite) TestQCSignatureInvalidFormat() { // change the verifier to fail the QC signature *qs.verifier = mocks.Verifier{} @@ -710,7 +710,7 @@ func (qs *QCSuite) TestQCSignatureInvalidFormat() { // In the validator, we previously checked the total weight of all signers meets the supermajority threshold, // which is a _positive_ number. Hence, there must be at least one signer. Hence, `Verifier.VerifyQC` // returning this error would be a symptom of a fatal internal bug. The Validator should _not_ interpret -// this error as an invalid QC / invalid block, i.e. it should _not_ return an `InvalidBlockError`. +// this error as an invalid QC / invalid block, i.e. it should _not_ return an `InvalidProposalError`. func (qs *QCSuite) TestQCEmptySigners() { *qs.verifier = mocks.Verifier{} qs.verifier.On("VerifyQC", mock.Anything, qs.qc.SigData, qs.block.View, qs.block.BlockID).Return( @@ -719,7 +719,7 @@ func (qs *QCSuite) TestQCEmptySigners() { // the Validator should _not_ interpret this as a invalid QC, but as an internal error err := qs.validator.ValidateQC(qs.qc) assert.True(qs.T(), model.IsInsufficientSignaturesError(err)) // unexpected error should be wrapped and propagated upwards - assert.False(qs.T(), model.IsInvalidBlockError(err), err, "should _not_ interpret this as a invalid QC, but as an internal error") + assert.False(qs.T(), model.IsInvalidProposalError(err), err, "should _not_ interpret this as a invalid QC, but as an internal error") } func TestValidateTC(t *testing.T) { diff --git a/consensus/hotstuff/vote_collector.go b/consensus/hotstuff/vote_collector.go index be5c6460723..157ef5338a7 100644 --- a/consensus/hotstuff/vote_collector.go +++ b/consensus/hotstuff/vote_collector.go @@ -59,7 +59,7 @@ type VoteCollector interface { // ProcessBlock performs validation of block signature and processes block with respected collector. // Calling this function will mark conflicting collector as stale and change state of valid collectors // It returns nil if the block is valid. - // It returns model.InvalidBlockError if block is invalid. + // It returns model.InvalidProposalError if block is invalid. // It returns other error if there is exception processing the block. ProcessBlock(block *model.Proposal) error @@ -115,6 +115,6 @@ type VoteProcessorFactory interface { // Create instantiates a VerifyingVoteProcessor for processing votes for a specific proposal. // Caller can be sure that proposal vote was successfully verified and processed. // Expected error returns during normal operations: - // * model.InvalidBlockError - proposal has invalid proposer vote + // * model.InvalidProposalError - proposal has invalid proposer vote Create(log zerolog.Logger, proposal *model.Proposal) (VerifyingVoteProcessor, error) } diff --git a/consensus/hotstuff/voteaggregator/vote_aggregator.go b/consensus/hotstuff/voteaggregator/vote_aggregator.go index be0ef5981dc..fadf5f17e07 100644 --- a/consensus/hotstuff/voteaggregator/vote_aggregator.go +++ b/consensus/hotstuff/voteaggregator/vote_aggregator.go @@ -246,7 +246,7 @@ func (va *VoteAggregator) processQueuedBlock(block *model.Proposal) error { err = collector.ProcessBlock(block) if err != nil { - if model.IsInvalidBlockError(err) { + if model.IsInvalidProposalError(err) { // We are attempting process a block which is invalid // This should never happen, because any component that feeds blocks into VoteAggregator // needs to make sure that it's submitting for processing ONLY valid blocks. diff --git a/consensus/hotstuff/voteaggregator/vote_aggregator_test.go b/consensus/hotstuff/voteaggregator/vote_aggregator_test.go index 5e753689177..006ab52b744 100644 --- a/consensus/hotstuff/voteaggregator/vote_aggregator_test.go +++ b/consensus/hotstuff/voteaggregator/vote_aggregator_test.go @@ -95,7 +95,7 @@ func (s *VoteAggregatorTestSuite) TestProcessInvalidBlock() { collector := mocks.NewVoteCollector(s.T()) collector.On("ProcessBlock", block).Run(func(_ mock.Arguments) { close(processed) - }).Return(model.InvalidBlockError{}) + }).Return(model.InvalidProposalError{}) s.collectors.On("GetOrCreateCollector", block.Block.View).Return(collector, true, nil).Once() // submit block for processing @@ -106,7 +106,7 @@ func (s *VoteAggregatorTestSuite) TestProcessInvalidBlock() { select { case err := <-s.errs: require.Error(s.T(), err) - require.False(s.T(), model.IsInvalidBlockError(err)) + require.False(s.T(), model.IsInvalidProposalError(err)) case <-time.After(100 * time.Millisecond): s.T().Fatalf("expected error but haven't received anything") } diff --git a/consensus/hotstuff/votecollector/factory.go b/consensus/hotstuff/votecollector/factory.go index 554c7675479..2c515fc052c 100644 --- a/consensus/hotstuff/votecollector/factory.go +++ b/consensus/hotstuff/votecollector/factory.go @@ -28,7 +28,7 @@ type baseFactory func(log zerolog.Logger, block *model.Block) (hotstuff.Verifyin // * delegates the creation of the actual instances to baseFactory // * adds the logic to verify the proposer's vote for its own block // Thereby, VoteProcessorFactory guarantees that only proposals with valid proposer -// vote are accepted (as per API specification). Otherwise, an `model.InvalidBlockError` +// vote are accepted (as per API specification). Otherwise, an `model.InvalidProposalError` // is returned. type VoteProcessorFactory struct { baseFactory baseFactory @@ -39,7 +39,7 @@ var _ hotstuff.VoteProcessorFactory = (*VoteProcessorFactory)(nil) // Create instantiates a VerifyingVoteProcessor for the given block proposal. // A VerifyingVoteProcessor are only created for proposals with valid proposer votes. // Expected error returns during normal operations: -// * model.InvalidBlockError - proposal has invalid proposer vote +// * model.InvalidProposalError - proposal has invalid proposer vote func (f *VoteProcessorFactory) Create(log zerolog.Logger, proposal *model.Proposal) (hotstuff.VerifyingVoteProcessor, error) { processor, err := f.baseFactory(log, proposal.Block) if err != nil { @@ -49,7 +49,7 @@ func (f *VoteProcessorFactory) Create(log zerolog.Logger, proposal *model.Propos err = processor.Process(proposal.ProposerVote()) if err != nil { if model.IsInvalidVoteError(err) { - return nil, model.NewInvalidBlockErrorf(proposal, "invalid proposer vote: %w", err) + return nil, model.NewInvalidProposalErrorf(proposal, "invalid proposer vote: %w", err) } return nil, fmt.Errorf("processing proposer's vote for block %v failed: %w", proposal.Block.BlockID, err) } diff --git a/consensus/hotstuff/votecollector/factory_test.go b/consensus/hotstuff/votecollector/factory_test.go index 52cbafe9955..9adeaef98f8 100644 --- a/consensus/hotstuff/votecollector/factory_test.go +++ b/consensus/hotstuff/votecollector/factory_test.go @@ -58,7 +58,7 @@ func TestVoteProcessorFactory_CreateWithInvalidVote(t *testing.T) { processor, err := voteProcessorFactory.Create(unittest.Logger(), proposal) require.Error(t, err) require.Nil(t, processor) - require.True(t, model.IsInvalidBlockError(err)) + require.True(t, model.IsInvalidProposalError(err)) mockedProcessor.AssertExpectations(t) }) @@ -80,7 +80,7 @@ func TestVoteProcessorFactory_CreateWithInvalidVote(t *testing.T) { require.ErrorIs(t, err, exception) require.Nil(t, processor) // an unexpected exception should _not_ be interpreted as the block being invalid - require.False(t, model.IsInvalidBlockError(err)) + require.False(t, model.IsInvalidProposalError(err)) mockedProcessor.AssertExpectations(t) }) @@ -107,7 +107,7 @@ func TestVoteProcessorFactory_CreateProcessException(t *testing.T) { require.ErrorIs(t, err, exception) require.Nil(t, processor) // an unexpected exception should _not_ be interpreted as the block being invalid - require.False(t, model.IsInvalidBlockError(err)) + require.False(t, model.IsInvalidProposalError(err)) mockedFactory.AssertExpectations(t) } diff --git a/engine/collection/compliance/core.go b/engine/collection/compliance/core.go index 39cb9e4663c..1bc3cbc410e 100644 --- a/engine/collection/compliance/core.go +++ b/engine/collection/compliance/core.go @@ -313,7 +313,7 @@ func (c *Core) processBlockProposal(proposal *cluster.Block) error { hotstuffProposal := model.ProposalFromFlow(header) err := c.validator.ValidateProposal(hotstuffProposal) if err != nil { - if invalidBlockErr, ok := model.AsInvalidBlockError(err); ok { + if invalidBlockErr, ok := model.AsInvalidProposalError(err); ok { c.proposalViolationNotifier.OnInvalidBlockDetected(*invalidBlockErr) return engine.NewInvalidInputErrorf("invalid block proposal: %w", err) } diff --git a/engine/collection/compliance/core_test.go b/engine/collection/compliance/core_test.go index c9fc0101784..b8db7dfe0f2 100644 --- a/engine/collection/compliance/core_test.go +++ b/engine/collection/compliance/core_test.go @@ -284,11 +284,11 @@ func (cs *CoreSuite) TestOnBlockProposal_FailsHotStuffValidation() { cs.Run("invalid block error", func() { // the block fails HotStuff validation *cs.validator = *hotstuff.NewValidator(cs.T()) - sentinelError := model.NewInvalidBlockErrorf(hotstuffProposal, "") + sentinelError := model.NewInvalidProposalErrorf(hotstuffProposal, "") cs.validator.On("ValidateProposal", hotstuffProposal).Return(sentinelError) cs.proposalViolationNotifier.On("OnInvalidBlockDetected", sentinelError).Return().Once() // we should notify VoteAggregator about the invalid block - cs.voteAggregator.On("InvalidBlock", hotstuffProposal).Return(nil) + cs.voteAggregator.On("InvalidProposal", hotstuffProposal).Return(nil) // the expected error should be handled within the Core err := cs.core.OnBlockProposal(originID, proposal) @@ -361,7 +361,7 @@ func (cs *CoreSuite) TestOnBlockProposal_FailsProtocolStateValidation() { cs.state.On("Final").Return(func() clusterint.Snapshot { return cs.snapshot }) cs.state.On("Extend", mock.Anything).Return(state.NewInvalidExtensionError("")) // we should notify VoteAggregator about the invalid block - cs.voteAggregator.On("InvalidBlock", hotstuffProposal).Return(nil) + cs.voteAggregator.On("InvalidProposal", hotstuffProposal).Return(nil) // the expected error should be handled within the Core err := cs.core.OnBlockProposal(originID, proposal) diff --git a/engine/common/follower/compliance_core.go b/engine/common/follower/compliance_core.go index c7e569d1e81..efcc0c82d67 100644 --- a/engine/common/follower/compliance_core.go +++ b/engine/common/follower/compliance_core.go @@ -136,14 +136,14 @@ func (c *ComplianceCore) OnBlockRange(originID flow.Identifier, batch []*flow.Bl // 1. The block has been signed by the legitimate primary for the view. This is important in case // there are multiple blocks for the view. We need to differentiate the following byzantine cases: // (i) Some other consensus node that is _not_ primary is trying to publish a block. - // This would result in the validation below failing with an `InvalidBlockError`. + // This would result in the validation below failing with an `InvalidProposalError`. // (ii) The legitimate primary for the view is equivocating. In this case, the validity check // below would pass. Though, the `PendingTree` would eventually notice this, when we connect // the equivocating blocks to the latest finalized block. // 2. The QC within the block is valid. A valid QC proves validity of all ancestors. err := c.validator.ValidateProposal(hotstuffProposal) if err != nil { - if invalidBlockError, ok := model.AsInvalidBlockError(err); ok { + if invalidBlockError, ok := model.AsInvalidProposalError(err); ok { c.proposalViolationNotifier.OnInvalidBlockDetected(*invalidBlockError) return nil } diff --git a/engine/common/follower/compliance_core_test.go b/engine/common/follower/compliance_core_test.go index ff5213c3478..fc9bdc5170e 100644 --- a/engine/common/follower/compliance_core_test.go +++ b/engine/common/follower/compliance_core_test.go @@ -165,7 +165,7 @@ func (s *CoreSuite) TestProcessingInvalidBlock() { blocks := unittest.ChainFixtureFrom(10, s.finalizedBlock) invalidProposal := model.ProposalFromFlow(blocks[len(blocks)-1].Header) - sentinelError := model.NewInvalidBlockErrorf(invalidProposal, "") + sentinelError := model.NewInvalidProposalErrorf(invalidProposal, "") s.validator.On("ValidateProposal", invalidProposal).Return(sentinelError).Once() s.followerConsumer.On("OnInvalidBlockDetected", sentinelError).Return().Once() err := s.core.OnBlockRange(s.originID, blocks) diff --git a/engine/consensus/compliance/core.go b/engine/consensus/compliance/core.go index 8939bb68a5a..d5e737714f3 100644 --- a/engine/consensus/compliance/core.go +++ b/engine/consensus/compliance/core.go @@ -323,7 +323,7 @@ func (c *Core) processBlockProposal(proposal *flow.Block) error { hotstuffProposal := model.ProposalFromFlow(header) err := c.validator.ValidateProposal(hotstuffProposal) if err != nil { - if invalidBlockErr, ok := model.AsInvalidBlockError(err); ok { + if invalidBlockErr, ok := model.AsInvalidProposalError(err); ok { c.proposalViolationNotifier.OnInvalidBlockDetected(*invalidBlockErr) return engine.NewInvalidInputErrorf("invalid block proposal: %w", err) } diff --git a/engine/consensus/compliance/core_test.go b/engine/consensus/compliance/core_test.go index fac752367ec..b48ae4375a5 100644 --- a/engine/consensus/compliance/core_test.go +++ b/engine/consensus/compliance/core_test.go @@ -367,11 +367,11 @@ func (cs *CoreSuite) TestOnBlockProposal_FailsHotStuffValidation() { cs.Run("invalid block error", func() { // the block fails HotStuff validation *cs.validator = *hotstuff.NewValidator(cs.T()) - sentinelError := model.NewInvalidBlockErrorf(hotstuffProposal, "") + sentinelError := model.NewInvalidProposalErrorf(hotstuffProposal, "") cs.validator.On("ValidateProposal", hotstuffProposal).Return(sentinelError) cs.proposalViolationNotifier.On("OnInvalidBlockDetected", sentinelError).Return().Once() // we should notify VoteAggregator about the invalid block - cs.voteAggregator.On("InvalidBlock", hotstuffProposal).Return(nil) + cs.voteAggregator.On("InvalidProposal", hotstuffProposal).Return(nil) // the expected error should be handled within the Core err := cs.core.OnBlockProposal(originID, proposal) @@ -443,7 +443,7 @@ func (cs *CoreSuite) TestOnBlockProposal_FailsProtocolStateValidation() { cs.state.On("Final").Return(func() protint.Snapshot { return cs.snapshot }) cs.state.On("Extend", mock.Anything, mock.Anything).Return(state.NewInvalidExtensionError("")) // we should notify VoteAggregator about the invalid block - cs.voteAggregator.On("InvalidBlock", hotstuffProposal).Return(nil) + cs.voteAggregator.On("InvalidProposal", hotstuffProposal).Return(nil) // the expected error should be handled within the Core err := cs.core.OnBlockProposal(originID, proposal) From 1f1b545e74d028eb4fc2be0751acc0c9510aa18e Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Fri, 12 May 2023 14:18:30 +0300 Subject: [PATCH 36/38] Update consensus/hotstuff/notifications/slashing_violation_consumer.go Co-authored-by: Alexander Hentschel --- consensus/hotstuff/notifications/slashing_violation_consumer.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/consensus/hotstuff/notifications/slashing_violation_consumer.go b/consensus/hotstuff/notifications/slashing_violation_consumer.go index af4fc385178..8b8b55ff886 100644 --- a/consensus/hotstuff/notifications/slashing_violation_consumer.go +++ b/consensus/hotstuff/notifications/slashing_violation_consumer.go @@ -15,6 +15,8 @@ type SlashingViolationsConsumer struct { } var _ hotstuff.ProposalViolationConsumer = (*SlashingViolationsConsumer)(nil) +var _ hotstuff.VoteAggregationViolationConsumer = (*SlashingViolationsConsumer)(nil) +var _ hotstuff.TimeoutAggregationViolationConsumer = (*SlashingViolationsConsumer)(nil) func NewSlashingViolationsConsumer(log zerolog.Logger) *SlashingViolationsConsumer { return &SlashingViolationsConsumer{ From 18f37579cf0461cf7b5ec7afee99f35559a56da6 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Fri, 12 May 2023 14:19:24 +0300 Subject: [PATCH 37/38] Renamed noop consumer --- .../hotstuff/integration/instance_test.go | 2 +- .../hotstuff/notifications/noop_consumer.go | 28 +++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/consensus/hotstuff/integration/instance_test.go b/consensus/hotstuff/integration/instance_test.go index 0f971248c70..469fe252d2a 100644 --- a/consensus/hotstuff/integration/instance_test.go +++ b/consensus/hotstuff/integration/instance_test.go @@ -85,7 +85,7 @@ type Instance struct { type MockedCommunicatorConsumer struct { notifications.NoopProposalViolationConsumer - notifications.NoopPartialConsumer + notifications.NoopParticipantConsumer notifications.NoopFinalizationConsumer *mocks.CommunicatorConsumer } diff --git a/consensus/hotstuff/notifications/noop_consumer.go b/consensus/hotstuff/notifications/noop_consumer.go index 568ff20a012..4ae0584a9d2 100644 --- a/consensus/hotstuff/notifications/noop_consumer.go +++ b/consensus/hotstuff/notifications/noop_consumer.go @@ -13,7 +13,7 @@ import ( type NoopConsumer struct { NoopProposalViolationConsumer NoopFinalizationConsumer - NoopPartialConsumer + NoopParticipantConsumer NoopCommunicatorConsumer } @@ -26,31 +26,31 @@ func NewNoopConsumer() *NoopConsumer { // no-op implementation of hotstuff.Consumer(but not nested interfaces) -type NoopPartialConsumer struct{} +type NoopParticipantConsumer struct{} -func (*NoopPartialConsumer) OnEventProcessed() {} +func (*NoopParticipantConsumer) OnEventProcessed() {} -func (*NoopPartialConsumer) OnStart(uint64) {} +func (*NoopParticipantConsumer) OnStart(uint64) {} -func (*NoopPartialConsumer) OnReceiveProposal(uint64, *model.Proposal) {} +func (*NoopParticipantConsumer) OnReceiveProposal(uint64, *model.Proposal) {} -func (*NoopPartialConsumer) OnReceiveQc(uint64, *flow.QuorumCertificate) {} +func (*NoopParticipantConsumer) OnReceiveQc(uint64, *flow.QuorumCertificate) {} -func (*NoopPartialConsumer) OnReceiveTc(uint64, *flow.TimeoutCertificate) {} +func (*NoopParticipantConsumer) OnReceiveTc(uint64, *flow.TimeoutCertificate) {} -func (*NoopPartialConsumer) OnPartialTc(uint64, *hotstuff.PartialTcCreated) {} +func (*NoopParticipantConsumer) OnPartialTc(uint64, *hotstuff.PartialTcCreated) {} -func (*NoopPartialConsumer) OnLocalTimeout(uint64) {} +func (*NoopParticipantConsumer) OnLocalTimeout(uint64) {} -func (*NoopPartialConsumer) OnViewChange(uint64, uint64) {} +func (*NoopParticipantConsumer) OnViewChange(uint64, uint64) {} -func (*NoopPartialConsumer) OnQcTriggeredViewChange(uint64, uint64, *flow.QuorumCertificate) {} +func (*NoopParticipantConsumer) OnQcTriggeredViewChange(uint64, uint64, *flow.QuorumCertificate) {} -func (*NoopPartialConsumer) OnTcTriggeredViewChange(uint64, uint64, *flow.TimeoutCertificate) {} +func (*NoopParticipantConsumer) OnTcTriggeredViewChange(uint64, uint64, *flow.TimeoutCertificate) {} -func (*NoopPartialConsumer) OnStartingTimeout(model.TimerInfo) {} +func (*NoopParticipantConsumer) OnStartingTimeout(model.TimerInfo) {} -func (*NoopPartialConsumer) OnCurrentViewDetails(uint64, uint64, flow.Identifier) {} +func (*NoopParticipantConsumer) OnCurrentViewDetails(uint64, uint64, flow.Identifier) {} // no-op implementation of hotstuff.FinalizationConsumer From bbc49b82de4c646655eb681370ede430c14296ec Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Fri, 12 May 2023 14:32:17 +0300 Subject: [PATCH 38/38] Fixed tests --- consensus/hotstuff/model/errors.go | 4 ++++ engine/collection/compliance/core_test.go | 4 ++-- engine/consensus/compliance/core_test.go | 4 ++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/consensus/hotstuff/model/errors.go b/consensus/hotstuff/model/errors.go index 3ee63351b06..4244d0ac531 100644 --- a/consensus/hotstuff/model/errors.go +++ b/consensus/hotstuff/model/errors.go @@ -185,6 +185,10 @@ func (e InvalidProposalError) Error() string { ) } +func (e InvalidProposalError) Unwrap() error { + return e.Err +} + // IsInvalidProposalError returns whether an error is InvalidProposalError func IsInvalidProposalError(err error) bool { var e InvalidProposalError diff --git a/engine/collection/compliance/core_test.go b/engine/collection/compliance/core_test.go index b8db7dfe0f2..81f19c2f3b1 100644 --- a/engine/collection/compliance/core_test.go +++ b/engine/collection/compliance/core_test.go @@ -288,7 +288,7 @@ func (cs *CoreSuite) TestOnBlockProposal_FailsHotStuffValidation() { cs.validator.On("ValidateProposal", hotstuffProposal).Return(sentinelError) cs.proposalViolationNotifier.On("OnInvalidBlockDetected", sentinelError).Return().Once() // we should notify VoteAggregator about the invalid block - cs.voteAggregator.On("InvalidProposal", hotstuffProposal).Return(nil) + cs.voteAggregator.On("InvalidBlock", hotstuffProposal).Return(nil) // the expected error should be handled within the Core err := cs.core.OnBlockProposal(originID, proposal) @@ -361,7 +361,7 @@ func (cs *CoreSuite) TestOnBlockProposal_FailsProtocolStateValidation() { cs.state.On("Final").Return(func() clusterint.Snapshot { return cs.snapshot }) cs.state.On("Extend", mock.Anything).Return(state.NewInvalidExtensionError("")) // we should notify VoteAggregator about the invalid block - cs.voteAggregator.On("InvalidProposal", hotstuffProposal).Return(nil) + cs.voteAggregator.On("InvalidBlock", hotstuffProposal).Return(nil) // the expected error should be handled within the Core err := cs.core.OnBlockProposal(originID, proposal) diff --git a/engine/consensus/compliance/core_test.go b/engine/consensus/compliance/core_test.go index b48ae4375a5..270a417411b 100644 --- a/engine/consensus/compliance/core_test.go +++ b/engine/consensus/compliance/core_test.go @@ -371,7 +371,7 @@ func (cs *CoreSuite) TestOnBlockProposal_FailsHotStuffValidation() { cs.validator.On("ValidateProposal", hotstuffProposal).Return(sentinelError) cs.proposalViolationNotifier.On("OnInvalidBlockDetected", sentinelError).Return().Once() // we should notify VoteAggregator about the invalid block - cs.voteAggregator.On("InvalidProposal", hotstuffProposal).Return(nil) + cs.voteAggregator.On("InvalidBlock", hotstuffProposal).Return(nil) // the expected error should be handled within the Core err := cs.core.OnBlockProposal(originID, proposal) @@ -443,7 +443,7 @@ func (cs *CoreSuite) TestOnBlockProposal_FailsProtocolStateValidation() { cs.state.On("Final").Return(func() protint.Snapshot { return cs.snapshot }) cs.state.On("Extend", mock.Anything, mock.Anything).Return(state.NewInvalidExtensionError("")) // we should notify VoteAggregator about the invalid block - cs.voteAggregator.On("InvalidProposal", hotstuffProposal).Return(nil) + cs.voteAggregator.On("InvalidBlock", hotstuffProposal).Return(nil) // the expected error should be handled within the Core err := cs.core.OnBlockProposal(originID, proposal)