-
Notifications
You must be signed in to change notification settings - Fork 674
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Detect if our node is behind the majority
This commit adds a mechanism that detects that our node is behind the majority of the stake. The intent is to later have this mechanism be the trigger for the bootstrapping mechanism. Currently, the bootstrapping mechanism is only active upon node boot, but not at a later point. The mechanism works in the following manner: - It intercepts the snowman engine's Chits message handling, and upon every reception of the Chits message, the mechanism that detects if the node is a straggler (a node with a ledger height behind the rest) may be invoked, if it wasn't invoked too recently. - The mechanism draws statistics from the validators known to it, and computes the latest accepted block for each validator. - The mechanism then proceeds to determine which blocks are pending to be processed (a block pending to be processed was not accepted). - The mechanism then collects a snapshot of all blocks it hasn't accepted yet, and the amount of stake that has accepted this block. - The mechanism then waits for its next invocation, in order to see if it has accepted blocks correlated with enough stake. - If there is too much stake that has accepted blocks by other nodes correlated to it that the node hasn't accepted, then the mechanism announces the node is behind, and returns the time period between the two invocations. - The mechanism sums the total time it has detected the node is behind, until a sampling concludes it is not behind, and then the total time is nullified. Signed-off-by: Yacov Manevich <[email protected]>
- Loading branch information
Showing
10 changed files
with
891 additions
and
13 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,9 @@ | ||
| // Copyright (C) 2019-2024, Ava Labs, Inc. All rights reserved. | ||
| // See the file LICENSE for licensing terms. | ||
|
|
||
| package ids | ||
|
|
||
| type NodeWeight struct { | ||
| Node NodeID | ||
| Weight uint64 | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,59 @@ | ||
| // Copyright (C) 2019-2024, Ava Labs, Inc. All rights reserved. | ||
| // See the file LICENSE for licensing terms. | ||
|
|
||
| package snowman | ||
|
|
||
| import ( | ||
| "context" | ||
| "time" | ||
|
|
||
| "go.uber.org/zap" | ||
|
|
||
| "github.com/ava-labs/avalanchego/ids" | ||
| "github.com/ava-labs/avalanchego/snow/engine/common" | ||
| ) | ||
|
|
||
| type EngineStragglerDetector struct { | ||
| Listener func(duration time.Duration) | ||
| Time func() time.Time | ||
| } | ||
|
|
||
| func (ed *EngineStragglerDetector) AttachToEngine(e *Engine) common.Engine { | ||
| minConfRatio := float64(e.Params.AlphaConfidence) / float64(e.Params.K) | ||
| sd := newStragglerDetector(ed.Time, e.Config.Ctx.Log, minConfRatio, e.Consensus.LastAccepted, | ||
| e.Config.ConnectedValidators.ConnectedValidators, e.Config.ConnectedValidators.ConnectedPercent, | ||
| e.Consensus.Processing, e.acceptedFrontiers.LastAccepted) | ||
| de := &DecoratedEngine{Engine: e} | ||
| de.decorate("Chits", func(e *Engine) { | ||
| behindDuration := sd.CheckIfWeAreStragglingBehind() | ||
| if behindDuration > 0 { | ||
| e.Config.Ctx.Log.Info("We are behind the rest of the network", zap.Float64("seconds", behindDuration.Seconds())) | ||
| } | ||
| e.metrics.stragglingDuration.Set(float64(behindDuration)) | ||
| ed.Listener(behindDuration) | ||
| }) | ||
|
|
||
| return de | ||
| } | ||
|
|
||
| type DecoratedEngine struct { | ||
| decorations map[string]func(*Engine) | ||
|
|
||
| *Engine | ||
| } | ||
|
|
||
| func (de *DecoratedEngine) decorate(method string, f func(*Engine)) { | ||
| if de.decorations == nil { | ||
| de.decorations = map[string]func(*Engine){} | ||
| } | ||
| de.decorations[method] = f | ||
| } | ||
|
|
||
| func (de *DecoratedEngine) Chits(ctx context.Context, nodeID ids.NodeID, requestID uint32, preferredID ids.ID, preferredIDAtHeight ids.ID, acceptedID ids.ID) error { | ||
| f, ok := de.decorations["Chits"] | ||
| if !ok { | ||
| panic("programming error: decorator for Chits not registered") | ||
| } | ||
| f(de.Engine) | ||
| return de.Engine.Chits(ctx, nodeID, requestID, preferredID, preferredIDAtHeight, acceptedID) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,75 @@ | ||
| // Copyright (C) 2019-2024, Ava Labs, Inc. All rights reserved. | ||
| // See the file LICENSE for licensing terms. | ||
|
|
||
| package snowman | ||
|
|
||
| import ( | ||
| "context" | ||
| "testing" | ||
| "time" | ||
|
|
||
| "github.com/stretchr/testify/require" | ||
|
|
||
| "github.com/ava-labs/avalanchego/ids" | ||
| "github.com/ava-labs/avalanchego/snow/consensus/snowman" | ||
| "github.com/ava-labs/avalanchego/snow/consensus/snowman/snowmantest" | ||
| ) | ||
|
|
||
| func TestEngineStragglerDetector(t *testing.T) { | ||
| require := require.New(t) | ||
|
|
||
| fakeClock := make(chan time.Time, 1) | ||
|
|
||
| conf := DefaultConfig(t) | ||
| peerID, _, sender, vm, engine := setup(t, conf) | ||
|
|
||
| parent := snowmantest.BuildChild(snowmantest.Genesis) | ||
| require.NoError(conf.Consensus.Add(parent)) | ||
|
|
||
| listenerShouldInvokeWith := []time.Duration{0, 0, time.Second * 2} | ||
|
|
||
| esd := &EngineStragglerDetector{ | ||
| Listener: func(duration time.Duration) { | ||
| require.Equal(listenerShouldInvokeWith[0], duration) | ||
| listenerShouldInvokeWith = listenerShouldInvokeWith[1:] | ||
| }, | ||
| Time: func() time.Time { | ||
| select { | ||
| case now := <-fakeClock: | ||
| return now | ||
| default: | ||
| require.Fail("should have a time.Time in the channel") | ||
| return time.Time{} | ||
| } | ||
| }, | ||
| } | ||
|
|
||
| decoratedEngine := esd.AttachToEngine(engine) | ||
|
|
||
| vm.GetBlockF = func(_ context.Context, blkID ids.ID) (snowman.Block, error) { | ||
| switch blkID { | ||
| case snowmantest.GenesisID: | ||
| return snowmantest.Genesis, nil | ||
| default: | ||
| return nil, errUnknownBlock | ||
| } | ||
| } | ||
|
|
||
| sender.SendGetF = func(_ context.Context, _ ids.NodeID, _ uint32, _ ids.ID) { | ||
| } | ||
| vm.ParseBlockF = func(_ context.Context, _ []byte) (snowman.Block, error) { | ||
| require.FailNow("should not be called") | ||
| return nil, nil | ||
| } | ||
|
|
||
| now := time.Now() | ||
| fakeClock <- now | ||
| require.NoError(decoratedEngine.Chits(context.Background(), peerID, 0, parent.ID(), parent.ID(), parent.ID())) | ||
| now = now.Add(time.Second * 2) | ||
| fakeClock <- now | ||
| require.NoError(decoratedEngine.Chits(context.Background(), peerID, 0, parent.ID(), parent.ID(), parent.ID())) | ||
| now = now.Add(time.Second * 2) | ||
| fakeClock <- now | ||
| require.NoError(decoratedEngine.Chits(context.Background(), peerID, 0, parent.ID(), parent.ID(), parent.ID())) | ||
| require.Empty(listenerShouldInvokeWith) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.