From f4c9674adb900794a7e084669a929aa9ead7d7e7 Mon Sep 17 00:00:00 2001 From: chris erway Date: Wed, 2 Aug 2023 13:38:19 -0400 Subject: [PATCH 001/232] add some pieces of dynamic lambda from feature/pipeline into agreement --- agreement/player.go | 50 ++++++++++++++++++++++++++++++++++++++++++--- config/consensus.go | 19 +++++++++++++++++ 2 files changed, 66 insertions(+), 3 deletions(-) diff --git a/agreement/player.go b/agreement/player.go index 1ae552b0b1..87fa7f2d53 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -17,6 +17,7 @@ package agreement import ( + "sort" "time" "github.com/algorand/go-algorand/config" @@ -52,6 +53,8 @@ type player struct { // Pending holds the player's proposalTable, which stores proposals that // must be verified after some vote has been verified. Pending proposalTable + + payloadArrivals []time.Duration } func (p *player) T() stateMachineTag { @@ -260,6 +263,45 @@ func (p *player) handleCheckpointEvent(r routerHandle, e checkpointEvent) []acti }} } +func (p *player) handleWinningPayloadArrival(payload proposal) { + // ignoring validatedAt + p.payloadArrivals = append(p.payloadArrivals, payload.receivedAt) +} + +// calculateFilterTimeout chooses the appropriate filter timeout for a new round. +func (p *player) calculateFilterTimeout(period period, ver protocol.ConsensusVersion) time.Duration { + proto := config.Consensus[ver] + + if !proto.DynamicFilterTimeout || period != 0 { + return FilterTimeout(period, ver) + } + + var dynamicDelay time.Duration + if proto.AgreementPipelineDelay <= 0 { + dynamicDelay = 0 + } else if proto.AgreementPipelineDelay > len(p.payloadArrivals) { + dynamicDelay = FilterTimeout(0, ver) + } else { + sortedArrivals := make([]time.Duration, len(p.payloadArrivals)) + copy(sortedArrivals[:], p.payloadArrivals[:]) + sort.Slice(sortedArrivals, func(i, j int) bool { return sortedArrivals[i] < sortedArrivals[j] }) + dynamicDelay = sortedArrivals[proto.AgreementPipelineDelay-1] + } + + // Make sure the dynamic delay is not too small; we want to + // evenly space out the pipelined rounds across FilterTimeout, + // which is the fastest we could agree on blocks anyway (not + // including the soft vote / cert vote times). + if proto.AgreementPipelineDepth > 0 { + evenSpacing := FilterTimeout(0, ver) / time.Duration(proto.AgreementPipelineDepth) + if dynamicDelay < evenSpacing { + dynamicDelay = evenSpacing + } + } + + return dynamicDelay +} + func (p *player) handleThresholdEvent(r routerHandle, e thresholdEvent) []action { r.t.timeR().RecThreshold(e) @@ -275,6 +317,7 @@ func (p *player) handleThresholdEvent(r routerHandle, e thresholdEvent) []action cert := Certificate(e.Bundle) a0 := ensureAction{Payload: res.Payload, Certificate: cert} actions = append(actions, a0) + p.handleWinningPayloadArrival(res.Payload) as := p.enterRound(r, e, p.Round+1) return append(actions, as...) } @@ -375,11 +418,11 @@ func (p *player) enterRound(r routerHandle, source event, target round) []action switch source := source.(type) { case roundInterruptionEvent: - p.Deadline = FilterTimeout(0, source.Proto.Version) + p.Deadline = p.calculateFilterTimeout(0, source.Proto.Version) case thresholdEvent: - p.Deadline = FilterTimeout(0, source.Proto) + p.Deadline = p.calculateFilterTimeout(0, source.Proto) case filterableMessageEvent: - p.Deadline = FilterTimeout(0, source.Proto.Version) + p.Deadline = p.calculateFilterTimeout(0, source.Proto.Version) } // update tracer state to match player @@ -594,6 +637,7 @@ func (p *player) handleMessageEvent(r routerHandle, e messageEvent) (actions []a cert := Certificate(freshestRes.Event.Bundle) a0 := ensureAction{Payload: e.Input.Proposal, Certificate: cert} actions = append(actions, a0) + p.handleWinningPayloadArrival(e.Input.Proposal) as := p.enterRound(r, delegatedE, cert.Round+1) return append(actions, as...) } diff --git a/config/consensus.go b/config/consensus.go index 8c217c18c1..1eed8b8c11 100644 --- a/config/consensus.go +++ b/config/consensus.go @@ -516,6 +516,25 @@ type ConsensusParams struct { // used by agreement for Circulation, and updates the calculation of StateProofOnlineTotalWeight used // by state proofs to use the same method (rather than excluding stake from the top N stakeholders as before). ExcludeExpiredCirculation bool + + // DynamicFilterTimeout + DynamicFilterTimeout bool + + // AgreementPipelineDepth specifies the maximum number of pipelined + // rounds that the agreement protocol can run ahead with. This is + // the maximum difference between the first uncommitted round and + // the round number that we run the agreement protocol for. + AgreementPipelineDepth int + + // AgreementPipelineDelayHistory specifies the number of past block arrivals + // that are measured to determine when to start pipelining the next block. + AgreementPipelineDelayHistory int + + // AgreementPipelineDelay specifies when the agreement code should start + // pipelining the next block, by choosing the delay time of the + // AgreementPipelineDelay'th slowest block to arrive out of the last + // AgreementPipelineDelayHistory lowest-credential block payloads. + AgreementPipelineDelay int } // PaysetCommitType enumerates possible ways for the block header to commit to From 8a987a43cd0bc1d470c80c4fd740a9ae0b46bdfb Mon Sep 17 00:00:00 2001 From: chris erway Date: Thu, 3 Aug 2023 09:18:23 -0400 Subject: [PATCH 002/232] add/adapt code from pipelinePlayer.resizeArrivals on feature/pipeline --- agreement/player.go | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/agreement/player.go b/agreement/player.go index 87fa7f2d53..3d4932786e 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -263,9 +263,20 @@ func (p *player) handleCheckpointEvent(r routerHandle, e checkpointEvent) []acti }} } -func (p *player) handleWinningPayloadArrival(payload proposal) { +func (p *player) handleWinningPayloadArrival(payload proposal, ver protocol.ConsensusVersion) { // ignoring validatedAt p.payloadArrivals = append(p.payloadArrivals, payload.receivedAt) + p.resizePayloadArrivals(ver) +} + +func (p *player) resizePayloadArrivals(ver protocol.ConsensusVersion) { + proto := config.Consensus[ver] + if len(p.payloadArrivals) > proto.AgreementPipelineDelayHistory { + p.payloadArrivals = p.payloadArrivals[len(p.payloadArrivals)-proto.AgreementPipelineDelayHistory:] + } + for len(p.payloadArrivals) < proto.AgreementPipelineDelayHistory { + p.payloadArrivals = append([]time.Duration{FilterTimeout(0, ver)}, p.payloadArrivals...) + } } // calculateFilterTimeout chooses the appropriate filter timeout for a new round. @@ -317,7 +328,7 @@ func (p *player) handleThresholdEvent(r routerHandle, e thresholdEvent) []action cert := Certificate(e.Bundle) a0 := ensureAction{Payload: res.Payload, Certificate: cert} actions = append(actions, a0) - p.handleWinningPayloadArrival(res.Payload) + p.handleWinningPayloadArrival(res.Payload, e.Proto) as := p.enterRound(r, e, p.Round+1) return append(actions, as...) } @@ -637,7 +648,7 @@ func (p *player) handleMessageEvent(r routerHandle, e messageEvent) (actions []a cert := Certificate(freshestRes.Event.Bundle) a0 := ensureAction{Payload: e.Input.Proposal, Certificate: cert} actions = append(actions, a0) - p.handleWinningPayloadArrival(e.Input.Proposal) + p.handleWinningPayloadArrival(e.Input.Proposal, e.Proto.Version) as := p.enterRound(r, delegatedE, cert.Round+1) return append(actions, as...) } From 02fd9f3c42240d6bf3f8982a5e42b7ac008ebc8e Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Tue, 8 Aug 2023 10:57:06 -0400 Subject: [PATCH 003/232] checkpoint: param cleanup --- agreement/player.go | 35 +++++++++++++++++++---------------- config/consensus.go | 21 ++++++++------------- 2 files changed, 27 insertions(+), 29 deletions(-) diff --git a/agreement/player.go b/agreement/player.go index 3d4932786e..550501e512 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -17,6 +17,7 @@ package agreement import ( + "fmt" "sort" "time" @@ -271,45 +272,47 @@ func (p *player) handleWinningPayloadArrival(payload proposal, ver protocol.Cons func (p *player) resizePayloadArrivals(ver protocol.ConsensusVersion) { proto := config.Consensus[ver] - if len(p.payloadArrivals) > proto.AgreementPipelineDelayHistory { - p.payloadArrivals = p.payloadArrivals[len(p.payloadArrivals)-proto.AgreementPipelineDelayHistory:] + if len(p.payloadArrivals) > proto.DynamicFilterPayloadArriavalHistory { + p.payloadArrivals = p.payloadArrivals[len(p.payloadArrivals)-proto.DynamicFilterPayloadArriavalHistory:] } - for len(p.payloadArrivals) < proto.AgreementPipelineDelayHistory { + for len(p.payloadArrivals) < proto.DynamicFilterPayloadArriavalHistory { p.payloadArrivals = append([]time.Duration{FilterTimeout(0, ver)}, p.payloadArrivals...) } } // calculateFilterTimeout chooses the appropriate filter timeout for a new round. func (p *player) calculateFilterTimeout(period period, ver protocol.ConsensusVersion) time.Duration { + proto := config.Consensus[ver] if !proto.DynamicFilterTimeout || period != 0 { + // Either dynamic lambda is disabled, or we're not in period 0 and + // therefore can't use dynamic lambda return FilterTimeout(period, ver) } var dynamicDelay time.Duration - if proto.AgreementPipelineDelay <= 0 { - dynamicDelay = 0 - } else if proto.AgreementPipelineDelay > len(p.payloadArrivals) { + if proto.DynamicFilterPayloadArriavalHistory <= 0 { + // we don't keep any history, use the default + dynamicDelay = FilterTimeout(0, ver) + } else if proto.DynamicFilterPayloadArriavalHistory > len(p.payloadArrivals) { + // not enough smaples, use the default dynamicDelay = FilterTimeout(0, ver) } else { sortedArrivals := make([]time.Duration, len(p.payloadArrivals)) copy(sortedArrivals[:], p.payloadArrivals[:]) sort.Slice(sortedArrivals, func(i, j int) bool { return sortedArrivals[i] < sortedArrivals[j] }) - dynamicDelay = sortedArrivals[proto.AgreementPipelineDelay-1] + dynamicDelay = sortedArrivals[proto.DynamicFilterPayloadArriavalHistory-1] } - // Make sure the dynamic delay is not too small; we want to - // evenly space out the pipelined rounds across FilterTimeout, - // which is the fastest we could agree on blocks anyway (not - // including the soft vote / cert vote times). - if proto.AgreementPipelineDepth > 0 { - evenSpacing := FilterTimeout(0, ver) / time.Duration(proto.AgreementPipelineDepth) - if dynamicDelay < evenSpacing { - dynamicDelay = evenSpacing - } + // Make sure the dynamic delay is not too small + if dynamicDelay < proto.DynamicFilterTimeoutLowerBound { + dynamicDelay = proto.DynamicFilterTimeoutLowerBound } + //dynamicDelay = FilterTimeout(period, ver) + fmt.Println("dynamic delay", dynamicDelay, "payload arrivals", len(p.payloadArrivals)) + return dynamicDelay } diff --git a/config/consensus.go b/config/consensus.go index 1eed8b8c11..02cc03ee9e 100644 --- a/config/consensus.go +++ b/config/consensus.go @@ -520,21 +520,11 @@ type ConsensusParams struct { // DynamicFilterTimeout DynamicFilterTimeout bool - // AgreementPipelineDepth specifies the maximum number of pipelined - // rounds that the agreement protocol can run ahead with. This is - // the maximum difference between the first uncommitted round and - // the round number that we run the agreement protocol for. - AgreementPipelineDepth int + DynamicFilterTimeoutLowerBound time.Duration - // AgreementPipelineDelayHistory specifies the number of past block arrivals + // DynamicFilterPayloadArriavalHistory specifies the number of past block arrivals // that are measured to determine when to start pipelining the next block. - AgreementPipelineDelayHistory int - - // AgreementPipelineDelay specifies when the agreement code should start - // pipelining the next block, by choosing the delay time of the - // AgreementPipelineDelay'th slowest block to arrive out of the last - // AgreementPipelineDelayHistory lowest-credential block payloads. - AgreementPipelineDelay int + DynamicFilterPayloadArriavalHistory int } // PaysetCommitType enumerates possible ways for the block header to commit to @@ -1366,6 +1356,10 @@ func initConsensusProtocols() { v38.AgreementFilterTimeoutPeriod0 = 3000 * time.Millisecond + v38.DynamicFilterTimeout = true + v38.DynamicFilterPayloadArriavalHistory = 32 + v38.DynamicFilterTimeoutLowerBound = time.Second + Consensus[protocol.ConsensusV38] = v38 // v37 can be upgraded to v38, with an update delay of 12h: @@ -1376,6 +1370,7 @@ func initConsensusProtocols() { // ConsensusFuture is used to test features that are implemented // but not yet released in a production protocol version. vFuture := v38 + vFuture.ApprovedUpgrades = map[protocol.ConsensusVersion]uint64{} vFuture.LogicSigVersion = 10 // When moving this to a release, put a new higher LogicSigVersion here From 3eb1c794a8a9cbfedd00a499d57c17c9151ff5a0 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Tue, 8 Aug 2023 15:54:45 -0400 Subject: [PATCH 004/232] reset history on non-0 periods --- agreement/agreementtest/simulate.go | 1 + agreement/player.go | 12 ++++++++++-- agreement/service_test.go | 18 ++++++++++++++++-- 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/agreement/agreementtest/simulate.go b/agreement/agreementtest/simulate.go index 8d8b713c68..35ff92de36 100644 --- a/agreement/agreementtest/simulate.go +++ b/agreement/agreementtest/simulate.go @@ -70,6 +70,7 @@ func (i *instant) TimeoutAt(d time.Duration) <-chan time.Time { } if d == agreement.FilterTimeout(0, protocol.ConsensusCurrentVersion) && !i.HasPending("pseudonode") { + //if d == priod0FilterTimeout && !i.HasPending("pseudonode") { close(ta) } return ta diff --git a/agreement/player.go b/agreement/player.go index 550501e512..d9a9a55754 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -285,9 +285,11 @@ func (p *player) calculateFilterTimeout(period period, ver protocol.ConsensusVer proto := config.Consensus[ver] + //return FilterTimeout(period, ver) + if !proto.DynamicFilterTimeout || period != 0 { // Either dynamic lambda is disabled, or we're not in period 0 and - // therefore can't use dynamic lambda + // therefore, can't use dynamic lambda return FilterTimeout(period, ver) } @@ -384,7 +386,13 @@ func (p *player) enterPeriod(r routerHandle, source thresholdEvent, target perio p.Step = soft p.Napping = false p.FastRecoveryDeadline = 0 // set immediately - p.Deadline = FilterTimeout(target, source.Proto) + + if target != 0 { + // We entered a non-0 period, we should reset the filter timeout + // calculation mechanism. + p.payloadArrivals = make([]time.Duration, 0) + } + p.Deadline = p.calculateFilterTimeout(target, source.Proto) // update tracer state to match player r.t.setMetadata(tracerMetadata{p.Round, p.Period, p.Step}) diff --git a/agreement/service_test.go b/agreement/service_test.go index e24f81890c..0fca7f9518 100644 --- a/agreement/service_test.go +++ b/agreement/service_test.go @@ -874,6 +874,20 @@ func simulateAgreementWithConsensusVersion(t *testing.T, numNodes int, numRounds simulateAgreementWithLedgerFactory(t, numNodes, numRounds, traceLevel, ledgerFactory) } +func maxFilterTimeoutPeriod0(services []*Service, version protocol.ConsensusVersion) time.Duration { + if len(services) == 0 { + return FilterTimeout(0, version) + } + maxFilter := services[0].persistStatus.calculateFilterTimeout(0, version) + for _, service := range services { + curr := service.persistStatus.calculateFilterTimeout(0, version) + if maxFilter < curr { + maxFilter = curr + } + } + return maxFilter +} + func simulateAgreementWithLedgerFactory(t *testing.T, numNodes int, numRounds int, traceLevel traceLevel, ledgerFactory func(map[basics.Address]basics.AccountData) Ledger) { _, baseLedger, cleanupFn, services, clocks, ledgers, activityMonitor := setupAgreement(t, numNodes, traceLevel, ledgerFactory) startRound := baseLedger.NextRound() @@ -888,10 +902,10 @@ func simulateAgreementWithLedgerFactory(t *testing.T, numNodes int, numRounds in // run round with round-specific consensus version first (since fix in #1896) version, _ := baseLedger.ConsensusVersion(ParamsRound(startRound)) - zeroes = runRound(clocks, activityMonitor, zeroes, FilterTimeout(0, version)) + zeroes = runRound(clocks, activityMonitor, zeroes, maxFilterTimeoutPeriod0(services, version)) for j := 1; j < numRounds; j++ { version, _ := baseLedger.ConsensusVersion(ParamsRound(baseLedger.NextRound() + basics.Round(j-1))) - zeroes = runRound(clocks, activityMonitor, zeroes, FilterTimeout(0, version)) + zeroes = runRound(clocks, activityMonitor, zeroes, maxFilterTimeoutPeriod0(services, version)) } for i := 0; i < numNodes; i++ { From 03a3c90db748f70c6b5038e81e0296b52dedd7e6 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Wed, 9 Aug 2023 15:15:10 -0400 Subject: [PATCH 005/232] test clock tweak --- agreement/player.go | 3 +-- agreement/service_test.go | 13 ++++++++++--- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/agreement/player.go b/agreement/player.go index d9a9a55754..f7fb008534 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -17,7 +17,6 @@ package agreement import ( - "fmt" "sort" "time" @@ -313,7 +312,7 @@ func (p *player) calculateFilterTimeout(period period, ver protocol.ConsensusVer } //dynamicDelay = FilterTimeout(period, ver) - fmt.Println("dynamic delay", dynamicDelay, "payload arrivals", len(p.payloadArrivals)) + //fmt.Println("dynamic delay", dynamicDelay, "payload arrivals", len(p.payloadArrivals)) return dynamicDelay } diff --git a/agreement/service_test.go b/agreement/service_test.go index 0fca7f9518..448b77c67d 100644 --- a/agreement/service_test.go +++ b/agreement/service_test.go @@ -874,7 +874,13 @@ func simulateAgreementWithConsensusVersion(t *testing.T, numNodes int, numRounds simulateAgreementWithLedgerFactory(t, numNodes, numRounds, traceLevel, ledgerFactory) } -func maxFilterTimeoutPeriod0(services []*Service, version protocol.ConsensusVersion) time.Duration { +func maxFilterTimeoutPeriod0(services []*Service, version protocol.ConsensusVersion, iter int) time.Duration { + if iter < 32 { // TODO: 32 is the history length we track for dynamic lambda. We need to change this param + return FilterTimeout(0, version) + } else { + return time.Second // this is the min filter timeout we're using. + } + if len(services) == 0 { return FilterTimeout(0, version) } @@ -902,10 +908,11 @@ func simulateAgreementWithLedgerFactory(t *testing.T, numNodes int, numRounds in // run round with round-specific consensus version first (since fix in #1896) version, _ := baseLedger.ConsensusVersion(ParamsRound(startRound)) - zeroes = runRound(clocks, activityMonitor, zeroes, maxFilterTimeoutPeriod0(services, version)) + zeroes = runRound(clocks, activityMonitor, zeroes, maxFilterTimeoutPeriod0(services, version, 0)) for j := 1; j < numRounds; j++ { + // TODO: remove this: time.Sleep(100 * time.Millisecond) version, _ := baseLedger.ConsensusVersion(ParamsRound(baseLedger.NextRound() + basics.Round(j-1))) - zeroes = runRound(clocks, activityMonitor, zeroes, maxFilterTimeoutPeriod0(services, version)) + zeroes = runRound(clocks, activityMonitor, zeroes, maxFilterTimeoutPeriod0(services, version, j)) } for i := 0; i < numNodes; i++ { From 0502b7c532075c3ab6d8b2c2c0aba9a4ab54b758 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Wed, 9 Aug 2023 15:56:43 -0400 Subject: [PATCH 006/232] tagged clock interface --- util/timers/frozen.go | 2 +- util/timers/interface.go | 14 +++++++-- util/timers/monotonic.go | 15 ++++++---- util/timers/monotonic_test.go | 55 ++++++++++++++++++++++++++++++----- 4 files changed, 71 insertions(+), 15 deletions(-) diff --git a/util/timers/frozen.go b/util/timers/frozen.go index e6487b1a8a..acc33f7bbe 100644 --- a/util/timers/frozen.go +++ b/util/timers/frozen.go @@ -38,7 +38,7 @@ func (m *Frozen) Zero() Clock { } // TimeoutAt returns a channel that will signal when the duration has elapsed. -func (m *Frozen) TimeoutAt(delta time.Duration) <-chan time.Time { +func (m *Frozen) TimeoutAt(delta time.Duration, timeoutType Timeout) <-chan time.Time { return m.timeoutCh } diff --git a/util/timers/interface.go b/util/timers/interface.go index e96aced757..fcb2dade80 100644 --- a/util/timers/interface.go +++ b/util/timers/interface.go @@ -21,6 +21,13 @@ import ( "time" ) +type Timeout int + +const ( + Filter Timeout = iota + Fast +) + // Clock provides timeout events which fire at some point after a point in time. type Clock interface { // Zero returns a reset Clock. TimeoutAt channels will use the point @@ -32,10 +39,13 @@ type Clock interface { Since() time.Duration // TimeoutAt returns a channel that fires delta time after Zero was called. + // timeoutType is specifies the reason for this timeout. If there are two + // timeouts of the same type at the same time, then only one of them fires. // If delta has already passed, it returns a closed channel. // - // TimeoutAt must be called after Zero; otherwise, the channel's behavior is undefined. - TimeoutAt(delta time.Duration) <-chan time.Time + // TimeoutAt must be called after Zero; otherwise, the channel's behavior is + // undefined. + TimeoutAt(delta time.Duration, timeoutType Timeout) <-chan time.Time // Encode serializes the Clock into a byte slice. Encode() []byte diff --git a/util/timers/monotonic.go b/util/timers/monotonic.go index 70db87da3e..01135d06c2 100644 --- a/util/timers/monotonic.go +++ b/util/timers/monotonic.go @@ -26,7 +26,7 @@ import ( // Monotonic uses the system's monotonic clock to emit timeouts. type Monotonic struct { zero time.Time - timeouts map[time.Duration]<-chan time.Time + timeouts map[Timeout]map[time.Duration]<-chan time.Time } // MakeMonotonicClock creates a new monotonic clock with a given zero point. @@ -44,11 +44,16 @@ func (m *Monotonic) Zero() Clock { } // TimeoutAt returns a channel that will signal when the duration has elapsed. -func (m *Monotonic) TimeoutAt(delta time.Duration) <-chan time.Time { +func (m *Monotonic) TimeoutAt(delta time.Duration, timeoutType Timeout) <-chan time.Time { if m.timeouts == nil { - m.timeouts = make(map[time.Duration]<-chan time.Time) + m.timeouts = make(map[Timeout]map[time.Duration]<-chan time.Time) } - timeoutCh, ok := m.timeouts[delta] + + if _, ok := m.timeouts[timeoutType]; !ok { + m.timeouts[timeoutType] = make(map[time.Duration]<-chan time.Time) + } + + timeoutCh, ok := m.timeouts[timeoutType][delta] if ok { return timeoutCh } @@ -62,7 +67,7 @@ func (m *Monotonic) TimeoutAt(delta time.Duration) <-chan time.Time { } else { timeoutCh = time.After(left) } - m.timeouts[delta] = timeoutCh + m.timeouts[timeoutType][delta] = timeoutCh return timeoutCh } diff --git a/util/timers/monotonic_test.go b/util/timers/monotonic_test.go index f8821b300b..ba847e1a98 100644 --- a/util/timers/monotonic_test.go +++ b/util/timers/monotonic_test.go @@ -17,10 +17,11 @@ package timers import ( - "github.com/algorand/go-algorand/test/partitiontest" "math/rand" "testing" "time" + + "github.com/algorand/go-algorand/test/partitiontest" ) func polled(ch <-chan time.Time) bool { @@ -42,7 +43,7 @@ func TestMonotonicDelta(t *testing.T) { d := time.Millisecond * 100 c = m.Zero() - ch = c.TimeoutAt(d) + ch = c.TimeoutAt(d, Filter) if polled(ch) { t.Errorf("channel fired ~100ms early") } @@ -52,7 +53,7 @@ func TestMonotonicDelta(t *testing.T) { t.Errorf("channel failed to fire at 100ms") } - ch = c.TimeoutAt(d / 2) + ch = c.TimeoutAt(d/2, Filter) if !polled(ch) { t.Errorf("channel failed to fire at 50ms") } @@ -66,7 +67,7 @@ func TestMonotonicZeroDelta(t *testing.T) { var ch <-chan time.Time c = m.Zero() - ch = c.TimeoutAt(0) + ch = c.TimeoutAt(0, Filter) if !polled(ch) { t.Errorf("read failed on channel at zero timeout") } @@ -80,7 +81,7 @@ func TestMonotonicNegativeDelta(t *testing.T) { var ch <-chan time.Time c = m.Zero() - ch = c.TimeoutAt(-time.Second) + ch = c.TimeoutAt(-time.Second, Filter) if !polled(ch) { t.Errorf("read failed on channel at negative timeout") } @@ -96,7 +97,7 @@ func TestMonotonicZeroTwice(t *testing.T) { d := time.Millisecond * 100 c = m.Zero() - ch = c.TimeoutAt(d) + ch = c.TimeoutAt(d, Filter) if polled(ch) { t.Errorf("channel fired ~100ms early") } @@ -107,7 +108,7 @@ func TestMonotonicZeroTwice(t *testing.T) { } c = c.Zero() - ch = c.TimeoutAt(d) + ch = c.TimeoutAt(d, Filter) if polled(ch) { t.Errorf("channel fired ~100ms early after call to Zero") } @@ -152,3 +153,43 @@ func TestMonotonicEncodeDecode(t *testing.T) { singleTest(c, "random") } } + +func TestTimeoutTypes(t *testing.T) { + partitiontest.PartitionTest(t) + + var m Monotonic + var c Clock + + d := time.Millisecond * 100 + + c = m.Zero() + ch1 := c.TimeoutAt(d, Filter) + ch2 := c.TimeoutAt(d, Fast) + if polled(ch1) { + t.Errorf("channel fired ~100ms early") + } + if polled(ch2) { + t.Errorf("channel fired ~100ms early") + } + + if ch1 == ch2 { + t.Errorf("equal channels for different timeout types") + } + + <-time.After(d * 2) + if !polled(ch1) { + t.Errorf("channel failed to fire at 100ms") + } + if !polled(ch2) { + t.Errorf("channel failed to fire at 100ms") + } + + ch1 = c.TimeoutAt(d/2, Filter) + if !polled(ch1) { + t.Errorf("channel failed to fire at 50ms") + } + ch2 = c.TimeoutAt(d/2, Filter) + if !polled(ch2) { + t.Errorf("channel failed to fire at 50ms") + } +} From b81bbd3d2d2afb577f3772b6cb7bf4f62254e354 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Wed, 9 Aug 2023 16:20:43 -0400 Subject: [PATCH 007/232] checkpoint: integrating the new clock --- agreement/agreementtest/simulate.go | 5 ++-- agreement/demux.go | 5 ++-- agreement/demux_test.go | 4 +-- agreement/fuzzer/networkFacade_test.go | 2 +- agreement/service_test.go | 34 ++++++++++++++++---------- util/timers/interface.go | 3 ++- 6 files changed, 31 insertions(+), 22 deletions(-) diff --git a/agreement/agreementtest/simulate.go b/agreement/agreementtest/simulate.go index 8d8b713c68..5b42ee37a2 100644 --- a/agreement/agreementtest/simulate.go +++ b/agreement/agreementtest/simulate.go @@ -31,7 +31,6 @@ import ( "github.com/algorand/go-algorand/crypto" "github.com/algorand/go-algorand/data/basics" "github.com/algorand/go-algorand/logging" - "github.com/algorand/go-algorand/protocol" "github.com/algorand/go-algorand/util/db" "github.com/algorand/go-algorand/util/timers" ) @@ -60,7 +59,7 @@ func (i *instant) Encode() []byte { return nil } -func (i *instant) TimeoutAt(d time.Duration) <-chan time.Time { +func (i *instant) TimeoutAt(d time.Duration, timeoutType timers.Timeout) <-chan time.Time { ta := make(chan time.Time) select { case <-i.timeoutAtCalled: @@ -69,7 +68,7 @@ func (i *instant) TimeoutAt(d time.Duration) <-chan time.Time { return ta } - if d == agreement.FilterTimeout(0, protocol.ConsensusCurrentVersion) && !i.HasPending("pseudonode") { + if timeoutType == timers.Filter && !i.HasPending("pseudonode") { close(ta) } return ta diff --git a/agreement/demux.go b/agreement/demux.go index f31c4d075d..8053c4443c 100644 --- a/agreement/demux.go +++ b/agreement/demux.go @@ -25,6 +25,7 @@ import ( "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/logging/logspec" "github.com/algorand/go-algorand/protocol" + "github.com/algorand/go-algorand/util/timers" ) const ( @@ -250,8 +251,8 @@ func (d *demux) next(s *Service, deadline time.Duration, fastDeadline time.Durat } ledgerNextRoundCh := s.Ledger.Wait(nextRound) - deadlineCh := s.Clock.TimeoutAt(deadline) - fastDeadlineCh := s.Clock.TimeoutAt(fastDeadline) + deadlineCh := s.Clock.TimeoutAt(deadline, timers.Deadline) + fastDeadlineCh := s.Clock.TimeoutAt(fastDeadline, timers.Fast) d.UpdateEventsQueue(eventQueueDemux, 0) d.monitor.dec(demuxCoserviceType) diff --git a/agreement/demux_test.go b/agreement/demux_test.go index 027dbc9e13..f5139f2d6a 100644 --- a/agreement/demux_test.go +++ b/agreement/demux_test.go @@ -428,8 +428,8 @@ func (t *demuxTester) Zero() timers.Clock { } // implement timers.Clock -func (t *demuxTester) TimeoutAt(delta time.Duration) <-chan time.Time { - if delta == fastTimeoutChTime { +func (t *demuxTester) TimeoutAt(delta time.Duration, timeoutType timers.Timeout) <-chan time.Time { + if timeoutType == timers.Fast { return nil } diff --git a/agreement/fuzzer/networkFacade_test.go b/agreement/fuzzer/networkFacade_test.go index 6db2d88c2e..e955c8e9c2 100644 --- a/agreement/fuzzer/networkFacade_test.go +++ b/agreement/fuzzer/networkFacade_test.go @@ -375,7 +375,7 @@ func (n *NetworkFacade) Rezero() { // Since implements the Clock interface. func (n *NetworkFacade) Since() time.Duration { return 0 } -func (n *NetworkFacade) TimeoutAt(d time.Duration) <-chan time.Time { +func (n *NetworkFacade) TimeoutAt(d time.Duration, timeoutType timers.Timeout) <-chan time.Time { defer n.timeoutAtInitOnce.Do(func() { n.timeoutAtInitWait.Done() }) diff --git a/agreement/service_test.go b/agreement/service_test.go index e24f81890c..6093d402e5 100644 --- a/agreement/service_test.go +++ b/agreement/service_test.go @@ -49,14 +49,14 @@ type testingClock struct { zeroes uint - TA map[time.Duration]chan time.Time // TimeoutAt + TA map[timers.Timeout]map[time.Duration]chan time.Time // TimeoutAt monitor *coserviceMonitor } func makeTestingClock(m *coserviceMonitor) *testingClock { c := new(testingClock) - c.TA = make(map[time.Duration]chan time.Time) + c.TA = make(map[timers.Timeout]map[time.Duration]chan time.Time) c.monitor = m return c } @@ -66,7 +66,7 @@ func (c *testingClock) Zero() timers.Clock { defer c.mu.Unlock() c.zeroes++ - c.TA = make(map[time.Duration]chan time.Time) + c.TA = make(map[timers.Timeout]map[time.Duration]chan time.Time) c.monitor.clearClock() return c } @@ -75,14 +75,18 @@ func (c *testingClock) Since() time.Duration { return 0 } -func (c *testingClock) TimeoutAt(d time.Duration) <-chan time.Time { +func (c *testingClock) TimeoutAt(d time.Duration, timeoutType timers.Timeout) <-chan time.Time { c.mu.Lock() defer c.mu.Unlock() - ta := c.TA[d] + if _, ok := c.TA[timeoutType]; !ok { + c.TA[timeoutType] = make(map[time.Duration]chan time.Time) + } + + ta := c.TA[timeoutType][d] if ta == nil { - c.TA[d] = make(chan time.Time) - ta = c.TA[d] + c.TA[timeoutType][d] = make(chan time.Time) + ta = c.TA[timeoutType][d] } return ta } @@ -99,14 +103,18 @@ func (c *testingClock) prepareToFire() { c.monitor.inc(clockCoserviceType) } -func (c *testingClock) fire(d time.Duration) { +func (c *testingClock) fire(d time.Duration, timeoutType timers.Timeout) { c.mu.Lock() defer c.mu.Unlock() - if c.TA[d] == nil { - c.TA[d] = make(chan time.Time) + if _, ok := c.TA[timeoutType]; !ok { + c.TA[timeoutType] = make(map[time.Duration]chan time.Time) + } + + if c.TA[timeoutType][d] == nil { + c.TA[timeoutType][d] = make(chan time.Time) } - close(c.TA[d]) + close(c.TA[timeoutType][d]) } type testingNetwork struct { @@ -830,12 +838,12 @@ func expectNoNewPeriod(clocks []timers.Clock, zeroes uint) (newzeroes uint) { return zeroes } -func triggerGlobalTimeout(d time.Duration, clocks []timers.Clock, activityMonitor *activityMonitor) { +func triggerGlobalTimeout(d time.Duration, timeoutType timers.Timeout, clocks []timers.Clock, activityMonitor *activityMonitor) { for i := range clocks { clocks[i].(*testingClock).prepareToFire() } for i := range clocks { - clocks[i].(*testingClock).fire(d) + clocks[i].(*testingClock).fire(d, timeoutType) } activityMonitor.waitForActivity() activityMonitor.waitForQuiet() diff --git a/util/timers/interface.go b/util/timers/interface.go index fcb2dade80..8b69218ef3 100644 --- a/util/timers/interface.go +++ b/util/timers/interface.go @@ -24,8 +24,9 @@ import ( type Timeout int const ( - Filter Timeout = iota + Deadline Timeout = iota Fast + Filter ) // Clock provides timeout events which fire at some point after a point in time. From 7ab2db77701bbf98556fb8d5ca6035ac7228054f Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Wed, 9 Aug 2023 17:55:26 -0400 Subject: [PATCH 008/232] checkpoint: integrating tagged timers --- Makefile | 2 +- agreement/agreementtest/simulate.go | 2 +- agreement/demux.go | 4 +- agreement/demux_test.go | 2 +- agreement/msgp_gen.go | 303 ++++++++++++++++++++++++++-- agreement/msgp_gen_test.go | 60 ++++++ agreement/persistence_test.go | 8 +- agreement/player.go | 30 ++- agreement/player_test.go | 3 +- agreement/service.go | 4 +- agreement/service_test.go | 169 +++++++++------- util/timers/interface.go | 2 +- 12 files changed, 476 insertions(+), 113 deletions(-) diff --git a/Makefile b/Makefile index 760dd42084..cf5113a8a8 100644 --- a/Makefile +++ b/Makefile @@ -91,7 +91,7 @@ ALGOD_API_PACKAGES := $(sort $(shell GOPATH=$(GOPATH) && GO111MODULE=off && cd d GOMOD_DIRS := ./tools/block-generator ./tools/x-repo-types -MSGP_GENERATE := ./protocol ./protocol/test ./crypto ./crypto/merklearray ./crypto/merklesignature ./crypto/stateproof ./data/basics ./data/transactions ./data/stateproofmsg ./data/committee ./data/bookkeeping ./data/hashable ./agreement ./rpcs ./network ./node ./ledger ./ledger/ledgercore ./ledger/store/trackerdb ./ledger/store/trackerdb/generickv ./ledger/encoded ./stateproof ./data/account ./daemon/algod/api/spec/v2 +MSGP_GENERATE := ./protocol ./protocol/test ./crypto ./crypto/merklearray ./crypto/merklesignature ./crypto/stateproof ./data/basics ./data/transactions ./data/stateproofmsg ./data/committee ./data/bookkeeping ./data/hashable ./agreement ./rpcs ./network ./node ./ledger ./ledger/ledgercore ./ledger/store/trackerdb ./ledger/store/trackerdb/generickv ./ledger/encoded ./stateproof ./data/account ./daemon/algod/api/spec/v2 ./util/timers default: build diff --git a/agreement/agreementtest/simulate.go b/agreement/agreementtest/simulate.go index 5b42ee37a2..5ba569b2ca 100644 --- a/agreement/agreementtest/simulate.go +++ b/agreement/agreementtest/simulate.go @@ -68,7 +68,7 @@ func (i *instant) TimeoutAt(d time.Duration, timeoutType timers.Timeout) <-chan return ta } - if timeoutType == timers.Filter && !i.HasPending("pseudonode") { + if timeoutType == timers.Filter && !i.HasPending("pseudonode") { //d == agreement.FilterTimeout(0, protocol.ConsensusCurrentVersion) && !i.HasPending("pseudonode") { close(ta) } return ta diff --git a/agreement/demux.go b/agreement/demux.go index 8053c4443c..28bbf4f4e9 100644 --- a/agreement/demux.go +++ b/agreement/demux.go @@ -191,7 +191,7 @@ func (d *demux) verifyBundle(ctx context.Context, m message, r round, p period, // next blocks until it observes an external input event of interest for the state machine. // // If ok is false, there are no more events so the agreement service should quit. -func (d *demux) next(s *Service, deadline time.Duration, fastDeadline time.Duration, currentRound round) (e externalEvent, ok bool) { +func (d *demux) next(s *Service, deadline Deadline, fastDeadline time.Duration, currentRound round) (e externalEvent, ok bool) { defer func() { if !ok { return @@ -251,7 +251,7 @@ func (d *demux) next(s *Service, deadline time.Duration, fastDeadline time.Durat } ledgerNextRoundCh := s.Ledger.Wait(nextRound) - deadlineCh := s.Clock.TimeoutAt(deadline, timers.Deadline) + deadlineCh := s.Clock.TimeoutAt(deadline.Deadline, deadline.Type) fastDeadlineCh := s.Clock.TimeoutAt(fastDeadline, timers.Fast) d.UpdateEventsQueue(eventQueueDemux, 0) diff --git a/agreement/demux_test.go b/agreement/demux_test.go index f5139f2d6a..f1747ef3d5 100644 --- a/agreement/demux_test.go +++ b/agreement/demux_test.go @@ -675,7 +675,7 @@ func (t *demuxTester) TestUsecase(testcase demuxTestUsecase) bool { close(s.quit) } - e, ok := dmx.next(s, time.Second, fastTimeoutChTime, 300) + e, ok := dmx.next(s, Deadline{Deadline: time.Second, Type: timers.Deadline}, fastTimeoutChTime, 300) if !assert.Equal(t, testcase.ok, ok) { return false diff --git a/agreement/msgp_gen.go b/agreement/msgp_gen.go index 19a803b759..6100bda791 100644 --- a/agreement/msgp_gen.go +++ b/agreement/msgp_gen.go @@ -13,6 +13,7 @@ import ( "github.com/algorand/go-algorand/data/bookkeeping" "github.com/algorand/go-algorand/data/committee" "github.com/algorand/go-algorand/protocol" + "github.com/algorand/go-algorand/util/timers" ) // The following msgp objects are implemented in this file: @@ -34,6 +35,15 @@ import ( // |-----> (*) MsgIsZero // |-----> ConsensusVersionViewMaxSize() // +// Deadline +// |-----> (*) MarshalMsg +// |-----> (*) CanMarshalMsg +// |-----> (*) UnmarshalMsg +// |-----> (*) CanUnmarshalMsg +// |-----> (*) Msgsize +// |-----> (*) MsgIsZero +// |-----> DeadlineMaxSize() +// // actionType // |-----> MarshalMsg // |-----> CanMarshalMsg @@ -961,6 +971,123 @@ func ConsensusVersionViewMaxSize() (s int) { return } +// MarshalMsg implements msgp.Marshaler +func (z *Deadline) MarshalMsg(b []byte) (o []byte) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 2 + // string "Deadline" + o = append(o, 0x82, 0xa8, 0x44, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65) + o = msgp.AppendDuration(o, (*z).Deadline) + // string "Type" + o = append(o, 0xa4, 0x54, 0x79, 0x70, 0x65) + o = (*z).Type.MarshalMsg(o) + return +} + +func (_ *Deadline) CanMarshalMsg(z interface{}) bool { + _, ok := (z).(*Deadline) + return ok +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *Deadline) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 int + var zb0002 bool + zb0001, zb0002, bts, err = msgp.ReadMapHeaderBytes(bts) + if _, ok := err.(msgp.TypeError); ok { + zb0001, zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + if zb0001 > 0 { + zb0001-- + (*z).Deadline, bts, err = msgp.ReadDurationBytes(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "Deadline") + return + } + } + if zb0001 > 0 { + zb0001-- + bts, err = (*z).Type.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "Type") + return + } + } + if zb0001 > 0 { + err = msgp.ErrTooManyArrayFields(zb0001) + if err != nil { + err = msgp.WrapError(err, "struct-from-array") + return + } + } + } else { + if err != nil { + err = msgp.WrapError(err) + return + } + if zb0002 { + (*z) = Deadline{} + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch string(field) { + case "Deadline": + (*z).Deadline, bts, err = msgp.ReadDurationBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Deadline") + return + } + case "Type": + bts, err = (*z).Type.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Type") + return + } + default: + err = msgp.ErrNoField(string(field)) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + } + o = bts + return +} + +func (_ *Deadline) CanUnmarshalMsg(z interface{}) bool { + _, ok := (z).(*Deadline) + return ok +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *Deadline) Msgsize() (s int) { + s = 1 + 9 + msgp.DurationSize + 5 + (*z).Type.Msgsize() + return +} + +// MsgIsZero returns whether this is a zero value +func (z *Deadline) MsgIsZero() bool { + return ((*z).Deadline == 0) && ((*z).Type.MsgIsZero()) +} + +// MaxSize returns a maximum valid message size for this message type +func DeadlineMaxSize() (s int) { + s = 1 + 9 + msgp.DurationSize + 5 + timers.TimeoutMaxSize() + return +} + // MarshalMsg implements msgp.Marshaler func (z actionType) MarshalMsg(b []byte) (o []byte) { o = msgp.Require(b, z.Msgsize()) @@ -3756,7 +3883,13 @@ func (z *player) MarshalMsg(b []byte) (o []byte) { // map header, size 8 // string "Deadline" o = append(o, 0x88, 0xa8, 0x44, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65) - o = msgp.AppendDuration(o, (*z).Deadline) + // map header, size 2 + // string "Deadline" + o = append(o, 0x82, 0xa8, 0x44, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65) + o = msgp.AppendDuration(o, (*z).Deadline.Deadline) + // string "Type" + o = append(o, 0xa4, 0x54, 0x79, 0x70, 0x65) + o = (*z).Deadline.Type.MarshalMsg(o) // string "FastRecoveryDeadline" o = append(o, 0xb4, 0x46, 0x61, 0x73, 0x74, 0x52, 0x65, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x79, 0x44, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65) o = msgp.AppendDuration(o, (*z).FastRecoveryDeadline) @@ -3845,10 +3978,74 @@ func (z *player) UnmarshalMsg(bts []byte) (o []byte, err error) { } if zb0001 > 0 { zb0001-- - (*z).Deadline, bts, err = msgp.ReadDurationBytes(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "Deadline") - return + var zb0006 int + var zb0007 bool + zb0006, zb0007, bts, err = msgp.ReadMapHeaderBytes(bts) + if _, ok := err.(msgp.TypeError); ok { + zb0006, zb0007, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "Deadline") + return + } + if zb0006 > 0 { + zb0006-- + (*z).Deadline.Deadline, bts, err = msgp.ReadDurationBytes(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "Deadline", "struct-from-array", "Deadline") + return + } + } + if zb0006 > 0 { + zb0006-- + bts, err = (*z).Deadline.Type.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "Deadline", "struct-from-array", "Type") + return + } + } + if zb0006 > 0 { + err = msgp.ErrTooManyArrayFields(zb0006) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "Deadline", "struct-from-array") + return + } + } + } else { + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "Deadline") + return + } + if zb0007 { + (*z).Deadline = Deadline{} + } + for zb0006 > 0 { + zb0006-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "Deadline") + return + } + switch string(field) { + case "Deadline": + (*z).Deadline.Deadline, bts, err = msgp.ReadDurationBytes(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "Deadline", "Deadline") + return + } + case "Type": + bts, err = (*z).Deadline.Type.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "Deadline", "Type") + return + } + default: + err = msgp.ErrNoField(string(field)) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "Deadline") + return + } + } + } } } if zb0001 > 0 { @@ -3906,39 +4103,103 @@ func (z *player) UnmarshalMsg(bts []byte) (o []byte, err error) { } case "Period": { - var zb0006 uint64 - zb0006, bts, err = msgp.ReadUint64Bytes(bts) + var zb0008 uint64 + zb0008, bts, err = msgp.ReadUint64Bytes(bts) if err != nil { err = msgp.WrapError(err, "Period") return } - (*z).Period = period(zb0006) + (*z).Period = period(zb0008) } case "Step": { - var zb0007 uint64 - zb0007, bts, err = msgp.ReadUint64Bytes(bts) + var zb0009 uint64 + zb0009, bts, err = msgp.ReadUint64Bytes(bts) if err != nil { err = msgp.WrapError(err, "Step") return } - (*z).Step = step(zb0007) + (*z).Step = step(zb0009) } case "LastConcluding": { - var zb0008 uint64 - zb0008, bts, err = msgp.ReadUint64Bytes(bts) + var zb0010 uint64 + zb0010, bts, err = msgp.ReadUint64Bytes(bts) if err != nil { err = msgp.WrapError(err, "LastConcluding") return } - (*z).LastConcluding = step(zb0008) + (*z).LastConcluding = step(zb0010) } case "Deadline": - (*z).Deadline, bts, err = msgp.ReadDurationBytes(bts) - if err != nil { - err = msgp.WrapError(err, "Deadline") - return + var zb0011 int + var zb0012 bool + zb0011, zb0012, bts, err = msgp.ReadMapHeaderBytes(bts) + if _, ok := err.(msgp.TypeError); ok { + zb0011, zb0012, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Deadline") + return + } + if zb0011 > 0 { + zb0011-- + (*z).Deadline.Deadline, bts, err = msgp.ReadDurationBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Deadline", "struct-from-array", "Deadline") + return + } + } + if zb0011 > 0 { + zb0011-- + bts, err = (*z).Deadline.Type.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Deadline", "struct-from-array", "Type") + return + } + } + if zb0011 > 0 { + err = msgp.ErrTooManyArrayFields(zb0011) + if err != nil { + err = msgp.WrapError(err, "Deadline", "struct-from-array") + return + } + } + } else { + if err != nil { + err = msgp.WrapError(err, "Deadline") + return + } + if zb0012 { + (*z).Deadline = Deadline{} + } + for zb0011 > 0 { + zb0011-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err, "Deadline") + return + } + switch string(field) { + case "Deadline": + (*z).Deadline.Deadline, bts, err = msgp.ReadDurationBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Deadline", "Deadline") + return + } + case "Type": + bts, err = (*z).Deadline.Type.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Deadline", "Type") + return + } + default: + err = msgp.ErrNoField(string(field)) + if err != nil { + err = msgp.WrapError(err, "Deadline") + return + } + } + } } case "Napping": (*z).Napping, bts, err = msgp.ReadBoolBytes(bts) @@ -3978,18 +4239,18 @@ func (_ *player) CanUnmarshalMsg(z interface{}) bool { // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message func (z *player) Msgsize() (s int) { - s = 1 + 6 + (*z).Round.Msgsize() + 7 + msgp.Uint64Size + 5 + msgp.Uint64Size + 15 + msgp.Uint64Size + 9 + msgp.DurationSize + 8 + msgp.BoolSize + 21 + msgp.DurationSize + 8 + (*z).Pending.Msgsize() + s = 1 + 6 + (*z).Round.Msgsize() + 7 + msgp.Uint64Size + 5 + msgp.Uint64Size + 15 + msgp.Uint64Size + 9 + 1 + 9 + msgp.DurationSize + 5 + (*z).Deadline.Type.Msgsize() + 8 + msgp.BoolSize + 21 + msgp.DurationSize + 8 + (*z).Pending.Msgsize() return } // MsgIsZero returns whether this is a zero value func (z *player) MsgIsZero() bool { - return ((*z).Round.MsgIsZero()) && ((*z).Period == 0) && ((*z).Step == 0) && ((*z).LastConcluding == 0) && ((*z).Deadline == 0) && ((*z).Napping == false) && ((*z).FastRecoveryDeadline == 0) && ((*z).Pending.MsgIsZero()) + return ((*z).Round.MsgIsZero()) && ((*z).Period == 0) && ((*z).Step == 0) && ((*z).LastConcluding == 0) && (((*z).Deadline.Deadline == 0) && ((*z).Deadline.Type.MsgIsZero())) && ((*z).Napping == false) && ((*z).FastRecoveryDeadline == 0) && ((*z).Pending.MsgIsZero()) } // MaxSize returns a maximum valid message size for this message type func PlayerMaxSize() (s int) { - s = 1 + 6 + basics.RoundMaxSize() + 7 + msgp.Uint64Size + 5 + msgp.Uint64Size + 15 + msgp.Uint64Size + 9 + msgp.DurationSize + 8 + msgp.BoolSize + 21 + msgp.DurationSize + 8 + ProposalTableMaxSize() + s = 1 + 6 + basics.RoundMaxSize() + 7 + msgp.Uint64Size + 5 + msgp.Uint64Size + 15 + msgp.Uint64Size + 9 + 1 + 9 + msgp.DurationSize + 5 + timers.TimeoutMaxSize() + 8 + msgp.BoolSize + 21 + msgp.DurationSize + 8 + ProposalTableMaxSize() return } diff --git a/agreement/msgp_gen_test.go b/agreement/msgp_gen_test.go index 99053ca4c3..2d3952add9 100644 --- a/agreement/msgp_gen_test.go +++ b/agreement/msgp_gen_test.go @@ -134,6 +134,66 @@ func BenchmarkUnmarshalConsensusVersionView(b *testing.B) { } } +func TestMarshalUnmarshalDeadline(t *testing.T) { + partitiontest.PartitionTest(t) + v := Deadline{} + bts := v.MarshalMsg(nil) + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func TestRandomizedEncodingDeadline(t *testing.T) { + protocol.RunEncodingTest(t, &Deadline{}) +} + +func BenchmarkMarshalMsgDeadline(b *testing.B) { + v := Deadline{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgDeadline(b *testing.B) { + v := Deadline{} + bts := make([]byte, 0, v.Msgsize()) + bts = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalDeadline(b *testing.B) { + v := Deadline{} + bts := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + func TestMarshalUnmarshalblockAssembler(t *testing.T) { partitiontest.PartitionTest(t) v := blockAssembler{} diff --git a/agreement/persistence_test.go b/agreement/persistence_test.go index fbd9323b09..ff4b90bb5b 100644 --- a/agreement/persistence_test.go +++ b/agreement/persistence_test.go @@ -38,7 +38,7 @@ func TestAgreementSerialization(t *testing.T) { // todo : we need to deserialize some more meaningful state. clock := timers.MakeMonotonicClock(time.Date(2015, 1, 2, 5, 6, 7, 8, time.UTC)) - status := player{Round: 350, Step: soft, Deadline: time.Duration(23) * time.Second} + status := player{Round: 350, Step: soft, Deadline: Deadline{Deadline: time.Duration(23) * time.Second, Type: timers.Deadline}} router := makeRootRouter(status) a := []action{checkpointAction{}, disconnectAction(messageEvent{}, nil)} @@ -59,7 +59,7 @@ func BenchmarkAgreementSerialization(b *testing.B) { b.SkipNow() clock := timers.MakeMonotonicClock(time.Date(2015, 1, 2, 5, 6, 7, 8, time.UTC)) - status := player{Round: 350, Step: soft, Deadline: time.Duration(23) * time.Second} + status := player{Round: 350, Step: soft, Deadline: Deadline{Deadline: time.Duration(23) * time.Second, Type: timers.Deadline}} router := makeRootRouter(status) a := []action{} @@ -74,7 +74,7 @@ func BenchmarkAgreementDeserialization(b *testing.B) { b.SkipNow() clock := timers.MakeMonotonicClock(time.Date(2015, 1, 2, 5, 6, 7, 8, time.UTC)) - status := player{Round: 350, Step: soft, Deadline: time.Duration(23) * time.Second} + status := player{Round: 350, Step: soft, Deadline: Deadline{Deadline: time.Duration(23) * time.Second, Type: timers.Deadline}} router := makeRootRouter(status) a := []action{} @@ -244,7 +244,7 @@ func TestDecodeFailures(t *testing.T) { clock := timers.MakeMonotonicClock(time.Date(2015, 1, 2, 5, 6, 7, 8, time.UTC)) ce := clock.Encode() log := makeServiceLogger(logging.Base()) - player := player{Round: 350, Step: soft, Deadline: time.Duration(23) * time.Second} + player := player{Round: 350, Step: soft, Deadline: Deadline{Deadline: time.Duration(23) * time.Second, Type: timers.Deadline}} router := makeRootRouter(player) pe := protocol.Encode(&player) re := protocol.Encode(&router) diff --git a/agreement/player.go b/agreement/player.go index 1ae552b0b1..a539e936cb 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -21,8 +21,15 @@ import ( "github.com/algorand/go-algorand/config" "github.com/algorand/go-algorand/protocol" + "github.com/algorand/go-algorand/util/timers" ) +type Deadline struct { + _struct struct{} `codec:","` + Deadline time.Duration + Type timers.Timeout +} + // The player implements the top-level state machine functionality of the // agreement protocol. type player struct { @@ -40,7 +47,7 @@ type player struct { // Deadline contains the time of the next timeout expected by the player // state machine (relevant to the start of the current period). - Deadline time.Duration + Deadline Deadline // Napping is set when the player is expecting a random timeout (i.e., // to determine when the player chooses to send a next-vote). Napping bool @@ -111,7 +118,8 @@ func (p *player) handle(r routerHandle, e event) []action { delta := time.Duration(e.RandomEntropy % uint64(upper-lower)) p.Napping = true - p.Deadline = lower + delta + p.Deadline.Deadline = lower + delta + p.Deadline.Type = timers.Deadline return actions } case roundInterruptionEvent: @@ -145,7 +153,8 @@ func (p *player) handleFastTimeout(r routerHandle, e timeoutEvent) []action { func (p *player) issueSoftVote(r routerHandle) (actions []action) { defer func() { - p.Deadline = deadlineTimeout + p.Deadline.Deadline = deadlineTimeout + p.Deadline.Type = timers.Deadline }() e := r.dispatch(*p, proposalFrozenEvent{}, proposalMachinePeriod, p.Round, p.Period, 0) @@ -213,7 +222,8 @@ func (p *player) issueNextVote(r routerHandle) []action { _, upper := p.Step.nextVoteRanges() p.Napping = false - p.Deadline = upper + p.Deadline.Deadline = upper + p.Deadline.Type = timers.Deadline return actions } @@ -327,7 +337,8 @@ func (p *player) enterPeriod(r routerHandle, source thresholdEvent, target perio p.Step = soft p.Napping = false p.FastRecoveryDeadline = 0 // set immediately - p.Deadline = FilterTimeout(target, source.Proto) + p.Deadline.Deadline = FilterTimeout(target, source.Proto) + p.Deadline.Type = timers.Filter // update tracer state to match player r.t.setMetadata(tracerMetadata{p.Round, p.Period, p.Step}) @@ -375,11 +386,14 @@ func (p *player) enterRound(r routerHandle, source event, target round) []action switch source := source.(type) { case roundInterruptionEvent: - p.Deadline = FilterTimeout(0, source.Proto.Version) + p.Deadline.Deadline = FilterTimeout(0, source.Proto.Version) + p.Deadline.Type = timers.Filter case thresholdEvent: - p.Deadline = FilterTimeout(0, source.Proto) + p.Deadline.Deadline = FilterTimeout(0, source.Proto) + p.Deadline.Type = timers.Filter case filterableMessageEvent: - p.Deadline = FilterTimeout(0, source.Proto.Version) + p.Deadline.Deadline = FilterTimeout(0, source.Proto.Version) + p.Deadline.Type = timers.Filter } // update tracer state to match player diff --git a/agreement/player_test.go b/agreement/player_test.go index 75987c2ed0..1cd0ed3700 100644 --- a/agreement/player_test.go +++ b/agreement/player_test.go @@ -28,6 +28,7 @@ import ( "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/protocol" "github.com/algorand/go-algorand/test/partitiontest" + "github.com/algorand/go-algorand/util/timers" ) var playerTracer tracer @@ -500,7 +501,7 @@ func TestPlayerLateBlockProposalPeriod0(t *testing.T) { func setupP(t *testing.T, r round, p period, s step) (plyr *player, pMachine ioAutomata, helper *voteMakerHelper) { // Set up a composed test machine starting at specified rps - rRouter := makeRootRouter(player{Round: r, Period: p, Step: s, Deadline: FilterTimeout(p, protocol.ConsensusCurrentVersion)}) + rRouter := makeRootRouter(player{Round: r, Period: p, Step: s, Deadline: Deadline{Deadline: FilterTimeout(p, protocol.ConsensusCurrentVersion), Type: timers.Filter}}) concreteMachine := ioAutomataConcretePlayer{rootRouter: &rRouter} plyr = concreteMachine.underlying() pMachine = &concreteMachine diff --git a/agreement/service.go b/agreement/service.go index 3cac126193..4f68c559f4 100644 --- a/agreement/service.go +++ b/agreement/service.go @@ -80,7 +80,7 @@ type parameters Parameters // externalDemuxSignals used to syncronize the external signals that goes to the demux with the main loop. type externalDemuxSignals struct { - Deadline time.Duration + Deadline Deadline FastRecoveryDeadline time.Duration CurrentRound round } @@ -212,7 +212,7 @@ func (s *Service) mainLoop(input <-chan externalEvent, output chan<- []action, r s.log.Errorf("unable to retrieve consensus version for round %d, defaulting to binary consensus version", nextRound) nextVersion = protocol.ConsensusCurrentVersion } - status = player{Round: nextRound, Step: soft, Deadline: FilterTimeout(0, nextVersion)} + status = player{Round: nextRound, Step: soft, Deadline: Deadline{Deadline: FilterTimeout(0, nextVersion), Type: timers.Filter}} router = makeRootRouter(status) a1 := pseudonodeAction{T: assemble, Round: s.Ledger.NextRound()} diff --git a/agreement/service_test.go b/agreement/service_test.go index 6093d402e5..e7fea84700 100644 --- a/agreement/service_test.go +++ b/agreement/service_test.go @@ -103,6 +103,22 @@ func (c *testingClock) prepareToFire() { c.monitor.inc(clockCoserviceType) } +func (c *testingClock) fireType(timeoutType timers.Timeout) { + c.mu.Lock() + defer c.mu.Unlock() + + if _, ok := c.TA[timeoutType]; !ok { + c.TA[timeoutType] = make(map[time.Duration]chan time.Time) + } + + for d := range c.TA[timeoutType] { + if c.TA[timeoutType][d] == nil { + c.TA[timeoutType][d] = make(chan time.Time) + } + close(c.TA[timeoutType][d]) + } +} + func (c *testingClock) fire(d time.Duration, timeoutType timers.Timeout) { c.mu.Lock() defer c.mu.Unlock() @@ -849,8 +865,19 @@ func triggerGlobalTimeout(d time.Duration, timeoutType timers.Timeout, clocks [] activityMonitor.waitForQuiet() } +func triggerGlobalTimeoutType(timeoutType timers.Timeout, clocks []timers.Clock, activityMonitor *activityMonitor) { + for i := range clocks { + clocks[i].(*testingClock).prepareToFire() + } + for i := range clocks { + clocks[i].(*testingClock).fireType(timeoutType) + } + activityMonitor.waitForActivity() + activityMonitor.waitForQuiet() +} + func runRound(clocks []timers.Clock, activityMonitor *activityMonitor, zeroes uint, filterTimeout time.Duration) (newzeroes uint) { - triggerGlobalTimeout(filterTimeout, clocks, activityMonitor) + triggerGlobalTimeout(filterTimeout, timers.Filter, clocks, activityMonitor) return expectNewPeriod(clocks, zeroes) } @@ -1047,23 +1074,23 @@ func TestAgreementFastRecoveryDownEarly(t *testing.T) { baseNetwork.dropAllSoftVotes() baseNetwork.dropAllSlowNextVotes() - triggerGlobalTimeout(FilterTimeout(0, version), clocks, activityMonitor) + triggerGlobalTimeout(FilterTimeout(0, version), timers.Filter, clocks, activityMonitor) zeroes = expectNoNewPeriod(clocks, zeroes) - triggerGlobalTimeout(deadlineTimeout, clocks, activityMonitor) + triggerGlobalTimeoutType(timers.Deadline, clocks, activityMonitor) zeroes = expectNoNewPeriod(clocks, zeroes) - triggerGlobalTimeout(0, clocks, activityMonitor) // activates fast partition recovery timer + triggerGlobalTimeout(0, timers.Fast, clocks, activityMonitor) // activates fast partition recovery timer zeroes = expectNoNewPeriod(clocks, zeroes) - triggerGlobalTimeout(firstFPR, clocks, activityMonitor) + triggerGlobalTimeout(firstFPR, timers.Fast, clocks, activityMonitor) zeroes = expectNewPeriod(clocks, zeroes) } // terminate on period 1 { baseNetwork.repairAll() - triggerGlobalTimeout(FilterTimeout(1, version), clocks, activityMonitor) + triggerGlobalTimeout(FilterTimeout(1, version), timers.Filter, clocks, activityMonitor) zeroes = expectNewPeriod(clocks, zeroes) } @@ -1104,13 +1131,13 @@ func TestAgreementFastRecoveryDownMiss(t *testing.T) { { // fail all steps baseNetwork.dropAllVotes() - triggerGlobalTimeout(FilterTimeout(0, version), clocks, activityMonitor) + triggerGlobalTimeout(FilterTimeout(0, version), timers.Filter, clocks, activityMonitor) zeroes = expectNoNewPeriod(clocks, zeroes) - triggerGlobalTimeout(deadlineTimeout, clocks, activityMonitor) + triggerGlobalTimeout(deadlineTimeout, timers.Deadline, clocks, activityMonitor) zeroes = expectNoNewPeriod(clocks, zeroes) - triggerGlobalTimeout(0, clocks, activityMonitor) // activates fast partition recovery timer + triggerGlobalTimeout(0, timers.Fast, clocks, activityMonitor) // activates fast partition recovery timer zeroes = expectNoNewPeriod(clocks, zeroes) firstClocks := clocks[:4] @@ -1120,7 +1147,7 @@ func TestAgreementFastRecoveryDownMiss(t *testing.T) { firstClocks[i].(*testingClock).prepareToFire() } for i := range firstClocks { - firstClocks[i].(*testingClock).fire(firstFPR) + firstClocks[i].(*testingClock).fire(firstFPR, timers.Fast) } activityMonitor.waitForActivity() activityMonitor.waitForQuiet() @@ -1131,20 +1158,20 @@ func TestAgreementFastRecoveryDownMiss(t *testing.T) { restClocks[i].(*testingClock).prepareToFire() } for i := range restClocks { - restClocks[i].(*testingClock).fire(firstFPR) + restClocks[i].(*testingClock).fire(firstFPR, timers.Fast) } activityMonitor.waitForActivity() activityMonitor.waitForQuiet() zeroes = expectNoNewPeriod(clocks, zeroes) - triggerGlobalTimeout(secondFPR, clocks, activityMonitor) + triggerGlobalTimeout(secondFPR, timers.Fast, clocks, activityMonitor) zeroes = expectNewPeriod(clocks, zeroes) } // terminate on period 1 { baseNetwork.repairAll() - triggerGlobalTimeout(FilterTimeout(1, version), clocks, activityMonitor) + triggerGlobalTimeout(FilterTimeout(1, version), timers.Filter, clocks, activityMonitor) zeroes = expectNewPeriod(clocks, zeroes) } @@ -1187,7 +1214,7 @@ func TestAgreementFastRecoveryLate(t *testing.T) { pocket := make(chan multicastParams, 100) closeFn := baseNetwork.pocketAllCertVotes(pocket) baseNetwork.dropAllSlowNextVotes() - triggerGlobalTimeout(FilterTimeout(0, version), clocks, activityMonitor) + triggerGlobalTimeout(FilterTimeout(0, version), timers.Filter, clocks, activityMonitor) zeroes = expectNoNewPeriod(clocks, zeroes) closeFn() @@ -1208,10 +1235,10 @@ func TestAgreementFastRecoveryLate(t *testing.T) { } } - triggerGlobalTimeout(deadlineTimeout, clocks, activityMonitor) + triggerGlobalTimeout(deadlineTimeout, timers.Deadline, clocks, activityMonitor) zeroes = expectNoNewPeriod(clocks, zeroes) - triggerGlobalTimeout(0, clocks, activityMonitor) // activates fast partition recovery timer + triggerGlobalTimeout(0, timers.Fast, clocks, activityMonitor) // activates fast partition recovery timer zeroes = expectNoNewPeriod(clocks, zeroes) baseNetwork.dropAllVotes() @@ -1222,7 +1249,7 @@ func TestAgreementFastRecoveryLate(t *testing.T) { firstClocks[i].(*testingClock).prepareToFire() } for i := range firstClocks { - firstClocks[i].(*testingClock).fire(firstFPR) + firstClocks[i].(*testingClock).fire(firstFPR, timers.Fast) } activityMonitor.waitForActivity() activityMonitor.waitForQuiet() @@ -1233,20 +1260,20 @@ func TestAgreementFastRecoveryLate(t *testing.T) { restClocks[i].(*testingClock).prepareToFire() } for i := range restClocks { - restClocks[i].(*testingClock).fire(firstFPR) + restClocks[i].(*testingClock).fire(firstFPR, timers.Fast) } activityMonitor.waitForActivity() activityMonitor.waitForQuiet() zeroes = expectNoNewPeriod(clocks, zeroes) - triggerGlobalTimeout(secondFPR, clocks, activityMonitor) + triggerGlobalTimeout(secondFPR, timers.Fast, clocks, activityMonitor) zeroes = expectNewPeriod(clocks, zeroes) } // terminate on period 1 { baseNetwork.repairAll() - triggerGlobalTimeout(FilterTimeout(1, version), clocks, activityMonitor) + triggerGlobalTimeout(FilterTimeout(1, version), timers.Filter, clocks, activityMonitor) zeroes = expectNewPeriod(clocks, zeroes) } @@ -1300,7 +1327,7 @@ func TestAgreementFastRecoveryRedo(t *testing.T) { pocket := make(chan multicastParams, 100) closeFn := baseNetwork.pocketAllCertVotes(pocket) baseNetwork.dropAllSlowNextVotes() - triggerGlobalTimeout(FilterTimeout(0, version), clocks, activityMonitor) + triggerGlobalTimeout(FilterTimeout(0, version), timers.Filter, clocks, activityMonitor) zeroes = expectNoNewPeriod(clocks, zeroes) closeFn() @@ -1321,10 +1348,10 @@ func TestAgreementFastRecoveryRedo(t *testing.T) { } } - triggerGlobalTimeout(deadlineTimeout, clocks, activityMonitor) + triggerGlobalTimeout(deadlineTimeout, timers.Deadline, clocks, activityMonitor) zeroes = expectNoNewPeriod(clocks, zeroes) - triggerGlobalTimeout(0, clocks, activityMonitor) // activates fast partition recovery timer + triggerGlobalTimeout(0, timers.Fast, clocks, activityMonitor) // activates fast partition recovery timer zeroes = expectNoNewPeriod(clocks, zeroes) baseNetwork.dropAllVotes() @@ -1335,7 +1362,7 @@ func TestAgreementFastRecoveryRedo(t *testing.T) { firstClocks[i].(*testingClock).prepareToFire() } for i := range firstClocks { - firstClocks[i].(*testingClock).fire(firstFPR) + firstClocks[i].(*testingClock).fire(firstFPR, timers.Fast) } activityMonitor.waitForActivity() activityMonitor.waitForQuiet() @@ -1346,26 +1373,26 @@ func TestAgreementFastRecoveryRedo(t *testing.T) { restClocks[i].(*testingClock).prepareToFire() } for i := range restClocks { - restClocks[i].(*testingClock).fire(firstFPR) + restClocks[i].(*testingClock).fire(firstFPR, timers.Fast) } activityMonitor.waitForActivity() activityMonitor.waitForQuiet() zeroes = expectNoNewPeriod(clocks, zeroes) - triggerGlobalTimeout(secondFPR, clocks, activityMonitor) + triggerGlobalTimeout(secondFPR, timers.Fast, clocks, activityMonitor) zeroes = expectNewPeriod(clocks, zeroes) } // fail period 1 with value again { baseNetwork.dropAllVotes() - triggerGlobalTimeout(FilterTimeout(1, version), clocks, activityMonitor) + triggerGlobalTimeout(FilterTimeout(1, version), timers.Filter, clocks, activityMonitor) zeroes = expectNoNewPeriod(clocks, zeroes) - triggerGlobalTimeout(deadlineTimeout, clocks, activityMonitor) + triggerGlobalTimeout(deadlineTimeout, timers.Deadline, clocks, activityMonitor) zeroes = expectNoNewPeriod(clocks, zeroes) - triggerGlobalTimeout(0, clocks, activityMonitor) // activates fast partition recovery timer + triggerGlobalTimeout(0, timers.Fast, clocks, activityMonitor) // activates fast partition recovery timer zeroes = expectNoNewPeriod(clocks, zeroes) baseNetwork.dropAllVotes() @@ -1376,7 +1403,7 @@ func TestAgreementFastRecoveryRedo(t *testing.T) { firstClocks[i].(*testingClock).prepareToFire() } for i := range firstClocks { - firstClocks[i].(*testingClock).fire(firstFPR) + firstClocks[i].(*testingClock).fire(firstFPR, timers.Fast) } activityMonitor.waitForActivity() activityMonitor.waitForQuiet() @@ -1387,20 +1414,20 @@ func TestAgreementFastRecoveryRedo(t *testing.T) { restClocks[i].(*testingClock).prepareToFire() } for i := range restClocks { - restClocks[i].(*testingClock).fire(firstFPR) + restClocks[i].(*testingClock).fire(firstFPR, timers.Fast) } activityMonitor.waitForActivity() activityMonitor.waitForQuiet() zeroes = expectNoNewPeriod(clocks, zeroes) - triggerGlobalTimeout(secondFPR, clocks, activityMonitor) + triggerGlobalTimeout(secondFPR, timers.Fast, clocks, activityMonitor) zeroes = expectNewPeriod(clocks, zeroes) } // terminate on period 2 { baseNetwork.repairAll() - triggerGlobalTimeout(FilterTimeout(2, version), clocks, activityMonitor) + triggerGlobalTimeout(FilterTimeout(2, version), timers.Filter, clocks, activityMonitor) zeroes = expectNewPeriod(clocks, zeroes) } @@ -1451,26 +1478,26 @@ func TestAgreementBlockReplayBug_b29ea57(t *testing.T) { // fail period 0 { baseNetwork.dropAllSoftVotes() - triggerGlobalTimeout(FilterTimeout(0, version), clocks, activityMonitor) + triggerGlobalTimeout(FilterTimeout(0, version), timers.Filter, clocks, activityMonitor) zeroes = expectNoNewPeriod(clocks, zeroes) - triggerGlobalTimeout(deadlineTimeout, clocks, activityMonitor) + triggerGlobalTimeout(deadlineTimeout, timers.Deadline, clocks, activityMonitor) zeroes = expectNewPeriod(clocks, zeroes) } // fail period 1 on bottom with block { - triggerGlobalTimeout(FilterTimeout(1, version), clocks, activityMonitor) + triggerGlobalTimeout(FilterTimeout(1, version), timers.Filter, clocks, activityMonitor) zeroes = expectNoNewPeriod(clocks, zeroes) - triggerGlobalTimeout(deadlineTimeout, clocks, activityMonitor) + triggerGlobalTimeout(deadlineTimeout, timers.Deadline, clocks, activityMonitor) zeroes = expectNewPeriod(clocks, zeroes) } // terminate on period 2 { baseNetwork.repairAll() - triggerGlobalTimeout(FilterTimeout(2, version), clocks, activityMonitor) + triggerGlobalTimeout(FilterTimeout(2, version), timers.Filter, clocks, activityMonitor) zeroes = expectNewPeriod(clocks, zeroes) } @@ -1511,12 +1538,12 @@ func TestAgreementLateCertBug(t *testing.T) { pocket := make(chan multicastParams, 100) { closeFn := baseNetwork.pocketAllCertVotes(pocket) - triggerGlobalTimeout(FilterTimeout(0, version), clocks, activityMonitor) + triggerGlobalTimeout(FilterTimeout(0, version), timers.Filter, clocks, activityMonitor) zeroes = expectNoNewPeriod(clocks, zeroes) closeFn() baseNetwork.repairAll() - triggerGlobalTimeout(deadlineTimeout, clocks, activityMonitor) + triggerGlobalTimeout(deadlineTimeout, timers.Filter, clocks, activityMonitor) zeroes = expectNewPeriod(clocks, zeroes) } @@ -1571,7 +1598,7 @@ func TestAgreementRecoverGlobalStartingValue(t *testing.T) { pocket := make(chan multicastParams, 100) closeFn := baseNetwork.pocketAllCertVotes(pocket) - triggerGlobalTimeout(FilterTimeout(0, version), clocks, activityMonitor) + triggerGlobalTimeout(FilterTimeout(0, version), timers.Filter, clocks, activityMonitor) zeroes = expectNoNewPeriod(clocks, zeroes) closeFn() @@ -1592,7 +1619,7 @@ func TestAgreementRecoverGlobalStartingValue(t *testing.T) { } } - triggerGlobalTimeout(deadlineTimeout, clocks, activityMonitor) + triggerGlobalTimeout(deadlineTimeout, timers.Deadline, clocks, activityMonitor) zeroes = expectNewPeriod(clocks, zeroes) require.Equal(t, 4, int(zeroes)) } @@ -1602,7 +1629,7 @@ func TestAgreementRecoverGlobalStartingValue(t *testing.T) { pocket := make(chan multicastParams, 100) closeFn := baseNetwork.pocketAllCertVotes(pocket) - triggerGlobalTimeout(FilterTimeout(1, version), clocks, activityMonitor) + triggerGlobalTimeout(FilterTimeout(1, version), timers.Filter, clocks, activityMonitor) zeroes = expectNoNewPeriod(clocks, zeroes) closeFn() @@ -1619,7 +1646,7 @@ func TestAgreementRecoverGlobalStartingValue(t *testing.T) { } } - triggerGlobalTimeout(deadlineTimeout, clocks, activityMonitor) + triggerGlobalTimeout(deadlineTimeout, timers.Deadline, clocks, activityMonitor) zeroes = expectNewPeriod(clocks, zeroes) require.Equal(t, 5, int(zeroes)) } @@ -1628,7 +1655,7 @@ func TestAgreementRecoverGlobalStartingValue(t *testing.T) { // todo: make more transparent, I want to kow what v we agreed on { baseNetwork.repairAll() - triggerGlobalTimeout(FilterTimeout(2, version), clocks, activityMonitor) + triggerGlobalTimeout(FilterTimeout(2, version), timers.Filter, clocks, activityMonitor) zeroes = expectNewPeriod(clocks, zeroes) require.Equal(t, 6, int(zeroes)) } @@ -1670,7 +1697,7 @@ func TestAgreementRecoverGlobalStartingValueBadProposal(t *testing.T) { { pocket := make(chan multicastParams, 100) closeFn := baseNetwork.pocketAllCertVotes(pocket) - triggerGlobalTimeout(FilterTimeout(0, version), clocks, activityMonitor) + triggerGlobalTimeout(FilterTimeout(0, version), timers.Filter, clocks, activityMonitor) zeroes = expectNoNewPeriod(clocks, zeroes) closeFn() @@ -1697,7 +1724,7 @@ func TestAgreementRecoverGlobalStartingValueBadProposal(t *testing.T) { } return params }) - triggerGlobalTimeout(deadlineTimeout, clocks, activityMonitor) + triggerGlobalTimeout(deadlineTimeout, timers.Deadline, clocks, activityMonitor) zeroes = expectNewPeriod(clocks, zeroes) require.Equal(t, 4, int(zeroes)) } @@ -1707,7 +1734,7 @@ func TestAgreementRecoverGlobalStartingValueBadProposal(t *testing.T) { baseNetwork.repairAll() pocket := make(chan multicastParams, 100) closeFn := baseNetwork.pocketAllCertVotes(pocket) - triggerGlobalTimeout(FilterTimeout(1, version), clocks, activityMonitor) + triggerGlobalTimeout(FilterTimeout(1, version), timers.Filter, clocks, activityMonitor) zeroes = expectNoNewPeriod(clocks, zeroes) closeFn() @@ -1723,7 +1750,7 @@ func TestAgreementRecoverGlobalStartingValueBadProposal(t *testing.T) { panic(errstr) } } - triggerGlobalTimeout(deadlineTimeout, clocks, activityMonitor) + triggerGlobalTimeout(deadlineTimeout, timers.Deadline, clocks, activityMonitor) zeroes = expectNewPeriod(clocks, zeroes) } @@ -1731,7 +1758,7 @@ func TestAgreementRecoverGlobalStartingValueBadProposal(t *testing.T) { // Finish in period 2 { baseNetwork.repairAll() - triggerGlobalTimeout(FilterTimeout(2, version), clocks, activityMonitor) + triggerGlobalTimeout(FilterTimeout(2, version), timers.Filter, clocks, activityMonitor) zeroes = expectNewPeriod(clocks, zeroes) require.Equal(t, 6, int(zeroes)) } @@ -1773,7 +1800,7 @@ func TestAgreementRecoverBothVAndBotQuorums(t *testing.T) { { pocket := make(chan multicastParams, 100) closeFn := baseNetwork.pocketAllSoftVotes(pocket) - triggerGlobalTimeout(FilterTimeout(0, version), clocks, activityMonitor) + triggerGlobalTimeout(FilterTimeout(0, version), timers.Filter, clocks, activityMonitor) zeroes = expectNoNewPeriod(clocks, zeroes) closeFn() pocketedSoft := make([]multicastParams, len(pocket)) @@ -1798,7 +1825,7 @@ func TestAgreementRecoverBothVAndBotQuorums(t *testing.T) { } // generate a bottom quorum; let only one node see it. baseNetwork.crown(0) - triggerGlobalTimeout(deadlineTimeout, clocks, activityMonitor) + triggerGlobalTimeout(deadlineTimeout, timers.Deadline, clocks, activityMonitor) if clocks[0].(*testingClock).zeroes != zeroes+1 { errstr := fmt.Sprintf("node 0 did not enter new period from bot quorum") panic(errstr) @@ -1817,12 +1844,12 @@ func TestAgreementRecoverBothVAndBotQuorums(t *testing.T) { // actually create the value quorum _, upper := (next).nextVoteRanges() - triggerGlobalTimeout(upper, clocks[1:], activityMonitor) // activates next timers + triggerGlobalTimeout(upper, timers.Deadline, clocks[1:], activityMonitor) // activates next timers zeroes = expectNoNewPeriod(clocks[1:], zeroes) lower, upper := (next + 1).nextVoteRanges() delta := time.Duration(testingRand{}.Uint64() % uint64(upper-lower)) - triggerGlobalTimeout(lower+delta, clocks[1:], activityMonitor) + triggerGlobalTimeout(lower+delta, timers.Deadline, clocks[1:], activityMonitor) zeroes = expectNewPeriod(clocks, zeroes) require.Equal(t, 4, int(zeroes)) } @@ -1832,7 +1859,7 @@ func TestAgreementRecoverBothVAndBotQuorums(t *testing.T) { baseNetwork.repairAll() pocket := make(chan multicastParams, 100) closeFn := baseNetwork.pocketAllCertVotes(pocket) - triggerGlobalTimeout(FilterTimeout(1, version), clocks, activityMonitor) + triggerGlobalTimeout(FilterTimeout(1, version), timers.Filter, clocks, activityMonitor) zeroes = expectNoNewPeriod(clocks, zeroes) closeFn() @@ -1849,14 +1876,14 @@ func TestAgreementRecoverBothVAndBotQuorums(t *testing.T) { } } - triggerGlobalTimeout(deadlineTimeout, clocks, activityMonitor) + triggerGlobalTimeout(deadlineTimeout, timers.Deadline, clocks, activityMonitor) zeroes = expectNewPeriod(clocks, zeroes) } // Finish in period 2 { baseNetwork.repairAll() - triggerGlobalTimeout(FilterTimeout(2, version), clocks, activityMonitor) + triggerGlobalTimeout(FilterTimeout(2, version), timers.Filter, clocks, activityMonitor) zeroes = expectNewPeriod(clocks, zeroes) require.Equal(t, 6, int(zeroes)) } @@ -1897,13 +1924,13 @@ func TestAgreementSlowPayloadsPreDeadline(t *testing.T) { pocket := make(chan multicastParams, 100) closeFn := baseNetwork.pocketAllCompound(pocket) // (takes effect next round) { - triggerGlobalTimeout(FilterTimeout(0, version), clocks, activityMonitor) + triggerGlobalTimeout(FilterTimeout(0, version), timers.Filter, clocks, activityMonitor) zeroes = expectNewPeriod(clocks, zeroes) } // run round with late payload { - triggerGlobalTimeout(FilterTimeout(0, version), clocks, activityMonitor) + triggerGlobalTimeout(FilterTimeout(0, version), timers.Filter, clocks, activityMonitor) zeroes = expectNoNewPeriod(clocks, zeroes) // release payloads; expect new round @@ -1955,15 +1982,15 @@ func TestAgreementSlowPayloadsPostDeadline(t *testing.T) { pocket := make(chan multicastParams, 100) closeFn := baseNetwork.pocketAllCompound(pocket) // (takes effect next round) { - triggerGlobalTimeout(FilterTimeout(0, version), clocks, activityMonitor) + triggerGlobalTimeout(FilterTimeout(0, version), timers.Filter, clocks, activityMonitor) zeroes = expectNewPeriod(clocks, zeroes) } // force network into period 1 by delaying proposals { - triggerGlobalTimeout(FilterTimeout(0, version), clocks, activityMonitor) + triggerGlobalTimeout(FilterTimeout(0, version), timers.Filter, clocks, activityMonitor) zeroes = expectNoNewPeriod(clocks, zeroes) - triggerGlobalTimeout(deadlineTimeout, clocks, activityMonitor) + triggerGlobalTimeout(deadlineTimeout, timers.Deadline, clocks, activityMonitor) zeroes = expectNewPeriod(clocks, zeroes) } @@ -1980,7 +2007,7 @@ func TestAgreementSlowPayloadsPostDeadline(t *testing.T) { activityMonitor.waitForQuiet() zeroes = expectNoNewPeriod(clocks, zeroes) - triggerGlobalTimeout(FilterTimeout(1, version), clocks, activityMonitor) + triggerGlobalTimeout(FilterTimeout(1, version), timers.Filter, clocks, activityMonitor) zeroes = expectNewPeriod(clocks, zeroes) } @@ -2020,11 +2047,11 @@ func TestAgreementLargePeriods(t *testing.T) { for p := 0; p < 60; p++ { { baseNetwork.partition(0, 1, 2) - triggerGlobalTimeout(FilterTimeout(period(p), version), clocks, activityMonitor) + triggerGlobalTimeout(FilterTimeout(period(p), version), timers.Filter, clocks, activityMonitor) zeroes = expectNoNewPeriod(clocks, zeroes) baseNetwork.repairAll() - triggerGlobalTimeout(deadlineTimeout, clocks, activityMonitor) + triggerGlobalTimeout(deadlineTimeout, timers.Deadline, clocks, activityMonitor) zeroes = expectNewPeriod(clocks, zeroes) require.Equal(t, 4+p, int(zeroes)) } @@ -2032,7 +2059,7 @@ func TestAgreementLargePeriods(t *testing.T) { // terminate { - triggerGlobalTimeout(FilterTimeout(60, version), clocks, activityMonitor) + triggerGlobalTimeout(FilterTimeout(60, version), timers.Filter, clocks, activityMonitor) zeroes = expectNewPeriod(clocks, zeroes) } @@ -2119,13 +2146,13 @@ func TestAgreementRegression_WrongPeriodPayloadVerificationCancellation_8ba23942 ch := validator.suspend() closeFn := baseNetwork.pocketAllCompound(pocket0) // (takes effect next round) { - triggerGlobalTimeout(FilterTimeout(0, version), clocks, activityMonitor) + triggerGlobalTimeout(FilterTimeout(0, version), timers.Filter, clocks, activityMonitor) zeroes = expectNewPeriod(clocks, zeroes) } // force network into period 1 by failing period 0, entering with bottom and no soft threshold (to prevent proposal value pinning) baseNetwork.dropAllSoftVotes() - triggerGlobalTimeout(FilterTimeout(0, version), clocks, activityMonitor) + triggerGlobalTimeout(FilterTimeout(0, version), timers.Filter, clocks, activityMonitor) zeroes = expectNoNewPeriod(clocks, zeroes) // resume delivery of payloads in following period @@ -2136,7 +2163,7 @@ func TestAgreementRegression_WrongPeriodPayloadVerificationCancellation_8ba23942 // release proposed blocks in a controlled manner to prevent oversubscription of verification pocket1 := make(chan multicastParams, 100) closeFn = baseNetwork.pocketAllCompound(pocket1) - triggerGlobalTimeout(deadlineTimeout, clocks, activityMonitor) + triggerGlobalTimeout(deadlineTimeout, timers.Deadline, clocks, activityMonitor) baseNetwork.repairAll() close(pocket1) { @@ -2320,7 +2347,7 @@ func TestAgreementCertificateDoesNotStallSingleRelay(t *testing.T) { return params }) // And with some hypothetical second relay the network achieves consensus on a certificate and block. - triggerGlobalTimeout(FilterTimeout(0, version), clocks, activityMonitor) + triggerGlobalTimeout(FilterTimeout(0, version), timers.Filter, clocks, activityMonitor) zeroes = expectNewPeriod(clocks[1:], zeroes) require.Equal(t, uint(3), clocks[0].(*testingClock).zeroes) close(pocketCert) @@ -2339,7 +2366,7 @@ func TestAgreementCertificateDoesNotStallSingleRelay(t *testing.T) { activityMonitor.waitForQuiet() // this relay must still relay initial messages. Note that payloads were already relayed with // the previous global timeout. - triggerGlobalTimeout(FilterTimeout(0, version), clocks[1:], activityMonitor) + triggerGlobalTimeout(FilterTimeout(0, version), timers.Filter, clocks[1:], activityMonitor) zeroes = expectNewPeriod(clocks[1:], zeroes) require.Equal(t, uint(3), clocks[0].(*testingClock).zeroes) diff --git a/util/timers/interface.go b/util/timers/interface.go index 8b69218ef3..7084368007 100644 --- a/util/timers/interface.go +++ b/util/timers/interface.go @@ -21,7 +21,7 @@ import ( "time" ) -type Timeout int +type Timeout int8 const ( Deadline Timeout = iota From 1fc2f90d35a38bbaa481c9218f510f13ede4e44f Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Wed, 9 Aug 2023 21:33:31 -0400 Subject: [PATCH 009/232] fix failing tests --- agreement/service_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/agreement/service_test.go b/agreement/service_test.go index e7fea84700..d9af5228a2 100644 --- a/agreement/service_test.go +++ b/agreement/service_test.go @@ -1543,7 +1543,7 @@ func TestAgreementLateCertBug(t *testing.T) { closeFn() baseNetwork.repairAll() - triggerGlobalTimeout(deadlineTimeout, timers.Filter, clocks, activityMonitor) + triggerGlobalTimeout(deadlineTimeout, timers.Deadline, clocks, activityMonitor) zeroes = expectNewPeriod(clocks, zeroes) } @@ -2436,6 +2436,6 @@ func TestAgreementServiceStartDeadline(t *testing.T) { default: require.Fail(t, "ready channel was empty while it should have contained a single entry") } - require.Equal(t, testConsensusParams.AgreementFilterTimeoutPeriod0, demuxSignal.Deadline) + require.Equal(t, testConsensusParams.AgreementFilterTimeoutPeriod0, demuxSignal.Deadline.Deadline) require.Equal(t, baseLedger.NextRound(), demuxSignal.CurrentRound) } From 7e9f9fcdfec36e9cd3d572bc1b916ddb6f2a283f Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Wed, 9 Aug 2023 23:25:18 -0400 Subject: [PATCH 010/232] names refactor --- agreement/agreementtest/simulate.go | 4 +- agreement/demux.go | 4 +- agreement/demux_test.go | 6 +-- agreement/fuzzer/networkFacade_test.go | 2 +- agreement/msgp_gen.go | 50 ++++++++++++------------ agreement/persistence_test.go | 8 ++-- agreement/player.go | 18 ++++----- agreement/player_test.go | 2 +- agreement/service.go | 2 +- agreement/service_test.go | 54 +++++++++++++------------- util/timers/frozen.go | 2 +- util/timers/interface.go | 8 ++-- util/timers/monotonic.go | 6 +-- util/timers/monotonic_test.go | 2 +- 14 files changed, 84 insertions(+), 84 deletions(-) diff --git a/agreement/agreementtest/simulate.go b/agreement/agreementtest/simulate.go index 5ba569b2ca..caf9e270d8 100644 --- a/agreement/agreementtest/simulate.go +++ b/agreement/agreementtest/simulate.go @@ -59,7 +59,7 @@ func (i *instant) Encode() []byte { return nil } -func (i *instant) TimeoutAt(d time.Duration, timeoutType timers.Timeout) <-chan time.Time { +func (i *instant) TimeoutAt(d time.Duration, timeoutType timers.TimeoutType) <-chan time.Time { ta := make(chan time.Time) select { case <-i.timeoutAtCalled: @@ -68,7 +68,7 @@ func (i *instant) TimeoutAt(d time.Duration, timeoutType timers.Timeout) <-chan return ta } - if timeoutType == timers.Filter && !i.HasPending("pseudonode") { //d == agreement.FilterTimeout(0, protocol.ConsensusCurrentVersion) && !i.HasPending("pseudonode") { + if timeoutType == timers.Filter && !i.HasPending("pseudonode") { close(ta) } return ta diff --git a/agreement/demux.go b/agreement/demux.go index 28bbf4f4e9..a0923e5185 100644 --- a/agreement/demux.go +++ b/agreement/demux.go @@ -251,8 +251,8 @@ func (d *demux) next(s *Service, deadline Deadline, fastDeadline time.Duration, } ledgerNextRoundCh := s.Ledger.Wait(nextRound) - deadlineCh := s.Clock.TimeoutAt(deadline.Deadline, deadline.Type) - fastDeadlineCh := s.Clock.TimeoutAt(fastDeadline, timers.Fast) + deadlineCh := s.Clock.TimeoutAt(deadline.Duration, deadline.Type) + fastDeadlineCh := s.Clock.TimeoutAt(fastDeadline, timers.FastRecovery) d.UpdateEventsQueue(eventQueueDemux, 0) d.monitor.dec(demuxCoserviceType) diff --git a/agreement/demux_test.go b/agreement/demux_test.go index f1747ef3d5..40610e9487 100644 --- a/agreement/demux_test.go +++ b/agreement/demux_test.go @@ -428,8 +428,8 @@ func (t *demuxTester) Zero() timers.Clock { } // implement timers.Clock -func (t *demuxTester) TimeoutAt(delta time.Duration, timeoutType timers.Timeout) <-chan time.Time { - if timeoutType == timers.Fast { +func (t *demuxTester) TimeoutAt(delta time.Duration, timeoutType timers.TimeoutType) <-chan time.Time { + if timeoutType == timers.FastRecovery { return nil } @@ -675,7 +675,7 @@ func (t *demuxTester) TestUsecase(testcase demuxTestUsecase) bool { close(s.quit) } - e, ok := dmx.next(s, Deadline{Deadline: time.Second, Type: timers.Deadline}, fastTimeoutChTime, 300) + e, ok := dmx.next(s, Deadline{Duration: time.Second, Type: timers.Deadline}, fastTimeoutChTime, 300) if !assert.Equal(t, testcase.ok, ok) { return false diff --git a/agreement/fuzzer/networkFacade_test.go b/agreement/fuzzer/networkFacade_test.go index e955c8e9c2..48678c7419 100644 --- a/agreement/fuzzer/networkFacade_test.go +++ b/agreement/fuzzer/networkFacade_test.go @@ -375,7 +375,7 @@ func (n *NetworkFacade) Rezero() { // Since implements the Clock interface. func (n *NetworkFacade) Since() time.Duration { return 0 } -func (n *NetworkFacade) TimeoutAt(d time.Duration, timeoutType timers.Timeout) <-chan time.Time { +func (n *NetworkFacade) TimeoutAt(d time.Duration, timeoutType timers.TimeoutType) <-chan time.Time { defer n.timeoutAtInitOnce.Do(func() { n.timeoutAtInitWait.Done() }) diff --git a/agreement/msgp_gen.go b/agreement/msgp_gen.go index 6100bda791..097efae2dd 100644 --- a/agreement/msgp_gen.go +++ b/agreement/msgp_gen.go @@ -975,9 +975,9 @@ func ConsensusVersionViewMaxSize() (s int) { func (z *Deadline) MarshalMsg(b []byte) (o []byte) { o = msgp.Require(b, z.Msgsize()) // map header, size 2 - // string "Deadline" - o = append(o, 0x82, 0xa8, 0x44, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65) - o = msgp.AppendDuration(o, (*z).Deadline) + // string "Duration" + o = append(o, 0x82, 0xa8, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e) + o = msgp.AppendDuration(o, (*z).Duration) // string "Type" o = append(o, 0xa4, 0x54, 0x79, 0x70, 0x65) o = (*z).Type.MarshalMsg(o) @@ -1004,9 +1004,9 @@ func (z *Deadline) UnmarshalMsg(bts []byte) (o []byte, err error) { } if zb0001 > 0 { zb0001-- - (*z).Deadline, bts, err = msgp.ReadDurationBytes(bts) + (*z).Duration, bts, err = msgp.ReadDurationBytes(bts) if err != nil { - err = msgp.WrapError(err, "struct-from-array", "Deadline") + err = msgp.WrapError(err, "struct-from-array", "Duration") return } } @@ -1041,10 +1041,10 @@ func (z *Deadline) UnmarshalMsg(bts []byte) (o []byte, err error) { return } switch string(field) { - case "Deadline": - (*z).Deadline, bts, err = msgp.ReadDurationBytes(bts) + case "Duration": + (*z).Duration, bts, err = msgp.ReadDurationBytes(bts) if err != nil { - err = msgp.WrapError(err, "Deadline") + err = msgp.WrapError(err, "Duration") return } case "Type": @@ -1079,12 +1079,12 @@ func (z *Deadline) Msgsize() (s int) { // MsgIsZero returns whether this is a zero value func (z *Deadline) MsgIsZero() bool { - return ((*z).Deadline == 0) && ((*z).Type.MsgIsZero()) + return ((*z).Duration == 0) && ((*z).Type.MsgIsZero()) } // MaxSize returns a maximum valid message size for this message type func DeadlineMaxSize() (s int) { - s = 1 + 9 + msgp.DurationSize + 5 + timers.TimeoutMaxSize() + s = 1 + 9 + msgp.DurationSize + 5 + timers.TimeoutTypeMaxSize() return } @@ -3884,9 +3884,9 @@ func (z *player) MarshalMsg(b []byte) (o []byte) { // string "Deadline" o = append(o, 0x88, 0xa8, 0x44, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65) // map header, size 2 - // string "Deadline" - o = append(o, 0x82, 0xa8, 0x44, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65) - o = msgp.AppendDuration(o, (*z).Deadline.Deadline) + // string "Duration" + o = append(o, 0x82, 0xa8, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e) + o = msgp.AppendDuration(o, (*z).Deadline.Duration) // string "Type" o = append(o, 0xa4, 0x54, 0x79, 0x70, 0x65) o = (*z).Deadline.Type.MarshalMsg(o) @@ -3989,9 +3989,9 @@ func (z *player) UnmarshalMsg(bts []byte) (o []byte, err error) { } if zb0006 > 0 { zb0006-- - (*z).Deadline.Deadline, bts, err = msgp.ReadDurationBytes(bts) + (*z).Deadline.Duration, bts, err = msgp.ReadDurationBytes(bts) if err != nil { - err = msgp.WrapError(err, "struct-from-array", "Deadline", "struct-from-array", "Deadline") + err = msgp.WrapError(err, "struct-from-array", "Deadline", "struct-from-array", "Duration") return } } @@ -4026,10 +4026,10 @@ func (z *player) UnmarshalMsg(bts []byte) (o []byte, err error) { return } switch string(field) { - case "Deadline": - (*z).Deadline.Deadline, bts, err = msgp.ReadDurationBytes(bts) + case "Duration": + (*z).Deadline.Duration, bts, err = msgp.ReadDurationBytes(bts) if err != nil { - err = msgp.WrapError(err, "struct-from-array", "Deadline", "Deadline") + err = msgp.WrapError(err, "struct-from-array", "Deadline", "Duration") return } case "Type": @@ -4143,9 +4143,9 @@ func (z *player) UnmarshalMsg(bts []byte) (o []byte, err error) { } if zb0011 > 0 { zb0011-- - (*z).Deadline.Deadline, bts, err = msgp.ReadDurationBytes(bts) + (*z).Deadline.Duration, bts, err = msgp.ReadDurationBytes(bts) if err != nil { - err = msgp.WrapError(err, "Deadline", "struct-from-array", "Deadline") + err = msgp.WrapError(err, "Deadline", "struct-from-array", "Duration") return } } @@ -4180,10 +4180,10 @@ func (z *player) UnmarshalMsg(bts []byte) (o []byte, err error) { return } switch string(field) { - case "Deadline": - (*z).Deadline.Deadline, bts, err = msgp.ReadDurationBytes(bts) + case "Duration": + (*z).Deadline.Duration, bts, err = msgp.ReadDurationBytes(bts) if err != nil { - err = msgp.WrapError(err, "Deadline", "Deadline") + err = msgp.WrapError(err, "Deadline", "Duration") return } case "Type": @@ -4245,12 +4245,12 @@ func (z *player) Msgsize() (s int) { // MsgIsZero returns whether this is a zero value func (z *player) MsgIsZero() bool { - return ((*z).Round.MsgIsZero()) && ((*z).Period == 0) && ((*z).Step == 0) && ((*z).LastConcluding == 0) && (((*z).Deadline.Deadline == 0) && ((*z).Deadline.Type.MsgIsZero())) && ((*z).Napping == false) && ((*z).FastRecoveryDeadline == 0) && ((*z).Pending.MsgIsZero()) + return ((*z).Round.MsgIsZero()) && ((*z).Period == 0) && ((*z).Step == 0) && ((*z).LastConcluding == 0) && (((*z).Deadline.Duration == 0) && ((*z).Deadline.Type.MsgIsZero())) && ((*z).Napping == false) && ((*z).FastRecoveryDeadline == 0) && ((*z).Pending.MsgIsZero()) } // MaxSize returns a maximum valid message size for this message type func PlayerMaxSize() (s int) { - s = 1 + 6 + basics.RoundMaxSize() + 7 + msgp.Uint64Size + 5 + msgp.Uint64Size + 15 + msgp.Uint64Size + 9 + 1 + 9 + msgp.DurationSize + 5 + timers.TimeoutMaxSize() + 8 + msgp.BoolSize + 21 + msgp.DurationSize + 8 + ProposalTableMaxSize() + s = 1 + 6 + basics.RoundMaxSize() + 7 + msgp.Uint64Size + 5 + msgp.Uint64Size + 15 + msgp.Uint64Size + 9 + 1 + 9 + msgp.DurationSize + 5 + timers.TimeoutTypeMaxSize() + 8 + msgp.BoolSize + 21 + msgp.DurationSize + 8 + ProposalTableMaxSize() return } diff --git a/agreement/persistence_test.go b/agreement/persistence_test.go index ff4b90bb5b..66036b57ce 100644 --- a/agreement/persistence_test.go +++ b/agreement/persistence_test.go @@ -38,7 +38,7 @@ func TestAgreementSerialization(t *testing.T) { // todo : we need to deserialize some more meaningful state. clock := timers.MakeMonotonicClock(time.Date(2015, 1, 2, 5, 6, 7, 8, time.UTC)) - status := player{Round: 350, Step: soft, Deadline: Deadline{Deadline: time.Duration(23) * time.Second, Type: timers.Deadline}} + status := player{Round: 350, Step: soft, Deadline: Deadline{Duration: time.Duration(23) * time.Second, Type: timers.Deadline}} router := makeRootRouter(status) a := []action{checkpointAction{}, disconnectAction(messageEvent{}, nil)} @@ -59,7 +59,7 @@ func BenchmarkAgreementSerialization(b *testing.B) { b.SkipNow() clock := timers.MakeMonotonicClock(time.Date(2015, 1, 2, 5, 6, 7, 8, time.UTC)) - status := player{Round: 350, Step: soft, Deadline: Deadline{Deadline: time.Duration(23) * time.Second, Type: timers.Deadline}} + status := player{Round: 350, Step: soft, Deadline: Deadline{Duration: time.Duration(23) * time.Second, Type: timers.Deadline}} router := makeRootRouter(status) a := []action{} @@ -74,7 +74,7 @@ func BenchmarkAgreementDeserialization(b *testing.B) { b.SkipNow() clock := timers.MakeMonotonicClock(time.Date(2015, 1, 2, 5, 6, 7, 8, time.UTC)) - status := player{Round: 350, Step: soft, Deadline: Deadline{Deadline: time.Duration(23) * time.Second, Type: timers.Deadline}} + status := player{Round: 350, Step: soft, Deadline: Deadline{Duration: time.Duration(23) * time.Second, Type: timers.Deadline}} router := makeRootRouter(status) a := []action{} @@ -244,7 +244,7 @@ func TestDecodeFailures(t *testing.T) { clock := timers.MakeMonotonicClock(time.Date(2015, 1, 2, 5, 6, 7, 8, time.UTC)) ce := clock.Encode() log := makeServiceLogger(logging.Base()) - player := player{Round: 350, Step: soft, Deadline: Deadline{Deadline: time.Duration(23) * time.Second, Type: timers.Deadline}} + player := player{Round: 350, Step: soft, Deadline: Deadline{Duration: time.Duration(23) * time.Second, Type: timers.Deadline}} router := makeRootRouter(player) pe := protocol.Encode(&player) re := protocol.Encode(&router) diff --git a/agreement/player.go b/agreement/player.go index a539e936cb..86627ccded 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -26,8 +26,8 @@ import ( type Deadline struct { _struct struct{} `codec:","` - Deadline time.Duration - Type timers.Timeout + Duration time.Duration + Type timers.TimeoutType } // The player implements the top-level state machine functionality of the @@ -118,7 +118,7 @@ func (p *player) handle(r routerHandle, e event) []action { delta := time.Duration(e.RandomEntropy % uint64(upper-lower)) p.Napping = true - p.Deadline.Deadline = lower + delta + p.Deadline.Duration = lower + delta p.Deadline.Type = timers.Deadline return actions } @@ -153,7 +153,7 @@ func (p *player) handleFastTimeout(r routerHandle, e timeoutEvent) []action { func (p *player) issueSoftVote(r routerHandle) (actions []action) { defer func() { - p.Deadline.Deadline = deadlineTimeout + p.Deadline.Duration = deadlineTimeout p.Deadline.Type = timers.Deadline }() @@ -222,7 +222,7 @@ func (p *player) issueNextVote(r routerHandle) []action { _, upper := p.Step.nextVoteRanges() p.Napping = false - p.Deadline.Deadline = upper + p.Deadline.Duration = upper p.Deadline.Type = timers.Deadline return actions } @@ -337,7 +337,7 @@ func (p *player) enterPeriod(r routerHandle, source thresholdEvent, target perio p.Step = soft p.Napping = false p.FastRecoveryDeadline = 0 // set immediately - p.Deadline.Deadline = FilterTimeout(target, source.Proto) + p.Deadline.Duration = FilterTimeout(target, source.Proto) p.Deadline.Type = timers.Filter // update tracer state to match player @@ -386,13 +386,13 @@ func (p *player) enterRound(r routerHandle, source event, target round) []action switch source := source.(type) { case roundInterruptionEvent: - p.Deadline.Deadline = FilterTimeout(0, source.Proto.Version) + p.Deadline.Duration = FilterTimeout(0, source.Proto.Version) p.Deadline.Type = timers.Filter case thresholdEvent: - p.Deadline.Deadline = FilterTimeout(0, source.Proto) + p.Deadline.Duration = FilterTimeout(0, source.Proto) p.Deadline.Type = timers.Filter case filterableMessageEvent: - p.Deadline.Deadline = FilterTimeout(0, source.Proto.Version) + p.Deadline.Duration = FilterTimeout(0, source.Proto.Version) p.Deadline.Type = timers.Filter } diff --git a/agreement/player_test.go b/agreement/player_test.go index 1cd0ed3700..9a48264470 100644 --- a/agreement/player_test.go +++ b/agreement/player_test.go @@ -501,7 +501,7 @@ func TestPlayerLateBlockProposalPeriod0(t *testing.T) { func setupP(t *testing.T, r round, p period, s step) (plyr *player, pMachine ioAutomata, helper *voteMakerHelper) { // Set up a composed test machine starting at specified rps - rRouter := makeRootRouter(player{Round: r, Period: p, Step: s, Deadline: Deadline{Deadline: FilterTimeout(p, protocol.ConsensusCurrentVersion), Type: timers.Filter}}) + rRouter := makeRootRouter(player{Round: r, Period: p, Step: s, Deadline: Deadline{Duration: FilterTimeout(p, protocol.ConsensusCurrentVersion), Type: timers.Filter}}) concreteMachine := ioAutomataConcretePlayer{rootRouter: &rRouter} plyr = concreteMachine.underlying() pMachine = &concreteMachine diff --git a/agreement/service.go b/agreement/service.go index 4f68c559f4..638944df7c 100644 --- a/agreement/service.go +++ b/agreement/service.go @@ -212,7 +212,7 @@ func (s *Service) mainLoop(input <-chan externalEvent, output chan<- []action, r s.log.Errorf("unable to retrieve consensus version for round %d, defaulting to binary consensus version", nextRound) nextVersion = protocol.ConsensusCurrentVersion } - status = player{Round: nextRound, Step: soft, Deadline: Deadline{Deadline: FilterTimeout(0, nextVersion), Type: timers.Filter}} + status = player{Round: nextRound, Step: soft, Deadline: Deadline{Duration: FilterTimeout(0, nextVersion), Type: timers.Filter}} router = makeRootRouter(status) a1 := pseudonodeAction{T: assemble, Round: s.Ledger.NextRound()} diff --git a/agreement/service_test.go b/agreement/service_test.go index d9af5228a2..f671708952 100644 --- a/agreement/service_test.go +++ b/agreement/service_test.go @@ -49,14 +49,14 @@ type testingClock struct { zeroes uint - TA map[timers.Timeout]map[time.Duration]chan time.Time // TimeoutAt + TA map[timers.TimeoutType]map[time.Duration]chan time.Time // TimeoutAt monitor *coserviceMonitor } func makeTestingClock(m *coserviceMonitor) *testingClock { c := new(testingClock) - c.TA = make(map[timers.Timeout]map[time.Duration]chan time.Time) + c.TA = make(map[timers.TimeoutType]map[time.Duration]chan time.Time) c.monitor = m return c } @@ -66,7 +66,7 @@ func (c *testingClock) Zero() timers.Clock { defer c.mu.Unlock() c.zeroes++ - c.TA = make(map[timers.Timeout]map[time.Duration]chan time.Time) + c.TA = make(map[timers.TimeoutType]map[time.Duration]chan time.Time) c.monitor.clearClock() return c } @@ -75,7 +75,7 @@ func (c *testingClock) Since() time.Duration { return 0 } -func (c *testingClock) TimeoutAt(d time.Duration, timeoutType timers.Timeout) <-chan time.Time { +func (c *testingClock) TimeoutAt(d time.Duration, timeoutType timers.TimeoutType) <-chan time.Time { c.mu.Lock() defer c.mu.Unlock() @@ -103,7 +103,7 @@ func (c *testingClock) prepareToFire() { c.monitor.inc(clockCoserviceType) } -func (c *testingClock) fireType(timeoutType timers.Timeout) { +func (c *testingClock) fireType(timeoutType timers.TimeoutType) { c.mu.Lock() defer c.mu.Unlock() @@ -119,7 +119,7 @@ func (c *testingClock) fireType(timeoutType timers.Timeout) { } } -func (c *testingClock) fire(d time.Duration, timeoutType timers.Timeout) { +func (c *testingClock) fire(d time.Duration, timeoutType timers.TimeoutType) { c.mu.Lock() defer c.mu.Unlock() @@ -854,7 +854,7 @@ func expectNoNewPeriod(clocks []timers.Clock, zeroes uint) (newzeroes uint) { return zeroes } -func triggerGlobalTimeout(d time.Duration, timeoutType timers.Timeout, clocks []timers.Clock, activityMonitor *activityMonitor) { +func triggerGlobalTimeout(d time.Duration, timeoutType timers.TimeoutType, clocks []timers.Clock, activityMonitor *activityMonitor) { for i := range clocks { clocks[i].(*testingClock).prepareToFire() } @@ -865,7 +865,7 @@ func triggerGlobalTimeout(d time.Duration, timeoutType timers.Timeout, clocks [] activityMonitor.waitForQuiet() } -func triggerGlobalTimeoutType(timeoutType timers.Timeout, clocks []timers.Clock, activityMonitor *activityMonitor) { +func triggerGlobalTimeoutType(timeoutType timers.TimeoutType, clocks []timers.Clock, activityMonitor *activityMonitor) { for i := range clocks { clocks[i].(*testingClock).prepareToFire() } @@ -1080,10 +1080,10 @@ func TestAgreementFastRecoveryDownEarly(t *testing.T) { triggerGlobalTimeoutType(timers.Deadline, clocks, activityMonitor) zeroes = expectNoNewPeriod(clocks, zeroes) - triggerGlobalTimeout(0, timers.Fast, clocks, activityMonitor) // activates fast partition recovery timer + triggerGlobalTimeout(0, timers.FastRecovery, clocks, activityMonitor) // activates fast partition recovery timer zeroes = expectNoNewPeriod(clocks, zeroes) - triggerGlobalTimeout(firstFPR, timers.Fast, clocks, activityMonitor) + triggerGlobalTimeout(firstFPR, timers.FastRecovery, clocks, activityMonitor) zeroes = expectNewPeriod(clocks, zeroes) } @@ -1137,7 +1137,7 @@ func TestAgreementFastRecoveryDownMiss(t *testing.T) { triggerGlobalTimeout(deadlineTimeout, timers.Deadline, clocks, activityMonitor) zeroes = expectNoNewPeriod(clocks, zeroes) - triggerGlobalTimeout(0, timers.Fast, clocks, activityMonitor) // activates fast partition recovery timer + triggerGlobalTimeout(0, timers.FastRecovery, clocks, activityMonitor) // activates fast partition recovery timer zeroes = expectNoNewPeriod(clocks, zeroes) firstClocks := clocks[:4] @@ -1147,7 +1147,7 @@ func TestAgreementFastRecoveryDownMiss(t *testing.T) { firstClocks[i].(*testingClock).prepareToFire() } for i := range firstClocks { - firstClocks[i].(*testingClock).fire(firstFPR, timers.Fast) + firstClocks[i].(*testingClock).fire(firstFPR, timers.FastRecovery) } activityMonitor.waitForActivity() activityMonitor.waitForQuiet() @@ -1158,13 +1158,13 @@ func TestAgreementFastRecoveryDownMiss(t *testing.T) { restClocks[i].(*testingClock).prepareToFire() } for i := range restClocks { - restClocks[i].(*testingClock).fire(firstFPR, timers.Fast) + restClocks[i].(*testingClock).fire(firstFPR, timers.FastRecovery) } activityMonitor.waitForActivity() activityMonitor.waitForQuiet() zeroes = expectNoNewPeriod(clocks, zeroes) - triggerGlobalTimeout(secondFPR, timers.Fast, clocks, activityMonitor) + triggerGlobalTimeout(secondFPR, timers.FastRecovery, clocks, activityMonitor) zeroes = expectNewPeriod(clocks, zeroes) } @@ -1238,7 +1238,7 @@ func TestAgreementFastRecoveryLate(t *testing.T) { triggerGlobalTimeout(deadlineTimeout, timers.Deadline, clocks, activityMonitor) zeroes = expectNoNewPeriod(clocks, zeroes) - triggerGlobalTimeout(0, timers.Fast, clocks, activityMonitor) // activates fast partition recovery timer + triggerGlobalTimeout(0, timers.FastRecovery, clocks, activityMonitor) // activates fast partition recovery timer zeroes = expectNoNewPeriod(clocks, zeroes) baseNetwork.dropAllVotes() @@ -1249,7 +1249,7 @@ func TestAgreementFastRecoveryLate(t *testing.T) { firstClocks[i].(*testingClock).prepareToFire() } for i := range firstClocks { - firstClocks[i].(*testingClock).fire(firstFPR, timers.Fast) + firstClocks[i].(*testingClock).fire(firstFPR, timers.FastRecovery) } activityMonitor.waitForActivity() activityMonitor.waitForQuiet() @@ -1260,13 +1260,13 @@ func TestAgreementFastRecoveryLate(t *testing.T) { restClocks[i].(*testingClock).prepareToFire() } for i := range restClocks { - restClocks[i].(*testingClock).fire(firstFPR, timers.Fast) + restClocks[i].(*testingClock).fire(firstFPR, timers.FastRecovery) } activityMonitor.waitForActivity() activityMonitor.waitForQuiet() zeroes = expectNoNewPeriod(clocks, zeroes) - triggerGlobalTimeout(secondFPR, timers.Fast, clocks, activityMonitor) + triggerGlobalTimeout(secondFPR, timers.FastRecovery, clocks, activityMonitor) zeroes = expectNewPeriod(clocks, zeroes) } @@ -1351,7 +1351,7 @@ func TestAgreementFastRecoveryRedo(t *testing.T) { triggerGlobalTimeout(deadlineTimeout, timers.Deadline, clocks, activityMonitor) zeroes = expectNoNewPeriod(clocks, zeroes) - triggerGlobalTimeout(0, timers.Fast, clocks, activityMonitor) // activates fast partition recovery timer + triggerGlobalTimeout(0, timers.FastRecovery, clocks, activityMonitor) // activates fast partition recovery timer zeroes = expectNoNewPeriod(clocks, zeroes) baseNetwork.dropAllVotes() @@ -1362,7 +1362,7 @@ func TestAgreementFastRecoveryRedo(t *testing.T) { firstClocks[i].(*testingClock).prepareToFire() } for i := range firstClocks { - firstClocks[i].(*testingClock).fire(firstFPR, timers.Fast) + firstClocks[i].(*testingClock).fire(firstFPR, timers.FastRecovery) } activityMonitor.waitForActivity() activityMonitor.waitForQuiet() @@ -1373,13 +1373,13 @@ func TestAgreementFastRecoveryRedo(t *testing.T) { restClocks[i].(*testingClock).prepareToFire() } for i := range restClocks { - restClocks[i].(*testingClock).fire(firstFPR, timers.Fast) + restClocks[i].(*testingClock).fire(firstFPR, timers.FastRecovery) } activityMonitor.waitForActivity() activityMonitor.waitForQuiet() zeroes = expectNoNewPeriod(clocks, zeroes) - triggerGlobalTimeout(secondFPR, timers.Fast, clocks, activityMonitor) + triggerGlobalTimeout(secondFPR, timers.FastRecovery, clocks, activityMonitor) zeroes = expectNewPeriod(clocks, zeroes) } @@ -1392,7 +1392,7 @@ func TestAgreementFastRecoveryRedo(t *testing.T) { triggerGlobalTimeout(deadlineTimeout, timers.Deadline, clocks, activityMonitor) zeroes = expectNoNewPeriod(clocks, zeroes) - triggerGlobalTimeout(0, timers.Fast, clocks, activityMonitor) // activates fast partition recovery timer + triggerGlobalTimeout(0, timers.FastRecovery, clocks, activityMonitor) // activates fast partition recovery timer zeroes = expectNoNewPeriod(clocks, zeroes) baseNetwork.dropAllVotes() @@ -1403,7 +1403,7 @@ func TestAgreementFastRecoveryRedo(t *testing.T) { firstClocks[i].(*testingClock).prepareToFire() } for i := range firstClocks { - firstClocks[i].(*testingClock).fire(firstFPR, timers.Fast) + firstClocks[i].(*testingClock).fire(firstFPR, timers.FastRecovery) } activityMonitor.waitForActivity() activityMonitor.waitForQuiet() @@ -1414,13 +1414,13 @@ func TestAgreementFastRecoveryRedo(t *testing.T) { restClocks[i].(*testingClock).prepareToFire() } for i := range restClocks { - restClocks[i].(*testingClock).fire(firstFPR, timers.Fast) + restClocks[i].(*testingClock).fire(firstFPR, timers.FastRecovery) } activityMonitor.waitForActivity() activityMonitor.waitForQuiet() zeroes = expectNoNewPeriod(clocks, zeroes) - triggerGlobalTimeout(secondFPR, timers.Fast, clocks, activityMonitor) + triggerGlobalTimeout(secondFPR, timers.FastRecovery, clocks, activityMonitor) zeroes = expectNewPeriod(clocks, zeroes) } @@ -2436,6 +2436,6 @@ func TestAgreementServiceStartDeadline(t *testing.T) { default: require.Fail(t, "ready channel was empty while it should have contained a single entry") } - require.Equal(t, testConsensusParams.AgreementFilterTimeoutPeriod0, demuxSignal.Deadline.Deadline) + require.Equal(t, testConsensusParams.AgreementFilterTimeoutPeriod0, demuxSignal.Deadline.Duration) require.Equal(t, baseLedger.NextRound(), demuxSignal.CurrentRound) } diff --git a/util/timers/frozen.go b/util/timers/frozen.go index acc33f7bbe..db5f068c3d 100644 --- a/util/timers/frozen.go +++ b/util/timers/frozen.go @@ -38,7 +38,7 @@ func (m *Frozen) Zero() Clock { } // TimeoutAt returns a channel that will signal when the duration has elapsed. -func (m *Frozen) TimeoutAt(delta time.Duration, timeoutType Timeout) <-chan time.Time { +func (m *Frozen) TimeoutAt(delta time.Duration, timeoutType TimeoutType) <-chan time.Time { return m.timeoutCh } diff --git a/util/timers/interface.go b/util/timers/interface.go index 7084368007..d8af682e4d 100644 --- a/util/timers/interface.go +++ b/util/timers/interface.go @@ -21,11 +21,11 @@ import ( "time" ) -type Timeout int8 +type TimeoutType int8 const ( - Deadline Timeout = iota - Fast + Deadline TimeoutType = iota + FastRecovery Filter ) @@ -46,7 +46,7 @@ type Clock interface { // // TimeoutAt must be called after Zero; otherwise, the channel's behavior is // undefined. - TimeoutAt(delta time.Duration, timeoutType Timeout) <-chan time.Time + TimeoutAt(delta time.Duration, timeoutType TimeoutType) <-chan time.Time // Encode serializes the Clock into a byte slice. Encode() []byte diff --git a/util/timers/monotonic.go b/util/timers/monotonic.go index 01135d06c2..dc12d2f076 100644 --- a/util/timers/monotonic.go +++ b/util/timers/monotonic.go @@ -26,7 +26,7 @@ import ( // Monotonic uses the system's monotonic clock to emit timeouts. type Monotonic struct { zero time.Time - timeouts map[Timeout]map[time.Duration]<-chan time.Time + timeouts map[TimeoutType]map[time.Duration]<-chan time.Time } // MakeMonotonicClock creates a new monotonic clock with a given zero point. @@ -44,9 +44,9 @@ func (m *Monotonic) Zero() Clock { } // TimeoutAt returns a channel that will signal when the duration has elapsed. -func (m *Monotonic) TimeoutAt(delta time.Duration, timeoutType Timeout) <-chan time.Time { +func (m *Monotonic) TimeoutAt(delta time.Duration, timeoutType TimeoutType) <-chan time.Time { if m.timeouts == nil { - m.timeouts = make(map[Timeout]map[time.Duration]<-chan time.Time) + m.timeouts = make(map[TimeoutType]map[time.Duration]<-chan time.Time) } if _, ok := m.timeouts[timeoutType]; !ok { diff --git a/util/timers/monotonic_test.go b/util/timers/monotonic_test.go index ba847e1a98..1ca9bd47c0 100644 --- a/util/timers/monotonic_test.go +++ b/util/timers/monotonic_test.go @@ -164,7 +164,7 @@ func TestTimeoutTypes(t *testing.T) { c = m.Zero() ch1 := c.TimeoutAt(d, Filter) - ch2 := c.TimeoutAt(d, Fast) + ch2 := c.TimeoutAt(d, FastRecovery) if polled(ch1) { t.Errorf("channel fired ~100ms early") } From b59136bd08d5988e97033e84751d66f37fe9ddec Mon Sep 17 00:00:00 2001 From: Nickolai Zeldovich Date: Thu, 10 Aug 2023 05:17:15 -0400 Subject: [PATCH 011/232] generic timeout type for timers.Clock --- agreement/agreementtest/simulate.go | 2 +- agreement/demux.go | 3 +- agreement/demux_test.go | 10 +- agreement/fuzzer/networkFacade_test.go | 2 +- agreement/msgp_gen.go | 270 ++++++++++--------------- agreement/persistence.go | 10 +- agreement/persistence_test.go | 26 +-- agreement/player.go | 17 +- agreement/player_test.go | 3 +- agreement/service.go | 14 +- agreement/service_test.go | 174 ++++++++-------- node/node.go | 6 +- util/timers/frozen.go | 22 +- util/timers/interface.go | 14 +- util/timers/monotonic.go | 22 +- util/timers/monotonic_test.go | 54 ++--- 16 files changed, 290 insertions(+), 359 deletions(-) diff --git a/agreement/agreementtest/simulate.go b/agreement/agreementtest/simulate.go index caf9e270d8..55bed68969 100644 --- a/agreement/agreementtest/simulate.go +++ b/agreement/agreementtest/simulate.go @@ -59,7 +59,7 @@ func (i *instant) Encode() []byte { return nil } -func (i *instant) TimeoutAt(d time.Duration, timeoutType timers.TimeoutType) <-chan time.Time { +func (i *instant) TimeoutAt(d time.Duration, timeoutType TimeoutType) <-chan time.Time { ta := make(chan time.Time) select { case <-i.timeoutAtCalled: diff --git a/agreement/demux.go b/agreement/demux.go index a0923e5185..060ef922b0 100644 --- a/agreement/demux.go +++ b/agreement/demux.go @@ -25,7 +25,6 @@ import ( "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/logging/logspec" "github.com/algorand/go-algorand/protocol" - "github.com/algorand/go-algorand/util/timers" ) const ( @@ -252,7 +251,7 @@ func (d *demux) next(s *Service, deadline Deadline, fastDeadline time.Duration, ledgerNextRoundCh := s.Ledger.Wait(nextRound) deadlineCh := s.Clock.TimeoutAt(deadline.Duration, deadline.Type) - fastDeadlineCh := s.Clock.TimeoutAt(fastDeadline, timers.FastRecovery) + fastDeadlineCh := s.Clock.TimeoutAt(fastDeadline, TimeoutFastRecovery) d.UpdateEventsQueue(eventQueueDemux, 0) d.monitor.dec(demuxCoserviceType) diff --git a/agreement/demux_test.go b/agreement/demux_test.go index 40610e9487..68e13dd950 100644 --- a/agreement/demux_test.go +++ b/agreement/demux_test.go @@ -422,14 +422,14 @@ func TestDemuxNext(t *testing.T) { } // implement timers.Clock -func (t *demuxTester) Zero() timers.Clock { +func (t *demuxTester) Zero() timers.Clock[TimeoutType] { // we don't care about this function in this test. return t } // implement timers.Clock -func (t *demuxTester) TimeoutAt(delta time.Duration, timeoutType timers.TimeoutType) <-chan time.Time { - if timeoutType == timers.FastRecovery { +func (t *demuxTester) TimeoutAt(delta time.Duration, timeoutType TimeoutType) <-chan time.Time { + if timeoutType == TimeoutFastRecovery { return nil } @@ -450,7 +450,7 @@ func (t *demuxTester) Encode() []byte { } // implement timers.Clock -func (t *demuxTester) Decode([]byte) (timers.Clock, error) { +func (t *demuxTester) Decode([]byte) (timers.Clock[TimeoutType], error) { // we don't care about this function in this test. return t, nil } @@ -675,7 +675,7 @@ func (t *demuxTester) TestUsecase(testcase demuxTestUsecase) bool { close(s.quit) } - e, ok := dmx.next(s, Deadline{Duration: time.Second, Type: timers.Deadline}, fastTimeoutChTime, 300) + e, ok := dmx.next(s, Deadline{Duration: time.Second, Type: TimeoutDeadline}, fastTimeoutChTime, 300) if !assert.Equal(t, testcase.ok, ok) { return false diff --git a/agreement/fuzzer/networkFacade_test.go b/agreement/fuzzer/networkFacade_test.go index 48678c7419..16c4bcf4cc 100644 --- a/agreement/fuzzer/networkFacade_test.go +++ b/agreement/fuzzer/networkFacade_test.go @@ -375,7 +375,7 @@ func (n *NetworkFacade) Rezero() { // Since implements the Clock interface. func (n *NetworkFacade) Since() time.Duration { return 0 } -func (n *NetworkFacade) TimeoutAt(d time.Duration, timeoutType timers.TimeoutType) <-chan time.Time { +func (n *NetworkFacade) TimeoutAt(d time.Duration, timeoutType TimeoutType) <-chan time.Time { defer n.timeoutAtInitOnce.Do(func() { n.timeoutAtInitWait.Done() }) diff --git a/agreement/msgp_gen.go b/agreement/msgp_gen.go index 097efae2dd..50036e72e8 100644 --- a/agreement/msgp_gen.go +++ b/agreement/msgp_gen.go @@ -13,7 +13,6 @@ import ( "github.com/algorand/go-algorand/data/bookkeeping" "github.com/algorand/go-algorand/data/committee" "github.com/algorand/go-algorand/protocol" - "github.com/algorand/go-algorand/util/timers" ) // The following msgp objects are implemented in this file: @@ -44,6 +43,15 @@ import ( // |-----> (*) MsgIsZero // |-----> DeadlineMaxSize() // +// TimeoutType +// |-----> MarshalMsg +// |-----> CanMarshalMsg +// |-----> (*) UnmarshalMsg +// |-----> (*) CanUnmarshalMsg +// |-----> Msgsize +// |-----> MsgIsZero +// |-----> TimeoutTypeMaxSize() +// // actionType // |-----> MarshalMsg // |-----> CanMarshalMsg @@ -980,7 +988,7 @@ func (z *Deadline) MarshalMsg(b []byte) (o []byte) { o = msgp.AppendDuration(o, (*z).Duration) // string "Type" o = append(o, 0xa4, 0x54, 0x79, 0x70, 0x65) - o = (*z).Type.MarshalMsg(o) + o = msgp.AppendInt8(o, int8((*z).Type)) return } @@ -1012,10 +1020,14 @@ func (z *Deadline) UnmarshalMsg(bts []byte) (o []byte, err error) { } if zb0001 > 0 { zb0001-- - bts, err = (*z).Type.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "Type") - return + { + var zb0003 int8 + zb0003, bts, err = msgp.ReadInt8Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "Type") + return + } + (*z).Type = TimeoutType(zb0003) } } if zb0001 > 0 { @@ -1048,10 +1060,14 @@ func (z *Deadline) UnmarshalMsg(bts []byte) (o []byte, err error) { return } case "Type": - bts, err = (*z).Type.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "Type") - return + { + var zb0004 int8 + zb0004, bts, err = msgp.ReadInt8Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Type") + return + } + (*z).Type = TimeoutType(zb0004) } default: err = msgp.ErrNoField(string(field)) @@ -1073,18 +1089,70 @@ func (_ *Deadline) CanUnmarshalMsg(z interface{}) bool { // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message func (z *Deadline) Msgsize() (s int) { - s = 1 + 9 + msgp.DurationSize + 5 + (*z).Type.Msgsize() + s = 1 + 9 + msgp.DurationSize + 5 + msgp.Int8Size return } // MsgIsZero returns whether this is a zero value func (z *Deadline) MsgIsZero() bool { - return ((*z).Duration == 0) && ((*z).Type.MsgIsZero()) + return ((*z).Duration == 0) && ((*z).Type == 0) } // MaxSize returns a maximum valid message size for this message type func DeadlineMaxSize() (s int) { - s = 1 + 9 + msgp.DurationSize + 5 + timers.TimeoutTypeMaxSize() + s = 1 + 9 + msgp.DurationSize + 5 + msgp.Int8Size + return +} + +// MarshalMsg implements msgp.Marshaler +func (z TimeoutType) MarshalMsg(b []byte) (o []byte) { + o = msgp.Require(b, z.Msgsize()) + o = msgp.AppendInt8(o, int8(z)) + return +} + +func (_ TimeoutType) CanMarshalMsg(z interface{}) bool { + _, ok := (z).(TimeoutType) + if !ok { + _, ok = (z).(*TimeoutType) + } + return ok +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *TimeoutType) UnmarshalMsg(bts []byte) (o []byte, err error) { + { + var zb0001 int8 + zb0001, bts, err = msgp.ReadInt8Bytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + (*z) = TimeoutType(zb0001) + } + o = bts + return +} + +func (_ *TimeoutType) CanUnmarshalMsg(z interface{}) bool { + _, ok := (z).(*TimeoutType) + return ok +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z TimeoutType) Msgsize() (s int) { + s = msgp.Int8Size + return +} + +// MsgIsZero returns whether this is a zero value +func (z TimeoutType) MsgIsZero() bool { + return z == 0 +} + +// MaxSize returns a maximum valid message size for this message type +func TimeoutTypeMaxSize() (s int) { + s = msgp.Int8Size return } @@ -3883,13 +3951,7 @@ func (z *player) MarshalMsg(b []byte) (o []byte) { // map header, size 8 // string "Deadline" o = append(o, 0x88, 0xa8, 0x44, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65) - // map header, size 2 - // string "Duration" - o = append(o, 0x82, 0xa8, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e) - o = msgp.AppendDuration(o, (*z).Deadline.Duration) - // string "Type" - o = append(o, 0xa4, 0x54, 0x79, 0x70, 0x65) - o = (*z).Deadline.Type.MarshalMsg(o) + o = (*z).Deadline.MarshalMsg(o) // string "FastRecoveryDeadline" o = append(o, 0xb4, 0x46, 0x61, 0x73, 0x74, 0x52, 0x65, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x79, 0x44, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65) o = msgp.AppendDuration(o, (*z).FastRecoveryDeadline) @@ -3978,74 +4040,10 @@ func (z *player) UnmarshalMsg(bts []byte) (o []byte, err error) { } if zb0001 > 0 { zb0001-- - var zb0006 int - var zb0007 bool - zb0006, zb0007, bts, err = msgp.ReadMapHeaderBytes(bts) - if _, ok := err.(msgp.TypeError); ok { - zb0006, zb0007, bts, err = msgp.ReadArrayHeaderBytes(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "Deadline") - return - } - if zb0006 > 0 { - zb0006-- - (*z).Deadline.Duration, bts, err = msgp.ReadDurationBytes(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "Deadline", "struct-from-array", "Duration") - return - } - } - if zb0006 > 0 { - zb0006-- - bts, err = (*z).Deadline.Type.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "Deadline", "struct-from-array", "Type") - return - } - } - if zb0006 > 0 { - err = msgp.ErrTooManyArrayFields(zb0006) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "Deadline", "struct-from-array") - return - } - } - } else { - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "Deadline") - return - } - if zb0007 { - (*z).Deadline = Deadline{} - } - for zb0006 > 0 { - zb0006-- - field, bts, err = msgp.ReadMapKeyZC(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "Deadline") - return - } - switch string(field) { - case "Duration": - (*z).Deadline.Duration, bts, err = msgp.ReadDurationBytes(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "Deadline", "Duration") - return - } - case "Type": - bts, err = (*z).Deadline.Type.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "Deadline", "Type") - return - } - default: - err = msgp.ErrNoField(string(field)) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "Deadline") - return - } - } - } + bts, err = (*z).Deadline.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "Deadline") + return } } if zb0001 > 0 { @@ -4103,103 +4101,39 @@ func (z *player) UnmarshalMsg(bts []byte) (o []byte, err error) { } case "Period": { - var zb0008 uint64 - zb0008, bts, err = msgp.ReadUint64Bytes(bts) + var zb0006 uint64 + zb0006, bts, err = msgp.ReadUint64Bytes(bts) if err != nil { err = msgp.WrapError(err, "Period") return } - (*z).Period = period(zb0008) + (*z).Period = period(zb0006) } case "Step": { - var zb0009 uint64 - zb0009, bts, err = msgp.ReadUint64Bytes(bts) + var zb0007 uint64 + zb0007, bts, err = msgp.ReadUint64Bytes(bts) if err != nil { err = msgp.WrapError(err, "Step") return } - (*z).Step = step(zb0009) + (*z).Step = step(zb0007) } case "LastConcluding": { - var zb0010 uint64 - zb0010, bts, err = msgp.ReadUint64Bytes(bts) + var zb0008 uint64 + zb0008, bts, err = msgp.ReadUint64Bytes(bts) if err != nil { err = msgp.WrapError(err, "LastConcluding") return } - (*z).LastConcluding = step(zb0010) + (*z).LastConcluding = step(zb0008) } case "Deadline": - var zb0011 int - var zb0012 bool - zb0011, zb0012, bts, err = msgp.ReadMapHeaderBytes(bts) - if _, ok := err.(msgp.TypeError); ok { - zb0011, zb0012, bts, err = msgp.ReadArrayHeaderBytes(bts) - if err != nil { - err = msgp.WrapError(err, "Deadline") - return - } - if zb0011 > 0 { - zb0011-- - (*z).Deadline.Duration, bts, err = msgp.ReadDurationBytes(bts) - if err != nil { - err = msgp.WrapError(err, "Deadline", "struct-from-array", "Duration") - return - } - } - if zb0011 > 0 { - zb0011-- - bts, err = (*z).Deadline.Type.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "Deadline", "struct-from-array", "Type") - return - } - } - if zb0011 > 0 { - err = msgp.ErrTooManyArrayFields(zb0011) - if err != nil { - err = msgp.WrapError(err, "Deadline", "struct-from-array") - return - } - } - } else { - if err != nil { - err = msgp.WrapError(err, "Deadline") - return - } - if zb0012 { - (*z).Deadline = Deadline{} - } - for zb0011 > 0 { - zb0011-- - field, bts, err = msgp.ReadMapKeyZC(bts) - if err != nil { - err = msgp.WrapError(err, "Deadline") - return - } - switch string(field) { - case "Duration": - (*z).Deadline.Duration, bts, err = msgp.ReadDurationBytes(bts) - if err != nil { - err = msgp.WrapError(err, "Deadline", "Duration") - return - } - case "Type": - bts, err = (*z).Deadline.Type.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "Deadline", "Type") - return - } - default: - err = msgp.ErrNoField(string(field)) - if err != nil { - err = msgp.WrapError(err, "Deadline") - return - } - } - } + bts, err = (*z).Deadline.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Deadline") + return } case "Napping": (*z).Napping, bts, err = msgp.ReadBoolBytes(bts) @@ -4239,18 +4173,18 @@ func (_ *player) CanUnmarshalMsg(z interface{}) bool { // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message func (z *player) Msgsize() (s int) { - s = 1 + 6 + (*z).Round.Msgsize() + 7 + msgp.Uint64Size + 5 + msgp.Uint64Size + 15 + msgp.Uint64Size + 9 + 1 + 9 + msgp.DurationSize + 5 + (*z).Deadline.Type.Msgsize() + 8 + msgp.BoolSize + 21 + msgp.DurationSize + 8 + (*z).Pending.Msgsize() + s = 1 + 6 + (*z).Round.Msgsize() + 7 + msgp.Uint64Size + 5 + msgp.Uint64Size + 15 + msgp.Uint64Size + 9 + (*z).Deadline.Msgsize() + 8 + msgp.BoolSize + 21 + msgp.DurationSize + 8 + (*z).Pending.Msgsize() return } // MsgIsZero returns whether this is a zero value func (z *player) MsgIsZero() bool { - return ((*z).Round.MsgIsZero()) && ((*z).Period == 0) && ((*z).Step == 0) && ((*z).LastConcluding == 0) && (((*z).Deadline.Duration == 0) && ((*z).Deadline.Type.MsgIsZero())) && ((*z).Napping == false) && ((*z).FastRecoveryDeadline == 0) && ((*z).Pending.MsgIsZero()) + return ((*z).Round.MsgIsZero()) && ((*z).Period == 0) && ((*z).Step == 0) && ((*z).LastConcluding == 0) && ((*z).Deadline.MsgIsZero()) && ((*z).Napping == false) && ((*z).FastRecoveryDeadline == 0) && ((*z).Pending.MsgIsZero()) } // MaxSize returns a maximum valid message size for this message type func PlayerMaxSize() (s int) { - s = 1 + 6 + basics.RoundMaxSize() + 7 + msgp.Uint64Size + 5 + msgp.Uint64Size + 15 + msgp.Uint64Size + 9 + 1 + 9 + msgp.DurationSize + 5 + timers.TimeoutTypeMaxSize() + 8 + msgp.BoolSize + 21 + msgp.DurationSize + 8 + ProposalTableMaxSize() + s = 1 + 6 + basics.RoundMaxSize() + 7 + msgp.Uint64Size + 5 + msgp.Uint64Size + 15 + msgp.Uint64Size + 9 + DeadlineMaxSize() + 8 + msgp.BoolSize + 21 + msgp.DurationSize + 8 + ProposalTableMaxSize() return } diff --git a/agreement/persistence.go b/agreement/persistence.go index 5e85955b30..a4b846ea13 100644 --- a/agreement/persistence.go +++ b/agreement/persistence.go @@ -53,7 +53,7 @@ func persistent(as []action) bool { } // encode serializes the current state into a byte array. -func encode(t timers.Clock, rr rootRouter, p player, a []action, reflect bool) (raw []byte) { +func encode(t timers.Clock[TimeoutType], rr rootRouter, p player, a []action, reflect bool) (raw []byte) { var s diskState if reflect { s.Router = protocol.EncodeReflect(rr) @@ -194,8 +194,8 @@ func restore(log logging.Logger, crash db.Accessor) (raw []byte, err error) { // decode process the incoming raw bytes array and attempt to reconstruct the agreement state objects. // // In all decoding errors, it returns the error code in err -func decode(raw []byte, t0 timers.Clock, log serviceLogger, reflect bool) (t timers.Clock, rr rootRouter, p player, a []action, err error) { - var t2 timers.Clock +func decode(raw []byte, t0 timers.Clock[TimeoutType], log serviceLogger, reflect bool) (t timers.Clock[TimeoutType], rr rootRouter, p player, a []action, err error) { + var t2 timers.Clock[TimeoutType] var rr2 rootRouter var p2 player a2 := []action{} @@ -280,7 +280,7 @@ type persistentRequest struct { step step raw []byte done chan error - clock timers.Clock + clock timers.Clock[TimeoutType] events chan<- externalEvent } @@ -302,7 +302,7 @@ func makeAsyncPersistenceLoop(log serviceLogger, crash db.Accessor, ledger Ledge } } -func (p *asyncPersistenceLoop) Enqueue(clock timers.Clock, round basics.Round, period period, step step, raw []byte, done chan error) (events <-chan externalEvent) { +func (p *asyncPersistenceLoop) Enqueue(clock timers.Clock[TimeoutType], round basics.Round, period period, step step, raw []byte, done chan error) (events <-chan externalEvent) { eventsChannel := make(chan externalEvent, 1) p.pending <- persistentRequest{ round: round, diff --git a/agreement/persistence_test.go b/agreement/persistence_test.go index 66036b57ce..e4c1020f6e 100644 --- a/agreement/persistence_test.go +++ b/agreement/persistence_test.go @@ -37,14 +37,14 @@ func TestAgreementSerialization(t *testing.T) { partitiontest.PartitionTest(t) // todo : we need to deserialize some more meaningful state. - clock := timers.MakeMonotonicClock(time.Date(2015, 1, 2, 5, 6, 7, 8, time.UTC)) - status := player{Round: 350, Step: soft, Deadline: Deadline{Duration: time.Duration(23) * time.Second, Type: timers.Deadline}} + clock := timers.MakeMonotonicClock[TimeoutType](time.Date(2015, 1, 2, 5, 6, 7, 8, time.UTC)) + status := player{Round: 350, Step: soft, Deadline: Deadline{Duration: time.Duration(23) * time.Second, Type: TimeoutDeadline}} router := makeRootRouter(status) a := []action{checkpointAction{}, disconnectAction(messageEvent{}, nil)} encodedBytes := encode(clock, router, status, a, false) - t0 := timers.MakeMonotonicClock(time.Date(2000, 0, 0, 0, 0, 0, 0, time.UTC)) + t0 := timers.MakeMonotonicClock[TimeoutType](time.Date(2000, 0, 0, 0, 0, 0, 0, time.UTC)) log := makeServiceLogger(logging.Base()) clock2, router2, status2, a2, err := decode(encodedBytes, t0, log, false) require.NoError(t, err) @@ -58,8 +58,8 @@ func BenchmarkAgreementSerialization(b *testing.B) { // todo : we need to deserialize some more meaningful state. b.SkipNow() - clock := timers.MakeMonotonicClock(time.Date(2015, 1, 2, 5, 6, 7, 8, time.UTC)) - status := player{Round: 350, Step: soft, Deadline: Deadline{Duration: time.Duration(23) * time.Second, Type: timers.Deadline}} + clock := timers.MakeMonotonicClock[TimeoutType](time.Date(2015, 1, 2, 5, 6, 7, 8, time.UTC)) + status := player{Round: 350, Step: soft, Deadline: Deadline{Duration: time.Duration(23) * time.Second, Type: TimeoutDeadline}} router := makeRootRouter(status) a := []action{} @@ -73,13 +73,13 @@ func BenchmarkAgreementDeserialization(b *testing.B) { // todo : we need to deserialize some more meaningful state. b.SkipNow() - clock := timers.MakeMonotonicClock(time.Date(2015, 1, 2, 5, 6, 7, 8, time.UTC)) - status := player{Round: 350, Step: soft, Deadline: Deadline{Duration: time.Duration(23) * time.Second, Type: timers.Deadline}} + clock := timers.MakeMonotonicClock[TimeoutType](time.Date(2015, 1, 2, 5, 6, 7, 8, time.UTC)) + status := player{Round: 350, Step: soft, Deadline: Deadline{Duration: time.Duration(23) * time.Second, Type: TimeoutDeadline}} router := makeRootRouter(status) a := []action{} encodedBytes := encode(clock, router, status, a, false) - t0 := timers.MakeMonotonicClock(time.Date(2000, 0, 0, 0, 0, 0, 0, time.UTC)) + t0 := timers.MakeMonotonicClock[TimeoutType](time.Date(2000, 0, 0, 0, 0, 0, 0, time.UTC)) log := makeServiceLogger(logging.Base()) b.ResetTimer() for n := 0; n < b.N; n++ { @@ -185,7 +185,7 @@ func TestRandomizedEncodingFullDiskState(t *testing.T) { for i := 0; i < 5000; i++ { router, player := randomizeDiskState() a := []action{} - clock := timers.MakeMonotonicClock(time.Date(2015, 1, 2, 5, 6, 7, 8, time.UTC)) + clock := timers.MakeMonotonicClock[TimeoutType](time.Date(2015, 1, 2, 5, 6, 7, 8, time.UTC)) log := makeServiceLogger(logging.Base()) e1 := encode(clock, router, player, a, true) e2 := encode(clock, router, player, a, false) @@ -201,7 +201,7 @@ func TestRandomizedEncodingFullDiskState(t *testing.T) { } func BenchmarkRandomizedEncode(b *testing.B) { - clock := timers.MakeMonotonicClock(time.Date(2015, 1, 2, 5, 6, 7, 8, time.UTC)) + clock := timers.MakeMonotonicClock[TimeoutType](time.Date(2015, 1, 2, 5, 6, 7, 8, time.UTC)) router, player := randomizeDiskState() a := []action{} b.ResetTimer() @@ -211,7 +211,7 @@ func BenchmarkRandomizedEncode(b *testing.B) { } func BenchmarkRandomizedDecode(b *testing.B) { - clock := timers.MakeMonotonicClock(time.Date(2015, 1, 2, 5, 6, 7, 8, time.UTC)) + clock := timers.MakeMonotonicClock[TimeoutType](time.Date(2015, 1, 2, 5, 6, 7, 8, time.UTC)) router, player := randomizeDiskState() a := []action{} ds := encode(clock, router, player, a, false) @@ -241,10 +241,10 @@ func TestEmptyMapDeserialization(t *testing.T) { func TestDecodeFailures(t *testing.T) { partitiontest.PartitionTest(t) - clock := timers.MakeMonotonicClock(time.Date(2015, 1, 2, 5, 6, 7, 8, time.UTC)) + clock := timers.MakeMonotonicClock[TimeoutType](time.Date(2015, 1, 2, 5, 6, 7, 8, time.UTC)) ce := clock.Encode() log := makeServiceLogger(logging.Base()) - player := player{Round: 350, Step: soft, Deadline: Deadline{Duration: time.Duration(23) * time.Second, Type: timers.Deadline}} + player := player{Round: 350, Step: soft, Deadline: Deadline{Duration: time.Duration(23) * time.Second, Type: TimeoutDeadline}} router := makeRootRouter(player) pe := protocol.Encode(&player) re := protocol.Encode(&router) diff --git a/agreement/player.go b/agreement/player.go index 86627ccded..3b3a746f72 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -21,13 +21,12 @@ import ( "github.com/algorand/go-algorand/config" "github.com/algorand/go-algorand/protocol" - "github.com/algorand/go-algorand/util/timers" ) type Deadline struct { _struct struct{} `codec:","` Duration time.Duration - Type timers.TimeoutType + Type TimeoutType } // The player implements the top-level state machine functionality of the @@ -119,7 +118,7 @@ func (p *player) handle(r routerHandle, e event) []action { p.Napping = true p.Deadline.Duration = lower + delta - p.Deadline.Type = timers.Deadline + p.Deadline.Type = TimeoutDeadline return actions } case roundInterruptionEvent: @@ -154,7 +153,7 @@ func (p *player) handleFastTimeout(r routerHandle, e timeoutEvent) []action { func (p *player) issueSoftVote(r routerHandle) (actions []action) { defer func() { p.Deadline.Duration = deadlineTimeout - p.Deadline.Type = timers.Deadline + p.Deadline.Type = TimeoutDeadline }() e := r.dispatch(*p, proposalFrozenEvent{}, proposalMachinePeriod, p.Round, p.Period, 0) @@ -223,7 +222,7 @@ func (p *player) issueNextVote(r routerHandle) []action { _, upper := p.Step.nextVoteRanges() p.Napping = false p.Deadline.Duration = upper - p.Deadline.Type = timers.Deadline + p.Deadline.Type = TimeoutDeadline return actions } @@ -338,7 +337,7 @@ func (p *player) enterPeriod(r routerHandle, source thresholdEvent, target perio p.Napping = false p.FastRecoveryDeadline = 0 // set immediately p.Deadline.Duration = FilterTimeout(target, source.Proto) - p.Deadline.Type = timers.Filter + p.Deadline.Type = TimeoutFilter // update tracer state to match player r.t.setMetadata(tracerMetadata{p.Round, p.Period, p.Step}) @@ -387,13 +386,13 @@ func (p *player) enterRound(r routerHandle, source event, target round) []action switch source := source.(type) { case roundInterruptionEvent: p.Deadline.Duration = FilterTimeout(0, source.Proto.Version) - p.Deadline.Type = timers.Filter + p.Deadline.Type = TimeoutFilter case thresholdEvent: p.Deadline.Duration = FilterTimeout(0, source.Proto) - p.Deadline.Type = timers.Filter + p.Deadline.Type = TimeoutFilter case filterableMessageEvent: p.Deadline.Duration = FilterTimeout(0, source.Proto.Version) - p.Deadline.Type = timers.Filter + p.Deadline.Type = TimeoutFilter } // update tracer state to match player diff --git a/agreement/player_test.go b/agreement/player_test.go index 9a48264470..84aad574ba 100644 --- a/agreement/player_test.go +++ b/agreement/player_test.go @@ -28,7 +28,6 @@ import ( "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/protocol" "github.com/algorand/go-algorand/test/partitiontest" - "github.com/algorand/go-algorand/util/timers" ) var playerTracer tracer @@ -501,7 +500,7 @@ func TestPlayerLateBlockProposalPeriod0(t *testing.T) { func setupP(t *testing.T, r round, p period, s step) (plyr *player, pMachine ioAutomata, helper *voteMakerHelper) { // Set up a composed test machine starting at specified rps - rRouter := makeRootRouter(player{Round: r, Period: p, Step: s, Deadline: Deadline{Duration: FilterTimeout(p, protocol.ConsensusCurrentVersion), Type: timers.Filter}}) + rRouter := makeRootRouter(player{Round: r, Period: p, Step: s, Deadline: Deadline{Duration: FilterTimeout(p, protocol.ConsensusCurrentVersion), Type: TimeoutFilter}}) concreteMachine := ioAutomataConcretePlayer{rootRouter: &rRouter} plyr = concreteMachine.underlying() pMachine = &concreteMachine diff --git a/agreement/service.go b/agreement/service.go index 638944df7c..7a519f6911 100644 --- a/agreement/service.go +++ b/agreement/service.go @@ -68,7 +68,7 @@ type Parameters struct { BlockFactory RandomSource EventsProcessingMonitor - timers.Clock + timers.Clock[TimeoutType] db.Accessor logging.Logger config.Local @@ -85,6 +85,14 @@ type externalDemuxSignals struct { CurrentRound round } +type TimeoutType int8 + +const ( + TimeoutDeadline TimeoutType = iota + TimeoutFastRecovery + TimeoutFilter +) + // MakeService creates a new Agreement Service instance given a set of Parameters. // // Call Start to start execution and Shutdown to finish execution. @@ -188,7 +196,7 @@ func (s *Service) demuxLoop(ctx context.Context, input chan<- externalEvent, out // 4. If necessary, persist state to disk. func (s *Service) mainLoop(input <-chan externalEvent, output chan<- []action, ready chan<- externalDemuxSignals) { // setup - var clock timers.Clock + var clock timers.Clock[TimeoutType] var router rootRouter var status player var a []action @@ -212,7 +220,7 @@ func (s *Service) mainLoop(input <-chan externalEvent, output chan<- []action, r s.log.Errorf("unable to retrieve consensus version for round %d, defaulting to binary consensus version", nextRound) nextVersion = protocol.ConsensusCurrentVersion } - status = player{Round: nextRound, Step: soft, Deadline: Deadline{Duration: FilterTimeout(0, nextVersion), Type: timers.Filter}} + status = player{Round: nextRound, Step: soft, Deadline: Deadline{Duration: FilterTimeout(0, nextVersion), Type: TimeoutFilter}} router = makeRootRouter(status) a1 := pseudonodeAction{T: assemble, Round: s.Ledger.NextRound()} diff --git a/agreement/service_test.go b/agreement/service_test.go index f671708952..783188ff55 100644 --- a/agreement/service_test.go +++ b/agreement/service_test.go @@ -49,24 +49,24 @@ type testingClock struct { zeroes uint - TA map[timers.TimeoutType]map[time.Duration]chan time.Time // TimeoutAt + TA map[TimeoutType]map[time.Duration]chan time.Time // TimeoutAt monitor *coserviceMonitor } func makeTestingClock(m *coserviceMonitor) *testingClock { c := new(testingClock) - c.TA = make(map[timers.TimeoutType]map[time.Duration]chan time.Time) + c.TA = make(map[TimeoutType]map[time.Duration]chan time.Time) c.monitor = m return c } -func (c *testingClock) Zero() timers.Clock { +func (c *testingClock) Zero() timers.Clock[TimeoutType] { c.mu.Lock() defer c.mu.Unlock() c.zeroes++ - c.TA = make(map[timers.TimeoutType]map[time.Duration]chan time.Time) + c.TA = make(map[TimeoutType]map[time.Duration]chan time.Time) c.monitor.clearClock() return c } @@ -75,7 +75,7 @@ func (c *testingClock) Since() time.Duration { return 0 } -func (c *testingClock) TimeoutAt(d time.Duration, timeoutType timers.TimeoutType) <-chan time.Time { +func (c *testingClock) TimeoutAt(d time.Duration, timeoutType TimeoutType) <-chan time.Time { c.mu.Lock() defer c.mu.Unlock() @@ -95,7 +95,7 @@ func (c *testingClock) Encode() []byte { return nil } -func (c *testingClock) Decode([]byte) (timers.Clock, error) { +func (c *testingClock) Decode([]byte) (timers.Clock[TimeoutType], error) { return makeTestingClock(nil), nil // TODO } @@ -103,7 +103,7 @@ func (c *testingClock) prepareToFire() { c.monitor.inc(clockCoserviceType) } -func (c *testingClock) fireType(timeoutType timers.TimeoutType) { +func (c *testingClock) fireType(timeoutType TimeoutType) { c.mu.Lock() defer c.mu.Unlock() @@ -119,7 +119,7 @@ func (c *testingClock) fireType(timeoutType timers.TimeoutType) { } } -func (c *testingClock) fire(d time.Duration, timeoutType timers.TimeoutType) { +func (c *testingClock) fire(d time.Duration, timeoutType TimeoutType) { c.mu.Lock() defer c.mu.Unlock() @@ -720,12 +720,12 @@ func (testingRand) Uint64() uint64 { return maxuint64 / 2 } -func setupAgreement(t *testing.T, numNodes int, traceLevel traceLevel, ledgerFactory func(map[basics.Address]basics.AccountData) Ledger) (*testingNetwork, Ledger, func(), []*Service, []timers.Clock, []Ledger, *activityMonitor) { +func setupAgreement(t *testing.T, numNodes int, traceLevel traceLevel, ledgerFactory func(map[basics.Address]basics.AccountData) Ledger) (*testingNetwork, Ledger, func(), []*Service, []timers.Clock[TimeoutType], []Ledger, *activityMonitor) { var validator testBlockValidator return setupAgreementWithValidator(t, numNodes, traceLevel, validator, ledgerFactory) } -func setupAgreementWithValidator(t *testing.T, numNodes int, traceLevel traceLevel, validator BlockValidator, ledgerFactory func(map[basics.Address]basics.AccountData) Ledger) (*testingNetwork, Ledger, func(), []*Service, []timers.Clock, []Ledger, *activityMonitor) { +func setupAgreementWithValidator(t *testing.T, numNodes int, traceLevel traceLevel, validator BlockValidator, ledgerFactory func(map[basics.Address]basics.AccountData) Ledger) (*testingNetwork, Ledger, func(), []*Service, []timers.Clock[TimeoutType], []Ledger, *activityMonitor) { bufCap := 1000 // max number of buffered messages // system state setup: keygen, stake initialization @@ -740,7 +740,7 @@ func setupAgreementWithValidator(t *testing.T, numNodes int, traceLevel traceLev log.SetLevel(logging.Debug) // node setup - clocks := make([]timers.Clock, numNodes) + clocks := make([]timers.Clock[TimeoutType], numNodes) ledgers := make([]Ledger, numNodes) dbAccessors := make([]db.Accessor, numNodes) services := make([]*Service, numNodes) @@ -833,7 +833,7 @@ func (m *coserviceMonitor) clearClock() { } } -func expectNewPeriod(clocks []timers.Clock, zeroes uint) (newzeroes uint) { +func expectNewPeriod(clocks []timers.Clock[TimeoutType], zeroes uint) (newzeroes uint) { zeroes++ for i := range clocks { if clocks[i].(*testingClock).zeroes != zeroes { @@ -844,7 +844,7 @@ func expectNewPeriod(clocks []timers.Clock, zeroes uint) (newzeroes uint) { return zeroes } -func expectNoNewPeriod(clocks []timers.Clock, zeroes uint) (newzeroes uint) { +func expectNoNewPeriod(clocks []timers.Clock[TimeoutType], zeroes uint) (newzeroes uint) { for i := range clocks { if clocks[i].(*testingClock).zeroes != zeroes { errstr := fmt.Sprintf("unexpected number of zeroes: %v != %v", clocks[i].(*testingClock).zeroes, zeroes) @@ -854,7 +854,7 @@ func expectNoNewPeriod(clocks []timers.Clock, zeroes uint) (newzeroes uint) { return zeroes } -func triggerGlobalTimeout(d time.Duration, timeoutType timers.TimeoutType, clocks []timers.Clock, activityMonitor *activityMonitor) { +func triggerGlobalTimeout(d time.Duration, timeoutType TimeoutType, clocks []timers.Clock[TimeoutType], activityMonitor *activityMonitor) { for i := range clocks { clocks[i].(*testingClock).prepareToFire() } @@ -865,7 +865,7 @@ func triggerGlobalTimeout(d time.Duration, timeoutType timers.TimeoutType, clock activityMonitor.waitForQuiet() } -func triggerGlobalTimeoutType(timeoutType timers.TimeoutType, clocks []timers.Clock, activityMonitor *activityMonitor) { +func triggerGlobalTimeoutType(timeoutType TimeoutType, clocks []timers.Clock[TimeoutType], activityMonitor *activityMonitor) { for i := range clocks { clocks[i].(*testingClock).prepareToFire() } @@ -876,8 +876,8 @@ func triggerGlobalTimeoutType(timeoutType timers.TimeoutType, clocks []timers.Cl activityMonitor.waitForQuiet() } -func runRound(clocks []timers.Clock, activityMonitor *activityMonitor, zeroes uint, filterTimeout time.Duration) (newzeroes uint) { - triggerGlobalTimeout(filterTimeout, timers.Filter, clocks, activityMonitor) +func runRound(clocks []timers.Clock[TimeoutType], activityMonitor *activityMonitor, zeroes uint, filterTimeout time.Duration) (newzeroes uint) { + triggerGlobalTimeout(filterTimeout, TimeoutFilter, clocks, activityMonitor) return expectNewPeriod(clocks, zeroes) } @@ -1074,23 +1074,23 @@ func TestAgreementFastRecoveryDownEarly(t *testing.T) { baseNetwork.dropAllSoftVotes() baseNetwork.dropAllSlowNextVotes() - triggerGlobalTimeout(FilterTimeout(0, version), timers.Filter, clocks, activityMonitor) + triggerGlobalTimeout(FilterTimeout(0, version), TimeoutFilter, clocks, activityMonitor) zeroes = expectNoNewPeriod(clocks, zeroes) - triggerGlobalTimeoutType(timers.Deadline, clocks, activityMonitor) + triggerGlobalTimeoutType(TimeoutDeadline, clocks, activityMonitor) zeroes = expectNoNewPeriod(clocks, zeroes) - triggerGlobalTimeout(0, timers.FastRecovery, clocks, activityMonitor) // activates fast partition recovery timer + triggerGlobalTimeout(0, TimeoutFastRecovery, clocks, activityMonitor) // activates fast partition recovery timer zeroes = expectNoNewPeriod(clocks, zeroes) - triggerGlobalTimeout(firstFPR, timers.FastRecovery, clocks, activityMonitor) + triggerGlobalTimeout(firstFPR, TimeoutFastRecovery, clocks, activityMonitor) zeroes = expectNewPeriod(clocks, zeroes) } // terminate on period 1 { baseNetwork.repairAll() - triggerGlobalTimeout(FilterTimeout(1, version), timers.Filter, clocks, activityMonitor) + triggerGlobalTimeout(FilterTimeout(1, version), TimeoutFilter, clocks, activityMonitor) zeroes = expectNewPeriod(clocks, zeroes) } @@ -1131,13 +1131,13 @@ func TestAgreementFastRecoveryDownMiss(t *testing.T) { { // fail all steps baseNetwork.dropAllVotes() - triggerGlobalTimeout(FilterTimeout(0, version), timers.Filter, clocks, activityMonitor) + triggerGlobalTimeout(FilterTimeout(0, version), TimeoutFilter, clocks, activityMonitor) zeroes = expectNoNewPeriod(clocks, zeroes) - triggerGlobalTimeout(deadlineTimeout, timers.Deadline, clocks, activityMonitor) + triggerGlobalTimeout(deadlineTimeout, TimeoutDeadline, clocks, activityMonitor) zeroes = expectNoNewPeriod(clocks, zeroes) - triggerGlobalTimeout(0, timers.FastRecovery, clocks, activityMonitor) // activates fast partition recovery timer + triggerGlobalTimeout(0, TimeoutFastRecovery, clocks, activityMonitor) // activates fast partition recovery timer zeroes = expectNoNewPeriod(clocks, zeroes) firstClocks := clocks[:4] @@ -1147,7 +1147,7 @@ func TestAgreementFastRecoveryDownMiss(t *testing.T) { firstClocks[i].(*testingClock).prepareToFire() } for i := range firstClocks { - firstClocks[i].(*testingClock).fire(firstFPR, timers.FastRecovery) + firstClocks[i].(*testingClock).fire(firstFPR, TimeoutFastRecovery) } activityMonitor.waitForActivity() activityMonitor.waitForQuiet() @@ -1158,20 +1158,20 @@ func TestAgreementFastRecoveryDownMiss(t *testing.T) { restClocks[i].(*testingClock).prepareToFire() } for i := range restClocks { - restClocks[i].(*testingClock).fire(firstFPR, timers.FastRecovery) + restClocks[i].(*testingClock).fire(firstFPR, TimeoutFastRecovery) } activityMonitor.waitForActivity() activityMonitor.waitForQuiet() zeroes = expectNoNewPeriod(clocks, zeroes) - triggerGlobalTimeout(secondFPR, timers.FastRecovery, clocks, activityMonitor) + triggerGlobalTimeout(secondFPR, TimeoutFastRecovery, clocks, activityMonitor) zeroes = expectNewPeriod(clocks, zeroes) } // terminate on period 1 { baseNetwork.repairAll() - triggerGlobalTimeout(FilterTimeout(1, version), timers.Filter, clocks, activityMonitor) + triggerGlobalTimeout(FilterTimeout(1, version), TimeoutFilter, clocks, activityMonitor) zeroes = expectNewPeriod(clocks, zeroes) } @@ -1214,7 +1214,7 @@ func TestAgreementFastRecoveryLate(t *testing.T) { pocket := make(chan multicastParams, 100) closeFn := baseNetwork.pocketAllCertVotes(pocket) baseNetwork.dropAllSlowNextVotes() - triggerGlobalTimeout(FilterTimeout(0, version), timers.Filter, clocks, activityMonitor) + triggerGlobalTimeout(FilterTimeout(0, version), TimeoutFilter, clocks, activityMonitor) zeroes = expectNoNewPeriod(clocks, zeroes) closeFn() @@ -1235,10 +1235,10 @@ func TestAgreementFastRecoveryLate(t *testing.T) { } } - triggerGlobalTimeout(deadlineTimeout, timers.Deadline, clocks, activityMonitor) + triggerGlobalTimeout(deadlineTimeout, TimeoutDeadline, clocks, activityMonitor) zeroes = expectNoNewPeriod(clocks, zeroes) - triggerGlobalTimeout(0, timers.FastRecovery, clocks, activityMonitor) // activates fast partition recovery timer + triggerGlobalTimeout(0, TimeoutFastRecovery, clocks, activityMonitor) // activates fast partition recovery timer zeroes = expectNoNewPeriod(clocks, zeroes) baseNetwork.dropAllVotes() @@ -1249,7 +1249,7 @@ func TestAgreementFastRecoveryLate(t *testing.T) { firstClocks[i].(*testingClock).prepareToFire() } for i := range firstClocks { - firstClocks[i].(*testingClock).fire(firstFPR, timers.FastRecovery) + firstClocks[i].(*testingClock).fire(firstFPR, TimeoutFastRecovery) } activityMonitor.waitForActivity() activityMonitor.waitForQuiet() @@ -1260,20 +1260,20 @@ func TestAgreementFastRecoveryLate(t *testing.T) { restClocks[i].(*testingClock).prepareToFire() } for i := range restClocks { - restClocks[i].(*testingClock).fire(firstFPR, timers.FastRecovery) + restClocks[i].(*testingClock).fire(firstFPR, TimeoutFastRecovery) } activityMonitor.waitForActivity() activityMonitor.waitForQuiet() zeroes = expectNoNewPeriod(clocks, zeroes) - triggerGlobalTimeout(secondFPR, timers.FastRecovery, clocks, activityMonitor) + triggerGlobalTimeout(secondFPR, TimeoutFastRecovery, clocks, activityMonitor) zeroes = expectNewPeriod(clocks, zeroes) } // terminate on period 1 { baseNetwork.repairAll() - triggerGlobalTimeout(FilterTimeout(1, version), timers.Filter, clocks, activityMonitor) + triggerGlobalTimeout(FilterTimeout(1, version), TimeoutFilter, clocks, activityMonitor) zeroes = expectNewPeriod(clocks, zeroes) } @@ -1327,7 +1327,7 @@ func TestAgreementFastRecoveryRedo(t *testing.T) { pocket := make(chan multicastParams, 100) closeFn := baseNetwork.pocketAllCertVotes(pocket) baseNetwork.dropAllSlowNextVotes() - triggerGlobalTimeout(FilterTimeout(0, version), timers.Filter, clocks, activityMonitor) + triggerGlobalTimeout(FilterTimeout(0, version), TimeoutFilter, clocks, activityMonitor) zeroes = expectNoNewPeriod(clocks, zeroes) closeFn() @@ -1348,10 +1348,10 @@ func TestAgreementFastRecoveryRedo(t *testing.T) { } } - triggerGlobalTimeout(deadlineTimeout, timers.Deadline, clocks, activityMonitor) + triggerGlobalTimeout(deadlineTimeout, TimeoutDeadline, clocks, activityMonitor) zeroes = expectNoNewPeriod(clocks, zeroes) - triggerGlobalTimeout(0, timers.FastRecovery, clocks, activityMonitor) // activates fast partition recovery timer + triggerGlobalTimeout(0, TimeoutFastRecovery, clocks, activityMonitor) // activates fast partition recovery timer zeroes = expectNoNewPeriod(clocks, zeroes) baseNetwork.dropAllVotes() @@ -1362,7 +1362,7 @@ func TestAgreementFastRecoveryRedo(t *testing.T) { firstClocks[i].(*testingClock).prepareToFire() } for i := range firstClocks { - firstClocks[i].(*testingClock).fire(firstFPR, timers.FastRecovery) + firstClocks[i].(*testingClock).fire(firstFPR, TimeoutFastRecovery) } activityMonitor.waitForActivity() activityMonitor.waitForQuiet() @@ -1373,26 +1373,26 @@ func TestAgreementFastRecoveryRedo(t *testing.T) { restClocks[i].(*testingClock).prepareToFire() } for i := range restClocks { - restClocks[i].(*testingClock).fire(firstFPR, timers.FastRecovery) + restClocks[i].(*testingClock).fire(firstFPR, TimeoutFastRecovery) } activityMonitor.waitForActivity() activityMonitor.waitForQuiet() zeroes = expectNoNewPeriod(clocks, zeroes) - triggerGlobalTimeout(secondFPR, timers.FastRecovery, clocks, activityMonitor) + triggerGlobalTimeout(secondFPR, TimeoutFastRecovery, clocks, activityMonitor) zeroes = expectNewPeriod(clocks, zeroes) } // fail period 1 with value again { baseNetwork.dropAllVotes() - triggerGlobalTimeout(FilterTimeout(1, version), timers.Filter, clocks, activityMonitor) + triggerGlobalTimeout(FilterTimeout(1, version), TimeoutFilter, clocks, activityMonitor) zeroes = expectNoNewPeriod(clocks, zeroes) - triggerGlobalTimeout(deadlineTimeout, timers.Deadline, clocks, activityMonitor) + triggerGlobalTimeout(deadlineTimeout, TimeoutDeadline, clocks, activityMonitor) zeroes = expectNoNewPeriod(clocks, zeroes) - triggerGlobalTimeout(0, timers.FastRecovery, clocks, activityMonitor) // activates fast partition recovery timer + triggerGlobalTimeout(0, TimeoutFastRecovery, clocks, activityMonitor) // activates fast partition recovery timer zeroes = expectNoNewPeriod(clocks, zeroes) baseNetwork.dropAllVotes() @@ -1403,7 +1403,7 @@ func TestAgreementFastRecoveryRedo(t *testing.T) { firstClocks[i].(*testingClock).prepareToFire() } for i := range firstClocks { - firstClocks[i].(*testingClock).fire(firstFPR, timers.FastRecovery) + firstClocks[i].(*testingClock).fire(firstFPR, TimeoutFastRecovery) } activityMonitor.waitForActivity() activityMonitor.waitForQuiet() @@ -1414,20 +1414,20 @@ func TestAgreementFastRecoveryRedo(t *testing.T) { restClocks[i].(*testingClock).prepareToFire() } for i := range restClocks { - restClocks[i].(*testingClock).fire(firstFPR, timers.FastRecovery) + restClocks[i].(*testingClock).fire(firstFPR, TimeoutFastRecovery) } activityMonitor.waitForActivity() activityMonitor.waitForQuiet() zeroes = expectNoNewPeriod(clocks, zeroes) - triggerGlobalTimeout(secondFPR, timers.FastRecovery, clocks, activityMonitor) + triggerGlobalTimeout(secondFPR, TimeoutFastRecovery, clocks, activityMonitor) zeroes = expectNewPeriod(clocks, zeroes) } // terminate on period 2 { baseNetwork.repairAll() - triggerGlobalTimeout(FilterTimeout(2, version), timers.Filter, clocks, activityMonitor) + triggerGlobalTimeout(FilterTimeout(2, version), TimeoutFilter, clocks, activityMonitor) zeroes = expectNewPeriod(clocks, zeroes) } @@ -1478,26 +1478,26 @@ func TestAgreementBlockReplayBug_b29ea57(t *testing.T) { // fail period 0 { baseNetwork.dropAllSoftVotes() - triggerGlobalTimeout(FilterTimeout(0, version), timers.Filter, clocks, activityMonitor) + triggerGlobalTimeout(FilterTimeout(0, version), TimeoutFilter, clocks, activityMonitor) zeroes = expectNoNewPeriod(clocks, zeroes) - triggerGlobalTimeout(deadlineTimeout, timers.Deadline, clocks, activityMonitor) + triggerGlobalTimeout(deadlineTimeout, TimeoutDeadline, clocks, activityMonitor) zeroes = expectNewPeriod(clocks, zeroes) } // fail period 1 on bottom with block { - triggerGlobalTimeout(FilterTimeout(1, version), timers.Filter, clocks, activityMonitor) + triggerGlobalTimeout(FilterTimeout(1, version), TimeoutFilter, clocks, activityMonitor) zeroes = expectNoNewPeriod(clocks, zeroes) - triggerGlobalTimeout(deadlineTimeout, timers.Deadline, clocks, activityMonitor) + triggerGlobalTimeout(deadlineTimeout, TimeoutDeadline, clocks, activityMonitor) zeroes = expectNewPeriod(clocks, zeroes) } // terminate on period 2 { baseNetwork.repairAll() - triggerGlobalTimeout(FilterTimeout(2, version), timers.Filter, clocks, activityMonitor) + triggerGlobalTimeout(FilterTimeout(2, version), TimeoutFilter, clocks, activityMonitor) zeroes = expectNewPeriod(clocks, zeroes) } @@ -1538,12 +1538,12 @@ func TestAgreementLateCertBug(t *testing.T) { pocket := make(chan multicastParams, 100) { closeFn := baseNetwork.pocketAllCertVotes(pocket) - triggerGlobalTimeout(FilterTimeout(0, version), timers.Filter, clocks, activityMonitor) + triggerGlobalTimeout(FilterTimeout(0, version), TimeoutFilter, clocks, activityMonitor) zeroes = expectNoNewPeriod(clocks, zeroes) closeFn() baseNetwork.repairAll() - triggerGlobalTimeout(deadlineTimeout, timers.Deadline, clocks, activityMonitor) + triggerGlobalTimeout(deadlineTimeout, TimeoutDeadline, clocks, activityMonitor) zeroes = expectNewPeriod(clocks, zeroes) } @@ -1598,7 +1598,7 @@ func TestAgreementRecoverGlobalStartingValue(t *testing.T) { pocket := make(chan multicastParams, 100) closeFn := baseNetwork.pocketAllCertVotes(pocket) - triggerGlobalTimeout(FilterTimeout(0, version), timers.Filter, clocks, activityMonitor) + triggerGlobalTimeout(FilterTimeout(0, version), TimeoutFilter, clocks, activityMonitor) zeroes = expectNoNewPeriod(clocks, zeroes) closeFn() @@ -1619,7 +1619,7 @@ func TestAgreementRecoverGlobalStartingValue(t *testing.T) { } } - triggerGlobalTimeout(deadlineTimeout, timers.Deadline, clocks, activityMonitor) + triggerGlobalTimeout(deadlineTimeout, TimeoutDeadline, clocks, activityMonitor) zeroes = expectNewPeriod(clocks, zeroes) require.Equal(t, 4, int(zeroes)) } @@ -1629,7 +1629,7 @@ func TestAgreementRecoverGlobalStartingValue(t *testing.T) { pocket := make(chan multicastParams, 100) closeFn := baseNetwork.pocketAllCertVotes(pocket) - triggerGlobalTimeout(FilterTimeout(1, version), timers.Filter, clocks, activityMonitor) + triggerGlobalTimeout(FilterTimeout(1, version), TimeoutFilter, clocks, activityMonitor) zeroes = expectNoNewPeriod(clocks, zeroes) closeFn() @@ -1646,7 +1646,7 @@ func TestAgreementRecoverGlobalStartingValue(t *testing.T) { } } - triggerGlobalTimeout(deadlineTimeout, timers.Deadline, clocks, activityMonitor) + triggerGlobalTimeout(deadlineTimeout, TimeoutDeadline, clocks, activityMonitor) zeroes = expectNewPeriod(clocks, zeroes) require.Equal(t, 5, int(zeroes)) } @@ -1655,7 +1655,7 @@ func TestAgreementRecoverGlobalStartingValue(t *testing.T) { // todo: make more transparent, I want to kow what v we agreed on { baseNetwork.repairAll() - triggerGlobalTimeout(FilterTimeout(2, version), timers.Filter, clocks, activityMonitor) + triggerGlobalTimeout(FilterTimeout(2, version), TimeoutFilter, clocks, activityMonitor) zeroes = expectNewPeriod(clocks, zeroes) require.Equal(t, 6, int(zeroes)) } @@ -1697,7 +1697,7 @@ func TestAgreementRecoverGlobalStartingValueBadProposal(t *testing.T) { { pocket := make(chan multicastParams, 100) closeFn := baseNetwork.pocketAllCertVotes(pocket) - triggerGlobalTimeout(FilterTimeout(0, version), timers.Filter, clocks, activityMonitor) + triggerGlobalTimeout(FilterTimeout(0, version), TimeoutFilter, clocks, activityMonitor) zeroes = expectNoNewPeriod(clocks, zeroes) closeFn() @@ -1724,7 +1724,7 @@ func TestAgreementRecoverGlobalStartingValueBadProposal(t *testing.T) { } return params }) - triggerGlobalTimeout(deadlineTimeout, timers.Deadline, clocks, activityMonitor) + triggerGlobalTimeout(deadlineTimeout, TimeoutDeadline, clocks, activityMonitor) zeroes = expectNewPeriod(clocks, zeroes) require.Equal(t, 4, int(zeroes)) } @@ -1734,7 +1734,7 @@ func TestAgreementRecoverGlobalStartingValueBadProposal(t *testing.T) { baseNetwork.repairAll() pocket := make(chan multicastParams, 100) closeFn := baseNetwork.pocketAllCertVotes(pocket) - triggerGlobalTimeout(FilterTimeout(1, version), timers.Filter, clocks, activityMonitor) + triggerGlobalTimeout(FilterTimeout(1, version), TimeoutFilter, clocks, activityMonitor) zeroes = expectNoNewPeriod(clocks, zeroes) closeFn() @@ -1750,7 +1750,7 @@ func TestAgreementRecoverGlobalStartingValueBadProposal(t *testing.T) { panic(errstr) } } - triggerGlobalTimeout(deadlineTimeout, timers.Deadline, clocks, activityMonitor) + triggerGlobalTimeout(deadlineTimeout, TimeoutDeadline, clocks, activityMonitor) zeroes = expectNewPeriod(clocks, zeroes) } @@ -1758,7 +1758,7 @@ func TestAgreementRecoverGlobalStartingValueBadProposal(t *testing.T) { // Finish in period 2 { baseNetwork.repairAll() - triggerGlobalTimeout(FilterTimeout(2, version), timers.Filter, clocks, activityMonitor) + triggerGlobalTimeout(FilterTimeout(2, version), TimeoutFilter, clocks, activityMonitor) zeroes = expectNewPeriod(clocks, zeroes) require.Equal(t, 6, int(zeroes)) } @@ -1800,7 +1800,7 @@ func TestAgreementRecoverBothVAndBotQuorums(t *testing.T) { { pocket := make(chan multicastParams, 100) closeFn := baseNetwork.pocketAllSoftVotes(pocket) - triggerGlobalTimeout(FilterTimeout(0, version), timers.Filter, clocks, activityMonitor) + triggerGlobalTimeout(FilterTimeout(0, version), TimeoutFilter, clocks, activityMonitor) zeroes = expectNoNewPeriod(clocks, zeroes) closeFn() pocketedSoft := make([]multicastParams, len(pocket)) @@ -1825,7 +1825,7 @@ func TestAgreementRecoverBothVAndBotQuorums(t *testing.T) { } // generate a bottom quorum; let only one node see it. baseNetwork.crown(0) - triggerGlobalTimeout(deadlineTimeout, timers.Deadline, clocks, activityMonitor) + triggerGlobalTimeout(deadlineTimeout, TimeoutDeadline, clocks, activityMonitor) if clocks[0].(*testingClock).zeroes != zeroes+1 { errstr := fmt.Sprintf("node 0 did not enter new period from bot quorum") panic(errstr) @@ -1844,12 +1844,12 @@ func TestAgreementRecoverBothVAndBotQuorums(t *testing.T) { // actually create the value quorum _, upper := (next).nextVoteRanges() - triggerGlobalTimeout(upper, timers.Deadline, clocks[1:], activityMonitor) // activates next timers + triggerGlobalTimeout(upper, TimeoutDeadline, clocks[1:], activityMonitor) // activates next timers zeroes = expectNoNewPeriod(clocks[1:], zeroes) lower, upper := (next + 1).nextVoteRanges() delta := time.Duration(testingRand{}.Uint64() % uint64(upper-lower)) - triggerGlobalTimeout(lower+delta, timers.Deadline, clocks[1:], activityMonitor) + triggerGlobalTimeout(lower+delta, TimeoutDeadline, clocks[1:], activityMonitor) zeroes = expectNewPeriod(clocks, zeroes) require.Equal(t, 4, int(zeroes)) } @@ -1859,7 +1859,7 @@ func TestAgreementRecoverBothVAndBotQuorums(t *testing.T) { baseNetwork.repairAll() pocket := make(chan multicastParams, 100) closeFn := baseNetwork.pocketAllCertVotes(pocket) - triggerGlobalTimeout(FilterTimeout(1, version), timers.Filter, clocks, activityMonitor) + triggerGlobalTimeout(FilterTimeout(1, version), TimeoutFilter, clocks, activityMonitor) zeroes = expectNoNewPeriod(clocks, zeroes) closeFn() @@ -1876,14 +1876,14 @@ func TestAgreementRecoverBothVAndBotQuorums(t *testing.T) { } } - triggerGlobalTimeout(deadlineTimeout, timers.Deadline, clocks, activityMonitor) + triggerGlobalTimeout(deadlineTimeout, TimeoutDeadline, clocks, activityMonitor) zeroes = expectNewPeriod(clocks, zeroes) } // Finish in period 2 { baseNetwork.repairAll() - triggerGlobalTimeout(FilterTimeout(2, version), timers.Filter, clocks, activityMonitor) + triggerGlobalTimeout(FilterTimeout(2, version), TimeoutFilter, clocks, activityMonitor) zeroes = expectNewPeriod(clocks, zeroes) require.Equal(t, 6, int(zeroes)) } @@ -1924,13 +1924,13 @@ func TestAgreementSlowPayloadsPreDeadline(t *testing.T) { pocket := make(chan multicastParams, 100) closeFn := baseNetwork.pocketAllCompound(pocket) // (takes effect next round) { - triggerGlobalTimeout(FilterTimeout(0, version), timers.Filter, clocks, activityMonitor) + triggerGlobalTimeout(FilterTimeout(0, version), TimeoutFilter, clocks, activityMonitor) zeroes = expectNewPeriod(clocks, zeroes) } // run round with late payload { - triggerGlobalTimeout(FilterTimeout(0, version), timers.Filter, clocks, activityMonitor) + triggerGlobalTimeout(FilterTimeout(0, version), TimeoutFilter, clocks, activityMonitor) zeroes = expectNoNewPeriod(clocks, zeroes) // release payloads; expect new round @@ -1982,15 +1982,15 @@ func TestAgreementSlowPayloadsPostDeadline(t *testing.T) { pocket := make(chan multicastParams, 100) closeFn := baseNetwork.pocketAllCompound(pocket) // (takes effect next round) { - triggerGlobalTimeout(FilterTimeout(0, version), timers.Filter, clocks, activityMonitor) + triggerGlobalTimeout(FilterTimeout(0, version), TimeoutFilter, clocks, activityMonitor) zeroes = expectNewPeriod(clocks, zeroes) } // force network into period 1 by delaying proposals { - triggerGlobalTimeout(FilterTimeout(0, version), timers.Filter, clocks, activityMonitor) + triggerGlobalTimeout(FilterTimeout(0, version), TimeoutFilter, clocks, activityMonitor) zeroes = expectNoNewPeriod(clocks, zeroes) - triggerGlobalTimeout(deadlineTimeout, timers.Deadline, clocks, activityMonitor) + triggerGlobalTimeout(deadlineTimeout, TimeoutDeadline, clocks, activityMonitor) zeroes = expectNewPeriod(clocks, zeroes) } @@ -2007,7 +2007,7 @@ func TestAgreementSlowPayloadsPostDeadline(t *testing.T) { activityMonitor.waitForQuiet() zeroes = expectNoNewPeriod(clocks, zeroes) - triggerGlobalTimeout(FilterTimeout(1, version), timers.Filter, clocks, activityMonitor) + triggerGlobalTimeout(FilterTimeout(1, version), TimeoutFilter, clocks, activityMonitor) zeroes = expectNewPeriod(clocks, zeroes) } @@ -2047,11 +2047,11 @@ func TestAgreementLargePeriods(t *testing.T) { for p := 0; p < 60; p++ { { baseNetwork.partition(0, 1, 2) - triggerGlobalTimeout(FilterTimeout(period(p), version), timers.Filter, clocks, activityMonitor) + triggerGlobalTimeout(FilterTimeout(period(p), version), TimeoutFilter, clocks, activityMonitor) zeroes = expectNoNewPeriod(clocks, zeroes) baseNetwork.repairAll() - triggerGlobalTimeout(deadlineTimeout, timers.Deadline, clocks, activityMonitor) + triggerGlobalTimeout(deadlineTimeout, TimeoutDeadline, clocks, activityMonitor) zeroes = expectNewPeriod(clocks, zeroes) require.Equal(t, 4+p, int(zeroes)) } @@ -2059,7 +2059,7 @@ func TestAgreementLargePeriods(t *testing.T) { // terminate { - triggerGlobalTimeout(FilterTimeout(60, version), timers.Filter, clocks, activityMonitor) + triggerGlobalTimeout(FilterTimeout(60, version), TimeoutFilter, clocks, activityMonitor) zeroes = expectNewPeriod(clocks, zeroes) } @@ -2146,13 +2146,13 @@ func TestAgreementRegression_WrongPeriodPayloadVerificationCancellation_8ba23942 ch := validator.suspend() closeFn := baseNetwork.pocketAllCompound(pocket0) // (takes effect next round) { - triggerGlobalTimeout(FilterTimeout(0, version), timers.Filter, clocks, activityMonitor) + triggerGlobalTimeout(FilterTimeout(0, version), TimeoutFilter, clocks, activityMonitor) zeroes = expectNewPeriod(clocks, zeroes) } // force network into period 1 by failing period 0, entering with bottom and no soft threshold (to prevent proposal value pinning) baseNetwork.dropAllSoftVotes() - triggerGlobalTimeout(FilterTimeout(0, version), timers.Filter, clocks, activityMonitor) + triggerGlobalTimeout(FilterTimeout(0, version), TimeoutFilter, clocks, activityMonitor) zeroes = expectNoNewPeriod(clocks, zeroes) // resume delivery of payloads in following period @@ -2163,7 +2163,7 @@ func TestAgreementRegression_WrongPeriodPayloadVerificationCancellation_8ba23942 // release proposed blocks in a controlled manner to prevent oversubscription of verification pocket1 := make(chan multicastParams, 100) closeFn = baseNetwork.pocketAllCompound(pocket1) - triggerGlobalTimeout(deadlineTimeout, timers.Deadline, clocks, activityMonitor) + triggerGlobalTimeout(deadlineTimeout, TimeoutDeadline, clocks, activityMonitor) baseNetwork.repairAll() close(pocket1) { @@ -2347,7 +2347,7 @@ func TestAgreementCertificateDoesNotStallSingleRelay(t *testing.T) { return params }) // And with some hypothetical second relay the network achieves consensus on a certificate and block. - triggerGlobalTimeout(FilterTimeout(0, version), timers.Filter, clocks, activityMonitor) + triggerGlobalTimeout(FilterTimeout(0, version), TimeoutFilter, clocks, activityMonitor) zeroes = expectNewPeriod(clocks[1:], zeroes) require.Equal(t, uint(3), clocks[0].(*testingClock).zeroes) close(pocketCert) @@ -2366,7 +2366,7 @@ func TestAgreementCertificateDoesNotStallSingleRelay(t *testing.T) { activityMonitor.waitForQuiet() // this relay must still relay initial messages. Note that payloads were already relayed with // the previous global timeout. - triggerGlobalTimeout(FilterTimeout(0, version), timers.Filter, clocks[1:], activityMonitor) + triggerGlobalTimeout(FilterTimeout(0, version), TimeoutFilter, clocks[1:], activityMonitor) zeroes = expectNewPeriod(clocks[1:], zeroes) require.Equal(t, uint(3), clocks[0].(*testingClock).zeroes) diff --git a/node/node.go b/node/node.go index 477c1b794c..1220751877 100644 --- a/node/node.go +++ b/node/node.go @@ -254,11 +254,11 @@ func MakeFull(log logging.Logger, rootDir string, cfg config.Local, phonebookAdd blockValidator := blockValidatorImpl{l: node.ledger, verificationPool: node.highPriorityCryptoVerificationPool} agreementLedger := makeAgreementLedger(node.ledger, node.net) - var agreementClock timers.Clock + var agreementClock timers.Clock[agreement.TimeoutType] if node.devMode { - agreementClock = timers.MakeFrozenClock() + agreementClock = timers.MakeFrozenClock[agreement.TimeoutType]() } else { - agreementClock = timers.MakeMonotonicClock(time.Now()) + agreementClock = timers.MakeMonotonicClock[agreement.TimeoutType](time.Now()) } agreementParameters := agreement.Parameters{ Logger: log, diff --git a/util/timers/frozen.go b/util/timers/frozen.go index db5f068c3d..f4400c3a82 100644 --- a/util/timers/frozen.go +++ b/util/timers/frozen.go @@ -21,42 +21,42 @@ import ( ) // Frozen is a dummy frozen clock that never fires. -type Frozen struct { +type Frozen[TimeoutType comparable] struct { timeoutCh chan time.Time } // MakeFrozenClock creates a new frozen clock. -func MakeFrozenClock() Clock { - return &Frozen{ +func MakeFrozenClock[TimeoutType comparable]() Clock[TimeoutType] { + return &Frozen[TimeoutType]{ timeoutCh: make(chan time.Time, 1), } } // Zero returns a new Clock reset to the current time. -func (m *Frozen) Zero() Clock { - return MakeFrozenClock() +func (m *Frozen[TimeoutType]) Zero() Clock[TimeoutType] { + return MakeFrozenClock[TimeoutType]() } // TimeoutAt returns a channel that will signal when the duration has elapsed. -func (m *Frozen) TimeoutAt(delta time.Duration, timeoutType TimeoutType) <-chan time.Time { +func (m *Frozen[TimeoutType]) TimeoutAt(delta time.Duration, timeoutType TimeoutType) <-chan time.Time { return m.timeoutCh } // Encode implements Clock.Encode. -func (m *Frozen) Encode() []byte { +func (m *Frozen[TimeoutType]) Encode() []byte { return []byte{} } // Decode implements Clock.Decode. -func (m *Frozen) Decode([]byte) (Clock, error) { - return MakeFrozenClock(), nil +func (m *Frozen[TimeoutType]) Decode([]byte) (Clock[TimeoutType], error) { + return MakeFrozenClock[TimeoutType](), nil } -func (m *Frozen) String() string { +func (m *Frozen[TimeoutType]) String() string { return "" } // Since implements the Clock interface. -func (m *Frozen) Since() time.Duration { +func (m *Frozen[TimeoutType]) Since() time.Duration { return 0 } diff --git a/util/timers/interface.go b/util/timers/interface.go index d8af682e4d..d217437e9e 100644 --- a/util/timers/interface.go +++ b/util/timers/interface.go @@ -21,19 +21,11 @@ import ( "time" ) -type TimeoutType int8 - -const ( - Deadline TimeoutType = iota - FastRecovery - Filter -) - // Clock provides timeout events which fire at some point after a point in time. -type Clock interface { +type Clock[TimeoutType comparable] interface { // Zero returns a reset Clock. TimeoutAt channels will use the point // at which Zero was called as their reference point. - Zero() Clock + Zero() Clock[TimeoutType] // Since returns the time spent between the last time the clock was zeroed out and the current // wall clock time. @@ -54,5 +46,5 @@ type Clock interface { // Decode deserializes the Clock from a byte slice. // A Clock which has been Decoded from an Encoded Clock should produce // the same timeouts as the original Clock. - Decode([]byte) (Clock, error) + Decode([]byte) (Clock[TimeoutType], error) } diff --git a/util/timers/monotonic.go b/util/timers/monotonic.go index dc12d2f076..1f84a7e6ce 100644 --- a/util/timers/monotonic.go +++ b/util/timers/monotonic.go @@ -24,27 +24,27 @@ import ( ) // Monotonic uses the system's monotonic clock to emit timeouts. -type Monotonic struct { +type Monotonic[TimeoutType comparable] struct { zero time.Time timeouts map[TimeoutType]map[time.Duration]<-chan time.Time } // MakeMonotonicClock creates a new monotonic clock with a given zero point. -func MakeMonotonicClock(zero time.Time) Clock { - return &Monotonic{ +func MakeMonotonicClock[TimeoutType comparable](zero time.Time) Clock[TimeoutType] { + return &Monotonic[TimeoutType]{ zero: zero, } } // Zero returns a new Clock reset to the current time. -func (m *Monotonic) Zero() Clock { +func (m *Monotonic[TimeoutType]) Zero() Clock[TimeoutType] { z := time.Now() logging.Base().Debugf("Clock zeroed to %v", z) - return MakeMonotonicClock(z) + return MakeMonotonicClock[TimeoutType](z) } // TimeoutAt returns a channel that will signal when the duration has elapsed. -func (m *Monotonic) TimeoutAt(delta time.Duration, timeoutType TimeoutType) <-chan time.Time { +func (m *Monotonic[TimeoutType]) TimeoutAt(delta time.Duration, timeoutType TimeoutType) <-chan time.Time { if m.timeouts == nil { m.timeouts = make(map[TimeoutType]map[time.Duration]<-chan time.Time) } @@ -72,12 +72,12 @@ func (m *Monotonic) TimeoutAt(delta time.Duration, timeoutType TimeoutType) <-ch } // Encode implements Clock.Encode. -func (m *Monotonic) Encode() []byte { +func (m *Monotonic[TimeoutType]) Encode() []byte { return protocol.EncodeReflect(m.zero) } // Decode implements Clock.Decode. -func (m *Monotonic) Decode(data []byte) (Clock, error) { +func (m *Monotonic[TimeoutType]) Decode(data []byte) (Clock[TimeoutType], error) { var zero time.Time err := protocol.DecodeReflect(data, &zero) if err == nil { @@ -85,14 +85,14 @@ func (m *Monotonic) Decode(data []byte) (Clock, error) { } else { logging.Base().Errorf("Clock decoded with zero at %v (err: %v)", zero, err) } - return MakeMonotonicClock(zero), err + return MakeMonotonicClock[TimeoutType](zero), err } -func (m *Monotonic) String() string { +func (m *Monotonic[TimeoutType]) String() string { return time.Time(m.zero).String() } // Since returns the time that has passed between the time the clock was last zeroed out and now -func (m *Monotonic) Since() time.Duration { +func (m *Monotonic[TimeoutType]) Since() time.Duration { return time.Since(m.zero) } diff --git a/util/timers/monotonic_test.go b/util/timers/monotonic_test.go index 1ca9bd47c0..912f1b8425 100644 --- a/util/timers/monotonic_test.go +++ b/util/timers/monotonic_test.go @@ -36,14 +36,14 @@ func polled(ch <-chan time.Time) bool { func TestMonotonicDelta(t *testing.T) { partitiontest.PartitionTest(t) - var m Monotonic - var c Clock + var m Monotonic[int] + var c Clock[int] var ch <-chan time.Time d := time.Millisecond * 100 c = m.Zero() - ch = c.TimeoutAt(d, Filter) + ch = c.TimeoutAt(d, 0) if polled(ch) { t.Errorf("channel fired ~100ms early") } @@ -53,7 +53,7 @@ func TestMonotonicDelta(t *testing.T) { t.Errorf("channel failed to fire at 100ms") } - ch = c.TimeoutAt(d/2, Filter) + ch = c.TimeoutAt(d/2, 0) if !polled(ch) { t.Errorf("channel failed to fire at 50ms") } @@ -62,12 +62,12 @@ func TestMonotonicDelta(t *testing.T) { func TestMonotonicZeroDelta(t *testing.T) { partitiontest.PartitionTest(t) - var m Monotonic - var c Clock + var m Monotonic[int] + var c Clock[int] var ch <-chan time.Time c = m.Zero() - ch = c.TimeoutAt(0, Filter) + ch = c.TimeoutAt(0, 0) if !polled(ch) { t.Errorf("read failed on channel at zero timeout") } @@ -76,12 +76,12 @@ func TestMonotonicZeroDelta(t *testing.T) { func TestMonotonicNegativeDelta(t *testing.T) { partitiontest.PartitionTest(t) - var m Monotonic - var c Clock + var m Monotonic[int] + var c Clock[int] var ch <-chan time.Time c = m.Zero() - ch = c.TimeoutAt(-time.Second, Filter) + ch = c.TimeoutAt(-time.Second, 0) if !polled(ch) { t.Errorf("read failed on channel at negative timeout") } @@ -90,14 +90,14 @@ func TestMonotonicNegativeDelta(t *testing.T) { func TestMonotonicZeroTwice(t *testing.T) { partitiontest.PartitionTest(t) - var m Monotonic - var c Clock + var m Monotonic[int] + var c Clock[int] var ch <-chan time.Time d := time.Millisecond * 100 c = m.Zero() - ch = c.TimeoutAt(d, Filter) + ch = c.TimeoutAt(d, 0) if polled(ch) { t.Errorf("channel fired ~100ms early") } @@ -108,7 +108,7 @@ func TestMonotonicZeroTwice(t *testing.T) { } c = c.Zero() - ch = c.TimeoutAt(d, Filter) + ch = c.TimeoutAt(d, 0) if polled(ch) { t.Errorf("channel fired ~100ms early after call to Zero") } @@ -122,21 +122,21 @@ func TestMonotonicZeroTwice(t *testing.T) { func TestMonotonicEncodeDecode(t *testing.T) { partitiontest.PartitionTest(t) - singleTest := func(c Clock, descr string) { + singleTest := func(c Clock[int], descr string) { data := c.Encode() c0, err := c.Decode(data) if err != nil { t.Errorf("decoding error: %v", err) } - if !time.Time(c.(*Monotonic).zero).Equal(time.Time(c0.(*Monotonic).zero)) { + if !time.Time(c.(*Monotonic[int]).zero).Equal(time.Time(c0.(*Monotonic[int]).zero)) { t.Errorf("%v clock not encoded properly: %v != %v", descr, c, c0) } } - var c Clock - var m Monotonic + var c Clock[int] + var m Monotonic[int] - c = Clock(&m) + c = Clock[int](&m) singleTest(c, "empty") c = c.Zero() @@ -145,8 +145,8 @@ func TestMonotonicEncodeDecode(t *testing.T) { now := time.Now() for i := 0; i < 100; i++ { r := time.Duration(rand.Int63()) - c = Clock( - &Monotonic{ + c = Clock[int]( + &Monotonic[int]{ zero: now.Add(r), }, ) @@ -157,14 +157,14 @@ func TestMonotonicEncodeDecode(t *testing.T) { func TestTimeoutTypes(t *testing.T) { partitiontest.PartitionTest(t) - var m Monotonic - var c Clock + var m Monotonic[int] + var c Clock[int] d := time.Millisecond * 100 c = m.Zero() - ch1 := c.TimeoutAt(d, Filter) - ch2 := c.TimeoutAt(d, FastRecovery) + ch1 := c.TimeoutAt(d, 0) + ch2 := c.TimeoutAt(d, 1) if polled(ch1) { t.Errorf("channel fired ~100ms early") } @@ -184,11 +184,11 @@ func TestTimeoutTypes(t *testing.T) { t.Errorf("channel failed to fire at 100ms") } - ch1 = c.TimeoutAt(d/2, Filter) + ch1 = c.TimeoutAt(d/2, 0) if !polled(ch1) { t.Errorf("channel failed to fire at 50ms") } - ch2 = c.TimeoutAt(d/2, Filter) + ch2 = c.TimeoutAt(d/2, 0) if !polled(ch2) { t.Errorf("channel failed to fire at 50ms") } From c3371fd1dd03e775c0117f039f0f426c7f6a85a3 Mon Sep 17 00:00:00 2001 From: chris erway Date: Wed, 2 Aug 2023 13:38:19 -0400 Subject: [PATCH 012/232] add some pieces of dynamic lambda from feature/pipeline into agreement --- agreement/player.go | 50 ++++++++++++++++++++++++++++++++++++++++++--- config/consensus.go | 19 +++++++++++++++++ 2 files changed, 66 insertions(+), 3 deletions(-) diff --git a/agreement/player.go b/agreement/player.go index 86627ccded..a64f43ab08 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -17,6 +17,7 @@ package agreement import ( + "sort" "time" "github.com/algorand/go-algorand/config" @@ -59,6 +60,8 @@ type player struct { // Pending holds the player's proposalTable, which stores proposals that // must be verified after some vote has been verified. Pending proposalTable + + payloadArrivals []time.Duration } func (p *player) T() stateMachineTag { @@ -270,6 +273,45 @@ func (p *player) handleCheckpointEvent(r routerHandle, e checkpointEvent) []acti }} } +func (p *player) handleWinningPayloadArrival(payload proposal) { + // ignoring validatedAt + p.payloadArrivals = append(p.payloadArrivals, payload.receivedAt) +} + +// calculateFilterTimeout chooses the appropriate filter timeout for a new round. +func (p *player) calculateFilterTimeout(period period, ver protocol.ConsensusVersion) time.Duration { + proto := config.Consensus[ver] + + if !proto.DynamicFilterTimeout || period != 0 { + return FilterTimeout(period, ver) + } + + var dynamicDelay time.Duration + if proto.AgreementPipelineDelay <= 0 { + dynamicDelay = 0 + } else if proto.AgreementPipelineDelay > len(p.payloadArrivals) { + dynamicDelay = FilterTimeout(0, ver) + } else { + sortedArrivals := make([]time.Duration, len(p.payloadArrivals)) + copy(sortedArrivals[:], p.payloadArrivals[:]) + sort.Slice(sortedArrivals, func(i, j int) bool { return sortedArrivals[i] < sortedArrivals[j] }) + dynamicDelay = sortedArrivals[proto.AgreementPipelineDelay-1] + } + + // Make sure the dynamic delay is not too small; we want to + // evenly space out the pipelined rounds across FilterTimeout, + // which is the fastest we could agree on blocks anyway (not + // including the soft vote / cert vote times). + if proto.AgreementPipelineDepth > 0 { + evenSpacing := FilterTimeout(0, ver) / time.Duration(proto.AgreementPipelineDepth) + if dynamicDelay < evenSpacing { + dynamicDelay = evenSpacing + } + } + + return dynamicDelay +} + func (p *player) handleThresholdEvent(r routerHandle, e thresholdEvent) []action { r.t.timeR().RecThreshold(e) @@ -285,6 +327,7 @@ func (p *player) handleThresholdEvent(r routerHandle, e thresholdEvent) []action cert := Certificate(e.Bundle) a0 := ensureAction{Payload: res.Payload, Certificate: cert} actions = append(actions, a0) + p.handleWinningPayloadArrival(res.Payload) as := p.enterRound(r, e, p.Round+1) return append(actions, as...) } @@ -386,13 +429,13 @@ func (p *player) enterRound(r routerHandle, source event, target round) []action switch source := source.(type) { case roundInterruptionEvent: - p.Deadline.Duration = FilterTimeout(0, source.Proto.Version) + p.Deadline.Duration = calculateFilterTimeout(0, source.Proto.Version) p.Deadline.Type = timers.Filter case thresholdEvent: - p.Deadline.Duration = FilterTimeout(0, source.Proto) + p.Deadline.Duration = calculateFilterTimeout(0, source.Proto) p.Deadline.Type = timers.Filter case filterableMessageEvent: - p.Deadline.Duration = FilterTimeout(0, source.Proto.Version) + p.Deadline.Duration = calculateFilterTimeout(0, source.Proto.Version) p.Deadline.Type = timers.Filter } @@ -608,6 +651,7 @@ func (p *player) handleMessageEvent(r routerHandle, e messageEvent) (actions []a cert := Certificate(freshestRes.Event.Bundle) a0 := ensureAction{Payload: e.Input.Proposal, Certificate: cert} actions = append(actions, a0) + p.handleWinningPayloadArrival(e.Input.Proposal) as := p.enterRound(r, delegatedE, cert.Round+1) return append(actions, as...) } diff --git a/config/consensus.go b/config/consensus.go index 8c217c18c1..1eed8b8c11 100644 --- a/config/consensus.go +++ b/config/consensus.go @@ -516,6 +516,25 @@ type ConsensusParams struct { // used by agreement for Circulation, and updates the calculation of StateProofOnlineTotalWeight used // by state proofs to use the same method (rather than excluding stake from the top N stakeholders as before). ExcludeExpiredCirculation bool + + // DynamicFilterTimeout + DynamicFilterTimeout bool + + // AgreementPipelineDepth specifies the maximum number of pipelined + // rounds that the agreement protocol can run ahead with. This is + // the maximum difference between the first uncommitted round and + // the round number that we run the agreement protocol for. + AgreementPipelineDepth int + + // AgreementPipelineDelayHistory specifies the number of past block arrivals + // that are measured to determine when to start pipelining the next block. + AgreementPipelineDelayHistory int + + // AgreementPipelineDelay specifies when the agreement code should start + // pipelining the next block, by choosing the delay time of the + // AgreementPipelineDelay'th slowest block to arrive out of the last + // AgreementPipelineDelayHistory lowest-credential block payloads. + AgreementPipelineDelay int } // PaysetCommitType enumerates possible ways for the block header to commit to From fc5877bdec1fd7cf0103d73af1e1e92b687a6bd6 Mon Sep 17 00:00:00 2001 From: chris erway Date: Thu, 3 Aug 2023 09:18:23 -0400 Subject: [PATCH 013/232] add/adapt code from pipelinePlayer.resizeArrivals on feature/pipeline --- agreement/player.go | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/agreement/player.go b/agreement/player.go index a64f43ab08..e6e54b88ba 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -273,9 +273,20 @@ func (p *player) handleCheckpointEvent(r routerHandle, e checkpointEvent) []acti }} } -func (p *player) handleWinningPayloadArrival(payload proposal) { +func (p *player) handleWinningPayloadArrival(payload proposal, ver protocol.ConsensusVersion) { // ignoring validatedAt p.payloadArrivals = append(p.payloadArrivals, payload.receivedAt) + p.resizePayloadArrivals(ver) +} + +func (p *player) resizePayloadArrivals(ver protocol.ConsensusVersion) { + proto := config.Consensus[ver] + if len(p.payloadArrivals) > proto.AgreementPipelineDelayHistory { + p.payloadArrivals = p.payloadArrivals[len(p.payloadArrivals)-proto.AgreementPipelineDelayHistory:] + } + for len(p.payloadArrivals) < proto.AgreementPipelineDelayHistory { + p.payloadArrivals = append([]time.Duration{FilterTimeout(0, ver)}, p.payloadArrivals...) + } } // calculateFilterTimeout chooses the appropriate filter timeout for a new round. @@ -327,7 +338,7 @@ func (p *player) handleThresholdEvent(r routerHandle, e thresholdEvent) []action cert := Certificate(e.Bundle) a0 := ensureAction{Payload: res.Payload, Certificate: cert} actions = append(actions, a0) - p.handleWinningPayloadArrival(res.Payload) + p.handleWinningPayloadArrival(res.Payload, e.Proto) as := p.enterRound(r, e, p.Round+1) return append(actions, as...) } @@ -651,7 +662,7 @@ func (p *player) handleMessageEvent(r routerHandle, e messageEvent) (actions []a cert := Certificate(freshestRes.Event.Bundle) a0 := ensureAction{Payload: e.Input.Proposal, Certificate: cert} actions = append(actions, a0) - p.handleWinningPayloadArrival(e.Input.Proposal) + p.handleWinningPayloadArrival(e.Input.Proposal, e.Proto.Version) as := p.enterRound(r, delegatedE, cert.Round+1) return append(actions, as...) } From b7f71465beb3811b7627138eadb06707129132b5 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Tue, 8 Aug 2023 10:57:06 -0400 Subject: [PATCH 014/232] checkpoint: param cleanup --- agreement/player.go | 35 +++++++++++++++++++---------------- config/consensus.go | 21 ++++++++------------- 2 files changed, 27 insertions(+), 29 deletions(-) diff --git a/agreement/player.go b/agreement/player.go index e6e54b88ba..bcda8831cc 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -17,6 +17,7 @@ package agreement import ( + "fmt" "sort" "time" @@ -281,45 +282,47 @@ func (p *player) handleWinningPayloadArrival(payload proposal, ver protocol.Cons func (p *player) resizePayloadArrivals(ver protocol.ConsensusVersion) { proto := config.Consensus[ver] - if len(p.payloadArrivals) > proto.AgreementPipelineDelayHistory { - p.payloadArrivals = p.payloadArrivals[len(p.payloadArrivals)-proto.AgreementPipelineDelayHistory:] + if len(p.payloadArrivals) > proto.DynamicFilterPayloadArriavalHistory { + p.payloadArrivals = p.payloadArrivals[len(p.payloadArrivals)-proto.DynamicFilterPayloadArriavalHistory:] } - for len(p.payloadArrivals) < proto.AgreementPipelineDelayHistory { + for len(p.payloadArrivals) < proto.DynamicFilterPayloadArriavalHistory { p.payloadArrivals = append([]time.Duration{FilterTimeout(0, ver)}, p.payloadArrivals...) } } // calculateFilterTimeout chooses the appropriate filter timeout for a new round. func (p *player) calculateFilterTimeout(period period, ver protocol.ConsensusVersion) time.Duration { + proto := config.Consensus[ver] if !proto.DynamicFilterTimeout || period != 0 { + // Either dynamic lambda is disabled, or we're not in period 0 and + // therefore can't use dynamic lambda return FilterTimeout(period, ver) } var dynamicDelay time.Duration - if proto.AgreementPipelineDelay <= 0 { - dynamicDelay = 0 - } else if proto.AgreementPipelineDelay > len(p.payloadArrivals) { + if proto.DynamicFilterPayloadArriavalHistory <= 0 { + // we don't keep any history, use the default + dynamicDelay = FilterTimeout(0, ver) + } else if proto.DynamicFilterPayloadArriavalHistory > len(p.payloadArrivals) { + // not enough smaples, use the default dynamicDelay = FilterTimeout(0, ver) } else { sortedArrivals := make([]time.Duration, len(p.payloadArrivals)) copy(sortedArrivals[:], p.payloadArrivals[:]) sort.Slice(sortedArrivals, func(i, j int) bool { return sortedArrivals[i] < sortedArrivals[j] }) - dynamicDelay = sortedArrivals[proto.AgreementPipelineDelay-1] + dynamicDelay = sortedArrivals[proto.DynamicFilterPayloadArriavalHistory-1] } - // Make sure the dynamic delay is not too small; we want to - // evenly space out the pipelined rounds across FilterTimeout, - // which is the fastest we could agree on blocks anyway (not - // including the soft vote / cert vote times). - if proto.AgreementPipelineDepth > 0 { - evenSpacing := FilterTimeout(0, ver) / time.Duration(proto.AgreementPipelineDepth) - if dynamicDelay < evenSpacing { - dynamicDelay = evenSpacing - } + // Make sure the dynamic delay is not too small + if dynamicDelay < proto.DynamicFilterTimeoutLowerBound { + dynamicDelay = proto.DynamicFilterTimeoutLowerBound } + //dynamicDelay = FilterTimeout(period, ver) + fmt.Println("dynamic delay", dynamicDelay, "payload arrivals", len(p.payloadArrivals)) + return dynamicDelay } diff --git a/config/consensus.go b/config/consensus.go index 1eed8b8c11..02cc03ee9e 100644 --- a/config/consensus.go +++ b/config/consensus.go @@ -520,21 +520,11 @@ type ConsensusParams struct { // DynamicFilterTimeout DynamicFilterTimeout bool - // AgreementPipelineDepth specifies the maximum number of pipelined - // rounds that the agreement protocol can run ahead with. This is - // the maximum difference between the first uncommitted round and - // the round number that we run the agreement protocol for. - AgreementPipelineDepth int + DynamicFilterTimeoutLowerBound time.Duration - // AgreementPipelineDelayHistory specifies the number of past block arrivals + // DynamicFilterPayloadArriavalHistory specifies the number of past block arrivals // that are measured to determine when to start pipelining the next block. - AgreementPipelineDelayHistory int - - // AgreementPipelineDelay specifies when the agreement code should start - // pipelining the next block, by choosing the delay time of the - // AgreementPipelineDelay'th slowest block to arrive out of the last - // AgreementPipelineDelayHistory lowest-credential block payloads. - AgreementPipelineDelay int + DynamicFilterPayloadArriavalHistory int } // PaysetCommitType enumerates possible ways for the block header to commit to @@ -1366,6 +1356,10 @@ func initConsensusProtocols() { v38.AgreementFilterTimeoutPeriod0 = 3000 * time.Millisecond + v38.DynamicFilterTimeout = true + v38.DynamicFilterPayloadArriavalHistory = 32 + v38.DynamicFilterTimeoutLowerBound = time.Second + Consensus[protocol.ConsensusV38] = v38 // v37 can be upgraded to v38, with an update delay of 12h: @@ -1376,6 +1370,7 @@ func initConsensusProtocols() { // ConsensusFuture is used to test features that are implemented // but not yet released in a production protocol version. vFuture := v38 + vFuture.ApprovedUpgrades = map[protocol.ConsensusVersion]uint64{} vFuture.LogicSigVersion = 10 // When moving this to a release, put a new higher LogicSigVersion here From 6781648e64ee0a8f330396c7bb439495daa60635 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Tue, 8 Aug 2023 15:54:45 -0400 Subject: [PATCH 015/232] reset history on non-0 periods --- agreement/player.go | 18 +++++++++++++----- agreement/service_test.go | 18 ++++++++++++++++-- 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/agreement/player.go b/agreement/player.go index bcda8831cc..acc21ae96a 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -295,9 +295,11 @@ func (p *player) calculateFilterTimeout(period period, ver protocol.ConsensusVer proto := config.Consensus[ver] + //return FilterTimeout(period, ver) + if !proto.DynamicFilterTimeout || period != 0 { // Either dynamic lambda is disabled, or we're not in period 0 and - // therefore can't use dynamic lambda + // therefore, can't use dynamic lambda return FilterTimeout(period, ver) } @@ -394,7 +396,13 @@ func (p *player) enterPeriod(r routerHandle, source thresholdEvent, target perio p.Step = soft p.Napping = false p.FastRecoveryDeadline = 0 // set immediately - p.Deadline.Duration = FilterTimeout(target, source.Proto) + + if target != 0 { + // We entered a non-0 period, we should reset the filter timeout + // calculation mechanism. + p.payloadArrivals = make([]time.Duration, 0) + } + p.Deadline.Duration = p.calculateFilterTimeout(target, source.Proto) p.Deadline.Type = timers.Filter // update tracer state to match player @@ -443,13 +451,13 @@ func (p *player) enterRound(r routerHandle, source event, target round) []action switch source := source.(type) { case roundInterruptionEvent: - p.Deadline.Duration = calculateFilterTimeout(0, source.Proto.Version) + p.Deadline.Duration = p.calculateFilterTimeout(0, source.Proto.Version) p.Deadline.Type = timers.Filter case thresholdEvent: - p.Deadline.Duration = calculateFilterTimeout(0, source.Proto) + p.Deadline.Duration = p.calculateFilterTimeout(0, source.Proto) p.Deadline.Type = timers.Filter case filterableMessageEvent: - p.Deadline.Duration = calculateFilterTimeout(0, source.Proto.Version) + p.Deadline.Duration = p.calculateFilterTimeout(0, source.Proto.Version) p.Deadline.Type = timers.Filter } diff --git a/agreement/service_test.go b/agreement/service_test.go index f671708952..de250c0b29 100644 --- a/agreement/service_test.go +++ b/agreement/service_test.go @@ -909,6 +909,20 @@ func simulateAgreementWithConsensusVersion(t *testing.T, numNodes int, numRounds simulateAgreementWithLedgerFactory(t, numNodes, numRounds, traceLevel, ledgerFactory) } +func maxFilterTimeoutPeriod0(services []*Service, version protocol.ConsensusVersion) time.Duration { + if len(services) == 0 { + return FilterTimeout(0, version) + } + maxFilter := services[0].persistStatus.calculateFilterTimeout(0, version) + for _, service := range services { + curr := service.persistStatus.calculateFilterTimeout(0, version) + if maxFilter < curr { + maxFilter = curr + } + } + return maxFilter +} + func simulateAgreementWithLedgerFactory(t *testing.T, numNodes int, numRounds int, traceLevel traceLevel, ledgerFactory func(map[basics.Address]basics.AccountData) Ledger) { _, baseLedger, cleanupFn, services, clocks, ledgers, activityMonitor := setupAgreement(t, numNodes, traceLevel, ledgerFactory) startRound := baseLedger.NextRound() @@ -923,10 +937,10 @@ func simulateAgreementWithLedgerFactory(t *testing.T, numNodes int, numRounds in // run round with round-specific consensus version first (since fix in #1896) version, _ := baseLedger.ConsensusVersion(ParamsRound(startRound)) - zeroes = runRound(clocks, activityMonitor, zeroes, FilterTimeout(0, version)) + zeroes = runRound(clocks, activityMonitor, zeroes, maxFilterTimeoutPeriod0(services, version)) for j := 1; j < numRounds; j++ { version, _ := baseLedger.ConsensusVersion(ParamsRound(baseLedger.NextRound() + basics.Round(j-1))) - zeroes = runRound(clocks, activityMonitor, zeroes, FilterTimeout(0, version)) + zeroes = runRound(clocks, activityMonitor, zeroes, maxFilterTimeoutPeriod0(services, version)) } for i := 0; i < numNodes; i++ { From 87096cb687d926545ec698f5b21bc9a489d0392d Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Wed, 9 Aug 2023 15:15:10 -0400 Subject: [PATCH 016/232] test clock tweak --- agreement/player.go | 3 +-- agreement/service_test.go | 13 ++++++++++--- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/agreement/player.go b/agreement/player.go index acc21ae96a..4929cd4edd 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -17,7 +17,6 @@ package agreement import ( - "fmt" "sort" "time" @@ -323,7 +322,7 @@ func (p *player) calculateFilterTimeout(period period, ver protocol.ConsensusVer } //dynamicDelay = FilterTimeout(period, ver) - fmt.Println("dynamic delay", dynamicDelay, "payload arrivals", len(p.payloadArrivals)) + //fmt.Println("dynamic delay", dynamicDelay, "payload arrivals", len(p.payloadArrivals)) return dynamicDelay } diff --git a/agreement/service_test.go b/agreement/service_test.go index de250c0b29..f6daf6f2f1 100644 --- a/agreement/service_test.go +++ b/agreement/service_test.go @@ -909,7 +909,13 @@ func simulateAgreementWithConsensusVersion(t *testing.T, numNodes int, numRounds simulateAgreementWithLedgerFactory(t, numNodes, numRounds, traceLevel, ledgerFactory) } -func maxFilterTimeoutPeriod0(services []*Service, version protocol.ConsensusVersion) time.Duration { +func maxFilterTimeoutPeriod0(services []*Service, version protocol.ConsensusVersion, iter int) time.Duration { + if iter < 32 { // TODO: 32 is the history length we track for dynamic lambda. We need to change this param + return FilterTimeout(0, version) + } else { + return time.Second // this is the min filter timeout we're using. + } + if len(services) == 0 { return FilterTimeout(0, version) } @@ -937,10 +943,11 @@ func simulateAgreementWithLedgerFactory(t *testing.T, numNodes int, numRounds in // run round with round-specific consensus version first (since fix in #1896) version, _ := baseLedger.ConsensusVersion(ParamsRound(startRound)) - zeroes = runRound(clocks, activityMonitor, zeroes, maxFilterTimeoutPeriod0(services, version)) + zeroes = runRound(clocks, activityMonitor, zeroes, maxFilterTimeoutPeriod0(services, version, 0)) for j := 1; j < numRounds; j++ { + // TODO: remove this: time.Sleep(100 * time.Millisecond) version, _ := baseLedger.ConsensusVersion(ParamsRound(baseLedger.NextRound() + basics.Round(j-1))) - zeroes = runRound(clocks, activityMonitor, zeroes, maxFilterTimeoutPeriod0(services, version)) + zeroes = runRound(clocks, activityMonitor, zeroes, maxFilterTimeoutPeriod0(services, version, j)) } for i := 0; i < numNodes; i++ { From 4077e115c51a371a26b5afbd88cb1eb5d0f29969 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Thu, 10 Aug 2023 11:00:35 -0400 Subject: [PATCH 017/232] avoid weird clock tweak --- agreement/msgp_gen.go | 138 +++++++++++++++++++------------------- agreement/service_test.go | 31 ++------- 2 files changed, 75 insertions(+), 94 deletions(-) diff --git a/agreement/msgp_gen.go b/agreement/msgp_gen.go index 097efae2dd..ee65757f52 100644 --- a/agreement/msgp_gen.go +++ b/agreement/msgp_gen.go @@ -3923,88 +3923,88 @@ func (_ *player) CanMarshalMsg(z interface{}) bool { func (z *player) UnmarshalMsg(bts []byte) (o []byte, err error) { var field []byte _ = field - var zb0001 int - var zb0002 bool - zb0001, zb0002, bts, err = msgp.ReadMapHeaderBytes(bts) + var zb0002 int + var zb0003 bool + zb0002, zb0003, bts, err = msgp.ReadMapHeaderBytes(bts) if _, ok := err.(msgp.TypeError); ok { - zb0001, zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + zb0002, zb0003, bts, err = msgp.ReadArrayHeaderBytes(bts) if err != nil { err = msgp.WrapError(err) return } - if zb0001 > 0 { - zb0001-- + if zb0002 > 0 { + zb0002-- bts, err = (*z).Round.UnmarshalMsg(bts) if err != nil { err = msgp.WrapError(err, "struct-from-array", "Round") return } } - if zb0001 > 0 { - zb0001-- + if zb0002 > 0 { + zb0002-- { - var zb0003 uint64 - zb0003, bts, err = msgp.ReadUint64Bytes(bts) + var zb0004 uint64 + zb0004, bts, err = msgp.ReadUint64Bytes(bts) if err != nil { err = msgp.WrapError(err, "struct-from-array", "Period") return } - (*z).Period = period(zb0003) + (*z).Period = period(zb0004) } } - if zb0001 > 0 { - zb0001-- + if zb0002 > 0 { + zb0002-- { - var zb0004 uint64 - zb0004, bts, err = msgp.ReadUint64Bytes(bts) + var zb0005 uint64 + zb0005, bts, err = msgp.ReadUint64Bytes(bts) if err != nil { err = msgp.WrapError(err, "struct-from-array", "Step") return } - (*z).Step = step(zb0004) + (*z).Step = step(zb0005) } } - if zb0001 > 0 { - zb0001-- + if zb0002 > 0 { + zb0002-- { - var zb0005 uint64 - zb0005, bts, err = msgp.ReadUint64Bytes(bts) + var zb0006 uint64 + zb0006, bts, err = msgp.ReadUint64Bytes(bts) if err != nil { err = msgp.WrapError(err, "struct-from-array", "LastConcluding") return } - (*z).LastConcluding = step(zb0005) + (*z).LastConcluding = step(zb0006) } } - if zb0001 > 0 { - zb0001-- - var zb0006 int - var zb0007 bool - zb0006, zb0007, bts, err = msgp.ReadMapHeaderBytes(bts) + if zb0002 > 0 { + zb0002-- + var zb0007 int + var zb0008 bool + zb0007, zb0008, bts, err = msgp.ReadMapHeaderBytes(bts) if _, ok := err.(msgp.TypeError); ok { - zb0006, zb0007, bts, err = msgp.ReadArrayHeaderBytes(bts) + zb0007, zb0008, bts, err = msgp.ReadArrayHeaderBytes(bts) if err != nil { err = msgp.WrapError(err, "struct-from-array", "Deadline") return } - if zb0006 > 0 { - zb0006-- + if zb0007 > 0 { + zb0007-- (*z).Deadline.Duration, bts, err = msgp.ReadDurationBytes(bts) if err != nil { err = msgp.WrapError(err, "struct-from-array", "Deadline", "struct-from-array", "Duration") return } } - if zb0006 > 0 { - zb0006-- + if zb0007 > 0 { + zb0007-- bts, err = (*z).Deadline.Type.UnmarshalMsg(bts) if err != nil { err = msgp.WrapError(err, "struct-from-array", "Deadline", "struct-from-array", "Type") return } } - if zb0006 > 0 { - err = msgp.ErrTooManyArrayFields(zb0006) + if zb0007 > 0 { + err = msgp.ErrTooManyArrayFields(zb0007) if err != nil { err = msgp.WrapError(err, "struct-from-array", "Deadline", "struct-from-array") return @@ -4015,11 +4015,11 @@ func (z *player) UnmarshalMsg(bts []byte) (o []byte, err error) { err = msgp.WrapError(err, "struct-from-array", "Deadline") return } - if zb0007 { + if zb0008 { (*z).Deadline = Deadline{} } - for zb0006 > 0 { - zb0006-- + for zb0007 > 0 { + zb0007-- field, bts, err = msgp.ReadMapKeyZC(bts) if err != nil { err = msgp.WrapError(err, "struct-from-array", "Deadline") @@ -4048,32 +4048,32 @@ func (z *player) UnmarshalMsg(bts []byte) (o []byte, err error) { } } } - if zb0001 > 0 { - zb0001-- + if zb0002 > 0 { + zb0002-- (*z).Napping, bts, err = msgp.ReadBoolBytes(bts) if err != nil { err = msgp.WrapError(err, "struct-from-array", "Napping") return } } - if zb0001 > 0 { - zb0001-- + if zb0002 > 0 { + zb0002-- (*z).FastRecoveryDeadline, bts, err = msgp.ReadDurationBytes(bts) if err != nil { err = msgp.WrapError(err, "struct-from-array", "FastRecoveryDeadline") return } } - if zb0001 > 0 { - zb0001-- + if zb0002 > 0 { + zb0002-- bts, err = (*z).Pending.UnmarshalMsg(bts) if err != nil { err = msgp.WrapError(err, "struct-from-array", "Pending") return } } - if zb0001 > 0 { - err = msgp.ErrTooManyArrayFields(zb0001) + if zb0002 > 0 { + err = msgp.ErrTooManyArrayFields(zb0002) if err != nil { err = msgp.WrapError(err, "struct-from-array") return @@ -4084,11 +4084,11 @@ func (z *player) UnmarshalMsg(bts []byte) (o []byte, err error) { err = msgp.WrapError(err) return } - if zb0002 { + if zb0003 { (*z) = player{} } - for zb0001 > 0 { - zb0001-- + for zb0002 > 0 { + zb0002-- field, bts, err = msgp.ReadMapKeyZC(bts) if err != nil { err = msgp.WrapError(err) @@ -4103,62 +4103,62 @@ func (z *player) UnmarshalMsg(bts []byte) (o []byte, err error) { } case "Period": { - var zb0008 uint64 - zb0008, bts, err = msgp.ReadUint64Bytes(bts) + var zb0009 uint64 + zb0009, bts, err = msgp.ReadUint64Bytes(bts) if err != nil { err = msgp.WrapError(err, "Period") return } - (*z).Period = period(zb0008) + (*z).Period = period(zb0009) } case "Step": { - var zb0009 uint64 - zb0009, bts, err = msgp.ReadUint64Bytes(bts) + var zb0010 uint64 + zb0010, bts, err = msgp.ReadUint64Bytes(bts) if err != nil { err = msgp.WrapError(err, "Step") return } - (*z).Step = step(zb0009) + (*z).Step = step(zb0010) } case "LastConcluding": { - var zb0010 uint64 - zb0010, bts, err = msgp.ReadUint64Bytes(bts) + var zb0011 uint64 + zb0011, bts, err = msgp.ReadUint64Bytes(bts) if err != nil { err = msgp.WrapError(err, "LastConcluding") return } - (*z).LastConcluding = step(zb0010) + (*z).LastConcluding = step(zb0011) } case "Deadline": - var zb0011 int - var zb0012 bool - zb0011, zb0012, bts, err = msgp.ReadMapHeaderBytes(bts) + var zb0012 int + var zb0013 bool + zb0012, zb0013, bts, err = msgp.ReadMapHeaderBytes(bts) if _, ok := err.(msgp.TypeError); ok { - zb0011, zb0012, bts, err = msgp.ReadArrayHeaderBytes(bts) + zb0012, zb0013, bts, err = msgp.ReadArrayHeaderBytes(bts) if err != nil { err = msgp.WrapError(err, "Deadline") return } - if zb0011 > 0 { - zb0011-- + if zb0012 > 0 { + zb0012-- (*z).Deadline.Duration, bts, err = msgp.ReadDurationBytes(bts) if err != nil { err = msgp.WrapError(err, "Deadline", "struct-from-array", "Duration") return } } - if zb0011 > 0 { - zb0011-- + if zb0012 > 0 { + zb0012-- bts, err = (*z).Deadline.Type.UnmarshalMsg(bts) if err != nil { err = msgp.WrapError(err, "Deadline", "struct-from-array", "Type") return } } - if zb0011 > 0 { - err = msgp.ErrTooManyArrayFields(zb0011) + if zb0012 > 0 { + err = msgp.ErrTooManyArrayFields(zb0012) if err != nil { err = msgp.WrapError(err, "Deadline", "struct-from-array") return @@ -4169,11 +4169,11 @@ func (z *player) UnmarshalMsg(bts []byte) (o []byte, err error) { err = msgp.WrapError(err, "Deadline") return } - if zb0012 { + if zb0013 { (*z).Deadline = Deadline{} } - for zb0011 > 0 { - zb0011-- + for zb0012 > 0 { + zb0012-- field, bts, err = msgp.ReadMapKeyZC(bts) if err != nil { err = msgp.WrapError(err, "Deadline") diff --git a/agreement/service_test.go b/agreement/service_test.go index f6daf6f2f1..6ae17faecc 100644 --- a/agreement/service_test.go +++ b/agreement/service_test.go @@ -880,6 +880,10 @@ func runRound(clocks []timers.Clock, activityMonitor *activityMonitor, zeroes ui triggerGlobalTimeout(filterTimeout, timers.Filter, clocks, activityMonitor) return expectNewPeriod(clocks, zeroes) } +func runRoundTriggerFilter(clocks []timers.Clock, activityMonitor *activityMonitor, zeroes uint) (newzeroes uint) { + triggerGlobalTimeoutType(timers.Filter, clocks, activityMonitor) + return expectNewPeriod(clocks, zeroes) +} func sanityCheck(startRound round, numRounds round, ledgers []Ledger) { for i := range ledgers { @@ -909,26 +913,6 @@ func simulateAgreementWithConsensusVersion(t *testing.T, numNodes int, numRounds simulateAgreementWithLedgerFactory(t, numNodes, numRounds, traceLevel, ledgerFactory) } -func maxFilterTimeoutPeriod0(services []*Service, version protocol.ConsensusVersion, iter int) time.Duration { - if iter < 32 { // TODO: 32 is the history length we track for dynamic lambda. We need to change this param - return FilterTimeout(0, version) - } else { - return time.Second // this is the min filter timeout we're using. - } - - if len(services) == 0 { - return FilterTimeout(0, version) - } - maxFilter := services[0].persistStatus.calculateFilterTimeout(0, version) - for _, service := range services { - curr := service.persistStatus.calculateFilterTimeout(0, version) - if maxFilter < curr { - maxFilter = curr - } - } - return maxFilter -} - func simulateAgreementWithLedgerFactory(t *testing.T, numNodes int, numRounds int, traceLevel traceLevel, ledgerFactory func(map[basics.Address]basics.AccountData) Ledger) { _, baseLedger, cleanupFn, services, clocks, ledgers, activityMonitor := setupAgreement(t, numNodes, traceLevel, ledgerFactory) startRound := baseLedger.NextRound() @@ -942,12 +926,9 @@ func simulateAgreementWithLedgerFactory(t *testing.T, numNodes int, numRounds in zeroes := expectNewPeriod(clocks, 0) // run round with round-specific consensus version first (since fix in #1896) - version, _ := baseLedger.ConsensusVersion(ParamsRound(startRound)) - zeroes = runRound(clocks, activityMonitor, zeroes, maxFilterTimeoutPeriod0(services, version, 0)) + zeroes = runRoundTriggerFilter(clocks, activityMonitor, zeroes) for j := 1; j < numRounds; j++ { - // TODO: remove this: time.Sleep(100 * time.Millisecond) - version, _ := baseLedger.ConsensusVersion(ParamsRound(baseLedger.NextRound() + basics.Round(j-1))) - zeroes = runRound(clocks, activityMonitor, zeroes, maxFilterTimeoutPeriod0(services, version, j)) + zeroes = runRoundTriggerFilter(clocks, activityMonitor, zeroes) } for i := 0; i < numNodes; i++ { From db1386c24ddc116d0776d19bbe8d9a142758eb7a Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Thu, 10 Aug 2023 11:42:18 -0400 Subject: [PATCH 018/232] msgp for tagged deadline --- util/timers/msgp_gen.go | 70 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 util/timers/msgp_gen.go diff --git a/util/timers/msgp_gen.go b/util/timers/msgp_gen.go new file mode 100644 index 0000000000..ae3b2729b5 --- /dev/null +++ b/util/timers/msgp_gen.go @@ -0,0 +1,70 @@ +package timers + +// Code generated by github.com/algorand/msgp DO NOT EDIT. + +import ( + "github.com/algorand/msgp/msgp" +) + +// The following msgp objects are implemented in this file: +// TimeoutType +// |-----> MarshalMsg +// |-----> CanMarshalMsg +// |-----> (*) UnmarshalMsg +// |-----> (*) CanUnmarshalMsg +// |-----> Msgsize +// |-----> MsgIsZero +// |-----> TimeoutTypeMaxSize() +// + +// MarshalMsg implements msgp.Marshaler +func (z TimeoutType) MarshalMsg(b []byte) (o []byte) { + o = msgp.Require(b, z.Msgsize()) + o = msgp.AppendInt8(o, int8(z)) + return +} + +func (_ TimeoutType) CanMarshalMsg(z interface{}) bool { + _, ok := (z).(TimeoutType) + if !ok { + _, ok = (z).(*TimeoutType) + } + return ok +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *TimeoutType) UnmarshalMsg(bts []byte) (o []byte, err error) { + { + var zb0001 int8 + zb0001, bts, err = msgp.ReadInt8Bytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + (*z) = TimeoutType(zb0001) + } + o = bts + return +} + +func (_ *TimeoutType) CanUnmarshalMsg(z interface{}) bool { + _, ok := (z).(*TimeoutType) + return ok +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z TimeoutType) Msgsize() (s int) { + s = msgp.Int8Size + return +} + +// MsgIsZero returns whether this is a zero value +func (z TimeoutType) MsgIsZero() bool { + return z == 0 +} + +// MaxSize returns a maximum valid message size for this message type +func TimeoutTypeMaxSize() (s int) { + s = msgp.Int8Size + return +} From 2f4eee1984e86104304993f630ced7384d2c92b4 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Thu, 10 Aug 2023 14:42:22 -0400 Subject: [PATCH 019/232] fix lint issues --- agreement/player.go | 2 ++ util/timers/interface.go | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/agreement/player.go b/agreement/player.go index 86627ccded..4cca86d3c7 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -24,6 +24,8 @@ import ( "github.com/algorand/go-algorand/util/timers" ) +// Deadline marks a timeout event of type Type that the player schedules to +// happen after Duration time. type Deadline struct { _struct struct{} `codec:","` Duration time.Duration diff --git a/util/timers/interface.go b/util/timers/interface.go index d8af682e4d..94ac21c0c1 100644 --- a/util/timers/interface.go +++ b/util/timers/interface.go @@ -21,11 +21,16 @@ import ( "time" ) +// TimeoutType annotates timeout events scheduled via TimeoutAt type TimeoutType int8 const ( + // Deadline annotates timeout events in the agreement protocol (e.g., for + // receiving a block) Deadline TimeoutType = iota + // FastRecovery annotates the fast recovery timeout in the agreement protocol FastRecovery + // Filter annotates the filter step timeout event in the agreement protocol Filter ) From 83d26c9986bdd994334100de275f2eb0a4c3a4ee Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Thu, 10 Aug 2023 14:50:12 -0400 Subject: [PATCH 020/232] fix lint issues --- agreement/player.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/agreement/player.go b/agreement/player.go index 2d3543d06d..414d9ad74e 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -296,8 +296,6 @@ func (p *player) calculateFilterTimeout(period period, ver protocol.ConsensusVer proto := config.Consensus[ver] - //return FilterTimeout(period, ver) - if !proto.DynamicFilterTimeout || period != 0 { // Either dynamic lambda is disabled, or we're not in period 0 and // therefore, can't use dynamic lambda @@ -309,7 +307,7 @@ func (p *player) calculateFilterTimeout(period period, ver protocol.ConsensusVer // we don't keep any history, use the default dynamicDelay = FilterTimeout(0, ver) } else if proto.DynamicFilterPayloadArriavalHistory > len(p.payloadArrivals) { - // not enough smaples, use the default + // not enough samples, use the default dynamicDelay = FilterTimeout(0, ver) } else { sortedArrivals := make([]time.Duration, len(p.payloadArrivals)) @@ -323,9 +321,6 @@ func (p *player) calculateFilterTimeout(period period, ver protocol.ConsensusVer dynamicDelay = proto.DynamicFilterTimeoutLowerBound } - //dynamicDelay = FilterTimeout(period, ver) - //fmt.Println("dynamic delay", dynamicDelay, "payload arrivals", len(p.payloadArrivals)) - return dynamicDelay } From 3039b1e62791b736af09114e6505751df606dadb Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Thu, 10 Aug 2023 15:13:10 -0400 Subject: [PATCH 021/232] parameter cleanup + logging --- agreement/player.go | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/agreement/player.go b/agreement/player.go index 414d9ad74e..79be574952 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -291,15 +291,15 @@ func (p *player) resizePayloadArrivals(ver protocol.ConsensusVersion) { } } -// calculateFilterTimeout chooses the appropriate filter timeout for a new round. -func (p *player) calculateFilterTimeout(period period, ver protocol.ConsensusVersion) time.Duration { +// calculateFilterTimeout chooses the appropriate filter timeout. +func (p *player) calculateFilterTimeout(ver protocol.ConsensusVersion, tracer *tracer) time.Duration { proto := config.Consensus[ver] - if !proto.DynamicFilterTimeout || period != 0 { + if !proto.DynamicFilterTimeout || p.Period != 0 { // Either dynamic lambda is disabled, or we're not in period 0 and // therefore, can't use dynamic lambda - return FilterTimeout(period, ver) + return FilterTimeout(p.Period, ver) } var dynamicDelay time.Duration @@ -318,7 +318,10 @@ func (p *player) calculateFilterTimeout(period period, ver protocol.ConsensusVer // Make sure the dynamic delay is not too small if dynamicDelay < proto.DynamicFilterTimeoutLowerBound { + tracer.log.Infof("Calculated dynamic delay = %v for round %v, period %v. It's too low, setting to %v\n", dynamicDelay, p.Round, p.Period, proto.DynamicFilterTimeoutLowerBound) dynamicDelay = proto.DynamicFilterTimeoutLowerBound + } else { + tracer.log.Infof("Calculated dynamic delay = %v for round %v, period %v.\n", dynamicDelay, p.Round, p.Period) } return dynamicDelay @@ -398,7 +401,7 @@ func (p *player) enterPeriod(r routerHandle, source thresholdEvent, target perio // calculation mechanism. p.payloadArrivals = make([]time.Duration, 0) } - p.Deadline.Duration = p.calculateFilterTimeout(target, source.Proto) + p.Deadline.Duration = p.calculateFilterTimeout(source.Proto, r.t) p.Deadline.Type = timers.Filter // update tracer state to match player @@ -447,13 +450,13 @@ func (p *player) enterRound(r routerHandle, source event, target round) []action switch source := source.(type) { case roundInterruptionEvent: - p.Deadline.Duration = p.calculateFilterTimeout(0, source.Proto.Version) + p.Deadline.Duration = p.calculateFilterTimeout(source.Proto.Version, r.t) p.Deadline.Type = timers.Filter case thresholdEvent: - p.Deadline.Duration = p.calculateFilterTimeout(0, source.Proto) + p.Deadline.Duration = p.calculateFilterTimeout(source.Proto, r.t) p.Deadline.Type = timers.Filter case filterableMessageEvent: - p.Deadline.Duration = p.calculateFilterTimeout(0, source.Proto.Version) + p.Deadline.Duration = p.calculateFilterTimeout(source.Proto.Version, r.t) p.Deadline.Type = timers.Filter } From ce6db010f2c0672e5fbe47d22ebbaa05fcae4ec7 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Thu, 10 Aug 2023 16:31:04 -0400 Subject: [PATCH 022/232] lint --- agreement/player.go | 16 +++++++++++----- config/consensus.go | 5 ++++- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/agreement/player.go b/agreement/player.go index 79be574952..b6944e7029 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -303,12 +303,13 @@ func (p *player) calculateFilterTimeout(ver protocol.ConsensusVersion, tracer *t } var dynamicDelay time.Duration + defaultDelay := FilterTimeout(0, ver) if proto.DynamicFilterPayloadArriavalHistory <= 0 { // we don't keep any history, use the default - dynamicDelay = FilterTimeout(0, ver) + dynamicDelay = defaultDelay } else if proto.DynamicFilterPayloadArriavalHistory > len(p.payloadArrivals) { // not enough samples, use the default - dynamicDelay = FilterTimeout(0, ver) + dynamicDelay = defaultDelay } else { sortedArrivals := make([]time.Duration, len(p.payloadArrivals)) copy(sortedArrivals[:], p.payloadArrivals[:]) @@ -316,11 +317,16 @@ func (p *player) calculateFilterTimeout(ver protocol.ConsensusVersion, tracer *t dynamicDelay = sortedArrivals[proto.DynamicFilterPayloadArriavalHistory-1] } - // Make sure the dynamic delay is not too small + // Make sure the dynamic delay is not too small or too large if dynamicDelay < proto.DynamicFilterTimeoutLowerBound { - tracer.log.Infof("Calculated dynamic delay = %v for round %v, period %v. It's too low, setting to %v\n", dynamicDelay, p.Round, p.Period, proto.DynamicFilterTimeoutLowerBound) + if tracer != nil { + tracer.log.Infof("Calculated dynamic delay = %v for round %v, period %v. It's too low, setting to %v\n", dynamicDelay, p.Round, p.Period, proto.DynamicFilterTimeoutLowerBound) + } dynamicDelay = proto.DynamicFilterTimeoutLowerBound - } else { + } else if dynamicDelay > defaultDelay { + dynamicDelay = defaultDelay + tracer.log.Warnf("Calculated dynamic delay = %v for round %v, period %v. It's higher than the default %v\n", dynamicDelay, p.Round, p.Period, defaultDelay) + } else if tracer != nil { tracer.log.Infof("Calculated dynamic delay = %v for round %v, period %v.\n", dynamicDelay, p.Round, p.Period) } diff --git a/config/consensus.go b/config/consensus.go index 02cc03ee9e..9fa3213c30 100644 --- a/config/consensus.go +++ b/config/consensus.go @@ -517,9 +517,12 @@ type ConsensusParams struct { // by state proofs to use the same method (rather than excluding stake from the top N stakeholders as before). ExcludeExpiredCirculation bool - // DynamicFilterTimeout + // DynamicFilterTimeout specifies whether the timeout for the filter step + // should be determined dynamically, at run-time. DynamicFilterTimeout bool + // DynamicFilterTimeoutLowerBound specifies a minimal duration that the + // filter timeout must meet. DynamicFilterTimeoutLowerBound time.Duration // DynamicFilterPayloadArriavalHistory specifies the number of past block arrivals From 0b8e8a5a8f402c578f722a999eadadf2b6e08aab Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Thu, 10 Aug 2023 16:43:37 -0400 Subject: [PATCH 023/232] less logging, params in vfuture --- agreement/player.go | 4 ++-- agreement/service_test.go | 12 ++++++++++++ config/consensus.go | 8 ++++---- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/agreement/player.go b/agreement/player.go index b6944e7029..dd038d835b 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -320,14 +320,14 @@ func (p *player) calculateFilterTimeout(ver protocol.ConsensusVersion, tracer *t // Make sure the dynamic delay is not too small or too large if dynamicDelay < proto.DynamicFilterTimeoutLowerBound { if tracer != nil { - tracer.log.Infof("Calculated dynamic delay = %v for round %v, period %v. It's too low, setting to %v\n", dynamicDelay, p.Round, p.Period, proto.DynamicFilterTimeoutLowerBound) + tracer.log.Debugf("Calculated dynamic delay = %v for round %v, period %v. It's too low, setting to %v\n", dynamicDelay, p.Round, p.Period, proto.DynamicFilterTimeoutLowerBound) } dynamicDelay = proto.DynamicFilterTimeoutLowerBound } else if dynamicDelay > defaultDelay { dynamicDelay = defaultDelay tracer.log.Warnf("Calculated dynamic delay = %v for round %v, period %v. It's higher than the default %v\n", dynamicDelay, p.Round, p.Period, defaultDelay) } else if tracer != nil { - tracer.log.Infof("Calculated dynamic delay = %v for round %v, period %v.\n", dynamicDelay, p.Round, p.Period) + tracer.log.Debugf("Calculated dynamic delay = %v for round %v, period %v.\n", dynamicDelay, p.Round, p.Period) } return dynamicDelay diff --git a/agreement/service_test.go b/agreement/service_test.go index 6ae17faecc..0fa6388ed9 100644 --- a/agreement/service_test.go +++ b/agreement/service_test.go @@ -1008,6 +1008,18 @@ func TestAgreementSynchronous5_50(t *testing.T) { simulateAgreement(t, 5, 50, disabled) } +func TestAgreementSynchronousFuture5_50(t *testing.T) { + partitiontest.PartitionTest(t) + + if testing.Short() { + t.Skip("Skipping agreement integration test") + } + consensusVersion := func(r basics.Round) (protocol.ConsensusVersion, error) { + return protocol.ConsensusFuture, nil + } + simulateAgreementWithConsensusVersion(t, 5, 50, disabled, consensusVersion) +} + func TestAgreementSynchronousFuture1(t *testing.T) { partitiontest.PartitionTest(t) diff --git a/config/consensus.go b/config/consensus.go index 9fa3213c30..5ab8f6931a 100644 --- a/config/consensus.go +++ b/config/consensus.go @@ -1359,10 +1359,6 @@ func initConsensusProtocols() { v38.AgreementFilterTimeoutPeriod0 = 3000 * time.Millisecond - v38.DynamicFilterTimeout = true - v38.DynamicFilterPayloadArriavalHistory = 32 - v38.DynamicFilterTimeoutLowerBound = time.Second - Consensus[protocol.ConsensusV38] = v38 // v37 can be upgraded to v38, with an update delay of 12h: @@ -1379,6 +1375,10 @@ func initConsensusProtocols() { vFuture.LogicSigVersion = 10 // When moving this to a release, put a new higher LogicSigVersion here vFuture.EnableLogicSigCostPooling = true + vFuture.DynamicFilterTimeout = true + vFuture.DynamicFilterTimeoutLowerBound = time.Second + vFuture.DynamicFilterPayloadArriavalHistory = 32 + Consensus[protocol.ConsensusFuture] = vFuture // vAlphaX versions are an separate series of consensus parameters and versions for alphanet From 75c808115356c34a4f872f1153bb744043ddd0aa Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Thu, 10 Aug 2023 18:13:33 -0400 Subject: [PATCH 024/232] enough rounds to activate dynamic lambda --- agreement/service_test.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/agreement/service_test.go b/agreement/service_test.go index 0fa6388ed9..1032b33944 100644 --- a/agreement/service_test.go +++ b/agreement/service_test.go @@ -1008,16 +1008,23 @@ func TestAgreementSynchronous5_50(t *testing.T) { simulateAgreement(t, 5, 50, disabled) } -func TestAgreementSynchronousFuture5_50(t *testing.T) { +func TestAgreementSynchronousFuture5_DynamicFilterRounds(t *testing.T) { partitiontest.PartitionTest(t) if testing.Short() { t.Skip("Skipping agreement integration test") } + consensusVersion := func(r basics.Round) (protocol.ConsensusVersion, error) { return protocol.ConsensusFuture, nil } - simulateAgreementWithConsensusVersion(t, 5, 50, disabled, consensusVersion) + + cfg := config.Consensus[protocol.ConsensusFuture] + if !cfg.DynamicFilterTimeout { + return + } + + simulateAgreementWithConsensusVersion(t, 5, cfg.DynamicFilterPayloadArriavalHistory+20, disabled, consensusVersion) } func TestAgreementSynchronousFuture1(t *testing.T) { From 3a5a1edd42b753f2650cd5ef6112ab3f77d78d76 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Thu, 10 Aug 2023 18:18:03 -0400 Subject: [PATCH 025/232] +msgp_gen test --- util/timers/msgp_gen_test.go | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 util/timers/msgp_gen_test.go diff --git a/util/timers/msgp_gen_test.go b/util/timers/msgp_gen_test.go new file mode 100644 index 0000000000..265bbec36f --- /dev/null +++ b/util/timers/msgp_gen_test.go @@ -0,0 +1,6 @@ +//go:build !skip_msgp_testing +// +build !skip_msgp_testing + +package timers + +// Code generated by github.com/algorand/msgp DO NOT EDIT. From aa603c9a63eeebb37f95fa1be380636e0d19d423 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Fri, 11 Aug 2023 11:42:03 -0400 Subject: [PATCH 026/232] skip sdk comparison --- tools/x-repo-types/xrt_test.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/tools/x-repo-types/xrt_test.go b/tools/x-repo-types/xrt_test.go index 4360b432d6..6fe353cedb 100644 --- a/tools/x-repo-types/xrt_test.go +++ b/tools/x-repo-types/xrt_test.go @@ -74,12 +74,14 @@ func TestCrossRepoTypes(t *testing.T) { yType: "EvalDelta", }, { - name: "goal-v-sdk-consensus", - xPkg: "github.com/algorand/go-algorand/config", - xType: "ConsensusParams", - yPkg: "github.com/algorand/go-algorand-sdk/v2/protocol/config", - yBranch: "develop", - yType: "ConsensusParams", + name: "goal-v-sdk-consensus", + xPkg: "github.com/algorand/go-algorand/config", + xType: "ConsensusParams", + yPkg: "github.com/algorand/go-algorand-sdk/v2/protocol/config", + yBranch: "develop", + yType: "ConsensusParams", + skip: true, + skipReason: `Changes to consensus parameters happen on the go-algorand repo, and cannot be synced in a single PR with changes to the go-algorand-sdk repo`, }, { name: "goal-v-sdk-blockheader", From cc5a872144a2f32b4ddb63d966195d20147c4f10 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Fri, 11 Aug 2023 14:36:55 -0400 Subject: [PATCH 027/232] rounds run faster now. Give the txn a little bit more time to live --- test/scripts/e2e_subs/limit-swap-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/scripts/e2e_subs/limit-swap-test.sh b/test/scripts/e2e_subs/limit-swap-test.sh index d89331300c..026d9b3609 100755 --- a/test/scripts/e2e_subs/limit-swap-test.sh +++ b/test/scripts/e2e_subs/limit-swap-test.sh @@ -42,7 +42,7 @@ echo "closeout part b, asset trader" # quick expiration, test closeout ROUND=$(goal node status | grep 'Last committed block:'|awk '{ print $4 }') -SETUP_ROUND=$((${ROUND} + 10)) +SETUP_ROUND=$((${ROUND} + 20)) TIMEOUT_ROUND=$((${SETUP_ROUND} + 1)) sed s/TMPL_ASSET/${ASSET_ID}/g < tools/teal/templates/limit-order-b.teal.tmpl | sed s/TMPL_SWAPN/137/g | sed s/TMPL_SWAPD/31337/g | sed s/TMPL_TIMEOUT/${TIMEOUT_ROUND}/g | sed s/TMPL_OWN/${ACCOUNT}/g | sed s/TMPL_FEE/100000/g | sed s/TMPL_MINTRD/10000/g > ${TEMPDIR}/limit-order-b.teal From 4d811fb02e9b621a7c0eb2ceaa8a3e03bf283360 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Mon, 14 Aug 2023 15:55:58 -0400 Subject: [PATCH 028/232] no need for msgp in timers anymore --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index a62b529dad..8301771718 100644 --- a/Makefile +++ b/Makefile @@ -86,7 +86,7 @@ ALGOD_API_PACKAGES := $(sort $(shell GOPATH=$(GOPATH) && GO111MODULE=off && cd d GOMOD_DIRS := ./tools/block-generator ./tools/x-repo-types -MSGP_GENERATE := ./protocol ./protocol/test ./crypto ./crypto/merklearray ./crypto/merklesignature ./crypto/stateproof ./data/basics ./data/transactions ./data/stateproofmsg ./data/committee ./data/bookkeeping ./data/hashable ./agreement ./rpcs ./network ./node ./ledger ./ledger/ledgercore ./ledger/store/trackerdb ./ledger/store/trackerdb/generickv ./ledger/encoded ./stateproof ./data/account ./daemon/algod/api/spec/v2 ./util/timers +MSGP_GENERATE := ./protocol ./protocol/test ./crypto ./crypto/merklearray ./crypto/merklesignature ./crypto/stateproof ./data/basics ./data/transactions ./data/stateproofmsg ./data/committee ./data/bookkeeping ./data/hashable ./agreement ./rpcs ./network ./node ./ledger ./ledger/ledgercore ./ledger/store/trackerdb ./ledger/store/trackerdb/generickv ./ledger/encoded ./stateproof ./data/account ./daemon/algod/api/spec/v2 default: build From ebbd32ba0e1e1d1e97d93788d0941f44fb1e0dac Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Mon, 14 Aug 2023 16:00:24 -0400 Subject: [PATCH 029/232] remove uneeded files --- util/timers/msgp_gen.go | 5 ----- util/timers/msgp_gen_test.go | 6 ------ 2 files changed, 11 deletions(-) delete mode 100644 util/timers/msgp_gen.go delete mode 100644 util/timers/msgp_gen_test.go diff --git a/util/timers/msgp_gen.go b/util/timers/msgp_gen.go deleted file mode 100644 index b3b8ebf5f7..0000000000 --- a/util/timers/msgp_gen.go +++ /dev/null @@ -1,5 +0,0 @@ -package timers - -// Code generated by github.com/algorand/msgp DO NOT EDIT. - -// The following msgp objects are implemented in this file: diff --git a/util/timers/msgp_gen_test.go b/util/timers/msgp_gen_test.go deleted file mode 100644 index 265bbec36f..0000000000 --- a/util/timers/msgp_gen_test.go +++ /dev/null @@ -1,6 +0,0 @@ -//go:build !skip_msgp_testing -// +build !skip_msgp_testing - -package timers - -// Code generated by github.com/algorand/msgp DO NOT EDIT. From be9acf666ca0f6813cde3b153913ddb9a8d13d06 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Tue, 15 Aug 2023 13:20:29 -0400 Subject: [PATCH 030/232] one deadline for each type --- agreement/service.go | 8 +++++ agreement/service_test.go | 70 +++++++++++++++------------------------ util/timers/monotonic.go | 33 ++++++++++-------- 3 files changed, 53 insertions(+), 58 deletions(-) diff --git a/agreement/service.go b/agreement/service.go index 7a519f6911..cea1db99a2 100644 --- a/agreement/service.go +++ b/agreement/service.go @@ -85,11 +85,19 @@ type externalDemuxSignals struct { CurrentRound round } +// TimeoutType taggs the type of a timeout, to distinguish simultaneous timeouts +// of different types. type TimeoutType int8 const ( + // TimeoutDeadline annotates timeout events in the agreement protocol (e.g., + // for receiving a block). TimeoutDeadline TimeoutType = iota + // TimeoutFastRecovery annotates the fast recovery timeout in the agreement + // protocol. TimeoutFastRecovery + // TimeoutFilter annotates the filter step timeout event in the agreement + // protocol. TimeoutFilter ) diff --git a/agreement/service_test.go b/agreement/service_test.go index 783188ff55..fc50c302d5 100644 --- a/agreement/service_test.go +++ b/agreement/service_test.go @@ -44,19 +44,24 @@ import ( "github.com/algorand/go-algorand/util/timers" ) +type testingTimeout struct { + delta time.Duration + ch chan time.Time +} + type testingClock struct { mu deadlock.Mutex zeroes uint - TA map[TimeoutType]map[time.Duration]chan time.Time // TimeoutAt + TA map[TimeoutType]testingTimeout // TimeoutAt monitor *coserviceMonitor } func makeTestingClock(m *coserviceMonitor) *testingClock { c := new(testingClock) - c.TA = make(map[TimeoutType]map[time.Duration]chan time.Time) + c.TA = make(map[TimeoutType]testingTimeout) c.monitor = m return c } @@ -66,7 +71,7 @@ func (c *testingClock) Zero() timers.Clock[TimeoutType] { defer c.mu.Unlock() c.zeroes++ - c.TA = make(map[TimeoutType]map[time.Duration]chan time.Time) + c.TA = make(map[TimeoutType]testingTimeout) c.monitor.clearClock() return c } @@ -79,16 +84,13 @@ func (c *testingClock) TimeoutAt(d time.Duration, timeoutType TimeoutType) <-cha c.mu.Lock() defer c.mu.Unlock() - if _, ok := c.TA[timeoutType]; !ok { - c.TA[timeoutType] = make(map[time.Duration]chan time.Time) + ta, ok := c.TA[timeoutType] + if !ok || ta.delta != d { + c.TA[timeoutType] = testingTimeout{delta: d, ch: make(chan time.Time)} + ta = c.TA[timeoutType] } - ta := c.TA[timeoutType][d] - if ta == nil { - c.TA[timeoutType][d] = make(chan time.Time) - ta = c.TA[timeoutType][d] - } - return ta + return ta.ch } func (c *testingClock) Encode() []byte { @@ -103,34 +105,14 @@ func (c *testingClock) prepareToFire() { c.monitor.inc(clockCoserviceType) } -func (c *testingClock) fireType(timeoutType TimeoutType) { - c.mu.Lock() - defer c.mu.Unlock() - - if _, ok := c.TA[timeoutType]; !ok { - c.TA[timeoutType] = make(map[time.Duration]chan time.Time) - } - - for d := range c.TA[timeoutType] { - if c.TA[timeoutType][d] == nil { - c.TA[timeoutType][d] = make(chan time.Time) - } - close(c.TA[timeoutType][d]) - } -} - -func (c *testingClock) fire(d time.Duration, timeoutType TimeoutType) { +func (c *testingClock) fire(timeoutType TimeoutType) { c.mu.Lock() defer c.mu.Unlock() if _, ok := c.TA[timeoutType]; !ok { - c.TA[timeoutType] = make(map[time.Duration]chan time.Time) - } - - if c.TA[timeoutType][d] == nil { - c.TA[timeoutType][d] = make(chan time.Time) + panic(fmt.Errorf("no timeout of type %v", timeoutType)) } - close(c.TA[timeoutType][d]) + close(c.TA[timeoutType].ch) } type testingNetwork struct { @@ -859,7 +841,7 @@ func triggerGlobalTimeout(d time.Duration, timeoutType TimeoutType, clocks []tim clocks[i].(*testingClock).prepareToFire() } for i := range clocks { - clocks[i].(*testingClock).fire(d, timeoutType) + clocks[i].(*testingClock).fire(timeoutType) } activityMonitor.waitForActivity() activityMonitor.waitForQuiet() @@ -870,7 +852,7 @@ func triggerGlobalTimeoutType(timeoutType TimeoutType, clocks []timers.Clock[Tim clocks[i].(*testingClock).prepareToFire() } for i := range clocks { - clocks[i].(*testingClock).fireType(timeoutType) + clocks[i].(*testingClock).fire(timeoutType) } activityMonitor.waitForActivity() activityMonitor.waitForQuiet() @@ -1147,7 +1129,7 @@ func TestAgreementFastRecoveryDownMiss(t *testing.T) { firstClocks[i].(*testingClock).prepareToFire() } for i := range firstClocks { - firstClocks[i].(*testingClock).fire(firstFPR, TimeoutFastRecovery) + firstClocks[i].(*testingClock).fire(TimeoutFastRecovery) } activityMonitor.waitForActivity() activityMonitor.waitForQuiet() @@ -1158,7 +1140,7 @@ func TestAgreementFastRecoveryDownMiss(t *testing.T) { restClocks[i].(*testingClock).prepareToFire() } for i := range restClocks { - restClocks[i].(*testingClock).fire(firstFPR, TimeoutFastRecovery) + restClocks[i].(*testingClock).fire(TimeoutFastRecovery) } activityMonitor.waitForActivity() activityMonitor.waitForQuiet() @@ -1249,7 +1231,7 @@ func TestAgreementFastRecoveryLate(t *testing.T) { firstClocks[i].(*testingClock).prepareToFire() } for i := range firstClocks { - firstClocks[i].(*testingClock).fire(firstFPR, TimeoutFastRecovery) + firstClocks[i].(*testingClock).fire(TimeoutFastRecovery) } activityMonitor.waitForActivity() activityMonitor.waitForQuiet() @@ -1260,7 +1242,7 @@ func TestAgreementFastRecoveryLate(t *testing.T) { restClocks[i].(*testingClock).prepareToFire() } for i := range restClocks { - restClocks[i].(*testingClock).fire(firstFPR, TimeoutFastRecovery) + restClocks[i].(*testingClock).fire(TimeoutFastRecovery) } activityMonitor.waitForActivity() activityMonitor.waitForQuiet() @@ -1362,7 +1344,7 @@ func TestAgreementFastRecoveryRedo(t *testing.T) { firstClocks[i].(*testingClock).prepareToFire() } for i := range firstClocks { - firstClocks[i].(*testingClock).fire(firstFPR, TimeoutFastRecovery) + firstClocks[i].(*testingClock).fire(TimeoutFastRecovery) } activityMonitor.waitForActivity() activityMonitor.waitForQuiet() @@ -1373,7 +1355,7 @@ func TestAgreementFastRecoveryRedo(t *testing.T) { restClocks[i].(*testingClock).prepareToFire() } for i := range restClocks { - restClocks[i].(*testingClock).fire(firstFPR, TimeoutFastRecovery) + restClocks[i].(*testingClock).fire(TimeoutFastRecovery) } activityMonitor.waitForActivity() activityMonitor.waitForQuiet() @@ -1403,7 +1385,7 @@ func TestAgreementFastRecoveryRedo(t *testing.T) { firstClocks[i].(*testingClock).prepareToFire() } for i := range firstClocks { - firstClocks[i].(*testingClock).fire(firstFPR, TimeoutFastRecovery) + firstClocks[i].(*testingClock).fire(TimeoutFastRecovery) } activityMonitor.waitForActivity() activityMonitor.waitForQuiet() @@ -1414,7 +1396,7 @@ func TestAgreementFastRecoveryRedo(t *testing.T) { restClocks[i].(*testingClock).prepareToFire() } for i := range restClocks { - restClocks[i].(*testingClock).fire(firstFPR, TimeoutFastRecovery) + restClocks[i].(*testingClock).fire(TimeoutFastRecovery) } activityMonitor.waitForActivity() activityMonitor.waitForQuiet() diff --git a/util/timers/monotonic.go b/util/timers/monotonic.go index 1f84a7e6ce..96207f3950 100644 --- a/util/timers/monotonic.go +++ b/util/timers/monotonic.go @@ -23,10 +23,15 @@ import ( "github.com/algorand/go-algorand/protocol" ) +type timeout struct { + delta time.Duration + ch <-chan time.Time +} + // Monotonic uses the system's monotonic clock to emit timeouts. type Monotonic[TimeoutType comparable] struct { zero time.Time - timeouts map[TimeoutType]map[time.Duration]<-chan time.Time + timeouts map[TimeoutType]timeout } // MakeMonotonicClock creates a new monotonic clock with a given zero point. @@ -46,29 +51,29 @@ func (m *Monotonic[TimeoutType]) Zero() Clock[TimeoutType] { // TimeoutAt returns a channel that will signal when the duration has elapsed. func (m *Monotonic[TimeoutType]) TimeoutAt(delta time.Duration, timeoutType TimeoutType) <-chan time.Time { if m.timeouts == nil { - m.timeouts = make(map[TimeoutType]map[time.Duration]<-chan time.Time) + m.timeouts = make(map[TimeoutType]timeout) } - if _, ok := m.timeouts[timeoutType]; !ok { - m.timeouts[timeoutType] = make(map[time.Duration]<-chan time.Time) + tmt, ok := m.timeouts[timeoutType] + if ok && tmt.delta == delta { + // if the new timeout is the same as the current one for that type, + // return the existing channel. + return tmt.ch } - timeoutCh, ok := m.timeouts[timeoutType][delta] - if ok { - return timeoutCh - } + tmt = timeout{delta: delta} target := m.zero.Add(delta) left := target.Sub(time.Now()) if left < 0 { - timeout := make(chan time.Time) - close(timeout) - timeoutCh = timeout + ch := make(chan time.Time) + close(ch) + tmt.ch = ch } else { - timeoutCh = time.After(left) + tmt.ch = time.After(left) } - m.timeouts[timeoutType][delta] = timeoutCh - return timeoutCh + m.timeouts[timeoutType] = tmt + return tmt.ch } // Encode implements Clock.Encode. From c51731afc176c567b0811f9a1435963a0ef2c54e Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Tue, 15 Aug 2023 14:27:28 -0400 Subject: [PATCH 031/232] 95th percentile deadline --- agreement/player.go | 12 ++++++------ config/consensus.go | 13 +++++++++---- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/agreement/player.go b/agreement/player.go index dd038d835b..8d2e7b6b2d 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -283,10 +283,10 @@ func (p *player) handleWinningPayloadArrival(payload proposal, ver protocol.Cons func (p *player) resizePayloadArrivals(ver protocol.ConsensusVersion) { proto := config.Consensus[ver] - if len(p.payloadArrivals) > proto.DynamicFilterPayloadArriavalHistory { - p.payloadArrivals = p.payloadArrivals[len(p.payloadArrivals)-proto.DynamicFilterPayloadArriavalHistory:] + if len(p.payloadArrivals) > proto.DynamicFilterCredentialArrivalHistory { + p.payloadArrivals = p.payloadArrivals[len(p.payloadArrivals)-proto.DynamicFilterCredentialArrivalHistory:] } - for len(p.payloadArrivals) < proto.DynamicFilterPayloadArriavalHistory { + for len(p.payloadArrivals) < proto.DynamicFilterCredentialArrivalHistory { p.payloadArrivals = append([]time.Duration{FilterTimeout(0, ver)}, p.payloadArrivals...) } } @@ -304,17 +304,17 @@ func (p *player) calculateFilterTimeout(ver protocol.ConsensusVersion, tracer *t var dynamicDelay time.Duration defaultDelay := FilterTimeout(0, ver) - if proto.DynamicFilterPayloadArriavalHistory <= 0 { + if proto.DynamicFilterCredentialArrivalHistory <= 0 { // we don't keep any history, use the default dynamicDelay = defaultDelay - } else if proto.DynamicFilterPayloadArriavalHistory > len(p.payloadArrivals) { + } else if proto.DynamicFilterCredentialArrivalHistory > len(p.payloadArrivals) { // not enough samples, use the default dynamicDelay = defaultDelay } else { sortedArrivals := make([]time.Duration, len(p.payloadArrivals)) copy(sortedArrivals[:], p.payloadArrivals[:]) sort.Slice(sortedArrivals, func(i, j int) bool { return sortedArrivals[i] < sortedArrivals[j] }) - dynamicDelay = sortedArrivals[proto.DynamicFilterPayloadArriavalHistory-1] + dynamicDelay = sortedArrivals[proto.DynamicFilterCredentialArrivalHistoryIdx] } // Make sure the dynamic delay is not too small or too large diff --git a/config/consensus.go b/config/consensus.go index 5ab8f6931a..46f0d77a17 100644 --- a/config/consensus.go +++ b/config/consensus.go @@ -525,9 +525,11 @@ type ConsensusParams struct { // filter timeout must meet. DynamicFilterTimeoutLowerBound time.Duration - // DynamicFilterPayloadArriavalHistory specifies the number of past block arrivals - // that are measured to determine when to start pipelining the next block. - DynamicFilterPayloadArriavalHistory int + // DynamicFilterCredentialArrivalHistory specifies the number of past credential arrivals + // that are measured to determine the next filter timeout. + DynamicFilterCredentialArrivalHistory int + + DynamicFilterCredentialArrivalHistoryIdx int } // PaysetCommitType enumerates possible ways for the block header to commit to @@ -1377,7 +1379,10 @@ func initConsensusProtocols() { vFuture.DynamicFilterTimeout = true vFuture.DynamicFilterTimeoutLowerBound = time.Second - vFuture.DynamicFilterPayloadArriavalHistory = 32 + // history window of 40, so we have enough statistics to calculate the 95th + // percentile, which is the timestamp at index 37 in the history array. + vFuture.DynamicFilterCredentialArrivalHistory = 40 + vFuture.DynamicFilterCredentialArrivalHistoryIdx = 37 Consensus[protocol.ConsensusFuture] = vFuture From e676f0a00cce61ecacf27fcc3b842de70c01ab88 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Tue, 15 Aug 2023 15:04:14 -0400 Subject: [PATCH 032/232] comment explaining why we store the deadline type --- agreement/player.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/agreement/player.go b/agreement/player.go index d9eaa09b4f..d93d89a404 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -28,7 +28,8 @@ import ( type Deadline struct { _struct struct{} `codec:","` Duration time.Duration - Type TimeoutType + // Type is used to allow tests fire timeouts of specific types. + Type TimeoutType } // The player implements the top-level state machine functionality of the From 9e0a3ff2c0eb85642f056608545bdc8389cd5b3f Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Tue, 15 Aug 2023 15:48:36 -0400 Subject: [PATCH 033/232] grace --- agreement/player.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agreement/player.go b/agreement/player.go index 8d2e7b6b2d..31540325de 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -314,7 +314,7 @@ func (p *player) calculateFilterTimeout(ver protocol.ConsensusVersion, tracer *t sortedArrivals := make([]time.Duration, len(p.payloadArrivals)) copy(sortedArrivals[:], p.payloadArrivals[:]) sort.Slice(sortedArrivals, func(i, j int) bool { return sortedArrivals[i] < sortedArrivals[j] }) - dynamicDelay = sortedArrivals[proto.DynamicFilterCredentialArrivalHistoryIdx] + dynamicDelay = sortedArrivals[proto.DynamicFilterCredentialArrivalHistoryIdx] + 50*time.Millisecond } // Make sure the dynamic delay is not too small or too large From 7d39273337558c7483e4263f19d5de2f40becdfd Mon Sep 17 00:00:00 2001 From: chris erway Date: Tue, 15 Aug 2023 15:51:26 -0400 Subject: [PATCH 034/232] add voteVerified ValidatedAt times --- agreement/demux.go | 2 ++ agreement/events.go | 7 ++++++- agreement/player.go | 6 ++++++ agreement/proposal.go | 2 ++ agreement/proposalStore.go | 5 +++++ agreement/vote.go | 5 +++++ 6 files changed, 26 insertions(+), 1 deletion(-) diff --git a/agreement/demux.go b/agreement/demux.go index a0923e5185..c2e200e927 100644 --- a/agreement/demux.go +++ b/agreement/demux.go @@ -204,6 +204,8 @@ func (d *demux) next(s *Service, deadline Deadline, fastDeadline time.Duration, e = e.(messageEvent).AttachValidatedAt(s.Clock.Since()) case payloadPresent, votePresent: e = e.(messageEvent).AttachReceivedAt(s.Clock.Since()) + case voteVerified: + e = e.(messageEvent).AttachValidatedAt(s.Clock.Since()) } }() diff --git a/agreement/events.go b/agreement/events.go index 52737e5f2c..8eba33006b 100644 --- a/agreement/events.go +++ b/agreement/events.go @@ -942,7 +942,12 @@ func (e checkpointEvent) AttachConsensusVersion(v ConsensusVersionView) external } func (e messageEvent) AttachValidatedAt(d time.Duration) messageEvent { - e.Input.Proposal.validatedAt = d + if e.T == payloadVerified { + e.Input.Proposal.validatedAt = d + } else if e.T == voteVerified && e.Input.UnauthenticatedVote.R.Step == 0 { + // if this is a proposal vote (step 0), record the receivedAt time on the vote + e.Input.Vote.validatedAt = d + } return e } diff --git a/agreement/player.go b/agreement/player.go index dd038d835b..67f8618f41 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -276,6 +276,12 @@ func (p *player) handleCheckpointEvent(r routerHandle, e checkpointEvent) []acti } func (p *player) handleWinningPayloadArrival(payload proposal, ver protocol.ConsensusVersion) { + + // look up lowest vote validatedAt time + //re := readLowestEvent{T: readLowestVote, Round: p.Round, Period: p.Period} + //re = r.dispatch(*p, re, proposalMachineRound, p.Round, p.Period, 0).(readLowestEvent) + //p.payloadArrivals = append(p.payloadArrivals, re.Vote.validatedAt) + // ignoring validatedAt p.payloadArrivals = append(p.payloadArrivals, payload.receivedAt) p.resizePayloadArrivals(ver) diff --git a/agreement/proposal.go b/agreement/proposal.go index bf021f2cfe..f2dde4bd6a 100644 --- a/agreement/proposal.go +++ b/agreement/proposal.go @@ -99,6 +99,8 @@ type proposal struct { // validated (and thus was ready to be delivered to the state // machine), relative to the zero of that round. validatedAt time.Duration + + voteValidatedAt time.Duration } func makeProposal(ve ValidatedBlock, pf crypto.VrfProof, origPer period, origProp basics.Address) proposal { diff --git a/agreement/proposalStore.go b/agreement/proposalStore.go index 080609de50..8751fbe540 100644 --- a/agreement/proposalStore.go +++ b/agreement/proposalStore.go @@ -272,6 +272,11 @@ func (store *proposalStore) handle(r routerHandle, p player, e event) event { a := stagedValue(p, r, p.Round, p.Period) authVote := ea.authenticator(p.Period) + + // attach vote receivedAt to payload + //ea = store.Assemblers[pv] + //ea.Payload.voteValidatedAt = authVote.validatedAt + if a.Proposal == pv { return committableEvent{Proposal: pv, Vote: authVote} } diff --git a/agreement/vote.go b/agreement/vote.go index af157ee3bb..bed20fe88f 100644 --- a/agreement/vote.go +++ b/agreement/vote.go @@ -18,6 +18,7 @@ package agreement import ( "fmt" + "time" "github.com/algorand/go-algorand/crypto" "github.com/algorand/go-algorand/data/basics" @@ -51,6 +52,10 @@ type ( R rawVote `codec:"r"` Cred committee.Credential `codec:"cred"` Sig crypto.OneTimeSignature `codec:"sig,omitempty,omitemptycheckstruct"` + + // validatedAt indicates the time at which this vote was verified (as a voteVerified messageEvent), + // relative to the zero of that round. It is only set for step 0. + validatedAt time.Duration } // unauthenticatedEquivocationVote is a pair of votes which has not From 272f0a6ed436f5095379e0c5d7176ad58055a47c Mon Sep 17 00:00:00 2001 From: chris erway Date: Tue, 15 Aug 2023 17:27:38 -0400 Subject: [PATCH 035/232] add two fields for backwards compatibility --- agreement/persistence.go | 4 ++++ agreement/player.go | 7 ++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/agreement/persistence.go b/agreement/persistence.go index a4b846ea13..0e5e75915a 100644 --- a/agreement/persistence.go +++ b/agreement/persistence.go @@ -244,6 +244,10 @@ func decode(raw []byte, t0 timers.Clock[TimeoutType], log serviceLogger, reflect return } } + if p2.OldDeadline != 0 { + p2.Deadline = Deadline{Duration: p2.OldDeadline, Type: TimeoutDeadline} + p2.OldDeadline = 0 // clear old value + } rr2 = makeRootRouter(p2) err = protocol.Decode(s.Router, &rr2) if err != nil { diff --git a/agreement/player.go b/agreement/player.go index d93d89a404..9bfb53654d 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -49,7 +49,12 @@ type player struct { // Deadline contains the time of the next timeout expected by the player // state machine (relevant to the start of the current period). - Deadline Deadline + Deadline Deadline `codec:"TimersDeadline"` + + // Deadline contains the time of the next timeout expected by the player + // state machine (relevant to the start of the current period). + OldDeadline time.Duration `codec:"Deadline,omitempty"` + // Napping is set when the player is expecting a random timeout (i.e., // to determine when the player chooses to send a next-vote). Napping bool From 089f4cf6cbad5d2269bb05cc80b4aa25be5e414b Mon Sep 17 00:00:00 2001 From: chris erway Date: Tue, 15 Aug 2023 18:11:07 -0400 Subject: [PATCH 036/232] update comment, rerun msgp and add test --- agreement/msgp_gen.go | 87 +++++++++++++++++++++++------------ agreement/persistence_test.go | 16 +++++++ agreement/player.go | 4 +- 3 files changed, 76 insertions(+), 31 deletions(-) diff --git a/agreement/msgp_gen.go b/agreement/msgp_gen.go index 50036e72e8..d083edd8e8 100644 --- a/agreement/msgp_gen.go +++ b/agreement/msgp_gen.go @@ -3948,31 +3948,46 @@ func PeriodRouterMaxSize() (s int) { // MarshalMsg implements msgp.Marshaler func (z *player) MarshalMsg(b []byte) (o []byte) { o = msgp.Require(b, z.Msgsize()) - // map header, size 8 - // string "Deadline" - o = append(o, 0x88, 0xa8, 0x44, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65) - o = (*z).Deadline.MarshalMsg(o) - // string "FastRecoveryDeadline" - o = append(o, 0xb4, 0x46, 0x61, 0x73, 0x74, 0x52, 0x65, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x79, 0x44, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65) - o = msgp.AppendDuration(o, (*z).FastRecoveryDeadline) - // string "LastConcluding" - o = append(o, 0xae, 0x4c, 0x61, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x69, 0x6e, 0x67) - o = msgp.AppendUint64(o, uint64((*z).LastConcluding)) - // string "Napping" - o = append(o, 0xa7, 0x4e, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67) - o = msgp.AppendBool(o, (*z).Napping) - // string "Pending" - o = append(o, 0xa7, 0x50, 0x65, 0x6e, 0x64, 0x69, 0x6e, 0x67) - o = (*z).Pending.MarshalMsg(o) - // string "Period" - o = append(o, 0xa6, 0x50, 0x65, 0x72, 0x69, 0x6f, 0x64) - o = msgp.AppendUint64(o, uint64((*z).Period)) - // string "Round" - o = append(o, 0xa5, 0x52, 0x6f, 0x75, 0x6e, 0x64) - o = (*z).Round.MarshalMsg(o) - // string "Step" - o = append(o, 0xa4, 0x53, 0x74, 0x65, 0x70) - o = msgp.AppendUint64(o, uint64((*z).Step)) + // omitempty: check for empty values + zb0001Len := uint32(9) + var zb0001Mask uint16 /* 10 bits */ + if (*z).OldDeadline == 0 { + zb0001Len-- + zb0001Mask |= 0x1 + } + // variable map header, size zb0001Len + o = append(o, 0x80|uint8(zb0001Len)) + if zb0001Len != 0 { + if (zb0001Mask & 0x1) == 0 { // if not empty + // string "Deadline" + o = append(o, 0xa8, 0x44, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65) + o = msgp.AppendDuration(o, (*z).OldDeadline) + } + // string "FastRecoveryDeadline" + o = append(o, 0xb4, 0x46, 0x61, 0x73, 0x74, 0x52, 0x65, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x79, 0x44, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65) + o = msgp.AppendDuration(o, (*z).FastRecoveryDeadline) + // string "LastConcluding" + o = append(o, 0xae, 0x4c, 0x61, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x69, 0x6e, 0x67) + o = msgp.AppendUint64(o, uint64((*z).LastConcluding)) + // string "Napping" + o = append(o, 0xa7, 0x4e, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67) + o = msgp.AppendBool(o, (*z).Napping) + // string "Pending" + o = append(o, 0xa7, 0x50, 0x65, 0x6e, 0x64, 0x69, 0x6e, 0x67) + o = (*z).Pending.MarshalMsg(o) + // string "Period" + o = append(o, 0xa6, 0x50, 0x65, 0x72, 0x69, 0x6f, 0x64) + o = msgp.AppendUint64(o, uint64((*z).Period)) + // string "Round" + o = append(o, 0xa5, 0x52, 0x6f, 0x75, 0x6e, 0x64) + o = (*z).Round.MarshalMsg(o) + // string "Step" + o = append(o, 0xa4, 0x53, 0x74, 0x65, 0x70) + o = msgp.AppendUint64(o, uint64((*z).Step)) + // string "TimersDeadline" + o = append(o, 0xae, 0x54, 0x69, 0x6d, 0x65, 0x72, 0x73, 0x44, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65) + o = (*z).Deadline.MarshalMsg(o) + } return } @@ -4046,6 +4061,14 @@ func (z *player) UnmarshalMsg(bts []byte) (o []byte, err error) { return } } + if zb0001 > 0 { + zb0001-- + (*z).OldDeadline, bts, err = msgp.ReadDurationBytes(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "OldDeadline") + return + } + } if zb0001 > 0 { zb0001-- (*z).Napping, bts, err = msgp.ReadBoolBytes(bts) @@ -4129,12 +4152,18 @@ func (z *player) UnmarshalMsg(bts []byte) (o []byte, err error) { } (*z).LastConcluding = step(zb0008) } - case "Deadline": + case "TimersDeadline": bts, err = (*z).Deadline.UnmarshalMsg(bts) if err != nil { err = msgp.WrapError(err, "Deadline") return } + case "Deadline": + (*z).OldDeadline, bts, err = msgp.ReadDurationBytes(bts) + if err != nil { + err = msgp.WrapError(err, "OldDeadline") + return + } case "Napping": (*z).Napping, bts, err = msgp.ReadBoolBytes(bts) if err != nil { @@ -4173,18 +4202,18 @@ func (_ *player) CanUnmarshalMsg(z interface{}) bool { // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message func (z *player) Msgsize() (s int) { - s = 1 + 6 + (*z).Round.Msgsize() + 7 + msgp.Uint64Size + 5 + msgp.Uint64Size + 15 + msgp.Uint64Size + 9 + (*z).Deadline.Msgsize() + 8 + msgp.BoolSize + 21 + msgp.DurationSize + 8 + (*z).Pending.Msgsize() + s = 1 + 6 + (*z).Round.Msgsize() + 7 + msgp.Uint64Size + 5 + msgp.Uint64Size + 15 + msgp.Uint64Size + 15 + (*z).Deadline.Msgsize() + 9 + msgp.DurationSize + 8 + msgp.BoolSize + 21 + msgp.DurationSize + 8 + (*z).Pending.Msgsize() return } // MsgIsZero returns whether this is a zero value func (z *player) MsgIsZero() bool { - return ((*z).Round.MsgIsZero()) && ((*z).Period == 0) && ((*z).Step == 0) && ((*z).LastConcluding == 0) && ((*z).Deadline.MsgIsZero()) && ((*z).Napping == false) && ((*z).FastRecoveryDeadline == 0) && ((*z).Pending.MsgIsZero()) + return ((*z).Round.MsgIsZero()) && ((*z).Period == 0) && ((*z).Step == 0) && ((*z).LastConcluding == 0) && ((*z).Deadline.MsgIsZero()) && ((*z).OldDeadline == 0) && ((*z).Napping == false) && ((*z).FastRecoveryDeadline == 0) && ((*z).Pending.MsgIsZero()) } // MaxSize returns a maximum valid message size for this message type func PlayerMaxSize() (s int) { - s = 1 + 6 + basics.RoundMaxSize() + 7 + msgp.Uint64Size + 5 + msgp.Uint64Size + 15 + msgp.Uint64Size + 9 + DeadlineMaxSize() + 8 + msgp.BoolSize + 21 + msgp.DurationSize + 8 + ProposalTableMaxSize() + s = 1 + 6 + basics.RoundMaxSize() + 7 + msgp.Uint64Size + 5 + msgp.Uint64Size + 15 + msgp.Uint64Size + 15 + DeadlineMaxSize() + 9 + msgp.DurationSize + 8 + msgp.BoolSize + 21 + msgp.DurationSize + 8 + ProposalTableMaxSize() return } diff --git a/agreement/persistence_test.go b/agreement/persistence_test.go index e4c1020f6e..38e793ed47 100644 --- a/agreement/persistence_test.go +++ b/agreement/persistence_test.go @@ -52,6 +52,22 @@ func TestAgreementSerialization(t *testing.T) { require.Equalf(t, router, router2, "Router wasn't serialized/deserialized correctly") require.Equalf(t, status, status2, "Status wasn't serialized/deserialized correctly") require.Equalf(t, a, a2, "Action wasn't serialized/deserialized correctly") + + // also check if old version gets "upgraded" as side effect of decode + clock3 := timers.MakeMonotonicClock[TimeoutType](time.Date(2015, 1, 2, 5, 6, 7, 8, time.UTC)) + status3 := player{Round: 350, Step: soft, OldDeadline: time.Duration(23) * time.Second} + router3 := makeRootRouter(status3) + a3 := []action{checkpointAction{}, disconnectAction(messageEvent{}, nil)} + + encodedBytes2 := encode(clock3, router3, status3, a3, false) + + t1 := timers.MakeMonotonicClock[TimeoutType](time.Date(2000, 0, 0, 0, 0, 0, 0, time.UTC)) + clock4, router4, status4, a4, err := decode(encodedBytes2, t1, log, false) + require.NoError(t, err) + require.Equalf(t, clock, clock4, "Clock wasn't serialized/deserialized correctly") + require.Equalf(t, status, status4, "Status wasn't serialized/deserialized correctly") + require.Equalf(t, router, router4, "Router wasn't serialized/deserialized correctly") + require.Equalf(t, a, a4, "Action wasn't serialized/deserialized correctly") } func BenchmarkAgreementSerialization(b *testing.B) { diff --git a/agreement/player.go b/agreement/player.go index 9bfb53654d..4d2ea94294 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -51,8 +51,8 @@ type player struct { // state machine (relevant to the start of the current period). Deadline Deadline `codec:"TimersDeadline"` - // Deadline contains the time of the next timeout expected by the player - // state machine (relevant to the start of the current period). + // OldDeadline contains the value of Deadline used from a previous version, + // for backwards compatibility when deserializing player. OldDeadline time.Duration `codec:"Deadline,omitempty"` // Napping is set when the player is expecting a random timeout (i.e., From 0a3fbea53daa95c54864119a81f58cfce7e032d5 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Wed, 16 Aug 2023 09:50:06 -0400 Subject: [PATCH 037/232] don't choose OldDeadline at random, since the unmarshalling logic is specific to the msgp decoder --- agreement/persistence_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/agreement/persistence_test.go b/agreement/persistence_test.go index 38e793ed47..ef738e86d6 100644 --- a/agreement/persistence_test.go +++ b/agreement/persistence_test.go @@ -187,11 +187,13 @@ func randomizeDiskState() (rr rootRouter, p player) { if err != nil { return } + rr2, err := protocol.RandomizeObject(&rootRouter{}) if err != nil { return } p = *(p2.(*player)) + p.OldDeadline = 0 rr = *(rr2.(*rootRouter)) return } From 1ace7889091f9737929e7fd9814d10304d051985 Mon Sep 17 00:00:00 2001 From: chris erway Date: Wed, 16 Aug 2023 10:39:42 -0400 Subject: [PATCH 038/232] add readLowestEvent from 63c03b907764a583132c55e255d3816d2aad49f8 (orig. from feature/pipeline) --- agreement/events.go | 40 ++++++++++++++++++++++++++++++++++++ agreement/proposalStore.go | 9 ++++++++ agreement/proposalTracker.go | 5 +++++ 3 files changed, 54 insertions(+) diff --git a/agreement/events.go b/agreement/events.go index 8eba33006b..e9fdf6eda2 100644 --- a/agreement/events.go +++ b/agreement/events.go @@ -197,6 +197,14 @@ const ( // readPinned is sent to the proposalStore to read the pinned value, if it exists. readPinned + // readLowestValue is sent to the proposalPeriodMachine to read the + // proposal-vote with the lowest credential. + readLowestValue + + // readLowestPayload is sent to the proposalStore to read the payload + // corresponding to the lowest-credential proposal-vote, if it exists. + readLowestPayload + /* * The following are event types that replace queries, and may warrant * a revision to make them more state-machine-esque. @@ -407,6 +415,38 @@ func (e newRoundEvent) ComparableStr() string { return e.String() } +type readLowestEvent struct { + // T is either readLowestValue or readLowestPayload + T eventType + + // Round and Period are the round and period for which to query + // the lowest-credential value and payload. This type of event + // is only sent for pipelining, which only makes sense for period + // 0, but the Period is here anyway to route to the appropriate + // proposalMachinePeriod. + Round round + Period period + + // Proposal holds the lowest-credential value. + Proposal proposalValue + // Payload holds the payload, if one exists (which is the case if PayloadOK is set). + Payload proposal + // PayloadOK is set if and only if a payload was received for the lowest-credential value. + PayloadOK bool +} + +func (e readLowestEvent) t() eventType { + return e.T +} + +func (e readLowestEvent) String() string { + return fmt.Sprintf("%v: %.5v", e.t().String(), e.Proposal.BlockDigest.String()) +} + +func (e readLowestEvent) ComparableStr() string { + return e.String() +} + type newPeriodEvent struct { // Period holds the latest period relevant to the proposalRoundMachine. Period period diff --git a/agreement/proposalStore.go b/agreement/proposalStore.go index 8751fbe540..6beaded2ec 100644 --- a/agreement/proposalStore.go +++ b/agreement/proposalStore.go @@ -357,6 +357,15 @@ func (store *proposalStore) handle(r routerHandle, p player, e event) event { se.Committable = ea.Assembled se.Payload = ea.Payload return se + case readLowestPayload: + re := e.(readLowestEvent) + re.T = readLowestValue + re = r.dispatch(p, re, proposalMachinePeriod, re.Round, re.Period, 0).(readLowestEvent) + re.T = readLowestPayload + ea := store.Assemblers[re.Proposal] + re.PayloadOK = ea.Assembled + re.Payload = ea.Payload + return re case readPinned: se := e.(pinnedValueEvent) ea := store.Assemblers[store.Pinned] // If pinned is bottom, assembled/payloadOK = false, payload = bottom diff --git a/agreement/proposalTracker.go b/agreement/proposalTracker.go index 59ffb77a28..7627cdd28d 100644 --- a/agreement/proposalTracker.go +++ b/agreement/proposalTracker.go @@ -165,6 +165,11 @@ func (t *proposalTracker) handle(r routerHandle, p player, e event) event { t.Freezer = t.Freezer.freeze() return e + case readLowestValue: + e := e.(readLowestEvent) + e.Proposal = t.Freezer.Lowest.R.Proposal + return e + case softThreshold, certThreshold: e := e.(thresholdEvent) t.Staging = e.Proposal From 911979f76557742861b64987bdfba0d9b50cef74 Mon Sep 17 00:00:00 2001 From: chris erway Date: Tue, 15 Aug 2023 15:40:15 -0400 Subject: [PATCH 039/232] add readLowestVote --- agreement/events.go | 8 +++++++- agreement/eventtype_string.go | 27 +++++++++++++++------------ agreement/proposalStore.go | 3 +++ agreement/proposalTracker.go | 5 +++++ 4 files changed, 30 insertions(+), 13 deletions(-) diff --git a/agreement/events.go b/agreement/events.go index e9fdf6eda2..73a789e2fa 100644 --- a/agreement/events.go +++ b/agreement/events.go @@ -198,9 +198,13 @@ const ( readPinned // readLowestValue is sent to the proposalPeriodMachine to read the - // proposal-vote with the lowest credential. + // proposalValue with the lowest credential. readLowestValue + // readLowestVote is sent to the proposalPeriodMachine to read the + // proposal-vote with the lowest credential. + readLowestVote + // readLowestPayload is sent to the proposalStore to read the payload // corresponding to the lowest-credential proposal-vote, if it exists. readLowestPayload @@ -429,6 +433,8 @@ type readLowestEvent struct { // Proposal holds the lowest-credential value. Proposal proposalValue + // Vote holds the lowest-credential vote. + Vote vote // Payload holds the payload, if one exists (which is the case if PayloadOK is set). Payload proposal // PayloadOK is set if and only if a payload was received for the lowest-credential value. diff --git a/agreement/eventtype_string.go b/agreement/eventtype_string.go index 9da84c1b98..471d93f36e 100644 --- a/agreement/eventtype_string.go +++ b/agreement/eventtype_string.go @@ -37,21 +37,24 @@ func _() { _ = x[newPeriod-26] _ = x[readStaging-27] _ = x[readPinned-28] - _ = x[voteFilterRequest-29] - _ = x[voteFilteredStep-30] - _ = x[nextThresholdStatusRequest-31] - _ = x[nextThresholdStatus-32] - _ = x[freshestBundleRequest-33] - _ = x[freshestBundle-34] - _ = x[dumpVotesRequest-35] - _ = x[dumpVotes-36] - _ = x[wrappedAction-37] - _ = x[checkpointReached-38] + _ = x[readLowestValue-29] + _ = x[readLowestVote-30] + _ = x[readLowestPayload-31] + _ = x[voteFilterRequest-32] + _ = x[voteFilteredStep-33] + _ = x[nextThresholdStatusRequest-34] + _ = x[nextThresholdStatus-35] + _ = x[freshestBundleRequest-36] + _ = x[freshestBundle-37] + _ = x[dumpVotesRequest-38] + _ = x[dumpVotes-39] + _ = x[wrappedAction-40] + _ = x[checkpointReached-41] } -const _eventType_name = "nonevotePresentpayloadPresentbundlePresentvoteVerifiedpayloadVerifiedbundleVerifiedroundInterruptiontimeoutfastTimeoutsoftThresholdcertThresholdnextThresholdproposalCommittableproposalAcceptedvoteFilteredvoteMalformedbundleFilteredbundleMalformedpayloadRejectedpayloadMalformedpayloadPipelinedpayloadAcceptedproposalFrozenvoteAcceptednewRoundnewPeriodreadStagingreadPinnedvoteFilterRequestvoteFilteredStepnextThresholdStatusRequestnextThresholdStatusfreshestBundleRequestfreshestBundledumpVotesRequestdumpVoteswrappedActioncheckpointReached" +const _eventType_name = "nonevotePresentpayloadPresentbundlePresentvoteVerifiedpayloadVerifiedbundleVerifiedroundInterruptiontimeoutfastTimeoutsoftThresholdcertThresholdnextThresholdproposalCommittableproposalAcceptedvoteFilteredvoteMalformedbundleFilteredbundleMalformedpayloadRejectedpayloadMalformedpayloadPipelinedpayloadAcceptedproposalFrozenvoteAcceptednewRoundnewPeriodreadStagingreadPinnedreadLowestValuereadLowestVotereadLowestPayloadvoteFilterRequestvoteFilteredStepnextThresholdStatusRequestnextThresholdStatusfreshestBundleRequestfreshestBundledumpVotesRequestdumpVoteswrappedActioncheckpointReached" -var _eventType_index = [...]uint16{0, 4, 15, 29, 42, 54, 69, 83, 100, 107, 118, 131, 144, 157, 176, 192, 204, 217, 231, 246, 261, 277, 293, 308, 322, 334, 342, 351, 362, 372, 389, 405, 431, 450, 471, 485, 501, 510, 523, 540} +var _eventType_index = [...]uint16{0, 4, 15, 29, 42, 54, 69, 83, 100, 107, 118, 131, 144, 157, 176, 192, 204, 217, 231, 246, 261, 277, 293, 308, 322, 334, 342, 351, 362, 372, 387, 401, 418, 435, 451, 477, 496, 517, 531, 547, 556, 569, 586} func (i eventType) String() string { if i >= eventType(len(_eventType_index)-1) { diff --git a/agreement/proposalStore.go b/agreement/proposalStore.go index 6beaded2ec..e53d6096cf 100644 --- a/agreement/proposalStore.go +++ b/agreement/proposalStore.go @@ -366,6 +366,9 @@ func (store *proposalStore) handle(r routerHandle, p player, e event) event { re.PayloadOK = ea.Assembled re.Payload = ea.Payload return re + case readLowestVote: + re := e.(readLowestEvent) + return r.dispatch(p, re, proposalMachinePeriod, re.Round, re.Period, 0).(readLowestEvent) case readPinned: se := e.(pinnedValueEvent) ea := store.Assemblers[store.Pinned] // If pinned is bottom, assembled/payloadOK = false, payload = bottom diff --git a/agreement/proposalTracker.go b/agreement/proposalTracker.go index 7627cdd28d..d56fcb31a8 100644 --- a/agreement/proposalTracker.go +++ b/agreement/proposalTracker.go @@ -170,6 +170,11 @@ func (t *proposalTracker) handle(r routerHandle, p player, e event) event { e.Proposal = t.Freezer.Lowest.R.Proposal return e + case readLowestVote: + e := e.(readLowestEvent) + e.Vote = t.Freezer.Lowest + return e + case softThreshold, certThreshold: e := e.(thresholdEvent) t.Staging = e.Proposal From 83489b5caabb1740ece82e2fa15db113b45d709f Mon Sep 17 00:00:00 2001 From: chris erway Date: Wed, 16 Aug 2023 11:28:29 -0400 Subject: [PATCH 040/232] update handleWinningPayloadArrival and hook up to player tests --- agreement/player.go | 22 +++++++++++----------- agreement/player_test.go | 28 +++++++++++++++++++++------- agreement/proposalTrackerContract.go | 2 +- 3 files changed, 33 insertions(+), 19 deletions(-) diff --git a/agreement/player.go b/agreement/player.go index 67f8618f41..00b4b9a6e3 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -275,15 +275,15 @@ func (p *player) handleCheckpointEvent(r routerHandle, e checkpointEvent) []acti }} } -func (p *player) handleWinningPayloadArrival(payload proposal, ver protocol.ConsensusVersion) { - - // look up lowest vote validatedAt time - //re := readLowestEvent{T: readLowestVote, Round: p.Round, Period: p.Period} - //re = r.dispatch(*p, re, proposalMachineRound, p.Round, p.Period, 0).(readLowestEvent) - //p.payloadArrivals = append(p.payloadArrivals, re.Vote.validatedAt) - - // ignoring validatedAt - p.payloadArrivals = append(p.payloadArrivals, payload.receivedAt) +// updateDynamicLambdaTimings is called at the end of the round (just before ensureAction +// is generated) to collect round timings for updating dynamic lambda. +func (p *player) updateDynamicLambdaTimings(r routerHandle, payload proposal, ver protocol.ConsensusVersion) { + // look up the validatedAt time of the winning proposal-vote + re := readLowestEvent{T: readLowestVote, Round: p.Round, Period: p.Period} + re = r.dispatch(*p, re, proposalMachineRound, p.Round, p.Period, 0).(readLowestEvent) + if re.Vote.validatedAt != 0 { + p.payloadArrivals = append(p.payloadArrivals, re.Vote.validatedAt) + } p.resizePayloadArrivals(ver) } @@ -354,7 +354,7 @@ func (p *player) handleThresholdEvent(r routerHandle, e thresholdEvent) []action cert := Certificate(e.Bundle) a0 := ensureAction{Payload: res.Payload, Certificate: cert} actions = append(actions, a0) - p.handleWinningPayloadArrival(res.Payload, e.Proto) + p.updateDynamicLambdaTimings(r, res.Payload, e.Proto) as := p.enterRound(r, e, p.Round+1) return append(actions, as...) } @@ -684,7 +684,7 @@ func (p *player) handleMessageEvent(r routerHandle, e messageEvent) (actions []a cert := Certificate(freshestRes.Event.Bundle) a0 := ensureAction{Payload: e.Input.Proposal, Certificate: cert} actions = append(actions, a0) - p.handleWinningPayloadArrival(e.Input.Proposal, e.Proto.Version) + p.updateDynamicLambdaTimings(r, e.Input.Proposal, e.Proto.Version) as := p.enterRound(r, delegatedE, cert.Round+1) return append(actions, as...) } diff --git a/agreement/player_test.go b/agreement/player_test.go index 9a48264470..d5053f6ef0 100644 --- a/agreement/player_test.go +++ b/agreement/player_test.go @@ -3248,6 +3248,7 @@ func TestPlayerRetainsReceivedValidatedAt(t *testing.T) { // send voteVerified message vVote := helper.MakeVerifiedVote(t, 0, r-1, p, propose, *pV) inMsg := messageEvent{T: voteVerified, Input: message{Vote: vVote, UnauthenticatedVote: vVote.u()}} + inMsg = inMsg.AttachValidatedAt(501 * time.Millisecond) err, panicErr := pM.transition(inMsg) require.NoError(t, err) require.NoError(t, panicErr) @@ -3260,7 +3261,13 @@ func TestPlayerRetainsReceivedValidatedAt(t *testing.T) { require.NoError(t, err) require.NoError(t, panicErr) - assertCorrectReceivedAtSet(t, pWhite, pM, helper, r, p, pP, pV, m) + assertCorrectReceivedAtSet(t, pWhite, pM, helper, r, p, pP, pV, m, protocol.ConsensusFuture) + + // assert lowest vote validateAt time was recorded into payloadArrivals + historyLen := config.Consensus[protocol.ConsensusFuture].DynamicFilterPayloadArriavalHistory + require.NotZero(t, historyLen) + require.Len(t, pWhite.payloadArrivals, historyLen) + require.Equal(t, 501*time.Millisecond, pWhite.payloadArrivals[historyLen-1]) } // test that ReceivedAt and ValidateAt timing information are retained in proposalStore @@ -3292,29 +3299,36 @@ func TestPlayerRetainsReceivedValidatedAtPP(t *testing.T) { // send voteVerified inMsg = messageEvent{T: voteVerified, Input: voteMsg, TaskIndex: 1} + inMsg = inMsg.AttachValidatedAt(502 * time.Millisecond) err, panicErr = pM.transition(inMsg) require.NoError(t, err) require.NoError(t, panicErr) - assertCorrectReceivedAtSet(t, pWhite, pM, helper, r, p, pP, pV, proposalMsg) + assertCorrectReceivedAtSet(t, pWhite, pM, helper, r, p, pP, pV, proposalMsg, protocol.ConsensusFuture) + + // assert lowest vote validateAt time was recorded into payloadArrivals + historyLen := config.Consensus[protocol.ConsensusFuture].DynamicFilterPayloadArriavalHistory + require.NotZero(t, historyLen) + require.Len(t, pWhite.payloadArrivals, historyLen) + require.Equal(t, 502*time.Millisecond, pWhite.payloadArrivals[historyLen-1]) } func assertCorrectReceivedAtSet(t *testing.T, pWhite *player, pM ioAutomata, helper *voteMakerHelper, - r round, p period, pP *proposal, pV *proposalValue, m message) { + r round, p period, pP *proposal, pV *proposalValue, m message, ver protocol.ConsensusVersion) { // make sure payload verify request verifyEvent := ev(cryptoAction{T: verifyPayload, M: m, Round: r - 1, Period: p, Step: propose, TaskIndex: 0}) require.Truef(t, pM.getTrace().Contains(verifyEvent), "Player should verify payload") // payloadVerified - inMsg := messageEvent{T: payloadVerified, Input: message{Proposal: *pP}, Proto: ConsensusVersionView{Version: protocol.ConsensusCurrentVersion}} + inMsg := messageEvent{T: payloadVerified, Input: message{Proposal: *pP}, Proto: ConsensusVersionView{Version: ver}} inMsg = inMsg.AttachValidatedAt(2 * time.Second) // call AttachValidatedAt like demux would err, panicErr := pM.transition(inMsg) require.NoError(t, err) require.NoError(t, panicErr) // gen cert to move into the next round - votes := make([]vote, int(cert.threshold(config.Consensus[protocol.ConsensusCurrentVersion]))) - for i := 0; i < int(cert.threshold(config.Consensus[protocol.ConsensusCurrentVersion])); i++ { + votes := make([]vote, int(cert.threshold(config.Consensus[ver]))) + for i := 0; i < int(cert.threshold(config.Consensus[ver])); i++ { votes[i] = helper.MakeVerifiedVote(t, i, r-1, p, cert, *pV) } bun := unauthenticatedBundle{ @@ -3331,7 +3345,7 @@ func assertCorrectReceivedAtSet(t *testing.T, pWhite *player, pM ioAutomata, hel }, UnauthenticatedBundle: bun, }, - Proto: ConsensusVersionView{Version: protocol.ConsensusCurrentVersion}, + Proto: ConsensusVersionView{Version: ver}, } err, panicErr = pM.transition(inMsg) require.NoError(t, err) diff --git a/agreement/proposalTrackerContract.go b/agreement/proposalTrackerContract.go index 2b995dfcac..bfc39170e8 100644 --- a/agreement/proposalTrackerContract.go +++ b/agreement/proposalTrackerContract.go @@ -32,7 +32,7 @@ type proposalTrackerContract struct { // TODO check concrete types of events func (c *proposalTrackerContract) pre(p player, in event) (pre []error) { switch in.t() { - case voteVerified, proposalFrozen, softThreshold, certThreshold, voteFilterRequest, readStaging: + case voteVerified, proposalFrozen, softThreshold, certThreshold, voteFilterRequest, readStaging, readLowestValue, readLowestPayload, readLowestVote: default: pre = append(pre, fmt.Errorf("incoming event has invalid type: %v", in.t())) } From 93c6a49b25f238650c7ee012709389dcb723ef7a Mon Sep 17 00:00:00 2001 From: chris erway Date: Wed, 16 Aug 2023 11:38:42 -0400 Subject: [PATCH 041/232] rerun msgp --- agreement/msgp_gen.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/agreement/msgp_gen.go b/agreement/msgp_gen.go index ee65757f52..d8de90ff82 100644 --- a/agreement/msgp_gen.go +++ b/agreement/msgp_gen.go @@ -4259,7 +4259,7 @@ func (z *proposal) MarshalMsg(b []byte) (o []byte) { o = msgp.Require(b, z.Msgsize()) // omitempty: check for empty values zb0004Len := uint32(29) - var zb0004Mask uint64 /* 38 bits */ + var zb0004Mask uint64 /* 39 bits */ if (*z).unauthenticatedProposal.Block.BlockHeader.RewardsState.RewardsLevel == 0 { zb0004Len-- zb0004Mask |= 0x40 @@ -11192,7 +11192,7 @@ func (z *vote) MarshalMsg(b []byte) (o []byte) { o = msgp.Require(b, z.Msgsize()) // omitempty: check for empty values zb0001Len := uint32(3) - var zb0001Mask uint8 /* 4 bits */ + var zb0001Mask uint8 /* 5 bits */ if (*z).Cred.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x2 From c7b9d9626edc17e13579fe1a528822a18d7c1b4b Mon Sep 17 00:00:00 2001 From: cce <51567+cce@users.noreply.github.com> Date: Wed, 16 Aug 2023 12:13:03 -0400 Subject: [PATCH 042/232] Update proposalStore.go remove commented out code from when we were trying to get receivedAt for proposal-votes --- agreement/proposalStore.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/agreement/proposalStore.go b/agreement/proposalStore.go index e53d6096cf..8f89f4fc73 100644 --- a/agreement/proposalStore.go +++ b/agreement/proposalStore.go @@ -272,11 +272,6 @@ func (store *proposalStore) handle(r routerHandle, p player, e event) event { a := stagedValue(p, r, p.Round, p.Period) authVote := ea.authenticator(p.Period) - - // attach vote receivedAt to payload - //ea = store.Assemblers[pv] - //ea.Payload.voteValidatedAt = authVote.validatedAt - if a.Proposal == pv { return committableEvent{Proposal: pv, Vote: authVote} } From 951ecd0ab1d79288d074a6e2b203fe24e92c1211 Mon Sep 17 00:00:00 2001 From: cce <51567+cce@users.noreply.github.com> Date: Wed, 16 Aug 2023 12:14:06 -0400 Subject: [PATCH 043/232] Update events.go fix comment to say validated instead of received --- agreement/events.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agreement/events.go b/agreement/events.go index 73a789e2fa..0458e9e9fa 100644 --- a/agreement/events.go +++ b/agreement/events.go @@ -991,7 +991,7 @@ func (e messageEvent) AttachValidatedAt(d time.Duration) messageEvent { if e.T == payloadVerified { e.Input.Proposal.validatedAt = d } else if e.T == voteVerified && e.Input.UnauthenticatedVote.R.Step == 0 { - // if this is a proposal vote (step 0), record the receivedAt time on the vote + // if this is a proposal vote (step 0), record the validatedAt time on the vote e.Input.Vote.validatedAt = d } return e From 905551b19e54ea8436c2c4fa6d44198848cecc99 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Wed, 16 Aug 2023 14:16:21 -0400 Subject: [PATCH 044/232] doc new parameter --- config/consensus.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config/consensus.go b/config/consensus.go index 46f0d77a17..f34ff896a6 100644 --- a/config/consensus.go +++ b/config/consensus.go @@ -529,6 +529,8 @@ type ConsensusParams struct { // that are measured to determine the next filter timeout. DynamicFilterCredentialArrivalHistory int + // DynamicFilterCredentialArrivalHistoryIdx specified which sample to use out of a sorted + // DynamicFilterCredentialArrivalHistory-sized array of time samples. DynamicFilterCredentialArrivalHistoryIdx int } From 2778bddac81c5d1809c3a25860923ddb0736e731 Mon Sep 17 00:00:00 2001 From: chris erway Date: Wed, 16 Aug 2023 14:33:22 -0400 Subject: [PATCH 045/232] add TestPlayerRetainsReceivedValidatedAtAVPP and comments --- agreement/player.go | 2 +- agreement/player_test.go | 70 +++++++++++++++++++++++++++++---- agreement/state_machine_test.go | 9 +++++ 3 files changed, 73 insertions(+), 8 deletions(-) diff --git a/agreement/player.go b/agreement/player.go index 00b4b9a6e3..abba0e631d 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -275,7 +275,7 @@ func (p *player) handleCheckpointEvent(r routerHandle, e checkpointEvent) []acti }} } -// updateDynamicLambdaTimings is called at the end of the round (just before ensureAction +// updateDynamicLambdaTimings is called at the end of a successful uninterrupted round (just after ensureAction // is generated) to collect round timings for updating dynamic lambda. func (p *player) updateDynamicLambdaTimings(r routerHandle, payload proposal, ver protocol.ConsensusVersion) { // look up the validatedAt time of the winning proposal-vote diff --git a/agreement/player_test.go b/agreement/player_test.go index d5053f6ef0..8506d8c5ba 100644 --- a/agreement/player_test.go +++ b/agreement/player_test.go @@ -3235,7 +3235,7 @@ func TestPlayerAlwaysResynchsPinnedValue(t *testing.T) { } // test that ReceivedAt and ValidateAt timing information are retained in proposalStore -// when the payloadPresent and payloadVerified events are processed, and that both timings +// when the payloadPresent, payloadVerified, and voteVerified events are processed, and that all timings // are available when the ensureAction is called for the block. func TestPlayerRetainsReceivedValidatedAt(t *testing.T) { partitiontest.PartitionTest(t) @@ -3271,8 +3271,8 @@ func TestPlayerRetainsReceivedValidatedAt(t *testing.T) { } // test that ReceivedAt and ValidateAt timing information are retained in proposalStore -// when the payloadPresent (as part of the CompoundMessage encoding used by PP messages) -// and payloadVerified events are processed, and that both timings +// when the payloadPresent (as part of the CompoundMessage encoding used by PP messages), +// payloadVerified, and voteVerified events are processed, and that all timings // are available when the ensureAction is called for the block. func TestPlayerRetainsReceivedValidatedAtPP(t *testing.T) { partitiontest.PartitionTest(t) @@ -3284,9 +3284,9 @@ func TestPlayerRetainsReceivedValidatedAtPP(t *testing.T) { // create a PP message for an arbitrary proposal/payload similar to setupCompoundMessage vVote := helper.MakeVerifiedVote(t, 0, r-1, p, propose, *pV) - voteMsg := message{Vote: vVote, UnauthenticatedVote: vVote.u()} + unverifiedVoteMsg := message{UnauthenticatedVote: vVote.u()} proposalMsg := message{UnauthenticatedProposal: pP.u()} - compoundMsg := messageEvent{T: votePresent, Input: voteMsg, + compoundMsg := messageEvent{T: votePresent, Input: unverifiedVoteMsg, Tail: &messageEvent{T: payloadPresent, Input: proposalMsg}} inMsg := compoundMsg.AttachReceivedAt(time.Second) // call AttachReceivedAt like demux would err, panicErr := pM.transition(inMsg) @@ -3294,11 +3294,12 @@ func TestPlayerRetainsReceivedValidatedAtPP(t *testing.T) { require.NoError(t, panicErr) // make sure vote verify requests - verifyEvent := ev(cryptoAction{T: verifyVote, M: voteMsg, Round: r - 1, Period: p, Step: propose, TaskIndex: 1}) + verifyEvent := ev(cryptoAction{T: verifyVote, M: unverifiedVoteMsg, Round: r - 1, Period: p, Step: propose, TaskIndex: 1}) require.Truef(t, pM.getTrace().Contains(verifyEvent), "Player should verify vote") // send voteVerified - inMsg = messageEvent{T: voteVerified, Input: voteMsg, TaskIndex: 1} + verifiedVoteMsg := message{Vote: vVote, UnauthenticatedVote: vVote.u()} + inMsg = messageEvent{T: voteVerified, Input: verifiedVoteMsg, TaskIndex: 1} inMsg = inMsg.AttachValidatedAt(502 * time.Millisecond) err, panicErr = pM.transition(inMsg) require.NoError(t, err) @@ -3313,6 +3314,61 @@ func TestPlayerRetainsReceivedValidatedAtPP(t *testing.T) { require.Equal(t, 502*time.Millisecond, pWhite.payloadArrivals[historyLen-1]) } +// test that ReceivedAt and ValidateAt timing information are retained in proposalStore +// when the voteVerified event comes in first (as part of the AV message before PP), +// then the payloadPresent (as part of the CompoundMessage encoding used by PP messages) +// and payloadVerified events are processed, and that all timings +// are available when the ensureAction is called for the block. +func TestPlayerRetainsReceivedValidatedAtAVPP(t *testing.T) { + partitiontest.PartitionTest(t) + + const r = round(20239) + const p = period(1001) + pWhite, pM, helper := setupP(t, r-1, p, soft) + pP, pV := helper.MakeRandomProposalPayload(t, r-1) + + // send votePresent message (mimicking the first AV message validating) + vVote := helper.MakeVerifiedVote(t, 0, r-1, p, propose, *pV) + unverifiedVoteMsg := message{UnauthenticatedVote: vVote.u()} + inMsg := messageEvent{T: votePresent, Input: unverifiedVoteMsg} + err, panicErr := pM.transition(inMsg) + require.NoError(t, err) + require.NoError(t, panicErr) + + // make sure vote verify requests + verifyEvent := ev(cryptoAction{T: verifyVote, M: unverifiedVoteMsg, Round: r - 1, Period: p, Step: propose, TaskIndex: 1}) + require.Truef(t, pM.getTrace().Contains(verifyEvent), "Player should verify vote") + + // send voteVerified + verifiedVoteMsg := message{Vote: vVote, UnauthenticatedVote: vVote.u()} + inMsg = messageEvent{T: voteVerified, Input: verifiedVoteMsg, TaskIndex: 1} + inMsg = inMsg.AttachValidatedAt(502 * time.Millisecond) + err, panicErr = pM.transition(inMsg) + require.NoError(t, err) + require.NoError(t, panicErr) + + // create a PP message for an arbitrary proposal/payload similar to setupCompoundMessage + proposalMsg := message{UnauthenticatedProposal: pP.u()} + compoundMsg := messageEvent{T: votePresent, Input: unverifiedVoteMsg, + Tail: &messageEvent{T: payloadPresent, Input: proposalMsg}} + inMsg = compoundMsg.AttachReceivedAt(time.Second) // call AttachReceivedAt like demux would + err, panicErr = pM.transition(inMsg) + require.NoError(t, err) + require.NoError(t, panicErr) + + // make sure no second request to verify this vote + verifyEvent = ev(cryptoAction{T: verifyVote, M: unverifiedVoteMsg, Round: r - 1, Period: p, Step: propose, TaskIndex: 1}) + require.Equal(t, 1, pM.getTrace().CountEvent(verifyEvent), "Player should not verify second vote") + + assertCorrectReceivedAtSet(t, pWhite, pM, helper, r, p, pP, pV, proposalMsg, protocol.ConsensusFuture) + + // assert lowest vote validateAt time was recorded into payloadArrivals + historyLen := config.Consensus[protocol.ConsensusFuture].DynamicFilterPayloadArriavalHistory + require.NotZero(t, historyLen) + require.Len(t, pWhite.payloadArrivals, historyLen) + require.Equal(t, 502*time.Millisecond, pWhite.payloadArrivals[historyLen-1]) +} + func assertCorrectReceivedAtSet(t *testing.T, pWhite *player, pM ioAutomata, helper *voteMakerHelper, r round, p period, pP *proposal, pV *proposalValue, m message, ver protocol.ConsensusVersion) { // make sure payload verify request diff --git a/agreement/state_machine_test.go b/agreement/state_machine_test.go index 0effd9fda1..69b31a9866 100644 --- a/agreement/state_machine_test.go +++ b/agreement/state_machine_test.go @@ -140,6 +140,15 @@ func (t ioTrace) Contains(e event) bool { }) } +func (t ioTrace) CountEvent(b event) (count int) { + for _, e := range t.events { + if e.ComparableStr() == b.ComparableStr() { + count++ + } + } + return +} + // for each event, passes it into the given fn; if returns true, returns true. func (t ioTrace) ContainsFn(compareFn func(b event) bool) bool { for _, ev := range t.events { From d50840564da7fe1c317c1c00c4fadae331026ab3 Mon Sep 17 00:00:00 2001 From: chris erway Date: Wed, 16 Aug 2023 15:05:28 -0400 Subject: [PATCH 046/232] remove readLowestValue and readLowestPayload --- agreement/demux.go | 5 ++++- agreement/events.go | 22 ++++------------------ agreement/eventtype_string.go | 28 +++++++++++++--------------- agreement/proposalStore.go | 9 --------- agreement/proposalTracker.go | 5 ----- agreement/proposalTrackerContract.go | 2 +- 6 files changed, 22 insertions(+), 49 deletions(-) diff --git a/agreement/demux.go b/agreement/demux.go index c2e200e927..11701101fb 100644 --- a/agreement/demux.go +++ b/agreement/demux.go @@ -205,7 +205,10 @@ func (d *demux) next(s *Service, deadline Deadline, fastDeadline time.Duration, case payloadPresent, votePresent: e = e.(messageEvent).AttachReceivedAt(s.Clock.Since()) case voteVerified: - e = e.(messageEvent).AttachValidatedAt(s.Clock.Since()) + // if this is a proposal vote (step 0), record the validatedAt time on the vote + if e.(messageEvent).Input.UnauthenticatedVote.R.Step == 0 { + e = e.(messageEvent).AttachValidatedAt(s.Clock.Since()) + } } }() diff --git a/agreement/events.go b/agreement/events.go index 0458e9e9fa..7f79efc198 100644 --- a/agreement/events.go +++ b/agreement/events.go @@ -197,18 +197,10 @@ const ( // readPinned is sent to the proposalStore to read the pinned value, if it exists. readPinned - // readLowestValue is sent to the proposalPeriodMachine to read the - // proposalValue with the lowest credential. - readLowestValue - // readLowestVote is sent to the proposalPeriodMachine to read the // proposal-vote with the lowest credential. readLowestVote - // readLowestPayload is sent to the proposalStore to read the payload - // corresponding to the lowest-credential proposal-vote, if it exists. - readLowestPayload - /* * The following are event types that replace queries, and may warrant * a revision to make them more state-machine-esque. @@ -431,14 +423,8 @@ type readLowestEvent struct { Round round Period period - // Proposal holds the lowest-credential value. - Proposal proposalValue // Vote holds the lowest-credential vote. Vote vote - // Payload holds the payload, if one exists (which is the case if PayloadOK is set). - Payload proposal - // PayloadOK is set if and only if a payload was received for the lowest-credential value. - PayloadOK bool } func (e readLowestEvent) t() eventType { @@ -446,7 +432,7 @@ func (e readLowestEvent) t() eventType { } func (e readLowestEvent) String() string { - return fmt.Sprintf("%v: %.5v", e.t().String(), e.Proposal.BlockDigest.String()) + return fmt.Sprintf("%s: %d\t%.10s\t%.5s", e.t().String(), e.Round, e.Period) } func (e readLowestEvent) ComparableStr() string { @@ -988,10 +974,10 @@ func (e checkpointEvent) AttachConsensusVersion(v ConsensusVersionView) external } func (e messageEvent) AttachValidatedAt(d time.Duration) messageEvent { - if e.T == payloadVerified { + switch e.T { + case payloadVerified: e.Input.Proposal.validatedAt = d - } else if e.T == voteVerified && e.Input.UnauthenticatedVote.R.Step == 0 { - // if this is a proposal vote (step 0), record the validatedAt time on the vote + case voteVerified: e.Input.Vote.validatedAt = d } return e diff --git a/agreement/eventtype_string.go b/agreement/eventtype_string.go index 471d93f36e..9973215b12 100644 --- a/agreement/eventtype_string.go +++ b/agreement/eventtype_string.go @@ -37,24 +37,22 @@ func _() { _ = x[newPeriod-26] _ = x[readStaging-27] _ = x[readPinned-28] - _ = x[readLowestValue-29] - _ = x[readLowestVote-30] - _ = x[readLowestPayload-31] - _ = x[voteFilterRequest-32] - _ = x[voteFilteredStep-33] - _ = x[nextThresholdStatusRequest-34] - _ = x[nextThresholdStatus-35] - _ = x[freshestBundleRequest-36] - _ = x[freshestBundle-37] - _ = x[dumpVotesRequest-38] - _ = x[dumpVotes-39] - _ = x[wrappedAction-40] - _ = x[checkpointReached-41] + _ = x[readLowestVote-29] + _ = x[voteFilterRequest-30] + _ = x[voteFilteredStep-31] + _ = x[nextThresholdStatusRequest-32] + _ = x[nextThresholdStatus-33] + _ = x[freshestBundleRequest-34] + _ = x[freshestBundle-35] + _ = x[dumpVotesRequest-36] + _ = x[dumpVotes-37] + _ = x[wrappedAction-38] + _ = x[checkpointReached-39] } -const _eventType_name = "nonevotePresentpayloadPresentbundlePresentvoteVerifiedpayloadVerifiedbundleVerifiedroundInterruptiontimeoutfastTimeoutsoftThresholdcertThresholdnextThresholdproposalCommittableproposalAcceptedvoteFilteredvoteMalformedbundleFilteredbundleMalformedpayloadRejectedpayloadMalformedpayloadPipelinedpayloadAcceptedproposalFrozenvoteAcceptednewRoundnewPeriodreadStagingreadPinnedreadLowestValuereadLowestVotereadLowestPayloadvoteFilterRequestvoteFilteredStepnextThresholdStatusRequestnextThresholdStatusfreshestBundleRequestfreshestBundledumpVotesRequestdumpVoteswrappedActioncheckpointReached" +const _eventType_name = "nonevotePresentpayloadPresentbundlePresentvoteVerifiedpayloadVerifiedbundleVerifiedroundInterruptiontimeoutfastTimeoutsoftThresholdcertThresholdnextThresholdproposalCommittableproposalAcceptedvoteFilteredvoteMalformedbundleFilteredbundleMalformedpayloadRejectedpayloadMalformedpayloadPipelinedpayloadAcceptedproposalFrozenvoteAcceptednewRoundnewPeriodreadStagingreadPinnedreadLowestVotevoteFilterRequestvoteFilteredStepnextThresholdStatusRequestnextThresholdStatusfreshestBundleRequestfreshestBundledumpVotesRequestdumpVoteswrappedActioncheckpointReached" -var _eventType_index = [...]uint16{0, 4, 15, 29, 42, 54, 69, 83, 100, 107, 118, 131, 144, 157, 176, 192, 204, 217, 231, 246, 261, 277, 293, 308, 322, 334, 342, 351, 362, 372, 387, 401, 418, 435, 451, 477, 496, 517, 531, 547, 556, 569, 586} +var _eventType_index = [...]uint16{0, 4, 15, 29, 42, 54, 69, 83, 100, 107, 118, 131, 144, 157, 176, 192, 204, 217, 231, 246, 261, 277, 293, 308, 322, 334, 342, 351, 362, 372, 386, 403, 419, 445, 464, 485, 499, 515, 524, 537, 554} func (i eventType) String() string { if i >= eventType(len(_eventType_index)-1) { diff --git a/agreement/proposalStore.go b/agreement/proposalStore.go index 8f89f4fc73..fdfecac5f2 100644 --- a/agreement/proposalStore.go +++ b/agreement/proposalStore.go @@ -352,15 +352,6 @@ func (store *proposalStore) handle(r routerHandle, p player, e event) event { se.Committable = ea.Assembled se.Payload = ea.Payload return se - case readLowestPayload: - re := e.(readLowestEvent) - re.T = readLowestValue - re = r.dispatch(p, re, proposalMachinePeriod, re.Round, re.Period, 0).(readLowestEvent) - re.T = readLowestPayload - ea := store.Assemblers[re.Proposal] - re.PayloadOK = ea.Assembled - re.Payload = ea.Payload - return re case readLowestVote: re := e.(readLowestEvent) return r.dispatch(p, re, proposalMachinePeriod, re.Round, re.Period, 0).(readLowestEvent) diff --git a/agreement/proposalTracker.go b/agreement/proposalTracker.go index d56fcb31a8..b9707d54dc 100644 --- a/agreement/proposalTracker.go +++ b/agreement/proposalTracker.go @@ -165,11 +165,6 @@ func (t *proposalTracker) handle(r routerHandle, p player, e event) event { t.Freezer = t.Freezer.freeze() return e - case readLowestValue: - e := e.(readLowestEvent) - e.Proposal = t.Freezer.Lowest.R.Proposal - return e - case readLowestVote: e := e.(readLowestEvent) e.Vote = t.Freezer.Lowest diff --git a/agreement/proposalTrackerContract.go b/agreement/proposalTrackerContract.go index bfc39170e8..a6fa2ffc36 100644 --- a/agreement/proposalTrackerContract.go +++ b/agreement/proposalTrackerContract.go @@ -32,7 +32,7 @@ type proposalTrackerContract struct { // TODO check concrete types of events func (c *proposalTrackerContract) pre(p player, in event) (pre []error) { switch in.t() { - case voteVerified, proposalFrozen, softThreshold, certThreshold, voteFilterRequest, readStaging, readLowestValue, readLowestPayload, readLowestVote: + case voteVerified, proposalFrozen, softThreshold, certThreshold, voteFilterRequest, readStaging, readLowestVote: default: pre = append(pre, fmt.Errorf("incoming event has invalid type: %v", in.t())) } From 6a31232bce2289f05a58a51a874547941b6370af Mon Sep 17 00:00:00 2001 From: chris erway Date: Wed, 16 Aug 2023 15:15:27 -0400 Subject: [PATCH 047/232] rename to lowestCredentialArrivals --- agreement/player.go | 36 +++++++++++++++++++++--------------- agreement/player_test.go | 12 ++++++------ 2 files changed, 27 insertions(+), 21 deletions(-) diff --git a/agreement/player.go b/agreement/player.go index abba0e631d..15a68316b8 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -63,7 +63,9 @@ type player struct { // must be verified after some vote has been verified. Pending proposalTable - payloadArrivals []time.Duration + // the history of arrival times of the lowest credential from previous + // ronuds, used for calculating dynamic lambda. + lowestCredentialArrivals []time.Duration } func (p *player) T() stateMachineTag { @@ -277,23 +279,27 @@ func (p *player) handleCheckpointEvent(r routerHandle, e checkpointEvent) []acti // updateDynamicLambdaTimings is called at the end of a successful uninterrupted round (just after ensureAction // is generated) to collect round timings for updating dynamic lambda. -func (p *player) updateDynamicLambdaTimings(r routerHandle, payload proposal, ver protocol.ConsensusVersion) { +func (p *player) updateDynamicLambdaTimings(r routerHandle, ver protocol.ConsensusVersion) { + // only append to lowestCredentialArrivals if this was a successful round completing in period 0. + if p.Period != 0 { + return + } // look up the validatedAt time of the winning proposal-vote re := readLowestEvent{T: readLowestVote, Round: p.Round, Period: p.Period} re = r.dispatch(*p, re, proposalMachineRound, p.Round, p.Period, 0).(readLowestEvent) if re.Vote.validatedAt != 0 { - p.payloadArrivals = append(p.payloadArrivals, re.Vote.validatedAt) + p.lowestCredentialArrivals = append(p.lowestCredentialArrivals, re.Vote.validatedAt) } - p.resizePayloadArrivals(ver) + p.resizeLowestCredentialArrivals(ver) } -func (p *player) resizePayloadArrivals(ver protocol.ConsensusVersion) { +func (p *player) resizeLowestCredentialArrivals(ver protocol.ConsensusVersion) { proto := config.Consensus[ver] - if len(p.payloadArrivals) > proto.DynamicFilterPayloadArriavalHistory { - p.payloadArrivals = p.payloadArrivals[len(p.payloadArrivals)-proto.DynamicFilterPayloadArriavalHistory:] + if len(p.lowestCredentialArrivals) > proto.DynamicFilterPayloadArriavalHistory { + p.lowestCredentialArrivals = p.lowestCredentialArrivals[len(p.lowestCredentialArrivals)-proto.DynamicFilterPayloadArriavalHistory:] } - for len(p.payloadArrivals) < proto.DynamicFilterPayloadArriavalHistory { - p.payloadArrivals = append([]time.Duration{FilterTimeout(0, ver)}, p.payloadArrivals...) + for len(p.lowestCredentialArrivals) < proto.DynamicFilterPayloadArriavalHistory { + p.lowestCredentialArrivals = append([]time.Duration{FilterTimeout(0, ver)}, p.lowestCredentialArrivals...) } } @@ -313,12 +319,12 @@ func (p *player) calculateFilterTimeout(ver protocol.ConsensusVersion, tracer *t if proto.DynamicFilterPayloadArriavalHistory <= 0 { // we don't keep any history, use the default dynamicDelay = defaultDelay - } else if proto.DynamicFilterPayloadArriavalHistory > len(p.payloadArrivals) { + } else if proto.DynamicFilterPayloadArriavalHistory > len(p.lowestCredentialArrivals) { // not enough samples, use the default dynamicDelay = defaultDelay } else { - sortedArrivals := make([]time.Duration, len(p.payloadArrivals)) - copy(sortedArrivals[:], p.payloadArrivals[:]) + sortedArrivals := make([]time.Duration, len(p.lowestCredentialArrivals)) + copy(sortedArrivals[:], p.lowestCredentialArrivals[:]) sort.Slice(sortedArrivals, func(i, j int) bool { return sortedArrivals[i] < sortedArrivals[j] }) dynamicDelay = sortedArrivals[proto.DynamicFilterPayloadArriavalHistory-1] } @@ -354,7 +360,7 @@ func (p *player) handleThresholdEvent(r routerHandle, e thresholdEvent) []action cert := Certificate(e.Bundle) a0 := ensureAction{Payload: res.Payload, Certificate: cert} actions = append(actions, a0) - p.updateDynamicLambdaTimings(r, res.Payload, e.Proto) + p.updateDynamicLambdaTimings(r, e.Proto) as := p.enterRound(r, e, p.Round+1) return append(actions, as...) } @@ -411,7 +417,7 @@ func (p *player) enterPeriod(r routerHandle, source thresholdEvent, target perio if target != 0 { // We entered a non-0 period, we should reset the filter timeout // calculation mechanism. - p.payloadArrivals = make([]time.Duration, 0) + p.lowestCredentialArrivals = make([]time.Duration, 0) } p.Deadline.Duration = p.calculateFilterTimeout(source.Proto, r.t) p.Deadline.Type = timers.Filter @@ -684,7 +690,7 @@ func (p *player) handleMessageEvent(r routerHandle, e messageEvent) (actions []a cert := Certificate(freshestRes.Event.Bundle) a0 := ensureAction{Payload: e.Input.Proposal, Certificate: cert} actions = append(actions, a0) - p.updateDynamicLambdaTimings(r, e.Input.Proposal, e.Proto.Version) + p.updateDynamicLambdaTimings(r, e.Proto.Version) as := p.enterRound(r, delegatedE, cert.Round+1) return append(actions, as...) } diff --git a/agreement/player_test.go b/agreement/player_test.go index 8506d8c5ba..d16400118c 100644 --- a/agreement/player_test.go +++ b/agreement/player_test.go @@ -3266,8 +3266,8 @@ func TestPlayerRetainsReceivedValidatedAt(t *testing.T) { // assert lowest vote validateAt time was recorded into payloadArrivals historyLen := config.Consensus[protocol.ConsensusFuture].DynamicFilterPayloadArriavalHistory require.NotZero(t, historyLen) - require.Len(t, pWhite.payloadArrivals, historyLen) - require.Equal(t, 501*time.Millisecond, pWhite.payloadArrivals[historyLen-1]) + require.Len(t, pWhite.lowestCredentialArrivals, historyLen) + require.Equal(t, 501*time.Millisecond, pWhite.lowestCredentialArrivals[historyLen-1]) } // test that ReceivedAt and ValidateAt timing information are retained in proposalStore @@ -3310,8 +3310,8 @@ func TestPlayerRetainsReceivedValidatedAtPP(t *testing.T) { // assert lowest vote validateAt time was recorded into payloadArrivals historyLen := config.Consensus[protocol.ConsensusFuture].DynamicFilterPayloadArriavalHistory require.NotZero(t, historyLen) - require.Len(t, pWhite.payloadArrivals, historyLen) - require.Equal(t, 502*time.Millisecond, pWhite.payloadArrivals[historyLen-1]) + require.Len(t, pWhite.lowestCredentialArrivals, historyLen) + require.Equal(t, 502*time.Millisecond, pWhite.lowestCredentialArrivals[historyLen-1]) } // test that ReceivedAt and ValidateAt timing information are retained in proposalStore @@ -3365,8 +3365,8 @@ func TestPlayerRetainsReceivedValidatedAtAVPP(t *testing.T) { // assert lowest vote validateAt time was recorded into payloadArrivals historyLen := config.Consensus[protocol.ConsensusFuture].DynamicFilterPayloadArriavalHistory require.NotZero(t, historyLen) - require.Len(t, pWhite.payloadArrivals, historyLen) - require.Equal(t, 502*time.Millisecond, pWhite.payloadArrivals[historyLen-1]) + require.Len(t, pWhite.lowestCredentialArrivals, historyLen) + require.Equal(t, 502*time.Millisecond, pWhite.lowestCredentialArrivals[historyLen-1]) } func assertCorrectReceivedAtSet(t *testing.T, pWhite *player, pM ioAutomata, helper *voteMakerHelper, From 6a4ea850eb7f97d29eed908734ce629af2cabeb4 Mon Sep 17 00:00:00 2001 From: chris erway Date: Wed, 16 Aug 2023 15:19:16 -0400 Subject: [PATCH 048/232] fix fmt string --- agreement/events.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agreement/events.go b/agreement/events.go index 7f79efc198..704eda3580 100644 --- a/agreement/events.go +++ b/agreement/events.go @@ -432,7 +432,7 @@ func (e readLowestEvent) t() eventType { } func (e readLowestEvent) String() string { - return fmt.Sprintf("%s: %d\t%.10s\t%.5s", e.t().String(), e.Round, e.Period) + return fmt.Sprintf("%s: %d %d", e.t().String(), e.Round, e.Period) } func (e readLowestEvent) ComparableStr() string { From e17850317c5af1bf9f7630cce761043acf7981b0 Mon Sep 17 00:00:00 2001 From: chris erway Date: Wed, 16 Aug 2023 15:21:45 -0400 Subject: [PATCH 049/232] fix TestPlayerRetainsReceivedValidatedAt tests --- agreement/events.go | 4 ++-- agreement/player_test.go | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/agreement/events.go b/agreement/events.go index 704eda3580..29341f3776 100644 --- a/agreement/events.go +++ b/agreement/events.go @@ -412,11 +412,11 @@ func (e newRoundEvent) ComparableStr() string { } type readLowestEvent struct { - // T is either readLowestValue or readLowestPayload + // T currently only supports readLowestVote T eventType // Round and Period are the round and period for which to query - // the lowest-credential value and payload. This type of event + // the lowest-credential vote, value or payload. This type of event // is only sent for pipelining, which only makes sense for period // 0, but the Period is here anyway to route to the appropriate // proposalMachinePeriod. diff --git a/agreement/player_test.go b/agreement/player_test.go index d16400118c..5662064a96 100644 --- a/agreement/player_test.go +++ b/agreement/player_test.go @@ -3241,7 +3241,7 @@ func TestPlayerRetainsReceivedValidatedAt(t *testing.T) { partitiontest.PartitionTest(t) const r = round(20239) - const p = period(1001) + const p = period(0) pWhite, pM, helper := setupP(t, r-1, p, soft) pP, pV := helper.MakeRandomProposalPayload(t, r-1) @@ -3278,7 +3278,7 @@ func TestPlayerRetainsReceivedValidatedAtPP(t *testing.T) { partitiontest.PartitionTest(t) const r = round(20239) - const p = period(1001) + const p = period(0) pWhite, pM, helper := setupP(t, r-1, p, soft) pP, pV := helper.MakeRandomProposalPayload(t, r-1) @@ -3323,7 +3323,7 @@ func TestPlayerRetainsReceivedValidatedAtAVPP(t *testing.T) { partitiontest.PartitionTest(t) const r = round(20239) - const p = period(1001) + const p = period(0) pWhite, pM, helper := setupP(t, r-1, p, soft) pP, pV := helper.MakeRandomProposalPayload(t, r-1) From bb94590d550196628290e5fbefd9b82c01980f48 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Wed, 16 Aug 2023 15:31:15 -0400 Subject: [PATCH 050/232] next receives Deadline-type timeouts --- agreement/demux.go | 5 ++--- agreement/demux_test.go | 2 +- agreement/service.go | 6 +++--- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/agreement/demux.go b/agreement/demux.go index 060ef922b0..c225396293 100644 --- a/agreement/demux.go +++ b/agreement/demux.go @@ -19,7 +19,6 @@ package agreement import ( "context" "fmt" - "time" "github.com/algorand/go-algorand/config" "github.com/algorand/go-algorand/logging" @@ -190,7 +189,7 @@ func (d *demux) verifyBundle(ctx context.Context, m message, r round, p period, // next blocks until it observes an external input event of interest for the state machine. // // If ok is false, there are no more events so the agreement service should quit. -func (d *demux) next(s *Service, deadline Deadline, fastDeadline time.Duration, currentRound round) (e externalEvent, ok bool) { +func (d *demux) next(s *Service, deadline Deadline, fastDeadline Deadline, currentRound round) (e externalEvent, ok bool) { defer func() { if !ok { return @@ -251,7 +250,7 @@ func (d *demux) next(s *Service, deadline Deadline, fastDeadline time.Duration, ledgerNextRoundCh := s.Ledger.Wait(nextRound) deadlineCh := s.Clock.TimeoutAt(deadline.Duration, deadline.Type) - fastDeadlineCh := s.Clock.TimeoutAt(fastDeadline, TimeoutFastRecovery) + fastDeadlineCh := s.Clock.TimeoutAt(fastDeadline.Duration, fastDeadline.Type) d.UpdateEventsQueue(eventQueueDemux, 0) d.monitor.dec(demuxCoserviceType) diff --git a/agreement/demux_test.go b/agreement/demux_test.go index 68e13dd950..c6aee6619c 100644 --- a/agreement/demux_test.go +++ b/agreement/demux_test.go @@ -675,7 +675,7 @@ func (t *demuxTester) TestUsecase(testcase demuxTestUsecase) bool { close(s.quit) } - e, ok := dmx.next(s, Deadline{Duration: time.Second, Type: TimeoutDeadline}, fastTimeoutChTime, 300) + e, ok := dmx.next(s, Deadline{Duration: time.Second, Type: TimeoutDeadline}, Deadline{Duration: fastTimeoutChTime, Type: TimeoutFastRecovery}, 300) if !assert.Equal(t, testcase.ok, ok) { return false diff --git a/agreement/service.go b/agreement/service.go index cea1db99a2..0911bdd6ad 100644 --- a/agreement/service.go +++ b/agreement/service.go @@ -19,7 +19,6 @@ package agreement //go:generate dbgen -i agree.sql -p agreement -n agree -o agreeInstall.go -h ../scripts/LICENSE_HEADER import ( "context" - "time" "github.com/algorand/go-algorand/config" "github.com/algorand/go-algorand/logging" @@ -81,7 +80,7 @@ type parameters Parameters // externalDemuxSignals used to syncronize the external signals that goes to the demux with the main loop. type externalDemuxSignals struct { Deadline Deadline - FastRecoveryDeadline time.Duration + FastRecoveryDeadline Deadline CurrentRound round } @@ -242,7 +241,8 @@ func (s *Service) mainLoop(input <-chan externalEvent, output chan<- []action, r for { output <- a - ready <- externalDemuxSignals{Deadline: status.Deadline, FastRecoveryDeadline: status.FastRecoveryDeadline, CurrentRound: status.Round} + fastRecoveryDeadline := Deadline{Duration: status.FastRecoveryDeadline, Type: TimeoutFastRecovery} + ready <- externalDemuxSignals{Deadline: status.Deadline, FastRecoveryDeadline: fastRecoveryDeadline, CurrentRound: status.Round} e, ok := <-input if !ok { break From 3580617aa90130589af2ddde630e6089cde33ee5 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Thu, 17 Aug 2023 13:39:29 -0400 Subject: [PATCH 051/232] address some of the comments --- agreement/player.go | 32 ++++++++++++-------------------- agreement/player_test.go | 18 +++++++++--------- agreement/service_test.go | 2 +- config/consensus.go | 21 ++++++++++++--------- tools/x-repo-types/xrt_test.go | 14 ++++++-------- 5 files changed, 40 insertions(+), 47 deletions(-) diff --git a/agreement/player.go b/agreement/player.go index 9512f22bf1..7b292fda31 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -295,45 +295,35 @@ func (p *player) updateDynamicLambdaTimings(r routerHandle, ver protocol.Consens if re.Vote.validatedAt != 0 { p.lowestCredentialArrivals = append(p.lowestCredentialArrivals, re.Vote.validatedAt) } - p.resizeLowestCredentialArrivals(ver) -} -func (p *player) resizeLowestCredentialArrivals(ver protocol.ConsensusVersion) { + // trim history to the last proto.DynamicFilterCredentialArrivalHistory samples. proto := config.Consensus[ver] if len(p.lowestCredentialArrivals) > proto.DynamicFilterCredentialArrivalHistory { p.lowestCredentialArrivals = p.lowestCredentialArrivals[len(p.lowestCredentialArrivals)-proto.DynamicFilterCredentialArrivalHistory:] } - for len(p.lowestCredentialArrivals) < proto.DynamicFilterCredentialArrivalHistory { - p.lowestCredentialArrivals = append([]time.Duration{FilterTimeout(0, ver)}, p.lowestCredentialArrivals...) - } } // calculateFilterTimeout chooses the appropriate filter timeout. func (p *player) calculateFilterTimeout(ver protocol.ConsensusVersion, tracer *tracer) time.Duration { proto := config.Consensus[ver] - - if !proto.DynamicFilterTimeout || p.Period != 0 { + if proto.DynamicFilterCredentialArrivalHistory <= 0 || p.Period != 0 { // Either dynamic lambda is disabled, or we're not in period 0 and // therefore, can't use dynamic lambda return FilterTimeout(p.Period, ver) } - var dynamicDelay time.Duration defaultDelay := FilterTimeout(0, ver) - if proto.DynamicFilterCredentialArrivalHistory <= 0 { - // we don't keep any history, use the default - dynamicDelay = defaultDelay - } else if proto.DynamicFilterCredentialArrivalHistory > len(p.lowestCredentialArrivals) { + if proto.DynamicFilterCredentialArrivalHistory > len(p.lowestCredentialArrivals) { // not enough samples, use the default - dynamicDelay = defaultDelay - } else { - sortedArrivals := make([]time.Duration, len(p.lowestCredentialArrivals)) - copy(sortedArrivals[:], p.lowestCredentialArrivals[:]) - sort.Slice(sortedArrivals, func(i, j int) bool { return sortedArrivals[i] < sortedArrivals[j] }) - dynamicDelay = sortedArrivals[proto.DynamicFilterCredentialArrivalHistoryIdx] + 50*time.Millisecond + return defaultDelay } + sortedArrivals := make([]time.Duration, len(p.lowestCredentialArrivals)) + copy(sortedArrivals[:], p.lowestCredentialArrivals[:]) + sort.Slice(sortedArrivals, func(i, j int) bool { return sortedArrivals[i] < sortedArrivals[j] }) + dynamicDelay := sortedArrivals[proto.DynamicFilterCredentialArrivalHistoryIdx] + proto.DynamicFilterGraceTime + // Make sure the dynamic delay is not too small or too large if dynamicDelay < proto.DynamicFilterTimeoutLowerBound { if tracer != nil { @@ -341,8 +331,10 @@ func (p *player) calculateFilterTimeout(ver protocol.ConsensusVersion, tracer *t } dynamicDelay = proto.DynamicFilterTimeoutLowerBound } else if dynamicDelay > defaultDelay { + if tracer != nil { + tracer.log.Debugf("Calculated dynamic delay = %v for round %v, period %v. It's higher than the default %v\n", dynamicDelay, p.Round, p.Period, defaultDelay) + } dynamicDelay = defaultDelay - tracer.log.Warnf("Calculated dynamic delay = %v for round %v, period %v. It's higher than the default %v\n", dynamicDelay, p.Round, p.Period, defaultDelay) } else if tracer != nil { tracer.log.Debugf("Calculated dynamic delay = %v for round %v, period %v.\n", dynamicDelay, p.Round, p.Period) } diff --git a/agreement/player_test.go b/agreement/player_test.go index b9f950a555..4d1a971272 100644 --- a/agreement/player_test.go +++ b/agreement/player_test.go @@ -3236,7 +3236,7 @@ func TestPlayerAlwaysResynchsPinnedValue(t *testing.T) { // test that ReceivedAt and ValidateAt timing information are retained in proposalStore // when the payloadPresent, payloadVerified, and voteVerified events are processed, and that all timings // are available when the ensureAction is called for the block. -func TestPlayerRetainsReceivedValidatedAt(t *testing.T) { +func TestPlayerRetainsReceivedValidatedAtOneSample(t *testing.T) { partitiontest.PartitionTest(t) const r = round(20239) @@ -3265,15 +3265,15 @@ func TestPlayerRetainsReceivedValidatedAt(t *testing.T) { // assert lowest vote validateAt time was recorded into payloadArrivals historyLen := config.Consensus[protocol.ConsensusFuture].DynamicFilterCredentialArrivalHistory require.NotZero(t, historyLen) - require.Len(t, pWhite.lowestCredentialArrivals, historyLen) - require.Equal(t, 501*time.Millisecond, pWhite.lowestCredentialArrivals[historyLen-1]) + require.Len(t, pWhite.lowestCredentialArrivals, 1) + require.Equal(t, 501*time.Millisecond, pWhite.lowestCredentialArrivals[0]) } // test that ReceivedAt and ValidateAt timing information are retained in proposalStore // when the payloadPresent (as part of the CompoundMessage encoding used by PP messages), // payloadVerified, and voteVerified events are processed, and that all timings // are available when the ensureAction is called for the block. -func TestPlayerRetainsReceivedValidatedAtPP(t *testing.T) { +func TestPlayerRetainsReceivedValidatedAtPPOneSample(t *testing.T) { partitiontest.PartitionTest(t) const r = round(20239) @@ -3309,8 +3309,8 @@ func TestPlayerRetainsReceivedValidatedAtPP(t *testing.T) { // assert lowest vote validateAt time was recorded into payloadArrivals historyLen := config.Consensus[protocol.ConsensusFuture].DynamicFilterCredentialArrivalHistory require.NotZero(t, historyLen) - require.Len(t, pWhite.lowestCredentialArrivals, historyLen) - require.Equal(t, 502*time.Millisecond, pWhite.lowestCredentialArrivals[historyLen-1]) + require.Len(t, pWhite.lowestCredentialArrivals, 1) + require.Equal(t, 502*time.Millisecond, pWhite.lowestCredentialArrivals[0]) } // test that ReceivedAt and ValidateAt timing information are retained in proposalStore @@ -3318,7 +3318,7 @@ func TestPlayerRetainsReceivedValidatedAtPP(t *testing.T) { // then the payloadPresent (as part of the CompoundMessage encoding used by PP messages) // and payloadVerified events are processed, and that all timings // are available when the ensureAction is called for the block. -func TestPlayerRetainsReceivedValidatedAtAVPP(t *testing.T) { +func TestPlayerRetainsReceivedValidatedAtAVPPOneSample(t *testing.T) { partitiontest.PartitionTest(t) const r = round(20239) @@ -3364,8 +3364,8 @@ func TestPlayerRetainsReceivedValidatedAtAVPP(t *testing.T) { // assert lowest vote validateAt time was recorded into payloadArrivals historyLen := config.Consensus[protocol.ConsensusFuture].DynamicFilterCredentialArrivalHistory require.NotZero(t, historyLen) - require.Len(t, pWhite.lowestCredentialArrivals, historyLen) - require.Equal(t, 502*time.Millisecond, pWhite.lowestCredentialArrivals[historyLen-1]) + require.Len(t, pWhite.lowestCredentialArrivals, 1) + require.Equal(t, 502*time.Millisecond, pWhite.lowestCredentialArrivals[0]) } func assertCorrectReceivedAtSet(t *testing.T, pWhite *player, pM ioAutomata, helper *voteMakerHelper, diff --git a/agreement/service_test.go b/agreement/service_test.go index f8822a6bfd..96977e812d 100644 --- a/agreement/service_test.go +++ b/agreement/service_test.go @@ -1002,7 +1002,7 @@ func TestAgreementSynchronousFuture5_DynamicFilterRounds(t *testing.T) { } cfg := config.Consensus[protocol.ConsensusFuture] - if !cfg.DynamicFilterTimeout { + if cfg.DynamicFilterCredentialArrivalHistory <= 0 { return } diff --git a/config/consensus.go b/config/consensus.go index f34ff896a6..79a2433ce6 100644 --- a/config/consensus.go +++ b/config/consensus.go @@ -517,21 +517,24 @@ type ConsensusParams struct { // by state proofs to use the same method (rather than excluding stake from the top N stakeholders as before). ExcludeExpiredCirculation bool - // DynamicFilterTimeout specifies whether the timeout for the filter step - // should be determined dynamically, at run-time. - DynamicFilterTimeout bool + // DynamicFilterCredentialArrivalHistory specifies the number of past + // credential arrivals that are measured to determine the next filter + // timeout. If DynamicFilterCredentialArrivalHistory <= 0, then the dynamic + // timeout feature is off and the filter step timeout is calculated using + // the static configuration. + DynamicFilterCredentialArrivalHistory int // DynamicFilterTimeoutLowerBound specifies a minimal duration that the // filter timeout must meet. DynamicFilterTimeoutLowerBound time.Duration - // DynamicFilterCredentialArrivalHistory specifies the number of past credential arrivals - // that are measured to determine the next filter timeout. - DynamicFilterCredentialArrivalHistory int - // DynamicFilterCredentialArrivalHistoryIdx specified which sample to use out of a sorted // DynamicFilterCredentialArrivalHistory-sized array of time samples. DynamicFilterCredentialArrivalHistoryIdx int + + // DynamicFilterGraceTime is additional extention to the dynamic filter time atop the + // one calculated based on the history of credential arrivals. + DynamicFilterGraceTime time.Duration } // PaysetCommitType enumerates possible ways for the block header to commit to @@ -1379,12 +1382,12 @@ func initConsensusProtocols() { vFuture.LogicSigVersion = 10 // When moving this to a release, put a new higher LogicSigVersion here vFuture.EnableLogicSigCostPooling = true - vFuture.DynamicFilterTimeout = true - vFuture.DynamicFilterTimeoutLowerBound = time.Second // history window of 40, so we have enough statistics to calculate the 95th // percentile, which is the timestamp at index 37 in the history array. vFuture.DynamicFilterCredentialArrivalHistory = 40 vFuture.DynamicFilterCredentialArrivalHistoryIdx = 37 + vFuture.DynamicFilterTimeoutLowerBound = time.Second + vFuture.DynamicFilterGraceTime = 50 * time.Millisecond Consensus[protocol.ConsensusFuture] = vFuture diff --git a/tools/x-repo-types/xrt_test.go b/tools/x-repo-types/xrt_test.go index 6fe353cedb..4360b432d6 100644 --- a/tools/x-repo-types/xrt_test.go +++ b/tools/x-repo-types/xrt_test.go @@ -74,14 +74,12 @@ func TestCrossRepoTypes(t *testing.T) { yType: "EvalDelta", }, { - name: "goal-v-sdk-consensus", - xPkg: "github.com/algorand/go-algorand/config", - xType: "ConsensusParams", - yPkg: "github.com/algorand/go-algorand-sdk/v2/protocol/config", - yBranch: "develop", - yType: "ConsensusParams", - skip: true, - skipReason: `Changes to consensus parameters happen on the go-algorand repo, and cannot be synced in a single PR with changes to the go-algorand-sdk repo`, + name: "goal-v-sdk-consensus", + xPkg: "github.com/algorand/go-algorand/config", + xType: "ConsensusParams", + yPkg: "github.com/algorand/go-algorand-sdk/v2/protocol/config", + yBranch: "develop", + yType: "ConsensusParams", }, { name: "goal-v-sdk-blockheader", From 53838ab47d3a77b40ad8471714ee3b2ab7b20714 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Thu, 17 Aug 2023 14:44:46 -0400 Subject: [PATCH 052/232] history window tests --- agreement/player_test.go | 162 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 161 insertions(+), 1 deletion(-) diff --git a/agreement/player_test.go b/agreement/player_test.go index 4d1a971272..40ae4a68e9 100644 --- a/agreement/player_test.go +++ b/agreement/player_test.go @@ -3269,6 +3269,50 @@ func TestPlayerRetainsReceivedValidatedAtOneSample(t *testing.T) { require.Equal(t, 501*time.Millisecond, pWhite.lowestCredentialArrivals[0]) } +// test that ReceivedAt and ValidateAt timing information are retained in proposalStore +// when the payloadPresent, payloadVerified, and voteVerified events are processed, and that all timings +// are available when the ensureAction is called for the block. The history should be kept for the last +// DynamicFilterCredentialArrivalHistory rounds. +func TestPlayerRetainsReceivedValidatedAtForHistoryWindow(t *testing.T) { + partitiontest.PartitionTest(t) + + const r = round(20239) + const p = period(0) + pWhite, pM, helper := setupP(t, r-1, p, soft) + + historyLen := config.Consensus[protocol.ConsensusFuture].DynamicFilterCredentialArrivalHistory + require.NotZero(t, historyLen) + + for i := 0; i < historyLen+1; i++ { + // send voteVerified message + pP, pV := helper.MakeRandomProposalPayload(t, r+round(i)-1) + vVote := helper.MakeVerifiedVote(t, 0, r+round(i)-1, p, propose, *pV) + inMsg := messageEvent{T: voteVerified, Input: message{Vote: vVote, UnauthenticatedVote: vVote.u()}} + timestamp := 500 + i + inMsg = inMsg.AttachValidatedAt(time.Duration(timestamp) * time.Millisecond) + err, panicErr := pM.transition(inMsg) + require.NoError(t, err) + require.NoError(t, panicErr) + + // send payloadPresent message + m := message{UnauthenticatedProposal: pP.u()} + inMsg = messageEvent{T: payloadPresent, Input: m} + inMsg = inMsg.AttachReceivedAt(time.Second) + err, panicErr = pM.transition(inMsg) + require.NoError(t, err) + require.NoError(t, panicErr) + moveToRound(t, pWhite, pM, helper, r+round(i), p, pP, pV, m, protocol.ConsensusFuture) + } + + // assert lowest vote validateAt time was recorded into payloadArrivals + require.Len(t, pWhite.lowestCredentialArrivals, historyLen) + for i := 0; i < historyLen; i++ { + // only the last historyLen samples are kept, so the first one is discarded + timestamp := 500 + i + 1 + require.Equal(t, time.Duration(timestamp)*time.Millisecond, pWhite.lowestCredentialArrivals[i]) + } +} + // test that ReceivedAt and ValidateAt timing information are retained in proposalStore // when the payloadPresent (as part of the CompoundMessage encoding used by PP messages), // payloadVerified, and voteVerified events are processed, and that all timings @@ -3313,6 +3357,60 @@ func TestPlayerRetainsReceivedValidatedAtPPOneSample(t *testing.T) { require.Equal(t, 502*time.Millisecond, pWhite.lowestCredentialArrivals[0]) } +// test that ReceivedAt and ValidateAt timing information are retained in +// proposalStore when the payloadPresent (as part of the CompoundMessage +// encoding used by PP messages), payloadVerified, and voteVerified events are +// processed, and that all timings are available when the ensureAction is called +// for the block. The history should be kept for the last +// DynamicFilterCredentialArrivalHistory rounds. +func TestPlayerRetainsReceivedValidatedAtPPForHistoryWindow(t *testing.T) { + partitiontest.PartitionTest(t) + + const r = round(20239) + const p = period(0) + pWhite, pM, helper := setupP(t, r-1, p, soft) + pP, pV := helper.MakeRandomProposalPayload(t, r-1) + + historyLen := config.Consensus[protocol.ConsensusFuture].DynamicFilterCredentialArrivalHistory + require.NotZero(t, historyLen) + + for i := 0; i < historyLen+1; i++ { + // create a PP message for an arbitrary proposal/payload similar to setupCompoundMessage + vVote := helper.MakeVerifiedVote(t, 0, r+round(i)-1, p, propose, *pV) + unverifiedVoteMsg := message{UnauthenticatedVote: vVote.u()} + proposalMsg := message{UnauthenticatedProposal: pP.u()} + compoundMsg := messageEvent{T: votePresent, Input: unverifiedVoteMsg, + Tail: &messageEvent{T: payloadPresent, Input: proposalMsg}} + + inMsg := compoundMsg.AttachReceivedAt(time.Second) // call AttachReceivedAt like demux would + err, panicErr := pM.transition(inMsg) + require.NoError(t, err) + require.NoError(t, panicErr) + + // make sure vote verify requests + verifyEvent := ev(cryptoAction{T: verifyVote, M: unverifiedVoteMsg, Round: r + round(i) - 1, Period: p, Step: propose, TaskIndex: 1}) + require.Truef(t, pM.getTrace().Contains(verifyEvent), "Player should verify vote") + + // send voteVerified + verifiedVoteMsg := message{Vote: vVote, UnauthenticatedVote: vVote.u()} + inMsg = messageEvent{T: voteVerified, Input: verifiedVoteMsg, TaskIndex: 1} + timestamp := 500 + i + inMsg = inMsg.AttachValidatedAt(time.Duration(timestamp) * time.Millisecond) + err, panicErr = pM.transition(inMsg) + require.NoError(t, err) + require.NoError(t, panicErr) + moveToRound(t, pWhite, pM, helper, r+round(i), p, pP, pV, proposalMsg, protocol.ConsensusFuture) + } + + // assert lowest vote validateAt time was recorded into payloadArrivals + require.Len(t, pWhite.lowestCredentialArrivals, historyLen) + for i := 0; i < historyLen; i++ { + // only the last historyLen samples are kept, so the first one is discarded + timestamp := 500 + i + 1 + require.Equal(t, time.Duration(timestamp)*time.Millisecond, pWhite.lowestCredentialArrivals[i]) + } +} + // test that ReceivedAt and ValidateAt timing information are retained in proposalStore // when the voteVerified event comes in first (as part of the AV message before PP), // then the payloadPresent (as part of the CompoundMessage encoding used by PP messages) @@ -3368,8 +3466,64 @@ func TestPlayerRetainsReceivedValidatedAtAVPPOneSample(t *testing.T) { require.Equal(t, 502*time.Millisecond, pWhite.lowestCredentialArrivals[0]) } -func assertCorrectReceivedAtSet(t *testing.T, pWhite *player, pM ioAutomata, helper *voteMakerHelper, +func TestPlayerRetainsReceivedValidatedAtAVPPHistoryWindow(t *testing.T) { + partitiontest.PartitionTest(t) + + const r = round(20239) + const p = period(0) + pWhite, pM, helper := setupP(t, r-1, p, soft) + + historyLen := config.Consensus[protocol.ConsensusFuture].DynamicFilterCredentialArrivalHistory + require.NotZero(t, historyLen) + + for i := 0; i < historyLen+1; i++ { + pP, pV := helper.MakeRandomProposalPayload(t, r+round(i)-1) + + // send votePresent message (mimicking the first AV message validating) + vVote := helper.MakeVerifiedVote(t, 0, r+round(i)-1, p, propose, *pV) + unverifiedVoteMsg := message{UnauthenticatedVote: vVote.u()} + inMsg := messageEvent{T: votePresent, Input: unverifiedVoteMsg} + err, panicErr := pM.transition(inMsg) + require.NoError(t, err) + require.NoError(t, panicErr) + + // make sure vote verify requests + verifyEvent := ev(cryptoAction{T: verifyVote, M: unverifiedVoteMsg, Round: r + round(i) - 1, Period: p, Step: propose, TaskIndex: 1}) + require.Truef(t, pM.getTrace().Contains(verifyEvent), "Player should verify vote") + + // send voteVerified + verifiedVoteMsg := message{Vote: vVote, UnauthenticatedVote: vVote.u()} + inMsg = messageEvent{T: voteVerified, Input: verifiedVoteMsg, TaskIndex: 1} + timestamp := 500 + i + inMsg = inMsg.AttachValidatedAt(time.Duration(timestamp) * time.Millisecond) + err, panicErr = pM.transition(inMsg) + require.NoError(t, err) + require.NoError(t, panicErr) + + // create a PP message for an arbitrary proposal/payload similar to setupCompoundMessage + proposalMsg := message{UnauthenticatedProposal: pP.u()} + compoundMsg := messageEvent{T: votePresent, Input: unverifiedVoteMsg, + Tail: &messageEvent{T: payloadPresent, Input: proposalMsg}} + inMsg = compoundMsg.AttachReceivedAt(time.Second) // call AttachReceivedAt like demux would + err, panicErr = pM.transition(inMsg) + require.NoError(t, err) + require.NoError(t, panicErr) + + moveToRound(t, pWhite, pM, helper, r+round(i), p, pP, pV, proposalMsg, protocol.ConsensusFuture) + } + + // assert lowest vote validateAt time was recorded into payloadArrivals + require.Len(t, pWhite.lowestCredentialArrivals, historyLen) + for i := 0; i < historyLen; i++ { + // only the last historyLen samples are kept, so the first one is discarded + timestamp := 500 + i + 1 + require.Equal(t, time.Duration(timestamp)*time.Millisecond, pWhite.lowestCredentialArrivals[i]) + } +} + +func moveToRound(t *testing.T, pWhite *player, pM ioAutomata, helper *voteMakerHelper, r round, p period, pP *proposal, pV *proposalValue, m message, ver protocol.ConsensusVersion) { + // make sure payload verify request verifyEvent := ev(cryptoAction{T: verifyPayload, M: m, Round: r - 1, Period: p, Step: propose, TaskIndex: 0}) require.Truef(t, pM.getTrace().Contains(verifyEvent), "Player should verify payload") @@ -3410,6 +3564,12 @@ func assertCorrectReceivedAtSet(t *testing.T, pWhite *player, pM ioAutomata, hel require.Equalf(t, period(0), pWhite.Period, "player did not enter period 0 in new round") commitEvent := ev(ensureAction{Certificate: Certificate(bun), Payload: *pP}) require.Truef(t, pM.getTrace().Contains(commitEvent), "Player should try to ensure block/digest on ledger") +} + +func assertCorrectReceivedAtSet(t *testing.T, pWhite *player, pM ioAutomata, helper *voteMakerHelper, + r round, p period, pP *proposal, pV *proposalValue, m message, ver protocol.ConsensusVersion) { + + moveToRound(t, pWhite, pM, helper, r, p, pP, pV, m, ver) // find and unwrap ensureAction from trace var ea ensureAction From ea35ee3a78cba9113eff2a3b5982ff3b7196c297 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Thu, 17 Aug 2023 16:09:46 -0400 Subject: [PATCH 053/232] nits --- agreement/player.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/agreement/player.go b/agreement/player.go index 7b292fda31..b439f9a750 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -69,7 +69,7 @@ type player struct { Pending proposalTable // the history of arrival times of the lowest credential from previous - // ronuds, used for calculating dynamic lambda. + // ronuds, used for calculating the filter timeout dynamically. lowestCredentialArrivals []time.Duration } @@ -414,7 +414,7 @@ func (p *player) enterPeriod(r routerHandle, source thresholdEvent, target perio if target != 0 { // We entered a non-0 period, we should reset the filter timeout // calculation mechanism. - p.lowestCredentialArrivals = make([]time.Duration, 0) + p.lowestCredentialArrivals = nil } p.Deadline.Duration = p.calculateFilterTimeout(source.Proto, r.t) p.Deadline.Type = TimeoutFilter From 0bcdf1968c12f79aa5a5aabb5ba7e76af4547303 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Thu, 17 Aug 2023 16:15:53 -0400 Subject: [PATCH 054/232] typo --- config/consensus.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/consensus.go b/config/consensus.go index 79a2433ce6..ef30c85d2d 100644 --- a/config/consensus.go +++ b/config/consensus.go @@ -532,7 +532,7 @@ type ConsensusParams struct { // DynamicFilterCredentialArrivalHistory-sized array of time samples. DynamicFilterCredentialArrivalHistoryIdx int - // DynamicFilterGraceTime is additional extention to the dynamic filter time atop the + // DynamicFilterGraceTime is additional extension to the dynamic filter time atop the // one calculated based on the history of credential arrivals. DynamicFilterGraceTime time.Duration } From 4cfb6aaa9eaaccfee15c99818780371ca86df206 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Thu, 17 Aug 2023 16:29:27 -0400 Subject: [PATCH 055/232] off switch in config --- agreement/player.go | 29 ++++++++++++++++++----------- config/consensus.go | 27 ++++++++++++++++++--------- 2 files changed, 36 insertions(+), 20 deletions(-) diff --git a/agreement/player.go b/agreement/player.go index b439f9a750..64ce7685a4 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -313,33 +313,40 @@ func (p *player) calculateFilterTimeout(ver protocol.ConsensusVersion, tracer *t return FilterTimeout(p.Period, ver) } - defaultDelay := FilterTimeout(0, ver) + defaultTimeout := FilterTimeout(0, ver) if proto.DynamicFilterCredentialArrivalHistory > len(p.lowestCredentialArrivals) { // not enough samples, use the default - return defaultDelay + return defaultTimeout } sortedArrivals := make([]time.Duration, len(p.lowestCredentialArrivals)) copy(sortedArrivals[:], p.lowestCredentialArrivals[:]) sort.Slice(sortedArrivals, func(i, j int) bool { return sortedArrivals[i] < sortedArrivals[j] }) - dynamicDelay := sortedArrivals[proto.DynamicFilterCredentialArrivalHistoryIdx] + proto.DynamicFilterGraceTime + dynamicTimeout := sortedArrivals[proto.DynamicFilterTimeoutCredentialArrivalHistoryIdx] + proto.DynamicFilterTimeoutGraceInterval // Make sure the dynamic delay is not too small or too large - if dynamicDelay < proto.DynamicFilterTimeoutLowerBound { + if dynamicTimeout < proto.DynamicFilterTimeoutLowerBound { if tracer != nil { - tracer.log.Debugf("Calculated dynamic delay = %v for round %v, period %v. It's too low, setting to %v\n", dynamicDelay, p.Round, p.Period, proto.DynamicFilterTimeoutLowerBound) + tracer.log.Debugf("Calculated dynamic delay = %v for round %v, period %v. It's too low, setting to %v\n", dynamicTimeout, p.Round, p.Period, proto.DynamicFilterTimeoutLowerBound) } - dynamicDelay = proto.DynamicFilterTimeoutLowerBound - } else if dynamicDelay > defaultDelay { + dynamicTimeout = proto.DynamicFilterTimeoutLowerBound + } else if dynamicTimeout > defaultTimeout { if tracer != nil { - tracer.log.Debugf("Calculated dynamic delay = %v for round %v, period %v. It's higher than the default %v\n", dynamicDelay, p.Round, p.Period, defaultDelay) + tracer.log.Debugf("Calculated dynamic delay = %v for round %v, period %v. It too high, so using the default %v\n", dynamicTimeout, p.Round, p.Period, defaultTimeout) } - dynamicDelay = defaultDelay + dynamicTimeout = defaultTimeout } else if tracer != nil { - tracer.log.Debugf("Calculated dynamic delay = %v for round %v, period %v.\n", dynamicDelay, p.Round, p.Period) + tracer.log.Debugf("Calculated dynamic delay = %v for round %v, period %v.\n", dynamicTimeout, p.Round, p.Period) } - return dynamicDelay + if !proto.DynamicFilterTimeout { + // If the dynamic filter is disabled, return the default filter timeout + // (after logging what the timeout would have been, if this feature were + // enabled). + return defaultTimeout + } + + return dynamicTimeout } func (p *player) handleThresholdEvent(r routerHandle, e thresholdEvent) []action { diff --git a/config/consensus.go b/config/consensus.go index ef30c85d2d..6d9dfeff51 100644 --- a/config/consensus.go +++ b/config/consensus.go @@ -528,13 +528,21 @@ type ConsensusParams struct { // filter timeout must meet. DynamicFilterTimeoutLowerBound time.Duration - // DynamicFilterCredentialArrivalHistoryIdx specified which sample to use out of a sorted - // DynamicFilterCredentialArrivalHistory-sized array of time samples. - DynamicFilterCredentialArrivalHistoryIdx int - - // DynamicFilterGraceTime is additional extension to the dynamic filter time atop the - // one calculated based on the history of credential arrivals. - DynamicFilterGraceTime time.Duration + // DynamicFilterTimeoutCredentialArrivalHistoryIdx specified which sample to + // use out of a sorted DynamicFilterCredentialArrivalHistory-sized array of + // time samples. + DynamicFilterTimeoutCredentialArrivalHistoryIdx int + + // DynamicFilterTimeoutGraceInterval is additional extension to the dynamic + // filter time atop the one calculated based on the history of credential + // arrivals. + DynamicFilterTimeoutGraceInterval time.Duration + + // DynamicFilterTimeout indicates whether the filter timeout is set + // dynamically, at run time, according to the recent history of credential + // arrival times or is set to a static value. Even if this flag disables the + // dynamic filter, it will be calculated and logged (but not used). + DynamicFilterTimeout bool } // PaysetCommitType enumerates possible ways for the block header to commit to @@ -1385,9 +1393,10 @@ func initConsensusProtocols() { // history window of 40, so we have enough statistics to calculate the 95th // percentile, which is the timestamp at index 37 in the history array. vFuture.DynamicFilterCredentialArrivalHistory = 40 - vFuture.DynamicFilterCredentialArrivalHistoryIdx = 37 + vFuture.DynamicFilterTimeoutCredentialArrivalHistoryIdx = 37 vFuture.DynamicFilterTimeoutLowerBound = time.Second - vFuture.DynamicFilterGraceTime = 50 * time.Millisecond + vFuture.DynamicFilterTimeoutGraceInterval = 50 * time.Millisecond + vFuture.DynamicFilterTimeout = true Consensus[protocol.ConsensusFuture] = vFuture From d459e78bf6208916d6766c84428ae8f4ef992b1c Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Thu, 17 Aug 2023 17:13:21 -0400 Subject: [PATCH 056/232] checkpoint: positive test --- agreement/player.go | 5 ++-- agreement/service_test.go | 59 +++++++++++++++++++++++++++++++++++---- 2 files changed, 56 insertions(+), 8 deletions(-) diff --git a/agreement/player.go b/agreement/player.go index 64ce7685a4..19b8345c28 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -17,6 +17,7 @@ package agreement import ( + "fmt" "sort" "time" @@ -292,7 +293,9 @@ func (p *player) updateDynamicLambdaTimings(r routerHandle, ver protocol.Consens // look up the validatedAt time of the winning proposal-vote re := readLowestEvent{T: readLowestVote, Round: p.Round, Period: p.Period} re = r.dispatch(*p, re, proposalMachineRound, p.Round, p.Period, 0).(readLowestEvent) + if re.Vote.validatedAt != 0 { + fmt.Println("update dynamic lambda") p.lowestCredentialArrivals = append(p.lowestCredentialArrivals, re.Vote.validatedAt) } @@ -305,14 +308,12 @@ func (p *player) updateDynamicLambdaTimings(r routerHandle, ver protocol.Consens // calculateFilterTimeout chooses the appropriate filter timeout. func (p *player) calculateFilterTimeout(ver protocol.ConsensusVersion, tracer *tracer) time.Duration { - proto := config.Consensus[ver] if proto.DynamicFilterCredentialArrivalHistory <= 0 || p.Period != 0 { // Either dynamic lambda is disabled, or we're not in period 0 and // therefore, can't use dynamic lambda return FilterTimeout(p.Period, ver) } - defaultTimeout := FilterTimeout(0, ver) if proto.DynamicFilterCredentialArrivalHistory > len(p.lowestCredentialArrivals) { // not enough samples, use the default diff --git a/agreement/service_test.go b/agreement/service_test.go index 96977e812d..33d28db651 100644 --- a/agreement/service_test.go +++ b/agreement/service_test.go @@ -93,6 +93,17 @@ func (c *testingClock) TimeoutAt(d time.Duration, timeoutType TimeoutType) <-cha return ta.ch } +func (c *testingClock) when(timeoutType TimeoutType) (time.Duration, error) { + c.mu.Lock() + defer c.mu.Unlock() + + ta, ok := c.TA[timeoutType] + if !ok { + return time.Duration(0), fmt.Errorf("no timeout of type, %v", timeoutType) + } + return ta.delta, nil +} + func (c *testingClock) Encode() []byte { return nil } @@ -884,18 +895,18 @@ func sanityCheck(startRound round, numRounds round, ledgers []Ledger) { } } -func simulateAgreement(t *testing.T, numNodes int, numRounds int, traceLevel traceLevel) { - simulateAgreementWithLedgerFactory(t, numNodes, numRounds, traceLevel, makeTestLedger) +func simulateAgreement(t *testing.T, numNodes int, numRounds int, traceLevel traceLevel) (filterTimeouts []time.Duration) { + return simulateAgreementWithLedgerFactory(t, numNodes, numRounds, traceLevel, makeTestLedger) } -func simulateAgreementWithConsensusVersion(t *testing.T, numNodes int, numRounds int, traceLevel traceLevel, consensusVersion func(basics.Round) (protocol.ConsensusVersion, error)) { +func simulateAgreementWithConsensusVersion(t *testing.T, numNodes int, numRounds int, traceLevel traceLevel, consensusVersion func(basics.Round) (protocol.ConsensusVersion, error)) (filterTimeouts []time.Duration) { ledgerFactory := func(data map[basics.Address]basics.AccountData) Ledger { return makeTestLedgerWithConsensusVersion(data, consensusVersion) } - simulateAgreementWithLedgerFactory(t, numNodes, numRounds, traceLevel, ledgerFactory) + return simulateAgreementWithLedgerFactory(t, numNodes, numRounds, traceLevel, ledgerFactory) } -func simulateAgreementWithLedgerFactory(t *testing.T, numNodes int, numRounds int, traceLevel traceLevel, ledgerFactory func(map[basics.Address]basics.AccountData) Ledger) { +func simulateAgreementWithLedgerFactory(t *testing.T, numNodes int, numRounds int, traceLevel traceLevel, ledgerFactory func(map[basics.Address]basics.AccountData) Ledger) []time.Duration { _, baseLedger, cleanupFn, services, clocks, ledgers, activityMonitor := setupAgreement(t, numNodes, traceLevel, ledgerFactory) startRound := baseLedger.NextRound() defer cleanupFn() @@ -907,9 +918,16 @@ func simulateAgreementWithLedgerFactory(t *testing.T, numNodes int, numRounds in activityMonitor.waitForQuiet() zeroes := expectNewPeriod(clocks, 0) + filterTimeouts := make([][]time.Duration, numNodes, numNodes) + // run round with round-specific consensus version first (since fix in #1896) zeroes = runRoundTriggerFilter(clocks, activityMonitor, zeroes) for j := 1; j < numRounds; j++ { + for srvIdx, clock := range clocks { + delta, err := clock.(*testingClock).when(TimeoutFilter) + require.NoError(t, err) + filterTimeouts[srvIdx] = append(filterTimeouts[srvIdx], delta) + } zeroes = runRoundTriggerFilter(clocks, activityMonitor, zeroes) } @@ -918,6 +936,19 @@ func simulateAgreementWithLedgerFactory(t *testing.T, numNodes int, numRounds in } sanityCheck(startRound, round(numRounds), ledgers) + + if len(clocks) == 0 { + return nil + } + + for rnd := 0; rnd < numRounds-1; rnd++ { + delta := filterTimeouts[0][rnd] + for srvIdx := range clocks { + require.Equal(t, delta, filterTimeouts[srvIdx][rnd]) + } + } + + return filterTimeouts[0] } func TestAgreementSynchronous1(t *testing.T) { @@ -1005,8 +1036,24 @@ func TestAgreementSynchronousFuture5_DynamicFilterRounds(t *testing.T) { if cfg.DynamicFilterCredentialArrivalHistory <= 0 { return } + if !cfg.DynamicFilterTimeout { + return + } - simulateAgreementWithConsensusVersion(t, 5, cfg.DynamicFilterCredentialArrivalHistory+20, disabled, consensusVersion) + rounds := cfg.DynamicFilterCredentialArrivalHistory + 20 + + filterTimeouts := simulateAgreementWithConsensusVersion(t, 5, rounds, disabled, consensusVersion) + require.Len(t, filterTimeouts, rounds-1) + for i := 1; i < cfg.DynamicFilterCredentialArrivalHistory-1; i++ { + require.Equal(t, filterTimeouts[i-1], filterTimeouts[i]) + } + + // dynamic filter timeout kicks in when history window is full + require.Less(t, filterTimeouts[cfg.DynamicFilterCredentialArrivalHistory-1], filterTimeouts[cfg.DynamicFilterCredentialArrivalHistory-2]) + + for i := cfg.DynamicFilterCredentialArrivalHistory; i < len(filterTimeouts); i++ { + require.Equal(t, filterTimeouts[i-1], filterTimeouts[i]) + } } func TestAgreementSynchronousFuture1(t *testing.T) { From bbfe2aeb0413a01464e633dfeda57c76cb9d69a7 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Fri, 18 Aug 2023 12:07:12 -0400 Subject: [PATCH 057/232] fix testingClock.Since to give non zero measurements --- agreement/player.go | 35 ++++++++++++++++------------------- agreement/service_test.go | 2 +- config/consensus.go | 2 +- 3 files changed, 18 insertions(+), 21 deletions(-) diff --git a/agreement/player.go b/agreement/player.go index 19b8345c28..6f4b4d9c51 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -17,7 +17,6 @@ package agreement import ( - "fmt" "sort" "time" @@ -283,9 +282,10 @@ func (p *player) handleCheckpointEvent(r routerHandle, e checkpointEvent) []acti }} } -// updateDynamicLambdaTimings is called at the end of a successful uninterrupted round (just after ensureAction -// is generated) to collect round timings for updating dynamic lambda. -func (p *player) updateDynamicLambdaTimings(r routerHandle, ver protocol.ConsensusVersion) { +// updateCredentialArrivalHistory is called at the end of a successful +// uninterrupted round (just after ensureAction is generated) to collect +// credential arrival times to dynamically set the filter timeout. +func (p *player) updateCredentialArrivalHistory(r routerHandle, ver protocol.ConsensusVersion) { // only append to lowestCredentialArrivals if this was a successful round completing in period 0. if p.Period != 0 { return @@ -294,10 +294,7 @@ func (p *player) updateDynamicLambdaTimings(r routerHandle, ver protocol.Consens re := readLowestEvent{T: readLowestVote, Round: p.Round, Period: p.Period} re = r.dispatch(*p, re, proposalMachineRound, p.Round, p.Period, 0).(readLowestEvent) - if re.Vote.validatedAt != 0 { - fmt.Println("update dynamic lambda") - p.lowestCredentialArrivals = append(p.lowestCredentialArrivals, re.Vote.validatedAt) - } + p.lowestCredentialArrivals = append(p.lowestCredentialArrivals, re.Vote.validatedAt) // trim history to the last proto.DynamicFilterCredentialArrivalHistory samples. proto := config.Consensus[ver] @@ -310,8 +307,8 @@ func (p *player) updateDynamicLambdaTimings(r routerHandle, ver protocol.Consens func (p *player) calculateFilterTimeout(ver protocol.ConsensusVersion, tracer *tracer) time.Duration { proto := config.Consensus[ver] if proto.DynamicFilterCredentialArrivalHistory <= 0 || p.Period != 0 { - // Either dynamic lambda is disabled, or we're not in period 0 and - // therefore, can't use dynamic lambda + // Either dynamic filter timeout is disabled, or we're not in period 0 + // and therefore, can't use dynamic timeout return FilterTimeout(p.Period, ver) } defaultTimeout := FilterTimeout(0, ver) @@ -325,25 +322,25 @@ func (p *player) calculateFilterTimeout(ver protocol.ConsensusVersion, tracer *t sort.Slice(sortedArrivals, func(i, j int) bool { return sortedArrivals[i] < sortedArrivals[j] }) dynamicTimeout := sortedArrivals[proto.DynamicFilterTimeoutCredentialArrivalHistoryIdx] + proto.DynamicFilterTimeoutGraceInterval - // Make sure the dynamic delay is not too small or too large + // Make sure the dynamic filter timeout is not too small nor too large if dynamicTimeout < proto.DynamicFilterTimeoutLowerBound { if tracer != nil { - tracer.log.Debugf("Calculated dynamic delay = %v for round %v, period %v. It's too low, setting to %v\n", dynamicTimeout, p.Round, p.Period, proto.DynamicFilterTimeoutLowerBound) + tracer.log.Debugf("Calculated dynamic filter timeout = %v for round %v, period %v. It's too low, setting to %v\n", dynamicTimeout, p.Round, p.Period, proto.DynamicFilterTimeoutLowerBound) } dynamicTimeout = proto.DynamicFilterTimeoutLowerBound } else if dynamicTimeout > defaultTimeout { if tracer != nil { - tracer.log.Debugf("Calculated dynamic delay = %v for round %v, period %v. It too high, so using the default %v\n", dynamicTimeout, p.Round, p.Period, defaultTimeout) + tracer.log.Debugf("Calculated dynamic filter timeout = %v for round %v, period %v. It too high, so using the default %v\n", dynamicTimeout, p.Round, p.Period, defaultTimeout) } dynamicTimeout = defaultTimeout } else if tracer != nil { - tracer.log.Debugf("Calculated dynamic delay = %v for round %v, period %v.\n", dynamicTimeout, p.Round, p.Period) + tracer.log.Debugf("Calculated dynamic filter timeout = %v for round %v, period %v.\n", dynamicTimeout, p.Round, p.Period) } if !proto.DynamicFilterTimeout { - // If the dynamic filter is disabled, return the default filter timeout - // (after logging what the timeout would have been, if this feature were - // enabled). + // If the dynamic filter timeout is disabled, return the default filter + // timeout (after logging what the timeout would have been, if this + // feature were enabled). return defaultTimeout } @@ -365,7 +362,7 @@ func (p *player) handleThresholdEvent(r routerHandle, e thresholdEvent) []action cert := Certificate(e.Bundle) a0 := ensureAction{Payload: res.Payload, Certificate: cert} actions = append(actions, a0) - p.updateDynamicLambdaTimings(r, e.Proto) + p.updateCredentialArrivalHistory(r, e.Proto) as := p.enterRound(r, e, p.Round+1) return append(actions, as...) } @@ -695,7 +692,7 @@ func (p *player) handleMessageEvent(r routerHandle, e messageEvent) (actions []a cert := Certificate(freshestRes.Event.Bundle) a0 := ensureAction{Payload: e.Input.Proposal, Certificate: cert} actions = append(actions, a0) - p.updateDynamicLambdaTimings(r, e.Proto.Version) + p.updateCredentialArrivalHistory(r, e.Proto.Version) as := p.enterRound(r, delegatedE, cert.Round+1) return append(actions, as...) } diff --git a/agreement/service_test.go b/agreement/service_test.go index 33d28db651..9a35191b10 100644 --- a/agreement/service_test.go +++ b/agreement/service_test.go @@ -77,7 +77,7 @@ func (c *testingClock) Zero() timers.Clock[TimeoutType] { } func (c *testingClock) Since() time.Duration { - return 0 + return 1 } func (c *testingClock) TimeoutAt(d time.Duration, timeoutType TimeoutType) <-chan time.Time { diff --git a/config/consensus.go b/config/consensus.go index 6d9dfeff51..a5644eb8c0 100644 --- a/config/consensus.go +++ b/config/consensus.go @@ -1394,7 +1394,7 @@ func initConsensusProtocols() { // percentile, which is the timestamp at index 37 in the history array. vFuture.DynamicFilterCredentialArrivalHistory = 40 vFuture.DynamicFilterTimeoutCredentialArrivalHistoryIdx = 37 - vFuture.DynamicFilterTimeoutLowerBound = time.Second + vFuture.DynamicFilterTimeoutLowerBound = 0 vFuture.DynamicFilterTimeoutGraceInterval = 50 * time.Millisecond vFuture.DynamicFilterTimeout = true From b88eab33adef7012cabe707a8c5c08548814ffac Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Fri, 18 Aug 2023 12:43:54 -0400 Subject: [PATCH 058/232] add negative test: make sure we discard history window when going to period > 0 --- agreement/service_test.go | 114 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 114 insertions(+) diff --git a/agreement/service_test.go b/agreement/service_test.go index 9a35191b10..bfd33d59ea 100644 --- a/agreement/service_test.go +++ b/agreement/service_test.go @@ -1056,6 +1056,120 @@ func TestAgreementSynchronousFuture5_DynamicFilterRounds(t *testing.T) { } } +func TestDynamicFilterTimeoutResets(t *testing.T) { + partitiontest.PartitionTest(t) + + if testing.Short() { + t.Skip("Skipping agreement integration test") + } + + version := protocol.ConsensusFuture + + cfg := config.Consensus[protocol.ConsensusFuture] + if cfg.DynamicFilterCredentialArrivalHistory <= 0 { + return + } + if !cfg.DynamicFilterTimeout { + return + } + + numNodes := 5 + + consensusVersion := func(r basics.Round) (protocol.ConsensusVersion, error) { + return protocol.ConsensusFuture, nil + } + ledgerFactory := func(data map[basics.Address]basics.AccountData) Ledger { + return makeTestLedgerWithConsensusVersion(data, consensusVersion) + } + + baseNetwork, baseLedger, cleanupFn, services, clocks, ledgers, activityMonitor := setupAgreement(t, numNodes, disabled, ledgerFactory) + startRound := baseLedger.NextRound() + defer cleanupFn() + + for i := 0; i < numNodes; i++ { + services[i].Start() + } + activityMonitor.waitForActivity() + activityMonitor.waitForQuiet() + zeroes := expectNewPeriod(clocks, 0) + + filterTimeouts := make([][]time.Duration, numNodes, numNodes) + + // run round with round-specific consensus version first (since fix in #1896) + zeroes = runRoundTriggerFilter(clocks, activityMonitor, zeroes) + for j := 1; j < cfg.DynamicFilterCredentialArrivalHistory+1; j++ { + for srvIdx, clock := range clocks { + delta, err := clock.(*testingClock).when(TimeoutFilter) + require.NoError(t, err) + filterTimeouts[srvIdx] = append(filterTimeouts[srvIdx], delta) + } + zeroes = runRoundTriggerFilter(clocks, activityMonitor, zeroes) + } + + for i := range clocks { + require.Len(t, filterTimeouts[i], cfg.DynamicFilterCredentialArrivalHistory) + for j := 1; j < cfg.DynamicFilterCredentialArrivalHistory-1; j++ { + require.Equal(t, filterTimeouts[i][j-1], filterTimeouts[i][j]) + } + fmt.Println(filterTimeouts[i]) + require.Less(t, filterTimeouts[i][cfg.DynamicFilterCredentialArrivalHistory-1], filterTimeouts[i][cfg.DynamicFilterCredentialArrivalHistory-2]) + } + + // force fast partition recovery into bottom + { + baseNetwork.dropAllSoftVotes() + baseNetwork.dropAllSlowNextVotes() + + triggerGlobalTimeout(FilterTimeout(0, version), TimeoutFilter, clocks, activityMonitor) + zeroes = expectNoNewPeriod(clocks, zeroes) + + triggerGlobalTimeoutType(TimeoutDeadline, clocks, activityMonitor) + zeroes = expectNoNewPeriod(clocks, zeroes) + + triggerGlobalTimeout(0, TimeoutFastRecovery, clocks, activityMonitor) // activates fast partition recovery timer + zeroes = expectNoNewPeriod(clocks, zeroes) + + triggerGlobalTimeout(firstFPR, TimeoutFastRecovery, clocks, activityMonitor) + zeroes = expectNewPeriod(clocks, zeroes) + } + + // terminate on period 1 + { + baseNetwork.repairAll() + triggerGlobalTimeout(FilterTimeout(1, version), TimeoutFilter, clocks, activityMonitor) + zeroes = expectNewPeriod(clocks, zeroes) + } + + filterTimeoutsPostRecovery := make([][]time.Duration, numNodes, numNodes) + + // run round with round-specific consensus version first (since fix in #1896) + zeroes = runRoundTriggerFilter(clocks, activityMonitor, zeroes) + for j := 1; j < cfg.DynamicFilterCredentialArrivalHistory+1; j++ { + for srvIdx, clock := range clocks { + delta, err := clock.(*testingClock).when(TimeoutFilter) + require.NoError(t, err) + filterTimeoutsPostRecovery[srvIdx] = append(filterTimeoutsPostRecovery[srvIdx], delta) + } + zeroes = runRoundTriggerFilter(clocks, activityMonitor, zeroes) + } + + for i := range clocks { + require.Len(t, filterTimeoutsPostRecovery[i], cfg.DynamicFilterCredentialArrivalHistory) + // check that history was discarded, so filter time increased back to its original default + require.Less(t, filterTimeouts[i][cfg.DynamicFilterCredentialArrivalHistory-1], filterTimeoutsPostRecovery[i][0]) + require.Equal(t, filterTimeouts[i][cfg.DynamicFilterCredentialArrivalHistory-2], filterTimeoutsPostRecovery[i][0]) + + // check that filter timeout was updated to at the end of the history window + for j := 1; j < cfg.DynamicFilterCredentialArrivalHistory-1; j++ { + require.Equal(t, filterTimeoutsPostRecovery[i][j-1], filterTimeoutsPostRecovery[i][j]) + } + fmt.Println(filterTimeoutsPostRecovery[i]) + require.Less(t, filterTimeoutsPostRecovery[i][cfg.DynamicFilterCredentialArrivalHistory-1], filterTimeoutsPostRecovery[i][cfg.DynamicFilterCredentialArrivalHistory-2]) + } + + sanityCheck(startRound, 2*round(cfg.DynamicFilterCredentialArrivalHistory+1)+1, ledgers) +} + func TestAgreementSynchronousFuture1(t *testing.T) { partitiontest.PartitionTest(t) From d14bde232328d55c5b7c18d44d30a99ea61b10fa Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Wed, 9 Aug 2023 15:56:43 -0400 Subject: [PATCH 059/232] tagged clock interface --- util/timers/interface.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/util/timers/interface.go b/util/timers/interface.go index d217437e9e..8a3e9cc73f 100644 --- a/util/timers/interface.go +++ b/util/timers/interface.go @@ -21,6 +21,13 @@ import ( "time" ) +type Timeout int + +const ( + Filter Timeout = iota + Fast +) + // Clock provides timeout events which fire at some point after a point in time. type Clock[TimeoutType comparable] interface { // Zero returns a reset Clock. TimeoutAt channels will use the point From b53d93e0dae62e63e7b583895fc65de633b1f7b5 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Wed, 9 Aug 2023 16:20:43 -0400 Subject: [PATCH 060/232] checkpoint: integrating the new clock --- util/timers/interface.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/util/timers/interface.go b/util/timers/interface.go index 8a3e9cc73f..34ec6933b5 100644 --- a/util/timers/interface.go +++ b/util/timers/interface.go @@ -24,8 +24,9 @@ import ( type Timeout int const ( - Filter Timeout = iota + Deadline Timeout = iota Fast + Filter ) // Clock provides timeout events which fire at some point after a point in time. From 85585bca140467076216711fd72f8e267dfece2a Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Wed, 9 Aug 2023 17:55:26 -0400 Subject: [PATCH 061/232] checkpoint: integrating tagged timers --- Makefile | 2 +- agreement/msgp_gen.go | 19 ++++++++++--------- agreement/player.go | 7 +++++++ util/timers/interface.go | 2 +- 4 files changed, 19 insertions(+), 11 deletions(-) diff --git a/Makefile b/Makefile index 8301771718..a62b529dad 100644 --- a/Makefile +++ b/Makefile @@ -86,7 +86,7 @@ ALGOD_API_PACKAGES := $(sort $(shell GOPATH=$(GOPATH) && GO111MODULE=off && cd d GOMOD_DIRS := ./tools/block-generator ./tools/x-repo-types -MSGP_GENERATE := ./protocol ./protocol/test ./crypto ./crypto/merklearray ./crypto/merklesignature ./crypto/stateproof ./data/basics ./data/transactions ./data/stateproofmsg ./data/committee ./data/bookkeeping ./data/hashable ./agreement ./rpcs ./network ./node ./ledger ./ledger/ledgercore ./ledger/store/trackerdb ./ledger/store/trackerdb/generickv ./ledger/encoded ./stateproof ./data/account ./daemon/algod/api/spec/v2 +MSGP_GENERATE := ./protocol ./protocol/test ./crypto ./crypto/merklearray ./crypto/merklesignature ./crypto/stateproof ./data/basics ./data/transactions ./data/stateproofmsg ./data/committee ./data/bookkeeping ./data/hashable ./agreement ./rpcs ./network ./node ./ledger ./ledger/ledgercore ./ledger/store/trackerdb ./ledger/store/trackerdb/generickv ./ledger/encoded ./stateproof ./data/account ./daemon/algod/api/spec/v2 ./util/timers default: build diff --git a/agreement/msgp_gen.go b/agreement/msgp_gen.go index d083edd8e8..3dc4773655 100644 --- a/agreement/msgp_gen.go +++ b/agreement/msgp_gen.go @@ -13,6 +13,7 @@ import ( "github.com/algorand/go-algorand/data/bookkeeping" "github.com/algorand/go-algorand/data/committee" "github.com/algorand/go-algorand/protocol" + "github.com/algorand/go-algorand/util/timers" ) // The following msgp objects are implemented in this file: @@ -4124,33 +4125,33 @@ func (z *player) UnmarshalMsg(bts []byte) (o []byte, err error) { } case "Period": { - var zb0006 uint64 - zb0006, bts, err = msgp.ReadUint64Bytes(bts) + var zb0008 uint64 + zb0008, bts, err = msgp.ReadUint64Bytes(bts) if err != nil { err = msgp.WrapError(err, "Period") return } - (*z).Period = period(zb0006) + (*z).Period = period(zb0008) } case "Step": { - var zb0007 uint64 - zb0007, bts, err = msgp.ReadUint64Bytes(bts) + var zb0009 uint64 + zb0009, bts, err = msgp.ReadUint64Bytes(bts) if err != nil { err = msgp.WrapError(err, "Step") return } - (*z).Step = step(zb0007) + (*z).Step = step(zb0009) } case "LastConcluding": { - var zb0008 uint64 - zb0008, bts, err = msgp.ReadUint64Bytes(bts) + var zb0010 uint64 + zb0010, bts, err = msgp.ReadUint64Bytes(bts) if err != nil { err = msgp.WrapError(err, "LastConcluding") return } - (*z).LastConcluding = step(zb0008) + (*z).LastConcluding = step(zb0010) } case "TimersDeadline": bts, err = (*z).Deadline.UnmarshalMsg(bts) diff --git a/agreement/player.go b/agreement/player.go index a3451ea0db..33dd2351f2 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -21,8 +21,15 @@ import ( "github.com/algorand/go-algorand/config" "github.com/algorand/go-algorand/protocol" + "github.com/algorand/go-algorand/util/timers" ) +type Deadline struct { + _struct struct{} `codec:","` + Deadline time.Duration + Type timers.Timeout +} + // The player implements the top-level state machine functionality of the // agreement protocol. type player struct { diff --git a/util/timers/interface.go b/util/timers/interface.go index 34ec6933b5..2e5f34c30f 100644 --- a/util/timers/interface.go +++ b/util/timers/interface.go @@ -21,7 +21,7 @@ import ( "time" ) -type Timeout int +type Timeout int8 const ( Deadline Timeout = iota From 5bce86215b5a5f3903e7acbcd342f8e62cbfad84 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Wed, 9 Aug 2023 21:33:31 -0400 Subject: [PATCH 062/232] fix failing tests --- agreement/service_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/agreement/service_test.go b/agreement/service_test.go index fc50c302d5..7cb7185912 100644 --- a/agreement/service_test.go +++ b/agreement/service_test.go @@ -1525,7 +1525,11 @@ func TestAgreementLateCertBug(t *testing.T) { closeFn() baseNetwork.repairAll() +<<<<<<< HEAD triggerGlobalTimeout(deadlineTimeout, TimeoutDeadline, clocks, activityMonitor) +======= + triggerGlobalTimeout(deadlineTimeout, timers.Deadline, clocks, activityMonitor) +>>>>>>> 1fc2f90d3 (fix failing tests) zeroes = expectNewPeriod(clocks, zeroes) } From 786e779e170f285863cf6a8ee2285491e99547e9 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Wed, 9 Aug 2023 23:25:18 -0400 Subject: [PATCH 063/232] names refactor --- agreement/player.go | 4 ++-- agreement/service_test.go | 4 ---- util/timers/interface.go | 6 +++--- 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/agreement/player.go b/agreement/player.go index 33dd2351f2..d389ae5afc 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -26,8 +26,8 @@ import ( type Deadline struct { _struct struct{} `codec:","` - Deadline time.Duration - Type timers.Timeout + Duration time.Duration + Type timers.TimeoutType } // The player implements the top-level state machine functionality of the diff --git a/agreement/service_test.go b/agreement/service_test.go index 7cb7185912..fc50c302d5 100644 --- a/agreement/service_test.go +++ b/agreement/service_test.go @@ -1525,11 +1525,7 @@ func TestAgreementLateCertBug(t *testing.T) { closeFn() baseNetwork.repairAll() -<<<<<<< HEAD triggerGlobalTimeout(deadlineTimeout, TimeoutDeadline, clocks, activityMonitor) -======= - triggerGlobalTimeout(deadlineTimeout, timers.Deadline, clocks, activityMonitor) ->>>>>>> 1fc2f90d3 (fix failing tests) zeroes = expectNewPeriod(clocks, zeroes) } diff --git a/util/timers/interface.go b/util/timers/interface.go index 2e5f34c30f..be368b5fcc 100644 --- a/util/timers/interface.go +++ b/util/timers/interface.go @@ -21,11 +21,11 @@ import ( "time" ) -type Timeout int8 +type TimeoutType int8 const ( - Deadline Timeout = iota - Fast + Deadline TimeoutType = iota + FastRecovery Filter ) From e02464fadb49e572cfb541f8b5de5f6554032393 Mon Sep 17 00:00:00 2001 From: chris erway Date: Wed, 2 Aug 2023 13:38:19 -0400 Subject: [PATCH 064/232] add some pieces of dynamic lambda from feature/pipeline into agreement --- agreement/player.go | 44 ++++++++++++++++++++++++++++++++++++++++++++ config/consensus.go | 19 +++++++++++++++++++ 2 files changed, 63 insertions(+) diff --git a/agreement/player.go b/agreement/player.go index d389ae5afc..63f08c474f 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -17,6 +17,7 @@ package agreement import ( + "sort" "time" "github.com/algorand/go-algorand/config" @@ -65,6 +66,8 @@ type player struct { // Pending holds the player's proposalTable, which stores proposals that // must be verified after some vote has been verified. Pending proposalTable + + payloadArrivals []time.Duration } func (p *player) T() stateMachineTag { @@ -273,6 +276,45 @@ func (p *player) handleCheckpointEvent(r routerHandle, e checkpointEvent) []acti }} } +func (p *player) handleWinningPayloadArrival(payload proposal) { + // ignoring validatedAt + p.payloadArrivals = append(p.payloadArrivals, payload.receivedAt) +} + +// calculateFilterTimeout chooses the appropriate filter timeout for a new round. +func (p *player) calculateFilterTimeout(period period, ver protocol.ConsensusVersion) time.Duration { + proto := config.Consensus[ver] + + if !proto.DynamicFilterTimeout || period != 0 { + return FilterTimeout(period, ver) + } + + var dynamicDelay time.Duration + if proto.AgreementPipelineDelay <= 0 { + dynamicDelay = 0 + } else if proto.AgreementPipelineDelay > len(p.payloadArrivals) { + dynamicDelay = FilterTimeout(0, ver) + } else { + sortedArrivals := make([]time.Duration, len(p.payloadArrivals)) + copy(sortedArrivals[:], p.payloadArrivals[:]) + sort.Slice(sortedArrivals, func(i, j int) bool { return sortedArrivals[i] < sortedArrivals[j] }) + dynamicDelay = sortedArrivals[proto.AgreementPipelineDelay-1] + } + + // Make sure the dynamic delay is not too small; we want to + // evenly space out the pipelined rounds across FilterTimeout, + // which is the fastest we could agree on blocks anyway (not + // including the soft vote / cert vote times). + if proto.AgreementPipelineDepth > 0 { + evenSpacing := FilterTimeout(0, ver) / time.Duration(proto.AgreementPipelineDepth) + if dynamicDelay < evenSpacing { + dynamicDelay = evenSpacing + } + } + + return dynamicDelay +} + func (p *player) handleThresholdEvent(r routerHandle, e thresholdEvent) []action { r.t.timeR().RecThreshold(e) @@ -288,6 +330,7 @@ func (p *player) handleThresholdEvent(r routerHandle, e thresholdEvent) []action cert := Certificate(e.Bundle) a0 := ensureAction{Payload: res.Payload, Certificate: cert} actions = append(actions, a0) + p.handleWinningPayloadArrival(res.Payload) as := p.enterRound(r, e, p.Round+1) return append(actions, as...) } @@ -607,6 +650,7 @@ func (p *player) handleMessageEvent(r routerHandle, e messageEvent) (actions []a cert := Certificate(freshestRes.Event.Bundle) a0 := ensureAction{Payload: e.Input.Proposal, Certificate: cert} actions = append(actions, a0) + p.handleWinningPayloadArrival(e.Input.Proposal) as := p.enterRound(r, delegatedE, cert.Round+1) return append(actions, as...) } diff --git a/config/consensus.go b/config/consensus.go index 8c217c18c1..1eed8b8c11 100644 --- a/config/consensus.go +++ b/config/consensus.go @@ -516,6 +516,25 @@ type ConsensusParams struct { // used by agreement for Circulation, and updates the calculation of StateProofOnlineTotalWeight used // by state proofs to use the same method (rather than excluding stake from the top N stakeholders as before). ExcludeExpiredCirculation bool + + // DynamicFilterTimeout + DynamicFilterTimeout bool + + // AgreementPipelineDepth specifies the maximum number of pipelined + // rounds that the agreement protocol can run ahead with. This is + // the maximum difference between the first uncommitted round and + // the round number that we run the agreement protocol for. + AgreementPipelineDepth int + + // AgreementPipelineDelayHistory specifies the number of past block arrivals + // that are measured to determine when to start pipelining the next block. + AgreementPipelineDelayHistory int + + // AgreementPipelineDelay specifies when the agreement code should start + // pipelining the next block, by choosing the delay time of the + // AgreementPipelineDelay'th slowest block to arrive out of the last + // AgreementPipelineDelayHistory lowest-credential block payloads. + AgreementPipelineDelay int } // PaysetCommitType enumerates possible ways for the block header to commit to From b7feca169104b89fd4736319768121b224144f07 Mon Sep 17 00:00:00 2001 From: chris erway Date: Thu, 3 Aug 2023 09:18:23 -0400 Subject: [PATCH 065/232] add/adapt code from pipelinePlayer.resizeArrivals on feature/pipeline --- agreement/player.go | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/agreement/player.go b/agreement/player.go index 63f08c474f..e1960e71b2 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -276,9 +276,20 @@ func (p *player) handleCheckpointEvent(r routerHandle, e checkpointEvent) []acti }} } -func (p *player) handleWinningPayloadArrival(payload proposal) { +func (p *player) handleWinningPayloadArrival(payload proposal, ver protocol.ConsensusVersion) { // ignoring validatedAt p.payloadArrivals = append(p.payloadArrivals, payload.receivedAt) + p.resizePayloadArrivals(ver) +} + +func (p *player) resizePayloadArrivals(ver protocol.ConsensusVersion) { + proto := config.Consensus[ver] + if len(p.payloadArrivals) > proto.AgreementPipelineDelayHistory { + p.payloadArrivals = p.payloadArrivals[len(p.payloadArrivals)-proto.AgreementPipelineDelayHistory:] + } + for len(p.payloadArrivals) < proto.AgreementPipelineDelayHistory { + p.payloadArrivals = append([]time.Duration{FilterTimeout(0, ver)}, p.payloadArrivals...) + } } // calculateFilterTimeout chooses the appropriate filter timeout for a new round. @@ -330,7 +341,7 @@ func (p *player) handleThresholdEvent(r routerHandle, e thresholdEvent) []action cert := Certificate(e.Bundle) a0 := ensureAction{Payload: res.Payload, Certificate: cert} actions = append(actions, a0) - p.handleWinningPayloadArrival(res.Payload) + p.handleWinningPayloadArrival(res.Payload, e.Proto) as := p.enterRound(r, e, p.Round+1) return append(actions, as...) } @@ -650,7 +661,7 @@ func (p *player) handleMessageEvent(r routerHandle, e messageEvent) (actions []a cert := Certificate(freshestRes.Event.Bundle) a0 := ensureAction{Payload: e.Input.Proposal, Certificate: cert} actions = append(actions, a0) - p.handleWinningPayloadArrival(e.Input.Proposal) + p.handleWinningPayloadArrival(e.Input.Proposal, e.Proto.Version) as := p.enterRound(r, delegatedE, cert.Round+1) return append(actions, as...) } From b999bee294f0bed99de60ad8e90d923f46904d48 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Tue, 8 Aug 2023 10:57:06 -0400 Subject: [PATCH 066/232] checkpoint: param cleanup --- agreement/player.go | 35 +++++++++++++++++++---------------- config/consensus.go | 21 ++++++++------------- 2 files changed, 27 insertions(+), 29 deletions(-) diff --git a/agreement/player.go b/agreement/player.go index e1960e71b2..9cfa862725 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -17,6 +17,7 @@ package agreement import ( + "fmt" "sort" "time" @@ -284,45 +285,47 @@ func (p *player) handleWinningPayloadArrival(payload proposal, ver protocol.Cons func (p *player) resizePayloadArrivals(ver protocol.ConsensusVersion) { proto := config.Consensus[ver] - if len(p.payloadArrivals) > proto.AgreementPipelineDelayHistory { - p.payloadArrivals = p.payloadArrivals[len(p.payloadArrivals)-proto.AgreementPipelineDelayHistory:] + if len(p.payloadArrivals) > proto.DynamicFilterPayloadArriavalHistory { + p.payloadArrivals = p.payloadArrivals[len(p.payloadArrivals)-proto.DynamicFilterPayloadArriavalHistory:] } - for len(p.payloadArrivals) < proto.AgreementPipelineDelayHistory { + for len(p.payloadArrivals) < proto.DynamicFilterPayloadArriavalHistory { p.payloadArrivals = append([]time.Duration{FilterTimeout(0, ver)}, p.payloadArrivals...) } } // calculateFilterTimeout chooses the appropriate filter timeout for a new round. func (p *player) calculateFilterTimeout(period period, ver protocol.ConsensusVersion) time.Duration { + proto := config.Consensus[ver] if !proto.DynamicFilterTimeout || period != 0 { + // Either dynamic lambda is disabled, or we're not in period 0 and + // therefore can't use dynamic lambda return FilterTimeout(period, ver) } var dynamicDelay time.Duration - if proto.AgreementPipelineDelay <= 0 { - dynamicDelay = 0 - } else if proto.AgreementPipelineDelay > len(p.payloadArrivals) { + if proto.DynamicFilterPayloadArriavalHistory <= 0 { + // we don't keep any history, use the default + dynamicDelay = FilterTimeout(0, ver) + } else if proto.DynamicFilterPayloadArriavalHistory > len(p.payloadArrivals) { + // not enough smaples, use the default dynamicDelay = FilterTimeout(0, ver) } else { sortedArrivals := make([]time.Duration, len(p.payloadArrivals)) copy(sortedArrivals[:], p.payloadArrivals[:]) sort.Slice(sortedArrivals, func(i, j int) bool { return sortedArrivals[i] < sortedArrivals[j] }) - dynamicDelay = sortedArrivals[proto.AgreementPipelineDelay-1] + dynamicDelay = sortedArrivals[proto.DynamicFilterPayloadArriavalHistory-1] } - // Make sure the dynamic delay is not too small; we want to - // evenly space out the pipelined rounds across FilterTimeout, - // which is the fastest we could agree on blocks anyway (not - // including the soft vote / cert vote times). - if proto.AgreementPipelineDepth > 0 { - evenSpacing := FilterTimeout(0, ver) / time.Duration(proto.AgreementPipelineDepth) - if dynamicDelay < evenSpacing { - dynamicDelay = evenSpacing - } + // Make sure the dynamic delay is not too small + if dynamicDelay < proto.DynamicFilterTimeoutLowerBound { + dynamicDelay = proto.DynamicFilterTimeoutLowerBound } + //dynamicDelay = FilterTimeout(period, ver) + fmt.Println("dynamic delay", dynamicDelay, "payload arrivals", len(p.payloadArrivals)) + return dynamicDelay } diff --git a/config/consensus.go b/config/consensus.go index 1eed8b8c11..02cc03ee9e 100644 --- a/config/consensus.go +++ b/config/consensus.go @@ -520,21 +520,11 @@ type ConsensusParams struct { // DynamicFilterTimeout DynamicFilterTimeout bool - // AgreementPipelineDepth specifies the maximum number of pipelined - // rounds that the agreement protocol can run ahead with. This is - // the maximum difference between the first uncommitted round and - // the round number that we run the agreement protocol for. - AgreementPipelineDepth int + DynamicFilterTimeoutLowerBound time.Duration - // AgreementPipelineDelayHistory specifies the number of past block arrivals + // DynamicFilterPayloadArriavalHistory specifies the number of past block arrivals // that are measured to determine when to start pipelining the next block. - AgreementPipelineDelayHistory int - - // AgreementPipelineDelay specifies when the agreement code should start - // pipelining the next block, by choosing the delay time of the - // AgreementPipelineDelay'th slowest block to arrive out of the last - // AgreementPipelineDelayHistory lowest-credential block payloads. - AgreementPipelineDelay int + DynamicFilterPayloadArriavalHistory int } // PaysetCommitType enumerates possible ways for the block header to commit to @@ -1366,6 +1356,10 @@ func initConsensusProtocols() { v38.AgreementFilterTimeoutPeriod0 = 3000 * time.Millisecond + v38.DynamicFilterTimeout = true + v38.DynamicFilterPayloadArriavalHistory = 32 + v38.DynamicFilterTimeoutLowerBound = time.Second + Consensus[protocol.ConsensusV38] = v38 // v37 can be upgraded to v38, with an update delay of 12h: @@ -1376,6 +1370,7 @@ func initConsensusProtocols() { // ConsensusFuture is used to test features that are implemented // but not yet released in a production protocol version. vFuture := v38 + vFuture.ApprovedUpgrades = map[protocol.ConsensusVersion]uint64{} vFuture.LogicSigVersion = 10 // When moving this to a release, put a new higher LogicSigVersion here From 15eaa70b2946c96080008e7a8e3d49ef3c8d6f41 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Tue, 8 Aug 2023 15:54:45 -0400 Subject: [PATCH 067/232] reset history on non-0 periods --- agreement/player.go | 4 +++- agreement/service_test.go | 18 ++++++++++++++++-- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/agreement/player.go b/agreement/player.go index 9cfa862725..f2ea29ea32 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -298,9 +298,11 @@ func (p *player) calculateFilterTimeout(period period, ver protocol.ConsensusVer proto := config.Consensus[ver] + //return FilterTimeout(period, ver) + if !proto.DynamicFilterTimeout || period != 0 { // Either dynamic lambda is disabled, or we're not in period 0 and - // therefore can't use dynamic lambda + // therefore, can't use dynamic lambda return FilterTimeout(period, ver) } diff --git a/agreement/service_test.go b/agreement/service_test.go index fc50c302d5..ef7db941bb 100644 --- a/agreement/service_test.go +++ b/agreement/service_test.go @@ -891,6 +891,20 @@ func simulateAgreementWithConsensusVersion(t *testing.T, numNodes int, numRounds simulateAgreementWithLedgerFactory(t, numNodes, numRounds, traceLevel, ledgerFactory) } +func maxFilterTimeoutPeriod0(services []*Service, version protocol.ConsensusVersion) time.Duration { + if len(services) == 0 { + return FilterTimeout(0, version) + } + maxFilter := services[0].persistStatus.calculateFilterTimeout(0, version) + for _, service := range services { + curr := service.persistStatus.calculateFilterTimeout(0, version) + if maxFilter < curr { + maxFilter = curr + } + } + return maxFilter +} + func simulateAgreementWithLedgerFactory(t *testing.T, numNodes int, numRounds int, traceLevel traceLevel, ledgerFactory func(map[basics.Address]basics.AccountData) Ledger) { _, baseLedger, cleanupFn, services, clocks, ledgers, activityMonitor := setupAgreement(t, numNodes, traceLevel, ledgerFactory) startRound := baseLedger.NextRound() @@ -905,10 +919,10 @@ func simulateAgreementWithLedgerFactory(t *testing.T, numNodes int, numRounds in // run round with round-specific consensus version first (since fix in #1896) version, _ := baseLedger.ConsensusVersion(ParamsRound(startRound)) - zeroes = runRound(clocks, activityMonitor, zeroes, FilterTimeout(0, version)) + zeroes = runRound(clocks, activityMonitor, zeroes, maxFilterTimeoutPeriod0(services, version)) for j := 1; j < numRounds; j++ { version, _ := baseLedger.ConsensusVersion(ParamsRound(baseLedger.NextRound() + basics.Round(j-1))) - zeroes = runRound(clocks, activityMonitor, zeroes, FilterTimeout(0, version)) + zeroes = runRound(clocks, activityMonitor, zeroes, maxFilterTimeoutPeriod0(services, version)) } for i := 0; i < numNodes; i++ { From 358c0eba6c44dfd5122f93d22651afd0f2f1b7cc Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Wed, 9 Aug 2023 15:15:10 -0400 Subject: [PATCH 068/232] test clock tweak --- agreement/player.go | 3 +-- agreement/service_test.go | 13 ++++++++++--- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/agreement/player.go b/agreement/player.go index f2ea29ea32..473f2a3bf5 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -17,7 +17,6 @@ package agreement import ( - "fmt" "sort" "time" @@ -326,7 +325,7 @@ func (p *player) calculateFilterTimeout(period period, ver protocol.ConsensusVer } //dynamicDelay = FilterTimeout(period, ver) - fmt.Println("dynamic delay", dynamicDelay, "payload arrivals", len(p.payloadArrivals)) + //fmt.Println("dynamic delay", dynamicDelay, "payload arrivals", len(p.payloadArrivals)) return dynamicDelay } diff --git a/agreement/service_test.go b/agreement/service_test.go index ef7db941bb..d61379cd91 100644 --- a/agreement/service_test.go +++ b/agreement/service_test.go @@ -891,7 +891,13 @@ func simulateAgreementWithConsensusVersion(t *testing.T, numNodes int, numRounds simulateAgreementWithLedgerFactory(t, numNodes, numRounds, traceLevel, ledgerFactory) } -func maxFilterTimeoutPeriod0(services []*Service, version protocol.ConsensusVersion) time.Duration { +func maxFilterTimeoutPeriod0(services []*Service, version protocol.ConsensusVersion, iter int) time.Duration { + if iter < 32 { // TODO: 32 is the history length we track for dynamic lambda. We need to change this param + return FilterTimeout(0, version) + } else { + return time.Second // this is the min filter timeout we're using. + } + if len(services) == 0 { return FilterTimeout(0, version) } @@ -919,10 +925,11 @@ func simulateAgreementWithLedgerFactory(t *testing.T, numNodes int, numRounds in // run round with round-specific consensus version first (since fix in #1896) version, _ := baseLedger.ConsensusVersion(ParamsRound(startRound)) - zeroes = runRound(clocks, activityMonitor, zeroes, maxFilterTimeoutPeriod0(services, version)) + zeroes = runRound(clocks, activityMonitor, zeroes, maxFilterTimeoutPeriod0(services, version, 0)) for j := 1; j < numRounds; j++ { + // TODO: remove this: time.Sleep(100 * time.Millisecond) version, _ := baseLedger.ConsensusVersion(ParamsRound(baseLedger.NextRound() + basics.Round(j-1))) - zeroes = runRound(clocks, activityMonitor, zeroes, maxFilterTimeoutPeriod0(services, version)) + zeroes = runRound(clocks, activityMonitor, zeroes, maxFilterTimeoutPeriod0(services, version, j)) } for i := 0; i < numNodes; i++ { From be5fe0fb781a5caf231aa95dbb9fb45902d8bc9b Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Thu, 10 Aug 2023 11:00:35 -0400 Subject: [PATCH 069/232] avoid weird clock tweak --- agreement/msgp_gen.go | 82 +++++++++++++++++++-------------------- agreement/service_test.go | 31 +++------------ 2 files changed, 47 insertions(+), 66 deletions(-) diff --git a/agreement/msgp_gen.go b/agreement/msgp_gen.go index 3dc4773655..bd04d34e0b 100644 --- a/agreement/msgp_gen.go +++ b/agreement/msgp_gen.go @@ -4001,57 +4001,57 @@ func (_ *player) CanMarshalMsg(z interface{}) bool { func (z *player) UnmarshalMsg(bts []byte) (o []byte, err error) { var field []byte _ = field - var zb0001 int - var zb0002 bool - zb0001, zb0002, bts, err = msgp.ReadMapHeaderBytes(bts) + var zb0002 int + var zb0003 bool + zb0002, zb0003, bts, err = msgp.ReadMapHeaderBytes(bts) if _, ok := err.(msgp.TypeError); ok { - zb0001, zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + zb0002, zb0003, bts, err = msgp.ReadArrayHeaderBytes(bts) if err != nil { err = msgp.WrapError(err) return } - if zb0001 > 0 { - zb0001-- + if zb0002 > 0 { + zb0002-- bts, err = (*z).Round.UnmarshalMsg(bts) if err != nil { err = msgp.WrapError(err, "struct-from-array", "Round") return } } - if zb0001 > 0 { - zb0001-- + if zb0002 > 0 { + zb0002-- { - var zb0003 uint64 - zb0003, bts, err = msgp.ReadUint64Bytes(bts) + var zb0004 uint64 + zb0004, bts, err = msgp.ReadUint64Bytes(bts) if err != nil { err = msgp.WrapError(err, "struct-from-array", "Period") return } - (*z).Period = period(zb0003) + (*z).Period = period(zb0004) } } - if zb0001 > 0 { - zb0001-- + if zb0002 > 0 { + zb0002-- { - var zb0004 uint64 - zb0004, bts, err = msgp.ReadUint64Bytes(bts) + var zb0005 uint64 + zb0005, bts, err = msgp.ReadUint64Bytes(bts) if err != nil { err = msgp.WrapError(err, "struct-from-array", "Step") return } - (*z).Step = step(zb0004) + (*z).Step = step(zb0005) } } - if zb0001 > 0 { - zb0001-- + if zb0002 > 0 { + zb0002-- { - var zb0005 uint64 - zb0005, bts, err = msgp.ReadUint64Bytes(bts) + var zb0006 uint64 + zb0006, bts, err = msgp.ReadUint64Bytes(bts) if err != nil { err = msgp.WrapError(err, "struct-from-array", "LastConcluding") return } - (*z).LastConcluding = step(zb0005) + (*z).LastConcluding = step(zb0006) } } if zb0001 > 0 { @@ -4070,32 +4070,32 @@ func (z *player) UnmarshalMsg(bts []byte) (o []byte, err error) { return } } - if zb0001 > 0 { - zb0001-- + if zb0002 > 0 { + zb0002-- (*z).Napping, bts, err = msgp.ReadBoolBytes(bts) if err != nil { err = msgp.WrapError(err, "struct-from-array", "Napping") return } } - if zb0001 > 0 { - zb0001-- + if zb0002 > 0 { + zb0002-- (*z).FastRecoveryDeadline, bts, err = msgp.ReadDurationBytes(bts) if err != nil { err = msgp.WrapError(err, "struct-from-array", "FastRecoveryDeadline") return } } - if zb0001 > 0 { - zb0001-- + if zb0002 > 0 { + zb0002-- bts, err = (*z).Pending.UnmarshalMsg(bts) if err != nil { err = msgp.WrapError(err, "struct-from-array", "Pending") return } } - if zb0001 > 0 { - err = msgp.ErrTooManyArrayFields(zb0001) + if zb0002 > 0 { + err = msgp.ErrTooManyArrayFields(zb0002) if err != nil { err = msgp.WrapError(err, "struct-from-array") return @@ -4106,11 +4106,11 @@ func (z *player) UnmarshalMsg(bts []byte) (o []byte, err error) { err = msgp.WrapError(err) return } - if zb0002 { + if zb0003 { (*z) = player{} } - for zb0001 > 0 { - zb0001-- + for zb0002 > 0 { + zb0002-- field, bts, err = msgp.ReadMapKeyZC(bts) if err != nil { err = msgp.WrapError(err) @@ -4125,33 +4125,33 @@ func (z *player) UnmarshalMsg(bts []byte) (o []byte, err error) { } case "Period": { - var zb0008 uint64 - zb0008, bts, err = msgp.ReadUint64Bytes(bts) + var zb0009 uint64 + zb0009, bts, err = msgp.ReadUint64Bytes(bts) if err != nil { err = msgp.WrapError(err, "Period") return } - (*z).Period = period(zb0008) + (*z).Period = period(zb0009) } case "Step": { - var zb0009 uint64 - zb0009, bts, err = msgp.ReadUint64Bytes(bts) + var zb0010 uint64 + zb0010, bts, err = msgp.ReadUint64Bytes(bts) if err != nil { err = msgp.WrapError(err, "Step") return } - (*z).Step = step(zb0009) + (*z).Step = step(zb0010) } case "LastConcluding": { - var zb0010 uint64 - zb0010, bts, err = msgp.ReadUint64Bytes(bts) + var zb0011 uint64 + zb0011, bts, err = msgp.ReadUint64Bytes(bts) if err != nil { err = msgp.WrapError(err, "LastConcluding") return } - (*z).LastConcluding = step(zb0010) + (*z).LastConcluding = step(zb0011) } case "TimersDeadline": bts, err = (*z).Deadline.UnmarshalMsg(bts) diff --git a/agreement/service_test.go b/agreement/service_test.go index d61379cd91..9afc61e119 100644 --- a/agreement/service_test.go +++ b/agreement/service_test.go @@ -862,6 +862,10 @@ func runRound(clocks []timers.Clock[TimeoutType], activityMonitor *activityMonit triggerGlobalTimeout(filterTimeout, TimeoutFilter, clocks, activityMonitor) return expectNewPeriod(clocks, zeroes) } +func runRoundTriggerFilter(clocks []timers.Clock, activityMonitor *activityMonitor, zeroes uint) (newzeroes uint) { + triggerGlobalTimeoutType(timers.Filter, clocks, activityMonitor) + return expectNewPeriod(clocks, zeroes) +} func sanityCheck(startRound round, numRounds round, ledgers []Ledger) { for i := range ledgers { @@ -891,26 +895,6 @@ func simulateAgreementWithConsensusVersion(t *testing.T, numNodes int, numRounds simulateAgreementWithLedgerFactory(t, numNodes, numRounds, traceLevel, ledgerFactory) } -func maxFilterTimeoutPeriod0(services []*Service, version protocol.ConsensusVersion, iter int) time.Duration { - if iter < 32 { // TODO: 32 is the history length we track for dynamic lambda. We need to change this param - return FilterTimeout(0, version) - } else { - return time.Second // this is the min filter timeout we're using. - } - - if len(services) == 0 { - return FilterTimeout(0, version) - } - maxFilter := services[0].persistStatus.calculateFilterTimeout(0, version) - for _, service := range services { - curr := service.persistStatus.calculateFilterTimeout(0, version) - if maxFilter < curr { - maxFilter = curr - } - } - return maxFilter -} - func simulateAgreementWithLedgerFactory(t *testing.T, numNodes int, numRounds int, traceLevel traceLevel, ledgerFactory func(map[basics.Address]basics.AccountData) Ledger) { _, baseLedger, cleanupFn, services, clocks, ledgers, activityMonitor := setupAgreement(t, numNodes, traceLevel, ledgerFactory) startRound := baseLedger.NextRound() @@ -924,12 +908,9 @@ func simulateAgreementWithLedgerFactory(t *testing.T, numNodes int, numRounds in zeroes := expectNewPeriod(clocks, 0) // run round with round-specific consensus version first (since fix in #1896) - version, _ := baseLedger.ConsensusVersion(ParamsRound(startRound)) - zeroes = runRound(clocks, activityMonitor, zeroes, maxFilterTimeoutPeriod0(services, version, 0)) + zeroes = runRoundTriggerFilter(clocks, activityMonitor, zeroes) for j := 1; j < numRounds; j++ { - // TODO: remove this: time.Sleep(100 * time.Millisecond) - version, _ := baseLedger.ConsensusVersion(ParamsRound(baseLedger.NextRound() + basics.Round(j-1))) - zeroes = runRound(clocks, activityMonitor, zeroes, maxFilterTimeoutPeriod0(services, version, j)) + zeroes = runRoundTriggerFilter(clocks, activityMonitor, zeroes) } for i := 0; i < numNodes; i++ { From 58ac6e83f94953c23489a0ad8295779a0e25c98a Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Thu, 10 Aug 2023 11:42:18 -0400 Subject: [PATCH 070/232] msgp for tagged deadline --- util/timers/msgp_gen.go | 70 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 util/timers/msgp_gen.go diff --git a/util/timers/msgp_gen.go b/util/timers/msgp_gen.go new file mode 100644 index 0000000000..ae3b2729b5 --- /dev/null +++ b/util/timers/msgp_gen.go @@ -0,0 +1,70 @@ +package timers + +// Code generated by github.com/algorand/msgp DO NOT EDIT. + +import ( + "github.com/algorand/msgp/msgp" +) + +// The following msgp objects are implemented in this file: +// TimeoutType +// |-----> MarshalMsg +// |-----> CanMarshalMsg +// |-----> (*) UnmarshalMsg +// |-----> (*) CanUnmarshalMsg +// |-----> Msgsize +// |-----> MsgIsZero +// |-----> TimeoutTypeMaxSize() +// + +// MarshalMsg implements msgp.Marshaler +func (z TimeoutType) MarshalMsg(b []byte) (o []byte) { + o = msgp.Require(b, z.Msgsize()) + o = msgp.AppendInt8(o, int8(z)) + return +} + +func (_ TimeoutType) CanMarshalMsg(z interface{}) bool { + _, ok := (z).(TimeoutType) + if !ok { + _, ok = (z).(*TimeoutType) + } + return ok +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *TimeoutType) UnmarshalMsg(bts []byte) (o []byte, err error) { + { + var zb0001 int8 + zb0001, bts, err = msgp.ReadInt8Bytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + (*z) = TimeoutType(zb0001) + } + o = bts + return +} + +func (_ *TimeoutType) CanUnmarshalMsg(z interface{}) bool { + _, ok := (z).(*TimeoutType) + return ok +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z TimeoutType) Msgsize() (s int) { + s = msgp.Int8Size + return +} + +// MsgIsZero returns whether this is a zero value +func (z TimeoutType) MsgIsZero() bool { + return z == 0 +} + +// MaxSize returns a maximum valid message size for this message type +func TimeoutTypeMaxSize() (s int) { + s = msgp.Int8Size + return +} From f679c57c50f8c523ec8918d56422f94f4e2f4f4c Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Thu, 10 Aug 2023 14:42:22 -0400 Subject: [PATCH 071/232] fix lint issues --- agreement/player.go | 2 ++ util/timers/interface.go | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/agreement/player.go b/agreement/player.go index 473f2a3bf5..9063371d14 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -25,6 +25,8 @@ import ( "github.com/algorand/go-algorand/util/timers" ) +// Deadline marks a timeout event of type Type that the player schedules to +// happen after Duration time. type Deadline struct { _struct struct{} `codec:","` Duration time.Duration diff --git a/util/timers/interface.go b/util/timers/interface.go index be368b5fcc..cbca039c84 100644 --- a/util/timers/interface.go +++ b/util/timers/interface.go @@ -21,11 +21,16 @@ import ( "time" ) +// TimeoutType annotates timeout events scheduled via TimeoutAt type TimeoutType int8 const ( + // Deadline annotates timeout events in the agreement protocol (e.g., for + // receiving a block) Deadline TimeoutType = iota + // FastRecovery annotates the fast recovery timeout in the agreement protocol FastRecovery + // Filter annotates the filter step timeout event in the agreement protocol Filter ) From a8b99985d03648e83f61e320cd2d458bdb50b532 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Thu, 10 Aug 2023 14:50:12 -0400 Subject: [PATCH 072/232] fix lint issues --- agreement/player.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/agreement/player.go b/agreement/player.go index 9063371d14..6bf1dd0389 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -299,8 +299,6 @@ func (p *player) calculateFilterTimeout(period period, ver protocol.ConsensusVer proto := config.Consensus[ver] - //return FilterTimeout(period, ver) - if !proto.DynamicFilterTimeout || period != 0 { // Either dynamic lambda is disabled, or we're not in period 0 and // therefore, can't use dynamic lambda @@ -312,7 +310,7 @@ func (p *player) calculateFilterTimeout(period period, ver protocol.ConsensusVer // we don't keep any history, use the default dynamicDelay = FilterTimeout(0, ver) } else if proto.DynamicFilterPayloadArriavalHistory > len(p.payloadArrivals) { - // not enough smaples, use the default + // not enough samples, use the default dynamicDelay = FilterTimeout(0, ver) } else { sortedArrivals := make([]time.Duration, len(p.payloadArrivals)) @@ -326,9 +324,6 @@ func (p *player) calculateFilterTimeout(period period, ver protocol.ConsensusVer dynamicDelay = proto.DynamicFilterTimeoutLowerBound } - //dynamicDelay = FilterTimeout(period, ver) - //fmt.Println("dynamic delay", dynamicDelay, "payload arrivals", len(p.payloadArrivals)) - return dynamicDelay } From 05cbd97088cb302d39e9a2bb1456ad5a9833bc1f Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Thu, 10 Aug 2023 15:13:10 -0400 Subject: [PATCH 073/232] parameter cleanup + logging --- agreement/player.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/agreement/player.go b/agreement/player.go index 6bf1dd0389..49a82f4772 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -294,15 +294,15 @@ func (p *player) resizePayloadArrivals(ver protocol.ConsensusVersion) { } } -// calculateFilterTimeout chooses the appropriate filter timeout for a new round. -func (p *player) calculateFilterTimeout(period period, ver protocol.ConsensusVersion) time.Duration { +// calculateFilterTimeout chooses the appropriate filter timeout. +func (p *player) calculateFilterTimeout(ver protocol.ConsensusVersion, tracer *tracer) time.Duration { proto := config.Consensus[ver] - if !proto.DynamicFilterTimeout || period != 0 { + if !proto.DynamicFilterTimeout || p.Period != 0 { // Either dynamic lambda is disabled, or we're not in period 0 and // therefore, can't use dynamic lambda - return FilterTimeout(period, ver) + return FilterTimeout(p.Period, ver) } var dynamicDelay time.Duration @@ -321,7 +321,10 @@ func (p *player) calculateFilterTimeout(period period, ver protocol.ConsensusVer // Make sure the dynamic delay is not too small if dynamicDelay < proto.DynamicFilterTimeoutLowerBound { + tracer.log.Infof("Calculated dynamic delay = %v for round %v, period %v. It's too low, setting to %v\n", dynamicDelay, p.Round, p.Period, proto.DynamicFilterTimeoutLowerBound) dynamicDelay = proto.DynamicFilterTimeoutLowerBound + } else { + tracer.log.Infof("Calculated dynamic delay = %v for round %v, period %v.\n", dynamicDelay, p.Round, p.Period) } return dynamicDelay From 06b0acd0b8275d9610be7138c8501edd837b4992 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Thu, 10 Aug 2023 16:31:04 -0400 Subject: [PATCH 074/232] lint --- agreement/player.go | 16 +++++++++++----- config/consensus.go | 5 ++++- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/agreement/player.go b/agreement/player.go index 49a82f4772..a9fa22b188 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -306,12 +306,13 @@ func (p *player) calculateFilterTimeout(ver protocol.ConsensusVersion, tracer *t } var dynamicDelay time.Duration + defaultDelay := FilterTimeout(0, ver) if proto.DynamicFilterPayloadArriavalHistory <= 0 { // we don't keep any history, use the default - dynamicDelay = FilterTimeout(0, ver) + dynamicDelay = defaultDelay } else if proto.DynamicFilterPayloadArriavalHistory > len(p.payloadArrivals) { // not enough samples, use the default - dynamicDelay = FilterTimeout(0, ver) + dynamicDelay = defaultDelay } else { sortedArrivals := make([]time.Duration, len(p.payloadArrivals)) copy(sortedArrivals[:], p.payloadArrivals[:]) @@ -319,11 +320,16 @@ func (p *player) calculateFilterTimeout(ver protocol.ConsensusVersion, tracer *t dynamicDelay = sortedArrivals[proto.DynamicFilterPayloadArriavalHistory-1] } - // Make sure the dynamic delay is not too small + // Make sure the dynamic delay is not too small or too large if dynamicDelay < proto.DynamicFilterTimeoutLowerBound { - tracer.log.Infof("Calculated dynamic delay = %v for round %v, period %v. It's too low, setting to %v\n", dynamicDelay, p.Round, p.Period, proto.DynamicFilterTimeoutLowerBound) + if tracer != nil { + tracer.log.Infof("Calculated dynamic delay = %v for round %v, period %v. It's too low, setting to %v\n", dynamicDelay, p.Round, p.Period, proto.DynamicFilterTimeoutLowerBound) + } dynamicDelay = proto.DynamicFilterTimeoutLowerBound - } else { + } else if dynamicDelay > defaultDelay { + dynamicDelay = defaultDelay + tracer.log.Warnf("Calculated dynamic delay = %v for round %v, period %v. It's higher than the default %v\n", dynamicDelay, p.Round, p.Period, defaultDelay) + } else if tracer != nil { tracer.log.Infof("Calculated dynamic delay = %v for round %v, period %v.\n", dynamicDelay, p.Round, p.Period) } diff --git a/config/consensus.go b/config/consensus.go index 02cc03ee9e..9fa3213c30 100644 --- a/config/consensus.go +++ b/config/consensus.go @@ -517,9 +517,12 @@ type ConsensusParams struct { // by state proofs to use the same method (rather than excluding stake from the top N stakeholders as before). ExcludeExpiredCirculation bool - // DynamicFilterTimeout + // DynamicFilterTimeout specifies whether the timeout for the filter step + // should be determined dynamically, at run-time. DynamicFilterTimeout bool + // DynamicFilterTimeoutLowerBound specifies a minimal duration that the + // filter timeout must meet. DynamicFilterTimeoutLowerBound time.Duration // DynamicFilterPayloadArriavalHistory specifies the number of past block arrivals From 32c942316bd3bd7d8f9c5836071ba939c6b2773c Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Thu, 10 Aug 2023 16:43:37 -0400 Subject: [PATCH 075/232] less logging, params in vfuture --- agreement/player.go | 4 ++-- agreement/service_test.go | 12 ++++++++++++ config/consensus.go | 8 ++++---- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/agreement/player.go b/agreement/player.go index a9fa22b188..3376922b68 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -323,14 +323,14 @@ func (p *player) calculateFilterTimeout(ver protocol.ConsensusVersion, tracer *t // Make sure the dynamic delay is not too small or too large if dynamicDelay < proto.DynamicFilterTimeoutLowerBound { if tracer != nil { - tracer.log.Infof("Calculated dynamic delay = %v for round %v, period %v. It's too low, setting to %v\n", dynamicDelay, p.Round, p.Period, proto.DynamicFilterTimeoutLowerBound) + tracer.log.Debugf("Calculated dynamic delay = %v for round %v, period %v. It's too low, setting to %v\n", dynamicDelay, p.Round, p.Period, proto.DynamicFilterTimeoutLowerBound) } dynamicDelay = proto.DynamicFilterTimeoutLowerBound } else if dynamicDelay > defaultDelay { dynamicDelay = defaultDelay tracer.log.Warnf("Calculated dynamic delay = %v for round %v, period %v. It's higher than the default %v\n", dynamicDelay, p.Round, p.Period, defaultDelay) } else if tracer != nil { - tracer.log.Infof("Calculated dynamic delay = %v for round %v, period %v.\n", dynamicDelay, p.Round, p.Period) + tracer.log.Debugf("Calculated dynamic delay = %v for round %v, period %v.\n", dynamicDelay, p.Round, p.Period) } return dynamicDelay diff --git a/agreement/service_test.go b/agreement/service_test.go index 9afc61e119..53cb2e083e 100644 --- a/agreement/service_test.go +++ b/agreement/service_test.go @@ -990,6 +990,18 @@ func TestAgreementSynchronous5_50(t *testing.T) { simulateAgreement(t, 5, 50, disabled) } +func TestAgreementSynchronousFuture5_50(t *testing.T) { + partitiontest.PartitionTest(t) + + if testing.Short() { + t.Skip("Skipping agreement integration test") + } + consensusVersion := func(r basics.Round) (protocol.ConsensusVersion, error) { + return protocol.ConsensusFuture, nil + } + simulateAgreementWithConsensusVersion(t, 5, 50, disabled, consensusVersion) +} + func TestAgreementSynchronousFuture1(t *testing.T) { partitiontest.PartitionTest(t) diff --git a/config/consensus.go b/config/consensus.go index 9fa3213c30..5ab8f6931a 100644 --- a/config/consensus.go +++ b/config/consensus.go @@ -1359,10 +1359,6 @@ func initConsensusProtocols() { v38.AgreementFilterTimeoutPeriod0 = 3000 * time.Millisecond - v38.DynamicFilterTimeout = true - v38.DynamicFilterPayloadArriavalHistory = 32 - v38.DynamicFilterTimeoutLowerBound = time.Second - Consensus[protocol.ConsensusV38] = v38 // v37 can be upgraded to v38, with an update delay of 12h: @@ -1379,6 +1375,10 @@ func initConsensusProtocols() { vFuture.LogicSigVersion = 10 // When moving this to a release, put a new higher LogicSigVersion here vFuture.EnableLogicSigCostPooling = true + vFuture.DynamicFilterTimeout = true + vFuture.DynamicFilterTimeoutLowerBound = time.Second + vFuture.DynamicFilterPayloadArriavalHistory = 32 + Consensus[protocol.ConsensusFuture] = vFuture // vAlphaX versions are an separate series of consensus parameters and versions for alphanet From 5f049d703165974f7076f86dcbf8f930005d8173 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Thu, 10 Aug 2023 18:13:33 -0400 Subject: [PATCH 076/232] enough rounds to activate dynamic lambda --- agreement/service_test.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/agreement/service_test.go b/agreement/service_test.go index 53cb2e083e..e70dd36028 100644 --- a/agreement/service_test.go +++ b/agreement/service_test.go @@ -990,16 +990,23 @@ func TestAgreementSynchronous5_50(t *testing.T) { simulateAgreement(t, 5, 50, disabled) } -func TestAgreementSynchronousFuture5_50(t *testing.T) { +func TestAgreementSynchronousFuture5_DynamicFilterRounds(t *testing.T) { partitiontest.PartitionTest(t) if testing.Short() { t.Skip("Skipping agreement integration test") } + consensusVersion := func(r basics.Round) (protocol.ConsensusVersion, error) { return protocol.ConsensusFuture, nil } - simulateAgreementWithConsensusVersion(t, 5, 50, disabled, consensusVersion) + + cfg := config.Consensus[protocol.ConsensusFuture] + if !cfg.DynamicFilterTimeout { + return + } + + simulateAgreementWithConsensusVersion(t, 5, cfg.DynamicFilterPayloadArriavalHistory+20, disabled, consensusVersion) } func TestAgreementSynchronousFuture1(t *testing.T) { From 85dba707e0da8303425ae8fa662230cfccd7de1a Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Thu, 10 Aug 2023 18:18:03 -0400 Subject: [PATCH 077/232] +msgp_gen test --- util/timers/msgp_gen_test.go | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 util/timers/msgp_gen_test.go diff --git a/util/timers/msgp_gen_test.go b/util/timers/msgp_gen_test.go new file mode 100644 index 0000000000..265bbec36f --- /dev/null +++ b/util/timers/msgp_gen_test.go @@ -0,0 +1,6 @@ +//go:build !skip_msgp_testing +// +build !skip_msgp_testing + +package timers + +// Code generated by github.com/algorand/msgp DO NOT EDIT. From 4a258eb1e6e51bc558cc365badaf5a6bdf7e55f4 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Fri, 11 Aug 2023 11:42:03 -0400 Subject: [PATCH 078/232] skip sdk comparison --- tools/x-repo-types/xrt_test.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/tools/x-repo-types/xrt_test.go b/tools/x-repo-types/xrt_test.go index 4360b432d6..6fe353cedb 100644 --- a/tools/x-repo-types/xrt_test.go +++ b/tools/x-repo-types/xrt_test.go @@ -74,12 +74,14 @@ func TestCrossRepoTypes(t *testing.T) { yType: "EvalDelta", }, { - name: "goal-v-sdk-consensus", - xPkg: "github.com/algorand/go-algorand/config", - xType: "ConsensusParams", - yPkg: "github.com/algorand/go-algorand-sdk/v2/protocol/config", - yBranch: "develop", - yType: "ConsensusParams", + name: "goal-v-sdk-consensus", + xPkg: "github.com/algorand/go-algorand/config", + xType: "ConsensusParams", + yPkg: "github.com/algorand/go-algorand-sdk/v2/protocol/config", + yBranch: "develop", + yType: "ConsensusParams", + skip: true, + skipReason: `Changes to consensus parameters happen on the go-algorand repo, and cannot be synced in a single PR with changes to the go-algorand-sdk repo`, }, { name: "goal-v-sdk-blockheader", From cbd2bbfe36695f866c5d20f916b3fa3ddd86ac1b Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Fri, 11 Aug 2023 14:36:55 -0400 Subject: [PATCH 079/232] rounds run faster now. Give the txn a little bit more time to live --- test/scripts/e2e_subs/limit-swap-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/scripts/e2e_subs/limit-swap-test.sh b/test/scripts/e2e_subs/limit-swap-test.sh index d89331300c..026d9b3609 100755 --- a/test/scripts/e2e_subs/limit-swap-test.sh +++ b/test/scripts/e2e_subs/limit-swap-test.sh @@ -42,7 +42,7 @@ echo "closeout part b, asset trader" # quick expiration, test closeout ROUND=$(goal node status | grep 'Last committed block:'|awk '{ print $4 }') -SETUP_ROUND=$((${ROUND} + 10)) +SETUP_ROUND=$((${ROUND} + 20)) TIMEOUT_ROUND=$((${SETUP_ROUND} + 1)) sed s/TMPL_ASSET/${ASSET_ID}/g < tools/teal/templates/limit-order-b.teal.tmpl | sed s/TMPL_SWAPN/137/g | sed s/TMPL_SWAPD/31337/g | sed s/TMPL_TIMEOUT/${TIMEOUT_ROUND}/g | sed s/TMPL_OWN/${ACCOUNT}/g | sed s/TMPL_FEE/100000/g | sed s/TMPL_MINTRD/10000/g > ${TEMPDIR}/limit-order-b.teal From 9c3c43b90e4929fafb886b68fce545c25185336a Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Tue, 15 Aug 2023 14:27:28 -0400 Subject: [PATCH 080/232] 95th percentile deadline --- agreement/player.go | 12 ++++++------ config/consensus.go | 13 +++++++++---- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/agreement/player.go b/agreement/player.go index 3376922b68..8500c932dc 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -286,10 +286,10 @@ func (p *player) handleWinningPayloadArrival(payload proposal, ver protocol.Cons func (p *player) resizePayloadArrivals(ver protocol.ConsensusVersion) { proto := config.Consensus[ver] - if len(p.payloadArrivals) > proto.DynamicFilterPayloadArriavalHistory { - p.payloadArrivals = p.payloadArrivals[len(p.payloadArrivals)-proto.DynamicFilterPayloadArriavalHistory:] + if len(p.payloadArrivals) > proto.DynamicFilterCredentialArrivalHistory { + p.payloadArrivals = p.payloadArrivals[len(p.payloadArrivals)-proto.DynamicFilterCredentialArrivalHistory:] } - for len(p.payloadArrivals) < proto.DynamicFilterPayloadArriavalHistory { + for len(p.payloadArrivals) < proto.DynamicFilterCredentialArrivalHistory { p.payloadArrivals = append([]time.Duration{FilterTimeout(0, ver)}, p.payloadArrivals...) } } @@ -307,17 +307,17 @@ func (p *player) calculateFilterTimeout(ver protocol.ConsensusVersion, tracer *t var dynamicDelay time.Duration defaultDelay := FilterTimeout(0, ver) - if proto.DynamicFilterPayloadArriavalHistory <= 0 { + if proto.DynamicFilterCredentialArrivalHistory <= 0 { // we don't keep any history, use the default dynamicDelay = defaultDelay - } else if proto.DynamicFilterPayloadArriavalHistory > len(p.payloadArrivals) { + } else if proto.DynamicFilterCredentialArrivalHistory > len(p.payloadArrivals) { // not enough samples, use the default dynamicDelay = defaultDelay } else { sortedArrivals := make([]time.Duration, len(p.payloadArrivals)) copy(sortedArrivals[:], p.payloadArrivals[:]) sort.Slice(sortedArrivals, func(i, j int) bool { return sortedArrivals[i] < sortedArrivals[j] }) - dynamicDelay = sortedArrivals[proto.DynamicFilterPayloadArriavalHistory-1] + dynamicDelay = sortedArrivals[proto.DynamicFilterCredentialArrivalHistoryIdx] } // Make sure the dynamic delay is not too small or too large diff --git a/config/consensus.go b/config/consensus.go index 5ab8f6931a..46f0d77a17 100644 --- a/config/consensus.go +++ b/config/consensus.go @@ -525,9 +525,11 @@ type ConsensusParams struct { // filter timeout must meet. DynamicFilterTimeoutLowerBound time.Duration - // DynamicFilterPayloadArriavalHistory specifies the number of past block arrivals - // that are measured to determine when to start pipelining the next block. - DynamicFilterPayloadArriavalHistory int + // DynamicFilterCredentialArrivalHistory specifies the number of past credential arrivals + // that are measured to determine the next filter timeout. + DynamicFilterCredentialArrivalHistory int + + DynamicFilterCredentialArrivalHistoryIdx int } // PaysetCommitType enumerates possible ways for the block header to commit to @@ -1377,7 +1379,10 @@ func initConsensusProtocols() { vFuture.DynamicFilterTimeout = true vFuture.DynamicFilterTimeoutLowerBound = time.Second - vFuture.DynamicFilterPayloadArriavalHistory = 32 + // history window of 40, so we have enough statistics to calculate the 95th + // percentile, which is the timestamp at index 37 in the history array. + vFuture.DynamicFilterCredentialArrivalHistory = 40 + vFuture.DynamicFilterCredentialArrivalHistoryIdx = 37 Consensus[protocol.ConsensusFuture] = vFuture From 60e7bf3a82131db8f23e16f040e454a7d60c2af0 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Tue, 15 Aug 2023 15:48:36 -0400 Subject: [PATCH 081/232] grace --- agreement/player.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agreement/player.go b/agreement/player.go index 8500c932dc..b57e988a63 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -317,7 +317,7 @@ func (p *player) calculateFilterTimeout(ver protocol.ConsensusVersion, tracer *t sortedArrivals := make([]time.Duration, len(p.payloadArrivals)) copy(sortedArrivals[:], p.payloadArrivals[:]) sort.Slice(sortedArrivals, func(i, j int) bool { return sortedArrivals[i] < sortedArrivals[j] }) - dynamicDelay = sortedArrivals[proto.DynamicFilterCredentialArrivalHistoryIdx] + dynamicDelay = sortedArrivals[proto.DynamicFilterCredentialArrivalHistoryIdx] + 50*time.Millisecond } // Make sure the dynamic delay is not too small or too large From 0d7ea4b6679c628a62d8b0373353ea12a35d9b02 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Wed, 16 Aug 2023 14:16:21 -0400 Subject: [PATCH 082/232] doc new parameter --- config/consensus.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config/consensus.go b/config/consensus.go index 46f0d77a17..f34ff896a6 100644 --- a/config/consensus.go +++ b/config/consensus.go @@ -529,6 +529,8 @@ type ConsensusParams struct { // that are measured to determine the next filter timeout. DynamicFilterCredentialArrivalHistory int + // DynamicFilterCredentialArrivalHistoryIdx specified which sample to use out of a sorted + // DynamicFilterCredentialArrivalHistory-sized array of time samples. DynamicFilterCredentialArrivalHistoryIdx int } From c8b59c3914aa7dbcc714d728e8803b07a3450a28 Mon Sep 17 00:00:00 2001 From: chris erway Date: Tue, 15 Aug 2023 15:51:26 -0400 Subject: [PATCH 083/232] add voteVerified ValidatedAt times --- agreement/demux.go | 2 ++ agreement/events.go | 7 ++++++- agreement/player.go | 6 ++++++ agreement/proposal.go | 2 ++ agreement/proposalStore.go | 5 +++++ agreement/vote.go | 5 +++++ 6 files changed, 26 insertions(+), 1 deletion(-) diff --git a/agreement/demux.go b/agreement/demux.go index c225396293..60b6d2d84b 100644 --- a/agreement/demux.go +++ b/agreement/demux.go @@ -202,6 +202,8 @@ func (d *demux) next(s *Service, deadline Deadline, fastDeadline Deadline, curre e = e.(messageEvent).AttachValidatedAt(s.Clock.Since()) case payloadPresent, votePresent: e = e.(messageEvent).AttachReceivedAt(s.Clock.Since()) + case voteVerified: + e = e.(messageEvent).AttachValidatedAt(s.Clock.Since()) } }() diff --git a/agreement/events.go b/agreement/events.go index 52737e5f2c..8eba33006b 100644 --- a/agreement/events.go +++ b/agreement/events.go @@ -942,7 +942,12 @@ func (e checkpointEvent) AttachConsensusVersion(v ConsensusVersionView) external } func (e messageEvent) AttachValidatedAt(d time.Duration) messageEvent { - e.Input.Proposal.validatedAt = d + if e.T == payloadVerified { + e.Input.Proposal.validatedAt = d + } else if e.T == voteVerified && e.Input.UnauthenticatedVote.R.Step == 0 { + // if this is a proposal vote (step 0), record the receivedAt time on the vote + e.Input.Vote.validatedAt = d + } return e } diff --git a/agreement/player.go b/agreement/player.go index b57e988a63..c2071ae8b1 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -279,6 +279,12 @@ func (p *player) handleCheckpointEvent(r routerHandle, e checkpointEvent) []acti } func (p *player) handleWinningPayloadArrival(payload proposal, ver protocol.ConsensusVersion) { + + // look up lowest vote validatedAt time + //re := readLowestEvent{T: readLowestVote, Round: p.Round, Period: p.Period} + //re = r.dispatch(*p, re, proposalMachineRound, p.Round, p.Period, 0).(readLowestEvent) + //p.payloadArrivals = append(p.payloadArrivals, re.Vote.validatedAt) + // ignoring validatedAt p.payloadArrivals = append(p.payloadArrivals, payload.receivedAt) p.resizePayloadArrivals(ver) diff --git a/agreement/proposal.go b/agreement/proposal.go index bf021f2cfe..f2dde4bd6a 100644 --- a/agreement/proposal.go +++ b/agreement/proposal.go @@ -99,6 +99,8 @@ type proposal struct { // validated (and thus was ready to be delivered to the state // machine), relative to the zero of that round. validatedAt time.Duration + + voteValidatedAt time.Duration } func makeProposal(ve ValidatedBlock, pf crypto.VrfProof, origPer period, origProp basics.Address) proposal { diff --git a/agreement/proposalStore.go b/agreement/proposalStore.go index 080609de50..8751fbe540 100644 --- a/agreement/proposalStore.go +++ b/agreement/proposalStore.go @@ -272,6 +272,11 @@ func (store *proposalStore) handle(r routerHandle, p player, e event) event { a := stagedValue(p, r, p.Round, p.Period) authVote := ea.authenticator(p.Period) + + // attach vote receivedAt to payload + //ea = store.Assemblers[pv] + //ea.Payload.voteValidatedAt = authVote.validatedAt + if a.Proposal == pv { return committableEvent{Proposal: pv, Vote: authVote} } diff --git a/agreement/vote.go b/agreement/vote.go index af157ee3bb..bed20fe88f 100644 --- a/agreement/vote.go +++ b/agreement/vote.go @@ -18,6 +18,7 @@ package agreement import ( "fmt" + "time" "github.com/algorand/go-algorand/crypto" "github.com/algorand/go-algorand/data/basics" @@ -51,6 +52,10 @@ type ( R rawVote `codec:"r"` Cred committee.Credential `codec:"cred"` Sig crypto.OneTimeSignature `codec:"sig,omitempty,omitemptycheckstruct"` + + // validatedAt indicates the time at which this vote was verified (as a voteVerified messageEvent), + // relative to the zero of that round. It is only set for step 0. + validatedAt time.Duration } // unauthenticatedEquivocationVote is a pair of votes which has not From 1c60257f680a4b05f713ecbde5389a6cba15df96 Mon Sep 17 00:00:00 2001 From: chris erway Date: Wed, 16 Aug 2023 10:39:42 -0400 Subject: [PATCH 084/232] add readLowestEvent from 63c03b907764a583132c55e255d3816d2aad49f8 (orig. from feature/pipeline) --- agreement/events.go | 40 ++++++++++++++++++++++++++++++++++++ agreement/proposalStore.go | 9 ++++++++ agreement/proposalTracker.go | 5 +++++ 3 files changed, 54 insertions(+) diff --git a/agreement/events.go b/agreement/events.go index 8eba33006b..e9fdf6eda2 100644 --- a/agreement/events.go +++ b/agreement/events.go @@ -197,6 +197,14 @@ const ( // readPinned is sent to the proposalStore to read the pinned value, if it exists. readPinned + // readLowestValue is sent to the proposalPeriodMachine to read the + // proposal-vote with the lowest credential. + readLowestValue + + // readLowestPayload is sent to the proposalStore to read the payload + // corresponding to the lowest-credential proposal-vote, if it exists. + readLowestPayload + /* * The following are event types that replace queries, and may warrant * a revision to make them more state-machine-esque. @@ -407,6 +415,38 @@ func (e newRoundEvent) ComparableStr() string { return e.String() } +type readLowestEvent struct { + // T is either readLowestValue or readLowestPayload + T eventType + + // Round and Period are the round and period for which to query + // the lowest-credential value and payload. This type of event + // is only sent for pipelining, which only makes sense for period + // 0, but the Period is here anyway to route to the appropriate + // proposalMachinePeriod. + Round round + Period period + + // Proposal holds the lowest-credential value. + Proposal proposalValue + // Payload holds the payload, if one exists (which is the case if PayloadOK is set). + Payload proposal + // PayloadOK is set if and only if a payload was received for the lowest-credential value. + PayloadOK bool +} + +func (e readLowestEvent) t() eventType { + return e.T +} + +func (e readLowestEvent) String() string { + return fmt.Sprintf("%v: %.5v", e.t().String(), e.Proposal.BlockDigest.String()) +} + +func (e readLowestEvent) ComparableStr() string { + return e.String() +} + type newPeriodEvent struct { // Period holds the latest period relevant to the proposalRoundMachine. Period period diff --git a/agreement/proposalStore.go b/agreement/proposalStore.go index 8751fbe540..6beaded2ec 100644 --- a/agreement/proposalStore.go +++ b/agreement/proposalStore.go @@ -357,6 +357,15 @@ func (store *proposalStore) handle(r routerHandle, p player, e event) event { se.Committable = ea.Assembled se.Payload = ea.Payload return se + case readLowestPayload: + re := e.(readLowestEvent) + re.T = readLowestValue + re = r.dispatch(p, re, proposalMachinePeriod, re.Round, re.Period, 0).(readLowestEvent) + re.T = readLowestPayload + ea := store.Assemblers[re.Proposal] + re.PayloadOK = ea.Assembled + re.Payload = ea.Payload + return re case readPinned: se := e.(pinnedValueEvent) ea := store.Assemblers[store.Pinned] // If pinned is bottom, assembled/payloadOK = false, payload = bottom diff --git a/agreement/proposalTracker.go b/agreement/proposalTracker.go index 59ffb77a28..7627cdd28d 100644 --- a/agreement/proposalTracker.go +++ b/agreement/proposalTracker.go @@ -165,6 +165,11 @@ func (t *proposalTracker) handle(r routerHandle, p player, e event) event { t.Freezer = t.Freezer.freeze() return e + case readLowestValue: + e := e.(readLowestEvent) + e.Proposal = t.Freezer.Lowest.R.Proposal + return e + case softThreshold, certThreshold: e := e.(thresholdEvent) t.Staging = e.Proposal From fbb42345acd13ef75eb832856e2b7adec09ea49e Mon Sep 17 00:00:00 2001 From: chris erway Date: Tue, 15 Aug 2023 15:40:15 -0400 Subject: [PATCH 085/232] add readLowestVote --- agreement/events.go | 8 +++++++- agreement/eventtype_string.go | 27 +++++++++++++++------------ agreement/proposalStore.go | 3 +++ agreement/proposalTracker.go | 5 +++++ 4 files changed, 30 insertions(+), 13 deletions(-) diff --git a/agreement/events.go b/agreement/events.go index e9fdf6eda2..73a789e2fa 100644 --- a/agreement/events.go +++ b/agreement/events.go @@ -198,9 +198,13 @@ const ( readPinned // readLowestValue is sent to the proposalPeriodMachine to read the - // proposal-vote with the lowest credential. + // proposalValue with the lowest credential. readLowestValue + // readLowestVote is sent to the proposalPeriodMachine to read the + // proposal-vote with the lowest credential. + readLowestVote + // readLowestPayload is sent to the proposalStore to read the payload // corresponding to the lowest-credential proposal-vote, if it exists. readLowestPayload @@ -429,6 +433,8 @@ type readLowestEvent struct { // Proposal holds the lowest-credential value. Proposal proposalValue + // Vote holds the lowest-credential vote. + Vote vote // Payload holds the payload, if one exists (which is the case if PayloadOK is set). Payload proposal // PayloadOK is set if and only if a payload was received for the lowest-credential value. diff --git a/agreement/eventtype_string.go b/agreement/eventtype_string.go index 9da84c1b98..471d93f36e 100644 --- a/agreement/eventtype_string.go +++ b/agreement/eventtype_string.go @@ -37,21 +37,24 @@ func _() { _ = x[newPeriod-26] _ = x[readStaging-27] _ = x[readPinned-28] - _ = x[voteFilterRequest-29] - _ = x[voteFilteredStep-30] - _ = x[nextThresholdStatusRequest-31] - _ = x[nextThresholdStatus-32] - _ = x[freshestBundleRequest-33] - _ = x[freshestBundle-34] - _ = x[dumpVotesRequest-35] - _ = x[dumpVotes-36] - _ = x[wrappedAction-37] - _ = x[checkpointReached-38] + _ = x[readLowestValue-29] + _ = x[readLowestVote-30] + _ = x[readLowestPayload-31] + _ = x[voteFilterRequest-32] + _ = x[voteFilteredStep-33] + _ = x[nextThresholdStatusRequest-34] + _ = x[nextThresholdStatus-35] + _ = x[freshestBundleRequest-36] + _ = x[freshestBundle-37] + _ = x[dumpVotesRequest-38] + _ = x[dumpVotes-39] + _ = x[wrappedAction-40] + _ = x[checkpointReached-41] } -const _eventType_name = "nonevotePresentpayloadPresentbundlePresentvoteVerifiedpayloadVerifiedbundleVerifiedroundInterruptiontimeoutfastTimeoutsoftThresholdcertThresholdnextThresholdproposalCommittableproposalAcceptedvoteFilteredvoteMalformedbundleFilteredbundleMalformedpayloadRejectedpayloadMalformedpayloadPipelinedpayloadAcceptedproposalFrozenvoteAcceptednewRoundnewPeriodreadStagingreadPinnedvoteFilterRequestvoteFilteredStepnextThresholdStatusRequestnextThresholdStatusfreshestBundleRequestfreshestBundledumpVotesRequestdumpVoteswrappedActioncheckpointReached" +const _eventType_name = "nonevotePresentpayloadPresentbundlePresentvoteVerifiedpayloadVerifiedbundleVerifiedroundInterruptiontimeoutfastTimeoutsoftThresholdcertThresholdnextThresholdproposalCommittableproposalAcceptedvoteFilteredvoteMalformedbundleFilteredbundleMalformedpayloadRejectedpayloadMalformedpayloadPipelinedpayloadAcceptedproposalFrozenvoteAcceptednewRoundnewPeriodreadStagingreadPinnedreadLowestValuereadLowestVotereadLowestPayloadvoteFilterRequestvoteFilteredStepnextThresholdStatusRequestnextThresholdStatusfreshestBundleRequestfreshestBundledumpVotesRequestdumpVoteswrappedActioncheckpointReached" -var _eventType_index = [...]uint16{0, 4, 15, 29, 42, 54, 69, 83, 100, 107, 118, 131, 144, 157, 176, 192, 204, 217, 231, 246, 261, 277, 293, 308, 322, 334, 342, 351, 362, 372, 389, 405, 431, 450, 471, 485, 501, 510, 523, 540} +var _eventType_index = [...]uint16{0, 4, 15, 29, 42, 54, 69, 83, 100, 107, 118, 131, 144, 157, 176, 192, 204, 217, 231, 246, 261, 277, 293, 308, 322, 334, 342, 351, 362, 372, 387, 401, 418, 435, 451, 477, 496, 517, 531, 547, 556, 569, 586} func (i eventType) String() string { if i >= eventType(len(_eventType_index)-1) { diff --git a/agreement/proposalStore.go b/agreement/proposalStore.go index 6beaded2ec..e53d6096cf 100644 --- a/agreement/proposalStore.go +++ b/agreement/proposalStore.go @@ -366,6 +366,9 @@ func (store *proposalStore) handle(r routerHandle, p player, e event) event { re.PayloadOK = ea.Assembled re.Payload = ea.Payload return re + case readLowestVote: + re := e.(readLowestEvent) + return r.dispatch(p, re, proposalMachinePeriod, re.Round, re.Period, 0).(readLowestEvent) case readPinned: se := e.(pinnedValueEvent) ea := store.Assemblers[store.Pinned] // If pinned is bottom, assembled/payloadOK = false, payload = bottom diff --git a/agreement/proposalTracker.go b/agreement/proposalTracker.go index 7627cdd28d..d56fcb31a8 100644 --- a/agreement/proposalTracker.go +++ b/agreement/proposalTracker.go @@ -170,6 +170,11 @@ func (t *proposalTracker) handle(r routerHandle, p player, e event) event { e.Proposal = t.Freezer.Lowest.R.Proposal return e + case readLowestVote: + e := e.(readLowestEvent) + e.Vote = t.Freezer.Lowest + return e + case softThreshold, certThreshold: e := e.(thresholdEvent) t.Staging = e.Proposal From f3aa45057789df7a6c2a991173baa48eefaa9660 Mon Sep 17 00:00:00 2001 From: chris erway Date: Wed, 16 Aug 2023 11:28:29 -0400 Subject: [PATCH 086/232] update handleWinningPayloadArrival and hook up to player tests --- agreement/player.go | 22 +++++++++++----------- agreement/player_test.go | 28 +++++++++++++++++++++------- agreement/proposalTrackerContract.go | 2 +- 3 files changed, 33 insertions(+), 19 deletions(-) diff --git a/agreement/player.go b/agreement/player.go index c2071ae8b1..6a644bc812 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -278,15 +278,15 @@ func (p *player) handleCheckpointEvent(r routerHandle, e checkpointEvent) []acti }} } -func (p *player) handleWinningPayloadArrival(payload proposal, ver protocol.ConsensusVersion) { - - // look up lowest vote validatedAt time - //re := readLowestEvent{T: readLowestVote, Round: p.Round, Period: p.Period} - //re = r.dispatch(*p, re, proposalMachineRound, p.Round, p.Period, 0).(readLowestEvent) - //p.payloadArrivals = append(p.payloadArrivals, re.Vote.validatedAt) - - // ignoring validatedAt - p.payloadArrivals = append(p.payloadArrivals, payload.receivedAt) +// updateDynamicLambdaTimings is called at the end of the round (just before ensureAction +// is generated) to collect round timings for updating dynamic lambda. +func (p *player) updateDynamicLambdaTimings(r routerHandle, payload proposal, ver protocol.ConsensusVersion) { + // look up the validatedAt time of the winning proposal-vote + re := readLowestEvent{T: readLowestVote, Round: p.Round, Period: p.Period} + re = r.dispatch(*p, re, proposalMachineRound, p.Round, p.Period, 0).(readLowestEvent) + if re.Vote.validatedAt != 0 { + p.payloadArrivals = append(p.payloadArrivals, re.Vote.validatedAt) + } p.resizePayloadArrivals(ver) } @@ -357,7 +357,7 @@ func (p *player) handleThresholdEvent(r routerHandle, e thresholdEvent) []action cert := Certificate(e.Bundle) a0 := ensureAction{Payload: res.Payload, Certificate: cert} actions = append(actions, a0) - p.handleWinningPayloadArrival(res.Payload, e.Proto) + p.updateDynamicLambdaTimings(r, res.Payload, e.Proto) as := p.enterRound(r, e, p.Round+1) return append(actions, as...) } @@ -677,7 +677,7 @@ func (p *player) handleMessageEvent(r routerHandle, e messageEvent) (actions []a cert := Certificate(freshestRes.Event.Bundle) a0 := ensureAction{Payload: e.Input.Proposal, Certificate: cert} actions = append(actions, a0) - p.handleWinningPayloadArrival(e.Input.Proposal, e.Proto.Version) + p.updateDynamicLambdaTimings(r, e.Input.Proposal, e.Proto.Version) as := p.enterRound(r, delegatedE, cert.Round+1) return append(actions, as...) } diff --git a/agreement/player_test.go b/agreement/player_test.go index 84aad574ba..df3878a2a9 100644 --- a/agreement/player_test.go +++ b/agreement/player_test.go @@ -3247,6 +3247,7 @@ func TestPlayerRetainsReceivedValidatedAt(t *testing.T) { // send voteVerified message vVote := helper.MakeVerifiedVote(t, 0, r-1, p, propose, *pV) inMsg := messageEvent{T: voteVerified, Input: message{Vote: vVote, UnauthenticatedVote: vVote.u()}} + inMsg = inMsg.AttachValidatedAt(501 * time.Millisecond) err, panicErr := pM.transition(inMsg) require.NoError(t, err) require.NoError(t, panicErr) @@ -3259,7 +3260,13 @@ func TestPlayerRetainsReceivedValidatedAt(t *testing.T) { require.NoError(t, err) require.NoError(t, panicErr) - assertCorrectReceivedAtSet(t, pWhite, pM, helper, r, p, pP, pV, m) + assertCorrectReceivedAtSet(t, pWhite, pM, helper, r, p, pP, pV, m, protocol.ConsensusFuture) + + // assert lowest vote validateAt time was recorded into payloadArrivals + historyLen := config.Consensus[protocol.ConsensusFuture].DynamicFilterPayloadArriavalHistory + require.NotZero(t, historyLen) + require.Len(t, pWhite.payloadArrivals, historyLen) + require.Equal(t, 501*time.Millisecond, pWhite.payloadArrivals[historyLen-1]) } // test that ReceivedAt and ValidateAt timing information are retained in proposalStore @@ -3291,29 +3298,36 @@ func TestPlayerRetainsReceivedValidatedAtPP(t *testing.T) { // send voteVerified inMsg = messageEvent{T: voteVerified, Input: voteMsg, TaskIndex: 1} + inMsg = inMsg.AttachValidatedAt(502 * time.Millisecond) err, panicErr = pM.transition(inMsg) require.NoError(t, err) require.NoError(t, panicErr) - assertCorrectReceivedAtSet(t, pWhite, pM, helper, r, p, pP, pV, proposalMsg) + assertCorrectReceivedAtSet(t, pWhite, pM, helper, r, p, pP, pV, proposalMsg, protocol.ConsensusFuture) + + // assert lowest vote validateAt time was recorded into payloadArrivals + historyLen := config.Consensus[protocol.ConsensusFuture].DynamicFilterPayloadArriavalHistory + require.NotZero(t, historyLen) + require.Len(t, pWhite.payloadArrivals, historyLen) + require.Equal(t, 502*time.Millisecond, pWhite.payloadArrivals[historyLen-1]) } func assertCorrectReceivedAtSet(t *testing.T, pWhite *player, pM ioAutomata, helper *voteMakerHelper, - r round, p period, pP *proposal, pV *proposalValue, m message) { + r round, p period, pP *proposal, pV *proposalValue, m message, ver protocol.ConsensusVersion) { // make sure payload verify request verifyEvent := ev(cryptoAction{T: verifyPayload, M: m, Round: r - 1, Period: p, Step: propose, TaskIndex: 0}) require.Truef(t, pM.getTrace().Contains(verifyEvent), "Player should verify payload") // payloadVerified - inMsg := messageEvent{T: payloadVerified, Input: message{Proposal: *pP}, Proto: ConsensusVersionView{Version: protocol.ConsensusCurrentVersion}} + inMsg := messageEvent{T: payloadVerified, Input: message{Proposal: *pP}, Proto: ConsensusVersionView{Version: ver}} inMsg = inMsg.AttachValidatedAt(2 * time.Second) // call AttachValidatedAt like demux would err, panicErr := pM.transition(inMsg) require.NoError(t, err) require.NoError(t, panicErr) // gen cert to move into the next round - votes := make([]vote, int(cert.threshold(config.Consensus[protocol.ConsensusCurrentVersion]))) - for i := 0; i < int(cert.threshold(config.Consensus[protocol.ConsensusCurrentVersion])); i++ { + votes := make([]vote, int(cert.threshold(config.Consensus[ver]))) + for i := 0; i < int(cert.threshold(config.Consensus[ver])); i++ { votes[i] = helper.MakeVerifiedVote(t, i, r-1, p, cert, *pV) } bun := unauthenticatedBundle{ @@ -3330,7 +3344,7 @@ func assertCorrectReceivedAtSet(t *testing.T, pWhite *player, pM ioAutomata, hel }, UnauthenticatedBundle: bun, }, - Proto: ConsensusVersionView{Version: protocol.ConsensusCurrentVersion}, + Proto: ConsensusVersionView{Version: ver}, } err, panicErr = pM.transition(inMsg) require.NoError(t, err) diff --git a/agreement/proposalTrackerContract.go b/agreement/proposalTrackerContract.go index 2b995dfcac..bfc39170e8 100644 --- a/agreement/proposalTrackerContract.go +++ b/agreement/proposalTrackerContract.go @@ -32,7 +32,7 @@ type proposalTrackerContract struct { // TODO check concrete types of events func (c *proposalTrackerContract) pre(p player, in event) (pre []error) { switch in.t() { - case voteVerified, proposalFrozen, softThreshold, certThreshold, voteFilterRequest, readStaging: + case voteVerified, proposalFrozen, softThreshold, certThreshold, voteFilterRequest, readStaging, readLowestValue, readLowestPayload, readLowestVote: default: pre = append(pre, fmt.Errorf("incoming event has invalid type: %v", in.t())) } From 822e946452571a4e20b995818efc6658a1c4d871 Mon Sep 17 00:00:00 2001 From: chris erway Date: Wed, 16 Aug 2023 11:38:42 -0400 Subject: [PATCH 087/232] rerun msgp --- agreement/msgp_gen.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/agreement/msgp_gen.go b/agreement/msgp_gen.go index bd04d34e0b..77d2cceed6 100644 --- a/agreement/msgp_gen.go +++ b/agreement/msgp_gen.go @@ -4223,7 +4223,7 @@ func (z *proposal) MarshalMsg(b []byte) (o []byte) { o = msgp.Require(b, z.Msgsize()) // omitempty: check for empty values zb0004Len := uint32(29) - var zb0004Mask uint64 /* 38 bits */ + var zb0004Mask uint64 /* 39 bits */ if (*z).unauthenticatedProposal.Block.BlockHeader.RewardsState.RewardsLevel == 0 { zb0004Len-- zb0004Mask |= 0x40 @@ -11156,7 +11156,7 @@ func (z *vote) MarshalMsg(b []byte) (o []byte) { o = msgp.Require(b, z.Msgsize()) // omitempty: check for empty values zb0001Len := uint32(3) - var zb0001Mask uint8 /* 4 bits */ + var zb0001Mask uint8 /* 5 bits */ if (*z).Cred.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x2 From a9e9bfce5fb4f08c8f1e6455bbc575a90edba596 Mon Sep 17 00:00:00 2001 From: cce <51567+cce@users.noreply.github.com> Date: Wed, 16 Aug 2023 12:13:03 -0400 Subject: [PATCH 088/232] Update proposalStore.go remove commented out code from when we were trying to get receivedAt for proposal-votes --- agreement/proposalStore.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/agreement/proposalStore.go b/agreement/proposalStore.go index e53d6096cf..8f89f4fc73 100644 --- a/agreement/proposalStore.go +++ b/agreement/proposalStore.go @@ -272,11 +272,6 @@ func (store *proposalStore) handle(r routerHandle, p player, e event) event { a := stagedValue(p, r, p.Round, p.Period) authVote := ea.authenticator(p.Period) - - // attach vote receivedAt to payload - //ea = store.Assemblers[pv] - //ea.Payload.voteValidatedAt = authVote.validatedAt - if a.Proposal == pv { return committableEvent{Proposal: pv, Vote: authVote} } From 97fcbc2f821dbf91620ec58167c41b952247c560 Mon Sep 17 00:00:00 2001 From: cce <51567+cce@users.noreply.github.com> Date: Wed, 16 Aug 2023 12:14:06 -0400 Subject: [PATCH 089/232] Update events.go fix comment to say validated instead of received --- agreement/events.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agreement/events.go b/agreement/events.go index 73a789e2fa..0458e9e9fa 100644 --- a/agreement/events.go +++ b/agreement/events.go @@ -991,7 +991,7 @@ func (e messageEvent) AttachValidatedAt(d time.Duration) messageEvent { if e.T == payloadVerified { e.Input.Proposal.validatedAt = d } else if e.T == voteVerified && e.Input.UnauthenticatedVote.R.Step == 0 { - // if this is a proposal vote (step 0), record the receivedAt time on the vote + // if this is a proposal vote (step 0), record the validatedAt time on the vote e.Input.Vote.validatedAt = d } return e From 8681dcad4b1407578d7a670786e387cfa0f43bac Mon Sep 17 00:00:00 2001 From: chris erway Date: Wed, 16 Aug 2023 14:33:22 -0400 Subject: [PATCH 090/232] add TestPlayerRetainsReceivedValidatedAtAVPP and comments --- agreement/player.go | 2 +- agreement/player_test.go | 70 +++++++++++++++++++++++++++++---- agreement/state_machine_test.go | 9 +++++ 3 files changed, 73 insertions(+), 8 deletions(-) diff --git a/agreement/player.go b/agreement/player.go index 6a644bc812..1e285471a2 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -278,7 +278,7 @@ func (p *player) handleCheckpointEvent(r routerHandle, e checkpointEvent) []acti }} } -// updateDynamicLambdaTimings is called at the end of the round (just before ensureAction +// updateDynamicLambdaTimings is called at the end of a successful uninterrupted round (just after ensureAction // is generated) to collect round timings for updating dynamic lambda. func (p *player) updateDynamicLambdaTimings(r routerHandle, payload proposal, ver protocol.ConsensusVersion) { // look up the validatedAt time of the winning proposal-vote diff --git a/agreement/player_test.go b/agreement/player_test.go index df3878a2a9..24c7da3e01 100644 --- a/agreement/player_test.go +++ b/agreement/player_test.go @@ -3234,7 +3234,7 @@ func TestPlayerAlwaysResynchsPinnedValue(t *testing.T) { } // test that ReceivedAt and ValidateAt timing information are retained in proposalStore -// when the payloadPresent and payloadVerified events are processed, and that both timings +// when the payloadPresent, payloadVerified, and voteVerified events are processed, and that all timings // are available when the ensureAction is called for the block. func TestPlayerRetainsReceivedValidatedAt(t *testing.T) { partitiontest.PartitionTest(t) @@ -3270,8 +3270,8 @@ func TestPlayerRetainsReceivedValidatedAt(t *testing.T) { } // test that ReceivedAt and ValidateAt timing information are retained in proposalStore -// when the payloadPresent (as part of the CompoundMessage encoding used by PP messages) -// and payloadVerified events are processed, and that both timings +// when the payloadPresent (as part of the CompoundMessage encoding used by PP messages), +// payloadVerified, and voteVerified events are processed, and that all timings // are available when the ensureAction is called for the block. func TestPlayerRetainsReceivedValidatedAtPP(t *testing.T) { partitiontest.PartitionTest(t) @@ -3283,9 +3283,9 @@ func TestPlayerRetainsReceivedValidatedAtPP(t *testing.T) { // create a PP message for an arbitrary proposal/payload similar to setupCompoundMessage vVote := helper.MakeVerifiedVote(t, 0, r-1, p, propose, *pV) - voteMsg := message{Vote: vVote, UnauthenticatedVote: vVote.u()} + unverifiedVoteMsg := message{UnauthenticatedVote: vVote.u()} proposalMsg := message{UnauthenticatedProposal: pP.u()} - compoundMsg := messageEvent{T: votePresent, Input: voteMsg, + compoundMsg := messageEvent{T: votePresent, Input: unverifiedVoteMsg, Tail: &messageEvent{T: payloadPresent, Input: proposalMsg}} inMsg := compoundMsg.AttachReceivedAt(time.Second) // call AttachReceivedAt like demux would err, panicErr := pM.transition(inMsg) @@ -3293,11 +3293,12 @@ func TestPlayerRetainsReceivedValidatedAtPP(t *testing.T) { require.NoError(t, panicErr) // make sure vote verify requests - verifyEvent := ev(cryptoAction{T: verifyVote, M: voteMsg, Round: r - 1, Period: p, Step: propose, TaskIndex: 1}) + verifyEvent := ev(cryptoAction{T: verifyVote, M: unverifiedVoteMsg, Round: r - 1, Period: p, Step: propose, TaskIndex: 1}) require.Truef(t, pM.getTrace().Contains(verifyEvent), "Player should verify vote") // send voteVerified - inMsg = messageEvent{T: voteVerified, Input: voteMsg, TaskIndex: 1} + verifiedVoteMsg := message{Vote: vVote, UnauthenticatedVote: vVote.u()} + inMsg = messageEvent{T: voteVerified, Input: verifiedVoteMsg, TaskIndex: 1} inMsg = inMsg.AttachValidatedAt(502 * time.Millisecond) err, panicErr = pM.transition(inMsg) require.NoError(t, err) @@ -3312,6 +3313,61 @@ func TestPlayerRetainsReceivedValidatedAtPP(t *testing.T) { require.Equal(t, 502*time.Millisecond, pWhite.payloadArrivals[historyLen-1]) } +// test that ReceivedAt and ValidateAt timing information are retained in proposalStore +// when the voteVerified event comes in first (as part of the AV message before PP), +// then the payloadPresent (as part of the CompoundMessage encoding used by PP messages) +// and payloadVerified events are processed, and that all timings +// are available when the ensureAction is called for the block. +func TestPlayerRetainsReceivedValidatedAtAVPP(t *testing.T) { + partitiontest.PartitionTest(t) + + const r = round(20239) + const p = period(1001) + pWhite, pM, helper := setupP(t, r-1, p, soft) + pP, pV := helper.MakeRandomProposalPayload(t, r-1) + + // send votePresent message (mimicking the first AV message validating) + vVote := helper.MakeVerifiedVote(t, 0, r-1, p, propose, *pV) + unverifiedVoteMsg := message{UnauthenticatedVote: vVote.u()} + inMsg := messageEvent{T: votePresent, Input: unverifiedVoteMsg} + err, panicErr := pM.transition(inMsg) + require.NoError(t, err) + require.NoError(t, panicErr) + + // make sure vote verify requests + verifyEvent := ev(cryptoAction{T: verifyVote, M: unverifiedVoteMsg, Round: r - 1, Period: p, Step: propose, TaskIndex: 1}) + require.Truef(t, pM.getTrace().Contains(verifyEvent), "Player should verify vote") + + // send voteVerified + verifiedVoteMsg := message{Vote: vVote, UnauthenticatedVote: vVote.u()} + inMsg = messageEvent{T: voteVerified, Input: verifiedVoteMsg, TaskIndex: 1} + inMsg = inMsg.AttachValidatedAt(502 * time.Millisecond) + err, panicErr = pM.transition(inMsg) + require.NoError(t, err) + require.NoError(t, panicErr) + + // create a PP message for an arbitrary proposal/payload similar to setupCompoundMessage + proposalMsg := message{UnauthenticatedProposal: pP.u()} + compoundMsg := messageEvent{T: votePresent, Input: unverifiedVoteMsg, + Tail: &messageEvent{T: payloadPresent, Input: proposalMsg}} + inMsg = compoundMsg.AttachReceivedAt(time.Second) // call AttachReceivedAt like demux would + err, panicErr = pM.transition(inMsg) + require.NoError(t, err) + require.NoError(t, panicErr) + + // make sure no second request to verify this vote + verifyEvent = ev(cryptoAction{T: verifyVote, M: unverifiedVoteMsg, Round: r - 1, Period: p, Step: propose, TaskIndex: 1}) + require.Equal(t, 1, pM.getTrace().CountEvent(verifyEvent), "Player should not verify second vote") + + assertCorrectReceivedAtSet(t, pWhite, pM, helper, r, p, pP, pV, proposalMsg, protocol.ConsensusFuture) + + // assert lowest vote validateAt time was recorded into payloadArrivals + historyLen := config.Consensus[protocol.ConsensusFuture].DynamicFilterPayloadArriavalHistory + require.NotZero(t, historyLen) + require.Len(t, pWhite.payloadArrivals, historyLen) + require.Equal(t, 502*time.Millisecond, pWhite.payloadArrivals[historyLen-1]) +} + func assertCorrectReceivedAtSet(t *testing.T, pWhite *player, pM ioAutomata, helper *voteMakerHelper, r round, p period, pP *proposal, pV *proposalValue, m message, ver protocol.ConsensusVersion) { // make sure payload verify request diff --git a/agreement/state_machine_test.go b/agreement/state_machine_test.go index 0effd9fda1..69b31a9866 100644 --- a/agreement/state_machine_test.go +++ b/agreement/state_machine_test.go @@ -140,6 +140,15 @@ func (t ioTrace) Contains(e event) bool { }) } +func (t ioTrace) CountEvent(b event) (count int) { + for _, e := range t.events { + if e.ComparableStr() == b.ComparableStr() { + count++ + } + } + return +} + // for each event, passes it into the given fn; if returns true, returns true. func (t ioTrace) ContainsFn(compareFn func(b event) bool) bool { for _, ev := range t.events { From ffc75d226b86a730e39315fc524ebb0931cfd98e Mon Sep 17 00:00:00 2001 From: chris erway Date: Wed, 16 Aug 2023 15:05:28 -0400 Subject: [PATCH 091/232] remove readLowestValue and readLowestPayload --- agreement/demux.go | 5 ++++- agreement/events.go | 22 ++++------------------ agreement/eventtype_string.go | 28 +++++++++++++--------------- agreement/proposalStore.go | 9 --------- agreement/proposalTracker.go | 5 ----- agreement/proposalTrackerContract.go | 2 +- 6 files changed, 22 insertions(+), 49 deletions(-) diff --git a/agreement/demux.go b/agreement/demux.go index 60b6d2d84b..cacdaf82d0 100644 --- a/agreement/demux.go +++ b/agreement/demux.go @@ -203,7 +203,10 @@ func (d *demux) next(s *Service, deadline Deadline, fastDeadline Deadline, curre case payloadPresent, votePresent: e = e.(messageEvent).AttachReceivedAt(s.Clock.Since()) case voteVerified: - e = e.(messageEvent).AttachValidatedAt(s.Clock.Since()) + // if this is a proposal vote (step 0), record the validatedAt time on the vote + if e.(messageEvent).Input.UnauthenticatedVote.R.Step == 0 { + e = e.(messageEvent).AttachValidatedAt(s.Clock.Since()) + } } }() diff --git a/agreement/events.go b/agreement/events.go index 0458e9e9fa..7f79efc198 100644 --- a/agreement/events.go +++ b/agreement/events.go @@ -197,18 +197,10 @@ const ( // readPinned is sent to the proposalStore to read the pinned value, if it exists. readPinned - // readLowestValue is sent to the proposalPeriodMachine to read the - // proposalValue with the lowest credential. - readLowestValue - // readLowestVote is sent to the proposalPeriodMachine to read the // proposal-vote with the lowest credential. readLowestVote - // readLowestPayload is sent to the proposalStore to read the payload - // corresponding to the lowest-credential proposal-vote, if it exists. - readLowestPayload - /* * The following are event types that replace queries, and may warrant * a revision to make them more state-machine-esque. @@ -431,14 +423,8 @@ type readLowestEvent struct { Round round Period period - // Proposal holds the lowest-credential value. - Proposal proposalValue // Vote holds the lowest-credential vote. Vote vote - // Payload holds the payload, if one exists (which is the case if PayloadOK is set). - Payload proposal - // PayloadOK is set if and only if a payload was received for the lowest-credential value. - PayloadOK bool } func (e readLowestEvent) t() eventType { @@ -446,7 +432,7 @@ func (e readLowestEvent) t() eventType { } func (e readLowestEvent) String() string { - return fmt.Sprintf("%v: %.5v", e.t().String(), e.Proposal.BlockDigest.String()) + return fmt.Sprintf("%s: %d\t%.10s\t%.5s", e.t().String(), e.Round, e.Period) } func (e readLowestEvent) ComparableStr() string { @@ -988,10 +974,10 @@ func (e checkpointEvent) AttachConsensusVersion(v ConsensusVersionView) external } func (e messageEvent) AttachValidatedAt(d time.Duration) messageEvent { - if e.T == payloadVerified { + switch e.T { + case payloadVerified: e.Input.Proposal.validatedAt = d - } else if e.T == voteVerified && e.Input.UnauthenticatedVote.R.Step == 0 { - // if this is a proposal vote (step 0), record the validatedAt time on the vote + case voteVerified: e.Input.Vote.validatedAt = d } return e diff --git a/agreement/eventtype_string.go b/agreement/eventtype_string.go index 471d93f36e..9973215b12 100644 --- a/agreement/eventtype_string.go +++ b/agreement/eventtype_string.go @@ -37,24 +37,22 @@ func _() { _ = x[newPeriod-26] _ = x[readStaging-27] _ = x[readPinned-28] - _ = x[readLowestValue-29] - _ = x[readLowestVote-30] - _ = x[readLowestPayload-31] - _ = x[voteFilterRequest-32] - _ = x[voteFilteredStep-33] - _ = x[nextThresholdStatusRequest-34] - _ = x[nextThresholdStatus-35] - _ = x[freshestBundleRequest-36] - _ = x[freshestBundle-37] - _ = x[dumpVotesRequest-38] - _ = x[dumpVotes-39] - _ = x[wrappedAction-40] - _ = x[checkpointReached-41] + _ = x[readLowestVote-29] + _ = x[voteFilterRequest-30] + _ = x[voteFilteredStep-31] + _ = x[nextThresholdStatusRequest-32] + _ = x[nextThresholdStatus-33] + _ = x[freshestBundleRequest-34] + _ = x[freshestBundle-35] + _ = x[dumpVotesRequest-36] + _ = x[dumpVotes-37] + _ = x[wrappedAction-38] + _ = x[checkpointReached-39] } -const _eventType_name = "nonevotePresentpayloadPresentbundlePresentvoteVerifiedpayloadVerifiedbundleVerifiedroundInterruptiontimeoutfastTimeoutsoftThresholdcertThresholdnextThresholdproposalCommittableproposalAcceptedvoteFilteredvoteMalformedbundleFilteredbundleMalformedpayloadRejectedpayloadMalformedpayloadPipelinedpayloadAcceptedproposalFrozenvoteAcceptednewRoundnewPeriodreadStagingreadPinnedreadLowestValuereadLowestVotereadLowestPayloadvoteFilterRequestvoteFilteredStepnextThresholdStatusRequestnextThresholdStatusfreshestBundleRequestfreshestBundledumpVotesRequestdumpVoteswrappedActioncheckpointReached" +const _eventType_name = "nonevotePresentpayloadPresentbundlePresentvoteVerifiedpayloadVerifiedbundleVerifiedroundInterruptiontimeoutfastTimeoutsoftThresholdcertThresholdnextThresholdproposalCommittableproposalAcceptedvoteFilteredvoteMalformedbundleFilteredbundleMalformedpayloadRejectedpayloadMalformedpayloadPipelinedpayloadAcceptedproposalFrozenvoteAcceptednewRoundnewPeriodreadStagingreadPinnedreadLowestVotevoteFilterRequestvoteFilteredStepnextThresholdStatusRequestnextThresholdStatusfreshestBundleRequestfreshestBundledumpVotesRequestdumpVoteswrappedActioncheckpointReached" -var _eventType_index = [...]uint16{0, 4, 15, 29, 42, 54, 69, 83, 100, 107, 118, 131, 144, 157, 176, 192, 204, 217, 231, 246, 261, 277, 293, 308, 322, 334, 342, 351, 362, 372, 387, 401, 418, 435, 451, 477, 496, 517, 531, 547, 556, 569, 586} +var _eventType_index = [...]uint16{0, 4, 15, 29, 42, 54, 69, 83, 100, 107, 118, 131, 144, 157, 176, 192, 204, 217, 231, 246, 261, 277, 293, 308, 322, 334, 342, 351, 362, 372, 386, 403, 419, 445, 464, 485, 499, 515, 524, 537, 554} func (i eventType) String() string { if i >= eventType(len(_eventType_index)-1) { diff --git a/agreement/proposalStore.go b/agreement/proposalStore.go index 8f89f4fc73..fdfecac5f2 100644 --- a/agreement/proposalStore.go +++ b/agreement/proposalStore.go @@ -352,15 +352,6 @@ func (store *proposalStore) handle(r routerHandle, p player, e event) event { se.Committable = ea.Assembled se.Payload = ea.Payload return se - case readLowestPayload: - re := e.(readLowestEvent) - re.T = readLowestValue - re = r.dispatch(p, re, proposalMachinePeriod, re.Round, re.Period, 0).(readLowestEvent) - re.T = readLowestPayload - ea := store.Assemblers[re.Proposal] - re.PayloadOK = ea.Assembled - re.Payload = ea.Payload - return re case readLowestVote: re := e.(readLowestEvent) return r.dispatch(p, re, proposalMachinePeriod, re.Round, re.Period, 0).(readLowestEvent) diff --git a/agreement/proposalTracker.go b/agreement/proposalTracker.go index d56fcb31a8..b9707d54dc 100644 --- a/agreement/proposalTracker.go +++ b/agreement/proposalTracker.go @@ -165,11 +165,6 @@ func (t *proposalTracker) handle(r routerHandle, p player, e event) event { t.Freezer = t.Freezer.freeze() return e - case readLowestValue: - e := e.(readLowestEvent) - e.Proposal = t.Freezer.Lowest.R.Proposal - return e - case readLowestVote: e := e.(readLowestEvent) e.Vote = t.Freezer.Lowest diff --git a/agreement/proposalTrackerContract.go b/agreement/proposalTrackerContract.go index bfc39170e8..a6fa2ffc36 100644 --- a/agreement/proposalTrackerContract.go +++ b/agreement/proposalTrackerContract.go @@ -32,7 +32,7 @@ type proposalTrackerContract struct { // TODO check concrete types of events func (c *proposalTrackerContract) pre(p player, in event) (pre []error) { switch in.t() { - case voteVerified, proposalFrozen, softThreshold, certThreshold, voteFilterRequest, readStaging, readLowestValue, readLowestPayload, readLowestVote: + case voteVerified, proposalFrozen, softThreshold, certThreshold, voteFilterRequest, readStaging, readLowestVote: default: pre = append(pre, fmt.Errorf("incoming event has invalid type: %v", in.t())) } From a3b1f485850fb01783e921742e5dd11bc8dfc0b7 Mon Sep 17 00:00:00 2001 From: chris erway Date: Wed, 16 Aug 2023 15:15:27 -0400 Subject: [PATCH 092/232] rename to lowestCredentialArrivals --- agreement/player.go | 24 +++++++++++++++--------- agreement/player_test.go | 12 ++++++------ 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/agreement/player.go b/agreement/player.go index 1e285471a2..df5a08d3fb 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -69,7 +69,9 @@ type player struct { // must be verified after some vote has been verified. Pending proposalTable - payloadArrivals []time.Duration + // the history of arrival times of the lowest credential from previous + // ronuds, used for calculating dynamic lambda. + lowestCredentialArrivals []time.Duration } func (p *player) T() stateMachineTag { @@ -280,17 +282,21 @@ func (p *player) handleCheckpointEvent(r routerHandle, e checkpointEvent) []acti // updateDynamicLambdaTimings is called at the end of a successful uninterrupted round (just after ensureAction // is generated) to collect round timings for updating dynamic lambda. -func (p *player) updateDynamicLambdaTimings(r routerHandle, payload proposal, ver protocol.ConsensusVersion) { +func (p *player) updateDynamicLambdaTimings(r routerHandle, ver protocol.ConsensusVersion) { + // only append to lowestCredentialArrivals if this was a successful round completing in period 0. + if p.Period != 0 { + return + } // look up the validatedAt time of the winning proposal-vote re := readLowestEvent{T: readLowestVote, Round: p.Round, Period: p.Period} re = r.dispatch(*p, re, proposalMachineRound, p.Round, p.Period, 0).(readLowestEvent) if re.Vote.validatedAt != 0 { - p.payloadArrivals = append(p.payloadArrivals, re.Vote.validatedAt) + p.lowestCredentialArrivals = append(p.lowestCredentialArrivals, re.Vote.validatedAt) } - p.resizePayloadArrivals(ver) + p.resizeLowestCredentialArrivals(ver) } -func (p *player) resizePayloadArrivals(ver protocol.ConsensusVersion) { +func (p *player) resizeLowestCredentialArrivals(ver protocol.ConsensusVersion) { proto := config.Consensus[ver] if len(p.payloadArrivals) > proto.DynamicFilterCredentialArrivalHistory { p.payloadArrivals = p.payloadArrivals[len(p.payloadArrivals)-proto.DynamicFilterCredentialArrivalHistory:] @@ -320,8 +326,8 @@ func (p *player) calculateFilterTimeout(ver protocol.ConsensusVersion, tracer *t // not enough samples, use the default dynamicDelay = defaultDelay } else { - sortedArrivals := make([]time.Duration, len(p.payloadArrivals)) - copy(sortedArrivals[:], p.payloadArrivals[:]) + sortedArrivals := make([]time.Duration, len(p.lowestCredentialArrivals)) + copy(sortedArrivals[:], p.lowestCredentialArrivals[:]) sort.Slice(sortedArrivals, func(i, j int) bool { return sortedArrivals[i] < sortedArrivals[j] }) dynamicDelay = sortedArrivals[proto.DynamicFilterCredentialArrivalHistoryIdx] + 50*time.Millisecond } @@ -357,7 +363,7 @@ func (p *player) handleThresholdEvent(r routerHandle, e thresholdEvent) []action cert := Certificate(e.Bundle) a0 := ensureAction{Payload: res.Payload, Certificate: cert} actions = append(actions, a0) - p.updateDynamicLambdaTimings(r, res.Payload, e.Proto) + p.updateDynamicLambdaTimings(r, e.Proto) as := p.enterRound(r, e, p.Round+1) return append(actions, as...) } @@ -677,7 +683,7 @@ func (p *player) handleMessageEvent(r routerHandle, e messageEvent) (actions []a cert := Certificate(freshestRes.Event.Bundle) a0 := ensureAction{Payload: e.Input.Proposal, Certificate: cert} actions = append(actions, a0) - p.updateDynamicLambdaTimings(r, e.Input.Proposal, e.Proto.Version) + p.updateDynamicLambdaTimings(r, e.Proto.Version) as := p.enterRound(r, delegatedE, cert.Round+1) return append(actions, as...) } diff --git a/agreement/player_test.go b/agreement/player_test.go index 24c7da3e01..0de0bc3f52 100644 --- a/agreement/player_test.go +++ b/agreement/player_test.go @@ -3265,8 +3265,8 @@ func TestPlayerRetainsReceivedValidatedAt(t *testing.T) { // assert lowest vote validateAt time was recorded into payloadArrivals historyLen := config.Consensus[protocol.ConsensusFuture].DynamicFilterPayloadArriavalHistory require.NotZero(t, historyLen) - require.Len(t, pWhite.payloadArrivals, historyLen) - require.Equal(t, 501*time.Millisecond, pWhite.payloadArrivals[historyLen-1]) + require.Len(t, pWhite.lowestCredentialArrivals, historyLen) + require.Equal(t, 501*time.Millisecond, pWhite.lowestCredentialArrivals[historyLen-1]) } // test that ReceivedAt and ValidateAt timing information are retained in proposalStore @@ -3309,8 +3309,8 @@ func TestPlayerRetainsReceivedValidatedAtPP(t *testing.T) { // assert lowest vote validateAt time was recorded into payloadArrivals historyLen := config.Consensus[protocol.ConsensusFuture].DynamicFilterPayloadArriavalHistory require.NotZero(t, historyLen) - require.Len(t, pWhite.payloadArrivals, historyLen) - require.Equal(t, 502*time.Millisecond, pWhite.payloadArrivals[historyLen-1]) + require.Len(t, pWhite.lowestCredentialArrivals, historyLen) + require.Equal(t, 502*time.Millisecond, pWhite.lowestCredentialArrivals[historyLen-1]) } // test that ReceivedAt and ValidateAt timing information are retained in proposalStore @@ -3364,8 +3364,8 @@ func TestPlayerRetainsReceivedValidatedAtAVPP(t *testing.T) { // assert lowest vote validateAt time was recorded into payloadArrivals historyLen := config.Consensus[protocol.ConsensusFuture].DynamicFilterPayloadArriavalHistory require.NotZero(t, historyLen) - require.Len(t, pWhite.payloadArrivals, historyLen) - require.Equal(t, 502*time.Millisecond, pWhite.payloadArrivals[historyLen-1]) + require.Len(t, pWhite.lowestCredentialArrivals, historyLen) + require.Equal(t, 502*time.Millisecond, pWhite.lowestCredentialArrivals[historyLen-1]) } func assertCorrectReceivedAtSet(t *testing.T, pWhite *player, pM ioAutomata, helper *voteMakerHelper, From 23941a2be8a6e7abacee563b4edab54f392f3896 Mon Sep 17 00:00:00 2001 From: chris erway Date: Wed, 16 Aug 2023 15:19:16 -0400 Subject: [PATCH 093/232] fix fmt string --- agreement/events.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agreement/events.go b/agreement/events.go index 7f79efc198..704eda3580 100644 --- a/agreement/events.go +++ b/agreement/events.go @@ -432,7 +432,7 @@ func (e readLowestEvent) t() eventType { } func (e readLowestEvent) String() string { - return fmt.Sprintf("%s: %d\t%.10s\t%.5s", e.t().String(), e.Round, e.Period) + return fmt.Sprintf("%s: %d %d", e.t().String(), e.Round, e.Period) } func (e readLowestEvent) ComparableStr() string { From 005dfbf6abf5c611b0114c0d62c7c11257857628 Mon Sep 17 00:00:00 2001 From: chris erway Date: Wed, 16 Aug 2023 15:21:45 -0400 Subject: [PATCH 094/232] fix TestPlayerRetainsReceivedValidatedAt tests --- agreement/events.go | 4 ++-- agreement/player_test.go | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/agreement/events.go b/agreement/events.go index 704eda3580..29341f3776 100644 --- a/agreement/events.go +++ b/agreement/events.go @@ -412,11 +412,11 @@ func (e newRoundEvent) ComparableStr() string { } type readLowestEvent struct { - // T is either readLowestValue or readLowestPayload + // T currently only supports readLowestVote T eventType // Round and Period are the round and period for which to query - // the lowest-credential value and payload. This type of event + // the lowest-credential vote, value or payload. This type of event // is only sent for pipelining, which only makes sense for period // 0, but the Period is here anyway to route to the appropriate // proposalMachinePeriod. diff --git a/agreement/player_test.go b/agreement/player_test.go index 0de0bc3f52..ba2ee443db 100644 --- a/agreement/player_test.go +++ b/agreement/player_test.go @@ -3240,7 +3240,7 @@ func TestPlayerRetainsReceivedValidatedAt(t *testing.T) { partitiontest.PartitionTest(t) const r = round(20239) - const p = period(1001) + const p = period(0) pWhite, pM, helper := setupP(t, r-1, p, soft) pP, pV := helper.MakeRandomProposalPayload(t, r-1) @@ -3277,7 +3277,7 @@ func TestPlayerRetainsReceivedValidatedAtPP(t *testing.T) { partitiontest.PartitionTest(t) const r = round(20239) - const p = period(1001) + const p = period(0) pWhite, pM, helper := setupP(t, r-1, p, soft) pP, pV := helper.MakeRandomProposalPayload(t, r-1) @@ -3322,7 +3322,7 @@ func TestPlayerRetainsReceivedValidatedAtAVPP(t *testing.T) { partitiontest.PartitionTest(t) const r = round(20239) - const p = period(1001) + const p = period(0) pWhite, pM, helper := setupP(t, r-1, p, soft) pP, pV := helper.MakeRandomProposalPayload(t, r-1) From 8a898219515bd3a1a7e97fb75935b49848325c87 Mon Sep 17 00:00:00 2001 From: Nickolai Zeldovich Date: Thu, 10 Aug 2023 05:17:15 -0400 Subject: [PATCH 095/232] generic timeout type for timers.Clock --- agreement/msgp_gen.go | 1 - agreement/player.go | 3 +-- agreement/service.go | 8 ++++++++ 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/agreement/msgp_gen.go b/agreement/msgp_gen.go index 77d2cceed6..9626cb8b59 100644 --- a/agreement/msgp_gen.go +++ b/agreement/msgp_gen.go @@ -13,7 +13,6 @@ import ( "github.com/algorand/go-algorand/data/bookkeeping" "github.com/algorand/go-algorand/data/committee" "github.com/algorand/go-algorand/protocol" - "github.com/algorand/go-algorand/util/timers" ) // The following msgp objects are implemented in this file: diff --git a/agreement/player.go b/agreement/player.go index df5a08d3fb..7f3bc14ef4 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -22,7 +22,6 @@ import ( "github.com/algorand/go-algorand/config" "github.com/algorand/go-algorand/protocol" - "github.com/algorand/go-algorand/util/timers" ) // Deadline marks a timeout event of type Type that the player schedules to @@ -30,7 +29,7 @@ import ( type Deadline struct { _struct struct{} `codec:","` Duration time.Duration - Type timers.TimeoutType + Type TimeoutType } // The player implements the top-level state machine functionality of the diff --git a/agreement/service.go b/agreement/service.go index 765f4270a9..c7e9462b9d 100644 --- a/agreement/service.go +++ b/agreement/service.go @@ -84,6 +84,14 @@ type externalDemuxSignals struct { CurrentRound round } +type TimeoutType int8 + +const ( + TimeoutDeadline TimeoutType = iota + TimeoutFastRecovery + TimeoutFilter +) + // MakeService creates a new Agreement Service instance given a set of Parameters. // // Call Start to start execution and Shutdown to finish execution. From f88b521dc7ff88e37a71d9d1d9929f25da599946 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Mon, 14 Aug 2023 15:55:58 -0400 Subject: [PATCH 096/232] no need for msgp in timers anymore --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index a62b529dad..8301771718 100644 --- a/Makefile +++ b/Makefile @@ -86,7 +86,7 @@ ALGOD_API_PACKAGES := $(sort $(shell GOPATH=$(GOPATH) && GO111MODULE=off && cd d GOMOD_DIRS := ./tools/block-generator ./tools/x-repo-types -MSGP_GENERATE := ./protocol ./protocol/test ./crypto ./crypto/merklearray ./crypto/merklesignature ./crypto/stateproof ./data/basics ./data/transactions ./data/stateproofmsg ./data/committee ./data/bookkeeping ./data/hashable ./agreement ./rpcs ./network ./node ./ledger ./ledger/ledgercore ./ledger/store/trackerdb ./ledger/store/trackerdb/generickv ./ledger/encoded ./stateproof ./data/account ./daemon/algod/api/spec/v2 ./util/timers +MSGP_GENERATE := ./protocol ./protocol/test ./crypto ./crypto/merklearray ./crypto/merklesignature ./crypto/stateproof ./data/basics ./data/transactions ./data/stateproofmsg ./data/committee ./data/bookkeeping ./data/hashable ./agreement ./rpcs ./network ./node ./ledger ./ledger/ledgercore ./ledger/store/trackerdb ./ledger/store/trackerdb/generickv ./ledger/encoded ./stateproof ./data/account ./daemon/algod/api/spec/v2 default: build From 8f884e84dc712dbd5f2f13d38e00b86320731ff3 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Mon, 14 Aug 2023 16:00:24 -0400 Subject: [PATCH 097/232] remove uneeded files --- util/timers/msgp_gen.go | 70 ------------------------------------ util/timers/msgp_gen_test.go | 6 ---- 2 files changed, 76 deletions(-) delete mode 100644 util/timers/msgp_gen.go delete mode 100644 util/timers/msgp_gen_test.go diff --git a/util/timers/msgp_gen.go b/util/timers/msgp_gen.go deleted file mode 100644 index ae3b2729b5..0000000000 --- a/util/timers/msgp_gen.go +++ /dev/null @@ -1,70 +0,0 @@ -package timers - -// Code generated by github.com/algorand/msgp DO NOT EDIT. - -import ( - "github.com/algorand/msgp/msgp" -) - -// The following msgp objects are implemented in this file: -// TimeoutType -// |-----> MarshalMsg -// |-----> CanMarshalMsg -// |-----> (*) UnmarshalMsg -// |-----> (*) CanUnmarshalMsg -// |-----> Msgsize -// |-----> MsgIsZero -// |-----> TimeoutTypeMaxSize() -// - -// MarshalMsg implements msgp.Marshaler -func (z TimeoutType) MarshalMsg(b []byte) (o []byte) { - o = msgp.Require(b, z.Msgsize()) - o = msgp.AppendInt8(o, int8(z)) - return -} - -func (_ TimeoutType) CanMarshalMsg(z interface{}) bool { - _, ok := (z).(TimeoutType) - if !ok { - _, ok = (z).(*TimeoutType) - } - return ok -} - -// UnmarshalMsg implements msgp.Unmarshaler -func (z *TimeoutType) UnmarshalMsg(bts []byte) (o []byte, err error) { - { - var zb0001 int8 - zb0001, bts, err = msgp.ReadInt8Bytes(bts) - if err != nil { - err = msgp.WrapError(err) - return - } - (*z) = TimeoutType(zb0001) - } - o = bts - return -} - -func (_ *TimeoutType) CanUnmarshalMsg(z interface{}) bool { - _, ok := (z).(*TimeoutType) - return ok -} - -// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message -func (z TimeoutType) Msgsize() (s int) { - s = msgp.Int8Size - return -} - -// MsgIsZero returns whether this is a zero value -func (z TimeoutType) MsgIsZero() bool { - return z == 0 -} - -// MaxSize returns a maximum valid message size for this message type -func TimeoutTypeMaxSize() (s int) { - s = msgp.Int8Size - return -} diff --git a/util/timers/msgp_gen_test.go b/util/timers/msgp_gen_test.go deleted file mode 100644 index 265bbec36f..0000000000 --- a/util/timers/msgp_gen_test.go +++ /dev/null @@ -1,6 +0,0 @@ -//go:build !skip_msgp_testing -// +build !skip_msgp_testing - -package timers - -// Code generated by github.com/algorand/msgp DO NOT EDIT. From cf06cad158b91522b65b76747e26c68b31fa7ac9 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Tue, 15 Aug 2023 13:20:29 -0400 Subject: [PATCH 098/232] one deadline for each type --- agreement/service.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/agreement/service.go b/agreement/service.go index c7e9462b9d..0911bdd6ad 100644 --- a/agreement/service.go +++ b/agreement/service.go @@ -84,11 +84,19 @@ type externalDemuxSignals struct { CurrentRound round } +// TimeoutType taggs the type of a timeout, to distinguish simultaneous timeouts +// of different types. type TimeoutType int8 const ( + // TimeoutDeadline annotates timeout events in the agreement protocol (e.g., + // for receiving a block). TimeoutDeadline TimeoutType = iota + // TimeoutFastRecovery annotates the fast recovery timeout in the agreement + // protocol. TimeoutFastRecovery + // TimeoutFilter annotates the filter step timeout event in the agreement + // protocol. TimeoutFilter ) From 3b61b8adb06266c6d45fcb49b2f3da78a6a12bdb Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Tue, 15 Aug 2023 15:04:14 -0400 Subject: [PATCH 099/232] comment explaining why we store the deadline type --- agreement/player.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/agreement/player.go b/agreement/player.go index 7f3bc14ef4..fa7969dafa 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -29,7 +29,8 @@ import ( type Deadline struct { _struct struct{} `codec:","` Duration time.Duration - Type TimeoutType + // Type is used to allow tests fire timeouts of specific types. + Type TimeoutType } // The player implements the top-level state machine functionality of the From 9ce34be810526c66b5ff86d2f96946df87611eb0 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Thu, 17 Aug 2023 13:39:29 -0400 Subject: [PATCH 100/232] address some of the comments --- agreement/player.go | 24 +++++++++++------------- agreement/player_test.go | 18 +++++++++--------- agreement/service_test.go | 2 +- config/consensus.go | 21 ++++++++++++--------- tools/x-repo-types/xrt_test.go | 14 ++++++-------- 5 files changed, 39 insertions(+), 40 deletions(-) diff --git a/agreement/player.go b/agreement/player.go index fa7969dafa..a8a1e6313f 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -293,10 +293,8 @@ func (p *player) updateDynamicLambdaTimings(r routerHandle, ver protocol.Consens if re.Vote.validatedAt != 0 { p.lowestCredentialArrivals = append(p.lowestCredentialArrivals, re.Vote.validatedAt) } - p.resizeLowestCredentialArrivals(ver) -} -func (p *player) resizeLowestCredentialArrivals(ver protocol.ConsensusVersion) { + // trim history to the last proto.DynamicFilterCredentialArrivalHistory samples. proto := config.Consensus[ver] if len(p.payloadArrivals) > proto.DynamicFilterCredentialArrivalHistory { p.payloadArrivals = p.payloadArrivals[len(p.payloadArrivals)-proto.DynamicFilterCredentialArrivalHistory:] @@ -310,28 +308,26 @@ func (p *player) resizeLowestCredentialArrivals(ver protocol.ConsensusVersion) { func (p *player) calculateFilterTimeout(ver protocol.ConsensusVersion, tracer *tracer) time.Duration { proto := config.Consensus[ver] - - if !proto.DynamicFilterTimeout || p.Period != 0 { + if proto.DynamicFilterCredentialArrivalHistory <= 0 || p.Period != 0 { // Either dynamic lambda is disabled, or we're not in period 0 and // therefore, can't use dynamic lambda return FilterTimeout(p.Period, ver) } - var dynamicDelay time.Duration defaultDelay := FilterTimeout(0, ver) if proto.DynamicFilterCredentialArrivalHistory <= 0 { // we don't keep any history, use the default dynamicDelay = defaultDelay } else if proto.DynamicFilterCredentialArrivalHistory > len(p.payloadArrivals) { // not enough samples, use the default - dynamicDelay = defaultDelay - } else { - sortedArrivals := make([]time.Duration, len(p.lowestCredentialArrivals)) - copy(sortedArrivals[:], p.lowestCredentialArrivals[:]) - sort.Slice(sortedArrivals, func(i, j int) bool { return sortedArrivals[i] < sortedArrivals[j] }) - dynamicDelay = sortedArrivals[proto.DynamicFilterCredentialArrivalHistoryIdx] + 50*time.Millisecond + return defaultDelay } + sortedArrivals := make([]time.Duration, len(p.lowestCredentialArrivals)) + copy(sortedArrivals[:], p.lowestCredentialArrivals[:]) + sort.Slice(sortedArrivals, func(i, j int) bool { return sortedArrivals[i] < sortedArrivals[j] }) + dynamicDelay := sortedArrivals[proto.DynamicFilterCredentialArrivalHistoryIdx] + proto.DynamicFilterGraceTime + // Make sure the dynamic delay is not too small or too large if dynamicDelay < proto.DynamicFilterTimeoutLowerBound { if tracer != nil { @@ -339,8 +335,10 @@ func (p *player) calculateFilterTimeout(ver protocol.ConsensusVersion, tracer *t } dynamicDelay = proto.DynamicFilterTimeoutLowerBound } else if dynamicDelay > defaultDelay { + if tracer != nil { + tracer.log.Debugf("Calculated dynamic delay = %v for round %v, period %v. It's higher than the default %v\n", dynamicDelay, p.Round, p.Period, defaultDelay) + } dynamicDelay = defaultDelay - tracer.log.Warnf("Calculated dynamic delay = %v for round %v, period %v. It's higher than the default %v\n", dynamicDelay, p.Round, p.Period, defaultDelay) } else if tracer != nil { tracer.log.Debugf("Calculated dynamic delay = %v for round %v, period %v.\n", dynamicDelay, p.Round, p.Period) } diff --git a/agreement/player_test.go b/agreement/player_test.go index ba2ee443db..65b49b5adc 100644 --- a/agreement/player_test.go +++ b/agreement/player_test.go @@ -3236,7 +3236,7 @@ func TestPlayerAlwaysResynchsPinnedValue(t *testing.T) { // test that ReceivedAt and ValidateAt timing information are retained in proposalStore // when the payloadPresent, payloadVerified, and voteVerified events are processed, and that all timings // are available when the ensureAction is called for the block. -func TestPlayerRetainsReceivedValidatedAt(t *testing.T) { +func TestPlayerRetainsReceivedValidatedAtOneSample(t *testing.T) { partitiontest.PartitionTest(t) const r = round(20239) @@ -3265,15 +3265,15 @@ func TestPlayerRetainsReceivedValidatedAt(t *testing.T) { // assert lowest vote validateAt time was recorded into payloadArrivals historyLen := config.Consensus[protocol.ConsensusFuture].DynamicFilterPayloadArriavalHistory require.NotZero(t, historyLen) - require.Len(t, pWhite.lowestCredentialArrivals, historyLen) - require.Equal(t, 501*time.Millisecond, pWhite.lowestCredentialArrivals[historyLen-1]) + require.Len(t, pWhite.lowestCredentialArrivals, 1) + require.Equal(t, 501*time.Millisecond, pWhite.lowestCredentialArrivals[0]) } // test that ReceivedAt and ValidateAt timing information are retained in proposalStore // when the payloadPresent (as part of the CompoundMessage encoding used by PP messages), // payloadVerified, and voteVerified events are processed, and that all timings // are available when the ensureAction is called for the block. -func TestPlayerRetainsReceivedValidatedAtPP(t *testing.T) { +func TestPlayerRetainsReceivedValidatedAtPPOneSample(t *testing.T) { partitiontest.PartitionTest(t) const r = round(20239) @@ -3309,8 +3309,8 @@ func TestPlayerRetainsReceivedValidatedAtPP(t *testing.T) { // assert lowest vote validateAt time was recorded into payloadArrivals historyLen := config.Consensus[protocol.ConsensusFuture].DynamicFilterPayloadArriavalHistory require.NotZero(t, historyLen) - require.Len(t, pWhite.lowestCredentialArrivals, historyLen) - require.Equal(t, 502*time.Millisecond, pWhite.lowestCredentialArrivals[historyLen-1]) + require.Len(t, pWhite.lowestCredentialArrivals, 1) + require.Equal(t, 502*time.Millisecond, pWhite.lowestCredentialArrivals[0]) } // test that ReceivedAt and ValidateAt timing information are retained in proposalStore @@ -3318,7 +3318,7 @@ func TestPlayerRetainsReceivedValidatedAtPP(t *testing.T) { // then the payloadPresent (as part of the CompoundMessage encoding used by PP messages) // and payloadVerified events are processed, and that all timings // are available when the ensureAction is called for the block. -func TestPlayerRetainsReceivedValidatedAtAVPP(t *testing.T) { +func TestPlayerRetainsReceivedValidatedAtAVPPOneSample(t *testing.T) { partitiontest.PartitionTest(t) const r = round(20239) @@ -3364,8 +3364,8 @@ func TestPlayerRetainsReceivedValidatedAtAVPP(t *testing.T) { // assert lowest vote validateAt time was recorded into payloadArrivals historyLen := config.Consensus[protocol.ConsensusFuture].DynamicFilterPayloadArriavalHistory require.NotZero(t, historyLen) - require.Len(t, pWhite.lowestCredentialArrivals, historyLen) - require.Equal(t, 502*time.Millisecond, pWhite.lowestCredentialArrivals[historyLen-1]) + require.Len(t, pWhite.lowestCredentialArrivals, 1) + require.Equal(t, 502*time.Millisecond, pWhite.lowestCredentialArrivals[0]) } func assertCorrectReceivedAtSet(t *testing.T, pWhite *player, pM ioAutomata, helper *voteMakerHelper, diff --git a/agreement/service_test.go b/agreement/service_test.go index e70dd36028..681ce89cdf 100644 --- a/agreement/service_test.go +++ b/agreement/service_test.go @@ -1002,7 +1002,7 @@ func TestAgreementSynchronousFuture5_DynamicFilterRounds(t *testing.T) { } cfg := config.Consensus[protocol.ConsensusFuture] - if !cfg.DynamicFilterTimeout { + if cfg.DynamicFilterCredentialArrivalHistory <= 0 { return } diff --git a/config/consensus.go b/config/consensus.go index f34ff896a6..79a2433ce6 100644 --- a/config/consensus.go +++ b/config/consensus.go @@ -517,21 +517,24 @@ type ConsensusParams struct { // by state proofs to use the same method (rather than excluding stake from the top N stakeholders as before). ExcludeExpiredCirculation bool - // DynamicFilterTimeout specifies whether the timeout for the filter step - // should be determined dynamically, at run-time. - DynamicFilterTimeout bool + // DynamicFilterCredentialArrivalHistory specifies the number of past + // credential arrivals that are measured to determine the next filter + // timeout. If DynamicFilterCredentialArrivalHistory <= 0, then the dynamic + // timeout feature is off and the filter step timeout is calculated using + // the static configuration. + DynamicFilterCredentialArrivalHistory int // DynamicFilterTimeoutLowerBound specifies a minimal duration that the // filter timeout must meet. DynamicFilterTimeoutLowerBound time.Duration - // DynamicFilterCredentialArrivalHistory specifies the number of past credential arrivals - // that are measured to determine the next filter timeout. - DynamicFilterCredentialArrivalHistory int - // DynamicFilterCredentialArrivalHistoryIdx specified which sample to use out of a sorted // DynamicFilterCredentialArrivalHistory-sized array of time samples. DynamicFilterCredentialArrivalHistoryIdx int + + // DynamicFilterGraceTime is additional extention to the dynamic filter time atop the + // one calculated based on the history of credential arrivals. + DynamicFilterGraceTime time.Duration } // PaysetCommitType enumerates possible ways for the block header to commit to @@ -1379,12 +1382,12 @@ func initConsensusProtocols() { vFuture.LogicSigVersion = 10 // When moving this to a release, put a new higher LogicSigVersion here vFuture.EnableLogicSigCostPooling = true - vFuture.DynamicFilterTimeout = true - vFuture.DynamicFilterTimeoutLowerBound = time.Second // history window of 40, so we have enough statistics to calculate the 95th // percentile, which is the timestamp at index 37 in the history array. vFuture.DynamicFilterCredentialArrivalHistory = 40 vFuture.DynamicFilterCredentialArrivalHistoryIdx = 37 + vFuture.DynamicFilterTimeoutLowerBound = time.Second + vFuture.DynamicFilterGraceTime = 50 * time.Millisecond Consensus[protocol.ConsensusFuture] = vFuture diff --git a/tools/x-repo-types/xrt_test.go b/tools/x-repo-types/xrt_test.go index 6fe353cedb..4360b432d6 100644 --- a/tools/x-repo-types/xrt_test.go +++ b/tools/x-repo-types/xrt_test.go @@ -74,14 +74,12 @@ func TestCrossRepoTypes(t *testing.T) { yType: "EvalDelta", }, { - name: "goal-v-sdk-consensus", - xPkg: "github.com/algorand/go-algorand/config", - xType: "ConsensusParams", - yPkg: "github.com/algorand/go-algorand-sdk/v2/protocol/config", - yBranch: "develop", - yType: "ConsensusParams", - skip: true, - skipReason: `Changes to consensus parameters happen on the go-algorand repo, and cannot be synced in a single PR with changes to the go-algorand-sdk repo`, + name: "goal-v-sdk-consensus", + xPkg: "github.com/algorand/go-algorand/config", + xType: "ConsensusParams", + yPkg: "github.com/algorand/go-algorand-sdk/v2/protocol/config", + yBranch: "develop", + yType: "ConsensusParams", }, { name: "goal-v-sdk-blockheader", From 1d7e6e44cca5a061445c0197ea9592e555235eba Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Thu, 17 Aug 2023 14:44:46 -0400 Subject: [PATCH 101/232] history window tests --- agreement/player_test.go | 162 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 161 insertions(+), 1 deletion(-) diff --git a/agreement/player_test.go b/agreement/player_test.go index 65b49b5adc..7d45c025a0 100644 --- a/agreement/player_test.go +++ b/agreement/player_test.go @@ -3269,6 +3269,50 @@ func TestPlayerRetainsReceivedValidatedAtOneSample(t *testing.T) { require.Equal(t, 501*time.Millisecond, pWhite.lowestCredentialArrivals[0]) } +// test that ReceivedAt and ValidateAt timing information are retained in proposalStore +// when the payloadPresent, payloadVerified, and voteVerified events are processed, and that all timings +// are available when the ensureAction is called for the block. The history should be kept for the last +// DynamicFilterCredentialArrivalHistory rounds. +func TestPlayerRetainsReceivedValidatedAtForHistoryWindow(t *testing.T) { + partitiontest.PartitionTest(t) + + const r = round(20239) + const p = period(0) + pWhite, pM, helper := setupP(t, r-1, p, soft) + + historyLen := config.Consensus[protocol.ConsensusFuture].DynamicFilterCredentialArrivalHistory + require.NotZero(t, historyLen) + + for i := 0; i < historyLen+1; i++ { + // send voteVerified message + pP, pV := helper.MakeRandomProposalPayload(t, r+round(i)-1) + vVote := helper.MakeVerifiedVote(t, 0, r+round(i)-1, p, propose, *pV) + inMsg := messageEvent{T: voteVerified, Input: message{Vote: vVote, UnauthenticatedVote: vVote.u()}} + timestamp := 500 + i + inMsg = inMsg.AttachValidatedAt(time.Duration(timestamp) * time.Millisecond) + err, panicErr := pM.transition(inMsg) + require.NoError(t, err) + require.NoError(t, panicErr) + + // send payloadPresent message + m := message{UnauthenticatedProposal: pP.u()} + inMsg = messageEvent{T: payloadPresent, Input: m} + inMsg = inMsg.AttachReceivedAt(time.Second) + err, panicErr = pM.transition(inMsg) + require.NoError(t, err) + require.NoError(t, panicErr) + moveToRound(t, pWhite, pM, helper, r+round(i), p, pP, pV, m, protocol.ConsensusFuture) + } + + // assert lowest vote validateAt time was recorded into payloadArrivals + require.Len(t, pWhite.lowestCredentialArrivals, historyLen) + for i := 0; i < historyLen; i++ { + // only the last historyLen samples are kept, so the first one is discarded + timestamp := 500 + i + 1 + require.Equal(t, time.Duration(timestamp)*time.Millisecond, pWhite.lowestCredentialArrivals[i]) + } +} + // test that ReceivedAt and ValidateAt timing information are retained in proposalStore // when the payloadPresent (as part of the CompoundMessage encoding used by PP messages), // payloadVerified, and voteVerified events are processed, and that all timings @@ -3313,6 +3357,60 @@ func TestPlayerRetainsReceivedValidatedAtPPOneSample(t *testing.T) { require.Equal(t, 502*time.Millisecond, pWhite.lowestCredentialArrivals[0]) } +// test that ReceivedAt and ValidateAt timing information are retained in +// proposalStore when the payloadPresent (as part of the CompoundMessage +// encoding used by PP messages), payloadVerified, and voteVerified events are +// processed, and that all timings are available when the ensureAction is called +// for the block. The history should be kept for the last +// DynamicFilterCredentialArrivalHistory rounds. +func TestPlayerRetainsReceivedValidatedAtPPForHistoryWindow(t *testing.T) { + partitiontest.PartitionTest(t) + + const r = round(20239) + const p = period(0) + pWhite, pM, helper := setupP(t, r-1, p, soft) + pP, pV := helper.MakeRandomProposalPayload(t, r-1) + + historyLen := config.Consensus[protocol.ConsensusFuture].DynamicFilterCredentialArrivalHistory + require.NotZero(t, historyLen) + + for i := 0; i < historyLen+1; i++ { + // create a PP message for an arbitrary proposal/payload similar to setupCompoundMessage + vVote := helper.MakeVerifiedVote(t, 0, r+round(i)-1, p, propose, *pV) + unverifiedVoteMsg := message{UnauthenticatedVote: vVote.u()} + proposalMsg := message{UnauthenticatedProposal: pP.u()} + compoundMsg := messageEvent{T: votePresent, Input: unverifiedVoteMsg, + Tail: &messageEvent{T: payloadPresent, Input: proposalMsg}} + + inMsg := compoundMsg.AttachReceivedAt(time.Second) // call AttachReceivedAt like demux would + err, panicErr := pM.transition(inMsg) + require.NoError(t, err) + require.NoError(t, panicErr) + + // make sure vote verify requests + verifyEvent := ev(cryptoAction{T: verifyVote, M: unverifiedVoteMsg, Round: r + round(i) - 1, Period: p, Step: propose, TaskIndex: 1}) + require.Truef(t, pM.getTrace().Contains(verifyEvent), "Player should verify vote") + + // send voteVerified + verifiedVoteMsg := message{Vote: vVote, UnauthenticatedVote: vVote.u()} + inMsg = messageEvent{T: voteVerified, Input: verifiedVoteMsg, TaskIndex: 1} + timestamp := 500 + i + inMsg = inMsg.AttachValidatedAt(time.Duration(timestamp) * time.Millisecond) + err, panicErr = pM.transition(inMsg) + require.NoError(t, err) + require.NoError(t, panicErr) + moveToRound(t, pWhite, pM, helper, r+round(i), p, pP, pV, proposalMsg, protocol.ConsensusFuture) + } + + // assert lowest vote validateAt time was recorded into payloadArrivals + require.Len(t, pWhite.lowestCredentialArrivals, historyLen) + for i := 0; i < historyLen; i++ { + // only the last historyLen samples are kept, so the first one is discarded + timestamp := 500 + i + 1 + require.Equal(t, time.Duration(timestamp)*time.Millisecond, pWhite.lowestCredentialArrivals[i]) + } +} + // test that ReceivedAt and ValidateAt timing information are retained in proposalStore // when the voteVerified event comes in first (as part of the AV message before PP), // then the payloadPresent (as part of the CompoundMessage encoding used by PP messages) @@ -3368,8 +3466,64 @@ func TestPlayerRetainsReceivedValidatedAtAVPPOneSample(t *testing.T) { require.Equal(t, 502*time.Millisecond, pWhite.lowestCredentialArrivals[0]) } -func assertCorrectReceivedAtSet(t *testing.T, pWhite *player, pM ioAutomata, helper *voteMakerHelper, +func TestPlayerRetainsReceivedValidatedAtAVPPHistoryWindow(t *testing.T) { + partitiontest.PartitionTest(t) + + const r = round(20239) + const p = period(0) + pWhite, pM, helper := setupP(t, r-1, p, soft) + + historyLen := config.Consensus[protocol.ConsensusFuture].DynamicFilterCredentialArrivalHistory + require.NotZero(t, historyLen) + + for i := 0; i < historyLen+1; i++ { + pP, pV := helper.MakeRandomProposalPayload(t, r+round(i)-1) + + // send votePresent message (mimicking the first AV message validating) + vVote := helper.MakeVerifiedVote(t, 0, r+round(i)-1, p, propose, *pV) + unverifiedVoteMsg := message{UnauthenticatedVote: vVote.u()} + inMsg := messageEvent{T: votePresent, Input: unverifiedVoteMsg} + err, panicErr := pM.transition(inMsg) + require.NoError(t, err) + require.NoError(t, panicErr) + + // make sure vote verify requests + verifyEvent := ev(cryptoAction{T: verifyVote, M: unverifiedVoteMsg, Round: r + round(i) - 1, Period: p, Step: propose, TaskIndex: 1}) + require.Truef(t, pM.getTrace().Contains(verifyEvent), "Player should verify vote") + + // send voteVerified + verifiedVoteMsg := message{Vote: vVote, UnauthenticatedVote: vVote.u()} + inMsg = messageEvent{T: voteVerified, Input: verifiedVoteMsg, TaskIndex: 1} + timestamp := 500 + i + inMsg = inMsg.AttachValidatedAt(time.Duration(timestamp) * time.Millisecond) + err, panicErr = pM.transition(inMsg) + require.NoError(t, err) + require.NoError(t, panicErr) + + // create a PP message for an arbitrary proposal/payload similar to setupCompoundMessage + proposalMsg := message{UnauthenticatedProposal: pP.u()} + compoundMsg := messageEvent{T: votePresent, Input: unverifiedVoteMsg, + Tail: &messageEvent{T: payloadPresent, Input: proposalMsg}} + inMsg = compoundMsg.AttachReceivedAt(time.Second) // call AttachReceivedAt like demux would + err, panicErr = pM.transition(inMsg) + require.NoError(t, err) + require.NoError(t, panicErr) + + moveToRound(t, pWhite, pM, helper, r+round(i), p, pP, pV, proposalMsg, protocol.ConsensusFuture) + } + + // assert lowest vote validateAt time was recorded into payloadArrivals + require.Len(t, pWhite.lowestCredentialArrivals, historyLen) + for i := 0; i < historyLen; i++ { + // only the last historyLen samples are kept, so the first one is discarded + timestamp := 500 + i + 1 + require.Equal(t, time.Duration(timestamp)*time.Millisecond, pWhite.lowestCredentialArrivals[i]) + } +} + +func moveToRound(t *testing.T, pWhite *player, pM ioAutomata, helper *voteMakerHelper, r round, p period, pP *proposal, pV *proposalValue, m message, ver protocol.ConsensusVersion) { + // make sure payload verify request verifyEvent := ev(cryptoAction{T: verifyPayload, M: m, Round: r - 1, Period: p, Step: propose, TaskIndex: 0}) require.Truef(t, pM.getTrace().Contains(verifyEvent), "Player should verify payload") @@ -3410,6 +3564,12 @@ func assertCorrectReceivedAtSet(t *testing.T, pWhite *player, pM ioAutomata, hel require.Equalf(t, period(0), pWhite.Period, "player did not enter period 0 in new round") commitEvent := ev(ensureAction{Certificate: Certificate(bun), Payload: *pP}) require.Truef(t, pM.getTrace().Contains(commitEvent), "Player should try to ensure block/digest on ledger") +} + +func assertCorrectReceivedAtSet(t *testing.T, pWhite *player, pM ioAutomata, helper *voteMakerHelper, + r round, p period, pP *proposal, pV *proposalValue, m message, ver protocol.ConsensusVersion) { + + moveToRound(t, pWhite, pM, helper, r, p, pP, pV, m, ver) // find and unwrap ensureAction from trace var ea ensureAction From 974a74b64d4ac2be3b92c16a86208c3a17d5d11a Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Thu, 17 Aug 2023 16:09:46 -0400 Subject: [PATCH 102/232] nits --- agreement/player.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agreement/player.go b/agreement/player.go index a8a1e6313f..f2d4919e5f 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -70,7 +70,7 @@ type player struct { Pending proposalTable // the history of arrival times of the lowest credential from previous - // ronuds, used for calculating dynamic lambda. + // ronuds, used for calculating the filter timeout dynamically. lowestCredentialArrivals []time.Duration } From b9c96137f53279b646b88a4980dbdfb5d9c7b4ef Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Thu, 17 Aug 2023 16:15:53 -0400 Subject: [PATCH 103/232] typo --- config/consensus.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/consensus.go b/config/consensus.go index 79a2433ce6..ef30c85d2d 100644 --- a/config/consensus.go +++ b/config/consensus.go @@ -532,7 +532,7 @@ type ConsensusParams struct { // DynamicFilterCredentialArrivalHistory-sized array of time samples. DynamicFilterCredentialArrivalHistoryIdx int - // DynamicFilterGraceTime is additional extention to the dynamic filter time atop the + // DynamicFilterGraceTime is additional extension to the dynamic filter time atop the // one calculated based on the history of credential arrivals. DynamicFilterGraceTime time.Duration } From effc9071ee7bde5457212be4aeb80a7177f9ea89 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Thu, 17 Aug 2023 16:29:27 -0400 Subject: [PATCH 104/232] off switch in config --- agreement/player.go | 27 +++++++++++++++++---------- config/consensus.go | 27 ++++++++++++++++++--------- 2 files changed, 35 insertions(+), 19 deletions(-) diff --git a/agreement/player.go b/agreement/player.go index f2d4919e5f..2f5bccbe79 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -320,30 +320,37 @@ func (p *player) calculateFilterTimeout(ver protocol.ConsensusVersion, tracer *t dynamicDelay = defaultDelay } else if proto.DynamicFilterCredentialArrivalHistory > len(p.payloadArrivals) { // not enough samples, use the default - return defaultDelay + return defaultTimeout } sortedArrivals := make([]time.Duration, len(p.lowestCredentialArrivals)) copy(sortedArrivals[:], p.lowestCredentialArrivals[:]) sort.Slice(sortedArrivals, func(i, j int) bool { return sortedArrivals[i] < sortedArrivals[j] }) - dynamicDelay := sortedArrivals[proto.DynamicFilterCredentialArrivalHistoryIdx] + proto.DynamicFilterGraceTime + dynamicTimeout := sortedArrivals[proto.DynamicFilterTimeoutCredentialArrivalHistoryIdx] + proto.DynamicFilterTimeoutGraceInterval // Make sure the dynamic delay is not too small or too large - if dynamicDelay < proto.DynamicFilterTimeoutLowerBound { + if dynamicTimeout < proto.DynamicFilterTimeoutLowerBound { if tracer != nil { - tracer.log.Debugf("Calculated dynamic delay = %v for round %v, period %v. It's too low, setting to %v\n", dynamicDelay, p.Round, p.Period, proto.DynamicFilterTimeoutLowerBound) + tracer.log.Debugf("Calculated dynamic delay = %v for round %v, period %v. It's too low, setting to %v\n", dynamicTimeout, p.Round, p.Period, proto.DynamicFilterTimeoutLowerBound) } - dynamicDelay = proto.DynamicFilterTimeoutLowerBound - } else if dynamicDelay > defaultDelay { + dynamicTimeout = proto.DynamicFilterTimeoutLowerBound + } else if dynamicTimeout > defaultTimeout { if tracer != nil { - tracer.log.Debugf("Calculated dynamic delay = %v for round %v, period %v. It's higher than the default %v\n", dynamicDelay, p.Round, p.Period, defaultDelay) + tracer.log.Debugf("Calculated dynamic delay = %v for round %v, period %v. It too high, so using the default %v\n", dynamicTimeout, p.Round, p.Period, defaultTimeout) } - dynamicDelay = defaultDelay + dynamicTimeout = defaultTimeout } else if tracer != nil { - tracer.log.Debugf("Calculated dynamic delay = %v for round %v, period %v.\n", dynamicDelay, p.Round, p.Period) + tracer.log.Debugf("Calculated dynamic delay = %v for round %v, period %v.\n", dynamicTimeout, p.Round, p.Period) + } + + if !proto.DynamicFilterTimeout { + // If the dynamic filter is disabled, return the default filter timeout + // (after logging what the timeout would have been, if this feature were + // enabled). + return defaultTimeout } - return dynamicDelay + return dynamicTimeout } func (p *player) handleThresholdEvent(r routerHandle, e thresholdEvent) []action { diff --git a/config/consensus.go b/config/consensus.go index ef30c85d2d..6d9dfeff51 100644 --- a/config/consensus.go +++ b/config/consensus.go @@ -528,13 +528,21 @@ type ConsensusParams struct { // filter timeout must meet. DynamicFilterTimeoutLowerBound time.Duration - // DynamicFilterCredentialArrivalHistoryIdx specified which sample to use out of a sorted - // DynamicFilterCredentialArrivalHistory-sized array of time samples. - DynamicFilterCredentialArrivalHistoryIdx int - - // DynamicFilterGraceTime is additional extension to the dynamic filter time atop the - // one calculated based on the history of credential arrivals. - DynamicFilterGraceTime time.Duration + // DynamicFilterTimeoutCredentialArrivalHistoryIdx specified which sample to + // use out of a sorted DynamicFilterCredentialArrivalHistory-sized array of + // time samples. + DynamicFilterTimeoutCredentialArrivalHistoryIdx int + + // DynamicFilterTimeoutGraceInterval is additional extension to the dynamic + // filter time atop the one calculated based on the history of credential + // arrivals. + DynamicFilterTimeoutGraceInterval time.Duration + + // DynamicFilterTimeout indicates whether the filter timeout is set + // dynamically, at run time, according to the recent history of credential + // arrival times or is set to a static value. Even if this flag disables the + // dynamic filter, it will be calculated and logged (but not used). + DynamicFilterTimeout bool } // PaysetCommitType enumerates possible ways for the block header to commit to @@ -1385,9 +1393,10 @@ func initConsensusProtocols() { // history window of 40, so we have enough statistics to calculate the 95th // percentile, which is the timestamp at index 37 in the history array. vFuture.DynamicFilterCredentialArrivalHistory = 40 - vFuture.DynamicFilterCredentialArrivalHistoryIdx = 37 + vFuture.DynamicFilterTimeoutCredentialArrivalHistoryIdx = 37 vFuture.DynamicFilterTimeoutLowerBound = time.Second - vFuture.DynamicFilterGraceTime = 50 * time.Millisecond + vFuture.DynamicFilterTimeoutGraceInterval = 50 * time.Millisecond + vFuture.DynamicFilterTimeout = true Consensus[protocol.ConsensusFuture] = vFuture From 52621f520d85158eaf7bba2dc28ff42c43b59fba Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Thu, 17 Aug 2023 17:13:21 -0400 Subject: [PATCH 105/232] checkpoint: positive test --- agreement/player.go | 4 +++- agreement/service_test.go | 44 ++++++++++++++++++++++++++++++++++----- 2 files changed, 42 insertions(+), 6 deletions(-) diff --git a/agreement/player.go b/agreement/player.go index 2f5bccbe79..ab9442f3af 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -17,6 +17,7 @@ package agreement import ( + "fmt" "sort" "time" @@ -290,7 +291,9 @@ func (p *player) updateDynamicLambdaTimings(r routerHandle, ver protocol.Consens // look up the validatedAt time of the winning proposal-vote re := readLowestEvent{T: readLowestVote, Round: p.Round, Period: p.Period} re = r.dispatch(*p, re, proposalMachineRound, p.Round, p.Period, 0).(readLowestEvent) + if re.Vote.validatedAt != 0 { + fmt.Println("update dynamic lambda") p.lowestCredentialArrivals = append(p.lowestCredentialArrivals, re.Vote.validatedAt) } @@ -306,7 +309,6 @@ func (p *player) updateDynamicLambdaTimings(r routerHandle, ver protocol.Consens // calculateFilterTimeout chooses the appropriate filter timeout. func (p *player) calculateFilterTimeout(ver protocol.ConsensusVersion, tracer *tracer) time.Duration { - proto := config.Consensus[ver] if proto.DynamicFilterCredentialArrivalHistory <= 0 || p.Period != 0 { // Either dynamic lambda is disabled, or we're not in period 0 and diff --git a/agreement/service_test.go b/agreement/service_test.go index 681ce89cdf..e718402028 100644 --- a/agreement/service_test.go +++ b/agreement/service_test.go @@ -93,6 +93,17 @@ func (c *testingClock) TimeoutAt(d time.Duration, timeoutType TimeoutType) <-cha return ta.ch } +func (c *testingClock) when(timeoutType TimeoutType) (time.Duration, error) { + c.mu.Lock() + defer c.mu.Unlock() + + ta, ok := c.TA[timeoutType] + if !ok { + return time.Duration(0), fmt.Errorf("no timeout of type, %v", timeoutType) + } + return ta.delta, nil +} + func (c *testingClock) Encode() []byte { return nil } @@ -884,18 +895,18 @@ func sanityCheck(startRound round, numRounds round, ledgers []Ledger) { } } -func simulateAgreement(t *testing.T, numNodes int, numRounds int, traceLevel traceLevel) { - simulateAgreementWithLedgerFactory(t, numNodes, numRounds, traceLevel, makeTestLedger) +func simulateAgreement(t *testing.T, numNodes int, numRounds int, traceLevel traceLevel) (filterTimeouts []time.Duration) { + return simulateAgreementWithLedgerFactory(t, numNodes, numRounds, traceLevel, makeTestLedger) } -func simulateAgreementWithConsensusVersion(t *testing.T, numNodes int, numRounds int, traceLevel traceLevel, consensusVersion func(basics.Round) (protocol.ConsensusVersion, error)) { +func simulateAgreementWithConsensusVersion(t *testing.T, numNodes int, numRounds int, traceLevel traceLevel, consensusVersion func(basics.Round) (protocol.ConsensusVersion, error)) (filterTimeouts []time.Duration) { ledgerFactory := func(data map[basics.Address]basics.AccountData) Ledger { return makeTestLedgerWithConsensusVersion(data, consensusVersion) } - simulateAgreementWithLedgerFactory(t, numNodes, numRounds, traceLevel, ledgerFactory) + return simulateAgreementWithLedgerFactory(t, numNodes, numRounds, traceLevel, ledgerFactory) } -func simulateAgreementWithLedgerFactory(t *testing.T, numNodes int, numRounds int, traceLevel traceLevel, ledgerFactory func(map[basics.Address]basics.AccountData) Ledger) { +func simulateAgreementWithLedgerFactory(t *testing.T, numNodes int, numRounds int, traceLevel traceLevel, ledgerFactory func(map[basics.Address]basics.AccountData) Ledger) []time.Duration { _, baseLedger, cleanupFn, services, clocks, ledgers, activityMonitor := setupAgreement(t, numNodes, traceLevel, ledgerFactory) startRound := baseLedger.NextRound() defer cleanupFn() @@ -907,9 +918,16 @@ func simulateAgreementWithLedgerFactory(t *testing.T, numNodes int, numRounds in activityMonitor.waitForQuiet() zeroes := expectNewPeriod(clocks, 0) + filterTimeouts := make([][]time.Duration, numNodes, numNodes) + // run round with round-specific consensus version first (since fix in #1896) zeroes = runRoundTriggerFilter(clocks, activityMonitor, zeroes) for j := 1; j < numRounds; j++ { + for srvIdx, clock := range clocks { + delta, err := clock.(*testingClock).when(TimeoutFilter) + require.NoError(t, err) + filterTimeouts[srvIdx] = append(filterTimeouts[srvIdx], delta) + } zeroes = runRoundTriggerFilter(clocks, activityMonitor, zeroes) } @@ -918,6 +936,19 @@ func simulateAgreementWithLedgerFactory(t *testing.T, numNodes int, numRounds in } sanityCheck(startRound, round(numRounds), ledgers) + + if len(clocks) == 0 { + return nil + } + + for rnd := 0; rnd < numRounds-1; rnd++ { + delta := filterTimeouts[0][rnd] + for srvIdx := range clocks { + require.Equal(t, delta, filterTimeouts[srvIdx][rnd]) + } + } + + return filterTimeouts[0] } func TestAgreementSynchronous1(t *testing.T) { @@ -1005,6 +1036,9 @@ func TestAgreementSynchronousFuture5_DynamicFilterRounds(t *testing.T) { if cfg.DynamicFilterCredentialArrivalHistory <= 0 { return } + if !cfg.DynamicFilterTimeout { + return + } simulateAgreementWithConsensusVersion(t, 5, cfg.DynamicFilterPayloadArriavalHistory+20, disabled, consensusVersion) } From 1962351ba83a3fa6cde0ac400fba1c0bceea9cd7 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Fri, 18 Aug 2023 12:07:12 -0400 Subject: [PATCH 106/232] fix testingClock.Since to give non zero measurements --- agreement/player.go | 35 ++++++++++++++++------------------- agreement/service_test.go | 2 +- config/consensus.go | 2 +- 3 files changed, 18 insertions(+), 21 deletions(-) diff --git a/agreement/player.go b/agreement/player.go index ab9442f3af..601d31cfd3 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -17,7 +17,6 @@ package agreement import ( - "fmt" "sort" "time" @@ -281,9 +280,10 @@ func (p *player) handleCheckpointEvent(r routerHandle, e checkpointEvent) []acti }} } -// updateDynamicLambdaTimings is called at the end of a successful uninterrupted round (just after ensureAction -// is generated) to collect round timings for updating dynamic lambda. -func (p *player) updateDynamicLambdaTimings(r routerHandle, ver protocol.ConsensusVersion) { +// updateCredentialArrivalHistory is called at the end of a successful +// uninterrupted round (just after ensureAction is generated) to collect +// credential arrival times to dynamically set the filter timeout. +func (p *player) updateCredentialArrivalHistory(r routerHandle, ver protocol.ConsensusVersion) { // only append to lowestCredentialArrivals if this was a successful round completing in period 0. if p.Period != 0 { return @@ -292,10 +292,7 @@ func (p *player) updateDynamicLambdaTimings(r routerHandle, ver protocol.Consens re := readLowestEvent{T: readLowestVote, Round: p.Round, Period: p.Period} re = r.dispatch(*p, re, proposalMachineRound, p.Round, p.Period, 0).(readLowestEvent) - if re.Vote.validatedAt != 0 { - fmt.Println("update dynamic lambda") - p.lowestCredentialArrivals = append(p.lowestCredentialArrivals, re.Vote.validatedAt) - } + p.lowestCredentialArrivals = append(p.lowestCredentialArrivals, re.Vote.validatedAt) // trim history to the last proto.DynamicFilterCredentialArrivalHistory samples. proto := config.Consensus[ver] @@ -311,8 +308,8 @@ func (p *player) updateDynamicLambdaTimings(r routerHandle, ver protocol.Consens func (p *player) calculateFilterTimeout(ver protocol.ConsensusVersion, tracer *tracer) time.Duration { proto := config.Consensus[ver] if proto.DynamicFilterCredentialArrivalHistory <= 0 || p.Period != 0 { - // Either dynamic lambda is disabled, or we're not in period 0 and - // therefore, can't use dynamic lambda + // Either dynamic filter timeout is disabled, or we're not in period 0 + // and therefore, can't use dynamic timeout return FilterTimeout(p.Period, ver) } @@ -330,25 +327,25 @@ func (p *player) calculateFilterTimeout(ver protocol.ConsensusVersion, tracer *t sort.Slice(sortedArrivals, func(i, j int) bool { return sortedArrivals[i] < sortedArrivals[j] }) dynamicTimeout := sortedArrivals[proto.DynamicFilterTimeoutCredentialArrivalHistoryIdx] + proto.DynamicFilterTimeoutGraceInterval - // Make sure the dynamic delay is not too small or too large + // Make sure the dynamic filter timeout is not too small nor too large if dynamicTimeout < proto.DynamicFilterTimeoutLowerBound { if tracer != nil { - tracer.log.Debugf("Calculated dynamic delay = %v for round %v, period %v. It's too low, setting to %v\n", dynamicTimeout, p.Round, p.Period, proto.DynamicFilterTimeoutLowerBound) + tracer.log.Debugf("Calculated dynamic filter timeout = %v for round %v, period %v. It's too low, setting to %v\n", dynamicTimeout, p.Round, p.Period, proto.DynamicFilterTimeoutLowerBound) } dynamicTimeout = proto.DynamicFilterTimeoutLowerBound } else if dynamicTimeout > defaultTimeout { if tracer != nil { - tracer.log.Debugf("Calculated dynamic delay = %v for round %v, period %v. It too high, so using the default %v\n", dynamicTimeout, p.Round, p.Period, defaultTimeout) + tracer.log.Debugf("Calculated dynamic filter timeout = %v for round %v, period %v. It too high, so using the default %v\n", dynamicTimeout, p.Round, p.Period, defaultTimeout) } dynamicTimeout = defaultTimeout } else if tracer != nil { - tracer.log.Debugf("Calculated dynamic delay = %v for round %v, period %v.\n", dynamicTimeout, p.Round, p.Period) + tracer.log.Debugf("Calculated dynamic filter timeout = %v for round %v, period %v.\n", dynamicTimeout, p.Round, p.Period) } if !proto.DynamicFilterTimeout { - // If the dynamic filter is disabled, return the default filter timeout - // (after logging what the timeout would have been, if this feature were - // enabled). + // If the dynamic filter timeout is disabled, return the default filter + // timeout (after logging what the timeout would have been, if this + // feature were enabled). return defaultTimeout } @@ -370,7 +367,7 @@ func (p *player) handleThresholdEvent(r routerHandle, e thresholdEvent) []action cert := Certificate(e.Bundle) a0 := ensureAction{Payload: res.Payload, Certificate: cert} actions = append(actions, a0) - p.updateDynamicLambdaTimings(r, e.Proto) + p.updateCredentialArrivalHistory(r, e.Proto) as := p.enterRound(r, e, p.Round+1) return append(actions, as...) } @@ -690,7 +687,7 @@ func (p *player) handleMessageEvent(r routerHandle, e messageEvent) (actions []a cert := Certificate(freshestRes.Event.Bundle) a0 := ensureAction{Payload: e.Input.Proposal, Certificate: cert} actions = append(actions, a0) - p.updateDynamicLambdaTimings(r, e.Proto.Version) + p.updateCredentialArrivalHistory(r, e.Proto.Version) as := p.enterRound(r, delegatedE, cert.Round+1) return append(actions, as...) } diff --git a/agreement/service_test.go b/agreement/service_test.go index e718402028..ff2b1cd84e 100644 --- a/agreement/service_test.go +++ b/agreement/service_test.go @@ -77,7 +77,7 @@ func (c *testingClock) Zero() timers.Clock[TimeoutType] { } func (c *testingClock) Since() time.Duration { - return 0 + return 1 } func (c *testingClock) TimeoutAt(d time.Duration, timeoutType TimeoutType) <-chan time.Time { diff --git a/config/consensus.go b/config/consensus.go index 6d9dfeff51..a5644eb8c0 100644 --- a/config/consensus.go +++ b/config/consensus.go @@ -1394,7 +1394,7 @@ func initConsensusProtocols() { // percentile, which is the timestamp at index 37 in the history array. vFuture.DynamicFilterCredentialArrivalHistory = 40 vFuture.DynamicFilterTimeoutCredentialArrivalHistoryIdx = 37 - vFuture.DynamicFilterTimeoutLowerBound = time.Second + vFuture.DynamicFilterTimeoutLowerBound = 0 vFuture.DynamicFilterTimeoutGraceInterval = 50 * time.Millisecond vFuture.DynamicFilterTimeout = true From 2d74e27bb9cfed047288db78f9daf492624a3744 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Fri, 18 Aug 2023 12:43:54 -0400 Subject: [PATCH 107/232] add negative test: make sure we discard history window when going to period > 0 --- agreement/service_test.go | 114 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 114 insertions(+) diff --git a/agreement/service_test.go b/agreement/service_test.go index ff2b1cd84e..18f53b9b16 100644 --- a/agreement/service_test.go +++ b/agreement/service_test.go @@ -1043,6 +1043,120 @@ func TestAgreementSynchronousFuture5_DynamicFilterRounds(t *testing.T) { simulateAgreementWithConsensusVersion(t, 5, cfg.DynamicFilterPayloadArriavalHistory+20, disabled, consensusVersion) } +func TestDynamicFilterTimeoutResets(t *testing.T) { + partitiontest.PartitionTest(t) + + if testing.Short() { + t.Skip("Skipping agreement integration test") + } + + version := protocol.ConsensusFuture + + cfg := config.Consensus[protocol.ConsensusFuture] + if cfg.DynamicFilterCredentialArrivalHistory <= 0 { + return + } + if !cfg.DynamicFilterTimeout { + return + } + + numNodes := 5 + + consensusVersion := func(r basics.Round) (protocol.ConsensusVersion, error) { + return protocol.ConsensusFuture, nil + } + ledgerFactory := func(data map[basics.Address]basics.AccountData) Ledger { + return makeTestLedgerWithConsensusVersion(data, consensusVersion) + } + + baseNetwork, baseLedger, cleanupFn, services, clocks, ledgers, activityMonitor := setupAgreement(t, numNodes, disabled, ledgerFactory) + startRound := baseLedger.NextRound() + defer cleanupFn() + + for i := 0; i < numNodes; i++ { + services[i].Start() + } + activityMonitor.waitForActivity() + activityMonitor.waitForQuiet() + zeroes := expectNewPeriod(clocks, 0) + + filterTimeouts := make([][]time.Duration, numNodes, numNodes) + + // run round with round-specific consensus version first (since fix in #1896) + zeroes = runRoundTriggerFilter(clocks, activityMonitor, zeroes) + for j := 1; j < cfg.DynamicFilterCredentialArrivalHistory+1; j++ { + for srvIdx, clock := range clocks { + delta, err := clock.(*testingClock).when(TimeoutFilter) + require.NoError(t, err) + filterTimeouts[srvIdx] = append(filterTimeouts[srvIdx], delta) + } + zeroes = runRoundTriggerFilter(clocks, activityMonitor, zeroes) + } + + for i := range clocks { + require.Len(t, filterTimeouts[i], cfg.DynamicFilterCredentialArrivalHistory) + for j := 1; j < cfg.DynamicFilterCredentialArrivalHistory-1; j++ { + require.Equal(t, filterTimeouts[i][j-1], filterTimeouts[i][j]) + } + fmt.Println(filterTimeouts[i]) + require.Less(t, filterTimeouts[i][cfg.DynamicFilterCredentialArrivalHistory-1], filterTimeouts[i][cfg.DynamicFilterCredentialArrivalHistory-2]) + } + + // force fast partition recovery into bottom + { + baseNetwork.dropAllSoftVotes() + baseNetwork.dropAllSlowNextVotes() + + triggerGlobalTimeout(FilterTimeout(0, version), TimeoutFilter, clocks, activityMonitor) + zeroes = expectNoNewPeriod(clocks, zeroes) + + triggerGlobalTimeoutType(TimeoutDeadline, clocks, activityMonitor) + zeroes = expectNoNewPeriod(clocks, zeroes) + + triggerGlobalTimeout(0, TimeoutFastRecovery, clocks, activityMonitor) // activates fast partition recovery timer + zeroes = expectNoNewPeriod(clocks, zeroes) + + triggerGlobalTimeout(firstFPR, TimeoutFastRecovery, clocks, activityMonitor) + zeroes = expectNewPeriod(clocks, zeroes) + } + + // terminate on period 1 + { + baseNetwork.repairAll() + triggerGlobalTimeout(FilterTimeout(1, version), TimeoutFilter, clocks, activityMonitor) + zeroes = expectNewPeriod(clocks, zeroes) + } + + filterTimeoutsPostRecovery := make([][]time.Duration, numNodes, numNodes) + + // run round with round-specific consensus version first (since fix in #1896) + zeroes = runRoundTriggerFilter(clocks, activityMonitor, zeroes) + for j := 1; j < cfg.DynamicFilterCredentialArrivalHistory+1; j++ { + for srvIdx, clock := range clocks { + delta, err := clock.(*testingClock).when(TimeoutFilter) + require.NoError(t, err) + filterTimeoutsPostRecovery[srvIdx] = append(filterTimeoutsPostRecovery[srvIdx], delta) + } + zeroes = runRoundTriggerFilter(clocks, activityMonitor, zeroes) + } + + for i := range clocks { + require.Len(t, filterTimeoutsPostRecovery[i], cfg.DynamicFilterCredentialArrivalHistory) + // check that history was discarded, so filter time increased back to its original default + require.Less(t, filterTimeouts[i][cfg.DynamicFilterCredentialArrivalHistory-1], filterTimeoutsPostRecovery[i][0]) + require.Equal(t, filterTimeouts[i][cfg.DynamicFilterCredentialArrivalHistory-2], filterTimeoutsPostRecovery[i][0]) + + // check that filter timeout was updated to at the end of the history window + for j := 1; j < cfg.DynamicFilterCredentialArrivalHistory-1; j++ { + require.Equal(t, filterTimeoutsPostRecovery[i][j-1], filterTimeoutsPostRecovery[i][j]) + } + fmt.Println(filterTimeoutsPostRecovery[i]) + require.Less(t, filterTimeoutsPostRecovery[i][cfg.DynamicFilterCredentialArrivalHistory-1], filterTimeoutsPostRecovery[i][cfg.DynamicFilterCredentialArrivalHistory-2]) + } + + sanityCheck(startRound, 2*round(cfg.DynamicFilterCredentialArrivalHistory+1)+1, ledgers) +} + func TestAgreementSynchronousFuture1(t *testing.T) { partitiontest.PartitionTest(t) From 34d5f94d7893ea7b3d30328d56a675cdd1b1305f Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Fri, 18 Aug 2023 15:36:09 -0400 Subject: [PATCH 108/232] rebase on master --- agreement/msgp_gen.go | 42 +++++++++++++++++++-------------------- agreement/player.go | 35 ++++++++++++-------------------- agreement/player_test.go | 6 +++--- agreement/service.go | 16 --------------- agreement/service_test.go | 6 +++--- util/timers/interface.go | 13 ------------ 6 files changed, 40 insertions(+), 78 deletions(-) diff --git a/agreement/msgp_gen.go b/agreement/msgp_gen.go index 9626cb8b59..4787d3d9bd 100644 --- a/agreement/msgp_gen.go +++ b/agreement/msgp_gen.go @@ -3949,16 +3949,16 @@ func PeriodRouterMaxSize() (s int) { func (z *player) MarshalMsg(b []byte) (o []byte) { o = msgp.Require(b, z.Msgsize()) // omitempty: check for empty values - zb0001Len := uint32(9) - var zb0001Mask uint16 /* 10 bits */ + zb0002Len := uint32(9) + var zb0002Mask uint16 /* 11 bits */ if (*z).OldDeadline == 0 { - zb0001Len-- - zb0001Mask |= 0x1 + zb0002Len-- + zb0002Mask |= 0x1 } - // variable map header, size zb0001Len - o = append(o, 0x80|uint8(zb0001Len)) - if zb0001Len != 0 { - if (zb0001Mask & 0x1) == 0 { // if not empty + // variable map header, size zb0002Len + o = append(o, 0x80|uint8(zb0002Len)) + if zb0002Len != 0 { + if (zb0002Mask & 0x1) == 0 { // if not empty // string "Deadline" o = append(o, 0xa8, 0x44, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65) o = msgp.AppendDuration(o, (*z).OldDeadline) @@ -4053,16 +4053,16 @@ func (z *player) UnmarshalMsg(bts []byte) (o []byte, err error) { (*z).LastConcluding = step(zb0006) } } - if zb0001 > 0 { - zb0001-- + if zb0002 > 0 { + zb0002-- bts, err = (*z).Deadline.UnmarshalMsg(bts) if err != nil { err = msgp.WrapError(err, "struct-from-array", "Deadline") return } } - if zb0001 > 0 { - zb0001-- + if zb0002 > 0 { + zb0002-- (*z).OldDeadline, bts, err = msgp.ReadDurationBytes(bts) if err != nil { err = msgp.WrapError(err, "struct-from-array", "OldDeadline") @@ -4124,33 +4124,33 @@ func (z *player) UnmarshalMsg(bts []byte) (o []byte, err error) { } case "Period": { - var zb0009 uint64 - zb0009, bts, err = msgp.ReadUint64Bytes(bts) + var zb0007 uint64 + zb0007, bts, err = msgp.ReadUint64Bytes(bts) if err != nil { err = msgp.WrapError(err, "Period") return } - (*z).Period = period(zb0009) + (*z).Period = period(zb0007) } case "Step": { - var zb0010 uint64 - zb0010, bts, err = msgp.ReadUint64Bytes(bts) + var zb0008 uint64 + zb0008, bts, err = msgp.ReadUint64Bytes(bts) if err != nil { err = msgp.WrapError(err, "Step") return } - (*z).Step = step(zb0010) + (*z).Step = step(zb0008) } case "LastConcluding": { - var zb0011 uint64 - zb0011, bts, err = msgp.ReadUint64Bytes(bts) + var zb0009 uint64 + zb0009, bts, err = msgp.ReadUint64Bytes(bts) if err != nil { err = msgp.WrapError(err, "LastConcluding") return } - (*z).LastConcluding = step(zb0011) + (*z).LastConcluding = step(zb0009) } case "TimersDeadline": bts, err = (*z).Deadline.UnmarshalMsg(bts) diff --git a/agreement/player.go b/agreement/player.go index 601d31cfd3..ca93693ceb 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -24,15 +24,6 @@ import ( "github.com/algorand/go-algorand/protocol" ) -// Deadline marks a timeout event of type Type that the player schedules to -// happen after Duration time. -type Deadline struct { - _struct struct{} `codec:","` - Duration time.Duration - // Type is used to allow tests fire timeouts of specific types. - Type TimeoutType -} - // The player implements the top-level state machine functionality of the // agreement protocol. type player struct { @@ -296,11 +287,8 @@ func (p *player) updateCredentialArrivalHistory(r routerHandle, ver protocol.Con // trim history to the last proto.DynamicFilterCredentialArrivalHistory samples. proto := config.Consensus[ver] - if len(p.payloadArrivals) > proto.DynamicFilterCredentialArrivalHistory { - p.payloadArrivals = p.payloadArrivals[len(p.payloadArrivals)-proto.DynamicFilterCredentialArrivalHistory:] - } - for len(p.payloadArrivals) < proto.DynamicFilterCredentialArrivalHistory { - p.payloadArrivals = append([]time.Duration{FilterTimeout(0, ver)}, p.payloadArrivals...) + if len(p.lowestCredentialArrivals) > proto.DynamicFilterCredentialArrivalHistory { + p.lowestCredentialArrivals = p.lowestCredentialArrivals[len(p.lowestCredentialArrivals)-proto.DynamicFilterCredentialArrivalHistory:] } } @@ -313,11 +301,8 @@ func (p *player) calculateFilterTimeout(ver protocol.ConsensusVersion, tracer *t return FilterTimeout(p.Period, ver) } - defaultDelay := FilterTimeout(0, ver) - if proto.DynamicFilterCredentialArrivalHistory <= 0 { - // we don't keep any history, use the default - dynamicDelay = defaultDelay - } else if proto.DynamicFilterCredentialArrivalHistory > len(p.payloadArrivals) { + defaultTimeout := FilterTimeout(0, ver) + if proto.DynamicFilterCredentialArrivalHistory > len(p.lowestCredentialArrivals) { // not enough samples, use the default return defaultTimeout } @@ -420,6 +405,12 @@ func (p *player) enterPeriod(r routerHandle, source thresholdEvent, target perio p.Step = soft p.Napping = false p.FastRecoveryDeadline = 0 // set immediately + + if target != 0 { + // We entered a non-0 period, we should reset the filter timeout + // calculation mechanism. + p.lowestCredentialArrivals = nil + } p.Deadline = Deadline{Duration: FilterTimeout(target, source.Proto), Type: TimeoutFilter} // update tracer state to match player @@ -468,11 +459,11 @@ func (p *player) enterRound(r routerHandle, source event, target round) []action switch source := source.(type) { case roundInterruptionEvent: - p.Deadline = Deadline{Duration: FilterTimeout(0, source.Proto.Version), Type: TimeoutFilter} + p.Deadline = Deadline{Duration: p.calculateFilterTimeout(source.Proto.Version, r.t), Type: TimeoutFilter} case thresholdEvent: - p.Deadline = Deadline{Duration: FilterTimeout(0, source.Proto), Type: TimeoutFilter} + p.Deadline = Deadline{Duration: p.calculateFilterTimeout(source.Proto, r.t), Type: TimeoutFilter} case filterableMessageEvent: - p.Deadline = Deadline{Duration: FilterTimeout(0, source.Proto.Version), Type: TimeoutFilter} + p.Deadline = Deadline{Duration: p.calculateFilterTimeout(source.Proto.Version, r.t), Type: TimeoutFilter} } // update tracer state to match player diff --git a/agreement/player_test.go b/agreement/player_test.go index 7d45c025a0..40ae4a68e9 100644 --- a/agreement/player_test.go +++ b/agreement/player_test.go @@ -3263,7 +3263,7 @@ func TestPlayerRetainsReceivedValidatedAtOneSample(t *testing.T) { assertCorrectReceivedAtSet(t, pWhite, pM, helper, r, p, pP, pV, m, protocol.ConsensusFuture) // assert lowest vote validateAt time was recorded into payloadArrivals - historyLen := config.Consensus[protocol.ConsensusFuture].DynamicFilterPayloadArriavalHistory + historyLen := config.Consensus[protocol.ConsensusFuture].DynamicFilterCredentialArrivalHistory require.NotZero(t, historyLen) require.Len(t, pWhite.lowestCredentialArrivals, 1) require.Equal(t, 501*time.Millisecond, pWhite.lowestCredentialArrivals[0]) @@ -3351,7 +3351,7 @@ func TestPlayerRetainsReceivedValidatedAtPPOneSample(t *testing.T) { assertCorrectReceivedAtSet(t, pWhite, pM, helper, r, p, pP, pV, proposalMsg, protocol.ConsensusFuture) // assert lowest vote validateAt time was recorded into payloadArrivals - historyLen := config.Consensus[protocol.ConsensusFuture].DynamicFilterPayloadArriavalHistory + historyLen := config.Consensus[protocol.ConsensusFuture].DynamicFilterCredentialArrivalHistory require.NotZero(t, historyLen) require.Len(t, pWhite.lowestCredentialArrivals, 1) require.Equal(t, 502*time.Millisecond, pWhite.lowestCredentialArrivals[0]) @@ -3460,7 +3460,7 @@ func TestPlayerRetainsReceivedValidatedAtAVPPOneSample(t *testing.T) { assertCorrectReceivedAtSet(t, pWhite, pM, helper, r, p, pP, pV, proposalMsg, protocol.ConsensusFuture) // assert lowest vote validateAt time was recorded into payloadArrivals - historyLen := config.Consensus[protocol.ConsensusFuture].DynamicFilterPayloadArriavalHistory + historyLen := config.Consensus[protocol.ConsensusFuture].DynamicFilterCredentialArrivalHistory require.NotZero(t, historyLen) require.Len(t, pWhite.lowestCredentialArrivals, 1) require.Equal(t, 502*time.Millisecond, pWhite.lowestCredentialArrivals[0]) diff --git a/agreement/service.go b/agreement/service.go index 0911bdd6ad..765f4270a9 100644 --- a/agreement/service.go +++ b/agreement/service.go @@ -84,22 +84,6 @@ type externalDemuxSignals struct { CurrentRound round } -// TimeoutType taggs the type of a timeout, to distinguish simultaneous timeouts -// of different types. -type TimeoutType int8 - -const ( - // TimeoutDeadline annotates timeout events in the agreement protocol (e.g., - // for receiving a block). - TimeoutDeadline TimeoutType = iota - // TimeoutFastRecovery annotates the fast recovery timeout in the agreement - // protocol. - TimeoutFastRecovery - // TimeoutFilter annotates the filter step timeout event in the agreement - // protocol. - TimeoutFilter -) - // MakeService creates a new Agreement Service instance given a set of Parameters. // // Call Start to start execution and Shutdown to finish execution. diff --git a/agreement/service_test.go b/agreement/service_test.go index 18f53b9b16..35d1d8d1d2 100644 --- a/agreement/service_test.go +++ b/agreement/service_test.go @@ -873,8 +873,8 @@ func runRound(clocks []timers.Clock[TimeoutType], activityMonitor *activityMonit triggerGlobalTimeout(filterTimeout, TimeoutFilter, clocks, activityMonitor) return expectNewPeriod(clocks, zeroes) } -func runRoundTriggerFilter(clocks []timers.Clock, activityMonitor *activityMonitor, zeroes uint) (newzeroes uint) { - triggerGlobalTimeoutType(timers.Filter, clocks, activityMonitor) +func runRoundTriggerFilter(clocks []timers.Clock[TimeoutType], activityMonitor *activityMonitor, zeroes uint) (newzeroes uint) { + triggerGlobalTimeoutType(TimeoutFilter, clocks, activityMonitor) return expectNewPeriod(clocks, zeroes) } @@ -1040,7 +1040,7 @@ func TestAgreementSynchronousFuture5_DynamicFilterRounds(t *testing.T) { return } - simulateAgreementWithConsensusVersion(t, 5, cfg.DynamicFilterPayloadArriavalHistory+20, disabled, consensusVersion) + simulateAgreementWithConsensusVersion(t, 5, cfg.DynamicFilterCredentialArrivalHistory+20, disabled, consensusVersion) } func TestDynamicFilterTimeoutResets(t *testing.T) { diff --git a/util/timers/interface.go b/util/timers/interface.go index cbca039c84..d217437e9e 100644 --- a/util/timers/interface.go +++ b/util/timers/interface.go @@ -21,19 +21,6 @@ import ( "time" ) -// TimeoutType annotates timeout events scheduled via TimeoutAt -type TimeoutType int8 - -const ( - // Deadline annotates timeout events in the agreement protocol (e.g., for - // receiving a block) - Deadline TimeoutType = iota - // FastRecovery annotates the fast recovery timeout in the agreement protocol - FastRecovery - // Filter annotates the filter step timeout event in the agreement protocol - Filter -) - // Clock provides timeout events which fire at some point after a point in time. type Clock[TimeoutType comparable] interface { // Zero returns a reset Clock. TimeoutAt channels will use the point From 2aef8afa6cff37fa8410a322437016d3dc788e1d Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Fri, 18 Aug 2023 16:02:30 -0400 Subject: [PATCH 109/232] remove unused variable --- agreement/proposal.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/agreement/proposal.go b/agreement/proposal.go index f2dde4bd6a..bf021f2cfe 100644 --- a/agreement/proposal.go +++ b/agreement/proposal.go @@ -99,8 +99,6 @@ type proposal struct { // validated (and thus was ready to be delivered to the state // machine), relative to the zero of that round. validatedAt time.Duration - - voteValidatedAt time.Duration } func makeProposal(ve ValidatedBlock, pf crypto.VrfProof, origPer period, origProp basics.Address) proposal { From 4af3e2e855b0e87e1dc5c43bb4036a8ce362ecda Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Fri, 18 Aug 2023 16:24:22 -0400 Subject: [PATCH 110/232] remove unneeded checks --- agreement/player.go | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/agreement/player.go b/agreement/player.go index 3b9f31459d..b27a58c139 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -313,16 +313,12 @@ func (p *player) calculateFilterTimeout(ver protocol.ConsensusVersion, tracer *t // Make sure the dynamic filter timeout is not too small nor too large if dynamicTimeout < proto.DynamicFilterTimeoutLowerBound { - if tracer != nil { - tracer.log.Debugf("Calculated dynamic filter timeout = %v for round %v, period %v. It's too low, setting to %v\n", dynamicTimeout, p.Round, p.Period, proto.DynamicFilterTimeoutLowerBound) - } + tracer.log.Debugf("Calculated dynamic filter timeout = %v for round %v, period %v. It's too low, setting to %v.\n", dynamicTimeout, p.Round, p.Period, proto.DynamicFilterTimeoutLowerBound) dynamicTimeout = proto.DynamicFilterTimeoutLowerBound } else if dynamicTimeout > defaultTimeout { - if tracer != nil { - tracer.log.Debugf("Calculated dynamic filter timeout = %v for round %v, period %v. It too high, so using the default %v\n", dynamicTimeout, p.Round, p.Period, defaultTimeout) - } + tracer.log.Debugf("Calculated dynamic filter timeout = %v for round %v, period %v. It's too high, using the default %v.\n", dynamicTimeout, p.Round, p.Period, defaultTimeout) dynamicTimeout = defaultTimeout - } else if tracer != nil { + } else { tracer.log.Debugf("Calculated dynamic filter timeout = %v for round %v, period %v.\n", dynamicTimeout, p.Round, p.Period) } From 7b0b9d2ca6d634f7ea56c2c1693d8e7e8a5c7eab Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Fri, 18 Aug 2023 17:06:22 -0400 Subject: [PATCH 111/232] shorter clamping --- agreement/player.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/agreement/player.go b/agreement/player.go index b27a58c139..7515a0e5e8 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -313,14 +313,12 @@ func (p *player) calculateFilterTimeout(ver protocol.ConsensusVersion, tracer *t // Make sure the dynamic filter timeout is not too small nor too large if dynamicTimeout < proto.DynamicFilterTimeoutLowerBound { - tracer.log.Debugf("Calculated dynamic filter timeout = %v for round %v, period %v. It's too low, setting to %v.\n", dynamicTimeout, p.Round, p.Period, proto.DynamicFilterTimeoutLowerBound) dynamicTimeout = proto.DynamicFilterTimeoutLowerBound - } else if dynamicTimeout > defaultTimeout { - tracer.log.Debugf("Calculated dynamic filter timeout = %v for round %v, period %v. It's too high, using the default %v.\n", dynamicTimeout, p.Round, p.Period, defaultTimeout) + } + if dynamicTimeout > defaultTimeout { dynamicTimeout = defaultTimeout - } else { - tracer.log.Debugf("Calculated dynamic filter timeout = %v for round %v, period %v.\n", dynamicTimeout, p.Round, p.Period) } + tracer.log.Debugf("round %d, period %d: dynamicTimeout = %d, clamped timeout = %d", p.Round, p.Period, dynamicTimeout, timeout) if !proto.DynamicFilterTimeout { // If the dynamic filter timeout is disabled, return the default filter From d47671e207c9150c18f170eb1935f2d10b23a0b2 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Sun, 20 Aug 2023 21:01:30 -0400 Subject: [PATCH 112/232] handle unfilled votes --- agreement/events.go | 3 +++ agreement/msgp_gen.go | 2 +- agreement/player.go | 3 +++ agreement/proposalTracker.go | 1 + 4 files changed, 8 insertions(+), 1 deletion(-) diff --git a/agreement/events.go b/agreement/events.go index 29341f3776..5bfd104936 100644 --- a/agreement/events.go +++ b/agreement/events.go @@ -425,6 +425,9 @@ type readLowestEvent struct { // Vote holds the lowest-credential vote. Vote vote + + // Filled indicates whether the Vote field is filled + Filled bool } func (e readLowestEvent) t() eventType { diff --git a/agreement/msgp_gen.go b/agreement/msgp_gen.go index 4787d3d9bd..53ee0d984f 100644 --- a/agreement/msgp_gen.go +++ b/agreement/msgp_gen.go @@ -4222,7 +4222,7 @@ func (z *proposal) MarshalMsg(b []byte) (o []byte) { o = msgp.Require(b, z.Msgsize()) // omitempty: check for empty values zb0004Len := uint32(29) - var zb0004Mask uint64 /* 39 bits */ + var zb0004Mask uint64 /* 38 bits */ if (*z).unauthenticatedProposal.Block.BlockHeader.RewardsState.RewardsLevel == 0 { zb0004Len-- zb0004Mask |= 0x40 diff --git a/agreement/player.go b/agreement/player.go index 7515a0e5e8..0adc27ab27 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -282,6 +282,9 @@ func (p *player) updateCredentialArrivalHistory(r routerHandle, ver protocol.Con // look up the validatedAt time of the winning proposal-vote re := readLowestEvent{T: readLowestVote, Round: p.Round, Period: p.Period} re = r.dispatch(*p, re, proposalMachineRound, p.Round, p.Period, 0).(readLowestEvent) + if !re.Filled { + return + } p.lowestCredentialArrivals = append(p.lowestCredentialArrivals, re.Vote.validatedAt) diff --git a/agreement/proposalTracker.go b/agreement/proposalTracker.go index b9707d54dc..8e29b27cfc 100644 --- a/agreement/proposalTracker.go +++ b/agreement/proposalTracker.go @@ -168,6 +168,7 @@ func (t *proposalTracker) handle(r routerHandle, p player, e event) event { case readLowestVote: e := e.(readLowestEvent) e.Vote = t.Freezer.Lowest + e.Filled = t.Freezer.Filled return e case softThreshold, certThreshold: From be77e7e222eedcda609d3bd45275be71442c0388 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Mon, 21 Aug 2023 06:45:30 -0400 Subject: [PATCH 113/232] no printing --- agreement/service_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/agreement/service_test.go b/agreement/service_test.go index bfd33d59ea..b5960437bb 100644 --- a/agreement/service_test.go +++ b/agreement/service_test.go @@ -1111,7 +1111,6 @@ func TestDynamicFilterTimeoutResets(t *testing.T) { for j := 1; j < cfg.DynamicFilterCredentialArrivalHistory-1; j++ { require.Equal(t, filterTimeouts[i][j-1], filterTimeouts[i][j]) } - fmt.Println(filterTimeouts[i]) require.Less(t, filterTimeouts[i][cfg.DynamicFilterCredentialArrivalHistory-1], filterTimeouts[i][cfg.DynamicFilterCredentialArrivalHistory-2]) } @@ -1163,7 +1162,6 @@ func TestDynamicFilterTimeoutResets(t *testing.T) { for j := 1; j < cfg.DynamicFilterCredentialArrivalHistory-1; j++ { require.Equal(t, filterTimeoutsPostRecovery[i][j-1], filterTimeoutsPostRecovery[i][j]) } - fmt.Println(filterTimeoutsPostRecovery[i]) require.Less(t, filterTimeoutsPostRecovery[i][cfg.DynamicFilterCredentialArrivalHistory-1], filterTimeoutsPostRecovery[i][cfg.DynamicFilterCredentialArrivalHistory-2]) } From b84d3abfbfcaba9af3de079e0b3275f7503490c3 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Mon, 21 Aug 2023 06:48:53 -0400 Subject: [PATCH 114/232] logging the right timeout --- agreement/player.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/agreement/player.go b/agreement/player.go index 0adc27ab27..b1b07bae33 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -315,13 +315,15 @@ func (p *player) calculateFilterTimeout(ver protocol.ConsensusVersion, tracer *t dynamicTimeout := sortedArrivals[proto.DynamicFilterTimeoutCredentialArrivalHistoryIdx] + proto.DynamicFilterTimeoutGraceInterval // Make sure the dynamic filter timeout is not too small nor too large + clampedTimeout := dynamicTimeout if dynamicTimeout < proto.DynamicFilterTimeoutLowerBound { - dynamicTimeout = proto.DynamicFilterTimeoutLowerBound + clampedTimeout = proto.DynamicFilterTimeoutLowerBound } if dynamicTimeout > defaultTimeout { - dynamicTimeout = defaultTimeout + clampedTimeout = defaultTimeout } - tracer.log.Debugf("round %d, period %d: dynamicTimeout = %d, clamped timeout = %d", p.Round, p.Period, dynamicTimeout, timeout) + tracer.log.Debugf("round %d, period %d: dynamicTimeout = %d, clamped timeout = %d", p.Round, p.Period, dynamicTimeout, clampedTimeout) + dynamicTimeout = clampedTimeout if !proto.DynamicFilterTimeout { // If the dynamic filter timeout is disabled, return the default filter From fe288a34380778d0e31008cb15eb3fc13686f57e Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Mon, 21 Aug 2023 10:48:33 -0400 Subject: [PATCH 115/232] fix --- agreement/player.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/agreement/player.go b/agreement/player.go index b1b07bae33..ffa89c9b0c 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -316,10 +316,10 @@ func (p *player) calculateFilterTimeout(ver protocol.ConsensusVersion, tracer *t // Make sure the dynamic filter timeout is not too small nor too large clampedTimeout := dynamicTimeout - if dynamicTimeout < proto.DynamicFilterTimeoutLowerBound { + if clampedTimeout < proto.DynamicFilterTimeoutLowerBound { clampedTimeout = proto.DynamicFilterTimeoutLowerBound } - if dynamicTimeout > defaultTimeout { + if clampedTimeout > defaultTimeout { clampedTimeout = defaultTimeout } tracer.log.Debugf("round %d, period %d: dynamicTimeout = %d, clamped timeout = %d", p.Round, p.Period, dynamicTimeout, clampedTimeout) From a9ef051223831c16d733ccf43c4baaf16bd6029b Mon Sep 17 00:00:00 2001 From: chris erway Date: Mon, 21 Aug 2023 11:29:38 -0400 Subject: [PATCH 116/232] add voteValidatedAt to telemetry --- agreement/actions.go | 21 +++++++++++++-------- agreement/player.go | 13 ++++++++----- logging/telemetryspec/event.go | 18 ++++++++++-------- 3 files changed, 31 insertions(+), 21 deletions(-) diff --git a/agreement/actions.go b/agreement/actions.go index 833dc8af36..dfe436334c 100644 --- a/agreement/actions.go +++ b/agreement/actions.go @@ -19,6 +19,7 @@ package agreement import ( "context" "fmt" + "time" "github.com/algorand/go-algorand/logging/logspec" "github.com/algorand/go-algorand/logging/telemetryspec" @@ -209,6 +210,9 @@ type ensureAction struct { Payload proposal // the certificate proving commitment Certificate Certificate + + // The time that the winning proposal-vote was validated, relative to the beginning of the round + voteValidatedAt time.Duration } func (a ensureAction) t() actionType { @@ -231,14 +235,15 @@ func (a ensureAction) do(ctx context.Context, s *Service) { logEvent.Type = logspec.RoundConcluded s.log.with(logEvent).Infof("committed round %d with pre-validated block %v", a.Certificate.Round, a.Certificate.Proposal) s.log.EventWithDetails(telemetryspec.Agreement, telemetryspec.BlockAcceptedEvent, telemetryspec.BlockAcceptedEventDetails{ - Address: a.Certificate.Proposal.OriginalProposer.String(), - Hash: a.Certificate.Proposal.BlockDigest.String(), - Round: uint64(a.Certificate.Round), - ValidatedAt: a.Payload.validatedAt, - ReceivedAt: a.Payload.receivedAt, - PreValidated: true, - PropBufLen: uint64(len(s.demux.rawProposals)), - VoteBufLen: uint64(len(s.demux.rawVotes)), + Address: a.Certificate.Proposal.OriginalProposer.String(), + Hash: a.Certificate.Proposal.BlockDigest.String(), + Round: uint64(a.Certificate.Round), + ValidatedAt: a.Payload.validatedAt, + ReceivedAt: a.Payload.receivedAt, + VoteValidatedAt: a.voteValidatedAt, + PreValidated: true, + PropBufLen: uint64(len(s.demux.rawProposals)), + VoteBufLen: uint64(len(s.demux.rawVotes)), }) s.Ledger.EnsureValidatedBlock(a.Payload.ve, a.Certificate) } else { diff --git a/agreement/player.go b/agreement/player.go index ffa89c9b0c..eb79bb18de 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -274,16 +274,18 @@ func (p *player) handleCheckpointEvent(r routerHandle, e checkpointEvent) []acti // updateCredentialArrivalHistory is called at the end of a successful // uninterrupted round (just after ensureAction is generated) to collect // credential arrival times to dynamically set the filter timeout. -func (p *player) updateCredentialArrivalHistory(r routerHandle, ver protocol.ConsensusVersion) { +// It returns the time of the lowest credential's arrival, if one was +// collected and added to lowestCredentialArrivals, or zero otherwise. +func (p *player) updateCredentialArrivalHistory(r routerHandle, ver protocol.ConsensusVersion) time.Duration { // only append to lowestCredentialArrivals if this was a successful round completing in period 0. if p.Period != 0 { - return + return 0 } // look up the validatedAt time of the winning proposal-vote re := readLowestEvent{T: readLowestVote, Round: p.Round, Period: p.Period} re = r.dispatch(*p, re, proposalMachineRound, p.Round, p.Period, 0).(readLowestEvent) if !re.Filled { - return + return 0 } p.lowestCredentialArrivals = append(p.lowestCredentialArrivals, re.Vote.validatedAt) @@ -293,6 +295,7 @@ func (p *player) updateCredentialArrivalHistory(r routerHandle, ver protocol.Con if len(p.lowestCredentialArrivals) > proto.DynamicFilterCredentialArrivalHistory { p.lowestCredentialArrivals = p.lowestCredentialArrivals[len(p.lowestCredentialArrivals)-proto.DynamicFilterCredentialArrivalHistory:] } + return re.Vote.validatedAt } // calculateFilterTimeout chooses the appropriate filter timeout. @@ -349,8 +352,8 @@ func (p *player) handleThresholdEvent(r routerHandle, e thresholdEvent) []action if res.Committable { cert := Certificate(e.Bundle) a0 := ensureAction{Payload: res.Payload, Certificate: cert} + a0.voteValidatedAt = p.updateCredentialArrivalHistory(r, e.Proto) actions = append(actions, a0) - p.updateCredentialArrivalHistory(r, e.Proto) as := p.enterRound(r, e, p.Round+1) return append(actions, as...) } @@ -675,8 +678,8 @@ func (p *player) handleMessageEvent(r routerHandle, e messageEvent) (actions []a if freshestRes.Ok && freshestRes.Event.t() == certThreshold && freshestRes.Event.Proposal == e.Input.Proposal.value() { cert := Certificate(freshestRes.Event.Bundle) a0 := ensureAction{Payload: e.Input.Proposal, Certificate: cert} + a0.voteValidatedAt = p.updateCredentialArrivalHistory(r, e.Proto.Version) actions = append(actions, a0) - p.updateCredentialArrivalHistory(r, e.Proto.Version) as := p.enterRound(r, delegatedE, cert.Round+1) return append(actions, as...) } diff --git a/logging/telemetryspec/event.go b/logging/telemetryspec/event.go index 4a8280591c..1617b0339c 100644 --- a/logging/telemetryspec/event.go +++ b/logging/telemetryspec/event.go @@ -89,14 +89,16 @@ const BlockAcceptedEvent Event = "BlockAccepted" // BlockAcceptedEventDetails contains details for the BlockAcceptedEvent type BlockAcceptedEventDetails struct { - Address string - Hash string - Round uint64 - ValidatedAt time.Duration - ReceivedAt time.Duration - PreValidated bool - PropBufLen uint64 - VoteBufLen uint64 + Address string + Hash string + Round uint64 + ValidatedAt time.Duration + ReceivedAt time.Duration + VoteValidatedAt time.Duration + DynamicFilterTimeout time.Duration + PreValidated bool + PropBufLen uint64 + VoteBufLen uint64 } // AccountRegisteredEvent event From 950de96aade58368e4d3aa14987a03066a302e13 Mon Sep 17 00:00:00 2001 From: chris erway Date: Tue, 22 Aug 2023 14:49:02 -0400 Subject: [PATCH 117/232] add dynamicFilterTimeout to BlockAcceptedEvent --- agreement/actions.go | 39 ++++++++++++++++++++++----------------- agreement/player.go | 9 +++++++++ 2 files changed, 31 insertions(+), 17 deletions(-) diff --git a/agreement/actions.go b/agreement/actions.go index dfe436334c..91276e22fc 100644 --- a/agreement/actions.go +++ b/agreement/actions.go @@ -213,6 +213,8 @@ type ensureAction struct { // The time that the winning proposal-vote was validated, relative to the beginning of the round voteValidatedAt time.Duration + // The dynamic filter timeout calculated for this round, even if not enabled, for reporting to telemetry. + dynamicFilterTimeout time.Duration } func (a ensureAction) t() actionType { @@ -235,15 +237,16 @@ func (a ensureAction) do(ctx context.Context, s *Service) { logEvent.Type = logspec.RoundConcluded s.log.with(logEvent).Infof("committed round %d with pre-validated block %v", a.Certificate.Round, a.Certificate.Proposal) s.log.EventWithDetails(telemetryspec.Agreement, telemetryspec.BlockAcceptedEvent, telemetryspec.BlockAcceptedEventDetails{ - Address: a.Certificate.Proposal.OriginalProposer.String(), - Hash: a.Certificate.Proposal.BlockDigest.String(), - Round: uint64(a.Certificate.Round), - ValidatedAt: a.Payload.validatedAt, - ReceivedAt: a.Payload.receivedAt, - VoteValidatedAt: a.voteValidatedAt, - PreValidated: true, - PropBufLen: uint64(len(s.demux.rawProposals)), - VoteBufLen: uint64(len(s.demux.rawVotes)), + Address: a.Certificate.Proposal.OriginalProposer.String(), + Hash: a.Certificate.Proposal.BlockDigest.String(), + Round: uint64(a.Certificate.Round), + ValidatedAt: a.Payload.validatedAt, + ReceivedAt: a.Payload.receivedAt, + VoteValidatedAt: a.voteValidatedAt, + DynamicFilterTimeout: a.dynamicFilterTimeout, + PreValidated: true, + PropBufLen: uint64(len(s.demux.rawProposals)), + VoteBufLen: uint64(len(s.demux.rawVotes)), }) s.Ledger.EnsureValidatedBlock(a.Payload.ve, a.Certificate) } else { @@ -251,14 +254,16 @@ func (a ensureAction) do(ctx context.Context, s *Service) { logEvent.Type = logspec.RoundConcluded s.log.with(logEvent).Infof("committed round %d with block %v", a.Certificate.Round, a.Certificate.Proposal) s.log.EventWithDetails(telemetryspec.Agreement, telemetryspec.BlockAcceptedEvent, telemetryspec.BlockAcceptedEventDetails{ - Address: a.Certificate.Proposal.OriginalProposer.String(), - Hash: a.Certificate.Proposal.BlockDigest.String(), - Round: uint64(a.Certificate.Round), - ValidatedAt: a.Payload.validatedAt, - ReceivedAt: a.Payload.receivedAt, - PreValidated: false, - PropBufLen: uint64(len(s.demux.rawProposals)), - VoteBufLen: uint64(len(s.demux.rawVotes)), + Address: a.Certificate.Proposal.OriginalProposer.String(), + Hash: a.Certificate.Proposal.BlockDigest.String(), + Round: uint64(a.Certificate.Round), + ValidatedAt: a.Payload.validatedAt, + ReceivedAt: a.Payload.receivedAt, + VoteValidatedAt: a.voteValidatedAt, + DynamicFilterTimeout: a.dynamicFilterTimeout, + PreValidated: false, + PropBufLen: uint64(len(s.demux.rawProposals)), + VoteBufLen: uint64(len(s.demux.rawVotes)), }) s.Ledger.EnsureBlock(block, a.Certificate) } diff --git a/agreement/player.go b/agreement/player.go index eb79bb18de..6be60db329 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -63,6 +63,11 @@ type player struct { // the history of arrival times of the lowest credential from previous // ronuds, used for calculating the filter timeout dynamically. lowestCredentialArrivals []time.Duration + + // The period 0 dynamic filter timeout calculated for this round, if set, + // even if dynamic filter timeouts are not enabled. It is used for reporting + // to telemetry. + dynamicFilterTimeout time.Duration } func (p *player) T() stateMachineTag { @@ -327,6 +332,8 @@ func (p *player) calculateFilterTimeout(ver protocol.ConsensusVersion, tracer *t } tracer.log.Debugf("round %d, period %d: dynamicTimeout = %d, clamped timeout = %d", p.Round, p.Period, dynamicTimeout, clampedTimeout) dynamicTimeout = clampedTimeout + // store dynamicFilterTimeout on the player for debugging & reporting + p.dynamicFilterTimeout = dynamicTimeout if !proto.DynamicFilterTimeout { // If the dynamic filter timeout is disabled, return the default filter @@ -353,6 +360,7 @@ func (p *player) handleThresholdEvent(r routerHandle, e thresholdEvent) []action cert := Certificate(e.Bundle) a0 := ensureAction{Payload: res.Payload, Certificate: cert} a0.voteValidatedAt = p.updateCredentialArrivalHistory(r, e.Proto) + a0.dynamicFilterTimeout = p.dynamicFilterTimeout actions = append(actions, a0) as := p.enterRound(r, e, p.Round+1) return append(actions, as...) @@ -679,6 +687,7 @@ func (p *player) handleMessageEvent(r routerHandle, e messageEvent) (actions []a cert := Certificate(freshestRes.Event.Bundle) a0 := ensureAction{Payload: e.Input.Proposal, Certificate: cert} a0.voteValidatedAt = p.updateCredentialArrivalHistory(r, e.Proto.Version) + a0.dynamicFilterTimeout = p.dynamicFilterTimeout actions = append(actions, a0) as := p.enterRound(r, delegatedE, cert.Round+1) return append(actions, as...) From ba6a55ac8ba260ec031bde21cb1a29ef97644b62 Mon Sep 17 00:00:00 2001 From: chris erway Date: Tue, 22 Aug 2023 14:51:53 -0400 Subject: [PATCH 118/232] rerun make msgp --- agreement/msgp_gen.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agreement/msgp_gen.go b/agreement/msgp_gen.go index 53ee0d984f..6c293e4edb 100644 --- a/agreement/msgp_gen.go +++ b/agreement/msgp_gen.go @@ -3950,7 +3950,7 @@ func (z *player) MarshalMsg(b []byte) (o []byte) { o = msgp.Require(b, z.Msgsize()) // omitempty: check for empty values zb0002Len := uint32(9) - var zb0002Mask uint16 /* 11 bits */ + var zb0002Mask uint16 /* 12 bits */ if (*z).OldDeadline == 0 { zb0002Len-- zb0002Mask |= 0x1 From 70163774d658a0b5ef516ae6fc38b2a3104e5574 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Tue, 22 Aug 2023 17:29:17 -0400 Subject: [PATCH 119/232] move dynamic filter timeout parameters to consts --- agreement/dynamicFilterTimeoutParams.go | 28 ++++++++++++++++ agreement/player.go | 17 +++++----- agreement/player_test.go | 39 +++++++++------------- agreement/service_test.go | 44 +++++++++++-------------- config/consensus.go | 27 --------------- 5 files changed, 71 insertions(+), 84 deletions(-) create mode 100644 agreement/dynamicFilterTimeoutParams.go diff --git a/agreement/dynamicFilterTimeoutParams.go b/agreement/dynamicFilterTimeoutParams.go new file mode 100644 index 0000000000..d749f730d0 --- /dev/null +++ b/agreement/dynamicFilterTimeoutParams.go @@ -0,0 +1,28 @@ +package agreement + +import "time" + +// This file contains parameters for the dynamic filter timeout mechanism. When +// this feature is enabled (dynamicFilterTimeout is true), these parameters +// should migrate to be consensus params. + +// DynamicFilterCredentialArrivalHistory specifies the number of past +// credential arrivals that are measured to determine the next filter +// timeout. If DynamicFilterCredentialArrivalHistory <= 0, then the dynamic +// timeout feature is off and the filter step timeout is calculated using +// the static configuration. +const dynamicFilterCredentialArrivalHistory int = 40 + +// DynamicFilterTimeoutLowerBound specifies a minimal duration that the +// filter timeout must meet. +const dynamicFilterTimeoutLowerBound time.Duration = 600 * time.Millisecond + +// DynamicFilterTimeoutCredentialArrivalHistoryIdx specified which sample to +// use out of a sorted DynamicFilterCredentialArrivalHistory-sized array of +// time samples. +const dynamicFilterTimeoutCredentialArrivalHistoryIdx int = 37 + +// DynamicFilterTimeoutGraceInterval is additional extension to the dynamic +// filter time atop the one calculated based on the history of credential +// arrivals. +const dynamicFilterTimeoutGraceInterval time.Duration = 50 * time.Millisecond diff --git a/agreement/player.go b/agreement/player.go index 6be60db329..3d726a2043 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -295,10 +295,9 @@ func (p *player) updateCredentialArrivalHistory(r routerHandle, ver protocol.Con p.lowestCredentialArrivals = append(p.lowestCredentialArrivals, re.Vote.validatedAt) - // trim history to the last proto.DynamicFilterCredentialArrivalHistory samples. - proto := config.Consensus[ver] - if len(p.lowestCredentialArrivals) > proto.DynamicFilterCredentialArrivalHistory { - p.lowestCredentialArrivals = p.lowestCredentialArrivals[len(p.lowestCredentialArrivals)-proto.DynamicFilterCredentialArrivalHistory:] + // trim history to the last dynamicFilterCredentialArrivalHistory samples. + if len(p.lowestCredentialArrivals) > dynamicFilterCredentialArrivalHistory { + p.lowestCredentialArrivals = p.lowestCredentialArrivals[len(p.lowestCredentialArrivals)-dynamicFilterCredentialArrivalHistory:] } return re.Vote.validatedAt } @@ -306,13 +305,13 @@ func (p *player) updateCredentialArrivalHistory(r routerHandle, ver protocol.Con // calculateFilterTimeout chooses the appropriate filter timeout. func (p *player) calculateFilterTimeout(ver protocol.ConsensusVersion, tracer *tracer) time.Duration { proto := config.Consensus[ver] - if proto.DynamicFilterCredentialArrivalHistory <= 0 || p.Period != 0 { + if dynamicFilterCredentialArrivalHistory <= 0 || p.Period != 0 { // Either dynamic filter timeout is disabled, or we're not in period 0 // and therefore, can't use dynamic timeout return FilterTimeout(p.Period, ver) } defaultTimeout := FilterTimeout(0, ver) - if proto.DynamicFilterCredentialArrivalHistory > len(p.lowestCredentialArrivals) { + if dynamicFilterCredentialArrivalHistory > len(p.lowestCredentialArrivals) { // not enough samples, use the default return defaultTimeout } @@ -320,12 +319,12 @@ func (p *player) calculateFilterTimeout(ver protocol.ConsensusVersion, tracer *t sortedArrivals := make([]time.Duration, len(p.lowestCredentialArrivals)) copy(sortedArrivals[:], p.lowestCredentialArrivals[:]) sort.Slice(sortedArrivals, func(i, j int) bool { return sortedArrivals[i] < sortedArrivals[j] }) - dynamicTimeout := sortedArrivals[proto.DynamicFilterTimeoutCredentialArrivalHistoryIdx] + proto.DynamicFilterTimeoutGraceInterval + dynamicTimeout := sortedArrivals[dynamicFilterTimeoutCredentialArrivalHistoryIdx] + dynamicFilterTimeoutGraceInterval // Make sure the dynamic filter timeout is not too small nor too large clampedTimeout := dynamicTimeout - if clampedTimeout < proto.DynamicFilterTimeoutLowerBound { - clampedTimeout = proto.DynamicFilterTimeoutLowerBound + if clampedTimeout < dynamicFilterTimeoutLowerBound { + clampedTimeout = dynamicFilterTimeoutLowerBound } if clampedTimeout > defaultTimeout { clampedTimeout = defaultTimeout diff --git a/agreement/player_test.go b/agreement/player_test.go index 40ae4a68e9..ba4fd23288 100644 --- a/agreement/player_test.go +++ b/agreement/player_test.go @@ -3263,8 +3263,7 @@ func TestPlayerRetainsReceivedValidatedAtOneSample(t *testing.T) { assertCorrectReceivedAtSet(t, pWhite, pM, helper, r, p, pP, pV, m, protocol.ConsensusFuture) // assert lowest vote validateAt time was recorded into payloadArrivals - historyLen := config.Consensus[protocol.ConsensusFuture].DynamicFilterCredentialArrivalHistory - require.NotZero(t, historyLen) + require.NotZero(t, dynamicFilterCredentialArrivalHistory) require.Len(t, pWhite.lowestCredentialArrivals, 1) require.Equal(t, 501*time.Millisecond, pWhite.lowestCredentialArrivals[0]) } @@ -3275,15 +3274,13 @@ func TestPlayerRetainsReceivedValidatedAtOneSample(t *testing.T) { // DynamicFilterCredentialArrivalHistory rounds. func TestPlayerRetainsReceivedValidatedAtForHistoryWindow(t *testing.T) { partitiontest.PartitionTest(t) - const r = round(20239) const p = period(0) pWhite, pM, helper := setupP(t, r-1, p, soft) - historyLen := config.Consensus[protocol.ConsensusFuture].DynamicFilterCredentialArrivalHistory - require.NotZero(t, historyLen) + require.NotZero(t, dynamicFilterCredentialArrivalHistory) - for i := 0; i < historyLen+1; i++ { + for i := 0; i < dynamicFilterCredentialArrivalHistory+1; i++ { // send voteVerified message pP, pV := helper.MakeRandomProposalPayload(t, r+round(i)-1) vVote := helper.MakeVerifiedVote(t, 0, r+round(i)-1, p, propose, *pV) @@ -3305,8 +3302,8 @@ func TestPlayerRetainsReceivedValidatedAtForHistoryWindow(t *testing.T) { } // assert lowest vote validateAt time was recorded into payloadArrivals - require.Len(t, pWhite.lowestCredentialArrivals, historyLen) - for i := 0; i < historyLen; i++ { + require.Len(t, pWhite.lowestCredentialArrivals, dynamicFilterCredentialArrivalHistory) + for i := 0; i < dynamicFilterCredentialArrivalHistory; i++ { // only the last historyLen samples are kept, so the first one is discarded timestamp := 500 + i + 1 require.Equal(t, time.Duration(timestamp)*time.Millisecond, pWhite.lowestCredentialArrivals[i]) @@ -3351,8 +3348,7 @@ func TestPlayerRetainsReceivedValidatedAtPPOneSample(t *testing.T) { assertCorrectReceivedAtSet(t, pWhite, pM, helper, r, p, pP, pV, proposalMsg, protocol.ConsensusFuture) // assert lowest vote validateAt time was recorded into payloadArrivals - historyLen := config.Consensus[protocol.ConsensusFuture].DynamicFilterCredentialArrivalHistory - require.NotZero(t, historyLen) + require.NotZero(t, dynamicFilterCredentialArrivalHistory) require.Len(t, pWhite.lowestCredentialArrivals, 1) require.Equal(t, 502*time.Millisecond, pWhite.lowestCredentialArrivals[0]) } @@ -3365,16 +3361,14 @@ func TestPlayerRetainsReceivedValidatedAtPPOneSample(t *testing.T) { // DynamicFilterCredentialArrivalHistory rounds. func TestPlayerRetainsReceivedValidatedAtPPForHistoryWindow(t *testing.T) { partitiontest.PartitionTest(t) - const r = round(20239) const p = period(0) pWhite, pM, helper := setupP(t, r-1, p, soft) pP, pV := helper.MakeRandomProposalPayload(t, r-1) - historyLen := config.Consensus[protocol.ConsensusFuture].DynamicFilterCredentialArrivalHistory - require.NotZero(t, historyLen) + require.NotZero(t, dynamicFilterCredentialArrivalHistory) - for i := 0; i < historyLen+1; i++ { + for i := 0; i < dynamicFilterCredentialArrivalHistory+1; i++ { // create a PP message for an arbitrary proposal/payload similar to setupCompoundMessage vVote := helper.MakeVerifiedVote(t, 0, r+round(i)-1, p, propose, *pV) unverifiedVoteMsg := message{UnauthenticatedVote: vVote.u()} @@ -3403,8 +3397,8 @@ func TestPlayerRetainsReceivedValidatedAtPPForHistoryWindow(t *testing.T) { } // assert lowest vote validateAt time was recorded into payloadArrivals - require.Len(t, pWhite.lowestCredentialArrivals, historyLen) - for i := 0; i < historyLen; i++ { + require.Len(t, pWhite.lowestCredentialArrivals, dynamicFilterCredentialArrivalHistory) + for i := 0; i < dynamicFilterCredentialArrivalHistory; i++ { // only the last historyLen samples are kept, so the first one is discarded timestamp := 500 + i + 1 require.Equal(t, time.Duration(timestamp)*time.Millisecond, pWhite.lowestCredentialArrivals[i]) @@ -3460,23 +3454,20 @@ func TestPlayerRetainsReceivedValidatedAtAVPPOneSample(t *testing.T) { assertCorrectReceivedAtSet(t, pWhite, pM, helper, r, p, pP, pV, proposalMsg, protocol.ConsensusFuture) // assert lowest vote validateAt time was recorded into payloadArrivals - historyLen := config.Consensus[protocol.ConsensusFuture].DynamicFilterCredentialArrivalHistory - require.NotZero(t, historyLen) + require.NotZero(t, dynamicFilterCredentialArrivalHistory) require.Len(t, pWhite.lowestCredentialArrivals, 1) require.Equal(t, 502*time.Millisecond, pWhite.lowestCredentialArrivals[0]) } func TestPlayerRetainsReceivedValidatedAtAVPPHistoryWindow(t *testing.T) { partitiontest.PartitionTest(t) - const r = round(20239) const p = period(0) pWhite, pM, helper := setupP(t, r-1, p, soft) - historyLen := config.Consensus[protocol.ConsensusFuture].DynamicFilterCredentialArrivalHistory - require.NotZero(t, historyLen) + require.NotZero(t, dynamicFilterCredentialArrivalHistory) - for i := 0; i < historyLen+1; i++ { + for i := 0; i < dynamicFilterCredentialArrivalHistory+1; i++ { pP, pV := helper.MakeRandomProposalPayload(t, r+round(i)-1) // send votePresent message (mimicking the first AV message validating) @@ -3513,8 +3504,8 @@ func TestPlayerRetainsReceivedValidatedAtAVPPHistoryWindow(t *testing.T) { } // assert lowest vote validateAt time was recorded into payloadArrivals - require.Len(t, pWhite.lowestCredentialArrivals, historyLen) - for i := 0; i < historyLen; i++ { + require.Len(t, pWhite.lowestCredentialArrivals, dynamicFilterCredentialArrivalHistory) + for i := 0; i < dynamicFilterCredentialArrivalHistory; i++ { // only the last historyLen samples are kept, so the first one is discarded timestamp := 500 + i + 1 require.Equal(t, time.Duration(timestamp)*time.Millisecond, pWhite.lowestCredentialArrivals[i]) diff --git a/agreement/service_test.go b/agreement/service_test.go index b5960437bb..6705b629c7 100644 --- a/agreement/service_test.go +++ b/agreement/service_test.go @@ -1033,25 +1033,23 @@ func TestAgreementSynchronousFuture5_DynamicFilterRounds(t *testing.T) { } cfg := config.Consensus[protocol.ConsensusFuture] - if cfg.DynamicFilterCredentialArrivalHistory <= 0 { - return - } - if !cfg.DynamicFilterTimeout { + require.True(t, cfg.DynamicFilterTimeout) + if dynamicFilterCredentialArrivalHistory <= 0 { return } - rounds := cfg.DynamicFilterCredentialArrivalHistory + 20 + rounds := dynamicFilterCredentialArrivalHistory + 20 filterTimeouts := simulateAgreementWithConsensusVersion(t, 5, rounds, disabled, consensusVersion) require.Len(t, filterTimeouts, rounds-1) - for i := 1; i < cfg.DynamicFilterCredentialArrivalHistory-1; i++ { + for i := 1; i < dynamicFilterCredentialArrivalHistory-1; i++ { require.Equal(t, filterTimeouts[i-1], filterTimeouts[i]) } // dynamic filter timeout kicks in when history window is full - require.Less(t, filterTimeouts[cfg.DynamicFilterCredentialArrivalHistory-1], filterTimeouts[cfg.DynamicFilterCredentialArrivalHistory-2]) + require.Less(t, filterTimeouts[dynamicFilterCredentialArrivalHistory-1], filterTimeouts[dynamicFilterCredentialArrivalHistory-2]) - for i := cfg.DynamicFilterCredentialArrivalHistory; i < len(filterTimeouts); i++ { + for i := dynamicFilterCredentialArrivalHistory; i < len(filterTimeouts); i++ { require.Equal(t, filterTimeouts[i-1], filterTimeouts[i]) } } @@ -1065,13 +1063,11 @@ func TestDynamicFilterTimeoutResets(t *testing.T) { version := protocol.ConsensusFuture - cfg := config.Consensus[protocol.ConsensusFuture] - if cfg.DynamicFilterCredentialArrivalHistory <= 0 { - return - } - if !cfg.DynamicFilterTimeout { + if dynamicFilterCredentialArrivalHistory <= 0 { return } + cfg := config.Consensus[protocol.ConsensusFuture] + require.True(t, cfg.DynamicFilterTimeout) numNodes := 5 @@ -1097,7 +1093,7 @@ func TestDynamicFilterTimeoutResets(t *testing.T) { // run round with round-specific consensus version first (since fix in #1896) zeroes = runRoundTriggerFilter(clocks, activityMonitor, zeroes) - for j := 1; j < cfg.DynamicFilterCredentialArrivalHistory+1; j++ { + for j := 1; j < dynamicFilterCredentialArrivalHistory+1; j++ { for srvIdx, clock := range clocks { delta, err := clock.(*testingClock).when(TimeoutFilter) require.NoError(t, err) @@ -1107,11 +1103,11 @@ func TestDynamicFilterTimeoutResets(t *testing.T) { } for i := range clocks { - require.Len(t, filterTimeouts[i], cfg.DynamicFilterCredentialArrivalHistory) - for j := 1; j < cfg.DynamicFilterCredentialArrivalHistory-1; j++ { + require.Len(t, filterTimeouts[i], dynamicFilterCredentialArrivalHistory) + for j := 1; j < dynamicFilterCredentialArrivalHistory-1; j++ { require.Equal(t, filterTimeouts[i][j-1], filterTimeouts[i][j]) } - require.Less(t, filterTimeouts[i][cfg.DynamicFilterCredentialArrivalHistory-1], filterTimeouts[i][cfg.DynamicFilterCredentialArrivalHistory-2]) + require.Less(t, filterTimeouts[i][dynamicFilterCredentialArrivalHistory-1], filterTimeouts[i][dynamicFilterCredentialArrivalHistory-2]) } // force fast partition recovery into bottom @@ -1143,7 +1139,7 @@ func TestDynamicFilterTimeoutResets(t *testing.T) { // run round with round-specific consensus version first (since fix in #1896) zeroes = runRoundTriggerFilter(clocks, activityMonitor, zeroes) - for j := 1; j < cfg.DynamicFilterCredentialArrivalHistory+1; j++ { + for j := 1; j < dynamicFilterCredentialArrivalHistory+1; j++ { for srvIdx, clock := range clocks { delta, err := clock.(*testingClock).when(TimeoutFilter) require.NoError(t, err) @@ -1153,19 +1149,19 @@ func TestDynamicFilterTimeoutResets(t *testing.T) { } for i := range clocks { - require.Len(t, filterTimeoutsPostRecovery[i], cfg.DynamicFilterCredentialArrivalHistory) + require.Len(t, filterTimeoutsPostRecovery[i], dynamicFilterCredentialArrivalHistory) // check that history was discarded, so filter time increased back to its original default - require.Less(t, filterTimeouts[i][cfg.DynamicFilterCredentialArrivalHistory-1], filterTimeoutsPostRecovery[i][0]) - require.Equal(t, filterTimeouts[i][cfg.DynamicFilterCredentialArrivalHistory-2], filterTimeoutsPostRecovery[i][0]) + require.Less(t, filterTimeouts[i][dynamicFilterCredentialArrivalHistory-1], filterTimeoutsPostRecovery[i][0]) + require.Equal(t, filterTimeouts[i][dynamicFilterCredentialArrivalHistory-2], filterTimeoutsPostRecovery[i][0]) // check that filter timeout was updated to at the end of the history window - for j := 1; j < cfg.DynamicFilterCredentialArrivalHistory-1; j++ { + for j := 1; j < dynamicFilterCredentialArrivalHistory-1; j++ { require.Equal(t, filterTimeoutsPostRecovery[i][j-1], filterTimeoutsPostRecovery[i][j]) } - require.Less(t, filterTimeoutsPostRecovery[i][cfg.DynamicFilterCredentialArrivalHistory-1], filterTimeoutsPostRecovery[i][cfg.DynamicFilterCredentialArrivalHistory-2]) + require.Less(t, filterTimeoutsPostRecovery[i][dynamicFilterCredentialArrivalHistory-1], filterTimeoutsPostRecovery[i][dynamicFilterCredentialArrivalHistory-2]) } - sanityCheck(startRound, 2*round(cfg.DynamicFilterCredentialArrivalHistory+1)+1, ledgers) + sanityCheck(startRound, 2*round(dynamicFilterCredentialArrivalHistory+1)+1, ledgers) } func TestAgreementSynchronousFuture1(t *testing.T) { diff --git a/config/consensus.go b/config/consensus.go index a5644eb8c0..819719f2af 100644 --- a/config/consensus.go +++ b/config/consensus.go @@ -517,27 +517,6 @@ type ConsensusParams struct { // by state proofs to use the same method (rather than excluding stake from the top N stakeholders as before). ExcludeExpiredCirculation bool - // DynamicFilterCredentialArrivalHistory specifies the number of past - // credential arrivals that are measured to determine the next filter - // timeout. If DynamicFilterCredentialArrivalHistory <= 0, then the dynamic - // timeout feature is off and the filter step timeout is calculated using - // the static configuration. - DynamicFilterCredentialArrivalHistory int - - // DynamicFilterTimeoutLowerBound specifies a minimal duration that the - // filter timeout must meet. - DynamicFilterTimeoutLowerBound time.Duration - - // DynamicFilterTimeoutCredentialArrivalHistoryIdx specified which sample to - // use out of a sorted DynamicFilterCredentialArrivalHistory-sized array of - // time samples. - DynamicFilterTimeoutCredentialArrivalHistoryIdx int - - // DynamicFilterTimeoutGraceInterval is additional extension to the dynamic - // filter time atop the one calculated based on the history of credential - // arrivals. - DynamicFilterTimeoutGraceInterval time.Duration - // DynamicFilterTimeout indicates whether the filter timeout is set // dynamically, at run time, according to the recent history of credential // arrival times or is set to a static value. Even if this flag disables the @@ -1390,12 +1369,6 @@ func initConsensusProtocols() { vFuture.LogicSigVersion = 10 // When moving this to a release, put a new higher LogicSigVersion here vFuture.EnableLogicSigCostPooling = true - // history window of 40, so we have enough statistics to calculate the 95th - // percentile, which is the timestamp at index 37 in the history array. - vFuture.DynamicFilterCredentialArrivalHistory = 40 - vFuture.DynamicFilterTimeoutCredentialArrivalHistoryIdx = 37 - vFuture.DynamicFilterTimeoutLowerBound = 0 - vFuture.DynamicFilterTimeoutGraceInterval = 50 * time.Millisecond vFuture.DynamicFilterTimeout = true Consensus[protocol.ConsensusFuture] = vFuture From 50a9ac084f7d703ae2b15dc99ed9761e444c5130 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Tue, 22 Aug 2023 18:09:57 -0400 Subject: [PATCH 120/232] +license --- agreement/dynamicFilterTimeoutParams.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/agreement/dynamicFilterTimeoutParams.go b/agreement/dynamicFilterTimeoutParams.go index d749f730d0..ab092d5221 100644 --- a/agreement/dynamicFilterTimeoutParams.go +++ b/agreement/dynamicFilterTimeoutParams.go @@ -1,3 +1,19 @@ +// Copyright (C) 2019-2023 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + package agreement import "time" From 21ecfec28f72964ca5b4305b277c4463ab88b8be Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Tue, 22 Aug 2023 18:23:37 -0400 Subject: [PATCH 121/232] document new event --- agreement/proposalTracker.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/agreement/proposalTracker.go b/agreement/proposalTracker.go index 8e29b27cfc..603a8fd40b 100644 --- a/agreement/proposalTracker.go +++ b/agreement/proposalTracker.go @@ -88,7 +88,7 @@ func (t *proposalTracker) underlying() listener { return t } -// A proposalTracker handles five types of events. +// A proposalTracker handles six types of events. // // - voteFilterRequest returns a voteFiltered event if a given proposal-vote // from a given sender has already been seen. Otherwise it returns an empty @@ -118,6 +118,9 @@ func (t *proposalTracker) underlying() listener { // - readStaging returns the a stagingValueEvent with the proposal-value // believed to be the staging value (i.e., sigma(S, r, p)) by the // proposalTracker in period p. +// +// - readLowestVote returns the vote with the lowest credential that was received so far. + func (t *proposalTracker) handle(r routerHandle, p player, e event) event { switch e.t() { case voteFilterRequest: From 4fc730403c2090e433b899ff846028017925311c Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Tue, 22 Aug 2023 18:47:37 -0400 Subject: [PATCH 122/232] continue tracking the best proposal even after freezing --- agreement/proposalTracker.go | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/agreement/proposalTracker.go b/agreement/proposalTracker.go index 8e29b27cfc..2f3e3113d4 100644 --- a/agreement/proposalTracker.go +++ b/agreement/proposalTracker.go @@ -33,12 +33,21 @@ type proposalSeeker struct { // Frozen is set once freeze is called. When Frozen is set, Lowest and // Filled will no longer be modified. Frozen bool + + lowestEvenIfLate vote + hasLowest bool } // accept compares a given vote with the current lowest-credentialled vote and // sets it if freeze has not been called. func (s proposalSeeker) accept(v vote) (proposalSeeker, error) { if s.Frozen { + // continue tracking and forwarding the lowest proposal even when frozen + if !s.hasLowest || v.Cred.Less(s.Lowest.Cred) { + s.lowestEvenIfLate = v + s.hasLowest = true + return s, nil + } return s, errProposalSeekerFrozen{} } @@ -48,6 +57,8 @@ func (s proposalSeeker) accept(v vote) (proposalSeeker, error) { s.Lowest = v s.Filled = true + s.lowestEvenIfLate = v + s.hasLowest = true return s, nil } @@ -167,8 +178,8 @@ func (t *proposalTracker) handle(r routerHandle, p player, e event) event { case readLowestVote: e := e.(readLowestEvent) - e.Vote = t.Freezer.Lowest - e.Filled = t.Freezer.Filled + e.Vote = t.Freezer.lowestEvenIfLate + e.Filled = t.Freezer.hasLowest return e case softThreshold, certThreshold: From e260bf7395976e42ba8198f23b9685d92c704f27 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Tue, 22 Aug 2023 19:35:58 -0400 Subject: [PATCH 123/232] use right cred --- agreement/proposalTracker.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agreement/proposalTracker.go b/agreement/proposalTracker.go index 2f3e3113d4..c89ea30a9b 100644 --- a/agreement/proposalTracker.go +++ b/agreement/proposalTracker.go @@ -43,7 +43,7 @@ type proposalSeeker struct { func (s proposalSeeker) accept(v vote) (proposalSeeker, error) { if s.Frozen { // continue tracking and forwarding the lowest proposal even when frozen - if !s.hasLowest || v.Cred.Less(s.Lowest.Cred) { + if !s.hasLowest || v.Cred.Less(s.lowestEvenIfLate.Cred) { s.lowestEvenIfLate = v s.hasLowest = true return s, nil From 52d9647afa3317a360c73591e53a5d56e4b1689a Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Tue, 22 Aug 2023 20:33:43 -0400 Subject: [PATCH 124/232] checkpoint: more backward compatible change --- agreement/events.go | 2 ++ agreement/player.go | 20 ++++++++++++++------ agreement/proposalTracker.go | 16 +++++++++------- agreement/proposalTracker_test.go | 10 ++++++---- 4 files changed, 31 insertions(+), 17 deletions(-) diff --git a/agreement/events.go b/agreement/events.go index 5bfd104936..152e3653bd 100644 --- a/agreement/events.go +++ b/agreement/events.go @@ -596,6 +596,8 @@ type filteredEvent struct { // {proposal,vote,bundle}{Filtered,Malformed} T eventType + StateUpdated bool + // Err is the reason cryptographic verification failed and is set for // events {proposal,vote,bundle}Malformed. Err *serializableError diff --git a/agreement/player.go b/agreement/player.go index 3d726a2043..8ae91ade69 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -608,16 +608,20 @@ func (p *player) handleMessageEvent(r routerHandle, e messageEvent) (actions []a }() ef := r.dispatch(*p, delegatedE, proposalMachine, 0, 0, 0) + relayOnly := false switch ef.t() { case voteMalformed: err := ef.(filteredEvent).Err return append(actions, disconnectAction(e, err)) case voteFiltered: - err := ef.(filteredEvent).Err - return append(actions, ignoreAction(e, err)) + relayOnly = true + if !ef.(filteredEvent).StateUpdated { + err := ef.(filteredEvent).Err + return append(actions, ignoreAction(e, err)) + } } - if e.t() == votePresent { + if e.t() == votePresent && !relayOnly { doneProcessing = false seq := p.Pending.push(e.Tail) uv := e.Input.UnauthenticatedVote @@ -700,6 +704,7 @@ func (p *player) handleMessageEvent(r routerHandle, e messageEvent) (actions []a case votePresent, voteVerified: ef := r.dispatch(*p, delegatedE, voteMachine, 0, 0, 0) + relayOnly := false switch ef.t() { case voteMalformed: // TODO Add Metrics here to capture telemetryspec.VoteRejectedEvent details @@ -707,10 +712,13 @@ func (p *player) handleMessageEvent(r routerHandle, e messageEvent) (actions []a err := makeSerErrf("rejected message since it was invalid: %v", ef.(filteredEvent).Err) return append(actions, disconnectAction(e, err)) case voteFiltered: - err := ef.(filteredEvent).Err - return append(actions, ignoreAction(e, err)) + relayOnly = true + if !ef.(filteredEvent).StateUpdated { + err := ef.(filteredEvent).Err + return append(actions, ignoreAction(e, err)) + } } - if e.t() == votePresent { + if e.t() == votePresent && !relayOnly { uv := e.Input.UnauthenticatedVote return append(actions, verifyVoteAction(e, uv.R.Round, uv.R.Period, 0)) } // else e.t() == voteVerified diff --git a/agreement/proposalTracker.go b/agreement/proposalTracker.go index c89ea30a9b..1443073b88 100644 --- a/agreement/proposalTracker.go +++ b/agreement/proposalTracker.go @@ -40,26 +40,27 @@ type proposalSeeker struct { // accept compares a given vote with the current lowest-credentialled vote and // sets it if freeze has not been called. -func (s proposalSeeker) accept(v vote) (proposalSeeker, error) { +func (s proposalSeeker) accept(v vote) (proposalSeeker, bool, error) { if s.Frozen { + updated := false // continue tracking and forwarding the lowest proposal even when frozen if !s.hasLowest || v.Cred.Less(s.lowestEvenIfLate.Cred) { s.lowestEvenIfLate = v s.hasLowest = true - return s, nil + updated = true } - return s, errProposalSeekerFrozen{} + return s, updated, errProposalSeekerFrozen{} } if s.Filled && !v.Cred.Less(s.Lowest.Cred) { - return s, errProposalSeekerNotLess{NewSender: v.R.Sender, LowestSender: s.Lowest.R.Sender} + return s, false, errProposalSeekerNotLess{NewSender: v.R.Sender, LowestSender: s.Lowest.R.Sender} } s.Lowest = v s.Filled = true s.lowestEvenIfLate = v s.hasLowest = true - return s, nil + return s, true, nil } // freeze freezes the state of the proposalSeeker so that future calls no longer @@ -157,11 +158,12 @@ func (t *proposalTracker) handle(r routerHandle, p player, e event) event { return filteredEvent{T: voteFiltered, Err: makeSerErr(err)} } + var updated bool var err error - t.Freezer, err = t.Freezer.accept(v) + t.Freezer, updated, err = t.Freezer.accept(v) if err != nil { err := errProposalTrackerPS{Sub: err} - return filteredEvent{T: voteFiltered, Err: makeSerErr(err)} + return filteredEvent{T: voteFiltered, StateUpdated: updated, Err: makeSerErr(err)} } return proposalAcceptedEvent{ diff --git a/agreement/proposalTracker_test.go b/agreement/proposalTracker_test.go index 641dee70e7..d830280ef7 100644 --- a/agreement/proposalTracker_test.go +++ b/agreement/proposalTracker_test.go @@ -65,31 +65,33 @@ func TestProposalTrackerProposalSeeker(t *testing.T) { assert.False(t, s.Filled) // issue events in the following order: 2, 3, 1, (freeze), 0 - s, err = s.accept(votes[2]) + s, _, err = s.accept(votes[2]) assert.NoError(t, err) assert.False(t, s.Frozen) assert.True(t, s.Filled) assert.True(t, s.Lowest.equals(votes[2])) - s, err = s.accept(votes[3]) + s, _, err = s.accept(votes[3]) assert.Error(t, err) assert.False(t, s.Frozen) assert.True(t, s.Filled) assert.True(t, s.Lowest.equals(votes[2])) - s, err = s.accept(votes[1]) + s, _, err = s.accept(votes[1]) assert.NoError(t, err) assert.False(t, s.Frozen) assert.True(t, s.Filled) assert.True(t, s.Lowest.equals(votes[1])) + lowestBeforeFreeze := s.Lowest s = s.freeze() assert.True(t, s.Frozen) assert.True(t, s.Filled) assert.True(t, s.Lowest.equals(votes[1])) - s, err = s.accept(votes[0]) + s, _, err = s.accept(votes[0]) assert.Error(t, err) + assert.Equal(t, s.Lowest, lowestBeforeFreeze) assert.True(t, s.Frozen) assert.True(t, s.Filled) assert.True(t, s.Lowest.equals(votes[1])) From 0df70820cad20e9f10b7de4c3d3d08fd1836f9d8 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Tue, 22 Aug 2023 21:32:55 -0400 Subject: [PATCH 125/232] relay only behavior --- agreement/player.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/agreement/player.go b/agreement/player.go index 8ae91ade69..d49dc99326 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -629,6 +629,9 @@ func (p *player) handleMessageEvent(r routerHandle, e messageEvent) (actions []a } v := e.Input.Vote a := relayAction(e, protocol.AgreementVoteTag, v.u()) + if relayOnly { + return append(actions, a) + } ep := ef.(proposalAcceptedEvent) if ep.PayloadOk { transmit := compoundMessage{ @@ -724,6 +727,9 @@ func (p *player) handleMessageEvent(r routerHandle, e messageEvent) (actions []a } // else e.t() == voteVerified v := e.Input.Vote actions = append(actions, relayAction(e, protocol.AgreementVoteTag, v.u())) + if relayOnly { + return actions + } a1 := p.handle(r, ef) return append(actions, a1...) From 8e83a2e69d93bad7e2b0c37958f7831d6d09e623 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Tue, 22 Aug 2023 21:35:21 -0400 Subject: [PATCH 126/232] doc line --- agreement/proposalTracker.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/agreement/proposalTracker.go b/agreement/proposalTracker.go index 1443073b88..7830d59b25 100644 --- a/agreement/proposalTracker.go +++ b/agreement/proposalTracker.go @@ -100,7 +100,7 @@ func (t *proposalTracker) underlying() listener { return t } -// A proposalTracker handles five types of events. +// A proposalTracker handles six types of events. // // - voteFilterRequest returns a voteFiltered event if a given proposal-vote // from a given sender has already been seen. Otherwise it returns an empty @@ -130,6 +130,8 @@ func (t *proposalTracker) underlying() listener { // - readStaging returns the a stagingValueEvent with the proposal-value // believed to be the staging value (i.e., sigma(S, r, p)) by the // proposalTracker in period p. +// +// - readLowestVote returns the vote with the lowest credential that was received so far. func (t *proposalTracker) handle(r routerHandle, p player, e event) event { switch e.t() { case voteFilterRequest: From 8b0984cf828be58eb89cb4fcc5c8eb6b8d6661a3 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Wed, 23 Aug 2023 15:35:24 -0400 Subject: [PATCH 127/232] keep old routers for longer --- agreement/router.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/agreement/router.go b/agreement/router.go index 32523ee8e9..642377e827 100644 --- a/agreement/router.go +++ b/agreement/router.go @@ -16,6 +16,8 @@ package agreement +import "github.com/algorand/go-algorand/config" + // A stateMachineTag uniquely identifies the type of a state machine. // // Rounds, periods, and steps may be used to further identify different state machine instances of the same type. @@ -119,6 +121,11 @@ func makeRootRouter(p player) (res rootRouter) { return } +// gcLag the maximal number of rounds that could pass before a credential from +// an honest party for an old round may arrive. It uses the +// dynamicFilterTimeoutLowerBound parameter as the minimal round time. +var gcLag = config.Protocol.SmallLambda / dynamicFilterTimeoutLowerBound + func (router *rootRouter) update(state player, r round, gc bool) { if router.proposalRoot == nil { router.proposalRoot = checkedListener{listener: &router.ProposalManager, listenerContract: proposalManagerContract{}} @@ -136,7 +143,11 @@ func (router *rootRouter) update(state player, r round, gc bool) { if gc { children := make(map[round]*roundRouter) for r, c := range router.Children { - if r >= state.Round { + // We may still receive credential messages from old rounds. Keep + // old round routers around, for as long as those credentials may + // arrive to keep track of them. + rr := r + round(gcLag) + if rr >= state.Round { children[r] = c } } From df7f645603b2bad935be8febd18758eacb60f75b Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Wed, 23 Aug 2023 17:35:40 -0400 Subject: [PATCH 128/232] take credential arrival times from credentialRoundLag ago --- agreement/player.go | 8 +++++--- agreement/player_test.go | 21 +++------------------ agreement/router.go | 6 +++--- agreement/service_test.go | 29 ++++++++++++++++------------- 4 files changed, 27 insertions(+), 37 deletions(-) diff --git a/agreement/player.go b/agreement/player.go index d49dc99326..baf750058d 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -286,9 +286,11 @@ func (p *player) updateCredentialArrivalHistory(r routerHandle, ver protocol.Con if p.Period != 0 { return 0 } - // look up the validatedAt time of the winning proposal-vote - re := readLowestEvent{T: readLowestVote, Round: p.Round, Period: p.Period} - re = r.dispatch(*p, re, proposalMachineRound, p.Round, p.Period, 0).(readLowestEvent) + // look up the validatedAt time of the winning proposal-vote from credentialRoundLag ago, + // by now we must have seen the lowest credential for that round. + round := p.Round.SubSaturate(round(credentialRoundLag)) + re := readLowestEvent{T: readLowestVote, Round: round, Period: p.Period} + re = r.dispatch(*p, re, proposalMachineRound, round, p.Period, 0).(readLowestEvent) if !re.Filled { return 0 } diff --git a/agreement/player_test.go b/agreement/player_test.go index ba4fd23288..0bcf688a8b 100644 --- a/agreement/player_test.go +++ b/agreement/player_test.go @@ -3261,11 +3261,6 @@ func TestPlayerRetainsReceivedValidatedAtOneSample(t *testing.T) { require.NoError(t, panicErr) assertCorrectReceivedAtSet(t, pWhite, pM, helper, r, p, pP, pV, m, protocol.ConsensusFuture) - - // assert lowest vote validateAt time was recorded into payloadArrivals - require.NotZero(t, dynamicFilterCredentialArrivalHistory) - require.Len(t, pWhite.lowestCredentialArrivals, 1) - require.Equal(t, 501*time.Millisecond, pWhite.lowestCredentialArrivals[0]) } // test that ReceivedAt and ValidateAt timing information are retained in proposalStore @@ -3280,7 +3275,7 @@ func TestPlayerRetainsReceivedValidatedAtForHistoryWindow(t *testing.T) { require.NotZero(t, dynamicFilterCredentialArrivalHistory) - for i := 0; i < dynamicFilterCredentialArrivalHistory+1; i++ { + for i := 0; i < dynamicFilterCredentialArrivalHistory+1+int(credentialRoundLag); i++ { // send voteVerified message pP, pV := helper.MakeRandomProposalPayload(t, r+round(i)-1) vVote := helper.MakeVerifiedVote(t, 0, r+round(i)-1, p, propose, *pV) @@ -3346,11 +3341,6 @@ func TestPlayerRetainsReceivedValidatedAtPPOneSample(t *testing.T) { require.NoError(t, panicErr) assertCorrectReceivedAtSet(t, pWhite, pM, helper, r, p, pP, pV, proposalMsg, protocol.ConsensusFuture) - - // assert lowest vote validateAt time was recorded into payloadArrivals - require.NotZero(t, dynamicFilterCredentialArrivalHistory) - require.Len(t, pWhite.lowestCredentialArrivals, 1) - require.Equal(t, 502*time.Millisecond, pWhite.lowestCredentialArrivals[0]) } // test that ReceivedAt and ValidateAt timing information are retained in @@ -3368,7 +3358,7 @@ func TestPlayerRetainsReceivedValidatedAtPPForHistoryWindow(t *testing.T) { require.NotZero(t, dynamicFilterCredentialArrivalHistory) - for i := 0; i < dynamicFilterCredentialArrivalHistory+1; i++ { + for i := 0; i < dynamicFilterCredentialArrivalHistory+1+int(credentialRoundLag); i++ { // create a PP message for an arbitrary proposal/payload similar to setupCompoundMessage vVote := helper.MakeVerifiedVote(t, 0, r+round(i)-1, p, propose, *pV) unverifiedVoteMsg := message{UnauthenticatedVote: vVote.u()} @@ -3452,11 +3442,6 @@ func TestPlayerRetainsReceivedValidatedAtAVPPOneSample(t *testing.T) { require.Equal(t, 1, pM.getTrace().CountEvent(verifyEvent), "Player should not verify second vote") assertCorrectReceivedAtSet(t, pWhite, pM, helper, r, p, pP, pV, proposalMsg, protocol.ConsensusFuture) - - // assert lowest vote validateAt time was recorded into payloadArrivals - require.NotZero(t, dynamicFilterCredentialArrivalHistory) - require.Len(t, pWhite.lowestCredentialArrivals, 1) - require.Equal(t, 502*time.Millisecond, pWhite.lowestCredentialArrivals[0]) } func TestPlayerRetainsReceivedValidatedAtAVPPHistoryWindow(t *testing.T) { @@ -3467,7 +3452,7 @@ func TestPlayerRetainsReceivedValidatedAtAVPPHistoryWindow(t *testing.T) { require.NotZero(t, dynamicFilterCredentialArrivalHistory) - for i := 0; i < dynamicFilterCredentialArrivalHistory+1; i++ { + for i := 0; i < dynamicFilterCredentialArrivalHistory+1+int(credentialRoundLag); i++ { pP, pV := helper.MakeRandomProposalPayload(t, r+round(i)-1) // send votePresent message (mimicking the first AV message validating) diff --git a/agreement/router.go b/agreement/router.go index 642377e827..c134fff8e7 100644 --- a/agreement/router.go +++ b/agreement/router.go @@ -121,10 +121,10 @@ func makeRootRouter(p player) (res rootRouter) { return } -// gcLag the maximal number of rounds that could pass before a credential from +// credentialRoundLag the maximal number of rounds that could pass before a credential from // an honest party for an old round may arrive. It uses the // dynamicFilterTimeoutLowerBound parameter as the minimal round time. -var gcLag = config.Protocol.SmallLambda / dynamicFilterTimeoutLowerBound +var credentialRoundLag = config.Protocol.SmallLambda / dynamicFilterTimeoutLowerBound func (router *rootRouter) update(state player, r round, gc bool) { if router.proposalRoot == nil { @@ -146,7 +146,7 @@ func (router *rootRouter) update(state player, r round, gc bool) { // We may still receive credential messages from old rounds. Keep // old round routers around, for as long as those credentials may // arrive to keep track of them. - rr := r + round(gcLag) + rr := r + round(credentialRoundLag) + 1 if rr >= state.Round { children[r] = c } diff --git a/agreement/service_test.go b/agreement/service_test.go index 6705b629c7..0c05f3f7b7 100644 --- a/agreement/service_test.go +++ b/agreement/service_test.go @@ -1038,18 +1038,19 @@ func TestAgreementSynchronousFuture5_DynamicFilterRounds(t *testing.T) { return } - rounds := dynamicFilterCredentialArrivalHistory + 20 + baseHistoryRounds := dynamicFilterCredentialArrivalHistory + int(credentialRoundLag) + rounds := baseHistoryRounds + 20 filterTimeouts := simulateAgreementWithConsensusVersion(t, 5, rounds, disabled, consensusVersion) require.Len(t, filterTimeouts, rounds-1) - for i := 1; i < dynamicFilterCredentialArrivalHistory-1; i++ { + for i := 1; i < baseHistoryRounds-1; i++ { require.Equal(t, filterTimeouts[i-1], filterTimeouts[i]) } // dynamic filter timeout kicks in when history window is full - require.Less(t, filterTimeouts[dynamicFilterCredentialArrivalHistory-1], filterTimeouts[dynamicFilterCredentialArrivalHistory-2]) + require.Less(t, filterTimeouts[baseHistoryRounds-1], filterTimeouts[baseHistoryRounds-2]) - for i := dynamicFilterCredentialArrivalHistory; i < len(filterTimeouts); i++ { + for i := baseHistoryRounds; i < len(filterTimeouts); i++ { require.Equal(t, filterTimeouts[i-1], filterTimeouts[i]) } } @@ -1091,9 +1092,11 @@ func TestDynamicFilterTimeoutResets(t *testing.T) { filterTimeouts := make([][]time.Duration, numNodes, numNodes) + baseHistoryRounds := dynamicFilterCredentialArrivalHistory + int(credentialRoundLag) + // run round with round-specific consensus version first (since fix in #1896) zeroes = runRoundTriggerFilter(clocks, activityMonitor, zeroes) - for j := 1; j < dynamicFilterCredentialArrivalHistory+1; j++ { + for j := 1; j < baseHistoryRounds+1; j++ { for srvIdx, clock := range clocks { delta, err := clock.(*testingClock).when(TimeoutFilter) require.NoError(t, err) @@ -1103,11 +1106,11 @@ func TestDynamicFilterTimeoutResets(t *testing.T) { } for i := range clocks { - require.Len(t, filterTimeouts[i], dynamicFilterCredentialArrivalHistory) - for j := 1; j < dynamicFilterCredentialArrivalHistory-1; j++ { + require.Len(t, filterTimeouts[i], baseHistoryRounds) + for j := 1; j < baseHistoryRounds-1; j++ { require.Equal(t, filterTimeouts[i][j-1], filterTimeouts[i][j]) } - require.Less(t, filterTimeouts[i][dynamicFilterCredentialArrivalHistory-1], filterTimeouts[i][dynamicFilterCredentialArrivalHistory-2]) + require.Less(t, filterTimeouts[i][baseHistoryRounds-1], filterTimeouts[i][baseHistoryRounds-2]) } // force fast partition recovery into bottom @@ -1139,7 +1142,7 @@ func TestDynamicFilterTimeoutResets(t *testing.T) { // run round with round-specific consensus version first (since fix in #1896) zeroes = runRoundTriggerFilter(clocks, activityMonitor, zeroes) - for j := 1; j < dynamicFilterCredentialArrivalHistory+1; j++ { + for j := 1; j < baseHistoryRounds+1; j++ { for srvIdx, clock := range clocks { delta, err := clock.(*testingClock).when(TimeoutFilter) require.NoError(t, err) @@ -1149,10 +1152,10 @@ func TestDynamicFilterTimeoutResets(t *testing.T) { } for i := range clocks { - require.Len(t, filterTimeoutsPostRecovery[i], dynamicFilterCredentialArrivalHistory) + require.Len(t, filterTimeoutsPostRecovery[i], baseHistoryRounds) // check that history was discarded, so filter time increased back to its original default - require.Less(t, filterTimeouts[i][dynamicFilterCredentialArrivalHistory-1], filterTimeoutsPostRecovery[i][0]) - require.Equal(t, filterTimeouts[i][dynamicFilterCredentialArrivalHistory-2], filterTimeoutsPostRecovery[i][0]) + require.Less(t, filterTimeouts[i][baseHistoryRounds-1], filterTimeoutsPostRecovery[i][0]) + require.Equal(t, filterTimeouts[i][baseHistoryRounds-2], filterTimeoutsPostRecovery[i][0]) // check that filter timeout was updated to at the end of the history window for j := 1; j < dynamicFilterCredentialArrivalHistory-1; j++ { @@ -1161,7 +1164,7 @@ func TestDynamicFilterTimeoutResets(t *testing.T) { require.Less(t, filterTimeoutsPostRecovery[i][dynamicFilterCredentialArrivalHistory-1], filterTimeoutsPostRecovery[i][dynamicFilterCredentialArrivalHistory-2]) } - sanityCheck(startRound, 2*round(dynamicFilterCredentialArrivalHistory+1)+1, ledgers) + sanityCheck(startRound, 2*round(baseHistoryRounds+1)+1, ledgers) } func TestAgreementSynchronousFuture1(t *testing.T) { From 8082f7918d3ce19a7a105fd29a17cce1683df899 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Wed, 23 Aug 2023 18:35:00 -0400 Subject: [PATCH 129/232] lint --- agreement/player.go | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/agreement/player.go b/agreement/player.go index baf750058d..ebea467a5f 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -288,9 +288,9 @@ func (p *player) updateCredentialArrivalHistory(r routerHandle, ver protocol.Con } // look up the validatedAt time of the winning proposal-vote from credentialRoundLag ago, // by now we must have seen the lowest credential for that round. - round := p.Round.SubSaturate(round(credentialRoundLag)) - re := readLowestEvent{T: readLowestVote, Round: round, Period: p.Period} - re = r.dispatch(*p, re, proposalMachineRound, round, p.Period, 0).(readLowestEvent) + credHistoryRound := p.Round.SubSaturate(round(credentialRoundLag)) + re := readLowestEvent{T: readLowestVote, Round: credHistoryRound, Period: p.Period} + re = r.dispatch(*p, re, proposalMachineRound, credHistoryRound, p.Period, 0).(readLowestEvent) if !re.Filled { return 0 } @@ -610,20 +610,21 @@ func (p *player) handleMessageEvent(r routerHandle, e messageEvent) (actions []a }() ef := r.dispatch(*p, delegatedE, proposalMachine, 0, 0, 0) - relayOnly := false switch ef.t() { case voteMalformed: err := ef.(filteredEvent).Err return append(actions, disconnectAction(e, err)) case voteFiltered: - relayOnly = true if !ef.(filteredEvent).StateUpdated { err := ef.(filteredEvent).Err return append(actions, ignoreAction(e, err)) } + v := e.Input.Vote + a := relayAction(e, protocol.AgreementVoteTag, v.u()) + return append(actions, a) } - if e.t() == votePresent && !relayOnly { + if e.t() == votePresent { doneProcessing = false seq := p.Pending.push(e.Tail) uv := e.Input.UnauthenticatedVote @@ -631,9 +632,6 @@ func (p *player) handleMessageEvent(r routerHandle, e messageEvent) (actions []a } v := e.Input.Vote a := relayAction(e, protocol.AgreementVoteTag, v.u()) - if relayOnly { - return append(actions, a) - } ep := ef.(proposalAcceptedEvent) if ep.PayloadOk { transmit := compoundMessage{ @@ -709,7 +707,6 @@ func (p *player) handleMessageEvent(r routerHandle, e messageEvent) (actions []a case votePresent, voteVerified: ef := r.dispatch(*p, delegatedE, voteMachine, 0, 0, 0) - relayOnly := false switch ef.t() { case voteMalformed: // TODO Add Metrics here to capture telemetryspec.VoteRejectedEvent details @@ -717,21 +714,20 @@ func (p *player) handleMessageEvent(r routerHandle, e messageEvent) (actions []a err := makeSerErrf("rejected message since it was invalid: %v", ef.(filteredEvent).Err) return append(actions, disconnectAction(e, err)) case voteFiltered: - relayOnly = true if !ef.(filteredEvent).StateUpdated { err := ef.(filteredEvent).Err return append(actions, ignoreAction(e, err)) } + v := e.Input.Vote + actions = append(actions, relayAction(e, protocol.AgreementVoteTag, v.u())) + return actions } - if e.t() == votePresent && !relayOnly { + if e.t() == votePresent { uv := e.Input.UnauthenticatedVote return append(actions, verifyVoteAction(e, uv.R.Round, uv.R.Period, 0)) } // else e.t() == voteVerified v := e.Input.Vote actions = append(actions, relayAction(e, protocol.AgreementVoteTag, v.u())) - if relayOnly { - return actions - } a1 := p.handle(r, ef) return append(actions, a1...) From aa341b27cc5a524965cd71bd2c92e86022b68576 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Thu, 24 Aug 2023 12:56:37 -0400 Subject: [PATCH 130/232] proposalManager doesn't filter old votes + test --- agreement/player.go | 2 +- agreement/player_test.go | 66 ++++++++++++++++++++++++++++++++++++ agreement/proposalManager.go | 10 ++++++ agreement/router.go | 5 +-- 4 files changed, 80 insertions(+), 3 deletions(-) diff --git a/agreement/player.go b/agreement/player.go index ebea467a5f..c337540631 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -288,7 +288,7 @@ func (p *player) updateCredentialArrivalHistory(r routerHandle, ver protocol.Con } // look up the validatedAt time of the winning proposal-vote from credentialRoundLag ago, // by now we must have seen the lowest credential for that round. - credHistoryRound := p.Round.SubSaturate(round(credentialRoundLag)) + credHistoryRound := p.Round.SubSaturate(round(credentialRoundLag + 1)) re := readLowestEvent{T: readLowestVote, Round: credHistoryRound, Period: p.Period} re = r.dispatch(*p, re, proposalMachineRound, credHistoryRound, p.Period, 0).(readLowestEvent) if !re.Filled { diff --git a/agreement/player_test.go b/agreement/player_test.go index 0bcf688a8b..2632cf74bc 100644 --- a/agreement/player_test.go +++ b/agreement/player_test.go @@ -25,6 +25,7 @@ import ( "github.com/algorand/go-algorand/config" "github.com/algorand/go-algorand/crypto" + "github.com/algorand/go-algorand/data/basics" "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/protocol" "github.com/algorand/go-algorand/test/partitiontest" @@ -3263,6 +3264,71 @@ func TestPlayerRetainsReceivedValidatedAtOneSample(t *testing.T) { assertCorrectReceivedAtSet(t, pWhite, pM, helper, r, p, pP, pV, m, protocol.ConsensusFuture) } +func TestPlayerRetainsLateReceivedValidatedAtOneSample(t *testing.T) { + partitiontest.PartitionTest(t) + + const r = round(20239) + const p = period(0) + pWhite, pM, helper := setupP(t, r-1, p, soft) + pP, pV := helper.MakeRandomProposalPayload(t, r-1) + + // Move to round r, no credentials arrived. + // send voteVerified message + vVote := helper.MakeVerifiedVote(t, 0, r-1, p, propose, *pV) + inMsg := messageEvent{T: voteVerified, Input: message{Vote: vVote, UnauthenticatedVote: vVote.u()}} + inMsg = inMsg.AttachValidatedAt(501 * time.Millisecond) + err, panicErr := pM.transition(inMsg) + require.NoError(t, err) + require.NoError(t, panicErr) + + // send payloadPresent message + m := message{UnauthenticatedProposal: pP.u()} + inMsg = messageEvent{T: payloadPresent, Input: m} + inMsg = inMsg.AttachReceivedAt(time.Second) + err, panicErr = pM.transition(inMsg) + require.NoError(t, err) + require.NoError(t, panicErr) + + assertCorrectReceivedAtSet(t, pWhite, pM, helper, r, p, pP, pV, m, protocol.ConsensusFuture) + require.Len(t, pWhite.lowestCredentialArrivals, 0) + + // Old credential arrives + // send voteVerified message + vVote = helper.MakeVerifiedVote(t, 0, r-basics.Round(credentialRoundLag+1), p, propose, *pV) + inMsg = messageEvent{T: voteVerified, Input: message{Vote: vVote, UnauthenticatedVote: vVote.u()}} + inMsg = inMsg.AttachValidatedAt(501 * time.Millisecond) + err, panicErr = pM.transition(inMsg) + require.NoError(t, err) + require.NoError(t, panicErr) + + // send payloadPresent message + m = message{UnauthenticatedProposal: pP.u()} + inMsg = messageEvent{T: payloadPresent, Input: m} + inMsg = inMsg.AttachReceivedAt(time.Second) + err, panicErr = pM.transition(inMsg) + require.NoError(t, err) + require.NoError(t, panicErr) + + // move to round r+1, triggering history update + vVote = helper.MakeVerifiedVote(t, 0, r, p, propose, *pV) + inMsg = messageEvent{T: voteVerified, Input: message{Vote: vVote, UnauthenticatedVote: vVote.u()}} + inMsg = inMsg.AttachValidatedAt(501 * time.Millisecond) + err, panicErr = pM.transition(inMsg) + require.NoError(t, err) + require.NoError(t, panicErr) + + // send payloadPresent message + m = message{UnauthenticatedProposal: pP.u()} + inMsg = messageEvent{T: payloadPresent, Input: m} + inMsg = inMsg.AttachReceivedAt(time.Second) + err, panicErr = pM.transition(inMsg) + require.NoError(t, err) + require.NoError(t, panicErr) + moveToRound(t, pWhite, pM, helper, r+1, p, pP, pV, m, protocol.ConsensusFuture) + + require.Len(t, pWhite.lowestCredentialArrivals, 1) +} + // test that ReceivedAt and ValidateAt timing information are retained in proposalStore // when the payloadPresent, payloadVerified, and voteVerified events are processed, and that all timings // are available when the ensureAction is called for the block. The history should be kept for the last diff --git a/agreement/proposalManager.go b/agreement/proposalManager.go index affa17f898..9b34e80789 100644 --- a/agreement/proposalManager.go +++ b/agreement/proposalManager.go @@ -18,6 +18,8 @@ package agreement import ( "fmt" + + "github.com/algorand/go-algorand/data/basics" ) // A proposalManager is a proposalMachine which applies relay rules to incoming @@ -232,6 +234,14 @@ func (m *proposalManager) filterProposalVote(p player, r routerHandle, uv unauth // voteFresh determines whether a proposal satisfies freshness rules. func proposalFresh(freshData freshnessData, vote unauthenticatedVote) error { + if freshData.PlayerRound <= vote.R.Round+basics.Round(credentialRoundLag)+1 && vote.R.Period == 0 { + if dynamicFilterCredentialArrivalHistory > 0 { + // continue processing old period 0 votes so we could track their + // arrival times and inform setting the filter timeout dynamically. + return nil + } + } + switch vote.R.Round { case freshData.PlayerRound: if freshData.PlayerPeriod != 0 && freshData.PlayerPeriod-1 > vote.R.Period { diff --git a/agreement/router.go b/agreement/router.go index c134fff8e7..2d06a9515b 100644 --- a/agreement/router.go +++ b/agreement/router.go @@ -16,7 +16,9 @@ package agreement -import "github.com/algorand/go-algorand/config" +import ( + "github.com/algorand/go-algorand/config" +) // A stateMachineTag uniquely identifies the type of a state machine. // @@ -212,7 +214,6 @@ func (router *roundRouter) update(state player, p period, gc bool) { // TODO may want regression test for correct pipelining behavior children[p] = c } - } router.Children = children } From 1b8f21daee70eeb3edc1a3ed8ce041b12ba6102f Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Thu, 24 Aug 2023 13:02:24 -0400 Subject: [PATCH 131/232] nits --- agreement/events.go | 2 ++ agreement/player.go | 2 +- agreement/proposalTracker.go | 12 ++++++------ agreement/router.go | 4 ++-- 4 files changed, 11 insertions(+), 9 deletions(-) diff --git a/agreement/events.go b/agreement/events.go index 152e3653bd..c09fc19c12 100644 --- a/agreement/events.go +++ b/agreement/events.go @@ -596,6 +596,8 @@ type filteredEvent struct { // {proposal,vote,bundle}{Filtered,Malformed} T eventType + // StateUpdated indicates whether the filtered message caused any change in + // the state machine StateUpdated bool // Err is the reason cryptographic verification failed and is set for diff --git a/agreement/player.go b/agreement/player.go index c337540631..1761d47995 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -288,7 +288,7 @@ func (p *player) updateCredentialArrivalHistory(r routerHandle, ver protocol.Con } // look up the validatedAt time of the winning proposal-vote from credentialRoundLag ago, // by now we must have seen the lowest credential for that round. - credHistoryRound := p.Round.SubSaturate(round(credentialRoundLag + 1)) + credHistoryRound := p.Round.SubSaturate(credentialRoundLag + 1) re := readLowestEvent{T: readLowestVote, Round: credHistoryRound, Period: p.Period} re = r.dispatch(*p, re, proposalMachineRound, credHistoryRound, p.Period, 0).(readLowestEvent) if !re.Filled { diff --git a/agreement/proposalTracker.go b/agreement/proposalTracker.go index 7830d59b25..04a5271948 100644 --- a/agreement/proposalTracker.go +++ b/agreement/proposalTracker.go @@ -34,8 +34,8 @@ type proposalSeeker struct { // Filled will no longer be modified. Frozen bool - lowestEvenIfLate vote - hasLowest bool + lowestEvenIfLate vote + hasLowestEvenIfLate bool } // accept compares a given vote with the current lowest-credentialled vote and @@ -44,9 +44,9 @@ func (s proposalSeeker) accept(v vote) (proposalSeeker, bool, error) { if s.Frozen { updated := false // continue tracking and forwarding the lowest proposal even when frozen - if !s.hasLowest || v.Cred.Less(s.lowestEvenIfLate.Cred) { + if !s.hasLowestEvenIfLate || v.Cred.Less(s.lowestEvenIfLate.Cred) { s.lowestEvenIfLate = v - s.hasLowest = true + s.hasLowestEvenIfLate = true updated = true } return s, updated, errProposalSeekerFrozen{} @@ -59,7 +59,7 @@ func (s proposalSeeker) accept(v vote) (proposalSeeker, bool, error) { s.Lowest = v s.Filled = true s.lowestEvenIfLate = v - s.hasLowest = true + s.hasLowestEvenIfLate = true return s, true, nil } @@ -183,7 +183,7 @@ func (t *proposalTracker) handle(r routerHandle, p player, e event) event { case readLowestVote: e := e.(readLowestEvent) e.Vote = t.Freezer.lowestEvenIfLate - e.Filled = t.Freezer.hasLowest + e.Filled = t.Freezer.hasLowestEvenIfLate return e case softThreshold, certThreshold: diff --git a/agreement/router.go b/agreement/router.go index 2d06a9515b..840097e1e4 100644 --- a/agreement/router.go +++ b/agreement/router.go @@ -126,7 +126,7 @@ func makeRootRouter(p player) (res rootRouter) { // credentialRoundLag the maximal number of rounds that could pass before a credential from // an honest party for an old round may arrive. It uses the // dynamicFilterTimeoutLowerBound parameter as the minimal round time. -var credentialRoundLag = config.Protocol.SmallLambda / dynamicFilterTimeoutLowerBound +var credentialRoundLag = round(config.Protocol.SmallLambda / dynamicFilterTimeoutLowerBound) func (router *rootRouter) update(state player, r round, gc bool) { if router.proposalRoot == nil { @@ -148,7 +148,7 @@ func (router *rootRouter) update(state player, r round, gc bool) { // We may still receive credential messages from old rounds. Keep // old round routers around, for as long as those credentials may // arrive to keep track of them. - rr := r + round(credentialRoundLag) + 1 + rr := r + credentialRoundLag + 1 if rr >= state.Round { children[r] = c } From df15f6b137cf5ab0c3a37325f0272addd478d155 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Thu, 24 Aug 2023 13:05:52 -0400 Subject: [PATCH 132/232] update comment --- agreement/events.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/agreement/events.go b/agreement/events.go index 5bfd104936..c494c66a85 100644 --- a/agreement/events.go +++ b/agreement/events.go @@ -415,11 +415,10 @@ type readLowestEvent struct { // T currently only supports readLowestVote T eventType - // Round and Period are the round and period for which to query - // the lowest-credential vote, value or payload. This type of event - // is only sent for pipelining, which only makes sense for period - // 0, but the Period is here anyway to route to the appropriate - // proposalMachinePeriod. + // Round and Period are the round and period for which to query the + // lowest-credential vote, value or payload. This type of event is only + // sent for reading the lowest period 0 credential, but the Period is here + // anyway to route to the appropriate proposalMachinePeriod. Round round Period period From 3e1592b0e484ec9d783402ef82f6ff2705266f81 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Thu, 24 Aug 2023 16:08:19 -0400 Subject: [PATCH 133/232] less conversions --- agreement/player_test.go | 3 +-- agreement/proposalManager.go | 4 +--- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/agreement/player_test.go b/agreement/player_test.go index eb11454a68..e2b300e162 100644 --- a/agreement/player_test.go +++ b/agreement/player_test.go @@ -25,7 +25,6 @@ import ( "github.com/algorand/go-algorand/config" "github.com/algorand/go-algorand/crypto" - "github.com/algorand/go-algorand/data/basics" "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/protocol" "github.com/algorand/go-algorand/test/partitiontest" @@ -3294,7 +3293,7 @@ func TestPlayerRetainsLateReceivedValidatedAtOneSample(t *testing.T) { // Old credential arrives // send voteVerified message - vVote = helper.MakeVerifiedVote(t, 0, r-basics.Round(credentialRoundLag+1), p, propose, *pV) + vVote = helper.MakeVerifiedVote(t, 0, r-credentialRoundLag+1, p, propose, *pV) inMsg = messageEvent{T: voteVerified, Input: message{Vote: vVote, UnauthenticatedVote: vVote.u()}} inMsg = inMsg.AttachValidatedAt(501 * time.Millisecond) err, panicErr = pM.transition(inMsg) diff --git a/agreement/proposalManager.go b/agreement/proposalManager.go index 0303c6ee87..675868ebf6 100644 --- a/agreement/proposalManager.go +++ b/agreement/proposalManager.go @@ -18,8 +18,6 @@ package agreement import ( "fmt" - - "github.com/algorand/go-algorand/data/basics" ) // A proposalManager is a proposalMachine which applies relay rules to incoming @@ -234,7 +232,7 @@ func (m *proposalManager) filterProposalVote(p player, r routerHandle, uv unauth // voteFresh determines whether a proposal satisfies freshness rules. func proposalFresh(freshData freshnessData, vote unauthenticatedVote) error { - if vote.R.Round < freshData.PlayerRound && freshData.PlayerRound <= vote.R.Round+basics.Round(credentialRoundLag)+1 && vote.R.Period == 0 && vote.R.Step == 0 { + if vote.R.Round < freshData.PlayerRound && freshData.PlayerRound <= vote.R.Round+credentialRoundLag+1 && vote.R.Period == 0 && vote.R.Step == 0 { if dynamicFilterCredentialArrivalHistory > 0 { // continue processing old period 0 votes so we could track their // arrival times and inform setting the filter timeout dynamically. From 128686e44d462f03793bf39434aed13566283a89 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Thu, 24 Aug 2023 18:48:14 -0400 Subject: [PATCH 134/232] off by 2 --- agreement/player_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agreement/player_test.go b/agreement/player_test.go index e2b300e162..804bb51572 100644 --- a/agreement/player_test.go +++ b/agreement/player_test.go @@ -3293,7 +3293,7 @@ func TestPlayerRetainsLateReceivedValidatedAtOneSample(t *testing.T) { // Old credential arrives // send voteVerified message - vVote = helper.MakeVerifiedVote(t, 0, r-credentialRoundLag+1, p, propose, *pV) + vVote = helper.MakeVerifiedVote(t, 0, r-credentialRoundLag-1, p, propose, *pV) inMsg = messageEvent{T: voteVerified, Input: message{Vote: vVote, UnauthenticatedVote: vVote.u()}} inMsg = inMsg.AttachValidatedAt(501 * time.Millisecond) err, panicErr = pM.transition(inMsg) From 9a4dfea4420e3be08568645d6555696e3a84684c Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Fri, 25 Aug 2023 09:26:03 -0400 Subject: [PATCH 135/232] var rename --- agreement/proposalTracker.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/agreement/proposalTracker.go b/agreement/proposalTracker.go index 04a5271948..c95de9e3bb 100644 --- a/agreement/proposalTracker.go +++ b/agreement/proposalTracker.go @@ -34,8 +34,8 @@ type proposalSeeker struct { // Filled will no longer be modified. Frozen bool - lowestEvenIfLate vote - hasLowestEvenIfLate bool + lowestAfterFreeze vote + hasLowestAfterFreeze bool } // accept compares a given vote with the current lowest-credentialled vote and @@ -44,9 +44,9 @@ func (s proposalSeeker) accept(v vote) (proposalSeeker, bool, error) { if s.Frozen { updated := false // continue tracking and forwarding the lowest proposal even when frozen - if !s.hasLowestEvenIfLate || v.Cred.Less(s.lowestEvenIfLate.Cred) { - s.lowestEvenIfLate = v - s.hasLowestEvenIfLate = true + if !s.hasLowestAfterFreeze || v.Cred.Less(s.lowestAfterFreeze.Cred) { + s.lowestAfterFreeze = v + s.hasLowestAfterFreeze = true updated = true } return s, updated, errProposalSeekerFrozen{} @@ -58,8 +58,8 @@ func (s proposalSeeker) accept(v vote) (proposalSeeker, bool, error) { s.Lowest = v s.Filled = true - s.lowestEvenIfLate = v - s.hasLowestEvenIfLate = true + s.lowestAfterFreeze = v + s.hasLowestAfterFreeze = true return s, true, nil } @@ -182,8 +182,8 @@ func (t *proposalTracker) handle(r routerHandle, p player, e event) event { case readLowestVote: e := e.(readLowestEvent) - e.Vote = t.Freezer.lowestEvenIfLate - e.Filled = t.Freezer.hasLowestEvenIfLate + e.Vote = t.Freezer.lowestAfterFreeze + e.Filled = t.Freezer.hasLowestAfterFreeze return e case softThreshold, certThreshold: From 4d72b15903837f22097812b1ddfce351fdb20588 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Fri, 25 Aug 2023 09:47:31 -0400 Subject: [PATCH 136/232] credential propagation bound is 2SmallLambda --- agreement/player.go | 6 ++++++ agreement/router.go | 18 +++++++++++++----- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/agreement/player.go b/agreement/player.go index 1761d47995..3902e80a29 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -286,6 +286,12 @@ func (p *player) updateCredentialArrivalHistory(r routerHandle, ver protocol.Con if p.Period != 0 { return 0 } + + if p.Round <= credentialRoundLag { + // not sufficiently many rounds had passed to collect any measurement + return 0 + } + // look up the validatedAt time of the winning proposal-vote from credentialRoundLag ago, // by now we must have seen the lowest credential for that round. credHistoryRound := p.Round.SubSaturate(credentialRoundLag + 1) diff --git a/agreement/router.go b/agreement/router.go index 840097e1e4..f733df943b 100644 --- a/agreement/router.go +++ b/agreement/router.go @@ -52,6 +52,19 @@ type routerHandle struct { src stateMachineTag } +// credentialRoundLag the maximal number of rounds that could pass before a credential from +// an honest party for an old round may arrive. It uses the +// dynamicFilterTimeoutLowerBound parameter as the minimal round time. +var credentialRoundLag round + +func init() { + // credential arrival time should be at most 2*config.Protocol.SmallLambda after it was sent + credentialRoundLag = round(2 * config.Protocol.SmallLambda / dynamicFilterTimeoutLowerBound) + if credentialRoundLag*round(dynamicFilterTimeoutLowerBound) < round(2*config.Protocol.SmallLambda) { + credentialRoundLag++ + } +} + // dispatch sends an event to the given state machine listener with the given stateMachineTag. // // If there are many state machines of this type (for instance, there is one voteMachineStep for each step) @@ -123,11 +136,6 @@ func makeRootRouter(p player) (res rootRouter) { return } -// credentialRoundLag the maximal number of rounds that could pass before a credential from -// an honest party for an old round may arrive. It uses the -// dynamicFilterTimeoutLowerBound parameter as the minimal round time. -var credentialRoundLag = round(config.Protocol.SmallLambda / dynamicFilterTimeoutLowerBound) - func (router *rootRouter) update(state player, r round, gc bool) { if router.proposalRoot == nil { router.proposalRoot = checkedListener{listener: &router.ProposalManager, listenerContract: proposalManagerContract{}} From 9413913329eee8f1aee167ce6ebd172c5457beb4 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Fri, 25 Aug 2023 20:05:46 -0400 Subject: [PATCH 137/232] document parameters --- agreement/dynamicFilterTimeoutParams.go | 7 ++++--- agreement/dynamicFilterTimeoutParams_test.go | 16 ++++++++++++++++ 2 files changed, 20 insertions(+), 3 deletions(-) create mode 100644 agreement/dynamicFilterTimeoutParams_test.go diff --git a/agreement/dynamicFilterTimeoutParams.go b/agreement/dynamicFilterTimeoutParams.go index ab092d5221..f24b7aac97 100644 --- a/agreement/dynamicFilterTimeoutParams.go +++ b/agreement/dynamicFilterTimeoutParams.go @@ -33,9 +33,10 @@ const dynamicFilterCredentialArrivalHistory int = 40 // filter timeout must meet. const dynamicFilterTimeoutLowerBound time.Duration = 600 * time.Millisecond -// DynamicFilterTimeoutCredentialArrivalHistoryIdx specified which sample to -// use out of a sorted DynamicFilterCredentialArrivalHistory-sized array of -// time samples. +// DynamicFilterTimeoutCredentialArrivalHistoryIdx specified which sample to use +// out of a sorted DynamicFilterCredentialArrivalHistory-sized array of time +// samples. The 95th percentile of dynamicFilterCredentialArrivalHistory = 40 +// sorted samples, is at index 37. const dynamicFilterTimeoutCredentialArrivalHistoryIdx int = 37 // DynamicFilterTimeoutGraceInterval is additional extension to the dynamic diff --git a/agreement/dynamicFilterTimeoutParams_test.go b/agreement/dynamicFilterTimeoutParams_test.go new file mode 100644 index 0000000000..c67af2050f --- /dev/null +++ b/agreement/dynamicFilterTimeoutParams_test.go @@ -0,0 +1,16 @@ +package agreement + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestSampleIndexIsValid(t *testing.T) { + require.Less(t, dynamicFilterTimeoutCredentialArrivalHistoryIdx, dynamicFilterCredentialArrivalHistory) +} + +func TestLowerBound(t *testing.T) { + require.Less(t, 20*time.Millisecond, dynamicFilterTimeoutLowerBound) +} From bab29e7c828c03f198bc38733fcd4f94fb897873 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Fri, 25 Aug 2023 20:22:44 -0400 Subject: [PATCH 138/232] test --- agreement/dynamicFilterTimeoutParams_test.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/agreement/dynamicFilterTimeoutParams_test.go b/agreement/dynamicFilterTimeoutParams_test.go index c67af2050f..23152b2411 100644 --- a/agreement/dynamicFilterTimeoutParams_test.go +++ b/agreement/dynamicFilterTimeoutParams_test.go @@ -8,7 +8,11 @@ import ( ) func TestSampleIndexIsValid(t *testing.T) { - require.Less(t, dynamicFilterTimeoutCredentialArrivalHistoryIdx, dynamicFilterCredentialArrivalHistory) + require.GreaterOrEqual(t, dynamicFilterCredentialArrivalHistory, 0) + require.GreaterOrEqual(t, dynamicFilterTimeoutCredentialArrivalHistoryIdx, 0) + if dynamicFilterCredentialArrivalHistory > 0 { + require.Less(t, dynamicFilterTimeoutCredentialArrivalHistoryIdx, dynamicFilterCredentialArrivalHistory) + } } func TestLowerBound(t *testing.T) { From 17c48d147dd90977093636a929e468da617f8188 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Fri, 25 Aug 2023 21:35:58 -0400 Subject: [PATCH 139/232] circular history buffer --- agreement/credentialArrivalHistory.go | 36 +++++++++++++++++++ agreement/player.go | 16 ++++----- agreement/player_test.go | 51 ++++++++++++++------------- 3 files changed, 70 insertions(+), 33 deletions(-) create mode 100644 agreement/credentialArrivalHistory.go diff --git a/agreement/credentialArrivalHistory.go b/agreement/credentialArrivalHistory.go new file mode 100644 index 0000000000..288970c9f4 --- /dev/null +++ b/agreement/credentialArrivalHistory.go @@ -0,0 +1,36 @@ +package agreement + +import "time" + +type credentialArrivalHistory struct { + history []time.Duration + writePtr int + full bool +} + +func newCredentialArrivalHistory(size int) *credentialArrivalHistory { + history := credentialArrivalHistory{history: make([]time.Duration, size, size)} + history.reset() + return &history +} + +func (history *credentialArrivalHistory) store(sample time.Duration) { + history.history[history.writePtr] = sample + history.writePtr++ + if history.writePtr == cap(history.history) { + history.full = true + history.writePtr = 0 + } +} + +func (history *credentialArrivalHistory) reset() { + history.writePtr = 0 + history.full = len(history.history) == 0 +} + +func (history *credentialArrivalHistory) isFull() bool { + if history == nil { + return false + } + return history.full +} diff --git a/agreement/player.go b/agreement/player.go index 3d726a2043..144547aeab 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -62,7 +62,7 @@ type player struct { // the history of arrival times of the lowest credential from previous // ronuds, used for calculating the filter timeout dynamically. - lowestCredentialArrivals []time.Duration + lowestCredentialArrivals *credentialArrivalHistory // The period 0 dynamic filter timeout calculated for this round, if set, // even if dynamic filter timeouts are not enabled. It is used for reporting @@ -293,12 +293,10 @@ func (p *player) updateCredentialArrivalHistory(r routerHandle, ver protocol.Con return 0 } - p.lowestCredentialArrivals = append(p.lowestCredentialArrivals, re.Vote.validatedAt) - - // trim history to the last dynamicFilterCredentialArrivalHistory samples. - if len(p.lowestCredentialArrivals) > dynamicFilterCredentialArrivalHistory { - p.lowestCredentialArrivals = p.lowestCredentialArrivals[len(p.lowestCredentialArrivals)-dynamicFilterCredentialArrivalHistory:] + if p.lowestCredentialArrivals == nil { + p.lowestCredentialArrivals = newCredentialArrivalHistory(dynamicFilterCredentialArrivalHistory) } + p.lowestCredentialArrivals.store(re.Vote.validatedAt) return re.Vote.validatedAt } @@ -311,13 +309,13 @@ func (p *player) calculateFilterTimeout(ver protocol.ConsensusVersion, tracer *t return FilterTimeout(p.Period, ver) } defaultTimeout := FilterTimeout(0, ver) - if dynamicFilterCredentialArrivalHistory > len(p.lowestCredentialArrivals) { + if !p.lowestCredentialArrivals.isFull() { // not enough samples, use the default return defaultTimeout } - sortedArrivals := make([]time.Duration, len(p.lowestCredentialArrivals)) - copy(sortedArrivals[:], p.lowestCredentialArrivals[:]) + sortedArrivals := make([]time.Duration, len(p.lowestCredentialArrivals.history)) + copy(sortedArrivals[:], p.lowestCredentialArrivals.history[:]) sort.Slice(sortedArrivals, func(i, j int) bool { return sortedArrivals[i] < sortedArrivals[j] }) dynamicTimeout := sortedArrivals[dynamicFilterTimeoutCredentialArrivalHistoryIdx] + dynamicFilterTimeoutGraceInterval diff --git a/agreement/player_test.go b/agreement/player_test.go index ba4fd23288..db7a9b1faa 100644 --- a/agreement/player_test.go +++ b/agreement/player_test.go @@ -3263,9 +3263,10 @@ func TestPlayerRetainsReceivedValidatedAtOneSample(t *testing.T) { assertCorrectReceivedAtSet(t, pWhite, pM, helper, r, p, pP, pV, m, protocol.ConsensusFuture) // assert lowest vote validateAt time was recorded into payloadArrivals - require.NotZero(t, dynamicFilterCredentialArrivalHistory) - require.Len(t, pWhite.lowestCredentialArrivals, 1) - require.Equal(t, 501*time.Millisecond, pWhite.lowestCredentialArrivals[0]) + require.NotNil(t, dynamicFilterCredentialArrivalHistory) + require.Equal(t, pWhite.lowestCredentialArrivals.writePtr, 1) + require.False(t, pWhite.lowestCredentialArrivals.isFull()) + require.Equal(t, 501*time.Millisecond, pWhite.lowestCredentialArrivals.history[0]) } // test that ReceivedAt and ValidateAt timing information are retained in proposalStore @@ -3278,9 +3279,9 @@ func TestPlayerRetainsReceivedValidatedAtForHistoryWindow(t *testing.T) { const p = period(0) pWhite, pM, helper := setupP(t, r-1, p, soft) - require.NotZero(t, dynamicFilterCredentialArrivalHistory) + require.NotNil(t, dynamicFilterCredentialArrivalHistory) - for i := 0; i < dynamicFilterCredentialArrivalHistory+1; i++ { + for i := 0; i < dynamicFilterCredentialArrivalHistory; i++ { // send voteVerified message pP, pV := helper.MakeRandomProposalPayload(t, r+round(i)-1) vVote := helper.MakeVerifiedVote(t, 0, r+round(i)-1, p, propose, *pV) @@ -3302,11 +3303,11 @@ func TestPlayerRetainsReceivedValidatedAtForHistoryWindow(t *testing.T) { } // assert lowest vote validateAt time was recorded into payloadArrivals - require.Len(t, pWhite.lowestCredentialArrivals, dynamicFilterCredentialArrivalHistory) + require.True(t, pWhite.lowestCredentialArrivals.isFull()) for i := 0; i < dynamicFilterCredentialArrivalHistory; i++ { // only the last historyLen samples are kept, so the first one is discarded - timestamp := 500 + i + 1 - require.Equal(t, time.Duration(timestamp)*time.Millisecond, pWhite.lowestCredentialArrivals[i]) + timestamp := 500 + i + require.Equal(t, time.Duration(timestamp)*time.Millisecond, pWhite.lowestCredentialArrivals.history[i]) } } @@ -3348,9 +3349,10 @@ func TestPlayerRetainsReceivedValidatedAtPPOneSample(t *testing.T) { assertCorrectReceivedAtSet(t, pWhite, pM, helper, r, p, pP, pV, proposalMsg, protocol.ConsensusFuture) // assert lowest vote validateAt time was recorded into payloadArrivals - require.NotZero(t, dynamicFilterCredentialArrivalHistory) - require.Len(t, pWhite.lowestCredentialArrivals, 1) - require.Equal(t, 502*time.Millisecond, pWhite.lowestCredentialArrivals[0]) + require.NotNil(t, dynamicFilterCredentialArrivalHistory) + require.Equal(t, pWhite.lowestCredentialArrivals.writePtr, 1) + require.False(t, pWhite.lowestCredentialArrivals.isFull()) + require.Equal(t, 502*time.Millisecond, pWhite.lowestCredentialArrivals.history[0]) } // test that ReceivedAt and ValidateAt timing information are retained in @@ -3366,9 +3368,9 @@ func TestPlayerRetainsReceivedValidatedAtPPForHistoryWindow(t *testing.T) { pWhite, pM, helper := setupP(t, r-1, p, soft) pP, pV := helper.MakeRandomProposalPayload(t, r-1) - require.NotZero(t, dynamicFilterCredentialArrivalHistory) + require.NotNil(t, dynamicFilterCredentialArrivalHistory) - for i := 0; i < dynamicFilterCredentialArrivalHistory+1; i++ { + for i := 0; i < dynamicFilterCredentialArrivalHistory; i++ { // create a PP message for an arbitrary proposal/payload similar to setupCompoundMessage vVote := helper.MakeVerifiedVote(t, 0, r+round(i)-1, p, propose, *pV) unverifiedVoteMsg := message{UnauthenticatedVote: vVote.u()} @@ -3397,11 +3399,11 @@ func TestPlayerRetainsReceivedValidatedAtPPForHistoryWindow(t *testing.T) { } // assert lowest vote validateAt time was recorded into payloadArrivals - require.Len(t, pWhite.lowestCredentialArrivals, dynamicFilterCredentialArrivalHistory) + require.True(t, pWhite.lowestCredentialArrivals.isFull()) for i := 0; i < dynamicFilterCredentialArrivalHistory; i++ { // only the last historyLen samples are kept, so the first one is discarded - timestamp := 500 + i + 1 - require.Equal(t, time.Duration(timestamp)*time.Millisecond, pWhite.lowestCredentialArrivals[i]) + timestamp := 500 + i + require.Equal(t, time.Duration(timestamp)*time.Millisecond, pWhite.lowestCredentialArrivals.history[i]) } } @@ -3454,9 +3456,10 @@ func TestPlayerRetainsReceivedValidatedAtAVPPOneSample(t *testing.T) { assertCorrectReceivedAtSet(t, pWhite, pM, helper, r, p, pP, pV, proposalMsg, protocol.ConsensusFuture) // assert lowest vote validateAt time was recorded into payloadArrivals - require.NotZero(t, dynamicFilterCredentialArrivalHistory) - require.Len(t, pWhite.lowestCredentialArrivals, 1) - require.Equal(t, 502*time.Millisecond, pWhite.lowestCredentialArrivals[0]) + require.NotNil(t, dynamicFilterCredentialArrivalHistory) + require.Equal(t, pWhite.lowestCredentialArrivals.writePtr, 1) + require.False(t, pWhite.lowestCredentialArrivals.isFull()) + require.Equal(t, 502*time.Millisecond, pWhite.lowestCredentialArrivals.history[0]) } func TestPlayerRetainsReceivedValidatedAtAVPPHistoryWindow(t *testing.T) { @@ -3465,9 +3468,9 @@ func TestPlayerRetainsReceivedValidatedAtAVPPHistoryWindow(t *testing.T) { const p = period(0) pWhite, pM, helper := setupP(t, r-1, p, soft) - require.NotZero(t, dynamicFilterCredentialArrivalHistory) + require.NotNil(t, dynamicFilterCredentialArrivalHistory) - for i := 0; i < dynamicFilterCredentialArrivalHistory+1; i++ { + for i := 0; i < dynamicFilterCredentialArrivalHistory; i++ { pP, pV := helper.MakeRandomProposalPayload(t, r+round(i)-1) // send votePresent message (mimicking the first AV message validating) @@ -3504,11 +3507,11 @@ func TestPlayerRetainsReceivedValidatedAtAVPPHistoryWindow(t *testing.T) { } // assert lowest vote validateAt time was recorded into payloadArrivals - require.Len(t, pWhite.lowestCredentialArrivals, dynamicFilterCredentialArrivalHistory) + require.True(t, pWhite.lowestCredentialArrivals.isFull()) for i := 0; i < dynamicFilterCredentialArrivalHistory; i++ { // only the last historyLen samples are kept, so the first one is discarded - timestamp := 500 + i + 1 - require.Equal(t, time.Duration(timestamp)*time.Millisecond, pWhite.lowestCredentialArrivals[i]) + timestamp := 500 + i + require.Equal(t, time.Duration(timestamp)*time.Millisecond, pWhite.lowestCredentialArrivals.history[i]) } } From 57c2e39a95d3887ef15a7787c63ad759a18c8bfe Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Fri, 25 Aug 2023 21:57:03 -0400 Subject: [PATCH 140/232] credential history tests --- agreement/credentialArrivalHistory_test.go | 60 ++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 agreement/credentialArrivalHistory_test.go diff --git a/agreement/credentialArrivalHistory_test.go b/agreement/credentialArrivalHistory_test.go new file mode 100644 index 0000000000..dcf9a9c581 --- /dev/null +++ b/agreement/credentialArrivalHistory_test.go @@ -0,0 +1,60 @@ +package agreement + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestCredentialHistoryStore(t *testing.T) { + size := 5 + buffer := newCredentialArrivalHistory(5) + // last store call overwrites the first one + for i := 0; i < size+1; i++ { + buffer.store(time.Duration(i)) + } + + require.True(t, buffer.isFull()) + require.Equal(t, time.Duration(size), buffer.history[0]) + for i := 1; i < size; i++ { + require.Equal(t, time.Duration(i), buffer.history[i]) + } +} + +func TestCredentialHistoryReset(t *testing.T) { + size := 5 + buffer := newCredentialArrivalHistory(5) + // last store call overwrites the first one + for i := 0; i < size+1; i++ { + buffer.store(time.Duration(i)) + } + + require.Equal(t, time.Duration(size), buffer.history[0]) + for i := 1; i < size; i++ { + require.Equal(t, time.Duration(i), buffer.history[i]) + } + require.True(t, buffer.isFull()) + buffer.reset() + require.False(t, buffer.isFull()) + buffer.store(time.Duration(100)) + require.Equal(t, time.Duration(100), buffer.history[0]) +} + +func TestCredentialHistoryIsFull(t *testing.T) { + var buffer *credentialArrivalHistory + require.False(t, buffer.isFull()) + + size := 5 + buffer = newCredentialArrivalHistory(size) + require.False(t, buffer.isFull()) + + for i := 0; i < size+10; i++ { + buffer.store(time.Duration(i)) + if i < size-1 { + require.False(t, buffer.isFull()) + } else { + require.True(t, buffer.isFull()) + } + } +} From a5bb3145c383addc05a00e0b2de10a6e0b77e044 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Fri, 25 Aug 2023 22:06:31 -0400 Subject: [PATCH 141/232] move logic to credentialhistory --- agreement/credentialArrivalHistory.go | 12 +++++++++++- agreement/credentialArrivalHistory_test.go | 18 ++++++++++++++++-- agreement/player.go | 6 +----- 3 files changed, 28 insertions(+), 8 deletions(-) diff --git a/agreement/credentialArrivalHistory.go b/agreement/credentialArrivalHistory.go index 288970c9f4..eaacdc3add 100644 --- a/agreement/credentialArrivalHistory.go +++ b/agreement/credentialArrivalHistory.go @@ -1,6 +1,9 @@ package agreement -import "time" +import ( + "sort" + "time" +) type credentialArrivalHistory struct { history []time.Duration @@ -34,3 +37,10 @@ func (history *credentialArrivalHistory) isFull() bool { } return history.full } + +func (history *credentialArrivalHistory) orderStatistics(idx int) time.Duration { + sortedArrivals := make([]time.Duration, len(history.history)) + copy(sortedArrivals[:], history.history[:]) + sort.Slice(sortedArrivals, func(i, j int) bool { return sortedArrivals[i] < sortedArrivals[j] }) + return sortedArrivals[idx] +} diff --git a/agreement/credentialArrivalHistory_test.go b/agreement/credentialArrivalHistory_test.go index dcf9a9c581..67df554732 100644 --- a/agreement/credentialArrivalHistory_test.go +++ b/agreement/credentialArrivalHistory_test.go @@ -9,7 +9,7 @@ import ( func TestCredentialHistoryStore(t *testing.T) { size := 5 - buffer := newCredentialArrivalHistory(5) + buffer := newCredentialArrivalHistory(size) // last store call overwrites the first one for i := 0; i < size+1; i++ { buffer.store(time.Duration(i)) @@ -24,7 +24,7 @@ func TestCredentialHistoryStore(t *testing.T) { func TestCredentialHistoryReset(t *testing.T) { size := 5 - buffer := newCredentialArrivalHistory(5) + buffer := newCredentialArrivalHistory(size) // last store call overwrites the first one for i := 0; i < size+1; i++ { buffer.store(time.Duration(i)) @@ -58,3 +58,17 @@ func TestCredentialHistoryIsFull(t *testing.T) { } } } + +func TestOrderStatistics(t *testing.T) { + size := 5 + buffer := newCredentialArrivalHistory(size) + require.False(t, buffer.isFull()) + + for i := 0; i < size; i++ { + buffer.store(time.Duration(size - i)) + } + + for i := 0; i < size; i++ { + require.Equal(t, time.Duration(i), buffer.orderStatistics(i)) + } +} diff --git a/agreement/player.go b/agreement/player.go index 144547aeab..5723d02f01 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -17,7 +17,6 @@ package agreement import ( - "sort" "time" "github.com/algorand/go-algorand/config" @@ -314,10 +313,7 @@ func (p *player) calculateFilterTimeout(ver protocol.ConsensusVersion, tracer *t return defaultTimeout } - sortedArrivals := make([]time.Duration, len(p.lowestCredentialArrivals.history)) - copy(sortedArrivals[:], p.lowestCredentialArrivals.history[:]) - sort.Slice(sortedArrivals, func(i, j int) bool { return sortedArrivals[i] < sortedArrivals[j] }) - dynamicTimeout := sortedArrivals[dynamicFilterTimeoutCredentialArrivalHistoryIdx] + dynamicFilterTimeoutGraceInterval + dynamicTimeout := p.lowestCredentialArrivals.orderStatistics(dynamicFilterTimeoutCredentialArrivalHistoryIdx) + dynamicFilterTimeoutGraceInterval // Make sure the dynamic filter timeout is not too small nor too large clampedTimeout := dynamicTimeout From 7a24b276b3e7c3b4cd58c61bc72e99b385abeed7 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Fri, 25 Aug 2023 22:36:15 -0400 Subject: [PATCH 142/232] efficiency remark --- agreement/credentialArrivalHistory.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/agreement/credentialArrivalHistory.go b/agreement/credentialArrivalHistory.go index eaacdc3add..15d7faceb1 100644 --- a/agreement/credentialArrivalHistory.go +++ b/agreement/credentialArrivalHistory.go @@ -39,6 +39,8 @@ func (history *credentialArrivalHistory) isFull() bool { } func (history *credentialArrivalHistory) orderStatistics(idx int) time.Duration { + // if history.history is long, then we could optimize this function to use + // the linear time order statistics algorithm. sortedArrivals := make([]time.Duration, len(history.history)) copy(sortedArrivals[:], history.history[:]) sort.Slice(sortedArrivals, func(i, j int) bool { return sortedArrivals[i] < sortedArrivals[j] }) From e32460ba2722ace7dcbca04ea39db14cd8666a75 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Fri, 25 Aug 2023 22:51:00 -0400 Subject: [PATCH 143/232] fix off by 1 in test --- agreement/credentialArrivalHistory_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/agreement/credentialArrivalHistory_test.go b/agreement/credentialArrivalHistory_test.go index 67df554732..b4c98cab0b 100644 --- a/agreement/credentialArrivalHistory_test.go +++ b/agreement/credentialArrivalHistory_test.go @@ -67,8 +67,9 @@ func TestOrderStatistics(t *testing.T) { for i := 0; i < size; i++ { buffer.store(time.Duration(size - i)) } + require.True(t, buffer.isFull()) for i := 0; i < size; i++ { - require.Equal(t, time.Duration(i), buffer.orderStatistics(i)) + require.Equal(t, time.Duration(i+1), buffer.orderStatistics(i)) } } From d8438b7785c887b0b84f71394c9383fda6a5707b Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Fri, 25 Aug 2023 22:54:07 -0400 Subject: [PATCH 144/232] sanity --- agreement/credentialArrivalHistory.go | 18 +++++++++++++++++- agreement/credentialArrivalHistory_test.go | 16 ++++++++++++++++ agreement/dynamicFilterTimeoutParams_test.go | 16 ++++++++++++++++ 3 files changed, 49 insertions(+), 1 deletion(-) diff --git a/agreement/credentialArrivalHistory.go b/agreement/credentialArrivalHistory.go index 15d7faceb1..5fdb99668d 100644 --- a/agreement/credentialArrivalHistory.go +++ b/agreement/credentialArrivalHistory.go @@ -1,3 +1,19 @@ +// Copyright (C) 2019-2023 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + package agreement import ( @@ -12,7 +28,7 @@ type credentialArrivalHistory struct { } func newCredentialArrivalHistory(size int) *credentialArrivalHistory { - history := credentialArrivalHistory{history: make([]time.Duration, size, size)} + history := credentialArrivalHistory{history: make([]time.Duration, size)} history.reset() return &history } diff --git a/agreement/credentialArrivalHistory_test.go b/agreement/credentialArrivalHistory_test.go index b4c98cab0b..2088c2a4e1 100644 --- a/agreement/credentialArrivalHistory_test.go +++ b/agreement/credentialArrivalHistory_test.go @@ -1,3 +1,19 @@ +// Copyright (C) 2019-2023 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + package agreement import ( diff --git a/agreement/dynamicFilterTimeoutParams_test.go b/agreement/dynamicFilterTimeoutParams_test.go index 23152b2411..8da2968b12 100644 --- a/agreement/dynamicFilterTimeoutParams_test.go +++ b/agreement/dynamicFilterTimeoutParams_test.go @@ -1,3 +1,19 @@ +// Copyright (C) 2019-2023 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + package agreement import ( From 253ab05146c4c26b978f607018ed83364b7aac2b Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Sat, 26 Aug 2023 16:05:56 -0400 Subject: [PATCH 145/232] fix unittest: now looking at specific rounds --- agreement/player_test.go | 51 ++++++++++++++++++++++++++++++++-------- 1 file changed, 41 insertions(+), 10 deletions(-) diff --git a/agreement/player_test.go b/agreement/player_test.go index 3b24e30710..6cf9a43550 100644 --- a/agreement/player_test.go +++ b/agreement/player_test.go @@ -3522,19 +3522,39 @@ func TestPlayerRetainsReceivedValidatedAtAVPPOneSample(t *testing.T) { const r = round(20239) const p = period(0) pWhite, pM, helper := setupP(t, r-1, p, soft) - pP, pV := helper.MakeRandomProposalPayload(t, r-credentialRoundLag-1) + pP, pV := helper.MakeRandomProposalPayload(t, r-1) + + // Move to round r, no credentials arrived. + // send voteVerified message + vVote := helper.MakeVerifiedVote(t, 0, r-1, p, propose, *pV) + inMsg := messageEvent{T: voteVerified, Input: message{Vote: vVote, UnauthenticatedVote: vVote.u()}} + inMsg = inMsg.AttachValidatedAt(501 * time.Millisecond) + err, panicErr := pM.transition(inMsg) + require.NoError(t, err) + require.NoError(t, panicErr) + + // send payloadPresent message + m := message{UnauthenticatedProposal: pP.u()} + inMsg = messageEvent{T: payloadPresent, Input: m} + inMsg = inMsg.AttachReceivedAt(time.Second) + err, panicErr = pM.transition(inMsg) + require.NoError(t, err) + require.NoError(t, panicErr) + + assertCorrectReceivedAtSet(t, pWhite, pM, helper, r, p, pP, pV, m, protocol.ConsensusFuture) + require.False(t, pWhite.lowestCredentialArrivals.isFull()) + require.Equal(t, pWhite.lowestCredentialArrivals.writePtr, 0) // send votePresent message (mimicking the first AV message validating) - vVote := helper.MakeVerifiedVote(t, 0, r-credentialRoundLag-1, p, propose, *pV) + vVote = helper.MakeVerifiedVote(t, 0, r-credentialRoundLag-1, p, propose, *pV) unverifiedVoteMsg := message{UnauthenticatedVote: vVote.u()} - inMsg := messageEvent{T: votePresent, Input: unverifiedVoteMsg} - err, panicErr := pM.transition(inMsg) + inMsg = messageEvent{T: votePresent, Input: unverifiedVoteMsg} + err, panicErr = pM.transition(inMsg) require.NoError(t, err) require.NoError(t, panicErr) // make sure vote verify requests - fmt.Println("targeting", r-credentialRoundLag-1) - verifyEvent := ev(cryptoAction{T: verifyVote, M: unverifiedVoteMsg, Round: r - credentialRoundLag - 1, Period: p, Step: propose, TaskIndex: 1}) + verifyEvent := ev(cryptoAction{T: verifyVote, M: unverifiedVoteMsg, Round: r - 1, Period: p, Step: propose, TaskIndex: 1}) require.Truef(t, pM.getTrace().Contains(verifyEvent), "Player should verify vote") // send voteVerified @@ -3554,11 +3574,22 @@ func TestPlayerRetainsReceivedValidatedAtAVPPOneSample(t *testing.T) { require.NoError(t, err) require.NoError(t, panicErr) - // make sure no second request to verify this vote - verifyEvent = ev(cryptoAction{T: verifyVote, M: unverifiedVoteMsg, Round: r - 1, Period: p, Step: propose, TaskIndex: 1}) - require.Equal(t, 1, pM.getTrace().CountEvent(verifyEvent), "Player should not verify second vote") + // move to round r+1, triggering history update + vVote = helper.MakeVerifiedVote(t, 0, r, p, propose, *pV) + inMsg = messageEvent{T: voteVerified, Input: message{Vote: vVote, UnauthenticatedVote: vVote.u()}} + inMsg = inMsg.AttachValidatedAt(time.Second) + err, panicErr = pM.transition(inMsg) + require.NoError(t, err) + require.NoError(t, panicErr) - assertCorrectReceivedAtSet(t, pWhite, pM, helper, r, p, pP, pV, proposalMsg, protocol.ConsensusFuture) + // send payloadPresent message + m = message{UnauthenticatedProposal: pP.u()} + inMsg = messageEvent{T: payloadPresent, Input: m} + inMsg = inMsg.AttachReceivedAt(time.Second) + err, panicErr = pM.transition(inMsg) + require.NoError(t, err) + require.NoError(t, panicErr) + moveToRound(t, pWhite, pM, helper, r+1, p, pP, pV, m, protocol.ConsensusFuture) // assert lowest vote validateAt time was recorded into payloadArrivals require.NotNil(t, dynamicFilterCredentialArrivalHistory) From 04f607a47862b688683734376fa77a63f1a5d0ca Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Sat, 26 Aug 2023 16:56:15 -0400 Subject: [PATCH 146/232] tighter checks --- agreement/player.go | 2 +- agreement/player_test.go | 12 ++++++------ agreement/proposalManager.go | 2 +- agreement/proposalManager_test.go | 2 +- agreement/router.go | 2 +- agreement/service_test.go | 12 ++++++------ 6 files changed, 16 insertions(+), 16 deletions(-) diff --git a/agreement/player.go b/agreement/player.go index d70a4b9aa3..bb868e2dee 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -293,7 +293,7 @@ func (p *player) updateCredentialArrivalHistory(r routerHandle, ver protocol.Con // look up the validatedAt time of the winning proposal-vote from credentialRoundLag ago, // by now we must have seen the lowest credential for that round. - credHistoryRound := p.Round.SubSaturate(credentialRoundLag + 1) + credHistoryRound := p.Round.SubSaturate(credentialRoundLag) re := readLowestEvent{T: readLowestVote, Round: credHistoryRound, Period: p.Period} re = r.dispatch(*p, re, proposalMachineRound, credHistoryRound, p.Period, 0).(readLowestEvent) if !re.Filled { diff --git a/agreement/player_test.go b/agreement/player_test.go index 6cf9a43550..6d22cb36f0 100644 --- a/agreement/player_test.go +++ b/agreement/player_test.go @@ -3295,7 +3295,7 @@ func TestPlayerRetainsLateReceivedValidatedAtOneSample(t *testing.T) { // Old credential arrives // send voteVerified message - vVote = helper.MakeVerifiedVote(t, 0, r-credentialRoundLag-1, p, propose, *pV) + vVote = helper.MakeVerifiedVote(t, 0, r-credentialRoundLag, p, propose, *pV) inMsg = messageEvent{T: voteVerified, Input: message{Vote: vVote, UnauthenticatedVote: vVote.u()}} inMsg = inMsg.AttachValidatedAt(501 * time.Millisecond) err, panicErr = pM.transition(inMsg) @@ -3346,7 +3346,7 @@ func TestPlayerRetainsReceivedValidatedAtForHistoryWindow(t *testing.T) { require.NotNil(t, dynamicFilterCredentialArrivalHistory) - for i := 0; i < dynamicFilterCredentialArrivalHistory+int(credentialRoundLag)+1; i++ { + for i := 0; i < dynamicFilterCredentialArrivalHistory+int(credentialRoundLag); i++ { // send voteVerified message pP, pV := helper.MakeRandomProposalPayload(t, r+round(i)-1) vVote := helper.MakeVerifiedVote(t, 0, r+round(i)-1, p, propose, *pV) @@ -3410,7 +3410,7 @@ func TestPlayerRetainsReceivedValidatedAtPPOneSample(t *testing.T) { require.Equal(t, pWhite.lowestCredentialArrivals.writePtr, 0) // create a PP message for the round we're going to take the sample from when round r-1 ends - vVote = helper.MakeVerifiedVote(t, 0, r-credentialRoundLag-1, p, propose, *pV) + vVote = helper.MakeVerifiedVote(t, 0, r-credentialRoundLag, p, propose, *pV) unverifiedVoteMsg := message{UnauthenticatedVote: vVote.u()} proposalMsg := message{UnauthenticatedProposal: pP.u()} compoundMsg := messageEvent{T: votePresent, Input: unverifiedVoteMsg, @@ -3474,7 +3474,7 @@ func TestPlayerRetainsReceivedValidatedAtPPForHistoryWindow(t *testing.T) { require.NotNil(t, dynamicFilterCredentialArrivalHistory) - for i := 0; i < dynamicFilterCredentialArrivalHistory+int(credentialRoundLag)+1; i++ { + for i := 0; i < dynamicFilterCredentialArrivalHistory+int(credentialRoundLag); i++ { // create a PP message for an arbitrary proposal/payload similar to setupCompoundMessage vVote := helper.MakeVerifiedVote(t, 0, r+round(i)-1, p, propose, *pV) unverifiedVoteMsg := message{UnauthenticatedVote: vVote.u()} @@ -3546,7 +3546,7 @@ func TestPlayerRetainsReceivedValidatedAtAVPPOneSample(t *testing.T) { require.Equal(t, pWhite.lowestCredentialArrivals.writePtr, 0) // send votePresent message (mimicking the first AV message validating) - vVote = helper.MakeVerifiedVote(t, 0, r-credentialRoundLag-1, p, propose, *pV) + vVote = helper.MakeVerifiedVote(t, 0, r-credentialRoundLag, p, propose, *pV) unverifiedVoteMsg := message{UnauthenticatedVote: vVote.u()} inMsg = messageEvent{T: votePresent, Input: unverifiedVoteMsg} err, panicErr = pM.transition(inMsg) @@ -3606,7 +3606,7 @@ func TestPlayerRetainsReceivedValidatedAtAVPPHistoryWindow(t *testing.T) { require.NotNil(t, dynamicFilterCredentialArrivalHistory) - for i := 0; i < dynamicFilterCredentialArrivalHistory+int(credentialRoundLag)+1; i++ { + for i := 0; i < dynamicFilterCredentialArrivalHistory+int(credentialRoundLag); i++ { pP, pV := helper.MakeRandomProposalPayload(t, r+round(i)-1) // send votePresent message (mimicking the first AV message validating) diff --git a/agreement/proposalManager.go b/agreement/proposalManager.go index 675868ebf6..4e72709988 100644 --- a/agreement/proposalManager.go +++ b/agreement/proposalManager.go @@ -232,7 +232,7 @@ func (m *proposalManager) filterProposalVote(p player, r routerHandle, uv unauth // voteFresh determines whether a proposal satisfies freshness rules. func proposalFresh(freshData freshnessData, vote unauthenticatedVote) error { - if vote.R.Round < freshData.PlayerRound && freshData.PlayerRound <= vote.R.Round+credentialRoundLag+1 && vote.R.Period == 0 && vote.R.Step == 0 { + if vote.R.Round < freshData.PlayerRound && freshData.PlayerRound <= vote.R.Round+credentialRoundLag && vote.R.Period == 0 && vote.R.Step == 0 { if dynamicFilterCredentialArrivalHistory > 0 { // continue processing old period 0 votes so we could track their // arrival times and inform setting the filter timeout dynamically. diff --git a/agreement/proposalManager_test.go b/agreement/proposalManager_test.go index 700fedb3d4..47cd8ce123 100644 --- a/agreement/proposalManager_test.go +++ b/agreement/proposalManager_test.go @@ -325,7 +325,7 @@ func TestProposalFreshAdjacentPeriods(t *testing.T) { // vote in credentialRoundLag period 0 should be fine pV = helper.MakeRandomProposalValue() - uv = helper.MakeUnauthenticatedVote(t, 0, r-credentialRoundLag-1, 0, 0, *pV) + uv = helper.MakeUnauthenticatedVote(t, 0, r-credentialRoundLag, 0, 0, *pV) inMsg = filterableMessageEvent{ FreshnessData: currentPlayerState, messageEvent: messageEvent{ diff --git a/agreement/router.go b/agreement/router.go index f733df943b..6ab144470f 100644 --- a/agreement/router.go +++ b/agreement/router.go @@ -156,7 +156,7 @@ func (router *rootRouter) update(state player, r round, gc bool) { // We may still receive credential messages from old rounds. Keep // old round routers around, for as long as those credentials may // arrive to keep track of them. - rr := r + credentialRoundLag + 1 + rr := r + credentialRoundLag if rr >= state.Round { children[r] = c } diff --git a/agreement/service_test.go b/agreement/service_test.go index f1e4d67a74..5e5d77efcc 100644 --- a/agreement/service_test.go +++ b/agreement/service_test.go @@ -1043,14 +1043,14 @@ func TestAgreementSynchronousFuture5_DynamicFilterRounds(t *testing.T) { filterTimeouts := simulateAgreementWithConsensusVersion(t, 5, rounds, disabled, consensusVersion) require.Len(t, filterTimeouts, rounds-1) - for i := 1; i < baseHistoryRounds; i++ { + for i := 1; i < baseHistoryRounds-1; i++ { require.Equal(t, filterTimeouts[i-1], filterTimeouts[i]) } // dynamic filter timeout kicks in when history window is full - require.Less(t, filterTimeouts[baseHistoryRounds], filterTimeouts[baseHistoryRounds-1]) + require.Less(t, filterTimeouts[baseHistoryRounds-1], filterTimeouts[baseHistoryRounds-2]) - for i := baseHistoryRounds + 1; i < len(filterTimeouts); i++ { + for i := baseHistoryRounds; i < len(filterTimeouts); i++ { require.Equal(t, filterTimeouts[i-1], filterTimeouts[i]) } } @@ -1107,10 +1107,10 @@ func TestDynamicFilterTimeoutResets(t *testing.T) { for i := range clocks { require.Len(t, filterTimeouts[i], baseHistoryRounds+1) - for j := 1; j < baseHistoryRounds-1; j++ { + for j := 1; j < baseHistoryRounds-2; j++ { require.Equal(t, filterTimeouts[i][j-1], filterTimeouts[i][j]) } - require.Less(t, filterTimeouts[i][baseHistoryRounds], filterTimeouts[i][baseHistoryRounds-1]) + require.Less(t, filterTimeouts[i][baseHistoryRounds-1], filterTimeouts[i][baseHistoryRounds-2]) } // force fast partition recovery into bottom @@ -1155,7 +1155,7 @@ func TestDynamicFilterTimeoutResets(t *testing.T) { require.Len(t, filterTimeoutsPostRecovery[i], baseHistoryRounds) // check that history was discarded, so filter time increased back to its original default require.Less(t, filterTimeouts[i][baseHistoryRounds], filterTimeoutsPostRecovery[i][0]) - require.Equal(t, filterTimeouts[i][baseHistoryRounds-1], filterTimeoutsPostRecovery[i][0]) + require.Equal(t, filterTimeouts[i][baseHistoryRounds-2], filterTimeoutsPostRecovery[i][0]) // check that filter timeout was updated to at the end of the history window for j := 1; j < dynamicFilterCredentialArrivalHistory-1; j++ { From 5e7ad382da0db1ce70ef46cdf94c8198b6c35548 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Sat, 26 Aug 2023 17:12:45 -0400 Subject: [PATCH 147/232] tighter checks in tests --- agreement/player.go | 6 +++--- agreement/proposalManager_test.go | 22 ++++++++++++++++++---- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/agreement/player.go b/agreement/player.go index bb868e2dee..2bc4efc001 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -286,14 +286,14 @@ func (p *player) updateCredentialArrivalHistory(r routerHandle, ver protocol.Con return 0 } - if p.Round <= credentialRoundLag { + if p.Round < credentialRoundLag { // not sufficiently many rounds had passed to collect any measurement return 0 } // look up the validatedAt time of the winning proposal-vote from credentialRoundLag ago, - // by now we must have seen the lowest credential for that round. - credHistoryRound := p.Round.SubSaturate(credentialRoundLag) + // by now we should have seen the lowest credential for that round. + credHistoryRound := p.Round - credentialRoundLag re := readLowestEvent{T: readLowestVote, Round: credHistoryRound, Period: p.Period} re = r.dispatch(*p, re, proposalMachineRound, credHistoryRound, p.Period, 0).(readLowestEvent) if !re.Filled { diff --git a/agreement/proposalManager_test.go b/agreement/proposalManager_test.go index 47cd8ce123..629f88eacf 100644 --- a/agreement/proposalManager_test.go +++ b/agreement/proposalManager_test.go @@ -323,7 +323,7 @@ func TestProposalFreshAdjacentPeriods(t *testing.T) { } b.AddInOutPair(inMsg, filteredEvent{T: voteFiltered}) - // vote in credentialRoundLag period 0 should be fine + // vote from credentialRoundLag rounds ago and period 0 should be fine pV = helper.MakeRandomProposalValue() uv = helper.MakeUnauthenticatedVote(t, 0, r-credentialRoundLag, 0, 0, *pV) inMsg = filterableMessageEvent{ @@ -337,9 +337,9 @@ func TestProposalFreshAdjacentPeriods(t *testing.T) { } b.AddInOutPair(inMsg, emptyEvent{}) - // vote in credentialRoundLag period > 0 should be filtered + // vote from credentialRoundLag ago and period > 0 should be filtered pV = helper.MakeRandomProposalValue() - uv = helper.MakeUnauthenticatedVote(t, 0, r-credentialRoundLag-1, 1, 0, *pV) + uv = helper.MakeUnauthenticatedVote(t, 0, r-credentialRoundLag, 1, 0, *pV) inMsg = filterableMessageEvent{ FreshnessData: currentPlayerState, messageEvent: messageEvent{ @@ -353,7 +353,21 @@ func TestProposalFreshAdjacentPeriods(t *testing.T) { // vote older than credentialRoundLag should be rejected even if period 0 pV = helper.MakeRandomProposalValue() - uv = helper.MakeUnauthenticatedVote(t, 0, r-credentialRoundLag-2, 0, 0, *pV) + uv = helper.MakeUnauthenticatedVote(t, 0, r-credentialRoundLag-1, 0, 0, *pV) + inMsg = filterableMessageEvent{ + FreshnessData: currentPlayerState, + messageEvent: messageEvent{ + T: votePresent, + Input: message{ + UnauthenticatedVote: uv, + }, + }, + } + b.AddInOutPair(inMsg, filteredEvent{T: voteFiltered}) + + // vote older than credentialRoundLag should be rejected on any period + pV = helper.MakeRandomProposalValue() + uv = helper.MakeUnauthenticatedVote(t, 0, r-credentialRoundLag-1, 1, 0, *pV) inMsg = filterableMessageEvent{ FreshnessData: currentPlayerState, messageEvent: messageEvent{ From e01ffe6bc52f4567544a97788b75a09ba6badf7b Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Sat, 26 Aug 2023 17:36:52 -0400 Subject: [PATCH 148/232] msgp --- agreement/msgp_gen.go | 106 +++++++++++++++++++++--------------------- 1 file changed, 53 insertions(+), 53 deletions(-) diff --git a/agreement/msgp_gen.go b/agreement/msgp_gen.go index 6c293e4edb..5a5efa7ddc 100644 --- a/agreement/msgp_gen.go +++ b/agreement/msgp_gen.go @@ -3949,16 +3949,16 @@ func PeriodRouterMaxSize() (s int) { func (z *player) MarshalMsg(b []byte) (o []byte) { o = msgp.Require(b, z.Msgsize()) // omitempty: check for empty values - zb0002Len := uint32(9) - var zb0002Mask uint16 /* 12 bits */ + zb0001Len := uint32(9) + var zb0001Mask uint16 /* 12 bits */ if (*z).OldDeadline == 0 { - zb0002Len-- - zb0002Mask |= 0x1 + zb0001Len-- + zb0001Mask |= 0x1 } - // variable map header, size zb0002Len - o = append(o, 0x80|uint8(zb0002Len)) - if zb0002Len != 0 { - if (zb0002Mask & 0x1) == 0 { // if not empty + // variable map header, size zb0001Len + o = append(o, 0x80|uint8(zb0001Len)) + if zb0001Len != 0 { + if (zb0001Mask & 0x1) == 0 { // if not empty // string "Deadline" o = append(o, 0xa8, 0x44, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65) o = msgp.AppendDuration(o, (*z).OldDeadline) @@ -4000,101 +4000,101 @@ func (_ *player) CanMarshalMsg(z interface{}) bool { func (z *player) UnmarshalMsg(bts []byte) (o []byte, err error) { var field []byte _ = field - var zb0002 int - var zb0003 bool - zb0002, zb0003, bts, err = msgp.ReadMapHeaderBytes(bts) + var zb0001 int + var zb0002 bool + zb0001, zb0002, bts, err = msgp.ReadMapHeaderBytes(bts) if _, ok := err.(msgp.TypeError); ok { - zb0002, zb0003, bts, err = msgp.ReadArrayHeaderBytes(bts) + zb0001, zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) if err != nil { err = msgp.WrapError(err) return } - if zb0002 > 0 { - zb0002-- + if zb0001 > 0 { + zb0001-- bts, err = (*z).Round.UnmarshalMsg(bts) if err != nil { err = msgp.WrapError(err, "struct-from-array", "Round") return } } - if zb0002 > 0 { - zb0002-- + if zb0001 > 0 { + zb0001-- { - var zb0004 uint64 - zb0004, bts, err = msgp.ReadUint64Bytes(bts) + var zb0003 uint64 + zb0003, bts, err = msgp.ReadUint64Bytes(bts) if err != nil { err = msgp.WrapError(err, "struct-from-array", "Period") return } - (*z).Period = period(zb0004) + (*z).Period = period(zb0003) } } - if zb0002 > 0 { - zb0002-- + if zb0001 > 0 { + zb0001-- { - var zb0005 uint64 - zb0005, bts, err = msgp.ReadUint64Bytes(bts) + var zb0004 uint64 + zb0004, bts, err = msgp.ReadUint64Bytes(bts) if err != nil { err = msgp.WrapError(err, "struct-from-array", "Step") return } - (*z).Step = step(zb0005) + (*z).Step = step(zb0004) } } - if zb0002 > 0 { - zb0002-- + if zb0001 > 0 { + zb0001-- { - var zb0006 uint64 - zb0006, bts, err = msgp.ReadUint64Bytes(bts) + var zb0005 uint64 + zb0005, bts, err = msgp.ReadUint64Bytes(bts) if err != nil { err = msgp.WrapError(err, "struct-from-array", "LastConcluding") return } - (*z).LastConcluding = step(zb0006) + (*z).LastConcluding = step(zb0005) } } - if zb0002 > 0 { - zb0002-- + if zb0001 > 0 { + zb0001-- bts, err = (*z).Deadline.UnmarshalMsg(bts) if err != nil { err = msgp.WrapError(err, "struct-from-array", "Deadline") return } } - if zb0002 > 0 { - zb0002-- + if zb0001 > 0 { + zb0001-- (*z).OldDeadline, bts, err = msgp.ReadDurationBytes(bts) if err != nil { err = msgp.WrapError(err, "struct-from-array", "OldDeadline") return } } - if zb0002 > 0 { - zb0002-- + if zb0001 > 0 { + zb0001-- (*z).Napping, bts, err = msgp.ReadBoolBytes(bts) if err != nil { err = msgp.WrapError(err, "struct-from-array", "Napping") return } } - if zb0002 > 0 { - zb0002-- + if zb0001 > 0 { + zb0001-- (*z).FastRecoveryDeadline, bts, err = msgp.ReadDurationBytes(bts) if err != nil { err = msgp.WrapError(err, "struct-from-array", "FastRecoveryDeadline") return } } - if zb0002 > 0 { - zb0002-- + if zb0001 > 0 { + zb0001-- bts, err = (*z).Pending.UnmarshalMsg(bts) if err != nil { err = msgp.WrapError(err, "struct-from-array", "Pending") return } } - if zb0002 > 0 { - err = msgp.ErrTooManyArrayFields(zb0002) + if zb0001 > 0 { + err = msgp.ErrTooManyArrayFields(zb0001) if err != nil { err = msgp.WrapError(err, "struct-from-array") return @@ -4105,11 +4105,11 @@ func (z *player) UnmarshalMsg(bts []byte) (o []byte, err error) { err = msgp.WrapError(err) return } - if zb0003 { + if zb0002 { (*z) = player{} } - for zb0002 > 0 { - zb0002-- + for zb0001 > 0 { + zb0001-- field, bts, err = msgp.ReadMapKeyZC(bts) if err != nil { err = msgp.WrapError(err) @@ -4124,33 +4124,33 @@ func (z *player) UnmarshalMsg(bts []byte) (o []byte, err error) { } case "Period": { - var zb0007 uint64 - zb0007, bts, err = msgp.ReadUint64Bytes(bts) + var zb0006 uint64 + zb0006, bts, err = msgp.ReadUint64Bytes(bts) if err != nil { err = msgp.WrapError(err, "Period") return } - (*z).Period = period(zb0007) + (*z).Period = period(zb0006) } case "Step": { - var zb0008 uint64 - zb0008, bts, err = msgp.ReadUint64Bytes(bts) + var zb0007 uint64 + zb0007, bts, err = msgp.ReadUint64Bytes(bts) if err != nil { err = msgp.WrapError(err, "Step") return } - (*z).Step = step(zb0008) + (*z).Step = step(zb0007) } case "LastConcluding": { - var zb0009 uint64 - zb0009, bts, err = msgp.ReadUint64Bytes(bts) + var zb0008 uint64 + zb0008, bts, err = msgp.ReadUint64Bytes(bts) if err != nil { err = msgp.WrapError(err, "LastConcluding") return } - (*z).LastConcluding = step(zb0009) + (*z).LastConcluding = step(zb0008) } case "TimersDeadline": bts, err = (*z).Deadline.UnmarshalMsg(bts) From 6e676cfce46429ec072b04f4fb3d8f1f3376558c Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Sun, 27 Aug 2023 09:02:19 -0400 Subject: [PATCH 149/232] partition test comments --- agreement/credentialArrivalHistory_test.go | 4 ++++ agreement/dynamicFilterTimeoutParams_test.go | 2 ++ 2 files changed, 6 insertions(+) diff --git a/agreement/credentialArrivalHistory_test.go b/agreement/credentialArrivalHistory_test.go index 2088c2a4e1..3b85a221ba 100644 --- a/agreement/credentialArrivalHistory_test.go +++ b/agreement/credentialArrivalHistory_test.go @@ -24,6 +24,7 @@ import ( ) func TestCredentialHistoryStore(t *testing.T) { + // partitiontest.PartitionTest(t) size := 5 buffer := newCredentialArrivalHistory(size) // last store call overwrites the first one @@ -39,6 +40,7 @@ func TestCredentialHistoryStore(t *testing.T) { } func TestCredentialHistoryReset(t *testing.T) { + // partitiontest.PartitionTest(t) size := 5 buffer := newCredentialArrivalHistory(size) // last store call overwrites the first one @@ -58,6 +60,7 @@ func TestCredentialHistoryReset(t *testing.T) { } func TestCredentialHistoryIsFull(t *testing.T) { + // partitiontest.PartitionTest(t) var buffer *credentialArrivalHistory require.False(t, buffer.isFull()) @@ -76,6 +79,7 @@ func TestCredentialHistoryIsFull(t *testing.T) { } func TestOrderStatistics(t *testing.T) { + // partitiontest.PartitionTest(t) size := 5 buffer := newCredentialArrivalHistory(size) require.False(t, buffer.isFull()) diff --git a/agreement/dynamicFilterTimeoutParams_test.go b/agreement/dynamicFilterTimeoutParams_test.go index 8da2968b12..71d755e674 100644 --- a/agreement/dynamicFilterTimeoutParams_test.go +++ b/agreement/dynamicFilterTimeoutParams_test.go @@ -24,6 +24,7 @@ import ( ) func TestSampleIndexIsValid(t *testing.T) { + // partitiontest.PartitionTest(t) require.GreaterOrEqual(t, dynamicFilterCredentialArrivalHistory, 0) require.GreaterOrEqual(t, dynamicFilterTimeoutCredentialArrivalHistoryIdx, 0) if dynamicFilterCredentialArrivalHistory > 0 { @@ -32,5 +33,6 @@ func TestSampleIndexIsValid(t *testing.T) { } func TestLowerBound(t *testing.T) { + // partitiontest.PartitionTest(t) require.Less(t, 20*time.Millisecond, dynamicFilterTimeoutLowerBound) } From 20d3c2ac446bd9aba8b92f6e71dfe4b877e078db Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Sun, 27 Aug 2023 09:27:06 -0400 Subject: [PATCH 150/232] fix lint issues, push lowerbound to 1/2s --- agreement/credentialArrivalHistory_test.go | 4 ++++ agreement/dynamicFilterTimeoutParams.go | 2 +- agreement/dynamicFilterTimeoutParams_test.go | 2 ++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/agreement/credentialArrivalHistory_test.go b/agreement/credentialArrivalHistory_test.go index 2088c2a4e1..3b85a221ba 100644 --- a/agreement/credentialArrivalHistory_test.go +++ b/agreement/credentialArrivalHistory_test.go @@ -24,6 +24,7 @@ import ( ) func TestCredentialHistoryStore(t *testing.T) { + // partitiontest.PartitionTest(t) size := 5 buffer := newCredentialArrivalHistory(size) // last store call overwrites the first one @@ -39,6 +40,7 @@ func TestCredentialHistoryStore(t *testing.T) { } func TestCredentialHistoryReset(t *testing.T) { + // partitiontest.PartitionTest(t) size := 5 buffer := newCredentialArrivalHistory(size) // last store call overwrites the first one @@ -58,6 +60,7 @@ func TestCredentialHistoryReset(t *testing.T) { } func TestCredentialHistoryIsFull(t *testing.T) { + // partitiontest.PartitionTest(t) var buffer *credentialArrivalHistory require.False(t, buffer.isFull()) @@ -76,6 +79,7 @@ func TestCredentialHistoryIsFull(t *testing.T) { } func TestOrderStatistics(t *testing.T) { + // partitiontest.PartitionTest(t) size := 5 buffer := newCredentialArrivalHistory(size) require.False(t, buffer.isFull()) diff --git a/agreement/dynamicFilterTimeoutParams.go b/agreement/dynamicFilterTimeoutParams.go index f24b7aac97..36348615b5 100644 --- a/agreement/dynamicFilterTimeoutParams.go +++ b/agreement/dynamicFilterTimeoutParams.go @@ -31,7 +31,7 @@ const dynamicFilterCredentialArrivalHistory int = 40 // DynamicFilterTimeoutLowerBound specifies a minimal duration that the // filter timeout must meet. -const dynamicFilterTimeoutLowerBound time.Duration = 600 * time.Millisecond +const dynamicFilterTimeoutLowerBound time.Duration = 500 * time.Millisecond // DynamicFilterTimeoutCredentialArrivalHistoryIdx specified which sample to use // out of a sorted DynamicFilterCredentialArrivalHistory-sized array of time diff --git a/agreement/dynamicFilterTimeoutParams_test.go b/agreement/dynamicFilterTimeoutParams_test.go index 8da2968b12..71d755e674 100644 --- a/agreement/dynamicFilterTimeoutParams_test.go +++ b/agreement/dynamicFilterTimeoutParams_test.go @@ -24,6 +24,7 @@ import ( ) func TestSampleIndexIsValid(t *testing.T) { + // partitiontest.PartitionTest(t) require.GreaterOrEqual(t, dynamicFilterCredentialArrivalHistory, 0) require.GreaterOrEqual(t, dynamicFilterTimeoutCredentialArrivalHistoryIdx, 0) if dynamicFilterCredentialArrivalHistory > 0 { @@ -32,5 +33,6 @@ func TestSampleIndexIsValid(t *testing.T) { } func TestLowerBound(t *testing.T) { + // partitiontest.PartitionTest(t) require.Less(t, 20*time.Millisecond, dynamicFilterTimeoutLowerBound) } From f2edc3f0af773ebf78d093aee0d896f84e618df5 Mon Sep 17 00:00:00 2001 From: yossigi Date: Mon, 28 Aug 2023 10:45:50 -0400 Subject: [PATCH 151/232] store method documentation Co-authored-by: cce <51567+cce@users.noreply.github.com> --- agreement/credentialArrivalHistory.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/agreement/credentialArrivalHistory.go b/agreement/credentialArrivalHistory.go index 5fdb99668d..3df96c3d61 100644 --- a/agreement/credentialArrivalHistory.go +++ b/agreement/credentialArrivalHistory.go @@ -33,6 +33,8 @@ func newCredentialArrivalHistory(size int) *credentialArrivalHistory { return &history } +// store saves a new sample into the circular buffer. +// If the buffer is full, it overwrites the oldest sample. func (history *credentialArrivalHistory) store(sample time.Duration) { history.history[history.writePtr] = sample history.writePtr++ From 5ca8a27e2bfe69c53a7b708ee7b9db6a3da984cb Mon Sep 17 00:00:00 2001 From: yossigi Date: Mon, 28 Aug 2023 10:46:27 -0400 Subject: [PATCH 152/232] credentialArrivalHistory documentation Co-authored-by: cce <51567+cce@users.noreply.github.com> --- agreement/credentialArrivalHistory.go | 1 + 1 file changed, 1 insertion(+) diff --git a/agreement/credentialArrivalHistory.go b/agreement/credentialArrivalHistory.go index 3df96c3d61..38b7d1b555 100644 --- a/agreement/credentialArrivalHistory.go +++ b/agreement/credentialArrivalHistory.go @@ -21,6 +21,7 @@ import ( "time" ) +// credentialArrivalHistory maintains a circular buffer of time.Duration samples. type credentialArrivalHistory struct { history []time.Duration writePtr int From bd2c576cc20a0df7d65fc76f94f697bb3e45eff7 Mon Sep 17 00:00:00 2001 From: yossigi Date: Mon, 28 Aug 2023 11:01:48 -0400 Subject: [PATCH 153/232] Update agreement/credentialArrivalHistory.go Co-authored-by: cce <51567+cce@users.noreply.github.com> --- agreement/credentialArrivalHistory.go | 1 + 1 file changed, 1 insertion(+) diff --git a/agreement/credentialArrivalHistory.go b/agreement/credentialArrivalHistory.go index 38b7d1b555..044686d26e 100644 --- a/agreement/credentialArrivalHistory.go +++ b/agreement/credentialArrivalHistory.go @@ -50,6 +50,7 @@ func (history *credentialArrivalHistory) reset() { history.full = len(history.history) == 0 } +// isFull checks if the circular buffer has been fully populated at least once. func (history *credentialArrivalHistory) isFull() bool { if history == nil { return false From 4c1e7062dfc96472f2ad15725231f31c54d81d94 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Mon, 28 Aug 2023 11:24:33 -0400 Subject: [PATCH 154/232] addressing pr comments --- agreement/credentialArrivalHistory.go | 23 ++++++++++++++++++++-- agreement/credentialArrivalHistory_test.go | 16 +++++++++++++++ 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/agreement/credentialArrivalHistory.go b/agreement/credentialArrivalHistory.go index 044686d26e..b9e304f50b 100644 --- a/agreement/credentialArrivalHistory.go +++ b/agreement/credentialArrivalHistory.go @@ -29,6 +29,9 @@ type credentialArrivalHistory struct { } func newCredentialArrivalHistory(size int) *credentialArrivalHistory { + if size < 0 { + panic("can't create CredentialArrivalHistory with negative size") + } history := credentialArrivalHistory{history: make([]time.Duration, size)} history.reset() return &history @@ -37,28 +40,44 @@ func newCredentialArrivalHistory(size int) *credentialArrivalHistory { // store saves a new sample into the circular buffer. // If the buffer is full, it overwrites the oldest sample. func (history *credentialArrivalHistory) store(sample time.Duration) { + if len(history.history) == 0 { + return + } + history.history[history.writePtr] = sample history.writePtr++ - if history.writePtr == cap(history.history) { + if history.writePtr == len(history.history) { history.full = true history.writePtr = 0 } } +// reset marks the history buffer as empty func (history *credentialArrivalHistory) reset() { history.writePtr = 0 - history.full = len(history.history) == 0 + history.full = false } // isFull checks if the circular buffer has been fully populated at least once. func (history *credentialArrivalHistory) isFull() bool { + // if the history struct wasn't allocated, then it can't be full if history == nil { return false } return history.full } +// orderStatistics returns the idx'th time duration in the sorted history array. +// It assumes that history is full and the idx is within the array bounds, and +// panics if either of these assumptions doesn't hold. func (history *credentialArrivalHistory) orderStatistics(idx int) time.Duration { + if !history.isFull() { + panic("history not full") + } + if idx < 0 || idx >= len(history.history) { + panic("index out of bounds") + } + // if history.history is long, then we could optimize this function to use // the linear time order statistics algorithm. sortedArrivals := make([]time.Duration, len(history.history)) diff --git a/agreement/credentialArrivalHistory_test.go b/agreement/credentialArrivalHistory_test.go index 3b85a221ba..79742b7a0a 100644 --- a/agreement/credentialArrivalHistory_test.go +++ b/agreement/credentialArrivalHistory_test.go @@ -78,6 +78,22 @@ func TestCredentialHistoryIsFull(t *testing.T) { } } +func TestCredentialHisotyZeroSize(t *testing.T) { + // partitiontest.PartitionTest(t) + var buffer *credentialArrivalHistory + require.False(t, buffer.isFull()) + + size := 0 + buffer = newCredentialArrivalHistory(size) + require.False(t, buffer.isFull()) + + // trying to store new samples won't panic but the history is never full + for i := 0; i < size+10; i++ { + buffer.store(time.Duration(i)) + require.False(t, buffer.isFull()) + } +} + func TestOrderStatistics(t *testing.T) { // partitiontest.PartitionTest(t) size := 5 From f6979bc4a7b65a929495650f5c84091988b90f2c Mon Sep 17 00:00:00 2001 From: yossigi Date: Mon, 28 Aug 2023 12:15:13 -0400 Subject: [PATCH 155/232] comment about dynamic lambda Co-authored-by: cce <51567+cce@users.noreply.github.com> --- agreement/player.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agreement/player.go b/agreement/player.go index 5723d02f01..9bfa83a99d 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -324,9 +324,9 @@ func (p *player) calculateFilterTimeout(ver protocol.ConsensusVersion, tracer *t clampedTimeout = defaultTimeout } tracer.log.Debugf("round %d, period %d: dynamicTimeout = %d, clamped timeout = %d", p.Round, p.Period, dynamicTimeout, clampedTimeout) - dynamicTimeout = clampedTimeout // store dynamicFilterTimeout on the player for debugging & reporting p.dynamicFilterTimeout = dynamicTimeout + dynamicTimeout = clampedTimeout if !proto.DynamicFilterTimeout { // If the dynamic filter timeout is disabled, return the default filter From 6a6eec00d03c9e13e390bad18b04ab26d202f9dd Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Mon, 28 Aug 2023 12:20:24 -0400 Subject: [PATCH 156/232] 1 line less --- agreement/player.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/agreement/player.go b/agreement/player.go index 9bfa83a99d..eb5310ecba 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -326,7 +326,6 @@ func (p *player) calculateFilterTimeout(ver protocol.ConsensusVersion, tracer *t tracer.log.Debugf("round %d, period %d: dynamicTimeout = %d, clamped timeout = %d", p.Round, p.Period, dynamicTimeout, clampedTimeout) // store dynamicFilterTimeout on the player for debugging & reporting p.dynamicFilterTimeout = dynamicTimeout - dynamicTimeout = clampedTimeout if !proto.DynamicFilterTimeout { // If the dynamic filter timeout is disabled, return the default filter @@ -335,7 +334,7 @@ func (p *player) calculateFilterTimeout(ver protocol.ConsensusVersion, tracer *t return defaultTimeout } - return dynamicTimeout + return clampedTimeout } func (p *player) handleThresholdEvent(r routerHandle, e thresholdEvent) []action { From 98524f0f8900bce0aacdc704bb5a02eba267942d Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Mon, 28 Aug 2023 14:12:15 -0400 Subject: [PATCH 157/232] credential history be non pointer --- agreement/credentialArrivalHistory.go | 8 ++------ agreement/credentialArrivalHistory_test.go | 14 +++++++------- agreement/player.go | 7 ++----- agreement/player_test.go | 6 ++++-- agreement/service.go | 2 +- 5 files changed, 16 insertions(+), 21 deletions(-) diff --git a/agreement/credentialArrivalHistory.go b/agreement/credentialArrivalHistory.go index b9e304f50b..b4bc48c976 100644 --- a/agreement/credentialArrivalHistory.go +++ b/agreement/credentialArrivalHistory.go @@ -28,13 +28,13 @@ type credentialArrivalHistory struct { full bool } -func newCredentialArrivalHistory(size int) *credentialArrivalHistory { +func makeCredentialArrivalHistory(size int) credentialArrivalHistory { if size < 0 { panic("can't create CredentialArrivalHistory with negative size") } history := credentialArrivalHistory{history: make([]time.Duration, size)} history.reset() - return &history + return history } // store saves a new sample into the circular buffer. @@ -60,10 +60,6 @@ func (history *credentialArrivalHistory) reset() { // isFull checks if the circular buffer has been fully populated at least once. func (history *credentialArrivalHistory) isFull() bool { - // if the history struct wasn't allocated, then it can't be full - if history == nil { - return false - } return history.full } diff --git a/agreement/credentialArrivalHistory_test.go b/agreement/credentialArrivalHistory_test.go index 79742b7a0a..89dfcb245c 100644 --- a/agreement/credentialArrivalHistory_test.go +++ b/agreement/credentialArrivalHistory_test.go @@ -26,7 +26,7 @@ import ( func TestCredentialHistoryStore(t *testing.T) { // partitiontest.PartitionTest(t) size := 5 - buffer := newCredentialArrivalHistory(size) + buffer := makeCredentialArrivalHistory(size) // last store call overwrites the first one for i := 0; i < size+1; i++ { buffer.store(time.Duration(i)) @@ -42,7 +42,7 @@ func TestCredentialHistoryStore(t *testing.T) { func TestCredentialHistoryReset(t *testing.T) { // partitiontest.PartitionTest(t) size := 5 - buffer := newCredentialArrivalHistory(size) + buffer := makeCredentialArrivalHistory(size) // last store call overwrites the first one for i := 0; i < size+1; i++ { buffer.store(time.Duration(i)) @@ -61,11 +61,11 @@ func TestCredentialHistoryReset(t *testing.T) { func TestCredentialHistoryIsFull(t *testing.T) { // partitiontest.PartitionTest(t) - var buffer *credentialArrivalHistory + var buffer credentialArrivalHistory require.False(t, buffer.isFull()) size := 5 - buffer = newCredentialArrivalHistory(size) + buffer = makeCredentialArrivalHistory(size) require.False(t, buffer.isFull()) for i := 0; i < size+10; i++ { @@ -80,11 +80,11 @@ func TestCredentialHistoryIsFull(t *testing.T) { func TestCredentialHisotyZeroSize(t *testing.T) { // partitiontest.PartitionTest(t) - var buffer *credentialArrivalHistory + var buffer credentialArrivalHistory require.False(t, buffer.isFull()) size := 0 - buffer = newCredentialArrivalHistory(size) + buffer = makeCredentialArrivalHistory(size) require.False(t, buffer.isFull()) // trying to store new samples won't panic but the history is never full @@ -97,7 +97,7 @@ func TestCredentialHisotyZeroSize(t *testing.T) { func TestOrderStatistics(t *testing.T) { // partitiontest.PartitionTest(t) size := 5 - buffer := newCredentialArrivalHistory(size) + buffer := makeCredentialArrivalHistory(size) require.False(t, buffer.isFull()) for i := 0; i < size; i++ { diff --git a/agreement/player.go b/agreement/player.go index eb5310ecba..72fe2f4556 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -61,7 +61,7 @@ type player struct { // the history of arrival times of the lowest credential from previous // ronuds, used for calculating the filter timeout dynamically. - lowestCredentialArrivals *credentialArrivalHistory + lowestCredentialArrivals credentialArrivalHistory // The period 0 dynamic filter timeout calculated for this round, if set, // even if dynamic filter timeouts are not enabled. It is used for reporting @@ -292,9 +292,6 @@ func (p *player) updateCredentialArrivalHistory(r routerHandle, ver protocol.Con return 0 } - if p.lowestCredentialArrivals == nil { - p.lowestCredentialArrivals = newCredentialArrivalHistory(dynamicFilterCredentialArrivalHistory) - } p.lowestCredentialArrivals.store(re.Vote.validatedAt) return re.Vote.validatedAt } @@ -410,7 +407,7 @@ func (p *player) enterPeriod(r routerHandle, source thresholdEvent, target perio if target != 0 { // We entered a non-0 period, we should reset the filter timeout // calculation mechanism. - p.lowestCredentialArrivals = nil + p.lowestCredentialArrivals.reset() } p.Deadline = Deadline{Duration: p.calculateFilterTimeout(source.Proto, r.t), Type: TimeoutFilter} diff --git a/agreement/player_test.go b/agreement/player_test.go index db7a9b1faa..210753d682 100644 --- a/agreement/player_test.go +++ b/agreement/player_test.go @@ -416,7 +416,8 @@ func testPlayerSetup() (player, rootRouter, testAccountData, testBlockFactory, L accs := testAccountData{addresses: addresses, vrfs: vrfSecrets, ots: otSecrets} round := ledger.NextRound() period := period(0) - player := player{Round: round, Period: period, Step: soft} + historyBuffer := makeCredentialArrivalHistory(dynamicFilterCredentialArrivalHistory) + player := player{Round: round, Period: period, Step: soft, lowestCredentialArrivals: historyBuffer} var p actor = ioLoggedActor{checkedActor{actor: &player, actorContract: playerContract{}}, playerTracer} router := routerFixture @@ -500,7 +501,8 @@ func TestPlayerLateBlockProposalPeriod0(t *testing.T) { func setupP(t *testing.T, r round, p period, s step) (plyr *player, pMachine ioAutomata, helper *voteMakerHelper) { // Set up a composed test machine starting at specified rps - rRouter := makeRootRouter(player{Round: r, Period: p, Step: s, Deadline: Deadline{Duration: FilterTimeout(p, protocol.ConsensusCurrentVersion), Type: TimeoutFilter}}) + history := makeCredentialArrivalHistory(dynamicFilterCredentialArrivalHistory) + rRouter := makeRootRouter(player{Round: r, Period: p, Step: s, Deadline: Deadline{Duration: FilterTimeout(p, protocol.ConsensusCurrentVersion), Type: TimeoutFilter}, lowestCredentialArrivals: history}) concreteMachine := ioAutomataConcretePlayer{rootRouter: &rRouter} plyr = concreteMachine.underlying() pMachine = &concreteMachine diff --git a/agreement/service.go b/agreement/service.go index 765f4270a9..3ad78152ad 100644 --- a/agreement/service.go +++ b/agreement/service.go @@ -211,7 +211,7 @@ func (s *Service) mainLoop(input <-chan externalEvent, output chan<- []action, r s.log.Errorf("unable to retrieve consensus version for round %d, defaulting to binary consensus version", nextRound) nextVersion = protocol.ConsensusCurrentVersion } - status = player{Round: nextRound, Step: soft, Deadline: Deadline{Duration: FilterTimeout(0, nextVersion), Type: TimeoutFilter}} + status = player{Round: nextRound, Step: soft, Deadline: Deadline{Duration: FilterTimeout(0, nextVersion), Type: TimeoutFilter}, lowestCredentialArrivals: makeCredentialArrivalHistory(dynamicFilterCredentialArrivalHistory)} router = makeRootRouter(status) a1 := pseudonodeAction{T: assemble, Round: s.Ledger.NextRound()} From 0d72d567400e60862c26cb80b114c59e0bb57aab Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Mon, 28 Aug 2023 14:24:00 -0400 Subject: [PATCH 158/232] more legible test --- agreement/credentialArrivalHistory_test.go | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/agreement/credentialArrivalHistory_test.go b/agreement/credentialArrivalHistory_test.go index 89dfcb245c..b88eada3d0 100644 --- a/agreement/credentialArrivalHistory_test.go +++ b/agreement/credentialArrivalHistory_test.go @@ -68,9 +68,22 @@ func TestCredentialHistoryIsFull(t *testing.T) { buffer = makeCredentialArrivalHistory(size) require.False(t, buffer.isFull()) - for i := 0; i < size+10; i++ { + for i := 1; i < size+10; i++ { + buffer.store(time.Duration(i)) + if i < size { + require.False(t, buffer.isFull()) + } else { + require.True(t, buffer.isFull()) + } + } + + // reset the buffer and then fill it again + buffer.reset() + require.False(t, buffer.isFull()) + + for i := 1; i < size+10; i++ { buffer.store(time.Duration(i)) - if i < size-1 { + if i < size { require.False(t, buffer.isFull()) } else { require.True(t, buffer.isFull()) From 79e1904378ab68ae921ab2ff8a1518e4ce0b0bc8 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Mon, 28 Aug 2023 14:55:59 -0400 Subject: [PATCH 159/232] partitiontest --- agreement/dynamicFilterTimeoutParams_test.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/agreement/dynamicFilterTimeoutParams_test.go b/agreement/dynamicFilterTimeoutParams_test.go index 71d755e674..7d03de7dea 100644 --- a/agreement/dynamicFilterTimeoutParams_test.go +++ b/agreement/dynamicFilterTimeoutParams_test.go @@ -20,11 +20,13 @@ import ( "testing" "time" + "github.com/algorand/go-algorand/test/partitiontest" "github.com/stretchr/testify/require" ) func TestSampleIndexIsValid(t *testing.T) { - // partitiontest.PartitionTest(t) + partitiontest.PartitionTest(t) + require.GreaterOrEqual(t, dynamicFilterCredentialArrivalHistory, 0) require.GreaterOrEqual(t, dynamicFilterTimeoutCredentialArrivalHistoryIdx, 0) if dynamicFilterCredentialArrivalHistory > 0 { @@ -33,6 +35,7 @@ func TestSampleIndexIsValid(t *testing.T) { } func TestLowerBound(t *testing.T) { - // partitiontest.PartitionTest(t) + partitiontest.PartitionTest(t) + require.Less(t, 20*time.Millisecond, dynamicFilterTimeoutLowerBound) } From 40935ed6262254801827f5cb0b2e34b217ec7407 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Mon, 28 Aug 2023 15:00:59 -0400 Subject: [PATCH 160/232] partitiontest --- agreement/credentialArrivalHistory_test.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/agreement/credentialArrivalHistory_test.go b/agreement/credentialArrivalHistory_test.go index b88eada3d0..c207583a30 100644 --- a/agreement/credentialArrivalHistory_test.go +++ b/agreement/credentialArrivalHistory_test.go @@ -20,11 +20,13 @@ import ( "testing" "time" + "github.com/algorand/go-algorand/test/partitiontest" "github.com/stretchr/testify/require" ) func TestCredentialHistoryStore(t *testing.T) { - // partitiontest.PartitionTest(t) + partitiontest.PartitionTest(t) + size := 5 buffer := makeCredentialArrivalHistory(size) // last store call overwrites the first one @@ -40,7 +42,8 @@ func TestCredentialHistoryStore(t *testing.T) { } func TestCredentialHistoryReset(t *testing.T) { - // partitiontest.PartitionTest(t) + partitiontest.PartitionTest(t) + size := 5 buffer := makeCredentialArrivalHistory(size) // last store call overwrites the first one @@ -92,7 +95,8 @@ func TestCredentialHistoryIsFull(t *testing.T) { } func TestCredentialHisotyZeroSize(t *testing.T) { - // partitiontest.PartitionTest(t) + partitiontest.PartitionTest(t) + var buffer credentialArrivalHistory require.False(t, buffer.isFull()) @@ -108,7 +112,8 @@ func TestCredentialHisotyZeroSize(t *testing.T) { } func TestOrderStatistics(t *testing.T) { - // partitiontest.PartitionTest(t) + partitiontest.PartitionTest(t) + size := 5 buffer := makeCredentialArrivalHistory(size) require.False(t, buffer.isFull()) From 27f1374e1aafbbaaea826437e603cbb35b01e8a5 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Mon, 28 Aug 2023 15:49:40 -0400 Subject: [PATCH 161/232] remove redundant check --- agreement/player.go | 9 ++------- agreement/player_test.go | 2 +- agreement/service_test.go | 4 ++-- 3 files changed, 5 insertions(+), 10 deletions(-) diff --git a/agreement/player.go b/agreement/player.go index 2c7d59d670..85015209b6 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -281,11 +281,6 @@ func (p *player) handleCheckpointEvent(r routerHandle, e checkpointEvent) []acti // It returns the time of the lowest credential's arrival, if one was // collected and added to lowestCredentialArrivals, or zero otherwise. func (p *player) updateCredentialArrivalHistory(r routerHandle, ver protocol.ConsensusVersion) time.Duration { - // only append to lowestCredentialArrivals if this was a successful round completing in period 0. - if p.Period != 0 { - return 0 - } - if p.Round < credentialRoundLag { // not sufficiently many rounds had passed to collect any measurement return 0 @@ -294,8 +289,8 @@ func (p *player) updateCredentialArrivalHistory(r routerHandle, ver protocol.Con // look up the validatedAt time of the winning proposal-vote from credentialRoundLag ago, // by now we should have seen the lowest credential for that round. credHistoryRound := p.Round - credentialRoundLag - re := readLowestEvent{T: readLowestVote, Round: credHistoryRound, Period: p.Period} - re = r.dispatch(*p, re, proposalMachineRound, credHistoryRound, p.Period, 0).(readLowestEvent) + re := readLowestEvent{T: readLowestVote, Round: credHistoryRound, Period: 0} + re = r.dispatch(*p, re, proposalMachineRound, credHistoryRound, 0, 0).(readLowestEvent) if !re.Filled { return 0 } diff --git a/agreement/player_test.go b/agreement/player_test.go index cb3dd4cf24..3eff1bb447 100644 --- a/agreement/player_test.go +++ b/agreement/player_test.go @@ -505,7 +505,7 @@ func setupP(t *testing.T, r round, p period, s step) (plyr *player, pMachine ioA rRouter := makeRootRouter(player{Round: r, Period: p, Step: s, Deadline: Deadline{Duration: FilterTimeout(p, protocol.ConsensusCurrentVersion), Type: TimeoutFilter}, lowestCredentialArrivals: history}) concreteMachine := ioAutomataConcretePlayer{rootRouter: &rRouter} plyr = concreteMachine.underlying() - plyr.lowestCredentialArrivals = newCredentialArrivalHistory(dynamicFilterCredentialArrivalHistory) + plyr.lowestCredentialArrivals = makeCredentialArrivalHistory(dynamicFilterCredentialArrivalHistory) pMachine = &concreteMachine helper = &voteMakerHelper{} helper.Setup() diff --git a/agreement/service_test.go b/agreement/service_test.go index 5e5d77efcc..6ff748fe7f 100644 --- a/agreement/service_test.go +++ b/agreement/service_test.go @@ -1158,10 +1158,10 @@ func TestDynamicFilterTimeoutResets(t *testing.T) { require.Equal(t, filterTimeouts[i][baseHistoryRounds-2], filterTimeoutsPostRecovery[i][0]) // check that filter timeout was updated to at the end of the history window - for j := 1; j < dynamicFilterCredentialArrivalHistory-1; j++ { + for j := 1; j < dynamicFilterCredentialArrivalHistory-2; j++ { require.Equal(t, filterTimeoutsPostRecovery[i][j-1], filterTimeoutsPostRecovery[i][j]) } - require.Less(t, filterTimeoutsPostRecovery[i][dynamicFilterCredentialArrivalHistory-1], filterTimeoutsPostRecovery[i][dynamicFilterCredentialArrivalHistory-2]) + require.Less(t, filterTimeoutsPostRecovery[i][dynamicFilterCredentialArrivalHistory-2], filterTimeoutsPostRecovery[i][dynamicFilterCredentialArrivalHistory-3]) } sanityCheck(startRound, 2*round(baseHistoryRounds+2), ledgers) From c1dea69c613ef243d40e71105be85ac1813e7355 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Mon, 28 Aug 2023 16:13:57 -0400 Subject: [PATCH 162/232] fix decoding --- agreement/persistence.go | 2 ++ agreement/persistence_test.go | 21 +++++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/agreement/persistence.go b/agreement/persistence.go index 0e5e75915a..67faca9b5b 100644 --- a/agreement/persistence.go +++ b/agreement/persistence.go @@ -228,6 +228,7 @@ func decode(raw []byte, t0 timers.Clock[TimeoutType], log serviceLogger, reflect if err != nil { return } + p2.lowestCredentialArrivals = makeCredentialArrivalHistory(dynamicFilterCredentialArrivalHistory) rr2 = makeRootRouter(p2) err = protocol.DecodeReflect(s.Router, &rr2) @@ -244,6 +245,7 @@ func decode(raw []byte, t0 timers.Clock[TimeoutType], log serviceLogger, reflect return } } + p2.lowestCredentialArrivals = makeCredentialArrivalHistory(dynamicFilterCredentialArrivalHistory) if p2.OldDeadline != 0 { p2.Deadline = Deadline{Duration: p2.OldDeadline, Type: TimeoutDeadline} p2.OldDeadline = 0 // clear old value diff --git a/agreement/persistence_test.go b/agreement/persistence_test.go index ef738e86d6..999e42c639 100644 --- a/agreement/persistence_test.go +++ b/agreement/persistence_test.go @@ -218,6 +218,27 @@ func TestRandomizedEncodingFullDiskState(t *testing.T) { } +func TestCredentialHistoryAllocated(t *testing.T) { + partitiontest.PartitionTest(t) + router, player := randomizeDiskState() + a := []action{} + clock := timers.MakeMonotonicClock[TimeoutType](time.Date(2015, 1, 2, 5, 6, 7, 8, time.UTC)) + log := makeServiceLogger(logging.Base()) + e1 := encode(clock, router, player, a, true) + e2 := encode(clock, router, player, a, false) + require.Equalf(t, e1, e2, "msgp and go-codec encodings differ: len(msgp)=%v, len(reflect)=%v", len(e1), len(e2)) + _, _, p1, _, err1 := decode(e1, clock, log, true) + _, _, p2, _, err2 := decode(e1, clock, log, false) + require.NoErrorf(t, err1, "reflect decoding failed") + require.NoErrorf(t, err2, "msgp decoding failed") + + require.Len(t, p1.lowestCredentialArrivals.history, dynamicFilterCredentialArrivalHistory) + require.Len(t, p2.lowestCredentialArrivals.history, dynamicFilterCredentialArrivalHistory) + emptyHistory := makeCredentialArrivalHistory(dynamicFilterCredentialArrivalHistory) + require.Equalf(t, p1.lowestCredentialArrivals, emptyHistory, "credential arrival history isn't empty") + require.Equalf(t, p2.lowestCredentialArrivals, emptyHistory, "credential arrival history isn't empty") +} + func BenchmarkRandomizedEncode(b *testing.B) { clock := timers.MakeMonotonicClock[TimeoutType](time.Date(2015, 1, 2, 5, 6, 7, 8, time.UTC)) router, player := randomizeDiskState() From 0ce9fea590b0fc9dc3912bd7c3937c538e3ec106 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Mon, 28 Aug 2023 16:27:35 -0400 Subject: [PATCH 163/232] allocate once at the end --- agreement/persistence.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/agreement/persistence.go b/agreement/persistence.go index 67faca9b5b..c699eab6b1 100644 --- a/agreement/persistence.go +++ b/agreement/persistence.go @@ -228,7 +228,6 @@ func decode(raw []byte, t0 timers.Clock[TimeoutType], log serviceLogger, reflect if err != nil { return } - p2.lowestCredentialArrivals = makeCredentialArrivalHistory(dynamicFilterCredentialArrivalHistory) rr2 = makeRootRouter(p2) err = protocol.DecodeReflect(s.Router, &rr2) @@ -245,7 +244,6 @@ func decode(raw []byte, t0 timers.Clock[TimeoutType], log serviceLogger, reflect return } } - p2.lowestCredentialArrivals = makeCredentialArrivalHistory(dynamicFilterCredentialArrivalHistory) if p2.OldDeadline != 0 { p2.Deadline = Deadline{Duration: p2.OldDeadline, Type: TimeoutDeadline} p2.OldDeadline = 0 // clear old value @@ -273,6 +271,8 @@ func decode(raw []byte, t0 timers.Clock[TimeoutType], log serviceLogger, reflect a2 = append(a2, act) } + p2.lowestCredentialArrivals = makeCredentialArrivalHistory(dynamicFilterCredentialArrivalHistory) + t = t2 rr = rr2 p = p2 From 7c1032dcf3a005303ca69b3ea24f4cadc908752a Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Mon, 28 Aug 2023 16:39:45 -0400 Subject: [PATCH 164/232] partition test --- agreement/credentialArrivalHistory_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agreement/credentialArrivalHistory_test.go b/agreement/credentialArrivalHistory_test.go index c207583a30..8626094dd8 100644 --- a/agreement/credentialArrivalHistory_test.go +++ b/agreement/credentialArrivalHistory_test.go @@ -63,7 +63,7 @@ func TestCredentialHistoryReset(t *testing.T) { } func TestCredentialHistoryIsFull(t *testing.T) { - // partitiontest.PartitionTest(t) + partitiontest.PartitionTest(t) var buffer credentialArrivalHistory require.False(t, buffer.isFull()) From ba1fc25f1ee702d66b1eb46e37b8720a14a25fde Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Mon, 28 Aug 2023 17:19:20 -0400 Subject: [PATCH 165/232] set player before passing to router --- agreement/persistence.go | 5 ++--- agreement/persistence_test.go | 2 +- test/scripts/e2e_subs/keyreg-teal-test.sh | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/agreement/persistence.go b/agreement/persistence.go index c699eab6b1..819806b69a 100644 --- a/agreement/persistence.go +++ b/agreement/persistence.go @@ -228,7 +228,7 @@ func decode(raw []byte, t0 timers.Clock[TimeoutType], log serviceLogger, reflect if err != nil { return } - + p2.lowestCredentialArrivals = makeCredentialArrivalHistory(dynamicFilterCredentialArrivalHistory) rr2 = makeRootRouter(p2) err = protocol.DecodeReflect(s.Router, &rr2) if err != nil { @@ -244,6 +244,7 @@ func decode(raw []byte, t0 timers.Clock[TimeoutType], log serviceLogger, reflect return } } + p2.lowestCredentialArrivals = makeCredentialArrivalHistory(dynamicFilterCredentialArrivalHistory) if p2.OldDeadline != 0 { p2.Deadline = Deadline{Duration: p2.OldDeadline, Type: TimeoutDeadline} p2.OldDeadline = 0 // clear old value @@ -271,8 +272,6 @@ func decode(raw []byte, t0 timers.Clock[TimeoutType], log serviceLogger, reflect a2 = append(a2, act) } - p2.lowestCredentialArrivals = makeCredentialArrivalHistory(dynamicFilterCredentialArrivalHistory) - t = t2 rr = rr2 p = p2 diff --git a/agreement/persistence_test.go b/agreement/persistence_test.go index 999e42c639..a3cc99755d 100644 --- a/agreement/persistence_test.go +++ b/agreement/persistence_test.go @@ -38,7 +38,7 @@ func TestAgreementSerialization(t *testing.T) { // todo : we need to deserialize some more meaningful state. clock := timers.MakeMonotonicClock[TimeoutType](time.Date(2015, 1, 2, 5, 6, 7, 8, time.UTC)) - status := player{Round: 350, Step: soft, Deadline: Deadline{Duration: time.Duration(23) * time.Second, Type: TimeoutDeadline}} + status := player{Round: 350, Step: soft, Deadline: Deadline{Duration: time.Duration(23) * time.Second, Type: TimeoutDeadline}, lowestCredentialArrivals: makeCredentialArrivalHistory(dynamicFilterCredentialArrivalHistory)} router := makeRootRouter(status) a := []action{checkpointAction{}, disconnectAction(messageEvent{}, nil)} diff --git a/test/scripts/e2e_subs/keyreg-teal-test.sh b/test/scripts/e2e_subs/keyreg-teal-test.sh index 61402a45e6..2d329e71fa 100755 --- a/test/scripts/e2e_subs/keyreg-teal-test.sh +++ b/test/scripts/e2e_subs/keyreg-teal-test.sh @@ -16,7 +16,7 @@ ACCOUNTA=$(${gcmd} account new|awk '{ print $6 }') ACCOUNTB=$(${gcmd} account new|awk '{ print $6 }') LEASE=YmxhaCBibGFoIGxlYXNlIHdoYXRldmVyIGJsYWghISE= -DUR=8 +DUR=12 PERIOD=8 EXPIRE=10000 FEE=100000 From b79b14cc22782adb7050110a07d8c1e2e5275621 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Tue, 29 Aug 2023 16:27:04 -0400 Subject: [PATCH 166/232] use special consensus version for tests --- agreement/service_test.go | 32 +++++++++++++++++++++----------- config/consensus.go | 2 -- tools/x-repo-types/xrt_test.go | 2 +- 3 files changed, 22 insertions(+), 14 deletions(-) diff --git a/agreement/service_test.go b/agreement/service_test.go index 6705b629c7..b2ecd42f31 100644 --- a/agreement/service_test.go +++ b/agreement/service_test.go @@ -1021,6 +1021,23 @@ func TestAgreementSynchronous5_50(t *testing.T) { simulateAgreement(t, 5, 50, disabled) } +func createDynamicFilterConfig() (version protocol.ConsensusVersion, consensusVersion func(r basics.Round) (protocol.ConsensusVersion, error), configCleanup func()) { + version = protocol.ConsensusVersion("test-protocol-filtertimeout") + protoParams := config.Consensus[protocol.ConsensusCurrentVersion] + protoParams.DynamicFilterTimeout = true + config.Consensus[version] = protoParams + + consensusVersion = func(r basics.Round) (protocol.ConsensusVersion, error) { + return version, nil + } + + configCleanup = func() { + delete(config.Consensus, version) + } + + return +} + func TestAgreementSynchronousFuture5_DynamicFilterRounds(t *testing.T) { partitiontest.PartitionTest(t) @@ -1028,12 +1045,9 @@ func TestAgreementSynchronousFuture5_DynamicFilterRounds(t *testing.T) { t.Skip("Skipping agreement integration test") } - consensusVersion := func(r basics.Round) (protocol.ConsensusVersion, error) { - return protocol.ConsensusFuture, nil - } + _, consensusVersion, configCleanup := createDynamicFilterConfig() + defer configCleanup() - cfg := config.Consensus[protocol.ConsensusFuture] - require.True(t, cfg.DynamicFilterTimeout) if dynamicFilterCredentialArrivalHistory <= 0 { return } @@ -1061,19 +1075,15 @@ func TestDynamicFilterTimeoutResets(t *testing.T) { t.Skip("Skipping agreement integration test") } - version := protocol.ConsensusFuture + version, consensusVersion, configCleanup := createDynamicFilterConfig() + defer configCleanup() if dynamicFilterCredentialArrivalHistory <= 0 { return } - cfg := config.Consensus[protocol.ConsensusFuture] - require.True(t, cfg.DynamicFilterTimeout) numNodes := 5 - consensusVersion := func(r basics.Round) (protocol.ConsensusVersion, error) { - return protocol.ConsensusFuture, nil - } ledgerFactory := func(data map[basics.Address]basics.AccountData) Ledger { return makeTestLedgerWithConsensusVersion(data, consensusVersion) } diff --git a/config/consensus.go b/config/consensus.go index 819719f2af..2a4ee1cf62 100644 --- a/config/consensus.go +++ b/config/consensus.go @@ -1369,8 +1369,6 @@ func initConsensusProtocols() { vFuture.LogicSigVersion = 10 // When moving this to a release, put a new higher LogicSigVersion here vFuture.EnableLogicSigCostPooling = true - vFuture.DynamicFilterTimeout = true - Consensus[protocol.ConsensusFuture] = vFuture // vAlphaX versions are an separate series of consensus parameters and versions for alphanet diff --git a/tools/x-repo-types/xrt_test.go b/tools/x-repo-types/xrt_test.go index 4360b432d6..8de6e83122 100644 --- a/tools/x-repo-types/xrt_test.go +++ b/tools/x-repo-types/xrt_test.go @@ -78,7 +78,7 @@ func TestCrossRepoTypes(t *testing.T) { xPkg: "github.com/algorand/go-algorand/config", xType: "ConsensusParams", yPkg: "github.com/algorand/go-algorand-sdk/v2/protocol/config", - yBranch: "develop", + yBranch: "b5f90353ea841fc2e3af9f77d42205c337c93dd2", yType: "ConsensusParams", }, { From 529278acb43f3fe97a041c17ea62890b8916f649 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Tue, 29 Aug 2023 16:38:41 -0400 Subject: [PATCH 167/232] revert e2e test changes --- test/scripts/e2e_subs/keyreg-teal-test.sh | 2 +- test/scripts/e2e_subs/limit-swap-test.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/scripts/e2e_subs/keyreg-teal-test.sh b/test/scripts/e2e_subs/keyreg-teal-test.sh index 2d329e71fa..61402a45e6 100755 --- a/test/scripts/e2e_subs/keyreg-teal-test.sh +++ b/test/scripts/e2e_subs/keyreg-teal-test.sh @@ -16,7 +16,7 @@ ACCOUNTA=$(${gcmd} account new|awk '{ print $6 }') ACCOUNTB=$(${gcmd} account new|awk '{ print $6 }') LEASE=YmxhaCBibGFoIGxlYXNlIHdoYXRldmVyIGJsYWghISE= -DUR=12 +DUR=8 PERIOD=8 EXPIRE=10000 FEE=100000 diff --git a/test/scripts/e2e_subs/limit-swap-test.sh b/test/scripts/e2e_subs/limit-swap-test.sh index 026d9b3609..d89331300c 100755 --- a/test/scripts/e2e_subs/limit-swap-test.sh +++ b/test/scripts/e2e_subs/limit-swap-test.sh @@ -42,7 +42,7 @@ echo "closeout part b, asset trader" # quick expiration, test closeout ROUND=$(goal node status | grep 'Last committed block:'|awk '{ print $4 }') -SETUP_ROUND=$((${ROUND} + 20)) +SETUP_ROUND=$((${ROUND} + 10)) TIMEOUT_ROUND=$((${SETUP_ROUND} + 1)) sed s/TMPL_ASSET/${ASSET_ID}/g < tools/teal/templates/limit-order-b.teal.tmpl | sed s/TMPL_SWAPN/137/g | sed s/TMPL_SWAPD/31337/g | sed s/TMPL_TIMEOUT/${TIMEOUT_ROUND}/g | sed s/TMPL_OWN/${ACCOUNT}/g | sed s/TMPL_FEE/100000/g | sed s/TMPL_MINTRD/10000/g > ${TEMPDIR}/limit-order-b.teal From 4fe6b66c1e7d5a1c2bb6a89f2ac40ba3d12859e9 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Tue, 29 Aug 2023 18:57:57 -0400 Subject: [PATCH 168/232] commented out dynamic filter flag assignment --- config/consensus.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/config/consensus.go b/config/consensus.go index 2a4ee1cf62..190e5ce16a 100644 --- a/config/consensus.go +++ b/config/consensus.go @@ -1369,6 +1369,10 @@ func initConsensusProtocols() { vFuture.LogicSigVersion = 10 // When moving this to a release, put a new higher LogicSigVersion here vFuture.EnableLogicSigCostPooling = true + // Setting DynamicFilterTimeout in vFuture will impact e2e test performance + // by reducing round time. Hence, it is commented out for now. + // vFuture.DynamicFilterTimeout = true + Consensus[protocol.ConsensusFuture] = vFuture // vAlphaX versions are an separate series of consensus parameters and versions for alphanet From bfa4e014a90ad7e5d4624688e39f23d0d815c1ca Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Wed, 30 Aug 2023 10:08:16 -0400 Subject: [PATCH 169/232] historical clocks --- agreement/actions.go | 15 +++++++++++++++ agreement/demux.go | 6 +++--- agreement/events.go | 22 +++++++++++++++------- agreement/player_test.go | 25 +++++++++++++++++++++++++ agreement/service.go | 4 ++++ 5 files changed, 62 insertions(+), 10 deletions(-) diff --git a/agreement/actions.go b/agreement/actions.go index 91276e22fc..7428d416db 100644 --- a/agreement/actions.go +++ b/agreement/actions.go @@ -316,6 +316,21 @@ func (a rezeroAction) String() string { func (a rezeroAction) do(ctx context.Context, s *Service) { s.Clock = s.Clock.Zero() + if _, ok := s.historicalClocks[a.Round]; !ok { + s.historicalClocks[a.Round] = s.Clock + } + + removeList := make([]round, 0) + + for r, _ := range s.historicalClocks { + if a.Round < r+credentialRoundLag { + removeList = append(removeList, r) + } + } + + for _, r := range removeList { + delete(s.historicalClocks, r) + } } type pseudonodeAction struct { diff --git a/agreement/demux.go b/agreement/demux.go index cacdaf82d0..ddeff5d9d1 100644 --- a/agreement/demux.go +++ b/agreement/demux.go @@ -199,13 +199,13 @@ func (d *demux) next(s *Service, deadline Deadline, fastDeadline Deadline, curre switch e.t() { case payloadVerified: - e = e.(messageEvent).AttachValidatedAt(s.Clock.Since()) + e = e.(messageEvent).AttachValidatedAt(s.historicalClocks) case payloadPresent, votePresent: - e = e.(messageEvent).AttachReceivedAt(s.Clock.Since()) + e = e.(messageEvent).AttachReceivedAt(s.historicalClocks) case voteVerified: // if this is a proposal vote (step 0), record the validatedAt time on the vote if e.(messageEvent).Input.UnauthenticatedVote.R.Step == 0 { - e = e.(messageEvent).AttachValidatedAt(s.Clock.Since()) + e = e.(messageEvent).AttachValidatedAt(s.historicalClocks) } } }() diff --git a/agreement/events.go b/agreement/events.go index a4031bf304..7f835d9ad4 100644 --- a/agreement/events.go +++ b/agreement/events.go @@ -18,10 +18,10 @@ package agreement import ( "fmt" - "time" "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/protocol" + "github.com/algorand/go-algorand/util/timers" ) // An event represents the communication of an event to a state machine. @@ -979,12 +979,16 @@ func (e checkpointEvent) AttachConsensusVersion(v ConsensusVersionView) external return e } -func (e messageEvent) AttachValidatedAt(d time.Duration) messageEvent { +func (e messageEvent) AttachValidatedAt(clocks map[round]timers.Clock[TimeoutType]) messageEvent { switch e.T { case payloadVerified: - e.Input.Proposal.validatedAt = d + if clock, ok := clocks[e.Input.Proposal.Round()]; ok { + e.Input.Proposal.validatedAt = clock.Since() + } case voteVerified: - e.Input.Vote.validatedAt = d + if clock, ok := clocks[e.Input.Vote.R.Round]; ok { + e.Input.Vote.validatedAt = clock.Since() + } } return e } @@ -992,9 +996,11 @@ func (e messageEvent) AttachValidatedAt(d time.Duration) messageEvent { // AttachReceivedAt looks for an unauthenticatedProposal inside a // payloadPresent or votePresent messageEvent, and attaches the given // time to the proposal's receivedAt field. -func (e messageEvent) AttachReceivedAt(d time.Duration) messageEvent { +func (e messageEvent) AttachReceivedAt(clocks map[round]timers.Clock[TimeoutType]) messageEvent { if e.T == payloadPresent { - e.Input.UnauthenticatedProposal.receivedAt = d + if clock, ok := clocks[e.Input.UnauthenticatedProposal.Round()]; ok { + e.Input.UnauthenticatedProposal.receivedAt = clock.Since() + } } else if e.T == votePresent { // Check for non-nil Tail, indicating this votePresent event // contains a synthetic payloadPresent event that was attached @@ -1003,7 +1009,9 @@ func (e messageEvent) AttachReceivedAt(d time.Duration) messageEvent { // The tail event is payloadPresent, serialized together // with the proposal vote as a single CompoundMessage // using a protocol.ProposalPayloadTag network message. - e.Tail.Input.UnauthenticatedProposal.receivedAt = d + if clock, ok := clocks[e.Tail.Input.UnauthenticatedProposal.Round()]; ok { + e.Tail.Input.UnauthenticatedProposal.receivedAt = clock.Since() + } } } return e diff --git a/agreement/player_test.go b/agreement/player_test.go index 3eff1bb447..e1c3723c85 100644 --- a/agreement/player_test.go +++ b/agreement/player_test.go @@ -28,6 +28,7 @@ import ( "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/protocol" "github.com/algorand/go-algorand/test/partitiontest" + "github.com/algorand/go-algorand/util/timers" ) var playerTracer tracer @@ -3236,6 +3237,30 @@ func TestPlayerAlwaysResynchsPinnedValue(t *testing.T) { require.Truef(t, trace.Contains(rePayloadEvent), "Player should relay payload even if not staged in previous period") } +type fixedTestingClock struct { + time.Duration +} + +func (c *fixedTestingClock) Zero() timers.Clock[TimeoutType] { + return c +} + +func (c *fixedTestingClock) Since() time.Duration { + return c.Duration +} + +func (c *fixedTestingClock) TimeoutAt(d time.Duration, timeoutType TimeoutType) <-chan time.Time { + return nil +} + +func (c *fixedTestingClock) Encode() []byte { + return nil +} + +func (c *fixedTestingClock) Decode([]byte) (timers.Clock[TimeoutType], error) { + return &fixedTestingClock{}, nil // TODO +} + // test that ReceivedAt and ValidateAt timing information are retained in proposalStore // when the payloadPresent, payloadVerified, and voteVerified events are processed, and that all timings // are available when the ensureAction is called for the block. diff --git a/agreement/service.go b/agreement/service.go index 3ad78152ad..01955809f1 100644 --- a/agreement/service.go +++ b/agreement/service.go @@ -56,6 +56,8 @@ type Service struct { persistRouter rootRouter persistStatus player persistActions []action + + historicalClocks map[round]timers.Clock[TimeoutType] } // Parameters holds the parameters necessary to run the agreement protocol. @@ -105,6 +107,8 @@ func MakeService(p Parameters) (*Service, error) { s.persistenceLoop = makeAsyncPersistenceLoop(s.log, s.Accessor, s.Ledger) + s.historicalClocks = make(map[round]timers.Clock[TimeoutType]) + return s, nil } From 6243be5d85dba521ec7a7b40294bc895e0f336b4 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Wed, 30 Aug 2023 12:15:28 -0400 Subject: [PATCH 170/232] timestamp messages from future rounds at 0 --- agreement/demux.go | 6 +- agreement/events.go | 28 ++++-- agreement/player_test.go | 194 ++++++++++++++++++++++++++++++++++----- 3 files changed, 195 insertions(+), 33 deletions(-) diff --git a/agreement/demux.go b/agreement/demux.go index cacdaf82d0..fbc3d66d54 100644 --- a/agreement/demux.go +++ b/agreement/demux.go @@ -199,13 +199,13 @@ func (d *demux) next(s *Service, deadline Deadline, fastDeadline Deadline, curre switch e.t() { case payloadVerified: - e = e.(messageEvent).AttachValidatedAt(s.Clock.Since()) + e = e.(messageEvent).AttachValidatedAt(s.Clock.Since(), currentRound) case payloadPresent, votePresent: - e = e.(messageEvent).AttachReceivedAt(s.Clock.Since()) + e = e.(messageEvent).AttachReceivedAt(s.Clock.Since(), currentRound) case voteVerified: // if this is a proposal vote (step 0), record the validatedAt time on the vote if e.(messageEvent).Input.UnauthenticatedVote.R.Step == 0 { - e = e.(messageEvent).AttachValidatedAt(s.Clock.Since()) + e = e.(messageEvent).AttachValidatedAt(s.Clock.Since(), currentRound) } } }() diff --git a/agreement/events.go b/agreement/events.go index c494c66a85..029e3e2f6e 100644 --- a/agreement/events.go +++ b/agreement/events.go @@ -975,12 +975,20 @@ func (e checkpointEvent) AttachConsensusVersion(v ConsensusVersionView) external return e } -func (e messageEvent) AttachValidatedAt(d time.Duration) messageEvent { +func (e messageEvent) AttachValidatedAt(d time.Duration, currentRound round) messageEvent { switch e.T { case payloadVerified: - e.Input.Proposal.validatedAt = d + if e.Input.Proposal.Round() > currentRound { + e.Input.Proposal.validatedAt = 1 + } else { + e.Input.Proposal.validatedAt = d + } case voteVerified: - e.Input.Vote.validatedAt = d + if e.Input.UnauthenticatedVote.R.Round > currentRound { + e.Input.Vote.validatedAt = 1 + } else { + e.Input.Vote.validatedAt = d + } } return e } @@ -988,9 +996,13 @@ func (e messageEvent) AttachValidatedAt(d time.Duration) messageEvent { // AttachReceivedAt looks for an unauthenticatedProposal inside a // payloadPresent or votePresent messageEvent, and attaches the given // time to the proposal's receivedAt field. -func (e messageEvent) AttachReceivedAt(d time.Duration) messageEvent { +func (e messageEvent) AttachReceivedAt(d time.Duration, currentRound round) messageEvent { if e.T == payloadPresent { - e.Input.UnauthenticatedProposal.receivedAt = d + if e.Input.UnauthenticatedProposal.Round() > currentRound { + e.Input.UnauthenticatedProposal.receivedAt = 1 + } else { + e.Input.UnauthenticatedProposal.receivedAt = d + } } else if e.T == votePresent { // Check for non-nil Tail, indicating this votePresent event // contains a synthetic payloadPresent event that was attached @@ -999,7 +1011,11 @@ func (e messageEvent) AttachReceivedAt(d time.Duration) messageEvent { // The tail event is payloadPresent, serialized together // with the proposal vote as a single CompoundMessage // using a protocol.ProposalPayloadTag network message. - e.Tail.Input.UnauthenticatedProposal.receivedAt = d + if e.Tail.Input.UnauthenticatedProposal.Round() > currentRound { + e.Tail.Input.UnauthenticatedProposal.receivedAt = 1 + } else { + e.Tail.Input.UnauthenticatedProposal.receivedAt = d + } } } return e diff --git a/agreement/player_test.go b/agreement/player_test.go index 210753d682..a9a9f1106d 100644 --- a/agreement/player_test.go +++ b/agreement/player_test.go @@ -3249,7 +3249,7 @@ func TestPlayerRetainsReceivedValidatedAtOneSample(t *testing.T) { // send voteVerified message vVote := helper.MakeVerifiedVote(t, 0, r-1, p, propose, *pV) inMsg := messageEvent{T: voteVerified, Input: message{Vote: vVote, UnauthenticatedVote: vVote.u()}} - inMsg = inMsg.AttachValidatedAt(501 * time.Millisecond) + inMsg = inMsg.AttachValidatedAt(501*time.Millisecond, r-1) err, panicErr := pM.transition(inMsg) require.NoError(t, err) require.NoError(t, panicErr) @@ -3257,20 +3257,65 @@ func TestPlayerRetainsReceivedValidatedAtOneSample(t *testing.T) { // send payloadPresent message m := message{UnauthenticatedProposal: pP.u()} inMsg = messageEvent{T: payloadPresent, Input: m} - inMsg = inMsg.AttachReceivedAt(time.Second) + inMsg = inMsg.AttachReceivedAt(time.Second, r-1) err, panicErr = pM.transition(inMsg) require.NoError(t, err) require.NoError(t, panicErr) - assertCorrectReceivedAtSet(t, pWhite, pM, helper, r, p, pP, pV, m, protocol.ConsensusFuture) + assertCorrectReceivedAtSet(t, pWhite, pM, helper, r, p, pP, pV, m, protocol.ConsensusFuture, time.Second) // assert lowest vote validateAt time was recorded into payloadArrivals - require.NotNil(t, dynamicFilterCredentialArrivalHistory) + require.NotZero(t, dynamicFilterCredentialArrivalHistory) require.Equal(t, pWhite.lowestCredentialArrivals.writePtr, 1) require.False(t, pWhite.lowestCredentialArrivals.isFull()) require.Equal(t, 501*time.Millisecond, pWhite.lowestCredentialArrivals.history[0]) } +// test that ReceivedAt and ValidateAt timing information are retained in +// proposalStore when the payloadPresent, payloadVerified, and voteVerified +// events are processed in the *preceding round*, and that all timings are +// available when the ensureAction is called for the block. +func TestPlayerRetainsEarlyReceivedValidatedAtOneSample(t *testing.T) { + partitiontest.PartitionTest(t) + const r = round(20239) + const p = period(0) + pWhite, pM, helper := setupP(t, r-1, p, soft) + pP, pV := helper.MakeRandomProposalPayload(t, r-1) + + require.NotZero(t, dynamicFilterCredentialArrivalHistory) + // move two rounds, so the measurement from round r gets inserted to the + // history window + + // create a PP message for an arbitrary proposal/payload similar to setupCompoundMessage + vVote := helper.MakeVerifiedVote(t, 0, r-1, p, propose, *pV) + unverifiedVoteMsg := message{UnauthenticatedVote: vVote.u()} + proposalMsg := message{UnauthenticatedProposal: pP.u()} + compoundMsg := messageEvent{T: votePresent, Input: unverifiedVoteMsg, + Tail: &messageEvent{T: payloadPresent, Input: proposalMsg}} + + inMsg := compoundMsg.AttachReceivedAt(time.Second, r-2) // call AttachReceivedAt like demux would + err, panicErr := pM.transition(inMsg) + require.NoError(t, err) + require.NoError(t, panicErr) + + // make sure vote verify requests + verifyEvent := ev(cryptoAction{T: verifyVote, M: unverifiedVoteMsg, Round: r - 1, Period: p, Step: propose, TaskIndex: 1}) + require.Truef(t, pM.getTrace().Contains(verifyEvent), "Player should verify vote") + + // send voteVerified + verifiedVoteMsg := message{Vote: vVote, UnauthenticatedVote: vVote.u()} + inMsg = messageEvent{T: voteVerified, Input: verifiedVoteMsg, TaskIndex: 1} + timestamp := 500 + inMsg = inMsg.AttachValidatedAt(time.Duration(timestamp)*time.Millisecond, r-2) + err, panicErr = pM.transition(inMsg) + require.NoError(t, err) + require.NoError(t, panicErr) + moveToRound(t, pWhite, pM, helper, r, p, pP, pV, proposalMsg, protocol.ConsensusFuture) + + // receive credential for the next round, check that it gets a timestamp of 1 + require.Equal(t, time.Duration(1), pWhite.lowestCredentialArrivals.history[0]) +} + // test that ReceivedAt and ValidateAt timing information are retained in proposalStore // when the payloadPresent, payloadVerified, and voteVerified events are processed, and that all timings // are available when the ensureAction is called for the block. The history should be kept for the last @@ -3281,7 +3326,7 @@ func TestPlayerRetainsReceivedValidatedAtForHistoryWindow(t *testing.T) { const p = period(0) pWhite, pM, helper := setupP(t, r-1, p, soft) - require.NotNil(t, dynamicFilterCredentialArrivalHistory) + require.NotZero(t, dynamicFilterCredentialArrivalHistory) for i := 0; i < dynamicFilterCredentialArrivalHistory; i++ { // send voteVerified message @@ -3289,7 +3334,7 @@ func TestPlayerRetainsReceivedValidatedAtForHistoryWindow(t *testing.T) { vVote := helper.MakeVerifiedVote(t, 0, r+round(i)-1, p, propose, *pV) inMsg := messageEvent{T: voteVerified, Input: message{Vote: vVote, UnauthenticatedVote: vVote.u()}} timestamp := 500 + i - inMsg = inMsg.AttachValidatedAt(time.Duration(timestamp) * time.Millisecond) + inMsg = inMsg.AttachValidatedAt(time.Duration(timestamp)*time.Millisecond, r+round(i)-1) err, panicErr := pM.transition(inMsg) require.NoError(t, err) require.NoError(t, panicErr) @@ -3297,7 +3342,7 @@ func TestPlayerRetainsReceivedValidatedAtForHistoryWindow(t *testing.T) { // send payloadPresent message m := message{UnauthenticatedProposal: pP.u()} inMsg = messageEvent{T: payloadPresent, Input: m} - inMsg = inMsg.AttachReceivedAt(time.Second) + inMsg = inMsg.AttachReceivedAt(time.Second, r+round(i)-1) err, panicErr = pM.transition(inMsg) require.NoError(t, err) require.NoError(t, panicErr) @@ -3331,7 +3376,7 @@ func TestPlayerRetainsReceivedValidatedAtPPOneSample(t *testing.T) { proposalMsg := message{UnauthenticatedProposal: pP.u()} compoundMsg := messageEvent{T: votePresent, Input: unverifiedVoteMsg, Tail: &messageEvent{T: payloadPresent, Input: proposalMsg}} - inMsg := compoundMsg.AttachReceivedAt(time.Second) // call AttachReceivedAt like demux would + inMsg := compoundMsg.AttachReceivedAt(time.Second, r-1) // call AttachReceivedAt like demux would err, panicErr := pM.transition(inMsg) require.NoError(t, err) require.NoError(t, panicErr) @@ -3343,20 +3388,65 @@ func TestPlayerRetainsReceivedValidatedAtPPOneSample(t *testing.T) { // send voteVerified verifiedVoteMsg := message{Vote: vVote, UnauthenticatedVote: vVote.u()} inMsg = messageEvent{T: voteVerified, Input: verifiedVoteMsg, TaskIndex: 1} - inMsg = inMsg.AttachValidatedAt(502 * time.Millisecond) + inMsg = inMsg.AttachValidatedAt(502*time.Millisecond, r-1) err, panicErr = pM.transition(inMsg) require.NoError(t, err) require.NoError(t, panicErr) - assertCorrectReceivedAtSet(t, pWhite, pM, helper, r, p, pP, pV, proposalMsg, protocol.ConsensusFuture) + assertCorrectReceivedAtSet(t, pWhite, pM, helper, r, p, pP, pV, proposalMsg, protocol.ConsensusFuture, time.Second) // assert lowest vote validateAt time was recorded into payloadArrivals - require.NotNil(t, dynamicFilterCredentialArrivalHistory) + require.NotZero(t, dynamicFilterCredentialArrivalHistory) require.Equal(t, pWhite.lowestCredentialArrivals.writePtr, 1) require.False(t, pWhite.lowestCredentialArrivals.isFull()) require.Equal(t, 502*time.Millisecond, pWhite.lowestCredentialArrivals.history[0]) } +// test that ReceivedAt and ValidateAt timing information are retained in +// proposalStore when the payloadPresent (as part of the CompoundMessage +// encoding used by PP messages), payloadVerified, and voteVerified events are +// processed one round early, and that all timings are available when the +// ensureAction is called for the block. +func TestPlayerRetainsEarlyReceivedValidatedAtPPOneSample(t *testing.T) { + partitiontest.PartitionTest(t) + + const r = round(20239) + const p = period(0) + pWhite, pM, helper := setupP(t, r-1, p, soft) + pP, pV := helper.MakeRandomProposalPayload(t, r-1) + + // create a PP message for an arbitrary proposal/payload similar to setupCompoundMessage + vVote := helper.MakeVerifiedVote(t, 0, r-1, p, propose, *pV) + unverifiedVoteMsg := message{UnauthenticatedVote: vVote.u()} + proposalMsg := message{UnauthenticatedProposal: pP.u()} + compoundMsg := messageEvent{T: votePresent, Input: unverifiedVoteMsg, + Tail: &messageEvent{T: payloadPresent, Input: proposalMsg}} + inMsg := compoundMsg.AttachReceivedAt(time.Second, r-2) // call AttachReceivedAt like demux would + err, panicErr := pM.transition(inMsg) + require.NoError(t, err) + require.NoError(t, panicErr) + + // make sure vote verify requests + verifyEvent := ev(cryptoAction{T: verifyVote, M: unverifiedVoteMsg, Round: r - 1, Period: p, Step: propose, TaskIndex: 1}) + require.Truef(t, pM.getTrace().Contains(verifyEvent), "Player should verify vote") + + // send voteVerified + verifiedVoteMsg := message{Vote: vVote, UnauthenticatedVote: vVote.u()} + inMsg = messageEvent{T: voteVerified, Input: verifiedVoteMsg, TaskIndex: 1} + inMsg = inMsg.AttachValidatedAt(502*time.Millisecond, r-2) + err, panicErr = pM.transition(inMsg) + require.NoError(t, err) + require.NoError(t, panicErr) + + assertCorrectReceivedAtSet(t, pWhite, pM, helper, r, p, pP, pV, proposalMsg, protocol.ConsensusFuture, time.Duration(1)) + + // assert lowest vote validateAt time was recorded into payloadArrivals + require.NotZero(t, dynamicFilterCredentialArrivalHistory) + require.Equal(t, pWhite.lowestCredentialArrivals.writePtr, 1) + require.False(t, pWhite.lowestCredentialArrivals.isFull()) + require.Equal(t, time.Duration(1), pWhite.lowestCredentialArrivals.history[0]) +} + // test that ReceivedAt and ValidateAt timing information are retained in // proposalStore when the payloadPresent (as part of the CompoundMessage // encoding used by PP messages), payloadVerified, and voteVerified events are @@ -3370,7 +3460,7 @@ func TestPlayerRetainsReceivedValidatedAtPPForHistoryWindow(t *testing.T) { pWhite, pM, helper := setupP(t, r-1, p, soft) pP, pV := helper.MakeRandomProposalPayload(t, r-1) - require.NotNil(t, dynamicFilterCredentialArrivalHistory) + require.NotZero(t, dynamicFilterCredentialArrivalHistory) for i := 0; i < dynamicFilterCredentialArrivalHistory; i++ { // create a PP message for an arbitrary proposal/payload similar to setupCompoundMessage @@ -3380,7 +3470,7 @@ func TestPlayerRetainsReceivedValidatedAtPPForHistoryWindow(t *testing.T) { compoundMsg := messageEvent{T: votePresent, Input: unverifiedVoteMsg, Tail: &messageEvent{T: payloadPresent, Input: proposalMsg}} - inMsg := compoundMsg.AttachReceivedAt(time.Second) // call AttachReceivedAt like demux would + inMsg := compoundMsg.AttachReceivedAt(time.Second, r+round(i)-1) // call AttachReceivedAt like demux would err, panicErr := pM.transition(inMsg) require.NoError(t, err) require.NoError(t, panicErr) @@ -3393,7 +3483,7 @@ func TestPlayerRetainsReceivedValidatedAtPPForHistoryWindow(t *testing.T) { verifiedVoteMsg := message{Vote: vVote, UnauthenticatedVote: vVote.u()} inMsg = messageEvent{T: voteVerified, Input: verifiedVoteMsg, TaskIndex: 1} timestamp := 500 + i - inMsg = inMsg.AttachValidatedAt(time.Duration(timestamp) * time.Millisecond) + inMsg = inMsg.AttachValidatedAt(time.Duration(timestamp)*time.Millisecond, r+round(i)-1) err, panicErr = pM.transition(inMsg) require.NoError(t, err) require.NoError(t, panicErr) @@ -3437,7 +3527,7 @@ func TestPlayerRetainsReceivedValidatedAtAVPPOneSample(t *testing.T) { // send voteVerified verifiedVoteMsg := message{Vote: vVote, UnauthenticatedVote: vVote.u()} inMsg = messageEvent{T: voteVerified, Input: verifiedVoteMsg, TaskIndex: 1} - inMsg = inMsg.AttachValidatedAt(502 * time.Millisecond) + inMsg = inMsg.AttachValidatedAt(502*time.Millisecond, r-1) err, panicErr = pM.transition(inMsg) require.NoError(t, err) require.NoError(t, panicErr) @@ -3446,7 +3536,7 @@ func TestPlayerRetainsReceivedValidatedAtAVPPOneSample(t *testing.T) { proposalMsg := message{UnauthenticatedProposal: pP.u()} compoundMsg := messageEvent{T: votePresent, Input: unverifiedVoteMsg, Tail: &messageEvent{T: payloadPresent, Input: proposalMsg}} - inMsg = compoundMsg.AttachReceivedAt(time.Second) // call AttachReceivedAt like demux would + inMsg = compoundMsg.AttachReceivedAt(time.Second, r-1) // call AttachReceivedAt like demux would err, panicErr = pM.transition(inMsg) require.NoError(t, err) require.NoError(t, panicErr) @@ -3455,22 +3545,78 @@ func TestPlayerRetainsReceivedValidatedAtAVPPOneSample(t *testing.T) { verifyEvent = ev(cryptoAction{T: verifyVote, M: unverifiedVoteMsg, Round: r - 1, Period: p, Step: propose, TaskIndex: 1}) require.Equal(t, 1, pM.getTrace().CountEvent(verifyEvent), "Player should not verify second vote") - assertCorrectReceivedAtSet(t, pWhite, pM, helper, r, p, pP, pV, proposalMsg, protocol.ConsensusFuture) + assertCorrectReceivedAtSet(t, pWhite, pM, helper, r, p, pP, pV, proposalMsg, protocol.ConsensusFuture, time.Second) // assert lowest vote validateAt time was recorded into payloadArrivals - require.NotNil(t, dynamicFilterCredentialArrivalHistory) + require.NotZero(t, dynamicFilterCredentialArrivalHistory) require.Equal(t, pWhite.lowestCredentialArrivals.writePtr, 1) require.False(t, pWhite.lowestCredentialArrivals.isFull()) require.Equal(t, 502*time.Millisecond, pWhite.lowestCredentialArrivals.history[0]) } +// test that ReceivedAt and ValidateAt timing information are retained in +// proposalStore when the voteVerified event comes in first (as part of the AV +// message before PP), then the payloadPresent (as part of the CompoundMessage +// encoding used by PP messages) and payloadVerified events are processed one +// round early, and that all timings are available when the ensureAction is +// called for the block. +func TestPlayerRetainsEarlyReceivedValidatedAtAVPPOneSample(t *testing.T) { + partitiontest.PartitionTest(t) + + const r = round(20239) + const p = period(0) + pWhite, pM, helper := setupP(t, r-1, p, soft) + pP, pV := helper.MakeRandomProposalPayload(t, r-1) + + // send votePresent message (mimicking the first AV message validating) + vVote := helper.MakeVerifiedVote(t, 0, r-1, p, propose, *pV) + unverifiedVoteMsg := message{UnauthenticatedVote: vVote.u()} + inMsg := messageEvent{T: votePresent, Input: unverifiedVoteMsg} + err, panicErr := pM.transition(inMsg) + require.NoError(t, err) + require.NoError(t, panicErr) + + // make sure vote verify requests + verifyEvent := ev(cryptoAction{T: verifyVote, M: unverifiedVoteMsg, Round: r - 1, Period: p, Step: propose, TaskIndex: 1}) + require.Truef(t, pM.getTrace().Contains(verifyEvent), "Player should verify vote") + + // send voteVerified + verifiedVoteMsg := message{Vote: vVote, UnauthenticatedVote: vVote.u()} + inMsg = messageEvent{T: voteVerified, Input: verifiedVoteMsg, TaskIndex: 1} + inMsg = inMsg.AttachValidatedAt(502*time.Millisecond, r-2) + err, panicErr = pM.transition(inMsg) + require.NoError(t, err) + require.NoError(t, panicErr) + + // create a PP message for an arbitrary proposal/payload similar to setupCompoundMessage + proposalMsg := message{UnauthenticatedProposal: pP.u()} + compoundMsg := messageEvent{T: votePresent, Input: unverifiedVoteMsg, + Tail: &messageEvent{T: payloadPresent, Input: proposalMsg}} + inMsg = compoundMsg.AttachReceivedAt(time.Second, r-2) // call AttachReceivedAt like demux would + err, panicErr = pM.transition(inMsg) + require.NoError(t, err) + require.NoError(t, panicErr) + + // make sure no second request to verify this vote + verifyEvent = ev(cryptoAction{T: verifyVote, M: unverifiedVoteMsg, Round: r - 1, Period: p, Step: propose, TaskIndex: 1}) + require.Equal(t, 1, pM.getTrace().CountEvent(verifyEvent), "Player should not verify second vote") + + assertCorrectReceivedAtSet(t, pWhite, pM, helper, r, p, pP, pV, proposalMsg, protocol.ConsensusFuture, time.Duration(1)) + + // assert lowest vote validateAt time was recorded into payloadArrivals + require.NotZero(t, dynamicFilterCredentialArrivalHistory) + require.Equal(t, pWhite.lowestCredentialArrivals.writePtr, 1) + require.False(t, pWhite.lowestCredentialArrivals.isFull()) + require.Equal(t, time.Duration(1), pWhite.lowestCredentialArrivals.history[0]) +} + func TestPlayerRetainsReceivedValidatedAtAVPPHistoryWindow(t *testing.T) { partitiontest.PartitionTest(t) const r = round(20239) const p = period(0) pWhite, pM, helper := setupP(t, r-1, p, soft) - require.NotNil(t, dynamicFilterCredentialArrivalHistory) + require.NotZero(t, dynamicFilterCredentialArrivalHistory) for i := 0; i < dynamicFilterCredentialArrivalHistory; i++ { pP, pV := helper.MakeRandomProposalPayload(t, r+round(i)-1) @@ -3491,7 +3637,7 @@ func TestPlayerRetainsReceivedValidatedAtAVPPHistoryWindow(t *testing.T) { verifiedVoteMsg := message{Vote: vVote, UnauthenticatedVote: vVote.u()} inMsg = messageEvent{T: voteVerified, Input: verifiedVoteMsg, TaskIndex: 1} timestamp := 500 + i - inMsg = inMsg.AttachValidatedAt(time.Duration(timestamp) * time.Millisecond) + inMsg = inMsg.AttachValidatedAt(time.Duration(timestamp)*time.Millisecond, r+round(i)-1) err, panicErr = pM.transition(inMsg) require.NoError(t, err) require.NoError(t, panicErr) @@ -3500,7 +3646,7 @@ func TestPlayerRetainsReceivedValidatedAtAVPPHistoryWindow(t *testing.T) { proposalMsg := message{UnauthenticatedProposal: pP.u()} compoundMsg := messageEvent{T: votePresent, Input: unverifiedVoteMsg, Tail: &messageEvent{T: payloadPresent, Input: proposalMsg}} - inMsg = compoundMsg.AttachReceivedAt(time.Second) // call AttachReceivedAt like demux would + inMsg = compoundMsg.AttachReceivedAt(time.Second, r+round(i)-1) // call AttachReceivedAt like demux would err, panicErr = pM.transition(inMsg) require.NoError(t, err) require.NoError(t, panicErr) @@ -3526,7 +3672,7 @@ func moveToRound(t *testing.T, pWhite *player, pM ioAutomata, helper *voteMakerH // payloadVerified inMsg := messageEvent{T: payloadVerified, Input: message{Proposal: *pP}, Proto: ConsensusVersionView{Version: ver}} - inMsg = inMsg.AttachValidatedAt(2 * time.Second) // call AttachValidatedAt like demux would + inMsg = inMsg.AttachValidatedAt(2*time.Second, r-1) // call AttachValidatedAt like demux would err, panicErr := pM.transition(inMsg) require.NoError(t, err) require.NoError(t, panicErr) @@ -3563,7 +3709,7 @@ func moveToRound(t *testing.T, pWhite *player, pM ioAutomata, helper *voteMakerH } func assertCorrectReceivedAtSet(t *testing.T, pWhite *player, pM ioAutomata, helper *voteMakerHelper, - r round, p period, pP *proposal, pV *proposalValue, m message, ver protocol.ConsensusVersion) { + r round, p period, pP *proposal, pV *proposalValue, m message, ver protocol.ConsensusVersion, validationTimestamp time.Duration) { moveToRound(t, pWhite, pM, helper, r, p, pP, pV, m, ver) @@ -3581,7 +3727,7 @@ func assertCorrectReceivedAtSet(t *testing.T, pWhite *player, pM ioAutomata, hel } require.True(t, foundEA) require.Equal(t, 2*time.Second, ea.Payload.validatedAt) - require.Equal(t, time.Second, ea.Payload.receivedAt) + require.Equal(t, validationTimestamp, ea.Payload.receivedAt) } // todo: test pipelined rounds, and round interruption From 869ee41dd219e41e2ce1b556f04fb888edef6db1 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Wed, 30 Aug 2023 12:20:20 -0400 Subject: [PATCH 171/232] verified/unauthenticated vote --- agreement/demux.go | 2 +- agreement/events.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/agreement/demux.go b/agreement/demux.go index fbc3d66d54..60ea2a8dd5 100644 --- a/agreement/demux.go +++ b/agreement/demux.go @@ -204,7 +204,7 @@ func (d *demux) next(s *Service, deadline Deadline, fastDeadline Deadline, curre e = e.(messageEvent).AttachReceivedAt(s.Clock.Since(), currentRound) case voteVerified: // if this is a proposal vote (step 0), record the validatedAt time on the vote - if e.(messageEvent).Input.UnauthenticatedVote.R.Step == 0 { + if e.(messageEvent).Input.Vote.R.Step == 0 { e = e.(messageEvent).AttachValidatedAt(s.Clock.Since(), currentRound) } } diff --git a/agreement/events.go b/agreement/events.go index 029e3e2f6e..8a8b8f2b5f 100644 --- a/agreement/events.go +++ b/agreement/events.go @@ -984,7 +984,7 @@ func (e messageEvent) AttachValidatedAt(d time.Duration, currentRound round) mes e.Input.Proposal.validatedAt = d } case voteVerified: - if e.Input.UnauthenticatedVote.R.Round > currentRound { + if e.Input.Vote.R.Round > currentRound { e.Input.Vote.validatedAt = 1 } else { e.Input.Vote.validatedAt = d From 254f768fbec480e0a43bd0579740e7fdaa3ecad2 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Wed, 30 Aug 2023 12:22:13 -0400 Subject: [PATCH 172/232] comment --- agreement/events.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/agreement/events.go b/agreement/events.go index 8a8b8f2b5f..faa4badebe 100644 --- a/agreement/events.go +++ b/agreement/events.go @@ -975,6 +975,9 @@ func (e checkpointEvent) AttachConsensusVersion(v ConsensusVersionView) external return e } +// AttachValidatedAt looks for a validated proposal or vote inside a +// payloadVerified or voteVerified messageEvent, and attaches the given time to +// the proposal's validatedAt field. func (e messageEvent) AttachValidatedAt(d time.Duration, currentRound round) messageEvent { switch e.T { case payloadVerified: From 929fa545bbe6a90483359abeb5fbaaa5f1413acc Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Wed, 30 Aug 2023 16:12:00 -0400 Subject: [PATCH 173/232] updated tests to match historical clocks state keeping --- agreement/player_test.go | 258 +++++++++++++++++++++++++++++++-------- 1 file changed, 210 insertions(+), 48 deletions(-) diff --git a/agreement/player_test.go b/agreement/player_test.go index 3b5943f386..12068d05f7 100644 --- a/agreement/player_test.go +++ b/agreement/player_test.go @@ -3248,13 +3248,21 @@ func TestPlayerRetainsReceivedValidatedAtOneSample(t *testing.T) { pP, pV := helper.MakeRandomProposalPayload(t, r-1) // send voteVerified message - vVote := helper.MakeVerifiedVote(t, 0, r-1, p, propose, *pV) + vVote := helper.MakeVerifiedVote(t, 0, r-credentialRoundLag-1, p, propose, *pV) inMsg := messageEvent{T: voteVerified, Input: message{Vote: vVote, UnauthenticatedVote: vVote.u()}} - inMsg = inMsg.AttachValidatedAt(501*time.Millisecond, r-1, nil) + inMsg = inMsg.AttachValidatedAt(501*time.Millisecond, r-credentialRoundLag-1, nil) err, panicErr := pM.transition(inMsg) require.NoError(t, err) require.NoError(t, panicErr) + // send voteVerified message + vVote = helper.MakeVerifiedVote(t, 0, r-1, p, propose, *pV) + inMsg = messageEvent{T: voteVerified, Input: message{Vote: vVote, UnauthenticatedVote: vVote.u()}} + inMsg = inMsg.AttachValidatedAt(501*time.Millisecond, r-1, nil) + err, panicErr = pM.transition(inMsg) + require.NoError(t, err) + require.NoError(t, panicErr) + // send payloadPresent message m := message{UnauthenticatedProposal: pP.u()} inMsg = messageEvent{T: payloadPresent, Input: m} @@ -3263,77 +3271,58 @@ func TestPlayerRetainsReceivedValidatedAtOneSample(t *testing.T) { require.NoError(t, err) require.NoError(t, panicErr) - assertCorrectReceivedAtSet(t, pWhite, pM, helper, r, p, pP, pV, m, protocol.ConsensusFuture) + assertCorrectReceivedAtSet(t, pWhite, pM, helper, r, p, pP, pV, m, protocol.ConsensusFuture, time.Second) + + // assert lowest vote validateAt time was recorded into payloadArrivals + require.NotZero(t, dynamicFilterCredentialArrivalHistory) + require.Equal(t, pWhite.lowestCredentialArrivals.writePtr, 1) + require.False(t, pWhite.lowestCredentialArrivals.isFull()) + require.Equal(t, 501*time.Millisecond, pWhite.lowestCredentialArrivals.history[0]) } -func TestPlayerRetainsLateReceivedValidatedAtOneSample(t *testing.T) { +// test that ReceivedAt and ValidateAt timing information are retained in +// proposalStore when the payloadPresent, payloadVerified, and voteVerified +// events are processed in the *preceding round*, and that all timings are +// available when the ensureAction is called for the block. +func TestPlayerRetainsEarlyReceivedValidatedAtOneSample(t *testing.T) { partitiontest.PartitionTest(t) - const r = round(20239) const p = period(0) pWhite, pM, helper := setupP(t, r-1, p, soft) - pP, pV := helper.MakeRandomProposalPayload(t, r-1) - // Move to round r, no credentials arrived. // send voteVerified message - vVote := helper.MakeVerifiedVote(t, 0, r-1, p, propose, *pV) + pP, pV := helper.MakeRandomProposalPayload(t, r-credentialRoundLag-1) + vVote := helper.MakeVerifiedVote(t, 0, r-credentialRoundLag-1, p, propose, *pV) inMsg := messageEvent{T: voteVerified, Input: message{Vote: vVote, UnauthenticatedVote: vVote.u()}} - inMsg = inMsg.AttachValidatedAt(501*time.Millisecond, r-1, nil) + inMsg = inMsg.AttachValidatedAt(401*time.Millisecond, r-credentialRoundLag-2, nil) err, panicErr := pM.transition(inMsg) require.NoError(t, err) require.NoError(t, panicErr) - // send payloadPresent message - m := message{UnauthenticatedProposal: pP.u()} - inMsg = messageEvent{T: payloadPresent, Input: m} - inMsg = inMsg.AttachReceivedAt(time.Second, r-1, nil) - err, panicErr = pM.transition(inMsg) - require.NoError(t, err) - require.NoError(t, panicErr) - - assertCorrectReceivedAtSet(t, pWhite, pM, helper, r, p, pP, pV, m, protocol.ConsensusFuture) - require.False(t, pWhite.lowestCredentialArrivals.isFull()) - require.Equal(t, pWhite.lowestCredentialArrivals.writePtr, 0) - - // Old credential arrives // send voteVerified message - vVote = helper.MakeVerifiedVote(t, 0, r-credentialRoundLag, p, propose, *pV) + pP, pV = helper.MakeRandomProposalPayload(t, r-1) + vVote = helper.MakeVerifiedVote(t, 0, r-1, p, propose, *pV) inMsg = messageEvent{T: voteVerified, Input: message{Vote: vVote, UnauthenticatedVote: vVote.u()}} - inMsg = inMsg.AttachValidatedAt(501*time.Millisecond, r-credentialRoundLag, nil) + inMsg = inMsg.AttachValidatedAt(501*time.Millisecond, r-1, nil) err, panicErr = pM.transition(inMsg) require.NoError(t, err) require.NoError(t, panicErr) // send payloadPresent message - m = message{UnauthenticatedProposal: pP.u()} + m := message{UnauthenticatedProposal: pP.u()} inMsg = messageEvent{T: payloadPresent, Input: m} - inMsg = inMsg.AttachReceivedAt(time.Second, r-credentialRoundLag, nil) - err, panicErr = pM.transition(inMsg) - require.NoError(t, err) - require.NoError(t, panicErr) - - // move to round r+1, triggering history update - vVote = helper.MakeVerifiedVote(t, 0, r, p, propose, *pV) - inMsg = messageEvent{T: voteVerified, Input: message{Vote: vVote, UnauthenticatedVote: vVote.u()}} - inMsg = inMsg.AttachValidatedAt(501*time.Millisecond, r, nil) + inMsg = inMsg.AttachReceivedAt(time.Second, r-1, nil) err, panicErr = pM.transition(inMsg) require.NoError(t, err) require.NoError(t, panicErr) - // send payloadPresent message - m = message{UnauthenticatedProposal: pP.u()} - inMsg = messageEvent{T: payloadPresent, Input: m} - inMsg = inMsg.AttachReceivedAt(time.Second, r, nil) - err, panicErr = pM.transition(inMsg) - require.NoError(t, err) - require.NoError(t, panicErr) - moveToRound(t, pWhite, pM, helper, r+1, p, pP, pV, m, protocol.ConsensusFuture) + assertCorrectReceivedAtSet(t, pWhite, pM, helper, r, p, pP, pV, m, protocol.ConsensusFuture, time.Second) // assert lowest vote validateAt time was recorded into payloadArrivals - require.NotNil(t, dynamicFilterCredentialArrivalHistory) + require.NotZero(t, dynamicFilterCredentialArrivalHistory) require.Equal(t, pWhite.lowestCredentialArrivals.writePtr, 1) require.False(t, pWhite.lowestCredentialArrivals.isFull()) - require.Equal(t, 501*time.Millisecond, pWhite.lowestCredentialArrivals.history[0]) + require.Equal(t, time.Duration(1), pWhite.lowestCredentialArrivals.history[0]) } // test that ReceivedAt and ValidateAt timing information are retained in proposalStore @@ -3407,7 +3396,7 @@ func TestPlayerRetainsReceivedValidatedAtPPOneSample(t *testing.T) { require.NoError(t, err) require.NoError(t, panicErr) - assertCorrectReceivedAtSet(t, pWhite, pM, helper, r, p, pP, pV, m, protocol.ConsensusFuture) + assertCorrectReceivedAtSet(t, pWhite, pM, helper, r, p, pP, pV, m, protocol.ConsensusFuture, time.Second) require.False(t, pWhite.lowestCredentialArrivals.isFull()) require.Equal(t, pWhite.lowestCredentialArrivals.writePtr, 0) @@ -3461,6 +3450,91 @@ func TestPlayerRetainsReceivedValidatedAtPPOneSample(t *testing.T) { require.Equal(t, 502*time.Millisecond, pWhite.lowestCredentialArrivals.history[0]) } +// test that ReceivedAt and ValidateAt timing information are retained in +// proposalStore when the payloadPresent (as part of the CompoundMessage +// encoding used by PP messages), payloadVerified, and voteVerified events are +// processed one round early, and that all timings are available when the +// ensureAction is called for the block. +func TestPlayerRetainsEarlyReceivedValidatedAtPPOneSample(t *testing.T) { + partitiontest.PartitionTest(t) + + const r = round(20239) + const p = period(0) + pWhite, pM, helper := setupP(t, r-1, p, soft) + pP, pV := helper.MakeRandomProposalPayload(t, r-1) + + // Move to round r, no credentials arrived. + // send voteVerified message + vVote := helper.MakeVerifiedVote(t, 0, r-1, p, propose, *pV) + inMsg := messageEvent{T: voteVerified, Input: message{Vote: vVote, UnauthenticatedVote: vVote.u()}} + inMsg = inMsg.AttachValidatedAt(501*time.Millisecond, r-1, nil) + err, panicErr := pM.transition(inMsg) + require.NoError(t, err) + require.NoError(t, panicErr) + + // send payloadPresent message + m := message{UnauthenticatedProposal: pP.u()} + inMsg = messageEvent{T: payloadPresent, Input: m} + inMsg = inMsg.AttachReceivedAt(time.Second, r-1, nil) + err, panicErr = pM.transition(inMsg) + require.NoError(t, err) + require.NoError(t, panicErr) + + assertCorrectReceivedAtSet(t, pWhite, pM, helper, r, p, pP, pV, m, protocol.ConsensusFuture, time.Second) + require.False(t, pWhite.lowestCredentialArrivals.isFull()) + require.Equal(t, pWhite.lowestCredentialArrivals.writePtr, 0) + + // create a PP message for the round we're going to take the sample from when round r-1 ends + // Now we're going to pretend we got the message one round early. + vVote = helper.MakeVerifiedVote(t, 0, r-credentialRoundLag, p, propose, *pV) + unverifiedVoteMsg := message{UnauthenticatedVote: vVote.u()} + proposalMsg := message{UnauthenticatedProposal: pP.u()} + compoundMsg := messageEvent{T: votePresent, Input: unverifiedVoteMsg, + Tail: &messageEvent{T: payloadPresent, Input: proposalMsg}} + inMsg = compoundMsg.AttachReceivedAt(time.Second, r-credentialRoundLag-1, nil) // call AttachReceivedAt like demux would + err, panicErr = pM.transition(inMsg) + require.NoError(t, err) + require.NoError(t, panicErr) + + verifiedVoteMsg := message{Vote: vVote, UnauthenticatedVote: vVote.u()} + inMsg = messageEvent{T: voteVerified, Input: verifiedVoteMsg, TaskIndex: 1} + inMsg = inMsg.AttachValidatedAt(502*time.Millisecond, r-credentialRoundLag, nil) + err, panicErr = pM.transition(inMsg) + require.NoError(t, err) + require.NoError(t, panicErr) + + // send payloadPresent message + m = message{UnauthenticatedProposal: pP.u()} + inMsg = messageEvent{T: payloadPresent, Input: m} + inMsg = inMsg.AttachReceivedAt(time.Second, r-credentialRoundLag, nil) + err, panicErr = pM.transition(inMsg) + require.NoError(t, err) + require.NoError(t, panicErr) + + // move to round r+1, triggering history update + vVote = helper.MakeVerifiedVote(t, 0, r, p, propose, *pV) + inMsg = messageEvent{T: voteVerified, Input: message{Vote: vVote, UnauthenticatedVote: vVote.u()}} + inMsg = inMsg.AttachValidatedAt(501*time.Millisecond, r, nil) + err, panicErr = pM.transition(inMsg) + require.NoError(t, err) + require.NoError(t, panicErr) + + // send payloadPresent message + m = message{UnauthenticatedProposal: pP.u()} + inMsg = messageEvent{T: payloadPresent, Input: m} + inMsg = inMsg.AttachReceivedAt(time.Second, r, nil) + err, panicErr = pM.transition(inMsg) + require.NoError(t, err) + require.NoError(t, panicErr) + moveToRound(t, pWhite, pM, helper, r+1, p, pP, pV, m, protocol.ConsensusFuture) + + // assert lowest vote validateAt time was recorded into payloadArrivals + require.NotNil(t, dynamicFilterCredentialArrivalHistory) + require.Equal(t, pWhite.lowestCredentialArrivals.writePtr, 1) + require.False(t, pWhite.lowestCredentialArrivals.isFull()) + require.Equal(t, 502*time.Millisecond, pWhite.lowestCredentialArrivals.history[0]) +} + // test that ReceivedAt and ValidateAt timing information are retained in // proposalStore when the payloadPresent (as part of the CompoundMessage // encoding used by PP messages), payloadVerified, and voteVerified events are @@ -3543,7 +3617,7 @@ func TestPlayerRetainsReceivedValidatedAtAVPPOneSample(t *testing.T) { require.NoError(t, err) require.NoError(t, panicErr) - assertCorrectReceivedAtSet(t, pWhite, pM, helper, r, p, pP, pV, m, protocol.ConsensusFuture) + assertCorrectReceivedAtSet(t, pWhite, pM, helper, r, p, pP, pV, m, protocol.ConsensusFuture, time.Second) require.False(t, pWhite.lowestCredentialArrivals.isFull()) require.Equal(t, pWhite.lowestCredentialArrivals.writePtr, 0) @@ -3600,6 +3674,94 @@ func TestPlayerRetainsReceivedValidatedAtAVPPOneSample(t *testing.T) { require.Equal(t, 502*time.Millisecond, pWhite.lowestCredentialArrivals.history[0]) } +// test that ReceivedAt and ValidateAt timing information are retained in +// proposalStore when the voteVerified event comes in first (as part of the AV +// message before PP), then the payloadPresent (as part of the CompoundMessage +// encoding used by PP messages) and payloadVerified events are processed one +// round early, and that all timings are available when the ensureAction is +// called for the block. +func TestPlayerRetainsEarlyReceivedValidatedAtAVPPOneSample(t *testing.T) { + partitiontest.PartitionTest(t) + + const r = round(20239) + const p = period(0) + pWhite, pM, helper := setupP(t, r-1, p, soft) + pP, pV := helper.MakeRandomProposalPayload(t, r-1) + + // Move to round r, no credentials arrived. + // send voteVerified message + vVote := helper.MakeVerifiedVote(t, 0, r-1, p, propose, *pV) + inMsg := messageEvent{T: voteVerified, Input: message{Vote: vVote, UnauthenticatedVote: vVote.u()}} + inMsg = inMsg.AttachValidatedAt(501*time.Millisecond, r-1, nil) + err, panicErr := pM.transition(inMsg) + require.NoError(t, err) + require.NoError(t, panicErr) + + // send payloadPresent message + m := message{UnauthenticatedProposal: pP.u()} + inMsg = messageEvent{T: payloadPresent, Input: m} + inMsg = inMsg.AttachReceivedAt(time.Second, r-1, nil) + err, panicErr = pM.transition(inMsg) + require.NoError(t, err) + require.NoError(t, panicErr) + + assertCorrectReceivedAtSet(t, pWhite, pM, helper, r, p, pP, pV, m, protocol.ConsensusFuture, time.Second) + require.False(t, pWhite.lowestCredentialArrivals.isFull()) + require.Equal(t, pWhite.lowestCredentialArrivals.writePtr, 0) + + // send votePresent message (mimicking the first AV message validating) + vVote = helper.MakeVerifiedVote(t, 0, r-credentialRoundLag, p, propose, *pV) + unverifiedVoteMsg := message{UnauthenticatedVote: vVote.u()} + inMsg = messageEvent{T: votePresent, Input: unverifiedVoteMsg} + err, panicErr = pM.transition(inMsg) + require.NoError(t, err) + require.NoError(t, panicErr) + + // make sure vote verify requests + verifyEvent := ev(cryptoAction{T: verifyVote, M: unverifiedVoteMsg, Round: r - 1, Period: p, Step: propose, TaskIndex: 1}) + require.Truef(t, pM.getTrace().Contains(verifyEvent), "Player should verify vote") + + // send voteVerified, pretend we're one round too early + verifiedVoteMsg := message{Vote: vVote, UnauthenticatedVote: vVote.u()} + inMsg = messageEvent{T: voteVerified, Input: verifiedVoteMsg, TaskIndex: 1} + inMsg = inMsg.AttachValidatedAt(502*time.Millisecond, r-credentialRoundLag-1, nil) + err, panicErr = pM.transition(inMsg) + require.NoError(t, err) + require.NoError(t, panicErr) + + // create a PP message for an arbitrary proposal/payload similar to setupCompoundMessage + proposalMsg := message{UnauthenticatedProposal: pP.u()} + compoundMsg := messageEvent{T: votePresent, Input: unverifiedVoteMsg, + Tail: &messageEvent{T: payloadPresent, Input: proposalMsg}} + inMsg = compoundMsg.AttachReceivedAt(time.Second, r-credentialRoundLag, nil) // call AttachReceivedAt like demux would + err, panicErr = pM.transition(inMsg) + require.NoError(t, err) + require.NoError(t, panicErr) + + // move to round r+1, triggering history update + vVote = helper.MakeVerifiedVote(t, 0, r, p, propose, *pV) + inMsg = messageEvent{T: voteVerified, Input: message{Vote: vVote, UnauthenticatedVote: vVote.u()}} + inMsg = inMsg.AttachValidatedAt(time.Second, r, nil) + err, panicErr = pM.transition(inMsg) + require.NoError(t, err) + require.NoError(t, panicErr) + + // send payloadPresent message + m = message{UnauthenticatedProposal: pP.u()} + inMsg = messageEvent{T: payloadPresent, Input: m} + inMsg = inMsg.AttachReceivedAt(time.Second, r, nil) + err, panicErr = pM.transition(inMsg) + require.NoError(t, err) + require.NoError(t, panicErr) + moveToRound(t, pWhite, pM, helper, r+1, p, pP, pV, m, protocol.ConsensusFuture) + + // assert lowest vote validateAt time was recorded into payloadArrivals + require.NotNil(t, dynamicFilterCredentialArrivalHistory) + require.Equal(t, pWhite.lowestCredentialArrivals.writePtr, 1) + require.False(t, pWhite.lowestCredentialArrivals.isFull()) + require.Equal(t, time.Duration(1), pWhite.lowestCredentialArrivals.history[0]) +} + func TestPlayerRetainsReceivedValidatedAtAVPPHistoryWindow(t *testing.T) { partitiontest.PartitionTest(t) const r = round(20239) @@ -3699,7 +3861,7 @@ func moveToRound(t *testing.T, pWhite *player, pM ioAutomata, helper *voteMakerH } func assertCorrectReceivedAtSet(t *testing.T, pWhite *player, pM ioAutomata, helper *voteMakerHelper, - r round, p period, pP *proposal, pV *proposalValue, m message, ver protocol.ConsensusVersion) { + r round, p period, pP *proposal, pV *proposalValue, m message, ver protocol.ConsensusVersion, validationTimestamp time.Duration) { moveToRound(t, pWhite, pM, helper, r, p, pP, pV, m, ver) @@ -3717,7 +3879,7 @@ func assertCorrectReceivedAtSet(t *testing.T, pWhite *player, pM ioAutomata, hel } require.True(t, foundEA) require.Equal(t, 2*time.Second, ea.Payload.validatedAt) - require.Equal(t, time.Second, ea.Payload.receivedAt) + require.Equal(t, validationTimestamp, ea.Payload.receivedAt) } // todo: test pipelined rounds, and round interruption From dc975c7bca7e992d7ff20cb2dcc60170968e7e53 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Wed, 30 Aug 2023 16:35:58 -0400 Subject: [PATCH 174/232] tests for late coming messages --- agreement/actions.go | 10 +- agreement/events.go | 7 +- agreement/player_test.go | 236 +++++++++++++++++++++++++++++++++++++++ agreement/service.go | 10 +- 4 files changed, 252 insertions(+), 11 deletions(-) diff --git a/agreement/actions.go b/agreement/actions.go index 7428d416db..5ed3c8c83b 100644 --- a/agreement/actions.go +++ b/agreement/actions.go @@ -322,14 +322,14 @@ func (a rezeroAction) do(ctx context.Context, s *Service) { removeList := make([]round, 0) - for r, _ := range s.historicalClocks { - if a.Round < r+credentialRoundLag { - removeList = append(removeList, r) + for round := range s.historicalClocks { + if a.Round < round+credentialRoundLag { + removeList = append(removeList, round) } } - for _, r := range removeList { - delete(s.historicalClocks, r) + for _, round := range removeList { + delete(s.historicalClocks, round) } } diff --git a/agreement/events.go b/agreement/events.go index e9f3e70a56..b2f0dabefb 100644 --- a/agreement/events.go +++ b/agreement/events.go @@ -22,7 +22,6 @@ import ( "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/protocol" - "github.com/algorand/go-algorand/util/timers" ) // An event represents the communication of an event to a state machine. @@ -980,7 +979,7 @@ func (e checkpointEvent) AttachConsensusVersion(v ConsensusVersionView) external return e } -func getTimestampForEvent(eventRound round, d time.Duration, currentRound round, historicalClocks map[round]timers.Clock[TimeoutType]) time.Duration { +func getTimestampForEvent(eventRound round, d time.Duration, currentRound round, historicalClocks map[round]historicalClock) time.Duration { if eventRound > currentRound { return time.Duration(1) } else if eventRound == currentRound { @@ -994,7 +993,7 @@ func getTimestampForEvent(eventRound round, d time.Duration, currentRound round, // AttachValidatedAt looks for a validated proposal or vote inside a // payloadVerified or voteVerified messageEvent, and attaches the given time to // the proposal's validatedAt field. -func (e messageEvent) AttachValidatedAt(d time.Duration, currentRound round, historicalClocks map[round]timers.Clock[TimeoutType]) messageEvent { +func (e messageEvent) AttachValidatedAt(d time.Duration, currentRound round, historicalClocks map[round]historicalClock) messageEvent { switch e.T { case payloadVerified: e.Input.Proposal.validatedAt = getTimestampForEvent(e.Input.Proposal.Round(), d, currentRound, historicalClocks) @@ -1007,7 +1006,7 @@ func (e messageEvent) AttachValidatedAt(d time.Duration, currentRound round, his // AttachReceivedAt looks for an unauthenticatedProposal inside a // payloadPresent or votePresent messageEvent, and attaches the given // time to the proposal's receivedAt field. -func (e messageEvent) AttachReceivedAt(d time.Duration, currentRound round, historicalClocks map[round]timers.Clock[TimeoutType]) messageEvent { +func (e messageEvent) AttachReceivedAt(d time.Duration, currentRound round, historicalClocks map[round]historicalClock) messageEvent { if e.T == payloadPresent { e.Input.UnauthenticatedProposal.receivedAt = getTimestampForEvent(e.Input.UnauthenticatedProposal.Round(), d, currentRound, historicalClocks) } else if e.T == votePresent { diff --git a/agreement/player_test.go b/agreement/player_test.go index 12068d05f7..bcd4b51675 100644 --- a/agreement/player_test.go +++ b/agreement/player_test.go @@ -3325,6 +3325,63 @@ func TestPlayerRetainsEarlyReceivedValidatedAtOneSample(t *testing.T) { require.Equal(t, time.Duration(1), pWhite.lowestCredentialArrivals.history[0]) } +type fixedHistoricalClock struct{} + +const fixedClockDuration = 900 * time.Millisecond + +func (fixedHistoricalClock) Since() time.Duration { + return fixedClockDuration +} + +// test that ReceivedAt and ValidateAt timing information are retained in +// proposalStore when the payloadPresent, payloadVerified, and voteVerified +// events are processed credentialRoundLag after the round they belong to, and +// that all timings are available when the ensureAction is called for the block. +func TestPlayerRetainsLateReceivedValidatedAtOneSample(t *testing.T) { + partitiontest.PartitionTest(t) + const r = round(20239) + const p = period(0) + pWhite, pM, helper := setupP(t, r-1, p, soft) + + historicalClocks := map[round]historicalClock{ + r - credentialRoundLag - 1: fixedHistoricalClock{}, + } + + // send voteVerified message + pP, pV := helper.MakeRandomProposalPayload(t, r-credentialRoundLag-1) + vVote := helper.MakeVerifiedVote(t, 0, r-credentialRoundLag-1, p, propose, *pV) + inMsg := messageEvent{T: voteVerified, Input: message{Vote: vVote, UnauthenticatedVote: vVote.u()}} + inMsg = inMsg.AttachValidatedAt(401*time.Millisecond, r-1, historicalClocks) + err, panicErr := pM.transition(inMsg) + require.NoError(t, err) + require.NoError(t, panicErr) + + // send voteVerified message + pP, pV = helper.MakeRandomProposalPayload(t, r-1) + vVote = helper.MakeVerifiedVote(t, 0, r-1, p, propose, *pV) + inMsg = messageEvent{T: voteVerified, Input: message{Vote: vVote, UnauthenticatedVote: vVote.u()}} + inMsg = inMsg.AttachValidatedAt(501*time.Millisecond, r-1, nil) + err, panicErr = pM.transition(inMsg) + require.NoError(t, err) + require.NoError(t, panicErr) + + // send payloadPresent message + m := message{UnauthenticatedProposal: pP.u()} + inMsg = messageEvent{T: payloadPresent, Input: m} + inMsg = inMsg.AttachReceivedAt(time.Second, r-1, nil) + err, panicErr = pM.transition(inMsg) + require.NoError(t, err) + require.NoError(t, panicErr) + + assertCorrectReceivedAtSet(t, pWhite, pM, helper, r, p, pP, pV, m, protocol.ConsensusFuture, time.Second) + + // assert lowest vote validateAt time was recorded into payloadArrivals + require.NotZero(t, dynamicFilterCredentialArrivalHistory) + require.Equal(t, pWhite.lowestCredentialArrivals.writePtr, 1) + require.False(t, pWhite.lowestCredentialArrivals.isFull()) + require.Equal(t, fixedClockDuration, pWhite.lowestCredentialArrivals.history[0]) +} + // test that ReceivedAt and ValidateAt timing information are retained in proposalStore // when the payloadPresent, payloadVerified, and voteVerified events are processed, and that all timings // are available when the ensureAction is called for the block. The history should be kept for the last @@ -3535,6 +3592,94 @@ func TestPlayerRetainsEarlyReceivedValidatedAtPPOneSample(t *testing.T) { require.Equal(t, 502*time.Millisecond, pWhite.lowestCredentialArrivals.history[0]) } +// test that ReceivedAt and ValidateAt timing information are retained in +// proposalStore when the payloadPresent (as part of the CompoundMessage +// encoding used by PP messages), payloadVerified, and voteVerified events are +// processed credentialRoundLag after the round they belong to, and that all +// timings are available when the ensureAction is called for the block. +func TestPlayerRetainsLateReceivedValidatedAtPPOneSample(t *testing.T) { + partitiontest.PartitionTest(t) + + const r = round(20239) + const p = period(0) + pWhite, pM, helper := setupP(t, r-1, p, soft) + pP, pV := helper.MakeRandomProposalPayload(t, r-1) + + // Move to round r, no credentials arrived. + // send voteVerified message + vVote := helper.MakeVerifiedVote(t, 0, r-1, p, propose, *pV) + inMsg := messageEvent{T: voteVerified, Input: message{Vote: vVote, UnauthenticatedVote: vVote.u()}} + inMsg = inMsg.AttachValidatedAt(501*time.Millisecond, r-1, nil) + err, panicErr := pM.transition(inMsg) + require.NoError(t, err) + require.NoError(t, panicErr) + + // send payloadPresent message + m := message{UnauthenticatedProposal: pP.u()} + inMsg = messageEvent{T: payloadPresent, Input: m} + inMsg = inMsg.AttachReceivedAt(time.Second, r-1, nil) + err, panicErr = pM.transition(inMsg) + require.NoError(t, err) + require.NoError(t, panicErr) + + assertCorrectReceivedAtSet(t, pWhite, pM, helper, r, p, pP, pV, m, protocol.ConsensusFuture, time.Second) + require.False(t, pWhite.lowestCredentialArrivals.isFull()) + require.Equal(t, pWhite.lowestCredentialArrivals.writePtr, 0) + + historicalClocks := map[round]historicalClock{ + r - credentialRoundLag: fixedHistoricalClock{}, + } + // create a PP message for the round we're going to take the sample from when round r-1 ends + // Now we're going to pretend we got the message credentialRoundLag too late. + vVote = helper.MakeVerifiedVote(t, 0, r-credentialRoundLag, p, propose, *pV) + unverifiedVoteMsg := message{UnauthenticatedVote: vVote.u()} + proposalMsg := message{UnauthenticatedProposal: pP.u()} + compoundMsg := messageEvent{T: votePresent, Input: unverifiedVoteMsg, + Tail: &messageEvent{T: payloadPresent, Input: proposalMsg}} + inMsg = compoundMsg.AttachReceivedAt(time.Second, r, historicalClocks) // call AttachReceivedAt like demux would + err, panicErr = pM.transition(inMsg) + require.NoError(t, err) + require.NoError(t, panicErr) + + verifiedVoteMsg := message{Vote: vVote, UnauthenticatedVote: vVote.u()} + inMsg = messageEvent{T: voteVerified, Input: verifiedVoteMsg, TaskIndex: 1} + inMsg = inMsg.AttachValidatedAt(502*time.Millisecond, r, historicalClocks) + err, panicErr = pM.transition(inMsg) + require.NoError(t, err) + require.NoError(t, panicErr) + + // send payloadPresent message + m = message{UnauthenticatedProposal: pP.u()} + inMsg = messageEvent{T: payloadPresent, Input: m} + inMsg = inMsg.AttachReceivedAt(time.Second, r-credentialRoundLag, historicalClocks) + err, panicErr = pM.transition(inMsg) + require.NoError(t, err) + require.NoError(t, panicErr) + + // move to round r+1, triggering history update + vVote = helper.MakeVerifiedVote(t, 0, r, p, propose, *pV) + inMsg = messageEvent{T: voteVerified, Input: message{Vote: vVote, UnauthenticatedVote: vVote.u()}} + inMsg = inMsg.AttachValidatedAt(501*time.Millisecond, r, nil) + err, panicErr = pM.transition(inMsg) + require.NoError(t, err) + require.NoError(t, panicErr) + + // send payloadPresent message + m = message{UnauthenticatedProposal: pP.u()} + inMsg = messageEvent{T: payloadPresent, Input: m} + inMsg = inMsg.AttachReceivedAt(time.Second, r, nil) + err, panicErr = pM.transition(inMsg) + require.NoError(t, err) + require.NoError(t, panicErr) + moveToRound(t, pWhite, pM, helper, r+1, p, pP, pV, m, protocol.ConsensusFuture) + + // assert lowest vote validateAt time was recorded into payloadArrivals + require.NotNil(t, dynamicFilterCredentialArrivalHistory) + require.Equal(t, pWhite.lowestCredentialArrivals.writePtr, 1) + require.False(t, pWhite.lowestCredentialArrivals.isFull()) + require.Equal(t, fixedClockDuration, pWhite.lowestCredentialArrivals.history[0]) +} + // test that ReceivedAt and ValidateAt timing information are retained in // proposalStore when the payloadPresent (as part of the CompoundMessage // encoding used by PP messages), payloadVerified, and voteVerified events are @@ -3762,6 +3907,97 @@ func TestPlayerRetainsEarlyReceivedValidatedAtAVPPOneSample(t *testing.T) { require.Equal(t, time.Duration(1), pWhite.lowestCredentialArrivals.history[0]) } +// test that ReceivedAt and ValidateAt timing information are retained in +// proposalStore when the voteVerified event comes in first (as part of the AV +// message before PP), then the payloadPresent (as part of the CompoundMessage +// encoding used by PP messages) and payloadVerified events are processed +// credentialRoundLag after the round they belong to, and that all timings are +// available when the ensureAction is called for the block. +func TestPlayerRetainsLateReceivedValidatedAtAVPPOneSample(t *testing.T) { + partitiontest.PartitionTest(t) + + const r = round(20239) + const p = period(0) + pWhite, pM, helper := setupP(t, r-1, p, soft) + pP, pV := helper.MakeRandomProposalPayload(t, r-1) + + // Move to round r, no credentials arrived. + // send voteVerified message + vVote := helper.MakeVerifiedVote(t, 0, r-1, p, propose, *pV) + inMsg := messageEvent{T: voteVerified, Input: message{Vote: vVote, UnauthenticatedVote: vVote.u()}} + inMsg = inMsg.AttachValidatedAt(501*time.Millisecond, r-1, nil) + err, panicErr := pM.transition(inMsg) + require.NoError(t, err) + require.NoError(t, panicErr) + + // send payloadPresent message + m := message{UnauthenticatedProposal: pP.u()} + inMsg = messageEvent{T: payloadPresent, Input: m} + inMsg = inMsg.AttachReceivedAt(time.Second, r-1, nil) + err, panicErr = pM.transition(inMsg) + require.NoError(t, err) + require.NoError(t, panicErr) + + assertCorrectReceivedAtSet(t, pWhite, pM, helper, r, p, pP, pV, m, protocol.ConsensusFuture, time.Second) + require.False(t, pWhite.lowestCredentialArrivals.isFull()) + require.Equal(t, pWhite.lowestCredentialArrivals.writePtr, 0) + + // send votePresent message (mimicking the first AV message validating) + vVote = helper.MakeVerifiedVote(t, 0, r-credentialRoundLag, p, propose, *pV) + unverifiedVoteMsg := message{UnauthenticatedVote: vVote.u()} + inMsg = messageEvent{T: votePresent, Input: unverifiedVoteMsg} + err, panicErr = pM.transition(inMsg) + require.NoError(t, err) + require.NoError(t, panicErr) + + // make sure vote verify requests + verifyEvent := ev(cryptoAction{T: verifyVote, M: unverifiedVoteMsg, Round: r - 1, Period: p, Step: propose, TaskIndex: 1}) + require.Truef(t, pM.getTrace().Contains(verifyEvent), "Player should verify vote") + + historicalClocks := map[round]historicalClock{ + r - credentialRoundLag: fixedHistoricalClock{}, + } + // send voteVerified, pretend we're credentialRoundLag after the message was sent + verifiedVoteMsg := message{Vote: vVote, UnauthenticatedVote: vVote.u()} + inMsg = messageEvent{T: voteVerified, Input: verifiedVoteMsg, TaskIndex: 1} + inMsg = inMsg.AttachValidatedAt(502*time.Millisecond, r, historicalClocks) + err, panicErr = pM.transition(inMsg) + require.NoError(t, err) + require.NoError(t, panicErr) + + // create a PP message for an arbitrary proposal/payload similar to setupCompoundMessage + proposalMsg := message{UnauthenticatedProposal: pP.u()} + compoundMsg := messageEvent{T: votePresent, Input: unverifiedVoteMsg, + Tail: &messageEvent{T: payloadPresent, Input: proposalMsg}} + inMsg = compoundMsg.AttachReceivedAt(time.Second, r-credentialRoundLag, nil) // call AttachReceivedAt like demux would + err, panicErr = pM.transition(inMsg) + require.NoError(t, err) + require.NoError(t, panicErr) + + // move to round r+1, triggering history update + vVote = helper.MakeVerifiedVote(t, 0, r, p, propose, *pV) + inMsg = messageEvent{T: voteVerified, Input: message{Vote: vVote, UnauthenticatedVote: vVote.u()}} + inMsg = inMsg.AttachValidatedAt(time.Second, r, nil) + err, panicErr = pM.transition(inMsg) + require.NoError(t, err) + require.NoError(t, panicErr) + + // send payloadPresent message + m = message{UnauthenticatedProposal: pP.u()} + inMsg = messageEvent{T: payloadPresent, Input: m} + inMsg = inMsg.AttachReceivedAt(time.Second, r, nil) + err, panicErr = pM.transition(inMsg) + require.NoError(t, err) + require.NoError(t, panicErr) + moveToRound(t, pWhite, pM, helper, r+1, p, pP, pV, m, protocol.ConsensusFuture) + + // assert lowest vote validateAt time was recorded into payloadArrivals + require.NotNil(t, dynamicFilterCredentialArrivalHistory) + require.Equal(t, pWhite.lowestCredentialArrivals.writePtr, 1) + require.False(t, pWhite.lowestCredentialArrivals.isFull()) + require.Equal(t, fixedClockDuration, pWhite.lowestCredentialArrivals.history[0]) +} + func TestPlayerRetainsReceivedValidatedAtAVPPHistoryWindow(t *testing.T) { partitiontest.PartitionTest(t) const r = round(20239) diff --git a/agreement/service.go b/agreement/service.go index 01955809f1..b0e45328bc 100644 --- a/agreement/service.go +++ b/agreement/service.go @@ -19,6 +19,7 @@ package agreement //go:generate dbgen -i agree.sql -p agreement -n agree -o agreeInstall.go -h ../scripts/LICENSE_HEADER import ( "context" + "time" "github.com/algorand/go-algorand/config" "github.com/algorand/go-algorand/logging" @@ -57,7 +58,7 @@ type Service struct { persistStatus player persistActions []action - historicalClocks map[round]timers.Clock[TimeoutType] + historicalClocks map[round]historicalClock } // Parameters holds the parameters necessary to run the agreement protocol. @@ -86,6 +87,11 @@ type externalDemuxSignals struct { CurrentRound round } +// an interface representing a clock from a previous round +type historicalClock interface { + Since() time.Duration +} + // MakeService creates a new Agreement Service instance given a set of Parameters. // // Call Start to start execution and Shutdown to finish execution. @@ -107,7 +113,7 @@ func MakeService(p Parameters) (*Service, error) { s.persistenceLoop = makeAsyncPersistenceLoop(s.log, s.Accessor, s.Ledger) - s.historicalClocks = make(map[round]timers.Clock[TimeoutType]) + s.historicalClocks = make(map[round]historicalClock) return s, nil } From 3eeaebe22caeebcef29f9de97580de7610629892 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Wed, 30 Aug 2023 17:21:06 -0400 Subject: [PATCH 175/232] historical clocks cleanup --- agreement/actions.go | 2 +- agreement/service_test.go | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/agreement/actions.go b/agreement/actions.go index 5ed3c8c83b..52608e4567 100644 --- a/agreement/actions.go +++ b/agreement/actions.go @@ -323,7 +323,7 @@ func (a rezeroAction) do(ctx context.Context, s *Service) { removeList := make([]round, 0) for round := range s.historicalClocks { - if a.Round < round+credentialRoundLag { + if a.Round > round+credentialRoundLag { removeList = append(removeList, round) } } diff --git a/agreement/service_test.go b/agreement/service_test.go index b4f1bbc32d..0a8196095a 100644 --- a/agreement/service_test.go +++ b/agreement/service_test.go @@ -935,6 +935,16 @@ func simulateAgreementWithLedgerFactory(t *testing.T, numNodes int, numRounds in services[i].Shutdown() } + // check that historical clocks map didn't get too large + for i := 0; i < numNodes; i++ { + require.LessOrEqual(t, len(services[i].historicalClocks), int(credentialRoundLag)+1, "too many historical clocks kept") + } + if numRounds >= int(credentialRoundLag) { + for i := 0; i < numNodes; i++ { + require.Equal(t, len(services[i].historicalClocks), int(credentialRoundLag)+1, "not enough historical clocks kept") + } + } + sanityCheck(startRound, round(numRounds), ledgers) if len(clocks) == 0 { @@ -1021,6 +1031,16 @@ func TestAgreementSynchronous5_50(t *testing.T) { simulateAgreement(t, 5, 50, disabled) } +func TestAgreementHistoricalClocksCleanup(t *testing.T) { + partitiontest.PartitionTest(t) + + if testing.Short() { + t.Skip("Skipping agreement integration test") + } + + simulateAgreement(t, 5, int(credentialRoundLag)+10, disabled) +} + func createDynamicFilterConfig() (version protocol.ConsensusVersion, consensusVersion func(r basics.Round) (protocol.ConsensusVersion, error), configCleanup func()) { version = protocol.ConsensusVersion("test-protocol-filtertimeout") protoParams := config.Consensus[protocol.ConsensusCurrentVersion] From 61d0aa0007ca0026e1a19dfed5aecb5499ae6271 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Wed, 30 Aug 2023 21:57:49 -0400 Subject: [PATCH 176/232] more checks that service keeps the right clocks --- agreement/service_test.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/agreement/service_test.go b/agreement/service_test.go index 0a8196095a..25b067975b 100644 --- a/agreement/service_test.go +++ b/agreement/service_test.go @@ -935,9 +935,18 @@ func simulateAgreementWithLedgerFactory(t *testing.T, numNodes int, numRounds in services[i].Shutdown() } + firstHistoricalClocksRound := startRound + if basics.Round(numRounds) > credentialRoundLag { + firstHistoricalClocksRound = startRound + basics.Round(numRounds) - credentialRoundLag + } + // check that historical clocks map didn't get too large for i := 0; i < numNodes; i++ { require.LessOrEqual(t, len(services[i].historicalClocks), int(credentialRoundLag)+1, "too many historical clocks kept") + for round := firstHistoricalClocksRound + 1; round <= startRound+basics.Round(numRounds); round++ { + _, has := services[i].historicalClocks[round] + require.True(t, has) + } } if numRounds >= int(credentialRoundLag) { for i := 0; i < numNodes; i++ { From e54e5503da9f6b4619271d16ad97abf5597a07d9 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Thu, 31 Aug 2023 13:42:10 -0400 Subject: [PATCH 177/232] fix sdk comparison test --- tools/x-repo-types/xrt_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/x-repo-types/xrt_test.go b/tools/x-repo-types/xrt_test.go index 8de6e83122..4360b432d6 100644 --- a/tools/x-repo-types/xrt_test.go +++ b/tools/x-repo-types/xrt_test.go @@ -78,7 +78,7 @@ func TestCrossRepoTypes(t *testing.T) { xPkg: "github.com/algorand/go-algorand/config", xType: "ConsensusParams", yPkg: "github.com/algorand/go-algorand-sdk/v2/protocol/config", - yBranch: "b5f90353ea841fc2e3af9f77d42205c337c93dd2", + yBranch: "develop", yType: "ConsensusParams", }, { From c490e92a6c15616df85a10b3027bfd386ab447b8 Mon Sep 17 00:00:00 2001 From: yossigi Date: Thu, 31 Aug 2023 15:48:10 -0400 Subject: [PATCH 178/232] Update agreement/proposalManager.go nit Co-authored-by: Pavel Zbitskiy <65323360+algorandskiy@users.noreply.github.com> --- agreement/proposalManager.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/agreement/proposalManager.go b/agreement/proposalManager.go index 4e72709988..18c14a29cd 100644 --- a/agreement/proposalManager.go +++ b/agreement/proposalManager.go @@ -232,7 +232,9 @@ func (m *proposalManager) filterProposalVote(p player, r routerHandle, uv unauth // voteFresh determines whether a proposal satisfies freshness rules. func proposalFresh(freshData freshnessData, vote unauthenticatedVote) error { - if vote.R.Round < freshData.PlayerRound && freshData.PlayerRound <= vote.R.Round+credentialRoundLag && vote.R.Period == 0 && vote.R.Step == 0 { + if vote.R.Round < freshData.PlayerRound && freshData.PlayerRound <= vote.R.Round+credentialRoundLag && + vote.R.Period == 0 && + vote.R.Step == propose { if dynamicFilterCredentialArrivalHistory > 0 { // continue processing old period 0 votes so we could track their // arrival times and inform setting the filter timeout dynamically. From 82a659f8a476a2e8eda3fd354d2de9d2eeef0ea4 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Thu, 31 Aug 2023 15:49:43 -0400 Subject: [PATCH 179/232] nits --- agreement/events.go | 6 ++++-- agreement/player.go | 7 ++++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/agreement/events.go b/agreement/events.go index b2f0dabefb..e89db243fa 100644 --- a/agreement/events.go +++ b/agreement/events.go @@ -982,9 +982,11 @@ func (e checkpointEvent) AttachConsensusVersion(v ConsensusVersionView) external func getTimestampForEvent(eventRound round, d time.Duration, currentRound round, historicalClocks map[round]historicalClock) time.Duration { if eventRound > currentRound { return time.Duration(1) - } else if eventRound == currentRound { + } + if eventRound == currentRound { return d - } else if clock, ok := historicalClocks[eventRound]; ok { + } + if clock, ok := historicalClocks[eventRound]; ok { return clock.Since() } return time.Duration(0) diff --git a/agreement/player.go b/agreement/player.go index 85015209b6..fd9f77266c 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -281,7 +281,12 @@ func (p *player) handleCheckpointEvent(r routerHandle, e checkpointEvent) []acti // It returns the time of the lowest credential's arrival, if one was // collected and added to lowestCredentialArrivals, or zero otherwise. func (p *player) updateCredentialArrivalHistory(r routerHandle, ver protocol.ConsensusVersion) time.Duration { - if p.Round < credentialRoundLag { + if p.Period != 0 { + // only append to lowestCredentialArrivals if this was a successful round completing in period 0. + return 0 + } + + if p.Round <= credentialRoundLag { // not sufficiently many rounds had passed to collect any measurement return 0 } From 916319984892d37bc1c607ed19e57eb6b5015835 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Thu, 31 Aug 2023 16:08:45 -0400 Subject: [PATCH 180/232] fmt --- agreement/proposalManager.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agreement/proposalManager.go b/agreement/proposalManager.go index 18c14a29cd..0eb90cfb30 100644 --- a/agreement/proposalManager.go +++ b/agreement/proposalManager.go @@ -233,7 +233,7 @@ func (m *proposalManager) filterProposalVote(p player, r routerHandle, uv unauth // voteFresh determines whether a proposal satisfies freshness rules. func proposalFresh(freshData freshnessData, vote unauthenticatedVote) error { if vote.R.Round < freshData.PlayerRound && freshData.PlayerRound <= vote.R.Round+credentialRoundLag && - vote.R.Period == 0 && + vote.R.Period == 0 && vote.R.Step == propose { if dynamicFilterCredentialArrivalHistory > 0 { // continue processing old period 0 votes so we could track their From ef88e2dc36730a1effefdfc91a1c3db54fd8aab2 Mon Sep 17 00:00:00 2001 From: chris erway Date: Fri, 1 Sep 2023 11:41:29 -0400 Subject: [PATCH 181/232] update TestProposalTrackerProposalSeeker --- agreement/events.go | 6 +++++- agreement/proposalTracker_test.go | 20 ++++++++++++++++---- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/agreement/events.go b/agreement/events.go index e89db243fa..1fad872e75 100644 --- a/agreement/events.go +++ b/agreement/events.go @@ -979,9 +979,13 @@ func (e checkpointEvent) AttachConsensusVersion(v ConsensusVersionView) external return e } +// This timestamp is assigned to messages that arrive for round R+1 while the current player +// is still waiting for quorum on R. +const pipelinedMessageTimestamp = time.Nanosecond + func getTimestampForEvent(eventRound round, d time.Duration, currentRound round, historicalClocks map[round]historicalClock) time.Duration { if eventRound > currentRound { - return time.Duration(1) + return pipelinedMessageTimestamp } if eventRound == currentRound { return d diff --git a/agreement/proposalTracker_test.go b/agreement/proposalTracker_test.go index d830280ef7..1b8a47aef6 100644 --- a/agreement/proposalTracker_test.go +++ b/agreement/proposalTracker_test.go @@ -63,38 +63,50 @@ func TestProposalTrackerProposalSeeker(t *testing.T) { var err error assert.False(t, s.Frozen) assert.False(t, s.Filled) + assert.False(t, s.hasLowestAfterFreeze) // issue events in the following order: 2, 3, 1, (freeze), 0 - s, _, err = s.accept(votes[2]) + var updated bool + s, updated, err = s.accept(votes[2]) assert.NoError(t, err) + assert.True(t, updated) assert.False(t, s.Frozen) assert.True(t, s.Filled) assert.True(t, s.Lowest.equals(votes[2])) + assert.True(t, s.hasLowestAfterFreeze) - s, _, err = s.accept(votes[3]) + s, updated, err = s.accept(votes[3]) assert.Error(t, err) + assert.False(t, updated) assert.False(t, s.Frozen) assert.True(t, s.Filled) assert.True(t, s.Lowest.equals(votes[2])) + assert.True(t, s.hasLowestAfterFreeze) - s, _, err = s.accept(votes[1]) + s, updated, err = s.accept(votes[1]) assert.NoError(t, err) + assert.True(t, updated) assert.False(t, s.Frozen) assert.True(t, s.Filled) assert.True(t, s.Lowest.equals(votes[1])) + assert.True(t, s.hasLowestAfterFreeze) lowestBeforeFreeze := s.Lowest s = s.freeze() assert.True(t, s.Frozen) assert.True(t, s.Filled) assert.True(t, s.Lowest.equals(votes[1])) + assert.True(t, s.hasLowestAfterFreeze) - s, _, err = s.accept(votes[0]) + s, updated, err = s.accept(votes[0]) assert.Error(t, err) + assert.True(t, updated) assert.Equal(t, s.Lowest, lowestBeforeFreeze) assert.True(t, s.Frozen) assert.True(t, s.Filled) assert.True(t, s.Lowest.equals(votes[1])) + assert.True(t, s.hasLowestAfterFreeze) + assert.True(t, s.lowestAfterFreeze.equals(votes[0])) } // mimics a proposalTracker, producing a trace of events From 8d6a5f1c410f2c70de0d5d3a34fd04bee5e18d5f Mon Sep 17 00:00:00 2001 From: yossigi Date: Fri, 1 Sep 2023 14:03:46 -0400 Subject: [PATCH 182/232] Update agreement/player.go Co-authored-by: cce <51567+cce@users.noreply.github.com> --- agreement/player.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/agreement/player.go b/agreement/player.go index fd9f77266c..723c22a88e 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -278,7 +278,8 @@ func (p *player) handleCheckpointEvent(r routerHandle, e checkpointEvent) []acti // updateCredentialArrivalHistory is called at the end of a successful // uninterrupted round (just after ensureAction is generated) to collect // credential arrival times to dynamically set the filter timeout. -// It returns the time of the lowest credential's arrival, if one was +// It returns the time of the lowest credential's arrival from +// credentialRoundLag rounds ago, if one was // collected and added to lowestCredentialArrivals, or zero otherwise. func (p *player) updateCredentialArrivalHistory(r routerHandle, ver protocol.ConsensusVersion) time.Duration { if p.Period != 0 { From 6d447539185f0ca9b584adcec4e9dfd9a91abde6 Mon Sep 17 00:00:00 2001 From: yossigi Date: Fri, 1 Sep 2023 14:04:45 -0400 Subject: [PATCH 183/232] Update agreement/proposalTracker.go Co-authored-by: cce <51567+cce@users.noreply.github.com> --- agreement/proposalTracker.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/agreement/proposalTracker.go b/agreement/proposalTracker.go index c95de9e3bb..7498eeaefd 100644 --- a/agreement/proposalTracker.go +++ b/agreement/proposalTracker.go @@ -34,6 +34,8 @@ type proposalSeeker struct { // Filled will no longer be modified. Frozen bool + // lowestAfterFreeze is used to track the lowest credential observed, even + // after the Lowest value has been frozen. lowestAfterFreeze vote hasLowestAfterFreeze bool } From 19f077cde03414e9a15d3497dc373055fabeb86e Mon Sep 17 00:00:00 2001 From: chris erway Date: Fri, 1 Sep 2023 14:13:33 -0400 Subject: [PATCH 184/232] add LowestAfterFreeze to readLowestEvent --- agreement/events.go | 11 ++++++++--- agreement/player.go | 11 ++++++----- agreement/proposalTracker.go | 6 ++++-- 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/agreement/events.go b/agreement/events.go index 1fad872e75..2c4cc21911 100644 --- a/agreement/events.go +++ b/agreement/events.go @@ -424,9 +424,14 @@ type readLowestEvent struct { // Vote holds the lowest-credential vote. Vote vote - - // Filled indicates whether the Vote field is filled - Filled bool + // LowestAfterFreeze holds the lowest-credential vote that was received, including + // after Vote has been frozen. + LowestAfterFreeze vote + + // Filled and HasLowestAfterFreeze indicates whether the Vote or LowestAfterFreeze + // fields are filled, respectively. + Filled bool + HasLowestAfterFreeze bool } func (e readLowestEvent) t() eventType { diff --git a/agreement/player.go b/agreement/player.go index fd9f77266c..895a60ae0a 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -278,8 +278,9 @@ func (p *player) handleCheckpointEvent(r routerHandle, e checkpointEvent) []acti // updateCredentialArrivalHistory is called at the end of a successful // uninterrupted round (just after ensureAction is generated) to collect // credential arrival times to dynamically set the filter timeout. -// It returns the time of the lowest credential's arrival, if one was -// collected and added to lowestCredentialArrivals, or zero otherwise. +// It returns the time of the lowest credential's arrival from +// credentialRoundLag rounds ago, if one was collected and added to +// lowestCredentialArrivals, or zero otherwise. func (p *player) updateCredentialArrivalHistory(r routerHandle, ver protocol.ConsensusVersion) time.Duration { if p.Period != 0 { // only append to lowestCredentialArrivals if this was a successful round completing in period 0. @@ -296,12 +297,12 @@ func (p *player) updateCredentialArrivalHistory(r routerHandle, ver protocol.Con credHistoryRound := p.Round - credentialRoundLag re := readLowestEvent{T: readLowestVote, Round: credHistoryRound, Period: 0} re = r.dispatch(*p, re, proposalMachineRound, credHistoryRound, 0, 0).(readLowestEvent) - if !re.Filled { + if !re.HasLowestAfterFreeze { return 0 } - p.lowestCredentialArrivals.store(re.Vote.validatedAt) - return re.Vote.validatedAt + p.lowestCredentialArrivals.store(re.LowestAfterFreeze.validatedAt) + return re.LowestAfterFreeze.validatedAt } // calculateFilterTimeout chooses the appropriate filter timeout. diff --git a/agreement/proposalTracker.go b/agreement/proposalTracker.go index c95de9e3bb..52d628e909 100644 --- a/agreement/proposalTracker.go +++ b/agreement/proposalTracker.go @@ -182,8 +182,10 @@ func (t *proposalTracker) handle(r routerHandle, p player, e event) event { case readLowestVote: e := e.(readLowestEvent) - e.Vote = t.Freezer.lowestAfterFreeze - e.Filled = t.Freezer.hasLowestAfterFreeze + e.Vote = t.Freezer.Lowest + e.Filled = t.Freezer.Filled + e.LowestAfterFreeze = t.Freezer.lowestAfterFreeze + e.HasLowestAfterFreeze = t.Freezer.hasLowestAfterFreeze return e case softThreshold, certThreshold: From 9ad9dbb5cf379f7f597e7dc546a7a15d42a874a9 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Fri, 1 Sep 2023 14:18:51 -0400 Subject: [PATCH 185/232] fix test off by 1 --- agreement/service_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agreement/service_test.go b/agreement/service_test.go index 25b067975b..f6ff27fd2b 100644 --- a/agreement/service_test.go +++ b/agreement/service_test.go @@ -1200,7 +1200,7 @@ func TestDynamicFilterTimeoutResets(t *testing.T) { for j := 1; j < dynamicFilterCredentialArrivalHistory-2; j++ { require.Equal(t, filterTimeoutsPostRecovery[i][j-1], filterTimeoutsPostRecovery[i][j]) } - require.Less(t, filterTimeoutsPostRecovery[i][dynamicFilterCredentialArrivalHistory-2], filterTimeoutsPostRecovery[i][dynamicFilterCredentialArrivalHistory-3]) + require.Less(t, filterTimeoutsPostRecovery[i][dynamicFilterCredentialArrivalHistory-1], filterTimeoutsPostRecovery[i][dynamicFilterCredentialArrivalHistory-2]) } sanityCheck(startRound, 2*round(baseHistoryRounds+2), ledgers) From 4f17cfd0872165faf9571c5b183d6588a564530f Mon Sep 17 00:00:00 2001 From: cce <51567+cce@users.noreply.github.com> Date: Fri, 1 Sep 2023 14:21:58 -0400 Subject: [PATCH 186/232] Update agreement/proposalTracker.go --- agreement/proposalTracker.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/agreement/proposalTracker.go b/agreement/proposalTracker.go index 8ae920954c..c108c92370 100644 --- a/agreement/proposalTracker.go +++ b/agreement/proposalTracker.go @@ -41,7 +41,8 @@ type proposalSeeker struct { } // accept compares a given vote with the current lowest-credentialled vote and -// sets it if freeze has not been called. +// sets it if freeze has not been called. Returns true if any internal proposalSeeker +// state has been updated, whether to Lowest or lowestAfterFreeze. func (s proposalSeeker) accept(v vote) (proposalSeeker, bool, error) { if s.Frozen { updated := false From 890b9bc7ed028f511922edccea108592eb60ff15 Mon Sep 17 00:00:00 2001 From: chris erway Date: Fri, 1 Sep 2023 14:22:35 -0400 Subject: [PATCH 187/232] fix fmt --- agreement/proposalTracker.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/agreement/proposalTracker.go b/agreement/proposalTracker.go index c108c92370..a9e0380bf1 100644 --- a/agreement/proposalTracker.go +++ b/agreement/proposalTracker.go @@ -34,8 +34,8 @@ type proposalSeeker struct { // Filled will no longer be modified. Frozen bool - // lowestAfterFreeze is used to track the lowest credential observed, even - // after the Lowest value has been frozen. + // lowestAfterFreeze is used to track the lowest credential observed, even + // after the Lowest value has been frozen. lowestAfterFreeze vote hasLowestAfterFreeze bool } From ebbb6f624c298f3ea090fbcd757e57c991b83848 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Fri, 1 Sep 2023 14:43:35 -0400 Subject: [PATCH 188/232] gating relay policy change behind consensus upgrade --- agreement/player.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/agreement/player.go b/agreement/player.go index 895a60ae0a..2c15dd87dd 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -612,7 +612,9 @@ func (p *player) handleMessageEvent(r routerHandle, e messageEvent) (actions []a err := ef.(filteredEvent).Err return append(actions, disconnectAction(e, err)) case voteFiltered: - if !ef.(filteredEvent).StateUpdated { + ver := e.Proto.Version + proto := config.Consensus[ver] + if !proto.DynamicFilterTimeout || !ef.(filteredEvent).StateUpdated { err := ef.(filteredEvent).Err return append(actions, ignoreAction(e, err)) } @@ -711,7 +713,9 @@ func (p *player) handleMessageEvent(r routerHandle, e messageEvent) (actions []a err := makeSerErrf("rejected message since it was invalid: %v", ef.(filteredEvent).Err) return append(actions, disconnectAction(e, err)) case voteFiltered: - if !ef.(filteredEvent).StateUpdated { + ver := e.Proto.Version + proto := config.Consensus[ver] + if !proto.DynamicFilterTimeout || !ef.(filteredEvent).StateUpdated { err := ef.(filteredEvent).Err return append(actions, ignoreAction(e, err)) } From 5b720e641e2edc8622ac73313f24aa4d770a0006 Mon Sep 17 00:00:00 2001 From: chris erway Date: Fri, 1 Sep 2023 16:55:14 -0400 Subject: [PATCH 189/232] split proposalFresh and proposalUsedForCredentialHistory --- agreement/player.go | 22 ++++++++++------------ agreement/player_permutation_test.go | 13 +++++-------- agreement/proposalManager.go | 13 ++++++++----- 3 files changed, 23 insertions(+), 25 deletions(-) diff --git a/agreement/player.go b/agreement/player.go index 2c15dd87dd..aaae4502e6 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -614,13 +614,12 @@ func (p *player) handleMessageEvent(r routerHandle, e messageEvent) (actions []a case voteFiltered: ver := e.Proto.Version proto := config.Consensus[ver] - if !proto.DynamicFilterTimeout || !ef.(filteredEvent).StateUpdated { - err := ef.(filteredEvent).Err - return append(actions, ignoreAction(e, err)) - } v := e.Input.Vote - a := relayAction(e, protocol.AgreementVoteTag, v.u()) - return append(actions, a) + if proto.DynamicFilterTimeout && ef.(filteredEvent).StateUpdated && proposalUsedForCredentialHistory(p.Round, v.u()) { + return append(actions, relayAction(e, protocol.AgreementVoteTag, v.u())) + } + err := ef.(filteredEvent).Err + return append(actions, ignoreAction(e, err)) } if e.t() == votePresent { @@ -715,13 +714,12 @@ func (p *player) handleMessageEvent(r routerHandle, e messageEvent) (actions []a case voteFiltered: ver := e.Proto.Version proto := config.Consensus[ver] - if !proto.DynamicFilterTimeout || !ef.(filteredEvent).StateUpdated { - err := ef.(filteredEvent).Err - return append(actions, ignoreAction(e, err)) - } v := e.Input.Vote - actions = append(actions, relayAction(e, protocol.AgreementVoteTag, v.u())) - return actions + if proto.DynamicFilterTimeout && ef.(filteredEvent).StateUpdated && proposalUsedForCredentialHistory(p.Round, v.u()) { + return append(actions, relayAction(e, protocol.AgreementVoteTag, v.u())) + } + err := ef.(filteredEvent).Err + return append(actions, ignoreAction(e, err)) } if e.t() == votePresent { uv := e.Input.UnauthenticatedVote diff --git a/agreement/player_permutation_test.go b/agreement/player_permutation_test.go index 5e8e247a2b..d965edc01d 100644 --- a/agreement/player_permutation_test.go +++ b/agreement/player_permutation_test.go @@ -467,21 +467,18 @@ func verifyPermutationExpectedActions(t *testing.T, playerN int, eventN int, hel } case playerNextRound: switch eventN { - case proposeVotePresentEventSamePeriod: - if p > 0 { - expectIgnore(t, trace, "Player should ignore msg from past rounds, player: %v, event: %v", playerN, eventN) - } else { - requireActionCount(t, trace, 1, playerN, eventN) - expectVerify(t, trace, "Player should verify period 0 msg from past rounds, player: %v, event: %v", playerN, eventN) - } case proposeVoteVerifiedEventSamePeriod: + // This case should never happen -- player is on R+1 and the voteVerified event is for R. + // Player will not queue up a verifyVoteAction for this vote (without DynamicFilterTimeout enabled). + // We are asserting the relay behavior player currently implements, but it is not possible given current + // code -- you would have filtered the votePresent for this vote. if p > 0 { expectIgnore(t, trace, "Player should ignore msg from past rounds, player: %v, event: %v", playerN, eventN) } else { requireActionCount(t, trace, 1, playerN, eventN) expectRelay(t, trace, "Player should relay period 0 msg from past rounds, player: %v, event: %v", playerN, eventN) } - case softVoteVerifiedEventSamePeriod, softVotePresentEventSamePeriod, proposeVoteVerifiedEventNextPeriod, payloadPresentEvent, payloadVerifiedEvent, payloadVerifiedEventNoMessageHandle, bundleVerifiedEventSamePeriod, bundlePresentEventSamePeriod: + case proposeVotePresentEventSamePeriod, softVoteVerifiedEventSamePeriod, softVotePresentEventSamePeriod, proposeVoteVerifiedEventNextPeriod, payloadPresentEvent, payloadVerifiedEvent, payloadVerifiedEventNoMessageHandle, bundleVerifiedEventSamePeriod, bundlePresentEventSamePeriod: requireActionCount(t, trace, 1, playerN, eventN) expectIgnore(t, trace, "Player should ignore msg from past rounds, player: %v, event: %v", playerN, eventN) case softVoteVerifiedErrorEventSamePeriod, proposeVoteVerifiedErrorEventSamePeriod, bundleVerifiedErrorEvent: diff --git a/agreement/proposalManager.go b/agreement/proposalManager.go index 0eb90cfb30..67d612ad20 100644 --- a/agreement/proposalManager.go +++ b/agreement/proposalManager.go @@ -150,7 +150,7 @@ func (m *proposalManager) handleMessageEvent(r routerHandle, p player, e filtera v := e.Input.Vote err := proposalFresh(e.FreshnessData, v.u()) - if err != nil { + if err != nil && !proposalUsedForCredentialHistory(e.FreshnessData.PlayerRound, v.u()) { err := makeSerErrf("proposalManager: ignoring proposal-vote due to age: %v", err) return filteredEvent{T: voteFiltered, Err: err} } @@ -230,18 +230,21 @@ func (m *proposalManager) filterProposalVote(p player, r routerHandle, uv unauth return nil } -// voteFresh determines whether a proposal satisfies freshness rules. -func proposalFresh(freshData freshnessData, vote unauthenticatedVote) error { - if vote.R.Round < freshData.PlayerRound && freshData.PlayerRound <= vote.R.Round+credentialRoundLag && +func proposalUsedForCredentialHistory(curRound round, vote unauthenticatedVote) bool { + if vote.R.Round < curRound && curRound <= vote.R.Round+credentialRoundLag && vote.R.Period == 0 && vote.R.Step == propose { if dynamicFilterCredentialArrivalHistory > 0 { // continue processing old period 0 votes so we could track their // arrival times and inform setting the filter timeout dynamically. - return nil + return true } } + return false +} +// voteFresh determines whether a proposal satisfies freshness rules. +func proposalFresh(freshData freshnessData, vote unauthenticatedVote) error { switch vote.R.Round { case freshData.PlayerRound: if freshData.PlayerPeriod != 0 && freshData.PlayerPeriod-1 > vote.R.Period { From 59ecd4e4148e52e8a2b01c6fa6eb3d5268c44642 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Fri, 1 Sep 2023 17:12:47 -0400 Subject: [PATCH 190/232] use named constant --- agreement/player_permutation_test.go | 18 ------------------ agreement/player_test.go | 4 ++-- 2 files changed, 2 insertions(+), 20 deletions(-) diff --git a/agreement/player_permutation_test.go b/agreement/player_permutation_test.go index d965edc01d..5d3a23a871 100644 --- a/agreement/player_permutation_test.go +++ b/agreement/player_permutation_test.go @@ -357,24 +357,6 @@ func expectRelay(t *testing.T, trace ioTrace, errMsg string, playerN int, eventN }), errMsg, playerN, eventN) } -func expectVerify(t *testing.T, trace ioTrace, errMsg string, playerN int, eventN int) { - require.Truef(t, trace.ContainsFn(func(b event) bool { - if b.t() != wrappedAction { - return false - } - - wrapper := b.(wrappedActionEvent) - if wrapper.action.t() != verifyVote { - return false - } - act := wrapper.action.(cryptoAction) - if act.T == verifyVote { - return true - } - return false - }), errMsg, playerN, eventN) -} - func expectDisconnect(t *testing.T, trace ioTrace, errMsg string, playerN int, eventN int) { require.Truef(t, trace.ContainsFn(func(b event) bool { if b.t() != wrappedAction { diff --git a/agreement/player_test.go b/agreement/player_test.go index bc01486878..67349e5cf9 100644 --- a/agreement/player_test.go +++ b/agreement/player_test.go @@ -3322,7 +3322,7 @@ func TestPlayerRetainsEarlyReceivedValidatedAtOneSample(t *testing.T) { require.NotZero(t, dynamicFilterCredentialArrivalHistory) require.Equal(t, pWhite.lowestCredentialArrivals.writePtr, 1) require.False(t, pWhite.lowestCredentialArrivals.isFull()) - require.Equal(t, time.Duration(1), pWhite.lowestCredentialArrivals.history[0]) + require.Equal(t, pipelinedMessageTimestamp, pWhite.lowestCredentialArrivals.history[0]) } type fixedHistoricalClock struct{} @@ -3903,7 +3903,7 @@ func TestPlayerRetainsEarlyReceivedValidatedAtAVPPOneSample(t *testing.T) { require.NotNil(t, dynamicFilterCredentialArrivalHistory) require.Equal(t, pWhite.lowestCredentialArrivals.writePtr, 1) require.False(t, pWhite.lowestCredentialArrivals.isFull()) - require.Equal(t, time.Duration(1), pWhite.lowestCredentialArrivals.history[0]) + require.Equal(t, pipelinedMessageTimestamp, pWhite.lowestCredentialArrivals.history[0]) } // test that ReceivedAt and ValidateAt timing information are retained in From b8b128abfadb22bb3927862af622353b6329d704 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Fri, 1 Sep 2023 20:57:06 -0400 Subject: [PATCH 191/232] relay vs ignore msg issue --- agreement/events.go | 2 ++ agreement/msgp_gen.go | 27 +++++++++++++++++---- agreement/player.go | 4 ++-- agreement/player_permutation_test.go | 35 ++++++++++++++++++++++++++-- agreement/proposalManager.go | 11 ++++++++- agreement/proposalTracker.go | 2 +- 6 files changed, 70 insertions(+), 11 deletions(-) diff --git a/agreement/events.go b/agreement/events.go index 2c4cc21911..e92119715d 100644 --- a/agreement/events.go +++ b/agreement/events.go @@ -285,6 +285,8 @@ type messageEvent struct { Cancelled bool Proto ConsensusVersionView + + FreshOnlyForCredentialHistoryTracking bool } func (e messageEvent) t() eventType { diff --git a/agreement/msgp_gen.go b/agreement/msgp_gen.go index 5a5efa7ddc..d0e71536de 100644 --- a/agreement/msgp_gen.go +++ b/agreement/msgp_gen.go @@ -3241,9 +3241,9 @@ func MessageMaxSize() (s int) { // MarshalMsg implements msgp.Marshaler func (z *messageEvent) MarshalMsg(b []byte) (o []byte) { o = msgp.Require(b, z.Msgsize()) - // map header, size 7 + // map header, size 8 // string "Cancelled" - o = append(o, 0x87, 0xa9, 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x6c, 0x65, 0x64) + o = append(o, 0x88, 0xa9, 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x6c, 0x65, 0x64) o = msgp.AppendBool(o, (*z).Cancelled) // string "Err" o = append(o, 0xa3, 0x45, 0x72, 0x72) @@ -3252,6 +3252,9 @@ func (z *messageEvent) MarshalMsg(b []byte) (o []byte) { } else { o = msgp.AppendString(o, string(*(*z).Err)) } + // string "FreshOnlyForCredentialHistoryTracking" + o = append(o, 0xd9, 0x25, 0x46, 0x72, 0x65, 0x73, 0x68, 0x4f, 0x6e, 0x6c, 0x79, 0x46, 0x6f, 0x72, 0x43, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x54, 0x72, 0x61, 0x63, 0x6b, 0x69, 0x6e, 0x67) + o = msgp.AppendBool(o, (*z).FreshOnlyForCredentialHistoryTracking) // string "Input" o = append(o, 0xa5, 0x49, 0x6e, 0x70, 0x75, 0x74) o = (*z).Input.MarshalMsg(o) @@ -3378,6 +3381,14 @@ func (z *messageEvent) UnmarshalMsg(bts []byte) (o []byte, err error) { return } } + if zb0001 > 0 { + zb0001-- + (*z).FreshOnlyForCredentialHistoryTracking, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "FreshOnlyForCredentialHistoryTracking") + return + } + } if zb0001 > 0 { err = msgp.ErrTooManyArrayFields(zb0001) if err != nil { @@ -3473,6 +3484,12 @@ func (z *messageEvent) UnmarshalMsg(bts []byte) (o []byte, err error) { err = msgp.WrapError(err, "Proto") return } + case "FreshOnlyForCredentialHistoryTracking": + (*z).FreshOnlyForCredentialHistoryTracking, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "FreshOnlyForCredentialHistoryTracking") + return + } default: err = msgp.ErrNoField(string(field)) if err != nil { @@ -3505,13 +3522,13 @@ func (z *messageEvent) Msgsize() (s int) { } else { s += (*z).Tail.Msgsize() } - s += 10 + msgp.BoolSize + 6 + (*z).Proto.Msgsize() + s += 10 + msgp.BoolSize + 6 + (*z).Proto.Msgsize() + 39 + msgp.BoolSize return } // MsgIsZero returns whether this is a zero value func (z *messageEvent) MsgIsZero() bool { - return ((*z).T == 0) && ((*z).Input.MsgIsZero()) && ((*z).Err == nil) && ((*z).TaskIndex == 0) && ((*z).Tail == nil) && ((*z).Cancelled == false) && ((*z).Proto.MsgIsZero()) + return ((*z).T == 0) && ((*z).Input.MsgIsZero()) && ((*z).Err == nil) && ((*z).TaskIndex == 0) && ((*z).Tail == nil) && ((*z).Cancelled == false) && ((*z).Proto.MsgIsZero()) && ((*z).FreshOnlyForCredentialHistoryTracking == false) } // MaxSize returns a maximum valid message size for this message type @@ -3520,7 +3537,7 @@ func MessageEventMaxSize() (s int) { panic("Unable to determine max size: String type string(*z.Err) is unbounded") s += 10 + msgp.Uint64Size + 5 s += MessageEventMaxSize() - s += 10 + msgp.BoolSize + 6 + ConsensusVersionViewMaxSize() + s += 10 + msgp.BoolSize + 6 + ConsensusVersionViewMaxSize() + 39 + msgp.BoolSize return } diff --git a/agreement/player.go b/agreement/player.go index aaae4502e6..743ccca52f 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -615,7 +615,7 @@ func (p *player) handleMessageEvent(r routerHandle, e messageEvent) (actions []a ver := e.Proto.Version proto := config.Consensus[ver] v := e.Input.Vote - if proto.DynamicFilterTimeout && ef.(filteredEvent).StateUpdated && proposalUsedForCredentialHistory(p.Round, v.u()) { + if proto.DynamicFilterTimeout && ef.(filteredEvent).StateUpdated { return append(actions, relayAction(e, protocol.AgreementVoteTag, v.u())) } err := ef.(filteredEvent).Err @@ -715,7 +715,7 @@ func (p *player) handleMessageEvent(r routerHandle, e messageEvent) (actions []a ver := e.Proto.Version proto := config.Consensus[ver] v := e.Input.Vote - if proto.DynamicFilterTimeout && ef.(filteredEvent).StateUpdated && proposalUsedForCredentialHistory(p.Round, v.u()) { + if proto.DynamicFilterTimeout && ef.(filteredEvent).StateUpdated { return append(actions, relayAction(e, protocol.AgreementVoteTag, v.u())) } err := ef.(filteredEvent).Err diff --git a/agreement/player_permutation_test.go b/agreement/player_permutation_test.go index 5d3a23a871..7c0537b423 100644 --- a/agreement/player_permutation_test.go +++ b/agreement/player_permutation_test.go @@ -374,6 +374,24 @@ func expectDisconnect(t *testing.T, trace ioTrace, errMsg string, playerN int, e }), errMsg, playerN, eventN) } +func expectVerify(t *testing.T, trace ioTrace, errMsg string, playerN int, eventN int) { + require.Truef(t, trace.ContainsFn(func(b event) bool { + if b.t() != wrappedAction { + return false + } + + wrapper := b.(wrappedActionEvent) + if wrapper.action.t() != verifyVote { + return false + } + act := wrapper.action.(cryptoAction) + if act.T == verifyVote { + return true + } + return false + }), errMsg, playerN, eventN) +} + func requireActionCount(t *testing.T, trace ioTrace, expectedCount, playerN, eventN int) { require.Equalf(t, trace.countAction(), expectedCount, "Player should not emit extra actions, player: %v, event: %v", playerN, eventN) } @@ -456,11 +474,24 @@ func verifyPermutationExpectedActions(t *testing.T, playerN int, eventN int, hel // code -- you would have filtered the votePresent for this vote. if p > 0 { expectIgnore(t, trace, "Player should ignore msg from past rounds, player: %v, event: %v", playerN, eventN) - } else { + } else if config.Consensus[protocol.ConsensusCurrentVersion].DynamicFilterTimeout { requireActionCount(t, trace, 1, playerN, eventN) expectRelay(t, trace, "Player should relay period 0 msg from past rounds, player: %v, event: %v", playerN, eventN) + } else { + requireActionCount(t, trace, 1, playerN, eventN) + expectIgnore(t, trace, "Player should ignore msg from past rounds, player: %v, event: %v", playerN, eventN) + } + case proposeVotePresentEventSamePeriod: + if p > 0 { + expectIgnore(t, trace, "Player should ignore msg from past rounds, player: %v, event: %v", playerN, eventN) + } else if config.Consensus[protocol.ConsensusCurrentVersion].DynamicFilterTimeout { + requireActionCount(t, trace, 1, playerN, eventN) + expectVerify(t, trace, "Player should verify period 0 msg from past rounds, player: %v, event: %v", playerN, eventN) + } else { + requireActionCount(t, trace, 1, playerN, eventN) + expectIgnore(t, trace, "Player should ignore msg from past rounds, player: %v, event: %v", playerN, eventN) } - case proposeVotePresentEventSamePeriod, softVoteVerifiedEventSamePeriod, softVotePresentEventSamePeriod, proposeVoteVerifiedEventNextPeriod, payloadPresentEvent, payloadVerifiedEvent, payloadVerifiedEventNoMessageHandle, bundleVerifiedEventSamePeriod, bundlePresentEventSamePeriod: + case softVoteVerifiedEventSamePeriod, softVotePresentEventSamePeriod, proposeVoteVerifiedEventNextPeriod, payloadPresentEvent, payloadVerifiedEvent, payloadVerifiedEventNoMessageHandle, bundleVerifiedEventSamePeriod, bundlePresentEventSamePeriod: requireActionCount(t, trace, 1, playerN, eventN) expectIgnore(t, trace, "Player should ignore msg from past rounds, player: %v, event: %v", playerN, eventN) case softVoteVerifiedErrorEventSamePeriod, proposeVoteVerifiedErrorEventSamePeriod, bundleVerifiedErrorEvent: diff --git a/agreement/proposalManager.go b/agreement/proposalManager.go index 67d612ad20..56274d504f 100644 --- a/agreement/proposalManager.go +++ b/agreement/proposalManager.go @@ -161,6 +161,10 @@ func (m *proposalManager) handleMessageEvent(r routerHandle, p player, e filtera r.t.timeRPlus1().RecVoteReceived(v) } + if err != nil { + e.messageEvent.FreshOnlyForCredentialHistoryTracking = true + } + return r.dispatch(p, e.messageEvent, proposalMachineRound, v.R.Round, v.R.Period, 0) case payloadPresent: @@ -218,7 +222,7 @@ func (m *proposalManager) handleMessageEvent(r routerHandle, p player, e filtera // filterVote filters a vote, checking if it is both fresh and not a duplicate. func (m *proposalManager) filterProposalVote(p player, r routerHandle, uv unauthenticatedVote, freshData freshnessData) error { err := proposalFresh(freshData, uv) - if err != nil { + if err != nil && !proposalUsedForCredentialHistory(freshData.PlayerRound, uv) { return fmt.Errorf("proposalManager: filtered proposal-vote due to age: %v", err) } @@ -245,6 +249,11 @@ func proposalUsedForCredentialHistory(curRound round, vote unauthenticatedVote) // voteFresh determines whether a proposal satisfies freshness rules. func proposalFresh(freshData freshnessData, vote unauthenticatedVote) error { + + // if proposalUsedForCredentialHistory(freshData.PlayerRound, vote) { + // return nil + // } + switch vote.R.Round { case freshData.PlayerRound: if freshData.PlayerPeriod != 0 && freshData.PlayerPeriod-1 > vote.R.Period { diff --git a/agreement/proposalTracker.go b/agreement/proposalTracker.go index a9e0380bf1..332d90f325 100644 --- a/agreement/proposalTracker.go +++ b/agreement/proposalTracker.go @@ -168,7 +168,7 @@ func (t *proposalTracker) handle(r routerHandle, p player, e event) event { t.Freezer, updated, err = t.Freezer.accept(v) if err != nil { err := errProposalTrackerPS{Sub: err} - return filteredEvent{T: voteFiltered, StateUpdated: updated, Err: makeSerErr(err)} + return filteredEvent{T: voteFiltered, StateUpdated: updated && e.FreshOnlyForCredentialHistoryTracking, Err: makeSerErr(err)} } return proposalAcceptedEvent{ From f6ed967b2b3e07605b0d888703ef5a3c840e5c9f Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Sat, 2 Sep 2023 14:34:36 -0400 Subject: [PATCH 192/232] checkpoint: relay rules --- agreement/events.go | 7 ++++-- agreement/msgp_gen.go | 27 +++++------------------ agreement/player.go | 15 +++++-------- agreement/proposalManager.go | 36 ++++++++++++++++++++++--------- agreement/proposalManager_test.go | 25 ++++++++++----------- agreement/proposalTracker.go | 2 +- 6 files changed, 55 insertions(+), 57 deletions(-) diff --git a/agreement/events.go b/agreement/events.go index e92119715d..279c60b5f0 100644 --- a/agreement/events.go +++ b/agreement/events.go @@ -285,8 +285,6 @@ type messageEvent struct { Cancelled bool Proto ConsensusVersionView - - FreshOnlyForCredentialHistoryTracking bool } func (e messageEvent) t() eventType { @@ -606,6 +604,11 @@ type filteredEvent struct { // the state machine StateUpdated bool + // ContinueProcessingVoteForCredentialTracking indicates whether its a vote + // present that we should continue processing only for tracking its + // credential arrival time. + ContinueProcessingVoteForCredentialTracking bool + // Err is the reason cryptographic verification failed and is set for // events {proposal,vote,bundle}Malformed. Err *serializableError diff --git a/agreement/msgp_gen.go b/agreement/msgp_gen.go index d0e71536de..5a5efa7ddc 100644 --- a/agreement/msgp_gen.go +++ b/agreement/msgp_gen.go @@ -3241,9 +3241,9 @@ func MessageMaxSize() (s int) { // MarshalMsg implements msgp.Marshaler func (z *messageEvent) MarshalMsg(b []byte) (o []byte) { o = msgp.Require(b, z.Msgsize()) - // map header, size 8 + // map header, size 7 // string "Cancelled" - o = append(o, 0x88, 0xa9, 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x6c, 0x65, 0x64) + o = append(o, 0x87, 0xa9, 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x6c, 0x65, 0x64) o = msgp.AppendBool(o, (*z).Cancelled) // string "Err" o = append(o, 0xa3, 0x45, 0x72, 0x72) @@ -3252,9 +3252,6 @@ func (z *messageEvent) MarshalMsg(b []byte) (o []byte) { } else { o = msgp.AppendString(o, string(*(*z).Err)) } - // string "FreshOnlyForCredentialHistoryTracking" - o = append(o, 0xd9, 0x25, 0x46, 0x72, 0x65, 0x73, 0x68, 0x4f, 0x6e, 0x6c, 0x79, 0x46, 0x6f, 0x72, 0x43, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x54, 0x72, 0x61, 0x63, 0x6b, 0x69, 0x6e, 0x67) - o = msgp.AppendBool(o, (*z).FreshOnlyForCredentialHistoryTracking) // string "Input" o = append(o, 0xa5, 0x49, 0x6e, 0x70, 0x75, 0x74) o = (*z).Input.MarshalMsg(o) @@ -3381,14 +3378,6 @@ func (z *messageEvent) UnmarshalMsg(bts []byte) (o []byte, err error) { return } } - if zb0001 > 0 { - zb0001-- - (*z).FreshOnlyForCredentialHistoryTracking, bts, err = msgp.ReadBoolBytes(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "FreshOnlyForCredentialHistoryTracking") - return - } - } if zb0001 > 0 { err = msgp.ErrTooManyArrayFields(zb0001) if err != nil { @@ -3484,12 +3473,6 @@ func (z *messageEvent) UnmarshalMsg(bts []byte) (o []byte, err error) { err = msgp.WrapError(err, "Proto") return } - case "FreshOnlyForCredentialHistoryTracking": - (*z).FreshOnlyForCredentialHistoryTracking, bts, err = msgp.ReadBoolBytes(bts) - if err != nil { - err = msgp.WrapError(err, "FreshOnlyForCredentialHistoryTracking") - return - } default: err = msgp.ErrNoField(string(field)) if err != nil { @@ -3522,13 +3505,13 @@ func (z *messageEvent) Msgsize() (s int) { } else { s += (*z).Tail.Msgsize() } - s += 10 + msgp.BoolSize + 6 + (*z).Proto.Msgsize() + 39 + msgp.BoolSize + s += 10 + msgp.BoolSize + 6 + (*z).Proto.Msgsize() return } // MsgIsZero returns whether this is a zero value func (z *messageEvent) MsgIsZero() bool { - return ((*z).T == 0) && ((*z).Input.MsgIsZero()) && ((*z).Err == nil) && ((*z).TaskIndex == 0) && ((*z).Tail == nil) && ((*z).Cancelled == false) && ((*z).Proto.MsgIsZero()) && ((*z).FreshOnlyForCredentialHistoryTracking == false) + return ((*z).T == 0) && ((*z).Input.MsgIsZero()) && ((*z).Err == nil) && ((*z).TaskIndex == 0) && ((*z).Tail == nil) && ((*z).Cancelled == false) && ((*z).Proto.MsgIsZero()) } // MaxSize returns a maximum valid message size for this message type @@ -3537,7 +3520,7 @@ func MessageEventMaxSize() (s int) { panic("Unable to determine max size: String type string(*z.Err) is unbounded") s += 10 + msgp.Uint64Size + 5 s += MessageEventMaxSize() - s += 10 + msgp.BoolSize + 6 + ConsensusVersionViewMaxSize() + 39 + msgp.BoolSize + s += 10 + msgp.BoolSize + 6 + ConsensusVersionViewMaxSize() return } diff --git a/agreement/player.go b/agreement/player.go index 743ccca52f..9d733ba62b 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -614,12 +614,13 @@ func (p *player) handleMessageEvent(r routerHandle, e messageEvent) (actions []a case voteFiltered: ver := e.Proto.Version proto := config.Consensus[ver] - v := e.Input.Vote - if proto.DynamicFilterTimeout && ef.(filteredEvent).StateUpdated { + if !proto.DynamicFilterTimeout || !ef.(filteredEvent).ContinueProcessingVoteForCredentialTracking { + err := ef.(filteredEvent).Err + return append(actions, ignoreAction(e, err)) + } else if proto.DynamicFilterTimeout && ef.(filteredEvent).StateUpdated { + v := e.Input.Vote return append(actions, relayAction(e, protocol.AgreementVoteTag, v.u())) } - err := ef.(filteredEvent).Err - return append(actions, ignoreAction(e, err)) } if e.t() == votePresent { @@ -712,12 +713,6 @@ func (p *player) handleMessageEvent(r routerHandle, e messageEvent) (actions []a err := makeSerErrf("rejected message since it was invalid: %v", ef.(filteredEvent).Err) return append(actions, disconnectAction(e, err)) case voteFiltered: - ver := e.Proto.Version - proto := config.Consensus[ver] - v := e.Input.Vote - if proto.DynamicFilterTimeout && ef.(filteredEvent).StateUpdated { - return append(actions, relayAction(e, protocol.AgreementVoteTag, v.u())) - } err := ef.(filteredEvent).Err return append(actions, ignoreAction(e, err)) } diff --git a/agreement/proposalManager.go b/agreement/proposalManager.go index 56274d504f..1f069f3d8a 100644 --- a/agreement/proposalManager.go +++ b/agreement/proposalManager.go @@ -133,8 +133,10 @@ func (m *proposalManager) handleMessageEvent(r routerHandle, p player, e filtera switch e.t() { case votePresent: err := m.filterProposalVote(p, r, e.Input.UnauthenticatedVote, e.FreshnessData) + // don't filter votes we get if they may be used for tracking credential arrival times if err != nil { - return filteredEvent{T: voteFiltered, Err: makeSerErr(err)} + credTrackingProcessing := proposalUsedForCredentialHistory(e.FreshnessData.PlayerRound, e.Input.UnauthenticatedVote) + return filteredEvent{T: voteFiltered, Err: makeSerErr(err), ContinueProcessingVoteForCredentialTracking: credTrackingProcessing} } return emptyEvent{} @@ -150,9 +152,14 @@ func (m *proposalManager) handleMessageEvent(r routerHandle, p player, e filtera v := e.Input.Vote err := proposalFresh(e.FreshnessData, v.u()) - if err != nil && !proposalUsedForCredentialHistory(e.FreshnessData.PlayerRound, v.u()) { - err := makeSerErrf("proposalManager: ignoring proposal-vote due to age: %v", err) - return filteredEvent{T: voteFiltered, Err: err} + keepForCredentialTracking := false + if err != nil { + // if we should keep processing this credential message only to record its timestamp, we continue + keepForCredentialTracking = proposalUsedForCredentialHistory(e.FreshnessData.PlayerRound, v.u()) + if !keepForCredentialTracking { + err := makeSerErrf("proposalManager: ignoring proposal-vote due to age: %v", err) + return filteredEvent{T: voteFiltered, Err: err} + } } if v.R.Round == p.Round { @@ -161,11 +168,20 @@ func (m *proposalManager) handleMessageEvent(r routerHandle, p player, e filtera r.t.timeRPlus1().RecVoteReceived(v) } - if err != nil { - e.messageEvent.FreshOnlyForCredentialHistoryTracking = true - } + e := r.dispatch(p, e.messageEvent, proposalMachineRound, v.R.Round, v.R.Period, 0) - return r.dispatch(p, e.messageEvent, proposalMachineRound, v.R.Round, v.R.Period, 0) + if keepForCredentialTracking { + // we only continued processing this vote to see whether it updates the credential arrival time + err := makeSerErrf("proposalManager: ignoring proposal-vote due to age: %v", err) + if e.t() == voteFiltered { + // indicate whether it updated + return filteredEvent{T: voteFiltered, Err: err, StateUpdated: e.(filteredEvent).StateUpdated} + } + // the proposalMachineRound didn't filter the vote, so it must have had a better credential, + // indicate that it did cause updating its state + return filteredEvent{T: voteFiltered, Err: err, StateUpdated: true} + } + return e case payloadPresent: propRound := e.Input.UnauthenticatedProposal.Round() @@ -222,13 +238,13 @@ func (m *proposalManager) handleMessageEvent(r routerHandle, p player, e filtera // filterVote filters a vote, checking if it is both fresh and not a duplicate. func (m *proposalManager) filterProposalVote(p player, r routerHandle, uv unauthenticatedVote, freshData freshnessData) error { err := proposalFresh(freshData, uv) - if err != nil && !proposalUsedForCredentialHistory(freshData.PlayerRound, uv) { + if err != nil { return fmt.Errorf("proposalManager: filtered proposal-vote due to age: %v", err) } qe := voteFilterRequestEvent{RawVote: uv.R} sawVote := r.dispatch(p, qe, proposalMachinePeriod, uv.R.Round, uv.R.Period, 0) - if sawVote.t() == voteFiltered { + if sawVote.t() == voteFiltered && !sawVote.(filteredEvent).StateUpdated { return fmt.Errorf("proposalManager: filtered proposal-vote: sender %v had already sent a vote in round %d period %d", uv.R.Sender, uv.R.Round, uv.R.Period) } return nil diff --git a/agreement/proposalManager_test.go b/agreement/proposalManager_test.go index 629f88eacf..b8ab96c5df 100644 --- a/agreement/proposalManager_test.go +++ b/agreement/proposalManager_test.go @@ -238,7 +238,7 @@ func TestProposalFreshAdjacentPeriods(t *testing.T) { }, }, } - b.AddInOutPair(inMsg, emptyEvent{}) + //b.AddInOutPair(inMsg, emptyEvent{}) // vote in same round p + 1 should also be fine pV = helper.MakeRandomProposalValue() @@ -252,7 +252,7 @@ func TestProposalFreshAdjacentPeriods(t *testing.T) { }, }, } - b.AddInOutPair(inMsg, emptyEvent{}) + //b.AddInOutPair(inMsg, emptyEvent{}) // vote in same round p+ 2 should be filtered pV = helper.MakeRandomProposalValue() @@ -266,7 +266,7 @@ func TestProposalFreshAdjacentPeriods(t *testing.T) { }, }, } - b.AddInOutPair(inMsg, filteredEvent{T: voteFiltered}) + //b.AddInOutPair(inMsg, filteredEvent{T: voteFiltered}) // vote in same round p - 2 should be filtered pV = helper.MakeRandomProposalValue() @@ -280,7 +280,7 @@ func TestProposalFreshAdjacentPeriods(t *testing.T) { }, }, } - b.AddInOutPair(inMsg, filteredEvent{T: voteFiltered}) + //b.AddInOutPair(inMsg, filteredEvent{T: voteFiltered}) // vote in r + 1 should be filtered unless period 0 pV = helper.MakeRandomProposalValue() @@ -294,7 +294,7 @@ func TestProposalFreshAdjacentPeriods(t *testing.T) { }, }, } - b.AddInOutPair(inMsg, filteredEvent{T: voteFiltered}) + //b.AddInOutPair(inMsg, filteredEvent{T: voteFiltered}) pV = helper.MakeRandomProposalValue() uv = helper.MakeUnauthenticatedVote(t, 0, r+1, 0, s, *pV) @@ -307,7 +307,7 @@ func TestProposalFreshAdjacentPeriods(t *testing.T) { }, }, } - b.AddInOutPair(inMsg, emptyEvent{}) + //b.AddInOutPair(inMsg, emptyEvent{}) // vote > r + 1 should be filtered pV = helper.MakeRandomProposalValue() @@ -321,9 +321,10 @@ func TestProposalFreshAdjacentPeriods(t *testing.T) { }, }, } - b.AddInOutPair(inMsg, filteredEvent{T: voteFiltered}) + //b.AddInOutPair(inMsg, filteredEvent{T: voteFiltered}) - // vote from credentialRoundLag rounds ago and period 0 should be fine + // vote from credentialRoundLag rounds ago and period 0 should continue + // processing only for the purpose of tracking credentials. pV = helper.MakeRandomProposalValue() uv = helper.MakeUnauthenticatedVote(t, 0, r-credentialRoundLag, 0, 0, *pV) inMsg = filterableMessageEvent{ @@ -335,7 +336,7 @@ func TestProposalFreshAdjacentPeriods(t *testing.T) { }, }, } - b.AddInOutPair(inMsg, emptyEvent{}) + b.AddInOutPair(inMsg, filteredEvent{T: voteFiltered, ContinueProcessingVoteForCredentialTracking: true}) // vote from credentialRoundLag ago and period > 0 should be filtered pV = helper.MakeRandomProposalValue() @@ -349,7 +350,7 @@ func TestProposalFreshAdjacentPeriods(t *testing.T) { }, }, } - b.AddInOutPair(inMsg, filteredEvent{T: voteFiltered}) + //b.AddInOutPair(inMsg, filteredEvent{T: voteFiltered}) // vote older than credentialRoundLag should be rejected even if period 0 pV = helper.MakeRandomProposalValue() @@ -363,7 +364,7 @@ func TestProposalFreshAdjacentPeriods(t *testing.T) { }, }, } - b.AddInOutPair(inMsg, filteredEvent{T: voteFiltered}) + //b.AddInOutPair(inMsg, filteredEvent{T: voteFiltered}) // vote older than credentialRoundLag should be rejected on any period pV = helper.MakeRandomProposalValue() @@ -377,7 +378,7 @@ func TestProposalFreshAdjacentPeriods(t *testing.T) { }, }, } - b.AddInOutPair(inMsg, filteredEvent{T: voteFiltered}) + //b.AddInOutPair(inMsg, filteredEvent{T: voteFiltered}) res, err := b.Build().Validate(pM) require.NoError(t, err) diff --git a/agreement/proposalTracker.go b/agreement/proposalTracker.go index 332d90f325..a9e0380bf1 100644 --- a/agreement/proposalTracker.go +++ b/agreement/proposalTracker.go @@ -168,7 +168,7 @@ func (t *proposalTracker) handle(r routerHandle, p player, e event) event { t.Freezer, updated, err = t.Freezer.accept(v) if err != nil { err := errProposalTrackerPS{Sub: err} - return filteredEvent{T: voteFiltered, StateUpdated: updated && e.FreshOnlyForCredentialHistoryTracking, Err: makeSerErr(err)} + return filteredEvent{T: voteFiltered, StateUpdated: updated, Err: makeSerErr(err)} } return proposalAcceptedEvent{ From b3b07ff6e2b04c4006e89088ef3f119eb31953e5 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Sat, 2 Sep 2023 14:44:46 -0400 Subject: [PATCH 193/232] relay action order --- agreement/player.go | 8 ++++++-- config/consensus.go | 2 ++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/agreement/player.go b/agreement/player.go index 9d733ba62b..f173ad3f56 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -614,12 +614,16 @@ func (p *player) handleMessageEvent(r routerHandle, e messageEvent) (actions []a case voteFiltered: ver := e.Proto.Version proto := config.Consensus[ver] - if !proto.DynamicFilterTimeout || !ef.(filteredEvent).ContinueProcessingVoteForCredentialTracking { + if !proto.DynamicFilterTimeout { err := ef.(filteredEvent).Err return append(actions, ignoreAction(e, err)) - } else if proto.DynamicFilterTimeout && ef.(filteredEvent).StateUpdated { + } + if ef.(filteredEvent).StateUpdated { v := e.Input.Vote return append(actions, relayAction(e, protocol.AgreementVoteTag, v.u())) + } else if !ef.(filteredEvent).ContinueProcessingVoteForCredentialTracking { + err := ef.(filteredEvent).Err + return append(actions, ignoreAction(e, err)) } } diff --git a/config/consensus.go b/config/consensus.go index 190e5ce16a..0827e910ef 100644 --- a/config/consensus.go +++ b/config/consensus.go @@ -1353,6 +1353,8 @@ func initConsensusProtocols() { v38.AgreementFilterTimeoutPeriod0 = 3000 * time.Millisecond + v38.DynamicFilterTimeout = true + Consensus[protocol.ConsensusV38] = v38 // v37 can be upgraded to v38, with an update delay of 12h: From 58db3f8dda627d21cc57c0c506af22d41554b5be Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Sat, 2 Sep 2023 14:45:39 -0400 Subject: [PATCH 194/232] x --- config/consensus.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/config/consensus.go b/config/consensus.go index 0827e910ef..190e5ce16a 100644 --- a/config/consensus.go +++ b/config/consensus.go @@ -1353,8 +1353,6 @@ func initConsensusProtocols() { v38.AgreementFilterTimeoutPeriod0 = 3000 * time.Millisecond - v38.DynamicFilterTimeout = true - Consensus[protocol.ConsensusV38] = v38 // v37 can be upgraded to v38, with an update delay of 12h: From 7038dfa5350c46153aab115896de1f281f6d28eb Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Sat, 2 Sep 2023 15:17:08 -0400 Subject: [PATCH 195/232] fix test by enabling dynamic filter timeout feature --- agreement/player.go | 7 +++++++ agreement/player_test.go | 21 ++++++++++++++++----- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/agreement/player.go b/agreement/player.go index f173ad3f56..b7536680bc 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -615,13 +615,20 @@ func (p *player) handleMessageEvent(r routerHandle, e messageEvent) (actions []a ver := e.Proto.Version proto := config.Consensus[ver] if !proto.DynamicFilterTimeout { + // Dynamic filter timeout feature disabled, so we filter the + // message as usual (keeping earlier behavior) err := ef.(filteredEvent).Err return append(actions, ignoreAction(e, err)) } if ef.(filteredEvent).StateUpdated { + // Dynamic filter timeout feature enabled, and current message + // updated the best credential arrival time v := e.Input.Vote return append(actions, relayAction(e, protocol.AgreementVoteTag, v.u())) } else if !ef.(filteredEvent).ContinueProcessingVoteForCredentialTracking { + // Dynamic filter timeout feature enabled, and current message + // may update the best credential arrival time, so we should + // continue processing. err := ef.(filteredEvent).Err return append(actions, ignoreAction(e, err)) } diff --git a/agreement/player_test.go b/agreement/player_test.go index 67349e5cf9..1053f805a0 100644 --- a/agreement/player_test.go +++ b/agreement/player_test.go @@ -2163,7 +2163,6 @@ func TestPlayerRePropagatesProposalPayload(t *testing.T) { require.NoError(t, err) require.NoError(t, panicErr) relayPayloadEvent = ev(networkAction{T: broadcast, Tag: protocol.ProposalPayloadTag, CompoundMessage: compoundMessage{Proposal: payloadNext.u()}}) - fmt.Println(relayPayloadEvent) require.Truef(t, pM.getTrace().Contains(relayPayloadEvent), "Player should relay staged payload over pinned payload on resynch") } @@ -3739,6 +3738,10 @@ func TestPlayerRetainsReceivedValidatedAtPPForHistoryWindow(t *testing.T) { func TestPlayerRetainsReceivedValidatedAtAVPPOneSample(t *testing.T) { partitiontest.PartitionTest(t) + // create a protocol version where dynamic lambda is enabled + version, _, configCleanup := createDynamicFilterConfig() + defer configCleanup() + const r = round(20239) const p = period(0) pWhite, pM, helper := setupP(t, r-1, p, soft) @@ -3761,14 +3764,14 @@ func TestPlayerRetainsReceivedValidatedAtAVPPOneSample(t *testing.T) { require.NoError(t, err) require.NoError(t, panicErr) - assertCorrectReceivedAtSet(t, pWhite, pM, helper, r, p, pP, pV, m, protocol.ConsensusFuture, time.Second) + assertCorrectReceivedAtSet(t, pWhite, pM, helper, r, p, pP, pV, m, version, time.Second) require.False(t, pWhite.lowestCredentialArrivals.isFull()) require.Equal(t, pWhite.lowestCredentialArrivals.writePtr, 0) // send votePresent message (mimicking the first AV message validating) vVote = helper.MakeVerifiedVote(t, 0, r-credentialRoundLag, p, propose, *pV) unverifiedVoteMsg := message{UnauthenticatedVote: vVote.u()} - inMsg = messageEvent{T: votePresent, Input: unverifiedVoteMsg} + inMsg = messageEvent{T: votePresent, Input: unverifiedVoteMsg, Proto: ConsensusVersionView{Version: version}} err, panicErr = pM.transition(inMsg) require.NoError(t, err) require.NoError(t, panicErr) @@ -3853,10 +3856,14 @@ func TestPlayerRetainsEarlyReceivedValidatedAtAVPPOneSample(t *testing.T) { require.False(t, pWhite.lowestCredentialArrivals.isFull()) require.Equal(t, pWhite.lowestCredentialArrivals.writePtr, 0) + // create a protocol version where dynamic filter is enabled + version, _, configCleanup := createDynamicFilterConfig() + defer configCleanup() + // send votePresent message (mimicking the first AV message validating) vVote = helper.MakeVerifiedVote(t, 0, r-credentialRoundLag, p, propose, *pV) unverifiedVoteMsg := message{UnauthenticatedVote: vVote.u()} - inMsg = messageEvent{T: votePresent, Input: unverifiedVoteMsg} + inMsg = messageEvent{T: votePresent, Input: unverifiedVoteMsg, Proto: ConsensusVersionView{Version: version}} err, panicErr = pM.transition(inMsg) require.NoError(t, err) require.NoError(t, panicErr) @@ -3941,10 +3948,14 @@ func TestPlayerRetainsLateReceivedValidatedAtAVPPOneSample(t *testing.T) { require.False(t, pWhite.lowestCredentialArrivals.isFull()) require.Equal(t, pWhite.lowestCredentialArrivals.writePtr, 0) + // create a protocol version where dynamic filter is enabled + version, _, configCleanup := createDynamicFilterConfig() + defer configCleanup() + // send votePresent message (mimicking the first AV message validating) vVote = helper.MakeVerifiedVote(t, 0, r-credentialRoundLag, p, propose, *pV) unverifiedVoteMsg := message{UnauthenticatedVote: vVote.u()} - inMsg = messageEvent{T: votePresent, Input: unverifiedVoteMsg} + inMsg = messageEvent{T: votePresent, Input: unverifiedVoteMsg, Proto: ConsensusVersionView{Version: version}} err, panicErr = pM.transition(inMsg) require.NoError(t, err) require.NoError(t, panicErr) From 44d862b7da0d4e9513f9feb834c14ad8d5f68f7f Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Sat, 2 Sep 2023 15:29:18 -0400 Subject: [PATCH 196/232] explicitly enable dyamic filter timeout for permutation test --- agreement/player_permutation_test.go | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/agreement/player_permutation_test.go b/agreement/player_permutation_test.go index 7c0537b423..bd023e7ce0 100644 --- a/agreement/player_permutation_test.go +++ b/agreement/player_permutation_test.go @@ -400,7 +400,7 @@ func requireTraceContains(t *testing.T, trace ioTrace, expected event, playerN, require.Truef(t, trace.Contains(expected), "Player should emit action, player: %v, event: %v", playerN, eventN) } -func verifyPermutationExpectedActions(t *testing.T, playerN int, eventN int, helper *voteMakerHelper, trace ioTrace) { +func verifyPermutationExpectedActions(t *testing.T, playerN int, eventN int, helper *voteMakerHelper, trace ioTrace, dynamicFilterTimeoutEnabled bool) { const r = round(209) const p = period(0) var payload = makeRandomProposalPayload(r) @@ -474,7 +474,7 @@ func verifyPermutationExpectedActions(t *testing.T, playerN int, eventN int, hel // code -- you would have filtered the votePresent for this vote. if p > 0 { expectIgnore(t, trace, "Player should ignore msg from past rounds, player: %v, event: %v", playerN, eventN) - } else if config.Consensus[protocol.ConsensusCurrentVersion].DynamicFilterTimeout { + } else if dynamicFilterTimeoutEnabled { requireActionCount(t, trace, 1, playerN, eventN) expectRelay(t, trace, "Player should relay period 0 msg from past rounds, player: %v, event: %v", playerN, eventN) } else { @@ -484,7 +484,7 @@ func verifyPermutationExpectedActions(t *testing.T, playerN int, eventN int, hel case proposeVotePresentEventSamePeriod: if p > 0 { expectIgnore(t, trace, "Player should ignore msg from past rounds, player: %v, event: %v", playerN, eventN) - } else if config.Consensus[protocol.ConsensusCurrentVersion].DynamicFilterTimeout { + } else if dynamicFilterTimeoutEnabled { requireActionCount(t, trace, 1, playerN, eventN) expectVerify(t, trace, "Player should verify period 0 msg from past rounds, player: %v, event: %v", playerN, eventN) } else { @@ -805,18 +805,30 @@ func verifyPermutationExpectedActions(t *testing.T, playerN int, eventN int, hel // Generates a set of player states, router states, and messageEvents and tests all permutations of them func TestPlayerPermutation(t *testing.T) { partitiontest.PartitionTest(t) + // check with/without setting dynamic filter timeout + playerPermutationCheck(t, false) + playerPermutationCheck(t, true) +} + +func playerPermutationCheck(t *testing.T, enableDynamicFilterTimeout bool) { + // create a protocol version where dynamic filter is enabled + version, _, configCleanup := createDynamicFilterConfig() + defer configCleanup() for i := 0; i < 7; i++ { for j := 0; j < 14; j++ { _, pMachine, helper := getPlayerPermutation(t, i) inMsg := getMessageEventPermutation(t, j, helper) + if enableDynamicFilterTimeout { + inMsg.Proto = ConsensusVersionView{Version: version} + } err, panicErr := pMachine.transition(inMsg) fmt.Println(pMachine.getTrace().events) fmt.Println("") require.NoErrorf(t, err, "player: %v, event: %v", i, j) require.NoErrorf(t, panicErr, "player: %v, event: %v", i, j) - verifyPermutationExpectedActions(t, i, j, helper, pMachine.getTrace()) + verifyPermutationExpectedActions(t, i, j, helper, pMachine.getTrace(), enableDynamicFilterTimeout) } } } From 707834016ea0e4cabb57a9fdf196c6d6fa590db5 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Sat, 2 Sep 2023 15:41:38 -0400 Subject: [PATCH 197/232] code cleanup --- agreement/proposalManager.go | 9 ++------- agreement/proposalManager_test.go | 20 ++++++++++---------- 2 files changed, 12 insertions(+), 17 deletions(-) diff --git a/agreement/proposalManager.go b/agreement/proposalManager.go index 1f069f3d8a..b0286bd6bd 100644 --- a/agreement/proposalManager.go +++ b/agreement/proposalManager.go @@ -133,8 +133,8 @@ func (m *proposalManager) handleMessageEvent(r routerHandle, p player, e filtera switch e.t() { case votePresent: err := m.filterProposalVote(p, r, e.Input.UnauthenticatedVote, e.FreshnessData) - // don't filter votes we get if they may be used for tracking credential arrival times if err != nil { + // mark filtered votes that may still update the best credential arrival time credTrackingProcessing := proposalUsedForCredentialHistory(e.FreshnessData.PlayerRound, e.Input.UnauthenticatedVote) return filteredEvent{T: voteFiltered, Err: makeSerErr(err), ContinueProcessingVoteForCredentialTracking: credTrackingProcessing} } @@ -244,7 +244,7 @@ func (m *proposalManager) filterProposalVote(p player, r routerHandle, uv unauth qe := voteFilterRequestEvent{RawVote: uv.R} sawVote := r.dispatch(p, qe, proposalMachinePeriod, uv.R.Round, uv.R.Period, 0) - if sawVote.t() == voteFiltered && !sawVote.(filteredEvent).StateUpdated { + if sawVote.t() == voteFiltered { return fmt.Errorf("proposalManager: filtered proposal-vote: sender %v had already sent a vote in round %d period %d", uv.R.Sender, uv.R.Round, uv.R.Period) } return nil @@ -265,11 +265,6 @@ func proposalUsedForCredentialHistory(curRound round, vote unauthenticatedVote) // voteFresh determines whether a proposal satisfies freshness rules. func proposalFresh(freshData freshnessData, vote unauthenticatedVote) error { - - // if proposalUsedForCredentialHistory(freshData.PlayerRound, vote) { - // return nil - // } - switch vote.R.Round { case freshData.PlayerRound: if freshData.PlayerPeriod != 0 && freshData.PlayerPeriod-1 > vote.R.Period { diff --git a/agreement/proposalManager_test.go b/agreement/proposalManager_test.go index b8ab96c5df..770677d456 100644 --- a/agreement/proposalManager_test.go +++ b/agreement/proposalManager_test.go @@ -238,7 +238,7 @@ func TestProposalFreshAdjacentPeriods(t *testing.T) { }, }, } - //b.AddInOutPair(inMsg, emptyEvent{}) + b.AddInOutPair(inMsg, emptyEvent{}) // vote in same round p + 1 should also be fine pV = helper.MakeRandomProposalValue() @@ -252,7 +252,7 @@ func TestProposalFreshAdjacentPeriods(t *testing.T) { }, }, } - //b.AddInOutPair(inMsg, emptyEvent{}) + b.AddInOutPair(inMsg, emptyEvent{}) // vote in same round p+ 2 should be filtered pV = helper.MakeRandomProposalValue() @@ -266,7 +266,7 @@ func TestProposalFreshAdjacentPeriods(t *testing.T) { }, }, } - //b.AddInOutPair(inMsg, filteredEvent{T: voteFiltered}) + b.AddInOutPair(inMsg, filteredEvent{T: voteFiltered}) // vote in same round p - 2 should be filtered pV = helper.MakeRandomProposalValue() @@ -280,7 +280,7 @@ func TestProposalFreshAdjacentPeriods(t *testing.T) { }, }, } - //b.AddInOutPair(inMsg, filteredEvent{T: voteFiltered}) + b.AddInOutPair(inMsg, filteredEvent{T: voteFiltered}) // vote in r + 1 should be filtered unless period 0 pV = helper.MakeRandomProposalValue() @@ -294,7 +294,7 @@ func TestProposalFreshAdjacentPeriods(t *testing.T) { }, }, } - //b.AddInOutPair(inMsg, filteredEvent{T: voteFiltered}) + b.AddInOutPair(inMsg, filteredEvent{T: voteFiltered}) pV = helper.MakeRandomProposalValue() uv = helper.MakeUnauthenticatedVote(t, 0, r+1, 0, s, *pV) @@ -307,7 +307,7 @@ func TestProposalFreshAdjacentPeriods(t *testing.T) { }, }, } - //b.AddInOutPair(inMsg, emptyEvent{}) + b.AddInOutPair(inMsg, emptyEvent{}) // vote > r + 1 should be filtered pV = helper.MakeRandomProposalValue() @@ -321,7 +321,7 @@ func TestProposalFreshAdjacentPeriods(t *testing.T) { }, }, } - //b.AddInOutPair(inMsg, filteredEvent{T: voteFiltered}) + b.AddInOutPair(inMsg, filteredEvent{T: voteFiltered}) // vote from credentialRoundLag rounds ago and period 0 should continue // processing only for the purpose of tracking credentials. @@ -350,7 +350,7 @@ func TestProposalFreshAdjacentPeriods(t *testing.T) { }, }, } - //b.AddInOutPair(inMsg, filteredEvent{T: voteFiltered}) + b.AddInOutPair(inMsg, filteredEvent{T: voteFiltered}) // vote older than credentialRoundLag should be rejected even if period 0 pV = helper.MakeRandomProposalValue() @@ -364,7 +364,7 @@ func TestProposalFreshAdjacentPeriods(t *testing.T) { }, }, } - //b.AddInOutPair(inMsg, filteredEvent{T: voteFiltered}) + b.AddInOutPair(inMsg, filteredEvent{T: voteFiltered}) // vote older than credentialRoundLag should be rejected on any period pV = helper.MakeRandomProposalValue() @@ -378,7 +378,7 @@ func TestProposalFreshAdjacentPeriods(t *testing.T) { }, }, } - //b.AddInOutPair(inMsg, filteredEvent{T: voteFiltered}) + b.AddInOutPair(inMsg, filteredEvent{T: voteFiltered}) res, err := b.Build().Validate(pM) require.NoError(t, err) From 291a48c06dff6a61c36307c836c8208a0b8c4772 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Sat, 2 Sep 2023 15:47:14 -0400 Subject: [PATCH 198/232] better comment --- agreement/player_permutation_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/agreement/player_permutation_test.go b/agreement/player_permutation_test.go index bd023e7ce0..56f31e513b 100644 --- a/agreement/player_permutation_test.go +++ b/agreement/player_permutation_test.go @@ -805,7 +805,8 @@ func verifyPermutationExpectedActions(t *testing.T, playerN int, eventN int, hel // Generates a set of player states, router states, and messageEvents and tests all permutations of them func TestPlayerPermutation(t *testing.T) { partitiontest.PartitionTest(t) - // check with/without setting dynamic filter timeout + // check with current consensus params and with consensus params that + // explicitly enable dynamic filter timeout playerPermutationCheck(t, false) playerPermutationCheck(t, true) } From f2fcc976cfbc7b1e2cd6f830dc8523b1cfe1473b Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Sat, 2 Sep 2023 22:04:04 -0400 Subject: [PATCH 199/232] switch to enum --- agreement/events.go | 20 ++++++++++++-------- agreement/player.go | 14 +++++++++----- agreement/proposalManager.go | 16 ++++++++++++---- agreement/proposalManager_test.go | 2 +- agreement/proposalTracker.go | 18 +++++++++--------- agreement/proposalTracker_test.go | 18 +++++++++--------- 6 files changed, 52 insertions(+), 36 deletions(-) diff --git a/agreement/events.go b/agreement/events.go index 279c60b5f0..21b82aee82 100644 --- a/agreement/events.go +++ b/agreement/events.go @@ -596,18 +596,22 @@ func (e payloadProcessedEvent) ComparableStr() string { return fmt.Sprintf("%v: %.5v", e.t().String(), e.Proposal.BlockDigest.String()) } +type CredentialTrackingEffect uint8 + +const ( + NoCredentialTrackingImpact CredentialTrackingEffect = iota + MayImpactCredentialTracking + NewBestCredential +) + type filteredEvent struct { // {proposal,vote,bundle}{Filtered,Malformed} T eventType - // StateUpdated indicates whether the filtered message caused any change in - // the state machine - StateUpdated bool - - // ContinueProcessingVoteForCredentialTracking indicates whether its a vote - // present that we should continue processing only for tracking its - // credential arrival time. - ContinueProcessingVoteForCredentialTracking bool + // CredentialTrackingNote indicates the impact of the filtered event on the + // credential tracking machinary used for dynamically setting the filter + // timeout. + CredentialTrackingNote CredentialTrackingEffect // Err is the reason cryptographic verification failed and is set for // events {proposal,vote,bundle}Malformed. diff --git a/agreement/player.go b/agreement/player.go index b7536680bc..a0c19f2af7 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -620,17 +620,21 @@ func (p *player) handleMessageEvent(r routerHandle, e messageEvent) (actions []a err := ef.(filteredEvent).Err return append(actions, ignoreAction(e, err)) } - if ef.(filteredEvent).StateUpdated { + switch ef.(filteredEvent).CredentialTrackingNote { + case NewBestCredential: // Dynamic filter timeout feature enabled, and current message // updated the best credential arrival time v := e.Input.Vote return append(actions, relayAction(e, protocol.AgreementVoteTag, v.u())) - } else if !ef.(filteredEvent).ContinueProcessingVoteForCredentialTracking { - // Dynamic filter timeout feature enabled, and current message - // may update the best credential arrival time, so we should - // continue processing. + case NoCredentialTrackingImpact: + // Dynamic filter timeout feature enabled, but current message + // may not update the best credential arrival time, so we should + // ignore it. err := ef.(filteredEvent).Err return append(actions, ignoreAction(e, err)) + // There is another case, where the message + // MayImpactCredentialTracking. This case does not return here, + // so we continue processing the message. } } diff --git a/agreement/proposalManager.go b/agreement/proposalManager.go index b0286bd6bd..8dea58949d 100644 --- a/agreement/proposalManager.go +++ b/agreement/proposalManager.go @@ -135,8 +135,11 @@ func (m *proposalManager) handleMessageEvent(r routerHandle, p player, e filtera err := m.filterProposalVote(p, r, e.Input.UnauthenticatedVote, e.FreshnessData) if err != nil { // mark filtered votes that may still update the best credential arrival time - credTrackingProcessing := proposalUsedForCredentialHistory(e.FreshnessData.PlayerRound, e.Input.UnauthenticatedVote) - return filteredEvent{T: voteFiltered, Err: makeSerErr(err), ContinueProcessingVoteForCredentialTracking: credTrackingProcessing} + credTrackingNote := NoCredentialTrackingImpact + if proposalUsedForCredentialHistory(e.FreshnessData.PlayerRound, e.Input.UnauthenticatedVote) { + credTrackingNote = MayImpactCredentialTracking + } + return filteredEvent{T: voteFiltered, Err: makeSerErr(err), CredentialTrackingNote: credTrackingNote} } return emptyEvent{} @@ -174,12 +177,17 @@ func (m *proposalManager) handleMessageEvent(r routerHandle, p player, e filtera // we only continued processing this vote to see whether it updates the credential arrival time err := makeSerErrf("proposalManager: ignoring proposal-vote due to age: %v", err) if e.t() == voteFiltered { + credNote := e.(filteredEvent).CredentialTrackingNote + if credNote != NewBestCredential && credNote != NoCredentialTrackingImpact { + // It should be impossible to hit this condition + credNote = NoCredentialTrackingImpact + } // indicate whether it updated - return filteredEvent{T: voteFiltered, Err: err, StateUpdated: e.(filteredEvent).StateUpdated} + return filteredEvent{T: voteFiltered, Err: err, CredentialTrackingNote: credNote} } // the proposalMachineRound didn't filter the vote, so it must have had a better credential, // indicate that it did cause updating its state - return filteredEvent{T: voteFiltered, Err: err, StateUpdated: true} + return filteredEvent{T: voteFiltered, Err: err, CredentialTrackingNote: NewBestCredential} } return e diff --git a/agreement/proposalManager_test.go b/agreement/proposalManager_test.go index 770677d456..863e16292e 100644 --- a/agreement/proposalManager_test.go +++ b/agreement/proposalManager_test.go @@ -336,7 +336,7 @@ func TestProposalFreshAdjacentPeriods(t *testing.T) { }, }, } - b.AddInOutPair(inMsg, filteredEvent{T: voteFiltered, ContinueProcessingVoteForCredentialTracking: true}) + b.AddInOutPair(inMsg, filteredEvent{T: voteFiltered, CredentialTrackingNote: MayImpactCredentialTracking}) // vote from credentialRoundLag ago and period > 0 should be filtered pV = helper.MakeRandomProposalValue() diff --git a/agreement/proposalTracker.go b/agreement/proposalTracker.go index a9e0380bf1..f93dfb3a82 100644 --- a/agreement/proposalTracker.go +++ b/agreement/proposalTracker.go @@ -43,27 +43,27 @@ type proposalSeeker struct { // accept compares a given vote with the current lowest-credentialled vote and // sets it if freeze has not been called. Returns true if any internal proposalSeeker // state has been updated, whether to Lowest or lowestAfterFreeze. -func (s proposalSeeker) accept(v vote) (proposalSeeker, bool, error) { +func (s proposalSeeker) accept(v vote) (proposalSeeker, CredentialTrackingEffect, error) { if s.Frozen { - updated := false + effect := NoCredentialTrackingImpact // continue tracking and forwarding the lowest proposal even when frozen if !s.hasLowestAfterFreeze || v.Cred.Less(s.lowestAfterFreeze.Cred) { s.lowestAfterFreeze = v s.hasLowestAfterFreeze = true - updated = true + effect = NewBestCredential } - return s, updated, errProposalSeekerFrozen{} + return s, effect, errProposalSeekerFrozen{} } if s.Filled && !v.Cred.Less(s.Lowest.Cred) { - return s, false, errProposalSeekerNotLess{NewSender: v.R.Sender, LowestSender: s.Lowest.R.Sender} + return s, NoCredentialTrackingImpact, errProposalSeekerNotLess{NewSender: v.R.Sender, LowestSender: s.Lowest.R.Sender} } s.Lowest = v s.Filled = true s.lowestAfterFreeze = v s.hasLowestAfterFreeze = true - return s, true, nil + return s, NewBestCredential, nil } // freeze freezes the state of the proposalSeeker so that future calls no longer @@ -163,12 +163,12 @@ func (t *proposalTracker) handle(r routerHandle, p player, e event) event { return filteredEvent{T: voteFiltered, Err: makeSerErr(err)} } - var updated bool + var effect CredentialTrackingEffect var err error - t.Freezer, updated, err = t.Freezer.accept(v) + t.Freezer, effect, err = t.Freezer.accept(v) if err != nil { err := errProposalTrackerPS{Sub: err} - return filteredEvent{T: voteFiltered, StateUpdated: updated, Err: makeSerErr(err)} + return filteredEvent{T: voteFiltered, CredentialTrackingNote: effect, Err: makeSerErr(err)} } return proposalAcceptedEvent{ diff --git a/agreement/proposalTracker_test.go b/agreement/proposalTracker_test.go index 1b8a47aef6..383ef57beb 100644 --- a/agreement/proposalTracker_test.go +++ b/agreement/proposalTracker_test.go @@ -66,26 +66,26 @@ func TestProposalTrackerProposalSeeker(t *testing.T) { assert.False(t, s.hasLowestAfterFreeze) // issue events in the following order: 2, 3, 1, (freeze), 0 - var updated bool - s, updated, err = s.accept(votes[2]) + var effect CredentialTrackingEffect + s, effect, err = s.accept(votes[2]) assert.NoError(t, err) - assert.True(t, updated) + assert.Equal(t, effect, NewBestCredential) assert.False(t, s.Frozen) assert.True(t, s.Filled) assert.True(t, s.Lowest.equals(votes[2])) assert.True(t, s.hasLowestAfterFreeze) - s, updated, err = s.accept(votes[3]) + s, effect, err = s.accept(votes[3]) assert.Error(t, err) - assert.False(t, updated) + assert.Equal(t, effect, NoCredentialTrackingImpact) assert.False(t, s.Frozen) assert.True(t, s.Filled) assert.True(t, s.Lowest.equals(votes[2])) assert.True(t, s.hasLowestAfterFreeze) - s, updated, err = s.accept(votes[1]) + s, effect, err = s.accept(votes[1]) assert.NoError(t, err) - assert.True(t, updated) + assert.Equal(t, effect, NewBestCredential) assert.False(t, s.Frozen) assert.True(t, s.Filled) assert.True(t, s.Lowest.equals(votes[1])) @@ -98,9 +98,9 @@ func TestProposalTrackerProposalSeeker(t *testing.T) { assert.True(t, s.Lowest.equals(votes[1])) assert.True(t, s.hasLowestAfterFreeze) - s, updated, err = s.accept(votes[0]) + s, effect, err = s.accept(votes[0]) assert.Error(t, err) - assert.True(t, updated) + assert.Equal(t, effect, NewBestCredential) assert.Equal(t, s.Lowest, lowestBeforeFreeze) assert.True(t, s.Frozen) assert.True(t, s.Filled) From 8ff160741d24daa2152a72426fc24c3c59edb3b7 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Sun, 3 Sep 2023 11:11:12 -0400 Subject: [PATCH 200/232] msgp --- agreement/msgp_gen.go | 61 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/agreement/msgp_gen.go b/agreement/msgp_gen.go index 5a5efa7ddc..328610e67d 100644 --- a/agreement/msgp_gen.go +++ b/agreement/msgp_gen.go @@ -34,6 +34,15 @@ import ( // |-----> (*) MsgIsZero // |-----> ConsensusVersionViewMaxSize() // +// CredentialTrackingEffect +// |-----> MarshalMsg +// |-----> CanMarshalMsg +// |-----> (*) UnmarshalMsg +// |-----> (*) CanUnmarshalMsg +// |-----> Msgsize +// |-----> MsgIsZero +// |-----> CredentialTrackingEffectMaxSize() +// // Deadline // |-----> (*) MarshalMsg // |-----> (*) CanMarshalMsg @@ -979,6 +988,58 @@ func ConsensusVersionViewMaxSize() (s int) { return } +// MarshalMsg implements msgp.Marshaler +func (z CredentialTrackingEffect) MarshalMsg(b []byte) (o []byte) { + o = msgp.Require(b, z.Msgsize()) + o = msgp.AppendUint8(o, uint8(z)) + return +} + +func (_ CredentialTrackingEffect) CanMarshalMsg(z interface{}) bool { + _, ok := (z).(CredentialTrackingEffect) + if !ok { + _, ok = (z).(*CredentialTrackingEffect) + } + return ok +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *CredentialTrackingEffect) UnmarshalMsg(bts []byte) (o []byte, err error) { + { + var zb0001 uint8 + zb0001, bts, err = msgp.ReadUint8Bytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + (*z) = CredentialTrackingEffect(zb0001) + } + o = bts + return +} + +func (_ *CredentialTrackingEffect) CanUnmarshalMsg(z interface{}) bool { + _, ok := (z).(*CredentialTrackingEffect) + return ok +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z CredentialTrackingEffect) Msgsize() (s int) { + s = msgp.Uint8Size + return +} + +// MsgIsZero returns whether this is a zero value +func (z CredentialTrackingEffect) MsgIsZero() bool { + return z == 0 +} + +// MaxSize returns a maximum valid message size for this message type +func CredentialTrackingEffectMaxSize() (s int) { + s = msgp.Uint8Size + return +} + // MarshalMsg implements msgp.Marshaler func (z *Deadline) MarshalMsg(b []byte) (o []byte) { o = msgp.Require(b, z.Msgsize()) From f1c1b464e0651b24752f46d6f36fdd93858cf4d7 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Sun, 3 Sep 2023 14:46:38 -0400 Subject: [PATCH 201/232] var name --- agreement/actions.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/agreement/actions.go b/agreement/actions.go index 52608e4567..b2373fb350 100644 --- a/agreement/actions.go +++ b/agreement/actions.go @@ -322,14 +322,14 @@ func (a rezeroAction) do(ctx context.Context, s *Service) { removeList := make([]round, 0) - for round := range s.historicalClocks { - if a.Round > round+credentialRoundLag { - removeList = append(removeList, round) + for rnd := range s.historicalClocks { + if a.Round > rnd+credentialRoundLag { + removeList = append(removeList, rnd) } } - for _, round := range removeList { - delete(s.historicalClocks, round) + for _, rnd := range removeList { + delete(s.historicalClocks, rnd) } } From fd838587bd9e21998631ae2bb31b72c60fb3a39e Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Sun, 3 Sep 2023 15:03:28 -0400 Subject: [PATCH 202/232] fix lint errors --- agreement/events.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/agreement/events.go b/agreement/events.go index 21b82aee82..cb5dfa483c 100644 --- a/agreement/events.go +++ b/agreement/events.go @@ -596,11 +596,22 @@ func (e payloadProcessedEvent) ComparableStr() string { return fmt.Sprintf("%v: %.5v", e.t().String(), e.Proposal.BlockDigest.String()) } +// CredentialTrackingEffect inficates the impact of an event that got filtered +// on the credential tracking mechanism for the purpose of tracking the time it +// took the best credential to arrive. type CredentialTrackingEffect uint8 const ( + // NoCredentialTrackingImpact indicates the filtered event had no impact on + // the credential tracking mechanism. NoCredentialTrackingImpact CredentialTrackingEffect = iota + + // MayImpactCredentialTracking indicates the filtered event could impact on + // the credential tracking mechanism and more processing may be required. MayImpactCredentialTracking + + // NewBestCredential indicates that the filtered event provided a new best + // credential for its round. NewBestCredential ) From 418edccf88e7fc07376506815861a16c7b6ffa15 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Sun, 3 Sep 2023 15:47:50 -0400 Subject: [PATCH 203/232] debug logging --- agreement/proposalManager.go | 1 + 1 file changed, 1 insertion(+) diff --git a/agreement/proposalManager.go b/agreement/proposalManager.go index 8dea58949d..9275cc7710 100644 --- a/agreement/proposalManager.go +++ b/agreement/proposalManager.go @@ -180,6 +180,7 @@ func (m *proposalManager) handleMessageEvent(r routerHandle, p player, e filtera credNote := e.(filteredEvent).CredentialTrackingNote if credNote != NewBestCredential && credNote != NoCredentialTrackingImpact { // It should be impossible to hit this condition + r.t.log.Debugf("vote verified may only be tagged with NewBestCredential/NoCredentialTrackingImpact") credNote = NoCredentialTrackingImpact } // indicate whether it updated From 6b2ca69a723f57dcac72aa67cbd82328c5a9b1c3 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Tue, 5 Sep 2023 13:01:08 -0400 Subject: [PATCH 204/232] combine loops --- agreement/actions.go | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/agreement/actions.go b/agreement/actions.go index b2373fb350..f9a0c26990 100644 --- a/agreement/actions.go +++ b/agreement/actions.go @@ -320,17 +320,12 @@ func (a rezeroAction) do(ctx context.Context, s *Service) { s.historicalClocks[a.Round] = s.Clock } - removeList := make([]round, 0) - + // garbage collect clocks that are too old for rnd := range s.historicalClocks { if a.Round > rnd+credentialRoundLag { - removeList = append(removeList, rnd) + delete(s.historicalClocks, rnd) } } - - for _, rnd := range removeList { - delete(s.historicalClocks, rnd) - } } type pseudonodeAction struct { From 50d7be4756444185a44610f4cb203b1022bdbf3f Mon Sep 17 00:00:00 2001 From: chris erway Date: Tue, 5 Sep 2023 13:11:40 -0400 Subject: [PATCH 205/232] simplify verifyPermutationExpectedActions --- agreement/player_permutation_test.go | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/agreement/player_permutation_test.go b/agreement/player_permutation_test.go index 56f31e513b..598cac0bfc 100644 --- a/agreement/player_permutation_test.go +++ b/agreement/player_permutation_test.go @@ -468,27 +468,21 @@ func verifyPermutationExpectedActions(t *testing.T, playerN int, eventN int, hel case playerNextRound: switch eventN { case proposeVoteVerifiedEventSamePeriod: + requireActionCount(t, trace, 1, playerN, eventN) // This case should never happen -- player is on R+1 and the voteVerified event is for R. // Player will not queue up a verifyVoteAction for this vote (without DynamicFilterTimeout enabled). // We are asserting the relay behavior player currently implements, but it is not possible given current // code -- you would have filtered the votePresent for this vote. - if p > 0 { - expectIgnore(t, trace, "Player should ignore msg from past rounds, player: %v, event: %v", playerN, eventN) - } else if dynamicFilterTimeoutEnabled { - requireActionCount(t, trace, 1, playerN, eventN) + if dynamicFilterTimeoutEnabled && p == 0 { expectRelay(t, trace, "Player should relay period 0 msg from past rounds, player: %v, event: %v", playerN, eventN) } else { - requireActionCount(t, trace, 1, playerN, eventN) expectIgnore(t, trace, "Player should ignore msg from past rounds, player: %v, event: %v", playerN, eventN) } case proposeVotePresentEventSamePeriod: - if p > 0 { - expectIgnore(t, trace, "Player should ignore msg from past rounds, player: %v, event: %v", playerN, eventN) - } else if dynamicFilterTimeoutEnabled { - requireActionCount(t, trace, 1, playerN, eventN) + requireActionCount(t, trace, 1, playerN, eventN) + if dynamicFilterTimeoutEnabled && p == 0 { expectVerify(t, trace, "Player should verify period 0 msg from past rounds, player: %v, event: %v", playerN, eventN) } else { - requireActionCount(t, trace, 1, playerN, eventN) expectIgnore(t, trace, "Player should ignore msg from past rounds, player: %v, event: %v", playerN, eventN) } case softVoteVerifiedEventSamePeriod, softVotePresentEventSamePeriod, proposeVoteVerifiedEventNextPeriod, payloadPresentEvent, payloadVerifiedEvent, payloadVerifiedEventNoMessageHandle, bundleVerifiedEventSamePeriod, bundlePresentEventSamePeriod: From 999ea5f4a1e45f95de1720ca8db6e6cd6d193cae Mon Sep 17 00:00:00 2001 From: yossigi Date: Tue, 5 Sep 2023 14:39:11 -0400 Subject: [PATCH 206/232] Apply suggestions from code review Co-authored-by: Pavel Zbitskiy <65323360+algorandskiy@users.noreply.github.com> --- agreement/player.go | 1 + 1 file changed, 1 insertion(+) diff --git a/agreement/player.go b/agreement/player.go index a0c19f2af7..ca101f609c 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -632,6 +632,7 @@ func (p *player) handleMessageEvent(r routerHandle, e messageEvent) (actions []a // ignore it. err := ef.(filteredEvent).Err return append(actions, ignoreAction(e, err)) + case MayImpactCredentialTracking: // There is another case, where the message // MayImpactCredentialTracking. This case does not return here, // so we continue processing the message. From a4ab04aebe104db845a958e0ba26e592f543b385 Mon Sep 17 00:00:00 2001 From: yossigi Date: Tue, 5 Sep 2023 14:41:10 -0400 Subject: [PATCH 207/232] Update agreement/player_permutation_test.go Co-authored-by: cce <51567+cce@users.noreply.github.com> --- agreement/player_permutation_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/agreement/player_permutation_test.go b/agreement/player_permutation_test.go index 598cac0bfc..3b24ba86b3 100644 --- a/agreement/player_permutation_test.go +++ b/agreement/player_permutation_test.go @@ -468,6 +468,7 @@ func verifyPermutationExpectedActions(t *testing.T, playerN int, eventN int, hel case playerNextRound: switch eventN { case proposeVoteVerifiedEventSamePeriod: + requireActionCount(t, trace, 1, playerN, eventN) requireActionCount(t, trace, 1, playerN, eventN) // This case should never happen -- player is on R+1 and the voteVerified event is for R. // Player will not queue up a verifyVoteAction for this vote (without DynamicFilterTimeout enabled). From cf65c531e247c98d028bfc15e7490b3e8342318a Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Tue, 5 Sep 2023 14:42:59 -0400 Subject: [PATCH 208/232] x --- agreement/player_permutation_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/agreement/player_permutation_test.go b/agreement/player_permutation_test.go index 3b24ba86b3..598cac0bfc 100644 --- a/agreement/player_permutation_test.go +++ b/agreement/player_permutation_test.go @@ -468,7 +468,6 @@ func verifyPermutationExpectedActions(t *testing.T, playerN int, eventN int, hel case playerNextRound: switch eventN { case proposeVoteVerifiedEventSamePeriod: - requireActionCount(t, trace, 1, playerN, eventN) requireActionCount(t, trace, 1, playerN, eventN) // This case should never happen -- player is on R+1 and the voteVerified event is for R. // Player will not queue up a verifyVoteAction for this vote (without DynamicFilterTimeout enabled). From 4ec269bbf5bba2aa5117b5599dbb0219194a28aa Mon Sep 17 00:00:00 2001 From: chris erway Date: Tue, 5 Sep 2023 14:55:54 -0400 Subject: [PATCH 209/232] fix typos --- agreement/events.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/agreement/events.go b/agreement/events.go index cb5dfa483c..f5f5b0eb7c 100644 --- a/agreement/events.go +++ b/agreement/events.go @@ -596,8 +596,8 @@ func (e payloadProcessedEvent) ComparableStr() string { return fmt.Sprintf("%v: %.5v", e.t().String(), e.Proposal.BlockDigest.String()) } -// CredentialTrackingEffect inficates the impact of an event that got filtered -// on the credential tracking mechanism for the purpose of tracking the time it +// CredentialTrackingEffect indicates the impact of an event that got filtered +// on the credential tracking mechanism, for the purpose of tracking the time it // took the best credential to arrive. type CredentialTrackingEffect uint8 @@ -620,7 +620,7 @@ type filteredEvent struct { T eventType // CredentialTrackingNote indicates the impact of the filtered event on the - // credential tracking machinary used for dynamically setting the filter + // credential tracking machinery used for dynamically setting the filter // timeout. CredentialTrackingNote CredentialTrackingEffect From 10788645c7ece53c3633ef2e03860a52b40f51f0 Mon Sep 17 00:00:00 2001 From: chris erway Date: Tue, 5 Sep 2023 17:05:30 -0400 Subject: [PATCH 210/232] rename and document CredentialTrackingNote enum values a little --- agreement/events.go | 12 +++++++----- agreement/player.go | 4 ++-- agreement/proposalManager.go | 6 +++--- agreement/proposalManager_test.go | 2 +- agreement/proposalTracker.go | 4 ++-- agreement/proposalTracker_test.go | 6 +++--- 6 files changed, 18 insertions(+), 16 deletions(-) diff --git a/agreement/events.go b/agreement/events.go index f5f5b0eb7c..0a1ae667e7 100644 --- a/agreement/events.go +++ b/agreement/events.go @@ -606,13 +606,15 @@ const ( // the credential tracking mechanism. NoCredentialTrackingImpact CredentialTrackingEffect = iota - // MayImpactCredentialTracking indicates the filtered event could impact on - // the credential tracking mechanism and more processing may be required. - MayImpactCredentialTracking + // UnverifiedBetterCredentialForTracking indicates the filtered event could impact + // the credential tracking mechanism and more processing (validation) may be required. + // It may be set by proposalManager when handling votePresent events. + UnverifiedBetterCredentialForTracking - // NewBestCredential indicates that the filtered event provided a new best + // VerifiedBetterCredentialForTracking indicates that the filtered event provided a new best // credential for its round. - NewBestCredential + // It may be set by proposalManager when handling voteVerified events. + VerifiedBetterCredentialForTracking ) type filteredEvent struct { diff --git a/agreement/player.go b/agreement/player.go index ca101f609c..72b7b4d129 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -621,7 +621,7 @@ func (p *player) handleMessageEvent(r routerHandle, e messageEvent) (actions []a return append(actions, ignoreAction(e, err)) } switch ef.(filteredEvent).CredentialTrackingNote { - case NewBestCredential: + case VerifiedBetterCredentialForTracking: // Dynamic filter timeout feature enabled, and current message // updated the best credential arrival time v := e.Input.Vote @@ -632,7 +632,7 @@ func (p *player) handleMessageEvent(r routerHandle, e messageEvent) (actions []a // ignore it. err := ef.(filteredEvent).Err return append(actions, ignoreAction(e, err)) - case MayImpactCredentialTracking: + case UnverifiedBetterCredentialForTracking: // There is another case, where the message // MayImpactCredentialTracking. This case does not return here, // so we continue processing the message. diff --git a/agreement/proposalManager.go b/agreement/proposalManager.go index 9275cc7710..e9f29dbc52 100644 --- a/agreement/proposalManager.go +++ b/agreement/proposalManager.go @@ -137,7 +137,7 @@ func (m *proposalManager) handleMessageEvent(r routerHandle, p player, e filtera // mark filtered votes that may still update the best credential arrival time credTrackingNote := NoCredentialTrackingImpact if proposalUsedForCredentialHistory(e.FreshnessData.PlayerRound, e.Input.UnauthenticatedVote) { - credTrackingNote = MayImpactCredentialTracking + credTrackingNote = UnverifiedBetterCredentialForTracking } return filteredEvent{T: voteFiltered, Err: makeSerErr(err), CredentialTrackingNote: credTrackingNote} } @@ -178,7 +178,7 @@ func (m *proposalManager) handleMessageEvent(r routerHandle, p player, e filtera err := makeSerErrf("proposalManager: ignoring proposal-vote due to age: %v", err) if e.t() == voteFiltered { credNote := e.(filteredEvent).CredentialTrackingNote - if credNote != NewBestCredential && credNote != NoCredentialTrackingImpact { + if credNote != VerifiedBetterCredentialForTracking && credNote != NoCredentialTrackingImpact { // It should be impossible to hit this condition r.t.log.Debugf("vote verified may only be tagged with NewBestCredential/NoCredentialTrackingImpact") credNote = NoCredentialTrackingImpact @@ -188,7 +188,7 @@ func (m *proposalManager) handleMessageEvent(r routerHandle, p player, e filtera } // the proposalMachineRound didn't filter the vote, so it must have had a better credential, // indicate that it did cause updating its state - return filteredEvent{T: voteFiltered, Err: err, CredentialTrackingNote: NewBestCredential} + return filteredEvent{T: voteFiltered, Err: err, CredentialTrackingNote: VerifiedBetterCredentialForTracking} } return e diff --git a/agreement/proposalManager_test.go b/agreement/proposalManager_test.go index 863e16292e..7212234116 100644 --- a/agreement/proposalManager_test.go +++ b/agreement/proposalManager_test.go @@ -336,7 +336,7 @@ func TestProposalFreshAdjacentPeriods(t *testing.T) { }, }, } - b.AddInOutPair(inMsg, filteredEvent{T: voteFiltered, CredentialTrackingNote: MayImpactCredentialTracking}) + b.AddInOutPair(inMsg, filteredEvent{T: voteFiltered, CredentialTrackingNote: UnverifiedBetterCredentialForTracking}) // vote from credentialRoundLag ago and period > 0 should be filtered pV = helper.MakeRandomProposalValue() diff --git a/agreement/proposalTracker.go b/agreement/proposalTracker.go index f93dfb3a82..5b48dda401 100644 --- a/agreement/proposalTracker.go +++ b/agreement/proposalTracker.go @@ -50,7 +50,7 @@ func (s proposalSeeker) accept(v vote) (proposalSeeker, CredentialTrackingEffect if !s.hasLowestAfterFreeze || v.Cred.Less(s.lowestAfterFreeze.Cred) { s.lowestAfterFreeze = v s.hasLowestAfterFreeze = true - effect = NewBestCredential + effect = VerifiedBetterCredentialForTracking } return s, effect, errProposalSeekerFrozen{} } @@ -63,7 +63,7 @@ func (s proposalSeeker) accept(v vote) (proposalSeeker, CredentialTrackingEffect s.Filled = true s.lowestAfterFreeze = v s.hasLowestAfterFreeze = true - return s, NewBestCredential, nil + return s, VerifiedBetterCredentialForTracking, nil } // freeze freezes the state of the proposalSeeker so that future calls no longer diff --git a/agreement/proposalTracker_test.go b/agreement/proposalTracker_test.go index 383ef57beb..41ff3906e2 100644 --- a/agreement/proposalTracker_test.go +++ b/agreement/proposalTracker_test.go @@ -69,7 +69,7 @@ func TestProposalTrackerProposalSeeker(t *testing.T) { var effect CredentialTrackingEffect s, effect, err = s.accept(votes[2]) assert.NoError(t, err) - assert.Equal(t, effect, NewBestCredential) + assert.Equal(t, effect, VerifiedBetterCredentialForTracking) assert.False(t, s.Frozen) assert.True(t, s.Filled) assert.True(t, s.Lowest.equals(votes[2])) @@ -85,7 +85,7 @@ func TestProposalTrackerProposalSeeker(t *testing.T) { s, effect, err = s.accept(votes[1]) assert.NoError(t, err) - assert.Equal(t, effect, NewBestCredential) + assert.Equal(t, effect, VerifiedBetterCredentialForTracking) assert.False(t, s.Frozen) assert.True(t, s.Filled) assert.True(t, s.Lowest.equals(votes[1])) @@ -100,7 +100,7 @@ func TestProposalTrackerProposalSeeker(t *testing.T) { s, effect, err = s.accept(votes[0]) assert.Error(t, err) - assert.Equal(t, effect, NewBestCredential) + assert.Equal(t, effect, VerifiedBetterCredentialForTracking) assert.Equal(t, s.Lowest, lowestBeforeFreeze) assert.True(t, s.Frozen) assert.True(t, s.Filled) From 51923447ded1b07a7b73ad004ff4adf7c2f932ff Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Tue, 5 Sep 2023 22:40:12 -0400 Subject: [PATCH 211/232] new test for late arriviong credentials where the first credential is better --- agreement/proposalManager.go | 2 +- agreement/proposalManager_test.go | 59 +++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 1 deletion(-) diff --git a/agreement/proposalManager.go b/agreement/proposalManager.go index e9f29dbc52..ee28fc3127 100644 --- a/agreement/proposalManager.go +++ b/agreement/proposalManager.go @@ -180,7 +180,7 @@ func (m *proposalManager) handleMessageEvent(r routerHandle, p player, e filtera credNote := e.(filteredEvent).CredentialTrackingNote if credNote != VerifiedBetterCredentialForTracking && credNote != NoCredentialTrackingImpact { // It should be impossible to hit this condition - r.t.log.Debugf("vote verified may only be tagged with NewBestCredential/NoCredentialTrackingImpact") + r.t.log.Debugf("vote verified may only be tagged with NewBestCredential/NoCredentialTrackingImpact but saw %v", credNote) credNote = NoCredentialTrackingImpact } // indicate whether it updated diff --git a/agreement/proposalManager_test.go b/agreement/proposalManager_test.go index 7212234116..a68b5fc984 100644 --- a/agreement/proposalManager_test.go +++ b/agreement/proposalManager_test.go @@ -209,6 +209,65 @@ func TestProposalManagerRejectsUnknownEvent(t *testing.T) { require.Errorf(t, panicErr, "proposalManager must reject bundleVerified event") } +func TestLateVotes(t *testing.T) { + partitiontest.PartitionTest(t) + + const r = 100 + const p = 3 + const s = soft + currentPlayerState := freshnessData{ + PlayerRound: r, + PlayerPeriod: p, + PlayerStep: s, + PlayerLastConcluding: 0, + } + b := testCaseBuilder{} + _, pM, helper := setupManager(t, r) + + // vote from credentialRoundLag rounds ago and period 0 should continue + // processing only for the purpose of tracking credentials. + pV := helper.MakeRandomProposalValue() + v1 := helper.MakeVerifiedVote(t, 0, r-credentialRoundLag, 0, 0, *pV) + inMsg1 := filterableMessageEvent{ + FreshnessData: currentPlayerState, + messageEvent: messageEvent{ + T: voteVerified, + Input: message{ + UnauthenticatedVote: v1.u(), + Vote: v1, + }, + }, + } + + v2 := helper.MakeVerifiedVote(t, 0, r-credentialRoundLag, 0, 0, *pV) + inMsg2 := filterableMessageEvent{ + FreshnessData: currentPlayerState, + messageEvent: messageEvent{ + T: voteVerified, + Input: message{ + UnauthenticatedVote: v2.u(), + Vote: v2, + }, + }, + } + + // Order the messages such that the first message's credential is lower + // (i.e., preffered). The first vote should be the best credential we get, + // so the second credential should be filtered without impacting the + // credential tracking mechanism. + if v1.Cred.Less(v2.Cred) { + b.AddInOutPair(inMsg1, filteredEvent{T: voteFiltered, CredentialTrackingNote: VerifiedBetterCredentialForTracking}) + b.AddInOutPair(inMsg2, filteredEvent{T: voteFiltered, CredentialTrackingNote: NoCredentialTrackingImpact}) + } else { + b.AddInOutPair(inMsg2, filteredEvent{T: voteFiltered, CredentialTrackingNote: VerifiedBetterCredentialForTracking}) + b.AddInOutPair(inMsg1, filteredEvent{T: voteFiltered, CredentialTrackingNote: NoCredentialTrackingImpact}) + } + + res, err := b.Build().Validate(pM) + require.NoError(t, err) + require.NoErrorf(t, res, "VerifiedVote resulted in unexpected output") +} + func TestProposalFreshAdjacentPeriods(t *testing.T) { partitiontest.PartitionTest(t) From 917e25f8f95544b1f960f3d3a76026cf42bd22c8 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Wed, 6 Sep 2023 15:46:58 -0400 Subject: [PATCH 212/232] check staging after updating credential tracking --- agreement/proposalTracker.go | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/agreement/proposalTracker.go b/agreement/proposalTracker.go index 5b48dda401..7f620a7d13 100644 --- a/agreement/proposalTracker.go +++ b/agreement/proposalTracker.go @@ -66,6 +66,11 @@ func (s proposalSeeker) accept(v vote) (proposalSeeker, CredentialTrackingEffect return s, VerifiedBetterCredentialForTracking, nil } +func (s *proposalSeeker) copyCredentialTrackingState(s2 proposalSeeker) { + s.hasLowestAfterFreeze = s2.hasLowestAfterFreeze + s.lowestAfterFreeze = s2.lowestAfterFreeze +} + // freeze freezes the state of the proposalSeeker so that future calls no longer // change its state. func (s proposalSeeker) freeze() proposalSeeker { @@ -158,19 +163,19 @@ func (t *proposalTracker) handle(r routerHandle, p player, e event) event { } t.Duplicate[v.R.Sender] = true - if t.Staging != bottom { - err := errProposalTrackerStaged{} - return filteredEvent{T: voteFiltered, Err: makeSerErr(err)} - } - - var effect CredentialTrackingEffect - var err error - t.Freezer, effect, err = t.Freezer.accept(v) + newFreezer, effect, err := t.Freezer.accept(v) + t.Freezer.copyCredentialTrackingState(newFreezer) if err != nil { err := errProposalTrackerPS{Sub: err} return filteredEvent{T: voteFiltered, CredentialTrackingNote: effect, Err: makeSerErr(err)} } + if t.Staging != bottom { + err = errProposalTrackerStaged{} + return filteredEvent{T: voteFiltered, Err: makeSerErr(err)} + } + t.Freezer = newFreezer + return proposalAcceptedEvent{ Round: v.R.Round, Period: v.R.Period, From 704494b99c3d8ec20903cb3c8b0740dccd6da019 Mon Sep 17 00:00:00 2001 From: yossigi Date: Wed, 6 Sep 2023 18:11:15 -0400 Subject: [PATCH 213/232] Update agreement/proposalTracker.go Co-authored-by: cce <51567+cce@users.noreply.github.com> --- agreement/proposalTracker.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/agreement/proposalTracker.go b/agreement/proposalTracker.go index 7f620a7d13..15bc37f5b3 100644 --- a/agreement/proposalTracker.go +++ b/agreement/proposalTracker.go @@ -165,14 +165,14 @@ func (t *proposalTracker) handle(r routerHandle, p player, e event) event { newFreezer, effect, err := t.Freezer.accept(v) t.Freezer.copyCredentialTrackingState(newFreezer) - if err != nil { - err := errProposalTrackerPS{Sub: err} + if t.Staging != bottom { + err = errProposalTrackerStaged{} return filteredEvent{T: voteFiltered, CredentialTrackingNote: effect, Err: makeSerErr(err)} } - if t.Staging != bottom { - err = errProposalTrackerStaged{} - return filteredEvent{T: voteFiltered, Err: makeSerErr(err)} + if err != nil { + err := errProposalTrackerPS{Sub: err} + return filteredEvent{T: voteFiltered, CredentialTrackingNote: effect, Err: makeSerErr(err)} } t.Freezer = newFreezer From da6e736df5fa67fb158846f7fb1a48847b5fe5fc Mon Sep 17 00:00:00 2001 From: chris erway Date: Thu, 7 Sep 2023 15:39:54 -0400 Subject: [PATCH 214/232] add TestPlayerRetainsReceivedValidatedAtForHistoryWindowLateBetter and update TestLateVotes --- agreement/player_test.go | 44 +++++++++++++++++++++++++++++++ agreement/proposalManager_test.go | 7 ++++- 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/agreement/player_test.go b/agreement/player_test.go index 1053f805a0..ecbec396a1 100644 --- a/agreement/player_test.go +++ b/agreement/player_test.go @@ -3387,6 +3387,15 @@ func TestPlayerRetainsLateReceivedValidatedAtOneSample(t *testing.T) { // DynamicFilterCredentialArrivalHistory rounds. func TestPlayerRetainsReceivedValidatedAtForHistoryWindow(t *testing.T) { partitiontest.PartitionTest(t) + testPlayerRetainsReceivedValidatedAtForHistoryWindow(t, false) +} + +func TestPlayerRetainsReceivedValidatedAtForHistoryWindowLateBetter(t *testing.T) { + partitiontest.PartitionTest(t) + testPlayerRetainsReceivedValidatedAtForHistoryWindow(t, true) +} + +func testPlayerRetainsReceivedValidatedAtForHistoryWindow(t *testing.T, addBetterLate bool) { const r = round(20239) const p = period(0) pWhite, pM, helper := setupP(t, r-1, p, soft) @@ -3397,6 +3406,21 @@ func TestPlayerRetainsReceivedValidatedAtForHistoryWindow(t *testing.T) { // send voteVerified message pP, pV := helper.MakeRandomProposalPayload(t, r+round(i)-1) vVote := helper.MakeVerifiedVote(t, 0, r+round(i)-1, p, propose, *pV) + var betterLateVote vote + if addBetterLate { + // set up better late proposal-vote from someone else, so it won't be a errProposalTrackerSenderDup + vVote2 := helper.MakeVerifiedVote(t, 1, r+round(i)-1, p, propose, *pV) + vVote.Cred.VrfOut = crypto.Digest{1} + vVote2.Cred.VrfOut = crypto.Digest{2} + if vVote2.Cred.Less(vVote.Cred) { + betterLateVote = vVote2 + } else { + betterLateVote = vVote + vVote = vVote2 + } + require.True(t, betterLateVote.Cred.Less(vVote.Cred)) + require.False(t, vVote.Cred.Less(betterLateVote.Cred)) + } inMsg := messageEvent{T: voteVerified, Input: message{Vote: vVote, UnauthenticatedVote: vVote.u()}} timestamp := 500 + i inMsg = inMsg.AttachValidatedAt(time.Duration(timestamp)*time.Millisecond, r+round(i)-1, nil) @@ -3412,6 +3436,16 @@ func TestPlayerRetainsReceivedValidatedAtForHistoryWindow(t *testing.T) { require.NoError(t, err) require.NoError(t, panicErr) moveToRound(t, pWhite, pM, helper, r+round(i), p, pP, pV, m, protocol.ConsensusFuture) + + // send better late voteVerified message + if addBetterLate { + inMsg = messageEvent{T: voteVerified, Input: message{Vote: betterLateVote, UnauthenticatedVote: betterLateVote.u()}} + timestamp := 600 + i + inMsg = inMsg.AttachValidatedAt(time.Duration(timestamp)*time.Millisecond, r+round(i)-1, nil) + err, panicErr = pM.transition(inMsg) + require.NoError(t, err) + require.NoError(t, panicErr) + } } // assert lowest vote validateAt time was recorded into payloadArrivals @@ -3419,6 +3453,9 @@ func TestPlayerRetainsReceivedValidatedAtForHistoryWindow(t *testing.T) { for i := 0; i < dynamicFilterCredentialArrivalHistory; i++ { // only the last historyLen samples are kept, so the first one is discarded timestamp := 500 + i + if addBetterLate { + timestamp = 600 + i + } require.Equal(t, time.Duration(timestamp)*time.Millisecond, pWhite.lowestCredentialArrivals.history[i]) } } @@ -4075,6 +4112,13 @@ func moveToRound(t *testing.T, pWhite *player, pM ioAutomata, helper *voteMakerH require.NoError(t, err) require.NoError(t, panicErr) + // // now, trigger soft vote timeout + // err, panicErr = pM.transition(makeTimeoutEvent()) + // require.NoError(t, err) + // require.NoError(t, panicErr) + // softVoteEvent := ev(pseudonodeAction{T: attest, Round: r - 1, Period: p, Step: soft, Proposal: *pV}) + // require.Truef(t, pM.getTrace().Contains(softVoteEvent), "Player should issue soft vote") + // gen cert to move into the next round votes := make([]vote, int(cert.threshold(config.Consensus[ver]))) for i := 0; i < int(cert.threshold(config.Consensus[ver])); i++ { diff --git a/agreement/proposalManager_test.go b/agreement/proposalManager_test.go index a68b5fc984..c85bcd3450 100644 --- a/agreement/proposalManager_test.go +++ b/agreement/proposalManager_test.go @@ -19,6 +19,7 @@ package agreement import ( "testing" + "github.com/algorand/go-algorand/crypto" "github.com/algorand/go-algorand/test/partitiontest" "github.com/stretchr/testify/require" ) @@ -252,13 +253,17 @@ func TestLateVotes(t *testing.T) { } // Order the messages such that the first message's credential is lower - // (i.e., preffered). The first vote should be the best credential we get, + // (i.e., preferred). The first vote should be the best credential we get, // so the second credential should be filtered without impacting the // credential tracking mechanism. + v1.Cred.VrfOut = crypto.Digest{1} + v2.Cred.VrfOut = crypto.Digest{2} if v1.Cred.Less(v2.Cred) { + require.False(t, v2.Cred.Less(v1.Cred)) b.AddInOutPair(inMsg1, filteredEvent{T: voteFiltered, CredentialTrackingNote: VerifiedBetterCredentialForTracking}) b.AddInOutPair(inMsg2, filteredEvent{T: voteFiltered, CredentialTrackingNote: NoCredentialTrackingImpact}) } else { + require.True(t, v2.Cred.Less(v1.Cred)) b.AddInOutPair(inMsg2, filteredEvent{T: voteFiltered, CredentialTrackingNote: VerifiedBetterCredentialForTracking}) b.AddInOutPair(inMsg1, filteredEvent{T: voteFiltered, CredentialTrackingNote: NoCredentialTrackingImpact}) } From b3b6cb5156ddf29e6d09ae444a0cb3f03ff13cc3 Mon Sep 17 00:00:00 2001 From: chris erway Date: Mon, 18 Sep 2023 16:31:40 -0400 Subject: [PATCH 215/232] don't persist agreement state for old rounds --- agreement/persistence.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/agreement/persistence.go b/agreement/persistence.go index 819806b69a..8711edf41b 100644 --- a/agreement/persistence.go +++ b/agreement/persistence.go @@ -55,6 +55,17 @@ func persistent(as []action) bool { // encode serializes the current state into a byte array. func encode(t timers.Clock[TimeoutType], rr rootRouter, p player, a []action, reflect bool) (raw []byte) { var s diskState + + // Don't persist state for old rounds + // rootRouter.update() may preserve roundRouters from credentialRoundLag rounds ago + children := make(map[round]*roundRouter) + for rnd, rndRouter := range rr.Children { + if rnd >= p.Round { + children[rnd] = rndRouter + } + } + rr.Children = children + if reflect { s.Router = protocol.EncodeReflect(rr) s.Player = protocol.EncodeReflect(p) From 363f5318dc95e84348f1f38794d393a09f60be6b Mon Sep 17 00:00:00 2001 From: cce <51567+cce@users.noreply.github.com> Date: Fri, 29 Sep 2023 14:24:24 -0400 Subject: [PATCH 216/232] dynamic-lambda: improve handling of duplicate votes and update player tests (#5755) --- agreement/actions.go | 34 ++ agreement/events.go | 2 +- agreement/player_permutation_test.go | 47 +- agreement/player_test.go | 755 +++++++++++---------------- agreement/proposalManager.go | 60 ++- agreement/state_machine_test.go | 11 +- 6 files changed, 415 insertions(+), 494 deletions(-) diff --git a/agreement/actions.go b/agreement/actions.go index f9a0c26990..80599014a4 100644 --- a/agreement/actions.go +++ b/agreement/actions.go @@ -66,6 +66,7 @@ type action interface { do(context.Context, *Service) String() string + ComparableString() string } type nonpersistent struct{} @@ -88,6 +89,8 @@ func (a noopAction) String() string { return a.t().String() } +func (a noopAction) ComparableString() string { return a.String() } + type networkAction struct { nonpersistent @@ -120,6 +123,13 @@ func (a networkAction) String() string { return fmt.Sprintf("%s: %2v", a.t().String(), a.Tag) } +func (a networkAction) ComparableString() string { + if a.Tag == protocol.AgreementVoteTag { + return fmt.Sprintf("%s: %2v: %3v-%2v-%2v", a.t().String(), a.Tag, a.UnauthenticatedVote.R.Round, a.UnauthenticatedVote.R.Period, a.UnauthenticatedVote.R.Step) + } + return a.String() +} + func (a networkAction) do(ctx context.Context, s *Service) { if a.T == broadcastVotes { tag := protocol.AgreementVoteTag @@ -192,6 +202,18 @@ func (a cryptoAction) String() string { return a.t().String() } +func (a cryptoAction) ComparableString() (s string) { + switch a.T { + case verifyVote: + s = fmt.Sprintf("%s: %3v-%2v TaskIndex %d", a.t().String(), a.Round, a.Period, a.TaskIndex) + case verifyPayload: + s = fmt.Sprintf("%s: %3v-%2v Pinned %v", a.t().String(), a.Round, a.Period, a.Pinned) + case verifyBundle: + s = fmt.Sprintf("%s: %3v-%2v-%2v", a.t().String(), a.Round, a.Period, a.Step) + } + return +} + func (a cryptoAction) do(ctx context.Context, s *Service) { switch a.T { case verifyVote: @@ -225,6 +247,8 @@ func (a ensureAction) String() string { return fmt.Sprintf("%s: %.5s: %v, %v, %.5s", a.t().String(), a.Payload.Digest().String(), a.Certificate.Round, a.Certificate.Period, a.Certificate.Proposal.BlockDigest.String()) } +func (a ensureAction) ComparableString() string { return a.String() } + func (a ensureAction) do(ctx context.Context, s *Service) { logEvent := logspec.AgreementEvent{ Hash: a.Certificate.Proposal.BlockDigest.String(), @@ -288,6 +312,8 @@ func (a stageDigestAction) String() string { return fmt.Sprintf("%s: %.5s. %v. %v", a.t().String(), a.Certificate.Proposal.BlockDigest.String(), a.Certificate.Round, a.Certificate.Period) } +func (a stageDigestAction) ComparableString() string { return a.String() } + func (a stageDigestAction) do(ctx context.Context, service *Service) { logEvent := logspec.AgreementEvent{ Hash: a.Certificate.Proposal.BlockDigest.String(), @@ -314,6 +340,10 @@ func (a rezeroAction) String() string { return a.t().String() } +func (a rezeroAction) ComparableString() string { + return fmt.Sprintf("%s: %d", a.t().String(), a.Round) +} + func (a rezeroAction) do(ctx context.Context, s *Service) { s.Clock = s.Clock.Zero() if _, ok := s.historicalClocks[a.Round]; !ok { @@ -346,6 +376,8 @@ func (a pseudonodeAction) String() string { return fmt.Sprintf("%v %3v-%2v-%2v: %.5v", a.t().String(), a.Round, a.Period, a.Step, a.Proposal.BlockDigest.String()) } +func (a pseudonodeAction) ComparableString() string { return a.String() } + func (a pseudonodeAction) persistent() bool { return a.T == attest } @@ -538,3 +570,5 @@ func (c checkpointAction) do(ctx context.Context, s *Service) { func (c checkpointAction) String() string { return c.t().String() } + +func (c checkpointAction) ComparableString() string { return c.String() } diff --git a/agreement/events.go b/agreement/events.go index 0a1ae667e7..8343313546 100644 --- a/agreement/events.go +++ b/agreement/events.go @@ -296,7 +296,7 @@ func (e messageEvent) String() string { } func (e messageEvent) ComparableStr() string { - return e.T.String() + return fmt.Sprintf("{T:%s %d Err:%v}", e.t().String(), e.ConsensusRound(), e.Err) } func (e messageEvent) ConsensusRound() round { diff --git a/agreement/player_permutation_test.go b/agreement/player_permutation_test.go index 598cac0bfc..bd2c2c84da 100644 --- a/agreement/player_permutation_test.go +++ b/agreement/player_permutation_test.go @@ -264,6 +264,7 @@ func getMessageEventPermutation(t *testing.T, n int, helper *voteMakerHelper) (e bun := unauthenticatedBundle{ Round: r, Period: p, + Step: cert, Proposal: pV, } e = messageEvent{ @@ -397,6 +398,10 @@ func requireActionCount(t *testing.T, trace ioTrace, expectedCount, playerN, eve } func requireTraceContains(t *testing.T, trace ioTrace, expected event, playerN, eventN int) { + if !trace.Contains(expected) { + t.Log("expected:", expected.ComparableStr()) + t.Log("trace:", trace.String()) + } require.Truef(t, trace.Contains(expected), "Player should emit action, player: %v, event: %v", playerN, eventN) } @@ -416,7 +421,7 @@ func verifyPermutationExpectedActions(t *testing.T, playerN int, eventN int, hel case softVotePresentEventSamePeriod: requireActionCount(t, trace, 1, playerN, eventN) vvote := helper.MakeVerifiedVote(t, 0, r, p, soft, pV) - a := cryptoAction{T: verifyVote, M: message{UnauthenticatedVote: vvote.u()}, TaskIndex: 0} + a := verifyVoteAction(messageEvent{Input: message{UnauthenticatedVote: vvote.u()}}, r, p, 0) requireTraceContains(t, trace, ev(a), playerN, eventN) case proposeVoteVerifiedEventNextPeriod: requireActionCount(t, trace, 1, playerN, eventN) @@ -431,7 +436,7 @@ func verifyPermutationExpectedActions(t *testing.T, playerN int, eventN int, hel case proposeVotePresentEventSamePeriod: requireActionCount(t, trace, 1, playerN, eventN) vvote := helper.MakeVerifiedVote(t, 0, r, p, propose, pV) - a := cryptoAction{T: verifyVote, M: message{UnauthenticatedVote: vvote.u()}, TaskIndex: 0} + a := verifyVoteAction(messageEvent{Input: message{UnauthenticatedVote: vvote.u()}}, r, p, 1) requireTraceContains(t, trace, ev(a), playerN, eventN) case payloadPresentEvent, payloadVerifiedEvent, payloadVerifiedEventNoMessageHandle: requireActionCount(t, trace, 1, playerN, eventN) @@ -453,8 +458,8 @@ func verifyPermutationExpectedActions(t *testing.T, playerN int, eventN int, hel for i := 0; i < int(cert.threshold(config.Consensus[protocol.ConsensusCurrentVersion])); i++ { votes[i] = helper.MakeVerifiedVote(t, i, r, p, cert, pV) } - bun := unauthenticatedBundle{Round: r, Period: p, Proposal: pV} - ca := cryptoAction{T: verifyBundle, M: message{Bundle: bundle{U: bun, Votes: votes}, UnauthenticatedBundle: bun}, TaskIndex: 0} + bun := unauthenticatedBundle{Round: r, Period: p, Step: cert, Proposal: pV} + ca := verifyBundleAction(messageEvent{Input: message{Bundle: bundle{U: bun, Votes: votes}, UnauthenticatedBundle: bun}}, r, p, cert) requireTraceContains(t, trace, ev(ca), playerN, eventN) case softVoteVerifiedErrorEventSamePeriod, proposeVoteVerifiedErrorEventSamePeriod, bundleVerifiedErrorEvent: requireActionCount(t, trace, 1, playerN, eventN) @@ -507,7 +512,7 @@ func verifyPermutationExpectedActions(t *testing.T, playerN int, eventN int, hel case softVotePresentEventSamePeriod: requireActionCount(t, trace, 1, playerN, eventN) vvote := helper.MakeVerifiedVote(t, 0, r, p, soft, pV) - a := cryptoAction{T: verifyVote, M: message{UnauthenticatedVote: vvote.u()}, TaskIndex: 0} + a := verifyVoteAction(messageEvent{Input: message{UnauthenticatedVote: vvote.u()}}, r, p, 0) requireTraceContains(t, trace, ev(a), playerN, eventN) case proposeVoteVerifiedEventNextPeriod: requireActionCount(t, trace, 1, playerN, eventN) @@ -522,7 +527,7 @@ func verifyPermutationExpectedActions(t *testing.T, playerN int, eventN int, hel case proposeVotePresentEventSamePeriod: requireActionCount(t, trace, 1, playerN, eventN) vvote := helper.MakeVerifiedVote(t, 0, r, p, propose, pV) - a := cryptoAction{T: verifyVote, M: message{UnauthenticatedVote: vvote.u()}, TaskIndex: 0} + a := verifyVoteAction(messageEvent{Input: message{UnauthenticatedVote: vvote.u()}, TaskIndex: 0}, r, p, 2) requireTraceContains(t, trace, ev(a), playerN, eventN) case payloadPresentEvent, payloadVerifiedEvent, payloadVerifiedEventNoMessageHandle: requireActionCount(t, trace, 1, playerN, eventN) @@ -549,7 +554,7 @@ func verifyPermutationExpectedActions(t *testing.T, playerN int, eventN int, hel case softVotePresentEventSamePeriod: requireActionCount(t, trace, 1, playerN, eventN) vvote := helper.MakeVerifiedVote(t, 0, r, p, soft, pV) - a := cryptoAction{T: verifyVote, M: message{UnauthenticatedVote: vvote.u()}, TaskIndex: 0} + a := verifyVoteAction(messageEvent{Input: message{UnauthenticatedVote: vvote.u()}}, r, p, 0) requireTraceContains(t, trace, ev(a), playerN, eventN) case proposeVoteVerifiedEventNextPeriod: requireActionCount(t, trace, 1, playerN, eventN) @@ -561,7 +566,7 @@ func verifyPermutationExpectedActions(t *testing.T, playerN int, eventN int, hel expectIgnore(t, trace, "Player should ignore proposalvvote already received: %v, event: %v", playerN, eventN) case payloadPresentEvent: requireActionCount(t, trace, 2, playerN, eventN) - ca := cryptoAction{T: verifyPayload, M: message{UnauthenticatedProposal: payload.u()}, TaskIndex: 0} + ca := verifyPayloadAction(messageEvent{Input: message{UnauthenticatedProposal: payload.u()}}, r, p, false) requireTraceContains(t, trace, ev(ca), playerN, eventN) na := networkAction{T: relay, Tag: protocol.ProposalPayloadTag, CompoundMessage: compoundMessage{Proposal: payload.u()}} requireTraceContains(t, trace, ev(na), playerN, eventN) @@ -588,8 +593,8 @@ func verifyPermutationExpectedActions(t *testing.T, playerN int, eventN int, hel for i := 0; i < int(cert.threshold(config.Consensus[protocol.ConsensusCurrentVersion])); i++ { votes[i] = helper.MakeVerifiedVote(t, i, r, p, cert, pV) } - bun := unauthenticatedBundle{Round: r, Period: p, Proposal: pV} - ca := cryptoAction{T: verifyBundle, M: message{Bundle: bundle{U: bun, Votes: votes}, UnauthenticatedBundle: bun}, TaskIndex: 0} + bun := unauthenticatedBundle{Round: r, Period: p, Step: cert, Proposal: pV} + ca := verifyBundleAction(messageEvent{Input: message{Bundle: bundle{U: bun, Votes: votes}, UnauthenticatedBundle: bun}}, r, p, cert) requireTraceContains(t, trace, ev(ca), playerN, eventN) case softVoteVerifiedErrorEventSamePeriod, proposeVoteVerifiedErrorEventSamePeriod, bundleVerifiedErrorEvent: requireActionCount(t, trace, 1, playerN, eventN) @@ -611,7 +616,7 @@ func verifyPermutationExpectedActions(t *testing.T, playerN int, eventN int, hel case softVotePresentEventSamePeriod: requireActionCount(t, trace, 1, playerN, eventN) vvote := helper.MakeVerifiedVote(t, 0, r, p, soft, pV) - a := cryptoAction{T: verifyVote, M: message{UnauthenticatedVote: vvote.u()}, TaskIndex: 0} + a := verifyVoteAction(messageEvent{Input: message{UnauthenticatedVote: vvote.u()}}, r, p, 0) requireTraceContains(t, trace, ev(a), playerN, eventN) case proposeVoteVerifiedEventNextPeriod: requireActionCount(t, trace, 1, playerN, eventN) @@ -623,7 +628,7 @@ func verifyPermutationExpectedActions(t *testing.T, playerN int, eventN int, hel expectIgnore(t, trace, "Player should ignore proposalvvote already received: %v, event: %v", playerN, eventN) case payloadPresentEvent: requireActionCount(t, trace, 2, playerN, eventN) - ca := cryptoAction{T: verifyPayload, M: message{UnauthenticatedProposal: payload.u()}, TaskIndex: 0} + ca := verifyPayloadAction(messageEvent{Input: message{UnauthenticatedProposal: payload.u()}}, r, p, false) requireTraceContains(t, trace, ev(ca), playerN, eventN) na := networkAction{T: relay, Tag: protocol.ProposalPayloadTag, CompoundMessage: compoundMessage{Proposal: payload.u()}} requireTraceContains(t, trace, ev(na), playerN, eventN) @@ -654,8 +659,8 @@ func verifyPermutationExpectedActions(t *testing.T, playerN int, eventN int, hel for i := 0; i < int(cert.threshold(config.Consensus[protocol.ConsensusCurrentVersion])); i++ { votes[i] = helper.MakeVerifiedVote(t, i, r, p, cert, pV) } - bun := unauthenticatedBundle{Round: r, Period: p, Proposal: pV} - ca := cryptoAction{T: verifyBundle, M: message{Bundle: bundle{U: bun, Votes: votes}, UnauthenticatedBundle: bun}, TaskIndex: 0} + bun := unauthenticatedBundle{Round: r, Period: p, Step: cert, Proposal: pV} + ca := verifyBundleAction(messageEvent{Input: message{Bundle: bundle{U: bun, Votes: votes}, UnauthenticatedBundle: bun}}, r, p, cert) requireTraceContains(t, trace, ev(ca), playerN, eventN) case softVoteVerifiedErrorEventSamePeriod, proposeVoteVerifiedErrorEventSamePeriod, bundleVerifiedErrorEvent: requireActionCount(t, trace, 1, playerN, eventN) @@ -677,7 +682,7 @@ func verifyPermutationExpectedActions(t *testing.T, playerN int, eventN int, hel case softVotePresentEventSamePeriod: requireActionCount(t, trace, 1, playerN, eventN) vvote := helper.MakeVerifiedVote(t, 0, r, p, soft, pV) - a := cryptoAction{T: verifyVote, M: message{UnauthenticatedVote: vvote.u()}, TaskIndex: 0} + a := verifyVoteAction(messageEvent{Input: message{UnauthenticatedVote: vvote.u()}}, r, p, 0) requireTraceContains(t, trace, ev(a), playerN, eventN) case proposeVoteVerifiedEventNextPeriod: requireActionCount(t, trace, 1, playerN, eventN) @@ -689,7 +694,7 @@ func verifyPermutationExpectedActions(t *testing.T, playerN int, eventN int, hel expectIgnore(t, trace, "Player should ignore proposalvvote already received: %v, event: %v", playerN, eventN) case payloadPresentEvent: requireActionCount(t, trace, 2, playerN, eventN) - ca := cryptoAction{T: verifyPayload, M: message{UnauthenticatedProposal: payload.u()}, TaskIndex: 0} + ca := verifyPayloadAction(messageEvent{Input: message{UnauthenticatedProposal: payload.u()}}, r, p, false) requireTraceContains(t, trace, ev(ca), playerN, eventN) na := networkAction{T: relay, Tag: protocol.ProposalPayloadTag, CompoundMessage: compoundMessage{Proposal: payload.u()}} requireTraceContains(t, trace, ev(na), playerN, eventN) @@ -720,8 +725,8 @@ func verifyPermutationExpectedActions(t *testing.T, playerN int, eventN int, hel for i := 0; i < int(cert.threshold(config.Consensus[protocol.ConsensusCurrentVersion])); i++ { votes[i] = helper.MakeVerifiedVote(t, i, r, p, cert, pV) } - bun := unauthenticatedBundle{Round: r, Period: p, Proposal: pV} - ca := cryptoAction{T: verifyBundle, M: message{Bundle: bundle{U: bun, Votes: votes}, UnauthenticatedBundle: bun}, TaskIndex: 0} + bun := unauthenticatedBundle{Round: r, Period: p, Step: cert, Proposal: pV} + ca := verifyBundleAction(messageEvent{Input: message{Bundle: bundle{U: bun, Votes: votes}, UnauthenticatedBundle: bun}}, r, p, cert) requireTraceContains(t, trace, ev(ca), playerN, eventN) case softVoteVerifiedErrorEventSamePeriod, proposeVoteVerifiedErrorEventSamePeriod, bundleVerifiedErrorEvent: requireActionCount(t, trace, 1, playerN, eventN) @@ -743,7 +748,7 @@ func verifyPermutationExpectedActions(t *testing.T, playerN int, eventN int, hel case softVotePresentEventSamePeriod: requireActionCount(t, trace, 1, playerN, eventN) vvote := helper.MakeVerifiedVote(t, 0, r, p, soft, pV) - a := cryptoAction{T: verifyVote, M: message{UnauthenticatedVote: vvote.u()}, TaskIndex: 0} + a := verifyVoteAction(messageEvent{Input: message{UnauthenticatedVote: vvote.u()}}, r, p, 0) requireTraceContains(t, trace, ev(a), playerN, eventN) case proposeVoteVerifiedEventNextPeriod: requireActionCount(t, trace, 1, playerN, eventN) @@ -777,8 +782,8 @@ func verifyPermutationExpectedActions(t *testing.T, playerN int, eventN int, hel for i := 0; i < int(cert.threshold(config.Consensus[protocol.ConsensusCurrentVersion])); i++ { votes[i] = helper.MakeVerifiedVote(t, i, r, p, cert, pV) } - bun := unauthenticatedBundle{Round: r, Period: p, Proposal: pV} - ca := cryptoAction{T: verifyBundle, M: message{Bundle: bundle{U: bun, Votes: votes}, UnauthenticatedBundle: bun}, TaskIndex: 0} + bun := unauthenticatedBundle{Round: r, Period: p, Step: cert, Proposal: pV} + ca := verifyBundleAction(messageEvent{Input: message{Bundle: bundle{U: bun, Votes: votes}, UnauthenticatedBundle: bun}}, r, p, cert) requireTraceContains(t, trace, ev(ca), playerN, eventN) case softVoteVerifiedErrorEventSamePeriod, proposeVoteVerifiedErrorEventSamePeriod, bundleVerifiedErrorEvent: requireActionCount(t, trace, 1, playerN, eventN) diff --git a/agreement/player_test.go b/agreement/player_test.go index ecbec396a1..80f51417f5 100644 --- a/agreement/player_test.go +++ b/agreement/player_test.go @@ -1619,7 +1619,7 @@ func TestPlayerReproposesNextValueBundleWithoutPayload(t *testing.T) { // check player fast-forwarded, rezeros, reproposed, relays next-value bundle require.Equalf(t, p, pWhite.Period, "player did not fast forward to new period") - zeroEvent := ev(rezeroAction{}) + zeroEvent := ev(rezeroAction{Round: r}) require.Truef(t, pM.getTrace().Contains(zeroEvent), "Player should reset clock") reproposeEvent := ev(pseudonodeAction{T: repropose, Round: r, Period: p, Proposal: *pV}) require.Truef(t, pM.getTrace().Contains(reproposeEvent), "Player should repropose from next-value quorum") @@ -1689,7 +1689,7 @@ func TestPlayerReproposesNextValueBundleRelaysPayload(t *testing.T) { // check player fast-forwarded, rezeros, reproposed, relays next-value bundle require.Equalf(t, p, pWhite.Period, "player did not fast forward to new period") - zeroEvent := ev(rezeroAction{}) + zeroEvent := ev(rezeroAction{Round: r}) require.Truef(t, pM.getTrace().Contains(zeroEvent), "Player should reset clock") reproposeEvent := ev(pseudonodeAction{T: repropose, Round: r, Period: p, Proposal: *pV}) require.Truef(t, pM.getTrace().Contains(reproposeEvent), "Player should repropose from next-value quorum") @@ -2417,7 +2417,7 @@ func TestPlayerRequestsVoteVerification(t *testing.T) { require.NoError(t, err) require.NoError(t, panicErr) - verifyEvent := ev(cryptoAction{T: verifyVote, M: m, TaskIndex: 0}) + verifyEvent := ev(cryptoAction{T: verifyVote, M: m, Round: r, Period: p, TaskIndex: 0}) require.Truef(t, pM.getTrace().Contains(verifyEvent), "Player should verify vote") } @@ -2440,7 +2440,7 @@ func TestPlayerRequestsProposalVoteVerification(t *testing.T) { require.NoError(t, err) require.NoError(t, panicErr) - verifyEvent := ev(cryptoAction{T: verifyVote, M: m, TaskIndex: 0}) + verifyEvent := ev(cryptoAction{T: verifyVote, M: m, Round: r, Period: p, TaskIndex: 1}) require.Truef(t, pM.getTrace().Contains(verifyEvent), "Player should verify vote") } @@ -2464,7 +2464,7 @@ func TestPlayerRequestsBundleVerification(t *testing.T) { err, panicErr := pM.transition(inMsg) require.NoError(t, err) require.NoError(t, panicErr) - verifyEvent := ev(cryptoAction{T: verifyBundle, M: m, TaskIndex: 0}) + verifyEvent := ev(cryptoAction{T: verifyBundle, M: m, Round: r, Period: p, TaskIndex: 0}) require.Truef(t, pM.getTrace().Contains(verifyEvent), "Player should verify bundle") } @@ -2501,7 +2501,7 @@ func TestPlayerRequestsPayloadVerification(t *testing.T) { require.NoError(t, panicErr) // make sure payload verify request - verifyEvent := ev(cryptoAction{T: verifyPayload, M: m, TaskIndex: 0}) + verifyEvent := ev(cryptoAction{T: verifyPayload, M: m, Round: r, Period: p, TaskIndex: 0}) require.Truef(t, pM.getTrace().Contains(verifyEvent), "Player should verify payload") } @@ -2537,8 +2537,7 @@ func TestPlayerRequestsPipelinedPayloadVerification(t *testing.T) { require.NoError(t, err) require.NoError(t, panicErr) // make sure no payload verify request, because its for the next round - verifyEvent := ev(cryptoAction{T: verifyPayload, M: m, TaskIndex: 0}) - require.Falsef(t, pM.getTrace().Contains(verifyEvent), "Player should not verify payload from r + 1") + require.Falsef(t, pM.getTrace().ContainsString(verifyPayload.String()), "Player should not verify payload from r + 1") // now enter next round pP, pV := helper.MakeRandomProposalPayload(t, r) @@ -2596,7 +2595,7 @@ func TestPlayerRequestsPipelinedPayloadVerification(t *testing.T) { require.Truef(t, pM.getTrace().Contains(commitEvent), "Player should try to ensure block/digest on ledger") // make sure we sent out pipelined payload verify requests - verifyEvent = ev(cryptoAction{T: verifyPayload}) + verifyEvent := ev(cryptoAction{T: verifyPayload, Round: r + 1, TaskIndex: 0}) require.Truef(t, pM.getTrace().Contains(verifyEvent), "Player should verify pipelined payload first seen in previous round") } @@ -2711,7 +2710,7 @@ func TestPlayerHandlesPipelinedThresholds(t *testing.T) { err, panicErr = pM.transition(inMsg) require.NoError(t, err) require.NoError(t, panicErr) - verifyEvent := ev(cryptoAction{T: verifyPayload}) + verifyEvent := ev(cryptoAction{T: verifyPayload, Round: r + 1}) require.Truef(t, pM.getTrace().Contains(verifyEvent), "Player should verify pipelined payload first seen in previous round") } @@ -3225,7 +3224,7 @@ func TestPlayerAlwaysResynchsPinnedValue(t *testing.T) { // Now, player should be in period 12, and should have tried to resychronize the pinned payload trace := pM.getTrace() require.Equalf(t, p, pWhite.Period, "player did not fast forward to new period") - zeroEvent := ev(rezeroAction{}) + zeroEvent := ev(rezeroAction{Round: r}) require.Truef(t, trace.Contains(zeroEvent), "Player should reset clock") resynchEvent := ev(networkAction{T: broadcast, Tag: protocol.VoteBundleTag, UnauthenticatedBundle: bun}) @@ -3241,42 +3240,58 @@ func TestPlayerAlwaysResynchsPinnedValue(t *testing.T) { func TestPlayerRetainsReceivedValidatedAtOneSample(t *testing.T) { partitiontest.PartitionTest(t) + version := protocol.ConsensusFuture const r = round(20239) - const p = period(0) + const p = period(131) pWhite, pM, helper := setupP(t, r-1, p, soft) pP, pV := helper.MakeRandomProposalPayload(t, r-1) - // send voteVerified message - vVote := helper.MakeVerifiedVote(t, 0, r-credentialRoundLag-1, p, propose, *pV) - inMsg := messageEvent{T: voteVerified, Input: message{Vote: vVote, UnauthenticatedVote: vVote.u()}} - inMsg = inMsg.AttachValidatedAt(501*time.Millisecond, r-credentialRoundLag-1, nil) - err, panicErr := pM.transition(inMsg) - require.NoError(t, err) - require.NoError(t, panicErr) + // send voteVerified message for round r-credentialRoundLag-1, then for r-1 + sendVoteVerified(t, helper, pM, 0, r-1, r-1, p, pV, 502*time.Millisecond, nil) + // send payloadPresent message for r-1 + sendPayloadPresent(t, pM, r-1, pP, time.Second, nil) - // send voteVerified message - vVote = helper.MakeVerifiedVote(t, 0, r-1, p, propose, *pV) - inMsg = messageEvent{T: voteVerified, Input: message{Vote: vVote, UnauthenticatedVote: vVote.u()}} - inMsg = inMsg.AttachValidatedAt(501*time.Millisecond, r-1, nil) - err, panicErr = pM.transition(inMsg) - require.NoError(t, err) - require.NoError(t, panicErr) + // advance player to R and check timings ensured for R-1 are correct + moveToRound(t, pWhite, pM, helper, r, p, pP, pV, 2*time.Second, version) + assertPayloadTimings(t, pM, r-1, pV, time.Second, 2*time.Second) +} - // send payloadPresent message - m := message{UnauthenticatedProposal: pP.u()} - inMsg = messageEvent{T: payloadPresent, Input: m} - inMsg = inMsg.AttachReceivedAt(time.Second, r-1, nil) - err, panicErr = pM.transition(inMsg) - require.NoError(t, err) - require.NoError(t, panicErr) +// test that ReceivedAt and ValidateAt timing information are retained in proposalStore +// when the payloadPresent, payloadVerified, and voteVerified events are processed, and that all timings +// are available when the ensureAction is called for the block. +func TestPlayerRetainsReceivedValidatedAtCredentialHistory(t *testing.T) { + partitiontest.PartitionTest(t) - assertCorrectReceivedAtSet(t, pWhite, pM, helper, r, p, pP, pV, m, protocol.ConsensusFuture, time.Second) + version := protocol.ConsensusFuture + const r = round(20239) + const p = period(0) + pWhite, pM, helper := setupP(t, r-credentialRoundLag-1, p, soft) - // assert lowest vote validateAt time was recorded into payloadArrivals - require.NotZero(t, dynamicFilterCredentialArrivalHistory) - require.Equal(t, pWhite.lowestCredentialArrivals.writePtr, 1) - require.False(t, pWhite.lowestCredentialArrivals.isFull()) - require.Equal(t, 501*time.Millisecond, pWhite.lowestCredentialArrivals.history[0]) + // send voteVerified and payloadPresent messages with timings, and advance through rounds r-credentialRoundLag-1 up to r-1 + voteVerifiedTiming := 501 * time.Millisecond + payloadPresentTiming := 1001 * time.Millisecond + payloadVerifiedTiming := 2001 * time.Millisecond + for rnd := r - credentialRoundLag - 1; rnd < r-1; rnd++ { + pP, pV := helper.MakeRandomProposalPayload(t, rnd) + sendVoteVerified(t, helper, pM, 0, rnd, rnd, p, pV, voteVerifiedTiming, nil) + sendPayloadPresent(t, pM, rnd, pP, payloadPresentTiming, nil) + moveToRound(t, pWhite, pM, helper, rnd+1, p, pP, pV, payloadVerifiedTiming, version) + + voteVerifiedTiming += time.Millisecond + payloadPresentTiming += time.Millisecond + payloadVerifiedTiming += time.Millisecond + } + + // send in voteVerified and payloadPresent for r-1 + pP, pV := helper.MakeRandomProposalPayload(t, r-1) + sendVoteVerified(t, helper, pM, 0, r-1, r-1, p, pV, 600*time.Millisecond, nil) + sendPayloadPresent(t, pM, r-1, pP, 1500*time.Millisecond, nil) + // advance player to R and check timings ensured for R-1 are correct + moveToRound(t, pWhite, pM, helper, r, p, pP, pV, 2500*time.Millisecond, version) + assertPayloadTimings(t, pM, r-1, pV, 1500*time.Millisecond, 2500*time.Millisecond) + + // player is looking up arrival times from r-roundLag ago so only the 501ms vote will be in lowestCredentialArrivals + assertSingleCredentialArrival(t, pWhite, 501*time.Millisecond) } // test that ReceivedAt and ValidateAt timing information are retained in @@ -3285,51 +3300,35 @@ func TestPlayerRetainsReceivedValidatedAtOneSample(t *testing.T) { // available when the ensureAction is called for the block. func TestPlayerRetainsEarlyReceivedValidatedAtOneSample(t *testing.T) { partitiontest.PartitionTest(t) + + version := protocol.ConsensusFuture const r = round(20239) const p = period(0) pWhite, pM, helper := setupP(t, r-1, p, soft) // send voteVerified message pP, pV := helper.MakeRandomProposalPayload(t, r-credentialRoundLag-1) - vVote := helper.MakeVerifiedVote(t, 0, r-credentialRoundLag-1, p, propose, *pV) - inMsg := messageEvent{T: voteVerified, Input: message{Vote: vVote, UnauthenticatedVote: vVote.u()}} - inMsg = inMsg.AttachValidatedAt(401*time.Millisecond, r-credentialRoundLag-2, nil) - err, panicErr := pM.transition(inMsg) - require.NoError(t, err) - require.NoError(t, panicErr) + sendVoteVerified(t, helper, pM, 0, r-credentialRoundLag-2, r-credentialRoundLag-1, p, pV, 401*time.Millisecond, nil) // send voteVerified message pP, pV = helper.MakeRandomProposalPayload(t, r-1) - vVote = helper.MakeVerifiedVote(t, 0, r-1, p, propose, *pV) - inMsg = messageEvent{T: voteVerified, Input: message{Vote: vVote, UnauthenticatedVote: vVote.u()}} - inMsg = inMsg.AttachValidatedAt(501*time.Millisecond, r-1, nil) - err, panicErr = pM.transition(inMsg) - require.NoError(t, err) - require.NoError(t, panicErr) + sendVoteVerified(t, helper, pM, 0, r-1, r-1, p, pV, 501*time.Millisecond, nil) // send payloadPresent message - m := message{UnauthenticatedProposal: pP.u()} - inMsg = messageEvent{T: payloadPresent, Input: m} - inMsg = inMsg.AttachReceivedAt(time.Second, r-1, nil) - err, panicErr = pM.transition(inMsg) - require.NoError(t, err) - require.NoError(t, panicErr) + sendPayloadPresent(t, pM, r-1, pP, time.Second, nil) - assertCorrectReceivedAtSet(t, pWhite, pM, helper, r, p, pP, pV, m, protocol.ConsensusFuture, time.Second) + // advance player to R and check timings ensured for R-1 are correct + moveToRound(t, pWhite, pM, helper, r, p, pP, pV, 2*time.Second, version) + assertPayloadTimings(t, pM, r-1, pV, time.Second, 2*time.Second) // assert lowest vote validateAt time was recorded into payloadArrivals - require.NotZero(t, dynamicFilterCredentialArrivalHistory) - require.Equal(t, pWhite.lowestCredentialArrivals.writePtr, 1) - require.False(t, pWhite.lowestCredentialArrivals.isFull()) - require.Equal(t, pipelinedMessageTimestamp, pWhite.lowestCredentialArrivals.history[0]) + assertSingleCredentialArrival(t, pWhite, pipelinedMessageTimestamp) } -type fixedHistoricalClock struct{} - -const fixedClockDuration = 900 * time.Millisecond +type fixedHistoricalClock time.Duration -func (fixedHistoricalClock) Since() time.Duration { - return fixedClockDuration +func (f fixedHistoricalClock) Since() time.Duration { + return time.Duration(f) } // test that ReceivedAt and ValidateAt timing information are retained in @@ -3338,47 +3337,33 @@ func (fixedHistoricalClock) Since() time.Duration { // that all timings are available when the ensureAction is called for the block. func TestPlayerRetainsLateReceivedValidatedAtOneSample(t *testing.T) { partitiontest.PartitionTest(t) + + version := protocol.ConsensusFuture const r = round(20239) const p = period(0) pWhite, pM, helper := setupP(t, r-1, p, soft) historicalClocks := map[round]historicalClock{ - r - credentialRoundLag - 1: fixedHistoricalClock{}, + r - credentialRoundLag - 1: fixedHistoricalClock(900 * time.Millisecond), } // send voteVerified message pP, pV := helper.MakeRandomProposalPayload(t, r-credentialRoundLag-1) - vVote := helper.MakeVerifiedVote(t, 0, r-credentialRoundLag-1, p, propose, *pV) - inMsg := messageEvent{T: voteVerified, Input: message{Vote: vVote, UnauthenticatedVote: vVote.u()}} - inMsg = inMsg.AttachValidatedAt(401*time.Millisecond, r-1, historicalClocks) - err, panicErr := pM.transition(inMsg) - require.NoError(t, err) - require.NoError(t, panicErr) + sendVoteVerified(t, helper, pM, 0, r-1, r-credentialRoundLag-1, p, pV, 401*time.Millisecond, historicalClocks) // send voteVerified message pP, pV = helper.MakeRandomProposalPayload(t, r-1) - vVote = helper.MakeVerifiedVote(t, 0, r-1, p, propose, *pV) - inMsg = messageEvent{T: voteVerified, Input: message{Vote: vVote, UnauthenticatedVote: vVote.u()}} - inMsg = inMsg.AttachValidatedAt(501*time.Millisecond, r-1, nil) - err, panicErr = pM.transition(inMsg) - require.NoError(t, err) - require.NoError(t, panicErr) + sendVoteVerified(t, helper, pM, 0, r-1, r-1, p, pV, 501*time.Millisecond, nil) // send payloadPresent message - m := message{UnauthenticatedProposal: pP.u()} - inMsg = messageEvent{T: payloadPresent, Input: m} - inMsg = inMsg.AttachReceivedAt(time.Second, r-1, nil) - err, panicErr = pM.transition(inMsg) - require.NoError(t, err) - require.NoError(t, panicErr) + sendPayloadPresent(t, pM, r-1, pP, time.Second, nil) - assertCorrectReceivedAtSet(t, pWhite, pM, helper, r, p, pP, pV, m, protocol.ConsensusFuture, time.Second) + // advance player to R and check timings ensured for R-1 are correct + moveToRound(t, pWhite, pM, helper, r, p, pP, pV, 2*time.Second, version) + assertPayloadTimings(t, pM, r-1, pV, time.Second, 2*time.Second) // assert lowest vote validateAt time was recorded into payloadArrivals - require.NotZero(t, dynamicFilterCredentialArrivalHistory) - require.Equal(t, pWhite.lowestCredentialArrivals.writePtr, 1) - require.False(t, pWhite.lowestCredentialArrivals.isFull()) - require.Equal(t, fixedClockDuration, pWhite.lowestCredentialArrivals.history[0]) + assertSingleCredentialArrival(t, pWhite, 900*time.Millisecond) } // test that ReceivedAt and ValidateAt timing information are retained in proposalStore @@ -3396,11 +3381,12 @@ func TestPlayerRetainsReceivedValidatedAtForHistoryWindowLateBetter(t *testing.T } func testPlayerRetainsReceivedValidatedAtForHistoryWindow(t *testing.T, addBetterLate bool) { + version := protocol.ConsensusFuture const r = round(20239) const p = period(0) pWhite, pM, helper := setupP(t, r-1, p, soft) - require.NotNil(t, dynamicFilterCredentialArrivalHistory) + require.NotZero(t, dynamicFilterCredentialArrivalHistory) for i := 0; i < dynamicFilterCredentialArrivalHistory+int(credentialRoundLag); i++ { // send voteVerified message @@ -3429,13 +3415,8 @@ func testPlayerRetainsReceivedValidatedAtForHistoryWindow(t *testing.T, addBette require.NoError(t, panicErr) // send payloadPresent message - m := message{UnauthenticatedProposal: pP.u()} - inMsg = messageEvent{T: payloadPresent, Input: m} - inMsg = inMsg.AttachReceivedAt(time.Second, r+round(i)-1, nil) - err, panicErr = pM.transition(inMsg) - require.NoError(t, err) - require.NoError(t, panicErr) - moveToRound(t, pWhite, pM, helper, r+round(i), p, pP, pV, m, protocol.ConsensusFuture) + sendPayloadPresent(t, pM, r+round(i)-1, pP, time.Second, nil) + moveToRound(t, pWhite, pM, helper, r+round(i), p, pP, pV, 2*time.Second, version) // send better late voteVerified message if addBetterLate { @@ -3467,6 +3448,8 @@ func testPlayerRetainsReceivedValidatedAtForHistoryWindow(t *testing.T, addBette func TestPlayerRetainsReceivedValidatedAtPPOneSample(t *testing.T) { partitiontest.PartitionTest(t) + version, _, configCleanup := createDynamicFilterConfig() + defer configCleanup() const r = round(20239) const p = period(0) pWhite, pM, helper := setupP(t, r-1, p, soft) @@ -3474,73 +3457,43 @@ func TestPlayerRetainsReceivedValidatedAtPPOneSample(t *testing.T) { // Move to round r, no credentials arrived. // send voteVerified message - vVote := helper.MakeVerifiedVote(t, 0, r-1, p, propose, *pV) - inMsg := messageEvent{T: voteVerified, Input: message{Vote: vVote, UnauthenticatedVote: vVote.u()}} - inMsg = inMsg.AttachValidatedAt(501*time.Millisecond, r-1, nil) - err, panicErr := pM.transition(inMsg) - require.NoError(t, err) - require.NoError(t, panicErr) + sendVoteVerified(t, helper, pM, 0, r-1, r-1, p, pV, 501*time.Millisecond, nil) // send payloadPresent message - m := message{UnauthenticatedProposal: pP.u()} - inMsg = messageEvent{T: payloadPresent, Input: m} - inMsg = inMsg.AttachReceivedAt(time.Second, r-1, nil) - err, panicErr = pM.transition(inMsg) - require.NoError(t, err) - require.NoError(t, panicErr) + sendPayloadPresent(t, pM, r-1, pP, time.Second, nil) - assertCorrectReceivedAtSet(t, pWhite, pM, helper, r, p, pP, pV, m, protocol.ConsensusFuture, time.Second) + moveToRound(t, pWhite, pM, helper, r, p, pP, pV, 2*time.Second, version) + assertPayloadTimings(t, pM, r-1, pV, time.Second, 2*time.Second) require.False(t, pWhite.lowestCredentialArrivals.isFull()) require.Equal(t, pWhite.lowestCredentialArrivals.writePtr, 0) + // XXX this behavior only happens if dynamic timeout enabled; test the other way + + historicalClocks := map[round]historicalClock{ + r - credentialRoundLag: fixedHistoricalClock(900 * time.Millisecond), + } // create a PP message for the round we're going to take the sample from when round r-1 ends - vVote = helper.MakeVerifiedVote(t, 0, r-credentialRoundLag, p, propose, *pV) - unverifiedVoteMsg := message{UnauthenticatedVote: vVote.u()} - proposalMsg := message{UnauthenticatedProposal: pP.u()} - compoundMsg := messageEvent{T: votePresent, Input: unverifiedVoteMsg, - Tail: &messageEvent{T: payloadPresent, Input: proposalMsg}} - inMsg = compoundMsg.AttachReceivedAt(time.Second, r-credentialRoundLag, nil) // call AttachReceivedAt like demux would - err, panicErr = pM.transition(inMsg) - require.NoError(t, err) - require.NoError(t, panicErr) + pP, pV = helper.MakeRandomProposalPayload(t, r-credentialRoundLag) + vVote := sendCompoundMessage(t, helper, pM, r, r-credentialRoundLag, p, pP, pV, time.Second, nil, version) - verifiedVoteMsg := message{Vote: vVote, UnauthenticatedVote: vVote.u()} - inMsg = messageEvent{T: voteVerified, Input: verifiedVoteMsg, TaskIndex: 1} - inMsg = inMsg.AttachValidatedAt(502*time.Millisecond, r-credentialRoundLag, nil) - err, panicErr = pM.transition(inMsg) - require.NoError(t, err) - require.NoError(t, panicErr) + verifyEvent := ev(verifyVoteAction(messageEvent{Input: message{UnauthenticatedVote: vVote.u()}}, r-credentialRoundLag, p, 1)) + require.Truef(t, pM.getTrace().Contains(verifyEvent), "Player should verify vote") + sendVoteVerifiedForVote(t, vVote, pM, r, 502*time.Millisecond, historicalClocks, 1) // send payloadPresent message - m = message{UnauthenticatedProposal: pP.u()} - inMsg = messageEvent{T: payloadPresent, Input: m} - inMsg = inMsg.AttachReceivedAt(time.Second, r-credentialRoundLag, nil) - err, panicErr = pM.transition(inMsg) - require.NoError(t, err) - require.NoError(t, panicErr) + sendPayloadPresent(t, pM, r, pP, time.Second, nil) // move to round r+1, triggering history update - vVote = helper.MakeVerifiedVote(t, 0, r, p, propose, *pV) - inMsg = messageEvent{T: voteVerified, Input: message{Vote: vVote, UnauthenticatedVote: vVote.u()}} - inMsg = inMsg.AttachValidatedAt(501*time.Millisecond, r, nil) - err, panicErr = pM.transition(inMsg) - require.NoError(t, err) - require.NoError(t, panicErr) + pP, pV = helper.MakeRandomProposalPayload(t, r) + sendVoteVerified(t, helper, pM, 0, r, r, p, pV, 501*time.Millisecond, nil) // send payloadPresent message - m = message{UnauthenticatedProposal: pP.u()} - inMsg = messageEvent{T: payloadPresent, Input: m} - inMsg = inMsg.AttachReceivedAt(time.Second, r, nil) - err, panicErr = pM.transition(inMsg) - require.NoError(t, err) - require.NoError(t, panicErr) - moveToRound(t, pWhite, pM, helper, r+1, p, pP, pV, m, protocol.ConsensusFuture) + sendPayloadPresent(t, pM, r, pP, time.Second, nil) + + moveToRound(t, pWhite, pM, helper, r+1, p, pP, pV, 2*time.Second, version) // assert lowest vote validateAt time was recorded into payloadArrivals - require.NotNil(t, dynamicFilterCredentialArrivalHistory) - require.Equal(t, pWhite.lowestCredentialArrivals.writePtr, 1) - require.False(t, pWhite.lowestCredentialArrivals.isFull()) - require.Equal(t, 502*time.Millisecond, pWhite.lowestCredentialArrivals.history[0]) + assertSingleCredentialArrival(t, pWhite, 900*time.Millisecond) } // test that ReceivedAt and ValidateAt timing information are retained in @@ -3550,6 +3503,10 @@ func TestPlayerRetainsReceivedValidatedAtPPOneSample(t *testing.T) { // ensureAction is called for the block. func TestPlayerRetainsEarlyReceivedValidatedAtPPOneSample(t *testing.T) { partitiontest.PartitionTest(t) + + version, _, configCleanup := createDynamicFilterConfig() + defer configCleanup() + const r = round(20239) const p = period(0) pWhite, pM, helper := setupP(t, r-1, p, soft) @@ -3557,74 +3514,40 @@ func TestPlayerRetainsEarlyReceivedValidatedAtPPOneSample(t *testing.T) { // Move to round r, no credentials arrived. // send voteVerified message - vVote := helper.MakeVerifiedVote(t, 0, r-1, p, propose, *pV) - inMsg := messageEvent{T: voteVerified, Input: message{Vote: vVote, UnauthenticatedVote: vVote.u()}} - inMsg = inMsg.AttachValidatedAt(501*time.Millisecond, r-1, nil) - err, panicErr := pM.transition(inMsg) - require.NoError(t, err) - require.NoError(t, panicErr) + sendVoteVerified(t, helper, pM, 0, r-1, r-1, p, pV, 501*time.Millisecond, nil) // send payloadPresent message - m := message{UnauthenticatedProposal: pP.u()} - inMsg = messageEvent{T: payloadPresent, Input: m} - inMsg = inMsg.AttachReceivedAt(time.Second, r-1, nil) - err, panicErr = pM.transition(inMsg) - require.NoError(t, err) - require.NoError(t, panicErr) + sendPayloadPresent(t, pM, r-1, pP, time.Second, nil) - assertCorrectReceivedAtSet(t, pWhite, pM, helper, r, p, pP, pV, m, protocol.ConsensusFuture, time.Second) + moveToRound(t, pWhite, pM, helper, r, p, pP, pV, 2*time.Second, version) + assertPayloadTimings(t, pM, r-1, pV, time.Second, 2*time.Second) require.False(t, pWhite.lowestCredentialArrivals.isFull()) require.Equal(t, pWhite.lowestCredentialArrivals.writePtr, 0) // create a PP message for the round we're going to take the sample from when round r-1 ends // Now we're going to pretend we got the message one round early. - vVote = helper.MakeVerifiedVote(t, 0, r-credentialRoundLag, p, propose, *pV) - unverifiedVoteMsg := message{UnauthenticatedVote: vVote.u()} - proposalMsg := message{UnauthenticatedProposal: pP.u()} - compoundMsg := messageEvent{T: votePresent, Input: unverifiedVoteMsg, - Tail: &messageEvent{T: payloadPresent, Input: proposalMsg}} - inMsg = compoundMsg.AttachReceivedAt(time.Second, r-credentialRoundLag-1, nil) // call AttachReceivedAt like demux would - err, panicErr = pM.transition(inMsg) - require.NoError(t, err) - require.NoError(t, panicErr) + pP, pV = helper.MakeRandomProposalPayload(t, r-credentialRoundLag) + vVote := sendCompoundMessage(t, helper, pM, r-credentialRoundLag-1, r-credentialRoundLag, p, pP, pV, time.Second, nil, version) - verifiedVoteMsg := message{Vote: vVote, UnauthenticatedVote: vVote.u()} - inMsg = messageEvent{T: voteVerified, Input: verifiedVoteMsg, TaskIndex: 1} - inMsg = inMsg.AttachValidatedAt(502*time.Millisecond, r-credentialRoundLag, nil) - err, panicErr = pM.transition(inMsg) - require.NoError(t, err) - require.NoError(t, panicErr) + // make sure vote verify requests + verifyEvent := ev(verifyVoteAction(messageEvent{Input: message{UnauthenticatedVote: vVote.u()}}, r-credentialRoundLag, p, 1)) + require.Truef(t, pM.getTrace().Contains(verifyEvent), "Player should verify vote") + + sendVoteVerifiedForVote(t, vVote, pM, r-credentialRoundLag, 502*time.Millisecond, nil, 1) // send payloadPresent message - m = message{UnauthenticatedProposal: pP.u()} - inMsg = messageEvent{T: payloadPresent, Input: m} - inMsg = inMsg.AttachReceivedAt(time.Second, r-credentialRoundLag, nil) - err, panicErr = pM.transition(inMsg) - require.NoError(t, err) - require.NoError(t, panicErr) + sendPayloadPresent(t, pM, r-credentialRoundLag, pP, time.Second, nil) // move to round r+1, triggering history update - vVote = helper.MakeVerifiedVote(t, 0, r, p, propose, *pV) - inMsg = messageEvent{T: voteVerified, Input: message{Vote: vVote, UnauthenticatedVote: vVote.u()}} - inMsg = inMsg.AttachValidatedAt(501*time.Millisecond, r, nil) - err, panicErr = pM.transition(inMsg) - require.NoError(t, err) - require.NoError(t, panicErr) + pP, pV = helper.MakeRandomProposalPayload(t, r) + sendVoteVerified(t, helper, pM, 0, r, r, p, pV, 501*time.Millisecond, nil) // send payloadPresent message - m = message{UnauthenticatedProposal: pP.u()} - inMsg = messageEvent{T: payloadPresent, Input: m} - inMsg = inMsg.AttachReceivedAt(time.Second, r, nil) - err, panicErr = pM.transition(inMsg) - require.NoError(t, err) - require.NoError(t, panicErr) - moveToRound(t, pWhite, pM, helper, r+1, p, pP, pV, m, protocol.ConsensusFuture) + sendPayloadPresent(t, pM, r, pP, time.Second, nil) + moveToRound(t, pWhite, pM, helper, r+1, p, pP, pV, 2*time.Second, version) // assert lowest vote validateAt time was recorded into payloadArrivals - require.NotNil(t, dynamicFilterCredentialArrivalHistory) - require.Equal(t, pWhite.lowestCredentialArrivals.writePtr, 1) - require.False(t, pWhite.lowestCredentialArrivals.isFull()) - require.Equal(t, 502*time.Millisecond, pWhite.lowestCredentialArrivals.history[0]) + assertSingleCredentialArrival(t, pWhite, 502*time.Millisecond) } // test that ReceivedAt and ValidateAt timing information are retained in @@ -3635,6 +3558,8 @@ func TestPlayerRetainsEarlyReceivedValidatedAtPPOneSample(t *testing.T) { func TestPlayerRetainsLateReceivedValidatedAtPPOneSample(t *testing.T) { partitiontest.PartitionTest(t) + version, _, configCleanup := createDynamicFilterConfig() + defer configCleanup() const r = round(20239) const p = period(0) pWhite, pM, helper := setupP(t, r-1, p, soft) @@ -3642,77 +3567,40 @@ func TestPlayerRetainsLateReceivedValidatedAtPPOneSample(t *testing.T) { // Move to round r, no credentials arrived. // send voteVerified message - vVote := helper.MakeVerifiedVote(t, 0, r-1, p, propose, *pV) - inMsg := messageEvent{T: voteVerified, Input: message{Vote: vVote, UnauthenticatedVote: vVote.u()}} - inMsg = inMsg.AttachValidatedAt(501*time.Millisecond, r-1, nil) - err, panicErr := pM.transition(inMsg) - require.NoError(t, err) - require.NoError(t, panicErr) + sendVoteVerified(t, helper, pM, 0, r-1, r-1, p, pV, 501*time.Millisecond, nil) // send payloadPresent message - m := message{UnauthenticatedProposal: pP.u()} - inMsg = messageEvent{T: payloadPresent, Input: m} - inMsg = inMsg.AttachReceivedAt(time.Second, r-1, nil) - err, panicErr = pM.transition(inMsg) - require.NoError(t, err) - require.NoError(t, panicErr) + sendPayloadPresent(t, pM, r-1, pP, time.Second, nil) - assertCorrectReceivedAtSet(t, pWhite, pM, helper, r, p, pP, pV, m, protocol.ConsensusFuture, time.Second) + // Go from round r-1 to r + moveToRound(t, pWhite, pM, helper, r, p, pP, pV, 2*time.Second, version) + assertPayloadTimings(t, pM, r-1, pV, time.Second, 2*time.Second) require.False(t, pWhite.lowestCredentialArrivals.isFull()) require.Equal(t, pWhite.lowestCredentialArrivals.writePtr, 0) historicalClocks := map[round]historicalClock{ - r - credentialRoundLag: fixedHistoricalClock{}, + r - credentialRoundLag: fixedHistoricalClock(900 * time.Millisecond), } // create a PP message for the round we're going to take the sample from when round r-1 ends // Now we're going to pretend we got the message credentialRoundLag too late. - vVote = helper.MakeVerifiedVote(t, 0, r-credentialRoundLag, p, propose, *pV) - unverifiedVoteMsg := message{UnauthenticatedVote: vVote.u()} - proposalMsg := message{UnauthenticatedProposal: pP.u()} - compoundMsg := messageEvent{T: votePresent, Input: unverifiedVoteMsg, - Tail: &messageEvent{T: payloadPresent, Input: proposalMsg}} - inMsg = compoundMsg.AttachReceivedAt(time.Second, r, historicalClocks) // call AttachReceivedAt like demux would - err, panicErr = pM.transition(inMsg) - require.NoError(t, err) - require.NoError(t, panicErr) + pP, pV = helper.MakeRandomProposalPayload(t, r-credentialRoundLag) + vVote := sendCompoundMessage(t, helper, pM, r, r-credentialRoundLag, p, pP, pV, time.Second, historicalClocks, version) - verifiedVoteMsg := message{Vote: vVote, UnauthenticatedVote: vVote.u()} - inMsg = messageEvent{T: voteVerified, Input: verifiedVoteMsg, TaskIndex: 1} - inMsg = inMsg.AttachValidatedAt(502*time.Millisecond, r, historicalClocks) - err, panicErr = pM.transition(inMsg) - require.NoError(t, err) - require.NoError(t, panicErr) + verifyEvent := ev(verifyVoteAction(messageEvent{Input: message{UnauthenticatedVote: vVote.u()}}, r-credentialRoundLag, p, 1)) + require.Truef(t, pM.getTrace().Contains(verifyEvent), "Player should verify vote") - // send payloadPresent message - m = message{UnauthenticatedProposal: pP.u()} - inMsg = messageEvent{T: payloadPresent, Input: m} - inMsg = inMsg.AttachReceivedAt(time.Second, r-credentialRoundLag, historicalClocks) - err, panicErr = pM.transition(inMsg) - require.NoError(t, err) - require.NoError(t, panicErr) + sendVoteVerifiedForVote(t, vVote, pM, r, 502*time.Millisecond, historicalClocks, 1) // move to round r+1, triggering history update - vVote = helper.MakeVerifiedVote(t, 0, r, p, propose, *pV) - inMsg = messageEvent{T: voteVerified, Input: message{Vote: vVote, UnauthenticatedVote: vVote.u()}} - inMsg = inMsg.AttachValidatedAt(501*time.Millisecond, r, nil) - err, panicErr = pM.transition(inMsg) - require.NoError(t, err) - require.NoError(t, panicErr) + pP, pV = helper.MakeRandomProposalPayload(t, r) + sendVoteVerified(t, helper, pM, 0, r, r, p, pV, 503*time.Millisecond, nil) // send payloadPresent message - m = message{UnauthenticatedProposal: pP.u()} - inMsg = messageEvent{T: payloadPresent, Input: m} - inMsg = inMsg.AttachReceivedAt(time.Second, r, nil) - err, panicErr = pM.transition(inMsg) - require.NoError(t, err) - require.NoError(t, panicErr) - moveToRound(t, pWhite, pM, helper, r+1, p, pP, pV, m, protocol.ConsensusFuture) + sendPayloadPresent(t, pM, r, pP, time.Second, nil) + moveToRound(t, pWhite, pM, helper, r+1, p, pP, pV, 2*time.Second, version) // assert lowest vote validateAt time was recorded into payloadArrivals - require.NotNil(t, dynamicFilterCredentialArrivalHistory) - require.Equal(t, pWhite.lowestCredentialArrivals.writePtr, 1) - require.False(t, pWhite.lowestCredentialArrivals.isFull()) - require.Equal(t, fixedClockDuration, pWhite.lowestCredentialArrivals.history[0]) + assertSingleCredentialArrival(t, pWhite, 900*time.Millisecond) } // test that ReceivedAt and ValidateAt timing information are retained in @@ -3723,39 +3611,29 @@ func TestPlayerRetainsLateReceivedValidatedAtPPOneSample(t *testing.T) { // DynamicFilterCredentialArrivalHistory rounds. func TestPlayerRetainsReceivedValidatedAtPPForHistoryWindow(t *testing.T) { partitiontest.PartitionTest(t) + + version := protocol.ConsensusFuture const r = round(20239) const p = period(0) pWhite, pM, helper := setupP(t, r-1, p, soft) - pP, pV := helper.MakeRandomProposalPayload(t, r-1) - require.NotNil(t, dynamicFilterCredentialArrivalHistory) + require.NotZero(t, dynamicFilterCredentialArrivalHistory) for i := 0; i < dynamicFilterCredentialArrivalHistory+int(credentialRoundLag); i++ { // create a PP message for an arbitrary proposal/payload similar to setupCompoundMessage - vVote := helper.MakeVerifiedVote(t, 0, r+round(i)-1, p, propose, *pV) - unverifiedVoteMsg := message{UnauthenticatedVote: vVote.u()} - proposalMsg := message{UnauthenticatedProposal: pP.u()} - compoundMsg := messageEvent{T: votePresent, Input: unverifiedVoteMsg, - Tail: &messageEvent{T: payloadPresent, Input: proposalMsg}} - - inMsg := compoundMsg.AttachReceivedAt(time.Second, r+round(i)-1, nil) // call AttachReceivedAt like demux would - err, panicErr := pM.transition(inMsg) - require.NoError(t, err) - require.NoError(t, panicErr) + pP, pV := helper.MakeRandomProposalPayload(t, r+round(i)-1) + vVote := sendCompoundMessage(t, helper, pM, r+round(i)-1, r+round(i)-1, p, pP, pV, time.Second, nil, version) // make sure vote verify requests - verifyEvent := ev(cryptoAction{T: verifyVote, M: unverifiedVoteMsg, Round: r + round(i) - 1, Period: p, Step: propose, TaskIndex: 1}) + taskIndex := uint64(i + 1) + verifyEvent := ev(verifyVoteAction(messageEvent{Input: message{UnauthenticatedVote: vVote.u()}}, r+round(i)-1, p, taskIndex)) require.Truef(t, pM.getTrace().Contains(verifyEvent), "Player should verify vote") // send voteVerified - verifiedVoteMsg := message{Vote: vVote, UnauthenticatedVote: vVote.u()} - inMsg = messageEvent{T: voteVerified, Input: verifiedVoteMsg, TaskIndex: 1} timestamp := 500 + i - inMsg = inMsg.AttachValidatedAt(time.Duration(timestamp)*time.Millisecond, r+round(i)-1, nil) - err, panicErr = pM.transition(inMsg) - require.NoError(t, err) - require.NoError(t, panicErr) - moveToRound(t, pWhite, pM, helper, r+round(i), p, pP, pV, proposalMsg, protocol.ConsensusFuture) + sendVoteVerifiedForVote(t, vVote, pM, r+round(i)-1, time.Duration(timestamp)*time.Millisecond, nil, taskIndex) + sendPayloadPresent(t, pM, r+round(i)-1, pP, time.Duration(timestamp)*time.Millisecond+time.Second, nil) + moveToRound(t, pWhite, pM, helper, r+round(i), p, pP, pV, 2*time.Second+time.Duration(timestamp)*time.Millisecond, version) } // assert lowest vote validateAt time was recorded into payloadArrivals @@ -3778,7 +3656,6 @@ func TestPlayerRetainsReceivedValidatedAtAVPPOneSample(t *testing.T) { // create a protocol version where dynamic lambda is enabled version, _, configCleanup := createDynamicFilterConfig() defer configCleanup() - const r = round(20239) const p = period(0) pWhite, pM, helper := setupP(t, r-1, p, soft) @@ -3786,76 +3663,41 @@ func TestPlayerRetainsReceivedValidatedAtAVPPOneSample(t *testing.T) { // Move to round r, no credentials arrived. // send voteVerified message - vVote := helper.MakeVerifiedVote(t, 0, r-1, p, propose, *pV) - inMsg := messageEvent{T: voteVerified, Input: message{Vote: vVote, UnauthenticatedVote: vVote.u()}} - inMsg = inMsg.AttachValidatedAt(501*time.Millisecond, r-1, nil) - err, panicErr := pM.transition(inMsg) - require.NoError(t, err) - require.NoError(t, panicErr) + sendVoteVerified(t, helper, pM, 0, r-1, r-1, p, pV, 501*time.Millisecond, nil) // send payloadPresent message - m := message{UnauthenticatedProposal: pP.u()} - inMsg = messageEvent{T: payloadPresent, Input: m} - inMsg = inMsg.AttachReceivedAt(time.Second, r-1, nil) - err, panicErr = pM.transition(inMsg) - require.NoError(t, err) - require.NoError(t, panicErr) + sendPayloadPresent(t, pM, r-1, pP, time.Second, nil) - assertCorrectReceivedAtSet(t, pWhite, pM, helper, r, p, pP, pV, m, version, time.Second) + moveToRound(t, pWhite, pM, helper, r, p, pP, pV, 2*time.Second, version) + assertPayloadTimings(t, pM, r-1, pV, time.Second, 2*time.Second) require.False(t, pWhite.lowestCredentialArrivals.isFull()) require.Equal(t, pWhite.lowestCredentialArrivals.writePtr, 0) // send votePresent message (mimicking the first AV message validating) - vVote = helper.MakeVerifiedVote(t, 0, r-credentialRoundLag, p, propose, *pV) - unverifiedVoteMsg := message{UnauthenticatedVote: vVote.u()} - inMsg = messageEvent{T: votePresent, Input: unverifiedVoteMsg, Proto: ConsensusVersionView{Version: version}} - err, panicErr = pM.transition(inMsg) - require.NoError(t, err) - require.NoError(t, panicErr) + pP, pV = helper.MakeRandomProposalPayload(t, r-credentialRoundLag) + vVote := sendVotePresent(t, helper, pM, 0, r-credentialRoundLag, p, pV, version) // make sure vote verify requests - verifyEvent := ev(cryptoAction{T: verifyVote, M: unverifiedVoteMsg, Round: r - 1, Period: p, Step: propose, TaskIndex: 1}) + unverifiedVoteMsg := message{UnauthenticatedVote: vVote.u()} + verifyEvent := ev(verifyVoteAction(messageEvent{Input: unverifiedVoteMsg}, r-credentialRoundLag, p, 1)) require.Truef(t, pM.getTrace().Contains(verifyEvent), "Player should verify vote") // send voteVerified - verifiedVoteMsg := message{Vote: vVote, UnauthenticatedVote: vVote.u()} - inMsg = messageEvent{T: voteVerified, Input: verifiedVoteMsg, TaskIndex: 1} - inMsg = inMsg.AttachValidatedAt(502*time.Millisecond, r-credentialRoundLag, nil) - err, panicErr = pM.transition(inMsg) - require.NoError(t, err) - require.NoError(t, panicErr) + sendVoteVerifiedForVote(t, vVote, pM, r-credentialRoundLag, 502*time.Millisecond, nil, 1) // create a PP message for an arbitrary proposal/payload similar to setupCompoundMessage - proposalMsg := message{UnauthenticatedProposal: pP.u()} - compoundMsg := messageEvent{T: votePresent, Input: unverifiedVoteMsg, - Tail: &messageEvent{T: payloadPresent, Input: proposalMsg}} - inMsg = compoundMsg.AttachReceivedAt(time.Second, r-credentialRoundLag, nil) // call AttachReceivedAt like demux would - err, panicErr = pM.transition(inMsg) - require.NoError(t, err) - require.NoError(t, panicErr) + sendCompoundMessageForVote(t, vVote, pM, r-credentialRoundLag, pP, time.Second, nil, version) // move to round r+1, triggering history update - vVote = helper.MakeVerifiedVote(t, 0, r, p, propose, *pV) - inMsg = messageEvent{T: voteVerified, Input: message{Vote: vVote, UnauthenticatedVote: vVote.u()}} - inMsg = inMsg.AttachValidatedAt(time.Second, r, nil) - err, panicErr = pM.transition(inMsg) - require.NoError(t, err) - require.NoError(t, panicErr) + pP, pV = helper.MakeRandomProposalPayload(t, r) + sendVoteVerified(t, helper, pM, 0, r, r, p, pV, time.Second, nil) // send payloadPresent message - m = message{UnauthenticatedProposal: pP.u()} - inMsg = messageEvent{T: payloadPresent, Input: m} - inMsg = inMsg.AttachReceivedAt(time.Second, r, nil) - err, panicErr = pM.transition(inMsg) - require.NoError(t, err) - require.NoError(t, panicErr) - moveToRound(t, pWhite, pM, helper, r+1, p, pP, pV, m, protocol.ConsensusFuture) + sendPayloadPresent(t, pM, r, pP, time.Second, nil) + moveToRound(t, pWhite, pM, helper, r+1, p, pP, pV, 2*time.Second, version) // assert lowest vote validateAt time was recorded into payloadArrivals - require.NotNil(t, dynamicFilterCredentialArrivalHistory) - require.Equal(t, pWhite.lowestCredentialArrivals.writePtr, 1) - require.False(t, pWhite.lowestCredentialArrivals.isFull()) - require.Equal(t, 502*time.Millisecond, pWhite.lowestCredentialArrivals.history[0]) + assertSingleCredentialArrival(t, pWhite, 502*time.Millisecond) } // test that ReceivedAt and ValidateAt timing information are retained in @@ -3867,6 +3709,7 @@ func TestPlayerRetainsReceivedValidatedAtAVPPOneSample(t *testing.T) { func TestPlayerRetainsEarlyReceivedValidatedAtAVPPOneSample(t *testing.T) { partitiontest.PartitionTest(t) + version := protocol.ConsensusFuture const r = round(20239) const p = period(0) pWhite, pM, helper := setupP(t, r-1, p, soft) @@ -3874,22 +3717,13 @@ func TestPlayerRetainsEarlyReceivedValidatedAtAVPPOneSample(t *testing.T) { // Move to round r, no credentials arrived. // send voteVerified message - vVote := helper.MakeVerifiedVote(t, 0, r-1, p, propose, *pV) - inMsg := messageEvent{T: voteVerified, Input: message{Vote: vVote, UnauthenticatedVote: vVote.u()}} - inMsg = inMsg.AttachValidatedAt(501*time.Millisecond, r-1, nil) - err, panicErr := pM.transition(inMsg) - require.NoError(t, err) - require.NoError(t, panicErr) + sendVoteVerified(t, helper, pM, 0, r-1, r-1, p, pV, 501*time.Millisecond, nil) // send payloadPresent message - m := message{UnauthenticatedProposal: pP.u()} - inMsg = messageEvent{T: payloadPresent, Input: m} - inMsg = inMsg.AttachReceivedAt(time.Second, r-1, nil) - err, panicErr = pM.transition(inMsg) - require.NoError(t, err) - require.NoError(t, panicErr) + sendPayloadPresent(t, pM, r-1, pP, time.Second, nil) - assertCorrectReceivedAtSet(t, pWhite, pM, helper, r, p, pP, pV, m, protocol.ConsensusFuture, time.Second) + moveToRound(t, pWhite, pM, helper, r, p, pP, pV, 2*time.Second, version) + assertPayloadTimings(t, pM, r-1, pV, time.Second, 2*time.Second) require.False(t, pWhite.lowestCredentialArrivals.isFull()) require.Equal(t, pWhite.lowestCredentialArrivals.writePtr, 0) @@ -3898,56 +3732,29 @@ func TestPlayerRetainsEarlyReceivedValidatedAtAVPPOneSample(t *testing.T) { defer configCleanup() // send votePresent message (mimicking the first AV message validating) - vVote = helper.MakeVerifiedVote(t, 0, r-credentialRoundLag, p, propose, *pV) - unverifiedVoteMsg := message{UnauthenticatedVote: vVote.u()} - inMsg = messageEvent{T: votePresent, Input: unverifiedVoteMsg, Proto: ConsensusVersionView{Version: version}} - err, panicErr = pM.transition(inMsg) - require.NoError(t, err) - require.NoError(t, panicErr) + pP, pV = helper.MakeRandomProposalPayload(t, r-credentialRoundLag) + vVote := sendVotePresent(t, helper, pM, 0, r-credentialRoundLag, p, pV, version) // make sure vote verify requests - verifyEvent := ev(cryptoAction{T: verifyVote, M: unverifiedVoteMsg, Round: r - 1, Period: p, Step: propose, TaskIndex: 1}) + verifyEvent := ev(verifyVoteAction(messageEvent{Input: message{UnauthenticatedVote: vVote.u()}}, r-credentialRoundLag, p, 1)) require.Truef(t, pM.getTrace().Contains(verifyEvent), "Player should verify vote") // send voteVerified, pretend we're one round too early - verifiedVoteMsg := message{Vote: vVote, UnauthenticatedVote: vVote.u()} - inMsg = messageEvent{T: voteVerified, Input: verifiedVoteMsg, TaskIndex: 1} - inMsg = inMsg.AttachValidatedAt(502*time.Millisecond, r-credentialRoundLag-1, nil) - err, panicErr = pM.transition(inMsg) - require.NoError(t, err) - require.NoError(t, panicErr) + sendVoteVerifiedForVote(t, vVote, pM, r-credentialRoundLag-1, 502*time.Millisecond, nil, 1) // create a PP message for an arbitrary proposal/payload similar to setupCompoundMessage - proposalMsg := message{UnauthenticatedProposal: pP.u()} - compoundMsg := messageEvent{T: votePresent, Input: unverifiedVoteMsg, - Tail: &messageEvent{T: payloadPresent, Input: proposalMsg}} - inMsg = compoundMsg.AttachReceivedAt(time.Second, r-credentialRoundLag, nil) // call AttachReceivedAt like demux would - err, panicErr = pM.transition(inMsg) - require.NoError(t, err) - require.NoError(t, panicErr) + sendCompoundMessageForVote(t, vVote, pM, r-credentialRoundLag, pP, time.Second, nil, version) // move to round r+1, triggering history update - vVote = helper.MakeVerifiedVote(t, 0, r, p, propose, *pV) - inMsg = messageEvent{T: voteVerified, Input: message{Vote: vVote, UnauthenticatedVote: vVote.u()}} - inMsg = inMsg.AttachValidatedAt(time.Second, r, nil) - err, panicErr = pM.transition(inMsg) - require.NoError(t, err) - require.NoError(t, panicErr) + pP, pV = helper.MakeRandomProposalPayload(t, r) + sendVoteVerified(t, helper, pM, 0, r, r, p, pV, time.Second, nil) // send payloadPresent message - m = message{UnauthenticatedProposal: pP.u()} - inMsg = messageEvent{T: payloadPresent, Input: m} - inMsg = inMsg.AttachReceivedAt(time.Second, r, nil) - err, panicErr = pM.transition(inMsg) - require.NoError(t, err) - require.NoError(t, panicErr) - moveToRound(t, pWhite, pM, helper, r+1, p, pP, pV, m, protocol.ConsensusFuture) + sendPayloadPresent(t, pM, r, pP, time.Second, nil) + moveToRound(t, pWhite, pM, helper, r+1, p, pP, pV, 2*time.Second, version) // assert lowest vote validateAt time was recorded into payloadArrivals - require.NotNil(t, dynamicFilterCredentialArrivalHistory) - require.Equal(t, pWhite.lowestCredentialArrivals.writePtr, 1) - require.False(t, pWhite.lowestCredentialArrivals.isFull()) - require.Equal(t, pipelinedMessageTimestamp, pWhite.lowestCredentialArrivals.history[0]) + assertSingleCredentialArrival(t, pWhite, pipelinedMessageTimestamp) } // test that ReceivedAt and ValidateAt timing information are retained in @@ -3959,6 +3766,7 @@ func TestPlayerRetainsEarlyReceivedValidatedAtAVPPOneSample(t *testing.T) { func TestPlayerRetainsLateReceivedValidatedAtAVPPOneSample(t *testing.T) { partitiontest.PartitionTest(t) + version := protocol.ConsensusFuture const r = round(20239) const p = period(0) pWhite, pM, helper := setupP(t, r-1, p, soft) @@ -3966,22 +3774,13 @@ func TestPlayerRetainsLateReceivedValidatedAtAVPPOneSample(t *testing.T) { // Move to round r, no credentials arrived. // send voteVerified message - vVote := helper.MakeVerifiedVote(t, 0, r-1, p, propose, *pV) - inMsg := messageEvent{T: voteVerified, Input: message{Vote: vVote, UnauthenticatedVote: vVote.u()}} - inMsg = inMsg.AttachValidatedAt(501*time.Millisecond, r-1, nil) - err, panicErr := pM.transition(inMsg) - require.NoError(t, err) - require.NoError(t, panicErr) + sendVoteVerified(t, helper, pM, 0, r-1, r-1, p, pV, 501*time.Millisecond, nil) // send payloadPresent message - m := message{UnauthenticatedProposal: pP.u()} - inMsg = messageEvent{T: payloadPresent, Input: m} - inMsg = inMsg.AttachReceivedAt(time.Second, r-1, nil) - err, panicErr = pM.transition(inMsg) - require.NoError(t, err) - require.NoError(t, panicErr) + sendPayloadPresent(t, pM, r-1, pP, time.Second, nil) - assertCorrectReceivedAtSet(t, pWhite, pM, helper, r, p, pP, pV, m, protocol.ConsensusFuture, time.Second) + moveToRound(t, pWhite, pM, helper, r, p, pP, pV, 2*time.Second, version) + assertPayloadTimings(t, pM, r-1, pV, time.Second, 2*time.Second) require.False(t, pWhite.lowestCredentialArrivals.isFull()) require.Equal(t, pWhite.lowestCredentialArrivals.writePtr, 0) @@ -3990,103 +3789,63 @@ func TestPlayerRetainsLateReceivedValidatedAtAVPPOneSample(t *testing.T) { defer configCleanup() // send votePresent message (mimicking the first AV message validating) - vVote = helper.MakeVerifiedVote(t, 0, r-credentialRoundLag, p, propose, *pV) - unverifiedVoteMsg := message{UnauthenticatedVote: vVote.u()} - inMsg = messageEvent{T: votePresent, Input: unverifiedVoteMsg, Proto: ConsensusVersionView{Version: version}} - err, panicErr = pM.transition(inMsg) - require.NoError(t, err) - require.NoError(t, panicErr) + pP, pV = helper.MakeRandomProposalPayload(t, r-credentialRoundLag) + vVote := sendVotePresent(t, helper, pM, 0, r-credentialRoundLag, p, pV, version) // make sure vote verify requests - verifyEvent := ev(cryptoAction{T: verifyVote, M: unverifiedVoteMsg, Round: r - 1, Period: p, Step: propose, TaskIndex: 1}) + verifyEvent := ev(verifyVoteAction(messageEvent{Input: message{UnauthenticatedVote: vVote.u()}}, r-credentialRoundLag, p, 1)) require.Truef(t, pM.getTrace().Contains(verifyEvent), "Player should verify vote") historicalClocks := map[round]historicalClock{ - r - credentialRoundLag: fixedHistoricalClock{}, + r - credentialRoundLag: fixedHistoricalClock(900 * time.Millisecond), } // send voteVerified, pretend we're credentialRoundLag after the message was sent - verifiedVoteMsg := message{Vote: vVote, UnauthenticatedVote: vVote.u()} - inMsg = messageEvent{T: voteVerified, Input: verifiedVoteMsg, TaskIndex: 1} - inMsg = inMsg.AttachValidatedAt(502*time.Millisecond, r, historicalClocks) - err, panicErr = pM.transition(inMsg) - require.NoError(t, err) - require.NoError(t, panicErr) + sendVoteVerifiedForVote(t, vVote, pM, r, 502*time.Millisecond, historicalClocks, 1) // create a PP message for an arbitrary proposal/payload similar to setupCompoundMessage - proposalMsg := message{UnauthenticatedProposal: pP.u()} - compoundMsg := messageEvent{T: votePresent, Input: unverifiedVoteMsg, - Tail: &messageEvent{T: payloadPresent, Input: proposalMsg}} - inMsg = compoundMsg.AttachReceivedAt(time.Second, r-credentialRoundLag, nil) // call AttachReceivedAt like demux would - err, panicErr = pM.transition(inMsg) - require.NoError(t, err) - require.NoError(t, panicErr) + sendCompoundMessageForVote(t, vVote, pM, r-credentialRoundLag, pP, time.Second, nil, version) // move to round r+1, triggering history update - vVote = helper.MakeVerifiedVote(t, 0, r, p, propose, *pV) - inMsg = messageEvent{T: voteVerified, Input: message{Vote: vVote, UnauthenticatedVote: vVote.u()}} - inMsg = inMsg.AttachValidatedAt(time.Second, r, nil) - err, panicErr = pM.transition(inMsg) - require.NoError(t, err) - require.NoError(t, panicErr) + pP, pV = helper.MakeRandomProposalPayload(t, r) + sendVoteVerified(t, helper, pM, 0, r, r, p, pV, time.Second, nil) // send payloadPresent message - m = message{UnauthenticatedProposal: pP.u()} - inMsg = messageEvent{T: payloadPresent, Input: m} - inMsg = inMsg.AttachReceivedAt(time.Second, r, nil) - err, panicErr = pM.transition(inMsg) - require.NoError(t, err) - require.NoError(t, panicErr) - moveToRound(t, pWhite, pM, helper, r+1, p, pP, pV, m, protocol.ConsensusFuture) + sendPayloadPresent(t, pM, r, pP, time.Second, nil) + moveToRound(t, pWhite, pM, helper, r+1, p, pP, pV, 2*time.Second, version) - // assert lowest vote validateAt time was recorded into payloadArrivals - require.NotNil(t, dynamicFilterCredentialArrivalHistory) - require.Equal(t, pWhite.lowestCredentialArrivals.writePtr, 1) - require.False(t, pWhite.lowestCredentialArrivals.isFull()) - require.Equal(t, fixedClockDuration, pWhite.lowestCredentialArrivals.history[0]) + // assert lowest vote validateAt time was recorded into lowestCredentialArrivals + assertSingleCredentialArrival(t, pWhite, 900*time.Millisecond) } func TestPlayerRetainsReceivedValidatedAtAVPPHistoryWindow(t *testing.T) { partitiontest.PartitionTest(t) + + version := protocol.ConsensusFuture const r = round(20239) const p = period(0) pWhite, pM, helper := setupP(t, r-1, p, soft) - require.NotNil(t, dynamicFilterCredentialArrivalHistory) + require.NotZero(t, dynamicFilterCredentialArrivalHistory) for i := 0; i < dynamicFilterCredentialArrivalHistory+int(credentialRoundLag); i++ { pP, pV := helper.MakeRandomProposalPayload(t, r+round(i)-1) // send votePresent message (mimicking the first AV message validating) - vVote := helper.MakeVerifiedVote(t, 0, r+round(i)-1, p, propose, *pV) - unverifiedVoteMsg := message{UnauthenticatedVote: vVote.u()} - inMsg := messageEvent{T: votePresent, Input: unverifiedVoteMsg} - err, panicErr := pM.transition(inMsg) - require.NoError(t, err) - require.NoError(t, panicErr) + vVote := sendVotePresent(t, helper, pM, 0, r+round(i)-1, p, pV, version) // make sure vote verify requests - verifyEvent := ev(cryptoAction{T: verifyVote, M: unverifiedVoteMsg, Round: r + round(i) - 1, Period: p, Step: propose, TaskIndex: 1}) + taskIndex := uint64(i + 1) + verifyEvent := ev(verifyVoteAction(messageEvent{Input: message{UnauthenticatedVote: vVote.u()}}, r+round(i)-1, p, taskIndex)) require.Truef(t, pM.getTrace().Contains(verifyEvent), "Player should verify vote") // send voteVerified - verifiedVoteMsg := message{Vote: vVote, UnauthenticatedVote: vVote.u()} - inMsg = messageEvent{T: voteVerified, Input: verifiedVoteMsg, TaskIndex: 1} timestamp := 500 + i - inMsg = inMsg.AttachValidatedAt(time.Duration(timestamp)*time.Millisecond, r+round(i)-1, nil) - err, panicErr = pM.transition(inMsg) - require.NoError(t, err) - require.NoError(t, panicErr) + sendVoteVerifiedForVote(t, vVote, pM, r+round(i)-1, time.Duration(timestamp)*time.Millisecond, nil, taskIndex) // create a PP message for an arbitrary proposal/payload similar to setupCompoundMessage - proposalMsg := message{UnauthenticatedProposal: pP.u()} - compoundMsg := messageEvent{T: votePresent, Input: unverifiedVoteMsg, - Tail: &messageEvent{T: payloadPresent, Input: proposalMsg}} - inMsg = compoundMsg.AttachReceivedAt(time.Second, r+round(i)-1, nil) // call AttachReceivedAt like demux would - err, panicErr = pM.transition(inMsg) - require.NoError(t, err) - require.NoError(t, panicErr) + sendCompoundMessageForVote(t, vVote, pM, r+round(i)-1, pP, time.Second, nil, version) - moveToRound(t, pWhite, pM, helper, r+round(i), p, pP, pV, proposalMsg, protocol.ConsensusFuture) + moveToRound(t, pWhite, pM, helper, r+round(i), p, pP, pV, 2*time.Second, version) } // assert lowest vote validateAt time was recorded into payloadArrivals @@ -4098,16 +3857,90 @@ func TestPlayerRetainsReceivedValidatedAtAVPPHistoryWindow(t *testing.T) { } } +// Helper function to send voteVerified message +func sendVoteVerified(t *testing.T, helper *voteMakerHelper, pM ioAutomata, addrIndex int, + curRound round, voteRound round, votePeriod period, pV *proposalValue, validatedAt time.Duration, + historicalClocks map[round]historicalClock) { + vVote := helper.MakeVerifiedVote(t, addrIndex, voteRound, votePeriod, propose, *pV) + sendVoteVerifiedForVote(t, vVote, pM, curRound, validatedAt, historicalClocks, 0) +} + +func sendVoteVerifiedForVote(t *testing.T, vVote vote, pM ioAutomata, + curRound round, validatedAt time.Duration, historicalClocks map[round]historicalClock, taskIndex uint64) { + inMsg := messageEvent{T: voteVerified, Input: message{Vote: vVote, UnauthenticatedVote: vVote.u()}, TaskIndex: taskIndex} + inMsg = inMsg.AttachValidatedAt(validatedAt, curRound, historicalClocks) + err, panicErr := pM.transition(inMsg) + require.NoError(t, err) + require.NoError(t, panicErr) +} + +func sendVotePresent(t *testing.T, helper *voteMakerHelper, pM ioAutomata, addrIndex int, + voteRound round, votePeriod period, pV *proposalValue, version protocol.ConsensusVersion) vote { + vVote := helper.MakeVerifiedVote(t, addrIndex, voteRound, votePeriod, propose, *pV) + inMsg := messageEvent{T: votePresent, Input: message{UnauthenticatedVote: vVote.u()}, Proto: ConsensusVersionView{Version: version}} + err, panicErr := pM.transition(inMsg) + require.NoError(t, err) + require.NoError(t, panicErr) + return vVote +} + +// Helper function to send payloadPresent message +func sendPayloadPresent(t *testing.T, pM ioAutomata, curRound round, pP *proposal, receivedAt time.Duration, historicalClocks map[round]historicalClock) { + m := message{UnauthenticatedProposal: pP.u()} + inMsg := messageEvent{T: payloadPresent, Input: m} + inMsg = inMsg.AttachReceivedAt(receivedAt, curRound, historicalClocks) + err, panicErr := pM.transition(inMsg) + require.NoError(t, err) + require.NoError(t, panicErr) +} + +// Helper function to send a compound PP message (votePresent + payloadPresent) +func sendCompoundMessage(t *testing.T, helper *voteMakerHelper, pM ioAutomata, curRound round, voteRound round, votePeriod period, pP *proposal, pV *proposalValue, receivedAt time.Duration, historicalClocks map[round]historicalClock, version protocol.ConsensusVersion) vote { + vVote := helper.MakeVerifiedVote(t, 0, voteRound, votePeriod, propose, *pV) + sendCompoundMessageForVote(t, vVote, pM, curRound, pP, receivedAt, historicalClocks, version) + return vVote +} + +func sendCompoundMessageForVote(t *testing.T, vVote vote, pM ioAutomata, curRound round, pP *proposal, receivedAt time.Duration, historicalClocks map[round]historicalClock, version protocol.ConsensusVersion) { + unverifiedVoteMsg := message{UnauthenticatedVote: vVote.u()} + proposalMsg := message{UnauthenticatedProposal: pP.u()} + compoundMsg := messageEvent{ + T: votePresent, + Input: unverifiedVoteMsg, + Tail: &messageEvent{ + T: payloadPresent, + Input: proposalMsg, + Proto: ConsensusVersionView{Version: version}, + }, + Proto: ConsensusVersionView{Version: version}, + } + inMsg := compoundMsg.AttachReceivedAt(receivedAt, curRound, historicalClocks) // call AttachReceivedAt like demux would + err, panicErr := pM.transition(inMsg) + require.NoError(t, err) + require.NoError(t, panicErr) +} + +// Helper function to assert lowest vote validateAt time was recorded into lowestCredentialArrivals +func assertSingleCredentialArrival(t *testing.T, pWhite *player, expectedTime time.Duration) { + require.NotZero(t, dynamicFilterCredentialArrivalHistory) + require.Equal(t, pWhite.lowestCredentialArrivals.writePtr, 1) + require.False(t, pWhite.lowestCredentialArrivals.isFull()) + require.Equal(t, expectedTime, pWhite.lowestCredentialArrivals.history[0]) +} + +// Helper function to submit payloadVerified message and a bundleVerified for a cert threshold +// to move into the next round. +// Assumes payloadPresent has alread been sent and the verifyPayload action has already requested. func moveToRound(t *testing.T, pWhite *player, pM ioAutomata, helper *voteMakerHelper, - r round, p period, pP *proposal, pV *proposalValue, m message, ver protocol.ConsensusVersion) { + r round, p period, pP *proposal, pV *proposalValue, validatedAt time.Duration, ver protocol.ConsensusVersion) { // make sure payload verify request - verifyEvent := ev(cryptoAction{T: verifyPayload, M: m, Round: r - 1, Period: p, Step: propose, TaskIndex: 0}) + verifyEvent := ev(verifyPayloadAction(messageEvent{Input: message{UnauthenticatedProposal: pP.u()}}, r-1, p, false)) require.Truef(t, pM.getTrace().Contains(verifyEvent), "Player should verify payload") // payloadVerified inMsg := messageEvent{T: payloadVerified, Input: message{Proposal: *pP}, Proto: ConsensusVersionView{Version: ver}} - inMsg = inMsg.AttachValidatedAt(2*time.Second, r-1, nil) // call AttachValidatedAt like demux would + inMsg = inMsg.AttachValidatedAt(validatedAt, r-1, nil) // call AttachValidatedAt like demux would err, panicErr := pM.transition(inMsg) require.NoError(t, err) require.NoError(t, panicErr) @@ -4150,10 +3983,8 @@ func moveToRound(t *testing.T, pWhite *player, pM ioAutomata, helper *voteMakerH require.Truef(t, pM.getTrace().Contains(commitEvent), "Player should try to ensure block/digest on ledger") } -func assertCorrectReceivedAtSet(t *testing.T, pWhite *player, pM ioAutomata, helper *voteMakerHelper, - r round, p period, pP *proposal, pV *proposalValue, m message, ver protocol.ConsensusVersion, validationTimestamp time.Duration) { - - moveToRound(t, pWhite, pM, helper, r, p, pP, pV, m, ver) +// inspect the ensureAction for round R and assert the correct payload timings +func assertPayloadTimings(t *testing.T, pM ioAutomata, r round, pV *proposalValue, receivedAt time.Duration, validatedAt time.Duration) { // find and unwrap ensureAction from trace var ea ensureAction @@ -4163,13 +3994,19 @@ func assertCorrectReceivedAtSet(t *testing.T, pWhite *player, pM ioAutomata, hel if wae.action.t() == ensure { require.False(t, foundEA) ea = wae.action.(ensureAction) - foundEA = true + // looking just for ensureAction on this round + if r == ea.Payload.Round() { + foundEA = true + break + } } } } require.True(t, foundEA) - require.Equal(t, 2*time.Second, ea.Payload.validatedAt) - require.Equal(t, validationTimestamp, ea.Payload.receivedAt) + require.Equal(t, *pV, ea.Certificate.Proposal) + require.Equal(t, r, ea.Payload.Round()) + require.Equal(t, validatedAt, ea.Payload.validatedAt) + require.Equal(t, receivedAt, ea.Payload.receivedAt) } // todo: test pipelined rounds, and round interruption diff --git a/agreement/proposalManager.go b/agreement/proposalManager.go index ee28fc3127..1cfaa30dd0 100644 --- a/agreement/proposalManager.go +++ b/agreement/proposalManager.go @@ -18,6 +18,8 @@ package agreement import ( "fmt" + + "github.com/algorand/go-algorand/data/basics" ) // A proposalManager is a proposalMachine which applies relay rules to incoming @@ -132,11 +134,12 @@ func (m *proposalManager) handleMessageEvent(r routerHandle, p player, e filtera switch e.t() { case votePresent: - err := m.filterProposalVote(p, r, e.Input.UnauthenticatedVote, e.FreshnessData) + verifyForCredHistory, err := m.filterProposalVote(p, r, e.Input.UnauthenticatedVote, e.FreshnessData) if err != nil { - // mark filtered votes that may still update the best credential arrival time credTrackingNote := NoCredentialTrackingImpact - if proposalUsedForCredentialHistory(e.FreshnessData.PlayerRound, e.Input.UnauthenticatedVote) { + if verifyForCredHistory { + // mark filtered votes that may still update the best credential arrival time + // the freshness check failed, but we still want to verify this proposal-vote for credential tracking credTrackingNote = UnverifiedBetterCredentialForTracking } return filteredEvent{T: voteFiltered, Err: makeSerErr(err), CredentialTrackingNote: credTrackingNote} @@ -244,19 +247,54 @@ func (m *proposalManager) handleMessageEvent(r routerHandle, p player, e filtera } } -// filterVote filters a vote, checking if it is both fresh and not a duplicate. -func (m *proposalManager) filterProposalVote(p player, r routerHandle, uv unauthenticatedVote, freshData freshnessData) error { +// errProposalManagerPVFreshness indicates that filterProposalVote failed the proposalFresh check. +type errProposalManagerPVNotFresh struct { + Reason error +} + +func (a errProposalManagerPVNotFresh) Error() string { + return fmt.Sprintf("proposalManager: filtered proposal-vote due to age: %v", a.Reason) +} + +// errProposalManagerPVDuplicate idnicates that filterProposalVote failed the duplicate check. +type errProposalManagerPVDuplicate struct { + Sender basics.Address + Round round + Period period +} + +func (d errProposalManagerPVDuplicate) Error() string { + return fmt.Sprintf("proposalManager: filtered proposal-vote: sender %v had already sent a vote in round %d period %d", d.Sender, d.Round, d.Period) +} + +// filterVote filters a vote, checking if it is both fresh and not a duplicate, returning +// an errProposalManagerPVNotFresh or errProposalManagerPVDuplicate if so, else nil. +// It also returns a bool indicating whether this proposal-vote should still be verified for tracking credential history. +func (m *proposalManager) filterProposalVote(p player, r routerHandle, uv unauthenticatedVote, freshData freshnessData) (bool, error) { + // check if the vote is within the credential history window + credHistory := proposalUsedForCredentialHistory(freshData.PlayerRound, uv) + + // checkDup asks proposalTracker if the vote is a duplicate, returning true if so + checkDup := func() bool { + qe := voteFilterRequestEvent{RawVote: uv.R} + sawVote := r.dispatch(p, qe, proposalMachinePeriod, uv.R.Round, uv.R.Period, 0) + return sawVote.t() == voteFiltered + } + + // check the vote against the current player's freshness rules err := proposalFresh(freshData, uv) if err != nil { - return fmt.Errorf("proposalManager: filtered proposal-vote due to age: %v", err) + // not fresh, but possibly useful for credential history: ensure not a duplicate + if credHistory && checkDup() { + credHistory = false + } + return credHistory, errProposalManagerPVNotFresh{Reason: err} } - qe := voteFilterRequestEvent{RawVote: uv.R} - sawVote := r.dispatch(p, qe, proposalMachinePeriod, uv.R.Round, uv.R.Period, 0) - if sawVote.t() == voteFiltered { - return fmt.Errorf("proposalManager: filtered proposal-vote: sender %v had already sent a vote in round %d period %d", uv.R.Sender, uv.R.Round, uv.R.Period) + if checkDup() { + return credHistory, errProposalManagerPVDuplicate{Sender: uv.R.Sender, Round: uv.R.Round, Period: uv.R.Period} } - return nil + return credHistory, nil } func proposalUsedForCredentialHistory(curRound round, vote unauthenticatedVote) bool { diff --git a/agreement/state_machine_test.go b/agreement/state_machine_test.go index 69b31a9866..681affb5b0 100644 --- a/agreement/state_machine_test.go +++ b/agreement/state_machine_test.go @@ -20,6 +20,7 @@ import ( "bytes" "fmt" "os" + "strings" "github.com/algorand/go-algorand/logging" ) @@ -111,7 +112,7 @@ func (t *ioTrace) String() string { var buf bytes.Buffer buf.WriteString("{\n") for i := 0; i < len(t.events); i++ { - buf.WriteString(fmt.Sprintf("\t%v |", t.events[i])) + buf.WriteString(fmt.Sprintf("\t%v |", t.events[i].ComparableStr())) if i%2 == 0 { buf.WriteString("\n") } @@ -140,6 +141,12 @@ func (t ioTrace) Contains(e event) bool { }) } +func (t ioTrace) ContainsString(s string) bool { + return t.ContainsFn(func(b event) bool { + return strings.Contains(b.ComparableStr(), s) + }) +} + func (t ioTrace) CountEvent(b event) (count int) { for _, e := range t.events { if e.ComparableStr() == b.ComparableStr() { @@ -550,7 +557,7 @@ func (e wrappedActionEvent) String() string { } func (e wrappedActionEvent) ComparableStr() string { - return e.action.String() + return e.action.ComparableString() } // ioAutomataConcretePlayer is a concrete wrapper around root router, implementing ioAutomata. From 0372df0f3ad8b0c902ff91a4587a5b63ffba7f34 Mon Sep 17 00:00:00 2001 From: cce <51567+cce@users.noreply.github.com> Date: Thu, 5 Oct 2023 13:47:11 -0400 Subject: [PATCH 217/232] Update agreement/proposalManager.go Co-authored-by: John Jannotti --- agreement/proposalManager.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agreement/proposalManager.go b/agreement/proposalManager.go index 1cfaa30dd0..b461d33a03 100644 --- a/agreement/proposalManager.go +++ b/agreement/proposalManager.go @@ -183,7 +183,7 @@ func (m *proposalManager) handleMessageEvent(r routerHandle, p player, e filtera credNote := e.(filteredEvent).CredentialTrackingNote if credNote != VerifiedBetterCredentialForTracking && credNote != NoCredentialTrackingImpact { // It should be impossible to hit this condition - r.t.log.Debugf("vote verified may only be tagged with NewBestCredential/NoCredentialTrackingImpact but saw %v", credNote) + r.t.log.Debugf("vote verified may only be tagged with VerifiedBetterCredential/NoCredentialTrackingImpact but saw %v", credNote) credNote = NoCredentialTrackingImpact } // indicate whether it updated From 41bdce05b3aadc16ea128b4238233c1c2c9d8c27 Mon Sep 17 00:00:00 2001 From: cce <51567+cce@users.noreply.github.com> Date: Thu, 5 Oct 2023 13:47:59 -0400 Subject: [PATCH 218/232] Update agreement/events.go Co-authored-by: John Jannotti --- agreement/events.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agreement/events.go b/agreement/events.go index 8343313546..6bf3b81df8 100644 --- a/agreement/events.go +++ b/agreement/events.go @@ -602,7 +602,7 @@ func (e payloadProcessedEvent) ComparableStr() string { type CredentialTrackingEffect uint8 const ( - // NoCredentialTrackingImpact indicates the filtered event had no impact on + // NoCredentialTrackingImpact indicates the filtered event would have no impact on // the credential tracking mechanism. NoCredentialTrackingImpact CredentialTrackingEffect = iota From 05db006549d03613be4275bc31cbcd06d162c4e3 Mon Sep 17 00:00:00 2001 From: cce <51567+cce@users.noreply.github.com> Date: Thu, 5 Oct 2023 13:48:49 -0400 Subject: [PATCH 219/232] Update agreement/events.go Co-authored-by: John Jannotti --- agreement/events.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agreement/events.go b/agreement/events.go index 6bf3b81df8..7ecc917fbd 100644 --- a/agreement/events.go +++ b/agreement/events.go @@ -611,7 +611,7 @@ const ( // It may be set by proposalManager when handling votePresent events. UnverifiedBetterCredentialForTracking - // VerifiedBetterCredentialForTracking indicates that the filtered event provided a new best + // VerifiedBetterCredentialForTracking indicates that the filtered event provides a new best // credential for its round. // It may be set by proposalManager when handling voteVerified events. VerifiedBetterCredentialForTracking From e6104d519c04508cc60685d9e9dddeec7ec59bc2 Mon Sep 17 00:00:00 2001 From: chris erway Date: Thu, 5 Oct 2023 14:17:15 -0400 Subject: [PATCH 220/232] CR changes --- agreement/actions.go | 18 +++++++++--------- agreement/player.go | 6 +++--- agreement/proposalManager.go | 6 +++--- agreement/proposalTracker.go | 8 ++++++-- agreement/proposalTracker_test.go | 7 +++++++ agreement/state_machine_test.go | 2 +- 6 files changed, 29 insertions(+), 18 deletions(-) diff --git a/agreement/actions.go b/agreement/actions.go index 80599014a4..136fb10bcf 100644 --- a/agreement/actions.go +++ b/agreement/actions.go @@ -66,7 +66,7 @@ type action interface { do(context.Context, *Service) String() string - ComparableString() string + ComparableStr() string } type nonpersistent struct{} @@ -89,7 +89,7 @@ func (a noopAction) String() string { return a.t().String() } -func (a noopAction) ComparableString() string { return a.String() } +func (a noopAction) ComparableStr() string { return a.String() } type networkAction struct { nonpersistent @@ -123,7 +123,7 @@ func (a networkAction) String() string { return fmt.Sprintf("%s: %2v", a.t().String(), a.Tag) } -func (a networkAction) ComparableString() string { +func (a networkAction) ComparableStr() string { if a.Tag == protocol.AgreementVoteTag { return fmt.Sprintf("%s: %2v: %3v-%2v-%2v", a.t().String(), a.Tag, a.UnauthenticatedVote.R.Round, a.UnauthenticatedVote.R.Period, a.UnauthenticatedVote.R.Step) } @@ -202,7 +202,7 @@ func (a cryptoAction) String() string { return a.t().String() } -func (a cryptoAction) ComparableString() (s string) { +func (a cryptoAction) ComparableStr() (s string) { switch a.T { case verifyVote: s = fmt.Sprintf("%s: %3v-%2v TaskIndex %d", a.t().String(), a.Round, a.Period, a.TaskIndex) @@ -247,7 +247,7 @@ func (a ensureAction) String() string { return fmt.Sprintf("%s: %.5s: %v, %v, %.5s", a.t().String(), a.Payload.Digest().String(), a.Certificate.Round, a.Certificate.Period, a.Certificate.Proposal.BlockDigest.String()) } -func (a ensureAction) ComparableString() string { return a.String() } +func (a ensureAction) ComparableStr() string { return a.String() } func (a ensureAction) do(ctx context.Context, s *Service) { logEvent := logspec.AgreementEvent{ @@ -312,7 +312,7 @@ func (a stageDigestAction) String() string { return fmt.Sprintf("%s: %.5s. %v. %v", a.t().String(), a.Certificate.Proposal.BlockDigest.String(), a.Certificate.Round, a.Certificate.Period) } -func (a stageDigestAction) ComparableString() string { return a.String() } +func (a stageDigestAction) ComparableStr() string { return a.String() } func (a stageDigestAction) do(ctx context.Context, service *Service) { logEvent := logspec.AgreementEvent{ @@ -340,7 +340,7 @@ func (a rezeroAction) String() string { return a.t().String() } -func (a rezeroAction) ComparableString() string { +func (a rezeroAction) ComparableStr() string { return fmt.Sprintf("%s: %d", a.t().String(), a.Round) } @@ -376,7 +376,7 @@ func (a pseudonodeAction) String() string { return fmt.Sprintf("%v %3v-%2v-%2v: %.5v", a.t().String(), a.Round, a.Period, a.Step, a.Proposal.BlockDigest.String()) } -func (a pseudonodeAction) ComparableString() string { return a.String() } +func (a pseudonodeAction) ComparableStr() string { return a.String() } func (a pseudonodeAction) persistent() bool { return a.T == attest @@ -571,4 +571,4 @@ func (c checkpointAction) String() string { return c.t().String() } -func (c checkpointAction) ComparableString() string { return c.String() } +func (c checkpointAction) ComparableStr() string { return c.String() } diff --git a/agreement/player.go b/agreement/player.go index 72b7b4d129..9c640e9603 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -633,9 +633,9 @@ func (p *player) handleMessageEvent(r routerHandle, e messageEvent) (actions []a err := ef.(filteredEvent).Err return append(actions, ignoreAction(e, err)) case UnverifiedBetterCredentialForTracking: - // There is another case, where the message - // MayImpactCredentialTracking. This case does not return here, - // so we continue processing the message. + // In this case, the vote may impact credential tracking, but needs to + // be validated. So we do not return here, and continue processing, so that + // the votePresent chdck below will make a verifyVoteAction for this vote. } } diff --git a/agreement/proposalManager.go b/agreement/proposalManager.go index b461d33a03..a68a64207d 100644 --- a/agreement/proposalManager.go +++ b/agreement/proposalManager.go @@ -161,7 +161,7 @@ func (m *proposalManager) handleMessageEvent(r routerHandle, p player, e filtera keepForCredentialTracking := false if err != nil { // if we should keep processing this credential message only to record its timestamp, we continue - keepForCredentialTracking = proposalUsedForCredentialHistory(e.FreshnessData.PlayerRound, v.u()) + keepForCredentialTracking = proposalUsefulForCredentialHistory(e.FreshnessData.PlayerRound, v.u()) if !keepForCredentialTracking { err := makeSerErrf("proposalManager: ignoring proposal-vote due to age: %v", err) return filteredEvent{T: voteFiltered, Err: err} @@ -272,7 +272,7 @@ func (d errProposalManagerPVDuplicate) Error() string { // It also returns a bool indicating whether this proposal-vote should still be verified for tracking credential history. func (m *proposalManager) filterProposalVote(p player, r routerHandle, uv unauthenticatedVote, freshData freshnessData) (bool, error) { // check if the vote is within the credential history window - credHistory := proposalUsedForCredentialHistory(freshData.PlayerRound, uv) + credHistory := proposalUsefulForCredentialHistory(freshData.PlayerRound, uv) // checkDup asks proposalTracker if the vote is a duplicate, returning true if so checkDup := func() bool { @@ -297,7 +297,7 @@ func (m *proposalManager) filterProposalVote(p player, r routerHandle, uv unauth return credHistory, nil } -func proposalUsedForCredentialHistory(curRound round, vote unauthenticatedVote) bool { +func proposalUsefulForCredentialHistory(curRound round, vote unauthenticatedVote) bool { if vote.R.Round < curRound && curRound <= vote.R.Round+credentialRoundLag && vote.R.Period == 0 && vote.R.Step == propose { diff --git a/agreement/proposalTracker.go b/agreement/proposalTracker.go index 15bc37f5b3..05cf1a3140 100644 --- a/agreement/proposalTracker.go +++ b/agreement/proposalTracker.go @@ -41,8 +41,12 @@ type proposalSeeker struct { } // accept compares a given vote with the current lowest-credentialled vote and -// sets it if freeze has not been called. Returns true if any internal proposalSeeker -// state has been updated, whether to Lowest or lowestAfterFreeze. +// sets it if freeze has not been called. Returns: +// - updated proposalSeeker state, +// - a CredentialTrackingEffect describing the usefulness of proposal-vote's +// credential for late credential tracking (for choosing dynamic filter timeout), +// - and an error if the proposal was not better than the lowest seen, or the +// seeker was already frozen. func (s proposalSeeker) accept(v vote) (proposalSeeker, CredentialTrackingEffect, error) { if s.Frozen { effect := NoCredentialTrackingImpact diff --git a/agreement/proposalTracker_test.go b/agreement/proposalTracker_test.go index 41ff3906e2..28807d63ba 100644 --- a/agreement/proposalTracker_test.go +++ b/agreement/proposalTracker_test.go @@ -74,6 +74,7 @@ func TestProposalTrackerProposalSeeker(t *testing.T) { assert.True(t, s.Filled) assert.True(t, s.Lowest.equals(votes[2])) assert.True(t, s.hasLowestAfterFreeze) + assert.Equal(t, s.Lowest, s.lowestAfterFreeze) s, effect, err = s.accept(votes[3]) assert.Error(t, err) @@ -82,6 +83,7 @@ func TestProposalTrackerProposalSeeker(t *testing.T) { assert.True(t, s.Filled) assert.True(t, s.Lowest.equals(votes[2])) assert.True(t, s.hasLowestAfterFreeze) + assert.Equal(t, s.Lowest, s.lowestAfterFreeze) s, effect, err = s.accept(votes[1]) assert.NoError(t, err) @@ -90,6 +92,7 @@ func TestProposalTrackerProposalSeeker(t *testing.T) { assert.True(t, s.Filled) assert.True(t, s.Lowest.equals(votes[1])) assert.True(t, s.hasLowestAfterFreeze) + assert.Equal(t, s.Lowest, s.lowestAfterFreeze) lowestBeforeFreeze := s.Lowest s = s.freeze() @@ -97,6 +100,7 @@ func TestProposalTrackerProposalSeeker(t *testing.T) { assert.True(t, s.Filled) assert.True(t, s.Lowest.equals(votes[1])) assert.True(t, s.hasLowestAfterFreeze) + assert.Equal(t, s.Lowest, s.lowestAfterFreeze) s, effect, err = s.accept(votes[0]) assert.Error(t, err) @@ -107,6 +111,9 @@ func TestProposalTrackerProposalSeeker(t *testing.T) { assert.True(t, s.Lowest.equals(votes[1])) assert.True(t, s.hasLowestAfterFreeze) assert.True(t, s.lowestAfterFreeze.equals(votes[0])) + assert.NotEqual(t, s.Lowest, s.lowestAfterFreeze) + assert.True(t, !s.Lowest.Cred.Less(s.lowestAfterFreeze.Cred)) + assert.True(t, s.lowestAfterFreeze.Cred.Less(s.Lowest.Cred)) } // mimics a proposalTracker, producing a trace of events diff --git a/agreement/state_machine_test.go b/agreement/state_machine_test.go index 681affb5b0..6946a051ec 100644 --- a/agreement/state_machine_test.go +++ b/agreement/state_machine_test.go @@ -557,7 +557,7 @@ func (e wrappedActionEvent) String() string { } func (e wrappedActionEvent) ComparableStr() string { - return e.action.ComparableString() + return e.action.ComparableStr() } // ioAutomataConcretePlayer is a concrete wrapper around root router, implementing ioAutomata. From f42cad32a4fd96e673d334bfe5923a8daa9d387f Mon Sep 17 00:00:00 2001 From: chris erway Date: Thu, 5 Oct 2023 14:37:50 -0400 Subject: [PATCH 221/232] rename LowestAfterFreeze => LowestIncludingLate --- agreement/events.go | 10 +++++----- agreement/player.go | 6 +++--- agreement/proposalTracker.go | 24 ++++++++++++------------ agreement/proposalTracker_test.go | 28 ++++++++++++++-------------- 4 files changed, 34 insertions(+), 34 deletions(-) diff --git a/agreement/events.go b/agreement/events.go index 7ecc917fbd..a58dbc1b9c 100644 --- a/agreement/events.go +++ b/agreement/events.go @@ -424,14 +424,14 @@ type readLowestEvent struct { // Vote holds the lowest-credential vote. Vote vote - // LowestAfterFreeze holds the lowest-credential vote that was received, including + // LowestIncludingLate holds the lowest-credential vote that was received, including // after Vote has been frozen. - LowestAfterFreeze vote + LowestIncludingLate vote - // Filled and HasLowestAfterFreeze indicates whether the Vote or LowestAfterFreeze + // Filled and HasLowestIncludingLate indicates whether the Vote or LowestIncludingLate // fields are filled, respectively. - Filled bool - HasLowestAfterFreeze bool + Filled bool + HasLowestIncludingLate bool } func (e readLowestEvent) t() eventType { diff --git a/agreement/player.go b/agreement/player.go index 9c640e9603..f0091fa3ef 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -297,12 +297,12 @@ func (p *player) updateCredentialArrivalHistory(r routerHandle, ver protocol.Con credHistoryRound := p.Round - credentialRoundLag re := readLowestEvent{T: readLowestVote, Round: credHistoryRound, Period: 0} re = r.dispatch(*p, re, proposalMachineRound, credHistoryRound, 0, 0).(readLowestEvent) - if !re.HasLowestAfterFreeze { + if !re.HasLowestIncludingLate { return 0 } - p.lowestCredentialArrivals.store(re.LowestAfterFreeze.validatedAt) - return re.LowestAfterFreeze.validatedAt + p.lowestCredentialArrivals.store(re.LowestIncludingLate.validatedAt) + return re.LowestIncludingLate.validatedAt } // calculateFilterTimeout chooses the appropriate filter timeout. diff --git a/agreement/proposalTracker.go b/agreement/proposalTracker.go index 05cf1a3140..6d4ea0e85d 100644 --- a/agreement/proposalTracker.go +++ b/agreement/proposalTracker.go @@ -34,10 +34,10 @@ type proposalSeeker struct { // Filled will no longer be modified. Frozen bool - // lowestAfterFreeze is used to track the lowest credential observed, even + // lowestIncludingLate is used to track the lowest credential observed, even // after the Lowest value has been frozen. - lowestAfterFreeze vote - hasLowestAfterFreeze bool + lowestIncludingLate vote + hasLowestIncludingLate bool } // accept compares a given vote with the current lowest-credentialled vote and @@ -51,9 +51,9 @@ func (s proposalSeeker) accept(v vote) (proposalSeeker, CredentialTrackingEffect if s.Frozen { effect := NoCredentialTrackingImpact // continue tracking and forwarding the lowest proposal even when frozen - if !s.hasLowestAfterFreeze || v.Cred.Less(s.lowestAfterFreeze.Cred) { - s.lowestAfterFreeze = v - s.hasLowestAfterFreeze = true + if !s.hasLowestIncludingLate || v.Cred.Less(s.lowestIncludingLate.Cred) { + s.lowestIncludingLate = v + s.hasLowestIncludingLate = true effect = VerifiedBetterCredentialForTracking } return s, effect, errProposalSeekerFrozen{} @@ -65,14 +65,14 @@ func (s proposalSeeker) accept(v vote) (proposalSeeker, CredentialTrackingEffect s.Lowest = v s.Filled = true - s.lowestAfterFreeze = v - s.hasLowestAfterFreeze = true + s.lowestIncludingLate = v + s.hasLowestIncludingLate = true return s, VerifiedBetterCredentialForTracking, nil } func (s *proposalSeeker) copyCredentialTrackingState(s2 proposalSeeker) { - s.hasLowestAfterFreeze = s2.hasLowestAfterFreeze - s.lowestAfterFreeze = s2.lowestAfterFreeze + s.hasLowestIncludingLate = s2.hasLowestIncludingLate + s.lowestIncludingLate = s2.lowestIncludingLate } // freeze freezes the state of the proposalSeeker so that future calls no longer @@ -196,8 +196,8 @@ func (t *proposalTracker) handle(r routerHandle, p player, e event) event { e := e.(readLowestEvent) e.Vote = t.Freezer.Lowest e.Filled = t.Freezer.Filled - e.LowestAfterFreeze = t.Freezer.lowestAfterFreeze - e.HasLowestAfterFreeze = t.Freezer.hasLowestAfterFreeze + e.LowestIncludingLate = t.Freezer.lowestIncludingLate + e.HasLowestIncludingLate = t.Freezer.hasLowestIncludingLate return e case softThreshold, certThreshold: diff --git a/agreement/proposalTracker_test.go b/agreement/proposalTracker_test.go index 28807d63ba..763fe37e8e 100644 --- a/agreement/proposalTracker_test.go +++ b/agreement/proposalTracker_test.go @@ -63,7 +63,7 @@ func TestProposalTrackerProposalSeeker(t *testing.T) { var err error assert.False(t, s.Frozen) assert.False(t, s.Filled) - assert.False(t, s.hasLowestAfterFreeze) + assert.False(t, s.hasLowestIncludingLate) // issue events in the following order: 2, 3, 1, (freeze), 0 var effect CredentialTrackingEffect @@ -73,8 +73,8 @@ func TestProposalTrackerProposalSeeker(t *testing.T) { assert.False(t, s.Frozen) assert.True(t, s.Filled) assert.True(t, s.Lowest.equals(votes[2])) - assert.True(t, s.hasLowestAfterFreeze) - assert.Equal(t, s.Lowest, s.lowestAfterFreeze) + assert.True(t, s.hasLowestIncludingLate) + assert.Equal(t, s.Lowest, s.lowestIncludingLate) s, effect, err = s.accept(votes[3]) assert.Error(t, err) @@ -82,8 +82,8 @@ func TestProposalTrackerProposalSeeker(t *testing.T) { assert.False(t, s.Frozen) assert.True(t, s.Filled) assert.True(t, s.Lowest.equals(votes[2])) - assert.True(t, s.hasLowestAfterFreeze) - assert.Equal(t, s.Lowest, s.lowestAfterFreeze) + assert.True(t, s.hasLowestIncludingLate) + assert.Equal(t, s.Lowest, s.lowestIncludingLate) s, effect, err = s.accept(votes[1]) assert.NoError(t, err) @@ -91,16 +91,16 @@ func TestProposalTrackerProposalSeeker(t *testing.T) { assert.False(t, s.Frozen) assert.True(t, s.Filled) assert.True(t, s.Lowest.equals(votes[1])) - assert.True(t, s.hasLowestAfterFreeze) - assert.Equal(t, s.Lowest, s.lowestAfterFreeze) + assert.True(t, s.hasLowestIncludingLate) + assert.Equal(t, s.Lowest, s.lowestIncludingLate) lowestBeforeFreeze := s.Lowest s = s.freeze() assert.True(t, s.Frozen) assert.True(t, s.Filled) assert.True(t, s.Lowest.equals(votes[1])) - assert.True(t, s.hasLowestAfterFreeze) - assert.Equal(t, s.Lowest, s.lowestAfterFreeze) + assert.True(t, s.hasLowestIncludingLate) + assert.Equal(t, s.Lowest, s.lowestIncludingLate) s, effect, err = s.accept(votes[0]) assert.Error(t, err) @@ -109,11 +109,11 @@ func TestProposalTrackerProposalSeeker(t *testing.T) { assert.True(t, s.Frozen) assert.True(t, s.Filled) assert.True(t, s.Lowest.equals(votes[1])) - assert.True(t, s.hasLowestAfterFreeze) - assert.True(t, s.lowestAfterFreeze.equals(votes[0])) - assert.NotEqual(t, s.Lowest, s.lowestAfterFreeze) - assert.True(t, !s.Lowest.Cred.Less(s.lowestAfterFreeze.Cred)) - assert.True(t, s.lowestAfterFreeze.Cred.Less(s.Lowest.Cred)) + assert.True(t, s.hasLowestIncludingLate) + assert.True(t, s.lowestIncludingLate.equals(votes[0])) + assert.NotEqual(t, s.Lowest, s.lowestIncludingLate) + assert.True(t, !s.Lowest.Cred.Less(s.lowestIncludingLate.Cred)) + assert.True(t, s.lowestIncludingLate.Cred.Less(s.Lowest.Cred)) } // mimics a proposalTracker, producing a trace of events From 461ebc804c58d6d9f687d60b6ffe36df637d856a Mon Sep 17 00:00:00 2001 From: chris erway Date: Thu, 5 Oct 2023 14:40:31 -0400 Subject: [PATCH 222/232] use switch in AttachReceivedAt --- agreement/events.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/agreement/events.go b/agreement/events.go index a58dbc1b9c..11c8ea31f3 100644 --- a/agreement/events.go +++ b/agreement/events.go @@ -1040,9 +1040,10 @@ func (e messageEvent) AttachValidatedAt(d time.Duration, currentRound round, his // payloadPresent or votePresent messageEvent, and attaches the given // time to the proposal's receivedAt field. func (e messageEvent) AttachReceivedAt(d time.Duration, currentRound round, historicalClocks map[round]historicalClock) messageEvent { - if e.T == payloadPresent { + switch e.T { + case payloadPresent: e.Input.UnauthenticatedProposal.receivedAt = getTimestampForEvent(e.Input.UnauthenticatedProposal.Round(), d, currentRound, historicalClocks) - } else if e.T == votePresent { + case votePresent: // Check for non-nil Tail, indicating this votePresent event // contains a synthetic payloadPresent event that was attached // to it by setupCompoundMessage. From cc6d9fb0550b37598e74d70eef3959c6c01c7d84 Mon Sep 17 00:00:00 2001 From: chris erway Date: Thu, 5 Oct 2023 14:49:28 -0400 Subject: [PATCH 223/232] fix encode to not replace nil rr.Children maps with empty maps --- agreement/persistence.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/agreement/persistence.go b/agreement/persistence.go index 8711edf41b..0c75808d64 100644 --- a/agreement/persistence.go +++ b/agreement/persistence.go @@ -58,13 +58,15 @@ func encode(t timers.Clock[TimeoutType], rr rootRouter, p player, a []action, re // Don't persist state for old rounds // rootRouter.update() may preserve roundRouters from credentialRoundLag rounds ago - children := make(map[round]*roundRouter) - for rnd, rndRouter := range rr.Children { - if rnd >= p.Round { - children[rnd] = rndRouter + if rr.Children != nil { + children := make(map[round]*roundRouter) + for rnd, rndRouter := range rr.Children { + if rnd >= p.Round { + children[rnd] = rndRouter + } } + rr.Children = children } - rr.Children = children if reflect { s.Router = protocol.EncodeReflect(rr) From dedcd0a06656a2c16d749f324b8176667d9075e9 Mon Sep 17 00:00:00 2001 From: chris erway Date: Thu, 5 Oct 2023 14:52:09 -0400 Subject: [PATCH 224/232] downgrade to nil in encoding rr.Children instead --- agreement/persistence.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/agreement/persistence.go b/agreement/persistence.go index 0c75808d64..b8510e446c 100644 --- a/agreement/persistence.go +++ b/agreement/persistence.go @@ -58,13 +58,15 @@ func encode(t timers.Clock[TimeoutType], rr rootRouter, p player, a []action, re // Don't persist state for old rounds // rootRouter.update() may preserve roundRouters from credentialRoundLag rounds ago - if rr.Children != nil { - children := make(map[round]*roundRouter) - for rnd, rndRouter := range rr.Children { - if rnd >= p.Round { - children[rnd] = rndRouter - } + children := make(map[round]*roundRouter) + for rnd, rndRouter := range rr.Children { + if rnd >= p.Round { + children[rnd] = rndRouter } + } + if len(children) == 0 { + rr.Children = nil + } else { rr.Children = children } From 9d3613e00bea7ce083f688ad1d53c6a63a4388d9 Mon Sep 17 00:00:00 2001 From: chris erway Date: Thu, 5 Oct 2023 21:14:19 -0400 Subject: [PATCH 225/232] re-run make msgp with latest master --- agreement/msgp_gen.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/agreement/msgp_gen.go b/agreement/msgp_gen.go index 25b1d68592..85a50ddaab 100644 --- a/agreement/msgp_gen.go +++ b/agreement/msgp_gen.go @@ -40,6 +40,7 @@ import ( // |-----> MarshalMsg // |-----> CanMarshalMsg // |-----> (*) UnmarshalMsg +// |-----> (*) UnmarshalMsgWithState // |-----> (*) CanUnmarshalMsg // |-----> Msgsize // |-----> MsgIsZero @@ -1070,7 +1071,12 @@ func (_ CredentialTrackingEffect) CanMarshalMsg(z interface{}) bool { } // UnmarshalMsg implements msgp.Unmarshaler -func (z *CredentialTrackingEffect) UnmarshalMsg(bts []byte) (o []byte, err error) { +func (z *CredentialTrackingEffect) UnmarshalMsgWithState(bts []byte, st msgp.UnmarshalState) (o []byte, err error) { + if st.AllowableDepth == 0 { + err = msgp.ErrMaxDepthExceeded{} + return + } + st.AllowableDepth-- { var zb0001 uint8 zb0001, bts, err = msgp.ReadUint8Bytes(bts) @@ -1084,6 +1090,9 @@ func (z *CredentialTrackingEffect) UnmarshalMsg(bts []byte) (o []byte, err error return } +func (z *CredentialTrackingEffect) UnmarshalMsg(bts []byte) (o []byte, err error) { + return z.UnmarshalMsgWithState(bts, msgp.DefaultUnmarshalState) +} func (_ *CredentialTrackingEffect) CanUnmarshalMsg(z interface{}) bool { _, ok := (z).(*CredentialTrackingEffect) return ok From 8c3da992cedfb31f060e7d1efa0ffb1130771a6c Mon Sep 17 00:00:00 2001 From: chris erway Date: Thu, 5 Oct 2023 21:21:39 -0400 Subject: [PATCH 226/232] update CredentialTracking => LateCredentialTracking --- agreement/events.go | 24 ++--- agreement/msgp_gen.go | 140 +++++++++++++++--------------- agreement/player.go | 8 +- agreement/proposalManager.go | 26 +++--- agreement/proposalManager_test.go | 10 +-- agreement/proposalTracker.go | 20 ++--- agreement/proposalTracker_test.go | 10 +-- 7 files changed, 119 insertions(+), 119 deletions(-) diff --git a/agreement/events.go b/agreement/events.go index 11c8ea31f3..fdc3c0161d 100644 --- a/agreement/events.go +++ b/agreement/events.go @@ -596,35 +596,35 @@ func (e payloadProcessedEvent) ComparableStr() string { return fmt.Sprintf("%v: %.5v", e.t().String(), e.Proposal.BlockDigest.String()) } -// CredentialTrackingEffect indicates the impact of an event that got filtered -// on the credential tracking mechanism, for the purpose of tracking the time it -// took the best credential to arrive. -type CredentialTrackingEffect uint8 +// LateCredentialTrackingEffect indicates the impact of a vote that was filtered (due to age) +// on the credential tracking system (in credentialArrivalHistory), for the purpose of tracking +// the time it took the best credential to arrive, even if it was late. +type LateCredentialTrackingEffect uint8 const ( - // NoCredentialTrackingImpact indicates the filtered event would have no impact on + // NoLateCredentialTrackingImpact indicates the filtered event would have no impact on // the credential tracking mechanism. - NoCredentialTrackingImpact CredentialTrackingEffect = iota + NoLateCredentialTrackingImpact LateCredentialTrackingEffect = iota - // UnverifiedBetterCredentialForTracking indicates the filtered event could impact + // UnverifiedLateCredentialForTracking indicates the filtered event could impact // the credential tracking mechanism and more processing (validation) may be required. // It may be set by proposalManager when handling votePresent events. - UnverifiedBetterCredentialForTracking + UnverifiedLateCredentialForTracking - // VerifiedBetterCredentialForTracking indicates that the filtered event provides a new best + // VerifiedBetterLateCredentialForTracking indicates that the filtered event provides a new best // credential for its round. // It may be set by proposalManager when handling voteVerified events. - VerifiedBetterCredentialForTracking + VerifiedBetterLateCredentialForTracking ) type filteredEvent struct { // {proposal,vote,bundle}{Filtered,Malformed} T eventType - // CredentialTrackingNote indicates the impact of the filtered event on the + // LateCredentialTrackingNote indicates the impact of the filtered event on the // credential tracking machinery used for dynamically setting the filter // timeout. - CredentialTrackingNote CredentialTrackingEffect + LateCredentialTrackingNote LateCredentialTrackingEffect // Err is the reason cryptographic verification failed and is set for // events {proposal,vote,bundle}Malformed. diff --git a/agreement/msgp_gen.go b/agreement/msgp_gen.go index 85a50ddaab..16679ce797 100644 --- a/agreement/msgp_gen.go +++ b/agreement/msgp_gen.go @@ -36,16 +36,6 @@ import ( // |-----> (*) MsgIsZero // |-----> ConsensusVersionViewMaxSize() // -// CredentialTrackingEffect -// |-----> MarshalMsg -// |-----> CanMarshalMsg -// |-----> (*) UnmarshalMsg -// |-----> (*) UnmarshalMsgWithState -// |-----> (*) CanUnmarshalMsg -// |-----> Msgsize -// |-----> MsgIsZero -// |-----> CredentialTrackingEffectMaxSize() -// // Deadline // |-----> (*) MarshalMsg // |-----> (*) CanMarshalMsg @@ -56,6 +46,16 @@ import ( // |-----> (*) MsgIsZero // |-----> DeadlineMaxSize() // +// LateCredentialTrackingEffect +// |-----> MarshalMsg +// |-----> CanMarshalMsg +// |-----> (*) UnmarshalMsg +// |-----> (*) UnmarshalMsgWithState +// |-----> (*) CanUnmarshalMsg +// |-----> Msgsize +// |-----> MsgIsZero +// |-----> LateCredentialTrackingEffectMaxSize() +// // TimeoutType // |-----> MarshalMsg // |-----> CanMarshalMsg @@ -1055,66 +1055,6 @@ func ConsensusVersionViewMaxSize() (s int) { return } -// MarshalMsg implements msgp.Marshaler -func (z CredentialTrackingEffect) MarshalMsg(b []byte) (o []byte) { - o = msgp.Require(b, z.Msgsize()) - o = msgp.AppendUint8(o, uint8(z)) - return -} - -func (_ CredentialTrackingEffect) CanMarshalMsg(z interface{}) bool { - _, ok := (z).(CredentialTrackingEffect) - if !ok { - _, ok = (z).(*CredentialTrackingEffect) - } - return ok -} - -// UnmarshalMsg implements msgp.Unmarshaler -func (z *CredentialTrackingEffect) UnmarshalMsgWithState(bts []byte, st msgp.UnmarshalState) (o []byte, err error) { - if st.AllowableDepth == 0 { - err = msgp.ErrMaxDepthExceeded{} - return - } - st.AllowableDepth-- - { - var zb0001 uint8 - zb0001, bts, err = msgp.ReadUint8Bytes(bts) - if err != nil { - err = msgp.WrapError(err) - return - } - (*z) = CredentialTrackingEffect(zb0001) - } - o = bts - return -} - -func (z *CredentialTrackingEffect) UnmarshalMsg(bts []byte) (o []byte, err error) { - return z.UnmarshalMsgWithState(bts, msgp.DefaultUnmarshalState) -} -func (_ *CredentialTrackingEffect) CanUnmarshalMsg(z interface{}) bool { - _, ok := (z).(*CredentialTrackingEffect) - return ok -} - -// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message -func (z CredentialTrackingEffect) Msgsize() (s int) { - s = msgp.Uint8Size - return -} - -// MsgIsZero returns whether this is a zero value -func (z CredentialTrackingEffect) MsgIsZero() bool { - return z == 0 -} - -// MaxSize returns a maximum valid message size for this message type -func CredentialTrackingEffectMaxSize() (s int) { - s = msgp.Uint8Size - return -} - // MarshalMsg implements msgp.Marshaler func (z *Deadline) MarshalMsg(b []byte) (o []byte) { o = msgp.Require(b, z.Msgsize()) @@ -1248,6 +1188,66 @@ func DeadlineMaxSize() (s int) { return } +// MarshalMsg implements msgp.Marshaler +func (z LateCredentialTrackingEffect) MarshalMsg(b []byte) (o []byte) { + o = msgp.Require(b, z.Msgsize()) + o = msgp.AppendUint8(o, uint8(z)) + return +} + +func (_ LateCredentialTrackingEffect) CanMarshalMsg(z interface{}) bool { + _, ok := (z).(LateCredentialTrackingEffect) + if !ok { + _, ok = (z).(*LateCredentialTrackingEffect) + } + return ok +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *LateCredentialTrackingEffect) UnmarshalMsgWithState(bts []byte, st msgp.UnmarshalState) (o []byte, err error) { + if st.AllowableDepth == 0 { + err = msgp.ErrMaxDepthExceeded{} + return + } + st.AllowableDepth-- + { + var zb0001 uint8 + zb0001, bts, err = msgp.ReadUint8Bytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + (*z) = LateCredentialTrackingEffect(zb0001) + } + o = bts + return +} + +func (z *LateCredentialTrackingEffect) UnmarshalMsg(bts []byte) (o []byte, err error) { + return z.UnmarshalMsgWithState(bts, msgp.DefaultUnmarshalState) +} +func (_ *LateCredentialTrackingEffect) CanUnmarshalMsg(z interface{}) bool { + _, ok := (z).(*LateCredentialTrackingEffect) + return ok +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z LateCredentialTrackingEffect) Msgsize() (s int) { + s = msgp.Uint8Size + return +} + +// MsgIsZero returns whether this is a zero value +func (z LateCredentialTrackingEffect) MsgIsZero() bool { + return z == 0 +} + +// MaxSize returns a maximum valid message size for this message type +func LateCredentialTrackingEffectMaxSize() (s int) { + s = msgp.Uint8Size + return +} + // MarshalMsg implements msgp.Marshaler func (z TimeoutType) MarshalMsg(b []byte) (o []byte) { o = msgp.Require(b, z.Msgsize()) diff --git a/agreement/player.go b/agreement/player.go index f0091fa3ef..4c4f565553 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -620,19 +620,19 @@ func (p *player) handleMessageEvent(r routerHandle, e messageEvent) (actions []a err := ef.(filteredEvent).Err return append(actions, ignoreAction(e, err)) } - switch ef.(filteredEvent).CredentialTrackingNote { - case VerifiedBetterCredentialForTracking: + switch ef.(filteredEvent).LateCredentialTrackingNote { + case VerifiedBetterLateCredentialForTracking: // Dynamic filter timeout feature enabled, and current message // updated the best credential arrival time v := e.Input.Vote return append(actions, relayAction(e, protocol.AgreementVoteTag, v.u())) - case NoCredentialTrackingImpact: + case NoLateCredentialTrackingImpact: // Dynamic filter timeout feature enabled, but current message // may not update the best credential arrival time, so we should // ignore it. err := ef.(filteredEvent).Err return append(actions, ignoreAction(e, err)) - case UnverifiedBetterCredentialForTracking: + case UnverifiedLateCredentialForTracking: // In this case, the vote may impact credential tracking, but needs to // be validated. So we do not return here, and continue processing, so that // the votePresent chdck below will make a verifyVoteAction for this vote. diff --git a/agreement/proposalManager.go b/agreement/proposalManager.go index a68a64207d..cf89063641 100644 --- a/agreement/proposalManager.go +++ b/agreement/proposalManager.go @@ -136,13 +136,13 @@ func (m *proposalManager) handleMessageEvent(r routerHandle, p player, e filtera case votePresent: verifyForCredHistory, err := m.filterProposalVote(p, r, e.Input.UnauthenticatedVote, e.FreshnessData) if err != nil { - credTrackingNote := NoCredentialTrackingImpact + credTrackingNote := NoLateCredentialTrackingImpact if verifyForCredHistory { // mark filtered votes that may still update the best credential arrival time // the freshness check failed, but we still want to verify this proposal-vote for credential tracking - credTrackingNote = UnverifiedBetterCredentialForTracking + credTrackingNote = UnverifiedLateCredentialForTracking } - return filteredEvent{T: voteFiltered, Err: makeSerErr(err), CredentialTrackingNote: credTrackingNote} + return filteredEvent{T: voteFiltered, Err: makeSerErr(err), LateCredentialTrackingNote: credTrackingNote} } return emptyEvent{} @@ -158,11 +158,11 @@ func (m *proposalManager) handleMessageEvent(r routerHandle, p player, e filtera v := e.Input.Vote err := proposalFresh(e.FreshnessData, v.u()) - keepForCredentialTracking := false + keepForLateCredentialTracking := false if err != nil { // if we should keep processing this credential message only to record its timestamp, we continue - keepForCredentialTracking = proposalUsefulForCredentialHistory(e.FreshnessData.PlayerRound, v.u()) - if !keepForCredentialTracking { + keepForLateCredentialTracking = proposalUsefulForCredentialHistory(e.FreshnessData.PlayerRound, v.u()) + if !keepForLateCredentialTracking { err := makeSerErrf("proposalManager: ignoring proposal-vote due to age: %v", err) return filteredEvent{T: voteFiltered, Err: err} } @@ -176,22 +176,22 @@ func (m *proposalManager) handleMessageEvent(r routerHandle, p player, e filtera e := r.dispatch(p, e.messageEvent, proposalMachineRound, v.R.Round, v.R.Period, 0) - if keepForCredentialTracking { + if keepForLateCredentialTracking { // we only continued processing this vote to see whether it updates the credential arrival time err := makeSerErrf("proposalManager: ignoring proposal-vote due to age: %v", err) if e.t() == voteFiltered { - credNote := e.(filteredEvent).CredentialTrackingNote - if credNote != VerifiedBetterCredentialForTracking && credNote != NoCredentialTrackingImpact { + credNote := e.(filteredEvent).LateCredentialTrackingNote + if credNote != VerifiedBetterLateCredentialForTracking && credNote != NoLateCredentialTrackingImpact { // It should be impossible to hit this condition - r.t.log.Debugf("vote verified may only be tagged with VerifiedBetterCredential/NoCredentialTrackingImpact but saw %v", credNote) - credNote = NoCredentialTrackingImpact + r.t.log.Debugf("vote verified may only be tagged with VerifiedBetterLateCredential/NoLateCredentialTrackingImpact but saw %v", credNote) + credNote = NoLateCredentialTrackingImpact } // indicate whether it updated - return filteredEvent{T: voteFiltered, Err: err, CredentialTrackingNote: credNote} + return filteredEvent{T: voteFiltered, Err: err, LateCredentialTrackingNote: credNote} } // the proposalMachineRound didn't filter the vote, so it must have had a better credential, // indicate that it did cause updating its state - return filteredEvent{T: voteFiltered, Err: err, CredentialTrackingNote: VerifiedBetterCredentialForTracking} + return filteredEvent{T: voteFiltered, Err: err, LateCredentialTrackingNote: VerifiedBetterLateCredentialForTracking} } return e diff --git a/agreement/proposalManager_test.go b/agreement/proposalManager_test.go index c85bcd3450..48481bd730 100644 --- a/agreement/proposalManager_test.go +++ b/agreement/proposalManager_test.go @@ -260,12 +260,12 @@ func TestLateVotes(t *testing.T) { v2.Cred.VrfOut = crypto.Digest{2} if v1.Cred.Less(v2.Cred) { require.False(t, v2.Cred.Less(v1.Cred)) - b.AddInOutPair(inMsg1, filteredEvent{T: voteFiltered, CredentialTrackingNote: VerifiedBetterCredentialForTracking}) - b.AddInOutPair(inMsg2, filteredEvent{T: voteFiltered, CredentialTrackingNote: NoCredentialTrackingImpact}) + b.AddInOutPair(inMsg1, filteredEvent{T: voteFiltered, LateCredentialTrackingNote: VerifiedBetterLateCredentialForTracking}) + b.AddInOutPair(inMsg2, filteredEvent{T: voteFiltered, LateCredentialTrackingNote: NoLateCredentialTrackingImpact}) } else { require.True(t, v2.Cred.Less(v1.Cred)) - b.AddInOutPair(inMsg2, filteredEvent{T: voteFiltered, CredentialTrackingNote: VerifiedBetterCredentialForTracking}) - b.AddInOutPair(inMsg1, filteredEvent{T: voteFiltered, CredentialTrackingNote: NoCredentialTrackingImpact}) + b.AddInOutPair(inMsg2, filteredEvent{T: voteFiltered, LateCredentialTrackingNote: VerifiedBetterLateCredentialForTracking}) + b.AddInOutPair(inMsg1, filteredEvent{T: voteFiltered, LateCredentialTrackingNote: NoLateCredentialTrackingImpact}) } res, err := b.Build().Validate(pM) @@ -400,7 +400,7 @@ func TestProposalFreshAdjacentPeriods(t *testing.T) { }, }, } - b.AddInOutPair(inMsg, filteredEvent{T: voteFiltered, CredentialTrackingNote: UnverifiedBetterCredentialForTracking}) + b.AddInOutPair(inMsg, filteredEvent{T: voteFiltered, LateCredentialTrackingNote: UnverifiedLateCredentialForTracking}) // vote from credentialRoundLag ago and period > 0 should be filtered pV = helper.MakeRandomProposalValue() diff --git a/agreement/proposalTracker.go b/agreement/proposalTracker.go index 6d4ea0e85d..de89f8e3e9 100644 --- a/agreement/proposalTracker.go +++ b/agreement/proposalTracker.go @@ -43,34 +43,34 @@ type proposalSeeker struct { // accept compares a given vote with the current lowest-credentialled vote and // sets it if freeze has not been called. Returns: // - updated proposalSeeker state, -// - a CredentialTrackingEffect describing the usefulness of proposal-vote's +// - a LateCredentialTrackingEffect describing the usefulness of proposal-vote's // credential for late credential tracking (for choosing dynamic filter timeout), // - and an error if the proposal was not better than the lowest seen, or the // seeker was already frozen. -func (s proposalSeeker) accept(v vote) (proposalSeeker, CredentialTrackingEffect, error) { +func (s proposalSeeker) accept(v vote) (proposalSeeker, LateCredentialTrackingEffect, error) { if s.Frozen { - effect := NoCredentialTrackingImpact + effect := NoLateCredentialTrackingImpact // continue tracking and forwarding the lowest proposal even when frozen if !s.hasLowestIncludingLate || v.Cred.Less(s.lowestIncludingLate.Cred) { s.lowestIncludingLate = v s.hasLowestIncludingLate = true - effect = VerifiedBetterCredentialForTracking + effect = VerifiedBetterLateCredentialForTracking } return s, effect, errProposalSeekerFrozen{} } if s.Filled && !v.Cred.Less(s.Lowest.Cred) { - return s, NoCredentialTrackingImpact, errProposalSeekerNotLess{NewSender: v.R.Sender, LowestSender: s.Lowest.R.Sender} + return s, NoLateCredentialTrackingImpact, errProposalSeekerNotLess{NewSender: v.R.Sender, LowestSender: s.Lowest.R.Sender} } s.Lowest = v s.Filled = true s.lowestIncludingLate = v s.hasLowestIncludingLate = true - return s, VerifiedBetterCredentialForTracking, nil + return s, VerifiedBetterLateCredentialForTracking, nil } -func (s *proposalSeeker) copyCredentialTrackingState(s2 proposalSeeker) { +func (s *proposalSeeker) copyLateCredentialTrackingState(s2 proposalSeeker) { s.hasLowestIncludingLate = s2.hasLowestIncludingLate s.lowestIncludingLate = s2.lowestIncludingLate } @@ -168,15 +168,15 @@ func (t *proposalTracker) handle(r routerHandle, p player, e event) event { t.Duplicate[v.R.Sender] = true newFreezer, effect, err := t.Freezer.accept(v) - t.Freezer.copyCredentialTrackingState(newFreezer) + t.Freezer.copyLateCredentialTrackingState(newFreezer) if t.Staging != bottom { err = errProposalTrackerStaged{} - return filteredEvent{T: voteFiltered, CredentialTrackingNote: effect, Err: makeSerErr(err)} + return filteredEvent{T: voteFiltered, LateCredentialTrackingNote: effect, Err: makeSerErr(err)} } if err != nil { err := errProposalTrackerPS{Sub: err} - return filteredEvent{T: voteFiltered, CredentialTrackingNote: effect, Err: makeSerErr(err)} + return filteredEvent{T: voteFiltered, LateCredentialTrackingNote: effect, Err: makeSerErr(err)} } t.Freezer = newFreezer diff --git a/agreement/proposalTracker_test.go b/agreement/proposalTracker_test.go index 763fe37e8e..164e9db40b 100644 --- a/agreement/proposalTracker_test.go +++ b/agreement/proposalTracker_test.go @@ -66,10 +66,10 @@ func TestProposalTrackerProposalSeeker(t *testing.T) { assert.False(t, s.hasLowestIncludingLate) // issue events in the following order: 2, 3, 1, (freeze), 0 - var effect CredentialTrackingEffect + var effect LateCredentialTrackingEffect s, effect, err = s.accept(votes[2]) assert.NoError(t, err) - assert.Equal(t, effect, VerifiedBetterCredentialForTracking) + assert.Equal(t, effect, VerifiedBetterLateCredentialForTracking) assert.False(t, s.Frozen) assert.True(t, s.Filled) assert.True(t, s.Lowest.equals(votes[2])) @@ -78,7 +78,7 @@ func TestProposalTrackerProposalSeeker(t *testing.T) { s, effect, err = s.accept(votes[3]) assert.Error(t, err) - assert.Equal(t, effect, NoCredentialTrackingImpact) + assert.Equal(t, effect, NoLateCredentialTrackingImpact) assert.False(t, s.Frozen) assert.True(t, s.Filled) assert.True(t, s.Lowest.equals(votes[2])) @@ -87,7 +87,7 @@ func TestProposalTrackerProposalSeeker(t *testing.T) { s, effect, err = s.accept(votes[1]) assert.NoError(t, err) - assert.Equal(t, effect, VerifiedBetterCredentialForTracking) + assert.Equal(t, effect, VerifiedBetterLateCredentialForTracking) assert.False(t, s.Frozen) assert.True(t, s.Filled) assert.True(t, s.Lowest.equals(votes[1])) @@ -104,7 +104,7 @@ func TestProposalTrackerProposalSeeker(t *testing.T) { s, effect, err = s.accept(votes[0]) assert.Error(t, err) - assert.Equal(t, effect, VerifiedBetterCredentialForTracking) + assert.Equal(t, effect, VerifiedBetterLateCredentialForTracking) assert.Equal(t, s.Lowest, lowestBeforeFreeze) assert.True(t, s.Frozen) assert.True(t, s.Filled) From b4f8598a5e139774154474480cb374a1d8f5db8e Mon Sep 17 00:00:00 2001 From: cce <51567+cce@users.noreply.github.com> Date: Wed, 11 Oct 2023 09:55:04 -0400 Subject: [PATCH 227/232] Update agreement/player.go Co-authored-by: Jason Paulos --- agreement/player.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agreement/player.go b/agreement/player.go index 4c4f565553..21d65ecf88 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -635,7 +635,7 @@ func (p *player) handleMessageEvent(r routerHandle, e messageEvent) (actions []a case UnverifiedLateCredentialForTracking: // In this case, the vote may impact credential tracking, but needs to // be validated. So we do not return here, and continue processing, so that - // the votePresent chdck below will make a verifyVoteAction for this vote. + // the votePresent check below will make a verifyVoteAction for this vote. } } From aa952c037719939619b4d7264f6c924a3be1ed51 Mon Sep 17 00:00:00 2001 From: chris erway Date: Wed, 11 Oct 2023 11:30:01 -0400 Subject: [PATCH 228/232] add clockForRound to support AttachReceivedAt/AttachValidatedAt --- agreement/actions.go | 5 +- agreement/demux.go | 6 +- agreement/events.go | 39 ++++---- agreement/player_test.go | 199 ++++++++++++++++++++------------------- agreement/service.go | 11 ++- 5 files changed, 137 insertions(+), 123 deletions(-) diff --git a/agreement/actions.go b/agreement/actions.go index 136fb10bcf..0e0d5f19c4 100644 --- a/agreement/actions.go +++ b/agreement/actions.go @@ -346,11 +346,14 @@ func (a rezeroAction) ComparableStr() string { func (a rezeroAction) do(ctx context.Context, s *Service) { s.Clock = s.Clock.Zero() + // Preserve the zero time of the new round a.Round (for + // period 0) for future use if a late proposal-vote arrives, + // for late credential tracking. if _, ok := s.historicalClocks[a.Round]; !ok { s.historicalClocks[a.Round] = s.Clock } - // garbage collect clocks that are too old + // Garbage collect clocks that are too old for rnd := range s.historicalClocks { if a.Round > rnd+credentialRoundLag { delete(s.historicalClocks, rnd) diff --git a/agreement/demux.go b/agreement/demux.go index 41f2f2df1c..7f888dc97f 100644 --- a/agreement/demux.go +++ b/agreement/demux.go @@ -199,13 +199,13 @@ func (d *demux) next(s *Service, deadline Deadline, fastDeadline Deadline, curre switch e.t() { case payloadVerified: - e = e.(messageEvent).AttachValidatedAt(s.Clock.Since(), currentRound, s.historicalClocks) + e = e.(messageEvent).AttachValidatedAt(clockForRound(currentRound, s.Clock, s.historicalClocks)) case payloadPresent, votePresent: - e = e.(messageEvent).AttachReceivedAt(s.Clock.Since(), currentRound, s.historicalClocks) + e = e.(messageEvent).AttachReceivedAt(clockForRound(currentRound, s.Clock, s.historicalClocks)) case voteVerified: // if this is a proposal vote (step 0), record the validatedAt time on the vote if e.(messageEvent).Input.Vote.R.Step == 0 { - e = e.(messageEvent).AttachValidatedAt(s.Clock.Since(), currentRound, s.historicalClocks) + e = e.(messageEvent).AttachValidatedAt(clockForRound(currentRound, s.Clock, s.historicalClocks)) } } }() diff --git a/agreement/events.go b/agreement/events.go index fdc3c0161d..ba9ed05c2e 100644 --- a/agreement/events.go +++ b/agreement/events.go @@ -1010,28 +1010,35 @@ func (e checkpointEvent) AttachConsensusVersion(v ConsensusVersionView) external // is still waiting for quorum on R. const pipelinedMessageTimestamp = time.Nanosecond -func getTimestampForEvent(eventRound round, d time.Duration, currentRound round, historicalClocks map[round]historicalClock) time.Duration { - if eventRound > currentRound { - return pipelinedMessageTimestamp - } - if eventRound == currentRound { - return d - } - if clock, ok := historicalClocks[eventRound]; ok { - return clock.Since() +type constantRoundZeroTimer time.Duration + +func (c constantRoundZeroTimer) Since() time.Duration { return time.Duration(c) } + +// clockForRound retrieves the roundZeroTimer used for AttachValidatedAt and AttachReceivedAt. +func clockForRound(currentRound round, currentClock roundZeroTimer, historicalClocks map[round]roundZeroTimer) func(round) roundZeroTimer { + return func(eventRound round) roundZeroTimer { + if eventRound > currentRound { + return constantRoundZeroTimer(pipelinedMessageTimestamp) + } + if eventRound == currentRound { + return currentClock + } + if clock, ok := historicalClocks[eventRound]; ok { + return clock + } + return constantRoundZeroTimer(0) } - return time.Duration(0) } // AttachValidatedAt looks for a validated proposal or vote inside a // payloadVerified or voteVerified messageEvent, and attaches the given time to // the proposal's validatedAt field. -func (e messageEvent) AttachValidatedAt(d time.Duration, currentRound round, historicalClocks map[round]historicalClock) messageEvent { +func (e messageEvent) AttachValidatedAt(getClock func(eventRound round) roundZeroTimer) messageEvent { switch e.T { case payloadVerified: - e.Input.Proposal.validatedAt = getTimestampForEvent(e.Input.Proposal.Round(), d, currentRound, historicalClocks) + e.Input.Proposal.validatedAt = getClock(e.Input.Proposal.Round()).Since() case voteVerified: - e.Input.Vote.validatedAt = getTimestampForEvent(e.Input.Vote.R.Round, d, currentRound, historicalClocks) + e.Input.Vote.validatedAt = getClock(e.Input.Vote.R.Round).Since() } return e } @@ -1039,10 +1046,10 @@ func (e messageEvent) AttachValidatedAt(d time.Duration, currentRound round, his // AttachReceivedAt looks for an unauthenticatedProposal inside a // payloadPresent or votePresent messageEvent, and attaches the given // time to the proposal's receivedAt field. -func (e messageEvent) AttachReceivedAt(d time.Duration, currentRound round, historicalClocks map[round]historicalClock) messageEvent { +func (e messageEvent) AttachReceivedAt(getClock func(eventRound round) roundZeroTimer) messageEvent { switch e.T { case payloadPresent: - e.Input.UnauthenticatedProposal.receivedAt = getTimestampForEvent(e.Input.UnauthenticatedProposal.Round(), d, currentRound, historicalClocks) + e.Input.UnauthenticatedProposal.receivedAt = getClock(e.Input.UnauthenticatedProposal.Round()).Since() case votePresent: // Check for non-nil Tail, indicating this votePresent event // contains a synthetic payloadPresent event that was attached @@ -1051,7 +1058,7 @@ func (e messageEvent) AttachReceivedAt(d time.Duration, currentRound round, hist // The tail event is payloadPresent, serialized together // with the proposal vote as a single CompoundMessage // using a protocol.ProposalPayloadTag network message. - e.Tail.Input.UnauthenticatedProposal.receivedAt = getTimestampForEvent(e.Tail.Input.UnauthenticatedProposal.Round(), d, currentRound, historicalClocks) + e.Tail.Input.UnauthenticatedProposal.receivedAt = getClock(e.Tail.Input.UnauthenticatedProposal.Round()).Since() } } return e diff --git a/agreement/player_test.go b/agreement/player_test.go index 80f51417f5..4f25d3f14d 100644 --- a/agreement/player_test.go +++ b/agreement/player_test.go @@ -3247,13 +3247,13 @@ func TestPlayerRetainsReceivedValidatedAtOneSample(t *testing.T) { pP, pV := helper.MakeRandomProposalPayload(t, r-1) // send voteVerified message for round r-credentialRoundLag-1, then for r-1 - sendVoteVerified(t, helper, pM, 0, r-1, r-1, p, pV, 502*time.Millisecond, nil) + sendVoteVerified(t, helper, pWhite, pM, 0, r-1, r-1, p, pV, 502*time.Millisecond, nil) // send payloadPresent message for r-1 - sendPayloadPresent(t, pM, r-1, pP, time.Second, nil) + sendPayloadPresent(t, pWhite, pM, r-1, pP, time.Second, nil) // advance player to R and check timings ensured for R-1 are correct moveToRound(t, pWhite, pM, helper, r, p, pP, pV, 2*time.Second, version) - assertPayloadTimings(t, pM, r-1, pV, time.Second, 2*time.Second) + assertPayloadTimings(t, pWhite, pM, r-1, pV, time.Second, 2*time.Second) } // test that ReceivedAt and ValidateAt timing information are retained in proposalStore @@ -3273,8 +3273,8 @@ func TestPlayerRetainsReceivedValidatedAtCredentialHistory(t *testing.T) { payloadVerifiedTiming := 2001 * time.Millisecond for rnd := r - credentialRoundLag - 1; rnd < r-1; rnd++ { pP, pV := helper.MakeRandomProposalPayload(t, rnd) - sendVoteVerified(t, helper, pM, 0, rnd, rnd, p, pV, voteVerifiedTiming, nil) - sendPayloadPresent(t, pM, rnd, pP, payloadPresentTiming, nil) + sendVoteVerified(t, helper, pWhite, pM, 0, rnd, rnd, p, pV, voteVerifiedTiming, nil) + sendPayloadPresent(t, pWhite, pM, rnd, pP, payloadPresentTiming, nil) moveToRound(t, pWhite, pM, helper, rnd+1, p, pP, pV, payloadVerifiedTiming, version) voteVerifiedTiming += time.Millisecond @@ -3284,11 +3284,11 @@ func TestPlayerRetainsReceivedValidatedAtCredentialHistory(t *testing.T) { // send in voteVerified and payloadPresent for r-1 pP, pV := helper.MakeRandomProposalPayload(t, r-1) - sendVoteVerified(t, helper, pM, 0, r-1, r-1, p, pV, 600*time.Millisecond, nil) - sendPayloadPresent(t, pM, r-1, pP, 1500*time.Millisecond, nil) + sendVoteVerified(t, helper, pWhite, pM, 0, r-1, r-1, p, pV, 600*time.Millisecond, nil) + sendPayloadPresent(t, pWhite, pM, r-1, pP, 1500*time.Millisecond, nil) // advance player to R and check timings ensured for R-1 are correct moveToRound(t, pWhite, pM, helper, r, p, pP, pV, 2500*time.Millisecond, version) - assertPayloadTimings(t, pM, r-1, pV, 1500*time.Millisecond, 2500*time.Millisecond) + assertPayloadTimings(t, pWhite, pM, r-1, pV, 1500*time.Millisecond, 2500*time.Millisecond) // player is looking up arrival times from r-roundLag ago so only the 501ms vote will be in lowestCredentialArrivals assertSingleCredentialArrival(t, pWhite, 501*time.Millisecond) @@ -3308,27 +3308,28 @@ func TestPlayerRetainsEarlyReceivedValidatedAtOneSample(t *testing.T) { // send voteVerified message pP, pV := helper.MakeRandomProposalPayload(t, r-credentialRoundLag-1) - sendVoteVerified(t, helper, pM, 0, r-credentialRoundLag-2, r-credentialRoundLag-1, p, pV, 401*time.Millisecond, nil) + sendVoteVerified(t, helper, pWhite, pM, 0, r-credentialRoundLag-2, r-credentialRoundLag-1, p, pV, 401*time.Millisecond, nil) // send voteVerified message pP, pV = helper.MakeRandomProposalPayload(t, r-1) - sendVoteVerified(t, helper, pM, 0, r-1, r-1, p, pV, 501*time.Millisecond, nil) + sendVoteVerified(t, helper, pWhite, pM, 0, r-1, r-1, p, pV, 501*time.Millisecond, nil) // send payloadPresent message - sendPayloadPresent(t, pM, r-1, pP, time.Second, nil) + sendPayloadPresent(t, pWhite, pM, r-1, pP, time.Second, nil) // advance player to R and check timings ensured for R-1 are correct moveToRound(t, pWhite, pM, helper, r, p, pP, pV, 2*time.Second, version) - assertPayloadTimings(t, pM, r-1, pV, time.Second, 2*time.Second) + assertPayloadTimings(t, pWhite, pM, r-1, pV, time.Second, 2*time.Second) // assert lowest vote validateAt time was recorded into payloadArrivals assertSingleCredentialArrival(t, pWhite, pipelinedMessageTimestamp) } -type fixedHistoricalClock time.Duration - -func (f fixedHistoricalClock) Since() time.Duration { - return time.Duration(f) +func testClockForRound(t *testing.T, pWhite *player, fixedDur time.Duration, currentRound round, historicalClocks map[round]roundZeroTimer) func(round) roundZeroTimer { + return func(eventRound round) roundZeroTimer { + //require.Equal(t, pWhite.Round, currentRound) // TODO make tests more realistic + return clockForRound(currentRound, constantRoundZeroTimer(fixedDur), historicalClocks)(eventRound) + } } // test that ReceivedAt and ValidateAt timing information are retained in @@ -3343,24 +3344,24 @@ func TestPlayerRetainsLateReceivedValidatedAtOneSample(t *testing.T) { const p = period(0) pWhite, pM, helper := setupP(t, r-1, p, soft) - historicalClocks := map[round]historicalClock{ - r - credentialRoundLag - 1: fixedHistoricalClock(900 * time.Millisecond), + historicalClocks := map[round]roundZeroTimer{ + r - credentialRoundLag - 1: constantRoundZeroTimer(900 * time.Millisecond), } // send voteVerified message pP, pV := helper.MakeRandomProposalPayload(t, r-credentialRoundLag-1) - sendVoteVerified(t, helper, pM, 0, r-1, r-credentialRoundLag-1, p, pV, 401*time.Millisecond, historicalClocks) + sendVoteVerified(t, helper, pWhite, pM, 0, r-1, r-credentialRoundLag-1, p, pV, 401*time.Millisecond, historicalClocks) // send voteVerified message pP, pV = helper.MakeRandomProposalPayload(t, r-1) - sendVoteVerified(t, helper, pM, 0, r-1, r-1, p, pV, 501*time.Millisecond, nil) + sendVoteVerified(t, helper, pWhite, pM, 0, r-1, r-1, p, pV, 501*time.Millisecond, nil) // send payloadPresent message - sendPayloadPresent(t, pM, r-1, pP, time.Second, nil) + sendPayloadPresent(t, pWhite, pM, r-1, pP, time.Second, nil) // advance player to R and check timings ensured for R-1 are correct moveToRound(t, pWhite, pM, helper, r, p, pP, pV, 2*time.Second, version) - assertPayloadTimings(t, pM, r-1, pV, time.Second, 2*time.Second) + assertPayloadTimings(t, pWhite, pM, r-1, pV, time.Second, 2*time.Second) // assert lowest vote validateAt time was recorded into payloadArrivals assertSingleCredentialArrival(t, pWhite, 900*time.Millisecond) @@ -3409,20 +3410,20 @@ func testPlayerRetainsReceivedValidatedAtForHistoryWindow(t *testing.T, addBette } inMsg := messageEvent{T: voteVerified, Input: message{Vote: vVote, UnauthenticatedVote: vVote.u()}} timestamp := 500 + i - inMsg = inMsg.AttachValidatedAt(time.Duration(timestamp)*time.Millisecond, r+round(i)-1, nil) + inMsg = inMsg.AttachValidatedAt(testClockForRound(t, pWhite, time.Duration(timestamp)*time.Millisecond, r+round(i)-1, nil)) err, panicErr := pM.transition(inMsg) require.NoError(t, err) require.NoError(t, panicErr) // send payloadPresent message - sendPayloadPresent(t, pM, r+round(i)-1, pP, time.Second, nil) + sendPayloadPresent(t, pWhite, pM, r+round(i)-1, pP, time.Second, nil) moveToRound(t, pWhite, pM, helper, r+round(i), p, pP, pV, 2*time.Second, version) // send better late voteVerified message if addBetterLate { inMsg = messageEvent{T: voteVerified, Input: message{Vote: betterLateVote, UnauthenticatedVote: betterLateVote.u()}} timestamp := 600 + i - inMsg = inMsg.AttachValidatedAt(time.Duration(timestamp)*time.Millisecond, r+round(i)-1, nil) + inMsg = inMsg.AttachValidatedAt(testClockForRound(t, pWhite, time.Duration(timestamp)*time.Millisecond, r+round(i)-1, nil)) err, panicErr = pM.transition(inMsg) require.NoError(t, err) require.NoError(t, panicErr) @@ -3457,38 +3458,38 @@ func TestPlayerRetainsReceivedValidatedAtPPOneSample(t *testing.T) { // Move to round r, no credentials arrived. // send voteVerified message - sendVoteVerified(t, helper, pM, 0, r-1, r-1, p, pV, 501*time.Millisecond, nil) + sendVoteVerified(t, helper, pWhite, pM, 0, r-1, r-1, p, pV, 501*time.Millisecond, nil) // send payloadPresent message - sendPayloadPresent(t, pM, r-1, pP, time.Second, nil) + sendPayloadPresent(t, pWhite, pM, r-1, pP, time.Second, nil) moveToRound(t, pWhite, pM, helper, r, p, pP, pV, 2*time.Second, version) - assertPayloadTimings(t, pM, r-1, pV, time.Second, 2*time.Second) + assertPayloadTimings(t, pWhite, pM, r-1, pV, time.Second, 2*time.Second) require.False(t, pWhite.lowestCredentialArrivals.isFull()) require.Equal(t, pWhite.lowestCredentialArrivals.writePtr, 0) // XXX this behavior only happens if dynamic timeout enabled; test the other way - historicalClocks := map[round]historicalClock{ - r - credentialRoundLag: fixedHistoricalClock(900 * time.Millisecond), + historicalClocks := map[round]roundZeroTimer{ + r - credentialRoundLag: constantRoundZeroTimer(900 * time.Millisecond), } // create a PP message for the round we're going to take the sample from when round r-1 ends pP, pV = helper.MakeRandomProposalPayload(t, r-credentialRoundLag) - vVote := sendCompoundMessage(t, helper, pM, r, r-credentialRoundLag, p, pP, pV, time.Second, nil, version) + vVote := sendCompoundMessage(t, helper, pWhite, pM, r, r-credentialRoundLag, p, pP, pV, time.Second, nil, version) verifyEvent := ev(verifyVoteAction(messageEvent{Input: message{UnauthenticatedVote: vVote.u()}}, r-credentialRoundLag, p, 1)) require.Truef(t, pM.getTrace().Contains(verifyEvent), "Player should verify vote") - sendVoteVerifiedForVote(t, vVote, pM, r, 502*time.Millisecond, historicalClocks, 1) + sendVoteVerifiedForVote(t, vVote, pWhite, pM, r, 502*time.Millisecond, historicalClocks, 1) // send payloadPresent message - sendPayloadPresent(t, pM, r, pP, time.Second, nil) + sendPayloadPresent(t, pWhite, pM, r, pP, time.Second, nil) // move to round r+1, triggering history update pP, pV = helper.MakeRandomProposalPayload(t, r) - sendVoteVerified(t, helper, pM, 0, r, r, p, pV, 501*time.Millisecond, nil) + sendVoteVerified(t, helper, pWhite, pM, 0, r, r, p, pV, 501*time.Millisecond, nil) // send payloadPresent message - sendPayloadPresent(t, pM, r, pP, time.Second, nil) + sendPayloadPresent(t, pWhite, pM, r, pP, time.Second, nil) moveToRound(t, pWhite, pM, helper, r+1, p, pP, pV, 2*time.Second, version) @@ -3514,36 +3515,36 @@ func TestPlayerRetainsEarlyReceivedValidatedAtPPOneSample(t *testing.T) { // Move to round r, no credentials arrived. // send voteVerified message - sendVoteVerified(t, helper, pM, 0, r-1, r-1, p, pV, 501*time.Millisecond, nil) + sendVoteVerified(t, helper, pWhite, pM, 0, r-1, r-1, p, pV, 501*time.Millisecond, nil) // send payloadPresent message - sendPayloadPresent(t, pM, r-1, pP, time.Second, nil) + sendPayloadPresent(t, pWhite, pM, r-1, pP, time.Second, nil) moveToRound(t, pWhite, pM, helper, r, p, pP, pV, 2*time.Second, version) - assertPayloadTimings(t, pM, r-1, pV, time.Second, 2*time.Second) + assertPayloadTimings(t, pWhite, pM, r-1, pV, time.Second, 2*time.Second) require.False(t, pWhite.lowestCredentialArrivals.isFull()) require.Equal(t, pWhite.lowestCredentialArrivals.writePtr, 0) // create a PP message for the round we're going to take the sample from when round r-1 ends // Now we're going to pretend we got the message one round early. pP, pV = helper.MakeRandomProposalPayload(t, r-credentialRoundLag) - vVote := sendCompoundMessage(t, helper, pM, r-credentialRoundLag-1, r-credentialRoundLag, p, pP, pV, time.Second, nil, version) + vVote := sendCompoundMessage(t, helper, pWhite, pM, r-credentialRoundLag-1, r-credentialRoundLag, p, pP, pV, time.Second, nil, version) // make sure vote verify requests verifyEvent := ev(verifyVoteAction(messageEvent{Input: message{UnauthenticatedVote: vVote.u()}}, r-credentialRoundLag, p, 1)) require.Truef(t, pM.getTrace().Contains(verifyEvent), "Player should verify vote") - sendVoteVerifiedForVote(t, vVote, pM, r-credentialRoundLag, 502*time.Millisecond, nil, 1) + sendVoteVerifiedForVote(t, vVote, pWhite, pM, r-credentialRoundLag, 502*time.Millisecond, nil, 1) // send payloadPresent message - sendPayloadPresent(t, pM, r-credentialRoundLag, pP, time.Second, nil) + sendPayloadPresent(t, pWhite, pM, r-credentialRoundLag, pP, time.Second, nil) // move to round r+1, triggering history update pP, pV = helper.MakeRandomProposalPayload(t, r) - sendVoteVerified(t, helper, pM, 0, r, r, p, pV, 501*time.Millisecond, nil) + sendVoteVerified(t, helper, pWhite, pM, 0, r, r, p, pV, 501*time.Millisecond, nil) // send payloadPresent message - sendPayloadPresent(t, pM, r, pP, time.Second, nil) + sendPayloadPresent(t, pWhite, pM, r, pP, time.Second, nil) moveToRound(t, pWhite, pM, helper, r+1, p, pP, pV, 2*time.Second, version) // assert lowest vote validateAt time was recorded into payloadArrivals @@ -3567,36 +3568,36 @@ func TestPlayerRetainsLateReceivedValidatedAtPPOneSample(t *testing.T) { // Move to round r, no credentials arrived. // send voteVerified message - sendVoteVerified(t, helper, pM, 0, r-1, r-1, p, pV, 501*time.Millisecond, nil) + sendVoteVerified(t, helper, pWhite, pM, 0, r-1, r-1, p, pV, 501*time.Millisecond, nil) // send payloadPresent message - sendPayloadPresent(t, pM, r-1, pP, time.Second, nil) + sendPayloadPresent(t, pWhite, pM, r-1, pP, time.Second, nil) // Go from round r-1 to r moveToRound(t, pWhite, pM, helper, r, p, pP, pV, 2*time.Second, version) - assertPayloadTimings(t, pM, r-1, pV, time.Second, 2*time.Second) + assertPayloadTimings(t, pWhite, pM, r-1, pV, time.Second, 2*time.Second) require.False(t, pWhite.lowestCredentialArrivals.isFull()) require.Equal(t, pWhite.lowestCredentialArrivals.writePtr, 0) - historicalClocks := map[round]historicalClock{ - r - credentialRoundLag: fixedHistoricalClock(900 * time.Millisecond), + historicalClocks := map[round]roundZeroTimer{ + r - credentialRoundLag: constantRoundZeroTimer(900 * time.Millisecond), } // create a PP message for the round we're going to take the sample from when round r-1 ends // Now we're going to pretend we got the message credentialRoundLag too late. pP, pV = helper.MakeRandomProposalPayload(t, r-credentialRoundLag) - vVote := sendCompoundMessage(t, helper, pM, r, r-credentialRoundLag, p, pP, pV, time.Second, historicalClocks, version) + vVote := sendCompoundMessage(t, helper, pWhite, pM, r, r-credentialRoundLag, p, pP, pV, time.Second, historicalClocks, version) verifyEvent := ev(verifyVoteAction(messageEvent{Input: message{UnauthenticatedVote: vVote.u()}}, r-credentialRoundLag, p, 1)) require.Truef(t, pM.getTrace().Contains(verifyEvent), "Player should verify vote") - sendVoteVerifiedForVote(t, vVote, pM, r, 502*time.Millisecond, historicalClocks, 1) + sendVoteVerifiedForVote(t, vVote, pWhite, pM, r, 502*time.Millisecond, historicalClocks, 1) // move to round r+1, triggering history update pP, pV = helper.MakeRandomProposalPayload(t, r) - sendVoteVerified(t, helper, pM, 0, r, r, p, pV, 503*time.Millisecond, nil) + sendVoteVerified(t, helper, pWhite, pM, 0, r, r, p, pV, 503*time.Millisecond, nil) // send payloadPresent message - sendPayloadPresent(t, pM, r, pP, time.Second, nil) + sendPayloadPresent(t, pWhite, pM, r, pP, time.Second, nil) moveToRound(t, pWhite, pM, helper, r+1, p, pP, pV, 2*time.Second, version) // assert lowest vote validateAt time was recorded into payloadArrivals @@ -3622,7 +3623,7 @@ func TestPlayerRetainsReceivedValidatedAtPPForHistoryWindow(t *testing.T) { for i := 0; i < dynamicFilterCredentialArrivalHistory+int(credentialRoundLag); i++ { // create a PP message for an arbitrary proposal/payload similar to setupCompoundMessage pP, pV := helper.MakeRandomProposalPayload(t, r+round(i)-1) - vVote := sendCompoundMessage(t, helper, pM, r+round(i)-1, r+round(i)-1, p, pP, pV, time.Second, nil, version) + vVote := sendCompoundMessage(t, helper, pWhite, pM, r+round(i)-1, r+round(i)-1, p, pP, pV, time.Second, nil, version) // make sure vote verify requests taskIndex := uint64(i + 1) @@ -3631,8 +3632,8 @@ func TestPlayerRetainsReceivedValidatedAtPPForHistoryWindow(t *testing.T) { // send voteVerified timestamp := 500 + i - sendVoteVerifiedForVote(t, vVote, pM, r+round(i)-1, time.Duration(timestamp)*time.Millisecond, nil, taskIndex) - sendPayloadPresent(t, pM, r+round(i)-1, pP, time.Duration(timestamp)*time.Millisecond+time.Second, nil) + sendVoteVerifiedForVote(t, vVote, pWhite, pM, r+round(i)-1, time.Duration(timestamp)*time.Millisecond, nil, taskIndex) + sendPayloadPresent(t, pWhite, pM, r+round(i)-1, pP, time.Duration(timestamp)*time.Millisecond+time.Second, nil) moveToRound(t, pWhite, pM, helper, r+round(i), p, pP, pV, 2*time.Second+time.Duration(timestamp)*time.Millisecond, version) } @@ -3663,19 +3664,19 @@ func TestPlayerRetainsReceivedValidatedAtAVPPOneSample(t *testing.T) { // Move to round r, no credentials arrived. // send voteVerified message - sendVoteVerified(t, helper, pM, 0, r-1, r-1, p, pV, 501*time.Millisecond, nil) + sendVoteVerified(t, helper, pWhite, pM, 0, r-1, r-1, p, pV, 501*time.Millisecond, nil) // send payloadPresent message - sendPayloadPresent(t, pM, r-1, pP, time.Second, nil) + sendPayloadPresent(t, pWhite, pM, r-1, pP, time.Second, nil) moveToRound(t, pWhite, pM, helper, r, p, pP, pV, 2*time.Second, version) - assertPayloadTimings(t, pM, r-1, pV, time.Second, 2*time.Second) + assertPayloadTimings(t, pWhite, pM, r-1, pV, time.Second, 2*time.Second) require.False(t, pWhite.lowestCredentialArrivals.isFull()) require.Equal(t, pWhite.lowestCredentialArrivals.writePtr, 0) // send votePresent message (mimicking the first AV message validating) pP, pV = helper.MakeRandomProposalPayload(t, r-credentialRoundLag) - vVote := sendVotePresent(t, helper, pM, 0, r-credentialRoundLag, p, pV, version) + vVote := sendVotePresent(t, helper, pWhite, pM, 0, r-credentialRoundLag, p, pV, version) // make sure vote verify requests unverifiedVoteMsg := message{UnauthenticatedVote: vVote.u()} @@ -3683,17 +3684,17 @@ func TestPlayerRetainsReceivedValidatedAtAVPPOneSample(t *testing.T) { require.Truef(t, pM.getTrace().Contains(verifyEvent), "Player should verify vote") // send voteVerified - sendVoteVerifiedForVote(t, vVote, pM, r-credentialRoundLag, 502*time.Millisecond, nil, 1) + sendVoteVerifiedForVote(t, vVote, pWhite, pM, r-credentialRoundLag, 502*time.Millisecond, nil, 1) // create a PP message for an arbitrary proposal/payload similar to setupCompoundMessage - sendCompoundMessageForVote(t, vVote, pM, r-credentialRoundLag, pP, time.Second, nil, version) + sendCompoundMessageForVote(t, vVote, pWhite, pM, r-credentialRoundLag, pP, time.Second, nil, version) // move to round r+1, triggering history update pP, pV = helper.MakeRandomProposalPayload(t, r) - sendVoteVerified(t, helper, pM, 0, r, r, p, pV, time.Second, nil) + sendVoteVerified(t, helper, pWhite, pM, 0, r, r, p, pV, time.Second, nil) // send payloadPresent message - sendPayloadPresent(t, pM, r, pP, time.Second, nil) + sendPayloadPresent(t, pWhite, pM, r, pP, time.Second, nil) moveToRound(t, pWhite, pM, helper, r+1, p, pP, pV, 2*time.Second, version) // assert lowest vote validateAt time was recorded into payloadArrivals @@ -3717,13 +3718,13 @@ func TestPlayerRetainsEarlyReceivedValidatedAtAVPPOneSample(t *testing.T) { // Move to round r, no credentials arrived. // send voteVerified message - sendVoteVerified(t, helper, pM, 0, r-1, r-1, p, pV, 501*time.Millisecond, nil) + sendVoteVerified(t, helper, pWhite, pM, 0, r-1, r-1, p, pV, 501*time.Millisecond, nil) // send payloadPresent message - sendPayloadPresent(t, pM, r-1, pP, time.Second, nil) + sendPayloadPresent(t, pWhite, pM, r-1, pP, time.Second, nil) moveToRound(t, pWhite, pM, helper, r, p, pP, pV, 2*time.Second, version) - assertPayloadTimings(t, pM, r-1, pV, time.Second, 2*time.Second) + assertPayloadTimings(t, pWhite, pM, r-1, pV, time.Second, 2*time.Second) require.False(t, pWhite.lowestCredentialArrivals.isFull()) require.Equal(t, pWhite.lowestCredentialArrivals.writePtr, 0) @@ -3733,24 +3734,24 @@ func TestPlayerRetainsEarlyReceivedValidatedAtAVPPOneSample(t *testing.T) { // send votePresent message (mimicking the first AV message validating) pP, pV = helper.MakeRandomProposalPayload(t, r-credentialRoundLag) - vVote := sendVotePresent(t, helper, pM, 0, r-credentialRoundLag, p, pV, version) + vVote := sendVotePresent(t, helper, pWhite, pM, 0, r-credentialRoundLag, p, pV, version) // make sure vote verify requests verifyEvent := ev(verifyVoteAction(messageEvent{Input: message{UnauthenticatedVote: vVote.u()}}, r-credentialRoundLag, p, 1)) require.Truef(t, pM.getTrace().Contains(verifyEvent), "Player should verify vote") // send voteVerified, pretend we're one round too early - sendVoteVerifiedForVote(t, vVote, pM, r-credentialRoundLag-1, 502*time.Millisecond, nil, 1) + sendVoteVerifiedForVote(t, vVote, pWhite, pM, r-credentialRoundLag-1, 502*time.Millisecond, nil, 1) // create a PP message for an arbitrary proposal/payload similar to setupCompoundMessage - sendCompoundMessageForVote(t, vVote, pM, r-credentialRoundLag, pP, time.Second, nil, version) + sendCompoundMessageForVote(t, vVote, pWhite, pM, r-credentialRoundLag, pP, time.Second, nil, version) // move to round r+1, triggering history update pP, pV = helper.MakeRandomProposalPayload(t, r) - sendVoteVerified(t, helper, pM, 0, r, r, p, pV, time.Second, nil) + sendVoteVerified(t, helper, pWhite, pM, 0, r, r, p, pV, time.Second, nil) // send payloadPresent message - sendPayloadPresent(t, pM, r, pP, time.Second, nil) + sendPayloadPresent(t, pWhite, pM, r, pP, time.Second, nil) moveToRound(t, pWhite, pM, helper, r+1, p, pP, pV, 2*time.Second, version) // assert lowest vote validateAt time was recorded into payloadArrivals @@ -3774,13 +3775,13 @@ func TestPlayerRetainsLateReceivedValidatedAtAVPPOneSample(t *testing.T) { // Move to round r, no credentials arrived. // send voteVerified message - sendVoteVerified(t, helper, pM, 0, r-1, r-1, p, pV, 501*time.Millisecond, nil) + sendVoteVerified(t, helper, pWhite, pM, 0, r-1, r-1, p, pV, 501*time.Millisecond, nil) // send payloadPresent message - sendPayloadPresent(t, pM, r-1, pP, time.Second, nil) + sendPayloadPresent(t, pWhite, pM, r-1, pP, time.Second, nil) moveToRound(t, pWhite, pM, helper, r, p, pP, pV, 2*time.Second, version) - assertPayloadTimings(t, pM, r-1, pV, time.Second, 2*time.Second) + assertPayloadTimings(t, pWhite, pM, r-1, pV, time.Second, 2*time.Second) require.False(t, pWhite.lowestCredentialArrivals.isFull()) require.Equal(t, pWhite.lowestCredentialArrivals.writePtr, 0) @@ -3790,27 +3791,27 @@ func TestPlayerRetainsLateReceivedValidatedAtAVPPOneSample(t *testing.T) { // send votePresent message (mimicking the first AV message validating) pP, pV = helper.MakeRandomProposalPayload(t, r-credentialRoundLag) - vVote := sendVotePresent(t, helper, pM, 0, r-credentialRoundLag, p, pV, version) + vVote := sendVotePresent(t, helper, pWhite, pM, 0, r-credentialRoundLag, p, pV, version) // make sure vote verify requests verifyEvent := ev(verifyVoteAction(messageEvent{Input: message{UnauthenticatedVote: vVote.u()}}, r-credentialRoundLag, p, 1)) require.Truef(t, pM.getTrace().Contains(verifyEvent), "Player should verify vote") - historicalClocks := map[round]historicalClock{ - r - credentialRoundLag: fixedHistoricalClock(900 * time.Millisecond), + historicalClocks := map[round]roundZeroTimer{ + r - credentialRoundLag: constantRoundZeroTimer(900 * time.Millisecond), } // send voteVerified, pretend we're credentialRoundLag after the message was sent - sendVoteVerifiedForVote(t, vVote, pM, r, 502*time.Millisecond, historicalClocks, 1) + sendVoteVerifiedForVote(t, vVote, pWhite, pM, r, 502*time.Millisecond, historicalClocks, 1) // create a PP message for an arbitrary proposal/payload similar to setupCompoundMessage - sendCompoundMessageForVote(t, vVote, pM, r-credentialRoundLag, pP, time.Second, nil, version) + sendCompoundMessageForVote(t, vVote, pWhite, pM, r-credentialRoundLag, pP, time.Second, nil, version) // move to round r+1, triggering history update pP, pV = helper.MakeRandomProposalPayload(t, r) - sendVoteVerified(t, helper, pM, 0, r, r, p, pV, time.Second, nil) + sendVoteVerified(t, helper, pWhite, pM, 0, r, r, p, pV, time.Second, nil) // send payloadPresent message - sendPayloadPresent(t, pM, r, pP, time.Second, nil) + sendPayloadPresent(t, pWhite, pM, r, pP, time.Second, nil) moveToRound(t, pWhite, pM, helper, r+1, p, pP, pV, 2*time.Second, version) // assert lowest vote validateAt time was recorded into lowestCredentialArrivals @@ -3831,7 +3832,7 @@ func TestPlayerRetainsReceivedValidatedAtAVPPHistoryWindow(t *testing.T) { pP, pV := helper.MakeRandomProposalPayload(t, r+round(i)-1) // send votePresent message (mimicking the first AV message validating) - vVote := sendVotePresent(t, helper, pM, 0, r+round(i)-1, p, pV, version) + vVote := sendVotePresent(t, helper, pWhite, pM, 0, r+round(i)-1, p, pV, version) // make sure vote verify requests taskIndex := uint64(i + 1) @@ -3840,10 +3841,10 @@ func TestPlayerRetainsReceivedValidatedAtAVPPHistoryWindow(t *testing.T) { // send voteVerified timestamp := 500 + i - sendVoteVerifiedForVote(t, vVote, pM, r+round(i)-1, time.Duration(timestamp)*time.Millisecond, nil, taskIndex) + sendVoteVerifiedForVote(t, vVote, pWhite, pM, r+round(i)-1, time.Duration(timestamp)*time.Millisecond, nil, taskIndex) // create a PP message for an arbitrary proposal/payload similar to setupCompoundMessage - sendCompoundMessageForVote(t, vVote, pM, r+round(i)-1, pP, time.Second, nil, version) + sendCompoundMessageForVote(t, vVote, pWhite, pM, r+round(i)-1, pP, time.Second, nil, version) moveToRound(t, pWhite, pM, helper, r+round(i), p, pP, pV, 2*time.Second, version) } @@ -3858,23 +3859,23 @@ func TestPlayerRetainsReceivedValidatedAtAVPPHistoryWindow(t *testing.T) { } // Helper function to send voteVerified message -func sendVoteVerified(t *testing.T, helper *voteMakerHelper, pM ioAutomata, addrIndex int, +func sendVoteVerified(t *testing.T, helper *voteMakerHelper, pWhite *player, pM ioAutomata, addrIndex int, curRound round, voteRound round, votePeriod period, pV *proposalValue, validatedAt time.Duration, - historicalClocks map[round]historicalClock) { + historicalClocks map[round]roundZeroTimer) { vVote := helper.MakeVerifiedVote(t, addrIndex, voteRound, votePeriod, propose, *pV) - sendVoteVerifiedForVote(t, vVote, pM, curRound, validatedAt, historicalClocks, 0) + sendVoteVerifiedForVote(t, vVote, pWhite, pM, curRound, validatedAt, historicalClocks, 0) } -func sendVoteVerifiedForVote(t *testing.T, vVote vote, pM ioAutomata, - curRound round, validatedAt time.Duration, historicalClocks map[round]historicalClock, taskIndex uint64) { +func sendVoteVerifiedForVote(t *testing.T, vVote vote, pWhite *player, pM ioAutomata, + curRound round, validatedAt time.Duration, historicalClocks map[round]roundZeroTimer, taskIndex uint64) { inMsg := messageEvent{T: voteVerified, Input: message{Vote: vVote, UnauthenticatedVote: vVote.u()}, TaskIndex: taskIndex} - inMsg = inMsg.AttachValidatedAt(validatedAt, curRound, historicalClocks) + inMsg = inMsg.AttachValidatedAt(testClockForRound(t, pWhite, validatedAt, curRound, historicalClocks)) err, panicErr := pM.transition(inMsg) require.NoError(t, err) require.NoError(t, panicErr) } -func sendVotePresent(t *testing.T, helper *voteMakerHelper, pM ioAutomata, addrIndex int, +func sendVotePresent(t *testing.T, helper *voteMakerHelper, pWhite *player, pM ioAutomata, addrIndex int, voteRound round, votePeriod period, pV *proposalValue, version protocol.ConsensusVersion) vote { vVote := helper.MakeVerifiedVote(t, addrIndex, voteRound, votePeriod, propose, *pV) inMsg := messageEvent{T: votePresent, Input: message{UnauthenticatedVote: vVote.u()}, Proto: ConsensusVersionView{Version: version}} @@ -3885,23 +3886,23 @@ func sendVotePresent(t *testing.T, helper *voteMakerHelper, pM ioAutomata, addrI } // Helper function to send payloadPresent message -func sendPayloadPresent(t *testing.T, pM ioAutomata, curRound round, pP *proposal, receivedAt time.Duration, historicalClocks map[round]historicalClock) { +func sendPayloadPresent(t *testing.T, pWhite *player, pM ioAutomata, curRound round, pP *proposal, receivedAt time.Duration, historicalClocks map[round]roundZeroTimer) { m := message{UnauthenticatedProposal: pP.u()} inMsg := messageEvent{T: payloadPresent, Input: m} - inMsg = inMsg.AttachReceivedAt(receivedAt, curRound, historicalClocks) + inMsg = inMsg.AttachReceivedAt(testClockForRound(t, pWhite, receivedAt, curRound, historicalClocks)) err, panicErr := pM.transition(inMsg) require.NoError(t, err) require.NoError(t, panicErr) } // Helper function to send a compound PP message (votePresent + payloadPresent) -func sendCompoundMessage(t *testing.T, helper *voteMakerHelper, pM ioAutomata, curRound round, voteRound round, votePeriod period, pP *proposal, pV *proposalValue, receivedAt time.Duration, historicalClocks map[round]historicalClock, version protocol.ConsensusVersion) vote { +func sendCompoundMessage(t *testing.T, helper *voteMakerHelper, pWhite *player, pM ioAutomata, curRound round, voteRound round, votePeriod period, pP *proposal, pV *proposalValue, receivedAt time.Duration, historicalClocks map[round]roundZeroTimer, version protocol.ConsensusVersion) vote { vVote := helper.MakeVerifiedVote(t, 0, voteRound, votePeriod, propose, *pV) - sendCompoundMessageForVote(t, vVote, pM, curRound, pP, receivedAt, historicalClocks, version) + sendCompoundMessageForVote(t, vVote, pWhite, pM, curRound, pP, receivedAt, historicalClocks, version) return vVote } -func sendCompoundMessageForVote(t *testing.T, vVote vote, pM ioAutomata, curRound round, pP *proposal, receivedAt time.Duration, historicalClocks map[round]historicalClock, version protocol.ConsensusVersion) { +func sendCompoundMessageForVote(t *testing.T, vVote vote, pWhite *player, pM ioAutomata, curRound round, pP *proposal, receivedAt time.Duration, historicalClocks map[round]roundZeroTimer, version protocol.ConsensusVersion) { unverifiedVoteMsg := message{UnauthenticatedVote: vVote.u()} proposalMsg := message{UnauthenticatedProposal: pP.u()} compoundMsg := messageEvent{ @@ -3914,7 +3915,7 @@ func sendCompoundMessageForVote(t *testing.T, vVote vote, pM ioAutomata, curRoun }, Proto: ConsensusVersionView{Version: version}, } - inMsg := compoundMsg.AttachReceivedAt(receivedAt, curRound, historicalClocks) // call AttachReceivedAt like demux would + inMsg := compoundMsg.AttachReceivedAt(testClockForRound(t, pWhite, receivedAt, curRound, historicalClocks)) // call AttachReceivedAt like demux would err, panicErr := pM.transition(inMsg) require.NoError(t, err) require.NoError(t, panicErr) @@ -3940,7 +3941,7 @@ func moveToRound(t *testing.T, pWhite *player, pM ioAutomata, helper *voteMakerH // payloadVerified inMsg := messageEvent{T: payloadVerified, Input: message{Proposal: *pP}, Proto: ConsensusVersionView{Version: ver}} - inMsg = inMsg.AttachValidatedAt(validatedAt, r-1, nil) // call AttachValidatedAt like demux would + inMsg = inMsg.AttachValidatedAt(testClockForRound(t, pWhite, validatedAt, r-1, nil)) // call AttachValidatedAt like demux would err, panicErr := pM.transition(inMsg) require.NoError(t, err) require.NoError(t, panicErr) @@ -3984,7 +3985,7 @@ func moveToRound(t *testing.T, pWhite *player, pM ioAutomata, helper *voteMakerH } // inspect the ensureAction for round R and assert the correct payload timings -func assertPayloadTimings(t *testing.T, pM ioAutomata, r round, pV *proposalValue, receivedAt time.Duration, validatedAt time.Duration) { +func assertPayloadTimings(t *testing.T, pWhite *player, pM ioAutomata, r round, pV *proposalValue, receivedAt time.Duration, validatedAt time.Duration) { // find and unwrap ensureAction from trace var ea ensureAction diff --git a/agreement/service.go b/agreement/service.go index 9a4a4dce8f..1e9b9e0d79 100644 --- a/agreement/service.go +++ b/agreement/service.go @@ -58,7 +58,8 @@ type Service struct { persistStatus player persistActions []action - historicalClocks map[round]historicalClock + // Retain the round period 0 + historicalClocks map[round]roundZeroTimer } // Parameters holds the parameters necessary to run the agreement protocol. @@ -87,8 +88,10 @@ type externalDemuxSignals struct { CurrentRound round } -// an interface representing a clock from a previous round -type historicalClock interface { +// an interface allowing for measuring the duration since a clock from a previous round, +// used for measuring the arrival time of a late proposal-vote, for the dynamic filter +// timeout feature +type roundZeroTimer interface { Since() time.Duration } @@ -118,7 +121,7 @@ func MakeService(p Parameters) (*Service, error) { s.persistenceLoop = makeAsyncPersistenceLoop(s.log, s.Accessor, s.Ledger) - s.historicalClocks = make(map[round]historicalClock) + s.historicalClocks = make(map[round]roundZeroTimer) return s, nil } From 81c4bc86cb4febc5c79d940250a7066a909e8258 Mon Sep 17 00:00:00 2001 From: chris erway Date: Wed, 11 Oct 2023 11:42:42 -0400 Subject: [PATCH 229/232] remove unused errProposalManagerPVFreshness and errProposalManagerPVDuplicate --- agreement/proposalManager.go | 27 +++------------------------ 1 file changed, 3 insertions(+), 24 deletions(-) diff --git a/agreement/proposalManager.go b/agreement/proposalManager.go index cf89063641..71bea3df53 100644 --- a/agreement/proposalManager.go +++ b/agreement/proposalManager.go @@ -18,8 +18,6 @@ package agreement import ( "fmt" - - "github.com/algorand/go-algorand/data/basics" ) // A proposalManager is a proposalMachine which applies relay rules to incoming @@ -247,26 +245,6 @@ func (m *proposalManager) handleMessageEvent(r routerHandle, p player, e filtera } } -// errProposalManagerPVFreshness indicates that filterProposalVote failed the proposalFresh check. -type errProposalManagerPVNotFresh struct { - Reason error -} - -func (a errProposalManagerPVNotFresh) Error() string { - return fmt.Sprintf("proposalManager: filtered proposal-vote due to age: %v", a.Reason) -} - -// errProposalManagerPVDuplicate idnicates that filterProposalVote failed the duplicate check. -type errProposalManagerPVDuplicate struct { - Sender basics.Address - Round round - Period period -} - -func (d errProposalManagerPVDuplicate) Error() string { - return fmt.Sprintf("proposalManager: filtered proposal-vote: sender %v had already sent a vote in round %d period %d", d.Sender, d.Round, d.Period) -} - // filterVote filters a vote, checking if it is both fresh and not a duplicate, returning // an errProposalManagerPVNotFresh or errProposalManagerPVDuplicate if so, else nil. // It also returns a bool indicating whether this proposal-vote should still be verified for tracking credential history. @@ -288,11 +266,12 @@ func (m *proposalManager) filterProposalVote(p player, r routerHandle, uv unauth if credHistory && checkDup() { credHistory = false } - return credHistory, errProposalManagerPVNotFresh{Reason: err} + return credHistory, fmt.Errorf("proposalManager: filtered proposal-vote due to age: %v", err) } if checkDup() { - return credHistory, errProposalManagerPVDuplicate{Sender: uv.R.Sender, Round: uv.R.Round, Period: uv.R.Period} + return credHistory, fmt.Errorf("proposalManager: filtered proposal-vote: sender %v had already sent a vote in round %d period %d", uv.R.Sender, uv.R.Round, uv.R.Period) + } return credHistory, nil } From 5923d763ef0f207e55186f6f6af9f0fb1031e753 Mon Sep 17 00:00:00 2001 From: chris erway Date: Wed, 11 Oct 2023 20:21:46 -0400 Subject: [PATCH 230/232] rerun "make msgp" --- agreement/msgp_gen.go | 70 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/agreement/msgp_gen.go b/agreement/msgp_gen.go index 16679ce797..a5c7b1215c 100644 --- a/agreement/msgp_gen.go +++ b/agreement/msgp_gen.go @@ -106,6 +106,16 @@ import ( // |-----> (*) MsgIsZero // |-----> CompoundMessageMaxSize() // +// constantRoundZeroTimer +// |-----> MarshalMsg +// |-----> CanMarshalMsg +// |-----> (*) UnmarshalMsg +// |-----> (*) UnmarshalMsgWithState +// |-----> (*) CanUnmarshalMsg +// |-----> Msgsize +// |-----> MsgIsZero +// |-----> ConstantRoundZeroTimerMaxSize() +// // diskState // |-----> (*) MarshalMsg // |-----> (*) CanMarshalMsg @@ -1989,6 +1999,66 @@ func CompoundMessageMaxSize() (s int) { return } +// MarshalMsg implements msgp.Marshaler +func (z constantRoundZeroTimer) MarshalMsg(b []byte) (o []byte) { + o = msgp.Require(b, z.Msgsize()) + o = msgp.AppendDuration(o, duration(z)) + return +} + +func (_ constantRoundZeroTimer) CanMarshalMsg(z interface{}) bool { + _, ok := (z).(constantRoundZeroTimer) + if !ok { + _, ok = (z).(*constantRoundZeroTimer) + } + return ok +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *constantRoundZeroTimer) UnmarshalMsgWithState(bts []byte, st msgp.UnmarshalState) (o []byte, err error) { + if st.AllowableDepth == 0 { + err = msgp.ErrMaxDepthExceeded{} + return + } + st.AllowableDepth-- + { + var zb0001 duration + zb0001, bts, err = msgp.ReadDurationBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + (*z) = constantRoundZeroTimer(zb0001) + } + o = bts + return +} + +func (z *constantRoundZeroTimer) UnmarshalMsg(bts []byte) (o []byte, err error) { + return z.UnmarshalMsgWithState(bts, msgp.DefaultUnmarshalState) +} +func (_ *constantRoundZeroTimer) CanUnmarshalMsg(z interface{}) bool { + _, ok := (z).(*constantRoundZeroTimer) + return ok +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z constantRoundZeroTimer) Msgsize() (s int) { + s = msgp.DurationSize + return +} + +// MsgIsZero returns whether this is a zero value +func (z constantRoundZeroTimer) MsgIsZero() bool { + return z == 0 +} + +// MaxSize returns a maximum valid message size for this message type +func ConstantRoundZeroTimerMaxSize() (s int) { + s = msgp.DurationSize + return +} + // MarshalMsg implements msgp.Marshaler func (z *diskState) MarshalMsg(b []byte) (o []byte) { o = msgp.Require(b, z.Msgsize()) From b62d48d38dbcd7ee79269f90fe16cb9375c87e21 Mon Sep 17 00:00:00 2001 From: chris erway Date: Thu, 12 Oct 2023 08:25:23 -0400 Subject: [PATCH 231/232] rename roundZeroTimer => roundStartTimer and //msgp:ignore constantRoundStartTimer --- agreement/events.go | 17 +++++----- agreement/msgp_gen.go | 70 ---------------------------------------- agreement/player_test.go | 32 +++++++++--------- agreement/service.go | 8 ++--- 4 files changed, 29 insertions(+), 98 deletions(-) diff --git a/agreement/events.go b/agreement/events.go index ba9ed05c2e..14aeb9facd 100644 --- a/agreement/events.go +++ b/agreement/events.go @@ -1010,15 +1010,16 @@ func (e checkpointEvent) AttachConsensusVersion(v ConsensusVersionView) external // is still waiting for quorum on R. const pipelinedMessageTimestamp = time.Nanosecond -type constantRoundZeroTimer time.Duration +//msgp:ignore constantRoundStartTimer +type constantRoundStartTimer time.Duration -func (c constantRoundZeroTimer) Since() time.Duration { return time.Duration(c) } +func (c constantRoundStartTimer) Since() time.Duration { return time.Duration(c) } // clockForRound retrieves the roundZeroTimer used for AttachValidatedAt and AttachReceivedAt. -func clockForRound(currentRound round, currentClock roundZeroTimer, historicalClocks map[round]roundZeroTimer) func(round) roundZeroTimer { - return func(eventRound round) roundZeroTimer { +func clockForRound(currentRound round, currentClock roundStartTimer, historicalClocks map[round]roundStartTimer) func(round) roundStartTimer { + return func(eventRound round) roundStartTimer { if eventRound > currentRound { - return constantRoundZeroTimer(pipelinedMessageTimestamp) + return constantRoundStartTimer(pipelinedMessageTimestamp) } if eventRound == currentRound { return currentClock @@ -1026,14 +1027,14 @@ func clockForRound(currentRound round, currentClock roundZeroTimer, historicalCl if clock, ok := historicalClocks[eventRound]; ok { return clock } - return constantRoundZeroTimer(0) + return constantRoundStartTimer(0) } } // AttachValidatedAt looks for a validated proposal or vote inside a // payloadVerified or voteVerified messageEvent, and attaches the given time to // the proposal's validatedAt field. -func (e messageEvent) AttachValidatedAt(getClock func(eventRound round) roundZeroTimer) messageEvent { +func (e messageEvent) AttachValidatedAt(getClock func(eventRound round) roundStartTimer) messageEvent { switch e.T { case payloadVerified: e.Input.Proposal.validatedAt = getClock(e.Input.Proposal.Round()).Since() @@ -1046,7 +1047,7 @@ func (e messageEvent) AttachValidatedAt(getClock func(eventRound round) roundZer // AttachReceivedAt looks for an unauthenticatedProposal inside a // payloadPresent or votePresent messageEvent, and attaches the given // time to the proposal's receivedAt field. -func (e messageEvent) AttachReceivedAt(getClock func(eventRound round) roundZeroTimer) messageEvent { +func (e messageEvent) AttachReceivedAt(getClock func(eventRound round) roundStartTimer) messageEvent { switch e.T { case payloadPresent: e.Input.UnauthenticatedProposal.receivedAt = getClock(e.Input.UnauthenticatedProposal.Round()).Since() diff --git a/agreement/msgp_gen.go b/agreement/msgp_gen.go index a5c7b1215c..16679ce797 100644 --- a/agreement/msgp_gen.go +++ b/agreement/msgp_gen.go @@ -106,16 +106,6 @@ import ( // |-----> (*) MsgIsZero // |-----> CompoundMessageMaxSize() // -// constantRoundZeroTimer -// |-----> MarshalMsg -// |-----> CanMarshalMsg -// |-----> (*) UnmarshalMsg -// |-----> (*) UnmarshalMsgWithState -// |-----> (*) CanUnmarshalMsg -// |-----> Msgsize -// |-----> MsgIsZero -// |-----> ConstantRoundZeroTimerMaxSize() -// // diskState // |-----> (*) MarshalMsg // |-----> (*) CanMarshalMsg @@ -1999,66 +1989,6 @@ func CompoundMessageMaxSize() (s int) { return } -// MarshalMsg implements msgp.Marshaler -func (z constantRoundZeroTimer) MarshalMsg(b []byte) (o []byte) { - o = msgp.Require(b, z.Msgsize()) - o = msgp.AppendDuration(o, duration(z)) - return -} - -func (_ constantRoundZeroTimer) CanMarshalMsg(z interface{}) bool { - _, ok := (z).(constantRoundZeroTimer) - if !ok { - _, ok = (z).(*constantRoundZeroTimer) - } - return ok -} - -// UnmarshalMsg implements msgp.Unmarshaler -func (z *constantRoundZeroTimer) UnmarshalMsgWithState(bts []byte, st msgp.UnmarshalState) (o []byte, err error) { - if st.AllowableDepth == 0 { - err = msgp.ErrMaxDepthExceeded{} - return - } - st.AllowableDepth-- - { - var zb0001 duration - zb0001, bts, err = msgp.ReadDurationBytes(bts) - if err != nil { - err = msgp.WrapError(err) - return - } - (*z) = constantRoundZeroTimer(zb0001) - } - o = bts - return -} - -func (z *constantRoundZeroTimer) UnmarshalMsg(bts []byte) (o []byte, err error) { - return z.UnmarshalMsgWithState(bts, msgp.DefaultUnmarshalState) -} -func (_ *constantRoundZeroTimer) CanUnmarshalMsg(z interface{}) bool { - _, ok := (z).(*constantRoundZeroTimer) - return ok -} - -// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message -func (z constantRoundZeroTimer) Msgsize() (s int) { - s = msgp.DurationSize - return -} - -// MsgIsZero returns whether this is a zero value -func (z constantRoundZeroTimer) MsgIsZero() bool { - return z == 0 -} - -// MaxSize returns a maximum valid message size for this message type -func ConstantRoundZeroTimerMaxSize() (s int) { - s = msgp.DurationSize - return -} - // MarshalMsg implements msgp.Marshaler func (z *diskState) MarshalMsg(b []byte) (o []byte) { o = msgp.Require(b, z.Msgsize()) diff --git a/agreement/player_test.go b/agreement/player_test.go index 4f25d3f14d..2bb2ffe818 100644 --- a/agreement/player_test.go +++ b/agreement/player_test.go @@ -3325,10 +3325,10 @@ func TestPlayerRetainsEarlyReceivedValidatedAtOneSample(t *testing.T) { assertSingleCredentialArrival(t, pWhite, pipelinedMessageTimestamp) } -func testClockForRound(t *testing.T, pWhite *player, fixedDur time.Duration, currentRound round, historicalClocks map[round]roundZeroTimer) func(round) roundZeroTimer { - return func(eventRound round) roundZeroTimer { +func testClockForRound(t *testing.T, pWhite *player, fixedDur time.Duration, currentRound round, historicalClocks map[round]roundStartTimer) func(round) roundStartTimer { + return func(eventRound round) roundStartTimer { //require.Equal(t, pWhite.Round, currentRound) // TODO make tests more realistic - return clockForRound(currentRound, constantRoundZeroTimer(fixedDur), historicalClocks)(eventRound) + return clockForRound(currentRound, constantRoundStartTimer(fixedDur), historicalClocks)(eventRound) } } @@ -3344,8 +3344,8 @@ func TestPlayerRetainsLateReceivedValidatedAtOneSample(t *testing.T) { const p = period(0) pWhite, pM, helper := setupP(t, r-1, p, soft) - historicalClocks := map[round]roundZeroTimer{ - r - credentialRoundLag - 1: constantRoundZeroTimer(900 * time.Millisecond), + historicalClocks := map[round]roundStartTimer{ + r - credentialRoundLag - 1: constantRoundStartTimer(900 * time.Millisecond), } // send voteVerified message @@ -3470,8 +3470,8 @@ func TestPlayerRetainsReceivedValidatedAtPPOneSample(t *testing.T) { // XXX this behavior only happens if dynamic timeout enabled; test the other way - historicalClocks := map[round]roundZeroTimer{ - r - credentialRoundLag: constantRoundZeroTimer(900 * time.Millisecond), + historicalClocks := map[round]roundStartTimer{ + r - credentialRoundLag: constantRoundStartTimer(900 * time.Millisecond), } // create a PP message for the round we're going to take the sample from when round r-1 ends pP, pV = helper.MakeRandomProposalPayload(t, r-credentialRoundLag) @@ -3579,8 +3579,8 @@ func TestPlayerRetainsLateReceivedValidatedAtPPOneSample(t *testing.T) { require.False(t, pWhite.lowestCredentialArrivals.isFull()) require.Equal(t, pWhite.lowestCredentialArrivals.writePtr, 0) - historicalClocks := map[round]roundZeroTimer{ - r - credentialRoundLag: constantRoundZeroTimer(900 * time.Millisecond), + historicalClocks := map[round]roundStartTimer{ + r - credentialRoundLag: constantRoundStartTimer(900 * time.Millisecond), } // create a PP message for the round we're going to take the sample from when round r-1 ends // Now we're going to pretend we got the message credentialRoundLag too late. @@ -3797,8 +3797,8 @@ func TestPlayerRetainsLateReceivedValidatedAtAVPPOneSample(t *testing.T) { verifyEvent := ev(verifyVoteAction(messageEvent{Input: message{UnauthenticatedVote: vVote.u()}}, r-credentialRoundLag, p, 1)) require.Truef(t, pM.getTrace().Contains(verifyEvent), "Player should verify vote") - historicalClocks := map[round]roundZeroTimer{ - r - credentialRoundLag: constantRoundZeroTimer(900 * time.Millisecond), + historicalClocks := map[round]roundStartTimer{ + r - credentialRoundLag: constantRoundStartTimer(900 * time.Millisecond), } // send voteVerified, pretend we're credentialRoundLag after the message was sent sendVoteVerifiedForVote(t, vVote, pWhite, pM, r, 502*time.Millisecond, historicalClocks, 1) @@ -3861,13 +3861,13 @@ func TestPlayerRetainsReceivedValidatedAtAVPPHistoryWindow(t *testing.T) { // Helper function to send voteVerified message func sendVoteVerified(t *testing.T, helper *voteMakerHelper, pWhite *player, pM ioAutomata, addrIndex int, curRound round, voteRound round, votePeriod period, pV *proposalValue, validatedAt time.Duration, - historicalClocks map[round]roundZeroTimer) { + historicalClocks map[round]roundStartTimer) { vVote := helper.MakeVerifiedVote(t, addrIndex, voteRound, votePeriod, propose, *pV) sendVoteVerifiedForVote(t, vVote, pWhite, pM, curRound, validatedAt, historicalClocks, 0) } func sendVoteVerifiedForVote(t *testing.T, vVote vote, pWhite *player, pM ioAutomata, - curRound round, validatedAt time.Duration, historicalClocks map[round]roundZeroTimer, taskIndex uint64) { + curRound round, validatedAt time.Duration, historicalClocks map[round]roundStartTimer, taskIndex uint64) { inMsg := messageEvent{T: voteVerified, Input: message{Vote: vVote, UnauthenticatedVote: vVote.u()}, TaskIndex: taskIndex} inMsg = inMsg.AttachValidatedAt(testClockForRound(t, pWhite, validatedAt, curRound, historicalClocks)) err, panicErr := pM.transition(inMsg) @@ -3886,7 +3886,7 @@ func sendVotePresent(t *testing.T, helper *voteMakerHelper, pWhite *player, pM i } // Helper function to send payloadPresent message -func sendPayloadPresent(t *testing.T, pWhite *player, pM ioAutomata, curRound round, pP *proposal, receivedAt time.Duration, historicalClocks map[round]roundZeroTimer) { +func sendPayloadPresent(t *testing.T, pWhite *player, pM ioAutomata, curRound round, pP *proposal, receivedAt time.Duration, historicalClocks map[round]roundStartTimer) { m := message{UnauthenticatedProposal: pP.u()} inMsg := messageEvent{T: payloadPresent, Input: m} inMsg = inMsg.AttachReceivedAt(testClockForRound(t, pWhite, receivedAt, curRound, historicalClocks)) @@ -3896,13 +3896,13 @@ func sendPayloadPresent(t *testing.T, pWhite *player, pM ioAutomata, curRound ro } // Helper function to send a compound PP message (votePresent + payloadPresent) -func sendCompoundMessage(t *testing.T, helper *voteMakerHelper, pWhite *player, pM ioAutomata, curRound round, voteRound round, votePeriod period, pP *proposal, pV *proposalValue, receivedAt time.Duration, historicalClocks map[round]roundZeroTimer, version protocol.ConsensusVersion) vote { +func sendCompoundMessage(t *testing.T, helper *voteMakerHelper, pWhite *player, pM ioAutomata, curRound round, voteRound round, votePeriod period, pP *proposal, pV *proposalValue, receivedAt time.Duration, historicalClocks map[round]roundStartTimer, version protocol.ConsensusVersion) vote { vVote := helper.MakeVerifiedVote(t, 0, voteRound, votePeriod, propose, *pV) sendCompoundMessageForVote(t, vVote, pWhite, pM, curRound, pP, receivedAt, historicalClocks, version) return vVote } -func sendCompoundMessageForVote(t *testing.T, vVote vote, pWhite *player, pM ioAutomata, curRound round, pP *proposal, receivedAt time.Duration, historicalClocks map[round]roundZeroTimer, version protocol.ConsensusVersion) { +func sendCompoundMessageForVote(t *testing.T, vVote vote, pWhite *player, pM ioAutomata, curRound round, pP *proposal, receivedAt time.Duration, historicalClocks map[round]roundStartTimer, version protocol.ConsensusVersion) { unverifiedVoteMsg := message{UnauthenticatedVote: vVote.u()} proposalMsg := message{UnauthenticatedProposal: pP.u()} compoundMsg := messageEvent{ diff --git a/agreement/service.go b/agreement/service.go index 1e9b9e0d79..433a17d5c6 100644 --- a/agreement/service.go +++ b/agreement/service.go @@ -58,8 +58,8 @@ type Service struct { persistStatus player persistActions []action - // Retain the round period 0 - historicalClocks map[round]roundZeroTimer + // Retain old rounds' period 0 start times. + historicalClocks map[round]roundStartTimer } // Parameters holds the parameters necessary to run the agreement protocol. @@ -91,7 +91,7 @@ type externalDemuxSignals struct { // an interface allowing for measuring the duration since a clock from a previous round, // used for measuring the arrival time of a late proposal-vote, for the dynamic filter // timeout feature -type roundZeroTimer interface { +type roundStartTimer interface { Since() time.Duration } @@ -121,7 +121,7 @@ func MakeService(p Parameters) (*Service, error) { s.persistenceLoop = makeAsyncPersistenceLoop(s.log, s.Accessor, s.Ledger) - s.historicalClocks = make(map[round]roundZeroTimer) + s.historicalClocks = make(map[round]roundStartTimer) return s, nil } From be63b9e3eb921f7b6e9ae1c54c4cc97971f3d24d Mon Sep 17 00:00:00 2001 From: chris erway Date: Thu, 12 Oct 2023 08:28:39 -0400 Subject: [PATCH 232/232] fix comment --- agreement/events.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agreement/events.go b/agreement/events.go index 14aeb9facd..afcfa3052d 100644 --- a/agreement/events.go +++ b/agreement/events.go @@ -1015,7 +1015,7 @@ type constantRoundStartTimer time.Duration func (c constantRoundStartTimer) Since() time.Duration { return time.Duration(c) } -// clockForRound retrieves the roundZeroTimer used for AttachValidatedAt and AttachReceivedAt. +// clockForRound retrieves the roundStartTimer used for AttachValidatedAt and AttachReceivedAt. func clockForRound(currentRound round, currentClock roundStartTimer, historicalClocks map[round]roundStartTimer) func(round) roundStartTimer { return func(eventRound round) roundStartTimer { if eventRound > currentRound {