Merge branch 'develop' into multiclientDenebE2E

beacon-chain/blockchain/chain_info.go

@@ -556,3 +556,18 @@ func (s *Service) RecentBlockSlot(root [32]byte) (primitives.Slot, error) {
 	defer s.cfg.ForkChoiceStore.RUnlock()
 	return s.cfg.ForkChoiceStore.Slot(root)
 }
+
+// inRegularSync applies the following heuristics to decide if the node is in
+// regular sync mode vs init sync mode using only forkchoice.
+// It checks that the highest received block is behind the current time by at least 2 epochs
+// and that it was imported at least one epoch late if both of these
+// tests pass then the node is in init sync. The caller of this function MUST
+// have a lock on forkchoice
+func (s *Service) inRegularSync() bool {
+	currentSlot := s.CurrentSlot()
+	fc := s.cfg.ForkChoiceStore
+	if currentSlot-fc.HighestReceivedBlockSlot() < 2*params.BeaconConfig().SlotsPerEpoch {
+		return true
+	}
+	return fc.HighestReceivedBlockDelay() < params.BeaconConfig().SlotsPerEpoch
+}

beacon-chain/blockchain/chain_info_test.go

@@ -593,3 +593,26 @@ func TestService_IsFinalized(t *testing.T) {
 	require.Equal(t, true, c.IsFinalized(ctx, br))
 	require.Equal(t, false, c.IsFinalized(ctx, [32]byte{'c'}))
 }
+
+func TestService_inRegularSync(t *testing.T) {
+	ctx := context.Background()
+	c := &Service{cfg: &config{ForkChoiceStore: doublylinkedtree.New()}, head: &head{root: [32]byte{'b'}}}
+	ojc := &ethpb.Checkpoint{Root: params.BeaconConfig().ZeroHash[:]}
+	ofc := &ethpb.Checkpoint{Root: params.BeaconConfig().ZeroHash[:]}
+	st, blkRoot, err := prepareForkchoiceState(ctx, 100, [32]byte{'a'}, [32]byte{}, params.BeaconConfig().ZeroHash, ojc, ofc)
+	require.NoError(t, err)
+	require.NoError(t, c.cfg.ForkChoiceStore.InsertNode(ctx, st, blkRoot))
+	require.Equal(t, false, c.inRegularSync())
+	c.SetGenesisTime(time.Now().Add(time.Second * time.Duration(-1*int64(params.BeaconConfig().SlotsPerEpoch)*int64(params.BeaconConfig().SecondsPerSlot))))
+	st, blkRoot, err = prepareForkchoiceState(ctx, 128, [32]byte{'b'}, [32]byte{'a'}, params.BeaconConfig().ZeroHash, ojc, ofc)
+	require.NoError(t, err)
+	require.NoError(t, c.cfg.ForkChoiceStore.InsertNode(ctx, st, blkRoot))
+	require.Equal(t, false, c.inRegularSync())
+
+	c.SetGenesisTime(time.Now().Add(time.Second * time.Duration(-5*int64(params.BeaconConfig().SlotsPerEpoch)*int64(params.BeaconConfig().SecondsPerSlot))))
+	require.Equal(t, true, c.inRegularSync())
+
+	c.SetGenesisTime(time.Now().Add(time.Second * time.Duration(-1*int64(params.BeaconConfig().SlotsPerEpoch)*int64(params.BeaconConfig().SecondsPerSlot))))
+	c.cfg.ForkChoiceStore.SetGenesisTime(uint64(time.Now().Unix()))
+	require.Equal(t, true, c.inRegularSync())
+}

beacon-chain/blockchain/process_block.go

@@ -67,7 +67,9 @@ func (s *Service) postBlockProcess(cfg *postBlockProcessConfig) error {
 	startTime := time.Now()
 	fcuArgs := &fcuConfig{}
 
-	defer s.handleSecondFCUCall(cfg, fcuArgs)
+	if s.inRegularSync() {
+		defer s.handleSecondFCUCall(cfg, fcuArgs)
+	}
 	defer s.sendLightClientFeeds(cfg)
 	defer s.sendStateFeedOnBlock(cfg)
 	defer reportProcessingTime(startTime)
@@ -580,10 +582,15 @@ func (s *Service) lateBlockTasks(ctx context.Context) {
 	if s.CurrentSlot() == s.HeadSlot() {
 		return
 	}
+	s.cfg.ForkChoiceStore.RLock()
+	defer s.cfg.ForkChoiceStore.RUnlock()
+	// return early if we are in init sync
+	if !s.inRegularSync() {
+		return
+	}
 	s.cfg.StateNotifier.StateFeed().Send(&feed.Event{
 		Type: statefeed.MissedSlot,
 	})
-
 	s.headLock.RLock()
 	headRoot := s.headRoot()
 	headState := s.headState(ctx)
@@ -598,18 +605,22 @@ func (s *Service) lateBlockTasks(ctx context.Context) {
 	if err := transition.UpdateNextSlotCache(ctx, lastRoot, lastState); err != nil {
 		log.WithError(err).Debug("could not update next slot state cache")
 	}
-	// handleEpochBoundary requires a forkchoice lock to obtain the target root.
-	s.cfg.ForkChoiceStore.RLock()
 	if err := s.handleEpochBoundary(ctx, currentSlot, headState, headRoot[:]); err != nil {
 		log.WithError(err).Error("lateBlockTasks: could not update epoch boundary caches")
 	}
-	s.cfg.ForkChoiceStore.RUnlock()
 	// return early if we already started building a block for the current
 	// head root
 	_, has := s.cfg.PayloadIDCache.PayloadID(s.CurrentSlot()+1, headRoot)
 	if has {
 		return
 	}
+
+	attribute := s.getPayloadAttribute(ctx, headState, s.CurrentSlot()+1, headRoot[:])
+	// return early if we are not proposing next slot
+	if attribute.IsEmpty() {
+		return
+	}
+
 	s.headLock.RLock()
 	headBlock, err := s.headBlock()
 	if err != nil {
@@ -620,18 +631,12 @@ func (s *Service) lateBlockTasks(ctx context.Context) {
 	s.headLock.RUnlock()
 
 	fcuArgs := &fcuConfig{
-		headState: headState,
-		headRoot:  headRoot,
-		headBlock: headBlock,
+		headState:  headState,
+		headRoot:   headRoot,
+		headBlock:  headBlock,
+		attributes: attribute,
 	}
-	fcuArgs.attributes = s.getPayloadAttribute(ctx, headState, s.CurrentSlot()+1, headRoot[:])
-	// return early if we are not proposing next slot
-	if fcuArgs.attributes.IsEmpty() {
-		return
-	}
-	s.cfg.ForkChoiceStore.RLock()
 	_, err = s.notifyForkchoiceUpdate(ctx, fcuArgs)
-	s.cfg.ForkChoiceStore.RUnlock()
 	if err != nil {
 		log.WithError(err).Debug("could not perform late block tasks: failed to update forkchoice with engine")
 	}

beacon-chain/blockchain/process_block_helpers.go

@@ -37,6 +37,9 @@ func (s *Service) getFCUArgs(cfg *postBlockProcessConfig, fcuArgs *fcuConfig) er
 	if err := s.getFCUArgsEarlyBlock(cfg, fcuArgs); err != nil {
 		return err
 	}
+	if !s.inRegularSync() {
+		return nil
+	}
 	slot := cfg.signed.Block().Slot()
 	if slots.WithinVotingWindow(uint64(s.genesisTime.Unix()), slot) {
 		return nil

beacon-chain/blockchain/receive_attestation.go

@@ -150,7 +150,9 @@ func (s *Service) UpdateHead(ctx context.Context, proposingSlot primitives.Slot)
 		headBlock:     headBlock,
 		proposingSlot: proposingSlot,
 	}
-	fcuArgs.attributes = s.getPayloadAttribute(ctx, headState, proposingSlot, newHeadRoot[:])
+	if s.inRegularSync() {
+		fcuArgs.attributes = s.getPayloadAttribute(ctx, headState, proposingSlot, newHeadRoot[:])
+	}
 	if fcuArgs.attributes != nil && s.shouldOverrideFCU(newHeadRoot, proposingSlot) {
 		return
 	}

beacon-chain/core/deneb/upgrade.go

@@ -57,6 +57,10 @@ func UpgradeToDeneb(state state.BeaconState) (state.BeaconState, error) {
 	if err != nil {
 		return nil, err
 	}
+	historicalRoots, err := state.HistoricalRoots()
+	if err != nil {
+		return nil, err
+	}
 
 	s := &ethpb.BeaconStateDeneb{
 		GenesisTime:           state.GenesisTime(),
@@ -70,7 +74,7 @@ func UpgradeToDeneb(state state.BeaconState) (state.BeaconState, error) {
 		LatestBlockHeader:           state.LatestBlockHeader(),
 		BlockRoots:                  state.BlockRoots(),
 		StateRoots:                  state.StateRoots(),
-		HistoricalRoots:             [][]byte{},
+		HistoricalRoots:             historicalRoots,
 		Eth1Data:                    state.Eth1Data(),
 		Eth1DataVotes:               state.Eth1DataVotes(),
 		Eth1DepositIndex:            state.Eth1DepositIndex(),

beacon-chain/core/deneb/upgrade_test.go

@@ -14,6 +14,7 @@ import (
 
 func TestUpgradeToDeneb(t *testing.T) {
 	st, _ := util.DeterministicGenesisStateCapella(t, params.BeaconConfig().MaxValidatorsPerCommittee)
+	require.NoError(t, st.SetHistoricalRoots([][]byte{{1}}))
 	preForkState := st.Copy()
 	mSt, err := deneb.UpgradeToDeneb(st)
 	require.NoError(t, err)
@@ -46,6 +47,12 @@ func TestUpgradeToDeneb(t *testing.T) {
 	require.NoError(t, err)
 	require.DeepSSZEqual(t, make([]uint64, numValidators), s)
 
+	hr1, err := preForkState.HistoricalRoots()
+	require.NoError(t, err)
+	hr2, err := mSt.HistoricalRoots()
+	require.NoError(t, err)
+	require.DeepEqual(t, hr1, hr2)
+
 	f := mSt.Fork()
 	require.DeepSSZEqual(t, &ethpb.Fork{
 		PreviousVersion: st.Fork().CurrentVersion,

beacon-chain/db/filesystem/blob.go

@@ -156,7 +156,7 @@ func (bs *BlobStorage) Save(sidecar blocks.VerifiedROBlob) error {
 	}
 	partialMoved = true
 	blobsWrittenCounter.Inc()
-	blobSaveLatency.Observe(time.Since(startTime).Seconds())
+	blobSaveLatency.Observe(float64(time.Since(startTime).Milliseconds()))
 	return nil
 }
 
@@ -180,7 +180,7 @@ func (bs *BlobStorage) Get(root [32]byte, idx uint64) (blocks.VerifiedROBlob, er
 		return blocks.VerifiedROBlob{}, err
 	}
 	defer func() {
-		blobFetchLatency.Observe(time.Since(startTime).Seconds())
+		blobFetchLatency.Observe(float64(time.Since(startTime).Milliseconds()))
 	}()
 	return verification.BlobSidecarNoop(ro)
 }

beacon-chain/db/filesystem/metrics.go

@@ -6,15 +6,15 @@ import (
 )
 
 var (
-	blobBuckets     = []float64{0.00003, 0.00005, 0.00007, 0.00009, 0.00011, 0.00013, 0.00015}
+	blobBuckets     = []float64{3, 5, 7, 9, 11, 13}
 	blobSaveLatency = promauto.NewHistogram(prometheus.HistogramOpts{
 		Name:    "blob_storage_save_latency",
-		Help:    "Latency of BlobSidecar storage save operations in seconds",
+		Help:    "Latency of BlobSidecar storage save operations in milliseconds",
 		Buckets: blobBuckets,
 	})
 	blobFetchLatency = promauto.NewHistogram(prometheus.HistogramOpts{
 		Name:    "blob_storage_get_latency",
-		Help:    "Latency of BlobSidecar storage get operations in seconds",
+		Help:    "Latency of BlobSidecar storage get operations in milliseconds",
 		Buckets: blobBuckets,
 	})
 	blobsPrunedCounter = promauto.NewCounter(prometheus.CounterOpts{

beacon-chain/forkchoice/doubly-linked-tree/store.go

@@ -240,6 +240,20 @@ func (f *ForkChoice) HighestReceivedBlockSlot() primitives.Slot {
 	return f.store.highestReceivedNode.slot
 }
 
+// HighestReceivedBlockSlotDelay returns the number of slots that the highest
+// received block was late when receiving it
+func (f *ForkChoice) HighestReceivedBlockDelay() primitives.Slot {
+	n := f.store.highestReceivedNode
+	if n == nil {
+		return 0
+	}
+	secs, err := slots.SecondsSinceSlotStart(n.slot, f.store.genesisTime, n.timestamp)
+	if err != nil {
+		return 0
+	}
+	return primitives.Slot(secs / params.BeaconConfig().SecondsPerSlot)
+}
+
 // ReceivedBlocksLastEpoch returns the number of blocks received in the last epoch
 func (f *ForkChoice) ReceivedBlocksLastEpoch() (uint64, error) {
 	count := uint64(0)

beacon-chain/forkchoice/doubly-linked-tree/store_test.go

@@ -333,26 +333,29 @@ func TestForkChoice_ReceivedBlocksLastEpoch(t *testing.T) {
 	require.NoError(t, err)
 	require.Equal(t, uint64(1), count)
 	require.Equal(t, primitives.Slot(1), f.HighestReceivedBlockSlot())
+	require.Equal(t, primitives.Slot(0), f.HighestReceivedBlockDelay())
 
 	// 64
 	// Received block last epoch is 1
 	_, err = s.insert(context.Background(), 64, [32]byte{'A'}, b, b, 1, 1)
 	require.NoError(t, err)
-	s.genesisTime = uint64(time.Now().Add(time.Duration(-64*int64(params.BeaconConfig().SecondsPerSlot)) * time.Second).Unix())
+	s.genesisTime = uint64(time.Now().Add(time.Duration((-64*int64(params.BeaconConfig().SecondsPerSlot))-1) * time.Second).Unix())
 	count, err = f.ReceivedBlocksLastEpoch()
 	require.NoError(t, err)
 	require.Equal(t, uint64(1), count)
 	require.Equal(t, primitives.Slot(64), f.HighestReceivedBlockSlot())
+	require.Equal(t, primitives.Slot(0), f.HighestReceivedBlockDelay())
 
 	// 64 65
 	// Received block last epoch is 2
 	_, err = s.insert(context.Background(), 65, [32]byte{'B'}, b, b, 1, 1)
 	require.NoError(t, err)
-	s.genesisTime = uint64(time.Now().Add(time.Duration(-65*int64(params.BeaconConfig().SecondsPerSlot)) * time.Second).Unix())
+	s.genesisTime = uint64(time.Now().Add(time.Duration(-66*int64(params.BeaconConfig().SecondsPerSlot)) * time.Second).Unix())
 	count, err = f.ReceivedBlocksLastEpoch()
 	require.NoError(t, err)
 	require.Equal(t, uint64(2), count)
 	require.Equal(t, primitives.Slot(65), f.HighestReceivedBlockSlot())
+	require.Equal(t, primitives.Slot(1), f.HighestReceivedBlockDelay())
 
 	// 64 65 66
 	// Received block last epoch is 3

beacon-chain/forkchoice/interfaces.go

@@ -64,6 +64,7 @@ type FastGetter interface {
 	FinalizedPayloadBlockHash() [32]byte
 	HasNode([32]byte) bool
 	HighestReceivedBlockSlot() primitives.Slot
+	HighestReceivedBlockDelay() primitives.Slot
 	IsCanonical(root [32]byte) bool
 	IsOptimistic(root [32]byte) (bool, error)
 	IsViableForCheckpoint(*forkchoicetypes.Checkpoint) (bool, error)

beacon-chain/forkchoice/ro.go

@@ -114,6 +114,13 @@ func (ro *ROForkChoice) HighestReceivedBlockSlot() primitives.Slot {
 	return ro.getter.HighestReceivedBlockSlot()
 }
 
+// HighestReceivedBlockDelay delegates to the underlying forkchoice call, under a lock.
+func (ro *ROForkChoice) HighestReceivedBlockDelay() primitives.Slot {
+	ro.l.RLock()
+	defer ro.l.RUnlock()
+	return ro.getter.HighestReceivedBlockDelay()
+}
+
 // ReceivedBlocksLastEpoch delegates to the underlying forkchoice call, under a lock.
 func (ro *ROForkChoice) ReceivedBlocksLastEpoch() (uint64, error) {
 	ro.l.RLock()

beacon-chain/forkchoice/ro_test.go

@@ -29,6 +29,7 @@ const (
 	unrealizedJustifiedPayloadBlockHashCalled
 	nodeCountCalled
 	highestReceivedBlockSlotCalled
+	highestReceivedBlockDelayCalled
 	receivedBlocksLastEpochCalled
 	weightCalled
 	isOptimisticCalled
@@ -113,6 +114,11 @@ func TestROLocking(t *testing.T) {
 			call: highestReceivedBlockSlotCalled,
 			cb:   func(g FastGetter) { g.HighestReceivedBlockSlot() },
 		},
+		{
+			name: "highestReceivedBlockDelayCalled",
+			call: highestReceivedBlockDelayCalled,
+			cb:   func(g FastGetter) { g.HighestReceivedBlockDelay() },
+		},
 		{
 			name: "receivedBlocksLastEpochCalled",
 			call: receivedBlocksLastEpochCalled,
@@ -245,6 +251,11 @@ func (ro *mockROForkchoice) HighestReceivedBlockSlot() primitives.Slot {
 	return 0
 }
 
+func (ro *mockROForkchoice) HighestReceivedBlockDelay() primitives.Slot {
+	ro.calls = append(ro.calls, highestReceivedBlockDelayCalled)
+	return 0
+}
+
 func (ro *mockROForkchoice) ReceivedBlocksLastEpoch() (uint64, error) {
 	ro.calls = append(ro.calls, receivedBlocksLastEpochCalled)
 	return 0, nil

beacon-chain/operations/attestations/prepare_forkchoice.go

@@ -42,7 +42,7 @@ func (s *Service) prepareForkChoiceAtts() {
 			switch slotInterval.Interval {
 			case 0:
 				duration := time.Since(t)
-				log.WithField("Duration", duration).Debug("aggregated unaggregated attestations")
+				log.WithField("Duration", duration).Debug("Aggregated unaggregated attestations")
 				batchForkChoiceAttsT1.Observe(float64(duration.Milliseconds()))
 			case 1:
 				batchForkChoiceAttsT2.Observe(float64(time.Since(t).Milliseconds()))

validator/accounts/iface/wallet.go

@@ -23,7 +23,7 @@ type Wallet interface {
 	// Read methods for important wallet and accounts-related files.
 	ReadFileAtPath(ctx context.Context, filePath string, fileName string) ([]byte, error)
 	// Write methods to persist important wallet and accounts-related files to disk.
-	WriteFileAtPath(ctx context.Context, pathName string, fileName string, data []byte) error
+	WriteFileAtPath(ctx context.Context, pathName string, fileName string, data []byte) (bool, error)
 	// Method for initializing a new keymanager.
 	InitializeKeymanager(ctx context.Context, cfg InitKeymanagerConfig) (keymanager.IKeymanager, error)
 }

validator/accounts/testing/mock.go

@@ -55,19 +55,19 @@ func (w *Wallet) Password() string {
 }
 
 // WriteFileAtPath --
-func (w *Wallet) WriteFileAtPath(_ context.Context, pathName, fileName string, data []byte) error {
+func (w *Wallet) WriteFileAtPath(_ context.Context, pathName, fileName string, data []byte) (bool, error) {
 	w.lock.Lock()
 	defer w.lock.Unlock()
 	if w.HasWriteFileError {
 		// reset the flag to not contaminate other tests
 		w.HasWriteFileError = false
-		return errors.New("could not write keystore file for accounts")
+		return false, errors.New("could not write keystore file for accounts")
 	}
 	if w.Files[pathName] == nil {
 		w.Files[pathName] = make(map[string][]byte)
 	}
 	w.Files[pathName][fileName] = data
-	return nil
+	return true, nil
 }
 
 // ReadFileAtPath --