allow individual calculation of validator balances across epoch bound…

…aries (#6416)

beacon_chain/networking/network_metadata.nim

@@ -343,7 +343,7 @@ proc getMetadataForNetwork*(networkName: string): Eth2NetworkMetadata =
  quit 1
 
  if networkName in ["goerli", "prater"]:
- warn "Goerli is deprecated and will stop being supported; https://blog.ethereum.org/2023/11/30/goerli-lts-update suggests migrating to Holesky or Sepolia"
+ warn "Goerli is deprecated and unsupported; https://blog.ethereum.org/2023/11/30/goerli-lts-update suggests migrating to Holesky or Sepolia"
 
  let metadata =
  when const_preset == "gnosis":

beacon_chain/spec/state_transition_epoch.nim

@@ -707,13 +707,11 @@ template get_flag_and_inactivity_delta(
  state: altair.BeaconState | bellatrix.BeaconState | capella.BeaconState |
  deneb.BeaconState | electra.BeaconState,
  base_reward_per_increment: Gwei, finality_delay: uint64,
- previous_epoch: Epoch,
- active_increments: uint64,
+ previous_epoch: Epoch, active_increments: uint64,
  penalty_denominator: uint64,
  epoch_participation: ptr EpochParticipationFlags,
- participating_increments: array[3, uint64],
- info: var altair.EpochInfo,
- vidx: ValidatorIndex
+ participating_increments: array[3, uint64], info: var altair.EpochInfo,
+ vidx: ValidatorIndex, inactivity_score: uint64
 ): (ValidatorIndex, Gwei, Gwei, Gwei, Gwei, Gwei, Gwei) =
  let
  base_reward = get_base_reward_increment(state, vidx, base_reward_per_increment)
@@ -751,7 +749,7 @@ template get_flag_and_inactivity_delta(
  0.Gwei
  else:
  let penalty_numerator =
- state.validators[vidx].effective_balance * state.inactivity_scores[vidx]
+ state.validators[vidx].effective_balance * inactivity_score
  penalty_numerator div penalty_denominator
 
  (vidx, reward(TIMELY_SOURCE_FLAG_INDEX),
@@ -804,7 +802,46 @@ iterator get_flag_and_inactivity_deltas*(
  yield get_flag_and_inactivity_delta(
  state, base_reward_per_increment, finality_delay, previous_epoch,
  active_increments, penalty_denominator, epoch_participation,
- participating_increments, info, vidx)
+ participating_increments, info, vidx, state.inactivity_scores[vidx])
+
+func get_flag_and_inactivity_delta_for_validator(
+ cfg: RuntimeConfig,
+ state: deneb.BeaconState | electra.BeaconState,
+ base_reward_per_increment: Gwei, info: var altair.EpochInfo,
+ finality_delay: uint64, vidx: ValidatorIndex, inactivity_score: Gwei):
+ Opt[(ValidatorIndex, Gwei, Gwei, Gwei, Gwei, Gwei, Gwei)] =
+ ## Return the deltas for a given ``flag_index`` by scanning through the
+ ## participation flags.
+ const INACTIVITY_PENALTY_QUOTIENT =
+ when state is altair.BeaconState:
+ INACTIVITY_PENALTY_QUOTIENT_ALTAIR
+ else:
+ INACTIVITY_PENALTY_QUOTIENT_BELLATRIX
+
+ static: doAssert ord(high(TimelyFlag)) == 2
+
+ let
+ previous_epoch = get_previous_epoch(state)
+ active_increments = get_active_increments(info)
+ penalty_denominator =
+ cfg.INACTIVITY_SCORE_BIAS * INACTIVITY_PENALTY_QUOTIENT
+ epoch_participation =
+ if previous_epoch == get_current_epoch(state):
+ unsafeAddr state.current_epoch_participation
+ else:
+ unsafeAddr state.previous_epoch_participation
+ participating_increments = [
+ get_unslashed_participating_increment(info, TIMELY_SOURCE_FLAG_INDEX),
+ get_unslashed_participating_increment(info, TIMELY_TARGET_FLAG_INDEX),
+ get_unslashed_participating_increment(info, TIMELY_HEAD_FLAG_INDEX)]
+
+ if not is_eligible_validator(info.validators[vidx]):
+ return Opt.none((ValidatorIndex, Gwei, Gwei, Gwei, Gwei, Gwei, Gwei))
+
+ Opt.some get_flag_and_inactivity_delta(
+ state, base_reward_per_increment, finality_delay, previous_epoch,
+ active_increments, penalty_denominator, epoch_participation,
+ participating_increments, info, vidx, inactivity_score.uint64)
 
 # https://github.com/ethereum/consensus-specs/blob/v1.4.0/specs/phase0/beacon-chain.md#rewards-and-penalties-1
 func process_rewards_and_penalties*(
@@ -1000,6 +1037,22 @@ func get_slashing_penalty*(validator: Validator,
 # https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.7/specs/phase0/beacon-chain.md#slashings
 # https://github.com/ethereum/consensus-specs/blob/v1.4.0/specs/altair/beacon-chain.md#slashings
 # https://github.com/ethereum/consensus-specs/blob/v1.5.0-alpha.3/specs/bellatrix/beacon-chain.md#slashings
+func get_slashing(
+ state: ForkyBeaconState, total_balance: Gwei, vidx: ValidatorIndex): Gwei =
+ # For efficiency reasons, it doesn't make sense to have process_slashings use
+ # this per-validator index version, but keep them parallel otherwise.
+ let
+ epoch = get_current_epoch(state)
+ adjusted_total_slashing_balance = get_adjusted_total_slashing_balance(
+ state, total_balance)
+
+ let validator = unsafeAddr state.validators.item(vidx)
+ if slashing_penalty_applies(validator[], epoch):
+ get_slashing_penalty(
+ validator[], adjusted_total_slashing_balance, total_balance)
+ else:
+ 0.Gwei
+
 func process_slashings*(state: var ForkyBeaconState, total_balance: Gwei) =
  let
  epoch = get_current_epoch(state)
@@ -1164,7 +1217,7 @@ template compute_inactivity_update(
  # TODO activeness already checked; remove redundant checks between
  # is_active_validator and is_unslashed_participating_index
  if is_unslashed_participating_index(
- state, TIMELY_TARGET_FLAG_INDEX, previous_epoch, index.ValidatorIndex):
+ state, TIMELY_TARGET_FLAG_INDEX, previous_epoch, index):
  inactivity_score -= min(1'u64, inactivity_score)
  else:
  inactivity_score += cfg.INACTIVITY_SCORE_BIAS
@@ -1195,6 +1248,7 @@ func process_inactivity_updates*(
 
  let
  pre_inactivity_score = state.inactivity_scores.asSeq()[index]
+ index = index.ValidatorIndex # intentional shadowing
  inactivity_score =
  compute_inactivity_update(cfg, state, info, pre_inactivity_score)
 
@@ -1507,3 +1561,108 @@ proc process_epoch*(
  process_sync_committee_updates(state)
 
  ok()
+
+proc get_validator_balance_after_epoch*(
+ cfg: RuntimeConfig,
+ state: deneb.BeaconState | electra.BeaconState,
+ flags: UpdateFlags, cache: var StateCache, info: var altair.EpochInfo,
+ index: ValidatorIndex): Gwei =
+ # Run a subset of process_epoch() which affects an individual validator,
+ # without modifying state itself
+ info.init(state) # TODO avoid quadratic aspects here
+
+ # Can't use process_justification_and_finalization(), but use its helper
+ # function. Used to calculate inactivity_score.
+ let jf_info =
+ # process_justification_and_finalization() skips first two epochs
+ if get_current_epoch(state) <= GENESIS_EPOCH + 1:
+ JustificationAndFinalizationInfo(
+ previous_justified_checkpoint: state.previous_justified_checkpoint,
+ current_justified_checkpoint: state.current_justified_checkpoint,
+ finalized_checkpoint: state.finalized_checkpoint,
+ justification_bits: state.justification_bits)
+ else:
+ weigh_justification_and_finalization(
+ state, info.balances.current_epoch,
+ info.balances.previous_epoch[TIMELY_TARGET_FLAG_INDEX],
+ info.balances.current_epoch_TIMELY_TARGET, flags)
+
+ # Used as part of process_rewards_and_penalties
+ let inactivity_score =
+ # process_inactivity_updates skips GENESIS_EPOCH and ineligible validators
+ if get_current_epoch(state) == GENESIS_EPOCH or
+ not is_eligible_validator(info.validators[index]):
+ 0.Gwei
+ else:
+ let
+ finality_delay =
+ get_previous_epoch(state) - jf_info.finalized_checkpoint.epoch
+ not_in_inactivity_leak = not is_in_inactivity_leak(finality_delay)
+ pre_inactivity_score = state.inactivity_scores.asSeq()[index]
+
+ # This is a template which uses not_in_inactivity_leak and index
+ compute_inactivity_update(cfg, state, info, pre_inactivity_score).Gwei
+
+ # process_rewards_and_penalties for a single validator
+ let reward_and_penalties_balance = block:
+ # process_rewards_and_penalties doesn't run at GENESIS_EPOCH
+ if get_current_epoch(state) == GENESIS_EPOCH:
+ state.balances.item(index)
+ else:
+ let
+ total_active_balance = info.balances.current_epoch
+ base_reward_per_increment = get_base_reward_per_increment(
+ total_active_balance)
+ finality_delay = get_finality_delay(state)
+
+ var balance = state.balances.item(index)
+ let maybeDelta = get_flag_and_inactivity_delta_for_validator(
+ cfg, state, base_reward_per_increment, info, finality_delay, index,
+ inactivity_score)
+ if maybeDelta.isOk:
+ # Can't use isErrOr in generics
+ let (validator_index, reward0, reward1, reward2, penalty0, penalty1, penalty2) =
+ maybeDelta.get
+ info.validators[validator_index].delta.rewards += reward0 + reward1 + reward2
+ info.validators[validator_index].delta.penalties += penalty0 + penalty1 + penalty2
+ increase_balance(balance, info.validators[index].delta.rewards)
+ decrease_balance(balance, info.validators[index].delta.penalties)
+ balance
+
+ # The two directly balance-changing operations, from Altair through Deneb,
+ # are these. The rest is necessary to look past a single epoch transition,
+ # but that's not the use case here.
+ var post_epoch_balance = reward_and_penalties_balance
+ decrease_balance(
+ post_epoch_balance,
+ get_slashing(state, info.balances.current_epoch, index))
+
+ # Electra adds process_pending_balance_deposit to the list of potential
+ # balance-changing epoch operations. This should probably be cached, so
+ # the 16+ invocations of this function each time, e.g., withdrawals are
+ # calculated don't repeat it, if it's empirically too expensive. Limits
+ # exist on how large this structure can get though.
+ when type(state).kind >= ConsensusFork.Electra:
+ let available_for_processing = state.deposit_balance_to_consume +
+ get_activation_exit_churn_limit(cfg, state, cache)
+ var processed_amount = 0.Gwei
+
+ for deposit in state.pending_balance_deposits:
+ let
+ validator = state.validators.item(deposit.index)
+ deposit_validator_index = ValidatorIndex.init(deposit.index).valueOr:
+ break
+
+ # Validator is exiting, postpone the deposit until after withdrawable epoch
+ if validator.exit_epoch < FAR_FUTURE_EPOCH:
+ if not(get_current_epoch(state) <= validator.withdrawable_epoch) and
+ deposit_validator_index == index:
+ increase_balance(post_epoch_balance, deposit.amount)
+ # Validator is not exiting, attempt to process deposit
+ else:
+ if not(processed_amount + deposit.amount > available_for_processing):
+ if deposit_validator_index == index:
+ increase_balance(post_epoch_balance, deposit.amount)
+ processed_amount += deposit.amount
+
+ post_epoch_balance

tests/consensus_spec/deneb/test_fixture_state_transition_epoch.nim

@@ -13,7 +13,7 @@ import
  chronicles,
  # Beacon chain internals
  ../../../beacon_chain/spec/[presets, state_transition_epoch],
- ../../../beacon_chain/spec/datatypes/[altair, deneb],
+ ../../../beacon_chain/spec/datatypes/altair,
  # Test utilities
  ../../testutil,
  ../fixtures_utils, ../os_ops,
@@ -22,6 +22,8 @@ import
 
 from std/sequtils import mapIt, toSeq
 from std/strutils import rsplit
+from ../../../beacon_chain/spec/datatypes/deneb import BeaconState
+from ../../teststateutil import checkPerValidatorBalanceCalc
 
 const
  RootDir = SszTestsDir/const_preset/"deneb"/"epoch_processing"
@@ -73,13 +75,15 @@ template runSuite(
 # ---------------------------------------------------------------
 runSuite(JustificationFinalizationDir, "Justification & Finalization"):
  let info = altair.EpochInfo.init(state)
+ check checkPerValidatorBalanceCalc(state)
  process_justification_and_finalization(state, info.balances)
  Result[void, cstring].ok()
 
 # Inactivity updates
 # ---------------------------------------------------------------
 runSuite(InactivityDir, "Inactivity"):
  let info = altair.EpochInfo.init(state)
+ check checkPerValidatorBalanceCalc(state)
  process_inactivity_updates(cfg, state, info)
  Result[void, cstring].ok()
 

tests/consensus_spec/electra/test_fixture_state_transition_epoch.nim

@@ -13,7 +13,7 @@ import
  chronicles,
  # Beacon chain internals
  ../../../beacon_chain/spec/[presets, state_transition_epoch],
- ../../../beacon_chain/spec/datatypes/[altair, electra],
+ ../../../beacon_chain/spec/datatypes/altair,
  # Test utilities
  ../../testutil,
  ../fixtures_utils, ../os_ops,
@@ -22,6 +22,8 @@ import
 
 from std/sequtils import mapIt, toSeq
 from std/strutils import rsplit
+from ../../../beacon_chain/spec/datatypes/electra import BeaconState
+from ../../teststateutil import checkPerValidatorBalanceCalc
 
 const
  RootDir = SszTestsDir/const_preset/"electra"/"epoch_processing"
@@ -76,13 +78,15 @@ template runSuite(
 # ---------------------------------------------------------------
 runSuite(JustificationFinalizationDir, "Justification & Finalization"):
  let info = altair.EpochInfo.init(state)
+ check checkPerValidatorBalanceCalc(state)
  process_justification_and_finalization(state, info.balances)
  Result[void, cstring].ok()
 
 # Inactivity updates
 # ---------------------------------------------------------------
 runSuite(InactivityDir, "Inactivity"):
  let info = altair.EpochInfo.init(state)
+ check checkPerValidatorBalanceCalc(state)
  process_inactivity_updates(cfg, state, info)
  Result[void, cstring].ok()
 

tests/consensus_spec/test_fixture_sanity_blocks.nim

@@ -11,7 +11,7 @@
 import
  chronicles,
  ../../beacon_chain/spec/forks,
- ../../beacon_chain/spec/state_transition,
+ ../../beacon_chain/spec/[state_transition, state_transition_epoch],
  ./os_ops,
  ../testutil
 
@@ -21,6 +21,7 @@ from ../../beacon_chain/spec/presets import
  const_preset, defaultRuntimeConfig
 from ./fixtures_utils import
  SSZ, SszTestsDir, hash_tree_root, parseTest, readSszBytes, toSszType
+from ../teststateutil import checkPerValidatorBalanceCalc
 
 proc runTest(
  consensusFork: static ConsensusFork,
@@ -52,6 +53,9 @@ proc runTest(
  discard state_transition(
  defaultRuntimeConfig, fhPreState[], blck, cache, info, flags = {},
  noRollback).expect("should apply block")
+ withState(fhPreState[]):
+ when consensusFork >= ConsensusFork.Deneb:
+ check checkPerValidatorBalanceCalc(forkyState.data)
  else:
  let res = state_transition(
  defaultRuntimeConfig, fhPreState[], blck, cache, info, flags = {},

tests/teststateutil.nim

@@ -14,6 +14,9 @@ import
  forks, state_transition, state_transition_block]
 
 from ".."/beacon_chain/bloomfilter import constructBloomFilter
+from ".."/beacon_chain/spec/state_transition_epoch import
+ get_validator_balance_after_epoch, process_epoch
+
 
 func round_multiple_down(x: Gwei, n: Gwei): Gwei =
  ## Round the input to the previous multiple of "n"
@@ -97,4 +100,19 @@ proc getTestStates*(
  doAssert getStateField(tmpState[], slot) == slot
 
  if tmpState[].kind == consensusFork:
- result.add assignClone(tmpState[])
+ result.add assignClone(tmpState[])
+
+proc checkPerValidatorBalanceCalc*(
+ state: deneb.BeaconState | electra.BeaconState): bool =
+ var
+ info: altair.EpochInfo
+ cache: StateCache
+ let tmpState = newClone(state) # slow, but tolerable for tests
+ discard process_epoch(defaultRuntimeConfig, tmpState[], {}, cache, info)
+ for i in 0 ..< tmpState.balances.len:
+ if tmpState.balances.item(i) != get_validator_balance_after_epoch(
+ defaultRuntimeConfig, state, default(UpdateFlags), cache, info,
+ i.ValidatorIndex):
+ return false
+
+ true