Cleanup stateless and block witness code. (#2295)

* Cleanup unneeded stateless and block witness code. Keeping MultiKeys which is used in the eth_getProofsByBlockNumber RPC endpoint which is needed for the Fluffy state network bridge. * Rename generateWitness flag to collectWitnessData to better describe what the flag does. We only collect the keys of the touched accounts and storage slots but no block witness generation is supported for now. * Move remaining stateless code into nimbus directory. * Add vmstate parameter to ChainRef to fix test. * Exclude *.in from check copyright year --------- Co-authored-by: jangko <[email protected]>
status-im · Jun 8, 2024 · db8c5b9 · db8c5b9
1 parent 0524fe8
commit db8c5b9
Show file tree

Hide file tree

Showing 68 changed files with 182 additions and 6,321 deletions.
diff --git a/nimbus/config.nim b/nimbus/config.nim
@@ -197,12 +197,6 @@ type
       defaultValueDesc: ""
       name: "verify-from" }: Option[uint64]
 
-    generateWitness* {.
-      hidden
-      desc: "Enable experimental generation and storage of block witnesses"
-      defaultValue: false
-      name: "generate-witness" }: bool
-
     evm* {.
       desc: "Load alternative EVM from EVMC-compatible shared library" & sharedLibText
       defaultValue: ""

diff --git a/nimbus/core/chain/chain_desc.nim b/nimbus/core/chain/chain_desc.nim
@@ -33,27 +33,29 @@ type
       ## Trigger extra validation, currently within `persistBlocks()`
       ## function only.
 
-    generateWitness: bool ##\
-      ## Enable generation of block witness, currently within `persistBlocks()`
-      ## function only.
-
     verifyFrom: BlockNumber ##\
       ## First block to when `extraValidation` will be applied (only
       ## effective if `extraValidation` is true.)
 
+    vmState: BaseVMState
+      ## If it's not nil, block validation will use this
+      ## If it's nil, a new vmState state will be created.
+
 # ------------------------------------------------------------------------------
 # Public constructors
 # ------------------------------------------------------------------------------
 
 proc newChain*(com: CommonRef,
-               extraValidation: bool): ChainRef =
+               extraValidation: bool,
+               vmState = BaseVMState(nil)): ChainRef =
   ## Constructor for the `Chain` descriptor object.
   ## The argument `extraValidation` enables extra block
   ## chain validation if set `true`.
   ChainRef(
     com: com,
     validateBlock: true,
     extraValidation: extraValidation,
+    vmState: vmState
   )
 
 func newChain*(com: CommonRef): ChainRef =
@@ -93,10 +95,6 @@ proc extraValidation*(c: ChainRef): bool =
   ## Getter
   c.extraValidation
 
-proc generateWitness*(c: ChainRef): bool =
-  ## Getter
-  c.generateWitness
-
 proc verifyFrom*(c: ChainRef): BlockNumber =
   ## Getter
   c.verifyFrom
@@ -121,11 +119,6 @@ proc `extraValidation=`*(c: ChainRef; extraValidation: bool) =
   ## extra block chain validation.
   c.extraValidation = extraValidation
 
-proc `generateWitness=`*(c: ChainRef; generateWitness: bool) =
-  ## Setter. If set `true`, the assignment value `generateWitness` enables
-  ## block witness generation.
-  c.generateWitness = generateWitness
-
 proc `verifyFrom=`*(c: ChainRef; verifyFrom: BlockNumber) =
   ## Setter. The  assignment value `verifyFrom` defines the first block where
   ## validation should start if the `Clique` field `extraValidation` was set

diff --git a/nimbus/core/chain/persist_blocks.nim b/nimbus/core/chain/persist_blocks.nim
@@ -111,11 +111,8 @@ proc persistBlocksImpl(c: ChainRef; headers: openArray[BlockHeader];
         body,
         checkSealOK = false) # TODO: how to checkseal from here
 
-    if c.generateWitness:
-      vmState.generateWitness = true
-
     let
-      validationResult = if c.validateBlock or c.generateWitness:
+      validationResult = if c.validateBlock:
                            vmState.processBlock(header, body)
                          else:
                            ValidationResult.OK
@@ -129,21 +126,6 @@ proc persistBlocksImpl(c: ChainRef; headers: openArray[BlockHeader];
     if validationResult != ValidationResult.OK:
       return err("Failed to validate block")
 
-    if c.generateWitness:
-      let dbTx = c.db.newTransaction()
-      defer: dbTx.dispose()
-
-      let
-        mkeys = vmState.stateDB.makeMultiKeys()
-        # Reset state to what it was before executing the block of transactions
-        initialState = BaseVMState.new(header, c.com).valueOr:
-                         return err("Failed to create vm state")
-        witness = initialState.buildWitness(mkeys)
-
-      dbTx.rollback()
-
-      c.db.setBlockWitness(header.blockHash(), witness)
-
     if NoPersistHeader notin flags:
       discard c.db.persistHeaderToDb(
         header, c.com.consensus == ConsensusType.POS, c.com.startOfHistory)

diff --git a/nimbus/core/executor/process_block.nim b/nimbus/core/executor/process_block.nim
@@ -114,7 +114,7 @@ proc procBlkEpilogue(vmState: BaseVMState;
     {.gcsafe, raises: [].} =
   # Reward beneficiary
   vmState.mutateStateDB:
-    if vmState.generateWitness:
+    if vmState.collectWitnessData:
       db.collectWitnessData()
 
     db.persist(clearEmptyAccount = vmState.determineFork >= FkSpurious)

diff --git a/nimbus/core/executor/process_transaction.nim b/nimbus/core/executor/process_transaction.nim
@@ -115,7 +115,7 @@ proc processTransactionImpl(
   else:
     res = err(txRes.error)
 
-  if vmState.generateWitness:
+  if vmState.collectWitnessData:
     vmState.stateDB.collectWitnessData()
   vmState.stateDB.persist(clearEmptyAccount = fork >= FkSpurious)
 

diff --git a/nimbus/core/tx_pool/tx_tasks/tx_packer.nim b/nimbus/core/tx_pool/tx_tasks/tx_packer.nim
@@ -108,7 +108,7 @@ proc runTxCommit(pst: TxPackerStateRef; item: TxItemRef; gasBurned: GasInt)
   vmState.stateDB.addBalance(xp.chain.feeRecipient, reward)
   xp.blockValue += reward
 
-  if vmState.generateWitness:
+  if vmState.collectWitnessData:
     vmState.stateDB.collectWitnessData()
 
   # Save accounts via persist() is not needed unless the fork is smaller
@@ -245,7 +245,7 @@ proc vmExecCommit(pst: TxPackerStateRef)
 
   # Reward beneficiary
   vmState.mutateStateDB:
-    if vmState.generateWitness:
+    if vmState.collectWitnessData:
       db.collectWitnessData()
     # Finish up, then vmState.stateDB.rootHash may be accessed
     db.persist(clearEmptyAccount = xp.chain.nextFork >= FkSpurious)

diff --git a/nimbus/db/core_db/core_apps_newapi.nim b/nimbus/db/core_db/core_apps_newapi.nim
@@ -984,20 +984,6 @@ proc haveBlockAndState*(db: CoreDbRef, headerHash: Hash256): bool =
   # see if stateRoot exists
   db.exists(header.stateRoot)
 
-proc getBlockWitness*(
-    db: CoreDbRef, blockHash: Hash256): Result[seq[byte], string] {.gcsafe.} =
-  let res = db.newKvt().get(blockHashToBlockWitnessKey(blockHash).toOpenArray)
-  if res.isErr():
-    err("Failed to get block witness from database: " & $res.error.error)
-  else:
-    ok(res.value())
-
-proc setBlockWitness*(db: CoreDbRef, blockHash: Hash256, witness: seq[byte]) =
-  let witnessKey = blockHashToBlockWitnessKey(blockHash)
-  db.newKvt.put(witnessKey.toOpenArray, witness).isOkOr:
-    warn logTxt "setBlockWitness()", witnessKey, action="put()", error=($$error)
-    return
-
 # ------------------------------------------------------------------------------
 # End
 # ------------------------------------------------------------------------------
diff --git a/nimbus/db/ledger/accounts_ledger.nim b/nimbus/db/ledger/accounts_ledger.nim
@@ -29,7 +29,7 @@ import
   chronicles,
   eth/[common, rlp],
   results,
-  ../../../stateless/multi_keys,
+  ../../stateless/multi_keys,
   "../.."/[constants, utils/utils],
   ../access_list as ac_access_list,
   ".."/[core_db, storage_types, transient_storage],

diff --git a/nimbus/db/ledger/base.nim b/nimbus/db/ledger/base.nim
@@ -14,7 +14,7 @@
 
 import
   eth/common,
-  ../../../stateless/multi_keys,
+  ../../stateless/multi_keys,
   ../core_db,
   ./base/[api_tracking, base_desc]
 

diff --git a/nimbus/evm/state.nim b/nimbus/evm/state.nim
@@ -13,7 +13,6 @@
 import
   std/[options, sets, strformat],
   eth/keys,
-  ../../stateless/[witness_from_tree, witness_types, multi_keys],
   ../db/ledger,
   ../common/[common, evmforks],
   ./interpreter/[op_codes, gas_costs],
@@ -259,31 +258,12 @@ proc `status=`*(vmState: BaseVMState, status: bool) =
  if status: vmState.flags.incl ExecutionOK
  else: vmState.flags.excl ExecutionOK
 
-proc generateWitness*(vmState: BaseVMState): bool =
-  GenerateWitness in vmState.flags
+proc collectWitnessData*(vmState: BaseVMState): bool =
+  CollectWitnessData in vmState.flags
 
-proc `generateWitness=`*(vmState: BaseVMState, status: bool) =
-  if status: vmState.flags.incl GenerateWitness
-  else: vmState.flags.excl GenerateWitness
-
-proc buildWitness*(
-    vmState: BaseVMState,
-    mkeys: MultiKeysRef): seq[byte] {.raises: [CatchableError].} =
-  let rootHash = vmState.stateDB.rootHash
-  let flags = if vmState.fork >= FkSpurious: {wfEIP170} else: {}
-
-  # A valid block having no transactions should return an empty witness
-  if mkeys.keys.len() == 0:
-    return @[]
-
-  # build witness from tree
-  var wb = initWitnessBuilder(vmState.com.db, rootHash, flags)
-  wb.buildWitness(mkeys)
-
-proc buildWitness*(
-    vmState: BaseVMState): seq[byte] {.raises: [CatchableError].} =
-  let mkeys = vmState.stateDB.makeMultiKeys()
-  buildWitness(vmState, mkeys)
+proc `collectWitnessData=`*(vmState: BaseVMState, status: bool) =
+  if status: vmState.flags.incl CollectWitnessData
+  else: vmState.flags.excl CollectWitnessData
 
 func forkDeterminationInfoForVMState*(vmState: BaseVMState): ForkDeterminationInfo =
   # FIXME-Adam: Is this timestamp right? Note that up above in blockNumber we add 1;

diff --git a/nimbus/evm/types.nim b/nimbus/evm/types.nim
@@ -36,7 +36,7 @@ const vm_use_recursion* = defined(evmc_enabled)
 type
   VMFlag* = enum
     ExecutionOK
-    GenerateWitness
+    CollectWitnessData
 
   BlockContext* = object
     timestamp*        : EthTime

diff --git a/nimbus/nimbus.nim b/nimbus/nimbus.nim
@@ -54,7 +54,6 @@ proc basicServices(nimbus: NimbusNode,
     nimbus.chainRef.extraValidation = 0 < verifyFrom
     nimbus.chainRef.verifyFrom = verifyFrom
 
-  nimbus.chainRef.generateWitness = conf.generateWitness
   nimbus.beaconEngine = BeaconEngineRef.new(nimbus.txPool, nimbus.chainRef)
 
 proc manageAccounts(nimbus: NimbusNode, conf: NimbusConf) =

diff --git a/nimbus/rpc/experimental.nim b/nimbus/rpc/experimental.nim
@@ -22,17 +22,17 @@ import
   ./filters,
   ../core/executor/process_block,
   ../db/ledger,
-  ../../stateless/[witness_verification, multi_keys],
+  ../stateless/multi_keys,
   ./p2p
 
 type
   BlockHeader = eth_types.BlockHeader
   ReadOnlyStateDB = state_db.ReadOnlyStateDB
 
-proc getBlockWitness*(
+proc getMultiKeys*(
     com: CommonRef,
     blockHeader: BlockHeader,
-    statePostExecution: bool): (MultiKeysRef, BlockWitness)
+    statePostExecution: bool): MultiKeysRef
     {.raises: [RlpError, BlockNotFound, ValueError, CatchableError].} =
 
   let
@@ -45,7 +45,7 @@ proc getBlockWitness*(
     vmState = BaseVMState.new(blockHeader, com).valueOr:
                 raise newException(ValueError, "Cannot create vm state")
 
-  vmState.generateWitness = true # Enable saving witness data
+  vmState.collectWitnessData = true # Enable saving witness data
   vmState.com.hardForkTransition(blockHeader)
 
   let dbTx = vmState.com.db.newTransaction()
@@ -57,16 +57,10 @@ proc getBlockWitness*(
 
   let mkeys = vmState.stateDB.makeMultiKeys()
 
-  if statePostExecution:
-    result = (mkeys, vmState.buildWitness(mkeys))
-  else:
-    # Use the initial state from prior to executing the block of transactions
-    let initialState = BaseVMState.new(blockHeader, com).valueOr:
-                         raise newException(ValueError, "Cannot create vm state")
-    result = (mkeys, initialState.buildWitness(mkeys))
-
   dbTx.rollback()
 
+  mkeys
+
 proc getBlockProofs*(
     accDB: ReadOnlyStateDB,
     mkeys: MultiKeysRef): seq[ProofResponse] {.raises: [RlpError].} =
@@ -95,19 +89,6 @@ proc setupExpRpc*(com: CommonRef, server: RpcServer) =
     let ac = newAccountStateDB(chainDB, header.stateRoot)
     result = ReadOnlyStateDB(ac)
 
-  server.rpc("exp_getWitnessByBlockNumber") do(quantityTag: BlockTag, statePostExecution: bool) -> seq[byte]:
-    ## Returns the block witness for a block by block number or tag.
-    ##
-    ## quantityTag: integer of a block number, or the string "earliest", "latest" or "pending", as in the default block parameter.
-    ## statePostExecution: bool which indicates whether to return the witness based on the state before or after executing the block.
-    ## Returns seq[byte]
-
-    let
-      blockHeader = chainDB.headerFromTag(quantityTag)
-      (_, witness) = getBlockWitness(com, blockHeader, statePostExecution)
-
-    return witness
-
   server.rpc("exp_getProofsByBlockNumber") do(quantityTag: BlockTag, statePostExecution: bool) -> seq[ProofResponse]:
     ## Returns the block proofs for a block by block number or tag.
     ##
@@ -117,7 +98,7 @@ proc setupExpRpc*(com: CommonRef, server: RpcServer) =
 
     let
       blockHeader = chainDB.headerFromTag(quantityTag)
-      (mkeys, _) = getBlockWitness(com, blockHeader, statePostExecution)
+      mkeys = getMultiKeys(com, blockHeader, statePostExecution)
 
     let accDB = if statePostExecution:
       getStateDB(blockHeader)

diff --git a/stateless/multi_keys.nim → nimbus/stateless/multi_keys.nim b/stateless/multi_keys.nim → nimbus/stateless/multi_keys.nim
@@ -9,11 +9,11 @@
 # according to those terms.
 
 import
-  eth/common, eth/trie/nibbles, algorithm,
-  ./witness_types
+  eth/common, eth/trie/nibbles, algorithm
 
 type
   KeyHash* = array[32, byte]
+  StorageSlot* = array[32, byte]
 
   KeyData* = object
     visited*: bool
@@ -47,6 +47,10 @@ type
     match*: bool
     group*: Group
 
+proc setBranchMaskBit(x: var uint, i: int) =
+  assert(i >= 0 and i < 17)
+  x = x or (1 shl i).uint
+
 func cmpHash(a, b: KeyHash): int =
   var i = 0
   var m = min(a.len, b.len)

diff --git a/nimbus/vm_state.nim b/nimbus/vm_state.nim
@@ -17,14 +17,13 @@ export
 export
   vms.`$`,
   vms.blockNumber,
-  vms.buildWitness,
   vms.coinbase,
   vms.determineFork,
   vms.difficultyOrPrevRandao,
   vms.baseFee,
   vms.forkDeterminationInfoForVMState,
-  vms.generateWitness,
-  vms.`generateWitness=`,
+  vms.collectWitnessData,
+  vms.`collectWitnessData=`,
   vms.getAncestorHash,
   vms.getAndClearLogEntries,
   vms.init,

diff --git a/scripts/check_copyright_year.sh b/scripts/check_copyright_year.sh
@@ -8,7 +8,7 @@
 # according to those terms.
 
 excluded_files="config.yaml|.gitmodules|.gitignore"
-excluded_extensions="json|md|png|txt|toml|gz|key|rlp|era1|cfg|py|sh"
+excluded_extensions="json|md|png|txt|toml|gz|key|rlp|era1|cfg|py|sh|in"
 
 current_year=$(date +"%Y")
 outdated_files=()