Merge branch 'master' into feat/stateful

# Conflicts: # Cargo.lock # Cargo.toml # crates/bin/src/commands/run_file.rs # crates/bin/src/utils.rs # crates/core/src/chunk.rs # crates/core/src/database.rs # crates/core/src/error.rs # crates/core/src/executor/builder.rs # crates/core/src/executor/mod.rs # crates/primitives/Cargo.toml # crates/primitives/src/lib.rs # crates/primitives/src/types/tx.rs
scroll-tech · Oct 21, 2024 · a94e7d3 · a94e7d3
2 parents 6975caa + d2220ca
commit a94e7d3
Show file tree

Hide file tree

Showing 18 changed files with 875 additions and 419 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -83,7 +83,7 @@ jobs:
       matrix:
         network: [ "mainnet" ]
         hardfork: [ "pre-bernoulli", "bernoulli", "curie", "darwin" ]
-        rust: [ "1.75", "nightly-2024-07-07" ]
+        rust: [ "1.81", "nightly-2024-07-07" ]
 
     steps:
       - uses: actions/checkout@v4

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -7,6 +7,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+
+- Add a new struct `LegacyStorageTrace` to support legacy storage trace support
+  ([#58](https://github.com/scroll-tech/stateless-block-verifier/pull/58))
+
+### Changed
+
+- `flatten_proofs` in `StorageTrace` is changed from `Vec<(B256, Bytes)` to `Vec<Bytes>`
+  since the node hash will be recalculated when adding to zktrie
+  ([#58](https://github.com/scroll-tech/stateless-block-verifier/pull/58))
+- `BlockTrace` now has a generic parameter `S` for the storage trace type, default to `StorageTrace`
+  ([#58](https://github.com/scroll-tech/stateless-block-verifier/pull/58))
+- rpc mode defaults to use legacy storage trace, using the flag `--flatten-proofs` to enable support of flatten proofs
+  ([#58](https://github.com/scroll-tech/stateless-block-verifier/pull/58))
+
+
 ## [2.0.0] - 2024-09-04
 
 ### Added

diff --git a/Cargo.toml b/Cargo.toml
@@ -5,7 +5,7 @@ resolver = "2"
 [workspace.package]
 version = "2.0.0"
 edition = "2021"
-rust-version = "1.75"
+rust-version = "1.81"
 authors = ["Scroll developers"]
 license = "MIT OR Apache-2.0"
 homepage = "https://github.com/scroll-tech/stateless-block-verifier"
@@ -15,14 +15,13 @@ repository = "https://github.com/scroll-tech/stateless-block-verifier"
 alloy = "0.3"
 hex = "0.4"
 once_cell = "1.19"
-rkyv = { version = "0.7", features = ["validation"] }
+rkyv = "0.8"
 thiserror = "1.0"
 tiny-keccak = "2.0"
 
 # dependencies from scroll-tech
 poseidon-bn254 = { git = "https://github.com/scroll-tech/poseidon-bn254", branch = "master", features = ["bn254"] }
-#zktrie-ng = { git = "https://github.com/scroll-tech/zktrie-ng", branch = "master", features = ["scroll"] }
-zktrie-ng = { path = "../zktrie-ng", features = ["scroll"] }
+zktrie-ng = { git = "https://github.com/scroll-tech/zktrie-ng", branch = "master", features = ["scroll"] }
 
 # binary dependencies
 anyhow = "1.0"
@@ -81,13 +80,13 @@ features = [
 ]
 
 [workspace.lints.rust]
-#missing-docs = "deny"
+missing-docs = "deny"
 missing-debug-implementations = "deny"
 
 [patch.crates-io]
 ff = { git = "https://github.com/scroll-tech/ff", branch = "feat/sp1" }
 
-# patched add rkyv support & MSRV 1.75
+# patched add rkyv support & MSRV 1.77
 ruint = { git = "https://github.com/scroll-tech/uint.git", branch = "v1.12.3" }
 
 alloy = { git = "https://github.com/scroll-tech/alloy.git", branch = "v0.3.0" }
@@ -96,8 +95,9 @@ alloy-eips = { git = "https://github.com/scroll-tech/alloy.git", branch = "v0.3.
 alloy-eip2930 = { git = "https://github.com/scroll-tech/alloy-eips", branch = "v0.1.0" }
 alloy-eip7702 = { git = "https://github.com/scroll-tech/alloy-eips", branch = "v0.1.0" }
 
-alloy-core = { git = "https://github.com/scroll-tech/alloy-core", branch = "v0.8.0"}
-alloy-primitives = { git = "https://github.com/scroll-tech/alloy-core", branch = "v0.8.0"}
+alloy-core = { git = "https://github.com/scroll-tech/alloy-core", branch = "v0.8.0" }
+alloy-primitives = { git = "https://github.com/scroll-tech/alloy-core", branch = "v0.8.0" }
+alloy-sol-types = {git = "https://github.com/scroll-tech/alloy-core", branch = "v0.8.0" }
 
 # for local development
 # [patch."https://github.com/scroll-tech/revm"]

diff --git a/crates/bin/src/commands/run_file.rs b/crates/bin/src/commands/run_file.rs
@@ -1,11 +1,12 @@
 use crate::utils;
-use anyhow::bail;
+use anyhow::{anyhow, bail};
 use clap::Args;
+use sbv::primitives::types::LegacyStorageTrace;
 use sbv::{
     core::{ChunkInfo, EvmExecutorBuilder, HardforkConfig},
-    primitives::{types::BlockTrace, zk_trie::db::HashMapDb, Block, B256},
+    primitives::{types::BlockTrace, zk_trie::db::kv::HashMapDb, Block, B256},
 };
-use std::rc::Rc;
+use std::panic::catch_unwind;
 use std::{cell::RefCell, path::PathBuf};
 use tiny_keccak::{Hasher, Keccak};
 use tokio::task::JoinSet;
@@ -44,7 +45,7 @@ impl RunFileCommand {
 
         while let Some(task) = tasks.join_next().await {
             if let Err(err) = task? {
-                bail!("{:?}", err);
+                dev_error!("{:?}", err);
             }
         }
 
@@ -75,12 +76,11 @@ impl RunFileCommand {
         }
 
         let fork_config = fork_config(traces[0].chain_id());
-        let (chunk_info, zktrie_db) = ChunkInfo::from_block_traces(&traces);
-        let zktrie_db = Rc::new(RefCell::new(zktrie_db));
+        let (chunk_info, mut zktrie_db) = ChunkInfo::from_block_traces(&traces);
 
         let tx_bytes_hasher = RefCell::new(Keccak::v256());
 
-        let mut executor = EvmExecutorBuilder::new(HashMapDb::default(), zktrie_db.clone())
+        let mut executor = EvmExecutorBuilder::new(HashMapDb::default(), &mut zktrie_db)
             .hardfork_config(fork_config)
             .with_hooks(&traces[0], |hooks| {
                 hooks.add_tx_rlp_handler(|_, rlp| {
@@ -94,7 +94,7 @@ impl RunFileCommand {
             executor.handle_block(trace)?;
         }
 
-        let post_state_root = executor.commit_changes(zktrie_db.clone())?;
+        let post_state_root = executor.commit_changes()?;
         if post_state_root != chunk_info.post_state_root() {
             bail!("post state root mismatch");
         }
@@ -116,19 +116,21 @@ async fn read_block_trace(path: &PathBuf) -> anyhow::Result<BlockTrace> {
 }
 
 fn deserialize_block_trace(trace: &str) -> anyhow::Result<BlockTrace> {
+    // Try to deserialize `BlockTrace` from JSON. In case of failure, try to
+    // deserialize `BlockTrace` from a JSON-RPC response that has the actual block
+    // trace nested in the value of the key "result".
     Ok(
-        // Try to deserialize `BlockTrace` from JSON. In case of failure, try to
-        // deserialize `BlockTrace` from a JSON-RPC response that has the actual block
-        // trace nested in the value of the key "result".
-        serde_json::from_str::<BlockTrace>(trace).or_else(|_| {
-            #[derive(serde::Deserialize, Default, Debug, Clone)]
-            pub struct BlockTraceJsonRpcResult {
-                pub result: BlockTrace,
-            }
-            Ok::<_, serde_json::Error>(
-                serde_json::from_str::<BlockTraceJsonRpcResult>(trace)?.result,
-            )
-        })?,
+        serde_json::from_str::<BlockTrace<LegacyStorageTrace>>(trace)
+            .or_else(|_| {
+                #[derive(serde::Deserialize, Default, Debug, Clone)]
+                pub struct BlockTraceJsonRpcResult {
+                    pub result: BlockTrace<LegacyStorageTrace>,
+                }
+                Ok::<_, serde_json::Error>(
+                    serde_json::from_str::<BlockTraceJsonRpcResult>(trace)?.result,
+                )
+            })?
+            .into(),
     )
 }
 
@@ -138,6 +140,26 @@ async fn run_trace(
 ) -> anyhow::Result<()> {
     let trace = read_block_trace(&path).await?;
     let fork_config = fork_config(trace.chain_id());
-    tokio::task::spawn_blocking(move || utils::verify(&trace, &fork_config)).await??;
+    if let Err(e) =
+        tokio::task::spawn_blocking(move || catch_unwind(|| utils::verify(&trace, &fork_config)))
+            .await?
+            .map_err(|e| {
+                e.downcast_ref::<&str>()
+                    .map(|s| anyhow!("task panics with: {s}"))
+                    .or_else(|| {
+                        e.downcast_ref::<String>()
+                            .map(|s| anyhow!("task panics with: {s}"))
+                    })
+                    .unwrap_or_else(|| anyhow!("task panics"))
+            })
+            .and_then(|r| r.map_err(anyhow::Error::from))
+    {
+        dev_error!(
+            "Error occurs when verifying block ({}): {:?}",
+            path.display(),
+            e
+        );
+        return Err(e);
+    }
     Ok(())
 }
diff --git a/crates/bin/src/commands/run_rpc.rs b/crates/bin/src/commands/run_rpc.rs
@@ -2,6 +2,7 @@ use crate::utils;
 use alloy::providers::{Provider, ProviderBuilder};
 use clap::Args;
 use futures::future::OptionFuture;
+use sbv::primitives::types::LegacyStorageTrace;
 use sbv::{
     core::HardforkConfig,
     primitives::{types::BlockTrace, Block},
@@ -19,6 +20,9 @@ pub struct RunRpcCommand {
     /// RPC URL
     #[arg(short, long, default_value = "http://localhost:8545")]
     url: Url,
+    /// Enable flatten proofs
+    #[arg(short, long)]
+    flatten_proofs: bool,
     /// Start Block number
     #[arg(short, long, default_value = "latest")]
     start_block: StartBlockSpec,
@@ -49,7 +53,11 @@ pub enum StartBlockSpec {
 
 impl RunRpcCommand {
     pub async fn run(self, fork_config: impl Fn(u64) -> HardforkConfig) -> anyhow::Result<()> {
-        dev_info!("Running RPC command with url: {}", self.url);
+        dev_info!(
+            "Running RPC command with url: {}, flatten proofs support: {}",
+            self.url,
+            self.flatten_proofs
+        );
         let provider = ProviderBuilder::new().on_http(self.url);
 
         let chain_id = provider.get_chain_id().await?;
@@ -75,21 +83,32 @@ impl RunRpcCommand {
             let rx = rx.clone();
             handles.spawn(async move {
                 while let Ok(block_number) = rx.recv().await {
-                    let l2_trace = _provider
-                        .raw_request::<_, BlockTrace>(
-                            "scroll_getBlockTraceByNumberOrHash".into(),
-                            (
-                                format!("0x{:x}", block_number),
-                                serde_json::json!({
-                                    "ExcludeExecutionResults": true,
-                                    "ExcludeTxStorageTraces": true,
-                                    "StorageProofFormat": "flatten",
-                                    "FlattenProofsOnly": true
-                                }),
-                            ),
-                        )
-                        .await
-                        .map_err(|e| (block_number, e.into()))?;
+                    let l2_trace: BlockTrace = if !self.flatten_proofs {
+                        let trace = _provider
+                            .raw_request::<_, BlockTrace<LegacyStorageTrace>>(
+                                "scroll_getBlockTraceByNumberOrHash".into(),
+                                (format!("0x{:x}", block_number),),
+                            )
+                            .await
+                            .map_err(|e| (block_number, e.into()))?;
+                        trace.into()
+                    } else {
+                        _provider
+                            .raw_request::<_, BlockTrace>(
+                                "scroll_getBlockTraceByNumberOrHash".into(),
+                                (
+                                    format!("0x{:x}", block_number),
+                                    serde_json::json!({
+                                        "ExcludeExecutionResults": true,
+                                        "ExcludeTxStorageTraces": true,
+                                        "StorageProofFormat": "flatten",
+                                        "FlattenProofsOnly": true
+                                    }),
+                                ),
+                            )
+                            .await
+                            .map_err(|e| (block_number, e.into()))?
+                    };
 
                     dev_info!(
                         "worker#{_idx}: load trace for block #{block_number}({})",

diff --git a/crates/bin/src/utils.rs b/crates/bin/src/utils.rs
@@ -1,9 +1,8 @@
+use sbv::primitives::zk_trie::db::NodeDb;
 use sbv::{
     core::{EvmExecutorBuilder, HardforkConfig, VerificationError},
-    primitives::{zk_trie::db::HashMapDb, Block},
+    primitives::{zk_trie::db::kv::HashMapDb, Block},
 };
-use std::cell::RefCell;
-use std::rc::Rc;
 
 pub fn verify<T: Block + Clone>(
     l2_trace: T,
@@ -37,19 +36,19 @@ fn verify_inner<T: Block + Clone>(
         .build()
         .unwrap();
 
-    let zktrie_db = cycle_track!(
+    let mut zktrie_db = cycle_track!(
         {
-            let mut zktrie_db = HashMapDb::default();
+            let mut zktrie_db = NodeDb::new(HashMapDb::default());
             measure_duration_millis!(
                 build_zktrie_db_duration_milliseconds,
                 l2_trace.build_zktrie_db(&mut zktrie_db).unwrap()
             );
-            Rc::new(RefCell::new(zktrie_db))
+            zktrie_db
         },
         "build ZktrieState"
     );
 
-    let mut executor = EvmExecutorBuilder::new(HashMapDb::default(), zktrie_db.clone())
+    let mut executor = EvmExecutorBuilder::new(HashMapDb::default(), &mut zktrie_db)
         .hardfork_config(*fork_config)
         .build(&l2_trace)?;
 
@@ -66,7 +65,7 @@ fn verify_inner<T: Block + Clone>(
         update_metrics_counter!(verification_error);
         e
     })?;
-    let revm_root_after = executor.commit_changes(zktrie_db.clone())?;
+    let revm_root_after = executor.commit_changes()?;
 
     #[cfg(feature = "profiling")]
     if let Ok(report) = guard.report().build() {

diff --git a/crates/core/src/chunk.rs b/crates/core/src/chunk.rs
@@ -1,5 +1,6 @@
 use revm::primitives::B256;
-use sbv_primitives::{zk_trie::db::HashMapDb, Block};
+use sbv_primitives::zk_trie::db::NodeDb;
+use sbv_primitives::{zk_trie::db::kv::HashMapDb, Block};
 use tiny_keccak::{Hasher, Keccak};
 
 /// A chunk is a set of continuous blocks.
@@ -21,7 +22,7 @@ pub struct ChunkInfo {
 
 impl ChunkInfo {
     /// Construct by block traces
-    pub fn from_block_traces<T: Block>(traces: &[T]) -> (Self, HashMapDb) {
+    pub fn from_block_traces<T: Block>(traces: &[T]) -> (Self, NodeDb<HashMapDb>) {
         let chain_id = traces.first().unwrap().chain_id();
         let prev_state_root = traces
             .first()
@@ -30,6 +31,7 @@ impl ChunkInfo {
         let post_state_root = traces.last().expect("at least 1 block needed").root_after();
         let withdraw_root = traces.last().unwrap().withdraw_root();
 
+        cycle_tracker_start!("Keccak::v256");
         let mut data_hasher = Keccak::v256();
         for trace in traces.iter() {
             trace.hash_da_header(&mut data_hasher);
@@ -39,14 +41,17 @@ impl ChunkInfo {
         }
         let mut data_hash = B256::ZERO;
         data_hasher.finalize(&mut data_hash.0);
+        cycle_tracker_end!("Keccak::v256");
 
-        let mut zktrie_db = HashMapDb::default();
+        let mut zktrie_db = NodeDb::new(HashMapDb::default());
+        cycle_tracker_start!("Block::build_zktrie_db");
         for trace in traces.iter() {
             measure_duration_millis!(
                 build_zktrie_db_duration_milliseconds,
                 trace.build_zktrie_db(&mut zktrie_db).unwrap()
             );
         }
+        cycle_tracker_end!("Block::build_zktrie_db");
 
         let info = ChunkInfo {
             chain_id,
@@ -116,7 +121,6 @@ mod tests {
     use revm::primitives::b256;
     use sbv_primitives::types::BlockTrace;
     use std::cell::RefCell;
-    use std::rc::Rc;
 
     const TRACES_STR: [&str; 4] = [
         include_str!("../../../testdata/mainnet_blocks/8370400.json"),
@@ -138,12 +142,11 @@ mod tests {
         });
 
         let fork_config = HardforkConfig::default_from_chain_id(traces[0].chain_id());
-        let (chunk_info, zktrie_db) = ChunkInfo::from_block_traces(&traces);
-        let zktrie_db = Rc::new(RefCell::new(zktrie_db));
+        let (chunk_info, mut zktrie_db) = ChunkInfo::from_block_traces(&traces);
 
         let tx_bytes_hasher = RefCell::new(Keccak::v256());
 
-        let mut executor = EvmExecutorBuilder::new(HashMapDb::default(), zktrie_db.clone())
+        let mut executor = EvmExecutorBuilder::new(HashMapDb::default(), &mut zktrie_db)
             .hardfork_config(fork_config)
             .with_hooks(&traces[0], |hooks| {
                 hooks.add_tx_rlp_handler(|_, rlp| {
@@ -158,7 +161,7 @@ mod tests {
             executor.handle_block(trace).unwrap();
         }
 
-        let post_state_root = executor.commit_changes(zktrie_db.clone()).unwrap();
+        let post_state_root = executor.commit_changes().unwrap();
         assert_eq!(post_state_root, chunk_info.post_state_root);
         drop(executor); // drop executor to release Rc<Keccek>