Esad/proving memory optimizations (#625)

* drop memory heavy stuff where we can * accumulate state diff map inside apply_soft_confirmations_from_sequencer_commitments * add some config files * update prover configs * update bitcoin da ids * use CumulativeStateDiff::default() instead of Default::default() Co-authored-by: Roman <[email protected]> * use itertools zip_eq * use default for itertools --------- Co-authored-by: Roman <[email protected]>
chainwayxyz · May 28, 2024 · 7323186 · 7323186
1 parent 5383987
commit 7323186
Show file tree

Hide file tree

Showing 19 changed files with 115 additions and 53 deletions.
diff --git a/.gitignore b/.gitignore
@@ -12,6 +12,7 @@ fuzz/Cargo.lock
 demo_data/
 sequencer-db/
 full-node-db/
+prover-db/
 /.vscode/*
 
 /docker/credentials/*

diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -74,7 +74,7 @@ async-trait = "0.1.71"
 anyhow = { version = "1.0.68", default-features = false }
 arbitrary = { version = "1.3.1", features = ["derive"] }
 alloy-rlp = "0.3"
-backoff = { version = "0.4.0", features = [ "futures", "tokio" ] }
+backoff = { version = "0.4.0", features = ["futures", "tokio"] }
 borsh = { version = "0.10.3", default-features = false }
 # TODO: Consider replacing this serialization format
 #     https://github.com/Sovereign-Labs/sovereign-sdk/issues/283
@@ -93,6 +93,7 @@ hex = { version = "0.4.3", default-features = false, features = [
     "alloc",
     "serde",
 ] }
+itertools = { version = "0.13.0", default-features = false }
 log-panics = { version = "2", features = ["with-backtrace"] }
 once_cell = { version = "1.19.0", default-features = false, features = [
     "alloc",

diff --git a/Makefile b/Makefile
@@ -16,6 +16,7 @@ clean: ## Cleans compiled
 clean-node: ## Cleans local dbs needed for sequencer and nodes
 	sudo rm -rf data/postgres
 	rm -rf sequencer-db
+	rm -rf prover-db
 	rm -rf full-node-db
 	rm test-da-dbs/*.db
 

diff --git a/bin/citrea/configs/bitcoin-regtest/prover_config.toml b/bin/citrea/configs/bitcoin-regtest/prover_config.toml
@@ -0,0 +1,2 @@
+proving_mode = "execute"
+proof_sampling_number = 500
diff --git a/bin/citrea/configs/bitcoin-regtest/prover_rollup_config.toml b/bin/citrea/configs/bitcoin-regtest/prover_rollup_config.toml
@@ -0,0 +1,28 @@
+[public_keys]
+sequencer_public_key = "204040e364c10f2bec9c1fe500a1cd4c247c89d650a01ed7e82caba867877c21"
+sequencer_da_pub_key = "02588d202afcc1ee4ab5254c7847ec25b9a135bbda0f2bc69ee1a714749fd77dc9"
+prover_da_pub_key = ""
+
+[da]
+# fill here
+node_url = ""
+# fill here
+node_username = ""
+# fill here                                       
+node_password = ""
+network = "regtest"
+
+[storage]
+# The path to the rollup's data directory. Paths that do not begin with `/` are interpreted as relative paths.
+path = "prover-db"
+
+[rpc]
+# the host and port to bind the rpc server for
+bind_host = "127.0.0.1"
+bind_port = 12346
+
+# We define the rollup's genesis to occur at block number `start_height`. The rollup will ignore
+# any blocks before this height
+[runner]
+sequencer_client_url = "http://0.0.0.0:12345"
+include_tx_body = false
diff --git a/bin/citrea/configs/mock/prover_config.toml b/bin/citrea/configs/mock/prover_config.toml
@@ -1,4 +1,5 @@
 proving_mode = "execute"
+proof_sampling_number = 500
 
 [db_config]
 db_host = "localhost"

diff --git a/bin/citrea/provers/risc0/guest-bitcoin/Cargo.lock b/bin/citrea/provers/risc0/guest-bitcoin/Cargo.lock
diff --git a/bin/citrea/provers/risc0/guest-mock/Cargo.lock b/bin/citrea/provers/risc0/guest-mock/Cargo.lock
diff --git a/bin/citrea/src/bitcoin_rollup.rs b/bin/citrea/src/bitcoin_rollup.rs
@@ -80,8 +80,8 @@ impl RollupBlueprint for BitcoinRollup {
     #[instrument(level = "trace", skip(self), ret)]
     fn get_code_commitment(&self) -> <Self::Vm as Zkvm>::CodeCommitment {
         Digest::from([
-            1860130309, 3212374340, 1571693462, 196802355, 3242449784, 3924610482, 1760955211,
-            1058326957,
+            3965686256, 604444860, 1238257798, 1624929768, 3378693295, 3187615180, 1121902498,
+            1170832510,
         ])
     }
 

diff --git a/crates/evm/Cargo.toml b/crates/evm/Cargo.toml
@@ -52,7 +52,7 @@ reth-rpc-types-compat = { workspace = true, optional = true }
 reth-rpc = { workspace = true, optional = true }
 revm-inspectors = { workspace = true, optional = true }
 secp256k1 = { workspace = true, optional = true }
-itertools = { version = "0.11.0", optional = true }
+itertools = { workspace = true, optional = true }
 
 [dev-dependencies]
 revm = { workspace = true, features = [

diff --git a/crates/sovereign-sdk/examples/demo-simple-stf/src/lib.rs b/crates/sovereign-sdk/examples/demo-simple-stf/src/lib.rs
@@ -5,8 +5,8 @@ use std::marker::PhantomData;
 use sha2::Digest;
 use sov_rollup_interface::da::{BlobReaderTrait, DaSpec};
 use sov_rollup_interface::soft_confirmation::SignedSoftConfirmationBatch;
-use sov_rollup_interface::stf::{BatchReceipt, SlotResult, StateDiff, StateTransitionFunction};
-use sov_rollup_interface::zk::{ValidityCondition, Zkvm};
+use sov_rollup_interface::stf::{BatchReceipt, SlotResult, StateTransitionFunction};
+use sov_rollup_interface::zk::{CumulativeStateDiff, ValidityCondition, Zkvm};
 
 /// An implementation of the [`StateTransitionFunction`]
 /// that is specifically designed to check if someone knows a preimage of a specific hash.
@@ -138,7 +138,7 @@ impl<Vm: Zkvm, Cond: ValidityCondition, Da: DaSpec> StateTransitionFunction<Vm,
         _slot_headers: std::collections::VecDeque<Vec<<Da as DaSpec>::BlockHeader>>,
         _validity_condition: &<Da as DaSpec>::ValidityCondition,
         _soft_batch: std::collections::VecDeque<Vec<SignedSoftConfirmationBatch>>,
-    ) -> (Self::StateRoot, StateDiff) {
+    ) -> (Self::StateRoot, CumulativeStateDiff) {
         todo!()
     }
 }
diff --git a/crates/sovereign-sdk/full-node/sov-stf-runner/src/mock/mod.rs b/crates/sovereign-sdk/full-node/sov-stf-runner/src/mock/mod.rs
@@ -1,9 +1,8 @@
 use std::marker::PhantomData;
 
-use sov_modules_api::StateDiff;
 use sov_rollup_interface::da::DaSpec;
 use sov_rollup_interface::stf::{BatchReceipt, SlotResult, StateTransitionFunction};
-use sov_rollup_interface::zk::{ValidityCondition, Zkvm};
+use sov_rollup_interface::zk::{CumulativeStateDiff, ValidityCondition, Zkvm};
 
 /// A mock implementation of the [`StateTransitionFunction`]
 #[derive(PartialEq, Debug, Clone, Eq, serde::Serialize, serde::Deserialize, Default)]
@@ -95,7 +94,7 @@ impl<Vm: Zkvm, Cond: ValidityCondition, Da: DaSpec> StateTransitionFunction<Vm,
         _soft_confirmations: std::collections::VecDeque<
             Vec<sov_modules_api::SignedSoftConfirmationBatch>,
         >,
-    ) -> (Self::StateRoot, StateDiff) {
+    ) -> (Self::StateRoot, CumulativeStateDiff) {
         todo!()
     }
 }
diff --git a/crates/sovereign-sdk/full-node/sov-stf-runner/src/runner.rs b/crates/sovereign-sdk/full-node/sov-stf-runner/src/runner.rs
@@ -844,9 +844,11 @@ where
                         Proof::Full(proof) => {
                             // hotfix for devnet deployment
                             // TODO: handle these deployed bug fixes better
-                            let code_commitment = if serde_json::to_string(&self.code_commitment)? == "[1860130309,3212374340,1571693462,196802355,3242449784,3924610482,1760955211,1058326957]" && soft_batch.da_slot_height <= 6787
+                            let code_commitment = if serde_json::to_string(&self.code_commitment)? == "[3965686256,604444860,1238257798,1624929768,3378693295,3187615180,1121902498,1170832510]" && soft_batch.da_slot_height <= 6787
                             {
                                 serde_json::from_str("[3565622432,3167720487,554880044,1689168947,3495902722,3638273228,1798052366,4175296332]").unwrap()
+                            } else if serde_json::to_string(&self.code_commitment)? == "[3965686256,604444860,1238257798,1624929768,3378693295,3187615180,1121902498,1170832510]" && soft_batch.da_slot_height > 6786 && soft_batch.da_slot_height <= 11075 {
+                                serde_json::from_str("[1860130309,3212374340,1571693462,196802355,3242449784,3924610482,1760955211,1058326957]").unwrap()
                             } else {
                                 self.code_commitment.clone()
                             };

diff --git a/crates/sovereign-sdk/full-node/sov-stf-runner/src/verifier.rs b/crates/sovereign-sdk/full-node/sov-stf-runner/src/verifier.rs
@@ -2,9 +2,7 @@ use std::marker::PhantomData;
 
 use sov_rollup_interface::da::{BlockHeaderTrait, DaVerifier};
 use sov_rollup_interface::stf::StateTransitionFunction;
-use sov_rollup_interface::zk::{
-    CumulativeStateDiff, StateTransition, StateTransitionData, Zkvm, ZkvmGuest,
-};
+use sov_rollup_interface::zk::{StateTransition, StateTransitionData, Zkvm, ZkvmGuest};
 
 /// Verifies a state transition
 pub struct StateTransitionVerifier<ST, Da, Zk>
@@ -82,12 +80,6 @@ where
             "Invalid final state root"
         );
 
-        // Collect state diffs into a BtreeMap
-        let state_diff: CumulativeStateDiff = state_diff
-            .into_iter()
-            // .map(|(k, v)| (k, v))
-            .collect();
-
         let out: StateTransition<Da::Spec, _> = StateTransition {
             initial_state_root: data.initial_state_root,
             final_state_root,

diff --git a/crates/sovereign-sdk/full-node/sov-stf-runner/tests/hash_stf.rs b/crates/sovereign-sdk/full-node/sov-stf-runner/tests/hash_stf.rs
@@ -3,12 +3,12 @@ use sov_mock_da::{
     MockAddress, MockBlob, MockBlock, MockBlockHeader, MockDaSpec, MockValidityCond,
 };
 use sov_mock_zkvm::MockZkvm;
-use sov_modules_api::{Context, StateDiff};
+use sov_modules_api::Context;
 use sov_modules_stf_blueprint::StfBlueprintTrait;
 use sov_prover_storage_manager::{new_orphan_storage, SnapshotManager};
 use sov_rollup_interface::da::{BlobReaderTrait, BlockHeaderTrait, DaSpec};
 use sov_rollup_interface::stf::{SlotResult, StateTransitionFunction};
-use sov_rollup_interface::zk::{ValidityCondition, Zkvm};
+use sov_rollup_interface::zk::{CumulativeStateDiff, ValidityCondition, Zkvm};
 use sov_state::storage::{NativeStorage, StorageKey, StorageValue};
 use sov_state::{
     ArrayWitness, DefaultStorageSpec, OrderedReadsAndWrites, Prefix, ProverStorage, Storage,
@@ -234,7 +234,7 @@ impl<Vm: Zkvm, Cond: ValidityCondition, Da: DaSpec> StateTransitionFunction<Vm,
         _soft_confirmations: std::collections::VecDeque<
             Vec<sov_modules_api::SignedSoftConfirmationBatch>,
         >,
-    ) -> (Self::StateRoot, StateDiff) {
+    ) -> (Self::StateRoot, CumulativeStateDiff) {
         todo!()
     }
 }

diff --git a/crates/sovereign-sdk/module-system/sov-modules-stf-blueprint/Cargo.toml b/crates/sovereign-sdk/module-system/sov-modules-stf-blueprint/Cargo.toml
@@ -19,7 +19,7 @@ serde = { workspace = true, features = ["derive"] }
 tracing = { workspace = true, optional = true }
 jmt = { workspace = true }
 hex = { workspace = true }
-
+itertools = { workspace = true }
 sov-rollup-interface = { path = "../../rollup-interface" }
 sov-state = { path = "../sov-state" }
 sov-modules-api = { path = "../sov-modules-api", default-features = false }
@@ -35,7 +35,12 @@ jsonrpsee = { workspace = true, features = ["server"], optional = true }
 [features]
 bench = ["sov-zk-cycle-macros", "risc0-zkvm", "risc0-zkvm-platform"]
 default = []
-native = ["sov-state/native", "sov-modules-api/native", "dep:tracing", "jsonrpsee"]
+native = [
+    "sov-state/native",
+    "sov-modules-api/native",
+    "dep:tracing",
+    "jsonrpsee",
+]
 
 [package.metadata.cargo-udeps.ignore]
 normal = [
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		proving_mode = "execute"
		proof_sampling_number = 500