From 97b6e5d96f557ce288f078b08c101cfe62c28a3d Mon Sep 17 00:00:00 2001 From: Rakan Al-Huneiti Date: Mon, 30 Sep 2024 14:31:13 +0300 Subject: [PATCH 01/28] Structured concurrency: Part 1 (#1214) * Implement managed tasks for SubscriptionManager * Fix clippy * Add comment about trace task * Use broadcast instead of mpsc for l2 blocks events --- Cargo.lock | 1 + crates/ethereum-rpc/Cargo.toml | 1 + crates/ethereum-rpc/src/lib.rs | 12 +- crates/ethereum-rpc/src/subscription.rs | 198 ++++++++++++++---------- crates/ethereum-rpc/src/trace.rs | 4 + 5 files changed, 123 insertions(+), 93 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b3c2e75a6..96819b0b8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2754,6 +2754,7 @@ dependencies = [ "borsh", "citrea-evm", "citrea-primitives", + "futures", "jsonrpsee", "parking_lot", "proptest", diff --git a/crates/ethereum-rpc/Cargo.toml b/crates/ethereum-rpc/Cargo.toml index 0c8dba08a..97539e828 100644 --- a/crates/ethereum-rpc/Cargo.toml +++ b/crates/ethereum-rpc/Cargo.toml @@ -17,6 +17,7 @@ anyhow = { workspace = true } borsh = { workspace = true } citrea-evm = { path = "../evm", features = ["native"] } citrea-primitives = { path = "../primitives" } +futures = { workspace = true } jsonrpsee = { workspace = true, features = ["http-client", "server"] } parking_lot = { workspace = true } rustc_version_runtime = { workspace = true } diff --git a/crates/ethereum-rpc/src/lib.rs b/crates/ethereum-rpc/src/lib.rs index 002126820..cf19183cf 100644 --- a/crates/ethereum-rpc/src/lib.rs +++ b/crates/ethereum-rpc/src/lib.rs @@ -24,7 +24,6 @@ use sov_modules_api::da::BlockHeaderTrait; use sov_modules_api::utils::to_jsonrpsee_error_object; use sov_modules_api::WorkingSet; use sov_rollup_interface::services::da::DaService; -use subscription::{handle_logs_subscription, handle_new_heads_subscription}; use tokio::join; use tokio::sync::broadcast; use trace::{debug_trace_by_block_number, handle_debug_trace_chain}; @@ -693,12 +692,12 @@ fn register_rpc_methods( match topic.as_str() { "newHeads" => { let subscription = pending.accept().await.unwrap(); - let rx = ethereum + ethereum .subscription_manager .as_ref() .unwrap() - .subscribe_new_heads(); - handle_new_heads_subscription(subscription, rx).await + .register_new_heads_subscription(subscription) + .await; } "logs" => { let filter: Filter = match params.next() { @@ -709,13 +708,12 @@ fn register_rpc_methods( } }; let subscription = pending.accept().await.unwrap(); - let rx = ethereum + ethereum .subscription_manager .as_ref() .unwrap() - .subscribe_logs() + .register_new_logs_subscription(filter, subscription) .await; - handle_logs_subscription(subscription, rx, filter).await } _ => { pending diff --git a/crates/ethereum-rpc/src/subscription.rs b/crates/ethereum-rpc/src/subscription.rs index bd414db17..0e3be324d 100644 --- a/crates/ethereum-rpc/src/subscription.rs +++ b/crates/ethereum-rpc/src/subscription.rs @@ -1,12 +1,19 @@ +use std::sync::Arc; + use citrea_evm::{log_matches_filter, Evm, Filter, LogResponse}; +use futures::future; use jsonrpsee::{SubscriptionMessage, SubscriptionSink}; use reth_rpc_types::{BlockNumberOrTag, RichBlock}; use sov_modules_api::WorkingSet; -use tokio::sync::broadcast; +use tokio::sync::{broadcast, mpsc, RwLock}; +use tokio::task::JoinHandle; pub(crate) struct SubscriptionManager { - new_heads_tx: broadcast::Sender, - logs_tx: broadcast::Sender>, + soft_confirmation_handle: JoinHandle<()>, + logs_notifier_handle: JoinHandle<()>, + heads_notifier_handle: JoinHandle<()>, + head_subscriptions: Arc>>, + logs_subscriptions: Arc>>, } impl SubscriptionManager { @@ -14,108 +21,96 @@ impl SubscriptionManager { storage: C::Storage, soft_confirmation_rx: broadcast::Receiver, ) -> Self { - let new_heads_tx = broadcast::channel(16).0; - let logs_tx = broadcast::channel(16).0; - let manager = Self { - new_heads_tx: new_heads_tx.clone(), - logs_tx: logs_tx.clone(), - }; - - let mut soft_confirmation_rx = soft_confirmation_rx; + let (new_heads_tx, new_heads_rx) = mpsc::channel(16); + let (logs_tx, logs_rx) = mpsc::channel(16); + + let head_subscriptions = Arc::new(RwLock::new(vec![])); + let logs_subscriptions = Arc::new(RwLock::new(vec![])); + + let soft_confirmation_rx = soft_confirmation_rx; // Spawn the task that will listen for new soft confirmation heights // and send the corresponding ethereum block to subscribers - tokio::spawn(async move { - let evm = Evm::::default(); - loop { - let Ok(height) = soft_confirmation_rx.recv().await else { - return; - }; - - if new_heads_tx.receiver_count() != 0 { - let mut working_set = WorkingSet::::new(storage.clone()); - let block = evm - .get_block_by_number( - Some(BlockNumberOrTag::Number(height)), - None, - &mut working_set, - ) - .expect("Error querying block from evm") - .expect("Received signal but evm block is not found"); - - // Only possible error is no receiver - let _ = new_heads_tx.send(block.clone()); - } + let soft_confirmation_handle = tokio::spawn(soft_confirmation_event_handler::( + storage, + soft_confirmation_rx, + new_heads_tx.clone(), + logs_tx.clone(), + )); - if logs_tx.receiver_count() != 0 { - let mut working_set = WorkingSet::::new(storage.clone()); - let logs = evm - .get_logs_in_block_range( - &mut working_set, - &Filter::default(), - height, - height, - ) - .expect("Error getting logs in block range"); - - // Only possible error is no receiver - let _ = logs_tx.send(logs); - } - } - }); + let logs_notifier_handle = tokio::spawn(logs_notifier(logs_rx, logs_subscriptions.clone())); + let heads_notifier_handle = + tokio::spawn(new_heads_notifier(new_heads_rx, head_subscriptions.clone())); - manager + Self { + soft_confirmation_handle, + logs_notifier_handle, + heads_notifier_handle, + head_subscriptions, + logs_subscriptions, + } } - pub(crate) fn subscribe_new_heads(&self) -> broadcast::Receiver { - self.new_heads_tx.subscribe() + pub async fn register_new_heads_subscription(&self, subscription: SubscriptionSink) { + let mut head_subscriptions = self.head_subscriptions.write().await; + head_subscriptions.retain(|s| !s.is_closed()); + head_subscriptions.push(subscription); } - pub(crate) async fn subscribe_logs(&self) -> broadcast::Receiver> { - self.logs_tx.subscribe() + pub async fn register_new_logs_subscription( + &self, + filter: Filter, + subscription: SubscriptionSink, + ) { + let mut logs_subscriptions = self.logs_subscriptions.write().await; + logs_subscriptions.retain(|(_, s)| !s.is_closed()); + logs_subscriptions.push((filter, subscription)); } } -pub async fn handle_new_heads_subscription( - subscription: SubscriptionSink, - mut rx: broadcast::Receiver, -) { - tokio::spawn(async move { - loop { - let Ok(block) = rx.recv().await else { - // Connection closed - return; - }; +impl Drop for SubscriptionManager { + fn drop(&mut self) { + self.soft_confirmation_handle.abort(); + self.logs_notifier_handle.abort(); + self.heads_notifier_handle.abort(); + } +} +pub async fn new_heads_notifier( + mut rx: mpsc::Receiver, + head_subscriptions: Arc>>, +) { + while let Some(block) = rx.recv().await { + // Acquire the read lock here to prevent starving the writes. + let subscriptions = head_subscriptions.read().await; + let mut send_tasks = vec![]; + for subscription in subscriptions.iter() { let msg = SubscriptionMessage::new( subscription.method_name(), subscription.subscription_id(), &block, ) .unwrap(); - let Ok(_) = subscription.send(msg).await else { - // Connection closed - return; - }; + send_tasks.push(subscription.send(msg)); } - }); + let _ = future::join_all(send_tasks).await; + // Drop lock to release the read lock. + drop(subscriptions); + } } -pub async fn handle_logs_subscription( - subscription: SubscriptionSink, - mut rx: broadcast::Receiver>, - filter: Filter, +pub async fn logs_notifier( + mut rx: mpsc::Receiver>, + logs_subscriptions: Arc>>, ) { - tokio::spawn(async move { - loop { - let Ok(logs) = rx.recv().await else { - // Connection closed - return; - }; - - for log in logs { + while let Some(logs) = rx.recv().await { + // Acquire the read lock here to prevent starving the writes. + let subscriptions = logs_subscriptions.read().await; + let mut send_tasks = vec![]; + for log in logs { + for (filter, subscription) in subscriptions.iter() { if log_matches_filter( &log.clone().try_into().unwrap(), - &filter, + filter, log.block_hash.as_ref().unwrap(), &log.block_number.as_ref().unwrap().to::(), ) { @@ -125,12 +120,43 @@ pub async fn handle_logs_subscription( &log, ) .unwrap(); - let Ok(_) = subscription.send(msg).await else { - // Connection closed - return; - }; + send_tasks.push(subscription.send(msg)); } } } - }); + let _ = future::join_all(send_tasks).await; + // Drop lock to release the read lock. + drop(subscriptions); + } +} + +pub async fn soft_confirmation_event_handler( + storage: C::Storage, + mut soft_confirmation_rx: broadcast::Receiver, + new_heads_tx: mpsc::Sender, + logs_tx: mpsc::Sender>, +) { + let evm = Evm::::default(); + while let Ok(height) = soft_confirmation_rx.recv().await { + let mut working_set = WorkingSet::::new(storage.clone()); + let block = evm + .get_block_by_number( + Some(BlockNumberOrTag::Number(height)), + None, + &mut working_set, + ) + .expect("Error querying block from evm") + .expect("Received signal but evm block is not found"); + + // Only possible error is no receiver + let _ = new_heads_tx.send(block.clone()).await; + + let mut working_set = WorkingSet::::new(storage.clone()); + let logs = evm + .get_logs_in_block_range(&mut working_set, &Filter::default(), height, height) + .expect("Error getting logs in block range"); + + // Only possible error is no receiver + let _ = logs_tx.send(logs).await; + } } diff --git a/crates/ethereum-rpc/src/trace.rs b/crates/ethereum-rpc/src/trace.rs index 2fa2ee9e3..7ca650931 100644 --- a/crates/ethereum-rpc/src/trace.rs +++ b/crates/ethereum-rpc/src/trace.rs @@ -82,6 +82,10 @@ pub async fn handle_debug_trace_chain::new(ethereum.storage.clone()); From 68f32a2225c0df94ee6725a348dd79a81f171fdf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ahmet=20Yaz=C4=B1c=C4=B1?= <75089142+yaziciahmet@users.noreply.github.com> Date: Mon, 30 Sep 2024 18:59:14 +0200 Subject: [PATCH 02/28] Fix typo (#1246) --- bin/citrea/tests/e2e/mod.rs | 2 +- bin/citrea/tests/e2e/proving.rs | 2 +- bin/citrea/tests/e2e/syncing.rs | 2 +- crates/prover/src/da_block_handler.rs | 2 +- .../sovereign-sdk/full-node/sov-stf-runner/src/config.rs | 8 ++++---- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/bin/citrea/tests/e2e/mod.rs b/bin/citrea/tests/e2e/mod.rs index 07aa9e8ba..fab958702 100644 --- a/bin/citrea/tests/e2e/mod.rs +++ b/bin/citrea/tests/e2e/mod.rs @@ -94,7 +94,7 @@ async fn test_all_flow() { Some(ProverConfig { proving_mode: sov_stf_runner::ProverGuestRunConfig::Execute, proof_sampling_number: 0, - enable_reocvery: true, + enable_recovery: true, }), rollup_config, None, diff --git a/bin/citrea/tests/e2e/proving.rs b/bin/citrea/tests/e2e/proving.rs index 428648b4b..d9440ba88 100644 --- a/bin/citrea/tests/e2e/proving.rs +++ b/bin/citrea/tests/e2e/proving.rs @@ -62,7 +62,7 @@ async fn full_node_verify_proof_and_store() { Some(ProverConfig { proving_mode: sov_stf_runner::ProverGuestRunConfig::Execute, proof_sampling_number: 0, - enable_reocvery: true, + enable_recovery: true, }), rollup_config, None, diff --git a/bin/citrea/tests/e2e/syncing.rs b/bin/citrea/tests/e2e/syncing.rs index 63954cf42..ab9c084db 100644 --- a/bin/citrea/tests/e2e/syncing.rs +++ b/bin/citrea/tests/e2e/syncing.rs @@ -295,7 +295,7 @@ async fn test_prover_sync_with_commitments() -> Result<(), anyhow::Error> { Some(ProverConfig { proving_mode: sov_stf_runner::ProverGuestRunConfig::Execute, proof_sampling_number: 0, - enable_reocvery: true, + enable_recovery: true, }), rollup_config, None, diff --git a/crates/prover/src/da_block_handler.rs b/crates/prover/src/da_block_handler.rs index f1806f852..96e265ac9 100644 --- a/crates/prover/src/da_block_handler.rs +++ b/crates/prover/src/da_block_handler.rs @@ -109,7 +109,7 @@ where } pub async fn run(mut self, start_l1_height: u64) { - if self.prover_config.enable_reocvery { + if self.prover_config.enable_recovery { if let Err(e) = self.check_and_recover_ongoing_proving_sessions().await { error!("Failed to recover ongoing proving sessions: {:?}", e); } diff --git a/crates/sovereign-sdk/full-node/sov-stf-runner/src/config.rs b/crates/sovereign-sdk/full-node/sov-stf-runner/src/config.rs index f0193828a..71c27ac5d 100644 --- a/crates/sovereign-sdk/full-node/sov-stf-runner/src/config.rs +++ b/crates/sovereign-sdk/full-node/sov-stf-runner/src/config.rs @@ -132,7 +132,7 @@ pub struct ProverConfig { /// Average number of commitments to prove pub proof_sampling_number: usize, /// If true prover will try to recover ongoing proving sessions - pub enable_reocvery: bool, + pub enable_recovery: bool, } impl Default for ProverConfig { @@ -140,7 +140,7 @@ impl Default for ProverConfig { Self { proving_mode: ProverGuestRunConfig::Execute, proof_sampling_number: 0, - enable_reocvery: true, + enable_recovery: true, } } } @@ -247,7 +247,7 @@ mod tests { let config = r#" proving_mode = "skip" proof_sampling_number = 500 - enable_reocvery = true + enable_recovery = true "#; let config_file = create_config_from(config); @@ -256,7 +256,7 @@ mod tests { let expected = ProverConfig { proving_mode: ProverGuestRunConfig::Skip, proof_sampling_number: 500, - enable_reocvery: true, + enable_recovery: true, }; assert_eq!(config, expected); } From 87e6fc8737ff735acc51713e0e22bdaee5b70af4 Mon Sep 17 00:00:00 2001 From: Rakan Al-Huneiti Date: Tue, 1 Oct 2024 13:26:50 +0300 Subject: [PATCH 03/28] Structured Concurrency: Part 2 (#1216) * Implement managed tasks for SubscriptionManager * Fix clippy * Add comment about trace task * Use broadcast instead of mpsc for l2 blocks events * Fix spawns in DaService * Add TaskManager * Use task manager in sequencer * Document TaskManager * Handle shutdown event * Use TaskTracker * Add comment about using a cancellation token * Use JoinHandles instead of TaskTracker * Use TaskManager in fullnode and prover * Improve bitcoin-da service * Force spawned tasks to accept a cancellation token * Use biased polling * Satisfy clippy * Address PR feedback * Fix checks --- Cargo.lock | 11 +- Cargo.toml | 1 + bin/citrea/src/rollup/bitcoin.rs | 2 +- crates/bitcoin-da/src/service.rs | 217 +++++++++--------- crates/common/Cargo.toml | 1 + crates/common/src/lib.rs | 1 + crates/common/src/tasks/manager.rs | 58 +++++ crates/common/src/tasks/mod.rs | 1 + crates/fullnode/Cargo.toml | 1 + crates/fullnode/src/da_block_handler.rs | 7 +- crates/fullnode/src/lib.rs | 2 +- crates/fullnode/src/runner.rs | 110 +++++---- crates/prover/Cargo.toml | 1 + crates/prover/src/da_block_handler.rs | 7 +- crates/prover/src/lib.rs | 2 +- crates/prover/src/runner.rs | 22 +- crates/sequencer/Cargo.toml | 1 + crates/sequencer/src/sequencer.rs | 56 +++-- .../sovereign-sdk/adapters/mock-da/Cargo.toml | 2 + .../adapters/mock-da/src/service.rs | 145 ++++++++---- 20 files changed, 413 insertions(+), 235 deletions(-) create mode 100644 crates/common/src/tasks/manager.rs create mode 100644 crates/common/src/tasks/mod.rs diff --git a/Cargo.lock b/Cargo.lock index 96819b0b8..ddb2457ba 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1758,6 +1758,7 @@ dependencies = [ "sov-db", "sov-rollup-interface", "tokio", + "tokio-util", "tower-http", "tracing", ] @@ -1840,6 +1841,7 @@ dependencies = [ "sov-stf-runner", "tempfile", "tokio", + "tokio-util", "tower", "tracing", ] @@ -1890,6 +1892,7 @@ dependencies = [ "sov-stf-runner", "tempfile", "tokio", + "tokio-util", "tower", "tracing", ] @@ -1964,6 +1967,7 @@ dependencies = [ "sov-stf-runner", "tempfile", "tokio", + "tokio-util", "tower", "tower-http", "tracing", @@ -7463,6 +7467,7 @@ dependencies = [ "tempfile", "tokio", "tokio-stream", + "tokio-util", "tracing", ] @@ -8285,14 +8290,16 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.11" +version = "0.7.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cf6b47b3771c49ac75ad09a6162f53ad4b8088b76ac60e8ec1455b31a189fe1" +checksum = "61e7c3654c13bcd040d4a03abee2c75b1d14a37b423cf5a813ceae1cc903ec6a" dependencies = [ "bytes", "futures-core", "futures-io", "futures-sink", + "futures-util", + "hashbrown 0.14.5", "pin-project-lite", "tokio", ] diff --git a/Cargo.toml b/Cargo.toml index ad3aec164..cf18746a0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -116,6 +116,7 @@ jsonrpsee = { version = "0.24.2", features = ["jsonrpsee-types"] } schemars = { version = "0.8.16", features = ["derive"] } tempfile = "3.8" tokio = { version = "1.39", features = ["full"] } +tokio-util = { version = "0.7.12", features = ["rt"] } num_cpus = "1.0" # Risc0 dependencies diff --git a/bin/citrea/src/rollup/bitcoin.rs b/bin/citrea/src/rollup/bitcoin.rs index f01b05831..6bc4fb380 100644 --- a/bin/citrea/src/rollup/bitcoin.rs +++ b/bin/citrea/src/rollup/bitcoin.rs @@ -147,7 +147,7 @@ impl RollupBlueprint for BitcoinRollup { // until forced transactions are implemented, // require_wallet_check is set false for full nodes. if require_wallet_check { - // spawn_da_queue only for sequencer and prover + // run only for sequencer and prover Arc::clone(&service).spawn_da_queue(rx); } Ok(service) diff --git a/crates/bitcoin-da/src/service.rs b/crates/bitcoin-da/src/service.rs index 1401b9838..e45dda033 100644 --- a/crates/bitcoin-da/src/service.rs +++ b/crates/bitcoin-da/src/service.rs @@ -9,7 +9,6 @@ use std::path::PathBuf; use std::sync::Arc; use anyhow::{bail, Context, Result}; -// use std::sync::Arc; use async_trait::async_trait; use backoff::future::retry as retry_backoff; use backoff::ExponentialBackoff; @@ -26,6 +25,7 @@ use sov_rollup_interface::da::{DaData, DaDataBatchProof, DaDataLightClient, DaSp use sov_rollup_interface::services::da::{DaService, SenderWithNotifier}; use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender}; use tokio::sync::oneshot::channel as oneshot_channel; +use tokio::{select, signal}; use tracing::{debug, error, info, instrument, trace}; use crate::helpers::builders::batch_proof_namespace::{ @@ -53,16 +53,15 @@ use crate::spec::{BitcoinSpec, RollupParams}; use crate::verifier::BitcoinVerifier; use crate::REVEAL_OUTPUT_AMOUNT; -/// A service that provides data and data availability proofs for Bitcoin -#[derive(Debug)] -pub struct BitcoinService { - client: Client, - network: bitcoin::Network, - da_private_key: Option, - reveal_light_client_prefix: Vec, - reveal_batch_prover_prefix: Vec, - inscribes_queue: UnboundedSender>>, - tx_backup_dir: PathBuf, +pub const FINALITY_DEPTH: u64 = 8; // blocks +const POLLING_INTERVAL: u64 = 10; // seconds + +#[derive(PartialEq, Eq, PartialOrd, Ord, core::hash::Hash)] +pub struct TxidWrapper(Txid); +impl From for [u8; 32] { + fn from(val: TxidWrapper) -> Self { + val.0.to_byte_array() + } } /// Runtime configuration for the DA service @@ -83,8 +82,17 @@ pub struct BitcoinServiceConfig { pub tx_backup_dir: String, } -pub const FINALITY_DEPTH: u64 = 8; // blocks -const POLLING_INTERVAL: u64 = 10; // seconds +/// A service that provides data and data availability proofs for Bitcoin +#[derive(Debug)] +pub struct BitcoinService { + client: Client, + network: bitcoin::Network, + da_private_key: Option, + reveal_light_client_prefix: Vec, + reveal_batch_prover_prefix: Vec, + inscribes_queue: UnboundedSender>>, + tx_backup_dir: PathBuf, +} impl BitcoinService { // Create a new instance of the DA service from the given configuration. @@ -172,84 +180,87 @@ impl BitcoinService { self: Arc, mut rx: UnboundedReceiver>>, ) { - // This is a queue of inscribe requests - tokio::task::spawn_blocking(|| { - tokio::runtime::Handle::current().block_on(async move { - let mut prev_utxo = match self.get_prev_utxo().await { - Ok(Some(prev_utxo)) => Some(prev_utxo), - Ok(None) => { - info!("No pending transactions found"); - None - } - Err(e) => { - error!(?e, "Failed to get pending transactions"); - None - } - }; + tokio::spawn(async move { + let mut prev_utxo = match self.get_prev_utxo().await { + Ok(Some(prev_utxo)) => Some(prev_utxo), + Ok(None) => { + info!("No pending transactions found"); + None + } + Err(e) => { + error!(?e, "Failed to get pending transactions"); + None + } + }; - trace!("BitcoinDA queue is initialized. Waiting for the first request..."); - - // We execute commit and reveal txs one by one to chain them - while let Some(request_opt) = rx.recv().await { - match request_opt { - Some(request) => { - trace!("A new request is received"); - let prev = prev_utxo.take(); - loop { - // Build and send tx with retries: - let fee_sat_per_vbyte = match self.get_fee_rate().await { - Ok(rate) => rate, - Err(e) => { - error!(?e, "Failed to call get_fee_rate. Retrying..."); - tokio::time::sleep(Duration::from_secs(1)).await; - continue; - } - }; - match self - .send_transaction_with_fee_rate( - prev.clone(), - request.da_data.clone(), - fee_sat_per_vbyte, - ) - .await - { - Ok(tx) => { - let tx_id = TxidWrapper(tx.id); - info!(%tx.id, "Sent tx to BitcoinDA"); - prev_utxo = Some(UTXO { - tx_id: tx.id, - vout: 0, - script_pubkey: tx.tx.output[0] - .script_pubkey - .to_hex_string(), - address: None, - amount: tx.tx.output[0].value.to_sat(), - confirmations: 0, - spendable: true, - solvable: true, - }); - - let _ = request.notify.send(Ok(tx_id)); - } - Err(e) => { - error!(?e, "Failed to send transaction to DA layer"); - tokio::time::sleep(Duration::from_secs(1)).await; - continue; + trace!("BitcoinDA queue is initialized. Waiting for the first request..."); + + loop { + select! { + request_opt = rx.recv() => { + if let Some(request_opt) = request_opt { + match request_opt { + Some(request) => { + trace!("A new request is received"); + let prev = prev_utxo.take(); + loop { + // Build and send tx with retries: + let fee_sat_per_vbyte = match self.get_fee_rate().await { + Ok(rate) => rate, + Err(e) => { + error!(?e, "Failed to call get_fee_rate. Retrying..."); + tokio::time::sleep(Duration::from_secs(1)).await; + continue; + } + }; + match self + .send_transaction_with_fee_rate( + prev.clone(), + request.da_data.clone(), + fee_sat_per_vbyte, + ) + .await + { + Ok(tx) => { + let tx_id = TxidWrapper(tx.id); + info!(%tx.id, "Sent tx to BitcoinDA"); + prev_utxo = Some(UTXO { + tx_id: tx.id, + vout: 0, + script_pubkey: tx.tx.output[0].script_pubkey.to_hex_string(), + address: None, + amount: tx.tx.output[0].value.to_sat(), + confirmations: 0, + spendable: true, + solvable: true, + }); + + let _ = request.notify.send(Ok(tx_id)); + } + Err(e) => { + error!(?e, "Failed to send transaction to DA layer"); + tokio::time::sleep(Duration::from_secs(1)).await; + continue; + } + } + break; } } - break; - } - } - None => { - info!("Shutdown signal received. Stopping BitcoinDA queue."); - break; + None => { + info!("Shutdown signal received. Stopping BitcoinDA queue."); + break; + } + } } + }, + _ = signal::ctrl_c() => { + return; } } + } - error!("BitcoinDA queue stopped"); - }); + error!("BitcoinDA queue stopped"); }); } @@ -541,30 +552,6 @@ impl BitcoinService { } } -#[derive(PartialEq, Eq, PartialOrd, Ord, core::hash::Hash)] -pub struct TxidWrapper(Txid); -impl From for [u8; 32] { - fn from(val: TxidWrapper) -> Self { - val.0.to_byte_array() - } -} - -fn calculate_witness_root(txdata: &[TransactionWrapper]) -> [u8; 32] { - let hashes = txdata - .iter() - .enumerate() - .map(|(i, t)| { - if i == 0 { - // Replace the first hash with zeroes. - Wtxid::all_zeros().to_raw_hash().to_byte_array() - } else { - t.compute_wtxid().to_raw_hash().to_byte_array() - } - }) - .collect(); - BitcoinMerkleTree::new(hashes).root() -} - #[async_trait] impl DaService for BitcoinService { type Spec = BitcoinSpec; @@ -977,6 +964,22 @@ pub fn get_relevant_blobs_from_txs( relevant_txs } +fn calculate_witness_root(txdata: &[TransactionWrapper]) -> [u8; 32] { + let hashes = txdata + .iter() + .enumerate() + .map(|(i, t)| { + if i == 0 { + // Replace the first hash with zeroes. + Wtxid::all_zeros().to_raw_hash().to_byte_array() + } else { + t.compute_wtxid().to_raw_hash().to_byte_array() + } + }) + .collect(); + BitcoinMerkleTree::new(hashes).root() +} + #[cfg(test)] mod tests { use core::str::FromStr; diff --git a/crates/common/Cargo.toml b/crates/common/Cargo.toml index 0557c63da..d90fd06fa 100644 --- a/crates/common/Cargo.toml +++ b/crates/common/Cargo.toml @@ -19,6 +19,7 @@ hyper = { workspace = true } jsonrpsee = { workspace = true, features = ["http-client", "server"] } lru = { workspace = true } tokio = { workspace = true } +tokio-util = { workspace = true } tower-http = { workspace = true } tracing = { workspace = true } diff --git a/crates/common/src/lib.rs b/crates/common/src/lib.rs index 49eaf34bc..9b1396d12 100644 --- a/crates/common/src/lib.rs +++ b/crates/common/src/lib.rs @@ -5,4 +5,5 @@ pub mod cache; pub mod da; pub mod error; pub mod rpc; +pub mod tasks; pub mod utils; diff --git a/crates/common/src/tasks/manager.rs b/crates/common/src/tasks/manager.rs new file mode 100644 index 000000000..3d47be61a --- /dev/null +++ b/crates/common/src/tasks/manager.rs @@ -0,0 +1,58 @@ +use std::future::Future; +use std::time::Duration; + +use tokio::task::JoinHandle; +use tokio::time::sleep; +use tokio_util::sync::CancellationToken; + +const WAIT_DURATION: u64 = 5; // 5 seconds + +/// TaskManager manages tasks spawned using tokio and keeps +/// track of handles so that these tasks are cancellable. +/// This provides a way to implement graceful shutdown of our +/// nodes by completing tasks as such read/write to DBs and then +/// performing the shutdown so that the database does not get corrupted. +pub struct TaskManager { + handles: Vec>, + cancellation_token: CancellationToken, +} + +impl Default for TaskManager { + fn default() -> Self { + Self { + handles: vec![], + cancellation_token: CancellationToken::new(), + } + } +} + +impl TaskManager { + /// Spawn a new asynchronous task. + /// + /// Tasks are forced to accept a cancellation token so that they can be notified + /// about the cancellation using the passed token. + pub fn spawn(&mut self, callback: F) + where + F: FnOnce(CancellationToken) -> Fut, + Fut: Future + Send + 'static, + { + let handle = tokio::spawn(callback(self.child_token())); + self.handles.push(handle); + } + + /// Notify all running tasks to stop. + pub async fn abort(&self) { + self.cancellation_token.cancel(); + + // provide tasks with some time to finish existing work + sleep(Duration::from_secs(WAIT_DURATION)).await; + } + + /// Provides a child cancellation token. + /// + /// This would enable us to pass this token into child tasks + /// so that all child tasks can be cancelled at once. + pub fn child_token(&self) -> CancellationToken { + self.cancellation_token.child_token() + } +} diff --git a/crates/common/src/tasks/mod.rs b/crates/common/src/tasks/mod.rs new file mode 100644 index 000000000..ff8de9eb9 --- /dev/null +++ b/crates/common/src/tasks/mod.rs @@ -0,0 +1 @@ +pub mod manager; diff --git a/crates/fullnode/Cargo.toml b/crates/fullnode/Cargo.toml index d8d94719c..fc4a197e1 100644 --- a/crates/fullnode/Cargo.toml +++ b/crates/fullnode/Cargo.toml @@ -34,6 +34,7 @@ rs_merkle = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } tokio = { workspace = true } +tokio-util = { workspace = true } tower = { workspace = true } tracing = { workspace = true } diff --git a/crates/fullnode/src/da_block_handler.rs b/crates/fullnode/src/da_block_handler.rs index 5d66f1f7a..a1df6d9f6 100644 --- a/crates/fullnode/src/da_block_handler.rs +++ b/crates/fullnode/src/da_block_handler.rs @@ -28,6 +28,7 @@ use sov_rollup_interface::zk::{Proof, ZkvmHost}; use tokio::select; use tokio::sync::{mpsc, Mutex}; use tokio::time::{sleep, Duration}; +use tokio_util::sync::CancellationToken; use tracing::{error, info, warn}; pub(crate) struct L1BlockHandler @@ -97,7 +98,7 @@ where } } - pub async fn run(mut self, start_l1_height: u64) { + pub async fn run(mut self, start_l1_height: u64, cancellation_token: CancellationToken) { let mut interval = tokio::time::interval(Duration::from_secs(1)); interval.tick().await; @@ -112,6 +113,10 @@ where loop { select! { + biased; + _ = cancellation_token.cancelled() => { + return; + } _ = &mut l1_sync_worker => {}, Some(l1_block) = l1_rx.recv() => { self.pending_l1_blocks.push_back(l1_block); diff --git a/crates/fullnode/src/lib.rs b/crates/fullnode/src/lib.rs index 60fae5416..108d8ccf1 100644 --- a/crates/fullnode/src/lib.rs +++ b/crates/fullnode/src/lib.rs @@ -34,7 +34,7 @@ impl FullNode { } /// Only run the rpc. - pub async fn run_rpc(self) -> Result<(), anyhow::Error> { + pub async fn run_rpc(mut self) -> Result<(), anyhow::Error> { self.runner.start_rpc_server(self.rpc_methods, None).await; Ok(()) } diff --git a/crates/fullnode/src/runner.rs b/crates/fullnode/src/runner.rs index a04a6d5b3..634af95b9 100644 --- a/crates/fullnode/src/runner.rs +++ b/crates/fullnode/src/runner.rs @@ -7,6 +7,7 @@ use backoff::future::retry as retry_backoff; use backoff::ExponentialBackoffBuilder; use citrea_common::cache::L1BlockCache; use citrea_common::da::get_da_block_at_height; +use citrea_common::tasks::manager::TaskManager; use citrea_primitives::types::SoftConfirmationHash; use jsonrpsee::core::client::Error as JsonrpseeError; use jsonrpsee::server::{BatchRequestConfig, RpcServiceBuilder, ServerBuilder}; @@ -25,9 +26,9 @@ use sov_rollup_interface::stf::StateTransitionFunction; use sov_rollup_interface::storage::HierarchicalStorageManager; use sov_rollup_interface::zk::{Zkvm, ZkvmHost}; use sov_stf_runner::{InitVariant, RollupPublicKeys, RpcConfig, RunnerConfig}; -use tokio::select; use tokio::sync::{broadcast, mpsc, oneshot, Mutex}; use tokio::time::{sleep, Duration}; +use tokio::{select, signal}; use tracing::{debug, error, info, instrument}; use crate::da_block_handler::L1BlockHandler; @@ -65,6 +66,7 @@ where sync_blocks_count: u64, fork_manager: ForkManager, soft_confirmation_tx: broadcast::Sender, + task_manager: TaskManager<()>, } impl CitreaFullnode @@ -149,12 +151,13 @@ where l1_block_cache: Arc::new(Mutex::new(L1BlockCache::new())), fork_manager, soft_confirmation_tx, + task_manager: TaskManager::default(), }) } /// Starts a RPC server with provided rpc methods. pub async fn start_rpc_server( - &self, + &mut self, methods: RpcModule<()>, channel: Option>, ) { @@ -178,43 +181,44 @@ where .layer(citrea_common::rpc::get_healthcheck_proxy_layer()); let rpc_middleware = RpcServiceBuilder::new().layer_fn(citrea_common::rpc::Logger); - let _handle = tokio::spawn(async move { - let server = ServerBuilder::default() - .max_connections(max_connections) - .max_subscriptions_per_connection(max_subscriptions_per_connection) - .max_request_body_size(max_request_body_size) - .max_response_body_size(max_response_body_size) - .set_batch_request_config(BatchRequestConfig::Limit(batch_requests_limit)) - .set_http_middleware(middleware) - .set_rpc_middleware(rpc_middleware) - .build([listen_address].as_ref()) - .await; - - match server { - Ok(server) => { - let bound_address = match server.local_addr() { - Ok(address) => address, - Err(e) => { - error!("{}", e); - return; - } - }; - if let Some(channel) = channel { - if let Err(e) = channel.send(bound_address) { - error!("Could not send bound_address {}: {}", bound_address, e); - return; + self.task_manager + .spawn(move |cancellation_token| async move { + let server = ServerBuilder::default() + .max_connections(max_connections) + .max_subscriptions_per_connection(max_subscriptions_per_connection) + .max_request_body_size(max_request_body_size) + .max_response_body_size(max_response_body_size) + .set_batch_request_config(BatchRequestConfig::Limit(batch_requests_limit)) + .set_http_middleware(middleware) + .set_rpc_middleware(rpc_middleware) + .build([listen_address].as_ref()) + .await; + + match server { + Ok(server) => { + let bound_address = match server.local_addr() { + Ok(address) => address, + Err(e) => { + error!("{}", e); + return; + } + }; + if let Some(channel) = channel { + if let Err(e) = channel.send(bound_address) { + error!("Could not send bound_address {}: {}", bound_address, e); + return; + } } - } - info!("Starting RPC server at {} ", &bound_address); + info!("Starting RPC server at {} ", &bound_address); - let _server_handle = server.start(methods); - futures::future::pending::<()>().await; - } - Err(e) => { - error!("Could not start RPC server: {}", e); + let _server_handle = server.start(methods); + cancellation_token.cancelled().await; + } + Err(e) => { + error!("Could not start RPC server: {}", e); + } } - } - }); + }); } async fn process_l2_block( @@ -325,19 +329,22 @@ where let accept_public_input_as_proven = self.accept_public_input_as_proven; let l1_block_cache = self.l1_block_cache.clone(); - tokio::spawn(async move { - let l1_block_handler = L1BlockHandler::::new( - ledger_db, - da_service, - sequencer_pub_key, - sequencer_da_pub_key, - prover_da_pub_key, - code_commitments_by_spec, - accept_public_input_as_proven, - l1_block_cache.clone(), - ); - l1_block_handler.run(start_l1_height).await - }); + self.task_manager + .spawn(move |cancellation_token| async move { + let l1_block_handler = L1BlockHandler::::new( + ledger_db, + da_service, + sequencer_pub_key, + sequencer_da_pub_key, + prover_da_pub_key, + code_commitments_by_spec, + accept_public_input_as_proven, + l1_block_cache.clone(), + ); + l1_block_handler + .run(start_l1_height, cancellation_token) + .await + }); let (l2_tx, mut l2_rx) = mpsc::channel(1); let l2_sync_worker = sync_l2::( @@ -392,6 +399,11 @@ where } } }, + _ = signal::ctrl_c() => { + info!("Shutting down"); + self.task_manager.abort().await; + return Ok(()); + } } } } diff --git a/crates/prover/Cargo.toml b/crates/prover/Cargo.toml index d819da12f..02bb3e34f 100644 --- a/crates/prover/Cargo.toml +++ b/crates/prover/Cargo.toml @@ -39,6 +39,7 @@ rayon = { workspace = true } rs_merkle = { workspace = true } serde = { workspace = true } tokio = { workspace = true } +tokio-util = { workspace = true } tower = { workspace = true } tracing = { workspace = true } diff --git a/crates/prover/src/da_block_handler.rs b/crates/prover/src/da_block_handler.rs index 96e265ac9..1bdaf7b4e 100644 --- a/crates/prover/src/da_block_handler.rs +++ b/crates/prover/src/da_block_handler.rs @@ -28,6 +28,7 @@ use sov_stf_runner::{ProverConfig, ProverService}; use tokio::select; use tokio::sync::{mpsc, Mutex}; use tokio::time::{sleep, Duration}; +use tokio_util::sync::CancellationToken; use tracing::{error, info, warn}; type CommitmentStateTransitionData = ( @@ -108,7 +109,7 @@ where } } - pub async fn run(mut self, start_l1_height: u64) { + pub async fn run(mut self, start_l1_height: u64, cancellation_token: CancellationToken) { if self.prover_config.enable_recovery { if let Err(e) = self.check_and_recover_ongoing_proving_sessions().await { error!("Failed to recover ongoing proving sessions: {:?}", e); @@ -133,6 +134,10 @@ where interval.tick().await; loop { select! { + biased; + _ = cancellation_token.cancelled() => { + return; + } _ = &mut l1_sync_worker => {}, Some(l1_block) = l1_rx.recv() => { self.pending_l1_blocks.push_back(l1_block); diff --git a/crates/prover/src/lib.rs b/crates/prover/src/lib.rs index 628269018..aac2e9e5d 100644 --- a/crates/prover/src/lib.rs +++ b/crates/prover/src/lib.rs @@ -36,7 +36,7 @@ impl Prover { } /// Only run the rpc. - pub async fn run_rpc(self) -> Result<(), anyhow::Error> { + pub async fn run_rpc(mut self) -> Result<(), anyhow::Error> { self.runner.start_rpc_server(self.rpc_methods, None).await; Ok(()) } diff --git a/crates/prover/src/runner.rs b/crates/prover/src/runner.rs index 551373701..668828389 100644 --- a/crates/prover/src/runner.rs +++ b/crates/prover/src/runner.rs @@ -9,6 +9,7 @@ use backoff::exponential::ExponentialBackoffBuilder; use backoff::future::retry as retry_backoff; use citrea_common::cache::L1BlockCache; use citrea_common::da::get_da_block_at_height; +use citrea_common::tasks::manager::TaskManager; use citrea_primitives::types::SoftConfirmationHash; use jsonrpsee::core::client::Error as JsonrpseeError; use jsonrpsee::server::{BatchRequestConfig, ServerBuilder}; @@ -28,9 +29,9 @@ use sov_rollup_interface::zk::ZkvmHost; use sov_stf_runner::{ InitVariant, ProverConfig, ProverService, RollupPublicKeys, RpcConfig, RunnerConfig, }; -use tokio::select; use tokio::sync::{broadcast, mpsc, oneshot, Mutex}; use tokio::time::sleep; +use tokio::{select, signal}; use tracing::{debug, error, info, instrument}; use crate::da_block_handler::L1BlockHandler; @@ -68,6 +69,7 @@ where sync_blocks_count: u64, fork_manager: ForkManager, soft_confirmation_tx: broadcast::Sender, + task_manager: TaskManager<()>, } impl CitreaProver @@ -157,12 +159,13 @@ where sync_blocks_count: runner_config.sync_blocks_count, fork_manager, soft_confirmation_tx, + task_manager: TaskManager::default(), }) } /// Starts a RPC server with provided rpc methods. pub async fn start_rpc_server( - &self, + &mut self, methods: RpcModule<()>, channel: Option>, ) { @@ -184,7 +187,7 @@ where let middleware = tower::ServiceBuilder::new().layer(citrea_common::rpc::get_cors_layer()); // .layer(citrea_common::rpc::get_healthcheck_proxy_layer()); - let _handle = tokio::spawn(async move { + self.task_manager.spawn(|cancellation_token| async move { let server = ServerBuilder::default() .max_connections(max_connections) .max_subscriptions_per_connection(max_subscriptions_per_connection) @@ -213,7 +216,7 @@ where info!("Starting RPC server at {} ", &bound_address); let _server_handle = server.start(methods); - futures::future::pending::<()>().await; + cancellation_token.cancelled().await; } Err(e) => { error!("Could not start RPC server: {}", e); @@ -249,7 +252,7 @@ where let code_commitments_by_spec = self.code_commitments_by_spec.clone(); let l1_block_cache = self.l1_block_cache.clone(); - tokio::spawn(async move { + self.task_manager.spawn(|cancellation_token| async move { let l1_block_handler = L1BlockHandler::::new( prover_config, @@ -262,7 +265,9 @@ where skip_submission_until_l1, l1_block_cache.clone(), ); - l1_block_handler.run(start_l1_height).await + l1_block_handler + .run(start_l1_height, cancellation_token) + .await }); // Create l2 sync worker task @@ -320,6 +325,11 @@ where } } }, + _ = signal::ctrl_c() => { + info!("Shutting down"); + self.task_manager.abort().await; + return Ok(()); + } } } } diff --git a/crates/sequencer/Cargo.toml b/crates/sequencer/Cargo.toml index 7552245b6..d51fc33c4 100644 --- a/crates/sequencer/Cargo.toml +++ b/crates/sequencer/Cargo.toml @@ -30,6 +30,7 @@ schnellru = "0.2.1" serde = { workspace = true } serde_json = { workspace = true } tokio = { workspace = true } +tokio-util = { workspace = true } tower = { workspace = true } tower-http = { workspace = true } tracing = { workspace = true } diff --git a/crates/sequencer/src/sequencer.rs b/crates/sequencer/src/sequencer.rs index b13fdedf8..585ecf50a 100644 --- a/crates/sequencer/src/sequencer.rs +++ b/crates/sequencer/src/sequencer.rs @@ -8,6 +8,7 @@ use anyhow::{anyhow, bail}; use backoff::future::retry as retry_backoff; use backoff::ExponentialBackoffBuilder; use borsh::BorshDeserialize; +use citrea_common::tasks::manager::TaskManager; use citrea_common::utils::merge_state_diffs; use citrea_evm::{CallMessage, Evm, RlpEvmTransaction, MIN_TRANSACTION_GAS}; use citrea_primitives::basefee::calculate_next_block_base_fee; @@ -46,9 +47,11 @@ use sov_rollup_interface::stf::StateTransitionFunction; use sov_rollup_interface::storage::HierarchicalStorageManager; use sov_rollup_interface::zk::ZkvmHost; use sov_stf_runner::{InitVariant, RollupPublicKeys, RpcConfig}; +use tokio::signal; use tokio::sync::oneshot::channel as oneshot_channel; use tokio::sync::{broadcast, mpsc}; use tokio::time::{sleep, Instant}; +use tokio_util::sync::CancellationToken; use tracing::{debug, error, info, instrument, trace, warn}; use crate::commitment_controller; @@ -95,6 +98,7 @@ where last_state_diff: StateDiff, fork_manager: ForkManager, soft_confirmation_tx: broadcast::Sender, + task_manager: TaskManager<()>, } enum L2BlockMode { @@ -165,6 +169,8 @@ where // Initialize the sequencer with the last state diff from DB. let last_state_diff = ledger_db.get_state_diff()?; + let task_manager = TaskManager::default(); + Ok(Self { da_service, mempool: Arc::new(pool), @@ -186,11 +192,12 @@ where last_state_diff, fork_manager, soft_confirmation_tx, + task_manager, }) } pub async fn start_rpc_server( - &self, + &mut self, channel: Option>, methods: RpcModule<()>, ) -> anyhow::Result<()> { @@ -214,7 +221,7 @@ where // .layer(citrea_common::rpc::get_healthcheck_proxy_layer()); let rpc_middleware = RpcServiceBuilder::new().layer_fn(citrea_common::rpc::Logger); - let _handle = tokio::spawn(async move { + self.task_manager.spawn(|cancellation_token| async move { let server = ServerBuilder::default() .max_connections(max_connections) .max_subscriptions_per_connection(max_subscriptions_per_connection) @@ -244,7 +251,7 @@ where info!("Starting RPC server at {} ", &bound_address); let _server_handle = server.start(methods); - futures::future::pending::<()>().await; + cancellation_token.cancelled().await; } Err(e) => { error!("Could not start RPC server: {}", e); @@ -852,11 +859,14 @@ where let (da_height_update_tx, mut da_height_update_rx) = mpsc::channel(1); let (da_commitment_tx, mut da_commitment_rx) = unbounded::(); - tokio::task::spawn(da_block_monitor( - self.da_service.clone(), - da_height_update_tx, - self.config.da_update_interval_ms, - )); + self.task_manager.spawn(|cancellation_token| { + da_block_monitor( + self.da_service.clone(), + da_height_update_tx, + self.config.da_update_interval_ms, + cancellation_token, + ) + }); let target_block_time = Duration::from_millis(self.config.block_production_interval_ms); @@ -959,6 +969,11 @@ where error!("Sequencer error: {}", e); } }; + }, + _ = signal::ctrl_c() => { + info!("Shutting down sequencer"); + self.task_manager.abort().await; + return Ok(()); } } } @@ -1181,21 +1196,30 @@ async fn da_block_monitor( da_service: Arc, sender: mpsc::Sender>, loop_interval: u64, + cancellation_token: CancellationToken, ) where Da: DaService, { loop { - let l1_data = match get_da_block_data(da_service.clone()).await { - Ok(l1_data) => l1_data, - Err(e) => { - error!("Could not fetch L1 data, {}", e); - continue; + tokio::select! { + biased; + _ = cancellation_token.cancelled() => { + return; } - }; + l1_data = get_da_block_data(da_service.clone()) => { + let l1_data = match l1_data { + Ok(l1_data) => l1_data, + Err(e) => { + error!("Could not fetch L1 data, {}", e); + continue; + } + }; - let _ = sender.send(l1_data).await; + let _ = sender.send(l1_data).await; - sleep(Duration::from_millis(loop_interval)).await; + sleep(Duration::from_millis(loop_interval)).await; + }, + } } } diff --git a/crates/sovereign-sdk/adapters/mock-da/Cargo.toml b/crates/sovereign-sdk/adapters/mock-da/Cargo.toml index f4d94dce2..af78f3ac6 100644 --- a/crates/sovereign-sdk/adapters/mock-da/Cargo.toml +++ b/crates/sovereign-sdk/adapters/mock-da/Cargo.toml @@ -20,6 +20,7 @@ hex = { workspace = true } lazy_static = { version = "1.4.0", optional = true } sha2 = { workspace = true } tokio = { workspace = true, optional = true } +tokio-util = { workspace = true, optional = true } futures = { workspace = true, optional = true } tokio-stream = { version = "0.1.14", features = ["full"], optional = true } pin-project = { workspace = true, optional = true } @@ -39,6 +40,7 @@ native = [ "dep:rusqlite", "dep:serde_json", "dep:tokio", + "dep:tokio-util", "dep:lazy_static", "dep:tokio-stream", "dep:futures", diff --git a/crates/sovereign-sdk/adapters/mock-da/src/service.rs b/crates/sovereign-sdk/adapters/mock-da/src/service.rs index ab97c65a9..e32d24c92 100644 --- a/crates/sovereign-sdk/adapters/mock-da/src/service.rs +++ b/crates/sovereign-sdk/adapters/mock-da/src/service.rs @@ -12,10 +12,11 @@ use sov_rollup_interface::da::{ BlobReaderTrait, BlockHeaderTrait, DaData, DaDataBatchProof, DaDataLightClient, DaSpec, Time, }; use sov_rollup_interface::services::da::{DaService, SenderWithNotifier, SlotData}; -use tokio::sync::mpsc::{unbounded_channel, UnboundedSender}; +use tokio::sync::broadcast::Receiver; +use tokio::sync::mpsc::{unbounded_channel, UnboundedReceiver, UnboundedSender}; use tokio::sync::{broadcast, Mutex as AsyncMutex, MutexGuard as AsyncMutexGuard}; -use tokio::time; -use tracing::instrument::Instrument; +use tokio::{select, time}; +use tokio_util::sync::CancellationToken; use crate::db_connector::DbConnector; use crate::types::{MockAddress, MockBlob, MockBlock, MockDaVerifier}; @@ -62,11 +63,38 @@ impl PlannedFork { } } -#[derive(Clone)] +#[pin_project] +/// Stream of finalized headers +pub struct MockDaBlockHeaderStream { + #[pin] + inner: tokio_stream::wrappers::BroadcastStream, +} + +impl MockDaBlockHeaderStream { + /// Create new stream of finalized headers + pub fn new(receiver: broadcast::Receiver) -> Self { + Self { + inner: tokio_stream::wrappers::BroadcastStream::new(receiver), + } + } +} + +impl futures::Stream for MockDaBlockHeaderStream { + type Item = Result; + + fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let this = self.project(); // Requires the pin-project crate or similar functionality + this.inner + .poll_next(cx) + .map(|opt| opt.map(|res| res.map_err(Into::into))) + } +} + /// DaService used in tests. /// Currently only supports single blob per block. /// Height of the first submitted block is 1. /// Submitted blocks are kept indefinitely in memory. +#[derive(Clone)] pub struct MockDaService { sequencer_da_address: MockAddress, // don't need a mutex, but DaService trait requires it by Sync trait @@ -75,8 +103,12 @@ pub struct MockDaService { blocks_to_finality: u32, /// Used for calculating correct finality from state of `blocks` finalized_header_sender: broadcast::Sender, + /// Used for sending transactions + transaction_queue_sender: + UnboundedSender::TransactionId>>>, wait_attempts: usize, planned_fork: Arc>>, + worker_handle: CancellationToken, } impl MockDaService { @@ -92,21 +124,60 @@ impl MockDaService { blocks_to_finality: u32, db_path: &Path, ) -> Self { - let (tx, rx1) = broadcast::channel(16); - // Spawn a task, so channel is never closed - tokio::spawn(async move { - let mut rx = rx1; - while let Ok(header) = rx.recv().instrument(tracing::Span::current()).await { - tracing::debug!("Finalized MockHeader: {}", header); - } - }); - Self { + let (transaction_queue_sender, transaction_queue_receiver) = + unbounded_channel::::TransactionId>>>(); + let (finalized_header_sender, finalized_header_receiver) = broadcast::channel(16); + + let da_service = Self { sequencer_da_address, blocks: Arc::new(AsyncMutex::new(DbConnector::new(db_path))), blocks_to_finality, - finalized_header_sender: tx, + finalized_header_sender, + transaction_queue_sender, wait_attempts: 100_0000, planned_fork: Arc::new(Mutex::new(None)), + worker_handle: CancellationToken::new(), + }; + + // Spawn the DA service worker task with a cancellation token + // so that when the DA service instance is dropped, the tasks are cancelled. + let cancellation_token = da_service.worker_handle.clone(); + let this = da_service.clone(); + tokio::spawn(this.da_service_worker( + transaction_queue_receiver, + finalized_header_receiver, + cancellation_token, + )); + + da_service + } + + async fn da_service_worker( + self, + mut transaction_queue_receiver: UnboundedReceiver< + Option::TransactionId>>, + >, + mut finalized_header_receiver: Receiver, + cancellation_token: CancellationToken, + ) { + loop { + select! { + biased; + _ = cancellation_token.cancelled() => { + return; + } + req = transaction_queue_receiver.recv() => { + if let Some(Some(req)) = req { + let res = self.send_transaction(req.da_data).await; + let _ = req.notify.send(res); + } + }, + header = finalized_header_receiver.recv() => { + if let Ok(header) = header { + tracing::debug!("Finalized MockHeader: {}", header); + } + }, + } } } @@ -274,33 +345,6 @@ impl MockDaService { } } -#[pin_project] -/// Stream of finalized headers -pub struct MockDaBlockHeaderStream { - #[pin] - inner: tokio_stream::wrappers::BroadcastStream, -} - -impl MockDaBlockHeaderStream { - /// Create new stream of finalized headers - pub fn new(receiver: broadcast::Receiver) -> Self { - Self { - inner: tokio_stream::wrappers::BroadcastStream::new(receiver), - } - } -} - -impl futures::Stream for MockDaBlockHeaderStream { - type Item = Result; - - fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - let this = self.project(); // Requires the pin-project crate or similar functionality - this.inner - .poll_next(cx) - .map(|opt| opt.map(|res| res.map_err(Into::into))) - } -} - #[async_trait] impl DaService for MockDaService { type Spec = MockDaSpec; @@ -443,15 +487,7 @@ impl DaService for MockDaService { fn get_send_transaction_queue( &self, ) -> UnboundedSender>> { - let (tx, mut rx) = unbounded_channel::>>(); - let this = self.clone(); - tokio::spawn(async move { - while let Some(Some(req)) = rx.recv().await { - let res = this.send_transaction(req.da_data).await; - let _ = req.notify.send(res); - } - }); - tx + self.transaction_queue_sender.clone() } async fn send_aggregated_zk_proof(&self, proof: &[u8]) -> Result { @@ -488,6 +524,12 @@ impl DaService for MockDaService { } } +impl Drop for MockDaService { + fn drop(&mut self) { + self.worker_handle.cancel(); + } +} + fn hash_to_array(bytes: &[u8]) -> [u8; 32] { let mut hasher = sha2::Sha256::new(); hasher.update(bytes); @@ -555,6 +597,9 @@ mod tests { // This prevents test for freezing in case of a bug // But we need to wait longer, as `MockDa let timeout_duration = Duration::from_millis(1000); + + // This task runs for as long as we are still expecting blocks and will stop eventually. + // Therefore, this is not considered to be an escaping task. tokio::spawn(async move { let mut received = Vec::with_capacity(expected_num_headers); for _ in 0..expected_num_headers { From 189ebe7a36f1f2704318bb031307ac4d5d7cd624 Mon Sep 17 00:00:00 2001 From: Rakan Al-Huneiti Date: Tue, 1 Oct 2024 13:43:27 +0300 Subject: [PATCH 04/28] Pin foundry (#1253) * Pin foundary * Add comment --- .github/workflows/checks.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 335a5fbc4..d685bbdac 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -442,7 +442,9 @@ jobs: - name: Install Foundry uses: foundry-rs/foundry-toolchain@v1 with: - version: nightly + # https://github.com/foundry-rs/foundry/releases/tag/nightly-25f24e677a6a32a62512ad4f561995589ac2c7dc + # This is the latest version known to work for us + version: nightly-25f24e677a6a32a62512ad4f561995589ac2c7dc - name: Set up Python uses: actions/setup-python@v4 From d4b7ec9545d26a4bfcb5142d1c4175d7ccbd75fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erce=20Can=20Bekt=C3=BCre?= Date: Tue, 1 Oct 2024 13:51:39 +0300 Subject: [PATCH 05/28] Remove puclihs mock da block script --- .github/workflows/checks.yml | 2 -- docs/run-dev.md | 6 +----- .../mock-dockerized/publish_da_block.sh | 20 ------------------- resources/hive/Dockerfile | 2 +- 4 files changed, 2 insertions(+), 28 deletions(-) delete mode 100755 resources/configs/mock-dockerized/publish_da_block.sh diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 335a5fbc4..50eb053f9 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -256,7 +256,6 @@ jobs: sleep 2 RUST_LOG=off ./target/release/citrea --rollup-config-path resources/configs/mock/rollup_config.toml --genesis-paths resources/genesis/mock/ & sleep 2 - ./resources/configs/mock-dockerized/publish_da_block.sh & cd ./bin/citrea/tests/evm/uniswap npx hardhat run --network citrea scripts/01_deploy.js npx hardhat run --network citrea scripts/02_swap.js @@ -298,7 +297,6 @@ jobs: sleep 2 RUST_LOG=off ./target/release/citrea --da-layer mock --rollup-config-path resources/configs/mock/rollup_config.toml --genesis-paths resources/genesis/mock/ & sleep 2 - # ./resources/configs/mock-dockerized/publish_da_block.sh & cd ./bin/citrea/tests/evm/web3_py python test.py diff --git a/docs/run-dev.md b/docs/run-dev.md index d2d22774c..2e55d33c7 100644 --- a/docs/run-dev.md +++ b/docs/run-dev.md @@ -34,11 +34,7 @@ _Optional_: Run full node on Mock DA: Full node RPC is accessible at `127.0.0.1:12346` -If test_mode is set to false in the sequencer config, the sequencer will publish blocks every 2 seconds. To also publish mock DA blocks, run this script: - -```sh -./configs/mock-dockerized/publish_da_block.sh -``` +If test_mode is set to false in the sequencer config, the sequencer will publish blocks every 2 seconds. ### Run on Bitcoin Regtest diff --git a/resources/configs/mock-dockerized/publish_da_block.sh b/resources/configs/mock-dockerized/publish_da_block.sh deleted file mode 100755 index b5fcf1d22..000000000 --- a/resources/configs/mock-dockerized/publish_da_block.sh +++ /dev/null @@ -1,20 +0,0 @@ -# script to auto send 'da_publishBlock' requests every 2 seconds -# TODO: read sequencer url from .toml files - -SLEEP_DURATION=60 -SEQUENCER_URL='http://0.0.0.0:12345' - -echo "Publishing da blocks every 60 seconds" -echo "Sequencer URL: $SEQUENCER_URL" - -while true; do - curl -s -o /dev/null --location $SEQUENCER_URL \ - --header 'Content-Type: application/json' \ - --data '{ - "jsonrpc": "2.0", - "method": "da_publishBlock", - "params": [], - "id": 1 - }' - sleep $SLEEP_DURATION -done diff --git a/resources/hive/Dockerfile b/resources/hive/Dockerfile index 4f9ff0d3c..393804061 100644 --- a/resources/hive/Dockerfile +++ b/resources/hive/Dockerfile @@ -18,4 +18,4 @@ RUN SKIP_GUEST_BUILD=1 cargo build --release --bin citrea EXPOSE 8545 -ENTRYPOINT ["sh", "-c", "./resources/configs/mock-dockerized/publish_da_block.sh & ./target/release/citrea --genesis-paths ./resources/genesis/mock-dockerized --rollup-config-path ./configs/mock-dockerized/rollup_config.toml --sequencer-config-path ./configs/mock-dockerized/sequencer_config.toml"] +ENTRYPOINT ["sh", "-c", "./target/release/citrea --genesis-paths ./resources/genesis/mock-dockerized --rollup-config-path ./configs/mock-dockerized/rollup_config.toml --sequencer-config-path ./configs/mock-dockerized/sequencer_config.toml"] From 559065b39ecf39b9e7af9c9fb2eb46a4752a7f7e Mon Sep 17 00:00:00 2001 From: Rakan Al-Huneiti Date: Tue, 1 Oct 2024 13:53:51 +0300 Subject: [PATCH 06/28] Use default block time on error (#1244) * Use default block time on error * Constant target_block_time * Remove import --- crates/sequencer/src/sequencer.rs | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/crates/sequencer/src/sequencer.rs b/crates/sequencer/src/sequencer.rs index 585ecf50a..8a0cd4c1f 100644 --- a/crates/sequencer/src/sequencer.rs +++ b/crates/sequencer/src/sequencer.rs @@ -50,7 +50,7 @@ use sov_stf_runner::{InitVariant, RollupPublicKeys, RpcConfig}; use tokio::signal; use tokio::sync::oneshot::channel as oneshot_channel; use tokio::sync::{broadcast, mpsc}; -use tokio::time::{sleep, Instant}; +use tokio::time::sleep; use tokio_util::sync::CancellationToken; use tracing::{debug, error, info, instrument, trace, warn}; @@ -943,22 +943,8 @@ where } - let instant = Instant::now(); match self.produce_l2_block(da_block, l1_fee_rate, L2BlockMode::NotEmpty).await { Ok((l1_block_number, state_diff_threshold_reached)) => { - // Set the next iteration's wait time to produce a block based on the - // previous block's execution time. - // This is mainly to make sure we account for the execution time to - // achieve consistent 2-second block production. - let parent_block_exec_time = instant.elapsed(); - - block_production_tick = tokio::time::interval( - target_block_time - .checked_sub(parent_block_exec_time) - .unwrap_or_default(), - ); - block_production_tick.tick().await; - last_used_l1_height = l1_block_number; if da_commitment_tx.unbounded_send(state_diff_threshold_reached).is_err() { From 37a862918b9b7c9b7c4de2f085f4877173a3701c Mon Sep 17 00:00:00 2001 From: jfldde <168934971+jfldde@users.noreply.github.com> Date: Tue, 1 Oct 2024 12:12:22 +0100 Subject: [PATCH 07/28] Dump logs and cleanup on assert failures (#1252) --- bin/citrea/tests/bitcoin_e2e/test_case.rs | 28 +++++++++++++---------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/bin/citrea/tests/bitcoin_e2e/test_case.rs b/bin/citrea/tests/bitcoin_e2e/test_case.rs index b78e8b86d..ee284df34 100644 --- a/bin/citrea/tests/bitcoin_e2e/test_case.rs +++ b/bin/citrea/tests/bitcoin_e2e/test_case.rs @@ -61,23 +61,27 @@ impl TestCaseRunner { /// /// This sets up the framework, executes the test, and ensures cleanup is performed even if a panic occurs. pub async fn run(mut self) -> Result<()> { + let mut framework = None; let result = panic::AssertUnwindSafe(async { - let mut framework = TestFramework::new(Self::generate_test_config()?).await?; - let test_result = self.run_test_case(&mut framework).await; - - if test_result.is_err() { - if let Err(e) = framework.dump_log() { - eprintln!("Error dumping log: {}", e); - } - } - - framework.stop().await?; - - test_result + framework = Some(TestFramework::new(Self::generate_test_config()?).await?); + let f = framework.as_mut().unwrap(); + self.run_test_case(f).await }) .catch_unwind() .await; + let f = framework + .as_mut() + .expect("Framework not correctly initialized"); + + if result.is_err() { + if let Err(e) = f.dump_log() { + eprintln!("Error dumping log: {}", e); + } + } + + f.stop().await?; + // Additional test cleanup self.0.cleanup().await?; From fbc24378ed694ce09b92cef06e46a3b85d360da5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erce=20Can=20Bekt=C3=BCre?= <47954181+ercecan@users.noreply.github.com> Date: Wed, 2 Oct 2024 17:45:31 +0300 Subject: [PATCH 08/28] Fix estiamte gas l1 fee issue when metamask max amount is selected (#1261) * Fix estiamte gas l1 fee issue when metamask max amount is selected * Fix tests * Remove unnecessary comment --- bin/citrea/tests/evm/tracing.rs | 12 +++---- crates/evm/src/evm/handler.rs | 8 ++--- crates/evm/src/query.rs | 33 +++++++++++++++---- .../adapters/mock-da/src/service.rs | 4 +-- 4 files changed, 39 insertions(+), 18 deletions(-) diff --git a/bin/citrea/tests/evm/tracing.rs b/bin/citrea/tests/evm/tracing.rs index 752a09e16..dd09fb458 100644 --- a/bin/citrea/tests/evm/tracing.rs +++ b/bin/citrea/tests/evm/tracing.rs @@ -116,14 +116,14 @@ async fn tracing_tests() -> Result<(), Box> { // It was replaced with the gas limit in our trace. let reth_json = serde_json::from_value::(json![{ "from": "0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266", - "gas": "0x679c", + "gas": "0x6a7d", "gasUsed": "0xba65", "to": "0xe7f1725e7734ce288f8367e1bb143e90bb3f0512", "input": "0xb7d5b6580000000000000000000000005fbdb2315678afecb367f032d93f642f64180aa30000000000000000000000000000000000000000000000000000000000000003", "calls": [ { "from": "0xe7f1725e7734ce288f8367e1bb143e90bb3f0512", - "gas": "0x5833", + "gas": "0x5b09", "gasUsed": "0x57f2", "to": "0x5fbdb2315678afecb367f032d93f642f64180aa3", "input": "0x60fe47b10000000000000000000000000000000000000000000000000000000000000003", @@ -178,7 +178,7 @@ async fn tracing_tests() -> Result<(), Box> { .await; let expected_send_eth_trace = serde_json::from_value::( - json![{"from":"0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266","gas":"0x1","gasUsed":"0x5208", + json![{"from":"0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266","gas":"0x1d1","gasUsed":"0x5208", "to":"0xf39fd6e51aad88f6f4ce6ab8827279cfffb92255","input":"0x","value":"0x4563918244f40000","type":"CALL"}], ).unwrap(); assert_eq!(send_eth_trace, CallTracer(expected_send_eth_trace.clone())); @@ -207,11 +207,11 @@ async fn tracing_tests() -> Result<(), Box> { ); let expected_call_get_trace = serde_json::from_value::( - json![{"from":"0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266","gas":"0x1886","gasUsed":"0x6b64","to":"0xe7f1725e7734ce288f8367e1bb143e90bb3f0512", + json![{"from":"0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266","gas":"0x19c7","gasUsed":"0x6b64","to":"0xe7f1725e7734ce288f8367e1bb143e90bb3f0512", "input":"0x35c152bd0000000000000000000000005fbdb2315678afecb367f032d93f642f64180aa3", "output":"0x0000000000000000000000000000000000000000000000000000000000000000", "calls":[{"from":"0xe7f1725e7734ce288f8367e1bb143e90bb3f0512", - "gas":"0xc3e","gasUsed":"0x996","to":"0x5fbdb2315678afecb367f032d93f642f64180aa3", + "gas":"0xd7a","gasUsed":"0x996","to":"0x5fbdb2315678afecb367f032d93f642f64180aa3", "input":"0x6d4ce63c","output":"0x0000000000000000000000000000000000000000000000000000000000000003","type":"STATICCALL"}], "value":"0x0","type":"CALL"}], ).unwrap(); @@ -291,7 +291,7 @@ async fn tracing_tests() -> Result<(), Box> { .await; let expected_top_call_only_call_get_trace = serde_json::from_value::( - json![{"from":"0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266","gas":"0x1886","gasUsed":"0x6b64","to":"0xe7f1725e7734ce288f8367e1bb143e90bb3f0512", + json![{"from":"0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266","gas":"0x19c7","gasUsed":"0x6b64","to":"0xe7f1725e7734ce288f8367e1bb143e90bb3f0512", "input":"0x35c152bd0000000000000000000000005fbdb2315678afecb367f032d93f642f64180aa3", "output":"0x0000000000000000000000000000000000000000000000000000000000000000", "calls":[], diff --git a/crates/evm/src/evm/handler.rs b/crates/evm/src/evm/handler.rs index 1d8906c0b..b3112066e 100644 --- a/crates/evm/src/evm/handler.rs +++ b/crates/evm/src/evm/handler.rs @@ -25,11 +25,11 @@ use crate::system_events::SYSTEM_SIGNER; use crate::{BASE_FEE_VAULT, L1_FEE_VAULT}; /// Eoa size is reduced because code_hash for eoas are None on state diff, converted to empty Keccak internally for evm operations -const DB_ACCOUNT_SIZE_EOA: usize = 42; +pub(crate) const DB_ACCOUNT_SIZE_EOA: usize = 42; const DB_ACCOUNT_SIZE_CONTRACT: usize = 75; /// Normally db account key is: 6 bytes of prefix ("Evm/a/") + 1 byte for size of remaining data + 20 bytes of address = 27 bytes -const DB_ACCOUNT_KEY_SIZE: usize = 27; +pub(crate) const DB_ACCOUNT_KEY_SIZE: usize = 27; /// Storage key is 59 bytes because of sov sdk prefix ("Evm/s/") const STORAGE_KEY_SIZE: usize = 59; @@ -70,9 +70,9 @@ Let's consider a batch of 1 block with the following transactions: If every user pays 0.75 of the balance state diff they created, the total balance state diff will be covered */ /// Nonce and balance are stored together so we use single constant -const NONCE_DISCOUNTED_PERCENTAGE: usize = 55; +pub(crate) const NONCE_DISCOUNTED_PERCENTAGE: usize = 55; const STORAGE_DISCOUNTED_PERCENTAGE: usize = 66; -const ACCOUNT_DISCOUNTED_PERCENTAGE: usize = 29; +pub(crate) const ACCOUNT_DISCOUNTED_PERCENTAGE: usize = 29; #[derive(Copy, Clone, Default, Debug)] pub struct TxInfo { diff --git a/crates/evm/src/query.rs b/crates/evm/src/query.rs index 9adacd625..1adda33dc 100644 --- a/crates/evm/src/query.rs +++ b/crates/evm/src/query.rs @@ -6,7 +6,6 @@ use alloy_eips::eip2930::AccessListWithGasUsed; use alloy_primitives::Uint; use alloy_rlp::Encodable; use jsonrpsee::core::RpcResult; -use reth_primitives::constants::GWEI_TO_WEI; use reth_primitives::TxKind::{Call, Create}; use reth_primitives::{ Block, BlockId, BlockNumberOrTag, SealedHeader, TransactionSignedEcRecovered, U256, U64, @@ -38,7 +37,10 @@ use crate::evm::call::prepare_call_env; use crate::evm::db::EvmDb; use crate::evm::primitive_types::{BlockEnv, Receipt, SealedBlock, TransactionSignedAndRecovered}; use crate::evm::DbAccount; -use crate::handler::TxInfo; +use crate::handler::{ + TxInfo, ACCOUNT_DISCOUNTED_PERCENTAGE, DB_ACCOUNT_KEY_SIZE, DB_ACCOUNT_SIZE_EOA, + NONCE_DISCOUNTED_PERCENTAGE, +}; use crate::rpc_helpers::*; use crate::{BloomFilter, Evm, EvmChainConfig, FilterBlockOption, FilterError}; @@ -71,8 +73,7 @@ impl EstimatedTxExpenses { /// Return total estimated gas used including evm gas and L1 fee. pub(crate) fn gas_with_l1_overhead(&self) -> U256 { // Actually not an L1 fee but l1_fee / base_fee. - let l1_fee_overhead = - U256::from(1).max(self.l1_fee / (self.base_fee + U256::from(GWEI_TO_WEI))); // assume 1 gwei priority fee + let l1_fee_overhead = U256::from(1).max(self.l1_fee.div_ceil(self.base_fee)); l1_fee_overhead + U256::from(self.gas_used) } } @@ -879,11 +880,31 @@ impl Evm { if let Ok((res, tx_info)) = res { if res.result.is_success() { + // If value is zero we should add extra balance transfer diff size assuming the first estimate gas was done by metamask + // we do this because on metamask when trying to send max amount to an address it will send 2 estimate_gas requests + // One with 0 value and the other with the remaining balance that extract from the current balance after the gas fee is deducted + // This causes the diff size to be lower than the actual diff size, and the tx to fail due to not enough l1 fee + let mut diff_size = tx_info.l1_diff_size; + let mut l1_fee = tx_info.l1_fee; + if tx_env.value.is_zero() { + // Calculation taken from diff size calculation in handler.rs + let balance_diff_size = + (DB_ACCOUNT_KEY_SIZE * ACCOUNT_DISCOUNTED_PERCENTAGE / 100) + as u64 + + ((DB_ACCOUNT_SIZE_EOA + DB_ACCOUNT_KEY_SIZE) + * NONCE_DISCOUNTED_PERCENTAGE + / 100) as u64; + + diff_size += balance_diff_size; + l1_fee = l1_fee.saturating_add( + U256::from(l1_fee_rate) * (U256::from(balance_diff_size)), + ); + } return Ok(EstimatedTxExpenses { gas_used: U64::from(MIN_TRANSACTION_GAS), base_fee: env_base_fee, - l1_fee: tx_info.l1_fee, - l1_diff_size: tx_info.l1_diff_size, + l1_fee, + l1_diff_size: diff_size, }); } } diff --git a/crates/sovereign-sdk/adapters/mock-da/src/service.rs b/crates/sovereign-sdk/adapters/mock-da/src/service.rs index e32d24c92..d14e74a0e 100644 --- a/crates/sovereign-sdk/adapters/mock-da/src/service.rs +++ b/crates/sovereign-sdk/adapters/mock-da/src/service.rs @@ -502,8 +502,8 @@ impl DaService for MockDaService { } async fn get_fee_rate(&self) -> Result { - // Mock constant - Ok(10_u128) + // Mock constant, use min possible in bitcoin + Ok(2500000000_u128) } async fn get_block_by_hash( From bbbe7bf7719efbfc9e5483867ce0a398108bc1f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erce=20Can=20Bekt=C3=BCre?= <47954181+ercecan@users.noreply.github.com> Date: Wed, 2 Oct 2024 18:29:12 +0300 Subject: [PATCH 09/28] Fix hive (#1263) * Fix configs * Remove publish mock block from docker * Update port for hive * Rename config in hive --- ...nfig.toml => sequencer_rollup_config.toml} | 7 ++----- resources/hive/Dockerfile | 3 ++- resources/hive/hive_publish_block.sh | 21 ------------------- 3 files changed, 4 insertions(+), 27 deletions(-) rename resources/configs/mock-dockerized/{rollup_config.toml => sequencer_rollup_config.toml} (84%) delete mode 100755 resources/hive/hive_publish_block.sh diff --git a/resources/configs/mock-dockerized/rollup_config.toml b/resources/configs/mock-dockerized/sequencer_rollup_config.toml similarity index 84% rename from resources/configs/mock-dockerized/rollup_config.toml rename to resources/configs/mock-dockerized/sequencer_rollup_config.toml index e3e26ba9f..94762cb30 100644 --- a/resources/configs/mock-dockerized/rollup_config.toml +++ b/resources/configs/mock-dockerized/sequencer_rollup_config.toml @@ -9,16 +9,13 @@ db_path = "resources/dbs/da-db" [storage] # The path to the rollup's data directory. Paths that do not begin with `/` are interpreted as relative paths. -path = "resources/dbs/full-node-db" +path = "resources/dbs/sequencer-db" db_max_open_files = 5000 [rpc] # the host and port to bind the rpc server for bind_host = "0.0.0.0" bind_port = 8545 +max_connections = 10000 enable_subscriptions = true max_subscriptions_per_connection = 100 - -[runner] -sequencer_client_url = "http://0.0.0.0:8545" -include_tx_body = false diff --git a/resources/hive/Dockerfile b/resources/hive/Dockerfile index 393804061..4200d6924 100644 --- a/resources/hive/Dockerfile +++ b/resources/hive/Dockerfile @@ -18,4 +18,5 @@ RUN SKIP_GUEST_BUILD=1 cargo build --release --bin citrea EXPOSE 8545 -ENTRYPOINT ["sh", "-c", "./target/release/citrea --genesis-paths ./resources/genesis/mock-dockerized --rollup-config-path ./configs/mock-dockerized/rollup_config.toml --sequencer-config-path ./configs/mock-dockerized/sequencer_config.toml"] +ENTRYPOINT ["sh", "-c", "./target/release/citrea --genesis-paths ./resources/genesis/mock-dockerized --rollup-config-path ./resources/configs/mock-dockerized/sequencer_rollup_config.toml --sequencer-config-path ./resources/configs/mock-dockerized/sequencer_config.toml"] + diff --git a/resources/hive/hive_publish_block.sh b/resources/hive/hive_publish_block.sh deleted file mode 100755 index ff29421cf..000000000 --- a/resources/hive/hive_publish_block.sh +++ /dev/null @@ -1,21 +0,0 @@ -# script to auto send 'citrea_testPublishBlock' requests every 2 seconds -# TODO: read sequencer url from .toml files - -SLEEP_DURATION=2 -SEQUENCER_URL='http://0.0.0.0:8545' - -echo "Publishing blocks every 2 seconds" -echo "Sequencer URL: $SEQUENCER_URL" - -while true; do - curl -s -o /dev/null --location $SEQUENCER_URL \ - --header 'Content-Type: application/json' \ - --data '{ - "jsonrpc": "2.0", - "method": "citrea_testPublishBlock", - "params": [], - "id": 1 - }' - - sleep $SLEEP_DURATION -done From 82bf52d067c1886d92c55c141c8fc062b1931164 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erce=20Can=20Bekt=C3=BCre?= <47954181+ercecan@users.noreply.github.com> Date: Thu, 3 Oct 2024 12:34:02 +0300 Subject: [PATCH 10/28] Refactor eth estimate gas l1 fee issue (#1264) * Refactor eth estimate gas l1 fee issue * Nits * Fix bug * Refactor * Do not ignore resources * Fix --------- Co-authored-by: Roman Proskuryakoff --- .dockerignore | 4 ++-- crates/evm/src/evm/handler.rs | 13 +++++++++---- crates/evm/src/query.rs | 12 ++---------- 3 files changed, 13 insertions(+), 16 deletions(-) diff --git a/.dockerignore b/.dockerignore index 58f2c33fa..8abb7b980 100644 --- a/.dockerignore +++ b/.dockerignore @@ -3,9 +3,8 @@ # include source files !/bin -!/resources/configs !/crates -!/resources/hive +!/resources !Cargo.lock !Cargo.toml !deny.toml @@ -15,6 +14,7 @@ !packages_to_publish.yml !rust-toolchain.toml !rustfmt.toml +!/da-db # include for vergen constants !/.git diff --git a/crates/evm/src/evm/handler.rs b/crates/evm/src/evm/handler.rs index b3112066e..06a7712f4 100644 --- a/crates/evm/src/evm/handler.rs +++ b/crates/evm/src/evm/handler.rs @@ -25,11 +25,11 @@ use crate::system_events::SYSTEM_SIGNER; use crate::{BASE_FEE_VAULT, L1_FEE_VAULT}; /// Eoa size is reduced because code_hash for eoas are None on state diff, converted to empty Keccak internally for evm operations -pub(crate) const DB_ACCOUNT_SIZE_EOA: usize = 42; +const DB_ACCOUNT_SIZE_EOA: usize = 42; const DB_ACCOUNT_SIZE_CONTRACT: usize = 75; /// Normally db account key is: 6 bytes of prefix ("Evm/a/") + 1 byte for size of remaining data + 20 bytes of address = 27 bytes -pub(crate) const DB_ACCOUNT_KEY_SIZE: usize = 27; +const DB_ACCOUNT_KEY_SIZE: usize = 27; /// Storage key is 59 bytes because of sov sdk prefix ("Evm/s/") const STORAGE_KEY_SIZE: usize = 59; @@ -70,9 +70,9 @@ Let's consider a batch of 1 block with the following transactions: If every user pays 0.75 of the balance state diff they created, the total balance state diff will be covered */ /// Nonce and balance are stored together so we use single constant -pub(crate) const NONCE_DISCOUNTED_PERCENTAGE: usize = 55; +const NONCE_DISCOUNTED_PERCENTAGE: usize = 55; const STORAGE_DISCOUNTED_PERCENTAGE: usize = 66; -pub(crate) const ACCOUNT_DISCOUNTED_PERCENTAGE: usize = 29; +const ACCOUNT_DISCOUNTED_PERCENTAGE: usize = 29; #[derive(Copy, Clone, Default, Debug)] pub struct TxInfo { @@ -618,3 +618,8 @@ fn decrease_caller_balance( let address = context.evm.env.tx.caller; change_balance(context, amount, false, address) } + +pub(crate) fn diff_size_send_eth_eoa() -> usize { + DB_ACCOUNT_KEY_SIZE * ACCOUNT_DISCOUNTED_PERCENTAGE / 100 + + (DB_ACCOUNT_SIZE_EOA + DB_ACCOUNT_KEY_SIZE) * NONCE_DISCOUNTED_PERCENTAGE / 100 +} diff --git a/crates/evm/src/query.rs b/crates/evm/src/query.rs index 1adda33dc..a8e71be60 100644 --- a/crates/evm/src/query.rs +++ b/crates/evm/src/query.rs @@ -37,10 +37,7 @@ use crate::evm::call::prepare_call_env; use crate::evm::db::EvmDb; use crate::evm::primitive_types::{BlockEnv, Receipt, SealedBlock, TransactionSignedAndRecovered}; use crate::evm::DbAccount; -use crate::handler::{ - TxInfo, ACCOUNT_DISCOUNTED_PERCENTAGE, DB_ACCOUNT_KEY_SIZE, DB_ACCOUNT_SIZE_EOA, - NONCE_DISCOUNTED_PERCENTAGE, -}; +use crate::handler::{diff_size_send_eth_eoa, TxInfo}; use crate::rpc_helpers::*; use crate::{BloomFilter, Evm, EvmChainConfig, FilterBlockOption, FilterError}; @@ -888,12 +885,7 @@ impl Evm { let mut l1_fee = tx_info.l1_fee; if tx_env.value.is_zero() { // Calculation taken from diff size calculation in handler.rs - let balance_diff_size = - (DB_ACCOUNT_KEY_SIZE * ACCOUNT_DISCOUNTED_PERCENTAGE / 100) - as u64 - + ((DB_ACCOUNT_SIZE_EOA + DB_ACCOUNT_KEY_SIZE) - * NONCE_DISCOUNTED_PERCENTAGE - / 100) as u64; + let balance_diff_size = diff_size_send_eth_eoa() as u64; diff_size += balance_diff_size; l1_fee = l1_fee.saturating_add( From 7c3440af162101e1fd9f7aa71578b420be3ef691 Mon Sep 17 00:00:00 2001 From: Rakan Al-Huneiti Date: Fri, 4 Oct 2024 10:58:27 +0300 Subject: [PATCH 11/28] Pruning skeleton (#1229) * Implement managed tasks for SubscriptionManager * Fix clippy * Add comment about trace task * Use broadcast instead of mpsc for l2 blocks events * Fix spawns in DaService * Add TaskManager * Use task manager in sequencer * Document TaskManager * Handle shutdown event * Use TaskTracker * Add comment about using a cancellation token * Use JoinHandles instead of TaskTracker * Use TaskManager in fullnode and prover * Improve bitcoin-da service * Force spawned tasks to accept a cancellation token * Use biased polling * Satisfy clippy * WIP * Add pruning tables * Pruning skeleton implementation * Use pruner in nodes * Use biased polling based on order * WIP * Fix how config is done * Derive default * Add logs * Let the tasks finish without panicing * Use pruning config in fullnode and prover * Add simple run test * Use option instead of PruningMode * Unneccessary changes * l2_receiver * Cleanup prints * Use last pruned block in calculation * Implement pruning criteria * Lint and add comment * Set the last_pruned_block to up_to_block value * Don't store config internally * Remove from constructor * Should not change * Move config value * Remove pruning from sequencer / prover --- Cargo.lock | 19 ++++ Cargo.toml | 1 + bin/citrea/tests/bitcoin_e2e/test_case.rs | 1 + bin/citrea/tests/test_helpers/mod.rs | 1 + crates/bitcoin-da/src/service.rs | 20 ---- crates/fullnode/Cargo.toml | 1 + crates/fullnode/src/runner.rs | 15 +++ .../tests/runner_initialization_tests.rs | 1 + crates/pruning/Cargo.toml | 29 +++++ crates/pruning/src/criteria.rs | 25 +++++ crates/pruning/src/lib.rs | 100 ++++++++++++++++++ crates/pruning/src/pruners/evm.rs | 10 ++ crates/pruning/src/pruners/ledger.rs | 8 ++ crates/pruning/src/pruners/mod.rs | 5 + crates/pruning/src/tests.rs | 44 ++++++++ .../full-node/db/sov-db/src/ledger_db/mod.rs | 24 ++++- .../db/sov-db/src/ledger_db/traits.rs | 6 ++ .../full-node/db/sov-db/src/schema/tables.rs | 6 ++ .../full-node/sov-stf-runner/Cargo.toml | 4 + .../full-node/sov-stf-runner/src/config.rs | 4 + resources/configs/mock/rollup_config.toml | 1 + 21 files changed, 301 insertions(+), 24 deletions(-) create mode 100644 crates/pruning/Cargo.toml create mode 100644 crates/pruning/src/criteria.rs create mode 100644 crates/pruning/src/lib.rs create mode 100644 crates/pruning/src/pruners/evm.rs create mode 100644 crates/pruning/src/pruners/ledger.rs create mode 100644 crates/pruning/src/pruners/mod.rs create mode 100644 crates/pruning/src/tests.rs diff --git a/Cargo.lock b/Cargo.lock index ddb2457ba..e2656d9a9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1820,6 +1820,7 @@ dependencies = [ "borsh", "citrea-common", "citrea-primitives", + "citrea-pruning", "futures", "hex", "jsonrpsee", @@ -1897,6 +1898,23 @@ dependencies = [ "tracing", ] +[[package]] +name = "citrea-pruning" +version = "0.5.0-rc.1" +dependencies = [ + "anyhow", + "citrea-evm", + "citrea-primitives", + "futures", + "serde", + "sov-db", + "sov-modules-api", + "tempfile", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "citrea-risc0-bonsai-adapter" version = "0.5.0-rc.1" @@ -7796,6 +7814,7 @@ dependencies = [ "anyhow", "async-trait", "borsh", + "citrea-pruning", "futures", "hex", "hyper 1.4.1", diff --git a/Cargo.toml b/Cargo.toml index cf18746a0..b0ab25c26 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,6 +5,7 @@ members = [ "bin/citrea", "crates/common", "crates/primitives", + "crates/pruning", "crates/bitcoin-da", "crates/evm", "crates/ethereum-rpc", diff --git a/bin/citrea/tests/bitcoin_e2e/test_case.rs b/bin/citrea/tests/bitcoin_e2e/test_case.rs index ee284df34..ff0e57797 100644 --- a/bin/citrea/tests/bitcoin_e2e/test_case.rs +++ b/bin/citrea/tests/bitcoin_e2e/test_case.rs @@ -167,6 +167,7 @@ impl TestCaseRunner { ), include_tx_body: true, accept_public_input_as_proven: Some(true), + pruning_config: None, sync_blocks_count: 10, }); diff --git a/bin/citrea/tests/test_helpers/mod.rs b/bin/citrea/tests/test_helpers/mod.rs index 26c8aea3b..232f8f163 100644 --- a/bin/citrea/tests/test_helpers/mod.rs +++ b/bin/citrea/tests/test_helpers/mod.rs @@ -139,6 +139,7 @@ pub fn create_default_rollup_config( sequencer_client_url: format!("http://localhost:{}", socket_addr.port()), accept_public_input_as_proven: Some(true), sync_blocks_count: 10, + pruning_config: None, }), NodeMode::SequencerNode => None, }, diff --git a/crates/bitcoin-da/src/service.rs b/crates/bitcoin-da/src/service.rs index e45dda033..0ed159ec6 100644 --- a/crates/bitcoin-da/src/service.rs +++ b/crates/bitcoin-da/src/service.rs @@ -1132,8 +1132,6 @@ mod tests { .await .expect("Failed to send transaction"); - println!("sent 1"); - da_service .send_transaction(DaData::SequencerCommitment(SequencerCommitment { merkle_root: [14; 32], @@ -1143,14 +1141,10 @@ mod tests { .await .expect("Failed to send transaction"); - println!("sent 2"); - println!("\n\nSend some BTC to this address: bcrt1qscttjdc3wypf7ttu0203sqgfz80a4q38cne693 and press enter\n\n"); let mut s = String::new(); std::io::stdin().read_line(&mut s).unwrap(); - println!("sent 3"); - let size = 2000; let blob = (0..size).map(|_| rand::random::()).collect::>(); @@ -1159,14 +1153,10 @@ mod tests { .await .expect("Failed to send transaction"); - println!("sent 4"); - println!("\n\nSend some BTC to this address: bcrt1qscttjdc3wypf7ttu0203sqgfz80a4q38cne693 and press enter\n\n"); let mut s = String::new(); std::io::stdin().read_line(&mut s).unwrap(); - println!("sent 5"); - let size = 600 * 1024; let blob = (0..size).map(|_| rand::random::()).collect::>(); @@ -1175,8 +1165,6 @@ mod tests { .await .expect("Failed to send transaction"); - println!("sent 6"); - // seq com different namespace get_service_wrong_namespace() .await @@ -1196,8 +1184,6 @@ mod tests { .await .expect("Failed to send transaction"); - println!("sent 7"); - // seq com incorrect pubkey and sig get_service_correct_sig_different_public_key() .await @@ -1218,8 +1204,6 @@ mod tests { .await .expect("Failed to send transaction"); - println!("sent 8"); - let size = 1200 * 1024; let blob = (0..size).map(|_| rand::random::()).collect::>(); @@ -1228,8 +1212,6 @@ mod tests { .await .expect("Failed to send transaction"); - println!("sent 9"); - da_service .send_transaction(DaData::SequencerCommitment(SequencerCommitment { merkle_root: [30; 32], @@ -1238,8 +1220,6 @@ mod tests { })) .await .expect("Failed to send transaction"); - - println!("sent 10"); } // #[tokio::test] diff --git a/crates/fullnode/Cargo.toml b/crates/fullnode/Cargo.toml index fc4a197e1..5e81ac479 100644 --- a/crates/fullnode/Cargo.toml +++ b/crates/fullnode/Cargo.toml @@ -12,6 +12,7 @@ repository.workspace = true # Citrea Deps citrea-common = { path = "../common" } citrea-primitives = { path = "../primitives", features = ["native"] } +citrea-pruning = { path = "../pruning" } sequencer-client = { path = "../sequencer-client" } # Sov SDK deps diff --git a/crates/fullnode/src/runner.rs b/crates/fullnode/src/runner.rs index 634af95b9..07dbb45eb 100644 --- a/crates/fullnode/src/runner.rs +++ b/crates/fullnode/src/runner.rs @@ -9,6 +9,7 @@ use citrea_common::cache::L1BlockCache; use citrea_common::da::get_da_block_at_height; use citrea_common::tasks::manager::TaskManager; use citrea_primitives::types::SoftConfirmationHash; +use citrea_pruning::{Pruner, PruningConfig}; use jsonrpsee::core::client::Error as JsonrpseeError; use jsonrpsee::server::{BatchRequestConfig, RpcServiceBuilder, ServerBuilder}; use jsonrpsee::RpcModule; @@ -66,6 +67,7 @@ where sync_blocks_count: u64, fork_manager: ForkManager, soft_confirmation_tx: broadcast::Sender, + pruning_config: Option, task_manager: TaskManager<()>, } @@ -151,6 +153,7 @@ where l1_block_cache: Arc::new(Mutex::new(L1BlockCache::new())), fork_manager, soft_confirmation_tx, + pruning_config: runner_config.pruning_config, task_manager: TaskManager::default(), }) } @@ -320,6 +323,18 @@ where } }; + if let Some(config) = &self.pruning_config { + let pruner = Pruner::::new( + config.clone(), + self.ledger_db.get_last_pruned_l2_height()?.unwrap_or(0), + self.soft_confirmation_tx.subscribe(), + self.ledger_db.clone(), + ); + + self.task_manager + .spawn(|cancellation_token| pruner.run(cancellation_token)); + } + let ledger_db = self.ledger_db.clone(); let da_service = self.da_service.clone(); let sequencer_pub_key = self.sequencer_pub_key.clone(); diff --git a/crates/fullnode/tests/runner_initialization_tests.rs b/crates/fullnode/tests/runner_initialization_tests.rs index dde9c335e..a5614bdbe 100644 --- a/crates/fullnode/tests/runner_initialization_tests.rs +++ b/crates/fullnode/tests/runner_initialization_tests.rs @@ -89,6 +89,7 @@ fn initialize_runner( include_tx_body: true, accept_public_input_as_proven: None, sync_blocks_count: 10, + pruning_config: None, }), da: MockDaConfig { sender_address: address, diff --git a/crates/pruning/Cargo.toml b/crates/pruning/Cargo.toml new file mode 100644 index 000000000..1c6c06640 --- /dev/null +++ b/crates/pruning/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "citrea-pruning" +version.workspace = true +authors.workspace = true +edition.workspace = true +homepage.workspace = true +license.workspace = true +publish.workspace = true +repository.workspace = true + +[dependencies] +# Citrea dependencies +citrea-evm = { path = "../evm", features = ["native"] } +citrea-primitives = { path = "../primitives", features = ["native"] } + +# Sov SDK deps +sov-db = { path = "../sovereign-sdk/full-node/db/sov-db" } +sov-modules-api = { path = "../sovereign-sdk/module-system/sov-modules-api", default-features = false } + +# 3rd-party dependencies +anyhow = { workspace = true } +futures = { workspace = true } +serde = { workspace = true } +tokio = { workspace = true } +tokio-util = { workspace = true } +tracing = { workspace = true } + +[dev-dependencies] +tempfile = { workspace = true } diff --git a/crates/pruning/src/criteria.rs b/crates/pruning/src/criteria.rs new file mode 100644 index 000000000..5f8482426 --- /dev/null +++ b/crates/pruning/src/criteria.rs @@ -0,0 +1,25 @@ +/// This defines the interface of a pruning criteria. +pub(crate) trait Criteria { + /// Decides whether pruning should be done or not. + /// + /// If None is returned, no pruning should happen. + /// Otherwise, it will return the `up_to_block` value. + fn should_prune(&self, last_pruned_block: u64, current_block_number: u64) -> Option; +} + +/// This distance criteria prunes blocks up to `last_pruned_block + distance`. +/// However, to keep `distance` amount of blocks, we have to wait for at least twice +/// the `distance` value to prune up to that point. +pub(crate) struct DistanceCriteria { + pub(crate) distance: u64, +} + +impl Criteria for DistanceCriteria { + fn should_prune(&self, last_pruned_block: u64, current_block_number: u64) -> Option { + let trigger_block = last_pruned_block + (2 * self.distance) + 1; + if current_block_number >= trigger_block { + return Some(last_pruned_block + self.distance); + } + None + } +} diff --git a/crates/pruning/src/lib.rs b/crates/pruning/src/lib.rs new file mode 100644 index 000000000..e6f6c3215 --- /dev/null +++ b/crates/pruning/src/lib.rs @@ -0,0 +1,100 @@ +use criteria::DistanceCriteria; +use futures::future; +use serde::{Deserialize, Serialize}; +use sov_db::ledger_db::SharedLedgerOps; +use tokio::select; +use tokio::sync::broadcast; +use tokio_util::sync::CancellationToken; +use tracing::{error, info}; + +use crate::criteria::Criteria; +use crate::pruners::{prune_evm, prune_ledger}; + +mod criteria; +mod pruners; +#[cfg(test)] +mod tests; + +/// A configuration type to define the behaviour of the pruner. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct PruningConfig { + /// Defines the number of blocks from the tip of the chain to remove. + pub distance: u64, +} + +impl Default for PruningConfig { + fn default() -> Self { + Self { distance: 256 } + } +} + +pub struct Pruner +where + DB: SharedLedgerOps, +{ + /// The last block number which was pruned. + last_pruned_block: u64, + /// A channel receiver which gets notified of new L2 blocks. + l2_receiver: broadcast::Receiver, + /// Access to ledger tables. + ledger_db: DB, + /// Criteria to decide pruning + criteria: Box, +} + +impl Pruner +where + DB: SharedLedgerOps + Send + Sync + Clone + 'static, +{ + pub fn new( + config: PruningConfig, + last_pruned_block: u64, + l2_receiver: broadcast::Receiver, + ledger_db: DB, + ) -> Self { + // distance is the only criteria implemented at the moment. + let criteria = Box::new(DistanceCriteria { + distance: config.distance, + }); + Self { + last_pruned_block, + l2_receiver, + ledger_db, + criteria, + } + } + + /// Prune everything + pub async fn prune(&self, up_to_block: u64) { + info!("Pruning up to L2 block: {}", up_to_block); + let ledger_db = self.ledger_db.clone(); + let ledger_pruning_handle = + tokio::task::spawn_blocking(move || prune_ledger(ledger_db, up_to_block)); + let evm_pruning_handle = tokio::task::spawn_blocking(move || prune_evm(up_to_block)); + + future::join_all([ledger_pruning_handle, evm_pruning_handle]).await; + } + + pub async fn run(mut self, cancellation_token: CancellationToken) { + loop { + select! { + biased; + _ = cancellation_token.cancelled() => { + // Store the last pruned l2 height in ledger DB to be restored in the next initialization. + if let Err(e) = self.ledger_db.set_last_pruned_l2_height(self.last_pruned_block) { + error!("Failed to store last pruned L2 height {}: {:?}", self.last_pruned_block, e); + } + return; + } + current_l2_block = self.l2_receiver.recv() => { + if let Ok(current_l2_block) = current_l2_block { + if let Some(up_to_block) = self.criteria.should_prune(self.last_pruned_block, current_l2_block) { + self.prune(up_to_block).await; + self.last_pruned_block = up_to_block; + } + } + }, + } + } + } +} diff --git a/crates/pruning/src/pruners/evm.rs b/crates/pruning/src/pruners/evm.rs new file mode 100644 index 000000000..af05397c7 --- /dev/null +++ b/crates/pruning/src/pruners/evm.rs @@ -0,0 +1,10 @@ +use citrea_evm::Evm; +use sov_modules_api::default_context::DefaultContext; +use tracing::debug; + +/// Prune evm +pub(crate) fn prune_evm(up_to_block: u64) { + debug!("Pruning EVM, up to L2 block {}", up_to_block); + let _evm = Evm::::default(); + // unimplemented!() +} diff --git a/crates/pruning/src/pruners/ledger.rs b/crates/pruning/src/pruners/ledger.rs new file mode 100644 index 000000000..06da6f26c --- /dev/null +++ b/crates/pruning/src/pruners/ledger.rs @@ -0,0 +1,8 @@ +use sov_db::ledger_db::SharedLedgerOps; +use tracing::debug; + +/// Prune ledger +pub(crate) fn prune_ledger(_ledger_db: DB, up_to_block: u64) { + debug!("Pruning Ledger, up to L2 block {}", up_to_block); + // unimplemented!() +} diff --git a/crates/pruning/src/pruners/mod.rs b/crates/pruning/src/pruners/mod.rs new file mode 100644 index 000000000..1c3b195be --- /dev/null +++ b/crates/pruning/src/pruners/mod.rs @@ -0,0 +1,5 @@ +mod evm; +mod ledger; + +pub(crate) use evm::*; +pub(crate) use ledger::*; diff --git a/crates/pruning/src/tests.rs b/crates/pruning/src/tests.rs new file mode 100644 index 000000000..3c1ec8207 --- /dev/null +++ b/crates/pruning/src/tests.rs @@ -0,0 +1,44 @@ +use std::thread::sleep; +use std::time::Duration; + +use sov_db::ledger_db::LedgerDB; +use sov_db::rocks_db_config::RocksdbConfig; +use tokio::sync::broadcast; +use tokio_util::sync::CancellationToken; + +use crate::criteria::{Criteria, DistanceCriteria}; +use crate::{Pruner, PruningConfig}; + +#[tokio::test(flavor = "multi_thread")] +async fn test_pruner_simple_run() { + let tmpdir = tempfile::tempdir().unwrap(); + let (sender, receiver) = broadcast::channel(1); + let cancellation_token = CancellationToken::new(); + + let ledger_db = LedgerDB::with_config(&RocksdbConfig::new(tmpdir.path(), None)).unwrap(); + let pruner = Pruner::new(PruningConfig { distance: 5 }, 0, receiver, ledger_db); + + tokio::spawn(pruner.run(cancellation_token.clone())); + + sleep(Duration::from_secs(1)); + + for i in 1..=10 { + let _ = sender.send(i); + } + + sleep(Duration::from_secs(1)); + + cancellation_token.cancel(); +} + +#[test] +pub fn test_should_prune() { + let criteria = DistanceCriteria { distance: 1000 }; + assert_eq!(criteria.should_prune(0, 1000), None); + assert_eq!(criteria.should_prune(0, 2000), None); + assert_eq!(criteria.should_prune(0, 2001), Some(1000)); + + assert_eq!(criteria.should_prune(1000, 2000), None); + assert_eq!(criteria.should_prune(1000, 3000), None); + assert_eq!(criteria.should_prune(1000, 3001), Some(2000)); +} diff --git a/crates/sovereign-sdk/full-node/db/sov-db/src/ledger_db/mod.rs b/crates/sovereign-sdk/full-node/db/sov-db/src/ledger_db/mod.rs index dc959bf43..cdd3c86d6 100644 --- a/crates/sovereign-sdk/full-node/db/sov-db/src/ledger_db/mod.rs +++ b/crates/sovereign-sdk/full-node/db/sov-db/src/ledger_db/mod.rs @@ -13,10 +13,10 @@ use tracing::instrument; use crate::rocks_db_config::RocksdbConfig; use crate::schema::tables::{ BatchByNumber, CommitmentsByNumber, L2GenesisStateRoot, L2RangeByL1Height, L2Witness, - LastSequencerCommitmentSent, LastStateDiff, MempoolTxs, PendingProvingSessions, - PendingSequencerCommitmentL2Range, ProofsBySlotNumber, ProverLastScannedSlot, ProverStateDiffs, - SlotByHash, SlotByNumber, SoftConfirmationByHash, SoftConfirmationByNumber, - SoftConfirmationStatus, VerifiedProofsBySlotNumber, LEDGER_TABLES, + LastPrunedBlock, LastSequencerCommitmentSent, LastStateDiff, MempoolTxs, + PendingProvingSessions, PendingSequencerCommitmentL2Range, ProofsBySlotNumber, + ProverLastScannedSlot, ProverStateDiffs, SlotByHash, SlotByNumber, SoftConfirmationByHash, + SoftConfirmationByNumber, SoftConfirmationStatus, VerifiedProofsBySlotNumber, LEDGER_TABLES, }; use crate::schema::types::{ split_tx_for_storage, BatchNumber, L2HeightRange, SlotNumber, StoredProof, StoredSlot, @@ -451,6 +451,22 @@ impl SharedLedgerOps for LedgerDB { Ok(()) } + + #[instrument(level = "trace", skip(self), err, ret)] + fn get_last_pruned_l2_height(&self) -> anyhow::Result> { + self.db.get::(&()) + } + + /// Set the last pruned L2 block number + #[instrument(level = "trace", skip(self), err, ret)] + fn set_last_pruned_l2_height(&self, l2_height: u64) -> anyhow::Result<()> { + let mut schema_batch = SchemaBatch::new(); + + schema_batch.put::(&(), &l2_height)?; + self.db.write_schemas(schema_batch)?; + + Ok(()) + } } impl ProverLedgerOps for LedgerDB { diff --git a/crates/sovereign-sdk/full-node/db/sov-db/src/ledger_db/traits.rs b/crates/sovereign-sdk/full-node/db/sov-db/src/ledger_db/traits.rs index efadb9908..c0a72e393 100644 --- a/crates/sovereign-sdk/full-node/db/sov-db/src/ledger_db/traits.rs +++ b/crates/sovereign-sdk/full-node/db/sov-db/src/ledger_db/traits.rs @@ -119,6 +119,12 @@ pub trait SharedLedgerOps { /// Set the last scanned slot fn set_last_scanned_l1_height(&self, l1_height: SlotNumber) -> Result<()>; + + /// Get the last pruned block number + fn get_last_pruned_l2_height(&self) -> Result>; + + /// Set the last pruned block number + fn set_last_pruned_l2_height(&self, l2_height: u64) -> Result<()>; } /// Node ledger operations diff --git a/crates/sovereign-sdk/full-node/db/sov-db/src/schema/tables.rs b/crates/sovereign-sdk/full-node/db/sov-db/src/schema/tables.rs index ef4128bc9..9fb1aae96 100644 --- a/crates/sovereign-sdk/full-node/db/sov-db/src/schema/tables.rs +++ b/crates/sovereign-sdk/full-node/db/sov-db/src/schema/tables.rs @@ -69,6 +69,7 @@ pub const LEDGER_TABLES: &[&str] = &[ MempoolTxs::table_name(), PendingProvingSessions::table_name(), ProverStateDiffs::table_name(), + LastPrunedBlock::table_name(), ]; /// A list of all tables used by the NativeDB. These tables store @@ -321,6 +322,11 @@ define_table_with_default_codec!( (ProverStateDiffs) BatchNumber => StateDiff ); +define_table_with_seek_key_codec!( + /// Stores the last pruned L2 block number + (LastPrunedBlock) () => u64 +); + impl KeyEncoder for NodeKey { fn encode_key(&self) -> sov_schema_db::schema::Result> { // 8 bytes for version, 4 each for the num_nibbles and bytes.len() fields, plus 1 byte per byte of nibllepath diff --git a/crates/sovereign-sdk/full-node/sov-stf-runner/Cargo.toml b/crates/sovereign-sdk/full-node/sov-stf-runner/Cargo.toml index 07b6ca97d..3d4d84efa 100644 --- a/crates/sovereign-sdk/full-node/sov-stf-runner/Cargo.toml +++ b/crates/sovereign-sdk/full-node/sov-stf-runner/Cargo.toml @@ -33,6 +33,9 @@ sov-db = { path = "../db/sov-db", optional = true } sov-modules-api = { path = "../../module-system/sov-modules-api", default-features = false } sov-rollup-interface = { path = "../../rollup-interface" } +# Citrea +citrea-pruning = { path = "../../../pruning", optional = true } + [dev-dependencies] sha2 = { workspace = true } tempfile = { workspace = true } @@ -58,4 +61,5 @@ native = [ "rand", "tower", "hyper", + "dep:citrea-pruning", ] diff --git a/crates/sovereign-sdk/full-node/sov-stf-runner/src/config.rs b/crates/sovereign-sdk/full-node/sov-stf-runner/src/config.rs index 71c27ac5d..c5cbac51b 100644 --- a/crates/sovereign-sdk/full-node/sov-stf-runner/src/config.rs +++ b/crates/sovereign-sdk/full-node/sov-stf-runner/src/config.rs @@ -2,6 +2,7 @@ use std::fs::File; use std::io::Read; use std::path::{Path, PathBuf}; +use citrea_pruning::PruningConfig; use serde::de::DeserializeOwned; use serde::{Deserialize, Serialize}; @@ -19,6 +20,8 @@ pub struct RunnerConfig { /// Number of blocks to request during sync #[serde(default = "default_sync_blocks_count")] pub sync_blocks_count: u64, + /// Configurations for pruning + pub pruning_config: Option, } /// RPC configuration. @@ -214,6 +217,7 @@ mod tests { include_tx_body: true, accept_public_input_as_proven: None, sync_blocks_count: 10, + pruning_config: None, }), da: sov_mock_da::MockDaConfig { sender_address: [0; 32].into(), diff --git a/resources/configs/mock/rollup_config.toml b/resources/configs/mock/rollup_config.toml index 5f468bd4e..a19c13be2 100644 --- a/resources/configs/mock/rollup_config.toml +++ b/resources/configs/mock/rollup_config.toml @@ -22,3 +22,4 @@ max_subscriptions_per_connection = 100 [runner] include_tx_body = false sequencer_client_url = "http://0.0.0.0:12345" +# pruning_config.distance = 10 From 348846ac2e382405ab979e8f23c201b324eb2bf4 Mon Sep 17 00:00:00 2001 From: jfldde <168934971+jfldde@users.noreply.github.com> Date: Fri, 4 Oct 2024 12:34:15 +0100 Subject: [PATCH 12/28] Derive SequencerClient (#1269) * Derive SequencerClient * Renames * Lint --- Cargo.lock | 1 + crates/sequencer/Cargo.toml | 3 +- crates/sequencer/src/rpc.rs | 182 +++++++++++++++++++----------- crates/sequencer/src/sequencer.rs | 4 +- 4 files changed, 120 insertions(+), 70 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e2656d9a9..d4d91689f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1946,6 +1946,7 @@ dependencies = [ "alloy-rlp", "alloy-sol-types", "anyhow", + "async-trait", "backoff", "borsh", "chrono", diff --git a/crates/sequencer/Cargo.toml b/crates/sequencer/Cargo.toml index d51fc33c4..1ed1b5472 100644 --- a/crates/sequencer/Cargo.toml +++ b/crates/sequencer/Cargo.toml @@ -16,6 +16,7 @@ resolver = "2" alloy-rlp = { workspace = true } alloy-sol-types = { workspace = true } anyhow = { workspace = true } +async-trait = { workspace = true } backoff = { workspace = true } borsh = { workspace = true } chrono = { workspace = true } @@ -23,7 +24,7 @@ digest = { workspace = true } futures = { workspace = true } hex = { workspace = true } hyper = { workspace = true } -jsonrpsee = { workspace = true, features = ["http-client", "server"] } +jsonrpsee = { workspace = true, features = ["http-client", "server", "client"] } parking_lot = { workspace = true } rs_merkle = { workspace = true } schnellru = "0.2.1" diff --git a/crates/sequencer/src/rpc.rs b/crates/sequencer/src/rpc.rs index 2f385bec2..e308b5705 100644 --- a/crates/sequencer/src/rpc.rs +++ b/crates/sequencer/src/rpc.rs @@ -2,9 +2,10 @@ use std::sync::Arc; use citrea_evm::Evm; use futures::channel::mpsc::UnboundedSender; +use jsonrpsee::core::RpcResult; +use jsonrpsee::proc_macros::rpc; use jsonrpsee::types::error::{INTERNAL_ERROR_CODE, INTERNAL_ERROR_MSG}; -use jsonrpsee::types::ErrorObjectOwned; -use jsonrpsee::RpcModule; +use jsonrpsee::types::{ErrorCode, ErrorObject, ErrorObjectOwned}; use parking_lot::Mutex; use reth_primitives::{Bytes, IntoRecoveredTransaction, B256}; use reth_rpc_eth_types::error::EthApiError; @@ -27,26 +28,56 @@ pub(crate) struct RpcContext RpcResult; + + #[method(name = "eth_getTransactionByHash")] + #[blocking] + fn eth_get_transaction_by_hash( + &self, + hash: B256, + mempool_only: Option, + ) -> RpcResult>; + + #[method(name = "citrea_sendRawDepositTransaction")] + #[blocking] + fn send_raw_deposit_transaction(&self, deposit: Bytes) -> RpcResult<()>; + + #[method(name = "citrea_testPublishBlock")] + async fn publish_test_block(&self) -> RpcResult<()>; +} + +pub struct SequencerRpcServerImpl< C: sov_modules_api::Context, DB: SequencerLedgerOps + Send + Sync + 'static, ->( - rpc_context: RpcContext, -) -> Result>, jsonrpsee::core::RegisterMethodError> { - let test_mode = rpc_context.test_mode; - let mut rpc = RpcModule::new(rpc_context); - rpc.register_async_method("eth_sendRawTransaction", |parameters, ctx, _| async move { - debug!("Sequencer: eth_sendRawTransaction"); - let data: Bytes = parameters.one()?; +> { + context: Arc>, +} + +impl + SequencerRpcServerImpl +{ + pub fn new(context: RpcContext) -> Self { + Self { + context: Arc::new(context), + } + } +} - // Only check if the signature is valid for now - let recovered: reth_primitives::PooledTransactionsElementEcRecovered = - recover_raw_transaction(data.clone())?; +#[async_trait::async_trait] +impl SequencerRpcServer + for SequencerRpcServerImpl +{ + async fn eth_send_raw_transaction(&self, data: Bytes) -> RpcResult { + debug!("Sequencer: eth_sendRawTransaction"); + let recovered = recover_raw_transaction(data.clone())?; let pool_transaction = EthPooledTransaction::from_pooled(recovered); - // submit the transaction to the pool with an `External` origin - let hash: B256 = ctx + let hash = self + .context .mempool .add_external_transaction(pool_transaction.clone()) .await @@ -58,48 +89,40 @@ pub(crate) fn create_rpc_module< .clone() .into_signed() .encode_enveloped(&mut rlp_encoded_tx); + // Do not return error here just log - if let Err(e) = ctx.ledger.insert_mempool_tx(hash.to_vec(), rlp_encoded_tx) { + if let Err(e) = self + .context + .ledger + .insert_mempool_tx(hash.to_vec(), rlp_encoded_tx) + { tracing::warn!("Failed to insert mempool tx into db: {:?}", e); } - Ok::(hash) - })?; - - if test_mode { - rpc.register_async_method("citrea_testPublishBlock", |_, ctx, _| async move { - debug!("Sequencer: citrea_testPublishBlock"); - ctx.l2_force_block_tx.unbounded_send(()).map_err(|e| { - ErrorObjectOwned::owned( - INTERNAL_ERROR_CODE, - INTERNAL_ERROR_MSG, - Some(format!("Could not send L2 force block transaction: {e}")), - ) - })?; - Ok::<(), ErrorObjectOwned>(()) - })?; + Ok(hash) } - rpc.register_blocking_method("eth_getTransactionByHash", move |parameters, ctx, _| { - let mut params = parameters.sequence(); - let hash: B256 = params.next()?; - let mempool_only: Result, ErrorObjectOwned> = params.optional_next(); + fn eth_get_transaction_by_hash( + &self, + hash: B256, + mempool_only: Option, + ) -> RpcResult> { debug!( "Sequencer: eth_getTransactionByHash({}, {:?})", hash, mempool_only ); - match ctx.mempool.get(&hash) { + match self.context.mempool.get(&hash) { Some(tx) => { let tx_signed_ec_recovered = tx.to_recovered_transaction(); // tx signed ec recovered let tx: reth_rpc_types::Transaction = from_recovered(tx_signed_ec_recovered); Ok::, ErrorObjectOwned>(Some(tx)) } None => match mempool_only { - Ok(Some(true)) => Ok::, ErrorObjectOwned>(None), + Some(true) => Ok::, ErrorObjectOwned>(None), _ => { let evm = Evm::::default(); - let mut working_set = WorkingSet::::new(ctx.storage.clone()); + let mut working_set = WorkingSet::::new(self.context.storage.clone()); match evm.get_transaction_by_hash(hash, &mut working_set) { Ok(tx) => Ok::, ErrorObjectOwned>(tx), @@ -108,39 +131,64 @@ pub(crate) fn create_rpc_module< } }, } - })?; - - rpc.register_blocking_method( - "citrea_sendRawDepositTransaction", - move |parameters, ctx, _| { - let mut params = parameters.sequence(); - let deposit: Bytes = params.next()?; + } - debug!("Sequencer: citrea_sendRawDepositTransaction"); + fn send_raw_deposit_transaction(&self, deposit: Bytes) -> RpcResult<()> { + debug!("Sequencer: citrea_sendRawDepositTransaction"); - let evm = Evm::::default(); - let mut working_set = WorkingSet::::new(ctx.storage.clone()); + let evm = Evm::::default(); + let mut working_set = WorkingSet::::new(self.context.storage.clone()); - let dep_tx = ctx - .deposit_mempool - .lock() - .make_deposit_tx_from_data(deposit.clone().into()); + let dep_tx = self + .context + .deposit_mempool + .lock() + .make_deposit_tx_from_data(deposit.clone().into()); - let tx_res = evm.get_call(dep_tx, None, None, None, &mut working_set); + let tx_res = evm.get_call(dep_tx, None, None, None, &mut working_set); - match tx_res { - Ok(hex_res) => { - tracing::debug!("Deposit tx processed successfully {}", hex_res); - ctx.deposit_mempool.lock().add_deposit_tx(deposit.to_vec()); - } - Err(e) => { - error!("Error processing deposit tx: {:?}", e); - return Err(e); - } + match tx_res { + Ok(hex_res) => { + tracing::debug!("Deposit tx processed successfully {}", hex_res); + self.context + .deposit_mempool + .lock() + .add_deposit_tx(deposit.to_vec()); + Ok(()) + } + Err(e) => { + error!("Error processing deposit tx: {:?}", e); + Err(e) } + } + } + + async fn publish_test_block(&self) -> RpcResult<()> { + if !self.context.test_mode { + return Err(ErrorObject::from(ErrorCode::MethodNotFound).to_owned()); + } + + debug!("Sequencer: citrea_testPublishBlock"); + self.context + .l2_force_block_tx + .unbounded_send(()) + .map_err(|e| { + ErrorObjectOwned::owned( + INTERNAL_ERROR_CODE, + INTERNAL_ERROR_MSG, + Some(format!("Could not send L2 force block transaction: {e}")), + ) + }) + } +} + +pub fn create_rpc_module< + C: sov_modules_api::Context, + DB: SequencerLedgerOps + Send + Sync + 'static, +>( + rpc_context: RpcContext, +) -> jsonrpsee::RpcModule> { + let server = SequencerRpcServerImpl::new(rpc_context); - Ok(()) - }, - )?; - Ok(rpc) + SequencerRpcServer::into_rpc(server) } diff --git a/crates/sequencer/src/sequencer.rs b/crates/sequencer/src/sequencer.rs index 8a0cd4c1f..2d2c1d0e1 100644 --- a/crates/sequencer/src/sequencer.rs +++ b/crates/sequencer/src/sequencer.rs @@ -758,7 +758,7 @@ where .filter_map( |mut blob| match DaDataBatchProof::try_from_slice(blob.full_data()) { Ok(da_data) - // we check on da pending txs of our wallet however let's keep consistency + // we check on da pending txs of our wallet however let's keep consistency if blob.sender().as_ref() == self.sequencer_da_pub_key.as_slice() => { match da_data { @@ -1072,7 +1072,7 @@ where mut rpc_methods: jsonrpsee::RpcModule<()>, ) -> Result, jsonrpsee::core::RegisterMethodError> { let rpc_context = self.create_rpc_context().await; - let rpc = create_rpc_module(rpc_context)?; + let rpc = create_rpc_module(rpc_context); rpc_methods.merge(rpc)?; Ok(rpc_methods) } From 07ec5784c4ba7b041f9524548d37592f41a74795 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ahmet=20Yaz=C4=B1c=C4=B1?= <75089142+yaziciahmet@users.noreply.github.com> Date: Fri, 4 Oct 2024 13:36:44 +0200 Subject: [PATCH 13/28] Improve estimate gas and create access list rpcs (#1265) * Remove unnecessary transact_to conversion * unwrap_or_default * More concise block_env initialization * Actually get highest gas limit from request and block env * One liner set * Allow unused * Cleanup prepare_call_env * Fix tests & lint * Replace unwrap * Set state to block in eth_call * Consume request in prepare_call_env * Lint * Set state before reading config * Remove unnecessary clone * Replace allow unused with feature native gate --- crates/evm/src/evm/call.rs | 82 +++++---- crates/evm/src/evm/handler.rs | 1 + crates/evm/src/query.rs | 284 ++++++++++++------------------- crates/evm/src/tests/tx_tests.rs | 4 +- 4 files changed, 160 insertions(+), 211 deletions(-) diff --git a/crates/evm/src/evm/call.rs b/crates/evm/src/evm/call.rs index 47e8929a3..21c9586c5 100644 --- a/crates/evm/src/evm/call.rs +++ b/crates/evm/src/evm/call.rs @@ -2,10 +2,10 @@ use std::cmp::min; -use reth_primitives::{TxKind, B256, U256}; +use reth_primitives::{B256, U256}; use reth_rpc_eth_types::error::{EthApiError, EthResult, RpcInvalidTransactionError}; use reth_rpc_types::TransactionRequest; -use revm::primitives::{TransactTo, TxEnv}; +use revm::primitives::{CfgEnvWithHandlerCfg, TxEnv}; use crate::caller_gas_allowance; use crate::primitive_types::BlockEnv; @@ -151,9 +151,7 @@ impl CallFees { } } -// https://github.com/paradigmxyz/reth/blob/d8677b4146f77c7c82d659c59b79b38caca78778/crates/rpc/rpc/src/eth/revm_utils.rs#L201 -// if from_balance is None, gas capping will not be applied -pub(crate) fn prepare_call_env( +pub(crate) fn create_txn_env( block_env: &BlockEnv, request: TransactionRequest, cap_to_balance: Option, @@ -172,21 +170,6 @@ pub(crate) fn prepare_call_env( chain_id, .. } = request; - let mut gas_price = gas_price.map(U256::from); - let mut max_fee_per_gas = max_fee_per_gas.map(U256::from); - let mut max_priority_fee_per_gas = max_priority_fee_per_gas.map(U256::from); - let gas = gas.map(U256::from); - - // TODO: write hardhat and unit tests for this - if max_fee_per_gas == Some(U256::ZERO) { - max_fee_per_gas = None; - } - if gas_price == Some(U256::ZERO) { - gas_price = None; - } - if max_priority_fee_per_gas == Some(U256::ZERO) { - max_priority_fee_per_gas = None; - } let CallFees { max_priority_fee_per_gas, @@ -194,9 +177,9 @@ pub(crate) fn prepare_call_env( // https://github.com/Sovereign-Labs/sovereign-sdk/issues/912 max_fee_per_blob_gas: _, } = CallFees::ensure_fees( - gas_price, - max_fee_per_gas, - max_priority_fee_per_gas, + gas_price.map(U256::from), + max_fee_per_gas.map(U256::from), + max_priority_fee_per_gas.map(U256::from), U256::from(block_env.basefee), // EIP-4844 related fields // https://github.com/Sovereign-Labs/sovereign-sdk/issues/912 @@ -205,9 +188,11 @@ pub(crate) fn prepare_call_env( None, )?; + // set gas limit initially to block gas limit let mut gas_limit = U256::from(block_env.gas_limit); + let request_gas_limit = gas.map(U256::from); - if let Some(request_gas_limit) = gas { + if let Some(request_gas_limit) = request_gas_limit { // gas limit is set by user in the request // we must make sure it does not exceed the block gas limit // otherwise there is a DoS vector @@ -227,15 +212,6 @@ pub(crate) fn prepare_call_env( gas_limit = min(gas_limit, max_gas_limit); } - let to = if let Some(kind) = to { - match kind { - TxKind::Create => TransactTo::Create, - TxKind::Call(address) => TransactTo::Call(address), - } - } else { - TransactTo::Create - }; - let env = TxEnv { gas_price, nonce, @@ -245,7 +221,7 @@ pub(crate) fn prepare_call_env( .map_err(|_| RpcInvalidTransactionError::GasUintOverflow)?, caller: from.unwrap_or_default(), gas_priority_fee: max_priority_fee_per_gas, - transact_to: to, + transact_to: to.unwrap_or_default(), value: value.unwrap_or_default(), data: input.try_into_unique_input()?.unwrap_or_default(), access_list: access_list.unwrap_or_default().to_vec(), @@ -259,6 +235,44 @@ pub(crate) fn prepare_call_env( Ok(env) } +// https://github.com/paradigmxyz/reth/blob/d8677b4146f77c7c82d659c59b79b38caca78778/crates/rpc/rpc/src/eth/revm_utils.rs#L201 +// if from_balance is None, gas capping will not be applied +pub(crate) fn prepare_call_env( + block_env: &BlockEnv, + cfg_env: &mut CfgEnvWithHandlerCfg, + mut request: TransactionRequest, + cap_to_balance: U256, +) -> EthResult { + // we want to disable this in eth_call, since this is common practice used by other node + // impls and providers + cfg_env.disable_block_gas_limit = true; + + // Disabled because eth_call is sometimes used with eoa senders + // See + cfg_env.disable_eip3607 = true; + + // The basefee should be ignored for eth_call + // See: + // + cfg_env.disable_base_fee = true; + + // set nonce to None so that the correct nonce is chosen by the EVM + request.nonce = None; + + // TODO: write hardhat and unit tests for this + if request.max_fee_per_gas == Some(0) { + request.max_fee_per_gas = None; + } + if request.gas_price == Some(0) { + request.gas_price = None; + } + if request.max_priority_fee_per_gas == Some(0) { + request.max_priority_fee_per_gas = None; + } + + create_txn_env(block_env, request.clone(), Some(cap_to_balance)) +} + #[cfg(test)] mod tests { use reth_primitives::constants::GWEI_TO_WEI; diff --git a/crates/evm/src/evm/handler.rs b/crates/evm/src/evm/handler.rs index 06a7712f4..ee63109f7 100644 --- a/crates/evm/src/evm/handler.rs +++ b/crates/evm/src/evm/handler.rs @@ -619,6 +619,7 @@ fn decrease_caller_balance( change_balance(context, amount, false, address) } +#[cfg(feature = "native")] pub(crate) fn diff_size_send_eth_eoa() -> usize { DB_ACCOUNT_KEY_SIZE * ACCOUNT_DISCOUNTED_PERCENTAGE / 100 + (DB_ACCOUNT_SIZE_EOA + DB_ACCOUNT_KEY_SIZE) * NONCE_DISCOUNTED_PERCENTAGE / 100 diff --git a/crates/evm/src/query.rs b/crates/evm/src/query.rs index a8e71be60..e64db104c 100644 --- a/crates/evm/src/query.rs +++ b/crates/evm/src/query.rs @@ -20,7 +20,6 @@ use reth_rpc_types::{ use reth_rpc_types_compat::block::from_primitive_with_hash; use revm::primitives::{ CfgEnvWithHandlerCfg, EVMError, ExecutionResult, HaltReason, InvalidTransaction, TransactTo, - TxEnv, }; use revm::{Database, DatabaseCommit}; use revm_inspectors::access_list::AccessListInspector; @@ -33,7 +32,7 @@ use sov_modules_api::WorkingSet; use crate::call::get_cfg_env; use crate::conversions::create_tx_env; use crate::error::rpc::ensure_success; -use crate::evm::call::prepare_call_env; +use crate::evm::call::{create_txn_env, prepare_call_env}; use crate::evm::db::EvmDb; use crate::evm::primitive_types::{BlockEnv, Receipt, SealedBlock, TransactionSignedAndRecovered}; use crate::evm::DbAccount; @@ -500,81 +499,46 @@ impl Evm { _block_overrides: Option>, working_set: &mut WorkingSet, ) -> RpcResult { - let mut block_env = match block_id { - Some(BlockId::Number(block_num)) => match block_num { - BlockNumberOrTag::Pending | BlockNumberOrTag::Latest => BlockEnv::from( - &self - .get_sealed_block_by_number(Some(BlockNumberOrTag::Latest), working_set) - .unwrap() - .expect("Genesis block must be set"), - ), - _ => { - let block = - match self.get_sealed_block_by_number(Some(block_num), working_set)? { - Some(block) => block, - None => return Err(EthApiError::UnknownBlockNumber.into()), - }; - - set_state_to_end_of_evm_block(block.header.number, working_set); - - BlockEnv::from(&block) - } - }, + let block_number = match block_id { + Some(BlockId::Number(block_num)) => block_num, Some(BlockId::Hash(block_hash)) => { let block_number = self .get_block_number_by_block_hash(block_hash.block_hash, working_set) .ok_or_else(|| EthApiError::UnknownBlockOrTxIndex)?; - - let block_env = BlockEnv::from( - &self - .get_sealed_block_by_number( - Some(BlockNumberOrTag::Number(block_number)), - working_set, - ) - .unwrap() - .expect("Block must be set"), - ); - - set_state_to_end_of_evm_block(block_number, working_set); - - block_env + BlockNumberOrTag::Number(block_number) } - None => BlockEnv::from( - &self - .get_sealed_block_by_number(Some(BlockNumberOrTag::Latest), working_set) - .unwrap() - .expect("Genesis block must be set"), - ), + None => BlockNumberOrTag::Latest, }; - let cfg = self - .cfg - .get(working_set) - .expect("EVM chain config should be set"); - let mut cfg_env = get_cfg_env(&block_env, cfg); + let (block_env, mut cfg_env) = { + let block = self + .get_sealed_block_by_number(Some(block_number), working_set)? + .ok_or(EthApiError::UnknownBlockNumber)?; + let block_env = BlockEnv::from(&block); - // set endpoint specific params - cfg_env.disable_eip3607 = true; - cfg_env.disable_base_fee = true; - // set higher block gas limit than usual - // but still cap it to prevent DoS - block_env.gas_limit = 100_000_000; + // Set evm state to block if needed + match block_number { + BlockNumberOrTag::Pending | BlockNumberOrTag::Latest => {} + _ => set_state_to_end_of_evm_block(block_env.number, working_set), + }; + + let cfg = self + .cfg + .get(working_set) + .expect("EVM chain config should be set"); + let cfg_env = get_cfg_env(&block_env, cfg); + + (block_env, cfg_env) + }; let mut evm_db = self.get_db(working_set); - let mut tx_env = prepare_call_env( - &block_env, - request.clone(), - Some( - evm_db - .basic(request.from.unwrap_or_default()) - .unwrap() - .unwrap_or_default() - .balance, - ), - )?; - // https://github.com/paradigmxyz/reth/issues/6574 - tx_env.nonce = None; + let cap_to_balance = evm_db + .basic(request.from.unwrap_or_default()) + .map_err(EthApiError::from)? + .unwrap_or_default() + .balance; + let tx_env = prepare_call_env(&block_env, &mut cfg_env, request, cap_to_balance)?; let result = match inspect( evm_db, @@ -616,59 +580,49 @@ impl Evm { ) -> RpcResult { let mut request = request.clone(); - let (l1_fee_rate, mut block_env, block_num) = { - let block = match self.get_sealed_block_by_number(block_number, working_set)? { - Some(block) => block, - None => return Err(EthApiError::UnknownBlockNumber.into()), - }; - let l1_fee_rate = block.l1_fee_rate; + let (l1_fee_rate, block_env, mut cfg_env) = { + let block = self + .get_sealed_block_by_number(block_number, working_set)? + .ok_or(EthApiError::UnknownBlockNumber)?; let block_env = BlockEnv::from(&block); - (l1_fee_rate, block_env, block.header.number) - }; + match block_number { + None | Some(BlockNumberOrTag::Pending | BlockNumberOrTag::Latest) => {} + _ => set_state_to_end_of_evm_block(block_env.number, working_set), + }; - match block_number { - None | Some(BlockNumberOrTag::Pending | BlockNumberOrTag::Latest) => {} - _ => { - set_state_to_end_of_evm_block(block_num, working_set); - } + let cfg = self + .cfg + .get(working_set) + .expect("EVM chain config should be set"); + let cfg_env = get_cfg_env(&block_env, cfg); + + (block.l1_fee_rate, block_env, cfg_env) }; - let cfg = self - .cfg - .get(working_set) - .expect("EVM chain config should be set"); - let mut cfg_env = get_cfg_env(&block_env, cfg); + // we want to disable this in eth_createAccessList, since this is common practice used by + // other node impls and providers + cfg_env.disable_block_gas_limit = true; - // set endpoint specific params - cfg_env.disable_eip3607 = true; + // The basefee should be ignored for eth_createAccessList + // See: + // cfg_env.disable_base_fee = true; - // set higher block gas limit than usual - // but still cap it to prevent DoS - block_env.gas_limit = 100_000_000; let mut evm_db = self.get_db(working_set); - let mut tx_env = prepare_call_env( - &block_env, - request.clone(), - Some( - evm_db - .basic(request.from.unwrap_or_default()) - .unwrap() - .unwrap_or_default() - .balance, - ), - )?; - let from = request.from.unwrap_or_default(); + let account = evm_db + .basic(from) + .map_err(EthApiError::from)? + .unwrap_or_default(); + + let tx_env = create_txn_env(&block_env, request.clone(), Some(account.balance))?; + let to = if let Some(Call(to)) = request.to { to } else { - let account = evm_db.basic(from).unwrap(); - - let nonce = account.unwrap_or_default().nonce; - from.create(nonce) + from.create(account.nonce) }; // can consume the list since we're not using the request anymore @@ -681,7 +635,7 @@ impl Evm { &mut evm_db, cfg_env.clone(), block_env, - tx_env.clone(), + tx_env, &mut inspector, ) .map_err(EthApiError::from)?; @@ -700,16 +654,9 @@ impl Evm { let access_list = inspector.into_access_list(); request.access_list = Some(access_list.clone()); - tx_env.access_list = access_list.to_vec(); - let estimated = self.estimate_gas_with_env( - request, - l1_fee_rate, - block_env, - cfg_env, - &mut tx_env, - working_set, - )?; + let estimated = + self.estimate_gas_with_env(request, l1_fee_rate, block_env, cfg_env, working_set)?; Ok(AccessListWithGasUsed { access_list, @@ -725,44 +672,27 @@ impl Evm { block_number: Option, working_set: &mut WorkingSet, ) -> RpcResult { - let (l1_fee_rate, block_env, block_num) = { - let block = match self.get_sealed_block_by_number(block_number, working_set)? { - Some(block) => block, - None => return Err(EthApiError::UnknownBlockNumber.into()), - }; - let l1_fee_rate = block.l1_fee_rate; + let (l1_fee_rate, block_env, cfg_env) = { + let block = self + .get_sealed_block_by_number(block_number, working_set)? + .ok_or(EthApiError::UnknownBlockNumber)?; let block_env = BlockEnv::from(&block); - (l1_fee_rate, block_env, block.header.number) - }; - - match block_number { - None | Some(BlockNumberOrTag::Pending | BlockNumberOrTag::Latest) => {} - _ => { - set_state_to_end_of_evm_block(block_num, working_set); - } - }; - - let mut tx_env = prepare_call_env(&block_env, request.clone(), None)?; + match block_number { + None | Some(BlockNumberOrTag::Pending | BlockNumberOrTag::Latest) => {} + _ => set_state_to_end_of_evm_block(block_env.number, working_set), + }; - let cfg = self - .cfg - .get(working_set) - .expect("EVM chain config should be set"); - let mut cfg_env = get_cfg_env(&block_env, cfg); + let cfg = self + .cfg + .get(working_set) + .expect("EVM chain config should be set"); + let cfg_env = get_cfg_env(&block_env, cfg); - // set endpoint specific params - cfg_env.disable_eip3607 = true; - cfg_env.disable_base_fee = true; + (block.l1_fee_rate, block_env, cfg_env) + }; - self.estimate_gas_with_env( - request, - l1_fee_rate, - block_env, - cfg_env, - &mut tx_env, - working_set, - ) + self.estimate_gas_with_env(request, l1_fee_rate, block_env, cfg_env, working_set) } /// Handler for: `eth_estimateGas` @@ -831,27 +761,37 @@ impl Evm { /// Inner gas estimator pub(crate) fn estimate_gas_with_env( &self, - request: reth_rpc_types::TransactionRequest, + mut request: reth_rpc_types::TransactionRequest, l1_fee_rate: u128, block_env: BlockEnv, - cfg_env: CfgEnvWithHandlerCfg, - tx_env: &mut TxEnv, + mut cfg_env: CfgEnvWithHandlerCfg, working_set: &mut WorkingSet, ) -> RpcResult { - let request_gas = request.gas; - let request_gas_price = request.gas_price; - let env_gas_limit = block_env.gas_limit.into(); - let env_base_fee = U256::from(block_env.basefee); + // Disabled because eth_estimateGas is sometimes used with eoa senders + // See + cfg_env.disable_eip3607 = true; - // get the highest possible gas limit, either the request's set value or the currently - // configured gas limit - let mut highest_gas_limit = request.gas.unwrap_or(env_gas_limit); + // The basefee should be ignored for eth_estimateGas and similar + // See: + // + cfg_env.disable_base_fee = true; + + // set nonce to None so that the correct nonce is chosen by the EVM + request.nonce = None; + + let request_gas_limit = request.gas; + let request_gas_price = request.gas_price; + let block_env_gas_limit = block_env.gas_limit.into(); + let block_env_base_fee = U256::from(block_env.basefee); let account = self .accounts - .get(&tx_env.caller, working_set) + .get(&request.from.unwrap_or_default(), working_set) .unwrap_or_default(); + // create tx env + let mut tx_env = create_txn_env(&block_env, request, Some(account.balance))?; + // if the request is a simple transfer we can optimize if tx_env.data.is_empty() { if let TransactTo::Call(to) = tx_env.transact_to { @@ -894,7 +834,7 @@ impl Evm { } return Ok(EstimatedTxExpenses { gas_used: U64::from(MIN_TRANSACTION_GAS), - base_fee: env_base_fee, + base_fee: block_env_base_fee, l1_fee, l1_diff_size: diff_size, }); @@ -904,20 +844,14 @@ impl Evm { } } - // check funds of the sender - if tx_env.gas_price > U256::ZERO { - // allowance is (balance - tx.value) / tx.gas_price - let allowance = ((account.balance - tx_env.value) / tx_env.gas_price).saturating_to(); - - if highest_gas_limit > allowance { - // cap the highest gas limit by max gas caller can afford with given gas price - highest_gas_limit = allowance; - } - } + // get the highest possible gas limit, either the request's set value or the currently + // configured gas limit + let highest_gas_limit = request_gas_limit + .map(|req_gas_limit| req_gas_limit.max(block_env_gas_limit)) + .unwrap_or(block_env_gas_limit); // if the provided gas limit is less than computed cap, use that - let gas_limit: u64 = std::cmp::min(tx_env.gas_limit, highest_gas_limit as u64); // highest_gas_limit is capped to u64::MAX - tx_env.gas_limit = gas_limit; + tx_env.gas_limit = std::cmp::min(tx_env.gas_limit, highest_gas_limit as u64); // highest_gas_limit is capped to u64::MAX let evm_db = self.get_db(working_set); @@ -936,7 +870,7 @@ impl Evm { { // if price or limit was included in the request then we can execute the request // again with the block's gas limit to check if revert is gas related or not - if request_gas.is_some() || request_gas_price.is_some() { + if request_gas_limit.is_some() || request_gas_price.is_some() { let evm_db = self.get_db(working_set); return Err(map_out_of_gas_err( block_env, @@ -960,7 +894,7 @@ impl Evm { ExecutionResult::Revert { output, .. } => { // if price or limit was included in the request then we can execute the request // again with the block's gas limit to check if revert is gas related or not - return if request_gas.is_some() || request_gas_price.is_some() { + return if request_gas_limit.is_some() || request_gas_price.is_some() { let evm_db = self.get_db(working_set); Err(map_out_of_gas_err( block_env, @@ -1081,7 +1015,7 @@ impl Evm { Ok(EstimatedTxExpenses { gas_used: U64::from(highest_gas_limit), - base_fee: env_base_fee, + base_fee: block_env_base_fee, l1_fee, l1_diff_size: diff_size, }) diff --git a/crates/evm/src/tests/tx_tests.rs b/crates/evm/src/tests/tx_tests.rs index e7ac1ab4c..53e3d78a5 100644 --- a/crates/evm/src/tests/tx_tests.rs +++ b/crates/evm/src/tests/tx_tests.rs @@ -11,7 +11,7 @@ use reth_primitives::{ use reth_rpc_types::request::{TransactionInput, TransactionRequest}; use revm::primitives::{TransactTo, TxEnv}; -use crate::evm::call::prepare_call_env; +use crate::evm::call::create_txn_env; use crate::evm::primitive_types::TransactionSignedAndRecovered; use crate::primitive_types::{Block, BlockEnv}; use crate::tests::DEFAULT_CHAIN_ID; @@ -108,7 +108,7 @@ fn prepare_call_env_conversion() { let block_env = BlockEnv::default(); - let tx_env = prepare_call_env(&block_env, request, None).unwrap(); + let tx_env = create_txn_env(&block_env, request, None).unwrap(); let expected = TxEnv { caller: from, gas_price: U256::from(100u64), From af745eb81afd5744bdf2854d239d5e0eec4affa6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ege=20Okan=20=C3=9Cnald=C4=B1?= <35339130+exeokan@users.noreply.github.com> Date: Sat, 5 Oct 2024 17:17:14 +0300 Subject: [PATCH 14/28] Add sys txs to evm tests (#1255) * modify config_push_contracts * modify call_multiple_test * modify tests in call_tests.rs * moving common functions to utils.rs * minor fixes * updated lock files * modify tests --------- Co-authored-by: Esad Yusuf Atik --- Cargo.lock | 1 + crates/evm/Cargo.toml | 1 + crates/evm/src/tests/call_tests.rs | 702 ++++++++++++------ .../tests/ef_tests/cases/blockchain_test.rs | 4 +- crates/evm/src/tests/genesis_tests.rs | 67 +- crates/evm/src/tests/hooks_tests.rs | 80 +- crates/evm/src/tests/queries/basic_queries.rs | 284 +++---- crates/evm/src/tests/queries/log_tests.rs | 19 +- crates/evm/src/tests/queries/mod.rs | 20 +- crates/evm/src/tests/sys_tx_tests.rs | 73 +- crates/evm/src/tests/utils.rs | 199 ++++- 11 files changed, 932 insertions(+), 518 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d4d91689f..e9f0153a5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1804,6 +1804,7 @@ dependencies = [ "sov-prover-storage-manager", "sov-rollup-interface", "sov-state", + "sov-stf-runner", "tempfile", "thiserror", "tracing", diff --git a/crates/evm/Cargo.toml b/crates/evm/Cargo.toml index 641a73007..e384051b4 100644 --- a/crates/evm/Cargo.toml +++ b/crates/evm/Cargo.toml @@ -59,6 +59,7 @@ revm = { workspace = true, features = ["optional_block_gas_limit", "optional_eip sov-modules-api = { path = "../sovereign-sdk/module-system/sov-modules-api", features = ["macros"] } sov-prover-storage-manager = { path = "../sovereign-sdk/full-node/sov-prover-storage-manager", features = ["test-utils"] } sov-rollup-interface = { path = "../sovereign-sdk/rollup-interface", features = ["testing"] } +sov-stf-runner = { path = "../sovereign-sdk/full-node/sov-stf-runner" } tempfile = { workspace = true } tracing-subscriber = { workspace = true } walkdir = "2.3.3" diff --git a/crates/evm/src/tests/call_tests.rs b/crates/evm/src/tests/call_tests.rs index 878643ca0..19693572a 100644 --- a/crates/evm/src/tests/call_tests.rs +++ b/crates/evm/src/tests/call_tests.rs @@ -1,9 +1,9 @@ use std::str::FromStr; use reth_primitives::constants::ETHEREUM_BLOCK_GAS_LIMIT; -use reth_primitives::{address, Address, BlockNumberOrTag, Bytes, TxKind, U64}; +use reth_primitives::{address, b256, Address, BlockNumberOrTag, Bytes, Log, LogData, TxKind, U64}; use reth_rpc_types::request::{TransactionInput, TransactionRequest}; -use revm::primitives::{SpecId, KECCAK_EMPTY, U256}; +use revm::primitives::{hex, SpecId, KECCAK_EMPTY, U256}; use sov_modules_api::default_context::DefaultContext; use sov_modules_api::hooks::HookSoftConfirmationInfo; use sov_modules_api::utils::generate_address; @@ -19,19 +19,22 @@ use crate::smart_contracts::{ SimpleStorageContract, TestContract, }; use crate::tests::test_signer::TestSigner; -use crate::tests::utils::get_evm; +use crate::tests::utils::{ + config_push_contracts, create_contract_message, create_contract_message_with_fee, + create_contract_transaction, get_evm, get_evm_config, get_evm_config_starting_base_fee, + publish_event_message, set_arg_message, +}; use crate::tests::DEFAULT_CHAIN_ID; use crate::{ AccountData, EvmConfig, RlpEvmTransaction, BASE_FEE_VAULT, L1_FEE_VAULT, PRIORITY_FEE_VAULT, }; - type C = DefaultContext; #[test] fn call_multiple_test() { let dev_signer1: TestSigner = TestSigner::new_random(); - let config = EvmConfig { + let mut config = EvmConfig { data: vec![AccountData { address: dev_signer1.address(), balance: U256::from_str("100000000000000000000").unwrap(), @@ -45,7 +48,7 @@ fn call_multiple_test() { spec: vec![(0, SpecId::SHANGHAI)].into_iter().collect(), ..Default::default() }; - + config_push_contracts(&mut config, None); let (mut evm, mut working_set) = get_evm(&config); let contract_addr = address!("819c5497b157177315e1204f52e588b393771719"); @@ -121,7 +124,6 @@ fn call_multiple_test() { .storage .get(&U256::ZERO, &mut working_set) .unwrap(); - assert_eq!(U256::from(set_arg + 3), storage_value); assert_eq!( @@ -132,50 +134,80 @@ fn call_multiple_test() { Receipt { receipt: reth_primitives::Receipt { tx_type: reth_primitives::TxType::Eip1559, - success: true, - cumulative_gas_used: 132943, - logs: vec![], + success: true, cumulative_gas_used: 50751, + logs: vec![] }, - gas_used: 132943, - log_index_start: 0, - l1_diff_size: 567, + gas_used: 50751, + log_index_start: 0, + l1_diff_size: 255 }, Receipt { receipt: reth_primitives::Receipt { tx_type: reth_primitives::TxType::Eip1559, success: true, - cumulative_gas_used: 176673, - logs: vec![], + cumulative_gas_used: 131371, + logs: vec![ + Log { + address: address!("3100000000000000000000000000000000000001"), + data: LogData::new( + vec![b256!("32eff959e2e8d1609edc4b39ccf75900aa6c1da5719f8432752963fdf008234f")], + Bytes::from_static(&hex!("000000000000000000000000000000000000000000000000000000000000000101010101010101010101010101010101010101010101010101010101010101010202020202020202020202020202020202020202020202020202020202020202")), + ).unwrap() + } + ]}, + gas_used: 80620, + log_index_start: 0, + l1_diff_size: 561 }, - gas_used: 43730, - log_index_start: 0, - l1_diff_size: 255, - }, - Receipt { - receipt: reth_primitives::Receipt { + Receipt { + receipt: reth_primitives::Receipt{ tx_type: reth_primitives::TxType::Eip1559, success: true, - cumulative_gas_used: 203303, - logs: vec![], + cumulative_gas_used: 300521, + logs: vec![ + Log { + address: address!("3100000000000000000000000000000000000002"), + data: LogData::new( + vec![b256!("fbe5b6cbafb274f445d7fed869dc77a838d8243a22c460de156560e8857cad03")], + Bytes::from_static(&hex!("0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000deaddeaddeaddeaddeaddeaddeaddeaddeaddead")), + ).unwrap() + }, + Log { + address: address!("3100000000000000000000000000000000000002"), + data: LogData::new( + vec![b256!("80bd1fdfe157286ce420ee763f91748455b249605748e5df12dad9844402bafc")], + Bytes::from_static(&hex!("000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000a0000000000000000000000000000000000000000000000000000000000000002d4a209fb3a961d8b1f4ec1caa220c6a50b815febc0b689ddf0b9ddfbf99cb74479e41ac0063066369747265611400000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000a08000000003b9aca006800000000000000000000000000000000000000000000")), + ).unwrap() + } + ] }, - gas_used: 26630, - log_index_start: 0, - l1_diff_size: 255, - }, - Receipt { - receipt: reth_primitives::Receipt { - tx_type: reth_primitives::TxType::Eip1559, - success: true, - cumulative_gas_used: 229933, - logs: vec![], + gas_used: 169150, + log_index_start: 1, + l1_diff_size: 1019 }, - gas_used: 26630, - log_index_start: 0, - l1_diff_size: 255, - } - ] + Receipt { + receipt: reth_primitives::Receipt { + tx_type: reth_primitives::TxType::Eip1559, + success: true, + cumulative_gas_used: 80620, + logs: vec![Log { + address: address!("3100000000000000000000000000000000000001"), + data: LogData::new( + vec![b256!("32eff959e2e8d1609edc4b39ccf75900aa6c1da5719f8432752963fdf008234f")], + Bytes::from_static(&hex!("000000000000000000000000000000000000000000000000000000000000000205050505050505050505050505050505050505050505050505050505050505052a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a")) + ).unwrap() + }] + }, + gas_used: 80620, + log_index_start: 0, + l1_diff_size: 561 + }, + Receipt { receipt: reth_primitives::Receipt { tx_type: reth_primitives::TxType::Eip1559, success: true, cumulative_gas_used: 213563, logs: vec![] }, gas_used: 132943, log_index_start: 1, l1_diff_size: 567 }, + Receipt { receipt: reth_primitives::Receipt { tx_type: reth_primitives::TxType::Eip1559, success: true, cumulative_gas_used: 257293, logs: vec![] }, gas_used: 43730, log_index_start: 1, l1_diff_size: 255 }, + Receipt { receipt: reth_primitives::Receipt { tx_type: reth_primitives::TxType::Eip1559, success: true, cumulative_gas_used: 283923, logs: vec![] }, gas_used: 26630, log_index_start: 1, l1_diff_size: 255 }, + Receipt { receipt: reth_primitives::Receipt { tx_type: reth_primitives::TxType::Eip1559, success: true, cumulative_gas_used: 310553, logs: vec![] }, + gas_used: 26630, log_index_start: 1, l1_diff_size: 255 }] ); - // checkout esad/fix-block-env-bug branch let tx = evm .get_transaction_by_block_number_and_index( @@ -255,32 +287,109 @@ fn call_test() { receipt: reth_primitives::Receipt { tx_type: reth_primitives::TxType::Eip1559, success: true, - cumulative_gas_used: 132943, - logs: vec![], + cumulative_gas_used: 50751, + logs: vec![] }, - gas_used: 132943, + gas_used: 50751, log_index_start: 0, - l1_diff_size: 567, + l1_diff_size: 255 }, Receipt { receipt: reth_primitives::Receipt { tx_type: reth_primitives::TxType::Eip1559, success: true, - cumulative_gas_used: 176673, - logs: vec![], + cumulative_gas_used: 131371, + logs: vec![ + Log { + address: address!("3100000000000000000000000000000000000001"), + data: LogData::new( + vec![b256!("32eff959e2e8d1609edc4b39ccf75900aa6c1da5719f8432752963fdf008234f")], + Bytes::from_static(&hex!("000000000000000000000000000000000000000000000000000000000000000101010101010101010101010101010101010101010101010101010101010101010202020202020202020202020202020202020202020202020202020202020202")) + ).unwrap() + } + ] }, - gas_used: 43730, + gas_used: 80620, log_index_start: 0, - l1_diff_size: 255, - } - ] - ) + l1_diff_size: 561 + }, + Receipt { + receipt: reth_primitives::Receipt { + tx_type: reth_primitives::TxType::Eip1559, + success: true, + cumulative_gas_used: 300521, + logs: vec![ + Log { + address: address!("3100000000000000000000000000000000000002"), + data: LogData::new( + vec![b256!("fbe5b6cbafb274f445d7fed869dc77a838d8243a22c460de156560e8857cad03")], + Bytes::from_static(&hex!("0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000deaddeaddeaddeaddeaddeaddeaddeaddeaddead")), + ).unwrap() + }, + Log { + address: address!("3100000000000000000000000000000000000002"), + data: LogData::new( + vec![b256!("80bd1fdfe157286ce420ee763f91748455b249605748e5df12dad9844402bafc")], + Bytes::from_static(&hex!("000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000a0000000000000000000000000000000000000000000000000000000000000002d4a209fb3a961d8b1f4ec1caa220c6a50b815febc0b689ddf0b9ddfbf99cb74479e41ac0063066369747265611400000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000a08000000003b9aca006800000000000000000000000000000000000000000000")) + ).unwrap() + } + ] + }, + gas_used: 169150, + log_index_start: 1, + l1_diff_size: 1019 + }, + Receipt { + receipt: reth_primitives::Receipt { + tx_type: reth_primitives::TxType::Eip1559, + success: true, + cumulative_gas_used: 80620, + logs: vec![ + Log { + address: address!("3100000000000000000000000000000000000001"), + data: LogData::new( + vec![b256!("32eff959e2e8d1609edc4b39ccf75900aa6c1da5719f8432752963fdf008234f")], + Bytes::from_static(&hex!("000000000000000000000000000000000000000000000000000000000000000205050505050505050505050505050505050505050505050505050505050505052a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a")) + ).unwrap() + } + ] + }, + gas_used: 80620, + log_index_start: 0, + l1_diff_size: 561 + }, + Receipt { + receipt: reth_primitives::Receipt { + tx_type: reth_primitives::TxType::Eip1559, + success: true, + cumulative_gas_used: 213563, + logs: vec![] + }, + gas_used: 132943, + log_index_start: 1, + l1_diff_size: 567 + }, + Receipt { + receipt: reth_primitives::Receipt { + tx_type: reth_primitives::TxType::Eip1559, + success: true, + cumulative_gas_used: 257293, + logs: vec![] + }, + gas_used: 43730, + log_index_start: 1, + l1_diff_size: 255 + }] + ); } #[test] fn failed_transaction_test() { let dev_signer: TestSigner = TestSigner::new_random(); - let (mut evm, mut working_set) = get_evm(&EvmConfig::default()); + let mut config = EvmConfig::default(); + config_push_contracts(&mut config, None); + + let (mut evm, mut working_set) = get_evm(&config); let working_set = &mut working_set; let l1_fee_rate = 0; let l2_height = 2; @@ -321,20 +430,98 @@ fn failed_transaction_test() { evm.call(call_message, &context, working_set).unwrap(); } - // assert no pending transaction + // assert one pending transaction (system transaction) let pending_txs = &evm.pending_transactions; - assert_eq!(pending_txs.len(), 0); + assert_eq!(pending_txs.len(), 1); evm.end_soft_confirmation_hook(&soft_confirmation_info, working_set); - // assert no pending transaction let pending_txs = &evm.pending_transactions; assert_eq!(pending_txs.len(), 0); - // Assert block does not have any transaction + assert_eq!( + evm.receipts + .iter(&mut working_set.accessory_state()) + .collect::>(), + [ + Receipt { + receipt: reth_primitives::Receipt { + tx_type: reth_primitives::TxType::Eip1559, + success: true, + cumulative_gas_used: 50751, + logs: vec![] + }, + gas_used: 50751, + log_index_start: 0, + l1_diff_size: 255 + }, + Receipt { + receipt: reth_primitives::Receipt{ + tx_type: reth_primitives::TxType::Eip1559, + success: true, + cumulative_gas_used: 131371, + logs: vec![ + Log { + address: address!("3100000000000000000000000000000000000001"), + data: LogData::new( + vec![b256!("32eff959e2e8d1609edc4b39ccf75900aa6c1da5719f8432752963fdf008234f")], + Bytes::from_static(&hex!("000000000000000000000000000000000000000000000000000000000000000101010101010101010101010101010101010101010101010101010101010101010202020202020202020202020202020202020202020202020202020202020202")) + ).unwrap() + } + ] + }, + gas_used: 80620, + log_index_start: 0, + l1_diff_size: 561 + }, + Receipt { + receipt: reth_primitives::Receipt { + tx_type: reth_primitives::TxType::Eip1559, + success: true, cumulative_gas_used: 300521, + logs: vec![ + Log { + address: address!("3100000000000000000000000000000000000002"), + data: LogData::new( + vec![b256!("fbe5b6cbafb274f445d7fed869dc77a838d8243a22c460de156560e8857cad03")], + Bytes::from_static(&hex!("0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000deaddeaddeaddeaddeaddeaddeaddeaddeaddead")) + ).unwrap() + }, + Log { + address: address!("3100000000000000000000000000000000000002"), + data: LogData::new( + vec![b256!("80bd1fdfe157286ce420ee763f91748455b249605748e5df12dad9844402bafc")], + Bytes::from_static(&hex!("000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000a0000000000000000000000000000000000000000000000000000000000000002d4a209fb3a961d8b1f4ec1caa220c6a50b815febc0b689ddf0b9ddfbf99cb74479e41ac0063066369747265611400000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000a08000000003b9aca006800000000000000000000000000000000000000000000")) + ).unwrap() + }] + }, + gas_used: 169150, + log_index_start: 1, + l1_diff_size: 1019 + }, + Receipt { + receipt: reth_primitives::Receipt { + tx_type: reth_primitives::TxType::Eip1559, + success: true, + cumulative_gas_used: 80620, + logs: vec![ + Log { + address: address!("3100000000000000000000000000000000000001"), + data: LogData::new( + vec![b256!("32eff959e2e8d1609edc4b39ccf75900aa6c1da5719f8432752963fdf008234f")], + Bytes::from_static(&hex!("000000000000000000000000000000000000000000000000000000000000000205050505050505050505050505050505050505050505050505050505050505052a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a")) + ).unwrap() + } + ] + }, + gas_used: 80620, + log_index_start: 0, + l1_diff_size: 561 + } + ] + ); let block = evm.blocks.last(&mut working_set.accessory_state()).unwrap(); assert_eq!(block.transactions.start, 0); - assert_eq!(block.transactions.end, 0); + assert_eq!(block.transactions.end, 3); } #[test] @@ -346,6 +533,7 @@ fn self_destruct_test() { let (config, dev_signer, contract_addr) = get_evm_config(U256::from_str("100000000000000000000").unwrap(), None); + let (mut evm, mut working_set) = get_evm(&config); let l1_fee_rate = 0; let mut l2_height = 2; @@ -691,33 +879,6 @@ fn test_block_gas_limit() { ); } -pub fn create_contract_message( - dev_signer: &TestSigner, - nonce: u64, - contract: T, -) -> RlpEvmTransaction { - dev_signer - .sign_default_transaction(TxKind::Create, contract.byte_code(), nonce, 0) - .unwrap() -} - -pub(crate) fn create_contract_message_with_fee( - dev_signer: &TestSigner, - nonce: u64, - contract: T, - max_fee_per_gas: u128, -) -> RlpEvmTransaction { - dev_signer - .sign_default_transaction_with_fee( - TxKind::Create, - contract.byte_code(), - nonce, - 0, - max_fee_per_gas, - ) - .unwrap() -} - pub(crate) fn create_contract_message_with_priority_fee( dev_signer: &TestSigner, nonce: u64, @@ -737,16 +898,6 @@ pub(crate) fn create_contract_message_with_priority_fee( .unwrap() } -pub(crate) fn create_contract_transaction( - dev_signer: &TestSigner, - nonce: u64, - contract: T, -) -> RlpEvmTransaction { - dev_signer - .sign_default_transaction(TxKind::Create, contract.byte_code(), nonce, 0) - .unwrap() -} - fn set_selfdestruct_arg_message( contract_addr: Address, dev_signer: &TestSigner, @@ -765,24 +916,6 @@ fn set_selfdestruct_arg_message( .unwrap() } -pub(crate) fn set_arg_message( - contract_addr: Address, - dev_signer: &TestSigner, - nonce: u64, - set_arg: u32, -) -> RlpEvmTransaction { - let contract = SimpleStorageContract::default(); - - dev_signer - .sign_default_transaction( - TxKind::Call(contract_addr), - contract.set_call_data(set_arg), - nonce, - 0, - ) - .unwrap() -} - fn set_arg_transaction( contract_addr: Address, dev_signer: &TestSigner, @@ -830,73 +963,6 @@ fn selfdestruct_message( .unwrap() } -pub(crate) fn publish_event_message( - contract_addr: Address, - signer: &TestSigner, - nonce: u64, - message: String, -) -> RlpEvmTransaction { - let contract = LogsContract::default(); - - signer - .sign_default_transaction( - TxKind::Call(contract_addr), - contract.publish_event(message), - nonce, - 0, - ) - .unwrap() -} - -pub(crate) fn get_evm_config( - signer_balance: U256, - block_gas_limit: Option, -) -> (EvmConfig, TestSigner, Address) { - let dev_signer: TestSigner = TestSigner::new_random(); - - let contract_addr = address!("819c5497b157177315e1204f52e588b393771719"); - let config = EvmConfig { - data: vec![AccountData { - address: dev_signer.address(), - balance: signer_balance, - code_hash: KECCAK_EMPTY, - code: Bytes::default(), - nonce: 0, - storage: Default::default(), - }], - spec: vec![(0, SpecId::SHANGHAI)].into_iter().collect(), - block_gas_limit: block_gas_limit.unwrap_or(ETHEREUM_BLOCK_GAS_LIMIT), - ..Default::default() - }; - (config, dev_signer, contract_addr) -} - -pub(crate) fn get_evm_config_starting_base_fee( - signer_balance: U256, - block_gas_limit: Option, - starting_base_fee: u64, -) -> (EvmConfig, TestSigner, Address) { - let dev_signer: TestSigner = TestSigner::new_random(); - - let contract_addr = address!("819c5497b157177315e1204f52e588b393771719"); - let config = EvmConfig { - data: vec![AccountData { - address: dev_signer.address(), - balance: signer_balance, - code_hash: KECCAK_EMPTY, - code: Bytes::default(), - nonce: 0, - storage: Default::default(), - }], - spec: vec![(0, SpecId::SHANGHAI)].into_iter().collect(), - block_gas_limit: block_gas_limit.unwrap_or(ETHEREUM_BLOCK_GAS_LIMIT), - starting_base_fee, - coinbase: PRIORITY_FEE_VAULT, - ..Default::default() - }; - (config, dev_signer, contract_addr) -} - #[test] fn test_l1_fee_success() { fn run_tx( @@ -961,8 +1027,8 @@ fn test_l1_fee_success() { .get(&dev_signer.address(), &mut working_set) .unwrap(); - let base_fee_valut = evm.accounts.get(&BASE_FEE_VAULT, &mut working_set).unwrap(); - let l1_fee_valut = evm.accounts.get(&L1_FEE_VAULT, &mut working_set).unwrap(); + let base_fee_vault = evm.accounts.get(&BASE_FEE_VAULT, &mut working_set).unwrap(); + let l1_fee_vault = evm.accounts.get(&L1_FEE_VAULT, &mut working_set).unwrap(); let coinbase_account = evm .accounts @@ -971,25 +1037,100 @@ fn test_l1_fee_success() { assert_eq!(config.coinbase, PRIORITY_FEE_VAULT); assert_eq!(db_account.balance, expected_balance); - assert_eq!(base_fee_valut.balance, expected_base_fee_vault_balance); + assert_eq!(base_fee_vault.balance, expected_base_fee_vault_balance); assert_eq!(coinbase_account.balance, expected_coinbase_balance); - assert_eq!(l1_fee_valut.balance, expected_l1_fee_vault_balance); + assert_eq!(l1_fee_vault.balance, expected_l1_fee_vault_balance); assert_eq!( evm.receipts .iter(&mut working_set.accessory_state()) .collect::>(), - [Receipt { - receipt: reth_primitives::Receipt { - tx_type: reth_primitives::TxType::Eip1559, - success: true, - cumulative_gas_used: 114235, - logs: vec![], + [ + Receipt { + receipt: reth_primitives::Receipt { + tx_type: reth_primitives::TxType::Eip1559, + success: true, + cumulative_gas_used: 50751, + logs: vec![] + }, + gas_used: 50751, + log_index_start: 0, + l1_diff_size: 255 }, - gas_used: 114235, - log_index_start: 0, - l1_diff_size: 479, - },] + Receipt { + receipt: reth_primitives::Receipt { + tx_type: reth_primitives::TxType::Eip1559, + success: true, + cumulative_gas_used: 131371, + logs: vec![ + Log { + address: address!("3100000000000000000000000000000000000001"), + data: LogData::new( + vec![b256!("32eff959e2e8d1609edc4b39ccf75900aa6c1da5719f8432752963fdf008234f")], + Bytes::from_static(&hex!("000000000000000000000000000000000000000000000000000000000000000101010101010101010101010101010101010101010101010101010101010101010202020202020202020202020202020202020202020202020202020202020202")) + ).unwrap() + } + ] + }, + gas_used: 80620, + log_index_start: 0, + l1_diff_size: 561 + }, + Receipt { + receipt: reth_primitives::Receipt { + tx_type: reth_primitives::TxType::Eip1559, + success: true, + cumulative_gas_used: 300521, + logs: vec![ + Log { + address: address!("3100000000000000000000000000000000000002"), + data: LogData::new( + vec![b256!("fbe5b6cbafb274f445d7fed869dc77a838d8243a22c460de156560e8857cad03")], + Bytes::from_static(&hex!("0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000deaddeaddeaddeaddeaddeaddeaddeaddeaddead")) + ).unwrap() + }, + Log { + address: address!("3100000000000000000000000000000000000002"), + data: LogData::new( + vec![b256!("80bd1fdfe157286ce420ee763f91748455b249605748e5df12dad9844402bafc")], + Bytes::from_static(&hex!("000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000a0000000000000000000000000000000000000000000000000000000000000002d4a209fb3a961d8b1f4ec1caa220c6a50b815febc0b689ddf0b9ddfbf99cb74479e41ac0063066369747265611400000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000a08000000003b9aca006800000000000000000000000000000000000000000000")) + ).unwrap() + }] + }, + gas_used: 169150, + log_index_start: 1, + l1_diff_size: 1019 + }, + Receipt { + receipt: reth_primitives::Receipt { + tx_type: reth_primitives::TxType::Eip1559, + success: true, + cumulative_gas_used: 80620, + logs: vec![ + Log { + address: address!("3100000000000000000000000000000000000001"), + data: LogData::new( + vec![b256!("32eff959e2e8d1609edc4b39ccf75900aa6c1da5719f8432752963fdf008234f")], + Bytes::from_static(&hex!("000000000000000000000000000000000000000000000000000000000000000205050505050505050505050505050505050505050505050505050505050505052a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a")) + ).unwrap() + }] + }, + gas_used: 80620, + log_index_start: 0, + l1_diff_size: 561 + }, + Receipt { + receipt: reth_primitives::Receipt { + tx_type: reth_primitives::TxType::Eip1559, + success: true, + cumulative_gas_used: 194855, + logs: vec![] + }, + gas_used: 114235, + log_index_start: 1, + l1_diff_size: 479 + } + ] ) } @@ -1061,8 +1202,68 @@ fn test_l1_fee_not_enough_funds() { assert!(call_result.is_ok()); - let block = evm.blocks.last(&mut working_set.accessory_state()).unwrap(); - assert!(block.transactions.is_empty()); + assert_eq!(evm.receipts + .iter(&mut working_set.accessory_state()) + .collect::>(), + [ + Receipt { + receipt: reth_primitives::Receipt { + tx_type: reth_primitives::TxType::Eip1559, + success: true, + cumulative_gas_used: 50751, + logs: vec![] + }, + gas_used: 50751, + log_index_start: 0, + l1_diff_size: 255 + }, + Receipt { + receipt: reth_primitives::Receipt { + tx_type: reth_primitives::TxType::Eip1559, + success: true, + cumulative_gas_used: 131371, + logs: vec![ + Log { + address: address!("3100000000000000000000000000000000000001"), + data: LogData::new( + vec![b256!("32eff959e2e8d1609edc4b39ccf75900aa6c1da5719f8432752963fdf008234f")], + Bytes::from_static(&hex!("000000000000000000000000000000000000000000000000000000000000000101010101010101010101010101010101010101010101010101010101010101010202020202020202020202020202020202020202020202020202020202020202")) + ).unwrap() + } + ] + }, + gas_used: 80620, + log_index_start: 0, + l1_diff_size: 561 + }, + Receipt { + receipt: reth_primitives::Receipt { + tx_type: reth_primitives::TxType::Eip1559, + success: true, + cumulative_gas_used: 300521, + logs: vec![ + Log { + address: address!("3100000000000000000000000000000000000002"), + data: LogData::new( + vec![b256!("fbe5b6cbafb274f445d7fed869dc77a838d8243a22c460de156560e8857cad03")], + Bytes::from_static(&hex!("0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000deaddeaddeaddeaddeaddeaddeaddeaddeaddead")) + ).unwrap() + }, + Log { + address: address!("3100000000000000000000000000000000000002"), + data: LogData::new( + vec![b256!("80bd1fdfe157286ce420ee763f91748455b249605748e5df12dad9844402bafc")], + Bytes::from_static(&hex!("000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000a0000000000000000000000000000000000000000000000000000000000000002d4a209fb3a961d8b1f4ec1caa220c6a50b815febc0b689ddf0b9ddfbf99cb74479e41ac0063066369747265611400000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000a08000000003b9aca006800000000000000000000000000000000000000000000")) + ).unwrap() + } + ] + }, + gas_used: 169150, + log_index_start: 1, + l1_diff_size: 1019 + } + ] + ); } evm.end_soft_confirmation_hook(&soft_confirmation_info, &mut working_set); @@ -1077,9 +1278,9 @@ fn test_l1_fee_not_enough_funds() { assert_eq!(db_account.balance, U256::from(1000000)); assert_eq!(db_account.nonce, 0); - // The coinbase was not created + // The coinbase balance is zero let db_coinbase = evm.accounts.get(&config.coinbase, &mut working_set); - assert!(db_coinbase.is_none()); + assert_eq!(db_coinbase.unwrap().balance, U256::from(0)); } #[test] @@ -1148,33 +1349,107 @@ fn test_l1_fee_halt() { evm.end_soft_confirmation_hook(&soft_confirmation_info, &mut working_set); evm.finalize_hook(&[99u8; 32].into(), &mut working_set.accessory_state()); - assert_eq!( - evm.receipts - .iter(&mut working_set.accessory_state()) - .collect::>(), + assert_eq!(evm.receipts + .iter(&mut working_set.accessory_state()) + .collect::>(), [ Receipt { receipt: reth_primitives::Receipt { tx_type: reth_primitives::TxType::Eip1559, success: true, - cumulative_gas_used: 106947, - logs: vec![], + cumulative_gas_used: 50751, + logs: vec![] }, - gas_used: 106947, + gas_used: 50751, log_index_start: 0, - l1_diff_size: 447, + l1_diff_size: 255 }, Receipt { receipt: reth_primitives::Receipt { tx_type: reth_primitives::TxType::Eip1559, - success: false, - cumulative_gas_used: 1106947, - logs: vec![], + success: true, + cumulative_gas_used: 131371, + logs: vec![ + Log { + address: address!("3100000000000000000000000000000000000001"), + data: LogData::new( + vec![b256!("32eff959e2e8d1609edc4b39ccf75900aa6c1da5719f8432752963fdf008234f")], + Bytes::from_static(&hex!("000000000000000000000000000000000000000000000000000000000000000101010101010101010101010101010101010101010101010101010101010101010202020202020202020202020202020202020202020202020202020202020202")) + ).unwrap() + } + ] }, - gas_used: 1000000, + gas_used: 80620, log_index_start: 0, - l1_diff_size: 96, + l1_diff_size: 561 + }, + Receipt { + receipt: reth_primitives::Receipt { + tx_type: reth_primitives::TxType::Eip1559, + success: true, + cumulative_gas_used: 300521, + logs: vec![ + Log { + address: address!("3100000000000000000000000000000000000002"), + data: LogData::new( + vec![b256!("fbe5b6cbafb274f445d7fed869dc77a838d8243a22c460de156560e8857cad03")], + Bytes::from_static(&hex!("0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000deaddeaddeaddeaddeaddeaddeaddeaddeaddead")) + ).unwrap() + }, + Log { + address: address!("3100000000000000000000000000000000000002"), + data: LogData::new( + vec![b256!("80bd1fdfe157286ce420ee763f91748455b249605748e5df12dad9844402bafc")], + Bytes::from_static(&hex!("000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000a0000000000000000000000000000000000000000000000000000000000000002d4a209fb3a961d8b1f4ec1caa220c6a50b815febc0b689ddf0b9ddfbf99cb74479e41ac0063066369747265611400000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000a08000000003b9aca006800000000000000000000000000000000000000000000")) + ).unwrap() + } + ] + }, + gas_used: 169150, + log_index_start: 1, + l1_diff_size: 1019 }, + Receipt { + receipt: reth_primitives::Receipt { + tx_type: reth_primitives::TxType::Eip1559, + success: true, + cumulative_gas_used: 80620, + logs: vec![ + Log { + address: address!("3100000000000000000000000000000000000001"), + data: LogData::new( + vec![b256!("32eff959e2e8d1609edc4b39ccf75900aa6c1da5719f8432752963fdf008234f")], + Bytes::from_static(&hex!("000000000000000000000000000000000000000000000000000000000000000205050505050505050505050505050505050505050505050505050505050505052a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a")) + ).unwrap() + }, + ] + }, + gas_used: 80620, + log_index_start: 0, + l1_diff_size: 561 + }, + Receipt { + receipt: reth_primitives::Receipt { + tx_type: reth_primitives::TxType::Eip1559, + success: true, + cumulative_gas_used: 187567, + logs: vec![] + }, + gas_used: 106947, + log_index_start: 1, + l1_diff_size: 447 + }, + Receipt { + receipt: reth_primitives::Receipt + { tx_type: reth_primitives::TxType::Eip1559, + success: false, + cumulative_gas_used: 1187567, + logs: vec![] + }, + gas_used: 1000000, + log_index_start: 1, + l1_diff_size: 96 + } ] ); @@ -1194,13 +1469,12 @@ fn test_l1_fee_halt() { expenses ) ); + let base_fee_vault = evm.accounts.get(&BASE_FEE_VAULT, &mut working_set).unwrap(); + let l1_fee_vault = evm.accounts.get(&L1_FEE_VAULT, &mut working_set).unwrap(); - let base_fee_valut = evm.accounts.get(&BASE_FEE_VAULT, &mut working_set).unwrap(); - let l1_fee_valut = evm.accounts.get(&L1_FEE_VAULT, &mut working_set).unwrap(); - - assert_eq!(base_fee_valut.balance, U256::from(1106947_u64 * 10000000)); + assert_eq!(base_fee_vault.balance, U256::from(1106947_u64 * 10000000)); assert_eq!( - l1_fee_valut.balance, + l1_fee_vault.balance, U256::from(447 + 96 + 2 * L1_FEE_OVERHEAD as u64) ); } diff --git a/crates/evm/src/tests/ef_tests/cases/blockchain_test.rs b/crates/evm/src/tests/ef_tests/cases/blockchain_test.rs index 4e799a309..008b8ac98 100644 --- a/crates/evm/src/tests/ef_tests/cases/blockchain_test.rs +++ b/crates/evm/src/tests/ef_tests/cases/blockchain_test.rs @@ -21,7 +21,7 @@ use crate::evm::DbAccount; use crate::primitive_types::Block; use crate::tests::ef_tests::models::{BlockchainTest, ForkSpec}; use crate::tests::ef_tests::{Case, Error, Suite}; -use crate::tests::utils::{commit, get_evm_with_storage}; +use crate::tests::utils::{commit, config_push_contracts, get_evm_with_storage}; use crate::{AccountData, Evm, EvmChainConfig, EvmConfig, RlpEvmTransaction, U256}; /// A handler for the blockchain test suite. @@ -135,7 +135,7 @@ impl Case for BlockchainTestCase { .par_bridge() .try_for_each(|case| { let mut evm_config = EvmConfig::default(); - + config_push_contracts(&mut evm_config, None); // Set this base fee based on what's set in genesis. let header = reth_primitives::Header { parent_hash: case.genesis_block_header.parent_hash, diff --git a/crates/evm/src/tests/genesis_tests.rs b/crates/evm/src/tests/genesis_tests.rs index 4af911f4d..0c89bc2f3 100644 --- a/crates/evm/src/tests/genesis_tests.rs +++ b/crates/evm/src/tests/genesis_tests.rs @@ -1,65 +1,19 @@ -use std::collections::HashMap; - use alloy_eips::eip1559::BaseFeeParams; use lazy_static::lazy_static; use reth_primitives::constants::{EMPTY_RECEIPTS, EMPTY_TRANSACTIONS, ETHEREUM_BLOCK_GAS_LIMIT}; use reth_primitives::hex_literal::hex; use reth_primitives::{ - Address, Bloom, Bytes, Header, SealedHeader, B256, EMPTY_OMMER_ROOT_HASH, KECCAK_EMPTY, U256, + Address, Bloom, Bytes, Header, SealedHeader, B256, EMPTY_OMMER_ROOT_HASH, U256, }; use revm::primitives::SpecId; use sov_modules_api::prelude::*; use crate::evm::primitive_types::SealedBlock; use crate::evm::{AccountInfo, EvmChainConfig}; -use crate::tests::utils::{get_evm, GENESIS_HASH, GENESIS_STATE_ROOT}; -use crate::{AccountData, EvmConfig}; +use crate::tests::utils::{get_evm, get_evm_test_config, GENESIS_HASH, GENESIS_STATE_ROOT}; +use crate::EvmConfig; lazy_static! { - pub(crate) static ref TEST_CONFIG: EvmConfig = EvmConfig { - data: vec![AccountData { - address: Address::from([1u8; 20]), - balance: U256::checked_mul(U256::from(1000), U256::pow(U256::from(10), U256::from(18))).unwrap(), // 1000 ETH - code_hash: KECCAK_EMPTY, - code: Bytes::default(), - nonce: 0, - storage: Default::default(), - }, - AccountData { - address:Address::from([2u8; 20]), - balance: U256::checked_mul(U256::from(1000), - U256::pow(U256::from(10), U256::from(18))).unwrap(), // 1000 ETH, - code_hash: hex!("4e8ee9adb469b245e3a5a8e58e9b733aaa857a9dce1982257531db8a2700aabf").into(), - code: hex!("60606040526000357c0100000000000000000000000000000000000000000000000000000000900463ffffffff168063a223e05d1461006a578063").into(), - storage: { - let mut storage = HashMap::new(); - storage.insert(U256::from(0), U256::from(0x4321)); - storage.insert( - U256::from_be_slice( - &hex!("6661e9d6d8b923d5bbaab1b96e1dd51ff6ea2a93520fdc9eb75d059238b8c5e9")[..], - ), - U256::from(8), - ); - - storage - }, - nonce: 1 - }], - spec: vec![(0, SpecId::BERLIN), (1, SpecId::SHANGHAI)] - .into_iter() - .collect(), - chain_id: 1000, - block_gas_limit: reth_primitives::constants::ETHEREUM_BLOCK_GAS_LIMIT, - coinbase: Address::from([3u8; 20]), - limit_contract_code_size: Some(5000), - starting_base_fee: 1000000000, - base_fee_params: BaseFeeParams::ethereum(), - timestamp: 0, - difficulty: U256::ZERO, - extra_data: Bytes::default(), - nonce: 0, - }; - pub(crate) static ref GENESIS_DA_TXS_COMMITMENT: B256 = B256::from(hex!( "4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b" )); @@ -68,16 +22,17 @@ lazy_static! { #[test] fn genesis_data() { - let (evm, mut working_set) = get_evm(&TEST_CONFIG); + let config = get_evm_test_config(); + let (evm, mut working_set) = get_evm(&config); - let account = &TEST_CONFIG.data[0]; + let account = &config.data[0]; let db_account = evm .accounts .get(&account.address, &mut working_set) .unwrap(); - let contract = &TEST_CONFIG.data[1]; + let contract = &config.data[1]; let contract_account = evm .accounts @@ -138,7 +93,7 @@ fn genesis_data() { #[test] fn genesis_cfg() { - let (evm, mut working_set) = get_evm(&TEST_CONFIG); + let (evm, mut working_set) = get_evm(&get_evm_test_config()); let cfg = evm.cfg.get(&mut working_set).unwrap(); assert_eq!( @@ -165,7 +120,7 @@ fn genesis_cfg_missing_specs() { #[test] fn genesis_empty_spec_defaults_to_shanghai() { - let mut config = TEST_CONFIG.clone(); + let mut config = get_evm_test_config(); config.spec.clear(); let (evm, mut working_set) = get_evm(&config); @@ -184,7 +139,7 @@ fn genesis_cfg_cancun() { #[test] fn genesis_block() { - let (evm, mut working_set) = get_evm(&TEST_CONFIG); + let (evm, mut working_set) = get_evm(&get_evm_test_config()); let mut accessory_state = working_set.accessory_state(); @@ -238,7 +193,7 @@ fn genesis_block() { #[test] fn genesis_head() { - let (evm, mut working_set) = get_evm(&TEST_CONFIG); + let (evm, mut working_set) = get_evm(&get_evm_test_config()); let head = evm.head.get(&mut working_set).unwrap(); assert_eq!(head.header.parent_hash, *GENESIS_HASH); diff --git a/crates/evm/src/tests/hooks_tests.rs b/crates/evm/src/tests/hooks_tests.rs index ee3cded17..a8b4d5678 100644 --- a/crates/evm/src/tests/hooks_tests.rs +++ b/crates/evm/src/tests/hooks_tests.rs @@ -9,12 +9,12 @@ use sov_modules_api::hooks::HookSoftConfirmationInfo; use sov_modules_api::{StateMapAccessor, StateValueAccessor, StateVecAccessor}; use sov_rollup_interface::spec::SpecId; -use super::genesis_tests::{GENESIS_DA_TXS_COMMITMENT, TEST_CONFIG}; +use super::genesis_tests::GENESIS_DA_TXS_COMMITMENT; use crate::evm::primitive_types::{ Block, BlockEnv, Receipt, SealedBlock, TransactionSignedAndRecovered, }; use crate::tests::genesis_tests::BENEFICIARY; -use crate::tests::utils::{get_evm, GENESIS_STATE_ROOT}; +use crate::tests::utils::{get_evm, get_evm_test_config, GENESIS_STATE_ROOT}; use crate::tests::DEFAULT_CHAIN_ID; use crate::PendingTransaction; @@ -24,7 +24,8 @@ lazy_static! { #[test] fn begin_soft_confirmation_hook_creates_pending_block() { - let (mut evm, mut working_set) = get_evm(&TEST_CONFIG); + let config = get_evm_test_config(); + let (mut evm, mut working_set) = get_evm(&config); let l1_fee_rate = 0; let l2_height = 2; let soft_confirmation_info = HookSoftConfirmationInfo { @@ -48,15 +49,16 @@ fn begin_soft_confirmation_hook_creates_pending_block() { coinbase: *BENEFICIARY, timestamp: 54, prevrandao: *DA_ROOT_HASH, - basefee: 765625000, - gas_limit: TEST_CONFIG.block_gas_limit, + basefee: 767816299, + gas_limit: config.block_gas_limit, } ); } #[test] fn end_soft_confirmation_hook_sets_head() { - let (mut evm, mut working_set) = get_evm(&TEST_CONFIG); + let config = get_evm_test_config(); + let (mut evm, mut working_set) = get_evm(&get_evm_test_config()); let mut pre_state_root = [0u8; 32]; pre_state_root.copy_from_slice(GENESIS_STATE_ROOT.as_ref()); @@ -98,28 +100,28 @@ fn end_soft_confirmation_hook_sets_head() { Block { header: Header { parent_hash: B256::from(hex!( - "3c83b326074b9430d4899991bbb06b8517315c50ca2cb17c11e1e972afce1b02" + "06c67dab6518e07a5df24039fd294e3f548026915ef1b8b6d597d421a18cc438" )), ommers_hash: EMPTY_OMMER_ROOT_HASH, - beneficiary: TEST_CONFIG.coinbase, + beneficiary: config.coinbase, state_root: KECCAK_EMPTY, transactions_root: B256::from(hex!( - "30eb5f6050df7ea18ca34cf3503f4713119315a2d3c11f892c5c8920acf816f4" + "fdf1049f7decef904ffdc7d55f8ca9c9c52ad655c8ddb7435025d86c97a253c0" )), receipts_root: B256::from(hex!( - "27036187b3f5e87d4306b396cf06c806da2cc9a0fef9b07c042e3b4304e01c64" + "e8271759b66c13c70ad0726ee34c9fd2574d429fd77d95f95b22f988565a1469" )), withdrawals_root: None, - logs_bloom: Bloom::default(), + logs_bloom: Bloom::new(hex!("00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000800000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008000000000000000000040000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000")), difficulty: U256::ZERO, number: 2, - gas_limit: TEST_CONFIG.block_gas_limit, + gas_limit: config.block_gas_limit, gas_used: 200u64, timestamp: 54, mix_hash: *DA_ROOT_HASH, nonce: 0, - base_fee_per_gas: Some(765625000), + base_fee_per_gas: Some(767816299), extra_data: Bytes::default(), blob_gas_used: None, excess_blob_gas: None, @@ -128,14 +130,14 @@ fn end_soft_confirmation_hook_sets_head() { }, l1_fee_rate: 0, l1_hash: B256::from(DA_ROOT_HASH.0), - transactions: 0..2 + transactions: 3..6 } ); } #[test] fn end_soft_confirmation_hook_moves_transactions_and_receipts() { - let (mut evm, mut working_set) = get_evm(&TEST_CONFIG); + let (mut evm, mut working_set) = get_evm(&get_evm_test_config()); let l1_fee_rate = 0; let l2_height = 2; @@ -165,31 +167,42 @@ fn end_soft_confirmation_hook_moves_transactions_and_receipts() { let tx2_hash = tx2.transaction.signed_transaction.hash; assert_eq!( - evm.transactions - .iter(&mut working_set.accessory_state()) - .collect::>(), - [tx1.transaction, tx2.transaction] + evm.receipts + .get(4, &mut working_set.accessory_state()) + .unwrap(), + tx1.receipt ); - assert_eq!( evm.receipts - .iter(&mut working_set.accessory_state()) - .collect::>(), - [tx1.receipt, tx2.receipt] + .get(5, &mut working_set.accessory_state()) + .unwrap(), + tx2.receipt + ); + assert_eq!( + evm.transactions + .get(4, &mut working_set.accessory_state()) + .unwrap(), + tx1.transaction + ); + assert_eq!( + evm.transactions + .get(5, &mut working_set.accessory_state()) + .unwrap(), + tx2.transaction ); assert_eq!( evm.transaction_hashes .get(&tx1_hash, &mut working_set.accessory_state()) .unwrap(), - 0 + 4 ); assert_eq!( evm.transaction_hashes .get(&tx2_hash, &mut working_set.accessory_state()) .unwrap(), - 1 + 5 ); assert_eq!(evm.pending_transactions.len(), 0); @@ -232,7 +245,8 @@ fn create_pending_transaction(hash: B256, index: u64) -> PendingTransaction { #[test] fn finalize_hook_creates_final_block() { - let (mut evm, mut working_set) = get_evm(&TEST_CONFIG); + let config = get_evm_test_config(); + let (mut evm, mut working_set) = get_evm(&config); // hack to get the root hash let binding = evm @@ -298,16 +312,16 @@ fn finalize_hook_creates_final_block() { let header = Header { parent_hash, ommers_hash: EMPTY_OMMER_ROOT_HASH, - beneficiary: TEST_CONFIG.coinbase, + beneficiary: config.coinbase, state_root: B256::from(root_hash), transactions_root: B256::from(hex!( - "30eb5f6050df7ea18ca34cf3503f4713119315a2d3c11f892c5c8920acf816f4" + "fdf1049f7decef904ffdc7d55f8ca9c9c52ad655c8ddb7435025d86c97a253c0" )), receipts_root: B256::from(hex!( - "27036187b3f5e87d4306b396cf06c806da2cc9a0fef9b07c042e3b4304e01c64" + "e8271759b66c13c70ad0726ee34c9fd2574d429fd77d95f95b22f988565a1469" )), withdrawals_root: None, - logs_bloom: Bloom::default(), + logs_bloom: Bloom::new(hex!("00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000800000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008000000000000000000040000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000")), difficulty: U256::ZERO, number: 2, gas_limit: 30000000, @@ -317,7 +331,7 @@ fn finalize_hook_creates_final_block() { "0505050505050505050505050505050505050505050505050505050505050505" )), nonce: 0, - base_fee_per_gas: Some(765625000), + base_fee_per_gas: Some(767816299), extra_data: Bytes::default(), blob_gas_used: None, excess_blob_gas: None, @@ -330,7 +344,7 @@ fn finalize_hook_creates_final_block() { header: header.seal_slow(), l1_fee_rate: 0, l1_hash: B256::from(DA_ROOT_HASH.0), - transactions: 0..2 + transactions: 3..6 } ); @@ -346,7 +360,7 @@ fn finalize_hook_creates_final_block() { #[test] fn begin_soft_confirmation_hook_appends_last_block_hashes() { - let (mut evm, mut working_set) = get_evm(&TEST_CONFIG); + let (mut evm, mut working_set) = get_evm(&get_evm_test_config()); // hack to get the root hash let binding = evm diff --git a/crates/evm/src/tests/queries/basic_queries.rs b/crates/evm/src/tests/queries/basic_queries.rs index 9cf1bb8ca..f8c7487b5 100644 --- a/crates/evm/src/tests/queries/basic_queries.rs +++ b/crates/evm/src/tests/queries/basic_queries.rs @@ -21,7 +21,7 @@ fn get_block_by_hash_test() { let third_block = evm .get_block_by_hash( - b256!("2d7962c316685635252886d6801a553139e94e3b7d2b678f8c9d974a54e24ab9"), + b256!("c8f53d2fb3a04b566938033716492ea98b203139a52bc9286bea45e7613e3bd3"), None, &mut working_set, ) @@ -114,11 +114,12 @@ fn get_transaction_by_block_hash_and_index_test() { .unwrap(); // doesn't exist - let result = evm.get_transaction_by_block_hash_and_index(hash, U64::from(4), &mut working_set); + let result = evm.get_transaction_by_block_hash_and_index(hash, U64::from(5), &mut working_set); assert_eq!(result, Ok(None)); let tx_hashes = [ + b256!("29640d82d763831afa07d23c967d6a3149a1fec2cde106a5b5abee6c319b61f3"), b256!("2ff3a833e99d5a97e26f912c2e855f95e2dda542c89131fea0d189889d384d99"), b256!("a69485c543cd51dc1856619f3ddb179416af040da2835a10405c856cd5fb41b8"), b256!("17fa953338b32b30795ccb62f050f1c9bcdd48f4793fb2d6d34290b444841271"), @@ -148,14 +149,14 @@ fn get_transaction_by_block_number_and_index_test() { // doesn't exist let result = evm.get_transaction_by_block_number_and_index( BlockNumberOrTag::Number(1), - U64::from(3), + U64::from(6), &mut working_set, ); assert_eq!(result, Ok(None)); // these should exist - for i in 0..3 { + for i in 0..6 { let result = evm.get_transaction_by_block_number_and_index( BlockNumberOrTag::Number(1), U64::from(i), @@ -166,6 +167,7 @@ fn get_transaction_by_block_number_and_index_test() { } let tx_hashes = [ + b256!("29640d82d763831afa07d23c967d6a3149a1fec2cde106a5b5abee6c319b61f3"), b256!("2ff3a833e99d5a97e26f912c2e855f95e2dda542c89131fea0d189889d384d99"), b256!("a69485c543cd51dc1856619f3ddb179416af040da2835a10405c856cd5fb41b8"), b256!("17fa953338b32b30795ccb62f050f1c9bcdd48f4793fb2d6d34290b444841271"), @@ -191,7 +193,7 @@ fn get_block_transaction_count_by_hash_test() { // Non-existent blockhash should return None assert_eq!(result, Ok(None)); - let bock_hash_1 = evm + let block_hash_1 = evm .get_block_by_number(Some(BlockNumberOrTag::Number(1)), None, &mut working_set) .unwrap() .unwrap() @@ -199,11 +201,11 @@ fn get_block_transaction_count_by_hash_test() { .hash .unwrap(); - let result = evm.eth_get_block_transaction_count_by_hash(bock_hash_1, &mut working_set); + let result = evm.eth_get_block_transaction_count_by_hash(block_hash_1, &mut working_set); - assert_eq!(result, Ok(Some(U256::from(3)))); + assert_eq!(result, Ok(Some(U256::from(6)))); - let bock_hash_2 = evm + let block_hash_2 = evm .get_block_by_number(Some(BlockNumberOrTag::Number(2)), None, &mut working_set) .unwrap() .unwrap() @@ -211,11 +213,10 @@ fn get_block_transaction_count_by_hash_test() { .hash .unwrap(); - let result = evm.eth_get_block_transaction_count_by_hash(bock_hash_2, &mut working_set); - - assert_eq!(result, Ok(Some(U256::from(4)))); + let result = evm.eth_get_block_transaction_count_by_hash(block_hash_2, &mut working_set); + assert_eq!(result, Ok(Some(U256::from(5)))); - let bock_hash_3 = evm + let block_hash_3 = evm .get_block_by_number(Some(BlockNumberOrTag::Number(3)), None, &mut working_set) .unwrap() .unwrap() @@ -223,9 +224,9 @@ fn get_block_transaction_count_by_hash_test() { .hash .unwrap(); - let result = evm.eth_get_block_transaction_count_by_hash(bock_hash_3, &mut working_set); + let result = evm.eth_get_block_transaction_count_by_hash(block_hash_3, &mut working_set); - assert_eq!(result, Ok(Some(U256::from(2)))); + assert_eq!(result, Ok(Some(U256::from(3)))); } #[test] @@ -239,15 +240,15 @@ fn get_block_transaction_count_by_number_test() { let result = evm .eth_get_block_transaction_count_by_number(BlockNumberOrTag::Number(1), &mut working_set); - assert_eq!(result, Ok(Some(U256::from(3)))); + assert_eq!(result, Ok(Some(U256::from(6)))); let result = evm .eth_get_block_transaction_count_by_number(BlockNumberOrTag::Number(2), &mut working_set); - assert_eq!(result, Ok(Some(U256::from(4)))); + assert_eq!(result, Ok(Some(U256::from(5)))); let result = evm .eth_get_block_transaction_count_by_number(BlockNumberOrTag::Number(3), &mut working_set); - assert_eq!(result, Ok(Some(U256::from(2)))); + assert_eq!(result, Ok(Some(U256::from(3)))); } #[test] @@ -465,32 +466,33 @@ fn call_test() { fn check_against_third_block(block: &Rich) { // details = false let mut inner_block = serde_json::from_value::(json!({ - "hash": "0x2d7962c316685635252886d6801a553139e94e3b7d2b678f8c9d974a54e24ab9", - "parentHash": "0x97dcd6347f726d864f7b4e2f279f535f0e050bdba9da89e9b2cc744897565432", - "sha3Uncles": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", - "miner": "0x0000000000000000000000000000000000000000", - "stateRoot": "0x6464646464646464646464646464646464646464646464646464646464646464", - "transactionsRoot": "0xef32d81a36e83472e84e033022e11d89a50d466cacc17bac6be1c981205330a3", - "receiptsRoot": "0xf966e7c620235a408862e853eb0cd7e74c28abac1dece96c4440cd5b991d9058", - "logsBloom": "0x00000000000000000000000000004000001000000000000000002000000000000000801000000000200000000000000000000000000000000000000000000000000020000000000000000000000000000000000000400000000000000000000008000000000000000000000000000400000000000000000000000000000000000040000000000000000000000800000000001100800000000000000000000000000000044000000000004000000000000000003000000000020001000000000000000000000000000000000000000000000000000000000000010000000000000000000000400000000000000000000000800000010000080000000000000000", + "baseFeePerGas": "0x2de0b039", "difficulty": "0x0", - "number": "0x2", - "gasLimit": "0x1c9c380", - "gasUsed": "0x19c14", - "timestamp": "0x18", "extraData": "0x", + "gasLimit": "0x1c9c380", + "gasUsed": "0x2d700", + "hash": "0xc8f53d2fb3a04b566938033716492ea98b203139a52bc9286bea45e7613e3bd3", + "logsBloom": "0x00000000000000000000000000004000001000000000000000002000000000000000801000000000200000000000000000000000000800000000000000000000000020000000000800000000000000000000000000400000000000000000000008000000000000000000000000000400000000000008000000000000000000040040000000000000000000000800000000001100800000000010000000000000000000044000000000004000000000000000003000000000020001000000000000000000000000000000000000000000000000000000000000010000000000000000000000400000000000000000000000800000010000080000000000000000", + "miner": "0x0000000000000000000000000000000000000000", "mixHash": "0x0808080808080808080808080808080808080808080808080808080808080808", "nonce": "0x0000000000000000", - "baseFeePerGas": "0x2dbf4076", + "number": "0x2", + "parentHash": "0x0e5059139f666213cee8b0306dec67ba4ca1f891fdd9e8bcc4acfd63f2b6b428", + "receiptsRoot": "0x2147ba909c0456b68d818b3e1bc80dc83c8c38e0ad3a91de36d0a940c97681de", + "sha3Uncles": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", + "size": "0x5c0", + "stateRoot": "0x6464646464646464646464646464646464646464646464646464646464646464", + "timestamp": "0x18", "totalDifficulty": "0x0", - "uncles": [], "transactions": [ + "0x29640d82d763831afa07d23c967d6a3149a1fec2cde106a5b5abee6c319b61f3", "0x2ff3a833e99d5a97e26f912c2e855f95e2dda542c89131fea0d189889d384d99", "0xa69485c543cd51dc1856619f3ddb179416af040da2835a10405c856cd5fb41b8", "0x17fa953338b32b30795ccb62f050f1c9bcdd48f4793fb2d6d34290b444841271", "0xd7e5b2bce65678b5e1a4430b1320b18a258fd5412e20bd5734f446124a9894e6" ], - "size": "0x54b", + "transactionsRoot": "0x246f34a58de3339f3079ad70f5f8614bcec07244bf1957e271c74468e262cb31", + "uncles": [] })).unwrap(); inner_block.other.insert( @@ -514,209 +516,243 @@ fn check_against_third_block(block: &Rich) { } fn check_against_third_block_receipts(receipts: Vec) { - let test_receipts = serde_json::from_value::>(json!([ + let test_receipts = serde_json::from_value::>(json!( + [ { - "transactionHash": "0x2ff3a833e99d5a97e26f912c2e855f95e2dda542c89131fea0d189889d384d99", + "blockHash": "0xc8f53d2fb3a04b566938033716492ea98b203139a52bc9286bea45e7613e3bd3", + "blockNumber": "0x2", + "contractAddress": null, + "cumulativeGasUsed": "0x13aec", + "effectiveGasPrice": "0x2de0b039", + "from": "0xdeaddeaddeaddeaddeaddeaddeaddeaddeaddead", + "gasUsed": "0x13aec", + "l1DiffSize": "0x231", + "l1FeeRate": "0x1", + "logs": [ + { + "address": "0x3100000000000000000000000000000000000001", + "blockHash": "0xc8f53d2fb3a04b566938033716492ea98b203139a52bc9286bea45e7613e3bd3", + "blockNumber": "0x2", + "blockTimestamp": "0x18", + "data": "0x000000000000000000000000000000000000000000000000000000000000000208080808080808080808080808080808080808080808080808080808080808082a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a", + "logIndex": "0x0", + "removed": false, + "topics": [ + "0x32eff959e2e8d1609edc4b39ccf75900aa6c1da5719f8432752963fdf008234f" + ], + "transactionHash": "0x29640d82d763831afa07d23c967d6a3149a1fec2cde106a5b5abee6c319b61f3", + "transactionIndex": "0x0" + } + ], + "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000800000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008000000000000000000040000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000", + "status": "0x1", + "to": "0x3100000000000000000000000000000000000001", + "transactionHash": "0x29640d82d763831afa07d23c967d6a3149a1fec2cde106a5b5abee6c319b61f3", "transactionIndex": "0x0", - "blockHash": "0x2d7962c316685635252886d6801a553139e94e3b7d2b678f8c9d974a54e24ab9", + "type": "0x2" + }, + { + "blockHash": "0xc8f53d2fb3a04b566938033716492ea98b203139a52bc9286bea45e7613e3bd3", "blockNumber": "0x2", - "cumulativeGasUsed": "0x6720", - "gasUsed": "0x6720", - "effectiveGasPrice": "0x2dbf4076", + "contractAddress": null, + "cumulativeGasUsed": "0x1a20c", + "effectiveGasPrice": "0x2de0b039", "from": "0x9e1abd37ec34bbc688b6a2b7d9387d9256cf1773", - "to": "0x819c5497b157177315e1204f52e588b393771719", - "l1FeeRate": "0x1", + "gasUsed": "0x6720", "l1DiffSize": "0x60", - "contractAddress": null, + "l1FeeRate": "0x1", "logs": [ { "address": "0x819c5497b157177315e1204f52e588b393771719", + "blockHash": "0xc8f53d2fb3a04b566938033716492ea98b203139a52bc9286bea45e7613e3bd3", + "blockNumber": "0x2", + "blockTimestamp": "0x18", + "data": "0x0000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000c48656c6c6f20576f726c64210000000000000000000000000000000000000000", + "logIndex": "0x1", + "removed": false, "topics": [ "0xa9943ee9804b5d456d8ad7b3b1b975a5aefa607e16d13936959976e776c4bec7", "0x0000000000000000000000009e1abd37ec34bbc688b6a2b7d9387d9256cf1773", "0x000000000000000000000000819c5497b157177315e1204f52e588b393771719", "0x6d91615c65c0e8f861b0fbfce2d9897fb942293e341eda10c91a6912c4f32668" ], - "data": "0x0000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000c48656c6c6f20576f726c64210000000000000000000000000000000000000000", - "blockHash": "0x2d7962c316685635252886d6801a553139e94e3b7d2b678f8c9d974a54e24ab9", - "blockNumber": "0x2", - "blockTimestamp": "0x18", "transactionHash": "0x2ff3a833e99d5a97e26f912c2e855f95e2dda542c89131fea0d189889d384d99", - "transactionIndex": "0x0", - "logIndex": "0x0", - "removed": false + "transactionIndex": "0x1" }, { "address": "0x819c5497b157177315e1204f52e588b393771719", + "blockHash": "0xc8f53d2fb3a04b566938033716492ea98b203139a52bc9286bea45e7613e3bd3", + "blockNumber": "0x2", + "blockTimestamp": "0x18", + "data": "0x", + "logIndex": "0x2", + "removed": false, "topics": [ "0xf16dfb875e436384c298237e04527f538a5eb71f60593cfbaae1ff23250d22a9", "0x000000000000000000000000819c5497b157177315e1204f52e588b393771719" ], - "data": "0x", - "blockHash": "0x2d7962c316685635252886d6801a553139e94e3b7d2b678f8c9d974a54e24ab9", - "blockNumber": "0x2", - "blockTimestamp": "0x18", "transactionHash": "0x2ff3a833e99d5a97e26f912c2e855f95e2dda542c89131fea0d189889d384d99", - "transactionIndex": "0x0", - "logIndex": "0x1", - "removed": false + "transactionIndex": "0x1" } ], "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000801000000000200000000000000000000000000000000000000000000000000000000000000000000000000000000000000000400000000000000000000008000000000000000000000000000400000000000000000000000000000000000000000000000000000000000800000000001000800000000000000000000000000000044000000000000000000000000000003000000000020000000000000000000000000000000000000000000000000000000000000000010000000000000000000000400000000000000000000000800000000000080000000000000000", "status": "0x1", + "to": "0x819c5497b157177315e1204f52e588b393771719", + "transactionHash": "0x2ff3a833e99d5a97e26f912c2e855f95e2dda542c89131fea0d189889d384d99", + "transactionIndex": "0x1", "type": "0x2" }, { - "transactionHash": "0xa69485c543cd51dc1856619f3ddb179416af040da2835a10405c856cd5fb41b8", - "transactionIndex": "0x1", - "blockHash": "0x2d7962c316685635252886d6801a553139e94e3b7d2b678f8c9d974a54e24ab9", + "blockHash": "0xc8f53d2fb3a04b566938033716492ea98b203139a52bc9286bea45e7613e3bd3", "blockNumber": "0x2", - "cumulativeGasUsed": "0xce1c", - "gasUsed": "0x66fc", - "effectiveGasPrice": "0x2dbf4076", + "contractAddress": null, + "cumulativeGasUsed": "0x20908", + "effectiveGasPrice": "0x2de0b039", "from": "0x9e1abd37ec34bbc688b6a2b7d9387d9256cf1773", - "to": "0x819c5497b157177315e1204f52e588b393771719", - "l1FeeRate": "0x1", + "gasUsed": "0x66fc", "l1DiffSize": "0x60", - "contractAddress": null, + "l1FeeRate": "0x1", "logs": [ { "address": "0x819c5497b157177315e1204f52e588b393771719", + "blockHash": "0xc8f53d2fb3a04b566938033716492ea98b203139a52bc9286bea45e7613e3bd3", + "blockNumber": "0x2", + "blockTimestamp": "0x18", + "data": "0x0000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000c48656c6c6f20576f726c64210000000000000000000000000000000000000000", + "logIndex": "0x3", + "removed": false, "topics": [ "0xa9943ee9804b5d456d8ad7b3b1b975a5aefa607e16d13936959976e776c4bec7", "0x0000000000000000000000009e1abd37ec34bbc688b6a2b7d9387d9256cf1773", "0x000000000000000000000000819c5497b157177315e1204f52e588b393771719", "0x63b901bb1c5ce387d96b2fa4dea95d718cf56095f6c1c7539385849cc23324e1" ], - "data": "0x0000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000c48656c6c6f20576f726c64210000000000000000000000000000000000000000", - "blockHash": "0x2d7962c316685635252886d6801a553139e94e3b7d2b678f8c9d974a54e24ab9", - "blockNumber": "0x2", - "blockTimestamp": "0x18", "transactionHash": "0xa69485c543cd51dc1856619f3ddb179416af040da2835a10405c856cd5fb41b8", - "transactionIndex": "0x1", - "logIndex": "0x2", - "removed": false + "transactionIndex": "0x2" }, { "address": "0x819c5497b157177315e1204f52e588b393771719", + "blockHash": "0xc8f53d2fb3a04b566938033716492ea98b203139a52bc9286bea45e7613e3bd3", + "blockNumber": "0x2", + "blockTimestamp": "0x18", + "data": "0x", + "logIndex": "0x4", + "removed": false, "topics": [ "0xf16dfb875e436384c298237e04527f538a5eb71f60593cfbaae1ff23250d22a9", "0x000000000000000000000000819c5497b157177315e1204f52e588b393771719" ], - "data": "0x", - "blockHash": "0x2d7962c316685635252886d6801a553139e94e3b7d2b678f8c9d974a54e24ab9", - "blockNumber": "0x2", - "blockTimestamp": "0x18", "transactionHash": "0xa69485c543cd51dc1856619f3ddb179416af040da2835a10405c856cd5fb41b8", - "transactionIndex": "0x1", - "logIndex": "0x3", - "removed": false + "transactionIndex": "0x2" } ], "logsBloom": "0x00000000000000000000000000000000001000000000000000002000000000000000801000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000400000000000000000000008000000000000000000000000000400000000000000000000000000000000000000000000000000000000000800000000001000800000000000000000000000000000044000000000000000000000000000001000000000020000000000000000000000000000000000000000000000000000000000000000010000000000000000000000400000000000000000000000800000000000000000000000000000", "status": "0x1", + "to": "0x819c5497b157177315e1204f52e588b393771719", + "transactionHash": "0xa69485c543cd51dc1856619f3ddb179416af040da2835a10405c856cd5fb41b8", + "transactionIndex": "0x2", "type": "0x2" }, { - "transactionHash": "0x17fa953338b32b30795ccb62f050f1c9bcdd48f4793fb2d6d34290b444841271", - "transactionIndex": "0x2", - "blockHash": "0x2d7962c316685635252886d6801a553139e94e3b7d2b678f8c9d974a54e24ab9", + "blockHash": "0xc8f53d2fb3a04b566938033716492ea98b203139a52bc9286bea45e7613e3bd3", "blockNumber": "0x2", - "cumulativeGasUsed": "0x13518", - "gasUsed": "0x66fc", - "effectiveGasPrice": "0x2dbf4076", + "contractAddress": null, + "cumulativeGasUsed": "0x27004", + "effectiveGasPrice": "0x2de0b039", "from": "0x9e1abd37ec34bbc688b6a2b7d9387d9256cf1773", - "to": "0x819c5497b157177315e1204f52e588b393771719", - "l1FeeRate": "0x1", + "gasUsed": "0x66fc", "l1DiffSize": "0x60", - "contractAddress": null, + "l1FeeRate": "0x1", "logs": [ { "address": "0x819c5497b157177315e1204f52e588b393771719", + "blockHash": "0xc8f53d2fb3a04b566938033716492ea98b203139a52bc9286bea45e7613e3bd3", + "blockNumber": "0x2", + "blockTimestamp": "0x18", + "data": "0x0000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000c48656c6c6f20576f726c64210000000000000000000000000000000000000000", + "logIndex": "0x5", + "removed": false, "topics": [ "0xa9943ee9804b5d456d8ad7b3b1b975a5aefa607e16d13936959976e776c4bec7", "0x0000000000000000000000009e1abd37ec34bbc688b6a2b7d9387d9256cf1773", "0x000000000000000000000000819c5497b157177315e1204f52e588b393771719", "0x5188fc8ba319bea37b8a074fdec21db88eef23191a849074ae8d6df8b2a32364" ], - "data": "0x0000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000c48656c6c6f20576f726c64210000000000000000000000000000000000000000", - "blockHash": "0x2d7962c316685635252886d6801a553139e94e3b7d2b678f8c9d974a54e24ab9", - "blockNumber": "0x2", - "blockTimestamp": "0x18", "transactionHash": "0x17fa953338b32b30795ccb62f050f1c9bcdd48f4793fb2d6d34290b444841271", - "transactionIndex": "0x2", - "logIndex": "0x4", - "removed": false + "transactionIndex": "0x3" }, { "address": "0x819c5497b157177315e1204f52e588b393771719", + "blockHash": "0xc8f53d2fb3a04b566938033716492ea98b203139a52bc9286bea45e7613e3bd3", + "blockNumber": "0x2", + "blockTimestamp": "0x18", + "data": "0x", + "logIndex": "0x6", + "removed": false, "topics": [ "0xf16dfb875e436384c298237e04527f538a5eb71f60593cfbaae1ff23250d22a9", "0x000000000000000000000000819c5497b157177315e1204f52e588b393771719" ], - "data": "0x", - "blockHash": "0x2d7962c316685635252886d6801a553139e94e3b7d2b678f8c9d974a54e24ab9", - "blockNumber": "0x2", - "blockTimestamp": "0x18", "transactionHash": "0x17fa953338b32b30795ccb62f050f1c9bcdd48f4793fb2d6d34290b444841271", - "transactionIndex": "0x2", - "logIndex": "0x5", - "removed": false + "transactionIndex": "0x3" } ], "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000801000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000400000000000000000000008000000000000000000000000000400000000000000000000000000000000000040000000000000000000000800000000001100800000000000000000000000000000044000000000000000000000000000001000000000020000000000000000000000000000000000000000000000000000000000000000010000000000000000000000400000000000000000000000800000010000000000000000000000", "status": "0x1", + "to": "0x819c5497b157177315e1204f52e588b393771719", + "transactionHash": "0x17fa953338b32b30795ccb62f050f1c9bcdd48f4793fb2d6d34290b444841271", + "transactionIndex": "0x3", "type": "0x2" }, { - "transactionHash": "0xd7e5b2bce65678b5e1a4430b1320b18a258fd5412e20bd5734f446124a9894e6", - "transactionIndex": "0x3", - "blockHash": "0x2d7962c316685635252886d6801a553139e94e3b7d2b678f8c9d974a54e24ab9", + "blockHash": "0xc8f53d2fb3a04b566938033716492ea98b203139a52bc9286bea45e7613e3bd3", "blockNumber": "0x2", - "cumulativeGasUsed": "0x19c14", - "gasUsed": "0x66fc", - "effectiveGasPrice": "0x2dbf4076", + "contractAddress": null, + "cumulativeGasUsed": "0x2d700", + "effectiveGasPrice": "0x2de0b039", "from": "0x9e1abd37ec34bbc688b6a2b7d9387d9256cf1773", - "to": "0x819c5497b157177315e1204f52e588b393771719", - "l1FeeRate": "0x1", + "gasUsed": "0x66fc", "l1DiffSize": "0x60", - "contractAddress": null, + "l1FeeRate": "0x1", "logs": [ { "address": "0x819c5497b157177315e1204f52e588b393771719", + "blockHash": "0xc8f53d2fb3a04b566938033716492ea98b203139a52bc9286bea45e7613e3bd3", + "blockNumber": "0x2", + "blockTimestamp": "0x18", + "data": "0x0000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000c48656c6c6f20576f726c64210000000000000000000000000000000000000000", + "logIndex": "0x7", + "removed": false, "topics": [ "0xa9943ee9804b5d456d8ad7b3b1b975a5aefa607e16d13936959976e776c4bec7", "0x0000000000000000000000009e1abd37ec34bbc688b6a2b7d9387d9256cf1773", "0x000000000000000000000000819c5497b157177315e1204f52e588b393771719", "0x29d61b64fc4b3d3e07e2692f6bc997236f115e546fae45393595f0cb0acbc4a0" ], - "data": "0x0000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000c48656c6c6f20576f726c64210000000000000000000000000000000000000000", - "blockHash": "0x2d7962c316685635252886d6801a553139e94e3b7d2b678f8c9d974a54e24ab9", - "blockNumber": "0x2", - "blockTimestamp": "0x18", "transactionHash": "0xd7e5b2bce65678b5e1a4430b1320b18a258fd5412e20bd5734f446124a9894e6", - "transactionIndex": "0x3", - "logIndex": "0x6", - "removed": false + "transactionIndex": "0x4" }, { "address": "0x819c5497b157177315e1204f52e588b393771719", + "blockHash": "0xc8f53d2fb3a04b566938033716492ea98b203139a52bc9286bea45e7613e3bd3", + "blockNumber": "0x2", + "blockTimestamp": "0x18", + "data": "0x", + "logIndex": "0x8", + "removed": false, "topics": [ "0xf16dfb875e436384c298237e04527f538a5eb71f60593cfbaae1ff23250d22a9", "0x000000000000000000000000819c5497b157177315e1204f52e588b393771719" ], - "data": "0x", - "blockHash": "0x2d7962c316685635252886d6801a553139e94e3b7d2b678f8c9d974a54e24ab9", - "blockNumber": "0x2", - "blockTimestamp": "0x18", "transactionHash": "0xd7e5b2bce65678b5e1a4430b1320b18a258fd5412e20bd5734f446124a9894e6", - "transactionIndex": "0x3", - "logIndex": "0x7", - "removed": false + "transactionIndex": "0x4" } ], "logsBloom": "0x00000000000000000000000000004000000000000000000000000000000000000000801000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000400000000000000000000008000000000000000000000000000400000000000000000000000000000000000000000000000000000000000800000000001000800000000000000000000000000000044000000000004000000000000000001000000000020001000000000000000000000000000000000000000000000000000000000000010000000000000000000000400000000000000000000000800000000000000000000000000000", "status": "0x1", + "to": "0x819c5497b157177315e1204f52e588b393771719", + "transactionHash": "0xd7e5b2bce65678b5e1a4430b1320b18a258fd5412e20bd5734f446124a9894e6", + "transactionIndex": "0x4", "type": "0x2" }])).unwrap(); diff --git a/crates/evm/src/tests/queries/log_tests.rs b/crates/evm/src/tests/queries/log_tests.rs index 00725a13f..0d7e2f696 100644 --- a/crates/evm/src/tests/queries/log_tests.rs +++ b/crates/evm/src/tests/queries/log_tests.rs @@ -12,9 +12,10 @@ use sov_rollup_interface::spec::SpecId; use crate::call::CallMessage; use crate::smart_contracts::LogsContract; -use crate::tests::call_tests::{create_contract_message, get_evm_config, publish_event_message}; use crate::tests::queries::init_evm; -use crate::tests::utils::get_evm; +use crate::tests::utils::{ + create_contract_message, get_evm, get_evm_config, publish_event_message, +}; use crate::{Filter, FilterBlockOption, FilterSet}; type C = DefaultContext; @@ -36,13 +37,12 @@ fn logs_for_filter_test() { }, &mut working_set, ); - assert_eq!(result, Err(EthApiError::UnknownBlockNumber.into())); let available_res = evm.eth_get_logs( Filter { block_option: FilterBlockOption::AtBlockHash(b256!( - "2d7962c316685635252886d6801a553139e94e3b7d2b678f8c9d974a54e24ab9" + "c8f53d2fb3a04b566938033716492ea98b203139a52bc9286bea45e7613e3bd3" )), address: FilterSet::default(), topics: [ @@ -56,7 +56,7 @@ fn logs_for_filter_test() { ); // TODO: Check this better. - assert_eq!(available_res.unwrap().len(), 8); + assert_eq!(available_res.unwrap().len(), 9); } #[test] @@ -152,8 +152,8 @@ fn log_filter_test_at_block_hash() { }; let rpc_logs = evm.eth_get_logs(filter, &mut working_set).unwrap(); - // should get all the logs - assert_eq!(rpc_logs.len(), 4); + // should get all the logs including system txs + assert_eq!(rpc_logs.len(), 5); // with address and without topics address.0.insert(contract_addr); @@ -164,7 +164,7 @@ fn log_filter_test_at_block_hash() { topics: topics.clone(), }; let rpc_logs = evm.eth_get_logs(filter, &mut working_set).unwrap(); - // 1) should get all the logs + // 1) should get all the logs with the contract address assert_eq!(rpc_logs.len(), 4); let empty_topic: FilterSet = FilterSet::default(); @@ -350,8 +350,7 @@ fn log_filter_test_with_range() { }; let rpc_logs = evm.eth_get_logs(filter, &mut working_set).unwrap(); - - assert_eq!(rpc_logs.len(), 4); + assert_eq!(rpc_logs.len(), 8); let soft_confirmation_info = HookSoftConfirmationInfo { l2_height, diff --git a/crates/evm/src/tests/queries/mod.rs b/crates/evm/src/tests/queries/mod.rs index bc224d73c..9afb1bd33 100644 --- a/crates/evm/src/tests/queries/mod.rs +++ b/crates/evm/src/tests/queries/mod.rs @@ -17,11 +17,11 @@ use crate::call::CallMessage; use crate::smart_contracts::{ CallerContract, LogsContract, SimplePayableContract, SimpleStorageContract, }; -use crate::tests::call_tests::{ - create_contract_transaction, publish_event_message, set_arg_message, -}; use crate::tests::test_signer::TestSigner; -use crate::tests::utils::{commit, get_evm_with_storage}; +use crate::tests::utils::{ + commit, config_push_contracts, create_contract_transaction, get_evm_with_storage, + publish_event_message, set_arg_message, +}; use crate::{AccountData, Evm, EvmConfig, RlpEvmTransaction}; type C = DefaultContext; @@ -33,7 +33,7 @@ type C = DefaultContext; fn init_evm() -> (Evm, WorkingSet, TestSigner, u64) { let dev_signer: TestSigner = TestSigner::new_random(); - let config = EvmConfig { + let mut config = EvmConfig { data: vec![AccountData { address: dev_signer.address(), balance: U256::from_str("100000000000000000000").unwrap(), @@ -47,7 +47,7 @@ fn init_evm() -> (Evm, WorkingSet, TestSigner, u64) { spec: vec![(0, SpecId::SHANGHAI)].into_iter().collect(), ..Default::default() }; - + config_push_contracts(&mut config, None); let (mut evm, mut working_set, prover_storage) = get_evm_with_storage(&config); let l1_fee_rate = 1; @@ -212,7 +212,7 @@ fn init_evm() -> (Evm, WorkingSet, TestSigner, u64) { pub fn init_evm_single_block() -> (Evm, WorkingSet, TestSigner) { let dev_signer: TestSigner = TestSigner::new_random(); - let config = EvmConfig { + let mut config = EvmConfig { data: vec![ AccountData { address: dev_signer.address(), @@ -234,7 +234,7 @@ pub fn init_evm_single_block() -> (Evm, WorkingSet, TestSigner) { spec: vec![(0, SpecId::SHANGHAI)].into_iter().collect(), ..Default::default() }; - + config_push_contracts(&mut config, None); let (mut evm, mut working_set, prover_storage) = get_evm_with_storage(&config); // let contract_addr: Address = Address::from_slice( @@ -294,7 +294,7 @@ pub fn init_evm_single_block() -> (Evm, WorkingSet, TestSigner) { pub fn init_evm_with_caller_contract() -> (Evm, WorkingSet, TestSigner, u64) { let dev_signer: TestSigner = TestSigner::new_random(); - let config = EvmConfig { + let mut config = EvmConfig { data: vec![AccountData { address: dev_signer.address(), balance: U256::from_str("100000000000000000000").unwrap(), // Setting initial balance @@ -306,7 +306,7 @@ pub fn init_evm_with_caller_contract() -> (Evm, WorkingSet, TestSigner, u6 spec: vec![(0, SpecId::SHANGHAI)].into_iter().collect(), ..Default::default() }; - + config_push_contracts(&mut config, None); let (mut evm, mut working_set, prover_storage) = get_evm_with_storage(&config); let contract_addr: Address = Address::from_slice( diff --git a/crates/evm/src/tests/sys_tx_tests.rs b/crates/evm/src/tests/sys_tx_tests.rs index 57cfc2e6c..863990965 100644 --- a/crates/evm/src/tests/sys_tx_tests.rs +++ b/crates/evm/src/tests/sys_tx_tests.rs @@ -18,13 +18,12 @@ use crate::evm::system_contracts::BitcoinLightClient; use crate::handler::L1_FEE_OVERHEAD; use crate::smart_contracts::{BlockHashContract, LogsContract}; use crate::system_contracts::{Bridge, ProxyAdmin}; -use crate::tests::call_tests::{ - create_contract_message, create_contract_message_with_fee, get_evm_config_starting_base_fee, - publish_event_message, -}; use crate::tests::test_signer::TestSigner; -use crate::tests::utils::get_evm; -use crate::{AccountData, EvmConfig, BASE_FEE_VAULT, L1_FEE_VAULT, SYSTEM_SIGNER}; +use crate::tests::utils::{ + config_push_contracts, create_contract_message, create_contract_message_with_fee, get_evm, + get_evm_config_starting_base_fee, publish_event_message, +}; +use crate::{AccountData, BASE_FEE_VAULT, L1_FEE_VAULT, SYSTEM_SIGNER}; type C = DefaultContext; @@ -33,8 +32,7 @@ fn test_sys_bitcoin_light_client() { let (mut config, dev_signer, _) = get_evm_config_starting_base_fee(U256::from_str("10000000000000").unwrap(), None, 1); - config_push_contracts(&mut config); - + config_push_contracts(&mut config, None); let (mut evm, mut working_set) = get_evm(&config); assert_eq!( @@ -291,7 +289,7 @@ fn test_sys_tx_gas_usage_effect_on_block_gas_limit() { 1, ); - config_push_contracts(&mut config); + config_push_contracts(&mut config, None); let (mut evm, mut working_set) = get_evm(&config); let l1_fee_rate = 0; @@ -416,7 +414,7 @@ fn test_bridge() { let (mut config, _, _) = get_evm_config_starting_base_fee(U256::from_str("1000000").unwrap(), None, 1); - config_push_contracts(&mut config); + config_push_contracts(&mut config, None); let (mut evm, mut working_set) = get_evm(&config); @@ -506,7 +504,7 @@ fn test_upgrade_light_client() { 1, ); - config_push_contracts(&mut config); + config_push_contracts(&mut config, None); // False bitcoin light client implementation, returns dead address on block hash query config.data.push(AccountData::new( @@ -622,7 +620,7 @@ fn test_change_upgrade_owner() { 1, ); - config_push_contracts(&mut config); + config_push_contracts(&mut config, None); let contract_owner = TestSigner::new( secp256k1::SecretKey::from_slice(&[ @@ -812,54 +810,3 @@ fn test_change_upgrade_owner() { .unwrap() ); } - -fn config_push_contracts(config: &mut EvmConfig) { - config.data.push(AccountData::new( - BitcoinLightClient::address(), - U256::ZERO, - Bytes::from_static(&hex!("60806040523661001357610011610017565b005b6100115b61001f610169565b6001600160a01b0316330361015f5760606001600160e01b0319600035166364d3180d60e11b810161005a5761005361019c565b9150610157565b63587086bd60e11b6001600160e01b031982160161007a576100536101f3565b63070d7c6960e41b6001600160e01b031982160161009a57610053610239565b621eb96f60e61b6001600160e01b03198216016100b95761005361026a565b63a39f25e560e01b6001600160e01b03198216016100d9576100536102aa565b60405162461bcd60e51b815260206004820152604260248201527f5472616e73706172656e745570677261646561626c6550726f78793a2061646d60448201527f696e2063616e6e6f742066616c6c6261636b20746f2070726f78792074617267606482015261195d60f21b608482015260a4015b60405180910390fd5b815160208301f35b6101676102be565b565b60007fb53127684a568b3173ae13b9f8a6016e243e63b6e8ee1178d6a717850b5d61035b546001600160a01b0316919050565b60606101a66102ce565b60006101b53660048184610683565b8101906101c291906106c9565b90506101df816040518060200160405280600081525060006102d9565b505060408051602081019091526000815290565b60606000806102053660048184610683565b81019061021291906106fa565b91509150610222828260016102d9565b604051806020016040528060008152509250505090565b60606102436102ce565b60006102523660048184610683565b81019061025f91906106c9565b90506101df81610305565b60606102746102ce565b600061027e610169565b604080516001600160a01b03831660208201529192500160405160208183030381529060405291505090565b60606102b46102ce565b600061027e61035c565b6101676102c961035c565b61036b565b341561016757600080fd5b6102e28361038f565b6000825111806102ef5750805b15610300576102fe83836103cf565b505b505050565b7f7e644d79422f17c01e4894b5f4f588d331ebfa28653d42ae832dc59e38c9798f61032e610169565b604080516001600160a01b03928316815291841660208301520160405180910390a1610359816103fb565b50565b60006103666104a4565b905090565b3660008037600080366000845af43d6000803e80801561038a573d6000f35b3d6000fd5b610398816104cc565b6040516001600160a01b038216907fbc7cd75a20ee27fd9adebab32041f755214dbc6bffa90cc0225b39da2e5c2d3b90600090a250565b60606103f4838360405180606001604052806027815260200161083860279139610560565b9392505050565b6001600160a01b0381166104605760405162461bcd60e51b815260206004820152602660248201527f455243313936373a206e65772061646d696e20697320746865207a65726f206160448201526564647265737360d01b606482015260840161014e565b807fb53127684a568b3173ae13b9f8a6016e243e63b6e8ee1178d6a717850b5d61035b80546001600160a01b0319166001600160a01b039290921691909117905550565b60007f360894a13ba1a3210667c828492db98dca3e2076cc3735a920a3ca505d382bbc61018d565b6001600160a01b0381163b6105395760405162461bcd60e51b815260206004820152602d60248201527f455243313936373a206e657720696d706c656d656e746174696f6e206973206e60448201526c1bdd08184818dbdb9d1c9858dd609a1b606482015260840161014e565b807f360894a13ba1a3210667c828492db98dca3e2076cc3735a920a3ca505d382bbc610483565b6060600080856001600160a01b03168560405161057d91906107e8565b600060405180830381855af49150503d80600081146105b8576040519150601f19603f3d011682016040523d82523d6000602084013e6105bd565b606091505b50915091506105ce868383876105d8565b9695505050505050565b60608315610647578251600003610640576001600160a01b0385163b6106405760405162461bcd60e51b815260206004820152601d60248201527f416464726573733a2063616c6c20746f206e6f6e2d636f6e7472616374000000604482015260640161014e565b5081610651565b6106518383610659565b949350505050565b8151156106695781518083602001fd5b8060405162461bcd60e51b815260040161014e9190610804565b6000808585111561069357600080fd5b838611156106a057600080fd5b5050820193919092039150565b80356001600160a01b03811681146106c457600080fd5b919050565b6000602082840312156106db57600080fd5b6103f4826106ad565b634e487b7160e01b600052604160045260246000fd5b6000806040838503121561070d57600080fd5b610716836106ad565b9150602083013567ffffffffffffffff81111561073257600080fd5b8301601f8101851361074357600080fd5b803567ffffffffffffffff81111561075d5761075d6106e4565b604051601f8201601f19908116603f0116810167ffffffffffffffff8111828210171561078c5761078c6106e4565b6040528181528282016020018710156107a457600080fd5b816020840160208301376000602083830101528093505050509250929050565b60005b838110156107df5781810151838201526020016107c7565b50506000910152565b600082516107fa8184602087016107c4565b9190910192915050565b60208152600082518060208401526108238160408501602087016107c4565b601f01601f1916919091016040019291505056fe416464726573733a206c6f772d6c6576656c2064656c65676174652063616c6c206661696c6564")), - 0, - [ - (U256::from_be_slice(&hex!("360894a13ba1a3210667c828492db98dca3e2076cc3735a920a3ca505d382bbc")), U256::from_be_slice(&hex!("0000000000000000000000003200000000000000000000000000000000000001"))), - (U256::from_be_slice(&hex!("b53127684a568b3173ae13b9f8a6016e243e63b6e8ee1178d6a717850b5d6103")), U256::from_be_slice(&hex!("00000000000000000000000031ffffffffffffffffffffffffffffffffffffff"))), - ].into_iter().collect(), - )); - - config.data.push(AccountData::new( - address!("3200000000000000000000000000000000000001"), - U256::ZERO, - Bytes::from_static(&hex!("608060405234801561001057600080fd5b50600436106100a95760003560e01c806357e871e71161007157806357e871e71461014c57806361b207e214610155578063a91d8b3d14610182578063d269a03e146101a2578063d761753e146101b5578063ee82ac5e146101e857600080fd5b80630466efc4146100ae5780630e27bc11146100e15780631f578333146100f657806334cdf78d146101095780634ffd344a14610129575b600080fd5b6100ce6100bc366004610598565b60009081526002602052604090205490565b6040519081526020015b60405180910390f35b6100f46100ef3660046105b1565b610208565b005b6100f4610104366004610598565b610330565b6100ce610117366004610598565b60016020526000908152604090205481565b61013c61013736600461061c565b6103de565b60405190151581526020016100d8565b6100ce60005481565b6100ce610163366004610598565b6000908152600160209081526040808320548352600290915290205490565b6100ce610190366004610598565b60026020526000908152604090205481565b61013c6101b036600461061c565b610404565b6101d073deaddeaddeaddeaddeaddeaddeaddeaddeaddead81565b6040516001600160a01b0390911681526020016100d8565b6100ce6101f6366004610598565b60009081526001602052604090205490565b3373deaddeaddeaddeaddeaddeaddeaddeaddeaddead146102705760405162461bcd60e51b815260206004820152601f60248201527f63616c6c6572206973206e6f74207468652073797374656d2063616c6c65720060448201526064015b60405180910390fd5b60008054908190036102b65760405162461bcd60e51b815260206004820152600f60248201526e139bdd081a5b9a5d1a585b1a5e9959608a1b6044820152606401610267565b60008181526001602081905260409091208490556102d5908290610677565b60009081558381526002602090815260409182902084905581518381529081018590529081018390527f32eff959e2e8d1609edc4b39ccf75900aa6c1da5719f8432752963fdf008234f9060600160405180910390a1505050565b3373deaddeaddeaddeaddeaddeaddeaddeaddeaddead146103935760405162461bcd60e51b815260206004820152601f60248201527f63616c6c6572206973206e6f74207468652073797374656d2063616c6c6572006044820152606401610267565b600054156103d95760405162461bcd60e51b8152602060048201526013602482015272105b1c9958591e481a5b9a5d1a585b1a5e9959606a1b6044820152606401610267565b600055565b6000858152600160205260408120546103fa908686868661040f565b9695505050505050565b60006103fa86868686865b6000858152600260209081526040808320548151601f870184900484028101840190925285825291610462918891849190899089908190840183828082843760009201919091525089925061046d915050565b979650505050505050565b6000838514801561047c575081155b801561048757508251155b15610494575060016104a3565b6104a0858486856104ab565b90505b949350505050565b6000602084516104bb9190610698565b156104c8575060006104a3565b83516000036104d9575060006104a3565b818560005b8651811015610548576104f2600284610698565b6001036105165761050f6105098883016020015190565b83610555565b915061052f565b61052c826105278984016020015190565b610555565b91505b60019290921c91610541602082610677565b90506104de565b5090931495945050505050565b6000610561838361056a565b90505b92915050565b60008260005281602052602060006040600060025afa50602060006020600060025afa505060005192915050565b6000602082840312156105aa57600080fd5b5035919050565b600080604083850312156105c457600080fd5b50508035926020909101359150565b60008083601f8401126105e557600080fd5b50813567ffffffffffffffff8111156105fd57600080fd5b60208301915083602082850101111561061557600080fd5b9250929050565b60008060008060006080868803121561063457600080fd5b8535945060208601359350604086013567ffffffffffffffff81111561065957600080fd5b610665888289016105d3565b96999598509660600135949350505050565b8082018082111561056457634e487b7160e01b600052601160045260246000fd5b6000826106b557634e487b7160e01b600052601260045260246000fd5b50069056")), - 0, - HashMap::new() - )); - - config.data.push(AccountData::new( - Bridge::address(), - U256::from_str("0x115EEC47F6CF7E35000000").unwrap(), - Bytes::from_static(&hex!("60806040523661001357610011610017565b005b6100115b61001f610169565b6001600160a01b0316330361015f5760606001600160e01b0319600035166364d3180d60e11b810161005a5761005361019c565b9150610157565b63587086bd60e11b6001600160e01b031982160161007a576100536101f3565b63070d7c6960e41b6001600160e01b031982160161009a57610053610239565b621eb96f60e61b6001600160e01b03198216016100b95761005361026a565b63a39f25e560e01b6001600160e01b03198216016100d9576100536102aa565b60405162461bcd60e51b815260206004820152604260248201527f5472616e73706172656e745570677261646561626c6550726f78793a2061646d60448201527f696e2063616e6e6f742066616c6c6261636b20746f2070726f78792074617267606482015261195d60f21b608482015260a4015b60405180910390fd5b815160208301f35b6101676102be565b565b60007fb53127684a568b3173ae13b9f8a6016e243e63b6e8ee1178d6a717850b5d61035b546001600160a01b0316919050565b60606101a66102ce565b60006101b53660048184610683565b8101906101c291906106c9565b90506101df816040518060200160405280600081525060006102d9565b505060408051602081019091526000815290565b60606000806102053660048184610683565b81019061021291906106fa565b91509150610222828260016102d9565b604051806020016040528060008152509250505090565b60606102436102ce565b60006102523660048184610683565b81019061025f91906106c9565b90506101df81610305565b60606102746102ce565b600061027e610169565b604080516001600160a01b03831660208201529192500160405160208183030381529060405291505090565b60606102b46102ce565b600061027e61035c565b6101676102c961035c565b61036b565b341561016757600080fd5b6102e28361038f565b6000825111806102ef5750805b15610300576102fe83836103cf565b505b505050565b7f7e644d79422f17c01e4894b5f4f588d331ebfa28653d42ae832dc59e38c9798f61032e610169565b604080516001600160a01b03928316815291841660208301520160405180910390a1610359816103fb565b50565b60006103666104a4565b905090565b3660008037600080366000845af43d6000803e80801561038a573d6000f35b3d6000fd5b610398816104cc565b6040516001600160a01b038216907fbc7cd75a20ee27fd9adebab32041f755214dbc6bffa90cc0225b39da2e5c2d3b90600090a250565b60606103f4838360405180606001604052806027815260200161083860279139610560565b9392505050565b6001600160a01b0381166104605760405162461bcd60e51b815260206004820152602660248201527f455243313936373a206e65772061646d696e20697320746865207a65726f206160448201526564647265737360d01b606482015260840161014e565b807fb53127684a568b3173ae13b9f8a6016e243e63b6e8ee1178d6a717850b5d61035b80546001600160a01b0319166001600160a01b039290921691909117905550565b60007f360894a13ba1a3210667c828492db98dca3e2076cc3735a920a3ca505d382bbc61018d565b6001600160a01b0381163b6105395760405162461bcd60e51b815260206004820152602d60248201527f455243313936373a206e657720696d706c656d656e746174696f6e206973206e60448201526c1bdd08184818dbdb9d1c9858dd609a1b606482015260840161014e565b807f360894a13ba1a3210667c828492db98dca3e2076cc3735a920a3ca505d382bbc610483565b6060600080856001600160a01b03168560405161057d91906107e8565b600060405180830381855af49150503d80600081146105b8576040519150601f19603f3d011682016040523d82523d6000602084013e6105bd565b606091505b50915091506105ce868383876105d8565b9695505050505050565b60608315610647578251600003610640576001600160a01b0385163b6106405760405162461bcd60e51b815260206004820152601d60248201527f416464726573733a2063616c6c20746f206e6f6e2d636f6e7472616374000000604482015260640161014e565b5081610651565b6106518383610659565b949350505050565b8151156106695781518083602001fd5b8060405162461bcd60e51b815260040161014e9190610804565b6000808585111561069357600080fd5b838611156106a057600080fd5b5050820193919092039150565b80356001600160a01b03811681146106c457600080fd5b919050565b6000602082840312156106db57600080fd5b6103f4826106ad565b634e487b7160e01b600052604160045260246000fd5b6000806040838503121561070d57600080fd5b610716836106ad565b9150602083013567ffffffffffffffff81111561073257600080fd5b8301601f8101851361074357600080fd5b803567ffffffffffffffff81111561075d5761075d6106e4565b604051601f8201601f19908116603f0116810167ffffffffffffffff8111828210171561078c5761078c6106e4565b6040528181528282016020018710156107a457600080fd5b816020840160208301376000602083830101528093505050509250929050565b60005b838110156107df5781810151838201526020016107c7565b50506000910152565b600082516107fa8184602087016107c4565b9190910192915050565b60208152600082518060208401526108238160408501602087016107c4565b601f01601f1916919091016040019291505056fe416464726573733a206c6f772d6c6576656c2064656c65676174652063616c6c206661696c6564")), - 0, - [ - (U256::from_be_slice(&hex!("360894a13ba1a3210667c828492db98dca3e2076cc3735a920a3ca505d382bbc")), U256::from_be_slice(&hex!("0000000000000000000000003200000000000000000000000000000000000002"))), - (U256::from_be_slice(&hex!("9016d09d72d40fdae2fd8ceac6b6234c7706214fd39c1cd1e609a0528c199300")), U256::from_be_slice(&hex!("000000000000000000000000f39fd6e51aad88f6f4ce6ab8827279cfffb92266"))), - (U256::from_be_slice(&hex!("b53127684a568b3173ae13b9f8a6016e243e63b6e8ee1178d6a717850b5d6103")), U256::from_be_slice(&hex!("00000000000000000000000031ffffffffffffffffffffffffffffffffffffff"))) - ].into_iter().collect(), - )); - - config.data.push(AccountData::new( - address!("3200000000000000000000000000000000000002"), - U256::ZERO, - Bytes::from_static(&hex!("60806040526004361061019c5760003560e01c80638786dba7116100ec578063d761753e1161008a578063e613ae0011610064578063e613ae001461048e578063f119a9bd146104a9578063f2fde38b146104c9578063f8e655d2146104e957600080fd5b8063d761753e14610431578063dd95c7c614610459578063e30c39781461047957600080fd5b8063a41c5cf3116100c6578063a41c5cf3146103af578063b3ab15fb146103c4578063bafa9eb2146103e4578063c045577b1461040457600080fd5b80638786dba71461037257806387f8bf56146103855780638da5cb5b1461039a57600080fd5b8063570ca73511610159578063715018a611610133578063715018a61461031357806374ab4a8314610328578063781952a81461034857806379ba50971461035d57600080fd5b8063570ca735146102945780635d3e3176146102d15780635e3cc740146102f357600080fd5b806311e53a01146101a1578063158ef93e146101e1578063198546231461020b5780634126013714610220578063419759f514610240578063471ba1e314610256575b600080fd5b3480156101ad57600080fd5b506101ce6101bc3660046129ee565b60276020526000908152604090205481565b6040519081526020015b60405180910390f35b3480156101ed57600080fd5b506000546101fb9060ff1681565b60405190151581526020016101d8565b61021e610219366004612a4b565b610509565b005b34801561022c57600080fd5b5061021e61023b366004612afb565b6106e2565b34801561024c57600080fd5b506101ce60215481565b34801561026257600080fd5b506102766102713660046129ee565b6108ef565b604080519283526001600160e01b03199091166020830152016101d8565b3480156102a057600080fd5b506000546102b99061010090046001600160a01b031681565b6040516001600160a01b0390911681526020016101d8565b3480156102dd57600080fd5b506102e6610920565b6040516101d89190612b96565b3480156102ff57600080fd5b5061021e61030e366004612be2565b6109ae565b34801561031f57600080fd5b5061021e610d65565b34801561033457600080fd5b5061021e610343366004612c1e565b610d79565b34801561035457600080fd5b506026546101ce565b34801561036957600080fd5b5061021e610f4c565b61021e610380366004612c88565b610f94565b34801561039157600080fd5b506102e66110bb565b3480156103a657600080fd5b506102b96110c8565b3480156103bb57600080fd5b506102e66110fd565b3480156103d057600080fd5b5061021e6103df366004612cb4565b61110a565b3480156103f057600080fd5b506101fb6103ff3660046129ee565b61117b565b34801561041057600080fd5b506101ce61041f3660046129ee565b60286020526000908152604090205481565b34801561043d57600080fd5b506102b973deaddeaddeaddeaddeaddeaddeaddeaddeaddead81565b34801561046557600080fd5b5061021e610474366004612be2565b6111a6565b34801561048557600080fd5b506102b9611606565b34801561049a57600080fd5b506102b96001603160981b0181565b3480156104b557600080fd5b5061021e6104c4366004612cdd565b61162f565b3480156104d557600080fd5b5061021e6104e4366004612cb4565b6116a3565b3480156104f557600080fd5b5061021e610504366004612d1e565b611728565b82811461054f5760405162461bcd60e51b815260206004820152600f60248201526e098cadccee8d040dad2e6dac2e8c6d608b1b60448201526064015b60405180910390fd5b60215461055d908490612d97565b34146105a55760405162461bcd60e51b8152602060048201526017602482015276125b9d985b1a59081dda5d1a191c985dc8185b5bdd5b9d604a1b6044820152606401610546565b60265460005b848110156106da57600060405180604001604052808888858181106105d2576105d2612dae565b9050602002013581526020018686858181106105f0576105f0612dae565b90506020020160208101906106059190612dc4565b6001600160e01b03191690526026805460018101825560009190915281517f744a2cf8fd7008e3d53b67916e73460df9fa5214e3ef23dd4259ca09493a359460029092029182015560208201517f744a2cf8fd7008e3d53b67916e73460df9fa5214e3ef23dd4259ca09493a3595909101805463ffffffff191660e09290921c91909117905590507f3311a04a346a103ac115cca33028a2bc82f1964805860d0d3fc84a2772496ada816106b98486612ddf565b426040516106c993929190612df2565b60405180910390a1506001016105ab565b505050505050565b3373deaddeaddeaddeaddeaddeaddeaddeaddeaddead146107455760405162461bcd60e51b815260206004820152601f60248201527f63616c6c6572206973206e6f74207468652073797374656d2063616c6c6572006044820152606401610546565b60005460ff16156107985760405162461bcd60e51b815260206004820152601f60248201527f436f6e747261637420697320616c726561647920696e697469616c697a6564006044820152606401610546565b806000036107e85760405162461bcd60e51b815260206004820152601a60248201527f4465706f73697420616d6f756e742063616e6e6f7420626520300000000000006044820152606401610546565b60008490036108095760405162461bcd60e51b815260040161054690612e20565b6000805460ff191660011790556023610823858783612ef0565b506024610831838583612ef0565b50602181905560008054610100600160a81b03191674deaddeaddeaddeaddeaddeaddeaddeaddeaddead001781556040805191825273deaddeaddeaddeaddeaddeaddeaddeaddeaddead60208301527ffbe5b6cbafb274f445d7fed869dc77a838d8243a22c460de156560e8857cad03910160405180910390a17f80bd1fdfe157286ce420ee763f91748455b249605748e5df12dad9844402bafc858585856040516108e09493929190612fd8565b60405180910390a15050505050565b602681815481106108ff57600080fd5b60009182526020909120600290910201805460019091015490915060e01b82565b6025805461092d90612e6d565b80601f016020809104026020016040519081016040528092919081815260200182805461095990612e6d565b80156109a65780601f1061097b576101008083540402835291602001916109a6565b820191906000526020600020905b81548152906001019060200180831161098957829003601f168201915b505050505081565b6109b7816117a2565b5060009050610a066109cc606084018461300a565b8080601f016020809104026020016040519081016040528093929190818152602001838380828437600092019190915250611ac592505050565b915060009050610a63610a1c606085018561300a565b8080601f016020809104026020016040519081016040528093929190818152602001838380828437600092019190915250610a5e925060019150869050613050565b611adc565b90506000610a7082611c54565b90506000610a88610a8383836020611cd6565b611d99565b90506000610aa9610a836020808651610aa19190613050565b869190611cd6565b600083815260276020526040812054919250819003610b015760405162461bcd60e51b815260206004820152601460248201527311195c1bdcda5d08191bc81b9bdd08195e1a5cdd60621b6044820152606401610546565b6000610b4d610b1360808a018a61300a565b8080601f01602080910402602001604051908101604052809392919081815260200183838082843760009201829052509250611e1e915050565b90506000610b5c826001611f01565b9050600060258054610b6d90612e6d565b915060009050610b7e838284611cd6565b9050610c148160258054610b9190612e6d565b80601f0160208091040260200160405190810160405280929190818152602001828054610bbd90612e6d565b8015610c0a5780601f10610bdf57610100808354040283529160200191610c0a565b820191906000526020600020905b815481529060010190602001808311610bed57829003601f168201915b5050505050612085565b610c605760405162461bcd60e51b815260206004820152601a60248201527f496e76616c696420736c6173684f7254616b65207363726970740000000000006044820152606401610546565b6000602881610c70600189613050565b815260200190815260200160002054905060008160001480610c9a5750610c968861215c565b8214155b905080610ce95760405162461bcd60e51b815260206004820152601960248201527f4f70657261746f72206973206e6f74206d616c6963696f7573000000000000006044820152606401610546565b600180896103e88110610cfe57610cfe612dae565b602091828204019190066101000a81548160ff0219169083151502179055507ff918cdaebea74c5a8c3b02d7404c162f507551b158202cedcba9b6a74eabdff288604051610d4e91815260200190565b60405180910390a150505050505050505050505050565b610d6d612169565b610d77600061219b565b565b610d82836117a2565b5060009050610dd3610d97604086018661300a565b8080601f0160208091040260200160405190810160405280939291908181526020018383808284376000920191909152508792506121d7915050565b90506000610de08261232a565b90506000610ded83612337565b9050600060268581548110610e0457610e04612dae565b60009182526020918290206040805180820190915260029290920201805480835260019091015460e01b6001600160e01b03191692820192909252915083148015610e665750816001600160e01b03191681602001516001600160e01b031916145b610ea65760405162461bcd60e51b81526020600482015260116024820152706e6f74206d61746368696e67205554584f60781b6044820152606401610546565b6000610eb86109cc60608a018a61300a565b915060009050610ece610a1c60608b018b61300a565b90506000610edb82611c54565b90506000610ee882611d99565b9050610ef38161215c565b60008a8152602860209081526040918290209290925580518b81529182018390527feedf47c2f61b040827944fd45e44ef6d742354b34e1af7dd99a56f444ec79347910160405180910390a15050505050505050505050565b3380610f56611606565b6001600160a01b031614610f885760405163118cdaa760e01b81526001600160a01b0382166004820152602401610546565b610f918161219b565b50565b6021543414610fdf5760405162461bcd60e51b8152602060048201526017602482015276125b9d985b1a59081dda5d1a191c985dc8185b5bdd5b9d604a1b6044820152606401610546565b6040805180820182528381526001600160e01b03198316602082019081526026805460018101825560009190915282517f744a2cf8fd7008e3d53b67916e73460df9fa5214e3ef23dd4259ca09493a3594600283029081019190915591517f744a2cf8fd7008e3d53b67916e73460df9fa5214e3ef23dd4259ca09493a3595909201805463ffffffff191660e09390931c9290921790915591519091907f3311a04a346a103ac115cca33028a2bc82f1964805860d0d3fc84a2772496ada906110ad90849084904290612df2565b60405180910390a150505050565b6024805461092d90612e6d565b6000807f9016d09d72d40fdae2fd8ceac6b6234c7706214fd39c1cd1e609a0528c1993005b546001600160a01b031692915050565b6023805461092d90612e6d565b611112612169565b60008054610100600160a81b0319166101006001600160a01b038481168281029390931793849055604080519290940416815260208101919091527ffbe5b6cbafb274f445d7fed869dc77a838d8243a22c460de156560e8857cad03910160405180910390a150565b6001816103e8811061118c57600080fd5b60209182820401919006915054906101000a900460ff1681565b60005461010090046001600160a01b031633146112055760405162461bcd60e51b815260206004820152601a60248201527f63616c6c6572206973206e6f7420746865206f70657261746f720000000000006044820152606401610546565b600080611211836117a2565b915091508060011461125e5760405162461bcd60e51b815260206004820152601660248201527513db9b1e481bdb99481a5b9c1d5d08185b1b1bddd95960521b6044820152606401610546565b60006113096112706020860186612dc4565b61127d604087018761300a565b8080601f0160208091040260200160405190810160405280939291908181526020018383808284376000920191909152506112bf92505050606088018861300a565b8080601f0160208091040260200160405190810160405280939291908181526020018383808284376000920191909152506113049250505060c0890160a08a01612dc4565b612344565b6000818152602760205260409020549091501561135d5760405162461bcd60e51b81526020600482015260126024820152711d1e125908185b1c9958591e481cdc195b9d60721b6044820152606401610546565b60226000815461136c90613063565b9182905550600082815260276020526040812091909155611393610b13608087018761300a565b905060006113a082611ac5565b915050806003146113eb5760405162461bcd60e51b8152602060048201526015602482015274496e76616c6964207769746e657373206974656d7360581b6044820152606401610546565b60006113f8836001611f01565b905060006023805461140990612e6d565b91506000905061141a838284611cd6565b905061142d8160238054610b9190612e6d565b6114725760405162461bcd60e51b8152602060048201526016602482015275125b9d985b1a590819195c1bdcda5d081cd8dc9a5c1d60521b6044820152606401610546565b60006114a66024805461148490612e6d565b86516114909250613050565b6024805461149d90612e6d565b87929150611cd6565b90506114b98160248054610b9190612e6d565b6114fd5760405162461bcd60e51b8152602060048201526015602482015274092dcecc2d8d2c840e6c6e4d2e0e840e6eaccccd2f605b1b6044820152606401610546565b600061150885612374565b602254604080518d8152602081018c90526001600160a01b038416818301524260608201526080810192909252519192507fa82453ca34121b3ecb910d957824e27c5dc6465315949facd15fb72886490058919081900360a00190a16021546040516000916001600160a01b038416918381818185875af1925050503d80600081146115b0576040519150601f19603f3d011682016040523d82523d6000602084013e6115b5565b606091505b50509050806115f85760405162461bcd60e51b815260206004820152600f60248201526e151c985b9cd9995c8819985a5b1959608a1b6044820152606401610546565b505050505050505050505050565b6000807f237e158222e3e6968b72b9db0d8043aacf074ad9f650f0d1606b4d82ee432c006110ed565b611637612169565b60008190036116585760405162461bcd60e51b815260040161054690612e20565b6025611665828483612ef0565b507f8578c80bdea3ff51431011ed88db9cb415de2cf64f9ed5e7137288268cbdeb2c828260405161169792919061307c565b60405180910390a15050565b6116ab612169565b7f237e158222e3e6968b72b9db0d8043aacf074ad9f650f0d1606b4d82ee432c0080546001600160a01b0319166001600160a01b03831690811782556116ef6110c8565b6001600160a01b03167f38d16b8cac22d99fc7c124b9cd0de2d3fa1faef420bfe791d8c362d765e2270060405160405180910390a35050565b611730612169565b60008390036117515760405162461bcd60e51b815260040161054690612e20565b602361175e848683612ef0565b50602461176c828483612ef0565b507f80bd1fdfe157286ce420ee763f91748455b249605748e5df12dad9844402bafc848484846040516110ad9493929190612fd8565b600080806118026117b66020860186612dc4565b6117c66040870160208801613090565b6117d3604088018861300a565b6117e060608a018a61300a565b6117ed60808c018c61300a565b6117fd60c08e0160a08f01612dc4565b6123aa565b905061184e611814604086018661300a565b8080601f0160208091040260200160405190810160405280939291908181526020018383808284376000920191909152506123de92505050565b61189a5760405162461bcd60e51b815260206004820152601d60248201527f56696e206973206e6f742070726f7065726c7920666f726d61747465640000006044820152606401610546565b6118e46118aa606086018661300a565b8080601f01602080910402602001604051908101604052809392919081815260200183838082843760009201919091525061248292505050565b6119305760405162461bcd60e51b815260206004820152601e60248201527f566f7574206973206e6f742070726f7065726c7920666f726d617474656400006044820152606401610546565b60006119426109cc604087018761300a565b91506119929050611956608087018761300a565b8080601f016020809104026020016040519081016040528093929190818152602001838380828437600092019190915250859250612519915050565b6119e85760405162461bcd60e51b815260206004820152602160248201527f5769746e657373206973206e6f742070726f7065726c7920666f726d617474656044820152601960fa1b6064820152608401610546565b6001603160981b01634ffd344a60e087013584611a0860c08a018a61300a565b8a61010001356040518663ffffffff1660e01b8152600401611a2e9594939291906130ba565b602060405180830381865afa158015611a4b573d6000803e3d6000fd5b505050506040513d601f19601f82011682018060405250810190611a6f91906130ec565b611abb5760405162461bcd60e51b815260206004820152601b60248201527f5472616e73616374696f6e206973206e6f7420696e20626c6f636b00000000006044820152606401610546565b9094909350915050565b600080611ad383600061258e565b91509150915091565b6060600080611aea85611ac5565b909250905060018201611b0f5760405162461bcd60e51b81526004016105469061310e565b808410611b525760405162461bcd60e51b81526020600482015260116024820152702b37baba103932b0b21037bb32b9393ab760791b6044820152606401610546565b600080611b60846001612ddf565b905060005b86811015611bde57611b778883612730565b92506000198303611bca5760405162461bcd60e51b815260206004820152601a60248201527f42616420566172496e7420696e207363726970745075626b65790000000000006044820152606401610546565b611bd48383612ddf565b9150600101611b65565b50611be98782612730565b91506000198203611c3c5760405162461bcd60e51b815260206004820152601a60248201527f42616420566172496e7420696e207363726970745075626b65790000000000006044820152606401610546565b611c47878284611cd6565b9450505050505b92915050565b606081600981518110611c6957611c69612dae565b6020910101516001600160f81b031916603560f91b14611c9757505060408051602081019091526000815290565b600082600a81518110611cac57611cac612dae565b01602001516001600160f81b031981169150611ccf908490600b9060f81c611cd6565b9392505050565b606081600003611cf55750604080516020810190915260008152611ccf565b6000611d018385612ddf565b90508381118015611d13575080855110155b611d555760405162461bcd60e51b8152602060048201526013602482015272536c696365206f7574206f6620626f756e647360681b6044820152606401610546565b604051915082604083010160405282825283850182038460208701018481015b80821015611d8e57815183830152602082019150611d75565b505050509392505050565b60008151600003611dac57506000919050565b81516020811115611e0a5760405162461bcd60e51b815260206004820152602260248201527f42797465732063616e6e6f74206265206d6f7265207468616e20333220627974604482015261657360f01b6064820152608401610546565b60209283015192036008029190911c919050565b606060008060005b84811015611e9757611e388683612794565b92506000198303611e835760405162461bcd60e51b815260206004820152601560248201527442616420566172496e7420696e207769746e65737360581b6044820152606401610546565b611e8d8383612ddf565b9150600101611e26565b50611ea28582612794565b91506000198203611eed5760405162461bcd60e51b815260206004820152601560248201527442616420566172496e7420696e207769746e65737360581b6044820152606401610546565b611ef8858284611cd6565b95945050505050565b6060600080611f0f85611ac5565b909250905060018201611f345760405162461bcd60e51b81526004016105469061310e565b808410611f765760405162461bcd60e51b815260206004820152601060248201526f2b34b7103932b0b21037bb32b9393ab760811b6044820152606401610546565b600080611f84846001612ddf565b905060005b8681101561200f57611f9b888361258e565b909550925060018301611fe55760405162461bcd60e51b815260206004820152601260248201527142616420566172496e7420696e206974656d60701b6044820152606401610546565b82611ff1866001612ddf565b611ffb9190612ddf565b6120059083612ddf565b9150600101611f89565b5061201a878261258e565b9094509150600182016120645760405162461bcd60e51b815260206004820152601260248201527142616420566172496e7420696e206974656d60701b6044820152606401610546565b611c47816120728685612ddf565b61207d906001612ddf565b899190611cd6565b8151815160009190811461209d576000915050611c4e565b60206000805b8383116120d357505084810151848201516020909201918082146120ce576000945050505050611c4e565b6120a3565b60006120e0602085613050565b90505b8481101561214e578681815181106120fd576120fd612dae565b602001015160f81c60f81b6001600160f81b03191688828151811061212457612124612dae565b01602001516001600160f81b0319161461214657600095505050505050611c4e565b6001016120e3565b506001979650505050505050565b6000611c4e826001612ddf565b336121726110c8565b6001600160a01b031614610d775760405163118cdaa760e01b8152336004820152602401610546565b7f237e158222e3e6968b72b9db0d8043aacf074ad9f650f0d1606b4d82ee432c0080546001600160a01b03191681556121d382612836565b5050565b60606000806121e585611ac5565b90925090506001820161220a5760405162461bcd60e51b81526004016105469061310e565b80841061224c5760405162461bcd60e51b815260206004820152601060248201526f2b34b7103932b0b21037bb32b9393ab760811b6044820152606401610546565b60008061225a846001612ddf565b905060005b868110156122d25761227188836128a7565b925060001983036122be5760405162461bcd60e51b815260206004820152601760248201527642616420566172496e7420696e2073637269707453696760481b6044820152606401610546565b6122c88383612ddf565b915060010161225f565b506122dd87826128a7565b91506000198203611c3c5760405162461bcd60e51b815260206004820152601760248201527642616420566172496e7420696e2073637269707453696760481b6044820152606401610546565b6020810151600090611c4e565b6000611c4e8260206128f0565b6000611ef8858585856040516020016123609493929190613150565b6040516020818303038152906040526128ff565b6000806023805461238490612e6d565b91506000905061239684836014611cd6565b61239f906131ad565b60601c949350505050565b60006123d08a8a8a8a8a8a8a8a8a60405160200161236099989796959493929190613200565b9a9950505050505050505050565b60008060006123ec84611ac5565b90925090508015806123ff575060001982145b1561240e575060009392505050565b600061241b836001612ddf565b905060005b82811015612475578551821061243c5750600095945050505050565b600061244887846128a7565b90506000198103612460575060009695505050505050565b61246a8184612ddf565b925050600101612420565b5093519093149392505050565b600080600061249084611ac5565b90925090508015806124a3575060001982145b156124b2575060009392505050565b60006124bf836001612ddf565b905060005b8281101561247557855182106124e05750600095945050505050565b60006124ec8784612730565b90506000198103612504575060009695505050505050565b61250e8184612ddf565b9250506001016124c4565b60008160000361252b57506000611c4e565b6000805b83811015612582578451821061254a57600092505050611c4e565b60006125568684612794565b9050600019810361256d5760009350505050611c4e565b6125778184612ddf565b92505060010161252f565b50835114905092915050565b600080600061259d8585612926565b90508060ff166000036125d25760008585815181106125be576125be612dae565b016020015190935060f81c91506127299050565b836125de826001613269565b60ff166125eb9190612ddf565b855110156126025760001960009250925050612729565b60008160ff166002036126465761263b612627612620876001612ddf565b88906128f0565b62ffff0060e882901c1660f89190911c1790565b61ffff16905061271f565b8160ff1660040361269557612688612662612620876001612ddf565b60d881901c63ff00ff001662ff00ff60e89290921c9190911617601081811b91901c1790565b63ffffffff16905061271f565b8160ff1660080361271f576127136126b1612620876001612ddf565b60c01c64ff000000ff600882811c91821665ff000000ff009390911b92831617601090811b6001600160401b031666ff00ff00ff00ff9290921667ff00ff00ff00ff009093169290921790911c65ffff0000ffff1617602081811c91901b1790565b6001600160401b031690505b60ff909116925090505b9250929050565b600061273d826009612ddf565b8351101561274e5750600019611c4e565b60008061276585612760866008612ddf565b61258e565b90925090506001820161277e5760001992505050611c4e565b8061278a836009612ddf565b611ef89190612ddf565b60008060006127a3858561258e565b9092509050600182016127bc5760001992505050611c4e565b6000806127ca846001612ddf565b905060005b8381101561282b576127e588612760848a612ddf565b9095509250600183016128015760001995505050505050611c4e565b8261280d866001612ddf565b6128179190612ddf565b6128219083612ddf565b91506001016127cf565b509695505050505050565b7f9016d09d72d40fdae2fd8ceac6b6234c7706214fd39c1cd1e609a0528c19930080546001600160a01b031981166001600160a01b03848116918217845560405192169182907f8be0079c531659141344cd1fd0a4f28419497f9722a3daafe3b4186f6b6457e090600090a3505050565b60008060006128b685856129ac565b9092509050600182016128cf5760001992505050611c4e565b806128db836025612ddf565b6128e59190612ddf565b611ef8906004612ddf565b6000611ccf8383016020015190565b60006020600083516020850160025afa50602060006020600060025afa5050600051919050565b600082828151811061293a5761293a612dae565b016020015160f81c60ff0361295157506008611c4e565b82828151811061296357612963612dae565b016020015160f81c60fe0361297a57506004611c4e565b82828151811061298c5761298c612dae565b016020015160f81c60fd036129a357506002611c4e565b50600092915050565b6000806129ba836025612ddf565b845110156129cf575060001990506000612729565b6000806129e186612760876024612ddf565b9097909650945050505050565b600060208284031215612a0057600080fd5b5035919050565b60008083601f840112612a1957600080fd5b5081356001600160401b03811115612a3057600080fd5b6020830191508360208260051b850101111561272957600080fd5b60008060008060408587031215612a6157600080fd5b84356001600160401b03811115612a7757600080fd5b612a8387828801612a07565b90955093505060208501356001600160401b03811115612aa257600080fd5b612aae87828801612a07565b95989497509550505050565b60008083601f840112612acc57600080fd5b5081356001600160401b03811115612ae357600080fd5b60208301915083602082850101111561272957600080fd5b600080600080600060608688031215612b1357600080fd5b85356001600160401b03811115612b2957600080fd5b612b3588828901612aba565b90965094505060208601356001600160401b03811115612b5457600080fd5b612b6088828901612aba565b96999598509660400135949350505050565b60005b83811015612b8d578181015183820152602001612b75565b50506000910152565b6020815260008251806020840152612bb5816040850160208701612b72565b601f01601f19169190910160400192915050565b60006101208284031215612bdc57600080fd5b50919050565b600060208284031215612bf457600080fd5b81356001600160401b03811115612c0a57600080fd5b612c1684828501612bc9565b949350505050565b600080600060608486031215612c3357600080fd5b83356001600160401b03811115612c4957600080fd5b612c5586828701612bc9565b9660208601359650604090950135949350505050565b80356001600160e01b031981168114612c8357600080fd5b919050565b60008060408385031215612c9b57600080fd5b82359150612cab60208401612c6b565b90509250929050565b600060208284031215612cc657600080fd5b81356001600160a01b0381168114611ccf57600080fd5b60008060208385031215612cf057600080fd5b82356001600160401b03811115612d0657600080fd5b612d1285828601612aba565b90969095509350505050565b60008060008060408587031215612d3457600080fd5b84356001600160401b03811115612d4a57600080fd5b612d5687828801612aba565b90955093505060208501356001600160401b03811115612d7557600080fd5b612aae87828801612aba565b634e487b7160e01b600052601160045260246000fd5b8082028115828204841417611c4e57611c4e612d81565b634e487b7160e01b600052603260045260246000fd5b600060208284031215612dd657600080fd5b611ccf82612c6b565b80820180821115611c4e57611c4e612d81565b835181526020938401516001600160e01b031916938101939093526040830191909152606082015260800190565b6020808252601e908201527f4465706f736974207363726970742063616e6e6f7420626520656d7074790000604082015260600190565b634e487b7160e01b600052604160045260246000fd5b600181811c90821680612e8157607f821691505b602082108103612bdc57634e487b7160e01b600052602260045260246000fd5b601f821115612eeb57806000526020600020601f840160051c81016020851015612ec85750805b601f840160051c820191505b81811015612ee85760008155600101612ed4565b50505b505050565b6001600160401b03831115612f0757612f07612e57565b612f1b83612f158354612e6d565b83612ea1565b6000601f841160018114612f4f5760008515612f375750838201355b600019600387901b1c1916600186901b178355612ee8565b600083815260209020601f19861690835b82811015612f805786850135825560209485019460019092019101612f60565b5086821015612f9d5760001960f88860031b161c19848701351681555b505060018560011b0183555050505050565b81835281816020850137506000828201602090810191909152601f909101601f19169091010190565b604081526000612fec604083018688612faf565b8281036020840152612fff818587612faf565b979650505050505050565b6000808335601e1984360301811261302157600080fd5b8301803591506001600160401b0382111561303b57600080fd5b60200191503681900382131561272957600080fd5b81810381811115611c4e57611c4e612d81565b60006001820161307557613075612d81565b5060010190565b602081526000612c16602083018486612faf565b6000602082840312156130a257600080fd5b81356001600160f01b031981168114611ccf57600080fd5b8581528460208201526080604082015260006130da608083018587612faf565b90508260608301529695505050505050565b6000602082840312156130fe57600080fd5b81518015158114611ccf57600080fd5b60208082526022908201527f52656164206f76657272756e20647572696e6720566172496e742070617273696040820152616e6760f01b606082015260800190565b6001600160e01b0319851681528351600090613173816004850160208901612b72565b84519083019061318a816004840160208901612b72565b6001600160e01b0319949094169301600481019390935250506008019392505050565b805160208201516bffffffffffffffffffffffff198116919060148210156131f9576bffffffffffffffffffffffff196bffffffffffffffffffffffff198360140360031b1b82161692505b5050919050565b6001600160e01b03198a1681526001600160f01b031989166004820152868860068301376000878201600681016000815287898237506000908701600601908152848682376001600160e01b031993909316929093019182525060040198975050505050505050565b60ff8181168382160190811115611c4e57611c4e612d8156")), - 0, - HashMap::new() - )); - - config.data.push(AccountData::new( - address!("31ffffffffffffffffffffffffffffffffffffff"), - U256::ZERO, - Bytes::from_static(&hex!("60806040526004361061007b5760003560e01c80639623609d1161004e5780639623609d1461011157806399a88ec414610124578063f2fde38b14610144578063f3b7dead1461016457600080fd5b8063204e1c7a14610080578063715018a6146100bc5780637eff275e146100d35780638da5cb5b146100f3575b600080fd5b34801561008c57600080fd5b506100a061009b366004610499565b610184565b6040516001600160a01b03909116815260200160405180910390f35b3480156100c857600080fd5b506100d1610215565b005b3480156100df57600080fd5b506100d16100ee3660046104bd565b610229565b3480156100ff57600080fd5b506000546001600160a01b03166100a0565b6100d161011f36600461050c565b610291565b34801561013057600080fd5b506100d161013f3660046104bd565b610300565b34801561015057600080fd5b506100d161015f366004610499565b610336565b34801561017057600080fd5b506100a061017f366004610499565b6103b4565b6000806000836001600160a01b03166040516101aa90635c60da1b60e01b815260040190565b600060405180830381855afa9150503d80600081146101e5576040519150601f19603f3d011682016040523d82523d6000602084013e6101ea565b606091505b5091509150816101f957600080fd5b8080602001905181019061020d91906105ea565b949350505050565b61021d6103da565b6102276000610434565b565b6102316103da565b6040516308f2839760e41b81526001600160a01b038281166004830152831690638f283970906024015b600060405180830381600087803b15801561027557600080fd5b505af1158015610289573d6000803e3d6000fd5b505050505050565b6102996103da565b60405163278f794360e11b81526001600160a01b03841690634f1ef2869034906102c99086908690600401610607565b6000604051808303818588803b1580156102e257600080fd5b505af11580156102f6573d6000803e3d6000fd5b5050505050505050565b6103086103da565b604051631b2ce7f360e11b81526001600160a01b038281166004830152831690633659cfe69060240161025b565b61033e6103da565b6001600160a01b0381166103a85760405162461bcd60e51b815260206004820152602660248201527f4f776e61626c653a206e6577206f776e657220697320746865207a65726f206160448201526564647265737360d01b60648201526084015b60405180910390fd5b6103b181610434565b50565b6000806000836001600160a01b03166040516101aa906303e1469160e61b815260040190565b6000546001600160a01b031633146102275760405162461bcd60e51b815260206004820181905260248201527f4f776e61626c653a2063616c6c6572206973206e6f7420746865206f776e6572604482015260640161039f565b600080546001600160a01b038381166001600160a01b0319831681178455604051919092169283917f8be0079c531659141344cd1fd0a4f28419497f9722a3daafe3b4186f6b6457e09190a35050565b6001600160a01b03811681146103b157600080fd5b6000602082840312156104ab57600080fd5b81356104b681610484565b9392505050565b600080604083850312156104d057600080fd5b82356104db81610484565b915060208301356104eb81610484565b809150509250929050565b634e487b7160e01b600052604160045260246000fd5b60008060006060848603121561052157600080fd5b833561052c81610484565b9250602084013561053c81610484565b9150604084013567ffffffffffffffff81111561055857600080fd5b8401601f8101861361056957600080fd5b803567ffffffffffffffff811115610583576105836104f6565b604051601f8201601f19908116603f0116810167ffffffffffffffff811182821017156105b2576105b26104f6565b6040528181528282016020018810156105ca57600080fd5b816020840160208301376000602083830101528093505050509250925092565b6000602082840312156105fc57600080fd5b81516104b681610484565b60018060a01b0383168152604060208201526000825180604084015260005b818110156106435760208186018101516060868401015201610626565b506000606082850101526060601f19601f830116840101915050939250505056")), - 0, - [ - (U256::from_be_slice(&hex!("0000000000000000000000000000000000000000000000000000000000000000")), U256::from_be_slice(&hex!("000000000000000000000000f39fd6e51aad88f6f4ce6ab8827279cfffb92266"))) - ].into_iter().collect(), - )); -} diff --git a/crates/evm/src/tests/utils.rs b/crates/evm/src/tests/utils.rs index ba941fa1f..f7f5edb30 100644 --- a/crates/evm/src/tests/utils.rs +++ b/crates/evm/src/tests/utils.rs @@ -1,23 +1,32 @@ +use std::collections::HashMap; +use std::path::Path; + +use alloy_eips::eip1559::BaseFeeParams; use lazy_static::lazy_static; +use reth_primitives::constants::ETHEREUM_BLOCK_GAS_LIMIT; use reth_primitives::hex_literal::hex; -use reth_primitives::B256; +use reth_primitives::{address, Address, Bytes, TxKind, B256}; +use revm::primitives::{SpecId, KECCAK_EMPTY, U256}; use sov_modules_api::default_context::DefaultContext; use sov_modules_api::hooks::HookSoftConfirmationInfo; use sov_modules_api::{Module, WorkingSet}; use sov_prover_storage_manager::{new_orphan_storage, SnapshotManager}; -use sov_rollup_interface::spec::SpecId; +use sov_rollup_interface::spec::SpecId as SovSpecId; use sov_state::{DefaultStorageSpec, ProverStorage, Storage}; +use sov_stf_runner::read_json_file; -use crate::{Evm, EvmConfig}; +use crate::smart_contracts::{LogsContract, SimpleStorageContract, TestContract}; +use crate::tests::test_signer::TestSigner; +use crate::{AccountData, Evm, EvmConfig, RlpEvmTransaction, PRIORITY_FEE_VAULT}; type C = DefaultContext; lazy_static! { pub(crate) static ref GENESIS_HASH: B256 = B256::from(hex!( - "9fcaf6e03bf17a3372e03c5f3aa293dee54f73545462f82ba7875710da4604d5" + "82d04e48b7bbcc7239fc123b0eda02586c5f2d45557d66a313183a7f1626f5a6" )); pub(crate) static ref GENESIS_STATE_ROOT: B256 = B256::from(hex!( - "5a4c1a83d16c771fa4221e0353ef5e2af558dbe11ce429e677914292428dec1c" + "5c5e936b06651c65b4e539500afa8563122173f04f1da9c812d717c0064c1051" )); } @@ -61,7 +70,7 @@ pub(crate) fn get_evm(config: &EvmConfig) -> (Evm, WorkingSet) { da_slot_height: 1, da_slot_txs_commitment: [2u8; 32], pre_state_root: root.to_vec(), - current_spec: SpecId::Genesis, + current_spec: SovSpecId::Genesis, pub_key: vec![], deposit_data: vec![], l1_fee_rate: 0, @@ -103,3 +112,181 @@ pub(crate) fn commit( root.0 } + +/// Loads the genesis configuration from the given path and pushes the accounts to the evm config +pub(crate) fn config_push_contracts(config: &mut EvmConfig, path: Option<&str>) { + let mut genesis_config: EvmConfig = read_json_file(Path::new( + path.unwrap_or("../../resources/test-data/integration-tests/evm.json"), + )) + .expect("Failed to read genesis configuration"); + config.data.append(&mut genesis_config.data); +} + +pub fn create_contract_message( + dev_signer: &TestSigner, + nonce: u64, + contract: T, +) -> RlpEvmTransaction { + dev_signer + .sign_default_transaction(TxKind::Create, contract.byte_code(), nonce, 0) + .unwrap() +} +pub(crate) fn create_contract_message_with_fee( + dev_signer: &TestSigner, + nonce: u64, + contract: T, + max_fee_per_gas: u128, +) -> RlpEvmTransaction { + dev_signer + .sign_default_transaction_with_fee( + TxKind::Create, + contract.byte_code(), + nonce, + 0, + max_fee_per_gas, + ) + .unwrap() +} +pub(crate) fn create_contract_transaction( + dev_signer: &TestSigner, + nonce: u64, + contract: T, +) -> RlpEvmTransaction { + dev_signer + .sign_default_transaction(TxKind::Create, contract.byte_code(), nonce, 0) + .unwrap() +} + +pub(crate) fn set_arg_message( + contract_addr: Address, + dev_signer: &TestSigner, + nonce: u64, + set_arg: u32, +) -> RlpEvmTransaction { + let contract = SimpleStorageContract::default(); + + dev_signer + .sign_default_transaction( + TxKind::Call(contract_addr), + contract.set_call_data(set_arg), + nonce, + 0, + ) + .unwrap() +} + +pub(crate) fn publish_event_message( + contract_addr: Address, + signer: &TestSigner, + nonce: u64, + message: String, +) -> RlpEvmTransaction { + let contract = LogsContract::default(); + + signer + .sign_default_transaction( + TxKind::Call(contract_addr), + contract.publish_event(message), + nonce, + 0, + ) + .unwrap() +} + +pub(crate) fn get_evm_config( + signer_balance: U256, + block_gas_limit: Option, +) -> (EvmConfig, TestSigner, Address) { + let dev_signer: TestSigner = TestSigner::new_random(); + + let contract_addr = address!("819c5497b157177315e1204f52e588b393771719"); + let mut config = EvmConfig { + data: vec![AccountData { + address: dev_signer.address(), + balance: signer_balance, + code_hash: KECCAK_EMPTY, + code: Bytes::default(), + nonce: 0, + storage: Default::default(), + }], + spec: vec![(0, SpecId::SHANGHAI)].into_iter().collect(), + block_gas_limit: block_gas_limit.unwrap_or(ETHEREUM_BLOCK_GAS_LIMIT), + ..Default::default() + }; + config_push_contracts(&mut config, None); + (config, dev_signer, contract_addr) +} + +pub(crate) fn get_evm_config_starting_base_fee( + signer_balance: U256, + block_gas_limit: Option, + starting_base_fee: u64, +) -> (EvmConfig, TestSigner, Address) { + let dev_signer: TestSigner = TestSigner::new_random(); + + let contract_addr = address!("819c5497b157177315e1204f52e588b393771719"); + let mut config = EvmConfig { + data: vec![AccountData { + address: dev_signer.address(), + balance: signer_balance, + code_hash: KECCAK_EMPTY, + code: Bytes::default(), + nonce: 0, + storage: Default::default(), + }], + spec: vec![(0, SpecId::SHANGHAI)].into_iter().collect(), + block_gas_limit: block_gas_limit.unwrap_or(ETHEREUM_BLOCK_GAS_LIMIT), + starting_base_fee, + coinbase: PRIORITY_FEE_VAULT, + ..Default::default() + }; + config_push_contracts(&mut config, None); + (config, dev_signer, contract_addr) +} +pub(crate) fn get_evm_test_config() -> EvmConfig { + let mut config = EvmConfig { + data: vec![AccountData { + address: Address::from([1u8; 20]), + balance: U256::checked_mul(U256::from(1000), U256::pow(U256::from(10), U256::from(18))).unwrap(), // 1000 ETH + code_hash: KECCAK_EMPTY, + code: Bytes::default(), + nonce: 0, + storage: Default::default(), + }, + AccountData { + address:Address::from([2u8; 20]), + balance: U256::checked_mul(U256::from(1000), + U256::pow(U256::from(10), U256::from(18))).unwrap(), // 1000 ETH, + code_hash: hex!("4e8ee9adb469b245e3a5a8e58e9b733aaa857a9dce1982257531db8a2700aabf").into(), + code: hex!("60606040526000357c0100000000000000000000000000000000000000000000000000000000900463ffffffff168063a223e05d1461006a578063").into(), + storage: { + let mut storage = HashMap::new(); + storage.insert(U256::from(0), U256::from(0x4321)); + storage.insert( + U256::from_be_slice( + &hex!("6661e9d6d8b923d5bbaab1b96e1dd51ff6ea2a93520fdc9eb75d059238b8c5e9")[..], + ), + U256::from(8), + ); + + storage + }, + nonce: 1 + }], + spec: vec![(0, SpecId::BERLIN), (1, SpecId::SHANGHAI)] + .into_iter() + .collect(), + chain_id: 1000, + block_gas_limit: reth_primitives::constants::ETHEREUM_BLOCK_GAS_LIMIT, + coinbase: Address::from([3u8; 20]), + limit_contract_code_size: Some(5000), + starting_base_fee: 1000000000, + base_fee_params: BaseFeeParams::ethereum(), + timestamp: 0, + difficulty: U256::ZERO, + extra_data: Bytes::default(), + nonce: 0, + }; + config_push_contracts(&mut config, None); + config +} From 84e8fd8eddd099d6539d10b902c21acffc793696 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ege=20Okan=20=C3=9Cnald=C4=B1?= <35339130+exeokan@users.noreply.github.com> Date: Mon, 7 Oct 2024 13:28:31 +0300 Subject: [PATCH 15/28] make prover config arg not optional (#1278) --- bin/citrea/src/rollup/mod.rs | 2 +- crates/prover/src/runner.rs | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bin/citrea/src/rollup/mod.rs b/bin/citrea/src/rollup/mod.rs index f6cf5b2ec..6bede76d2 100644 --- a/bin/citrea/src/rollup/mod.rs +++ b/bin/citrea/src/rollup/mod.rs @@ -333,7 +333,7 @@ pub trait CitreaRollupBlueprint: RollupBlueprint { storage_manager, init_variant, Arc::new(prover_service), - Some(prover_config), + prover_config, code_commitments_by_spec, fork_manager, soft_confirmation_tx, diff --git a/crates/prover/src/runner.rs b/crates/prover/src/runner.rs index 668828389..59fdd973e 100644 --- a/crates/prover/src/runner.rs +++ b/crates/prover/src/runner.rs @@ -63,7 +63,7 @@ where sequencer_pub_key: Vec, sequencer_da_pub_key: Vec, phantom: std::marker::PhantomData, - prover_config: Option, + prover_config: ProverConfig, code_commitments_by_spec: HashMap, l1_block_cache: Arc>>, sync_blocks_count: u64, @@ -107,7 +107,7 @@ where mut storage_manager: Sm, init_variant: InitVariant, prover_service: Arc, - prover_config: Option, + prover_config: ProverConfig, code_commitments_by_spec: HashMap, fork_manager: ForkManager, soft_confirmation_tx: broadcast::Sender, @@ -244,7 +244,7 @@ where }; let ledger_db = self.ledger_db.clone(); - let prover_config = self.prover_config.clone().unwrap(); + let prover_config = self.prover_config.clone(); let prover_service = self.prover_service.clone(); let da_service = self.da_service.clone(); let sequencer_pub_key = self.sequencer_pub_key.clone(); From 462565b8d60ec5c2ccf911d55c4088ced2b80d6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ahmet=20Yaz=C4=B1c=C4=B1?= <75089142+yaziciahmet@users.noreply.github.com> Date: Mon, 7 Oct 2024 17:11:38 +0200 Subject: [PATCH 16/28] Pin to 27.1 (#1279) --- bin/citrea/tests/bitcoin_e2e/config/bitcoin.rs | 2 +- bin/citrea/tests/bitcoin_e2e/config/docker.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/citrea/tests/bitcoin_e2e/config/bitcoin.rs b/bin/citrea/tests/bitcoin_e2e/config/bitcoin.rs index 447bad3e6..734c29792 100644 --- a/bin/citrea/tests/bitcoin_e2e/config/bitcoin.rs +++ b/bin/citrea/tests/bitcoin_e2e/config/bitcoin.rs @@ -29,7 +29,7 @@ impl Default for BitcoinConfig { .into_path(), extra_args: Vec::new(), network: Network::Regtest, - docker_image: Some("bitcoin/bitcoin:latest".to_string()), + docker_image: Some("bitcoin/bitcoin:27.1-alpine".to_string()), env: Vec::new(), idx: 0, } diff --git a/bin/citrea/tests/bitcoin_e2e/config/docker.rs b/bin/citrea/tests/bitcoin_e2e/config/docker.rs index 4de442660..5f57a469c 100644 --- a/bin/citrea/tests/bitcoin_e2e/config/docker.rs +++ b/bin/citrea/tests/bitcoin_e2e/config/docker.rs @@ -34,7 +34,7 @@ impl From<&BitcoinConfig> for DockerConfig { image: v .docker_image .clone() - .unwrap_or_else(|| "bitcoin/bitcoin:latest".to_string()), + .unwrap_or_else(|| "bitcoin/bitcoin:27.1-alpine".to_string()), cmd: args, log_path: v.data_dir.join("regtest").join("debug.log"), volume: VolumeConfig { From 5aa4b9ff9e3b59452f9a6e34030d7a5185550df7 Mon Sep 17 00:00:00 2001 From: jfldde <168934971+jfldde@users.noreply.github.com> Date: Tue, 8 Oct 2024 09:35:41 +0100 Subject: [PATCH 17/28] E2E tests using citrea-e2e (#1277) * E2E tests using citrea-e2e * Lint * Set CITREA path * With github.workspace * Target citrea-e2e main * Update citrea-e2e rev * Use debug build * Update citrea-e2e --- .github/workflows/checks.yml | 2 + Cargo.lock | 125 ++++-- Cargo.toml | 2 - bin/citrea/Cargo.toml | 5 +- bin/citrea/tests/all_tests.rs | 1 - bin/citrea/tests/bitcoin_e2e/bitcoin.rs | 366 ------------------ .../bitcoin_e2e/{tests => }/bitcoin_test.rs | 11 +- .../tests/bitcoin_e2e/config/bitcoin.rs | 64 --- bin/citrea/tests/bitcoin_e2e/config/docker.rs | 77 ---- bin/citrea/tests/bitcoin_e2e/config/mod.rs | 30 -- bin/citrea/tests/bitcoin_e2e/config/rollup.rs | 68 ---- bin/citrea/tests/bitcoin_e2e/config/test.rs | 12 - .../tests/bitcoin_e2e/config/test_case.rs | 73 ---- bin/citrea/tests/bitcoin_e2e/config/utils.rs | 14 - bin/citrea/tests/bitcoin_e2e/docker.rs | 282 -------------- bin/citrea/tests/bitcoin_e2e/framework.rs | 200 ---------- bin/citrea/tests/bitcoin_e2e/full_node.rs | 181 --------- .../bitcoin_e2e/{tests => }/mempool_accept.rs | 12 +- bin/citrea/tests/bitcoin_e2e/mod.rs | 20 +- bin/citrea/tests/bitcoin_e2e/node.rs | 165 -------- bin/citrea/tests/bitcoin_e2e/prover.rs | 140 ------- .../bitcoin_e2e/{tests => }/prover_test.rs | 30 +- bin/citrea/tests/bitcoin_e2e/sequencer.rs | 143 ------- .../{tests => }/sequencer_commitments.rs | 62 +-- .../bitcoin_e2e/{tests => }/sequencer_test.rs | 47 ++- bin/citrea/tests/bitcoin_e2e/test_case.rs | 344 ---------------- bin/citrea/tests/bitcoin_e2e/tests/mod.rs | 5 - bin/citrea/tests/bitcoin_e2e/utils.rs | 149 ------- crates/sequencer/src/lib.rs | 1 + .../sov-modules-macros/Cargo.toml | 2 +- resources/scripts/cycle-diff.sh | 4 +- 31 files changed, 184 insertions(+), 2453 deletions(-) delete mode 100644 bin/citrea/tests/bitcoin_e2e/bitcoin.rs rename bin/citrea/tests/bitcoin_e2e/{tests => }/bitcoin_test.rs (92%) delete mode 100644 bin/citrea/tests/bitcoin_e2e/config/bitcoin.rs delete mode 100644 bin/citrea/tests/bitcoin_e2e/config/docker.rs delete mode 100644 bin/citrea/tests/bitcoin_e2e/config/mod.rs delete mode 100644 bin/citrea/tests/bitcoin_e2e/config/rollup.rs delete mode 100644 bin/citrea/tests/bitcoin_e2e/config/test.rs delete mode 100644 bin/citrea/tests/bitcoin_e2e/config/test_case.rs delete mode 100644 bin/citrea/tests/bitcoin_e2e/config/utils.rs delete mode 100644 bin/citrea/tests/bitcoin_e2e/docker.rs delete mode 100644 bin/citrea/tests/bitcoin_e2e/framework.rs delete mode 100644 bin/citrea/tests/bitcoin_e2e/full_node.rs rename bin/citrea/tests/bitcoin_e2e/{tests => }/mempool_accept.rs (83%) delete mode 100644 bin/citrea/tests/bitcoin_e2e/node.rs delete mode 100644 bin/citrea/tests/bitcoin_e2e/prover.rs rename bin/citrea/tests/bitcoin_e2e/{tests => }/prover_test.rs (93%) delete mode 100644 bin/citrea/tests/bitcoin_e2e/sequencer.rs rename bin/citrea/tests/bitcoin_e2e/{tests => }/sequencer_commitments.rs (84%) rename bin/citrea/tests/bitcoin_e2e/{tests => }/sequencer_test.rs (76%) delete mode 100644 bin/citrea/tests/bitcoin_e2e/test_case.rs delete mode 100644 bin/citrea/tests/bitcoin_e2e/tests/mod.rs delete mode 100644 bin/citrea/tests/bitcoin_e2e/utils.rs diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index fb25e1ad9..e0e7efa52 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -224,6 +224,7 @@ jobs: RUST_BACKTRACE: 1 USE_DOCKER: "true" SHORT_PREFIX: 1 + CITREA: ${{ github.workspace }}/target/debug/citrea - name: Upload coverage uses: codecov/codecov-action@v4 with: @@ -395,6 +396,7 @@ jobs: BONSAI_API_KEY: ${{ secrets.BONSAI_API_KEY }} # TODO: remove this once we don't use the client on tests USE_DOCKER: "true" SHORT_PREFIX: 1 + CITREA: ${{ github.workspace }}/target/debug/citrea system-contracts: strategy: diff --git a/Cargo.lock b/Cargo.lock index e9f0153a5..dfa2c2c6b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1225,7 +1225,7 @@ dependencies = [ "serde", "serde_json", "sha2", - "sov-rollup-interface", + "sov-rollup-interface 0.5.0-rc.1", "thiserror", "tokio", "tracing", @@ -1693,9 +1693,9 @@ dependencies = [ "bitcoin", "bitcoin-da", "bitcoincore-rpc", - "bollard", "borsh", "citrea-common", + "citrea-e2e", "citrea-evm", "citrea-fullnode", "citrea-primitives", @@ -1705,13 +1705,11 @@ dependencies = [ "citrea-stf", "clap", "ethereum-rpc", - "futures", "hex", "jsonrpsee", "log", "log-panics", "proptest", - "rand 0.8.5", "regex", "reqwest 0.12.5", "reth-primitives", @@ -1728,18 +1726,17 @@ dependencies = [ "sha2", "soft-confirmation-rule-enforcer", "sov-db", - "sov-ledger-rpc", + "sov-ledger-rpc 0.5.0-rc.1", "sov-mock-da", "sov-modules-api", "sov-modules-rollup-blueprint", "sov-modules-stf-blueprint", "sov-prover-storage-manager", - "sov-rollup-interface", + "sov-rollup-interface 0.5.0-rc.1", "sov-state", "sov-stf-runner", "tempfile", "tokio", - "toml", "tracing", "tracing-subscriber 0.3.18", ] @@ -1756,13 +1753,39 @@ dependencies = [ "jsonrpsee", "lru", "sov-db", - "sov-rollup-interface", + "sov-rollup-interface 0.5.0-rc.1", "tokio", "tokio-util", "tower-http", "tracing", ] +[[package]] +name = "citrea-e2e" +version = "0.1.0" +source = "git+https://github.com/chainwayxyz/citrea-e2e?branch=main#669d14f9b5ada394d045c63f4067fbeaecfe9aea" +dependencies = [ + "anyhow", + "async-trait", + "bitcoin", + "bitcoincore-rpc", + "bollard", + "futures", + "hex", + "jsonrpsee", + "log", + "rand 0.8.5", + "serde", + "serde_json", + "sov-ledger-rpc 0.5.0-rc.1 (git+https://github.com/chainwayxyz/citrea?rev=82bf52d)", + "sov-rollup-interface 0.5.0-rc.1 (git+https://github.com/chainwayxyz/citrea?rev=82bf52d)", + "syn 1.0.109", + "tempfile", + "tokio", + "toml", + "which", +] + [[package]] name = "citrea-evm" version = "0.5.0-rc.1" @@ -1802,7 +1825,7 @@ dependencies = [ "serde_json", "sov-modules-api", "sov-prover-storage-manager", - "sov-rollup-interface", + "sov-rollup-interface 0.5.0-rc.1", "sov-state", "sov-stf-runner", "tempfile", @@ -1838,7 +1861,7 @@ dependencies = [ "sov-modules-rollup-blueprint", "sov-modules-stf-blueprint", "sov-prover-storage-manager", - "sov-rollup-interface", + "sov-rollup-interface 0.5.0-rc.1", "sov-state", "sov-stf-runner", "tempfile", @@ -1856,7 +1879,7 @@ dependencies = [ "anyhow", "reth-primitives", "serde", - "sov-rollup-interface", + "sov-rollup-interface 0.5.0-rc.1", "tokio", "tracing", ] @@ -1890,7 +1913,7 @@ dependencies = [ "sov-modules-api", "sov-modules-rollup-blueprint", "sov-modules-stf-blueprint", - "sov-rollup-interface", + "sov-rollup-interface 0.5.0-rc.1", "sov-stf-runner", "tempfile", "tokio", @@ -1936,7 +1959,7 @@ dependencies = [ "serde", "sov-db", "sov-risc0-adapter", - "sov-rollup-interface", + "sov-rollup-interface 0.5.0-rc.1", "tracing", ] @@ -1982,7 +2005,7 @@ dependencies = [ "sov-modules-api", "sov-modules-rollup-blueprint", "sov-modules-stf-blueprint", - "sov-rollup-interface", + "sov-rollup-interface 0.5.0-rc.1", "sov-state", "sov-stf-runner", "tempfile", @@ -2011,7 +2034,7 @@ dependencies = [ "sov-accounts", "sov-modules-api", "sov-modules-stf-blueprint", - "sov-rollup-interface", + "sov-rollup-interface 0.5.0-rc.1", "sov-state", "sov-stf-runner", "tracing", @@ -2420,7 +2443,7 @@ dependencies = [ "sha2", "sov-mock-da", "sov-mock-zkvm", - "sov-rollup-interface", + "sov-rollup-interface 0.5.0-rc.1", ] [[package]] @@ -2444,7 +2467,7 @@ dependencies = [ "sov-modules-api", "sov-modules-stf-blueprint", "sov-prover-storage-manager", - "sov-rollup-interface", + "sov-rollup-interface 0.5.0-rc.1", "sov-sequencer-registry", "sov-state", "sov-stf-runner", @@ -2793,7 +2816,7 @@ dependencies = [ "serde_json", "sov-db", "sov-modules-api", - "sov-rollup-interface", + "sov-rollup-interface 0.5.0-rc.1", "tokio", "tracing", ] @@ -3769,7 +3792,7 @@ dependencies = [ "serde", "sov-modules-api", "sov-prover-storage-manager", - "sov-rollup-interface", + "sov-rollup-interface 0.5.0-rc.1", "sov-schema-db", "sov-state", "tempfile", @@ -7044,7 +7067,7 @@ dependencies = [ "reth-rpc-types", "serde", "serde_json", - "sov-rollup-interface", + "sov-rollup-interface 0.5.0-rc.1", "tokio", "tracing", ] @@ -7317,7 +7340,7 @@ dependencies = [ "sov-mock-da", "sov-modules-api", "sov-prover-storage-manager", - "sov-rollup-interface", + "sov-rollup-interface 0.5.0-rc.1", "sov-state", "tempfile", "tracing", @@ -7444,7 +7467,7 @@ dependencies = [ "rocksdb", "serde", "sha2", - "sov-rollup-interface", + "sov-rollup-interface 0.5.0-rc.1", "sov-schema-db", "tempfile", "tokio", @@ -7462,11 +7485,21 @@ dependencies = [ "serde_json", "sov-db", "sov-modules-api", - "sov-rollup-interface", + "sov-rollup-interface 0.5.0-rc.1", "tempfile", "tokio", ] +[[package]] +name = "sov-ledger-rpc" +version = "0.5.0-rc.1" +source = "git+https://github.com/chainwayxyz/citrea?rev=82bf52d#82bf52d067c1886d92c55c141c8fc062b1931164" +dependencies = [ + "jsonrpsee", + "serde", + "sov-rollup-interface 0.5.0-rc.1 (git+https://github.com/chainwayxyz/citrea?rev=82bf52d)", +] + [[package]] name = "sov-mock-da" version = "0.5.0-rc.1" @@ -7483,7 +7516,7 @@ dependencies = [ "serde", "serde_json", "sha2", - "sov-rollup-interface", + "sov-rollup-interface 0.5.0-rc.1", "tempfile", "tokio", "tokio-stream", @@ -7499,7 +7532,7 @@ dependencies = [ "bincode", "borsh", "serde", - "sov-rollup-interface", + "sov-rollup-interface 0.5.0-rc.1", ] [[package]] @@ -7544,7 +7577,7 @@ dependencies = [ "sov-modules-core", "sov-modules-macros", "sov-prover-storage-manager", - "sov-rollup-interface", + "sov-rollup-interface 0.5.0-rc.1", "sov-state", "tempfile", "thiserror", @@ -7572,7 +7605,7 @@ dependencies = [ "sov-modules-api", "sov-modules-core", "sov-prover-storage-manager", - "sov-rollup-interface", + "sov-rollup-interface 0.5.0-rc.1", "sov-state", "tempfile", "thiserror", @@ -7592,7 +7625,7 @@ dependencies = [ "serde_json", "sov-modules-api", "sov-modules-core", - "sov-rollup-interface", + "sov-rollup-interface 0.5.0-rc.1", "sov-state", "syn 1.0.109", "trybuild", @@ -7611,10 +7644,10 @@ dependencies = [ "serde_json", "sov-cli", "sov-db", - "sov-ledger-rpc", + "sov-ledger-rpc 0.5.0-rc.1", "sov-modules-api", "sov-modules-stf-blueprint", - "sov-rollup-interface", + "sov-rollup-interface 0.5.0-rc.1", "sov-state", "sov-stf-runner", "tokio", @@ -7636,7 +7669,7 @@ dependencies = [ "rs_merkle", "serde", "sov-modules-api", - "sov-rollup-interface", + "sov-rollup-interface 0.5.0-rc.1", "sov-state", "sov-zk-cycle-macros", "sov-zk-cycle-utils", @@ -7673,7 +7706,7 @@ dependencies = [ "sha2", "sov-db", "sov-mock-da", - "sov-rollup-interface", + "sov-rollup-interface 0.5.0-rc.1", "sov-schema-db", "sov-state", "tempfile", @@ -7695,7 +7728,7 @@ dependencies = [ "risc0-zkvm", "risc0-zkvm-platform", "serde", - "sov-rollup-interface", + "sov-rollup-interface 0.5.0-rc.1", "sov-zk-cycle-utils", ] @@ -7713,7 +7746,7 @@ dependencies = [ "sov-bank", "sov-mock-da", "sov-modules-api", - "sov-rollup-interface", + "sov-rollup-interface 0.5.0-rc.1", "tokio", ] @@ -7739,6 +7772,26 @@ dependencies = [ "tracing", ] +[[package]] +name = "sov-rollup-interface" +version = "0.5.0-rc.1" +source = "git+https://github.com/chainwayxyz/citrea?rev=82bf52d#82bf52d067c1886d92c55c141c8fc062b1931164" +dependencies = [ + "anyhow", + "async-trait", + "borsh", + "bytes", + "digest 0.10.7", + "futures", + "hex", + "proptest", + "serde", + "sha2", + "thiserror", + "tokio", + "tracing", +] + [[package]] name = "sov-schema-db" version = "0.5.0-rc.1" @@ -7803,7 +7856,7 @@ dependencies = [ "sha2", "sov-db", "sov-modules-core", - "sov-rollup-interface", + "sov-rollup-interface 0.5.0-rc.1", "sov-zk-cycle-macros", "tempfile", "thiserror", @@ -7829,7 +7882,7 @@ dependencies = [ "sov-mock-da", "sov-modules-api", "sov-prover-storage-manager", - "sov-rollup-interface", + "sov-rollup-interface 0.5.0-rc.1", "sov-state", "tempfile", "thiserror", diff --git a/Cargo.toml b/Cargo.toml index b0ab25c26..c0d132682 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -166,8 +166,6 @@ tower-http = { version = "0.5.0", features = ["full"] } tower = { version = "0.4.13", features = ["full"] } hyper = { version = "1.4.0" } -bollard = { version = "0.17.1" } - [patch.'https://github.com/eigerco/celestia-node-rs.git'] # Uncomment to apply local changes # celestia-proto = { path = "../celestia-node-rs/proto" } diff --git a/bin/citrea/Cargo.toml b/bin/citrea/Cargo.toml index a98926519..1b8f65e50 100644 --- a/bin/citrea/Cargo.toml +++ b/bin/citrea/Cargo.toml @@ -86,10 +86,7 @@ rustc_version_runtime = { workspace = true } # bitcoin-e2e dependencies bitcoin.workspace = true bitcoincore-rpc.workspace = true -bollard.workspace = true -futures.workspace = true -rand.workspace = true -toml.workspace = true +citrea-e2e = { git = "https://github.com/chainwayxyz/citrea-e2e", branch = "main" } [features] default = [] # Deviate from convention by making the "native" feature active by default. This aligns with how this package is meant to be used (as a binary first, library second). diff --git a/bin/citrea/tests/all_tests.rs b/bin/citrea/tests/all_tests.rs index 72112d863..6a4aceaf1 100644 --- a/bin/citrea/tests/all_tests.rs +++ b/bin/citrea/tests/all_tests.rs @@ -3,7 +3,6 @@ mod e2e; mod bitcoin_e2e; - mod evm; mod mempool; mod soft_confirmation_rule_enforcer; diff --git a/bin/citrea/tests/bitcoin_e2e/bitcoin.rs b/bin/citrea/tests/bitcoin_e2e/bitcoin.rs deleted file mode 100644 index 6bab7dc8f..000000000 --- a/bin/citrea/tests/bitcoin_e2e/bitcoin.rs +++ /dev/null @@ -1,366 +0,0 @@ -use std::collections::HashSet; -use std::path::PathBuf; -use std::sync::Arc; -use std::time::{Duration, Instant}; - -use anyhow::{bail, Context}; -use async_trait::async_trait; -use bitcoin::Address; -use bitcoin_da::service::{get_relevant_blobs_from_txs, FINALITY_DEPTH}; -use bitcoin_da::spec::blob::BlobWithSender; -use bitcoincore_rpc::json::AddressType::Bech32m; -use bitcoincore_rpc::{Auth, Client, RpcApi}; -use citrea_primitives::REVEAL_BATCH_PROOF_PREFIX; -use futures::TryStreamExt; -use tokio::process::Command; -use tokio::sync::OnceCell; -use tokio::time::sleep; - -use super::config::BitcoinConfig; -use super::docker::DockerEnv; -use super::framework::TestContext; -use super::node::{LogProvider, Node, Restart, SpawnOutput}; -use super::Result; -use crate::bitcoin_e2e::node::NodeKind; - -pub struct BitcoinNode { - spawn_output: SpawnOutput, - pub config: BitcoinConfig, - client: Client, - gen_addr: OnceCell
, - docker_env: Arc>, -} - -impl BitcoinNode { - pub async fn new(config: &BitcoinConfig, docker: Arc>) -> Result { - let spawn_output = Self::spawn(config, &docker).await?; - - let rpc_url = format!( - "http://127.0.0.1:{}/wallet/{}", - config.rpc_port, - NodeKind::Bitcoin - ); - let client = Client::new( - &rpc_url, - Auth::UserPass(config.rpc_user.clone(), config.rpc_password.clone()), - ) - .await - .context("Failed to create RPC client")?; - - wait_for_rpc_ready(&client, None).await?; - - Ok(Self { - spawn_output, - config: config.clone(), - client, - gen_addr: OnceCell::new(), - docker_env: docker, - }) - } - - pub async fn wait_mempool_len( - &self, - target_len: usize, - timeout: Option, - ) -> Result<()> { - let timeout = timeout.unwrap_or(Duration::from_secs(300)); - let start = Instant::now(); - while start.elapsed() < timeout { - let mempool_len = self.get_raw_mempool().await?.len(); - if mempool_len >= target_len { - return Ok(()); - } - sleep(Duration::from_millis(500)).await; - } - bail!("Timeout waiting for mempool to reach length {}", target_len) - } - - pub async fn fund_wallet(&self, name: String, blocks: u64) -> Result<()> { - let rpc_url = format!("http://127.0.0.1:{}/wallet/{}", self.config.rpc_port, name); - let client = Client::new( - &rpc_url, - Auth::UserPass( - self.config.rpc_user.clone(), - self.config.rpc_password.clone(), - ), - ) - .await - .context("Failed to create RPC client")?; - - let gen_addr = client - .get_new_address(None, Some(Bech32m)) - .await? - .assume_checked(); - client.generate_to_address(blocks, &gen_addr).await?; - Ok(()) - } - - pub async fn get_finalized_height(&self) -> Result { - Ok(self.get_block_count().await? - FINALITY_DEPTH + 1) - } - - pub async fn get_relevant_blobs_from_block(&self, height: u64) -> Result> { - let hash = self.get_block_hash(height).await?; - let block = self.get_block(&hash).await?; - - Ok(get_relevant_blobs_from_txs( - block.txdata, - REVEAL_BATCH_PROOF_PREFIX, - )) - } - - async fn wait_for_shutdown(&self) -> Result<()> { - let timeout_duration = Duration::from_secs(30); - let start = std::time::Instant::now(); - - while start.elapsed() < timeout_duration { - if !self.is_process_running().await? { - println!("Bitcoin daemon has stopped successfully"); - return Ok(()); - } - sleep(Duration::from_millis(200)).await; - } - - bail!("Timeout waiting for Bitcoin daemon to stop") - } - - async fn is_process_running(&self) -> Result { - let data_dir = &self.config.data_dir; - let output = Command::new("pgrep") - .args(["-f", &format!("bitcoind.*{}", data_dir.display())]) - .output() - .await?; - - Ok(output.status.success()) - } - - // Infallible, discard already loaded errors - async fn load_wallets(&self) { - let _ = self.load_wallet(&NodeKind::Bitcoin.to_string()).await; - let _ = self.load_wallet(&NodeKind::Sequencer.to_string()).await; - let _ = self.load_wallet(&NodeKind::Prover.to_string()).await; - } - - // Switch this over to Node signature once we add support for docker to citrea nodes - async fn spawn(config: &BitcoinConfig, docker: &Arc>) -> Result { - match docker.as_ref() { - Some(docker) => docker.spawn(config.into()).await, - None => ::spawn(config), - } - } -} - -#[async_trait] -impl RpcApi for BitcoinNode { - async fn call serde::de::Deserialize<'a>>( - &self, - cmd: &str, - args: &[serde_json::Value], - ) -> bitcoincore_rpc::Result { - self.client.call(cmd, args).await - } - - // Override deprecated generate method. - // Uses or lazy init gen_addr and forward to `generate_to_address` - async fn generate( - &self, - block_num: u64, - _maxtries: Option, - ) -> bitcoincore_rpc::Result> { - let addr = self - .gen_addr - .get_or_init(|| async { - self.client - .get_new_address(None, Some(Bech32m)) - .await - .expect("Failed to generate address") - .assume_checked() - }) - .await; - - self.generate_to_address(block_num, addr).await - } -} - -impl Node for BitcoinNode { - type Config = BitcoinConfig; - type Client = Client; - - fn spawn(config: &Self::Config) -> Result { - let args = config.args(); - println!("Running bitcoind with args : {args:?}"); - - Command::new("bitcoind") - .args(&args) - .kill_on_drop(true) - .envs(config.env.clone()) - .spawn() - .context("Failed to spawn bitcoind process") - .map(SpawnOutput::Child) - } - - fn spawn_output(&mut self) -> &mut SpawnOutput { - &mut self.spawn_output - } - - async fn wait_for_ready(&self, timeout: Option) -> Result<()> { - println!("Waiting for ready"); - let start = Instant::now(); - let timeout = timeout.unwrap_or(Duration::from_secs(30)); - while start.elapsed() < timeout { - if wait_for_rpc_ready(&self.client, Some(timeout)) - .await - .is_ok() - { - return Ok(()); - } - tokio::time::sleep(Duration::from_millis(500)).await; - } - anyhow::bail!("Node failed to become ready within the specified timeout") - } - - fn client(&self) -> &Self::Client { - &self.client - } - - fn env(&self) -> Vec<(&'static str, &'static str)> { - self.config.env.clone() - } - - fn config_mut(&mut self) -> &mut Self::Config { - &mut self.config - } -} - -impl Restart for BitcoinNode { - async fn wait_until_stopped(&mut self) -> Result<()> { - self.client.stop().await?; - self.stop().await?; - - match &self.spawn_output { - SpawnOutput::Child(_) => self.wait_for_shutdown().await, - SpawnOutput::Container(output) => { - let Some(env) = self.docker_env.as_ref() else { - bail!("Missing docker environment") - }; - env.docker.stop_container(&output.id, None).await?; - - env.docker - .wait_container::(&output.id, None) - .try_collect::>() - .await?; - env.docker.remove_container(&output.id, None).await?; - println!("Docker container {} succesfully removed", output.id); - Ok(()) - } - } - } - - async fn start(&mut self, config: Option) -> Result<()> { - if let Some(config) = config { - self.config = config - } - self.spawn_output = Self::spawn(&self.config, &self.docker_env).await?; - - self.wait_for_ready(None).await?; - - // Reload wallets after restart - self.load_wallets().await; - - Ok(()) - } -} - -impl LogProvider for BitcoinNode { - fn kind(&self) -> NodeKind { - NodeKind::Bitcoin - } - - fn log_path(&self) -> PathBuf { - self.config.data_dir.join("regtest").join("debug.log") - } -} - -pub struct BitcoinNodeCluster { - inner: Vec, -} - -impl BitcoinNodeCluster { - pub async fn new(ctx: &TestContext) -> Result { - let n_nodes = ctx.config.test_case.n_nodes; - let mut cluster = Self { - inner: Vec::with_capacity(n_nodes), - }; - for config in ctx.config.bitcoin.iter() { - let node = BitcoinNode::new(config, Arc::clone(&ctx.docker)).await?; - cluster.inner.push(node) - } - - Ok(cluster) - } - - pub async fn stop_all(&mut self) -> Result<()> { - for node in &mut self.inner { - RpcApi::stop(node).await?; - node.stop().await?; - } - Ok(()) - } - - pub async fn wait_for_sync(&self, timeout: Duration) -> Result<()> { - let start = Instant::now(); - while start.elapsed() < timeout { - let mut heights = HashSet::new(); - for node in &self.inner { - let height = node.get_block_count().await?; - heights.insert(height); - } - - if heights.len() == 1 { - return Ok(()); - } - - sleep(Duration::from_secs(1)).await; - } - bail!("Nodes failed to sync within the specified timeout") - } - - // Connect all bitcoin nodes between them - pub async fn connect_nodes(&self) -> Result<()> { - for (i, from_node) in self.inner.iter().enumerate() { - for (j, to_node) in self.inner.iter().enumerate() { - if i != j { - let ip = match &to_node.spawn_output { - SpawnOutput::Container(container) => container.ip.clone(), - _ => "127.0.0.1".to_string(), - }; - - let add_node_arg = format!("{}:{}", ip, to_node.config.p2p_port); - from_node.add_node(&add_node_arg).await?; - } - } - } - Ok(()) - } - - pub fn get(&self, index: usize) -> Option<&BitcoinNode> { - self.inner.get(index) - } - - #[allow(unused)] - pub fn get_mut(&mut self, index: usize) -> Option<&mut BitcoinNode> { - self.inner.get_mut(index) - } -} - -async fn wait_for_rpc_ready(client: &Client, timeout: Option) -> Result<()> { - let start = Instant::now(); - let timeout = timeout.unwrap_or(Duration::from_secs(300)); - while start.elapsed() < timeout { - match client.get_blockchain_info().await { - Ok(_) => return Ok(()), - Err(_) => sleep(Duration::from_millis(500)).await, - } - } - Err(anyhow::anyhow!("Timeout waiting for RPC to be ready")) -} diff --git a/bin/citrea/tests/bitcoin_e2e/tests/bitcoin_test.rs b/bin/citrea/tests/bitcoin_e2e/bitcoin_test.rs similarity index 92% rename from bin/citrea/tests/bitcoin_e2e/tests/bitcoin_test.rs rename to bin/citrea/tests/bitcoin_e2e/bitcoin_test.rs index 4cd0580b7..166838eb1 100644 --- a/bin/citrea/tests/bitcoin_e2e/tests/bitcoin_test.rs +++ b/bin/citrea/tests/bitcoin_e2e/bitcoin_test.rs @@ -4,12 +4,11 @@ use anyhow::bail; use async_trait::async_trait; use bitcoincore_rpc::json::IndexStatus; use bitcoincore_rpc::RpcApi; - -use crate::bitcoin_e2e::config::{BitcoinConfig, TestCaseConfig}; -use crate::bitcoin_e2e::framework::TestFramework; -use crate::bitcoin_e2e::node::Restart; -use crate::bitcoin_e2e::test_case::{TestCase, TestCaseRunner}; -use crate::bitcoin_e2e::Result; +use citrea_e2e::config::{BitcoinConfig, TestCaseConfig}; +use citrea_e2e::framework::TestFramework; +use citrea_e2e::test_case::{TestCase, TestCaseRunner}; +use citrea_e2e::traits::Restart; +use citrea_e2e::Result; struct BasicSyncTest; diff --git a/bin/citrea/tests/bitcoin_e2e/config/bitcoin.rs b/bin/citrea/tests/bitcoin_e2e/config/bitcoin.rs deleted file mode 100644 index 734c29792..000000000 --- a/bin/citrea/tests/bitcoin_e2e/config/bitcoin.rs +++ /dev/null @@ -1,64 +0,0 @@ -use std::path::PathBuf; - -use bitcoin::Network; -use tempfile::TempDir; - -#[derive(Debug, Clone)] -pub struct BitcoinConfig { - pub p2p_port: u16, - pub rpc_port: u16, - pub rpc_user: String, - pub rpc_password: String, - pub data_dir: PathBuf, - pub extra_args: Vec<&'static str>, - pub network: Network, - pub docker_image: Option, - pub env: Vec<(&'static str, &'static str)>, - pub idx: usize, -} - -impl Default for BitcoinConfig { - fn default() -> Self { - Self { - p2p_port: 0, - rpc_port: 0, - rpc_user: "user".to_string(), - rpc_password: "password".to_string(), - data_dir: TempDir::new() - .expect("Failed to create temporary directory") - .into_path(), - extra_args: Vec::new(), - network: Network::Regtest, - docker_image: Some("bitcoin/bitcoin:27.1-alpine".to_string()), - env: Vec::new(), - idx: 0, - } - } -} - -impl BitcoinConfig { - fn base_args(&self) -> Vec { - vec![ - "-regtest".to_string(), - format!("-datadir={}", self.data_dir.display()), - format!("-port={}", self.p2p_port), - format!("-rpcport={}", self.rpc_port), - format!("-rpcuser={}", self.rpc_user), - format!("-rpcpassword={}", self.rpc_password), - "-server".to_string(), - "-daemonwait".to_string(), - "-txindex".to_string(), - "-addresstype=bech32m".to_string(), - "-debug=net".to_string(), - "-debug=rpc".to_string(), - ] - } - - pub fn args(&self) -> Vec { - [ - self.base_args(), - self.extra_args.iter().map(|&s| s.to_string()).collect(), - ] - .concat() - } -} diff --git a/bin/citrea/tests/bitcoin_e2e/config/docker.rs b/bin/citrea/tests/bitcoin_e2e/config/docker.rs deleted file mode 100644 index 5f57a469c..000000000 --- a/bin/citrea/tests/bitcoin_e2e/config/docker.rs +++ /dev/null @@ -1,77 +0,0 @@ -use std::path::PathBuf; - -use super::{BitcoinConfig, FullSequencerConfig}; -use crate::bitcoin_e2e::utils::get_genesis_path; - -#[derive(Debug)] -pub struct VolumeConfig { - pub name: String, - pub target: String, -} - -#[derive(Debug)] -pub struct DockerConfig { - pub ports: Vec, - pub image: String, - pub cmd: Vec, - pub log_path: PathBuf, - pub volume: VolumeConfig, -} - -impl From<&BitcoinConfig> for DockerConfig { - fn from(v: &BitcoinConfig) -> Self { - let mut args = v.args(); - - // Docker specific args - args.extend([ - "-rpcallowip=0.0.0.0/0".to_string(), - "-rpcbind=0.0.0.0".to_string(), - "-daemonwait=0".to_string(), - ]); - - Self { - ports: vec![v.rpc_port, v.p2p_port], - image: v - .docker_image - .clone() - .unwrap_or_else(|| "bitcoin/bitcoin:27.1-alpine".to_string()), - cmd: args, - log_path: v.data_dir.join("regtest").join("debug.log"), - volume: VolumeConfig { - name: format!("bitcoin-{}", v.idx), - target: "/home/bitcoin/.bitcoin".to_string(), - }, - } - } -} - -impl From<&FullSequencerConfig> for DockerConfig { - fn from(v: &FullSequencerConfig) -> Self { - let args = vec![ - "--da-layer".to_string(), - "bitcoin".to_string(), - "--rollup-config-path".to_string(), - "sequencer_rollup_config.toml".to_string(), - "--sequencer-config-path".to_string(), - "sequencer_config.toml".to_string(), - "--genesis-paths".to_string(), - get_genesis_path(v.dir.parent().expect("Couldn't get parent dir")) - .display() - .to_string(), - ]; - - Self { - ports: vec![v.rollup.rpc.bind_port], - image: v - .docker_image - .clone() - .unwrap_or_else(|| "citrea:latest".to_string()), // Default to local image - cmd: args, - log_path: v.dir.join("stdout"), - volume: VolumeConfig { - name: "sequencer".to_string(), - target: "/sequencer/data".to_string(), - }, - } - } -} diff --git a/bin/citrea/tests/bitcoin_e2e/config/mod.rs b/bin/citrea/tests/bitcoin_e2e/config/mod.rs deleted file mode 100644 index 9bb845371..000000000 --- a/bin/citrea/tests/bitcoin_e2e/config/mod.rs +++ /dev/null @@ -1,30 +0,0 @@ -mod bitcoin; -mod docker; -mod rollup; -mod test; -mod test_case; -mod utils; - -use std::path::PathBuf; - -pub use bitcoin::BitcoinConfig; -pub use citrea_sequencer::SequencerConfig; -pub use docker::DockerConfig; -pub use rollup::{default_rollup_config, RollupConfig}; -pub use sov_stf_runner::ProverConfig; -pub use test::TestConfig; -pub use test_case::{TestCaseConfig, TestCaseEnv}; -pub use utils::config_to_file; - -#[derive(Clone, Debug)] -pub struct FullL2NodeConfig { - pub node: T, - pub rollup: RollupConfig, - pub docker_image: Option, - pub dir: PathBuf, - pub env: Vec<(&'static str, &'static str)>, -} - -pub type FullSequencerConfig = FullL2NodeConfig; -pub type FullProverConfig = FullL2NodeConfig; -pub type FullFullNodeConfig = FullL2NodeConfig<()>; diff --git a/bin/citrea/tests/bitcoin_e2e/config/rollup.rs b/bin/citrea/tests/bitcoin_e2e/config/rollup.rs deleted file mode 100644 index 3d83d99fb..000000000 --- a/bin/citrea/tests/bitcoin_e2e/config/rollup.rs +++ /dev/null @@ -1,68 +0,0 @@ -use bitcoin_da::service::BitcoinServiceConfig; -use sov_stf_runner::{FullNodeConfig, RollupPublicKeys, RpcConfig, StorageConfig}; -use tempfile::TempDir; - -use super::BitcoinConfig; -use crate::bitcoin_e2e::utils::get_tx_backup_dir; -pub type RollupConfig = FullNodeConfig; - -pub fn default_rollup_config() -> RollupConfig { - RollupConfig { - rpc: RpcConfig { - bind_host: "127.0.0.1".into(), - bind_port: 0, - max_connections: 100, - max_request_body_size: 10 * 1024 * 1024, - max_response_body_size: 10 * 1024 * 1024, - batch_requests_limit: 50, - enable_subscriptions: true, - max_subscriptions_per_connection: 100, - }, - storage: StorageConfig { - path: TempDir::new() - .expect("Failed to create temporary directory") - .into_path(), - db_max_open_files: None, - }, - runner: None, - da: BitcoinServiceConfig { - node_url: String::new(), - node_username: String::from("user"), - node_password: String::from("password"), - network: bitcoin::Network::Regtest, - da_private_key: None, - tx_backup_dir: get_tx_backup_dir(), - }, - public_keys: RollupPublicKeys { - sequencer_public_key: vec![ - 32, 64, 64, 227, 100, 193, 15, 43, 236, 156, 31, 229, 0, 161, 205, 76, 36, 124, - 137, 214, 80, 160, 30, 215, 232, 44, 171, 168, 103, 135, 124, 33, - ], - // private key [4, 95, 252, 129, 163, 193, 253, 179, 175, 19, 89, 219, 242, 209, 20, 176, 179, 239, 191, 127, 41, 204, 156, 93, 160, 18, 103, 170, 57, 210, 199, 141] - // Private Key (WIF): KwNDSCvKqZqFWLWN1cUzvMiJQ7ck6ZKqR6XBqVKyftPZtvmbE6YD - sequencer_da_pub_key: vec![ - 3, 136, 195, 18, 11, 187, 25, 37, 38, 109, 184, 237, 247, 208, 131, 219, 162, 70, - 35, 174, 234, 47, 239, 247, 60, 51, 174, 242, 247, 112, 186, 222, 30, - ], - // private key [117, 186, 249, 100, 208, 116, 89, 70, 0, 54, 110, 91, 17, 26, 29, 168, 248, 107, 46, 254, 45, 34, 218, 81, 200, 216, 33, 38, 160, 252, 172, 114] - // Private Key (WIF): L1AZdJXzDGGENBBPZGSL7dKJnwn5xSKqzszgK6CDwiBGThYQEVTo - prover_da_pub_key: vec![ - 2, 138, 232, 157, 214, 46, 7, 210, 235, 33, 105, 239, 71, 169, 105, 233, 239, 84, - 172, 112, 13, 54, 9, 206, 106, 138, 251, 218, 15, 28, 137, 112, 127, - ], - }, - } -} - -impl From for BitcoinServiceConfig { - fn from(v: BitcoinConfig) -> Self { - Self { - node_url: format!("127.0.0.1:{}", v.rpc_port), - node_username: v.rpc_user, - node_password: v.rpc_password, - network: v.network, - da_private_key: None, - tx_backup_dir: "".to_string(), - } - } -} diff --git a/bin/citrea/tests/bitcoin_e2e/config/test.rs b/bin/citrea/tests/bitcoin_e2e/config/test.rs deleted file mode 100644 index 3f33d4ff9..000000000 --- a/bin/citrea/tests/bitcoin_e2e/config/test.rs +++ /dev/null @@ -1,12 +0,0 @@ -use super::bitcoin::BitcoinConfig; -use super::test_case::TestCaseConfig; -use super::{FullFullNodeConfig, FullProverConfig, FullSequencerConfig}; - -#[derive(Clone)] -pub struct TestConfig { - pub test_case: TestCaseConfig, - pub bitcoin: Vec, - pub sequencer: FullSequencerConfig, - pub prover: FullProverConfig, - pub full_node: FullFullNodeConfig, -} diff --git a/bin/citrea/tests/bitcoin_e2e/config/test_case.rs b/bin/citrea/tests/bitcoin_e2e/config/test_case.rs deleted file mode 100644 index caaeca5e6..000000000 --- a/bin/citrea/tests/bitcoin_e2e/config/test_case.rs +++ /dev/null @@ -1,73 +0,0 @@ -use std::path::PathBuf; -use std::time::Duration; - -use tempfile::TempDir; - -#[derive(Clone, Default)] -pub struct TestCaseEnv { - pub test: Vec<(&'static str, &'static str)>, - pub full_node: Vec<(&'static str, &'static str)>, - pub sequencer: Vec<(&'static str, &'static str)>, - pub prover: Vec<(&'static str, &'static str)>, - pub bitcoin: Vec<(&'static str, &'static str)>, -} - -impl TestCaseEnv { - // Base env that should apply to every test cases - fn base_env() -> Vec<(&'static str, &'static str)> { - vec![("NO_COLOR", "1")] - } - - fn test_env(&self) -> Vec<(&'static str, &'static str)> { - [Self::base_env(), self.test.clone()].concat() - } - - pub fn sequencer(&self) -> Vec<(&'static str, &'static str)> { - [self.test_env(), self.sequencer.clone()].concat() - } - - pub fn prover(&self) -> Vec<(&'static str, &'static str)> { - [self.test_env(), self.prover.clone()].concat() - } - - pub fn full_node(&self) -> Vec<(&'static str, &'static str)> { - [self.test_env(), self.full_node.clone()].concat() - } - - pub fn bitcoin(&self) -> Vec<(&'static str, &'static str)> { - [self.test_env(), self.bitcoin.clone()].concat() - } -} - -#[derive(Clone)] -pub struct TestCaseConfig { - pub n_nodes: usize, - pub with_sequencer: bool, - pub with_full_node: bool, - pub with_prover: bool, - #[allow(unused)] - pub timeout: Duration, - pub dir: PathBuf, - pub docker: bool, - // Either a relative dir from workspace root, i.e. "./resources/genesis/devnet" - // Or an absolute path. - // Defaults to resources/genesis/bitcoin-regtest - pub genesis_dir: Option, -} - -impl Default for TestCaseConfig { - fn default() -> Self { - TestCaseConfig { - n_nodes: 1, - with_sequencer: true, - with_prover: false, - with_full_node: false, - timeout: Duration::from_secs(60), - dir: TempDir::new() - .expect("Failed to create temporary directory") - .into_path(), - docker: std::env::var("USE_DOCKER").map_or(false, |v| v.parse().unwrap_or(false)), - genesis_dir: None, - } - } -} diff --git a/bin/citrea/tests/bitcoin_e2e/config/utils.rs b/bin/citrea/tests/bitcoin_e2e/config/utils.rs deleted file mode 100644 index 49beb1563..000000000 --- a/bin/citrea/tests/bitcoin_e2e/config/utils.rs +++ /dev/null @@ -1,14 +0,0 @@ -use std::path::Path; - -use serde::Serialize; - -pub fn config_to_file(config: &C, path: &P) -> std::io::Result<()> -where - C: Serialize, - P: AsRef, -{ - let toml = - toml::to_string(config).map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))?; - std::fs::write(path, toml)?; - Ok(()) -} diff --git a/bin/citrea/tests/bitcoin_e2e/docker.rs b/bin/citrea/tests/bitcoin_e2e/docker.rs deleted file mode 100644 index 44194d115..000000000 --- a/bin/citrea/tests/bitcoin_e2e/docker.rs +++ /dev/null @@ -1,282 +0,0 @@ -use std::collections::{HashMap, HashSet}; -use std::io::{stdout, Write}; -use std::path::PathBuf; - -use anyhow::{anyhow, Context, Result}; -use bollard::container::{Config, LogOutput, LogsOptions, NetworkingConfig}; -use bollard::image::CreateImageOptions; -use bollard::models::{EndpointSettings, Mount, PortBinding}; -use bollard::network::CreateNetworkOptions; -use bollard::secret::MountTypeEnum; -use bollard::service::HostConfig; -use bollard::volume::CreateVolumeOptions; -use bollard::Docker; -use futures::StreamExt; -use tokio::fs::File; -use tokio::io::AsyncWriteExt; -use tokio::task::JoinHandle; - -use super::config::DockerConfig; -use super::node::SpawnOutput; -use super::utils::generate_test_id; -use crate::bitcoin_e2e::node::ContainerSpawnOutput; - -pub struct DockerEnv { - pub docker: Docker, - pub network_id: String, - pub network_name: String, - id: String, - volumes: HashSet, -} - -impl DockerEnv { - pub async fn new(n_nodes: usize) -> Result { - let docker = - Docker::connect_with_local_defaults().context("Failed to connect to Docker")?; - let test_id = generate_test_id(); - let (network_id, network_name) = Self::create_network(&docker, &test_id).await?; - let volumes = Self::create_volumes(&docker, &test_id, n_nodes).await?; - - Ok(Self { - docker, - network_id, - network_name, - id: test_id, - volumes, - }) - } - - async fn create_volumes( - docker: &Docker, - test_case_id: &str, - n_nodes: usize, - ) -> Result> { - let volume_configs = vec![("bitcoin", n_nodes)]; - let mut volumes = HashSet::new(); - - for (name, n) in volume_configs { - for i in 0..n { - let volume_name = format!("{name}-{i}-{test_case_id}"); - docker - .create_volume(CreateVolumeOptions { - name: volume_name.clone(), - driver: "local".to_string(), - driver_opts: HashMap::new(), - labels: HashMap::new(), - }) - .await?; - - volumes.insert(volume_name); - } - } - - Ok(volumes) - } - - async fn create_network(docker: &Docker, test_case_id: &str) -> Result<(String, String)> { - let network_name = format!("test_network_{}", test_case_id); - let options = CreateNetworkOptions { - name: network_name.clone(), - check_duplicate: true, - driver: "bridge".to_string(), - ..Default::default() - }; - - let id = docker - .create_network(options) - .await? - .id - .context("Error getting network id")?; - Ok((id, network_name)) - } - - pub async fn spawn(&self, config: DockerConfig) -> Result { - println!("Spawning docker with config {config:#?}"); - let exposed_ports: HashMap> = config - .ports - .iter() - .map(|port| (format!("{}/tcp", port), HashMap::new())) - .collect(); - - let port_bindings: HashMap>> = config - .ports - .iter() - .map(|port| { - ( - format!("{}/tcp", port), - Some(vec![PortBinding { - host_ip: Some("0.0.0.0".to_string()), - host_port: Some(port.to_string()), - }]), - ) - }) - .collect(); - - let mut network_config = HashMap::new(); - network_config.insert(self.network_id.clone(), EndpointSettings::default()); - - let volume_name = format!("{}-{}", config.volume.name, self.id); - let mount = Mount { - target: Some(config.volume.target.clone()), - source: Some(volume_name), - typ: Some(MountTypeEnum::VOLUME), - ..Default::default() - }; - - let container_config = Config { - image: Some(config.image), - cmd: Some(config.cmd), - exposed_ports: Some(exposed_ports), - host_config: Some(HostConfig { - port_bindings: Some(port_bindings), - // binds: Some(vec![config.dir]), - mounts: Some(vec![mount]), - ..Default::default() - }), - networking_config: Some(NetworkingConfig { - endpoints_config: network_config, - }), - tty: Some(true), - ..Default::default() - }; - - let image = container_config - .image - .as_ref() - .context("Image not specified in config")?; - self.ensure_image_exists(image).await?; - - // println!("options :{options:?}"); - // println!("config :{container_config:?}"); - - let container = self - .docker - .create_container::(None, container_config) - .await - .map_err(|e| anyhow!("Failed to create Docker container {e}"))?; - - self.docker - .start_container::(&container.id, None) - .await - .context("Failed to start Docker container")?; - - let inspect_result = self.docker.inspect_container(&container.id, None).await?; - let ip_address = inspect_result - .network_settings - .and_then(|ns| ns.networks) - .and_then(|networks| { - networks - .values() - .next() - .and_then(|network| network.ip_address.clone()) - }) - .context("Failed to get container IP address")?; - - // Extract container logs to host - // This spawns a background task to continuously stream logs from the container. - // The task will run until the container is stopped or removed during cleanup. - Self::extract_container_logs(self.docker.clone(), container.id.clone(), config.log_path); - - Ok(SpawnOutput::Container(ContainerSpawnOutput { - id: container.id, - ip: ip_address, - })) - } - - async fn ensure_image_exists(&self, image: &str) -> Result<()> { - let images = self - .docker - .list_images::(None) - .await - .context("Failed to list Docker images")?; - if images - .iter() - .any(|img| img.repo_tags.contains(&image.to_string())) - { - return Ok(()); - } - - println!("Pulling image: {}", image); - let options = Some(CreateImageOptions { - from_image: image, - ..Default::default() - }); - - let mut stream = self.docker.create_image(options, None, None); - while let Some(result) = stream.next().await { - match result { - Ok(info) => { - if let (Some(status), Some(progress)) = (info.status, info.progress) { - print!("\r{}: {} ", status, progress); - stdout().flush().unwrap(); - } - } - Err(e) => return Err(anyhow::anyhow!("Failed to pull image: {}", e)), - } - } - println!("Image succesfully pulled"); - - Ok(()) - } - - pub async fn cleanup(&self) -> Result<()> { - let containers = self.docker.list_containers::(None).await?; - for container in containers { - if let (Some(id), Some(networks)) = ( - container.id, - container.network_settings.and_then(|ns| ns.networks), - ) { - if networks.contains_key(&self.network_name) { - self.docker.stop_container(&id, None).await?; - self.docker.remove_container(&id, None).await?; - } - } - } - - self.docker.remove_network(&self.network_name).await?; - - for volume_name in &self.volumes { - self.docker.remove_volume(volume_name, None).await?; - } - - Ok(()) - } - - fn extract_container_logs( - docker: Docker, - container_id: String, - log_path: PathBuf, - ) -> JoinHandle> { - tokio::spawn(async move { - if let Some(parent) = log_path.parent() { - tokio::fs::create_dir_all(parent) - .await - .context("Failed to create log directory")?; - } - let mut log_file = File::create(log_path) - .await - .context("Failed to create log file")?; - let mut log_stream = docker.logs::( - &container_id, - Some(LogsOptions { - follow: true, - stdout: true, - stderr: true, - ..Default::default() - }), - ); - - while let Some(Ok(log_output)) = log_stream.next().await { - let log_line = match log_output { - LogOutput::Console { message } | LogOutput::StdOut { message } => message, - _ => continue, - }; - log_file - .write_all(&log_line) - .await - .context("Failed to write log line")?; - } - Ok(()) - }) - } -} diff --git a/bin/citrea/tests/bitcoin_e2e/framework.rs b/bin/citrea/tests/bitcoin_e2e/framework.rs deleted file mode 100644 index eae3ce926..000000000 --- a/bin/citrea/tests/bitcoin_e2e/framework.rs +++ /dev/null @@ -1,200 +0,0 @@ -use std::future::Future; -use std::sync::Arc; - -use bitcoincore_rpc::RpcApi; - -use super::bitcoin::BitcoinNodeCluster; -use super::config::TestConfig; -use super::docker::DockerEnv; -use super::full_node::FullNode; -use super::node::{LogProvider, LogProviderErased, Node, NodeKind}; -use super::sequencer::Sequencer; -use super::Result; -use crate::bitcoin_e2e::prover::Prover; -use crate::bitcoin_e2e::utils::tail_file; - -pub struct TestContext { - pub config: TestConfig, - pub docker: Arc>, -} - -impl TestContext { - async fn new(config: TestConfig) -> Self { - let docker = if config.test_case.docker { - Some(DockerEnv::new(config.test_case.n_nodes).await.unwrap()) - } else { - None - }; - Self { - config, - docker: Arc::new(docker), - } - } -} - -pub struct TestFramework { - ctx: TestContext, - pub bitcoin_nodes: BitcoinNodeCluster, - pub sequencer: Option, - pub prover: Option, - pub full_node: Option, - show_logs: bool, - pub initial_da_height: u64, -} - -async fn create_optional(pred: bool, f: impl Future>) -> Result> { - if pred { - Ok(Some(f.await?)) - } else { - Ok(None) - } -} - -impl TestFramework { - pub async fn new(config: TestConfig) -> Result { - anyhow::ensure!( - config.test_case.n_nodes > 0, - "At least one bitcoin node has to be running" - ); - - let ctx = TestContext::new(config).await; - - let bitcoin_nodes = BitcoinNodeCluster::new(&ctx).await?; - - // tokio::time::sleep(std::time::Duration::from_secs(30)).await; - Ok(Self { - bitcoin_nodes, - sequencer: None, - prover: None, - full_node: None, - ctx, - show_logs: true, - initial_da_height: 0, - }) - } - - pub async fn init_nodes(&mut self) -> Result<()> { - // Has to initialize sequencer first since prover and full node depend on it - self.sequencer = create_optional( - self.ctx.config.test_case.with_sequencer, - Sequencer::new(&self.ctx), - ) - .await?; - - (self.prover, self.full_node) = tokio::try_join!( - create_optional( - self.ctx.config.test_case.with_prover, - Prover::new(&self.ctx) - ), - create_optional( - self.ctx.config.test_case.with_full_node, - FullNode::new(&self.ctx) - ), - )?; - - Ok(()) - } - - fn get_nodes_as_log_provider(&self) -> Vec<&dyn LogProviderErased> { - vec![ - self.bitcoin_nodes.get(0).map(LogProvider::as_erased), - self.sequencer.as_ref().map(LogProvider::as_erased), - self.full_node.as_ref().map(LogProvider::as_erased), - self.prover.as_ref().map(LogProvider::as_erased), - ] - .into_iter() - .flatten() - .collect() - } - - pub fn show_log_paths(&self) { - if self.show_logs { - println!( - "Logs available at {}", - self.ctx.config.test_case.dir.display() - ); - - for node in self.get_nodes_as_log_provider() { - println!( - "{} logs available at : {}", - node.kind(), - node.log_path().display() - ); - } - } - } - - pub fn dump_log(&self) -> Result<()> { - println!("Dumping logs:"); - - let n_lines = std::env::var("TAIL_N_LINES") - .ok() - .and_then(|v| v.parse::().ok()) - .unwrap_or(25); - for node in self.get_nodes_as_log_provider() { - println!("{} logs (last {n_lines} lines):", node.kind()); - if let Err(e) = tail_file(&node.log_path(), n_lines) { - eprint!("{e}"); - } - } - Ok(()) - } - - pub async fn stop(&mut self) -> Result<()> { - println!("Stopping framework..."); - - if let Some(sequencer) = &mut self.sequencer { - let _ = sequencer.stop().await; - println!("Successfully stopped sequencer"); - } - - if let Some(prover) = &mut self.prover { - let _ = prover.stop().await; - println!("Successfully stopped prover"); - } - - if let Some(full_node) = &mut self.full_node { - let _ = full_node.stop().await; - println!("Successfully stopped full_node"); - } - - let _ = self.bitcoin_nodes.stop_all().await; - println!("Successfully stopped bitcoin nodes"); - - if let Some(docker) = self.ctx.docker.as_ref() { - let _ = docker.cleanup().await; - println!("Successfully cleaned docker"); - } - - Ok(()) - } - - pub async fn fund_da_wallets(&mut self) -> Result<()> { - let da = self.bitcoin_nodes.get(0).unwrap(); - - da.create_wallet(&NodeKind::Sequencer.to_string(), None, None, None, None) - .await?; - da.create_wallet(&NodeKind::Prover.to_string(), None, None, None, None) - .await?; - da.create_wallet(&NodeKind::Bitcoin.to_string(), None, None, None, None) - .await?; - - let blocks_to_mature = 100; - let blocks_to_fund = 25; - if self.ctx.config.test_case.with_sequencer { - da.fund_wallet(NodeKind::Sequencer.to_string(), blocks_to_fund) - .await?; - } - - if self.ctx.config.test_case.with_prover { - da.fund_wallet(NodeKind::Prover.to_string(), blocks_to_fund) - .await?; - } - da.fund_wallet(NodeKind::Bitcoin.to_string(), blocks_to_fund) - .await?; - - da.generate(blocks_to_mature, None).await?; - self.initial_da_height = da.get_block_count().await?; - Ok(()) - } -} diff --git a/bin/citrea/tests/bitcoin_e2e/full_node.rs b/bin/citrea/tests/bitcoin_e2e/full_node.rs deleted file mode 100644 index d08720375..000000000 --- a/bin/citrea/tests/bitcoin_e2e/full_node.rs +++ /dev/null @@ -1,181 +0,0 @@ -use std::fs::File; -use std::net::SocketAddr; -use std::path::PathBuf; -use std::process::Stdio; - -use anyhow::{bail, Context}; -use sov_rollup_interface::rpc::{SequencerCommitmentResponse, VerifiedProofResponse}; -use tokio::process::Command; -use tokio::time::{sleep, Duration, Instant}; - -use super::config::{config_to_file, FullFullNodeConfig, TestConfig}; -use super::framework::TestContext; -use super::node::{LogProvider, Node, NodeKind, SpawnOutput}; -use super::utils::{get_citrea_path, get_stderr_path, get_stdout_path, retry}; -use super::Result; -use crate::bitcoin_e2e::utils::get_genesis_path; -use crate::evm::make_test_client; -use crate::test_client::TestClient; - -#[allow(unused)] -pub struct FullNode { - spawn_output: SpawnOutput, - config: FullFullNodeConfig, - pub client: Box, -} - -impl FullNode { - pub async fn new(ctx: &TestContext) -> Result { - let TestConfig { - full_node: full_node_config, - .. - } = &ctx.config; - - let spawn_output = Self::spawn(full_node_config)?; - - let socket_addr = SocketAddr::new( - full_node_config - .rollup - .rpc - .bind_host - .parse() - .context("Failed to parse bind host")?, - full_node_config.rollup.rpc.bind_port, - ); - let client = retry(|| async { make_test_client(socket_addr).await }, None).await?; - - Ok(Self { - spawn_output, - config: full_node_config.clone(), - client, - }) - } - - pub async fn wait_for_sequencer_commitments( - &self, - height: u64, - timeout: Option, - ) -> Result> { - let start = Instant::now(); - let timeout = timeout.unwrap_or(Duration::from_secs(30)); - - loop { - if start.elapsed() >= timeout { - bail!("FullNode failed to get sequencer commitments within the specified timeout"); - } - - match self - .client - .ledger_get_sequencer_commitments_on_slot_by_number(height) - .await - { - Ok(Some(commitments)) => return Ok(commitments), - Ok(None) => sleep(Duration::from_millis(500)).await, - Err(e) => bail!("Error fetching sequencer commitments: {}", e), - } - } - } - - pub async fn wait_for_zkproofs( - &self, - height: u64, - timeout: Option, - ) -> Result> { - let start = Instant::now(); - let timeout = timeout.unwrap_or(Duration::from_secs(30)); - - loop { - if start.elapsed() >= timeout { - bail!("FullNode failed to get zkproofs within the specified timeout"); - } - - match self - .client - .ledger_get_verified_proofs_by_slot_height(height) - .await - { - Some(proofs) => return Ok(proofs), - None => sleep(Duration::from_millis(500)).await, - } - } - } -} - -impl Node for FullNode { - type Config = FullFullNodeConfig; - type Client = TestClient; - - fn spawn(config: &Self::Config) -> Result { - let citrea = get_citrea_path(); - let dir = &config.dir; - - let stdout_file = - File::create(get_stdout_path(dir)).context("Failed to create stdout file")?; - let stderr_file = - File::create(get_stderr_path(dir)).context("Failed to create stderr file")?; - - let rollup_config_path = dir.join("full_node_rollup_config.toml"); - config_to_file(&config.rollup, &rollup_config_path)?; - - Command::new(citrea) - .arg("--da-layer") - .arg("bitcoin") - .arg("--rollup-config-path") - .arg(rollup_config_path) - .arg("--genesis-paths") - .arg(get_genesis_path( - dir.parent().expect("Couldn't get parent dir"), - )) - .envs(config.env.clone()) - .stdout(Stdio::from(stdout_file)) - .stderr(Stdio::from(stderr_file)) - .kill_on_drop(true) - .spawn() - .context("Failed to spawn citrea process") - .map(SpawnOutput::Child) - } - - fn spawn_output(&mut self) -> &mut SpawnOutput { - &mut self.spawn_output - } - - async fn wait_for_ready(&self, timeout: Option) -> Result<()> { - let start = Instant::now(); - - let timeout = timeout.unwrap_or(Duration::from_secs(30)); - while start.elapsed() < timeout { - if self - .client - .ledger_get_head_soft_confirmation() - .await - .is_ok() - { - return Ok(()); - } - sleep(Duration::from_millis(500)).await; - } - bail!("FullNode failed to become ready within the specified timeout") - } - - fn client(&self) -> &Self::Client { - &self.client - } - - fn env(&self) -> Vec<(&'static str, &'static str)> { - self.config.env.clone() - } - - fn config_mut(&mut self) -> &mut Self::Config { - &mut self.config - } -} - -impl LogProvider for FullNode { - fn kind(&self) -> NodeKind { - NodeKind::FullNode - } - - fn log_path(&self) -> PathBuf { - get_stdout_path(&self.config.dir) - } -} diff --git a/bin/citrea/tests/bitcoin_e2e/tests/mempool_accept.rs b/bin/citrea/tests/bitcoin_e2e/mempool_accept.rs similarity index 83% rename from bin/citrea/tests/bitcoin_e2e/tests/mempool_accept.rs rename to bin/citrea/tests/bitcoin_e2e/mempool_accept.rs index 1e715b814..ab2d32990 100644 --- a/bin/citrea/tests/bitcoin_e2e/tests/mempool_accept.rs +++ b/bin/citrea/tests/bitcoin_e2e/mempool_accept.rs @@ -2,11 +2,11 @@ use async_trait::async_trait; use bitcoin_da::service::FINALITY_DEPTH; use bitcoincore_rpc::RpcApi; -use crate::bitcoin_e2e::config::BitcoinConfig; -use crate::bitcoin_e2e::framework::TestFramework; -use crate::bitcoin_e2e::node::L2Node; -use crate::bitcoin_e2e::test_case::{TestCase, TestCaseRunner}; -use crate::bitcoin_e2e::Result; +use citrea_e2e::config::BitcoinConfig; +use citrea_e2e::framework::TestFramework; +use citrea_e2e::test_case::{TestCase, TestCaseRunner}; +use citrea_e2e::traits::L2Node; +use citrea_e2e::Result; struct MempoolAcceptTest; @@ -31,7 +31,7 @@ impl TestCase for MempoolAcceptTest { // publish min_soft_conf_per_commitment - 1 confirmations, no commitments should be sent for _ in 0..min_soft_conf_per_commitment { - sequencer.client.send_publish_batch_request().await; + sequencer.client.send_publish_batch_request().await?; } sequencer .wait_for_l2_height(min_soft_conf_per_commitment, None) diff --git a/bin/citrea/tests/bitcoin_e2e/mod.rs b/bin/citrea/tests/bitcoin_e2e/mod.rs index 9cf7f4d7b..e951a6b23 100644 --- a/bin/citrea/tests/bitcoin_e2e/mod.rs +++ b/bin/citrea/tests/bitcoin_e2e/mod.rs @@ -1,15 +1,5 @@ -mod bitcoin; -pub mod config; -mod docker; -pub mod framework; -mod full_node; -pub mod node; -mod prover; -mod sequencer; -pub mod test_case; - -mod tests; - -mod utils; - -pub(crate) type Result = anyhow::Result; +pub mod bitcoin_test; +// pub mod mempool_accept; +pub mod prover_test; +pub mod sequencer_commitments; +pub mod sequencer_test; diff --git a/bin/citrea/tests/bitcoin_e2e/node.rs b/bin/citrea/tests/bitcoin_e2e/node.rs deleted file mode 100644 index d81ddb144..000000000 --- a/bin/citrea/tests/bitcoin_e2e/node.rs +++ /dev/null @@ -1,165 +0,0 @@ -use std::fmt; -use std::path::PathBuf; -use std::time::Duration; - -use anyhow::Context; -use bollard::container::StopContainerOptions; -use bollard::Docker; -use tokio::process::Child; - -use super::Result; -use crate::test_client::TestClient; -use crate::test_helpers::wait_for_l2_block; - -#[derive(Debug)] -pub enum NodeKind { - Bitcoin, - Prover, - Sequencer, - FullNode, -} - -impl fmt::Display for NodeKind { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - NodeKind::Bitcoin => write!(f, "bitcoin"), - NodeKind::Prover => write!(f, "prover"), - NodeKind::Sequencer => write!(f, "sequencer"), - NodeKind::FullNode => write!(f, "full-node"), - } - } -} - -#[derive(Debug)] -pub struct ContainerSpawnOutput { - pub id: String, - pub ip: String, -} - -#[derive(Debug)] -pub enum SpawnOutput { - Child(Child), - Container(ContainerSpawnOutput), -} - -/// The Node trait defines the common interface shared between -/// BitcoinNode, Prover, Sequencer and FullNode -pub(crate) trait Node { - type Config; - type Client; - - /// Spawn a new node with specific config and return its child - fn spawn(test_config: &Self::Config) -> Result; - fn spawn_output(&mut self) -> &mut SpawnOutput; - - fn config_mut(&mut self) -> &mut Self::Config; - - /// Stops the running node - async fn stop(&mut self) -> Result<()> { - match self.spawn_output() { - SpawnOutput::Child(process) => { - process - .kill() - .await - .context("Failed to kill child process")?; - Ok(()) - } - SpawnOutput::Container(ContainerSpawnOutput { id, .. }) => { - println!("Stopping container {id}"); - let docker = - Docker::connect_with_local_defaults().context("Failed to connect to Docker")?; - docker - .stop_container(id, Some(StopContainerOptions { t: 10 })) - .await - .context("Failed to stop Docker container")?; - Ok(()) - } - } - } - - /// Wait for the node to be reachable by its client. - async fn wait_for_ready(&self, timeout: Option) -> Result<()>; - - fn client(&self) -> &Self::Client; - - #[allow(unused)] - fn env(&self) -> Vec<(&'static str, &'static str)> { - Vec::new() - } -} - -pub trait L2Node: Node { - async fn wait_for_l2_height(&self, height: u64, timeout: Option); -} - -impl L2Node for T -where - T: Node, -{ - async fn wait_for_l2_height(&self, height: u64, timeout: Option) { - wait_for_l2_block(self.client(), height, timeout).await - } -} - -// Two patterns supported : -// - Call wait_until_stopped, runs any extra commands needed for testing purposes, call start again. -// - Call restart if you need to wait for node to be fully shutdown and brough back up with new config. -pub trait Restart: Node { - async fn wait_until_stopped(&mut self) -> Result<()>; - async fn start(&mut self, new_config: Option) -> Result<()>; - - // Default implementation to support waiting for node to be fully shutdown and brough back up with new config. - async fn restart(&mut self, new_config: Option) -> Result<()> { - self.wait_until_stopped().await?; - self.start(new_config).await - } -} - -impl Restart for T -where - T: L2Node, -{ - async fn wait_until_stopped(&mut self) -> Result<()> { - self.stop().await?; - match self.spawn_output() { - SpawnOutput::Child(pid) => pid.wait().await?, - SpawnOutput::Container(_) => unimplemented!("L2 nodes don't run in docker yet"), - }; - Ok(()) - } - - async fn start(&mut self, new_config: Option) -> Result<()> { - let config = self.config_mut(); - if let Some(new_config) = new_config { - *config = new_config - } - *self.spawn_output() = Self::spawn(config)?; - self.wait_for_ready(None).await - } -} - -pub trait LogProvider: Node { - fn kind(&self) -> NodeKind; - fn log_path(&self) -> PathBuf; - fn as_erased(&self) -> &dyn LogProviderErased - where - Self: Sized, - { - self - } -} - -pub trait LogProviderErased { - fn kind(&self) -> NodeKind; - fn log_path(&self) -> PathBuf; -} - -impl LogProviderErased for T { - fn kind(&self) -> NodeKind { - LogProvider::kind(self) - } - - fn log_path(&self) -> PathBuf { - LogProvider::log_path(self) - } -} diff --git a/bin/citrea/tests/bitcoin_e2e/prover.rs b/bin/citrea/tests/bitcoin_e2e/prover.rs deleted file mode 100644 index 54a5f3a05..000000000 --- a/bin/citrea/tests/bitcoin_e2e/prover.rs +++ /dev/null @@ -1,140 +0,0 @@ -use std::fs::File; -use std::net::SocketAddr; -use std::path::PathBuf; -use std::process::Stdio; - -use anyhow::Context; -use tokio::process::Command; -use tokio::time::{sleep, Duration, Instant}; - -use super::config::{config_to_file, FullProverConfig, TestConfig}; -use super::framework::TestContext; -use super::node::{LogProvider, Node, NodeKind, SpawnOutput}; -use super::utils::{get_citrea_path, get_stderr_path, get_stdout_path, retry}; -use super::Result; -use crate::bitcoin_e2e::utils::get_genesis_path; -use crate::evm::make_test_client; -use crate::test_client::TestClient; -use crate::test_helpers::wait_for_prover_l1_height; - -#[allow(unused)] -pub struct Prover { - spawn_output: SpawnOutput, - config: FullProverConfig, - pub client: Box, -} - -impl Prover { - pub async fn new(ctx: &TestContext) -> Result { - let TestConfig { - prover: prover_config, - .. - } = &ctx.config; - - let spawn_output = Self::spawn(prover_config)?; - - let socket_addr = SocketAddr::new( - prover_config - .rollup - .rpc - .bind_host - .parse() - .context("Failed to parse bind host")?, - prover_config.rollup.rpc.bind_port, - ); - let client = retry(|| async { make_test_client(socket_addr).await }, None).await?; - - Ok(Self { - spawn_output, - config: prover_config.to_owned(), - client, - }) - } - - pub async fn wait_for_l1_height(&self, height: u64, timeout: Option) -> Result<()> { - wait_for_prover_l1_height(&self.client, height, timeout).await - } -} - -impl Node for Prover { - type Config = FullProverConfig; - type Client = TestClient; - - fn spawn(config: &Self::Config) -> Result { - let citrea = get_citrea_path(); - let dir = &config.dir; - - let stdout_file = - File::create(get_stdout_path(dir)).context("Failed to create stdout file")?; - let stderr_file = - File::create(get_stderr_path(dir)).context("Failed to create stderr file")?; - - let config_path = dir.join("prover_config.toml"); - config_to_file(&config.node, &config_path)?; - - let rollup_config_path = dir.join("prover_rollup_config.toml"); - config_to_file(&config.rollup, &rollup_config_path)?; - - Command::new(citrea) - .arg("--da-layer") - .arg("bitcoin") - .arg("--rollup-config-path") - .arg(rollup_config_path) - .arg("--prover-config-path") - .arg(config_path) - .arg("--genesis-paths") - .arg(get_genesis_path( - dir.parent().expect("Couldn't get parent dir"), - )) - .envs(config.env.clone()) - .stdout(Stdio::from(stdout_file)) - .stderr(Stdio::from(stderr_file)) - .kill_on_drop(true) - .spawn() - .context("Failed to spawn citrea process") - .map(SpawnOutput::Child) - } - - fn spawn_output(&mut self) -> &mut SpawnOutput { - &mut self.spawn_output - } - - async fn wait_for_ready(&self, timeout: Option) -> Result<()> { - let start = Instant::now(); - let timeout = timeout.unwrap_or(Duration::from_secs(30)); - while start.elapsed() < timeout { - if self - .client - .ledger_get_head_soft_confirmation() - .await - .is_ok() - { - return Ok(()); - } - sleep(Duration::from_millis(500)).await; - } - anyhow::bail!("Prover failed to become ready within the specified timeout") - } - - fn client(&self) -> &Self::Client { - &self.client - } - - fn env(&self) -> Vec<(&'static str, &'static str)> { - self.config.env.clone() - } - - fn config_mut(&mut self) -> &mut Self::Config { - &mut self.config - } -} - -impl LogProvider for Prover { - fn kind(&self) -> NodeKind { - NodeKind::Prover - } - - fn log_path(&self) -> PathBuf { - get_stdout_path(&self.config.dir) - } -} diff --git a/bin/citrea/tests/bitcoin_e2e/tests/prover_test.rs b/bin/citrea/tests/bitcoin_e2e/prover_test.rs similarity index 93% rename from bin/citrea/tests/bitcoin_e2e/tests/prover_test.rs rename to bin/citrea/tests/bitcoin_e2e/prover_test.rs index 7cbf6ac60..d9deac1f8 100644 --- a/bin/citrea/tests/bitcoin_e2e/tests/prover_test.rs +++ b/bin/citrea/tests/bitcoin_e2e/prover_test.rs @@ -6,18 +6,16 @@ use async_trait::async_trait; use bitcoin_da::service::{BitcoinService, BitcoinServiceConfig, TxidWrapper, FINALITY_DEPTH}; use bitcoin_da::spec::RollupParams; use bitcoincore_rpc::RpcApi; +use citrea_e2e::config::{SequencerConfig, TestCaseConfig}; +use citrea_e2e::framework::TestFramework; +use citrea_e2e::node::NodeKind; +use citrea_e2e::test_case::{TestCase, TestCaseRunner}; +use citrea_e2e::Result; use citrea_primitives::{REVEAL_BATCH_PROOF_PREFIX, REVEAL_LIGHT_CLIENT_PREFIX}; use sov_rollup_interface::da::{DaData, SequencerCommitment}; use sov_rollup_interface::services::da::SenderWithNotifier; use tokio::sync::mpsc::UnboundedSender; -use crate::bitcoin_e2e::config::{SequencerConfig, TestCaseConfig}; -use crate::bitcoin_e2e::framework::TestFramework; -use crate::bitcoin_e2e::node::NodeKind; -use crate::bitcoin_e2e::test_case::{TestCase, TestCaseRunner}; -use crate::bitcoin_e2e::utils::get_tx_backup_dir; -use crate::bitcoin_e2e::Result; - /// This is a basic prover test showcasing spawning a bitcoin node as DA, a sequencer and a prover. /// It generates soft confirmations and wait until it reaches the first commitment. /// It asserts that the blob inscribe txs have been sent. @@ -61,14 +59,11 @@ impl TestCase for BasicProverTest { // Generate confirmed UTXOs da.generate(120, None).await?; - let seq_height0 = sequencer.client.eth_block_number().await; - assert_eq!(seq_height0, 0); - let min_soft_confirmations_per_commitment = sequencer.min_soft_confirmations_per_commitment(); for _ in 0..min_soft_confirmations_per_commitment { - sequencer.client.send_publish_batch_request().await; + sequencer.client.send_publish_batch_request().await?; } da.generate(FINALITY_DEPTH, None).await?; @@ -163,7 +158,11 @@ impl TestCase for SkipPreprovenCommitmentsTest { // somehow resubmitted the same commitment. "045FFC81A3C1FDB3AF1359DBF2D114B0B3EFBF7F29CC9C5DA01267AA39D2C78D".to_owned(), ), - tx_backup_dir: get_tx_backup_dir(), + tx_backup_dir: Self::test_config() + .dir + .join("tx_backup_dir") + .display() + .to_string(), }; let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); // Keep sender for cleanup @@ -186,14 +185,11 @@ impl TestCase for SkipPreprovenCommitmentsTest { // Generate 1 FINALIZED DA block. da.generate(1 + FINALITY_DEPTH, None).await?; - let seq_height0 = sequencer.client.eth_block_number().await; - assert_eq!(seq_height0, 0); - let min_soft_confirmations_per_commitment = sequencer.min_soft_confirmations_per_commitment(); for _ in 0..min_soft_confirmations_per_commitment { - sequencer.client.send_publish_batch_request().await; + sequencer.client.send_publish_batch_request().await?; } da.generate(FINALITY_DEPTH, None).await?; @@ -263,7 +259,7 @@ impl TestCase for SkipPreprovenCommitmentsTest { // Trigger a new commitment. for _ in 0..min_soft_confirmations_per_commitment { - sequencer.client.send_publish_batch_request().await; + sequencer.client.send_publish_batch_request().await?; } // Wait for the sequencer commitment to be submitted & accepted. diff --git a/bin/citrea/tests/bitcoin_e2e/sequencer.rs b/bin/citrea/tests/bitcoin_e2e/sequencer.rs deleted file mode 100644 index 37d5fc1f2..000000000 --- a/bin/citrea/tests/bitcoin_e2e/sequencer.rs +++ /dev/null @@ -1,143 +0,0 @@ -use std::fs::File; -use std::net::SocketAddr; -use std::path::PathBuf; -use std::process::Stdio; - -use anyhow::Context; -use tokio::process::Command; -use tokio::time::{sleep, Duration, Instant}; - -use super::config::{config_to_file, FullSequencerConfig, TestConfig}; -use super::framework::TestContext; -use super::node::{LogProvider, Node, NodeKind, SpawnOutput}; -use super::utils::{get_citrea_path, get_stderr_path, get_stdout_path, retry}; -use super::Result; -use crate::bitcoin_e2e::utils::get_genesis_path; -use crate::evm::make_test_client; -use crate::test_client::TestClient; - -#[allow(unused)] -pub struct Sequencer { - spawn_output: SpawnOutput, - config: FullSequencerConfig, - pub client: Box, -} - -impl Sequencer { - pub async fn new(ctx: &TestContext) -> Result { - let TestConfig { - sequencer: config, .. - } = &ctx.config; - - let spawn_output = Self::spawn(config)?; - - let socket_addr = SocketAddr::new( - config - .rollup - .rpc - .bind_host - .parse() - .context("Failed to parse bind host")?, - config.rollup.rpc.bind_port, - ); - - let client = retry(|| async { make_test_client(socket_addr).await }, None).await?; - - Ok(Self { - spawn_output, - config: config.clone(), - client, - }) - } - - pub fn dir(&self) -> &PathBuf { - &self.config.dir - } - - pub fn min_soft_confirmations_per_commitment(&self) -> u64 { - self.config.node.min_soft_confirmations_per_commitment - } -} - -impl Node for Sequencer { - type Config = FullSequencerConfig; - type Client = TestClient; - - fn spawn(config: &Self::Config) -> Result { - let citrea = get_citrea_path(); - let dir = &config.dir; - - let stdout_file = - File::create(get_stdout_path(dir)).context("Failed to create stdout file")?; - let stderr_file = - File::create(get_stderr_path(dir)).context("Failed to create stderr file")?; - - let config_path = dir.join("sequencer_config.toml"); - config_to_file(&config.node, &config_path)?; - - let rollup_config_path = dir.join("sequencer_rollup_config.toml"); - config_to_file(&config.rollup, &rollup_config_path)?; - - Command::new(citrea) - .arg("--da-layer") - .arg("bitcoin") - .arg("--rollup-config-path") - .arg(rollup_config_path) - .arg("--sequencer-config-path") - .arg(config_path) - .arg("--genesis-paths") - .arg(get_genesis_path( - dir.parent().expect("Couldn't get parent dir"), - )) - .envs(config.env.clone()) - .stdout(Stdio::from(stdout_file)) - .stderr(Stdio::from(stderr_file)) - .kill_on_drop(true) - .spawn() - .context("Failed to spawn citrea process") - .map(SpawnOutput::Child) - } - - fn spawn_output(&mut self) -> &mut SpawnOutput { - &mut self.spawn_output - } - - async fn wait_for_ready(&self, timeout: Option) -> Result<()> { - let start = Instant::now(); - let timeout = timeout.unwrap_or(Duration::from_secs(30)); - while start.elapsed() < timeout { - if self - .client - .ledger_get_head_soft_confirmation() - .await - .is_ok() - { - return Ok(()); - } - sleep(Duration::from_millis(500)).await; - } - anyhow::bail!("Sequencer failed to become ready within the specified timeout") - } - - fn client(&self) -> &Self::Client { - &self.client - } - - fn env(&self) -> Vec<(&'static str, &'static str)> { - self.config.env.clone() - } - - fn config_mut(&mut self) -> &mut Self::Config { - &mut self.config - } -} - -impl LogProvider for Sequencer { - fn kind(&self) -> NodeKind { - NodeKind::Sequencer - } - - fn log_path(&self) -> PathBuf { - get_stdout_path(self.dir()) - } -} diff --git a/bin/citrea/tests/bitcoin_e2e/tests/sequencer_commitments.rs b/bin/citrea/tests/bitcoin_e2e/sequencer_commitments.rs similarity index 84% rename from bin/citrea/tests/bitcoin_e2e/tests/sequencer_commitments.rs rename to bin/citrea/tests/bitcoin_e2e/sequencer_commitments.rs index dcbb2506e..623d1855a 100644 --- a/bin/citrea/tests/bitcoin_e2e/tests/sequencer_commitments.rs +++ b/bin/citrea/tests/bitcoin_e2e/sequencer_commitments.rs @@ -1,22 +1,18 @@ use async_trait::async_trait; use bitcoin::hashes::Hash; -use bitcoin_da::service::FINALITY_DEPTH; -use bitcoin_da::spec::BitcoinSpec; +use bitcoin_da::service::{get_relevant_blobs_from_txs, FINALITY_DEPTH}; use bitcoincore_rpc::RpcApi; use borsh::BorshDeserialize; +use citrea_e2e::bitcoin::BitcoinNode; +use citrea_e2e::config::{SequencerConfig, TestCaseConfig}; +use citrea_e2e::framework::TestFramework; +use citrea_e2e::sequencer::Sequencer; +use citrea_e2e::test_case::{TestCase, TestCaseRunner}; +use citrea_e2e::Result; +use citrea_primitives::REVEAL_BATCH_PROOF_PREFIX; use rs_merkle::algorithms::Sha256; use rs_merkle::MerkleTree; -use sov_modules_api::BlobReaderTrait; -use sov_rollup_interface::da::DaData; - -use crate::bitcoin_e2e::bitcoin::BitcoinNode; -use crate::bitcoin_e2e::config::{SequencerConfig, TestCaseConfig}; -use crate::bitcoin_e2e::framework::TestFramework; -use crate::bitcoin_e2e::node::L2Node; -use crate::bitcoin_e2e::sequencer::Sequencer; -use crate::bitcoin_e2e::test_case::{TestCase, TestCaseRunner}; -use crate::bitcoin_e2e::Result; - +use sov_rollup_interface::da::{BlobReaderTrait, DaData}; struct LedgerGetCommitmentsProverTest; #[async_trait] @@ -41,11 +37,11 @@ impl TestCase for LedgerGetCommitmentsProverTest { sequencer.min_soft_confirmations_per_commitment(); for _ in 0..min_soft_confirmations_per_commitment { - sequencer.client.send_publish_batch_request().await; + sequencer.client.send_publish_batch_request().await?; } sequencer .wait_for_l2_height(min_soft_confirmations_per_commitment, None) - .await; + .await?; // Wait for blob tx to hit the mempool da.wait_mempool_len(1, None).await?; @@ -115,13 +111,13 @@ impl TestCase for LedgerGetCommitmentsTest { sequencer.min_soft_confirmations_per_commitment(); for _ in 0..min_soft_confirmations_per_commitment { - sequencer.client.send_publish_batch_request().await; + sequencer.client.send_publish_batch_request().await?; } // disable this since it's the only difference from other tests?? // da.generate(1, None).await?; - // sequencer.client.send_publish_batch_request().await; + // sequencer.client.send_publish_batch_request().await?; // Wait for blob tx to hit the mempool da.wait_mempool_len(1, None).await?; @@ -131,7 +127,7 @@ impl TestCase for LedgerGetCommitmentsTest { full_node .wait_for_l2_height(min_soft_confirmations_per_commitment, None) - .await; + .await?; let finalized_height = da.get_finalized_height().await?; @@ -185,11 +181,11 @@ impl TestCase for SequencerSendCommitmentsToDaTest { // publish min_soft_confirmations_per_commitment - 1 confirmations, no commitments should be sent for _ in 0..min_soft_confirmations_per_commitment - 1 { - sequencer.client.send_publish_batch_request().await; + sequencer.client.send_publish_batch_request().await?; } sequencer .wait_for_l2_height(min_soft_confirmations_per_commitment - 1, None) - .await; + .await?; da.generate(FINALITY_DEPTH, None).await?; tokio::time::sleep(std::time::Duration::from_millis(1000)).await; @@ -197,24 +193,27 @@ impl TestCase for SequencerSendCommitmentsToDaTest { let finalized_height = da.get_finalized_height().await?; for height in initial_height..finalized_height { - let mut blobs = da.get_relevant_blobs_from_block(height).await?; + let hash = da.get_block_hash(height).await?; + let block = da.get_block(&hash).await?; + + let mut blobs = get_relevant_blobs_from_txs(block.txdata, REVEAL_BATCH_PROOF_PREFIX); for mut blob in blobs.drain(0..) { - let data = blob.full_data(); + let data = BlobReaderTrait::full_data(&mut blob); assert_eq!(data, &[] as &[u8]); } } // Publish one more L2 block and send commitment - sequencer.client.send_publish_batch_request().await; + sequencer.client.send_publish_batch_request().await?; sequencer .wait_for_l2_height( min_soft_confirmations_per_commitment + FINALITY_DEPTH - 1, None, ) - .await; + .await?; // Wait for blob tx to hit the mempool da.wait_mempool_len(1, None).await?; @@ -230,14 +229,14 @@ impl TestCase for SequencerSendCommitmentsToDaTest { .await?; for _ in 0..min_soft_confirmations_per_commitment { - sequencer.client.send_publish_batch_request().await; + sequencer.client.send_publish_batch_request().await?; } sequencer .wait_for_l2_height( end_l2_block + min_soft_confirmations_per_commitment + FINALITY_DEPTH - 2, None, ) - .await; + .await?; // Wait for blob tx to hit the mempool da.wait_mempool_len(1, None).await?; @@ -265,13 +264,16 @@ impl SequencerSendCommitmentsToDaTest { let finalized_height = da.get_finalized_height().await?; // Extract and verify the commitment from the block - let mut blobs = da.get_relevant_blobs_from_block(finalized_height).await?; + let hash = da.get_block_hash(finalized_height).await?; + let block = da.get_block(&hash).await?; + + let mut blobs = get_relevant_blobs_from_txs(block.txdata, REVEAL_BATCH_PROOF_PREFIX); assert_eq!(blobs.len(), 1); let mut blob = blobs.pop().unwrap(); - let data = blob.full_data(); + let data = BlobReaderTrait::full_data(&mut blob); let commitment = DaData::try_from_slice(data).unwrap(); @@ -287,8 +289,8 @@ impl SequencerSendCommitmentsToDaTest { soft_confirmations.push( sequencer .client - .ledger_get_soft_confirmation_by_number::(i) - .await + .ledger_get_soft_confirmation_by_number(i) + .await? .unwrap(), ); } diff --git a/bin/citrea/tests/bitcoin_e2e/tests/sequencer_test.rs b/bin/citrea/tests/bitcoin_e2e/sequencer_test.rs similarity index 76% rename from bin/citrea/tests/bitcoin_e2e/tests/sequencer_test.rs rename to bin/citrea/tests/bitcoin_e2e/sequencer_test.rs index 10edbba4b..fa66f1df0 100644 --- a/bin/citrea/tests/bitcoin_e2e/tests/sequencer_test.rs +++ b/bin/citrea/tests/bitcoin_e2e/sequencer_test.rs @@ -1,13 +1,11 @@ use anyhow::bail; use async_trait::async_trait; -use bitcoin_da::spec::BitcoinSpec; use bitcoincore_rpc::RpcApi; -use citrea_sequencer::SequencerConfig; - -use crate::bitcoin_e2e::framework::TestFramework; -use crate::bitcoin_e2e::node::{L2Node, Restart}; -use crate::bitcoin_e2e::test_case::{TestCase, TestCaseRunner}; -use crate::bitcoin_e2e::Result; +use citrea_e2e::config::SequencerConfig; +use citrea_e2e::framework::TestFramework; +use citrea_e2e::test_case::{TestCase, TestCaseRunner}; +use citrea_e2e::traits::Restart; +use citrea_e2e::Result; struct BasicSequencerTest; @@ -22,15 +20,26 @@ impl TestCase for BasicSequencerTest { bail!("bitcoind not running. Test cannot run with bitcoind runnign as DA") }; - let seq_height0 = sequencer.client.eth_block_number().await; - assert_eq!(seq_height0, 0); + sequencer.client.send_publish_batch_request().await?; + + let head_batch0 = sequencer + .client + .ledger_get_head_soft_confirmation() + .await? + .unwrap(); + assert_eq!(head_batch0.l2_height, 1); + + sequencer.client.send_publish_batch_request().await?; - sequencer.client.send_publish_batch_request().await; da.generate(1, None).await?; - sequencer.wait_for_l2_height(1, None).await; - let seq_height1 = sequencer.client.eth_block_number().await; - assert_eq!(seq_height1, 1); + sequencer.client.wait_for_l2_block(1, None).await?; + let head_batch1 = sequencer + .client + .ledger_get_head_soft_confirmation() + .await? + .unwrap(); + assert_eq!(head_batch1.l2_height, 2); Ok(()) } @@ -68,7 +77,7 @@ impl TestCase for SequencerMissedDaBlocksTest { // Create initial DA blocks da.generate(3, None).await?; - sequencer.client.send_publish_batch_request().await; + sequencer.client.send_publish_batch_request().await?; sequencer.wait_until_stopped().await?; @@ -79,15 +88,13 @@ impl TestCase for SequencerMissedDaBlocksTest { sequencer.start(None).await?; for _ in 0..10 { - sequencer.client.send_publish_batch_request().await; + sequencer.client.send_publish_batch_request().await?; } let head_soft_confirmation_height = sequencer .client .ledger_get_head_soft_confirmation_height() - .await - .unwrap() - .unwrap(); + .await?; let mut last_used_l1_height = initial_l1_height; @@ -98,8 +105,8 @@ impl TestCase for SequencerMissedDaBlocksTest { for i in 1..=head_soft_confirmation_height { let soft_confirmation = sequencer .client - .ledger_get_soft_confirmation_by_number::(i) - .await + .ledger_get_soft_confirmation_by_number(i) + .await? .unwrap(); if i == 1 { diff --git a/bin/citrea/tests/bitcoin_e2e/test_case.rs b/bin/citrea/tests/bitcoin_e2e/test_case.rs deleted file mode 100644 index ff0e57797..000000000 --- a/bin/citrea/tests/bitcoin_e2e/test_case.rs +++ /dev/null @@ -1,344 +0,0 @@ -//! This module provides the TestCaseRunner and TestCase trait for running and defining test cases. -//! It handles setup, execution, and cleanup of test environments. - -use std::panic::{self}; -use std::path::{Path, PathBuf}; -use std::time::Duration; - -use anyhow::{bail, Context}; -use async_trait::async_trait; -use bitcoin_da::service::BitcoinServiceConfig; -use citrea_sequencer::SequencerConfig; -use futures::FutureExt; -use sov_stf_runner::{ProverConfig, RpcConfig, RunnerConfig, StorageConfig}; - -use super::config::{ - default_rollup_config, BitcoinConfig, FullFullNodeConfig, FullProverConfig, - FullSequencerConfig, RollupConfig, TestCaseConfig, TestCaseEnv, TestConfig, -}; -use super::framework::TestFramework; -use super::node::NodeKind; -use super::utils::{copy_directory, get_available_port, get_tx_backup_dir}; -use super::Result; -use crate::bitcoin_e2e::node::Node; -use crate::bitcoin_e2e::utils::{get_default_genesis_path, get_workspace_root}; - -// TestCaseRunner manages the lifecycle of a test case, including setup, execution, and cleanup. -/// It creates a test framework with the associated configs, spawns required nodes, connects them, -/// runs the test case, and performs cleanup afterwards. The `run` method handles any panics that -/// might occur during test execution and takes care of cleaning up and stopping the child processes. -pub struct TestCaseRunner(T); - -impl TestCaseRunner { - /// Creates a new TestCaseRunner with the given test case. - pub fn new(test_case: T) -> Self { - Self(test_case) - } - - /// Internal method to fund the wallets, connect the nodes, wait for them to be ready. - async fn prepare(&self, f: &mut TestFramework) -> Result<()> { - f.fund_da_wallets().await?; - f.init_nodes().await?; - f.show_log_paths(); - f.bitcoin_nodes.connect_nodes().await?; - - if let Some(sequencer) = &f.sequencer { - sequencer - .wait_for_ready(Some(Duration::from_secs(5))) - .await?; - } - - Ok(()) - } - - async fn run_test_case(&mut self, f: &mut TestFramework) -> Result<()> { - self.prepare(f).await?; - self.0.setup(f).await?; - self.0.run_test(f).await - } - - /// Executes the test case, handling any panics and performing cleanup. - /// - /// This sets up the framework, executes the test, and ensures cleanup is performed even if a panic occurs. - pub async fn run(mut self) -> Result<()> { - let mut framework = None; - let result = panic::AssertUnwindSafe(async { - framework = Some(TestFramework::new(Self::generate_test_config()?).await?); - let f = framework.as_mut().unwrap(); - self.run_test_case(f).await - }) - .catch_unwind() - .await; - - let f = framework - .as_mut() - .expect("Framework not correctly initialized"); - - if result.is_err() { - if let Err(e) = f.dump_log() { - eprintln!("Error dumping log: {}", e); - } - } - - f.stop().await?; - - // Additional test cleanup - self.0.cleanup().await?; - - match result { - Ok(Ok(())) => Ok(()), - Ok(Err(e)) => Err(e), - Err(panic_error) => { - let panic_msg = panic_error - .downcast_ref::() - .map(|s| s.to_string()) - .unwrap_or_else(|| "Unknown panic".to_string()); - bail!(panic_msg) - } - } - } - - fn generate_test_config() -> Result { - let test_case = T::test_config(); - let env = T::test_env(); - let bitcoin = T::bitcoin_config(); - let prover = T::prover_config(); - let sequencer = T::sequencer_config(); - let sequencer_rollup = default_rollup_config(); - let prover_rollup = default_rollup_config(); - let full_node_rollup = default_rollup_config(); - - let [bitcoin_dir, dbs_dir, prover_dir, sequencer_dir, full_node_dir, genesis_dir] = - create_dirs(&test_case.dir)?; - - copy_genesis_dir(&test_case.genesis_dir, &genesis_dir)?; - - let mut bitcoin_confs = vec![]; - for i in 0..test_case.n_nodes { - let data_dir = bitcoin_dir.join(i.to_string()); - std::fs::create_dir_all(&data_dir) - .with_context(|| format!("Failed to create {} directory", data_dir.display()))?; - - let p2p_port = get_available_port()?; - let rpc_port = get_available_port()?; - - bitcoin_confs.push(BitcoinConfig { - p2p_port, - rpc_port, - data_dir, - env: env.bitcoin().clone(), - idx: i, - ..bitcoin.clone() - }) - } - - // Target first bitcoin node as DA for now - let da_config: BitcoinServiceConfig = bitcoin_confs[0].clone().into(); - - let sequencer_rollup = { - let bind_port = get_available_port()?; - let node_kind = NodeKind::Sequencer.to_string(); - RollupConfig { - da: BitcoinServiceConfig { - da_private_key: Some( - "045FFC81A3C1FDB3AF1359DBF2D114B0B3EFBF7F29CC9C5DA01267AA39D2C78D" - .to_string(), - ), - node_url: format!("http://{}/wallet/{}", da_config.node_url, node_kind), - tx_backup_dir: get_tx_backup_dir(), - ..da_config.clone() - }, - storage: StorageConfig { - path: dbs_dir.join(format!("{}-db", node_kind)), - db_max_open_files: None, - }, - rpc: RpcConfig { - bind_port, - ..sequencer_rollup.rpc - }, - ..sequencer_rollup - } - }; - - let runner_config = Some(RunnerConfig { - sequencer_client_url: format!( - "http://{}:{}", - sequencer_rollup.rpc.bind_host, sequencer_rollup.rpc.bind_port, - ), - include_tx_body: true, - accept_public_input_as_proven: Some(true), - pruning_config: None, - sync_blocks_count: 10, - }); - - let prover_rollup = { - let bind_port = get_available_port()?; - let node_kind = NodeKind::Prover.to_string(); - RollupConfig { - da: BitcoinServiceConfig { - da_private_key: Some( - "75BAF964D074594600366E5B111A1DA8F86B2EFE2D22DA51C8D82126A0FCAC72" - .to_string(), - ), - node_url: format!("http://{}/wallet/{}", da_config.node_url, node_kind), - tx_backup_dir: get_tx_backup_dir(), - ..da_config.clone() - }, - storage: StorageConfig { - path: dbs_dir.join(format!("{}-db", node_kind)), - db_max_open_files: None, - }, - rpc: RpcConfig { - bind_port, - ..prover_rollup.rpc - }, - runner: runner_config.clone(), - ..prover_rollup - } - }; - - let full_node_rollup = { - let bind_port = get_available_port()?; - let node_kind = NodeKind::FullNode.to_string(); - RollupConfig { - da: BitcoinServiceConfig { - node_url: format!( - "http://{}/wallet/{}", - da_config.node_url, - NodeKind::Bitcoin // Use default wallet - ), - tx_backup_dir: get_tx_backup_dir(), - ..da_config.clone() - }, - storage: StorageConfig { - path: dbs_dir.join(format!("{}-db", node_kind)), - db_max_open_files: None, - }, - rpc: RpcConfig { - bind_port, - ..full_node_rollup.rpc - }, - runner: runner_config.clone(), - ..full_node_rollup - } - }; - - Ok(TestConfig { - bitcoin: bitcoin_confs, - sequencer: FullSequencerConfig { - rollup: sequencer_rollup, - dir: sequencer_dir, - docker_image: None, - node: sequencer, - env: env.sequencer(), - }, - prover: FullProverConfig { - rollup: prover_rollup, - dir: prover_dir, - docker_image: None, - node: prover, - env: env.prover(), - }, - full_node: FullFullNodeConfig { - rollup: full_node_rollup, - dir: full_node_dir, - docker_image: None, - node: (), - env: env.full_node(), - }, - test_case, - }) - } -} - -/// Defines the interface for implementing test cases. -/// -/// This trait should be implemented by every test case to define the configuration -/// and inner test logic. It provides default configurations that should be sane for most test cases, -/// which can be overridden by implementing the associated methods. -#[async_trait] -pub trait TestCase: Send + Sync + 'static { - /// Returns the test case configuration. - /// Override this method to provide custom test configurations. - fn test_config() -> TestCaseConfig { - TestCaseConfig::default() - } - - /// Returns the test case env. - /// Override this method to provide custom env per node. - fn test_env() -> TestCaseEnv { - TestCaseEnv::default() - } - - /// Returns the Bitcoin configuration for the test. - /// Override this method to provide a custom Bitcoin configuration. - fn bitcoin_config() -> BitcoinConfig { - BitcoinConfig::default() - } - - /// Returns the sequencer configuration for the test. - /// Override this method to provide a custom sequencer configuration. - fn sequencer_config() -> SequencerConfig { - SequencerConfig::default() - } - - /// Returns the prover configuration for the test. - /// Override this method to provide a custom prover configuration. - fn prover_config() -> ProverConfig { - ProverConfig::default() - } - - /// Returns the test setup - /// Override this method to add custom initialization logic - async fn setup(&self, _framework: &mut TestFramework) -> Result<()> { - Ok(()) - } - - /// Implements the actual test logic. - /// - /// This method is where the test case should be implemented. It receives - /// a reference to the TestFramework, which provides access to the test environment. - /// - /// # Arguments - /// * `framework` - A reference to the TestFramework instance - async fn run_test(&mut self, framework: &mut TestFramework) -> Result<()>; - - async fn cleanup(&self) -> Result<()> { - Ok(()) - } -} - -fn create_dirs(base_dir: &Path) -> Result<[PathBuf; 6]> { - let paths = [ - NodeKind::Bitcoin.to_string(), - "dbs".to_string(), - NodeKind::Prover.to_string(), - NodeKind::Sequencer.to_string(), - NodeKind::FullNode.to_string(), - "genesis".to_string(), - ] - .map(|dir| base_dir.join(dir)); - - for path in &paths { - std::fs::create_dir_all(path) - .with_context(|| format!("Failed to create {} directory", path.display()))?; - } - - Ok(paths) -} - -fn copy_genesis_dir(genesis_dir: &Option, target_dir: &Path) -> std::io::Result<()> { - let genesis_dir = - genesis_dir - .as_ref() - .map(PathBuf::from) - .map_or_else(get_default_genesis_path, |dir| { - if dir.is_absolute() { - dir - } else { - get_workspace_root().join(dir) - } - }); - - copy_directory(genesis_dir, target_dir) -} diff --git a/bin/citrea/tests/bitcoin_e2e/tests/mod.rs b/bin/citrea/tests/bitcoin_e2e/tests/mod.rs deleted file mode 100644 index e951a6b23..000000000 --- a/bin/citrea/tests/bitcoin_e2e/tests/mod.rs +++ /dev/null @@ -1,5 +0,0 @@ -pub mod bitcoin_test; -// pub mod mempool_accept; -pub mod prover_test; -pub mod sequencer_commitments; -pub mod sequencer_test; diff --git a/bin/citrea/tests/bitcoin_e2e/utils.rs b/bin/citrea/tests/bitcoin_e2e/utils.rs deleted file mode 100644 index 05e395f9c..000000000 --- a/bin/citrea/tests/bitcoin_e2e/utils.rs +++ /dev/null @@ -1,149 +0,0 @@ -use std::fs::File; -use std::future::Future; -use std::io::{BufRead, BufReader}; -use std::net::TcpListener; -use std::path::{Path, PathBuf}; -use std::{fs, io}; - -use anyhow::bail; -use rand::distributions::Alphanumeric; -use rand::{thread_rng, Rng}; -use tokio::time::{sleep, Duration, Instant}; - -use super::Result; - -pub fn get_available_port() -> Result { - let listener = TcpListener::bind("127.0.0.1:0")?; - Ok(listener.local_addr()?.port()) -} - -pub fn get_workspace_root() -> PathBuf { - let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); - manifest_dir - .ancestors() - .nth(2) - .expect("Failed to find workspace root") - .to_path_buf() -} - -/// Get citrea path from CITREA env or resolves to debug build. -pub fn get_citrea_path() -> PathBuf { - std::env::var("CITREA").map_or_else( - |_| { - let workspace_root = get_workspace_root(); - let mut path = workspace_root.to_path_buf(); - path.push("target"); - path.push("debug"); - path.push("citrea"); - path - }, - PathBuf::from, - ) -} - -pub fn get_stdout_path(dir: &Path) -> PathBuf { - dir.join("stdout.log") -} - -pub fn get_stderr_path(dir: &Path) -> PathBuf { - dir.join("stderr.log") -} - -/// Get genesis path from resources -/// TODO: assess need for customable genesis path in e2e tests -pub fn get_default_genesis_path() -> PathBuf { - let workspace_root = get_workspace_root(); - let mut path = workspace_root.to_path_buf(); - path.push("resources"); - path.push("genesis"); - path.push("bitcoin-regtest"); - path -} - -pub fn get_genesis_path(dir: &Path) -> PathBuf { - dir.join("genesis") -} - -pub fn generate_test_id() -> String { - thread_rng() - .sample_iter(&Alphanumeric) - .take(10) - .map(char::from) - .collect() -} - -pub fn copy_directory(src: impl AsRef, dst: impl AsRef) -> io::Result<()> { - let src = src.as_ref(); - let dst = dst.as_ref(); - - if !dst.exists() { - fs::create_dir_all(dst)?; - } - - for entry in fs::read_dir(src)? { - let entry = entry?; - let ty = entry.file_type()?; - let file_name = entry.file_name(); - let src_path = src.join(&file_name); - let dst_path = dst.join(&file_name); - - if ty.is_dir() { - copy_directory(&src_path, &dst_path)?; - } else { - fs::copy(&src_path, &dst_path)?; - } - } - - Ok(()) -} - -pub(crate) async fn retry(f: F, timeout: Option) -> Result -where - F: Fn() -> Fut, - Fut: Future>, -{ - let start = Instant::now(); - let timeout = start + timeout.unwrap_or_else(|| Duration::from_secs(5)); - - loop { - match tokio::time::timeout_at(timeout, f()).await { - Ok(Ok(result)) => return Ok(result), - Ok(Err(e)) => { - if Instant::now() >= timeout { - return Err(e); - } - sleep(Duration::from_millis(500)).await; - } - Err(elapsed) => bail!("Timeout expired {elapsed}"), - } - } -} - -pub fn tail_file(path: &Path, lines: usize) -> Result<()> { - let file = File::open(path)?; - let reader = BufReader::new(file); - let mut last_lines = Vec::with_capacity(lines); - - for line in reader.lines() { - let line = line?; - if last_lines.len() >= lines { - last_lines.remove(0); - } - last_lines.push(line); - } - - for line in last_lines { - println!("{}", line); - } - - Ok(()) -} - -pub fn get_tx_backup_dir() -> String { - let workspace_root = get_workspace_root(); - let mut path = workspace_root.to_path_buf(); - path.push("resources"); - path.push("bitcoin"); - path.push("inscription_txs"); - path.to_str().expect("Failed to convert path").to_string() -} diff --git a/crates/sequencer/src/lib.rs b/crates/sequencer/src/lib.rs index 8c8940595..e896f94d9 100644 --- a/crates/sequencer/src/lib.rs +++ b/crates/sequencer/src/lib.rs @@ -10,6 +10,7 @@ mod utils; use std::net::SocketAddr; pub use config::{SequencerConfig, SequencerMempoolConfig}; +pub use rpc::SequencerRpcClient; pub use sequencer::CitreaSequencer; use sov_db::ledger_db::LedgerDB; use sov_modules_rollup_blueprint::RollupBlueprint; diff --git a/crates/sovereign-sdk/module-system/sov-modules-macros/Cargo.toml b/crates/sovereign-sdk/module-system/sov-modules-macros/Cargo.toml index 96edb26ef..888062cd6 100644 --- a/crates/sovereign-sdk/module-system/sov-modules-macros/Cargo.toml +++ b/crates/sovereign-sdk/module-system/sov-modules-macros/Cargo.toml @@ -37,7 +37,7 @@ jsonrpsee = { workspace = true, features = ["http-client", "server"], optional = proc-macro2 = "1.0" quote = "1.0" serde_json = { workspace = true } -syn = { version = "1.0", features = ["full"] } +syn = { version = "1.0", features = ["full", "extra-traits"] } [features] default = ["native"] diff --git a/resources/scripts/cycle-diff.sh b/resources/scripts/cycle-diff.sh index cc509229f..3ea503417 100755 --- a/resources/scripts/cycle-diff.sh +++ b/resources/scripts/cycle-diff.sh @@ -16,7 +16,7 @@ # # Environment Variables: # BASE_BRANCH: The branch to compare against (default: nightly) -# TEST_NAME: The test file to run (default: bitcoin_e2e::tests::prover_test::basic_prover_test) +# TEST_NAME: The test file to run (default: bitcoin_e2e::prover_test::basic_prover_test) # TARGET_PCT: The threshold percentage for regression detection (default: 3) # NUM_RUNS: Number of times to run the test for averaging (default: 1) # @@ -38,7 +38,7 @@ set -euo pipefail BASE_BRANCH=${BASE_BRANCH:-"nightly"} -TEST_NAME=${TEST_NAME:-"bitcoin_e2e::tests::prover_test::basic_prover_test"} +TEST_NAME=${TEST_NAME:-"bitcoin_e2e::prover_test::basic_prover_test"} TARGET_PCT=${TARGET_PCT:-3} COMPARISON_FILE=${COMPARISON_FILE:-"comparison_results.log"} NUM_RUNS=${NUM_RUNS:-1} From 567548dd02cbe28ddca9eb70dd4183fab0044487 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erce=20Can=20Bekt=C3=BCre?= <47954181+ercecan@users.noreply.github.com> Date: Tue, 8 Oct 2024 20:27:41 +0300 Subject: [PATCH 18/28] Prover generate input rpc (#1280) * WIP Implement prover generate rpcg * It compiles but stf is modified * Context * Fix Context: Send * Merge fix * Move function from common to prover * Remove code duplicate and unnecessary log * Add optional parameter to break commitments into groups * Return input as string --------- Co-authored-by: Roman Proskuryakoff --- Cargo.lock | 3 + crates/common/Cargo.toml | 2 + crates/common/src/utils.rs | 41 ++- crates/fullnode/src/da_block_handler.rs | 2 +- crates/prover/Cargo.toml | 3 +- crates/prover/src/da_block_handler.rs | 330 ++++++++---------- crates/prover/src/lib.rs | 5 +- crates/prover/src/rpc.rs | 329 +++++++++++++++++ crates/prover/src/runner.rs | 46 ++- .../sov-modules-core/src/common/witness.rs | 4 +- .../rollup-interface/src/state_machine/stf.rs | 8 +- 11 files changed, 567 insertions(+), 206 deletions(-) create mode 100644 crates/prover/src/rpc.rs diff --git a/Cargo.lock b/Cargo.lock index dfa2c2c6b..d8e786bff 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1747,8 +1747,10 @@ version = "0.5.0-rc.1" dependencies = [ "anyhow", "backoff", + "borsh", "citrea-primitives", "futures", + "hex", "hyper 1.4.1", "jsonrpsee", "lru", @@ -1911,6 +1913,7 @@ dependencies = [ "sov-mock-da", "sov-mock-zkvm", "sov-modules-api", + "sov-modules-core", "sov-modules-rollup-blueprint", "sov-modules-stf-blueprint", "sov-rollup-interface 0.5.0-rc.1", diff --git a/crates/common/Cargo.toml b/crates/common/Cargo.toml index d90fd06fa..eb39d2ef5 100644 --- a/crates/common/Cargo.toml +++ b/crates/common/Cargo.toml @@ -14,7 +14,9 @@ resolver = "2" # 3rd-party deps anyhow = { workspace = true } backoff = { workspace = true } +borsh = { workspace = true } futures = { workspace = true } +hex = { workspace = true } hyper = { workspace = true } jsonrpsee = { workspace = true, features = ["http-client", "server"] } lru = { workspace = true } diff --git a/crates/common/src/utils.rs b/crates/common/src/utils.rs index 6da20c216..1ba578959 100644 --- a/crates/common/src/utils.rs +++ b/crates/common/src/utils.rs @@ -1,9 +1,11 @@ use std::collections::{HashMap, HashSet}; +use borsh::de::BorshDeserialize; use sov_db::ledger_db::SharedLedgerOps; use sov_db::schema::types::BatchNumber; -use sov_rollup_interface::da::SequencerCommitment; +use sov_rollup_interface::da::{BlobReaderTrait, DaDataBatchProof, DaSpec, SequencerCommitment}; use sov_rollup_interface::rpc::SoftConfirmationStatus; +use sov_rollup_interface::services::da::DaService; use sov_rollup_interface::stf::StateDiff; pub fn merge_state_diffs(old_diff: StateDiff, new_diff: StateDiff) -> StateDiff { @@ -13,18 +15,6 @@ pub fn merge_state_diffs(old_diff: StateDiff, new_diff: StateDiff) -> StateDiff new_diff_map.into_iter().collect() } -/// Remove finalized commitments using the end block number of the L2 range. -/// This is basically filtering out finalized soft confirmations. -pub fn filter_out_finalized_commitments( - ledger_db: &DB, - sequencer_commitments: &[SequencerCommitment], -) -> anyhow::Result<(Vec, Vec)> { - filter_out_commitments_by_status( - ledger_db, - sequencer_commitments, - SoftConfirmationStatus::Finalized, - ) -} /// Remove proven commitments using the end block number of the L2 range. /// This is basically filtering out proven soft confirmations. pub fn filter_out_proven_commitments( @@ -38,7 +28,7 @@ pub fn filter_out_proven_commitments( ) } -pub fn filter_out_commitments_by_status( +fn filter_out_commitments_by_status( ledger_db: &DB, sequencer_commitments: &[SequencerCommitment], exclude_status: SoftConfirmationStatus, @@ -76,7 +66,7 @@ pub fn filter_out_commitments_by_status( } pub fn check_l2_range_exists( - ledger_db: DB, + ledger_db: &DB, first_l2_height_of_l1: u64, last_l2_height_of_l1: u64, ) -> bool { @@ -89,3 +79,24 @@ pub fn check_l2_range_exists( } false } + +pub fn extract_sequencer_commitments( + sequencer_da_pub_key: &[u8], + da_data: &mut [<::Spec as DaSpec>::BlobTransaction], +) -> Vec { + let mut sequencer_commitments = vec![]; + // if we don't do this, the zk circuit can't read the sequencer commitments + da_data.iter_mut().for_each(|blob| { + blob.full_data(); + }); + da_data.iter_mut().for_each(|tx| { + let data = DaDataBatchProof::try_from_slice(tx.full_data()); + // Check for commitment + if tx.sender().as_ref() == sequencer_da_pub_key { + if let Ok(DaDataBatchProof::SequencerCommitment(seq_com)) = data { + sequencer_commitments.push(seq_com); + } + } + }); + sequencer_commitments +} diff --git a/crates/fullnode/src/da_block_handler.rs b/crates/fullnode/src/da_block_handler.rs index a1df6d9f6..ec4726dff 100644 --- a/crates/fullnode/src/da_block_handler.rs +++ b/crates/fullnode/src/da_block_handler.rs @@ -160,7 +160,7 @@ where // If the L2 range does not exist, we break off the current process call // We retry the L1 block at a later tick. if !check_l2_range_exists( - self.ledger_db.clone(), + &self.ledger_db, sequencer_commitments[0].l2_start_block_number, sequencer_commitments[sequencer_commitments.len() - 1].l2_end_block_number, ) { diff --git a/crates/prover/Cargo.toml b/crates/prover/Cargo.toml index 02bb3e34f..39bbe5b5b 100644 --- a/crates/prover/Cargo.toml +++ b/crates/prover/Cargo.toml @@ -19,6 +19,7 @@ sequencer-client = { path = "../sequencer-client" } # Sov SDK deps sov-db = { path = "../sovereign-sdk/full-node/db/sov-db" } sov-modules-api = { path = "../sovereign-sdk/module-system/sov-modules-api", default-features = false } +sov-modules-core = { path = "../sovereign-sdk/module-system/sov-modules-core" } sov-modules-rollup-blueprint = { path = "../sovereign-sdk/module-system/sov-modules-rollup-blueprint" } sov-modules-stf-blueprint = { path = "../sovereign-sdk/module-system/sov-modules-stf-blueprint", features = ["native"] } sov-rollup-interface = { path = "../sovereign-sdk/rollup-interface" } @@ -31,7 +32,7 @@ backoff = { workspace = true } borsh = { workspace = true } futures = { workspace = true } hex = { workspace = true } -jsonrpsee = { workspace = true } +jsonrpsee = { workspace = true, features = ["http-client", "server", "client"] } num_cpus = { workspace = true } parking_lot = { workspace = true } rand = { workspace = true } diff --git a/crates/prover/src/da_block_handler.rs b/crates/prover/src/da_block_handler.rs index 1bdaf7b4e..5f455626b 100644 --- a/crates/prover/src/da_block_handler.rs +++ b/crates/prover/src/da_block_handler.rs @@ -10,7 +10,8 @@ use borsh::{BorshDeserialize, BorshSerialize}; use citrea_common::cache::L1BlockCache; use citrea_common::da::get_da_block_at_height; use citrea_common::utils::{ - check_l2_range_exists, filter_out_proven_commitments, merge_state_diffs, + check_l2_range_exists, extract_sequencer_commitments, filter_out_proven_commitments, + merge_state_diffs, }; use citrea_primitives::MAX_TXBODY_SIZE; use rand::Rng; @@ -18,10 +19,11 @@ use serde::de::DeserializeOwned; use serde::Serialize; use sov_db::ledger_db::ProverLedgerOps; use sov_db::schema::types::{BatchNumber, SlotNumber, StoredProof, StoredStateTransition}; -use sov_modules_api::{BlobReaderTrait, DaSpec, SignedSoftConfirmation, StateDiff, Zkvm}; -use sov_rollup_interface::da::{BlockHeaderTrait, DaDataBatchProof, SequencerCommitment}; +use sov_modules_api::{BlobReaderTrait, DaSpec, StateDiff, Zkvm}; +use sov_rollup_interface::da::{BlockHeaderTrait, SequencerCommitment}; use sov_rollup_interface::rpc::SoftConfirmationStatus; use sov_rollup_interface::services::da::{DaService, SlotData}; +use sov_rollup_interface::soft_confirmation::SignedSoftConfirmation; use sov_rollup_interface::spec::SpecId; use sov_rollup_interface::zk::{Proof, StateTransitionData, ZkvmHost}; use sov_stf_runner::{ProverConfig, ProverService}; @@ -176,7 +178,10 @@ where blob.full_data(); }); let mut sequencer_commitments: Vec = - self.extract_sequencer_commitments(l1_block.header().hash().into(), &mut da_data); + extract_sequencer_commitments::( + self.sequencer_da_pub_key.as_slice(), + &mut da_data, + ); if sequencer_commitments.is_empty() { info!("No sequencer commitment found at height {}", l1_height,); @@ -207,7 +212,7 @@ where // the outer loop / select to make room for other tasks to run. // We retry the L1 block there as well. if !check_l2_range_exists( - self.ledger_db.clone(), + &self.ledger_db, sequencer_commitments[0].l2_start_block_number, sequencer_commitments[sequencer_commitments.len() - 1].l2_end_block_number, ) { @@ -246,8 +251,10 @@ where let hash = da_block_header_of_commitments.hash(); if should_prove { - let sequencer_commitments_groups = - self.break_sequencer_commitments_into_groups(&sequencer_commitments)?; + let sequencer_commitments_groups = break_sequencer_commitments_into_groups( + &self.ledger_db, + &sequencer_commitments, + )?; let submitted_proofs = self .ledger_db @@ -296,87 +303,6 @@ where Ok(()) } - fn extract_sequencer_commitments( - &self, - l1_block_hash: [u8; 32], - da_data: &mut [<::Spec as DaSpec>::BlobTransaction], - ) -> Vec { - let mut sequencer_commitments = vec![]; - // if we don't do this, the zk circuit can't read the sequencer commitments - da_data.iter_mut().for_each(|blob| { - blob.full_data(); - }); - da_data.iter_mut().for_each(|tx| { - let data = DaDataBatchProof::try_from_slice(tx.full_data()); - // Check for commitment - if tx.sender().as_ref() == self.sequencer_da_pub_key.as_slice() { - if let Ok(DaDataBatchProof::SequencerCommitment(seq_com)) = data { - sequencer_commitments.push(seq_com); - } else { - tracing::warn!( - "Found broken DA data in block 0x{}: {:?}", - hex::encode(l1_block_hash), - data - ); - } - } - }); - sequencer_commitments - } - - fn break_sequencer_commitments_into_groups( - &self, - sequencer_commitments: &[SequencerCommitment], - ) -> anyhow::Result>> { - let mut result_range = vec![]; - - let mut range = 0usize..=0usize; - let mut cumulative_state_diff = StateDiff::new(); - for (index, sequencer_commitment) in sequencer_commitments.iter().enumerate() { - let mut sequencer_commitment_state_diff = StateDiff::new(); - for l2_height in sequencer_commitment.l2_start_block_number - ..=sequencer_commitment.l2_end_block_number - { - let state_diff = self - .ledger_db - .get_l2_state_diff(BatchNumber(l2_height))? - .ok_or(anyhow!( - "Could not find state diff for L2 range {}-{}", - sequencer_commitment.l2_start_block_number, - sequencer_commitment.l2_end_block_number - ))?; - sequencer_commitment_state_diff = - merge_state_diffs(sequencer_commitment_state_diff, state_diff); - } - cumulative_state_diff = merge_state_diffs( - cumulative_state_diff, - sequencer_commitment_state_diff.clone(), - ); - - let compressed_state_diff = compress_blob(&borsh::to_vec(&cumulative_state_diff)?); - - // Threshold is checked by comparing compressed state diff size as the data will be compressed before it is written on DA - let state_diff_threshold_reached = compressed_state_diff.len() > MAX_TXBODY_SIZE; - - if state_diff_threshold_reached { - // We've exceeded the limit with the current commitments - // so we have to stop at the previous one. - result_range.push(range); - - // Reset the cumulative state diff to be equal to the current commitment state diff - cumulative_state_diff = sequencer_commitment_state_diff; - range = index..=index; - } else { - range = *range.start()..=index; - } - } - - // If the last group hasn't been reset because it has not reached the threshold, - // Add it anyway - result_range.push(range); - Ok(result_range) - } - async fn create_state_transition_data( &self, sequencer_commitments: &[SequencerCommitment], @@ -394,12 +320,13 @@ where state_transition_witnesses, soft_confirmations, da_block_headers_of_soft_confirmations, - ) = self - .get_state_transition_data_from_commitments( - &sequencer_commitments[sequencer_commitments_range.clone()], - &self.da_service, - ) - .await?; + ) = get_state_transition_data_from_commitments( + &sequencer_commitments[sequencer_commitments_range.clone()], + &self.da_service, + &self.ledger_db, + &self.l1_block_cache, + ) + .await?; let initial_state_root = self .ledger_db .get_l2_state_root::(first_l2_height_of_l1 - 1)? @@ -446,87 +373,6 @@ where Ok(transition_data) } - async fn get_state_transition_data_from_commitments( - &self, - sequencer_commitments: &[SequencerCommitment], - da_service: &Arc, - ) -> Result, anyhow::Error> { - let mut state_transition_witnesses: VecDeque> = VecDeque::new(); - let mut soft_confirmations: VecDeque> = VecDeque::new(); - let mut da_block_headers_of_soft_confirmations: VecDeque< - Vec<<::Spec as DaSpec>::BlockHeader>, - > = VecDeque::new(); - for sequencer_commitment in sequencer_commitments.to_owned().iter() { - // get the l2 height ranges of each seq_commitments - let mut witnesses = vec![]; - let start_l2 = sequencer_commitment.l2_start_block_number; - let end_l2 = sequencer_commitment.l2_end_block_number; - let soft_confirmations_in_commitment = match self - .ledger_db - .get_soft_confirmation_range(&(BatchNumber(start_l2)..=BatchNumber(end_l2))) - { - Ok(soft_confirmations) => soft_confirmations, - Err(e) => { - return Err(anyhow!( - "Failed to get soft confirmations from the ledger db: {}", - e - )); - } - }; - let mut commitment_soft_confirmations = vec![]; - let mut da_block_headers_to_push: Vec< - <::Spec as DaSpec>::BlockHeader, - > = vec![]; - for soft_confirmation in soft_confirmations_in_commitment { - if da_block_headers_to_push.is_empty() - || da_block_headers_to_push.last().unwrap().height() - != soft_confirmation.da_slot_height - { - let filtered_block = match get_da_block_at_height( - da_service, - soft_confirmation.da_slot_height, - self.l1_block_cache.clone(), - ) - .await - { - Ok(block) => block, - Err(_) => { - return Err(anyhow!( - "Error while fetching DA block at height: {}", - soft_confirmation.da_slot_height - )); - } - }; - da_block_headers_to_push.push(filtered_block.header().clone()); - } - let signed_soft_confirmation: SignedSoftConfirmation = - soft_confirmation.clone().into(); - commitment_soft_confirmations.push(signed_soft_confirmation.clone()); - } - soft_confirmations.push_back(commitment_soft_confirmations); - - da_block_headers_of_soft_confirmations.push_back(da_block_headers_to_push); - for l2_height in sequencer_commitment.l2_start_block_number - ..=sequencer_commitment.l2_end_block_number - { - let witness = match self.ledger_db.get_l2_witness::(l2_height) { - Ok(witness) => witness, - Err(e) => { - return Err(anyhow!("Failed to get witness from the ledger db: {}", e)) - } - }; - - witnesses.push(witness.expect("A witness must be present")); - } - state_transition_witnesses.push_back(witnesses); - } - Ok(( - state_transition_witnesses, - soft_confirmations, - da_block_headers_of_soft_confirmations, - )) - } - async fn prove_state_transition( &self, transition_data: StateTransitionData, @@ -730,3 +576,137 @@ async fn sync_l1( sleep(Duration::from_secs(2)).await; } } + +pub(crate) async fn get_state_transition_data_from_commitments< + Da: DaService, + DB: ProverLedgerOps, + Witness: DeserializeOwned, +>( + sequencer_commitments: &[SequencerCommitment], + da_service: &Arc, + ledger_db: &DB, + l1_block_cache: &Arc>>, +) -> Result, anyhow::Error> { + let mut state_transition_witnesses: VecDeque> = VecDeque::new(); + let mut soft_confirmations: VecDeque> = VecDeque::new(); + let mut da_block_headers_of_soft_confirmations: VecDeque< + Vec<<::Spec as DaSpec>::BlockHeader>, + > = VecDeque::new(); + for sequencer_commitment in sequencer_commitments.to_owned().iter() { + // get the l2 height ranges of each seq_commitments + let mut witnesses = vec![]; + let start_l2 = sequencer_commitment.l2_start_block_number; + let end_l2 = sequencer_commitment.l2_end_block_number; + let soft_confirmations_in_commitment = match ledger_db + .get_soft_confirmation_range(&(BatchNumber(start_l2)..=BatchNumber(end_l2))) + { + Ok(soft_confirmations) => soft_confirmations, + Err(e) => { + return Err(anyhow!( + "Failed to get soft confirmations from the ledger db: {}", + e + )); + } + }; + let mut commitment_soft_confirmations = vec![]; + let mut da_block_headers_to_push: Vec<<::Spec as DaSpec>::BlockHeader> = + vec![]; + for soft_confirmation in soft_confirmations_in_commitment { + if da_block_headers_to_push.is_empty() + || da_block_headers_to_push.last().unwrap().height() + != soft_confirmation.da_slot_height + { + let filtered_block = match get_da_block_at_height( + da_service, + soft_confirmation.da_slot_height, + l1_block_cache.clone(), + ) + .await + { + Ok(block) => block, + Err(_) => { + return Err(anyhow!( + "Error while fetching DA block at height: {}", + soft_confirmation.da_slot_height + )); + } + }; + da_block_headers_to_push.push(filtered_block.header().clone()); + } + let signed_soft_confirmation: SignedSoftConfirmation = soft_confirmation.clone().into(); + commitment_soft_confirmations.push(signed_soft_confirmation.clone()); + } + soft_confirmations.push_back(commitment_soft_confirmations); + + da_block_headers_of_soft_confirmations.push_back(da_block_headers_to_push); + for l2_height in + sequencer_commitment.l2_start_block_number..=sequencer_commitment.l2_end_block_number + { + let witness = match ledger_db.get_l2_witness::(l2_height) { + Ok(witness) => witness, + Err(e) => return Err(anyhow!("Failed to get witness from the ledger db: {}", e)), + }; + + witnesses.push(witness.expect("A witness must be present")); + } + state_transition_witnesses.push_back(witnesses); + } + Ok(( + state_transition_witnesses, + soft_confirmations, + da_block_headers_of_soft_confirmations, + )) +} + +pub(crate) fn break_sequencer_commitments_into_groups( + ledger_db: &DB, + sequencer_commitments: &[SequencerCommitment], +) -> anyhow::Result>> { + let mut result_range = vec![]; + + let mut range = 0usize..=0usize; + let mut cumulative_state_diff = StateDiff::new(); + for (index, sequencer_commitment) in sequencer_commitments.iter().enumerate() { + let mut sequencer_commitment_state_diff = StateDiff::new(); + for l2_height in + sequencer_commitment.l2_start_block_number..=sequencer_commitment.l2_end_block_number + { + let state_diff = + ledger_db + .get_l2_state_diff(BatchNumber(l2_height))? + .ok_or(anyhow!( + "Could not find state diff for L2 range {}-{}", + sequencer_commitment.l2_start_block_number, + sequencer_commitment.l2_end_block_number + ))?; + sequencer_commitment_state_diff = + merge_state_diffs(sequencer_commitment_state_diff, state_diff); + } + cumulative_state_diff = merge_state_diffs( + cumulative_state_diff, + sequencer_commitment_state_diff.clone(), + ); + + let compressed_state_diff = compress_blob(&borsh::to_vec(&cumulative_state_diff)?); + + // Threshold is checked by comparing compressed state diff size as the data will be compressed before it is written on DA + let state_diff_threshold_reached = compressed_state_diff.len() > MAX_TXBODY_SIZE; + + if state_diff_threshold_reached { + // We've exceeded the limit with the current commitments + // so we have to stop at the previous one. + result_range.push(range); + + // Reset the cumulative state diff to be equal to the current commitment state diff + cumulative_state_diff = sequencer_commitment_state_diff; + range = index..=index; + } else { + range = *range.start()..=index; + } + } + + // If the last group hasn't been reset because it has not reached the threshold, + // Add it anyway + result_range.push(range); + Ok(result_range) +} diff --git a/crates/prover/src/lib.rs b/crates/prover/src/lib.rs index aac2e9e5d..0bc2caca0 100644 --- a/crates/prover/src/lib.rs +++ b/crates/prover/src/lib.rs @@ -10,6 +10,7 @@ mod da_block_handler; pub mod prover_service; mod runner; pub use runner::*; +mod rpc; /// Dependencies needed to run the rollup. pub struct Prover { @@ -37,7 +38,7 @@ impl Prover { /// Only run the rpc. pub async fn run_rpc(mut self) -> Result<(), anyhow::Error> { - self.runner.start_rpc_server(self.rpc_methods, None).await; + self.runner.start_rpc_server(self.rpc_methods, None).await?; Ok(()) } @@ -47,7 +48,7 @@ impl Prover { channel: Option>, ) -> Result<(), anyhow::Error> { let mut runner = self.runner; - runner.start_rpc_server(self.rpc_methods, channel).await; + runner.start_rpc_server(self.rpc_methods, channel).await?; runner.run().await?; Ok(()) diff --git a/crates/prover/src/rpc.rs b/crates/prover/src/rpc.rs new file mode 100644 index 000000000..89f7662e3 --- /dev/null +++ b/crates/prover/src/rpc.rs @@ -0,0 +1,329 @@ +use std::marker::PhantomData; +use std::sync::Arc; + +use borsh::BorshSerialize; +use citrea_common::cache::L1BlockCache; +use citrea_common::utils::{ + check_l2_range_exists, extract_sequencer_commitments, filter_out_proven_commitments, +}; +use jsonrpsee::core::RpcResult; +use jsonrpsee::proc_macros::rpc; +use jsonrpsee::types::error::{INTERNAL_ERROR_CODE, INTERNAL_ERROR_MSG}; +use jsonrpsee::types::ErrorObjectOwned; +use serde::{Deserialize, Serialize}; +use sov_db::ledger_db::ProverLedgerOps; +use sov_db::schema::types::BatchNumber; +use sov_modules_core::{Spec, Storage}; +use sov_rollup_interface::da::{DaSpec, SequencerCommitment}; +use sov_rollup_interface::services::da::{DaService, SlotData}; +use sov_rollup_interface::zk::StateTransitionData; +use tokio::sync::Mutex; +use tracing::{debug, error}; + +use crate::da_block_handler::{ + break_sequencer_commitments_into_groups, get_state_transition_data_from_commitments, +}; + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ProverInputResponse { + pub commitment_range: (u32, u32), + pub l1_block_height: u64, + pub encoded_serialized_state_transition_data: String, +} + +pub(crate) struct RpcContext +where + C: sov_modules_api::Context, + Da: DaService, + DB: ProverLedgerOps, +{ + pub da_service: Arc, + pub ledger: DB, + pub sequencer_da_pub_key: Vec, + pub sequencer_pub_key: Vec, + pub l1_block_cache: Arc>>, + pub phantom: PhantomData C>, +} + +#[rpc(client, server)] +pub trait ProverRpc { + /// Generate state transition data for the given L1 block height, and return the data as a borsh serialized hex string. + #[method(name = "prover_generateInput")] + async fn generate_input( + &self, + l1_height: u64, + group_commitments: Option, + ) -> RpcResult>; +} + +pub struct ProverRpcServerImpl +where + C: sov_modules_api::Context, + Da: DaService, + DB: ProverLedgerOps + Send + Sync + 'static, +{ + context: Arc>, +} + +impl ProverRpcServerImpl +where + C: sov_modules_api::Context, + Da: DaService, + DB: ProverLedgerOps + Send + Sync + 'static, +{ + pub fn new(context: RpcContext) -> Self { + Self { + context: Arc::new(context), + } + } +} + +#[async_trait::async_trait] +impl + ProverRpcServer for ProverRpcServerImpl +{ + async fn generate_input( + &self, + l1_height: u64, + group_commitments: Option, + ) -> RpcResult> { + debug!("Prover: prover_generateInput"); + + let l1_block: ::FilteredBlock = self + .context + .da_service + .get_block_at(l1_height) + .await + .map_err(|e| { + ErrorObjectOwned::owned( + INTERNAL_ERROR_CODE, + INTERNAL_ERROR_MSG, + Some(format!("{e}",)), + ) + })?; + + let mut da_data: Vec<<::Spec as DaSpec>::BlobTransaction> = + self.context.da_service.extract_relevant_blobs(&l1_block); + + let mut sequencer_commitments: Vec = extract_sequencer_commitments::( + self.context.sequencer_da_pub_key.as_slice(), + &mut da_data, + ); + + if sequencer_commitments.is_empty() { + return Err(ErrorObjectOwned::owned( + INTERNAL_ERROR_CODE, + INTERNAL_ERROR_MSG, + Some(format!( + "No sequencer commitments found in block: {l1_height}", + )), + )); + } + + // Make sure all sequencer commitments are stored in ascending order. + // We sort before checking ranges to prevent substraction errors. + sequencer_commitments.sort(); + + // If the L2 range does not exist, we break off the local loop getting back to + // the outer loop / select to make room for other tasks to run. + // We retry the L1 block there as well. + let start_block_number = sequencer_commitments[0].l2_start_block_number; + let end_block_number = + sequencer_commitments[sequencer_commitments.len() - 1].l2_end_block_number; + + // If range is not synced yet return error + if !check_l2_range_exists(&self.context.ledger, start_block_number, end_block_number) { + return Err(ErrorObjectOwned::owned( + INTERNAL_ERROR_CODE, + INTERNAL_ERROR_MSG, + Some(format!( + "L2 Range of commitments is not synced yet: {start_block_number} - {end_block_number}" + )), + )); + } + + let (sequencer_commitments, preproven_commitments) = + filter_out_proven_commitments(&self.context.ledger, &sequencer_commitments).map_err( + |e| { + error!("Error filtering out proven commitments: {:?}", e); + ErrorObjectOwned::owned( + INTERNAL_ERROR_CODE, + INTERNAL_ERROR_MSG, + Some(format!("{e}",)), + ) + }, + )?; + + if sequencer_commitments.is_empty() { + return Err(ErrorObjectOwned::owned( + INTERNAL_ERROR_CODE, + INTERNAL_ERROR_MSG, + Some(format!( + "All sequencer commitments are duplicates from a former DA block {}", + l1_height + )), + )); + } + + let da_block_header_of_commitments: <::Spec as DaSpec>::BlockHeader = + l1_block.header().clone(); + + let mut state_transition_responses = Vec::new(); + + let ranges = match group_commitments { + Some(true) => break_sequencer_commitments_into_groups( + &self.context.ledger, + &sequencer_commitments, + ) + .map_err(|e| { + error!("Error breaking sequencer commitments into groups: {:?}", e); + ErrorObjectOwned::owned( + INTERNAL_ERROR_CODE, + INTERNAL_ERROR_MSG, + Some(format!("{e}",)), + ) + })?, + _ => vec![(0..=sequencer_commitments.len() - 1)], + }; + + for sequencer_commitments_range in ranges { + let first_l2_height_of_l1 = + sequencer_commitments[*sequencer_commitments_range.start()].l2_start_block_number; + let last_l2_height_of_l1 = + sequencer_commitments[*sequencer_commitments_range.end()].l2_end_block_number; + let ( + state_transition_witnesses, + soft_confirmations, + da_block_headers_of_soft_confirmations, + ) = get_state_transition_data_from_commitments( + &sequencer_commitments[sequencer_commitments_range.clone()], + &self.context.da_service, + &self.context.ledger, + &self.context.l1_block_cache, + ) + .await + .map_err(|e| { + error!( + "Error getting state transition data from commitments: {:?}", + e + ); + ErrorObjectOwned::owned( + INTERNAL_ERROR_CODE, + INTERNAL_ERROR_MSG, + Some(format!("{e}",)), + ) + })?; + let initial_state_root = self + .context + .ledger + .get_l2_state_root::<<::Storage as Storage>::Root>( + first_l2_height_of_l1 - 1, + ) + .map_err(|e| { + error!("Error getting initial state root: {:?}", e); + ErrorObjectOwned::owned( + INTERNAL_ERROR_CODE, + INTERNAL_ERROR_MSG, + Some(format!("{e}",)), + ) + })? + .expect("There should be a state root"); + let initial_batch_hash = self + .context + .ledger + .get_soft_confirmation_by_number(&BatchNumber(first_l2_height_of_l1)) + .map_err(|e| { + error!("Error getting initial batch hash: {:?}", e); + ErrorObjectOwned::owned( + INTERNAL_ERROR_CODE, + INTERNAL_ERROR_MSG, + Some(format!("{e}",)), + ) + })? + .ok_or(ErrorObjectOwned::owned( + INTERNAL_ERROR_CODE, + INTERNAL_ERROR_MSG, + Some(format!( + "Could not find soft batch at height {}", + first_l2_height_of_l1 + )), + ))? + .prev_hash; + + let final_state_root = self + .context + .ledger + .get_l2_state_root::<<::Storage as Storage>::Root>(last_l2_height_of_l1) + .map_err(|e| { + error!("Error getting final state root: {:?}", e); + ErrorObjectOwned::owned( + INTERNAL_ERROR_CODE, + INTERNAL_ERROR_MSG, + Some(format!("{e}",)), + ) + })? + .expect("There should be a state root"); + + let (inclusion_proof, completeness_proof) = self + .context + .da_service + .get_extraction_proof(&l1_block, &da_data) + .await; + + let state_transition_data: StateTransitionData< + <::Storage as Storage>::Root, + <::Storage as Storage>::Witness, + Da::Spec, + > = StateTransitionData { + initial_state_root, + final_state_root, + initial_batch_hash, + da_data: da_data.clone(), + da_block_header_of_commitments: da_block_header_of_commitments.clone(), + inclusion_proof, + completeness_proof, + soft_confirmations, + state_transition_witnesses, + da_block_headers_of_soft_confirmations, + preproven_commitments: preproven_commitments.to_vec(), + sequencer_commitments_range: ( + *sequencer_commitments_range.start() as u32, + *sequencer_commitments_range.end() as u32, + ), + sequencer_public_key: self.context.sequencer_pub_key.clone(), + sequencer_da_public_key: self.context.sequencer_da_pub_key.clone(), + }; + let serialized_state_transition = serialize_state_transition(state_transition_data); + + let response = ProverInputResponse { + commitment_range: ( + *sequencer_commitments_range.start() as u32, + *sequencer_commitments_range.end() as u32, + ), + l1_block_height: l1_height, + encoded_serialized_state_transition_data: hex::encode(serialized_state_transition), + }; + + state_transition_responses.push(response); + } + + Ok(state_transition_responses) + } +} + +fn serialize_state_transition(item: T) -> Vec { + borsh::to_vec(&item).expect("Risc0 hint serialization is infallible") +} + +pub fn create_rpc_module( + rpc_context: RpcContext, +) -> jsonrpsee::RpcModule> +where + C: sov_modules_api::Context, + Da: DaService, + DB: ProverLedgerOps + Send + Sync + 'static, +{ + let server = ProverRpcServerImpl::new(rpc_context); + + ProverRpcServer::into_rpc(server) +} diff --git a/crates/prover/src/runner.rs b/crates/prover/src/runner.rs index 59fdd973e..e0b7b304e 100644 --- a/crates/prover/src/runner.rs +++ b/crates/prover/src/runner.rs @@ -4,7 +4,7 @@ use std::net::SocketAddr; use std::sync::Arc; use std::time::Duration; -use anyhow::bail; +use anyhow::{anyhow, bail}; use backoff::exponential::ExponentialBackoffBuilder; use backoff::future::retry as retry_backoff; use citrea_common::cache::L1BlockCache; @@ -35,6 +35,7 @@ use tokio::{select, signal}; use tracing::{debug, error, info, instrument}; use crate::da_block_handler::L1BlockHandler; +use crate::rpc::{create_rpc_module, RpcContext}; type StateRoot = >::StateRoot; @@ -163,20 +164,44 @@ where }) } + /// Creates a shared RpcContext with all required data. + fn create_rpc_context(&self) -> RpcContext { + RpcContext { + ledger: self.ledger_db.clone(), + da_service: self.da_service.clone(), + sequencer_da_pub_key: self.sequencer_da_pub_key.clone(), + sequencer_pub_key: self.sequencer_pub_key.clone(), + l1_block_cache: self.l1_block_cache.clone(), + phantom: std::marker::PhantomData, + } + } + + /// Updates the given RpcModule with Prover methods. + pub fn register_rpc_methods( + &self, + mut rpc_methods: jsonrpsee::RpcModule<()>, + ) -> Result, jsonrpsee::core::RegisterMethodError> { + let rpc_context = self.create_rpc_context(); + let rpc = create_rpc_module(rpc_context); + rpc_methods.merge(rpc)?; + Ok(rpc_methods) + } + /// Starts a RPC server with provided rpc methods. pub async fn start_rpc_server( &mut self, methods: RpcModule<()>, channel: Option>, - ) { - let bind_host = match self.rpc_config.bind_host.parse() { - Ok(bind_host) => bind_host, - Err(e) => { - error!("Failed to parse bind host: {}", e); - return; - } - }; - let listen_address = SocketAddr::new(bind_host, self.rpc_config.bind_port); + ) -> anyhow::Result<()> { + let methods = self.register_rpc_methods(methods)?; + + let listen_address = SocketAddr::new( + self.rpc_config + .bind_host + .parse() + .map_err(|e| anyhow!("Failed to parse bind host: {}", e))?, + self.rpc_config.bind_port, + ); let max_connections = self.rpc_config.max_connections; let max_subscriptions_per_connection = self.rpc_config.max_subscriptions_per_connection; @@ -223,6 +248,7 @@ where } } }); + Ok(()) } /// Runs the rollup. diff --git a/crates/sovereign-sdk/module-system/sov-modules-core/src/common/witness.rs b/crates/sovereign-sdk/module-system/sov-modules-core/src/common/witness.rs index 71693fcae..db4212fbd 100644 --- a/crates/sovereign-sdk/module-system/sov-modules-core/src/common/witness.rs +++ b/crates/sovereign-sdk/module-system/sov-modules-core/src/common/witness.rs @@ -14,7 +14,9 @@ use serde::Serialize; /// they were added via [`Witness::add_hint`]. // TODO: Refactor witness trait so it only require Serialize / Deserialize // https://github.com/Sovereign-Labs/sovereign-sdk/issues/263 -pub trait Witness: Default + BorshDeserialize + Serialize + DeserializeOwned { +pub trait Witness: + Default + BorshSerialize + BorshDeserialize + Serialize + DeserializeOwned +{ /// Adds a serializable "hint" to the witness value, which can be later /// read by the zkVM circuit. /// diff --git a/crates/sovereign-sdk/rollup-interface/src/state_machine/stf.rs b/crates/sovereign-sdk/rollup-interface/src/state_machine/stf.rs index e1e7e8a57..1e7b000f4 100644 --- a/crates/sovereign-sdk/rollup-interface/src/state_machine/stf.rs +++ b/crates/sovereign-sdk/rollup-interface/src/state_machine/stf.rs @@ -193,7 +193,13 @@ pub trait StateTransitionFunction { /// Witness is a data that is produced during actual batch execution /// or validated together with proof during verification - type Witness: Default + BorshDeserialize + Serialize + DeserializeOwned + Send + Sync; + type Witness: Default + + BorshSerialize + + BorshDeserialize + + Serialize + + DeserializeOwned + + Send + + Sync; /// The validity condition that must be verified outside of the Vm type Condition: ValidityCondition; From 970b14773e454dcb3cbfe168ab5fbe85cae91ce4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ege=20Okan=20=C3=9Cnald=C4=B1?= <35339130+exeokan@users.noreply.github.com> Date: Wed, 9 Oct 2024 10:33:58 +0300 Subject: [PATCH 19/28] Move node configs to citrea-common (#1286) * move node configs to a seperate crate * fix udeps * remove native feature * move SequencerConfig * move config to citrea-common * fix use statements --- Cargo.lock | 11 +- bin/citrea/src/main.rs | 3 +- bin/citrea/src/rollup/bitcoin.rs | 4 +- bin/citrea/src/rollup/mock.rs | 2 +- bin/citrea/src/rollup/mod.rs | 5 +- bin/citrea/src/test_rpc.rs | 4 +- bin/citrea/tests/e2e/mod.rs | 3 +- bin/citrea/tests/e2e/proving.rs | 3 +- bin/citrea/tests/e2e/reopen.rs | 3 +- bin/citrea/tests/e2e/sequencer_behaviour.rs | 2 +- bin/citrea/tests/e2e/sequencer_replacement.rs | 2 +- bin/citrea/tests/e2e/syncing.rs | 3 +- bin/citrea/tests/e2e/tx_propagation.rs | 2 +- bin/citrea/tests/evm/archival_state.rs | 2 +- bin/citrea/tests/evm/fee.rs | 2 +- bin/citrea/tests/evm/gas_price.rs | 2 +- bin/citrea/tests/evm/mod.rs | 2 +- bin/citrea/tests/evm/subscription.rs | 2 +- bin/citrea/tests/evm/tracing.rs | 2 +- bin/citrea/tests/mempool/mod.rs | 2 +- .../soft_confirmation_rule_enforcer/mod.rs | 2 +- bin/citrea/tests/test_helpers/mod.rs | 8 +- crates/common/Cargo.toml | 8 ++ .../sov-stf-runner => common}/src/config.rs | 115 +++++++++++++++- crates/common/src/lib.rs | 2 + crates/fullnode/src/runner.rs | 3 +- .../tests/runner_initialization_tests.rs | 6 +- crates/prover/src/da_block_handler.rs | 3 +- .../prover/src/prover_service/parallel/mod.rs | 2 +- crates/prover/src/runner.rs | 5 +- crates/sequencer/src/config.rs | 129 ------------------ crates/sequencer/src/lib.rs | 3 +- crates/sequencer/src/mempool.rs | 2 +- crates/sequencer/src/sequencer.rs | 4 +- .../examples/demo-stf/Cargo.toml | 2 - .../full-node/sov-stf-runner/Cargo.toml | 7 - .../full-node/sov-stf-runner/src/lib.rs | 5 - .../sov-modules-rollup-blueprint/Cargo.toml | 1 + .../sov-modules-rollup-blueprint/src/lib.rs | 3 +- 39 files changed, 173 insertions(+), 198 deletions(-) rename crates/{sovereign-sdk/full-node/sov-stf-runner => common}/src/config.rs (66%) delete mode 100644 crates/sequencer/src/config.rs diff --git a/Cargo.lock b/Cargo.lock index d8e786bff..d180677df 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1749,15 +1749,21 @@ dependencies = [ "backoff", "borsh", "citrea-primitives", + "citrea-pruning", "futures", "hex", "hyper 1.4.1", "jsonrpsee", "lru", + "serde", "sov-db", + "sov-mock-da", "sov-rollup-interface 0.5.0-rc.1", + "sov-stf-runner", + "tempfile", "tokio", "tokio-util", + "toml", "tower-http", "tracing", ] @@ -2477,7 +2483,6 @@ dependencies = [ "sov-value-setter", "tempfile", "tokio", - "toml", "tracing", ] @@ -7641,6 +7646,7 @@ dependencies = [ "anyhow", "async-trait", "borsh", + "citrea-common", "hex", "jsonrpsee", "serde", @@ -7872,7 +7878,6 @@ dependencies = [ "anyhow", "async-trait", "borsh", - "citrea-pruning", "futures", "hex", "hyper 1.4.1", @@ -7882,7 +7887,6 @@ dependencies = [ "serde_json", "sha2", "sov-db", - "sov-mock-da", "sov-modules-api", "sov-prover-storage-manager", "sov-rollup-interface 0.5.0-rc.1", @@ -7890,7 +7894,6 @@ dependencies = [ "tempfile", "thiserror", "tokio", - "toml", "tower", "tracing", ] diff --git a/bin/citrea/src/main.rs b/bin/citrea/src/main.rs index 886b239d6..3e6d20a09 100644 --- a/bin/citrea/src/main.rs +++ b/bin/citrea/src/main.rs @@ -3,14 +3,13 @@ use core::fmt::Debug as DebugTrait; use anyhow::Context as _; use bitcoin_da::service::BitcoinServiceConfig; use citrea::{initialize_logging, BitcoinRollup, CitreaRollupBlueprint, MockDemoRollup}; -use citrea_sequencer::SequencerConfig; +use citrea_common::{from_toml_path, FullNodeConfig, ProverConfig, SequencerConfig}; use citrea_stf::genesis_config::GenesisPaths; use clap::Parser; use sov_mock_da::MockDaConfig; use sov_modules_api::Spec; use sov_modules_rollup_blueprint::RollupBlueprint; use sov_state::storage::NativeStorage; -use sov_stf_runner::{from_toml_path, FullNodeConfig, ProverConfig}; use tracing::{error, instrument}; #[cfg(test)] diff --git a/bin/citrea/src/rollup/bitcoin.rs b/bin/citrea/src/rollup/bitcoin.rs index 6bc4fb380..63555ba81 100644 --- a/bin/citrea/src/rollup/bitcoin.rs +++ b/bin/citrea/src/rollup/bitcoin.rs @@ -6,6 +6,7 @@ use bitcoin_da::service::{BitcoinService, BitcoinServiceConfig, TxidWrapper}; use bitcoin_da::spec::{BitcoinSpec, RollupParams}; use bitcoin_da::verifier::BitcoinVerifier; use citrea_common::rpc::register_healthcheck_rpc; +use citrea_common::{FullNodeConfig, ProverConfig}; use citrea_primitives::{REVEAL_BATCH_PROOF_PREFIX, REVEAL_LIGHT_CLIENT_PREFIX}; use citrea_prover::prover_service::ParallelProverService; use citrea_risc0_bonsai_adapter::host::Risc0BonsaiHost; @@ -23,7 +24,6 @@ use sov_rollup_interface::services::da::SenderWithNotifier; use sov_rollup_interface::spec::SpecId; use sov_rollup_interface::zk::{Zkvm, ZkvmHost}; use sov_state::{DefaultStorageSpec, Storage, ZkStorage}; -use sov_stf_runner::{FullNodeConfig, ProverConfig}; use tokio::sync::broadcast; use tokio::sync::mpsc::unbounded_channel; use tracing::instrument; @@ -105,7 +105,7 @@ impl RollupBlueprint for BitcoinRollup { #[instrument(level = "trace", skip_all, err)] fn create_storage_manager( &self, - rollup_config: &sov_stf_runner::FullNodeConfig, + rollup_config: &citrea_common::FullNodeConfig, ) -> Result { let storage_config = StorageConfig { path: rollup_config.storage.path.clone(), diff --git a/bin/citrea/src/rollup/mock.rs b/bin/citrea/src/rollup/mock.rs index 2ba57f2bb..756ed35de 100644 --- a/bin/citrea/src/rollup/mock.rs +++ b/bin/citrea/src/rollup/mock.rs @@ -3,6 +3,7 @@ use std::sync::Arc; use async_trait::async_trait; use citrea_common::rpc::register_healthcheck_rpc; +use citrea_common::{FullNodeConfig, ProverConfig}; use citrea_prover::prover_service::ParallelProverService; use citrea_risc0_bonsai_adapter::host::Risc0BonsaiHost; use citrea_risc0_bonsai_adapter::Digest; @@ -18,7 +19,6 @@ use sov_prover_storage_manager::ProverStorageManager; use sov_rollup_interface::spec::SpecId; use sov_rollup_interface::zk::{Zkvm, ZkvmHost}; use sov_state::{DefaultStorageSpec, Storage, ZkStorage}; -use sov_stf_runner::{FullNodeConfig, ProverConfig}; use tokio::sync::broadcast; use crate::CitreaRollupBlueprint; diff --git a/bin/citrea/src/rollup/mod.rs b/bin/citrea/src/rollup/mod.rs index 6bede76d2..79c842bf8 100644 --- a/bin/citrea/src/rollup/mod.rs +++ b/bin/citrea/src/rollup/mod.rs @@ -2,10 +2,11 @@ use std::sync::Arc; use anyhow::anyhow; use async_trait::async_trait; +use citrea_common::{FullNodeConfig, ProverConfig, SequencerConfig}; use citrea_fullnode::{CitreaFullnode, FullNode}; use citrea_primitives::forks::FORKS; use citrea_prover::{CitreaProver, Prover}; -use citrea_sequencer::{CitreaSequencer, Sequencer, SequencerConfig}; +use citrea_sequencer::{CitreaSequencer, Sequencer}; use sov_db::ledger_db::SharedLedgerOps; use sov_db::rocks_db_config::RocksdbConfig; use sov_db::schema::types::BatchNumber; @@ -15,7 +16,7 @@ use sov_modules_rollup_blueprint::RollupBlueprint; use sov_modules_stf_blueprint::{Runtime as RuntimeTrait, StfBlueprint}; use sov_rollup_interface::fork::ForkManager; use sov_state::storage::NativeStorage; -use sov_stf_runner::{FullNodeConfig, InitVariant, ProverConfig}; +use sov_stf_runner::InitVariant; use tokio::sync::broadcast; use tracing::{info, instrument}; diff --git a/bin/citrea/src/test_rpc.rs b/bin/citrea/src/test_rpc.rs index 1dcd1f517..8e274001a 100644 --- a/bin/citrea/src/test_rpc.rs +++ b/bin/citrea/src/test_rpc.rs @@ -1,3 +1,5 @@ +#[cfg(test)] +use citrea_common::RpcConfig; use hex::ToHex; use reqwest::header::CONTENT_TYPE; use sha2::Digest; @@ -7,8 +9,6 @@ use sov_mock_da::MockDaSpec; #[cfg(test)] use sov_modules_api::DaSpec; use sov_rollup_interface::stf::{Event, SoftConfirmationReceipt, TransactionReceipt}; -#[cfg(test)] -use sov_stf_runner::RpcConfig; struct TestExpect { payload: serde_json::Value, diff --git a/bin/citrea/tests/e2e/mod.rs b/bin/citrea/tests/e2e/mod.rs index fab958702..82a971a87 100644 --- a/bin/citrea/tests/e2e/mod.rs +++ b/bin/citrea/tests/e2e/mod.rs @@ -12,14 +12,13 @@ use std::path::{Path, PathBuf}; use std::str::FromStr; use std::time::Duration; +use citrea_common::{ProverConfig, SequencerConfig}; use citrea_evm::smart_contracts::SimpleStorageContract; -use citrea_sequencer::SequencerConfig; use citrea_stf::genesis_config::GenesisPaths; use reth_primitives::{Address, BlockNumberOrTag, U256}; use sov_mock_da::{MockAddress, MockDaService}; use sov_rollup_interface::rpc::{LastVerifiedProofResponse, SoftConfirmationStatus}; use sov_rollup_interface::services::da::DaService; -use sov_stf_runner::ProverConfig; use tokio::task::JoinHandle; use crate::evm::{init_test_rollup, make_test_client}; diff --git a/bin/citrea/tests/e2e/proving.rs b/bin/citrea/tests/e2e/proving.rs index d9440ba88..8d743c449 100644 --- a/bin/citrea/tests/e2e/proving.rs +++ b/bin/citrea/tests/e2e/proving.rs @@ -1,12 +1,11 @@ /// Prover node, proving and full node proof verification related tests use std::time::Duration; -use citrea_sequencer::SequencerConfig; +use citrea_common::{ProverConfig, SequencerConfig}; use citrea_stf::genesis_config::GenesisPaths; use sov_mock_da::{MockAddress, MockDaService}; use sov_rollup_interface::rpc::SoftConfirmationStatus; use sov_rollup_interface::services::da::DaService; -use sov_stf_runner::ProverConfig; use crate::evm::make_test_client; use crate::test_helpers::{ diff --git a/bin/citrea/tests/e2e/reopen.rs b/bin/citrea/tests/e2e/reopen.rs index a6c749761..8dfcc1e1b 100644 --- a/bin/citrea/tests/e2e/reopen.rs +++ b/bin/citrea/tests/e2e/reopen.rs @@ -4,11 +4,10 @@ use std::str::FromStr; use std::time::Duration; -use citrea_sequencer::SequencerConfig; +use citrea_common::{ProverConfig, SequencerConfig}; use citrea_stf::genesis_config::GenesisPaths; use reth_primitives::{Address, BlockNumberOrTag}; use sov_mock_da::{MockAddress, MockDaService}; -use sov_stf_runner::ProverConfig; use tokio::runtime::Runtime; use tokio::time::sleep; diff --git a/bin/citrea/tests/e2e/sequencer_behaviour.rs b/bin/citrea/tests/e2e/sequencer_behaviour.rs index aa511a3ff..9ca63939c 100644 --- a/bin/citrea/tests/e2e/sequencer_behaviour.rs +++ b/bin/citrea/tests/e2e/sequencer_behaviour.rs @@ -6,7 +6,7 @@ use alloy::consensus::{Signed, TxEip1559, TxEnvelope}; use alloy::signers::local::PrivateKeySigner; use alloy::signers::Signer; use alloy_rlp::{BytesMut, Encodable}; -use citrea_sequencer::{SequencerConfig, SequencerMempoolConfig}; +use citrea_common::{SequencerConfig, SequencerMempoolConfig}; use citrea_stf::genesis_config::GenesisPaths; use reth_primitives::{Address, BlockNumberOrTag}; use sov_mock_da::{MockAddress, MockDaService, MockDaSpec}; diff --git a/bin/citrea/tests/e2e/sequencer_replacement.rs b/bin/citrea/tests/e2e/sequencer_replacement.rs index c12410cca..48fc4587d 100644 --- a/bin/citrea/tests/e2e/sequencer_replacement.rs +++ b/bin/citrea/tests/e2e/sequencer_replacement.rs @@ -6,7 +6,7 @@ use std::time::Duration; use alloy::consensus::{Signed, TxEip1559, TxEnvelope}; use alloy_rlp::Decodable; -use citrea_sequencer::{SequencerConfig, SequencerMempoolConfig}; +use citrea_common::{SequencerConfig, SequencerMempoolConfig}; use citrea_stf::genesis_config::GenesisPaths; use reth_primitives::{Address, BlockNumberOrTag}; use sov_db::ledger_db::{LedgerDB, SequencerLedgerOps}; diff --git a/bin/citrea/tests/e2e/syncing.rs b/bin/citrea/tests/e2e/syncing.rs index ab9c084db..0a36535d7 100644 --- a/bin/citrea/tests/e2e/syncing.rs +++ b/bin/citrea/tests/e2e/syncing.rs @@ -2,14 +2,13 @@ use std::str::FromStr; use std::time::Duration; -use citrea_sequencer::SequencerConfig; +use citrea_common::{ProverConfig, SequencerConfig}; use citrea_stf::genesis_config::GenesisPaths; use ethereum_rpc::LayerStatus; use reth_primitives::{Address, BlockNumberOrTag}; use sov_mock_da::{MockAddress, MockDaService, MockDaSpec, MockHash}; use sov_rollup_interface::da::{DaDataLightClient, DaSpec}; use sov_rollup_interface::services::da::DaService; -use sov_stf_runner::ProverConfig; use tokio::time::sleep; use crate::e2e::{execute_blocks, initialize_test, TestConfig}; diff --git a/bin/citrea/tests/e2e/tx_propagation.rs b/bin/citrea/tests/e2e/tx_propagation.rs index 21b22e7ab..f6557e9d0 100644 --- a/bin/citrea/tests/e2e/tx_propagation.rs +++ b/bin/citrea/tests/e2e/tx_propagation.rs @@ -1,7 +1,7 @@ /// Tests that check the full node's ability to send a transaction to the sequencer. use std::str::FromStr; -use citrea_sequencer::SequencerConfig; +use citrea_common::SequencerConfig; use citrea_stf::genesis_config::GenesisPaths; use reth_primitives::{Address, BlockNumberOrTag, TxHash}; diff --git a/bin/citrea/tests/evm/archival_state.rs b/bin/citrea/tests/evm/archival_state.rs index fbc8e07ea..be76071e6 100644 --- a/bin/citrea/tests/evm/archival_state.rs +++ b/bin/citrea/tests/evm/archival_state.rs @@ -1,8 +1,8 @@ use std::str::FromStr; use std::time::Duration; +use citrea_common::SequencerConfig; use citrea_evm::smart_contracts::SimpleStorageContract; -use citrea_sequencer::SequencerConfig; use citrea_stf::genesis_config::GenesisPaths; use reth_primitives::{Address, BlockId, BlockNumberOrTag, Bytes, B256, U256}; use tokio::time::sleep; diff --git a/bin/citrea/tests/evm/fee.rs b/bin/citrea/tests/evm/fee.rs index 5ffbec8d4..94e7fdcba 100644 --- a/bin/citrea/tests/evm/fee.rs +++ b/bin/citrea/tests/evm/fee.rs @@ -1,6 +1,6 @@ use std::time::Duration; -use citrea_sequencer::SequencerConfig; +use citrea_common::SequencerConfig; use citrea_stf::genesis_config::GenesisPaths; use reth_primitives::BlockNumberOrTag; diff --git a/bin/citrea/tests/evm/gas_price.rs b/bin/citrea/tests/evm/gas_price.rs index d3b227dd9..83ecffb5b 100644 --- a/bin/citrea/tests/evm/gas_price.rs +++ b/bin/citrea/tests/evm/gas_price.rs @@ -3,8 +3,8 @@ use std::time::Duration; use alloy::signers::local::PrivateKeySigner; use alloy::signers::Signer; +use citrea_common::SequencerConfig; use citrea_evm::smart_contracts::SimpleStorageContract; -use citrea_sequencer::SequencerConfig; use citrea_stf::genesis_config::GenesisPaths; use reth_primitives::{BlockNumberOrTag, U256}; diff --git a/bin/citrea/tests/evm/mod.rs b/bin/citrea/tests/evm/mod.rs index bad5b251c..309ee4905 100644 --- a/bin/citrea/tests/evm/mod.rs +++ b/bin/citrea/tests/evm/mod.rs @@ -3,10 +3,10 @@ use std::str::FromStr; use alloy::signers::local::PrivateKeySigner; use alloy::signers::Signer; +use citrea_common::SequencerConfig; // use citrea::initialize_logging; use citrea_evm::smart_contracts::{LogsContract, SimpleStorageContract, TestContract}; use citrea_evm::system_contracts::BitcoinLightClient; -use citrea_sequencer::SequencerConfig; use citrea_stf::genesis_config::GenesisPaths; use reth_primitives::{Address, BlockId, BlockNumberOrTag, Bytes, U256}; use sov_rollup_interface::CITREA_VERSION; diff --git a/bin/citrea/tests/evm/subscription.rs b/bin/citrea/tests/evm/subscription.rs index 61724285e..2a0b48e2b 100644 --- a/bin/citrea/tests/evm/subscription.rs +++ b/bin/citrea/tests/evm/subscription.rs @@ -4,9 +4,9 @@ use std::time::Duration; use alloy_primitives::FixedBytes; use alloy_sol_types::SolEvent; +use citrea_common::SequencerConfig; use citrea_evm::smart_contracts::{AnotherLogEvent, LogEvent, LogsContract, TestContract}; use citrea_evm::{Filter, LogResponse}; -use citrea_sequencer::SequencerConfig; // use citrea::initialize_logging; use citrea_stf::genesis_config::GenesisPaths; use reth_primitives::{keccak256, Address}; diff --git a/bin/citrea/tests/evm/tracing.rs b/bin/citrea/tests/evm/tracing.rs index dd09fb458..de08e1411 100644 --- a/bin/citrea/tests/evm/tracing.rs +++ b/bin/citrea/tests/evm/tracing.rs @@ -1,8 +1,8 @@ use std::str::FromStr; +use citrea_common::SequencerConfig; // use citrea::initialize_logging; use citrea_evm::smart_contracts::{CallerContract, SimpleStorageContract}; -use citrea_sequencer::SequencerConfig; use citrea_stf::genesis_config::GenesisPaths; use reth_primitives::{Address, BlockNumberOrTag}; use reth_rpc_types::trace::geth::GethTrace::{self, CallTracer, FourByteTracer}; diff --git a/bin/citrea/tests/mempool/mod.rs b/bin/citrea/tests/mempool/mod.rs index 9e046b190..0fe91211b 100644 --- a/bin/citrea/tests/mempool/mod.rs +++ b/bin/citrea/tests/mempool/mod.rs @@ -3,7 +3,7 @@ use std::str::FromStr; use alloy::signers::local::PrivateKeySigner; use alloy::signers::Signer; -use citrea_sequencer::SequencerConfig; +use citrea_common::SequencerConfig; use citrea_stf::genesis_config::GenesisPaths; use reth_primitives::{Address, BlockNumberOrTag}; use tokio::task::JoinHandle; diff --git a/bin/citrea/tests/soft_confirmation_rule_enforcer/mod.rs b/bin/citrea/tests/soft_confirmation_rule_enforcer/mod.rs index dc13c71c0..99827bf2c 100644 --- a/bin/citrea/tests/soft_confirmation_rule_enforcer/mod.rs +++ b/bin/citrea/tests/soft_confirmation_rule_enforcer/mod.rs @@ -1,6 +1,6 @@ use std::time::Duration; -use citrea_sequencer::SequencerConfig; +use citrea_common::SequencerConfig; use citrea_stf::genesis_config::GenesisPaths; use sov_mock_da::{MockAddress, MockDaService}; use tokio::time::sleep; diff --git a/bin/citrea/tests/test_helpers/mod.rs b/bin/citrea/tests/test_helpers/mod.rs index 232f8f163..ee2fa729c 100644 --- a/bin/citrea/tests/test_helpers/mod.rs +++ b/bin/citrea/tests/test_helpers/mod.rs @@ -5,8 +5,11 @@ use std::time::{Duration, SystemTime}; use anyhow::bail; use borsh::BorshDeserialize; use citrea::{CitreaRollupBlueprint, MockDemoRollup}; +use citrea_common::{ + FullNodeConfig, ProverConfig, RollupPublicKeys, RpcConfig, RunnerConfig, SequencerConfig, + StorageConfig, +}; use citrea_primitives::TEST_PRIVATE_KEY; -use citrea_sequencer::SequencerConfig; use citrea_stf::genesis_config::GenesisPaths; use sov_mock_da::{MockAddress, MockBlock, MockDaConfig, MockDaService}; use sov_modules_api::default_signature::private_key::DefaultPrivateKey; @@ -14,9 +17,6 @@ use sov_modules_api::PrivateKey; use sov_rollup_interface::da::{BlobReaderTrait, DaData, SequencerCommitment}; use sov_rollup_interface::services::da::{DaService, SlotData}; use sov_rollup_interface::zk::Proof; -use sov_stf_runner::{ - FullNodeConfig, ProverConfig, RollupPublicKeys, RpcConfig, RunnerConfig, StorageConfig, -}; use tempfile::TempDir; use tokio::sync::oneshot; use tokio::time::sleep; diff --git a/crates/common/Cargo.toml b/crates/common/Cargo.toml index eb39d2ef5..80dc92fbe 100644 --- a/crates/common/Cargo.toml +++ b/crates/common/Cargo.toml @@ -20,14 +20,22 @@ hex = { workspace = true } hyper = { workspace = true } jsonrpsee = { workspace = true, features = ["http-client", "server"] } lru = { workspace = true } +serde = { workspace = true } tokio = { workspace = true } tokio-util = { workspace = true } +toml = { workspace = true } tower-http = { workspace = true } tracing = { workspace = true } # Sov SDK deps sov-db = { path = "../sovereign-sdk/full-node/db/sov-db" } sov-rollup-interface = { path = "../sovereign-sdk/rollup-interface" } +sov-stf-runner = { path = "../sovereign-sdk/full-node/sov-stf-runner" } # Citrea citrea-primitives = { path = "../primitives/" } +citrea-pruning = { path = "../pruning" } + +[dev-dependencies] +sov-mock-da = { path = "../sovereign-sdk/adapters/mock-da", features = ["native"] } +tempfile = { workspace = true } diff --git a/crates/sovereign-sdk/full-node/sov-stf-runner/src/config.rs b/crates/common/src/config.rs similarity index 66% rename from crates/sovereign-sdk/full-node/sov-stf-runner/src/config.rs rename to crates/common/src/config.rs index c5cbac51b..3726d7310 100644 --- a/crates/sovereign-sdk/full-node/sov-stf-runner/src/config.rs +++ b/crates/common/src/config.rs @@ -5,9 +5,7 @@ use std::path::{Path, PathBuf}; use citrea_pruning::PruningConfig; use serde::de::DeserializeOwned; use serde::{Deserialize, Serialize}; - -use crate::ProverGuestRunConfig; - +use sov_stf_runner::ProverGuestRunConfig; /// Runner configuration. #[derive(Debug, Clone, PartialEq, Deserialize, Serialize)] pub struct RunnerConfig { @@ -163,6 +161,74 @@ pub fn from_toml_path, R: DeserializeOwned>(path: P) -> anyhow::R Ok(result) } +/// Rollup Configuration +#[derive(Debug, Clone, PartialEq, Deserialize, Serialize)] +pub struct SequencerConfig { + /// Private key of the sequencer + pub private_key: String, + /// Min. soft confirmaitons for sequencer to commit + pub min_soft_confirmations_per_commitment: u64, + /// Whether or not the sequencer is running in test mode + pub test_mode: bool, + /// Limit for the number of deposit transactions to be included in the block + pub deposit_mempool_fetch_limit: usize, + /// Sequencer specific mempool config + pub mempool_conf: SequencerMempoolConfig, + /// DA layer update loop interval in ms + pub da_update_interval_ms: u64, + /// Block production interval in ms + pub block_production_interval_ms: u64, +} + +impl Default for SequencerConfig { + fn default() -> Self { + SequencerConfig { + private_key: "1212121212121212121212121212121212121212121212121212121212121212" + .to_string(), + min_soft_confirmations_per_commitment: 4, + test_mode: true, + deposit_mempool_fetch_limit: 10, + block_production_interval_ms: 100, + da_update_interval_ms: 100, + mempool_conf: Default::default(), + } + } +} + +/// Mempool Config for the sequencer +/// Read: https://github.com/ledgerwatch/erigon/wiki/Transaction-Pool-Design +#[derive(Debug, Clone, PartialEq, Deserialize, Serialize)] +pub struct SequencerMempoolConfig { + /// Max number of transactions in the pending sub-pool + pub pending_tx_limit: u64, + /// Max megabytes of transactions in the pending sub-pool + pub pending_tx_size: u64, + /// Max number of transactions in the queued sub-pool + pub queue_tx_limit: u64, + /// Max megabytes of transactions in the queued sub-pool + pub queue_tx_size: u64, + /// Max number of transactions in the base-fee sub-pool + pub base_fee_tx_limit: u64, + /// Max megabytes of transactions in the base-fee sub-pool + pub base_fee_tx_size: u64, + /// Max number of executable transaction slots guaranteed per account + pub max_account_slots: u64, +} + +impl Default for SequencerMempoolConfig { + fn default() -> Self { + Self { + pending_tx_limit: 100000, + pending_tx_size: 200, + queue_tx_limit: 100000, + queue_tx_size: 200, + base_fee_tx_limit: 100000, + base_fee_tx_size: 200, + max_account_slots: 16, + } + } +} + #[cfg(test)] mod tests { use std::io::Write; @@ -264,4 +330,47 @@ mod tests { }; assert_eq!(config, expected); } + #[test] + fn test_correct_config_sequencer() { + let config = r#" + private_key = "1212121212121212121212121212121212121212121212121212121212121212" + min_soft_confirmations_per_commitment = 123 + test_mode = false + deposit_mempool_fetch_limit = 10 + da_update_interval_ms = 1000 + block_production_interval_ms = 1000 + [mempool_conf] + pending_tx_limit = 100000 + pending_tx_size = 200 + queue_tx_limit = 100000 + queue_tx_size = 200 + base_fee_tx_limit = 100000 + base_fee_tx_size = 200 + max_account_slots = 16 + "#; + + let config_file = create_config_from(config); + + let config: SequencerConfig = from_toml_path(config_file.path()).unwrap(); + + let expected = SequencerConfig { + private_key: "1212121212121212121212121212121212121212121212121212121212121212" + .to_string(), + min_soft_confirmations_per_commitment: 123, + test_mode: false, + deposit_mempool_fetch_limit: 10, + mempool_conf: SequencerMempoolConfig { + pending_tx_limit: 100000, + pending_tx_size: 200, + queue_tx_limit: 100000, + queue_tx_size: 200, + base_fee_tx_limit: 100000, + base_fee_tx_size: 200, + max_account_slots: 16, + }, + da_update_interval_ms: 1000, + block_production_interval_ms: 1000, + }; + assert_eq!(config, expected); + } } diff --git a/crates/common/src/lib.rs b/crates/common/src/lib.rs index 9b1396d12..395ad9ab9 100644 --- a/crates/common/src/lib.rs +++ b/crates/common/src/lib.rs @@ -2,8 +2,10 @@ #![forbid(unsafe_code)] pub mod cache; +pub mod config; pub mod da; pub mod error; pub mod rpc; pub mod tasks; pub mod utils; +pub use config::*; diff --git a/crates/fullnode/src/runner.rs b/crates/fullnode/src/runner.rs index 07dbb45eb..93269bfbc 100644 --- a/crates/fullnode/src/runner.rs +++ b/crates/fullnode/src/runner.rs @@ -8,6 +8,7 @@ use backoff::ExponentialBackoffBuilder; use citrea_common::cache::L1BlockCache; use citrea_common::da::get_da_block_at_height; use citrea_common::tasks::manager::TaskManager; +use citrea_common::{RollupPublicKeys, RpcConfig, RunnerConfig}; use citrea_primitives::types::SoftConfirmationHash; use citrea_pruning::{Pruner, PruningConfig}; use jsonrpsee::core::client::Error as JsonrpseeError; @@ -26,7 +27,7 @@ pub use sov_rollup_interface::stf::BatchReceipt; use sov_rollup_interface::stf::StateTransitionFunction; use sov_rollup_interface::storage::HierarchicalStorageManager; use sov_rollup_interface::zk::{Zkvm, ZkvmHost}; -use sov_stf_runner::{InitVariant, RollupPublicKeys, RpcConfig, RunnerConfig}; +use sov_stf_runner::InitVariant; use tokio::sync::{broadcast, mpsc, oneshot, Mutex}; use tokio::time::{sleep, Duration}; use tokio::{select, signal}; diff --git a/crates/fullnode/tests/runner_initialization_tests.rs b/crates/fullnode/tests/runner_initialization_tests.rs index a5614bdbe..3c30b90e0 100644 --- a/crates/fullnode/tests/runner_initialization_tests.rs +++ b/crates/fullnode/tests/runner_initialization_tests.rs @@ -1,6 +1,7 @@ use std::collections::HashMap; use std::sync::Arc; +use citrea_common::{FullNodeConfig, RollupPublicKeys, RpcConfig, RunnerConfig, StorageConfig}; use citrea_fullnode::CitreaFullnode; use sov_db::ledger_db::LedgerDB; use sov_db::rocks_db_config::RocksdbConfig; @@ -10,10 +11,7 @@ use sov_prover_storage_manager::ProverStorageManager; use sov_rollup_interface::fork::{Fork, ForkManager}; use sov_rollup_interface::spec::SpecId; use sov_state::DefaultStorageSpec; -use sov_stf_runner::{ - FullNodeConfig, InitVariant, RollupPublicKeys, RpcConfig, RunnerConfig, StorageConfig, -}; - +use sov_stf_runner::InitVariant; mod hash_stf; use hash_stf::HashStf; diff --git a/crates/prover/src/da_block_handler.rs b/crates/prover/src/da_block_handler.rs index 5f455626b..b128b9ec8 100644 --- a/crates/prover/src/da_block_handler.rs +++ b/crates/prover/src/da_block_handler.rs @@ -13,6 +13,7 @@ use citrea_common::utils::{ check_l2_range_exists, extract_sequencer_commitments, filter_out_proven_commitments, merge_state_diffs, }; +use citrea_common::ProverConfig; use citrea_primitives::MAX_TXBODY_SIZE; use rand::Rng; use serde::de::DeserializeOwned; @@ -26,7 +27,7 @@ use sov_rollup_interface::services::da::{DaService, SlotData}; use sov_rollup_interface::soft_confirmation::SignedSoftConfirmation; use sov_rollup_interface::spec::SpecId; use sov_rollup_interface::zk::{Proof, StateTransitionData, ZkvmHost}; -use sov_stf_runner::{ProverConfig, ProverService}; +use sov_stf_runner::ProverService; use tokio::select; use tokio::sync::{mpsc, Mutex}; use tokio::time::{sleep, Duration}; diff --git a/crates/prover/src/prover_service/parallel/mod.rs b/crates/prover/src/prover_service/parallel/mod.rs index 1c7d79205..8b61796b9 100644 --- a/crates/prover/src/prover_service/parallel/mod.rs +++ b/crates/prover/src/prover_service/parallel/mod.rs @@ -3,6 +3,7 @@ use std::sync::Arc; use async_trait::async_trait; use borsh::{BorshDeserialize, BorshSerialize}; +use citrea_common::config::ProverConfig; use citrea_stf::verifier::StateTransitionVerifier; use parking_lot::Mutex; use prover::Prover; @@ -13,7 +14,6 @@ use sov_rollup_interface::da::{DaData, DaSpec}; use sov_rollup_interface::services::da::DaService; use sov_rollup_interface::stf::StateTransitionFunction; use sov_rollup_interface::zk::{Proof, StateTransitionData, ZkvmHost}; -use sov_stf_runner::config::ProverConfig; use sov_stf_runner::{ ProofProcessingStatus, ProverGuestRunConfig, ProverService, ProverServiceError, WitnessSubmissionStatus, diff --git a/crates/prover/src/runner.rs b/crates/prover/src/runner.rs index e0b7b304e..1e5e2d523 100644 --- a/crates/prover/src/runner.rs +++ b/crates/prover/src/runner.rs @@ -10,6 +10,7 @@ use backoff::future::retry as retry_backoff; use citrea_common::cache::L1BlockCache; use citrea_common::da::get_da_block_at_height; use citrea_common::tasks::manager::TaskManager; +use citrea_common::{ProverConfig, RollupPublicKeys, RpcConfig, RunnerConfig}; use citrea_primitives::types::SoftConfirmationHash; use jsonrpsee::core::client::Error as JsonrpseeError; use jsonrpsee::server::{BatchRequestConfig, ServerBuilder}; @@ -26,9 +27,7 @@ use sov_rollup_interface::services::da::DaService; use sov_rollup_interface::spec::SpecId; use sov_rollup_interface::stf::StateTransitionFunction; use sov_rollup_interface::zk::ZkvmHost; -use sov_stf_runner::{ - InitVariant, ProverConfig, ProverService, RollupPublicKeys, RpcConfig, RunnerConfig, -}; +use sov_stf_runner::{InitVariant, ProverService}; use tokio::sync::{broadcast, mpsc, oneshot, Mutex}; use tokio::time::sleep; use tokio::{select, signal}; diff --git a/crates/sequencer/src/config.rs b/crates/sequencer/src/config.rs deleted file mode 100644 index 48fdd2d48..000000000 --- a/crates/sequencer/src/config.rs +++ /dev/null @@ -1,129 +0,0 @@ -use serde::{Deserialize, Serialize}; - -/// Rollup Configuration -#[derive(Debug, Clone, PartialEq, Deserialize, Serialize)] -pub struct SequencerConfig { - /// Private key of the sequencer - pub private_key: String, - /// Min. soft confirmaitons for sequencer to commit - pub min_soft_confirmations_per_commitment: u64, - /// Whether or not the sequencer is running in test mode - pub test_mode: bool, - /// Limit for the number of deposit transactions to be included in the block - pub deposit_mempool_fetch_limit: usize, - /// Sequencer specific mempool config - pub mempool_conf: SequencerMempoolConfig, - /// DA layer update loop interval in ms - pub da_update_interval_ms: u64, - /// Block production interval in ms - pub block_production_interval_ms: u64, -} - -impl Default for SequencerConfig { - fn default() -> Self { - SequencerConfig { - private_key: "1212121212121212121212121212121212121212121212121212121212121212" - .to_string(), - min_soft_confirmations_per_commitment: 4, - test_mode: true, - deposit_mempool_fetch_limit: 10, - block_production_interval_ms: 100, - da_update_interval_ms: 100, - mempool_conf: Default::default(), - } - } -} - -/// Mempool Config for the sequencer -/// Read: https://github.com/ledgerwatch/erigon/wiki/Transaction-Pool-Design -#[derive(Debug, Clone, PartialEq, Deserialize, Serialize)] -pub struct SequencerMempoolConfig { - /// Max number of transactions in the pending sub-pool - pub pending_tx_limit: u64, - /// Max megabytes of transactions in the pending sub-pool - pub pending_tx_size: u64, - /// Max number of transactions in the queued sub-pool - pub queue_tx_limit: u64, - /// Max megabytes of transactions in the queued sub-pool - pub queue_tx_size: u64, - /// Max number of transactions in the base-fee sub-pool - pub base_fee_tx_limit: u64, - /// Max megabytes of transactions in the base-fee sub-pool - pub base_fee_tx_size: u64, - /// Max number of executable transaction slots guaranteed per account - pub max_account_slots: u64, -} - -impl Default for SequencerMempoolConfig { - fn default() -> Self { - Self { - pending_tx_limit: 100000, - pending_tx_size: 200, - queue_tx_limit: 100000, - queue_tx_size: 200, - base_fee_tx_limit: 100000, - base_fee_tx_size: 200, - max_account_slots: 16, - } - } -} - -#[cfg(test)] -mod tests { - use std::io::Write; - - use sov_stf_runner::from_toml_path; - use tempfile::NamedTempFile; - - use super::*; - - fn create_config_from(content: &str) -> NamedTempFile { - let mut config_file = NamedTempFile::new().unwrap(); - config_file.write_all(content.as_bytes()).unwrap(); - config_file - } - - #[test] - fn test_correct_config_sequencer() { - let config = r#" - private_key = "1212121212121212121212121212121212121212121212121212121212121212" - min_soft_confirmations_per_commitment = 123 - test_mode = false - deposit_mempool_fetch_limit = 10 - da_update_interval_ms = 1000 - block_production_interval_ms = 1000 - [mempool_conf] - pending_tx_limit = 100000 - pending_tx_size = 200 - queue_tx_limit = 100000 - queue_tx_size = 200 - base_fee_tx_limit = 100000 - base_fee_tx_size = 200 - max_account_slots = 16 - "#; - - let config_file = create_config_from(config); - - let config: SequencerConfig = from_toml_path(config_file.path()).unwrap(); - - let expected = SequencerConfig { - private_key: "1212121212121212121212121212121212121212121212121212121212121212" - .to_string(), - min_soft_confirmations_per_commitment: 123, - test_mode: false, - deposit_mempool_fetch_limit: 10, - mempool_conf: SequencerMempoolConfig { - pending_tx_limit: 100000, - pending_tx_size: 200, - queue_tx_limit: 100000, - queue_tx_size: 200, - base_fee_tx_limit: 100000, - base_fee_tx_size: 200, - max_account_slots: 16, - }, - da_update_interval_ms: 1000, - block_production_interval_ms: 1000, - }; - assert_eq!(config, expected); - } -} diff --git a/crates/sequencer/src/lib.rs b/crates/sequencer/src/lib.rs index e896f94d9..a590df559 100644 --- a/crates/sequencer/src/lib.rs +++ b/crates/sequencer/src/lib.rs @@ -1,5 +1,4 @@ mod commitment_controller; -mod config; mod db_provider; mod deposit_data_mempool; mod mempool; @@ -9,7 +8,7 @@ mod utils; use std::net::SocketAddr; -pub use config::{SequencerConfig, SequencerMempoolConfig}; +pub use citrea_common::{SequencerConfig, SequencerMempoolConfig}; pub use rpc::SequencerRpcClient; pub use sequencer::CitreaSequencer; use sov_db::ledger_db::LedgerDB; diff --git a/crates/sequencer/src/mempool.rs b/crates/sequencer/src/mempool.rs index 3a784909b..e477fdd2b 100644 --- a/crates/sequencer/src/mempool.rs +++ b/crates/sequencer/src/mempool.rs @@ -1,6 +1,7 @@ use std::sync::Arc; use anyhow::{anyhow, bail}; +use citrea_common::SequencerMempoolConfig; use citrea_evm::SYSTEM_SIGNER; use reth_chainspec::{Chain, ChainSpecBuilder}; use reth_primitives::{Genesis, TxHash}; @@ -13,7 +14,6 @@ use reth_transaction_pool::{ TransactionPool, TransactionPoolExt, TransactionValidationTaskExecutor, ValidPoolTransaction, }; -use crate::config::SequencerMempoolConfig; pub use crate::db_provider::DbProvider; type CitreaMempoolImpl = Pool< diff --git a/crates/sequencer/src/sequencer.rs b/crates/sequencer/src/sequencer.rs index 2d2c1d0e1..99f35992d 100644 --- a/crates/sequencer/src/sequencer.rs +++ b/crates/sequencer/src/sequencer.rs @@ -10,6 +10,7 @@ use backoff::ExponentialBackoffBuilder; use borsh::BorshDeserialize; use citrea_common::tasks::manager::TaskManager; use citrea_common::utils::merge_state_diffs; +use citrea_common::{RollupPublicKeys, RpcConfig, SequencerConfig}; use citrea_evm::{CallMessage, Evm, RlpEvmTransaction, MIN_TRANSACTION_GAS}; use citrea_primitives::basefee::calculate_next_block_base_fee; use citrea_primitives::types::SoftConfirmationHash; @@ -46,7 +47,7 @@ use sov_rollup_interface::services::da::{DaService, SenderWithNotifier}; use sov_rollup_interface::stf::StateTransitionFunction; use sov_rollup_interface::storage::HierarchicalStorageManager; use sov_rollup_interface::zk::ZkvmHost; -use sov_stf_runner::{InitVariant, RollupPublicKeys, RpcConfig}; +use sov_stf_runner::InitVariant; use tokio::signal; use tokio::sync::oneshot::channel as oneshot_channel; use tokio::sync::{broadcast, mpsc}; @@ -55,7 +56,6 @@ use tokio_util::sync::CancellationToken; use tracing::{debug, error, info, instrument, trace, warn}; use crate::commitment_controller; -use crate::config::SequencerConfig; use crate::db_provider::DbProvider; use crate::deposit_data_mempool::DepositDataMempool; use crate::mempool::CitreaMempool; diff --git a/crates/sovereign-sdk/examples/demo-stf/Cargo.toml b/crates/sovereign-sdk/examples/demo-stf/Cargo.toml index 1dd210e63..1c88b6fa0 100644 --- a/crates/sovereign-sdk/examples/demo-stf/Cargo.toml +++ b/crates/sovereign-sdk/examples/demo-stf/Cargo.toml @@ -16,7 +16,6 @@ borsh = { workspace = true } serde = { workspace = true } serde_json = { workspace = true, optional = true } clap = { workspace = true, optional = true } -toml = { workspace = true, optional = true } jsonrpsee = { workspace = true, features = [ "http-client", "server", @@ -67,7 +66,6 @@ native = [ "soft-confirmation-rule-enforcer/native", "jsonrpsee", "tokio", - "toml", ] serde = [ "sov-bank/serde", diff --git a/crates/sovereign-sdk/full-node/sov-stf-runner/Cargo.toml b/crates/sovereign-sdk/full-node/sov-stf-runner/Cargo.toml index 3d4d84efa..76058e7b0 100644 --- a/crates/sovereign-sdk/full-node/sov-stf-runner/Cargo.toml +++ b/crates/sovereign-sdk/full-node/sov-stf-runner/Cargo.toml @@ -24,7 +24,6 @@ serde = { workspace = true } serde_json = { workspace = true } thiserror = { workspace = true, optional = true } tokio = { workspace = true, optional = true } -toml = { workspace = true, optional = true } tower = { workspace = true, optional = true } tracing = { workspace = true, optional = true } @@ -33,14 +32,10 @@ sov-db = { path = "../db/sov-db", optional = true } sov-modules-api = { path = "../../module-system/sov-modules-api", default-features = false } sov-rollup-interface = { path = "../../rollup-interface" } -# Citrea -citrea-pruning = { path = "../../../pruning", optional = true } - [dev-dependencies] sha2 = { workspace = true } tempfile = { workspace = true } -sov-mock-da = { path = "../../adapters/mock-da", features = ["native"] } sov-modules-api = { path = "../../module-system/sov-modules-api", features = ["native"] } sov-prover-storage-manager = { path = "../sov-prover-storage-manager", features = ["test-utils"] } sov-state = { path = "../../module-system/sov-state", features = ["native"] } @@ -52,7 +47,6 @@ native = [ "sov-db", "sov-modules-api/native", "jsonrpsee", - "toml", "tokio", "tracing", "futures", @@ -61,5 +55,4 @@ native = [ "rand", "tower", "hyper", - "dep:citrea-pruning", ] diff --git a/crates/sovereign-sdk/full-node/sov-stf-runner/src/lib.rs b/crates/sovereign-sdk/full-node/sov-stf-runner/src/lib.rs index 700f8fab4..2eff97fbc 100644 --- a/crates/sovereign-sdk/full-node/sov-stf-runner/src/lib.rs +++ b/crates/sovereign-sdk/full-node/sov-stf-runner/src/lib.rs @@ -1,9 +1,6 @@ #![deny(missing_docs)] #![doc = include_str!("../README.md")] -#[cfg(feature = "native")] -/// Config -pub mod config; /// Testing utilities. #[cfg(feature = "mock")] pub mod mock; @@ -16,8 +13,6 @@ use std::path::Path; #[cfg(feature = "native")] use anyhow::Context; #[cfg(feature = "native")] -pub use config::*; -#[cfg(feature = "native")] pub use prover_service::*; #[cfg(feature = "native")] use sov_modules_api::{DaSpec, Zkvm}; diff --git a/crates/sovereign-sdk/module-system/sov-modules-rollup-blueprint/Cargo.toml b/crates/sovereign-sdk/module-system/sov-modules-rollup-blueprint/Cargo.toml index 706a2f309..8bee6e5de 100644 --- a/crates/sovereign-sdk/module-system/sov-modules-rollup-blueprint/Cargo.toml +++ b/crates/sovereign-sdk/module-system/sov-modules-rollup-blueprint/Cargo.toml @@ -12,6 +12,7 @@ resolver = "2" description = "This crate contains abstractions needed to create a new rollup" [dependencies] +citrea-common = { path = "../../../common" } sov-cli = { path = "../../module-system/sov-cli" } sov-modules-api = { path = "../../module-system/sov-modules-api", features = ["native"] } sov-rollup-interface = { path = "../../rollup-interface", features = ["native"] } diff --git a/crates/sovereign-sdk/module-system/sov-modules-rollup-blueprint/src/lib.rs b/crates/sovereign-sdk/module-system/sov-modules-rollup-blueprint/src/lib.rs index 74ed15aec..386cc20c1 100644 --- a/crates/sovereign-sdk/module-system/sov-modules-rollup-blueprint/src/lib.rs +++ b/crates/sovereign-sdk/module-system/sov-modules-rollup-blueprint/src/lib.rs @@ -8,6 +8,7 @@ use std::collections::HashMap; use std::sync::Arc; use async_trait::async_trait; +use citrea_common::{FullNodeConfig, ProverConfig}; pub use runtime_rpc::*; use sov_db::ledger_db::LedgerDB; use sov_db::rocks_db_config::RocksdbConfig; @@ -18,7 +19,7 @@ use sov_rollup_interface::spec::SpecId; use sov_rollup_interface::storage::HierarchicalStorageManager; use sov_rollup_interface::zk::{Zkvm, ZkvmHost}; use sov_state::Storage; -use sov_stf_runner::{FullNodeConfig, ProverConfig, ProverService}; +use sov_stf_runner::ProverService; use tokio::sync::broadcast; pub use wallet::*; From b156b08c49c0b62d8280611f66adcd55f0998ade Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erce=20Can=20Bekt=C3=BCre?= <47954181+ercecan@users.noreply.github.com> Date: Wed, 9 Oct 2024 14:26:17 +0300 Subject: [PATCH 20/28] Update e2e test framework and fix tests (#1305) * Update e2e test framework and fix tests * Fix bug * Update ci binary env key * Test if new fix works * Update bitcoincore-rpc version * Dprint * Try against fix prover config rev * Target main HEAD rev --------- Co-authored-by: jfldde <168934971+jfldde@users.noreply.github.com> --- .github/workflows/checks.yml | 4 ++-- Cargo.lock | 10 ++++------ Cargo.toml | 2 +- bin/citrea/Cargo.toml | 2 +- bin/citrea/tests/bitcoin_e2e/prover_test.rs | 16 +++++++++------- .../tests/bitcoin_e2e/sequencer_commitments.rs | 4 ++-- 6 files changed, 19 insertions(+), 19 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index e0e7efa52..93dfe4b1e 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -224,7 +224,7 @@ jobs: RUST_BACKTRACE: 1 USE_DOCKER: "true" SHORT_PREFIX: 1 - CITREA: ${{ github.workspace }}/target/debug/citrea + CITREA_E2E_TEST_BINARY: ${{ github.workspace }}/target/debug/citrea - name: Upload coverage uses: codecov/codecov-action@v4 with: @@ -396,7 +396,7 @@ jobs: BONSAI_API_KEY: ${{ secrets.BONSAI_API_KEY }} # TODO: remove this once we don't use the client on tests USE_DOCKER: "true" SHORT_PREFIX: 1 - CITREA: ${{ github.workspace }}/target/debug/citrea + CITREA_E2E_TEST_BINARY: ${{ github.workspace }}/target/debug/citrea system-contracts: strategy: diff --git a/Cargo.lock b/Cargo.lock index d180677df..fe0eaac23 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1270,7 +1270,7 @@ dependencies = [ [[package]] name = "bitcoincore-rpc" version = "0.18.0" -source = "git+https://github.com/chainwayxyz/rust-bitcoincore-rpc.git?rev=0ae498d#0ae498da2b6aa6e89b1a1bc9d484eb55919718ba" +source = "git+https://github.com/chainwayxyz/rust-bitcoincore-rpc.git?rev=ede8097#ede8097e0f0fce5d69e9cab6f85fda39e788a97b" dependencies = [ "async-trait", "bitcoincore-rpc-json", @@ -1285,7 +1285,7 @@ dependencies = [ [[package]] name = "bitcoincore-rpc-json" version = "0.18.0" -source = "git+https://github.com/chainwayxyz/rust-bitcoincore-rpc.git?rev=0ae498d#0ae498da2b6aa6e89b1a1bc9d484eb55919718ba" +source = "git+https://github.com/chainwayxyz/rust-bitcoincore-rpc.git?rev=ede8097#ede8097e0f0fce5d69e9cab6f85fda39e788a97b" dependencies = [ "bitcoin", "serde", @@ -1771,7 +1771,7 @@ dependencies = [ [[package]] name = "citrea-e2e" version = "0.1.0" -source = "git+https://github.com/chainwayxyz/citrea-e2e?branch=main#669d14f9b5ada394d045c63f4067fbeaecfe9aea" +source = "git+https://github.com/chainwayxyz/citrea-e2e?rev=a96abcf#a96abcfe145b9d2467e2f6a5b996cf458bb2d079" dependencies = [ "anyhow", "async-trait", @@ -1781,17 +1781,15 @@ dependencies = [ "futures", "hex", "jsonrpsee", - "log", "rand 0.8.5", "serde", "serde_json", "sov-ledger-rpc 0.5.0-rc.1 (git+https://github.com/chainwayxyz/citrea?rev=82bf52d)", "sov-rollup-interface 0.5.0-rc.1 (git+https://github.com/chainwayxyz/citrea?rev=82bf52d)", - "syn 1.0.109", "tempfile", "tokio", "toml", - "which", + "tracing", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index c0d132682..ed4c64a3b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -185,4 +185,4 @@ ed25519-dalek = { git = "https://github.com/risc0/curve25519-dalek", tag = "curv crypto-bigint = { git = "https://github.com/risc0/RustCrypto-crypto-bigint", tag = "v0.5.5-risczero.0" } secp256k1 = { git = "https://github.com/Sovereign-Labs/rust-secp256k1.git", branch = "risc0-compatible-0-29-0" } k256 = { git = "https://github.com/risc0/RustCrypto-elliptic-curves", tag = "k256/v0.13.3-risczero.0" } -bitcoincore-rpc = { version = "0.18.0", git = "https://github.com/chainwayxyz/rust-bitcoincore-rpc.git", rev = "0ae498d" } +bitcoincore-rpc = { version = "0.18.0", git = "https://github.com/chainwayxyz/rust-bitcoincore-rpc.git", rev = "ede8097" } diff --git a/bin/citrea/Cargo.toml b/bin/citrea/Cargo.toml index 1b8f65e50..7959bcd4a 100644 --- a/bin/citrea/Cargo.toml +++ b/bin/citrea/Cargo.toml @@ -86,7 +86,7 @@ rustc_version_runtime = { workspace = true } # bitcoin-e2e dependencies bitcoin.workspace = true bitcoincore-rpc.workspace = true -citrea-e2e = { git = "https://github.com/chainwayxyz/citrea-e2e", branch = "main" } +citrea-e2e = { git = "https://github.com/chainwayxyz/citrea-e2e", rev = "a96abcf" } [features] default = [] # Deviate from convention by making the "native" feature active by default. This aligns with how this package is meant to be used (as a binary first, library second). diff --git a/bin/citrea/tests/bitcoin_e2e/prover_test.rs b/bin/citrea/tests/bitcoin_e2e/prover_test.rs index d9deac1f8..0e0a22e88 100644 --- a/bin/citrea/tests/bitcoin_e2e/prover_test.rs +++ b/bin/citrea/tests/bitcoin_e2e/prover_test.rs @@ -26,7 +26,7 @@ struct BasicProverTest; impl TestCase for BasicProverTest { fn test_config() -> TestCaseConfig { TestCaseConfig { - with_prover: true, + with_batch_prover: true, with_full_node: true, ..Default::default() } @@ -44,8 +44,8 @@ impl TestCase for BasicProverTest { bail!("Sequencer not running. Set TestCaseConfig with_sequencer to true") }; - let Some(prover) = &f.prover else { - bail!("Prover not running. Set TestCaseConfig with_prover to true") + let Some(batch_prover) = &f.batch_prover else { + bail!("Batch Prover not running. Set TestCaseConfig with_prover to true") }; let Some(full_node) = &f.full_node else { @@ -73,7 +73,9 @@ impl TestCase for BasicProverTest { da.generate(FINALITY_DEPTH, None).await?; let finalized_height = da.get_finalized_height().await?; - prover.wait_for_l1_height(finalized_height, None).await?; + batch_prover + .wait_for_l1_height(finalized_height, None) + .await?; da.generate(FINALITY_DEPTH, None).await?; let proofs = full_node @@ -114,7 +116,7 @@ struct SkipPreprovenCommitmentsTest { impl TestCase for SkipPreprovenCommitmentsTest { fn test_config() -> TestCaseConfig { TestCaseConfig { - with_prover: true, + with_batch_prover: true, with_full_node: true, ..Default::default() } @@ -125,8 +127,8 @@ impl TestCase for SkipPreprovenCommitmentsTest { bail!("Sequencer not running. Set TestCaseConfig with_sequencer to true") }; - let Some(prover) = &f.prover else { - bail!("Prover not running. Set TestCaseConfig with_prover to true") + let Some(prover) = &f.batch_prover else { + bail!("Batch Prover not running. Set TestCaseConfig with_prover to true") }; let Some(full_node) = &f.full_node else { diff --git a/bin/citrea/tests/bitcoin_e2e/sequencer_commitments.rs b/bin/citrea/tests/bitcoin_e2e/sequencer_commitments.rs index 623d1855a..b72c3071c 100644 --- a/bin/citrea/tests/bitcoin_e2e/sequencer_commitments.rs +++ b/bin/citrea/tests/bitcoin_e2e/sequencer_commitments.rs @@ -19,7 +19,7 @@ struct LedgerGetCommitmentsProverTest; impl TestCase for LedgerGetCommitmentsProverTest { fn test_config() -> TestCaseConfig { TestCaseConfig { - with_prover: true, + with_batch_prover: true, ..Default::default() } } @@ -31,7 +31,7 @@ impl TestCase for LedgerGetCommitmentsProverTest { async fn run_test(&mut self, f: &mut TestFramework) -> Result<()> { let sequencer = f.sequencer.as_ref().unwrap(); let da = f.bitcoin_nodes.get(0).expect("DA not running."); - let prover = f.prover.as_ref().unwrap(); + let prover = f.batch_prover.as_ref().unwrap(); let min_soft_confirmations_per_commitment = sequencer.min_soft_confirmations_per_commitment(); From 095f2c172d410b690862f2fb36af4696dd35d4bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=87etin?= Date: Wed, 9 Oct 2024 14:53:50 +0300 Subject: [PATCH 21/28] build and publish a new image for every commit to the nightly branch (#1309) --- .github/workflows/nightly_build_push.yml | 182 +++++++++++++++++++++++ build-push/nightly/Dockerfile | 12 ++ 2 files changed, 194 insertions(+) create mode 100644 .github/workflows/nightly_build_push.yml create mode 100644 build-push/nightly/Dockerfile diff --git a/.github/workflows/nightly_build_push.yml b/.github/workflows/nightly_build_push.yml new file mode 100644 index 000000000..a0e6e589a --- /dev/null +++ b/.github/workflows/nightly_build_push.yml @@ -0,0 +1,182 @@ +name: nightly-build-and-push + +on: + push: + branches: + - nightly + +env: + EXPECTED_BITCOIN_DA_ID: ${{ vars.EXPECTED_BITCOIN_DA_ID }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + IMAGE_TAG: ${{ github.sha }} + +jobs: + + validate_DA_ID_format: + runs-on: ubuntu-latest + steps: + - name: Validate EXPECTED_BITCOIN_DA_ID format + run: | + echo "Raw EXPECTED_BITCOIN_DA_ID value:" + echo "$EXPECTED_BITCOIN_DA_ID" + + echo "Length of EXPECTED_BITCOIN_DA_ID: ${#EXPECTED_BITCOIN_DA_ID}" + + if [ -z "${EXPECTED_BITCOIN_DA_ID// }" ]; then + echo "Error: EXPECTED_BITCOIN_DA_ID is not set, empty, or contains only spaces" + exit 1 + fi + + # Remove any trailing newline or carriage return + EXPECTED_BITCOIN_DA_ID=$(echo "$EXPECTED_BITCOIN_DA_ID" | tr -d '\n\r') + + # Count commas and spaces + comma_count=$(echo "$EXPECTED_BITCOIN_DA_ID" | tr -cd ',' | wc -c) + space_count=$(echo "$EXPECTED_BITCOIN_DA_ID" | tr -cd ' ' | wc -c) + + echo "Number of commas: $comma_count" + echo "Number of spaces: $space_count" + + # Split the string into an array and trim each element + IFS=', ' read -ra raw_numbers <<< "$EXPECTED_BITCOIN_DA_ID" + numbers=() + for num in "${raw_numbers[@]}"; do + trimmed_num=$(echo "$num" | tr -d '[:space:]') # Remove all whitespace + numbers+=("$trimmed_num") + done + + echo "Number of elements after splitting and trimming: ${#numbers[@]}" + + # Check if there are exactly 8 numbers + if [ ${#numbers[@]} -ne 8 ]; then + echo "Error: EXPECTED_BITCOIN_DA_ID should contain exactly 8 numbers" + echo "Actual number of elements: ${#numbers[@]}" + exit 1 + fi + + # Check if all numbers are valid u32 + for i in "${!numbers[@]}"; do + num=${numbers[$i]} + echo "Checking number $((i+1)): '$num'" + echo "Hex representation: $(echo -n "$num" | xxd -p)" + if ! [[ $num =~ ^[0-9]+$ ]]; then + echo "Error: '$num' is not composed of digits only" + exit 1 + fi + if [ $num -gt 4294967295 ]; then + echo "Error: '$num' is greater than 4294967295" + exit 1 + fi + done + + # Reconstruct the trimmed DA_ID + trimmed_da_id=$(IFS=', '; echo "${numbers[*]}") + + # Final check + if [ $comma_count -eq 7 ] && [ $space_count -eq 7 ] && [ ${#numbers[@]} -eq 8 ]; then + echo "EXPECTED_BITCOIN_DA_ID is valid:" + echo "- Contains 7 commas" + echo "- Contains 7 spaces" + echo "- Contains 8 valid u32 numbers" + echo "Original value: $EXPECTED_BITCOIN_DA_ID" + echo "Trimmed value: $trimmed_da_id" + else + echo "Error: EXPECTED_BITCOIN_DA_ID format is incorrect" + echo "- Comma count: $comma_count (should be 7)" + echo "- Space count: $space_count (should be 7)" + echo "- Number count: ${#numbers[@]} (should be 8)" + exit 1 + fi + + linux_amd64_binary_extraction: + needs: validate_DA_ID_format + runs-on: ubicloud-standard-30 + strategy: + matrix: + include: + - short_prefix: 1 + short_prefix_value: "-short-prefix" + - short_prefix: 0 + short_prefix_value: "" + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install Dependencies + run: | + sudo apt update && sudo apt -y install curl gcc cpp cmake clang llvm + sudo apt -y autoremove && sudo apt clean && sudo rm -rf /var/lib/apt/lists/* + + - name: Install Rust + run: | + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y + rustup install 1.79.0 + rustup default 1.79.0 + + - name: Install Cargo Binstall + run: | + cargo install --version 1.7.0 cargo-binstall + + - name: Install cargo-risczero + run: | + cargo binstall cargo-risczero@1.0.5 --no-confirm + + - name: Install risc0-zkvm toolchain + run: cargo risczero install --version r0.1.79.0-2 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Build Project + env: + REPR_GUEST_BUILD: 1 + SHORT_PREFIX: ${{ matrix.short_prefix }} + SKIP_GUEST_BUILD: 0 + run: | + cargo build --release + + - name: Check BITCOIN_DA_ID + id: check-id + run: | + RESULT=$(grep -R "BITCOIN_DA_ID" target/ || echo "Grep failed") + EXPECTED_BITCOIN_DA_ID=$(echo "${{ env.EXPECTED_BITCOIN_DA_ID }}" | tr -d '\n\r') + if echo "$RESULT" | grep -q "$EXPECTED_BITCOIN_DA_ID"; then + echo "Check passed successfully." + echo "Expected: BITCOIN_DA_ID ${{ env.EXPECTED_BITCOIN_DA_ID }} " + echo "Actual: $RESULT" + + else + echo "Check failed. Expected: BITCOIN_DA_ID ${{ env.EXPECTED_BITCOIN_DA_ID }} " + echo "Actual: $RESULT" + exit 1 + fi + + - name: Copy binary to devops/nightly + run: | + cp target/release/citrea devops/nightly/citrea + chmod +x devops/nightly/citrea + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ vars.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Build Docker image + uses: docker/build-push-action@v6 + with: + file: ./devops/nightly/Dockerfile + context: ./devops/nightly + tags: ${{ vars.DOCKERHUB_USERNAME }}/citrea:${{ env.IMAGE_TAG }}${{ matrix.short_prefix_value }} + platforms: linux/amd64 + push: true + load: false + provenance: false + + + diff --git a/build-push/nightly/Dockerfile b/build-push/nightly/Dockerfile new file mode 100644 index 000000000..c9ebc1135 --- /dev/null +++ b/build-push/nightly/Dockerfile @@ -0,0 +1,12 @@ +FROM debian:bookworm-slim + +RUN apt-get update && \ + apt-get install -y --no-install-recommends ca-certificates && \ + update-ca-certificates && \ + rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +COPY citrea /app/citrea + +ENTRYPOINT ["./citrea"] \ No newline at end of file From d8708dee4deda6c89721445df786f9fb4b20e7ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erce=20Can=20Bekt=C3=BCre?= <47954181+ercecan@users.noreply.github.com> Date: Wed, 9 Oct 2024 14:57:09 +0300 Subject: [PATCH 22/28] Get fee recommendation from mempool space (#1302) * Get fee recommendation from mempool space * Fix bug * Construct mempool space endpoint by network --------- Co-authored-by: Esad Yusuf Atik --- Cargo.lock | 1 + crates/bitcoin-da/Cargo.toml | 2 ++ crates/bitcoin-da/src/service.rs | 60 ++++++++++++++++++++++++++++++-- 3 files changed, 60 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fe0eaac23..7f1cc2ff9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1222,6 +1222,7 @@ dependencies = [ "hex", "pin-project", "rand 0.8.5", + "reqwest 0.12.5", "serde", "serde_json", "sha2", diff --git a/crates/bitcoin-da/Cargo.toml b/crates/bitcoin-da/Cargo.toml index 1a6f25ac1..ab4504c01 100644 --- a/crates/bitcoin-da/Cargo.toml +++ b/crates/bitcoin-da/Cargo.toml @@ -23,6 +23,7 @@ borsh = { workspace = true } hex = { workspace = true, features = ["serde"] } pin-project = { workspace = true, optional = true, features = [] } rand = { workspace = true } +reqwest = { workspace = true, optional = true } serde = { workspace = true } serde_json = { workspace = true, features = ["raw_value"] } thiserror = { workspace = true } @@ -46,4 +47,5 @@ native = [ "sov-rollup-interface/native", "dep:citrea-primitives", "dep:bitcoincore-rpc", + "dep:reqwest", ] diff --git a/crates/bitcoin-da/src/service.rs b/crates/bitcoin-da/src/service.rs index 0ed159ec6..09224a043 100644 --- a/crates/bitcoin-da/src/service.rs +++ b/crates/bitcoin-da/src/service.rs @@ -56,6 +56,9 @@ use crate::REVEAL_OUTPUT_AMOUNT; pub const FINALITY_DEPTH: u64 = 8; // blocks const POLLING_INTERVAL: u64 = 10; // seconds +const MEMPOOL_SPACE_URL: &str = "https://mempool.space/"; +const MEMPOOL_SPACE_RECOMMENDED_FEE_ENDPOINT: &str = "api/v1/fees/recommended"; + #[derive(PartialEq, Eq, PartialOrd, Ord, core::hash::Hash)] pub struct TxidWrapper(Txid); impl From for [u8; 32] { @@ -544,8 +547,13 @@ impl BitcoinService { #[instrument(level = "trace", skip_all, ret)] pub async fn get_fee_rate_as_sat_vb(&self) -> Result { - let smart_fee = self.client.estimate_smart_fee(1, None).await?; - let sat_vkb = smart_fee.fee_rate.map_or(1000, |rate| rate.to_sat()); + // If network is regtest or signet, mempool space is not available + let smart_fee = get_fee_rate_from_mempool_space(self.network).await?.or(self + .client + .estimate_smart_fee(1, None) + .await? + .fee_rate); + let sat_vkb = smart_fee.map_or(1000, |rate| rate.to_sat()); tracing::debug!("Fee rate: {} sat/vb", sat_vkb / 1000); Ok(sat_vkb / 1000) @@ -980,6 +988,32 @@ fn calculate_witness_root(txdata: &[TransactionWrapper]) -> [u8; 32] { BitcoinMerkleTree::new(hashes).root() } +pub(crate) async fn get_fee_rate_from_mempool_space( + network: bitcoin::Network, +) -> Result> { + let url = match network { + bitcoin::Network::Bitcoin => format!( + // Mainnet + "{}{}", + MEMPOOL_SPACE_URL, MEMPOOL_SPACE_RECOMMENDED_FEE_ENDPOINT + ), + bitcoin::Network::Testnet => format!( + "{}testnet4/{}", + MEMPOOL_SPACE_URL, MEMPOOL_SPACE_RECOMMENDED_FEE_ENDPOINT + ), + _ => return Ok(None), + }; + let fee_rate = reqwest::get(url) + .await? + .json::() + .await? + .get("fastestFee") + .and_then(|fee| fee.as_u64()) + .map(|fee| Amount::from_sat(fee * 1000)); // multiply by 1000 to convert to sat/vkb + + Ok(fee_rate) +} + #[cfg(test)] mod tests { use core::str::FromStr; @@ -995,7 +1029,7 @@ mod tests { use sov_rollup_interface::da::{DaVerifier, SequencerCommitment}; use sov_rollup_interface::services::da::{DaService, SlotData}; - use super::BitcoinService; + use super::{get_fee_rate_from_mempool_space, BitcoinService}; use crate::helpers::parsers::parse_hex_transaction; use crate::helpers::test_utils::{get_mock_data, get_mock_txs}; use crate::service::BitcoinServiceConfig; @@ -1453,4 +1487,24 @@ mod tests { "Publickey recovered incorrectly!" ); } + + #[tokio::test] + async fn test_mempool_space_fee_rate() { + let _fee_rate = get_fee_rate_from_mempool_space(bitcoin::Network::Bitcoin) + .await + .unwrap() + .unwrap(); + let _fee_rate = get_fee_rate_from_mempool_space(bitcoin::Network::Testnet) + .await + .unwrap() + .unwrap(); + assert!(get_fee_rate_from_mempool_space(bitcoin::Network::Regtest) + .await + .unwrap() + .is_none()); + assert!(get_fee_rate_from_mempool_space(bitcoin::Network::Signet) + .await + .unwrap() + .is_none()); + } } From a3cee8db54cc227fb8dd86966135e49611a22074 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=87etin?= Date: Wed, 9 Oct 2024 15:20:14 +0300 Subject: [PATCH 23/28] new path fix for nightly (#1310) --- .github/workflows/nightly_build_push.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/nightly_build_push.yml b/.github/workflows/nightly_build_push.yml index a0e6e589a..e053ba5c6 100644 --- a/.github/workflows/nightly_build_push.yml +++ b/.github/workflows/nightly_build_push.yml @@ -150,10 +150,10 @@ jobs: exit 1 fi - - name: Copy binary to devops/nightly + - name: Copy binary to build-push/nightly run: | - cp target/release/citrea devops/nightly/citrea - chmod +x devops/nightly/citrea + cp target/release/citrea build-push/nightly/citrea + chmod +x build-push/nightly/citrea - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -170,8 +170,8 @@ jobs: - name: Build Docker image uses: docker/build-push-action@v6 with: - file: ./devops/nightly/Dockerfile - context: ./devops/nightly + file: ./build-push/nightly/Dockerfile + context: ./build-push/nightly tags: ${{ vars.DOCKERHUB_USERNAME }}/citrea:${{ env.IMAGE_TAG }}${{ matrix.short_prefix_value }} platforms: linux/amd64 push: true From 31a2b5285c89df7526731c72d2e71756d3534012 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ege=20Okan=20=C3=9Cnald=C4=B1?= <35339130+exeokan@users.noreply.github.com> Date: Wed, 9 Oct 2024 16:31:38 +0300 Subject: [PATCH 24/28] Enable pending block tag in simulation endpoints (#1303) * treat pending tag same as latest * fix lint * return new sealed block for pending * handle pending tag externally * revert enabling pending in some endpoints * get blockenv instead of sealed block * address review comments * implement tests for eth_call, eth_estimateGas, eth_createAccessList pending blocks * rename tests --------- Co-authored-by: eyusufatik --- crates/ethereum-rpc/src/lib.rs | 9 +- crates/ethereum-rpc/src/trace.rs | 8 +- crates/evm/Cargo.toml | 2 +- crates/evm/src/query.rs | 120 +++++++++++++----- crates/evm/src/tests/call_tests.rs | 38 +++++- .../src/tests/queries/estimate_gas_tests.rs | 61 ++++++++- 6 files changed, 197 insertions(+), 41 deletions(-) diff --git a/crates/ethereum-rpc/src/lib.rs b/crates/ethereum-rpc/src/lib.rs index cf19183cf..960369bbd 100644 --- a/crates/ethereum-rpc/src/lib.rs +++ b/crates/ethereum-rpc/src/lib.rs @@ -421,10 +421,15 @@ fn register_rpc_methods( let mut working_set = WorkingSet::::new(ethereum.storage.clone()); let evm = Evm::::default(); + let latest_block_number: u64 = evm.block_number(&mut working_set)?.saturating_to(); + let block_number = match block_number { BlockNumberOrTag::Number(block_number) => block_number, - BlockNumberOrTag::Latest => evm.block_number(&mut working_set)?.saturating_to(), - _ => return Err(EthApiError::Unsupported("Earliest, pending, safe and finalized are not supported for debug_traceBlockByNumber").into()), + BlockNumberOrTag::Latest => latest_block_number, + _ => return Err(EthApiError::Unsupported( + "Earliest, pending, safe and finalized are not supported for debug_traceBlockByNumber", + ) + .into()), }; debug_trace_by_block_number(block_number, None, ðereum, &evm, &mut working_set, opts) diff --git a/crates/ethereum-rpc/src/trace.rs b/crates/ethereum-rpc/src/trace.rs index 7ca650931..81738a00a 100644 --- a/crates/ethereum-rpc/src/trace.rs +++ b/crates/ethereum-rpc/src/trace.rs @@ -61,9 +61,11 @@ pub async fn handle_debug_trace_chain latest_block_number, _ => { - pending.reject(EthApiError::Unsupported( - "Earliest, pending, safe and finalized are not supported for traceChain end block", - )).await; + pending + .reject(EthApiError::Unsupported( + "Earliest, pending, safe and finalized are not supported for traceChain end block", + )) + .await; return; } }; diff --git a/crates/evm/Cargo.toml b/crates/evm/Cargo.toml index e384051b4..d0c41eb7a 100644 --- a/crates/evm/Cargo.toml +++ b/crates/evm/Cargo.toml @@ -59,7 +59,7 @@ revm = { workspace = true, features = ["optional_block_gas_limit", "optional_eip sov-modules-api = { path = "../sovereign-sdk/module-system/sov-modules-api", features = ["macros"] } sov-prover-storage-manager = { path = "../sovereign-sdk/full-node/sov-prover-storage-manager", features = ["test-utils"] } sov-rollup-interface = { path = "../sovereign-sdk/rollup-interface", features = ["testing"] } -sov-stf-runner = { path = "../sovereign-sdk/full-node/sov-stf-runner" } +sov-stf-runner = { path = "../sovereign-sdk/full-node/sov-stf-runner", features = ["native"] } tempfile = { workspace = true } tracing-subscriber = { workspace = true } walkdir = "2.3.3" diff --git a/crates/evm/src/query.rs b/crates/evm/src/query.rs index e64db104c..038c1b88a 100644 --- a/crates/evm/src/query.rs +++ b/crates/evm/src/query.rs @@ -5,6 +5,7 @@ use alloy_consensus::Eip658Value; use alloy_eips::eip2930::AccessListWithGasUsed; use alloy_primitives::Uint; use alloy_rlp::Encodable; +use citrea_primitives::basefee::calculate_next_block_base_fee; use jsonrpsee::core::RpcResult; use reth_primitives::TxKind::{Call, Create}; use reth_primitives::{ @@ -39,7 +40,6 @@ use crate::evm::DbAccount; use crate::handler::{diff_size_send_eth_eoa, TxInfo}; use crate::rpc_helpers::*; use crate::{BloomFilter, Evm, EvmChainConfig, FilterBlockOption, FilterError}; - /// Gas per transaction not creating a contract. pub const MIN_TRANSACTION_GAS: u64 = 21_000u64; @@ -148,7 +148,6 @@ impl Evm { Some(sealed_block) => sealed_block, None => return Ok(None), // if block doesn't exist return null }; - // Build rpc header response let mut header = from_primitive_with_hash(sealed_block.header.clone()); header.total_difficulty = Some(header.difficulty); @@ -511,11 +510,15 @@ impl Evm { }; let (block_env, mut cfg_env) = { - let block = self - .get_sealed_block_by_number(Some(block_number), working_set)? - .ok_or(EthApiError::UnknownBlockNumber)?; - let block_env = BlockEnv::from(&block); - + let block_env = match block_number { + BlockNumberOrTag::Pending => get_pending_block_env(self, working_set), + _ => { + let block = self + .get_sealed_block_by_number(Some(block_number), working_set)? + .ok_or(EthApiError::UnknownBlockNumber)?; + BlockEnv::from(&block) + } + }; // Set evm state to block if needed match block_number { BlockNumberOrTag::Pending | BlockNumberOrTag::Latest => {} @@ -581,11 +584,22 @@ impl Evm { let mut request = request.clone(); let (l1_fee_rate, block_env, mut cfg_env) = { - let block = self - .get_sealed_block_by_number(block_number, working_set)? - .ok_or(EthApiError::UnknownBlockNumber)?; - let block_env = BlockEnv::from(&block); - + let (l1_fee_rate, block_env) = match block_number { + Some(BlockNumberOrTag::Pending) => { + let l1_fee_rate = self + .blocks + .last(&mut working_set.accessory_state()) + .expect("Head block must be set") + .l1_fee_rate; + (l1_fee_rate, get_pending_block_env(self, working_set)) + } + _ => { + let block = self + .get_sealed_block_by_number(block_number, working_set)? + .ok_or(EthApiError::UnknownBlockNumber)?; + (block.l1_fee_rate, BlockEnv::from(&block)) + } + }; match block_number { None | Some(BlockNumberOrTag::Pending | BlockNumberOrTag::Latest) => {} _ => set_state_to_end_of_evm_block(block_env.number, working_set), @@ -597,7 +611,7 @@ impl Evm { .expect("EVM chain config should be set"); let cfg_env = get_cfg_env(&block_env, cfg); - (block.l1_fee_rate, block_env, cfg_env) + (l1_fee_rate, block_env, cfg_env) }; // we want to disable this in eth_createAccessList, since this is common practice used by @@ -673,23 +687,29 @@ impl Evm { working_set: &mut WorkingSet, ) -> RpcResult { let (l1_fee_rate, block_env, cfg_env) = { - let block = self - .get_sealed_block_by_number(block_number, working_set)? - .ok_or(EthApiError::UnknownBlockNumber)?; - let block_env = BlockEnv::from(&block); - - match block_number { - None | Some(BlockNumberOrTag::Pending | BlockNumberOrTag::Latest) => {} - _ => set_state_to_end_of_evm_block(block_env.number, working_set), + let (l1_fee_rate, block_env) = match block_number { + Some(BlockNumberOrTag::Pending) => { + let l1_fee_rate = self + .blocks + .last(&mut working_set.accessory_state()) + .expect("Head block must be set") + .l1_fee_rate; + (l1_fee_rate, get_pending_block_env(self, working_set)) + } + _ => { + let block = self + .get_sealed_block_by_number(block_number, working_set)? + .ok_or(EthApiError::UnknownBlockNumber)?; + (block.l1_fee_rate, BlockEnv::from(&block)) + } }; - let cfg = self .cfg .get(working_set) .expect("EVM chain config should be set"); let cfg_env = get_cfg_env(&block_env, cfg); - (block.l1_fee_rate, block_env, cfg_env) + (l1_fee_rate, block_env, cfg_env) }; self.estimate_gas_with_env(request, l1_fee_rate, block_env, cfg_env, working_set) @@ -1364,13 +1384,15 @@ impl Evm { block_id: &BlockNumberOrTag, working_set: &mut WorkingSet, ) -> Result { + let latest_block_number = self + .blocks + .last(&mut working_set.accessory_state()) + .map(|block| block.header.number) + .expect("Head block must be set"); match block_id { BlockNumberOrTag::Earliest => Ok(0), - BlockNumberOrTag::Latest => Ok(self - .blocks - .last(&mut working_set.accessory_state()) - .map(|block| block.header.number) - .expect("Head block must be set")), + BlockNumberOrTag::Latest => Ok(latest_block_number), + BlockNumberOrTag::Pending => Err(EthApiError::UnknownBlockNumber), BlockNumberOrTag::Number(block_number) => { if *block_number < self.blocks.len(&mut working_set.accessory_state()) as u64 { Ok(*block_number) @@ -1379,7 +1401,7 @@ impl Evm { } } _ => Err(EthApiError::InvalidParams( - "Please provide a number or earliest/latest tag".to_string(), + "Please provide a number or earliest/latest/pending tag".to_string(), )), } } @@ -1673,3 +1695,43 @@ fn set_state_to_end_of_evm_block( // so every block is offset by 1 working_set.set_archival_version(block_number + 1); } + +/// Creates the next blocks `BlockEnv` based on the latest block +/// Also updates `Evm::latest_block_hashes` with the new block hash +fn get_pending_block_env( + evm: &Evm, + working_set: &mut WorkingSet, +) -> BlockEnv { + let latest_block = evm + .blocks + .last(&mut working_set.accessory_state()) + .expect("Head block must be set"); + + evm.latest_block_hashes.set( + &U256::from(latest_block.header.number), + &latest_block.header.hash(), + working_set, + ); + + let cfg = evm + .cfg + .get(working_set) + .expect("EVM chain config should be set"); + + let mut block_env = BlockEnv::from(&latest_block); + block_env.number += 1; + block_env.basefee = calculate_next_block_base_fee( + latest_block.header.gas_used as u128, + latest_block.header.gas_limit as u128, + latest_block.header.base_fee_per_gas, + cfg.base_fee_params, + ) + .unwrap_or_default(); + + if block_env.number > 256 { + evm.latest_block_hashes + .remove(&U256::from(block_env.number - 257), working_set); + } + + block_env +} diff --git a/crates/evm/src/tests/call_tests.rs b/crates/evm/src/tests/call_tests.rs index 19693572a..0e87bfabd 100644 --- a/crates/evm/src/tests/call_tests.rs +++ b/crates/evm/src/tests/call_tests.rs @@ -1,5 +1,6 @@ use std::str::FromStr; +use alloy_eips::BlockId; use reth_primitives::constants::ETHEREUM_BLOCK_GAS_LIMIT; use reth_primitives::{address, b256, Address, BlockNumberOrTag, Bytes, Log, LogData, TxKind, U64}; use reth_rpc_types::request::{TransactionInput, TransactionRequest}; @@ -786,10 +787,7 @@ fn test_block_hash_in_evm() { for i in 0..=1000 { request.input.input = Some(BlockHashContract::default().get_block_hash(i).into()); let resp = evm.get_call(request.clone(), None, None, None, &mut working_set); - if !(260..=515).contains(&i) { - // Should be 0, there is more than 256 blocks between the last block and the block number - assert_eq!(resp.unwrap().to_vec(), vec![0u8; 32]); - } else { + if (260..=515).contains(&i) { // Should be equal to the hash in accessory state let block = evm .blocks @@ -798,8 +796,40 @@ fn test_block_hash_in_evm() { resp.unwrap().to_vec(), block.unwrap().header.hash().to_vec() ); + } else { + // Should be 0, there is more than 256 blocks between the last block and the block number + assert_eq!(resp.unwrap().to_vec(), vec![0u8; 32]); } } + + // last produced block is 516, eth_call with pending should return latest block's hash + let latest_block = evm.blocks.get(516, &mut working_set.accessory_state()); + request.input.input = Some(BlockHashContract::default().get_block_hash(516).into()); + + let resp = evm.get_call( + request.clone(), + Some(BlockId::pending()), + None, + None, + &mut working_set, + ); + + assert_eq!( + resp.unwrap().to_vec(), + latest_block.unwrap().header.hash().to_vec() + ); + + // but not 260's hash + request.input.input = Some(BlockHashContract::default().get_block_hash(260).into()); + let resp = evm.get_call( + request.clone(), + Some(BlockId::pending()), + None, + None, + &mut working_set, + ); + + assert_eq!(resp.unwrap().to_vec(), vec![0u8; 32]); } #[test] diff --git a/crates/evm/src/tests/queries/estimate_gas_tests.rs b/crates/evm/src/tests/queries/estimate_gas_tests.rs index eaf062bed..e461d1bdd 100644 --- a/crates/evm/src/tests/queries/estimate_gas_tests.rs +++ b/crates/evm/src/tests/queries/estimate_gas_tests.rs @@ -20,7 +20,7 @@ use crate::{EstimatedDiffSize, Evm}; type C = DefaultContext; #[test] -fn payable_contract_value_test() { +fn test_payable_contract_value() { let (evm, mut working_set, signer) = init_evm_single_block(); let tx_req = TransactionRequest { @@ -338,7 +338,7 @@ fn test_access_list() { } #[test] -fn estimate_gas_with_varied_inputs_test() { +fn test_estimate_gas_with_varied_inputs() { let (evm, mut working_set, signer, _) = init_evm(); let simple_call_data = 0; @@ -363,6 +363,63 @@ fn estimate_gas_with_varied_inputs_test() { ); } +#[test] +fn test_pending_env() { + let (evm, mut working_set, signer) = init_evm_single_block(); + + let tx_req = TransactionRequest { + from: Some(signer.address()), + to: Some(TxKind::Call(address!( + "819c5497b157177315e1204f52e588b393771719" + ))), // Address of the payable contract. + gas: Some(100000), + gas_price: Some(100000000), + max_fee_per_gas: None, + max_priority_fee_per_gas: None, + value: Some(U256::from(3100000)), + input: TransactionInput { + input: None, + data: None, + }, + nonce: Some(1u64), + chain_id: Some(1u64), + access_list: None, + max_fee_per_blob_gas: None, + blob_versioned_hashes: None, + transaction_type: None, + sidecar: None, + }; + + let result = evm + .eth_estimate_gas( + tx_req.clone(), + Some(BlockNumberOrTag::Latest), + &mut working_set, + ) + .unwrap(); + + let result_pending = evm.eth_estimate_gas( + tx_req.clone(), + Some(BlockNumberOrTag::Pending), + &mut working_set, + ); + assert_eq!(result_pending.unwrap(), result); + + let result = evm + .create_access_list(tx_req.clone(), None, &mut working_set) + .unwrap(); + + let result_pending = evm + .create_access_list( + tx_req.clone(), + Some(BlockNumberOrTag::Pending), + &mut working_set, + ) + .unwrap(); + + assert_eq!(result_pending, result); +} + fn test_estimate_gas_with_input( evm: &Evm, working_set: &mut WorkingSet, From 1f5782fd946ee68029afc9599f7f9ffbcec01fd2 Mon Sep 17 00:00:00 2001 From: Rakan Al-Huneiti Date: Wed, 9 Oct 2024 17:59:07 +0300 Subject: [PATCH 25/28] Implement state and block overrides (#1270) * Implement state and block overrides * Update comment * apply_block_overrides func * apply_state_overrides func * Use a single func for replacing account storage * Clippy * Address feedback Use Reth's BlockOverrides with saturating_to to convert back to u64 * Test for block overrides * Test for state overrides * Comment tests * Create a fresh working set * Remove box * remove alloy-serde --------- Co-authored-by: eyusufatik --- crates/evm/src/evm/db.rs | 54 +++++-- crates/evm/src/query.rs | 18 ++- crates/evm/src/rpc_helpers/mod.rs | 117 ++++++++++++++- crates/evm/src/tests/call_tests.rs | 134 ++++++++++++++++++ crates/evm/src/tests/queries/basic_queries.rs | 16 +-- .../src/tests/queries/estimate_gas_tests.rs | 4 +- .../evm/src/tests/queries/evm_call_tests.rs | 104 +++++++++++++- crates/evm/src/tests/queries/log_tests.rs | 2 +- crates/evm/src/tests/queries/mod.rs | 12 +- 9 files changed, 422 insertions(+), 39 deletions(-) diff --git a/crates/evm/src/evm/db.rs b/crates/evm/src/evm/db.rs index d29c52eee..370c30cf2 100644 --- a/crates/evm/src/evm/db.rs +++ b/crates/evm/src/evm/db.rs @@ -1,3 +1,6 @@ +#[cfg(feature = "native")] +use std::collections::HashMap; + use reth_primitives::{Address, B256}; use revm::primitives::{AccountInfo as ReVmAccountInfo, Bytecode, U256}; use revm::Database; @@ -6,6 +9,23 @@ use sov_state::codec::BcsCodec; use super::{AccountInfo, DbAccount}; +// infallible +#[derive(Debug, Clone, Eq, PartialEq)] +pub enum DBError {} + +impl std::fmt::Display for DBError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "EVM Infallible DBError") + } +} + +// impl stdError for dberror +impl std::error::Error for DBError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + None + } +} + pub(crate) struct EvmDb<'a, C: sov_modules_api::Context> { pub(crate) accounts: sov_modules_api::StateMap, pub(crate) code: sov_modules_api::StateMap, @@ -27,22 +47,32 @@ impl<'a, C: sov_modules_api::Context> EvmDb<'a, C> { working_set, } } -} -// infallible -#[derive(Debug, Clone, Eq, PartialEq)] -pub enum DBError {} + #[cfg(feature = "native")] + pub(crate) fn override_block_hash(&mut self, number: u64, hash: B256) { + self.last_block_hashes + .set(&U256::from(number), &hash, self.working_set); + } -impl std::fmt::Display for DBError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "EVM Infallible DBError") + #[cfg(feature = "native")] + pub(crate) fn override_account(&mut self, account: &Address, info: AccountInfo) { + self.accounts.set(account, &info, self.working_set); } -} -// impl stdError for dberror -impl std::error::Error for DBError { - fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { - None + #[cfg(feature = "native")] + pub(crate) fn override_set_account_storage( + &mut self, + account: &Address, + state_diff: HashMap, + ) { + let db_account = DbAccount::new(*account); + for (slot, value) in state_diff { + db_account.storage.set( + &U256::from_be_bytes(slot.0), + &U256::from_be_bytes(value.0), + self.working_set, + ); + } } } diff --git a/crates/evm/src/query.rs b/crates/evm/src/query.rs index 038c1b88a..767f17d77 100644 --- a/crates/evm/src/query.rs +++ b/crates/evm/src/query.rs @@ -13,9 +13,10 @@ use reth_primitives::{ }; use reth_provider::ProviderError; use reth_rpc_eth_types::error::{EthApiError, EthResult, RevertError, RpcInvalidTransactionError}; +use reth_rpc_types::state::StateOverride; use reth_rpc_types::trace::geth::{GethDebugTracingOptions, GethTrace}; use reth_rpc_types::{ - AnyReceiptEnvelope, AnyTransactionReceipt, Log, OtherFields, ReceiptWithBloom, + AnyReceiptEnvelope, AnyTransactionReceipt, BlockOverrides, Log, OtherFields, ReceiptWithBloom, TransactionReceipt, }; use reth_rpc_types_compat::block::from_primitive_with_hash; @@ -494,8 +495,8 @@ impl Evm { &self, request: reth_rpc_types::TransactionRequest, block_id: Option, - _state_overrides: Option, - _block_overrides: Option>, + state_overrides: Option, + block_overrides: Option, working_set: &mut WorkingSet, ) -> RpcResult { let block_number = match block_id { @@ -509,7 +510,7 @@ impl Evm { None => BlockNumberOrTag::Latest, }; - let (block_env, mut cfg_env) = { + let (mut block_env, mut cfg_env) = { let block_env = match block_number { BlockNumberOrTag::Pending => get_pending_block_env(self, working_set), _ => { @@ -519,6 +520,7 @@ impl Evm { BlockEnv::from(&block) } }; + // Set evm state to block if needed match block_number { BlockNumberOrTag::Pending | BlockNumberOrTag::Latest => {} @@ -536,6 +538,14 @@ impl Evm { let mut evm_db = self.get_db(working_set); + if let Some(mut block_overrides) = block_overrides { + apply_block_overrides(&mut block_env, &mut block_overrides, &mut evm_db); + } + + if let Some(state_overrides) = state_overrides { + apply_state_overrides(state_overrides, &mut evm_db)?; + } + let cap_to_balance = evm_db .basic(request.from.unwrap_or_default()) .map_err(EthApiError::from)? diff --git a/crates/evm/src/rpc_helpers/mod.rs b/crates/evm/src/rpc_helpers/mod.rs index bf905b40b..2019e5762 100644 --- a/crates/evm/src/rpc_helpers/mod.rs +++ b/crates/evm/src/rpc_helpers/mod.rs @@ -1,9 +1,120 @@ +use std::collections::HashMap; + +pub use filter::*; +pub use log_utils::*; +pub use responses::*; +use reth_primitives::{keccak256, Address}; +use reth_rpc_eth_types::{EthApiError, EthResult}; +use reth_rpc_types::state::AccountOverride; +use reth_rpc_types::BlockOverrides; +use revm::Database; + mod filter; mod log_utils; mod responses; mod tracing_utils; -pub use filter::*; -pub use log_utils::*; -pub use responses::*; pub(crate) use tracing_utils::*; + +use crate::db::EvmDb; +#[cfg(feature = "native")] +use crate::primitive_types::BlockEnv; + +#[cfg(feature = "native")] +/// Applies all instances [`AccountOverride`] to the [`EvmDb`]. +pub(crate) fn apply_state_overrides( + state_overrides: HashMap, + db: &mut EvmDb, +) -> EthResult<()> { + for (address, account_overrides) in state_overrides { + apply_account_override(address, account_overrides, db)?; + } + + Ok(()) +} + +#[cfg(feature = "native")] +/// Applies a single [`AccountOverride`] to the [`EvmDb`]. +pub(crate) fn apply_account_override( + account: Address, + account_override: AccountOverride, + db: &mut EvmDb, +) -> EthResult<()> { + // we need to fetch the account via the `DatabaseRef` to not update the state of the account, + // which is modified via `Database::basic_ref` + let mut account_info = db.basic(account)?.unwrap_or_default(); + + if let Some(nonce) = account_override.nonce { + account_info.nonce = nonce; + } + if let Some(code) = account_override.code { + account_info.code_hash = keccak256(code); + } + if let Some(balance) = account_override.balance { + account_info.balance = balance; + } + + db.override_account(&account, account_info.into()); + + // We ensure that not both state and state_diff are set. + // If state is set, we must mark the account as "NewlyCreated", so that the old storage + // isn't read from + match (account_override.state, account_override.state_diff) { + (Some(_), Some(_)) => return Err(EthApiError::BothStateAndStateDiffInOverride(account)), + (None, None) => { + // nothing to do + } + (Some(new_account_state), None) => { + db.override_set_account_storage(&account, new_account_state); + } + (None, Some(account_state_diff)) => { + db.override_set_account_storage(&account, account_state_diff); + } + }; + + Ok(()) +} + +#[cfg(feature = "native")] +/// Applies all instances of [`BlockOverride`] to the [`EvmDb`]. +pub(crate) fn apply_block_overrides( + block_env: &mut BlockEnv, + block_overrides: &mut BlockOverrides, + db: &mut EvmDb, +) { + if let Some(block_hashes) = block_overrides.block_hash.take() { + // override block hashes + for (num, hash) in block_hashes { + db.override_block_hash(num, hash); + } + } + + let BlockOverrides { + number, + time, + gas_limit, + coinbase, + random, + base_fee, + block_hash: _, + difficulty: _, + } = *block_overrides; + if let Some(number) = number { + block_env.number = number.saturating_to(); + } + if let Some(time) = time { + block_env.timestamp = time; + } + if let Some(gas_limit) = gas_limit { + block_env.gas_limit = gas_limit; + } + if let Some(coinbase) = coinbase { + block_env.coinbase = coinbase; + } + if let Some(random) = random { + block_env.prevrandao = random; + } + if let Some(base_fee) = base_fee { + block_env.basefee = base_fee.saturating_to(); + } +} diff --git a/crates/evm/src/tests/call_tests.rs b/crates/evm/src/tests/call_tests.rs index 0e87bfabd..a87680d3e 100644 --- a/crates/evm/src/tests/call_tests.rs +++ b/crates/evm/src/tests/call_tests.rs @@ -1,9 +1,11 @@ +use std::collections::BTreeMap; use std::str::FromStr; use alloy_eips::BlockId; use reth_primitives::constants::ETHEREUM_BLOCK_GAS_LIMIT; use reth_primitives::{address, b256, Address, BlockNumberOrTag, Bytes, Log, LogData, TxKind, U64}; use reth_rpc_types::request::{TransactionInput, TransactionRequest}; +use reth_rpc_types::BlockOverrides; use revm::primitives::{hex, SpecId, KECCAK_EMPTY, U256}; use sov_modules_api::default_context::DefaultContext; use sov_modules_api::hooks::HookSoftConfirmationInfo; @@ -1508,3 +1510,135 @@ fn test_l1_fee_halt() { U256::from(447 + 96 + 2 * L1_FEE_OVERHEAD as u64) ); } + +#[test] +fn test_call_with_block_overrides() { + let (config, dev_signer, contract_addr) = + get_evm_config(U256::from_str("100000000000000000000").unwrap(), None); + + let (mut evm, mut working_set) = get_evm(&config); + let l1_fee_rate = 0; + let mut l2_height = 2; + + let soft_confirmation_info = HookSoftConfirmationInfo { + l2_height, + da_slot_hash: [5u8; 32], + da_slot_height: 1, + da_slot_txs_commitment: [42u8; 32], + pre_state_root: [10u8; 32].to_vec(), + current_spec: SovSpecId::Genesis, + pub_key: vec![], + deposit_data: vec![], + l1_fee_rate, + timestamp: 0, + }; + + // Deploy block hashes contract + let sender_address = generate_address::("sender"); + evm.begin_soft_confirmation_hook(&soft_confirmation_info, &mut working_set); + { + let sequencer_address = generate_address::("sequencer"); + let context = C::new( + sender_address, + sequencer_address, + l2_height, + SovSpecId::Genesis, + l1_fee_rate, + ); + + let deploy_message = create_contract_message(&dev_signer, 0, BlockHashContract::default()); + + evm.call( + CallMessage { + txs: vec![deploy_message], + }, + &context, + &mut working_set, + ) + .unwrap(); + } + evm.end_soft_confirmation_hook(&soft_confirmation_info, &mut working_set); + evm.finalize_hook(&[99u8; 32].into(), &mut working_set.accessory_state()); + + // Create empty EVM blocks + for _i in 0..10 { + let l1_fee_rate = 0; + let soft_confirmation_info = HookSoftConfirmationInfo { + l2_height, + da_slot_hash: [5u8; 32], + da_slot_height: 1, + da_slot_txs_commitment: [42u8; 32], + pre_state_root: [99u8; 32].to_vec(), + current_spec: SovSpecId::Genesis, + pub_key: vec![], + deposit_data: vec![], + l1_fee_rate, + timestamp: 0, + }; + evm.begin_soft_confirmation_hook(&soft_confirmation_info, &mut working_set); + evm.end_soft_confirmation_hook(&soft_confirmation_info, &mut working_set); + evm.finalize_hook(&[99u8; 32].into(), &mut working_set.accessory_state()); + + l2_height += 1; + } + + // Construct block override with custom hashes + let mut block_hashes = BTreeMap::new(); + block_hashes.insert(1, [1; 32].into()); + block_hashes.insert(2, [2; 32].into()); + + // Call with block overrides and check that the hash for 1st block is what we want + let call_result = evm + .get_call( + TransactionRequest { + from: None, + to: Some(TxKind::Call(contract_addr)), + input: TransactionInput::new(BlockHashContract::default().get_block_hash(1).into()), + ..Default::default() + }, + None, + None, + Some(BlockOverrides { + number: None, + difficulty: None, + time: None, + gas_limit: None, + coinbase: None, + random: None, + base_fee: None, + block_hash: Some(block_hashes.clone()), + }), + &mut working_set, + ) + .unwrap(); + + let expected_hash = Bytes::from_iter([1; 32]); + assert_eq!(call_result, expected_hash); + + // Call with block overrides and check that the hash for 2nd block is what we want + let call_result = evm + .get_call( + TransactionRequest { + from: None, + to: Some(TxKind::Call(contract_addr)), + input: TransactionInput::new(BlockHashContract::default().get_block_hash(2).into()), + ..Default::default() + }, + None, + None, + Some(BlockOverrides { + number: None, + difficulty: None, + time: None, + gas_limit: None, + coinbase: None, + random: None, + base_fee: None, + block_hash: Some(block_hashes), + }), + &mut working_set, + ) + .unwrap(); + let expected_hash = Bytes::from_iter([2; 32]); + assert_eq!(call_result, expected_hash); +} diff --git a/crates/evm/src/tests/queries/basic_queries.rs b/crates/evm/src/tests/queries/basic_queries.rs index f8c7487b5..3ee5886c9 100644 --- a/crates/evm/src/tests/queries/basic_queries.rs +++ b/crates/evm/src/tests/queries/basic_queries.rs @@ -13,7 +13,7 @@ use crate::tests::queries::init_evm; #[test] fn get_block_by_hash_test() { // make a block - let (evm, mut working_set, _, _) = init_evm(); + let (evm, mut working_set, _, _, _) = init_evm(); let result = evm.get_block_by_hash([5u8; 32].into(), Some(false), &mut working_set); @@ -34,7 +34,7 @@ fn get_block_by_hash_test() { #[test] fn get_block_by_number_test() { // make a block - let (evm, mut working_set, _, _) = init_evm(); + let (evm, mut working_set, _, _, _) = init_evm(); let result = evm.get_block_by_number( Some(BlockNumberOrTag::Number(1000)), @@ -60,7 +60,7 @@ fn get_block_by_number_test() { #[test] fn get_block_receipts_test() { // make a block - let (evm, mut working_set, _, _) = init_evm(); + let (evm, mut working_set, _, _, _) = init_evm(); let result = evm.get_block_receipts( BlockId::Number(BlockNumberOrTag::Number(1000)), @@ -91,7 +91,7 @@ fn get_block_receipts_test() { #[test] fn get_transaction_by_block_hash_and_index_test() { - let (evm, mut working_set, _, _) = init_evm(); + let (evm, mut working_set, _, _, _) = init_evm(); let result = evm.get_transaction_by_block_hash_and_index( [0u8; 32].into(), @@ -136,7 +136,7 @@ fn get_transaction_by_block_hash_and_index_test() { #[test] fn get_transaction_by_block_number_and_index_test() { - let (evm, mut working_set, _, _) = init_evm(); + let (evm, mut working_set, _, _, _) = init_evm(); let result = evm.get_transaction_by_block_number_and_index( BlockNumberOrTag::Number(100), @@ -186,7 +186,7 @@ fn get_transaction_by_block_number_and_index_test() { #[test] fn get_block_transaction_count_by_hash_test() { - let (evm, mut working_set, _, _) = init_evm(); + let (evm, mut working_set, _, _, _) = init_evm(); let result = evm.eth_get_block_transaction_count_by_hash(B256::from([0u8; 32]), &mut working_set); @@ -231,7 +231,7 @@ fn get_block_transaction_count_by_hash_test() { #[test] fn get_block_transaction_count_by_number_test() { - let (evm, mut working_set, _, _) = init_evm(); + let (evm, mut working_set, _, _, _) = init_evm(); let result = evm .eth_get_block_transaction_count_by_number(BlockNumberOrTag::Number(5), &mut working_set); @@ -253,7 +253,7 @@ fn get_block_transaction_count_by_number_test() { #[test] fn call_test() { - let (evm, mut working_set, signer, _) = init_evm(); + let (evm, mut working_set, _, signer, _) = init_evm(); let fail_result = evm.get_call( TransactionRequest { diff --git a/crates/evm/src/tests/queries/estimate_gas_tests.rs b/crates/evm/src/tests/queries/estimate_gas_tests.rs index e461d1bdd..157053fff 100644 --- a/crates/evm/src/tests/queries/estimate_gas_tests.rs +++ b/crates/evm/src/tests/queries/estimate_gas_tests.rs @@ -338,8 +338,8 @@ fn test_access_list() { } #[test] -fn test_estimate_gas_with_varied_inputs() { - let (evm, mut working_set, signer, _) = init_evm(); +fn estimate_gas_with_varied_inputs_test() { + let (evm, mut working_set, _, signer, _) = init_evm(); let simple_call_data = 0; let simple_result = diff --git a/crates/evm/src/tests/queries/evm_call_tests.rs b/crates/evm/src/tests/queries/evm_call_tests.rs index 167aa92a4..8acd761d9 100644 --- a/crates/evm/src/tests/queries/evm_call_tests.rs +++ b/crates/evm/src/tests/queries/evm_call_tests.rs @@ -1,9 +1,11 @@ +use std::collections::HashMap; use std::str::FromStr; use jsonrpsee::core::RpcResult; use reth_primitives::{address, Address, BlockNumberOrTag, Bytes, TxKind}; use reth_rpc_eth_types::RpcInvalidTransactionError; use reth_rpc_types::request::{TransactionInput, TransactionRequest}; +use reth_rpc_types::state::AccountOverride; use reth_rpc_types::BlockId; use revm::primitives::U256; use sov_modules_api::hooks::HookSoftConfirmationInfo; @@ -18,7 +20,7 @@ use crate::Evm; #[test] fn call_contract_without_value() { - let (evm, mut working_set, signer, _) = init_evm(); + let (evm, mut working_set, _, signer, _) = init_evm(); let contract = SimpleStorageContract::default(); let contract_address = Address::from_str("0xeeb03d20dae810f52111b853b31c8be6f30f4cd3").unwrap(); @@ -66,7 +68,7 @@ fn call_contract_without_value() { #[test] fn test_state_change() { - let (mut evm, mut working_set, signer, l2_height) = init_evm(); + let (mut evm, mut working_set, _, signer, l2_height) = init_evm(); let balance_1 = evm.get_balance(signer.address(), None, &mut working_set); @@ -112,7 +114,7 @@ fn test_state_change() { #[test] fn call_contract_with_value_transfer() { - let (evm, mut working_set, signer, _) = init_evm(); + let (evm, mut working_set, _, signer, _) = init_evm(); let contract = SimpleStorageContract::default(); let contract_address = Address::from_str("0xeeb03d20dae810f52111b853b31c8be6f30f4cd3").unwrap(); @@ -138,7 +140,7 @@ fn call_contract_with_value_transfer() { #[test] fn call_contract_with_invalid_nonce() { - let (evm, mut working_set, signer, _) = init_evm(); + let (evm, mut working_set, _, signer, _) = init_evm(); let contract = SimpleStorageContract::default(); let contract_address = Address::from_str("0xeeb03d20dae810f52111b853b31c8be6f30f4cd3").unwrap(); @@ -188,7 +190,7 @@ fn call_contract_with_invalid_nonce() { #[test] fn call_to_nonexistent_contract() { - let (evm, mut working_set, signer, _) = init_evm(); + let (evm, mut working_set, _, signer, _) = init_evm(); let nonexistent_contract_address = Address::from_str("0x000000000000000000000000000000000000dead").unwrap(); @@ -216,7 +218,7 @@ fn call_to_nonexistent_contract() { #[test] fn call_with_high_gas_price() { - let (evm, mut working_set, signer, _) = init_evm(); + let (evm, mut working_set, _, signer, _) = init_evm(); let contract = SimpleStorageContract::default(); let contract_address = Address::from_str("0xeeb03d20dae810f52111b853b31c8be6f30f4cd3").unwrap(); @@ -246,7 +248,7 @@ fn call_with_high_gas_price() { #[test] fn test_eip1559_fields_call() { - let (evm, mut working_set, signer, _) = init_evm(); + let (evm, mut working_set, _, signer, _) = init_evm(); let default_result = eth_call_eip1559( &evm, @@ -506,3 +508,91 @@ fn gas_price_call_test() { assert!(result_high_fees.is_ok()); working_set.unset_archival_version(); } + +#[test] +fn test_call_with_state_overrides() { + let (evm, mut working_set, prover_storage, signer, _) = init_evm(); + + let contract = SimpleStorageContract::default(); + let contract_address = Address::from_str("0xeeb03d20dae810f52111b853b31c8be6f30f4cd3").unwrap(); + + // Get value of contract before state override + let call_result_without_state_override = evm + .get_call( + TransactionRequest { + from: Some(signer.address()), + to: Some(TxKind::Call(contract_address)), + input: TransactionInput::new(contract.get_call_data().into()), + ..Default::default() + }, + None, + None, + None, + &mut working_set, + ) + .unwrap(); + + assert_eq!( + call_result_without_state_override, + U256::from(478).to_be_bytes_vec() + ); + + // Override the state and check returned value + let mut state = HashMap::new(); + state.insert(U256::from(0).into(), U256::from(15).into()); + + let mut state_override = HashMap::new(); + state_override.insert( + contract_address, + AccountOverride { + balance: None, + nonce: None, + code: None, + state: Some(state), + state_diff: None, + }, + ); + let call_result_with_state_override = evm + .get_call( + TransactionRequest { + from: Some(signer.address()), + to: Some(TxKind::Call(contract_address)), + input: TransactionInput::new(contract.get_call_data().into()), + ..Default::default() + }, + None, + Some(state_override), + None, + &mut working_set, + ) + .unwrap(); + + assert_eq!( + call_result_with_state_override, + U256::from(15).to_be_bytes_vec() + ); + + // Start with a fresh working set, because the previous one was for a separate RPC call. + let mut working_set = WorkingSet::new(prover_storage); + + // Get value of contract AFTER state override, this MUST be the original value. + let call_result_without_state_override = evm + .get_call( + TransactionRequest { + from: Some(signer.address()), + to: Some(TxKind::Call(contract_address)), + input: TransactionInput::new(contract.get_call_data().into()), + ..Default::default() + }, + None, + None, + None, + &mut working_set, + ) + .unwrap(); + + assert_eq!( + call_result_without_state_override, + U256::from(478).to_be_bytes_vec() + ); +} diff --git a/crates/evm/src/tests/queries/log_tests.rs b/crates/evm/src/tests/queries/log_tests.rs index 0d7e2f696..8a80963aa 100644 --- a/crates/evm/src/tests/queries/log_tests.rs +++ b/crates/evm/src/tests/queries/log_tests.rs @@ -22,7 +22,7 @@ type C = DefaultContext; #[test] fn logs_for_filter_test() { - let (evm, mut working_set, _, _) = init_evm(); + let (evm, mut working_set, _, _, _) = init_evm(); let result = evm.eth_get_logs( Filter { diff --git a/crates/evm/src/tests/queries/mod.rs b/crates/evm/src/tests/queries/mod.rs index 9afb1bd33..f6ecb8955 100644 --- a/crates/evm/src/tests/queries/mod.rs +++ b/crates/evm/src/tests/queries/mod.rs @@ -11,7 +11,9 @@ use sov_modules_api::default_context::DefaultContext; use sov_modules_api::hooks::HookSoftConfirmationInfo; use sov_modules_api::utils::generate_address; use sov_modules_api::{Context, Module, WorkingSet}; +use sov_prover_storage_manager::SnapshotManager; use sov_rollup_interface::spec::SpecId as SovSpecId; +use sov_state::{DefaultStorageSpec, ProverStorage}; use crate::call::CallMessage; use crate::smart_contracts::{ @@ -30,7 +32,13 @@ type C = DefaultContext; /// Block 1 has 3 transactions /// Block 2 has 4 transactions /// Block 3 has 2 transactions -fn init_evm() -> (Evm, WorkingSet, TestSigner, u64) { +fn init_evm() -> ( + Evm, + WorkingSet, + ProverStorage, + TestSigner, + u64, +) { let dev_signer: TestSigner = TestSigner::new_random(); let mut config = EvmConfig { @@ -206,7 +214,7 @@ fn init_evm() -> (Evm, WorkingSet, TestSigner, u64) { let working_set: WorkingSet = WorkingSet::new(prover_storage.clone()); - (evm, working_set, dev_signer, l2_height) + (evm, working_set, prover_storage, dev_signer, l2_height) } pub fn init_evm_single_block() -> (Evm, WorkingSet, TestSigner) { From 07058a5fd0b99500f0ca43bdee4f5184e7fe339b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ahmet=20Yaz=C4=B1c=C4=B1?= <75089142+yaziciahmet@users.noreply.github.com> Date: Wed, 9 Oct 2024 23:44:43 +0200 Subject: [PATCH 26/28] Use spawn_blocking for da queue (#1311) --- crates/bitcoin-da/src/service.rs | 143 ++++++++++++++++--------------- 1 file changed, 74 insertions(+), 69 deletions(-) diff --git a/crates/bitcoin-da/src/service.rs b/crates/bitcoin-da/src/service.rs index 09224a043..8a6a99b4d 100644 --- a/crates/bitcoin-da/src/service.rs +++ b/crates/bitcoin-da/src/service.rs @@ -183,85 +183,90 @@ impl BitcoinService { self: Arc, mut rx: UnboundedReceiver>>, ) { - tokio::spawn(async move { - let mut prev_utxo = match self.get_prev_utxo().await { - Ok(Some(prev_utxo)) => Some(prev_utxo), - Ok(None) => { - info!("No pending transactions found"); - None - } - Err(e) => { - error!(?e, "Failed to get pending transactions"); - None - } - }; + // This should be spawn_blocking, since it is a CPU-bound worker. + // When spawned with tokio::spawn, it blocks other futures and + // disrupts tokio runtime. + tokio::task::spawn_blocking(|| { + tokio::runtime::Handle::current().block_on(async move { + let mut prev_utxo = match self.get_prev_utxo().await { + Ok(Some(prev_utxo)) => Some(prev_utxo), + Ok(None) => { + info!("No pending transactions found"); + None + } + Err(e) => { + error!(?e, "Failed to get pending transactions"); + None + } + }; - trace!("BitcoinDA queue is initialized. Waiting for the first request..."); - - loop { - select! { - request_opt = rx.recv() => { - if let Some(request_opt) = request_opt { - match request_opt { - Some(request) => { - trace!("A new request is received"); - let prev = prev_utxo.take(); - loop { - // Build and send tx with retries: - let fee_sat_per_vbyte = match self.get_fee_rate().await { - Ok(rate) => rate, - Err(e) => { - error!(?e, "Failed to call get_fee_rate. Retrying..."); - tokio::time::sleep(Duration::from_secs(1)).await; - continue; - } - }; - match self - .send_transaction_with_fee_rate( - prev.clone(), - request.da_data.clone(), - fee_sat_per_vbyte, - ) - .await - { - Ok(tx) => { - let tx_id = TxidWrapper(tx.id); - info!(%tx.id, "Sent tx to BitcoinDA"); - prev_utxo = Some(UTXO { - tx_id: tx.id, - vout: 0, - script_pubkey: tx.tx.output[0].script_pubkey.to_hex_string(), - address: None, - amount: tx.tx.output[0].value.to_sat(), - confirmations: 0, - spendable: true, - solvable: true, - }); - - let _ = request.notify.send(Ok(tx_id)); - } - Err(e) => { - error!(?e, "Failed to send transaction to DA layer"); - tokio::time::sleep(Duration::from_secs(1)).await; - continue; + trace!("BitcoinDA queue is initialized. Waiting for the first request..."); + + loop { + select! { + request_opt = rx.recv() => { + if let Some(request_opt) = request_opt { + match request_opt { + Some(request) => { + trace!("A new request is received"); + let prev = prev_utxo.take(); + loop { + // Build and send tx with retries: + let fee_sat_per_vbyte = match self.get_fee_rate().await { + Ok(rate) => rate, + Err(e) => { + error!(?e, "Failed to call get_fee_rate. Retrying..."); + tokio::time::sleep(Duration::from_secs(1)).await; + continue; + } + }; + match self + .send_transaction_with_fee_rate( + prev.clone(), + request.da_data.clone(), + fee_sat_per_vbyte, + ) + .await + { + Ok(tx) => { + let tx_id = TxidWrapper(tx.id); + info!(%tx.id, "Sent tx to BitcoinDA"); + prev_utxo = Some(UTXO { + tx_id: tx.id, + vout: 0, + script_pubkey: tx.tx.output[0].script_pubkey.to_hex_string(), + address: None, + amount: tx.tx.output[0].value.to_sat(), + confirmations: 0, + spendable: true, + solvable: true, + }); + + let _ = request.notify.send(Ok(tx_id)); + } + Err(e) => { + error!(?e, "Failed to send transaction to DA layer"); + tokio::time::sleep(Duration::from_secs(1)).await; + continue; + } } + break; } - break; } - } - None => { - info!("Shutdown signal received. Stopping BitcoinDA queue."); - break; + None => { + info!("Shutdown signal received. Stopping BitcoinDA queue."); + break; + } } } + }, + _ = signal::ctrl_c() => { + return; } - }, - _ = signal::ctrl_c() => { - return; } } - } + }); error!("BitcoinDA queue stopped"); }); From 10b0c636121bcb57fc83ece51d1e5a113da4b770 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ahmet=20Yaz=C4=B1c=C4=B1?= <75089142+yaziciahmet@users.noreply.github.com> Date: Wed, 9 Oct 2024 23:45:20 +0200 Subject: [PATCH 27/28] Fetch smart fee only if none (#1312) --- crates/bitcoin-da/src/service.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/crates/bitcoin-da/src/service.rs b/crates/bitcoin-da/src/service.rs index 8a6a99b4d..03286f324 100644 --- a/crates/bitcoin-da/src/service.rs +++ b/crates/bitcoin-da/src/service.rs @@ -553,11 +553,10 @@ impl BitcoinService { #[instrument(level = "trace", skip_all, ret)] pub async fn get_fee_rate_as_sat_vb(&self) -> Result { // If network is regtest or signet, mempool space is not available - let smart_fee = get_fee_rate_from_mempool_space(self.network).await?.or(self - .client - .estimate_smart_fee(1, None) - .await? - .fee_rate); + let smart_fee = match get_fee_rate_from_mempool_space(self.network).await? { + Some(fee_rate) => Some(fee_rate), + None => self.client.estimate_smart_fee(1, None).await?.fee_rate, + }; let sat_vkb = smart_fee.map_or(1000, |rate| rate.to_sat()); tracing::debug!("Fee rate: {} sat/vb", sat_vkb / 1000); From 341f2417db1f69fbd3e8b1811023add9608ae39c Mon Sep 17 00:00:00 2001 From: Esad Yusuf Atik Date: Thu, 10 Oct 2024 01:16:02 +0300 Subject: [PATCH 28/28] update run doc and changelog (#1315) --- CHANGELOG.md | 8 ++++++++ docs/run-testnet.md | 4 ++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ee21af61e..30590d4e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## v0.5.3 (2024-10-10) +- `eth_call` RPC now supports state and block overrides. ([#1270](https://github.com/chainwayxyz/citrea/pull/1270)) +- `eth_call`, `eth_estimateGas` and `eth_createAccessList` RPCs now supports "pending" block tag. ([#1303](https://github.com/chainwayxyz/citrea/pull/1303)) +- Bitcoin DA adapter uses mempool.space API for fee estimation. ([#1302](https://github.com/chainwayxyz/citrea/pull/1302)) +- New RPC for prover node: `prover_generateInput`. ([#1280](https://github.com/chainwayxyz/citrea/pull/1280)) +- Enhance `eth_estimateGas` RPC L1 fee estimatation. ([#1261](https://github.com/chainwayxyz/citrea/pull/1261)) +- Structured concurrency and graceful shutdown: fixes breaking storage on shutdown while syncing for the first time. ([#1214](https://github.com/chainwayxyz/citrea/pull/1214) and [#1216](https://github.com/chainwayxyz/citrea/pull/1216)) + ## v0.5.2 (2024-09-30) - Added config for disableing prover proving session recovery. ([#1241](https://github.com/chainwayxyz/citrea/pull/1241)) - Nodes now log each RPC request and response. ([#1236](https://github.com/chainwayxyz/citrea/pull/1236)) diff --git a/docs/run-testnet.md b/docs/run-testnet.md index 3d2f9fa84..10d37abad 100644 --- a/docs/run-testnet.md +++ b/docs/run-testnet.md @@ -117,12 +117,12 @@ Finally run this command to run your Citrea full node: Mac: ```sh -./citrea-v0.5.2-osx-arm64 --da-layer bitcoin --rollup-config-path ./rollup_config.toml --genesis-paths ./genesis +./citrea-v0.5.3-osx-arm64 --da-layer bitcoin --rollup-config-path ./rollup_config.toml --genesis-paths ./genesis ``` Linux: ```sh -./citrea-v0.5.2-linux-amd64 --da-layer bitcoin --rollup-config-path ./rollup_config.toml --genesis-paths ./genesis +./citrea-v0.5.3-linux-amd64 --da-layer bitcoin --rollup-config-path ./rollup_config.toml --genesis-paths ./genesis ``` Your full node should be serving RPC at `http://0.0.0.0:8080` now.