diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 633c37e..e2e3847 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -34,6 +34,30 @@ jobs: - run: cargo check --all-targets - run: cargo clippy -- --deny warnings + spec-tests: + name: Spec tests + runs-on: 'ubuntu-latest' + steps: + - uses: actions/checkout@v3 + - uses: actions/cache@v2 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + - name: Install rust + uses: dtolnay/rust-toolchain@master + with: + toolchain: ${{ matrix.rust || 'stable' }} + targets: ${{ matrix.target }} + - name: Downloading ethereum/tests + run: git clone https://github.com/ethereum/tests ethereum-tests + - name: Downloading EELS fixtures released at Cancun + run: curl -LO https://github.com/ethereum/execution-spec-tests/releases/download/v2.1.1/fixtures.tar.gz && tar -xzf fixtures.tar.gz + - name: Test specs (EELS and ethereum/tests) + run: cargo test --features testing + tests: name: Tests ${{ matrix.name }} needs: [style] diff --git a/.gitignore b/.gitignore index ea8c4bf..52b69b8 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ /target +ethereum-tests/ +fixtures/ \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index b6cecb1..68a520f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7663,6 +7663,7 @@ dependencies = [ "reth-basic-payload-builder", "reth-chain-state", "reth-chainspec", + "reth-cli", "reth-cli-util", "reth-consensus", "reth-db", diff --git a/Cargo.toml b/Cargo.toml index 5751dca..d4cbfc2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,6 +21,7 @@ reth-ethereum-consensus = { git = "https://github.com/paradigmxyz/reth", rev = " reth-chainspec = { git = "https://github.com/paradigmxyz/reth", rev = "8f61af0136e1a20119832925081c341ae89b93f0" } reth-chain-state = { git = "https://github.com/paradigmxyz/reth", rev = "8f61af0136e1a20119832925081c341ae89b93f0" } reth-consensus = { git = "https://github.com/paradigmxyz/reth", rev = "8f61af0136e1a20119832925081c341ae89b93f0" } +reth-cli = { git = "https://github.com/paradigmxyz/reth", rev = "8f61af0136e1a20119832925081c341ae89b93f0" } reth-cli-util = { git = "https://github.com/paradigmxyz/reth", rev = "8f61af0136e1a20119832925081c341ae89b93f0" } # reth-auto-seal-consensus = { git = "https://github.com/paradigmxyz/reth", rev = "8f61af0136e1a20119832925081c341ae89b93f0" } reth-prune-types = { git = "https://github.com/paradigmxyz/reth", rev = "8f61af0136e1a20119832925081c341ae89b93f0" } @@ -80,4 +81,5 @@ libc = "0.2" [features] default = ["jemalloc"] jemalloc = ["dep:tikv-jemallocator"] - +testing = [] +failing-tests = [] \ No newline at end of file diff --git a/scripts/chiado_genesis_alloc.json b/scripts/chiado_genesis_alloc.json index 117bb7f..bec79e3 100644 --- a/scripts/chiado_genesis_alloc.json +++ b/scripts/chiado_genesis_alloc.json @@ -61,7 +61,8 @@ }, "registrar": "0x6000000000000000000000000000000000000000" }, - "eip1559collector": "0x1559000000000000000000000000000000000000" + "eip1559collector": "0x1559000000000000000000000000000000000000", + "depositContractAddress": "0xbabe2bed00000000000000000000000000000003" }, "baseFeePerGas": "0x3b9aca00", "difficulty": "0x01", diff --git a/src/lib.rs b/src/lib.rs index 2ca4a0a..cae746d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -32,6 +32,7 @@ mod evm_config; pub mod execute; mod gnosis; mod payload_builder; +mod testing; #[derive(Debug, Clone, Default, PartialEq, Eq, clap::Args)] #[command(next_help_heading = "Gnosis")] diff --git a/src/testing/assert.rs b/src/testing/assert.rs new file mode 100644 index 0000000..4724ad3 --- /dev/null +++ b/src/testing/assert.rs @@ -0,0 +1,18 @@ +//! Various assertion helpers. + +use crate::testing::Error; +use std::fmt::Debug; + +/// A helper like `assert_eq!` that instead returns `Err(Error::Assertion)` on failure. +pub fn assert_equal(left: T, right: T, msg: &str) -> Result<(), Error> +where + T: PartialEq + Debug, +{ + if left == right { + Ok(()) + } else { + Err(Error::Assertion(format!( + "{msg}\n left `{left:?}`,\n right `{right:?}`" + ))) + } +} diff --git a/src/testing/case.rs b/src/testing/case.rs new file mode 100644 index 0000000..0535b47 --- /dev/null +++ b/src/testing/case.rs @@ -0,0 +1,42 @@ +//! Test case definitions + +use crate::testing::result::{CaseResult, Error}; +use std::{ + fmt::Debug, + path::{Path, PathBuf}, +}; + +/// A single test case, capable of loading a JSON description of itself and running it. +/// +/// See for test specs. +pub trait Case: Debug + Sync + Sized { + /// A description of the test. + fn description(&self) -> String { + "no description".to_string() + } + + /// Load the test from the given file path. + /// + /// The file can be assumed to be a valid EF test case as described on . + fn load(path: &Path) -> Result; + + /// Run the test. + fn run(&self) -> Result<(), Error>; +} + +/// A container for multiple test cases. +#[derive(Debug)] +pub struct Cases { + /// The contained test cases and the path to each test. + pub test_cases: Vec<(PathBuf, T)>, +} + +impl Cases { + /// Run the contained test cases. + pub fn run(&self) -> Vec { + self.test_cases + .iter() + .map(|(path, case)| CaseResult::new(path, case, case.run())) + .collect() + } +} diff --git a/src/testing/cases/blockchain_test.rs b/src/testing/cases/blockchain_test.rs new file mode 100644 index 0000000..8fb21c7 --- /dev/null +++ b/src/testing/cases/blockchain_test.rs @@ -0,0 +1,296 @@ +//! Test runners for `BlockchainTests` in + +use crate::{ + execute::GnosisExecutorProvider, + testing::{ + models::{BlockchainTest, ForkSpec}, + Case, Error, Suite, + }, +}; +use alloy_rlp::Decodable; +use rayon::iter::{ParallelBridge, ParallelIterator}; +use reth_chainspec::ChainSpec; +use reth_cli::chainspec::parse_genesis; +use reth_primitives::{BlockBody, SealedBlock, StaticFileSegment}; +use reth_provider::{ + providers::StaticFileWriter, test_utils::create_test_provider_factory_with_chain_spec, + DatabaseProviderFactory, HashingWriter, StaticFileProviderFactory, +}; +use reth_stages::{stages::ExecutionStage, ExecInput, Stage}; +use std::{ + collections::BTreeMap, + fs, + path::{Path, PathBuf}, + sync::Arc, +}; + +/// A handler for the blockchain test suite. +#[derive(Debug)] +pub struct BlockchainTests { + suite: String, +} + +impl BlockchainTests { + /// Create a new handler for a subset of the blockchain test suite. + pub const fn new(suite: String) -> Self { + Self { suite } + } +} + +impl Suite for BlockchainTests { + type Case = BlockchainTestCase; + + fn suite_name(&self) -> String { + self.suite.clone() + } +} + +/// An Ethereum blockchain test. +#[derive(Debug, PartialEq, Eq)] +pub struct BlockchainTestCase { + tests: BTreeMap, + skip: bool, +} + +impl Case for BlockchainTestCase { + fn load(path: &Path) -> Result { + Ok(Self { + tests: { + let s = fs::read_to_string(path).map_err(|error| Error::Io { + path: path.into(), + error, + })?; + let test = + serde_json::from_str(&s).map_err(|error| Error::CouldNotDeserialize { + path: path.into(), + error, + })?; + test + }, + skip: should_skip(path), + }) + } + + /// Runs the test cases for the Ethereum Forks test suite. + /// + /// # Errors + /// Returns an error if the test is flagged for skipping or encounters issues during execution. + fn run(&self) -> Result<(), Error> { + // If the test is marked for skipping, return a Skipped error immediately. + if self.skip { + return Err(Error::Skipped); + } + + let chainspec_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("scripts") + .join("chiado_genesis_alloc.json"); + let original_chain_spec: ChainSpec = parse_genesis(chainspec_path.to_str().unwrap()) + .unwrap() + .into(); + // let chain_spec: Arc = Arc::new(chain_spec); + + // Iterate through test cases, filtering by the network type to exclude specific forks. + self.tests + .values() + .filter(|case| { + !matches!( + case.network, + ForkSpec::ByzantiumToConstantinopleAt5 + | ForkSpec::Frontier + | ForkSpec::Homestead + | ForkSpec::Byzantium + | ForkSpec::Istanbul + | ForkSpec::Berlin + | ForkSpec::London + | ForkSpec::Constantinople + | ForkSpec::ConstantinopleFix + | ForkSpec::MergeEOF + | ForkSpec::MergeMeterInitCode + | ForkSpec::MergePush0 + | ForkSpec::Unknown + ) + }) + .par_bridge() + .try_for_each(|case| { + // Create a new test database and initialize a provider for the test case. + let mut chain_spec: ChainSpec = case.network.clone().into(); + chain_spec.genesis.config.extra_fields.insert( + String::from("eip1559collector"), + original_chain_spec + .genesis + .config + .extra_fields + .get("eip1559collector") + .unwrap() + .clone(), + ); + chain_spec.genesis.config.extra_fields.insert( + String::from("blockRewardsContract"), + original_chain_spec + .genesis + .config + .extra_fields + .get("blockRewardsContract") + .unwrap() + .clone(), + ); + chain_spec.deposit_contract = original_chain_spec.deposit_contract; + let chain_spec: Arc = Arc::new(chain_spec); + let provider = create_test_provider_factory_with_chain_spec(chain_spec.clone()); + + let provider = provider.database_provider_rw().unwrap(); + + // provider. + + // Insert initial test state into the provider. + provider.insert_historical_block( + SealedBlock::new( + case.genesis_block_header.clone().into(), + BlockBody::default(), + ) + .try_seal_with_senders() + .unwrap(), + )?; + case.pre.write_to_db(provider.tx_ref())?; + + // Initialize receipts static file with genesis + { + let static_file_provider = provider.static_file_provider(); + let mut receipts_writer = static_file_provider + .latest_writer(StaticFileSegment::Receipts) + .unwrap(); + receipts_writer.increment_block(0).unwrap(); + receipts_writer.commit_without_sync_all().unwrap(); + } + + // Decode and insert blocks, creating a chain of blocks for the test case. + let last_block = case.blocks.iter().try_fold(None, |_, block| { + let decoded = SealedBlock::decode(&mut block.rlp.as_ref())?; + provider.insert_historical_block( + decoded.clone().try_seal_with_senders().unwrap(), + )?; + Ok::, Error>(Some(decoded)) + })?; + provider + .static_file_provider() + .latest_writer(StaticFileSegment::Headers) + .unwrap() + .commit_without_sync_all() + .unwrap(); + + let gnosis_executor_provider = GnosisExecutorProvider::gnosis(chain_spec.clone()); + + // Execute the execution stage using the EVM processor factory for the test case + // network. + let result = ExecutionStage::new_with_executor(gnosis_executor_provider).execute( + &provider, + ExecInput { + target: last_block.as_ref().map(|b| b.number), + checkpoint: None, + }, + ); + if let Err(e) = result { + return Err(Error::Custom( + format!("error in execution stage {:?}", e).to_string(), + )); + } + + // Validate the post-state for the test case. + match (&case.post_state, &case.post_state_hash) { + (Some(state), None) => { + // Validate accounts in the state against the provider's database. + for (&address, account) in state { + account.assert_db(address, provider.tx_ref())?; + } + } + (None, Some(expected_state_root)) => { + // Insert state hashes into the provider based on the expected state root. + let last_block = last_block.unwrap_or_default(); + provider.insert_hashes( + 0..=last_block.number, + last_block.hash(), + *expected_state_root, + )?; + } + _ => { + return Err(Error::MissingPostState); + } + } + + // Drop the provider without committing to the database. + drop(provider); + Ok(()) + })?; + + Ok(()) + } +} + +/// Returns whether the test at the given path should be skipped. +/// +/// Some tests are edge cases that cannot happen on mainnet, while others are skipped for +/// convenience (e.g. they take a long time to run) or are temporarily disabled. +/// +/// The reason should be documented in a comment above the file name(s). +pub fn should_skip(path: &Path) -> bool { + let path_str = path.to_str().expect("Path is not valid UTF-8"); + let test_name = path.file_name().unwrap().to_str().unwrap(); + let test_folder = path + .parent() + .unwrap() + .file_name() + .unwrap() + .to_str() + .unwrap(); + + // executnig all tests for now + false && matches!( + test_name, + // funky test with `bigint 0x00` value in json :) not possible to happen on mainnet and require + // custom json parser. https://github.com/ethereum/tests/issues/971 + | "ValueOverflow.json" + | "ValueOverflowParis.json" + + // txbyte is of type 02 and we don't parse tx bytes for this test to fail. + | "typeTwoBerlin.json" + + // Test checks if nonce overflows. We are handling this correctly but we are not parsing + // exception in testsuite There are more nonce overflow tests that are in internal + // call/create, and those tests are passing and are enabled. + | "CreateTransactionHighNonce.json" + + // Test check if gas price overflows, we handle this correctly but does not match tests specific + // exception. + | "HighGasPrice.json" + | "HighGasPriceParis.json" + + // Skip test where basefee/accesslist/difficulty is present but it shouldn't be supported in + // London/Berlin/TheMerge. https://github.com/ethereum/tests/blob/5b7e1ab3ffaf026d99d20b17bb30f533a2c80c8b/GeneralStateTests/stExample/eip1559.json#L130 + // It is expected to not execute these tests. + | "accessListExample.json" + | "basefeeExample.json" + | "eip1559.json" + | "mergeTest.json" + + // These tests are passing, but they take a lot of time to execute so we are going to skip them. + | "loopExp.json" + | "Call50000_sha256.json" + | "static_Call50000_sha256.json" + | "loopMul.json" + | "CALLBlake2f_MaxRounds.json" + | "shiftCombinations.json" + ) + || matches!( + test_folder, + // These tests are passing, but they take a lot of time to execute + | "stTimeConsuming" + ) + // Ignore outdated EOF tests that haven't been updated for Cancun yet. + || path_contains(path_str, &["EIPTests", "stEOF"]) +} + +/// `str::contains` but for a path. Takes into account the OS path separator (`/` or `\`). +fn path_contains(path_str: &str, rhs: &[&str]) -> bool { + let rhs = rhs.join(std::path::MAIN_SEPARATOR_STR); + path_str.contains(&rhs) +} diff --git a/src/testing/cases/mod.rs b/src/testing/cases/mod.rs new file mode 100644 index 0000000..cfde4ac --- /dev/null +++ b/src/testing/cases/mod.rs @@ -0,0 +1,3 @@ +//! Specific test case handler implementations. + +pub mod blockchain_test; diff --git a/src/testing/mod.rs b/src/testing/mod.rs new file mode 100644 index 0000000..99aace8 --- /dev/null +++ b/src/testing/mod.rs @@ -0,0 +1,15 @@ +#![allow(dead_code)] +#![cfg(test)] +pub mod case; +pub mod result; +pub mod suite; + +pub mod assert; +pub mod cases; +pub mod models; + +pub use case::Case; +pub use result::Error; +pub use suite::Suite; + +pub mod tests; diff --git a/src/testing/models.rs b/src/testing/models.rs new file mode 100644 index 0000000..76e8863 --- /dev/null +++ b/src/testing/models.rs @@ -0,0 +1,499 @@ +//! Shared models for + +use crate::testing::{assert::assert_equal, Error}; +use alloy_consensus::Header as RethHeader; +use alloy_eips::eip4895::Withdrawals; +use alloy_primitives::{keccak256, Address, Bloom, Bytes, B256, B64, U256}; +use reth_chainspec::{ChainSpec, ChainSpecBuilder}; +use reth_db::tables; +use reth_db_api::{ + cursor::DbDupCursorRO, + transaction::{DbTx, DbTxMut}, +}; +use reth_primitives::{Account as RethAccount, Bytecode, SealedHeader, StorageEntry}; +use serde::Deserialize; +use std::{collections::BTreeMap, ops::Deref}; + +/// The definition of a blockchain test. +#[derive(Debug, PartialEq, Eq, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct BlockchainTest { + /// Genesis block header. + pub genesis_block_header: Header, + /// RLP encoded genesis block. + #[serde(rename = "genesisRLP")] + pub genesis_rlp: Option, + /// Block data. + pub blocks: Vec, + /// The expected post state. + pub post_state: Option>, + /// The expected post state merkle root. + pub post_state_hash: Option, + /// The test pre-state. + pub pre: State, + /// Hash of the best block. + pub lastblockhash: B256, + /// Network spec. + pub network: ForkSpec, + #[serde(default)] + /// Engine spec. + pub seal_engine: SealEngine, +} + +/// A block header in an Ethereum blockchain test. +#[derive(Debug, PartialEq, Eq, Clone, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +pub struct Header { + /// Bloom filter. + pub bloom: Bloom, + /// Coinbase. + pub coinbase: Address, + /// Difficulty. + pub difficulty: U256, + /// Extra data. + pub extra_data: Bytes, + /// Gas limit. + pub gas_limit: U256, + /// Gas used. + pub gas_used: U256, + /// Block Hash. + pub hash: B256, + /// Mix hash. + pub mix_hash: B256, + /// Seal nonce. + pub nonce: B64, + /// Block number. + pub number: U256, + /// Parent hash. + pub parent_hash: B256, + /// Receipt trie. + pub receipt_trie: B256, + /// State root. + pub state_root: B256, + /// Timestamp. + pub timestamp: U256, + /// Transactions trie. + pub transactions_trie: B256, + /// Uncle hash. + pub uncle_hash: B256, + /// Base fee per gas. + pub base_fee_per_gas: Option, + /// Withdrawals root. + pub withdrawals_root: Option, + /// Blob gas used. + pub blob_gas_used: Option, + /// Excess blob gas. + pub excess_blob_gas: Option, + /// Parent beacon block root. + pub parent_beacon_block_root: Option, + /// Requests root. + pub requests_root: Option, + /// Target blobs per block. + pub target_blobs_per_block: Option, +} + +impl From
for SealedHeader { + fn from(value: Header) -> Self { + let header = RethHeader { + base_fee_per_gas: value.base_fee_per_gas.map(|v| v.to::()), + beneficiary: value.coinbase, + difficulty: value.difficulty, + extra_data: value.extra_data, + gas_limit: value.gas_limit.to::(), + gas_used: value.gas_used.to::(), + mix_hash: value.mix_hash, + nonce: u64::from_be_bytes(value.nonce.0).into(), + number: value.number.to::(), + timestamp: value.timestamp.to::(), + transactions_root: value.transactions_trie, + receipts_root: value.receipt_trie, + ommers_hash: value.uncle_hash, + state_root: value.state_root, + parent_hash: value.parent_hash, + logs_bloom: value.bloom, + withdrawals_root: value.withdrawals_root, + blob_gas_used: value.blob_gas_used.map(|v| v.to::()), + excess_blob_gas: value.excess_blob_gas.map(|v| v.to::()), + parent_beacon_block_root: value.parent_beacon_block_root, + requests_hash: value.requests_root, + target_blobs_per_block: value.target_blobs_per_block.map(|v| v.to::()), + }; + Self::new(header, value.hash) + } +} + +/// A block in an Ethereum blockchain test. +#[derive(Debug, PartialEq, Eq, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +pub struct Block { + /// Block header. + pub block_header: Option
, + /// RLP encoded block bytes + pub rlp: Bytes, + /// Transactions + pub transactions: Option>, + /// Uncle/ommer headers + pub uncle_headers: Option>, + /// Transaction Sequence + pub transaction_sequence: Option>, + /// Withdrawals + pub withdrawals: Option, +} + +/// Transaction sequence in block +#[derive(Debug, PartialEq, Eq, Deserialize, Default)] +#[serde(deny_unknown_fields)] +#[serde(rename_all = "camelCase")] +pub struct TransactionSequence { + exception: String, + raw_bytes: Bytes, + valid: String, +} + +/// Ethereum blockchain test data state. +#[derive(Clone, Debug, Eq, PartialEq, Deserialize, Default)] +pub struct State(BTreeMap); + +impl State { + /// Write the state to the database. + pub fn write_to_db(&self, tx: &impl DbTxMut) -> Result<(), Error> { + for (&address, account) in &self.0 { + let hashed_address = keccak256(address); + let has_code = !account.code.is_empty(); + let code_hash = has_code.then(|| keccak256(&account.code)); + let reth_account = RethAccount { + balance: account.balance, + nonce: account.nonce.to::(), + bytecode_hash: code_hash, + }; + tx.put::(address, reth_account)?; + tx.put::(hashed_address, reth_account)?; + if let Some(code_hash) = code_hash { + tx.put::(code_hash, Bytecode::new_raw(account.code.clone()))?; + } + account + .storage + .iter() + .filter(|(_, v)| !v.is_zero()) + .try_for_each(|(k, v)| { + let storage_key = B256::from_slice(&k.to_be_bytes::<32>()); + tx.put::( + address, + StorageEntry { + key: storage_key, + value: *v, + }, + )?; + tx.put::( + hashed_address, + StorageEntry { + key: keccak256(storage_key), + value: *v, + }, + ) + })?; + } + + Ok(()) + } +} + +impl Deref for State { + type Target = BTreeMap; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +/// An account. +#[derive(Debug, PartialEq, Eq, Deserialize, Clone, Default)] +#[serde(deny_unknown_fields)] +pub struct Account { + /// Balance. + pub balance: U256, + /// Code. + pub code: Bytes, + /// Nonce. + pub nonce: U256, + /// Storage. + pub storage: BTreeMap, +} + +impl Account { + /// Check that the account matches what is in the database. + /// + /// In case of a mismatch, `Err(Error::Assertion)` is returned. + pub fn assert_db(&self, address: Address, tx: &impl DbTx) -> Result<(), Error> { + let account = tx + .get::(address)? + .ok_or_else(|| { + Error::Assertion(format!( + "Expected account ({address}) is missing from DB: {self:?}" + )) + })?; + + dbg!("Account for {}: {:?}", address, account); + + assert_equal(self.balance, account.balance, "Balance does not match")?; + assert_equal(self.nonce.to(), account.nonce, "Nonce does not match")?; + + if let Some(bytecode_hash) = account.bytecode_hash { + assert_equal( + keccak256(&self.code), + bytecode_hash, + "Bytecode does not match", + )?; + } else { + assert_equal( + self.code.is_empty(), + true, + "Expected empty bytecode, got bytecode in db.", + )?; + } + + let mut storage_cursor = tx.cursor_dup_read::()?; + dbg!("self.storage: {:?}", self.storage.clone()); + // dbg!("printing all storage entries"); + // let mut all = storage_cursor.walk(None).unwrap(); + // while let Ok(entry) = all.next().unwrap() { + // dbg!("Storage entries: {:?}", entry); + // } + for (slot, value) in &self.storage { + if let Some(entry) = + storage_cursor.seek_by_key_subkey(address, B256::new(slot.to_be_bytes()))? + { + dbg!("Storage entry: {:?}", entry); + if U256::from_be_bytes(entry.key.0) == *slot { + assert_equal( + *value, + entry.value, + &format!("Storage for slot {slot:?} does not match"), + )?; + } else { + return Err(Error::Assertion(format!( + "Slot {slot:?} is missing from the database. Expected {value:?}" + ))); + } + } else { + return Err(Error::Assertion(format!( + "Slot {slot:?} is missing from the database. Expected {value:?}" + ))); + } + } + + Ok(()) + } +} + +/// Fork specification. +#[derive(Debug, PartialEq, Eq, PartialOrd, Hash, Ord, Clone, Deserialize)] +pub enum ForkSpec { + /// Frontier + Frontier, + /// Frontier to Homestead + FrontierToHomesteadAt5, + /// Homestead + Homestead, + /// Homestead to Tangerine + HomesteadToDaoAt5, + /// Homestead to Tangerine + HomesteadToEIP150At5, + /// Tangerine + EIP150, + /// Spurious Dragon + EIP158, // EIP-161: State trie clearing + /// Spurious Dragon to Byzantium + EIP158ToByzantiumAt5, + /// Byzantium + Byzantium, + /// Byzantium to Constantinople + ByzantiumToConstantinopleAt5, // SKIPPED + /// Byzantium to Constantinople + ByzantiumToConstantinopleFixAt5, + /// Constantinople + Constantinople, // SKIPPED + /// Constantinople fix + ConstantinopleFix, + /// Istanbul + Istanbul, + /// Berlin + Berlin, + /// Berlin to London + BerlinToLondonAt5, + /// London + London, + /// Paris aka The Merge + Merge, + /// Shanghai + Shanghai, + /// Merge EOF test + #[serde(alias = "Merge+3540+3670")] + MergeEOF, + /// After Merge Init Code test + #[serde(alias = "Merge+3860")] + MergeMeterInitCode, + /// After Merge plus new PUSH0 opcode + #[serde(alias = "Merge+3855")] + MergePush0, + /// Cancun + Cancun, + /// Fork Spec which is unknown to us + #[serde(other)] + Unknown, +} + +impl From for ChainSpec { + fn from(fork_spec: ForkSpec) -> Self { + let spec_builder = ChainSpecBuilder::mainnet(); + + match fork_spec { + ForkSpec::Frontier => spec_builder.frontier_activated(), + ForkSpec::Homestead | ForkSpec::FrontierToHomesteadAt5 => { + spec_builder.homestead_activated() + } + ForkSpec::EIP150 | ForkSpec::HomesteadToDaoAt5 | ForkSpec::HomesteadToEIP150At5 => { + spec_builder.tangerine_whistle_activated() + } + ForkSpec::EIP158 => spec_builder.spurious_dragon_activated(), + ForkSpec::Byzantium + | ForkSpec::EIP158ToByzantiumAt5 + | ForkSpec::ConstantinopleFix + | ForkSpec::ByzantiumToConstantinopleFixAt5 => spec_builder.byzantium_activated(), + ForkSpec::Istanbul => spec_builder.istanbul_activated(), + ForkSpec::Berlin => spec_builder.berlin_activated(), + ForkSpec::London | ForkSpec::BerlinToLondonAt5 => spec_builder.london_activated(), + ForkSpec::Merge + | ForkSpec::MergeEOF + | ForkSpec::MergeMeterInitCode + | ForkSpec::MergePush0 => spec_builder.paris_activated(), + ForkSpec::Shanghai => spec_builder.shanghai_activated(), + ForkSpec::Cancun => spec_builder.cancun_activated(), + ForkSpec::ByzantiumToConstantinopleAt5 | ForkSpec::Constantinople => { + panic!("Overridden with PETERSBURG") + } + ForkSpec::Unknown => { + panic!("Unknown fork"); + } + } + .build() + } +} + +/// Possible seal engines. +#[derive(Debug, PartialEq, Eq, Default, Deserialize)] +pub enum SealEngine { + /// No consensus checks. + #[default] + NoProof, +} + +/// Ethereum blockchain test transaction data. +#[derive(Debug, PartialEq, Eq, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Transaction { + /// Transaction type + #[serde(rename = "type")] + pub transaction_type: Option, + /// Data. + pub data: Bytes, + /// Gas limit. + pub gas_limit: U256, + /// Gas price. + pub gas_price: Option, + /// Nonce. + pub nonce: U256, + /// Signature r part. + pub r: U256, + /// Signature s part. + pub s: U256, + /// Parity bit. + pub v: U256, + /// Transaction value. + pub value: U256, + /// Chain ID. + pub chain_id: Option, + /// Access list. + pub access_list: Option, + /// Max fee per gas. + pub max_fee_per_gas: Option, + /// Max priority fee per gas + pub max_priority_fee_per_gas: Option, + /// Transaction hash. + pub hash: Option, +} + +/// Access list item +#[derive(Debug, PartialEq, Eq, Deserialize, Clone)] +#[serde(rename_all = "camelCase")] +pub struct AccessListItem { + /// Account address + pub address: Address, + /// Storage key. + pub storage_keys: Vec, +} + +/// Access list. +pub type AccessList = Vec; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn header_deserialize() { + let test = r#"{ + "baseFeePerGas" : "0x0a", + "bloom" : "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "coinbase" : "0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba", + "difficulty" : "0x020000", + "extraData" : "0x00", + "gasLimit" : "0x10000000000000", + "gasUsed" : "0x10000000000000", + "hash" : "0x7ebfee2a2c785fef181b8ffd92d4a48a0660ec000f465f309757e3f092d13882", + "mixHash" : "0x0000000000000000000000000000000000000000000000000000000000000000", + "nonce" : "0x0000000000000000", + "number" : "0x01", + "parentHash" : "0xa8f2eb2ea9dccbf725801eef5a31ce59bada431e888dfd5501677cc4365dc3be", + "receiptTrie" : "0xbdd943f5c62ae0299324244a0f65524337ada9817e18e1764631cc1424f3a293", + "stateRoot" : "0xc9c6306ee3e5acbaabe8e2fa28a10c12e27bad1d1aacc271665149f70519f8b0", + "timestamp" : "0x03e8", + "transactionsTrie" : "0xf5893b055ca05e4f14d1792745586a1376e218180bd56bd96b2b024e1dc78300", + "uncleHash" : "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347" + }"#; + let res = serde_json::from_str::
(test); + assert!( + res.is_ok(), + "Failed to deserialize Header with error: {res:?}" + ); + } + + #[test] + fn transaction_deserialize() { + let test = r#"[ + { + "accessList" : [ + ], + "chainId" : "0x01", + "data" : "0x693c61390000000000000000000000000000000000000000000000000000000000000000", + "gasLimit" : "0x10000000000000", + "maxFeePerGas" : "0x07d0", + "maxPriorityFeePerGas" : "0x00", + "nonce" : "0x01", + "r" : "0x5fecc3972a35c9e341b41b0c269d9a7325e13269fb01c2f64cbce1046b3441c8", + "s" : "0x7d4d0eda0e4ebd53c5d0b6fc35c600b317f8fa873b3963ab623ec9cec7d969bd", + "sender" : "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b", + "to" : "0xcccccccccccccccccccccccccccccccccccccccc", + "type" : "0x02", + "v" : "0x01", + "value" : "0x00" + } + ]"#; + + let res = serde_json::from_str::>(test); + assert!( + res.is_ok(), + "Failed to deserialize transaction with error: {res:?}" + ); + } +} diff --git a/src/testing/result.rs b/src/testing/result.rs new file mode 100644 index 0000000..7d08007 --- /dev/null +++ b/src/testing/result.rs @@ -0,0 +1,138 @@ +//! Test results and errors + +use crate::testing::Case; +use reth_db::DatabaseError; +use reth_provider::ProviderError; +use std::path::{Path, PathBuf}; +use thiserror::Error; + +/// Test errors +/// +/// # Note +/// +/// `Error::Skipped` should not be treated as a test failure. +#[derive(Debug, Error)] +#[non_exhaustive] +pub enum Error { + /// The test was skipped + #[error("test was skipped")] + Skipped, + /// No post state found in test + #[error("no post state found for validation")] + MissingPostState, + /// An IO error occurred + #[error("an error occurred interacting with the file system at {path}: {error}")] + Io { + /// The path to the file or directory + path: PathBuf, + /// The specific error + #[source] + error: std::io::Error, + }, + /// A deserialization error occurred + #[error("an error occurred deserializing the test at {path}: {error}")] + CouldNotDeserialize { + /// The path to the file we wanted to deserialize + path: PathBuf, + /// The specific error + #[source] + error: serde_json::Error, + }, + /// A database error occurred. + #[error(transparent)] + Database(#[from] DatabaseError), + /// A test assertion failed. + #[error("test failed: {0}")] + Assertion(String), + /// An error internally in reth occurred. + #[error("test failed: {0}")] + Provider(#[from] ProviderError), + /// An error occurred while decoding RLP. + #[error("an error occurred deserializing RLP: {0}")] + RlpDecode(#[from] alloy_rlp::Error), + /// Custom error message + #[error("{0}")] + Custom(String), +} + +/// The result of running a test. +#[derive(Debug)] +pub struct CaseResult { + /// A description of the test. + pub desc: String, + /// The full path to the test. + pub path: PathBuf, + /// The result of the test. + pub result: Result<(), Error>, +} + +impl CaseResult { + /// Create a new test result. + pub fn new(path: &Path, case: &impl Case, result: Result<(), Error>) -> Self { + Self { + desc: case.description(), + path: path.into(), + result, + } + } +} + +/// Assert that all the given tests passed and print the results to stdout. +pub(crate) fn assert_tests_pass(suite_name: &str, path: &Path, results: &[CaseResult]) { + let (passed, failed, skipped) = categorize_results(results); + + print_results(suite_name, path, &passed, &failed, &skipped); + + assert!(failed.is_empty(), "Some tests failed (see above)"); +} + +/// Categorize test results into `(passed, failed, skipped)`. +pub(crate) fn categorize_results( + results: &[CaseResult], +) -> (Vec<&CaseResult>, Vec<&CaseResult>, Vec<&CaseResult>) { + let mut passed = Vec::new(); + let mut failed = Vec::new(); + let mut skipped = Vec::new(); + + for case in results { + match case.result.as_ref().err() { + Some(Error::Skipped) => skipped.push(case), + Some(_) => failed.push(case), + None => passed.push(case), + } + } + + (passed, failed, skipped) +} + +/// Display the given test results to stdout. +pub(crate) fn print_results( + suite_name: &str, + path: &Path, + passed: &[&CaseResult], + failed: &[&CaseResult], + skipped: &[&CaseResult], +) { + println!("Suite: {suite_name} (at {})", path.display()); + println!( + "Ran {} tests ({} passed, {} failed, {} skipped)", + passed.len() + failed.len() + skipped.len(), + passed.len(), + failed.len(), + skipped.len() + ); + + for case in skipped { + println!("[S] Case {} skipped", case.path.display()); + } + + for case in failed { + let error = case.result.as_ref().unwrap_err(); + println!( + "[!] Case {} failed (description: {}): {}", + case.path.display(), + case.desc, + error + ); + } +} diff --git a/src/testing/suite.rs b/src/testing/suite.rs new file mode 100644 index 0000000..9b6076b --- /dev/null +++ b/src/testing/suite.rs @@ -0,0 +1,75 @@ +//! Abstractions for groups of tests. + +use crate::testing::{ + case::{Case, Cases}, + result::assert_tests_pass, +}; +use std::path::{Path, PathBuf}; +use walkdir::{DirEntry, WalkDir}; + +/// A collection of tests. +pub trait Suite { + /// The type of test cases in this suite. + type Case: Case; + + /// The name of the test suite used to locate the individual test cases. + /// + /// # Example + /// + /// - `GeneralStateTests` + /// - `BlockchainTests/InvalidBlocks` + /// - `BlockchainTests/TransitionTests` + fn suite_name(&self) -> String; + + /// Load an run each contained test case. + /// + /// # Note + /// + /// This recursively finds every test description in the resulting path. + fn run(&self) { + // Build the path to the test suite directory + // let suite_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + // .join("fixtures") + // .join(self.suite_name()); + dbg!("suit path", self.suite_name()); + let suite_path = match self.suite_name().starts_with("blockchain_tests") { + true => PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("fixtures") + .join(self.suite_name()), + false => PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("ethereum-tests") + .join(self.suite_name()), + }; + + // Verify that the path exists + assert!( + suite_path.exists(), + "Test suite path does not exist: {suite_path:?}" + ); + + // Find all files with the ".json" extension in the test suite directory + let test_cases = find_all_files_with_extension(&suite_path, ".json") + .into_iter() + .map(|test_case_path| { + let case = Self::Case::load(&test_case_path).expect("test case should load"); + (test_case_path, case) + }) + .collect(); + + // Run the test cases and collect the results + let results = Cases { test_cases }.run(); + + // Assert that all tests in the suite pass + assert_tests_pass(&self.suite_name(), &suite_path, &results); + } +} + +/// Recursively find all files with a given extension. +fn find_all_files_with_extension(path: &Path, extension: &str) -> Vec { + WalkDir::new(path) + .into_iter() + .filter_map(Result::ok) + .filter(|e| e.file_name().to_string_lossy().ends_with(extension)) + .map(DirEntry::into_path) + .collect() +} diff --git a/src/testing/tests/mod.rs b/src/testing/tests/mod.rs new file mode 100644 index 0000000..14f0038 --- /dev/null +++ b/src/testing/tests/mod.rs @@ -0,0 +1 @@ +mod tests; diff --git a/src/testing/tests/tests.rs b/src/testing/tests/tests.rs new file mode 100644 index 0000000..d39965f --- /dev/null +++ b/src/testing/tests/tests.rs @@ -0,0 +1,137 @@ +#![allow(missing_docs)] +#![cfg(feature = "testing")] +#![cfg(test)] + +macro_rules! general_state_test { + ($test_name:ident, $fork_or_testname:ident $(, $test:ident, $testname:ident)?) => { + #[test] + fn $test_name() { + // if test and testname is empty, then return BlockchainTests::new(format!("GeneralStateTests/{}", stringify!($dir))).run(); + if stringify!($($test)?).is_empty() && stringify!($($testname)?).is_empty() { + return BlockchainTests::new(format!("BlockchainTests/GeneralStateTests/{}", stringify!($fork_or_testname))).run(); + } + $(BlockchainTests::new(format!("blockchain_tests/{}/{}/{}", stringify!($fork_or_testname), stringify!($test), stringify!($testname))).run();)? + } + }; +} + +#[allow(missing_docs)] +mod general_state_tests { + use crate::testing::{cases::blockchain_test::BlockchainTests, suite::Suite}; + + ///////////////////////////// TESTS FROM EXECUTION LAYER SPEC TESTS ///////////////////////////// + general_state_test!(modexp, byzantium, eip198_modexp_precompile, modexp); + general_state_test!(acl, berlin, eip2930_access_list, acl); + general_state_test!(dup, frontier, opcodes, dup); + general_state_test!( + call_and_callcode_gas_calculation, + frontier, + opcodes, + call_and_callcode_gas_calculation + ); + general_state_test!(chainid, istanbul, eip1344_chainid, chainid); + general_state_test!( + dynamic_create2_selfdestruct_collision, + cancun, + eip6780_selfdestruct, + dynamic_create2_selfdestruct_collision + ); + general_state_test!(selfdestruct, cancun, eip6780_selfdestruct, selfdestruct); + general_state_test!( + reentrancy_selfdestruct_revert, + cancun, + eip6780_selfdestruct, + reentrancy_selfdestruct_revert + ); + general_state_test!( + warm_coinbase, + shanghai, + eip3651_warm_coinbase, + warm_coinbase + ); + general_state_test!(push0, shanghai, eip3855_push0, push0); + general_state_test!(yul_example, homestead, yul, yul_example); + general_state_test!( + selfdestruct_balance_bug, + paris, + security, + selfdestruct_balance_bug + ); + + #[cfg(feature = "failing-tests")] + mod failing_eels_tests { + use super::*; + general_state_test!(withdrawals, shanghai, eip4895_withdrawals, withdrawals); + general_state_test!(initcode, shanghai, eip3860_initcode, initcode); + } + + /////////////////////////////////// TESTS FROM ETHEREUM/TESTS /////////////////////////////////// + general_state_test!(st_args_zero_one_balance, stArgsZeroOneBalance); + general_state_test!(st_attack, stAttackTest); + general_state_test!(st_bugs, stBugs); + general_state_test!(st_call_codes, stCallCodes); + general_state_test!(st_call_create_call_code, stCallCreateCallCodeTest); + general_state_test!( + st_call_delegate_codes_call_code_homestead, + stCallDelegateCodesCallCodeHomestead + ); + general_state_test!( + st_call_delegate_codes_homestead, + stCallDelegateCodesHomestead + ); + general_state_test!(st_chain_id, stChainId); + general_state_test!(st_code_copy_test, stCodeCopyTest); + general_state_test!(st_code_size_limit, stCodeSizeLimit); + general_state_test!(st_delegate_call_test_homestead, stDelegatecallTestHomestead); + general_state_test!(st_eip150_gas_prices, stEIP150singleCodeGasPrices); + general_state_test!(st_eip150, stEIP150Specific); + general_state_test!(st_eip158, stEIP158Specific); + general_state_test!(st_eip2930, stEIP2930); + general_state_test!(st_homestead, stHomesteadSpecific); + general_state_test!(st_init_code, stInitCodeTest); + general_state_test!(st_log, stLogTests); + general_state_test!(st_mem_expanding_eip150_calls, stMemExpandingEIP150Calls); + general_state_test!(st_memory_stress, stMemoryStressTest); + general_state_test!(st_memory, stMemoryTest); + general_state_test!(st_non_zero_calls, stNonZeroCallsTest); + general_state_test!(st_precompiles, stPreCompiledContracts); + general_state_test!(st_precompiles2, stPreCompiledContracts2); + general_state_test!(st_random, stRandom); + general_state_test!(st_random2, stRandom2); + general_state_test!(st_refund, stRefundTest); + general_state_test!(st_return, stReturnDataTest); + general_state_test!(st_self_balance, stSelfBalance); + general_state_test!(st_shift, stShift); + general_state_test!(st_sload, stSLoadTest); + general_state_test!(st_solidity, stSolidityTest); + general_state_test!(st_static_flag, stStaticFlagEnabled); + general_state_test!(st_system_operations, stSystemOperationsTest); + general_state_test!(st_time_consuming, stTimeConsuming); + general_state_test!(st_wallet, stWalletTest); + general_state_test!(st_zero_calls_revert, stZeroCallsRevert); + general_state_test!(st_zero_calls, stZeroCallsTest); + general_state_test!(st_zero_knowledge, stZeroKnowledge); + general_state_test!(st_zero_knowledge2, stZeroKnowledge2); + + #[cfg(feature = "failing-tests")] + mod failing_ethereum_tests { + use super::*; + general_state_test!(shanghai, Shanghai); + general_state_test!(st_bad_opcode, stBadOpcode); + general_state_test!(st_create2, stCreate2); + general_state_test!(st_create, stCreateTest); + general_state_test!(st_eip1559, stEIP1559); + general_state_test!(st_eip3607, stEIP3607); + general_state_test!(st_example, stExample); + general_state_test!(st_ext_codehash, stExtCodeHash); + general_state_test!(st_quadratic_complexity, stQuadraticComplexityTest); + general_state_test!(st_recursive_create, stRecursiveCreate); + general_state_test!(st_revert, stRevertTest); + general_state_test!(st_special, stSpecialTest); + general_state_test!(st_sstore, stSStoreTest); + general_state_test!(st_stack, stStackTests); + general_state_test!(st_static_call, stStaticCall); // passing, but conflicts with rewards contract + general_state_test!(st_transaction, stTransactionTest); + general_state_test!(vm_tests, VMTests); + } +}