From 8a5cc34ba24715a5d39abc88abca5b73ae834aa5 Mon Sep 17 00:00:00 2001 From: Qiuhao Li Date: Sat, 10 Aug 2024 16:31:08 +0800 Subject: [PATCH 1/2] feat(invariant): support fuzz with random msg.value --- crates/evm/evm/src/executors/invariant/mod.rs | 6 ++-- .../evm/evm/src/executors/invariant/replay.rs | 7 ++-- .../evm/evm/src/executors/invariant/shrink.rs | 7 ++-- .../evm/fuzz/src/invariant/call_override.rs | 4 ++- crates/evm/fuzz/src/invariant/mod.rs | 4 ++- crates/evm/fuzz/src/lib.rs | 19 +++++++++-- crates/evm/fuzz/src/strategies/invariants.rs | 29 +++++++++++++--- crates/forge/src/runner.rs | 1 + crates/forge/tests/it/invariant.rs | 23 ++++++++++++- .../invariant/common/InvariantMsgValue.t.sol | 33 +++++++++++++++++++ .../common/InvariantSelectorsWeight.t.sol | 17 +++++++--- 11 files changed, 129 insertions(+), 21 deletions(-) create mode 100644 testdata/default/fuzz/invariant/common/InvariantMsgValue.t.sol diff --git a/crates/evm/evm/src/executors/invariant/mod.rs b/crates/evm/evm/src/executors/invariant/mod.rs index 91143f4b232e..a2f74b5338b1 100644 --- a/crates/evm/evm/src/executors/invariant/mod.rs +++ b/crates/evm/evm/src/executors/invariant/mod.rs @@ -314,7 +314,9 @@ impl<'a> InvariantExecutor<'a> { let tx = current_run.inputs.last().ok_or_else(|| { TestCaseError::fail("No input generated to call fuzzed target.") })?; - + if current_run.executor.get_balance(tx.sender)? < tx.value { + current_run.executor.set_balance(tx.sender, tx.value)?; + } // Execute call from the randomly generated sequence and commit state changes. let call_result = current_run .executor @@ -322,7 +324,7 @@ impl<'a> InvariantExecutor<'a> { tx.sender, tx.call_details.target, tx.call_details.calldata.clone(), - U256::ZERO, + tx.value, ) .map_err(|e| { TestCaseError::fail(format!("Could not make raw evm call: {e}")) diff --git a/crates/evm/evm/src/executors/invariant/replay.rs b/crates/evm/evm/src/executors/invariant/replay.rs index 9de5bf531714..394f0fce5bde 100644 --- a/crates/evm/evm/src/executors/invariant/replay.rs +++ b/crates/evm/evm/src/executors/invariant/replay.rs @@ -16,7 +16,6 @@ use foundry_evm_traces::{load_contracts, TraceKind, TraceMode, Traces}; use indicatif::ProgressBar; use parking_lot::RwLock; use proptest::test_runner::TestError; -use revm::primitives::U256; use std::sync::Arc; /// Replays a call sequence for collecting logs and traces. @@ -41,11 +40,14 @@ pub fn replay_run( // Replay each call from the sequence, collect logs, traces and coverage. for tx in inputs { + if executor.get_balance(tx.sender)? < tx.value { + executor.set_balance(tx.sender, tx.value)?; + } let call_result = executor.transact_raw( tx.sender, tx.call_details.target, tx.call_details.calldata.clone(), - U256::ZERO, + tx.value, )?; logs.extend(call_result.logs); traces.push((TraceKind::Execution, call_result.traces.clone().unwrap())); @@ -66,6 +68,7 @@ pub fn replay_run( tx.sender, tx.call_details.target, &tx.call_details.calldata, + tx.value, &ided_contracts, call_result.traces, )); diff --git a/crates/evm/evm/src/executors/invariant/shrink.rs b/crates/evm/evm/src/executors/invariant/shrink.rs index 5559ec821724..a79fc0e95b24 100644 --- a/crates/evm/evm/src/executors/invariant/shrink.rs +++ b/crates/evm/evm/src/executors/invariant/shrink.rs @@ -4,7 +4,7 @@ use crate::executors::{ }, Executor, }; -use alloy_primitives::{Address, Bytes, U256}; +use alloy_primitives::{Address, Bytes}; use foundry_evm_fuzz::invariant::BasicTxDetails; use indicatif::ProgressBar; use proptest::bits::{BitSetLike, VarBitSet}; @@ -153,11 +153,14 @@ pub fn check_sequence( // Apply the call sequence. for call_index in sequence { let tx = &calls[call_index]; + if executor.get_balance(tx.sender)? < tx.value { + executor.set_balance(tx.sender, tx.value)?; + } let call_result = executor.transact_raw( tx.sender, tx.call_details.target, tx.call_details.calldata.clone(), - U256::ZERO, + tx.value )?; if call_result.reverted && fail_on_revert { // Candidate sequence fails test. diff --git a/crates/evm/fuzz/src/invariant/call_override.rs b/crates/evm/fuzz/src/invariant/call_override.rs index dddf591526f0..ca666f96bc91 100644 --- a/crates/evm/fuzz/src/invariant/call_override.rs +++ b/crates/evm/fuzz/src/invariant/call_override.rs @@ -1,5 +1,6 @@ use super::{BasicTxDetails, CallDetails}; use alloy_primitives::Address; +use alloy_primitives::U256; use parking_lot::{Mutex, RwLock}; use proptest::{ option::weighted, @@ -75,12 +76,13 @@ impl RandomCallGenerator { *self.target_reference.write() = original_caller; // `original_caller` has a 80% chance of being the `new_target`. + // TODO: Support msg.value > 0 for call_override let choice = self .strategy .new_tree(&mut self.runner.lock()) .unwrap() .current() - .map(|call_details| BasicTxDetails { sender, call_details }); + .map(|call_details| BasicTxDetails { sender, call_details, value: U256::ZERO }); self.last_sequence.write().push(choice.clone()); choice diff --git a/crates/evm/fuzz/src/invariant/mod.rs b/crates/evm/fuzz/src/invariant/mod.rs index d6b3e574d8b1..9db2d9cfeae8 100644 --- a/crates/evm/fuzz/src/invariant/mod.rs +++ b/crates/evm/fuzz/src/invariant/mod.rs @@ -1,5 +1,5 @@ use alloy_json_abi::{Function, JsonAbi}; -use alloy_primitives::{Address, Bytes, Selector}; +use alloy_primitives::{Address, Bytes, Selector, U256}; use itertools::Either; use parking_lot::Mutex; use std::{collections::BTreeMap, sync::Arc}; @@ -205,6 +205,8 @@ pub struct BasicTxDetails { pub sender: Address, // Transaction call details. pub call_details: CallDetails, + // Transaction value. + pub value: U256, } /// Call details of a transaction generated to fuzz invariant target. diff --git a/crates/evm/fuzz/src/lib.rs b/crates/evm/fuzz/src/lib.rs index 8f24cba3048c..0c4354db12f5 100644 --- a/crates/evm/fuzz/src/lib.rs +++ b/crates/evm/fuzz/src/lib.rs @@ -9,7 +9,7 @@ extern crate tracing; use alloy_dyn_abi::{DynSolValue, JsonAbiExt}; -use alloy_primitives::{Address, Bytes, Log}; +use alloy_primitives::{Address, Bytes, Log, U256}; use foundry_common::{calc, contracts::ContractsByAddress, evm::Breakpoints}; use foundry_evm_coverage::HitMaps; use foundry_evm_traces::CallTraceArena; @@ -44,6 +44,8 @@ pub struct BaseCounterExample { pub addr: Option
, /// The data to provide pub calldata: Bytes, + /// The number of wei sent + pub value: Option, /// Contract name if it exists pub contract_name: Option, /// Function signature if it exists @@ -61,9 +63,11 @@ impl BaseCounterExample { sender: Address, addr: Address, bytes: &Bytes, + value: U256, contracts: &ContractsByAddress, traces: Option, ) -> Self { + let value = if value == U256::ZERO {None} else {Some(value)}; if let Some((name, abi)) = &contracts.get(&addr) { if let Some(func) = abi.functions().find(|f| f.selector() == bytes[..4]) { // skip the function selector when decoding @@ -72,6 +76,7 @@ impl BaseCounterExample { sender: Some(sender), addr: Some(addr), calldata: bytes.clone(), + value, contract_name: Some(name.clone()), signature: Some(func.signature()), args: Some( @@ -87,6 +92,7 @@ impl BaseCounterExample { sender: Some(sender), addr: Some(addr), calldata: bytes.clone(), + value, contract_name: None, signature: None, args: None, @@ -104,6 +110,7 @@ impl BaseCounterExample { sender: None, addr: None, calldata: bytes, + value: None, contract_name: None, signature: None, args: Some(foundry_common::fmt::format_tokens(&args).format(", ").to_string()), @@ -133,9 +140,15 @@ impl fmt::Display for BaseCounterExample { } if let Some(args) = &self.args { - write!(f, " args=[{args}]") + write!(f, " args=[{args}]")? } else { - write!(f, " args=[]") + write!(f, " args=[]")? + } + + if let Some(value) = &self.value { + write!(f, " value=[{value}]") + } else { + write!(f, "") } } } diff --git a/crates/evm/fuzz/src/strategies/invariants.rs b/crates/evm/fuzz/src/strategies/invariants.rs index c7d04dd1ab02..b9deab0409b1 100644 --- a/crates/evm/fuzz/src/strategies/invariants.rs +++ b/crates/evm/fuzz/src/strategies/invariants.rs @@ -4,8 +4,8 @@ use crate::{ strategies::{fuzz_calldata_from_state, fuzz_param, EvmFuzzState}, FuzzFixtures, }; -use alloy_json_abi::Function; -use alloy_primitives::Address; +use alloy_json_abi::{Function, StateMutability}; +use alloy_primitives::{Address, U256}; use parking_lot::RwLock; use proptest::prelude::*; use rand::seq::IteratorRandom; @@ -70,15 +70,16 @@ pub fn invariant_strat( let functions = contracts.fuzzed_functions(); let (target_address, target_function) = selector.select(functions); let sender = select_random_sender(&fuzz_state, senders.clone(), dictionary_weight); + let value = select_random_msg_value(&fuzz_state, dictionary_weight, target_function); let call_details = fuzz_contract_with_calldata( &fuzz_state, &fuzz_fixtures, *target_address, target_function.clone(), ); - (sender, call_details) + (sender, call_details, value) }) - .prop_map(|(sender, call_details)| BasicTxDetails { sender, call_details }) + .prop_map(|(sender, call_details, value)| BasicTxDetails { sender, call_details, value }) } /// Strategy to select a sender address: @@ -104,6 +105,26 @@ fn select_random_sender( } } +/// Strategy to select a msg.value: +/// * If the target function is not payable, the msg.value is zero. +/// * If the target function is payable, the msg.value either a random number or from the dictionary. +fn select_random_msg_value( + fuzz_state: &EvmFuzzState, + dictionary_weight: u32, + func: &Function, +) -> impl Strategy { + if func.state_mutability != StateMutability::Payable { + (0..1).prop_map(move |_value| U256::ZERO).boxed() + } else { + assert!(dictionary_weight <= 100, "dictionary_weight must be <= 100"); + proptest::prop_oneof![ + 100 - dictionary_weight => fuzz_param(&alloy_dyn_abi::DynSolType::Uint(96)), + dictionary_weight => fuzz_param_from_state(&alloy_dyn_abi::DynSolType::Uint(96), fuzz_state), + ] + .prop_map(move |value| value.as_uint().unwrap().0).boxed() + } +} + /// Given a function, it returns a proptest strategy which generates valid abi-encoded calldata /// for that function's input types. pub fn fuzz_contract_with_calldata( diff --git a/crates/forge/src/runner.rs b/crates/forge/src/runner.rs index 6ed06e525e39..a09deb1b02c4 100644 --- a/crates/forge/src/runner.rs +++ b/crates/forge/src/runner.rs @@ -508,6 +508,7 @@ impl<'a> ContractRunner<'a> { target: seq.addr.unwrap_or_default(), calldata: seq.calldata.clone(), }, + value: seq.value.unwrap_or(U256::ZERO), }) .collect::>(); if let Ok((success, replayed_entirely)) = check_sequence( diff --git a/crates/forge/tests/it/invariant.rs b/crates/forge/tests/it/invariant.rs index bd9286713b1d..660866232202 100644 --- a/crates/forge/tests/it/invariant.rs +++ b/crates/forge/tests/it/invariant.rs @@ -588,6 +588,27 @@ async fn test_invariant_scrape_values() { ); } +#[tokio::test(flavor = "multi_thread")] +async fn test_invariant_msg_value() { + let filter = Filter::new(".*", ".*", ".*fuzz/invariant/common/InvariantMsgValue.t.sol"); + let results = TEST_DATA_DEFAULT.runner().test_collect(&filter); + assert_multiple( + &results, + BTreeMap::from([ + ( + "default/fuzz/invariant/common/InvariantMsgValue.t.sol:InvariantMsgValue", + vec![( + "invariant_msg_value_not_found()", + false, + Some("revert: CBA with 2,msg.value>0.1234,0 found".into()), + None, + None, + )], + ), + ]), + ); +} + #[tokio::test(flavor = "multi_thread")] async fn test_invariant_roll_fork_handler() { let filter = Filter::new(".*", ".*", ".*fuzz/invariant/common/InvariantRollFork.t.sol"); @@ -672,7 +693,7 @@ async fn test_invariant_selectors_weight() { let mut runner = TEST_DATA_DEFAULT.runner(); runner.test_options.fuzz.seed = Some(U256::from(119u32)); runner.test_options.invariant.runs = 1; - runner.test_options.invariant.depth = 10; + runner.test_options.invariant.depth = 5000; let results = runner.test_collect(&filter); assert_multiple( &results, diff --git a/testdata/default/fuzz/invariant/common/InvariantMsgValue.t.sol b/testdata/default/fuzz/invariant/common/InvariantMsgValue.t.sol new file mode 100644 index 000000000000..1850424673f3 --- /dev/null +++ b/testdata/default/fuzz/invariant/common/InvariantMsgValue.t.sol @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +pragma solidity ^0.8.13; + +import "ds-test/test.sol"; + +contract Pay { + uint256 private counter; + bool public found; // CBA with 2,msg.value>0.1234,0 + + function A(uint8 x) external { + if (counter == 2 && x == 0) found = true; else counter = 0; + } + function B() external payable { + if (counter == 1 && msg.value > 0.1234 ether) counter++; else counter = 0; + } + function C(uint8 x) external { + if (counter == 0 && x == 2) counter++; + } +} + +contract InvariantMsgValue is DSTest { + Pay target; + + function setUp() public { + target = new Pay(); + } + + /// forge-config: default.invariant.runs = 2000 + function invariant_msg_value_not_found() public view { + require(!target.found(), "CBA with 2,msg.value>0.1234,0 found"); + } +} + diff --git a/testdata/default/fuzz/invariant/common/InvariantSelectorsWeight.t.sol b/testdata/default/fuzz/invariant/common/InvariantSelectorsWeight.t.sol index aea46f41859b..f1573069e740 100644 --- a/testdata/default/fuzz/invariant/common/InvariantSelectorsWeight.t.sol +++ b/testdata/default/fuzz/invariant/common/InvariantSelectorsWeight.t.sol @@ -45,11 +45,18 @@ contract InvariantSelectorsWeightTest is DSTest { function afterInvariant() public { // selector hits uniformly distributed, see https://github.com/foundry-rs/foundry/issues/2986 - assertEq(handlerOne.hit1(), 2); - assertEq(handlerTwo.hit2(), 2); - assertEq(handlerTwo.hit3(), 3); - assertEq(handlerTwo.hit4(), 1); - assertEq(handlerTwo.hit5(), 2); + uint256[5] memory hits = [handlerOne.hit1(), handlerTwo.hit2(), handlerTwo.hit3(), handlerTwo.hit4(), handlerTwo.hit5()]; + + uint256 hits_sum; + for (uint i = 0; i < hits.length; i++) { + hits_sum += hits[i]; + } + uint256 average = (hits_sum) / hits.length; + for (uint i = 0; i < hits.length; i++) { + uint256 delta = average > hits[i] ? average - hits[i] : hits[i] - average; + uint256 delta_scaled = delta * 100 / average; + require(delta_scaled <= 10, "Selectors Delta > 10%"); + } } function invariant_selectors_weight() public view {} From 2bb11a4ee6dbad649bccad6a24891f19719dc1bb Mon Sep 17 00:00:00 2001 From: QiuhaoLi Date: Tue, 13 Aug 2024 12:50:42 +0800 Subject: [PATCH 2/2] fix fmt --- crates/evm/evm/src/executors/invariant/shrink.rs | 2 +- crates/evm/fuzz/src/invariant/call_override.rs | 3 +-- crates/evm/fuzz/src/lib.rs | 2 +- crates/evm/fuzz/src/strategies/invariants.rs | 3 ++- crates/forge/tests/it/invariant.rs | 6 ++---- .../default/fuzz/invariant/common/InvariantMsgValue.t.sol | 6 ++++-- .../fuzz/invariant/common/InvariantSelectorsWeight.t.sol | 7 ++++--- 7 files changed, 15 insertions(+), 14 deletions(-) diff --git a/crates/evm/evm/src/executors/invariant/shrink.rs b/crates/evm/evm/src/executors/invariant/shrink.rs index a79fc0e95b24..4a2ff2b767bc 100644 --- a/crates/evm/evm/src/executors/invariant/shrink.rs +++ b/crates/evm/evm/src/executors/invariant/shrink.rs @@ -160,7 +160,7 @@ pub fn check_sequence( tx.sender, tx.call_details.target, tx.call_details.calldata.clone(), - tx.value + tx.value, )?; if call_result.reverted && fail_on_revert { // Candidate sequence fails test. diff --git a/crates/evm/fuzz/src/invariant/call_override.rs b/crates/evm/fuzz/src/invariant/call_override.rs index ca666f96bc91..b55ad30323b9 100644 --- a/crates/evm/fuzz/src/invariant/call_override.rs +++ b/crates/evm/fuzz/src/invariant/call_override.rs @@ -1,6 +1,5 @@ use super::{BasicTxDetails, CallDetails}; -use alloy_primitives::Address; -use alloy_primitives::U256; +use alloy_primitives::{Address, U256}; use parking_lot::{Mutex, RwLock}; use proptest::{ option::weighted, diff --git a/crates/evm/fuzz/src/lib.rs b/crates/evm/fuzz/src/lib.rs index 0c4354db12f5..932885a4c931 100644 --- a/crates/evm/fuzz/src/lib.rs +++ b/crates/evm/fuzz/src/lib.rs @@ -67,7 +67,7 @@ impl BaseCounterExample { contracts: &ContractsByAddress, traces: Option, ) -> Self { - let value = if value == U256::ZERO {None} else {Some(value)}; + let value = if value == U256::ZERO { None } else { Some(value) }; if let Some((name, abi)) = &contracts.get(&addr) { if let Some(func) = abi.functions().find(|f| f.selector() == bytes[..4]) { // skip the function selector when decoding diff --git a/crates/evm/fuzz/src/strategies/invariants.rs b/crates/evm/fuzz/src/strategies/invariants.rs index b9deab0409b1..cd6578521f0f 100644 --- a/crates/evm/fuzz/src/strategies/invariants.rs +++ b/crates/evm/fuzz/src/strategies/invariants.rs @@ -107,7 +107,8 @@ fn select_random_sender( /// Strategy to select a msg.value: /// * If the target function is not payable, the msg.value is zero. -/// * If the target function is payable, the msg.value either a random number or from the dictionary. +/// * If the target function is payable, the msg.value either a random number or +/// * from the dictionary. fn select_random_msg_value( fuzz_state: &EvmFuzzState, dictionary_weight: u32, diff --git a/crates/forge/tests/it/invariant.rs b/crates/forge/tests/it/invariant.rs index 660866232202..1500a3937918 100644 --- a/crates/forge/tests/it/invariant.rs +++ b/crates/forge/tests/it/invariant.rs @@ -594,8 +594,7 @@ async fn test_invariant_msg_value() { let results = TEST_DATA_DEFAULT.runner().test_collect(&filter); assert_multiple( &results, - BTreeMap::from([ - ( + BTreeMap::from([( "default/fuzz/invariant/common/InvariantMsgValue.t.sol:InvariantMsgValue", vec![( "invariant_msg_value_not_found()", @@ -604,8 +603,7 @@ async fn test_invariant_msg_value() { None, None, )], - ), - ]), + )]), ); } diff --git a/testdata/default/fuzz/invariant/common/InvariantMsgValue.t.sol b/testdata/default/fuzz/invariant/common/InvariantMsgValue.t.sol index 1850424673f3..a623e2efc778 100644 --- a/testdata/default/fuzz/invariant/common/InvariantMsgValue.t.sol +++ b/testdata/default/fuzz/invariant/common/InvariantMsgValue.t.sol @@ -8,10 +8,12 @@ contract Pay { bool public found; // CBA with 2,msg.value>0.1234,0 function A(uint8 x) external { - if (counter == 2 && x == 0) found = true; else counter = 0; + if (counter == 2 && x == 0) found = true; + else counter = 0; } function B() external payable { - if (counter == 1 && msg.value > 0.1234 ether) counter++; else counter = 0; + if (counter == 1 && msg.value > 0.1234 ether) counter++; + else counter = 0; } function C(uint8 x) external { if (counter == 0 && x == 2) counter++; diff --git a/testdata/default/fuzz/invariant/common/InvariantSelectorsWeight.t.sol b/testdata/default/fuzz/invariant/common/InvariantSelectorsWeight.t.sol index f1573069e740..99f92e0c2369 100644 --- a/testdata/default/fuzz/invariant/common/InvariantSelectorsWeight.t.sol +++ b/testdata/default/fuzz/invariant/common/InvariantSelectorsWeight.t.sol @@ -45,14 +45,15 @@ contract InvariantSelectorsWeightTest is DSTest { function afterInvariant() public { // selector hits uniformly distributed, see https://github.com/foundry-rs/foundry/issues/2986 - uint256[5] memory hits = [handlerOne.hit1(), handlerTwo.hit2(), handlerTwo.hit3(), handlerTwo.hit4(), handlerTwo.hit5()]; + uint256[5] memory hits = + [handlerOne.hit1(), handlerTwo.hit2(), handlerTwo.hit3(), handlerTwo.hit4(), handlerTwo.hit5()]; uint256 hits_sum; - for (uint i = 0; i < hits.length; i++) { + for (uint256 i = 0; i < hits.length; i++) { hits_sum += hits[i]; } uint256 average = (hits_sum) / hits.length; - for (uint i = 0; i < hits.length; i++) { + for (uint256 i = 0; i < hits.length; i++) { uint256 delta = average > hits[i] ? average - hits[i] : hits[i] - average; uint256 delta_scaled = delta * 100 / average; require(delta_scaled <= 10, "Selectors Delta > 10%");