From e1000176a31140b2abd79c47653cbc4bb1a6808a Mon Sep 17 00:00:00 2001 From: Tom French <15848336+TomAFrench@users.noreply.github.com> Date: Tue, 18 Jun 2024 19:52:45 +0100 Subject: [PATCH] feat: add fuzzer for Noir programs (#5251) # Description ## Problem\* Step towards #5249 ## Summary\* This PR adds a very simple fuzzer which can be used to find inputs to Noir programs which fail to execute. The motivation for this is to eventually allow `nargo test` to run on noir functions with arguments and automatically fuzz them in a similar fashion to `forge test`. It's currently very hit and miss on how quickly it can zero in on failing cases. For example, the program below is near-unfuzzable currently. ```rust fn main(x: u32, y: u32) { assert(x != y); } ``` ## Additional Context ## Documentation\* Check one: - [ ] No documentation needed. - [ ] Documentation included in this PR. - [x] **[For Experimental Features]** Documentation to be submitted in a separate PR. # PR Checklist\* - [x] I have tested the changes locally. - [x] I have formatted the changes with [Prettier](https://prettier.io/) and/or `cargo fmt` on default settings. --- Cargo.lock | 14 +++ Cargo.toml | 2 + .../noirc_frontend/src/elaborator/lints.rs | 9 -- compiler/noirc_frontend/src/elaborator/mod.rs | 1 - .../src/hir/resolution/resolver.rs | 10 +- noir_stdlib/src/uint128.nr | 18 +++- tooling/fuzzer/Cargo.toml | 18 ++++ tooling/fuzzer/src/lib.rs | 93 ++++++++++++++++++ tooling/fuzzer/src/strategies/int.rs | 83 ++++++++++++++++ tooling/fuzzer/src/strategies/mod.rs | 97 +++++++++++++++++++ tooling/fuzzer/src/strategies/uint.rs | 68 +++++++++++++ tooling/fuzzer/src/types.rs | 42 ++++++++ tooling/nargo_cli/Cargo.toml | 2 + tooling/nargo_cli/src/cli/test_cmd.rs | 52 ++++++++-- tooling/nargo_cli/tests/stdlib-tests.rs | 57 +++++++++-- tooling/noirc_abi/Cargo.toml | 3 +- 16 files changed, 524 insertions(+), 45 deletions(-) create mode 100644 tooling/fuzzer/Cargo.toml create mode 100644 tooling/fuzzer/src/lib.rs create mode 100644 tooling/fuzzer/src/strategies/int.rs create mode 100644 tooling/fuzzer/src/strategies/mod.rs create mode 100644 tooling/fuzzer/src/strategies/uint.rs create mode 100644 tooling/fuzzer/src/types.rs diff --git a/Cargo.lock b/Cargo.lock index 16988432243..cacfc06327d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2517,6 +2517,7 @@ dependencies = [ "nargo_fmt", "nargo_toml", "noir_debugger", + "noir_fuzzer", "noir_lsp", "noirc_abi", "noirc_artifacts", @@ -2529,6 +2530,7 @@ dependencies = [ "pprof 0.13.0", "predicates 2.1.5", "prettytable-rs", + "proptest", "rayon", "serde", "serde_json", @@ -2661,6 +2663,18 @@ dependencies = [ "thiserror", ] +[[package]] +name = "noir_fuzzer" +version = "0.31.0" +dependencies = [ + "acvm", + "nargo", + "noirc_abi", + "noirc_artifacts", + "proptest", + "rand 0.8.5", +] + [[package]] name = "noir_grumpkin" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 5fc9936de91..129661d157d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,6 +15,7 @@ members = [ # Crates related to tooling built on top of the Noir compiler "tooling/lsp", "tooling/debugger", + "tooling/fuzzer", "tooling/nargo", "tooling/nargo_fmt", "tooling/nargo_cli", @@ -69,6 +70,7 @@ noirc_frontend = { path = "compiler/noirc_frontend" } noirc_printable_type = { path = "compiler/noirc_printable_type" } # Noir tooling workspace dependencies +noir_fuzzer = { path = "tooling/fuzzer" } nargo = { path = "tooling/nargo" } nargo_fmt = { path = "tooling/nargo_fmt" } nargo_toml = { path = "tooling/nargo_toml" } diff --git a/compiler/noirc_frontend/src/elaborator/lints.rs b/compiler/noirc_frontend/src/elaborator/lints.rs index 4859ac5f97c..b86912940eb 100644 --- a/compiler/noirc_frontend/src/elaborator/lints.rs +++ b/compiler/noirc_frontend/src/elaborator/lints.rs @@ -130,15 +130,6 @@ pub(super) fn recursive_non_entrypoint_function( } } -/// Test functions cannot have arguments in order to be executable. -pub(super) fn test_function_with_args(func: &NoirFunction) -> Option { - if func.attributes().is_test_function() && !func.parameters().is_empty() { - Some(ResolverError::TestFunctionHasParameters { span: func.name_ident().span() }) - } else { - None - } -} - /// Check that we are not passing a mutable reference from a constrained runtime to an unconstrained runtime. pub(super) fn unconstrained_function_args( function_args: &[(Type, ExprId, Span)], diff --git a/compiler/noirc_frontend/src/elaborator/mod.rs b/compiler/noirc_frontend/src/elaborator/mod.rs index 053de6e1735..91d6ba71927 100644 --- a/compiler/noirc_frontend/src/elaborator/mod.rs +++ b/compiler/noirc_frontend/src/elaborator/mod.rs @@ -581,7 +581,6 @@ impl<'context> Elaborator<'context> { self.run_lint(|elaborator| { lints::low_level_function_outside_stdlib(func, elaborator.crate_id).map(Into::into) }); - self.run_lint(|_| lints::test_function_with_args(func).map(Into::into)); self.run_lint(|_| { lints::recursive_non_entrypoint_function(func, is_entry_point).map(Into::into) }); diff --git a/compiler/noirc_frontend/src/hir/resolution/resolver.rs b/compiler/noirc_frontend/src/hir/resolution/resolver.rs index 1dcace82372..97de66be817 100644 --- a/compiler/noirc_frontend/src/hir/resolution/resolver.rs +++ b/compiler/noirc_frontend/src/hir/resolution/resolver.rs @@ -23,7 +23,7 @@ use crate::hir_def::expr::{ use crate::hir_def::function::FunctionBody; use crate::hir_def::traits::{Trait, TraitConstraint}; use crate::macros_api::SecondaryAttribute; -use crate::token::{Attributes, FunctionAttribute}; +use crate::token::Attributes; use regex::Regex; use std::collections::{BTreeMap, BTreeSet, HashSet}; use std::rc::Rc; @@ -1043,14 +1043,6 @@ impl<'a> Resolver<'a> { }); } - if matches!(attributes.function, Some(FunctionAttribute::Test { .. })) - && !parameters.is_empty() - { - self.push_err(ResolverError::TestFunctionHasParameters { - span: func.name_ident().span(), - }); - } - let mut typ = Type::Function(parameter_types, return_type, Box::new(Type::Unit)); if !generics.is_empty() { diff --git a/noir_stdlib/src/uint128.nr b/noir_stdlib/src/uint128.nr index 173fa54863a..829ab09ee1e 100644 --- a/noir_stdlib/src/uint128.nr +++ b/noir_stdlib/src/uint128.nr @@ -319,13 +319,12 @@ mod tests { use crate::uint128::{U128, pow64, pow63}; #[test] - fn test_not() { - let num = U128::from_u64s_le(0, 0); + fn test_not(lo: u64, hi: u64) { + let num = U128::from_u64s_le(lo, hi); let not_num = num.not(); - let max_u64: Field = pow64 - 1; - assert_eq(not_num.hi, max_u64); - assert_eq(not_num.lo, max_u64); + assert_eq(not_num.hi, (hi.not() as Field)); + assert_eq(not_num.lo, (lo.not() as Field)); let not_not_num = not_num.not(); assert_eq(num, not_not_num); @@ -493,6 +492,15 @@ mod tests { let end = a.to_integer(); assert_eq(start, end); } + + #[test] + fn integer_conversions_fuzz(lo: u64, hi: u64) { + let start: Field = (lo as Field) + pow64 * (hi as Field); + let a = U128::from_integer(start); + let end = a.to_integer(); + assert_eq(start, end); + } + #[test] fn test_wrapping_mul() { // 1*0==0 diff --git a/tooling/fuzzer/Cargo.toml b/tooling/fuzzer/Cargo.toml new file mode 100644 index 00000000000..ef49d707d6a --- /dev/null +++ b/tooling/fuzzer/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "noir_fuzzer" +description = "A fuzzer for Noir programs" +version.workspace = true +authors.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +acvm.workspace = true +nargo.workspace = true +noirc_artifacts.workspace = true +noirc_abi.workspace = true +proptest.workspace = true +rand.workspace = true diff --git a/tooling/fuzzer/src/lib.rs b/tooling/fuzzer/src/lib.rs new file mode 100644 index 00000000000..42dccc1dc83 --- /dev/null +++ b/tooling/fuzzer/src/lib.rs @@ -0,0 +1,93 @@ +//! This module has been adapted from Foundry's fuzzing implementation for the EVM. +//! https://github.com/foundry-rs/foundry/blob/6a85dbaa62f1c305f31cab37781232913055ae28/crates/evm/evm/src/executors/fuzz/mod.rs#L40 +//! +//! Code is used under the MIT license. + +use acvm::{blackbox_solver::StubbedBlackBoxSolver, FieldElement}; +use noirc_abi::InputMap; +use proptest::test_runner::{TestCaseError, TestError, TestRunner}; + +mod strategies; +mod types; + +use types::{CaseOutcome, CounterExampleOutcome, FuzzOutcome, FuzzTestResult}; + +use noirc_artifacts::program::ProgramArtifact; + +use nargo::ops::{execute_program, DefaultForeignCallExecutor}; + +/// An executor for Noir programs which which provides fuzzing support using [`proptest`]. +/// +/// After instantiation, calling `fuzz` will proceed to hammer the program with +/// inputs, until it finds a counterexample. The provided [`TestRunner`] contains all the +/// configuration which can be overridden via [environment variables](proptest::test_runner::Config) +pub struct FuzzedExecutor { + /// The program to be fuzzed + program: ProgramArtifact, + + /// The fuzzer + runner: TestRunner, +} + +impl FuzzedExecutor { + /// Instantiates a fuzzed executor given a testrunner + pub fn new(program: ProgramArtifact, runner: TestRunner) -> Self { + Self { program, runner } + } + + /// Fuzzes the provided program. + pub fn fuzz(&self) -> FuzzTestResult { + let strategy = strategies::arb_input_map(&self.program.abi); + + let run_result: Result<(), TestError> = + self.runner.clone().run(&strategy, |input_map| { + let fuzz_res = self.single_fuzz(input_map)?; + + match fuzz_res { + FuzzOutcome::Case(_) => Ok(()), + FuzzOutcome::CounterExample(CounterExampleOutcome { + exit_reason: status, + .. + }) => Err(TestCaseError::fail(status)), + } + }); + + match run_result { + Ok(()) => FuzzTestResult { success: true, reason: None, counterexample: None }, + + Err(TestError::Abort(reason)) => FuzzTestResult { + success: false, + reason: Some(reason.to_string()), + counterexample: None, + }, + Err(TestError::Fail(reason, counterexample)) => { + let reason = reason.to_string(); + let reason = if reason.is_empty() { None } else { Some(reason) }; + + FuzzTestResult { success: false, reason, counterexample: Some(counterexample) } + } + } + } + + /// Granular and single-step function that runs only one fuzz and returns either a `CaseOutcome` + /// or a `CounterExampleOutcome` + pub fn single_fuzz(&self, input_map: InputMap) -> Result { + let initial_witness = self.program.abi.encode(&input_map, None).unwrap(); + let result = execute_program( + &self.program.bytecode, + initial_witness, + &StubbedBlackBoxSolver, + &mut DefaultForeignCallExecutor::::new(false, None), + ); + + // TODO: Add handling for `vm.assume` equivalent + + match result { + Ok(_) => Ok(FuzzOutcome::Case(CaseOutcome { case: input_map })), + Err(err) => Ok(FuzzOutcome::CounterExample(CounterExampleOutcome { + exit_reason: err.to_string(), + counterexample: input_map, + })), + } + } +} diff --git a/tooling/fuzzer/src/strategies/int.rs b/tooling/fuzzer/src/strategies/int.rs new file mode 100644 index 00000000000..d11cafcfae5 --- /dev/null +++ b/tooling/fuzzer/src/strategies/int.rs @@ -0,0 +1,83 @@ +use proptest::{ + strategy::{NewTree, Strategy}, + test_runner::TestRunner, +}; +use rand::Rng; + +/// Strategy for signed ints (up to i128). +/// The strategy combines 2 different strategies, each assigned a specific weight: +/// 1. Generate purely random value in a range. This will first choose bit size uniformly (up `bits` +/// param). Then generate a value for this bit size. +/// 2. Generate a random value around the edges (+/- 3 around min, 0 and max possible value) +#[derive(Debug)] +pub struct IntStrategy { + /// Bit size of int (e.g. 128) + bits: usize, + /// The weight for edge cases (+/- 3 around 0 and max possible value) + edge_weight: usize, + /// The weight for purely random values + random_weight: usize, +} + +impl IntStrategy { + /// Create a new strategy. + /// # Arguments + /// * `bits` - Size of int in bits + pub fn new(bits: usize) -> Self { + Self { bits, edge_weight: 10usize, random_weight: 50usize } + } + + fn generate_edge_tree(&self, runner: &mut TestRunner) -> NewTree { + let rng = runner.rng(); + + let offset = rng.gen_range(0..4); + // Choose if we want values around min, -0, +0, or max + let kind = rng.gen_range(0..4); + let start = match kind { + 0 => self.type_min() + offset, + 1 => -offset - 1i128, + 2 => offset, + 3 => self.type_max() - offset, + _ => unreachable!(), + }; + Ok(proptest::num::i128::BinarySearch::new(start)) + } + + fn generate_random_tree(&self, runner: &mut TestRunner) -> NewTree { + let rng = runner.rng(); + + let start: i128 = rng.gen_range(self.type_min()..=self.type_max()); + Ok(proptest::num::i128::BinarySearch::new(start)) + } + + fn type_max(&self) -> i128 { + if self.bits < 128 { + (1i128 << (self.bits - 1)) - 1 + } else { + i128::MAX + } + } + + fn type_min(&self) -> i128 { + if self.bits < 128 { + -(1i128 << (self.bits - 1)) + } else { + i128::MIN + } + } +} + +impl Strategy for IntStrategy { + type Tree = proptest::num::i128::BinarySearch; + type Value = i128; + + fn new_tree(&self, runner: &mut TestRunner) -> NewTree { + let total_weight = self.random_weight + self.edge_weight; + let bias = runner.rng().gen_range(0..total_weight); + // randomly select one of 2 strategies + match bias { + x if x < self.edge_weight => self.generate_edge_tree(runner), + _ => self.generate_random_tree(runner), + } + } +} diff --git a/tooling/fuzzer/src/strategies/mod.rs b/tooling/fuzzer/src/strategies/mod.rs new file mode 100644 index 00000000000..f5b03953ba8 --- /dev/null +++ b/tooling/fuzzer/src/strategies/mod.rs @@ -0,0 +1,97 @@ +use int::IntStrategy; +use prop::collection::vec; +use proptest::prelude::*; + +use acvm::{AcirField, FieldElement}; + +use noirc_abi::{input_parser::InputValue, Abi, AbiType, InputMap, Sign}; +use std::collections::BTreeMap; +use uint::UintStrategy; + +mod int; +mod uint; + +proptest::prop_compose! { + pub(super) fn arb_field_from_integer(bit_size: u32)(value: u128)-> FieldElement { + let width = (bit_size % 128).clamp(1, 127); + let max_value = 2u128.pow(width) - 1; + let value = value % max_value; + FieldElement::from(value) + } +} + +pub(super) fn arb_value_from_abi_type(abi_type: &AbiType) -> SBoxedStrategy { + match abi_type { + AbiType::Field => vec(any::(), 32) + .prop_map(|bytes| InputValue::Field(FieldElement::from_be_bytes_reduce(&bytes))) + .sboxed(), + AbiType::Integer { width, sign } if sign == &Sign::Unsigned => { + UintStrategy::new(*width as usize) + .prop_map(|uint| InputValue::Field(uint.into())) + .sboxed() + } + AbiType::Integer { width, .. } => { + let shift = 2i128.pow(*width); + IntStrategy::new(*width as usize) + .prop_map(move |mut int| { + if int < 0 { + int += shift + } + InputValue::Field(int.into()) + }) + .sboxed() + } + AbiType::Boolean => { + any::().prop_map(|val| InputValue::Field(FieldElement::from(val))).sboxed() + } + + AbiType::String { length } => { + // Strings only allow ASCII characters as each character must be able to be represented by a single byte. + let string_regex = format!("[[:ascii:]]{{{length}}}"); + proptest::string::string_regex(&string_regex) + .expect("parsing of regex should always succeed") + .prop_map(InputValue::String) + .sboxed() + } + AbiType::Array { length, typ } => { + let length = *length as usize; + let elements = vec(arb_value_from_abi_type(typ), length..=length); + + elements.prop_map(InputValue::Vec).sboxed() + } + + AbiType::Struct { fields, .. } => { + let fields: Vec> = fields + .iter() + .map(|(name, typ)| (Just(name.clone()), arb_value_from_abi_type(typ)).sboxed()) + .collect(); + + fields + .prop_map(|fields| { + let fields: BTreeMap<_, _> = fields.into_iter().collect(); + InputValue::Struct(fields) + }) + .sboxed() + } + + AbiType::Tuple { fields } => { + let fields: Vec<_> = fields.iter().map(arb_value_from_abi_type).collect(); + fields.prop_map(InputValue::Vec).sboxed() + } + } +} + +pub(super) fn arb_input_map(abi: &Abi) -> BoxedStrategy { + let values: Vec<_> = abi + .parameters + .iter() + .map(|param| (Just(param.name.clone()), arb_value_from_abi_type(¶m.typ))) + .collect(); + + values + .prop_map(|values| { + let input_map: InputMap = values.into_iter().collect(); + input_map + }) + .boxed() +} diff --git a/tooling/fuzzer/src/strategies/uint.rs b/tooling/fuzzer/src/strategies/uint.rs new file mode 100644 index 00000000000..5021e832b97 --- /dev/null +++ b/tooling/fuzzer/src/strategies/uint.rs @@ -0,0 +1,68 @@ +use proptest::{ + strategy::{NewTree, Strategy}, + test_runner::TestRunner, +}; +use rand::Rng; + +/// Value tree for unsigned ints (up to u128). +/// The strategy combines 2 different strategies, each assigned a specific weight: +/// 1. Generate purely random value in a range. This will first choose bit size uniformly (up `bits` +/// param). Then generate a value for this bit size. +/// 2. Generate a random value around the edges (+/- 3 around 0 and max possible value) +#[derive(Debug)] +pub struct UintStrategy { + /// Bit size of uint (e.g. 128) + bits: usize, + + /// The weight for edge cases (+/- 3 around 0 and max possible value) + edge_weight: usize, + /// The weight for purely random values + random_weight: usize, +} + +impl UintStrategy { + /// Create a new strategy. + /// # Arguments + /// * `bits` - Size of uint in bits + pub fn new(bits: usize) -> Self { + Self { bits, edge_weight: 10usize, random_weight: 50usize } + } + + fn generate_edge_tree(&self, runner: &mut TestRunner) -> NewTree { + let rng = runner.rng(); + // Choose if we want values around 0 or max + let is_min = rng.gen_bool(0.5); + let offset = rng.gen_range(0..4); + let start = if is_min { offset } else { self.type_max().saturating_sub(offset) }; + Ok(proptest::num::u128::BinarySearch::new(start)) + } + + fn generate_random_tree(&self, runner: &mut TestRunner) -> NewTree { + let rng = runner.rng(); + let start = rng.gen_range(0..=self.type_max()); + + Ok(proptest::num::u128::BinarySearch::new(start)) + } + + fn type_max(&self) -> u128 { + if self.bits < 128 { + (1 << self.bits) - 1 + } else { + u128::MAX + } + } +} + +impl Strategy for UintStrategy { + type Tree = proptest::num::u128::BinarySearch; + type Value = u128; + fn new_tree(&self, runner: &mut TestRunner) -> NewTree { + let total_weight = self.random_weight + self.edge_weight; + let bias = runner.rng().gen_range(0..total_weight); + // randomly select one of 2 strategies + match bias { + x if x < self.edge_weight => self.generate_edge_tree(runner), + _ => self.generate_random_tree(runner), + } + } +} diff --git a/tooling/fuzzer/src/types.rs b/tooling/fuzzer/src/types.rs new file mode 100644 index 00000000000..dbd4ba94ec1 --- /dev/null +++ b/tooling/fuzzer/src/types.rs @@ -0,0 +1,42 @@ +use noirc_abi::InputMap; + +type CounterExample = InputMap; + +/// The outcome of a fuzz test +#[derive(Debug)] +pub struct FuzzTestResult { + /// Whether the test case was successful. This means that the program executed + /// properly, or that there was a constraint failure and that the test was expected to fail + /// (has the `should_fail` attribute) + pub success: bool, + + /// If there was a constraint failure, this field will be populated. Note that the test can + /// still be successful (i.e self.success == true) when it's expected to fail. + pub reason: Option, + + /// Minimal reproduction test case for failing fuzz tests + pub counterexample: Option, +} + +/// Returned by a single fuzz in the case of a successful run +#[derive(Debug)] +pub struct CaseOutcome { + /// Data of a single fuzz test case + pub case: InputMap, +} + +/// Returned by a single fuzz when a counterexample has been discovered +#[derive(Debug)] +pub struct CounterExampleOutcome { + /// Minimal reproduction test case for failing test + pub counterexample: CounterExample, + /// The status of the call + pub exit_reason: String, +} + +/// Outcome of a single fuzz +#[derive(Debug)] +pub enum FuzzOutcome { + Case(CaseOutcome), + CounterExample(CounterExampleOutcome), +} diff --git a/tooling/nargo_cli/Cargo.toml b/tooling/nargo_cli/Cargo.toml index c3beafb7db5..b9d7d7e3e48 100644 --- a/tooling/nargo_cli/Cargo.toml +++ b/tooling/nargo_cli/Cargo.toml @@ -32,6 +32,7 @@ noirc_driver.workspace = true noirc_frontend = { workspace = true, features = ["bn254"] } noirc_abi.workspace = true noirc_errors.workspace = true +noir_fuzzer.workspace = true noirc_artifacts.workspace = true acvm = { workspace = true, features = ["bn254"] } bn254_blackbox_solver.workspace = true @@ -50,6 +51,7 @@ color-eyre = "0.6.2" tokio = { version = "1.0", features = ["io-std", "rt"] } dap.workspace = true clap-markdown = { git = "https://github.com/noir-lang/clap-markdown", rev = "450d759532c88f0dba70891ceecdbc9ff8f25d2b", optional = true } +proptest.workspace = true notify = "6.1.1" notify-debouncer-full = "0.3.1" diff --git a/tooling/nargo_cli/src/cli/test_cmd.rs b/tooling/nargo_cli/src/cli/test_cmd.rs index 1cda8d8382e..de9e8dc5d7c 100644 --- a/tooling/nargo_cli/src/cli/test_cmd.rs +++ b/tooling/nargo_cli/src/cli/test_cmd.rs @@ -10,7 +10,8 @@ use nargo::{ }; use nargo_toml::{get_package_manifest, resolve_workspace_from_toml, PackageSelection}; use noirc_driver::{ - check_crate, file_manager_with_stdlib, CompileOptions, NOIR_ARTIFACT_VERSION_STRING, + check_crate, compile_no_check, file_manager_with_stdlib, CompileOptions, + NOIR_ARTIFACT_VERSION_STRING, }; use noirc_frontend::{ graph::CrateName, @@ -185,14 +186,47 @@ fn run_test + Default>( let blackbox_solver = S::default(); - nargo::ops::run_test( - &blackbox_solver, - &mut context, - test_function, - show_output, - foreign_call_resolver_url, - compile_options, - ) + let test_function_has_no_arguments = context + .def_interner + .function_meta(&test_function.get_id()) + .function_signature() + .0 + .is_empty(); + + if test_function_has_no_arguments { + nargo::ops::run_test( + &blackbox_solver, + &mut context, + test_function, + show_output, + foreign_call_resolver_url, + compile_options, + ) + } else { + use noir_fuzzer::FuzzedExecutor; + use proptest::test_runner::TestRunner; + + let compiled_program = + compile_no_check(&mut context, compile_options, test_function.get_id(), None, false); + match compiled_program { + Ok(compiled_program) => { + let runner = TestRunner::default(); + + let fuzzer = FuzzedExecutor::new(compiled_program.into(), runner); + + let result = fuzzer.fuzz(); + if result.success { + TestStatus::Pass + } else { + TestStatus::Fail { + message: result.reason.unwrap_or_default(), + error_diagnostic: None, + } + } + } + Err(err) => TestStatus::CompileError(err.into()), + } + } } fn get_tests_in_package( diff --git a/tooling/nargo_cli/tests/stdlib-tests.rs b/tooling/nargo_cli/tests/stdlib-tests.rs index 70857b4b65e..5badd76605a 100644 --- a/tooling/nargo_cli/tests/stdlib-tests.rs +++ b/tooling/nargo_cli/tests/stdlib-tests.rs @@ -3,7 +3,7 @@ use std::{collections::BTreeMap, path::PathBuf}; use acvm::blackbox_solver::StubbedBlackBoxSolver; use fm::FileManager; -use noirc_driver::{check_crate, file_manager_with_stdlib, CompileOptions}; +use noirc_driver::{check_crate, compile_no_check, file_manager_with_stdlib, CompileOptions}; use noirc_frontend::hir::FunctionNameMatch; use nargo::{ @@ -47,15 +47,52 @@ fn run_stdlib_tests() { let test_report: Vec<(String, TestStatus)> = test_functions .into_iter() .map(|(test_name, test_function)| { - let status = run_test( - &StubbedBlackBoxSolver, - &mut context, - &test_function, - false, - None, - &CompileOptions::default(), - ); - + let test_function_has_no_arguments = context + .def_interner + .function_meta(&test_function.get_id()) + .function_signature() + .0 + .is_empty(); + + let status = if test_function_has_no_arguments { + run_test( + &StubbedBlackBoxSolver, + &mut context, + &test_function, + false, + None, + &CompileOptions::default(), + ) + } else { + use noir_fuzzer::FuzzedExecutor; + use proptest::test_runner::TestRunner; + + let compiled_program = compile_no_check( + &mut context, + &CompileOptions::default(), + test_function.get_id(), + None, + false, + ); + match compiled_program { + Ok(compiled_program) => { + let runner = TestRunner::default(); + + let fuzzer = FuzzedExecutor::new(compiled_program.into(), runner); + + let result = fuzzer.fuzz(); + if result.success { + TestStatus::Pass + } else { + TestStatus::Fail { + message: result.reason.unwrap_or_default(), + error_diagnostic: None, + } + } + } + Err(err) => TestStatus::CompileError(err.into()), + } + }; (test_name, status) }) .collect(); diff --git a/tooling/noirc_abi/Cargo.toml b/tooling/noirc_abi/Cargo.toml index e8f57060367..4c0c1f75e42 100644 --- a/tooling/noirc_abi/Cargo.toml +++ b/tooling/noirc_abi/Cargo.toml @@ -25,7 +25,6 @@ strum_macros = "0.24" proptest.workspace = true proptest-derive.workspace = true - [features] bn254 = ["acvm/bn254"] -bls12_381 = ["acvm/bls12_381"] \ No newline at end of file +bls12_381 = ["acvm/bls12_381"]