From 9b2e97246d9aef581f7c47e8ab561a019274b06b Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Fri, 11 Oct 2024 14:36:02 -0500 Subject: [PATCH] Change how env vars work in `differential` fuzzer (#9457) * Change how env vars work in `differential` fuzzer This commit updates the processing of the `ALLOWED_*` environment variables to work differently than before. Previously the list of engines and module-generation-strategies were filtered based on `ALLOWED_*` environment variables but this meant that the meaning of a fuzz input changed depending on environment variables. This commit instead replaces the handling of these environment variables to ensure that the meaning of the fuzz input doesn't change depending on their values. Instead fuzz test cases are early-thrown-out if they request an engine that's disabled or a module-generation-strategy that's disabled. * Update fuzz/fuzz_targets/differential.rs Co-authored-by: Trevor Elliott --------- Co-authored-by: Trevor Elliott --- crates/fuzzing/src/oracles/engine.rs | 17 +++++++++++------ fuzz/fuzz_targets/differential.rs | 20 +++++++++++++++----- 2 files changed, 26 insertions(+), 11 deletions(-) diff --git a/crates/fuzzing/src/oracles/engine.rs b/crates/fuzzing/src/oracles/engine.rs index 5c7cf69c547a..887f8382b36c 100644 --- a/crates/fuzzing/src/oracles/engine.rs +++ b/crates/fuzzing/src/oracles/engine.rs @@ -93,14 +93,17 @@ pub fn setup_engine_runtimes() { /// Build a list of allowed values from the given `defaults` using the /// `env_list`. /// +/// The entries in `defaults` are preserved, in order, and are replaced with +/// `None` in the returned list if they are disabled. +/// /// ``` /// # use wasmtime_fuzzing::oracles::engine::build_allowed_env_list; /// // Passing no `env_list` returns the defaults: -/// assert_eq!(build_allowed_env_list(None, &["a"]), vec!["a"]); +/// assert_eq!(build_allowed_env_list(None, &["a"]), vec![Some("a")]); /// // We can build up a subset of the defaults: -/// assert_eq!(build_allowed_env_list(Some(vec!["b".to_string()]), &["a","b"]), vec!["b"]); +/// assert_eq!(build_allowed_env_list(Some(vec!["b".to_string()]), &["a","b"]), vec![None, Some("b")]); /// // Alternately we can subtract from the defaults: -/// assert_eq!(build_allowed_env_list(Some(vec!["-a".to_string()]), &["a","b"]), vec!["b"]); +/// assert_eq!(build_allowed_env_list(Some(vec!["-a".to_string()]), &["a","b"]), vec![None, Some("b")]); /// ``` /// ```should_panic /// # use wasmtime_fuzzing::oracles::engine::build_allowed_env_list; @@ -116,7 +119,7 @@ pub fn setup_engine_runtimes() { pub fn build_allowed_env_list<'a>( env_list: Option>, defaults: &[&'a str], -) -> Vec<&'a str> { +) -> Vec> { if let Some(configured) = &env_list { // Check that the names are either all additions or all subtractions. let subtract_from_defaults = configured.iter().all(|c| c.starts_with("-")); @@ -141,12 +144,14 @@ pub fn build_allowed_env_list<'a>( for &d in defaults { let mentioned = configured.iter().any(|c| &c[start..] == d); if (add_from_defaults && mentioned) || (subtract_from_defaults && !mentioned) { - allowed.push(d); + allowed.push(Some(d)); + } else { + allowed.push(None); } } allowed } else { - defaults.to_vec() + defaults.iter().copied().map(Some).collect() } } diff --git a/fuzz/fuzz_targets/differential.rs b/fuzz/fuzz_targets/differential.rs index 802c8edeb9e8..5cc3fb4b4011 100644 --- a/fuzz/fuzz_targets/differential.rs +++ b/fuzz/fuzz_targets/differential.rs @@ -22,8 +22,8 @@ static SETUP: Once = Once::new(); // - ALLOWED_ENGINES=wasmi,spec cargo +nightly fuzz run ... // - ALLOWED_ENGINES=-v8 cargo +nightly fuzz run ... // - ALLOWED_MODULES=single-inst cargo +nightly fuzz run ... -static ALLOWED_ENGINES: Mutex> = Mutex::new(vec![]); -static ALLOWED_MODULES: Mutex> = Mutex::new(vec![]); +static ALLOWED_ENGINES: Mutex>> = Mutex::new(vec![]); +static ALLOWED_MODULES: Mutex>> = Mutex::new(vec![]); // Statistics about what's actually getting executed during fuzzing static STATS: RuntimeStats = RuntimeStats::new(); @@ -73,7 +73,13 @@ fn execute_one(data: &[u8]) -> Result<()> { // Choose an engine that Wasmtime will be differentially executed against. // The chosen engine is then created, which might update `config`, and // returned as a trait object. - let lhs = u.choose(&allowed_engines)?; + let lhs = match *u.choose(&allowed_engines)? { + Some(engine) => engine, + None => { + log::debug!("test case uses a runtime-disabled engine"); + return Ok(()); + } + }; let mut lhs = match engine::build(&mut u, lhs, &mut config)? { Some(engine) => engine, // The chosen engine does not have support compiled into the fuzzer, @@ -100,8 +106,12 @@ fn execute_one(data: &[u8]) -> Result<()> { panic!("unable to generate a module to fuzz against; check `ALLOWED_MODULES`") } let wasm = match *u.choose(&allowed_modules)? { - "wasm-smith" => build_wasm_smith_module(&mut u, &config)?, - "single-inst" => build_single_inst_module(&mut u, &config)?, + Some("wasm-smith") => build_wasm_smith_module(&mut u, &config)?, + Some("single-inst") => build_single_inst_module(&mut u, &config)?, + None => { + log::debug!("test case uses a runtime-disabled module strategy"); + return Ok(()); + } _ => unreachable!(), };