Change how env vars work in differential fuzzer (#9457)

* Change how env vars work in `differential` fuzzer

This commit updates the processing of the `ALLOWED_*` environment
variables to work differently than before. Previously the list of
engines and module-generation-strategies were filtered based on
`ALLOWED_*` environment variables but this meant that the meaning of a
fuzz input changed depending on environment variables. This commit
instead replaces the handling of these environment variables to ensure
that the meaning of the fuzz input doesn't change depending on their
values. Instead fuzz test cases are early-thrown-out if they request an
engine that's disabled or a module-generation-strategy that's disabled.

* Update fuzz/fuzz_targets/differential.rs

Co-authored-by: Trevor Elliott <[email protected]>

---------

Co-authored-by: Trevor Elliott <[email protected]>

crates/fuzzing/src/oracles/engine.rs

@@ -93,14 +93,17 @@ pub fn setup_engine_runtimes() {
 /// Build a list of allowed values from the given `defaults` using the
 /// `env_list`.
 ///
+/// The entries in `defaults` are preserved, in order, and are replaced with
+/// `None` in the returned list if they are disabled.
+///
 /// ```
 /// # use wasmtime_fuzzing::oracles::engine::build_allowed_env_list;
 /// // Passing no `env_list` returns the defaults:
-/// assert_eq!(build_allowed_env_list(None, &["a"]), vec!["a"]);
+/// assert_eq!(build_allowed_env_list(None, &["a"]), vec![Some("a")]);
 /// // We can build up a subset of the defaults:
-/// assert_eq!(build_allowed_env_list(Some(vec!["b".to_string()]), &["a","b"]), vec!["b"]);
+/// assert_eq!(build_allowed_env_list(Some(vec!["b".to_string()]), &["a","b"]), vec![None, Some("b")]);
 /// // Alternately we can subtract from the defaults:
-/// assert_eq!(build_allowed_env_list(Some(vec!["-a".to_string()]), &["a","b"]), vec!["b"]);
+/// assert_eq!(build_allowed_env_list(Some(vec!["-a".to_string()]), &["a","b"]), vec![None, Some("b")]);
 /// ```
 /// ```should_panic
 /// # use wasmtime_fuzzing::oracles::engine::build_allowed_env_list;
@@ -116,7 +119,7 @@ pub fn setup_engine_runtimes() {
 pub fn build_allowed_env_list<'a>(
     env_list: Option<Vec<String>>,
     defaults: &[&'a str],
-) -> Vec<&'a str> {
+) -> Vec<Option<&'a str>> {
     if let Some(configured) = &env_list {
         // Check that the names are either all additions or all subtractions.
         let subtract_from_defaults = configured.iter().all(|c| c.starts_with("-"));
@@ -141,12 +144,14 @@ pub fn build_allowed_env_list<'a>(
         for &d in defaults {
             let mentioned = configured.iter().any(|c| &c[start..] == d);
             if (add_from_defaults && mentioned) || (subtract_from_defaults && !mentioned) {
-                allowed.push(d);
+                allowed.push(Some(d));
+            } else {
+                allowed.push(None);
             }
         }
         allowed
     } else {
-        defaults.to_vec()
+        defaults.iter().copied().map(Some).collect()
     }
 }
 

fuzz/fuzz_targets/differential.rs

@@ -22,8 +22,8 @@ static SETUP: Once = Once::new();
 // - ALLOWED_ENGINES=wasmi,spec cargo +nightly fuzz run ...
 // - ALLOWED_ENGINES=-v8 cargo +nightly fuzz run ...
 // - ALLOWED_MODULES=single-inst cargo +nightly fuzz run ...
-static ALLOWED_ENGINES: Mutex<Vec<&str>> = Mutex::new(vec![]);
-static ALLOWED_MODULES: Mutex<Vec<&str>> = Mutex::new(vec![]);
+static ALLOWED_ENGINES: Mutex<Vec<Option<&str>>> = Mutex::new(vec![]);
+static ALLOWED_MODULES: Mutex<Vec<Option<&str>>> = Mutex::new(vec![]);
 
 // Statistics about what's actually getting executed during fuzzing
 static STATS: RuntimeStats = RuntimeStats::new();
@@ -73,7 +73,13 @@ fn execute_one(data: &[u8]) -> Result<()> {
     // Choose an engine that Wasmtime will be differentially executed against.
     // The chosen engine is then created, which might update `config`, and
     // returned as a trait object.
-    let lhs = u.choose(&allowed_engines)?;
+    let lhs = match *u.choose(&allowed_engines)? {
+        Some(engine) => engine,
+        None => {
+            log::debug!("test case uses a runtime-disabled engine");
+            return Ok(());
+        }
+    };
     let mut lhs = match engine::build(&mut u, lhs, &mut config)? {
         Some(engine) => engine,
         // The chosen engine does not have support compiled into the fuzzer,
@@ -100,8 +106,12 @@ fn execute_one(data: &[u8]) -> Result<()> {
         panic!("unable to generate a module to fuzz against; check `ALLOWED_MODULES`")
     }
     let wasm = match *u.choose(&allowed_modules)? {
-        "wasm-smith" => build_wasm_smith_module(&mut u, &config)?,
-        "single-inst" => build_single_inst_module(&mut u, &config)?,
+        Some("wasm-smith") => build_wasm_smith_module(&mut u, &config)?,
+        Some("single-inst") => build_single_inst_module(&mut u, &config)?,
+        None => {
+            log::debug!("test case uses a runtime-disabled module strategy");
+            return Ok(());
+        }
         _ => unreachable!(),
     };