Skip to content

Commit

Permalink
fuzzing: provide a better error management
Browse files Browse the repository at this point in the history
  • Loading branch information
@sylvestre
sylvestre committed Nov 9, 2023
1 parent 8698be5 commit 06c4b34
Show file tree
Hide file tree
Showing 3 changed files with 89 additions and 86 deletions.
54 changes: 54 additions & 0 deletions fuzz/fuzz_targets/fuzz_common.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,7 @@ where
}

uumain_exit_status = uumain_function(args.to_owned().into_iter());

io::stdout().flush().unwrap();
io::stderr().flush().unwrap();

Expand Down Expand Up @@ -180,3 +181,56 @@ pub fn run_gnu_cmd(
Err((stdout, stderr, exit_code))
}
}

pub fn compare_result(
test_type: &str,
input: &str,
rust_stdout: &str,
gnu_stdout: &str,
rust_stderr: &str,
gnu_stderr: &str,
rust_exit_code: i32,
gnu_exit_code: i32,
fail_on_stderr_diff: bool,
) {
println!("Test Type: {}", test_type);
println!("Input: {}", input);

let mut discrepancies = Vec::new();
let mut should_panic = false;

if rust_stdout.trim() != gnu_stdout.trim() {
discrepancies.push("stdout differs");
println!("Rust stdout: {}", rust_stdout);
println!("GNU stdout: {}", gnu_stdout);
should_panic = true;
}
if rust_stderr.trim() != gnu_stderr.trim() {
discrepancies.push("stderr differs");
println!("Rust stderr: {}", rust_stderr);
println!("GNU stderr: {}", gnu_stderr);
if fail_on_stderr_diff {
should_panic = true;
}
}
if rust_exit_code != gnu_exit_code {
discrepancies.push("exit code differs");
println!("Rust exit code: {}", rust_exit_code);
println!("GNU exit code: {}", gnu_exit_code);
should_panic = true;
}

if discrepancies.is_empty() {
println!("All outputs and exit codes matched.");
} else {
println!("Discrepancy detected: {}", discrepancies.join(", "));
if should_panic {
panic!("Test failed for {}: {}", test_type, input);
} else {
println!(
"Test completed with discrepancies for {}: {}",
test_type, input
);
}
}
}
61 changes: 17 additions & 44 deletions fuzz/fuzz_targets/fuzz_expr.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,9 @@ use rand::Rng;
use std::{env, ffi::OsString};

mod fuzz_common;
use crate::fuzz_common::{generate_and_run_uumain, run_gnu_cmd};
use crate::fuzz_common::{compare_result, generate_and_run_uumain, run_gnu_cmd};

static CMD_PATH: &str = "expr";
static CMD_PATH: &str = "/usr/bin/expr";

fn generate_random_string(max_length: usize) -> String {
let mut rng = rand::thread_rng();
Expand Down Expand Up @@ -84,52 +84,25 @@ fuzz_target!(|_data: &[u8]| {
let mut args = vec![OsString::from("expr")];
args.extend(expr.split_whitespace().map(OsString::from));

let (rust_stdout, rust_stderr, uumain_exit_code) = generate_and_run_uumain(&args, uumain);

// Use C locale to avoid false positives, like in https://github.com/uutils/coreutils/issues/5378,
// because uutils expr doesn't support localization yet
// TODO remove once uutils expr supports localization
env::set_var("LC_COLLATE", "C");

// Run GNU expr with the provided arguments and compare the output
match run_gnu_cmd(CMD_PATH, &args[1..], true) {
Ok((gnu_stdout, gnu_stderr, gnu_exit_code)) => {
let gnu_stdout = gnu_stdout.trim().to_owned();
if uumain_exit_code != gnu_exit_code {
println!("Expression: {}", expr);

println!("GNU stderr: {}", gnu_stderr);
println!("Rust stderr: {}", rust_stderr);

println!("Rust code: {}", uumain_exit_code);
println!("GNU code: {}", gnu_exit_code);
panic!("Different error codes");
}
if rust_stdout == gnu_stdout {
println!(
"Outputs matched for expression: {} => Result: {}",
expr, rust_stdout
);
} else {
println!("Expression: {}", expr);
println!("Rust output: {}", rust_stdout);
println!("GNU output: {}", gnu_stdout);
panic!("Different output between Rust & GNU");
}
}
let (rust_stdout, rust_stderr, uumain_exit_code) = generate_and_run_uumain(&args, uumain);

Err((_gnu_stdout, gnu_stderr, _gnu_exit_code)) => {
if rust_stderr == gnu_stderr {
println!(
"GNU execution failed for input: {} stderr: {}",
expr, rust_stderr
);
} else {
println!("Input: {}", expr);
println!("Rust stderr: {}", rust_stderr);
println!("GNU stderr: {}", gnu_stderr);
panic!("Different stderr between Rust & GNU");
}
}
}
let (gnu_stdout, gnu_stderr, gnu_exit_code) =
run_gnu_cmd(CMD_PATH, &args[1..], false).unwrap_or_else(|e| e);

compare_result(
"expr",
&format!("{:?}", &args[1..]),
&rust_stdout,
&gnu_stdout,
&rust_stderr,
&gnu_stderr,
uumain_exit_code,
gnu_exit_code,
false, // Set to true if you want to fail on stderr diff
);
});
60 changes: 18 additions & 42 deletions fuzz/fuzz_targets/fuzz_test.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ use rand::Rng;
use std::ffi::OsString;

mod fuzz_common;
use crate::fuzz_common::{generate_and_run_uumain, run_gnu_cmd};
use crate::fuzz_common::{compare_result, generate_and_run_uumain, run_gnu_cmd};

#[derive(PartialEq, Debug, Clone)]
enum ArgType {
Expand All @@ -26,7 +26,7 @@ enum ArgType {
// Add any other types as needed
}

static CMD_PATH: &str = "test";
static CMD_PATH: &str = "/usr/bin/test";

fn generate_random_string(max_length: usize) -> String {
let mut rng = rand::thread_rng();
Expand Down Expand Up @@ -204,44 +204,20 @@ fuzz_target!(|_data: &[u8]| {
args.push(OsString::from(generate_test_arg()));
}

let (rust_stdout, rust_stderr, uumain_exit_status) = generate_and_run_uumain(&args, uumain);

// Run GNU test with the provided arguments and compare the output
match run_gnu_cmd(CMD_PATH, &args[1..], false) {
Ok((gnu_stdout, gnu_stderr, gnu_exit_status)) => {
let gnu_stdout = gnu_stdout.trim().to_owned();
println!("gnu_exit_status {}", gnu_exit_status);
println!("uumain_exit_status {}", uumain_exit_status);
if rust_stdout != gnu_stdout || uumain_exit_status != gnu_exit_status {
println!("Discrepancy detected!");
println!("Test: {:?}", &args[1..]);
println!("Rust output: {}", rust_stdout);
println!("GNU output: {}", gnu_stdout);

println!("Rust stderr: {}", rust_stderr);
println!("GNU stderr: {}", gnu_stderr);
println!("My exit status: {}", uumain_exit_status);
println!("GNU exit status: {}", gnu_exit_status);
panic!();
} else {
println!(
"Outputs and exit statuses matched for expression {:?}",
&args[1..]
);
}
}
Err((_gnu_stdout, gnu_stderr, _gnu_exit_code)) => {
if rust_stderr == gnu_stderr {
println!(
"GNU execution failed for input: {:?} stderr: {}",
args, rust_stderr
);
} else {
println!("Input: {:?}", args);
println!("Rust stderr: {}", rust_stderr);
println!("GNU stderr: {}", gnu_stderr);
panic!("Different stderr between Rust & GNU");
}
}
}
let (rust_stdout, rust_stderr, uumain_exit_code) = generate_and_run_uumain(&args, uumain);

let (gnu_stdout, gnu_stderr, gnu_exit_code) =
run_gnu_cmd(CMD_PATH, &args[1..], false).unwrap_or_else(|e| e);

compare_result(
"test",
&format!("{:?}", &args[1..]),
&rust_stdout,
&gnu_stdout,
&rust_stderr,
&gnu_stderr,
uumain_exit_code,
gnu_exit_code,
false, // Set to true if you want to fail on stderr diff
);
});

0 comments on commit 06c4b34

Please sign in to comment.