From 01d38daa17929c1894dd8d4e117fd9059f44fc72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20=C4=8Euri=C5=A1?= Date: Fri, 13 Oct 2023 16:31:49 +0200 Subject: [PATCH 01/26] yara-x-dumper init --- Cargo.lock | 9 ++++++++ Cargo.toml | 2 ++ yara-x-cli/Cargo.toml | 1 + yara-x-cli/src/commands/dump.rs | 33 ++++++++++++++++++++++++++++ yara-x-cli/src/commands/mod.rs | 2 ++ yara-x-cli/src/main.rs | 2 ++ yara-x-dump/Cargo.toml | 19 +++++++++++++++++ yara-x-dump/src/lib.rs | 38 +++++++++++++++++++++++++++++++++ yara-x/fuzz/Cargo.toml | 2 ++ 9 files changed, 108 insertions(+) create mode 100644 yara-x-cli/src/commands/dump.rs create mode 100644 yara-x-dump/Cargo.toml create mode 100644 yara-x-dump/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index a567e0afa..e9c8173b9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4115,10 +4115,19 @@ dependencies = [ "wild", "yansi", "yara-x", + "yara-x-dump", "yara-x-fmt", "yara-x-parser", ] +[[package]] +name = "yara-x-dump" +version = "0.1.0" +dependencies = [ + "pretty_assertions", + "thiserror", +] + [[package]] name = "yara-x-fmt" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index f55841ba1..ca3bf5ad4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,6 +16,7 @@ rust-version = "1.70.0" members = [ "yara-x", "yara-x-cli", + "yara-x-dump", "yara-x-fmt", "yara-x-macros", "yara-x-parser", @@ -78,6 +79,7 @@ wasmtime = "12.0.2" yaml-rust = "0.4.5" yansi = "0.5.1" yara-x = { path = "yara-x" } +yara-x-dump = { path = "yara-x-dump" } yara-x-fmt = { path = "yara-x-fmt" } yara-x-macros = { path = "yara-x-macros" } yara-x-parser = { path = "yara-x-parser" } diff --git a/yara-x-cli/Cargo.toml b/yara-x-cli/Cargo.toml index 3a0ed7fff..48383413b 100644 --- a/yara-x-cli/Cargo.toml +++ b/yara-x-cli/Cargo.toml @@ -45,6 +45,7 @@ protobuf = { workspace = true } serde_json = { workspace = true } yansi = { workspace = true } yara-x = { workspace = true } +yara-x-dump = { workspace = true } yara-x-parser = { workspace = true, features = ["ascii-tree"] } yara-x-fmt = { workspace = true } diff --git a/yara-x-cli/src/commands/dump.rs b/yara-x-cli/src/commands/dump.rs new file mode 100644 index 000000000..007005918 --- /dev/null +++ b/yara-x-cli/src/commands/dump.rs @@ -0,0 +1,33 @@ +use clap::{arg, value_parser, ArgAction, ArgMatches, Command}; +use std::fs::File; +use std::io::stdin; +use std::path::PathBuf; + +use yara_x_dump::Dumper; + +pub fn dump() -> Command { + super::command("dump").about("Dump information about binary files").arg( + arg!() + .help("Path to binary file") + .value_parser(value_parser!(PathBuf)) + .action(ArgAction::Append) + .required(false), + ) +} + +pub fn exec_dump(args: &ArgMatches) -> anyhow::Result<()> { + let file = args.get_one::("FILE"); + + let dumper = Dumper::new(); + + if let Some(file) = file { + println!("Dumping file: {:?}", file.as_path()); + let input = File::open(file.as_path())?; + dumper.dump(input)?; + } else { + println!("Dumping stdin"); + dumper.dump(stdin())?; + } + + Ok(()) +} diff --git a/yara-x-cli/src/commands/mod.rs b/yara-x-cli/src/commands/mod.rs index f6370db38..269345bd7 100644 --- a/yara-x-cli/src/commands/mod.rs +++ b/yara-x-cli/src/commands/mod.rs @@ -1,12 +1,14 @@ mod check; mod compile; mod debug; +mod dump; mod fmt; mod scan; pub use check::*; pub use compile::*; pub use debug::*; +pub use dump::*; pub use fmt::*; pub use scan::*; diff --git a/yara-x-cli/src/main.rs b/yara-x-cli/src/main.rs index eb12f0f59..fe25b6ab2 100644 --- a/yara-x-cli/src/main.rs +++ b/yara-x-cli/src/main.rs @@ -43,6 +43,7 @@ fn main() -> anyhow::Result<()> { commands::check(), commands::debug(), commands::fmt(), + commands::dump(), ]) .get_matches_from(wild::args()); @@ -58,6 +59,7 @@ fn main() -> anyhow::Result<()> { Some(("debug", args)) => commands::exec_debug(args), Some(("check", args)) => commands::exec_check(args), Some(("fmt", args)) => commands::exec_fmt(args), + Some(("dump", args)) => commands::exec_dump(args), Some(("scan", args)) => commands::exec_scan(args), Some(("compile", args)) => commands::exec_compile(args), _ => unreachable!(), diff --git a/yara-x-dump/Cargo.toml b/yara-x-dump/Cargo.toml new file mode 100644 index 000000000..e43423346 --- /dev/null +++ b/yara-x-dump/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "yara-x-dump" +version = "0.1.0" +description = "A YARA module for dumping file information" +authors = ["Tomas Duris "] +edition = "2021" +readme.workspace = true +license.workspace = true +homepage.workspace = true +rust-version.workspace = true + +[lib] +crate-type = ["rlib", "cdylib"] + +[dependencies] +thiserror = { workspace = true } + +[dev-dependencies] +pretty_assertions = { workspace = true } \ No newline at end of file diff --git a/yara-x-dump/src/lib.rs b/yara-x-dump/src/lib.rs new file mode 100644 index 000000000..f778a7f75 --- /dev/null +++ b/yara-x-dump/src/lib.rs @@ -0,0 +1,38 @@ +use std::io; +use thiserror::Error; + +/// Errors returned by [`Dumper::dump`]. +#[derive(Error, Debug)] +#[allow(clippy::large_enum_variant)] +pub enum Error { + /// Error while reading from input. + #[error("Read error")] + ReadError(io::Error), +} + +/// Dumps information about binary files. +pub struct Dumper {} + +impl Default for Dumper { + fn default() -> Self { + Self::new() + } +} + +// Dumper public API. +impl Dumper { + /// Creates a new dumper. + pub fn new() -> Self { + Dumper {} + } + + pub fn dump(&self, mut input: R) -> Result<(), Error> + where + R: io::Read, + { + let mut buffer = Vec::new(); + input.read_to_end(&mut buffer).map_err(Error::ReadError)?; + println!("Buffer: {:?}", buffer); + Ok(()) + } +} diff --git a/yara-x/fuzz/Cargo.toml b/yara-x/fuzz/Cargo.toml index 0b8cd1a6d..28758c18f 100644 --- a/yara-x/fuzz/Cargo.toml +++ b/yara-x/fuzz/Cargo.toml @@ -1,8 +1,10 @@ [package] name = "yara-x-fuzz" version = "0.1.0" +authors = ["Tomas Duris "] publish = false edition = "2021" +description = "Fuzzing harness for YARA-X modules" [package.metadata] cargo-fuzz = true From 829ca56f6ffa85d8f562d41e652206702959e9ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20=C4=8Euri=C5=A1?= Date: Mon, 16 Oct 2023 15:56:26 +0200 Subject: [PATCH 02/26] add basic output for hardcoded module --- Cargo.lock | 3 +++ yara-x-dump/Cargo.toml | 4 ++++ yara-x-dump/src/lib.rs | 23 ++++++++++++++++++++++- 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index e9c8173b9..472261de7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4124,8 +4124,11 @@ dependencies = [ name = "yara-x-dump" version = "0.1.0" dependencies = [ + "indent", "pretty_assertions", + "protobuf", "thiserror", + "yara-x", ] [[package]] diff --git a/yara-x-dump/Cargo.toml b/yara-x-dump/Cargo.toml index e43423346..08f879720 100644 --- a/yara-x-dump/Cargo.toml +++ b/yara-x-dump/Cargo.toml @@ -14,6 +14,10 @@ crate-type = ["rlib", "cdylib"] [dependencies] thiserror = { workspace = true } +yara-x = { workspace = true } +protobuf = { workspace = true } + +indent = "0.1.1" [dev-dependencies] pretty_assertions = { workspace = true } \ No newline at end of file diff --git a/yara-x-dump/src/lib.rs b/yara-x-dump/src/lib.rs index f778a7f75..f6b6c9c3a 100644 --- a/yara-x-dump/src/lib.rs +++ b/yara-x-dump/src/lib.rs @@ -1,5 +1,7 @@ +use indent::indent_all_by; use std::io; use thiserror::Error; +use yara_x; /// Errors returned by [`Dumper::dump`]. #[derive(Error, Debug)] @@ -32,7 +34,26 @@ impl Dumper { { let mut buffer = Vec::new(); input.read_to_end(&mut buffer).map_err(Error::ReadError)?; - println!("Buffer: {:?}", buffer); + + // Construct a dummy YARA rule that only imports the module. + let rule = r#"import "macho" rule test { condition: false } "#; + + // Compile the rule. + let rules = yara_x::compile(rule).unwrap(); + + let mut scanner = yara_x::Scanner::new(&rules); + + let scan_results = + scanner.scan(&buffer).expect("scan should not fail"); + + let output = + scan_results.module_output("macho").unwrap_or_else(|| { + panic!("module `macho` should produce some output") + }); + + // Get a text representation of the module's output. + let output = protobuf::text_format::print_to_string_pretty(output); + println!("{}", indent_all_by(4, output)); Ok(()) } } From 85ea33ae9b7ca3e728e144c459673a798d6ac6f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20=C4=8Euri=C5=A1?= Date: Thu, 19 Oct 2023 15:38:03 +0200 Subject: [PATCH 03/26] use all builtin modules for output --- yara-x-dump/src/lib.rs | 34 +++++++++++++++++++++++----------- yara-x/src/lib.rs | 2 ++ yara-x/src/modules/mod.rs | 4 ++++ 3 files changed, 29 insertions(+), 11 deletions(-) diff --git a/yara-x-dump/src/lib.rs b/yara-x-dump/src/lib.rs index f6b6c9c3a..52cbb0b85 100644 --- a/yara-x-dump/src/lib.rs +++ b/yara-x-dump/src/lib.rs @@ -35,25 +35,37 @@ impl Dumper { let mut buffer = Vec::new(); input.read_to_end(&mut buffer).map_err(Error::ReadError)?; - // Construct a dummy YARA rule that only imports the module. - let rule = r#"import "macho" rule test { condition: false } "#; + let import_statements = yara_x::get_builtin_modules_names() + .iter() + .map(|module_name| format!("import \"{}\"", module_name)) + .collect::>() + .join("\n"); + let rule = format!( + r#"{} rule test {{ condition: false }}"#, + import_statements + ); + + println!("{}", rule.as_str()); // Compile the rule. - let rules = yara_x::compile(rule).unwrap(); + let rules = yara_x::compile(rule.as_str()).unwrap(); let mut scanner = yara_x::Scanner::new(&rules); let scan_results = scanner.scan(&buffer).expect("scan should not fail"); - let output = - scan_results.module_output("macho").unwrap_or_else(|| { - panic!("module `macho` should produce some output") - }); - - // Get a text representation of the module's output. - let output = protobuf::text_format::print_to_string_pretty(output); - println!("{}", indent_all_by(4, output)); + for (mod_name, mod_output) in scan_results.module_outputs() { + // Get a text representation of the module's output. + println!( + "{}: {}", + mod_name, + indent_all_by( + 4, + protobuf::text_format::print_to_string_pretty(mod_output) + ) + ); + } Ok(()) } } diff --git a/yara-x/src/lib.rs b/yara-x/src/lib.rs index 021ad2377..d36ce55e5 100644 --- a/yara-x/src/lib.rs +++ b/yara-x/src/lib.rs @@ -49,6 +49,8 @@ pub use compiler::Error; pub use compiler::Rules; pub use compiler::SerializationError; +pub use modules::get_builtin_modules_names; + pub use scanner::Match; pub use scanner::Matches; pub use scanner::MatchingRules; diff --git a/yara-x/src/modules/mod.rs b/yara-x/src/modules/mod.rs index ac42fc6c9..a4ec43f01 100644 --- a/yara-x/src/modules/mod.rs +++ b/yara-x/src/modules/mod.rs @@ -26,6 +26,10 @@ include!("modules.rs"); /// Type of module's main function. type MainFn = fn(&[u8]) -> Box; +pub fn get_builtin_modules_names() -> Vec<&'static str> { + BUILTIN_MODULES.keys().cloned().collect() +} + /// Describes a YARA module. pub(crate) struct Module { /// Pointer to the module's main function. From 2222d9f9c7c1c1db0274b51ce30f8979e59e3de3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20=C4=8Euri=C5=A1?= Date: Fri, 20 Oct 2023 17:32:47 +0200 Subject: [PATCH 04/26] user defined modules WiP --- yara-x-cli/src/commands/dump.rs | 41 +++++++++++++++++++++++++-------- yara-x-dump/src/lib.rs | 11 +++++++-- 2 files changed, 41 insertions(+), 11 deletions(-) diff --git a/yara-x-cli/src/commands/dump.rs b/yara-x-cli/src/commands/dump.rs index 007005918..ac1d5243c 100644 --- a/yara-x-cli/src/commands/dump.rs +++ b/yara-x-cli/src/commands/dump.rs @@ -3,30 +3,53 @@ use std::fs::File; use std::io::stdin; use std::path::PathBuf; +use yara_x::get_builtin_modules_names; use yara_x_dump::Dumper; pub fn dump() -> Command { - super::command("dump").about("Dump information about binary files").arg( - arg!() - .help("Path to binary file") - .value_parser(value_parser!(PathBuf)) - .action(ArgAction::Append) - .required(false), - ) + super::command("dump") + .about("Dump information about binary files") + .arg( + arg!() + .help("Path to binary file") + .value_parser(value_parser!(PathBuf)) + .action(ArgAction::Append) + .required(false), + ) + .arg( + arg!(--"modules" ) + .help("Name of the module or comma-separated list of modules to be used for parsing") + .value_parser(value_parser!(String)) + .value_delimiter(',') + .required(false), + ) } pub fn exec_dump(args: &ArgMatches) -> anyhow::Result<()> { let file = args.get_one::("FILE"); + let modules = args + .get_many::("modules") + .unwrap_or_default() + .map(|s| s.to_string()) + .collect::>(); + + // Validate modules + let supported_modules = get_builtin_modules_names(); + for module in &modules { + if !supported_modules.contains(&module.as_str()) { + anyhow::bail!("Unsupported module: {}. Supported modules for --modules argument are: {}", module, supported_modules.join(", ")); + } + } let dumper = Dumper::new(); if let Some(file) = file { println!("Dumping file: {:?}", file.as_path()); let input = File::open(file.as_path())?; - dumper.dump(input)?; + dumper.dump(input, modules)?; } else { println!("Dumping stdin"); - dumper.dump(stdin())?; + dumper.dump(stdin(), modules)?; } Ok(()) diff --git a/yara-x-dump/src/lib.rs b/yara-x-dump/src/lib.rs index 52cbb0b85..b7009a841 100644 --- a/yara-x-dump/src/lib.rs +++ b/yara-x-dump/src/lib.rs @@ -28,25 +28,31 @@ impl Dumper { Dumper {} } - pub fn dump(&self, mut input: R) -> Result<(), Error> + pub fn dump( + &self, + mut input: R, + modules: Vec, + ) -> Result<(), Error> where R: io::Read, { let mut buffer = Vec::new(); input.read_to_end(&mut buffer).map_err(Error::ReadError)?; + println!("desired modules: {}", modules.join(", ")); + // Create a rule that imports all the built-in modules. let import_statements = yara_x::get_builtin_modules_names() .iter() .map(|module_name| format!("import \"{}\"", module_name)) .collect::>() .join("\n"); + // Create a dummy rule let rule = format!( r#"{} rule test {{ condition: false }}"#, import_statements ); - println!("{}", rule.as_str()); // Compile the rule. let rules = yara_x::compile(rule.as_str()).unwrap(); @@ -55,6 +61,7 @@ impl Dumper { let scan_results = scanner.scan(&buffer).expect("scan should not fail"); + // Iterate over the modules' outputs. for (mod_name, mod_output) in scan_results.module_outputs() { // Get a text representation of the module's output. println!( From fc5945dad665c5d146664ae1533be28f0bdd0da9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20=C4=8Euri=C5=A1?= Date: Mon, 23 Oct 2023 10:46:37 +0200 Subject: [PATCH 05/26] user defined modules --- Cargo.lock | 1 + yara-x-cli/src/commands/dump.rs | 29 +++++++++-------------------- yara-x-cli/src/commands/mod.rs | 17 ++++++++++++++++- yara-x-dump/Cargo.toml | 1 + yara-x-dump/src/lib.rs | 19 ++++++++++++++----- 5 files changed, 41 insertions(+), 26 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 472261de7..eff1b597b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4128,6 +4128,7 @@ dependencies = [ "pretty_assertions", "protobuf", "thiserror", + "yansi", "yara-x", ] diff --git a/yara-x-cli/src/commands/dump.rs b/yara-x-cli/src/commands/dump.rs index ac1d5243c..070740195 100644 --- a/yara-x-cli/src/commands/dump.rs +++ b/yara-x-cli/src/commands/dump.rs @@ -1,11 +1,12 @@ -use clap::{arg, value_parser, ArgAction, ArgMatches, Command}; +use clap::{arg, value_parser, Arg, ArgAction, ArgMatches, Command}; use std::fs::File; use std::io::stdin; use std::path::PathBuf; -use yara_x::get_builtin_modules_names; use yara_x_dump::Dumper; +use crate::commands::modules_parser; + pub fn dump() -> Command { super::command("dump") .about("Dump information about binary files") @@ -17,29 +18,17 @@ pub fn dump() -> Command { .required(false), ) .arg( - arg!(--"modules" ) - .help("Name of the module or comma-separated list of modules to be used for parsing") - .value_parser(value_parser!(String)) - .value_delimiter(',') - .required(false), + Arg::new("modules") + .long("modules") + .help("Name of the module or comma-separated list of modules to be used for parsing") + .required(false) + .value_parser(modules_parser), ) } pub fn exec_dump(args: &ArgMatches) -> anyhow::Result<()> { let file = args.get_one::("FILE"); - let modules = args - .get_many::("modules") - .unwrap_or_default() - .map(|s| s.to_string()) - .collect::>(); - - // Validate modules - let supported_modules = get_builtin_modules_names(); - for module in &modules { - if !supported_modules.contains(&module.as_str()) { - anyhow::bail!("Unsupported module: {}. Supported modules for --modules argument are: {}", module, supported_modules.join(", ")); - } - } + let modules = args.get_one::>("modules"); let dumper = Dumper::new(); diff --git a/yara-x-cli/src/commands/mod.rs b/yara-x-cli/src/commands/mod.rs index 269345bd7..805848782 100644 --- a/yara-x-cli/src/commands/mod.rs +++ b/yara-x-cli/src/commands/mod.rs @@ -23,7 +23,7 @@ use crossterm::tty::IsTty; use serde_json::Value; use superconsole::{Component, Line, Lines, Span, SuperConsole}; -use yara_x::{Compiler, Rules}; +use yara_x::{get_builtin_modules_names, Compiler, Rules}; use yara_x_parser::SourceCode; use crate::walk::DirWalker; @@ -39,6 +39,21 @@ pub fn command(name: &'static str) -> Command { ) } +fn modules_parser(option: &str) -> Result, anyhow::Error> { + let modules = option.split(',').map(|s| s.to_string()).collect::>(); + let supported_modules = get_builtin_modules_names(); + for module in &modules { + if !supported_modules.contains(&module.as_str()) { + anyhow::bail!( + "Unsupported module: {}. Supported modules for --modules argument are: {}", + module, + supported_modules.join(", ") + ); + } + } + Ok(modules) +} + fn external_var_parser( option: &str, ) -> Result<(String, serde_json::Value), anyhow::Error> { diff --git a/yara-x-dump/Cargo.toml b/yara-x-dump/Cargo.toml index 08f879720..4796d5b94 100644 --- a/yara-x-dump/Cargo.toml +++ b/yara-x-dump/Cargo.toml @@ -16,6 +16,7 @@ crate-type = ["rlib", "cdylib"] thiserror = { workspace = true } yara-x = { workspace = true } protobuf = { workspace = true } +yansi = { workspace = true } indent = "0.1.1" diff --git a/yara-x-dump/src/lib.rs b/yara-x-dump/src/lib.rs index b7009a841..de9368985 100644 --- a/yara-x-dump/src/lib.rs +++ b/yara-x-dump/src/lib.rs @@ -1,6 +1,7 @@ use indent::indent_all_by; use std::io; use thiserror::Error; +use yansi::Color::Cyan; use yara_x; /// Errors returned by [`Dumper::dump`]. @@ -31,7 +32,7 @@ impl Dumper { pub fn dump( &self, mut input: R, - modules: Vec, + modules: Option<&Vec>, ) -> Result<(), Error> where R: io::Read, @@ -39,9 +40,17 @@ impl Dumper { let mut buffer = Vec::new(); input.read_to_end(&mut buffer).map_err(Error::ReadError)?; - println!("desired modules: {}", modules.join(", ")); + let import_modules = if let Some(modules) = modules { + modules.clone() + } else { + yara_x::get_builtin_modules_names() + .into_iter() + .map(|s| s.to_string()) + .collect() + }; + // Create a rule that imports all the built-in modules. - let import_statements = yara_x::get_builtin_modules_names() + let import_statements = import_modules .iter() .map(|module_name| format!("import \"{}\"", module_name)) .collect::>() @@ -65,8 +74,8 @@ impl Dumper { for (mod_name, mod_output) in scan_results.module_outputs() { // Get a text representation of the module's output. println!( - "{}: {}", - mod_name, + ">>> {}:\n{}<<<", + Cyan.paint(mod_name).bold(), indent_all_by( 4, protobuf::text_format::print_to_string_pretty(mod_output) From 67f78d24d838290d77e3816d939c0222a225df9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20=C4=8Euri=C5=A1?= Date: Tue, 24 Oct 2023 12:05:12 +0200 Subject: [PATCH 06/26] multiple serilization formats implemented --- Cargo.lock | 27 ++++++++++++++++++++++++ yara-x-dump/Cargo.toml | 3 +++ yara-x-dump/src/lib.rs | 39 ++++++++++++++++++++++++++++------- yara-x-dump/src/serializer.rs | 38 ++++++++++++++++++++++++++++++++++ 4 files changed, 99 insertions(+), 8 deletions(-) create mode 100644 yara-x-dump/src/serializer.rs diff --git a/Cargo.lock b/Cargo.lock index eff1b597b..be42647b6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2303,6 +2303,17 @@ dependencies = [ "thiserror", ] +[[package]] +name = "protobuf-json-mapping" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "523039a90666b229b5260fb91c20686ef309b9d1b1fc3cacb283a0895753ec44" +dependencies = [ + "protobuf", + "protobuf-support", + "thiserror", +] + [[package]] name = "protobuf-parse" version = "3.3.0" @@ -2706,11 +2717,24 @@ version = "1.0.108" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b" dependencies = [ + "indexmap 2.1.0", "itoa", "ryu", "serde", ] +[[package]] +name = "serde_yaml" +version = "0.8.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "578a7433b776b56a35785ed5ce9a7e777ac0598aac5a6dd1b4b18a307c7fc71b" +dependencies = [ + "indexmap 1.9.3", + "ryu", + "serde", + "yaml-rust", +] + [[package]] name = "sha1" version = "0.10.6" @@ -4127,6 +4151,9 @@ dependencies = [ "indent", "pretty_assertions", "protobuf", + "protobuf-json-mapping", + "serde_json", + "serde_yaml", "thiserror", "yansi", "yara-x", diff --git a/yara-x-dump/Cargo.toml b/yara-x-dump/Cargo.toml index 4796d5b94..36723db1d 100644 --- a/yara-x-dump/Cargo.toml +++ b/yara-x-dump/Cargo.toml @@ -19,6 +19,9 @@ protobuf = { workspace = true } yansi = { workspace = true } indent = "0.1.1" +serde_json = { version = "1.0.1", features = ["preserve_order"] } +serde_yaml = { version = "0.8.26" } +protobuf-json-mapping = "3.3.0" [dev-dependencies] pretty_assertions = { workspace = true } \ No newline at end of file diff --git a/yara-x-dump/src/lib.rs b/yara-x-dump/src/lib.rs index de9368985..812f09011 100644 --- a/yara-x-dump/src/lib.rs +++ b/yara-x-dump/src/lib.rs @@ -1,9 +1,14 @@ -use indent::indent_all_by; +mod serializer; +use protobuf_json_mapping::print_to_string; +use serde_json; +use serde_yaml; use std::io; use thiserror::Error; use yansi::Color::Cyan; use yara_x; +use crate::serializer::get_serializer; + /// Errors returned by [`Dumper::dump`]. #[derive(Error, Debug)] #[allow(clippy::large_enum_variant)] @@ -11,6 +16,18 @@ pub enum Error { /// Error while reading from input. #[error("Read error")] ReadError(io::Error), + /// Error while serializing protobuf messages. + #[error("Serialization error")] + SerializationError(#[from] protobuf_json_mapping::PrintError), + /// Error while parsing JSON strings. + #[error("Parsing JSON error")] + ParsingJSONError(#[from] serde_json::Error), + /// Error while parsing YAML strings. + #[error("Parsing YAML error")] + ParsingYAMLError(#[from] serde_yaml::Error), + /// Error for unsupported serilization formats. + #[error("Unsupported serilization format")] + UnsupportedFormat, } /// Dumps information about binary files. @@ -29,6 +46,7 @@ impl Dumper { Dumper {} } + /// Dumps information about the binary file. pub fn dump( &self, mut input: R, @@ -40,6 +58,7 @@ impl Dumper { let mut buffer = Vec::new(); input.read_to_end(&mut buffer).map_err(Error::ReadError)?; + // Get the list of modules to import. let import_modules = if let Some(modules) = modules { modules.clone() } else { @@ -70,16 +89,20 @@ impl Dumper { let scan_results = scanner.scan(&buffer).expect("scan should not fail"); - // Iterate over the modules' outputs. + // Iterate over the modules' outputs and get serialized results to + // print. for (mod_name, mod_output) in scan_results.module_outputs() { - // Get a text representation of the module's output. + let desired_output = "json"; + let json_output = print_to_string(mod_output)?; + + let serializer = get_serializer(desired_output)?; + + let serialized_result = serializer.serialize(json_output)?; + println!( - ">>> {}:\n{}<<<", + ">>>\n{}:\n{}\n<<<", Cyan.paint(mod_name).bold(), - indent_all_by( - 4, - protobuf::text_format::print_to_string_pretty(mod_output) - ) + serialized_result ); } Ok(()) diff --git a/yara-x-dump/src/serializer.rs b/yara-x-dump/src/serializer.rs new file mode 100644 index 000000000..16c6ad173 --- /dev/null +++ b/yara-x-dump/src/serializer.rs @@ -0,0 +1,38 @@ +use crate::Error; + +// A struct that represents serializers +struct JsonSerializer; +struct YamlSerializer; + +// A trait for any type that can serialize a message +pub(crate) trait Serializer { + fn serialize(&self, message: String) -> Result; +} + +// Implement the trait for the JSON serializer +impl Serializer for JsonSerializer { + fn serialize(&self, message: String) -> Result { + let value = serde_json::from_str::(&message)?; + Ok(serde_json::to_string_pretty(&value)?) + } +} + +// Implement the trait for the YAML serializer +impl Serializer for YamlSerializer { + fn serialize(&self, message: String) -> Result { + let value = serde_json::from_str::(&message)?; + Ok(serde_yaml::to_string(&value)?) + } +} + +// A function that returns a trait object based on the format +pub fn get_serializer(format: &str) -> Result, Error> { + match format { + // Return a JSON serializer + "json" => Ok(Box::new(JsonSerializer)), + // Return a YAML serializer + "yaml" => Ok(Box::new(YamlSerializer)), + // Return an error if the format is unsupported + _ => Err(Error::UnsupportedFormat), + } +} From 46b45551412f46fcca13ef0fcdf7815aef5eeaee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20=C4=8Euri=C5=A1?= Date: Wed, 25 Oct 2023 13:23:40 +0200 Subject: [PATCH 07/26] toml,xml added, output can be specified with CLI --- Cargo.lock | 80 ++++++++++++++++++++++++++++++++- yara-x-cli/src/commands/dump.rs | 14 ++++-- yara-x-dump/Cargo.toml | 4 +- yara-x-dump/src/lib.rs | 14 ++++-- yara-x-dump/src/serializer.rs | 31 +++++++++++++ 5 files changed, 133 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index be42647b6..3211a6012 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1466,7 +1466,7 @@ dependencies = [ "log", "num-format", "once_cell", - "quick-xml", + "quick-xml 0.26.0", "rgb", "str_stack", ] @@ -2429,6 +2429,15 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "quick-xml" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11bafc859c6815fbaffbbbf4229ecb767ac913fecb27f9ad4343662e9ef099ea" +dependencies = [ + "memchr", +] + [[package]] name = "quick-xml" version = "0.26.0" @@ -2723,6 +2732,15 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_spanned" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12022b835073e5b11e90a14f86838ceb1c8fb0325b72416845c487ac0fa95e80" +dependencies = [ + "serde", +] + [[package]] name = "serde_yaml" version = "0.8.26" @@ -3182,6 +3200,40 @@ dependencies = [ "serde", ] +[[package]] +name = "toml" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ef75d881185fd2df4a040793927c153d863651108a93c7e17a9e591baa95cc6" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3550f4e9685620ac18a50ed434eb3aec30db8ba93b0287467bca5826ea25baf1" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.20.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "380f9e8120405471f7c9ad1860a713ef5ece6a670c7eae39225e477340f32fc4" +dependencies = [ + "indexmap 2.1.0", + "serde", + "serde_spanned", + "toml_datetime", + "winnow", +] + [[package]] name = "typenum" version = "1.17.0" @@ -3510,7 +3562,7 @@ dependencies = [ "rustix", "serde", "sha2 0.10.8", - "toml", + "toml 0.5.11", "windows-sys 0.48.0", "zstd", ] @@ -3997,6 +4049,15 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" +[[package]] +name = "winnow" +version = "0.5.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3b801d0e0a6726477cc207f60162da452f3a95adb368399bef20a946e06f65c" +dependencies = [ + "memchr", +] + [[package]] name = "wit-parser" version = "0.9.2" @@ -4022,6 +4083,19 @@ dependencies = [ "tap", ] +[[package]] +name = "xml2json-rs" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6ba4fb1256176d2eb02e25065efade8e43215a4d0743cb823991aea314077ea" +dependencies = [ + "lazy_static", + "quick-xml 0.23.1", + "regex", + "serde", + "serde_json", +] + [[package]] name = "yaml-rust" version = "0.4.5" @@ -4155,6 +4229,8 @@ dependencies = [ "serde_json", "serde_yaml", "thiserror", + "toml 0.8.4", + "xml2json-rs", "yansi", "yara-x", ] diff --git a/yara-x-cli/src/commands/dump.rs b/yara-x-cli/src/commands/dump.rs index 070740195..889121ac8 100644 --- a/yara-x-cli/src/commands/dump.rs +++ b/yara-x-cli/src/commands/dump.rs @@ -1,4 +1,4 @@ -use clap::{arg, value_parser, Arg, ArgAction, ArgMatches, Command}; +use clap::{arg, value_parser, Arg, ArgMatches, Command}; use std::fs::File; use std::io::stdin; use std::path::PathBuf; @@ -14,7 +14,12 @@ pub fn dump() -> Command { arg!() .help("Path to binary file") .value_parser(value_parser!(PathBuf)) - .action(ArgAction::Append) + .required(false), + ) + .arg( + arg!(-o --"output-format" ) + .help("Desired output format") + .value_parser(value_parser!(String)) .required(false), ) .arg( @@ -28,6 +33,7 @@ pub fn dump() -> Command { pub fn exec_dump(args: &ArgMatches) -> anyhow::Result<()> { let file = args.get_one::("FILE"); + let output_format = args.get_one::("output-format"); let modules = args.get_one::>("modules"); let dumper = Dumper::new(); @@ -35,10 +41,10 @@ pub fn exec_dump(args: &ArgMatches) -> anyhow::Result<()> { if let Some(file) = file { println!("Dumping file: {:?}", file.as_path()); let input = File::open(file.as_path())?; - dumper.dump(input, modules)?; + dumper.dump(input, modules, output_format)?; } else { println!("Dumping stdin"); - dumper.dump(stdin(), modules)?; + dumper.dump(stdin(), modules, output_format)?; } Ok(()) diff --git a/yara-x-dump/Cargo.toml b/yara-x-dump/Cargo.toml index 36723db1d..8e12bb49d 100644 --- a/yara-x-dump/Cargo.toml +++ b/yara-x-dump/Cargo.toml @@ -20,8 +20,10 @@ yansi = { workspace = true } indent = "0.1.1" serde_json = { version = "1.0.1", features = ["preserve_order"] } -serde_yaml = { version = "0.8.26" } +serde_yaml = "0.8.26" +toml = "0.8.4" protobuf-json-mapping = "3.3.0" +xml2json-rs = "1.0.1" [dev-dependencies] pretty_assertions = { workspace = true } \ No newline at end of file diff --git a/yara-x-dump/src/lib.rs b/yara-x-dump/src/lib.rs index 812f09011..3ef67c4ac 100644 --- a/yara-x-dump/src/lib.rs +++ b/yara-x-dump/src/lib.rs @@ -25,6 +25,12 @@ pub enum Error { /// Error while parsing YAML strings. #[error("Parsing YAML error")] ParsingYAMLError(#[from] serde_yaml::Error), + /// Error while parsing TOML strings. + #[error("Parsing TOML error")] + ParsingTOMLError(#[from] toml::ser::Error), + /// Error while parsing XML strings. + #[error("Parsing XML error")] + ParsingXMLError(#[from] xml2json_rs::X2JError), /// Error for unsupported serilization formats. #[error("Unsupported serilization format")] UnsupportedFormat, @@ -51,6 +57,7 @@ impl Dumper { &self, mut input: R, modules: Option<&Vec>, + output_format: Option<&String>, ) -> Result<(), Error> where R: io::Read, @@ -92,10 +99,11 @@ impl Dumper { // Iterate over the modules' outputs and get serialized results to // print. for (mod_name, mod_output) in scan_results.module_outputs() { - let desired_output = "json"; let json_output = print_to_string(mod_output)?; - - let serializer = get_serializer(desired_output)?; + let serializer = get_serializer(match output_format { + Some(format) => format, + None => "json", + })?; let serialized_result = serializer.serialize(json_output)?; diff --git a/yara-x-dump/src/serializer.rs b/yara-x-dump/src/serializer.rs index 16c6ad173..283f77a9b 100644 --- a/yara-x-dump/src/serializer.rs +++ b/yara-x-dump/src/serializer.rs @@ -3,6 +3,8 @@ use crate::Error; // A struct that represents serializers struct JsonSerializer; struct YamlSerializer; +struct TomlSerializer; +struct XmlSerializer; // A trait for any type that can serialize a message pub(crate) trait Serializer { @@ -25,6 +27,31 @@ impl Serializer for YamlSerializer { } } +// Implement the trait for the TOML serializer +impl Serializer for TomlSerializer { + fn serialize(&self, message: String) -> Result { + let value = serde_json::from_str::(&message)?; + Ok(toml::to_string_pretty(&value)?) + } +} + +// Implement the trait for the XML serializer +impl Serializer for XmlSerializer { + fn serialize(&self, message: String) -> Result { + let mut xml_builder = xml2json_rs::XmlConfig::new() + .rendering(xml2json_rs::Indentation::new(b' ', 2)) + .decl(xml2json_rs::Declaration::new( + xml2json_rs::Version::XML10, + Some(xml2json_rs::Encoding::UTF8), + Some(true), + )) + .root_name("file") + .finalize(); + let xml = xml_builder.build_from_json_string(&message)?; + Ok(xml) + } +} + // A function that returns a trait object based on the format pub fn get_serializer(format: &str) -> Result, Error> { match format { @@ -32,6 +59,10 @@ pub fn get_serializer(format: &str) -> Result, Error> { "json" => Ok(Box::new(JsonSerializer)), // Return a YAML serializer "yaml" => Ok(Box::new(YamlSerializer)), + // Return a TOML serializer + "toml" => Ok(Box::new(TomlSerializer)), + // Return an XML serializer + "xml" => Ok(Box::new(XmlSerializer)), // Return an error if the format is unsupported _ => Err(Error::UnsupportedFormat), } From 04911a16e4120d47906741073a94ae20b60628ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20=C4=8Euri=C5=A1?= Date: Mon, 30 Oct 2023 13:22:00 +0100 Subject: [PATCH 08/26] human-readable format WiP --- Cargo.lock | 1 + yara-x-dump/Cargo.toml | 1 + yara-x-dump/src/lib.rs | 31 ++++++++++++++---- yara-x-dump/src/serializer.rs | 47 +++++++++++++++++++++++++++ yara-x-proto/src/yara.proto | 1 + yara-x/src/modules/protos/macho.proto | 2 +- 6 files changed, 75 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3211a6012..368cc4f24 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4233,6 +4233,7 @@ dependencies = [ "xml2json-rs", "yansi", "yara-x", + "yara-x-proto", ] [[package]] diff --git a/yara-x-dump/Cargo.toml b/yara-x-dump/Cargo.toml index 8e12bb49d..f2c049c2b 100644 --- a/yara-x-dump/Cargo.toml +++ b/yara-x-dump/Cargo.toml @@ -15,6 +15,7 @@ crate-type = ["rlib", "cdylib"] [dependencies] thiserror = { workspace = true } yara-x = { workspace = true } +yara-x-proto = { workspace = true } protobuf = { workspace = true } yansi = { workspace = true } diff --git a/yara-x-dump/src/lib.rs b/yara-x-dump/src/lib.rs index 3ef67c4ac..a9aba1dbf 100644 --- a/yara-x-dump/src/lib.rs +++ b/yara-x-dump/src/lib.rs @@ -1,4 +1,5 @@ mod serializer; +use protobuf::reflect::MessageRef; use protobuf_json_mapping::print_to_string; use serde_json; use serde_yaml; @@ -7,7 +8,7 @@ use thiserror::Error; use yansi::Color::Cyan; use yara_x; -use crate::serializer::get_serializer; +use crate::serializer::{get_human_readable_output, get_serializer}; /// Errors returned by [`Dumper::dump`]. #[derive(Error, Debug)] @@ -99,13 +100,29 @@ impl Dumper { // Iterate over the modules' outputs and get serialized results to // print. for (mod_name, mod_output) in scan_results.module_outputs() { - let json_output = print_to_string(mod_output)?; - let serializer = get_serializer(match output_format { - Some(format) => format, - None => "json", - })?; + let serialized_result; - let serialized_result = serializer.serialize(json_output)?; + match output_format { + // Output is desired to be human-readable. + Some(format) if format == "human-readable" => { + serialized_result = get_human_readable_output( + &MessageRef::from(mod_output), + ); + } + // Serialize output for other given formats. + Some(format) => { + let json_output = print_to_string(mod_output)?; + let serializer = get_serializer(format)?; + + serialized_result = serializer.serialize(json_output)?; + } + // Default to human-readable output. + None => { + serialized_result = get_human_readable_output( + &MessageRef::from(mod_output), + ); + } + } println!( ">>>\n{}:\n{}\n<<<", diff --git a/yara-x-dump/src/serializer.rs b/yara-x-dump/src/serializer.rs index 283f77a9b..34985a95a 100644 --- a/yara-x-dump/src/serializer.rs +++ b/yara-x-dump/src/serializer.rs @@ -1,3 +1,8 @@ +use protobuf::reflect::MessageRef; +use protobuf::reflect::ReflectFieldRef; +use protobuf::reflect::ReflectValueRef; +use yara_x_proto::exts::field_options; + use crate::Error; // A struct that represents serializers @@ -67,3 +72,45 @@ pub fn get_serializer(format: &str) -> Result, Error> { _ => Err(Error::UnsupportedFormat), } } + +pub fn get_human_readable_output(msg: &MessageRef) -> String { + let desc = msg.descriptor_dyn(); + + for f in desc.fields() { + match f.get_reflect(&**msg) { + ReflectFieldRef::Map(map) => { + println!("{:?}", map) + } + ReflectFieldRef::Repeated(repeated) => { + println!("{:?}", repeated) + } + ReflectFieldRef::Optional(optional) => { + if let Some(options) = field_options.get(&f.proto().options) { + if options.hex_value.unwrap_or(false) { + match optional.value().unwrap() { + ReflectValueRef::Message(m) => {} + ReflectValueRef::Enum(d, v) => {} + ReflectValueRef::String(s) => {} + ReflectValueRef::Bytes(b) => {} + ReflectValueRef::I32(v) => {} + ReflectValueRef::I64(v) => {} + ReflectValueRef::U32(v) => { + println!("{:x}", v) + } + ReflectValueRef::U64(v) => {} + ReflectValueRef::Bool(v) => {} + ReflectValueRef::F32(v) => {} + ReflectValueRef::F64(v) => {} + } + } + } + println!("{:?}", optional.value()) + } + } + if let Some(options) = field_options.get(&f.proto().options) { + println!("{:?}", options) + } + } + + return "Test".to_string(); +} diff --git a/yara-x-proto/src/yara.proto b/yara-x-proto/src/yara.proto index f3620b896..ee27357d2 100644 --- a/yara-x-proto/src/yara.proto +++ b/yara-x-proto/src/yara.proto @@ -16,6 +16,7 @@ message ModuleOptions { message FieldOptions { optional string name = 1; optional bool ignore = 2; + optional bool hex_value = 3; } message MessageOptions { diff --git a/yara-x/src/modules/protos/macho.proto b/yara-x/src/modules/protos/macho.proto index 73eef853a..44cf1a58e 100644 --- a/yara-x/src/modules/protos/macho.proto +++ b/yara-x/src/modules/protos/macho.proto @@ -73,7 +73,7 @@ message File { message Macho { // Set Mach-O header and basic fields - optional uint32 magic = 1; + optional uint32 magic = 1 [(yara.field_options).hex_value = true]; optional uint32 cputype = 2; optional uint32 cpusubtype = 3; optional uint32 filetype = 4; From fb111a8b2c0e6630afc5f7e1b0199fcaee3cf3b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20=C4=8Euri=C5=A1?= Date: Tue, 31 Oct 2023 22:42:34 +0100 Subject: [PATCH 09/26] human-readable format is similar to YAML --- Cargo.lock | 1 + yara-x-dump/Cargo.toml | 1 + yara-x-dump/src/lib.rs | 12 ++- yara-x-dump/src/serializer.rs | 193 ++++++++++++++++++++++++++++------ 4 files changed, 173 insertions(+), 34 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 368cc4f24..b13e5e154 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4226,6 +4226,7 @@ dependencies = [ "pretty_assertions", "protobuf", "protobuf-json-mapping", + "protobuf-support", "serde_json", "serde_yaml", "thiserror", diff --git a/yara-x-dump/Cargo.toml b/yara-x-dump/Cargo.toml index f2c049c2b..783354871 100644 --- a/yara-x-dump/Cargo.toml +++ b/yara-x-dump/Cargo.toml @@ -24,6 +24,7 @@ serde_json = { version = "1.0.1", features = ["preserve_order"] } serde_yaml = "0.8.26" toml = "0.8.4" protobuf-json-mapping = "3.3.0" +protobuf-support = "3.3.0" xml2json-rs = "1.0.1" [dev-dependencies] diff --git a/yara-x-dump/src/lib.rs b/yara-x-dump/src/lib.rs index a9aba1dbf..c41cf0755 100644 --- a/yara-x-dump/src/lib.rs +++ b/yara-x-dump/src/lib.rs @@ -100,13 +100,17 @@ impl Dumper { // Iterate over the modules' outputs and get serialized results to // print. for (mod_name, mod_output) in scan_results.module_outputs() { - let serialized_result; + let mut serialized_result = String::new(); + let mut is_first_line = false; match output_format { // Output is desired to be human-readable. Some(format) if format == "human-readable" => { serialized_result = get_human_readable_output( &MessageRef::from(mod_output), + &mut serialized_result, + 0, + &mut is_first_line, ); } // Serialize output for other given formats. @@ -118,12 +122,16 @@ impl Dumper { } // Default to human-readable output. None => { - serialized_result = get_human_readable_output( + get_human_readable_output( &MessageRef::from(mod_output), + &mut serialized_result, + 0, + &mut is_first_line, ); } } + // Print the result. println!( ">>>\n{}:\n{}\n<<<", Cyan.paint(mod_name).bold(), diff --git a/yara-x-dump/src/serializer.rs b/yara-x-dump/src/serializer.rs index 34985a95a..fcc47de03 100644 --- a/yara-x-dump/src/serializer.rs +++ b/yara-x-dump/src/serializer.rs @@ -1,6 +1,9 @@ +use protobuf::descriptor::FieldDescriptorProto; use protobuf::reflect::MessageRef; use protobuf::reflect::ReflectFieldRef; use protobuf::reflect::ReflectValueRef; +use protobuf_support::text_format::quote_bytes_to; +use std::fmt::Write; use yara_x_proto::exts::field_options; use crate::Error; @@ -11,12 +14,12 @@ struct YamlSerializer; struct TomlSerializer; struct XmlSerializer; -// A trait for any type that can serialize a message +/// A trait for any type that can serialize a message pub(crate) trait Serializer { fn serialize(&self, message: String) -> Result; } -// Implement the trait for the JSON serializer +/// Implement the trait for the JSON serializer impl Serializer for JsonSerializer { fn serialize(&self, message: String) -> Result { let value = serde_json::from_str::(&message)?; @@ -24,7 +27,7 @@ impl Serializer for JsonSerializer { } } -// Implement the trait for the YAML serializer +/// Implement the trait for the YAML serializer impl Serializer for YamlSerializer { fn serialize(&self, message: String) -> Result { let value = serde_json::from_str::(&message)?; @@ -32,7 +35,7 @@ impl Serializer for YamlSerializer { } } -// Implement the trait for the TOML serializer +/// Implement the trait for the TOML serializer impl Serializer for TomlSerializer { fn serialize(&self, message: String) -> Result { let value = serde_json::from_str::(&message)?; @@ -40,9 +43,10 @@ impl Serializer for TomlSerializer { } } -// Implement the trait for the XML serializer +/// Implement the trait for the XML serializer impl Serializer for XmlSerializer { fn serialize(&self, message: String) -> Result { + // Create a new XML builder and get the XML let mut xml_builder = xml2json_rs::XmlConfig::new() .rendering(xml2json_rs::Indentation::new(b' ', 2)) .decl(xml2json_rs::Declaration::new( @@ -57,7 +61,7 @@ impl Serializer for XmlSerializer { } } -// A function that returns a trait object based on the format +/// A function that returns a trait object based on the format pub fn get_serializer(format: &str) -> Result, Error> { match format { // Return a JSON serializer @@ -73,44 +77,169 @@ pub fn get_serializer(format: &str) -> Result, Error> { } } -pub fn get_human_readable_output(msg: &MessageRef) -> String { +// Print a field name with correct indentation +fn print_field_name( + buf: &mut String, + field_name: &str, + indent: usize, + is_first_line: &mut bool, +) { + let mut indentation = " ".repeat(indent); + + if field_name.is_empty() { + return; + } + + if *is_first_line { + if !indentation.is_empty() { + indentation.pop(); + indentation.pop(); + } + write!(buf, "{}- {}: ", indentation, field_name).unwrap(); + *is_first_line = false; + } else { + write!(buf, "{}{}: ", indentation, field_name).unwrap(); + } +} + +// Print a field value with correct indentation for multiple value formats +fn print_field_value( + buf: &mut String, + value: ReflectValueRef, + is_hex: bool, + indent: usize, + is_first_line: &mut bool, +) { + match value { + ReflectValueRef::Message(m) => { + *is_first_line = true; + get_human_readable_output(&m, buf, indent + 1, is_first_line); + } + ReflectValueRef::Enum(d, v) => match d.value_by_number(v) { + Some(e) => writeln!(buf, "{}", e.name()).unwrap(), + None => writeln!(buf, "{}", v).unwrap(), + }, + ReflectValueRef::String(s) => { + quote_bytes_to(s.as_bytes(), buf); + buf.push_str("\n"); + } + ReflectValueRef::Bytes(b) => { + quote_bytes_to(b, buf); + buf.push_str("\n"); + } + ReflectValueRef::I32(v) => { + let field_value = + if is_hex { format!("0x{:x}", v) } else { v.to_string() }; + writeln!(buf, "{}", field_value).unwrap(); + } + ReflectValueRef::I64(v) => { + let field_value = + if is_hex { format!("0x{:x}", v) } else { v.to_string() }; + writeln!(buf, "{}", field_value).unwrap(); + } + ReflectValueRef::U32(v) => { + let field_value = + if is_hex { format!("0x{:x}", v) } else { v.to_string() }; + writeln!(buf, "{}", field_value).unwrap(); + } + ReflectValueRef::U64(v) => { + let field_value = + if is_hex { format!("0x{:x}", v) } else { v.to_string() }; + writeln!(buf, "{}", field_value).unwrap(); + } + ReflectValueRef::Bool(v) => { + writeln!(buf, "{}", v).unwrap(); + } + ReflectValueRef::F32(v) => { + writeln!(buf, "{}", v).unwrap(); + } + ReflectValueRef::F64(v) => { + writeln!(buf, "{}", v).unwrap(); + } + } +} + +// Print a field name and value +fn print_field( + buf: &mut String, + field_name: &str, + value: ReflectValueRef, + field_descriptor: &FieldDescriptorProto, + indent: usize, + is_first_line: &mut bool, +) { + let mut is_hex: bool = false; + + if let Some(options) = field_options.get(&field_descriptor.options) { + is_hex = options.hex_value.unwrap_or(false) + } + + print_field_name(buf, field_name, indent, is_first_line); + print_field_value(buf, value, is_hex, indent, is_first_line); +} + +/// A function that returns a human-readable output +pub fn get_human_readable_output( + msg: &MessageRef, + buf: &mut String, + indent: usize, + first_line: &mut bool, +) -> String { let desc = msg.descriptor_dyn(); + // Iterate over the fields of the message for f in desc.fields() { + let indentation = " ".repeat(indent); + + // Match the field type match f.get_reflect(&**msg) { ReflectFieldRef::Map(map) => { - println!("{:?}", map) + if map.is_empty() { + continue; + } + writeln!(buf, "{}{}:", indentation, f.name()).unwrap(); + for (k, v) in &map { + print_field( + buf, + "", + k, + &f.proto(), + indent + 1, + first_line, + ); + print_field( + buf, + "", + v, + &f.proto(), + indent + 1, + first_line, + ); + } } ReflectFieldRef::Repeated(repeated) => { - println!("{:?}", repeated) + if repeated.is_empty() { + continue; + } + writeln!(buf, "{}{}:", indentation, f.name()).unwrap(); + for v in repeated { + print_field(buf, "", v, &f.proto(), indent, first_line); + } } ReflectFieldRef::Optional(optional) => { - if let Some(options) = field_options.get(&f.proto().options) { - if options.hex_value.unwrap_or(false) { - match optional.value().unwrap() { - ReflectValueRef::Message(m) => {} - ReflectValueRef::Enum(d, v) => {} - ReflectValueRef::String(s) => {} - ReflectValueRef::Bytes(b) => {} - ReflectValueRef::I32(v) => {} - ReflectValueRef::I64(v) => {} - ReflectValueRef::U32(v) => { - println!("{:x}", v) - } - ReflectValueRef::U64(v) => {} - ReflectValueRef::Bool(v) => {} - ReflectValueRef::F32(v) => {} - ReflectValueRef::F64(v) => {} - } - } + if let Some(v) = optional.value() { + print_field( + buf, + f.name(), + v, + &f.proto(), + indent, + first_line, + ); } - println!("{:?}", optional.value()) } } - if let Some(options) = field_options.get(&f.proto().options) { - println!("{:?}", options) - } } - return "Test".to_string(); + return buf.to_string(); } From 7d5a844a3d50461d77fee16a72ffcedb16631e31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20=C4=8Euri=C5=A1?= Date: Wed, 1 Nov 2023 11:06:46 +0100 Subject: [PATCH 10/26] colors and comments in human readable format, timestamp support, macho dylib versions changed to strings and tests added --- Cargo.lock | 68 ++++++++++ yara-x-dump/Cargo.toml | 1 + yara-x-dump/src/serializer.rs | 124 +++++++++++++++--- yara-x-proto/src/yara.proto | 1 + yara-x/src/modules/macho/mod.rs | 26 +++- yara-x/src/modules/macho/tests/mod.rs | 7 + .../macho/tests/testdata/macho_ppc_file.out | 4 +- .../testdata/macho_x86_64_dylib_file.out | 8 +- .../macho/tests/testdata/macho_x86_file.out | 4 +- .../macho/tests/testdata/tiny_universal.out | 8 +- yara-x/src/modules/protos/macho.proto | 30 ++--- 11 files changed, 234 insertions(+), 47 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b13e5e154..79ed77f27 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -65,6 +65,21 @@ dependencies = [ "alloc-no-stdlib", ] +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + [[package]] name = "anes" version = "0.1.6" @@ -426,6 +441,20 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "chrono" +version = "0.4.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "js-sys", + "num-traits", + "wasm-bindgen", + "windows-targets 0.48.5", +] + [[package]] name = "ciborium" version = "0.2.1" @@ -551,6 +580,12 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "core-foundation-sys" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" + [[package]] name = "cpp_demangle" version = "0.3.5" @@ -1350,6 +1385,29 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" +[[package]] +name = "iana-time-zone" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8326b86b6cff230b97d0d312a6c40a60726df3332e721f72a1b035f451663b20" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + [[package]] name = "id-arena" version = "2.2.1" @@ -3902,6 +3960,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows-core" +version = "0.51.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1f8cf84f35d2db49a46868f947758c7a1138116f7fac3bc844f43ade1292e64" +dependencies = [ + "windows-targets 0.48.5", +] + [[package]] name = "windows-sys" version = "0.42.0" @@ -4222,6 +4289,7 @@ dependencies = [ name = "yara-x-dump" version = "0.1.0" dependencies = [ + "chrono", "indent", "pretty_assertions", "protobuf", diff --git a/yara-x-dump/Cargo.toml b/yara-x-dump/Cargo.toml index 783354871..a71b1d6db 100644 --- a/yara-x-dump/Cargo.toml +++ b/yara-x-dump/Cargo.toml @@ -26,6 +26,7 @@ toml = "0.8.4" protobuf-json-mapping = "3.3.0" protobuf-support = "3.3.0" xml2json-rs = "1.0.1" +chrono = "0.4.31" [dev-dependencies] pretty_assertions = { workspace = true } \ No newline at end of file diff --git a/yara-x-dump/src/serializer.rs b/yara-x-dump/src/serializer.rs index fcc47de03..827174ab3 100644 --- a/yara-x-dump/src/serializer.rs +++ b/yara-x-dump/src/serializer.rs @@ -1,9 +1,11 @@ +use chrono::prelude::{DateTime, NaiveDateTime, Utc}; use protobuf::descriptor::FieldDescriptorProto; use protobuf::reflect::MessageRef; use protobuf::reflect::ReflectFieldRef; use protobuf::reflect::ReflectValueRef; use protobuf_support::text_format::quote_bytes_to; use std::fmt::Write; +use yansi::Color::{Green, Yellow}; use yara_x_proto::exts::field_options; use crate::Error; @@ -14,6 +16,17 @@ struct YamlSerializer; struct TomlSerializer; struct XmlSerializer; +struct ValueOptions { + is_hex: bool, + is_timestamp: bool, +} + +impl ValueOptions { + fn new() -> Self { + ValueOptions { is_hex: false, is_timestamp: false } + } +} + /// A trait for any type that can serialize a message pub(crate) trait Serializer { fn serialize(&self, message: String) -> Result; @@ -95,7 +108,14 @@ fn print_field_name( indentation.pop(); indentation.pop(); } - write!(buf, "{}- {}: ", indentation, field_name).unwrap(); + write!( + buf, + "{}{} {}: ", + indentation, + Yellow.paint("-").bold(), + field_name + ) + .unwrap(); *is_first_line = false; } else { write!(buf, "{}{}: ", indentation, field_name).unwrap(); @@ -106,7 +126,7 @@ fn print_field_name( fn print_field_value( buf: &mut String, value: ReflectValueRef, - is_hex: bool, + value_options: &ValueOptions, indent: usize, is_first_line: &mut bool, ) { @@ -128,23 +148,74 @@ fn print_field_value( buf.push_str("\n"); } ReflectValueRef::I32(v) => { - let field_value = - if is_hex { format!("0x{:x}", v) } else { v.to_string() }; + let field_value = if value_options.is_hex { + format!("{} (0x{:x})", v, v) + } else if value_options.is_timestamp { + format!( + "{} ({})", + v, + DateTime::::from_naive_utc_and_offset( + NaiveDateTime::from_timestamp_opt(v as i64, 0) + .unwrap(), + Utc, + ) + ) + } else { + v.to_string() + }; writeln!(buf, "{}", field_value).unwrap(); } ReflectValueRef::I64(v) => { - let field_value = - if is_hex { format!("0x{:x}", v) } else { v.to_string() }; + let field_value = if value_options.is_hex { + format!("{} (0x{:x})", v, v) + } else if value_options.is_timestamp { + format!( + "{} ({})", + v, + DateTime::::from_naive_utc_and_offset( + NaiveDateTime::from_timestamp_opt(v, 0).unwrap(), + Utc, + ) + ) + } else { + v.to_string() + }; writeln!(buf, "{}", field_value).unwrap(); } ReflectValueRef::U32(v) => { - let field_value = - if is_hex { format!("0x{:x}", v) } else { v.to_string() }; + let field_value = if value_options.is_hex { + format!("{} (0x{:x})", v, v) + } else if value_options.is_timestamp { + format!( + "{} ({})", + v, + DateTime::::from_naive_utc_and_offset( + NaiveDateTime::from_timestamp_opt(v as i64, 0) + .unwrap(), + Utc, + ) + ) + } else { + v.to_string() + }; writeln!(buf, "{}", field_value).unwrap(); } ReflectValueRef::U64(v) => { - let field_value = - if is_hex { format!("0x{:x}", v) } else { v.to_string() }; + let field_value = if value_options.is_hex { + format!("{} (0x{:x})", v, v) + } else if value_options.is_timestamp { + format!( + "{} ({})", + v, + DateTime::::from_naive_utc_and_offset( + NaiveDateTime::from_timestamp_opt(v as i64, 0) + .unwrap(), + Utc, + ) + ) + } else { + v.to_string() + }; writeln!(buf, "{}", field_value).unwrap(); } ReflectValueRef::Bool(v) => { @@ -168,14 +239,16 @@ fn print_field( indent: usize, is_first_line: &mut bool, ) { - let mut is_hex: bool = false; - - if let Some(options) = field_options.get(&field_descriptor.options) { - is_hex = options.hex_value.unwrap_or(false) - } + let value_options = field_options + .get(&field_descriptor.options) + .map(|options| ValueOptions { + is_hex: options.hex_value.unwrap_or(false), + is_timestamp: options.timestamp.unwrap_or(false), + }) + .unwrap_or(ValueOptions::new()); print_field_name(buf, field_name, indent, is_first_line); - print_field_value(buf, value, is_hex, indent, is_first_line); + print_field_value(buf, value, &value_options, indent, is_first_line); } /// A function that returns a human-readable output @@ -189,7 +262,7 @@ pub fn get_human_readable_output( // Iterate over the fields of the message for f in desc.fields() { - let indentation = " ".repeat(indent); + let indentation = " ".repeat(indent); // Match the field type match f.get_reflect(&**msg) { @@ -221,7 +294,22 @@ pub fn get_human_readable_output( if repeated.is_empty() { continue; } - writeln!(buf, "{}{}:", indentation, f.name()).unwrap(); + writeln!( + buf, + "{}{} {} {}", + indentation, + Green.paint("# Nested").italic(), + Green.paint(f.name()).italic(), + Green.paint("structure").italic() + ) + .unwrap(); + writeln!( + buf, + "{}{}:", + indentation, + Yellow.paint(f.name()).bold() + ) + .unwrap(); for v in repeated { print_field(buf, "", v, &f.proto(), indent, first_line); } diff --git a/yara-x-proto/src/yara.proto b/yara-x-proto/src/yara.proto index ee27357d2..fa21f2b20 100644 --- a/yara-x-proto/src/yara.proto +++ b/yara-x-proto/src/yara.proto @@ -17,6 +17,7 @@ message FieldOptions { optional string name = 1; optional bool ignore = 2; optional bool hex_value = 3; + optional bool timestamp = 4; } message MessageOptions { diff --git a/yara-x/src/modules/macho/mod.rs b/yara-x/src/modules/macho/mod.rs index 126c5ccde..bf1da5036 100644 --- a/yara-x/src/modules/macho/mod.rs +++ b/yara-x/src/modules/macho/mod.rs @@ -526,6 +526,24 @@ fn should_swap_bytes(magic: u32) -> bool { matches!(magic, MH_CIGAM | MH_CIGAM_64 | FAT_CIGAM | FAT_CIGAM_64) } +/// Convert a decimal number representation to a version string representation. +/// The decimal number is expected to be in the format +/// `major(rest of digits).minor(previous 2 digits).patch(last 2 digits)`. +/// +/// # Arguments +/// +/// * `decimal_number`: The decimal number to convert. +/// +/// # Returns +/// +/// A string representation of the version number. +fn convert_to_version_string(decimal_number: u32) -> String { + let major = decimal_number >> 16; + let minor = (decimal_number >> 8) & 0xFF; + let patch = decimal_number & 0xFF; + format!("{}.{}.{}", major, minor, patch) +} + /// Convert a Mach-O Relative Virtual Address (RVA) to an offset within the /// file. /// @@ -1556,8 +1574,12 @@ fn handle_dylib_command( .to_string(), ), timestamp: Some(dy.dylib.timestamp), - compatibility_version: Some(dy.dylib.compatibility_version), - current_version: Some(dy.dylib.current_version), + compatibility_version: Some(convert_to_version_string( + dy.dylib.compatibility_version, + )), + current_version: Some(convert_to_version_string( + dy.dylib.current_version, + )), ..Default::default() }; macho_file.dylibs.push(dylib); diff --git a/yara-x/src/modules/macho/tests/mod.rs b/yara-x/src/modules/macho/tests/mod.rs index d1fa13dfd..e95c05b14 100644 --- a/yara-x/src/modules/macho/tests/mod.rs +++ b/yara-x/src/modules/macho/tests/mod.rs @@ -131,6 +131,13 @@ fn test_fat_is_32() { assert_eq!(fat_is_32(FAT_CIGAM_64), false); } +#[test] +fn test_convert_to_version_string() { + assert_eq!(convert_to_version_string(65536), "1.0.0"); + assert_eq!(convert_to_version_string(102895360), "1570.15.0"); + assert_eq!(convert_to_version_string(0), "0.0.0"); +} + #[test] fn test_should_swap_bytes() { assert_eq!(should_swap_bytes(MH_CIGAM), true); diff --git a/yara-x/src/modules/macho/tests/testdata/macho_ppc_file.out b/yara-x/src/modules/macho/tests/testdata/macho_ppc_file.out index 1798e72ce..336da472f 100644 --- a/yara-x/src/modules/macho/tests/testdata/macho_ppc_file.out +++ b/yara-x/src/modules/macho/tests/testdata/macho_ppc_file.out @@ -230,7 +230,7 @@ segments { dylibs { name: "/usr/lib/libSystem.B.dylib" timestamp: 1111112572 - compatibility_version: 65536 - current_version: 4653313 + compatibility_version: "1.0.0" + current_version: "71.1.1" } entry_point: 3768 diff --git a/yara-x/src/modules/macho/tests/testdata/macho_x86_64_dylib_file.out b/yara-x/src/modules/macho/tests/testdata/macho_x86_64_dylib_file.out index e7d9595c8..c5c88a074 100644 --- a/yara-x/src/modules/macho/tests/testdata/macho_x86_64_dylib_file.out +++ b/yara-x/src/modules/macho/tests/testdata/macho_x86_64_dylib_file.out @@ -78,12 +78,12 @@ segments { dylibs { name: "fact_x86_64.dylib" timestamp: 1 - compatibility_version: 0 - current_version: 0 + compatibility_version: "0.0.0" + current_version: "0.0.0" } dylibs { name: "/usr/lib/libSystem.B.dylib" timestamp: 2 - compatibility_version: 65536 - current_version: 79495168 + compatibility_version: "1.0.0" + current_version: "1213.0.0" } diff --git a/yara-x/src/modules/macho/tests/testdata/macho_x86_file.out b/yara-x/src/modules/macho/tests/testdata/macho_x86_file.out index e8ebc004b..b0e890827 100644 --- a/yara-x/src/modules/macho/tests/testdata/macho_x86_file.out +++ b/yara-x/src/modules/macho/tests/testdata/macho_x86_file.out @@ -152,8 +152,8 @@ segments { dylibs { name: "/usr/lib/libSystem.B.dylib" timestamp: 2 - compatibility_version: 65536 - current_version: 79495168 + compatibility_version: "1.0.0" + current_version: "1213.0.0" } entry_point: 3728 stack_size: 0 diff --git a/yara-x/src/modules/macho/tests/testdata/tiny_universal.out b/yara-x/src/modules/macho/tests/testdata/tiny_universal.out index 34befd9fc..fab280f7b 100644 --- a/yara-x/src/modules/macho/tests/testdata/tiny_universal.out +++ b/yara-x/src/modules/macho/tests/testdata/tiny_universal.out @@ -169,8 +169,8 @@ file { dylibs { name: "/usr/lib/libSystem.B.dylib" timestamp: 2 - compatibility_version: 65536 - current_version: 79495168 + compatibility_version: "1.0.0" + current_version: "1213.0.0" } entry_point: 3808 stack_size: 0 @@ -352,8 +352,8 @@ file { dylibs { name: "/usr/lib/libSystem.B.dylib" timestamp: 2 - compatibility_version: 65536 - current_version: 79495168 + compatibility_version: "1.0.0" + current_version: "1213.0.0" } entry_point: 3808 stack_size: 0 diff --git a/yara-x/src/modules/protos/macho.proto b/yara-x/src/modules/protos/macho.proto index 44cf1a58e..ecd36bbdd 100644 --- a/yara-x/src/modules/protos/macho.proto +++ b/yara-x/src/modules/protos/macho.proto @@ -11,21 +11,21 @@ option (yara.module_options) = { message Dylib { optional string name = 1; - optional uint32 timestamp = 2; - optional uint32 compatibility_version = 3; - optional uint32 current_version = 4; + optional uint32 timestamp = 2 [(yara.field_options).timestamp = true]; + optional string compatibility_version = 3; + optional string current_version = 4; } message Section { optional string segname = 1; optional string sectname = 2; - optional uint64 addr = 3; - optional uint64 size = 4; + optional uint64 addr = 3 [(yara.field_options).hex_value = true]; + optional uint64 size = 4 [(yara.field_options).hex_value = true]; optional uint32 offset = 5; optional uint32 align = 6; optional uint32 reloff = 7; optional uint32 nreloc = 8; - optional uint32 flags = 9; + optional uint32 flags = 9 [(yara.field_options).hex_value = true]; optional uint32 reserved1 = 10; optional uint32 reserved2 = 11; optional uint32 reserved3 = 12; @@ -35,14 +35,14 @@ message Segment { optional uint32 cmd = 1; optional uint32 cmdsize = 2; optional string segname = 3; - optional uint64 vmaddr = 4; - optional uint64 vmsize = 5; + optional uint64 vmaddr = 4 [(yara.field_options).hex_value = true]; + optional uint64 vmsize = 5 [(yara.field_options).hex_value = true]; optional uint64 fileoff = 6; optional uint64 filesize = 7; - optional uint32 maxprot = 8; - optional uint32 initprot = 9; + optional uint32 maxprot = 8 [(yara.field_options).hex_value = true]; + optional uint32 initprot = 9 [(yara.field_options).hex_value = true]; optional uint32 nsects = 10; - optional uint32 flags = 11; + optional uint32 flags = 11 [(yara.field_options).hex_value = true]; repeated Section sections = 12; } @@ -56,13 +56,13 @@ message FatArch { } message File { - optional uint32 magic = 1; + optional uint32 magic = 1 [(yara.field_options).hex_value = true]; optional uint32 cputype = 2; optional uint32 cpusubtype = 3; optional uint32 filetype = 4; optional uint32 ncmds = 5; optional uint32 sizeofcmds = 6; - optional uint32 flags = 7; + optional uint32 flags = 7 [(yara.field_options).hex_value = true]; optional uint32 reserved = 8; optional uint64 number_of_segments = 9; repeated Segment segments = 10; @@ -79,7 +79,7 @@ message Macho { optional uint32 filetype = 4; optional uint32 ncmds = 5; optional uint32 sizeofcmds = 6; - optional uint32 flags = 7; + optional uint32 flags = 7 [(yara.field_options).hex_value = true]; optional uint32 reserved = 8; optional uint64 number_of_segments = 9; repeated Segment segments = 10; @@ -88,7 +88,7 @@ message Macho { optional uint64 stack_size = 13; // Add fields for Mach-O fat binary header - optional uint32 fat_magic = 14; + optional uint32 fat_magic = 14 [(yara.field_options).hex_value = true]; optional uint32 nfat_arch = 15; repeated FatArch fat_arch = 16; From cd6854b9a51a03f5441924314fc2b2aa4c3a2455 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20=C4=8Euri=C5=A1?= Date: Thu, 2 Nov 2023 14:14:45 +0100 Subject: [PATCH 11/26] added colors for fields, module filtering, protobuf map printing --- yara-x-dump/src/lib.rs | 36 +++++- yara-x-dump/src/serializer.rs | 155 ++++++++++++++++++++------ yara-x-proto/src/yara.proto | 1 + yara-x/src/modules/protos/lnk.proto | 1 + yara-x/src/modules/protos/macho.proto | 1 + 5 files changed, 158 insertions(+), 36 deletions(-) diff --git a/yara-x-dump/src/lib.rs b/yara-x-dump/src/lib.rs index c41cf0755..689cb8605 100644 --- a/yara-x-dump/src/lib.rs +++ b/yara-x-dump/src/lib.rs @@ -1,5 +1,7 @@ mod serializer; -use protobuf::reflect::MessageRef; +use protobuf::{ + reflect::MessageRef, reflect::ReflectValueRef::Bool, MessageDyn, +}; use protobuf_json_mapping::print_to_string; use serde_json; use serde_yaml; @@ -7,6 +9,7 @@ use std::io; use thiserror::Error; use yansi::Color::Cyan; use yara_x; +use yara_x_proto::exts::module_options; use crate::serializer::{get_human_readable_output, get_serializer}; @@ -53,6 +56,28 @@ impl Dumper { Dumper {} } + // Checks if the module output is valid by checking the validity flag. + fn module_is_valid(&self, mod_output: &dyn MessageDyn) -> bool { + if let Some(module_desc) = module_options + .get(&mod_output.descriptor_dyn().file_descriptor_proto().options) + { + if let Some(validity_flag_str) = + module_desc.validity_flag.as_deref() + { + if let Some(field) = mod_output + .descriptor_dyn() + .field_by_name(validity_flag_str) + { + if let Some(value) = field.get_singular(mod_output) { + return value != Bool(false); + } + } + } + } + + false + } + /// Dumps information about the binary file. pub fn dump( &self, @@ -103,6 +128,15 @@ impl Dumper { let mut serialized_result = String::new(); let mut is_first_line = false; + // Skip empty outputs or invalid outputs that are not requested. + if mod_output.compute_size_dyn() == 0 + || (!self.module_is_valid(mod_output) + && !modules + .unwrap_or(&vec![]) + .contains(&mod_name.to_string())) + { + continue; + } match output_format { // Output is desired to be human-readable. Some(format) if format == "human-readable" => { diff --git a/yara-x-dump/src/serializer.rs b/yara-x-dump/src/serializer.rs index 827174ab3..06de28ad7 100644 --- a/yara-x-dump/src/serializer.rs +++ b/yara-x-dump/src/serializer.rs @@ -5,7 +5,7 @@ use protobuf::reflect::ReflectFieldRef; use protobuf::reflect::ReflectValueRef; use protobuf_support::text_format::quote_bytes_to; use std::fmt::Write; -use yansi::Color::{Green, Yellow}; +use yansi::Color::{Blue, Green, Yellow}; use yara_x_proto::exts::field_options; use crate::Error; @@ -97,7 +97,7 @@ fn print_field_name( indent: usize, is_first_line: &mut bool, ) { - let mut indentation = " ".repeat(indent); + let mut indentation = get_indentation(indent); if field_name.is_empty() { return; @@ -113,12 +113,12 @@ fn print_field_name( "{}{} {}: ", indentation, Yellow.paint("-").bold(), - field_name + Blue.paint(field_name) ) .unwrap(); *is_first_line = false; } else { - write!(buf, "{}{}: ", indentation, field_name).unwrap(); + write!(buf, "{}{}: ", indentation, Blue.paint(field_name)).unwrap(); } } @@ -222,14 +222,26 @@ fn print_field_value( writeln!(buf, "{}", v).unwrap(); } ReflectValueRef::F32(v) => { - writeln!(buf, "{}", v).unwrap(); + writeln!(buf, "{:.1}", v).unwrap(); } ReflectValueRef::F64(v) => { - writeln!(buf, "{}", v).unwrap(); + writeln!(buf, "{:.1}", v).unwrap(); } } } +// Get the value options for a field +fn get_value_options(field_descriptor: &FieldDescriptorProto) -> ValueOptions { + let value_options = field_options + .get(&field_descriptor.options) + .map(|options| ValueOptions { + is_hex: options.hex_value.unwrap_or(false), + is_timestamp: options.timestamp.unwrap_or(false), + }) + .unwrap_or(ValueOptions::new()); + value_options +} + // Print a field name and value fn print_field( buf: &mut String, @@ -239,18 +251,17 @@ fn print_field( indent: usize, is_first_line: &mut bool, ) { - let value_options = field_options - .get(&field_descriptor.options) - .map(|options| ValueOptions { - is_hex: options.hex_value.unwrap_or(false), - is_timestamp: options.timestamp.unwrap_or(false), - }) - .unwrap_or(ValueOptions::new()); + let value_options = get_value_options(field_descriptor); print_field_name(buf, field_name, indent, is_first_line); print_field_value(buf, value, &value_options, indent, is_first_line); } +// Get indentation level +fn get_indentation(indent: usize) -> String { + " ".repeat(indent) +} + /// A function that returns a human-readable output pub fn get_human_readable_output( msg: &MessageRef, @@ -262,24 +273,40 @@ pub fn get_human_readable_output( // Iterate over the fields of the message for f in desc.fields() { - let indentation = " ".repeat(indent); - // Match the field type match f.get_reflect(&**msg) { ReflectFieldRef::Map(map) => { if map.is_empty() { continue; } - writeln!(buf, "{}{}:", indentation, f.name()).unwrap(); + writeln!( + buf, + "{}{}:", + get_indentation(indent), + Yellow.paint(f.name()).bold() + ) + .unwrap(); for (k, v) in &map { - print_field( - buf, - "", - k, - &f.proto(), - indent + 1, - first_line, - ); + match v { + ReflectValueRef::Message(_) => { + writeln!( + buf, + "{}{}:", + get_indentation(indent + 1), + Blue.paint(k) + ) + .unwrap(); + } + _ => { + write!( + buf, + "{}{}: ", + get_indentation(indent + 1), + Blue.paint(k) + ) + .unwrap(); + } + } print_field( buf, "", @@ -297,7 +324,7 @@ pub fn get_human_readable_output( writeln!( buf, "{}{} {} {}", - indentation, + get_indentation(indent), Green.paint("# Nested").italic(), Green.paint(f.name()).italic(), Green.paint("structure").italic() @@ -306,24 +333,82 @@ pub fn get_human_readable_output( writeln!( buf, "{}{}:", - indentation, + get_indentation(indent), Yellow.paint(f.name()).bold() ) .unwrap(); for v in repeated { - print_field(buf, "", v, &f.proto(), indent, first_line); + match v { + ReflectValueRef::Message(_) => { + print_field( + buf, + "", + v, + &f.proto(), + indent, + first_line, + ); + } + _ => { + write!( + buf, + "{} {} ", + get_indentation(indent), + Yellow.paint("-").bold(), + ) + .unwrap(); + print_field( + buf, + "", + v, + &f.proto(), + indent, + first_line, + ); + } + } } } ReflectFieldRef::Optional(optional) => { if let Some(v) = optional.value() { - print_field( - buf, - f.name(), - v, - &f.proto(), - indent, - first_line, - ); + match v { + ReflectValueRef::Message(_) => { + writeln!( + buf, + "{}{} {} {}", + get_indentation(indent), + Green.paint("# Nested").italic(), + Green.paint(f.name()).italic(), + Green.paint("structure").italic() + ) + .unwrap(); + writeln!( + buf, + "{}{}:", + get_indentation(indent), + Yellow.paint(f.name()).bold() + ) + .unwrap(); + print_field( + buf, + "", + v, + &f.proto(), + indent, + first_line, + ); + } + _ => { + print_field( + buf, + f.name(), + v, + &f.proto(), + indent, + first_line, + ); + } + } } } } diff --git a/yara-x-proto/src/yara.proto b/yara-x-proto/src/yara.proto index fa21f2b20..0a32ba156 100644 --- a/yara-x-proto/src/yara.proto +++ b/yara-x-proto/src/yara.proto @@ -11,6 +11,7 @@ message ModuleOptions { required string name = 1; required string root_message = 2; optional string rust_module = 3; + optional string validity_flag = 4; } message FieldOptions { diff --git a/yara-x/src/modules/protos/lnk.proto b/yara-x/src/modules/protos/lnk.proto index 049d2f149..807117deb 100644 --- a/yara-x/src/modules/protos/lnk.proto +++ b/yara-x/src/modules/protos/lnk.proto @@ -7,6 +7,7 @@ option (yara.module_options) = { name : "lnk" root_message: "lnk.Lnk" rust_module: "lnk" + validity_flag: "is_lnk" }; enum FileAttributes { diff --git a/yara-x/src/modules/protos/macho.proto b/yara-x/src/modules/protos/macho.proto index ecd36bbdd..449fa22ad 100644 --- a/yara-x/src/modules/protos/macho.proto +++ b/yara-x/src/modules/protos/macho.proto @@ -7,6 +7,7 @@ option (yara.module_options) = { name : "macho" root_message: "macho.Macho" rust_module: "macho" + validity_flag: "magic" }; message Dylib { From c6b38f2eddefc5e36a21ebb0a9b20ec1a759e918 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20=C4=8Euri=C5=A1?= Date: Fri, 3 Nov 2023 16:02:30 +0100 Subject: [PATCH 12/26] added custom colors and support for skipping 0x00 bytes in the middle of a string --- yara-x-dump/src/serializer.rs | 41 ++++++++++++++++++++------------- yara-x/src/modules/macho/mod.rs | 12 +++++----- 2 files changed, 31 insertions(+), 22 deletions(-) diff --git a/yara-x-dump/src/serializer.rs b/yara-x-dump/src/serializer.rs index 06de28ad7..fd0ddf298 100644 --- a/yara-x-dump/src/serializer.rs +++ b/yara-x-dump/src/serializer.rs @@ -5,7 +5,7 @@ use protobuf::reflect::ReflectFieldRef; use protobuf::reflect::ReflectValueRef; use protobuf_support::text_format::quote_bytes_to; use std::fmt::Write; -use yansi::Color::{Blue, Green, Yellow}; +use yansi::Color; use yara_x_proto::exts::field_options; use crate::Error; @@ -16,6 +16,14 @@ struct YamlSerializer; struct TomlSerializer; struct XmlSerializer; +struct Colors; + +impl Colors { + const GREEN: Color = Color::RGB(51, 255, 153); + const BLUE: Color = Color::RGB(51, 51, 255); + const YELLOW: Color = Color::RGB(255, 255, 102); +} + struct ValueOptions { is_hex: bool, is_timestamp: bool, @@ -112,13 +120,14 @@ fn print_field_name( buf, "{}{} {}: ", indentation, - Yellow.paint("-").bold(), - Blue.paint(field_name) + Colors::YELLOW.paint("-").bold(), + Colors::BLUE.paint(field_name) ) .unwrap(); *is_first_line = false; } else { - write!(buf, "{}{}: ", indentation, Blue.paint(field_name)).unwrap(); + write!(buf, "{}{}: ", indentation, Colors::BLUE.paint(field_name)) + .unwrap(); } } @@ -283,7 +292,7 @@ pub fn get_human_readable_output( buf, "{}{}:", get_indentation(indent), - Yellow.paint(f.name()).bold() + Colors::YELLOW.paint(f.name()).bold() ) .unwrap(); for (k, v) in &map { @@ -293,7 +302,7 @@ pub fn get_human_readable_output( buf, "{}{}:", get_indentation(indent + 1), - Blue.paint(k) + Colors::BLUE.paint(k) ) .unwrap(); } @@ -302,7 +311,7 @@ pub fn get_human_readable_output( buf, "{}{}: ", get_indentation(indent + 1), - Blue.paint(k) + Colors::BLUE.paint(k) ) .unwrap(); } @@ -325,16 +334,16 @@ pub fn get_human_readable_output( buf, "{}{} {} {}", get_indentation(indent), - Green.paint("# Nested").italic(), - Green.paint(f.name()).italic(), - Green.paint("structure").italic() + Colors::GREEN.paint("# Nested").italic(), + Colors::GREEN.paint(f.name()).italic(), + Colors::GREEN.paint("structure").italic() ) .unwrap(); writeln!( buf, "{}{}:", get_indentation(indent), - Yellow.paint(f.name()).bold() + Colors::YELLOW.paint(f.name()).bold() ) .unwrap(); for v in repeated { @@ -354,7 +363,7 @@ pub fn get_human_readable_output( buf, "{} {} ", get_indentation(indent), - Yellow.paint("-").bold(), + Colors::YELLOW.paint("-").bold(), ) .unwrap(); print_field( @@ -377,16 +386,16 @@ pub fn get_human_readable_output( buf, "{}{} {} {}", get_indentation(indent), - Green.paint("# Nested").italic(), - Green.paint(f.name()).italic(), - Green.paint("structure").italic() + Colors::GREEN.paint("# Nested").italic(), + Colors::GREEN.paint(f.name()).italic(), + Colors::GREEN.paint("structure").italic() ) .unwrap(); writeln!( buf, "{}{}:", get_indentation(indent), - Yellow.paint(f.name()).bold() + Colors::YELLOW.paint(f.name()).bold() ) .unwrap(); print_field( diff --git a/yara-x/src/modules/macho/mod.rs b/yara-x/src/modules/macho/mod.rs index bf1da5036..42bcd2417 100644 --- a/yara-x/src/modules/macho/mod.rs +++ b/yara-x/src/modules/macho/mod.rs @@ -1640,7 +1640,7 @@ fn handle_segment_command( segname: Some( std::str::from_utf8(&sg.segname) .unwrap_or_default() - .trim_end_matches('\0') + .replace('\0', "") .to_string(), ), vmaddr: Some(sg.vmaddr as u64), @@ -1673,13 +1673,13 @@ fn handle_segment_command( segname: Some( std::str::from_utf8(&sec.segname) .unwrap_or_default() - .trim_end_matches('\0') + .replace('\0', "") .to_string(), ), sectname: Some( std::str::from_utf8(&sec.sectname) .unwrap_or_default() - .trim_end_matches('\0') + .replace('\0', "") .to_string(), ), addr: Some(sec.addr as u64), @@ -1760,7 +1760,7 @@ fn handle_segment_command_64( segname: Some( std::str::from_utf8(&sg.segname) .unwrap_or_default() - .trim_end_matches('\0') + .replace('\0', "") .to_string(), ), vmaddr: Some(sg.vmaddr), @@ -1793,13 +1793,13 @@ fn handle_segment_command_64( segname: Some( std::str::from_utf8(&sec.segname) .unwrap_or_default() - .trim_end_matches('\0') + .replace('\0', "") .to_string(), ), sectname: Some( std::str::from_utf8(&sec.sectname) .unwrap_or_default() - .trim_end_matches('\0') + .replace('\0', "") .to_string(), ), addr: Some(sec.addr), From bb4bc0dd18277f7b2ea31f08aa154d3c784343cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20=C4=8Euri=C5=A1?= Date: Mon, 6 Nov 2023 12:50:31 +0100 Subject: [PATCH 13/26] code formatting --- yara-x-cli/src/commands/dump.rs | 27 ++++-- yara-x-cli/src/commands/mod.rs | 17 +--- yara-x-dump/src/lib.rs | 38 +++----- yara-x-dump/src/serializer.rs | 165 +++++++++++++++----------------- 4 files changed, 109 insertions(+), 138 deletions(-) diff --git a/yara-x-cli/src/commands/dump.rs b/yara-x-cli/src/commands/dump.rs index 889121ac8..42339eb86 100644 --- a/yara-x-cli/src/commands/dump.rs +++ b/yara-x-cli/src/commands/dump.rs @@ -1,11 +1,11 @@ -use clap::{arg, value_parser, Arg, ArgMatches, Command}; +use clap::{arg, value_parser, Arg, ArgAction, ArgMatches, Command}; use std::fs::File; use std::io::stdin; use std::path::PathBuf; use yara_x_dump::Dumper; -use crate::commands::modules_parser; +use yara_x::get_builtin_modules_names; pub fn dump() -> Command { super::command("dump") @@ -27,25 +27,32 @@ pub fn dump() -> Command { .long("modules") .help("Name of the module or comma-separated list of modules to be used for parsing") .required(false) - .value_parser(modules_parser), + .action(ArgAction::Append) + .value_parser(get_builtin_modules_names()), ) } pub fn exec_dump(args: &ArgMatches) -> anyhow::Result<()> { let file = args.get_one::("FILE"); let output_format = args.get_one::("output-format"); - let modules = args.get_one::>("modules"); - let dumper = Dumper::new(); + // get vector of modules + let modules: Vec<&str> = args + .get_many::("modules") + .unwrap_or_default() + .map(|s| s.as_str()) + .collect(); - if let Some(file) = file { + let dumper = Dumper::default(); + + let input: Box = if let Some(file) = file { println!("Dumping file: {:?}", file.as_path()); - let input = File::open(file.as_path())?; - dumper.dump(input, modules, output_format)?; + Box::new(File::open(file.as_path())?) } else { println!("Dumping stdin"); - dumper.dump(stdin(), modules, output_format)?; - } + Box::new(stdin()) + }; + dumper.dump(input, modules, output_format)?; Ok(()) } diff --git a/yara-x-cli/src/commands/mod.rs b/yara-x-cli/src/commands/mod.rs index 805848782..269345bd7 100644 --- a/yara-x-cli/src/commands/mod.rs +++ b/yara-x-cli/src/commands/mod.rs @@ -23,7 +23,7 @@ use crossterm::tty::IsTty; use serde_json::Value; use superconsole::{Component, Line, Lines, Span, SuperConsole}; -use yara_x::{get_builtin_modules_names, Compiler, Rules}; +use yara_x::{Compiler, Rules}; use yara_x_parser::SourceCode; use crate::walk::DirWalker; @@ -39,21 +39,6 @@ pub fn command(name: &'static str) -> Command { ) } -fn modules_parser(option: &str) -> Result, anyhow::Error> { - let modules = option.split(',').map(|s| s.to_string()).collect::>(); - let supported_modules = get_builtin_modules_names(); - for module in &modules { - if !supported_modules.contains(&module.as_str()) { - anyhow::bail!( - "Unsupported module: {}. Supported modules for --modules argument are: {}", - module, - supported_modules.join(", ") - ); - } - } - Ok(modules) -} - fn external_var_parser( option: &str, ) -> Result<(String, serde_json::Value), anyhow::Error> { diff --git a/yara-x-dump/src/lib.rs b/yara-x-dump/src/lib.rs index 689cb8605..c40887d16 100644 --- a/yara-x-dump/src/lib.rs +++ b/yara-x-dump/src/lib.rs @@ -3,12 +3,9 @@ use protobuf::{ reflect::MessageRef, reflect::ReflectValueRef::Bool, MessageDyn, }; use protobuf_json_mapping::print_to_string; -use serde_json; -use serde_yaml; use std::io; use thiserror::Error; use yansi::Color::Cyan; -use yara_x; use yara_x_proto::exts::module_options; use crate::serializer::{get_human_readable_output, get_serializer}; @@ -38,24 +35,17 @@ pub enum Error { /// Error for unsupported serilization formats. #[error("Unsupported serilization format")] UnsupportedFormat, + /// Error for formatting problems + #[error("Formatting Error")] + FormattingError(#[from] std::fmt::Error), } /// Dumps information about binary files. +#[derive(Debug, Default, Clone)] pub struct Dumper {} -impl Default for Dumper { - fn default() -> Self { - Self::new() - } -} - // Dumper public API. impl Dumper { - /// Creates a new dumper. - pub fn new() -> Self { - Dumper {} - } - // Checks if the module output is valid by checking the validity flag. fn module_is_valid(&self, mod_output: &dyn MessageDyn) -> bool { if let Some(module_desc) = module_options @@ -82,7 +72,7 @@ impl Dumper { pub fn dump( &self, mut input: R, - modules: Option<&Vec>, + modules: Vec<&str>, output_format: Option<&String>, ) -> Result<(), Error> where @@ -92,13 +82,10 @@ impl Dumper { input.read_to_end(&mut buffer).map_err(Error::ReadError)?; // Get the list of modules to import. - let import_modules = if let Some(modules) = modules { + let import_modules = if !modules.is_empty() { modules.clone() } else { yara_x::get_builtin_modules_names() - .into_iter() - .map(|s| s.to_string()) - .collect() }; // Create a rule that imports all the built-in modules. @@ -131,28 +118,27 @@ impl Dumper { // Skip empty outputs or invalid outputs that are not requested. if mod_output.compute_size_dyn() == 0 || (!self.module_is_valid(mod_output) - && !modules - .unwrap_or(&vec![]) - .contains(&mod_name.to_string())) + && !modules.contains(&mod_name)) { continue; } match output_format { // Output is desired to be human-readable. Some(format) if format == "human-readable" => { - serialized_result = get_human_readable_output( + get_human_readable_output( &MessageRef::from(mod_output), &mut serialized_result, 0, &mut is_first_line, - ); + )?; } // Serialize output for other given formats. Some(format) => { let json_output = print_to_string(mod_output)?; let serializer = get_serializer(format)?; - serialized_result = serializer.serialize(json_output)?; + serialized_result = + serializer.serialize(json_output.as_str())?; } // Default to human-readable output. None => { @@ -161,7 +147,7 @@ impl Dumper { &mut serialized_result, 0, &mut is_first_line, - ); + )?; } } diff --git a/yara-x-dump/src/serializer.rs b/yara-x-dump/src/serializer.rs index fd0ddf298..b10dcc12e 100644 --- a/yara-x-dump/src/serializer.rs +++ b/yara-x-dump/src/serializer.rs @@ -24,49 +24,44 @@ impl Colors { const YELLOW: Color = Color::RGB(255, 255, 102); } +#[derive(Debug, Default, Clone)] struct ValueOptions { is_hex: bool, is_timestamp: bool, } -impl ValueOptions { - fn new() -> Self { - ValueOptions { is_hex: false, is_timestamp: false } - } -} - /// A trait for any type that can serialize a message pub(crate) trait Serializer { - fn serialize(&self, message: String) -> Result; + fn serialize(&self, message: &str) -> Result; } /// Implement the trait for the JSON serializer impl Serializer for JsonSerializer { - fn serialize(&self, message: String) -> Result { - let value = serde_json::from_str::(&message)?; + fn serialize(&self, message: &str) -> Result { + let value = serde_json::from_str::(message)?; Ok(serde_json::to_string_pretty(&value)?) } } /// Implement the trait for the YAML serializer impl Serializer for YamlSerializer { - fn serialize(&self, message: String) -> Result { - let value = serde_json::from_str::(&message)?; + fn serialize(&self, message: &str) -> Result { + let value = serde_json::from_str::(message)?; Ok(serde_yaml::to_string(&value)?) } } /// Implement the trait for the TOML serializer impl Serializer for TomlSerializer { - fn serialize(&self, message: String) -> Result { - let value = serde_json::from_str::(&message)?; + fn serialize(&self, message: &str) -> Result { + let value = serde_json::from_str::(message)?; Ok(toml::to_string_pretty(&value)?) } } /// Implement the trait for the XML serializer impl Serializer for XmlSerializer { - fn serialize(&self, message: String) -> Result { + fn serialize(&self, message: &str) -> Result { // Create a new XML builder and get the XML let mut xml_builder = xml2json_rs::XmlConfig::new() .rendering(xml2json_rs::Indentation::new(b' ', 2)) @@ -77,13 +72,15 @@ impl Serializer for XmlSerializer { )) .root_name("file") .finalize(); - let xml = xml_builder.build_from_json_string(&message)?; + let xml = xml_builder.build_from_json_string(message)?; Ok(xml) } } /// A function that returns a trait object based on the format -pub fn get_serializer(format: &str) -> Result, Error> { +pub(crate) fn get_serializer( + format: &str, +) -> Result, Error> { match format { // Return a JSON serializer "json" => Ok(Box::new(JsonSerializer)), @@ -104,31 +101,33 @@ fn print_field_name( field_name: &str, indent: usize, is_first_line: &mut bool, -) { +) -> Result<(), Error> { let mut indentation = get_indentation(indent); - if field_name.is_empty() { - return; - } - - if *is_first_line { - if !indentation.is_empty() { - indentation.pop(); - indentation.pop(); + if !field_name.is_empty() { + if *is_first_line { + if !indentation.is_empty() { + indentation.pop(); + indentation.pop(); + } + write!( + buf, + "{}{} {}: ", + indentation, + Colors::YELLOW.paint("-").bold(), + Colors::BLUE.paint(field_name) + )?; + *is_first_line = false; + } else { + write!( + buf, + "{}{}: ", + indentation, + Colors::BLUE.paint(field_name) + )?; } - write!( - buf, - "{}{} {}: ", - indentation, - Colors::YELLOW.paint("-").bold(), - Colors::BLUE.paint(field_name) - ) - .unwrap(); - *is_first_line = false; - } else { - write!(buf, "{}{}: ", indentation, Colors::BLUE.paint(field_name)) - .unwrap(); } + Ok(()) } // Print a field value with correct indentation for multiple value formats @@ -138,23 +137,23 @@ fn print_field_value( value_options: &ValueOptions, indent: usize, is_first_line: &mut bool, -) { +) -> Result<(), Error> { match value { ReflectValueRef::Message(m) => { *is_first_line = true; - get_human_readable_output(&m, buf, indent + 1, is_first_line); + get_human_readable_output(&m, buf, indent + 1, is_first_line)?; } ReflectValueRef::Enum(d, v) => match d.value_by_number(v) { - Some(e) => writeln!(buf, "{}", e.name()).unwrap(), - None => writeln!(buf, "{}", v).unwrap(), + Some(e) => writeln!(buf, "{}", e.name())?, + None => writeln!(buf, "{}", v)?, }, ReflectValueRef::String(s) => { quote_bytes_to(s.as_bytes(), buf); - buf.push_str("\n"); + buf.push('\n'); } ReflectValueRef::Bytes(b) => { quote_bytes_to(b, buf); - buf.push_str("\n"); + buf.push('\n'); } ReflectValueRef::I32(v) => { let field_value = if value_options.is_hex { @@ -172,7 +171,7 @@ fn print_field_value( } else { v.to_string() }; - writeln!(buf, "{}", field_value).unwrap(); + writeln!(buf, "{}", field_value)?; } ReflectValueRef::I64(v) => { let field_value = if value_options.is_hex { @@ -189,7 +188,7 @@ fn print_field_value( } else { v.to_string() }; - writeln!(buf, "{}", field_value).unwrap(); + writeln!(buf, "{}", field_value)?; } ReflectValueRef::U32(v) => { let field_value = if value_options.is_hex { @@ -207,7 +206,7 @@ fn print_field_value( } else { v.to_string() }; - writeln!(buf, "{}", field_value).unwrap(); + writeln!(buf, "{}", field_value)?; } ReflectValueRef::U64(v) => { let field_value = if value_options.is_hex { @@ -225,30 +224,31 @@ fn print_field_value( } else { v.to_string() }; - writeln!(buf, "{}", field_value).unwrap(); + writeln!(buf, "{}", field_value)?; } ReflectValueRef::Bool(v) => { - writeln!(buf, "{}", v).unwrap(); + writeln!(buf, "{}", v)?; } ReflectValueRef::F32(v) => { - writeln!(buf, "{:.1}", v).unwrap(); + writeln!(buf, "{:.1}", v)?; } ReflectValueRef::F64(v) => { - writeln!(buf, "{:.1}", v).unwrap(); + writeln!(buf, "{:.1}", v)?; } } + Ok(()) } // Get the value options for a field fn get_value_options(field_descriptor: &FieldDescriptorProto) -> ValueOptions { - let value_options = field_options + field_options .get(&field_descriptor.options) .map(|options| ValueOptions { - is_hex: options.hex_value.unwrap_or(false), - is_timestamp: options.timestamp.unwrap_or(false), + // Default for boolean is false + is_hex: options.hex_value.unwrap_or_default(), + is_timestamp: options.timestamp.unwrap_or_default(), }) - .unwrap_or(ValueOptions::new()); - value_options + .unwrap_or_default() } // Print a field name and value @@ -259,11 +259,12 @@ fn print_field( field_descriptor: &FieldDescriptorProto, indent: usize, is_first_line: &mut bool, -) { +) -> Result<(), Error> { let value_options = get_value_options(field_descriptor); - print_field_name(buf, field_name, indent, is_first_line); - print_field_value(buf, value, &value_options, indent, is_first_line); + print_field_name(buf, field_name, indent, is_first_line)?; + print_field_value(buf, value, &value_options, indent, is_first_line)?; + Ok(()) } // Get indentation level @@ -277,7 +278,7 @@ pub fn get_human_readable_output( buf: &mut String, indent: usize, first_line: &mut bool, -) -> String { +) -> Result<(), Error> { let desc = msg.descriptor_dyn(); // Iterate over the fields of the message @@ -293,8 +294,7 @@ pub fn get_human_readable_output( "{}{}:", get_indentation(indent), Colors::YELLOW.paint(f.name()).bold() - ) - .unwrap(); + )?; for (k, v) in &map { match v { ReflectValueRef::Message(_) => { @@ -303,8 +303,7 @@ pub fn get_human_readable_output( "{}{}:", get_indentation(indent + 1), Colors::BLUE.paint(k) - ) - .unwrap(); + )?; } _ => { write!( @@ -312,18 +311,17 @@ pub fn get_human_readable_output( "{}{}: ", get_indentation(indent + 1), Colors::BLUE.paint(k) - ) - .unwrap(); + )?; } } print_field( buf, "", v, - &f.proto(), + f.proto(), indent + 1, first_line, - ); + )?; } } ReflectFieldRef::Repeated(repeated) => { @@ -337,15 +335,13 @@ pub fn get_human_readable_output( Colors::GREEN.paint("# Nested").italic(), Colors::GREEN.paint(f.name()).italic(), Colors::GREEN.paint("structure").italic() - ) - .unwrap(); + )?; writeln!( buf, "{}{}:", get_indentation(indent), Colors::YELLOW.paint(f.name()).bold() - ) - .unwrap(); + )?; for v in repeated { match v { ReflectValueRef::Message(_) => { @@ -353,10 +349,10 @@ pub fn get_human_readable_output( buf, "", v, - &f.proto(), + f.proto(), indent, first_line, - ); + )?; } _ => { write!( @@ -364,16 +360,15 @@ pub fn get_human_readable_output( "{} {} ", get_indentation(indent), Colors::YELLOW.paint("-").bold(), - ) - .unwrap(); + )?; print_field( buf, "", v, - &f.proto(), + f.proto(), indent, first_line, - ); + )?; } } } @@ -389,33 +384,31 @@ pub fn get_human_readable_output( Colors::GREEN.paint("# Nested").italic(), Colors::GREEN.paint(f.name()).italic(), Colors::GREEN.paint("structure").italic() - ) - .unwrap(); + )?; writeln!( buf, "{}{}:", get_indentation(indent), Colors::YELLOW.paint(f.name()).bold() - ) - .unwrap(); + )?; print_field( buf, "", v, - &f.proto(), + f.proto(), indent, first_line, - ); + )?; } _ => { print_field( buf, f.name(), v, - &f.proto(), + f.proto(), indent, first_line, - ); + )?; } } } @@ -423,5 +416,5 @@ pub fn get_human_readable_output( } } - return buf.to_string(); + Ok(()) } From 58de81a1a9c243496f60a4ab3c4568528b17fe44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20=C4=8Euri=C5=A1?= Date: Mon, 6 Nov 2023 17:25:45 +0100 Subject: [PATCH 14/26] added documentation and tests --- Cargo.lock | 170 ++++++++++++++++- yara-x-cli/src/commands/dump.rs | 5 +- yara-x-dump/Cargo.toml | 8 +- yara-x-dump/src/lib.rs | 44 ++++- yara-x-dump/src/serializer.rs | 154 ++++++++++++++- yara-x-dump/src/tests/mod.rs | 98 ++++++++++ .../macho_x86_file.automatic.None.out | 147 +++++++++++++++ ...acho_x86_file.automatic.human-readable.out | 147 +++++++++++++++ .../macho_x86_file.automatic.json.out | 172 +++++++++++++++++ .../macho_x86_file.automatic.toml.out | 163 ++++++++++++++++ .../testdata/macho_x86_file.automatic.xml.out | 165 ++++++++++++++++ .../macho_x86_file.automatic.yaml.out | 144 ++++++++++++++ .../src/tests/testdata/macho_x86_file.in | 73 ++++++++ .../testdata/macho_x86_file.lnk.None.out | 5 + .../macho_x86_file.lnk.human-readable.out | 5 + .../testdata/macho_x86_file.lnk.json.out | 6 + .../testdata/macho_x86_file.lnk.toml.out | 5 + .../tests/testdata/macho_x86_file.lnk.xml.out | 7 + .../testdata/macho_x86_file.lnk.yaml.out | 6 + .../testdata/macho_x86_file.macho.None.out | 147 +++++++++++++++ .../macho_x86_file.macho.human-readable.out | 147 +++++++++++++++ .../testdata/macho_x86_file.macho.json.out | 172 +++++++++++++++++ .../testdata/macho_x86_file.macho.toml.out | 163 ++++++++++++++++ .../testdata/macho_x86_file.macho.xml.out | 165 ++++++++++++++++ .../testdata/macho_x86_file.macho.yaml.out | 144 ++++++++++++++ .../macho_x86_file.macho_lnk.None.out | 151 +++++++++++++++ ...acho_x86_file.macho_lnk.human-readable.out | 151 +++++++++++++++ .../macho_x86_file.macho_lnk.json.out | 177 ++++++++++++++++++ .../macho_x86_file.macho_lnk.toml.out | 167 +++++++++++++++++ .../testdata/macho_x86_file.macho_lnk.xml.out | 171 +++++++++++++++++ .../macho_x86_file.macho_lnk.yaml.out | 149 +++++++++++++++ 31 files changed, 3415 insertions(+), 13 deletions(-) create mode 100644 yara-x-dump/src/tests/mod.rs create mode 100644 yara-x-dump/src/tests/testdata/macho_x86_file.automatic.None.out create mode 100644 yara-x-dump/src/tests/testdata/macho_x86_file.automatic.human-readable.out create mode 100644 yara-x-dump/src/tests/testdata/macho_x86_file.automatic.json.out create mode 100644 yara-x-dump/src/tests/testdata/macho_x86_file.automatic.toml.out create mode 100644 yara-x-dump/src/tests/testdata/macho_x86_file.automatic.xml.out create mode 100644 yara-x-dump/src/tests/testdata/macho_x86_file.automatic.yaml.out create mode 100644 yara-x-dump/src/tests/testdata/macho_x86_file.in create mode 100644 yara-x-dump/src/tests/testdata/macho_x86_file.lnk.None.out create mode 100644 yara-x-dump/src/tests/testdata/macho_x86_file.lnk.human-readable.out create mode 100644 yara-x-dump/src/tests/testdata/macho_x86_file.lnk.json.out create mode 100644 yara-x-dump/src/tests/testdata/macho_x86_file.lnk.toml.out create mode 100644 yara-x-dump/src/tests/testdata/macho_x86_file.lnk.xml.out create mode 100644 yara-x-dump/src/tests/testdata/macho_x86_file.lnk.yaml.out create mode 100644 yara-x-dump/src/tests/testdata/macho_x86_file.macho.None.out create mode 100644 yara-x-dump/src/tests/testdata/macho_x86_file.macho.human-readable.out create mode 100644 yara-x-dump/src/tests/testdata/macho_x86_file.macho.json.out create mode 100644 yara-x-dump/src/tests/testdata/macho_x86_file.macho.toml.out create mode 100644 yara-x-dump/src/tests/testdata/macho_x86_file.macho.xml.out create mode 100644 yara-x-dump/src/tests/testdata/macho_x86_file.macho.yaml.out create mode 100644 yara-x-dump/src/tests/testdata/macho_x86_file.macho_lnk.None.out create mode 100644 yara-x-dump/src/tests/testdata/macho_x86_file.macho_lnk.human-readable.out create mode 100644 yara-x-dump/src/tests/testdata/macho_x86_file.macho_lnk.json.out create mode 100644 yara-x-dump/src/tests/testdata/macho_x86_file.macho_lnk.toml.out create mode 100644 yara-x-dump/src/tests/testdata/macho_x86_file.macho_lnk.xml.out create mode 100644 yara-x-dump/src/tests/testdata/macho_x86_file.macho_lnk.yaml.out diff --git a/Cargo.lock b/Cargo.lock index 79ed77f27..42b6031ef 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1205,6 +1205,101 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" +[[package]] +name = "futures" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da0290714b38af9b4a7b094b8a37086d1b4e61f2df9122c3cad2577669145335" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff4dd66668b557604244583e3e1e1eada8c5c2e96a6d0d6653ede395b78bbacb" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb1d22c66e66d9d72e1758f0bd7d4fd0bee04cad842ee34587d68c07e45d088c" + +[[package]] +name = "futures-executor" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f4fb8693db0cf099eadcca0efe2a5a22e4550f98ed16aba6c48700da29597bc" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bf34a163b5c4c52d0478a4d757da8fb65cabef42ba90515efee0f6f9fa45aaa" + +[[package]] +name = "futures-macro" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.38", +] + +[[package]] +name = "futures-sink" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e36d3378ee38c2a36ad710c5d30c2911d752cb941c00c72dbabfb786a7970817" + +[[package]] +name = "futures-task" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "efd193069b0ddadc69c46389b740bbccdd97203899b48d09c5f7969591d6bae2" + +[[package]] +name = "futures-timer" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c" + +[[package]] +name = "futures-util" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a19526d624e703a3179b3d322efec918b6246ea0fa51d41124525f00f1cc8104" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + [[package]] name = "fxhash" version = "0.2.1" @@ -1634,9 +1729,9 @@ checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67" [[package]] name = "libc" -version = "0.2.149" +version = "0.2.150" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a08173bc88b7955d1b3145aa561539096c421ac8debde8cbc3612ec635fee29b" +checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c" [[package]] name = "libloading" @@ -2255,6 +2350,18 @@ dependencies = [ "siphasher", ] +[[package]] +name = "pin-project-lite" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + [[package]] name = "pkg-config" version = "0.3.27" @@ -2659,6 +2766,12 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" +[[package]] +name = "relative-path" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c707298afce11da2efef2f600116fa93ffa7a032b5d7b628aa17711ec81383ca" + [[package]] name = "rgb" version = "0.8.37" @@ -2668,6 +2781,35 @@ dependencies = [ "bytemuck", ] +[[package]] +name = "rstest" +version = "0.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97eeab2f3c0a199bc4be135c36c924b6590b88c377d416494288c14f2db30199" +dependencies = [ + "futures", + "futures-timer", + "rstest_macros", + "rustc_version", +] + +[[package]] +name = "rstest_macros" +version = "0.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d428f8247852f894ee1be110b375111b586d4fa431f6c46e64ba5a0dcccbe605" +dependencies = [ + "cfg-if", + "glob", + "proc-macro2", + "quote", + "regex", + "relative-path", + "rustc_version", + "syn 2.0.38", + "unicode-ident", +] + [[package]] name = "rustc-demangle" version = "0.1.23" @@ -2680,6 +2822,15 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" +[[package]] +name = "rustc_version" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +dependencies = [ + "semver 1.0.20", +] + [[package]] name = "rustix" version = "0.38.21" @@ -2933,6 +3084,15 @@ dependencies = [ "walkdir", ] +[[package]] +name = "slab" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" +dependencies = [ + "autocfg", +] + [[package]] name = "slice-group-by" version = "0.3.1" @@ -4289,14 +4449,20 @@ dependencies = [ name = "yara-x-dump" version = "0.1.0" dependencies = [ + "anyhow", "chrono", + "globwalk", + "goldenfile", + "ihex", "indent", "pretty_assertions", "protobuf", "protobuf-json-mapping", "protobuf-support", + "rstest", "serde_json", "serde_yaml", + "tempfile", "thiserror", "toml 0.8.4", "xml2json-rs", diff --git a/yara-x-cli/src/commands/dump.rs b/yara-x-cli/src/commands/dump.rs index 42339eb86..f1a6e2cbd 100644 --- a/yara-x-cli/src/commands/dump.rs +++ b/yara-x-cli/src/commands/dump.rs @@ -25,6 +25,7 @@ pub fn dump() -> Command { .arg( Arg::new("modules") .long("modules") + .short('m') .help("Name of the module or comma-separated list of modules to be used for parsing") .required(false) .action(ArgAction::Append) @@ -46,13 +47,11 @@ pub fn exec_dump(args: &ArgMatches) -> anyhow::Result<()> { let dumper = Dumper::default(); let input: Box = if let Some(file) = file { - println!("Dumping file: {:?}", file.as_path()); Box::new(File::open(file.as_path())?) } else { - println!("Dumping stdin"); Box::new(stdin()) }; - dumper.dump(input, modules, output_format)?; + println!("{}", dumper.dump(input, modules, output_format)?); Ok(()) } diff --git a/yara-x-dump/Cargo.toml b/yara-x-dump/Cargo.toml index a71b1d6db..2f1e40542 100644 --- a/yara-x-dump/Cargo.toml +++ b/yara-x-dump/Cargo.toml @@ -13,6 +13,7 @@ rust-version.workspace = true crate-type = ["rlib", "cdylib"] [dependencies] +anyhow = { workspace = true } thiserror = { workspace = true } yara-x = { workspace = true } yara-x-proto = { workspace = true } @@ -29,4 +30,9 @@ xml2json-rs = "1.0.1" chrono = "0.4.31" [dev-dependencies] -pretty_assertions = { workspace = true } \ No newline at end of file +pretty_assertions = { workspace = true } +goldenfile = "1.5.2" +ihex = "3.0.0" +globwalk = { workspace = true } +tempfile = "3.8.1" +rstest = "0.18.2" diff --git a/yara-x-dump/src/lib.rs b/yara-x-dump/src/lib.rs index c40887d16..6e93a5224 100644 --- a/yara-x-dump/src/lib.rs +++ b/yara-x-dump/src/lib.rs @@ -3,6 +3,7 @@ use protobuf::{ reflect::MessageRef, reflect::ReflectValueRef::Bool, MessageDyn, }; use protobuf_json_mapping::print_to_string; +use std::fmt::Write; use std::io; use thiserror::Error; use yansi::Color::Cyan; @@ -10,6 +11,9 @@ use yara_x_proto::exts::module_options; use crate::serializer::{get_human_readable_output, get_serializer}; +#[cfg(test)] +mod tests; + /// Errors returned by [`Dumper::dump`]. #[derive(Error, Debug)] #[allow(clippy::large_enum_variant)] @@ -35,7 +39,7 @@ pub enum Error { /// Error for unsupported serilization formats. #[error("Unsupported serilization format")] UnsupportedFormat, - /// Error for formatting problems + /// Error while formatting output #[error("Formatting Error")] FormattingError(#[from] std::fmt::Error), } @@ -47,17 +51,31 @@ pub struct Dumper {} // Dumper public API. impl Dumper { // Checks if the module output is valid by checking the validity flag. + // + // # Arguments + // + // * `mod_output`: The module output to check. + // + // # Returns + // + // * `true` if the module output is valid, `false` otherwise. fn module_is_valid(&self, mod_output: &dyn MessageDyn) -> bool { + // Get the module options. if let Some(module_desc) = module_options .get(&mod_output.descriptor_dyn().file_descriptor_proto().options) { + // Get the field name which is considered as the validity flag. if let Some(validity_flag_str) = module_desc.validity_flag.as_deref() { + // Get the validity flag value. if let Some(field) = mod_output .descriptor_dyn() .field_by_name(validity_flag_str) { + // Check if the validity flag is set. + // Validity flag is set if the value present and is not + // false. if let Some(value) = field.get_singular(mod_output) { return value != Bool(false); } @@ -69,16 +87,29 @@ impl Dumper { } /// Dumps information about the binary file. + /// + /// # Arguments + /// + /// * `input`: The input to read from. + /// * `modules`: The list of modules to import. + /// * `output_format`: The desired output format. + /// + /// # Returns + /// + /// Returns a `Result<(), Error>` indicating whether the operation was + /// successful or not. pub fn dump( &self, mut input: R, modules: Vec<&str>, output_format: Option<&String>, - ) -> Result<(), Error> + ) -> Result where R: io::Read, { let mut buffer = Vec::new(); + let mut result = String::new(); + input.read_to_end(&mut buffer).map_err(Error::ReadError)?; // Get the list of modules to import. @@ -106,6 +137,7 @@ impl Dumper { let mut scanner = yara_x::Scanner::new(&rules); + // Scan the buffer and get the results. let scan_results = scanner.scan(&buffer).expect("scan should not fail"); @@ -151,13 +183,13 @@ impl Dumper { } } - // Print the result. - println!( + write!( + result, ">>>\n{}:\n{}\n<<<", Cyan.paint(mod_name).bold(), serialized_result - ); + )?; } - Ok(()) + Ok(result) } } diff --git a/yara-x-dump/src/serializer.rs b/yara-x-dump/src/serializer.rs index b10dcc12e..2a7cdc4a6 100644 --- a/yara-x-dump/src/serializer.rs +++ b/yara-x-dump/src/serializer.rs @@ -10,12 +10,13 @@ use yara_x_proto::exts::field_options; use crate::Error; -// A struct that represents serializers +// A struct that represents serializers for different formats struct JsonSerializer; struct YamlSerializer; struct TomlSerializer; struct XmlSerializer; +// A struct that represents colors for output struct Colors; impl Colors { @@ -24,6 +25,7 @@ impl Colors { const YELLOW: Color = Color::RGB(255, 255, 102); } +// A struct that represents options for a field values #[derive(Debug, Default, Clone)] struct ValueOptions { is_hex: bool, @@ -32,6 +34,24 @@ struct ValueOptions { /// A trait for any type that can serialize a message pub(crate) trait Serializer { + /// Serialize a message + /// + /// # Arguments + /// + /// * `message`: The message to serialize + /// + /// # Returns + /// + /// Returns a `Result` where the `String` is the serialized + /// message in specified format and the `Error` is any error that occurred + /// during serialization + /// + /// # Errors + /// + /// * `Error::ParsingJSONError`: If the message is not a valid JSON + /// * `Error::ParsingYAMLError`: If the message is not a valid YAML + /// * `Error::ParsingTOMLError`: If the message is not a valid TOML + /// * `Error::ParsingXMLError`: If the message is not a valid XML fn serialize(&self, message: &str) -> Result; } @@ -78,6 +98,20 @@ impl Serializer for XmlSerializer { } /// A function that returns a trait object based on the format +/// +/// # Arguments +/// +/// * `format`: The format to return the trait object for +/// +/// # Returns +/// +/// Returns a `Result, Error>` where the `Box` is the trait object for the specified format and the `Error` +/// is any error that occurred during the process +/// +/// # Errors +/// +/// * `Error::UnsupportedFormat`: If the format is unsupported pub(crate) fn get_serializer( format: &str, ) -> Result, Error> { @@ -96,6 +130,24 @@ pub(crate) fn get_serializer( } // Print a field name with correct indentation +// +// # Arguments +// +// * `buf`: The buffer to write the field name to +// * `field_name`: The field name to write +// * `indent`: The indentation level +// * `is_first_line`: A boolean that indicates if the field name is the first +// line +// +// # Returns +// +// Returns a `Result<(), Error>` where the `Error` is any error that occurred +// during the process +// +// # Errors +// +// * `Error::FormattingError`: If the field name could not be written to the +// buffer fn print_field_name( buf: &mut String, field_name: &str, @@ -104,7 +156,10 @@ fn print_field_name( ) -> Result<(), Error> { let mut indentation = get_indentation(indent); + // If the field name is not empty, print it if !field_name.is_empty() { + // If the field name is the first line, print the indentation with a + // dash and the field name if *is_first_line { if !indentation.is_empty() { indentation.pop(); @@ -118,6 +173,8 @@ fn print_field_name( Colors::BLUE.paint(field_name) )?; *is_first_line = false; + // If the field name is not the first line, print the indentation and + // the field name } else { write!( buf, @@ -131,6 +188,25 @@ fn print_field_name( } // Print a field value with correct indentation for multiple value formats +// +// # Arguments +// +// * `buf`: The buffer to write the field value to +// * `value`: The field value to write +// * `value_options`: The value options for the field value +// * `indent`: The indentation level +// * `is_first_line`: A boolean that indicates if the field value is the first +// line +// +// # Returns +// +// Returns a `Result<(), Error>` where the `Error` is any error that occurred +// during the process +// +// # Errors +// +// * `Error::FormattingError`: If the field value could not be written to the +// buffer fn print_field_value( buf: &mut String, value: ReflectValueRef, @@ -138,9 +214,11 @@ fn print_field_value( indent: usize, is_first_line: &mut bool, ) -> Result<(), Error> { + // Match the field value type and print it in desired format match value { ReflectValueRef::Message(m) => { *is_first_line = true; + // Recursively print the message get_human_readable_output(&m, buf, indent + 1, is_first_line)?; } ReflectValueRef::Enum(d, v) => match d.value_by_number(v) { @@ -156,8 +234,11 @@ fn print_field_value( buf.push('\n'); } ReflectValueRef::I32(v) => { + // If the value has hex option turned on, print it in hex format let field_value = if value_options.is_hex { format!("{} (0x{:x})", v, v) + // If the value has timestamp option turned on, print it in + // timestamp format } else if value_options.is_timestamp { format!( "{} ({})", @@ -168,14 +249,18 @@ fn print_field_value( Utc, ) ) + // Otherwise, print it as a normal integer } else { v.to_string() }; writeln!(buf, "{}", field_value)?; } ReflectValueRef::I64(v) => { + // If the value has hex option turned on, print it in hex format let field_value = if value_options.is_hex { format!("{} (0x{:x})", v, v) + // If the value has timestamp option turned on, print it in + // timestamp format } else if value_options.is_timestamp { format!( "{} ({})", @@ -185,14 +270,18 @@ fn print_field_value( Utc, ) ) + // Otherwise, print it as a normal integer } else { v.to_string() }; writeln!(buf, "{}", field_value)?; } ReflectValueRef::U32(v) => { + // If the value has hex option turned on, print it in hex format let field_value = if value_options.is_hex { format!("{} (0x{:x})", v, v) + // If the value has timestamp option turned on, print it in + // timestamp format } else if value_options.is_timestamp { format!( "{} ({})", @@ -203,14 +292,18 @@ fn print_field_value( Utc, ) ) + // Otherwise, print it as a normal integer } else { v.to_string() }; writeln!(buf, "{}", field_value)?; } ReflectValueRef::U64(v) => { + // If the value has hex option turned on, print it in hex format let field_value = if value_options.is_hex { format!("{} (0x{:x})", v, v) + // If the value has timestamp option turned on, print it in + // timestamp format } else if value_options.is_timestamp { format!( "{} ({})", @@ -221,6 +314,7 @@ fn print_field_value( Utc, ) ) + // Otherwise, print it as a normal integer } else { v.to_string() }; @@ -240,6 +334,14 @@ fn print_field_value( } // Get the value options for a field +// +// # Arguments +// +// * `field_descriptor`: The field descriptor to get the value options for +// +// # Returns +// +// Returns a `ValueOptions` which is the value options for the field fn get_value_options(field_descriptor: &FieldDescriptorProto) -> ValueOptions { field_options .get(&field_descriptor.options) @@ -252,6 +354,22 @@ fn get_value_options(field_descriptor: &FieldDescriptorProto) -> ValueOptions { } // Print a field name and value +// +// # Arguments +// +// * `buf`: The buffer to write the field name and value to +// * `field_name`: The field name to write +// * `value`: The field value to write +// * `field_descriptor`: The field descriptor to get the value options for +// * `indent`: The indentation level +// * `is_first_line`: A boolean that indicates if the field name and value is +// the first line +// +// # Returns +// +// Returns a `Result<(), Error>` where the `Error` is any error that occurred +// during the process +// fn print_field( buf: &mut String, field_name: &str, @@ -268,11 +386,32 @@ fn print_field( } // Get indentation level +// +// # Arguments +// +// * `indent`: The indentation level +// +// # Returns +// +// Returns a `String` which represents the indentation level fn get_indentation(indent: usize) -> String { " ".repeat(indent) } /// A function that returns a human-readable output +/// +/// # Arguments +/// +/// * `msg`: The message to get the human-readable output for +/// * `buf`: The buffer to write the human-readable output to +/// * `indent`: The indentation level +/// * `first_line`: A boolean that indicates if the field name and value is +/// the first line +/// +/// # Returns +/// +/// Returns a `Result<(), Error>` where the `Error` is any error that occurred +/// during the process pub fn get_human_readable_output( msg: &MessageRef, buf: &mut String, @@ -285,6 +424,7 @@ pub fn get_human_readable_output( for f in desc.fields() { // Match the field type match f.get_reflect(&**msg) { + // If the field is a message, print it recursively ReflectFieldRef::Map(map) => { if map.is_empty() { continue; @@ -295,8 +435,10 @@ pub fn get_human_readable_output( get_indentation(indent), Colors::YELLOW.paint(f.name()).bold() )?; + // Iterate over the map for (k, v) in &map { match v { + // If the value is a message, print it recursively ReflectValueRef::Message(_) => { writeln!( buf, @@ -305,6 +447,7 @@ pub fn get_human_readable_output( Colors::BLUE.paint(k) )?; } + // Otherwise, print the field name _ => { write!( buf, @@ -314,6 +457,7 @@ pub fn get_human_readable_output( )?; } } + // Print the field value print_field( buf, "", @@ -324,6 +468,8 @@ pub fn get_human_readable_output( )?; } } + // If the field is a repeated field, print nested structure without + // repeating the field name ReflectFieldRef::Repeated(repeated) => { if repeated.is_empty() { continue; @@ -342,8 +488,10 @@ pub fn get_human_readable_output( get_indentation(indent), Colors::YELLOW.paint(f.name()).bold() )?; + // Iterate over the repeated field for v in repeated { match v { + // If the value is a message, print it recursively ReflectValueRef::Message(_) => { print_field( buf, @@ -354,6 +502,7 @@ pub fn get_human_readable_output( first_line, )?; } + // Otherwise, print the field value _ => { write!( buf, @@ -373,9 +522,11 @@ pub fn get_human_readable_output( } } } + // If the field is a singular field, print it ReflectFieldRef::Optional(optional) => { if let Some(v) = optional.value() { match v { + // If the value is a message, print it recursively ReflectValueRef::Message(_) => { writeln!( buf, @@ -400,6 +551,7 @@ pub fn get_human_readable_output( first_line, )?; } + // Otherwise, print the field value _ => { print_field( buf, diff --git a/yara-x-dump/src/tests/mod.rs b/yara-x-dump/src/tests/mod.rs new file mode 100644 index 000000000..ed5e78fd4 --- /dev/null +++ b/yara-x-dump/src/tests/mod.rs @@ -0,0 +1,98 @@ +use rstest::rstest; +use std::fs; +use std::io::Write; +use std::path::Path; +use tempfile::NamedTempFile; + +pub fn create_binary_from_ihex>( + path: P, +) -> anyhow::Result> { + let contents = fs::read_to_string(path)?; + let mut reader = ihex::Reader::new(&contents); + let mut data = Vec::new(); + while let Some(Ok(record)) = reader.next() { + if let ihex::Record::Data { value, .. } = record { + data.extend(value); + } + } + Ok(data) +} + +#[rstest( + module, + output_format, + case(&vec!["macho"], "json"), + case(&vec!["macho"], "yaml"), + case(&vec!["macho"], "toml"), + case(&vec!["macho"], "xml"), + case(&vec!["macho"], "human-readable"), + case(&vec!["macho"], "None"), + case(&vec!["lnk"], "json"), + case(&vec!["lnk"], "yaml"), + case(&vec!["lnk"], "toml"), + case(&vec!["lnk"], "xml"), + case(&vec!["lnk"], "human-readable"), + case(&vec!["lnk"], "None"), + case(&vec!["macho", "lnk"], "json"), + case(&vec!["macho", "lnk"], "yaml"), + case(&vec!["macho", "lnk"], "toml"), + case(&vec!["macho", "lnk"], "xml"), + case(&vec!["macho", "lnk"], "human-readable"), + case(&vec!["macho", "lnk"], "None"), + case(&vec![], "json"), + case(&vec![], "yaml"), + case(&vec![], "toml"), + case(&vec![], "xml"), + case(&vec![], "human-readable"), + case(&vec![], "None"), +)] +fn test_dumper(module: &Vec<&str>, output_format: &str) { + // Create goldenfile mint. + let mut mint = goldenfile::Mint::new("."); + + for entry in globwalk::glob("src/tests/testdata/*.in").unwrap().flatten() { + // Path to the .in file. + println!("{:?}", entry); + let in_path = entry.into_path(); + + // Change the module to "automatic" if it's an empty vector. + let module_name = if module.is_empty() { + "automatic".to_string() + } else { + module.join("_") + }; + + // Create a unique test name based on the combination of module and + // output_format. + let test_name = format!( + "{}.{}.{}.out", + in_path.with_extension("").to_str().unwrap(), + module_name, + output_format + ); + + let mut input = NamedTempFile::new().unwrap(); + let input_stream = input.reopen().unwrap(); + let bytes = create_binary_from_ihex(&in_path).unwrap_or_else(|err| { + panic!("error reading ihex file {:?}: {:?}", in_path, err) + }); + input.write_all(&bytes).unwrap(); + + let dumper = crate::Dumper::default(); + let output = if output_format == "None" { + dumper.dump(input_stream, module.clone(), None) + } else { + dumper.dump( + input_stream, + module.clone(), + Some(&output_format.to_string()), + ) + } + .unwrap(); + + // Create a goldenfile test + let mut output_file = mint.new_goldenfile(test_name).unwrap(); + + write!(output_file, "{}", output).unwrap(); + } +} diff --git a/yara-x-dump/src/tests/testdata/macho_x86_file.automatic.None.out b/yara-x-dump/src/tests/testdata/macho_x86_file.automatic.None.out new file mode 100644 index 000000000..446b6f89c --- /dev/null +++ b/yara-x-dump/src/tests/testdata/macho_x86_file.automatic.None.out @@ -0,0 +1,147 @@ +>>> +macho: +magic: 4277009102 (0xfeedface) +cputype: 7 +cpusubtype: 3 +filetype: 2 +ncmds: 16 +sizeofcmds: 1060 +flags: 18874501 (0x1200085) +number_of_segments: 4 +# Nested segments structure +segments: + - cmd: 1 + cmdsize: 56 + segname: "__PAGEZERO" + vmaddr: 0 (0x0) + vmsize: 4096 (0x1000) + fileoff: 0 + filesize: 0 + maxprot: 0 (0x0) + initprot: 0 (0x0) + nsects: 0 + flags: 0 (0x0) + - cmd: 1 + cmdsize: 396 + segname: "__TEXT" + vmaddr: 4096 (0x1000) + vmsize: 4096 (0x1000) + fileoff: 0 + filesize: 4096 + maxprot: 7 (0x7) + initprot: 5 (0x5) + nsects: 5 + flags: 0 (0x0) + # Nested sections structure + sections: + - segname: "__TEXT" + sectname: "__text" + addr: 7824 (0x1e90) + size: 166 (0xa6) + offset: 3728 + align: 4 + reloff: 0 + nreloc: 0 + flags: 2147484672 (0x80000400) + reserved1: 0 + reserved2: 0 + - segname: "__TEXT" + sectname: "__symbol_stub" + addr: 7990 (0x1f36) + size: 12 (0xc) + offset: 3894 + align: 1 + reloff: 0 + nreloc: 0 + flags: 2147484936 (0x80000508) + reserved1: 0 + reserved2: 6 + - segname: "__TEXT" + sectname: "__stub_helper" + addr: 8004 (0x1f44) + size: 32 (0x20) + offset: 3908 + align: 2 + reloff: 0 + nreloc: 0 + flags: 2147484928 (0x80000500) + reserved1: 0 + reserved2: 0 + - segname: "__TEXT" + sectname: "__cstring" + addr: 8036 (0x1f64) + size: 69 (0x45) + offset: 3940 + align: 0 + reloff: 0 + nreloc: 0 + flags: 2 (0x2) + reserved1: 0 + reserved2: 0 + - segname: "__TEXT" + sectname: "__unwind_info" + addr: 8108 (0x1fac) + size: 72 (0x48) + offset: 4012 + align: 2 + reloff: 0 + nreloc: 0 + flags: 0 (0x0) + reserved1: 0 + reserved2: 0 + - cmd: 1 + cmdsize: 192 + segname: "__DATA" + vmaddr: 8192 (0x2000) + vmsize: 4096 (0x1000) + fileoff: 4096 + filesize: 4096 + maxprot: 7 (0x7) + initprot: 3 (0x3) + nsects: 2 + flags: 0 (0x0) + # Nested sections structure + sections: + - segname: "__DATA" + sectname: "__nl_symbol_ptr" + addr: 8192 (0x2000) + size: 8 (0x8) + offset: 4096 + align: 2 + reloff: 0 + nreloc: 0 + flags: 6 (0x6) + reserved1: 2 + reserved2: 0 + - segname: "__DATA" + sectname: "__la_symbol_ptr" + addr: 8200 (0x2008) + size: 8 (0x8) + offset: 4104 + align: 2 + reloff: 0 + nreloc: 0 + flags: 7 (0x7) + reserved1: 4 + reserved2: 0 + - cmd: 1 + cmdsize: 56 + segname: "__LINKEDIT" + vmaddr: 12288 (0x3000) + vmsize: 4096 (0x1000) + fileoff: 8192 + filesize: 280 + maxprot: 7 (0x7) + initprot: 1 (0x1) + nsects: 0 + flags: 0 (0x0) +# Nested dylibs structure +dylibs: + - name: "/usr/lib/libSystem.B.dylib" + timestamp: 2 (1970-01-01 00:00:02 UTC) + compatibility_version: "1.0.0" + current_version: "1213.0.0" +entry_point: 3728 +stack_size: 0 + +<<< \ No newline at end of file diff --git a/yara-x-dump/src/tests/testdata/macho_x86_file.automatic.human-readable.out b/yara-x-dump/src/tests/testdata/macho_x86_file.automatic.human-readable.out new file mode 100644 index 000000000..446b6f89c --- /dev/null +++ b/yara-x-dump/src/tests/testdata/macho_x86_file.automatic.human-readable.out @@ -0,0 +1,147 @@ +>>> +macho: +magic: 4277009102 (0xfeedface) +cputype: 7 +cpusubtype: 3 +filetype: 2 +ncmds: 16 +sizeofcmds: 1060 +flags: 18874501 (0x1200085) +number_of_segments: 4 +# Nested segments structure +segments: + - cmd: 1 + cmdsize: 56 + segname: "__PAGEZERO" + vmaddr: 0 (0x0) + vmsize: 4096 (0x1000) + fileoff: 0 + filesize: 0 + maxprot: 0 (0x0) + initprot: 0 (0x0) + nsects: 0 + flags: 0 (0x0) + - cmd: 1 + cmdsize: 396 + segname: "__TEXT" + vmaddr: 4096 (0x1000) + vmsize: 4096 (0x1000) + fileoff: 0 + filesize: 4096 + maxprot: 7 (0x7) + initprot: 5 (0x5) + nsects: 5 + flags: 0 (0x0) + # Nested sections structure + sections: + - segname: "__TEXT" + sectname: "__text" + addr: 7824 (0x1e90) + size: 166 (0xa6) + offset: 3728 + align: 4 + reloff: 0 + nreloc: 0 + flags: 2147484672 (0x80000400) + reserved1: 0 + reserved2: 0 + - segname: "__TEXT" + sectname: "__symbol_stub" + addr: 7990 (0x1f36) + size: 12 (0xc) + offset: 3894 + align: 1 + reloff: 0 + nreloc: 0 + flags: 2147484936 (0x80000508) + reserved1: 0 + reserved2: 6 + - segname: "__TEXT" + sectname: "__stub_helper" + addr: 8004 (0x1f44) + size: 32 (0x20) + offset: 3908 + align: 2 + reloff: 0 + nreloc: 0 + flags: 2147484928 (0x80000500) + reserved1: 0 + reserved2: 0 + - segname: "__TEXT" + sectname: "__cstring" + addr: 8036 (0x1f64) + size: 69 (0x45) + offset: 3940 + align: 0 + reloff: 0 + nreloc: 0 + flags: 2 (0x2) + reserved1: 0 + reserved2: 0 + - segname: "__TEXT" + sectname: "__unwind_info" + addr: 8108 (0x1fac) + size: 72 (0x48) + offset: 4012 + align: 2 + reloff: 0 + nreloc: 0 + flags: 0 (0x0) + reserved1: 0 + reserved2: 0 + - cmd: 1 + cmdsize: 192 + segname: "__DATA" + vmaddr: 8192 (0x2000) + vmsize: 4096 (0x1000) + fileoff: 4096 + filesize: 4096 + maxprot: 7 (0x7) + initprot: 3 (0x3) + nsects: 2 + flags: 0 (0x0) + # Nested sections structure + sections: + - segname: "__DATA" + sectname: "__nl_symbol_ptr" + addr: 8192 (0x2000) + size: 8 (0x8) + offset: 4096 + align: 2 + reloff: 0 + nreloc: 0 + flags: 6 (0x6) + reserved1: 2 + reserved2: 0 + - segname: "__DATA" + sectname: "__la_symbol_ptr" + addr: 8200 (0x2008) + size: 8 (0x8) + offset: 4104 + align: 2 + reloff: 0 + nreloc: 0 + flags: 7 (0x7) + reserved1: 4 + reserved2: 0 + - cmd: 1 + cmdsize: 56 + segname: "__LINKEDIT" + vmaddr: 12288 (0x3000) + vmsize: 4096 (0x1000) + fileoff: 8192 + filesize: 280 + maxprot: 7 (0x7) + initprot: 1 (0x1) + nsects: 0 + flags: 0 (0x0) +# Nested dylibs structure +dylibs: + - name: "/usr/lib/libSystem.B.dylib" + timestamp: 2 (1970-01-01 00:00:02 UTC) + compatibility_version: "1.0.0" + current_version: "1213.0.0" +entry_point: 3728 +stack_size: 0 + +<<< \ No newline at end of file diff --git a/yara-x-dump/src/tests/testdata/macho_x86_file.automatic.json.out b/yara-x-dump/src/tests/testdata/macho_x86_file.automatic.json.out new file mode 100644 index 000000000..e2eae2a1c --- /dev/null +++ b/yara-x-dump/src/tests/testdata/macho_x86_file.automatic.json.out @@ -0,0 +1,172 @@ +>>> +macho: +{ + "magic": 4277009102, + "cputype": 7, + "cpusubtype": 3, + "filetype": 2, + "ncmds": 16, + "sizeofcmds": 1060, + "flags": 18874501, + "numberOfSegments": "4", + "segments": [ + { + "cmd": 1, + "cmdsize": 56, + "segname": "__PAGEZERO", + "vmaddr": "0", + "vmsize": "4096", + "fileoff": "0", + "filesize": "0", + "maxprot": 0, + "initprot": 0, + "nsects": 0, + "flags": 0 + }, + { + "cmd": 1, + "cmdsize": 396, + "segname": "__TEXT", + "vmaddr": "4096", + "vmsize": "4096", + "fileoff": "0", + "filesize": "4096", + "maxprot": 7, + "initprot": 5, + "nsects": 5, + "flags": 0, + "sections": [ + { + "segname": "__TEXT", + "sectname": "__text", + "addr": "7824", + "size": "166", + "offset": 3728, + "align": 4, + "reloff": 0, + "nreloc": 0, + "flags": 2147484672, + "reserved1": 0, + "reserved2": 0 + }, + { + "segname": "__TEXT", + "sectname": "__symbol_stub", + "addr": "7990", + "size": "12", + "offset": 3894, + "align": 1, + "reloff": 0, + "nreloc": 0, + "flags": 2147484936, + "reserved1": 0, + "reserved2": 6 + }, + { + "segname": "__TEXT", + "sectname": "__stub_helper", + "addr": "8004", + "size": "32", + "offset": 3908, + "align": 2, + "reloff": 0, + "nreloc": 0, + "flags": 2147484928, + "reserved1": 0, + "reserved2": 0 + }, + { + "segname": "__TEXT", + "sectname": "__cstring", + "addr": "8036", + "size": "69", + "offset": 3940, + "align": 0, + "reloff": 0, + "nreloc": 0, + "flags": 2, + "reserved1": 0, + "reserved2": 0 + }, + { + "segname": "__TEXT", + "sectname": "__unwind_info", + "addr": "8108", + "size": "72", + "offset": 4012, + "align": 2, + "reloff": 0, + "nreloc": 0, + "flags": 0, + "reserved1": 0, + "reserved2": 0 + } + ] + }, + { + "cmd": 1, + "cmdsize": 192, + "segname": "__DATA", + "vmaddr": "8192", + "vmsize": "4096", + "fileoff": "4096", + "filesize": "4096", + "maxprot": 7, + "initprot": 3, + "nsects": 2, + "flags": 0, + "sections": [ + { + "segname": "__DATA", + "sectname": "__nl_symbol_ptr", + "addr": "8192", + "size": "8", + "offset": 4096, + "align": 2, + "reloff": 0, + "nreloc": 0, + "flags": 6, + "reserved1": 2, + "reserved2": 0 + }, + { + "segname": "__DATA", + "sectname": "__la_symbol_ptr", + "addr": "8200", + "size": "8", + "offset": 4104, + "align": 2, + "reloff": 0, + "nreloc": 0, + "flags": 7, + "reserved1": 4, + "reserved2": 0 + } + ] + }, + { + "cmd": 1, + "cmdsize": 56, + "segname": "__LINKEDIT", + "vmaddr": "12288", + "vmsize": "4096", + "fileoff": "8192", + "filesize": "280", + "maxprot": 7, + "initprot": 1, + "nsects": 0, + "flags": 0 + } + ], + "dylibs": [ + { + "name": "/usr/lib/libSystem.B.dylib", + "timestamp": 2, + "compatibilityVersion": "1.0.0", + "currentVersion": "1213.0.0" + } + ], + "entryPoint": "3728", + "stackSize": "0" +} +<<< \ No newline at end of file diff --git a/yara-x-dump/src/tests/testdata/macho_x86_file.automatic.toml.out b/yara-x-dump/src/tests/testdata/macho_x86_file.automatic.toml.out new file mode 100644 index 000000000..1d7844c93 --- /dev/null +++ b/yara-x-dump/src/tests/testdata/macho_x86_file.automatic.toml.out @@ -0,0 +1,163 @@ +>>> +macho: +cpusubtype = 3 +cputype = 7 +entryPoint = "3728" +filetype = 2 +flags = 18874501 +magic = 4277009102 +ncmds = 16 +numberOfSegments = "4" +sizeofcmds = 1060 +stackSize = "0" + +[[dylibs]] +compatibilityVersion = "1.0.0" +currentVersion = "1213.0.0" +name = "/usr/lib/libSystem.B.dylib" +timestamp = 2 + +[[segments]] +cmd = 1 +cmdsize = 56 +fileoff = "0" +filesize = "0" +flags = 0 +initprot = 0 +maxprot = 0 +nsects = 0 +segname = "__PAGEZERO" +vmaddr = "0" +vmsize = "4096" + +[[segments]] +cmd = 1 +cmdsize = 396 +fileoff = "0" +filesize = "4096" +flags = 0 +initprot = 5 +maxprot = 7 +nsects = 5 +segname = "__TEXT" +vmaddr = "4096" +vmsize = "4096" + +[[segments.sections]] +addr = "7824" +align = 4 +flags = 2147484672 +nreloc = 0 +offset = 3728 +reloff = 0 +reserved1 = 0 +reserved2 = 0 +sectname = "__text" +segname = "__TEXT" +size = "166" + +[[segments.sections]] +addr = "7990" +align = 1 +flags = 2147484936 +nreloc = 0 +offset = 3894 +reloff = 0 +reserved1 = 0 +reserved2 = 6 +sectname = "__symbol_stub" +segname = "__TEXT" +size = "12" + +[[segments.sections]] +addr = "8004" +align = 2 +flags = 2147484928 +nreloc = 0 +offset = 3908 +reloff = 0 +reserved1 = 0 +reserved2 = 0 +sectname = "__stub_helper" +segname = "__TEXT" +size = "32" + +[[segments.sections]] +addr = "8036" +align = 0 +flags = 2 +nreloc = 0 +offset = 3940 +reloff = 0 +reserved1 = 0 +reserved2 = 0 +sectname = "__cstring" +segname = "__TEXT" +size = "69" + +[[segments.sections]] +addr = "8108" +align = 2 +flags = 0 +nreloc = 0 +offset = 4012 +reloff = 0 +reserved1 = 0 +reserved2 = 0 +sectname = "__unwind_info" +segname = "__TEXT" +size = "72" + +[[segments]] +cmd = 1 +cmdsize = 192 +fileoff = "4096" +filesize = "4096" +flags = 0 +initprot = 3 +maxprot = 7 +nsects = 2 +segname = "__DATA" +vmaddr = "8192" +vmsize = "4096" + +[[segments.sections]] +addr = "8192" +align = 2 +flags = 6 +nreloc = 0 +offset = 4096 +reloff = 0 +reserved1 = 2 +reserved2 = 0 +sectname = "__nl_symbol_ptr" +segname = "__DATA" +size = "8" + +[[segments.sections]] +addr = "8200" +align = 2 +flags = 7 +nreloc = 0 +offset = 4104 +reloff = 0 +reserved1 = 4 +reserved2 = 0 +sectname = "__la_symbol_ptr" +segname = "__DATA" +size = "8" + +[[segments]] +cmd = 1 +cmdsize = 56 +fileoff = "8192" +filesize = "280" +flags = 0 +initprot = 1 +maxprot = 7 +nsects = 0 +segname = "__LINKEDIT" +vmaddr = "12288" +vmsize = "4096" + +<<< \ No newline at end of file diff --git a/yara-x-dump/src/tests/testdata/macho_x86_file.automatic.xml.out b/yara-x-dump/src/tests/testdata/macho_x86_file.automatic.xml.out new file mode 100644 index 000000000..48d3c0daa --- /dev/null +++ b/yara-x-dump/src/tests/testdata/macho_x86_file.automatic.xml.out @@ -0,0 +1,165 @@ +>>> +macho: + + + 4277009102 + 7 + 3 + 2 + 16 + 1060 + 18874501 + 4 + + 1 + 56 + __PAGEZERO + 0 + 4096 + 0 + 0 + 0 + 0 + 0 + 0 + + + 1 + 396 + __TEXT + 4096 + 4096 + 0 + 4096 + 7 + 5 + 5 + 0 + + __TEXT + __text + 7824 + 166 + 3728 + 4 + 0 + 0 + 2147484672 + 0 + 0 + + + __TEXT + __symbol_stub + 7990 + 12 + 3894 + 1 + 0 + 0 + 2147484936 + 0 + 6 + + + __TEXT + __stub_helper + 8004 + 32 + 3908 + 2 + 0 + 0 + 2147484928 + 0 + 0 + + + __TEXT + __cstring + 8036 + 69 + 3940 + 0 + 0 + 0 + 2 + 0 + 0 + + + __TEXT + __unwind_info + 8108 + 72 + 4012 + 2 + 0 + 0 + 0 + 0 + 0 + + + + 1 + 192 + __DATA + 8192 + 4096 + 4096 + 4096 + 7 + 3 + 2 + 0 + + __DATA + __nl_symbol_ptr + 8192 + 8 + 4096 + 2 + 0 + 0 + 6 + 2 + 0 + + + __DATA + __la_symbol_ptr + 8200 + 8 + 4104 + 2 + 0 + 0 + 7 + 4 + 0 + + + + 1 + 56 + __LINKEDIT + 12288 + 4096 + 8192 + 280 + 7 + 1 + 0 + 0 + + + /usr/lib/libSystem.B.dylib + 2 + 1.0.0 + 1213.0.0 + + 3728 + 0 + +<<< \ No newline at end of file diff --git a/yara-x-dump/src/tests/testdata/macho_x86_file.automatic.yaml.out b/yara-x-dump/src/tests/testdata/macho_x86_file.automatic.yaml.out new file mode 100644 index 000000000..ef6fbce6a --- /dev/null +++ b/yara-x-dump/src/tests/testdata/macho_x86_file.automatic.yaml.out @@ -0,0 +1,144 @@ +>>> +macho: +--- +magic: 4277009102 +cputype: 7 +cpusubtype: 3 +filetype: 2 +ncmds: 16 +sizeofcmds: 1060 +flags: 18874501 +numberOfSegments: "4" +segments: + - cmd: 1 + cmdsize: 56 + segname: __PAGEZERO + vmaddr: "0" + vmsize: "4096" + fileoff: "0" + filesize: "0" + maxprot: 0 + initprot: 0 + nsects: 0 + flags: 0 + - cmd: 1 + cmdsize: 396 + segname: __TEXT + vmaddr: "4096" + vmsize: "4096" + fileoff: "0" + filesize: "4096" + maxprot: 7 + initprot: 5 + nsects: 5 + flags: 0 + sections: + - segname: __TEXT + sectname: __text + addr: "7824" + size: "166" + offset: 3728 + align: 4 + reloff: 0 + nreloc: 0 + flags: 2147484672 + reserved1: 0 + reserved2: 0 + - segname: __TEXT + sectname: __symbol_stub + addr: "7990" + size: "12" + offset: 3894 + align: 1 + reloff: 0 + nreloc: 0 + flags: 2147484936 + reserved1: 0 + reserved2: 6 + - segname: __TEXT + sectname: __stub_helper + addr: "8004" + size: "32" + offset: 3908 + align: 2 + reloff: 0 + nreloc: 0 + flags: 2147484928 + reserved1: 0 + reserved2: 0 + - segname: __TEXT + sectname: __cstring + addr: "8036" + size: "69" + offset: 3940 + align: 0 + reloff: 0 + nreloc: 0 + flags: 2 + reserved1: 0 + reserved2: 0 + - segname: __TEXT + sectname: __unwind_info + addr: "8108" + size: "72" + offset: 4012 + align: 2 + reloff: 0 + nreloc: 0 + flags: 0 + reserved1: 0 + reserved2: 0 + - cmd: 1 + cmdsize: 192 + segname: __DATA + vmaddr: "8192" + vmsize: "4096" + fileoff: "4096" + filesize: "4096" + maxprot: 7 + initprot: 3 + nsects: 2 + flags: 0 + sections: + - segname: __DATA + sectname: __nl_symbol_ptr + addr: "8192" + size: "8" + offset: 4096 + align: 2 + reloff: 0 + nreloc: 0 + flags: 6 + reserved1: 2 + reserved2: 0 + - segname: __DATA + sectname: __la_symbol_ptr + addr: "8200" + size: "8" + offset: 4104 + align: 2 + reloff: 0 + nreloc: 0 + flags: 7 + reserved1: 4 + reserved2: 0 + - cmd: 1 + cmdsize: 56 + segname: __LINKEDIT + vmaddr: "12288" + vmsize: "4096" + fileoff: "8192" + filesize: "280" + maxprot: 7 + initprot: 1 + nsects: 0 + flags: 0 +dylibs: + - name: /usr/lib/libSystem.B.dylib + timestamp: 2 + compatibilityVersion: 1.0.0 + currentVersion: 1213.0.0 +entryPoint: "3728" +stackSize: "0" + +<<< \ No newline at end of file diff --git a/yara-x-dump/src/tests/testdata/macho_x86_file.in b/yara-x-dump/src/tests/testdata/macho_x86_file.in new file mode 100644 index 000000000..b81321cb0 --- /dev/null +++ b/yara-x-dump/src/tests/testdata/macho_x86_file.in @@ -0,0 +1,73 @@ +:10000000CEFAEDFE07000000030000000200000031 +:100010001000000024040000850020010100000001 +:10002000380000005F5F504147455A45524F00007D +:1000300000000000000000000010000000000000B0 +:1000400000000000000000000000000000000000B0 +:1000500000000000010000008C0100005F5F5445BB +:1000600058540000000000000000000000100000D4 +:100070000010000000000000001000000700000059 +:100080000500000005000000000000005F5F7465CF +:100090007874000000000000000000005F5F54451D +:1000A000585400000000000000000000901E0000F6 +:1000B000A6000000900E00000400000000000000F8 +:1000C00000000000000400800000000000000000AC +:1000D0005F5F73796D626F6C5F73747562000000AF +:1000E0005F5F54455854000000000000000000000D +:1000F000361F00000C000000360F00000100000059 +:100100000000000000000000080500800000000062 +:10011000060000005F5F737475625F68656C7065F0 +:10012000720000005F5F544558540000000000005A +:1001300000000000441F000020000000440F0000E9 +:100140000200000000000000000000000005008028 +:1001500000000000000000005F5F63737472696E4E +:1001600067000000000000005F5F54455854000025 +:100170000000000000000000641F000045000000B7 +:10018000640F0000000000000000000000000000FC +:100190000200000000000000000000005F5F756EBC +:1001A00077696E645F696E666F0000005F5F54453B +:1001B000585400000000000000000000AC1F0000C8 +:1001C00048000000AC0F000002000000000000002A +:1001D000000000000000000000000000000000001F +:1001E00001000000C00000005F5F44415441000076 +:1001F00000000000000000000020000000100000CF +:1002000000100000001000000700000003000000C4 +:1002100002000000000000005F5F6E6C5F73796D8C +:10022000626F6C5F707472005F5F44415441000004 +:100230000000000000000000002000000800000096 +:10024000001000000200000000000000000000009C +:100250000600000002000000000000005F5F6C610B +:100260005F73796D626F6C5F707472005F5F4441A1 +:1002700054410000000000000000000008200000C1 +:10028000080000000810000002000000000000004C +:100290000000000007000000040000000000000053 +:1002A00001000000380000005F5F4C494E4B4544A0 +:1002B0004954000000000000003000000010000061 +:1002C00000200000180100000700000001000000ED +:1002D000000000000000000022000080300000004C +:1002E0000020000010000000102000001800000096 +:1002F0000000000000000000282000001C0000009A +:10030000442000002C000000020000001800000043 +:100310008820000005000000DC2000003C000000F8 +:100320000B00000050000000000000000000000072 +:1003300000000000020000000200000003000000B6 +:1003400000000000000000000000000000000000AD +:100350000000000000000000C420000006000000B3 +:10036000000000000000000000000000000000008D +:100370000E0000001C0000000C0000002F757372BE +:100380002F6C69622F64796C640000001B00000010 +:10039000180000005FB5950F40253D4FA8FB96481B +:1003A000C1740790240000001000000000090A003A +:1003B000000A0A002A0000001000000000000000EF +:1003C000000000002800008018000000900E0000CF +:1003D0000000000000000000000000000C00000011 +:1003E0003400000018000000020000000000BD04FE +:1003F000000001002F7573722F6C69622F6C6962A7 +:1004000053797374656D2E422E64796C69620000B5 +:100410002600000010000000702000000400000012 +:1004200029000000100000007420000000000000FF +:100430002B000000100000007420000014000000D9 +:1004400000000000000000000000000000000000AC +:10045000000000000000000000000000000000009C +:040460000000000098 +:00000001FF + diff --git a/yara-x-dump/src/tests/testdata/macho_x86_file.lnk.None.out b/yara-x-dump/src/tests/testdata/macho_x86_file.lnk.None.out new file mode 100644 index 000000000..9eff539a9 --- /dev/null +++ b/yara-x-dump/src/tests/testdata/macho_x86_file.lnk.None.out @@ -0,0 +1,5 @@ +>>> +lnk: +is_lnk: false + +<<< \ No newline at end of file diff --git a/yara-x-dump/src/tests/testdata/macho_x86_file.lnk.human-readable.out b/yara-x-dump/src/tests/testdata/macho_x86_file.lnk.human-readable.out new file mode 100644 index 000000000..9eff539a9 --- /dev/null +++ b/yara-x-dump/src/tests/testdata/macho_x86_file.lnk.human-readable.out @@ -0,0 +1,5 @@ +>>> +lnk: +is_lnk: false + +<<< \ No newline at end of file diff --git a/yara-x-dump/src/tests/testdata/macho_x86_file.lnk.json.out b/yara-x-dump/src/tests/testdata/macho_x86_file.lnk.json.out new file mode 100644 index 000000000..d1389a5f1 --- /dev/null +++ b/yara-x-dump/src/tests/testdata/macho_x86_file.lnk.json.out @@ -0,0 +1,6 @@ +>>> +lnk: +{ + "isLnk": false +} +<<< \ No newline at end of file diff --git a/yara-x-dump/src/tests/testdata/macho_x86_file.lnk.toml.out b/yara-x-dump/src/tests/testdata/macho_x86_file.lnk.toml.out new file mode 100644 index 000000000..94c59c00d --- /dev/null +++ b/yara-x-dump/src/tests/testdata/macho_x86_file.lnk.toml.out @@ -0,0 +1,5 @@ +>>> +lnk: +isLnk = false + +<<< \ No newline at end of file diff --git a/yara-x-dump/src/tests/testdata/macho_x86_file.lnk.xml.out b/yara-x-dump/src/tests/testdata/macho_x86_file.lnk.xml.out new file mode 100644 index 000000000..931137069 --- /dev/null +++ b/yara-x-dump/src/tests/testdata/macho_x86_file.lnk.xml.out @@ -0,0 +1,7 @@ +>>> +lnk: + + + false + +<<< \ No newline at end of file diff --git a/yara-x-dump/src/tests/testdata/macho_x86_file.lnk.yaml.out b/yara-x-dump/src/tests/testdata/macho_x86_file.lnk.yaml.out new file mode 100644 index 000000000..cca5783af --- /dev/null +++ b/yara-x-dump/src/tests/testdata/macho_x86_file.lnk.yaml.out @@ -0,0 +1,6 @@ +>>> +lnk: +--- +isLnk: false + +<<< \ No newline at end of file diff --git a/yara-x-dump/src/tests/testdata/macho_x86_file.macho.None.out b/yara-x-dump/src/tests/testdata/macho_x86_file.macho.None.out new file mode 100644 index 000000000..446b6f89c --- /dev/null +++ b/yara-x-dump/src/tests/testdata/macho_x86_file.macho.None.out @@ -0,0 +1,147 @@ +>>> +macho: +magic: 4277009102 (0xfeedface) +cputype: 7 +cpusubtype: 3 +filetype: 2 +ncmds: 16 +sizeofcmds: 1060 +flags: 18874501 (0x1200085) +number_of_segments: 4 +# Nested segments structure +segments: + - cmd: 1 + cmdsize: 56 + segname: "__PAGEZERO" + vmaddr: 0 (0x0) + vmsize: 4096 (0x1000) + fileoff: 0 + filesize: 0 + maxprot: 0 (0x0) + initprot: 0 (0x0) + nsects: 0 + flags: 0 (0x0) + - cmd: 1 + cmdsize: 396 + segname: "__TEXT" + vmaddr: 4096 (0x1000) + vmsize: 4096 (0x1000) + fileoff: 0 + filesize: 4096 + maxprot: 7 (0x7) + initprot: 5 (0x5) + nsects: 5 + flags: 0 (0x0) + # Nested sections structure + sections: + - segname: "__TEXT" + sectname: "__text" + addr: 7824 (0x1e90) + size: 166 (0xa6) + offset: 3728 + align: 4 + reloff: 0 + nreloc: 0 + flags: 2147484672 (0x80000400) + reserved1: 0 + reserved2: 0 + - segname: "__TEXT" + sectname: "__symbol_stub" + addr: 7990 (0x1f36) + size: 12 (0xc) + offset: 3894 + align: 1 + reloff: 0 + nreloc: 0 + flags: 2147484936 (0x80000508) + reserved1: 0 + reserved2: 6 + - segname: "__TEXT" + sectname: "__stub_helper" + addr: 8004 (0x1f44) + size: 32 (0x20) + offset: 3908 + align: 2 + reloff: 0 + nreloc: 0 + flags: 2147484928 (0x80000500) + reserved1: 0 + reserved2: 0 + - segname: "__TEXT" + sectname: "__cstring" + addr: 8036 (0x1f64) + size: 69 (0x45) + offset: 3940 + align: 0 + reloff: 0 + nreloc: 0 + flags: 2 (0x2) + reserved1: 0 + reserved2: 0 + - segname: "__TEXT" + sectname: "__unwind_info" + addr: 8108 (0x1fac) + size: 72 (0x48) + offset: 4012 + align: 2 + reloff: 0 + nreloc: 0 + flags: 0 (0x0) + reserved1: 0 + reserved2: 0 + - cmd: 1 + cmdsize: 192 + segname: "__DATA" + vmaddr: 8192 (0x2000) + vmsize: 4096 (0x1000) + fileoff: 4096 + filesize: 4096 + maxprot: 7 (0x7) + initprot: 3 (0x3) + nsects: 2 + flags: 0 (0x0) + # Nested sections structure + sections: + - segname: "__DATA" + sectname: "__nl_symbol_ptr" + addr: 8192 (0x2000) + size: 8 (0x8) + offset: 4096 + align: 2 + reloff: 0 + nreloc: 0 + flags: 6 (0x6) + reserved1: 2 + reserved2: 0 + - segname: "__DATA" + sectname: "__la_symbol_ptr" + addr: 8200 (0x2008) + size: 8 (0x8) + offset: 4104 + align: 2 + reloff: 0 + nreloc: 0 + flags: 7 (0x7) + reserved1: 4 + reserved2: 0 + - cmd: 1 + cmdsize: 56 + segname: "__LINKEDIT" + vmaddr: 12288 (0x3000) + vmsize: 4096 (0x1000) + fileoff: 8192 + filesize: 280 + maxprot: 7 (0x7) + initprot: 1 (0x1) + nsects: 0 + flags: 0 (0x0) +# Nested dylibs structure +dylibs: + - name: "/usr/lib/libSystem.B.dylib" + timestamp: 2 (1970-01-01 00:00:02 UTC) + compatibility_version: "1.0.0" + current_version: "1213.0.0" +entry_point: 3728 +stack_size: 0 + +<<< \ No newline at end of file diff --git a/yara-x-dump/src/tests/testdata/macho_x86_file.macho.human-readable.out b/yara-x-dump/src/tests/testdata/macho_x86_file.macho.human-readable.out new file mode 100644 index 000000000..446b6f89c --- /dev/null +++ b/yara-x-dump/src/tests/testdata/macho_x86_file.macho.human-readable.out @@ -0,0 +1,147 @@ +>>> +macho: +magic: 4277009102 (0xfeedface) +cputype: 7 +cpusubtype: 3 +filetype: 2 +ncmds: 16 +sizeofcmds: 1060 +flags: 18874501 (0x1200085) +number_of_segments: 4 +# Nested segments structure +segments: + - cmd: 1 + cmdsize: 56 + segname: "__PAGEZERO" + vmaddr: 0 (0x0) + vmsize: 4096 (0x1000) + fileoff: 0 + filesize: 0 + maxprot: 0 (0x0) + initprot: 0 (0x0) + nsects: 0 + flags: 0 (0x0) + - cmd: 1 + cmdsize: 396 + segname: "__TEXT" + vmaddr: 4096 (0x1000) + vmsize: 4096 (0x1000) + fileoff: 0 + filesize: 4096 + maxprot: 7 (0x7) + initprot: 5 (0x5) + nsects: 5 + flags: 0 (0x0) + # Nested sections structure + sections: + - segname: "__TEXT" + sectname: "__text" + addr: 7824 (0x1e90) + size: 166 (0xa6) + offset: 3728 + align: 4 + reloff: 0 + nreloc: 0 + flags: 2147484672 (0x80000400) + reserved1: 0 + reserved2: 0 + - segname: "__TEXT" + sectname: "__symbol_stub" + addr: 7990 (0x1f36) + size: 12 (0xc) + offset: 3894 + align: 1 + reloff: 0 + nreloc: 0 + flags: 2147484936 (0x80000508) + reserved1: 0 + reserved2: 6 + - segname: "__TEXT" + sectname: "__stub_helper" + addr: 8004 (0x1f44) + size: 32 (0x20) + offset: 3908 + align: 2 + reloff: 0 + nreloc: 0 + flags: 2147484928 (0x80000500) + reserved1: 0 + reserved2: 0 + - segname: "__TEXT" + sectname: "__cstring" + addr: 8036 (0x1f64) + size: 69 (0x45) + offset: 3940 + align: 0 + reloff: 0 + nreloc: 0 + flags: 2 (0x2) + reserved1: 0 + reserved2: 0 + - segname: "__TEXT" + sectname: "__unwind_info" + addr: 8108 (0x1fac) + size: 72 (0x48) + offset: 4012 + align: 2 + reloff: 0 + nreloc: 0 + flags: 0 (0x0) + reserved1: 0 + reserved2: 0 + - cmd: 1 + cmdsize: 192 + segname: "__DATA" + vmaddr: 8192 (0x2000) + vmsize: 4096 (0x1000) + fileoff: 4096 + filesize: 4096 + maxprot: 7 (0x7) + initprot: 3 (0x3) + nsects: 2 + flags: 0 (0x0) + # Nested sections structure + sections: + - segname: "__DATA" + sectname: "__nl_symbol_ptr" + addr: 8192 (0x2000) + size: 8 (0x8) + offset: 4096 + align: 2 + reloff: 0 + nreloc: 0 + flags: 6 (0x6) + reserved1: 2 + reserved2: 0 + - segname: "__DATA" + sectname: "__la_symbol_ptr" + addr: 8200 (0x2008) + size: 8 (0x8) + offset: 4104 + align: 2 + reloff: 0 + nreloc: 0 + flags: 7 (0x7) + reserved1: 4 + reserved2: 0 + - cmd: 1 + cmdsize: 56 + segname: "__LINKEDIT" + vmaddr: 12288 (0x3000) + vmsize: 4096 (0x1000) + fileoff: 8192 + filesize: 280 + maxprot: 7 (0x7) + initprot: 1 (0x1) + nsects: 0 + flags: 0 (0x0) +# Nested dylibs structure +dylibs: + - name: "/usr/lib/libSystem.B.dylib" + timestamp: 2 (1970-01-01 00:00:02 UTC) + compatibility_version: "1.0.0" + current_version: "1213.0.0" +entry_point: 3728 +stack_size: 0 + +<<< \ No newline at end of file diff --git a/yara-x-dump/src/tests/testdata/macho_x86_file.macho.json.out b/yara-x-dump/src/tests/testdata/macho_x86_file.macho.json.out new file mode 100644 index 000000000..e2eae2a1c --- /dev/null +++ b/yara-x-dump/src/tests/testdata/macho_x86_file.macho.json.out @@ -0,0 +1,172 @@ +>>> +macho: +{ + "magic": 4277009102, + "cputype": 7, + "cpusubtype": 3, + "filetype": 2, + "ncmds": 16, + "sizeofcmds": 1060, + "flags": 18874501, + "numberOfSegments": "4", + "segments": [ + { + "cmd": 1, + "cmdsize": 56, + "segname": "__PAGEZERO", + "vmaddr": "0", + "vmsize": "4096", + "fileoff": "0", + "filesize": "0", + "maxprot": 0, + "initprot": 0, + "nsects": 0, + "flags": 0 + }, + { + "cmd": 1, + "cmdsize": 396, + "segname": "__TEXT", + "vmaddr": "4096", + "vmsize": "4096", + "fileoff": "0", + "filesize": "4096", + "maxprot": 7, + "initprot": 5, + "nsects": 5, + "flags": 0, + "sections": [ + { + "segname": "__TEXT", + "sectname": "__text", + "addr": "7824", + "size": "166", + "offset": 3728, + "align": 4, + "reloff": 0, + "nreloc": 0, + "flags": 2147484672, + "reserved1": 0, + "reserved2": 0 + }, + { + "segname": "__TEXT", + "sectname": "__symbol_stub", + "addr": "7990", + "size": "12", + "offset": 3894, + "align": 1, + "reloff": 0, + "nreloc": 0, + "flags": 2147484936, + "reserved1": 0, + "reserved2": 6 + }, + { + "segname": "__TEXT", + "sectname": "__stub_helper", + "addr": "8004", + "size": "32", + "offset": 3908, + "align": 2, + "reloff": 0, + "nreloc": 0, + "flags": 2147484928, + "reserved1": 0, + "reserved2": 0 + }, + { + "segname": "__TEXT", + "sectname": "__cstring", + "addr": "8036", + "size": "69", + "offset": 3940, + "align": 0, + "reloff": 0, + "nreloc": 0, + "flags": 2, + "reserved1": 0, + "reserved2": 0 + }, + { + "segname": "__TEXT", + "sectname": "__unwind_info", + "addr": "8108", + "size": "72", + "offset": 4012, + "align": 2, + "reloff": 0, + "nreloc": 0, + "flags": 0, + "reserved1": 0, + "reserved2": 0 + } + ] + }, + { + "cmd": 1, + "cmdsize": 192, + "segname": "__DATA", + "vmaddr": "8192", + "vmsize": "4096", + "fileoff": "4096", + "filesize": "4096", + "maxprot": 7, + "initprot": 3, + "nsects": 2, + "flags": 0, + "sections": [ + { + "segname": "__DATA", + "sectname": "__nl_symbol_ptr", + "addr": "8192", + "size": "8", + "offset": 4096, + "align": 2, + "reloff": 0, + "nreloc": 0, + "flags": 6, + "reserved1": 2, + "reserved2": 0 + }, + { + "segname": "__DATA", + "sectname": "__la_symbol_ptr", + "addr": "8200", + "size": "8", + "offset": 4104, + "align": 2, + "reloff": 0, + "nreloc": 0, + "flags": 7, + "reserved1": 4, + "reserved2": 0 + } + ] + }, + { + "cmd": 1, + "cmdsize": 56, + "segname": "__LINKEDIT", + "vmaddr": "12288", + "vmsize": "4096", + "fileoff": "8192", + "filesize": "280", + "maxprot": 7, + "initprot": 1, + "nsects": 0, + "flags": 0 + } + ], + "dylibs": [ + { + "name": "/usr/lib/libSystem.B.dylib", + "timestamp": 2, + "compatibilityVersion": "1.0.0", + "currentVersion": "1213.0.0" + } + ], + "entryPoint": "3728", + "stackSize": "0" +} +<<< \ No newline at end of file diff --git a/yara-x-dump/src/tests/testdata/macho_x86_file.macho.toml.out b/yara-x-dump/src/tests/testdata/macho_x86_file.macho.toml.out new file mode 100644 index 000000000..1d7844c93 --- /dev/null +++ b/yara-x-dump/src/tests/testdata/macho_x86_file.macho.toml.out @@ -0,0 +1,163 @@ +>>> +macho: +cpusubtype = 3 +cputype = 7 +entryPoint = "3728" +filetype = 2 +flags = 18874501 +magic = 4277009102 +ncmds = 16 +numberOfSegments = "4" +sizeofcmds = 1060 +stackSize = "0" + +[[dylibs]] +compatibilityVersion = "1.0.0" +currentVersion = "1213.0.0" +name = "/usr/lib/libSystem.B.dylib" +timestamp = 2 + +[[segments]] +cmd = 1 +cmdsize = 56 +fileoff = "0" +filesize = "0" +flags = 0 +initprot = 0 +maxprot = 0 +nsects = 0 +segname = "__PAGEZERO" +vmaddr = "0" +vmsize = "4096" + +[[segments]] +cmd = 1 +cmdsize = 396 +fileoff = "0" +filesize = "4096" +flags = 0 +initprot = 5 +maxprot = 7 +nsects = 5 +segname = "__TEXT" +vmaddr = "4096" +vmsize = "4096" + +[[segments.sections]] +addr = "7824" +align = 4 +flags = 2147484672 +nreloc = 0 +offset = 3728 +reloff = 0 +reserved1 = 0 +reserved2 = 0 +sectname = "__text" +segname = "__TEXT" +size = "166" + +[[segments.sections]] +addr = "7990" +align = 1 +flags = 2147484936 +nreloc = 0 +offset = 3894 +reloff = 0 +reserved1 = 0 +reserved2 = 6 +sectname = "__symbol_stub" +segname = "__TEXT" +size = "12" + +[[segments.sections]] +addr = "8004" +align = 2 +flags = 2147484928 +nreloc = 0 +offset = 3908 +reloff = 0 +reserved1 = 0 +reserved2 = 0 +sectname = "__stub_helper" +segname = "__TEXT" +size = "32" + +[[segments.sections]] +addr = "8036" +align = 0 +flags = 2 +nreloc = 0 +offset = 3940 +reloff = 0 +reserved1 = 0 +reserved2 = 0 +sectname = "__cstring" +segname = "__TEXT" +size = "69" + +[[segments.sections]] +addr = "8108" +align = 2 +flags = 0 +nreloc = 0 +offset = 4012 +reloff = 0 +reserved1 = 0 +reserved2 = 0 +sectname = "__unwind_info" +segname = "__TEXT" +size = "72" + +[[segments]] +cmd = 1 +cmdsize = 192 +fileoff = "4096" +filesize = "4096" +flags = 0 +initprot = 3 +maxprot = 7 +nsects = 2 +segname = "__DATA" +vmaddr = "8192" +vmsize = "4096" + +[[segments.sections]] +addr = "8192" +align = 2 +flags = 6 +nreloc = 0 +offset = 4096 +reloff = 0 +reserved1 = 2 +reserved2 = 0 +sectname = "__nl_symbol_ptr" +segname = "__DATA" +size = "8" + +[[segments.sections]] +addr = "8200" +align = 2 +flags = 7 +nreloc = 0 +offset = 4104 +reloff = 0 +reserved1 = 4 +reserved2 = 0 +sectname = "__la_symbol_ptr" +segname = "__DATA" +size = "8" + +[[segments]] +cmd = 1 +cmdsize = 56 +fileoff = "8192" +filesize = "280" +flags = 0 +initprot = 1 +maxprot = 7 +nsects = 0 +segname = "__LINKEDIT" +vmaddr = "12288" +vmsize = "4096" + +<<< \ No newline at end of file diff --git a/yara-x-dump/src/tests/testdata/macho_x86_file.macho.xml.out b/yara-x-dump/src/tests/testdata/macho_x86_file.macho.xml.out new file mode 100644 index 000000000..48d3c0daa --- /dev/null +++ b/yara-x-dump/src/tests/testdata/macho_x86_file.macho.xml.out @@ -0,0 +1,165 @@ +>>> +macho: + + + 4277009102 + 7 + 3 + 2 + 16 + 1060 + 18874501 + 4 + + 1 + 56 + __PAGEZERO + 0 + 4096 + 0 + 0 + 0 + 0 + 0 + 0 + + + 1 + 396 + __TEXT + 4096 + 4096 + 0 + 4096 + 7 + 5 + 5 + 0 + + __TEXT + __text + 7824 + 166 + 3728 + 4 + 0 + 0 + 2147484672 + 0 + 0 + + + __TEXT + __symbol_stub + 7990 + 12 + 3894 + 1 + 0 + 0 + 2147484936 + 0 + 6 + + + __TEXT + __stub_helper + 8004 + 32 + 3908 + 2 + 0 + 0 + 2147484928 + 0 + 0 + + + __TEXT + __cstring + 8036 + 69 + 3940 + 0 + 0 + 0 + 2 + 0 + 0 + + + __TEXT + __unwind_info + 8108 + 72 + 4012 + 2 + 0 + 0 + 0 + 0 + 0 + + + + 1 + 192 + __DATA + 8192 + 4096 + 4096 + 4096 + 7 + 3 + 2 + 0 + + __DATA + __nl_symbol_ptr + 8192 + 8 + 4096 + 2 + 0 + 0 + 6 + 2 + 0 + + + __DATA + __la_symbol_ptr + 8200 + 8 + 4104 + 2 + 0 + 0 + 7 + 4 + 0 + + + + 1 + 56 + __LINKEDIT + 12288 + 4096 + 8192 + 280 + 7 + 1 + 0 + 0 + + + /usr/lib/libSystem.B.dylib + 2 + 1.0.0 + 1213.0.0 + + 3728 + 0 + +<<< \ No newline at end of file diff --git a/yara-x-dump/src/tests/testdata/macho_x86_file.macho.yaml.out b/yara-x-dump/src/tests/testdata/macho_x86_file.macho.yaml.out new file mode 100644 index 000000000..ef6fbce6a --- /dev/null +++ b/yara-x-dump/src/tests/testdata/macho_x86_file.macho.yaml.out @@ -0,0 +1,144 @@ +>>> +macho: +--- +magic: 4277009102 +cputype: 7 +cpusubtype: 3 +filetype: 2 +ncmds: 16 +sizeofcmds: 1060 +flags: 18874501 +numberOfSegments: "4" +segments: + - cmd: 1 + cmdsize: 56 + segname: __PAGEZERO + vmaddr: "0" + vmsize: "4096" + fileoff: "0" + filesize: "0" + maxprot: 0 + initprot: 0 + nsects: 0 + flags: 0 + - cmd: 1 + cmdsize: 396 + segname: __TEXT + vmaddr: "4096" + vmsize: "4096" + fileoff: "0" + filesize: "4096" + maxprot: 7 + initprot: 5 + nsects: 5 + flags: 0 + sections: + - segname: __TEXT + sectname: __text + addr: "7824" + size: "166" + offset: 3728 + align: 4 + reloff: 0 + nreloc: 0 + flags: 2147484672 + reserved1: 0 + reserved2: 0 + - segname: __TEXT + sectname: __symbol_stub + addr: "7990" + size: "12" + offset: 3894 + align: 1 + reloff: 0 + nreloc: 0 + flags: 2147484936 + reserved1: 0 + reserved2: 6 + - segname: __TEXT + sectname: __stub_helper + addr: "8004" + size: "32" + offset: 3908 + align: 2 + reloff: 0 + nreloc: 0 + flags: 2147484928 + reserved1: 0 + reserved2: 0 + - segname: __TEXT + sectname: __cstring + addr: "8036" + size: "69" + offset: 3940 + align: 0 + reloff: 0 + nreloc: 0 + flags: 2 + reserved1: 0 + reserved2: 0 + - segname: __TEXT + sectname: __unwind_info + addr: "8108" + size: "72" + offset: 4012 + align: 2 + reloff: 0 + nreloc: 0 + flags: 0 + reserved1: 0 + reserved2: 0 + - cmd: 1 + cmdsize: 192 + segname: __DATA + vmaddr: "8192" + vmsize: "4096" + fileoff: "4096" + filesize: "4096" + maxprot: 7 + initprot: 3 + nsects: 2 + flags: 0 + sections: + - segname: __DATA + sectname: __nl_symbol_ptr + addr: "8192" + size: "8" + offset: 4096 + align: 2 + reloff: 0 + nreloc: 0 + flags: 6 + reserved1: 2 + reserved2: 0 + - segname: __DATA + sectname: __la_symbol_ptr + addr: "8200" + size: "8" + offset: 4104 + align: 2 + reloff: 0 + nreloc: 0 + flags: 7 + reserved1: 4 + reserved2: 0 + - cmd: 1 + cmdsize: 56 + segname: __LINKEDIT + vmaddr: "12288" + vmsize: "4096" + fileoff: "8192" + filesize: "280" + maxprot: 7 + initprot: 1 + nsects: 0 + flags: 0 +dylibs: + - name: /usr/lib/libSystem.B.dylib + timestamp: 2 + compatibilityVersion: 1.0.0 + currentVersion: 1213.0.0 +entryPoint: "3728" +stackSize: "0" + +<<< \ No newline at end of file diff --git a/yara-x-dump/src/tests/testdata/macho_x86_file.macho_lnk.None.out b/yara-x-dump/src/tests/testdata/macho_x86_file.macho_lnk.None.out new file mode 100644 index 000000000..aee684dd0 --- /dev/null +++ b/yara-x-dump/src/tests/testdata/macho_x86_file.macho_lnk.None.out @@ -0,0 +1,151 @@ +>>> +lnk: +is_lnk: false + +<<<>>> +macho: +magic: 4277009102 (0xfeedface) +cputype: 7 +cpusubtype: 3 +filetype: 2 +ncmds: 16 +sizeofcmds: 1060 +flags: 18874501 (0x1200085) +number_of_segments: 4 +# Nested segments structure +segments: + - cmd: 1 + cmdsize: 56 + segname: "__PAGEZERO" + vmaddr: 0 (0x0) + vmsize: 4096 (0x1000) + fileoff: 0 + filesize: 0 + maxprot: 0 (0x0) + initprot: 0 (0x0) + nsects: 0 + flags: 0 (0x0) + - cmd: 1 + cmdsize: 396 + segname: "__TEXT" + vmaddr: 4096 (0x1000) + vmsize: 4096 (0x1000) + fileoff: 0 + filesize: 4096 + maxprot: 7 (0x7) + initprot: 5 (0x5) + nsects: 5 + flags: 0 (0x0) + # Nested sections structure + sections: + - segname: "__TEXT" + sectname: "__text" + addr: 7824 (0x1e90) + size: 166 (0xa6) + offset: 3728 + align: 4 + reloff: 0 + nreloc: 0 + flags: 2147484672 (0x80000400) + reserved1: 0 + reserved2: 0 + - segname: "__TEXT" + sectname: "__symbol_stub" + addr: 7990 (0x1f36) + size: 12 (0xc) + offset: 3894 + align: 1 + reloff: 0 + nreloc: 0 + flags: 2147484936 (0x80000508) + reserved1: 0 + reserved2: 6 + - segname: "__TEXT" + sectname: "__stub_helper" + addr: 8004 (0x1f44) + size: 32 (0x20) + offset: 3908 + align: 2 + reloff: 0 + nreloc: 0 + flags: 2147484928 (0x80000500) + reserved1: 0 + reserved2: 0 + - segname: "__TEXT" + sectname: "__cstring" + addr: 8036 (0x1f64) + size: 69 (0x45) + offset: 3940 + align: 0 + reloff: 0 + nreloc: 0 + flags: 2 (0x2) + reserved1: 0 + reserved2: 0 + - segname: "__TEXT" + sectname: "__unwind_info" + addr: 8108 (0x1fac) + size: 72 (0x48) + offset: 4012 + align: 2 + reloff: 0 + nreloc: 0 + flags: 0 (0x0) + reserved1: 0 + reserved2: 0 + - cmd: 1 + cmdsize: 192 + segname: "__DATA" + vmaddr: 8192 (0x2000) + vmsize: 4096 (0x1000) + fileoff: 4096 + filesize: 4096 + maxprot: 7 (0x7) + initprot: 3 (0x3) + nsects: 2 + flags: 0 (0x0) + # Nested sections structure + sections: + - segname: "__DATA" + sectname: "__nl_symbol_ptr" + addr: 8192 (0x2000) + size: 8 (0x8) + offset: 4096 + align: 2 + reloff: 0 + nreloc: 0 + flags: 6 (0x6) + reserved1: 2 + reserved2: 0 + - segname: "__DATA" + sectname: "__la_symbol_ptr" + addr: 8200 (0x2008) + size: 8 (0x8) + offset: 4104 + align: 2 + reloff: 0 + nreloc: 0 + flags: 7 (0x7) + reserved1: 4 + reserved2: 0 + - cmd: 1 + cmdsize: 56 + segname: "__LINKEDIT" + vmaddr: 12288 (0x3000) + vmsize: 4096 (0x1000) + fileoff: 8192 + filesize: 280 + maxprot: 7 (0x7) + initprot: 1 (0x1) + nsects: 0 + flags: 0 (0x0) +# Nested dylibs structure +dylibs: + - name: "/usr/lib/libSystem.B.dylib" + timestamp: 2 (1970-01-01 00:00:02 UTC) + compatibility_version: "1.0.0" + current_version: "1213.0.0" +entry_point: 3728 +stack_size: 0 + +<<< \ No newline at end of file diff --git a/yara-x-dump/src/tests/testdata/macho_x86_file.macho_lnk.human-readable.out b/yara-x-dump/src/tests/testdata/macho_x86_file.macho_lnk.human-readable.out new file mode 100644 index 000000000..aee684dd0 --- /dev/null +++ b/yara-x-dump/src/tests/testdata/macho_x86_file.macho_lnk.human-readable.out @@ -0,0 +1,151 @@ +>>> +lnk: +is_lnk: false + +<<<>>> +macho: +magic: 4277009102 (0xfeedface) +cputype: 7 +cpusubtype: 3 +filetype: 2 +ncmds: 16 +sizeofcmds: 1060 +flags: 18874501 (0x1200085) +number_of_segments: 4 +# Nested segments structure +segments: + - cmd: 1 + cmdsize: 56 + segname: "__PAGEZERO" + vmaddr: 0 (0x0) + vmsize: 4096 (0x1000) + fileoff: 0 + filesize: 0 + maxprot: 0 (0x0) + initprot: 0 (0x0) + nsects: 0 + flags: 0 (0x0) + - cmd: 1 + cmdsize: 396 + segname: "__TEXT" + vmaddr: 4096 (0x1000) + vmsize: 4096 (0x1000) + fileoff: 0 + filesize: 4096 + maxprot: 7 (0x7) + initprot: 5 (0x5) + nsects: 5 + flags: 0 (0x0) + # Nested sections structure + sections: + - segname: "__TEXT" + sectname: "__text" + addr: 7824 (0x1e90) + size: 166 (0xa6) + offset: 3728 + align: 4 + reloff: 0 + nreloc: 0 + flags: 2147484672 (0x80000400) + reserved1: 0 + reserved2: 0 + - segname: "__TEXT" + sectname: "__symbol_stub" + addr: 7990 (0x1f36) + size: 12 (0xc) + offset: 3894 + align: 1 + reloff: 0 + nreloc: 0 + flags: 2147484936 (0x80000508) + reserved1: 0 + reserved2: 6 + - segname: "__TEXT" + sectname: "__stub_helper" + addr: 8004 (0x1f44) + size: 32 (0x20) + offset: 3908 + align: 2 + reloff: 0 + nreloc: 0 + flags: 2147484928 (0x80000500) + reserved1: 0 + reserved2: 0 + - segname: "__TEXT" + sectname: "__cstring" + addr: 8036 (0x1f64) + size: 69 (0x45) + offset: 3940 + align: 0 + reloff: 0 + nreloc: 0 + flags: 2 (0x2) + reserved1: 0 + reserved2: 0 + - segname: "__TEXT" + sectname: "__unwind_info" + addr: 8108 (0x1fac) + size: 72 (0x48) + offset: 4012 + align: 2 + reloff: 0 + nreloc: 0 + flags: 0 (0x0) + reserved1: 0 + reserved2: 0 + - cmd: 1 + cmdsize: 192 + segname: "__DATA" + vmaddr: 8192 (0x2000) + vmsize: 4096 (0x1000) + fileoff: 4096 + filesize: 4096 + maxprot: 7 (0x7) + initprot: 3 (0x3) + nsects: 2 + flags: 0 (0x0) + # Nested sections structure + sections: + - segname: "__DATA" + sectname: "__nl_symbol_ptr" + addr: 8192 (0x2000) + size: 8 (0x8) + offset: 4096 + align: 2 + reloff: 0 + nreloc: 0 + flags: 6 (0x6) + reserved1: 2 + reserved2: 0 + - segname: "__DATA" + sectname: "__la_symbol_ptr" + addr: 8200 (0x2008) + size: 8 (0x8) + offset: 4104 + align: 2 + reloff: 0 + nreloc: 0 + flags: 7 (0x7) + reserved1: 4 + reserved2: 0 + - cmd: 1 + cmdsize: 56 + segname: "__LINKEDIT" + vmaddr: 12288 (0x3000) + vmsize: 4096 (0x1000) + fileoff: 8192 + filesize: 280 + maxprot: 7 (0x7) + initprot: 1 (0x1) + nsects: 0 + flags: 0 (0x0) +# Nested dylibs structure +dylibs: + - name: "/usr/lib/libSystem.B.dylib" + timestamp: 2 (1970-01-01 00:00:02 UTC) + compatibility_version: "1.0.0" + current_version: "1213.0.0" +entry_point: 3728 +stack_size: 0 + +<<< \ No newline at end of file diff --git a/yara-x-dump/src/tests/testdata/macho_x86_file.macho_lnk.json.out b/yara-x-dump/src/tests/testdata/macho_x86_file.macho_lnk.json.out new file mode 100644 index 000000000..0b75d1830 --- /dev/null +++ b/yara-x-dump/src/tests/testdata/macho_x86_file.macho_lnk.json.out @@ -0,0 +1,177 @@ +>>> +lnk: +{ + "isLnk": false +} +<<<>>> +macho: +{ + "magic": 4277009102, + "cputype": 7, + "cpusubtype": 3, + "filetype": 2, + "ncmds": 16, + "sizeofcmds": 1060, + "flags": 18874501, + "numberOfSegments": "4", + "segments": [ + { + "cmd": 1, + "cmdsize": 56, + "segname": "__PAGEZERO", + "vmaddr": "0", + "vmsize": "4096", + "fileoff": "0", + "filesize": "0", + "maxprot": 0, + "initprot": 0, + "nsects": 0, + "flags": 0 + }, + { + "cmd": 1, + "cmdsize": 396, + "segname": "__TEXT", + "vmaddr": "4096", + "vmsize": "4096", + "fileoff": "0", + "filesize": "4096", + "maxprot": 7, + "initprot": 5, + "nsects": 5, + "flags": 0, + "sections": [ + { + "segname": "__TEXT", + "sectname": "__text", + "addr": "7824", + "size": "166", + "offset": 3728, + "align": 4, + "reloff": 0, + "nreloc": 0, + "flags": 2147484672, + "reserved1": 0, + "reserved2": 0 + }, + { + "segname": "__TEXT", + "sectname": "__symbol_stub", + "addr": "7990", + "size": "12", + "offset": 3894, + "align": 1, + "reloff": 0, + "nreloc": 0, + "flags": 2147484936, + "reserved1": 0, + "reserved2": 6 + }, + { + "segname": "__TEXT", + "sectname": "__stub_helper", + "addr": "8004", + "size": "32", + "offset": 3908, + "align": 2, + "reloff": 0, + "nreloc": 0, + "flags": 2147484928, + "reserved1": 0, + "reserved2": 0 + }, + { + "segname": "__TEXT", + "sectname": "__cstring", + "addr": "8036", + "size": "69", + "offset": 3940, + "align": 0, + "reloff": 0, + "nreloc": 0, + "flags": 2, + "reserved1": 0, + "reserved2": 0 + }, + { + "segname": "__TEXT", + "sectname": "__unwind_info", + "addr": "8108", + "size": "72", + "offset": 4012, + "align": 2, + "reloff": 0, + "nreloc": 0, + "flags": 0, + "reserved1": 0, + "reserved2": 0 + } + ] + }, + { + "cmd": 1, + "cmdsize": 192, + "segname": "__DATA", + "vmaddr": "8192", + "vmsize": "4096", + "fileoff": "4096", + "filesize": "4096", + "maxprot": 7, + "initprot": 3, + "nsects": 2, + "flags": 0, + "sections": [ + { + "segname": "__DATA", + "sectname": "__nl_symbol_ptr", + "addr": "8192", + "size": "8", + "offset": 4096, + "align": 2, + "reloff": 0, + "nreloc": 0, + "flags": 6, + "reserved1": 2, + "reserved2": 0 + }, + { + "segname": "__DATA", + "sectname": "__la_symbol_ptr", + "addr": "8200", + "size": "8", + "offset": 4104, + "align": 2, + "reloff": 0, + "nreloc": 0, + "flags": 7, + "reserved1": 4, + "reserved2": 0 + } + ] + }, + { + "cmd": 1, + "cmdsize": 56, + "segname": "__LINKEDIT", + "vmaddr": "12288", + "vmsize": "4096", + "fileoff": "8192", + "filesize": "280", + "maxprot": 7, + "initprot": 1, + "nsects": 0, + "flags": 0 + } + ], + "dylibs": [ + { + "name": "/usr/lib/libSystem.B.dylib", + "timestamp": 2, + "compatibilityVersion": "1.0.0", + "currentVersion": "1213.0.0" + } + ], + "entryPoint": "3728", + "stackSize": "0" +} +<<< \ No newline at end of file diff --git a/yara-x-dump/src/tests/testdata/macho_x86_file.macho_lnk.toml.out b/yara-x-dump/src/tests/testdata/macho_x86_file.macho_lnk.toml.out new file mode 100644 index 000000000..70af5ca36 --- /dev/null +++ b/yara-x-dump/src/tests/testdata/macho_x86_file.macho_lnk.toml.out @@ -0,0 +1,167 @@ +>>> +lnk: +isLnk = false + +<<<>>> +macho: +cpusubtype = 3 +cputype = 7 +entryPoint = "3728" +filetype = 2 +flags = 18874501 +magic = 4277009102 +ncmds = 16 +numberOfSegments = "4" +sizeofcmds = 1060 +stackSize = "0" + +[[dylibs]] +compatibilityVersion = "1.0.0" +currentVersion = "1213.0.0" +name = "/usr/lib/libSystem.B.dylib" +timestamp = 2 + +[[segments]] +cmd = 1 +cmdsize = 56 +fileoff = "0" +filesize = "0" +flags = 0 +initprot = 0 +maxprot = 0 +nsects = 0 +segname = "__PAGEZERO" +vmaddr = "0" +vmsize = "4096" + +[[segments]] +cmd = 1 +cmdsize = 396 +fileoff = "0" +filesize = "4096" +flags = 0 +initprot = 5 +maxprot = 7 +nsects = 5 +segname = "__TEXT" +vmaddr = "4096" +vmsize = "4096" + +[[segments.sections]] +addr = "7824" +align = 4 +flags = 2147484672 +nreloc = 0 +offset = 3728 +reloff = 0 +reserved1 = 0 +reserved2 = 0 +sectname = "__text" +segname = "__TEXT" +size = "166" + +[[segments.sections]] +addr = "7990" +align = 1 +flags = 2147484936 +nreloc = 0 +offset = 3894 +reloff = 0 +reserved1 = 0 +reserved2 = 6 +sectname = "__symbol_stub" +segname = "__TEXT" +size = "12" + +[[segments.sections]] +addr = "8004" +align = 2 +flags = 2147484928 +nreloc = 0 +offset = 3908 +reloff = 0 +reserved1 = 0 +reserved2 = 0 +sectname = "__stub_helper" +segname = "__TEXT" +size = "32" + +[[segments.sections]] +addr = "8036" +align = 0 +flags = 2 +nreloc = 0 +offset = 3940 +reloff = 0 +reserved1 = 0 +reserved2 = 0 +sectname = "__cstring" +segname = "__TEXT" +size = "69" + +[[segments.sections]] +addr = "8108" +align = 2 +flags = 0 +nreloc = 0 +offset = 4012 +reloff = 0 +reserved1 = 0 +reserved2 = 0 +sectname = "__unwind_info" +segname = "__TEXT" +size = "72" + +[[segments]] +cmd = 1 +cmdsize = 192 +fileoff = "4096" +filesize = "4096" +flags = 0 +initprot = 3 +maxprot = 7 +nsects = 2 +segname = "__DATA" +vmaddr = "8192" +vmsize = "4096" + +[[segments.sections]] +addr = "8192" +align = 2 +flags = 6 +nreloc = 0 +offset = 4096 +reloff = 0 +reserved1 = 2 +reserved2 = 0 +sectname = "__nl_symbol_ptr" +segname = "__DATA" +size = "8" + +[[segments.sections]] +addr = "8200" +align = 2 +flags = 7 +nreloc = 0 +offset = 4104 +reloff = 0 +reserved1 = 4 +reserved2 = 0 +sectname = "__la_symbol_ptr" +segname = "__DATA" +size = "8" + +[[segments]] +cmd = 1 +cmdsize = 56 +fileoff = "8192" +filesize = "280" +flags = 0 +initprot = 1 +maxprot = 7 +nsects = 0 +segname = "__LINKEDIT" +vmaddr = "12288" +vmsize = "4096" + +<<< \ No newline at end of file diff --git a/yara-x-dump/src/tests/testdata/macho_x86_file.macho_lnk.xml.out b/yara-x-dump/src/tests/testdata/macho_x86_file.macho_lnk.xml.out new file mode 100644 index 000000000..91c635bd6 --- /dev/null +++ b/yara-x-dump/src/tests/testdata/macho_x86_file.macho_lnk.xml.out @@ -0,0 +1,171 @@ +>>> +lnk: + + + false + +<<<>>> +macho: + + + 4277009102 + 7 + 3 + 2 + 16 + 1060 + 18874501 + 4 + + 1 + 56 + __PAGEZERO + 0 + 4096 + 0 + 0 + 0 + 0 + 0 + 0 + + + 1 + 396 + __TEXT + 4096 + 4096 + 0 + 4096 + 7 + 5 + 5 + 0 + + __TEXT + __text + 7824 + 166 + 3728 + 4 + 0 + 0 + 2147484672 + 0 + 0 + + + __TEXT + __symbol_stub + 7990 + 12 + 3894 + 1 + 0 + 0 + 2147484936 + 0 + 6 + + + __TEXT + __stub_helper + 8004 + 32 + 3908 + 2 + 0 + 0 + 2147484928 + 0 + 0 + + + __TEXT + __cstring + 8036 + 69 + 3940 + 0 + 0 + 0 + 2 + 0 + 0 + + + __TEXT + __unwind_info + 8108 + 72 + 4012 + 2 + 0 + 0 + 0 + 0 + 0 + + + + 1 + 192 + __DATA + 8192 + 4096 + 4096 + 4096 + 7 + 3 + 2 + 0 + + __DATA + __nl_symbol_ptr + 8192 + 8 + 4096 + 2 + 0 + 0 + 6 + 2 + 0 + + + __DATA + __la_symbol_ptr + 8200 + 8 + 4104 + 2 + 0 + 0 + 7 + 4 + 0 + + + + 1 + 56 + __LINKEDIT + 12288 + 4096 + 8192 + 280 + 7 + 1 + 0 + 0 + + + /usr/lib/libSystem.B.dylib + 2 + 1.0.0 + 1213.0.0 + + 3728 + 0 + +<<< \ No newline at end of file diff --git a/yara-x-dump/src/tests/testdata/macho_x86_file.macho_lnk.yaml.out b/yara-x-dump/src/tests/testdata/macho_x86_file.macho_lnk.yaml.out new file mode 100644 index 000000000..145753fb4 --- /dev/null +++ b/yara-x-dump/src/tests/testdata/macho_x86_file.macho_lnk.yaml.out @@ -0,0 +1,149 @@ +>>> +lnk: +--- +isLnk: false + +<<<>>> +macho: +--- +magic: 4277009102 +cputype: 7 +cpusubtype: 3 +filetype: 2 +ncmds: 16 +sizeofcmds: 1060 +flags: 18874501 +numberOfSegments: "4" +segments: + - cmd: 1 + cmdsize: 56 + segname: __PAGEZERO + vmaddr: "0" + vmsize: "4096" + fileoff: "0" + filesize: "0" + maxprot: 0 + initprot: 0 + nsects: 0 + flags: 0 + - cmd: 1 + cmdsize: 396 + segname: __TEXT + vmaddr: "4096" + vmsize: "4096" + fileoff: "0" + filesize: "4096" + maxprot: 7 + initprot: 5 + nsects: 5 + flags: 0 + sections: + - segname: __TEXT + sectname: __text + addr: "7824" + size: "166" + offset: 3728 + align: 4 + reloff: 0 + nreloc: 0 + flags: 2147484672 + reserved1: 0 + reserved2: 0 + - segname: __TEXT + sectname: __symbol_stub + addr: "7990" + size: "12" + offset: 3894 + align: 1 + reloff: 0 + nreloc: 0 + flags: 2147484936 + reserved1: 0 + reserved2: 6 + - segname: __TEXT + sectname: __stub_helper + addr: "8004" + size: "32" + offset: 3908 + align: 2 + reloff: 0 + nreloc: 0 + flags: 2147484928 + reserved1: 0 + reserved2: 0 + - segname: __TEXT + sectname: __cstring + addr: "8036" + size: "69" + offset: 3940 + align: 0 + reloff: 0 + nreloc: 0 + flags: 2 + reserved1: 0 + reserved2: 0 + - segname: __TEXT + sectname: __unwind_info + addr: "8108" + size: "72" + offset: 4012 + align: 2 + reloff: 0 + nreloc: 0 + flags: 0 + reserved1: 0 + reserved2: 0 + - cmd: 1 + cmdsize: 192 + segname: __DATA + vmaddr: "8192" + vmsize: "4096" + fileoff: "4096" + filesize: "4096" + maxprot: 7 + initprot: 3 + nsects: 2 + flags: 0 + sections: + - segname: __DATA + sectname: __nl_symbol_ptr + addr: "8192" + size: "8" + offset: 4096 + align: 2 + reloff: 0 + nreloc: 0 + flags: 6 + reserved1: 2 + reserved2: 0 + - segname: __DATA + sectname: __la_symbol_ptr + addr: "8200" + size: "8" + offset: 4104 + align: 2 + reloff: 0 + nreloc: 0 + flags: 7 + reserved1: 4 + reserved2: 0 + - cmd: 1 + cmdsize: 56 + segname: __LINKEDIT + vmaddr: "12288" + vmsize: "4096" + fileoff: "8192" + filesize: "280" + maxprot: 7 + initprot: 1 + nsects: 0 + flags: 0 +dylibs: + - name: /usr/lib/libSystem.B.dylib + timestamp: 2 + compatibilityVersion: 1.0.0 + currentVersion: 1213.0.0 +entryPoint: "3728" +stackSize: "0" + +<<< \ No newline at end of file From ec8ca96080ae3414a1ed7547ed9249ed72fcb6a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20=C4=8Euri=C5=A1?= Date: Tue, 7 Nov 2023 09:30:26 +0100 Subject: [PATCH 15/26] add module documentation --- docs/Module Developer's Guide.md | 44 ++++++++++++++++++++++++++++---- 1 file changed, 39 insertions(+), 5 deletions(-) diff --git a/docs/Module Developer's Guide.md b/docs/Module Developer's Guide.md index 33af9ae42..f6bb262aa 100644 --- a/docs/Module Developer's Guide.md +++ b/docs/Module Developer's Guide.md @@ -49,6 +49,7 @@ option (YARA.module_options) = { name : "text" root_message: "text.Text" rust_module: "text" + validity_flag: "num_lines" (optional) }; message Text { @@ -105,6 +106,7 @@ option (yara.module_options) = { name : "text" root_message: "text.Text" rust_module: "text" + validity_flag: "num_lines" (optional) }; ``` @@ -115,16 +117,28 @@ file, but one describing a module. In fact, you can put any `.proto` file in the files is describing a YARA module. Only files containing a `yara.module_options` section will define a module. -Options `name` and `root_message` are required, while `rust_module` is optional. -The `name` option defines the module's name. This is the name that will be used -for importing the module in a YARA rule, in this case our module will be imported +Options `name` and `root_message` are required, while `rust_module` and `validity_flag` +are optional. The `name` option defines the module's name. This is the name that will be +used for importing the module in a YARA rule, in this case our module will be imported with `import "text"`. The `root_message` option indicates which is the module's root structure, it must contain the name of some structure (a.k.a message) defined in the `.proto` file. In our case the value for `root_message` is `"text.Text"` because we have defined our module's structure in a message named `Text`, which is under package `text`. In general the value in this field will have the form `package.Message`, except if the `package` statement is missing, in which case -it would be the name of the message alone (i.e: `Text`). +it would be the name of the message alone (i.e: `Text`). The `validity_flag` field +is used by `yr dump` module in order to mark specific field as the one that +determines if the module is valid or not. If this field has a value after parsing and +this value is not `false` then the module is considered valid and module output will +be shown. Module without this flag will be simply skipped in `yr dump` output. +For example `lnk` module has a field `is_lnk` which is marked as validity flag by setting +`validity_flag: "is_lnk"`. Important thing is that this field has to be in root message +structure. If `is_lnk` is `false` then module output is considered invalid and will be +skipped in `yr dump` output. Every other value of `is_lnk` will be considered valid +and module output will be shown. This is due to some modules not having any kind of +`is_valid` field and we need to mark other field as validity flag. `Macho` module has a +field `magic` which is considered as validity flag field. If this field is set to some +value then module output will be shown as we can consider module parsing to be successful. The `root_message` field is required because your `.proto` file can define multiple messages, and YARA needs to know which of them is considered the root @@ -144,7 +158,27 @@ This is a very simple structure with only two integer fields. Notice that the numbers after the `=` signs are not the values for those fields, they are actually field tags (i.e: a unique number identifying each field in a message). This may be confusing if you are not familiar with protobuf's syntax, so again: -explore the protobuf's [documentation](https://developers.google.com/protocol-buffers). +explore the protobuf's [documentation](https://developers.google.com/protocol-buffers). + +One thing that can be done with integer fields is to represent them in some other way. +This optional representation is shown in `yr dump` crate output. This crate provides +multiple output formats as JSON, XML, YAML, TOML and human-readable. The last mentioned +is the only one that provides also custom representation for integer numbers. Let's say +for some fields it makes sense to show them as hexadecimal numbers. This can be done by +adding `[(yara.field_options).hex_value = true];` descriptor to the field. For example: + +``` +message Macho { + optional uint32 magic = 1 [(yara.field_options).hex_value = true]; +} +``` + +This will mark magic field as a hexadecimal number and it will be shown as +`magic: 4277009103 (0xfeedfacf)`. Other format that is supported right now is +for timestamps. If you want to show some integer field as a timestamp you can do it by +setting `[(yara.field_options).timestamp = true];` descriptor to the field and +timestamps will be shown in human-readable format. + Also notice that we are defining our fields as `optional`. In `proto2` fields must be either `optional` or `required`, while in `proto3` they are always From e017dbdde8b20582140e91a6f55881e6cb6721df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20=C4=8Euri=C5=A1?= Date: Tue, 7 Nov 2023 09:31:12 +0100 Subject: [PATCH 16/26] remove unnecessary optional marking --- docs/Module Developer's Guide.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Module Developer's Guide.md b/docs/Module Developer's Guide.md index f6bb262aa..5c47e1849 100644 --- a/docs/Module Developer's Guide.md +++ b/docs/Module Developer's Guide.md @@ -106,7 +106,7 @@ option (yara.module_options) = { name : "text" root_message: "text.Text" rust_module: "text" - validity_flag: "num_lines" (optional) + validity_flag: "num_lines" }; ``` From 4e15ca51a4de21d37a9bb54f061bc1f38ecd350c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20=C4=8Euri=C5=A1?= Date: Tue, 7 Nov 2023 10:17:34 +0100 Subject: [PATCH 17/26] clippy warnings fixed --- yara-x/src/modules/macho/mod.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/yara-x/src/modules/macho/mod.rs b/yara-x/src/modules/macho/mod.rs index 42bcd2417..00e869aa1 100644 --- a/yara-x/src/modules/macho/mod.rs +++ b/yara-x/src/modules/macho/mod.rs @@ -1640,8 +1640,7 @@ fn handle_segment_command( segname: Some( std::str::from_utf8(&sg.segname) .unwrap_or_default() - .replace('\0', "") - .to_string(), + .replace('\0', ""), ), vmaddr: Some(sg.vmaddr as u64), vmsize: Some(sg.vmsize as u64), @@ -1760,8 +1759,7 @@ fn handle_segment_command_64( segname: Some( std::str::from_utf8(&sg.segname) .unwrap_or_default() - .replace('\0', "") - .to_string(), + .replace('\0', ""), ), vmaddr: Some(sg.vmaddr), vmsize: Some(sg.vmsize), From 624c525ed1f383b8e650b200e163a7ae0d514692 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20=C4=8Euri=C5=A1?= Date: Tue, 7 Nov 2023 10:19:17 +0100 Subject: [PATCH 18/26] remove unused dependancies --- Cargo.lock | 2 -- yara-x-dump/Cargo.toml | 2 -- 2 files changed, 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 42b6031ef..1097902da 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4454,8 +4454,6 @@ dependencies = [ "globwalk", "goldenfile", "ihex", - "indent", - "pretty_assertions", "protobuf", "protobuf-json-mapping", "protobuf-support", diff --git a/yara-x-dump/Cargo.toml b/yara-x-dump/Cargo.toml index 2f1e40542..05d6f99b6 100644 --- a/yara-x-dump/Cargo.toml +++ b/yara-x-dump/Cargo.toml @@ -20,7 +20,6 @@ yara-x-proto = { workspace = true } protobuf = { workspace = true } yansi = { workspace = true } -indent = "0.1.1" serde_json = { version = "1.0.1", features = ["preserve_order"] } serde_yaml = "0.8.26" toml = "0.8.4" @@ -30,7 +29,6 @@ xml2json-rs = "1.0.1" chrono = "0.4.31" [dev-dependencies] -pretty_assertions = { workspace = true } goldenfile = "1.5.2" ihex = "3.0.0" globwalk = { workspace = true } From 3cdf7ac2b216605b311321844e94babe715ff88c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20=C4=8Euri=C5=A1?= Date: Wed, 8 Nov 2023 14:00:37 +0100 Subject: [PATCH 19/26] yr dump decoupling --- Cargo.lock | 2 +- yara-x-cli/Cargo.toml | 1 + yara-x-cli/src/commands/dump.rs | 102 ++++- yara-x-dump/Cargo.toml | 1 - yara-x-dump/src/lib.rs | 160 ++----- yara-x-dump/src/serializer.rs | 4 +- yara-x-dump/src/tests/mod.rs | 78 +--- .../macho_x86_file.automatic.None.out | 147 ------ ...acho_x86_file.automatic.human-readable.out | 147 ------ .../macho_x86_file.automatic.json.out | 172 ------- .../macho_x86_file.automatic.toml.out | 163 ------- .../testdata/macho_x86_file.automatic.xml.out | 165 ------- .../macho_x86_file.automatic.yaml.out | 144 ------ .../src/tests/testdata/macho_x86_file.in | 418 +++++++++++++++--- .../testdata/macho_x86_file.lnk.None.out | 5 - .../macho_x86_file.lnk.human-readable.out | 5 - .../testdata/macho_x86_file.lnk.json.out | 6 - .../testdata/macho_x86_file.lnk.toml.out | 5 - .../tests/testdata/macho_x86_file.lnk.xml.out | 7 - .../testdata/macho_x86_file.lnk.yaml.out | 6 - .../testdata/macho_x86_file.macho.None.out | 147 ------ .../macho_x86_file.macho.human-readable.out | 147 ------ .../testdata/macho_x86_file.macho.json.out | 172 ------- .../testdata/macho_x86_file.macho.toml.out | 163 ------- .../testdata/macho_x86_file.macho.xml.out | 165 ------- .../testdata/macho_x86_file.macho.yaml.out | 144 ------ .../macho_x86_file.macho_lnk.None.out | 151 ------- ...acho_x86_file.macho_lnk.human-readable.out | 151 ------- .../macho_x86_file.macho_lnk.json.out | 177 -------- .../macho_x86_file.macho_lnk.toml.out | 167 ------- .../testdata/macho_x86_file.macho_lnk.xml.out | 171 ------- .../macho_x86_file.macho_lnk.yaml.out | 149 ------- yara-x-dump/src/tests/testdata/test.proto | 7 + 33 files changed, 497 insertions(+), 3152 deletions(-) delete mode 100644 yara-x-dump/src/tests/testdata/macho_x86_file.automatic.None.out delete mode 100644 yara-x-dump/src/tests/testdata/macho_x86_file.automatic.human-readable.out delete mode 100644 yara-x-dump/src/tests/testdata/macho_x86_file.automatic.json.out delete mode 100644 yara-x-dump/src/tests/testdata/macho_x86_file.automatic.toml.out delete mode 100644 yara-x-dump/src/tests/testdata/macho_x86_file.automatic.xml.out delete mode 100644 yara-x-dump/src/tests/testdata/macho_x86_file.automatic.yaml.out delete mode 100644 yara-x-dump/src/tests/testdata/macho_x86_file.lnk.None.out delete mode 100644 yara-x-dump/src/tests/testdata/macho_x86_file.lnk.human-readable.out delete mode 100644 yara-x-dump/src/tests/testdata/macho_x86_file.lnk.json.out delete mode 100644 yara-x-dump/src/tests/testdata/macho_x86_file.lnk.toml.out delete mode 100644 yara-x-dump/src/tests/testdata/macho_x86_file.lnk.xml.out delete mode 100644 yara-x-dump/src/tests/testdata/macho_x86_file.lnk.yaml.out delete mode 100644 yara-x-dump/src/tests/testdata/macho_x86_file.macho.None.out delete mode 100644 yara-x-dump/src/tests/testdata/macho_x86_file.macho.human-readable.out delete mode 100644 yara-x-dump/src/tests/testdata/macho_x86_file.macho.json.out delete mode 100644 yara-x-dump/src/tests/testdata/macho_x86_file.macho.toml.out delete mode 100644 yara-x-dump/src/tests/testdata/macho_x86_file.macho.xml.out delete mode 100644 yara-x-dump/src/tests/testdata/macho_x86_file.macho.yaml.out delete mode 100644 yara-x-dump/src/tests/testdata/macho_x86_file.macho_lnk.None.out delete mode 100644 yara-x-dump/src/tests/testdata/macho_x86_file.macho_lnk.human-readable.out delete mode 100644 yara-x-dump/src/tests/testdata/macho_x86_file.macho_lnk.json.out delete mode 100644 yara-x-dump/src/tests/testdata/macho_x86_file.macho_lnk.toml.out delete mode 100644 yara-x-dump/src/tests/testdata/macho_x86_file.macho_lnk.xml.out delete mode 100644 yara-x-dump/src/tests/testdata/macho_x86_file.macho_lnk.yaml.out create mode 100644 yara-x-dump/src/tests/testdata/test.proto diff --git a/Cargo.lock b/Cargo.lock index 1097902da..e1ff02cb0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4443,6 +4443,7 @@ dependencies = [ "yara-x-dump", "yara-x-fmt", "yara-x-parser", + "yara-x-proto", ] [[package]] @@ -4465,7 +4466,6 @@ dependencies = [ "toml 0.8.4", "xml2json-rs", "yansi", - "yara-x", "yara-x-proto", ] diff --git a/yara-x-cli/Cargo.toml b/yara-x-cli/Cargo.toml index 48383413b..e2d1920a8 100644 --- a/yara-x-cli/Cargo.toml +++ b/yara-x-cli/Cargo.toml @@ -48,6 +48,7 @@ yara-x = { workspace = true } yara-x-dump = { workspace = true } yara-x-parser = { workspace = true, features = ["ascii-tree"] } yara-x-fmt = { workspace = true } +yara-x-proto = { workspace = true } crossbeam = "0.8.2" crossterm = "0.27.0" diff --git a/yara-x-cli/src/commands/dump.rs b/yara-x-cli/src/commands/dump.rs index f1a6e2cbd..c69e1f049 100644 --- a/yara-x-cli/src/commands/dump.rs +++ b/yara-x-cli/src/commands/dump.rs @@ -1,12 +1,22 @@ use clap::{arg, value_parser, Arg, ArgAction, ArgMatches, Command}; +use protobuf::{reflect::ReflectValueRef::Bool, MessageDyn}; +use std::fmt::Write; use std::fs::File; use std::io::stdin; use std::path::PathBuf; +use yansi::Color::Cyan; +use yara_x_proto::exts::module_options; use yara_x_dump::Dumper; use yara_x::get_builtin_modules_names; +/// Creates the `dump` command. +/// The `dump` command dumps information about binary files. +/// +/// # Returns +/// +/// Returns a `Command` struct that represents the `dump` command. pub fn dump() -> Command { super::command("dump") .about("Dump information about binary files") @@ -33,7 +43,53 @@ pub fn dump() -> Command { ) } +// Checks if the module output is valid by checking the validity flag. +// +// # Arguments +// +// * `mod_output`: The module output to check. +// +// # Returns +// +// * `true` if the module output is valid, `false` otherwise. +fn module_is_valid(mod_output: &dyn MessageDyn) -> bool { + // Get the module options. + if let Some(module_desc) = module_options + .get(&mod_output.descriptor_dyn().file_descriptor_proto().options) + { + // Get the field name which is considered as the validity flag. + if let Some(validity_flag_str) = module_desc.validity_flag.as_deref() { + // Get the validity flag value. + if let Some(field) = + mod_output.descriptor_dyn().field_by_name(validity_flag_str) + { + // Check if the validity flag is set. + // Validity flag is set if the value present and is not + // false. + if let Some(value) = field.get_singular(mod_output) { + return value != Bool(false); + } + } + } + } + + false +} + +/// Executes the `dump` command. +/// +/// # Arguments +/// +/// * `args`: The arguments passed to the `dump` command. +/// +/// # Returns +/// +/// Returns a `Result<(), anyhow::Error>` indicating whether the operation was +/// successful or not. pub fn exec_dump(args: &ArgMatches) -> anyhow::Result<()> { + let mut buffer = Vec::new(); + let mut result = String::new(); + let file = args.get_one::("FILE"); let output_format = args.get_one::("output-format"); @@ -46,12 +102,54 @@ pub fn exec_dump(args: &ArgMatches) -> anyhow::Result<()> { let dumper = Dumper::default(); - let input: Box = if let Some(file) = file { + // Get the input. + let mut input: Box = if let Some(file) = file { Box::new(File::open(file.as_path())?) } else { Box::new(stdin()) }; - println!("{}", dumper.dump(input, modules, output_format)?); + input.read_to_end(&mut buffer)?; + + // Get the list of modules to import. + let import_modules = if !modules.is_empty() { + modules.clone() + } else { + yara_x::get_builtin_modules_names() + }; + + // Create a rule that imports all the built-in modules. + let import_statements = import_modules + .iter() + .map(|module_name| format!("import \"{}\"", module_name)) + .collect::>() + .join("\n"); + + // Create a dummy rule + let rule = + format!(r#"{} rule test {{ condition: false }}"#, import_statements); + + // Compile the rule. + let rules = yara_x::compile(rule.as_str()).unwrap(); + + let mut scanner = yara_x::Scanner::new(&rules); + + // Scan the buffer and get the results. + let scan_results = scanner.scan(&buffer).expect("scan should not fail"); + + for (mod_name, mod_output) in scan_results.module_outputs() { + if mod_output.compute_size_dyn() != 0 + && (module_is_valid(mod_output) || modules.contains(&mod_name)) + { + write!( + result, + ">>>\n{}:\n{}<<<\n", + Cyan.paint(mod_name).bold(), + dumper.dump(mod_output, output_format)? + )?; + } + } + + println!("{}", result); Ok(()) } diff --git a/yara-x-dump/Cargo.toml b/yara-x-dump/Cargo.toml index 05d6f99b6..2d9e7e9c6 100644 --- a/yara-x-dump/Cargo.toml +++ b/yara-x-dump/Cargo.toml @@ -15,7 +15,6 @@ crate-type = ["rlib", "cdylib"] [dependencies] anyhow = { workspace = true } thiserror = { workspace = true } -yara-x = { workspace = true } yara-x-proto = { workspace = true } protobuf = { workspace = true } yansi = { workspace = true } diff --git a/yara-x-dump/src/lib.rs b/yara-x-dump/src/lib.rs index 6e93a5224..ca5438603 100644 --- a/yara-x-dump/src/lib.rs +++ b/yara-x-dump/src/lib.rs @@ -1,13 +1,9 @@ mod serializer; -use protobuf::{ - reflect::MessageRef, reflect::ReflectValueRef::Bool, MessageDyn, -}; +use protobuf::{reflect::MessageRef, MessageDyn}; use protobuf_json_mapping::print_to_string; -use std::fmt::Write; + use std::io; use thiserror::Error; -use yansi::Color::Cyan; -use yara_x_proto::exts::module_options; use crate::serializer::{get_human_readable_output, get_serializer}; @@ -50,42 +46,6 @@ pub struct Dumper {} // Dumper public API. impl Dumper { - // Checks if the module output is valid by checking the validity flag. - // - // # Arguments - // - // * `mod_output`: The module output to check. - // - // # Returns - // - // * `true` if the module output is valid, `false` otherwise. - fn module_is_valid(&self, mod_output: &dyn MessageDyn) -> bool { - // Get the module options. - if let Some(module_desc) = module_options - .get(&mod_output.descriptor_dyn().file_descriptor_proto().options) - { - // Get the field name which is considered as the validity flag. - if let Some(validity_flag_str) = - module_desc.validity_flag.as_deref() - { - // Get the validity flag value. - if let Some(field) = mod_output - .descriptor_dyn() - .field_by_name(validity_flag_str) - { - // Check if the validity flag is set. - // Validity flag is set if the value present and is not - // false. - if let Some(value) = field.get_singular(mod_output) { - return value != Bool(false); - } - } - } - } - - false - } - /// Dumps information about the binary file. /// /// # Arguments @@ -98,98 +58,46 @@ impl Dumper { /// /// Returns a `Result<(), Error>` indicating whether the operation was /// successful or not. - pub fn dump( + pub fn dump( &self, - mut input: R, - modules: Vec<&str>, + mod_output: &dyn MessageDyn, output_format: Option<&String>, - ) -> Result - where - R: io::Read, - { - let mut buffer = Vec::new(); - let mut result = String::new(); - - input.read_to_end(&mut buffer).map_err(Error::ReadError)?; - - // Get the list of modules to import. - let import_modules = if !modules.is_empty() { - modules.clone() - } else { - yara_x::get_builtin_modules_names() - }; - - // Create a rule that imports all the built-in modules. - let import_statements = import_modules - .iter() - .map(|module_name| format!("import \"{}\"", module_name)) - .collect::>() - .join("\n"); - - // Create a dummy rule - let rule = format!( - r#"{} rule test {{ condition: false }}"#, - import_statements - ); - - // Compile the rule. - let rules = yara_x::compile(rule.as_str()).unwrap(); - - let mut scanner = yara_x::Scanner::new(&rules); - - // Scan the buffer and get the results. - let scan_results = - scanner.scan(&buffer).expect("scan should not fail"); - + ) -> Result { // Iterate over the modules' outputs and get serialized results to // print. - for (mod_name, mod_output) in scan_results.module_outputs() { - let mut serialized_result = String::new(); - let mut is_first_line = false; - // Skip empty outputs or invalid outputs that are not requested. - if mod_output.compute_size_dyn() == 0 - || (!self.module_is_valid(mod_output) - && !modules.contains(&mod_name)) - { - continue; + let mut serialized_result = String::new(); + let mut is_first_line = false; + + match output_format { + // Output is desired to be human-readable. + Some(format) if format == "human-readable" => { + get_human_readable_output( + &MessageRef::from(mod_output), + &mut serialized_result, + 0, + &mut is_first_line, + )?; } - match output_format { - // Output is desired to be human-readable. - Some(format) if format == "human-readable" => { - get_human_readable_output( - &MessageRef::from(mod_output), - &mut serialized_result, - 0, - &mut is_first_line, - )?; - } - // Serialize output for other given formats. - Some(format) => { - let json_output = print_to_string(mod_output)?; - let serializer = get_serializer(format)?; + // Serialize output for other given formats. + Some(format) => { + let json_output = print_to_string(mod_output)?; + let serializer = get_serializer(format)?; - serialized_result = - serializer.serialize(json_output.as_str())?; - } - // Default to human-readable output. - None => { - get_human_readable_output( - &MessageRef::from(mod_output), - &mut serialized_result, - 0, - &mut is_first_line, - )?; - } + serialized_result = + serializer.serialize(json_output.as_str())?; + } + // Default to human-readable output. + None => { + get_human_readable_output( + &MessageRef::from(mod_output), + &mut serialized_result, + 0, + &mut is_first_line, + )?; } - - write!( - result, - ">>>\n{}:\n{}\n<<<", - Cyan.paint(mod_name).bold(), - serialized_result - )?; } - Ok(result) + + Ok(serialized_result) } } diff --git a/yara-x-dump/src/serializer.rs b/yara-x-dump/src/serializer.rs index 2a7cdc4a6..c0359387d 100644 --- a/yara-x-dump/src/serializer.rs +++ b/yara-x-dump/src/serializer.rs @@ -59,7 +59,7 @@ pub(crate) trait Serializer { impl Serializer for JsonSerializer { fn serialize(&self, message: &str) -> Result { let value = serde_json::from_str::(message)?; - Ok(serde_json::to_string_pretty(&value)?) + Ok(serde_json::to_string_pretty(&value)? + "\n") } } @@ -92,7 +92,7 @@ impl Serializer for XmlSerializer { )) .root_name("file") .finalize(); - let xml = xml_builder.build_from_json_string(message)?; + let xml = xml_builder.build_from_json_string(message)? + "\n"; Ok(xml) } } diff --git a/yara-x-dump/src/tests/mod.rs b/yara-x-dump/src/tests/mod.rs index ed5e78fd4..b1ebc916e 100644 --- a/yara-x-dump/src/tests/mod.rs +++ b/yara-x-dump/src/tests/mod.rs @@ -1,52 +1,10 @@ +use protobuf::text_format::parse_from_str; use rstest::rstest; use std::fs; use std::io::Write; -use std::path::Path; -use tempfile::NamedTempFile; -pub fn create_binary_from_ihex>( - path: P, -) -> anyhow::Result> { - let contents = fs::read_to_string(path)?; - let mut reader = ihex::Reader::new(&contents); - let mut data = Vec::new(); - while let Some(Ok(record)) = reader.next() { - if let ihex::Record::Data { value, .. } = record { - data.extend(value); - } - } - Ok(data) -} - -#[rstest( - module, - output_format, - case(&vec!["macho"], "json"), - case(&vec!["macho"], "yaml"), - case(&vec!["macho"], "toml"), - case(&vec!["macho"], "xml"), - case(&vec!["macho"], "human-readable"), - case(&vec!["macho"], "None"), - case(&vec!["lnk"], "json"), - case(&vec!["lnk"], "yaml"), - case(&vec!["lnk"], "toml"), - case(&vec!["lnk"], "xml"), - case(&vec!["lnk"], "human-readable"), - case(&vec!["lnk"], "None"), - case(&vec!["macho", "lnk"], "json"), - case(&vec!["macho", "lnk"], "yaml"), - case(&vec!["macho", "lnk"], "toml"), - case(&vec!["macho", "lnk"], "xml"), - case(&vec!["macho", "lnk"], "human-readable"), - case(&vec!["macho", "lnk"], "None"), - case(&vec![], "json"), - case(&vec![], "yaml"), - case(&vec![], "toml"), - case(&vec![], "xml"), - case(&vec![], "human-readable"), - case(&vec![], "None"), -)] -fn test_dumper(module: &Vec<&str>, output_format: &str) { +#[rstest(output_format, case("json"))] +fn test_dumper(output_format: &str) { // Create goldenfile mint. let mut mint = goldenfile::Mint::new("."); @@ -55,40 +13,18 @@ fn test_dumper(module: &Vec<&str>, output_format: &str) { println!("{:?}", entry); let in_path = entry.into_path(); - // Change the module to "automatic" if it's an empty vector. - let module_name = if module.is_empty() { - "automatic".to_string() - } else { - module.join("_") - }; - // Create a unique test name based on the combination of module and // output_format. let test_name = format!( - "{}.{}.{}.out", + "{}.{}.out", in_path.with_extension("").to_str().unwrap(), - module_name, output_format ); - - let mut input = NamedTempFile::new().unwrap(); - let input_stream = input.reopen().unwrap(); - let bytes = create_binary_from_ihex(&in_path).unwrap_or_else(|err| { - panic!("error reading ihex file {:?}: {:?}", in_path, err) - }); - input.write_all(&bytes).unwrap(); + let input = fs::read_to_string(in_path).expect("Unable to read"); + let protobuf_module_output = parse_from_str(&input).unwrap(); let dumper = crate::Dumper::default(); - let output = if output_format == "None" { - dumper.dump(input_stream, module.clone(), None) - } else { - dumper.dump( - input_stream, - module.clone(), - Some(&output_format.to_string()), - ) - } - .unwrap(); + let output = dumper.dump(protobuf_module_output, output_format); // Create a goldenfile test let mut output_file = mint.new_goldenfile(test_name).unwrap(); diff --git a/yara-x-dump/src/tests/testdata/macho_x86_file.automatic.None.out b/yara-x-dump/src/tests/testdata/macho_x86_file.automatic.None.out deleted file mode 100644 index 446b6f89c..000000000 --- a/yara-x-dump/src/tests/testdata/macho_x86_file.automatic.None.out +++ /dev/null @@ -1,147 +0,0 @@ ->>> -macho: -magic: 4277009102 (0xfeedface) -cputype: 7 -cpusubtype: 3 -filetype: 2 -ncmds: 16 -sizeofcmds: 1060 -flags: 18874501 (0x1200085) -number_of_segments: 4 -# Nested segments structure -segments: - - cmd: 1 - cmdsize: 56 - segname: "__PAGEZERO" - vmaddr: 0 (0x0) - vmsize: 4096 (0x1000) - fileoff: 0 - filesize: 0 - maxprot: 0 (0x0) - initprot: 0 (0x0) - nsects: 0 - flags: 0 (0x0) - - cmd: 1 - cmdsize: 396 - segname: "__TEXT" - vmaddr: 4096 (0x1000) - vmsize: 4096 (0x1000) - fileoff: 0 - filesize: 4096 - maxprot: 7 (0x7) - initprot: 5 (0x5) - nsects: 5 - flags: 0 (0x0) - # Nested sections structure - sections: - - segname: "__TEXT" - sectname: "__text" - addr: 7824 (0x1e90) - size: 166 (0xa6) - offset: 3728 - align: 4 - reloff: 0 - nreloc: 0 - flags: 2147484672 (0x80000400) - reserved1: 0 - reserved2: 0 - - segname: "__TEXT" - sectname: "__symbol_stub" - addr: 7990 (0x1f36) - size: 12 (0xc) - offset: 3894 - align: 1 - reloff: 0 - nreloc: 0 - flags: 2147484936 (0x80000508) - reserved1: 0 - reserved2: 6 - - segname: "__TEXT" - sectname: "__stub_helper" - addr: 8004 (0x1f44) - size: 32 (0x20) - offset: 3908 - align: 2 - reloff: 0 - nreloc: 0 - flags: 2147484928 (0x80000500) - reserved1: 0 - reserved2: 0 - - segname: "__TEXT" - sectname: "__cstring" - addr: 8036 (0x1f64) - size: 69 (0x45) - offset: 3940 - align: 0 - reloff: 0 - nreloc: 0 - flags: 2 (0x2) - reserved1: 0 - reserved2: 0 - - segname: "__TEXT" - sectname: "__unwind_info" - addr: 8108 (0x1fac) - size: 72 (0x48) - offset: 4012 - align: 2 - reloff: 0 - nreloc: 0 - flags: 0 (0x0) - reserved1: 0 - reserved2: 0 - - cmd: 1 - cmdsize: 192 - segname: "__DATA" - vmaddr: 8192 (0x2000) - vmsize: 4096 (0x1000) - fileoff: 4096 - filesize: 4096 - maxprot: 7 (0x7) - initprot: 3 (0x3) - nsects: 2 - flags: 0 (0x0) - # Nested sections structure - sections: - - segname: "__DATA" - sectname: "__nl_symbol_ptr" - addr: 8192 (0x2000) - size: 8 (0x8) - offset: 4096 - align: 2 - reloff: 0 - nreloc: 0 - flags: 6 (0x6) - reserved1: 2 - reserved2: 0 - - segname: "__DATA" - sectname: "__la_symbol_ptr" - addr: 8200 (0x2008) - size: 8 (0x8) - offset: 4104 - align: 2 - reloff: 0 - nreloc: 0 - flags: 7 (0x7) - reserved1: 4 - reserved2: 0 - - cmd: 1 - cmdsize: 56 - segname: "__LINKEDIT" - vmaddr: 12288 (0x3000) - vmsize: 4096 (0x1000) - fileoff: 8192 - filesize: 280 - maxprot: 7 (0x7) - initprot: 1 (0x1) - nsects: 0 - flags: 0 (0x0) -# Nested dylibs structure -dylibs: - - name: "/usr/lib/libSystem.B.dylib" - timestamp: 2 (1970-01-01 00:00:02 UTC) - compatibility_version: "1.0.0" - current_version: "1213.0.0" -entry_point: 3728 -stack_size: 0 - -<<< \ No newline at end of file diff --git a/yara-x-dump/src/tests/testdata/macho_x86_file.automatic.human-readable.out b/yara-x-dump/src/tests/testdata/macho_x86_file.automatic.human-readable.out deleted file mode 100644 index 446b6f89c..000000000 --- a/yara-x-dump/src/tests/testdata/macho_x86_file.automatic.human-readable.out +++ /dev/null @@ -1,147 +0,0 @@ ->>> -macho: -magic: 4277009102 (0xfeedface) -cputype: 7 -cpusubtype: 3 -filetype: 2 -ncmds: 16 -sizeofcmds: 1060 -flags: 18874501 (0x1200085) -number_of_segments: 4 -# Nested segments structure -segments: - - cmd: 1 - cmdsize: 56 - segname: "__PAGEZERO" - vmaddr: 0 (0x0) - vmsize: 4096 (0x1000) - fileoff: 0 - filesize: 0 - maxprot: 0 (0x0) - initprot: 0 (0x0) - nsects: 0 - flags: 0 (0x0) - - cmd: 1 - cmdsize: 396 - segname: "__TEXT" - vmaddr: 4096 (0x1000) - vmsize: 4096 (0x1000) - fileoff: 0 - filesize: 4096 - maxprot: 7 (0x7) - initprot: 5 (0x5) - nsects: 5 - flags: 0 (0x0) - # Nested sections structure - sections: - - segname: "__TEXT" - sectname: "__text" - addr: 7824 (0x1e90) - size: 166 (0xa6) - offset: 3728 - align: 4 - reloff: 0 - nreloc: 0 - flags: 2147484672 (0x80000400) - reserved1: 0 - reserved2: 0 - - segname: "__TEXT" - sectname: "__symbol_stub" - addr: 7990 (0x1f36) - size: 12 (0xc) - offset: 3894 - align: 1 - reloff: 0 - nreloc: 0 - flags: 2147484936 (0x80000508) - reserved1: 0 - reserved2: 6 - - segname: "__TEXT" - sectname: "__stub_helper" - addr: 8004 (0x1f44) - size: 32 (0x20) - offset: 3908 - align: 2 - reloff: 0 - nreloc: 0 - flags: 2147484928 (0x80000500) - reserved1: 0 - reserved2: 0 - - segname: "__TEXT" - sectname: "__cstring" - addr: 8036 (0x1f64) - size: 69 (0x45) - offset: 3940 - align: 0 - reloff: 0 - nreloc: 0 - flags: 2 (0x2) - reserved1: 0 - reserved2: 0 - - segname: "__TEXT" - sectname: "__unwind_info" - addr: 8108 (0x1fac) - size: 72 (0x48) - offset: 4012 - align: 2 - reloff: 0 - nreloc: 0 - flags: 0 (0x0) - reserved1: 0 - reserved2: 0 - - cmd: 1 - cmdsize: 192 - segname: "__DATA" - vmaddr: 8192 (0x2000) - vmsize: 4096 (0x1000) - fileoff: 4096 - filesize: 4096 - maxprot: 7 (0x7) - initprot: 3 (0x3) - nsects: 2 - flags: 0 (0x0) - # Nested sections structure - sections: - - segname: "__DATA" - sectname: "__nl_symbol_ptr" - addr: 8192 (0x2000) - size: 8 (0x8) - offset: 4096 - align: 2 - reloff: 0 - nreloc: 0 - flags: 6 (0x6) - reserved1: 2 - reserved2: 0 - - segname: "__DATA" - sectname: "__la_symbol_ptr" - addr: 8200 (0x2008) - size: 8 (0x8) - offset: 4104 - align: 2 - reloff: 0 - nreloc: 0 - flags: 7 (0x7) - reserved1: 4 - reserved2: 0 - - cmd: 1 - cmdsize: 56 - segname: "__LINKEDIT" - vmaddr: 12288 (0x3000) - vmsize: 4096 (0x1000) - fileoff: 8192 - filesize: 280 - maxprot: 7 (0x7) - initprot: 1 (0x1) - nsects: 0 - flags: 0 (0x0) -# Nested dylibs structure -dylibs: - - name: "/usr/lib/libSystem.B.dylib" - timestamp: 2 (1970-01-01 00:00:02 UTC) - compatibility_version: "1.0.0" - current_version: "1213.0.0" -entry_point: 3728 -stack_size: 0 - -<<< \ No newline at end of file diff --git a/yara-x-dump/src/tests/testdata/macho_x86_file.automatic.json.out b/yara-x-dump/src/tests/testdata/macho_x86_file.automatic.json.out deleted file mode 100644 index e2eae2a1c..000000000 --- a/yara-x-dump/src/tests/testdata/macho_x86_file.automatic.json.out +++ /dev/null @@ -1,172 +0,0 @@ ->>> -macho: -{ - "magic": 4277009102, - "cputype": 7, - "cpusubtype": 3, - "filetype": 2, - "ncmds": 16, - "sizeofcmds": 1060, - "flags": 18874501, - "numberOfSegments": "4", - "segments": [ - { - "cmd": 1, - "cmdsize": 56, - "segname": "__PAGEZERO", - "vmaddr": "0", - "vmsize": "4096", - "fileoff": "0", - "filesize": "0", - "maxprot": 0, - "initprot": 0, - "nsects": 0, - "flags": 0 - }, - { - "cmd": 1, - "cmdsize": 396, - "segname": "__TEXT", - "vmaddr": "4096", - "vmsize": "4096", - "fileoff": "0", - "filesize": "4096", - "maxprot": 7, - "initprot": 5, - "nsects": 5, - "flags": 0, - "sections": [ - { - "segname": "__TEXT", - "sectname": "__text", - "addr": "7824", - "size": "166", - "offset": 3728, - "align": 4, - "reloff": 0, - "nreloc": 0, - "flags": 2147484672, - "reserved1": 0, - "reserved2": 0 - }, - { - "segname": "__TEXT", - "sectname": "__symbol_stub", - "addr": "7990", - "size": "12", - "offset": 3894, - "align": 1, - "reloff": 0, - "nreloc": 0, - "flags": 2147484936, - "reserved1": 0, - "reserved2": 6 - }, - { - "segname": "__TEXT", - "sectname": "__stub_helper", - "addr": "8004", - "size": "32", - "offset": 3908, - "align": 2, - "reloff": 0, - "nreloc": 0, - "flags": 2147484928, - "reserved1": 0, - "reserved2": 0 - }, - { - "segname": "__TEXT", - "sectname": "__cstring", - "addr": "8036", - "size": "69", - "offset": 3940, - "align": 0, - "reloff": 0, - "nreloc": 0, - "flags": 2, - "reserved1": 0, - "reserved2": 0 - }, - { - "segname": "__TEXT", - "sectname": "__unwind_info", - "addr": "8108", - "size": "72", - "offset": 4012, - "align": 2, - "reloff": 0, - "nreloc": 0, - "flags": 0, - "reserved1": 0, - "reserved2": 0 - } - ] - }, - { - "cmd": 1, - "cmdsize": 192, - "segname": "__DATA", - "vmaddr": "8192", - "vmsize": "4096", - "fileoff": "4096", - "filesize": "4096", - "maxprot": 7, - "initprot": 3, - "nsects": 2, - "flags": 0, - "sections": [ - { - "segname": "__DATA", - "sectname": "__nl_symbol_ptr", - "addr": "8192", - "size": "8", - "offset": 4096, - "align": 2, - "reloff": 0, - "nreloc": 0, - "flags": 6, - "reserved1": 2, - "reserved2": 0 - }, - { - "segname": "__DATA", - "sectname": "__la_symbol_ptr", - "addr": "8200", - "size": "8", - "offset": 4104, - "align": 2, - "reloff": 0, - "nreloc": 0, - "flags": 7, - "reserved1": 4, - "reserved2": 0 - } - ] - }, - { - "cmd": 1, - "cmdsize": 56, - "segname": "__LINKEDIT", - "vmaddr": "12288", - "vmsize": "4096", - "fileoff": "8192", - "filesize": "280", - "maxprot": 7, - "initprot": 1, - "nsects": 0, - "flags": 0 - } - ], - "dylibs": [ - { - "name": "/usr/lib/libSystem.B.dylib", - "timestamp": 2, - "compatibilityVersion": "1.0.0", - "currentVersion": "1213.0.0" - } - ], - "entryPoint": "3728", - "stackSize": "0" -} -<<< \ No newline at end of file diff --git a/yara-x-dump/src/tests/testdata/macho_x86_file.automatic.toml.out b/yara-x-dump/src/tests/testdata/macho_x86_file.automatic.toml.out deleted file mode 100644 index 1d7844c93..000000000 --- a/yara-x-dump/src/tests/testdata/macho_x86_file.automatic.toml.out +++ /dev/null @@ -1,163 +0,0 @@ ->>> -macho: -cpusubtype = 3 -cputype = 7 -entryPoint = "3728" -filetype = 2 -flags = 18874501 -magic = 4277009102 -ncmds = 16 -numberOfSegments = "4" -sizeofcmds = 1060 -stackSize = "0" - -[[dylibs]] -compatibilityVersion = "1.0.0" -currentVersion = "1213.0.0" -name = "/usr/lib/libSystem.B.dylib" -timestamp = 2 - -[[segments]] -cmd = 1 -cmdsize = 56 -fileoff = "0" -filesize = "0" -flags = 0 -initprot = 0 -maxprot = 0 -nsects = 0 -segname = "__PAGEZERO" -vmaddr = "0" -vmsize = "4096" - -[[segments]] -cmd = 1 -cmdsize = 396 -fileoff = "0" -filesize = "4096" -flags = 0 -initprot = 5 -maxprot = 7 -nsects = 5 -segname = "__TEXT" -vmaddr = "4096" -vmsize = "4096" - -[[segments.sections]] -addr = "7824" -align = 4 -flags = 2147484672 -nreloc = 0 -offset = 3728 -reloff = 0 -reserved1 = 0 -reserved2 = 0 -sectname = "__text" -segname = "__TEXT" -size = "166" - -[[segments.sections]] -addr = "7990" -align = 1 -flags = 2147484936 -nreloc = 0 -offset = 3894 -reloff = 0 -reserved1 = 0 -reserved2 = 6 -sectname = "__symbol_stub" -segname = "__TEXT" -size = "12" - -[[segments.sections]] -addr = "8004" -align = 2 -flags = 2147484928 -nreloc = 0 -offset = 3908 -reloff = 0 -reserved1 = 0 -reserved2 = 0 -sectname = "__stub_helper" -segname = "__TEXT" -size = "32" - -[[segments.sections]] -addr = "8036" -align = 0 -flags = 2 -nreloc = 0 -offset = 3940 -reloff = 0 -reserved1 = 0 -reserved2 = 0 -sectname = "__cstring" -segname = "__TEXT" -size = "69" - -[[segments.sections]] -addr = "8108" -align = 2 -flags = 0 -nreloc = 0 -offset = 4012 -reloff = 0 -reserved1 = 0 -reserved2 = 0 -sectname = "__unwind_info" -segname = "__TEXT" -size = "72" - -[[segments]] -cmd = 1 -cmdsize = 192 -fileoff = "4096" -filesize = "4096" -flags = 0 -initprot = 3 -maxprot = 7 -nsects = 2 -segname = "__DATA" -vmaddr = "8192" -vmsize = "4096" - -[[segments.sections]] -addr = "8192" -align = 2 -flags = 6 -nreloc = 0 -offset = 4096 -reloff = 0 -reserved1 = 2 -reserved2 = 0 -sectname = "__nl_symbol_ptr" -segname = "__DATA" -size = "8" - -[[segments.sections]] -addr = "8200" -align = 2 -flags = 7 -nreloc = 0 -offset = 4104 -reloff = 0 -reserved1 = 4 -reserved2 = 0 -sectname = "__la_symbol_ptr" -segname = "__DATA" -size = "8" - -[[segments]] -cmd = 1 -cmdsize = 56 -fileoff = "8192" -filesize = "280" -flags = 0 -initprot = 1 -maxprot = 7 -nsects = 0 -segname = "__LINKEDIT" -vmaddr = "12288" -vmsize = "4096" - -<<< \ No newline at end of file diff --git a/yara-x-dump/src/tests/testdata/macho_x86_file.automatic.xml.out b/yara-x-dump/src/tests/testdata/macho_x86_file.automatic.xml.out deleted file mode 100644 index 48d3c0daa..000000000 --- a/yara-x-dump/src/tests/testdata/macho_x86_file.automatic.xml.out +++ /dev/null @@ -1,165 +0,0 @@ ->>> -macho: - - - 4277009102 - 7 - 3 - 2 - 16 - 1060 - 18874501 - 4 - - 1 - 56 - __PAGEZERO - 0 - 4096 - 0 - 0 - 0 - 0 - 0 - 0 - - - 1 - 396 - __TEXT - 4096 - 4096 - 0 - 4096 - 7 - 5 - 5 - 0 - - __TEXT - __text - 7824 - 166 - 3728 - 4 - 0 - 0 - 2147484672 - 0 - 0 - - - __TEXT - __symbol_stub - 7990 - 12 - 3894 - 1 - 0 - 0 - 2147484936 - 0 - 6 - - - __TEXT - __stub_helper - 8004 - 32 - 3908 - 2 - 0 - 0 - 2147484928 - 0 - 0 - - - __TEXT - __cstring - 8036 - 69 - 3940 - 0 - 0 - 0 - 2 - 0 - 0 - - - __TEXT - __unwind_info - 8108 - 72 - 4012 - 2 - 0 - 0 - 0 - 0 - 0 - - - - 1 - 192 - __DATA - 8192 - 4096 - 4096 - 4096 - 7 - 3 - 2 - 0 - - __DATA - __nl_symbol_ptr - 8192 - 8 - 4096 - 2 - 0 - 0 - 6 - 2 - 0 - - - __DATA - __la_symbol_ptr - 8200 - 8 - 4104 - 2 - 0 - 0 - 7 - 4 - 0 - - - - 1 - 56 - __LINKEDIT - 12288 - 4096 - 8192 - 280 - 7 - 1 - 0 - 0 - - - /usr/lib/libSystem.B.dylib - 2 - 1.0.0 - 1213.0.0 - - 3728 - 0 - -<<< \ No newline at end of file diff --git a/yara-x-dump/src/tests/testdata/macho_x86_file.automatic.yaml.out b/yara-x-dump/src/tests/testdata/macho_x86_file.automatic.yaml.out deleted file mode 100644 index ef6fbce6a..000000000 --- a/yara-x-dump/src/tests/testdata/macho_x86_file.automatic.yaml.out +++ /dev/null @@ -1,144 +0,0 @@ ->>> -macho: ---- -magic: 4277009102 -cputype: 7 -cpusubtype: 3 -filetype: 2 -ncmds: 16 -sizeofcmds: 1060 -flags: 18874501 -numberOfSegments: "4" -segments: - - cmd: 1 - cmdsize: 56 - segname: __PAGEZERO - vmaddr: "0" - vmsize: "4096" - fileoff: "0" - filesize: "0" - maxprot: 0 - initprot: 0 - nsects: 0 - flags: 0 - - cmd: 1 - cmdsize: 396 - segname: __TEXT - vmaddr: "4096" - vmsize: "4096" - fileoff: "0" - filesize: "4096" - maxprot: 7 - initprot: 5 - nsects: 5 - flags: 0 - sections: - - segname: __TEXT - sectname: __text - addr: "7824" - size: "166" - offset: 3728 - align: 4 - reloff: 0 - nreloc: 0 - flags: 2147484672 - reserved1: 0 - reserved2: 0 - - segname: __TEXT - sectname: __symbol_stub - addr: "7990" - size: "12" - offset: 3894 - align: 1 - reloff: 0 - nreloc: 0 - flags: 2147484936 - reserved1: 0 - reserved2: 6 - - segname: __TEXT - sectname: __stub_helper - addr: "8004" - size: "32" - offset: 3908 - align: 2 - reloff: 0 - nreloc: 0 - flags: 2147484928 - reserved1: 0 - reserved2: 0 - - segname: __TEXT - sectname: __cstring - addr: "8036" - size: "69" - offset: 3940 - align: 0 - reloff: 0 - nreloc: 0 - flags: 2 - reserved1: 0 - reserved2: 0 - - segname: __TEXT - sectname: __unwind_info - addr: "8108" - size: "72" - offset: 4012 - align: 2 - reloff: 0 - nreloc: 0 - flags: 0 - reserved1: 0 - reserved2: 0 - - cmd: 1 - cmdsize: 192 - segname: __DATA - vmaddr: "8192" - vmsize: "4096" - fileoff: "4096" - filesize: "4096" - maxprot: 7 - initprot: 3 - nsects: 2 - flags: 0 - sections: - - segname: __DATA - sectname: __nl_symbol_ptr - addr: "8192" - size: "8" - offset: 4096 - align: 2 - reloff: 0 - nreloc: 0 - flags: 6 - reserved1: 2 - reserved2: 0 - - segname: __DATA - sectname: __la_symbol_ptr - addr: "8200" - size: "8" - offset: 4104 - align: 2 - reloff: 0 - nreloc: 0 - flags: 7 - reserved1: 4 - reserved2: 0 - - cmd: 1 - cmdsize: 56 - segname: __LINKEDIT - vmaddr: "12288" - vmsize: "4096" - fileoff: "8192" - filesize: "280" - maxprot: 7 - initprot: 1 - nsects: 0 - flags: 0 -dylibs: - - name: /usr/lib/libSystem.B.dylib - timestamp: 2 - compatibilityVersion: 1.0.0 - currentVersion: 1213.0.0 -entryPoint: "3728" -stackSize: "0" - -<<< \ No newline at end of file diff --git a/yara-x-dump/src/tests/testdata/macho_x86_file.in b/yara-x-dump/src/tests/testdata/macho_x86_file.in index b81321cb0..de9aa4582 100644 --- a/yara-x-dump/src/tests/testdata/macho_x86_file.in +++ b/yara-x-dump/src/tests/testdata/macho_x86_file.in @@ -1,73 +1,345 @@ -:10000000CEFAEDFE07000000030000000200000031 -:100010001000000024040000850020010100000001 -:10002000380000005F5F504147455A45524F00007D -:1000300000000000000000000010000000000000B0 -:1000400000000000000000000000000000000000B0 -:1000500000000000010000008C0100005F5F5445BB -:1000600058540000000000000000000000100000D4 -:100070000010000000000000001000000700000059 -:100080000500000005000000000000005F5F7465CF -:100090007874000000000000000000005F5F54451D -:1000A000585400000000000000000000901E0000F6 -:1000B000A6000000900E00000400000000000000F8 -:1000C00000000000000400800000000000000000AC -:1000D0005F5F73796D626F6C5F73747562000000AF -:1000E0005F5F54455854000000000000000000000D -:1000F000361F00000C000000360F00000100000059 -:100100000000000000000000080500800000000062 -:10011000060000005F5F737475625F68656C7065F0 -:10012000720000005F5F544558540000000000005A -:1001300000000000441F000020000000440F0000E9 -:100140000200000000000000000000000005008028 -:1001500000000000000000005F5F63737472696E4E -:1001600067000000000000005F5F54455854000025 -:100170000000000000000000641F000045000000B7 -:10018000640F0000000000000000000000000000FC -:100190000200000000000000000000005F5F756EBC -:1001A00077696E645F696E666F0000005F5F54453B -:1001B000585400000000000000000000AC1F0000C8 -:1001C00048000000AC0F000002000000000000002A -:1001D000000000000000000000000000000000001F -:1001E00001000000C00000005F5F44415441000076 -:1001F00000000000000000000020000000100000CF -:1002000000100000001000000700000003000000C4 -:1002100002000000000000005F5F6E6C5F73796D8C -:10022000626F6C5F707472005F5F44415441000004 -:100230000000000000000000002000000800000096 -:10024000001000000200000000000000000000009C -:100250000600000002000000000000005F5F6C610B -:100260005F73796D626F6C5F707472005F5F4441A1 -:1002700054410000000000000000000008200000C1 -:10028000080000000810000002000000000000004C -:100290000000000007000000040000000000000053 -:1002A00001000000380000005F5F4C494E4B4544A0 -:1002B0004954000000000000003000000010000061 -:1002C00000200000180100000700000001000000ED -:1002D000000000000000000022000080300000004C -:1002E0000020000010000000102000001800000096 -:1002F0000000000000000000282000001C0000009A -:10030000442000002C000000020000001800000043 -:100310008820000005000000DC2000003C000000F8 -:100320000B00000050000000000000000000000072 -:1003300000000000020000000200000003000000B6 -:1003400000000000000000000000000000000000AD -:100350000000000000000000C420000006000000B3 -:10036000000000000000000000000000000000008D -:100370000E0000001C0000000C0000002F757372BE -:100380002F6C69622F64796C640000001B00000010 -:10039000180000005FB5950F40253D4FA8FB96481B -:1003A000C1740790240000001000000000090A003A -:1003B000000A0A002A0000001000000000000000EF -:1003C000000000002800008018000000900E0000CF -:1003D0000000000000000000000000000C00000011 -:1003E0003400000018000000020000000000BD04FE -:1003F000000001002F7573722F6C69622F6C6962A7 -:1004000053797374656D2E422E64796C69620000B5 -:100410002600000010000000702000000400000012 -:1004200029000000100000007420000000000000FF -:100430002B000000100000007420000014000000D9 -:1004400000000000000000000000000000000000AC -:10045000000000000000000000000000000000009C -:040460000000000098 -:00000001FF - +magic: 4277009103 +cputype: 16777223 +cpusubtype: 3 +filetype: 2 +ncmds: 21 +sizeofcmds: 2472 +flags: 10485893 +reserved: 0 +number_of_segments: 4 +segments { + cmd: 25 + cmdsize: 72 + segname: "__PAGEZERO" + vmaddr: 0 + vmsize: 4294967296 + fileoff: 0 + filesize: 0 + maxprot: 0 + initprot: 0 + nsects: 0 + flags: 0 +} +segments { + cmd: 25 + cmdsize: 712 + segname: "__TEXT" + vmaddr: 4294967296 + vmsize: 237568 + fileoff: 0 + filesize: 237568 + maxprot: 5 + initprot: 5 + nsects: 8 + flags: 0 + sections { + segname: "__TEXT" + sectname: "__text" + addr: 4294970912 + size: 212922 + offset: 3616 + align: 5 + reloff: 0 + nreloc: 0 + flags: 2147484672 + reserved1: 0 + reserved2: 0 + reserved3: 0 + } + sections { + segname: "__TEXT" + sectname: "__stubs" + addr: 4295183834 + size: 630 + offset: 216538 + align: 1 + reloff: 0 + nreloc: 0 + flags: 2147484680 + reserved1: 0 + reserved2: 6 + reserved3: 0 + } + sections { + segname: "__TEXT" + sectname: "__stub_helper" + addr: 4295184464 + size: 1066 + offset: 217168 + align: 2 + reloff: 0 + nreloc: 0 + flags: 2147484672 + reserved1: 0 + reserved2: 0 + reserved3: 0 + } + sections { + segname: "__TEXT" + sectname: "__cstring" + addr: 4295185536 + size: 4935 + offset: 218240 + align: 4 + reloff: 0 + nreloc: 0 + flags: 2 + reserved1: 0 + reserved2: 0 + reserved3: 0 + } + sections { + segname: "__TEXT" + sectname: "__const" + addr: 4295190480 + size: 4976 + offset: 223184 + align: 4 + reloff: 0 + nreloc: 0 + flags: 0 + reserved1: 0 + reserved2: 0 + reserved3: 0 + } + sections { + segname: "__TEXT" + sectname: "__gcc_except_tab" + addr: 4295195456 + size: 5844 + offset: 228160 + align: 2 + reloff: 0 + nreloc: 0 + flags: 0 + reserved1: 0 + reserved2: 0 + reserved3: 0 + } + sections { + segname: "__TEXT" + sectname: "__unwind_info" + addr: 4295201300 + size: 3416 + offset: 234004 + align: 2 + reloff: 0 + nreloc: 0 + flags: 0 + reserved1: 0 + reserved2: 0 + reserved3: 0 + } + sections { + segname: "__TEXT" + sectname: "__eh_frame" + addr: 4295204720 + size: 128 + offset: 237424 + align: 3 + reloff: 0 + nreloc: 0 + flags: 0 + reserved1: 0 + reserved2: 0 + reserved3: 0 + } +} +segments { + cmd: 25 + cmdsize: 872 + segname: "__DATA" + vmaddr: 4295204864 + vmsize: 20480 + fileoff: 237568 + filesize: 12288 + maxprot: 3 + initprot: 3 + nsects: 10 + flags: 0 + sections { + segname: "__DATA" + sectname: "__nl_symbol_ptr" + addr: 4295204864 + size: 8 + offset: 237568 + align: 3 + reloff: 0 + nreloc: 0 + flags: 6 + reserved1: 105 + reserved2: 0 + reserved3: 0 + } + sections { + segname: "__DATA" + sectname: "__got" + addr: 4295204872 + size: 72 + offset: 237576 + align: 3 + reloff: 0 + nreloc: 0 + flags: 6 + reserved1: 106 + reserved2: 0 + reserved3: 0 + } + sections { + segname: "__DATA" + sectname: "__la_symbol_ptr" + addr: 4295204944 + size: 840 + offset: 237648 + align: 3 + reloff: 0 + nreloc: 0 + flags: 7 + reserved1: 115 + reserved2: 0 + reserved3: 0 + } + sections { + segname: "__DATA" + sectname: "__mod_init_func" + addr: 4295205784 + size: 8 + offset: 238488 + align: 3 + reloff: 0 + nreloc: 0 + flags: 9 + reserved1: 0 + reserved2: 0 + reserved3: 0 + } + sections { + segname: "__DATA" + sectname: "__const" + addr: 4295205792 + size: 8336 + offset: 238496 + align: 4 + reloff: 0 + nreloc: 0 + flags: 0 + reserved1: 0 + reserved2: 0 + reserved3: 0 + } + sections { + segname: "__DATA" + sectname: "__data" + addr: 4295214128 + size: 808 + offset: 246832 + align: 3 + reloff: 0 + nreloc: 0 + flags: 0 + reserved1: 0 + reserved2: 0 + reserved3: 0 + } + sections { + segname: "__DATA" + sectname: "__thread_vars" + addr: 4295214936 + size: 24 + offset: 247640 + align: 3 + reloff: 0 + nreloc: 0 + flags: 19 + reserved1: 0 + reserved2: 0 + reserved3: 0 + } + sections { + segname: "__DATA" + sectname: "__thread_data" + addr: 4295214960 + size: 16 + offset: 247664 + align: 3 + reloff: 0 + nreloc: 0 + flags: 17 + reserved1: 0 + reserved2: 0 + reserved3: 0 + } + sections { + segname: "__DATA" + sectname: "__bss" + addr: 4295214976 + size: 5364 + offset: 0 + align: 4 + reloff: 0 + nreloc: 0 + flags: 1 + reserved1: 0 + reserved2: 0 + reserved3: 0 + } + sections { + segname: "__DATA" + sectname: "__common" + addr: 4295220344 + size: 1576 + offset: 0 + align: 3 + reloff: 0 + nreloc: 0 + flags: 1 + reserved1: 0 + reserved2: 0 + reserved3: 0 + } +} +segments { + cmd: 25 + cmdsize: 72 + segname: "__LINKEDIT" + vmaddr: 4295225344 + vmsize: 32768 + fileoff: 249856 + filesize: 30656 + maxprot: 1 + initprot: 1 + nsects: 0 + flags: 0 +} +dylibs { + name: "/System/Library/Frameworks/MediaPlayer.framework/Versions/A/MediaPlayer" + timestamp: 2 + compatibility_version: "1.0.0" + current_version: "1.0.0" +} +dylibs { + name: "@rpath/Electron Framework.framework/Electron Framework" + timestamp: 2 + compatibility_version: "0.0.0" + current_version: "0.0.0" +} +dylibs { + name: "/usr/lib/libsandbox.1.dylib" + timestamp: 2 + compatibility_version: "1.0.0" + current_version: "1.0.0" +} +dylibs { + name: "/usr/lib/libSystem.B.dylib" + timestamp: 2 + compatibility_version: "1.0.0" + current_version: "1252.250.1" +} +dylibs { + name: "/System/Library/Frameworks/CoreFoundation.framework/Versions/A/CoreFoundation" + timestamp: 2 + compatibility_version: "150.0.0" + current_version: "1570.15.0" +} +entry_point: 3616 +stack_size: 0 diff --git a/yara-x-dump/src/tests/testdata/macho_x86_file.lnk.None.out b/yara-x-dump/src/tests/testdata/macho_x86_file.lnk.None.out deleted file mode 100644 index 9eff539a9..000000000 --- a/yara-x-dump/src/tests/testdata/macho_x86_file.lnk.None.out +++ /dev/null @@ -1,5 +0,0 @@ ->>> -lnk: -is_lnk: false - -<<< \ No newline at end of file diff --git a/yara-x-dump/src/tests/testdata/macho_x86_file.lnk.human-readable.out b/yara-x-dump/src/tests/testdata/macho_x86_file.lnk.human-readable.out deleted file mode 100644 index 9eff539a9..000000000 --- a/yara-x-dump/src/tests/testdata/macho_x86_file.lnk.human-readable.out +++ /dev/null @@ -1,5 +0,0 @@ ->>> -lnk: -is_lnk: false - -<<< \ No newline at end of file diff --git a/yara-x-dump/src/tests/testdata/macho_x86_file.lnk.json.out b/yara-x-dump/src/tests/testdata/macho_x86_file.lnk.json.out deleted file mode 100644 index d1389a5f1..000000000 --- a/yara-x-dump/src/tests/testdata/macho_x86_file.lnk.json.out +++ /dev/null @@ -1,6 +0,0 @@ ->>> -lnk: -{ - "isLnk": false -} -<<< \ No newline at end of file diff --git a/yara-x-dump/src/tests/testdata/macho_x86_file.lnk.toml.out b/yara-x-dump/src/tests/testdata/macho_x86_file.lnk.toml.out deleted file mode 100644 index 94c59c00d..000000000 --- a/yara-x-dump/src/tests/testdata/macho_x86_file.lnk.toml.out +++ /dev/null @@ -1,5 +0,0 @@ ->>> -lnk: -isLnk = false - -<<< \ No newline at end of file diff --git a/yara-x-dump/src/tests/testdata/macho_x86_file.lnk.xml.out b/yara-x-dump/src/tests/testdata/macho_x86_file.lnk.xml.out deleted file mode 100644 index 931137069..000000000 --- a/yara-x-dump/src/tests/testdata/macho_x86_file.lnk.xml.out +++ /dev/null @@ -1,7 +0,0 @@ ->>> -lnk: - - - false - -<<< \ No newline at end of file diff --git a/yara-x-dump/src/tests/testdata/macho_x86_file.lnk.yaml.out b/yara-x-dump/src/tests/testdata/macho_x86_file.lnk.yaml.out deleted file mode 100644 index cca5783af..000000000 --- a/yara-x-dump/src/tests/testdata/macho_x86_file.lnk.yaml.out +++ /dev/null @@ -1,6 +0,0 @@ ->>> -lnk: ---- -isLnk: false - -<<< \ No newline at end of file diff --git a/yara-x-dump/src/tests/testdata/macho_x86_file.macho.None.out b/yara-x-dump/src/tests/testdata/macho_x86_file.macho.None.out deleted file mode 100644 index 446b6f89c..000000000 --- a/yara-x-dump/src/tests/testdata/macho_x86_file.macho.None.out +++ /dev/null @@ -1,147 +0,0 @@ ->>> -macho: -magic: 4277009102 (0xfeedface) -cputype: 7 -cpusubtype: 3 -filetype: 2 -ncmds: 16 -sizeofcmds: 1060 -flags: 18874501 (0x1200085) -number_of_segments: 4 -# Nested segments structure -segments: - - cmd: 1 - cmdsize: 56 - segname: "__PAGEZERO" - vmaddr: 0 (0x0) - vmsize: 4096 (0x1000) - fileoff: 0 - filesize: 0 - maxprot: 0 (0x0) - initprot: 0 (0x0) - nsects: 0 - flags: 0 (0x0) - - cmd: 1 - cmdsize: 396 - segname: "__TEXT" - vmaddr: 4096 (0x1000) - vmsize: 4096 (0x1000) - fileoff: 0 - filesize: 4096 - maxprot: 7 (0x7) - initprot: 5 (0x5) - nsects: 5 - flags: 0 (0x0) - # Nested sections structure - sections: - - segname: "__TEXT" - sectname: "__text" - addr: 7824 (0x1e90) - size: 166 (0xa6) - offset: 3728 - align: 4 - reloff: 0 - nreloc: 0 - flags: 2147484672 (0x80000400) - reserved1: 0 - reserved2: 0 - - segname: "__TEXT" - sectname: "__symbol_stub" - addr: 7990 (0x1f36) - size: 12 (0xc) - offset: 3894 - align: 1 - reloff: 0 - nreloc: 0 - flags: 2147484936 (0x80000508) - reserved1: 0 - reserved2: 6 - - segname: "__TEXT" - sectname: "__stub_helper" - addr: 8004 (0x1f44) - size: 32 (0x20) - offset: 3908 - align: 2 - reloff: 0 - nreloc: 0 - flags: 2147484928 (0x80000500) - reserved1: 0 - reserved2: 0 - - segname: "__TEXT" - sectname: "__cstring" - addr: 8036 (0x1f64) - size: 69 (0x45) - offset: 3940 - align: 0 - reloff: 0 - nreloc: 0 - flags: 2 (0x2) - reserved1: 0 - reserved2: 0 - - segname: "__TEXT" - sectname: "__unwind_info" - addr: 8108 (0x1fac) - size: 72 (0x48) - offset: 4012 - align: 2 - reloff: 0 - nreloc: 0 - flags: 0 (0x0) - reserved1: 0 - reserved2: 0 - - cmd: 1 - cmdsize: 192 - segname: "__DATA" - vmaddr: 8192 (0x2000) - vmsize: 4096 (0x1000) - fileoff: 4096 - filesize: 4096 - maxprot: 7 (0x7) - initprot: 3 (0x3) - nsects: 2 - flags: 0 (0x0) - # Nested sections structure - sections: - - segname: "__DATA" - sectname: "__nl_symbol_ptr" - addr: 8192 (0x2000) - size: 8 (0x8) - offset: 4096 - align: 2 - reloff: 0 - nreloc: 0 - flags: 6 (0x6) - reserved1: 2 - reserved2: 0 - - segname: "__DATA" - sectname: "__la_symbol_ptr" - addr: 8200 (0x2008) - size: 8 (0x8) - offset: 4104 - align: 2 - reloff: 0 - nreloc: 0 - flags: 7 (0x7) - reserved1: 4 - reserved2: 0 - - cmd: 1 - cmdsize: 56 - segname: "__LINKEDIT" - vmaddr: 12288 (0x3000) - vmsize: 4096 (0x1000) - fileoff: 8192 - filesize: 280 - maxprot: 7 (0x7) - initprot: 1 (0x1) - nsects: 0 - flags: 0 (0x0) -# Nested dylibs structure -dylibs: - - name: "/usr/lib/libSystem.B.dylib" - timestamp: 2 (1970-01-01 00:00:02 UTC) - compatibility_version: "1.0.0" - current_version: "1213.0.0" -entry_point: 3728 -stack_size: 0 - -<<< \ No newline at end of file diff --git a/yara-x-dump/src/tests/testdata/macho_x86_file.macho.human-readable.out b/yara-x-dump/src/tests/testdata/macho_x86_file.macho.human-readable.out deleted file mode 100644 index 446b6f89c..000000000 --- a/yara-x-dump/src/tests/testdata/macho_x86_file.macho.human-readable.out +++ /dev/null @@ -1,147 +0,0 @@ ->>> -macho: -magic: 4277009102 (0xfeedface) -cputype: 7 -cpusubtype: 3 -filetype: 2 -ncmds: 16 -sizeofcmds: 1060 -flags: 18874501 (0x1200085) -number_of_segments: 4 -# Nested segments structure -segments: - - cmd: 1 - cmdsize: 56 - segname: "__PAGEZERO" - vmaddr: 0 (0x0) - vmsize: 4096 (0x1000) - fileoff: 0 - filesize: 0 - maxprot: 0 (0x0) - initprot: 0 (0x0) - nsects: 0 - flags: 0 (0x0) - - cmd: 1 - cmdsize: 396 - segname: "__TEXT" - vmaddr: 4096 (0x1000) - vmsize: 4096 (0x1000) - fileoff: 0 - filesize: 4096 - maxprot: 7 (0x7) - initprot: 5 (0x5) - nsects: 5 - flags: 0 (0x0) - # Nested sections structure - sections: - - segname: "__TEXT" - sectname: "__text" - addr: 7824 (0x1e90) - size: 166 (0xa6) - offset: 3728 - align: 4 - reloff: 0 - nreloc: 0 - flags: 2147484672 (0x80000400) - reserved1: 0 - reserved2: 0 - - segname: "__TEXT" - sectname: "__symbol_stub" - addr: 7990 (0x1f36) - size: 12 (0xc) - offset: 3894 - align: 1 - reloff: 0 - nreloc: 0 - flags: 2147484936 (0x80000508) - reserved1: 0 - reserved2: 6 - - segname: "__TEXT" - sectname: "__stub_helper" - addr: 8004 (0x1f44) - size: 32 (0x20) - offset: 3908 - align: 2 - reloff: 0 - nreloc: 0 - flags: 2147484928 (0x80000500) - reserved1: 0 - reserved2: 0 - - segname: "__TEXT" - sectname: "__cstring" - addr: 8036 (0x1f64) - size: 69 (0x45) - offset: 3940 - align: 0 - reloff: 0 - nreloc: 0 - flags: 2 (0x2) - reserved1: 0 - reserved2: 0 - - segname: "__TEXT" - sectname: "__unwind_info" - addr: 8108 (0x1fac) - size: 72 (0x48) - offset: 4012 - align: 2 - reloff: 0 - nreloc: 0 - flags: 0 (0x0) - reserved1: 0 - reserved2: 0 - - cmd: 1 - cmdsize: 192 - segname: "__DATA" - vmaddr: 8192 (0x2000) - vmsize: 4096 (0x1000) - fileoff: 4096 - filesize: 4096 - maxprot: 7 (0x7) - initprot: 3 (0x3) - nsects: 2 - flags: 0 (0x0) - # Nested sections structure - sections: - - segname: "__DATA" - sectname: "__nl_symbol_ptr" - addr: 8192 (0x2000) - size: 8 (0x8) - offset: 4096 - align: 2 - reloff: 0 - nreloc: 0 - flags: 6 (0x6) - reserved1: 2 - reserved2: 0 - - segname: "__DATA" - sectname: "__la_symbol_ptr" - addr: 8200 (0x2008) - size: 8 (0x8) - offset: 4104 - align: 2 - reloff: 0 - nreloc: 0 - flags: 7 (0x7) - reserved1: 4 - reserved2: 0 - - cmd: 1 - cmdsize: 56 - segname: "__LINKEDIT" - vmaddr: 12288 (0x3000) - vmsize: 4096 (0x1000) - fileoff: 8192 - filesize: 280 - maxprot: 7 (0x7) - initprot: 1 (0x1) - nsects: 0 - flags: 0 (0x0) -# Nested dylibs structure -dylibs: - - name: "/usr/lib/libSystem.B.dylib" - timestamp: 2 (1970-01-01 00:00:02 UTC) - compatibility_version: "1.0.0" - current_version: "1213.0.0" -entry_point: 3728 -stack_size: 0 - -<<< \ No newline at end of file diff --git a/yara-x-dump/src/tests/testdata/macho_x86_file.macho.json.out b/yara-x-dump/src/tests/testdata/macho_x86_file.macho.json.out deleted file mode 100644 index e2eae2a1c..000000000 --- a/yara-x-dump/src/tests/testdata/macho_x86_file.macho.json.out +++ /dev/null @@ -1,172 +0,0 @@ ->>> -macho: -{ - "magic": 4277009102, - "cputype": 7, - "cpusubtype": 3, - "filetype": 2, - "ncmds": 16, - "sizeofcmds": 1060, - "flags": 18874501, - "numberOfSegments": "4", - "segments": [ - { - "cmd": 1, - "cmdsize": 56, - "segname": "__PAGEZERO", - "vmaddr": "0", - "vmsize": "4096", - "fileoff": "0", - "filesize": "0", - "maxprot": 0, - "initprot": 0, - "nsects": 0, - "flags": 0 - }, - { - "cmd": 1, - "cmdsize": 396, - "segname": "__TEXT", - "vmaddr": "4096", - "vmsize": "4096", - "fileoff": "0", - "filesize": "4096", - "maxprot": 7, - "initprot": 5, - "nsects": 5, - "flags": 0, - "sections": [ - { - "segname": "__TEXT", - "sectname": "__text", - "addr": "7824", - "size": "166", - "offset": 3728, - "align": 4, - "reloff": 0, - "nreloc": 0, - "flags": 2147484672, - "reserved1": 0, - "reserved2": 0 - }, - { - "segname": "__TEXT", - "sectname": "__symbol_stub", - "addr": "7990", - "size": "12", - "offset": 3894, - "align": 1, - "reloff": 0, - "nreloc": 0, - "flags": 2147484936, - "reserved1": 0, - "reserved2": 6 - }, - { - "segname": "__TEXT", - "sectname": "__stub_helper", - "addr": "8004", - "size": "32", - "offset": 3908, - "align": 2, - "reloff": 0, - "nreloc": 0, - "flags": 2147484928, - "reserved1": 0, - "reserved2": 0 - }, - { - "segname": "__TEXT", - "sectname": "__cstring", - "addr": "8036", - "size": "69", - "offset": 3940, - "align": 0, - "reloff": 0, - "nreloc": 0, - "flags": 2, - "reserved1": 0, - "reserved2": 0 - }, - { - "segname": "__TEXT", - "sectname": "__unwind_info", - "addr": "8108", - "size": "72", - "offset": 4012, - "align": 2, - "reloff": 0, - "nreloc": 0, - "flags": 0, - "reserved1": 0, - "reserved2": 0 - } - ] - }, - { - "cmd": 1, - "cmdsize": 192, - "segname": "__DATA", - "vmaddr": "8192", - "vmsize": "4096", - "fileoff": "4096", - "filesize": "4096", - "maxprot": 7, - "initprot": 3, - "nsects": 2, - "flags": 0, - "sections": [ - { - "segname": "__DATA", - "sectname": "__nl_symbol_ptr", - "addr": "8192", - "size": "8", - "offset": 4096, - "align": 2, - "reloff": 0, - "nreloc": 0, - "flags": 6, - "reserved1": 2, - "reserved2": 0 - }, - { - "segname": "__DATA", - "sectname": "__la_symbol_ptr", - "addr": "8200", - "size": "8", - "offset": 4104, - "align": 2, - "reloff": 0, - "nreloc": 0, - "flags": 7, - "reserved1": 4, - "reserved2": 0 - } - ] - }, - { - "cmd": 1, - "cmdsize": 56, - "segname": "__LINKEDIT", - "vmaddr": "12288", - "vmsize": "4096", - "fileoff": "8192", - "filesize": "280", - "maxprot": 7, - "initprot": 1, - "nsects": 0, - "flags": 0 - } - ], - "dylibs": [ - { - "name": "/usr/lib/libSystem.B.dylib", - "timestamp": 2, - "compatibilityVersion": "1.0.0", - "currentVersion": "1213.0.0" - } - ], - "entryPoint": "3728", - "stackSize": "0" -} -<<< \ No newline at end of file diff --git a/yara-x-dump/src/tests/testdata/macho_x86_file.macho.toml.out b/yara-x-dump/src/tests/testdata/macho_x86_file.macho.toml.out deleted file mode 100644 index 1d7844c93..000000000 --- a/yara-x-dump/src/tests/testdata/macho_x86_file.macho.toml.out +++ /dev/null @@ -1,163 +0,0 @@ ->>> -macho: -cpusubtype = 3 -cputype = 7 -entryPoint = "3728" -filetype = 2 -flags = 18874501 -magic = 4277009102 -ncmds = 16 -numberOfSegments = "4" -sizeofcmds = 1060 -stackSize = "0" - -[[dylibs]] -compatibilityVersion = "1.0.0" -currentVersion = "1213.0.0" -name = "/usr/lib/libSystem.B.dylib" -timestamp = 2 - -[[segments]] -cmd = 1 -cmdsize = 56 -fileoff = "0" -filesize = "0" -flags = 0 -initprot = 0 -maxprot = 0 -nsects = 0 -segname = "__PAGEZERO" -vmaddr = "0" -vmsize = "4096" - -[[segments]] -cmd = 1 -cmdsize = 396 -fileoff = "0" -filesize = "4096" -flags = 0 -initprot = 5 -maxprot = 7 -nsects = 5 -segname = "__TEXT" -vmaddr = "4096" -vmsize = "4096" - -[[segments.sections]] -addr = "7824" -align = 4 -flags = 2147484672 -nreloc = 0 -offset = 3728 -reloff = 0 -reserved1 = 0 -reserved2 = 0 -sectname = "__text" -segname = "__TEXT" -size = "166" - -[[segments.sections]] -addr = "7990" -align = 1 -flags = 2147484936 -nreloc = 0 -offset = 3894 -reloff = 0 -reserved1 = 0 -reserved2 = 6 -sectname = "__symbol_stub" -segname = "__TEXT" -size = "12" - -[[segments.sections]] -addr = "8004" -align = 2 -flags = 2147484928 -nreloc = 0 -offset = 3908 -reloff = 0 -reserved1 = 0 -reserved2 = 0 -sectname = "__stub_helper" -segname = "__TEXT" -size = "32" - -[[segments.sections]] -addr = "8036" -align = 0 -flags = 2 -nreloc = 0 -offset = 3940 -reloff = 0 -reserved1 = 0 -reserved2 = 0 -sectname = "__cstring" -segname = "__TEXT" -size = "69" - -[[segments.sections]] -addr = "8108" -align = 2 -flags = 0 -nreloc = 0 -offset = 4012 -reloff = 0 -reserved1 = 0 -reserved2 = 0 -sectname = "__unwind_info" -segname = "__TEXT" -size = "72" - -[[segments]] -cmd = 1 -cmdsize = 192 -fileoff = "4096" -filesize = "4096" -flags = 0 -initprot = 3 -maxprot = 7 -nsects = 2 -segname = "__DATA" -vmaddr = "8192" -vmsize = "4096" - -[[segments.sections]] -addr = "8192" -align = 2 -flags = 6 -nreloc = 0 -offset = 4096 -reloff = 0 -reserved1 = 2 -reserved2 = 0 -sectname = "__nl_symbol_ptr" -segname = "__DATA" -size = "8" - -[[segments.sections]] -addr = "8200" -align = 2 -flags = 7 -nreloc = 0 -offset = 4104 -reloff = 0 -reserved1 = 4 -reserved2 = 0 -sectname = "__la_symbol_ptr" -segname = "__DATA" -size = "8" - -[[segments]] -cmd = 1 -cmdsize = 56 -fileoff = "8192" -filesize = "280" -flags = 0 -initprot = 1 -maxprot = 7 -nsects = 0 -segname = "__LINKEDIT" -vmaddr = "12288" -vmsize = "4096" - -<<< \ No newline at end of file diff --git a/yara-x-dump/src/tests/testdata/macho_x86_file.macho.xml.out b/yara-x-dump/src/tests/testdata/macho_x86_file.macho.xml.out deleted file mode 100644 index 48d3c0daa..000000000 --- a/yara-x-dump/src/tests/testdata/macho_x86_file.macho.xml.out +++ /dev/null @@ -1,165 +0,0 @@ ->>> -macho: - - - 4277009102 - 7 - 3 - 2 - 16 - 1060 - 18874501 - 4 - - 1 - 56 - __PAGEZERO - 0 - 4096 - 0 - 0 - 0 - 0 - 0 - 0 - - - 1 - 396 - __TEXT - 4096 - 4096 - 0 - 4096 - 7 - 5 - 5 - 0 - - __TEXT - __text - 7824 - 166 - 3728 - 4 - 0 - 0 - 2147484672 - 0 - 0 - - - __TEXT - __symbol_stub - 7990 - 12 - 3894 - 1 - 0 - 0 - 2147484936 - 0 - 6 - - - __TEXT - __stub_helper - 8004 - 32 - 3908 - 2 - 0 - 0 - 2147484928 - 0 - 0 - - - __TEXT - __cstring - 8036 - 69 - 3940 - 0 - 0 - 0 - 2 - 0 - 0 - - - __TEXT - __unwind_info - 8108 - 72 - 4012 - 2 - 0 - 0 - 0 - 0 - 0 - - - - 1 - 192 - __DATA - 8192 - 4096 - 4096 - 4096 - 7 - 3 - 2 - 0 - - __DATA - __nl_symbol_ptr - 8192 - 8 - 4096 - 2 - 0 - 0 - 6 - 2 - 0 - - - __DATA - __la_symbol_ptr - 8200 - 8 - 4104 - 2 - 0 - 0 - 7 - 4 - 0 - - - - 1 - 56 - __LINKEDIT - 12288 - 4096 - 8192 - 280 - 7 - 1 - 0 - 0 - - - /usr/lib/libSystem.B.dylib - 2 - 1.0.0 - 1213.0.0 - - 3728 - 0 - -<<< \ No newline at end of file diff --git a/yara-x-dump/src/tests/testdata/macho_x86_file.macho.yaml.out b/yara-x-dump/src/tests/testdata/macho_x86_file.macho.yaml.out deleted file mode 100644 index ef6fbce6a..000000000 --- a/yara-x-dump/src/tests/testdata/macho_x86_file.macho.yaml.out +++ /dev/null @@ -1,144 +0,0 @@ ->>> -macho: ---- -magic: 4277009102 -cputype: 7 -cpusubtype: 3 -filetype: 2 -ncmds: 16 -sizeofcmds: 1060 -flags: 18874501 -numberOfSegments: "4" -segments: - - cmd: 1 - cmdsize: 56 - segname: __PAGEZERO - vmaddr: "0" - vmsize: "4096" - fileoff: "0" - filesize: "0" - maxprot: 0 - initprot: 0 - nsects: 0 - flags: 0 - - cmd: 1 - cmdsize: 396 - segname: __TEXT - vmaddr: "4096" - vmsize: "4096" - fileoff: "0" - filesize: "4096" - maxprot: 7 - initprot: 5 - nsects: 5 - flags: 0 - sections: - - segname: __TEXT - sectname: __text - addr: "7824" - size: "166" - offset: 3728 - align: 4 - reloff: 0 - nreloc: 0 - flags: 2147484672 - reserved1: 0 - reserved2: 0 - - segname: __TEXT - sectname: __symbol_stub - addr: "7990" - size: "12" - offset: 3894 - align: 1 - reloff: 0 - nreloc: 0 - flags: 2147484936 - reserved1: 0 - reserved2: 6 - - segname: __TEXT - sectname: __stub_helper - addr: "8004" - size: "32" - offset: 3908 - align: 2 - reloff: 0 - nreloc: 0 - flags: 2147484928 - reserved1: 0 - reserved2: 0 - - segname: __TEXT - sectname: __cstring - addr: "8036" - size: "69" - offset: 3940 - align: 0 - reloff: 0 - nreloc: 0 - flags: 2 - reserved1: 0 - reserved2: 0 - - segname: __TEXT - sectname: __unwind_info - addr: "8108" - size: "72" - offset: 4012 - align: 2 - reloff: 0 - nreloc: 0 - flags: 0 - reserved1: 0 - reserved2: 0 - - cmd: 1 - cmdsize: 192 - segname: __DATA - vmaddr: "8192" - vmsize: "4096" - fileoff: "4096" - filesize: "4096" - maxprot: 7 - initprot: 3 - nsects: 2 - flags: 0 - sections: - - segname: __DATA - sectname: __nl_symbol_ptr - addr: "8192" - size: "8" - offset: 4096 - align: 2 - reloff: 0 - nreloc: 0 - flags: 6 - reserved1: 2 - reserved2: 0 - - segname: __DATA - sectname: __la_symbol_ptr - addr: "8200" - size: "8" - offset: 4104 - align: 2 - reloff: 0 - nreloc: 0 - flags: 7 - reserved1: 4 - reserved2: 0 - - cmd: 1 - cmdsize: 56 - segname: __LINKEDIT - vmaddr: "12288" - vmsize: "4096" - fileoff: "8192" - filesize: "280" - maxprot: 7 - initprot: 1 - nsects: 0 - flags: 0 -dylibs: - - name: /usr/lib/libSystem.B.dylib - timestamp: 2 - compatibilityVersion: 1.0.0 - currentVersion: 1213.0.0 -entryPoint: "3728" -stackSize: "0" - -<<< \ No newline at end of file diff --git a/yara-x-dump/src/tests/testdata/macho_x86_file.macho_lnk.None.out b/yara-x-dump/src/tests/testdata/macho_x86_file.macho_lnk.None.out deleted file mode 100644 index aee684dd0..000000000 --- a/yara-x-dump/src/tests/testdata/macho_x86_file.macho_lnk.None.out +++ /dev/null @@ -1,151 +0,0 @@ ->>> -lnk: -is_lnk: false - -<<<>>> -macho: -magic: 4277009102 (0xfeedface) -cputype: 7 -cpusubtype: 3 -filetype: 2 -ncmds: 16 -sizeofcmds: 1060 -flags: 18874501 (0x1200085) -number_of_segments: 4 -# Nested segments structure -segments: - - cmd: 1 - cmdsize: 56 - segname: "__PAGEZERO" - vmaddr: 0 (0x0) - vmsize: 4096 (0x1000) - fileoff: 0 - filesize: 0 - maxprot: 0 (0x0) - initprot: 0 (0x0) - nsects: 0 - flags: 0 (0x0) - - cmd: 1 - cmdsize: 396 - segname: "__TEXT" - vmaddr: 4096 (0x1000) - vmsize: 4096 (0x1000) - fileoff: 0 - filesize: 4096 - maxprot: 7 (0x7) - initprot: 5 (0x5) - nsects: 5 - flags: 0 (0x0) - # Nested sections structure - sections: - - segname: "__TEXT" - sectname: "__text" - addr: 7824 (0x1e90) - size: 166 (0xa6) - offset: 3728 - align: 4 - reloff: 0 - nreloc: 0 - flags: 2147484672 (0x80000400) - reserved1: 0 - reserved2: 0 - - segname: "__TEXT" - sectname: "__symbol_stub" - addr: 7990 (0x1f36) - size: 12 (0xc) - offset: 3894 - align: 1 - reloff: 0 - nreloc: 0 - flags: 2147484936 (0x80000508) - reserved1: 0 - reserved2: 6 - - segname: "__TEXT" - sectname: "__stub_helper" - addr: 8004 (0x1f44) - size: 32 (0x20) - offset: 3908 - align: 2 - reloff: 0 - nreloc: 0 - flags: 2147484928 (0x80000500) - reserved1: 0 - reserved2: 0 - - segname: "__TEXT" - sectname: "__cstring" - addr: 8036 (0x1f64) - size: 69 (0x45) - offset: 3940 - align: 0 - reloff: 0 - nreloc: 0 - flags: 2 (0x2) - reserved1: 0 - reserved2: 0 - - segname: "__TEXT" - sectname: "__unwind_info" - addr: 8108 (0x1fac) - size: 72 (0x48) - offset: 4012 - align: 2 - reloff: 0 - nreloc: 0 - flags: 0 (0x0) - reserved1: 0 - reserved2: 0 - - cmd: 1 - cmdsize: 192 - segname: "__DATA" - vmaddr: 8192 (0x2000) - vmsize: 4096 (0x1000) - fileoff: 4096 - filesize: 4096 - maxprot: 7 (0x7) - initprot: 3 (0x3) - nsects: 2 - flags: 0 (0x0) - # Nested sections structure - sections: - - segname: "__DATA" - sectname: "__nl_symbol_ptr" - addr: 8192 (0x2000) - size: 8 (0x8) - offset: 4096 - align: 2 - reloff: 0 - nreloc: 0 - flags: 6 (0x6) - reserved1: 2 - reserved2: 0 - - segname: "__DATA" - sectname: "__la_symbol_ptr" - addr: 8200 (0x2008) - size: 8 (0x8) - offset: 4104 - align: 2 - reloff: 0 - nreloc: 0 - flags: 7 (0x7) - reserved1: 4 - reserved2: 0 - - cmd: 1 - cmdsize: 56 - segname: "__LINKEDIT" - vmaddr: 12288 (0x3000) - vmsize: 4096 (0x1000) - fileoff: 8192 - filesize: 280 - maxprot: 7 (0x7) - initprot: 1 (0x1) - nsects: 0 - flags: 0 (0x0) -# Nested dylibs structure -dylibs: - - name: "/usr/lib/libSystem.B.dylib" - timestamp: 2 (1970-01-01 00:00:02 UTC) - compatibility_version: "1.0.0" - current_version: "1213.0.0" -entry_point: 3728 -stack_size: 0 - -<<< \ No newline at end of file diff --git a/yara-x-dump/src/tests/testdata/macho_x86_file.macho_lnk.human-readable.out b/yara-x-dump/src/tests/testdata/macho_x86_file.macho_lnk.human-readable.out deleted file mode 100644 index aee684dd0..000000000 --- a/yara-x-dump/src/tests/testdata/macho_x86_file.macho_lnk.human-readable.out +++ /dev/null @@ -1,151 +0,0 @@ ->>> -lnk: -is_lnk: false - -<<<>>> -macho: -magic: 4277009102 (0xfeedface) -cputype: 7 -cpusubtype: 3 -filetype: 2 -ncmds: 16 -sizeofcmds: 1060 -flags: 18874501 (0x1200085) -number_of_segments: 4 -# Nested segments structure -segments: - - cmd: 1 - cmdsize: 56 - segname: "__PAGEZERO" - vmaddr: 0 (0x0) - vmsize: 4096 (0x1000) - fileoff: 0 - filesize: 0 - maxprot: 0 (0x0) - initprot: 0 (0x0) - nsects: 0 - flags: 0 (0x0) - - cmd: 1 - cmdsize: 396 - segname: "__TEXT" - vmaddr: 4096 (0x1000) - vmsize: 4096 (0x1000) - fileoff: 0 - filesize: 4096 - maxprot: 7 (0x7) - initprot: 5 (0x5) - nsects: 5 - flags: 0 (0x0) - # Nested sections structure - sections: - - segname: "__TEXT" - sectname: "__text" - addr: 7824 (0x1e90) - size: 166 (0xa6) - offset: 3728 - align: 4 - reloff: 0 - nreloc: 0 - flags: 2147484672 (0x80000400) - reserved1: 0 - reserved2: 0 - - segname: "__TEXT" - sectname: "__symbol_stub" - addr: 7990 (0x1f36) - size: 12 (0xc) - offset: 3894 - align: 1 - reloff: 0 - nreloc: 0 - flags: 2147484936 (0x80000508) - reserved1: 0 - reserved2: 6 - - segname: "__TEXT" - sectname: "__stub_helper" - addr: 8004 (0x1f44) - size: 32 (0x20) - offset: 3908 - align: 2 - reloff: 0 - nreloc: 0 - flags: 2147484928 (0x80000500) - reserved1: 0 - reserved2: 0 - - segname: "__TEXT" - sectname: "__cstring" - addr: 8036 (0x1f64) - size: 69 (0x45) - offset: 3940 - align: 0 - reloff: 0 - nreloc: 0 - flags: 2 (0x2) - reserved1: 0 - reserved2: 0 - - segname: "__TEXT" - sectname: "__unwind_info" - addr: 8108 (0x1fac) - size: 72 (0x48) - offset: 4012 - align: 2 - reloff: 0 - nreloc: 0 - flags: 0 (0x0) - reserved1: 0 - reserved2: 0 - - cmd: 1 - cmdsize: 192 - segname: "__DATA" - vmaddr: 8192 (0x2000) - vmsize: 4096 (0x1000) - fileoff: 4096 - filesize: 4096 - maxprot: 7 (0x7) - initprot: 3 (0x3) - nsects: 2 - flags: 0 (0x0) - # Nested sections structure - sections: - - segname: "__DATA" - sectname: "__nl_symbol_ptr" - addr: 8192 (0x2000) - size: 8 (0x8) - offset: 4096 - align: 2 - reloff: 0 - nreloc: 0 - flags: 6 (0x6) - reserved1: 2 - reserved2: 0 - - segname: "__DATA" - sectname: "__la_symbol_ptr" - addr: 8200 (0x2008) - size: 8 (0x8) - offset: 4104 - align: 2 - reloff: 0 - nreloc: 0 - flags: 7 (0x7) - reserved1: 4 - reserved2: 0 - - cmd: 1 - cmdsize: 56 - segname: "__LINKEDIT" - vmaddr: 12288 (0x3000) - vmsize: 4096 (0x1000) - fileoff: 8192 - filesize: 280 - maxprot: 7 (0x7) - initprot: 1 (0x1) - nsects: 0 - flags: 0 (0x0) -# Nested dylibs structure -dylibs: - - name: "/usr/lib/libSystem.B.dylib" - timestamp: 2 (1970-01-01 00:00:02 UTC) - compatibility_version: "1.0.0" - current_version: "1213.0.0" -entry_point: 3728 -stack_size: 0 - -<<< \ No newline at end of file diff --git a/yara-x-dump/src/tests/testdata/macho_x86_file.macho_lnk.json.out b/yara-x-dump/src/tests/testdata/macho_x86_file.macho_lnk.json.out deleted file mode 100644 index 0b75d1830..000000000 --- a/yara-x-dump/src/tests/testdata/macho_x86_file.macho_lnk.json.out +++ /dev/null @@ -1,177 +0,0 @@ ->>> -lnk: -{ - "isLnk": false -} -<<<>>> -macho: -{ - "magic": 4277009102, - "cputype": 7, - "cpusubtype": 3, - "filetype": 2, - "ncmds": 16, - "sizeofcmds": 1060, - "flags": 18874501, - "numberOfSegments": "4", - "segments": [ - { - "cmd": 1, - "cmdsize": 56, - "segname": "__PAGEZERO", - "vmaddr": "0", - "vmsize": "4096", - "fileoff": "0", - "filesize": "0", - "maxprot": 0, - "initprot": 0, - "nsects": 0, - "flags": 0 - }, - { - "cmd": 1, - "cmdsize": 396, - "segname": "__TEXT", - "vmaddr": "4096", - "vmsize": "4096", - "fileoff": "0", - "filesize": "4096", - "maxprot": 7, - "initprot": 5, - "nsects": 5, - "flags": 0, - "sections": [ - { - "segname": "__TEXT", - "sectname": "__text", - "addr": "7824", - "size": "166", - "offset": 3728, - "align": 4, - "reloff": 0, - "nreloc": 0, - "flags": 2147484672, - "reserved1": 0, - "reserved2": 0 - }, - { - "segname": "__TEXT", - "sectname": "__symbol_stub", - "addr": "7990", - "size": "12", - "offset": 3894, - "align": 1, - "reloff": 0, - "nreloc": 0, - "flags": 2147484936, - "reserved1": 0, - "reserved2": 6 - }, - { - "segname": "__TEXT", - "sectname": "__stub_helper", - "addr": "8004", - "size": "32", - "offset": 3908, - "align": 2, - "reloff": 0, - "nreloc": 0, - "flags": 2147484928, - "reserved1": 0, - "reserved2": 0 - }, - { - "segname": "__TEXT", - "sectname": "__cstring", - "addr": "8036", - "size": "69", - "offset": 3940, - "align": 0, - "reloff": 0, - "nreloc": 0, - "flags": 2, - "reserved1": 0, - "reserved2": 0 - }, - { - "segname": "__TEXT", - "sectname": "__unwind_info", - "addr": "8108", - "size": "72", - "offset": 4012, - "align": 2, - "reloff": 0, - "nreloc": 0, - "flags": 0, - "reserved1": 0, - "reserved2": 0 - } - ] - }, - { - "cmd": 1, - "cmdsize": 192, - "segname": "__DATA", - "vmaddr": "8192", - "vmsize": "4096", - "fileoff": "4096", - "filesize": "4096", - "maxprot": 7, - "initprot": 3, - "nsects": 2, - "flags": 0, - "sections": [ - { - "segname": "__DATA", - "sectname": "__nl_symbol_ptr", - "addr": "8192", - "size": "8", - "offset": 4096, - "align": 2, - "reloff": 0, - "nreloc": 0, - "flags": 6, - "reserved1": 2, - "reserved2": 0 - }, - { - "segname": "__DATA", - "sectname": "__la_symbol_ptr", - "addr": "8200", - "size": "8", - "offset": 4104, - "align": 2, - "reloff": 0, - "nreloc": 0, - "flags": 7, - "reserved1": 4, - "reserved2": 0 - } - ] - }, - { - "cmd": 1, - "cmdsize": 56, - "segname": "__LINKEDIT", - "vmaddr": "12288", - "vmsize": "4096", - "fileoff": "8192", - "filesize": "280", - "maxprot": 7, - "initprot": 1, - "nsects": 0, - "flags": 0 - } - ], - "dylibs": [ - { - "name": "/usr/lib/libSystem.B.dylib", - "timestamp": 2, - "compatibilityVersion": "1.0.0", - "currentVersion": "1213.0.0" - } - ], - "entryPoint": "3728", - "stackSize": "0" -} -<<< \ No newline at end of file diff --git a/yara-x-dump/src/tests/testdata/macho_x86_file.macho_lnk.toml.out b/yara-x-dump/src/tests/testdata/macho_x86_file.macho_lnk.toml.out deleted file mode 100644 index 70af5ca36..000000000 --- a/yara-x-dump/src/tests/testdata/macho_x86_file.macho_lnk.toml.out +++ /dev/null @@ -1,167 +0,0 @@ ->>> -lnk: -isLnk = false - -<<<>>> -macho: -cpusubtype = 3 -cputype = 7 -entryPoint = "3728" -filetype = 2 -flags = 18874501 -magic = 4277009102 -ncmds = 16 -numberOfSegments = "4" -sizeofcmds = 1060 -stackSize = "0" - -[[dylibs]] -compatibilityVersion = "1.0.0" -currentVersion = "1213.0.0" -name = "/usr/lib/libSystem.B.dylib" -timestamp = 2 - -[[segments]] -cmd = 1 -cmdsize = 56 -fileoff = "0" -filesize = "0" -flags = 0 -initprot = 0 -maxprot = 0 -nsects = 0 -segname = "__PAGEZERO" -vmaddr = "0" -vmsize = "4096" - -[[segments]] -cmd = 1 -cmdsize = 396 -fileoff = "0" -filesize = "4096" -flags = 0 -initprot = 5 -maxprot = 7 -nsects = 5 -segname = "__TEXT" -vmaddr = "4096" -vmsize = "4096" - -[[segments.sections]] -addr = "7824" -align = 4 -flags = 2147484672 -nreloc = 0 -offset = 3728 -reloff = 0 -reserved1 = 0 -reserved2 = 0 -sectname = "__text" -segname = "__TEXT" -size = "166" - -[[segments.sections]] -addr = "7990" -align = 1 -flags = 2147484936 -nreloc = 0 -offset = 3894 -reloff = 0 -reserved1 = 0 -reserved2 = 6 -sectname = "__symbol_stub" -segname = "__TEXT" -size = "12" - -[[segments.sections]] -addr = "8004" -align = 2 -flags = 2147484928 -nreloc = 0 -offset = 3908 -reloff = 0 -reserved1 = 0 -reserved2 = 0 -sectname = "__stub_helper" -segname = "__TEXT" -size = "32" - -[[segments.sections]] -addr = "8036" -align = 0 -flags = 2 -nreloc = 0 -offset = 3940 -reloff = 0 -reserved1 = 0 -reserved2 = 0 -sectname = "__cstring" -segname = "__TEXT" -size = "69" - -[[segments.sections]] -addr = "8108" -align = 2 -flags = 0 -nreloc = 0 -offset = 4012 -reloff = 0 -reserved1 = 0 -reserved2 = 0 -sectname = "__unwind_info" -segname = "__TEXT" -size = "72" - -[[segments]] -cmd = 1 -cmdsize = 192 -fileoff = "4096" -filesize = "4096" -flags = 0 -initprot = 3 -maxprot = 7 -nsects = 2 -segname = "__DATA" -vmaddr = "8192" -vmsize = "4096" - -[[segments.sections]] -addr = "8192" -align = 2 -flags = 6 -nreloc = 0 -offset = 4096 -reloff = 0 -reserved1 = 2 -reserved2 = 0 -sectname = "__nl_symbol_ptr" -segname = "__DATA" -size = "8" - -[[segments.sections]] -addr = "8200" -align = 2 -flags = 7 -nreloc = 0 -offset = 4104 -reloff = 0 -reserved1 = 4 -reserved2 = 0 -sectname = "__la_symbol_ptr" -segname = "__DATA" -size = "8" - -[[segments]] -cmd = 1 -cmdsize = 56 -fileoff = "8192" -filesize = "280" -flags = 0 -initprot = 1 -maxprot = 7 -nsects = 0 -segname = "__LINKEDIT" -vmaddr = "12288" -vmsize = "4096" - -<<< \ No newline at end of file diff --git a/yara-x-dump/src/tests/testdata/macho_x86_file.macho_lnk.xml.out b/yara-x-dump/src/tests/testdata/macho_x86_file.macho_lnk.xml.out deleted file mode 100644 index 91c635bd6..000000000 --- a/yara-x-dump/src/tests/testdata/macho_x86_file.macho_lnk.xml.out +++ /dev/null @@ -1,171 +0,0 @@ ->>> -lnk: - - - false - -<<<>>> -macho: - - - 4277009102 - 7 - 3 - 2 - 16 - 1060 - 18874501 - 4 - - 1 - 56 - __PAGEZERO - 0 - 4096 - 0 - 0 - 0 - 0 - 0 - 0 - - - 1 - 396 - __TEXT - 4096 - 4096 - 0 - 4096 - 7 - 5 - 5 - 0 - - __TEXT - __text - 7824 - 166 - 3728 - 4 - 0 - 0 - 2147484672 - 0 - 0 - - - __TEXT - __symbol_stub - 7990 - 12 - 3894 - 1 - 0 - 0 - 2147484936 - 0 - 6 - - - __TEXT - __stub_helper - 8004 - 32 - 3908 - 2 - 0 - 0 - 2147484928 - 0 - 0 - - - __TEXT - __cstring - 8036 - 69 - 3940 - 0 - 0 - 0 - 2 - 0 - 0 - - - __TEXT - __unwind_info - 8108 - 72 - 4012 - 2 - 0 - 0 - 0 - 0 - 0 - - - - 1 - 192 - __DATA - 8192 - 4096 - 4096 - 4096 - 7 - 3 - 2 - 0 - - __DATA - __nl_symbol_ptr - 8192 - 8 - 4096 - 2 - 0 - 0 - 6 - 2 - 0 - - - __DATA - __la_symbol_ptr - 8200 - 8 - 4104 - 2 - 0 - 0 - 7 - 4 - 0 - - - - 1 - 56 - __LINKEDIT - 12288 - 4096 - 8192 - 280 - 7 - 1 - 0 - 0 - - - /usr/lib/libSystem.B.dylib - 2 - 1.0.0 - 1213.0.0 - - 3728 - 0 - -<<< \ No newline at end of file diff --git a/yara-x-dump/src/tests/testdata/macho_x86_file.macho_lnk.yaml.out b/yara-x-dump/src/tests/testdata/macho_x86_file.macho_lnk.yaml.out deleted file mode 100644 index 145753fb4..000000000 --- a/yara-x-dump/src/tests/testdata/macho_x86_file.macho_lnk.yaml.out +++ /dev/null @@ -1,149 +0,0 @@ ->>> -lnk: ---- -isLnk: false - -<<<>>> -macho: ---- -magic: 4277009102 -cputype: 7 -cpusubtype: 3 -filetype: 2 -ncmds: 16 -sizeofcmds: 1060 -flags: 18874501 -numberOfSegments: "4" -segments: - - cmd: 1 - cmdsize: 56 - segname: __PAGEZERO - vmaddr: "0" - vmsize: "4096" - fileoff: "0" - filesize: "0" - maxprot: 0 - initprot: 0 - nsects: 0 - flags: 0 - - cmd: 1 - cmdsize: 396 - segname: __TEXT - vmaddr: "4096" - vmsize: "4096" - fileoff: "0" - filesize: "4096" - maxprot: 7 - initprot: 5 - nsects: 5 - flags: 0 - sections: - - segname: __TEXT - sectname: __text - addr: "7824" - size: "166" - offset: 3728 - align: 4 - reloff: 0 - nreloc: 0 - flags: 2147484672 - reserved1: 0 - reserved2: 0 - - segname: __TEXT - sectname: __symbol_stub - addr: "7990" - size: "12" - offset: 3894 - align: 1 - reloff: 0 - nreloc: 0 - flags: 2147484936 - reserved1: 0 - reserved2: 6 - - segname: __TEXT - sectname: __stub_helper - addr: "8004" - size: "32" - offset: 3908 - align: 2 - reloff: 0 - nreloc: 0 - flags: 2147484928 - reserved1: 0 - reserved2: 0 - - segname: __TEXT - sectname: __cstring - addr: "8036" - size: "69" - offset: 3940 - align: 0 - reloff: 0 - nreloc: 0 - flags: 2 - reserved1: 0 - reserved2: 0 - - segname: __TEXT - sectname: __unwind_info - addr: "8108" - size: "72" - offset: 4012 - align: 2 - reloff: 0 - nreloc: 0 - flags: 0 - reserved1: 0 - reserved2: 0 - - cmd: 1 - cmdsize: 192 - segname: __DATA - vmaddr: "8192" - vmsize: "4096" - fileoff: "4096" - filesize: "4096" - maxprot: 7 - initprot: 3 - nsects: 2 - flags: 0 - sections: - - segname: __DATA - sectname: __nl_symbol_ptr - addr: "8192" - size: "8" - offset: 4096 - align: 2 - reloff: 0 - nreloc: 0 - flags: 6 - reserved1: 2 - reserved2: 0 - - segname: __DATA - sectname: __la_symbol_ptr - addr: "8200" - size: "8" - offset: 4104 - align: 2 - reloff: 0 - nreloc: 0 - flags: 7 - reserved1: 4 - reserved2: 0 - - cmd: 1 - cmdsize: 56 - segname: __LINKEDIT - vmaddr: "12288" - vmsize: "4096" - fileoff: "8192" - filesize: "280" - maxprot: 7 - initprot: 1 - nsects: 0 - flags: 0 -dylibs: - - name: /usr/lib/libSystem.B.dylib - timestamp: 2 - compatibilityVersion: 1.0.0 - currentVersion: 1213.0.0 -entryPoint: "3728" -stackSize: "0" - -<<< \ No newline at end of file diff --git a/yara-x-dump/src/tests/testdata/test.proto b/yara-x-dump/src/tests/testdata/test.proto new file mode 100644 index 000000000..8d28ce740 --- /dev/null +++ b/yara-x-dump/src/tests/testdata/test.proto @@ -0,0 +1,7 @@ +syntax = "proto2"; + +message MyMessage { + optional int32 field1 = 1; + optional string field2 = 2; + repeated int32 repeated_field = 3; +} From aa5ad9d9ae45d2e077daa20a6887cd946af5eebb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20=C4=8Euri=C5=A1?= Date: Wed, 8 Nov 2023 16:46:55 +0100 Subject: [PATCH 20/26] yr dump decoupled --- Cargo.lock | 2 + yara-x-dump/Cargo.toml | 6 + yara-x-dump/build.rs | 11 + yara-x-dump/src/lib.rs | 3 + yara-x-dump/src/serializer.rs | 3 +- yara-x-dump/src/tests/mod.rs | 30 +- yara-x-dump/src/tests/protos/dumper.proto | 14 + yara-x-dump/src/tests/protos/test.proto | 24 ++ .../tests/testdata/macho_x86_file.None.out | 13 + .../macho_x86_file.human-readable.out | 13 + .../src/tests/testdata/macho_x86_file.in | 352 +----------------- .../tests/testdata/macho_x86_file.json.out | 18 + .../tests/testdata/macho_x86_file.toml.out | 14 + .../src/tests/testdata/macho_x86_file.xml.out | 17 + .../tests/testdata/macho_x86_file.yaml.out | 12 + yara-x-dump/src/tests/testdata/test.proto | 7 - 16 files changed, 185 insertions(+), 354 deletions(-) create mode 100644 yara-x-dump/build.rs create mode 100644 yara-x-dump/src/tests/protos/dumper.proto create mode 100644 yara-x-dump/src/tests/protos/test.proto create mode 100644 yara-x-dump/src/tests/testdata/macho_x86_file.None.out create mode 100644 yara-x-dump/src/tests/testdata/macho_x86_file.human-readable.out create mode 100644 yara-x-dump/src/tests/testdata/macho_x86_file.json.out create mode 100644 yara-x-dump/src/tests/testdata/macho_x86_file.toml.out create mode 100644 yara-x-dump/src/tests/testdata/macho_x86_file.xml.out create mode 100644 yara-x-dump/src/tests/testdata/macho_x86_file.yaml.out delete mode 100644 yara-x-dump/src/tests/testdata/test.proto diff --git a/Cargo.lock b/Cargo.lock index e1ff02cb0..3140825d9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4456,7 +4456,9 @@ dependencies = [ "goldenfile", "ihex", "protobuf", + "protobuf-codegen", "protobuf-json-mapping", + "protobuf-parse", "protobuf-support", "rstest", "serde_json", diff --git a/yara-x-dump/Cargo.toml b/yara-x-dump/Cargo.toml index 2d9e7e9c6..6138de3de 100644 --- a/yara-x-dump/Cargo.toml +++ b/yara-x-dump/Cargo.toml @@ -33,3 +33,9 @@ ihex = "3.0.0" globwalk = { workspace = true } tempfile = "3.8.1" rstest = "0.18.2" + + +[build-dependencies] +protobuf = { workspace = true } +protobuf-codegen = { workspace = true } +protobuf-parse = { workspace = true } \ No newline at end of file diff --git a/yara-x-dump/build.rs b/yara-x-dump/build.rs new file mode 100644 index 000000000..8b9f47942 --- /dev/null +++ b/yara-x-dump/build.rs @@ -0,0 +1,11 @@ +use protobuf_codegen::Codegen; + +fn main() { + Codegen::new() + .pure() + .cargo_out_dir("protos") + .include("src/tests/protos") + .input("src/tests/protos/dumper.proto") + .input("src/tests/protos/test.proto") + .run_from_script(); +} diff --git a/yara-x-dump/src/lib.rs b/yara-x-dump/src/lib.rs index ca5438603..d208eaeba 100644 --- a/yara-x-dump/src/lib.rs +++ b/yara-x-dump/src/lib.rs @@ -10,6 +10,9 @@ use crate::serializer::{get_human_readable_output, get_serializer}; #[cfg(test)] mod tests; +pub use test::*; +include!(concat!(env!("OUT_DIR"), "/protos/mod.rs")); + /// Errors returned by [`Dumper::dump`]. #[derive(Error, Debug)] #[allow(clippy::large_enum_variant)] diff --git a/yara-x-dump/src/serializer.rs b/yara-x-dump/src/serializer.rs index c0359387d..83ad118da 100644 --- a/yara-x-dump/src/serializer.rs +++ b/yara-x-dump/src/serializer.rs @@ -6,7 +6,8 @@ use protobuf::reflect::ReflectValueRef; use protobuf_support::text_format::quote_bytes_to; use std::fmt::Write; use yansi::Color; -use yara_x_proto::exts::field_options; + +use crate::dumper::exts::field_options; use crate::Error; diff --git a/yara-x-dump/src/tests/mod.rs b/yara-x-dump/src/tests/mod.rs index b1ebc916e..e70ea2437 100644 --- a/yara-x-dump/src/tests/mod.rs +++ b/yara-x-dump/src/tests/mod.rs @@ -1,10 +1,21 @@ use protobuf::text_format::parse_from_str; +use protobuf::MessageDyn; use rstest::rstest; use std::fs; use std::io::Write; -#[rstest(output_format, case("json"))] -fn test_dumper(output_format: &str) { +use crate::Dumper; + +#[rstest( + output_format, + case("json"), + case("yaml"), + case("toml"), + case("xml"), + case("human-readable"), + case("None") +)] +fn test_dumper(output_format: String) { // Create goldenfile mint. let mut mint = goldenfile::Mint::new("."); @@ -21,10 +32,19 @@ fn test_dumper(output_format: &str) { output_format ); let input = fs::read_to_string(in_path).expect("Unable to read"); - let protobuf_module_output = parse_from_str(&input).unwrap(); - let dumper = crate::Dumper::default(); - let output = dumper.dump(protobuf_module_output, output_format); + let test = parse_from_str::(&input).unwrap(); + + let dumper = Dumper::default(); + let output = if output_format == "None" { + dumper.dump(&test as &dyn MessageDyn, None) + } else { + dumper.dump( + &test as &dyn MessageDyn, + Some(&output_format.to_string()), + ) + } + .unwrap(); // Create a goldenfile test let mut output_file = mint.new_goldenfile(test_name).unwrap(); diff --git a/yara-x-dump/src/tests/protos/dumper.proto b/yara-x-dump/src/tests/protos/dumper.proto new file mode 100644 index 000000000..38a6005dd --- /dev/null +++ b/yara-x-dump/src/tests/protos/dumper.proto @@ -0,0 +1,14 @@ +syntax = "proto2"; + +package dumper; + +import "google/protobuf/descriptor.proto"; + +message FieldOptions { + optional bool hex_value = 3; + optional bool timestamp = 4; + } + +extend google.protobuf.FieldOptions { + optional FieldOptions field_options = 51504; + } \ No newline at end of file diff --git a/yara-x-dump/src/tests/protos/test.proto b/yara-x-dump/src/tests/protos/test.proto new file mode 100644 index 000000000..f5ff756a6 --- /dev/null +++ b/yara-x-dump/src/tests/protos/test.proto @@ -0,0 +1,24 @@ +syntax = "proto2"; + +import "dumper.proto"; + +package test; + +message Segment { + optional uint32 nested1 = 1; + optional uint64 nested2 = 2 [(dumper.field_options).hex_value = true]; + optional uint32 timestamp = 3 [(dumper.field_options).timestamp = true]; +} + +message OptionalNested { + optional uint32 onested1 = 1; + optional uint64 onested2 = 2 [(dumper.field_options).hex_value = true]; + map map_string_string = 3; +} + +message MyMessage { + optional int32 field1 = 1 [(dumper.field_options).hex_value = true]; + optional string field2 = 2; + repeated Segment segments = 3; + optional OptionalNested optional = 4; +} diff --git a/yara-x-dump/src/tests/testdata/macho_x86_file.None.out b/yara-x-dump/src/tests/testdata/macho_x86_file.None.out new file mode 100644 index 000000000..003d9a37e --- /dev/null +++ b/yara-x-dump/src/tests/testdata/macho_x86_file.None.out @@ -0,0 +1,13 @@ +field1: 123 (0x7b) +field2: "test" +# Nested segments structure +segments: + - nested1: 456 + nested2: 789 (0x315) + timestamp: 123456789 (1973-11-29 21:33:09 UTC) +# Nested optional structure +optional: + - onested1: 123 + onested2: 456 (0x1c8) + map_string_string: + foo: "bar" diff --git a/yara-x-dump/src/tests/testdata/macho_x86_file.human-readable.out b/yara-x-dump/src/tests/testdata/macho_x86_file.human-readable.out new file mode 100644 index 000000000..003d9a37e --- /dev/null +++ b/yara-x-dump/src/tests/testdata/macho_x86_file.human-readable.out @@ -0,0 +1,13 @@ +field1: 123 (0x7b) +field2: "test" +# Nested segments structure +segments: + - nested1: 456 + nested2: 789 (0x315) + timestamp: 123456789 (1973-11-29 21:33:09 UTC) +# Nested optional structure +optional: + - onested1: 123 + onested2: 456 (0x1c8) + map_string_string: + foo: "bar" diff --git a/yara-x-dump/src/tests/testdata/macho_x86_file.in b/yara-x-dump/src/tests/testdata/macho_x86_file.in index de9aa4582..3213164d3 100644 --- a/yara-x-dump/src/tests/testdata/macho_x86_file.in +++ b/yara-x-dump/src/tests/testdata/macho_x86_file.in @@ -1,345 +1,15 @@ -magic: 4277009103 -cputype: 16777223 -cpusubtype: 3 -filetype: 2 -ncmds: 21 -sizeofcmds: 2472 -flags: 10485893 -reserved: 0 -number_of_segments: 4 +field1: 123 +field2: "test" segments { - cmd: 25 - cmdsize: 72 - segname: "__PAGEZERO" - vmaddr: 0 - vmsize: 4294967296 - fileoff: 0 - filesize: 0 - maxprot: 0 - initprot: 0 - nsects: 0 - flags: 0 + nested1: 456 + nested2: 789 + timestamp: 123456789 } -segments { - cmd: 25 - cmdsize: 712 - segname: "__TEXT" - vmaddr: 4294967296 - vmsize: 237568 - fileoff: 0 - filesize: 237568 - maxprot: 5 - initprot: 5 - nsects: 8 - flags: 0 - sections { - segname: "__TEXT" - sectname: "__text" - addr: 4294970912 - size: 212922 - offset: 3616 - align: 5 - reloff: 0 - nreloc: 0 - flags: 2147484672 - reserved1: 0 - reserved2: 0 - reserved3: 0 - } - sections { - segname: "__TEXT" - sectname: "__stubs" - addr: 4295183834 - size: 630 - offset: 216538 - align: 1 - reloff: 0 - nreloc: 0 - flags: 2147484680 - reserved1: 0 - reserved2: 6 - reserved3: 0 - } - sections { - segname: "__TEXT" - sectname: "__stub_helper" - addr: 4295184464 - size: 1066 - offset: 217168 - align: 2 - reloff: 0 - nreloc: 0 - flags: 2147484672 - reserved1: 0 - reserved2: 0 - reserved3: 0 - } - sections { - segname: "__TEXT" - sectname: "__cstring" - addr: 4295185536 - size: 4935 - offset: 218240 - align: 4 - reloff: 0 - nreloc: 0 - flags: 2 - reserved1: 0 - reserved2: 0 - reserved3: 0 - } - sections { - segname: "__TEXT" - sectname: "__const" - addr: 4295190480 - size: 4976 - offset: 223184 - align: 4 - reloff: 0 - nreloc: 0 - flags: 0 - reserved1: 0 - reserved2: 0 - reserved3: 0 - } - sections { - segname: "__TEXT" - sectname: "__gcc_except_tab" - addr: 4295195456 - size: 5844 - offset: 228160 - align: 2 - reloff: 0 - nreloc: 0 - flags: 0 - reserved1: 0 - reserved2: 0 - reserved3: 0 - } - sections { - segname: "__TEXT" - sectname: "__unwind_info" - addr: 4295201300 - size: 3416 - offset: 234004 - align: 2 - reloff: 0 - nreloc: 0 - flags: 0 - reserved1: 0 - reserved2: 0 - reserved3: 0 - } - sections { - segname: "__TEXT" - sectname: "__eh_frame" - addr: 4295204720 - size: 128 - offset: 237424 - align: 3 - reloff: 0 - nreloc: 0 - flags: 0 - reserved1: 0 - reserved2: 0 - reserved3: 0 - } -} -segments { - cmd: 25 - cmdsize: 872 - segname: "__DATA" - vmaddr: 4295204864 - vmsize: 20480 - fileoff: 237568 - filesize: 12288 - maxprot: 3 - initprot: 3 - nsects: 10 - flags: 0 - sections { - segname: "__DATA" - sectname: "__nl_symbol_ptr" - addr: 4295204864 - size: 8 - offset: 237568 - align: 3 - reloff: 0 - nreloc: 0 - flags: 6 - reserved1: 105 - reserved2: 0 - reserved3: 0 - } - sections { - segname: "__DATA" - sectname: "__got" - addr: 4295204872 - size: 72 - offset: 237576 - align: 3 - reloff: 0 - nreloc: 0 - flags: 6 - reserved1: 106 - reserved2: 0 - reserved3: 0 - } - sections { - segname: "__DATA" - sectname: "__la_symbol_ptr" - addr: 4295204944 - size: 840 - offset: 237648 - align: 3 - reloff: 0 - nreloc: 0 - flags: 7 - reserved1: 115 - reserved2: 0 - reserved3: 0 - } - sections { - segname: "__DATA" - sectname: "__mod_init_func" - addr: 4295205784 - size: 8 - offset: 238488 - align: 3 - reloff: 0 - nreloc: 0 - flags: 9 - reserved1: 0 - reserved2: 0 - reserved3: 0 +optional { + onested1: 123 + onested2: 456 + map_string_string { + key: "foo" + value: "bar" } - sections { - segname: "__DATA" - sectname: "__const" - addr: 4295205792 - size: 8336 - offset: 238496 - align: 4 - reloff: 0 - nreloc: 0 - flags: 0 - reserved1: 0 - reserved2: 0 - reserved3: 0 - } - sections { - segname: "__DATA" - sectname: "__data" - addr: 4295214128 - size: 808 - offset: 246832 - align: 3 - reloff: 0 - nreloc: 0 - flags: 0 - reserved1: 0 - reserved2: 0 - reserved3: 0 - } - sections { - segname: "__DATA" - sectname: "__thread_vars" - addr: 4295214936 - size: 24 - offset: 247640 - align: 3 - reloff: 0 - nreloc: 0 - flags: 19 - reserved1: 0 - reserved2: 0 - reserved3: 0 - } - sections { - segname: "__DATA" - sectname: "__thread_data" - addr: 4295214960 - size: 16 - offset: 247664 - align: 3 - reloff: 0 - nreloc: 0 - flags: 17 - reserved1: 0 - reserved2: 0 - reserved3: 0 - } - sections { - segname: "__DATA" - sectname: "__bss" - addr: 4295214976 - size: 5364 - offset: 0 - align: 4 - reloff: 0 - nreloc: 0 - flags: 1 - reserved1: 0 - reserved2: 0 - reserved3: 0 - } - sections { - segname: "__DATA" - sectname: "__common" - addr: 4295220344 - size: 1576 - offset: 0 - align: 3 - reloff: 0 - nreloc: 0 - flags: 1 - reserved1: 0 - reserved2: 0 - reserved3: 0 - } -} -segments { - cmd: 25 - cmdsize: 72 - segname: "__LINKEDIT" - vmaddr: 4295225344 - vmsize: 32768 - fileoff: 249856 - filesize: 30656 - maxprot: 1 - initprot: 1 - nsects: 0 - flags: 0 -} -dylibs { - name: "/System/Library/Frameworks/MediaPlayer.framework/Versions/A/MediaPlayer" - timestamp: 2 - compatibility_version: "1.0.0" - current_version: "1.0.0" -} -dylibs { - name: "@rpath/Electron Framework.framework/Electron Framework" - timestamp: 2 - compatibility_version: "0.0.0" - current_version: "0.0.0" -} -dylibs { - name: "/usr/lib/libsandbox.1.dylib" - timestamp: 2 - compatibility_version: "1.0.0" - current_version: "1.0.0" -} -dylibs { - name: "/usr/lib/libSystem.B.dylib" - timestamp: 2 - compatibility_version: "1.0.0" - current_version: "1252.250.1" -} -dylibs { - name: "/System/Library/Frameworks/CoreFoundation.framework/Versions/A/CoreFoundation" - timestamp: 2 - compatibility_version: "150.0.0" - current_version: "1570.15.0" } -entry_point: 3616 -stack_size: 0 diff --git a/yara-x-dump/src/tests/testdata/macho_x86_file.json.out b/yara-x-dump/src/tests/testdata/macho_x86_file.json.out new file mode 100644 index 000000000..e6ab96391 --- /dev/null +++ b/yara-x-dump/src/tests/testdata/macho_x86_file.json.out @@ -0,0 +1,18 @@ +{ + "field1": 123, + "field2": "test", + "segments": [ + { + "nested1": 456, + "nested2": "789", + "timestamp": 123456789 + } + ], + "optional": { + "onested1": 123, + "onested2": "456", + "mapStringString": { + "foo": "bar" + } + } +} diff --git a/yara-x-dump/src/tests/testdata/macho_x86_file.toml.out b/yara-x-dump/src/tests/testdata/macho_x86_file.toml.out new file mode 100644 index 000000000..3c0dda9b2 --- /dev/null +++ b/yara-x-dump/src/tests/testdata/macho_x86_file.toml.out @@ -0,0 +1,14 @@ +field1 = 123 +field2 = "test" + +[[segments]] +nested1 = 456 +nested2 = "789" +timestamp = 123456789 + +[optional] +onested1 = 123 +onested2 = "456" + +[optional.mapStringString] +foo = "bar" diff --git a/yara-x-dump/src/tests/testdata/macho_x86_file.xml.out b/yara-x-dump/src/tests/testdata/macho_x86_file.xml.out new file mode 100644 index 000000000..733f3faba --- /dev/null +++ b/yara-x-dump/src/tests/testdata/macho_x86_file.xml.out @@ -0,0 +1,17 @@ + + + 123 + test + + 456 + 789 + 123456789 + + + 123 + 456 + + bar + + + diff --git a/yara-x-dump/src/tests/testdata/macho_x86_file.yaml.out b/yara-x-dump/src/tests/testdata/macho_x86_file.yaml.out new file mode 100644 index 000000000..58237c145 --- /dev/null +++ b/yara-x-dump/src/tests/testdata/macho_x86_file.yaml.out @@ -0,0 +1,12 @@ +--- +field1: 123 +field2: test +segments: + - nested1: 456 + nested2: "789" + timestamp: 123456789 +optional: + onested1: 123 + onested2: "456" + mapStringString: + foo: bar diff --git a/yara-x-dump/src/tests/testdata/test.proto b/yara-x-dump/src/tests/testdata/test.proto deleted file mode 100644 index 8d28ce740..000000000 --- a/yara-x-dump/src/tests/testdata/test.proto +++ /dev/null @@ -1,7 +0,0 @@ -syntax = "proto2"; - -message MyMessage { - optional int32 field1 = 1; - optional string field2 = 2; - repeated int32 repeated_field = 3; -} From 5e2844c53f15ec906f2e002b0232a710f7685d47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20=C4=8Euri=C5=A1?= Date: Wed, 8 Nov 2023 16:54:12 +0100 Subject: [PATCH 21/26] remove unused dependancies --- Cargo.lock | 2 -- yara-x-dump/Cargo.toml | 2 -- 2 files changed, 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3140825d9..5344803e9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4454,7 +4454,6 @@ dependencies = [ "chrono", "globwalk", "goldenfile", - "ihex", "protobuf", "protobuf-codegen", "protobuf-json-mapping", @@ -4468,7 +4467,6 @@ dependencies = [ "toml 0.8.4", "xml2json-rs", "yansi", - "yara-x-proto", ] [[package]] diff --git a/yara-x-dump/Cargo.toml b/yara-x-dump/Cargo.toml index 6138de3de..ce6c9616e 100644 --- a/yara-x-dump/Cargo.toml +++ b/yara-x-dump/Cargo.toml @@ -15,7 +15,6 @@ crate-type = ["rlib", "cdylib"] [dependencies] anyhow = { workspace = true } thiserror = { workspace = true } -yara-x-proto = { workspace = true } protobuf = { workspace = true } yansi = { workspace = true } @@ -29,7 +28,6 @@ chrono = "0.4.31" [dev-dependencies] goldenfile = "1.5.2" -ihex = "3.0.0" globwalk = { workspace = true } tempfile = "3.8.1" rstest = "0.18.2" From 157bee20653ec5d3c049cf33a20b3be0a8e18bd0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20=C4=8Euri=C5=A1?= Date: Thu, 9 Nov 2023 15:16:03 +0100 Subject: [PATCH 22/26] unnecessary formats removed and code refactored --- Cargo.lock | 28 ++- docs/Module Developer's Guide.md | 13 +- yara-x-cli/Cargo.toml | 4 +- yara-x-cli/src/commands/dump.rs | 65 ++++-- yara-x-dump/Cargo.toml | 1 - yara-x-dump/src/lib.rs | 70 +----- yara-x-dump/src/serializer.rs | 210 ++++++------------ yara-x-dump/src/tests/mod.rs | 39 +--- yara-x-dump/src/tests/protos/dumper.proto | 3 +- yara-x-dump/src/tests/protos/test.proto | 8 +- .../tests/testdata/macho_x86_file.None.out | 13 -- .../macho_x86_file.human-readable.out | 13 -- .../tests/testdata/macho_x86_file.json.out | 18 -- .../src/tests/testdata/macho_x86_file.out | 13 ++ .../tests/testdata/macho_x86_file.toml.out | 14 -- .../src/tests/testdata/macho_x86_file.xml.out | 17 -- .../tests/testdata/macho_x86_file.yaml.out | 12 - yara-x-proto/src/yara.proto | 3 +- yara-x/src/modules/protos/macho.proto | 28 +-- 19 files changed, 204 insertions(+), 368 deletions(-) delete mode 100644 yara-x-dump/src/tests/testdata/macho_x86_file.None.out delete mode 100644 yara-x-dump/src/tests/testdata/macho_x86_file.human-readable.out delete mode 100644 yara-x-dump/src/tests/testdata/macho_x86_file.json.out create mode 100644 yara-x-dump/src/tests/testdata/macho_x86_file.out delete mode 100644 yara-x-dump/src/tests/testdata/macho_x86_file.toml.out delete mode 100644 yara-x-dump/src/tests/testdata/macho_x86_file.xml.out delete mode 100644 yara-x-dump/src/tests/testdata/macho_x86_file.yaml.out diff --git a/Cargo.lock b/Cargo.lock index 5344803e9..649434db7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -500,6 +500,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac495e00dcec98c83465d5ad66c5c4fabd652fd6686e7c6269b117e729a6f17b" dependencies = [ "clap_builder", + "clap_derive", ] [[package]] @@ -514,6 +515,18 @@ dependencies = [ "strsim", ] +[[package]] +name = "clap_derive" +version = "4.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf9804afaaf59a91e75b022a30fb7229a7901f60c755489cc61c9b423b836442" +dependencies = [ + "heck 0.4.1", + "proc-macro2", + "quote", + "syn 2.0.38", +] + [[package]] name = "clap_lex" version = "0.6.0" @@ -526,6 +539,18 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" +[[package]] +name = "colored_json" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66203a5372a3ffbef29a90b4f8ed98eae3be5662bc7922c7dee1cb57bd40f9a8" +dependencies = [ + "is-terminal", + "serde", + "serde_json", + "yansi", +] + [[package]] name = "compact_str" version = "0.7.1" @@ -4426,6 +4451,7 @@ dependencies = [ "anyhow", "ascii_tree", "clap", + "colored_json", "crossbeam", "crossterm 0.27.0", "enable-ansi-support", @@ -4435,6 +4461,7 @@ dependencies = [ "log", "pprof", "protobuf", + "protobuf-json-mapping", "serde_json", "superconsole", "wild", @@ -4456,7 +4483,6 @@ dependencies = [ "goldenfile", "protobuf", "protobuf-codegen", - "protobuf-json-mapping", "protobuf-parse", "protobuf-support", "rstest", diff --git a/docs/Module Developer's Guide.md b/docs/Module Developer's Guide.md index 5c47e1849..a16de22c7 100644 --- a/docs/Module Developer's Guide.md +++ b/docs/Module Developer's Guide.md @@ -162,22 +162,21 @@ explore the protobuf's [documentation](https://developers.google.com/protocol-bu One thing that can be done with integer fields is to represent them in some other way. This optional representation is shown in `yr dump` crate output. This crate provides -multiple output formats as JSON, XML, YAML, TOML and human-readable. The last mentioned -is the only one that provides also custom representation for integer numbers. Let's say +two output formats: JSON and YAML. Both can be shown in colored output via `-c|--color` option. The last mentioned also provides custom representation for integer numbers. Let's say for some fields it makes sense to show them as hexadecimal numbers. This can be done by -adding `[(yara.field_options).hex_value = true];` descriptor to the field. For example: +adding `[(yara.field_options).yaml_fmt = ""];` descriptor to the field. Currently supported formats are: hexadecimal number and human-readable timestamp. For example: ``` message Macho { - optional uint32 magic = 1 [(yara.field_options).hex_value = true]; + optional uint32 magic = 1 [(yara.field_options).yml_fmt= "x"]; } ``` This will mark magic field as a hexadecimal number and it will be shown as -`magic: 4277009103 (0xfeedfacf)`. Other format that is supported right now is +`magic: 0xfeedfacf` instead of `4277009103`. Other format that is supported right now is for timestamps. If you want to show some integer field as a timestamp you can do it by -setting `[(yara.field_options).timestamp = true];` descriptor to the field and -timestamps will be shown in human-readable format. +setting `[(yara.field_options).yml_fmt = "t"];` descriptor to the field and +human readable timestamps will be shown in YAML comment after its integer value. Also notice that we are defining our fields as `optional`. In `proto2` fields diff --git a/yara-x-cli/Cargo.toml b/yara-x-cli/Cargo.toml index e2d1920a8..256be9ebd 100644 --- a/yara-x-cli/Cargo.toml +++ b/yara-x-cli/Cargo.toml @@ -36,12 +36,13 @@ logging = ["dep:log", "dep:env_logger"] [dependencies] ascii_tree = { workspace = true } anyhow = { workspace = true } -clap = { workspace = true, features=["cargo"] } +clap = { workspace = true, features=["cargo", "derive"] } globwalk = { workspace = true } enable-ansi-support = { workspace = true } env_logger = { workspace = true , optional = true } log = { workspace = true, optional = true } protobuf = { workspace = true } +protobuf-json-mapping = "3.3.0" serde_json = { workspace = true } yansi = { workspace = true } yara-x = { workspace = true } @@ -56,3 +57,4 @@ indent = "0.1.1" pprof = { version = "0.12.1", features = ["flamegraph"], optional=true } superconsole = "0.2.0" wild = "2.1.0" +colored_json = "4.0.0" diff --git a/yara-x-cli/src/commands/dump.rs b/yara-x-cli/src/commands/dump.rs index c69e1f049..b15a38a75 100644 --- a/yara-x-cli/src/commands/dump.rs +++ b/yara-x-cli/src/commands/dump.rs @@ -1,16 +1,28 @@ -use clap::{arg, value_parser, Arg, ArgAction, ArgMatches, Command}; +use clap::{ + arg, value_parser, Arg, ArgAction, ArgMatches, Command, ValueEnum, +}; + +use colored_json::ToColoredJson; use protobuf::{reflect::ReflectValueRef::Bool, MessageDyn}; +use protobuf_json_mapping::print_to_string; use std::fmt::Write; use std::fs::File; use std::io::stdin; +use std::io::Read; use std::path::PathBuf; -use yansi::Color::Cyan; +use yansi::{Color::Cyan, Paint}; use yara_x_proto::exts::module_options; use yara_x_dump::Dumper; use yara_x::get_builtin_modules_names; +#[derive(Debug, Clone, ValueEnum)] +enum OutputFormats { + JSON, + YAML, +} + /// Creates the `dump` command. /// The `dump` command dumps information about binary files. /// @@ -29,9 +41,13 @@ pub fn dump() -> Command { .arg( arg!(-o --"output-format" ) .help("Desired output format") - .value_parser(value_parser!(String)) + .value_parser(value_parser!(OutputFormats)) .required(false), ) + .arg( + arg!(-c - -"color") + .help("Use colorful output") + ) .arg( Arg::new("modules") .long("modules") @@ -91,7 +107,13 @@ pub fn exec_dump(args: &ArgMatches) -> anyhow::Result<()> { let mut result = String::new(); let file = args.get_one::("FILE"); - let output_format = args.get_one::("output-format"); + let output_format = args.get_one::("output-format"); + let colors_flag = args.get_flag("color"); + + // Disable colors if the flag is not set. + if !colors_flag { + Paint::disable(); + } // get vector of modules let modules: Vec<&str> = args @@ -100,17 +122,13 @@ pub fn exec_dump(args: &ArgMatches) -> anyhow::Result<()> { .map(|s| s.as_str()) .collect(); - let dumper = Dumper::default(); - // Get the input. - let mut input: Box = if let Some(file) = file { - Box::new(File::open(file.as_path())?) + if let Some(file) = file { + File::open(file.as_path())?.read_to_end(&mut buffer)? } else { - Box::new(stdin()) + stdin().read_to_end(&mut buffer)? }; - input.read_to_end(&mut buffer)?; - // Get the list of modules to import. let import_modules = if !modules.is_empty() { modules.clone() @@ -141,12 +159,25 @@ pub fn exec_dump(args: &ArgMatches) -> anyhow::Result<()> { if mod_output.compute_size_dyn() != 0 && (module_is_valid(mod_output) || modules.contains(&mod_name)) { - write!( - result, - ">>>\n{}:\n{}<<<\n", - Cyan.paint(mod_name).bold(), - dumper.dump(mod_output, output_format)? - )?; + match output_format { + Some(OutputFormats::JSON) => { + writeln!( + result, + ">>>\n{}:\n{}\n<<<", + Cyan.paint(mod_name).bold(), + print_to_string(mod_output)?.to_colored_json_auto()? + )?; + } + Some(OutputFormats::YAML) | None => { + let dumper = Dumper::default(); + write!( + result, + ">>>\n{}:\n{}<<<", + Cyan.paint(mod_name).bold(), + dumper.dump(mod_output)? + )?; + } + } } } diff --git a/yara-x-dump/Cargo.toml b/yara-x-dump/Cargo.toml index ce6c9616e..d07a10753 100644 --- a/yara-x-dump/Cargo.toml +++ b/yara-x-dump/Cargo.toml @@ -21,7 +21,6 @@ yansi = { workspace = true } serde_json = { version = "1.0.1", features = ["preserve_order"] } serde_yaml = "0.8.26" toml = "0.8.4" -protobuf-json-mapping = "3.3.0" protobuf-support = "3.3.0" xml2json-rs = "1.0.1" chrono = "0.4.31" diff --git a/yara-x-dump/src/lib.rs b/yara-x-dump/src/lib.rs index d208eaeba..be7234266 100644 --- a/yara-x-dump/src/lib.rs +++ b/yara-x-dump/src/lib.rs @@ -1,12 +1,9 @@ mod serializer; use protobuf::{reflect::MessageRef, MessageDyn}; -use protobuf_json_mapping::print_to_string; +use serializer::get_yaml; -use std::io; use thiserror::Error; -use crate::serializer::{get_human_readable_output, get_serializer}; - #[cfg(test)] mod tests; @@ -17,27 +14,6 @@ include!(concat!(env!("OUT_DIR"), "/protos/mod.rs")); #[derive(Error, Debug)] #[allow(clippy::large_enum_variant)] pub enum Error { - /// Error while reading from input. - #[error("Read error")] - ReadError(io::Error), - /// Error while serializing protobuf messages. - #[error("Serialization error")] - SerializationError(#[from] protobuf_json_mapping::PrintError), - /// Error while parsing JSON strings. - #[error("Parsing JSON error")] - ParsingJSONError(#[from] serde_json::Error), - /// Error while parsing YAML strings. - #[error("Parsing YAML error")] - ParsingYAMLError(#[from] serde_yaml::Error), - /// Error while parsing TOML strings. - #[error("Parsing TOML error")] - ParsingTOMLError(#[from] toml::ser::Error), - /// Error while parsing XML strings. - #[error("Parsing XML error")] - ParsingXMLError(#[from] xml2json_rs::X2JError), - /// Error for unsupported serilization formats. - #[error("Unsupported serilization format")] - UnsupportedFormat, /// Error while formatting output #[error("Formatting Error")] FormattingError(#[from] std::fmt::Error), @@ -53,53 +29,25 @@ impl Dumper { /// /// # Arguments /// - /// * `input`: The input to read from. - /// * `modules`: The list of modules to import. - /// * `output_format`: The desired output format. + /// * `mod_output` - The module output to be dumped. /// /// # Returns /// /// Returns a `Result<(), Error>` indicating whether the operation was /// successful or not. - pub fn dump( - &self, - mod_output: &dyn MessageDyn, - output_format: Option<&String>, - ) -> Result { + pub fn dump(&self, mod_output: &dyn MessageDyn) -> Result { // Iterate over the modules' outputs and get serialized results to // print. let mut serialized_result = String::new(); let mut is_first_line = false; - match output_format { - // Output is desired to be human-readable. - Some(format) if format == "human-readable" => { - get_human_readable_output( - &MessageRef::from(mod_output), - &mut serialized_result, - 0, - &mut is_first_line, - )?; - } - // Serialize output for other given formats. - Some(format) => { - let json_output = print_to_string(mod_output)?; - let serializer = get_serializer(format)?; - - serialized_result = - serializer.serialize(json_output.as_str())?; - } - // Default to human-readable output. - None => { - get_human_readable_output( - &MessageRef::from(mod_output), - &mut serialized_result, - 0, - &mut is_first_line, - )?; - } - } + get_yaml( + &MessageRef::from(mod_output), + &mut serialized_result, + 0, + &mut is_first_line, + )?; Ok(serialized_result) } diff --git a/yara-x-dump/src/serializer.rs b/yara-x-dump/src/serializer.rs index 83ad118da..918f697eb 100644 --- a/yara-x-dump/src/serializer.rs +++ b/yara-x-dump/src/serializer.rs @@ -6,24 +6,20 @@ use protobuf::reflect::ReflectValueRef; use protobuf_support::text_format::quote_bytes_to; use std::fmt::Write; use yansi::Color; +use yansi::Paint; use crate::dumper::exts::field_options; use crate::Error; -// A struct that represents serializers for different formats -struct JsonSerializer; -struct YamlSerializer; -struct TomlSerializer; -struct XmlSerializer; - // A struct that represents colors for output struct Colors; impl Colors { - const GREEN: Color = Color::RGB(51, 255, 153); - const BLUE: Color = Color::RGB(51, 51, 255); - const YELLOW: Color = Color::RGB(255, 255, 102); + const GREEN: Color = Color::Green; + const BLUE: Color = Color::Blue; + const YELLOW: Color = Color::Yellow; + const BROWN: Color = Color::RGB(222, 184, 135); } // A struct that represents options for a field values @@ -33,101 +29,17 @@ struct ValueOptions { is_timestamp: bool, } -/// A trait for any type that can serialize a message -pub(crate) trait Serializer { - /// Serialize a message - /// - /// # Arguments - /// - /// * `message`: The message to serialize - /// - /// # Returns - /// - /// Returns a `Result` where the `String` is the serialized - /// message in specified format and the `Error` is any error that occurred - /// during serialization - /// - /// # Errors - /// - /// * `Error::ParsingJSONError`: If the message is not a valid JSON - /// * `Error::ParsingYAMLError`: If the message is not a valid YAML - /// * `Error::ParsingTOMLError`: If the message is not a valid TOML - /// * `Error::ParsingXMLError`: If the message is not a valid XML - fn serialize(&self, message: &str) -> Result; -} - -/// Implement the trait for the JSON serializer -impl Serializer for JsonSerializer { - fn serialize(&self, message: &str) -> Result { - let value = serde_json::from_str::(message)?; - Ok(serde_json::to_string_pretty(&value)? + "\n") - } -} - -/// Implement the trait for the YAML serializer -impl Serializer for YamlSerializer { - fn serialize(&self, message: &str) -> Result { - let value = serde_json::from_str::(message)?; - Ok(serde_yaml::to_string(&value)?) - } -} - -/// Implement the trait for the TOML serializer -impl Serializer for TomlSerializer { - fn serialize(&self, message: &str) -> Result { - let value = serde_json::from_str::(message)?; - Ok(toml::to_string_pretty(&value)?) - } -} - -/// Implement the trait for the XML serializer -impl Serializer for XmlSerializer { - fn serialize(&self, message: &str) -> Result { - // Create a new XML builder and get the XML - let mut xml_builder = xml2json_rs::XmlConfig::new() - .rendering(xml2json_rs::Indentation::new(b' ', 2)) - .decl(xml2json_rs::Declaration::new( - xml2json_rs::Version::XML10, - Some(xml2json_rs::Encoding::UTF8), - Some(true), - )) - .root_name("file") - .finalize(); - let xml = xml_builder.build_from_json_string(message)? + "\n"; - Ok(xml) - } -} - -/// A function that returns a trait object based on the format -/// -/// # Arguments -/// -/// * `format`: The format to return the trait object for -/// -/// # Returns -/// -/// Returns a `Result, Error>` where the `Box` is the trait object for the specified format and the `Error` -/// is any error that occurred during the process -/// -/// # Errors -/// -/// * `Error::UnsupportedFormat`: If the format is unsupported -pub(crate) fn get_serializer( - format: &str, -) -> Result, Error> { - match format { - // Return a JSON serializer - "json" => Ok(Box::new(JsonSerializer)), - // Return a YAML serializer - "yaml" => Ok(Box::new(YamlSerializer)), - // Return a TOML serializer - "toml" => Ok(Box::new(TomlSerializer)), - // Return an XML serializer - "xml" => Ok(Box::new(XmlSerializer)), - // Return an error if the format is unsupported - _ => Err(Error::UnsupportedFormat), - } +// Write a value as a comment +// +// # Arguments +// +// * `value`: The value to write +// +// # Returns +// +// Returns a `String` which represents the value as a comment +fn write_as_a_comment(value: String) -> Paint { + Colors::BROWN.paint(format!("{} {}", "#", value)) } // Print a field name with correct indentation @@ -171,7 +83,7 @@ fn print_field_name( "{}{} {}: ", indentation, Colors::YELLOW.paint("-").bold(), - Colors::BLUE.paint(field_name) + Colors::BLUE.paint(field_name).bold() )?; *is_first_line = false; // If the field name is not the first line, print the indentation and @@ -181,7 +93,7 @@ fn print_field_name( buf, "{}{}: ", indentation, - Colors::BLUE.paint(field_name) + Colors::BLUE.paint(field_name).bold() )?; } } @@ -220,15 +132,20 @@ fn print_field_value( ReflectValueRef::Message(m) => { *is_first_line = true; // Recursively print the message - get_human_readable_output(&m, buf, indent + 1, is_first_line)?; + get_yaml(&m, buf, indent + 1, is_first_line)?; } ReflectValueRef::Enum(d, v) => match d.value_by_number(v) { Some(e) => writeln!(buf, "{}", e.name())?, None => writeln!(buf, "{}", v)?, }, ReflectValueRef::String(s) => { - quote_bytes_to(s.as_bytes(), buf); - buf.push('\n'); + writeln!( + buf, + "{}{}{}", + Colors::GREEN.paint("\""), + Colors::GREEN.paint(s), + Colors::GREEN.paint("\""), + )?; } ReflectValueRef::Bytes(b) => { quote_bytes_to(b, buf); @@ -237,17 +154,20 @@ fn print_field_value( ReflectValueRef::I32(v) => { // If the value has hex option turned on, print it in hex format let field_value = if value_options.is_hex { - format!("{} (0x{:x})", v, v) + format!("0x{:x}", v) // If the value has timestamp option turned on, print it in // timestamp format } else if value_options.is_timestamp { format!( - "{} ({})", + "{} {}", v, - DateTime::::from_naive_utc_and_offset( - NaiveDateTime::from_timestamp_opt(v as i64, 0) - .unwrap(), - Utc, + write_as_a_comment( + DateTime::::from_naive_utc_and_offset( + NaiveDateTime::from_timestamp_opt(v as i64, 0) + .unwrap(), + Utc, + ) + .to_string() ) ) // Otherwise, print it as a normal integer @@ -259,12 +179,12 @@ fn print_field_value( ReflectValueRef::I64(v) => { // If the value has hex option turned on, print it in hex format let field_value = if value_options.is_hex { - format!("{} (0x{:x})", v, v) + format!("0x{:x}", v) // If the value has timestamp option turned on, print it in // timestamp format } else if value_options.is_timestamp { format!( - "{} ({})", + "{} {}", v, DateTime::::from_naive_utc_and_offset( NaiveDateTime::from_timestamp_opt(v, 0).unwrap(), @@ -280,17 +200,20 @@ fn print_field_value( ReflectValueRef::U32(v) => { // If the value has hex option turned on, print it in hex format let field_value = if value_options.is_hex { - format!("{} (0x{:x})", v, v) + format!("0x{:x}", v) // If the value has timestamp option turned on, print it in // timestamp format } else if value_options.is_timestamp { format!( - "{} ({})", + "{} {}", v, - DateTime::::from_naive_utc_and_offset( - NaiveDateTime::from_timestamp_opt(v as i64, 0) - .unwrap(), - Utc, + write_as_a_comment( + DateTime::::from_naive_utc_and_offset( + NaiveDateTime::from_timestamp_opt(v as i64, 0) + .unwrap(), + Utc, + ) + .to_string() ) ) // Otherwise, print it as a normal integer @@ -302,17 +225,20 @@ fn print_field_value( ReflectValueRef::U64(v) => { // If the value has hex option turned on, print it in hex format let field_value = if value_options.is_hex { - format!("{} (0x{:x})", v, v) + format!("0x{:x}", v) // If the value has timestamp option turned on, print it in // timestamp format } else if value_options.is_timestamp { format!( - "{} ({})", + "{} {}", v, - DateTime::::from_naive_utc_and_offset( - NaiveDateTime::from_timestamp_opt(v as i64, 0) - .unwrap(), - Utc, + write_as_a_comment( + DateTime::::from_naive_utc_and_offset( + NaiveDateTime::from_timestamp_opt(v as i64, 0) + .unwrap(), + Utc, + ) + .to_string() ) ) // Otherwise, print it as a normal integer @@ -348,8 +274,8 @@ fn get_value_options(field_descriptor: &FieldDescriptorProto) -> ValueOptions { .get(&field_descriptor.options) .map(|options| ValueOptions { // Default for boolean is false - is_hex: options.hex_value.unwrap_or_default(), - is_timestamp: options.timestamp.unwrap_or_default(), + is_hex: options.yaml_fmt() == "x", + is_timestamp: options.yaml_fmt() == "t", }) .unwrap_or_default() } @@ -399,7 +325,7 @@ fn get_indentation(indent: usize) -> String { " ".repeat(indent) } -/// A function that returns a human-readable output +/// A function that returns a YAML output /// /// # Arguments /// @@ -413,7 +339,7 @@ fn get_indentation(indent: usize) -> String { /// /// Returns a `Result<(), Error>` where the `Error` is any error that occurred /// during the process -pub fn get_human_readable_output( +pub fn get_yaml( msg: &MessageRef, buf: &mut String, indent: usize, @@ -445,7 +371,7 @@ pub fn get_human_readable_output( buf, "{}{}:", get_indentation(indent + 1), - Colors::BLUE.paint(k) + Colors::BLUE.paint(k).bold() )?; } // Otherwise, print the field name @@ -454,7 +380,7 @@ pub fn get_human_readable_output( buf, "{}{}: ", get_indentation(indent + 1), - Colors::BLUE.paint(k) + Colors::BLUE.paint(k).bold() )?; } } @@ -477,11 +403,9 @@ pub fn get_human_readable_output( } writeln!( buf, - "{}{} {} {}", + "{}{}", get_indentation(indent), - Colors::GREEN.paint("# Nested").italic(), - Colors::GREEN.paint(f.name()).italic(), - Colors::GREEN.paint("structure").italic() + write_as_a_comment("Nested ".to_string() + f.name()), )?; writeln!( buf, @@ -531,11 +455,11 @@ pub fn get_human_readable_output( ReflectValueRef::Message(_) => { writeln!( buf, - "{}{} {} {}", + "{}{}", get_indentation(indent), - Colors::GREEN.paint("# Nested").italic(), - Colors::GREEN.paint(f.name()).italic(), - Colors::GREEN.paint("structure").italic() + write_as_a_comment( + "Nested ".to_string() + f.name() + ), )?; writeln!( buf, diff --git a/yara-x-dump/src/tests/mod.rs b/yara-x-dump/src/tests/mod.rs index e70ea2437..92e9d826f 100644 --- a/yara-x-dump/src/tests/mod.rs +++ b/yara-x-dump/src/tests/mod.rs @@ -1,21 +1,16 @@ use protobuf::text_format::parse_from_str; use protobuf::MessageDyn; -use rstest::rstest; use std::fs; use std::io::Write; +use yansi::Paint; use crate::Dumper; -#[rstest( - output_format, - case("json"), - case("yaml"), - case("toml"), - case("xml"), - case("human-readable"), - case("None") -)] -fn test_dumper(output_format: String) { +#[test] +fn test_dumper() { + // Disable colors for testing. + Paint::disable(); + // Create goldenfile mint. let mut mint = goldenfile::Mint::new("."); @@ -24,30 +19,18 @@ fn test_dumper(output_format: String) { println!("{:?}", entry); let in_path = entry.into_path(); - // Create a unique test name based on the combination of module and - // output_format. - let test_name = format!( - "{}.{}.out", - in_path.with_extension("").to_str().unwrap(), - output_format - ); + // Path to the .out file. + let out_path = in_path.with_extension("out"); + let input = fs::read_to_string(in_path).expect("Unable to read"); let test = parse_from_str::(&input).unwrap(); let dumper = Dumper::default(); - let output = if output_format == "None" { - dumper.dump(&test as &dyn MessageDyn, None) - } else { - dumper.dump( - &test as &dyn MessageDyn, - Some(&output_format.to_string()), - ) - } - .unwrap(); + let output = dumper.dump(&test as &dyn MessageDyn).unwrap(); // Create a goldenfile test - let mut output_file = mint.new_goldenfile(test_name).unwrap(); + let mut output_file = mint.new_goldenfile(out_path).unwrap(); write!(output_file, "{}", output).unwrap(); } diff --git a/yara-x-dump/src/tests/protos/dumper.proto b/yara-x-dump/src/tests/protos/dumper.proto index 38a6005dd..cf6f6b1b8 100644 --- a/yara-x-dump/src/tests/protos/dumper.proto +++ b/yara-x-dump/src/tests/protos/dumper.proto @@ -5,8 +5,7 @@ package dumper; import "google/protobuf/descriptor.proto"; message FieldOptions { - optional bool hex_value = 3; - optional bool timestamp = 4; + optional string yaml_fmt = 3; } extend google.protobuf.FieldOptions { diff --git a/yara-x-dump/src/tests/protos/test.proto b/yara-x-dump/src/tests/protos/test.proto index f5ff756a6..5c5f11b3b 100644 --- a/yara-x-dump/src/tests/protos/test.proto +++ b/yara-x-dump/src/tests/protos/test.proto @@ -6,18 +6,18 @@ package test; message Segment { optional uint32 nested1 = 1; - optional uint64 nested2 = 2 [(dumper.field_options).hex_value = true]; - optional uint32 timestamp = 3 [(dumper.field_options).timestamp = true]; + optional uint64 nested2 = 2 [(dumper.field_options).yaml_fmt = "x"]; + optional uint32 timestamp = 3 [(dumper.field_options).yaml_fmt = "t"]; } message OptionalNested { optional uint32 onested1 = 1; - optional uint64 onested2 = 2 [(dumper.field_options).hex_value = true]; + optional uint64 onested2 = 2 [(dumper.field_options).yaml_fmt = "x"]; map map_string_string = 3; } message MyMessage { - optional int32 field1 = 1 [(dumper.field_options).hex_value = true]; + optional int32 field1 = 1 [(dumper.field_options).yaml_fmt = "x"]; optional string field2 = 2; repeated Segment segments = 3; optional OptionalNested optional = 4; diff --git a/yara-x-dump/src/tests/testdata/macho_x86_file.None.out b/yara-x-dump/src/tests/testdata/macho_x86_file.None.out deleted file mode 100644 index 003d9a37e..000000000 --- a/yara-x-dump/src/tests/testdata/macho_x86_file.None.out +++ /dev/null @@ -1,13 +0,0 @@ -field1: 123 (0x7b) -field2: "test" -# Nested segments structure -segments: - - nested1: 456 - nested2: 789 (0x315) - timestamp: 123456789 (1973-11-29 21:33:09 UTC) -# Nested optional structure -optional: - - onested1: 123 - onested2: 456 (0x1c8) - map_string_string: - foo: "bar" diff --git a/yara-x-dump/src/tests/testdata/macho_x86_file.human-readable.out b/yara-x-dump/src/tests/testdata/macho_x86_file.human-readable.out deleted file mode 100644 index 003d9a37e..000000000 --- a/yara-x-dump/src/tests/testdata/macho_x86_file.human-readable.out +++ /dev/null @@ -1,13 +0,0 @@ -field1: 123 (0x7b) -field2: "test" -# Nested segments structure -segments: - - nested1: 456 - nested2: 789 (0x315) - timestamp: 123456789 (1973-11-29 21:33:09 UTC) -# Nested optional structure -optional: - - onested1: 123 - onested2: 456 (0x1c8) - map_string_string: - foo: "bar" diff --git a/yara-x-dump/src/tests/testdata/macho_x86_file.json.out b/yara-x-dump/src/tests/testdata/macho_x86_file.json.out deleted file mode 100644 index e6ab96391..000000000 --- a/yara-x-dump/src/tests/testdata/macho_x86_file.json.out +++ /dev/null @@ -1,18 +0,0 @@ -{ - "field1": 123, - "field2": "test", - "segments": [ - { - "nested1": 456, - "nested2": "789", - "timestamp": 123456789 - } - ], - "optional": { - "onested1": 123, - "onested2": "456", - "mapStringString": { - "foo": "bar" - } - } -} diff --git a/yara-x-dump/src/tests/testdata/macho_x86_file.out b/yara-x-dump/src/tests/testdata/macho_x86_file.out new file mode 100644 index 000000000..e0881e988 --- /dev/null +++ b/yara-x-dump/src/tests/testdata/macho_x86_file.out @@ -0,0 +1,13 @@ +field1: 0x7b +field2: "test" +# Nested segments +segments: + - nested1: 456 + nested2: 0x315 + timestamp: 123456789 # 1973-11-29 21:33:09 UTC +# Nested optional +optional: + - onested1: 123 + onested2: 0x1c8 + map_string_string: + foo: "bar" diff --git a/yara-x-dump/src/tests/testdata/macho_x86_file.toml.out b/yara-x-dump/src/tests/testdata/macho_x86_file.toml.out deleted file mode 100644 index 3c0dda9b2..000000000 --- a/yara-x-dump/src/tests/testdata/macho_x86_file.toml.out +++ /dev/null @@ -1,14 +0,0 @@ -field1 = 123 -field2 = "test" - -[[segments]] -nested1 = 456 -nested2 = "789" -timestamp = 123456789 - -[optional] -onested1 = 123 -onested2 = "456" - -[optional.mapStringString] -foo = "bar" diff --git a/yara-x-dump/src/tests/testdata/macho_x86_file.xml.out b/yara-x-dump/src/tests/testdata/macho_x86_file.xml.out deleted file mode 100644 index 733f3faba..000000000 --- a/yara-x-dump/src/tests/testdata/macho_x86_file.xml.out +++ /dev/null @@ -1,17 +0,0 @@ - - - 123 - test - - 456 - 789 - 123456789 - - - 123 - 456 - - bar - - - diff --git a/yara-x-dump/src/tests/testdata/macho_x86_file.yaml.out b/yara-x-dump/src/tests/testdata/macho_x86_file.yaml.out deleted file mode 100644 index 58237c145..000000000 --- a/yara-x-dump/src/tests/testdata/macho_x86_file.yaml.out +++ /dev/null @@ -1,12 +0,0 @@ ---- -field1: 123 -field2: test -segments: - - nested1: 456 - nested2: "789" - timestamp: 123456789 -optional: - onested1: 123 - onested2: "456" - mapStringString: - foo: bar diff --git a/yara-x-proto/src/yara.proto b/yara-x-proto/src/yara.proto index 0a32ba156..26be91ee9 100644 --- a/yara-x-proto/src/yara.proto +++ b/yara-x-proto/src/yara.proto @@ -17,8 +17,7 @@ message ModuleOptions { message FieldOptions { optional string name = 1; optional bool ignore = 2; - optional bool hex_value = 3; - optional bool timestamp = 4; + optional string yaml_fmt = 3; } message MessageOptions { diff --git a/yara-x/src/modules/protos/macho.proto b/yara-x/src/modules/protos/macho.proto index 449fa22ad..09c30755b 100644 --- a/yara-x/src/modules/protos/macho.proto +++ b/yara-x/src/modules/protos/macho.proto @@ -12,7 +12,7 @@ option (yara.module_options) = { message Dylib { optional string name = 1; - optional uint32 timestamp = 2 [(yara.field_options).timestamp = true]; + optional uint32 timestamp = 2 [(yara.field_options).yaml_fmt = "t"]; optional string compatibility_version = 3; optional string current_version = 4; } @@ -20,13 +20,13 @@ message Dylib { message Section { optional string segname = 1; optional string sectname = 2; - optional uint64 addr = 3 [(yara.field_options).hex_value = true]; - optional uint64 size = 4 [(yara.field_options).hex_value = true]; + optional uint64 addr = 3 [(yara.field_options).yaml_fmt = "x"]; + optional uint64 size = 4 [(yara.field_options).yaml_fmt = "x"]; optional uint32 offset = 5; optional uint32 align = 6; optional uint32 reloff = 7; optional uint32 nreloc = 8; - optional uint32 flags = 9 [(yara.field_options).hex_value = true]; + optional uint32 flags = 9 [(yara.field_options).yaml_fmt = "x"]; optional uint32 reserved1 = 10; optional uint32 reserved2 = 11; optional uint32 reserved3 = 12; @@ -36,14 +36,14 @@ message Segment { optional uint32 cmd = 1; optional uint32 cmdsize = 2; optional string segname = 3; - optional uint64 vmaddr = 4 [(yara.field_options).hex_value = true]; - optional uint64 vmsize = 5 [(yara.field_options).hex_value = true]; + optional uint64 vmaddr = 4 [(yara.field_options).yaml_fmt = "x"]; + optional uint64 vmsize = 5 [(yara.field_options).yaml_fmt = "x"]; optional uint64 fileoff = 6; optional uint64 filesize = 7; - optional uint32 maxprot = 8 [(yara.field_options).hex_value = true]; - optional uint32 initprot = 9 [(yara.field_options).hex_value = true]; + optional uint32 maxprot = 8 [(yara.field_options).yaml_fmt = "x"]; + optional uint32 initprot = 9 [(yara.field_options).yaml_fmt = "x"]; optional uint32 nsects = 10; - optional uint32 flags = 11 [(yara.field_options).hex_value = true]; + optional uint32 flags = 11 [(yara.field_options).yaml_fmt = "x"]; repeated Section sections = 12; } @@ -57,13 +57,13 @@ message FatArch { } message File { - optional uint32 magic = 1 [(yara.field_options).hex_value = true]; + optional uint32 magic = 1 [(yara.field_options).yaml_fmt = "x"]; optional uint32 cputype = 2; optional uint32 cpusubtype = 3; optional uint32 filetype = 4; optional uint32 ncmds = 5; optional uint32 sizeofcmds = 6; - optional uint32 flags = 7 [(yara.field_options).hex_value = true]; + optional uint32 flags = 7 [(yara.field_options).yaml_fmt = "x"]; optional uint32 reserved = 8; optional uint64 number_of_segments = 9; repeated Segment segments = 10; @@ -74,13 +74,13 @@ message File { message Macho { // Set Mach-O header and basic fields - optional uint32 magic = 1 [(yara.field_options).hex_value = true]; + optional uint32 magic = 1 [(yara.field_options).yaml_fmt = "x"]; optional uint32 cputype = 2; optional uint32 cpusubtype = 3; optional uint32 filetype = 4; optional uint32 ncmds = 5; optional uint32 sizeofcmds = 6; - optional uint32 flags = 7 [(yara.field_options).hex_value = true]; + optional uint32 flags = 7 [(yara.field_options).yaml_fmt = "x"]; optional uint32 reserved = 8; optional uint64 number_of_segments = 9; repeated Segment segments = 10; @@ -89,7 +89,7 @@ message Macho { optional uint64 stack_size = 13; // Add fields for Mach-O fat binary header - optional uint32 fat_magic = 14 [(yara.field_options).hex_value = true]; + optional uint32 fat_magic = 14 [(yara.field_options).yaml_fmt = "x"]; optional uint32 nfat_arch = 15; repeated FatArch fat_arch = 16; From ce664667a48c02a932f83c464347dd45e611a563 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20=C4=8Euri=C5=A1?= Date: Thu, 9 Nov 2023 15:39:13 +0100 Subject: [PATCH 23/26] fix clippy warnings --- Cargo.lock | 256 +------------------------------- yara-x-cli/src/commands/dump.rs | 8 +- yara-x-dump/Cargo.toml | 5 - 3 files changed, 6 insertions(+), 263 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 649434db7..9c6dcb472 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1230,101 +1230,6 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" -[[package]] -name = "futures" -version = "0.3.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da0290714b38af9b4a7b094b8a37086d1b4e61f2df9122c3cad2577669145335" -dependencies = [ - "futures-channel", - "futures-core", - "futures-executor", - "futures-io", - "futures-sink", - "futures-task", - "futures-util", -] - -[[package]] -name = "futures-channel" -version = "0.3.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff4dd66668b557604244583e3e1e1eada8c5c2e96a6d0d6653ede395b78bbacb" -dependencies = [ - "futures-core", - "futures-sink", -] - -[[package]] -name = "futures-core" -version = "0.3.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb1d22c66e66d9d72e1758f0bd7d4fd0bee04cad842ee34587d68c07e45d088c" - -[[package]] -name = "futures-executor" -version = "0.3.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f4fb8693db0cf099eadcca0efe2a5a22e4550f98ed16aba6c48700da29597bc" -dependencies = [ - "futures-core", - "futures-task", - "futures-util", -] - -[[package]] -name = "futures-io" -version = "0.3.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bf34a163b5c4c52d0478a4d757da8fb65cabef42ba90515efee0f6f9fa45aaa" - -[[package]] -name = "futures-macro" -version = "0.3.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.38", -] - -[[package]] -name = "futures-sink" -version = "0.3.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e36d3378ee38c2a36ad710c5d30c2911d752cb941c00c72dbabfb786a7970817" - -[[package]] -name = "futures-task" -version = "0.3.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efd193069b0ddadc69c46389b740bbccdd97203899b48d09c5f7969591d6bae2" - -[[package]] -name = "futures-timer" -version = "3.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c" - -[[package]] -name = "futures-util" -version = "0.3.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a19526d624e703a3179b3d322efec918b6246ea0fa51d41124525f00f1cc8104" -dependencies = [ - "futures-channel", - "futures-core", - "futures-io", - "futures-macro", - "futures-sink", - "futures-task", - "memchr", - "pin-project-lite", - "pin-utils", - "slab", -] - [[package]] name = "fxhash" version = "0.2.1" @@ -1644,7 +1549,7 @@ dependencies = [ "log", "num-format", "once_cell", - "quick-xml 0.26.0", + "quick-xml", "rgb", "str_stack", ] @@ -2375,18 +2280,6 @@ dependencies = [ "siphasher", ] -[[package]] -name = "pin-project-lite" -version = "0.2.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" - -[[package]] -name = "pin-utils" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" - [[package]] name = "pkg-config" version = "0.3.27" @@ -2619,15 +2512,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "quick-xml" -version = "0.23.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11bafc859c6815fbaffbbbf4229ecb767ac913fecb27f9ad4343662e9ef099ea" -dependencies = [ - "memchr", -] - [[package]] name = "quick-xml" version = "0.26.0" @@ -2791,12 +2675,6 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" -[[package]] -name = "relative-path" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c707298afce11da2efef2f600116fa93ffa7a032b5d7b628aa17711ec81383ca" - [[package]] name = "rgb" version = "0.8.37" @@ -2806,35 +2684,6 @@ dependencies = [ "bytemuck", ] -[[package]] -name = "rstest" -version = "0.18.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97eeab2f3c0a199bc4be135c36c924b6590b88c377d416494288c14f2db30199" -dependencies = [ - "futures", - "futures-timer", - "rstest_macros", - "rustc_version", -] - -[[package]] -name = "rstest_macros" -version = "0.18.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d428f8247852f894ee1be110b375111b586d4fa431f6c46e64ba5a0dcccbe605" -dependencies = [ - "cfg-if", - "glob", - "proc-macro2", - "quote", - "regex", - "relative-path", - "rustc_version", - "syn 2.0.38", - "unicode-ident", -] - [[package]] name = "rustc-demangle" version = "0.1.23" @@ -2847,15 +2696,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" -[[package]] -name = "rustc_version" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" -dependencies = [ - "semver 1.0.20", -] - [[package]] name = "rustix" version = "0.38.21" @@ -2960,33 +2800,11 @@ version = "1.0.108" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b" dependencies = [ - "indexmap 2.1.0", "itoa", "ryu", "serde", ] -[[package]] -name = "serde_spanned" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12022b835073e5b11e90a14f86838ceb1c8fb0325b72416845c487ac0fa95e80" -dependencies = [ - "serde", -] - -[[package]] -name = "serde_yaml" -version = "0.8.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "578a7433b776b56a35785ed5ce9a7e777ac0598aac5a6dd1b4b18a307c7fc71b" -dependencies = [ - "indexmap 1.9.3", - "ryu", - "serde", - "yaml-rust", -] - [[package]] name = "sha1" version = "0.10.6" @@ -3109,15 +2927,6 @@ dependencies = [ "walkdir", ] -[[package]] -name = "slab" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" -dependencies = [ - "autocfg", -] - [[package]] name = "slice-group-by" version = "0.3.1" @@ -3443,40 +3252,6 @@ dependencies = [ "serde", ] -[[package]] -name = "toml" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ef75d881185fd2df4a040793927c153d863651108a93c7e17a9e591baa95cc6" -dependencies = [ - "serde", - "serde_spanned", - "toml_datetime", - "toml_edit", -] - -[[package]] -name = "toml_datetime" -version = "0.6.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3550f4e9685620ac18a50ed434eb3aec30db8ba93b0287467bca5826ea25baf1" -dependencies = [ - "serde", -] - -[[package]] -name = "toml_edit" -version = "0.20.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "380f9e8120405471f7c9ad1860a713ef5ece6a670c7eae39225e477340f32fc4" -dependencies = [ - "indexmap 2.1.0", - "serde", - "serde_spanned", - "toml_datetime", - "winnow", -] - [[package]] name = "typenum" version = "1.17.0" @@ -3805,7 +3580,7 @@ dependencies = [ "rustix", "serde", "sha2 0.10.8", - "toml 0.5.11", + "toml", "windows-sys 0.48.0", "zstd", ] @@ -4301,15 +4076,6 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" -[[package]] -name = "winnow" -version = "0.5.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3b801d0e0a6726477cc207f60162da452f3a95adb368399bef20a946e06f65c" -dependencies = [ - "memchr", -] - [[package]] name = "wit-parser" version = "0.9.2" @@ -4335,19 +4101,6 @@ dependencies = [ "tap", ] -[[package]] -name = "xml2json-rs" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6ba4fb1256176d2eb02e25065efade8e43215a4d0743cb823991aea314077ea" -dependencies = [ - "lazy_static", - "quick-xml 0.23.1", - "regex", - "serde", - "serde_json", -] - [[package]] name = "yaml-rust" version = "0.4.5" @@ -4485,13 +4238,8 @@ dependencies = [ "protobuf-codegen", "protobuf-parse", "protobuf-support", - "rstest", - "serde_json", - "serde_yaml", "tempfile", "thiserror", - "toml 0.8.4", - "xml2json-rs", "yansi", ] diff --git a/yara-x-cli/src/commands/dump.rs b/yara-x-cli/src/commands/dump.rs index b15a38a75..4309b217b 100644 --- a/yara-x-cli/src/commands/dump.rs +++ b/yara-x-cli/src/commands/dump.rs @@ -19,8 +19,8 @@ use yara_x::get_builtin_modules_names; #[derive(Debug, Clone, ValueEnum)] enum OutputFormats { - JSON, - YAML, + Json, + Yaml, } /// Creates the `dump` command. @@ -160,7 +160,7 @@ pub fn exec_dump(args: &ArgMatches) -> anyhow::Result<()> { && (module_is_valid(mod_output) || modules.contains(&mod_name)) { match output_format { - Some(OutputFormats::JSON) => { + Some(OutputFormats::Json) => { writeln!( result, ">>>\n{}:\n{}\n<<<", @@ -168,7 +168,7 @@ pub fn exec_dump(args: &ArgMatches) -> anyhow::Result<()> { print_to_string(mod_output)?.to_colored_json_auto()? )?; } - Some(OutputFormats::YAML) | None => { + Some(OutputFormats::Yaml) | None => { let dumper = Dumper::default(); write!( result, diff --git a/yara-x-dump/Cargo.toml b/yara-x-dump/Cargo.toml index d07a10753..46b8a318c 100644 --- a/yara-x-dump/Cargo.toml +++ b/yara-x-dump/Cargo.toml @@ -18,18 +18,13 @@ thiserror = { workspace = true } protobuf = { workspace = true } yansi = { workspace = true } -serde_json = { version = "1.0.1", features = ["preserve_order"] } -serde_yaml = "0.8.26" -toml = "0.8.4" protobuf-support = "3.3.0" -xml2json-rs = "1.0.1" chrono = "0.4.31" [dev-dependencies] goldenfile = "1.5.2" globwalk = { workspace = true } tempfile = "3.8.1" -rstest = "0.18.2" [build-dependencies] From 59c6d3cb55defe0c508981d74be10c959d662925 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20=C4=8Euri=C5=A1?= Date: Fri, 10 Nov 2023 14:11:26 +0100 Subject: [PATCH 24/26] fix issues with yaml format --- yara-x-dump/Cargo.toml | 1 - yara-x-dump/src/serializer.rs | 79 ++++++++++--------- yara-x-dump/src/tests/protos/test.proto | 5 +- .../src/tests/testdata/macho_x86_file.in | 10 ++- .../src/tests/testdata/macho_x86_file.out | 10 ++- 5 files changed, 60 insertions(+), 45 deletions(-) diff --git a/yara-x-dump/Cargo.toml b/yara-x-dump/Cargo.toml index 46b8a318c..68eb34987 100644 --- a/yara-x-dump/Cargo.toml +++ b/yara-x-dump/Cargo.toml @@ -26,7 +26,6 @@ goldenfile = "1.5.2" globwalk = { workspace = true } tempfile = "3.8.1" - [build-dependencies] protobuf = { workspace = true } protobuf-codegen = { workspace = true } diff --git a/yara-x-dump/src/serializer.rs b/yara-x-dump/src/serializer.rs index 918f697eb..2a4b7da81 100644 --- a/yara-x-dump/src/serializer.rs +++ b/yara-x-dump/src/serializer.rs @@ -3,7 +3,7 @@ use protobuf::descriptor::FieldDescriptorProto; use protobuf::reflect::MessageRef; use protobuf::reflect::ReflectFieldRef; use protobuf::reflect::ReflectValueRef; -use protobuf_support::text_format::quote_bytes_to; +use protobuf_support::text_format::escape_bytes_to; use std::fmt::Write; use yansi::Color; use yansi::Paint; @@ -29,6 +29,25 @@ struct ValueOptions { is_timestamp: bool, } +// Quote bytes function takes from protobuf-support crate +// and modified to return a String instead of writing to a buffer +// to allow colorizing the output +// +// # Arguments +// +// * `bytes`: The bytes to quote +// +// # Returns +// +// Returns a `String` which represents the quoted bytes +pub fn quote_bytes(bytes: &[u8]) -> String { + let mut result = String::new(); + result.push('"'); + escape_bytes_to(bytes, &mut result); + result.push('"'); + result +} + // Write a value as a comment // // # Arguments @@ -67,24 +86,14 @@ fn print_field_name( indent: usize, is_first_line: &mut bool, ) -> Result<(), Error> { - let mut indentation = get_indentation(indent); + let indentation = get_indentation(indent); // If the field name is not empty, print it if !field_name.is_empty() { // If the field name is the first line, print the indentation with a // dash and the field name if *is_first_line { - if !indentation.is_empty() { - indentation.pop(); - indentation.pop(); - } - write!( - buf, - "{}{} {}: ", - indentation, - Colors::YELLOW.paint("-").bold(), - Colors::BLUE.paint(field_name).bold() - )?; + write!(buf, "{}: ", Colors::BLUE.paint(field_name).bold())?; *is_first_line = false; // If the field name is not the first line, print the indentation and // the field name @@ -141,15 +150,12 @@ fn print_field_value( ReflectValueRef::String(s) => { writeln!( buf, - "{}{}{}", - Colors::GREEN.paint("\""), - Colors::GREEN.paint(s), - Colors::GREEN.paint("\""), + "{}", + Colors::GREEN.paint(quote_bytes(s.as_bytes())) )?; } ReflectValueRef::Bytes(b) => { - quote_bytes_to(b, buf); - buf.push('\n'); + writeln!(buf, "{}", Colors::GREEN.paint(quote_bytes(b)))?; } ReflectValueRef::I32(v) => { // If the value has hex option turned on, print it in hex format @@ -339,7 +345,7 @@ fn get_indentation(indent: usize) -> String { /// /// Returns a `Result<(), Error>` where the `Error` is any error that occurred /// during the process -pub fn get_yaml( +pub(crate) fn get_yaml( msg: &MessageRef, buf: &mut String, indent: usize, @@ -371,7 +377,11 @@ pub fn get_yaml( buf, "{}{}:", get_indentation(indent + 1), - Colors::BLUE.paint(k).bold() + Colors::BLUE + .paint(quote_bytes( + k.to_string().as_bytes() + )) + .bold() )?; } // Otherwise, print the field name @@ -380,7 +390,11 @@ pub fn get_yaml( buf, "{}{}: ", get_indentation(indent + 1), - Colors::BLUE.paint(k).bold() + Colors::BLUE + .paint(quote_bytes( + k.to_string().as_bytes() + )) + .bold() )?; } } @@ -401,12 +415,6 @@ pub fn get_yaml( if repeated.is_empty() { continue; } - writeln!( - buf, - "{}{}", - get_indentation(indent), - write_as_a_comment("Nested ".to_string() + f.name()), - )?; writeln!( buf, "{}{}:", @@ -418,6 +426,12 @@ pub fn get_yaml( match v { // If the value is a message, print it recursively ReflectValueRef::Message(_) => { + write!( + buf, + "{} {} ", + get_indentation(indent), + Colors::YELLOW.paint("-").bold(), + )?; print_field( buf, "", @@ -453,20 +467,13 @@ pub fn get_yaml( match v { // If the value is a message, print it recursively ReflectValueRef::Message(_) => { - writeln!( - buf, - "{}{}", - get_indentation(indent), - write_as_a_comment( - "Nested ".to_string() + f.name() - ), - )?; writeln!( buf, "{}{}:", get_indentation(indent), Colors::YELLOW.paint(f.name()).bold() )?; + write!(buf, "test",)?; print_field( buf, "", diff --git a/yara-x-dump/src/tests/protos/test.proto b/yara-x-dump/src/tests/protos/test.proto index 5c5f11b3b..579019093 100644 --- a/yara-x-dump/src/tests/protos/test.proto +++ b/yara-x-dump/src/tests/protos/test.proto @@ -19,6 +19,7 @@ message OptionalNested { message MyMessage { optional int32 field1 = 1 [(dumper.field_options).yaml_fmt = "x"]; optional string field2 = 2; - repeated Segment segments = 3; - optional OptionalNested optional = 4; + required string field3 = 3; + repeated Segment segments = 4; + optional OptionalNested optional = 5; } diff --git a/yara-x-dump/src/tests/testdata/macho_x86_file.in b/yara-x-dump/src/tests/testdata/macho_x86_file.in index 3213164d3..b28131b41 100644 --- a/yara-x-dump/src/tests/testdata/macho_x86_file.in +++ b/yara-x-dump/src/tests/testdata/macho_x86_file.in @@ -1,15 +1,21 @@ field1: 123 field2: "test" +field3: "test\ntest" segments { nested1: 456 nested2: 789 timestamp: 123456789 } +segments { + nested1: 100000 + nested2: 200000 + timestamp: 999999999 +} optional { onested1: 123 onested2: 456 map_string_string { - key: "foo" - value: "bar" + key: "foo\nfoo" + value: "bar\nbar" } } diff --git a/yara-x-dump/src/tests/testdata/macho_x86_file.out b/yara-x-dump/src/tests/testdata/macho_x86_file.out index e0881e988..dc851edf1 100644 --- a/yara-x-dump/src/tests/testdata/macho_x86_file.out +++ b/yara-x-dump/src/tests/testdata/macho_x86_file.out @@ -1,13 +1,15 @@ field1: 0x7b field2: "test" -# Nested segments +field3: "test\ntest" segments: - nested1: 456 nested2: 0x315 timestamp: 123456789 # 1973-11-29 21:33:09 UTC -# Nested optional + - nested1: 100000 + nested2: 0x30d40 + timestamp: 999999999 # 2001-09-09 01:46:39 UTC optional: - - onested1: 123 +testonested1: 123 onested2: 0x1c8 map_string_string: - foo: "bar" + "foo\nfoo": "bar\nbar" From 2209400205d63a90de8134fa2620844dc2d044f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20=C4=8Euri=C5=A1?= Date: Fri, 10 Nov 2023 16:43:53 +0100 Subject: [PATCH 25/26] make use of new module API --- Cargo.lock | 17 +- Cargo.toml | 2 +- yara-x-cli/Cargo.toml | 1 + yara-x-cli/src/commands/dump.rs | 177 ++++++++++-------- yara-x-dump/src/serializer.rs | 1 - .../src/tests/testdata/macho_x86_file.out | 2 +- 6 files changed, 116 insertions(+), 84 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9c6dcb472..8e800c273 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1708,7 +1708,7 @@ dependencies = [ "serde-wasm-bindgen", "serde_json", "strum", - "strum_macros", + "strum_macros 0.24.3", "wasm-bindgen", ] @@ -2800,6 +2800,7 @@ version = "1.0.108" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b" dependencies = [ + "indexmap 2.1.0", "itoa", "ryu", "serde", @@ -3014,6 +3015,19 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "strum_macros" +version = "0.25.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23dc1fa9ac9c169a78ba62f0b841814b7abae11bdd047b9c58f893439e309ea0" +dependencies = [ + "heck 0.4.1", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.38", +] + [[package]] name = "superconsole" version = "0.2.0" @@ -4216,6 +4230,7 @@ dependencies = [ "protobuf", "protobuf-json-mapping", "serde_json", + "strum_macros 0.25.3", "superconsole", "wild", "yansi", diff --git a/Cargo.toml b/Cargo.toml index ca3bf5ad4..e4c4fa099 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -71,7 +71,7 @@ regex-automata = { git = "https://github.com/plusvic/regex.git", rev="423493d" } rustc-hash = "1.1.0" smallvec = "1.10.0" serde = "1.0.156" -serde_json = "1.0.107" +serde_json = { version = "1.0.108", features = ["preserve_order"] } thiserror = "1.0.40" uuid = "1.4.1" walrus = "0.20.1" diff --git a/yara-x-cli/Cargo.toml b/yara-x-cli/Cargo.toml index 256be9ebd..f36daca64 100644 --- a/yara-x-cli/Cargo.toml +++ b/yara-x-cli/Cargo.toml @@ -58,3 +58,4 @@ pprof = { version = "0.12.1", features = ["flamegraph"], optional=true } superconsole = "0.2.0" wild = "2.1.0" colored_json = "4.0.0" +strum_macros = "0.25" diff --git a/yara-x-cli/src/commands/dump.rs b/yara-x-cli/src/commands/dump.rs index 4309b217b..9c86c5bea 100644 --- a/yara-x-cli/src/commands/dump.rs +++ b/yara-x-cli/src/commands/dump.rs @@ -1,21 +1,27 @@ +use anyhow::Error; use clap::{ arg, value_parser, Arg, ArgAction, ArgMatches, Command, ValueEnum, }; use colored_json::ToColoredJson; -use protobuf::{reflect::ReflectValueRef::Bool, MessageDyn}; +use protobuf::MessageDyn; use protobuf_json_mapping::print_to_string; use std::fmt::Write; use std::fs::File; use std::io::stdin; use std::io::Read; use std::path::PathBuf; +use strum_macros::Display; use yansi::{Color::Cyan, Paint}; -use yara_x_proto::exts::module_options; use yara_x_dump::Dumper; -use yara_x::get_builtin_modules_names; +#[derive(Debug, Clone, ValueEnum, Display)] +enum SupportedModules { + Lnk, + Macho, + Elf, +} #[derive(Debug, Clone, ValueEnum)] enum OutputFormats { @@ -55,41 +61,49 @@ pub fn dump() -> Command { .help("Name of the module or comma-separated list of modules to be used for parsing") .required(false) .action(ArgAction::Append) - .value_parser(get_builtin_modules_names()), + .value_parser(value_parser!(SupportedModules)), ) } -// Checks if the module output is valid by checking the validity flag. +// Obtains information about a module by calling dumper crate. // // # Arguments // -// * `mod_output`: The module output to check. +// * `output_format`: The output format. +// * `module`: The module name. +// * `output`: The output protobuf structure to be dumped. +// * `result`: String where the result is stored. // // # Returns // -// * `true` if the module output is valid, `false` otherwise. -fn module_is_valid(mod_output: &dyn MessageDyn) -> bool { - // Get the module options. - if let Some(module_desc) = module_options - .get(&mod_output.descriptor_dyn().file_descriptor_proto().options) - { - // Get the field name which is considered as the validity flag. - if let Some(validity_flag_str) = module_desc.validity_flag.as_deref() { - // Get the validity flag value. - if let Some(field) = - mod_output.descriptor_dyn().field_by_name(validity_flag_str) - { - // Check if the validity flag is set. - // Validity flag is set if the value present and is not - // false. - if let Some(value) = field.get_singular(mod_output) { - return value != Bool(false); - } - } +// Returns a `Result<(), Error>` indicating whether the operation was +// successful or not. +fn obtain_module_info( + output_format: Option<&OutputFormats>, + module: &SupportedModules, + output: &dyn MessageDyn, + result: &mut String, +) -> Result<(), Error> { + match output_format { + Some(OutputFormats::Json) => { + writeln!( + result, + ">>>\n{}:\n{}\n<<<", + Cyan.paint(module).bold(), + print_to_string(&*output)?.to_colored_json_auto()? + )?; + } + Some(OutputFormats::Yaml) | None => { + let dumper = Dumper::default(); + write!( + result, + ">>>\n{}:\n{}<<<", + Cyan.paint(module).bold(), + dumper.dump(&*output)? + )?; } } - - false + Ok(()) } /// Executes the `dump` command. @@ -108,6 +122,7 @@ pub fn exec_dump(args: &ArgMatches) -> anyhow::Result<()> { let file = args.get_one::("FILE"); let output_format = args.get_one::("output-format"); + let modules = args.get_many::("modules"); let colors_flag = args.get_flag("color"); // Disable colors if the flag is not set. @@ -115,13 +130,6 @@ pub fn exec_dump(args: &ArgMatches) -> anyhow::Result<()> { Paint::disable(); } - // get vector of modules - let modules: Vec<&str> = args - .get_many::("modules") - .unwrap_or_default() - .map(|s| s.as_str()) - .collect(); - // Get the input. if let Some(file) = file { File::open(file.as_path())?.read_to_end(&mut buffer)? @@ -129,54 +137,63 @@ pub fn exec_dump(args: &ArgMatches) -> anyhow::Result<()> { stdin().read_to_end(&mut buffer)? }; - // Get the list of modules to import. - let import_modules = if !modules.is_empty() { - modules.clone() - } else { - yara_x::get_builtin_modules_names() - }; - - // Create a rule that imports all the built-in modules. - let import_statements = import_modules - .iter() - .map(|module_name| format!("import \"{}\"", module_name)) - .collect::>() - .join("\n"); - - // Create a dummy rule - let rule = - format!(r#"{} rule test {{ condition: false }}"#, import_statements); - - // Compile the rule. - let rules = yara_x::compile(rule.as_str()).unwrap(); - - let mut scanner = yara_x::Scanner::new(&rules); - - // Scan the buffer and get the results. - let scan_results = scanner.scan(&buffer).expect("scan should not fail"); - - for (mod_name, mod_output) in scan_results.module_outputs() { - if mod_output.compute_size_dyn() != 0 - && (module_is_valid(mod_output) || modules.contains(&mod_name)) - { - match output_format { - Some(OutputFormats::Json) => { - writeln!( - result, - ">>>\n{}:\n{}\n<<<", - Cyan.paint(mod_name).bold(), - print_to_string(mod_output)?.to_colored_json_auto()? - )?; + if modules.is_some() { + for module in modules.unwrap() { + if let Some(output) = match module { + SupportedModules::Lnk => { + yara_x::mods::invoke_mod_dyn::(&buffer) } - Some(OutputFormats::Yaml) | None => { - let dumper = Dumper::default(); - write!( - result, - ">>>\n{}:\n{}<<<", - Cyan.paint(mod_name).bold(), - dumper.dump(mod_output)? - )?; + SupportedModules::Macho => yara_x::mods::invoke_mod_dyn::< + yara_x::mods::Macho, + >(&buffer), + SupportedModules::Elf => { + yara_x::mods::invoke_mod_dyn::(&buffer) } + } { + obtain_module_info( + output_format, + module, + &*output, + &mut result, + )?; + } + } + } else { + // Module was not specified therefore we have to obtain ouput for every supported module and decide which is valid. + if let Some(lnk_output) = + yara_x::mods::invoke_mod::(&buffer) + { + if lnk_output.is_lnk() { + obtain_module_info( + output_format, + &SupportedModules::Lnk, + &*lnk_output, + &mut result, + )?; + } + } + if let Some(macho_output) = + yara_x::mods::invoke_mod::(&buffer) + { + if macho_output.has_magic() { + obtain_module_info( + output_format, + &SupportedModules::Macho, + &*macho_output, + &mut result, + )?; + } + } + if let Some(elf_output) = + yara_x::mods::invoke_mod::(&buffer) + { + if elf_output.has_type() { + obtain_module_info( + output_format, + &SupportedModules::Elf, + &*elf_output, + &mut result, + )?; } } } diff --git a/yara-x-dump/src/serializer.rs b/yara-x-dump/src/serializer.rs index 2a4b7da81..c21ad87b7 100644 --- a/yara-x-dump/src/serializer.rs +++ b/yara-x-dump/src/serializer.rs @@ -473,7 +473,6 @@ pub(crate) fn get_yaml( get_indentation(indent), Colors::YELLOW.paint(f.name()).bold() )?; - write!(buf, "test",)?; print_field( buf, "", diff --git a/yara-x-dump/src/tests/testdata/macho_x86_file.out b/yara-x-dump/src/tests/testdata/macho_x86_file.out index dc851edf1..b0502b30a 100644 --- a/yara-x-dump/src/tests/testdata/macho_x86_file.out +++ b/yara-x-dump/src/tests/testdata/macho_x86_file.out @@ -9,7 +9,7 @@ segments: nested2: 0x30d40 timestamp: 999999999 # 2001-09-09 01:46:39 UTC optional: -testonested1: 123 +onested1: 123 onested2: 0x1c8 map_string_string: "foo\nfoo": "bar\nbar" From 188e30a2dacc7fa074e3d59c8662221cb6c419d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20=C4=8Euri=C5=A1?= Date: Fri, 10 Nov 2023 17:05:30 +0100 Subject: [PATCH 26/26] remove validity check from yara-x and fix clippy warnings --- docs/Module Developer's Guide.md | 27 +++++++---------------- yara-x-cli/src/commands/dump.rs | 8 +++---- yara-x-dump/Cargo.toml | 2 +- yara-x-dump/src/tests/protos/dumper.proto | 2 +- yara-x-proto/src/yara.proto | 1 - yara-x/src/lib.rs | 2 -- yara-x/src/modules/mod.rs | 4 ---- yara-x/src/modules/protos/lnk.proto | 1 - yara-x/src/modules/protos/macho.proto | 1 - 9 files changed, 14 insertions(+), 34 deletions(-) diff --git a/docs/Module Developer's Guide.md b/docs/Module Developer's Guide.md index a16de22c7..a5c11968c 100644 --- a/docs/Module Developer's Guide.md +++ b/docs/Module Developer's Guide.md @@ -49,7 +49,6 @@ option (YARA.module_options) = { name : "text" root_message: "text.Text" rust_module: "text" - validity_flag: "num_lines" (optional) }; message Text { @@ -106,7 +105,6 @@ option (yara.module_options) = { name : "text" root_message: "text.Text" rust_module: "text" - validity_flag: "num_lines" }; ``` @@ -117,8 +115,8 @@ file, but one describing a module. In fact, you can put any `.proto` file in the files is describing a YARA module. Only files containing a `yara.module_options` section will define a module. -Options `name` and `root_message` are required, while `rust_module` and `validity_flag` -are optional. The `name` option defines the module's name. This is the name that will be +Options `name` and `root_message` are required, while `rust_module` is optional. +The `name` option defines the module's name. This is the name that will be used for importing the module in a YARA rule, in this case our module will be imported with `import "text"`. The `root_message` option indicates which is the module's root structure, it must contain the name of some structure (a.k.a message) defined @@ -126,19 +124,7 @@ in the `.proto` file. In our case the value for `root_message` is `"text.Text"` because we have defined our module's structure in a message named `Text`, which is under package `text`. In general the value in this field will have the form `package.Message`, except if the `package` statement is missing, in which case -it would be the name of the message alone (i.e: `Text`). The `validity_flag` field -is used by `yr dump` module in order to mark specific field as the one that -determines if the module is valid or not. If this field has a value after parsing and -this value is not `false` then the module is considered valid and module output will -be shown. Module without this flag will be simply skipped in `yr dump` output. -For example `lnk` module has a field `is_lnk` which is marked as validity flag by setting -`validity_flag: "is_lnk"`. Important thing is that this field has to be in root message -structure. If `is_lnk` is `false` then module output is considered invalid and will be -skipped in `yr dump` output. Every other value of `is_lnk` will be considered valid -and module output will be shown. This is due to some modules not having any kind of -`is_valid` field and we need to mark other field as validity flag. `Macho` module has a -field `magic` which is considered as validity flag field. If this field is set to some -value then module output will be shown as we can consider module parsing to be successful. +it would be the name of the message alone (i.e: `Text`). The `root_message` field is required because your `.proto` file can define multiple messages, and YARA needs to know which of them is considered the root @@ -162,9 +148,12 @@ explore the protobuf's [documentation](https://developers.google.com/protocol-bu One thing that can be done with integer fields is to represent them in some other way. This optional representation is shown in `yr dump` crate output. This crate provides -two output formats: JSON and YAML. Both can be shown in colored output via `-c|--color` option. The last mentioned also provides custom representation for integer numbers. Let's say +two output formats: JSON and YAML. Both can be shown in colored output via `-c|--color` option. +The last mentioned also provides custom representation for integer numbers. Let's say for some fields it makes sense to show them as hexadecimal numbers. This can be done by -adding `[(yara.field_options).yaml_fmt = ""];` descriptor to the field. Currently supported formats are: hexadecimal number and human-readable timestamp. For example: +adding `[(yara.field_options).yaml_fmt = ""];` descriptor to the field. +Currently supported formats are: hexadecimal number and human-readable timestamp. +For example: ``` message Macho { diff --git a/yara-x-cli/src/commands/dump.rs b/yara-x-cli/src/commands/dump.rs index 9c86c5bea..0dd6c78dc 100644 --- a/yara-x-cli/src/commands/dump.rs +++ b/yara-x-cli/src/commands/dump.rs @@ -90,7 +90,7 @@ fn obtain_module_info( result, ">>>\n{}:\n{}\n<<<", Cyan.paint(module).bold(), - print_to_string(&*output)?.to_colored_json_auto()? + print_to_string(output)?.to_colored_json_auto()? )?; } Some(OutputFormats::Yaml) | None => { @@ -99,7 +99,7 @@ fn obtain_module_info( result, ">>>\n{}:\n{}<<<", Cyan.paint(module).bold(), - dumper.dump(&*output)? + dumper.dump(output)? )?; } } @@ -137,8 +137,8 @@ pub fn exec_dump(args: &ArgMatches) -> anyhow::Result<()> { stdin().read_to_end(&mut buffer)? }; - if modules.is_some() { - for module in modules.unwrap() { + if let Some(modules) = modules { + for module in modules { if let Some(output) = match module { SupportedModules::Lnk => { yara_x::mods::invoke_mod_dyn::(&buffer) diff --git a/yara-x-dump/Cargo.toml b/yara-x-dump/Cargo.toml index 68eb34987..9056f2682 100644 --- a/yara-x-dump/Cargo.toml +++ b/yara-x-dump/Cargo.toml @@ -29,4 +29,4 @@ tempfile = "3.8.1" [build-dependencies] protobuf = { workspace = true } protobuf-codegen = { workspace = true } -protobuf-parse = { workspace = true } \ No newline at end of file +protobuf-parse = { workspace = true } diff --git a/yara-x-dump/src/tests/protos/dumper.proto b/yara-x-dump/src/tests/protos/dumper.proto index cf6f6b1b8..c4170cf1a 100644 --- a/yara-x-dump/src/tests/protos/dumper.proto +++ b/yara-x-dump/src/tests/protos/dumper.proto @@ -10,4 +10,4 @@ message FieldOptions { extend google.protobuf.FieldOptions { optional FieldOptions field_options = 51504; - } \ No newline at end of file + } diff --git a/yara-x-proto/src/yara.proto b/yara-x-proto/src/yara.proto index 26be91ee9..475e44eb2 100644 --- a/yara-x-proto/src/yara.proto +++ b/yara-x-proto/src/yara.proto @@ -11,7 +11,6 @@ message ModuleOptions { required string name = 1; required string root_message = 2; optional string rust_module = 3; - optional string validity_flag = 4; } message FieldOptions { diff --git a/yara-x/src/lib.rs b/yara-x/src/lib.rs index d36ce55e5..021ad2377 100644 --- a/yara-x/src/lib.rs +++ b/yara-x/src/lib.rs @@ -49,8 +49,6 @@ pub use compiler::Error; pub use compiler::Rules; pub use compiler::SerializationError; -pub use modules::get_builtin_modules_names; - pub use scanner::Match; pub use scanner::Matches; pub use scanner::MatchingRules; diff --git a/yara-x/src/modules/mod.rs b/yara-x/src/modules/mod.rs index a4ec43f01..ac42fc6c9 100644 --- a/yara-x/src/modules/mod.rs +++ b/yara-x/src/modules/mod.rs @@ -26,10 +26,6 @@ include!("modules.rs"); /// Type of module's main function. type MainFn = fn(&[u8]) -> Box; -pub fn get_builtin_modules_names() -> Vec<&'static str> { - BUILTIN_MODULES.keys().cloned().collect() -} - /// Describes a YARA module. pub(crate) struct Module { /// Pointer to the module's main function. diff --git a/yara-x/src/modules/protos/lnk.proto b/yara-x/src/modules/protos/lnk.proto index 807117deb..049d2f149 100644 --- a/yara-x/src/modules/protos/lnk.proto +++ b/yara-x/src/modules/protos/lnk.proto @@ -7,7 +7,6 @@ option (yara.module_options) = { name : "lnk" root_message: "lnk.Lnk" rust_module: "lnk" - validity_flag: "is_lnk" }; enum FileAttributes { diff --git a/yara-x/src/modules/protos/macho.proto b/yara-x/src/modules/protos/macho.proto index 09c30755b..6cd1c0678 100644 --- a/yara-x/src/modules/protos/macho.proto +++ b/yara-x/src/modules/protos/macho.proto @@ -7,7 +7,6 @@ option (yara.module_options) = { name : "macho" root_message: "macho.Macho" rust_module: "macho" - validity_flag: "magic" }; message Dylib {