diff --git a/Cargo.lock b/Cargo.lock index a567e0afa..8e800c273 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -65,6 +65,21 @@ dependencies = [ "alloc-no-stdlib", ] +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + [[package]] name = "anes" version = "0.1.6" @@ -426,6 +441,20 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "chrono" +version = "0.4.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "js-sys", + "num-traits", + "wasm-bindgen", + "windows-targets 0.48.5", +] + [[package]] name = "ciborium" version = "0.2.1" @@ -471,6 +500,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac495e00dcec98c83465d5ad66c5c4fabd652fd6686e7c6269b117e729a6f17b" dependencies = [ "clap_builder", + "clap_derive", ] [[package]] @@ -485,6 +515,18 @@ dependencies = [ "strsim", ] +[[package]] +name = "clap_derive" +version = "4.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf9804afaaf59a91e75b022a30fb7229a7901f60c755489cc61c9b423b836442" +dependencies = [ + "heck 0.4.1", + "proc-macro2", + "quote", + "syn 2.0.38", +] + [[package]] name = "clap_lex" version = "0.6.0" @@ -497,6 +539,18 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" +[[package]] +name = "colored_json" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66203a5372a3ffbef29a90b4f8ed98eae3be5662bc7922c7dee1cb57bd40f9a8" +dependencies = [ + "is-terminal", + "serde", + "serde_json", + "yansi", +] + [[package]] name = "compact_str" version = "0.7.1" @@ -551,6 +605,12 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "core-foundation-sys" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" + [[package]] name = "cpp_demangle" version = "0.3.5" @@ -1350,6 +1410,29 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" +[[package]] +name = "iana-time-zone" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8326b86b6cff230b97d0d312a6c40a60726df3332e721f72a1b035f451663b20" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + [[package]] name = "id-arena" version = "2.2.1" @@ -1576,9 +1659,9 @@ checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67" [[package]] name = "libc" -version = "0.2.149" +version = "0.2.150" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a08173bc88b7955d1b3145aa561539096c421ac8debde8cbc3612ec635fee29b" +checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c" [[package]] name = "libloading" @@ -1625,7 +1708,7 @@ dependencies = [ "serde-wasm-bindgen", "serde_json", "strum", - "strum_macros", + "strum_macros 0.24.3", "wasm-bindgen", ] @@ -2303,6 +2386,17 @@ dependencies = [ "thiserror", ] +[[package]] +name = "protobuf-json-mapping" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "523039a90666b229b5260fb91c20686ef309b9d1b1fc3cacb283a0895753ec44" +dependencies = [ + "protobuf", + "protobuf-support", + "thiserror", +] + [[package]] name = "protobuf-parse" version = "3.3.0" @@ -2706,6 +2800,7 @@ version = "1.0.108" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b" dependencies = [ + "indexmap 2.1.0", "itoa", "ryu", "serde", @@ -2920,6 +3015,19 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "strum_macros" +version = "0.25.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23dc1fa9ac9c169a78ba62f0b841814b7abae11bdd047b9c58f893439e309ea0" +dependencies = [ + "heck 0.4.1", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.38", +] + [[package]] name = "superconsole" version = "0.2.0" @@ -3826,6 +3934,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows-core" +version = "0.51.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1f8cf84f35d2db49a46868f947758c7a1138116f7fac3bc844f43ade1292e64" +dependencies = [ + "windows-targets 0.48.5", +] + [[package]] name = "windows-sys" version = "0.42.0" @@ -4101,6 +4218,7 @@ dependencies = [ "anyhow", "ascii_tree", "clap", + "colored_json", "crossbeam", "crossterm 0.27.0", "enable-ansi-support", @@ -4110,13 +4228,34 @@ dependencies = [ "log", "pprof", "protobuf", + "protobuf-json-mapping", "serde_json", + "strum_macros 0.25.3", "superconsole", "wild", "yansi", "yara-x", + "yara-x-dump", "yara-x-fmt", "yara-x-parser", + "yara-x-proto", +] + +[[package]] +name = "yara-x-dump" +version = "0.1.0" +dependencies = [ + "anyhow", + "chrono", + "globwalk", + "goldenfile", + "protobuf", + "protobuf-codegen", + "protobuf-parse", + "protobuf-support", + "tempfile", + "thiserror", + "yansi", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index f55841ba1..e4c4fa099 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,6 +16,7 @@ rust-version = "1.70.0" members = [ "yara-x", "yara-x-cli", + "yara-x-dump", "yara-x-fmt", "yara-x-macros", "yara-x-parser", @@ -70,7 +71,7 @@ regex-automata = { git = "https://github.com/plusvic/regex.git", rev="423493d" } rustc-hash = "1.1.0" smallvec = "1.10.0" serde = "1.0.156" -serde_json = "1.0.107" +serde_json = { version = "1.0.108", features = ["preserve_order"] } thiserror = "1.0.40" uuid = "1.4.1" walrus = "0.20.1" @@ -78,6 +79,7 @@ wasmtime = "12.0.2" yaml-rust = "0.4.5" yansi = "0.5.1" yara-x = { path = "yara-x" } +yara-x-dump = { path = "yara-x-dump" } yara-x-fmt = { path = "yara-x-fmt" } yara-x-macros = { path = "yara-x-macros" } yara-x-parser = { path = "yara-x-parser" } diff --git a/docs/Module Developer's Guide.md b/docs/Module Developer's Guide.md index 33af9ae42..a5c11968c 100644 --- a/docs/Module Developer's Guide.md +++ b/docs/Module Developer's Guide.md @@ -115,9 +115,9 @@ file, but one describing a module. In fact, you can put any `.proto` file in the files is describing a YARA module. Only files containing a `yara.module_options` section will define a module. -Options `name` and `root_message` are required, while `rust_module` is optional. -The `name` option defines the module's name. This is the name that will be used -for importing the module in a YARA rule, in this case our module will be imported +Options `name` and `root_message` are required, while `rust_module` is optional. +The `name` option defines the module's name. This is the name that will be +used for importing the module in a YARA rule, in this case our module will be imported with `import "text"`. The `root_message` option indicates which is the module's root structure, it must contain the name of some structure (a.k.a message) defined in the `.proto` file. In our case the value for `root_message` is `"text.Text"` @@ -144,7 +144,29 @@ This is a very simple structure with only two integer fields. Notice that the numbers after the `=` signs are not the values for those fields, they are actually field tags (i.e: a unique number identifying each field in a message). This may be confusing if you are not familiar with protobuf's syntax, so again: -explore the protobuf's [documentation](https://developers.google.com/protocol-buffers). +explore the protobuf's [documentation](https://developers.google.com/protocol-buffers). + +One thing that can be done with integer fields is to represent them in some other way. +This optional representation is shown in `yr dump` crate output. This crate provides +two output formats: JSON and YAML. Both can be shown in colored output via `-c|--color` option. +The last mentioned also provides custom representation for integer numbers. Let's say +for some fields it makes sense to show them as hexadecimal numbers. This can be done by +adding `[(yara.field_options).yaml_fmt = ""];` descriptor to the field. +Currently supported formats are: hexadecimal number and human-readable timestamp. +For example: + +``` +message Macho { + optional uint32 magic = 1 [(yara.field_options).yml_fmt= "x"]; +} +``` + +This will mark magic field as a hexadecimal number and it will be shown as +`magic: 0xfeedfacf` instead of `4277009103`. Other format that is supported right now is +for timestamps. If you want to show some integer field as a timestamp you can do it by +setting `[(yara.field_options).yml_fmt = "t"];` descriptor to the field and +human readable timestamps will be shown in YAML comment after its integer value. + Also notice that we are defining our fields as `optional`. In `proto2` fields must be either `optional` or `required`, while in `proto3` they are always diff --git a/yara-x-cli/Cargo.toml b/yara-x-cli/Cargo.toml index 3a0ed7fff..f36daca64 100644 --- a/yara-x-cli/Cargo.toml +++ b/yara-x-cli/Cargo.toml @@ -36,17 +36,20 @@ logging = ["dep:log", "dep:env_logger"] [dependencies] ascii_tree = { workspace = true } anyhow = { workspace = true } -clap = { workspace = true, features=["cargo"] } +clap = { workspace = true, features=["cargo", "derive"] } globwalk = { workspace = true } enable-ansi-support = { workspace = true } env_logger = { workspace = true , optional = true } log = { workspace = true, optional = true } protobuf = { workspace = true } +protobuf-json-mapping = "3.3.0" serde_json = { workspace = true } yansi = { workspace = true } yara-x = { workspace = true } +yara-x-dump = { workspace = true } yara-x-parser = { workspace = true, features = ["ascii-tree"] } yara-x-fmt = { workspace = true } +yara-x-proto = { workspace = true } crossbeam = "0.8.2" crossterm = "0.27.0" @@ -54,3 +57,5 @@ indent = "0.1.1" pprof = { version = "0.12.1", features = ["flamegraph"], optional=true } superconsole = "0.2.0" wild = "2.1.0" +colored_json = "4.0.0" +strum_macros = "0.25" diff --git a/yara-x-cli/src/commands/dump.rs b/yara-x-cli/src/commands/dump.rs new file mode 100644 index 000000000..0dd6c78dc --- /dev/null +++ b/yara-x-cli/src/commands/dump.rs @@ -0,0 +1,203 @@ +use anyhow::Error; +use clap::{ + arg, value_parser, Arg, ArgAction, ArgMatches, Command, ValueEnum, +}; + +use colored_json::ToColoredJson; +use protobuf::MessageDyn; +use protobuf_json_mapping::print_to_string; +use std::fmt::Write; +use std::fs::File; +use std::io::stdin; +use std::io::Read; +use std::path::PathBuf; +use strum_macros::Display; +use yansi::{Color::Cyan, Paint}; + +use yara_x_dump::Dumper; + +#[derive(Debug, Clone, ValueEnum, Display)] +enum SupportedModules { + Lnk, + Macho, + Elf, +} + +#[derive(Debug, Clone, ValueEnum)] +enum OutputFormats { + Json, + Yaml, +} + +/// Creates the `dump` command. +/// The `dump` command dumps information about binary files. +/// +/// # Returns +/// +/// Returns a `Command` struct that represents the `dump` command. +pub fn dump() -> Command { + super::command("dump") + .about("Dump information about binary files") + .arg( + arg!() + .help("Path to binary file") + .value_parser(value_parser!(PathBuf)) + .required(false), + ) + .arg( + arg!(-o --"output-format" ) + .help("Desired output format") + .value_parser(value_parser!(OutputFormats)) + .required(false), + ) + .arg( + arg!(-c - -"color") + .help("Use colorful output") + ) + .arg( + Arg::new("modules") + .long("modules") + .short('m') + .help("Name of the module or comma-separated list of modules to be used for parsing") + .required(false) + .action(ArgAction::Append) + .value_parser(value_parser!(SupportedModules)), + ) +} + +// Obtains information about a module by calling dumper crate. +// +// # Arguments +// +// * `output_format`: The output format. +// * `module`: The module name. +// * `output`: The output protobuf structure to be dumped. +// * `result`: String where the result is stored. +// +// # Returns +// +// Returns a `Result<(), Error>` indicating whether the operation was +// successful or not. +fn obtain_module_info( + output_format: Option<&OutputFormats>, + module: &SupportedModules, + output: &dyn MessageDyn, + result: &mut String, +) -> Result<(), Error> { + match output_format { + Some(OutputFormats::Json) => { + writeln!( + result, + ">>>\n{}:\n{}\n<<<", + Cyan.paint(module).bold(), + print_to_string(output)?.to_colored_json_auto()? + )?; + } + Some(OutputFormats::Yaml) | None => { + let dumper = Dumper::default(); + write!( + result, + ">>>\n{}:\n{}<<<", + Cyan.paint(module).bold(), + dumper.dump(output)? + )?; + } + } + Ok(()) +} + +/// Executes the `dump` command. +/// +/// # Arguments +/// +/// * `args`: The arguments passed to the `dump` command. +/// +/// # Returns +/// +/// Returns a `Result<(), anyhow::Error>` indicating whether the operation was +/// successful or not. +pub fn exec_dump(args: &ArgMatches) -> anyhow::Result<()> { + let mut buffer = Vec::new(); + let mut result = String::new(); + + let file = args.get_one::("FILE"); + let output_format = args.get_one::("output-format"); + let modules = args.get_many::("modules"); + let colors_flag = args.get_flag("color"); + + // Disable colors if the flag is not set. + if !colors_flag { + Paint::disable(); + } + + // Get the input. + if let Some(file) = file { + File::open(file.as_path())?.read_to_end(&mut buffer)? + } else { + stdin().read_to_end(&mut buffer)? + }; + + if let Some(modules) = modules { + for module in modules { + if let Some(output) = match module { + SupportedModules::Lnk => { + yara_x::mods::invoke_mod_dyn::(&buffer) + } + SupportedModules::Macho => yara_x::mods::invoke_mod_dyn::< + yara_x::mods::Macho, + >(&buffer), + SupportedModules::Elf => { + yara_x::mods::invoke_mod_dyn::(&buffer) + } + } { + obtain_module_info( + output_format, + module, + &*output, + &mut result, + )?; + } + } + } else { + // Module was not specified therefore we have to obtain ouput for every supported module and decide which is valid. + if let Some(lnk_output) = + yara_x::mods::invoke_mod::(&buffer) + { + if lnk_output.is_lnk() { + obtain_module_info( + output_format, + &SupportedModules::Lnk, + &*lnk_output, + &mut result, + )?; + } + } + if let Some(macho_output) = + yara_x::mods::invoke_mod::(&buffer) + { + if macho_output.has_magic() { + obtain_module_info( + output_format, + &SupportedModules::Macho, + &*macho_output, + &mut result, + )?; + } + } + if let Some(elf_output) = + yara_x::mods::invoke_mod::(&buffer) + { + if elf_output.has_type() { + obtain_module_info( + output_format, + &SupportedModules::Elf, + &*elf_output, + &mut result, + )?; + } + } + } + + println!("{}", result); + Ok(()) +} diff --git a/yara-x-cli/src/commands/mod.rs b/yara-x-cli/src/commands/mod.rs index f6370db38..269345bd7 100644 --- a/yara-x-cli/src/commands/mod.rs +++ b/yara-x-cli/src/commands/mod.rs @@ -1,12 +1,14 @@ mod check; mod compile; mod debug; +mod dump; mod fmt; mod scan; pub use check::*; pub use compile::*; pub use debug::*; +pub use dump::*; pub use fmt::*; pub use scan::*; diff --git a/yara-x-cli/src/main.rs b/yara-x-cli/src/main.rs index eb12f0f59..fe25b6ab2 100644 --- a/yara-x-cli/src/main.rs +++ b/yara-x-cli/src/main.rs @@ -43,6 +43,7 @@ fn main() -> anyhow::Result<()> { commands::check(), commands::debug(), commands::fmt(), + commands::dump(), ]) .get_matches_from(wild::args()); @@ -58,6 +59,7 @@ fn main() -> anyhow::Result<()> { Some(("debug", args)) => commands::exec_debug(args), Some(("check", args)) => commands::exec_check(args), Some(("fmt", args)) => commands::exec_fmt(args), + Some(("dump", args)) => commands::exec_dump(args), Some(("scan", args)) => commands::exec_scan(args), Some(("compile", args)) => commands::exec_compile(args), _ => unreachable!(), diff --git a/yara-x-dump/Cargo.toml b/yara-x-dump/Cargo.toml new file mode 100644 index 000000000..9056f2682 --- /dev/null +++ b/yara-x-dump/Cargo.toml @@ -0,0 +1,32 @@ +[package] +name = "yara-x-dump" +version = "0.1.0" +description = "A YARA module for dumping file information" +authors = ["Tomas Duris "] +edition = "2021" +readme.workspace = true +license.workspace = true +homepage.workspace = true +rust-version.workspace = true + +[lib] +crate-type = ["rlib", "cdylib"] + +[dependencies] +anyhow = { workspace = true } +thiserror = { workspace = true } +protobuf = { workspace = true } +yansi = { workspace = true } + +protobuf-support = "3.3.0" +chrono = "0.4.31" + +[dev-dependencies] +goldenfile = "1.5.2" +globwalk = { workspace = true } +tempfile = "3.8.1" + +[build-dependencies] +protobuf = { workspace = true } +protobuf-codegen = { workspace = true } +protobuf-parse = { workspace = true } diff --git a/yara-x-dump/build.rs b/yara-x-dump/build.rs new file mode 100644 index 000000000..8b9f47942 --- /dev/null +++ b/yara-x-dump/build.rs @@ -0,0 +1,11 @@ +use protobuf_codegen::Codegen; + +fn main() { + Codegen::new() + .pure() + .cargo_out_dir("protos") + .include("src/tests/protos") + .input("src/tests/protos/dumper.proto") + .input("src/tests/protos/test.proto") + .run_from_script(); +} diff --git a/yara-x-dump/src/lib.rs b/yara-x-dump/src/lib.rs new file mode 100644 index 000000000..be7234266 --- /dev/null +++ b/yara-x-dump/src/lib.rs @@ -0,0 +1,54 @@ +mod serializer; +use protobuf::{reflect::MessageRef, MessageDyn}; +use serializer::get_yaml; + +use thiserror::Error; + +#[cfg(test)] +mod tests; + +pub use test::*; +include!(concat!(env!("OUT_DIR"), "/protos/mod.rs")); + +/// Errors returned by [`Dumper::dump`]. +#[derive(Error, Debug)] +#[allow(clippy::large_enum_variant)] +pub enum Error { + /// Error while formatting output + #[error("Formatting Error")] + FormattingError(#[from] std::fmt::Error), +} + +/// Dumps information about binary files. +#[derive(Debug, Default, Clone)] +pub struct Dumper {} + +// Dumper public API. +impl Dumper { + /// Dumps information about the binary file. + /// + /// # Arguments + /// + /// * `mod_output` - The module output to be dumped. + /// + /// # Returns + /// + /// Returns a `Result<(), Error>` indicating whether the operation was + /// successful or not. + pub fn dump(&self, mod_output: &dyn MessageDyn) -> Result { + // Iterate over the modules' outputs and get serialized results to + // print. + + let mut serialized_result = String::new(); + let mut is_first_line = false; + + get_yaml( + &MessageRef::from(mod_output), + &mut serialized_result, + 0, + &mut is_first_line, + )?; + + Ok(serialized_result) + } +} diff --git a/yara-x-dump/src/serializer.rs b/yara-x-dump/src/serializer.rs new file mode 100644 index 000000000..c21ad87b7 --- /dev/null +++ b/yara-x-dump/src/serializer.rs @@ -0,0 +1,503 @@ +use chrono::prelude::{DateTime, NaiveDateTime, Utc}; +use protobuf::descriptor::FieldDescriptorProto; +use protobuf::reflect::MessageRef; +use protobuf::reflect::ReflectFieldRef; +use protobuf::reflect::ReflectValueRef; +use protobuf_support::text_format::escape_bytes_to; +use std::fmt::Write; +use yansi::Color; +use yansi::Paint; + +use crate::dumper::exts::field_options; + +use crate::Error; + +// A struct that represents colors for output +struct Colors; + +impl Colors { + const GREEN: Color = Color::Green; + const BLUE: Color = Color::Blue; + const YELLOW: Color = Color::Yellow; + const BROWN: Color = Color::RGB(222, 184, 135); +} + +// A struct that represents options for a field values +#[derive(Debug, Default, Clone)] +struct ValueOptions { + is_hex: bool, + is_timestamp: bool, +} + +// Quote bytes function takes from protobuf-support crate +// and modified to return a String instead of writing to a buffer +// to allow colorizing the output +// +// # Arguments +// +// * `bytes`: The bytes to quote +// +// # Returns +// +// Returns a `String` which represents the quoted bytes +pub fn quote_bytes(bytes: &[u8]) -> String { + let mut result = String::new(); + result.push('"'); + escape_bytes_to(bytes, &mut result); + result.push('"'); + result +} + +// Write a value as a comment +// +// # Arguments +// +// * `value`: The value to write +// +// # Returns +// +// Returns a `String` which represents the value as a comment +fn write_as_a_comment(value: String) -> Paint { + Colors::BROWN.paint(format!("{} {}", "#", value)) +} + +// Print a field name with correct indentation +// +// # Arguments +// +// * `buf`: The buffer to write the field name to +// * `field_name`: The field name to write +// * `indent`: The indentation level +// * `is_first_line`: A boolean that indicates if the field name is the first +// line +// +// # Returns +// +// Returns a `Result<(), Error>` where the `Error` is any error that occurred +// during the process +// +// # Errors +// +// * `Error::FormattingError`: If the field name could not be written to the +// buffer +fn print_field_name( + buf: &mut String, + field_name: &str, + indent: usize, + is_first_line: &mut bool, +) -> Result<(), Error> { + let indentation = get_indentation(indent); + + // If the field name is not empty, print it + if !field_name.is_empty() { + // If the field name is the first line, print the indentation with a + // dash and the field name + if *is_first_line { + write!(buf, "{}: ", Colors::BLUE.paint(field_name).bold())?; + *is_first_line = false; + // If the field name is not the first line, print the indentation and + // the field name + } else { + write!( + buf, + "{}{}: ", + indentation, + Colors::BLUE.paint(field_name).bold() + )?; + } + } + Ok(()) +} + +// Print a field value with correct indentation for multiple value formats +// +// # Arguments +// +// * `buf`: The buffer to write the field value to +// * `value`: The field value to write +// * `value_options`: The value options for the field value +// * `indent`: The indentation level +// * `is_first_line`: A boolean that indicates if the field value is the first +// line +// +// # Returns +// +// Returns a `Result<(), Error>` where the `Error` is any error that occurred +// during the process +// +// # Errors +// +// * `Error::FormattingError`: If the field value could not be written to the +// buffer +fn print_field_value( + buf: &mut String, + value: ReflectValueRef, + value_options: &ValueOptions, + indent: usize, + is_first_line: &mut bool, +) -> Result<(), Error> { + // Match the field value type and print it in desired format + match value { + ReflectValueRef::Message(m) => { + *is_first_line = true; + // Recursively print the message + get_yaml(&m, buf, indent + 1, is_first_line)?; + } + ReflectValueRef::Enum(d, v) => match d.value_by_number(v) { + Some(e) => writeln!(buf, "{}", e.name())?, + None => writeln!(buf, "{}", v)?, + }, + ReflectValueRef::String(s) => { + writeln!( + buf, + "{}", + Colors::GREEN.paint(quote_bytes(s.as_bytes())) + )?; + } + ReflectValueRef::Bytes(b) => { + writeln!(buf, "{}", Colors::GREEN.paint(quote_bytes(b)))?; + } + ReflectValueRef::I32(v) => { + // If the value has hex option turned on, print it in hex format + let field_value = if value_options.is_hex { + format!("0x{:x}", v) + // If the value has timestamp option turned on, print it in + // timestamp format + } else if value_options.is_timestamp { + format!( + "{} {}", + v, + write_as_a_comment( + DateTime::::from_naive_utc_and_offset( + NaiveDateTime::from_timestamp_opt(v as i64, 0) + .unwrap(), + Utc, + ) + .to_string() + ) + ) + // Otherwise, print it as a normal integer + } else { + v.to_string() + }; + writeln!(buf, "{}", field_value)?; + } + ReflectValueRef::I64(v) => { + // If the value has hex option turned on, print it in hex format + let field_value = if value_options.is_hex { + format!("0x{:x}", v) + // If the value has timestamp option turned on, print it in + // timestamp format + } else if value_options.is_timestamp { + format!( + "{} {}", + v, + DateTime::::from_naive_utc_and_offset( + NaiveDateTime::from_timestamp_opt(v, 0).unwrap(), + Utc, + ) + ) + // Otherwise, print it as a normal integer + } else { + v.to_string() + }; + writeln!(buf, "{}", field_value)?; + } + ReflectValueRef::U32(v) => { + // If the value has hex option turned on, print it in hex format + let field_value = if value_options.is_hex { + format!("0x{:x}", v) + // If the value has timestamp option turned on, print it in + // timestamp format + } else if value_options.is_timestamp { + format!( + "{} {}", + v, + write_as_a_comment( + DateTime::::from_naive_utc_and_offset( + NaiveDateTime::from_timestamp_opt(v as i64, 0) + .unwrap(), + Utc, + ) + .to_string() + ) + ) + // Otherwise, print it as a normal integer + } else { + v.to_string() + }; + writeln!(buf, "{}", field_value)?; + } + ReflectValueRef::U64(v) => { + // If the value has hex option turned on, print it in hex format + let field_value = if value_options.is_hex { + format!("0x{:x}", v) + // If the value has timestamp option turned on, print it in + // timestamp format + } else if value_options.is_timestamp { + format!( + "{} {}", + v, + write_as_a_comment( + DateTime::::from_naive_utc_and_offset( + NaiveDateTime::from_timestamp_opt(v as i64, 0) + .unwrap(), + Utc, + ) + .to_string() + ) + ) + // Otherwise, print it as a normal integer + } else { + v.to_string() + }; + writeln!(buf, "{}", field_value)?; + } + ReflectValueRef::Bool(v) => { + writeln!(buf, "{}", v)?; + } + ReflectValueRef::F32(v) => { + writeln!(buf, "{:.1}", v)?; + } + ReflectValueRef::F64(v) => { + writeln!(buf, "{:.1}", v)?; + } + } + Ok(()) +} + +// Get the value options for a field +// +// # Arguments +// +// * `field_descriptor`: The field descriptor to get the value options for +// +// # Returns +// +// Returns a `ValueOptions` which is the value options for the field +fn get_value_options(field_descriptor: &FieldDescriptorProto) -> ValueOptions { + field_options + .get(&field_descriptor.options) + .map(|options| ValueOptions { + // Default for boolean is false + is_hex: options.yaml_fmt() == "x", + is_timestamp: options.yaml_fmt() == "t", + }) + .unwrap_or_default() +} + +// Print a field name and value +// +// # Arguments +// +// * `buf`: The buffer to write the field name and value to +// * `field_name`: The field name to write +// * `value`: The field value to write +// * `field_descriptor`: The field descriptor to get the value options for +// * `indent`: The indentation level +// * `is_first_line`: A boolean that indicates if the field name and value is +// the first line +// +// # Returns +// +// Returns a `Result<(), Error>` where the `Error` is any error that occurred +// during the process +// +fn print_field( + buf: &mut String, + field_name: &str, + value: ReflectValueRef, + field_descriptor: &FieldDescriptorProto, + indent: usize, + is_first_line: &mut bool, +) -> Result<(), Error> { + let value_options = get_value_options(field_descriptor); + + print_field_name(buf, field_name, indent, is_first_line)?; + print_field_value(buf, value, &value_options, indent, is_first_line)?; + Ok(()) +} + +// Get indentation level +// +// # Arguments +// +// * `indent`: The indentation level +// +// # Returns +// +// Returns a `String` which represents the indentation level +fn get_indentation(indent: usize) -> String { + " ".repeat(indent) +} + +/// A function that returns a YAML output +/// +/// # Arguments +/// +/// * `msg`: The message to get the human-readable output for +/// * `buf`: The buffer to write the human-readable output to +/// * `indent`: The indentation level +/// * `first_line`: A boolean that indicates if the field name and value is +/// the first line +/// +/// # Returns +/// +/// Returns a `Result<(), Error>` where the `Error` is any error that occurred +/// during the process +pub(crate) fn get_yaml( + msg: &MessageRef, + buf: &mut String, + indent: usize, + first_line: &mut bool, +) -> Result<(), Error> { + let desc = msg.descriptor_dyn(); + + // Iterate over the fields of the message + for f in desc.fields() { + // Match the field type + match f.get_reflect(&**msg) { + // If the field is a message, print it recursively + ReflectFieldRef::Map(map) => { + if map.is_empty() { + continue; + } + writeln!( + buf, + "{}{}:", + get_indentation(indent), + Colors::YELLOW.paint(f.name()).bold() + )?; + // Iterate over the map + for (k, v) in &map { + match v { + // If the value is a message, print it recursively + ReflectValueRef::Message(_) => { + writeln!( + buf, + "{}{}:", + get_indentation(indent + 1), + Colors::BLUE + .paint(quote_bytes( + k.to_string().as_bytes() + )) + .bold() + )?; + } + // Otherwise, print the field name + _ => { + write!( + buf, + "{}{}: ", + get_indentation(indent + 1), + Colors::BLUE + .paint(quote_bytes( + k.to_string().as_bytes() + )) + .bold() + )?; + } + } + // Print the field value + print_field( + buf, + "", + v, + f.proto(), + indent + 1, + first_line, + )?; + } + } + // If the field is a repeated field, print nested structure without + // repeating the field name + ReflectFieldRef::Repeated(repeated) => { + if repeated.is_empty() { + continue; + } + writeln!( + buf, + "{}{}:", + get_indentation(indent), + Colors::YELLOW.paint(f.name()).bold() + )?; + // Iterate over the repeated field + for v in repeated { + match v { + // If the value is a message, print it recursively + ReflectValueRef::Message(_) => { + write!( + buf, + "{} {} ", + get_indentation(indent), + Colors::YELLOW.paint("-").bold(), + )?; + print_field( + buf, + "", + v, + f.proto(), + indent, + first_line, + )?; + } + // Otherwise, print the field value + _ => { + write!( + buf, + "{} {} ", + get_indentation(indent), + Colors::YELLOW.paint("-").bold(), + )?; + print_field( + buf, + "", + v, + f.proto(), + indent, + first_line, + )?; + } + } + } + } + // If the field is a singular field, print it + ReflectFieldRef::Optional(optional) => { + if let Some(v) = optional.value() { + match v { + // If the value is a message, print it recursively + ReflectValueRef::Message(_) => { + writeln!( + buf, + "{}{}:", + get_indentation(indent), + Colors::YELLOW.paint(f.name()).bold() + )?; + print_field( + buf, + "", + v, + f.proto(), + indent, + first_line, + )?; + } + // Otherwise, print the field value + _ => { + print_field( + buf, + f.name(), + v, + f.proto(), + indent, + first_line, + )?; + } + } + } + } + } + } + + Ok(()) +} diff --git a/yara-x-dump/src/tests/mod.rs b/yara-x-dump/src/tests/mod.rs new file mode 100644 index 000000000..92e9d826f --- /dev/null +++ b/yara-x-dump/src/tests/mod.rs @@ -0,0 +1,37 @@ +use protobuf::text_format::parse_from_str; +use protobuf::MessageDyn; +use std::fs; +use std::io::Write; +use yansi::Paint; + +use crate::Dumper; + +#[test] +fn test_dumper() { + // Disable colors for testing. + Paint::disable(); + + // Create goldenfile mint. + let mut mint = goldenfile::Mint::new("."); + + for entry in globwalk::glob("src/tests/testdata/*.in").unwrap().flatten() { + // Path to the .in file. + println!("{:?}", entry); + let in_path = entry.into_path(); + + // Path to the .out file. + let out_path = in_path.with_extension("out"); + + let input = fs::read_to_string(in_path).expect("Unable to read"); + + let test = parse_from_str::(&input).unwrap(); + + let dumper = Dumper::default(); + let output = dumper.dump(&test as &dyn MessageDyn).unwrap(); + + // Create a goldenfile test + let mut output_file = mint.new_goldenfile(out_path).unwrap(); + + write!(output_file, "{}", output).unwrap(); + } +} diff --git a/yara-x-dump/src/tests/protos/dumper.proto b/yara-x-dump/src/tests/protos/dumper.proto new file mode 100644 index 000000000..c4170cf1a --- /dev/null +++ b/yara-x-dump/src/tests/protos/dumper.proto @@ -0,0 +1,13 @@ +syntax = "proto2"; + +package dumper; + +import "google/protobuf/descriptor.proto"; + +message FieldOptions { + optional string yaml_fmt = 3; + } + +extend google.protobuf.FieldOptions { + optional FieldOptions field_options = 51504; + } diff --git a/yara-x-dump/src/tests/protos/test.proto b/yara-x-dump/src/tests/protos/test.proto new file mode 100644 index 000000000..579019093 --- /dev/null +++ b/yara-x-dump/src/tests/protos/test.proto @@ -0,0 +1,25 @@ +syntax = "proto2"; + +import "dumper.proto"; + +package test; + +message Segment { + optional uint32 nested1 = 1; + optional uint64 nested2 = 2 [(dumper.field_options).yaml_fmt = "x"]; + optional uint32 timestamp = 3 [(dumper.field_options).yaml_fmt = "t"]; +} + +message OptionalNested { + optional uint32 onested1 = 1; + optional uint64 onested2 = 2 [(dumper.field_options).yaml_fmt = "x"]; + map map_string_string = 3; +} + +message MyMessage { + optional int32 field1 = 1 [(dumper.field_options).yaml_fmt = "x"]; + optional string field2 = 2; + required string field3 = 3; + repeated Segment segments = 4; + optional OptionalNested optional = 5; +} diff --git a/yara-x-dump/src/tests/testdata/macho_x86_file.in b/yara-x-dump/src/tests/testdata/macho_x86_file.in new file mode 100644 index 000000000..b28131b41 --- /dev/null +++ b/yara-x-dump/src/tests/testdata/macho_x86_file.in @@ -0,0 +1,21 @@ +field1: 123 +field2: "test" +field3: "test\ntest" +segments { + nested1: 456 + nested2: 789 + timestamp: 123456789 +} +segments { + nested1: 100000 + nested2: 200000 + timestamp: 999999999 +} +optional { + onested1: 123 + onested2: 456 + map_string_string { + key: "foo\nfoo" + value: "bar\nbar" + } +} diff --git a/yara-x-dump/src/tests/testdata/macho_x86_file.out b/yara-x-dump/src/tests/testdata/macho_x86_file.out new file mode 100644 index 000000000..b0502b30a --- /dev/null +++ b/yara-x-dump/src/tests/testdata/macho_x86_file.out @@ -0,0 +1,15 @@ +field1: 0x7b +field2: "test" +field3: "test\ntest" +segments: + - nested1: 456 + nested2: 0x315 + timestamp: 123456789 # 1973-11-29 21:33:09 UTC + - nested1: 100000 + nested2: 0x30d40 + timestamp: 999999999 # 2001-09-09 01:46:39 UTC +optional: +onested1: 123 + onested2: 0x1c8 + map_string_string: + "foo\nfoo": "bar\nbar" diff --git a/yara-x-proto/src/yara.proto b/yara-x-proto/src/yara.proto index f3620b896..475e44eb2 100644 --- a/yara-x-proto/src/yara.proto +++ b/yara-x-proto/src/yara.proto @@ -16,6 +16,7 @@ message ModuleOptions { message FieldOptions { optional string name = 1; optional bool ignore = 2; + optional string yaml_fmt = 3; } message MessageOptions { diff --git a/yara-x/fuzz/Cargo.toml b/yara-x/fuzz/Cargo.toml index 0b8cd1a6d..28758c18f 100644 --- a/yara-x/fuzz/Cargo.toml +++ b/yara-x/fuzz/Cargo.toml @@ -1,8 +1,10 @@ [package] name = "yara-x-fuzz" version = "0.1.0" +authors = ["Tomas Duris "] publish = false edition = "2021" +description = "Fuzzing harness for YARA-X modules" [package.metadata] cargo-fuzz = true diff --git a/yara-x/src/modules/macho/mod.rs b/yara-x/src/modules/macho/mod.rs index 126c5ccde..00e869aa1 100644 --- a/yara-x/src/modules/macho/mod.rs +++ b/yara-x/src/modules/macho/mod.rs @@ -526,6 +526,24 @@ fn should_swap_bytes(magic: u32) -> bool { matches!(magic, MH_CIGAM | MH_CIGAM_64 | FAT_CIGAM | FAT_CIGAM_64) } +/// Convert a decimal number representation to a version string representation. +/// The decimal number is expected to be in the format +/// `major(rest of digits).minor(previous 2 digits).patch(last 2 digits)`. +/// +/// # Arguments +/// +/// * `decimal_number`: The decimal number to convert. +/// +/// # Returns +/// +/// A string representation of the version number. +fn convert_to_version_string(decimal_number: u32) -> String { + let major = decimal_number >> 16; + let minor = (decimal_number >> 8) & 0xFF; + let patch = decimal_number & 0xFF; + format!("{}.{}.{}", major, minor, patch) +} + /// Convert a Mach-O Relative Virtual Address (RVA) to an offset within the /// file. /// @@ -1556,8 +1574,12 @@ fn handle_dylib_command( .to_string(), ), timestamp: Some(dy.dylib.timestamp), - compatibility_version: Some(dy.dylib.compatibility_version), - current_version: Some(dy.dylib.current_version), + compatibility_version: Some(convert_to_version_string( + dy.dylib.compatibility_version, + )), + current_version: Some(convert_to_version_string( + dy.dylib.current_version, + )), ..Default::default() }; macho_file.dylibs.push(dylib); @@ -1618,8 +1640,7 @@ fn handle_segment_command( segname: Some( std::str::from_utf8(&sg.segname) .unwrap_or_default() - .trim_end_matches('\0') - .to_string(), + .replace('\0', ""), ), vmaddr: Some(sg.vmaddr as u64), vmsize: Some(sg.vmsize as u64), @@ -1651,13 +1672,13 @@ fn handle_segment_command( segname: Some( std::str::from_utf8(&sec.segname) .unwrap_or_default() - .trim_end_matches('\0') + .replace('\0', "") .to_string(), ), sectname: Some( std::str::from_utf8(&sec.sectname) .unwrap_or_default() - .trim_end_matches('\0') + .replace('\0', "") .to_string(), ), addr: Some(sec.addr as u64), @@ -1738,8 +1759,7 @@ fn handle_segment_command_64( segname: Some( std::str::from_utf8(&sg.segname) .unwrap_or_default() - .trim_end_matches('\0') - .to_string(), + .replace('\0', ""), ), vmaddr: Some(sg.vmaddr), vmsize: Some(sg.vmsize), @@ -1771,13 +1791,13 @@ fn handle_segment_command_64( segname: Some( std::str::from_utf8(&sec.segname) .unwrap_or_default() - .trim_end_matches('\0') + .replace('\0', "") .to_string(), ), sectname: Some( std::str::from_utf8(&sec.sectname) .unwrap_or_default() - .trim_end_matches('\0') + .replace('\0', "") .to_string(), ), addr: Some(sec.addr), diff --git a/yara-x/src/modules/macho/tests/mod.rs b/yara-x/src/modules/macho/tests/mod.rs index d1fa13dfd..e95c05b14 100644 --- a/yara-x/src/modules/macho/tests/mod.rs +++ b/yara-x/src/modules/macho/tests/mod.rs @@ -131,6 +131,13 @@ fn test_fat_is_32() { assert_eq!(fat_is_32(FAT_CIGAM_64), false); } +#[test] +fn test_convert_to_version_string() { + assert_eq!(convert_to_version_string(65536), "1.0.0"); + assert_eq!(convert_to_version_string(102895360), "1570.15.0"); + assert_eq!(convert_to_version_string(0), "0.0.0"); +} + #[test] fn test_should_swap_bytes() { assert_eq!(should_swap_bytes(MH_CIGAM), true); diff --git a/yara-x/src/modules/macho/tests/testdata/macho_ppc_file.out b/yara-x/src/modules/macho/tests/testdata/macho_ppc_file.out index 1798e72ce..336da472f 100644 --- a/yara-x/src/modules/macho/tests/testdata/macho_ppc_file.out +++ b/yara-x/src/modules/macho/tests/testdata/macho_ppc_file.out @@ -230,7 +230,7 @@ segments { dylibs { name: "/usr/lib/libSystem.B.dylib" timestamp: 1111112572 - compatibility_version: 65536 - current_version: 4653313 + compatibility_version: "1.0.0" + current_version: "71.1.1" } entry_point: 3768 diff --git a/yara-x/src/modules/macho/tests/testdata/macho_x86_64_dylib_file.out b/yara-x/src/modules/macho/tests/testdata/macho_x86_64_dylib_file.out index e7d9595c8..c5c88a074 100644 --- a/yara-x/src/modules/macho/tests/testdata/macho_x86_64_dylib_file.out +++ b/yara-x/src/modules/macho/tests/testdata/macho_x86_64_dylib_file.out @@ -78,12 +78,12 @@ segments { dylibs { name: "fact_x86_64.dylib" timestamp: 1 - compatibility_version: 0 - current_version: 0 + compatibility_version: "0.0.0" + current_version: "0.0.0" } dylibs { name: "/usr/lib/libSystem.B.dylib" timestamp: 2 - compatibility_version: 65536 - current_version: 79495168 + compatibility_version: "1.0.0" + current_version: "1213.0.0" } diff --git a/yara-x/src/modules/macho/tests/testdata/macho_x86_file.out b/yara-x/src/modules/macho/tests/testdata/macho_x86_file.out index e8ebc004b..b0e890827 100644 --- a/yara-x/src/modules/macho/tests/testdata/macho_x86_file.out +++ b/yara-x/src/modules/macho/tests/testdata/macho_x86_file.out @@ -152,8 +152,8 @@ segments { dylibs { name: "/usr/lib/libSystem.B.dylib" timestamp: 2 - compatibility_version: 65536 - current_version: 79495168 + compatibility_version: "1.0.0" + current_version: "1213.0.0" } entry_point: 3728 stack_size: 0 diff --git a/yara-x/src/modules/macho/tests/testdata/tiny_universal.out b/yara-x/src/modules/macho/tests/testdata/tiny_universal.out index 34befd9fc..fab280f7b 100644 --- a/yara-x/src/modules/macho/tests/testdata/tiny_universal.out +++ b/yara-x/src/modules/macho/tests/testdata/tiny_universal.out @@ -169,8 +169,8 @@ file { dylibs { name: "/usr/lib/libSystem.B.dylib" timestamp: 2 - compatibility_version: 65536 - current_version: 79495168 + compatibility_version: "1.0.0" + current_version: "1213.0.0" } entry_point: 3808 stack_size: 0 @@ -352,8 +352,8 @@ file { dylibs { name: "/usr/lib/libSystem.B.dylib" timestamp: 2 - compatibility_version: 65536 - current_version: 79495168 + compatibility_version: "1.0.0" + current_version: "1213.0.0" } entry_point: 3808 stack_size: 0 diff --git a/yara-x/src/modules/protos/macho.proto b/yara-x/src/modules/protos/macho.proto index 73eef853a..6cd1c0678 100644 --- a/yara-x/src/modules/protos/macho.proto +++ b/yara-x/src/modules/protos/macho.proto @@ -11,21 +11,21 @@ option (yara.module_options) = { message Dylib { optional string name = 1; - optional uint32 timestamp = 2; - optional uint32 compatibility_version = 3; - optional uint32 current_version = 4; + optional uint32 timestamp = 2 [(yara.field_options).yaml_fmt = "t"]; + optional string compatibility_version = 3; + optional string current_version = 4; } message Section { optional string segname = 1; optional string sectname = 2; - optional uint64 addr = 3; - optional uint64 size = 4; + optional uint64 addr = 3 [(yara.field_options).yaml_fmt = "x"]; + optional uint64 size = 4 [(yara.field_options).yaml_fmt = "x"]; optional uint32 offset = 5; optional uint32 align = 6; optional uint32 reloff = 7; optional uint32 nreloc = 8; - optional uint32 flags = 9; + optional uint32 flags = 9 [(yara.field_options).yaml_fmt = "x"]; optional uint32 reserved1 = 10; optional uint32 reserved2 = 11; optional uint32 reserved3 = 12; @@ -35,14 +35,14 @@ message Segment { optional uint32 cmd = 1; optional uint32 cmdsize = 2; optional string segname = 3; - optional uint64 vmaddr = 4; - optional uint64 vmsize = 5; + optional uint64 vmaddr = 4 [(yara.field_options).yaml_fmt = "x"]; + optional uint64 vmsize = 5 [(yara.field_options).yaml_fmt = "x"]; optional uint64 fileoff = 6; optional uint64 filesize = 7; - optional uint32 maxprot = 8; - optional uint32 initprot = 9; + optional uint32 maxprot = 8 [(yara.field_options).yaml_fmt = "x"]; + optional uint32 initprot = 9 [(yara.field_options).yaml_fmt = "x"]; optional uint32 nsects = 10; - optional uint32 flags = 11; + optional uint32 flags = 11 [(yara.field_options).yaml_fmt = "x"]; repeated Section sections = 12; } @@ -56,13 +56,13 @@ message FatArch { } message File { - optional uint32 magic = 1; + optional uint32 magic = 1 [(yara.field_options).yaml_fmt = "x"]; optional uint32 cputype = 2; optional uint32 cpusubtype = 3; optional uint32 filetype = 4; optional uint32 ncmds = 5; optional uint32 sizeofcmds = 6; - optional uint32 flags = 7; + optional uint32 flags = 7 [(yara.field_options).yaml_fmt = "x"]; optional uint32 reserved = 8; optional uint64 number_of_segments = 9; repeated Segment segments = 10; @@ -73,13 +73,13 @@ message File { message Macho { // Set Mach-O header and basic fields - optional uint32 magic = 1; + optional uint32 magic = 1 [(yara.field_options).yaml_fmt = "x"]; optional uint32 cputype = 2; optional uint32 cpusubtype = 3; optional uint32 filetype = 4; optional uint32 ncmds = 5; optional uint32 sizeofcmds = 6; - optional uint32 flags = 7; + optional uint32 flags = 7 [(yara.field_options).yaml_fmt = "x"]; optional uint32 reserved = 8; optional uint64 number_of_segments = 9; repeated Segment segments = 10; @@ -88,7 +88,7 @@ message Macho { optional uint64 stack_size = 13; // Add fields for Mach-O fat binary header - optional uint32 fat_magic = 14; + optional uint32 fat_magic = 14 [(yara.field_options).yaml_fmt = "x"]; optional uint32 nfat_arch = 15; repeated FatArch fat_arch = 16;