From e810ba40b59963ce523b6cc3beb52d9e1d079164 Mon Sep 17 00:00:00 2001 From: Jacob Latonis Date: Sun, 4 Feb 2024 09:49:17 -0600 Subject: [PATCH 01/55] chore: Fix deprecated node16 warnings in actions workflows (#80) --- .github/workflows/coverage.yaml | 2 +- .github/workflows/tests.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/coverage.yaml b/.github/workflows/coverage.yaml index 0311a0390..1bbc1fd29 100644 --- a/.github/workflows/coverage.yaml +++ b/.github/workflows/coverage.yaml @@ -27,7 +27,7 @@ jobs: run: cargo llvm-cov --workspace --lib --lcov --output-path lcov.info - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v4 with: files: lcov.info fail_ci_if_error: true \ No newline at end of file diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index b86c62971..8aa2c8386 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -63,7 +63,7 @@ jobs: uses: actions/checkout@v4 - name: Setup cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: | ~/.cargo/registry From 6d3067fd490850bedbf9923332d9d5bf4b065983 Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Sun, 4 Feb 2024 16:54:31 +0100 Subject: [PATCH 02/55] chore: upgrade `env_logger` to version 0.11.1 --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 41d97ab58..1065f554d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -45,7 +45,7 @@ clap = "4.4.8" crc32fast = "1.3.2" criterion = "0.5.1" enable-ansi-support = "0.2.1" -env_logger = "0.10.0" +env_logger = "0.11.1" fmmap = "0.3.2" globwalk = "0.9.0" indenter = "0.3.3" From c0bc880200f0586e81e5feb58ffe7c6cf067ae37 Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Sun, 4 Feb 2024 16:55:11 +0100 Subject: [PATCH 03/55] chore: upgrade `authenticode-parser` to version 0.4.0 --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 1065f554d..4a3c21e52 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,7 +34,7 @@ anyhow = "1.0.75" ariadne = "0.3.0" array-bytes = "6.2.0" ascii_tree = "0.1.1" -authenticode-parser = "0.3.2" +authenticode-parser = "0.4.0" base64 = "0.21.0" bincode = "1.3.3" bitmask = "0.5.0" From a81ddd2cb120ebd9f2fc89faf6b548bb06d77f85 Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Sun, 4 Feb 2024 17:03:54 +0100 Subject: [PATCH 04/55] chore: bump MSRV from 1.71.0 to 1.73.0 --- .github/workflows/code_health.yaml | 4 ++-- .github/workflows/tests.yaml | 2 +- Cargo.toml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/code_health.yaml b/.github/workflows/code_health.yaml index a4ae90246..4be86fc90 100644 --- a/.github/workflows/code_health.yaml +++ b/.github/workflows/code_health.yaml @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@1.71.0 + - uses: dtolnay/rust-toolchain@1.73.0 with: components: clippy - run: cargo clippy --tests --no-deps --all-features -- --deny clippy::all @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@1.71.0 + - uses: dtolnay/rust-toolchain@1.73.0 with: components: rustfmt - run: cargo fmt --all --check diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 8aa2c8386..d5a2596ca 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -22,7 +22,7 @@ jobs: include: - build: msrv os: ubuntu-latest - rust: 1.71.0 + rust: 1.73.0 args: "" - build: stable diff --git a/Cargo.toml b/Cargo.toml index 4a3c21e52..840d252e0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,7 +12,7 @@ license = "MIT" # .github/workflows/code_health.yaml # # TODO: after bumping to 1.73 stop using the `num` crate. -rust-version = "1.71.0" +rust-version = "1.73.0" [workspace] members = [ From 752d74bddb91dbf5f6cd087b3b471f105507ba11 Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Sun, 4 Feb 2024 17:33:50 +0100 Subject: [PATCH 05/55] chore: remove dependency from `num` crate --- Cargo.lock | 44 ++++++++++++++--------------- Cargo.toml | 2 -- yara-x-parser/Cargo.toml | 2 +- yara-x-parser/src/parser/cst2ast.rs | 6 ++-- yara-x-parser/src/parser/mod.rs | 5 +--- yara-x/Cargo.toml | 1 - yara-x/src/modules/dotnet/parser.rs | 13 ++++----- yara-x/src/modules/pe/mod.rs | 4 +-- yara-x/src/modules/pe/parser.rs | 3 +- yara-x/src/scanner/mod.rs | 15 ++++------ 10 files changed, 40 insertions(+), 55 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4ca5c3209..85bc95b52 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -90,9 +90,9 @@ checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" [[package]] name = "anstream" -version = "0.6.5" +version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d664a92ecae85fd0a7392615844904654d1d5f5514837f471ddef4a057aba1b6" +checksum = "6e2e1ebcb11de5c03c67de28a7df593d32191b44939c482e97702baaaa6ab6a5" dependencies = [ "anstyle", "anstyle-parse", @@ -189,18 +189,18 @@ dependencies = [ [[package]] name = "authenticode-parser" -version = "0.3.2" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a157336d4bedbed5d11a2389a589dadfdd26aa4ed5e1425fe04681fb1a5518a0" +checksum = "604103d623672bbfcea96b1f20b75ee433012a14421fc87660ada85e74add9f9" dependencies = [ "authenticode-parser-sys", ] [[package]] name = "authenticode-parser-sys" -version = "0.3.1" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "038c06837a7af6b971836a34532d905d29d4aac692b66fda9b9634912c2043e7" +checksum = "214a42da4790c46cf73660c83fe50bc1ecd4462362dfcf10f4a64de2776e2364" dependencies = [ "cc", ] @@ -1143,17 +1143,27 @@ dependencies = [ "syn 2.0.43", ] +[[package]] +name = "env_filter" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a009aa4810eb158359dda09d0c87378e4bbb89b5a801f016885a4707ba24f7ea" +dependencies = [ + "log", + "regex", +] + [[package]] name = "env_logger" -version = "0.10.1" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95b3f3e67048839cb0d0781f445682a35113da7121f7c949db0e2be96a4fbece" +checksum = "05e7cf40684ae96ade6232ed84582f40ce0a66efcd43a5117aef610534f8e0b8" dependencies = [ + "anstream", + "anstyle", + "env_filter", "humantime", - "is-terminal", "log", - "regex", - "termcolor", ] [[package]] @@ -3198,15 +3208,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "termcolor" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff1bc3d3f05aff0403e8ac0d92ced918ec05b666a43f83297ccef5bea8a3d449" -dependencies = [ - "winapi-util", -] - [[package]] name = "terminfo" version = "0.7.5" @@ -4324,7 +4325,6 @@ dependencies = [ "memchr", "memx", "nom 7.1.3", - "num", "num-derive 0.4.1", "num-traits", "pretty_assertions", @@ -4417,7 +4417,7 @@ dependencies = [ "indenter", "itertools 0.12.0", "lazy_static", - "num", + "num-traits", "pest", "pest_derive", "pretty_assertions", diff --git a/Cargo.toml b/Cargo.toml index 840d252e0..c029da4e2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,7 +11,6 @@ license = "MIT" # .github/workflows/tests.yaml # .github/workflows/code_health.yaml # -# TODO: after bumping to 1.73 stop using the `num` crate. rust-version = "1.73.0" [workspace] @@ -65,7 +64,6 @@ tlsh-fixed = "0.1.1" memchr = "2.6.4" memx = "0.1.28" nom = "7.1.3" -num = "0.4.1" num-traits = "0.2" num-derive = "0.4.1" pest = "2.7.5" diff --git a/yara-x-parser/Cargo.toml b/yara-x-parser/Cargo.toml index 9ceaf9cff..ff9f2244a 100644 --- a/yara-x-parser/Cargo.toml +++ b/yara-x-parser/Cargo.toml @@ -28,7 +28,7 @@ bitmask = { workspace = true } bstr = { workspace = true } itertools = { workspace = true } lazy_static = { workspace = true } -num = { workspace = true } +num-traits = { workspace = true } pest = { workspace = true } pest_derive = { workspace = true } thiserror = { workspace = true } diff --git a/yara-x-parser/src/parser/cst2ast.rs b/yara-x-parser/src/parser/cst2ast.rs index 6f7356d77..6fe043f1a 100644 --- a/yara-x-parser/src/parser/cst2ast.rs +++ b/yara-x-parser/src/parser/cst2ast.rs @@ -8,7 +8,7 @@ use std::str; use crate::ast; use bstr::{BStr, BString, ByteSlice, ByteVec}; use lazy_static::lazy_static; -use num::{Bounded, CheckedMul, FromPrimitive, Integer}; +use num_traits::{Bounded, CheckedMul, FromPrimitive, Num}; use pest::iterators::Pair; use pest::pratt_parser::{Assoc, Op, PrattParser}; @@ -1779,14 +1779,14 @@ fn iterator_from_cst<'src>( } /// From a CST node corresponding to the grammar rule `integer_lit`, returns -/// the the corresponding integer. This is a generic function that can be used +/// the corresponding integer. This is a generic function that can be used /// for obtaining any type of integer, like u8, i64, etc. fn integer_lit_from_cst<'src, T>( ctx: &Context<'src, '_>, integer_lit: CSTNode<'src>, ) -> Result where - T: Integer + Bounded + CheckedMul + FromPrimitive + std::fmt::Display, + T: Num + Bounded + CheckedMul + FromPrimitive + std::fmt::Display, { expect!(integer_lit, GrammarRule::integer_lit); diff --git a/yara-x-parser/src/parser/mod.rs b/yara-x-parser/src/parser/mod.rs index 1842ec4cf..e36e40d57 100644 --- a/yara-x-parser/src/parser/mod.rs +++ b/yara-x-parser/src/parser/mod.rs @@ -273,10 +273,7 @@ impl<'a> Parser<'a> { // the Unicode replacement characters that takes 3 bytes. // This way the span ends at a valid UTF-8 character // boundary. - // TODO: `usize::next_multiple_of` was stabilized in Rust 1.73. - // Once we bump the MSRV to 1.73 we can stop using `num`. - // https://doc.rust-lang.org/std/primitive.u32.html#method.div_ceil - span_start + num::Integer::next_multiple_of(&error_len, &3) + span_start + error_len.next_multiple_of(3) } else { span_start }; diff --git a/yara-x/Cargo.toml b/yara-x/Cargo.toml index 45d4e381c..a2e8abfc9 100644 --- a/yara-x/Cargo.toml +++ b/yara-x/Cargo.toml @@ -155,7 +155,6 @@ sha2 = { workspace = true, optional = true } memchr = { workspace = true } memx = { workspace = true } nom = { workspace = true, optional = true } -num = { workspace = true } num-derive = { workspace = true } num-traits = { workspace = true } protobuf = { workspace = true } diff --git a/yara-x/src/modules/dotnet/parser.rs b/yara-x/src/modules/dotnet/parser.rs index 5116bcd94..a41210806 100644 --- a/yara-x/src/modules/dotnet/parser.rs +++ b/yara-x/src/modules/dotnet/parser.rs @@ -15,6 +15,7 @@ use nom::number::complete::{le_u16, le_u32, le_u64, u8}; use nom::sequence::tuple; use nom::{bits, IResult, Parser}; use num_derive::FromPrimitive; +use num_traits; use protobuf::MessageField; use uuid::Uuid; @@ -379,11 +380,7 @@ impl<'a> Dotnet<'a> { // The name is padded with zeroes up to the next 4 bytes boundary, // lets consume the padding, which includes the null-terminator. - // TODO: `usize::next_multiple_of` was stabilized in Rust 1.73. - // Once we bump the MSRV to 1.73 we can stop using `num`. - // https://doc.rust-lang.org/std/primitive.u32.html#method.div_ceil - let padding = - num::Integer::next_multiple_of(&(name.len() + 1), &4) - name.len(); + let padding = (name.len() + 1).next_multiple_of(4) - name.len(); let (remaining, _) = take(padding)(remaining)?; @@ -1081,7 +1078,7 @@ impl<'a> Dotnet<'a> { } let (mut remainder, type_) = - map_opt(u8, num::FromPrimitive::from_u8)(input)?; + map_opt(u8, num_traits::FromPrimitive::from_u8)(input)?; *depth += 1; @@ -1643,8 +1640,8 @@ impl<'a> Dotnet<'a> { ) -> impl FnMut(&'a [u8]) -> IResult<&'a [u8], Constant> + '_ { map( tuple(( - map_opt(u8, num::FromPrimitive::from_u8), // type - u8, // padding + map_opt(u8, num_traits::FromPrimitive::from_u8), // type + u8, // padding self.coded_index(&[ Table::Field, Table::Param, diff --git a/yara-x/src/modules/pe/mod.rs b/yara-x/src/modules/pe/mod.rs index 508d807ca..de21debd8 100644 --- a/yara-x/src/modules/pe/mod.rs +++ b/yara-x/src/modules/pe/mod.rs @@ -114,8 +114,8 @@ fn calculate_checksum(ctx: &mut ScanContext) -> Option { let data = ctx.scanned_data(); let mut sum: u32 = 0; - // The parser first try to read a u32, if not enough data is available, - // it tries to read a u16, and if still there's no data available it + // The parser first try to read an u32, if not enough data is available, + // it tries to read an u16, and if still there's no data available it // tries to read a byte. In all cases the result is promoted to u32. This // emulates the padding at the end of the data if necessary. let data_parser = alt(( diff --git a/yara-x/src/modules/pe/parser.rs b/yara-x/src/modules/pe/parser.rs index ddcd6b31b..eccebbf91 100644 --- a/yara-x/src/modules/pe/parser.rs +++ b/yara-x/src/modules/pe/parser.rs @@ -1197,8 +1197,7 @@ impl<'a> PE<'a> { /// Round up an offset to the next 32-bit boundary. fn round_up(offset: O) -> usize { - // TODO: use usize:div_ceil when we bump the MSRV to 1.73.0. - num::Integer::div_ceil(&offset.to_usize(), &4) * 4 + offset.to_usize().div_ceil(4) * 4 } /// Parses the PE resources. diff --git a/yara-x/src/scanner/mod.rs b/yara-x/src/scanner/mod.rs index 5e94ca380..6faaa26a1 100644 --- a/yara-x/src/scanner/mod.rs +++ b/yara-x/src/scanner/mod.rs @@ -177,18 +177,13 @@ impl<'r> Scanner<'r> { // the N-th bit is set if pattern with PatternId = N matched. The // bitmap starts right after the bitmap that contains matching // information for rules. - // - // TODO: `u32::div_ceil` was stabilized in Rust 1.73. Once we bump - // the MSRV to 1.73 we can stop using `num`. - // https://doc.rust-lang.org/std/primitive.u32.html#method.div_ceil - let matching_patterns_bitmap_base = MATCHING_RULES_BITMAP_BASE as u32 - + num::Integer::div_ceil(&num_rules, &8); + let matching_patterns_bitmap_base = + MATCHING_RULES_BITMAP_BASE as u32 + num_rules.div_ceil(8); // Compute the required memory size in 64KB pages. - let mem_size = num::Integer::div_ceil( - &(matching_patterns_bitmap_base - + num::Integer::div_ceil(&num_patterns, &8)), - &65536, + let mem_size = u32::div_ceil( + matching_patterns_bitmap_base + num_patterns.div_ceil(8), + 65536, ); let matching_patterns_bitmap_base = Global::new( From db7a74c0ad11d680f483a0c641b666f8edce8ff0 Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Sun, 4 Feb 2024 17:45:29 +0100 Subject: [PATCH 06/55] style: remove unnecessary hashes around string literals --- yara-x/src/compiler/ir/hex2hir.rs | 4 +- yara-x/src/tests/mod.rs | 132 +++++++++++++++--------------- 2 files changed, 66 insertions(+), 70 deletions(-) diff --git a/yara-x/src/compiler/ir/hex2hir.rs b/yara-x/src/compiler/ir/hex2hir.rs index 12749d31c..8079f6e2b 100644 --- a/yara-x/src/compiler/ir/hex2hir.rs +++ b/yara-x/src/compiler/ir/hex2hir.rs @@ -98,11 +98,11 @@ mod tests { fn hex_byte_to_hir() { let hir = super::hex_byte_hir_from_ast(&HexByte { value: 0x00, mask: 0x00 }); - assert_eq!(hir.to_string(), r#"(?-u:[\x00-\xFF])"#); + assert_eq!(hir.to_string(), r"(?-u:[\x00-\xFF])"); let hir = super::hex_byte_hir_from_ast(&HexByte { value: 0x10, mask: 0xf0 }); - assert_eq!(hir.to_string(), r#"(?-u:[\x10-\x1F])"#); + assert_eq!(hir.to_string(), r"(?-u:[\x10-\x1F])"); let hir = super::hex_byte_hir_from_ast(&HexByte { value: 0x02, mask: 0x0f }); diff --git a/yara-x/src/tests/mod.rs b/yara-x/src/tests/mod.rs index 3b0660597..6e0f10855 100644 --- a/yara-x/src/tests/mod.rs +++ b/yara-x/src/tests/mod.rs @@ -1144,16 +1144,16 @@ fn regexp_patterns_3() { pattern_match!(r#"/[a-c-e]/"#, b"b", b"b"); pattern_match!(r#"/[a-c-e]/"#, b"-", b"-"); pattern_false!(r#"/[a-c-e]/"#, b"d"); - pattern_match!(r#"/a[\-b]/"#, b"a-", b"a-"); - pattern_match!(r#"/a[\-b]/"#, b"ab", b"ab"); + pattern_match!(r"/a[\-b]/", b"a-", b"a-"); + pattern_match!(r"/a[\-b]/", b"ab", b"ab"); pattern_match!(r#"/a]/"#, b"a]", b"a]"); pattern_match!(r#"/a[]]b/"#, b"a]b", b"a]b"); pattern_match!(r#"/[a-z]-b/"#, b"c-b-c", b"c-b"); pattern_match!(r#"/a[]-]b/"#, b"a]b", b"a]b"); pattern_match!(r#"/a[]-]b/"#, b"a-b", b"a-b"); - pattern_match!(r#"/[\.-z]+/"#, b"...abc", b"...abc"); - pattern_match!(r#"/[\.-]+/"#, b"...abc", b"..."); - pattern_match!(r#"/a[\]]b/"#, b"a]b", b"a]b"); + pattern_match!(r"/[\.-z]+/", b"...abc", b"...abc"); + pattern_match!(r"/[\.-]+/", b"...abc", b"..."); + pattern_match!(r"/a[\]]b/", b"a]b", b"a]b"); pattern_match!(r#"/a[^bc]d/"#, b"aed", b"aed"); pattern_false!(r#"/a[^bc]d/"#, b"abd"); pattern_match!(r#"/a[^-b]c/"#, b"adc", b"adc"); @@ -1161,50 +1161,46 @@ fn regexp_patterns_3() { pattern_false!(r#"/a[^]b]c/"#, b"a]c"); pattern_match!(r#"/a[^]b]c/"#, b"adc", b"adc"); pattern_match!(r#"/[^ab]+/"#, b"cde", b"cde"); - pattern_match!(r#"/a[\s]b/"#, b"a b", b"a b"); - pattern_false!(r#"/a[\S]b/"#, b"a b"); - pattern_match!(r#"/a[\d]b/"#, b"a1b", b"a1b"); - pattern_false!(r#"/a[\D]b/"#, b"a1b"); - pattern_match!(r#"/a\sb/"#, b"a b", b"a b"); - pattern_match!(r#"/a\sb/"#, b"a\tb", b"a\tb"); - pattern_match!(r#"/a\sb/"#, b"a\rb", b"a\rb"); - pattern_match!(r#"/a\sb/"#, b"a\nb", b"a\nb"); - pattern_match!(r#"/a\sb/"#, b"a\x0bb", b"a\x0bb"); - pattern_match!(r#"/a\sb/"#, b"a\x0cb", b"a\x0cb"); - pattern_false!(r#"/a\Sb/"#, b"a b"); - pattern_false!(r#"/a\Sb/"#, b"a\tb"); - pattern_false!(r#"/a\Sb/"#, b"a\rb"); - pattern_false!(r#"/a\Sb/"#, b"a\nb"); - pattern_false!(r#"/a\Sb/"#, b"a\x0bb"); - pattern_false!(r#"/a\Sb/"#, b"a\x0cb"); - pattern_match!(r#"/a[\s]*b/"#, b"a \t\r\n\x0b\x0cb", b"a \t\r\n\x0b\x0cb"); - pattern_match!( - r#"/a[^\S]*b/"#, - b"a \t\r\n\x0b\x0cb", - b"a \t\r\n\x0b\x0cb" - ); - pattern_match!(r#"/foo[^\s]*/"#, b"foobar\n", b"foobar"); - pattern_match!(r#"/foo[^\s]*/"#, b"foobar\r\n", b"foobar"); - pattern_match!(r#"/\n\r\t\f\a/"#, b"\n\r\t\x0c\x07", b"\n\r\t\x0c\x07"); - pattern_match!( - r#"/[\n][\r][\t][\f][\a]/"#, + pattern_match!(r"/a[\s]b/", b"a b", b"a b"); + pattern_false!(r"/a[\S]b/", b"a b"); + pattern_match!(r"/a[\d]b/", b"a1b", b"a1b"); + pattern_false!(r"/a[\D]b/", b"a1b"); + pattern_match!(r"/a\sb/", b"a b", b"a b"); + pattern_match!(r"/a\sb/", b"a\tb", b"a\tb"); + pattern_match!(r"/a\sb/", b"a\rb", b"a\rb"); + pattern_match!(r"/a\sb/", b"a\nb", b"a\nb"); + pattern_match!(r"/a\sb/", b"a\x0bb", b"a\x0bb"); + pattern_match!(r"/a\sb/", b"a\x0cb", b"a\x0cb"); + pattern_false!(r"/a\Sb/", b"a b"); + pattern_false!(r"/a\Sb/", b"a\tb"); + pattern_false!(r"/a\Sb/", b"a\rb"); + pattern_false!(r"/a\Sb/", b"a\nb"); + pattern_false!(r"/a\Sb/", b"a\x0bb"); + pattern_false!(r"/a\Sb/", b"a\x0cb"); + pattern_match!(r"/a[\s]*b/", b"a \t\r\n\x0b\x0cb", b"a \t\r\n\x0b\x0cb"); + pattern_match!(r"/a[^\S]*b/", b"a \t\r\n\x0b\x0cb", b"a \t\r\n\x0b\x0cb"); + pattern_match!(r"/foo[^\s]*/", b"foobar\n", b"foobar"); + pattern_match!(r"/foo[^\s]*/", b"foobar\r\n", b"foobar"); + pattern_match!(r"/\n\r\t\f\a/", b"\n\r\t\x0c\x07", b"\n\r\t\x0c\x07"); + pattern_match!( + r"/[\n][\r][\t][\f][\a]/", b"\n\r\t\x0c\x07", b"\n\r\t\x0c\x07" ); - pattern_match!(r#"/\x01\x02\x03/"#, b"\x01\x02\x03", b"\x01\x02\x03"); - pattern_match!(r#"/[\x01-\x03]+/"#, b"\x01\x02\x03", b"\x01\x02\x03"); - pattern_false!(r#"/[\x00-\x02]+/"#, b"\x03\x04\x05"); - pattern_match!(r#"/[\x5D]/"#, b"]", b"]"); - pattern_match!(r#"/a\wc/"#, b"abc", b"abc"); - pattern_match!(r#"/a\wc/"#, b"a_c", b"a_c"); - pattern_match!(r#"/a\wc/"#, b"a0c", b"a0c"); - pattern_false!(r#"/a\wc/"#, b"a*c"); - pattern_match!(r#"/\w+/"#, b"--ab_cd0123--", b"ab_cd0123"); - pattern_match!(r#"/[\w]+/"#, b"--ab_cd0123--", b"ab_cd0123"); - pattern_match!(r#"/\D+/"#, b"1234abc5678", b"abc"); - pattern_match!(r#"/[\d]+/"#, b"0123456789", b"0123456789"); - pattern_match!(r#"/[\D]+/"#, b"1234abc5678", b"abc"); - pattern_match!(r#"/[\da-fA-F]+/"#, b"123abcDEF", b"123abcDEF"); + pattern_match!(r"/\x01\x02\x03/", b"\x01\x02\x03", b"\x01\x02\x03"); + pattern_match!(r"/[\x01-\x03]+/", b"\x01\x02\x03", b"\x01\x02\x03"); + pattern_false!(r"/[\x00-\x02]+/", b"\x03\x04\x05"); + pattern_match!(r"/[\x5D]/", b"]", b"]"); + pattern_match!(r"/a\wc/", b"abc", b"abc"); + pattern_match!(r"/a\wc/", b"a_c", b"a_c"); + pattern_match!(r"/a\wc/", b"a0c", b"a0c"); + pattern_false!(r"/a\wc/", b"a*c"); + pattern_match!(r"/\w+/", b"--ab_cd0123--", b"ab_cd0123"); + pattern_match!(r"/[\w]+/", b"--ab_cd0123--", b"ab_cd0123"); + pattern_match!(r"/\D+/", b"1234abc5678", b"abc"); + pattern_match!(r"/[\d]+/", b"0123456789", b"0123456789"); + pattern_match!(r"/[\D]+/", b"1234abc5678", b"abc"); + pattern_match!(r"/[\da-fA-F]+/", b"123abcDEF", b"123abcDEF"); pattern_match!(r#"/(abc|)ef/"#, b"abcdef", b"ef"); pattern_match!(r#"/(abc|)ef/"#, b"abcef", b"abcef"); pattern_match!(r#"/(abc|)ef/"#, b"abcef", b"abcef"); @@ -1220,7 +1216,7 @@ fn regexp_patterns_3() { pattern_match!(r#"/a([bc]*)(c+d)/"#, b"abcd", b"abcd"); pattern_match!(r#"/a[bcd]*dcdcde/"#, b"adcdcde", b"adcdcde"); pattern_false!(r#"/a[bcd]+dcdcde/"#, b"adcdcde"); - pattern_match!(r#"/\((.*), (.*)\)/"#, b"(a, b)", b"(a, b)"); + pattern_match!(r"/\((.*), (.*)\)/", b"(a, b)", b"(a, b)"); pattern_match!(r#"/whatever| x. x/"#, b" xy x", b" xy x"); pattern_match!(r#"/^abc/"#, b"abc", b"abc"); pattern_match!(r#"/^abc/"#, b"abcd", b"abc"); @@ -1244,26 +1240,26 @@ fn regexp_patterns_3() { pattern_false!(r#"/(bc+d$|ef*g.|h?i(j|k))/"#, b"effg"); pattern_false!(r#"/(bc+d$|ef*g.|h?i(j|k))/"#, b"bcdd"); pattern_match!(r#"/(bc+d$|ef*g.|h?i(j|k))/"#, b"reffgz", b"effgz"); - pattern_match!(r#"/\babc/"#, b"abc", b"abc"); - pattern_match!(r#"/abc\b/"#, b"abc", b"abc"); - pattern_false!(r#"/\babc/"#, b"1abc"); - pattern_false!(r#"/abc\b/"#, b"abc1"); - pattern_match!(r#"/abc\s\b/"#, b"abc x", b"abc "); - pattern_false!(r#"/abc\s\b/"#, b"abc "); - pattern_match!(r#"/\babc\b/"#, b" abc ", b"abc"); - pattern_match!(r#"/\b\w\w\w\b/"#, b" abc ", b"abc"); - pattern_match!(r#"/\w\w\w\b/"#, b"abcd", b"bcd"); - pattern_match!(r#"/\b\w\w\w/"#, b"abcd", b"abc"); - pattern_false!(r#"/\b\w\w\w\b/"#, b"abcd"); - pattern_false!(r#"/\Babc/"#, b"abc"); - pattern_false!(r#"/abc\B/"#, b"abc"); - pattern_match!(r#"/\Babc/"#, b"1abc", b"abc"); - pattern_match!(r#"/abc\B/"#, b"abc1", b"abc"); - pattern_false!(r#"/abc\s\B/"#, b"abc x"); - pattern_match!(r#"/abc\s\B/"#, b"abc ", b"abc "); - pattern_match!(r#"/\w\w\w\B/"#, b"abcd", b"abc"); - pattern_match!(r#"/\B\w\w\w/"#, b"abcd", b"bcd"); - pattern_false!(r#"/\B\w\w\w\B/"#, b"abcd"); + pattern_match!(r"/\babc/", b"abc", b"abc"); + pattern_match!(r"/abc\b/", b"abc", b"abc"); + pattern_false!(r"/\babc/", b"1abc"); + pattern_false!(r"/abc\b/", b"abc1"); + pattern_match!(r"/abc\s\b/", b"abc x", b"abc "); + pattern_false!(r"/abc\s\b/", b"abc "); + pattern_match!(r"/\babc\b/", b" abc ", b"abc"); + pattern_match!(r"/\b\w\w\w\b/", b" abc ", b"abc"); + pattern_match!(r"/\w\w\w\b/", b"abcd", b"bcd"); + pattern_match!(r"/\b\w\w\w/", b"abcd", b"abc"); + pattern_false!(r"/\b\w\w\w\b/", b"abcd"); + pattern_false!(r"/\Babc/", b"abc"); + pattern_false!(r"/abc\B/", b"abc"); + pattern_match!(r"/\Babc/", b"1abc", b"abc"); + pattern_match!(r"/abc\B/", b"abc1", b"abc"); + pattern_false!(r"/abc\s\B/", b"abc x"); + pattern_match!(r"/abc\s\B/", b"abc ", b"abc "); + pattern_match!(r"/\w\w\w\B/", b"abcd", b"abc"); + pattern_match!(r"/\B\w\w\w/", b"abcd", b"bcd"); + pattern_false!(r"/\B\w\w\w\B/", b"abcd"); pattern_false!(r#"/a.b/"#, b"a\nb"); pattern_false!(r#"/a.*b/"#, b"acc\nccb"); pattern_match!(r#"/foo/"#, b"foo", b"foo"); @@ -1286,7 +1282,7 @@ fn regexp_patterns_3() { pattern_match!(r#"/foo|bar|baz/i"#, b"BAR", b"BAR"); pattern_match!(r#"/foo|bar|baz/i"#, b"BAZ", b"BAZ"); - pattern_match!(r#"/foo\x01bar/"#, b"foo\x01bar", b"foo\x01bar"); + pattern_match!(r"/foo\x01bar/", b"foo\x01bar", b"foo\x01bar"); /* TODO: YARA accepts unicode characters in regexps but regexp_syntax either From 390a1a852c90bd44d52ab4ef254924c3b56261fd Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Sun, 4 Feb 2024 18:09:17 +0100 Subject: [PATCH 07/55] style: fix clippy warnings --- yara-x/src/modules/dotnet/parser.rs | 1 + yara-x/src/modules/elf/parser.rs | 44 +++++++++++++---------------- yara-x/src/modules/pe/mod.rs | 2 +- yara-x/src/modules/pe/parser.rs | 22 +++++---------- yara-x/src/scanner/mod.rs | 20 ++++++------- 5 files changed, 37 insertions(+), 52 deletions(-) diff --git a/yara-x/src/modules/dotnet/parser.rs b/yara-x/src/modules/dotnet/parser.rs index a41210806..b67daeaed 100644 --- a/yara-x/src/modules/dotnet/parser.rs +++ b/yara-x/src/modules/dotnet/parser.rs @@ -2320,6 +2320,7 @@ enum Type { String = 0xe, Ptr = 0xf, ByRef = 0x10, + #[allow(clippy::enum_variant_names)] ValueType = 0x11, Class = 0x12, Var = 0x13, diff --git a/yara-x/src/modules/elf/parser.rs b/yara-x/src/modules/elf/parser.rs index a5cf6beef..7461a7e87 100644 --- a/yara-x/src/modules/elf/parser.rs +++ b/yara-x/src/modules/elf/parser.rs @@ -2,7 +2,7 @@ use std::mem; use std::ops::Range; use nom::bytes::complete::{take, take_till}; -use nom::combinator::{map_res, verify}; +use nom::combinator::{map, map_res, verify}; use nom::multi::{count, many0}; use nom::number::complete::{le_u32, u16, u32, u64, u8}; use nom::number::Endianness; @@ -79,17 +79,11 @@ impl ElfParser { // Parse the executable header. let (_remainder, ehdr) = self.parse_ehdr()(remainder)?; - self.result.type_ = ehdr - .type_ - .try_into() - .ok() - .map(EnumOrUnknown::::from_i32); + self.result.type_ = + Some(EnumOrUnknown::::from_i32(ehdr.type_.into())); - self.result.machine = ehdr - .machine - .try_into() - .ok() - .map(EnumOrUnknown::::from_i32); + self.result.machine = + Some(EnumOrUnknown::::from_i32(ehdr.machine.into())); self.result.sh_offset = Some(ehdr.sh_offset); self.result.sh_entry_size = Some(ehdr.sh_entry_size.into()); @@ -376,14 +370,14 @@ impl ElfParser { phdr.alignment, ), ) = tuple(( - u32(self.endianness), // type_ - map_res(u32(self.endianness), |v| v.try_into()), // offset - map_res(u32(self.endianness), |v| v.try_into()), // virt_addr - map_res(u32(self.endianness), |v| v.try_into()), // phys_addr - map_res(u32(self.endianness), |v| v.try_into()), // file_size - map_res(u32(self.endianness), |v| v.try_into()), // mem_size - u32(self.endianness), // flags - map_res(u32(self.endianness), |v| v.try_into()), // alignment + u32(self.endianness), // type_ + map(u32(self.endianness), |v| v.into()), // offset + map(u32(self.endianness), |v| v.into()), // virt_addr + map(u32(self.endianness), |v| v.into()), // phys_addr + map(u32(self.endianness), |v| v.into()), // file_size + map(u32(self.endianness), |v| v.into()), // mem_size + u32(self.endianness), // flags + map(u32(self.endianness), |v| v.into()), // alignment ))(input)?; Ok((remainder, phdr)) @@ -499,12 +493,12 @@ impl ElfParser { sym.shndx, ), ) = tuple(( - u32(self.endianness), // name - map_res(u32(self.endianness), |v| v.try_into()), // value - map_res(u32(self.endianness), |v| v.try_into()), // size - u8, // info - u8, // other - u16(self.endianness), // shndx + u32(self.endianness), // name + map(u32(self.endianness), |v| v.into()), // value + map(u32(self.endianness), |v| v.into()), // size + u8, // info + u8, // other + u16(self.endianness), // shndx ))(input)?; Ok((remainder, sym)) diff --git a/yara-x/src/modules/pe/mod.rs b/yara-x/src/modules/pe/mod.rs index de21debd8..95880e13d 100644 --- a/yara-x/src/modules/pe/mod.rs +++ b/yara-x/src/modules/pe/mod.rs @@ -68,7 +68,7 @@ fn rva_to_offset(ctx: &ScanContext, rva: i64) -> Option { pe.file_alignment?, pe.section_alignment?, )?; - Some(offset.try_into().unwrap()) + Some(offset.into()) } /// Returns the PE checksum, as calculated by YARA. diff --git a/yara-x/src/modules/pe/parser.rs b/yara-x/src/modules/pe/parser.rs index eccebbf91..671be88c3 100644 --- a/yara-x/src/modules/pe/parser.rs +++ b/yara-x/src/modules/pe/parser.rs @@ -655,7 +655,7 @@ impl<'a> PE<'a> { .ok_or(Err::Error(Error::new(input, ErrorKind::Tag)))?; // The u32 that follows the "Rich" tag is the XOR key used for - // for encrypting the Rich data. + // encrypting the Rich data. let (remainder, key) = le_u32(&input[rich_tag_pos + 4..])?; // Search for the "DanS" tag that indicates the start of the rich @@ -1935,12 +1935,10 @@ impl From> for protos::pe::PE { let mut result = protos::pe::PE::new(); result.set_is_pe(true); - result.machine = pe + result.machine = Some(EnumOrUnknown::::from_i32(pe .pe_hdr .machine - .try_into() - .ok() - .map(EnumOrUnknown::::from_i32); + .into())); result.set_timestamp(pe.pe_hdr.timestamp); result.set_characteristics(pe.pe_hdr.characteristics.into()); @@ -1949,19 +1947,13 @@ impl From> for protos::pe::PE { result.set_number_of_symbols(pe.pe_hdr.number_of_symbols); result.set_size_of_optional_header(pe.pe_hdr.size_of_optional_header.into()); - result.opthdr_magic = pe + result.opthdr_magic = Some(EnumOrUnknown::::from_i32(pe .optional_hdr - .magic - .try_into() - .ok() - .map(EnumOrUnknown::::from_i32); + .magic.into())); - result.subsystem = pe + result.subsystem = Some(EnumOrUnknown::::from_i32(pe .optional_hdr - .subsystem - .try_into() - .ok() - .map(EnumOrUnknown::::from_i32); + .subsystem.into())); result.set_size_of_code(pe.optional_hdr.size_of_code); result.set_base_of_code(pe.optional_hdr.base_of_code); diff --git a/yara-x/src/scanner/mod.rs b/yara-x/src/scanner/mod.rs index 6faaa26a1..503f91bd1 100644 --- a/yara-x/src/scanner/mod.rs +++ b/yara-x/src/scanner/mod.rs @@ -408,17 +408,15 @@ impl<'r> Scanner<'r> { if self.timeout.is_some() { INIT_HEARTBEAT.call_once(|| { thread::spawn(|| loop { - loop { - thread::sleep(Duration::from_secs(1)); - ENGINE.increment_epoch(); - HEARTBEAT_COUNTER - .fetch_update( - Ordering::SeqCst, - Ordering::SeqCst, - |x| Some(x + 1), - ) - .unwrap(); - } + thread::sleep(Duration::from_secs(1)); + ENGINE.increment_epoch(); + HEARTBEAT_COUNTER + .fetch_update( + Ordering::SeqCst, + Ordering::SeqCst, + |x| Some(x + 1), + ) + .unwrap(); }); }); } From e6c134767558e8f80c927fd8290a4eb30c6e9402 Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Sun, 4 Feb 2024 18:20:34 +0100 Subject: [PATCH 08/55] style: fix clippy warning --- yara-x/src/modules/test_proto2/tests/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yara-x/src/modules/test_proto2/tests/mod.rs b/yara-x/src/modules/test_proto2/tests/mod.rs index 934c613e6..9842c10d8 100644 --- a/yara-x/src/modules/test_proto2/tests/mod.rs +++ b/yara-x/src/modules/test_proto2/tests/mod.rs @@ -44,7 +44,7 @@ fn test_proto2_module() { r#"test_proto2.double_one * test_proto2.float_one == 1.0"# ); - condition_true!(r#"test_proto2.double_one \ 2 == 0.5"#); + condition_true!(r"test_proto2.double_one \ 2 == 0.5"); condition_true!(r#"test_proto2.nested.nested_int64_zero == 0"#); condition_true!(r#"test_proto2.nested.nested_int64_one == 1"#); From ffa67968969bf8bb21240bab2fdc781f8581fa7d Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Sun, 4 Feb 2024 18:25:42 +0100 Subject: [PATCH 09/55] chore: upgrade dependencies --- Cargo.lock | 514 +++++++++++++++++++++++++++-------------------------- 1 file changed, 258 insertions(+), 256 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 85bc95b52..fe45ec8a5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -30,9 +30,9 @@ dependencies = [ [[package]] name = "ahash" -version = "0.8.6" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91429305e9f0a25f6205c5b8e0d2db09e0708a7a6df0f42212bb56c32c8ac97a" +checksum = "77c3a9648d43b9cd48db467b3f87fdd6e146bcc88ab0180006cef2179fe11d01" dependencies = [ "cfg-if", "const-random", @@ -104,9 +104,9 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7079075b41f533b8c61d2a4d073c4676e1f8b249ff94a393b0595db304e0dd87" +checksum = "2faccea4cc4ab4a667ce676a30e8ec13922a692c99bb8f5b11f1502c72e04220" [[package]] name = "anstyle-parse" @@ -138,9 +138,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.77" +version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9d19de80eff169429ac1e9f48fffb163916b448a44e8e046186232046d9e1f9" +checksum = "080e9890a082662b09c1ad45f567faeeb47f22b5fb23895fbe1e651e718e25ca" [[package]] name = "arbitrary" @@ -178,13 +178,13 @@ checksum = "ca6c635b3aa665c649ad1415f1573c85957dfa47690ec27aebe7ec17efe3c643" [[package]] name = "async-trait" -version = "0.1.75" +version = "0.1.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdf6721fb0140e4f897002dd086c06f6c27775df19cfe1fccb21181a48fd2c98" +checksum = "c980ee35e870bd1a4d2c8294d4c04d0499e67bca1e4b5cefcc693c2fa00caea9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.43", + "syn 2.0.48", ] [[package]] @@ -234,9 +234,9 @@ checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" [[package]] name = "base64" -version = "0.21.5" +version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35636a1494ede3b646cc98f74f8e62c773a38a659ebc777a2cf26b9b74171df9" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" [[package]] name = "base64ct" @@ -259,7 +259,7 @@ version = "0.68.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "726e4313eb6ec35d2730258ad4e15b547ee75d6afaa1361a922e78e59b7d8078" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.4.2", "cexpr", "clang-sys", "lazy_static", @@ -270,7 +270,7 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn 2.0.43", + "syn 2.0.48", ] [[package]] @@ -281,9 +281,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.4.1" +version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" +checksum = "ed570934406eb16438a4e976b1b4500774099c13b8cb96eec99f620f05090ddf" [[package]] name = "bitmask" @@ -360,7 +360,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c48f0051a4b4c5e0b6d365cd04af53aeaa209e3cc15ec2cdb69e73cc87fbd0dc" dependencies = [ "memchr", - "regex-automata 0.4.3", + "regex-automata 0.4.5", "serde", ] @@ -378,9 +378,9 @@ checksum = "e1e5f035d16fc623ae5f74981db80a439803888314e3a555fd6f04acd51a3205" [[package]] name = "bytemuck" -version = "1.14.0" +version = "1.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "374d28ec25809ee0e23827c2ab573d729e293f281dfe393500e7ad618baa61c6" +checksum = "ed2490600f404f2b94c167e31d3ed1d5f3c225a0f3b80230053b3e0b7b962bd9" [[package]] name = "byteorder" @@ -441,7 +441,7 @@ checksum = "4acbb09d9ee8e23699b9634375c72795d095bf268439da88562cf9b501f181fa" dependencies = [ "camino", "cargo-platform", - "semver 1.0.20", + "semver 1.0.21", "serde", "serde_json", ] @@ -488,23 +488,23 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.31" +version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38" +checksum = "9f13690e35a5e4ace198e7beea2895d29f3a9cc55015fcebe6336bd2010af9eb" dependencies = [ "android-tzdata", "iana-time-zone", "js-sys", "num-traits", "wasm-bindgen", - "windows-targets 0.48.5", + "windows-targets 0.52.0", ] [[package]] name = "ciborium" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "effd91f6c78e5a4ace8a5d3c0b6bfaec9e2baaef55f3efc00e45fb2e477ee926" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" dependencies = [ "ciborium-io", "ciborium-ll", @@ -513,15 +513,15 @@ dependencies = [ [[package]] name = "ciborium-io" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdf919175532b369853f5d5e20b26b43112613fd6fe7aee757e35f7a44642656" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" [[package]] name = "ciborium-ll" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "defaa24ecc093c77630e6c15e17c51f5e187bf35ee514f4e2d67baaa96dae22b" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" dependencies = [ "ciborium-io", "half", @@ -539,9 +539,9 @@ dependencies = [ [[package]] name = "clang-sys" -version = "1.6.1" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c688fc74432808e3eb684cae8830a86be1d66a2bd58e1f248ed0960a590baf6f" +checksum = "67523a3b4be3ce1989d607a828d036249522dd9c1c8de7f4dd2dae43a37369d1" dependencies = [ "glob", "libc", @@ -550,9 +550,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.12" +version = "4.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcfab8ba68f3668e89f6ff60f5b205cea56aa7b769451a59f34b8682f51c056d" +checksum = "1e578d6ec4194633722ccf9544794b71b1385c3c027efe0c55db226fc880865c" dependencies = [ "clap_builder", "clap_derive", @@ -560,9 +560,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.4.12" +version = "4.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb7fb5e4e979aec3be7791562fcba452f94ad85e954da024396433e0e25a79e9" +checksum = "4df4df40ec50c46000231c914968278b1eb05098cf8f1b3a518a95030e71d1c7" dependencies = [ "anstream", "anstyle", @@ -579,7 +579,7 @@ dependencies = [ "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.43", + "syn 2.0.48", ] [[package]] @@ -620,14 +620,14 @@ dependencies = [ [[package]] name = "console" -version = "0.15.7" +version = "0.15.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c926e00cc70edefdc64d3a5ff31cc65bb97a3460097762bd23afb4d8145fccf8" +checksum = "0e1f83fc076bd6dd27517eacdf25fef6c4dfe5f1d7448bafaaf3a26f13b5e4eb" dependencies = [ "encode_unicode", "lazy_static", "libc", - "windows-sys 0.45.0", + "windows-sys 0.52.0", ] [[package]] @@ -691,9 +691,9 @@ dependencies = [ [[package]] name = "cpufeatures" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce420fe07aecd3e67c5f910618fe65e94158f6dcc0adf44e00d69ce2bdfe0fd0" +checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" dependencies = [ "libc", ] @@ -854,11 +854,10 @@ dependencies = [ [[package]] name = "crossbeam" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6eb9105919ca8e40d437fc9cbb8f1975d916f1bd28afe795a48aae32a2cc8920" +checksum = "1137cd7e7fc0fb5d3c5a8678be38ec56e819125d8d7907411fe24ccb943faca8" dependencies = [ - "cfg-if", "crossbeam-channel", "crossbeam-deque", "crossbeam-epoch", @@ -868,54 +867,46 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.10" +version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82a9b73a36529d9c47029b9fb3a6f0ea3cc916a261195352ba19e770fc1748b2" +checksum = "176dc175b78f56c0f321911d9c8eb2b77a78a4860b9c19db83835fea1a46649b" dependencies = [ - "cfg-if", "crossbeam-utils", ] [[package]] name = "crossbeam-deque" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fca89a0e215bab21874660c67903c5f143333cab1da83d041c7ded6053774751" +checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" dependencies = [ - "cfg-if", "crossbeam-epoch", "crossbeam-utils", ] [[package]] name = "crossbeam-epoch" -version = "0.9.17" +version = "0.9.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e3681d554572a651dda4186cd47240627c3d0114d45a95f6ad27f2f22e7548d" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" dependencies = [ - "autocfg", - "cfg-if", "crossbeam-utils", ] [[package]] name = "crossbeam-queue" -version = "0.3.10" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adc6598521bb5a83d491e8c1fe51db7296019d2ca3cb93cc6c2a20369a4d78a2" +checksum = "df0346b5d5e76ac2fe4e327c5fd1118d6be7c51dfb18f9b7922923f287471e35" dependencies = [ - "cfg-if", "crossbeam-utils", ] [[package]] name = "crossbeam-utils" -version = "0.8.18" +version = "0.8.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3a430a770ebd84726f584a90ee7f020d28db52c6d02138900f22341f866d39c" -dependencies = [ - "cfg-if", -] +checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" [[package]] name = "crossterm" @@ -939,7 +930,7 @@ version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f476fe445d41c9e991fd07515a6f463074b782242ccf4a5b7b1d1012e70824df" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.4.2", "crossterm_winapi", "libc", "mio", @@ -1036,9 +1027,9 @@ checksum = "5729f5117e208430e437df2f4843f5e5952997175992d1414f94c57d61e270b4" [[package]] name = "deranged" -version = "0.3.10" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8eb30d70a07a3b04884d2677f06bec33509dc67ca60d92949e5535352d3191dc" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" dependencies = [ "powerfmt", ] @@ -1140,7 +1131,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.43", + "syn 2.0.48", ] [[package]] @@ -1321,7 +1312,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27d12c0aed7f1e24276a241aadc4cb8ea9f83000f34bc062b7cc2d51e3b0fabd" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.4.2", "debugid", "fxhash", "serde", @@ -1340,9 +1331,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe9006bed769170c11f845cf00c7c1e9092aeb3f268e007c3e760ac68008070f" +checksum = "190092ea657667030ac6a35e305e62fc4dd69fd98ac98631e5d3a2b1575a12b5" dependencies = [ "cfg-if", "libc", @@ -1367,7 +1358,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" dependencies = [ "fallible-iterator 0.3.0", - "indexmap 2.1.0", + "indexmap 2.2.2", "stable_deref_trait", ] @@ -1386,17 +1377,17 @@ dependencies = [ "aho-corasick", "bstr 1.9.0", "log", - "regex-automata 0.4.3", + "regex-automata 0.4.5", "regex-syntax 0.8.2", ] [[package]] name = "globwalk" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baba373693f105316dff9ebdae694118df59cf8fb3fc21b2acf1e294a3893794" +checksum = "0bf760ebf69878d9fd8f110c89703d90ce35095324d1f1edcb595c63945ee757" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.4.2", "ignore", "walkdir", ] @@ -1415,9 +1406,13 @@ dependencies = [ [[package]] name = "half" -version = "1.8.2" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7" +checksum = "bc52e53916c08643f1b56ec082790d1e86a32e58dc5268f897f313fbae7b4872" +dependencies = [ + "cfg-if", + "crunchy", +] [[package]] name = "hashbrown" @@ -1460,9 +1455,9 @@ checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" [[package]] name = "hermit-abi" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" +checksum = "5d3d0e0f38255e7fa3cf31335b3a56f05febd18025f4db5ef7a0cfb4f8da651f" [[package]] name = "hex" @@ -1496,9 +1491,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "iana-time-zone" -version = "0.1.58" +version = "0.1.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8326b86b6cff230b97d0d312a6c40a60726df3332e721f72a1b035f451663b20" +checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -1531,15 +1526,15 @@ checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" [[package]] name = "ignore" -version = "0.4.21" +version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "747ad1b4ae841a78e8aba0d63adbfbeaea26b517b63705d47856b73015d27060" +checksum = "b46810df39e66e925525d6e38ce1e7f6e1d208f72dc39757880fcb66e2c58af1" dependencies = [ "crossbeam-deque", "globset", "log", "memchr", - "regex-automata 0.4.3", + "regex-automata 0.4.5", "same-file", "walkdir", "winapi-util", @@ -1588,9 +1583,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.1.0" +version = "2.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f" +checksum = "824b2ae422412366ba479e8111fd301f7b5faece8149317bb81925979a53f520" dependencies = [ "equivalent", "hashbrown 0.14.3", @@ -1610,7 +1605,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "321f0f839cd44a4686e9504b0a62b4d69a50b62072144c71c68f5873c167b8d9" dependencies = [ "ahash", - "indexmap 2.1.0", + "indexmap 2.2.2", "is-terminal", "itoa", "log", @@ -1658,9 +1653,9 @@ dependencies = [ [[package]] name = "itertools" -version = "0.12.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25db6b064527c5d482d0423354fcd07a89a2dfe07b67892e62411946db7f07b0" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" dependencies = [ "either", ] @@ -1702,9 +1697,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.66" +version = "0.3.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cee9c64da59eae3b50095c18d3e74f8b73c0b86d2792824ff01bbce68ba229ca" +checksum = "9a1d36f1235bc969acba30b7f5990b864423a6068a10f7c90ae8f0112e3a59d1" dependencies = [ "wasm-bindgen", ] @@ -1735,18 +1730,18 @@ checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67" [[package]] name = "libc" -version = "0.2.151" +version = "0.2.153" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "302d7ab3130588088d277783b1e2d2e10c9e9e4a16dd9050e6ec93fb3e7048f4" +checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" [[package]] name = "libloading" -version = "0.7.4" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b67380fd3b2fbe7527a606e18729d21c6f3951633d0500574c4dc22d2d638b9f" +checksum = "c571b676ddfc9a8c12f1f3d3085a7b163966a8fd8098a90640953ce5f6170161" dependencies = [ "cfg-if", - "winapi", + "windows-sys 0.48.0", ] [[package]] @@ -1755,7 +1750,7 @@ version = "0.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85c833ca1e66078851dba29046874e38f08b2c883700aa29a03ddd3b23814ee8" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.4.2", "libc", "redox_syscall", ] @@ -1771,7 +1766,7 @@ dependencies = [ "compact_str", "fraction", "include_dir", - "itertools 0.12.0", + "itertools 0.12.1", "lingua-english-language-model", "lingua-french-language-model", "lingua-german-language-model", @@ -1832,29 +1827,29 @@ checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" [[package]] name = "linkme" -version = "0.3.20" +version = "0.3.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3ae8aae8e1d516e0a3ceee1219eded7f73741607e4227bf11ef2c3e31580427" +checksum = "8b53ad6a33de58864705954edb5ad5d571a010f9e296865ed43dc72a5621b430" dependencies = [ "linkme-impl", ] [[package]] name = "linkme-impl" -version = "0.3.20" +version = "0.3.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad083d767be37e709a232ae2a244445ed032bb9c6bf7d9442dd416ba5a7b7264" +checksum = "04e542a18c94a9b6fcc7adb090fa3ba6b79ee220a16404f325672729f32a66ff" dependencies = [ "proc-macro2", "quote", - "syn 2.0.43", + "syn 2.0.48", ] [[package]] name = "linux-raw-sys" -version = "0.4.12" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4cd1a83af159aa67994778be9070f0ae1bd732942279cabb14f86f986a21456" +checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" [[package]] name = "lock_api" @@ -1910,9 +1905,9 @@ dependencies = [ [[package]] name = "memmap2" -version = "0.9.3" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45fd3a57831bf88bc63f8cebc0cf956116276e97fef3966103e96416209f7c92" +checksum = "fe751422e4a8caa417e13c3ea66452215d7d63e19e604f4980461212f3ae1322" dependencies = [ "libc", ] @@ -1967,9 +1962,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" +checksum = "9d811f3e15f28568be3407c8e7fdb6514c1cda3cb30683f15b6a1a1dc4ea14a7" dependencies = [ "adler", ] @@ -2063,6 +2058,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + [[package]] name = "num-derive" version = "0.3.3" @@ -2082,7 +2083,7 @@ checksum = "cfb77679af88f8b125209d354a202862602672222e7f2313fdd6dc349bad4712" dependencies = [ "proc-macro2", "quote", - "syn 2.0.43", + "syn 2.0.48", ] [[package]] @@ -2145,7 +2146,7 @@ checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" dependencies = [ "crc32fast", "hashbrown 0.14.3", - "indexmap 2.1.0", + "indexmap 2.2.2", "memchr", ] @@ -2176,6 +2177,15 @@ dependencies = [ "num-traits", ] +[[package]] +name = "ordered-float" +version = "4.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a76df7075c7d4d01fdcb46c912dd17fba5b60c78ea480b475f2b6ab6f666584e" +dependencies = [ + "num-traits", +] + [[package]] name = "parking_lot" version = "0.12.1" @@ -2222,7 +2232,7 @@ dependencies = [ "regex", "regex-syntax 0.7.5 (registry+https://github.com/rust-lang/crates.io-index)", "structmeta", - "syn 2.0.43", + "syn 2.0.48", ] [[package]] @@ -2262,9 +2272,9 @@ checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" [[package]] name = "pest" -version = "2.7.5" +version = "2.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae9cee2a55a544be8b89dc6848072af97a20f2422603c10865be2a42b580fff5" +checksum = "1f200d8d83c44a45b21764d1916299752ca035d15ecd46faca3e9a2a2bf6ad06" dependencies = [ "memchr", "thiserror", @@ -2273,9 +2283,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.7.5" +version = "2.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81d78524685f5ef2a3b3bd1cafbc9fcabb036253d9b1463e726a91cd16e2dfc2" +checksum = "bcd6ab1236bbdb3a49027e920e693192ebfe8913f6d60e294de57463a493cfde" dependencies = [ "pest", "pest_generator", @@ -2283,22 +2293,22 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.7.5" +version = "2.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68bd1206e71118b5356dae5ddc61c8b11e28b09ef6a31acbd15ea48a28e0c227" +checksum = "2a31940305ffc96863a735bef7c7994a00b325a7138fdbc5bda0f1a0476d3275" dependencies = [ "pest", "pest_meta", "proc-macro2", "quote", - "syn 2.0.43", + "syn 2.0.48", ] [[package]] name = "pest_meta" -version = "2.7.5" +version = "2.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c747191d4ad9e4a4ab9c8798f1e82a39affe7ef9648390b7e5548d18e099de6" +checksum = "a7ff62f5259e53b78d1af898941cdcdccfae7385cf7d793a6e55de5d05bb4b7d" dependencies = [ "once_cell", "pest", @@ -2354,7 +2364,7 @@ dependencies = [ "phf_shared 0.11.2", "proc-macro2", "quote", - "syn 2.0.43", + "syn 2.0.48", ] [[package]] @@ -2377,9 +2387,9 @@ dependencies = [ [[package]] name = "pkg-config" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69d3587f8a9e599cc7ec2c00e331f71c4e69a5f9a4b8a6efd5b07466b9736f9a" +checksum = "2900ede94e305130c13ddd391e0ab7cbaeb783945ae07a279c268cb05109c6cb" [[package]] name = "plotters" @@ -2454,9 +2464,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.71" +version = "1.0.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75cb1540fadbd5b8fbccc4dddad2734eba435053f725621c070711a14bb5f4b8" +checksum = "e2422ad645d89c99f8f3e6b88a9fdeca7fabeac836b1002371c4367c8f984aae" dependencies = [ "unicode-ident", ] @@ -2534,11 +2544,11 @@ dependencies = [ [[package]] name = "pulldown-cmark" -version = "0.9.3" +version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77a1a2f1f0a7ecff9c31abbe177637be0e97a0aef46cf8738ece09327985d998" +checksum = "57206b407293d2bcd3af849ce869d52068623f19e1b5ff8e8778e3309439682b" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.4.2", "memchr", "unicase", ] @@ -2624,9 +2634,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.33" +version = "1.0.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" dependencies = [ "proc-macro2", ] @@ -2669,9 +2679,9 @@ dependencies = [ [[package]] name = "rayon" -version = "1.8.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c27db03db7734835b3f53954b534c91069375ce6ccaa2e065441e07d9b6cdb1" +checksum = "fa7237101a77a10773db45d62004a272517633fbcc3df19d96455ede1122e051" dependencies = [ "either", "rayon-core", @@ -2679,9 +2689,9 @@ dependencies = [ [[package]] name = "rayon-core" -version = "1.12.0" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ce3fb6ad83f861aac485e76e1985cd109d9a3713802152be56c3b1f0e0658ed" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" dependencies = [ "crossbeam-deque", "crossbeam-utils", @@ -2722,13 +2732,13 @@ dependencies = [ [[package]] name = "regex" -version = "1.10.2" +version = "1.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343" +checksum = "b62dbe01f0b06f9d8dc7d49e05a0785f153b00b2c227856282f671e0318c9b15" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.4.3", + "regex-automata 0.4.5", "regex-syntax 0.8.2", ] @@ -2750,9 +2760,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.3" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f" +checksum = "5bb987efffd3c6d0d8f5f89510bb458559eab11e4f869acb20bf845e016259cd" dependencies = [ "aho-corasick", "memchr", @@ -2799,11 +2809,11 @@ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" [[package]] name = "rustix" -version = "0.38.28" +version = "0.38.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72e572a5e8ca657d7366229cdde4bd14c4eb5499a9573d4d366fe1b599daa316" +checksum = "6ea3e1a662af26cd7a3ba09c0297a31af215563ecf42817c98df621387f4e949" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.4.2", "errno", "libc", "linux-raw-sys", @@ -2848,9 +2858,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.20" +version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "836fa6a3e1e547f9a2c4040802ec865b5d85f4014efe00555d7090a3dcaa1090" +checksum = "b97ed7a9823b74f99c7742f5336af7be5ecd3eeafcb1507d1fa93347b1d589b0" dependencies = [ "serde", ] @@ -2866,9 +2876,9 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.193" +version = "1.0.196" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25dd9975e68d0cb5aa1120c288333fc98731bd1dd12f561e468ea4728c042b89" +checksum = "870026e60fa08c69f064aa766c10f10b1d62db9ccd4d0abb206472bee0ce3b32" dependencies = [ "serde_derive", ] @@ -2886,22 +2896,22 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.193" +version = "1.0.196" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3" +checksum = "33c85360c95e7d137454dc81d9a4ed2b8efd8fbe19cee57357b32b9771fccb67" dependencies = [ "proc-macro2", "quote", - "syn 2.0.43", + "syn 2.0.48", ] [[package]] name = "serde_json" -version = "1.0.108" +version = "1.0.113" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b" +checksum = "69801b70b1c3dac963ecb03a364ba0ceda9cf60c71cfe475e99864759c8b8a79" dependencies = [ - "indexmap 2.1.0", + "indexmap 2.2.2", "itoa", "ryu", "serde", @@ -2990,9 +3000,9 @@ dependencies = [ [[package]] name = "similar" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2aeaf503862c419d66959f5d7ca015337d864e9c49485d771b732e2a20453597" +checksum = "32fea41aca09ee824cc9724996433064c89f7777e60762749a4170a14abbfa21" dependencies = [ "bstr 0.2.17", "unicode-segmentation", @@ -3037,9 +3047,9 @@ checksum = "826167069c09b99d56f31e9ae5c99049e932a98c9dc2dac47645b08dbbf76ba7" [[package]] name = "smallvec" -version = "1.11.2" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dccd0940a2dcdf68d092b8cbab7dc0ad8fa938bf95787e1b916b0e3d0e8e970" +checksum = "e6ecd384b10a64542d77071bd64bd7b231f4ed5940fba55e98c3de13824cf3d7" dependencies = [ "serde", ] @@ -3083,7 +3093,7 @@ dependencies = [ "proc-macro2", "quote", "structmeta-derive", - "syn 2.0.43", + "syn 2.0.48", ] [[package]] @@ -3094,7 +3104,7 @@ checksum = "a60bcaff7397072dca0017d1db428e30d5002e00b6847703e2e42005c95fbe00" dependencies = [ "proc-macro2", "quote", - "syn 2.0.43", + "syn 2.0.48", ] [[package]] @@ -3113,7 +3123,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.43", + "syn 2.0.48", ] [[package]] @@ -3174,9 +3184,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.43" +version = "2.0.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee659fb5f3d355364e1f3e5bc10fb82068efbf824a1e9d1c9504244a6469ad53" +checksum = "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f" dependencies = [ "proc-macro2", "quote", @@ -3191,9 +3201,9 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] name = "target-lexicon" -version = "0.12.12" +version = "0.12.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14c39fd04924ca3a864207c66fc2cd7d22d7c016007f9ce846cbb9326331930a" +checksum = "69758bda2e78f098e4ccb393021a0963bb3442eac05f135c30f61b7370bbafae" [[package]] name = "tempfile" @@ -3251,7 +3261,7 @@ dependencies = [ "nix 0.24.3", "num-derive 0.3.3", "num-traits", - "ordered-float", + "ordered-float 3.9.2", "pest", "pest_derive", "phf 0.10.1", @@ -3268,37 +3278,38 @@ dependencies = [ "vtparse", "wezterm-bidi", "wezterm-color-types", - "wezterm-dynamic", + "wezterm-dynamic 0.1.0", "winapi", ] [[package]] name = "thiserror" -version = "1.0.52" +version = "1.0.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83a48fd946b02c0a526b2e9481c8e2a17755e47039164a86c4070446e3a4614d" +checksum = "d54378c645627613241d077a3a79db965db602882668f9136ac42af9ecb730ad" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.52" +version = "1.0.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7fbe9b594d6568a6a1443250a7e67d80b74e1e96f6d1715e1e21cc1888291d3" +checksum = "fa0faa943b50f3db30a20aa7e265dbc66076993efed8463e8de414e5d06d3471" dependencies = [ "proc-macro2", "quote", - "syn 2.0.43", + "syn 2.0.48", ] [[package]] name = "time" -version = "0.3.31" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f657ba42c3f86e7680e53c8cd3af8abbe56b5491790b46e22e19c0d57463583e" +checksum = "c8248b6521bb14bc45b4067159b9b6ad792e2d6d754d6c41fb50e29fefe38749" dependencies = [ "deranged", + "num-conv", "powerfmt", "serde", "time-core", @@ -3403,9 +3414,9 @@ checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" [[package]] name = "uuid" -version = "1.6.1" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e395fcf16a7a3d8127ec99782007af141946b4795001f876d54fb0d55978560" +checksum = "f00cc9702ca12d3c81455259621e676d0f7251cec66a21e98fe2e9a37db93b2a" dependencies = [ "getrandom", ] @@ -3471,9 +3482,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.89" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ed0d4f68a3015cc185aff4db9506a015f4b96f95303897bfa23f846db54064e" +checksum = "b1223296a201415c7fad14792dbefaace9bd52b62d33453ade1c5b5f07555406" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -3481,24 +3492,24 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.89" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b56f625e64f3a1084ded111c4d5f477df9f8c92df113852fa5a374dbda78826" +checksum = "fcdc935b63408d58a32f8cc9738a0bffd8f05cc7c002086c6ef20b7312ad9dcd" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn 2.0.43", + "syn 2.0.48", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.89" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0162dbf37223cd2afce98f3d0785506dcb8d266223983e4b5b525859e6e182b2" +checksum = "3e4c238561b2d428924c49815533a8b9121c664599558a5d9ec51f8a1740a999" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3506,22 +3517,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.89" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0eb82fcb7930ae6219a7ecfd55b217f5f0893484b7a13022ebb2b2bf20b5283" +checksum = "bae1abb6806dc1ad9e560ed242107c0f6c84335f1749dd4e8ddb012ebd5e25a7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.43", + "syn 2.0.48", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.89" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ab9b36309365056cd639da3134bf87fa8f3d86008abf99e612384a6eecd459f" +checksum = "4d91413b1c31d7539ba5ef2451af3f0b833a005eb27a631cec32bc0635a8602b" [[package]] name = "wasm-encoder" @@ -3543,9 +3554,9 @@ dependencies = [ [[package]] name = "wasm-encoder" -version = "0.38.1" +version = "0.41.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ad2b51884de9c7f4fe2fd1043fccb8dcad4b1e29558146ee57a144d15779f3f" +checksum = "e09bca7d6388637d27fb5edbeab11f56bfabcef8743c55ae34370e1e5030a071" dependencies = [ "leb128", ] @@ -3562,28 +3573,29 @@ version = "0.116.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a58e28b80dd8340cb07b8242ae654756161f6fc8d0038123d679b7b99964fa50" dependencies = [ - "indexmap 2.1.0", - "semver 1.0.20", + "indexmap 2.2.2", + "semver 1.0.21", ] [[package]] name = "wasmparser" -version = "0.118.1" +version = "0.121.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95ee9723b928e735d53000dec9eae7b07a60e490c85ab54abb66659fc61bfcd9" +checksum = "953cf6a7606ab31382cb1caa5ae403e77ba70c7f8e12eeda167e7040d42bfda8" dependencies = [ - "indexmap 2.1.0", - "semver 1.0.20", + "bitflags 2.4.2", + "indexmap 2.2.2", + "semver 1.0.21", ] [[package]] name = "wasmprinter" -version = "0.2.75" +version = "0.2.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d027eb8294904fc715ac0870cebe6b0271e96b90605ee21511e7565c4ce568c" +checksum = "05e32c13c59fdc64d3f6998a1d52eb1d362b6904a88b754190ccb85661ad577a" dependencies = [ "anyhow", - "wasmparser 0.118.1", + "wasmparser 0.121.0", ] [[package]] @@ -3598,7 +3610,7 @@ dependencies = [ "bumpalo", "cfg-if", "fxprof-processed-profile", - "indexmap 2.1.0", + "indexmap 2.2.2", "libc", "log", "object", @@ -3639,7 +3651,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aba5bf44d044d25892c03fb3534373936ee204141ff92bac8297787ac7f22318" dependencies = [ "anyhow", - "base64 0.21.5", + "base64 0.21.7", "bincode", "directories-next", "log", @@ -3661,7 +3673,7 @@ dependencies = [ "anyhow", "proc-macro2", "quote", - "syn 2.0.43", + "syn 2.0.48", "wasmtime-component-util", "wasmtime-wit-bindgen", "wit-parser", @@ -3723,7 +3735,7 @@ dependencies = [ "anyhow", "cranelift-entity", "gimli 0.28.1", - "indexmap 2.1.0", + "indexmap 2.2.2", "log", "object", "serde", @@ -3808,7 +3820,7 @@ dependencies = [ "anyhow", "cc", "cfg-if", - "indexmap 2.1.0", + "indexmap 2.2.2", "libc", "log", "mach", @@ -3849,7 +3861,7 @@ checksum = "f50f51f8d79bfd2aa8e9d9a0ae7c2d02b45fe412e62ff1b87c0c81b07c738231" dependencies = [ "proc-macro2", "quote", - "syn 2.0.43", + "syn 2.0.48", ] [[package]] @@ -3860,7 +3872,7 @@ checksum = "4b804dfd3d0c0d6d37aa21026fe7772ba1a769c89ee4f5c4f13b82d91d75216f" dependencies = [ "anyhow", "heck 0.4.1", - "indexmap 2.1.0", + "indexmap 2.2.2", "wit-parser", ] @@ -3872,30 +3884,31 @@ checksum = "9b6060bc082cc32d9a45587c7640e29e3c7b89ada82677ac25d87850aaccb368" [[package]] name = "wast" -version = "69.0.1" +version = "70.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1ee37317321afde358e4d7593745942c48d6d17e0e6e943704de9bbee121e7a" +checksum = "a3d5061300042ff5065123dae1e27d00c03f567d34a2937c8472255148a216dc" dependencies = [ + "bumpalo", "leb128", "memchr", "unicode-width", - "wasm-encoder 0.38.1", + "wasm-encoder 0.41.0", ] [[package]] name = "wat" -version = "1.0.82" +version = "1.0.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aeb338ee8dee4d4cd05e6426683f21c5087dc7cfc8903e839ccf48d43332da3c" +checksum = "afd7357b6cc46d46a2509c43dcb1dd4131dafbf4e75562d87017b5a05ffad2d6" dependencies = [ "wast", ] [[package]] name = "web-sys" -version = "0.3.66" +version = "0.3.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50c24a44ec86bb68fbecd1b3efed7e85ea5621b39b35ef2766b66cd984f8010f" +checksum = "58cd2333b6e0be7a39605f0e255892fd7418a682d8da8fe042fe25128794d2ed" dependencies = [ "js-sys", "wasm-bindgen", @@ -3903,12 +3916,12 @@ dependencies = [ [[package]] name = "wezterm-bidi" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1560382cf39b0fa92473eae4d5b3772f88c63202cbf5a72c35db72ba99e66c36" +checksum = "0c0a6e355560527dd2d1cf7890652f4f09bb3433b6aadade4c9b5ed76de5f3ec" dependencies = [ "log", - "wezterm-dynamic", + "wezterm-dynamic 0.2.0", ] [[package]] @@ -3920,7 +3933,7 @@ dependencies = [ "csscolorparser", "deltae", "lazy_static", - "wezterm-dynamic", + "wezterm-dynamic 0.1.0", ] [[package]] @@ -3930,7 +3943,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a75e78c0cc60a76de5d93f9dad05651105351e151b6446ab305514945d7588aa" dependencies = [ "log", - "ordered-float", + "ordered-float 3.9.2", + "strsim", + "thiserror", + "wezterm-dynamic-derive", +] + +[[package]] +name = "wezterm-dynamic" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfb128bacfa86734e07681fb6068e34c144698e84ee022d6e009145d1abb77b5" +dependencies = [ + "log", + "ordered-float 4.2.0", "strsim", "thiserror", "wezterm-dynamic-derive", @@ -3961,9 +3987,9 @@ dependencies = [ [[package]] name = "wild" -version = "2.2.0" +version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10d01931a94d5a115a53f95292f51d316856b68a035618eb831bbba593a30b67" +checksum = "a3131afc8c575281e1e80f36ed6a092aa502c08b18ed7524e86fbbb12bb410e1" dependencies = [ "glob", ] @@ -4001,11 +4027,11 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windows-core" -version = "0.51.1" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1f8cf84f35d2db49a46868f947758c7a1138116f7fac3bc844f43ade1292e64" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" dependencies = [ - "windows-targets 0.48.5", + "windows-targets 0.52.0", ] [[package]] @@ -4023,15 +4049,6 @@ dependencies = [ "windows_x86_64_msvc 0.42.2", ] -[[package]] -name = "windows-sys" -version = "0.45.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" -dependencies = [ - "windows-targets 0.42.2", -] - [[package]] name = "windows-sys" version = "0.48.0" @@ -4050,21 +4067,6 @@ dependencies = [ "windows-targets 0.52.0", ] -[[package]] -name = "windows-targets" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" -dependencies = [ - "windows_aarch64_gnullvm 0.42.2", - "windows_aarch64_msvc 0.42.2", - "windows_i686_gnu 0.42.2", - "windows_i686_msvc 0.42.2", - "windows_x86_64_gnu 0.42.2", - "windows_x86_64_gnullvm 0.42.2", - "windows_x86_64_msvc 0.42.2", -] - [[package]] name = "windows-targets" version = "0.48.5" @@ -4223,15 +4225,15 @@ checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" [[package]] name = "wit-parser" -version = "0.13.0" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15df6b7b28ce94b8be39d8df5cb21a08a4f3b9f33b631aedb4aa5776f785ead3" +checksum = "df4913a2219096373fd6512adead1fb77ecdaa59d7fc517972a7d30b12f625be" dependencies = [ "anyhow", "id-arena", - "indexmap 2.1.0", + "indexmap 2.2.2", "log", - "semver 1.0.20", + "semver 1.0.21", "serde", "serde_derive", "serde_json", @@ -4274,7 +4276,7 @@ version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3a33a79127a0f98c6f273af4f421dafaea80620163ea4f6b3ef4df0a6e08b77" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.4.2", "lazy_static", "thiserror", "yara-sys", @@ -4302,7 +4304,7 @@ dependencies = [ "array-bytes", "ascii_tree", "authenticode-parser", - "base64 0.21.5", + "base64 0.21.7", "bincode", "bitmask", "bitvec", @@ -4314,9 +4316,9 @@ dependencies = [ "globwalk", "goldenfile", "ihex", - "indexmap 2.1.0", + "indexmap 2.2.2", "intaglio", - "itertools 0.12.0", + "itertools 0.12.1", "lazy_static", "lingua", "linkme", @@ -4411,11 +4413,11 @@ version = "0.1.0" dependencies = [ "ariadne", "ascii_tree", - "base64 0.21.5", + "base64 0.21.7", "bitmask", "bstr 1.9.0", "indenter", - "itertools 0.12.0", + "itertools 0.12.1", "lazy_static", "num-traits", "pest", @@ -4443,7 +4445,7 @@ dependencies = [ "chrono", "globwalk", "goldenfile", - "itertools 0.12.0", + "itertools 0.12.1", "protobuf", "protobuf-codegen", "protobuf-parse", @@ -4479,7 +4481,7 @@ checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.43", + "syn 2.0.48", ] [[package]] From c0b25aeab5fe617d99bdfecc557c52f4a2e422bc Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Sun, 4 Feb 2024 18:41:08 +0100 Subject: [PATCH 10/55] chore: enable `auto-color` feature for `env_logger` crate. --- yara-x-cli/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yara-x-cli/Cargo.toml b/yara-x-cli/Cargo.toml index 3e9c0ef7d..947b69e75 100644 --- a/yara-x-cli/Cargo.toml +++ b/yara-x-cli/Cargo.toml @@ -39,7 +39,7 @@ anyhow = { workspace = true } clap = { workspace = true, features=["cargo", "derive"] } globwalk = { workspace = true } enable-ansi-support = { workspace = true } -env_logger = { workspace = true , optional = true } +env_logger = { workspace = true , optional = true, features = ["auto-color"] } log = { workspace = true, optional = true } protobuf = { workspace = true } protobuf-json-mapping = { workspace = true } From 065d2f96332a6a3cc42842869326ad79163983d6 Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Sun, 4 Feb 2024 18:44:23 +0100 Subject: [PATCH 11/55] ci: upgrade `cargo-udeps` to version 0.1.45 --- .github/workflows/code_health.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/code_health.yaml b/.github/workflows/code_health.yaml index 4be86fc90..f85254e3e 100644 --- a/.github/workflows/code_health.yaml +++ b/.github/workflows/code_health.yaml @@ -36,5 +36,5 @@ jobs: - name: Run cargo-udeps uses: aig787/cargo-udeps-action@v1 with: - version: v0.1.35 + version: v0.1.45 args: '--all-targets' \ No newline at end of file From ee7557595850870671e685f7237ffb6547271173 Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Sun, 4 Feb 2024 18:50:57 +0100 Subject: [PATCH 12/55] chore: upgrade `wasmtime` to version 17.0.0 --- Cargo.lock | 166 +++++++++++++++++++++++------------------------------ Cargo.toml | 2 +- 2 files changed, 73 insertions(+), 95 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fe45ec8a5..2b7c36a69 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -700,18 +700,18 @@ dependencies = [ [[package]] name = "cranelift-bforest" -version = "0.102.1" +version = "0.104.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e7e56668d2263f92b691cb9e4a2fcb186ca0384941fe420484322fa559c3329" +checksum = "d819feeda4c420a18f1e28236ca0ce1177b22bf7c8a44ddee92dfe40de15bcf0" dependencies = [ "cranelift-entity", ] [[package]] name = "cranelift-codegen" -version = "0.102.1" +version = "0.104.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a9ff61938bf11615f55b80361288c68865318025632ea73c65c0b44fa16283c" +checksum = "e9b8d03d5bdbca7e5f72b0e0a0f69933ed1f09e24be6c075aa6fe3f802b0cc0c" dependencies = [ "bumpalo", "cranelift-bforest", @@ -730,33 +730,33 @@ dependencies = [ [[package]] name = "cranelift-codegen-meta" -version = "0.102.1" +version = "0.104.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50656bf19e3d4a153b404ff835b8b59e924cfa3682ebe0d3df408994f37983f6" +checksum = "a3fd3664e38e51649b17dc30cfdd561273fe2f590dcd013fb75d9eabc6272dfb" dependencies = [ "cranelift-codegen-shared", ] [[package]] name = "cranelift-codegen-shared" -version = "0.102.1" +version = "0.104.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "388041deeb26109f1ea73c1812ea26bfd406c94cbce0bb5230aa44277e43b209" +checksum = "4b031ec5e605828975952622b5a77d49126f20ffe88d33719a0af66b23a0fc36" [[package]] name = "cranelift-control" -version = "0.102.1" +version = "0.104.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b39b7c512ffac527e5b5df9beae3d67ab85d07dca6d88942c16195439fedd1d3" +checksum = "fada054d017cf2ed8f7ed2336e0517fc1b19e6825be1790de9eb00c94788362b" dependencies = [ "arbitrary", ] [[package]] name = "cranelift-entity" -version = "0.102.1" +version = "0.104.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdb25f573701284fe2bcf88209d405342125df00764b396c923e11eafc94d892" +checksum = "177b6f94ae8de6348eb45bf977c79ab9e3c40fc3ac8cb7ed8109560ea39bee7d" dependencies = [ "serde", "serde_derive", @@ -764,9 +764,9 @@ dependencies = [ [[package]] name = "cranelift-frontend" -version = "0.102.1" +version = "0.104.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e57374fd11d72cf9ffb85ff64506ed831440818318f58d09f45b4185e5e9c376" +checksum = "ebebd23a69a23e3ddea78e98ff3a2de222e88c8e045d81ef4a72f042e0d79dbd" dependencies = [ "cranelift-codegen", "log", @@ -776,15 +776,15 @@ dependencies = [ [[package]] name = "cranelift-isle" -version = "0.102.1" +version = "0.104.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae769b235f6ea2f86623a3ff157cc04a4ff131dc9fe782c2ebd35f272043581e" +checksum = "1571bfc14df8966d12c6121b5325026591a4b4009e22fea0fe3765ab7cd33b96" [[package]] name = "cranelift-native" -version = "0.102.1" +version = "0.104.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3dc7bfb8f13a0526fe20db338711d9354729b861c336978380bb10f7f17dd207" +checksum = "35a69c37e0c10b46fe5527f2397ac821046efbf5f7ec112c8b84df25712f465b" dependencies = [ "cranelift-codegen", "libc", @@ -793,9 +793,9 @@ dependencies = [ [[package]] name = "cranelift-wasm" -version = "0.102.1" +version = "0.104.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c5f41a4af931b756be05af0dd374ce200aae2d52cea16b0beb07e8b52732c35" +checksum = "9b3fef8bbceb8cb56d3f1778b0418d75c5cf12ec571a35fc01eb41abb0227a25" dependencies = [ "cranelift-codegen", "cranelift-entity", @@ -803,7 +803,7 @@ dependencies = [ "itertools 0.10.5", "log", "smallvec", - "wasmparser 0.116.1", + "wasmparser 0.118.1", "wasmtime-types", ] @@ -2446,12 +2446,6 @@ dependencies = [ "thiserror", ] -[[package]] -name = "ppv-lite86" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" - [[package]] name = "pretty_assertions" version = "1.4.0" @@ -2653,18 +2647,6 @@ version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ - "libc", - "rand_chacha", - "rand_core", -] - -[[package]] -name = "rand_chacha" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" -dependencies = [ - "ppv-lite86", "rand_core", ] @@ -2673,9 +2655,6 @@ name = "rand_core" version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" -dependencies = [ - "getrandom", -] [[package]] name = "rayon" @@ -3545,9 +3524,9 @@ dependencies = [ [[package]] name = "wasm-encoder" -version = "0.36.2" +version = "0.38.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "822b645bf4f2446b949776ffca47e2af60b167209ffb70814ef8779d299cd421" +checksum = "0ad2b51884de9c7f4fe2fd1043fccb8dcad4b1e29558146ee57a144d15779f3f" dependencies = [ "leb128", ] @@ -3569,9 +3548,9 @@ checksum = "449167e2832691a1bff24cde28d2804e90e09586a448c8e76984792c44334a6b" [[package]] name = "wasmparser" -version = "0.116.1" +version = "0.118.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a58e28b80dd8340cb07b8242ae654756161f6fc8d0038123d679b7b99964fa50" +checksum = "95ee9723b928e735d53000dec9eae7b07a60e490c85ab54abb66659fc61bfcd9" dependencies = [ "indexmap 2.2.2", "semver 1.0.21", @@ -3600,9 +3579,9 @@ dependencies = [ [[package]] name = "wasmtime" -version = "15.0.1" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "642e12d108e800215263e3b95972977f473957923103029d7d617db701d67ba4" +checksum = "910fabce77e660f0e0e41cfd5f69fc8bf020a025f059718846e918db7177f469" dependencies = [ "anyhow", "async-trait", @@ -3616,14 +3595,13 @@ dependencies = [ "object", "once_cell", "paste", - "psm", "rayon", "serde", "serde_derive", "serde_json", "target-lexicon", - "wasm-encoder 0.36.2", - "wasmparser 0.116.1", + "wasm-encoder 0.38.1", + "wasmparser 0.118.1", "wasmtime-cache", "wasmtime-component-macro", "wasmtime-cranelift", @@ -3632,23 +3610,23 @@ dependencies = [ "wasmtime-jit", "wasmtime-runtime", "wat", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "wasmtime-asm-macros" -version = "15.0.1" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "beada8bb15df52503de0a4c58de4357bfd2f96d9a44a6e547bad11efdd988b47" +checksum = "37288142e9b4a61655a3bcbdc7316c2e4bb9e776b10ce3dd758f8186b4469572" dependencies = [ "cfg-if", ] [[package]] name = "wasmtime-cache" -version = "15.0.1" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aba5bf44d044d25892c03fb3534373936ee204141ff92bac8297787ac7f22318" +checksum = "45cbd74a636f09d2108f9405c79857f061e19323e4abeed22e837cfe7b08a22b" dependencies = [ "anyhow", "base64 0.21.7", @@ -3660,15 +3638,15 @@ dependencies = [ "serde_derive", "sha2 0.10.8", "toml", - "windows-sys 0.48.0", + "windows-sys 0.52.0", "zstd", ] [[package]] name = "wasmtime-component-macro" -version = "15.0.1" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56ccba556991465cca68d5a54769684bcf489fb532059da55105f851642d52c1" +checksum = "ad63de18eb42e586386b6091f787c82707cbd5ac5e9343216dba1976190cd03a" dependencies = [ "anyhow", "proc-macro2", @@ -3681,15 +3659,15 @@ dependencies = [ [[package]] name = "wasmtime-component-util" -version = "15.0.1" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05492a177a6006cb73f034d6e9a6fad6da55b23c4398835cb0012b5fa51ecf67" +checksum = "7e0a160c0c44369aa4bee6d311a8e4366943bab1651040cc8b0fcec2c9eb8906" [[package]] name = "wasmtime-cranelift" -version = "15.0.1" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe2e7532f1d6adbcc57e69bb6a7c503f0859076d07a9b4b6aabe8021ff8a05fd" +checksum = "3734cc01b7cd37bc62fdbcd9529ca9547440052d4b3886cfdec3b8081a5d3647" dependencies = [ "anyhow", "cfg-if", @@ -3704,7 +3682,7 @@ dependencies = [ "object", "target-lexicon", "thiserror", - "wasmparser 0.116.1", + "wasmparser 0.118.1", "wasmtime-cranelift-shared", "wasmtime-environ", "wasmtime-versioned-export-macros", @@ -3712,9 +3690,9 @@ dependencies = [ [[package]] name = "wasmtime-cranelift-shared" -version = "15.0.1" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c98d5378a856cbf058d36278627dfabf0ed68a888142958c7ae8e6af507dafa" +checksum = "e0eb33cd30c47844aa228d4d0030587e65c1108343f311fe9f7248b5bd9cb65c" dependencies = [ "anyhow", "cranelift-codegen", @@ -3728,9 +3706,9 @@ dependencies = [ [[package]] name = "wasmtime-environ" -version = "15.0.1" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6d33a9f421da810a070cd56add9bc51f852bd66afbb8b920489d6242f15b70e" +checksum = "9a3a056b041fdea604f0972e2fae97958e7748d629a55180228348baefdfc217" dependencies = [ "anyhow", "cranelift-entity", @@ -3742,15 +3720,15 @@ dependencies = [ "serde_derive", "target-lexicon", "thiserror", - "wasmparser 0.116.1", + "wasmparser 0.118.1", "wasmtime-types", ] [[package]] name = "wasmtime-fiber" -version = "15.0.1" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "404741f4c6d7f4e043be2e8b466406a2aee289ccdba22bf9eba6399921121b97" +checksum = "43987d0977c07f15c3608c2f255870c127ffd19e35eeedb1ac1dccedf9932a42" dependencies = [ "anyhow", "cc", @@ -3758,14 +3736,14 @@ dependencies = [ "rustix", "wasmtime-asm-macros", "wasmtime-versioned-export-macros", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "wasmtime-jit" -version = "15.0.1" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d0994a86d6dca5f7d9740d7f2bd0568be06d2014a550361dc1c397d289d81ef" +checksum = "9b3e48395ac672b386ed588d97a9612aa13a345008f26466f0dfb2a91628aa9f" dependencies = [ "addr2line", "anyhow", @@ -3785,14 +3763,14 @@ dependencies = [ "wasmtime-jit-debug", "wasmtime-jit-icache-coherence", "wasmtime-runtime", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "wasmtime-jit-debug" -version = "15.0.1" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e0c4b74e606d1462d648631d5bc328e3d5b14e7f9d3ff93bc6db062fb8c5cd8" +checksum = "dd21fd0f5ca68681d3d5b636eea00f182d0f9d764144469e9257fd7e3f55ae0e" dependencies = [ "object", "once_cell", @@ -3802,20 +3780,20 @@ dependencies = [ [[package]] name = "wasmtime-jit-icache-coherence" -version = "15.0.1" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3090a69ba1476979e090aa7ed4bc759178bafdb65b22f98b9ba24fc6e7e578d5" +checksum = "bdc26415bb89e9ccd3bdc498fef63aabf665c4c0dd710c107691deb9694955da" dependencies = [ "cfg-if", "libc", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "wasmtime-runtime" -version = "15.0.1" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b993ac8380385ed67bf71b51b9553edcf1ab0801b78a805a067de581b9a3e88a" +checksum = "0abddaf17912aabaf39be0802d5eba9a002e956e902d1ebd438a2fe1c88769a2" dependencies = [ "anyhow", "cc", @@ -3827,37 +3805,37 @@ dependencies = [ "memfd", "memoffset 0.9.0", "paste", - "rand", + "psm", "rustix", "sptr", - "wasm-encoder 0.36.2", + "wasm-encoder 0.38.1", "wasmtime-asm-macros", "wasmtime-environ", "wasmtime-fiber", "wasmtime-jit-debug", "wasmtime-versioned-export-macros", "wasmtime-wmemcheck", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "wasmtime-types" -version = "15.0.1" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b5778112fcab2dc3d4371f4203ab8facf0c453dd94312b0a88dd662955e64e0" +checksum = "b35a95cdc1433729085beab42c0a5c742b431f25b17c40d7718e46df63d5ffc7" dependencies = [ "cranelift-entity", "serde", "serde_derive", "thiserror", - "wasmparser 0.116.1", + "wasmparser 0.118.1", ] [[package]] name = "wasmtime-versioned-export-macros" -version = "15.0.1" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f50f51f8d79bfd2aa8e9d9a0ae7c2d02b45fe412e62ff1b87c0c81b07c738231" +checksum = "fad322733fe67e45743784d8b1df452bcb54f581572a4f1a646a4332deecbcc2" dependencies = [ "proc-macro2", "quote", @@ -3866,9 +3844,9 @@ dependencies = [ [[package]] name = "wasmtime-wit-bindgen" -version = "15.0.1" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b804dfd3d0c0d6d37aa21026fe7772ba1a769c89ee4f5c4f13b82d91d75216f" +checksum = "41e5675998fdc74495afdd90ad2bd221206a258075b23048af0535a969b07893" dependencies = [ "anyhow", "heck 0.4.1", @@ -3878,9 +3856,9 @@ dependencies = [ [[package]] name = "wasmtime-wmemcheck" -version = "15.0.1" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b6060bc082cc32d9a45587c7640e29e3c7b89ada82677ac25d87850aaccb368" +checksum = "b20a19e10d8cb50b45412fb21192982b7ce85c0122dc33bb71f1813e25dc6e52" [[package]] name = "wast" diff --git a/Cargo.toml b/Cargo.toml index c029da4e2..7bb9c5c01 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -82,7 +82,7 @@ serde_json = "1.0" thiserror = "1.0.50" uuid = "1.4.1" walrus = "0.20.2" -wasmtime = "15.0.1" +wasmtime = "17.0.0" yaml-rust = "0.4.5" yansi = "0.5.1" yara-x = { path = "yara-x" } From 99fee50044d10586f1ea50596da97e198d7d6134 Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Sun, 4 Feb 2024 19:02:29 +0100 Subject: [PATCH 13/55] chore: upgrade `yara` crate to version 0.24.0 --- Cargo.lock | 8 ++++---- yara-x/Cargo.toml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2b7c36a69..09c6f367f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4250,9 +4250,9 @@ checksum = "1367295b8f788d371ce2dbc842c7b709c73ee1364d30351dd300ec2203b12377" [[package]] name = "yara" -version = "0.23.0" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3a33a79127a0f98c6f273af4f421dafaea80620163ea4f6b3ef4df0a6e08b77" +checksum = "b40e0a3ad51d2ae8cb016c38bd5af7885352094caf1bbeb5ba37e06d0247e605" dependencies = [ "bitflags 2.4.2", "lazy_static", @@ -4262,9 +4262,9 @@ dependencies = [ [[package]] name = "yara-sys" -version = "0.23.0" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8e3e4616bc8bb87d2b162b984cf3d023132f10c64c669c2a67261b08bff3af4" +checksum = "7b155296fb1c32113e689085ab8c249d774910f4e600f8eb612793a59cea4ff1" dependencies = [ "bindgen", "cc", diff --git a/yara-x/Cargo.toml b/yara-x/Cargo.toml index a2e8abfc9..fe45bf5e8 100644 --- a/yara-x/Cargo.toml +++ b/yara-x/Cargo.toml @@ -189,7 +189,7 @@ ihex = "3.0.0" goldenfile = "1.6.0" pretty_assertions = { workspace = true } wasmprinter = "0.2.73" -yara = { version = "0.23.0", features = ["vendored"] } +yara = { version = "0.24.0", features = ["vendored"] } yara-x-proto-yaml = { workspace = true } zip = "0.6.6" From db30d885bbc201f25ccda16b984ee1f58562a0f1 Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Mon, 5 Feb 2024 12:25:16 +0100 Subject: [PATCH 14/55] ci: add missing token to code coverage action --- .github/workflows/coverage.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/coverage.yaml b/.github/workflows/coverage.yaml index 1bbc1fd29..abc60e491 100644 --- a/.github/workflows/coverage.yaml +++ b/.github/workflows/coverage.yaml @@ -29,5 +29,6 @@ jobs: - name: Upload coverage to Codecov uses: codecov/codecov-action@v4 with: + token: ${{ secrets.CODECOV_TOKEN }} files: lcov.info fail_ci_if_error: true \ No newline at end of file From 698a510be91c24565b0691b975af08611b1e40a7 Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Mon, 5 Feb 2024 12:40:39 +0100 Subject: [PATCH 15/55] ci: run the coverage GitHub workflow when `coverage.yaml` is changed. --- .github/workflows/coverage.yaml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/coverage.yaml b/.github/workflows/coverage.yaml index abc60e491..da517c782 100644 --- a/.github/workflows/coverage.yaml +++ b/.github/workflows/coverage.yaml @@ -1,10 +1,13 @@ name: Code coverage with grcov on: - schedule: # Run at every day at 2:15. + schedule: - cron: '15 2 * * *' - + # Also run then this file is changed. + push: + paths: + - '.github/workflows/coverage.yaml' jobs: coverage: name: Coverage From 1956ccd7e6ee186425bd6d9d6f492f63e84ad7c1 Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Mon, 5 Feb 2024 13:11:02 +0100 Subject: [PATCH 16/55] chore: turn off clippy warning --- yara-x/src/modules/dotnet/parser.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/yara-x/src/modules/dotnet/parser.rs b/yara-x/src/modules/dotnet/parser.rs index b67daeaed..a752d51a6 100644 --- a/yara-x/src/modules/dotnet/parser.rs +++ b/yara-x/src/modules/dotnet/parser.rs @@ -2193,6 +2193,7 @@ struct StreamHeader<'a> { struct BlobIndex(u32); /// An index into the `#GUID` stream. +#[allow(dead_code)] #[derive(Clone, Copy)] struct GuidIndex(u32); From d772a67637bb49568e0ccd104a44f799aa8a7f89 Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Mon, 5 Feb 2024 13:27:32 +0100 Subject: [PATCH 17/55] chore: add profile for building the binary with link-time optimization --- Cargo.toml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 7bb9c5c01..25ed777a4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -92,8 +92,12 @@ yara-x-parser = { path = "yara-x-parser" } yara-x-proto = { path = "yara-x-proto" } yara-x-proto-yaml = { path = "yara-x-proto-yaml" } - -[profile.release] -# debug = 1 # Include debug information in the binary. -# lto = true # Enable link-time optimization. -# strip = true # Automatically strip symbols from the binary. \ No newline at end of file +# Special profile that builds a release binary with link-time optimization. +# Compiling with this profile takes a while, but the resulting binary is +# smaller and better optimized. For building with this profile use: +# +# cargo build --profile release-lto +[profile.release-lto] +inherits = "release" +lto = true +codegen-units = 1 From 244393665c4eac827b04045c159fd65e1a6a23b1 Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Mon, 5 Feb 2024 17:10:20 +0100 Subject: [PATCH 18/55] chore: remove dependency from `byteorder` --- Cargo.lock | 1 - Cargo.toml | 1 - yara-x/Cargo.toml | 2 -- yara-x/src/modules/pe/parser.rs | 5 ++--- 4 files changed, 2 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 09c6f367f..6b7c7fce4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4287,7 +4287,6 @@ dependencies = [ "bitmask", "bitvec", "bstr 1.9.0", - "byteorder", "crc32fast", "criterion", "fmmap", diff --git a/Cargo.toml b/Cargo.toml index 25ed777a4..bd81b684c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -39,7 +39,6 @@ bincode = "1.3.3" bitmask = "0.5.0" bitvec = "1.0.1" bstr = "1.8.0" -byteorder = "1.5.0" clap = "4.4.8" crc32fast = "1.3.2" criterion = "0.5.1" diff --git a/yara-x/Cargo.toml b/yara-x/Cargo.toml index fe45bf5e8..e4209c5ce 100644 --- a/yara-x/Cargo.toml +++ b/yara-x/Cargo.toml @@ -77,7 +77,6 @@ lnk-module = [ # The `macho` module parses Mach-O files. macho-module = [ - "dep:byteorder", "dep:nom", ] @@ -140,7 +139,6 @@ bincode = { workspace = true } bitmask = { workspace = true } bitvec = { workspace = true } bstr = { workspace = true, features=["serde"] } -byteorder = { workspace = true, optional = true } crc32fast = { workspace = true, optional = true } fmmap = { workspace = true } indexmap = { workspace = true, features=["serde"] } diff --git a/yara-x/src/modules/pe/parser.rs b/yara-x/src/modules/pe/parser.rs index 671be88c3..720225dde 100644 --- a/yara-x/src/modules/pe/parser.rs +++ b/yara-x/src/modules/pe/parser.rs @@ -12,7 +12,6 @@ use authenticode_parser::{ CounterSignatureVerify, }; use bstr::{BStr, ByteSlice}; -use byteorder::{ByteOrder, LE}; use itertools::Itertools; use memchr::memmem; use nom::branch::{alt, permutation}; @@ -421,7 +420,7 @@ impl<'a> PE<'a> { const IMAGE_DEBUG_TYPE_CODEVIEW: u32 = 2; const RICH_TAG: &'static [u8] = &[0x52_u8, 0x69, 0x63, 0x68]; - const DANS_TAG: &'static [u8] = &[0x44_u8, 0x61, 0x6e, 0x53]; + const DANS_TAG: u32 = 0x536e6144; const SIZE_OF_PE_SIGNATURE: usize = 4; // size of PE signature (PE\0\0). const SIZE_OF_FILE_HEADER: usize = 20; // size of IMAGE_FILE_HEADER @@ -660,7 +659,7 @@ impl<'a> PE<'a> { // Search for the "DanS" tag that indicates the start of the rich // data. This tag appears encrypted with the XOR key. - let dans_tag = key ^ LE::read_u32(Self::DANS_TAG); + let dans_tag = key ^ Self::DANS_TAG; let dans_tag_pos = memmem::rfind( &input[..rich_tag_pos], From 0443b6e24fe94b51f5c6c8e0503959b5e2859f0a Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Mon, 5 Feb 2024 17:33:20 +0100 Subject: [PATCH 19/55] chore: add feature `openssl-static` for linking OpenSSL statically. --- yara-x/Cargo.toml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/yara-x/Cargo.toml b/yara-x/Cargo.toml index e4209c5ce..a57f7171e 100644 --- a/yara-x/Cargo.toml +++ b/yara-x/Cargo.toml @@ -10,6 +10,19 @@ homepage.workspace = true rust-version.workspace = true [features] +# When this feature is enabled the OpenSSL library is linked statically. +# However, you need to specify the path where the library and its header files +# are located by setting the environment variable OPENSSL_DIR. For example: +# +# OPENSSL_DIR=/usr/local/Cellar/openssl@3/3.2.0_1/ cargo build --feature openssl-static +# +# The directory specified in OPENSSL_DIR must contain subdirectories /include +# and /lib. The latter must contain `libcrypto.a`. +# +# You can also set OPENSSL_INCLUDE_DIR and OPENSSL_LIB_DIR, for setting the +# two directories independently. +openssl-static = ['authenticode-parser/openssl-static'] + # Enables constant folding. When constant folding is enabled, expressions # like `2+2+2` and `true or false`, whose value can be determined at compile # time, will be reduced to its final value, instead of producing code that From e7374d61df19d15f0061306994e19658dad50681 Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Fri, 9 Feb 2024 13:50:10 +0100 Subject: [PATCH 20/55] chore: rename top-level directories. In the future we want to add a `go` directory that contains the Go bindings for YARA-X. Having a `yara-x-go` directory is not ideal, because the import path for that Go module would be `github.com/VirusTotal/yara-x/yara-x-go`. Instead we want the more natural `github.com/VirusTotal/yara-x/go`. However, adding a `go` directory makes the current naming incoherent, but with this change everything fits well again. --- .github/workflows/python.yaml | 8 ++--- Cargo.toml | 29 +++++++++--------- {yara-x-cli => cli}/Cargo.toml | 0 {yara-x-cli => cli}/src/commands/check.rs | 0 {yara-x-cli => cli}/src/commands/compile.rs | 0 {yara-x-cli => cli}/src/commands/debug.rs | 0 {yara-x-cli => cli}/src/commands/dump.rs | 0 {yara-x-cli => cli}/src/commands/fmt.rs | 0 {yara-x-cli => cli}/src/commands/mod.rs | 0 {yara-x-cli => cli}/src/commands/scan.rs | 4 +-- {yara-x-cli => cli}/src/help.rs | 0 {yara-x-cli => cli}/src/main.rs | 0 {yara-x-cli => cli}/src/walk.rs | 0 {yara-x-fmt => fmt}/Cargo.lock | 0 {yara-x-fmt => fmt}/Cargo.toml | 0 {yara-x-fmt => fmt}/src/align.rs | 0 {yara-x-fmt => fmt}/src/bubble.rs | 0 {yara-x-fmt => fmt}/src/comments.rs | 0 {yara-x-fmt => fmt}/src/indentation.rs | 0 {yara-x-fmt => fmt}/src/lib.rs | 0 {yara-x-fmt => fmt}/src/processor/mod.rs | 0 {yara-x-fmt => fmt}/src/processor/tests.rs | 0 .../src/testdata/test1.formatted | 0 .../src/testdata/test1.unformatted | 0 .../src/testdata/test10.formatted | 0 .../src/testdata/test10.unformatted | 0 .../src/testdata/test11.formatted | 0 .../src/testdata/test11.unformatted | 0 .../src/testdata/test12.formatted | 0 .../src/testdata/test12.unformatted | 0 .../src/testdata/test13.formatted | 0 .../src/testdata/test13.unformatted | 0 .../src/testdata/test14.formatted | 0 .../src/testdata/test14.unformatted | 0 .../src/testdata/test15.formatted | 0 .../src/testdata/test15.unformatted | 0 .../src/testdata/test16.formatted | 0 .../src/testdata/test16.unformatted | 0 .../src/testdata/test17.formatted | 0 .../src/testdata/test17.unformatted | 0 .../src/testdata/test18.formatted | 0 .../src/testdata/test18.unformatted | 0 .../src/testdata/test19.formatted | 0 .../src/testdata/test19.unformatted | 0 .../src/testdata/test2.formatted | 0 .../src/testdata/test2.unformatted | 0 .../src/testdata/test20.formatted | 0 .../src/testdata/test20.unformatted | 0 .../src/testdata/test21.formatted | 0 .../src/testdata/test21.unformatted | 0 .../src/testdata/test22.formatted | 0 .../src/testdata/test22.unformatted | 0 .../src/testdata/test23.formatted | 0 .../src/testdata/test23.unformatted | 0 .../src/testdata/test24.formatted | 0 .../src/testdata/test24.unformatted | 0 .../src/testdata/test25.formatted | 0 .../src/testdata/test25.unformatted | 0 .../src/testdata/test26.formatted | 0 .../src/testdata/test26.unformatted | 0 .../src/testdata/test27.formatted | 0 .../src/testdata/test27.unformatted | 0 .../src/testdata/test28.formatted | 0 .../src/testdata/test28.unformatted | 0 .../src/testdata/test29.formatted | 0 .../src/testdata/test29.unformatted | 0 .../src/testdata/test3.formatted | 0 .../src/testdata/test3.unformatted | 0 .../src/testdata/test4.formatted | 0 .../src/testdata/test4.unformatted | 0 .../src/testdata/test5.formatted | 0 .../src/testdata/test5.unformatted | 0 .../src/testdata/test6.formatted | 0 .../src/testdata/test6.unformatted | 0 .../src/testdata/test7.formatted | 0 .../src/testdata/test7.unformatted | 0 .../src/testdata/test8.formatted | 0 .../src/testdata/test8.unformatted | 0 .../src/testdata/test9.formatted | 0 .../src/testdata/test9.unformatted | 0 {yara-x-fmt => fmt}/src/tests.rs | 0 {yara-x-fmt => fmt}/src/tokens/mod.rs | 0 {yara-x-fmt => fmt}/src/tokens/tests.rs | 0 {yara-x-fmt => fmt}/src/trailing_spaces.rs | 0 {yara-x => lib}/Cargo.toml | 0 {yara-x => lib}/benches/benches.rs | 0 {yara-x => lib}/build.rs | 12 ++++---- {yara-x => lib}/fuzz/.gitignore | 0 {yara-x => lib}/fuzz/Cargo.lock | 0 {yara-x => lib}/fuzz/Cargo.toml | 0 .../fuzz/fuzz_targets/dotnet_parser.rs | 0 .../fuzz/fuzz_targets/elf_parser.rs | 0 .../fuzz/fuzz_targets/lnk_parser.rs | 0 .../fuzz/fuzz_targets/macho_fat_parser.rs | 0 .../fuzz/fuzz_targets/macho_parser.rs | 0 .../fuzz/fuzz_targets/pe_parser.rs | 0 .../fuzz/fuzz_targets/rule_compiler.rs | 0 {yara-x => lib}/src/compiler/atoms/mask.rs | 0 {yara-x => lib}/src/compiler/atoms/mod.rs | 0 {yara-x => lib}/src/compiler/atoms/quality.rs | 0 {yara-x => lib}/src/compiler/base64.rs | 0 {yara-x => lib}/src/compiler/context.rs | 0 {yara-x => lib}/src/compiler/emit.rs | 0 {yara-x => lib}/src/compiler/errors.rs | 0 {yara-x => lib}/src/compiler/ir/ast2ir.rs | 0 {yara-x => lib}/src/compiler/ir/hex2hir.rs | 0 {yara-x => lib}/src/compiler/ir/mod.rs | 0 {yara-x => lib}/src/compiler/ir/utils.rs | 0 {yara-x => lib}/src/compiler/mod.rs | 0 {yara-x => lib}/src/compiler/rules.rs | 0 {yara-x => lib}/src/compiler/tests/errors.rs | 0 {yara-x => lib}/src/compiler/tests/mod.rs | 0 .../src/compiler/tests/warnings.rs | 0 {yara-x => lib}/src/lib.rs | 0 {yara-x => lib}/src/modules/console.rs | 0 {yara-x => lib}/src/modules/dotnet/mod.rs | 0 {yara-x => lib}/src/modules/dotnet/parser.rs | 0 ...e3152020fc7964243fc845cad499d11f9ab.in.zip | Bin ...6a0e3152020fc7964243fc845cad499d11f9ab.out | 0 ...985a2f69fe1b94f3a83cfbba4e4104c2b26.in.zip | Bin ...e07985a2f69fe1b94f3a83cfbba4e4104c2b26.out | 0 ...62bd542e5cb948d1a4ea5a575a14d6b9d6ee849.in | 0 ...542e5cb948d1a4ea5a575a14d6b9d6ee849.in.zip | Bin ...2bd542e5cb948d1a4ea5a575a14d6b9d6ee849.out | 0 ...a62d286de5493c25c000a00f5fbc6e7b746.in.zip | Bin ...030a62d286de5493c25c000a00f5fbc6e7b746.out | 0 ...9358c1068950f849c612ab808225b359db0bb8c.in | 0 ...c1068950f849c612ab808225b359db0bb8c.in.zip | Bin ...358c1068950f849c612ab808225b359db0bb8c.out | 0 ...e98d03e054ca2b88ef5a41f72b5787841b3.in.zip | Bin ...940e98d03e054ca2b88ef5a41f72b5787841b3.out | 0 ...092af954dad65df534bff603b9afcb49cd4.in.zip | Bin ...af2092af954dad65df534bff603b9afcb49cd4.out | 0 {yara-x => lib}/src/modules/elf/mod.rs | 0 {yara-x => lib}/src/modules/elf/parser.rs | 0 {yara-x => lib}/src/modules/elf/tests/mod.rs | 0 ...c2ee6f0d961abc71fbe105f69f23b54bcd3.in.zip | Bin ...cbbc2ee6f0d961abc71fbe105f69f23b54bcd3.out | 0 ...c6034e6f717bac38b1574bc00408c3a90ca.in.zip | Bin ...ea5c6034e6f717bac38b1574bc00408c3a90ca.out | 0 ...c93818264222518c11107aae7d586fabd9e.in.zip | Bin ...db8c93818264222518c11107aae7d586fabd9e.out | 0 ...7527947c0c10a5d9632084a3dee1e828fa7.in.zip | Bin ...5617527947c0c10a5d9632084a3dee1e828fa7.out | 0 ...a0425aa892e4b358bf5a9bf4304e571df1b.in.zip | Bin ...900a0425aa892e4b358bf5a9bf4304e571df1b.out | 0 ...1e4e1d3a026356da4043095f9b96ebce51c.in.zip | Bin ...8b51e4e1d3a026356da4043095f9b96ebce51c.out | 0 ...3a294f58042436d30b5f5126c16edbdc1a7.in.zip | Bin ...fe23a294f58042436d30b5f5126c16edbdc1a7.out | 0 ...3e5d157c0628f5a348e64f63962265d1434.in.zip | Bin ...7b03e5d157c0628f5a348e64f63962265d1434.out | 0 ...dc599ca4fb159be50ef740b31a32dfbcf67.in.zip | Bin ...1d5dc599ca4fb159be50ef740b31a32dfbcf67.out | 0 {yara-x => lib}/src/modules/hash/mod.rs | 0 {yara-x => lib}/src/modules/hash/tests/mod.rs | 0 {yara-x => lib}/src/modules/lnk/mod.rs | 0 {yara-x => lib}/src/modules/lnk/parser.rs | 0 .../lnk/tests/testdata/lnk-extradata-1.in.zip | Bin .../lnk/tests/testdata/lnk-extradata-1.out | 0 .../lnk/tests/testdata/lnk-extradata-2.in.zip | Bin .../lnk/tests/testdata/lnk-extradata-2.out | 0 .../lnk/tests/testdata/lnk-malformed.in.zip | Bin .../lnk/tests/testdata/lnk-malformed.out | 0 .../lnk/tests/testdata/lnk-network.in.zip | Bin .../lnk/tests/testdata/lnk-network.out | 0 .../lnk/tests/testdata/lnk-overlay.in.zip | Bin .../lnk/tests/testdata/lnk-overlay.out | 0 .../lnk/tests/testdata/lnk-standard.in.zip | Bin .../lnk/tests/testdata/lnk-standard.out | 0 {yara-x => lib}/src/modules/macho/mod.rs | 0 {yara-x => lib}/src/modules/macho/parser.rs | 0 .../src/modules/macho/tests/mod.rs | 0 ...aae4d21de61f776e2405324c498ef52b21b.in.zip | Bin ...8bfaae4d21de61f776e2405324c498ef52b21b.out | 0 ...afb2bb8114803e8b26c0652ce4afccb94e1.in.zip | Bin ...a6dafb2bb8114803e8b26c0652ce4afccb94e1.out | 0 ...a00bae94f868ce65a3352177dd6a75797ff.in.zip | Bin ...b53a00bae94f868ce65a3352177dd6a75797ff.out | 0 ...a394e400276ef05b93daa78717daffcd803.in.zip | Bin ...79fa394e400276ef05b93daa78717daffcd803.out | 0 ...3c8beebeb768e712bd3b754b5cd3e151356.in.zip | Bin ...1543c8beebeb768e712bd3b754b5cd3e151356.out | 0 ...cb8cd171119a7bf7a329ae18e5b34cdb85f.in.zip | Bin ...422cb8cd171119a7bf7a329ae18e5b34cdb85f.out | 0 ...79cb0e7394c7b5065f74b7ce8c52ace75f1.in.zip | Bin ...18d79cb0e7394c7b5065f74b7ce8c52ace75f1.out | 0 ...9a5bf085619bbde2c4146b0cc00ebad21c8.in.zip | Bin ...d329a5bf085619bbde2c4146b0cc00ebad21c8.out | 0 .../tests/testdata/macho_ppc_file.in.zip | Bin .../macho/tests/testdata/macho_ppc_file.out | 0 .../testdata/macho_x86_64_dylib_file.in.zip | Bin .../testdata/macho_x86_64_dylib_file.out | 0 .../tests/testdata/macho_x86_file.in.zip | Bin .../macho/tests/testdata/macho_x86_file.out | 0 .../testdata/macho_x86_object_file.in.zip | Bin .../tests/testdata/macho_x86_object_file.out | 0 .../macho/tests/testdata/tiny_macho.in.zip | Bin .../macho/tests/testdata/tiny_macho.out | 0 .../tests/testdata/tiny_universal.in.zip | Bin .../macho/tests/testdata/tiny_universal.out | 0 {yara-x => lib}/src/modules/math.rs | 0 {yara-x => lib}/src/modules/mod.rs | 0 {yara-x => lib}/src/modules/modules.rs | 0 {yara-x => lib}/src/modules/pe/mod.rs | 0 {yara-x => lib}/src/modules/pe/parser.rs | 0 {yara-x => lib}/src/modules/pe/rva2off.rs | 0 {yara-x => lib}/src/modules/pe/tests/mod.rs | 0 ...97104f54a75683a5adee235401778279818.in.zip | Bin ...22597104f54a75683a5adee235401778279818.out | 0 ...dc8dd28a968c6b4d3b88acdd58ce2d3b885.in.zip | Bin ...15cdc8dd28a968c6b4d3b88acdd58ce2d3b885.out | 0 ...135267a9b0156642a9596a62e85c9998cc9.in.zip | Bin ...2a4135267a9b0156642a9596a62e85c9998cc9.out | 0 ...5cd882c80e5afc511c4f7b2e0e193968f7f.in.zip | Bin ...2d55cd882c80e5afc511c4f7b2e0e193968f7f.out | 0 ...2f56b21e48fd67044e69e7a2ae76db631e5.in.zip | Bin ...eb02f56b21e48fd67044e69e7a2ae76db631e5.out | 0 ...5dbec84b9d71a6abbacb26c14e83f5897e4.in.zip | Bin ...ee35dbec84b9d71a6abbacb26c14e83f5897e4.out | 0 ...2c0fad490cb2165f371f53b61941eedf072.in.zip | Bin ...4742c0fad490cb2165f371f53b61941eedf072.out | 0 ...9f272c62d1889216b7a6f1141571ec12649.in.zip | Bin ...7a19f272c62d1889216b7a6f1141571ec12649.out | 0 ...4d7f095eb395779de0ad1ac946914dfa34c.in.zip | Bin ...8c44d7f095eb395779de0ad1ac946914dfa34c.out | 0 ...fea184ce9b58320891a2d72d4ec93766f14.in.zip | Bin ...bdcfea184ce9b58320891a2d72d4ec93766f14.out | 0 ...f004d49ec0c1806080fa72e960529cba14c.in.zip | Bin ...ea8f004d49ec0c1806080fa72e960529cba14c.out | 0 ...0eb69005158f981bc84560e9a5dde103d90.in.zip | Bin ...23b0eb69005158f981bc84560e9a5dde103d90.out | 0 ...ea4181f6477cfb61f2130f37d014ce21888.in.zip | Bin ...06dea4181f6477cfb61f2130f37d014ce21888.out | 0 ...42ccfb04ccedd3ada1e8c26939c726a4c8e.in.zip | Bin ...6ad42ccfb04ccedd3ada1e8c26939c726a4c8e.out | 0 ...eb966dbf9e198c94e3d74a9e9260e5f9870.in.zip | Bin ...02eeb966dbf9e198c94e3d74a9e9260e5f9870.out | 0 ...bbafd719dcc7cfc28f405ad3bc2783c6a12.in.zip | Bin ...c21bbafd719dcc7cfc28f405ad3bc2783c6a12.out | 0 ...185f177fb9daaa2649b4669a24f2ee7e3ba.in.zip | Bin ...2fe185f177fb9daaa2649b4669a24f2ee7e3ba.out | 0 ...a0029d9d4e7903cd63291946b334c314831.in.zip | Bin ...318a0029d9d4e7903cd63291946b334c314831.out | 0 ...073b860e3b4d42e081f86d27bdb1cf6ede4.in.zip | Bin ...540073b860e3b4d42e081f86d27bdb1cf6ede4.out | 0 ...17d29de9dfc156666c98581dfeccc07548e.in.zip | Bin ...00c17d29de9dfc156666c98581dfeccc07548e.out | 0 ...8e0c9710e5a787b18c6948f2eedd9338984.in.zip | Bin ...e218e0c9710e5a787b18c6948f2eedd9338984.out | 0 ...58fef40dad54532ee4355b86bc129f27345.in.zip | Bin ...19258fef40dad54532ee4355b86bc129f27345.out | 0 ...f8d3920141a39b502e870348ce3b254eb80.in.zip | Bin ...f13f8d3920141a39b502e870348ce3b254eb80.out | 0 ...2e0826b06cd15eb6dec9bf3df6465216dd8.in.zip | Bin ...7c12e0826b06cd15eb6dec9bf3df6465216dd8.out | 0 ...173e10a4086fc9d7606fac6e945c0c15ca2.in.zip | Bin ...609173e10a4086fc9d7606fac6e945c0c15ca2.out | 0 .../src/modules/protos/console.proto | 0 .../src/modules/protos/dotnet.proto | 0 {yara-x => lib}/src/modules/protos/elf.proto | 0 {yara-x => lib}/src/modules/protos/hash.proto | 0 {yara-x => lib}/src/modules/protos/lnk.proto | 0 .../src/modules/protos/macho.proto | 0 {yara-x => lib}/src/modules/protos/math.proto | 0 {yara-x => lib}/src/modules/protos/mods.proto | 0 {yara-x => lib}/src/modules/protos/pe.proto | 0 .../src/modules/protos/string.proto | 0 .../src/modules/protos/test_proto2.proto | 0 .../src/modules/protos/test_proto3.proto | 0 {yara-x => lib}/src/modules/protos/text.proto | 0 {yara-x => lib}/src/modules/protos/time.proto | 0 {yara-x => lib}/src/modules/string.rs | 0 .../src/modules/test_proto2/mod.rs | 0 .../src/modules/test_proto2/tests/mod.rs | 0 .../src/modules/test_proto3/mod.rs | 0 {yara-x => lib}/src/modules/tests.rs | 0 {yara-x => lib}/src/modules/text.rs | 0 {yara-x => lib}/src/modules/time.rs | 0 {yara-x => lib}/src/re/bitmapset.rs | 0 {yara-x => lib}/src/re/fast/compiler.rs | 0 {yara-x => lib}/src/re/fast/fastvm.rs | 0 {yara-x => lib}/src/re/fast/instr.rs | 0 {yara-x => lib}/src/re/fast/mod.rs | 0 {yara-x => lib}/src/re/hir.rs | 0 {yara-x => lib}/src/re/mod.rs | 0 {yara-x => lib}/src/re/parser.rs | 0 {yara-x => lib}/src/re/thompson/compiler.rs | 0 {yara-x => lib}/src/re/thompson/instr.rs | 0 {yara-x => lib}/src/re/thompson/mod.rs | 0 {yara-x => lib}/src/re/thompson/pikevm.rs | 0 {yara-x => lib}/src/re/thompson/tests.rs | 0 {yara-x => lib}/src/scanner/context.rs | 0 {yara-x => lib}/src/scanner/matches.rs | 0 {yara-x => lib}/src/scanner/mod.rs | 2 +- {yara-x => lib}/src/scanner/tests.rs | 16 +++++----- {yara-x => lib}/src/string_pool.rs | 0 {yara-x => lib}/src/symbols/mod.rs | 0 {yara-x => lib}/src/tests/mod.rs | 0 {yara-x => lib}/src/tests/testdata/jumps.bin | Bin {yara-x => lib}/src/types/array.rs | 0 {yara-x => lib}/src/types/func.rs | 0 {yara-x => lib}/src/types/map.rs | 0 {yara-x => lib}/src/types/mod.rs | 0 {yara-x => lib}/src/types/structure.rs | 0 {yara-x => lib}/src/variables.rs | 0 {yara-x => lib}/src/wasm/builder.rs | 0 {yara-x => lib}/src/wasm/mod.rs | 0 {yara-x => lib}/src/wasm/string.rs | 0 {yara-x-macros => macros}/Cargo.lock | 0 {yara-x-macros => macros}/Cargo.toml | 0 {yara-x-macros => macros}/src/error.rs | 0 {yara-x-macros => macros}/src/lib.rs | 0 .../src/module_export.rs | 0 {yara-x-macros => macros}/src/module_main.rs | 0 {yara-x-macros => macros}/src/span.rs | 0 {yara-x-macros => macros}/src/wasm_export.rs | 0 {yara-x-parser => parser}/Cargo.lock | 0 {yara-x-parser => parser}/Cargo.toml | 0 .../src/ast/ascii_tree.rs | 0 {yara-x-parser => parser}/src/ast/mod.rs | 0 {yara-x-parser => parser}/src/ast/span.rs | 0 {yara-x-parser => parser}/src/cst/mod.rs | 0 {yara-x-parser => parser}/src/lib.rs | 0 .../src/parser/context.rs | 0 .../src/parser/cst2ast.rs | 0 .../src/parser/errors.rs | 0 .../src/parser/grammar.pest | 0 {yara-x-parser => parser}/src/parser/mod.rs | 0 .../src/parser/tests/ast.rs | 0 .../src/parser/tests/cst.rs | 0 .../src/parser/tests/errors.rs | 0 .../src/parser/tests/mod.rs | 0 .../parser/tests/testdata/arithmetic_ops.yaml | 0 .../parser/tests/testdata/bitwise_ops.yaml | 0 .../src/parser/tests/testdata/for_stmt.yaml | 0 .../src/parser/tests/testdata/general.yaml | 0 .../parser/tests/testdata/hex_patterns.yaml | 0 .../src/parser/tests/testdata/literals.yaml | 0 .../src/parser/tests/testdata/meta.yaml | 0 .../src/parser/tests/testdata/of_stmt.yaml | 0 .../tests/testdata/regexp_patterns.yaml | 0 .../src/parser/tests/testdata/shift_ops.yaml | 0 .../src/parser/tests/testdata/string_ops.yaml | 0 .../parser/tests/testdata/text_patterns.yaml | 0 {yara-x-parser => parser}/src/report.rs | 0 {yara-x-parser => parser}/src/warnings.rs | 0 {yara-x-proto-yaml => proto-yaml}/Cargo.toml | 0 {yara-x-proto-yaml => proto-yaml}/build.rs | 0 {yara-x-proto-yaml => proto-yaml}/src/lib.rs | 0 .../src/tests/mod.rs | 0 .../src/tests/test.proto | 0 .../src/tests/testdata/1.in | 0 .../src/tests/testdata/1.out | 0 .../src/yaml.proto | 0 {yara-x-proto => proto}/Cargo.lock | 0 {yara-x-proto => proto}/Cargo.toml | 0 {yara-x-proto => proto}/build.rs | 0 {yara-x-proto => proto}/src/lib.rs | 0 {yara-x-proto => proto}/src/yara.proto | 0 {yara-x-py => py}/Cargo.toml | 0 {yara-x-py => py}/build.rs | 0 {yara-x-py => py}/pyproject.toml | 0 {yara-x-py => py}/src/lib.rs | 8 ++--- {yara-x-py => py}/tests/test_api.py | 10 +++--- 365 files changed, 45 insertions(+), 44 deletions(-) rename {yara-x-cli => cli}/Cargo.toml (100%) rename {yara-x-cli => cli}/src/commands/check.rs (100%) rename {yara-x-cli => cli}/src/commands/compile.rs (100%) rename {yara-x-cli => cli}/src/commands/debug.rs (100%) rename {yara-x-cli => cli}/src/commands/dump.rs (100%) rename {yara-x-cli => cli}/src/commands/fmt.rs (100%) rename {yara-x-cli => cli}/src/commands/mod.rs (100%) rename {yara-x-cli => cli}/src/commands/scan.rs (99%) rename {yara-x-cli => cli}/src/help.rs (100%) rename {yara-x-cli => cli}/src/main.rs (100%) rename {yara-x-cli => cli}/src/walk.rs (100%) rename {yara-x-fmt => fmt}/Cargo.lock (100%) rename {yara-x-fmt => fmt}/Cargo.toml (100%) rename {yara-x-fmt => fmt}/src/align.rs (100%) rename {yara-x-fmt => fmt}/src/bubble.rs (100%) rename {yara-x-fmt => fmt}/src/comments.rs (100%) rename {yara-x-fmt => fmt}/src/indentation.rs (100%) rename {yara-x-fmt => fmt}/src/lib.rs (100%) rename {yara-x-fmt => fmt}/src/processor/mod.rs (100%) rename {yara-x-fmt => fmt}/src/processor/tests.rs (100%) rename {yara-x-fmt => fmt}/src/testdata/test1.formatted (100%) rename {yara-x-fmt => fmt}/src/testdata/test1.unformatted (100%) rename {yara-x-fmt => fmt}/src/testdata/test10.formatted (100%) rename {yara-x-fmt => fmt}/src/testdata/test10.unformatted (100%) rename {yara-x-fmt => fmt}/src/testdata/test11.formatted (100%) rename {yara-x-fmt => fmt}/src/testdata/test11.unformatted (100%) rename {yara-x-fmt => fmt}/src/testdata/test12.formatted (100%) rename {yara-x-fmt => fmt}/src/testdata/test12.unformatted (100%) rename {yara-x-fmt => fmt}/src/testdata/test13.formatted (100%) rename {yara-x-fmt => fmt}/src/testdata/test13.unformatted (100%) rename {yara-x-fmt => fmt}/src/testdata/test14.formatted (100%) rename {yara-x-fmt => fmt}/src/testdata/test14.unformatted (100%) rename {yara-x-fmt => fmt}/src/testdata/test15.formatted (100%) rename {yara-x-fmt => fmt}/src/testdata/test15.unformatted (100%) rename {yara-x-fmt => fmt}/src/testdata/test16.formatted (100%) rename {yara-x-fmt => fmt}/src/testdata/test16.unformatted (100%) rename {yara-x-fmt => fmt}/src/testdata/test17.formatted (100%) rename {yara-x-fmt => fmt}/src/testdata/test17.unformatted (100%) rename {yara-x-fmt => fmt}/src/testdata/test18.formatted (100%) rename {yara-x-fmt => fmt}/src/testdata/test18.unformatted (100%) rename {yara-x-fmt => fmt}/src/testdata/test19.formatted (100%) rename {yara-x-fmt => fmt}/src/testdata/test19.unformatted (100%) rename {yara-x-fmt => fmt}/src/testdata/test2.formatted (100%) rename {yara-x-fmt => fmt}/src/testdata/test2.unformatted (100%) rename {yara-x-fmt => fmt}/src/testdata/test20.formatted (100%) rename {yara-x-fmt => fmt}/src/testdata/test20.unformatted (100%) rename {yara-x-fmt => fmt}/src/testdata/test21.formatted (100%) rename {yara-x-fmt => fmt}/src/testdata/test21.unformatted (100%) rename {yara-x-fmt => fmt}/src/testdata/test22.formatted (100%) rename {yara-x-fmt => fmt}/src/testdata/test22.unformatted (100%) rename {yara-x-fmt => fmt}/src/testdata/test23.formatted (100%) rename {yara-x-fmt => fmt}/src/testdata/test23.unformatted (100%) rename {yara-x-fmt => fmt}/src/testdata/test24.formatted (100%) rename {yara-x-fmt => fmt}/src/testdata/test24.unformatted (100%) rename {yara-x-fmt => fmt}/src/testdata/test25.formatted (100%) rename {yara-x-fmt => fmt}/src/testdata/test25.unformatted (100%) rename {yara-x-fmt => fmt}/src/testdata/test26.formatted (100%) rename {yara-x-fmt => fmt}/src/testdata/test26.unformatted (100%) rename {yara-x-fmt => fmt}/src/testdata/test27.formatted (100%) rename {yara-x-fmt => fmt}/src/testdata/test27.unformatted (100%) rename {yara-x-fmt => fmt}/src/testdata/test28.formatted (100%) rename {yara-x-fmt => fmt}/src/testdata/test28.unformatted (100%) rename {yara-x-fmt => fmt}/src/testdata/test29.formatted (100%) rename {yara-x-fmt => fmt}/src/testdata/test29.unformatted (100%) rename {yara-x-fmt => fmt}/src/testdata/test3.formatted (100%) rename {yara-x-fmt => fmt}/src/testdata/test3.unformatted (100%) rename {yara-x-fmt => fmt}/src/testdata/test4.formatted (100%) rename {yara-x-fmt => fmt}/src/testdata/test4.unformatted (100%) rename {yara-x-fmt => fmt}/src/testdata/test5.formatted (100%) rename {yara-x-fmt => fmt}/src/testdata/test5.unformatted (100%) rename {yara-x-fmt => fmt}/src/testdata/test6.formatted (100%) rename {yara-x-fmt => fmt}/src/testdata/test6.unformatted (100%) rename {yara-x-fmt => fmt}/src/testdata/test7.formatted (100%) rename {yara-x-fmt => fmt}/src/testdata/test7.unformatted (100%) rename {yara-x-fmt => fmt}/src/testdata/test8.formatted (100%) rename {yara-x-fmt => fmt}/src/testdata/test8.unformatted (100%) rename {yara-x-fmt => fmt}/src/testdata/test9.formatted (100%) rename {yara-x-fmt => fmt}/src/testdata/test9.unformatted (100%) rename {yara-x-fmt => fmt}/src/tests.rs (100%) rename {yara-x-fmt => fmt}/src/tokens/mod.rs (100%) rename {yara-x-fmt => fmt}/src/tokens/tests.rs (100%) rename {yara-x-fmt => fmt}/src/trailing_spaces.rs (100%) rename {yara-x => lib}/Cargo.toml (100%) rename {yara-x => lib}/benches/benches.rs (100%) rename {yara-x => lib}/build.rs (93%) rename {yara-x => lib}/fuzz/.gitignore (100%) rename {yara-x => lib}/fuzz/Cargo.lock (100%) rename {yara-x => lib}/fuzz/Cargo.toml (100%) rename {yara-x => lib}/fuzz/fuzz_targets/dotnet_parser.rs (100%) rename {yara-x => lib}/fuzz/fuzz_targets/elf_parser.rs (100%) rename {yara-x => lib}/fuzz/fuzz_targets/lnk_parser.rs (100%) rename {yara-x => lib}/fuzz/fuzz_targets/macho_fat_parser.rs (100%) rename {yara-x => lib}/fuzz/fuzz_targets/macho_parser.rs (100%) rename {yara-x => lib}/fuzz/fuzz_targets/pe_parser.rs (100%) rename {yara-x => lib}/fuzz/fuzz_targets/rule_compiler.rs (100%) rename {yara-x => lib}/src/compiler/atoms/mask.rs (100%) rename {yara-x => lib}/src/compiler/atoms/mod.rs (100%) rename {yara-x => lib}/src/compiler/atoms/quality.rs (100%) rename {yara-x => lib}/src/compiler/base64.rs (100%) rename {yara-x => lib}/src/compiler/context.rs (100%) rename {yara-x => lib}/src/compiler/emit.rs (100%) rename {yara-x => lib}/src/compiler/errors.rs (100%) rename {yara-x => lib}/src/compiler/ir/ast2ir.rs (100%) rename {yara-x => lib}/src/compiler/ir/hex2hir.rs (100%) rename {yara-x => lib}/src/compiler/ir/mod.rs (100%) rename {yara-x => lib}/src/compiler/ir/utils.rs (100%) rename {yara-x => lib}/src/compiler/mod.rs (100%) rename {yara-x => lib}/src/compiler/rules.rs (100%) rename {yara-x => lib}/src/compiler/tests/errors.rs (100%) rename {yara-x => lib}/src/compiler/tests/mod.rs (100%) rename {yara-x => lib}/src/compiler/tests/warnings.rs (100%) rename {yara-x => lib}/src/lib.rs (100%) rename {yara-x => lib}/src/modules/console.rs (100%) rename {yara-x => lib}/src/modules/dotnet/mod.rs (100%) rename {yara-x => lib}/src/modules/dotnet/parser.rs (100%) rename {yara-x => lib}/src/modules/dotnet/tests/testdata/0224ad9739f5fa64a2a63e23ef6a0e3152020fc7964243fc845cad499d11f9ab.in.zip (100%) rename {yara-x => lib}/src/modules/dotnet/tests/testdata/0224ad9739f5fa64a2a63e23ef6a0e3152020fc7964243fc845cad499d11f9ab.out (100%) rename {yara-x => lib}/src/modules/dotnet/tests/testdata/4a8e3e00ce7fd88da587fbb187e07985a2f69fe1b94f3a83cfbba4e4104c2b26.in.zip (100%) rename {yara-x => lib}/src/modules/dotnet/tests/testdata/4a8e3e00ce7fd88da587fbb187e07985a2f69fe1b94f3a83cfbba4e4104c2b26.out (100%) rename {yara-x => lib}/src/modules/dotnet/tests/testdata/605ebe5b5f4b94e1a73e0ad1162bd542e5cb948d1a4ea5a575a14d6b9d6ee849.in (100%) rename {yara-x => lib}/src/modules/dotnet/tests/testdata/605ebe5b5f4b94e1a73e0ad1162bd542e5cb948d1a4ea5a575a14d6b9d6ee849.in.zip (100%) rename {yara-x => lib}/src/modules/dotnet/tests/testdata/605ebe5b5f4b94e1a73e0ad1162bd542e5cb948d1a4ea5a575a14d6b9d6ee849.out (100%) rename {yara-x => lib}/src/modules/dotnet/tests/testdata/6d72a599724c4ff367b2c00b48030a62d286de5493c25c000a00f5fbc6e7b746.in.zip (100%) rename {yara-x => lib}/src/modules/dotnet/tests/testdata/6d72a599724c4ff367b2c00b48030a62d286de5493c25c000a00f5fbc6e7b746.out (100%) rename {yara-x => lib}/src/modules/dotnet/tests/testdata/86a1e48cfc843eabfe1b468ef9358c1068950f849c612ab808225b359db0bb8c.in (100%) rename {yara-x => lib}/src/modules/dotnet/tests/testdata/86a1e48cfc843eabfe1b468ef9358c1068950f849c612ab808225b359db0bb8c.in.zip (100%) rename {yara-x => lib}/src/modules/dotnet/tests/testdata/86a1e48cfc843eabfe1b468ef9358c1068950f849c612ab808225b359db0bb8c.out (100%) rename {yara-x => lib}/src/modules/dotnet/tests/testdata/88d0d0692e675cd5fafb57db4f940e98d03e054ca2b88ef5a41f72b5787841b3.in.zip (100%) rename {yara-x => lib}/src/modules/dotnet/tests/testdata/88d0d0692e675cd5fafb57db4f940e98d03e054ca2b88ef5a41f72b5787841b3.out (100%) rename {yara-x => lib}/src/modules/dotnet/tests/testdata/984750efd1cb94e5ca7b366863af2092af954dad65df534bff603b9afcb49cd4.in.zip (100%) rename {yara-x => lib}/src/modules/dotnet/tests/testdata/984750efd1cb94e5ca7b366863af2092af954dad65df534bff603b9afcb49cd4.out (100%) rename {yara-x => lib}/src/modules/elf/mod.rs (100%) rename {yara-x => lib}/src/modules/elf/parser.rs (100%) rename {yara-x => lib}/src/modules/elf/tests/mod.rs (100%) rename {yara-x => lib}/src/modules/elf/tests/testdata/246f379d911603775271609e45cbbc2ee6f0d961abc71fbe105f69f23b54bcd3.in.zip (100%) rename {yara-x => lib}/src/modules/elf/tests/testdata/246f379d911603775271609e45cbbc2ee6f0d961abc71fbe105f69f23b54bcd3.out (100%) rename {yara-x => lib}/src/modules/elf/tests/testdata/558ca94cffacf78aa9aebb8983ea5c6034e6f717bac38b1574bc00408c3a90ca.in.zip (100%) rename {yara-x => lib}/src/modules/elf/tests/testdata/558ca94cffacf78aa9aebb8983ea5c6034e6f717bac38b1574bc00408c3a90ca.out (100%) rename {yara-x => lib}/src/modules/elf/tests/testdata/5ddb04cb08755e692fa540c51cdb8c93818264222518c11107aae7d586fabd9e.in.zip (100%) rename {yara-x => lib}/src/modules/elf/tests/testdata/5ddb04cb08755e692fa540c51cdb8c93818264222518c11107aae7d586fabd9e.out (100%) rename {yara-x => lib}/src/modules/elf/tests/testdata/7c673873f6aca25d450216af9b5617527947c0c10a5d9632084a3dee1e828fa7.in.zip (100%) rename {yara-x => lib}/src/modules/elf/tests/testdata/7c673873f6aca25d450216af9b5617527947c0c10a5d9632084a3dee1e828fa7.out (100%) rename {yara-x => lib}/src/modules/elf/tests/testdata/8bfe885838b4d1fba194b761ca900a0425aa892e4b358bf5a9bf4304e571df1b.in.zip (100%) rename {yara-x => lib}/src/modules/elf/tests/testdata/8bfe885838b4d1fba194b761ca900a0425aa892e4b358bf5a9bf4304e571df1b.out (100%) rename {yara-x => lib}/src/modules/elf/tests/testdata/ac64f17c42eec4ec357e5fe9e58b51e4e1d3a026356da4043095f9b96ebce51c.in.zip (100%) rename {yara-x => lib}/src/modules/elf/tests/testdata/ac64f17c42eec4ec357e5fe9e58b51e4e1d3a026356da4043095f9b96ebce51c.out (100%) rename {yara-x => lib}/src/modules/elf/tests/testdata/cc7ca38e635918e4928fe7d6fffe23a294f58042436d30b5f5126c16edbdc1a7.in.zip (100%) rename {yara-x => lib}/src/modules/elf/tests/testdata/cc7ca38e635918e4928fe7d6fffe23a294f58042436d30b5f5126c16edbdc1a7.out (100%) rename {yara-x => lib}/src/modules/elf/tests/testdata/f040356a60cde1047266c841237b03e5d157c0628f5a348e64f63962265d1434.in.zip (100%) rename {yara-x => lib}/src/modules/elf/tests/testdata/f040356a60cde1047266c841237b03e5d157c0628f5a348e64f63962265d1434.out (100%) rename {yara-x => lib}/src/modules/elf/tests/testdata/ff8a8921d672e5ce1676fb35001d5dc599ca4fb159be50ef740b31a32dfbcf67.in.zip (100%) rename {yara-x => lib}/src/modules/elf/tests/testdata/ff8a8921d672e5ce1676fb35001d5dc599ca4fb159be50ef740b31a32dfbcf67.out (100%) rename {yara-x => lib}/src/modules/hash/mod.rs (100%) rename {yara-x => lib}/src/modules/hash/tests/mod.rs (100%) rename {yara-x => lib}/src/modules/lnk/mod.rs (100%) rename {yara-x => lib}/src/modules/lnk/parser.rs (100%) rename {yara-x => lib}/src/modules/lnk/tests/testdata/lnk-extradata-1.in.zip (100%) rename {yara-x => lib}/src/modules/lnk/tests/testdata/lnk-extradata-1.out (100%) rename {yara-x => lib}/src/modules/lnk/tests/testdata/lnk-extradata-2.in.zip (100%) rename {yara-x => lib}/src/modules/lnk/tests/testdata/lnk-extradata-2.out (100%) rename {yara-x => lib}/src/modules/lnk/tests/testdata/lnk-malformed.in.zip (100%) rename {yara-x => lib}/src/modules/lnk/tests/testdata/lnk-malformed.out (100%) rename {yara-x => lib}/src/modules/lnk/tests/testdata/lnk-network.in.zip (100%) rename {yara-x => lib}/src/modules/lnk/tests/testdata/lnk-network.out (100%) rename {yara-x => lib}/src/modules/lnk/tests/testdata/lnk-overlay.in.zip (100%) rename {yara-x => lib}/src/modules/lnk/tests/testdata/lnk-overlay.out (100%) rename {yara-x => lib}/src/modules/lnk/tests/testdata/lnk-standard.in.zip (100%) rename {yara-x => lib}/src/modules/lnk/tests/testdata/lnk-standard.out (100%) rename {yara-x => lib}/src/modules/macho/mod.rs (100%) rename {yara-x => lib}/src/modules/macho/parser.rs (100%) rename {yara-x => lib}/src/modules/macho/tests/mod.rs (100%) rename {yara-x => lib}/src/modules/macho/tests/testdata/01ac68a14f0ff5faa72bb33e768bfaae4d21de61f776e2405324c498ef52b21b.in.zip (100%) rename {yara-x => lib}/src/modules/macho/tests/testdata/01ac68a14f0ff5faa72bb33e768bfaae4d21de61f776e2405324c498ef52b21b.out (100%) rename {yara-x => lib}/src/modules/macho/tests/testdata/3d27411e4465ae14ab2d02e1f2a6dafb2bb8114803e8b26c0652ce4afccb94e1.in.zip (100%) rename {yara-x => lib}/src/modules/macho/tests/testdata/3d27411e4465ae14ab2d02e1f2a6dafb2bb8114803e8b26c0652ce4afccb94e1.out (100%) rename {yara-x => lib}/src/modules/macho/tests/testdata/4d0725d5c506c7ec26be89ec12b53a00bae94f868ce65a3352177dd6a75797ff.in.zip (100%) rename {yara-x => lib}/src/modules/macho/tests/testdata/4d0725d5c506c7ec26be89ec12b53a00bae94f868ce65a3352177dd6a75797ff.out (100%) rename {yara-x => lib}/src/modules/macho/tests/testdata/5fad2944f6d9e36f6dc21ad3ea79fa394e400276ef05b93daa78717daffcd803.in.zip (100%) rename {yara-x => lib}/src/modules/macho/tests/testdata/5fad2944f6d9e36f6dc21ad3ea79fa394e400276ef05b93daa78717daffcd803.out (100%) rename {yara-x => lib}/src/modules/macho/tests/testdata/8962a76d0aeaee3326cf840de11543c8beebeb768e712bd3b754b5cd3e151356.in.zip (100%) rename {yara-x => lib}/src/modules/macho/tests/testdata/8962a76d0aeaee3326cf840de11543c8beebeb768e712bd3b754b5cd3e151356.out (100%) rename {yara-x => lib}/src/modules/macho/tests/testdata/94b2f690c776c4d847845be21b422cb8cd171119a7bf7a329ae18e5b34cdb85f.in.zip (100%) rename {yara-x => lib}/src/modules/macho/tests/testdata/94b2f690c776c4d847845be21b422cb8cd171119a7bf7a329ae18e5b34cdb85f.out (100%) rename {yara-x => lib}/src/modules/macho/tests/testdata/a1da3d22c5ec85143faf5f69fb18d79cb0e7394c7b5065f74b7ce8c52ace75f1.in.zip (100%) rename {yara-x => lib}/src/modules/macho/tests/testdata/a1da3d22c5ec85143faf5f69fb18d79cb0e7394c7b5065f74b7ce8c52ace75f1.out (100%) rename {yara-x => lib}/src/modules/macho/tests/testdata/edf47dd000b9fdf4519fb61c28d329a5bf085619bbde2c4146b0cc00ebad21c8.in.zip (100%) rename {yara-x => lib}/src/modules/macho/tests/testdata/edf47dd000b9fdf4519fb61c28d329a5bf085619bbde2c4146b0cc00ebad21c8.out (100%) rename {yara-x => lib}/src/modules/macho/tests/testdata/macho_ppc_file.in.zip (100%) rename {yara-x => lib}/src/modules/macho/tests/testdata/macho_ppc_file.out (100%) rename {yara-x => lib}/src/modules/macho/tests/testdata/macho_x86_64_dylib_file.in.zip (100%) rename {yara-x => lib}/src/modules/macho/tests/testdata/macho_x86_64_dylib_file.out (100%) rename {yara-x => lib}/src/modules/macho/tests/testdata/macho_x86_file.in.zip (100%) rename {yara-x => lib}/src/modules/macho/tests/testdata/macho_x86_file.out (100%) rename {yara-x => lib}/src/modules/macho/tests/testdata/macho_x86_object_file.in.zip (100%) rename {yara-x => lib}/src/modules/macho/tests/testdata/macho_x86_object_file.out (100%) rename {yara-x => lib}/src/modules/macho/tests/testdata/tiny_macho.in.zip (100%) rename {yara-x => lib}/src/modules/macho/tests/testdata/tiny_macho.out (100%) rename {yara-x => lib}/src/modules/macho/tests/testdata/tiny_universal.in.zip (100%) rename {yara-x => lib}/src/modules/macho/tests/testdata/tiny_universal.out (100%) rename {yara-x => lib}/src/modules/math.rs (100%) rename {yara-x => lib}/src/modules/mod.rs (100%) rename {yara-x => lib}/src/modules/modules.rs (100%) rename {yara-x => lib}/src/modules/pe/mod.rs (100%) rename {yara-x => lib}/src/modules/pe/parser.rs (100%) rename {yara-x => lib}/src/modules/pe/rva2off.rs (100%) rename {yara-x => lib}/src/modules/pe/tests/mod.rs (100%) rename {yara-x => lib}/src/modules/pe/tests/testdata/04ac6dd0c1cc33a49962ee0f3222597104f54a75683a5adee235401778279818.in.zip (100%) rename {yara-x => lib}/src/modules/pe/tests/testdata/04ac6dd0c1cc33a49962ee0f3222597104f54a75683a5adee235401778279818.out (100%) rename {yara-x => lib}/src/modules/pe/tests/testdata/079a472d22290a94ebb212aa8015cdc8dd28a968c6b4d3b88acdd58ce2d3b885.in.zip (100%) rename {yara-x => lib}/src/modules/pe/tests/testdata/079a472d22290a94ebb212aa8015cdc8dd28a968c6b4d3b88acdd58ce2d3b885.out (100%) rename {yara-x => lib}/src/modules/pe/tests/testdata/09e7d832320e51bcc80b9aecde2a4135267a9b0156642a9596a62e85c9998cc9.in.zip (100%) rename {yara-x => lib}/src/modules/pe/tests/testdata/09e7d832320e51bcc80b9aecde2a4135267a9b0156642a9596a62e85c9998cc9.out (100%) rename {yara-x => lib}/src/modules/pe/tests/testdata/0ba6042247d90a187919dd88dc2d55cd882c80e5afc511c4f7b2e0e193968f7f.in.zip (100%) rename {yara-x => lib}/src/modules/pe/tests/testdata/0ba6042247d90a187919dd88dc2d55cd882c80e5afc511c4f7b2e0e193968f7f.out (100%) rename {yara-x => lib}/src/modules/pe/tests/testdata/23e72ce7e9cdbc80c0095484ebeb02f56b21e48fd67044e69e7a2ae76db631e5.in.zip (100%) rename {yara-x => lib}/src/modules/pe/tests/testdata/23e72ce7e9cdbc80c0095484ebeb02f56b21e48fd67044e69e7a2ae76db631e5.out (100%) rename {yara-x => lib}/src/modules/pe/tests/testdata/2775d97f8bdb3311ace960a42eee35dbec84b9d71a6abbacb26c14e83f5897e4.in.zip (100%) rename {yara-x => lib}/src/modules/pe/tests/testdata/2775d97f8bdb3311ace960a42eee35dbec84b9d71a6abbacb26c14e83f5897e4.out (100%) rename {yara-x => lib}/src/modules/pe/tests/testdata/29eeeecf2c458ea3da1ce9d6d54742c0fad490cb2165f371f53b61941eedf072.in.zip (100%) rename {yara-x => lib}/src/modules/pe/tests/testdata/29eeeecf2c458ea3da1ce9d6d54742c0fad490cb2165f371f53b61941eedf072.out (100%) rename {yara-x => lib}/src/modules/pe/tests/testdata/2d80c403b5c50f8bbacb65f58e7a19f272c62d1889216b7a6f1141571ec12649.in.zip (100%) rename {yara-x => lib}/src/modules/pe/tests/testdata/2d80c403b5c50f8bbacb65f58e7a19f272c62d1889216b7a6f1141571ec12649.out (100%) rename {yara-x => lib}/src/modules/pe/tests/testdata/2e9c671b8a0411f2b397544b368c44d7f095eb395779de0ad1ac946914dfa34c.in.zip (100%) rename {yara-x => lib}/src/modules/pe/tests/testdata/2e9c671b8a0411f2b397544b368c44d7f095eb395779de0ad1ac946914dfa34c.out (100%) rename {yara-x => lib}/src/modules/pe/tests/testdata/55cfd3bcea1aa352b4687c4d45bdcfea184ce9b58320891a2d72d4ec93766f14.in.zip (100%) rename {yara-x => lib}/src/modules/pe/tests/testdata/55cfd3bcea1aa352b4687c4d45bdcfea184ce9b58320891a2d72d4ec93766f14.out (100%) rename {yara-x => lib}/src/modules/pe/tests/testdata/6c2abf4b80a87e63eee2996e5cea8f004d49ec0c1806080fa72e960529cba14c.in.zip (100%) rename {yara-x => lib}/src/modules/pe/tests/testdata/6c2abf4b80a87e63eee2996e5cea8f004d49ec0c1806080fa72e960529cba14c.out (100%) rename {yara-x => lib}/src/modules/pe/tests/testdata/99df28014fae5f213c8decfde423b0eb69005158f981bc84560e9a5dde103d90.in.zip (100%) rename {yara-x => lib}/src/modules/pe/tests/testdata/99df28014fae5f213c8decfde423b0eb69005158f981bc84560e9a5dde103d90.out (100%) rename {yara-x => lib}/src/modules/pe/tests/testdata/9bcf79a99ffbb1bd649503ce1406dea4181f6477cfb61f2130f37d014ce21888.in.zip (100%) rename {yara-x => lib}/src/modules/pe/tests/testdata/9bcf79a99ffbb1bd649503ce1406dea4181f6477cfb61f2130f37d014ce21888.out (100%) rename {yara-x => lib}/src/modules/pe/tests/testdata/af3f20a9272489cbef4281c8c86ad42ccfb04ccedd3ada1e8c26939c726a4c8e.in.zip (100%) rename {yara-x => lib}/src/modules/pe/tests/testdata/af3f20a9272489cbef4281c8c86ad42ccfb04ccedd3ada1e8c26939c726a4c8e.out (100%) rename {yara-x => lib}/src/modules/pe/tests/testdata/b8543d3aceec5a754292393f6602eeb966dbf9e198c94e3d74a9e9260e5f9870.in.zip (100%) rename {yara-x => lib}/src/modules/pe/tests/testdata/b8543d3aceec5a754292393f6602eeb966dbf9e198c94e3d74a9e9260e5f9870.out (100%) rename {yara-x => lib}/src/modules/pe/tests/testdata/bd82090d9c6e23c1e2708550f4c21bbafd719dcc7cfc28f405ad3bc2783c6a12.in.zip (100%) rename {yara-x => lib}/src/modules/pe/tests/testdata/bd82090d9c6e23c1e2708550f4c21bbafd719dcc7cfc28f405ad3bc2783c6a12.out (100%) rename {yara-x => lib}/src/modules/pe/tests/testdata/c6f9709feccf42f2d9e22057182fe185f177fb9daaa2649b4669a24f2ee7e3ba.in.zip (100%) rename {yara-x => lib}/src/modules/pe/tests/testdata/c6f9709feccf42f2d9e22057182fe185f177fb9daaa2649b4669a24f2ee7e3ba.out (100%) rename {yara-x => lib}/src/modules/pe/tests/testdata/c703e95b9ec0ca955b1bcf33e1318a0029d9d4e7903cd63291946b334c314831.in.zip (100%) rename {yara-x => lib}/src/modules/pe/tests/testdata/c703e95b9ec0ca955b1bcf33e1318a0029d9d4e7903cd63291946b334c314831.out (100%) rename {yara-x => lib}/src/modules/pe/tests/testdata/c704cca0fe4c9bdee18a302952540073b860e3b4d42e081f86d27bdb1cf6ede4.in.zip (100%) rename {yara-x => lib}/src/modules/pe/tests/testdata/c704cca0fe4c9bdee18a302952540073b860e3b4d42e081f86d27bdb1cf6ede4.out (100%) rename {yara-x => lib}/src/modules/pe/tests/testdata/d009038e4e371f9cdc3a96923700c17d29de9dfc156666c98581dfeccc07548e.in.zip (100%) rename {yara-x => lib}/src/modules/pe/tests/testdata/d009038e4e371f9cdc3a96923700c17d29de9dfc156666c98581dfeccc07548e.out (100%) rename {yara-x => lib}/src/modules/pe/tests/testdata/db6a9934570fa98a93a979e7e0e218e0c9710e5a787b18c6948f2eedd9338984.in.zip (100%) rename {yara-x => lib}/src/modules/pe/tests/testdata/db6a9934570fa98a93a979e7e0e218e0c9710e5a787b18c6948f2eedd9338984.out (100%) rename {yara-x => lib}/src/modules/pe/tests/testdata/e3d45a2865818756068757d7e319258fef40dad54532ee4355b86bc129f27345.in.zip (100%) rename {yara-x => lib}/src/modules/pe/tests/testdata/e3d45a2865818756068757d7e319258fef40dad54532ee4355b86bc129f27345.out (100%) rename {yara-x => lib}/src/modules/pe/tests/testdata/e5b038a7a8578bbe020784877bf13f8d3920141a39b502e870348ce3b254eb80.in.zip (100%) rename {yara-x => lib}/src/modules/pe/tests/testdata/e5b038a7a8578bbe020784877bf13f8d3920141a39b502e870348ce3b254eb80.out (100%) rename {yara-x => lib}/src/modules/pe/tests/testdata/eda21be1b234c84acb8257f2f77c12e0826b06cd15eb6dec9bf3df6465216dd8.in.zip (100%) rename {yara-x => lib}/src/modules/pe/tests/testdata/eda21be1b234c84acb8257f2f77c12e0826b06cd15eb6dec9bf3df6465216dd8.out (100%) rename {yara-x => lib}/src/modules/pe/tests/testdata/ee82699133743266399721dadb609173e10a4086fc9d7606fac6e945c0c15ca2.in.zip (100%) rename {yara-x => lib}/src/modules/pe/tests/testdata/ee82699133743266399721dadb609173e10a4086fc9d7606fac6e945c0c15ca2.out (100%) rename {yara-x => lib}/src/modules/protos/console.proto (100%) rename {yara-x => lib}/src/modules/protos/dotnet.proto (100%) rename {yara-x => lib}/src/modules/protos/elf.proto (100%) rename {yara-x => lib}/src/modules/protos/hash.proto (100%) rename {yara-x => lib}/src/modules/protos/lnk.proto (100%) rename {yara-x => lib}/src/modules/protos/macho.proto (100%) rename {yara-x => lib}/src/modules/protos/math.proto (100%) rename {yara-x => lib}/src/modules/protos/mods.proto (100%) rename {yara-x => lib}/src/modules/protos/pe.proto (100%) rename {yara-x => lib}/src/modules/protos/string.proto (100%) rename {yara-x => lib}/src/modules/protos/test_proto2.proto (100%) rename {yara-x => lib}/src/modules/protos/test_proto3.proto (100%) rename {yara-x => lib}/src/modules/protos/text.proto (100%) rename {yara-x => lib}/src/modules/protos/time.proto (100%) rename {yara-x => lib}/src/modules/string.rs (100%) rename {yara-x => lib}/src/modules/test_proto2/mod.rs (100%) rename {yara-x => lib}/src/modules/test_proto2/tests/mod.rs (100%) rename {yara-x => lib}/src/modules/test_proto3/mod.rs (100%) rename {yara-x => lib}/src/modules/tests.rs (100%) rename {yara-x => lib}/src/modules/text.rs (100%) rename {yara-x => lib}/src/modules/time.rs (100%) rename {yara-x => lib}/src/re/bitmapset.rs (100%) rename {yara-x => lib}/src/re/fast/compiler.rs (100%) rename {yara-x => lib}/src/re/fast/fastvm.rs (100%) rename {yara-x => lib}/src/re/fast/instr.rs (100%) rename {yara-x => lib}/src/re/fast/mod.rs (100%) rename {yara-x => lib}/src/re/hir.rs (100%) rename {yara-x => lib}/src/re/mod.rs (100%) rename {yara-x => lib}/src/re/parser.rs (100%) rename {yara-x => lib}/src/re/thompson/compiler.rs (100%) rename {yara-x => lib}/src/re/thompson/instr.rs (100%) rename {yara-x => lib}/src/re/thompson/mod.rs (100%) rename {yara-x => lib}/src/re/thompson/pikevm.rs (100%) rename {yara-x => lib}/src/re/thompson/tests.rs (100%) rename {yara-x => lib}/src/scanner/context.rs (100%) rename {yara-x => lib}/src/scanner/matches.rs (100%) rename {yara-x => lib}/src/scanner/mod.rs (99%) rename {yara-x => lib}/src/scanner/tests.rs (95%) rename {yara-x => lib}/src/string_pool.rs (100%) rename {yara-x => lib}/src/symbols/mod.rs (100%) rename {yara-x => lib}/src/tests/mod.rs (100%) rename {yara-x => lib}/src/tests/testdata/jumps.bin (100%) rename {yara-x => lib}/src/types/array.rs (100%) rename {yara-x => lib}/src/types/func.rs (100%) rename {yara-x => lib}/src/types/map.rs (100%) rename {yara-x => lib}/src/types/mod.rs (100%) rename {yara-x => lib}/src/types/structure.rs (100%) rename {yara-x => lib}/src/variables.rs (100%) rename {yara-x => lib}/src/wasm/builder.rs (100%) rename {yara-x => lib}/src/wasm/mod.rs (100%) rename {yara-x => lib}/src/wasm/string.rs (100%) rename {yara-x-macros => macros}/Cargo.lock (100%) rename {yara-x-macros => macros}/Cargo.toml (100%) rename {yara-x-macros => macros}/src/error.rs (100%) rename {yara-x-macros => macros}/src/lib.rs (100%) rename {yara-x-macros => macros}/src/module_export.rs (100%) rename {yara-x-macros => macros}/src/module_main.rs (100%) rename {yara-x-macros => macros}/src/span.rs (100%) rename {yara-x-macros => macros}/src/wasm_export.rs (100%) rename {yara-x-parser => parser}/Cargo.lock (100%) rename {yara-x-parser => parser}/Cargo.toml (100%) rename {yara-x-parser => parser}/src/ast/ascii_tree.rs (100%) rename {yara-x-parser => parser}/src/ast/mod.rs (100%) rename {yara-x-parser => parser}/src/ast/span.rs (100%) rename {yara-x-parser => parser}/src/cst/mod.rs (100%) rename {yara-x-parser => parser}/src/lib.rs (100%) rename {yara-x-parser => parser}/src/parser/context.rs (100%) rename {yara-x-parser => parser}/src/parser/cst2ast.rs (100%) rename {yara-x-parser => parser}/src/parser/errors.rs (100%) rename {yara-x-parser => parser}/src/parser/grammar.pest (100%) rename {yara-x-parser => parser}/src/parser/mod.rs (100%) rename {yara-x-parser => parser}/src/parser/tests/ast.rs (100%) rename {yara-x-parser => parser}/src/parser/tests/cst.rs (100%) rename {yara-x-parser => parser}/src/parser/tests/errors.rs (100%) rename {yara-x-parser => parser}/src/parser/tests/mod.rs (100%) rename {yara-x-parser => parser}/src/parser/tests/testdata/arithmetic_ops.yaml (100%) rename {yara-x-parser => parser}/src/parser/tests/testdata/bitwise_ops.yaml (100%) rename {yara-x-parser => parser}/src/parser/tests/testdata/for_stmt.yaml (100%) rename {yara-x-parser => parser}/src/parser/tests/testdata/general.yaml (100%) rename {yara-x-parser => parser}/src/parser/tests/testdata/hex_patterns.yaml (100%) rename {yara-x-parser => parser}/src/parser/tests/testdata/literals.yaml (100%) rename {yara-x-parser => parser}/src/parser/tests/testdata/meta.yaml (100%) rename {yara-x-parser => parser}/src/parser/tests/testdata/of_stmt.yaml (100%) rename {yara-x-parser => parser}/src/parser/tests/testdata/regexp_patterns.yaml (100%) rename {yara-x-parser => parser}/src/parser/tests/testdata/shift_ops.yaml (100%) rename {yara-x-parser => parser}/src/parser/tests/testdata/string_ops.yaml (100%) rename {yara-x-parser => parser}/src/parser/tests/testdata/text_patterns.yaml (100%) rename {yara-x-parser => parser}/src/report.rs (100%) rename {yara-x-parser => parser}/src/warnings.rs (100%) rename {yara-x-proto-yaml => proto-yaml}/Cargo.toml (100%) rename {yara-x-proto-yaml => proto-yaml}/build.rs (100%) rename {yara-x-proto-yaml => proto-yaml}/src/lib.rs (100%) rename {yara-x-proto-yaml => proto-yaml}/src/tests/mod.rs (100%) rename {yara-x-proto-yaml => proto-yaml}/src/tests/test.proto (100%) rename {yara-x-proto-yaml => proto-yaml}/src/tests/testdata/1.in (100%) rename {yara-x-proto-yaml => proto-yaml}/src/tests/testdata/1.out (100%) rename {yara-x-proto-yaml => proto-yaml}/src/yaml.proto (100%) rename {yara-x-proto => proto}/Cargo.lock (100%) rename {yara-x-proto => proto}/Cargo.toml (100%) rename {yara-x-proto => proto}/build.rs (100%) rename {yara-x-proto => proto}/src/lib.rs (100%) rename {yara-x-proto => proto}/src/yara.proto (100%) rename {yara-x-py => py}/Cargo.toml (100%) rename {yara-x-py => py}/build.rs (100%) rename {yara-x-py => py}/pyproject.toml (100%) rename {yara-x-py => py}/src/lib.rs (99%) rename {yara-x-py => py}/tests/test_api.py (95%) diff --git a/.github/workflows/python.yaml b/.github/workflows/python.yaml index 430117328..9f022fec6 100644 --- a/.github/workflows/python.yaml +++ b/.github/workflows/python.yaml @@ -47,8 +47,8 @@ jobs: pwd python -m pip install --upgrade pip maturin python -m pip install pytest - maturin develop --manifest-path yara-x-py/Cargo.toml - pytest yara-x-py + maturin develop --manifest-path py/Cargo.toml + pytest py # Non-windows - name: Test Python - Non-Windows @@ -58,6 +58,6 @@ jobs: virtualenv venv; source venv/bin/activate python -m pip install --upgrade pip maturin python -m pip install pytest - maturin develop --manifest-path yara-x-py/Cargo.toml - pytest yara-x-py + maturin develop --manifest-path py/Cargo.toml + pytest py diff --git a/Cargo.toml b/Cargo.toml index bd81b684c..80e030c44 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,14 +15,15 @@ rust-version = "1.73.0" [workspace] members = [ - "yara-x", - "yara-x-cli", - "yara-x-fmt", - "yara-x-macros", - "yara-x-parser", - "yara-x-proto", - "yara-x-proto-yaml", - "yara-x-py", + "lib", + "clib", + "cli", + "fmt", + "macros", + "parser", + "proto", + "proto-yaml", + "py", ] resolver = "2" @@ -84,12 +85,12 @@ walrus = "0.20.2" wasmtime = "17.0.0" yaml-rust = "0.4.5" yansi = "0.5.1" -yara-x = { path = "yara-x" } -yara-x-fmt = { path = "yara-x-fmt" } -yara-x-macros = { path = "yara-x-macros" } -yara-x-parser = { path = "yara-x-parser" } -yara-x-proto = { path = "yara-x-proto" } -yara-x-proto-yaml = { path = "yara-x-proto-yaml" } +yara-x = { path = "lib" } +yara-x-fmt = { path = "fmt" } +yara-x-macros = { path = "macros" } +yara-x-parser = { path = "parser" } +yara-x-proto = { path = "proto" } +yara-x-proto-yaml = { path = "proto-yaml" } # Special profile that builds a release binary with link-time optimization. # Compiling with this profile takes a while, but the resulting binary is diff --git a/yara-x-cli/Cargo.toml b/cli/Cargo.toml similarity index 100% rename from yara-x-cli/Cargo.toml rename to cli/Cargo.toml diff --git a/yara-x-cli/src/commands/check.rs b/cli/src/commands/check.rs similarity index 100% rename from yara-x-cli/src/commands/check.rs rename to cli/src/commands/check.rs diff --git a/yara-x-cli/src/commands/compile.rs b/cli/src/commands/compile.rs similarity index 100% rename from yara-x-cli/src/commands/compile.rs rename to cli/src/commands/compile.rs diff --git a/yara-x-cli/src/commands/debug.rs b/cli/src/commands/debug.rs similarity index 100% rename from yara-x-cli/src/commands/debug.rs rename to cli/src/commands/debug.rs diff --git a/yara-x-cli/src/commands/dump.rs b/cli/src/commands/dump.rs similarity index 100% rename from yara-x-cli/src/commands/dump.rs rename to cli/src/commands/dump.rs diff --git a/yara-x-cli/src/commands/fmt.rs b/cli/src/commands/fmt.rs similarity index 100% rename from yara-x-cli/src/commands/fmt.rs rename to cli/src/commands/fmt.rs diff --git a/yara-x-cli/src/commands/mod.rs b/cli/src/commands/mod.rs similarity index 100% rename from yara-x-cli/src/commands/mod.rs rename to cli/src/commands/mod.rs diff --git a/yara-x-cli/src/commands/scan.rs b/cli/src/commands/scan.rs similarity index 99% rename from yara-x-cli/src/commands/scan.rs rename to cli/src/commands/scan.rs index aaaa77de0..90a066c1e 100644 --- a/yara-x-cli/src/commands/scan.rs +++ b/cli/src/commands/scan.rs @@ -290,13 +290,13 @@ fn print_matching_rules( format!( "{}:{} {}", Cyan.paint(matching_rule.namespace()).bold(), - Cyan.paint(matching_rule.name()).bold(), + Cyan.paint(matching_rule.identifier()).bold(), file_path.display(), ) } else { format!( "{} {}", - Cyan.paint(matching_rule.name()).bold(), + Cyan.paint(matching_rule.identifier()).bold(), file_path.display() ) }; diff --git a/yara-x-cli/src/help.rs b/cli/src/help.rs similarity index 100% rename from yara-x-cli/src/help.rs rename to cli/src/help.rs diff --git a/yara-x-cli/src/main.rs b/cli/src/main.rs similarity index 100% rename from yara-x-cli/src/main.rs rename to cli/src/main.rs diff --git a/yara-x-cli/src/walk.rs b/cli/src/walk.rs similarity index 100% rename from yara-x-cli/src/walk.rs rename to cli/src/walk.rs diff --git a/yara-x-fmt/Cargo.lock b/fmt/Cargo.lock similarity index 100% rename from yara-x-fmt/Cargo.lock rename to fmt/Cargo.lock diff --git a/yara-x-fmt/Cargo.toml b/fmt/Cargo.toml similarity index 100% rename from yara-x-fmt/Cargo.toml rename to fmt/Cargo.toml diff --git a/yara-x-fmt/src/align.rs b/fmt/src/align.rs similarity index 100% rename from yara-x-fmt/src/align.rs rename to fmt/src/align.rs diff --git a/yara-x-fmt/src/bubble.rs b/fmt/src/bubble.rs similarity index 100% rename from yara-x-fmt/src/bubble.rs rename to fmt/src/bubble.rs diff --git a/yara-x-fmt/src/comments.rs b/fmt/src/comments.rs similarity index 100% rename from yara-x-fmt/src/comments.rs rename to fmt/src/comments.rs diff --git a/yara-x-fmt/src/indentation.rs b/fmt/src/indentation.rs similarity index 100% rename from yara-x-fmt/src/indentation.rs rename to fmt/src/indentation.rs diff --git a/yara-x-fmt/src/lib.rs b/fmt/src/lib.rs similarity index 100% rename from yara-x-fmt/src/lib.rs rename to fmt/src/lib.rs diff --git a/yara-x-fmt/src/processor/mod.rs b/fmt/src/processor/mod.rs similarity index 100% rename from yara-x-fmt/src/processor/mod.rs rename to fmt/src/processor/mod.rs diff --git a/yara-x-fmt/src/processor/tests.rs b/fmt/src/processor/tests.rs similarity index 100% rename from yara-x-fmt/src/processor/tests.rs rename to fmt/src/processor/tests.rs diff --git a/yara-x-fmt/src/testdata/test1.formatted b/fmt/src/testdata/test1.formatted similarity index 100% rename from yara-x-fmt/src/testdata/test1.formatted rename to fmt/src/testdata/test1.formatted diff --git a/yara-x-fmt/src/testdata/test1.unformatted b/fmt/src/testdata/test1.unformatted similarity index 100% rename from yara-x-fmt/src/testdata/test1.unformatted rename to fmt/src/testdata/test1.unformatted diff --git a/yara-x-fmt/src/testdata/test10.formatted b/fmt/src/testdata/test10.formatted similarity index 100% rename from yara-x-fmt/src/testdata/test10.formatted rename to fmt/src/testdata/test10.formatted diff --git a/yara-x-fmt/src/testdata/test10.unformatted b/fmt/src/testdata/test10.unformatted similarity index 100% rename from yara-x-fmt/src/testdata/test10.unformatted rename to fmt/src/testdata/test10.unformatted diff --git a/yara-x-fmt/src/testdata/test11.formatted b/fmt/src/testdata/test11.formatted similarity index 100% rename from yara-x-fmt/src/testdata/test11.formatted rename to fmt/src/testdata/test11.formatted diff --git a/yara-x-fmt/src/testdata/test11.unformatted b/fmt/src/testdata/test11.unformatted similarity index 100% rename from yara-x-fmt/src/testdata/test11.unformatted rename to fmt/src/testdata/test11.unformatted diff --git a/yara-x-fmt/src/testdata/test12.formatted b/fmt/src/testdata/test12.formatted similarity index 100% rename from yara-x-fmt/src/testdata/test12.formatted rename to fmt/src/testdata/test12.formatted diff --git a/yara-x-fmt/src/testdata/test12.unformatted b/fmt/src/testdata/test12.unformatted similarity index 100% rename from yara-x-fmt/src/testdata/test12.unformatted rename to fmt/src/testdata/test12.unformatted diff --git a/yara-x-fmt/src/testdata/test13.formatted b/fmt/src/testdata/test13.formatted similarity index 100% rename from yara-x-fmt/src/testdata/test13.formatted rename to fmt/src/testdata/test13.formatted diff --git a/yara-x-fmt/src/testdata/test13.unformatted b/fmt/src/testdata/test13.unformatted similarity index 100% rename from yara-x-fmt/src/testdata/test13.unformatted rename to fmt/src/testdata/test13.unformatted diff --git a/yara-x-fmt/src/testdata/test14.formatted b/fmt/src/testdata/test14.formatted similarity index 100% rename from yara-x-fmt/src/testdata/test14.formatted rename to fmt/src/testdata/test14.formatted diff --git a/yara-x-fmt/src/testdata/test14.unformatted b/fmt/src/testdata/test14.unformatted similarity index 100% rename from yara-x-fmt/src/testdata/test14.unformatted rename to fmt/src/testdata/test14.unformatted diff --git a/yara-x-fmt/src/testdata/test15.formatted b/fmt/src/testdata/test15.formatted similarity index 100% rename from yara-x-fmt/src/testdata/test15.formatted rename to fmt/src/testdata/test15.formatted diff --git a/yara-x-fmt/src/testdata/test15.unformatted b/fmt/src/testdata/test15.unformatted similarity index 100% rename from yara-x-fmt/src/testdata/test15.unformatted rename to fmt/src/testdata/test15.unformatted diff --git a/yara-x-fmt/src/testdata/test16.formatted b/fmt/src/testdata/test16.formatted similarity index 100% rename from yara-x-fmt/src/testdata/test16.formatted rename to fmt/src/testdata/test16.formatted diff --git a/yara-x-fmt/src/testdata/test16.unformatted b/fmt/src/testdata/test16.unformatted similarity index 100% rename from yara-x-fmt/src/testdata/test16.unformatted rename to fmt/src/testdata/test16.unformatted diff --git a/yara-x-fmt/src/testdata/test17.formatted b/fmt/src/testdata/test17.formatted similarity index 100% rename from yara-x-fmt/src/testdata/test17.formatted rename to fmt/src/testdata/test17.formatted diff --git a/yara-x-fmt/src/testdata/test17.unformatted b/fmt/src/testdata/test17.unformatted similarity index 100% rename from yara-x-fmt/src/testdata/test17.unformatted rename to fmt/src/testdata/test17.unformatted diff --git a/yara-x-fmt/src/testdata/test18.formatted b/fmt/src/testdata/test18.formatted similarity index 100% rename from yara-x-fmt/src/testdata/test18.formatted rename to fmt/src/testdata/test18.formatted diff --git a/yara-x-fmt/src/testdata/test18.unformatted b/fmt/src/testdata/test18.unformatted similarity index 100% rename from yara-x-fmt/src/testdata/test18.unformatted rename to fmt/src/testdata/test18.unformatted diff --git a/yara-x-fmt/src/testdata/test19.formatted b/fmt/src/testdata/test19.formatted similarity index 100% rename from yara-x-fmt/src/testdata/test19.formatted rename to fmt/src/testdata/test19.formatted diff --git a/yara-x-fmt/src/testdata/test19.unformatted b/fmt/src/testdata/test19.unformatted similarity index 100% rename from yara-x-fmt/src/testdata/test19.unformatted rename to fmt/src/testdata/test19.unformatted diff --git a/yara-x-fmt/src/testdata/test2.formatted b/fmt/src/testdata/test2.formatted similarity index 100% rename from yara-x-fmt/src/testdata/test2.formatted rename to fmt/src/testdata/test2.formatted diff --git a/yara-x-fmt/src/testdata/test2.unformatted b/fmt/src/testdata/test2.unformatted similarity index 100% rename from yara-x-fmt/src/testdata/test2.unformatted rename to fmt/src/testdata/test2.unformatted diff --git a/yara-x-fmt/src/testdata/test20.formatted b/fmt/src/testdata/test20.formatted similarity index 100% rename from yara-x-fmt/src/testdata/test20.formatted rename to fmt/src/testdata/test20.formatted diff --git a/yara-x-fmt/src/testdata/test20.unformatted b/fmt/src/testdata/test20.unformatted similarity index 100% rename from yara-x-fmt/src/testdata/test20.unformatted rename to fmt/src/testdata/test20.unformatted diff --git a/yara-x-fmt/src/testdata/test21.formatted b/fmt/src/testdata/test21.formatted similarity index 100% rename from yara-x-fmt/src/testdata/test21.formatted rename to fmt/src/testdata/test21.formatted diff --git a/yara-x-fmt/src/testdata/test21.unformatted b/fmt/src/testdata/test21.unformatted similarity index 100% rename from yara-x-fmt/src/testdata/test21.unformatted rename to fmt/src/testdata/test21.unformatted diff --git a/yara-x-fmt/src/testdata/test22.formatted b/fmt/src/testdata/test22.formatted similarity index 100% rename from yara-x-fmt/src/testdata/test22.formatted rename to fmt/src/testdata/test22.formatted diff --git a/yara-x-fmt/src/testdata/test22.unformatted b/fmt/src/testdata/test22.unformatted similarity index 100% rename from yara-x-fmt/src/testdata/test22.unformatted rename to fmt/src/testdata/test22.unformatted diff --git a/yara-x-fmt/src/testdata/test23.formatted b/fmt/src/testdata/test23.formatted similarity index 100% rename from yara-x-fmt/src/testdata/test23.formatted rename to fmt/src/testdata/test23.formatted diff --git a/yara-x-fmt/src/testdata/test23.unformatted b/fmt/src/testdata/test23.unformatted similarity index 100% rename from yara-x-fmt/src/testdata/test23.unformatted rename to fmt/src/testdata/test23.unformatted diff --git a/yara-x-fmt/src/testdata/test24.formatted b/fmt/src/testdata/test24.formatted similarity index 100% rename from yara-x-fmt/src/testdata/test24.formatted rename to fmt/src/testdata/test24.formatted diff --git a/yara-x-fmt/src/testdata/test24.unformatted b/fmt/src/testdata/test24.unformatted similarity index 100% rename from yara-x-fmt/src/testdata/test24.unformatted rename to fmt/src/testdata/test24.unformatted diff --git a/yara-x-fmt/src/testdata/test25.formatted b/fmt/src/testdata/test25.formatted similarity index 100% rename from yara-x-fmt/src/testdata/test25.formatted rename to fmt/src/testdata/test25.formatted diff --git a/yara-x-fmt/src/testdata/test25.unformatted b/fmt/src/testdata/test25.unformatted similarity index 100% rename from yara-x-fmt/src/testdata/test25.unformatted rename to fmt/src/testdata/test25.unformatted diff --git a/yara-x-fmt/src/testdata/test26.formatted b/fmt/src/testdata/test26.formatted similarity index 100% rename from yara-x-fmt/src/testdata/test26.formatted rename to fmt/src/testdata/test26.formatted diff --git a/yara-x-fmt/src/testdata/test26.unformatted b/fmt/src/testdata/test26.unformatted similarity index 100% rename from yara-x-fmt/src/testdata/test26.unformatted rename to fmt/src/testdata/test26.unformatted diff --git a/yara-x-fmt/src/testdata/test27.formatted b/fmt/src/testdata/test27.formatted similarity index 100% rename from yara-x-fmt/src/testdata/test27.formatted rename to fmt/src/testdata/test27.formatted diff --git a/yara-x-fmt/src/testdata/test27.unformatted b/fmt/src/testdata/test27.unformatted similarity index 100% rename from yara-x-fmt/src/testdata/test27.unformatted rename to fmt/src/testdata/test27.unformatted diff --git a/yara-x-fmt/src/testdata/test28.formatted b/fmt/src/testdata/test28.formatted similarity index 100% rename from yara-x-fmt/src/testdata/test28.formatted rename to fmt/src/testdata/test28.formatted diff --git a/yara-x-fmt/src/testdata/test28.unformatted b/fmt/src/testdata/test28.unformatted similarity index 100% rename from yara-x-fmt/src/testdata/test28.unformatted rename to fmt/src/testdata/test28.unformatted diff --git a/yara-x-fmt/src/testdata/test29.formatted b/fmt/src/testdata/test29.formatted similarity index 100% rename from yara-x-fmt/src/testdata/test29.formatted rename to fmt/src/testdata/test29.formatted diff --git a/yara-x-fmt/src/testdata/test29.unformatted b/fmt/src/testdata/test29.unformatted similarity index 100% rename from yara-x-fmt/src/testdata/test29.unformatted rename to fmt/src/testdata/test29.unformatted diff --git a/yara-x-fmt/src/testdata/test3.formatted b/fmt/src/testdata/test3.formatted similarity index 100% rename from yara-x-fmt/src/testdata/test3.formatted rename to fmt/src/testdata/test3.formatted diff --git a/yara-x-fmt/src/testdata/test3.unformatted b/fmt/src/testdata/test3.unformatted similarity index 100% rename from yara-x-fmt/src/testdata/test3.unformatted rename to fmt/src/testdata/test3.unformatted diff --git a/yara-x-fmt/src/testdata/test4.formatted b/fmt/src/testdata/test4.formatted similarity index 100% rename from yara-x-fmt/src/testdata/test4.formatted rename to fmt/src/testdata/test4.formatted diff --git a/yara-x-fmt/src/testdata/test4.unformatted b/fmt/src/testdata/test4.unformatted similarity index 100% rename from yara-x-fmt/src/testdata/test4.unformatted rename to fmt/src/testdata/test4.unformatted diff --git a/yara-x-fmt/src/testdata/test5.formatted b/fmt/src/testdata/test5.formatted similarity index 100% rename from yara-x-fmt/src/testdata/test5.formatted rename to fmt/src/testdata/test5.formatted diff --git a/yara-x-fmt/src/testdata/test5.unformatted b/fmt/src/testdata/test5.unformatted similarity index 100% rename from yara-x-fmt/src/testdata/test5.unformatted rename to fmt/src/testdata/test5.unformatted diff --git a/yara-x-fmt/src/testdata/test6.formatted b/fmt/src/testdata/test6.formatted similarity index 100% rename from yara-x-fmt/src/testdata/test6.formatted rename to fmt/src/testdata/test6.formatted diff --git a/yara-x-fmt/src/testdata/test6.unformatted b/fmt/src/testdata/test6.unformatted similarity index 100% rename from yara-x-fmt/src/testdata/test6.unformatted rename to fmt/src/testdata/test6.unformatted diff --git a/yara-x-fmt/src/testdata/test7.formatted b/fmt/src/testdata/test7.formatted similarity index 100% rename from yara-x-fmt/src/testdata/test7.formatted rename to fmt/src/testdata/test7.formatted diff --git a/yara-x-fmt/src/testdata/test7.unformatted b/fmt/src/testdata/test7.unformatted similarity index 100% rename from yara-x-fmt/src/testdata/test7.unformatted rename to fmt/src/testdata/test7.unformatted diff --git a/yara-x-fmt/src/testdata/test8.formatted b/fmt/src/testdata/test8.formatted similarity index 100% rename from yara-x-fmt/src/testdata/test8.formatted rename to fmt/src/testdata/test8.formatted diff --git a/yara-x-fmt/src/testdata/test8.unformatted b/fmt/src/testdata/test8.unformatted similarity index 100% rename from yara-x-fmt/src/testdata/test8.unformatted rename to fmt/src/testdata/test8.unformatted diff --git a/yara-x-fmt/src/testdata/test9.formatted b/fmt/src/testdata/test9.formatted similarity index 100% rename from yara-x-fmt/src/testdata/test9.formatted rename to fmt/src/testdata/test9.formatted diff --git a/yara-x-fmt/src/testdata/test9.unformatted b/fmt/src/testdata/test9.unformatted similarity index 100% rename from yara-x-fmt/src/testdata/test9.unformatted rename to fmt/src/testdata/test9.unformatted diff --git a/yara-x-fmt/src/tests.rs b/fmt/src/tests.rs similarity index 100% rename from yara-x-fmt/src/tests.rs rename to fmt/src/tests.rs diff --git a/yara-x-fmt/src/tokens/mod.rs b/fmt/src/tokens/mod.rs similarity index 100% rename from yara-x-fmt/src/tokens/mod.rs rename to fmt/src/tokens/mod.rs diff --git a/yara-x-fmt/src/tokens/tests.rs b/fmt/src/tokens/tests.rs similarity index 100% rename from yara-x-fmt/src/tokens/tests.rs rename to fmt/src/tokens/tests.rs diff --git a/yara-x-fmt/src/trailing_spaces.rs b/fmt/src/trailing_spaces.rs similarity index 100% rename from yara-x-fmt/src/trailing_spaces.rs rename to fmt/src/trailing_spaces.rs diff --git a/yara-x/Cargo.toml b/lib/Cargo.toml similarity index 100% rename from yara-x/Cargo.toml rename to lib/Cargo.toml diff --git a/yara-x/benches/benches.rs b/lib/benches/benches.rs similarity index 100% rename from yara-x/benches/benches.rs rename to lib/benches/benches.rs diff --git a/yara-x/build.rs b/lib/build.rs similarity index 93% rename from yara-x/build.rs rename to lib/build.rs index 91cfded4a..8a09de910 100644 --- a/yara-x/build.rs +++ b/lib/build.rs @@ -20,15 +20,15 @@ fn main() { .pure() .cargo_out_dir("protos") //.out_dir("src/modules/protos") - .include("../yara-x-proto/src") - .include("../yara-x-proto-yaml/src") + .include("../proto/src") + .include("../proto-yaml/src") .include("src/modules/protos") - .input("../yara-x-proto/src/yara.proto") - .input("../yara-x-proto-yaml/src/yaml.proto"); + .input("../proto/src/yara.proto") + .input("../proto-yaml/src/yaml.proto"); proto_parser - .include("../yara-x-proto/src") - .include("../yara-x-proto-yaml/src") + .include("../proto/src") + .include("../proto-yaml/src") .include("src/modules/protos"); for entry in fs::read_dir("src/modules/protos").unwrap() { diff --git a/yara-x/fuzz/.gitignore b/lib/fuzz/.gitignore similarity index 100% rename from yara-x/fuzz/.gitignore rename to lib/fuzz/.gitignore diff --git a/yara-x/fuzz/Cargo.lock b/lib/fuzz/Cargo.lock similarity index 100% rename from yara-x/fuzz/Cargo.lock rename to lib/fuzz/Cargo.lock diff --git a/yara-x/fuzz/Cargo.toml b/lib/fuzz/Cargo.toml similarity index 100% rename from yara-x/fuzz/Cargo.toml rename to lib/fuzz/Cargo.toml diff --git a/yara-x/fuzz/fuzz_targets/dotnet_parser.rs b/lib/fuzz/fuzz_targets/dotnet_parser.rs similarity index 100% rename from yara-x/fuzz/fuzz_targets/dotnet_parser.rs rename to lib/fuzz/fuzz_targets/dotnet_parser.rs diff --git a/yara-x/fuzz/fuzz_targets/elf_parser.rs b/lib/fuzz/fuzz_targets/elf_parser.rs similarity index 100% rename from yara-x/fuzz/fuzz_targets/elf_parser.rs rename to lib/fuzz/fuzz_targets/elf_parser.rs diff --git a/yara-x/fuzz/fuzz_targets/lnk_parser.rs b/lib/fuzz/fuzz_targets/lnk_parser.rs similarity index 100% rename from yara-x/fuzz/fuzz_targets/lnk_parser.rs rename to lib/fuzz/fuzz_targets/lnk_parser.rs diff --git a/yara-x/fuzz/fuzz_targets/macho_fat_parser.rs b/lib/fuzz/fuzz_targets/macho_fat_parser.rs similarity index 100% rename from yara-x/fuzz/fuzz_targets/macho_fat_parser.rs rename to lib/fuzz/fuzz_targets/macho_fat_parser.rs diff --git a/yara-x/fuzz/fuzz_targets/macho_parser.rs b/lib/fuzz/fuzz_targets/macho_parser.rs similarity index 100% rename from yara-x/fuzz/fuzz_targets/macho_parser.rs rename to lib/fuzz/fuzz_targets/macho_parser.rs diff --git a/yara-x/fuzz/fuzz_targets/pe_parser.rs b/lib/fuzz/fuzz_targets/pe_parser.rs similarity index 100% rename from yara-x/fuzz/fuzz_targets/pe_parser.rs rename to lib/fuzz/fuzz_targets/pe_parser.rs diff --git a/yara-x/fuzz/fuzz_targets/rule_compiler.rs b/lib/fuzz/fuzz_targets/rule_compiler.rs similarity index 100% rename from yara-x/fuzz/fuzz_targets/rule_compiler.rs rename to lib/fuzz/fuzz_targets/rule_compiler.rs diff --git a/yara-x/src/compiler/atoms/mask.rs b/lib/src/compiler/atoms/mask.rs similarity index 100% rename from yara-x/src/compiler/atoms/mask.rs rename to lib/src/compiler/atoms/mask.rs diff --git a/yara-x/src/compiler/atoms/mod.rs b/lib/src/compiler/atoms/mod.rs similarity index 100% rename from yara-x/src/compiler/atoms/mod.rs rename to lib/src/compiler/atoms/mod.rs diff --git a/yara-x/src/compiler/atoms/quality.rs b/lib/src/compiler/atoms/quality.rs similarity index 100% rename from yara-x/src/compiler/atoms/quality.rs rename to lib/src/compiler/atoms/quality.rs diff --git a/yara-x/src/compiler/base64.rs b/lib/src/compiler/base64.rs similarity index 100% rename from yara-x/src/compiler/base64.rs rename to lib/src/compiler/base64.rs diff --git a/yara-x/src/compiler/context.rs b/lib/src/compiler/context.rs similarity index 100% rename from yara-x/src/compiler/context.rs rename to lib/src/compiler/context.rs diff --git a/yara-x/src/compiler/emit.rs b/lib/src/compiler/emit.rs similarity index 100% rename from yara-x/src/compiler/emit.rs rename to lib/src/compiler/emit.rs diff --git a/yara-x/src/compiler/errors.rs b/lib/src/compiler/errors.rs similarity index 100% rename from yara-x/src/compiler/errors.rs rename to lib/src/compiler/errors.rs diff --git a/yara-x/src/compiler/ir/ast2ir.rs b/lib/src/compiler/ir/ast2ir.rs similarity index 100% rename from yara-x/src/compiler/ir/ast2ir.rs rename to lib/src/compiler/ir/ast2ir.rs diff --git a/yara-x/src/compiler/ir/hex2hir.rs b/lib/src/compiler/ir/hex2hir.rs similarity index 100% rename from yara-x/src/compiler/ir/hex2hir.rs rename to lib/src/compiler/ir/hex2hir.rs diff --git a/yara-x/src/compiler/ir/mod.rs b/lib/src/compiler/ir/mod.rs similarity index 100% rename from yara-x/src/compiler/ir/mod.rs rename to lib/src/compiler/ir/mod.rs diff --git a/yara-x/src/compiler/ir/utils.rs b/lib/src/compiler/ir/utils.rs similarity index 100% rename from yara-x/src/compiler/ir/utils.rs rename to lib/src/compiler/ir/utils.rs diff --git a/yara-x/src/compiler/mod.rs b/lib/src/compiler/mod.rs similarity index 100% rename from yara-x/src/compiler/mod.rs rename to lib/src/compiler/mod.rs diff --git a/yara-x/src/compiler/rules.rs b/lib/src/compiler/rules.rs similarity index 100% rename from yara-x/src/compiler/rules.rs rename to lib/src/compiler/rules.rs diff --git a/yara-x/src/compiler/tests/errors.rs b/lib/src/compiler/tests/errors.rs similarity index 100% rename from yara-x/src/compiler/tests/errors.rs rename to lib/src/compiler/tests/errors.rs diff --git a/yara-x/src/compiler/tests/mod.rs b/lib/src/compiler/tests/mod.rs similarity index 100% rename from yara-x/src/compiler/tests/mod.rs rename to lib/src/compiler/tests/mod.rs diff --git a/yara-x/src/compiler/tests/warnings.rs b/lib/src/compiler/tests/warnings.rs similarity index 100% rename from yara-x/src/compiler/tests/warnings.rs rename to lib/src/compiler/tests/warnings.rs diff --git a/yara-x/src/lib.rs b/lib/src/lib.rs similarity index 100% rename from yara-x/src/lib.rs rename to lib/src/lib.rs diff --git a/yara-x/src/modules/console.rs b/lib/src/modules/console.rs similarity index 100% rename from yara-x/src/modules/console.rs rename to lib/src/modules/console.rs diff --git a/yara-x/src/modules/dotnet/mod.rs b/lib/src/modules/dotnet/mod.rs similarity index 100% rename from yara-x/src/modules/dotnet/mod.rs rename to lib/src/modules/dotnet/mod.rs diff --git a/yara-x/src/modules/dotnet/parser.rs b/lib/src/modules/dotnet/parser.rs similarity index 100% rename from yara-x/src/modules/dotnet/parser.rs rename to lib/src/modules/dotnet/parser.rs diff --git a/yara-x/src/modules/dotnet/tests/testdata/0224ad9739f5fa64a2a63e23ef6a0e3152020fc7964243fc845cad499d11f9ab.in.zip b/lib/src/modules/dotnet/tests/testdata/0224ad9739f5fa64a2a63e23ef6a0e3152020fc7964243fc845cad499d11f9ab.in.zip similarity index 100% rename from yara-x/src/modules/dotnet/tests/testdata/0224ad9739f5fa64a2a63e23ef6a0e3152020fc7964243fc845cad499d11f9ab.in.zip rename to lib/src/modules/dotnet/tests/testdata/0224ad9739f5fa64a2a63e23ef6a0e3152020fc7964243fc845cad499d11f9ab.in.zip diff --git a/yara-x/src/modules/dotnet/tests/testdata/0224ad9739f5fa64a2a63e23ef6a0e3152020fc7964243fc845cad499d11f9ab.out b/lib/src/modules/dotnet/tests/testdata/0224ad9739f5fa64a2a63e23ef6a0e3152020fc7964243fc845cad499d11f9ab.out similarity index 100% rename from yara-x/src/modules/dotnet/tests/testdata/0224ad9739f5fa64a2a63e23ef6a0e3152020fc7964243fc845cad499d11f9ab.out rename to lib/src/modules/dotnet/tests/testdata/0224ad9739f5fa64a2a63e23ef6a0e3152020fc7964243fc845cad499d11f9ab.out diff --git a/yara-x/src/modules/dotnet/tests/testdata/4a8e3e00ce7fd88da587fbb187e07985a2f69fe1b94f3a83cfbba4e4104c2b26.in.zip b/lib/src/modules/dotnet/tests/testdata/4a8e3e00ce7fd88da587fbb187e07985a2f69fe1b94f3a83cfbba4e4104c2b26.in.zip similarity index 100% rename from yara-x/src/modules/dotnet/tests/testdata/4a8e3e00ce7fd88da587fbb187e07985a2f69fe1b94f3a83cfbba4e4104c2b26.in.zip rename to lib/src/modules/dotnet/tests/testdata/4a8e3e00ce7fd88da587fbb187e07985a2f69fe1b94f3a83cfbba4e4104c2b26.in.zip diff --git a/yara-x/src/modules/dotnet/tests/testdata/4a8e3e00ce7fd88da587fbb187e07985a2f69fe1b94f3a83cfbba4e4104c2b26.out b/lib/src/modules/dotnet/tests/testdata/4a8e3e00ce7fd88da587fbb187e07985a2f69fe1b94f3a83cfbba4e4104c2b26.out similarity index 100% rename from yara-x/src/modules/dotnet/tests/testdata/4a8e3e00ce7fd88da587fbb187e07985a2f69fe1b94f3a83cfbba4e4104c2b26.out rename to lib/src/modules/dotnet/tests/testdata/4a8e3e00ce7fd88da587fbb187e07985a2f69fe1b94f3a83cfbba4e4104c2b26.out diff --git a/yara-x/src/modules/dotnet/tests/testdata/605ebe5b5f4b94e1a73e0ad1162bd542e5cb948d1a4ea5a575a14d6b9d6ee849.in b/lib/src/modules/dotnet/tests/testdata/605ebe5b5f4b94e1a73e0ad1162bd542e5cb948d1a4ea5a575a14d6b9d6ee849.in similarity index 100% rename from yara-x/src/modules/dotnet/tests/testdata/605ebe5b5f4b94e1a73e0ad1162bd542e5cb948d1a4ea5a575a14d6b9d6ee849.in rename to lib/src/modules/dotnet/tests/testdata/605ebe5b5f4b94e1a73e0ad1162bd542e5cb948d1a4ea5a575a14d6b9d6ee849.in diff --git a/yara-x/src/modules/dotnet/tests/testdata/605ebe5b5f4b94e1a73e0ad1162bd542e5cb948d1a4ea5a575a14d6b9d6ee849.in.zip b/lib/src/modules/dotnet/tests/testdata/605ebe5b5f4b94e1a73e0ad1162bd542e5cb948d1a4ea5a575a14d6b9d6ee849.in.zip similarity index 100% rename from yara-x/src/modules/dotnet/tests/testdata/605ebe5b5f4b94e1a73e0ad1162bd542e5cb948d1a4ea5a575a14d6b9d6ee849.in.zip rename to lib/src/modules/dotnet/tests/testdata/605ebe5b5f4b94e1a73e0ad1162bd542e5cb948d1a4ea5a575a14d6b9d6ee849.in.zip diff --git a/yara-x/src/modules/dotnet/tests/testdata/605ebe5b5f4b94e1a73e0ad1162bd542e5cb948d1a4ea5a575a14d6b9d6ee849.out b/lib/src/modules/dotnet/tests/testdata/605ebe5b5f4b94e1a73e0ad1162bd542e5cb948d1a4ea5a575a14d6b9d6ee849.out similarity index 100% rename from yara-x/src/modules/dotnet/tests/testdata/605ebe5b5f4b94e1a73e0ad1162bd542e5cb948d1a4ea5a575a14d6b9d6ee849.out rename to lib/src/modules/dotnet/tests/testdata/605ebe5b5f4b94e1a73e0ad1162bd542e5cb948d1a4ea5a575a14d6b9d6ee849.out diff --git a/yara-x/src/modules/dotnet/tests/testdata/6d72a599724c4ff367b2c00b48030a62d286de5493c25c000a00f5fbc6e7b746.in.zip b/lib/src/modules/dotnet/tests/testdata/6d72a599724c4ff367b2c00b48030a62d286de5493c25c000a00f5fbc6e7b746.in.zip similarity index 100% rename from yara-x/src/modules/dotnet/tests/testdata/6d72a599724c4ff367b2c00b48030a62d286de5493c25c000a00f5fbc6e7b746.in.zip rename to lib/src/modules/dotnet/tests/testdata/6d72a599724c4ff367b2c00b48030a62d286de5493c25c000a00f5fbc6e7b746.in.zip diff --git a/yara-x/src/modules/dotnet/tests/testdata/6d72a599724c4ff367b2c00b48030a62d286de5493c25c000a00f5fbc6e7b746.out b/lib/src/modules/dotnet/tests/testdata/6d72a599724c4ff367b2c00b48030a62d286de5493c25c000a00f5fbc6e7b746.out similarity index 100% rename from yara-x/src/modules/dotnet/tests/testdata/6d72a599724c4ff367b2c00b48030a62d286de5493c25c000a00f5fbc6e7b746.out rename to lib/src/modules/dotnet/tests/testdata/6d72a599724c4ff367b2c00b48030a62d286de5493c25c000a00f5fbc6e7b746.out diff --git a/yara-x/src/modules/dotnet/tests/testdata/86a1e48cfc843eabfe1b468ef9358c1068950f849c612ab808225b359db0bb8c.in b/lib/src/modules/dotnet/tests/testdata/86a1e48cfc843eabfe1b468ef9358c1068950f849c612ab808225b359db0bb8c.in similarity index 100% rename from yara-x/src/modules/dotnet/tests/testdata/86a1e48cfc843eabfe1b468ef9358c1068950f849c612ab808225b359db0bb8c.in rename to lib/src/modules/dotnet/tests/testdata/86a1e48cfc843eabfe1b468ef9358c1068950f849c612ab808225b359db0bb8c.in diff --git a/yara-x/src/modules/dotnet/tests/testdata/86a1e48cfc843eabfe1b468ef9358c1068950f849c612ab808225b359db0bb8c.in.zip b/lib/src/modules/dotnet/tests/testdata/86a1e48cfc843eabfe1b468ef9358c1068950f849c612ab808225b359db0bb8c.in.zip similarity index 100% rename from yara-x/src/modules/dotnet/tests/testdata/86a1e48cfc843eabfe1b468ef9358c1068950f849c612ab808225b359db0bb8c.in.zip rename to lib/src/modules/dotnet/tests/testdata/86a1e48cfc843eabfe1b468ef9358c1068950f849c612ab808225b359db0bb8c.in.zip diff --git a/yara-x/src/modules/dotnet/tests/testdata/86a1e48cfc843eabfe1b468ef9358c1068950f849c612ab808225b359db0bb8c.out b/lib/src/modules/dotnet/tests/testdata/86a1e48cfc843eabfe1b468ef9358c1068950f849c612ab808225b359db0bb8c.out similarity index 100% rename from yara-x/src/modules/dotnet/tests/testdata/86a1e48cfc843eabfe1b468ef9358c1068950f849c612ab808225b359db0bb8c.out rename to lib/src/modules/dotnet/tests/testdata/86a1e48cfc843eabfe1b468ef9358c1068950f849c612ab808225b359db0bb8c.out diff --git a/yara-x/src/modules/dotnet/tests/testdata/88d0d0692e675cd5fafb57db4f940e98d03e054ca2b88ef5a41f72b5787841b3.in.zip b/lib/src/modules/dotnet/tests/testdata/88d0d0692e675cd5fafb57db4f940e98d03e054ca2b88ef5a41f72b5787841b3.in.zip similarity index 100% rename from yara-x/src/modules/dotnet/tests/testdata/88d0d0692e675cd5fafb57db4f940e98d03e054ca2b88ef5a41f72b5787841b3.in.zip rename to lib/src/modules/dotnet/tests/testdata/88d0d0692e675cd5fafb57db4f940e98d03e054ca2b88ef5a41f72b5787841b3.in.zip diff --git a/yara-x/src/modules/dotnet/tests/testdata/88d0d0692e675cd5fafb57db4f940e98d03e054ca2b88ef5a41f72b5787841b3.out b/lib/src/modules/dotnet/tests/testdata/88d0d0692e675cd5fafb57db4f940e98d03e054ca2b88ef5a41f72b5787841b3.out similarity index 100% rename from yara-x/src/modules/dotnet/tests/testdata/88d0d0692e675cd5fafb57db4f940e98d03e054ca2b88ef5a41f72b5787841b3.out rename to lib/src/modules/dotnet/tests/testdata/88d0d0692e675cd5fafb57db4f940e98d03e054ca2b88ef5a41f72b5787841b3.out diff --git a/yara-x/src/modules/dotnet/tests/testdata/984750efd1cb94e5ca7b366863af2092af954dad65df534bff603b9afcb49cd4.in.zip b/lib/src/modules/dotnet/tests/testdata/984750efd1cb94e5ca7b366863af2092af954dad65df534bff603b9afcb49cd4.in.zip similarity index 100% rename from yara-x/src/modules/dotnet/tests/testdata/984750efd1cb94e5ca7b366863af2092af954dad65df534bff603b9afcb49cd4.in.zip rename to lib/src/modules/dotnet/tests/testdata/984750efd1cb94e5ca7b366863af2092af954dad65df534bff603b9afcb49cd4.in.zip diff --git a/yara-x/src/modules/dotnet/tests/testdata/984750efd1cb94e5ca7b366863af2092af954dad65df534bff603b9afcb49cd4.out b/lib/src/modules/dotnet/tests/testdata/984750efd1cb94e5ca7b366863af2092af954dad65df534bff603b9afcb49cd4.out similarity index 100% rename from yara-x/src/modules/dotnet/tests/testdata/984750efd1cb94e5ca7b366863af2092af954dad65df534bff603b9afcb49cd4.out rename to lib/src/modules/dotnet/tests/testdata/984750efd1cb94e5ca7b366863af2092af954dad65df534bff603b9afcb49cd4.out diff --git a/yara-x/src/modules/elf/mod.rs b/lib/src/modules/elf/mod.rs similarity index 100% rename from yara-x/src/modules/elf/mod.rs rename to lib/src/modules/elf/mod.rs diff --git a/yara-x/src/modules/elf/parser.rs b/lib/src/modules/elf/parser.rs similarity index 100% rename from yara-x/src/modules/elf/parser.rs rename to lib/src/modules/elf/parser.rs diff --git a/yara-x/src/modules/elf/tests/mod.rs b/lib/src/modules/elf/tests/mod.rs similarity index 100% rename from yara-x/src/modules/elf/tests/mod.rs rename to lib/src/modules/elf/tests/mod.rs diff --git a/yara-x/src/modules/elf/tests/testdata/246f379d911603775271609e45cbbc2ee6f0d961abc71fbe105f69f23b54bcd3.in.zip b/lib/src/modules/elf/tests/testdata/246f379d911603775271609e45cbbc2ee6f0d961abc71fbe105f69f23b54bcd3.in.zip similarity index 100% rename from yara-x/src/modules/elf/tests/testdata/246f379d911603775271609e45cbbc2ee6f0d961abc71fbe105f69f23b54bcd3.in.zip rename to lib/src/modules/elf/tests/testdata/246f379d911603775271609e45cbbc2ee6f0d961abc71fbe105f69f23b54bcd3.in.zip diff --git a/yara-x/src/modules/elf/tests/testdata/246f379d911603775271609e45cbbc2ee6f0d961abc71fbe105f69f23b54bcd3.out b/lib/src/modules/elf/tests/testdata/246f379d911603775271609e45cbbc2ee6f0d961abc71fbe105f69f23b54bcd3.out similarity index 100% rename from yara-x/src/modules/elf/tests/testdata/246f379d911603775271609e45cbbc2ee6f0d961abc71fbe105f69f23b54bcd3.out rename to lib/src/modules/elf/tests/testdata/246f379d911603775271609e45cbbc2ee6f0d961abc71fbe105f69f23b54bcd3.out diff --git a/yara-x/src/modules/elf/tests/testdata/558ca94cffacf78aa9aebb8983ea5c6034e6f717bac38b1574bc00408c3a90ca.in.zip b/lib/src/modules/elf/tests/testdata/558ca94cffacf78aa9aebb8983ea5c6034e6f717bac38b1574bc00408c3a90ca.in.zip similarity index 100% rename from yara-x/src/modules/elf/tests/testdata/558ca94cffacf78aa9aebb8983ea5c6034e6f717bac38b1574bc00408c3a90ca.in.zip rename to lib/src/modules/elf/tests/testdata/558ca94cffacf78aa9aebb8983ea5c6034e6f717bac38b1574bc00408c3a90ca.in.zip diff --git a/yara-x/src/modules/elf/tests/testdata/558ca94cffacf78aa9aebb8983ea5c6034e6f717bac38b1574bc00408c3a90ca.out b/lib/src/modules/elf/tests/testdata/558ca94cffacf78aa9aebb8983ea5c6034e6f717bac38b1574bc00408c3a90ca.out similarity index 100% rename from yara-x/src/modules/elf/tests/testdata/558ca94cffacf78aa9aebb8983ea5c6034e6f717bac38b1574bc00408c3a90ca.out rename to lib/src/modules/elf/tests/testdata/558ca94cffacf78aa9aebb8983ea5c6034e6f717bac38b1574bc00408c3a90ca.out diff --git a/yara-x/src/modules/elf/tests/testdata/5ddb04cb08755e692fa540c51cdb8c93818264222518c11107aae7d586fabd9e.in.zip b/lib/src/modules/elf/tests/testdata/5ddb04cb08755e692fa540c51cdb8c93818264222518c11107aae7d586fabd9e.in.zip similarity index 100% rename from yara-x/src/modules/elf/tests/testdata/5ddb04cb08755e692fa540c51cdb8c93818264222518c11107aae7d586fabd9e.in.zip rename to lib/src/modules/elf/tests/testdata/5ddb04cb08755e692fa540c51cdb8c93818264222518c11107aae7d586fabd9e.in.zip diff --git a/yara-x/src/modules/elf/tests/testdata/5ddb04cb08755e692fa540c51cdb8c93818264222518c11107aae7d586fabd9e.out b/lib/src/modules/elf/tests/testdata/5ddb04cb08755e692fa540c51cdb8c93818264222518c11107aae7d586fabd9e.out similarity index 100% rename from yara-x/src/modules/elf/tests/testdata/5ddb04cb08755e692fa540c51cdb8c93818264222518c11107aae7d586fabd9e.out rename to lib/src/modules/elf/tests/testdata/5ddb04cb08755e692fa540c51cdb8c93818264222518c11107aae7d586fabd9e.out diff --git a/yara-x/src/modules/elf/tests/testdata/7c673873f6aca25d450216af9b5617527947c0c10a5d9632084a3dee1e828fa7.in.zip b/lib/src/modules/elf/tests/testdata/7c673873f6aca25d450216af9b5617527947c0c10a5d9632084a3dee1e828fa7.in.zip similarity index 100% rename from yara-x/src/modules/elf/tests/testdata/7c673873f6aca25d450216af9b5617527947c0c10a5d9632084a3dee1e828fa7.in.zip rename to lib/src/modules/elf/tests/testdata/7c673873f6aca25d450216af9b5617527947c0c10a5d9632084a3dee1e828fa7.in.zip diff --git a/yara-x/src/modules/elf/tests/testdata/7c673873f6aca25d450216af9b5617527947c0c10a5d9632084a3dee1e828fa7.out b/lib/src/modules/elf/tests/testdata/7c673873f6aca25d450216af9b5617527947c0c10a5d9632084a3dee1e828fa7.out similarity index 100% rename from yara-x/src/modules/elf/tests/testdata/7c673873f6aca25d450216af9b5617527947c0c10a5d9632084a3dee1e828fa7.out rename to lib/src/modules/elf/tests/testdata/7c673873f6aca25d450216af9b5617527947c0c10a5d9632084a3dee1e828fa7.out diff --git a/yara-x/src/modules/elf/tests/testdata/8bfe885838b4d1fba194b761ca900a0425aa892e4b358bf5a9bf4304e571df1b.in.zip b/lib/src/modules/elf/tests/testdata/8bfe885838b4d1fba194b761ca900a0425aa892e4b358bf5a9bf4304e571df1b.in.zip similarity index 100% rename from yara-x/src/modules/elf/tests/testdata/8bfe885838b4d1fba194b761ca900a0425aa892e4b358bf5a9bf4304e571df1b.in.zip rename to lib/src/modules/elf/tests/testdata/8bfe885838b4d1fba194b761ca900a0425aa892e4b358bf5a9bf4304e571df1b.in.zip diff --git a/yara-x/src/modules/elf/tests/testdata/8bfe885838b4d1fba194b761ca900a0425aa892e4b358bf5a9bf4304e571df1b.out b/lib/src/modules/elf/tests/testdata/8bfe885838b4d1fba194b761ca900a0425aa892e4b358bf5a9bf4304e571df1b.out similarity index 100% rename from yara-x/src/modules/elf/tests/testdata/8bfe885838b4d1fba194b761ca900a0425aa892e4b358bf5a9bf4304e571df1b.out rename to lib/src/modules/elf/tests/testdata/8bfe885838b4d1fba194b761ca900a0425aa892e4b358bf5a9bf4304e571df1b.out diff --git a/yara-x/src/modules/elf/tests/testdata/ac64f17c42eec4ec357e5fe9e58b51e4e1d3a026356da4043095f9b96ebce51c.in.zip b/lib/src/modules/elf/tests/testdata/ac64f17c42eec4ec357e5fe9e58b51e4e1d3a026356da4043095f9b96ebce51c.in.zip similarity index 100% rename from yara-x/src/modules/elf/tests/testdata/ac64f17c42eec4ec357e5fe9e58b51e4e1d3a026356da4043095f9b96ebce51c.in.zip rename to lib/src/modules/elf/tests/testdata/ac64f17c42eec4ec357e5fe9e58b51e4e1d3a026356da4043095f9b96ebce51c.in.zip diff --git a/yara-x/src/modules/elf/tests/testdata/ac64f17c42eec4ec357e5fe9e58b51e4e1d3a026356da4043095f9b96ebce51c.out b/lib/src/modules/elf/tests/testdata/ac64f17c42eec4ec357e5fe9e58b51e4e1d3a026356da4043095f9b96ebce51c.out similarity index 100% rename from yara-x/src/modules/elf/tests/testdata/ac64f17c42eec4ec357e5fe9e58b51e4e1d3a026356da4043095f9b96ebce51c.out rename to lib/src/modules/elf/tests/testdata/ac64f17c42eec4ec357e5fe9e58b51e4e1d3a026356da4043095f9b96ebce51c.out diff --git a/yara-x/src/modules/elf/tests/testdata/cc7ca38e635918e4928fe7d6fffe23a294f58042436d30b5f5126c16edbdc1a7.in.zip b/lib/src/modules/elf/tests/testdata/cc7ca38e635918e4928fe7d6fffe23a294f58042436d30b5f5126c16edbdc1a7.in.zip similarity index 100% rename from yara-x/src/modules/elf/tests/testdata/cc7ca38e635918e4928fe7d6fffe23a294f58042436d30b5f5126c16edbdc1a7.in.zip rename to lib/src/modules/elf/tests/testdata/cc7ca38e635918e4928fe7d6fffe23a294f58042436d30b5f5126c16edbdc1a7.in.zip diff --git a/yara-x/src/modules/elf/tests/testdata/cc7ca38e635918e4928fe7d6fffe23a294f58042436d30b5f5126c16edbdc1a7.out b/lib/src/modules/elf/tests/testdata/cc7ca38e635918e4928fe7d6fffe23a294f58042436d30b5f5126c16edbdc1a7.out similarity index 100% rename from yara-x/src/modules/elf/tests/testdata/cc7ca38e635918e4928fe7d6fffe23a294f58042436d30b5f5126c16edbdc1a7.out rename to lib/src/modules/elf/tests/testdata/cc7ca38e635918e4928fe7d6fffe23a294f58042436d30b5f5126c16edbdc1a7.out diff --git a/yara-x/src/modules/elf/tests/testdata/f040356a60cde1047266c841237b03e5d157c0628f5a348e64f63962265d1434.in.zip b/lib/src/modules/elf/tests/testdata/f040356a60cde1047266c841237b03e5d157c0628f5a348e64f63962265d1434.in.zip similarity index 100% rename from yara-x/src/modules/elf/tests/testdata/f040356a60cde1047266c841237b03e5d157c0628f5a348e64f63962265d1434.in.zip rename to lib/src/modules/elf/tests/testdata/f040356a60cde1047266c841237b03e5d157c0628f5a348e64f63962265d1434.in.zip diff --git a/yara-x/src/modules/elf/tests/testdata/f040356a60cde1047266c841237b03e5d157c0628f5a348e64f63962265d1434.out b/lib/src/modules/elf/tests/testdata/f040356a60cde1047266c841237b03e5d157c0628f5a348e64f63962265d1434.out similarity index 100% rename from yara-x/src/modules/elf/tests/testdata/f040356a60cde1047266c841237b03e5d157c0628f5a348e64f63962265d1434.out rename to lib/src/modules/elf/tests/testdata/f040356a60cde1047266c841237b03e5d157c0628f5a348e64f63962265d1434.out diff --git a/yara-x/src/modules/elf/tests/testdata/ff8a8921d672e5ce1676fb35001d5dc599ca4fb159be50ef740b31a32dfbcf67.in.zip b/lib/src/modules/elf/tests/testdata/ff8a8921d672e5ce1676fb35001d5dc599ca4fb159be50ef740b31a32dfbcf67.in.zip similarity index 100% rename from yara-x/src/modules/elf/tests/testdata/ff8a8921d672e5ce1676fb35001d5dc599ca4fb159be50ef740b31a32dfbcf67.in.zip rename to lib/src/modules/elf/tests/testdata/ff8a8921d672e5ce1676fb35001d5dc599ca4fb159be50ef740b31a32dfbcf67.in.zip diff --git a/yara-x/src/modules/elf/tests/testdata/ff8a8921d672e5ce1676fb35001d5dc599ca4fb159be50ef740b31a32dfbcf67.out b/lib/src/modules/elf/tests/testdata/ff8a8921d672e5ce1676fb35001d5dc599ca4fb159be50ef740b31a32dfbcf67.out similarity index 100% rename from yara-x/src/modules/elf/tests/testdata/ff8a8921d672e5ce1676fb35001d5dc599ca4fb159be50ef740b31a32dfbcf67.out rename to lib/src/modules/elf/tests/testdata/ff8a8921d672e5ce1676fb35001d5dc599ca4fb159be50ef740b31a32dfbcf67.out diff --git a/yara-x/src/modules/hash/mod.rs b/lib/src/modules/hash/mod.rs similarity index 100% rename from yara-x/src/modules/hash/mod.rs rename to lib/src/modules/hash/mod.rs diff --git a/yara-x/src/modules/hash/tests/mod.rs b/lib/src/modules/hash/tests/mod.rs similarity index 100% rename from yara-x/src/modules/hash/tests/mod.rs rename to lib/src/modules/hash/tests/mod.rs diff --git a/yara-x/src/modules/lnk/mod.rs b/lib/src/modules/lnk/mod.rs similarity index 100% rename from yara-x/src/modules/lnk/mod.rs rename to lib/src/modules/lnk/mod.rs diff --git a/yara-x/src/modules/lnk/parser.rs b/lib/src/modules/lnk/parser.rs similarity index 100% rename from yara-x/src/modules/lnk/parser.rs rename to lib/src/modules/lnk/parser.rs diff --git a/yara-x/src/modules/lnk/tests/testdata/lnk-extradata-1.in.zip b/lib/src/modules/lnk/tests/testdata/lnk-extradata-1.in.zip similarity index 100% rename from yara-x/src/modules/lnk/tests/testdata/lnk-extradata-1.in.zip rename to lib/src/modules/lnk/tests/testdata/lnk-extradata-1.in.zip diff --git a/yara-x/src/modules/lnk/tests/testdata/lnk-extradata-1.out b/lib/src/modules/lnk/tests/testdata/lnk-extradata-1.out similarity index 100% rename from yara-x/src/modules/lnk/tests/testdata/lnk-extradata-1.out rename to lib/src/modules/lnk/tests/testdata/lnk-extradata-1.out diff --git a/yara-x/src/modules/lnk/tests/testdata/lnk-extradata-2.in.zip b/lib/src/modules/lnk/tests/testdata/lnk-extradata-2.in.zip similarity index 100% rename from yara-x/src/modules/lnk/tests/testdata/lnk-extradata-2.in.zip rename to lib/src/modules/lnk/tests/testdata/lnk-extradata-2.in.zip diff --git a/yara-x/src/modules/lnk/tests/testdata/lnk-extradata-2.out b/lib/src/modules/lnk/tests/testdata/lnk-extradata-2.out similarity index 100% rename from yara-x/src/modules/lnk/tests/testdata/lnk-extradata-2.out rename to lib/src/modules/lnk/tests/testdata/lnk-extradata-2.out diff --git a/yara-x/src/modules/lnk/tests/testdata/lnk-malformed.in.zip b/lib/src/modules/lnk/tests/testdata/lnk-malformed.in.zip similarity index 100% rename from yara-x/src/modules/lnk/tests/testdata/lnk-malformed.in.zip rename to lib/src/modules/lnk/tests/testdata/lnk-malformed.in.zip diff --git a/yara-x/src/modules/lnk/tests/testdata/lnk-malformed.out b/lib/src/modules/lnk/tests/testdata/lnk-malformed.out similarity index 100% rename from yara-x/src/modules/lnk/tests/testdata/lnk-malformed.out rename to lib/src/modules/lnk/tests/testdata/lnk-malformed.out diff --git a/yara-x/src/modules/lnk/tests/testdata/lnk-network.in.zip b/lib/src/modules/lnk/tests/testdata/lnk-network.in.zip similarity index 100% rename from yara-x/src/modules/lnk/tests/testdata/lnk-network.in.zip rename to lib/src/modules/lnk/tests/testdata/lnk-network.in.zip diff --git a/yara-x/src/modules/lnk/tests/testdata/lnk-network.out b/lib/src/modules/lnk/tests/testdata/lnk-network.out similarity index 100% rename from yara-x/src/modules/lnk/tests/testdata/lnk-network.out rename to lib/src/modules/lnk/tests/testdata/lnk-network.out diff --git a/yara-x/src/modules/lnk/tests/testdata/lnk-overlay.in.zip b/lib/src/modules/lnk/tests/testdata/lnk-overlay.in.zip similarity index 100% rename from yara-x/src/modules/lnk/tests/testdata/lnk-overlay.in.zip rename to lib/src/modules/lnk/tests/testdata/lnk-overlay.in.zip diff --git a/yara-x/src/modules/lnk/tests/testdata/lnk-overlay.out b/lib/src/modules/lnk/tests/testdata/lnk-overlay.out similarity index 100% rename from yara-x/src/modules/lnk/tests/testdata/lnk-overlay.out rename to lib/src/modules/lnk/tests/testdata/lnk-overlay.out diff --git a/yara-x/src/modules/lnk/tests/testdata/lnk-standard.in.zip b/lib/src/modules/lnk/tests/testdata/lnk-standard.in.zip similarity index 100% rename from yara-x/src/modules/lnk/tests/testdata/lnk-standard.in.zip rename to lib/src/modules/lnk/tests/testdata/lnk-standard.in.zip diff --git a/yara-x/src/modules/lnk/tests/testdata/lnk-standard.out b/lib/src/modules/lnk/tests/testdata/lnk-standard.out similarity index 100% rename from yara-x/src/modules/lnk/tests/testdata/lnk-standard.out rename to lib/src/modules/lnk/tests/testdata/lnk-standard.out diff --git a/yara-x/src/modules/macho/mod.rs b/lib/src/modules/macho/mod.rs similarity index 100% rename from yara-x/src/modules/macho/mod.rs rename to lib/src/modules/macho/mod.rs diff --git a/yara-x/src/modules/macho/parser.rs b/lib/src/modules/macho/parser.rs similarity index 100% rename from yara-x/src/modules/macho/parser.rs rename to lib/src/modules/macho/parser.rs diff --git a/yara-x/src/modules/macho/tests/mod.rs b/lib/src/modules/macho/tests/mod.rs similarity index 100% rename from yara-x/src/modules/macho/tests/mod.rs rename to lib/src/modules/macho/tests/mod.rs diff --git a/yara-x/src/modules/macho/tests/testdata/01ac68a14f0ff5faa72bb33e768bfaae4d21de61f776e2405324c498ef52b21b.in.zip b/lib/src/modules/macho/tests/testdata/01ac68a14f0ff5faa72bb33e768bfaae4d21de61f776e2405324c498ef52b21b.in.zip similarity index 100% rename from yara-x/src/modules/macho/tests/testdata/01ac68a14f0ff5faa72bb33e768bfaae4d21de61f776e2405324c498ef52b21b.in.zip rename to lib/src/modules/macho/tests/testdata/01ac68a14f0ff5faa72bb33e768bfaae4d21de61f776e2405324c498ef52b21b.in.zip diff --git a/yara-x/src/modules/macho/tests/testdata/01ac68a14f0ff5faa72bb33e768bfaae4d21de61f776e2405324c498ef52b21b.out b/lib/src/modules/macho/tests/testdata/01ac68a14f0ff5faa72bb33e768bfaae4d21de61f776e2405324c498ef52b21b.out similarity index 100% rename from yara-x/src/modules/macho/tests/testdata/01ac68a14f0ff5faa72bb33e768bfaae4d21de61f776e2405324c498ef52b21b.out rename to lib/src/modules/macho/tests/testdata/01ac68a14f0ff5faa72bb33e768bfaae4d21de61f776e2405324c498ef52b21b.out diff --git a/yara-x/src/modules/macho/tests/testdata/3d27411e4465ae14ab2d02e1f2a6dafb2bb8114803e8b26c0652ce4afccb94e1.in.zip b/lib/src/modules/macho/tests/testdata/3d27411e4465ae14ab2d02e1f2a6dafb2bb8114803e8b26c0652ce4afccb94e1.in.zip similarity index 100% rename from yara-x/src/modules/macho/tests/testdata/3d27411e4465ae14ab2d02e1f2a6dafb2bb8114803e8b26c0652ce4afccb94e1.in.zip rename to lib/src/modules/macho/tests/testdata/3d27411e4465ae14ab2d02e1f2a6dafb2bb8114803e8b26c0652ce4afccb94e1.in.zip diff --git a/yara-x/src/modules/macho/tests/testdata/3d27411e4465ae14ab2d02e1f2a6dafb2bb8114803e8b26c0652ce4afccb94e1.out b/lib/src/modules/macho/tests/testdata/3d27411e4465ae14ab2d02e1f2a6dafb2bb8114803e8b26c0652ce4afccb94e1.out similarity index 100% rename from yara-x/src/modules/macho/tests/testdata/3d27411e4465ae14ab2d02e1f2a6dafb2bb8114803e8b26c0652ce4afccb94e1.out rename to lib/src/modules/macho/tests/testdata/3d27411e4465ae14ab2d02e1f2a6dafb2bb8114803e8b26c0652ce4afccb94e1.out diff --git a/yara-x/src/modules/macho/tests/testdata/4d0725d5c506c7ec26be89ec12b53a00bae94f868ce65a3352177dd6a75797ff.in.zip b/lib/src/modules/macho/tests/testdata/4d0725d5c506c7ec26be89ec12b53a00bae94f868ce65a3352177dd6a75797ff.in.zip similarity index 100% rename from yara-x/src/modules/macho/tests/testdata/4d0725d5c506c7ec26be89ec12b53a00bae94f868ce65a3352177dd6a75797ff.in.zip rename to lib/src/modules/macho/tests/testdata/4d0725d5c506c7ec26be89ec12b53a00bae94f868ce65a3352177dd6a75797ff.in.zip diff --git a/yara-x/src/modules/macho/tests/testdata/4d0725d5c506c7ec26be89ec12b53a00bae94f868ce65a3352177dd6a75797ff.out b/lib/src/modules/macho/tests/testdata/4d0725d5c506c7ec26be89ec12b53a00bae94f868ce65a3352177dd6a75797ff.out similarity index 100% rename from yara-x/src/modules/macho/tests/testdata/4d0725d5c506c7ec26be89ec12b53a00bae94f868ce65a3352177dd6a75797ff.out rename to lib/src/modules/macho/tests/testdata/4d0725d5c506c7ec26be89ec12b53a00bae94f868ce65a3352177dd6a75797ff.out diff --git a/yara-x/src/modules/macho/tests/testdata/5fad2944f6d9e36f6dc21ad3ea79fa394e400276ef05b93daa78717daffcd803.in.zip b/lib/src/modules/macho/tests/testdata/5fad2944f6d9e36f6dc21ad3ea79fa394e400276ef05b93daa78717daffcd803.in.zip similarity index 100% rename from yara-x/src/modules/macho/tests/testdata/5fad2944f6d9e36f6dc21ad3ea79fa394e400276ef05b93daa78717daffcd803.in.zip rename to lib/src/modules/macho/tests/testdata/5fad2944f6d9e36f6dc21ad3ea79fa394e400276ef05b93daa78717daffcd803.in.zip diff --git a/yara-x/src/modules/macho/tests/testdata/5fad2944f6d9e36f6dc21ad3ea79fa394e400276ef05b93daa78717daffcd803.out b/lib/src/modules/macho/tests/testdata/5fad2944f6d9e36f6dc21ad3ea79fa394e400276ef05b93daa78717daffcd803.out similarity index 100% rename from yara-x/src/modules/macho/tests/testdata/5fad2944f6d9e36f6dc21ad3ea79fa394e400276ef05b93daa78717daffcd803.out rename to lib/src/modules/macho/tests/testdata/5fad2944f6d9e36f6dc21ad3ea79fa394e400276ef05b93daa78717daffcd803.out diff --git a/yara-x/src/modules/macho/tests/testdata/8962a76d0aeaee3326cf840de11543c8beebeb768e712bd3b754b5cd3e151356.in.zip b/lib/src/modules/macho/tests/testdata/8962a76d0aeaee3326cf840de11543c8beebeb768e712bd3b754b5cd3e151356.in.zip similarity index 100% rename from yara-x/src/modules/macho/tests/testdata/8962a76d0aeaee3326cf840de11543c8beebeb768e712bd3b754b5cd3e151356.in.zip rename to lib/src/modules/macho/tests/testdata/8962a76d0aeaee3326cf840de11543c8beebeb768e712bd3b754b5cd3e151356.in.zip diff --git a/yara-x/src/modules/macho/tests/testdata/8962a76d0aeaee3326cf840de11543c8beebeb768e712bd3b754b5cd3e151356.out b/lib/src/modules/macho/tests/testdata/8962a76d0aeaee3326cf840de11543c8beebeb768e712bd3b754b5cd3e151356.out similarity index 100% rename from yara-x/src/modules/macho/tests/testdata/8962a76d0aeaee3326cf840de11543c8beebeb768e712bd3b754b5cd3e151356.out rename to lib/src/modules/macho/tests/testdata/8962a76d0aeaee3326cf840de11543c8beebeb768e712bd3b754b5cd3e151356.out diff --git a/yara-x/src/modules/macho/tests/testdata/94b2f690c776c4d847845be21b422cb8cd171119a7bf7a329ae18e5b34cdb85f.in.zip b/lib/src/modules/macho/tests/testdata/94b2f690c776c4d847845be21b422cb8cd171119a7bf7a329ae18e5b34cdb85f.in.zip similarity index 100% rename from yara-x/src/modules/macho/tests/testdata/94b2f690c776c4d847845be21b422cb8cd171119a7bf7a329ae18e5b34cdb85f.in.zip rename to lib/src/modules/macho/tests/testdata/94b2f690c776c4d847845be21b422cb8cd171119a7bf7a329ae18e5b34cdb85f.in.zip diff --git a/yara-x/src/modules/macho/tests/testdata/94b2f690c776c4d847845be21b422cb8cd171119a7bf7a329ae18e5b34cdb85f.out b/lib/src/modules/macho/tests/testdata/94b2f690c776c4d847845be21b422cb8cd171119a7bf7a329ae18e5b34cdb85f.out similarity index 100% rename from yara-x/src/modules/macho/tests/testdata/94b2f690c776c4d847845be21b422cb8cd171119a7bf7a329ae18e5b34cdb85f.out rename to lib/src/modules/macho/tests/testdata/94b2f690c776c4d847845be21b422cb8cd171119a7bf7a329ae18e5b34cdb85f.out diff --git a/yara-x/src/modules/macho/tests/testdata/a1da3d22c5ec85143faf5f69fb18d79cb0e7394c7b5065f74b7ce8c52ace75f1.in.zip b/lib/src/modules/macho/tests/testdata/a1da3d22c5ec85143faf5f69fb18d79cb0e7394c7b5065f74b7ce8c52ace75f1.in.zip similarity index 100% rename from yara-x/src/modules/macho/tests/testdata/a1da3d22c5ec85143faf5f69fb18d79cb0e7394c7b5065f74b7ce8c52ace75f1.in.zip rename to lib/src/modules/macho/tests/testdata/a1da3d22c5ec85143faf5f69fb18d79cb0e7394c7b5065f74b7ce8c52ace75f1.in.zip diff --git a/yara-x/src/modules/macho/tests/testdata/a1da3d22c5ec85143faf5f69fb18d79cb0e7394c7b5065f74b7ce8c52ace75f1.out b/lib/src/modules/macho/tests/testdata/a1da3d22c5ec85143faf5f69fb18d79cb0e7394c7b5065f74b7ce8c52ace75f1.out similarity index 100% rename from yara-x/src/modules/macho/tests/testdata/a1da3d22c5ec85143faf5f69fb18d79cb0e7394c7b5065f74b7ce8c52ace75f1.out rename to lib/src/modules/macho/tests/testdata/a1da3d22c5ec85143faf5f69fb18d79cb0e7394c7b5065f74b7ce8c52ace75f1.out diff --git a/yara-x/src/modules/macho/tests/testdata/edf47dd000b9fdf4519fb61c28d329a5bf085619bbde2c4146b0cc00ebad21c8.in.zip b/lib/src/modules/macho/tests/testdata/edf47dd000b9fdf4519fb61c28d329a5bf085619bbde2c4146b0cc00ebad21c8.in.zip similarity index 100% rename from yara-x/src/modules/macho/tests/testdata/edf47dd000b9fdf4519fb61c28d329a5bf085619bbde2c4146b0cc00ebad21c8.in.zip rename to lib/src/modules/macho/tests/testdata/edf47dd000b9fdf4519fb61c28d329a5bf085619bbde2c4146b0cc00ebad21c8.in.zip diff --git a/yara-x/src/modules/macho/tests/testdata/edf47dd000b9fdf4519fb61c28d329a5bf085619bbde2c4146b0cc00ebad21c8.out b/lib/src/modules/macho/tests/testdata/edf47dd000b9fdf4519fb61c28d329a5bf085619bbde2c4146b0cc00ebad21c8.out similarity index 100% rename from yara-x/src/modules/macho/tests/testdata/edf47dd000b9fdf4519fb61c28d329a5bf085619bbde2c4146b0cc00ebad21c8.out rename to lib/src/modules/macho/tests/testdata/edf47dd000b9fdf4519fb61c28d329a5bf085619bbde2c4146b0cc00ebad21c8.out diff --git a/yara-x/src/modules/macho/tests/testdata/macho_ppc_file.in.zip b/lib/src/modules/macho/tests/testdata/macho_ppc_file.in.zip similarity index 100% rename from yara-x/src/modules/macho/tests/testdata/macho_ppc_file.in.zip rename to lib/src/modules/macho/tests/testdata/macho_ppc_file.in.zip diff --git a/yara-x/src/modules/macho/tests/testdata/macho_ppc_file.out b/lib/src/modules/macho/tests/testdata/macho_ppc_file.out similarity index 100% rename from yara-x/src/modules/macho/tests/testdata/macho_ppc_file.out rename to lib/src/modules/macho/tests/testdata/macho_ppc_file.out diff --git a/yara-x/src/modules/macho/tests/testdata/macho_x86_64_dylib_file.in.zip b/lib/src/modules/macho/tests/testdata/macho_x86_64_dylib_file.in.zip similarity index 100% rename from yara-x/src/modules/macho/tests/testdata/macho_x86_64_dylib_file.in.zip rename to lib/src/modules/macho/tests/testdata/macho_x86_64_dylib_file.in.zip diff --git a/yara-x/src/modules/macho/tests/testdata/macho_x86_64_dylib_file.out b/lib/src/modules/macho/tests/testdata/macho_x86_64_dylib_file.out similarity index 100% rename from yara-x/src/modules/macho/tests/testdata/macho_x86_64_dylib_file.out rename to lib/src/modules/macho/tests/testdata/macho_x86_64_dylib_file.out diff --git a/yara-x/src/modules/macho/tests/testdata/macho_x86_file.in.zip b/lib/src/modules/macho/tests/testdata/macho_x86_file.in.zip similarity index 100% rename from yara-x/src/modules/macho/tests/testdata/macho_x86_file.in.zip rename to lib/src/modules/macho/tests/testdata/macho_x86_file.in.zip diff --git a/yara-x/src/modules/macho/tests/testdata/macho_x86_file.out b/lib/src/modules/macho/tests/testdata/macho_x86_file.out similarity index 100% rename from yara-x/src/modules/macho/tests/testdata/macho_x86_file.out rename to lib/src/modules/macho/tests/testdata/macho_x86_file.out diff --git a/yara-x/src/modules/macho/tests/testdata/macho_x86_object_file.in.zip b/lib/src/modules/macho/tests/testdata/macho_x86_object_file.in.zip similarity index 100% rename from yara-x/src/modules/macho/tests/testdata/macho_x86_object_file.in.zip rename to lib/src/modules/macho/tests/testdata/macho_x86_object_file.in.zip diff --git a/yara-x/src/modules/macho/tests/testdata/macho_x86_object_file.out b/lib/src/modules/macho/tests/testdata/macho_x86_object_file.out similarity index 100% rename from yara-x/src/modules/macho/tests/testdata/macho_x86_object_file.out rename to lib/src/modules/macho/tests/testdata/macho_x86_object_file.out diff --git a/yara-x/src/modules/macho/tests/testdata/tiny_macho.in.zip b/lib/src/modules/macho/tests/testdata/tiny_macho.in.zip similarity index 100% rename from yara-x/src/modules/macho/tests/testdata/tiny_macho.in.zip rename to lib/src/modules/macho/tests/testdata/tiny_macho.in.zip diff --git a/yara-x/src/modules/macho/tests/testdata/tiny_macho.out b/lib/src/modules/macho/tests/testdata/tiny_macho.out similarity index 100% rename from yara-x/src/modules/macho/tests/testdata/tiny_macho.out rename to lib/src/modules/macho/tests/testdata/tiny_macho.out diff --git a/yara-x/src/modules/macho/tests/testdata/tiny_universal.in.zip b/lib/src/modules/macho/tests/testdata/tiny_universal.in.zip similarity index 100% rename from yara-x/src/modules/macho/tests/testdata/tiny_universal.in.zip rename to lib/src/modules/macho/tests/testdata/tiny_universal.in.zip diff --git a/yara-x/src/modules/macho/tests/testdata/tiny_universal.out b/lib/src/modules/macho/tests/testdata/tiny_universal.out similarity index 100% rename from yara-x/src/modules/macho/tests/testdata/tiny_universal.out rename to lib/src/modules/macho/tests/testdata/tiny_universal.out diff --git a/yara-x/src/modules/math.rs b/lib/src/modules/math.rs similarity index 100% rename from yara-x/src/modules/math.rs rename to lib/src/modules/math.rs diff --git a/yara-x/src/modules/mod.rs b/lib/src/modules/mod.rs similarity index 100% rename from yara-x/src/modules/mod.rs rename to lib/src/modules/mod.rs diff --git a/yara-x/src/modules/modules.rs b/lib/src/modules/modules.rs similarity index 100% rename from yara-x/src/modules/modules.rs rename to lib/src/modules/modules.rs diff --git a/yara-x/src/modules/pe/mod.rs b/lib/src/modules/pe/mod.rs similarity index 100% rename from yara-x/src/modules/pe/mod.rs rename to lib/src/modules/pe/mod.rs diff --git a/yara-x/src/modules/pe/parser.rs b/lib/src/modules/pe/parser.rs similarity index 100% rename from yara-x/src/modules/pe/parser.rs rename to lib/src/modules/pe/parser.rs diff --git a/yara-x/src/modules/pe/rva2off.rs b/lib/src/modules/pe/rva2off.rs similarity index 100% rename from yara-x/src/modules/pe/rva2off.rs rename to lib/src/modules/pe/rva2off.rs diff --git a/yara-x/src/modules/pe/tests/mod.rs b/lib/src/modules/pe/tests/mod.rs similarity index 100% rename from yara-x/src/modules/pe/tests/mod.rs rename to lib/src/modules/pe/tests/mod.rs diff --git a/yara-x/src/modules/pe/tests/testdata/04ac6dd0c1cc33a49962ee0f3222597104f54a75683a5adee235401778279818.in.zip b/lib/src/modules/pe/tests/testdata/04ac6dd0c1cc33a49962ee0f3222597104f54a75683a5adee235401778279818.in.zip similarity index 100% rename from yara-x/src/modules/pe/tests/testdata/04ac6dd0c1cc33a49962ee0f3222597104f54a75683a5adee235401778279818.in.zip rename to lib/src/modules/pe/tests/testdata/04ac6dd0c1cc33a49962ee0f3222597104f54a75683a5adee235401778279818.in.zip diff --git a/yara-x/src/modules/pe/tests/testdata/04ac6dd0c1cc33a49962ee0f3222597104f54a75683a5adee235401778279818.out b/lib/src/modules/pe/tests/testdata/04ac6dd0c1cc33a49962ee0f3222597104f54a75683a5adee235401778279818.out similarity index 100% rename from yara-x/src/modules/pe/tests/testdata/04ac6dd0c1cc33a49962ee0f3222597104f54a75683a5adee235401778279818.out rename to lib/src/modules/pe/tests/testdata/04ac6dd0c1cc33a49962ee0f3222597104f54a75683a5adee235401778279818.out diff --git a/yara-x/src/modules/pe/tests/testdata/079a472d22290a94ebb212aa8015cdc8dd28a968c6b4d3b88acdd58ce2d3b885.in.zip b/lib/src/modules/pe/tests/testdata/079a472d22290a94ebb212aa8015cdc8dd28a968c6b4d3b88acdd58ce2d3b885.in.zip similarity index 100% rename from yara-x/src/modules/pe/tests/testdata/079a472d22290a94ebb212aa8015cdc8dd28a968c6b4d3b88acdd58ce2d3b885.in.zip rename to lib/src/modules/pe/tests/testdata/079a472d22290a94ebb212aa8015cdc8dd28a968c6b4d3b88acdd58ce2d3b885.in.zip diff --git a/yara-x/src/modules/pe/tests/testdata/079a472d22290a94ebb212aa8015cdc8dd28a968c6b4d3b88acdd58ce2d3b885.out b/lib/src/modules/pe/tests/testdata/079a472d22290a94ebb212aa8015cdc8dd28a968c6b4d3b88acdd58ce2d3b885.out similarity index 100% rename from yara-x/src/modules/pe/tests/testdata/079a472d22290a94ebb212aa8015cdc8dd28a968c6b4d3b88acdd58ce2d3b885.out rename to lib/src/modules/pe/tests/testdata/079a472d22290a94ebb212aa8015cdc8dd28a968c6b4d3b88acdd58ce2d3b885.out diff --git a/yara-x/src/modules/pe/tests/testdata/09e7d832320e51bcc80b9aecde2a4135267a9b0156642a9596a62e85c9998cc9.in.zip b/lib/src/modules/pe/tests/testdata/09e7d832320e51bcc80b9aecde2a4135267a9b0156642a9596a62e85c9998cc9.in.zip similarity index 100% rename from yara-x/src/modules/pe/tests/testdata/09e7d832320e51bcc80b9aecde2a4135267a9b0156642a9596a62e85c9998cc9.in.zip rename to lib/src/modules/pe/tests/testdata/09e7d832320e51bcc80b9aecde2a4135267a9b0156642a9596a62e85c9998cc9.in.zip diff --git a/yara-x/src/modules/pe/tests/testdata/09e7d832320e51bcc80b9aecde2a4135267a9b0156642a9596a62e85c9998cc9.out b/lib/src/modules/pe/tests/testdata/09e7d832320e51bcc80b9aecde2a4135267a9b0156642a9596a62e85c9998cc9.out similarity index 100% rename from yara-x/src/modules/pe/tests/testdata/09e7d832320e51bcc80b9aecde2a4135267a9b0156642a9596a62e85c9998cc9.out rename to lib/src/modules/pe/tests/testdata/09e7d832320e51bcc80b9aecde2a4135267a9b0156642a9596a62e85c9998cc9.out diff --git a/yara-x/src/modules/pe/tests/testdata/0ba6042247d90a187919dd88dc2d55cd882c80e5afc511c4f7b2e0e193968f7f.in.zip b/lib/src/modules/pe/tests/testdata/0ba6042247d90a187919dd88dc2d55cd882c80e5afc511c4f7b2e0e193968f7f.in.zip similarity index 100% rename from yara-x/src/modules/pe/tests/testdata/0ba6042247d90a187919dd88dc2d55cd882c80e5afc511c4f7b2e0e193968f7f.in.zip rename to lib/src/modules/pe/tests/testdata/0ba6042247d90a187919dd88dc2d55cd882c80e5afc511c4f7b2e0e193968f7f.in.zip diff --git a/yara-x/src/modules/pe/tests/testdata/0ba6042247d90a187919dd88dc2d55cd882c80e5afc511c4f7b2e0e193968f7f.out b/lib/src/modules/pe/tests/testdata/0ba6042247d90a187919dd88dc2d55cd882c80e5afc511c4f7b2e0e193968f7f.out similarity index 100% rename from yara-x/src/modules/pe/tests/testdata/0ba6042247d90a187919dd88dc2d55cd882c80e5afc511c4f7b2e0e193968f7f.out rename to lib/src/modules/pe/tests/testdata/0ba6042247d90a187919dd88dc2d55cd882c80e5afc511c4f7b2e0e193968f7f.out diff --git a/yara-x/src/modules/pe/tests/testdata/23e72ce7e9cdbc80c0095484ebeb02f56b21e48fd67044e69e7a2ae76db631e5.in.zip b/lib/src/modules/pe/tests/testdata/23e72ce7e9cdbc80c0095484ebeb02f56b21e48fd67044e69e7a2ae76db631e5.in.zip similarity index 100% rename from yara-x/src/modules/pe/tests/testdata/23e72ce7e9cdbc80c0095484ebeb02f56b21e48fd67044e69e7a2ae76db631e5.in.zip rename to lib/src/modules/pe/tests/testdata/23e72ce7e9cdbc80c0095484ebeb02f56b21e48fd67044e69e7a2ae76db631e5.in.zip diff --git a/yara-x/src/modules/pe/tests/testdata/23e72ce7e9cdbc80c0095484ebeb02f56b21e48fd67044e69e7a2ae76db631e5.out b/lib/src/modules/pe/tests/testdata/23e72ce7e9cdbc80c0095484ebeb02f56b21e48fd67044e69e7a2ae76db631e5.out similarity index 100% rename from yara-x/src/modules/pe/tests/testdata/23e72ce7e9cdbc80c0095484ebeb02f56b21e48fd67044e69e7a2ae76db631e5.out rename to lib/src/modules/pe/tests/testdata/23e72ce7e9cdbc80c0095484ebeb02f56b21e48fd67044e69e7a2ae76db631e5.out diff --git a/yara-x/src/modules/pe/tests/testdata/2775d97f8bdb3311ace960a42eee35dbec84b9d71a6abbacb26c14e83f5897e4.in.zip b/lib/src/modules/pe/tests/testdata/2775d97f8bdb3311ace960a42eee35dbec84b9d71a6abbacb26c14e83f5897e4.in.zip similarity index 100% rename from yara-x/src/modules/pe/tests/testdata/2775d97f8bdb3311ace960a42eee35dbec84b9d71a6abbacb26c14e83f5897e4.in.zip rename to lib/src/modules/pe/tests/testdata/2775d97f8bdb3311ace960a42eee35dbec84b9d71a6abbacb26c14e83f5897e4.in.zip diff --git a/yara-x/src/modules/pe/tests/testdata/2775d97f8bdb3311ace960a42eee35dbec84b9d71a6abbacb26c14e83f5897e4.out b/lib/src/modules/pe/tests/testdata/2775d97f8bdb3311ace960a42eee35dbec84b9d71a6abbacb26c14e83f5897e4.out similarity index 100% rename from yara-x/src/modules/pe/tests/testdata/2775d97f8bdb3311ace960a42eee35dbec84b9d71a6abbacb26c14e83f5897e4.out rename to lib/src/modules/pe/tests/testdata/2775d97f8bdb3311ace960a42eee35dbec84b9d71a6abbacb26c14e83f5897e4.out diff --git a/yara-x/src/modules/pe/tests/testdata/29eeeecf2c458ea3da1ce9d6d54742c0fad490cb2165f371f53b61941eedf072.in.zip b/lib/src/modules/pe/tests/testdata/29eeeecf2c458ea3da1ce9d6d54742c0fad490cb2165f371f53b61941eedf072.in.zip similarity index 100% rename from yara-x/src/modules/pe/tests/testdata/29eeeecf2c458ea3da1ce9d6d54742c0fad490cb2165f371f53b61941eedf072.in.zip rename to lib/src/modules/pe/tests/testdata/29eeeecf2c458ea3da1ce9d6d54742c0fad490cb2165f371f53b61941eedf072.in.zip diff --git a/yara-x/src/modules/pe/tests/testdata/29eeeecf2c458ea3da1ce9d6d54742c0fad490cb2165f371f53b61941eedf072.out b/lib/src/modules/pe/tests/testdata/29eeeecf2c458ea3da1ce9d6d54742c0fad490cb2165f371f53b61941eedf072.out similarity index 100% rename from yara-x/src/modules/pe/tests/testdata/29eeeecf2c458ea3da1ce9d6d54742c0fad490cb2165f371f53b61941eedf072.out rename to lib/src/modules/pe/tests/testdata/29eeeecf2c458ea3da1ce9d6d54742c0fad490cb2165f371f53b61941eedf072.out diff --git a/yara-x/src/modules/pe/tests/testdata/2d80c403b5c50f8bbacb65f58e7a19f272c62d1889216b7a6f1141571ec12649.in.zip b/lib/src/modules/pe/tests/testdata/2d80c403b5c50f8bbacb65f58e7a19f272c62d1889216b7a6f1141571ec12649.in.zip similarity index 100% rename from yara-x/src/modules/pe/tests/testdata/2d80c403b5c50f8bbacb65f58e7a19f272c62d1889216b7a6f1141571ec12649.in.zip rename to lib/src/modules/pe/tests/testdata/2d80c403b5c50f8bbacb65f58e7a19f272c62d1889216b7a6f1141571ec12649.in.zip diff --git a/yara-x/src/modules/pe/tests/testdata/2d80c403b5c50f8bbacb65f58e7a19f272c62d1889216b7a6f1141571ec12649.out b/lib/src/modules/pe/tests/testdata/2d80c403b5c50f8bbacb65f58e7a19f272c62d1889216b7a6f1141571ec12649.out similarity index 100% rename from yara-x/src/modules/pe/tests/testdata/2d80c403b5c50f8bbacb65f58e7a19f272c62d1889216b7a6f1141571ec12649.out rename to lib/src/modules/pe/tests/testdata/2d80c403b5c50f8bbacb65f58e7a19f272c62d1889216b7a6f1141571ec12649.out diff --git a/yara-x/src/modules/pe/tests/testdata/2e9c671b8a0411f2b397544b368c44d7f095eb395779de0ad1ac946914dfa34c.in.zip b/lib/src/modules/pe/tests/testdata/2e9c671b8a0411f2b397544b368c44d7f095eb395779de0ad1ac946914dfa34c.in.zip similarity index 100% rename from yara-x/src/modules/pe/tests/testdata/2e9c671b8a0411f2b397544b368c44d7f095eb395779de0ad1ac946914dfa34c.in.zip rename to lib/src/modules/pe/tests/testdata/2e9c671b8a0411f2b397544b368c44d7f095eb395779de0ad1ac946914dfa34c.in.zip diff --git a/yara-x/src/modules/pe/tests/testdata/2e9c671b8a0411f2b397544b368c44d7f095eb395779de0ad1ac946914dfa34c.out b/lib/src/modules/pe/tests/testdata/2e9c671b8a0411f2b397544b368c44d7f095eb395779de0ad1ac946914dfa34c.out similarity index 100% rename from yara-x/src/modules/pe/tests/testdata/2e9c671b8a0411f2b397544b368c44d7f095eb395779de0ad1ac946914dfa34c.out rename to lib/src/modules/pe/tests/testdata/2e9c671b8a0411f2b397544b368c44d7f095eb395779de0ad1ac946914dfa34c.out diff --git a/yara-x/src/modules/pe/tests/testdata/55cfd3bcea1aa352b4687c4d45bdcfea184ce9b58320891a2d72d4ec93766f14.in.zip b/lib/src/modules/pe/tests/testdata/55cfd3bcea1aa352b4687c4d45bdcfea184ce9b58320891a2d72d4ec93766f14.in.zip similarity index 100% rename from yara-x/src/modules/pe/tests/testdata/55cfd3bcea1aa352b4687c4d45bdcfea184ce9b58320891a2d72d4ec93766f14.in.zip rename to lib/src/modules/pe/tests/testdata/55cfd3bcea1aa352b4687c4d45bdcfea184ce9b58320891a2d72d4ec93766f14.in.zip diff --git a/yara-x/src/modules/pe/tests/testdata/55cfd3bcea1aa352b4687c4d45bdcfea184ce9b58320891a2d72d4ec93766f14.out b/lib/src/modules/pe/tests/testdata/55cfd3bcea1aa352b4687c4d45bdcfea184ce9b58320891a2d72d4ec93766f14.out similarity index 100% rename from yara-x/src/modules/pe/tests/testdata/55cfd3bcea1aa352b4687c4d45bdcfea184ce9b58320891a2d72d4ec93766f14.out rename to lib/src/modules/pe/tests/testdata/55cfd3bcea1aa352b4687c4d45bdcfea184ce9b58320891a2d72d4ec93766f14.out diff --git a/yara-x/src/modules/pe/tests/testdata/6c2abf4b80a87e63eee2996e5cea8f004d49ec0c1806080fa72e960529cba14c.in.zip b/lib/src/modules/pe/tests/testdata/6c2abf4b80a87e63eee2996e5cea8f004d49ec0c1806080fa72e960529cba14c.in.zip similarity index 100% rename from yara-x/src/modules/pe/tests/testdata/6c2abf4b80a87e63eee2996e5cea8f004d49ec0c1806080fa72e960529cba14c.in.zip rename to lib/src/modules/pe/tests/testdata/6c2abf4b80a87e63eee2996e5cea8f004d49ec0c1806080fa72e960529cba14c.in.zip diff --git a/yara-x/src/modules/pe/tests/testdata/6c2abf4b80a87e63eee2996e5cea8f004d49ec0c1806080fa72e960529cba14c.out b/lib/src/modules/pe/tests/testdata/6c2abf4b80a87e63eee2996e5cea8f004d49ec0c1806080fa72e960529cba14c.out similarity index 100% rename from yara-x/src/modules/pe/tests/testdata/6c2abf4b80a87e63eee2996e5cea8f004d49ec0c1806080fa72e960529cba14c.out rename to lib/src/modules/pe/tests/testdata/6c2abf4b80a87e63eee2996e5cea8f004d49ec0c1806080fa72e960529cba14c.out diff --git a/yara-x/src/modules/pe/tests/testdata/99df28014fae5f213c8decfde423b0eb69005158f981bc84560e9a5dde103d90.in.zip b/lib/src/modules/pe/tests/testdata/99df28014fae5f213c8decfde423b0eb69005158f981bc84560e9a5dde103d90.in.zip similarity index 100% rename from yara-x/src/modules/pe/tests/testdata/99df28014fae5f213c8decfde423b0eb69005158f981bc84560e9a5dde103d90.in.zip rename to lib/src/modules/pe/tests/testdata/99df28014fae5f213c8decfde423b0eb69005158f981bc84560e9a5dde103d90.in.zip diff --git a/yara-x/src/modules/pe/tests/testdata/99df28014fae5f213c8decfde423b0eb69005158f981bc84560e9a5dde103d90.out b/lib/src/modules/pe/tests/testdata/99df28014fae5f213c8decfde423b0eb69005158f981bc84560e9a5dde103d90.out similarity index 100% rename from yara-x/src/modules/pe/tests/testdata/99df28014fae5f213c8decfde423b0eb69005158f981bc84560e9a5dde103d90.out rename to lib/src/modules/pe/tests/testdata/99df28014fae5f213c8decfde423b0eb69005158f981bc84560e9a5dde103d90.out diff --git a/yara-x/src/modules/pe/tests/testdata/9bcf79a99ffbb1bd649503ce1406dea4181f6477cfb61f2130f37d014ce21888.in.zip b/lib/src/modules/pe/tests/testdata/9bcf79a99ffbb1bd649503ce1406dea4181f6477cfb61f2130f37d014ce21888.in.zip similarity index 100% rename from yara-x/src/modules/pe/tests/testdata/9bcf79a99ffbb1bd649503ce1406dea4181f6477cfb61f2130f37d014ce21888.in.zip rename to lib/src/modules/pe/tests/testdata/9bcf79a99ffbb1bd649503ce1406dea4181f6477cfb61f2130f37d014ce21888.in.zip diff --git a/yara-x/src/modules/pe/tests/testdata/9bcf79a99ffbb1bd649503ce1406dea4181f6477cfb61f2130f37d014ce21888.out b/lib/src/modules/pe/tests/testdata/9bcf79a99ffbb1bd649503ce1406dea4181f6477cfb61f2130f37d014ce21888.out similarity index 100% rename from yara-x/src/modules/pe/tests/testdata/9bcf79a99ffbb1bd649503ce1406dea4181f6477cfb61f2130f37d014ce21888.out rename to lib/src/modules/pe/tests/testdata/9bcf79a99ffbb1bd649503ce1406dea4181f6477cfb61f2130f37d014ce21888.out diff --git a/yara-x/src/modules/pe/tests/testdata/af3f20a9272489cbef4281c8c86ad42ccfb04ccedd3ada1e8c26939c726a4c8e.in.zip b/lib/src/modules/pe/tests/testdata/af3f20a9272489cbef4281c8c86ad42ccfb04ccedd3ada1e8c26939c726a4c8e.in.zip similarity index 100% rename from yara-x/src/modules/pe/tests/testdata/af3f20a9272489cbef4281c8c86ad42ccfb04ccedd3ada1e8c26939c726a4c8e.in.zip rename to lib/src/modules/pe/tests/testdata/af3f20a9272489cbef4281c8c86ad42ccfb04ccedd3ada1e8c26939c726a4c8e.in.zip diff --git a/yara-x/src/modules/pe/tests/testdata/af3f20a9272489cbef4281c8c86ad42ccfb04ccedd3ada1e8c26939c726a4c8e.out b/lib/src/modules/pe/tests/testdata/af3f20a9272489cbef4281c8c86ad42ccfb04ccedd3ada1e8c26939c726a4c8e.out similarity index 100% rename from yara-x/src/modules/pe/tests/testdata/af3f20a9272489cbef4281c8c86ad42ccfb04ccedd3ada1e8c26939c726a4c8e.out rename to lib/src/modules/pe/tests/testdata/af3f20a9272489cbef4281c8c86ad42ccfb04ccedd3ada1e8c26939c726a4c8e.out diff --git a/yara-x/src/modules/pe/tests/testdata/b8543d3aceec5a754292393f6602eeb966dbf9e198c94e3d74a9e9260e5f9870.in.zip b/lib/src/modules/pe/tests/testdata/b8543d3aceec5a754292393f6602eeb966dbf9e198c94e3d74a9e9260e5f9870.in.zip similarity index 100% rename from yara-x/src/modules/pe/tests/testdata/b8543d3aceec5a754292393f6602eeb966dbf9e198c94e3d74a9e9260e5f9870.in.zip rename to lib/src/modules/pe/tests/testdata/b8543d3aceec5a754292393f6602eeb966dbf9e198c94e3d74a9e9260e5f9870.in.zip diff --git a/yara-x/src/modules/pe/tests/testdata/b8543d3aceec5a754292393f6602eeb966dbf9e198c94e3d74a9e9260e5f9870.out b/lib/src/modules/pe/tests/testdata/b8543d3aceec5a754292393f6602eeb966dbf9e198c94e3d74a9e9260e5f9870.out similarity index 100% rename from yara-x/src/modules/pe/tests/testdata/b8543d3aceec5a754292393f6602eeb966dbf9e198c94e3d74a9e9260e5f9870.out rename to lib/src/modules/pe/tests/testdata/b8543d3aceec5a754292393f6602eeb966dbf9e198c94e3d74a9e9260e5f9870.out diff --git a/yara-x/src/modules/pe/tests/testdata/bd82090d9c6e23c1e2708550f4c21bbafd719dcc7cfc28f405ad3bc2783c6a12.in.zip b/lib/src/modules/pe/tests/testdata/bd82090d9c6e23c1e2708550f4c21bbafd719dcc7cfc28f405ad3bc2783c6a12.in.zip similarity index 100% rename from yara-x/src/modules/pe/tests/testdata/bd82090d9c6e23c1e2708550f4c21bbafd719dcc7cfc28f405ad3bc2783c6a12.in.zip rename to lib/src/modules/pe/tests/testdata/bd82090d9c6e23c1e2708550f4c21bbafd719dcc7cfc28f405ad3bc2783c6a12.in.zip diff --git a/yara-x/src/modules/pe/tests/testdata/bd82090d9c6e23c1e2708550f4c21bbafd719dcc7cfc28f405ad3bc2783c6a12.out b/lib/src/modules/pe/tests/testdata/bd82090d9c6e23c1e2708550f4c21bbafd719dcc7cfc28f405ad3bc2783c6a12.out similarity index 100% rename from yara-x/src/modules/pe/tests/testdata/bd82090d9c6e23c1e2708550f4c21bbafd719dcc7cfc28f405ad3bc2783c6a12.out rename to lib/src/modules/pe/tests/testdata/bd82090d9c6e23c1e2708550f4c21bbafd719dcc7cfc28f405ad3bc2783c6a12.out diff --git a/yara-x/src/modules/pe/tests/testdata/c6f9709feccf42f2d9e22057182fe185f177fb9daaa2649b4669a24f2ee7e3ba.in.zip b/lib/src/modules/pe/tests/testdata/c6f9709feccf42f2d9e22057182fe185f177fb9daaa2649b4669a24f2ee7e3ba.in.zip similarity index 100% rename from yara-x/src/modules/pe/tests/testdata/c6f9709feccf42f2d9e22057182fe185f177fb9daaa2649b4669a24f2ee7e3ba.in.zip rename to lib/src/modules/pe/tests/testdata/c6f9709feccf42f2d9e22057182fe185f177fb9daaa2649b4669a24f2ee7e3ba.in.zip diff --git a/yara-x/src/modules/pe/tests/testdata/c6f9709feccf42f2d9e22057182fe185f177fb9daaa2649b4669a24f2ee7e3ba.out b/lib/src/modules/pe/tests/testdata/c6f9709feccf42f2d9e22057182fe185f177fb9daaa2649b4669a24f2ee7e3ba.out similarity index 100% rename from yara-x/src/modules/pe/tests/testdata/c6f9709feccf42f2d9e22057182fe185f177fb9daaa2649b4669a24f2ee7e3ba.out rename to lib/src/modules/pe/tests/testdata/c6f9709feccf42f2d9e22057182fe185f177fb9daaa2649b4669a24f2ee7e3ba.out diff --git a/yara-x/src/modules/pe/tests/testdata/c703e95b9ec0ca955b1bcf33e1318a0029d9d4e7903cd63291946b334c314831.in.zip b/lib/src/modules/pe/tests/testdata/c703e95b9ec0ca955b1bcf33e1318a0029d9d4e7903cd63291946b334c314831.in.zip similarity index 100% rename from yara-x/src/modules/pe/tests/testdata/c703e95b9ec0ca955b1bcf33e1318a0029d9d4e7903cd63291946b334c314831.in.zip rename to lib/src/modules/pe/tests/testdata/c703e95b9ec0ca955b1bcf33e1318a0029d9d4e7903cd63291946b334c314831.in.zip diff --git a/yara-x/src/modules/pe/tests/testdata/c703e95b9ec0ca955b1bcf33e1318a0029d9d4e7903cd63291946b334c314831.out b/lib/src/modules/pe/tests/testdata/c703e95b9ec0ca955b1bcf33e1318a0029d9d4e7903cd63291946b334c314831.out similarity index 100% rename from yara-x/src/modules/pe/tests/testdata/c703e95b9ec0ca955b1bcf33e1318a0029d9d4e7903cd63291946b334c314831.out rename to lib/src/modules/pe/tests/testdata/c703e95b9ec0ca955b1bcf33e1318a0029d9d4e7903cd63291946b334c314831.out diff --git a/yara-x/src/modules/pe/tests/testdata/c704cca0fe4c9bdee18a302952540073b860e3b4d42e081f86d27bdb1cf6ede4.in.zip b/lib/src/modules/pe/tests/testdata/c704cca0fe4c9bdee18a302952540073b860e3b4d42e081f86d27bdb1cf6ede4.in.zip similarity index 100% rename from yara-x/src/modules/pe/tests/testdata/c704cca0fe4c9bdee18a302952540073b860e3b4d42e081f86d27bdb1cf6ede4.in.zip rename to lib/src/modules/pe/tests/testdata/c704cca0fe4c9bdee18a302952540073b860e3b4d42e081f86d27bdb1cf6ede4.in.zip diff --git a/yara-x/src/modules/pe/tests/testdata/c704cca0fe4c9bdee18a302952540073b860e3b4d42e081f86d27bdb1cf6ede4.out b/lib/src/modules/pe/tests/testdata/c704cca0fe4c9bdee18a302952540073b860e3b4d42e081f86d27bdb1cf6ede4.out similarity index 100% rename from yara-x/src/modules/pe/tests/testdata/c704cca0fe4c9bdee18a302952540073b860e3b4d42e081f86d27bdb1cf6ede4.out rename to lib/src/modules/pe/tests/testdata/c704cca0fe4c9bdee18a302952540073b860e3b4d42e081f86d27bdb1cf6ede4.out diff --git a/yara-x/src/modules/pe/tests/testdata/d009038e4e371f9cdc3a96923700c17d29de9dfc156666c98581dfeccc07548e.in.zip b/lib/src/modules/pe/tests/testdata/d009038e4e371f9cdc3a96923700c17d29de9dfc156666c98581dfeccc07548e.in.zip similarity index 100% rename from yara-x/src/modules/pe/tests/testdata/d009038e4e371f9cdc3a96923700c17d29de9dfc156666c98581dfeccc07548e.in.zip rename to lib/src/modules/pe/tests/testdata/d009038e4e371f9cdc3a96923700c17d29de9dfc156666c98581dfeccc07548e.in.zip diff --git a/yara-x/src/modules/pe/tests/testdata/d009038e4e371f9cdc3a96923700c17d29de9dfc156666c98581dfeccc07548e.out b/lib/src/modules/pe/tests/testdata/d009038e4e371f9cdc3a96923700c17d29de9dfc156666c98581dfeccc07548e.out similarity index 100% rename from yara-x/src/modules/pe/tests/testdata/d009038e4e371f9cdc3a96923700c17d29de9dfc156666c98581dfeccc07548e.out rename to lib/src/modules/pe/tests/testdata/d009038e4e371f9cdc3a96923700c17d29de9dfc156666c98581dfeccc07548e.out diff --git a/yara-x/src/modules/pe/tests/testdata/db6a9934570fa98a93a979e7e0e218e0c9710e5a787b18c6948f2eedd9338984.in.zip b/lib/src/modules/pe/tests/testdata/db6a9934570fa98a93a979e7e0e218e0c9710e5a787b18c6948f2eedd9338984.in.zip similarity index 100% rename from yara-x/src/modules/pe/tests/testdata/db6a9934570fa98a93a979e7e0e218e0c9710e5a787b18c6948f2eedd9338984.in.zip rename to lib/src/modules/pe/tests/testdata/db6a9934570fa98a93a979e7e0e218e0c9710e5a787b18c6948f2eedd9338984.in.zip diff --git a/yara-x/src/modules/pe/tests/testdata/db6a9934570fa98a93a979e7e0e218e0c9710e5a787b18c6948f2eedd9338984.out b/lib/src/modules/pe/tests/testdata/db6a9934570fa98a93a979e7e0e218e0c9710e5a787b18c6948f2eedd9338984.out similarity index 100% rename from yara-x/src/modules/pe/tests/testdata/db6a9934570fa98a93a979e7e0e218e0c9710e5a787b18c6948f2eedd9338984.out rename to lib/src/modules/pe/tests/testdata/db6a9934570fa98a93a979e7e0e218e0c9710e5a787b18c6948f2eedd9338984.out diff --git a/yara-x/src/modules/pe/tests/testdata/e3d45a2865818756068757d7e319258fef40dad54532ee4355b86bc129f27345.in.zip b/lib/src/modules/pe/tests/testdata/e3d45a2865818756068757d7e319258fef40dad54532ee4355b86bc129f27345.in.zip similarity index 100% rename from yara-x/src/modules/pe/tests/testdata/e3d45a2865818756068757d7e319258fef40dad54532ee4355b86bc129f27345.in.zip rename to lib/src/modules/pe/tests/testdata/e3d45a2865818756068757d7e319258fef40dad54532ee4355b86bc129f27345.in.zip diff --git a/yara-x/src/modules/pe/tests/testdata/e3d45a2865818756068757d7e319258fef40dad54532ee4355b86bc129f27345.out b/lib/src/modules/pe/tests/testdata/e3d45a2865818756068757d7e319258fef40dad54532ee4355b86bc129f27345.out similarity index 100% rename from yara-x/src/modules/pe/tests/testdata/e3d45a2865818756068757d7e319258fef40dad54532ee4355b86bc129f27345.out rename to lib/src/modules/pe/tests/testdata/e3d45a2865818756068757d7e319258fef40dad54532ee4355b86bc129f27345.out diff --git a/yara-x/src/modules/pe/tests/testdata/e5b038a7a8578bbe020784877bf13f8d3920141a39b502e870348ce3b254eb80.in.zip b/lib/src/modules/pe/tests/testdata/e5b038a7a8578bbe020784877bf13f8d3920141a39b502e870348ce3b254eb80.in.zip similarity index 100% rename from yara-x/src/modules/pe/tests/testdata/e5b038a7a8578bbe020784877bf13f8d3920141a39b502e870348ce3b254eb80.in.zip rename to lib/src/modules/pe/tests/testdata/e5b038a7a8578bbe020784877bf13f8d3920141a39b502e870348ce3b254eb80.in.zip diff --git a/yara-x/src/modules/pe/tests/testdata/e5b038a7a8578bbe020784877bf13f8d3920141a39b502e870348ce3b254eb80.out b/lib/src/modules/pe/tests/testdata/e5b038a7a8578bbe020784877bf13f8d3920141a39b502e870348ce3b254eb80.out similarity index 100% rename from yara-x/src/modules/pe/tests/testdata/e5b038a7a8578bbe020784877bf13f8d3920141a39b502e870348ce3b254eb80.out rename to lib/src/modules/pe/tests/testdata/e5b038a7a8578bbe020784877bf13f8d3920141a39b502e870348ce3b254eb80.out diff --git a/yara-x/src/modules/pe/tests/testdata/eda21be1b234c84acb8257f2f77c12e0826b06cd15eb6dec9bf3df6465216dd8.in.zip b/lib/src/modules/pe/tests/testdata/eda21be1b234c84acb8257f2f77c12e0826b06cd15eb6dec9bf3df6465216dd8.in.zip similarity index 100% rename from yara-x/src/modules/pe/tests/testdata/eda21be1b234c84acb8257f2f77c12e0826b06cd15eb6dec9bf3df6465216dd8.in.zip rename to lib/src/modules/pe/tests/testdata/eda21be1b234c84acb8257f2f77c12e0826b06cd15eb6dec9bf3df6465216dd8.in.zip diff --git a/yara-x/src/modules/pe/tests/testdata/eda21be1b234c84acb8257f2f77c12e0826b06cd15eb6dec9bf3df6465216dd8.out b/lib/src/modules/pe/tests/testdata/eda21be1b234c84acb8257f2f77c12e0826b06cd15eb6dec9bf3df6465216dd8.out similarity index 100% rename from yara-x/src/modules/pe/tests/testdata/eda21be1b234c84acb8257f2f77c12e0826b06cd15eb6dec9bf3df6465216dd8.out rename to lib/src/modules/pe/tests/testdata/eda21be1b234c84acb8257f2f77c12e0826b06cd15eb6dec9bf3df6465216dd8.out diff --git a/yara-x/src/modules/pe/tests/testdata/ee82699133743266399721dadb609173e10a4086fc9d7606fac6e945c0c15ca2.in.zip b/lib/src/modules/pe/tests/testdata/ee82699133743266399721dadb609173e10a4086fc9d7606fac6e945c0c15ca2.in.zip similarity index 100% rename from yara-x/src/modules/pe/tests/testdata/ee82699133743266399721dadb609173e10a4086fc9d7606fac6e945c0c15ca2.in.zip rename to lib/src/modules/pe/tests/testdata/ee82699133743266399721dadb609173e10a4086fc9d7606fac6e945c0c15ca2.in.zip diff --git a/yara-x/src/modules/pe/tests/testdata/ee82699133743266399721dadb609173e10a4086fc9d7606fac6e945c0c15ca2.out b/lib/src/modules/pe/tests/testdata/ee82699133743266399721dadb609173e10a4086fc9d7606fac6e945c0c15ca2.out similarity index 100% rename from yara-x/src/modules/pe/tests/testdata/ee82699133743266399721dadb609173e10a4086fc9d7606fac6e945c0c15ca2.out rename to lib/src/modules/pe/tests/testdata/ee82699133743266399721dadb609173e10a4086fc9d7606fac6e945c0c15ca2.out diff --git a/yara-x/src/modules/protos/console.proto b/lib/src/modules/protos/console.proto similarity index 100% rename from yara-x/src/modules/protos/console.proto rename to lib/src/modules/protos/console.proto diff --git a/yara-x/src/modules/protos/dotnet.proto b/lib/src/modules/protos/dotnet.proto similarity index 100% rename from yara-x/src/modules/protos/dotnet.proto rename to lib/src/modules/protos/dotnet.proto diff --git a/yara-x/src/modules/protos/elf.proto b/lib/src/modules/protos/elf.proto similarity index 100% rename from yara-x/src/modules/protos/elf.proto rename to lib/src/modules/protos/elf.proto diff --git a/yara-x/src/modules/protos/hash.proto b/lib/src/modules/protos/hash.proto similarity index 100% rename from yara-x/src/modules/protos/hash.proto rename to lib/src/modules/protos/hash.proto diff --git a/yara-x/src/modules/protos/lnk.proto b/lib/src/modules/protos/lnk.proto similarity index 100% rename from yara-x/src/modules/protos/lnk.proto rename to lib/src/modules/protos/lnk.proto diff --git a/yara-x/src/modules/protos/macho.proto b/lib/src/modules/protos/macho.proto similarity index 100% rename from yara-x/src/modules/protos/macho.proto rename to lib/src/modules/protos/macho.proto diff --git a/yara-x/src/modules/protos/math.proto b/lib/src/modules/protos/math.proto similarity index 100% rename from yara-x/src/modules/protos/math.proto rename to lib/src/modules/protos/math.proto diff --git a/yara-x/src/modules/protos/mods.proto b/lib/src/modules/protos/mods.proto similarity index 100% rename from yara-x/src/modules/protos/mods.proto rename to lib/src/modules/protos/mods.proto diff --git a/yara-x/src/modules/protos/pe.proto b/lib/src/modules/protos/pe.proto similarity index 100% rename from yara-x/src/modules/protos/pe.proto rename to lib/src/modules/protos/pe.proto diff --git a/yara-x/src/modules/protos/string.proto b/lib/src/modules/protos/string.proto similarity index 100% rename from yara-x/src/modules/protos/string.proto rename to lib/src/modules/protos/string.proto diff --git a/yara-x/src/modules/protos/test_proto2.proto b/lib/src/modules/protos/test_proto2.proto similarity index 100% rename from yara-x/src/modules/protos/test_proto2.proto rename to lib/src/modules/protos/test_proto2.proto diff --git a/yara-x/src/modules/protos/test_proto3.proto b/lib/src/modules/protos/test_proto3.proto similarity index 100% rename from yara-x/src/modules/protos/test_proto3.proto rename to lib/src/modules/protos/test_proto3.proto diff --git a/yara-x/src/modules/protos/text.proto b/lib/src/modules/protos/text.proto similarity index 100% rename from yara-x/src/modules/protos/text.proto rename to lib/src/modules/protos/text.proto diff --git a/yara-x/src/modules/protos/time.proto b/lib/src/modules/protos/time.proto similarity index 100% rename from yara-x/src/modules/protos/time.proto rename to lib/src/modules/protos/time.proto diff --git a/yara-x/src/modules/string.rs b/lib/src/modules/string.rs similarity index 100% rename from yara-x/src/modules/string.rs rename to lib/src/modules/string.rs diff --git a/yara-x/src/modules/test_proto2/mod.rs b/lib/src/modules/test_proto2/mod.rs similarity index 100% rename from yara-x/src/modules/test_proto2/mod.rs rename to lib/src/modules/test_proto2/mod.rs diff --git a/yara-x/src/modules/test_proto2/tests/mod.rs b/lib/src/modules/test_proto2/tests/mod.rs similarity index 100% rename from yara-x/src/modules/test_proto2/tests/mod.rs rename to lib/src/modules/test_proto2/tests/mod.rs diff --git a/yara-x/src/modules/test_proto3/mod.rs b/lib/src/modules/test_proto3/mod.rs similarity index 100% rename from yara-x/src/modules/test_proto3/mod.rs rename to lib/src/modules/test_proto3/mod.rs diff --git a/yara-x/src/modules/tests.rs b/lib/src/modules/tests.rs similarity index 100% rename from yara-x/src/modules/tests.rs rename to lib/src/modules/tests.rs diff --git a/yara-x/src/modules/text.rs b/lib/src/modules/text.rs similarity index 100% rename from yara-x/src/modules/text.rs rename to lib/src/modules/text.rs diff --git a/yara-x/src/modules/time.rs b/lib/src/modules/time.rs similarity index 100% rename from yara-x/src/modules/time.rs rename to lib/src/modules/time.rs diff --git a/yara-x/src/re/bitmapset.rs b/lib/src/re/bitmapset.rs similarity index 100% rename from yara-x/src/re/bitmapset.rs rename to lib/src/re/bitmapset.rs diff --git a/yara-x/src/re/fast/compiler.rs b/lib/src/re/fast/compiler.rs similarity index 100% rename from yara-x/src/re/fast/compiler.rs rename to lib/src/re/fast/compiler.rs diff --git a/yara-x/src/re/fast/fastvm.rs b/lib/src/re/fast/fastvm.rs similarity index 100% rename from yara-x/src/re/fast/fastvm.rs rename to lib/src/re/fast/fastvm.rs diff --git a/yara-x/src/re/fast/instr.rs b/lib/src/re/fast/instr.rs similarity index 100% rename from yara-x/src/re/fast/instr.rs rename to lib/src/re/fast/instr.rs diff --git a/yara-x/src/re/fast/mod.rs b/lib/src/re/fast/mod.rs similarity index 100% rename from yara-x/src/re/fast/mod.rs rename to lib/src/re/fast/mod.rs diff --git a/yara-x/src/re/hir.rs b/lib/src/re/hir.rs similarity index 100% rename from yara-x/src/re/hir.rs rename to lib/src/re/hir.rs diff --git a/yara-x/src/re/mod.rs b/lib/src/re/mod.rs similarity index 100% rename from yara-x/src/re/mod.rs rename to lib/src/re/mod.rs diff --git a/yara-x/src/re/parser.rs b/lib/src/re/parser.rs similarity index 100% rename from yara-x/src/re/parser.rs rename to lib/src/re/parser.rs diff --git a/yara-x/src/re/thompson/compiler.rs b/lib/src/re/thompson/compiler.rs similarity index 100% rename from yara-x/src/re/thompson/compiler.rs rename to lib/src/re/thompson/compiler.rs diff --git a/yara-x/src/re/thompson/instr.rs b/lib/src/re/thompson/instr.rs similarity index 100% rename from yara-x/src/re/thompson/instr.rs rename to lib/src/re/thompson/instr.rs diff --git a/yara-x/src/re/thompson/mod.rs b/lib/src/re/thompson/mod.rs similarity index 100% rename from yara-x/src/re/thompson/mod.rs rename to lib/src/re/thompson/mod.rs diff --git a/yara-x/src/re/thompson/pikevm.rs b/lib/src/re/thompson/pikevm.rs similarity index 100% rename from yara-x/src/re/thompson/pikevm.rs rename to lib/src/re/thompson/pikevm.rs diff --git a/yara-x/src/re/thompson/tests.rs b/lib/src/re/thompson/tests.rs similarity index 100% rename from yara-x/src/re/thompson/tests.rs rename to lib/src/re/thompson/tests.rs diff --git a/yara-x/src/scanner/context.rs b/lib/src/scanner/context.rs similarity index 100% rename from yara-x/src/scanner/context.rs rename to lib/src/scanner/context.rs diff --git a/yara-x/src/scanner/matches.rs b/lib/src/scanner/matches.rs similarity index 100% rename from yara-x/src/scanner/matches.rs rename to lib/src/scanner/matches.rs diff --git a/yara-x/src/scanner/mod.rs b/lib/src/scanner/mod.rs similarity index 99% rename from yara-x/src/scanner/mod.rs rename to lib/src/scanner/mod.rs index 503f91bd1..62509c5bd 100644 --- a/yara-x/src/scanner/mod.rs +++ b/lib/src/scanner/mod.rs @@ -816,7 +816,7 @@ pub struct Rule<'a, 'r> { impl<'a, 'r> Rule<'a, 'r> { /// Returns the rule's name. - pub fn name(&self) -> &'r str { + pub fn identifier(&self) -> &'r str { self.rules.ident_pool().get(self.rule_info.ident_id).unwrap() } diff --git a/yara-x/src/scanner/tests.rs b/lib/src/scanner/tests.rs similarity index 95% rename from yara-x/src/scanner/tests.rs rename to lib/src/scanner/tests.rs index 07aac939f..307a9ec7f 100644 --- a/yara-x/src/scanner/tests.rs +++ b/lib/src/scanner/tests.rs @@ -22,18 +22,18 @@ rule rule_4 { condition: false } let mut iter = results.matching_rules(); assert_eq!(iter.len(), 2); - assert_eq!(iter.next().unwrap().name(), "rule_1"); + assert_eq!(iter.next().unwrap().identifier(), "rule_1"); assert_eq!(iter.len(), 1); - assert_eq!(iter.next().unwrap().name(), "rule_3"); + assert_eq!(iter.next().unwrap().identifier(), "rule_3"); assert_eq!(iter.len(), 0); assert!(iter.next().is_none()); let mut iter = results.non_matching_rules(); assert_eq!(iter.len(), 2); - assert_eq!(iter.next().unwrap().name(), "rule_2"); + assert_eq!(iter.next().unwrap().identifier(), "rule_2"); assert_eq!(iter.len(), 1); - assert_eq!(iter.next().unwrap().name(), "rule_4"); + assert_eq!(iter.next().unwrap().identifier(), "rule_4"); assert_eq!(iter.len(), 0); assert!(iter.next().is_none()); } @@ -372,16 +372,16 @@ fn global_rules() { assert_eq!(results.matching_rules().len(), 1); let mut matching = results.matching_rules(); - assert_eq!(matching.next().unwrap().name(), "matching"); + assert_eq!(matching.next().unwrap().identifier(), "matching"); assert!(matching.next().is_none()); let mut non_matching = results.non_matching_rules(); // `global_true` and `non_matching` don't match because they are in the // namespace as `global_false`. - assert_eq!(non_matching.next().unwrap().name(), "global_true"); - assert_eq!(non_matching.next().unwrap().name(), "non_matching"); - assert_eq!(non_matching.next().unwrap().name(), "global_false"); + assert_eq!(non_matching.next().unwrap().identifier(), "global_true"); + assert_eq!(non_matching.next().unwrap().identifier(), "non_matching"); + assert_eq!(non_matching.next().unwrap().identifier(), "global_false"); assert!(non_matching.next().is_none()); } diff --git a/yara-x/src/string_pool.rs b/lib/src/string_pool.rs similarity index 100% rename from yara-x/src/string_pool.rs rename to lib/src/string_pool.rs diff --git a/yara-x/src/symbols/mod.rs b/lib/src/symbols/mod.rs similarity index 100% rename from yara-x/src/symbols/mod.rs rename to lib/src/symbols/mod.rs diff --git a/yara-x/src/tests/mod.rs b/lib/src/tests/mod.rs similarity index 100% rename from yara-x/src/tests/mod.rs rename to lib/src/tests/mod.rs diff --git a/yara-x/src/tests/testdata/jumps.bin b/lib/src/tests/testdata/jumps.bin similarity index 100% rename from yara-x/src/tests/testdata/jumps.bin rename to lib/src/tests/testdata/jumps.bin diff --git a/yara-x/src/types/array.rs b/lib/src/types/array.rs similarity index 100% rename from yara-x/src/types/array.rs rename to lib/src/types/array.rs diff --git a/yara-x/src/types/func.rs b/lib/src/types/func.rs similarity index 100% rename from yara-x/src/types/func.rs rename to lib/src/types/func.rs diff --git a/yara-x/src/types/map.rs b/lib/src/types/map.rs similarity index 100% rename from yara-x/src/types/map.rs rename to lib/src/types/map.rs diff --git a/yara-x/src/types/mod.rs b/lib/src/types/mod.rs similarity index 100% rename from yara-x/src/types/mod.rs rename to lib/src/types/mod.rs diff --git a/yara-x/src/types/structure.rs b/lib/src/types/structure.rs similarity index 100% rename from yara-x/src/types/structure.rs rename to lib/src/types/structure.rs diff --git a/yara-x/src/variables.rs b/lib/src/variables.rs similarity index 100% rename from yara-x/src/variables.rs rename to lib/src/variables.rs diff --git a/yara-x/src/wasm/builder.rs b/lib/src/wasm/builder.rs similarity index 100% rename from yara-x/src/wasm/builder.rs rename to lib/src/wasm/builder.rs diff --git a/yara-x/src/wasm/mod.rs b/lib/src/wasm/mod.rs similarity index 100% rename from yara-x/src/wasm/mod.rs rename to lib/src/wasm/mod.rs diff --git a/yara-x/src/wasm/string.rs b/lib/src/wasm/string.rs similarity index 100% rename from yara-x/src/wasm/string.rs rename to lib/src/wasm/string.rs diff --git a/yara-x-macros/Cargo.lock b/macros/Cargo.lock similarity index 100% rename from yara-x-macros/Cargo.lock rename to macros/Cargo.lock diff --git a/yara-x-macros/Cargo.toml b/macros/Cargo.toml similarity index 100% rename from yara-x-macros/Cargo.toml rename to macros/Cargo.toml diff --git a/yara-x-macros/src/error.rs b/macros/src/error.rs similarity index 100% rename from yara-x-macros/src/error.rs rename to macros/src/error.rs diff --git a/yara-x-macros/src/lib.rs b/macros/src/lib.rs similarity index 100% rename from yara-x-macros/src/lib.rs rename to macros/src/lib.rs diff --git a/yara-x-macros/src/module_export.rs b/macros/src/module_export.rs similarity index 100% rename from yara-x-macros/src/module_export.rs rename to macros/src/module_export.rs diff --git a/yara-x-macros/src/module_main.rs b/macros/src/module_main.rs similarity index 100% rename from yara-x-macros/src/module_main.rs rename to macros/src/module_main.rs diff --git a/yara-x-macros/src/span.rs b/macros/src/span.rs similarity index 100% rename from yara-x-macros/src/span.rs rename to macros/src/span.rs diff --git a/yara-x-macros/src/wasm_export.rs b/macros/src/wasm_export.rs similarity index 100% rename from yara-x-macros/src/wasm_export.rs rename to macros/src/wasm_export.rs diff --git a/yara-x-parser/Cargo.lock b/parser/Cargo.lock similarity index 100% rename from yara-x-parser/Cargo.lock rename to parser/Cargo.lock diff --git a/yara-x-parser/Cargo.toml b/parser/Cargo.toml similarity index 100% rename from yara-x-parser/Cargo.toml rename to parser/Cargo.toml diff --git a/yara-x-parser/src/ast/ascii_tree.rs b/parser/src/ast/ascii_tree.rs similarity index 100% rename from yara-x-parser/src/ast/ascii_tree.rs rename to parser/src/ast/ascii_tree.rs diff --git a/yara-x-parser/src/ast/mod.rs b/parser/src/ast/mod.rs similarity index 100% rename from yara-x-parser/src/ast/mod.rs rename to parser/src/ast/mod.rs diff --git a/yara-x-parser/src/ast/span.rs b/parser/src/ast/span.rs similarity index 100% rename from yara-x-parser/src/ast/span.rs rename to parser/src/ast/span.rs diff --git a/yara-x-parser/src/cst/mod.rs b/parser/src/cst/mod.rs similarity index 100% rename from yara-x-parser/src/cst/mod.rs rename to parser/src/cst/mod.rs diff --git a/yara-x-parser/src/lib.rs b/parser/src/lib.rs similarity index 100% rename from yara-x-parser/src/lib.rs rename to parser/src/lib.rs diff --git a/yara-x-parser/src/parser/context.rs b/parser/src/parser/context.rs similarity index 100% rename from yara-x-parser/src/parser/context.rs rename to parser/src/parser/context.rs diff --git a/yara-x-parser/src/parser/cst2ast.rs b/parser/src/parser/cst2ast.rs similarity index 100% rename from yara-x-parser/src/parser/cst2ast.rs rename to parser/src/parser/cst2ast.rs diff --git a/yara-x-parser/src/parser/errors.rs b/parser/src/parser/errors.rs similarity index 100% rename from yara-x-parser/src/parser/errors.rs rename to parser/src/parser/errors.rs diff --git a/yara-x-parser/src/parser/grammar.pest b/parser/src/parser/grammar.pest similarity index 100% rename from yara-x-parser/src/parser/grammar.pest rename to parser/src/parser/grammar.pest diff --git a/yara-x-parser/src/parser/mod.rs b/parser/src/parser/mod.rs similarity index 100% rename from yara-x-parser/src/parser/mod.rs rename to parser/src/parser/mod.rs diff --git a/yara-x-parser/src/parser/tests/ast.rs b/parser/src/parser/tests/ast.rs similarity index 100% rename from yara-x-parser/src/parser/tests/ast.rs rename to parser/src/parser/tests/ast.rs diff --git a/yara-x-parser/src/parser/tests/cst.rs b/parser/src/parser/tests/cst.rs similarity index 100% rename from yara-x-parser/src/parser/tests/cst.rs rename to parser/src/parser/tests/cst.rs diff --git a/yara-x-parser/src/parser/tests/errors.rs b/parser/src/parser/tests/errors.rs similarity index 100% rename from yara-x-parser/src/parser/tests/errors.rs rename to parser/src/parser/tests/errors.rs diff --git a/yara-x-parser/src/parser/tests/mod.rs b/parser/src/parser/tests/mod.rs similarity index 100% rename from yara-x-parser/src/parser/tests/mod.rs rename to parser/src/parser/tests/mod.rs diff --git a/yara-x-parser/src/parser/tests/testdata/arithmetic_ops.yaml b/parser/src/parser/tests/testdata/arithmetic_ops.yaml similarity index 100% rename from yara-x-parser/src/parser/tests/testdata/arithmetic_ops.yaml rename to parser/src/parser/tests/testdata/arithmetic_ops.yaml diff --git a/yara-x-parser/src/parser/tests/testdata/bitwise_ops.yaml b/parser/src/parser/tests/testdata/bitwise_ops.yaml similarity index 100% rename from yara-x-parser/src/parser/tests/testdata/bitwise_ops.yaml rename to parser/src/parser/tests/testdata/bitwise_ops.yaml diff --git a/yara-x-parser/src/parser/tests/testdata/for_stmt.yaml b/parser/src/parser/tests/testdata/for_stmt.yaml similarity index 100% rename from yara-x-parser/src/parser/tests/testdata/for_stmt.yaml rename to parser/src/parser/tests/testdata/for_stmt.yaml diff --git a/yara-x-parser/src/parser/tests/testdata/general.yaml b/parser/src/parser/tests/testdata/general.yaml similarity index 100% rename from yara-x-parser/src/parser/tests/testdata/general.yaml rename to parser/src/parser/tests/testdata/general.yaml diff --git a/yara-x-parser/src/parser/tests/testdata/hex_patterns.yaml b/parser/src/parser/tests/testdata/hex_patterns.yaml similarity index 100% rename from yara-x-parser/src/parser/tests/testdata/hex_patterns.yaml rename to parser/src/parser/tests/testdata/hex_patterns.yaml diff --git a/yara-x-parser/src/parser/tests/testdata/literals.yaml b/parser/src/parser/tests/testdata/literals.yaml similarity index 100% rename from yara-x-parser/src/parser/tests/testdata/literals.yaml rename to parser/src/parser/tests/testdata/literals.yaml diff --git a/yara-x-parser/src/parser/tests/testdata/meta.yaml b/parser/src/parser/tests/testdata/meta.yaml similarity index 100% rename from yara-x-parser/src/parser/tests/testdata/meta.yaml rename to parser/src/parser/tests/testdata/meta.yaml diff --git a/yara-x-parser/src/parser/tests/testdata/of_stmt.yaml b/parser/src/parser/tests/testdata/of_stmt.yaml similarity index 100% rename from yara-x-parser/src/parser/tests/testdata/of_stmt.yaml rename to parser/src/parser/tests/testdata/of_stmt.yaml diff --git a/yara-x-parser/src/parser/tests/testdata/regexp_patterns.yaml b/parser/src/parser/tests/testdata/regexp_patterns.yaml similarity index 100% rename from yara-x-parser/src/parser/tests/testdata/regexp_patterns.yaml rename to parser/src/parser/tests/testdata/regexp_patterns.yaml diff --git a/yara-x-parser/src/parser/tests/testdata/shift_ops.yaml b/parser/src/parser/tests/testdata/shift_ops.yaml similarity index 100% rename from yara-x-parser/src/parser/tests/testdata/shift_ops.yaml rename to parser/src/parser/tests/testdata/shift_ops.yaml diff --git a/yara-x-parser/src/parser/tests/testdata/string_ops.yaml b/parser/src/parser/tests/testdata/string_ops.yaml similarity index 100% rename from yara-x-parser/src/parser/tests/testdata/string_ops.yaml rename to parser/src/parser/tests/testdata/string_ops.yaml diff --git a/yara-x-parser/src/parser/tests/testdata/text_patterns.yaml b/parser/src/parser/tests/testdata/text_patterns.yaml similarity index 100% rename from yara-x-parser/src/parser/tests/testdata/text_patterns.yaml rename to parser/src/parser/tests/testdata/text_patterns.yaml diff --git a/yara-x-parser/src/report.rs b/parser/src/report.rs similarity index 100% rename from yara-x-parser/src/report.rs rename to parser/src/report.rs diff --git a/yara-x-parser/src/warnings.rs b/parser/src/warnings.rs similarity index 100% rename from yara-x-parser/src/warnings.rs rename to parser/src/warnings.rs diff --git a/yara-x-proto-yaml/Cargo.toml b/proto-yaml/Cargo.toml similarity index 100% rename from yara-x-proto-yaml/Cargo.toml rename to proto-yaml/Cargo.toml diff --git a/yara-x-proto-yaml/build.rs b/proto-yaml/build.rs similarity index 100% rename from yara-x-proto-yaml/build.rs rename to proto-yaml/build.rs diff --git a/yara-x-proto-yaml/src/lib.rs b/proto-yaml/src/lib.rs similarity index 100% rename from yara-x-proto-yaml/src/lib.rs rename to proto-yaml/src/lib.rs diff --git a/yara-x-proto-yaml/src/tests/mod.rs b/proto-yaml/src/tests/mod.rs similarity index 100% rename from yara-x-proto-yaml/src/tests/mod.rs rename to proto-yaml/src/tests/mod.rs diff --git a/yara-x-proto-yaml/src/tests/test.proto b/proto-yaml/src/tests/test.proto similarity index 100% rename from yara-x-proto-yaml/src/tests/test.proto rename to proto-yaml/src/tests/test.proto diff --git a/yara-x-proto-yaml/src/tests/testdata/1.in b/proto-yaml/src/tests/testdata/1.in similarity index 100% rename from yara-x-proto-yaml/src/tests/testdata/1.in rename to proto-yaml/src/tests/testdata/1.in diff --git a/yara-x-proto-yaml/src/tests/testdata/1.out b/proto-yaml/src/tests/testdata/1.out similarity index 100% rename from yara-x-proto-yaml/src/tests/testdata/1.out rename to proto-yaml/src/tests/testdata/1.out diff --git a/yara-x-proto-yaml/src/yaml.proto b/proto-yaml/src/yaml.proto similarity index 100% rename from yara-x-proto-yaml/src/yaml.proto rename to proto-yaml/src/yaml.proto diff --git a/yara-x-proto/Cargo.lock b/proto/Cargo.lock similarity index 100% rename from yara-x-proto/Cargo.lock rename to proto/Cargo.lock diff --git a/yara-x-proto/Cargo.toml b/proto/Cargo.toml similarity index 100% rename from yara-x-proto/Cargo.toml rename to proto/Cargo.toml diff --git a/yara-x-proto/build.rs b/proto/build.rs similarity index 100% rename from yara-x-proto/build.rs rename to proto/build.rs diff --git a/yara-x-proto/src/lib.rs b/proto/src/lib.rs similarity index 100% rename from yara-x-proto/src/lib.rs rename to proto/src/lib.rs diff --git a/yara-x-proto/src/yara.proto b/proto/src/yara.proto similarity index 100% rename from yara-x-proto/src/yara.proto rename to proto/src/yara.proto diff --git a/yara-x-py/Cargo.toml b/py/Cargo.toml similarity index 100% rename from yara-x-py/Cargo.toml rename to py/Cargo.toml diff --git a/yara-x-py/build.rs b/py/build.rs similarity index 100% rename from yara-x-py/build.rs rename to py/build.rs diff --git a/yara-x-py/pyproject.toml b/py/pyproject.toml similarity index 100% rename from yara-x-py/pyproject.toml rename to py/pyproject.toml diff --git a/yara-x-py/src/lib.rs b/py/src/lib.rs similarity index 99% rename from yara-x-py/src/lib.rs rename to py/src/lib.rs index 27db1b0a6..011b02d3e 100644 --- a/yara-x-py/src/lib.rs +++ b/py/src/lib.rs @@ -260,7 +260,7 @@ impl ScanResults { /// Represents a rule that matched while scanning some data. #[pyclass] struct Rule { - name: String, + identifier: String, namespace: String, patterns: Vec>, } @@ -269,8 +269,8 @@ struct Rule { impl Rule { #[getter] /// Returns the rule's name. - fn name(&self) -> &str { - self.name.as_str() + fn identifier(&self) -> &str { + self.identifier.as_str() } /// Returns the rule's namespace. @@ -426,7 +426,7 @@ fn rule_to_py(py: Python, rule: yrx::Rule) -> PyResult> { Py::new( py, Rule { - name: rule.name().to_string(), + identifier: rule.identifier().to_string(), namespace: rule.namespace().to_string(), patterns: rule .patterns() diff --git a/yara-x-py/tests/test_api.py b/py/tests/test_api.py similarity index 95% rename from yara-x-py/tests/test_api.py rename to py/tests/test_api.py index 1a86d152c..7de5466b8 100644 --- a/yara-x-py/tests/test_api.py +++ b/py/tests/test_api.py @@ -82,7 +82,7 @@ def test_namespaces(): assert len(matching_rules) == 2 - assert matching_rules[0].name == 'foo' + assert matching_rules[0].identifier == 'foo' assert matching_rules[0].namespace == 'foo' assert len(matching_rules[0].patterns) == 1 assert matching_rules[0].patterns[0].identifier == '$foo' @@ -91,7 +91,7 @@ def test_namespaces(): assert matching_rules[0].patterns[0].matches[0].length == 3 assert matching_rules[0].patterns[0].matches[0].xor_key is None - assert matching_rules[1].name == 'bar' + assert matching_rules[1].identifier == 'bar' assert matching_rules[1].namespace == 'bar' assert len(matching_rules[1].patterns) == 1 assert matching_rules[1].patterns[0].identifier == '$bar' @@ -104,7 +104,7 @@ def test_compile_and_scan(): rules = yara_x.compile('rule foo {strings: $a = "foo" condition: $a}') matching_rules = rules.scan(b'foobar').matching_rules assert len(matching_rules) == 1 - assert matching_rules[0].name == 'foo' + assert matching_rules[0].identifier == 'foo' assert matching_rules[0].namespace == 'default' assert len(matching_rules[0].patterns) == 1 assert matching_rules[0].patterns[0].identifier == '$a' @@ -114,7 +114,7 @@ def test_compiler_and_scanner(): rules = yara_x.compile('rule foo {strings: $a = "foo" condition: $a}') matching_rules = rules.scan(b'foobar').matching_rules assert len(matching_rules) == 1 - assert matching_rules[0].name == 'foo' + assert matching_rules[0].identifier == 'foo' assert matching_rules[0].namespace == 'default' assert len(matching_rules[0].patterns) == 1 assert matching_rules[0].patterns[0].identifier == '$a' @@ -124,7 +124,7 @@ def test_xor_key(): rules = yara_x.compile('rule foo {strings: $a = "foo" xor condition: $a}') matching_rules = rules.scan(b'\xCC\xC5\xC5').matching_rules assert len(matching_rules) == 1 - assert matching_rules[0].name == 'foo' + assert matching_rules[0].identifier == 'foo' assert matching_rules[0].namespace == 'default' assert len(matching_rules[0].patterns) == 1 assert matching_rules[0].patterns[0].identifier == '$a' From a93f275ffb4b574ef5dce65b1cf5a0ec2498dd3c Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Fri, 9 Feb 2024 13:51:39 +0100 Subject: [PATCH 21/55] feat: draft implementation of C and Go bindings for YARA-X --- Cargo.lock | 100 ++++++++++++- clib/Cargo.toml | 19 +++ clib/build.rs | 18 +++ clib/cbindgen.toml | 117 +++++++++++++++ clib/include/yara-x.h | 162 +++++++++++++++++++++ clib/src/lib.rs | 325 ++++++++++++++++++++++++++++++++++++++++++ go/go.mod | 10 ++ go/go.sum | 9 ++ go/main.go | 179 +++++++++++++++++++++++ go/scanner.go | 116 +++++++++++++++ go/scanner_test.go | 29 ++++ 11 files changed, 1080 insertions(+), 4 deletions(-) create mode 100644 clib/Cargo.toml create mode 100644 clib/build.rs create mode 100644 clib/cbindgen.toml create mode 100644 clib/include/yara-x.h create mode 100644 clib/src/lib.rs create mode 100644 go/go.mod create mode 100644 go/go.sum create mode 100644 go/main.go create mode 100644 go/scanner.go create mode 100644 go/scanner_test.go diff --git a/Cargo.lock b/Cargo.lock index 6b7c7fce4..e1530ad9d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -187,6 +187,17 @@ dependencies = [ "syn 2.0.48", ] +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi 0.1.19", + "libc", + "winapi", +] + [[package]] name = "authenticode-parser" version = "0.4.0" @@ -461,6 +472,25 @@ dependencies = [ "rustversion", ] +[[package]] +name = "cbindgen" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da6bc11b07529f16944307272d5bd9b22530bc7d05751717c9d416586cedab49" +dependencies = [ + "clap 3.2.25", + "heck 0.4.1", + "indexmap 1.9.3", + "log", + "proc-macro2", + "quote", + "serde", + "serde_json", + "syn 1.0.109", + "tempfile", + "toml", +] + [[package]] name = "cc" version = "1.0.83" @@ -548,6 +578,21 @@ dependencies = [ "libloading", ] +[[package]] +name = "clap" +version = "3.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123" +dependencies = [ + "atty", + "bitflags 1.3.2", + "clap_lex 0.2.4", + "indexmap 1.9.3", + "strsim", + "termcolor", + "textwrap", +] + [[package]] name = "clap" version = "4.4.18" @@ -566,7 +611,7 @@ checksum = "4df4df40ec50c46000231c914968278b1eb05098cf8f1b3a518a95030e71d1c7" dependencies = [ "anstream", "anstyle", - "clap_lex", + "clap_lex 0.6.0", "strsim", ] @@ -582,6 +627,15 @@ dependencies = [ "syn 2.0.48", ] +[[package]] +name = "clap_lex" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5" +dependencies = [ + "os_str_bytes", +] + [[package]] name = "clap_lex" version = "0.6.0" @@ -825,7 +879,7 @@ dependencies = [ "anes", "cast", "ciborium", - "clap", + "clap 4.4.18", "criterion-plot", "is-terminal", "itertools 0.10.5", @@ -1453,6 +1507,15 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + [[package]] name = "hermit-abi" version = "0.3.4" @@ -1637,7 +1700,7 @@ version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0bad00257d07be169d870ab665980b06cdb366d792ad690bf2e76876dc503455" dependencies = [ - "hermit-abi", + "hermit-abi 0.3.4", "rustix", "windows-sys 0.52.0", ] @@ -2186,6 +2249,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "os_str_bytes" +version = "6.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1" + [[package]] name = "parking_lot" version = "0.12.1" @@ -3197,6 +3266,15 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "termcolor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" +dependencies = [ + "winapi-util", +] + [[package]] name = "terminfo" version = "0.7.5" @@ -3261,6 +3339,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "textwrap" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" + [[package]] name = "thiserror" version = "1.0.56" @@ -4333,13 +4417,21 @@ dependencies = [ "zip", ] +[[package]] +name = "yara-x-c" +version = "0.1.0" +dependencies = [ + "cbindgen", + "yara-x", +] + [[package]] name = "yara-x-cli" version = "0.1.0" dependencies = [ "anyhow", "ascii_tree", - "clap", + "clap 4.4.18", "colored_json", "crossbeam", "crossterm 0.27.0", diff --git a/clib/Cargo.toml b/clib/Cargo.toml new file mode 100644 index 000000000..50e73b3ab --- /dev/null +++ b/clib/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "yara-x-c" +version.workspace = true +authors.workspace = true +edition.workspace = true +description.workspace = true +readme.workspace = true +license.workspace = true +homepage.workspace = true + +[lib] +name = "yara_x_c" +crate-type = ["staticlib", "cdylib"] + +[dependencies] +yara-x = { workspace = true } + +[build-dependencies] +cbindgen = "0.26.0" \ No newline at end of file diff --git a/clib/build.rs b/clib/build.rs new file mode 100644 index 000000000..a8d64d4e9 --- /dev/null +++ b/clib/build.rs @@ -0,0 +1,18 @@ +use std::env; + +fn main() { + println!("cargo:rerun-if-changed=src"); + println!("cargo:rerun-if-changed=cbindgen.toml"); + + let crate_dir = env::var("CARGO_MANIFEST_DIR").unwrap(); + let output_file = "include/yara-x.h".to_owned(); + + match cbindgen::generate(crate_dir) { + Ok(header) => { + header.write_to_file(output_file); + } + Err(err) => { + panic!("{}", err) + } + } +} diff --git a/clib/cbindgen.toml b/clib/cbindgen.toml new file mode 100644 index 000000000..21d5ab7f6 --- /dev/null +++ b/clib/cbindgen.toml @@ -0,0 +1,117 @@ + + +language = "C" +cpp_compat = false + + + +############## Options for Wrapping the Contents of the Header ################# + +# header = "/* Text to put at the beginning of the generated file. Probably a license. */" +# trailer = "/* Text to put at the end of the generated file */" +include_guard = "YARA_X" +pragma_once = true +autogen_warning = "// This file is autogenerated by cbindgen. Don't modify it manually." +include_version = true +# namespace = "my_namespace" +namespaces = [] +using_namespaces = [] +sys_includes = [] +includes = [] +no_includes = false +after_includes = "" + + +############################ Code Style Options ################################ + +braces = "SameLine" +line_length = 80 +tab_width = 2 +documentation = true +documentation_style = "c99" +documentation_length = "full" +line_endings = "LF" + +############################# Codegen Options ################################## + +style = "both" +sort_by = "None" # default for `fn.sort_by` and `const.sort_by` +usize_is_size_t = true + +[defines] +# "target_os = freebsd" = "DEFINE_FREEBSD" +# "feature = serde" = "DEFINE_SERDE" + +[export] +include = [] +exclude = [] +item_types = [] +renaming_overrides_prefixing = false + + +[export.rename] + + +[export.body] + + +[export.mangle] + + +[fn] +rename_args = "None" +# must_use = "MUST_USE_FUNC" +# no_return = "NO_RETURN" +# prefix = "START_FUNC" +# postfix = "END_FUNC" +args = "vertical" + +[struct] +rename_fields = "None" +# must_use = "MUST_USE_STRUCT" +derive_constructor = false +derive_eq = false +derive_neq = false +derive_lt = false +derive_lte = false +derive_gt = false +derive_gte = false + + +[enum] +rename_variants = "None" +# must_use = "MUST_USE_ENUM" +add_sentinel = false +prefix_with_name = false +derive_helper_methods = false +derive_const_casts = false +derive_mut_casts = false +# cast_assert_name = "ASSERT" +derive_tagged_enum_destructor = false +derive_tagged_enum_copy_constructor = false +enum_class = true +private_default_tagged_enum_constructor = false + + +[const] +allow_static_const = true +allow_constexpr = false + +[macro_expansion] +bitflags = false + + +############## Options for How Your Rust library Should Be Parsed ############## + +[parse] +parse_deps = false +# include = [] +exclude = [] +clean = false +extra_bindings = [] + +[parse.expand] +crates = [] +all_features = false +default_features = true +features = [] \ No newline at end of file diff --git a/clib/include/yara-x.h b/clib/include/yara-x.h new file mode 100644 index 000000000..51486f91e --- /dev/null +++ b/clib/include/yara-x.h @@ -0,0 +1,162 @@ +#ifndef YARA_X +#define YARA_X + +#pragma once + +/* Generated with cbindgen:0.26.0 */ + +// This file is autogenerated by cbindgen. Don't modify it manually. + +#include +#include +#include +#include +#include + + +typedef enum YRX_ERROR { + SUCCESS, + PANIC, + SYNTAX_ERROR, + INVALID_ARGUMENT, +} YRX_ERROR; + +// A pattern within a rule. +typedef struct YRX_PATTERN YRX_PATTERN; + +// The set of patterns declared in a YARA rule. +typedef struct YRX_PATTERNS YRX_PATTERNS; + +// A single YARA rule. +typedef struct YRX_RULE YRX_RULE; + +// A set of compiled YARA rules. +typedef struct YRX_RULES YRX_RULES; + +// A scanner that scans data with a set of compiled YARA rules. +typedef struct YRX_SCANNER YRX_SCANNER; + +// Callback function passed to the scanner via [`yrx_scanner_on_matching_rule`] +// which receives notifications about matching rules. +// +// The callback receives a pointer to the matching rule, represented by a +// [`YRX_RULE`] structure. This pointer is guaranteed to be valid while the +// callback function is being executed, but it may be freed after the callback +// function returns, so you cannot use the pointer outside the callback. +// +// It also receives the `user_data` pointer that was passed to the +// [`yrx_scanner_on_matching_rule`] function, which can point to arbitrary +// data owned by the user. +typedef void (*YRX_ON_MATCHING_RULE)(const struct YRX_RULE *rule, + void *user_data); + +// Contains information about a pattern match. +typedef struct YRX_MATCH { + size_t offset; + size_t length; +} YRX_MATCH; + +// Compiles YARA source code and creates a [`YRX_RULES`] object that contains +// the compiled rules. +// +// The rules must be destroyed with [`yrx_rules_destroy`]. +enum YRX_ERROR yrx_compile(const char *src, + struct YRX_RULES **rules); + +// Creates a [`YRX_SCANNER`] object that can be used for scanning data with +// the provided [`YRX_RULES`]. +// +// It's ok to pass the same [`YRX_RULES`] to multiple scanners, and use each +// scanner from a different thread. The scanner can be used as many times as +// you want, and it must be destroyed with [`yrx_scanner_destroy`]. Also, the +// scanner is valid as long as the rules are not destroyed, so, always destroy +// the [`YRX_SCANNER`] object before the [`YRX_RULES`] object. +enum YRX_ERROR yrx_scanner_create(const struct YRX_RULES *rules, + struct YRX_SCANNER **scanner); + +// Scans a data buffer. +enum YRX_ERROR yrx_scanner_scan(struct YRX_SCANNER *scanner, + const uint8_t *data, + size_t len); + +// Sets a callback function that is called by the scanner for each rule that +// matched during a scan. +// +// The `user_data` pointer can be used to provide additional context to your +// callback function. If the callback is not set, the scanner doesn't notify +// about matching rules. +// +// See [`YRX_ON_MATCHING_RULE`] for more details. +enum YRX_ERROR yrx_scanner_on_matching_rule(struct YRX_SCANNER *scanner, + YRX_ON_MATCHING_RULE callback, + void *user_data); + +// Destroys a [`YRX_SCANNER`] object. +void yrx_scanner_destroy(struct YRX_SCANNER *scanner); + +// Destroys a [`YRX_RULES`] object. +void yrx_rules_destroy(struct YRX_RULES *rules); + +// Returns the name of the rule represented by [`YRX_RULE`]. +// +// Arguments `ident` and `len` are output parameters that receive pointers +// to a `const uint8_t*` and `size_t`, where this function will leave a pointer +// to the rule's name and its length, respectively. The rule's name is *NOT* +// null-terminated, and the pointer will be valid as long as the [`YRX_RULES`] +// object that contains the rule is not freed. The name is guaranteed to be a +// valid UTF-8 string. +enum YRX_ERROR yrx_rule_identifier(const struct YRX_RULE *rule, + const uint8_t **ident, + size_t *len); + +// Returns the namespace of the rule represented by [`YRX_RULE`]. +// +// Arguments `ns` and `len` are output parameters that receive pointers to a +// `const uint8_t*` and `size_t`, where this function will leave a pointer +// to the rule's namespace and its length, respectively. The namespace is *NOT* +// null-terminated, and the pointer will be valid as long as the [`YRX_RULES`] +// object that contains the rule is not freed. The namespace is guaranteed to +// be a valid UTF-8 string. +enum YRX_ERROR yrx_rule_namespace(const struct YRX_RULE *rule, + const uint8_t **ns, + size_t *len); + +// Returns the all patterns defined by a rule, each pattern contains +// information about whether it matched or not, and where in the data it +// matched. +// +// The [`YRX_PATTERNS`] object must be destroyed with [`yrx_patterns_destroy`]. +const struct YRX_PATTERNS *yrx_rule_patterns(const struct YRX_RULE *rule); + +// Returns the number of patterns in a given [`YRX_PATTERNS`] object. +int32_t yrx_patterns_count(const struct YRX_PATTERNS *patterns); + +// Returns the pattern with the give `index`, from a set of patterns represented +// by a [`YRX_PATTERNS`] object. +// +// The index must be between 0 and the value returned by [`yrx_patterns_count`], +// otherwise the result will be a null pointer. The result is also a null +// pointer if `patterns` is null. +const struct YRX_PATTERN *yrx_patterns_get(struct YRX_PATTERNS *patterns, + size_t index); + +// Destroys a [`YRX_PATTERNS`] object. +void yrx_patterns_destroy(struct YRX_PATTERNS *patterns); + +// Returns the identifier of rule's pattern represented by [`YRX_PATTERN`]. +// +// Arguments `ident` and `len` are output parameters that receive pointers to a +// `const uint8_t*` and `size_t`, where this function will leave a pointer +// to the rule's name and its length, respectively. The identifier is *NOT* +// null-terminated, and the pointer will be valid as long as the [`YRX_RULES`] +// object that contains the rule is not freed. The name is guaranteed to be a +// valid UTF-8 string. +enum YRX_ERROR yrx_pattern_identifier(const struct YRX_PATTERN *pattern, + const uint8_t **ident, + size_t *len); + +enum YRX_ERROR yrx_pattern_matches(const struct YRX_PATTERN *pattern, + const struct YRX_MATCH **matches, + size_t *len); + +#endif /* YARA_X */ diff --git a/clib/src/lib.rs b/clib/src/lib.rs new file mode 100644 index 000000000..f5c262f51 --- /dev/null +++ b/clib/src/lib.rs @@ -0,0 +1,325 @@ +#![allow(non_camel_case_types)] +#![allow(clippy::missing_safety_doc)] +#![allow(clippy::not_unsafe_ptr_arg_deref)] + +use std::ffi::{c_char, CStr}; +use std::slice; + +#[repr(C)] +pub enum YRX_ERROR { + SUCCESS, + PANIC, + SYNTAX_ERROR, + INVALID_ARGUMENT, +} + +/// A set of compiled YARA rules. +pub struct YRX_RULES(yara_x::Rules); + +/// A single YARA rule. +pub struct YRX_RULE<'a, 'r>(yara_x::Rule<'a, 'r>); + +/// A pattern within a rule. +pub struct YRX_PATTERN { + identifier: String, + matches: Vec, +} + +/// Contains information about a pattern match. +#[repr(C)] +pub struct YRX_MATCH { + pub offset: usize, + pub length: usize, +} + +/// The set of patterns declared in a YARA rule. +pub struct YRX_PATTERNS(Vec); + +/// A scanner that scans data with a set of compiled YARA rules. +pub struct YRX_SCANNER<'s> { + inner: yara_x::Scanner<'s>, + on_matching_rule: Option<(YRX_ON_MATCHING_RULE, *mut std::ffi::c_void)>, +} + +/// Compiles YARA source code and creates a [`YRX_RULES`] object that contains +/// the compiled rules. +/// +/// The rules must be destroyed with [`yrx_rules_destroy`]. +#[no_mangle] +pub unsafe extern "C" fn yrx_compile( + src: *const c_char, + rules: &mut *mut YRX_RULES, +) -> YRX_ERROR { + let c_str = CStr::from_ptr(src); + + match yara_x::compile(c_str.to_bytes()) { + Ok(r) => *rules = Box::into_raw(Box::new(YRX_RULES(r))), + Err(err) => return YRX_ERROR::SYNTAX_ERROR, + }; + + YRX_ERROR::SUCCESS +} + +/// Creates a [`YRX_SCANNER`] object that can be used for scanning data with +/// the provided [`YRX_RULES`]. +/// +/// It's ok to pass the same [`YRX_RULES`] to multiple scanners, and use each +/// scanner from a different thread. The scanner can be used as many times as +/// you want, and it must be destroyed with [`yrx_scanner_destroy`]. Also, the +/// scanner is valid as long as the rules are not destroyed, so, always destroy +/// the [`YRX_SCANNER`] object before the [`YRX_RULES`] object. +#[no_mangle] +pub unsafe extern "C" fn yrx_scanner_create( + rules: *const YRX_RULES, + scanner: &mut *mut YRX_SCANNER, +) -> YRX_ERROR { + if let Some(rules) = rules.as_ref() { + let s = yara_x::Scanner::new(&rules.0); + *scanner = Box::into_raw(Box::new(YRX_SCANNER { + inner: s, + on_matching_rule: None, + })); + YRX_ERROR::SUCCESS + } else { + YRX_ERROR::INVALID_ARGUMENT + } +} + +/// Scans a data buffer. +#[no_mangle] +pub unsafe extern "C" fn yrx_scanner_scan( + scanner: *mut YRX_SCANNER, + data: *const u8, + len: usize, +) -> YRX_ERROR { + if scanner.is_null() { + return YRX_ERROR::INVALID_ARGUMENT; + } + + // Data is allowed to be null as long as len is 0. This case is handled + // as an empty slice. + if data.is_null() && len > 0 { + return YRX_ERROR::INVALID_ARGUMENT; + } + + let scanner = scanner.as_mut().unwrap(); + let data = slice::from_raw_parts(data, len); + let scan_results = scanner.inner.scan(data); + + if let Err(_) = scan_results { + return YRX_ERROR::PANIC; + } + + let scan_results = scan_results.unwrap(); + + if let Some((callback, user_data)) = scanner.on_matching_rule { + for r in scan_results.matching_rules() { + let rule = YRX_RULE(r); + callback(&rule as *const YRX_RULE, user_data); + } + } + + YRX_ERROR::SUCCESS +} + +/// Callback function passed to the scanner via [`yrx_scanner_on_matching_rule`] +/// which receives notifications about matching rules. +/// +/// The callback receives a pointer to the matching rule, represented by a +/// [`YRX_RULE`] structure. This pointer is guaranteed to be valid while the +/// callback function is being executed, but it may be freed after the callback +/// function returns, so you cannot use the pointer outside the callback. +/// +/// It also receives the `user_data` pointer that was passed to the +/// [`yrx_scanner_on_matching_rule`] function, which can point to arbitrary +/// data owned by the user. +type YRX_ON_MATCHING_RULE = extern "C" fn( + rule: *const YRX_RULE, + user_data: *mut std::ffi::c_void, +) -> (); + +/// Sets a callback function that is called by the scanner for each rule that +/// matched during a scan. +/// +/// The `user_data` pointer can be used to provide additional context to your +/// callback function. If the callback is not set, the scanner doesn't notify +/// about matching rules. +/// +/// See [`YRX_ON_MATCHING_RULE`] for more details. +#[no_mangle] +pub unsafe extern "C" fn yrx_scanner_on_matching_rule( + scanner: *mut YRX_SCANNER, + callback: YRX_ON_MATCHING_RULE, + user_data: *mut std::ffi::c_void, +) -> YRX_ERROR { + if let Some(scanner) = scanner.as_mut() { + scanner.on_matching_rule = Some((callback, user_data)); + YRX_ERROR::SUCCESS + } else { + YRX_ERROR::INVALID_ARGUMENT + } +} + +/// Destroys a [`YRX_SCANNER`] object. +#[no_mangle] +pub unsafe extern "C" fn yrx_scanner_destroy(scanner: *mut YRX_SCANNER) { + drop(Box::from_raw(scanner)) +} + +/// Destroys a [`YRX_RULES`] object. +#[no_mangle] +pub unsafe extern "C" fn yrx_rules_destroy(rules: *mut YRX_RULES) { + drop(Box::from_raw(rules)) +} + +/// Returns the name of the rule represented by [`YRX_RULE`]. +/// +/// Arguments `ident` and `len` are output parameters that receive pointers +/// to a `const uint8_t*` and `size_t`, where this function will leave a pointer +/// to the rule's name and its length, respectively. The rule's name is *NOT* +/// null-terminated, and the pointer will be valid as long as the [`YRX_RULES`] +/// object that contains the rule is not freed. The name is guaranteed to be a +/// valid UTF-8 string. +#[no_mangle] +pub unsafe extern "C" fn yrx_rule_identifier( + rule: *const YRX_RULE, + ident: &mut *const u8, + len: &mut usize, +) -> YRX_ERROR { + if let Some(rule) = rule.as_ref() { + *ident = rule.0.identifier().as_ptr(); + *len = rule.0.identifier().len(); + YRX_ERROR::SUCCESS + } else { + YRX_ERROR::INVALID_ARGUMENT + } +} + +/// Returns the namespace of the rule represented by [`YRX_RULE`]. +/// +/// Arguments `ns` and `len` are output parameters that receive pointers to a +/// `const uint8_t*` and `size_t`, where this function will leave a pointer +/// to the rule's namespace and its length, respectively. The namespace is *NOT* +/// null-terminated, and the pointer will be valid as long as the [`YRX_RULES`] +/// object that contains the rule is not freed. The namespace is guaranteed to +/// be a valid UTF-8 string. +#[no_mangle] +pub unsafe extern "C" fn yrx_rule_namespace( + rule: *const YRX_RULE, + ns: &mut *const u8, + len: &mut usize, +) -> YRX_ERROR { + if let Some(rule) = rule.as_ref() { + *ns = rule.0.namespace().as_ptr(); + *len = rule.0.namespace().len(); + YRX_ERROR::SUCCESS + } else { + YRX_ERROR::INVALID_ARGUMENT + } +} + +/// Returns the all patterns defined by a rule, each pattern contains +/// information about whether it matched or not, and where in the data it +/// matched. +/// +/// The [`YRX_PATTERNS`] object must be destroyed with [`yrx_patterns_destroy`]. +#[no_mangle] +pub unsafe extern "C" fn yrx_rule_patterns( + rule: *const YRX_RULE, +) -> *const YRX_PATTERNS { + if let Some(rule) = rule.as_ref() { + return Box::into_raw(Box::new(YRX_PATTERNS( + rule.0 + .patterns() + .map(|pat| YRX_PATTERN { + identifier: pat.identifier().to_string(), + matches: pat + .matches() + .map(|m| YRX_MATCH { + offset: m.range().start, + length: m.range().len(), + }) + .collect(), + }) + .collect(), + ))); + } else { + std::ptr::null() + } +} + +/// Returns the number of patterns in a given [`YRX_PATTERNS`] object. +#[no_mangle] +pub unsafe extern "C" fn yrx_patterns_count( + patterns: *const YRX_PATTERNS, +) -> i32 { + if let Some(patterns) = patterns.as_ref() { + patterns.0.len() as i32 + } else { + -1 + } +} + +/// Returns the pattern with the give `index`, from a set of patterns represented +/// by a [`YRX_PATTERNS`] object. +/// +/// The index must be between 0 and the value returned by [`yrx_patterns_count`], +/// otherwise the result will be a null pointer. The result is also a null +/// pointer if `patterns` is null. +#[no_mangle] +pub unsafe extern "C" fn yrx_patterns_get( + patterns: *mut YRX_PATTERNS, + index: usize, +) -> *const YRX_PATTERN { + if let Some(pattern) = + patterns.as_ref().and_then(|patterns| patterns.0.get(index)) + { + pattern + } else { + std::ptr::null() + } +} + +/// Destroys a [`YRX_PATTERNS`] object. +#[no_mangle] +pub unsafe extern "C" fn yrx_patterns_destroy(patterns: *mut YRX_PATTERNS) { + drop(Box::from_raw(patterns)) +} + +/// Returns the identifier of rule's pattern represented by [`YRX_PATTERN`]. +/// +/// Arguments `ident` and `len` are output parameters that receive pointers to a +/// `const uint8_t*` and `size_t`, where this function will leave a pointer +/// to the rule's name and its length, respectively. The identifier is *NOT* +/// null-terminated, and the pointer will be valid as long as the [`YRX_RULES`] +/// object that contains the rule is not freed. The name is guaranteed to be a +/// valid UTF-8 string. +#[no_mangle] +pub unsafe extern "C" fn yrx_pattern_identifier( + pattern: *const YRX_PATTERN, + ident: &mut *const u8, + len: &mut usize, +) -> YRX_ERROR { + if let Some(pattern) = pattern.as_ref() { + *ident = pattern.identifier.as_ptr(); + *len = pattern.identifier.len(); + YRX_ERROR::SUCCESS + } else { + YRX_ERROR::INVALID_ARGUMENT + } +} + +#[no_mangle] +pub unsafe extern "C" fn yrx_pattern_matches( + pattern: *const YRX_PATTERN, + matches: &mut *const YRX_MATCH, + len: &mut usize, +) -> YRX_ERROR { + if let Some(pattern) = pattern.as_ref() { + *matches = pattern.matches.as_ptr(); + *len = pattern.matches.len(); + YRX_ERROR::SUCCESS + } else { + YRX_ERROR::INVALID_ARGUMENT + } +} diff --git a/go/go.mod b/go/go.mod new file mode 100644 index 000000000..2d4da5b8f --- /dev/null +++ b/go/go.mod @@ -0,0 +1,10 @@ +module github.com/VirusTotal/yara-x/go + +go 1.18 + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/stretchr/testify v1.8.4 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/go/go.sum b/go/go.sum new file mode 100644 index 000000000..8cf66553b --- /dev/null +++ b/go/go.sum @@ -0,0 +1,9 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/go/main.go b/go/main.go new file mode 100644 index 000000000..edc6564b0 --- /dev/null +++ b/go/main.go @@ -0,0 +1,179 @@ +// Package yara_x provides Go bindings to the YARA-X library. +// +// Use the static_link build tag for linking the YARA-X Rust library statically. +// Notice however that this only works if the Rust library itself links all +// its dependencies statically. +package yara_x + +// #cgo CFLAGS: -I${SRCDIR}/../clib/include +// #cgo !static_link LDFLAGS: -L${SRCDIR}/../target/release -lyara_x_c +// #cgo static_link LDFLAGS: ${SRCDIR}/../target/release/libyara_x_c.a +// #import +import "C" +import ( + "runtime" + "unsafe" +) + +// Compile receives YARA source code and returns compiled Rules that can be +// used for scanning data. +func Compile(source string) *Rules { + s := C.CString(source) + r := &Rules{} + C.yrx_compile(s, &(r.cRules)) + runtime.SetFinalizer(r, (*Rules).Destroy) + return r +} + +// Rules represents a set of compiled YARA rules. +type Rules struct{ cRules *C.YRX_RULES } + +// Destroy destroys the compiled YARA rules represented by Rules. +// +// Calling this method directly is not necessary, it will be invoked by the +// garbage collector when the rules are not used anymore. +func (r *Rules) Destroy() { + if r.cRules != nil { + C.yrx_rules_destroy(r.cRules) + r.cRules = nil + } + runtime.SetFinalizer(r, nil) +} + +// Rule represents a YARA rule. +type Rule struct { + namespace string + identifier string + cPatterns *C.YRX_PATTERNS + patterns []Pattern +} + +// Creates a new Rule from it's C counterpart. +func newRule(cRule *C.YRX_RULE) *Rule { + var str *C.uint8_t + var len C.size_t + + if C.yrx_rule_namespace(cRule, &str, &len) != C.SUCCESS { + panic("yrx_rule_namespace failed") + } + + namespace := C.GoStringN((*C.char)(unsafe.Pointer(str)), C.int(len)) + + if C.yrx_rule_identifier(cRule, &str, &len) != C.SUCCESS { + panic("yrx_rule_name failed") + } + + identifier := C.GoStringN((*C.char)(unsafe.Pointer(str)), C.int(len)) + + cPatterns := C.yrx_rule_patterns(cRule) + + if cPatterns == nil { + panic("yrx_rule_patterns failed") + } + + rule := &Rule{namespace, identifier, cPatterns, nil} + runtime.SetFinalizer(rule, (*Rule).destroy) + return rule +} + +func (r *Rule) destroy() { + if r.cPatterns != nil { + C.yrx_patterns_destroy(r.cPatterns) + r.cPatterns = nil + } + runtime.SetFinalizer(r, nil) +} + +// Identifier returns the rule's identifier. +func (r *Rule) Identifier() string { + return r.identifier +} + +// Namespace returns the rule's namespace. +func (r *Rule) Namespace() string { + return r.namespace +} + +// Patterns returns the patterns defined by this rule. +func (r *Rule) Patterns() []Pattern { + if r.patterns != nil { + return r.patterns + } + + numPatterns := int(C.yrx_patterns_count(r.cPatterns)) + r.patterns = make([]Pattern, numPatterns) + + for i := 0; i < numPatterns; i++ { + pattern := C.yrx_patterns_get(r.cPatterns, C.ulong(i)) + if pattern == nil { + panic("yrx_patterns_get failed") + } + + var str *C.uint8_t + var len C.size_t + + if C.yrx_pattern_identifier(pattern, &str, &len) != C.SUCCESS { + panic("yrx_pattern_identifier failed") + } + + identifier := C.GoStringN((*C.char)(unsafe.Pointer(str)), C.int(len)) + + var ptrMatches *C.YRX_MATCH + var numMatches C.size_t + + if C.yrx_pattern_matches(pattern, &ptrMatches, &numMatches) != C.SUCCESS { + panic("yrx_pattern_matches failed") + } + + matches := make([]Match, int(numMatches)) + + for i := 0; i < int(numMatches); i++ { + match := *(*C.YRX_MATCH)(unsafe.Add( + unsafe.Pointer(ptrMatches), + i*C.sizeof_YRX_MATCH)) + matches[i] = Match{ + offset: uint(match.offset), + length: uint(match.length), + } + } + + r.patterns[i] = Pattern{ + identifier: identifier, + matches: matches, + } + } + return r.patterns +} + +// Pattern represents a pattern in a Rule. +type Pattern struct { + identifier string + matches []Match +} + +// Identifier returns the pattern's identifier (i.e: `$a`, `$foo`). +func (p *Pattern) Identifier() string { + return p.identifier +} + +// Matches returns the matches found for this pattern. +func (p *Pattern) Matches() []Match { + return p.matches +} + +// Match contains information about the offset where a match occurred and +// the length of the match. +type Match struct { + offset uint + length uint +} + +// Offset returns the offset within the scanned data where a match occurred. +func (m *Match) Offset() uint { + return m.offset +} + +// Length returns the length of a match in bytes. +func (m *Match) Length() uint { + return m.length +} diff --git a/go/scanner.go b/go/scanner.go new file mode 100644 index 000000000..f5247dd0d --- /dev/null +++ b/go/scanner.go @@ -0,0 +1,116 @@ +package yara_x + +import "C" +import ( + "runtime" + "runtime/cgo" + "unsafe" +) + +// #import +// void onMatchingRule(YRX_RULE*, void*); +// char fault(size_t p); +import "C" + +// Scanner scans data with a set of compiled YARA rules. +type Scanner struct { + // Pointer to C-side scanner. + inner *C.YRX_SCANNER + // The Scanner holds a pointer to the Rules it uses in order to prevent + // Rules from being garbage collected while the scanner is in use. If Rules + // is garbage collected the associated C.YRX_RULES object is destroyed + // before the C.YRX_SCANNER object. + rules *Rules + // Handle to the scanner itself that is passed to C callbacks. Notice that + // this is not actually the handle but a pointer to the handle, and the + // memory that stores the handle is allocated via C.malloc because the + // pointer is passed to C code. We don't want the garbage collector messing + // with the memory that holds the handle. + // + // Go 1.21 introduces https://pkg.go.dev/runtime#Pinner.Pin, which allows + // to pin a Go object in memory, guaranteeing that the garbage collector + // won't move it to another memory location. We could pin the Scanner + // struct and pass a pointer to the scanner to the C code, making this + // handle unnecessary. At this time (Feb 2024) Go 1.21 is only 6 months + // old, and we want to support older versions. + handle *cgo.Handle + // Rules that matched during the last scan. + matchingRules []*Rule +} + +type ScanResults struct{} + +// NewScanner creates a Scanner that will use the provided YARA rules. +// +// It's safe to pass the same Rules to multiple scanners, and use each scanner +// on a separate goroutine for performing multiple scans in parallel with the +// same set of rules. +func NewScanner(r *Rules) *Scanner { + s := &Scanner{rules: r} + // TODO: handle error returned by yrx_scanner_create + C.yrx_scanner_create(r.cRules, &s.inner) + + // Allocate the memory that will hold the handle. This memory is allocated + // using C.malloc because a pointer to it is passed to C code, and we don't + // want the garbage collector messing with it. + s.handle = (*cgo.Handle)(C.malloc(C.size_t(unsafe.Sizeof(s.handle)))) + + // Create a new handle that points to the scanner, and store it in the + // newly allocated memory. + *s.handle = cgo.NewHandle(s) + + C.yrx_scanner_on_matching_rule( + s.inner, + C.YRX_ON_MATCHING_RULE(C.onMatchingRule), + unsafe.Pointer(s.handle)) + + runtime.SetFinalizer(s, (*Scanner).Destroy) + return s +} + +// Scan scans the provided data with the Rules associated to the Scanner. +func (s *Scanner) Scan(buf []byte) []*Rule { + var ptr *C.uint8_t + // When `buf` is an empty slice `ptr` will be nil. That's ok, because + // yrx_scanner_scan allows the data pointer to be null as long as the data + // size is 0. + if len(buf) > 0 { + ptr = (*C.uint8_t)(unsafe.Pointer(&(buf[0]))) + } + s.matchingRules = nil + // TODO: handle errors + C.yrx_scanner_scan(s.inner, ptr, C.size_t(len(buf))) + // Ensure that s is not finalized until yrx_scanner_scan returns. + runtime.KeepAlive(s) + return s.matchingRules +} + +// Destroy destroys the scanner. +// +// Calling this method directly is not necessary, it will be invoked by the +// garbage collector when the scanner is not used anymore. +func (s *Scanner) Destroy() { + if s.inner != nil { + C.yrx_scanner_destroy(s.inner) + s.handle.Delete() + C.free(unsafe.Pointer(s.handle)) + s.inner = nil + } + runtime.SetFinalizer(s, nil) +} + +// This is the callback function called every time a YARA rule matches. +// +//export onMatchingRule +func onMatchingRule(rule *C.YRX_RULE, handlePtr unsafe.Pointer) { + handle := *(*cgo.Handle)(handlePtr) + scanner, ok := handle.Value().(*Scanner) + if !ok { + panic("onMatchingRule didn't receive a Scanner") + } + scanner.matchingRules = append(scanner.matchingRules, newRule(rule)) +} + + + + diff --git a/go/scanner_test.go b/go/scanner_test.go new file mode 100644 index 000000000..4da75efcc --- /dev/null +++ b/go/scanner_test.go @@ -0,0 +1,29 @@ +package yara_x + +import "testing" +import "github.com/stretchr/testify/assert" + + +func TestScanner1(t *testing.T) { + s := NewScanner(Compile("rule t { condition: true }")) + matchingRules := s.Scan([]byte{}) + + assert.Len(t, matchingRules, 1) + assert.Equal(t, "t", matchingRules[0].Name()) + assert.Equal(t, "default", matchingRules[0].Namespace()) + assert.Len(t, matchingRules[0].Patterns(), 0) +} + +func TestScanner2(t *testing.T) { + s := NewScanner(Compile(`rule t { strings: $bar = "bar" condition: $bar }`)) + matchingRules := s.Scan([]byte("foobar")) + + assert.Len(t, matchingRules, 1) + assert.Equal(t, "t", matchingRules[0].Name()) + assert.Equal(t, "default", matchingRules[0].Namespace()) + + assert.Len(t, matchingRules[0].Patterns(), 1) + assert.Equal(t, "$bar", matchingRules[0].Patterns()[0].Identifier()) + assert.Equal(t, uint(3), matchingRules[0].Patterns()[0].Matches()[0].Offset()) + assert.Equal(t, uint(3), matchingRules[0].Patterns()[0].Matches()[0].Length()) +} \ No newline at end of file From d5f84ad956af7304e3b5cf999de71da3ccf5408a Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Fri, 9 Feb 2024 14:03:00 +0100 Subject: [PATCH 22/55] chore: individual descriptions for each crate --- Cargo.toml | 1 - cli/Cargo.toml | 4 +++- clib/Cargo.toml | 4 +++- fmt/Cargo.toml | 4 +++- lib/Cargo.toml | 4 +++- parser/Cargo.toml | 4 +++- proto-yaml/Cargo.toml | 4 +++- proto/Cargo.toml | 4 +++- py/Cargo.toml | 3 +++ 9 files changed, 24 insertions(+), 8 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 80e030c44..5874db0a3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,5 @@ [workspace.package] version = "0.1.0" -description = "An experimental implementation of YARA in Rust" authors = ["Victor M. Alvarez "] edition = "2021" homepage = "https://github.com/VirusTotal/yara-x" diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 947b69e75..8781ec9dc 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -1,9 +1,11 @@ [package] name = "yara-x-cli" +description = """ +A command-line interface for YARA-X. +""" version.workspace = true authors.workspace = true edition.workspace = true -description.workspace = true readme.workspace = true license.workspace = true homepage.workspace = true diff --git a/clib/Cargo.toml b/clib/Cargo.toml index 50e73b3ab..1f481410f 100644 --- a/clib/Cargo.toml +++ b/clib/Cargo.toml @@ -1,9 +1,11 @@ [package] name = "yara-x-c" +description = """ +A C API for the YARA-X library. +""" version.workspace = true authors.workspace = true edition.workspace = true -description.workspace = true readme.workspace = true license.workspace = true homepage.workspace = true diff --git a/fmt/Cargo.toml b/fmt/Cargo.toml index 1cd8682a2..95d3453c1 100644 --- a/fmt/Cargo.toml +++ b/fmt/Cargo.toml @@ -1,9 +1,11 @@ [package] name = "yara-x-fmt" +description = """ +A code-formatting library for YARA rules. +""" version.workspace = true authors.workspace = true edition.workspace = true -description.workspace = true readme.workspace = true license.workspace = true homepage.workspace = true diff --git a/lib/Cargo.toml b/lib/Cargo.toml index a57f7171e..663ec85e3 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -1,9 +1,11 @@ [package] name = "yara-x" +description = """ +A pure Rust implmentation of YARA-X. +""" version.workspace = true authors.workspace = true edition.workspace = true -description.workspace = true readme.workspace = true license.workspace = true homepage.workspace = true diff --git a/parser/Cargo.toml b/parser/Cargo.toml index ff9f2244a..199761e68 100644 --- a/parser/Cargo.toml +++ b/parser/Cargo.toml @@ -1,9 +1,11 @@ [package] name = "yara-x-parser" +description = """ +A parsing library for YARA rules. +""" version.workspace = true authors.workspace = true edition.workspace = true -description.workspace = true readme.workspace = true license.workspace = true homepage.workspace = true diff --git a/proto-yaml/Cargo.toml b/proto-yaml/Cargo.toml index 3bfb51616..b5e0dd3d2 100644 --- a/proto-yaml/Cargo.toml +++ b/proto-yaml/Cargo.toml @@ -1,7 +1,9 @@ [package] name = "yara-x-proto-yaml" +description = """ +Library the converts protocol buffers into YAML. +""" version.workspace = true -description.workspace = true authors.workspace = true edition.workspace = true homepage.workspace = true diff --git a/proto/Cargo.toml b/proto/Cargo.toml index f55c05ef1..88857bde0 100644 --- a/proto/Cargo.toml +++ b/proto/Cargo.toml @@ -1,6 +1,8 @@ [package] name = "yara-x-proto" -description = "Protocol buffer with options for YARA modules" +description = """ +Protocol buffer with options for YARA modules. +""" version.workspace = true authors.workspace = true edition.workspace = true diff --git a/py/Cargo.toml b/py/Cargo.toml index e7bf70061..5228cb845 100644 --- a/py/Cargo.toml +++ b/py/Cargo.toml @@ -1,5 +1,8 @@ [package] name = "yara-x-py" +description = """ +Python bindings for YARA-X library. +""" version.workspace = true edition.workspace = true From 000d72824e22104fe4d9b6423c27f8cb06fdfa46 Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Fri, 9 Feb 2024 14:09:53 +0100 Subject: [PATCH 23/55] chore: rename `clib` directory to `capi` --- Cargo.toml | 2 +- {clib => capi}/Cargo.toml | 0 {clib => capi}/build.rs | 0 {clib => capi}/cbindgen.toml | 0 {clib => capi}/include/yara-x.h | 0 {clib => capi}/src/lib.rs | 0 6 files changed, 1 insertion(+), 1 deletion(-) rename {clib => capi}/Cargo.toml (100%) rename {clib => capi}/build.rs (100%) rename {clib => capi}/cbindgen.toml (100%) rename {clib => capi}/include/yara-x.h (100%) rename {clib => capi}/src/lib.rs (100%) diff --git a/Cargo.toml b/Cargo.toml index 5874db0a3..1ca76aa3c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,7 +15,7 @@ rust-version = "1.73.0" [workspace] members = [ "lib", - "clib", + "capi", "cli", "fmt", "macros", diff --git a/clib/Cargo.toml b/capi/Cargo.toml similarity index 100% rename from clib/Cargo.toml rename to capi/Cargo.toml diff --git a/clib/build.rs b/capi/build.rs similarity index 100% rename from clib/build.rs rename to capi/build.rs diff --git a/clib/cbindgen.toml b/capi/cbindgen.toml similarity index 100% rename from clib/cbindgen.toml rename to capi/cbindgen.toml diff --git a/clib/include/yara-x.h b/capi/include/yara-x.h similarity index 100% rename from clib/include/yara-x.h rename to capi/include/yara-x.h diff --git a/clib/src/lib.rs b/capi/src/lib.rs similarity index 100% rename from clib/src/lib.rs rename to capi/src/lib.rs From 82ede926a3081e8be28b4d72ffe58bf790bfcaf4 Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Fri, 9 Feb 2024 14:11:09 +0100 Subject: [PATCH 24/55] style: fix clippy warning --- capi/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/capi/src/lib.rs b/capi/src/lib.rs index f5c262f51..dde190cd6 100644 --- a/capi/src/lib.rs +++ b/capi/src/lib.rs @@ -54,7 +54,7 @@ pub unsafe extern "C" fn yrx_compile( match yara_x::compile(c_str.to_bytes()) { Ok(r) => *rules = Box::into_raw(Box::new(YRX_RULES(r))), - Err(err) => return YRX_ERROR::SYNTAX_ERROR, + Err(_) => return YRX_ERROR::SYNTAX_ERROR, }; YRX_ERROR::SUCCESS From 409e8013dfb4e9b27d0dda3e246559eeed29fb96 Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Fri, 9 Feb 2024 14:12:58 +0100 Subject: [PATCH 25/55] style: fix clippy warning --- capi/src/lib.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/capi/src/lib.rs b/capi/src/lib.rs index dde190cd6..ef16fba19 100644 --- a/capi/src/lib.rs +++ b/capi/src/lib.rs @@ -106,7 +106,8 @@ pub unsafe extern "C" fn yrx_scanner_scan( let data = slice::from_raw_parts(data, len); let scan_results = scanner.inner.scan(data); - if let Err(_) = scan_results { + if scan_results.is_err() { + // TODO: return appropriate error return YRX_ERROR::PANIC; } From 78ae51afcbfae05416168c7e036a9efc6e16caa8 Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Fri, 9 Feb 2024 15:18:16 +0100 Subject: [PATCH 26/55] fix: some fixes in Go package --- go/main.go | 2 +- go/scanner.go | 1 - go/scanner_test.go | 4 ++-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/go/main.go b/go/main.go index edc6564b0..1491e044d 100644 --- a/go/main.go +++ b/go/main.go @@ -5,7 +5,7 @@ // its dependencies statically. package yara_x -// #cgo CFLAGS: -I${SRCDIR}/../clib/include +// #cgo CFLAGS: -I${SRCDIR}/../capi/include // #cgo !static_link LDFLAGS: -L${SRCDIR}/../target/release -lyara_x_c // #cgo static_link LDFLAGS: ${SRCDIR}/../target/release/libyara_x_c.a // #import diff --git a/go/scanner.go b/go/scanner.go index f5247dd0d..c564276f8 100644 --- a/go/scanner.go +++ b/go/scanner.go @@ -9,7 +9,6 @@ import ( // #import // void onMatchingRule(YRX_RULE*, void*); -// char fault(size_t p); import "C" // Scanner scans data with a set of compiled YARA rules. diff --git a/go/scanner_test.go b/go/scanner_test.go index 4da75efcc..e2db8e8b9 100644 --- a/go/scanner_test.go +++ b/go/scanner_test.go @@ -9,7 +9,7 @@ func TestScanner1(t *testing.T) { matchingRules := s.Scan([]byte{}) assert.Len(t, matchingRules, 1) - assert.Equal(t, "t", matchingRules[0].Name()) + assert.Equal(t, "t", matchingRules[0].Identifier()) assert.Equal(t, "default", matchingRules[0].Namespace()) assert.Len(t, matchingRules[0].Patterns(), 0) } @@ -19,7 +19,7 @@ func TestScanner2(t *testing.T) { matchingRules := s.Scan([]byte("foobar")) assert.Len(t, matchingRules, 1) - assert.Equal(t, "t", matchingRules[0].Name()) + assert.Equal(t, "t", matchingRules[0].Identifier()) assert.Equal(t, "default", matchingRules[0].Namespace()) assert.Len(t, matchingRules[0].Patterns(), 1) From cc080fcd7151f304ce80c65c291af56365e6ec33 Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Fri, 9 Feb 2024 18:13:30 +0100 Subject: [PATCH 27/55] ci: add GitHub workflow for testing the Go package (#81) At this time the Go package is tested only in Linux and MacOS. Building the Go package in Windows is a bit more difficult because it requires MSYS2. --- .github/workflows/golang.yaml | 67 +++++++++++++++++++++++++++++++++++ go/main.go | 4 +-- go/scanner.go | 2 +- 3 files changed, 70 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/golang.yaml diff --git a/.github/workflows/golang.yaml b/.github/workflows/golang.yaml new file mode 100644 index 000000000..9a7cf6917 --- /dev/null +++ b/.github/workflows/golang.yaml @@ -0,0 +1,67 @@ +name: Golang package + +on: + push: + paths: + - 'go/**' + - '.github/workflows/golang.yaml' + +permissions: + contents: read + +jobs: + test: + strategy: + fail-fast: false + matrix: + go-version: [ '1.19', '1.20', '1.21.x' ] + os: [ubuntu-latest, macos-latest] + runs-on: ${{ matrix.os }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Go ${{ matrix.go-version }} + uses: actions/setup-go@v4 + with: + go-version: ${{ matrix.go-version }} + + - name: Setup Rust + uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + + # Install OpenSSL on Windows + - name: Install OpenSSL + if: runner.os == 'Windows' + id: vcpkg + uses: johnwason/vcpkg-action@v5 + with: + pkgs: openssl + triplet: x64-windows-static + token: ${{ github.token }} + + - name: Set OPENSSL_DIR environment variable + if: runner.os == 'Windows' + shell: bash + run: echo "OPENSSL_DIR=${{ github.workspace }}\\vcpkg\\installed\\x64-windows-static" >> $GITHUB_ENV + + # Install OpenSSL on Linux + - name: Install dependencies + if: runner.os == 'Linux' + run: | + sudo apt-get update + sudo apt-get install -y libssl-dev + + - name: Build Rust library + run: cargo build --release -p yara-x -p yara-x-c + + - name: Set LD_LIBRARY_PATH environment variable + if: runner.os == 'Linux' + shell: bash + run: echo "LD_LIBRARY_PATH=${{ github.workspace }}/target/release" >> $GITHUB_ENV + + - name: Run Go tests + run: | + cd go + go test \ No newline at end of file diff --git a/go/main.go b/go/main.go index 1491e044d..aa64b1393 100644 --- a/go/main.go +++ b/go/main.go @@ -8,7 +8,7 @@ package yara_x // #cgo CFLAGS: -I${SRCDIR}/../capi/include // #cgo !static_link LDFLAGS: -L${SRCDIR}/../target/release -lyara_x_c // #cgo static_link LDFLAGS: ${SRCDIR}/../target/release/libyara_x_c.a -// #import +// #include import "C" import ( "runtime" @@ -104,7 +104,7 @@ func (r *Rule) Patterns() []Pattern { r.patterns = make([]Pattern, numPatterns) for i := 0; i < numPatterns; i++ { - pattern := C.yrx_patterns_get(r.cPatterns, C.ulong(i)) + pattern := C.yrx_patterns_get(r.cPatterns, C.size_t(i)) if pattern == nil { panic("yrx_patterns_get failed") } diff --git a/go/scanner.go b/go/scanner.go index c564276f8..25e28f7ed 100644 --- a/go/scanner.go +++ b/go/scanner.go @@ -7,7 +7,7 @@ import ( "unsafe" ) -// #import +// #include // void onMatchingRule(YRX_RULE*, void*); import "C" From f5f8959acb3d0c052754eb46b93b5f66df2d1fa6 Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Fri, 9 Feb 2024 18:23:11 +0100 Subject: [PATCH 28/55] chore: rename `libyara_x_c` to `libyara_x_capi` --- Cargo.lock | 2 +- capi/Cargo.toml | 4 ++-- go/main.go | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e1530ad9d..08909a9ac 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4418,7 +4418,7 @@ dependencies = [ ] [[package]] -name = "yara-x-c" +name = "yara-x-capi" version = "0.1.0" dependencies = [ "cbindgen", diff --git a/capi/Cargo.toml b/capi/Cargo.toml index 1f481410f..e5d2b6c72 100644 --- a/capi/Cargo.toml +++ b/capi/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "yara-x-c" +name = "yara-x-capi" description = """ A C API for the YARA-X library. """ @@ -11,7 +11,7 @@ license.workspace = true homepage.workspace = true [lib] -name = "yara_x_c" +name = "yara_x_capi" crate-type = ["staticlib", "cdylib"] [dependencies] diff --git a/go/main.go b/go/main.go index aa64b1393..5b3fe74e1 100644 --- a/go/main.go +++ b/go/main.go @@ -6,9 +6,9 @@ package yara_x // #cgo CFLAGS: -I${SRCDIR}/../capi/include -// #cgo !static_link LDFLAGS: -L${SRCDIR}/../target/release -lyara_x_c -// #cgo static_link LDFLAGS: ${SRCDIR}/../target/release/libyara_x_c.a -// #include +// #cgo !static_link LDFLAGS: -L${SRCDIR}/../target/release -lyara_x_capi +// #cgo static_link LDFLAGS: ${SRCDIR}/../target/release/libyara_x_capi.a +// #import import "C" import ( "runtime" From bc1d22a8447def0564ec5368698b46eaf1c3c8df Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Fri, 9 Feb 2024 18:23:45 +0100 Subject: [PATCH 29/55] ci: fix path in GitHub work flow --- .github/workflows/python.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python.yaml b/.github/workflows/python.yaml index 9f022fec6..aca84dc11 100644 --- a/.github/workflows/python.yaml +++ b/.github/workflows/python.yaml @@ -3,7 +3,7 @@ name: Python extension on: push: paths: - - 'yara-x-py/**' + - 'py/**' - '.github/workflows/python.yaml' permissions: From b52751ee48e3ab6d5071e30031b099245a7e8234 Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Fri, 9 Feb 2024 18:37:46 +0100 Subject: [PATCH 30/55] ci: fix broken workflow due to package name change --- .github/workflows/golang.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/golang.yaml b/.github/workflows/golang.yaml index 9a7cf6917..8012e99e3 100644 --- a/.github/workflows/golang.yaml +++ b/.github/workflows/golang.yaml @@ -54,7 +54,7 @@ jobs: sudo apt-get install -y libssl-dev - name: Build Rust library - run: cargo build --release -p yara-x -p yara-x-c + run: cargo build --release -p yara-x -p yara-x-capi - name: Set LD_LIBRARY_PATH environment variable if: runner.os == 'Linux' From 7501103ef887444185c30434af09ec2ed0f0fead Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Fri, 9 Feb 2024 19:08:35 +0100 Subject: [PATCH 31/55] refactor(capi): move scanner-related types and functions to their own file. --- capi/include/yara-x.h | 74 ++++++++++++------------ capi/src/lib.rs | 131 ++++++------------------------------------ capi/src/scanner.rs | 116 +++++++++++++++++++++++++++++++++++++ 3 files changed, 170 insertions(+), 151 deletions(-) create mode 100644 capi/src/scanner.rs diff --git a/capi/include/yara-x.h b/capi/include/yara-x.h index 51486f91e..f783a39a5 100644 --- a/capi/include/yara-x.h +++ b/capi/include/yara-x.h @@ -36,6 +36,12 @@ typedef struct YRX_RULES YRX_RULES; // A scanner that scans data with a set of compiled YARA rules. typedef struct YRX_SCANNER YRX_SCANNER; +// Contains information about a pattern match. +typedef struct YRX_MATCH { + size_t offset; + size_t length; +} YRX_MATCH; + // Callback function passed to the scanner via [`yrx_scanner_on_matching_rule`] // which receives notifications about matching rules. // @@ -50,12 +56,6 @@ typedef struct YRX_SCANNER YRX_SCANNER; typedef void (*YRX_ON_MATCHING_RULE)(const struct YRX_RULE *rule, void *user_data); -// Contains information about a pattern match. -typedef struct YRX_MATCH { - size_t offset; - size_t length; -} YRX_MATCH; - // Compiles YARA source code and creates a [`YRX_RULES`] object that contains // the compiled rules. // @@ -63,37 +63,6 @@ typedef struct YRX_MATCH { enum YRX_ERROR yrx_compile(const char *src, struct YRX_RULES **rules); -// Creates a [`YRX_SCANNER`] object that can be used for scanning data with -// the provided [`YRX_RULES`]. -// -// It's ok to pass the same [`YRX_RULES`] to multiple scanners, and use each -// scanner from a different thread. The scanner can be used as many times as -// you want, and it must be destroyed with [`yrx_scanner_destroy`]. Also, the -// scanner is valid as long as the rules are not destroyed, so, always destroy -// the [`YRX_SCANNER`] object before the [`YRX_RULES`] object. -enum YRX_ERROR yrx_scanner_create(const struct YRX_RULES *rules, - struct YRX_SCANNER **scanner); - -// Scans a data buffer. -enum YRX_ERROR yrx_scanner_scan(struct YRX_SCANNER *scanner, - const uint8_t *data, - size_t len); - -// Sets a callback function that is called by the scanner for each rule that -// matched during a scan. -// -// The `user_data` pointer can be used to provide additional context to your -// callback function. If the callback is not set, the scanner doesn't notify -// about matching rules. -// -// See [`YRX_ON_MATCHING_RULE`] for more details. -enum YRX_ERROR yrx_scanner_on_matching_rule(struct YRX_SCANNER *scanner, - YRX_ON_MATCHING_RULE callback, - void *user_data); - -// Destroys a [`YRX_SCANNER`] object. -void yrx_scanner_destroy(struct YRX_SCANNER *scanner); - // Destroys a [`YRX_RULES`] object. void yrx_rules_destroy(struct YRX_RULES *rules); @@ -159,4 +128,35 @@ enum YRX_ERROR yrx_pattern_matches(const struct YRX_PATTERN *pattern, const struct YRX_MATCH **matches, size_t *len); +// Creates a [`YRX_SCANNER`] object that can be used for scanning data with +// the provided [`YRX_RULES`]. +// +// It's ok to pass the same [`YRX_RULES`] to multiple scanners, and use each +// scanner from a different thread. The scanner can be used as many times as +// you want, and it must be destroyed with [`yrx_scanner_destroy`]. Also, the +// scanner is valid as long as the rules are not destroyed, so, always destroy +// the [`YRX_SCANNER`] object before the [`YRX_RULES`] object. +enum YRX_ERROR yrx_scanner_create(const struct YRX_RULES *rules, + struct YRX_SCANNER **scanner); + +// Scans a data buffer. +enum YRX_ERROR yrx_scanner_scan(struct YRX_SCANNER *scanner, + const uint8_t *data, + size_t len); + +// Sets a callback function that is called by the scanner for each rule that +// matched during a scan. +// +// The `user_data` pointer can be used to provide additional context to your +// callback function. If the callback is not set, the scanner doesn't notify +// about matching rules. +// +// See [`YRX_ON_MATCHING_RULE`] for more details. +enum YRX_ERROR yrx_scanner_on_matching_rule(struct YRX_SCANNER *scanner, + YRX_ON_MATCHING_RULE callback, + void *user_data); + +// Destroys a [`YRX_SCANNER`] object. +void yrx_scanner_destroy(struct YRX_SCANNER *scanner); + #endif /* YARA_X */ diff --git a/capi/src/lib.rs b/capi/src/lib.rs index ef16fba19..9ea69b441 100644 --- a/capi/src/lib.rs +++ b/capi/src/lib.rs @@ -1,9 +1,25 @@ +/*! C bindings for the YARA-X library. + +This crate defines the C-compatible API that C/C++ programs can use for +interfacing with the YARA-X Rust library. When this crate is built, the header +file `capi/include/yara-x.h` is generated automatically using [`cbindgen`][1], +together with dynamic-linking and static-linking versions of a `libyara-x-capi` +that can be found in the `target` directory. + +This crate is not intended to be used by other Rust programs. + +[1]: https://github.com/mozilla/cbindgen +*/ + #![allow(non_camel_case_types)] #![allow(clippy::missing_safety_doc)] #![allow(clippy::not_unsafe_ptr_arg_deref)] use std::ffi::{c_char, CStr}; -use std::slice; + +mod scanner; + +pub use scanner::*; #[repr(C)] pub enum YRX_ERROR { @@ -35,12 +51,6 @@ pub struct YRX_MATCH { /// The set of patterns declared in a YARA rule. pub struct YRX_PATTERNS(Vec); -/// A scanner that scans data with a set of compiled YARA rules. -pub struct YRX_SCANNER<'s> { - inner: yara_x::Scanner<'s>, - on_matching_rule: Option<(YRX_ON_MATCHING_RULE, *mut std::ffi::c_void)>, -} - /// Compiles YARA source code and creates a [`YRX_RULES`] object that contains /// the compiled rules. /// @@ -60,113 +70,6 @@ pub unsafe extern "C" fn yrx_compile( YRX_ERROR::SUCCESS } -/// Creates a [`YRX_SCANNER`] object that can be used for scanning data with -/// the provided [`YRX_RULES`]. -/// -/// It's ok to pass the same [`YRX_RULES`] to multiple scanners, and use each -/// scanner from a different thread. The scanner can be used as many times as -/// you want, and it must be destroyed with [`yrx_scanner_destroy`]. Also, the -/// scanner is valid as long as the rules are not destroyed, so, always destroy -/// the [`YRX_SCANNER`] object before the [`YRX_RULES`] object. -#[no_mangle] -pub unsafe extern "C" fn yrx_scanner_create( - rules: *const YRX_RULES, - scanner: &mut *mut YRX_SCANNER, -) -> YRX_ERROR { - if let Some(rules) = rules.as_ref() { - let s = yara_x::Scanner::new(&rules.0); - *scanner = Box::into_raw(Box::new(YRX_SCANNER { - inner: s, - on_matching_rule: None, - })); - YRX_ERROR::SUCCESS - } else { - YRX_ERROR::INVALID_ARGUMENT - } -} - -/// Scans a data buffer. -#[no_mangle] -pub unsafe extern "C" fn yrx_scanner_scan( - scanner: *mut YRX_SCANNER, - data: *const u8, - len: usize, -) -> YRX_ERROR { - if scanner.is_null() { - return YRX_ERROR::INVALID_ARGUMENT; - } - - // Data is allowed to be null as long as len is 0. This case is handled - // as an empty slice. - if data.is_null() && len > 0 { - return YRX_ERROR::INVALID_ARGUMENT; - } - - let scanner = scanner.as_mut().unwrap(); - let data = slice::from_raw_parts(data, len); - let scan_results = scanner.inner.scan(data); - - if scan_results.is_err() { - // TODO: return appropriate error - return YRX_ERROR::PANIC; - } - - let scan_results = scan_results.unwrap(); - - if let Some((callback, user_data)) = scanner.on_matching_rule { - for r in scan_results.matching_rules() { - let rule = YRX_RULE(r); - callback(&rule as *const YRX_RULE, user_data); - } - } - - YRX_ERROR::SUCCESS -} - -/// Callback function passed to the scanner via [`yrx_scanner_on_matching_rule`] -/// which receives notifications about matching rules. -/// -/// The callback receives a pointer to the matching rule, represented by a -/// [`YRX_RULE`] structure. This pointer is guaranteed to be valid while the -/// callback function is being executed, but it may be freed after the callback -/// function returns, so you cannot use the pointer outside the callback. -/// -/// It also receives the `user_data` pointer that was passed to the -/// [`yrx_scanner_on_matching_rule`] function, which can point to arbitrary -/// data owned by the user. -type YRX_ON_MATCHING_RULE = extern "C" fn( - rule: *const YRX_RULE, - user_data: *mut std::ffi::c_void, -) -> (); - -/// Sets a callback function that is called by the scanner for each rule that -/// matched during a scan. -/// -/// The `user_data` pointer can be used to provide additional context to your -/// callback function. If the callback is not set, the scanner doesn't notify -/// about matching rules. -/// -/// See [`YRX_ON_MATCHING_RULE`] for more details. -#[no_mangle] -pub unsafe extern "C" fn yrx_scanner_on_matching_rule( - scanner: *mut YRX_SCANNER, - callback: YRX_ON_MATCHING_RULE, - user_data: *mut std::ffi::c_void, -) -> YRX_ERROR { - if let Some(scanner) = scanner.as_mut() { - scanner.on_matching_rule = Some((callback, user_data)); - YRX_ERROR::SUCCESS - } else { - YRX_ERROR::INVALID_ARGUMENT - } -} - -/// Destroys a [`YRX_SCANNER`] object. -#[no_mangle] -pub unsafe extern "C" fn yrx_scanner_destroy(scanner: *mut YRX_SCANNER) { - drop(Box::from_raw(scanner)) -} - /// Destroys a [`YRX_RULES`] object. #[no_mangle] pub unsafe extern "C" fn yrx_rules_destroy(rules: *mut YRX_RULES) { diff --git a/capi/src/scanner.rs b/capi/src/scanner.rs new file mode 100644 index 000000000..af644d2f6 --- /dev/null +++ b/capi/src/scanner.rs @@ -0,0 +1,116 @@ +use std::slice; + +use crate::{YRX_ERROR, YRX_RULE, YRX_RULES}; + +/// A scanner that scans data with a set of compiled YARA rules. +pub struct YRX_SCANNER<'s> { + inner: yara_x::Scanner<'s>, + on_matching_rule: Option<(YRX_ON_MATCHING_RULE, *mut std::ffi::c_void)>, +} + +/// Creates a [`YRX_SCANNER`] object that can be used for scanning data with +/// the provided [`YRX_RULES`]. +/// +/// It's ok to pass the same [`YRX_RULES`] to multiple scanners, and use each +/// scanner from a different thread. The scanner can be used as many times as +/// you want, and it must be destroyed with [`yrx_scanner_destroy`]. Also, the +/// scanner is valid as long as the rules are not destroyed, so, always destroy +/// the [`YRX_SCANNER`] object before the [`YRX_RULES`] object. +#[no_mangle] +pub unsafe extern "C" fn yrx_scanner_create( + rules: *const YRX_RULES, + scanner: &mut *mut YRX_SCANNER, +) -> YRX_ERROR { + if let Some(rules) = rules.as_ref() { + let s = yara_x::Scanner::new(&rules.0); + *scanner = Box::into_raw(Box::new(YRX_SCANNER { + inner: s, + on_matching_rule: None, + })); + YRX_ERROR::SUCCESS + } else { + YRX_ERROR::INVALID_ARGUMENT + } +} + +/// Scans a data buffer. +#[no_mangle] +pub unsafe extern "C" fn yrx_scanner_scan( + scanner: *mut YRX_SCANNER, + data: *const u8, + len: usize, +) -> YRX_ERROR { + if scanner.is_null() { + return YRX_ERROR::INVALID_ARGUMENT; + } + + // Data is allowed to be null as long as len is 0. This case is handled + // as an empty slice. + if data.is_null() && len > 0 { + return YRX_ERROR::INVALID_ARGUMENT; + } + + let scanner = scanner.as_mut().unwrap(); + let data = slice::from_raw_parts(data, len); + let scan_results = scanner.inner.scan(data); + + if scan_results.is_err() { + // TODO: return appropriate error + return YRX_ERROR::PANIC; + } + + let scan_results = scan_results.unwrap(); + + if let Some((callback, user_data)) = scanner.on_matching_rule { + for r in scan_results.matching_rules() { + let rule = YRX_RULE(r); + callback(&rule as *const YRX_RULE, user_data); + } + } + + YRX_ERROR::SUCCESS +} + +/// Callback function passed to the scanner via [`yrx_scanner_on_matching_rule`] +/// which receives notifications about matching rules. +/// +/// The callback receives a pointer to the matching rule, represented by a +/// [`YRX_RULE`] structure. This pointer is guaranteed to be valid while the +/// callback function is being executed, but it may be freed after the callback +/// function returns, so you cannot use the pointer outside the callback. +/// +/// It also receives the `user_data` pointer that was passed to the +/// [`yrx_scanner_on_matching_rule`] function, which can point to arbitrary +/// data owned by the user. +type YRX_ON_MATCHING_RULE = extern "C" fn( + rule: *const YRX_RULE, + user_data: *mut std::ffi::c_void, +) -> (); + +/// Sets a callback function that is called by the scanner for each rule that +/// matched during a scan. +/// +/// The `user_data` pointer can be used to provide additional context to your +/// callback function. If the callback is not set, the scanner doesn't notify +/// about matching rules. +/// +/// See [`YRX_ON_MATCHING_RULE`] for more details. +#[no_mangle] +pub unsafe extern "C" fn yrx_scanner_on_matching_rule( + scanner: *mut YRX_SCANNER, + callback: YRX_ON_MATCHING_RULE, + user_data: *mut std::ffi::c_void, +) -> YRX_ERROR { + if let Some(scanner) = scanner.as_mut() { + scanner.on_matching_rule = Some((callback, user_data)); + YRX_ERROR::SUCCESS + } else { + YRX_ERROR::INVALID_ARGUMENT + } +} + +/// Destroys a [`YRX_SCANNER`] object. +#[no_mangle] +pub unsafe extern "C" fn yrx_scanner_destroy(scanner: *mut YRX_SCANNER) { + drop(Box::from_raw(scanner)) +} From 6cf62f4d574bb832cb8d7bbee3f16476157ba33a Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Sat, 10 Feb 2024 22:52:39 +0100 Subject: [PATCH 32/55] feat: implement `ExactSizeIterator`trait for `Patterns` --- lib/src/scanner/mod.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lib/src/scanner/mod.rs b/lib/src/scanner/mod.rs index 62509c5bd..a5219ba1e 100644 --- a/lib/src/scanner/mod.rs +++ b/lib/src/scanner/mod.rs @@ -831,6 +831,7 @@ impl<'a, 'r> Rule<'a, 'r> { ctx: self.ctx, data: self.data, iterator: self.rule_info.patterns.iter(), + len: self.rule_info.patterns.len(), } } } @@ -840,6 +841,7 @@ pub struct Patterns<'a, 'r> { ctx: &'a ScanContext<'r>, data: &'a ScannedData<'a>, iterator: Iter<'a, (IdentId, PatternId)>, + len: usize, } impl<'a, 'r> Iterator for Patterns<'a, 'r> { @@ -856,6 +858,13 @@ impl<'a, 'r> Iterator for Patterns<'a, 'r> { } } +impl<'a, 'r> ExactSizeIterator for Patterns<'a, 'r> { + #[inline] + fn len(&self) -> usize { + self.len + } +} + /// Represents a pattern defined by a rule. pub struct Pattern<'a, 'r> { ctx: &'a ScanContext<'r>, From be21199c14327c77b0567abd131f9145d89e288b Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Tue, 13 Feb 2024 16:33:39 +0100 Subject: [PATCH 33/55] feat: advance in the implementation of the C and Golang APIs --- capi/include/yara-x.h | 185 +++++++++++++++++++--------- capi/src/compiler.rs | 219 ++++++++++++++++++++++++++++++++++ capi/src/lib.rs | 208 ++++++++++++++++---------------- capi/src/scanner.rs | 106 ++++++++++++---- go/compiler.go | 95 +++++++++++++++ go/compiler_test.go | 54 +++++++++ go/main.go | 70 ++++------- go/scanner.go | 56 ++++++--- go/scanner_test.go | 29 ++++- lib/src/compiler/errors.rs | 9 +- lib/src/compiler/mod.rs | 10 +- lib/src/compiler/tests/mod.rs | 16 +-- lib/src/scanner/mod.rs | 3 +- lib/src/variables.rs | 2 +- parser/src/parser/errors.rs | 3 +- 15 files changed, 792 insertions(+), 273 deletions(-) create mode 100644 capi/src/compiler.rs create mode 100644 go/compiler.go create mode 100644 go/compiler_test.go diff --git a/capi/include/yara-x.h b/capi/include/yara-x.h index f783a39a5..025a8dcb9 100644 --- a/capi/include/yara-x.h +++ b/capi/include/yara-x.h @@ -14,18 +14,19 @@ #include -typedef enum YRX_ERROR { +typedef enum YRX_RESULT { SUCCESS, PANIC, SYNTAX_ERROR, + VARIABLE_ERROR, + SCAN_ERROR, + SCAN_TIMEOUT, + INVALID_IDENTIFIER, INVALID_ARGUMENT, -} YRX_ERROR; +} YRX_RESULT; -// A pattern within a rule. -typedef struct YRX_PATTERN YRX_PATTERN; - -// The set of patterns declared in a YARA rule. -typedef struct YRX_PATTERNS YRX_PATTERNS; +// A compiler that takes YARA source code and produces compiled rules. +typedef struct YRX_COMPILER YRX_COMPILER; // A single YARA rule. typedef struct YRX_RULE YRX_RULE; @@ -42,6 +43,28 @@ typedef struct YRX_MATCH { size_t length; } YRX_MATCH; +// A pattern within a rule. +typedef struct YRX_PATTERN { + // Pattern's identifier (i.e: $a, $foo) + char *identifier; + // Number of matches found for this pattern. + size_t num_matches; + // Pointer to an array of YRX_MATCH structures describing the matches + // for this pattern. The array has num_matches items. If num_matches is + // zero this pointer is invalid and should not be de-referenced. + struct YRX_MATCH *matches; +} YRX_PATTERN; + +// A set of patterns declared in a YARA rule. +typedef struct YRX_PATTERNS { + // Number of patterns. + size_t num_patterns; + // Pointer to an array of YRX_PATTERN structures. The array has + // num_patterns items. If num_patterns is zero this pointer is invalid + // and should not be de-referenced. + struct YRX_PATTERN *patterns; +} YRX_PATTERNS; + // Callback function passed to the scanner via [`yrx_scanner_on_matching_rule`] // which receives notifications about matching rules. // @@ -60,8 +83,8 @@ typedef void (*YRX_ON_MATCHING_RULE)(const struct YRX_RULE *rule, // the compiled rules. // // The rules must be destroyed with [`yrx_rules_destroy`]. -enum YRX_ERROR yrx_compile(const char *src, - struct YRX_RULES **rules); +enum YRX_RESULT yrx_compile(const char *src, + struct YRX_RULES **rules); // Destroys a [`YRX_RULES`] object. void yrx_rules_destroy(struct YRX_RULES *rules); @@ -74,9 +97,9 @@ void yrx_rules_destroy(struct YRX_RULES *rules); // null-terminated, and the pointer will be valid as long as the [`YRX_RULES`] // object that contains the rule is not freed. The name is guaranteed to be a // valid UTF-8 string. -enum YRX_ERROR yrx_rule_identifier(const struct YRX_RULE *rule, - const uint8_t **ident, - size_t *len); +enum YRX_RESULT yrx_rule_identifier(const struct YRX_RULE *rule, + const uint8_t **ident, + size_t *len); // Returns the namespace of the rule represented by [`YRX_RULE`]. // @@ -86,47 +109,79 @@ enum YRX_ERROR yrx_rule_identifier(const struct YRX_RULE *rule, // null-terminated, and the pointer will be valid as long as the [`YRX_RULES`] // object that contains the rule is not freed. The namespace is guaranteed to // be a valid UTF-8 string. -enum YRX_ERROR yrx_rule_namespace(const struct YRX_RULE *rule, - const uint8_t **ns, - size_t *len); - -// Returns the all patterns defined by a rule, each pattern contains -// information about whether it matched or not, and where in the data it -// matched. -// -// The [`YRX_PATTERNS`] object must be destroyed with [`yrx_patterns_destroy`]. -const struct YRX_PATTERNS *yrx_rule_patterns(const struct YRX_RULE *rule); - -// Returns the number of patterns in a given [`YRX_PATTERNS`] object. -int32_t yrx_patterns_count(const struct YRX_PATTERNS *patterns); +enum YRX_RESULT yrx_rule_namespace(const struct YRX_RULE *rule, + const uint8_t **ns, + size_t *len); -// Returns the pattern with the give `index`, from a set of patterns represented -// by a [`YRX_PATTERNS`] object. +// Returns all the patterns defined by a rule. // -// The index must be between 0 and the value returned by [`yrx_patterns_count`], -// otherwise the result will be a null pointer. The result is also a null -// pointer if `patterns` is null. -const struct YRX_PATTERN *yrx_patterns_get(struct YRX_PATTERNS *patterns, - size_t index); +// Each pattern contains information about whether it matched or not, and where +// in the data it matched. The patterns are represented by a [`YRX_PATTERNS`] +// object that must be destroyed with [`yrx_patterns_destroy`] when not needed +// anymore. +struct YRX_PATTERNS *yrx_rule_patterns(const struct YRX_RULE *rule); // Destroys a [`YRX_PATTERNS`] object. void yrx_patterns_destroy(struct YRX_PATTERNS *patterns); -// Returns the identifier of rule's pattern represented by [`YRX_PATTERN`]. +// Creates a [`YRX_COMPILER`] object. +enum YRX_RESULT yrx_compiler_create(struct YRX_COMPILER **compiler); + +// Destroys a [`YRX_COMPILER`] object. +void yrx_compiler_destroy(struct YRX_COMPILER *compiler); + +// Adds a YARA source code to be compiled. // -// Arguments `ident` and `len` are output parameters that receive pointers to a -// `const uint8_t*` and `size_t`, where this function will leave a pointer -// to the rule's name and its length, respectively. The identifier is *NOT* -// null-terminated, and the pointer will be valid as long as the [`YRX_RULES`] -// object that contains the rule is not freed. The name is guaranteed to be a -// valid UTF-8 string. -enum YRX_ERROR yrx_pattern_identifier(const struct YRX_PATTERN *pattern, - const uint8_t **ident, - size_t *len); +// This function can be called multiple times. +enum YRX_RESULT yrx_compiler_add_source(struct YRX_COMPILER *compiler, + const char *src); -enum YRX_ERROR yrx_pattern_matches(const struct YRX_PATTERN *pattern, - const struct YRX_MATCH **matches, - size_t *len); +// Returns the error message for the most recent error returned by the +// compiler. +// +// The returned pointer is only valid until the next call to any of the +// yrx_compiler_xxxx functions. A call any of these functions can modify +// the last error, rendering the pointer to a previous error message +// invalid. Also, the pointer will be null if the compiler hasn't returned +// any error. +const char *yrx_compiler_last_error(const struct YRX_COMPILER *compiler); + +// Creates a new namespace. +// +// Further calls to `yrx_compiler_add_source` will put the rules under the +// newly created namespace. +// +// The `namespace` argument must be pointer to null-terminated UTF-8 string. +// If the string is not valid UTF-8 the result is an `INVALID_ARGUMENT` error. +enum YRX_RESULT yrx_compiler_new_namespace(struct YRX_COMPILER *compiler, + const char *namespace_); + +// Defines a global variable of string type and sets its initial value. +enum YRX_RESULT yrx_compiler_define_global_str(struct YRX_COMPILER *compiler, + const char *ident, + const char *value); + +// Defines a global variable of bool type and sets its initial value. +enum YRX_RESULT yrx_compiler_define_global_bool(struct YRX_COMPILER *compiler, + const char *ident, + bool value); + +// Defines a global variable of integer type and sets its initial value. +enum YRX_RESULT yrx_compiler_define_global_int(struct YRX_COMPILER *compiler, + const char *ident, + int64_t value); + +// Defines a global variable of float type and sets its initial value. +enum YRX_RESULT yrx_compiler_define_global_float(struct YRX_COMPILER *compiler, + const char *ident, + double value); + +// Builds the source code previously added to the compiler. +// +// After calling this function the compiler is reset to its initial state, +// you can keep using it by adding more sources and calling this function +// again. +struct YRX_RULES *yrx_compiler_build(struct YRX_COMPILER *compiler); // Creates a [`YRX_SCANNER`] object that can be used for scanning data with // the provided [`YRX_RULES`]. @@ -136,13 +191,26 @@ enum YRX_ERROR yrx_pattern_matches(const struct YRX_PATTERN *pattern, // you want, and it must be destroyed with [`yrx_scanner_destroy`]. Also, the // scanner is valid as long as the rules are not destroyed, so, always destroy // the [`YRX_SCANNER`] object before the [`YRX_RULES`] object. -enum YRX_ERROR yrx_scanner_create(const struct YRX_RULES *rules, - struct YRX_SCANNER **scanner); +enum YRX_RESULT yrx_scanner_create(const struct YRX_RULES *rules, + struct YRX_SCANNER **scanner); + +// Destroys a [`YRX_SCANNER`] object. +void yrx_scanner_destroy(struct YRX_SCANNER *scanner); + +// Sets a timeout (in seconds) for scan operations. +// +// The scan functions will return a timeout error once the provided timeout +// duration has elapsed. The scanner will make every effort to stop promptly +// after the designated timeout duration. However, in some cases, particularly +// with rules containing only a few patterns, the scanner could potentially +// continue running for a longer period than the specified timeout. +enum YRX_RESULT yrx_scanner_timeout(struct YRX_SCANNER *scanner, + uint64_t timeout); // Scans a data buffer. -enum YRX_ERROR yrx_scanner_scan(struct YRX_SCANNER *scanner, - const uint8_t *data, - size_t len); +enum YRX_RESULT yrx_scanner_scan(struct YRX_SCANNER *scanner, + const uint8_t *data, + size_t len); // Sets a callback function that is called by the scanner for each rule that // matched during a scan. @@ -152,11 +220,18 @@ enum YRX_ERROR yrx_scanner_scan(struct YRX_SCANNER *scanner, // about matching rules. // // See [`YRX_ON_MATCHING_RULE`] for more details. -enum YRX_ERROR yrx_scanner_on_matching_rule(struct YRX_SCANNER *scanner, - YRX_ON_MATCHING_RULE callback, - void *user_data); +enum YRX_RESULT yrx_scanner_on_matching_rule(struct YRX_SCANNER *scanner, + YRX_ON_MATCHING_RULE callback, + void *user_data); -// Destroys a [`YRX_SCANNER`] object. -void yrx_scanner_destroy(struct YRX_SCANNER *scanner); +// Returns the error message for the most recent error returned by the +// scanner. +// +// The returned pointer is only valid until the next call to any of the +// yrx_scanner_xxxx functions. A call any of these functions can modify +// the last error, rendering the pointer to a previous error message +// invalid. Also, the pointer will be null if the scanner hasn't returned +// any error. +const char *yrx_scanner_last_error(const struct YRX_SCANNER *scanner); #endif /* YARA_X */ diff --git a/capi/src/compiler.rs b/capi/src/compiler.rs new file mode 100644 index 000000000..a2a5d8161 --- /dev/null +++ b/capi/src/compiler.rs @@ -0,0 +1,219 @@ +use crate::{YRX_RESULT, YRX_RULES}; +use std::ffi::{c_char, CStr, CString}; +use std::mem; + +/// A compiler that takes YARA source code and produces compiled rules. +pub struct YRX_COMPILER<'a> { + inner: yara_x::Compiler<'a>, + last_error: Option, +} + +/// Creates a [`YRX_COMPILER`] object. +#[no_mangle] +pub unsafe extern "C" fn yrx_compiler_create( + compiler: &mut *mut YRX_COMPILER, +) -> YRX_RESULT { + *compiler = Box::into_raw(Box::new(YRX_COMPILER { + inner: yara_x::Compiler::new(), + last_error: None, + })); + YRX_RESULT::SUCCESS +} + +/// Destroys a [`YRX_COMPILER`] object. +#[no_mangle] +pub unsafe extern "C" fn yrx_compiler_destroy(compiler: *mut YRX_COMPILER) { + drop(Box::from_raw(compiler)) +} + +/// Adds a YARA source code to be compiled. +/// +/// This function can be called multiple times. +#[no_mangle] +pub unsafe extern "C" fn yrx_compiler_add_source( + compiler: *mut YRX_COMPILER, + src: *const c_char, +) -> YRX_RESULT { + let compiler = if let Some(compiler) = compiler.as_mut() { + compiler + } else { + return YRX_RESULT::INVALID_ARGUMENT; + }; + + let src = CStr::from_ptr(src); + + match compiler.inner.add_source(src.to_bytes()) { + Ok(_) => { + compiler.last_error = None; + YRX_RESULT::SUCCESS + } + Err(err) => { + compiler.last_error = Some(CString::new(err.to_string()).unwrap()); + YRX_RESULT::SYNTAX_ERROR + } + } +} + +/// Returns the error message for the most recent error returned by the +/// compiler. +/// +/// The returned pointer is only valid until the next call to any of the +/// yrx_compiler_xxxx functions. A call any of these functions can modify +/// the last error, rendering the pointer to a previous error message +/// invalid. Also, the pointer will be null if the compiler hasn't returned +/// any error. +#[no_mangle] +pub unsafe extern "C" fn yrx_compiler_last_error( + compiler: *const YRX_COMPILER, +) -> *const c_char { + let compiler = if let Some(compiler) = compiler.as_ref() { + compiler + } else { + return std::ptr::null(); + }; + + if let Some(last_error) = compiler.last_error.as_ref() { + last_error.as_ptr() + } else { + std::ptr::null() + } +} + +/// Creates a new namespace. +/// +/// Further calls to `yrx_compiler_add_source` will put the rules under the +/// newly created namespace. +/// +/// The `namespace` argument must be pointer to null-terminated UTF-8 string. +/// If the string is not valid UTF-8 the result is an `INVALID_ARGUMENT` error. +#[no_mangle] +pub unsafe extern "C" fn yrx_compiler_new_namespace( + compiler: *mut YRX_COMPILER, + namespace: *const c_char, +) -> YRX_RESULT { + let compiler = if let Some(compiler) = compiler.as_mut() { + compiler + } else { + return YRX_RESULT::INVALID_ARGUMENT; + }; + + let namespace = if let Ok(namespace) = CStr::from_ptr(namespace).to_str() { + namespace + } else { + return YRX_RESULT::INVALID_ARGUMENT; + }; + + compiler.inner.new_namespace(namespace); + + YRX_RESULT::SUCCESS +} + +/// Defines a global variable and sets its initial value. +/// +/// Global variables must be defined before using `yrx_compiler_add_source` +/// for adding any YARA source code that uses those variables. The variable +/// will retain its initial value when the compiled rules are used for +/// scanning data, however each scanner can change the variable’s initial +/// value by calling `yrx_scanner_set_global`. +unsafe fn yrx_compiler_define_global< + T: TryInto, +>( + compiler: *mut YRX_COMPILER, + ident: *const c_char, + value: T, +) -> YRX_RESULT { + let compiler = if let Some(compiler) = compiler.as_mut() { + compiler + } else { + return YRX_RESULT::INVALID_ARGUMENT; + }; + + let ident = if let Ok(ident) = CStr::from_ptr(ident).to_str() { + ident + } else { + return YRX_RESULT::INVALID_ARGUMENT; + }; + + match compiler.inner.define_global(ident, value) { + Ok(_) => { + compiler.last_error = None; + YRX_RESULT::SUCCESS + } + Err(err) => { + compiler.last_error = Some(CString::new(err.to_string()).unwrap()); + YRX_RESULT::VARIABLE_ERROR + } + } +} + +/// Defines a global variable of string type and sets its initial value. +#[no_mangle] +pub unsafe extern "C" fn yrx_compiler_define_global_str( + compiler: *mut YRX_COMPILER, + ident: *const c_char, + value: *const c_char, +) -> YRX_RESULT { + let value = if let Ok(value) = CStr::from_ptr(value).to_str() { + value + } else { + return YRX_RESULT::INVALID_ARGUMENT; + }; + + yrx_compiler_define_global(compiler, ident, value) +} + +/// Defines a global variable of bool type and sets its initial value. +#[no_mangle] +pub unsafe extern "C" fn yrx_compiler_define_global_bool( + compiler: *mut YRX_COMPILER, + ident: *const c_char, + value: bool, +) -> YRX_RESULT { + yrx_compiler_define_global(compiler, ident, value) +} + +/// Defines a global variable of integer type and sets its initial value. +#[no_mangle] +pub unsafe extern "C" fn yrx_compiler_define_global_int( + compiler: *mut YRX_COMPILER, + ident: *const c_char, + value: i64, +) -> YRX_RESULT { + yrx_compiler_define_global(compiler, ident, value) +} + +/// Defines a global variable of float type and sets its initial value. +#[no_mangle] +pub unsafe extern "C" fn yrx_compiler_define_global_float( + compiler: *mut YRX_COMPILER, + ident: *const c_char, + value: f64, +) -> YRX_RESULT { + yrx_compiler_define_global(compiler, ident, value) +} + +/// Builds the source code previously added to the compiler. +/// +/// After calling this function the compiler is reset to its initial state, +/// you can keep using it by adding more sources and calling this function +/// again. +#[no_mangle] +pub unsafe extern "C" fn yrx_compiler_build( + compiler: *mut YRX_COMPILER, +) -> *mut YRX_RULES { + let compiler = if let Some(compiler) = compiler.as_mut() { + compiler + } else { + return std::ptr::null_mut(); + }; + + // As the build() method consumes the compiler, we need to take ownership + // of it, but that implies that the inner compiler in the YRX_COMPILER + // object must be replaced with something else, either a null value or a + // new compiler.It is replaced with a new compiler, so that users of the + // C API can keep using the YRX_COMPILER object after calling + // yrx_compiler_build. + let compiler = mem::replace(&mut compiler.inner, yara_x::Compiler::new()); + + Box::into_raw(Box::new(YRX_RULES(compiler.build()))) +} diff --git a/capi/src/lib.rs b/capi/src/lib.rs index 9ea69b441..4ee8b4d4d 100644 --- a/capi/src/lib.rs +++ b/capi/src/lib.rs @@ -15,17 +15,24 @@ This crate is not intended to be used by other Rust programs. #![allow(clippy::missing_safety_doc)] #![allow(clippy::not_unsafe_ptr_arg_deref)] -use std::ffi::{c_char, CStr}; +use std::ffi::{c_char, CStr, CString}; +use std::mem::ManuallyDrop; +use std::ptr::slice_from_raw_parts_mut; +mod compiler; mod scanner; pub use scanner::*; #[repr(C)] -pub enum YRX_ERROR { +pub enum YRX_RESULT { SUCCESS, PANIC, SYNTAX_ERROR, + VARIABLE_ERROR, + SCAN_ERROR, + SCAN_TIMEOUT, + INVALID_IDENTIFIER, INVALID_ARGUMENT, } @@ -35,10 +42,51 @@ pub struct YRX_RULES(yara_x::Rules); /// A single YARA rule. pub struct YRX_RULE<'a, 'r>(yara_x::Rule<'a, 'r>); +/// A set of patterns declared in a YARA rule. +#[repr(C)] +pub struct YRX_PATTERNS { + /// Number of patterns. + num_patterns: usize, + /// Pointer to an array of YRX_PATTERN structures. The array has + /// num_patterns items. If num_patterns is zero this pointer is invalid + /// and should not be de-referenced. + patterns: *mut YRX_PATTERN, +} + +impl Drop for YRX_PATTERNS { + fn drop(&mut self) { + unsafe { + drop(Box::from_raw(slice_from_raw_parts_mut( + self.patterns, + self.num_patterns, + ))); + } + } +} + /// A pattern within a rule. +#[repr(C)] pub struct YRX_PATTERN { - identifier: String, - matches: Vec, + /// Pattern's identifier (i.e: $a, $foo) + identifier: *mut c_char, + /// Number of matches found for this pattern. + num_matches: usize, + /// Pointer to an array of YRX_MATCH structures describing the matches + /// for this pattern. The array has num_matches items. If num_matches is + /// zero this pointer is invalid and should not be de-referenced. + matches: *mut YRX_MATCH, +} + +impl Drop for YRX_PATTERN { + fn drop(&mut self) { + unsafe { + drop(CString::from_raw(self.identifier)); + drop(Box::from_raw(slice_from_raw_parts_mut( + self.matches, + self.num_matches, + ))); + } + } } /// Contains information about a pattern match. @@ -48,9 +96,6 @@ pub struct YRX_MATCH { pub length: usize, } -/// The set of patterns declared in a YARA rule. -pub struct YRX_PATTERNS(Vec); - /// Compiles YARA source code and creates a [`YRX_RULES`] object that contains /// the compiled rules. /// @@ -59,15 +104,18 @@ pub struct YRX_PATTERNS(Vec); pub unsafe extern "C" fn yrx_compile( src: *const c_char, rules: &mut *mut YRX_RULES, -) -> YRX_ERROR { +) -> YRX_RESULT { let c_str = CStr::from_ptr(src); match yara_x::compile(c_str.to_bytes()) { Ok(r) => *rules = Box::into_raw(Box::new(YRX_RULES(r))), - Err(_) => return YRX_ERROR::SYNTAX_ERROR, + Err(_) => { + // TODO: handle error + return YRX_RESULT::SYNTAX_ERROR; + } }; - YRX_ERROR::SUCCESS + YRX_RESULT::SUCCESS } /// Destroys a [`YRX_RULES`] object. @@ -89,13 +137,13 @@ pub unsafe extern "C" fn yrx_rule_identifier( rule: *const YRX_RULE, ident: &mut *const u8, len: &mut usize, -) -> YRX_ERROR { +) -> YRX_RESULT { if let Some(rule) = rule.as_ref() { *ident = rule.0.identifier().as_ptr(); *len = rule.0.identifier().len(); - YRX_ERROR::SUCCESS + YRX_RESULT::SUCCESS } else { - YRX_ERROR::INVALID_ARGUMENT + YRX_RESULT::INVALID_ARGUMENT } } @@ -112,118 +160,62 @@ pub unsafe extern "C" fn yrx_rule_namespace( rule: *const YRX_RULE, ns: &mut *const u8, len: &mut usize, -) -> YRX_ERROR { +) -> YRX_RESULT { if let Some(rule) = rule.as_ref() { *ns = rule.0.namespace().as_ptr(); *len = rule.0.namespace().len(); - YRX_ERROR::SUCCESS + YRX_RESULT::SUCCESS } else { - YRX_ERROR::INVALID_ARGUMENT + YRX_RESULT::INVALID_ARGUMENT } } -/// Returns the all patterns defined by a rule, each pattern contains -/// information about whether it matched or not, and where in the data it -/// matched. +/// Returns all the patterns defined by a rule. /// -/// The [`YRX_PATTERNS`] object must be destroyed with [`yrx_patterns_destroy`]. +/// Each pattern contains information about whether it matched or not, and where +/// in the data it matched. The patterns are represented by a [`YRX_PATTERNS`] +/// object that must be destroyed with [`yrx_patterns_destroy`] when not needed +/// anymore. #[no_mangle] pub unsafe extern "C" fn yrx_rule_patterns( rule: *const YRX_RULE, -) -> *const YRX_PATTERNS { - if let Some(rule) = rule.as_ref() { - return Box::into_raw(Box::new(YRX_PATTERNS( - rule.0 - .patterns() - .map(|pat| YRX_PATTERN { - identifier: pat.identifier().to_string(), - matches: pat - .matches() - .map(|m| YRX_MATCH { - offset: m.range().start, - length: m.range().len(), - }) - .collect(), - }) - .collect(), - ))); - } else { - std::ptr::null() +) -> *mut YRX_PATTERNS { + let patterns_iter = rule.as_ref().unwrap().0.patterns(); + let mut patterns = Vec::with_capacity(patterns_iter.len()); + + for pattern in patterns_iter { + let matches = pattern + .matches() + .map(|m| YRX_MATCH { + offset: m.range().start, + length: m.range().len(), + }) + .collect::>() + .into_boxed_slice(); + + // Prevent `matches` from being dropped at the end of the current + // scope. We are taking a pointer to `matches` and storing it in a + // YRX_PATTERN structure. The `YRX_PATTERN::drop` method takes care + // of dropping the slice of matches. + let mut matches = ManuallyDrop::new(matches); + + patterns.push(YRX_PATTERN { + identifier: CString::new(pattern.identifier()).unwrap().into_raw(), + num_matches: matches.len(), + matches: matches.as_mut_ptr(), + }); } -} -/// Returns the number of patterns in a given [`YRX_PATTERNS`] object. -#[no_mangle] -pub unsafe extern "C" fn yrx_patterns_count( - patterns: *const YRX_PATTERNS, -) -> i32 { - if let Some(patterns) = patterns.as_ref() { - patterns.0.len() as i32 - } else { - -1 - } -} + let mut patterns = ManuallyDrop::new(patterns); -/// Returns the pattern with the give `index`, from a set of patterns represented -/// by a [`YRX_PATTERNS`] object. -/// -/// The index must be between 0 and the value returned by [`yrx_patterns_count`], -/// otherwise the result will be a null pointer. The result is also a null -/// pointer if `patterns` is null. -#[no_mangle] -pub unsafe extern "C" fn yrx_patterns_get( - patterns: *mut YRX_PATTERNS, - index: usize, -) -> *const YRX_PATTERN { - if let Some(pattern) = - patterns.as_ref().and_then(|patterns| patterns.0.get(index)) - { - pattern - } else { - std::ptr::null() - } + Box::into_raw(Box::new(YRX_PATTERNS { + num_patterns: patterns.len(), + patterns: patterns.as_mut_ptr(), + })) } /// Destroys a [`YRX_PATTERNS`] object. #[no_mangle] pub unsafe extern "C" fn yrx_patterns_destroy(patterns: *mut YRX_PATTERNS) { - drop(Box::from_raw(patterns)) -} - -/// Returns the identifier of rule's pattern represented by [`YRX_PATTERN`]. -/// -/// Arguments `ident` and `len` are output parameters that receive pointers to a -/// `const uint8_t*` and `size_t`, where this function will leave a pointer -/// to the rule's name and its length, respectively. The identifier is *NOT* -/// null-terminated, and the pointer will be valid as long as the [`YRX_RULES`] -/// object that contains the rule is not freed. The name is guaranteed to be a -/// valid UTF-8 string. -#[no_mangle] -pub unsafe extern "C" fn yrx_pattern_identifier( - pattern: *const YRX_PATTERN, - ident: &mut *const u8, - len: &mut usize, -) -> YRX_ERROR { - if let Some(pattern) = pattern.as_ref() { - *ident = pattern.identifier.as_ptr(); - *len = pattern.identifier.len(); - YRX_ERROR::SUCCESS - } else { - YRX_ERROR::INVALID_ARGUMENT - } -} - -#[no_mangle] -pub unsafe extern "C" fn yrx_pattern_matches( - pattern: *const YRX_PATTERN, - matches: &mut *const YRX_MATCH, - len: &mut usize, -) -> YRX_ERROR { - if let Some(pattern) = pattern.as_ref() { - *matches = pattern.matches.as_ptr(); - *len = pattern.matches.len(); - YRX_ERROR::SUCCESS - } else { - YRX_ERROR::INVALID_ARGUMENT - } + drop(Box::from_raw(patterns)); } diff --git a/capi/src/scanner.rs b/capi/src/scanner.rs index af644d2f6..6c3896a7d 100644 --- a/capi/src/scanner.rs +++ b/capi/src/scanner.rs @@ -1,11 +1,16 @@ +use std::ffi::{c_char, CString}; use std::slice; +use std::time::Duration; +use yara_x::ScanError; -use crate::{YRX_ERROR, YRX_RULE, YRX_RULES}; +use crate::compiler::YRX_COMPILER; +use crate::{YRX_RESULT, YRX_RULE, YRX_RULES}; /// A scanner that scans data with a set of compiled YARA rules. pub struct YRX_SCANNER<'s> { inner: yara_x::Scanner<'s>, on_matching_rule: Option<(YRX_ON_MATCHING_RULE, *mut std::ffi::c_void)>, + last_error: Option, } /// Creates a [`YRX_SCANNER`] object that can be used for scanning data with @@ -20,17 +25,48 @@ pub struct YRX_SCANNER<'s> { pub unsafe extern "C" fn yrx_scanner_create( rules: *const YRX_RULES, scanner: &mut *mut YRX_SCANNER, -) -> YRX_ERROR { - if let Some(rules) = rules.as_ref() { - let s = yara_x::Scanner::new(&rules.0); - *scanner = Box::into_raw(Box::new(YRX_SCANNER { - inner: s, - on_matching_rule: None, - })); - YRX_ERROR::SUCCESS +) -> YRX_RESULT { + let rules = if let Some(rules) = rules.as_ref() { + rules } else { - YRX_ERROR::INVALID_ARGUMENT + return YRX_RESULT::INVALID_ARGUMENT; + }; + + *scanner = Box::into_raw(Box::new(YRX_SCANNER { + inner: yara_x::Scanner::new(&rules.0), + on_matching_rule: None, + last_error: None, + })); + + YRX_RESULT::SUCCESS +} + +/// Destroys a [`YRX_SCANNER`] object. +#[no_mangle] +pub unsafe extern "C" fn yrx_scanner_destroy(scanner: *mut YRX_SCANNER) { + drop(Box::from_raw(scanner)) +} + +/// Sets a timeout (in seconds) for scan operations. +/// +/// The scan functions will return a timeout error once the provided timeout +/// duration has elapsed. The scanner will make every effort to stop promptly +/// after the designated timeout duration. However, in some cases, particularly +/// with rules containing only a few patterns, the scanner could potentially +/// continue running for a longer period than the specified timeout. +#[no_mangle] +pub unsafe extern "C" fn yrx_scanner_timeout( + scanner: *mut YRX_SCANNER, + timeout: u64, +) -> YRX_RESULT { + if scanner.is_null() { + return YRX_RESULT::INVALID_ARGUMENT; } + + let scanner = scanner.as_mut().unwrap(); + scanner.inner.timeout(Duration::from_secs(timeout)); + + YRX_RESULT::SUCCESS } /// Scans a data buffer. @@ -39,24 +75,27 @@ pub unsafe extern "C" fn yrx_scanner_scan( scanner: *mut YRX_SCANNER, data: *const u8, len: usize, -) -> YRX_ERROR { +) -> YRX_RESULT { if scanner.is_null() { - return YRX_ERROR::INVALID_ARGUMENT; + return YRX_RESULT::INVALID_ARGUMENT; } // Data is allowed to be null as long as len is 0. This case is handled // as an empty slice. if data.is_null() && len > 0 { - return YRX_ERROR::INVALID_ARGUMENT; + return YRX_RESULT::INVALID_ARGUMENT; } let scanner = scanner.as_mut().unwrap(); let data = slice::from_raw_parts(data, len); let scan_results = scanner.inner.scan(data); - if scan_results.is_err() { - // TODO: return appropriate error - return YRX_ERROR::PANIC; + if let Err(err) = scan_results { + scanner.last_error = Some(CString::new(err.to_string()).unwrap()); + return match err { + ScanError::Timeout => YRX_RESULT::SCAN_TIMEOUT, + _ => YRX_RESULT::SCAN_ERROR, + }; } let scan_results = scan_results.unwrap(); @@ -68,7 +107,7 @@ pub unsafe extern "C" fn yrx_scanner_scan( } } - YRX_ERROR::SUCCESS + YRX_RESULT::SUCCESS } /// Callback function passed to the scanner via [`yrx_scanner_on_matching_rule`] @@ -82,7 +121,7 @@ pub unsafe extern "C" fn yrx_scanner_scan( /// It also receives the `user_data` pointer that was passed to the /// [`yrx_scanner_on_matching_rule`] function, which can point to arbitrary /// data owned by the user. -type YRX_ON_MATCHING_RULE = extern "C" fn( +pub type YRX_ON_MATCHING_RULE = extern "C" fn( rule: *const YRX_RULE, user_data: *mut std::ffi::c_void, ) -> (); @@ -100,17 +139,36 @@ pub unsafe extern "C" fn yrx_scanner_on_matching_rule( scanner: *mut YRX_SCANNER, callback: YRX_ON_MATCHING_RULE, user_data: *mut std::ffi::c_void, -) -> YRX_ERROR { +) -> YRX_RESULT { if let Some(scanner) = scanner.as_mut() { scanner.on_matching_rule = Some((callback, user_data)); - YRX_ERROR::SUCCESS + YRX_RESULT::SUCCESS } else { - YRX_ERROR::INVALID_ARGUMENT + YRX_RESULT::INVALID_ARGUMENT } } -/// Destroys a [`YRX_SCANNER`] object. +/// Returns the error message for the most recent error returned by the +/// scanner. +/// +/// The returned pointer is only valid until the next call to any of the +/// yrx_scanner_xxxx functions. A call any of these functions can modify +/// the last error, rendering the pointer to a previous error message +/// invalid. Also, the pointer will be null if the scanner hasn't returned +/// any error. #[no_mangle] -pub unsafe extern "C" fn yrx_scanner_destroy(scanner: *mut YRX_SCANNER) { - drop(Box::from_raw(scanner)) +pub unsafe extern "C" fn yrx_scanner_last_error( + scanner: *const YRX_SCANNER, +) -> *const c_char { + let scanner = if let Some(scanner) = scanner.as_ref() { + scanner + } else { + return std::ptr::null(); + }; + + if let Some(last_error) = scanner.last_error.as_ref() { + last_error.as_ptr() + } else { + std::ptr::null() + } } diff --git a/go/compiler.go b/go/compiler.go new file mode 100644 index 000000000..b010da29b --- /dev/null +++ b/go/compiler.go @@ -0,0 +1,95 @@ +package yara_x + +// #include +import "C" +import ( + "errors" + "runtime" + "unsafe" +) + +type Compiler struct { + cCompiler *C.YRX_COMPILER +} + +func NewCompiler() *Compiler { + c := &Compiler{} + C.yrx_compiler_create(&c.cCompiler) + runtime.SetFinalizer(c, (*Compiler).Destroy) + return c +} + +func (c *Compiler) AddSource(src string) error { + cSrc := C.CString(src) + defer C.free(unsafe.Pointer(cSrc)) + var err error + if C.yrx_compiler_add_source(c.cCompiler, cSrc) == C.SYNTAX_ERROR { + err = errors.New(C.GoString(C.yrx_compiler_last_error(c.cCompiler))) + } + // After the call to yrx_compiler_add_source, c is not live anymore and + // the garbage collector could try to free it and call the finalizer while + // yrx_compiler_add_source is being executed. This ensure that c is alive + // until yrx_compiler_add_source finishes. + runtime.KeepAlive(c) + return err +} + +func (c *Compiler) NewNamespace(namespace string) { + cNamespace := C.CString(namespace) + defer C.free(unsafe.Pointer(cNamespace)) + result := C.yrx_compiler_new_namespace(c.cCompiler, cNamespace) + if result != C.SUCCESS { + panic("yrx_compiler_new_namespace failed") + } + runtime.KeepAlive(c) +} + +func (c *Compiler) DefineGlobalInt(ident string, value int64) error { + cIdent := C.CString(ident) + defer C.free(unsafe.Pointer(cIdent)) + var err error + if C.yrx_compiler_define_global_int(c.cCompiler, cIdent, C.int64_t(value)) == C.VARIABLE_ERROR { + err = errors.New(C.GoString(C.yrx_compiler_last_error(c.cCompiler))) + } + runtime.KeepAlive(c) + return err +} + +func (c *Compiler) DefineGlobalBool(ident string, value bool) error { + cIdent := C.CString(ident) + defer C.free(unsafe.Pointer(cIdent)) + var err error + if C.yrx_compiler_define_global_bool(c.cCompiler, cIdent, C.bool(value)) == C.VARIABLE_ERROR { + err = errors.New(C.GoString(C.yrx_compiler_last_error(c.cCompiler))) + } + runtime.KeepAlive(c) + return err +} + +func (c *Compiler) DefineGlobalStr(ident string, value string) error { + cIdent := C.CString(ident) + defer C.free(unsafe.Pointer(cIdent)) + cValue := C.CString(value) + defer C.free(unsafe.Pointer(cValue)) + var err error + if C.yrx_compiler_define_global_str(c.cCompiler, cIdent, cValue) == C.VARIABLE_ERROR { + err = errors.New(C.GoString(C.yrx_compiler_last_error(c.cCompiler))) + } + runtime.KeepAlive(c) + return err +} + +func (c *Compiler) Build() *Rules { + r := &Rules{cRules: C.yrx_compiler_build(c.cCompiler)} + runtime.SetFinalizer(r, (*Rules).Destroy) + runtime.KeepAlive(c) + return r +} + +func (c *Compiler) Destroy() { + if c.cCompiler != nil { + C.yrx_compiler_destroy(c.cCompiler) + c.cCompiler = nil + } + runtime.SetFinalizer(c, nil) +} \ No newline at end of file diff --git a/go/compiler_test.go b/go/compiler_test.go new file mode 100644 index 000000000..2d8ec0fbf --- /dev/null +++ b/go/compiler_test.go @@ -0,0 +1,54 @@ +package yara_x + +import ( + "github.com/stretchr/testify/assert" + "testing" +) + +func TestNamespaces(t *testing.T) { + c := NewCompiler() + c.NewNamespace("foo") + c.AddSource("rule test { condition: true }") + c.NewNamespace("bar") + c.AddSource("rule test { condition: true }") + + s := NewScanner(c.Build()) + matchingRules, _ := s.Scan([]byte{}) + + assert.Len(t, matchingRules, 2) +} + +func TestVariables(t *testing.T) { + c := NewCompiler() + + c.DefineGlobalInt("var", 1234) + c.AddSource("rule test { condition: var == 1234 }") + matchingRules, _ := NewScanner(c.Build()).Scan([]byte{}) + assert.Len(t, matchingRules, 1) + + c.DefineGlobalInt("var", -1234) + c.AddSource("rule test { condition: var == -1234 }") + matchingRules, _ = NewScanner(c.Build()).Scan([]byte{}) + assert.Len(t, matchingRules, 1) + + c.DefineGlobalBool("var", true) + c.AddSource("rule test { condition: var }") + matchingRules, _ = NewScanner(c.Build()).Scan([]byte{}) + assert.Len(t, matchingRules, 1) + + c.DefineGlobalBool("var", false) + c.AddSource("rule test { condition: var }") + matchingRules, _ = NewScanner(c.Build()).Scan([]byte{}) + assert.Len(t, matchingRules, 0) + + c.DefineGlobalStr("var", "foo") + c.AddSource("rule test { condition: var == \"foo\" }") + matchingRules, _ = NewScanner(c.Build()).Scan([]byte{}) + assert.Len(t, matchingRules, 1) +} + +func TestError(t *testing.T) { + c := NewCompiler() + err := c.AddSource("rule test { condition: foo }") + assert.Error(t, err) +} \ No newline at end of file diff --git a/go/main.go b/go/main.go index 5b3fe74e1..8437d5f2d 100644 --- a/go/main.go +++ b/go/main.go @@ -17,12 +17,12 @@ import ( // Compile receives YARA source code and returns compiled Rules that can be // used for scanning data. -func Compile(source string) *Rules { - s := C.CString(source) - r := &Rules{} - C.yrx_compile(s, &(r.cRules)) - runtime.SetFinalizer(r, (*Rules).Destroy) - return r +func Compile(source string) (*Rules, error) { + c := NewCompiler() + if err := c.AddSource(source); err != nil { + return nil, err + } + return c.Build(), nil } // Rules represents a set of compiled YARA rules. @@ -45,7 +45,7 @@ type Rule struct { namespace string identifier string cPatterns *C.YRX_PATTERNS - patterns []Pattern + patterns []Pattern } // Creates a new Rule from it's C counterpart. @@ -65,22 +65,19 @@ func newRule(cRule *C.YRX_RULE) *Rule { identifier := C.GoStringN((*C.char)(unsafe.Pointer(str)), C.int(len)) - cPatterns := C.yrx_rule_patterns(cRule) - - if cPatterns == nil { - panic("yrx_rule_patterns failed") + rule := &Rule{ + namespace, + identifier, + C.yrx_rule_patterns(cRule), + nil, } - rule := &Rule{namespace, identifier, cPatterns, nil} runtime.SetFinalizer(rule, (*Rule).destroy) return rule } func (r *Rule) destroy() { - if r.cPatterns != nil { - C.yrx_patterns_destroy(r.cPatterns) - r.cPatterns = nil - } + C.yrx_patterns_destroy(r.cPatterns) runtime.SetFinalizer(r, nil) } @@ -96,52 +93,33 @@ func (r *Rule) Namespace() string { // Patterns returns the patterns defined by this rule. func (r *Rule) Patterns() []Pattern { + // If this method was called before, return the patterns already cached. if r.patterns != nil { return r.patterns } - numPatterns := int(C.yrx_patterns_count(r.cPatterns)) + numPatterns := int(r.cPatterns.num_patterns) + cPatterns := unsafe.Slice(r.cPatterns.patterns, numPatterns) r.patterns = make([]Pattern, numPatterns) - for i := 0; i < numPatterns; i++ { - pattern := C.yrx_patterns_get(r.cPatterns, C.size_t(i)) - if pattern == nil { - panic("yrx_patterns_get failed") - } - - var str *C.uint8_t - var len C.size_t - - if C.yrx_pattern_identifier(pattern, &str, &len) != C.SUCCESS { - panic("yrx_pattern_identifier failed") - } + for i, pattern := range cPatterns { + numMatches := int(pattern.num_matches) + cMatches := unsafe.Slice(pattern.matches, numMatches) + matches := make([]Match, numMatches) - identifier := C.GoStringN((*C.char)(unsafe.Pointer(str)), C.int(len)) - - var ptrMatches *C.YRX_MATCH - var numMatches C.size_t - - if C.yrx_pattern_matches(pattern, &ptrMatches, &numMatches) != C.SUCCESS { - panic("yrx_pattern_matches failed") - } - - matches := make([]Match, int(numMatches)) - - for i := 0; i < int(numMatches); i++ { - match := *(*C.YRX_MATCH)(unsafe.Add( - unsafe.Pointer(ptrMatches), - i*C.sizeof_YRX_MATCH)) - matches[i] = Match{ + for j, match := range cMatches { + matches[j] = Match{ offset: uint(match.offset), length: uint(match.length), } } r.patterns[i] = Pattern{ - identifier: identifier, + identifier: C.GoString(pattern.identifier), matches: matches, } } + return r.patterns } diff --git a/go/scanner.go b/go/scanner.go index 25e28f7ed..c64430dd4 100644 --- a/go/scanner.go +++ b/go/scanner.go @@ -2,8 +2,11 @@ package yara_x import "C" import ( + "errors" + "math" "runtime" "runtime/cgo" + "time" "unsafe" ) @@ -14,7 +17,7 @@ import "C" // Scanner scans data with a set of compiled YARA rules. type Scanner struct { // Pointer to C-side scanner. - inner *C.YRX_SCANNER + cScanner *C.YRX_SCANNER // The Scanner holds a pointer to the Rules it uses in order to prevent // Rules from being garbage collected while the scanner is in use. If Rules // is garbage collected the associated C.YRX_RULES object is destroyed @@ -46,8 +49,9 @@ type ScanResults struct{} // same set of rules. func NewScanner(r *Rules) *Scanner { s := &Scanner{rules: r} - // TODO: handle error returned by yrx_scanner_create - C.yrx_scanner_create(r.cRules, &s.inner) + if C.yrx_scanner_create(r.cRules, &s.cScanner) != C.SUCCESS { + panic("yrx_scanner_create failed") + } // Allocate the memory that will hold the handle. This memory is allocated // using C.malloc because a pointer to it is passed to C code, and we don't @@ -59,7 +63,7 @@ func NewScanner(r *Rules) *Scanner { *s.handle = cgo.NewHandle(s) C.yrx_scanner_on_matching_rule( - s.inner, + s.cScanner, C.YRX_ON_MATCHING_RULE(C.onMatchingRule), unsafe.Pointer(s.handle)) @@ -67,8 +71,22 @@ func NewScanner(r *Rules) *Scanner { return s } +// Timeout sets a timeout for scan operations. +// +// The Scan method will return a timeout error once the provided timeout +// duration has elapsed. The scanner will make every effort to stop promptly +// after the designated timeout duration. However, in some cases, particularly +// with rules containing only a few patterns, the scanner could potentially +// continue running for a longer period than the specified timeout. +func (s *Scanner) Timeout(timeout time.Duration) { + C.yrx_scanner_timeout(s.cScanner, C.uint64_t(math.Ceil(timeout.Seconds()))) + runtime.KeepAlive(s) +} + +var ErrTimeout = errors.New("timeout") + // Scan scans the provided data with the Rules associated to the Scanner. -func (s *Scanner) Scan(buf []byte) []*Rule { +func (s *Scanner) Scan(buf []byte) ([]*Rule, error) { var ptr *C.uint8_t // When `buf` is an empty slice `ptr` will be nil. That's ok, because // yrx_scanner_scan allows the data pointer to be null as long as the data @@ -76,12 +94,20 @@ func (s *Scanner) Scan(buf []byte) []*Rule { if len(buf) > 0 { ptr = (*C.uint8_t)(unsafe.Pointer(&(buf[0]))) } + s.matchingRules = nil - // TODO: handle errors - C.yrx_scanner_scan(s.inner, ptr, C.size_t(len(buf))) - // Ensure that s is not finalized until yrx_scanner_scan returns. - runtime.KeepAlive(s) - return s.matchingRules + + var err error + switch r := C.yrx_scanner_scan(s.cScanner, ptr, C.size_t(len(buf))); r { + case C.SUCCESS: + err = nil + case C.SCAN_TIMEOUT: + err = ErrTimeout + default: + err = errors.New(C.GoString(C.yrx_scanner_last_error(s.cScanner))) + } + + return s.matchingRules, err } // Destroy destroys the scanner. @@ -89,11 +115,11 @@ func (s *Scanner) Scan(buf []byte) []*Rule { // Calling this method directly is not necessary, it will be invoked by the // garbage collector when the scanner is not used anymore. func (s *Scanner) Destroy() { - if s.inner != nil { - C.yrx_scanner_destroy(s.inner) + if s.cScanner != nil { + C.yrx_scanner_destroy(s.cScanner) s.handle.Delete() C.free(unsafe.Pointer(s.handle)) - s.inner = nil + s.cScanner = nil } runtime.SetFinalizer(s, nil) } @@ -109,7 +135,3 @@ func onMatchingRule(rule *C.YRX_RULE, handlePtr unsafe.Pointer) { } scanner.matchingRules = append(scanner.matchingRules, newRule(rule)) } - - - - diff --git a/go/scanner_test.go b/go/scanner_test.go index e2db8e8b9..dfe13b3a5 100644 --- a/go/scanner_test.go +++ b/go/scanner_test.go @@ -1,12 +1,17 @@ package yara_x -import "testing" +import ( + "bytes" + "runtime" + "testing" + "time" +) import "github.com/stretchr/testify/assert" - func TestScanner1(t *testing.T) { - s := NewScanner(Compile("rule t { condition: true }")) - matchingRules := s.Scan([]byte{}) + r, _ := Compile("rule t { condition: true }") + s := NewScanner(r) + matchingRules, _:= s.Scan([]byte{}) assert.Len(t, matchingRules, 1) assert.Equal(t, "t", matchingRules[0].Identifier()) @@ -15,8 +20,9 @@ func TestScanner1(t *testing.T) { } func TestScanner2(t *testing.T) { - s := NewScanner(Compile(`rule t { strings: $bar = "bar" condition: $bar }`)) - matchingRules := s.Scan([]byte("foobar")) + r, _ := Compile(`rule t { strings: $bar = "bar" condition: $bar }`) + s := NewScanner(r) + matchingRules, _ := s.Scan([]byte("foobar")) assert.Len(t, matchingRules, 1) assert.Equal(t, "t", matchingRules[0].Identifier()) @@ -26,4 +32,15 @@ func TestScanner2(t *testing.T) { assert.Equal(t, "$bar", matchingRules[0].Patterns()[0].Identifier()) assert.Equal(t, uint(3), matchingRules[0].Patterns()[0].Matches()[0].Offset()) assert.Equal(t, uint(3), matchingRules[0].Patterns()[0].Matches()[0].Length()) + + s.Destroy() + runtime.GC() +} + +func TestScannerTimeout(t *testing.T) { + r, _ := Compile("rule t { strings: $a = /a(.*)*a/ condition: $a }") + s := NewScanner(r) + s.Timeout(1*time.Second) + _, err := s.Scan(bytes.Repeat([]byte("a"), 9000)) + assert.ErrorIs(t, err, ErrTimeout) } \ No newline at end of file diff --git a/lib/src/compiler/errors.rs b/lib/src/compiler/errors.rs index 3820eddc0..08227cfee 100644 --- a/lib/src/compiler/errors.rs +++ b/lib/src/compiler/errors.rs @@ -3,6 +3,7 @@ use std::io; use thiserror::Error; +use crate::VariableError; use yara_x_macros::Error as DeriveError; use yara_x_parser::ast::Span; use yara_x_parser::report::ReportBuilder; @@ -29,16 +30,20 @@ pub enum SerializationError { pub struct EmitWasmError(#[from] anyhow::Error); /// Errors returned by the compiler. -#[derive(Error, Debug)] +#[derive(Error, Debug, Eq, PartialEq)] pub enum Error { #[error(transparent)] ParseError(#[from] ParseError), #[error(transparent)] CompileError(#[from] CompileError), + + #[error(transparent)] + VariableError(#[from] VariableError), } /// Error produced while compiling rules. +#[derive(Eq, PartialEq)] pub struct CompileError(Box); impl CompileError { @@ -69,7 +74,7 @@ impl From for CompileError { impl std::error::Error for CompileError {} /// An error occurred during the compilation process. -#[derive(DeriveError)] +#[derive(DeriveError, Eq, PartialEq)] #[non_exhaustive] pub enum CompileErrorInfo { #[error("wrong type")] diff --git a/lib/src/compiler/mod.rs b/lib/src/compiler/mod.rs index 362ecd449..3283d943d 100644 --- a/lib/src/compiler/mod.rs +++ b/lib/src/compiler/mod.rs @@ -370,19 +370,21 @@ impl<'a> Compiler<'a> { &mut self, ident: &str, value: T, - ) -> Result<&mut Self, VariableError> + ) -> Result<&mut Self, Error> where - VariableError: From<>::Error>, + Error: From<>::Error>, { if !is_valid_identifier(ident) { - return Err(VariableError::InvalidIdentifier(ident.to_string())); + return Err( + VariableError::InvalidIdentifier(ident.to_string()).into() + ); } let var: Variable = value.try_into()?; let type_value: TypeValue = var.into(); if self.root_struct.add_field(ident, type_value).is_some() { - return Err(VariableError::AlreadyExists(ident.to_string())); + return Err(VariableError::AlreadyExists(ident.to_string()).into()); } self.global_symbols diff --git a/lib/src/compiler/tests/mod.rs b/lib/src/compiler/tests/mod.rs index ec4a89a30..50c0a2fe1 100644 --- a/lib/src/compiler/tests/mod.rs +++ b/lib/src/compiler/tests/mod.rs @@ -6,7 +6,7 @@ use crate::compiler::{ SerializationError, SubPattern, Var, VarStack, VariableError, }; use crate::types::Type; -use crate::{compile, Compiler, Rules, Scanner}; +use crate::{compile, Compiler, Error, Rules, Scanner}; mod errors; mod warnings; @@ -123,7 +123,9 @@ fn globals() { assert_eq!( compiler.define_global("#invalid", true).err().unwrap(), - VariableError::InvalidIdentifier("#invalid".to_string()) + Error::VariableError(VariableError::InvalidIdentifier( + "#invalid".to_string() + )) ); let mut compiler = Compiler::new(); @@ -135,7 +137,7 @@ fn globals() { .define_global("a", false) .err() .unwrap(), - VariableError::AlreadyExists("a".to_string()) + Error::VariableError(VariableError::AlreadyExists("a".to_string())) ); let mut compiler = Compiler::new(); @@ -463,28 +465,28 @@ fn globals_json() { Compiler::new() .define_global("invalid_array", json!([1, "foo", 3])) .unwrap_err(), - VariableError::InvalidArray + Error::VariableError(VariableError::InvalidArray) ); assert_eq!( Compiler::new() .define_global("invalid_array", json!([1, [2, 3], 4])) .unwrap_err(), - VariableError::InvalidArray + Error::VariableError(VariableError::InvalidArray) ); assert_eq!( Compiler::new() .define_global("invalid_array", json!([1, null])) .unwrap_err(), - VariableError::InvalidArray + Error::VariableError(VariableError::InvalidArray) ); assert_eq!( Compiler::new() .define_global("invalid_array", json!({ "foo": null })) .unwrap_err(), - VariableError::UnexpectedNull + Error::VariableError(VariableError::UnexpectedNull) ); } diff --git a/lib/src/scanner/mod.rs b/lib/src/scanner/mod.rs index a5219ba1e..b94f37f43 100644 --- a/lib/src/scanner/mod.rs +++ b/lib/src/scanner/mod.rs @@ -250,8 +250,7 @@ impl<'r> Scanner<'r> { /// Sets a timeout for scan operations. /// /// The scan functions will return an [ScanError::Timeout] once the - /// provided timeout duration has elapsed. It's important to note that the - /// timeout might not be entirely precise, the scanner will make every + /// provided timeout duration has elapsed. The scanner will make every /// effort to stop promptly after the designated timeout duration. However, /// in some cases, particularly with rules containing only a few patterns, /// the scanner could potentially continue running for a longer period than diff --git a/lib/src/variables.rs b/lib/src/variables.rs index 4bcaf83c9..2bdc72ec7 100644 --- a/lib/src/variables.rs +++ b/lib/src/variables.rs @@ -23,7 +23,7 @@ use crate::types::{Array, TypeValue, Value}; pub struct Variable(TypeValue); /// Errors returned while defining or setting variables. -#[derive(Error, Debug, PartialEq)] +#[derive(Error, Debug, Eq, PartialEq)] pub enum VariableError { /// The variable has not being defined. Before calling /// [`crate::Scanner::set_global`] the variable must be defined with a diff --git a/parser/src/parser/errors.rs b/parser/src/parser/errors.rs index 78ad48b2c..c36353993 100644 --- a/parser/src/parser/errors.rs +++ b/parser/src/parser/errors.rs @@ -12,6 +12,7 @@ use crate::report::ReportType; /// relevant for that specific error. This information is usually contained /// inside the detailed report itself, but having access to the individual /// pieces is useful for applications that can't rely on text-based reports. +#[derive(Eq, PartialEq)] pub struct Error(Box); impl Error { @@ -37,7 +38,7 @@ impl std::error::Error for Error {} /// Additional information about an error occurred during parsing. #[rustfmt::skip] -#[derive(Err)] +#[derive(Err, Eq, PartialEq)] pub enum ErrorInfo { #[error("syntax error")] #[label("{error_msg}", error_span)] From 2736f238b3b8ac42235e1760ebecca0103df6eef Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Tue, 13 Feb 2024 17:08:59 +0100 Subject: [PATCH 34/55] tests: add some test cases for the C API. --- capi/include/yara-x.h | 4 ++++ capi/src/lib.rs | 3 +++ capi/src/scanner.rs | 14 +++++++++++--- capi/src/tests.rs | 44 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 62 insertions(+), 3 deletions(-) create mode 100644 capi/src/tests.rs diff --git a/capi/include/yara-x.h b/capi/include/yara-x.h index 025a8dcb9..42dd2360c 100644 --- a/capi/include/yara-x.h +++ b/capi/include/yara-x.h @@ -208,6 +208,10 @@ enum YRX_RESULT yrx_scanner_timeout(struct YRX_SCANNER *scanner, uint64_t timeout); // Scans a data buffer. +// +// `data` can be null as long as `len` is 0. In such cases its handled as +// empty data. Some YARA rules (i.e: `rule dummy { condition: true }`) can +// match even with empty data. enum YRX_RESULT yrx_scanner_scan(struct YRX_SCANNER *scanner, const uint8_t *data, size_t len); diff --git a/capi/src/lib.rs b/capi/src/lib.rs index 4ee8b4d4d..d044310bf 100644 --- a/capi/src/lib.rs +++ b/capi/src/lib.rs @@ -22,6 +22,9 @@ use std::ptr::slice_from_raw_parts_mut; mod compiler; mod scanner; +#[cfg(test)] +mod tests; + pub use scanner::*; #[repr(C)] diff --git a/capi/src/scanner.rs b/capi/src/scanner.rs index 6c3896a7d..a9004addd 100644 --- a/capi/src/scanner.rs +++ b/capi/src/scanner.rs @@ -3,7 +3,6 @@ use std::slice; use std::time::Duration; use yara_x::ScanError; -use crate::compiler::YRX_COMPILER; use crate::{YRX_RESULT, YRX_RULE, YRX_RULES}; /// A scanner that scans data with a set of compiled YARA rules. @@ -70,6 +69,10 @@ pub unsafe extern "C" fn yrx_scanner_timeout( } /// Scans a data buffer. +/// +/// `data` can be null as long as `len` is 0. In such cases its handled as +/// empty data. Some YARA rules (i.e: `rule dummy { condition: true }`) can +/// match even with empty data. #[no_mangle] pub unsafe extern "C" fn yrx_scanner_scan( scanner: *mut YRX_SCANNER, @@ -80,14 +83,19 @@ pub unsafe extern "C" fn yrx_scanner_scan( return YRX_RESULT::INVALID_ARGUMENT; } - // Data is allowed to be null as long as len is 0. This case is handled + // `data` is allowed to be null as long as `len` is 0. This case is handled // as an empty slice. if data.is_null() && len > 0 { return YRX_RESULT::INVALID_ARGUMENT; } + let data = if data.is_null() || len == 0 { + &[] + } else { + slice::from_raw_parts(data, len) + }; + let scanner = scanner.as_mut().unwrap(); - let data = slice::from_raw_parts(data, len); let scan_results = scanner.inner.scan(data); if let Err(err) = scan_results { diff --git a/capi/src/tests.rs b/capi/src/tests.rs new file mode 100644 index 000000000..8797e2d87 --- /dev/null +++ b/capi/src/tests.rs @@ -0,0 +1,44 @@ +use crate::compiler::{ + yrx_compiler_add_source, yrx_compiler_build, yrx_compiler_create, +}; +use crate::{ + yrx_scanner_create, yrx_scanner_on_matching_rule, yrx_scanner_scan, + YRX_RULE, +}; +use std::ffi::{c_void, CString}; + +extern "C" fn callback(_rule: *const YRX_RULE, user_data: *mut c_void) { + let ptr = user_data as *mut i32; + let matches = unsafe { ptr.as_mut().unwrap() }; + *matches += 1; +} + +#[test] +fn capi() { + unsafe { + let mut compiler = std::ptr::null_mut(); + yrx_compiler_create(&mut compiler); + + let src = + CString::new(b"rule test {condition: true}".to_vec()).unwrap(); + + yrx_compiler_add_source(compiler, src.as_ptr()); + + let rules = yrx_compiler_build(compiler); + + let mut scanner = std::ptr::null_mut(); + yrx_scanner_create(rules, &mut scanner); + + let mut matches = 0; + + yrx_scanner_on_matching_rule( + scanner, + callback, + &mut matches as *mut i32 as *mut c_void, + ); + + yrx_scanner_scan(scanner, std::ptr::null(), 0); + + assert_eq!(matches, 1); + } +} From 995ae4872d4304f1fb72940e1112e2bb5123bd41 Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Tue, 13 Feb 2024 17:44:43 +0100 Subject: [PATCH 35/55] chore: temporarily get the `authenticode-parser` from clone Get `authenticode-parser` from https://github.com/plusvic/rs-authenticode-parser until https://github.com/vthib/authenticode-parser/issues/13 is fixed. --- Cargo.lock | 6 ++---- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 08909a9ac..6a71ce065 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -201,8 +201,7 @@ dependencies = [ [[package]] name = "authenticode-parser" version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "604103d623672bbfcea96b1f20b75ee433012a14421fc87660ada85e74add9f9" +source = "git+https://github.com/plusvic/rs-authenticode-parser.git?rev=9be0b333#9be0b33347ae20dc4787c5b6e536c4ecfdd5362e" dependencies = [ "authenticode-parser-sys", ] @@ -210,8 +209,7 @@ dependencies = [ [[package]] name = "authenticode-parser-sys" version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "214a42da4790c46cf73660c83fe50bc1ecd4462362dfcf10f4a64de2776e2364" +source = "git+https://github.com/plusvic/rs-authenticode-parser.git?rev=9be0b333#9be0b33347ae20dc4787c5b6e536c4ecfdd5362e" dependencies = [ "cc", ] diff --git a/Cargo.toml b/Cargo.toml index 1ca76aa3c..21bfd416c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,7 +33,7 @@ anyhow = "1.0.75" ariadne = "0.3.0" array-bytes = "6.2.0" ascii_tree = "0.1.1" -authenticode-parser = "0.4.0" +authenticode-parser = { git = "https://github.com/plusvic/rs-authenticode-parser.git", rev= "9be0b333" } base64 = "0.21.0" bincode = "1.3.3" bitmask = "0.5.0" From 9b93c30d673afdaede54031454646600f139a24d Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Wed, 14 Feb 2024 09:59:28 +0100 Subject: [PATCH 36/55] tests: increase test case coverage for C API --- capi/src/tests.rs | 39 +++++++++++++++++++++++++++++++++++---- 1 file changed, 35 insertions(+), 4 deletions(-) diff --git a/capi/src/tests.rs b/capi/src/tests.rs index 8797e2d87..9240f044e 100644 --- a/capi/src/tests.rs +++ b/capi/src/tests.rs @@ -1,9 +1,12 @@ use crate::compiler::{ yrx_compiler_add_source, yrx_compiler_build, yrx_compiler_create, + yrx_compiler_define_global_bool, yrx_compiler_define_global_float, + yrx_compiler_define_global_int, yrx_compiler_define_global_str, + yrx_compiler_destroy, yrx_compiler_new_namespace, }; use crate::{ - yrx_scanner_create, yrx_scanner_on_matching_rule, yrx_scanner_scan, - YRX_RULE, + yrx_scanner_create, yrx_scanner_destroy, yrx_scanner_on_matching_rule, + yrx_scanner_scan, YRX_RULE, }; use std::ffi::{c_void, CString}; @@ -19,13 +22,40 @@ fn capi() { let mut compiler = std::ptr::null_mut(); yrx_compiler_create(&mut compiler); - let src = - CString::new(b"rule test {condition: true}".to_vec()).unwrap(); + let src = CString::new( + b"rule test {\ + condition: \ + some_bool and \ + some_str == \"some_str\" and \ + some_int == 1 and \ + some_float == 3.14 \ + }" + .to_vec(), + ) + .unwrap(); + let some_bool = CString::new(b"some_bool".to_vec()).unwrap(); + let some_str = CString::new(b"some_str".to_vec()).unwrap(); + let some_int = CString::new(b"some_int".to_vec()).unwrap(); + let some_float = CString::new(b"some_float".to_vec()).unwrap(); + + yrx_compiler_define_global_int(compiler, some_int.as_ptr(), 1); + yrx_compiler_define_global_float(compiler, some_float.as_ptr(), 3.14); + yrx_compiler_define_global_bool(compiler, some_bool.as_ptr(), true); + yrx_compiler_define_global_str( + compiler, + some_str.as_ptr(), + some_str.as_ptr(), + ); + + let namespace = CString::new(b"foo".to_vec()).unwrap(); + yrx_compiler_new_namespace(compiler, namespace.as_ptr()); yrx_compiler_add_source(compiler, src.as_ptr()); let rules = yrx_compiler_build(compiler); + yrx_compiler_destroy(compiler); + let mut scanner = std::ptr::null_mut(); yrx_scanner_create(rules, &mut scanner); @@ -38,6 +68,7 @@ fn capi() { ); yrx_scanner_scan(scanner, std::ptr::null(), 0); + yrx_scanner_destroy(scanner); assert_eq!(matches, 1); } From 4f19761991774c4acc5345388d8ec2c026450ebb Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Wed, 14 Feb 2024 10:13:12 +0100 Subject: [PATCH 37/55] style: fix clippy warning --- capi/src/tests.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/capi/src/tests.rs b/capi/src/tests.rs index 9240f044e..296295f7b 100644 --- a/capi/src/tests.rs +++ b/capi/src/tests.rs @@ -28,7 +28,7 @@ fn capi() { some_bool and \ some_str == \"some_str\" and \ some_int == 1 and \ - some_float == 3.14 \ + some_float == 1.5 \ }" .to_vec(), ) @@ -40,7 +40,7 @@ fn capi() { let some_float = CString::new(b"some_float".to_vec()).unwrap(); yrx_compiler_define_global_int(compiler, some_int.as_ptr(), 1); - yrx_compiler_define_global_float(compiler, some_float.as_ptr(), 3.14); + yrx_compiler_define_global_float(compiler, some_float.as_ptr(), 1.5); yrx_compiler_define_global_bool(compiler, some_bool.as_ptr(), true); yrx_compiler_define_global_str( compiler, From daf87a1f36259db2bcd199eb6f3266546d8aa435 Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Fri, 16 Feb 2024 10:59:58 +0100 Subject: [PATCH 38/55] chore: upgrade `authenticode-parser` to version 0.5.0 --- Cargo.lock | 10 ++++++---- Cargo.toml | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6a71ce065..acf92525d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -200,16 +200,18 @@ dependencies = [ [[package]] name = "authenticode-parser" -version = "0.4.0" -source = "git+https://github.com/plusvic/rs-authenticode-parser.git?rev=9be0b333#9be0b33347ae20dc4787c5b6e536c4ecfdd5362e" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b04c8a6168fa3d7733652b4604dbad1c0587b16382ad24b082d2de7ee7296087" dependencies = [ "authenticode-parser-sys", ] [[package]] name = "authenticode-parser-sys" -version = "0.4.0" -source = "git+https://github.com/plusvic/rs-authenticode-parser.git?rev=9be0b333#9be0b33347ae20dc4787c5b6e536c4ecfdd5362e" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6fec48978e90f3740da7ba30f501beb08d1729dabdcc685e163226b564f3737" dependencies = [ "cc", ] diff --git a/Cargo.toml b/Cargo.toml index 21bfd416c..e259236f3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,7 +33,7 @@ anyhow = "1.0.75" ariadne = "0.3.0" array-bytes = "6.2.0" ascii_tree = "0.1.1" -authenticode-parser = { git = "https://github.com/plusvic/rs-authenticode-parser.git", rev= "9be0b333" } +authenticode-parser = "0.5.0" base64 = "0.21.0" bincode = "1.3.3" bitmask = "0.5.0" From 0261a8f8532e3f7b7bbd9cf946903d843f88c6bd Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Fri, 16 Feb 2024 14:17:12 +0100 Subject: [PATCH 39/55] feat: add support for the `YRX_MOD_PROTO_DIRS` environment variable With this environment variable you can specify additional directories where to find `.proto` files with module definitions. This variable can contain a single directory path or a comma-separated list of directory path. The directories will be walked recursively looking for `.proto` files containing module definitions. Notice however that these modules are data-only, they can't contain functions because for adding functions to a YARA module the `.proto` file must be accompanied by Rust source files that implement the functions. --- lib/Cargo.toml | 1 + lib/build.rs | 57 +++++++++++++++++++++++++++++++++++++------------- 2 files changed, 44 insertions(+), 14 deletions(-) diff --git a/lib/Cargo.toml b/lib/Cargo.toml index 663ec85e3..6b5bad82e 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -190,6 +190,7 @@ yara-x-proto = { workspace = true } lingua = { version = "1.6.0", optional = true, default-features = false, features = ["english", "german", "french", "spanish"] } [build-dependencies] +globwalk = { workspace = true } protobuf = { workspace = true } protobuf-codegen = { workspace = true } protobuf-parse = { workspace = true } diff --git a/lib/build.rs b/lib/build.rs index 8a09de910..e196a7456 100644 --- a/lib/build.rs +++ b/lib/build.rs @@ -19,25 +19,54 @@ fn main() { proto_compiler .pure() .cargo_out_dir("protos") - //.out_dir("src/modules/protos") .include("../proto/src") .include("../proto-yaml/src") - .include("src/modules/protos") .input("../proto/src/yara.proto") .input("../proto-yaml/src/yaml.proto"); - proto_parser - .include("../proto/src") - .include("../proto-yaml/src") - .include("src/modules/protos"); - - for entry in fs::read_dir("src/modules/protos").unwrap() { - let entry = entry.unwrap(); - let path = entry.path(); - if let Some(extension) = path.extension() { - if extension == "proto" { - proto_compiler.input(&path); - proto_parser.input(&path); + proto_parser.include("../proto/src").include("../proto-yaml/src"); + + // These are the directories where we are going to search for the .proto + // files that contain module definitions. Initially, the only directory + // is the one included in this repository. But the YRX_MOD_PROTOS_DIRS + // environment variable specify additional directories. These directories + // are walked recursively looking for .proto files. + let mut proto_dirs = vec!["src/modules/protos".to_string()]; + + // The YRX_MOD_PROTOS_DIRS environment variable can contain a + // comma-separated list of directories with additional module definitions + // in `.proto` files. Notice however that these modules are data-only, + // they can't contain functions because for adding functions to a YARA + // module the `.proto` file must be accompanied by Rust source files that + // implement the functions. + if let Ok(var) = env::var("YRX_MOD_PROTO_DIRS") { + proto_dirs.extend( + var.split(',').map(|s| s.to_string()).collect::>(), + ); + } + + for dir in &proto_dirs { + let dir = fs::canonicalize(dir).unwrap(); + proto_compiler.include(&dir); + proto_parser.include(&dir); + + let walker = globwalk::GlobWalkerBuilder::from_patterns(dir, &["**"]) + .follow_links(true) + .build() + .unwrap(); + + for entry in walker { + let entry = entry.unwrap(); + let path = fs::canonicalize(entry.path()).unwrap(); + + if let Some(extension) = path.extension() { + if extension == "proto" { + proto_compiler.input(&path); + proto_parser.input(&path); + let dir = path.with_file_name(""); + proto_compiler.include(&dir); + proto_parser.include(&dir); + } } } } From 42cca1b74d4f93f0e25dc495b5c471b47bef462c Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Fri, 16 Feb 2024 14:35:22 +0100 Subject: [PATCH 40/55] chore: upgrade `goldenfile` and `chrono` crates --- Cargo.lock | 4 ++-- Cargo.toml | 2 ++ lib/Cargo.toml | 2 +- proto-yaml/Cargo.toml | 4 ++-- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index acf92525d..39234178c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -518,9 +518,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.33" +version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f13690e35a5e4ace198e7beea2895d29f3a9cc55015fcebe6336bd2010af9eb" +checksum = "5bc015644b92d5890fab7489e49d21f879d5c990186827d42ec511919404f38b" dependencies = [ "android-tzdata", "iana-time-zone", diff --git a/Cargo.toml b/Cargo.toml index e259236f3..ad24d80ac 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -39,6 +39,7 @@ bincode = "1.3.3" bitmask = "0.5.0" bitvec = "1.0.1" bstr = "1.8.0" +chrono = "0.4.34" clap = "4.4.8" crc32fast = "1.3.2" criterion = "0.5.1" @@ -46,6 +47,7 @@ enable-ansi-support = "0.2.1" env_logger = "0.11.1" fmmap = "0.3.2" globwalk = "0.9.0" +goldenfile = "1.6.0" indenter = "0.3.3" indexmap = "2.1.0" intaglio = "1.9.1" diff --git a/lib/Cargo.toml b/lib/Cargo.toml index 6b5bad82e..6a0556191 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -199,8 +199,8 @@ yara-x-proto = { workspace = true } [dev-dependencies] criterion = { workspace = true } globwalk = { workspace = true } +goldenfile = { workspace = true } ihex = "3.0.0" -goldenfile = "1.6.0" pretty_assertions = { workspace = true } wasmprinter = "0.2.73" yara = { version = "0.24.0", features = ["vendored"] } diff --git a/proto-yaml/Cargo.toml b/proto-yaml/Cargo.toml index b5e0dd3d2..4b5e108b2 100644 --- a/proto-yaml/Cargo.toml +++ b/proto-yaml/Cargo.toml @@ -12,15 +12,15 @@ license.workspace = true rust-version.workspace = true [dependencies] +chrono = { workspace = true } protobuf = { workspace = true } itertools = { workspace = true } yansi = { workspace = true } protobuf-support = "3.3.0" -chrono = "0.4.0" [dev-dependencies] -goldenfile = "1.5.2" +goldenfile = { workspace = true } globwalk = { workspace = true } [build-dependencies] From dc98dfbe7d4bbeba6a3c27a18b6e8d3d14fef5c7 Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Fri, 16 Feb 2024 19:04:11 +0100 Subject: [PATCH 41/55] refact: re-write `yara.proto` as proto3 This is for compatibility with an internal version of `yara.proto` that we have at VirusTotal. --- lib/build.rs | 10 +++++----- lib/src/types/structure.rs | 20 ++++++++++++++------ proto/src/yara.proto | 30 +++++++++++++++--------------- 3 files changed, 34 insertions(+), 26 deletions(-) diff --git a/lib/build.rs b/lib/build.rs index e196a7456..eeee7d653 100644 --- a/lib/build.rs +++ b/lib/build.rs @@ -89,7 +89,7 @@ fn main() { yara_module_options.get(&proto_file.options) { modules.push(( - module_options.name.unwrap(), + module_options.name, proto_file .name .unwrap() @@ -97,7 +97,7 @@ fn main() { .unwrap() .to_string(), module_options.rust_module, - module_options.root_message.unwrap(), + module_options.root_message, )); } } @@ -134,7 +134,7 @@ fn main() { let rust_mod = m.2; let root_message = m.3; - if let Some(rust_mod) = &rust_mod { + if !rust_mod.is_empty() { write!( modules_rs, r#" @@ -147,13 +147,13 @@ mod {rust_mod};"#, // If the YARA module has an associated Rust module, this module must // have a function named "main". If the YARA module doesn't have an // associated YARA module, the main function is set to None. - let main_fn = if let Some(rust_mod) = &rust_mod { + let main_fn = if !rust_mod.is_empty() { format!("Some({}::__main__ as MainFn)", rust_mod) } else { "None".to_string() }; - let rust_mod_name = if let Some(rust_mod) = &rust_mod { + let rust_mod_name = if !rust_mod.is_empty() { format!(r#"Some("{}")"#, rust_mod) } else { "None".to_string() diff --git a/lib/src/types/structure.rs b/lib/src/types/structure.rs index e16e123e1..f9fecd54c 100644 --- a/lib/src/types/structure.rs +++ b/lib/src/types/structure.rs @@ -412,7 +412,7 @@ impl Struct { if let Some(options) = module_options.get(&file_descriptor.proto().options) { - options.root_message.unwrap() == msg_descriptor.full_name() + options.root_message == msg_descriptor.full_name() } else { false } @@ -438,7 +438,11 @@ impl Struct { if let Some(options) = enum_options.get(&enum_descriptor.proto().options) { - options.name.unwrap_or_else(|| enum_descriptor.name().to_owned()) + if options.name.is_empty() { + enum_descriptor.name().to_owned() + } else { + options.name.clone() + } } else { enum_descriptor.name().to_owned() } @@ -480,7 +484,7 @@ impl Struct { if let Some(options) = enum_options.get(&enum_descriptor.proto().options) { - options.inline.unwrap_or(false) + options.inline } else { false } @@ -525,7 +529,7 @@ impl Struct { if let Some(options) = enum_value.get(&enum_value_descriptor.proto().options) { - options.i64.unwrap_or_else(|| enum_value_descriptor.value() as i64) + options.i64 } else { enum_value_descriptor.value() as i64 } @@ -548,7 +552,11 @@ impl Struct { if let Some(options) = field_options.get(&field_descriptor.proto().options) { - options.name.unwrap_or_else(|| field_descriptor.name().to_owned()) + if options.name.is_empty() { + field_descriptor.name().to_owned() + } else { + options.name.clone() + } } else { field_descriptor.name().to_owned() } @@ -567,7 +575,7 @@ impl Struct { if let Some(options) = field_options.get(&field_descriptor.proto().options) { - options.ignore.unwrap_or(false) + options.ignore } else { false } diff --git a/proto/src/yara.proto b/proto/src/yara.proto index f3620b896..669a2e300 100644 --- a/proto/src/yara.proto +++ b/proto/src/yara.proto @@ -1,52 +1,52 @@ // Protocol buffer that specifies the options that can be used in other protos // for controlling the generation of YARA modules. -syntax = "proto2"; +syntax = "proto3"; package yara; import "google/protobuf/descriptor.proto"; message ModuleOptions { - required string name = 1; - required string root_message = 2; - optional string rust_module = 3; + string name = 1; + string root_message = 2; + string rust_module = 3; } message FieldOptions { - optional string name = 1; - optional bool ignore = 2; + string name = 1; + bool ignore = 2; } message MessageOptions { - optional string name = 1; + string name = 1; } message EnumOptions { - optional string name = 1; - optional bool inline = 2; + string name = 1; + bool inline = 2; } message EnumValueOptions { - optional int64 i64 = 1; + int64 i64 = 1; } extend google.protobuf.FileOptions { - optional ModuleOptions module_options = 51503; + ModuleOptions module_options = 51503; } extend google.protobuf.FieldOptions { - optional FieldOptions field_options = 51504; + FieldOptions field_options = 51504; } extend google.protobuf.MessageOptions { - optional MessageOptions message_options = 51505; + MessageOptions message_options = 51505; } extend google.protobuf.EnumOptions { - optional EnumOptions enum_options = 51506; + EnumOptions enum_options = 51506; } extend google.protobuf.EnumValueOptions { - optional EnumValueOptions enum_value = 51507; + EnumValueOptions enum_value = 51507; } \ No newline at end of file From ba3f2f5050a74fdf7103a1e6ed669b252577dc03 Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Sun, 18 Feb 2024 12:07:23 +0100 Subject: [PATCH 42/55] Create codeql.yml --- .github/workflows/codeql.yml | 69 ++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 .github/workflows/codeql.yml diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 000000000..6c536c61d --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,69 @@ +name: "CodeQL" + +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] + schedule: + - cron: '36 18 * * 4' + +jobs: + analyze: + name: Analyze + # Runner size impacts CodeQL analysis time. To learn more, please see: + # - https://gh.io/recommended-hardware-resources-for-running-codeql + # - https://gh.io/supported-runners-and-hardware-resources + # - https://gh.io/using-larger-runners + # Consider using larger runners for possible analysis time improvements. + runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }} + timeout-minutes: ${{ (matrix.language == 'swift' && 120) || 360 }} + permissions: + # required for all workflows + security-events: write + + # only required for workflows in private repositories + actions: read + contents: read + + strategy: + fail-fast: false + matrix: + language: [ 'go', 'python' ] + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + + # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs + # queries: security-extended,security-and-quality + + + # Autobuild attempts to build any compiled languages (C/C++, C#, Go, Java, or Swift). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v3 + + # ℹ️ Command-line programs to run using the OS shell. + # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun + + # If the Autobuild fails above, remove it and uncomment the following three lines. + # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. + + # - run: | + # echo "Run, Build Application using script" + # ./location_of_script_within_repo/buildscript.sh + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v3 + with: + category: "/language:${{matrix.language}}" From 1d75a56732aa247b54d6ee77b16dbc9bd75e52c5 Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Sun, 18 Feb 2024 13:26:59 +0100 Subject: [PATCH 43/55] refactor: replace the `YRX_MOD_PROTO_DIRS` env variable with `YRX_PROTOC_CONFIG_FILE` Instead of having an environment variable that points to a directory with `.proto` files, now the environment variable points to a JSON file, that in turns contains the list of `.proto` files that should be included in the compilation. This way we can control which individual `.proto` files are included, in a directory that could contain other unrelated `.proto` files that don't need to be compiled. --- Cargo.lock | 38 ++++++++-------- Cargo.toml | 8 ++-- lib/Cargo.toml | 2 + lib/build.rs | 116 +++++++++++++++++++++++++++++++------------------ 4 files changed, 101 insertions(+), 63 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 39234178c..fec967866 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2536,20 +2536,18 @@ dependencies = [ [[package]] name = "protobuf" -version = "3.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b65f4a8ec18723a734e5dc09c173e0abf9690432da5340285d536edcb4dac190" +version = "4.0.0-alpha.0" +source = "git+https://github.com/plusvic/rust-protobuf.git?rev=b484d8a7#b484d8a7025db760c4fcf6994bda39f73452c6d7" dependencies = [ "once_cell", - "protobuf-support", + "protobuf-support 4.0.0-alpha.0", "thiserror", ] [[package]] name = "protobuf-codegen" -version = "3.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e85514a216b1c73111d9032e26cc7a5ecb1bb3d4d9539e91fb72a4395060f78" +version = "4.0.0-alpha.0" +source = "git+https://github.com/plusvic/rust-protobuf.git?rev=b484d8a7#b484d8a7025db760c4fcf6994bda39f73452c6d7" dependencies = [ "anyhow", "once_cell", @@ -2562,26 +2560,24 @@ dependencies = [ [[package]] name = "protobuf-json-mapping" -version = "3.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "523039a90666b229b5260fb91c20686ef309b9d1b1fc3cacb283a0895753ec44" +version = "4.0.0-alpha.0" +source = "git+https://github.com/plusvic/rust-protobuf.git?rev=b484d8a7#b484d8a7025db760c4fcf6994bda39f73452c6d7" dependencies = [ "protobuf", - "protobuf-support", + "protobuf-support 4.0.0-alpha.0", "thiserror", ] [[package]] name = "protobuf-parse" -version = "3.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77d6fbd6697c9e531873e81cec565a85e226b99a0f10e1acc079be057fe2fcba" +version = "4.0.0-alpha.0" +source = "git+https://github.com/plusvic/rust-protobuf.git?rev=b484d8a7#b484d8a7025db760c4fcf6994bda39f73452c6d7" dependencies = [ "anyhow", - "indexmap 1.9.3", + "indexmap 2.2.2", "log", "protobuf", - "protobuf-support", + "protobuf-support 4.0.0-alpha.0", "tempfile", "thiserror", "which", @@ -2596,6 +2592,14 @@ dependencies = [ "thiserror", ] +[[package]] +name = "protobuf-support" +version = "4.0.0-alpha.0" +source = "git+https://github.com/plusvic/rust-protobuf.git?rev=b484d8a7#b484d8a7025db760c4fcf6994bda39f73452c6d7" +dependencies = [ + "thiserror", +] + [[package]] name = "psm" version = "0.1.21" @@ -4518,7 +4522,7 @@ dependencies = [ "protobuf", "protobuf-codegen", "protobuf-parse", - "protobuf-support", + "protobuf-support 3.3.0", "yansi 0.5.1", ] diff --git a/Cargo.toml b/Cargo.toml index ad24d80ac..886e77ecd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -70,10 +70,10 @@ num-derive = "0.4.1" pest = "2.7.5" pest_derive = "2.7.5" pretty_assertions = "1.4.0" -protobuf = "3.3.0" -protobuf-codegen = "3.3.0" -protobuf-json-mapping = "3.3.0" -protobuf-parse = "3.3.0" +protobuf = { git = "https://github.com/plusvic/rust-protobuf.git", rev="b484d8a7" } +protobuf-codegen = { git = "https://github.com/plusvic/rust-protobuf.git", rev="b484d8a7" } +protobuf-json-mapping = { git = "https://github.com/plusvic/rust-protobuf.git", rev="b484d8a7" } +protobuf-parse = { git = "https://github.com/plusvic/rust-protobuf.git", rev="b484d8a7" } regex-syntax = { git = "https://github.com/plusvic/regex.git", rev="423493d" } regex-automata = { git = "https://github.com/plusvic/regex.git", rev="423493d" } rustc-hash = "1.1.0" diff --git a/lib/Cargo.toml b/lib/Cargo.toml index 6a0556191..22e081be2 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -194,6 +194,8 @@ globwalk = { workspace = true } protobuf = { workspace = true } protobuf-codegen = { workspace = true } protobuf-parse = { workspace = true } +serde = { workspace = true, features=["rc"] } +serde_json = { workspace = true } yara-x-proto = { workspace = true } [dev-dependencies] diff --git a/lib/build.rs b/lib/build.rs index eeee7d653..cabd1f909 100644 --- a/lib/build.rs +++ b/lib/build.rs @@ -1,12 +1,21 @@ use std::fs::File; -use std::io::Write; +use std::io::{BufReader, Write}; use std::path::Path; use std::{env, fs}; use protobuf_codegen::Codegen; use protobuf_parse::Parser; +use serde::{Deserialize, Serialize}; +use serde_json; + use yara_x_proto::exts::module_options as yara_module_options; +#[derive(Serialize, Deserialize)] +struct ProtocConfig { + includes: Vec, + inputs: Vec, +} + fn main() { println!("cargo:rerun-if-changed=src/modules"); println!("cargo:rerun-if-changed=src/modules/protos"); @@ -21,53 +30,76 @@ fn main() { .cargo_out_dir("protos") .include("../proto/src") .include("../proto-yaml/src") + .include("src/modules/protos") .input("../proto/src/yara.proto") .input("../proto-yaml/src/yaml.proto"); - proto_parser.include("../proto/src").include("../proto-yaml/src"); - - // These are the directories where we are going to search for the .proto - // files that contain module definitions. Initially, the only directory - // is the one included in this repository. But the YRX_MOD_PROTOS_DIRS - // environment variable specify additional directories. These directories - // are walked recursively looking for .proto files. - let mut proto_dirs = vec!["src/modules/protos".to_string()]; - - // The YRX_MOD_PROTOS_DIRS environment variable can contain a - // comma-separated list of directories with additional module definitions - // in `.proto` files. Notice however that these modules are data-only, - // they can't contain functions because for adding functions to a YARA - // module the `.proto` file must be accompanied by Rust source files that - // implement the functions. - if let Ok(var) = env::var("YRX_MOD_PROTO_DIRS") { - proto_dirs.extend( - var.split(',').map(|s| s.to_string()).collect::>(), - ); + proto_parser + .include("../proto/src") + .include("../proto-yaml/src") + .include("src/modules/protos"); + + // All `.proto` files in src/modules/protos must be compiled + for entry in fs::read_dir("src/modules/protos").unwrap() { + let entry = entry.unwrap(); + let path = entry.path(); + if let Some(extension) = path.extension() { + if extension == "proto" { + proto_compiler.input(&path); + proto_parser.input(&path); + } + } } - for dir in &proto_dirs { - let dir = fs::canonicalize(dir).unwrap(); - proto_compiler.include(&dir); - proto_parser.include(&dir); - - let walker = globwalk::GlobWalkerBuilder::from_patterns(dir, &["**"]) - .follow_links(true) - .build() - .unwrap(); + // If the YRX_PROTOC_CONFIG_FILE environment variable is set, it must + // contain the path to a JSON file that indicates additional include + // directories and input files passed to the `protoc` compiler. The path + // to the JSON file must be either an absolute path or a path relative to + // this `build.rs` file. + // + // The JSON file must be similar to this: + // + // { + // "includes": [ + // "../../vt-protos/protos/tools", + // "../../vt-protos/protos" + // ], + // "inputs": [ + // "../../vt-protos/protos/titan.proto", + // "../../vt-protos/protos/filetypes.proto", + // "../../vt-protos/protos/sandbox.proto", + // "../../vt-protos/protos/vtnet.proto", + // "../../vt-protos/protos/submitter.proto", + // "../../vt-protos/protos/analysis.proto", + // "../../vt-protos/protos/tools/net_analysis.proto", + // "../../vt-protos/protos/tools/snort.proto", + // "../../vt-protos/protos/tools/suricata.proto", + // "../../vt-protos/protos/tools/tshark.proto", + // "../../vt-protos/protos/sigma.proto", + // "../../vt-protos/protos/relationships.proto" + // ] + // } + // + // Paths in the "includes" and "inputs" lists must also be absolute or + // relative to this `build.rs` file. + if let Ok(path) = env::var("YRX_PROTOC_CONFIG_FILE") { + let file = File::open(path.as_str()) + .expect(&*format!("error opening {}", path)); + + let reader = BufReader::new(file); + let config: ProtocConfig = serde_json::from_reader(reader) + .expect(&*format!("invalid config file {}", path)); + + for path in config.includes { + let path = fs::canonicalize(path).unwrap(); + proto_compiler.include(&path); + proto_parser.include(&path); + } - for entry in walker { - let entry = entry.unwrap(); - let path = fs::canonicalize(entry.path()).unwrap(); - - if let Some(extension) = path.extension() { - if extension == "proto" { - proto_compiler.input(&path); - proto_parser.input(&path); - let dir = path.with_file_name(""); - proto_compiler.include(&dir); - proto_parser.include(&dir); - } - } + for path in config.inputs { + let path = fs::canonicalize(path).unwrap(); + proto_compiler.input(&path); + proto_parser.input(&path); } } From a4b12d8b9be2a67b6b17b55c72bdaf12ac26ee62 Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Sun, 18 Feb 2024 14:48:17 +0100 Subject: [PATCH 44/55] feat: control the name of the feature associated to each YARA module from the protobuf defining the module Now each YARA module can define the name of the cargo feature that enables/disables the module. This is done using the new `cargo_feature` option, like in: ``` option (yara.module_options) = { name : "text" root_message: "text.Text" rust_module: "text" cargo_feature: "text-module" }; ``` The `cargo_feature` option is not required. If missing the module will always be enabled. --- docs/Module Developer's Guide.md | 31 +++++++++++------- lib/build.rs | 40 ++++++++++++++---------- lib/src/modules/protos/console.proto | 1 + lib/src/modules/protos/dotnet.proto | 1 + lib/src/modules/protos/elf.proto | 1 + lib/src/modules/protos/hash.proto | 1 + lib/src/modules/protos/lnk.proto | 1 + lib/src/modules/protos/macho.proto | 1 + lib/src/modules/protos/math.proto | 1 + lib/src/modules/protos/pe.proto | 1 + lib/src/modules/protos/string.proto | 1 + lib/src/modules/protos/test_proto2.proto | 4 +++ lib/src/modules/protos/test_proto3.proto | 4 +++ lib/src/modules/protos/text.proto | 3 ++ lib/src/modules/protos/time.proto | 1 + lib/src/types/structure.rs | 20 ++++-------- mod.cfg | 20 ++++++++++++ proto/src/yara.proto | 31 +++++++++--------- 18 files changed, 107 insertions(+), 56 deletions(-) create mode 100644 mod.cfg diff --git a/docs/Module Developer's Guide.md b/docs/Module Developer's Guide.md index 663df2b2e..881f70d9b 100644 --- a/docs/Module Developer's Guide.md +++ b/docs/Module Developer's Guide.md @@ -12,7 +12,6 @@ For illustrative purposes we are going to create a `text` module that allows creating YARA rules for plain-text files, based on the number of lines and words -- [Module Developer's Guide](#module-developers-guide) - [Defining the module's structure](#defining-the-modules-structure) - [Proto2 vs Proto3](#proto2-vs-proto3) - [Tweaking the module's YAML output](#tweaking-the-modules-yaml-output) @@ -65,6 +64,7 @@ option (yara.module_options) = { name : "text" root_message: "text.Text" rust_module: "text" + cargo_feature: "text-module" }; message Text { @@ -121,6 +121,7 @@ option (yara.module_options) = { name : "text" root_message: "text.Text" rust_module: "text" + cargo_feature: "text-module" }; ``` @@ -131,16 +132,24 @@ file, but one describing a module. In fact, you can put any `.proto` file in the files is describing a YARA module. Only files containing a `yara.module_options` section will define a module. -Options `name` and `root_message` are required, while `rust_module` is optional. -The `name` option defines the module's name. This is the name that will be used -for importing the module in a YARA rule, in this case our module will be imported -with `import "text"`. The `root_message` option indicates which is the module's -root structure, it must contain the name of some structure (a.k.a message) defined -in the `.proto` file. In our case the value for `root_message` is `"text.Text"` -because we have defined our module's structure in a message named `Text`, which -is under package `text`. In general the value in this field will have the form -`package.Message`, except if the `package` statement is missing, in which case -it would be the name of the message alone (i.e: `Text`). +Options `name` and `root_message` are required, while `rust_module` and +`cargo_feature` are optional. The `name` option defines the module's name. This +is the name that will be used for importing the module in a YARA rule, in this +case our module will be imported with `import "text"`. + +The `cargo_feature` option indicates the name of the feature that controls whether +the module is built or not. If this option is not specified the module is always +built, but if you specify a feature name, this feature name must also be included +in the `Cargo.toml` file, and the module will be built only when this `cargo` +feature is enabled. + +The `root_message` option a very important option indicating which is the module's +root structure, it must contain the name of some structure (a.k.a. message) +defined in the `.proto` file. In our case the value for `root_message` is +`"text.Text"` because we have defined our module's structure in a message named +`Text`, which is under package `text`. In general the value in this field will +have the form `package.Message`, except if the `package` statement is missing, +in which case it would be the name of the message alone (i.e: `Text`). The `root_message` field is required because your `.proto` file can define multiple messages, and YARA needs to know which of them is considered the root diff --git a/lib/build.rs b/lib/build.rs index cabd1f909..32e4c3857 100644 --- a/lib/build.rs +++ b/lib/build.rs @@ -121,7 +121,7 @@ fn main() { yara_module_options.get(&proto_file.options) { modules.push(( - module_options.name, + module_options.name.unwrap(), proto_file .name .unwrap() @@ -129,7 +129,8 @@ fn main() { .unwrap() .to_string(), module_options.rust_module, - module_options.root_message, + module_options.cargo_feature, + module_options.root_message.unwrap(), )); } } @@ -164,37 +165,44 @@ fn main() { let name = m.0; let proto_mod = m.1; let rust_mod = m.2; - let root_message = m.3; - - if !rust_mod.is_empty() { - write!( - modules_rs, - r#" -#[cfg(feature = "{name}-module")] -mod {rust_mod};"#, - ) - .unwrap(); - } + let cargo_feature = m.3; + let root_message = m.4; // If the YARA module has an associated Rust module, this module must // have a function named "main". If the YARA module doesn't have an // associated YARA module, the main function is set to None. - let main_fn = if !rust_mod.is_empty() { + let main_fn = if let Some(rust_mod) = &rust_mod { format!("Some({}::__main__ as MainFn)", rust_mod) } else { "None".to_string() }; - let rust_mod_name = if !rust_mod.is_empty() { + let rust_mod_name = if let Some(rust_mod) = &rust_mod { format!(r#"Some("{}")"#, rust_mod) } else { "None".to_string() }; + let cfg_feature = if let Some(cargo_feature) = &cargo_feature { + format!(r#"#[cfg(feature = "{cargo_feature}")]"#) + } else { + "".to_string() + }; + + if let Some(rust_mod) = &rust_mod { + write!( + modules_rs, + r#" +{cfg_feature} +mod {rust_mod};"#, + ) + .unwrap(); + } + write!( add_modules_rs, r#" -#[cfg(feature = "{name}-module")] +{cfg_feature} add_module!(modules, "{name}", {proto_mod}, "{root_message}", {rust_mod_name}, {main_fn});"#, ) .unwrap(); diff --git a/lib/src/modules/protos/console.proto b/lib/src/modules/protos/console.proto index 05ce59fc7..40d615584 100644 --- a/lib/src/modules/protos/console.proto +++ b/lib/src/modules/protos/console.proto @@ -7,6 +7,7 @@ option (yara.module_options) = { name : "console" root_message: "console.Console" rust_module: "console" + cargo_feature: "console-module" }; message Console { diff --git a/lib/src/modules/protos/dotnet.proto b/lib/src/modules/protos/dotnet.proto index bb5239e2d..6eb817ce2 100644 --- a/lib/src/modules/protos/dotnet.proto +++ b/lib/src/modules/protos/dotnet.proto @@ -7,6 +7,7 @@ option (yara.module_options) = { name : "dotnet" root_message: "dotnet.Dotnet" rust_module: "dotnet" + cargo_feature: "dotnet-module" }; message Dotnet { diff --git a/lib/src/modules/protos/elf.proto b/lib/src/modules/protos/elf.proto index b99e5bb27..006f88cb2 100644 --- a/lib/src/modules/protos/elf.proto +++ b/lib/src/modules/protos/elf.proto @@ -7,6 +7,7 @@ option (yara.module_options) = { name : "elf" root_message: "elf.ELF" rust_module: "elf" + cargo_feature: "elf-module" }; message ELF { diff --git a/lib/src/modules/protos/hash.proto b/lib/src/modules/protos/hash.proto index 488f2a871..888b435a7 100644 --- a/lib/src/modules/protos/hash.proto +++ b/lib/src/modules/protos/hash.proto @@ -7,6 +7,7 @@ option (yara.module_options) = { name : "hash" root_message: "hash.Hash" rust_module: "hash" + cargo_feature: "hash-module" }; message Hash { diff --git a/lib/src/modules/protos/lnk.proto b/lib/src/modules/protos/lnk.proto index 049d2f149..df4305bff 100644 --- a/lib/src/modules/protos/lnk.proto +++ b/lib/src/modules/protos/lnk.proto @@ -7,6 +7,7 @@ option (yara.module_options) = { name : "lnk" root_message: "lnk.Lnk" rust_module: "lnk" + cargo_feature: "lnk-module" }; enum FileAttributes { diff --git a/lib/src/modules/protos/macho.proto b/lib/src/modules/protos/macho.proto index 8dfd00475..886784349 100644 --- a/lib/src/modules/protos/macho.proto +++ b/lib/src/modules/protos/macho.proto @@ -8,6 +8,7 @@ option (yara.module_options) = { name : "macho" root_message: "macho.Macho" rust_module: "macho" + cargo_feature: "macho-module" }; message Dylib { diff --git a/lib/src/modules/protos/math.proto b/lib/src/modules/protos/math.proto index edd5ed3bb..7a74645da 100644 --- a/lib/src/modules/protos/math.proto +++ b/lib/src/modules/protos/math.proto @@ -7,6 +7,7 @@ option (yara.module_options) = { name : "math" root_message: "math.Math" rust_module: "math" + cargo_feature: "math-module" }; message Math { diff --git a/lib/src/modules/protos/pe.proto b/lib/src/modules/protos/pe.proto index 37204d27b..2da3b5e58 100644 --- a/lib/src/modules/protos/pe.proto +++ b/lib/src/modules/protos/pe.proto @@ -9,6 +9,7 @@ option (yara.module_options) = { name : "pe" root_message: "pe.PE" rust_module: "pe" + cargo_feature: "pe-module" }; message PE { diff --git a/lib/src/modules/protos/string.proto b/lib/src/modules/protos/string.proto index 4d5a34b3a..286b44410 100644 --- a/lib/src/modules/protos/string.proto +++ b/lib/src/modules/protos/string.proto @@ -7,6 +7,7 @@ option (yara.module_options) = { name : "string" root_message: "string.String" rust_module: "string" + cargo_feature: "string-module" }; message String { diff --git a/lib/src/modules/protos/test_proto2.proto b/lib/src/modules/protos/test_proto2.proto index 668970bd2..434daaf98 100644 --- a/lib/src/modules/protos/test_proto2.proto +++ b/lib/src/modules/protos/test_proto2.proto @@ -45,6 +45,10 @@ option (yara.module_options) = { // in the data structure defined by this proto file, and don't need to have // any associated code. rust_module: "test_proto2" + + // The name of the feature that controls whether this module is compiled or + // not. A feature with this name must be added to the Cargo.toml file. + cargo_feature: "test_proto2-module" }; /// Top-level structure for this module. diff --git a/lib/src/modules/protos/test_proto3.proto b/lib/src/modules/protos/test_proto3.proto index d6235e3d5..1163073c6 100644 --- a/lib/src/modules/protos/test_proto3.proto +++ b/lib/src/modules/protos/test_proto3.proto @@ -45,6 +45,10 @@ option (yara.module_options) = { // in the data structure defined by this proto file, and don't need to have // any associated code. rust_module: "test_proto3" + + // The name of the feature that controls whether this module is compiled or + // not. A feature with this name must be added to the Cargo.toml file. + cargo_feature: "test_proto3-module" }; /// Top-level structure for this module. diff --git a/lib/src/modules/protos/text.proto b/lib/src/modules/protos/text.proto index e66eaf29e..b0e38fc48 100644 --- a/lib/src/modules/protos/text.proto +++ b/lib/src/modules/protos/text.proto @@ -15,6 +15,9 @@ option (yara.module_options) = { // The Rust module implementing this YARA module is named `text`. It can // be found in `src/modules/text.rs`. rust_module: "text" + // The feature that controls whether this module is compiled or not is named + // `text-module`. + cargo_feature: "text-module" }; // This is the module's root structure. diff --git a/lib/src/modules/protos/time.proto b/lib/src/modules/protos/time.proto index 0056c55fe..79351d4cd 100644 --- a/lib/src/modules/protos/time.proto +++ b/lib/src/modules/protos/time.proto @@ -7,6 +7,7 @@ option (yara.module_options) = { name : "time" root_message: "time.Time" rust_module: "time" + cargo_feature: "time-module" }; message Time { diff --git a/lib/src/types/structure.rs b/lib/src/types/structure.rs index f9fecd54c..e16e123e1 100644 --- a/lib/src/types/structure.rs +++ b/lib/src/types/structure.rs @@ -412,7 +412,7 @@ impl Struct { if let Some(options) = module_options.get(&file_descriptor.proto().options) { - options.root_message == msg_descriptor.full_name() + options.root_message.unwrap() == msg_descriptor.full_name() } else { false } @@ -438,11 +438,7 @@ impl Struct { if let Some(options) = enum_options.get(&enum_descriptor.proto().options) { - if options.name.is_empty() { - enum_descriptor.name().to_owned() - } else { - options.name.clone() - } + options.name.unwrap_or_else(|| enum_descriptor.name().to_owned()) } else { enum_descriptor.name().to_owned() } @@ -484,7 +480,7 @@ impl Struct { if let Some(options) = enum_options.get(&enum_descriptor.proto().options) { - options.inline + options.inline.unwrap_or(false) } else { false } @@ -529,7 +525,7 @@ impl Struct { if let Some(options) = enum_value.get(&enum_value_descriptor.proto().options) { - options.i64 + options.i64.unwrap_or_else(|| enum_value_descriptor.value() as i64) } else { enum_value_descriptor.value() as i64 } @@ -552,11 +548,7 @@ impl Struct { if let Some(options) = field_options.get(&field_descriptor.proto().options) { - if options.name.is_empty() { - field_descriptor.name().to_owned() - } else { - options.name.clone() - } + options.name.unwrap_or_else(|| field_descriptor.name().to_owned()) } else { field_descriptor.name().to_owned() } @@ -575,7 +567,7 @@ impl Struct { if let Some(options) = field_options.get(&field_descriptor.proto().options) { - options.ignore + options.ignore.unwrap_or(false) } else { false } diff --git a/mod.cfg b/mod.cfg new file mode 100644 index 000000000..1f1e27969 --- /dev/null +++ b/mod.cfg @@ -0,0 +1,20 @@ +{ + "includes": [ + "../../atlas/vt-protos/protos/tools", + "../../atlas/vt-protos/protos" + ], + "inputs": [ + "../../atlas/vt-protos/protos/titan.proto", + "../../atlas/vt-protos/protos/filetypes.proto", + "../../atlas/vt-protos/protos/sandbox.proto", + "../../atlas/vt-protos/protos/vtnet.proto", + "../../atlas/vt-protos/protos/submitter.proto", + "../../atlas/vt-protos/protos/analysis.proto", + "../../atlas/vt-protos/protos/tools/net_analysis.proto", + "../../atlas/vt-protos/protos/tools/snort.proto", + "../../atlas/vt-protos/protos/tools/suricata.proto", + "../../atlas/vt-protos/protos/tools/tshark.proto", + "../../atlas/vt-protos/protos/sigma.proto", + "../../atlas/vt-protos/protos/relationships.proto" + ] +} \ No newline at end of file diff --git a/proto/src/yara.proto b/proto/src/yara.proto index 669a2e300..dbf731326 100644 --- a/proto/src/yara.proto +++ b/proto/src/yara.proto @@ -1,52 +1,53 @@ // Protocol buffer that specifies the options that can be used in other protos // for controlling the generation of YARA modules. -syntax = "proto3"; +syntax = "proto2"; package yara; import "google/protobuf/descriptor.proto"; message ModuleOptions { - string name = 1; - string root_message = 2; - string rust_module = 3; + required string name = 1; + required string root_message = 2; + optional string rust_module = 3; + optional string cargo_feature = 4; } message FieldOptions { - string name = 1; - bool ignore = 2; + optional string name = 1; + optional bool ignore = 2; } message MessageOptions { - string name = 1; + optional string name = 1; } message EnumOptions { - string name = 1; - bool inline = 2; + optional string name = 1; + optional bool inline = 2; } message EnumValueOptions { - int64 i64 = 1; + optional int64 i64 = 1; } extend google.protobuf.FileOptions { - ModuleOptions module_options = 51503; + optional ModuleOptions module_options = 51503; } extend google.protobuf.FieldOptions { - FieldOptions field_options = 51504; + optional FieldOptions field_options = 51504; } extend google.protobuf.MessageOptions { - MessageOptions message_options = 51505; + optional MessageOptions message_options = 51505; } extend google.protobuf.EnumOptions { - EnumOptions enum_options = 51506; + optional EnumOptions enum_options = 51506; } extend google.protobuf.EnumValueOptions { - EnumValueOptions enum_value = 51507; + optional EnumValueOptions enum_value = 51507; } \ No newline at end of file From c3a5e084f6feeeb6707b798ff8889d42a33f32f2 Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Sun, 18 Feb 2024 14:59:27 +0100 Subject: [PATCH 45/55] style: fix clippy warnings --- lib/build.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/build.rs b/lib/build.rs index 32e4c3857..5273a5303 100644 --- a/lib/build.rs +++ b/lib/build.rs @@ -84,11 +84,11 @@ fn main() { // relative to this `build.rs` file. if let Ok(path) = env::var("YRX_PROTOC_CONFIG_FILE") { let file = File::open(path.as_str()) - .expect(&*format!("error opening {}", path)); + .expect(*format!("error opening {}", path)); let reader = BufReader::new(file); let config: ProtocConfig = serde_json::from_reader(reader) - .expect(&*format!("invalid config file {}", path)); + .expect(*format!("invalid config file {}", path)); for path in config.includes { let path = fs::canonicalize(path).unwrap(); From ef05df26671bb396da54d076981dc4504f25c352 Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Sun, 18 Feb 2024 15:00:01 +0100 Subject: [PATCH 46/55] style: fix clippy warning --- lib/build.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/build.rs b/lib/build.rs index 5273a5303..365a79340 100644 --- a/lib/build.rs +++ b/lib/build.rs @@ -6,7 +6,6 @@ use std::{env, fs}; use protobuf_codegen::Codegen; use protobuf_parse::Parser; use serde::{Deserialize, Serialize}; -use serde_json; use yara_x_proto::exts::module_options as yara_module_options; From d3356138af8e5324369bf0820652e6b4698fc0b8 Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Sun, 18 Feb 2024 17:10:33 +0100 Subject: [PATCH 47/55] chore: fix compilation --- lib/build.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/build.rs b/lib/build.rs index 365a79340..264f081a6 100644 --- a/lib/build.rs +++ b/lib/build.rs @@ -83,11 +83,11 @@ fn main() { // relative to this `build.rs` file. if let Ok(path) = env::var("YRX_PROTOC_CONFIG_FILE") { let file = File::open(path.as_str()) - .expect(*format!("error opening {}", path)); + .expect(&format!("error opening {}", path)); let reader = BufReader::new(file); let config: ProtocConfig = serde_json::from_reader(reader) - .expect(*format!("invalid config file {}", path)); + .expect(&format!("invalid config file {}", path)); for path in config.includes { let path = fs::canonicalize(path).unwrap(); From 414b25c24f4e935928f5b7c72c2ca8069c9d0b0b Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Sun, 18 Feb 2024 18:40:17 +0100 Subject: [PATCH 48/55] style: fix more clippy warnings --- lib/build.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/build.rs b/lib/build.rs index 264f081a6..ab656a21d 100644 --- a/lib/build.rs +++ b/lib/build.rs @@ -83,11 +83,11 @@ fn main() { // relative to this `build.rs` file. if let Ok(path) = env::var("YRX_PROTOC_CONFIG_FILE") { let file = File::open(path.as_str()) - .expect(&format!("error opening {}", path)); + .unwrap_or_else(|_| panic!("error opening {}", path)); let reader = BufReader::new(file); let config: ProtocConfig = serde_json::from_reader(reader) - .expect(&format!("invalid config file {}", path)); + .unwrap_or_else(|_| panic!("invalid config file {}", path)); for path in config.includes { let path = fs::canonicalize(path).unwrap(); From b02e76831c320d7a2141282722d04d67a73a3cc4 Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Sun, 18 Feb 2024 19:46:15 +0100 Subject: [PATCH 49/55] chore: add `derive` feature to `serde` dependency for build script --- lib/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/Cargo.toml b/lib/Cargo.toml index 22e081be2..a695f4b6d 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -194,7 +194,7 @@ globwalk = { workspace = true } protobuf = { workspace = true } protobuf-codegen = { workspace = true } protobuf-parse = { workspace = true } -serde = { workspace = true, features=["rc"] } +serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } yara-x-proto = { workspace = true } From 96488ccab229b5fa44b692821ee8aa149e5cb8ef Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Mon, 19 Feb 2024 13:27:18 +0100 Subject: [PATCH 50/55] refactor: merge CompileError and CompileErrorInfo in a single type. (#82) --- lib/src/compiler/errors.rs | 35 +------ lib/src/compiler/ir/ast2ir.rs | 190 +++++++++++++++------------------- lib/src/compiler/mod.rs | 50 ++++----- lib/src/lib.rs | 1 - 4 files changed, 114 insertions(+), 162 deletions(-) diff --git a/lib/src/compiler/errors.rs b/lib/src/compiler/errors.rs index 08227cfee..29fa76bbb 100644 --- a/lib/src/compiler/errors.rs +++ b/lib/src/compiler/errors.rs @@ -36,47 +36,16 @@ pub enum Error { ParseError(#[from] ParseError), #[error(transparent)] - CompileError(#[from] CompileError), + CompileError(#[from] Box), #[error(transparent)] VariableError(#[from] VariableError), } -/// Error produced while compiling rules. -#[derive(Eq, PartialEq)] -pub struct CompileError(Box); - -impl CompileError { - /// Returns additional information about the error. - pub fn info(&self) -> &CompileErrorInfo { - self.0.as_ref() - } -} - -impl Debug for CompileError { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "{:?}", self.0) - } -} - -impl Display for CompileError { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", self.0) - } -} - -impl From for CompileError { - fn from(value: CompileErrorInfo) -> Self { - Self(Box::new(value)) - } -} - -impl std::error::Error for CompileError {} - /// An error occurred during the compilation process. #[derive(DeriveError, Eq, PartialEq)] #[non_exhaustive] -pub enum CompileErrorInfo { +pub enum CompileError { #[error("wrong type")] #[label( "expression should be {expected_types}, but is `{actual_type}`", diff --git a/lib/src/compiler/ir/ast2ir.rs b/lib/src/compiler/ir/ast2ir.rs index c966a4d82..7ef06eb9d 100644 --- a/lib/src/compiler/ir/ast2ir.rs +++ b/lib/src/compiler/ir/ast2ir.rs @@ -17,7 +17,7 @@ use crate::compiler::ir::{ MatchAnchor, Of, OfItems, Pattern, PatternFlagSet, PatternFlags, PatternIdx, PatternInRule, Quantifier, Range, RegexpPattern, }; -use crate::compiler::{CompileContext, CompileError, CompileErrorInfo}; +use crate::compiler::{CompileContext, CompileError}; use crate::modules::BUILTIN_MODULES; use crate::re; use crate::re::parser::Error; @@ -27,18 +27,18 @@ use crate::types::{Map, Regexp, Type, TypeValue, Value}; pub(in crate::compiler) fn patterns_from_ast<'src>( report_builder: &ReportBuilder, patterns: Option<&Vec>>, -) -> Result>, CompileError> { +) -> Result>, Box> { patterns .into_iter() .flatten() .map(|p| pattern_from_ast(report_builder, p)) - .collect::>, CompileError>>() + .collect::>, Box>>() } fn pattern_from_ast<'src>( report_builder: &ReportBuilder, pattern: &ast::Pattern<'src>, -) -> Result, CompileError> { +) -> Result, Box> { match pattern { ast::Pattern::Text(pattern) => { Ok(text_pattern_from_ast(report_builder, pattern)?) @@ -55,7 +55,7 @@ fn pattern_from_ast<'src>( pub(in crate::compiler) fn text_pattern_from_ast<'src>( _report_builder: &ReportBuilder, pattern: &ast::TextPattern<'src>, -) -> Result, CompileError> { +) -> Result, Box> { let mut flags = PatternFlagSet::none(); if pattern.modifiers.ascii().is_some() @@ -108,7 +108,6 @@ pub(in crate::compiler) fn text_pattern_from_ast<'src>( base64_alphabet: base64_alphabet.map(String::from), base64wide_alphabet: base64wide_alphabet.map(String::from), anchored_at: None, - text: pattern.text.as_ref().into(), }), }) @@ -117,7 +116,7 @@ pub(in crate::compiler) fn text_pattern_from_ast<'src>( pub(in crate::compiler) fn hex_pattern_from_ast<'src>( _report_builder: &ReportBuilder, pattern: &ast::HexPattern<'src>, -) -> Result, CompileError> { +) -> Result, Box> { Ok(PatternInRule { identifier: pattern.identifier.name, pattern: Pattern::Regexp(RegexpPattern { @@ -131,7 +130,7 @@ pub(in crate::compiler) fn hex_pattern_from_ast<'src>( pub(in crate::compiler) fn regexp_pattern_from_ast<'src>( report_builder: &ReportBuilder, pattern: &ast::RegexpPattern<'src>, -) -> Result, CompileError> { +) -> Result, Box> { let mut flags = PatternFlagSet::none(); if pattern.modifiers.ascii().is_some() @@ -206,10 +205,10 @@ pub(in crate::compiler) fn regexp_pattern_from_ast<'src>( pub(in crate::compiler) fn expr_from_ast( ctx: &mut CompileContext, expr: &ast::Expr, -) -> Result { +) -> Result> { match expr { ast::Expr::Entrypoint { span } => { - Err(CompileError::from(CompileErrorInfo::entrypoint_unsupported( + Err(Box::new(CompileError::entrypoint_unsupported( ctx.report_builder, *span, Some("use `pe.entry_point`, `elf.entry_point` or `macho.entry_point`".to_string()), @@ -344,8 +343,7 @@ pub(in crate::compiler) fn expr_from_ast( }; if symbol.is_none() { - return Err(CompileError::from( - CompileErrorInfo::unknown_identifier( + return Err(Box::new(CompileError::unknown_identifier( ctx.report_builder, ident.name.to_string(), ident.span(), @@ -380,8 +378,7 @@ pub(in crate::compiler) fn expr_from_ast( let current_rule = ctx.get_current_rule(); let used_rule = ctx.get_rule(*rule_id); if current_rule.is_global && !used_rule.is_global { - return Err(CompileError::from( - CompileErrorInfo::wrong_rule_dependency( + return Err(Box::new(CompileError::wrong_rule_dependency( ctx.report_builder, ctx.ident_pool .get(current_rule.ident_id) @@ -588,8 +585,7 @@ pub(in crate::compiler) fn expr_from_ast( // The type of the key/index expression should correspond // with the type of the map's keys. if key_ty != ty { - return Err(CompileError::from( - CompileErrorInfo::wrong_type( + return Err(Box::new(CompileError::wrong_type( ctx.report_builder, format!("`{}`", key_ty), ty.to_string(), @@ -605,7 +601,7 @@ pub(in crate::compiler) fn expr_from_ast( }))) } type_value => { - Err(CompileError::from(CompileErrorInfo::wrong_type( + Err(Box::new(CompileError::wrong_type( ctx.report_builder, format!("`{}` or `{}`", Type::Array, Type::Map), type_value.ty().to_string(), @@ -620,7 +616,7 @@ pub(in crate::compiler) fn expr_from_ast( pub(in crate::compiler) fn bool_expr_from_ast( ctx: &mut CompileContext, ast: &ast::Expr, -) -> Result { +) -> Result> { let expr = expr_from_ast(ctx, ast)?; warn_if_not_bool(ctx, expr.ty(), ast.span()); Ok(expr) @@ -629,7 +625,7 @@ pub(in crate::compiler) fn bool_expr_from_ast( fn of_expr_from_ast( ctx: &mut CompileContext, of: &ast::Of, -) -> Result { +) -> Result> { let quantifier = quantifier_from_ast(ctx, &of.quantifier)?; // Create new stack frame with 5 slots: // 1 slot for the loop variable, a bool in this case. @@ -646,7 +642,7 @@ fn of_expr_from_ast( check_type(ctx, expr.ty(), e.span(), &[Type::Bool])?; Ok(expr) }) - .collect::, CompileError>>()?; + .collect::, Box>>()?; let num_items = tuple.len(); (OfItems::BoolExprTuple(tuple), num_items) @@ -734,7 +730,7 @@ fn of_expr_from_ast( fn for_of_expr_from_ast( ctx: &mut CompileContext, for_of: &ast::ForOf, -) -> Result { +) -> Result> { let quantifier = quantifier_from_ast(ctx, &for_of.quantifier)?; let pattern_set = pattern_set_from_ast(ctx, &for_of.pattern_set)?; // Create new stack frame with 5 slots: @@ -771,7 +767,7 @@ fn for_of_expr_from_ast( fn for_in_expr_from_ast( ctx: &mut CompileContext, for_in: &ast::ForIn, -) -> Result { +) -> Result> { let quantifier = quantifier_from_ast(ctx, &for_in.quantifier)?; let iterable = iterable_from_ast(ctx, &for_in.iterable)?; @@ -782,7 +778,7 @@ fn for_in_expr_from_ast( // the type of the first item in the tuple as the type of the // loop variable. Notice that we are using `clone_without_value` // instead of `clone`, because we want a TypeValue with the same - // type than the first item in the tuple, but we don't want to + // type as the first item in the tuple, but we don't want to // clone its actual value if known. The actual value for the // loop variable is not known until the loop is executed. vec![expressions @@ -814,15 +810,13 @@ fn for_in_expr_from_ast( if loop_vars.len() != expected_vars.len() { let span = loop_vars.first().unwrap().span(); let span = span.combine(&loop_vars.last().unwrap().span()); - return Err(CompileError::from( - CompileErrorInfo::assignment_mismatch( - ctx.report_builder, - loop_vars.len() as u8, - expected_vars.len() as u8, - for_in.iterable.span(), - span, - ), - )); + return Err(Box::new(CompileError::assignment_mismatch( + ctx.report_builder, + loop_vars.len() as u8, + expected_vars.len() as u8, + for_in.iterable.span(), + span, + ))); } // Create stack frame with capacity for the loop variables, plus 4 @@ -866,7 +860,7 @@ fn for_in_expr_from_ast( fn iterable_from_ast( ctx: &mut CompileContext, iter: &ast::Iterable, -) -> Result { +) -> Result> { match iter { ast::Iterable::Range(range) => { Ok(Iterable::Range(range_from_ast(ctx, range)?)) @@ -898,8 +892,8 @@ fn iterable_from_ast( // type mismatch. if let Some((prev_ty, prev_span)) = prev { if prev_ty != ty { - return Err(CompileError::from( - CompileErrorInfo::mismatching_types( + return Err(Box::new( + CompileError::mismatching_types( ctx.report_builder, prev_ty.to_string(), ty.to_string(), @@ -920,7 +914,7 @@ fn iterable_from_ast( fn anchor_from_ast( ctx: &mut CompileContext, anchor: &Option, -) -> Result { +) -> Result> { match anchor { Some(ast::MatchAnchor::At(at_)) => Ok(MatchAnchor::At(Box::new( non_negative_integer_from_ast(ctx, &at_.expr)?, @@ -935,7 +929,7 @@ fn anchor_from_ast( fn range_from_ast( ctx: &mut CompileContext, range: &ast::Range, -) -> Result { +) -> Result> { let lower_bound = Box::new(non_negative_integer_from_ast(ctx, &range.lower_bound)?); @@ -952,7 +946,7 @@ fn range_from_ast( ) = (lower_bound.type_value(), upper_bound.type_value()) { if lower_bound > upper_bound { - return Err(CompileError::from(CompileErrorInfo::invalid_range( + return Err(Box::new(CompileError::invalid_range( ctx.report_builder, range.span, ))); @@ -965,7 +959,7 @@ fn range_from_ast( fn non_negative_integer_from_ast( ctx: &mut CompileContext, expr: &ast::Expr, -) -> Result { +) -> Result> { let span = expr.span(); let expr = expr_from_ast(ctx, expr)?; let type_value = expr.type_value(); @@ -974,12 +968,10 @@ fn non_negative_integer_from_ast( if let TypeValue::Integer(Value::Const(value)) = type_value { if value < 0 { - return Err(CompileError::from( - CompileErrorInfo::unexpected_negative_number( - ctx.report_builder, - span, - ), - )); + return Err(Box::new(CompileError::unexpected_negative_number( + ctx.report_builder, + span, + ))); } } @@ -990,7 +982,7 @@ fn integer_in_range_from_ast( ctx: &mut CompileContext, expr: &ast::Expr, range: RangeInclusive, -) -> Result { +) -> Result> { let span = expr.span(); let expr = expr_from_ast(ctx, expr)?; let type_value = expr.type_value(); @@ -1001,14 +993,12 @@ fn integer_in_range_from_ast( // the given range. if let TypeValue::Integer(Value::Const(value)) = type_value { if !range.contains(&value) { - return Err(CompileError::from( - CompileErrorInfo::number_out_of_range( - ctx.report_builder, - *range.start(), - *range.end(), - span, - ), - )); + return Err(Box::new(CompileError::number_out_of_range( + ctx.report_builder, + *range.start(), + *range.end(), + span, + ))); } } @@ -1018,7 +1008,7 @@ fn integer_in_range_from_ast( fn quantifier_from_ast( ctx: &mut CompileContext, quantifier: &ast::Quantifier, -) -> Result { +) -> Result> { match quantifier { ast::Quantifier::None { .. } => Ok(Quantifier::None), ast::Quantifier::All { .. } => Ok(Quantifier::All), @@ -1040,7 +1030,7 @@ fn quantifier_from_ast( fn pattern_set_from_ast( ctx: &mut CompileContext, pattern_set: &ast::PatternSet, -) -> Result, CompileError> { +) -> Result, Box> { let pattern_indexes = match pattern_set { // `x of them` ast::PatternSet::Them { span } => { @@ -1050,16 +1040,11 @@ fn pattern_set_from_ast( .collect(); if pattern_indexes.is_empty() { - return Err(CompileError::from( - CompileErrorInfo::empty_pattern_set( - ctx.report_builder, - *span, - Some( - "this rule doesn't define any patterns" - .to_string(), - ), - ), - )); + return Err(Box::new(CompileError::empty_pattern_set( + ctx.report_builder, + *span, + Some("this rule doesn't define any patterns".to_string()), + ))); } // Make all the patterns in the set non-anchorable. @@ -1077,16 +1062,14 @@ fn pattern_set_from_ast( .iter() .any(|pattern| item.matches(pattern.identifier())) { - return Err(CompileError::from( - CompileErrorInfo::empty_pattern_set( - ctx.report_builder, - item.span(), - Some(format!( - "`{}` doesn't match any pattern identifier", - item.identifier, - )), - ), - )); + return Err(Box::new(CompileError::empty_pattern_set( + ctx.report_builder, + item.span(), + Some(format!( + "`{}` doesn't match any pattern identifier", + item.identifier, + )), + ))); } } let mut pattern_indexes = Vec::new(); @@ -1111,7 +1094,7 @@ fn pattern_set_from_ast( fn func_call_from_ast( ctx: &mut CompileContext, func_call: &ast::FuncCall, -) -> Result { +) -> Result> { let callable = expr_from_ast(ctx, &func_call.callable)?; let type_value = callable.type_value(); @@ -1126,7 +1109,7 @@ fn func_call_from_ast( .args .iter() .map(|arg| expr_from_ast(ctx, arg)) - .collect::, CompileError>>()?; + .collect::, Box>>()?; let arg_types: Vec = args.iter().map(|arg| arg.ty()).collect(); @@ -1158,7 +1141,7 @@ fn func_call_from_ast( // No matching signature was found, that means that the arguments // provided were incorrect. if matching_signature.is_none() { - return Err(CompileError::from(CompileErrorInfo::wrong_arguments( + return Err(Box::new(CompileError::wrong_arguments( ctx.report_builder, func_call.args_span, Some(format!( @@ -1193,7 +1176,7 @@ fn func_call_from_ast( fn matches_expr_from_ast( ctx: &mut CompileContext, expr: &ast::BinaryExpr, -) -> Result { +) -> Result> { let lhs_span = expr.lhs.span(); let rhs_span = expr.rhs.span(); @@ -1217,11 +1200,11 @@ fn check_type( ty: Type, span: Span, accepted_types: &[Type], -) -> Result<(), CompileError> { +) -> Result<(), Box> { if accepted_types.contains(&ty) { Ok(()) } else { - Err(CompileError::from(CompileErrorInfo::wrong_type( + Err(Box::new(CompileError::wrong_type( ctx.report_builder, ErrorInfo::join_with_or(accepted_types, true), ty.to_string(), @@ -1235,11 +1218,11 @@ fn check_type2( expr: &ast::Expr, ty: Type, accepted_types: &[Type], -) -> Result<(), CompileError> { +) -> Result<(), Box> { if accepted_types.contains(&ty) { Ok(()) } else { - Err(CompileError::from(CompileErrorInfo::wrong_type( + Err(Box::new(CompileError::wrong_type( ctx.report_builder, ErrorInfo::join_with_or(accepted_types, true), ty.to_string(), @@ -1256,7 +1239,7 @@ fn check_operands( rhs_span: Span, accepted_types: &[Type], compatible_types: &[Type], -) -> Result<(), CompileError> { +) -> Result<(), Box> { // Both types must be known. assert!(!matches!(lhs_ty, Type::Unknown)); assert!(!matches!(rhs_ty, Type::Unknown)); @@ -1276,7 +1259,7 @@ fn check_operands( }; if !types_are_compatible { - return Err(CompileError::from(CompileErrorInfo::mismatching_types( + return Err(Box::new(CompileError::mismatching_types( ctx.report_builder, lhs_ty.to_string(), rhs_ty.to_string(), @@ -1295,27 +1278,27 @@ fn re_error_to_compile_error( ) -> CompileError { match err { Error::SyntaxError { msg, span } => { - CompileError::from(CompileErrorInfo::invalid_regexp( + CompileError::invalid_regexp( report_builder, msg, // the error span is relative to the start of the regexp, not to // the start of the source file, here we make it relative to the // source file. regexp.span.subspan(span.start.offset, span.end.offset), - )) + ) } Error::MixedGreediness { is_greedy_1, is_greedy_2, span_1, span_2, - } => CompileError::from(CompileErrorInfo::mixed_greediness( + } => CompileError::mixed_greediness( report_builder, if is_greedy_1 { "greedy" } else { "non-greedy" }.to_string(), if is_greedy_2 { "greedy" } else { "non-greedy" }.to_string(), regexp.span.subspan(span_1.start.offset, span_1.end.offset), regexp.span.subspan(span_2.start.offset, span_2.end.offset), - )), + ), } } @@ -1356,7 +1339,7 @@ macro_rules! gen_unary_op { fn $name( ctx: &mut CompileContext, expr: &ast::UnaryExpr, - ) -> Result { + ) -> Result> { let operand = Box::new(expr_from_ast(ctx, &expr.operand)?); // The `not` operator accepts integers, floats and strings because @@ -1369,7 +1352,7 @@ macro_rules! gen_unary_op { )?; let check_fn: - Option Result<(), CompileError>> + Option Result<(), Box>> = $check_fn; if let Some(check_fn) = check_fn { @@ -1392,7 +1375,7 @@ macro_rules! gen_binary_op { fn $name( ctx: &mut CompileContext, expr: &ast::BinaryExpr, - ) -> Result { + ) -> Result> { let lhs_span = expr.lhs.span(); let rhs_span = expr.rhs.span(); @@ -1410,7 +1393,7 @@ macro_rules! gen_binary_op { )?; let check_fn: - Option Result<(), CompileError>> + Option Result<(), Box>> = $check_fn; if let Some(check_fn) = check_fn { @@ -1433,7 +1416,7 @@ macro_rules! gen_string_op { fn $name( ctx: &mut CompileContext, expr: &ast::BinaryExpr, - ) -> Result { + ) -> Result> { let lhs_span = expr.lhs.span(); let rhs_span = expr.rhs.span(); @@ -1466,17 +1449,17 @@ macro_rules! gen_n_ary_operation { fn $name( ctx: &mut CompileContext, expr: &ast::NAryExpr, - ) -> Result { + ) -> Result> { let accepted_types = &[$( $accepted_types ),+]; let compatible_types = &[$( $compatible_types ),+]; let operands_hir: Vec = expr .operands() .map(|expr| expr_from_ast(ctx, expr)) - .collect::, CompileError>>()?; + .collect::, Box>>()?; let check_fn: - Option Result<(), CompileError>> + Option Result<(), Box>> = $check_fn; // Make sure that all operands have one of the accepted types. @@ -1507,8 +1490,7 @@ macro_rules! gen_n_ary_operation { }; if !types_are_compatible { - return Err(CompileError::from( - CompileErrorInfo::mismatching_types( + return Err(Box::new(CompileError::mismatching_types( ctx.report_builder, lhs_ty.to_string(), rhs_ty.to_string(), @@ -1630,8 +1612,8 @@ gen_binary_op!( Some(|ctx, _lhs, rhs, _lhs_span, rhs_span| { if let TypeValue::Integer(Value::Const(value)) = rhs.type_value() { if value < 0 { - return Err(CompileError::from( - CompileErrorInfo::unexpected_negative_number( + return Err(Box::new( + CompileError::unexpected_negative_number( ctx.report_builder, rhs_span, ), @@ -1650,8 +1632,8 @@ gen_binary_op!( Some(|ctx, _lhs, rhs, _lhs_span, rhs_span| { if let TypeValue::Integer(Value::Const(value)) = rhs.type_value() { if value < 0 { - return Err(CompileError::from( - CompileErrorInfo::unexpected_negative_number( + return Err(Box::new( + CompileError::unexpected_negative_number( ctx.report_builder, rhs_span, ), diff --git a/lib/src/compiler/mod.rs b/lib/src/compiler/mod.rs index 3283d943d..303b5ee9c 100644 --- a/lib/src/compiler/mod.rs +++ b/lib/src/compiler/mod.rs @@ -558,24 +558,22 @@ impl<'a> Compiler<'a> { fn check_for_existing_identifier( &self, ident: &Ident, - ) -> Result<(), CompileError> { + ) -> Result<(), Box> { if let Some(symbol) = self.symbol_table.lookup(ident.name) { return match symbol.kind() { SymbolKind::Rule(rule_id) => { - Err(CompileError::from(CompileErrorInfo::duplicate_rule( + Err(Box::new(CompileError::duplicate_rule( &self.report_builder, ident.name.to_string(), ident.span, self.rules.get(rule_id.0 as usize).unwrap().ident_span, ))) } - _ => Err(CompileError::from( - CompileErrorInfo::conflicting_rule_identifier( - &self.report_builder, - ident.name.to_string(), - ident.span, - ), - )), + _ => Err(Box::new(CompileError::conflicting_rule_identifier( + &self.report_builder, + ident.name.to_string(), + ident.span, + ))), }; } Ok(()) @@ -635,14 +633,17 @@ impl<'a> Compiler<'a> { /// creates a new field with the same name than the module in the /// top-level structure `self.root_struct` that contains all the /// imported modules. This field is created only if it don't exist yet. - fn import_module(&mut self, import: &Import) -> Result<(), CompileError> { + fn import_module( + &mut self, + import: &Import, + ) -> Result<(), Box> { let module_name = import.module_name.as_str(); let module = BUILTIN_MODULES.get(module_name); // Does a module with the given name actually exist? ... if module.is_none() { // The module does not exist, that's an error. - return Err(CompileError::from(CompileErrorInfo::unknown_module( + return Err(Box::new(CompileError::unknown_module( &self.report_builder, module_name.to_string(), import.span(), @@ -703,7 +704,7 @@ impl<'a> Compiler<'a> { } impl<'a> Compiler<'a> { - fn c_rule(&mut self, rule: &ast::Rule) -> Result<(), CompileError> { + fn c_rule(&mut self, rule: &ast::Rule) -> Result<(), Box> { // Check if another rule, module or variable has the same identifier // and return an error in that case. self.check_for_existing_identifier(&rule.identifier)?; @@ -1064,7 +1065,7 @@ impl<'a> Compiler<'a> { pattern: RegexpPattern, anchored_at: Option, span: Span, - ) -> Result<(), CompileError> { + ) -> Result<(), Box> { // Try splitting the regexp into multiple chained sub-patterns if it // contains large gaps. For example, `{ 01 02 03 [-] 04 05 06 }` is // split into `{ 01 02 03 }` and `{ 04 05 06 }`, where `{ 04 05 06 }` @@ -1227,7 +1228,7 @@ impl<'a> Compiler<'a> { trailing: &[ChainedPattern], flags: PatternFlagSet, span: Span, - ) -> Result<(), CompileError> { + ) -> Result<(), Box> { let ascii = flags.contains(PatternFlags::Ascii); let wide = flags.contains(PatternFlags::Wide); let case_insensitive = flags.contains(PatternFlags::Nocase); @@ -1371,7 +1372,7 @@ impl<'a> Compiler<'a> { &mut self, hir: &re::hir::Hir, span: Span, - ) -> Result<(Vec, bool), CompileError> { + ) -> Result<(Vec, bool), Box> { // When the `fast-regexp` feature is enabled, try to compile the regexp // for `FastVM` first, if the it fails with `Error::FastIncompatible`, // the regexp is not compatible for `FastVM` and `PikeVM` must be used @@ -1394,18 +1395,16 @@ impl<'a> Compiler<'a> { ); let mut atoms = result.map_err(|err| match err { - re::Error::TooLarge => { - CompileError::from(CompileErrorInfo::invalid_regexp( - &self.report_builder, - "regexp is too large".to_string(), - span, - )) - } + re::Error::TooLarge => Box::new(CompileError::invalid_regexp( + &self.report_builder, + "regexp is too large".to_string(), + span, + )), _ => unreachable!(), })?; if matches!(hir.minimum_len(), Some(0)) { - return Err(CompileError::from(CompileErrorInfo::invalid_regexp( + return Err(Box::new(CompileError::invalid_regexp( &self.report_builder, "this regexp can match empty strings".to_string(), span, @@ -1473,7 +1472,10 @@ impl<'a> Compiler<'a> { ) } - fn c_imports(&mut self, imports: &[Import]) -> Result<(), CompileError> { + fn c_imports( + &mut self, + imports: &[Import], + ) -> Result<(), Box> { for import in imports { // Import the module. This updates `self.root_struct` if // necessary. diff --git a/lib/src/lib.rs b/lib/src/lib.rs index 1e94ed762..51e666fb4 100644 --- a/lib/src/lib.rs +++ b/lib/src/lib.rs @@ -43,7 +43,6 @@ assert_eq!(results.matching_rules().len(), 1); pub use compiler::compile; pub use compiler::CompileError; -pub use compiler::CompileErrorInfo; pub use compiler::Compiler; pub use compiler::Error; pub use compiler::Rules; From b4b54e7f21c5c099d6eff1a4bd68527fae3f49db Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Mon, 19 Feb 2024 17:35:07 +0100 Subject: [PATCH 51/55] feat: implement API for setting the output data for a YARA module --- capi/include/yara-x.h | 37 ++++++++ capi/src/scanner.rs | 96 ++++++++++++++++--- go/go.mod | 1 + go/go.sum | 7 ++ go/scanner.go | 60 ++++++++++++ lib/src/modules/mod.rs | 2 +- lib/src/scanner/context.rs | 7 +- lib/src/scanner/mod.rs | 184 +++++++++++++++++++++++++++++-------- lib/src/scanner/tests.rs | 80 ++++++++++++++++ lib/src/types/structure.rs | 19 +--- 10 files changed, 424 insertions(+), 69 deletions(-) diff --git a/capi/include/yara-x.h b/capi/include/yara-x.h index 42dd2360c..a398eb10f 100644 --- a/capi/include/yara-x.h +++ b/capi/include/yara-x.h @@ -228,6 +228,43 @@ enum YRX_RESULT yrx_scanner_on_matching_rule(struct YRX_SCANNER *scanner, YRX_ON_MATCHING_RULE callback, void *user_data); +// Specifies the output data structure for a module. +// +// Each YARA module generates an output consisting of a data structure that +// contains information about the scanned file. This data structure is represented +// by a Protocol Buffer. Typically, you won't need to provide this output data +// yourself, as the YARA module automatically generates different outputs for +// each file it scans. +// +// However, there are two scenarios in which you may want to provide the output +// for a module yourself: +// +// 1) When the module does not produce any output on its own. +// 2) When you already know the output of the module for the upcoming file to +// be scanned, and you prefer to reuse this data instead of generating it again. +// +// Case 1) applies to certain modules lacking a main function, thus incapable of +// producing any output on their own. For such modules, you must set the output +// before scanning the associated data. Since the module's output typically varies +// with each scanned file, you need to call [yrx_scanner_set_module_output] prior +// to each invocation of [yrx_scanner_scan]. Once [yrx_scanner_scan] is executed, +// the module's output is consumed and will be empty unless set again before the +// subsequent call. +// +// Case 2) applies when you have previously stored the module's output for certain +// scanned data. In such cases, when rescanning the data, you can utilize this +// function to supply the module's output, thereby preventing redundant computation +// by the module. This optimization enhances performance by eliminating the need +// for the module to reparse the scanned data. +// +// The `name` argument is either a YARA module name (i.e: "pe", "elf", "dotnet", +// etc.) or the fully-qualified name of the protobuf message associated to +// the module. +enum YRX_RESULT yrx_scanner_set_module_output(struct YRX_SCANNER *scanner, + const char *name, + const uint8_t *data, + size_t len); + // Returns the error message for the most recent error returned by the // scanner. // diff --git a/capi/src/scanner.rs b/capi/src/scanner.rs index a9004addd..82c26b6d6 100644 --- a/capi/src/scanner.rs +++ b/capi/src/scanner.rs @@ -1,4 +1,4 @@ -use std::ffi::{c_char, CString}; +use std::ffi::{c_char, CStr, CString}; use std::slice; use std::time::Duration; use yara_x::ScanError; @@ -83,16 +83,9 @@ pub unsafe extern "C" fn yrx_scanner_scan( return YRX_RESULT::INVALID_ARGUMENT; } - // `data` is allowed to be null as long as `len` is 0. This case is handled - // as an empty slice. - if data.is_null() && len > 0 { - return YRX_RESULT::INVALID_ARGUMENT; - } - - let data = if data.is_null() || len == 0 { - &[] - } else { - slice::from_raw_parts(data, len) + let data = match slice_from_ptr_and_len(data, len) { + Some(data) => data, + None => return YRX_RESULT::INVALID_ARGUMENT, }; let scanner = scanner.as_mut().unwrap(); @@ -156,6 +149,70 @@ pub unsafe extern "C" fn yrx_scanner_on_matching_rule( } } +/// Specifies the output data structure for a module. +/// +/// Each YARA module generates an output consisting of a data structure that +/// contains information about the scanned file. This data structure is represented +/// by a Protocol Buffer. Typically, you won't need to provide this output data +/// yourself, as the YARA module automatically generates different outputs for +/// each file it scans. +/// +/// However, there are two scenarios in which you may want to provide the output +/// for a module yourself: +/// +/// 1) When the module does not produce any output on its own. +/// 2) When you already know the output of the module for the upcoming file to +/// be scanned, and you prefer to reuse this data instead of generating it again. +/// +/// Case 1) applies to certain modules lacking a main function, thus incapable of +/// producing any output on their own. For such modules, you must set the output +/// before scanning the associated data. Since the module's output typically varies +/// with each scanned file, you need to call [yrx_scanner_set_module_output] prior +/// to each invocation of [yrx_scanner_scan]. Once [yrx_scanner_scan] is executed, +/// the module's output is consumed and will be empty unless set again before the +/// subsequent call. +/// +/// Case 2) applies when you have previously stored the module's output for certain +/// scanned data. In such cases, when rescanning the data, you can utilize this +/// function to supply the module's output, thereby preventing redundant computation +/// by the module. This optimization enhances performance by eliminating the need +/// for the module to reparse the scanned data. +/// +/// The `name` argument is either a YARA module name (i.e: "pe", "elf", "dotnet", +/// etc.) or the fully-qualified name of the protobuf message associated to +/// the module. +#[no_mangle] +pub unsafe extern "C" fn yrx_scanner_set_module_output( + scanner: *mut YRX_SCANNER, + name: *const c_char, + data: *const u8, + len: usize, +) -> YRX_RESULT { + if scanner.is_null() { + return YRX_RESULT::INVALID_ARGUMENT; + } + + let module_name = match CStr::from_ptr(name).to_str() { + Ok(name) => name, + Err(_) => return YRX_RESULT::INVALID_ARGUMENT, + }; + + let data = match slice_from_ptr_and_len(data, len) { + Some(data) => data, + None => return YRX_RESULT::INVALID_ARGUMENT, + }; + + let scanner = scanner.as_mut().unwrap(); + + match scanner.inner.set_module_output_raw(module_name, data) { + Ok(_) => YRX_RESULT::SUCCESS, + Err(err) => { + scanner.last_error = Some(CString::new(err.to_string()).unwrap()); + YRX_RESULT::SCAN_ERROR + } + } +} + /// Returns the error message for the most recent error returned by the /// scanner. /// @@ -180,3 +237,20 @@ pub unsafe extern "C" fn yrx_scanner_last_error( std::ptr::null() } } + +unsafe fn slice_from_ptr_and_len<'a>( + data: *const u8, + len: usize, +) -> Option<&'a [u8]> { + // `data` is allowed to be null as long as `len` is 0. That's equivalent + // to an empty slice. + if data.is_null() && len > 0 { + return None; + } + let data = if data.is_null() || len == 0 { + &[] + } else { + slice::from_raw_parts(data, len) + }; + Some(data) +} diff --git a/go/go.mod b/go/go.mod index 2d4da5b8f..65294f96b 100644 --- a/go/go.mod +++ b/go/go.mod @@ -4,6 +4,7 @@ go 1.18 require ( github.com/davecgh/go-spew v1.1.1 // indirect + github.com/golang/protobuf v1.5.3 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/stretchr/testify v1.8.4 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go/go.sum b/go/go.sum index 8cf66553b..af49a2ab8 100644 --- a/go/go.sum +++ b/go/go.sum @@ -1,9 +1,16 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/go/scanner.go b/go/scanner.go index c64430dd4..78836fc36 100644 --- a/go/scanner.go +++ b/go/scanner.go @@ -10,6 +10,10 @@ import ( "unsafe" ) +import ( + "github.com/golang/protobuf/proto" +) + // #include // void onMatchingRule(YRX_RULE*, void*); import "C" @@ -85,6 +89,62 @@ func (s *Scanner) Timeout(timeout time.Duration) { var ErrTimeout = errors.New("timeout") + +// SetModuleOutput sets the output data for a YARA module. +// +// Each YARA module generates an output consisting of a data structure that +// contains information about the scanned file. This data structure is represented +// by a Protocol Buffer. Typically, you won't need to provide this data yourself, +// as the YARA module automatically generates different outputs for each file +// it scans. +// +// However, there are two scenarios in which you may want to provide the output +// for a module yourself: +// +// 1) When the module does not produce any output on its own. +// 2) When you already know the output of the module for the upcoming file to +// be scanned, and you prefer to reuse this data instead of generating it again. +// +// Case 1) applies to certain modules lacking a main function, thus incapable of +// producing any output on their own. For such modules, you must set the output +// before scanning the associated data. Since the module's output typically varies +// with each scanned file, you need to call [yrx_scanner_set_module_output] prior +// to each invocation of [yrx_scanner_scan]. Once [yrx_scanner_scan] is executed, +// the module's output is consumed and will be empty unless set again before the +// subsequent call. +// +// Case 2) applies when you have previously stored the module's output for certain +// scanned data. In such cases, when rescanning the data, you can utilize this +// function to supply the module's output, thereby preventing redundant computation +// by the module. This optimization enhances performance by eliminating the need +// for the module to reparse the scanned data. +// +// The data argument must be a Protocol Buffer message corresponding to any of +// the existing YARA modules. +func (s *Scanner) SetModuleOutput(data proto.Message) error { + var err error + var buf []byte + + if buf, err = proto.Marshal(data); err != nil { + return err + } + + var ptr *C.uint8_t + if len(buf) > 0 { + ptr = (*C.uint8_t)(unsafe.Pointer(&(buf[0]))) + } + + name := C.CString(proto.MessageReflect(data).Descriptor().FullName()) + defer C.free(unsafe.Pointer(name)) + + if r := C.yrx_scanner_set_module_output(s.cScanner, name, ptr, C.size_t(len(buf))); r != C.SUCCESS { + err = errors.New(C.GoString(C.yrx_scanner_last_error(s.cScanner))) + } + + runtime.KeepAlive(s) + return err +} + // Scan scans the provided data with the Rules associated to the Scanner. func (s *Scanner) Scan(buf []byte) ([]*Rule, error) { var ptr *C.uint8_t diff --git a/lib/src/modules/mod.rs b/lib/src/modules/mod.rs index f88dd617a..b18ce5c70 100644 --- a/lib/src/modules/mod.rs +++ b/lib/src/modules/mod.rs @@ -153,7 +153,7 @@ pub mod mods { /// YARA modules typically parse specific file formats, returning structures /// that contain information about the file. These structures are used in YARA /// rules for expressing powerful and rich conditions. However, being able to - /// access this information outside of YARA rules can also be beneficial. + /// access this information outside YARA rules can also be beneficial. /// ///
/// diff --git a/lib/src/scanner/context.rs b/lib/src/scanner/context.rs index 6ef41df6b..5684c1471 100644 --- a/lib/src/scanner/context.rs +++ b/lib/src/scanner/context.rs @@ -73,8 +73,13 @@ pub(crate) struct ScanContext<'r> { pub main_memory: Option, /// Hash map that contains the protobuf messages returned by YARA modules. /// Keys are the fully qualified protobuf message name, and values are - /// the message returned by the corresponding module. + /// the message returned by the main function of the corresponding module. pub module_outputs: FxHashMap>, + /// Hash map that contains the protobuf messages that has been explicitly + /// provided by the user to be used as module outputs during the next scan + /// operation. Keys are the fully qualified protobuf message names, and + /// values are the protobuf messages set with [`Scanner::set_module_output`]. + pub user_provided_module_outputs: FxHashMap>, /// Hash map that tracks the matches occurred during a scan. The keys /// are the PatternId of the matching pattern, and values are a list /// of matches. diff --git a/lib/src/scanner/mod.rs b/lib/src/scanner/mod.rs index b94f37f43..0646f7d76 100644 --- a/lib/src/scanner/mod.rs +++ b/lib/src/scanner/mod.rs @@ -55,6 +55,12 @@ pub enum ScanError { /// Could not map the scanned file into memory. #[error("can not map `{path}`: {source}")] MapError { path: PathBuf, source: fmmap::error::Error }, + /// Could not deserialize the protobuf message for some YARA module. + #[error("can not deserialize protobuf message for YARA module `{module}`: {err}")] + ProtoError { module: String, err: protobuf::Error }, + /// The module is unknown. + #[error("unknown module")] + UnknownModule, } /// Global counter that gets incremented every 1 second by a dedicated thread. @@ -128,6 +134,7 @@ impl<'r> Scanner<'r> { global_matching_rules: FxHashMap::default(), main_memory: None, module_outputs: FxHashMap::default(), + user_provided_module_outputs: FxHashMap::default(), pattern_matches: FxHashMap::default(), unconfirmed_matches: FxHashMap::default(), deadline: 0, @@ -366,6 +373,103 @@ impl<'r> Scanner<'r> { Ok(self) } + + /// Sets the output data for a YARA module. + /// + /// Each YARA module generates an output consisting of a data structure that + /// contains information about the scanned file. This data structure is + /// represented by a Protocol Buffer message. Typically, you won't need to + /// provide this data yourself, as the YARA module automatically generates + /// different outputs for each file it scans. + /// + /// However, there are two scenarios in which you may want to provide the + /// output for a module yourself: + /// + /// 1) When the module does not produce any output on its own. + /// 2) When you already know the output of the module for the upcoming file + /// to be scanned, and you prefer to reuse this data instead of generating + /// it again. + /// + /// Case 1) applies to certain modules lacking a main function, thus + /// incapable of producing any output on their own. For such modules, you + /// must set the output before scanning the associated data. Since the + /// module's output typically varies with each scanned file, you need to + /// call [`Scanner::set_module_output`] prior to each invocation of + /// [`Scanner::scan`]. Once [`Scanner::scan`] is executed, the module's + /// output is consumed and will be empty unless set again before the + /// subsequent call. + /// + /// Case 2) applies when you have previously stored the module's output for + /// certain scanned data. In such cases, when rescanning the data, you can + /// utilize this function to supply the module's output, thereby preventing + /// redundant computation by the module. This optimization enhances + /// performance by eliminating the need for the module to reparse the + /// scanned data. + /// + ///
+ /// + /// The `data` argument must be a Protocol Buffer message corresponding + /// to any of the existing YARA modules. + pub fn set_module_output( + &mut self, + data: Box, + ) -> Result<(), ScanError> { + let descriptor = data.descriptor_dyn(); + let full_name = descriptor.full_name(); + + // Check if the protobuf message passed to this function corresponds + // with any of the existing modules. + if !BUILTIN_MODULES + .iter() + .any(|m| m.1.root_struct_descriptor.full_name() == full_name) + { + return Err(ScanError::UnknownModule); + } + + self.wasm_store + .data_mut() + .user_provided_module_outputs + .insert(full_name.to_string(), data); + + Ok(()) + } + + /// Similar to [`Scanner::set_module_output`], but receives a module name + /// and the protobuf message as raw data. + /// + /// `name` can be either the YARA module name (i.e: "pe", "elf", "dotnet", + /// etc.) or the fully-qualified name for the protobuf message associated + /// to the module (i.e: "pe.PE", "elf.ELF", "dotnet.Dotnet", etc.). + pub fn set_module_output_raw( + &mut self, + name: &str, + data: &[u8], + ) -> Result<(), ScanError> { + // Try to find the module by name first, if not found, then try + // to find a module where the fully-qualified name for its protobuf + // message matches the `name` arguments. + let descriptor = if let Some(module) = BUILTIN_MODULES.get(name) { + Some(&module.root_struct_descriptor) + } else { + BUILTIN_MODULES.values().find_map(|module| { + if module.root_struct_descriptor.full_name() == name { + Some(&module.root_struct_descriptor) + } else { + None + } + }) + }; + + if descriptor.is_none() { + return Err(ScanError::UnknownModule); + } + + self.set_module_output( + descriptor.unwrap().parse_from_bytes(data).map_err(|err| { + ScanError::ProtoError { module: name.to_string(), err } + })?, + ) + } } impl<'r> Scanner<'r> { @@ -441,41 +545,43 @@ impl<'r> Scanner<'r> { for module_name in ctx.compiled_rules.imports() { // Lookup the module in the list of built-in modules. let module = modules::BUILTIN_MODULES.get(module_name).unwrap(); - - // Call the module's main function, if any. This function returns - // a data structure serialized as a protocol buffer. The format of - // the data is specified by the .proto file associated to the - // module. - let module_output = if let Some(main_fn) = module.main_fn { - main_fn(data.as_ref()) + let root_struct_name = module.root_struct_descriptor.full_name(); + + // If the user already provided some output for the module by + // calling `Scanner::set_module_output`, use that output. If not, + // call the module's main function (if the module has a main + // function) for getting its output. + let module_output = if let Some(output) = + ctx.user_provided_module_outputs.remove(root_struct_name) + { + Some(output) } else { - // Implement the case in which the module doesn't have a main - // function and the serialized data should be provided by the - // user. - todo!() + module.main_fn.map(|main_fn| main_fn(data.as_ref())) }; - // Make sure that the module is returning a protobuf message of the - // expected type. - debug_assert_eq!( - module_output.descriptor_dyn().full_name(), - module.root_struct_descriptor.full_name(), - "main function of module `{}` must return `{}`, but returned `{}`", - module_name, - module.root_struct_descriptor.full_name(), - module_output.descriptor_dyn().full_name(), - ); - - // Make sure that the module is returning a protobuf message where - // all required fields are initialized. This only applies to - // proto2, proto3 doesn't have "required" fields, all - // fields are optional. - debug_assert!( - module_output.is_initialized_dyn(), - "module `{}` returned a protobuf `{}` where some required fields are not initialized ", - module_name, - module.root_struct_descriptor.full_name() - ); + if let Some(module_output) = &module_output { + // Make sure that the module is returning a protobuf message of the + // expected type. + debug_assert_eq!( + module_output.descriptor_dyn().full_name(), + module.root_struct_descriptor.full_name(), + "main function of module `{}` must return `{}`, but returned `{}`", + module_name, + module.root_struct_descriptor.full_name(), + module_output.descriptor_dyn().full_name(), + ); + + // Make sure that the module is returning a protobuf message where + // all required fields are initialized. This only applies to + // proto2, proto3 doesn't have "required" fields, all + // fields are optional. + debug_assert!( + module_output.is_initialized_dyn(), + "module `{}` returned a protobuf `{}` where some required fields are not initialized ", + module_name, + module.root_struct_descriptor.full_name() + ); + } // When constant folding is enabled we don't need to generate // structure fields for enums. This is because during the @@ -491,16 +597,16 @@ impl<'r> Scanner<'r> { let generate_fields_for_enums = !cfg!(feature = "constant-folding"); - let module_struct = Struct::from_proto_msg( - module_output.deref(), + let module_struct = Struct::from_proto_descriptor_and_msg( + &module.root_struct_descriptor, + module_output.as_deref(), generate_fields_for_enums, ); - // Update the module's output in stored in ScanContext. - ctx.module_outputs.insert( - module_output.descriptor_dyn().full_name().to_string(), - module_output, - ); + if let Some(module_output) = module_output { + ctx.module_outputs + .insert(root_struct_name.to_string(), module_output); + } // The data structure obtained from the module is added to the // root structure. Any data from previous scans will be replaced diff --git a/lib/src/scanner/tests.rs b/lib/src/scanner/tests.rs index 307a9ec7f..501ba9366 100644 --- a/lib/src/scanner/tests.rs +++ b/lib/src/scanner/tests.rs @@ -1,6 +1,8 @@ use pretty_assertions::assert_eq; use protobuf::MessageDyn; +use protobuf::{Message, MessageFull}; +use crate::mods; use crate::scanner::Scanner; use crate::variables::VariableError; @@ -471,3 +473,81 @@ fn max_matches_per_pattern() { // number of matches must be reset to 0 for the new scan. assert_eq!(scanner.scan(b"foo").unwrap().matching_rules().len(), 1); } + +#[test] +fn set_module_output() { + let mut compiler = crate::Compiler::new(); + + compiler + .add_source( + r#" + import "pe" + rule test { + condition: + pe.entry_point == 1 + } + "#, + ) + .unwrap(); + + let rules = compiler.build(); + + let mut scanner = Scanner::new(&rules); + let mut pe_data = Box::new(mods::PE::new()); + + pe_data.set_entry_point(1); + pe_data.set_is_pe(true); + + let pe_data_raw = pe_data.write_to_bytes().unwrap(); + + scanner.set_module_output(pe_data).unwrap(); + + // The data being scanned is empty, but we set the output for the PE module + // by ourselves. + let scan_results = scanner.scan(b"").expect("scan should not fail"); + assert_eq!(scan_results.matching_rules().len(), 1); + + // In this second call we haven't set a value for entry point, so it's + // undefined. + let scan_results = scanner.scan(b"").expect("scan should not fail"); + assert_eq!(scan_results.matching_rules().len(), 0); + + // This should fail because `foobar` is not a valid module name. + assert_eq!( + scanner + .set_module_output_raw("foobar", &[]) + .err() + .unwrap() + .to_string() + .as_str(), + "unknown module" + ); + + // This should fail while trying to parse the empty slice as the protobuf + // corresponding to the `pe` module. + assert_eq!( + scanner + .set_module_output_raw("pe", &[]) + .err() + .unwrap() + .to_string() + .as_str(), + "can not deserialize protobuf message for YARA module `pe`: Message `PE` is missing required fields" + ); + + // Now test by passing a valid protobuf for the PE module. + scanner.set_module_output_raw("pe", pe_data_raw.as_slice()).unwrap(); + let scan_results = scanner.scan(b"").expect("scan should not fail"); + assert_eq!(scan_results.matching_rules().len(), 1); + + // Try calling `set_module_output_raw` but this time pass the fully-qualified + // name of the protobuf message, instead of the module name. + scanner + .set_module_output_raw( + mods::PE::descriptor().full_name(), + pe_data_raw.as_slice(), + ) + .unwrap(); + let scan_results = scanner.scan(b"").expect("scan should not fail"); + assert_eq!(scan_results.matching_rules().len(), 1); +} diff --git a/lib/src/types/structure.rs b/lib/src/types/structure.rs index e16e123e1..7440af3c9 100644 --- a/lib/src/types/structure.rs +++ b/lib/src/types/structure.rs @@ -43,8 +43,8 @@ pub(crate) struct StructField { /// is also represented by one of these structures. /// /// The structures that represent a YARA module are created from the protobuf -/// associated to that module. Functions [`Struct::from_proto_msg`] and -/// [`Struct::from_proto_descriptor_and_msg`] are used for that purpose. +/// associated to that module. Function [`Struct::from_proto_descriptor_and_msg`] +/// is used for that purpose. #[derive(Debug, Serialize, Deserialize, Default)] pub(crate) struct Struct { /// Fields in this structure. @@ -172,21 +172,6 @@ impl Struct { self.fields.get_mut(name) } - /// Creates a [`Struct`] from a protobuf message. - /// - /// See [`Self::from_proto_descriptor_and_msg`] for details. - #[inline] - pub fn from_proto_msg( - msg: &dyn MessageDyn, - generate_fields_for_enums: bool, - ) -> Self { - Self::from_proto_descriptor_and_msg( - &msg.descriptor_dyn(), - Some(msg), - generate_fields_for_enums, - ) - } - /// Creates a [`Struct`] from a protobuf message descriptor. /// /// Receives the [`MessageDescriptor`] corresponding to the protobuf From c2cfba22641ef1a349f4644de9cc794ccd8750f7 Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Mon, 19 Feb 2024 17:52:43 +0100 Subject: [PATCH 52/55] chore: add `protobuf` package to `go.mod` --- go/go.mod | 1 + go/go.sum | 2 ++ go/scanner.go | 1 - 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go/go.mod b/go/go.mod index 65294f96b..bbc09e8b1 100644 --- a/go/go.mod +++ b/go/go.mod @@ -7,5 +7,6 @@ require ( github.com/golang/protobuf v1.5.3 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/stretchr/testify v1.8.4 // indirect + google.golang.org/protobuf v1.32.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go/go.sum b/go/go.sum index af49a2ab8..6d46dc46d 100644 --- a/go/go.sum +++ b/go/go.sum @@ -11,6 +11,8 @@ github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXl golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= +google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/go/scanner.go b/go/scanner.go index 78836fc36..5aa2332d0 100644 --- a/go/scanner.go +++ b/go/scanner.go @@ -89,7 +89,6 @@ func (s *Scanner) Timeout(timeout time.Duration) { var ErrTimeout = errors.New("timeout") - // SetModuleOutput sets the output data for a YARA module. // // Each YARA module generates an output consisting of a data structure that From ad33ea67e14efbe4129173c1c8ae57ac5c687a3d Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Mon, 19 Feb 2024 18:09:09 +0100 Subject: [PATCH 53/55] fix(go): add required type cast --- go/scanner.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/scanner.go b/go/scanner.go index 5aa2332d0..8a8e5cb57 100644 --- a/go/scanner.go +++ b/go/scanner.go @@ -133,7 +133,7 @@ func (s *Scanner) SetModuleOutput(data proto.Message) error { ptr = (*C.uint8_t)(unsafe.Pointer(&(buf[0]))) } - name := C.CString(proto.MessageReflect(data).Descriptor().FullName()) + name := C.CString(string(proto.MessageReflect(data).Descriptor().FullName())) defer C.free(unsafe.Pointer(name)) if r := C.yrx_scanner_set_module_output(s.cScanner, name, ptr, C.size_t(len(buf))); r != C.SUCCESS { From 0dbc8a7d9d868ad1c04ea005d815182e239516f9 Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Mon, 19 Feb 2024 18:09:48 +0100 Subject: [PATCH 54/55] docs: add example in `yara_x::mods` --- lib/src/modules/mod.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lib/src/modules/mod.rs b/lib/src/modules/mod.rs index b18ce5c70..3c7416053 100644 --- a/lib/src/modules/mod.rs +++ b/lib/src/modules/mod.rs @@ -130,6 +130,14 @@ pub mod mods { This allows external projects to benefit from YARA's file-parsing capabilities for their own purposes. + + # Example + + ```rust + # use yara_x; + # let data = &[]; + let pe_info = yara_x::mods::invoke_mod::(data); + ``` */ /// Data structure returned by the `dotnet` module. From 40d982ffad26daa59f4be11428e20f75a84d39be Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Mon, 19 Feb 2024 20:54:26 +0100 Subject: [PATCH 55/55] chore: fix typos --- lib/src/compiler/base64.rs | 8 ++++---- lib/src/lib.rs | 2 +- lib/src/modules/mod.rs | 2 +- lib/src/re/fast/compiler.rs | 17 ++++++++--------- lib/src/re/mod.rs | 2 +- 5 files changed, 15 insertions(+), 16 deletions(-) diff --git a/lib/src/compiler/base64.rs b/lib/src/compiler/base64.rs index ce207bd7b..330dad5f2 100644 --- a/lib/src/compiler/base64.rs +++ b/lib/src/compiler/base64.rs @@ -30,14 +30,14 @@ use bstr::{BString, ByteVec}; /// /// `LUxvcmVtIGlwc3VtIGRvbG9yIHNpdCBhbWV0` /// -/// The good news is that there are only 3 string that we need to look for: -/// `aXBzdW`, `lwc3Vt` or `pcHN1b`. If some string `S` contains "ipsum", the +/// The good news is that there are only 3 strings that we need to look for: +/// `aXBzdW`, `lwc3Vt` and `pcHN1b`. If some string `S` contains "ipsum", the /// result of `base64(S)` must contain one of these three patterns. /// -/// These three patterns are the results of `base64(S)`, `base64("X" + S)` +/// These three patterns are the result of `base64(S)`, `base64("X" + S)` /// and `base64("XX" + S)`, after removing the characters that are affected /// by the "X" at the beginning of the pattern, or that could be affected -/// if more bytes bytes are added after "ipsum". +/// if more bytes are added after "ipsum". /// /// padding + S base64(padding + S) final pattern /// "ipsum" -> "aXBzdW0" -> "aXBzdW" diff --git a/lib/src/lib.rs b/lib/src/lib.rs index 51e666fb4..d1c66b0e8 100644 --- a/lib/src/lib.rs +++ b/lib/src/lib.rs @@ -79,7 +79,7 @@ mod wasm; mod tests; mod utils { - /// Tries to match `target` as an the enum variant `pat`. Returns the + /// Tries to match `target` as the enum variant `pat`. Returns the /// inner value contained in the variant, or panics if `target` does /// not match `pat`. /// diff --git a/lib/src/modules/mod.rs b/lib/src/modules/mod.rs index 3c7416053..a013437cd 100644 --- a/lib/src/modules/mod.rs +++ b/lib/src/modules/mod.rs @@ -34,7 +34,7 @@ pub(crate) struct Module { /// module (e.g: "test_proto2"). pub rust_module_name: Option<&'static str>, /// A [`MessageDescriptor`] that describes the module's structure. This - /// corresponds to the the protobuf message declared in the "root_message" + /// corresponds to the protobuf message declared in the "root_message" /// for the YARA module. It allows iterating the fields declared by the /// module and obtaining their names and types. pub root_struct_descriptor: MessageDescriptor, diff --git a/lib/src/re/fast/compiler.rs b/lib/src/re/fast/compiler.rs index 248500cde..2f522ed90 100644 --- a/lib/src/re/fast/compiler.rs +++ b/lib/src/re/fast/compiler.rs @@ -184,8 +184,8 @@ impl Compiler { } // If the pattern was decomposed into more than one piece its atoms can - // not be exact. The atoms could be exact if the pattern was is a single - // piece and it fits completely in the atom. + // not be exact. The atoms could be exact if the pattern was a single + // piece, and it fits completely in the atom. if pieces.len() > 1 { for atom in atoms.iter_mut() { atom.set_exact(false); @@ -274,16 +274,15 @@ struct PatternSplitter { impl PatternSplitter { fn finish_literal(&mut self) -> Option { - // If the `bytes` is empty return `None` because empty literals - // are ignored, except when when we are inside an alternation, - // where empty literals are accepted in cases like `(abc|)`. + // If the `bytes` is empty return `None` because empty literals are + // ignored, except when we are inside an alternation, where empty + // literals are accepted in cases like `(abc|)`. if !self.in_alternation && self.bytes.is_empty() { return None; } - // If all bytes in the mask are 0xff the piece is a - // Literal and the mask is not necessary, if not, - // the piece is a MaskedLiteral. In both cases the mask - // and the bytes are reset to empty vectors. + // If all bytes in the mask are 0xff the piece is a Literal and the + // mask is not necessary, if not, the piece is a MaskedLiteral. In both + // cases the mask and the bytes are reset to empty vectors. if self.mask.iter().all(|&b| b == 0xff) { self.mask.clear(); Some(Pattern::Literal(mem::take(&mut self.bytes))) diff --git a/lib/src/re/mod.rs b/lib/src/re/mod.rs index 5861f751d..aae1d141b 100644 --- a/lib/src/re/mod.rs +++ b/lib/src/re/mod.rs @@ -3,7 +3,7 @@ The parsing of regular expressions is actually done by the [`regex-syntax`][1] crate, which produces a high-level intermediate representation (HIR) for a given regular expression in text form. This crates provides its own [`hir::Hir`] -type, but is is just a thin wrapper around the [`regex_syntax::hir::Hir`] type. +type, but is just a thin wrapper around the [`regex_syntax::hir::Hir`] type. Both regexp patterns and hex patterns are converted into a [`hir::Hir`], as every YARA hex pattern can be boiled down to a regular expression. Both kinds