diff --git a/.github/workflows/golang.yaml b/.github/workflows/golang.yaml index e56070799..7cc29a26e 100644 --- a/.github/workflows/golang.yaml +++ b/.github/workflows/golang.yaml @@ -14,7 +14,7 @@ jobs: strategy: fail-fast: false matrix: - go-version: [ '1.19', '1.20', '1.21.x' ] + go-version: [ '1.19', '1.20', '1.21.x', '1.22.x' ] os: [ ubuntu-latest, macos-latest ] runs-on: ${{ matrix.os }} steps: diff --git a/.github/workflows/python.yaml b/.github/workflows/python.yaml index 01b98550f..3e78e3fae 100644 --- a/.github/workflows/python.yaml +++ b/.github/workflows/python.yaml @@ -14,7 +14,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [ 3.8, 3.9, "3.10", "3.11" ] + python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12" ] os: [ ubuntu-latest, macos-latest, windows-latest ] runs-on: ${{ matrix.os }} steps: @@ -34,7 +34,7 @@ jobs: python -m pip install pytest maturin develop --manifest-path py/Cargo.toml pytest py - + # Non-windows - name: Test Python - Non-Windows if: runner.os != 'Windows' diff --git a/Cargo.lock b/Cargo.lock index 6f9610528..03ace986d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -574,9 +574,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.7" +version = "4.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5db83dced34638ad474f39f250d7fea9598bdd239eaced1bdf45d597da0f433f" +checksum = "84b3edb18336f4df585bc9aa31dd99c036dfa5dc5e9a2939a722a188f3a8970d" dependencies = [ "clap_builder", "clap_derive", @@ -584,9 +584,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.7" +version = "4.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7e204572485eb3fbf28f871612191521df159bc3e15a9f5064c66dba3a8c05f" +checksum = "c1c09dd5ada6c6c78075d6fd0da3f90d8080651e2d6cc8eb2f1aaa4034ced708" dependencies = [ "anstream", "anstyle", @@ -596,18 +596,18 @@ dependencies = [ [[package]] name = "clap_complete" -version = "4.5.5" +version = "4.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2020fa13af48afc65a9a87335bda648309ab3d154cd03c7ff95b378c7ed39c4" +checksum = "1d598e88f6874d4b888ed40c71efbcbf4076f1dfbae128a08a8c9e45f710605d" dependencies = [ - "clap 4.5.7", + "clap 4.5.8", ] [[package]] name = "clap_derive" -version = "4.5.5" +version = "4.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c780290ccf4fb26629baa7a1081e68ced113f1d3ec302fa5948f1c381ebf06c6" +checksum = "2bac35c6dafb060fd4d275d9a4ffae97917c13a6327903a8be2153cd964f7085" dependencies = [ "heck 0.5.0", "proc-macro2", @@ -2062,9 +2062,9 @@ checksum = "9374ef4228402d4b7e403e5838cb880d9ee663314b0a900d5a6aabf0c213552e" [[package]] name = "log" -version = "0.4.21" +version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" +checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" [[package]] name = "lzma-rs" @@ -4793,7 +4793,7 @@ checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" [[package]] name = "yara-x" -version = "0.4.0" +version = "0.5.0" dependencies = [ "aho-corasick", "anyhow", @@ -4862,7 +4862,7 @@ dependencies = [ [[package]] name = "yara-x-capi" -version = "0.4.0" +version = "0.5.0" dependencies = [ "cbindgen", "yara-x", @@ -4870,12 +4870,12 @@ dependencies = [ [[package]] name = "yara-x-cli" -version = "0.4.0" +version = "0.5.0" dependencies = [ "anyhow", "ascii_tree", "chardetng", - "clap 4.5.7", + "clap 4.5.8", "clap_complete", "colored_json", "crossbeam", @@ -4901,7 +4901,7 @@ dependencies = [ [[package]] name = "yara-x-fmt" -version = "0.4.0" +version = "0.5.0" dependencies = [ "anyhow", "bitmask", @@ -4913,7 +4913,7 @@ dependencies = [ [[package]] name = "yara-x-macros" -version = "0.4.0" +version = "0.5.0" dependencies = [ "convert_case", "darling", @@ -4924,7 +4924,7 @@ dependencies = [ [[package]] name = "yara-x-parser" -version = "0.4.0" +version = "0.5.0" dependencies = [ "annotate-snippets", "ascii_tree", @@ -4946,7 +4946,7 @@ dependencies = [ [[package]] name = "yara-x-proto" -version = "0.4.0" +version = "0.5.0" dependencies = [ "protobuf", "protobuf-codegen", @@ -4955,7 +4955,7 @@ dependencies = [ [[package]] name = "yara-x-proto-yaml" -version = "0.4.0" +version = "0.5.0" dependencies = [ "chrono", "globwalk", @@ -4970,7 +4970,7 @@ dependencies = [ [[package]] name = "yara-x-py" -version = "0.4.0" +version = "0.5.0" dependencies = [ "protobuf-json-mapping", "pyo3", diff --git a/Cargo.toml b/Cargo.toml index 53da965e6..f180702db 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace.package] -version = "0.4.0" +version = "0.5.0" authors = ["Victor M. Alvarez "] edition = "2021" homepage = "https://github.com/VirusTotal/yara-x" @@ -31,7 +31,7 @@ resolver = "2" [workspace.dependencies] aho-corasick = "1.1.3" -annotate-snippets = "0.11.2" +annotate-snippets = "0.11.4" anyhow = "1.0.86" array-bytes = "6.2.3" ascii_tree = "0.1.1" @@ -42,8 +42,8 @@ bitvec = "1.0.1" bstr = "1.9.1" cbindgen = "0.26.0" chrono = "0.4.38" -clap = "4.5.4" -clap_complete = "4.5.2" +clap = "4.5.7" +clap_complete = "4.5.7" const-oid = "0.9.6" crc32fast = "1.4.2" der-parser = "9.0.0" @@ -62,13 +62,13 @@ intaglio = "1.9.1" itertools = "0.13.0" lazy_static = "1.4.0" line-span = "0.1.5" -linkme = "0.3.26" -log = "0.4.21" +linkme = "0.3.27" +log = "0.4.22" magic = "0.16.2" md2 = "0.10.2" md-5 = "0.10.6" -memchr = "2.7.2" -memx = "0.1.30" +memchr = "2.7.4" +memx = "0.1.32" nom = "7.1.3" num-traits = "0.2.19" num-derive = "0.4.2" @@ -83,8 +83,8 @@ protobuf-json-mapping = "3.5.0" protobuf-parse = "3.5.0" protobuf-support = "3.5.0" rayon = "1.10.0" -regex-syntax = "0.8.3" -regex-automata = "0.4.6" +regex-syntax = "0.8.4" +regex-automata = "0.4.7" roxmltree = "0.20.0" rsa = "0.9.6" rustc-hash = "2.0.0" @@ -104,11 +104,11 @@ x509-parser = "0.16.0" yaml-rust = "0.4.5" yansi = "1.0.1" yara-x = { path = "lib" } -yara-x-fmt = { path = "fmt", version = "0.4.0" } -yara-x-macros = { path = "macros", version = "0.4.0" } -yara-x-parser = { path = "parser", version = "0.4.0" } -yara-x-proto = { path = "proto", version = "0.4.0" } -yara-x-proto-yaml = { path = "proto-yaml", version = "0.4.0" } +yara-x-fmt = { path = "fmt", version = "0.5.0" } +yara-x-macros = { path = "macros", version = "0.5.0" } +yara-x-parser = { path = "parser", version = "0.5.0" } +yara-x-proto = { path = "proto", version = "0.5.0" } +yara-x-proto-yaml = { path = "proto-yaml", version = "0.5.0" } zip = "2.1.1" # Special profile that builds a release binary with link-time optimization. diff --git a/go/compiler_test.go b/go/compiler_test.go index daaea261f..9ffe0c295 100644 --- a/go/compiler_test.go +++ b/go/compiler_test.go @@ -42,7 +42,7 @@ func TestRelaxedReSyntax(t *testing.T) { func TestErrorOnSlowPattern(t *testing.T) { _, err := Compile(` - rule test { strings: $a = /a.*b/ condition: $a }`, + rule test { strings: $a = /a.*/ condition: $a }`, ErrorOnSlowPattern(true)) assert.Error(t, err) } @@ -107,7 +107,7 @@ func TestVariables(t *testing.T) { func TestError(t *testing.T) { _, err := Compile("rule test { condition: foo }") - assert.EqualError(t, err, `error: unknown identifier `+"`foo`"+` + assert.EqualError(t, err, `error[E107]: unknown identifier `+"`foo`"+` --> line:1:24 | 1 | rule test { condition: foo } diff --git a/lib/src/scanner/context.rs b/lib/src/scanner/context.rs index 02632e5bb..8cb7c0786 100644 --- a/lib/src/scanner/context.rs +++ b/lib/src/scanner/context.rs @@ -58,8 +58,10 @@ pub(crate) struct ScanContext<'r> { /// `matching_rules` map, and then moved to this vector once the scan /// finishes. pub private_matching_rules: Vec, - /// Map containing the IDs of rules that matched. - pub matching_rules: FxHashMap>, + /// Map containing the IDs of rules that matched. Using an `IndexMap` + /// because we want to keep the insertion order, so that rules in + /// namespaces that were declared first, appear first in scan results. + pub matching_rules: IndexMap>, /// Compiled rules for this scan. pub compiled_rules: &'r Rules, /// Structure that contains top-level symbols, like module names diff --git a/lib/src/scanner/mod.rs b/lib/src/scanner/mod.rs index 3d34a6700..46ac45da3 100644 --- a/lib/src/scanner/mod.rs +++ b/lib/src/scanner/mod.rs @@ -151,7 +151,7 @@ impl<'r> Scanner<'r> { scanned_data_len: 0, private_matching_rules: Vec::new(), non_private_matching_rules: Vec::new(), - matching_rules: FxHashMap::default(), + matching_rules: IndexMap::new(), main_memory: None, module_outputs: FxHashMap::default(), user_provided_module_outputs: FxHashMap::default(), @@ -1020,7 +1020,7 @@ impl<'a, 'r> Metadata<'a, 'r> { /// condition: /// true /// } - /// "#).unwrap(); + /// "#).unwrap(); /// /// let mut scanner = yara_x::Scanner::new(&rules); /// diff --git a/lib/src/scanner/tests.rs b/lib/src/scanner/tests.rs index a495cd42d..f18c0235e 100644 --- a/lib/src/scanner/tests.rs +++ b/lib/src/scanner/tests.rs @@ -52,7 +52,7 @@ fn matches() { $c = "baz" condition: any of them - } + } "#, ) .unwrap(); @@ -90,7 +90,7 @@ fn metadata() { quux = "qu\x00x" condition: true - } + } "#, ) .unwrap(); @@ -138,7 +138,7 @@ fn xor_matches() { $a = "mississippi" xor condition: $a - } + } "#, ) .unwrap(); @@ -172,7 +172,7 @@ fn reuse_scanner() { rule test { condition: test_proto2.file_size == 3 - } + } "#, ) .unwrap(); @@ -214,7 +214,7 @@ fn module_output() { rule test { condition: test_proto2.file_size == 3 - } + } "#, ) .unwrap(); @@ -241,7 +241,7 @@ fn module_outputs() { rule test { condition: test_proto2.file_size == 3 - } + } "#, ) .unwrap(); @@ -276,7 +276,7 @@ fn variables_1() { rule test { condition: bool_var - } + } "#, ) .unwrap(); @@ -346,7 +346,7 @@ fn variables_2() { condition: some_bool and some_str == "foo" - } + } "#, ) .unwrap(); @@ -402,7 +402,7 @@ fn global_rules() { .add_source( r#" // This rule is always true. - private rule const_true { + private rule const_true { condition: true } @@ -429,7 +429,7 @@ fn global_rules() { .new_namespace("matching") .add_source( r#" - // This rule matches because it is in separate namespace not + // This rule matches because it is in separate namespace not // which is not affected by the global rule. rule matching { condition: @@ -622,3 +622,27 @@ fn set_module_output() { let scan_results = scanner.scan(b"").expect("scan should not fail"); assert_eq!(scan_results.matching_rules().len(), 1); } + +#[test] +fn namespaces() { + let mut compiler = crate::Compiler::new(); + + compiler + .new_namespace("foo") + .add_source(r#"rule foo {strings: $foo = "foo" condition: $foo }"#) + .unwrap() + .new_namespace("bar") + .add_source(r#"rule bar {strings: $bar = "bar" condition: $bar }"#) + .unwrap(); + + let rules = compiler.build(); + let mut scanner = Scanner::new(&rules); + let scan_results = scanner.scan(b"foobar").expect("scan should not fail"); + let matching_rules: Vec<_> = scan_results.matching_rules().collect(); + + assert_eq!(matching_rules.len(), 2); + assert_eq!(matching_rules[0].identifier(), "foo"); + assert_eq!(matching_rules[0].namespace(), "foo"); + assert_eq!(matching_rules[1].identifier(), "bar"); + assert_eq!(matching_rules[1].namespace(), "bar"); +} diff --git a/py/tests/test_api.py b/py/tests/test_api.py index 3aa7b5de9..e1a02d61d 100644 --- a/py/tests/test_api.py +++ b/py/tests/test_api.py @@ -24,9 +24,9 @@ def test_relaxed_re_syntax(): def test_error_on_slow_pattern(): - compiler = yara_x.Compiler(error_on_slow_pattern=True) - with pytest.raises(yara_x.CompileError): - compiler.add_source(r'rule test {strings: $a = /a.*b/ condition: $a}') + compiler = yara_x.Compiler(error_on_slow_pattern=True) + with pytest.raises(yara_x.CompileError): + compiler.add_source(r'rule test {strings: $a = /a.*/ condition: $a}') def test_int_globals(): @@ -117,7 +117,7 @@ def test_namespaces(): def test_metadata(): - rules = yara_x.compile(''' + rules = yara_x.compile(''' rule test { meta: foo = 1 @@ -130,17 +130,17 @@ def test_metadata(): } ''') - matching_rules = rules.scan(b'').matching_rules - - assert matching_rules[0].metadata == ( - ("foo", 1), - ("bar", 2.0), - ("baz", True), - ("qux", "qux"), - ("quux", "qu\0x") - ) - - + matching_rules = rules.scan(b'').matching_rules + + assert matching_rules[0].metadata == ( + ("foo", 1), + ("bar", 2.0), + ("baz", True), + ("qux", "qux"), + ("quux", "qu\0x") + ) + + def test_compile_and_scan(): rules = yara_x.compile('rule foo {strings: $a = "foo" condition: $a}') matching_rules = rules.scan(b'foobar').matching_rules