Merge branch 'VirusTotal:main' into numeric_underscores

Cargo.toml

@@ -43,7 +43,6 @@ chrono = "0.4.34"
 clap = "4.4.8"
 crc32fast = "1.3.2"
 criterion = "0.5.1"
-cryptographic-message-syntax = "0.26.0"
 enable-ansi-support = "0.2.1"
 env_logger = "0.11.1"
 fmmap = "0.3.2"
@@ -71,12 +70,12 @@ num-derive = "0.4.1"
 pest = "2.7.5"
 pest_derive = "2.7.5"
 pretty_assertions = "1.4.0"
-protobuf = { git = "https://github.com/plusvic/rust-protobuf.git", rev="b484d8a7" }
-protobuf-codegen = { git = "https://github.com/plusvic/rust-protobuf.git", rev="b484d8a7" }
-protobuf-json-mapping = { git = "https://github.com/plusvic/rust-protobuf.git", rev="b484d8a7" }
-protobuf-parse = { git = "https://github.com/plusvic/rust-protobuf.git", rev="b484d8a7" }
-regex-syntax = { git = "https://github.com/plusvic/regex.git", rev="423493d" }
-regex-automata = { git = "https://github.com/plusvic/regex.git", rev="423493d" }
+protobuf = { git = "https://github.com/plusvic/rust-protobuf.git", rev = "b484d8a7" }
+protobuf-codegen = { git = "https://github.com/plusvic/rust-protobuf.git", rev = "b484d8a7" }
+protobuf-json-mapping = { git = "https://github.com/plusvic/rust-protobuf.git", rev = "b484d8a7" }
+protobuf-parse = { git = "https://github.com/plusvic/rust-protobuf.git", rev = "b484d8a7" }
+regex-syntax = { git = "https://github.com/plusvic/regex.git", rev = "423493d" }
+regex-automata = { git = "https://github.com/plusvic/regex.git", rev = "423493d" }
 roxmltree = "0.19.0"
 rustc-hash = "1.1.0"
 smallvec = "1.10.0"

capi/include/yara-x.h

@@ -294,4 +294,24 @@ enum YRX_RESULT yrx_scanner_set_module_output(struct YRX_SCANNER *scanner,
                                               const uint8_t *data,
                                               size_t len);
 
+// Sets the value of a global variable of type string.
+enum YRX_RESULT yrx_scanner_set_global_str(struct YRX_SCANNER *scanner,
+                                           const char *ident,
+                                           const char *value);
+
+// Sets the value of a global variable of type bool.
+enum YRX_RESULT yrx_scanner_set_global_bool(struct YRX_SCANNER *scanner,
+                                            const char *ident,
+                                            bool value);
+
+// Sets the value of a global variable of type int.
+enum YRX_RESULT yrx_scanner_set_global_int(struct YRX_SCANNER *scanner,
+                                           const char *ident,
+                                           int64_t value);
+
+// Sets the value of a global variable of type float.
+enum YRX_RESULT yrx_scanner_set_global_float(struct YRX_SCANNER *scanner,
+                                             const char *ident,
+                                             double value);
+
 #endif /* YARA_X */

capi/src/lib.rs

@@ -325,9 +325,11 @@ pub unsafe extern "C" fn yrx_buffer_destroy(buf: *mut YRX_BUFFER) {
 /// the most recent function was successfully.
 #[no_mangle]
 pub unsafe extern "C" fn yrx_last_error() -> *const c_char {
-    if let Some(last_error) = LAST_ERROR.with(|_| None::<CString>) {
-        last_error.as_ptr()
-    } else {
-        std::ptr::null()
-    }
+    LAST_ERROR.with_borrow(|last_error| {
+        if let Some(last_error) = last_error {
+            last_error.as_ptr()
+        } else {
+            std::ptr::null()
+        }
+    })
 }

capi/src/scanner.rs

@@ -215,6 +215,82 @@ pub unsafe extern "C" fn yrx_scanner_set_module_output(
     }
 }
 
+unsafe extern "C" fn yrx_scanner_set_global<
+    T: TryInto<yara_x::Variable, Error = yara_x::VariableError>,
+>(
+    scanner: *mut YRX_SCANNER,
+    ident: *const c_char,
+    value: T,
+) -> YRX_RESULT {
+    if scanner.is_null() {
+        return YRX_RESULT::INVALID_ARGUMENT;
+    }
+
+    let ident = match CStr::from_ptr(ident).to_str() {
+        Ok(ident) => ident,
+        Err(_) => return YRX_RESULT::INVALID_ARGUMENT,
+    };
+
+    let scanner = scanner.as_mut().unwrap();
+
+    match scanner.inner.set_global(ident, value) {
+        Ok(_) => {
+            LAST_ERROR.set(None);
+            YRX_RESULT::SUCCESS
+        }
+        Err(err) => {
+            LAST_ERROR.set(Some(CString::new(err.to_string()).unwrap()));
+            YRX_RESULT::SCAN_ERROR
+        }
+    }
+}
+
+/// Sets the value of a global variable of type string.
+#[no_mangle]
+pub unsafe extern "C" fn yrx_scanner_set_global_str(
+    scanner: *mut YRX_SCANNER,
+    ident: *const c_char,
+    value: *const c_char,
+) -> YRX_RESULT {
+    let value = if let Ok(value) = CStr::from_ptr(value).to_str() {
+        value
+    } else {
+        return YRX_RESULT::INVALID_ARGUMENT;
+    };
+
+    yrx_scanner_set_global(scanner, ident, value)
+}
+
+/// Sets the value of a global variable of type bool.
+#[no_mangle]
+pub unsafe extern "C" fn yrx_scanner_set_global_bool(
+    scanner: *mut YRX_SCANNER,
+    ident: *const c_char,
+    value: bool,
+) -> YRX_RESULT {
+    yrx_scanner_set_global(scanner, ident, value)
+}
+
+/// Sets the value of a global variable of type int.
+#[no_mangle]
+pub unsafe extern "C" fn yrx_scanner_set_global_int(
+    scanner: *mut YRX_SCANNER,
+    ident: *const c_char,
+    value: i64,
+) -> YRX_RESULT {
+    yrx_scanner_set_global(scanner, ident, value)
+}
+
+/// Sets the value of a global variable of type float.
+#[no_mangle]
+pub unsafe extern "C" fn yrx_scanner_set_global_float(
+    scanner: *mut YRX_SCANNER,
+    ident: *const c_char,
+    value: f64,
+) -> YRX_RESULT {
+    yrx_scanner_set_global(scanner, ident, value)
+}
+
 unsafe fn slice_from_ptr_and_len<'a>(
     data: *const u8,
     len: usize,

capi/src/tests.rs

@@ -7,7 +7,7 @@ use crate::compiler::{
 use crate::{
     yrx_buffer_destroy, yrx_rules_deserialize, yrx_rules_serialize,
     yrx_scanner_create, yrx_scanner_destroy, yrx_scanner_on_matching_rule,
-    yrx_scanner_scan, YRX_BUFFER, YRX_RULE,
+    yrx_scanner_scan, yrx_scanner_set_global_bool, YRX_BUFFER, YRX_RULE,
 };
 use std::ffi::{c_void, CString};
 
@@ -75,8 +75,16 @@ fn capi() {
         );
 
         yrx_scanner_scan(scanner, std::ptr::null(), 0);
-        yrx_scanner_destroy(scanner);
-
         assert_eq!(matches, 1);
+
+        matches = 0;
+
+        // After changing the value of `some_bool` to false, the rule doesn't
+        // match anymore.
+        yrx_scanner_set_global_bool(scanner, some_bool.as_ptr(), false);
+        yrx_scanner_scan(scanner, std::ptr::null(), 0);
+        assert_eq!(matches, 0);
+
+        yrx_scanner_destroy(scanner);
     }
 }

go/compiler_test.go

@@ -71,11 +71,18 @@ func TestVariables(t *testing.T) {
 	assert.Len(t, matchingRules, 1)
 
 	err := c.DefineGlobal("var", struct{}{})
-	assert.Errorf(t, err, "variable `var` has unsupported type: struct{}")
+	assert.EqualError(t, err, "variable `var` has unsupported type: struct {}")
 }
 
 func TestError(t *testing.T) {
 	c := NewCompiler()
 	err := c.AddSource("rule test { condition: foo }")
-	assert.Error(t, err)
+	assert.EqualError(t, err, `error: unknown identifier `+"`foo`"+`
+   ╭─[line:1:24]
+   │
+ 1 │ rule test { condition: foo }
+   │                        ─┬─  
+   │                         ╰─── this identifier has not been declared
+───╯
+`)
 }

go/scanner.go

@@ -3,6 +3,7 @@ package yara_x
 import "C"
 import (
 	"errors"
+	"fmt"
 	"math"
 	"runtime"
 	"runtime/cgo"
@@ -89,6 +90,46 @@ func (s *Scanner) Timeout(timeout time.Duration) {
 
 var ErrTimeout = errors.New("timeout")
 
+// SetGlobal sets the value of a global variable.
+//
+// The variable must has been previously defined by calling Compiler.DefineGlobal
+// and the type it has during the definition must match the type of the new
+// value.
+//
+// The variable will retain the new value in subsequent scans, unless this
+// function is called again for setting a new value.
+func (s *Scanner) SetGlobal(ident string, value interface{}) error {
+	cIdent := C.CString(ident)
+	defer C.free(unsafe.Pointer(cIdent))
+	var ret C.int
+
+	runtime.LockOSThread()
+	defer runtime.UnlockOSThread()
+
+	switch v := value.(type) {
+	case int:
+		ret = C.int(C.yrx_scanner_set_global_int(s.cScanner, cIdent, C.int64_t(v)))
+	case bool:
+		ret = C.int(C.yrx_scanner_set_global_bool(s.cScanner, cIdent, C.bool(v)))
+	case string:
+		cValue := C.CString(v)
+		defer C.free(unsafe.Pointer(cValue))
+		ret = C.int(C.yrx_scanner_set_global_str(s.cScanner, cIdent, cValue))
+	case float64:
+		ret = C.int(C.yrx_scanner_set_global_float(s.cScanner, cIdent, C.double(v)))
+	default:
+		return fmt.Errorf("variable `%s` has unsupported type: %T", ident, v)
+	}
+
+	runtime.KeepAlive(s)
+
+	if ret == C.VARIABLE_ERROR {
+		return errors.New(C.GoString(C.yrx_last_error()))
+	}
+
+	return nil
+}
+
 // SetModuleOutput sets the output data for a YARA module.
 //
 // Each YARA module generates an output consisting of a data structure that

go/scanner_test.go

@@ -37,6 +37,20 @@ func TestScanner2(t *testing.T) {
 	runtime.GC()
 }
 
+func TestScanner3(t *testing.T) {
+	r, _ := Compile(
+		`rule t { condition: var_bool }`,
+		GlobalVars(map[string]interface{}{"var_bool": true}))
+
+	s := NewScanner(r)
+	matchingRules, _ := s.Scan([]byte{})
+	assert.Len(t, matchingRules, 1)
+
+	s.SetGlobal("var_bool", false)
+	matchingRules, _ = s.Scan([]byte{})
+	assert.Len(t, matchingRules, 0)
+}
+
 func TestScannerTimeout(t *testing.T) {
 	r, _ := Compile("rule t { strings: $a = /a(.*)*a/ condition: $a }")
 	s := NewScanner(r)

lib/Cargo.toml

@@ -93,7 +93,6 @@ lnk-module = [
 # The `macho` module parses Mach-O files.
 macho-module = [
     "dep:nom",
-    "dep:cryptographic-message-syntax",
     "dep:roxmltree",
 ]
 
@@ -157,7 +156,6 @@ bitmask = { workspace = true }
 bitvec = { workspace = true }
 bstr = { workspace = true, features = ["serde"] }
 crc32fast = { workspace = true, optional = true }
-cryptographic-message-syntax = { workspace = true, optional = true }
 fmmap = { workspace = true }
 indexmap = { workspace = true, features = ["serde"] }
 intaglio = { workspace = true }
@@ -178,8 +176,8 @@ rustc-hash = { workspace = true }
 regex-syntax = { workspace = true }
 regex-automata = { workspace = true }
 roxmltree = { workspace = true, optional = true }
-smallvec = { workspace = true, features=["serde"] }
-serde = { workspace = true, features=["rc"] }
+smallvec = { workspace = true, features = ["serde"] }
+serde = { workspace = true, features = ["rc"] }
 serde_json = { workspace = true }
 thiserror = { workspace = true }
 tlsh-fixed = { workspace = true, optional = true }

lib/src/compiler/rules.rs

@@ -206,7 +206,7 @@ impl Rules {
         self.rules.get(rule_id.0 as usize).unwrap()
     }
 
-    /// Returns an slice with the individual rules that were compiled.
+    /// Returns a slice with the individual rules that were compiled.
     #[inline]
     pub(crate) fn rules(&self) -> &[RuleInfo] {
         self.rules.as_slice()
@@ -233,8 +233,8 @@ impl Rules {
         let re = types::Regexp::new(self.regexp_pool.get(regexp_id).unwrap());
 
         let mut parser = regex_syntax::ast::parse::ParserBuilder::new()
-            // This the custom configuration option that turns-on support for
-            // the `{,n}`. This option doesn't exist in the official
+            // This is the custom configuration option that turns-on support
+            // for the `{,n}` syntax. This option doesn't exist in the official
             // `regex_syntax` crate.
             .empty_min_range(true)
             .build();
@@ -249,7 +249,18 @@ impl Rules {
 
         let hir = translator.translate(re.naked(), &ast).unwrap();
 
-        regex_automata::meta::Builder::new().build_from_hir(&hir).unwrap()
+        // Set a size limit for the NFA automata. The default limit (10MB) is
+        // too small for certain regexps seen in YARA rules in the wild, see:
+        // https://github.com/VirusTotal/yara-x/issues/85
+        let config = regex_automata::meta::Config::new()
+            .nfa_size_limit(Some(50 * 1024 * 1024));
+
+        regex_automata::meta::Builder::new()
+            .configure(config)
+            .build_from_hir(&hir)
+            .unwrap_or_else(|err| {
+                panic!("error compiling regex `{}`: {:#?}", re.as_str(), err)
+            })
     }
 
     /// Returns a sub-pattern by [`SubPatternId`].