Merge branch 'main' into macho-symtab

.github/workflows/python.yaml

@@ -1,107 +1,15 @@
-# This file is autogenerated by maturin v1.0.1
-# To update, run
-#
-#    maturin generate-ci github
-#
 name: Python extension
 
 on:
   push:
     paths:
       - 'yara-x-py/**'
+      - '.github/workflows/python.yaml'
 
 permissions:
   contents: read
 
 jobs:
-  linux:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        target: [x86_64, aarch64]
-    steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v4
-        with:
-          python-version: "3.10"
-      - name: Build wheels
-        uses: PyO3/maturin-action@v1
-        with:
-          target: ${{ matrix.target }}
-          args: --release --out dist --find-interpreter
-          working-directory: yara-x-py
-          sccache: "true"
-          manylinux: auto
-      - name: Upload wheels
-        uses: actions/upload-artifact@v3
-        with:
-          name: wheels
-          path: dist
-
-  windows:
-    runs-on: windows-latest
-    strategy:
-      matrix:
-        target: [x64]
-    steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v4
-        with:
-          python-version: "3.10"
-          architecture: ${{ matrix.target }}
-      - name: Build wheels
-        uses: PyO3/maturin-action@v1
-        with:
-          target: ${{ matrix.target }}
-          args: --release --out dist --find-interpreter
-          working-directory: yara-x-py
-          sccache: "true"
-      - name: Upload wheels
-        uses: actions/upload-artifact@v3
-        with:
-          name: wheels
-          path: dist
-
-  macos:
-    runs-on: macos-latest
-    strategy:
-      matrix:
-        target: [x86_64, aarch64]
-    steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v4
-        with:
-          python-version: "3.10"
-      - name: Build wheels
-        uses: PyO3/maturin-action@v1
-        with:
-          target: ${{ matrix.target }}
-          args: --release --out dist --find-interpreter
-          working-directory: yara-x-py
-          sccache: "true"
-      - name: Upload wheels
-        uses: actions/upload-artifact@v3
-        with:
-          name: wheels
-          path: dist
-
-  release:
-    runs-on: ubuntu-latest
-    if: "startsWith(github.ref, 'refs/tags/')"
-    needs: [linux, windows, macos]
-    steps:
-      - uses: actions/download-artifact@v3
-        with:
-          name: wheels
-      - name: Publish to PyPI
-        uses: PyO3/maturin-action@v1
-        env:
-          MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_API_TOKEN }}
-        with:
-          command: upload
-          args: --skip-existing *
-          working-directory: yara-x-py
-
   test:
     strategy:
       fail-fast: false
@@ -115,22 +23,41 @@ jobs:
       - uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
-      - name: Test Python - Non-Windows
-        if: runner.os != 'Windows'
+
+      # Windows
+      - name: Install OpenSSL
+        if: runner.os == 'Windows'   
+        id: vcpkg
+        uses: johnwason/vcpkg-action@v5
+        with:
+          pkgs: openssl
+          triplet: x64-windows-static
+          token: ${{ github.token }}
+
+      - name: Set OPENSSL_DIR environment variable
+        if: runner.os == 'Windows'   
+        shell: bash
+        run: echo "OPENSSL_DIR=${{ github.workspace }}\\vcpkg\\installed\\x64-windows-static" >> $GITHUB_ENV
+
+      - name: Test Python - Windows
+        if: runner.os == 'Windows'
         run: |
           pip install virtualenv
-          virtualenv venv; source venv/bin/activate
+          virtualenv venv; venv\Scripts\activate.ps1
+          pwd
           python -m pip install --upgrade pip maturin
           python -m pip install pytest
           maturin develop --manifest-path yara-x-py/Cargo.toml
           pytest yara-x-py
-      - name: Test Python - Windows
-        if: runner.os == 'Windows'
+          
+      # Non-windows
+      - name: Test Python - Non-Windows
+        if: runner.os != 'Windows'
         run: |
           pip install virtualenv
-          virtualenv venv; venv\Scripts\activate.ps1
-          pwd
+          virtualenv venv; source venv/bin/activate
           python -m pip install --upgrade pip maturin
           python -m pip install pytest
           maturin develop --manifest-path yara-x-py/Cargo.toml
           pytest yara-x-py
+

.github/workflows/tests.yaml

@@ -56,7 +56,7 @@ jobs:
           - build: no-default-features
             os: ubuntu-latest
             rust: stable
-            args: "--package yara-x --no-default-features --features=test_proto2-module,test_proto3-module,time-module,hash-module,macho-module,math-module,lnk-module,elf-module,pe-module,dotnet-module"
+            args: "--package yara-x --no-default-features --features=test_proto2-module,test_proto3-module,time-module,hash-module,macho-module,math-module,lnk-module,elf-module,pe-module,dotnet-module,console-module"
 
     steps:
       - name: Checkout sources

Cargo.toml

@@ -74,6 +74,7 @@ pest_derive = "2.7.5"
 pretty_assertions = "1.4.0"
 protobuf = "3.3.0"
 protobuf-codegen = "3.3.0"
+protobuf-json-mapping = "3.3.0"
 protobuf-parse = "3.3.0"
 regex-syntax = { git = "https://github.com/plusvic/regex.git", rev="423493d" }
 regex-automata = { git = "https://github.com/plusvic/regex.git", rev="423493d" }

yara-x-cli/Cargo.toml

@@ -42,7 +42,7 @@ enable-ansi-support = { workspace = true }
 env_logger = { workspace = true , optional = true }
 log = { workspace = true, optional = true }
 protobuf = { workspace = true }
-protobuf-json-mapping = "3.3.0"
+protobuf-json-mapping = { workspace = true }
 serde_json = { workspace = true, features = ["preserve_order"] }
 yansi = { workspace = true }
 yara-x = { workspace = true }
@@ -53,7 +53,6 @@ yara-x-fmt = { workspace = true }
 colored_json = "4.0.0"
 crossbeam = "0.8.2"
 crossterm = "0.27.0"
-indent = "0.1.1"
 pprof = { version = "0.13.0", features = ["flamegraph"], optional=true }
 strum_macros = "0.25"
 superconsole = "0.2.0"

yara-x-cli/src/commands/check.rs

@@ -70,8 +70,8 @@ pub fn exec_check(args: &ArgMatches) -> anyhow::Result<()> {
     w.walk(
         rules_path,
         CheckState::new(),
-        || {},
-        |file_path, state, output, _| {
+        |_, _| {},
+        |state, output, file_path, _| {
             let src = fs::read(file_path.clone())
                 .with_context(|| {
                     format!("can not read `{}`", file_path.display())

yara-x-cli/src/commands/scan.rs

@@ -1,6 +1,5 @@
 use std::cmp::min;
 use std::fs::File;
-use std::io::Read;
 use std::path::{Path, PathBuf};
 use std::sync::atomic::{AtomicUsize, Ordering};
 use std::sync::Mutex;
@@ -9,7 +8,6 @@ use std::time::{Duration, Instant};
 use anyhow::{bail, Context, Error};
 use clap::{arg, value_parser, Arg, ArgAction, ArgMatches, Command};
 use crossbeam::channel::Sender;
-use indent::indent_all_by;
 use superconsole::style::Stylize;
 use superconsole::{Component, Line, Lines, Span};
 use yansi::Color::{Cyan, Red, Yellow};
@@ -49,8 +47,8 @@ pub fn scan() -> Command {
                 .value_parser(value_parser!(usize))
         )
         .arg(
-            arg!(-D --"dump-module-output")
-                .help("Dumps the data produced by modules")
+            arg!(--"disable-console-logs")
+                .help("Disable printing console log messages")
         )
         .arg(
             arg!(-n - -"negate")
@@ -104,7 +102,7 @@ pub fn exec_scan(args: &ArgMatches) -> anyhow::Result<()> {
     let path_as_namespace = args.get_flag("path-as-namespace");
     let skip_larger = args.get_one::<u64>("skip-larger");
     let negate = args.get_flag("negate");
-    let dump_module_output = args.get_flag("dump-module-output");
+    let disable_console_logs = args.get_flag("disable-console-logs");
     let timeout = args.get_one::<u64>("timeout");
 
     let mut external_vars: Option<Vec<(String, serde_json::Value)>> = args
@@ -121,16 +119,10 @@ pub fn exec_scan(args: &ArgMatches) -> anyhow::Result<()> {
 
         let rules_path = rules_path.next().unwrap();
 
-        let mut file = File::open(rules_path)
+        let file = File::open(rules_path)
             .with_context(|| format!("can not open {:?}", &rules_path))?;
 
-        let mut data = Vec::new();
-
-        File::read_to_end(&mut file, &mut data)
-            .with_context(|| format!("can not read {:?}", &rules_path))?;
-
-        // TODO: implement Rules::deserialize_from reader
-        let rules = Rules::deserialize(data.as_slice())?;
+        let rules = Rules::deserialize_from(file)?;
 
         // If the user is defining external variables, make sure that these
         // variables are valid. A scanner is created only with the purpose
@@ -174,18 +166,31 @@ pub fn exec_scan(args: &ArgMatches) -> anyhow::Result<()> {
     w.walk(
         path,
         state,
-        || {
+        // Initialization
+        |_, output| {
             let mut scanner = Scanner::new(rules_ref);
+
+            if !disable_console_logs {
+                let output = output.clone();
+                scanner.console_log(move |msg| {
+                    output
+                        .send(Message::Error(format!("{}", Yellow.paint(msg))))
+                        .unwrap();
+                });
+            }
+
             if let Some(ref vars) = external_vars {
                 for (ident, value) in vars {
                     // It's ok to use `unwrap()`, this can not fail because
                     // we already verified that external variables are correct.
                     scanner.set_global(ident.as_str(), value).unwrap();
                 }
             }
+
             scanner
         },
-        |file_path, state, output, scanner| {
+        // File handler. Called for every file found while walking the path.
+        |state, output, file_path, scanner| {
             let elapsed_time = Instant::elapsed(&start_time);
 
             if let Some(timeout) = timeout.checked_sub(elapsed_time) {
@@ -238,28 +243,11 @@ pub fn exec_scan(args: &ArgMatches) -> anyhow::Result<()> {
                 );
             };
 
-            if dump_module_output {
-                for (mod_name, mod_output) in scan_results.module_outputs() {
-                    output
-                        .send(Message::Info(format!(
-                            ">>> {} {}\n{}<<<",
-                            Yellow.paint(mod_name).bold(),
-                            file_path.display(),
-                            indent_all_by(
-                                4,
-                                protobuf::text_format::print_to_string_pretty(
-                                    mod_output,
-                                )
-                            ),
-                        )))
-                        .unwrap();
-                }
-            }
-
             state.num_scanned_files.fetch_add(1, Ordering::Relaxed);
 
             Ok(())
         },
+        // Error handler
         |err, output| {
             let _ = output.send(Message::Error(format!(
                 "{} {}: {}",
@@ -319,14 +307,17 @@ fn print_matching_rules(
             let limit = print_strings_limit.unwrap_or(&120);
             for p in matching_rule.patterns() {
                 for m in p.matches() {
+                    let match_range = m.range();
+                    let match_data = m.data();
+
                     let mut msg = format!(
                         "{:#x}:{}:{}: ",
-                        m.range.start,
-                        m.range.len(),
+                        match_range.start,
+                        match_range.len(),
                         p.identifier(),
                     );
 
-                    for b in &m.data[..min(m.data.len(), *limit)] {
+                    for b in &match_data[..min(match_data.len(), *limit)] {
                         for c in b.escape_ascii() {
                             msg.push_str(format!("{}", c as char).as_str());
                         }