Merge branch 'limit_json_chunks'

Cargo.toml

@@ -108,9 +108,9 @@ csv = "1.3"
 plist = "1.6"
 
 # jq
-jaq-core = "1.2"
-jaq-std = "1.2"
-jaq-interpret = "1.2"
+jaq-core = "1.4"
+jaq-std = "1.4"
+jaq-interpret = "1.4"
 jaq-parse = "1.0"
 jaq-syn = "1.1"
 

justfile

@@ -11,13 +11,6 @@ o________________INIT_COMMANDS: _default
 clean:
   cargo clean
 
-install_python_venv:
-  cd py_scnr && python3 -m venv .venv
-  cd py_scnr && pip install -r requirements.txt
-  # cd py_scnr && pip freeze > requirements.txt
-  echo "now call ---->" 
-  echo "source ./py_scnr/.venv/bin/activate"
-
 # execute all commands to check workspace health, if this command pass, CI should pass as well
 all: clean test check check_deny install
 
@@ -104,13 +97,30 @@ install:
 # ==================================================================================================
 o________________DEPS_COMMANDS: _default
 
-# Installs build tools & dependencies 
-install_tooling:
+# Installs cargo tools
+install_cargo_tools:
   cargo install cargo-deny
   cargo install --locked maturin
-  sudo apt install python3-venv
-  sudo apt install python3-pip
-
-
 
+# Installs python virtual env requirements
+install_python_venv:
+  cd py_scnr && python3 -m venv .venv
+  cd py_scnr && pip install -r requirements.txt
+  # cd py_scnr && pip freeze > requirements.txt
+  echo "now call ---->" 
+  echo "source ./py_scnr/.venv/bin/activate"
 
+# Installs build tools & dependencies
+[linux]
+install_tooling: install_cargo_tools && install_python_venv
+  sudo apt install python3-venv
+  sudo apt install python3-pip
+  pip install virtualenv
+
+# Installs build tools & dependencies
+[macos]
+install_tooling: install_cargo_tools && install_python_venv
+  brew install python pipx
+  pipx ensurepath
+  pipx install pip
+  pip install virtualenv

py_scnr/requirements.txt

@@ -1 +1,2 @@
 maturin==1.4.0
+virtualenv

py_scnr/src/iterators.rs

@@ -43,17 +43,13 @@ type JqInnerIterator = Box<dyn Iterator<Item = serde_json::Value> + Send>;
 
 impl JqIterator {
   pub fn new(result: ScanResult, query: &str) -> Result<Self, PyScnrError> {
-    let filter = scnr_core::jq::make_jq_filter(query)?;
+    let filter = scnr_core::jq::JqFilter::new(query)?;
 
     let iter = result
       .into_iter()
       .filter_map(|c| c.map_err(|e| tracing::error!("{e:?}")).ok())
       .filter_map(|c| c.content.json().map(|json| (c.rel_path, json)))
-      .flat_map(move |(_path, json)| {
-        scnr_core::jq::jq_from_filter(json, filter.clone())
-          .map_err(|e| tracing::error!("{e:?}"))
-          .unwrap_or_default()
-      });
+      .flat_map(move |(_path, json)| filter.run(json).map_err(|e| tracing::error!("{e:?}")).unwrap_or_default());
 
     Ok(Self { iter: Box::new(iter) })
   }

py_scnr/src/lib.rs

@@ -41,41 +41,47 @@ fn activate_verbose(verbose: bool) {
 }
 
 #[pyfunction]
-#[pyo3(signature = (*, input = DEFAULT_INPUT.to_string(), filter=vec![], starter=vec![], cfg=vec![], profile=CfgProfile::default(), verbose=false))]
+#[allow(clippy::too_many_arguments)]
+#[pyo3(signature = (*, input = DEFAULT_INPUT.to_string(), filter=vec![], starter=vec![], cfg=vec![], profile=CfgProfile::default(), print_file_names=false, pretty_print=false, verbose=false))]
 fn scan(
   input: String,
   filter: Vec<String>,
   starter: Vec<Plugin>,
   cfg: Vec<(String, Plugin)>,
   profile: CfgProfile,
+  print_file_names: bool,
+  pretty_print: bool,
   verbose: bool,
 ) -> Result<ScanResultIterator, PyScnrError> {
   activate_verbose(verbose);
   let starter = to_scnr_starter(starter);
   let cfg = to_scnr_cfg(cfg);
   let profile = profile.into();
-  let common = CommonArgs { input, filter, starter, cfg, profile };
+  let common = CommonArgs { input, filter, starter, cfg, profile, print_file_names, pretty_print };
   let scanner = scnr::get_scanner_from_options(&common)?;
   let result = scanner.scan()?;
   Ok(result.into())
 }
 
 #[pyfunction]
-#[pyo3(signature = (*, input = DEFAULT_INPUT.to_string(), query = DEFAULT_JQ_QUERY, filter=vec![], starter=vec![], cfg=vec![], profile=CfgProfile::default(), verbose=false))]
+#[allow(clippy::too_many_arguments)]
+#[pyo3(signature = (*, input = DEFAULT_INPUT.to_string(), query = DEFAULT_JQ_QUERY, filter=vec![], starter=vec![], cfg=vec![], profile=CfgProfile::default(), print_file_names=false, pretty_print=false, verbose=false))]
 fn jq(
   input: String,
   query: &str,
   filter: Vec<String>,
   starter: Vec<Plugin>,
   cfg: Vec<(String, Plugin)>,
   profile: CfgProfile,
+  print_file_names: bool,
+  pretty_print: bool,
   verbose: bool,
 ) -> Result<JqIterator, PyScnrError> {
   activate_verbose(verbose);
   let starter = to_scnr_starter(starter);
   let cfg = to_scnr_cfg(cfg);
   let profile = profile.into();
-  let common = CommonArgs { input, filter, starter, cfg, profile };
+  let common = CommonArgs { input, filter, starter, cfg, profile, print_file_names, pretty_print };
   let scanner = scnr::get_scanner_from_options(&common)?;
   let result = scanner.scan()?;
   let iterator = JqIterator::new(result, query)?;

rust-toolchain.toml

@@ -1,5 +1,5 @@
 [toolchain]
-channel = "1.76"
+channel = "stable"
 
 # https://rust-lang.github.io/rustup-components-history/
 components = ["rustfmt", "clippy"]

scnr/src/lib.rs

@@ -1,12 +1,13 @@
 use options::CommonArgs;
-use scnr_core::{filter::Glob, Scanner};
+use scnr_core::{filter::Glob, Scanner, ScannerOptions};
 
 pub mod options;
 pub mod profiles;
 pub use scnr_core as core;
 
 pub fn get_scanner_from_options(common_args: &CommonArgs) -> Result<Scanner, anyhow::Error> {
-  let picker = profiles::get_plugin_picker(common_args.profile, &common_args.cfg, &common_args.starter)?;
+  let options = ScannerOptions::default();
+  let picker = profiles::get_plugin_picker(common_args.profile, &common_args.cfg, &common_args.starter, &options)?;
   let scanner = Scanner::new(&common_args.input, picker);
   let scanner = config_scanner_filter(scanner, &common_args.filter)?;
   Ok(scanner)

scnr/src/main.rs

@@ -1,8 +1,8 @@
 #![allow(clippy::default_trait_access, clippy::module_name_repetitions, clippy::wildcard_imports)]
 #![deny(clippy::expect_used, clippy::unwrap_used, clippy::panic)]
 
-use scnr_core::{bin_repr, jq, Scanner};
-use std::io::Write;
+use scnr_core::{bin_repr, jq, Content, Scanner};
+use std::{io::Write, path::Path};
 
 use scnr::options::*;
 
@@ -26,6 +26,32 @@ fn main() -> anyhow::Result<()> {
   Ok(())
 }
 
+fn print_path(out: &mut impl Write, path: &Path, options: &CommonArgs) -> anyhow::Result<()> {
+  if options.print_file_names {
+    writeln!(out, "{}", path.display())?;
+  }
+  Ok(())
+}
+
+fn print_content(out: &mut impl Write, content: &Content, options: &CommonArgs) -> anyhow::Result<()> {
+  match &content {
+    scnr_core::Content::Json(json) => {
+      let consume_out = &mut *out;
+      if options.pretty_print {
+        serde_json::to_writer_pretty(consume_out, &json)?;
+      } else {
+        serde_json::to_writer(consume_out, &json)?;
+      }
+    }
+    scnr_core::Content::Text(text) => writeln!(out, "{text}")?,
+    scnr_core::Content::Bytes(bytes) => writeln!(out, "{}", bin_repr::BinRepr::Base64.to_string(bytes))?,
+  }
+
+  writeln!(out)?;
+
+  Ok(())
+}
+
 #[tracing::instrument(skip(scanner), err)]
 fn scan(scanner: Scanner, args: ScanArgs) -> anyhow::Result<()> {
   let stdout = std::io::stdout();
@@ -36,12 +62,8 @@ fn scan(scanner: Scanner, args: ScanArgs) -> anyhow::Result<()> {
   for content in iter {
     match content {
       Ok(content) => {
-        writeln!(lock, "{}", content.rel_path.display())?;
-        match content.content {
-          scnr_core::Content::Json(json) => serde_json::to_writer_pretty(&mut lock, &json)?,
-          scnr_core::Content::Text(text) => writeln!(lock, "{text}")?,
-          scnr_core::Content::Bytes(bytes) => writeln!(lock, "{}", bin_repr::BinRepr::Base64.to_string(&bytes))?,
-        }
+        print_path(&mut lock, &content.rel_path, &args.common)?;
+        print_content(&mut lock, &content.content, &args.common)?;
       }
       Err(err) => tracing::error!("{err:?}"),
     }
@@ -55,25 +77,24 @@ fn jq(scanner: Scanner, args: JqArgs) -> anyhow::Result<()> {
   let stdout = std::io::stdout();
   let mut lock = stdout.lock();
 
-  let jq_filter = jq::make_jq_filter(&args.query)?;
+  let jq_filter = jq::JqFilter::new(&args.query)?;
 
   let iter = scanner.scan()?;
 
   for content in iter {
     match content {
       Ok(content) => {
         if let Some(json) = content.content.json() {
-          for element in jq::jq_from_filter(json, jq_filter.clone())? {
-            serde_json::to_writer_pretty(&mut lock, &element)?;
+          print_path(&mut lock, &content.rel_path, &args.common)?;
+          for element in jq_filter.run(json)? {
+            print_content(&mut lock, &Content::Json(element), &args.common)?;
           }
         }
       }
       Err(err) => tracing::error!("{err:?}"),
     }
   }
 
-  writeln!(lock)?;
-
   Ok(())
 }
 
@@ -141,13 +162,13 @@ mod tests {
   #[test]
   fn sample_to_console() -> anyhow::Result<()> {
     let samples = get_samples_path()?;
-    test_scnr_scan_output(&format!("scnr scan -i {samples}"), 24, 7, 4, 2)
+    test_scnr_scan_output(&format!("scnr scan -i {samples}"), 40, 7, 4, 2)
   }
 
   #[test]
   fn sample_to_console_sysdiag_profil() -> anyhow::Result<()> {
     let samples = get_samples_path()?;
-    test_scnr_scan_output(&format!("scnr scan -i {samples} -p sysdiagnose"), 39, 7, 1, 3)
+    test_scnr_scan_output(&format!("scnr scan -i {samples} -p sysdiagnose"), 55, 7, 1, 3)
   }
 
   fn test_scnr_scan_output(

scnr/src/options.rs

@@ -47,11 +47,25 @@ pub struct CommonArgs {
 
   #[arg(short, long, default_value_t = CfgProfile::default(),  help = "Plugins configuration profile to start with. Profiles are cfg bundles and can be then overridden by cfg args")]
   pub profile: CfgProfile,
+
+  #[arg(long, short = 'n', help = "DO print the file names (before the content)")]
+  pub print_file_names: bool,
+
+  #[arg(long, short = 'b', help = "DO pretty(beautiful) print the output")]
+  pub pretty_print: bool,
 }
 
 impl Default for CommonArgs {
   fn default() -> Self {
-    CommonArgs { input: DEFAULT_INPUT.to_string(), filter: vec![], profile: CfgProfile::default(), cfg: vec![], starter: vec![] }
+    CommonArgs {
+      input: DEFAULT_INPUT.to_string(),
+      filter: vec![],
+      profile: CfgProfile::default(),
+      cfg: vec![],
+      starter: vec![],
+      print_file_names: false,
+      pretty_print: false,
+    }
   }
 }
 
@@ -179,7 +193,7 @@ mod tests {
   #[test]
   fn parse_cmd_2() {
     let cmd =
-      "scnr -v extract --output /tmp -f *.json --filter=**/*.xml --force -p sysdiagnose --cfg img.svg=json --cfg *.toml=text -s file-system";
+      "scnr -v extract --output /tmp -f *.json --filter=**/*.xml --force -p sysdiagnose --cfg img.svg=json --cfg *.toml=text -s file-system -nb";
     let opts = Opts::parse_from(cmd.split(' '));
     assert!(opts.verbose);
     assert_eq!(
@@ -191,6 +205,8 @@ mod tests {
           profile: CfgProfile::Sysdiagnose,
           cfg: vec![("img.svg".into(), Plugin::Json), ("*.toml".into(), Plugin::Text)],
           starter: vec![Plugin::FileSystem],
+          print_file_names: true,
+          pretty_print: true
         },
         output: PathBuf::from("/tmp"),
         force: true,