From 0e51b4afa4544eda4290e46c4d0c8407f5bee01b Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Fri, 8 Mar 2024 00:31:23 +0900 Subject: [PATCH 1/5] feat: added low-memory-mode option #1298 --- src/afterfact.rs | 14 ++++++++++++++ src/detections/configs.rs | 21 +++++++++++++++++++-- src/detections/detection.rs | 5 +++++ src/detections/rule/condition_parser.rs | 1 + src/detections/rule/count.rs | 1 + src/detections/rule/matchers.rs | 1 + src/detections/rule/mod.rs | 1 + src/detections/rule/selectionnodes.rs | 1 + src/detections/utils.rs | 1 + src/main.rs | 5 +++++ src/options/htmlreport.rs | 4 ++++ src/options/profile.rs | 3 +++ src/yaml.rs | 1 + 13 files changed, 57 insertions(+), 2 deletions(-) diff --git a/src/afterfact.rs b/src/afterfact.rs index 85f665073..4b811f93b 100644 --- a/src/afterfact.rs +++ b/src/afterfact.rs @@ -2291,6 +2291,7 @@ mod tests { remove_duplicate_detections: false, no_wizard: true, include_status: None, + low_memory_mode: false, }, geo_ip: None, output: Some(Path::new("./test_emit_csv.csv").to_path_buf()), @@ -2381,6 +2382,7 @@ mod tests { remove_duplicate_detections: false, no_wizard: true, include_status: None, + low_memory_mode: false, }; let ch = mock_ch_filter .get(&CompactString::from("security")) @@ -2625,6 +2627,7 @@ mod tests { remove_duplicate_detections: false, no_wizard: true, include_status: None, + low_memory_mode: false, }, geo_ip: None, output: Some(Path::new("./test_emit_csv_multiline.csv").to_path_buf()), @@ -2725,6 +2728,7 @@ mod tests { remove_duplicate_detections: false, no_wizard: true, include_status: None, + low_memory_mode: false, }; let ch = mock_ch_filter .get(&CompactString::from("security")) @@ -2953,6 +2957,7 @@ mod tests { remove_duplicate_detections: false, no_wizard: true, include_status: None, + low_memory_mode: false, }, geo_ip: None, output: Some(Path::new("./test_emit_csv_remove_duplicate.csv").to_path_buf()), @@ -3043,6 +3048,7 @@ mod tests { remove_duplicate_detections: false, no_wizard: true, include_status: None, + low_memory_mode: false, }; let ch = mock_ch_filter .get(&CompactString::from("security")) @@ -3282,6 +3288,7 @@ mod tests { remove_duplicate_detections: false, no_wizard: true, include_status: None, + low_memory_mode: false, }, geo_ip: None, output: Some(Path::new("./test_emit_csv_remove_duplicate.json").to_path_buf()), @@ -3372,6 +3379,7 @@ mod tests { remove_duplicate_detections: false, no_wizard: true, include_status: None, + low_memory_mode: false, }; let ch = mock_ch_filter .get(&CompactString::from("security")) @@ -3684,6 +3692,7 @@ mod tests { remove_duplicate_detections: false, no_wizard: true, include_status: None, + low_memory_mode: false, }, geo_ip: None, output: Some(Path::new("./test_multiple_data_in_details.json").to_path_buf()), @@ -3775,6 +3784,7 @@ mod tests { remove_duplicate_detections: false, no_wizard: true, include_status: None, + low_memory_mode: false, }; let ch = mock_ch_filter .get(&CompactString::from("security")) @@ -4032,6 +4042,7 @@ mod tests { remove_duplicate_detections: false, no_wizard: true, include_status: None, + low_memory_mode: false, }, geo_ip: None, output: Some(Path::new("./test_emit_csv_json.json").to_path_buf()), @@ -4123,6 +4134,7 @@ mod tests { remove_duplicate_detections: false, no_wizard: true, include_status: None, + low_memory_mode: false, }; let ch = mock_ch_filter .get(&CompactString::from("security")) @@ -4305,6 +4317,7 @@ mod tests { remove_duplicate_detections: false, no_wizard: true, include_status: None, + low_memory_mode: false, }, geo_ip: None, output: Some(Path::new("./test_emit_csv_jsonl.jsonl").to_path_buf()), @@ -4396,6 +4409,7 @@ mod tests { remove_duplicate_detections: false, no_wizard: true, include_status: None, + low_memory_mode: false, }; let ch = mock_ch_filter .get(&CompactString::from("security")) diff --git a/src/detections/configs.rs b/src/detections/configs.rs index 13f19dff9..f8c0cfd20 100644 --- a/src/detections/configs.rs +++ b/src/detections/configs.rs @@ -557,9 +557,12 @@ impl StoredStatic { .collect(), _ => HashSet::default(), }; - + let is_low_memory = match &input_config.as_ref().unwrap().action { + Some(Action::CsvTimeline(opt)) => opt.output_options.low_memory_mode, + Some(Action::JsonTimeline(opt)) => opt.output_options.low_memory_mode, + _ => false, + }; let mut ret = StoredStatic { - is_low_memory: false, config: input_config.as_ref().unwrap().to_owned(), config_path: config_path.to_path_buf(), ch_config: create_output_filter_config( @@ -674,6 +677,7 @@ impl StoredStatic { enable_recover_records, timeline_offset, include_status, + is_low_memory, }; ret.profiles = load_profile( check_setting_path( @@ -1545,6 +1549,10 @@ pub struct OutputOption { /// Do not ask questions. Scan for all events and alerts. #[arg(help_heading = Some("General Options"), short = 'w', long = "no-wizard", display_order = 400)] pub no_wizard: bool, + + /// low-memory-mode + #[arg(help_heading = Some("General Options"), long = "low-memory-mode", display_order = 380)] + pub low_memory_mode: bool, } #[derive(Copy, Args, Clone, Debug)] @@ -2218,6 +2226,7 @@ fn extract_output_options(config: &Config) -> Option { remove_duplicate_detections: false, no_wizard: option.no_wizard, include_status: option.include_status.clone(), + low_memory_mode: false, }), Action::EidMetrics(option) => Some(OutputOption { input_args: option.input_args.clone(), @@ -2258,6 +2267,7 @@ fn extract_output_options(config: &Config) -> Option { remove_duplicate_detections: false, no_wizard: true, include_status: None, + low_memory_mode: false, }), Action::LogonSummary(option) => Some(OutputOption { input_args: option.input_args.clone(), @@ -2298,6 +2308,7 @@ fn extract_output_options(config: &Config) -> Option { remove_duplicate_detections: false, no_wizard: true, include_status: None, + low_memory_mode: false, }), Action::ComputerMetrics(option) => Some(OutputOption { input_args: option.input_args.clone(), @@ -2347,6 +2358,7 @@ fn extract_output_options(config: &Config) -> Option { remove_duplicate_detections: false, no_wizard: true, include_status: None, + low_memory_mode: false, }), Action::Search(option) => Some(OutputOption { input_args: option.input_args.clone(), @@ -2396,6 +2408,7 @@ fn extract_output_options(config: &Config) -> Option { remove_duplicate_detections: false, no_wizard: true, include_status: None, + low_memory_mode: false, }), Action::SetDefaultProfile(option) => Some(OutputOption { input_args: InputOption { @@ -2451,6 +2464,7 @@ fn extract_output_options(config: &Config) -> Option { remove_duplicate_detections: false, no_wizard: true, include_status: None, + low_memory_mode: false, }), Action::UpdateRules(option) => Some(OutputOption { input_args: InputOption { @@ -2506,6 +2520,7 @@ fn extract_output_options(config: &Config) -> Option { remove_duplicate_detections: false, no_wizard: true, include_status: None, + low_memory_mode: false, }), _ => None, } @@ -2758,6 +2773,7 @@ mod tests { remove_duplicate_detections: false, no_wizard: true, include_status: None, + low_memory_mode: false, }, geo_ip: None, output: None, @@ -2833,6 +2849,7 @@ mod tests { remove_duplicate_detections: false, no_wizard: true, include_status: None, + low_memory_mode: false, }, geo_ip: None, output: None, diff --git a/src/detections/detection.rs b/src/detections/detection.rs index 6d5233fd8..d274cdfd9 100644 --- a/src/detections/detection.rs +++ b/src/detections/detection.rs @@ -1292,6 +1292,7 @@ mod tests { remove_duplicate_detections: false, no_wizard: true, include_status: None, + low_memory_mode: false, }, geo_ip: None, output: None, @@ -1553,6 +1554,7 @@ mod tests { remove_duplicate_detections: false, no_wizard: true, include_status: None, + low_memory_mode: false, }, geo_ip: Some(Path::new("test_files/mmdb").to_path_buf()), output: Some(Path::new("./test_emit_csv.csv").to_path_buf()), @@ -1689,6 +1691,7 @@ mod tests { remove_duplicate_detections: false, no_wizard: true, include_status: None, + low_memory_mode: false, }, geo_ip: Some(Path::new("test_files/mmdb").to_path_buf()), output: Some(Path::new("./test_emit_csv.csv").to_path_buf()), @@ -1820,6 +1823,7 @@ mod tests { remove_duplicate_detections: false, no_wizard: true, include_status: None, + low_memory_mode: false, }, geo_ip: None, output: Some(Path::new("./test_emit_csv.csv").to_path_buf()), @@ -1967,6 +1971,7 @@ mod tests { remove_duplicate_detections: false, no_wizard: true, include_status: None, + low_memory_mode: false, }, geo_ip: None, output: Some(Path::new("./test_emit_csv.csv").to_path_buf()), diff --git a/src/detections/rule/condition_parser.rs b/src/detections/rule/condition_parser.rs index c0d8c4dde..e018cc725 100644 --- a/src/detections/rule/condition_parser.rs +++ b/src/detections/rule/condition_parser.rs @@ -508,6 +508,7 @@ mod tests { remove_duplicate_detections: false, no_wizard: true, include_status: None, + low_memory_mode: false, }, geo_ip: None, output: None, diff --git a/src/detections/rule/count.rs b/src/detections/rule/count.rs index d1ba40b2e..82f365ef3 100644 --- a/src/detections/rule/count.rs +++ b/src/detections/rule/count.rs @@ -636,6 +636,7 @@ mod tests { remove_duplicate_detections: false, no_wizard: true, include_status: None, + low_memory_mode: false, }, geo_ip: None, output: None, diff --git a/src/detections/rule/matchers.rs b/src/detections/rule/matchers.rs index c237df86a..818420b79 100644 --- a/src/detections/rule/matchers.rs +++ b/src/detections/rule/matchers.rs @@ -871,6 +871,7 @@ mod tests { remove_duplicate_detections: false, no_wizard: true, include_status: None, + low_memory_mode: false, }, geo_ip: None, output: None, diff --git a/src/detections/rule/mod.rs b/src/detections/rule/mod.rs index 9e2e26dd9..f07ef4ddc 100644 --- a/src/detections/rule/mod.rs +++ b/src/detections/rule/mod.rs @@ -453,6 +453,7 @@ mod tests { remove_duplicate_detections: false, no_wizard: true, include_status: None, + low_memory_mode: false, }, geo_ip: None, output: None, diff --git a/src/detections/rule/selectionnodes.rs b/src/detections/rule/selectionnodes.rs index 1df26cdd0..2eff7c07d 100644 --- a/src/detections/rule/selectionnodes.rs +++ b/src/detections/rule/selectionnodes.rs @@ -576,6 +576,7 @@ mod tests { remove_duplicate_detections: false, no_wizard: true, include_status: None, + low_memory_mode: false, }, geo_ip: None, output: None, diff --git a/src/detections/utils.rs b/src/detections/utils.rs index 6d98135e0..4d6a18869 100644 --- a/src/detections/utils.rs +++ b/src/detections/utils.rs @@ -1075,6 +1075,7 @@ mod tests { remove_duplicate_detections: false, no_wizard: true, include_status: None, + low_memory_mode: false, }, geo_ip: None, output: None, diff --git a/src/main.rs b/src/main.rs index ced5953cf..df949b750 100644 --- a/src/main.rs +++ b/src/main.rs @@ -2234,6 +2234,7 @@ mod tests { remove_duplicate_detections: false, no_wizard: true, include_status: None, + low_memory_mode: false, }, geo_ip: None, output: None, @@ -2405,6 +2406,7 @@ mod tests { remove_duplicate_detections: false, no_wizard: true, include_status: None, + low_memory_mode: false, }, geo_ip: None, output: Some(Path::new("overwrite.csv").to_path_buf()), @@ -2490,6 +2492,7 @@ mod tests { remove_duplicate_detections: false, no_wizard: true, include_status: None, + low_memory_mode: false, }, geo_ip: None, output: Some(Path::new("overwrite.csv").to_path_buf()), @@ -2574,6 +2577,7 @@ mod tests { remove_duplicate_detections: false, no_wizard: true, include_status: None, + low_memory_mode: false, }, geo_ip: None, output: Some(Path::new("overwrite.json").to_path_buf()), @@ -2659,6 +2663,7 @@ mod tests { remove_duplicate_detections: false, no_wizard: true, include_status: None, + low_memory_mode: false, }, geo_ip: None, output: Some(Path::new("overwrite.json").to_path_buf()), diff --git a/src/options/htmlreport.rs b/src/options/htmlreport.rs index 2bd6bef6f..ddfa3723d 100644 --- a/src/options/htmlreport.rs +++ b/src/options/htmlreport.rs @@ -302,6 +302,7 @@ mod tests { remove_duplicate_detections: false, no_wizard: true, include_status: None, + low_memory_mode: false, }, geo_ip: None, output: None, @@ -369,6 +370,7 @@ mod tests { remove_duplicate_detections: false, no_wizard: true, include_status: None, + low_memory_mode: false, }, geo_ip: None, output: None, @@ -439,6 +441,7 @@ mod tests { remove_duplicate_detections: false, no_wizard: true, include_status: None, + low_memory_mode: false, }, jsonl_timeline: false, geo_ip: None, @@ -506,6 +509,7 @@ mod tests { remove_duplicate_detections: false, no_wizard: true, include_status: None, + low_memory_mode: false, }, jsonl_timeline: false, geo_ip: None, diff --git a/src/options/profile.rs b/src/options/profile.rs index 1f45f4fd0..0e71bf1f1 100644 --- a/src/options/profile.rs +++ b/src/options/profile.rs @@ -537,6 +537,7 @@ mod tests { remove_duplicate_detections: false, no_wizard: true, include_status: None, + low_memory_mode: false, }, geo_ip: None, output: None, @@ -615,6 +616,7 @@ mod tests { remove_duplicate_detections: false, no_wizard: true, include_status: None, + low_memory_mode: false, }, geo_ip: None, output: None, @@ -723,6 +725,7 @@ mod tests { remove_duplicate_detections: false, no_wizard: true, include_status: None, + low_memory_mode: false, }, geo_ip: None, output: None, diff --git a/src/yaml.rs b/src/yaml.rs index 11df79ff9..759f65533 100644 --- a/src/yaml.rs +++ b/src/yaml.rs @@ -766,6 +766,7 @@ mod tests { remove_duplicate_detections: false, no_wizard: true, include_status: None, + low_memory_mode: false, }, geo_ip: None, output: None, From 01127aedf7a12b8b9548b4a04f72fe35504f4fa5 Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Fri, 8 Mar 2024 00:35:43 +0900 Subject: [PATCH 2/5] feat(configs): modified low-memory-mode option and description #1298 --- src/detections/configs.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/detections/configs.rs b/src/detections/configs.rs index f8c0cfd20..c779cab7f 100644 --- a/src/detections/configs.rs +++ b/src/detections/configs.rs @@ -1550,8 +1550,8 @@ pub struct OutputOption { #[arg(help_heading = Some("General Options"), short = 'w', long = "no-wizard", display_order = 400)] pub no_wizard: bool, - /// low-memory-mode - #[arg(help_heading = Some("General Options"), long = "low-memory-mode", display_order = 380)] + /// Scan with the minimal amount of memory by not sorting events + #[arg(help_heading = Some("General Options"), short='s', long = "low-memory-mode", display_order = 380)] pub low_memory_mode: bool, } From 7b8ae6eab41f4ac39b9f97474e71d2b77f4728fc Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Fri, 8 Mar 2024 00:47:22 +0900 Subject: [PATCH 3/5] feat(configs): added conflicts of remove-duplicate-detecions, remove_duplicate_data with low-memory-mode #1298 --- src/detections/configs.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/detections/configs.rs b/src/detections/configs.rs index c779cab7f..1b0603116 100644 --- a/src/detections/configs.rs +++ b/src/detections/configs.rs @@ -1535,15 +1535,16 @@ pub struct OutputOption { /// Duplicate field data will be replaced with "DUP" #[arg( - help_heading = Some("Output"), + help_heading = Some("General Options"), short = 'R', long = "remove-duplicate-data", + conflicts_with = "low_memory_mode", display_order = 440 )] pub remove_duplicate_data: bool, /// Remove duplicate detections (default: disabled) - #[arg(help_heading = Some("Output"), short = 'X', long = "remove-duplicate-detections", display_order = 441)] + #[arg(help_heading = Some("General Options"), short = 'X', long = "remove-duplicate-detections", conflicts_with = "low_memory_mode", display_order = 441)] pub remove_duplicate_detections: bool, /// Do not ask questions. Scan for all events and alerts. From c304c12b7e1a1a263d04fa742abd1d77ef56ca5c Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Fri, 8 Mar 2024 08:53:50 +0900 Subject: [PATCH 4/5] UI(configs): to adapt review, moved from Input to Genral Options #1298 --- src/detections/configs.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/detections/configs.rs b/src/detections/configs.rs index 1b0603116..3b1956370 100644 --- a/src/detections/configs.rs +++ b/src/detections/configs.rs @@ -918,7 +918,7 @@ impl Action { #[derive(Args, Clone, Debug)] pub struct DetectCommonOption { /// Scan JSON formatted logs instead of .evtx (.json or .jsonl) - #[arg(help_heading = Some("Input"), short = 'J', long = "JSON-input", conflicts_with = "live_analysis", display_order = 390)] + #[arg(help_heading = Some("General Options"), short = 'J', long = "JSON-input", conflicts_with = "live_analysis", display_order = 390)] pub json_input: bool, /// Specify additional evtx file extensions (ex: evtx_data) @@ -1535,7 +1535,7 @@ pub struct OutputOption { /// Duplicate field data will be replaced with "DUP" #[arg( - help_heading = Some("General Options"), + help_heading = Some("Output"), short = 'R', long = "remove-duplicate-data", conflicts_with = "low_memory_mode", @@ -1544,7 +1544,7 @@ pub struct OutputOption { pub remove_duplicate_data: bool, /// Remove duplicate detections (default: disabled) - #[arg(help_heading = Some("General Options"), short = 'X', long = "remove-duplicate-detections", conflicts_with = "low_memory_mode", display_order = 441)] + #[arg(help_heading = Some("Output"), short = 'X', long = "remove-duplicate-detections", conflicts_with = "low_memory_mode", display_order = 441)] pub remove_duplicate_detections: bool, /// Do not ask questions. Scan for all events and alerts. @@ -1587,7 +1587,7 @@ pub struct InputOption { pub live_analysis: bool, /// Carve evtx records from slack space (default: disabled) - #[arg(help_heading = Some("Input"), short = 'x', long = "recover-records", conflicts_with = "json_input", display_order = 440)] + #[arg(help_heading = Some("General Options"), short = 'x', long = "recover-records", conflicts_with = "json_input", display_order = 440)] pub recover_records: bool, /// Scan recent events based on an offset (ex: 1y, 3M, 30d, 24h, 30m) @@ -1657,7 +1657,7 @@ pub struct ComputerMetricsOption { pub common_options: CommonOptions, /// Scan JSON formatted logs instead of .evtx (.json or .jsonl) - #[arg(help_heading = Some("Input"), short = 'J', long = "JSON-input", conflicts_with = "live_analysis", display_order = 390)] + #[arg(help_heading = Some("General Options"), short = 'J', long = "JSON-input", conflicts_with = "live_analysis", display_order = 390)] pub json_input: bool, /// Specify additional evtx file extensions (ex: evtx_data) From 20d217765b1782e2f3fb454a0e932a8509666990 Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Sat, 9 Mar 2024 00:36:02 +0900 Subject: [PATCH 5/5] fix(configs, main): to erase erased fixed value assignment #1298 --- src/detections/configs.rs | 1 - src/main.rs | 1 - 2 files changed, 2 deletions(-) diff --git a/src/detections/configs.rs b/src/detections/configs.rs index 3b1956370..e6ff44abe 100644 --- a/src/detections/configs.rs +++ b/src/detections/configs.rs @@ -698,7 +698,6 @@ impl StoredStatic { .unwrap(), Some(&ret), ); - ret.is_low_memory = false; ret } /// detailsのdefault値をファイルから読み取る関数 diff --git a/src/main.rs b/src/main.rs index df949b750..a7c8ba44a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -77,7 +77,6 @@ fn main() { let mut config_reader = ConfigReader::new(); // コマンドのパース情報を作成してstatic変数に格納する let mut stored_static = StoredStatic::create_static_data(config_reader.config); - //stored_static.is_low_memory = true; config_reader.config = None; let mut app = App::new(stored_static.thread_number); app.exec(&mut config_reader.app, &mut stored_static);