From 0cd1d80072dddeffdb636e59ffe0765fd9129225 Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Fri, 17 Nov 2023 12:47:43 +0900 Subject: [PATCH 01/16] feat: added print count of rules in the scan wizard #1206 --- src/main.rs | 124 ++++++++++++++++++++++++++++------- src/yaml.rs | 183 +++++++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 275 insertions(+), 32 deletions(-) diff --git a/src/main.rs b/src/main.rs index 6191fef32..2c4e0c5d8 100644 --- a/src/main.rs +++ b/src/main.rs @@ -24,7 +24,7 @@ use hayabusa::detections::utils::{ check_setting_path, get_writable_color, output_and_data_stack_for_html, output_duration, output_profile_name, }; -use hayabusa::options; +use hayabusa::{options, yaml}; use hayabusa::options::htmlreport::{self, HTML_REPORTER}; use hayabusa::options::pivot::create_output; use hayabusa::options::pivot::PIVOT_KEYWORD; @@ -50,6 +50,7 @@ use std::path::Path; use std::ptr::null_mut; use std::sync::Arc; use std::time::Duration; +use std::u128; use std::{ env, fs::{self, File}, @@ -1012,8 +1013,66 @@ impl App { || stored_static.computer_metrics_flag || stored_static.output_option.as_ref().unwrap().no_wizard) { + let mut rule_counter_wizard_map = HashMap::new(); + yaml::count_rules(&stored_static.output_option.as_ref().unwrap().rules, &filter::exclude_ids(stored_static), stored_static, &mut rule_counter_wizard_map); + let level_map:HashMap<&str, u128> = HashMap::from([ + ("INFORMATIONAL", 1), + ("LOW", 2), + ("MEDIUM", 3), + ("HIGH", 4), + ("CRITICAL", 5), + ]); println!("Scan wizard:"); println!(); + let calcurate_wizard_rule_count = |exclude_noisytarget_flag: bool, exclude_noisy_status: Vec<&str>, min_level: &str, target_status: Vec<&str>, target_tags: Vec<&str>| -> HashMap { + let mut ret = HashMap::new(); + if exclude_noisytarget_flag { + for s in exclude_noisy_status { + let mut ret_cnt = 0; + if let Some(target_status_count) = rule_counter_wizard_map.get(s) { + target_status_count.iter().for_each(| (rule_level, value)| { + let doc_level_num = level_map.get(rule_level.to_uppercase().as_str()).unwrap_or(&1); + let args_level_num = level_map.get(min_level.to_uppercase().as_str()).unwrap_or(&1); + if doc_level_num >= args_level_num { + ret_cnt += value.iter().map(|(_, cnt )| cnt).sum::() + } + }); + } + ret.insert(CompactString::from(s), ret_cnt); + } + } else { + let all_status_flag = target_status.contains(&"*"); + for s in rule_counter_wizard_map.keys() { + // 指定されたstatusに合致しないものは集計をスキップする + if (exclude_noisy_status.contains(&s.as_str()) || !target_status.contains(&s.as_str())) && !all_status_flag { + continue; + } + let mut ret_cnt = 0; + if let Some(target_status_count) = rule_counter_wizard_map.get(s) { + target_status_count.iter().for_each(| (rule_level, value)| { + let doc_level_num = level_map.get(rule_level.to_uppercase().as_str()).unwrap_or(&1); + let args_level_num = level_map.get(min_level.to_uppercase().as_str()).unwrap_or(&1); + if doc_level_num >= args_level_num { + if !target_tags.is_empty() { + for (tag, cnt) in value.iter() { + if target_tags.contains(&tag.as_str()) { + let matched_tag_cnt = ret.entry(tag.clone()); + *matched_tag_cnt.or_insert(0) += cnt; + } + } + } else { + ret_cnt += value.iter().map(|(_, cnt )| cnt).sum::() + } + } + }); + if !exclude_noisy_status.contains(&s.as_str()) { + ret.insert(s.clone(), ret_cnt); + } + } + } + } + ret + }; let selections_status = &[ ("1. Core ( status: test, stable | level: high, critical )", (vec!["test", "stable"], "high")), ("2. Core+ ( status: test, stable | level: medium, high, critical )", (vec!["test", "stable"], "medium")), @@ -1022,23 +1081,35 @@ impl App { ("5. All event and alert rules ( status: * | level: informational+ )", (vec!["*"], "informational")), ]; - let selections = selections_status.iter().map(|x| x.0).collect_vec(); - let selected_index = Select::with_theme(&ColorfulTheme::default()) - .with_prompt("Which set of detection rules would you like to load?") - .default(0) - .items(selections.as_slice()) - .interact() - .unwrap(); - status_append_output = Some(format!( - "- selected detection rule sets: {}", - selections_status[selected_index].0 - )); - stored_static.output_option.as_mut().unwrap().min_level = - selections_status[selected_index].1 .1.into(); + let sections_rule_cnt = selections_status.iter().map(|(_, (status, min_level))| { + calcurate_wizard_rule_count(false, [].to_vec(), min_level, status.to_vec(), [].to_vec()) + }).collect_vec(); + let selection_status_items = &[ + format!("1. Core ({} rules) ( status: test, stable | level: high, critical )", sections_rule_cnt[0].iter().map(|(_, cnt)| cnt).sum::()), + format!("2. Core+ ({} rules) ( status: test, stable | level: medium, high, critical )", sections_rule_cnt[1].iter().map(|(_, cnt)| cnt).sum::()), + format!("3. Core++ ({} rules) ( status: experimental, test, stable | level: medium, high, critical )", sections_rule_cnt[2].iter().map(|(_, cnt)| cnt).sum::()), + format!("4. All alert rules ({} rules) ( status: * | level: low+ )", sections_rule_cnt[3].iter().map(|(_, cnt)| cnt).sum::()), + format!("5. All event and alert rules ({} rules) ( status: * | level: informational+ )", sections_rule_cnt[4].iter().map(|(_, cnt)| cnt).sum::()) + ]; - stored_static.include_status.extend( - selections_status[selected_index] - .1 + let selected_index = Select::with_theme(&ColorfulTheme::default()) + .with_prompt("Which set of detection rules would you like to load?") + .default(0) + .items(selection_status_items.as_slice()) + .interact() + .unwrap(); + status_append_output = Some(format!( + "- selected detection rule sets: {}", + selections_status[selected_index].0 + )); + stored_static.output_option.as_mut().unwrap().min_level = + selections_status[selected_index].1 .1.into(); + + let exclude_noisy_cnt = calcurate_wizard_rule_count(true, ["exclude", "noisy", "deprecated", "unsupported"].to_vec(), selections_status[selected_index].1.1, [].to_vec(), [].to_vec()); + + stored_static.include_status.extend( + selections_status[selected_index] + .1 .0 .iter() .map(|x| x.to_owned().into()), @@ -1046,10 +1117,12 @@ impl App { let mut output_option = stored_static.output_option.clone().unwrap(); let exclude_tags = output_option.exclude_tag.get_or_insert_with(Vec::new); + let tags_cnt = calcurate_wizard_rule_count(false, [].to_vec(), selections_status[selected_index].1.1, selections_status[selected_index].1.0.clone(), ["detection.emerging_threats", "detection.threat_hunting", "sysmon"].to_vec()); // If anything other than "4. All alert rules" or "5. All event and alert rules" was selected, ask questions about tags. if selected_index < 3 { + let prompt_fmt = format!("Include Emerging Threats rules? ({} rules)", tags_cnt.get("detection.emerging_threats").unwrap_or(&0)); let et_rules_load_flag = Confirm::with_theme(&ColorfulTheme::default()) - .with_prompt("Include Emerging Threats rules?") + .with_prompt(prompt_fmt) .default(true) .show_default(true) .interact() @@ -1058,8 +1131,9 @@ impl App { if !et_rules_load_flag { exclude_tags.push("detection.emerging_threats".into()); } + let prompt_fmt = format!("Include Threat Hunting rules? ({} rules)", tags_cnt.get("detection.threat_hunting").unwrap_or(&0)); let th_rules_load_flag = Confirm::with_theme(&ColorfulTheme::default()) - .with_prompt("Include Threat Hunting rules?") + .with_prompt(prompt_fmt) .default(false) .show_default(true) .interact() @@ -1070,8 +1144,9 @@ impl App { } } // deprecated rules load prompt + let prompt_fmt = format!("Include deprecated rules? ({} rules)", exclude_noisy_cnt.get("deprecated").unwrap_or(&0)); let dep_rules_load_flag = Confirm::with_theme(&ColorfulTheme::default()) - .with_prompt("Include deprecated rules?") + .with_prompt(prompt_fmt) .default(false) .show_default(true) .interact() @@ -1085,8 +1160,9 @@ impl App { } // noisy rules load prompt + let prompt_fmt = format!("Include noisy rules? ({} rules)", tags_cnt.get("noisy").unwrap_or(&0)); let noisy_rules_load_flag = Confirm::with_theme(&ColorfulTheme::default()) - .with_prompt("Include noisy rules?") + .with_prompt(prompt_fmt) .default(false) .show_default(true) .interact() @@ -1100,8 +1176,9 @@ impl App { } // unsupported rules load prompt + let prompt_fmt = format!("Include unsupported rules? ({} rules)", exclude_noisy_cnt.get("unsupported").unwrap_or(&0)); let unsupported_rules_load_flag = Confirm::with_theme(&ColorfulTheme::default()) - .with_prompt("Include unsupported rules?") + .with_prompt(prompt_fmt) .default(false) .show_default(true) .interact() @@ -1114,8 +1191,9 @@ impl App { .enable_unsupported_rules = true; } + let prompt_fmt = format!("Include sysmon rules? ({} rules)", tags_cnt.get("sysmon").unwrap_or(&0)); let sysmon_rules_load_flag = Confirm::with_theme(&ColorfulTheme::default()) - .with_prompt("Include sysmon rules?") + .with_prompt(prompt_fmt) .default(true) .show_default(true) .interact() diff --git a/src/yaml.rs b/src/yaml.rs index 328df971a..12573c360 100644 --- a/src/yaml.rs +++ b/src/yaml.rs @@ -8,10 +8,10 @@ use crate::detections::utils; use crate::filter::RuleExclude; use compact_str::CompactString; use hashbrown::{HashMap, HashSet}; +use itertools::Itertools; use std::ffi::OsStr; use std::fs; -use std::io; -use std::io::{BufReader, Read}; +use std::io::{self, BufReader, Read}; use std::path::{Path, PathBuf}; use yaml_rust::YamlLoader; @@ -54,7 +54,7 @@ impl ParseYaml { } } - pub fn read_file(&self, path: PathBuf) -> Result { + pub fn read_file(path: PathBuf) -> Result { let mut file_content = String::new(); let mut fr = fs::File::open(path) @@ -116,7 +116,7 @@ impl ParseYaml { } // 個別のファイルの読み込みは即終了としない。 - let read_content = self.read_file(path.as_ref().to_path_buf()); + let read_content = Self::read_file(path.as_ref().to_path_buf()); if read_content.is_err() { let errmsg = format!( "fail to read file: {}\n{} ", @@ -203,7 +203,7 @@ impl ParseYaml { } // 個別のファイルの読み込みは即終了としない。 - let read_content = self.read_file(path); + let read_content = Self::read_file(path); if read_content.is_err() { let errmsg = format!( "fail to read file: {}\n{} ", @@ -479,6 +479,172 @@ impl ParseYaml { } } + /// wizardへのルール数表示のためのstatus/level/tagsごとに階層化させてカウントする + pub fn count_rules>( + path: P, + exclude_ids: &RuleExclude, + stored_static: &StoredStatic, + result_container: &mut HashMap>> + ) -> HashMap>> { + let metadata = fs::metadata(path.as_ref()); + if metadata.is_err() { + return HashMap::default(); + } + let mut yaml_docs = vec![]; + if metadata.unwrap().file_type().is_file() { + // 拡張子がymlでないファイルは無視 + if path + .as_ref() + .to_path_buf() + .extension() + .unwrap_or_else(|| OsStr::new("")) + != "yml" + { + return HashMap::default(); + } + + // 個別のファイルの読み込みは即終了としない。 + let read_content = ParseYaml::read_file(path.as_ref().to_path_buf()); + if read_content.is_err() { + return HashMap::default(); + } + + // ここも個別のファイルの読み込みは即終了としない。 + let yaml_contents = YamlLoader::load_from_str(&read_content.unwrap()); + if yaml_contents.is_err() { + return HashMap::default(); + } + + yaml_docs.extend(yaml_contents.unwrap().into_iter().map(|yaml_content| { + let filepath = format!("{}", path.as_ref().to_path_buf().display()); + (filepath, yaml_content) + })); + } else { + let entries = fs::read_dir(path); + if entries.is_err() { + return HashMap::default(); + } + yaml_docs = entries.unwrap().try_fold(vec![], |mut ret, entry| { + let entry = entry?; + // フォルダは再帰的に呼び出す。 + if entry.file_type()?.is_dir() { + count_rules( + entry.path(), + exclude_ids, + stored_static, + result_container + ); + return io::Result::Ok(ret); + } + // ファイル以外は無視 + if !entry.file_type()?.is_file() { + return io::Result::Ok(ret); + } + + // 拡張子がymlでないファイルは無視 + let path = entry.path(); + if path.extension().unwrap_or_else(|| OsStr::new("")) != "yml" { + return io::Result::Ok(ret); + } + + let path_str = path.to_str().unwrap(); + // ignore if yml file in .git folder. + if utils::contains_str(path_str, "/.git/") + || utils::contains_str(path_str, "\\.git\\") + { + return io::Result::Ok(ret); + } + + // ignore if tool test yml file in hayabusa-rules. + if utils::contains_str(path_str, "rules/tools/sigmac/test_files") + || utils::contains_str(path_str, "rules\\tools\\sigmac\\test_files") + { + return io::Result::Ok(ret); + } + + // 個別のファイルの読み込みは即終了としない。 + let read_content = ParseYaml::read_file(path); + if read_content.is_err() { + return io::Result::Ok(ret); + } + + // ここも個別のファイルの読み込みは即終了としない。 + let yaml_contents = YamlLoader::load_from_str(&read_content.unwrap()); + if yaml_contents.is_err() { + let errmsg = format!( + "Failed to parse yml: {}\n{} ", + entry.path().display(), + yaml_contents.unwrap_err() + ); + if stored_static.verbose_flag { + AlertMessage::warn(&errmsg)?; + } + if !stored_static.quiet_errors_flag { + ERROR_LOG_STACK + .lock() + .unwrap() + .push(format!("[WARN] {errmsg}")); + } + return io::Result::Ok(ret); + } + + let yaml_contents = yaml_contents.unwrap().into_iter().map(|yaml_content| { + let filepath = format!("{}", entry.path().display()); + (filepath, yaml_content) + }); + ret.extend(yaml_contents); + io::Result::Ok(ret) + }).unwrap_or_default(); + } + yaml_docs.into_iter().for_each(|(_filepath, yaml_doc)| { + //除外されたルールは無視する + let empty = vec![]; + let rule_id = &yaml_doc["id"].as_str(); + let rule_tags_vec = yaml_doc["tags"].as_vec().unwrap_or(&empty); + let included_target_tag_vec = { + let target_wizard_tags = ["detection.emerging_threats", "detection.threat_hunting", "sysmon"]; + rule_tags_vec.iter().filter(|x| target_wizard_tags.contains(&x.as_str().unwrap_or_default())).filter_map(|s| s.as_str()).collect_vec() + }; + if rule_id.is_some() { + if let Some(v) = exclude_ids + .no_use_rule + .get(&rule_id.unwrap_or(&String::default()).to_string()) + { + let entry_key = if utils::contains_str(v, "exclude_rule") { + "excluded" + } else { + "noisy" + }; + let counter = result_container.entry(entry_key.into()).or_insert(HashMap::new()); + *counter.entry(yaml_doc["level"] + .as_str() + .unwrap_or("informational") + .to_uppercase().into()).or_insert(HashMap::new()).entry(yaml_doc["status"].as_str().unwrap_or("undefined").to_lowercase().into()).or_insert(0) += 1; + return; + } + } + + if let Some(s) = yaml_doc["status"].as_str() { + // wizard用の初期カウンティングではstatusとlevelの内容を確認したうえで以降の処理は行わないようにする + let counter = result_container.entry(s.into()).or_insert(HashMap::new()); + if included_target_tag_vec.is_empty() { + *counter.entry(yaml_doc["level"] + .as_str() + .unwrap_or("informational") + .to_uppercase().into()).or_insert(HashMap::new()).entry("other".into()).or_insert(0) += 1; + } else { + for tag in included_target_tag_vec { + *counter.entry(yaml_doc["level"] + .as_str() + .unwrap_or("informational") + .to_uppercase().into()).or_insert(HashMap::new()).entry(tag.into()).or_insert(0) += 1; + } + } + } + }); + result_container.to_owned() + } + #[cfg(test)] mod tests { @@ -492,6 +658,7 @@ mod tests { use crate::detections::configs::StoredStatic; use crate::filter; use crate::yaml; + use crate::yaml::ParseYaml; use crate::yaml::RuleExclude; use compact_str::CompactString; use hashbrown::HashMap; @@ -600,9 +767,8 @@ mod tests { #[test] fn test_read_yaml() { - let yaml = yaml::ParseYaml::new(&create_dummy_stored_static()); let path = Path::new("test_files/rules/yaml/1.yml"); - let ret = yaml.read_file(path.to_path_buf()).unwrap(); + let ret = ParseYaml::read_file(path.to_path_buf()).unwrap(); let rule = YamlLoader::load_from_str(&ret).unwrap(); for i in rule { if i["title"].as_str().unwrap() == "Sysmon Check command lines" { @@ -617,9 +783,8 @@ mod tests { #[test] fn test_failed_read_yaml() { - let yaml = yaml::ParseYaml::new(&create_dummy_stored_static()); let path = Path::new("test_files/rules/yaml/error.yml"); - let ret = yaml.read_file(path.to_path_buf()).unwrap(); + let ret = ParseYaml::read_file(path.to_path_buf()).unwrap(); let rule = YamlLoader::load_from_str(&ret); assert!(rule.is_err()); } From a43e362334d05a22a4780b1209f13b052fccb80b Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Fri, 17 Nov 2023 12:48:37 +0900 Subject: [PATCH 02/16] refactor: fixed Clippy error --- src/afterfact.rs | 9 ++-- src/detections/configs.rs | 6 +-- src/detections/field_data_map.rs | 2 +- src/main.rs | 2 +- src/options/level_tuning.rs | 2 +- src/options/profile.rs | 90 ++++---------------------------- 6 files changed, 19 insertions(+), 92 deletions(-) diff --git a/src/afterfact.rs b/src/afterfact.rs index c56e07e8d..ea2d8fff1 100644 --- a/src/afterfact.rs +++ b/src/afterfact.rs @@ -72,7 +72,7 @@ pub fn set_output_color(no_color_flag: bool) -> HashMap { return; } let empty = &"".to_string(); - let level = CompactString::new(line.get(0).unwrap_or(empty).to_lowercase()); + let level = CompactString::new(line.first().unwrap_or(empty).to_lowercase()); let convert_color_result = hex::decode(line.get(1).unwrap_or(empty).trim()); if convert_color_result.is_err() { AlertMessage::warn(&format!( @@ -498,7 +498,7 @@ fn emit_csv( let level_suffix = get_level_suffix(detect_info.level.as_str()); let author_list = author_list_cache .entry(detect_info.rulepath.clone()) - .or_insert_with(|| extract_author_name(&detect_info.rulepath, stored_static)) + .or_insert_with(|| extract_author_name(&detect_info.rulepath)) .clone(); let author_str = author_list.iter().join(", "); detect_rule_authors.insert(detect_info.rulepath.to_owned(), author_str.into()); @@ -1792,9 +1792,8 @@ fn output_detected_rule_authors( } /// 与えられたyaml_pathからauthorの名前を抽出して配列で返却する関数 -fn extract_author_name(yaml_path: &str, stored_static: &StoredStatic) -> Nested { - let parser = ParseYaml::new(stored_static); - let contents = match parser.read_file(Path::new(&yaml_path).to_path_buf()) { +fn extract_author_name(yaml_path: &str) -> Nested { + let contents = match ParseYaml::read_file(Path::new(&yaml_path).to_path_buf()) { Ok(yaml) => Some(yaml), Err(e) => { AlertMessage::alert(&e).ok(); diff --git a/src/detections/configs.rs b/src/detections/configs.rs index 6762054ad..fd55e54d6 100644 --- a/src/detections/configs.rs +++ b/src/detections/configs.rs @@ -667,7 +667,7 @@ impl StoredStatic { lines .iter() .try_for_each(|line| -> Result<(), String> { - let provider = match line.get(0) { + let provider = match line.first() { Some(_provider) => _provider.trim(), _ => { return Result::Err( @@ -2015,7 +2015,7 @@ pub fn load_eventkey_alias(path: &str) -> EventKeyAliasConfig { } let empty = &"".to_string(); - let alias = line.get(0).unwrap_or(empty); + let alias = line.first().unwrap_or(empty); let event_key = line.get(1).unwrap_or(empty); if alias.is_empty() || event_key.is_empty() { return; @@ -2502,7 +2502,7 @@ fn load_eventcode_info(path: &str) -> EventInfoConfig { } let empty = &"".to_string(); - let channel = line.get(0).unwrap_or(empty); + let channel = line.first().unwrap_or(empty); let eventcode = line.get(1).unwrap_or(empty); let event_title = line.get(2).unwrap_or(empty); infodata = EventInfo { diff --git a/src/detections/field_data_map.rs b/src/detections/field_data_map.rs index 3b6ee9041..855a53ed1 100644 --- a/src/detections/field_data_map.rs +++ b/src/detections/field_data_map.rs @@ -166,7 +166,7 @@ mod tests { fn build_yaml(s: &str) -> Yaml { YamlLoader::load_from_str(s) .unwrap_or_default() - .get(0) + .first() .unwrap() .clone() } diff --git a/src/main.rs b/src/main.rs index 2c4e0c5d8..1282f9078 100644 --- a/src/main.rs +++ b/src/main.rs @@ -689,7 +689,7 @@ impl App { } Action::ListProfiles(_) => { let profile_list = - options::profile::get_profile_list("config/profiles.yaml", stored_static); + options::profile::get_profile_list("config/profiles.yaml"); write_color_buffer( &BufferWriter::stdout(ColorChoice::Always), None, diff --git a/src/options/level_tuning.rs b/src/options/level_tuning.rs index e8642ba87..90910bddb 100644 --- a/src/options/level_tuning.rs +++ b/src/options/level_tuning.rs @@ -25,7 +25,7 @@ impl LevelTuning { let mut tuning_map: HashMap = HashMap::new(); read_result.iter().try_for_each(|line| -> Result<(), String> { // 1つ目の要素も存在しない場合はread_csvの段階で読み飛ばされるためget(0)がNoneにはならない - let id = line.get(0).unwrap(); + let id = line.first().unwrap(); if !configs::IDS_REGEX.is_match(id) { return Result::Err(format!("Failed to read level tuning file. {id} is not correct id format, fix it.")); } diff --git a/src/options/profile.rs b/src/options/profile.rs index 1be2f49cd..6a7223218 100644 --- a/src/options/profile.rs +++ b/src/options/profile.rs @@ -135,10 +135,8 @@ impl From<&str> for Profile { // 指定されたパスのprofileを読み込む処理 fn read_profile_data( profile_path: &str, - stored_static: &StoredStatic, ) -> Result, String> { - let yml = yaml::ParseYaml::new(stored_static); - if let Ok(loaded_profile) = yml.read_file(Path::new(profile_path).to_path_buf()) { + if let Ok(loaded_profile) = yaml::ParseYaml::read_file(Path::new(profile_path).to_path_buf()) { match YamlLoader::load_from_str(&loaded_profile) { Ok(profile_yml) => Ok(profile_yml), Err(e) => Err(format!("Parse error: {profile_path}. {e}")), @@ -175,7 +173,7 @@ pub fn load_profile( }; let profile_all: Vec = if profile.is_none() { - match read_profile_data(default_profile_path, opt_stored_static.unwrap()) { + match read_profile_data(default_profile_path) { Ok(data) => data, Err(e) => { AlertMessage::alert(&e).ok(); @@ -183,7 +181,7 @@ pub fn load_profile( } } } else { - match read_profile_data(profile_path, opt_stored_static.unwrap()) { + match read_profile_data(profile_path) { Ok(data) => data, Err(e) => { AlertMessage::alert(&e).ok(); @@ -270,7 +268,7 @@ pub fn set_default_profile( profile_path: &str, stored_static: &StoredStatic, ) -> Result<(), String> { - let profile_data: Vec = match read_profile_data(profile_path, stored_static) { + let profile_data: Vec = match read_profile_data(profile_path) { Ok(data) => data, Err(e) => { AlertMessage::alert(&e).ok(); @@ -351,13 +349,12 @@ pub fn set_default_profile( } /// Get profile name and tag list in yaml file. -pub fn get_profile_list(profile_path: &str, stored_static: &StoredStatic) -> Nested> { +pub fn get_profile_list(profile_path: &str) -> Nested> { let ymls = match read_profile_data( check_setting_path(&CURRENT_EXE_PATH.to_path_buf(), profile_path, true) .unwrap() .to_str() - .unwrap(), - stored_static, + .unwrap() ) { Ok(data) => data, Err(e) => { @@ -389,7 +386,7 @@ mod tests { use crate::detections::configs::{ Action, CommonOptions, Config, CsvOutputOption, DetectCommonOption, InputOption, - OutputOption, StoredStatic, UpdateOption, GEOIP_DB_PARSER, + OutputOption, StoredStatic, GEOIP_DB_PARSER, }; use crate::options::profile::{get_profile_list, load_profile, Profile}; use compact_str::CompactString; @@ -419,71 +416,9 @@ mod tests { #[test] fn test_get_profile_list_err_load() { - let dummy_stored_static = - create_dummy_stored_static(Action::CsvTimeline(CsvOutputOption { - output_options: OutputOption { - input_args: InputOption { - directory: None, - filepath: None, - live_analysis: false, - recover_records: false, - timeline_offset: None, - }, - profile: None, - enable_deprecated_rules: false, - exclude_status: None, - min_level: "informational".to_string(), - exact_level: None, - enable_noisy_rules: false, - end_timeline: None, - start_timeline: None, - eid_filter: false, - european_time: false, - iso_8601: false, - rfc_2822: false, - rfc_3339: false, - us_military_time: false, - us_time: false, - utc: false, - visualize_timeline: false, - rules: Path::new("./rules").to_path_buf(), - html_report: None, - no_summary: false, - common_options: CommonOptions { - no_color: false, - quiet: false, - }, - detect_common_options: DetectCommonOption { - evtx_file_ext: None, - thread_number: None, - quiet_errors: false, - config: Path::new("./rules/config").to_path_buf(), - verbose: false, - json_input: false, - include_computer: None, - exclude_computer: None, - }, - enable_unsupported_rules: false, - clobber: false, - proven_rules: false, - include_tag: None, - exclude_tag: None, - include_category: None, - exclude_category: None, - include_eid: None, - exclude_eid: None, - no_field: false, - remove_duplicate_data: false, - remove_duplicate_detections: false, - no_wizard: true, - }, - geo_ip: None, - output: None, - multiline: false, - })); assert_eq!( Nested::>::new(), - get_profile_list("test_files/not_exist_path", &dummy_stored_static) + get_profile_list("test_files/not_exist_path") ); } @@ -831,14 +766,7 @@ mod tests { assert_eq!( expect, get_profile_list( - "test_files/config/profiles.yaml", - &create_dummy_stored_static(Action::UpdateRules(UpdateOption { - rules: Path::new("./rules").to_path_buf(), - common_options: CommonOptions { - no_color: false, - quiet: false, - }, - })) + "test_files/config/profiles.yaml" ) ); } From a23dbe30508d14f6ad4aac8cc30454a3345f065d Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Fri, 17 Nov 2023 12:49:11 +0900 Subject: [PATCH 03/16] style: cargo fmt --- src/main.rs | 148 +++++++++++++++++++-------- src/options/profile.rs | 13 +-- src/yaml.rs | 222 ++++++++++++++++++++++++----------------- 3 files changed, 239 insertions(+), 144 deletions(-) diff --git a/src/main.rs b/src/main.rs index 1282f9078..cb16461ed 100644 --- a/src/main.rs +++ b/src/main.rs @@ -24,7 +24,6 @@ use hayabusa::detections::utils::{ check_setting_path, get_writable_color, output_and_data_stack_for_html, output_duration, output_profile_name, }; -use hayabusa::{options, yaml}; use hayabusa::options::htmlreport::{self, HTML_REPORTER}; use hayabusa::options::pivot::create_output; use hayabusa::options::pivot::PIVOT_KEYWORD; @@ -34,6 +33,7 @@ use hayabusa::timeline::computer_metrics::countup_event_by_computer; use hayabusa::{afterfact::after_fact, detections::utils}; use hayabusa::{detections::configs, timeline::timelines::Timeline}; use hayabusa::{detections::utils::write_color_buffer, filter}; +use hayabusa::{options, yaml}; use indicatif::ProgressBar; use indicatif::{ProgressDrawTarget, ProgressStyle}; use itertools::Itertools; @@ -688,8 +688,7 @@ impl App { return; } Action::ListProfiles(_) => { - let profile_list = - options::profile::get_profile_list("config/profiles.yaml"); + let profile_list = options::profile::get_profile_list("config/profiles.yaml"); write_color_buffer( &BufferWriter::stdout(ColorChoice::Always), None, @@ -1014,8 +1013,13 @@ impl App { || stored_static.output_option.as_ref().unwrap().no_wizard) { let mut rule_counter_wizard_map = HashMap::new(); - yaml::count_rules(&stored_static.output_option.as_ref().unwrap().rules, &filter::exclude_ids(stored_static), stored_static, &mut rule_counter_wizard_map); - let level_map:HashMap<&str, u128> = HashMap::from([ + yaml::count_rules( + &stored_static.output_option.as_ref().unwrap().rules, + &filter::exclude_ids(stored_static), + stored_static, + &mut rule_counter_wizard_map, + ); + let level_map: HashMap<&str, u128> = HashMap::from([ ("INFORMATIONAL", 1), ("LOW", 2), ("MEDIUM", 3), @@ -1024,17 +1028,26 @@ impl App { ]); println!("Scan wizard:"); println!(); - let calcurate_wizard_rule_count = |exclude_noisytarget_flag: bool, exclude_noisy_status: Vec<&str>, min_level: &str, target_status: Vec<&str>, target_tags: Vec<&str>| -> HashMap { + let calcurate_wizard_rule_count = |exclude_noisytarget_flag: bool, + exclude_noisy_status: Vec<&str>, + min_level: &str, + target_status: Vec<&str>, + target_tags: Vec<&str>| + -> HashMap { let mut ret = HashMap::new(); if exclude_noisytarget_flag { for s in exclude_noisy_status { let mut ret_cnt = 0; if let Some(target_status_count) = rule_counter_wizard_map.get(s) { - target_status_count.iter().for_each(| (rule_level, value)| { - let doc_level_num = level_map.get(rule_level.to_uppercase().as_str()).unwrap_or(&1); - let args_level_num = level_map.get(min_level.to_uppercase().as_str()).unwrap_or(&1); + target_status_count.iter().for_each(|(rule_level, value)| { + let doc_level_num = level_map + .get(rule_level.to_uppercase().as_str()) + .unwrap_or(&1); + let args_level_num = level_map + .get(min_level.to_uppercase().as_str()) + .unwrap_or(&1); if doc_level_num >= args_level_num { - ret_cnt += value.iter().map(|(_, cnt )| cnt).sum::() + ret_cnt += value.iter().map(|(_, cnt)| cnt).sum::() } }); } @@ -1044,14 +1057,21 @@ impl App { let all_status_flag = target_status.contains(&"*"); for s in rule_counter_wizard_map.keys() { // 指定されたstatusに合致しないものは集計をスキップする - if (exclude_noisy_status.contains(&s.as_str()) || !target_status.contains(&s.as_str())) && !all_status_flag { + if (exclude_noisy_status.contains(&s.as_str()) + || !target_status.contains(&s.as_str())) + && !all_status_flag + { continue; } let mut ret_cnt = 0; if let Some(target_status_count) = rule_counter_wizard_map.get(s) { - target_status_count.iter().for_each(| (rule_level, value)| { - let doc_level_num = level_map.get(rule_level.to_uppercase().as_str()).unwrap_or(&1); - let args_level_num = level_map.get(min_level.to_uppercase().as_str()).unwrap_or(&1); + target_status_count.iter().for_each(|(rule_level, value)| { + let doc_level_num = level_map + .get(rule_level.to_uppercase().as_str()) + .unwrap_or(&1); + let args_level_num = level_map + .get(min_level.to_uppercase().as_str()) + .unwrap_or(&1); if doc_level_num >= args_level_num { if !target_tags.is_empty() { for (tag, cnt) in value.iter() { @@ -1061,7 +1081,7 @@ impl App { } } } else { - ret_cnt += value.iter().map(|(_, cnt )| cnt).sum::() + ret_cnt += value.iter().map(|(_, cnt)| cnt).sum::() } } }); @@ -1081,9 +1101,18 @@ impl App { ("5. All event and alert rules ( status: * | level: informational+ )", (vec!["*"], "informational")), ]; - let sections_rule_cnt = selections_status.iter().map(|(_, (status, min_level))| { - calcurate_wizard_rule_count(false, [].to_vec(), min_level, status.to_vec(), [].to_vec()) - }).collect_vec(); + let sections_rule_cnt = selections_status + .iter() + .map(|(_, (status, min_level))| { + calcurate_wizard_rule_count( + false, + [].to_vec(), + min_level, + status.to_vec(), + [].to_vec(), + ) + }) + .collect_vec(); let selection_status_items = &[ format!("1. Core ({} rules) ( status: test, stable | level: high, critical )", sections_rule_cnt[0].iter().map(|(_, cnt)| cnt).sum::()), format!("2. Core+ ({} rules) ( status: test, stable | level: medium, high, critical )", sections_rule_cnt[1].iter().map(|(_, cnt)| cnt).sum::()), @@ -1093,23 +1122,29 @@ impl App { ]; let selected_index = Select::with_theme(&ColorfulTheme::default()) - .with_prompt("Which set of detection rules would you like to load?") - .default(0) - .items(selection_status_items.as_slice()) - .interact() - .unwrap(); - status_append_output = Some(format!( - "- selected detection rule sets: {}", - selections_status[selected_index].0 - )); - stored_static.output_option.as_mut().unwrap().min_level = - selections_status[selected_index].1 .1.into(); - - let exclude_noisy_cnt = calcurate_wizard_rule_count(true, ["exclude", "noisy", "deprecated", "unsupported"].to_vec(), selections_status[selected_index].1.1, [].to_vec(), [].to_vec()); - - stored_static.include_status.extend( - selections_status[selected_index] - .1 + .with_prompt("Which set of detection rules would you like to load?") + .default(0) + .items(selection_status_items.as_slice()) + .interact() + .unwrap(); + status_append_output = Some(format!( + "- selected detection rule sets: {}", + selections_status[selected_index].0 + )); + stored_static.output_option.as_mut().unwrap().min_level = + selections_status[selected_index].1 .1.into(); + + let exclude_noisy_cnt = calcurate_wizard_rule_count( + true, + ["exclude", "noisy", "deprecated", "unsupported"].to_vec(), + selections_status[selected_index].1 .1, + [].to_vec(), + [].to_vec(), + ); + + stored_static.include_status.extend( + selections_status[selected_index] + .1 .0 .iter() .map(|x| x.to_owned().into()), @@ -1117,10 +1152,24 @@ impl App { let mut output_option = stored_static.output_option.clone().unwrap(); let exclude_tags = output_option.exclude_tag.get_or_insert_with(Vec::new); - let tags_cnt = calcurate_wizard_rule_count(false, [].to_vec(), selections_status[selected_index].1.1, selections_status[selected_index].1.0.clone(), ["detection.emerging_threats", "detection.threat_hunting", "sysmon"].to_vec()); + let tags_cnt = calcurate_wizard_rule_count( + false, + [].to_vec(), + selections_status[selected_index].1 .1, + selections_status[selected_index].1 .0.clone(), + [ + "detection.emerging_threats", + "detection.threat_hunting", + "sysmon", + ] + .to_vec(), + ); // If anything other than "4. All alert rules" or "5. All event and alert rules" was selected, ask questions about tags. if selected_index < 3 { - let prompt_fmt = format!("Include Emerging Threats rules? ({} rules)", tags_cnt.get("detection.emerging_threats").unwrap_or(&0)); + let prompt_fmt = format!( + "Include Emerging Threats rules? ({} rules)", + tags_cnt.get("detection.emerging_threats").unwrap_or(&0) + ); let et_rules_load_flag = Confirm::with_theme(&ColorfulTheme::default()) .with_prompt(prompt_fmt) .default(true) @@ -1131,7 +1180,10 @@ impl App { if !et_rules_load_flag { exclude_tags.push("detection.emerging_threats".into()); } - let prompt_fmt = format!("Include Threat Hunting rules? ({} rules)", tags_cnt.get("detection.threat_hunting").unwrap_or(&0)); + let prompt_fmt = format!( + "Include Threat Hunting rules? ({} rules)", + tags_cnt.get("detection.threat_hunting").unwrap_or(&0) + ); let th_rules_load_flag = Confirm::with_theme(&ColorfulTheme::default()) .with_prompt(prompt_fmt) .default(false) @@ -1144,7 +1196,10 @@ impl App { } } // deprecated rules load prompt - let prompt_fmt = format!("Include deprecated rules? ({} rules)", exclude_noisy_cnt.get("deprecated").unwrap_or(&0)); + let prompt_fmt = format!( + "Include deprecated rules? ({} rules)", + exclude_noisy_cnt.get("deprecated").unwrap_or(&0) + ); let dep_rules_load_flag = Confirm::with_theme(&ColorfulTheme::default()) .with_prompt(prompt_fmt) .default(false) @@ -1160,7 +1215,10 @@ impl App { } // noisy rules load prompt - let prompt_fmt = format!("Include noisy rules? ({} rules)", tags_cnt.get("noisy").unwrap_or(&0)); + let prompt_fmt = format!( + "Include noisy rules? ({} rules)", + tags_cnt.get("noisy").unwrap_or(&0) + ); let noisy_rules_load_flag = Confirm::with_theme(&ColorfulTheme::default()) .with_prompt(prompt_fmt) .default(false) @@ -1176,7 +1234,10 @@ impl App { } // unsupported rules load prompt - let prompt_fmt = format!("Include unsupported rules? ({} rules)", exclude_noisy_cnt.get("unsupported").unwrap_or(&0)); + let prompt_fmt = format!( + "Include unsupported rules? ({} rules)", + exclude_noisy_cnt.get("unsupported").unwrap_or(&0) + ); let unsupported_rules_load_flag = Confirm::with_theme(&ColorfulTheme::default()) .with_prompt(prompt_fmt) .default(false) @@ -1191,7 +1252,10 @@ impl App { .enable_unsupported_rules = true; } - let prompt_fmt = format!("Include sysmon rules? ({} rules)", tags_cnt.get("sysmon").unwrap_or(&0)); + let prompt_fmt = format!( + "Include sysmon rules? ({} rules)", + tags_cnt.get("sysmon").unwrap_or(&0) + ); let sysmon_rules_load_flag = Confirm::with_theme(&ColorfulTheme::default()) .with_prompt(prompt_fmt) .default(true) diff --git a/src/options/profile.rs b/src/options/profile.rs index 6a7223218..9db943256 100644 --- a/src/options/profile.rs +++ b/src/options/profile.rs @@ -133,9 +133,7 @@ impl From<&str> for Profile { } // 指定されたパスのprofileを読み込む処理 -fn read_profile_data( - profile_path: &str, -) -> Result, String> { +fn read_profile_data(profile_path: &str) -> Result, String> { if let Ok(loaded_profile) = yaml::ParseYaml::read_file(Path::new(profile_path).to_path_buf()) { match YamlLoader::load_from_str(&loaded_profile) { Ok(profile_yml) => Ok(profile_yml), @@ -354,7 +352,7 @@ pub fn get_profile_list(profile_path: &str) -> Nested> { check_setting_path(&CURRENT_EXE_PATH.to_path_buf(), profile_path, true) .unwrap() .to_str() - .unwrap() + .unwrap(), ) { Ok(data) => data, Err(e) => { @@ -763,11 +761,6 @@ mod tests { expect.push(vec!["standard".to_string(), "%Timestamp%, %Computer%, %Channel%, %EventID%, %Level%, %MitreTags%, %RecordID%, %RuleTitle%, %Details%".to_string()]); expect.push(vec!["verbose-1".to_string(), "%Timestamp%, %Computer%, %Channel%, %EventID%, %Level%, %MitreTags%, %RecordID%, %RuleTitle%, %Details%, %RuleFile%, %EvtxFile%".to_string()]); expect.push(vec!["verbose-2".to_string(), "%Timestamp%, %Computer%, %Channel%, %EventID%, %Level%, %MitreTags%, %RecordID%, %RuleTitle%, %Details%, %AllFieldInfo%".to_string()]); - assert_eq!( - expect, - get_profile_list( - "test_files/config/profiles.yaml" - ) - ); + assert_eq!(expect, get_profile_list("test_files/config/profiles.yaml")); } } diff --git a/src/yaml.rs b/src/yaml.rs index 12573c360..36a4cf937 100644 --- a/src/yaml.rs +++ b/src/yaml.rs @@ -479,61 +479,61 @@ impl ParseYaml { } } - /// wizardへのルール数表示のためのstatus/level/tagsごとに階層化させてカウントする - pub fn count_rules>( - path: P, - exclude_ids: &RuleExclude, - stored_static: &StoredStatic, - result_container: &mut HashMap>> - ) -> HashMap>> { - let metadata = fs::metadata(path.as_ref()); - if metadata.is_err() { +/// wizardへのルール数表示のためのstatus/level/tagsごとに階層化させてカウントする +pub fn count_rules>( + path: P, + exclude_ids: &RuleExclude, + stored_static: &StoredStatic, + result_container: &mut HashMap< + CompactString, + HashMap>, + >, +) -> HashMap>> { + let metadata = fs::metadata(path.as_ref()); + if metadata.is_err() { + return HashMap::default(); + } + let mut yaml_docs = vec![]; + if metadata.unwrap().file_type().is_file() { + // 拡張子がymlでないファイルは無視 + if path + .as_ref() + .to_path_buf() + .extension() + .unwrap_or_else(|| OsStr::new("")) + != "yml" + { return HashMap::default(); } - let mut yaml_docs = vec![]; - if metadata.unwrap().file_type().is_file() { - // 拡張子がymlでないファイルは無視 - if path - .as_ref() - .to_path_buf() - .extension() - .unwrap_or_else(|| OsStr::new("")) - != "yml" - { - return HashMap::default(); - } - // 個別のファイルの読み込みは即終了としない。 - let read_content = ParseYaml::read_file(path.as_ref().to_path_buf()); - if read_content.is_err() { - return HashMap::default(); - } + // 個別のファイルの読み込みは即終了としない。 + let read_content = ParseYaml::read_file(path.as_ref().to_path_buf()); + if read_content.is_err() { + return HashMap::default(); + } - // ここも個別のファイルの読み込みは即終了としない。 - let yaml_contents = YamlLoader::load_from_str(&read_content.unwrap()); - if yaml_contents.is_err() { - return HashMap::default(); - } + // ここも個別のファイルの読み込みは即終了としない。 + let yaml_contents = YamlLoader::load_from_str(&read_content.unwrap()); + if yaml_contents.is_err() { + return HashMap::default(); + } - yaml_docs.extend(yaml_contents.unwrap().into_iter().map(|yaml_content| { - let filepath = format!("{}", path.as_ref().to_path_buf().display()); - (filepath, yaml_content) - })); - } else { - let entries = fs::read_dir(path); - if entries.is_err() { - return HashMap::default(); - } - yaml_docs = entries.unwrap().try_fold(vec![], |mut ret, entry| { + yaml_docs.extend(yaml_contents.unwrap().into_iter().map(|yaml_content| { + let filepath = format!("{}", path.as_ref().to_path_buf().display()); + (filepath, yaml_content) + })); + } else { + let entries = fs::read_dir(path); + if entries.is_err() { + return HashMap::default(); + } + yaml_docs = entries + .unwrap() + .try_fold(vec![], |mut ret, entry| { let entry = entry?; // フォルダは再帰的に呼び出す。 if entry.file_type()?.is_dir() { - count_rules( - entry.path(), - exclude_ids, - stored_static, - result_container - ); + count_rules(entry.path(), exclude_ids, stored_static, result_container); return io::Result::Ok(ret); } // ファイル以外は無視 @@ -594,56 +594,94 @@ impl ParseYaml { }); ret.extend(yaml_contents); io::Result::Ok(ret) - }).unwrap_or_default(); - } - yaml_docs.into_iter().for_each(|(_filepath, yaml_doc)| { - //除外されたルールは無視する - let empty = vec![]; - let rule_id = &yaml_doc["id"].as_str(); - let rule_tags_vec = yaml_doc["tags"].as_vec().unwrap_or(&empty); - let included_target_tag_vec = { - let target_wizard_tags = ["detection.emerging_threats", "detection.threat_hunting", "sysmon"]; - rule_tags_vec.iter().filter(|x| target_wizard_tags.contains(&x.as_str().unwrap_or_default())).filter_map(|s| s.as_str()).collect_vec() - }; - if rule_id.is_some() { - if let Some(v) = exclude_ids - .no_use_rule - .get(&rule_id.unwrap_or(&String::default()).to_string()) - { - let entry_key = if utils::contains_str(v, "exclude_rule") { - "excluded" - } else { - "noisy" - }; - let counter = result_container.entry(entry_key.into()).or_insert(HashMap::new()); - *counter.entry(yaml_doc["level"] - .as_str() - .unwrap_or("informational") - .to_uppercase().into()).or_insert(HashMap::new()).entry(yaml_doc["status"].as_str().unwrap_or("undefined").to_lowercase().into()).or_insert(0) += 1; - return; - } + }) + .unwrap_or_default(); + } + yaml_docs.into_iter().for_each(|(_filepath, yaml_doc)| { + //除外されたルールは無視する + let empty = vec![]; + let rule_id = &yaml_doc["id"].as_str(); + let rule_tags_vec = yaml_doc["tags"].as_vec().unwrap_or(&empty); + let included_target_tag_vec = { + let target_wizard_tags = [ + "detection.emerging_threats", + "detection.threat_hunting", + "sysmon", + ]; + rule_tags_vec + .iter() + .filter(|x| target_wizard_tags.contains(&x.as_str().unwrap_or_default())) + .filter_map(|s| s.as_str()) + .collect_vec() + }; + if rule_id.is_some() { + if let Some(v) = exclude_ids + .no_use_rule + .get(&rule_id.unwrap_or(&String::default()).to_string()) + { + let entry_key = if utils::contains_str(v, "exclude_rule") { + "excluded" + } else { + "noisy" + }; + let counter = result_container + .entry(entry_key.into()) + .or_insert(HashMap::new()); + *counter + .entry( + yaml_doc["level"] + .as_str() + .unwrap_or("informational") + .to_uppercase() + .into(), + ) + .or_insert(HashMap::new()) + .entry( + yaml_doc["status"] + .as_str() + .unwrap_or("undefined") + .to_lowercase() + .into(), + ) + .or_insert(0) += 1; + return; } + } - if let Some(s) = yaml_doc["status"].as_str() { - // wizard用の初期カウンティングではstatusとlevelの内容を確認したうえで以降の処理は行わないようにする - let counter = result_container.entry(s.into()).or_insert(HashMap::new()); - if included_target_tag_vec.is_empty() { - *counter.entry(yaml_doc["level"] - .as_str() - .unwrap_or("informational") - .to_uppercase().into()).or_insert(HashMap::new()).entry("other".into()).or_insert(0) += 1; - } else { - for tag in included_target_tag_vec { - *counter.entry(yaml_doc["level"] + if let Some(s) = yaml_doc["status"].as_str() { + // wizard用の初期カウンティングではstatusとlevelの内容を確認したうえで以降の処理は行わないようにする + let counter = result_container.entry(s.into()).or_insert(HashMap::new()); + if included_target_tag_vec.is_empty() { + *counter + .entry( + yaml_doc["level"] .as_str() .unwrap_or("informational") - .to_uppercase().into()).or_insert(HashMap::new()).entry(tag.into()).or_insert(0) += 1; - } - } + .to_uppercase() + .into(), + ) + .or_insert(HashMap::new()) + .entry("other".into()) + .or_insert(0) += 1; + } else { + for tag in included_target_tag_vec { + *counter + .entry( + yaml_doc["level"] + .as_str() + .unwrap_or("informational") + .to_uppercase() + .into(), + ) + .or_insert(HashMap::new()) + .entry(tag.into()) + .or_insert(0) += 1; + } } - }); - result_container.to_owned() - } + } + }); + result_container.to_owned() +} #[cfg(test)] mod tests { From 6a074955ab4810e90d4be5bc1678df2adeb1cc1a Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Sat, 18 Nov 2023 21:21:48 +0900 Subject: [PATCH 04/16] docs(CHANGELOG): added next version template --- CHANGELOG-Japanese.md | 18 ++++++++++++++++++ CHANGELOG.md | 18 ++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/CHANGELOG-Japanese.md b/CHANGELOG-Japanese.md index b729656ff..98baa2fd2 100644 --- a/CHANGELOG-Japanese.md +++ b/CHANGELOG-Japanese.md @@ -1,5 +1,23 @@ # 変更点 +## 2.11.0 [2023/XX/XX] "XXX Release" + +**新機能:** + +- xxx + +**改善:** + +- xxx + +**バグ修正:** + +- xxx + +**その他:** + +- xxx + ## 2.10.1 [2023/11/12] "Kamemushi Release" **改善:** diff --git a/CHANGELOG.md b/CHANGELOG.md index 32942e58a..753ab5be4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,23 @@ # Changes +## 2.11.0 [2023/XX/XX] "XX Release" + +**New Features:** + +- XXX + +**Enhancements:** + +- XXX + +**Bug Fixes:** + +- XXX + +**Other:** + +- XXX + ## 2.10.1 [2023/11/13] "Kamemushi Release" **Enhancements:** From fef2a345df69141959f5a5ab03b6ea88cea9440a Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Sat, 18 Nov 2023 21:23:36 +0900 Subject: [PATCH 05/16] docs(CHANGELOG): added #1206 --- CHANGELOG-Japanese.md | 2 +- CHANGELOG.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG-Japanese.md b/CHANGELOG-Japanese.md index 98baa2fd2..027c14946 100644 --- a/CHANGELOG-Japanese.md +++ b/CHANGELOG-Japanese.md @@ -8,7 +8,7 @@ **改善:** -- xxx +- スキャンウィザードにルール数を追加した. (#1206) (@hitenkoku) **バグ修正:** diff --git a/CHANGELOG.md b/CHANGELOG.md index 753ab5be4..518b153cc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ **Enhancements:** -- XXX +- Added rule count in the scan wizard. (#1206) (@hitenkoku) **Bug Fixes:** From 691ee18dd5af99e8748cf8c950bc1ccb430d566e Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Sun, 19 Nov 2023 13:41:02 +0900 Subject: [PATCH 06/16] feat(main): fixed wizard output included exclude status #1206 --- src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main.rs b/src/main.rs index bf326c5f7..c52197828 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1106,7 +1106,7 @@ impl App { .map(|(_, (status, min_level))| { calcurate_wizard_rule_count( false, - [].to_vec(), + ["exclude", "noisy", "deprecated", "unsupported"].to_vec(), min_level, status.to_vec(), [].to_vec(), From f8cf6e6a7ad7927c542bd49ba4053ed8df63d2fa Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Sun, 19 Nov 2023 13:54:26 +0900 Subject: [PATCH 07/16] fix(main): fixed typo #1206 --- src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main.rs b/src/main.rs index c52197828..8e2d51b53 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1106,7 +1106,7 @@ impl App { .map(|(_, (status, min_level))| { calcurate_wizard_rule_count( false, - ["exclude", "noisy", "deprecated", "unsupported"].to_vec(), + ["excluded", "noisy", "deprecated", "unsupported"].to_vec(), min_level, status.to_vec(), [].to_vec(), From 0d771429c51a75497e70c640cd7c7b4966e1fc56 Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Sun, 19 Nov 2023 16:44:31 +0900 Subject: [PATCH 08/16] fix(main): fixed noisy rule count miss in wizard output #1205 --- src/main.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main.rs b/src/main.rs index 8e2d51b53..0399e859e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1136,7 +1136,7 @@ impl App { let exclude_noisy_cnt = calcurate_wizard_rule_count( true, - ["exclude", "noisy", "deprecated", "unsupported"].to_vec(), + ["exclude", "excluded", "noisy", "deprecated", "unsupported"].to_vec(), selections_status[selected_index].1 .1, [].to_vec(), [].to_vec(), @@ -1217,7 +1217,7 @@ impl App { // noisy rules load prompt let prompt_fmt = format!( "Include noisy rules? ({} rules)", - tags_cnt.get("noisy").unwrap_or(&0) + exclude_noisy_cnt.get("noisy").unwrap_or(&0) ); let noisy_rules_load_flag = Confirm::with_theme(&ColorfulTheme::default()) .with_prompt(prompt_fmt) From c9ad9a0ede1b565e96c06ef23005ae03e4598265 Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Thu, 23 Nov 2023 11:26:27 +0900 Subject: [PATCH 09/16] feat(yaml): modified rule count from id base to file base #1206 --- src/yaml.rs | 55 +++++++++++++++++++++-------------------------------- 1 file changed, 22 insertions(+), 33 deletions(-) diff --git a/src/yaml.rs b/src/yaml.rs index 36a4cf937..b93eb6691 100644 --- a/src/yaml.rs +++ b/src/yaml.rs @@ -284,11 +284,8 @@ impl ParseYaml { } if let Some(id) = rule_id { if !stored_static.target_ruleids.is_target(id, true) { - if !self.loaded_rule_ids.contains(*id) { - let entry = self.rule_load_cnt.entry("excluded".into()).or_insert(0); - *entry += 1; - self.loaded_rule_ids.insert(CompactString::from(*id)); - } + let entry = self.rule_load_cnt.entry("excluded".into()).or_insert(0); + *entry += 1; return Option::None; } } @@ -299,12 +296,9 @@ impl ParseYaml { *status_cnt += 1; }; - let mut up_rule_load_cnt = |status: &str, rule_id: &str| { - if !self.loaded_rule_ids.contains(rule_id) { - let entry = self.rule_load_cnt.entry(status.into()).or_insert(0); - *entry += 1; - self.loaded_rule_ids.insert(rule_id.into()); - } + let mut up_rule_load_cnt = |status: &str| { + let entry = self.rule_load_cnt.entry(status.into()).or_insert(0); + *entry += 1; }; // 指定されたレベルより低いルールは無視する @@ -318,7 +312,7 @@ impl ParseYaml { if doc_level_num < args_level_num || (target_level_num != &0_u128 && doc_level_num != target_level_num) { - up_rule_load_cnt("excluded", rule_id.unwrap_or(&String::default())); + up_rule_load_cnt("excluded"); return Option::None; } @@ -329,7 +323,7 @@ impl ParseYaml { || !(is_contained_include_status_all_allowed || stored_static.include_status.contains(s)) { - up_rule_load_cnt("excluded", rule_id.unwrap_or(&String::default())); + up_rule_load_cnt("excluded"); return Option::None; } if exist_output_opt @@ -380,13 +374,13 @@ impl ParseYaml { if !include_category.is_empty() && !include_category.contains(&category_in_rule.to_string()) { - up_rule_load_cnt("excluded", rule_id.unwrap_or(&String::default())); + up_rule_load_cnt("excluded"); return Option::None; } if !exclude_category.is_empty() && exclude_category.contains(&category_in_rule.to_string()) { - up_rule_load_cnt("excluded", rule_id.unwrap_or(&String::default())); + up_rule_load_cnt("excluded"); return Option::None; } } @@ -413,11 +407,11 @@ impl ParseYaml { target_tags.contains(&tag.as_str().unwrap_or_default().to_string()) }); if !is_match { - up_rule_load_cnt("excluded", rule_id.unwrap_or(&String::default())); + up_rule_load_cnt("excluded"); return Option::None; } } else { - up_rule_load_cnt("excluded", rule_id.unwrap_or(&String::default())); + up_rule_load_cnt("excluded"); return Option::None; } } @@ -444,29 +438,24 @@ impl ParseYaml { exclude_target_tags.contains(&tag.as_str().unwrap_or_default().to_string()) }); if is_match { - up_rule_load_cnt("excluded", rule_id.unwrap_or(&String::default())); + up_rule_load_cnt("excluded"); return Option::None; } } } - if !self - .loaded_rule_ids - .contains(rule_id.unwrap_or(&String::default())) - { - self.rulecounter.insert( - yaml_doc["ruletype"].as_str().unwrap_or("Other").into(), - self.rulecounter - .get(yaml_doc["ruletype"].as_str().unwrap_or("Other")) - .unwrap_or(&0) - + 1, - ); + self.rulecounter.insert( + yaml_doc["ruletype"].as_str().unwrap_or("Other").into(), + self.rulecounter + .get(yaml_doc["ruletype"].as_str().unwrap_or("Other")) + .unwrap_or(&0) + + 1, + ); - up_rule_status_cnt(status.unwrap_or("undefined")); + up_rule_status_cnt(status.unwrap_or("undefined")); - self.loaded_rule_ids - .insert(rule_id.unwrap_or(&String::default()).into()); - } + self.loaded_rule_ids + .insert(rule_id.unwrap_or(&String::default()).into()); if stored_static.verbose_flag { println!("Loaded rule: {filepath}"); From a34f3ae6fdfb0f5f71740d3bda8c0ca494f7692c Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Thu, 23 Nov 2023 18:48:07 +0900 Subject: [PATCH 10/16] feat(main): added noisy rule count in wizard section count #1206 --- src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main.rs b/src/main.rs index 0399e859e..8a2cf98d6 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1106,7 +1106,7 @@ impl App { .map(|(_, (status, min_level))| { calcurate_wizard_rule_count( false, - ["excluded", "noisy", "deprecated", "unsupported"].to_vec(), + ["excluded", "deprecated", "unsupported"].to_vec(), min_level, status.to_vec(), [].to_vec(), From bdb4b008b8addc391eece9e877dd1ed6c4525df5 Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Thu, 23 Nov 2023 19:26:41 +0900 Subject: [PATCH 11/16] feat(main): no output when include/exclude rule count is 0 in wizard #1206 --- src/main.rs | 178 ++++++++++++++++++++++++++++------------------------ 1 file changed, 96 insertions(+), 82 deletions(-) diff --git a/src/main.rs b/src/main.rs index 8a2cf98d6..2a34ff6cb 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1051,7 +1051,9 @@ impl App { } }); } - ret.insert(CompactString::from(s), ret_cnt); + if ret_cnt > 0 { + ret.insert(CompactString::from(s), ret_cnt); + } } } else { let all_status_flag = target_status.contains(&"*"); @@ -1085,7 +1087,7 @@ impl App { } } }); - if !exclude_noisy_status.contains(&s.as_str()) { + if !exclude_noisy_status.contains(&s.as_str()) && ret_cnt > 0 { ret.insert(s.clone(), ret_cnt); } } @@ -1166,106 +1168,118 @@ impl App { ); // If anything other than "4. All alert rules" or "5. All event and alert rules" was selected, ask questions about tags. if selected_index < 3 { + if let Some(et_cnt) = tags_cnt.get("detection.emerging_threats") { + let prompt_fmt = format!( + "Include Emerging Threats rules? ({} rules)", + et_cnt + ); + let et_rules_load_flag = Confirm::with_theme(&ColorfulTheme::default()) + .with_prompt(prompt_fmt) + .default(true) + .show_default(true) + .interact() + .unwrap(); + // If no is selected, then add "--exclude-tags detection.emerging_threats" + if !et_rules_load_flag { + exclude_tags.push("detection.emerging_threats".into()); + } + } + if let Some(th_cnt) = tags_cnt.get("detection.threat_hunting") { + let prompt_fmt = format!( + "Include Threat Hunting rules? ({} rules)", + th_cnt + ); + let th_rules_load_flag = Confirm::with_theme(&ColorfulTheme::default()) + .with_prompt(prompt_fmt) + .default(false) + .show_default(true) + .interact() + .unwrap(); + // If no is selected, then add "--exclude-tags detection.threat_hunting" + if !th_rules_load_flag { + exclude_tags.push("detection.threat_hunting".into()); + } + } + } + if let Some(dep_cnt) = exclude_noisy_cnt.get("deprecated") { + // deprecated rules load prompt let prompt_fmt = format!( - "Include Emerging Threats rules? ({} rules)", - tags_cnt.get("detection.emerging_threats").unwrap_or(&0) + "Include deprecated rules? ({} rules)", + dep_cnt ); - let et_rules_load_flag = Confirm::with_theme(&ColorfulTheme::default()) + let dep_rules_load_flag = Confirm::with_theme(&ColorfulTheme::default()) .with_prompt(prompt_fmt) - .default(true) + .default(false) .show_default(true) .interact() .unwrap(); - // If no is selected, then add "--exclude-tags detection.emerging_threats" - if !et_rules_load_flag { - exclude_tags.push("detection.emerging_threats".into()); + if dep_rules_load_flag { + stored_static + .output_option + .as_mut() + .unwrap() + .enable_deprecated_rules = true; } + } + + if let Some(noisy_cnt) = exclude_noisy_cnt.get("noisy") { + // noisy rules load prompt let prompt_fmt = format!( - "Include Threat Hunting rules? ({} rules)", - tags_cnt.get("detection.threat_hunting").unwrap_or(&0) + "Include noisy rules? ({} rules)", + noisy_cnt ); - let th_rules_load_flag = Confirm::with_theme(&ColorfulTheme::default()) + let noisy_rules_load_flag = Confirm::with_theme(&ColorfulTheme::default()) .with_prompt(prompt_fmt) .default(false) .show_default(true) .interact() .unwrap(); - // If no is selected, then add "--exclude-tags detection.threat_hunting" - if !th_rules_load_flag { - exclude_tags.push("detection.threat_hunting".into()); + if noisy_rules_load_flag { + stored_static + .output_option + .as_mut() + .unwrap() + .enable_noisy_rules = true; } } - // deprecated rules load prompt - let prompt_fmt = format!( - "Include deprecated rules? ({} rules)", - exclude_noisy_cnt.get("deprecated").unwrap_or(&0) - ); - let dep_rules_load_flag = Confirm::with_theme(&ColorfulTheme::default()) - .with_prompt(prompt_fmt) - .default(false) - .show_default(true) - .interact() - .unwrap(); - if dep_rules_load_flag { - stored_static - .output_option - .as_mut() - .unwrap() - .enable_deprecated_rules = true; - } - // noisy rules load prompt - let prompt_fmt = format!( - "Include noisy rules? ({} rules)", - exclude_noisy_cnt.get("noisy").unwrap_or(&0) - ); - let noisy_rules_load_flag = Confirm::with_theme(&ColorfulTheme::default()) - .with_prompt(prompt_fmt) - .default(false) - .show_default(true) - .interact() - .unwrap(); - if noisy_rules_load_flag { - stored_static - .output_option - .as_mut() - .unwrap() - .enable_noisy_rules = true; - } - - // unsupported rules load prompt - let prompt_fmt = format!( - "Include unsupported rules? ({} rules)", - exclude_noisy_cnt.get("unsupported").unwrap_or(&0) - ); - let unsupported_rules_load_flag = Confirm::with_theme(&ColorfulTheme::default()) - .with_prompt(prompt_fmt) - .default(false) - .show_default(true) - .interact() - .unwrap(); - if unsupported_rules_load_flag { - stored_static - .output_option - .as_mut() - .unwrap() - .enable_unsupported_rules = true; + if let Some(unsup_cnt) = exclude_noisy_cnt.get("unsupported") { + // unsupported rules load prompt + let prompt_fmt = format!( + "Include unsupported rules? ({} rules)", + unsup_cnt + ); + let unsupported_rules_load_flag = Confirm::with_theme(&ColorfulTheme::default()) + .with_prompt(prompt_fmt) + .default(false) + .show_default(true) + .interact() + .unwrap(); + if unsupported_rules_load_flag { + stored_static + .output_option + .as_mut() + .unwrap() + .enable_unsupported_rules = true; + } } - let prompt_fmt = format!( - "Include sysmon rules? ({} rules)", - tags_cnt.get("sysmon").unwrap_or(&0) - ); - let sysmon_rules_load_flag = Confirm::with_theme(&ColorfulTheme::default()) - .with_prompt(prompt_fmt) - .default(true) - .show_default(true) - .interact() - .unwrap(); + if let Some(sysmon_cnt) = tags_cnt.get("sysmon") { + let prompt_fmt = format!( + "Include sysmon rules? ({} rules)", + sysmon_cnt + ); + let sysmon_rules_load_flag = Confirm::with_theme(&ColorfulTheme::default()) + .with_prompt(prompt_fmt) + .default(true) + .show_default(true) + .interact() + .unwrap(); - // If no is selected, then add "--exclude-tags sysmon" - if !sysmon_rules_load_flag { - exclude_tags.push("sysmon".into()); + // If no is selected, then add "--exclude-tags sysmon" + if !sysmon_rules_load_flag { + exclude_tags.push("sysmon".into()); + } } if !exclude_tags.is_empty() { From 65886a8469bda1a62234771e1e2f129ca0c851b8 Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Thu, 23 Nov 2023 19:27:13 +0900 Subject: [PATCH 12/16] style: cargo fmt --- src/main.rs | 30 ++++++------------------------ 1 file changed, 6 insertions(+), 24 deletions(-) diff --git a/src/main.rs b/src/main.rs index 2a34ff6cb..ea27f9911 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1169,10 +1169,7 @@ impl App { // If anything other than "4. All alert rules" or "5. All event and alert rules" was selected, ask questions about tags. if selected_index < 3 { if let Some(et_cnt) = tags_cnt.get("detection.emerging_threats") { - let prompt_fmt = format!( - "Include Emerging Threats rules? ({} rules)", - et_cnt - ); + let prompt_fmt = format!("Include Emerging Threats rules? ({} rules)", et_cnt); let et_rules_load_flag = Confirm::with_theme(&ColorfulTheme::default()) .with_prompt(prompt_fmt) .default(true) @@ -1185,10 +1182,7 @@ impl App { } } if let Some(th_cnt) = tags_cnt.get("detection.threat_hunting") { - let prompt_fmt = format!( - "Include Threat Hunting rules? ({} rules)", - th_cnt - ); + let prompt_fmt = format!("Include Threat Hunting rules? ({} rules)", th_cnt); let th_rules_load_flag = Confirm::with_theme(&ColorfulTheme::default()) .with_prompt(prompt_fmt) .default(false) @@ -1203,10 +1197,7 @@ impl App { } if let Some(dep_cnt) = exclude_noisy_cnt.get("deprecated") { // deprecated rules load prompt - let prompt_fmt = format!( - "Include deprecated rules? ({} rules)", - dep_cnt - ); + let prompt_fmt = format!("Include deprecated rules? ({} rules)", dep_cnt); let dep_rules_load_flag = Confirm::with_theme(&ColorfulTheme::default()) .with_prompt(prompt_fmt) .default(false) @@ -1224,10 +1215,7 @@ impl App { if let Some(noisy_cnt) = exclude_noisy_cnt.get("noisy") { // noisy rules load prompt - let prompt_fmt = format!( - "Include noisy rules? ({} rules)", - noisy_cnt - ); + let prompt_fmt = format!("Include noisy rules? ({} rules)", noisy_cnt); let noisy_rules_load_flag = Confirm::with_theme(&ColorfulTheme::default()) .with_prompt(prompt_fmt) .default(false) @@ -1245,10 +1233,7 @@ impl App { if let Some(unsup_cnt) = exclude_noisy_cnt.get("unsupported") { // unsupported rules load prompt - let prompt_fmt = format!( - "Include unsupported rules? ({} rules)", - unsup_cnt - ); + let prompt_fmt = format!("Include unsupported rules? ({} rules)", unsup_cnt); let unsupported_rules_load_flag = Confirm::with_theme(&ColorfulTheme::default()) .with_prompt(prompt_fmt) .default(false) @@ -1265,10 +1250,7 @@ impl App { } if let Some(sysmon_cnt) = tags_cnt.get("sysmon") { - let prompt_fmt = format!( - "Include sysmon rules? ({} rules)", - sysmon_cnt - ); + let prompt_fmt = format!("Include sysmon rules? ({} rules)", sysmon_cnt); let sysmon_rules_load_flag = Confirm::with_theme(&ColorfulTheme::default()) .with_prompt(prompt_fmt) .default(true) From e65e1d40f02ae9317027b0063ebabf797d8d298d Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Thu, 23 Nov 2023 19:35:17 +0900 Subject: [PATCH 13/16] feat(main): no asking about deprecated and unsupported excluded when All alert or event selected #1206 --- src/main.rs | 70 +++++++++++++++++++++++++++-------------------------- 1 file changed, 36 insertions(+), 34 deletions(-) diff --git a/src/main.rs b/src/main.rs index ea27f9911..dddf8d8c8 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1194,22 +1194,42 @@ impl App { exclude_tags.push("detection.threat_hunting".into()); } } - } - if let Some(dep_cnt) = exclude_noisy_cnt.get("deprecated") { - // deprecated rules load prompt - let prompt_fmt = format!("Include deprecated rules? ({} rules)", dep_cnt); - let dep_rules_load_flag = Confirm::with_theme(&ColorfulTheme::default()) - .with_prompt(prompt_fmt) - .default(false) - .show_default(true) - .interact() - .unwrap(); - if dep_rules_load_flag { - stored_static - .output_option - .as_mut() - .unwrap() - .enable_deprecated_rules = true; + } else { + // If "4. All alert rules" or "5. All event and alert rules" was selected, ask questions about deprecated and unsupported rules. + if let Some(dep_cnt) = exclude_noisy_cnt.get("deprecated") { + // deprecated rules load prompt + let prompt_fmt = format!("Include deprecated rules? ({} rules)", dep_cnt); + let dep_rules_load_flag = Confirm::with_theme(&ColorfulTheme::default()) + .with_prompt(prompt_fmt) + .default(false) + .show_default(true) + .interact() + .unwrap(); + if dep_rules_load_flag { + stored_static + .output_option + .as_mut() + .unwrap() + .enable_deprecated_rules = true; + } + } + if let Some(unsup_cnt) = exclude_noisy_cnt.get("unsupported") { + // unsupported rules load prompt + let prompt_fmt = format!("Include unsupported rules? ({} rules)", unsup_cnt); + let unsupported_rules_load_flag = + Confirm::with_theme(&ColorfulTheme::default()) + .with_prompt(prompt_fmt) + .default(false) + .show_default(true) + .interact() + .unwrap(); + if unsupported_rules_load_flag { + stored_static + .output_option + .as_mut() + .unwrap() + .enable_unsupported_rules = true; + } } } @@ -1231,24 +1251,6 @@ impl App { } } - if let Some(unsup_cnt) = exclude_noisy_cnt.get("unsupported") { - // unsupported rules load prompt - let prompt_fmt = format!("Include unsupported rules? ({} rules)", unsup_cnt); - let unsupported_rules_load_flag = Confirm::with_theme(&ColorfulTheme::default()) - .with_prompt(prompt_fmt) - .default(false) - .show_default(true) - .interact() - .unwrap(); - if unsupported_rules_load_flag { - stored_static - .output_option - .as_mut() - .unwrap() - .enable_unsupported_rules = true; - } - } - if let Some(sysmon_cnt) = tags_cnt.get("sysmon") { let prompt_fmt = format!("Include sysmon rules? ({} rules)", sysmon_cnt); let sysmon_rules_load_flag = Confirm::with_theme(&ColorfulTheme::default()) From 7aa6ba5d868f37cecdebc6bd801140e53e8a38bb Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Sun, 26 Nov 2023 22:32:02 +0900 Subject: [PATCH 14/16] fix(main, yaml): removed duplicated rule count by tag in wizard #1206 --- src/main.rs | 27 +++++++++++----------- src/yaml.rs | 64 +++++++++++++++++++++++++++++++---------------------- 2 files changed, 52 insertions(+), 39 deletions(-) diff --git a/src/main.rs b/src/main.rs index dddf8d8c8..05ba8afdd 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1033,7 +1033,7 @@ impl App { min_level: &str, target_status: Vec<&str>, target_tags: Vec<&str>| - -> HashMap { + -> HashMap { let mut ret = HashMap::new(); if exclude_noisytarget_flag { for s in exclude_noisy_status { @@ -1047,7 +1047,7 @@ impl App { .get(min_level.to_uppercase().as_str()) .unwrap_or(&1); if doc_level_num >= args_level_num { - ret_cnt += value.iter().map(|(_, cnt)| cnt).sum::() + ret_cnt += value.iter().map(|(_, cnt)| cnt).sum::() } }); } @@ -1083,11 +1083,11 @@ impl App { } } } else { - ret_cnt += value.iter().map(|(_, cnt)| cnt).sum::() + ret_cnt += value.iter().map(|(t, cnt)| cnt).sum::() } } }); - if !exclude_noisy_status.contains(&s.as_str()) && ret_cnt > 0 { + if ret_cnt > 0 { ret.insert(s.clone(), ret_cnt); } } @@ -1106,21 +1106,22 @@ impl App { let sections_rule_cnt = selections_status .iter() .map(|(_, (status, min_level))| { - calcurate_wizard_rule_count( + let ret = calcurate_wizard_rule_count( false, - ["excluded", "deprecated", "unsupported"].to_vec(), + ["excluded", "deprecated", "unsupported", "noisy"].to_vec(), min_level, status.to_vec(), [].to_vec(), - ) + ); + ret }) .collect_vec(); let selection_status_items = &[ - format!("1. Core ({} rules) ( status: test, stable | level: high, critical )", sections_rule_cnt[0].iter().map(|(_, cnt)| cnt).sum::()), - format!("2. Core+ ({} rules) ( status: test, stable | level: medium, high, critical )", sections_rule_cnt[1].iter().map(|(_, cnt)| cnt).sum::()), - format!("3. Core++ ({} rules) ( status: experimental, test, stable | level: medium, high, critical )", sections_rule_cnt[2].iter().map(|(_, cnt)| cnt).sum::()), - format!("4. All alert rules ({} rules) ( status: * | level: low+ )", sections_rule_cnt[3].iter().map(|(_, cnt)| cnt).sum::()), - format!("5. All event and alert rules ({} rules) ( status: * | level: informational+ )", sections_rule_cnt[4].iter().map(|(_, cnt)| cnt).sum::()) + format!("1. Core ({} rules) ( status: test, stable | level: high, critical )", sections_rule_cnt[0].iter().map(|(_, cnt)| cnt).sum::()), + format!("2. Core+ ({} rules) ( status: test, stable | level: medium, high, critical )", sections_rule_cnt[1].iter().map(|(_, cnt)| cnt).sum::()), + format!("3. Core++ ({} rules) ( status: experimental, test, stable | level: medium, high, critical )", sections_rule_cnt[2].iter().map(|(_, cnt)| cnt).sum::()), + format!("4. All alert rules ({} rules) ( status: * | level: low+ )", sections_rule_cnt[3].iter().map(|(_, cnt)| cnt).sum::()), + format!("5. All event and alert rules ({} rules) ( status: * | level: informational+ )", sections_rule_cnt[4].iter().map(|(_, cnt)| cnt).sum::()) ]; let selected_index = Select::with_theme(&ColorfulTheme::default()) @@ -1138,7 +1139,7 @@ impl App { let exclude_noisy_cnt = calcurate_wizard_rule_count( true, - ["exclude", "excluded", "noisy", "deprecated", "unsupported"].to_vec(), + ["excluded", "noisy", "deprecated", "unsupported"].to_vec(), selections_status[selected_index].1 .1, [].to_vec(), [].to_vec(), diff --git a/src/yaml.rs b/src/yaml.rs index b93eb6691..404220626 100644 --- a/src/yaml.rs +++ b/src/yaml.rs @@ -269,7 +269,6 @@ impl ParseYaml { // テスト用のルール(ID:000...0)の場合はexcluded ruleのカウントから除外するようにする if v != "00000000-0000-0000-0000-000000000000" { let entry = self.rule_load_cnt.entry(entry_key.into()).or_insert(0); - self.loaded_rule_ids.insert(v.into()); *entry += 1; } let enable_noisy_rules = if let Some(o) = stored_static.output_option.as_ref() { @@ -454,9 +453,6 @@ impl ParseYaml { up_rule_status_cnt(status.unwrap_or("undefined")); - self.loaded_rule_ids - .insert(rule_id.unwrap_or(&String::default()).into()); - if stored_static.verbose_flag { println!("Loaded rule: {filepath}"); } @@ -475,9 +471,9 @@ pub fn count_rules>( stored_static: &StoredStatic, result_container: &mut HashMap< CompactString, - HashMap>, + HashMap>, >, -) -> HashMap>> { +) -> HashMap>> { let metadata = fs::metadata(path.as_ref()); if metadata.is_err() { return HashMap::default(); @@ -613,26 +609,29 @@ pub fn count_rules>( } else { "noisy" }; - let counter = result_container - .entry(entry_key.into()) - .or_insert(HashMap::new()); - *counter - .entry( - yaml_doc["level"] - .as_str() - .unwrap_or("informational") - .to_uppercase() - .into(), - ) - .or_insert(HashMap::new()) - .entry( - yaml_doc["status"] - .as_str() - .unwrap_or("undefined") - .to_lowercase() - .into(), - ) - .or_insert(0) += 1; + // テスト用のルール(ID:000...0)の場合はexcluded ruleのカウントから除外するようにする + if v != "00000000-0000-0000-0000-000000000000" { + let counter = result_container + .entry(entry_key.into()) + .or_insert(HashMap::new()); + *counter + .entry( + yaml_doc["level"] + .as_str() + .unwrap_or("informational") + .to_uppercase() + .into(), + ) + .or_insert(HashMap::new()) + .entry( + yaml_doc["status"] + .as_str() + .unwrap_or("undefined") + .to_lowercase() + .into(), + ) + .or_insert(0) += 1; + } return; } } @@ -653,6 +652,19 @@ pub fn count_rules>( .entry("other".into()) .or_insert(0) += 1; } else { + if included_target_tag_vec.len() > 1 { + *counter + .entry( + yaml_doc["level"] + .as_str() + .unwrap_or("informational") + .to_uppercase() + .into(), + ) + .or_insert(HashMap::new()) + .entry("duplicated".into()) + .or_insert(0) -= (included_target_tag_vec.len() - 1) as i128; + } for tag in included_target_tag_vec { *counter .entry( From 71ef72bf6e54fbd640dd56774014895fea2c0c5e Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Mon, 27 Nov 2023 09:24:24 +0900 Subject: [PATCH 15/16] style: fixed cargo clippy error --- src/main.rs | 7 ++--- src/options/profile.rs | 63 ------------------------------------------ 2 files changed, 3 insertions(+), 67 deletions(-) diff --git a/src/main.rs b/src/main.rs index 056db2927..f31cf51f2 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1083,7 +1083,7 @@ impl App { } } } else { - ret_cnt += value.iter().map(|(t, cnt)| cnt).sum::() + ret_cnt += value.iter().map(|(_, cnt)| cnt).sum::() } } }); @@ -1106,14 +1106,13 @@ impl App { let sections_rule_cnt = selections_status .iter() .map(|(_, (status, min_level))| { - let ret = calcurate_wizard_rule_count( + calcurate_wizard_rule_count( false, ["excluded", "deprecated", "unsupported", "noisy"].to_vec(), min_level, status.to_vec(), [].to_vec(), - ); - ret + ) }) .collect_vec(); let selection_status_items = &[ diff --git a/src/options/profile.rs b/src/options/profile.rs index 5f8082039..395776a77 100644 --- a/src/options/profile.rs +++ b/src/options/profile.rs @@ -414,69 +414,6 @@ mod tests { #[test] fn test_get_profile_list_err_load() { - let dummy_stored_static = - create_dummy_stored_static(Action::CsvTimeline(CsvOutputOption { - output_options: OutputOption { - input_args: InputOption { - directory: None, - filepath: None, - live_analysis: false, - recover_records: false, - timeline_offset: None, - }, - profile: None, - enable_deprecated_rules: false, - exclude_status: None, - min_level: "informational".to_string(), - exact_level: None, - enable_noisy_rules: false, - end_timeline: None, - start_timeline: None, - eid_filter: false, - european_time: false, - iso_8601: false, - rfc_2822: false, - rfc_3339: false, - us_military_time: false, - us_time: false, - utc: false, - visualize_timeline: false, - rules: Path::new("./rules").to_path_buf(), - html_report: None, - no_summary: false, - common_options: CommonOptions { - no_color: false, - quiet: false, - }, - detect_common_options: DetectCommonOption { - evtx_file_ext: None, - thread_number: None, - quiet_errors: false, - config: Path::new("./rules/config").to_path_buf(), - verbose: false, - json_input: false, - include_computer: None, - exclude_computer: None, - }, - enable_unsupported_rules: false, - clobber: false, - proven_rules: false, - include_tag: None, - exclude_tag: None, - include_category: None, - exclude_category: None, - include_eid: None, - exclude_eid: None, - no_field: false, - no_pwsh_field_extraction: false, - remove_duplicate_data: false, - remove_duplicate_detections: false, - no_wizard: true, - }, - geo_ip: None, - output: None, - multiline: false, - })); assert_eq!( Nested::>::new(), get_profile_list("test_files/not_exist_path") From fe7280088a878747d0e030a781e567a8a665fc59 Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Mon, 27 Nov 2023 21:10:08 +0900 Subject: [PATCH 16/16] fix(main): removed excluded rule count #1206 --- src/main.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main.rs b/src/main.rs index f31cf51f2..57314c14f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1116,11 +1116,11 @@ impl App { }) .collect_vec(); let selection_status_items = &[ - format!("1. Core ({} rules) ( status: test, stable | level: high, critical )", sections_rule_cnt[0].iter().map(|(_, cnt)| cnt).sum::()), - format!("2. Core+ ({} rules) ( status: test, stable | level: medium, high, critical )", sections_rule_cnt[1].iter().map(|(_, cnt)| cnt).sum::()), - format!("3. Core++ ({} rules) ( status: experimental, test, stable | level: medium, high, critical )", sections_rule_cnt[2].iter().map(|(_, cnt)| cnt).sum::()), - format!("4. All alert rules ({} rules) ( status: * | level: low+ )", sections_rule_cnt[3].iter().map(|(_, cnt)| cnt).sum::()), - format!("5. All event and alert rules ({} rules) ( status: * | level: informational+ )", sections_rule_cnt[4].iter().map(|(_, cnt)| cnt).sum::()) + format!("1. Core ({} rules) ( status: test, stable | level: high, critical )", sections_rule_cnt[0].iter().map(|(_, cnt)| cnt).sum::() - sections_rule_cnt[0].get("excluded").unwrap_or(&0)), + format!("2. Core+ ({} rules) ( status: test, stable | level: medium, high, critical )", sections_rule_cnt[1].iter().map(|(_, cnt)| cnt).sum::() - sections_rule_cnt[1].get("excluded").unwrap_or(&0)), + format!("3. Core++ ({} rules) ( status: experimental, test, stable | level: medium, high, critical )", sections_rule_cnt[2].iter().map(|(_, cnt)| cnt).sum::() - sections_rule_cnt[2].get("excluded").unwrap_or(&0)), + format!("4. All alert rules ({} rules) ( status: * | level: low+ )", sections_rule_cnt[3].iter().map(|(_, cnt)| cnt).sum::() - sections_rule_cnt[3].get("excluded").unwrap_or(&0)), + format!("5. All event and alert rules ({} rules) ( status: * | level: informational+ )", sections_rule_cnt[4].iter().map(|(_, cnt)| cnt).sum::() - sections_rule_cnt[4].get("excluded").unwrap_or(&0)) ]; let selected_index = Select::with_theme(&ColorfulTheme::default())