From f5a67376e03e5e9ba212f4e82d3f21de822d0942 Mon Sep 17 00:00:00 2001 From: fukusuket <41001169+fukusuket@users.noreply.github.com> Date: Sat, 28 Sep 2024 10:43:40 +0900 Subject: [PATCH] feat: support encoded rules to avoid AV false positives --- src/afterfact.rs | 105 +++++++++++++++++++----------------- src/detections/detection.rs | 2 + src/detections/message.rs | 1 + src/main.rs | 91 +++++++++++++++++++++---------- src/yaml.rs | 84 ++++++++++++++++++++++++++--- 5 files changed, 198 insertions(+), 85 deletions(-) diff --git a/src/afterfact.rs b/src/afterfact.rs index 7572a2095..637ef2715 100644 --- a/src/afterfact.rs +++ b/src/afterfact.rs @@ -23,19 +23,17 @@ use num_format::{Locale, ToFormattedString}; use termcolor::{Buffer, BufferWriter, Color, ColorChoice, ColorSpec, WriteColor}; use terminal_size::terminal_size; use terminal_size::Width; -use yaml_rust::YamlLoader; use crate::detections::configs::{ Action, OutputOption, StoredStatic, CONTROL_CHAT_REPLACE_MAP, CURRENT_EXE_PATH, GEOIP_DB_PARSER, }; use crate::detections::message::{AlertMessage, DetectInfo, COMPUTER_MITRE_ATTCK_MAP, LEVEL_FULL}; use crate::detections::utils::{ - self, format_time, get_writable_color, output_and_data_stack_for_html, parse_csv, - write_color_buffer, + self, check_setting_path, format_time, get_writable_color, output_and_data_stack_for_html, + parse_csv, write_color_buffer, }; use crate::options::htmlreport; use crate::options::profile::Profile; -use crate::yaml::ParseYaml; use rust_embed::Embed; lazy_static! { @@ -524,7 +522,7 @@ fn calc_statistic_info( let author_list = afterfact_info .author_list_cache .entry(detect_info.rulepath.clone()) - .or_insert_with(|| extract_author_name(&detect_info.rulepath)) + .or_insert_with(|| extract_author_name(&detect_info.ruleauther)) .clone(); let author_str = author_list.iter().join(", "); afterfact_info @@ -1572,18 +1570,34 @@ fn _print_detection_summary_tables( LEVEL_FULL.get(level[1].as_str()).unwrap(), LEVEL_FULL.get(level[1].as_str()).unwrap() )); + let rule_path = stored_static.output_option.as_ref().unwrap().rules.clone(); + let rule_encoded = + check_setting_path(&CURRENT_EXE_PATH.to_path_buf(), "encoded_rules.yml", true) + .unwrap(); + let is_encoded_rule = rule_encoded.exists() && rule_encoded.to_path_buf() == rule_path; for x in sorted_detections.iter() { let not_found_str = CompactString::from_str("").unwrap(); let rule_path = rule_title_path_map.get(x.0).unwrap_or(¬_found_str); - html_output_stock.push(format!( - "- [{}]({}) ({}) - {}", - x.0, - &rule_path.replace('\\', "/"), - x.1.to_formatted_string(&Locale::en), - rule_detect_author_map - .get(rule_path) - .unwrap_or(¬_found_str) - )); + if is_encoded_rule { + html_output_stock.push(format!( + "- {} ({}) - {}", + x.0, + x.1.to_formatted_string(&Locale::en), + rule_detect_author_map + .get(rule_path) + .unwrap_or(¬_found_str) + )); + } else { + html_output_stock.push(format!( + "- [{}]({}) ({}) - {}", + x.0, + &rule_path.replace('\\', "/"), + x.1.to_formatted_string(&Locale::en), + rule_detect_author_map + .get(rule_path) + .unwrap_or(¬_found_str) + )); + } } html_output_stock.push(""); } @@ -2178,44 +2192,24 @@ fn output_detected_rule_authors( } /// 与えられたyaml_pathからauthorの名前を抽出して配列で返却する関数 -fn extract_author_name(yaml_path: &str) -> Nested { - let contents = match ParseYaml::read_file(&Path::new(&yaml_path).to_path_buf()) { - Ok(yaml) => Some(yaml), - Err(e) => { - AlertMessage::alert(&e).ok(); - None - } - }; - if contents.is_none() { - // 対象のファイルが存在しなかった場合は空配列を返す(検知しているルールに対して行うため、ここは通る想定はないが、ファイルが検知途中で削除された場合などを考慮して追加) - return Nested::new(); +fn extract_author_name(author: &str) -> Nested { + let mut ret = Nested::::new(); + for author in author.split(',').map(|s| { + // 各要素の括弧以降の記載は名前としないためtmpの一番最初の要素のみを参照する + // データの中にdouble quote と single quoteが入っているためここで除外する + s.split('(').next().unwrap_or_default().to_string() + }) { + ret.extend(author.split(';')); } - for yaml in YamlLoader::load_from_str(&contents.unwrap()) - .unwrap_or_default() - .into_iter() - { - if let Some(author) = yaml["author"].as_str() { - let mut ret = Nested::::new(); - for author in author.split(',').map(|s| { - // 各要素の括弧以降の記載は名前としないためtmpの一番最初の要素のみを参照する - // データの中にdouble quote と single quoteが入っているためここで除外する - s.split('(').next().unwrap_or_default().to_string() - }) { - ret.extend(author.split(';')); - } - return ret - .iter() - .map(|r| { - r.split('/') - .map(|p| p.trim().replace(['"', '\''], "")) - .collect::() - }) - .collect(); - }; - } - // ここまで来た場合は要素がない場合なので空配列を返す - Nested::new() + return ret + .iter() + .map(|r| { + r.split('/') + .map(|p| p.trim().replace(['"', '\''], "")) + .collect::() + }) + .collect(); } ///MITRE ATTCKのTacticsの属性を持つルールに検知したコンピュータ名をhtml出力するための文字列をhtml_output_stockに追加する関数 @@ -2504,6 +2498,7 @@ mod tests { rulepath: CompactString::from(test_rulepath), ruleid: test_rule_id.into(), ruletitle: CompactString::from(test_title), + ruleauther: CompactString::from("test_author"), level: CompactString::from(test_level), computername: CompactString::from(test_computername2), eventid: CompactString::from(test_eventid), @@ -2528,6 +2523,7 @@ mod tests { rulepath: CompactString::from(test_rulepath), ruleid: test_rule_id.into(), ruletitle: CompactString::from(test_title), + ruleauther: CompactString::from("test_author"), level: CompactString::from(test_level), computername: CompactString::from(test_computername), eventid: CompactString::from(test_eventid), @@ -2851,6 +2847,7 @@ mod tests { rulepath: CompactString::from(test_rulepath), ruleid: test_rule_id.into(), ruletitle: CompactString::from(test_title), + ruleauther: CompactString::from("test_author"), level: CompactString::from(test_level), computername: CompactString::from(test_computername2), eventid: CompactString::from(test_eventid), @@ -2875,6 +2872,7 @@ mod tests { rulepath: CompactString::from(test_rulepath), ruleid: test_rule_id.into(), ruletitle: CompactString::from(test_title), + ruleauther: CompactString::from("test_author"), level: CompactString::from(test_level), computername: CompactString::from(test_computername), eventid: CompactString::from(test_eventid), @@ -3178,6 +3176,7 @@ mod tests { rulepath: CompactString::from(test_rulepath), ruleid: test_rule_id.into(), ruletitle: CompactString::from(test_title), + ruleauther: CompactString::from("test_author"), level: CompactString::from(test_level), computername: CompactString::from(test_computername2), eventid: CompactString::from(test_eventid), @@ -3202,6 +3201,7 @@ mod tests { rulepath: CompactString::from(test_rulepath), ruleid: test_rule_id.into(), ruletitle: CompactString::from(test_title), + ruleauther: CompactString::from("test_author"), level: CompactString::from(test_level), computername: CompactString::from(test_computername), eventid: CompactString::from(test_eventid), @@ -3515,6 +3515,7 @@ mod tests { rulepath: CompactString::from(test_rulepath), ruleid: test_rule_id.into(), ruletitle: CompactString::from(test_title), + ruleauther: CompactString::from("test_author"), level: CompactString::from(test_level), computername: CompactString::from(test_computername2), eventid: CompactString::from(test_eventid), @@ -3539,6 +3540,7 @@ mod tests { rulepath: CompactString::from(test_rulepath), ruleid: test_rule_id.into(), ruletitle: CompactString::from(test_title), + ruleauther: CompactString::from("test_author"), level: CompactString::from(test_level), computername: CompactString::from(test_computername), eventid: CompactString::from(test_eventid), @@ -3924,6 +3926,7 @@ mod tests { rulepath: CompactString::from(test_rulepath), ruleid: test_rule_id.into(), ruletitle: CompactString::from(test_title), + ruleauther: CompactString::from("test_author"), level: CompactString::from(test_level), computername: CompactString::from(test_computername), eventid: CompactString::from(test_eventid), @@ -4278,6 +4281,7 @@ mod tests { rulepath: CompactString::from(test_rulepath), ruleid: test_rule_id.into(), ruletitle: CompactString::from(test_title), + ruleauther: CompactString::from("test_author"), level: CompactString::from(test_level), computername: CompactString::from(test_computername2), eventid: CompactString::from(test_eventid), @@ -4558,6 +4562,7 @@ mod tests { rulepath: CompactString::from(test_rulepath), ruleid: test_rule_id.into(), ruletitle: CompactString::from(test_title), + ruleauther: CompactString::from("test_author"), level: CompactString::from(test_level), computername: CompactString::from(test_computername2), eventid: CompactString::from(test_eventid), diff --git a/src/detections/detection.rs b/src/detections/detection.rs index d3f1d30f7..e64b22213 100644 --- a/src/detections/detection.rs +++ b/src/detections/detection.rs @@ -731,6 +731,7 @@ impl Detection { rulepath: CompactString::from(&rule.rulepath), ruleid: CompactString::from(rule.yaml["id"].as_str().unwrap_or("-")), ruletitle: CompactString::from(rule.yaml["title"].as_str().unwrap_or("-")), + ruleauther: CompactString::from(rule.yaml["author"].as_str().unwrap_or("-")), level: CompactString::from( LEVEL_ABBR_MAP .get(&level.as_str()) @@ -997,6 +998,7 @@ impl Detection { rulepath: CompactString::from(&rule.rulepath), ruleid: CompactString::from(rule.yaml["id"].as_str().unwrap_or("-")), ruletitle: CompactString::from(rule.yaml["title"].as_str().unwrap_or("-")), + ruleauther: CompactString::from(rule.yaml["author"].as_str().unwrap_or("-")), level: CompactString::from( LEVEL_ABBR_MAP .get(str_level) diff --git a/src/detections/message.rs b/src/detections/message.rs index 39cb9f154..595057070 100644 --- a/src/detections/message.rs +++ b/src/detections/message.rs @@ -36,6 +36,7 @@ pub struct DetectInfo { pub rulepath: CompactString, pub ruleid: CompactString, pub ruletitle: CompactString, + pub ruleauther: CompactString, pub level: CompactString, pub computername: CompactString, pub eventid: CompactString, diff --git a/src/main.rs b/src/main.rs index a3644a7d3..3e01704e1 100644 --- a/src/main.rs +++ b/src/main.rs @@ -8,7 +8,7 @@ use std::borrow::BorrowMut; use std::ffi::{OsStr, OsString}; use std::fmt::Display; use std::fmt::Write as _; -use std::io::{BufWriter, Write}; +use std::io::{copy, BufWriter, Write}; use std::path::Path; use std::ptr::null_mut; use std::sync::Arc; @@ -29,20 +29,6 @@ use dialoguer::Confirm; use dialoguer::{theme::ColorfulTheme, Select}; use evtx::{EvtxParser, ParserSettings, RecordAllocation}; use hashbrown::{HashMap, HashSet}; -use indicatif::ProgressBar; -use indicatif::{ProgressDrawTarget, ProgressStyle}; -use itertools::Itertools; -use libmimalloc_sys::mi_stats_print_out; -use mimalloc::MiMalloc; -use nested::Nested; -use num_format::{Locale, ToFormattedString}; -use rust_embed::Embed; -use serde_json::{Map, Value}; -use termcolor::{BufferWriter, Color, ColorChoice}; -use tokio::runtime::Runtime; -use tokio::spawn; -use tokio::task::JoinHandle; - use hayabusa::afterfact::{self, AfterfactInfo, AfterfactWriter}; use hayabusa::debug::checkpoint_process_timer::CHECKPOINT; use hayabusa::detections::configs::{ @@ -66,8 +52,22 @@ use hayabusa::timeline::computer_metrics::countup_event_by_computer; use hayabusa::{detections::configs, timeline::timelines::Timeline}; use hayabusa::{detections::utils::write_color_buffer, filter}; use hayabusa::{options, yaml}; +use indicatif::ProgressBar; +use indicatif::{ProgressDrawTarget, ProgressStyle}; #[cfg(target_os = "windows")] use is_elevated::is_elevated; +use itertools::Itertools; +use libmimalloc_sys::mi_stats_print_out; +use mimalloc::MiMalloc; +use nested::Nested; +use num_format::{Locale, ToFormattedString}; +use rust_embed::Embed; +use serde_json::{Map, Value}; +use termcolor::{BufferWriter, Color, ColorChoice}; +use tokio::runtime::Runtime; +use tokio::spawn; +use tokio::task::JoinHandle; +use ureq::get; #[derive(Embed)] #[folder = "art/"] @@ -267,9 +267,18 @@ impl App { Action::CsvTimeline(_) | Action::JsonTimeline(_) => { // カレントディレクトリ以外からの実行の際にrulesオプションの指定がないとエラーが発生することを防ぐための処理 if stored_static.output_option.as_ref().unwrap().rules == Path::new("./rules") { - stored_static.output_option.as_mut().unwrap().rules = - utils::check_setting_path(&CURRENT_EXE_PATH.to_path_buf(), "rules", true) - .unwrap(); + if Path::new("./encoded_rules.yml").exists() { + stored_static.output_option.as_mut().unwrap().rules = check_setting_path( + &CURRENT_EXE_PATH.to_path_buf(), + "encoded_rules.yml", + true, + ) + .unwrap(); + } else { + stored_static.output_option.as_mut().unwrap().rules = + check_setting_path(&CURRENT_EXE_PATH.to_path_buf(), "rules", true) + .unwrap(); + } } // rule configのフォルダ、ファイルを確認してエラーがあった場合は終了とする if let Err(e) = utils::check_rule_config(&stored_static.config_path) { @@ -544,24 +553,50 @@ impl App { let latest_version_data = Update::get_latest_hayabusa_version().unwrap_or_default(); let now_version = &format!("v{}", env!("CARGO_PKG_VERSION")); stored_static.include_status.insert("*".into()); - match Update::update_rules(update_target.unwrap().to_str().unwrap(), stored_static) - { - Ok(output) => { - if output != "You currently have the latest rules." { + let rule_encoded = + check_setting_path(&CURRENT_EXE_PATH.to_path_buf(), "encoded_rules.yml", true) + .unwrap(); + if rule_encoded.exists() { + let url = "https://raw.githubusercontent.com/Yamato-Security/hayabusa-encoded-rules/main/encoded_rules.yml"; + match get(url).call() { + Ok(res) => { + let mut dst = File::create(Path::new("./encoded_rules.yml")).unwrap(); + copy(&mut res.into_reader(), &mut dst).unwrap(); write_color_buffer( &BufferWriter::stdout(ColorChoice::Always), None, - "Rules updated successfully.", + "Rules(encoded_rules.yml) updated successfully.", true, ) .ok(); } - } - Err(e) => { - if e.message().is_empty() { + Err(_) => { AlertMessage::alert("Failed to update rules.").ok(); - } else { - AlertMessage::alert(&format!("Failed to update rules. {e:?} ")).ok(); + } + } + } else { + match Update::update_rules( + update_target.unwrap().to_str().unwrap(), + stored_static, + ) { + Ok(output) => { + if output != "You currently have the latest rules." { + write_color_buffer( + &BufferWriter::stdout(ColorChoice::Always), + None, + "Rules updated successfully.", + true, + ) + .ok(); + } + } + Err(e) => { + if e.message().is_empty() { + AlertMessage::alert("Failed to update rules.").ok(); + } else { + AlertMessage::alert(&format!("Failed to update rules. {e:?} ")) + .ok(); + } } } } diff --git a/src/yaml.rs b/src/yaml.rs index 4ac858766..b41c392eb 100644 --- a/src/yaml.rs +++ b/src/yaml.rs @@ -71,6 +71,17 @@ impl ParseYaml { Ok(file_content) } + fn read_encoded_file(path: &PathBuf) -> Result { + let mut fr = fs::File::open(path) + .map(BufReader::new) + .map_err(|e| e.to_string())?; + let mut encrypted_content = Vec::new(); + let _ = fr.read_to_end(&mut encrypted_content); + let decode_content = encrypted_content.iter().map(|&b| b ^ 0xAA).collect(); // key: 0xAA + let decode_string = String::from_utf8(decode_content).expect("Invalid UTF-8 sequence"); + Ok(decode_string) + } + fn update_correlation_counts(&mut self, yaml_docs: &Vec) { for doc in yaml_docs { if let Some(correlation) = doc["correlation"].as_hash() { @@ -96,6 +107,7 @@ impl ParseYaml { } } } + pub fn read_dir>( &mut self, path: P, @@ -143,9 +155,20 @@ impl ParseYaml { { return io::Result::Ok(String::default()); } - // 個別のファイルの読み込みは即終了としない。 - let read_content = Self::read_file(&path.as_ref().to_path_buf()); + let mut is_encoded = false; + let read_content = if path + .as_ref() + .to_path_buf() + .file_name() + .unwrap_or_else(|| OsStr::new("")) + == "encoded_rules.yml" + { + is_encoded = true; + Self::read_encoded_file(&path.as_ref().to_path_buf()) + } else { + Self::read_file(&path.as_ref().to_path_buf()) + }; if read_content.is_err() { let errmsg = format!( "fail to read file: {}\n{} ", @@ -170,7 +193,14 @@ impl ParseYaml { Ok(contents) => { Self::update_correlation_counts(self, &contents); yaml_docs.extend(contents.into_iter().map(|yaml_content| { - let filepath = format!("{}", path.as_ref().to_path_buf().display()); + let filepath = if is_encoded { + yaml_content["rulefile"] + .as_str() + .unwrap_or_default() + .to_string() + } else { + format!("{}", path.as_ref().to_path_buf().display()) + }; (filepath, yaml_content) })); } @@ -526,7 +556,20 @@ pub fn count_rules>( } // 個別のファイルの読み込みは即終了としない。 - let read_content = ParseYaml::read_file(&path.as_ref().to_path_buf()); + let mut is_encoded = false; + let read_content = if path + .as_ref() + .to_path_buf() + .file_name() + .unwrap_or_else(|| OsStr::new("")) + == "encoded_rules.yml" + { + is_encoded = true; + ParseYaml::read_encoded_file(&path.as_ref().to_path_buf()) + } else { + ParseYaml::read_file(&path.as_ref().to_path_buf()) + }; + if read_content.is_err() { return HashMap::default(); } @@ -538,7 +581,14 @@ pub fn count_rules>( } yaml_docs.extend(yaml_contents.unwrap().into_iter().map(|yaml_content| { - let filepath = format!("{}", path.as_ref().to_path_buf().display()); + let filepath = if is_encoded { + yaml_content["rulefile"] + .as_str() + .unwrap_or_default() + .to_string() + } else { + format!("{}", path.as_ref().to_path_buf().display()) + }; (filepath, yaml_content) })); } else { @@ -720,7 +770,6 @@ pub fn count_rules>( #[cfg(test)] mod tests { - use crate::detections::configs::Action; use crate::detections::configs::CommonOptions; use crate::detections::configs::Config; @@ -736,7 +785,9 @@ mod tests { use compact_str::CompactString; use hashbrown::HashMap; use hashbrown::HashSet; - use std::path::Path; + use std::fs::File; + use std::io::Write; + use std::path::{Path, PathBuf}; use yaml_rust::YamlLoader; fn create_dummy_stored_static() -> StoredStatic { @@ -1263,4 +1314,23 @@ mod tests { .unwrap(); assert_eq!(yaml.files.len(), 5); } + + #[test] + fn test_read_encoded_file() { + let test_path = PathBuf::from("test_encoded_file"); + let encoded_content: Vec = vec![ + b'H' ^ 0xAA, + b'e' ^ 0xAA, + b'l' ^ 0xAA, + b'l' ^ 0xAA, + b'o' ^ 0xAA, + ]; + let mut file = File::create(&test_path).expect("Failed to create test file"); + file.write_all(&encoded_content) + .expect("Failed to write to test file"); + let result = ParseYaml::read_encoded_file(&test_path); + std::fs::remove_file(&test_path).expect("Failed to delete test file"); + assert!(result.is_ok()); + assert_eq!(result.unwrap(), "Hello"); + } }