From 5263a63c53e6c0a6c96c062e048171e3b811f729 Mon Sep 17 00:00:00 2001 From: fukusuket <41001169+fukusuket@users.noreply.github.com> Date: Sat, 5 Oct 2024 05:03:40 +0900 Subject: [PATCH 1/4] feat: load config files from a single file --- src/detections/configs.rs | 51 +++++++++++++++++++++++++--- src/detections/field_data_map.rs | 18 ++++++++++ src/detections/utils.rs | 57 +++++++++++++++++++++++--------- src/filter.rs | 46 ++++++++++++++++---------- src/main.rs | 23 ++++++++++++- 5 files changed, 155 insertions(+), 40 deletions(-) diff --git a/src/detections/configs.rs b/src/detections/configs.rs index edafed7bb..f584c1176 100644 --- a/src/detections/configs.rs +++ b/src/detections/configs.rs @@ -14,9 +14,11 @@ use itertools::Itertools; use lazy_static::lazy_static; use regex::Regex; use std::env::current_exe; +use std::fs::File; +use std::io::BufRead; use std::path::{Path, PathBuf}; use std::sync::RwLock; -use std::{fs, process}; +use std::{fs, io, process}; use terminal_size::{terminal_size, Width}; use yaml_rust::{Yaml, YamlLoader}; @@ -39,6 +41,37 @@ lazy_static! { .match_kind(MatchKind::LeftmostLongest) .build(["🛂r", "🛂n", "🛂t"]) .unwrap(); + pub static ref ONE_CONFIG_MAP: HashMap = + read_one_config_file(Path::new("rules_config_files.txt")).unwrap_or_default(); +} + +fn read_one_config_file(file_path: &Path) -> io::Result> { + let file = File::open(file_path)?; + let reader = io::BufReader::new(file); + + let mut sections = HashMap::new(); + let mut current_path = String::new(); + let mut current_content = String::new(); + let mut in_content = false; + + for line in reader.lines() { + let line = line?; + if line.starts_with("---FILE_START---") { + current_path.clear(); + current_content.clear(); + in_content = false; + } else if let Some(path) = line.strip_prefix("path: ") { + current_path = path.to_string(); + } else if line.starts_with("---CONTENT---") { + in_content = true; + } else if line.starts_with("---FILE_END---") { + sections.insert(current_path.clone(), current_content.clone()); + } else if in_content { + current_content.push_str(&line); + current_content.push('\n'); + } + } + Ok(sections) } pub struct ConfigReader { @@ -248,16 +281,24 @@ impl StoredStatic { ) .unwrap() }); - if !geo_ip_file_path.exists() { + if !geo_ip_file_path.exists() + && !ONE_CONFIG_MAP.contains_key("geoip_field_mapping.yaml") + { AlertMessage::alert( "Could not find the geoip_field_mapping.yaml config file. Please run update-rules." ) .ok(); process::exit(1); } - let geo_ip_mapping = if let Ok(loaded_yaml) = - YamlLoader::load_from_str(&fs::read_to_string(geo_ip_file_path).unwrap()) - { + let contents = if ONE_CONFIG_MAP.contains_key("geoip_field_mapping.yaml") { + ONE_CONFIG_MAP + .get("geoip_field_mapping.yaml") + .unwrap() + .as_str() + } else { + &fs::read_to_string(geo_ip_file_path).unwrap() + }; + let geo_ip_mapping = if let Ok(loaded_yaml) = YamlLoader::load_from_str(contents) { loaded_yaml } else { AlertMessage::alert("Parse error in geoip_field_mapping.yaml.").ok(); diff --git a/src/detections/field_data_map.rs b/src/detections/field_data_map.rs index f041ce3aa..c45108243 100644 --- a/src/detections/field_data_map.rs +++ b/src/detections/field_data_map.rs @@ -1,3 +1,4 @@ +use crate::detections::configs::ONE_CONFIG_MAP; use crate::detections::field_data_map::FieldDataConverter::{HexToDecimal, ReplaceStr}; use crate::detections::message::AlertMessage; use crate::detections::utils::get_serde_number_to_string; @@ -165,6 +166,23 @@ fn load_yaml_files(dir_path: &Path) -> Result, String> { } pub fn create_field_data_map(dir_path: &Path) -> Option { + let one_config_values: Vec = ONE_CONFIG_MAP + .iter() + .filter(|(key, _)| key.contains(".yaml") && !key.contains("geoip_field_mapping.yaml")) + .map(|(_, value)| value.clone()) + .collect(); + if !one_config_values.is_empty() { + let yaml_contents: Vec = one_config_values + .iter() + .flat_map(|value| YamlLoader::load_from_str(value).unwrap_or_default()) + .collect(); + return Some( + yaml_contents + .into_iter() + .map(build_field_data_map) + .collect(), + ); + } let yaml_data = load_yaml_files(dir_path); match yaml_data { Ok(y) => Some(y.into_iter().map(build_field_data_map).collect()), diff --git a/src/detections/utils.rs b/src/detections/utils.rs index 941effc0a..09e7acc39 100644 --- a/src/detections/utils.rs +++ b/src/detections/utils.rs @@ -26,7 +26,7 @@ use termcolor::{BufferWriter, ColorSpec, WriteColor}; use termcolor::{Color, ColorChoice}; use tokio::runtime::{Builder, Runtime}; -use crate::detections::configs::CURRENT_EXE_PATH; +use crate::detections::configs::{CURRENT_EXE_PATH, ONE_CONFIG_MAP}; use crate::detections::field_data_map::{convert_field_data, FieldDataMap, FieldDataMapKey}; use crate::detections::field_extract::extract_fields; use crate::options::htmlreport; @@ -93,13 +93,24 @@ pub fn read_txt(filename: &str) -> Result, String> { } else { filename.to_string() }; + let re = Regex::new(r".*/").unwrap(); + let one_config_path = &re.replace(filename, "").to_string(); + if ONE_CONFIG_MAP.contains_key(one_config_path) { + return Ok(Nested::from_iter( + ONE_CONFIG_MAP + .get(one_config_path) + .unwrap() + .lines() + .map(|s| s.to_string()), + )); + } let f = File::open(filepath); if f.is_err() { let errmsg = format!("Cannot open file. [file:{filename}]"); - return Result::Err(errmsg); + return Err(errmsg); } let reader = BufReader::new(f.unwrap()); - Result::Ok(Nested::from_iter( + Ok(Nested::from_iter( reader.lines().map(|line| line.unwrap_or_default()), )) } @@ -167,18 +178,24 @@ pub fn read_json_to_value(path: &str) -> Result>, } pub fn read_csv(filename: &str) -> Result>, String> { + let re = Regex::new(r".*/").unwrap(); + let one_config_path = &re.replace(filename, "").to_string(); + if ONE_CONFIG_MAP.contains_key(one_config_path) { + let csv_res = parse_csv(ONE_CONFIG_MAP.get(one_config_path).unwrap()); + return Ok(csv_res); + } let f = File::open(filename); if f.is_err() { - return Result::Err(format!("Cannot open file. [file:{filename}]")); + return Err(format!("Cannot open file. [file:{filename}]")); } let mut contents: String = String::new(); let read_res = f.unwrap().read_to_string(&mut contents); if let Err(e) = read_res { - return Result::Err(e.to_string()); + return Err(e.to_string()); } let csv_res = parse_csv(&contents); - Result::Ok(csv_res) + Ok(csv_res) } pub fn parse_csv(file_contents: &str) -> Nested> { @@ -553,7 +570,10 @@ pub fn make_ascii_titlecase(s: &str) -> CompactString { /// base_path/path が存在するかを確認し、存在しなければカレントディレクトリを参照するpathを返す関数 pub fn check_setting_path(base_path: &Path, path: &str, ignore_err: bool) -> Option { - if base_path.join(path).exists() { + let re = Regex::new(r".*/").unwrap(); + if ONE_CONFIG_MAP.contains_key(&re.replace(path, "").to_string()) { + Some(path.into()) + } else if base_path.join(path).exists() { Some(base_path.join(path)) } else if ignore_err { Some(Path::new(path).to_path_buf()) @@ -564,6 +584,20 @@ pub fn check_setting_path(base_path: &Path, path: &str, ignore_err: bool) -> Opt /// rule configのファイルの所在を確認する関数。 pub fn check_rule_config(config_path: &PathBuf) -> Result<(), String> { + // 各種ファイルを確認する + let files = vec![ + "channel_abbreviations.txt", + "target_event_IDs.txt", + "default_details.txt", + "level_tuning.txt", + "channel_eid_info.txt", + "eventkey_alias.txt", + ]; + let all_keys_present = files.iter().all(|key| ONE_CONFIG_MAP.contains_key(*key)); + if all_keys_present { + return Ok(()); + } + // rules/configのフォルダが存在するかを確認する let exist_rule_config_folder = if config_path == &CURRENT_EXE_PATH.to_path_buf() { check_setting_path(config_path, "rules/config", false).is_some() @@ -574,15 +608,6 @@ pub fn check_rule_config(config_path: &PathBuf) -> Result<(), String> { return Err("The required rules and config files were not found. Please download them with the update-rules command.".to_string()); } - // 各種ファイルを確認する - let files = vec![ - "channel_abbreviations.txt", - "target_event_IDs.txt", - "default_details.txt", - "level_tuning.txt", - "channel_eid_info.txt", - "eventkey_alias.txt", - ]; let mut not_exist_file = vec![]; for file in &files { if check_setting_path(config_path, file, false).is_none() { diff --git a/src/filter.rs b/src/filter.rs index 0d84ef866..69cce4c8c 100644 --- a/src/filter.rs +++ b/src/filter.rs @@ -1,4 +1,4 @@ -use crate::detections::configs::{self, StoredStatic}; +use crate::detections::configs::{self, StoredStatic, ONE_CONFIG_MAP}; use crate::detections::message::{AlertMessage, ERROR_LOG_STACK}; use crate::detections::rule::RuleNode; use evtx::EvtxParser; @@ -58,24 +58,34 @@ pub fn exclude_ids(stored_static: &StoredStatic) -> RuleExclude { impl RuleExclude { fn insert_ids(&mut self, filename: &str, stored_static: &StoredStatic) { - let f = File::open(filename); - if f.is_err() { - if stored_static.verbose_flag { - AlertMessage::warn(&format!("{filename} does not exist")).ok(); - } - if !stored_static.quiet_errors_flag { - ERROR_LOG_STACK - .lock() - .unwrap() - .push(format!("{filename} does not exist")); + let re = Regex::new(r".*/").unwrap(); + let one_config_path = &re.replace(filename, "").to_string(); + let lines: Vec = if ONE_CONFIG_MAP.contains_key(one_config_path) { + ONE_CONFIG_MAP + .get(one_config_path) + .unwrap() + .split('\n') + .map(|s| s.to_string()) + .collect() + } else { + let f = File::open(filename); + if f.is_err() { + if stored_static.verbose_flag { + AlertMessage::warn(&format!("{filename} does not exist")).ok(); + } + if !stored_static.quiet_errors_flag { + ERROR_LOG_STACK + .lock() + .unwrap() + .push(format!("{filename} does not exist")); + } + return; } - return; - } - let reader = BufReader::new(f.unwrap()); - for v in reader.lines() { - let v = v.unwrap().split('#').collect::>()[0] - .trim() - .to_string(); + let reader = BufReader::new(f.unwrap()); + reader.lines().map_while(Result::ok).collect() + }; + for v in lines { + let v = v.split('#').collect::>()[0].trim().to_string(); if v.is_empty() || !configs::IDS_REGEX.is_match(&v) { // 空行は無視する。IDの検証 continue; diff --git a/src/main.rs b/src/main.rs index 8dd93e16d..a6b7846d9 100644 --- a/src/main.rs +++ b/src/main.rs @@ -33,7 +33,7 @@ use hayabusa::afterfact::{self, AfterfactInfo, AfterfactWriter}; use hayabusa::debug::checkpoint_process_timer::CHECKPOINT; use hayabusa::detections::configs::{ load_pivot_keywords, Action, ConfigReader, EventKeyAliasConfig, StoredStatic, TargetEventTime, - TargetIds, CURRENT_EXE_PATH, STORED_EKEY_ALIAS, STORED_STATIC, + TargetIds, CURRENT_EXE_PATH, ONE_CONFIG_MAP, STORED_EKEY_ALIAS, STORED_STATIC, }; use hayabusa::detections::detection::{self, EvtxRecordInfo}; use hayabusa::detections::message::{AlertMessage, DetectInfo, ERROR_LOG_STACK}; @@ -574,6 +574,27 @@ impl App { AlertMessage::alert("Failed to update rules.").ok(); } } + + if !ONE_CONFIG_MAP.is_empty() { + let url = "https://raw.githubusercontent.com/Yamato-Security/hayabusa-encoded-rules/refs/heads/main/rules_config_files.txt"; + match get(url).call() { + Ok(res) => { + let mut dst = + File::create(Path::new("./rules_config_files.txt")).unwrap(); + copy(&mut res.into_reader(), &mut dst).unwrap(); + write_color_buffer( + &BufferWriter::stdout(ColorChoice::Always), + None, + "Config file rules_config_files.txt updated successfully.", + true, + ) + .ok(); + } + Err(_) => { + AlertMessage::alert("Failed to update config file.").ok(); + } + } + } } else { match Update::update_rules( update_target.unwrap().to_str().unwrap(), From ec30d4a05e995bf0e1d42961d67ef509619e5588 Mon Sep 17 00:00:00 2001 From: fukusuket <41001169+fukusuket@users.noreply.github.com> Date: Sat, 5 Oct 2024 05:35:40 +0900 Subject: [PATCH 2/4] fix: load encoded_rules.yml when pivot-keywords-list --- src/main.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/main.rs b/src/main.rs index a6b7846d9..1ba233b08 100644 --- a/src/main.rs +++ b/src/main.rs @@ -443,6 +443,14 @@ impl App { .to_str() .unwrap(), ); + if Path::new("./encoded_rules.yml").exists() { + stored_static.output_option.as_mut().unwrap().rules = check_setting_path( + &CURRENT_EXE_PATH.to_path_buf(), + "encoded_rules.yml", + true, + ) + .unwrap(); + } // pivot 機能でファイルを出力する際に同名ファイルが既に存在していた場合はエラー文を出して終了する。 let mut error_flag = false; From 0d07ef7613eb97623272f04bff18c2654cd7b8de Mon Sep 17 00:00:00 2001 From: Yamato Security <71482215+YamatoSecurity@users.noreply.github.com> Date: Sat, 5 Oct 2024 09:34:02 +0900 Subject: [PATCH 3/4] update changelog --- CHANGELOG-Japanese.md | 3 ++- CHANGELOG.md | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG-Japanese.md b/CHANGELOG-Japanese.md index f34de12de..7aafa22dd 100644 --- a/CHANGELOG-Japanese.md +++ b/CHANGELOG-Japanese.md @@ -4,10 +4,11 @@ **新機能:** -- `fieldref`モディファイアに対応した。(#1409) (@hitenkoku) +- `fieldref`モディファイア(`equalsfield`モディファイアのエリアス)に対応した。(#1409) (@hitenkoku) - XORエンコードされたルールをサポートし、端末に置かれるファイルを最小限に抑えるとともに、ルールに過検知するアンチウイルス製品を回避する。(#1419) (@fukusuket) - リリースページで、この機能を設定済みのパッケージを含める予定。手動で設定したい場合は、[encoded_rules.yml](https://github.com/Yamato-Security/hayabusa-encoded-rules/raw/refs/heads/main/encoded_rules.yml)をダウンロードして、Hayabusaのルートフォルダに置いてください。このファイルは、hayabusa-rulesリポジトリ内のルールから作成されており、ルールが更新されるたびに自動的にアップデートされる。configディレクトリ以外のrulesフォルダ内のファイルは、まだ単一ファイルに含まれていないので削除してください。 - 注意: -Hオプションで生成されるレポートは、ルールへのリンクを作成せず、ルール名だけが出力される。 +- `rules/config`の設定ファイルが単一のファイル[rules_config_files.txt](https://github.com/Yamato-Security/hayabusa-encoded-rules/raw/refs/heads/main/rules_config_files.txt)からロードされるようになり、ライブ調査のためにターゲットシステムに保存する必要があるファイル数が減った。(#1420) (@fukusuket) ## 2.17.0 [2024/08/23] "HITCON Community Release" diff --git a/CHANGELOG.md b/CHANGELOG.md index ccc1accdf..1d33d1801 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,10 +4,11 @@ **New Features:** -- Support for the `fieldref` modifier. (#1409) (@hitenkoku) +- Support for the `fieldref` modifier (alias to the `equalsfield` modifier). (#1409) (@hitenkoku) - Support for XOR encoded rules to minimize files put on the system as well as bypass anti-virus products that give false positives on rules. (#1419) (@fukusuket) - We will include packages in the Releases page that are already configured to use this. If you wanted to manually configure this though, download [encoded_rules.yml](https://github.com/Yamato-Security/hayabusa-encoded-rules/raw/refs/heads/main/encoded_rules.yml) and place it in the Hayabusa's root folder. This file is created from the rules in the hayabusa-rules repository and is automatically updated anytime there is a rule update. Delete all of the files inside the `rules` folder except for the `config` directory as those files are not yet contained in a single file. - Note: The report generated by the `-H` option cannot create a link to the rule (only the rule name is outputted.) + - `rules/config` config files are now loaded from a single file [rules_config_files.txt](https://github.com/Yamato-Security/hayabusa-encoded-rules/raw/refs/heads/main/rules_config_files.txt) to reduce the number of files needed to be stored on a target system for live response. (#1420) (@fukusuket) ## 2.17.0 [2024/08/23] "HITCON Community Release" From d39f4318d8f4c4bde973d6bb6118f4ee9dc7d0d4 Mon Sep 17 00:00:00 2001 From: fukusuket <41001169+fukusuket@users.noreply.github.com> Date: Sat, 5 Oct 2024 10:16:01 +0900 Subject: [PATCH 4/4] chg: add rules && encoded_rules.yml incompatible check --- src/main.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/main.rs b/src/main.rs index 1ba233b08..3b4e73d75 100644 --- a/src/main.rs +++ b/src/main.rs @@ -172,6 +172,12 @@ impl App { return; } + if Path::new("encoded_rules.yml").exists() && Path::new("rules").exists() { + println!("You have the rules directory and encoded_rules.yml in your path. Please delete one of them."); + println!(); + return; + } + // カレントディレクトリ以外からの実行の際にrules-configオプションの指定がないとエラーが発生することを防ぐための処理 if stored_static.config_path == Path::new("./rules/config") { stored_static.config_path =