Merge pull request #1423 from Yamato-Security/1420-load-config-from-s…

…igle-file

feat: load config files from a single file

CHANGELOG-Japanese.md

@@ -4,10 +4,11 @@
 
 **新機能:**
 
-- `fieldref`モディファイアに対応した。(#1409) (@hitenkoku)
+- `fieldref`モディファイア(`equalsfield`モディファイアのエリアス)に対応した。(#1409) (@hitenkoku)
 - XORエンコードされたルールをサポートし、端末に置かれるファイルを最小限に抑えるとともに、ルールに過検知するアンチウイルス製品を回避する。(#1419) (@fukusuket)
   - リリースページで、この機能を設定済みのパッケージを含める予定。手動で設定したい場合は、[encoded_rules.yml](https://github.com/Yamato-Security/hayabusa-encoded-rules/raw/refs/heads/main/encoded_rules.yml)をダウンロードして、Hayabusaのルートフォルダに置いてください。このファイルは、hayabusa-rulesリポジトリ内のルールから作成されており、ルールが更新されるたびに自動的にアップデートされる。configディレクトリ以外のrulesフォルダ内のファイルは、まだ単一ファイルに含まれていないので削除してください。
   - 注意: -Hオプションで生成されるレポートは、ルールへのリンクを作成せず、ルール名だけが出力される。
+- `rules/config`の設定ファイルが単一のファイル[rules_config_files.txt](https://github.com/Yamato-Security/hayabusa-encoded-rules/raw/refs/heads/main/rules_config_files.txt)からロードされるようになり、ライブ調査のためにターゲットシステムに保存する必要があるファイル数が減った。(#1420) (@fukusuket)
 
 ## 2.17.0 [2024/08/23] "HITCON Community Release"
 

CHANGELOG.md

@@ -4,10 +4,11 @@
 
 **New Features:**
 
-- Support for the `fieldref` modifier. (#1409) (@hitenkoku)
+- Support for the `fieldref` modifier (alias to the `equalsfield` modifier). (#1409) (@hitenkoku)
 - Support for XOR encoded rules to minimize files put on the system as well as bypass anti-virus products that give false positives on rules. (#1419) (@fukusuket)
   - We will include packages in the Releases page that are already configured to use this. If you wanted to manually configure this though, download [encoded_rules.yml](https://github.com/Yamato-Security/hayabusa-encoded-rules/raw/refs/heads/main/encoded_rules.yml) and place it in the Hayabusa's root folder. This file is created from the rules in the hayabusa-rules repository and is automatically updated anytime there is a rule update. Delete all of the files inside the `rules` folder except for the `config` directory as those files are not yet contained in a single file.
   - Note: The report generated by the `-H` option cannot create a link to the rule (only the rule name is outputted.) 
+  - `rules/config` config files are now loaded from a single file [rules_config_files.txt](https://github.com/Yamato-Security/hayabusa-encoded-rules/raw/refs/heads/main/rules_config_files.txt) to reduce the number of files needed to be stored on a target system for live response. (#1420) (@fukusuket)
 
 ## 2.17.0 [2024/08/23] "HITCON Community Release"
 

src/detections/configs.rs

@@ -14,9 +14,11 @@ use itertools::Itertools;
 use lazy_static::lazy_static;
 use regex::Regex;
 use std::env::current_exe;
+use std::fs::File;
+use std::io::BufRead;
 use std::path::{Path, PathBuf};
 use std::sync::RwLock;
-use std::{fs, process};
+use std::{fs, io, process};
 use terminal_size::{terminal_size, Width};
 use yaml_rust::{Yaml, YamlLoader};
 
@@ -39,6 +41,37 @@ lazy_static! {
         .match_kind(MatchKind::LeftmostLongest)
         .build(["🛂r", "🛂n", "🛂t"])
         .unwrap();
+    pub static ref ONE_CONFIG_MAP: HashMap<String, String> =
+        read_one_config_file(Path::new("rules_config_files.txt")).unwrap_or_default();
+}
+
+fn read_one_config_file(file_path: &Path) -> io::Result<HashMap<String, String>> {
+    let file = File::open(file_path)?;
+    let reader = io::BufReader::new(file);
+
+    let mut sections = HashMap::new();
+    let mut current_path = String::new();
+    let mut current_content = String::new();
+    let mut in_content = false;
+
+    for line in reader.lines() {
+        let line = line?;
+        if line.starts_with("---FILE_START---") {
+            current_path.clear();
+            current_content.clear();
+            in_content = false;
+        } else if let Some(path) = line.strip_prefix("path: ") {
+            current_path = path.to_string();
+        } else if line.starts_with("---CONTENT---") {
+            in_content = true;
+        } else if line.starts_with("---FILE_END---") {
+            sections.insert(current_path.clone(), current_content.clone());
+        } else if in_content {
+            current_content.push_str(&line);
+            current_content.push('\n');
+        }
+    }
+    Ok(sections)
 }
 
 pub struct ConfigReader {
@@ -248,16 +281,24 @@ impl StoredStatic {
                         )
                         .unwrap()
                     });
-            if !geo_ip_file_path.exists() {
+            if !geo_ip_file_path.exists()
+                && !ONE_CONFIG_MAP.contains_key("geoip_field_mapping.yaml")
+            {
                 AlertMessage::alert(
                     "Could not find the geoip_field_mapping.yaml config file. Please run update-rules."
                 )
                 .ok();
                 process::exit(1);
             }
-            let geo_ip_mapping = if let Ok(loaded_yaml) =
-                YamlLoader::load_from_str(&fs::read_to_string(geo_ip_file_path).unwrap())
-            {
+            let contents = if ONE_CONFIG_MAP.contains_key("geoip_field_mapping.yaml") {
+                ONE_CONFIG_MAP
+                    .get("geoip_field_mapping.yaml")
+                    .unwrap()
+                    .as_str()
+            } else {
+                &fs::read_to_string(geo_ip_file_path).unwrap()
+            };
+            let geo_ip_mapping = if let Ok(loaded_yaml) = YamlLoader::load_from_str(contents) {
                 loaded_yaml
             } else {
                 AlertMessage::alert("Parse error in geoip_field_mapping.yaml.").ok();

src/detections/field_data_map.rs

@@ -1,3 +1,4 @@
+use crate::detections::configs::ONE_CONFIG_MAP;
 use crate::detections::field_data_map::FieldDataConverter::{HexToDecimal, ReplaceStr};
 use crate::detections::message::AlertMessage;
 use crate::detections::utils::get_serde_number_to_string;
@@ -165,6 +166,23 @@ fn load_yaml_files(dir_path: &Path) -> Result<Vec<Yaml>, String> {
 }
 
 pub fn create_field_data_map(dir_path: &Path) -> Option<FieldDataMap> {
+    let one_config_values: Vec<String> = ONE_CONFIG_MAP
+        .iter()
+        .filter(|(key, _)| key.contains(".yaml") && !key.contains("geoip_field_mapping.yaml"))
+        .map(|(_, value)| value.clone())
+        .collect();
+    if !one_config_values.is_empty() {
+        let yaml_contents: Vec<Yaml> = one_config_values
+            .iter()
+            .flat_map(|value| YamlLoader::load_from_str(value).unwrap_or_default())
+            .collect();
+        return Some(
+            yaml_contents
+                .into_iter()
+                .map(build_field_data_map)
+                .collect(),
+        );
+    }
     let yaml_data = load_yaml_files(dir_path);
     match yaml_data {
         Ok(y) => Some(y.into_iter().map(build_field_data_map).collect()),

src/detections/utils.rs

@@ -26,7 +26,7 @@ use termcolor::{BufferWriter, ColorSpec, WriteColor};
 use termcolor::{Color, ColorChoice};
 use tokio::runtime::{Builder, Runtime};
 
-use crate::detections::configs::CURRENT_EXE_PATH;
+use crate::detections::configs::{CURRENT_EXE_PATH, ONE_CONFIG_MAP};
 use crate::detections::field_data_map::{convert_field_data, FieldDataMap, FieldDataMapKey};
 use crate::detections::field_extract::extract_fields;
 use crate::options::htmlreport;
@@ -93,13 +93,24 @@ pub fn read_txt(filename: &str) -> Result<Nested<String>, String> {
     } else {
         filename.to_string()
     };
+    let re = Regex::new(r".*/").unwrap();
+    let one_config_path = &re.replace(filename, "").to_string();
+    if ONE_CONFIG_MAP.contains_key(one_config_path) {
+        return Ok(Nested::from_iter(
+            ONE_CONFIG_MAP
+                .get(one_config_path)
+                .unwrap()
+                .lines()
+                .map(|s| s.to_string()),
+        ));
+    }
     let f = File::open(filepath);
     if f.is_err() {
         let errmsg = format!("Cannot open file. [file:{filename}]");
-        return Result::Err(errmsg);
+        return Err(errmsg);
     }
     let reader = BufReader::new(f.unwrap());
-    Result::Ok(Nested::from_iter(
+    Ok(Nested::from_iter(
         reader.lines().map(|line| line.unwrap_or_default()),
     ))
 }
@@ -167,18 +178,24 @@ pub fn read_json_to_value(path: &str) -> Result<Box<dyn Iterator<Item = Value>>,
 }
 
 pub fn read_csv(filename: &str) -> Result<Nested<Vec<String>>, String> {
+    let re = Regex::new(r".*/").unwrap();
+    let one_config_path = &re.replace(filename, "").to_string();
+    if ONE_CONFIG_MAP.contains_key(one_config_path) {
+        let csv_res = parse_csv(ONE_CONFIG_MAP.get(one_config_path).unwrap());
+        return Ok(csv_res);
+    }
     let f = File::open(filename);
     if f.is_err() {
-        return Result::Err(format!("Cannot open file. [file:{filename}]"));
+        return Err(format!("Cannot open file. [file:{filename}]"));
     }
     let mut contents: String = String::new();
     let read_res = f.unwrap().read_to_string(&mut contents);
     if let Err(e) = read_res {
-        return Result::Err(e.to_string());
+        return Err(e.to_string());
     }
 
     let csv_res = parse_csv(&contents);
-    Result::Ok(csv_res)
+    Ok(csv_res)
 }
 
 pub fn parse_csv(file_contents: &str) -> Nested<Vec<String>> {
@@ -553,7 +570,10 @@ pub fn make_ascii_titlecase(s: &str) -> CompactString {
 
 /// base_path/path が存在するかを確認し、存在しなければカレントディレクトリを参照するpathを返す関数
 pub fn check_setting_path(base_path: &Path, path: &str, ignore_err: bool) -> Option<PathBuf> {
-    if base_path.join(path).exists() {
+    let re = Regex::new(r".*/").unwrap();
+    if ONE_CONFIG_MAP.contains_key(&re.replace(path, "").to_string()) {
+        Some(path.into())
+    } else if base_path.join(path).exists() {
         Some(base_path.join(path))
     } else if ignore_err {
         Some(Path::new(path).to_path_buf())
@@ -564,6 +584,20 @@ pub fn check_setting_path(base_path: &Path, path: &str, ignore_err: bool) -> Opt
 
 /// rule configのファイルの所在を確認する関数。
 pub fn check_rule_config(config_path: &PathBuf) -> Result<(), String> {
+    // 各種ファイルを確認する
+    let files = vec![
+        "channel_abbreviations.txt",
+        "target_event_IDs.txt",
+        "default_details.txt",
+        "level_tuning.txt",
+        "channel_eid_info.txt",
+        "eventkey_alias.txt",
+    ];
+    let all_keys_present = files.iter().all(|key| ONE_CONFIG_MAP.contains_key(*key));
+    if all_keys_present {
+        return Ok(());
+    }
+
     // rules/configのフォルダが存在するかを確認する
     let exist_rule_config_folder = if config_path == &CURRENT_EXE_PATH.to_path_buf() {
         check_setting_path(config_path, "rules/config", false).is_some()
@@ -574,15 +608,6 @@ pub fn check_rule_config(config_path: &PathBuf) -> Result<(), String> {
         return Err("The required rules and config files were not found. Please download them with the update-rules command.".to_string());
     }
 
-    // 各種ファイルを確認する
-    let files = vec![
-        "channel_abbreviations.txt",
-        "target_event_IDs.txt",
-        "default_details.txt",
-        "level_tuning.txt",
-        "channel_eid_info.txt",
-        "eventkey_alias.txt",
-    ];
     let mut not_exist_file = vec![];
     for file in &files {
         if check_setting_path(config_path, file, false).is_none() {

src/filter.rs

@@ -1,4 +1,4 @@
-use crate::detections::configs::{self, StoredStatic};
+use crate::detections::configs::{self, StoredStatic, ONE_CONFIG_MAP};
 use crate::detections::message::{AlertMessage, ERROR_LOG_STACK};
 use crate::detections::rule::RuleNode;
 use evtx::EvtxParser;
@@ -58,24 +58,34 @@ pub fn exclude_ids(stored_static: &StoredStatic) -> RuleExclude {
 
 impl RuleExclude {
     fn insert_ids(&mut self, filename: &str, stored_static: &StoredStatic) {
-        let f = File::open(filename);
-        if f.is_err() {
-            if stored_static.verbose_flag {
-                AlertMessage::warn(&format!("{filename} does not exist")).ok();
-            }
-            if !stored_static.quiet_errors_flag {
-                ERROR_LOG_STACK
-                    .lock()
-                    .unwrap()
-                    .push(format!("{filename} does not exist"));
+        let re = Regex::new(r".*/").unwrap();
+        let one_config_path = &re.replace(filename, "").to_string();
+        let lines: Vec<String> = if ONE_CONFIG_MAP.contains_key(one_config_path) {
+            ONE_CONFIG_MAP
+                .get(one_config_path)
+                .unwrap()
+                .split('\n')
+                .map(|s| s.to_string())
+                .collect()
+        } else {
+            let f = File::open(filename);
+            if f.is_err() {
+                if stored_static.verbose_flag {
+                    AlertMessage::warn(&format!("{filename} does not exist")).ok();
+                }
+                if !stored_static.quiet_errors_flag {
+                    ERROR_LOG_STACK
+                        .lock()
+                        .unwrap()
+                        .push(format!("{filename} does not exist"));
+                }
+                return;
             }
-            return;
-        }
-        let reader = BufReader::new(f.unwrap());
-        for v in reader.lines() {
-            let v = v.unwrap().split('#').collect::<Vec<&str>>()[0]
-                .trim()
-                .to_string();
+            let reader = BufReader::new(f.unwrap());
+            reader.lines().map_while(Result::ok).collect()
+        };
+        for v in lines {
+            let v = v.split('#').collect::<Vec<&str>>()[0].trim().to_string();
             if v.is_empty() || !configs::IDS_REGEX.is_match(&v) {
                 // 空行は無視する。IDの検証
                 continue;

src/main.rs

@@ -33,7 +33,7 @@ use hayabusa::afterfact::{self, AfterfactInfo, AfterfactWriter};
 use hayabusa::debug::checkpoint_process_timer::CHECKPOINT;
 use hayabusa::detections::configs::{
     load_pivot_keywords, Action, ConfigReader, EventKeyAliasConfig, StoredStatic, TargetEventTime,
-    TargetIds, CURRENT_EXE_PATH, STORED_EKEY_ALIAS, STORED_STATIC,
+    TargetIds, CURRENT_EXE_PATH, ONE_CONFIG_MAP, STORED_EKEY_ALIAS, STORED_STATIC,
 };
 use hayabusa::detections::detection::{self, EvtxRecordInfo};
 use hayabusa::detections::message::{AlertMessage, DetectInfo, ERROR_LOG_STACK};
@@ -172,6 +172,12 @@ impl App {
             return;
         }
 
+        if Path::new("encoded_rules.yml").exists() && Path::new("rules").exists() {
+            println!("You have the rules directory and encoded_rules.yml in your path. Please delete one of them.");
+            println!();
+            return;
+        }
+
         // カレントディレクトリ以外からの実行の際にrules-configオプションの指定がないとエラーが発生することを防ぐための処理
         if stored_static.config_path == Path::new("./rules/config") {
             stored_static.config_path =
@@ -443,6 +449,14 @@ impl App {
                     .to_str()
                     .unwrap(),
                 );
+                if Path::new("./encoded_rules.yml").exists() {
+                    stored_static.output_option.as_mut().unwrap().rules = check_setting_path(
+                        &CURRENT_EXE_PATH.to_path_buf(),
+                        "encoded_rules.yml",
+                        true,
+                    )
+                    .unwrap();
+                }
 
                 // pivot 機能でファイルを出力する際に同名ファイルが既に存在していた場合はエラー文を出して終了する。
                 let mut error_flag = false;
@@ -574,6 +588,27 @@ impl App {
                             AlertMessage::alert("Failed to update rules.").ok();
                         }
                     }
+
+                    if !ONE_CONFIG_MAP.is_empty() {
+                        let url = "https://raw.githubusercontent.com/Yamato-Security/hayabusa-encoded-rules/refs/heads/main/rules_config_files.txt";
+                        match get(url).call() {
+                            Ok(res) => {
+                                let mut dst =
+                                    File::create(Path::new("./rules_config_files.txt")).unwrap();
+                                copy(&mut res.into_reader(), &mut dst).unwrap();
+                                write_color_buffer(
+                                    &BufferWriter::stdout(ColorChoice::Always),
+                                    None,
+                                    "Config file rules_config_files.txt updated successfully.",
+                                    true,
+                                )
+                                .ok();
+                            }
+                            Err(_) => {
+                                AlertMessage::alert("Failed to update config file.").ok();
+                            }
+                        }
+                    }
                 } else {
                     match Update::update_rules(
                         update_target.unwrap().to_str().unwrap(),