Merge pull request #1527 from Yamato-Security/1434-support-expand

feat: add support `expand` modifier

.github/workflows/integration-test.yml

@@ -56,6 +56,15 @@ jobs:
       - name: eid-metrics(-o)
         run: cd main && cargo run --release -- eid-metrics -d ../hayabusa-sample-evtx -q -o out.csv -C
 
+      - name: expand-list
+        run: cd main && cargo run --release -- expand-list -q
+
+      - name: extract-base64
+        run: cd main && cargo run --release -- extract-base64 -d ../hayabusa-sample-evtx -q
+
+      - name: extract-base64(-o)
+        run: cd main && cargo run --release -- extract-base64 -d ../hayabusa-sample-evtx -q -o out.csv -C
+
       - name: json-timeline
         run: cd main && cargo run --release -- json-timeline -d ../hayabusa-sample-evtx -o out.json -q -w -D -n -u
 

CHANGELOG-Japanese.md

@@ -4,8 +4,9 @@
 
 **新機能:**
 
-Base64文字列を抽出して、デコードする`extract-base64`コマンドを追加した。(#1512) (@fukusuket)
-`expand`修飾子が入っているルールで使用されるプレースホルダー名を出力する`expand-list`コマンドを追加した。(#1513) (@fukuseket)
+- Base64文字列を抽出して、デコードする`extract-base64`コマンドを追加した。(#1512) (@fukusuket)
+- `expand`修飾子が入っているルールで使用されるプレースホルダー名を出力する`expand-list`コマンドを追加した。(#1513) (@fukuseket)
+- `expand`フィールド修飾子に対応した。 (#1434) (@fukusuket)
 
 **バグ修正:**
 

CHANGELOG.md

@@ -4,8 +4,9 @@
 
 **New Features:**
 
-New `extract-base64` command to extract and decode base64 strings from events. (#1512) (@fukusuket)
-New `expand-list` command to output placeholder names used for rules with the `expand` modifier. (#1513) (@fukuseket)
+- New `extract-base64` command to extract and decode base64 strings from events. (#1512) (@fukusuket)
+- New `expand-list` command to output placeholder names used for rules with the `expand` modifier. (#1513) (@fukuseket)
+- Support for `expand` field modifiers. (#1434) (@fukusuket)
 
 **Bug Fixes:**
 

src/detections/detection.rs

@@ -140,6 +140,7 @@ impl Detection {
         // parse rule files
         let mut ret = rulefile_loader
             .files
+            .clone()
             .into_iter()
             .map(|rule_file_tuple| rule::create_rule(rule_file_tuple.0, rule_file_tuple.1))
             .filter_map(return_if_success)
@@ -151,15 +152,7 @@ impl Detection {
             || stored_static.computer_metrics_flag
             || stored_static.log_metrics_flag)
         {
-            Detection::print_rule_load_info(
-                &rulefile_loader.rulecounter,
-                &rulefile_loader.rule_load_cnt,
-                &rulefile_loader.rule_status_cnt,
-                &rulefile_loader.rule_cor_cnt,
-                &rulefile_loader.rule_cor_ref_cnt,
-                &parseerror_count,
-                stored_static,
-            );
+            Detection::print_rule_load_info(&rulefile_loader, &parseerror_count, stored_static);
         }
         ret
     }
@@ -1119,14 +1112,16 @@ impl Detection {
     }
 
     pub fn print_rule_load_info(
-        rc: &HashMap<CompactString, u128>,
-        ld_rc: &HashMap<CompactString, u128>,
-        st_rc: &HashMap<CompactString, u128>,
-        cor_rc: &HashMap<CompactString, u128>,
-        cor_ref_rc: &HashMap<CompactString, u128>,
+        parse_yaml: &ParseYaml,
         err_rc: &u128,
         stored_static: &StoredStatic,
     ) {
+        let rc = &parse_yaml.rulecounter;
+        let ld_rc = &parse_yaml.rule_load_cnt;
+        let st_rc = &parse_yaml.rule_status_cnt;
+        let cor_rc = &parse_yaml.rule_cor_cnt;
+        let cor_ref_rc = &parse_yaml.rule_cor_ref_cnt;
+
         let mut sorted_ld_rc: Vec<(&CompactString, &u128)> = ld_rc.iter().collect();
         sorted_ld_rc.sort_by(|a, b| a.0.cmp(b.0));
         let mut html_report_stock = Nested::<String>::new();
@@ -1294,6 +1289,56 @@ impl Detection {
             println!();
         }
 
+        let expand_total = parse_yaml.rule_expand_cnt;
+        let expand_enabled_total = parse_yaml.rule_expand_enabled_cnt;
+        let key = "Expand rules: ";
+        let val = format!(
+            "{} ({:.2}%)",
+            expand_total.to_formatted_string(&Locale::en),
+            (expand_total as f64) / (total_loaded_rule_cnt as f64) * 100.0
+        );
+        write_color_buffer(
+            &BufferWriter::stdout(ColorChoice::Always),
+            get_writable_color(
+                Some(Color::Rgb(0, 255, 0)),
+                stored_static.common_options.no_color,
+            ),
+            key,
+            false,
+        )
+        .ok();
+        write_color_buffer(
+            &BufferWriter::stdout(ColorChoice::Always),
+            get_writable_color(None, stored_static.common_options.no_color),
+            val.as_str(),
+            true,
+        )
+        .ok();
+        let key = "Enabled expand rules: ";
+        let val = format!(
+            "{} ({:.2}%)",
+            expand_enabled_total.to_formatted_string(&Locale::en),
+            (expand_enabled_total as f64) / (total_loaded_rule_cnt as f64) * 100.0
+        );
+        write_color_buffer(
+            &BufferWriter::stdout(ColorChoice::Always),
+            get_writable_color(
+                Some(Color::Rgb(0, 255, 0)),
+                stored_static.common_options.no_color,
+            ),
+            key,
+            false,
+        )
+        .ok();
+        write_color_buffer(
+            &BufferWriter::stdout(ColorChoice::Always),
+            None,
+            val.as_str(),
+            true,
+        )
+        .ok();
+        println!();
+
         let mut sorted_rc: Vec<(&CompactString, &u128)> = rc.iter().collect();
         sorted_rc.sort_by(|a, b| a.0.cmp(b.0));
         sorted_rc.into_iter().for_each(|(key, value)| {

src/lib.rs

@@ -6,5 +6,7 @@ pub mod notify;
 pub mod options;
 pub mod timeline;
 pub mod yaml;
+pub mod yaml_expand;
+
 #[macro_use]
 extern crate horrorshow;

src/yaml.rs

@@ -1,11 +1,12 @@
 extern crate serde_derive;
 extern crate yaml_rust2;
 
-use crate::detections::configs::{self, StoredStatic};
+use crate::detections::configs::{self, StoredStatic, CURRENT_EXE_PATH};
 use crate::detections::message::AlertMessage;
 use crate::detections::message::ERROR_LOG_STACK;
 use crate::detections::utils;
 use crate::filter::RuleExclude;
+use crate::yaml_expand::{process_yaml, read_expand_files};
 use compact_str::CompactString;
 use hashbrown::{HashMap, HashSet};
 use itertools::Itertools;
@@ -16,12 +17,14 @@ use std::path::{Path, PathBuf};
 use yaml_rust2::{Yaml, YamlLoader};
 
 pub struct ParseYaml {
-    pub files: Vec<(String, yaml_rust2::Yaml)>,
+    pub files: Vec<(String, Yaml)>,
     pub rulecounter: HashMap<CompactString, u128>,
     pub rule_load_cnt: HashMap<CompactString, u128>,
     pub rule_status_cnt: HashMap<CompactString, u128>,
     pub rule_cor_cnt: HashMap<CompactString, u128>,
     pub rule_cor_ref_cnt: HashMap<CompactString, u128>,
+    pub rule_expand_cnt: u128,
+    pub rule_expand_enabled_cnt: u128,
     pub errorrule_count: u128,
     pub exclude_status: HashSet<String>,
     pub level_map: HashMap<String, u128>,
@@ -45,6 +48,8 @@ impl ParseYaml {
             ]),
             rule_cor_cnt: Default::default(),
             rule_cor_ref_cnt: Default::default(),
+            rule_expand_cnt: Default::default(),
+            rule_expand_enabled_cnt: Default::default(),
             errorrule_count: 0,
             exclude_status: configs::convert_option_vecs_to_hs(exclude_status_vec.as_ref()),
             level_map: HashMap::from([
@@ -143,6 +148,7 @@ impl ParseYaml {
             }
             return io::Result::Ok(String::default());
         }
+        let expand_map = read_expand_files(CURRENT_EXE_PATH.join("config/expand"));
         let mut yaml_docs = vec![];
         if metadata.unwrap().file_type().is_file() {
             // 拡張子がymlでないファイルは無視
@@ -318,6 +324,15 @@ impl ParseYaml {
         }
         let exist_output_opt = stored_static.output_option.is_some();
         let files = yaml_docs.into_iter().filter_map(|(filepath, yaml_doc)| {
+            let yaml_doc = match &expand_map {
+                Ok(map) => process_yaml(
+                    &yaml_doc,
+                    map,
+                    &mut self.rule_expand_cnt,
+                    &mut self.rule_expand_enabled_cnt,
+                ),
+                Err(_) => yaml_doc,
+            };
             //除外されたルールは無視する
             let rule_id = &yaml_doc["id"].as_str();
             if rule_id.is_some() {