Merge pull request #1377 from Yamato-Security/1371-display-data-field…

…-index-string

chg: display `Data` field index string

CHANGELOG-Japanese.md

@@ -9,6 +9,9 @@
 - デフォルトで低メモリモードを有効にした。`-s, --low-memory-mode`は、`-s, --sort-events` - 出力/保存する前に結果をソートする。(注意: より多くのメモリを消費する。）(#1361) (@hitenkoku)
   - 注意: `-R, --remove-duplicate-data`または`-X, --remove-duplicate-detections`を使用するには、ソートを有効にする必要がある。
 - Sigma相関ルールが参照しているルールは、デフォルトで結果を出力しないようにした。`generate: true`を指定すると、出力される。 (#1367) (@fukusuket)
+- `Data`フィールドは、すべて`Data`フィールドとして、またはJSONの配列としてではなく、インデックス化された文字列として表示されるようになった。(#1371) (@fukusuket)
+  - 前: `"Data": ["17514", "Multiprocessor Free", "Service Pack 1"]`
+  - 後: `"Data[3]": "17514", "Data[4]": "Multiprocessor Free", "Data[5]": "Service Pack 1"`
 
 ## 2.16.0 [2024/06/11]
 

CHANGELOG.md

@@ -9,6 +9,9 @@
 - Enabled low memory mode by default. `-s, --low-memory-mode` is now `-s, --sort-events` - Sort events before outputting results. (warning: this uses much more memory!). (#1361) (@hitenkoku)
   - Note: you need to enable sorting in order to use  `-R, --remove-duplicate-data` and `-X, --remove-duplicate-detections`.
 - Sigma correlation reference rules now do not output alerts by default. You can enable them by adding `generate: true`. (#1367) (@fukusuket)
+- `Data` fields are now displayed as indexed strings instead of as all `Data` fields or in an array for JSON. (#1371) (@fukusuket)
+  - Before: `"Data": ["17514", "Multiprocessor Free", "Service Pack 1"]`
+  - After: `"Data[3]": "17514", "Data[4]": "Multiprocessor Free", "Data[5]": "Service Pack 1"`
 
 ## 2.16.0 [2024/06/11]
 

src/afterfact.rs

@@ -1,42 +1,40 @@
-use crate::detections::configs::{
-    Action, OutputOption, StoredStatic, CONTROL_CHAT_REPLACE_MAP, CURRENT_EXE_PATH, GEOIP_DB_PARSER,
-};
-use crate::detections::message::{AlertMessage, DetectInfo, COMPUTER_MITRE_ATTCK_MAP, LEVEL_FULL};
-use crate::detections::utils::{
-    self, format_time, get_writable_color, output_and_data_stack_for_html, write_color_buffer,
-};
-use crate::options::htmlreport;
-use crate::options::profile::Profile;
-use crate::yaml::ParseYaml;
+use std::cmp::{self, min, Ordering};
+use std::error::Error;
+use std::fs::File;
+use std::io::{self, BufWriter, Write};
+use std::path::Path;
+use std::process;
+use std::str::FromStr;
+
 use aho_corasick::{AhoCorasick, AhoCorasickBuilder, MatchKind};
 use chrono::{DateTime, Local, TimeZone, Utc};
 use comfy_table::modifiers::UTF8_ROUND_CORNERS;
 use comfy_table::presets::UTF8_FULL;
+use comfy_table::*;
 use compact_str::CompactString;
-use hashbrown::hash_map::RawEntryMut;
-use terminal_size::terminal_size;
-
 use csv::{QuoteStyle, Writer, WriterBuilder};
+use hashbrown::hash_map::RawEntryMut;
+use hashbrown::{HashMap, HashSet};
 use itertools::Itertools;
 use krapslog::{build_sparkline, build_time_markers};
+use lazy_static::lazy_static;
 use nested::Nested;
-use std::path::Path;
-use std::str::FromStr;
-use yaml_rust::YamlLoader;
-
-use comfy_table::*;
-use hashbrown::{HashMap, HashSet};
 use num_format::{Locale, ToFormattedString};
-use std::cmp::{self, min, Ordering};
-use std::error::Error;
-
-use std::io::{self, BufWriter, Write};
-
-use lazy_static::lazy_static;
-use std::fs::File;
-use std::process;
 use termcolor::{Buffer, BufferWriter, Color, ColorChoice, ColorSpec, WriteColor};
+use terminal_size::terminal_size;
 use terminal_size::Width;
+use yaml_rust::YamlLoader;
+
+use crate::detections::configs::{
+    Action, OutputOption, StoredStatic, CONTROL_CHAT_REPLACE_MAP, CURRENT_EXE_PATH, GEOIP_DB_PARSER,
+};
+use crate::detections::message::{AlertMessage, DetectInfo, COMPUTER_MITRE_ATTCK_MAP, LEVEL_FULL};
+use crate::detections::utils::{
+    self, format_time, get_writable_color, output_and_data_stack_for_html, write_color_buffer,
+};
+use crate::options::htmlreport;
+use crate::options::profile::Profile;
+use crate::yaml::ParseYaml;
 
 lazy_static! {
     // ここで字句解析するときに使う正規表現の一覧を定義する。
@@ -2204,7 +2202,15 @@ fn _output_html_computer_by_mitre_attck(html_output_stock: &mut Nested<String>)
 
 #[cfg(test)]
 mod tests {
-    use super::create_output_color_map;
+    use std::fs::{read_to_string, remove_file};
+    use std::path::Path;
+
+    use chrono::NaiveDateTime;
+    use chrono::{Local, TimeZone, Utc};
+    use compact_str::CompactString;
+    use hashbrown::HashMap;
+    use serde_json::Value;
+
     use crate::afterfact::format_time;
     use crate::afterfact::init_writer;
     use crate::afterfact::output_afterfact_inner;
@@ -2226,13 +2232,8 @@ mod tests {
     use crate::detections::message::DetectInfo;
     use crate::detections::utils;
     use crate::options::profile::{load_profile, Profile};
-    use chrono::NaiveDateTime;
-    use chrono::{Local, TimeZone, Utc};
-    use compact_str::CompactString;
-    use hashbrown::HashMap;
-    use serde_json::Value;
-    use std::fs::{read_to_string, remove_file};
-    use std::path::Path;
+
+    use super::create_output_color_map;
 
     #[test]
     fn test_emit_csv_output() {
@@ -3934,7 +3935,7 @@ mod tests {
                 ),
                 (
                     "RecordInformation",
-                    CompactString::from("{\n        \"CommandRLine\": \"hoge\",\n        \"Data\": [\"xxx\", \"yyy\"]\n    }"),
+                    CompactString::from("{\n        \"CommandRLine\": \"hoge\",\n        \"Data[1]\": \"xxx\",\n        \"Data[2]\": \"yyy\"\n    }"),
                 ),
                 (
                     "RuleFile",

src/detections/utils.rs

@@ -2,42 +2,39 @@ extern crate base64;
 extern crate csv;
 extern crate regex;
 
-use crate::detections::configs::CURRENT_EXE_PATH;
-use crate::options::htmlreport;
-
-use compact_str::{CompactString, ToCompactString};
-use hashbrown::{HashMap, HashSet};
-use itertools::Itertools;
-use nested::Nested;
-use std::path::{Path, PathBuf};
-use std::thread::available_parallelism;
-
-use chrono::Local;
-use termcolor::{Color, ColorChoice};
-
-use tokio::runtime::{Builder, Runtime};
-
-use chrono::{DateTime, TimeZone, Utc};
-use regex::Regex;
-use serde_json::{json, Error, Map, Value};
 use std::cmp::Ordering;
 use std::fs::{read_to_string, File};
 use std::io::prelude::*;
 use std::io::{BufRead, BufReader};
+use std::path::{Path, PathBuf};
 use std::str;
 use std::string::String;
+use std::thread::available_parallelism;
 use std::vec;
 use std::{fs, io};
+
+use chrono::Local;
+use chrono::{DateTime, TimeZone, Utc};
+use compact_str::{CompactString, ToCompactString};
+use hashbrown::{HashMap, HashSet};
+use itertools::Itertools;
+use memchr::memmem;
+use nested::Nested;
+use regex::Regex;
+use serde_json::{json, Error, Map, Value};
 use termcolor::{BufferWriter, ColorSpec, WriteColor};
+use termcolor::{Color, ColorChoice};
+use tokio::runtime::{Builder, Runtime};
+
+use crate::detections::configs::CURRENT_EXE_PATH;
+use crate::detections::field_data_map::{convert_field_data, FieldDataMap, FieldDataMapKey};
+use crate::detections::field_extract::extract_fields;
+use crate::options::htmlreport;
 
 use super::configs::{EventKeyAliasConfig, OutputOption, STORED_EKEY_ALIAS};
 use super::detection::EvtxRecordInfo;
 use super::message::AlertMessage;
 
-use crate::detections::field_data_map::{convert_field_data, FieldDataMap, FieldDataMapKey};
-use crate::detections::field_extract::extract_fields;
-use memchr::memmem;
-
 pub fn concat_selection_key(key_list: &Nested<String>) -> String {
     return key_list
         .iter()
@@ -396,7 +393,7 @@ pub fn create_recordinfos(
     _collect_recordinfo(
         &mut vec![],
         "",
-        0,
+        -1,
         record,
         record,
         &mut output,
@@ -437,7 +434,7 @@ pub fn create_recordinfos(
 fn _collect_recordinfo<'a>(
     keys: &mut Vec<&'a str>,
     parent_key: &'a str,
-    arr_index: usize,
+    arr_index: i8,
     org_value: &'a Value,
     cur_value: &'a Value,
     output: &mut HashSet<(String, String)>,
@@ -449,7 +446,7 @@ fn _collect_recordinfo<'a>(
                 _collect_recordinfo(
                     keys,
                     parent_key,
-                    i,
+                    i as i8,
                     org_value,
                     sub_value,
                     output,
@@ -471,7 +468,15 @@ fn _collect_recordinfo<'a>(
                     continue;
                 }
 
-                _collect_recordinfo(keys, key, 0, org_value, value, output, filed_data_converter);
+                _collect_recordinfo(
+                    keys,
+                    key,
+                    -1,
+                    org_value,
+                    value,
+                    output,
+                    filed_data_converter,
+                );
             }
             if !parent_key.is_empty() {
                 keys.pop();
@@ -492,10 +497,10 @@ fn _collect_recordinfo<'a>(
                     };
                     acc
                 });
-                if arr_index > 0 {
+                let key = if arr_index >= 0 {
                     let (field_data_map, field_data_map_key) = filed_data_converter;
                     let i = arr_index + 1;
-                    let field = format!("{parent_key}[{i}]",).to_lowercase();
+                    let field = format!("{parent_key}[{i}]").to_lowercase();
                     if let Some(map) = field_data_map {
                         let converted_str = convert_field_data(
                             map,
@@ -508,8 +513,11 @@ fn _collect_recordinfo<'a>(
                             strval = converted_str.to_string();
                         }
                     }
-                }
-                output.insert((parent_key.to_string(), strval));
+                    format!("{parent_key}[{i}]")
+                } else {
+                    parent_key.to_string()
+                };
+                output.insert((key, strval));
             }
         }
     }
@@ -763,6 +771,13 @@ pub fn remove_sp_char(record_value: CompactString) -> CompactString {
 mod tests {
     use std::path::Path;
 
+    use chrono::NaiveDate;
+    use compact_str::CompactString;
+    use hashbrown::{HashMap, HashSet};
+    use nested::Nested;
+    use regex::Regex;
+    use serde_json::Value;
+
     use crate::detections::field_data_map::FieldDataMapKey;
     use crate::{
         detections::{
@@ -774,12 +789,6 @@ mod tests {
         },
         options::htmlreport::HTML_REPORTER,
     };
-    use chrono::NaiveDate;
-    use compact_str::CompactString;
-    use hashbrown::{HashMap, HashSet};
-    use nested::Nested;
-    use regex::Regex;
-    use serde_json::Value;
 
     use super::{output_duration, output_profile_name};
 
@@ -833,7 +842,7 @@ mod tests {
             Ok(record) => {
                 let ret = utils::create_recordinfos(&record, &FieldDataMapKey::default(), &None);
                 // Systemは除外される/属性(_attributesも除外される)/key順に並ぶ
-                let expected = "Binary: hogehoge ¦ Data:  ¦ Data: Data1 ¦ Data: DataData2 ¦ Data: DataDataData3"
+                let expected = "Binary: hogehoge ¦ Data[1]: Data1 ¦ Data[2]: DataData2 ¦ Data[3]:  ¦ Data[4]: DataDataData3"
                     .to_string();
                 assert_eq!(ret.join(" ¦ "), expected);
             }