Merge pull request #1190 from Yamato-Security/fix/#1189

fixed removed `AllFieldInfo` and `ExtraFieldInfo`  field newline and tab char

src/afterfact.rs

@@ -1581,9 +1581,9 @@ pub fn output_json_str(
                         output_stock.push(format!("    \"{key}\": {{"));
                     };
                     for (idx, contents) in details_target_stock.iter().enumerate() {
-                        let (key, value) = contents.split_once(": ").unwrap_or_default();
+                        let (key, value) = contents.split_once(':').unwrap_or_default();
                         let output_key = _convert_valid_json_str(&[key], false);
-                        let fmted_val = _convert_valid_json_str(&[value], false);
+                        let fmted_val = _convert_valid_json_str(&[value.trim_start()], false);
 
                         if idx != details_target_stock.len() - 1 {
                             output_stock.push(format!(

src/detections/message.rs

@@ -127,10 +127,9 @@ pub fn insert(
     ),
 ) {
     let mut record_details_info_map = HashMap::new();
-    let mut sp_removed_details_in_record_trim_newline = vec![];
     if !is_agg {
         //ここの段階でdetailsの内容でaliasを置き換えた内容と各種、key,valueの組み合わせのmapを取得する
-        let (removed_sp_parsed_detail, details_in_record) = parse_message(
+        let (removed_sp_parsed_detail, mut details_in_record) = parse_message(
             event_record,
             &output,
             eventkey_alias,
@@ -140,14 +139,13 @@ pub fn insert(
         );
 
         let mut sp_removed_details_in_record = vec![];
-        details_in_record.iter().for_each(|v| {
-            sp_removed_details_in_record.push(remove_sp_char(v.clone(), true));
-            sp_removed_details_in_record_trim_newline.push(remove_sp_char(v.clone(), false));
+        details_in_record.drain(..).for_each(|v| {
+            sp_removed_details_in_record.push(remove_sp_char(v.clone()));
         });
         record_details_info_map.insert("#Details".into(), sp_removed_details_in_record);
         // 特殊文字の除外のためのretain処理
         // Details内にある改行文字は除外しないために絵文字を含めた特殊な文字に変換することで対応する
-        let parsed_detail = remove_sp_char(removed_sp_parsed_detail, true);
+        let parsed_detail = remove_sp_char(removed_sp_parsed_detail);
         detect_info.detail = if parsed_detail.is_empty() {
             CompactString::from("-")
         } else {
@@ -227,12 +225,18 @@ pub fn insert(
                 }
                 let record_details_info_ref = record_details_info_map.clone();
                 let profile_all_field_info_prof = record_details_info_ref.get("#AllFieldInfo");
-                let details_splits: HashSet<&str> =
-                    HashSet::from_iter(sp_removed_details_in_record_trim_newline.iter().map(|x| {
-                        let v = x.split_once(": ").unwrap_or_default().1;
-                        // 末尾のカンマが含まれている場合と含まれていない場合でExtraFieldInfoでの一致判定が変わってしまうため判定用のハッシュセットの末尾のカンマを削除する
-                        v.strip_suffix(',').unwrap_or(v)
-                    }));
+                let empty = vec![];
+                let details_splits: HashSet<&str> = HashSet::from_iter(
+                    record_details_info_ref
+                        .get("#Details")
+                        .unwrap_or(&empty)
+                        .iter()
+                        .map(|x| {
+                            let v = x.split_once(": ").unwrap_or_default().1;
+                            // 末尾のカンマが含まれている場合と含まれていない場合でExtraFieldInfoでの一致判定が変わってしまうため判定用のハッシュセットの末尾のカンマを削除する
+                            v.strip_suffix(',').unwrap_or(v)
+                        }),
+                );
                 let profile_all_field_info = if let Some(all_field_info_val) =
                     profile_all_field_info_prof
                 {

src/detections/utils.rs

@@ -73,7 +73,7 @@ pub fn value_to_string(value: &Value) -> Option<String> {
         Value::Null => Option::None,
         Value::Bool(b) => Option::Some(b.to_string()),
         Value::Number(n) => Option::Some(n.to_string()),
-        Value::String(s) => Option::Some(s.trim().to_string()),
+        Value::String(s) => Option::Some(s.to_string()),
         Value::Array(_) => Option::None,
         Value::Object(_) => Option::None,
     }
@@ -218,7 +218,7 @@ pub fn get_serde_number_to_string(
     if value.is_string() {
         let val_str = value.as_str().unwrap_or("");
         if val_str.ends_with(',') {
-            Some(CompactString::from(val_str.strip_suffix(',').unwrap()))
+            Some(CompactString::from(val_str))
         } else {
             Option::Some(CompactString::from(val_str))
         }
@@ -398,12 +398,12 @@ pub fn create_recordinfos(
                 if let Some(converted_str) =
                     convert_field_data(map, field_data_map_key, &key.to_lowercase(), value)
                 {
-                    let val = remove_sp_char(converted_str, true);
-                    return format!("{key}: {}", val.strip_suffix(',').unwrap_or(&val)).into();
+                    let val = remove_sp_char(converted_str);
+                    return format!("{key}: {val}",).into();
                 }
             }
-            let val = remove_sp_char(value.into(), true);
-            format!("{key}: {}", val.strip_suffix(',').unwrap_or(&val)).into()
+            let val = remove_sp_char(value.into());
+            format!("{key}: {val}").into()
         })
         .collect()
 }
@@ -448,8 +448,10 @@ fn _collect_recordinfo<'a>(
             // 一番子の要素の値しか収集しない
             let strval = value_to_string(value);
             if let Some(strval) = strval {
-                let strval = strval.trim().chars().fold(String::default(), |mut acc, c| {
-                    if c.is_control() || c.is_ascii_whitespace() {
+                let strval = strval.chars().fold(String::default(), |mut acc, c| {
+                    if (c.is_control() || c.is_ascii_whitespace())
+                        && !['\r', '\n', '\t'].contains(&c)
+                    {
                         acc.push(' ');
                     } else {
                         acc.push(c);
@@ -692,22 +694,11 @@ pub fn output_duration(d: Duration) -> String {
     format!("{h:02}:{m:02}:{s:02}.{ms:03}")
 }
 
-pub fn remove_sp_char(record_value: CompactString, remain_newline: bool) -> CompactString {
-    let mut newline_replaced_cs: String = if remain_newline {
-        record_value
-            .replace('\n', "🛂n")
-            .replace('\r', "🛂r")
-            .replace('\t', "🛂t")
-    } else {
-        record_value.chars().fold(String::default(), |mut acc, c| {
-            if c.is_control() || c.is_ascii_whitespace() {
-                acc.push(' ');
-            } else {
-                acc.push(c);
-            };
-            acc
-        })
-    };
+pub fn remove_sp_char(record_value: CompactString) -> CompactString {
+    let mut newline_replaced_cs: String = record_value
+        .replace('\n', "🛂n")
+        .replace('\r', "🛂r")
+        .replace('\t', "🛂t");
     let mut prev = 'a';
     newline_replaced_cs.retain(|ch| {
         let retain_flag = (prev == ' ' && ch == ' ') || ch.is_control();