diff --git a/src/afterfact.rs b/src/afterfact.rs index ae1250012..d810b53eb 100644 --- a/src/afterfact.rs +++ b/src/afterfact.rs @@ -328,6 +328,7 @@ fn emit_csv( // remove duplicate dataのための前レコード分の情報を保持する変数 let mut prev_message: HashMap = HashMap::new(); + let mut prev_details_convert_map: HashMap> = HashMap::new(); for (message_idx, time) in MESSAGEKEYS .lock() .unwrap() @@ -410,8 +411,10 @@ fn emit_csv( jsonl_output_flag, GEOIP_DB_PARSER.read().unwrap().is_some(), remove_duplicate_data_flag, + &[&detect_info.details_convert_map, &prev_details_convert_map], ); prev_message = result.1; + prev_details_convert_map = detect_info.details_convert_map.clone(); wtr.write_field(format!("{{ {} }}", &result.0))?; } else if json_output_flag { // JSON output @@ -422,8 +425,10 @@ fn emit_csv( jsonl_output_flag, GEOIP_DB_PARSER.read().unwrap().is_some(), remove_duplicate_data_flag, + &[&detect_info.details_convert_map, &prev_details_convert_map], ); prev_message = result.1; + prev_details_convert_map = detect_info.details_convert_map.clone(); wtr.write_field(&result.0)?; wtr.write_field("}")?; } else { @@ -1426,6 +1431,7 @@ pub fn output_json_str( jsonl_output_flag: bool, is_included_geo_ip: bool, remove_duplicate_flag: bool, + details_infos: &[&HashMap>], ) -> (String, HashMap) { let mut target: Vec = vec![]; let mut target_ext_field = Vec::new(); @@ -1435,15 +1441,32 @@ pub fn output_json_str( for (field_name, profile) in ext_field.iter() { match profile { Profile::Details(_) | Profile::AllFieldInfo(_) | Profile::ExtraFieldInfo(_) => { - if prev_message - .get(field_name) - .unwrap_or(&Profile::Literal("-".into())) - .to_value() - == profile.to_value() - { + let details_key = match profile { + Profile::Details(_) => "Details", + Profile::AllFieldInfo(_) => "AllFieldInfo", + Profile::ExtraFieldInfo(_) => "ExtraFieldInfo", + _ => "", + }; + + let empty = vec![]; + let now = details_infos[0] + .get(format!("#{details_key}").as_str()) + .unwrap_or(&empty); + let prev = details_infos[1] + .get(format!("#{details_key}").as_str()) + .unwrap_or(&empty); + let dup_flag = (!profile.to_value().is_empty() + && prev_message + .get(field_name) + .unwrap_or(&Profile::Literal("".into())) + .to_value() + == profile.to_value()) + || (!&now.is_empty() && !&prev.is_empty() && now == prev); + if dup_flag { // 合致する場合は前回レコード分のメッセージを更新する合致している場合は出力用のフィールドマップの内容を変更する。 // 合致しているので前回分のメッセージは更新しない - target_ext_field.push((field_name.clone(), profile.convert(&"DUP".into()))); + //DUPという通常の文字列を出すためにProfile::Literalを使用する + target_ext_field.push((field_name.clone(), Profile::Literal("DUP".into()))); } else { // 合致しない場合は前回レコード分のメッセージを更新する next_prev_message.insert(field_name.clone(), profile.clone()); @@ -1466,6 +1489,7 @@ pub fn output_json_str( "TgtCountry", "TgtCity", ]; + let valid_key_add_to_details: Vec<&str> = key_add_to_details .iter() .filter(|target_key| { @@ -1477,7 +1501,13 @@ pub fn output_json_str( for (key, profile) in target_ext_field.iter() { let val = profile.to_value(); let vec_data = _get_json_vec(profile, &val.to_string()); - if !key_add_to_details.contains(&key.as_str()) && vec_data.is_empty() { + if (!key_add_to_details.contains(&key.as_str()) + && !matches!( + profile, + Profile::AllFieldInfo(_) | Profile::ExtraFieldInfo(_) + )) + && vec_data.is_empty() + { let tmp_val: Vec<&str> = val.split(": ").collect(); let output_val = _convert_valid_json_str(&tmp_val, matches!(profile, Profile::AllFieldInfo(_))); @@ -1509,133 +1539,33 @@ pub fn output_json_str( Profile::Details(_) | Profile::AllFieldInfo(_) | Profile::ExtraFieldInfo(_) => { let mut output_stock: Vec = vec![]; output_stock.push(format!(" \"{key}\": {{")); - let mut stocked_value: Vec> = vec![]; - let mut key_index_stock = vec![]; - for detail_contents in vec_data.iter() { - // 分解してキーとなりえる箇所を抽出する - let mut tmp_stock = vec![]; - let mut space_split_contents = detail_contents.split(' '); - while let Some(sp) = space_split_contents.next() { - let first_character = - char::from_str(&sp.chars().next().unwrap_or('-').to_string()) - .unwrap_or_default(); - if !sp.contains(['\\', '"', '🛂']) - && first_character.is_uppercase() - && !sp.starts_with(['-', '/']) - && sp.ends_with(':') - && sp.len() > 2 - && sp != "Number:" - { - key_index_stock.push(sp.replace(':', "")); - if sp == "Payload:" { - stocked_value.push(vec![]); - stocked_value.push( - space_split_contents.map(|s| s.to_string()).collect(), - ); - break; - } else { - stocked_value.push(tmp_stock); - tmp_stock = vec![]; - } - } else if (first_character.is_lowercase() - || first_character.is_numeric()) - && sp.ends_with(';') - && sp.len() < 5 - && key_index_stock.len() > 1 - && key_index_stock.last().unwrap_or(&String::default()) != "Cmdline" - { - let last_key = key_index_stock.pop().unwrap_or_default(); - let mut last_stocked_value = - stocked_value.pop().unwrap_or_default(); - last_stocked_value.push(format!("{last_key}: {sp}")); - stocked_value.push(last_stocked_value); - } else { - tmp_stock.push(sp.to_owned()); - } - } - if !tmp_stock.is_empty() { - stocked_value.push(tmp_stock); - } - } - if stocked_value - .iter() - .counts_by(|x| x.len()) - .get(&0) - .unwrap_or(&0) - != &key_index_stock.len() - { - if let Some((target_idx, _)) = key_index_stock - .iter() - .enumerate() - .rfind(|(_, y)| "CmdLine" == *y) - { - let cmd_line_vec_idx_len = - stocked_value[2 * (target_idx + 1) - 1].len(); - stocked_value[2 * (target_idx + 1) - 1][cmd_line_vec_idx_len - 1] - .push_str(&format!(" {}:", key_index_stock[target_idx + 1])); - key_index_stock.remove(target_idx + 1); - } - } - let mut key_idx = 0; - let mut output_value_stock = String::default(); - for (value_idx, value) in stocked_value.iter().enumerate() { - if key_idx >= key_index_stock.len() { - break; - } - let mut tmp = if value_idx == 0 && !value.is_empty() { - key.as_str() - } else { - key_index_stock[key_idx].as_str() - }; - if !output_value_stock.is_empty() { - let separate_chr = - if key_index_stock[key_idx].starts_with("ScriptBlock") { - " | " - } else { - ": " - }; - output_value_stock.push_str(separate_chr); - } - output_value_stock.push_str(&value.join(" ")); - //1つまえのキーの段階で以降にvalueの配列で区切りとなる空の配列が存在しているかを確認する - let is_remain_split_stock = key_index_stock.len() > 1 - && key_idx == key_index_stock.len() - 2 - && value_idx < stocked_value.len() - 1 - && !output_value_stock.is_empty() - && !stocked_value[value_idx + 1..] - .iter() - .any(|remain_value| remain_value.is_empty()); - if (value_idx < stocked_value.len() - 1 - && stocked_value[value_idx + 1].is_empty() - && key_idx != key_index_stock.len() - 1) - || is_remain_split_stock - { - // 次の要素を確認して、存在しないもしくは、キーが入っているとなった場合現在ストックしている内容が出力していいことが確定するので出力処理を行う - let output_tmp = format!("{tmp}: {output_value_stock}"); - let output: Vec<&str> = output_tmp.split(": ").collect(); - let key = _convert_valid_json_str(&[output[0]], false); - let fmted_val = _convert_valid_json_str(&output, false); + let details_key = match profile { + Profile::Details(_) => "Details", + Profile::AllFieldInfo(_) => "AllFieldInfo", + Profile::ExtraFieldInfo(_) => "ExtraFieldInfo", + _ => "", + }; + // 個々の段階でDetails, AllFieldInfo, ExtraFieldInfoの要素はdetails_infosに格納されているのでunwrapする + let details_stocks = details_infos[0] + .get(&CompactString::from(format!("#{details_key}"))) + .unwrap(); + for (idx, contents) in details_stocks.iter().enumerate() { + let (key, value) = contents.split_once(": ").unwrap_or_default(); + let output_key = _convert_valid_json_str(&[key], false); + let fmted_val = _convert_valid_json_str(&[value], false); + + if idx != details_stocks.len() - 1 { output_stock.push(format!( "{},", _create_json_output_format( - &key, + &output_key, &fmted_val, key.starts_with('\"'), fmted_val.starts_with('\"'), 8 ) )); - output_value_stock.clear(); - tmp = ""; - key_idx += 1; - } - if value_idx == stocked_value.len() - 1 - && !(tmp.is_empty() && stocked_value.is_empty()) - { - let output_tmp = format!("{tmp}: {output_value_stock}"); - let output: Vec<&str> = output_tmp.split(": ").collect(); - let key = _convert_valid_json_str(&[output[0]], false); - let fmted_val = _convert_valid_json_str(&output, false); + } else { let last_contents_end = if is_included_geo_ip && !valid_key_add_to_details.is_empty() { "," @@ -1652,7 +1582,6 @@ pub fn output_json_str( 8, ) )); - key_idx += 1; } } if is_included_geo_ip { diff --git a/src/detections/detection.rs b/src/detections/detection.rs index 5e98aba93..8b576792a 100644 --- a/src/detections/detection.rs +++ b/src/detections/detection.rs @@ -655,6 +655,7 @@ impl Detection { _ => {} } } + //ルール側にdetailsの項目があればそれをそのまま出力し、そうでない場合はproviderとeventidの組で設定したdetailsの項目を出力する let details_fmt_str = match rule.yaml["details"].as_str() { Some(s) => s.to_string(), None => match stored_static @@ -662,10 +663,11 @@ impl Detection { .get(&CompactString::from(format!("{provider}_{eid}"))) { Some(str) => str.to_string(), - None => create_recordinfos(&record_info.record, &FieldDataMapKey::default(), &None), + None => create_recordinfos(&record_info.record, &FieldDataMapKey::default(), &None) + .join(" ¦ "), }, }; - let field_data_map_key = if stored_static.field_data_map.is_none() { + let field_data_map_key: FieldDataMapKey = if stored_static.field_data_map.is_none() { FieldDataMapKey::default() } else { FieldDataMapKey { @@ -693,13 +695,14 @@ impl Detection { detail: CompactString::default(), ext_field: stored_static.profiles.as_ref().unwrap().to_owned(), is_condition: false, + details_convert_map: HashMap::default(), }; message::insert( &record_info.record, CompactString::new(details_fmt_str), detect_info, time, - &mut profile_converter, + &profile_converter, (false, is_json_timeline, included_all_field_info_flag), ( eventkey_alias, @@ -911,6 +914,7 @@ impl Detection { detail: output, ext_field: stored_static.profiles.as_ref().unwrap().to_owned(), is_condition: true, + details_convert_map: HashMap::default(), }; let binding = STORED_EKEY_ALIAS.read().unwrap(); let eventkey_alias = binding.as_ref().unwrap(); @@ -921,7 +925,7 @@ impl Detection { CompactString::new(rule.yaml["details"].as_str().unwrap_or("-")), detect_info, agg_result.start_timedate, - &mut profile_converter, + &profile_converter, (true, is_json_timeline, false), (eventkey_alias, &field_data_map_key, &None), ) @@ -1144,7 +1148,7 @@ impl Detection { is_csv_output: bool, ) -> CompactString { for alias in target_alias { - let search_data = message::parse_message( + let (search_data, _) = message::parse_message( record, CompactString::from(alias), eventkey_alias, diff --git a/src/detections/field_data_map.rs b/src/detections/field_data_map.rs index 1b294b001..21c4c4fe0 100644 --- a/src/detections/field_data_map.rs +++ b/src/detections/field_data_map.rs @@ -330,7 +330,7 @@ mod tests { Ok(record) => { let ret = utils::create_recordinfos(&record, &key, &Some(map)); let expected = "ElevatedToken: NO ¦ ImpersonationLevel: A ¦ NewProcessId: 6528 ¦ ProcessId: 1100".to_string(); - assert_eq!(ret, expected); + assert_eq!(ret.join(" ¦ "), expected); } Err(_) => { panic!("Failed to parse json record."); diff --git a/src/detections/message.rs b/src/detections/message.rs index 0cb342ae9..3946b9c9c 100644 --- a/src/detections/message.rs +++ b/src/detections/message.rs @@ -16,7 +16,7 @@ use lazy_static::lazy_static; use nested::Nested; use regex::Regex; use serde_json::Value; -use std::borrow::Borrow; + use std::env; use std::fs::{create_dir, File}; use std::io::{self, BufWriter, Write}; @@ -26,7 +26,7 @@ use termcolor::{BufferWriter, ColorChoice}; use super::configs::EventKeyAliasConfig; -#[derive(Debug, Clone, PartialEq, Eq, Hash, Default)] +#[derive(Debug, Clone, PartialEq, Eq, Default)] pub struct DetectInfo { pub rulepath: CompactString, pub ruleid: CompactString, @@ -37,6 +37,7 @@ pub struct DetectInfo { pub detail: CompactString, pub ext_field: Vec<(CompactString, Profile)>, pub is_condition: bool, + pub details_convert_map: HashMap>, } pub struct AlertMessage {} @@ -115,7 +116,7 @@ pub fn insert( output: CompactString, mut detect_info: DetectInfo, time: DateTime, - profile_converter: &mut HashMap<&str, Profile>, + profile_converter: &HashMap<&str, Profile>, (is_agg, is_json_timeline, included_all_field_info): (bool, bool, bool), (eventkey_alias, field_data_map_key, field_data_map): ( &EventKeyAliasConfig, @@ -123,9 +124,11 @@ pub fn insert( &Option, ), ) { + let mut record_details_info_map = HashMap::new(); if !is_agg { let mut prev = 'a'; - let mut removed_sp_parsed_detail = parse_message( + //ここの段階でdetailsの内容でaliasを置き換えた内容と各種、key,valueの組み合わせのmapを取得する + let (mut removed_sp_parsed_detail, alias_hash_map) = parse_message( event_record, output, eventkey_alias, @@ -133,6 +136,8 @@ pub fn insert( field_data_map_key, field_data_map, ); + record_details_info_map.insert("#Details".into(), alias_hash_map); + // 特殊文字の除外のためのretain処理 removed_sp_parsed_detail.retain(|ch| { let retain_flag = prev == ' ' && ch == ' ' && ch.is_control(); if !retain_flag { @@ -140,6 +145,8 @@ pub fn insert( } !retain_flag }); + + // Details内にある改行文字は除外しないために絵文字を含めた特殊な文字に変換することで対応する let parsed_detail = removed_sp_parsed_detail .replace('\n', "🛂n") .replace('\r', "🛂r") @@ -154,79 +161,106 @@ pub fn insert( for (key, profile) in detect_info.ext_field.iter() { match profile { Details(_) => { + // Detailsの要素がすでにreplaced_profilesに存在する場合は次の処理に進み let existed_flag = replaced_profiles .iter() .any(|(_, y)| matches!(y, Details(_))); if existed_flag { continue; } - if detect_info.borrow().detail.is_empty() { + if detect_info.detail.is_empty() { + //Detailsの中身が何も入っていない場合はそのままの値を入れる replaced_profiles.push((key.to_owned(), profile.to_owned())); } else { replaced_profiles .push((key.to_owned(), Details(detect_info.detail.clone().into()))); + // メモリの節約のためにDetailsの中身を空にする detect_info.detail = CompactString::default(); } } AllFieldInfo(_) => { - let existed_flag = replaced_profiles - .iter() - .any(|(_, y)| matches!(y, AllFieldInfo(_))); - if existed_flag { - continue; - } if is_agg { replaced_profiles.push((key.to_owned(), AllFieldInfo("-".into()))); } else { - let rec = + let recinfos = utils::create_recordinfos(event_record, field_data_map_key, field_data_map); - let rec = if rec.is_empty() { "-".to_string() } else { rec }; - replaced_profiles.push((key.to_owned(), AllFieldInfo(rec.into()))); + let rec = if recinfos.is_empty() { + "-".to_string() + } else if !is_json_timeline { + recinfos.join(" ¦ ") + } else { + String::default() + }; + if rec.is_empty() { + record_details_info_map.insert("#AllFieldInfo".into(), recinfos); + replaced_profiles.push((key.to_owned(), AllFieldInfo("".into()))); + } else { + replaced_profiles.push((key.to_owned(), AllFieldInfo(rec.into()))); + } } } Literal(_) => replaced_profiles.push((key.to_owned(), profile.to_owned())), ExtraFieldInfo(_) => { - let mut profile_all_field_info_prof = None; - let mut profile_details_prof = None; - replaced_profiles.iter().for_each(|(_, y)| match y { - AllFieldInfo(_) => profile_all_field_info_prof = Some(y.to_value()), - Details(_) => profile_details_prof = Some(y.to_value()), - _ => {} - }); - let profile_details = - profile_details_prof.unwrap_or(detect_info.detail.clone().into()); + let empty = vec![]; + let record_details_info_ref = record_details_info_map.clone(); + let profile_all_field_info_prof = record_details_info_ref.get("#AllFieldInfo"); + let details_splits: HashSet<&str> = HashSet::from_iter( + record_details_info_ref + .get("#Details") + .unwrap_or(&empty) + .iter() + .map(|x| x.split_once(": ").unwrap_or_default().1), + ); let profile_all_field_info = if let Some(all_field_info_val) = profile_all_field_info_prof { - all_field_info_val + all_field_info_val.to_owned() } else if is_agg { if included_all_field_info { // AllFieldInfoがまだ読み込まれていない場合は、AllFieldInfoを追加する replaced_profiles.push((key.to_owned(), AllFieldInfo("-".into()))); } - "-".to_string() + vec![] } else { - let rec = + let recinfos = utils::create_recordinfos(event_record, field_data_map_key, field_data_map); - let rec = if rec.is_empty() { "-".to_string() } else { rec }; + let rec = if recinfos.is_empty() { + "-".to_string() + } else if !is_json_timeline { + recinfos.join(" ¦ ") + } else { + String::default() + }; + if included_all_field_info { - replaced_profiles.push((key.to_owned(), AllFieldInfo(rec.clone().into()))); + if rec.is_empty() { + record_details_info_map + .insert("#AllFieldInfo".into(), recinfos.clone()); + replaced_profiles.push((key.to_owned(), AllFieldInfo("".into()))); + } else { + replaced_profiles + .push((key.to_owned(), AllFieldInfo(rec.clone().into()))); + } } - rec + recinfos }; - let details_splits: HashSet<&str> = HashSet::from_iter( - profile_details - .split(" ¦ ") - .map(|x| x.split_once(": ").unwrap_or_default().1), - ); - let extra_field_val = profile_all_field_info - .split(" ¦ ") + let mut extra_field_val = profile_all_field_info + .iter() .filter(|x| { let value = x.split_once(": ").unwrap_or_default().1; !details_splits.contains(value) }) - .join(" ¦ "); - replaced_profiles.push((key.to_owned(), ExtraFieldInfo(extra_field_val.into()))); + .map(|y| y.to_owned()); + if is_json_timeline { + record_details_info_map + .insert("#ExtraFieldInfo".into(), extra_field_val.collect()); + replaced_profiles.push((key.to_owned(), ExtraFieldInfo("".into()))); + } else { + replaced_profiles.push(( + key.to_owned(), + ExtraFieldInfo(extra_field_val.join(" ¦ ").into()), + )); + } } SrcASN(_) | SrcCountry(_) | SrcCity(_) | TgtASN(_) | TgtCountry(_) | TgtCity(_) => { replaced_profiles.push(( @@ -236,26 +270,25 @@ pub fn insert( } _ => { if let Some(p) = profile_converter.get(key.as_str()) { - replaced_profiles.push(( - key.to_owned(), - profile.convert(&parse_message( - event_record, - CompactString::new(p.to_value()), - eventkey_alias, - is_json_timeline, - field_data_map_key, - field_data_map, - )), - )) + let (parsed_message, _) = &parse_message( + event_record, + CompactString::new(p.to_value()), + eventkey_alias, + is_json_timeline, + field_data_map_key, + field_data_map, + ); + replaced_profiles.push((key.to_owned(), profile.convert(parsed_message))) } } } } detect_info.ext_field = replaced_profiles; + detect_info.details_convert_map = record_details_info_map; insert_message(detect_info, time) } -/// メッセージ内の%で囲まれた箇所をエイリアスとしてをレコード情報を参照して置き換える関数 +/// メッセージ内の%で囲まれた箇所をエイリアスとしてレコード情報を参照して置き換える関数 pub fn parse_message( event_record: &Value, output: CompactString, @@ -263,9 +296,13 @@ pub fn parse_message( json_timeline_flag: bool, field_data_map_key: &FieldDataMapKey, field_data_map: &Option, -) -> CompactString { - let mut return_message = output; - let mut hash_map: HashMap = HashMap::new(); +) -> (CompactString, Vec) { + let mut return_message = output.clone(); + let mut hash_map: HashMap> = HashMap::new(); + let detail_key: Vec<&str> = output + .split(" ¦ ") + .map(|x| x.split_once(": ").unwrap_or_default().0) + .collect(); for caps in ALIASREGEX.captures_iter(&return_message) { let full_target_str = &caps[0]; let target_str = full_target_str @@ -314,23 +351,31 @@ pub fn parse_message( converted_str.unwrap_or(hash_value) }; if json_timeline_flag { - hash_map.insert(CompactString::from(full_target_str), field_data); + hash_map.insert(CompactString::from(full_target_str), [field_data].to_vec()); } else { hash_map.insert( CompactString::from(full_target_str), - field_data.split_ascii_whitespace().join(" ").into(), + [field_data.split_ascii_whitespace().join(" ").into()].to_vec(), ); } } } else { - hash_map.insert(CompactString::from(full_target_str), "n/a".into()); + hash_map.insert( + CompactString::from(full_target_str), + ["n/a".into()].to_vec(), + ); } } - - for (k, v) in hash_map { - return_message = CompactString::new(return_message.replace(k.as_str(), v.as_str())); + let mut details_key_and_value: Vec = vec![]; + for (i, (k, v)) in hash_map.iter().enumerate() { + // JSON出力の場合は各種のaliasを置き換える処理はafterfactの出力用の関数で行うため、ここでは行わない + if !json_timeline_flag { + return_message = CompactString::new(return_message.replace(k.as_str(), v[0].as_str())); + } else { + details_key_and_value.push(format!("{}: {}", detail_key[i], v[0]).into()); + } } - return_message + (return_message, details_key_and_value) } /// メッセージを返す diff --git a/src/detections/utils.rs b/src/detections/utils.rs index 1e548bd66..664f224b4 100644 --- a/src/detections/utils.rs +++ b/src/detections/utils.rs @@ -375,7 +375,7 @@ pub fn create_recordinfos( record: &Value, field_data_map_key: &FieldDataMapKey, field_data_map: &Option, -) -> String { +) -> Vec { let mut output = HashSet::new(); _collect_recordinfo(&mut vec![], "", record, &mut output); @@ -398,13 +398,13 @@ pub fn create_recordinfos( convert_field_data(map, field_data_map_key, &key.to_lowercase(), value) { let val = converted_str.strip_suffix(',').unwrap_or(&converted_str); - return format!("{key}: {val}"); + return format!("{key}: {val}").into(); } } let val = value.strip_suffix(',').unwrap_or(value); - format!("{key}: {val}") + format!("{key}: {val}").into() }) - .join(" ¦ ") + .collect() } /** diff --git a/src/timeline/search.rs b/src/timeline/search.rs index ecf664a6c..893293c41 100644 --- a/src/timeline/search.rs +++ b/src/timeline/search.rs @@ -310,7 +310,7 @@ fn extract_search_event_info( let datainfo = utils::create_recordinfos(&record.record, &FieldDataMapKey::default(), &None); let allfieldinfo = if !datainfo.is_empty() { - datainfo.into() + datainfo.join(" ¦ ").into() } else { CompactString::new("-") }; @@ -465,6 +465,7 @@ pub fn search_result_dsp_msg( jsonl_output, false, false, + &[&HashMap::default(), &HashMap::default()], ); file_wtr