From 3482e10e3b8474b6634fc7159f8302960931f584 Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Tue, 5 Sep 2023 13:59:20 +0900 Subject: [PATCH 01/24] style(message): fixed clippy warning --- src/detections/message.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/detections/message.rs b/src/detections/message.rs index 0cb342ae9..c7b0a53e3 100644 --- a/src/detections/message.rs +++ b/src/detections/message.rs @@ -16,7 +16,6 @@ use lazy_static::lazy_static; use nested::Nested; use regex::Regex; use serde_json::Value; -use std::borrow::Borrow; use std::env; use std::fs::{create_dir, File}; use std::io::{self, BufWriter, Write}; @@ -115,7 +114,7 @@ pub fn insert( output: CompactString, mut detect_info: DetectInfo, time: DateTime, - profile_converter: &mut HashMap<&str, Profile>, + profile_converter: &HashMap<&str, Profile>, (is_agg, is_json_timeline, included_all_field_info): (bool, bool, bool), (eventkey_alias, field_data_map_key, field_data_map): ( &EventKeyAliasConfig, @@ -160,7 +159,7 @@ pub fn insert( if existed_flag { continue; } - if detect_info.borrow().detail.is_empty() { + if detect_info.detail.is_empty() { replaced_profiles.push((key.to_owned(), profile.to_owned())); } else { replaced_profiles From 33a538c7d6cf54e451f2263ea79d848d43976b70 Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Tue, 5 Sep 2023 14:00:20 +0900 Subject: [PATCH 02/24] feat(message): added message when os error 123 is happend #1166 --- src/detections/message.rs | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/src/detections/message.rs b/src/detections/message.rs index c7b0a53e3..12d98f6a5 100644 --- a/src/detections/message.rs +++ b/src/detections/message.rs @@ -364,17 +364,13 @@ impl AlertMessage { create_dir(path.parent().unwrap()).ok(); } let mut error_log_writer = BufWriter::new(File::create(path).unwrap()); + let mut error_contents = env::args().collect::>().iter().join(" "); + // 複数言語対応のためエラーコード123の数値部分を参照する + if error_contents.ends_with("123)") { + error_contents = format!("{error_contents}. When specifying a directory path in Windows, do not include a trailing slash at the end of the path."); + } error_log_writer - .write_all( - format!( - "user input: {:?}\n", - format_args!( - "{}", - env::args().collect::>().iter().join(" ") - ) - ) - .as_bytes(), - ) + .write_all(format!("user input: {error_contents:?}\n").as_bytes()) .ok(); let error_logs = ERROR_LOG_STACK.lock().unwrap(); error_logs.iter().for_each(|error_log| { From 9052619260db5591464782ec7ec8a8da6893d2eb Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Tue, 5 Sep 2023 14:06:06 +0900 Subject: [PATCH 03/24] docs(CHANGELOG): added next version CHANGELOG format --- CHANGELOG-Japanese.md | 18 ++++++++++++++++++ CHANGELOG.md | 22 ++++++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/CHANGELOG-Japanese.md b/CHANGELOG-Japanese.md index e2f7a15ba..6e6ea61a4 100644 --- a/CHANGELOG-Japanese.md +++ b/CHANGELOG-Japanese.md @@ -1,5 +1,23 @@ # 変更点 +## 2.9.0 [2023/XX/XX] "xxx Release" + +**新機能:** + +- XXX + +**改善:** + +- XXX + +**バグ修正:** + +- XXX + +**その他:** + +- XXX + ## 2.8.0 [2023/09/01] "Double X Release" **新機能:** diff --git a/CHANGELOG.md b/CHANGELOG.md index d8916cac3..5b7c5d96b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,27 @@ # Changes +## 2.9.0 [2023/XX/XX] "xxx Release" + +**New Features:** + +- XXX + +**New Features:** + +- XXX + +**Enhancements:** + +- XXX + +**Bug Fixes:** + +- XXX + +**Other:** + +- XXX + ## 2.8.0 [2023/09/01] "Double X Release" **New Features:** From 55ad767e50ec784abcc75c2775835a0db9a2728e Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Tue, 5 Sep 2023 14:14:33 +0900 Subject: [PATCH 04/24] docs(CHANGELOG): added #1166 --- CHANGELOG-Japanese.md | 2 +- CHANGELOG.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG-Japanese.md b/CHANGELOG-Japanese.md index 6e6ea61a4..d41dab3a2 100644 --- a/CHANGELOG-Japanese.md +++ b/CHANGELOG-Japanese.md @@ -8,7 +8,7 @@ **改善:** -- XXX +- ディレクトリパスの指定にバックスラッシュを使用すべきではないことを示すエラーメッセージを追加した。 (#1166) (@hitenkoku, 提案者: @joswr1ght) **バグ修正:** diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b7c5d96b..ef01a956e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ **Enhancements:** -- XXX +- Added error message to indicate that trailing backslashes should not be used in a directory path specification. (#1166) (@hitenkoku, thanks for the suggestion from @joswr1ght) **Bug Fixes:** From 50ef449b78187e5ee8fdbcb39aadf09f784a2400 Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Tue, 5 Sep 2023 14:32:22 +0900 Subject: [PATCH 05/24] style: fixed cargo clippy error --- src/detections/configs.rs | 4 ++-- src/detections/detection.rs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/detections/configs.rs b/src/detections/configs.rs index f9595f2ec..60cc6c033 100644 --- a/src/detections/configs.rs +++ b/src/detections/configs.rs @@ -3,7 +3,7 @@ use crate::detections::message::AlertMessage; use crate::detections::utils; use crate::options::geoip_search::GeoIPSearch; use crate::options::htmlreport; -use crate::options::pivot::{PivotKeyword, PIVOT_KEYWORD}; +use crate::options::pivot::PIVOT_KEYWORD; use crate::options::profile::{load_profile, Profile}; use aho_corasick::{AhoCorasick, AhoCorasickBuilder, MatchKind}; use chrono::{DateTime, Days, Duration, Local, Months, Utc}; @@ -2033,7 +2033,7 @@ pub fn load_pivot_keywords(path: &str) { .write() .unwrap() .entry(key.to_string()) - .or_insert_with(PivotKeyword::new); + .or_default(); PIVOT_KEYWORD .write() diff --git a/src/detections/detection.rs b/src/detections/detection.rs index 5e98aba93..8d8e962ec 100644 --- a/src/detections/detection.rs +++ b/src/detections/detection.rs @@ -699,7 +699,7 @@ impl Detection { CompactString::new(details_fmt_str), detect_info, time, - &mut profile_converter, + &profile_converter, (false, is_json_timeline, included_all_field_info_flag), ( eventkey_alias, @@ -921,7 +921,7 @@ impl Detection { CompactString::new(rule.yaml["details"].as_str().unwrap_or("-")), detect_info, agg_result.start_timedate, - &mut profile_converter, + &profile_converter, (true, is_json_timeline, false), (eventkey_alias, &field_data_map_key, &None), ) From 89ce2c472218c1b39a81b1a150edb2f389099998 Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Wed, 6 Sep 2023 10:41:00 +0900 Subject: [PATCH 06/24] style(afterfact): fixed clippy error --- src/afterfact.rs | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/afterfact.rs b/src/afterfact.rs index 6c392a4c5..6eb714ffa 100644 --- a/src/afterfact.rs +++ b/src/afterfact.rs @@ -2055,7 +2055,7 @@ mod tests { is_condition: false, }, expect_time, - &mut profile_converter, + &profile_converter, (false, false, false), (&eventkey_alias, &FieldDataMapKey::default(), &None), ); @@ -2077,7 +2077,7 @@ mod tests { is_condition: false, }, expect_time, - &mut profile_converter, + &profile_converter, (false, false, false), (&eventkey_alias, &FieldDataMapKey::default(), &None), ); @@ -2378,7 +2378,7 @@ mod tests { is_condition: false, }, expect_time, - &mut profile_converter, + &profile_converter, (false, false, true), (&eventkey_alias, &FieldDataMapKey::default(), &None), ); @@ -2400,7 +2400,7 @@ mod tests { is_condition: false, }, expect_time, - &mut profile_converter, + &profile_converter, (false, false, true), (&eventkey_alias, &FieldDataMapKey::default(), &None), ); @@ -2691,7 +2691,7 @@ mod tests { is_condition: false, }, expect_time, - &mut profile_converter, + &profile_converter, (false, false, false), (&eventkey_alias, &FieldDataMapKey::default(), &None), ); @@ -2713,7 +2713,7 @@ mod tests { is_condition: false, }, expect_time, - &mut profile_converter, + &profile_converter, (false, false, false), (&eventkey_alias, &FieldDataMapKey::default(), &None), ); @@ -3012,7 +3012,7 @@ mod tests { is_condition: false, }, expect_time, - &mut profile_converter, + &profile_converter, (false, false, false), (&eventkey_alias, &FieldDataMapKey::default(), &None), ); @@ -3034,7 +3034,7 @@ mod tests { is_condition: false, }, expect_time, - &mut profile_converter, + &profile_converter, (false, false, false), (&eventkey_alias, &FieldDataMapKey::default(), &None), ); @@ -3556,7 +3556,7 @@ mod tests { is_condition: false, }, expect_time, - &mut profile_converter, + &profile_converter, (false, false, false), (&eventkey_alias, &FieldDataMapKey::default(), &None), ); @@ -3816,7 +3816,7 @@ mod tests { is_condition: false, }, expect_time, - &mut profile_converter, + &profile_converter, (false, false, false), (&eventkey_alias, &FieldDataMapKey::default(), &None), ); From 43c8016a94fabe4cc44d5749e1b95450a284fce7 Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Tue, 5 Sep 2023 13:59:20 +0900 Subject: [PATCH 07/24] style(message): fixed clippy warning --- src/detections/message.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/detections/message.rs b/src/detections/message.rs index 0cb342ae9..c7b0a53e3 100644 --- a/src/detections/message.rs +++ b/src/detections/message.rs @@ -16,7 +16,6 @@ use lazy_static::lazy_static; use nested::Nested; use regex::Regex; use serde_json::Value; -use std::borrow::Borrow; use std::env; use std::fs::{create_dir, File}; use std::io::{self, BufWriter, Write}; @@ -115,7 +114,7 @@ pub fn insert( output: CompactString, mut detect_info: DetectInfo, time: DateTime, - profile_converter: &mut HashMap<&str, Profile>, + profile_converter: &HashMap<&str, Profile>, (is_agg, is_json_timeline, included_all_field_info): (bool, bool, bool), (eventkey_alias, field_data_map_key, field_data_map): ( &EventKeyAliasConfig, @@ -160,7 +159,7 @@ pub fn insert( if existed_flag { continue; } - if detect_info.borrow().detail.is_empty() { + if detect_info.detail.is_empty() { replaced_profiles.push((key.to_owned(), profile.to_owned())); } else { replaced_profiles From 1edc73db0050882793fa5bd5192709a3510c2a13 Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Tue, 5 Sep 2023 14:32:22 +0900 Subject: [PATCH 08/24] style: fixed cargo clippy error --- src/detections/configs.rs | 4 ++-- src/detections/detection.rs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/detections/configs.rs b/src/detections/configs.rs index f9595f2ec..60cc6c033 100644 --- a/src/detections/configs.rs +++ b/src/detections/configs.rs @@ -3,7 +3,7 @@ use crate::detections::message::AlertMessage; use crate::detections::utils; use crate::options::geoip_search::GeoIPSearch; use crate::options::htmlreport; -use crate::options::pivot::{PivotKeyword, PIVOT_KEYWORD}; +use crate::options::pivot::PIVOT_KEYWORD; use crate::options::profile::{load_profile, Profile}; use aho_corasick::{AhoCorasick, AhoCorasickBuilder, MatchKind}; use chrono::{DateTime, Days, Duration, Local, Months, Utc}; @@ -2033,7 +2033,7 @@ pub fn load_pivot_keywords(path: &str) { .write() .unwrap() .entry(key.to_string()) - .or_insert_with(PivotKeyword::new); + .or_default(); PIVOT_KEYWORD .write() diff --git a/src/detections/detection.rs b/src/detections/detection.rs index 5e98aba93..8d8e962ec 100644 --- a/src/detections/detection.rs +++ b/src/detections/detection.rs @@ -699,7 +699,7 @@ impl Detection { CompactString::new(details_fmt_str), detect_info, time, - &mut profile_converter, + &profile_converter, (false, is_json_timeline, included_all_field_info_flag), ( eventkey_alias, @@ -921,7 +921,7 @@ impl Detection { CompactString::new(rule.yaml["details"].as_str().unwrap_or("-")), detect_info, agg_result.start_timedate, - &mut profile_converter, + &profile_converter, (true, is_json_timeline, false), (eventkey_alias, &field_data_map_key, &None), ) From 0fab76875180803b1dbf3e0ea95d7f0af573cf97 Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Wed, 6 Sep 2023 10:41:00 +0900 Subject: [PATCH 09/24] style(afterfact): fixed clippy error --- src/afterfact.rs | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/afterfact.rs b/src/afterfact.rs index 6c392a4c5..6eb714ffa 100644 --- a/src/afterfact.rs +++ b/src/afterfact.rs @@ -2055,7 +2055,7 @@ mod tests { is_condition: false, }, expect_time, - &mut profile_converter, + &profile_converter, (false, false, false), (&eventkey_alias, &FieldDataMapKey::default(), &None), ); @@ -2077,7 +2077,7 @@ mod tests { is_condition: false, }, expect_time, - &mut profile_converter, + &profile_converter, (false, false, false), (&eventkey_alias, &FieldDataMapKey::default(), &None), ); @@ -2378,7 +2378,7 @@ mod tests { is_condition: false, }, expect_time, - &mut profile_converter, + &profile_converter, (false, false, true), (&eventkey_alias, &FieldDataMapKey::default(), &None), ); @@ -2400,7 +2400,7 @@ mod tests { is_condition: false, }, expect_time, - &mut profile_converter, + &profile_converter, (false, false, true), (&eventkey_alias, &FieldDataMapKey::default(), &None), ); @@ -2691,7 +2691,7 @@ mod tests { is_condition: false, }, expect_time, - &mut profile_converter, + &profile_converter, (false, false, false), (&eventkey_alias, &FieldDataMapKey::default(), &None), ); @@ -2713,7 +2713,7 @@ mod tests { is_condition: false, }, expect_time, - &mut profile_converter, + &profile_converter, (false, false, false), (&eventkey_alias, &FieldDataMapKey::default(), &None), ); @@ -3012,7 +3012,7 @@ mod tests { is_condition: false, }, expect_time, - &mut profile_converter, + &profile_converter, (false, false, false), (&eventkey_alias, &FieldDataMapKey::default(), &None), ); @@ -3034,7 +3034,7 @@ mod tests { is_condition: false, }, expect_time, - &mut profile_converter, + &profile_converter, (false, false, false), (&eventkey_alias, &FieldDataMapKey::default(), &None), ); @@ -3556,7 +3556,7 @@ mod tests { is_condition: false, }, expect_time, - &mut profile_converter, + &profile_converter, (false, false, false), (&eventkey_alias, &FieldDataMapKey::default(), &None), ); @@ -3816,7 +3816,7 @@ mod tests { is_condition: false, }, expect_time, - &mut profile_converter, + &profile_converter, (false, false, false), (&eventkey_alias, &FieldDataMapKey::default(), &None), ); From 42cb5460a1eca584ba2b17caa46fe2d81ef1f561 Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Thu, 7 Sep 2023 01:31:20 +0900 Subject: [PATCH 10/24] Revert "feat(message): added message when os error 123 is happend #1166" This reverts commit 33a538c7d6cf54e451f2263ea79d848d43976b70. --- src/detections/message.rs | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/detections/message.rs b/src/detections/message.rs index 12d98f6a5..c7b0a53e3 100644 --- a/src/detections/message.rs +++ b/src/detections/message.rs @@ -364,13 +364,17 @@ impl AlertMessage { create_dir(path.parent().unwrap()).ok(); } let mut error_log_writer = BufWriter::new(File::create(path).unwrap()); - let mut error_contents = env::args().collect::>().iter().join(" "); - // 複数言語対応のためエラーコード123の数値部分を参照する - if error_contents.ends_with("123)") { - error_contents = format!("{error_contents}. When specifying a directory path in Windows, do not include a trailing slash at the end of the path."); - } error_log_writer - .write_all(format!("user input: {error_contents:?}\n").as_bytes()) + .write_all( + format!( + "user input: {:?}\n", + format_args!( + "{}", + env::args().collect::>().iter().join(" ") + ) + ) + .as_bytes(), + ) .ok(); let error_logs = ERROR_LOG_STACK.lock().unwrap(); error_logs.iter().for_each(|error_log| { From 5108978c03cac81462f119d58a91977e82a7d80e Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Thu, 7 Sep 2023 01:32:28 +0900 Subject: [PATCH 11/24] feat(main): added message when os error is happened in directory option #1166 --- src/main.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main.rs b/src/main.rs index ab071c7a4..331c6024a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -880,7 +880,10 @@ impl App { } let entries = fs::read_dir(dirpath); if entries.is_err() { - let errmsg = format!("{}", entries.unwrap_err()); + let mut errmsg = format!("{}", entries.unwrap_err()); + if errmsg.ends_with("123)") { + errmsg = format!("{errmsg}. When specifying a directory path in Windows, do not include a trailing slash at the end of the path."); + } if stored_static.verbose_flag { AlertMessage::alert(&errmsg).ok(); } From a9d9a0939a97e1a354299e9956194e6f2db90852 Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Thu, 7 Sep 2023 01:34:48 +0900 Subject: [PATCH 12/24] feat(field_data_map): added message when os error is happend in config option #1166 --- src/detections/field_data_map.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/detections/field_data_map.rs b/src/detections/field_data_map.rs index 1b294b001..f09de36c2 100644 --- a/src/detections/field_data_map.rs +++ b/src/detections/field_data_map.rs @@ -133,7 +133,10 @@ fn load_yaml_files(dir_path: &Path) -> Result, String> { .flatten() .collect()), Err(e) => { - let msg = format!("Failed to open field mapping dir[{path}]."); + let mut msg = format!("Failed to open field mapping dir[{path}]. ",); + if e.to_string().ends_with("123)") { + msg = format!("{msg}. When specifying a directory path in Windows, do not include a trailing slash at the end of the path."); + } AlertMessage::warn(&msg).ok(); Err(e.to_string()) } From 24ff7395744669c47323befe84c709de54062fee Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Thu, 7 Sep 2023 01:35:51 +0900 Subject: [PATCH 13/24] feat(yaml): added message when os error is happened in rule option #1166 --- src/yaml.rs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/yaml.rs b/src/yaml.rs index 0fcb97e1e..873a2ee39 100644 --- a/src/yaml.rs +++ b/src/yaml.rs @@ -77,10 +77,19 @@ impl ParseYaml { ) -> io::Result { let metadata = fs::metadata(path.as_ref()); if metadata.is_err() { - let errmsg = format!( - "fail to read metadata of file: {}", + let err_contents = if let Err(e) = metadata { + e.to_string() + } else { + String::default() + }; + let mut errmsg = format!( + "fail to read metadata of file: {} {}", path.as_ref().to_path_buf().display(), + err_contents ); + if err_contents.ends_with("123)") { + errmsg = format!("{errmsg}. When specifying a directory path in Windows, do not include a trailing slash at the end of the path."); + } if stored_static.verbose_flag { AlertMessage::alert(&errmsg)?; } From 97e744e0aa8a231f42274a7d5b32a8262fea6da2 Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Thu, 7 Sep 2023 01:36:57 +0900 Subject: [PATCH 14/24] build(Cargo): updated next version --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b4f8ae18f..89dc271c9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -836,7 +836,7 @@ dependencies = [ [[package]] name = "hayabusa" -version = "2.8.0" +version = "2.9.0-dev" dependencies = [ "aho-corasick", "base64", diff --git a/Cargo.toml b/Cargo.toml index 3b889ce89..38296ef9b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "hayabusa" -version = "2.8.0" +version = "2.9.0-dev" repository = "https://github.com/Yamato-Security/hayabusa" authors = ["Yamato Security @SecurityYamato"] edition = "2021" From f49110366b228c15303a07cafa52138f3645af9c Mon Sep 17 00:00:00 2001 From: Yamato Security <71482215+YamatoSecurity@users.noreply.github.com> Date: Thu, 7 Sep 2023 16:49:54 +0900 Subject: [PATCH 15/24] optimize max detect records --- CHANGELOG-Japanese.md | 18 ++++++++++++++++++ CHANGELOG.md | 22 ++++++++++++++++++++++ src/main.rs | 5 +++-- 3 files changed, 43 insertions(+), 2 deletions(-) diff --git a/CHANGELOG-Japanese.md b/CHANGELOG-Japanese.md index e2f7a15ba..49f3de1bd 100644 --- a/CHANGELOG-Japanese.md +++ b/CHANGELOG-Japanese.md @@ -1,5 +1,23 @@ # 変更点 +## 2.9.0 [2023/XX/XX] "xxx Release" + +**新機能:** + +- XXX + +**改善:** + +- 一度に読み込むレコード数の最適化。(#1175) (@yamatosecurity) + +**バグ修正:** + +- XXX + +**その他:** + +- XXX + ## 2.8.0 [2023/09/01] "Double X Release" **新機能:** diff --git a/CHANGELOG.md b/CHANGELOG.md index d8916cac3..2a6ec3ae1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,27 @@ # Changes +## 2.9.0 [2023/XX/XX] "xxx Release" + +**New Features:** + +- XXX + +**New Features:** + +- XXX + +**Enhancements:** + +- Optimized the number of records to load at a time. (#1175) (@yamatosecurity) + +**Bug Fixes:** + +- XXX + +**Other:** + +- XXX + ## 2.8.0 [2023/09/01] "Double X Release" **New Features:** diff --git a/src/main.rs b/src/main.rs index ab071c7a4..2347cb5ee 100644 --- a/src/main.rs +++ b/src/main.rs @@ -64,8 +64,9 @@ use is_elevated::is_elevated; #[global_allocator] static GLOBAL: MiMalloc = MiMalloc; -// 一度にtimelineやdetectionを実行する行数 -const MAX_DETECT_RECORDS: usize = 5000; +// 一度に読み込んで、スキャンするレコード数 +// The number of records to load and scan at a time. 1000 gave the fastest results and lowest memory usage in test benchmarks. +const MAX_DETECT_RECORDS: usize = 1000; fn main() { let mut config_reader = ConfigReader::new(); From bb016032700ce993e55cb878c2f73e56f2932702 Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Sat, 9 Sep 2023 13:25:04 +0900 Subject: [PATCH 16/24] feat(main): replaced backslashes of paths in progress bar with slash #1172 --- src/main.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main.rs b/src/main.rs index ab071c7a4..f0e8b63fd 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1058,7 +1058,10 @@ impl App { *STORED_EKEY_ALIAS.write().unwrap() = Some(stored_static.eventkey_alias.clone()); *STORED_STATIC.write().unwrap() = Some(stored_static.clone()); for evtx_file in evtx_files { - let pb_msg = format!("{:?}", &evtx_file); + let pb_msg = format!( + "{:?}", + &evtx_file.to_str().unwrap_or_default().replace('\\', "/") + ); pb.set_message(pb_msg); let cnt_tmp: usize; From 221474e2c5b4f543f30f53f468cb39fec6552b46 Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Sat, 9 Sep 2023 13:28:47 +0900 Subject: [PATCH 17/24] docs(CHANGELOG): added #1172 --- CHANGELOG-Japanese.md | 1 + CHANGELOG.md | 1 + 2 files changed, 2 insertions(+) diff --git a/CHANGELOG-Japanese.md b/CHANGELOG-Japanese.md index 49f3de1bd..da6eb47be 100644 --- a/CHANGELOG-Japanese.md +++ b/CHANGELOG-Japanese.md @@ -9,6 +9,7 @@ **改善:** - 一度に読み込むレコード数の最適化。(#1175) (@yamatosecurity) +- プログレスバー内にあるバックスラッシュの表示をスラッシュに変更した。 (#1172) (@hitenkoku) **バグ修正:** diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a6ec3ae1..786906cfc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ **Enhancements:** - Optimized the number of records to load at a time. (#1175) (@yamatosecurity) +- Replaced backslashes in paths in the progress bar with slash. (#1172) (@hitenkoku) **Bug Fixes:** From 36b8667bb105e7edb2f10058b1986b27d16cc990 Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Sat, 9 Sep 2023 13:33:33 +0900 Subject: [PATCH 18/24] build(Cargo): cargo update --- Cargo.lock | 88 ++++++++++++++++++++++-------------------------------- 1 file changed, 35 insertions(+), 53 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b4f8ae18f..66b829644 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -194,9 +194,9 @@ checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" [[package]] name = "bytes" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" +checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" [[package]] name = "bytesize" @@ -262,25 +262,24 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.28" +version = "0.4.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95ed24df0632f708f5f6d8082675bef2596f7084dee3dd55f632290bf35bfe0f" +checksum = "defd4e7873dbddba6c7c91e199c7fcb946abc4a6a4ac3195400bcfb01b5de877" dependencies = [ "android-tzdata", "iana-time-zone", "js-sys", "num-traits", "serde", - "time 0.1.45", "wasm-bindgen", "windows-targets 0.48.5", ] [[package]] name = "cidr-utils" -version = "0.5.10" +version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdfa36f04861d39453affe1cf084ce2d6554021a84eb6f31ebdeafb6fb92a01c" +checksum = "2315f7119b7146d6a883de6acd63ddf96071b5f79d9d98d2adaa84d749f6abf1" dependencies = [ "debug-helper", "num-bigint", @@ -335,7 +334,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.29", + "syn 2.0.31", ] [[package]] @@ -788,7 +787,7 @@ checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" dependencies = [ "cfg-if", "libc", - "wasi 0.11.0+wasi-snapshot-preview1", + "wasi", ] [[package]] @@ -1232,9 +1231,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.6.2" +version = "2.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5486aed0026218e61b8a01d5fbd5a0a134649abb71a0e53b7bc088529dced86e" +checksum = "8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c" [[package]] name = "memmap2" @@ -1289,7 +1288,7 @@ checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" dependencies = [ "libc", "log", - "wasi 0.11.0+wasi-snapshot-preview1", + "wasi", "windows-sys 0.48.0", ] @@ -1456,9 +1455,9 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" [[package]] name = "object" -version = "0.32.0" +version = "0.32.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77ac5bbd07aea88c60a577a1ce218075ffd59208b2d7ca97adf9bfc5aeb21ebe" +checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0" dependencies = [ "memchr", ] @@ -1492,7 +1491,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.29", + "syn 2.0.31", ] [[package]] @@ -1503,18 +1502,18 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-src" -version = "111.27.0+1.1.1v" +version = "300.1.3+3.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06e8f197c82d7511c5b014030c9b1efeda40d7d5f99d23b4ceed3524a5e63f02" +checksum = "cd2c101a165fff9935e34def4669595ab1c7847943c42be86e21503e482be107" dependencies = [ "cc", ] [[package]] name = "openssl-sys" -version = "0.9.92" +version = "0.9.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db7e971c2c2bba161b2d2fdf37080177eff520b3bc044787c7f1f5f9e78d869b" +checksum = "db4d56a4c0478783083cfafcc42493dd4a981d41669da64b4572a2a089b51b1d" dependencies = [ "cc", "libc", @@ -1720,9 +1719,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.9.4" +version = "1.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12de2eff854e5fa4b1295edd650e227e9d8fb0c9e90b12e7f36d6a6811791a29" +checksum = "697061221ea1b4a94a624f67d0ae2bfe4e22b8a17b6a192afb11046542cc8c47" dependencies = [ "aho-corasick", "memchr", @@ -1732,9 +1731,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.3.7" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49530408a136e16e5b486e883fbb6ba058e8e4e8ae6621a77b048b314336e629" +checksum = "c2f401f4955220693b56f8ec66ee9c78abffd8d1c4f23dc41a23839eb88f0795" dependencies = [ "aho-corasick", "memchr", @@ -1934,7 +1933,7 @@ checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.29", + "syn 2.0.31", ] [[package]] @@ -2149,9 +2148,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.29" +version = "2.0.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c324c494eba9d92503e6f1ef2e6df781e78f6a7705a0202d9801b198807d518a" +checksum = "718fa2415bcb8d8bd775917a1bf12a7931b6dfa890753378538118181e0cb398" dependencies = [ "proc-macro2", "quote", @@ -2204,33 +2203,22 @@ checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" [[package]] name = "thiserror" -version = "1.0.47" +version = "1.0.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97a802ec30afc17eee47b2855fc72e0c4cd62be9b4efe6591edde0ec5bd68d8f" +checksum = "9d6d7a740b8a666a7e828dd00da9c0dc290dff53154ea77ac109281de90589b7" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.47" +version = "1.0.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bb623b56e39ab7dcd4b1b98bb6c8f8d907ed255b18de254088016b27a8ee19b" +checksum = "49922ecae66cc8a249b77e68d1d0623c1b2c514f0060c27cdc68bd62a1219d35" dependencies = [ "proc-macro2", "quote", - "syn 2.0.29", -] - -[[package]] -name = "time" -version = "0.1.45" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a" -dependencies = [ - "libc", - "wasi 0.10.0+wasi-snapshot-preview1", - "winapi", + "syn 2.0.31", ] [[package]] @@ -2343,7 +2331,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.29", + "syn 2.0.31", ] [[package]] @@ -2435,20 +2423,14 @@ checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] name = "walkdir" -version = "2.3.3" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36df944cda56c7d8d8b7496af378e6b16de9284591917d307c9b4d313c44e698" +checksum = "d71d857dc86794ca4c280d616f7da00d2dbfd8cd788846559a6813e6aa4b54ee" dependencies = [ "same-file", "winapi-util", ] -[[package]] -name = "wasi" -version = "0.10.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" - [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" @@ -2476,7 +2458,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.29", + "syn 2.0.31", "wasm-bindgen-shared", ] @@ -2498,7 +2480,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.29", + "syn 2.0.31", "wasm-bindgen-backend", "wasm-bindgen-shared", ] From e868dc678b05ecc2149429b5d480557c9f9f2018 Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Tue, 5 Sep 2023 13:59:20 +0900 Subject: [PATCH 19/24] style(message): fixed clippy warning --- src/detections/message.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/detections/message.rs b/src/detections/message.rs index 0cb342ae9..c7b0a53e3 100644 --- a/src/detections/message.rs +++ b/src/detections/message.rs @@ -16,7 +16,6 @@ use lazy_static::lazy_static; use nested::Nested; use regex::Regex; use serde_json::Value; -use std::borrow::Borrow; use std::env; use std::fs::{create_dir, File}; use std::io::{self, BufWriter, Write}; @@ -115,7 +114,7 @@ pub fn insert( output: CompactString, mut detect_info: DetectInfo, time: DateTime, - profile_converter: &mut HashMap<&str, Profile>, + profile_converter: &HashMap<&str, Profile>, (is_agg, is_json_timeline, included_all_field_info): (bool, bool, bool), (eventkey_alias, field_data_map_key, field_data_map): ( &EventKeyAliasConfig, @@ -160,7 +159,7 @@ pub fn insert( if existed_flag { continue; } - if detect_info.borrow().detail.is_empty() { + if detect_info.detail.is_empty() { replaced_profiles.push((key.to_owned(), profile.to_owned())); } else { replaced_profiles From 1b29f8044ac36d33359951cc7f1246eeeb239ffa Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Tue, 5 Sep 2023 14:32:22 +0900 Subject: [PATCH 20/24] style: fixed cargo clippy error --- src/detections/configs.rs | 4 ++-- src/detections/detection.rs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/detections/configs.rs b/src/detections/configs.rs index f9595f2ec..60cc6c033 100644 --- a/src/detections/configs.rs +++ b/src/detections/configs.rs @@ -3,7 +3,7 @@ use crate::detections::message::AlertMessage; use crate::detections::utils; use crate::options::geoip_search::GeoIPSearch; use crate::options::htmlreport; -use crate::options::pivot::{PivotKeyword, PIVOT_KEYWORD}; +use crate::options::pivot::PIVOT_KEYWORD; use crate::options::profile::{load_profile, Profile}; use aho_corasick::{AhoCorasick, AhoCorasickBuilder, MatchKind}; use chrono::{DateTime, Days, Duration, Local, Months, Utc}; @@ -2033,7 +2033,7 @@ pub fn load_pivot_keywords(path: &str) { .write() .unwrap() .entry(key.to_string()) - .or_insert_with(PivotKeyword::new); + .or_default(); PIVOT_KEYWORD .write() diff --git a/src/detections/detection.rs b/src/detections/detection.rs index 5e98aba93..8d8e962ec 100644 --- a/src/detections/detection.rs +++ b/src/detections/detection.rs @@ -699,7 +699,7 @@ impl Detection { CompactString::new(details_fmt_str), detect_info, time, - &mut profile_converter, + &profile_converter, (false, is_json_timeline, included_all_field_info_flag), ( eventkey_alias, @@ -921,7 +921,7 @@ impl Detection { CompactString::new(rule.yaml["details"].as_str().unwrap_or("-")), detect_info, agg_result.start_timedate, - &mut profile_converter, + &profile_converter, (true, is_json_timeline, false), (eventkey_alias, &field_data_map_key, &None), ) From b1595f222be9f49cb0f9185468cf64d90c9056c5 Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Wed, 6 Sep 2023 10:41:00 +0900 Subject: [PATCH 21/24] style(afterfact): fixed clippy error --- src/afterfact.rs | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/afterfact.rs b/src/afterfact.rs index 6c392a4c5..6eb714ffa 100644 --- a/src/afterfact.rs +++ b/src/afterfact.rs @@ -2055,7 +2055,7 @@ mod tests { is_condition: false, }, expect_time, - &mut profile_converter, + &profile_converter, (false, false, false), (&eventkey_alias, &FieldDataMapKey::default(), &None), ); @@ -2077,7 +2077,7 @@ mod tests { is_condition: false, }, expect_time, - &mut profile_converter, + &profile_converter, (false, false, false), (&eventkey_alias, &FieldDataMapKey::default(), &None), ); @@ -2378,7 +2378,7 @@ mod tests { is_condition: false, }, expect_time, - &mut profile_converter, + &profile_converter, (false, false, true), (&eventkey_alias, &FieldDataMapKey::default(), &None), ); @@ -2400,7 +2400,7 @@ mod tests { is_condition: false, }, expect_time, - &mut profile_converter, + &profile_converter, (false, false, true), (&eventkey_alias, &FieldDataMapKey::default(), &None), ); @@ -2691,7 +2691,7 @@ mod tests { is_condition: false, }, expect_time, - &mut profile_converter, + &profile_converter, (false, false, false), (&eventkey_alias, &FieldDataMapKey::default(), &None), ); @@ -2713,7 +2713,7 @@ mod tests { is_condition: false, }, expect_time, - &mut profile_converter, + &profile_converter, (false, false, false), (&eventkey_alias, &FieldDataMapKey::default(), &None), ); @@ -3012,7 +3012,7 @@ mod tests { is_condition: false, }, expect_time, - &mut profile_converter, + &profile_converter, (false, false, false), (&eventkey_alias, &FieldDataMapKey::default(), &None), ); @@ -3034,7 +3034,7 @@ mod tests { is_condition: false, }, expect_time, - &mut profile_converter, + &profile_converter, (false, false, false), (&eventkey_alias, &FieldDataMapKey::default(), &None), ); @@ -3556,7 +3556,7 @@ mod tests { is_condition: false, }, expect_time, - &mut profile_converter, + &profile_converter, (false, false, false), (&eventkey_alias, &FieldDataMapKey::default(), &None), ); @@ -3816,7 +3816,7 @@ mod tests { is_condition: false, }, expect_time, - &mut profile_converter, + &profile_converter, (false, false, false), (&eventkey_alias, &FieldDataMapKey::default(), &None), ); From b7fbefaceb3f295deb5c239d3a31baf8068fba27 Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Sat, 9 Sep 2023 14:20:34 +0900 Subject: [PATCH 22/24] refactor(detection/afterfact): fixed cargo clippy warn --- src/afterfact.rs | 44 ++++++++++++++++++------------------- src/detections/detection.rs | 25 +++++++++++---------- 2 files changed, 35 insertions(+), 34 deletions(-) diff --git a/src/afterfact.rs b/src/afterfact.rs index 6eb714ffa..7d08426a1 100644 --- a/src/afterfact.rs +++ b/src/afterfact.rs @@ -1828,6 +1828,7 @@ mod tests { use crate::detections::message::DetectInfo; use crate::detections::utils; use crate::options::profile::{load_profile, Profile}; + use chrono::NaiveDateTime; use chrono::{Local, TimeZone, Utc}; use compact_str::CompactString; use hashbrown::HashMap; @@ -1858,9 +1859,9 @@ mod tests { let test_attack = "execution/txxxx.yyy"; let test_recinfo = "CommandRLine: hoge"; let test_record_id = "11111"; - let expect_time = Utc - .datetime_from_str("1996-02-27T01:05:01Z", "%Y-%m-%dT%H:%M:%SZ") - .unwrap(); + let expect_naivetime = + NaiveDateTime::parse_from_str("1996-02-27T01:05:01Z", "%Y-%m-%dT%H:%M:%SZ").unwrap(); + let expect_time = Utc.from_local_datetime(&expect_naivetime).unwrap(); let expect_tz = expect_time.with_timezone(&Utc); let dummy_action = Action::CsvTimeline(CsvOutputOption { output_options: OutputOption { @@ -2182,9 +2183,9 @@ mod tests { let test_attack = "execution/txxxx.yyy"; let test_recinfo = "CommandRLine: hoge ¦ Test1: hogetest1 ¦ Test2: hogetest2"; let test_record_id = "11111"; - let expect_time = Utc - .datetime_from_str("1996-02-27T01:05:01Z", "%Y-%m-%dT%H:%M:%SZ") - .unwrap(); + let expect_naivetime = + NaiveDateTime::parse_from_str("1996-02-27T01:05:01Z", "%Y-%m-%dT%H:%M:%SZ").unwrap(); + let expect_time = Utc.from_local_datetime(&expect_naivetime).unwrap(); let expect_tz = expect_time.with_timezone(&Utc); let dummy_action = Action::CsvTimeline(CsvOutputOption { output_options: OutputOption { @@ -2494,9 +2495,9 @@ mod tests { let test_attack = "execution/txxxx.yyy"; let test_recinfo = "CommandRLine: hoge"; let test_record_id = "11111"; - let expect_time = Utc - .datetime_from_str("1996-02-27T01:05:01Z", "%Y-%m-%dT%H:%M:%SZ") - .unwrap(); + let expect_naivetime = + NaiveDateTime::parse_from_str("1996-02-27T01:05:01Z", "%Y-%m-%dT%H:%M:%SZ").unwrap(); + let expect_time = Utc.from_local_datetime(&expect_naivetime).unwrap(); let expect_tz = expect_time.with_timezone(&Utc); let dummy_action = Action::CsvTimeline(CsvOutputOption { output_options: OutputOption { @@ -2815,9 +2816,9 @@ mod tests { let test_attack = "execution/txxxx.yyy"; let test_recinfo = "CommandRLine: hoge"; let test_record_id = "11111"; - let expect_time = Utc - .datetime_from_str("1996-02-27T01:05:01Z", "%Y-%m-%dT%H:%M:%SZ") - .unwrap(); + let expect_naivetime = + NaiveDateTime::parse_from_str("1996-02-27T01:05:01Z", "%Y-%m-%dT%H:%M:%SZ").unwrap(); + let expect_time = Utc.from_local_datetime(&expect_naivetime).unwrap(); let expect_tz = expect_time.with_timezone(&Utc); let dummy_action = Action::JsonTimeline(JSONOutputOption { output_options: OutputOption { @@ -3200,10 +3201,9 @@ mod tests { let output = "displaytest"; let test_recinfo = "testinfo"; let test_recid = "22222"; - - let test_timestamp = Utc - .datetime_from_str("1996-02-27T01:05:01Z", "%Y-%m-%dT%H:%M:%SZ") - .unwrap(); + let test_naivetime = + NaiveDateTime::parse_from_str("1996-02-27T01:05:01Z", "%Y-%m-%dT%H:%M:%SZ").unwrap(); + let test_timestamp = Utc.from_local_datetime(&test_naivetime).unwrap(); let expect_header = "Timestamp ‖ Computer ‖ Channel ‖ EventID ‖ Level ‖ RecordID ‖ RuleTitle ‖ Details ‖ RecordInformation\n"; let expect_tz = test_timestamp.with_timezone(&Local); @@ -3358,9 +3358,9 @@ mod tests { let test_attack = "execution/txxxx.yyy"; let test_recinfo = "CommandRLine: hoge"; let test_record_id = "11111"; - let expect_time = Utc - .datetime_from_str("1996-02-27T01:05:01Z", "%Y-%m-%dT%H:%M:%SZ") - .unwrap(); + let expect_naivetime = + NaiveDateTime::parse_from_str("1996-02-27T01:05:01Z", "%Y-%m-%dT%H:%M:%SZ").unwrap(); + let expect_time = Utc.from_local_datetime(&expect_naivetime).unwrap(); let expect_tz = expect_time.with_timezone(&Utc); let json_dummy_action = Action::JsonTimeline(JSONOutputOption { output_options: OutputOption { @@ -3618,9 +3618,9 @@ mod tests { let test_attack = "execution/txxxx.yyy"; let test_recinfo = "CommandRLine: hoge"; let test_record_id = "11111"; - let expect_time = Utc - .datetime_from_str("1996-02-27T01:05:01Z", "%Y-%m-%dT%H:%M:%SZ") - .unwrap(); + let expect_naivetime = + NaiveDateTime::parse_from_str("1996-02-27T01:05:01Z", "%Y-%m-%dT%H:%M:%SZ").unwrap(); + let expect_time = Utc.from_local_datetime(&expect_naivetime).unwrap(); let expect_tz = expect_time.with_timezone(&Utc); let json_dummy_action = Action::JsonTimeline(JSONOutputOption { output_options: OutputOption { diff --git a/src/detections/detection.rs b/src/detections/detection.rs index 8d8e962ec..07027d615 100644 --- a/src/detections/detection.rs +++ b/src/detections/detection.rs @@ -1182,6 +1182,7 @@ mod tests { use crate::detections::utils; use crate::filter; use crate::options::profile::Profile; + use chrono::NaiveDateTime; use chrono::TimeZone; use chrono::Utc; use compact_str::CompactString; @@ -1449,9 +1450,9 @@ mod tests { fn test_insert_message_with_geoip() { let test_filepath: &str = "test.evtx"; let test_rulepath: &str = "test-rule.yml"; - let expect_time = Utc - .datetime_from_str("1996-02-27T01:05:01Z", "%Y-%m-%dT%H:%M:%SZ") - .unwrap(); + let expect_naivetime = + NaiveDateTime::parse_from_str("1996-02-27T01:05:01Z", "%Y-%m-%dT%H:%M:%SZ").unwrap(); + let expect_time = Utc.from_local_datetime(&expect_naivetime).unwrap(); let dummy_action = Action::CsvTimeline(CsvOutputOption { output_options: OutputOption { input_args: InputOption { @@ -1583,9 +1584,9 @@ mod tests { fn test_filtered_insert_message_with_geoip() { let test_filepath: &str = "test.evtx"; let test_rulepath: &str = "test-rule.yml"; - let expect_time = Utc - .datetime_from_str("1996-02-27T01:05:01Z", "%Y-%m-%dT%H:%M:%SZ") - .unwrap(); + let expect_naivetime = + NaiveDateTime::parse_from_str("1996-02-27T01:05:01Z", "%Y-%m-%dT%H:%M:%SZ").unwrap(); + let expect_time = Utc.from_local_datetime(&expect_naivetime).unwrap(); let dummy_action = Action::CsvTimeline(CsvOutputOption { output_options: OutputOption { input_args: InputOption { @@ -1713,9 +1714,9 @@ mod tests { #[test] fn test_insert_message_extra_field_info() { let test_filepath: &str = "test.evtx"; - let expect_time = Utc - .datetime_from_str("1996-02-27T01:05:01Z", "%Y-%m-%dT%H:%M:%SZ") - .unwrap(); + let expect_naivetime = + NaiveDateTime::parse_from_str("1996-02-27T01:05:01Z", "%Y-%m-%dT%H:%M:%SZ").unwrap(); + let expect_time = Utc.from_local_datetime(&expect_naivetime).unwrap(); let dummy_action = Action::CsvTimeline(CsvOutputOption { output_options: OutputOption { input_args: InputOption { @@ -1856,9 +1857,9 @@ mod tests { #[test] fn test_insert_message_multiline_ruleauthor() { let test_filepath: &str = "test.evtx"; - let expect_time = Utc - .datetime_from_str("1996-02-27T01:05:01Z", "%Y-%m-%dT%H:%M:%SZ") - .unwrap(); + let expect_naivetime = + NaiveDateTime::parse_from_str("1996-02-27T01:05:01Z", "%Y-%m-%dT%H:%M:%SZ").unwrap(); + let expect_time = Utc.from_local_datetime(&expect_naivetime).unwrap(); let dummy_action = Action::CsvTimeline(CsvOutputOption { output_options: OutputOption { input_args: InputOption { From baa3ea890a1462911019f87c493473e18a8acacd Mon Sep 17 00:00:00 2001 From: Yamato Security <71482215+YamatoSecurity@users.noreply.github.com> Date: Sun, 10 Sep 2023 07:30:20 +0900 Subject: [PATCH 23/24] update changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 786906cfc..c67fb998c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,7 @@ **Enhancements:** - Optimized the number of records to load at a time. (#1175) (@yamatosecurity) -- Replaced backslashes in paths in the progress bar with slash. (#1172) (@hitenkoku) +- Replaced double backslashes in paths in under the progress bar on Windows systems with single forward slashes. (#1172) (@hitenkoku) **Bug Fixes:** From afe320305fa244e35c67e1b07b5d5e8eadd8ac25 Mon Sep 17 00:00:00 2001 From: Yamato Security <71482215+YamatoSecurity@users.noreply.github.com> Date: Sun, 10 Sep 2023 07:36:48 +0900 Subject: [PATCH 24/24] update changelog and error message --- CHANGELOG.md | 2 +- src/detections/field_data_map.rs | 2 +- src/main.rs | 2 +- src/yaml.rs | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f1f6674cd..b368e9f38 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ **Enhancements:** -- Added error message to indicate that trailing backslashes should not be used in a directory path specification. (#1166) (@hitenkoku, thanks for the suggestion from @joswr1ght) +- Added an error message to indicate that when you can't load evtx files in Windows due to specifying a directory path with spaces in it, you need to remove the trailing backslash. (#1166) (@hitenkoku, thanks for the suggestion from @joswr1ght) - Optimized the number of records to load at a time. (#1175) (@yamatosecurity) **Bug Fixes:** diff --git a/src/detections/field_data_map.rs b/src/detections/field_data_map.rs index f09de36c2..c2aca4b5e 100644 --- a/src/detections/field_data_map.rs +++ b/src/detections/field_data_map.rs @@ -135,7 +135,7 @@ fn load_yaml_files(dir_path: &Path) -> Result, String> { Err(e) => { let mut msg = format!("Failed to open field mapping dir[{path}]. ",); if e.to_string().ends_with("123)") { - msg = format!("{msg}. When specifying a directory path in Windows, do not include a trailing slash at the end of the path."); + msg = format!("{msg}. You may not be able to load evtx files when there are spaces in the directory path. Please enclose the path with double quotes and remove any trailing slash at the end of the path."); } AlertMessage::warn(&msg).ok(); Err(e.to_string()) diff --git a/src/main.rs b/src/main.rs index 969ab9231..696280ba3 100644 --- a/src/main.rs +++ b/src/main.rs @@ -883,7 +883,7 @@ impl App { if entries.is_err() { let mut errmsg = format!("{}", entries.unwrap_err()); if errmsg.ends_with("123)") { - errmsg = format!("{errmsg}. When specifying a directory path in Windows, do not include a trailing slash at the end of the path."); + errmsg = format!("{errmsg}. You may not be able to load evtx files when there are spaces in the directory path. Please enclose the path with double quotes and remove any trailing slash at the end of the path."); } if stored_static.verbose_flag { AlertMessage::alert(&errmsg).ok(); diff --git a/src/yaml.rs b/src/yaml.rs index 873a2ee39..367688971 100644 --- a/src/yaml.rs +++ b/src/yaml.rs @@ -88,7 +88,7 @@ impl ParseYaml { err_contents ); if err_contents.ends_with("123)") { - errmsg = format!("{errmsg}. When specifying a directory path in Windows, do not include a trailing slash at the end of the path."); + errmsg = format!("{errmsg}. You may not be able to load evtx files when there are spaces in the directory path. Please enclose the path with double quotes and remove any trailing slash at the end of the path."); } if stored_static.verbose_flag { AlertMessage::alert(&errmsg)?;