From 193a2a949d2c12c13705e9f414fb4fa64b97b354 Mon Sep 17 00:00:00 2001 From: fukusuket <41001169+fukusuket@users.noreply.github.com> Date: Mon, 3 Jun 2024 02:15:55 +0900 Subject: [PATCH 1/5] feat: add support for sigma correlation Event count --- src/detections/detection.rs | 52 ++--- src/detections/rule/correlation_parser.rs | 254 ++++++++++++++++++++++ src/detections/rule/mod.rs | 60 +++-- src/main.rs | 72 +++--- 4 files changed, 367 insertions(+), 71 deletions(-) create mode 100644 src/detections/rule/correlation_parser.rs diff --git a/src/detections/detection.rs b/src/detections/detection.rs index 862ff9a14..9a830fb2c 100644 --- a/src/detections/detection.rs +++ b/src/detections/detection.rs @@ -1,38 +1,38 @@ extern crate csv; -use crate::detections::configs::Action; -use crate::detections::utils::{create_recordinfos, format_time, write_color_buffer}; -use crate::options::profile::Profile::{ - self, Channel, Computer, EventID, EvtxFile, Level, MitreTactics, MitreTags, OtherTags, - Provider, RecordID, RecoveredRecord, RenderedMessage, RuleAuthor, RuleCreationDate, RuleFile, - RuleID, RuleModifiedDate, RuleTitle, SrcASN, SrcCity, SrcCountry, Status, TgtASN, TgtCity, - TgtCountry, Timestamp, -}; +use std::default::Default; +use std::fmt::Write; +use std::path::Path; +use std::sync::Arc; + use chrono::{TimeZone, Utc}; use compact_str::CompactString; +use hashbrown::HashMap; use itertools::Itertools; use nested::Nested; use num_format::{Locale, ToFormattedString}; -use std::default::Default; +use serde_json::Value; use termcolor::{BufferWriter, Color, ColorChoice}; +use tokio::{runtime::Runtime, spawn, task::JoinHandle}; use yaml_rust::Yaml; +use crate::detections::configs::Action; +use crate::detections::configs::STORED_EKEY_ALIAS; +use crate::detections::field_data_map::FieldDataMapKey; use crate::detections::message::{AlertMessage, DetectInfo, ERROR_LOG_STACK, TAGS_CONFIG}; use crate::detections::rule::{self, AggResult, RuleNode}; +use crate::detections::utils::{create_recordinfos, format_time, write_color_buffer}; use crate::detections::utils::{get_serde_number_to_string, make_ascii_titlecase}; use crate::filter; use crate::options::htmlreport; use crate::options::pivot::insert_pivot_keyword; +use crate::options::profile::Profile::{ + self, Channel, Computer, EventID, EvtxFile, Level, MitreTactics, MitreTags, OtherTags, + Provider, RecordID, RecoveredRecord, RenderedMessage, RuleAuthor, RuleCreationDate, RuleFile, + RuleID, RuleModifiedDate, RuleTitle, SrcASN, SrcCity, SrcCountry, Status, TgtASN, TgtCity, + TgtCountry, Timestamp, +}; use crate::yaml::ParseYaml; -use hashbrown::HashMap; -use serde_json::Value; -use std::fmt::Write; -use std::path::Path; - -use crate::detections::configs::STORED_EKEY_ALIAS; -use crate::detections::field_data_map::FieldDataMapKey; -use std::sync::Arc; -use tokio::{runtime::Runtime, spawn, task::JoinHandle}; use super::configs::{ EventKeyAliasConfig, StoredStatic, GEOIP_DB_PARSER, GEOIP_DB_YAML, GEOIP_FILTER, STORED_STATIC, @@ -1190,6 +1190,15 @@ impl Detection { #[cfg(test)] mod tests { + use std::path::Path; + + use chrono::TimeZone; + use chrono::Utc; + use compact_str::CompactString; + use serde_json::Value; + use yaml_rust::Yaml; + use yaml_rust::YamlLoader; + use crate::detections; use crate::detections::configs::load_eventkey_alias; use crate::detections::configs::Action; @@ -1209,13 +1218,6 @@ mod tests { use crate::detections::utils; use crate::filter; use crate::options::profile::Profile; - use chrono::TimeZone; - use chrono::Utc; - use compact_str::CompactString; - use serde_json::Value; - use std::path::Path; - use yaml_rust::Yaml; - use yaml_rust::YamlLoader; fn create_dummy_stored_static() -> StoredStatic { StoredStatic::create_static_data(Some(Config { diff --git a/src/detections/rule/correlation_parser.rs b/src/detections/rule/correlation_parser.rs new file mode 100644 index 000000000..166e047e3 --- /dev/null +++ b/src/detections/rule/correlation_parser.rs @@ -0,0 +1,254 @@ +use std::error::Error; + +use yaml_rust::Yaml; + +use crate::detections::configs::StoredStatic; +use crate::detections::rule::aggregation_parser::{ + AggregationConditionToken, AggregationParseInfo, +}; +use crate::detections::rule::count::TimeFrameInfo; +use crate::detections::rule::selectionnodes::OrSelectionNode; +use crate::detections::rule::{DetectionNode, RuleNode}; + +fn is_related_rule(rule_node: &RuleNode, id_or_title: &str) -> bool { + if let Some(hash) = rule_node.yaml.as_hash() { + if let Some(id) = hash.get(&Yaml::String("id".to_string())) { + if id.as_str() == Some(id_or_title) { + return true; + } + } + if let Some(title) = hash.get(&Yaml::String("title".to_string())) { + if title.as_str() == Some(id_or_title) { + return true; + } + } + } + false +} + +fn parse_condition(yaml: &Yaml) -> Result<(AggregationConditionToken, i64), Box> { + if let Some(hash) = yaml.as_hash() { + if let Some(condition) = hash.get(&Yaml::String("condition".to_string())) { + if let Some(condition_hash) = condition.as_hash() { + if let Some((key, value)) = condition_hash.into_iter().next() { + let key_str = key.as_str().ok_or("Failed to convert key to string")?; + let token = match key_str { + "eq" => AggregationConditionToken::EQ, + "lte" => AggregationConditionToken::LE, + "gte" => AggregationConditionToken::GE, + "lt" => AggregationConditionToken::LT, + "gt" => AggregationConditionToken::GT, + _ => return Err(format!("Invalid condition token: {}", key_str).into()), + }; + let value_num = value.as_i64().ok_or("Failed to convert value to i64")?; + return Ok((token, value_num)); + } + } + } + } + Err("Failed to parse condition".into()) +} + +fn to_or_selection_node(related_rule_nodes: Vec) -> OrSelectionNode { + let mut or_selection_node = OrSelectionNode::new(); + for rule_node in related_rule_nodes { + or_selection_node + .child_nodes + .push(rule_node.detection.condition.unwrap()); + } + or_selection_node +} + +fn get_related_rules_id(yaml: &Yaml) -> Result, Box> { + let correlation = yaml["correlation"] + .as_hash() + .ok_or("Failed to get 'correlation'")?; + let rules_yaml = correlation + .get(&Yaml::String("rules".to_string())) + .ok_or("Failed to get 'rules'")?; + + let mut rules = Vec::new(); + for rule_yaml in rules_yaml + .as_vec() + .ok_or("Failed to convert 'rules' to Vec")? + { + let rule = rule_yaml + .as_str() + .ok_or("Failed to convert rule to string")? + .to_string(); + rules.push(rule); + } + + Ok(rules) +} + +fn get_group_by_from_yaml(yaml: &Yaml) -> Result> { + let correlation = yaml["correlation"] + .as_hash() + .ok_or("Failed to get 'correlation'")?; + let group_by_yaml = correlation + .get(&Yaml::String("group-by".to_string())) + .ok_or("Failed to get 'group-by'")?; + + let mut group_by = Vec::new(); + for group_by_yaml in group_by_yaml + .as_vec() + .ok_or("Failed to convert 'group-by' to Vec")? + { + let group = group_by_yaml + .as_str() + .ok_or("Failed to convert group to string")? + .to_string(); + group_by.push(group); + } + + Ok(group_by.join("_")) +} +fn parse_tframe(value: String) -> Result> { + let ttype; + let mut target_val = value.as_str(); + if target_val.ends_with('s') { + ttype = "s"; + } else if target_val.ends_with('m') { + ttype = "m"; + } else if target_val.ends_with('h') { + ttype = "h"; + } else if target_val.ends_with('d') { + ttype = "d"; + } else { + return Err("Invalid time frame".into()); + } + if !ttype.is_empty() { + target_val = &value[..value.len() - 1]; + } + Ok(TimeFrameInfo { + timetype: ttype.to_string(), + timenum: target_val.parse::(), + }) +} + +fn create_related_rule_nodes( + related_rules_ids: Vec, + other_rules: &[RuleNode], + stored_static: &StoredStatic, +) -> Vec { + let mut related_rule_nodes: Vec = Vec::new(); + for id in related_rules_ids { + for other_rule in other_rules { + if is_related_rule(other_rule, &id) { + let mut node = RuleNode::new(other_rule.rulepath.clone(), other_rule.yaml.clone()); + let _ = node.init(stored_static); + related_rule_nodes.push(node); + } + } + } + related_rule_nodes +} + +fn create_detection( + rule_node: &RuleNode, + related_rule_nodes: Vec, +) -> Result> { + let condition = parse_condition(&rule_node.yaml["correlation"])?; + let group_by = get_group_by_from_yaml(&rule_node.yaml)?; + let timespan = rule_node.yaml["correlation"]["timespan"].as_str().unwrap(); + let time_frame = parse_tframe(timespan.to_string())?; + let nodes = to_or_selection_node(related_rule_nodes); + let agg_info = AggregationParseInfo { + _field_name: None, + _by_field_name: Some(group_by), + _cmp_op: condition.0, + _cmp_num: condition.1, + }; + Ok(DetectionNode::new_with_data( + Some(Box::new(nodes)), + Some(agg_info), + Some(time_frame), + )) +} + +pub fn parse_correlation_rules( + rule_nodes: Vec, + stored_static: &StoredStatic, +) -> Vec { + let (correlation_rules, other_rules): (Vec, Vec) = rule_nodes + .into_iter() + .partition(|rule_node| !rule_node.yaml["correlation"].is_badvalue()); + let mut parsed_rules: Vec = correlation_rules + .into_iter() + .map(|rule_node| { + let related_rules_ids = get_related_rules_id(&rule_node.yaml).unwrap(); + let related_rules = + create_related_rule_nodes(related_rules_ids, &other_rules, stored_static); + let detection = create_detection(&rule_node, related_rules).unwrap(); + RuleNode::new_with_detection(rule_node.rulepath, rule_node.yaml, detection) + }) + .collect(); + parsed_rules.extend(other_rules); + parsed_rules +} + +#[cfg(test)] +mod tests { + use yaml_rust::YamlLoader; + + use super::*; + + #[test] + fn test_parse_condition_valid() { + let yaml_str = r#" + condition: + gte: 3 + "#; + let yaml = &YamlLoader::load_from_str(yaml_str).unwrap()[0]; + let result = parse_condition(yaml); + assert!(result.is_ok()); + let (_, value) = result.unwrap(); + assert_eq!(value, 3); + } + + #[test] + fn test_parse_condition_invalid_token() { + let yaml_str = r#" + condition: + invalid_token: 3 + "#; + let yaml = &YamlLoader::load_from_str(yaml_str).unwrap()[0]; + let result = parse_condition(yaml); + assert!(result.is_err()); + } + + #[test] + fn test_parse_condition_invalid_value() { + let yaml_str = r#" + condition: + gte: invalid_value + "#; + let yaml = &YamlLoader::load_from_str(yaml_str).unwrap()[0]; + let result = parse_condition(yaml); + assert!(result.is_err()); + } + + #[test] + fn test_get_rules_from_yaml() { + let yaml_str = r#" + title: Many failed logins to the same computer + id: 0e95725d-7320-415d-80f7-004da920fc11 + correlation: + type: event_count + rules: + - e87bd730-df45-4ae9-85de-6c75369c5d29 # Logon Failure (Wrong Password) + - 8afa97ce-a217-4f7c-aced-3e320a57756d # Logon Failure (User Does Not Exist) + group-by: + - Computer + timespan: 5m + condition: + gte: 3 + "#; + let yaml = &YamlLoader::load_from_str(yaml_str).unwrap()[0]; + let result = get_related_rules_id(yaml).unwrap(); + assert_eq!(result.len(), 2); + assert_eq!(result[0], "e87bd730-df45-4ae9-85de-6c75369c5d29"); + assert_eq!(result[1], "8afa97ce-a217-4f7c-aced-3e320a57756d"); + } +} diff --git a/src/detections/rule/mod.rs b/src/detections/rule/mod.rs index 8b9fcbc72..d2951febf 100644 --- a/src/detections/rule/mod.rs +++ b/src/detections/rule/mod.rs @@ -1,31 +1,32 @@ extern crate regex; -use chrono::{DateTime, Utc}; +use std::{fmt::Debug, sync::Arc, vec}; +use chrono::{DateTime, Utc}; use hashbrown::HashMap; use nested::Nested; -use std::{fmt::Debug, sync::Arc, vec}; - use yaml_rust::Yaml; -mod matchers; -mod selectionnodes; -use self::selectionnodes::{LeafSelectionNode, SelectionNode}; -mod aggregation_parser; +use super::configs::{EventKeyAliasConfig, StoredStatic}; +use super::detection::EvtxRecordInfo; + use self::aggregation_parser::AggregationParseInfo; +use self::count::{AggRecordTimeInfo, TimeFrameInfo}; +use self::selectionnodes::{LeafSelectionNode, SelectionNode}; +mod aggregation_parser; mod condition_parser; +pub mod correlation_parser; mod count; -use self::count::{AggRecordTimeInfo, TimeFrameInfo}; - -use super::configs::{EventKeyAliasConfig, StoredStatic}; -use super::detection::EvtxRecordInfo; +mod matchers; +mod selectionnodes; pub fn create_rule(rulepath: String, yaml: Yaml) -> RuleNode { RuleNode::new(rulepath, yaml) } -/// Ruleファイルを表すノード +/// Ruleファイルを表すノ +/// ード pub struct RuleNode { pub rulepath: String, pub yaml: Yaml, @@ -49,8 +50,24 @@ impl RuleNode { } } + fn new_with_detection( + rule_path: String, + yaml_data: Yaml, + detection: DetectionNode, + ) -> RuleNode { + RuleNode { + rulepath: rule_path, + yaml: yaml_data, + detection, + countdata: HashMap::new(), + } + } + pub fn init(&mut self, stored_static: &StoredStatic) -> Result<(), Vec> { let mut errmsgs: Vec = vec![]; + if !&self.yaml["correlation"].is_badvalue() { + return Result::Ok(()); + } // detection node initialization let detection_result = self.detection.init(&self.yaml["detection"], stored_static); @@ -158,6 +175,19 @@ impl DetectionNode { } } + pub fn new_with_data( + condition: Option>, + aggregation_condition: Option, + timeframe: Option, + ) -> DetectionNode { + DetectionNode { + name_to_selection: HashMap::new(), + condition, + aggregation_condition, + timeframe, + } + } + fn init( &mut self, detection_yaml: &Yaml, @@ -376,7 +406,8 @@ impl AggResult { mod tests { use std::path::Path; - use super::RuleNode; + use yaml_rust::YamlLoader; + use crate::detections::{ self, configs::{ @@ -386,7 +417,8 @@ mod tests { rule::create_rule, utils, }; - use yaml_rust::YamlLoader; + + use super::RuleNode; fn create_dummy_stored_static() -> StoredStatic { StoredStatic::create_static_data(Some(Config { diff --git a/src/main.rs b/src/main.rs index 5fc6a7ea9..a1499f9ed 100644 --- a/src/main.rs +++ b/src/main.rs @@ -4,6 +4,22 @@ extern crate maxminddb; extern crate serde; extern crate serde_derive; +use std::borrow::BorrowMut; +use std::ffi::{OsStr, OsString}; +use std::fmt::Display; +use std::fmt::Write as _; +use std::io::{BufWriter, Write}; +use std::path::Path; +use std::ptr::null_mut; +use std::sync::Arc; +use std::time::Duration; +use std::{ + env, + fs::{self, File}, + path::PathBuf, + vec, +}; + use bytesize::ByteSize; use chrono::{DateTime, Datelike, Local, NaiveDateTime, Utc}; use clap::Command; @@ -13,6 +29,19 @@ use dialoguer::Confirm; use dialoguer::{theme::ColorfulTheme, Select}; use evtx::{EvtxParser, ParserSettings, RecordAllocation}; use hashbrown::{HashMap, HashSet}; +use indicatif::ProgressBar; +use indicatif::{ProgressDrawTarget, ProgressStyle}; +use itertools::Itertools; +use libmimalloc_sys::mi_stats_print_out; +use mimalloc::MiMalloc; +use nested::Nested; +use num_format::{Locale, ToFormattedString}; +use serde_json::{Map, Value}; +use termcolor::{BufferWriter, Color, ColorChoice}; +use tokio::runtime::Runtime; +use tokio::spawn; +use tokio::task::JoinHandle; + use hayabusa::afterfact::{self, AfterfactInfo, AfterfactWriter}; use hayabusa::debug::checkpoint_process_timer::CHECKPOINT; use hayabusa::detections::configs::{ @@ -21,6 +50,7 @@ use hayabusa::detections::configs::{ }; use hayabusa::detections::detection::{self, EvtxRecordInfo}; use hayabusa::detections::message::{AlertMessage, DetectInfo, ERROR_LOG_STACK}; +use hayabusa::detections::rule::correlation_parser::parse_correlation_rules; use hayabusa::detections::rule::{get_detection_keys, RuleNode}; use hayabusa::detections::utils; use hayabusa::detections::utils::{ @@ -36,34 +66,6 @@ use hayabusa::timeline::computer_metrics::countup_event_by_computer; use hayabusa::{detections::configs, timeline::timelines::Timeline}; use hayabusa::{detections::utils::write_color_buffer, filter}; use hayabusa::{options, yaml}; -use indicatif::ProgressBar; -use indicatif::{ProgressDrawTarget, ProgressStyle}; -use itertools::Itertools; -use libmimalloc_sys::mi_stats_print_out; -use mimalloc::MiMalloc; -use nested::Nested; -use num_format::{Locale, ToFormattedString}; -use serde_json::{Map, Value}; -use std::borrow::BorrowMut; -use std::ffi::{OsStr, OsString}; -use std::fmt::Display; -use std::fmt::Write as _; -use std::io::{BufWriter, Write}; -use std::path::Path; -use std::ptr::null_mut; -use std::sync::Arc; -use std::time::Duration; -use std::{ - env, - fs::{self, File}, - path::PathBuf, - vec, -}; -use termcolor::{BufferWriter, Color, ColorChoice}; -use tokio::runtime::Runtime; -use tokio::spawn; -use tokio::task::JoinHandle; - #[cfg(target_os = "windows")] use is_elevated::is_elevated; @@ -1455,7 +1457,10 @@ impl App { println!("{evtx_files_after_channel_filter}"); } if !stored_static.enable_all_rules { - rule_files.retain(|r| channel_filter.rulepathes.contains(&r.rulepath)); + rule_files.retain(|r| { + channel_filter.rulepathes.contains(&r.rulepath) + || !r.yaml["correlation"].is_badvalue() + }); let rules_after_channel_filter = format!( "Detection rules enabled after channel filter: {}", (rule_files.len()).to_formatted_string(&Locale::en) @@ -1464,6 +1469,7 @@ impl App { println!(); } } + rule_files = parse_correlation_rules(rule_files, stored_static); output_profile_name(&stored_static.output_option, true); println!(); println!("Scanning in progress. Please wait."); @@ -2326,9 +2332,11 @@ mod tests { path::Path, }; - use crate::App; use chrono::Local; use hashbrown::HashSet; + use itertools::Itertools; + use yaml_rust::YamlLoader; + use hayabusa::{ afterfact::{self, AfterfactInfo}, detections::{ @@ -2344,8 +2352,8 @@ mod tests { options::htmlreport::HTML_REPORTER, timeline::timelines::Timeline, }; - use itertools::Itertools; - use yaml_rust::YamlLoader; + + use crate::App; fn create_dummy_stored_static() -> StoredStatic { StoredStatic::create_static_data(Some(Config { From 30a2f455bf0c682b4db6eb2f55b38d2c5015a83e Mon Sep 17 00:00:00 2001 From: fukusuket <41001169+fukusuket@users.noreply.github.com> Date: Mon, 3 Jun 2024 10:25:06 +0900 Subject: [PATCH 2/5] chg: add handling for rule parse error --- src/detections/detection.rs | 4 +- src/detections/rule/correlation_parser.rs | 99 ++++++++++++++++++----- src/main.rs | 2 - 3 files changed, 84 insertions(+), 21 deletions(-) diff --git a/src/detections/detection.rs b/src/detections/detection.rs index 9a830fb2c..0e327dead 100644 --- a/src/detections/detection.rs +++ b/src/detections/detection.rs @@ -20,6 +20,7 @@ use crate::detections::configs::Action; use crate::detections::configs::STORED_EKEY_ALIAS; use crate::detections::field_data_map::FieldDataMapKey; use crate::detections::message::{AlertMessage, DetectInfo, ERROR_LOG_STACK, TAGS_CONFIG}; +use crate::detections::rule::correlation_parser::parse_correlation_rules; use crate::detections::rule::{self, AggResult, RuleNode}; use crate::detections::utils::{create_recordinfos, format_time, write_color_buffer}; use crate::detections::utils::{get_serde_number_to_string, make_ascii_titlecase}; @@ -134,12 +135,13 @@ impl Detection { None }; // parse rule files - let ret = rulefile_loader + let mut ret = rulefile_loader .files .into_iter() .map(|rule_file_tuple| rule::create_rule(rule_file_tuple.0, rule_file_tuple.1)) .filter_map(return_if_success) .collect(); + ret = parse_correlation_rules(ret, stored_static, &mut parseerror_count); if !(stored_static.logon_summary_flag || stored_static.search_flag || stored_static.metrics_flag diff --git a/src/detections/rule/correlation_parser.rs b/src/detections/rule/correlation_parser.rs index 166e047e3..6dfa0de26 100644 --- a/src/detections/rule/correlation_parser.rs +++ b/src/detections/rule/correlation_parser.rs @@ -3,6 +3,7 @@ use std::error::Error; use yaml_rust::Yaml; use crate::detections::configs::StoredStatic; +use crate::detections::message::{AlertMessage, ERROR_LOG_STACK}; use crate::detections::rule::aggregation_parser::{ AggregationConditionToken, AggregationParseInfo, }; @@ -31,7 +32,9 @@ fn parse_condition(yaml: &Yaml) -> Result<(AggregationConditionToken, i64), Box< if let Some(condition) = hash.get(&Yaml::String("condition".to_string())) { if let Some(condition_hash) = condition.as_hash() { if let Some((key, value)) = condition_hash.into_iter().next() { - let key_str = key.as_str().ok_or("Failed to convert key to string")?; + let key_str = key + .as_str() + .ok_or("Failed to convert condition key to string")?; let token = match key_str { "eq" => AggregationConditionToken::EQ, "lte" => AggregationConditionToken::LE, @@ -40,7 +43,9 @@ fn parse_condition(yaml: &Yaml) -> Result<(AggregationConditionToken, i64), Box< "gt" => AggregationConditionToken::GT, _ => return Err(format!("Invalid condition token: {}", key_str).into()), }; - let value_num = value.as_i64().ok_or("Failed to convert value to i64")?; + let value_num = value + .as_i64() + .ok_or("Failed to convert condition value to i64")?; return Ok((token, value_num)); } } @@ -151,25 +156,53 @@ fn create_detection( ) -> Result> { let condition = parse_condition(&rule_node.yaml["correlation"])?; let group_by = get_group_by_from_yaml(&rule_node.yaml)?; - let timespan = rule_node.yaml["correlation"]["timespan"].as_str().unwrap(); - let time_frame = parse_tframe(timespan.to_string())?; - let nodes = to_or_selection_node(related_rule_nodes); - let agg_info = AggregationParseInfo { - _field_name: None, - _by_field_name: Some(group_by), - _cmp_op: condition.0, - _cmp_num: condition.1, - }; - Ok(DetectionNode::new_with_data( - Some(Box::new(nodes)), - Some(agg_info), - Some(time_frame), - )) + let timespan = rule_node.yaml["correlation"]["timespan"].as_str(); + match timespan { + None => Err("Failed to get 'timespan'".into()), + Some(timespan) => { + let time_frame = parse_tframe(timespan.to_string())?; + let nodes = to_or_selection_node(related_rule_nodes); + let agg_info = AggregationParseInfo { + _field_name: None, + _by_field_name: Some(group_by), + _cmp_op: condition.0, + _cmp_num: condition.1, + }; + Ok(DetectionNode::new_with_data( + Some(Box::new(nodes)), + Some(agg_info), + Some(time_frame), + )) + } + } +} + +fn error_log( + rule_path: &str, + reason: &str, + stored_static: &StoredStatic, + parseerror_count: &mut u128, +) { + let msg = format!( + "Failed to parse rule. (FilePath : {}) {}", + rule_path, reason + ); + if stored_static.verbose_flag { + AlertMessage::alert(msg.as_str()).ok(); + } + if !stored_static.quiet_errors_flag { + ERROR_LOG_STACK + .lock() + .unwrap() + .push(format!("[WARN] {msg}")); + } + *parseerror_count += 1; } pub fn parse_correlation_rules( rule_nodes: Vec, stored_static: &StoredStatic, + parseerror_count: &mut u128, ) -> Vec { let (correlation_rules, other_rules): (Vec, Vec) = rule_nodes .into_iter() @@ -177,10 +210,40 @@ pub fn parse_correlation_rules( let mut parsed_rules: Vec = correlation_rules .into_iter() .map(|rule_node| { - let related_rules_ids = get_related_rules_id(&rule_node.yaml).unwrap(); + if rule_node.yaml["correlation"]["type"].as_str() != Some("event_count") { + let m = "The type of correlations rule only supports event_count."; + error_log(&rule_node.rulepath, m, stored_static, parseerror_count); + return rule_node; + } + let related_rules_ids = get_related_rules_id(&rule_node.yaml); + let related_rules_ids = match related_rules_ids { + Ok(related_rules_ids) => related_rules_ids, + Err(_) => { + let m = "Related rule not found."; + error_log(&rule_node.rulepath, m, stored_static, parseerror_count); + return rule_node; + } + }; + if related_rules_ids.is_empty() { + let m = "Related rule not found."; + error_log(&rule_node.rulepath, m, stored_static, parseerror_count); + return rule_node; + } let related_rules = create_related_rule_nodes(related_rules_ids, &other_rules, stored_static); - let detection = create_detection(&rule_node, related_rules).unwrap(); + let detection = create_detection(&rule_node, related_rules); + let detection = match detection { + Ok(detection) => detection, + Err(e) => { + error_log( + &rule_node.rulepath, + e.to_string().as_str(), + stored_static, + parseerror_count, + ); + return rule_node; + } + }; RuleNode::new_with_detection(rule_node.rulepath, rule_node.yaml, detection) }) .collect(); diff --git a/src/main.rs b/src/main.rs index a1499f9ed..5c66890a3 100644 --- a/src/main.rs +++ b/src/main.rs @@ -50,7 +50,6 @@ use hayabusa::detections::configs::{ }; use hayabusa::detections::detection::{self, EvtxRecordInfo}; use hayabusa::detections::message::{AlertMessage, DetectInfo, ERROR_LOG_STACK}; -use hayabusa::detections::rule::correlation_parser::parse_correlation_rules; use hayabusa::detections::rule::{get_detection_keys, RuleNode}; use hayabusa::detections::utils; use hayabusa::detections::utils::{ @@ -1469,7 +1468,6 @@ impl App { println!(); } } - rule_files = parse_correlation_rules(rule_files, stored_static); output_profile_name(&stored_static.output_option, true); println!(); println!("Scanning in progress. Please wait."); From b3445624e81d567df1cfff4aa6e4a136562fb213 Mon Sep 17 00:00:00 2001 From: fukusuket <41001169+fukusuket@users.noreply.github.com> Date: Mon, 3 Jun 2024 10:56:10 +0900 Subject: [PATCH 3/5] fix: multiple key agg case --- src/detections/rule/correlation_parser.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/detections/rule/correlation_parser.rs b/src/detections/rule/correlation_parser.rs index 6dfa0de26..881b0e547 100644 --- a/src/detections/rule/correlation_parser.rs +++ b/src/detections/rule/correlation_parser.rs @@ -4,12 +4,12 @@ use yaml_rust::Yaml; use crate::detections::configs::StoredStatic; use crate::detections::message::{AlertMessage, ERROR_LOG_STACK}; +use crate::detections::rule::{DetectionNode, RuleNode}; use crate::detections::rule::aggregation_parser::{ AggregationConditionToken, AggregationParseInfo, }; use crate::detections::rule::count::TimeFrameInfo; use crate::detections::rule::selectionnodes::OrSelectionNode; -use crate::detections::rule::{DetectionNode, RuleNode}; fn is_related_rule(rule_node: &RuleNode, id_or_title: &str) -> bool { if let Some(hash) = rule_node.yaml.as_hash() { @@ -107,7 +107,7 @@ fn get_group_by_from_yaml(yaml: &Yaml) -> Result> { group_by.push(group); } - Ok(group_by.join("_")) + Ok(group_by.join(",")) } fn parse_tframe(value: String) -> Result> { let ttype; From 0b9c359747c0560c4463c9ce8b5ee3df13ce904b Mon Sep 17 00:00:00 2001 From: fukusuket <41001169+fukusuket@users.noreply.github.com> Date: Mon, 3 Jun 2024 10:57:57 +0900 Subject: [PATCH 4/5] fix: cargo fmt --- src/detections/rule/correlation_parser.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/detections/rule/correlation_parser.rs b/src/detections/rule/correlation_parser.rs index 881b0e547..85823823a 100644 --- a/src/detections/rule/correlation_parser.rs +++ b/src/detections/rule/correlation_parser.rs @@ -4,12 +4,12 @@ use yaml_rust::Yaml; use crate::detections::configs::StoredStatic; use crate::detections::message::{AlertMessage, ERROR_LOG_STACK}; -use crate::detections::rule::{DetectionNode, RuleNode}; use crate::detections::rule::aggregation_parser::{ AggregationConditionToken, AggregationParseInfo, }; use crate::detections::rule::count::TimeFrameInfo; use crate::detections::rule::selectionnodes::OrSelectionNode; +use crate::detections::rule::{DetectionNode, RuleNode}; fn is_related_rule(rule_node: &RuleNode, id_or_title: &str) -> bool { if let Some(hash) = rule_node.yaml.as_hash() { From f1a170656334fc5dd137c74c58e6b9674f745d2d Mon Sep 17 00:00:00 2001 From: Yamato Security <71482215+YamatoSecurity@users.noreply.github.com> Date: Thu, 6 Jun 2024 06:39:42 +0900 Subject: [PATCH 5/5] update changelog --- CHANGELOG-Japanese.md | 1 + CHANGELOG.md | 1 + 2 files changed, 2 insertions(+) diff --git a/CHANGELOG-Japanese.md b/CHANGELOG-Japanese.md index 6a4edfc21..eeedb52ee 100644 --- a/CHANGELOG-Japanese.md +++ b/CHANGELOG-Japanese.md @@ -10,6 +10,7 @@ - [Mimikatz Use](https://github.com/SigmaHQ/sigma/blob/master/rules/windows/builtin/win_alert_mimikatz_keywords.yml) - デフォルトでは、適用可能なルールを持つ`.evtx`ファイルのみ読み込む。たとえば、さまざまなイベントログのディレクトリをスキャンしている場合でも、 `Channel: Security` を探すルールのみを有効にした場合、Hayabusaは`Security`以外のすべてのイベントログを無視します。ベンチマークでは、通常のスキャンで約10％、単一のルールでスキャンする場合は最大60％以上のパフォーマンス向上が得られる。チャネルに関係なくすべての`.evtx`ファイルを読み込みたい場合は、`csv-timeline` と `json-timeline` の `-a、--scan-all-evtx-files` オプションでこのフィルタリングをオフにすることができる。(#1318) (@fukusuket) - 注意: チャンネルフィルタリングは .evtx ファイルにのみ適用され、`-J, --json-input`オプションを使用してイベントログをJSONファイルから読み込む際に`-A`または`-a`を指定するとエラーが発生する。(#1345) (@fukusuket) +- Sigma CorrelationのEvent Countに対応した。 (#1337) (@fukusuket) **改善:** diff --git a/CHANGELOG.md b/CHANGELOG.md index 6effd323d..71efd3f03 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ - [Mimikatz Use](https://github.com/SigmaHQ/sigma/blob/master/rules/windows/builtin/win_alert_mimikatz_keywords.yml) - By default now, `.evtx` files that have applicable rules will be loaded. So for example, if you are scanning a directory of various event logs but only enable a rule that is looking for `Channel: Security` then Hayabusa will ignore all non-security event logs. In our benchmarks, this gives a speed benefit of around 10% with normal scans and up to 60%+ performance increase when scanning with a single rule. If you want to load all `.evtx` files regardless of channel, then you can turn off this filtering with the `-a, --scan-all-evtx-files` option in `csv-timeline` and `json-timeline`. (#1318) (@fukusuket) - Note: Channel filtering only works with .evtx files and you will receive an error if you try to load event logs from a JSON file with `-J, --json-input` and also specify `-A` or `-a`. (#1345) (@fukusuket) +- Support for Sigma Correlation's Event Count. (#1337) (@fukusuket) **Enhancements:**