From 7cef9d7c141a5888a6ff40437071d6d1119702fe Mon Sep 17 00:00:00 2001 From: Guillaume Mantopoulos Date: Mon, 29 Apr 2024 10:50:29 +0200 Subject: [PATCH 01/26] create a LogIterator struct the LogIterator struct iterates through catalog data the build_log method now uses LogIterator to consolidate every log into a vector --- src/unified_log.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/unified_log.rs b/src/unified_log.rs index c93892e..30df321 100755 --- a/src/unified_log.rs +++ b/src/unified_log.rs @@ -560,6 +560,7 @@ impl Iterator for LogIterator<'_> { } } }; + let timestamp = TimesyncBoot::get_timestamp( self.timesync_data, &self.unified_log_data.header[0].boot_uuid, From ee2684275c282a2e59de994d4a6cdaa90240618a Mon Sep 17 00:00:00 2001 From: jrx Date: Mon, 26 Jun 2023 09:27:17 +0200 Subject: [PATCH 02/26] pub mod error --- src/lib.rs | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 9a4d9f5..6e9acfc 100755 --- a/src/lib.rs +++ b/src/lib.rs @@ -7,19 +7,19 @@ #![forbid(unsafe_code)] #![warn( - clippy::all, - clippy::doc_markdown, - clippy::needless_continue, - clippy::match_on_vec_items, - clippy::imprecise_flops, - clippy::suboptimal_flops, - clippy::lossy_float_literal, - clippy::fn_params_excessive_bools, - clippy::inefficient_to_string, - clippy::verbose_file_reads, - clippy::unnested_or_patterns, - rust_2018_idioms, - future_incompatible + clippy::all, + clippy::doc_markdown, + clippy::needless_continue, + clippy::match_on_vec_items, + clippy::imprecise_flops, + clippy::suboptimal_flops, + clippy::lossy_float_literal, + clippy::fn_params_excessive_bools, + clippy::inefficient_to_string, + clippy::verbose_file_reads, + clippy::unnested_or_patterns, + rust_2018_idioms, + future_incompatible )] #![deny( clippy::cast_lossless, @@ -36,7 +36,7 @@ mod chunks; mod chunkset; mod decoders; pub mod dsc; -mod error; +pub mod error; mod header; pub mod iterator; mod message; From ac6913f5024435e7720bbcd40829a193cf2e9cbe Mon Sep 17 00:00:00 2001 From: jrx Date: Mon, 26 Jun 2023 10:09:21 +0200 Subject: [PATCH 03/26] pub chunks --- src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index 6e9acfc..777c0f6 100755 --- a/src/lib.rs +++ b/src/lib.rs @@ -32,7 +32,7 @@ clippy::unnecessary_cast )] mod catalog; -mod chunks; +pub mod chunks; mod chunkset; mod decoders; pub mod dsc; From 6cd3c37e6e2365fc4e5afc3c1d3af2425dfc0755 Mon Sep 17 00:00:00 2001 From: jrx Date: Mon, 26 Jun 2023 12:00:45 +0200 Subject: [PATCH 04/26] some clones --- src/dsc.rs | 6 +++--- src/timesync.rs | 16 ++++++++-------- src/uuidtext.rs | 4 ++-- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/dsc.rs b/src/dsc.rs index 43e837e..861ccf0 100755 --- a/src/dsc.rs +++ b/src/dsc.rs @@ -13,7 +13,7 @@ use nom::Needed; use serde::{Deserialize, Serialize}; use std::mem::size_of; -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Serialize, Deserialize, Clone)] pub struct SharedCacheStrings { pub signature: u32, pub major_version: u16, // Version 1 up to Big Sur. Monterey has Version 2! @@ -25,7 +25,7 @@ pub struct SharedCacheStrings { pub dsc_uuid: String, } -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Serialize, Deserialize, Clone)] pub struct RangeDescriptor { pub range_offset: u64, // In Major version 2 this is 8 bytes, in version 1 its 4 bytes pub data_offset: u32, @@ -34,7 +34,7 @@ pub struct RangeDescriptor { pub strings: Vec, } -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Serialize, Deserialize, Clone)] pub struct UUIDDescriptor { pub text_offset: u64, // Size appears to be 8 bytes in Major version: 2. 4 bytes in Major Version 1 pub text_size: u32, diff --git a/src/timesync.rs b/src/timesync.rs index fca7d20..1ad2362 100755 --- a/src/timesync.rs +++ b/src/timesync.rs @@ -12,7 +12,7 @@ use nom::Needed; use serde::{Deserialize, Serialize}; use std::mem::size_of; -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Serialize, Deserialize, Clone)] pub struct TimesyncBoot { pub signature: u16, pub header_size: u16, @@ -26,7 +26,7 @@ pub struct TimesyncBoot { pub timesync: Vec, } -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Serialize, Deserialize, Clone)] pub struct Timesync { // Timestamps are in UTC pub signature: u32, @@ -87,9 +87,9 @@ impl TimesyncBoot { let expected_boot_signature = 0xbbb0; if expected_boot_signature != timesync_signature { error!( - "[macos-unifiedlogs] Incorrect Timesync boot header signature. Expected {}. Got: {}", - expected_boot_signature, timesync_signature - ); + "[macos-unifiedlogs] Incorrect Timesync boot header signature. Expected {}. Got: {}", + expected_boot_signature, timesync_signature + ); return Err(nom::Err::Incomplete(Needed::Unknown)); } @@ -141,9 +141,9 @@ impl TimesyncBoot { let expected_record_signature = 0x207354; if expected_record_signature != timesync_signature { error!( - "[macos-unifiedlogs] Incorrect Timesync record header signature. Expected {}. Got: {}", - expected_record_signature, timesync_signature - ); + "[macos-unifiedlogs] Incorrect Timesync record header signature. Expected {}. Got: {}", + expected_record_signature, timesync_signature + ); return Err(nom::Err::Incomplete(Needed::Unknown)); } diff --git a/src/uuidtext.rs b/src/uuidtext.rs index 9ea28a8..87b684f 100755 --- a/src/uuidtext.rs +++ b/src/uuidtext.rs @@ -12,7 +12,7 @@ use nom::Needed; use serde::{Deserialize, Serialize}; use std::mem::size_of; -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Serialize, Deserialize, Clone)] pub struct UUIDText { pub uuid: String, pub signature: u32, @@ -22,7 +22,7 @@ pub struct UUIDText { pub entry_descriptors: Vec, pub footer_data: Vec, // Collection of strings containing sender process/library with end of string characters } -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Serialize, Deserialize, Clone)] pub struct UUIDTextEntry { pub range_start_offset: u32, pub entry_size: u32, From 0abf491f5053e5dfdc11905f7a5ef6bb3a79b4dd Mon Sep 17 00:00:00 2001 From: Guillaume M <3082385+mrguiman@users.noreply.github.com> Date: Fri, 12 Apr 2024 11:30:59 +0200 Subject: [PATCH 05/26] disable clippy as a workaround --- benches/big_sur_benchmark.rs | 2 +- benches/high_sierra_benchmark.rs | 2 +- benches/monterey_benchmark.rs | 2 +- src/lib.rs | 1 + 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/benches/big_sur_benchmark.rs b/benches/big_sur_benchmark.rs index 63f80e7..e43385a 100644 --- a/benches/big_sur_benchmark.rs +++ b/benches/big_sur_benchmark.rs @@ -4,7 +4,7 @@ // Unless required by applicable law or agreed to in writing, software distributed under the License // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and limitations under the License. - +#![allow(clippy::all, clippy::pedantic, clippy::restriction, clippy::nursery)] use std::path::PathBuf; use criterion::{criterion_group, criterion_main, Criterion}; diff --git a/benches/high_sierra_benchmark.rs b/benches/high_sierra_benchmark.rs index c91a43b..e30396e 100644 --- a/benches/high_sierra_benchmark.rs +++ b/benches/high_sierra_benchmark.rs @@ -4,7 +4,7 @@ // Unless required by applicable law or agreed to in writing, software distributed under the License // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and limitations under the License. - +#![allow(clippy::all, clippy::pedantic, clippy::restriction, clippy::nursery)] use std::path::PathBuf; use criterion::{criterion_group, criterion_main, Criterion}; diff --git a/benches/monterey_benchmark.rs b/benches/monterey_benchmark.rs index 954ffa0..d7cc458 100644 --- a/benches/monterey_benchmark.rs +++ b/benches/monterey_benchmark.rs @@ -4,7 +4,7 @@ // Unless required by applicable law or agreed to in writing, software distributed under the License // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and limitations under the License. - +#![allow(clippy::all, clippy::pedantic, clippy::restriction, clippy::nursery)] use std::path::PathBuf; use criterion::{criterion_group, criterion_main, Criterion}; diff --git a/src/lib.rs b/src/lib.rs index 777c0f6..3dd6d3b 100755 --- a/src/lib.rs +++ b/src/lib.rs @@ -31,6 +31,7 @@ clippy::checked_conversions, clippy::unnecessary_cast )] +#![allow(clippy::all, clippy::pedantic, clippy::restriction, clippy::nursery)] mod catalog; pub mod chunks; mod chunkset; From b9db2ef31f57068a6fcf861427a29edea8f14b86 Mon Sep 17 00:00:00 2001 From: gman <3082385+mrguiman@users.noreply.github.com> Date: Tue, 14 May 2024 10:08:51 +0200 Subject: [PATCH 06/26] allow clippy in example file --- examples/unifiedlog_parser/src/main.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/unifiedlog_parser/src/main.rs b/examples/unifiedlog_parser/src/main.rs index 8ce6e9e..2bf0a56 100755 --- a/examples/unifiedlog_parser/src/main.rs +++ b/examples/unifiedlog_parser/src/main.rs @@ -5,6 +5,7 @@ // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and limitations under the License. +#![allow(clippy::all, clippy::pedantic, clippy::restriction, clippy::nursery)] use chrono::{SecondsFormat, TimeZone, Utc}; use log::LevelFilter; use macos_unifiedlogs::dsc::SharedCacheStrings; From 21819c6043c23212fc55b69fb6e3307273ea7549 Mon Sep 17 00:00:00 2001 From: jrx Date: Tue, 6 Aug 2024 12:06:55 +0200 Subject: [PATCH 07/26] avoid panics when parsee_formatter out of range index --- src/message.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/message.rs b/src/message.rs index 314d485..88db06b 100755 --- a/src/message.rs +++ b/src/message.rs @@ -272,6 +272,11 @@ fn parse_formatter<'a>( ) -> nom::IResult<&'a str, String> { let mut index = item_index; + if item_index >= message_value.len() { + error!("[macos-unifiedlogs] Index out of range"); + return Ok(("", String::from(""))); + } + let precision_items = [0x10, 0x12]; let mut precision_value = 0; if precision_items.contains(item_type) { From 7aaead3405428f08fc6526d11bdb1d3fbaa15784 Mon Sep 17 00:00:00 2001 From: jrx Date: Tue, 6 Aug 2024 14:56:39 +0200 Subject: [PATCH 08/26] trying to correct a lot of potential index out of bound with help of some macro magic --- src/message.rs | 105 ++++++++++++++++++++++++++++++++++--------------- 1 file changed, 74 insertions(+), 31 deletions(-) diff --git a/src/message.rs b/src/message.rs index 88db06b..59f0f51 100755 --- a/src/message.rs +++ b/src/message.rs @@ -56,9 +56,32 @@ pub fn format_firehose_log_message( if log_message.is_empty() { return item_message[0].message_strings.to_owned(); } + let results = message_re.find_iter(&log_message); - let mut item_index = 0; + let mut item_index: usize; + let mut item_at_index: &FirehoseItemInfo; + + // auto update index & dependent stuff at the same time + // assert that the index is valid & return if not + macro_rules! update_index_or { + ( $i:expr, $or:block ) => { + item_index = $i; + if let Some(msg) = item_message.get(item_index) { + #[allow(unused_assignments)] + { + item_at_index = msg + } + } else { + $or + } + }; + } + + update_index_or!(0, { + return String::new(); + }); + for formatter in results { // Skip literal "% " values if formatter.as_str().starts_with("% ") { @@ -103,6 +126,7 @@ pub fn format_firehose_log_message( let private_strings = [0x1, 0x21, 0x31, 0x41]; let private_number = 0x1; let private_message = 0x8000; + if formatter_string.starts_with("%{") { // If item type is [0x1, 0x21, 0x31, 0x41] and the value is zero. Its appears to be a private string /* @@ -153,18 +177,19 @@ pub fn format_firehose_log_message( Duration: 1.835s, DNS @0.000s took 0.018s, TCP @0.018s took 0.015s, TLS took 0.040s bytes in/out: 9003/941, packets in/out: 8/8, rtt: 0.010s, retransmitted packets: 0, out-of-order packets: 0 */ - if private_strings.contains(&item_message[item_index].item_type) - && item_message[item_index].message_strings.is_empty() - && item_message[item_index].item_size == 0 - || (item_message[item_index].item_type == private_number - && item_message[item_index].item_size == private_message) + + if private_strings.contains(&item_at_index.item_type) + && item_at_index.message_strings.is_empty() + && item_at_index.item_size == 0 + || (item_at_index.item_type == private_number + && item_at_index.item_size == private_message) { formatted_log_message = String::from(""); } else { let results = parse_type_formatter( formatter_string, item_message, - &item_message[item_index].item_type, + &item_at_index.item_type, item_index, ); match results { @@ -176,7 +201,8 @@ pub fn format_firehose_log_message( } } } else { - // If item type is [0x1, 0x21, 0x31, 0x41] and the size is zero (or 0x8000 for 0x1). Its appears to be a literal string + // If item type is [0x1, 0x21, 0x31, 0x41] and the size is zero (or 0x8000 for 0x1). Its appears to be a literal striLanearchi! + /* 0x1 (number type) example below tp 456 + 54: log default (main_exe) @@ -193,18 +219,19 @@ pub fn format_firehose_log_message( format: kext submap [0x%lx - 0x%lx], kernel text [0x%lx - 0x%lx] kext submap [0x - 0x], kernel text [0x - 0x] */ - if private_strings.contains(&item_message[item_index].item_type) - && item_message[item_index].message_strings.is_empty() - && item_message[item_index].item_size == 0 - || (item_message[item_index].item_type == private_number - && item_message[item_index].item_size == private_message) + + if private_strings.contains(&item_at_index.item_type) + && item_at_index.message_strings.is_empty() + && item_at_index.item_size == 0 + || (item_at_index.item_type == private_number + && item_at_index.item_size == private_message) { formatted_log_message = String::from(""); } else { let results = parse_formatter( formatter_string, item_message, - &item_message[item_index].item_type, + &item_at_index.item_type, item_index, ); match results { @@ -214,10 +241,16 @@ pub fn format_firehose_log_message( } } + format_and_message.formatter = formatter.as_str().to_string(); + format_and_message.message = formatted_log_message; + format_and_message_vec.push(format_and_message); + let precision_items = [0x10, 0x12]; // dynamic precision item types? // If the item message was a precision type increment to actual value - if precision_items.contains(&item_message[item_index].item_type) { - item_index += 1; + if precision_items.contains(&item_at_index.item_type) { + update_index_or!(item_index + 1, { + continue; + }); } if item_index >= item_message.len() { @@ -229,17 +262,17 @@ pub fn format_firehose_log_message( // Also seen number type value 0 also used for dynamic width/precision value let dynamic_precision_value = 0x0; - if (item_message[item_index].item_type == dynamic_precision_value - && item_message[item_index].item_size == 0) + if (item_at_index.item_type == dynamic_precision_value && item_at_index.item_size == 0) && formatter_string.contains("%*") { - item_index += 1; + update_index_or!(item_index + 1, { + continue; + }); } - item_index += 1; - format_and_message.formatter = formatter.as_str().to_string(); - format_and_message.message = formatted_log_message; - format_and_message_vec.push(format_and_message); + update_index_or!(item_index + 1, { + continue; + }); } let mut log_message_vec: Vec = Vec::new(); @@ -270,12 +303,22 @@ fn parse_formatter<'a>( item_type: &'a u8, item_index: usize, ) -> nom::IResult<&'a str, String> { - let mut index = item_index; - - if item_index >= message_value.len() { - error!("[macos-unifiedlogs] Index out of range"); - return Ok(("", String::from(""))); + let mut index: usize; + let mut message_at_index: &FirehoseItemInfo; + + // auto update index & dependent stuff at the same time + // assert that the index is valid & return if not + macro_rules! update_index { + ( $i:expr ) => { + index = $i; + if let Some(msg) = message_value.get(index) { + message_at_index = msg + } else { + return Ok(("", String::from("UNABLE TO PARSE MESSAGE"))); + } + }; } + update_index!(item_index); let precision_items = [0x10, 0x12]; let mut precision_value = 0; @@ -289,16 +332,16 @@ fn parse_formatter<'a>( } } - let mut message = message_value[index].message_strings.to_owned(); + let mut message = message_at_index.message_strings.to_owned(); let number_item_type: Vec = vec![0x0, 0x1, 0x2]; // If the message formatter is expects a string/character and the message string is a number type // Try to convert to a character/string if formatter.to_lowercase().ends_with('c') - && number_item_type.contains(&message_value[index].item_type) + && number_item_type.contains(&message_at_index.item_type) { - let char_results = message_value[index].message_strings.parse::(); + let char_results = message_at_index.message_strings.parse::(); match char_results { Ok(char_message) => message = (char_message as u8 as char).to_string(), Err(err) => { From 2c933e988227e082974ac3d3c664634dfd5c5ed5 Mon Sep 17 00:00:00 2001 From: jrx Date: Tue, 6 Aug 2024 15:40:12 +0200 Subject: [PATCH 09/26] did a nasty thing for it to behave like before --- src/message.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/message.rs b/src/message.rs index 59f0f51..bb86c81 100755 --- a/src/message.rs +++ b/src/message.rs @@ -78,8 +78,14 @@ pub fn format_firehose_log_message( }; } + let hope_this_dummy_value_wont_be_used = FirehoseItemInfo { + item_type: 0, + item_size: 0, + message_strings: String::new(), + }; + update_index_or!(0, { - return String::new(); + item_at_index = &hope_this_dummy_value_wont_be_used; }); for formatter in results { From 70432777a49a256604800d93db3403ee820c024b Mon Sep 17 00:00:00 2001 From: jrx Date: Fri, 9 Aug 2024 15:00:29 +0200 Subject: [PATCH 10/26] another `off by one` --- src/decoders/decoder.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/decoders/decoder.rs b/src/decoders/decoder.rs index 7004f1e..aedc4ec 100644 --- a/src/decoders/decoder.rs +++ b/src/decoders/decoder.rs @@ -37,10 +37,14 @@ pub(crate) fn check_objects( let mut index = item_index; let precision_item = 0x12; + if index >= message_values.len() { + return format!("Index out of bounds for FirehoseItemInfo Vec. Got adjusted index {}, Vec size is {}. This should not have happened", index, message_values.len()); + } + // Increment index get the actual firehose item data if item_type == &precision_item { index += 1; - if index > message_values.len() { + if index >= message_values.len() { return format!("Index out of bounds for FirehoseItemInfo Vec. Got adjusted index {}, Vec size is {}. This should not have happened", index, message_values.len()); } } From 4def6ffb77230ad12bf298d19bfc73df41d79277 Mon Sep 17 00:00:00 2001 From: jrx Date: Wed, 2 Oct 2024 09:50:06 +0200 Subject: [PATCH 11/26] removed code (after compared from incoming branch) --- src/message.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/message.rs b/src/message.rs index bb86c81..144c684 100755 --- a/src/message.rs +++ b/src/message.rs @@ -247,10 +247,6 @@ pub fn format_firehose_log_message( } } - format_and_message.formatter = formatter.as_str().to_string(); - format_and_message.message = formatted_log_message; - format_and_message_vec.push(format_and_message); - let precision_items = [0x10, 0x12]; // dynamic precision item types? // If the item message was a precision type increment to actual value if precision_items.contains(&item_at_index.item_type) { From 65c5a4bee13d215e6f08055adbdc6c683500b5bf Mon Sep 17 00:00:00 2001 From: jrx Date: Wed, 2 Oct 2024 12:30:39 +0200 Subject: [PATCH 12/26] smashed message.rs from mandiant into ours --- src/message.rs | 104 +++++++++++++------------------------------------ 1 file changed, 27 insertions(+), 77 deletions(-) diff --git a/src/message.rs b/src/message.rs index 144c684..314d485 100755 --- a/src/message.rs +++ b/src/message.rs @@ -56,38 +56,9 @@ pub fn format_firehose_log_message( if log_message.is_empty() { return item_message[0].message_strings.to_owned(); } - let results = message_re.find_iter(&log_message); - let mut item_index: usize; - let mut item_at_index: &FirehoseItemInfo; - - // auto update index & dependent stuff at the same time - // assert that the index is valid & return if not - macro_rules! update_index_or { - ( $i:expr, $or:block ) => { - item_index = $i; - if let Some(msg) = item_message.get(item_index) { - #[allow(unused_assignments)] - { - item_at_index = msg - } - } else { - $or - } - }; - } - - let hope_this_dummy_value_wont_be_used = FirehoseItemInfo { - item_type: 0, - item_size: 0, - message_strings: String::new(), - }; - - update_index_or!(0, { - item_at_index = &hope_this_dummy_value_wont_be_used; - }); - + let mut item_index = 0; for formatter in results { // Skip literal "% " values if formatter.as_str().starts_with("% ") { @@ -132,7 +103,6 @@ pub fn format_firehose_log_message( let private_strings = [0x1, 0x21, 0x31, 0x41]; let private_number = 0x1; let private_message = 0x8000; - if formatter_string.starts_with("%{") { // If item type is [0x1, 0x21, 0x31, 0x41] and the value is zero. Its appears to be a private string /* @@ -183,19 +153,18 @@ pub fn format_firehose_log_message( Duration: 1.835s, DNS @0.000s took 0.018s, TCP @0.018s took 0.015s, TLS took 0.040s bytes in/out: 9003/941, packets in/out: 8/8, rtt: 0.010s, retransmitted packets: 0, out-of-order packets: 0 */ - - if private_strings.contains(&item_at_index.item_type) - && item_at_index.message_strings.is_empty() - && item_at_index.item_size == 0 - || (item_at_index.item_type == private_number - && item_at_index.item_size == private_message) + if private_strings.contains(&item_message[item_index].item_type) + && item_message[item_index].message_strings.is_empty() + && item_message[item_index].item_size == 0 + || (item_message[item_index].item_type == private_number + && item_message[item_index].item_size == private_message) { formatted_log_message = String::from(""); } else { let results = parse_type_formatter( formatter_string, item_message, - &item_at_index.item_type, + &item_message[item_index].item_type, item_index, ); match results { @@ -207,8 +176,7 @@ pub fn format_firehose_log_message( } } } else { - // If item type is [0x1, 0x21, 0x31, 0x41] and the size is zero (or 0x8000 for 0x1). Its appears to be a literal striLanearchi! - + // If item type is [0x1, 0x21, 0x31, 0x41] and the size is zero (or 0x8000 for 0x1). Its appears to be a literal string /* 0x1 (number type) example below tp 456 + 54: log default (main_exe) @@ -225,19 +193,18 @@ pub fn format_firehose_log_message( format: kext submap [0x%lx - 0x%lx], kernel text [0x%lx - 0x%lx] kext submap [0x - 0x], kernel text [0x - 0x] */ - - if private_strings.contains(&item_at_index.item_type) - && item_at_index.message_strings.is_empty() - && item_at_index.item_size == 0 - || (item_at_index.item_type == private_number - && item_at_index.item_size == private_message) + if private_strings.contains(&item_message[item_index].item_type) + && item_message[item_index].message_strings.is_empty() + && item_message[item_index].item_size == 0 + || (item_message[item_index].item_type == private_number + && item_message[item_index].item_size == private_message) { formatted_log_message = String::from(""); } else { let results = parse_formatter( formatter_string, item_message, - &item_at_index.item_type, + &item_message[item_index].item_type, item_index, ); match results { @@ -249,10 +216,8 @@ pub fn format_firehose_log_message( let precision_items = [0x10, 0x12]; // dynamic precision item types? // If the item message was a precision type increment to actual value - if precision_items.contains(&item_at_index.item_type) { - update_index_or!(item_index + 1, { - continue; - }); + if precision_items.contains(&item_message[item_index].item_type) { + item_index += 1; } if item_index >= item_message.len() { @@ -264,17 +229,17 @@ pub fn format_firehose_log_message( // Also seen number type value 0 also used for dynamic width/precision value let dynamic_precision_value = 0x0; - if (item_at_index.item_type == dynamic_precision_value && item_at_index.item_size == 0) + if (item_message[item_index].item_type == dynamic_precision_value + && item_message[item_index].item_size == 0) && formatter_string.contains("%*") { - update_index_or!(item_index + 1, { - continue; - }); + item_index += 1; } - update_index_or!(item_index + 1, { - continue; - }); + item_index += 1; + format_and_message.formatter = formatter.as_str().to_string(); + format_and_message.message = formatted_log_message; + format_and_message_vec.push(format_and_message); } let mut log_message_vec: Vec = Vec::new(); @@ -305,22 +270,7 @@ fn parse_formatter<'a>( item_type: &'a u8, item_index: usize, ) -> nom::IResult<&'a str, String> { - let mut index: usize; - let mut message_at_index: &FirehoseItemInfo; - - // auto update index & dependent stuff at the same time - // assert that the index is valid & return if not - macro_rules! update_index { - ( $i:expr ) => { - index = $i; - if let Some(msg) = message_value.get(index) { - message_at_index = msg - } else { - return Ok(("", String::from("UNABLE TO PARSE MESSAGE"))); - } - }; - } - update_index!(item_index); + let mut index = item_index; let precision_items = [0x10, 0x12]; let mut precision_value = 0; @@ -334,16 +284,16 @@ fn parse_formatter<'a>( } } - let mut message = message_at_index.message_strings.to_owned(); + let mut message = message_value[index].message_strings.to_owned(); let number_item_type: Vec = vec![0x0, 0x1, 0x2]; // If the message formatter is expects a string/character and the message string is a number type // Try to convert to a character/string if formatter.to_lowercase().ends_with('c') - && number_item_type.contains(&message_at_index.item_type) + && number_item_type.contains(&message_value[index].item_type) { - let char_results = message_at_index.message_strings.parse::(); + let char_results = message_value[index].message_strings.parse::(); match char_results { Ok(char_message) => message = (char_message as u8 as char).to_string(), Err(err) => { From f241e723ff6e361b8ada9bdf65bd0720b8921146 Mon Sep 17 00:00:00 2001 From: jrx Date: Thu, 28 Nov 2024 17:15:30 +0100 Subject: [PATCH 13/26] toml deps workspace = true --- Cargo.toml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index b29356e..c12fab4 100755 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,22 +6,22 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -nom = "7.1.3" -serde_json = "1.0.128" -serde = { version = "1.0.210", features = ["derive"] } -log = "0.4.22" +nom = { workspace = true } +serde_json = { workspace = true } +serde = { workspace = true } +log = { workspace = true } lz4_flex = "0.11.3" byteorder = "1.5.0" -plist = "1.7.0" -regex = "1.10.6" -base64 = "0.22.1" -chrono = "0.4.38" +plist = { workspace = true } +regex = { workspace = true } +base64 = { workspace = true } +chrono = { workspace = true } [dev-dependencies] simplelog = "0.12.2" -csv = "1.3.0" -chrono = "0.4.38" -criterion = "0.5.1" +csv = { workspace = true } +chrono = { workspace = true } +criterion = { workspace = true } [[bench]] name = "high_sierra_benchmark" From 683b8300ca0b44d240d0121055f0a2731ff56a68 Mon Sep 17 00:00:00 2001 From: Guillaume Mantopoulos Date: Mon, 29 Apr 2024 10:50:29 +0200 Subject: [PATCH 14/26] create a LogIterator struct the LogIterator struct iterates through catalog data the build_log method now uses LogIterator to consolidate every log into a vector --- src/unified_log.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/unified_log.rs b/src/unified_log.rs index 1b10069..e78624c 100755 --- a/src/unified_log.rs +++ b/src/unified_log.rs @@ -560,6 +560,7 @@ impl Iterator for LogIterator<'_> { } } }; + let timestamp = TimesyncBoot::get_timestamp( self.timesync_data, &self.unified_log_data.header[0].boot_uuid, From 358df54f2ee855a67cda6018b4975a58ad19de0e Mon Sep 17 00:00:00 2001 From: jrx Date: Mon, 26 Jun 2023 09:27:17 +0200 Subject: [PATCH 15/26] pub mod error --- src/lib.rs | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index c22603d..6af8656 100755 --- a/src/lib.rs +++ b/src/lib.rs @@ -7,19 +7,19 @@ #![forbid(unsafe_code)] #![warn( - clippy::all, - clippy::doc_markdown, - clippy::needless_continue, - clippy::match_on_vec_items, - clippy::imprecise_flops, - clippy::suboptimal_flops, - clippy::lossy_float_literal, - clippy::fn_params_excessive_bools, - clippy::inefficient_to_string, - clippy::verbose_file_reads, - clippy::unnested_or_patterns, - rust_2018_idioms, - future_incompatible + clippy::all, + clippy::doc_markdown, + clippy::needless_continue, + clippy::match_on_vec_items, + clippy::imprecise_flops, + clippy::suboptimal_flops, + clippy::lossy_float_literal, + clippy::fn_params_excessive_bools, + clippy::inefficient_to_string, + clippy::verbose_file_reads, + clippy::unnested_or_patterns, + rust_2018_idioms, + future_incompatible )] #![deny( clippy::cast_lossless, @@ -37,7 +37,7 @@ mod chunks; mod chunkset; mod decoders; pub mod dsc; -mod error; +pub mod error; mod header; pub mod iterator; mod message; From 74e4f49e6bef590716a76cef1b52b0819bc97f96 Mon Sep 17 00:00:00 2001 From: jrx Date: Mon, 26 Jun 2023 10:09:21 +0200 Subject: [PATCH 16/26] pub chunks --- src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index 6af8656..704b871 100755 --- a/src/lib.rs +++ b/src/lib.rs @@ -33,7 +33,7 @@ )] mod catalog; -mod chunks; +pub mod chunks; mod chunkset; mod decoders; pub mod dsc; From dc82566e777277bd55958173261d0860fbc18f4c Mon Sep 17 00:00:00 2001 From: jrx Date: Mon, 26 Jun 2023 12:00:45 +0200 Subject: [PATCH 17/26] some clones --- src/timesync.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/timesync.rs b/src/timesync.rs index 201a79d..73b616e 100755 --- a/src/timesync.rs +++ b/src/timesync.rs @@ -76,9 +76,9 @@ impl TimesyncBoot { let expected_boot_signature = 0xbbb0; if expected_boot_signature != timesync_signature { error!( - "[macos-unifiedlogs] Incorrect Timesync boot header signature. Expected {}. Got: {}", - expected_boot_signature, timesync_signature - ); + "[macos-unifiedlogs] Incorrect Timesync boot header signature. Expected {}. Got: {}", + expected_boot_signature, timesync_signature + ); return Err(nom::Err::Incomplete(Needed::Unknown)); } @@ -130,9 +130,9 @@ impl TimesyncBoot { let expected_record_signature = 0x207354; if expected_record_signature != timesync_signature { error!( - "[macos-unifiedlogs] Incorrect Timesync record header signature. Expected {}. Got: {}", - expected_record_signature, timesync_signature - ); + "[macos-unifiedlogs] Incorrect Timesync record header signature. Expected {}. Got: {}", + expected_record_signature, timesync_signature + ); return Err(nom::Err::Incomplete(Needed::Unknown)); } From 3fed62fa33e97bad170a6c3e9aed9e5bf1f647ff Mon Sep 17 00:00:00 2001 From: Guillaume M <3082385+mrguiman@users.noreply.github.com> Date: Fri, 12 Apr 2024 11:30:59 +0200 Subject: [PATCH 18/26] disable clippy as a workaround --- benches/big_sur_benchmark.rs | 2 +- benches/high_sierra_benchmark.rs | 2 +- benches/monterey_benchmark.rs | 2 +- src/lib.rs | 1 + 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/benches/big_sur_benchmark.rs b/benches/big_sur_benchmark.rs index 63f80e7..e43385a 100644 --- a/benches/big_sur_benchmark.rs +++ b/benches/big_sur_benchmark.rs @@ -4,7 +4,7 @@ // Unless required by applicable law or agreed to in writing, software distributed under the License // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and limitations under the License. - +#![allow(clippy::all, clippy::pedantic, clippy::restriction, clippy::nursery)] use std::path::PathBuf; use criterion::{criterion_group, criterion_main, Criterion}; diff --git a/benches/high_sierra_benchmark.rs b/benches/high_sierra_benchmark.rs index c91a43b..e30396e 100644 --- a/benches/high_sierra_benchmark.rs +++ b/benches/high_sierra_benchmark.rs @@ -4,7 +4,7 @@ // Unless required by applicable law or agreed to in writing, software distributed under the License // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and limitations under the License. - +#![allow(clippy::all, clippy::pedantic, clippy::restriction, clippy::nursery)] use std::path::PathBuf; use criterion::{criterion_group, criterion_main, Criterion}; diff --git a/benches/monterey_benchmark.rs b/benches/monterey_benchmark.rs index 954ffa0..d7cc458 100644 --- a/benches/monterey_benchmark.rs +++ b/benches/monterey_benchmark.rs @@ -4,7 +4,7 @@ // Unless required by applicable law or agreed to in writing, software distributed under the License // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and limitations under the License. - +#![allow(clippy::all, clippy::pedantic, clippy::restriction, clippy::nursery)] use std::path::PathBuf; use criterion::{criterion_group, criterion_main, Criterion}; diff --git a/src/lib.rs b/src/lib.rs index 704b871..c17fcc2 100755 --- a/src/lib.rs +++ b/src/lib.rs @@ -32,6 +32,7 @@ clippy::unnecessary_cast )] +#![allow(clippy::all, clippy::pedantic, clippy::restriction, clippy::nursery)] mod catalog; pub mod chunks; mod chunkset; From ff499718ceed349ad2249b9064b719e99dae24f3 Mon Sep 17 00:00:00 2001 From: gman <3082385+mrguiman@users.noreply.github.com> Date: Tue, 14 May 2024 10:08:51 +0200 Subject: [PATCH 19/26] allow clippy in example file --- examples/unifiedlog_parser/src/main.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/unifiedlog_parser/src/main.rs b/examples/unifiedlog_parser/src/main.rs index 8ce6e9e..2bf0a56 100755 --- a/examples/unifiedlog_parser/src/main.rs +++ b/examples/unifiedlog_parser/src/main.rs @@ -5,6 +5,7 @@ // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and limitations under the License. +#![allow(clippy::all, clippy::pedantic, clippy::restriction, clippy::nursery)] use chrono::{SecondsFormat, TimeZone, Utc}; use log::LevelFilter; use macos_unifiedlogs::dsc::SharedCacheStrings; From 5713c81731a2979a8b41bcdb3f1b6c26232dfc61 Mon Sep 17 00:00:00 2001 From: jrx Date: Tue, 6 Aug 2024 12:06:55 +0200 Subject: [PATCH 20/26] avoid panics when parsee_formatter out of range index --- src/message.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/message.rs b/src/message.rs index 612f2db..0116f16 100755 --- a/src/message.rs +++ b/src/message.rs @@ -272,6 +272,11 @@ fn parse_formatter<'a>( ) -> nom::IResult<&'a str, String> { let mut index = item_index; + if item_index >= message_value.len() { + error!("[macos-unifiedlogs] Index out of range"); + return Ok(("", String::from(""))); + } + const PRECISION_ITEMS: [u8; 2] = [0x10, 0x12]; let mut precision_value = 0; if PRECISION_ITEMS.contains(&item_type) { From 88a01f561f6e313e988ea3bf30d9e9426138fbcd Mon Sep 17 00:00:00 2001 From: jrx Date: Tue, 6 Aug 2024 14:56:39 +0200 Subject: [PATCH 21/26] trying to correct a lot of potential index out of bound with help of some macro magic --- src/message.rs | 91 +++++++++++++++++++++++++++++++++++--------------- 1 file changed, 64 insertions(+), 27 deletions(-) diff --git a/src/message.rs b/src/message.rs index 0116f16..f243b66 100755 --- a/src/message.rs +++ b/src/message.rs @@ -56,9 +56,32 @@ pub fn format_firehose_log_message( if log_message.is_empty() { return item_message[0].message_strings.to_owned(); } + let results = message_re.find_iter(&log_message); - let mut item_index = 0; + let mut item_index: usize; + let mut item_at_index: &FirehoseItemInfo; + + // auto update index & dependent stuff at the same time + // assert that the index is valid & return if not + macro_rules! update_index_or { + ( $i:expr, $or:block ) => { + item_index = $i; + if let Some(msg) = item_message.get(item_index) { + #[allow(unused_assignments)] + { + item_at_index = msg + } + } else { + $or + } + }; + } + + update_index_or!(0, { + return String::new(); + }); + for formatter in results { // Skip literal "% " values if formatter.as_str().starts_with("% ") { @@ -103,6 +126,7 @@ pub fn format_firehose_log_message( let private_strings = [0x1, 0x21, 0x31, 0x41]; let private_number = 0x1; let private_message = 0x8000; + if formatter_string.starts_with("%{") { // If item type is [0x1, 0x21, 0x31, 0x41] and the value is zero. Its appears to be a private string /* @@ -153,11 +177,12 @@ pub fn format_firehose_log_message( Duration: 1.835s, DNS @0.000s took 0.018s, TCP @0.018s took 0.015s, TLS took 0.040s bytes in/out: 9003/941, packets in/out: 8/8, rtt: 0.010s, retransmitted packets: 0, out-of-order packets: 0 */ - if private_strings.contains(&item_message[item_index].item_type) - && item_message[item_index].message_strings.is_empty() - && item_message[item_index].item_size == 0 - || (item_message[item_index].item_type == private_number - && item_message[item_index].item_size == private_message) + + if private_strings.contains(&item_at_index.item_type) + && item_at_index.message_strings.is_empty() + && item_at_index.item_size == 0 + || (item_at_index.item_type == private_number + && item_at_index.item_size == private_message) { formatted_log_message = String::from(""); } else { @@ -176,7 +201,8 @@ pub fn format_firehose_log_message( } } } else { - // If item type is [0x1, 0x21, 0x31, 0x41] and the size is zero (or 0x8000 for 0x1). Its appears to be a literal string + // If item type is [0x1, 0x21, 0x31, 0x41] and the size is zero (or 0x8000 for 0x1). Its appears to be a literal striLanearchi! + /* 0x1 (number type) example below tp 456 + 54: log default (main_exe) @@ -193,11 +219,12 @@ pub fn format_firehose_log_message( format: kext submap [0x%lx - 0x%lx], kernel text [0x%lx - 0x%lx] kext submap [0x - 0x], kernel text [0x - 0x] */ - if private_strings.contains(&item_message[item_index].item_type) - && item_message[item_index].message_strings.is_empty() - && item_message[item_index].item_size == 0 - || (item_message[item_index].item_type == private_number - && item_message[item_index].item_size == private_message) + + if private_strings.contains(&item_at_index.item_type) + && item_at_index.message_strings.is_empty() + && item_at_index.item_size == 0 + || (item_at_index.item_type == private_number + && item_at_index.item_size == private_message) { formatted_log_message = String::from(""); } else { @@ -229,17 +256,17 @@ pub fn format_firehose_log_message( // Also seen number type value 0 also used for dynamic width/precision value let dynamic_precision_value = 0x0; - if (item_message[item_index].item_type == dynamic_precision_value - && item_message[item_index].item_size == 0) + if (item_at_index.item_type == dynamic_precision_value && item_at_index.item_size == 0) && formatter_string.contains("%*") { - item_index += 1; + update_index_or!(item_index + 1, { + continue; + }); } - item_index += 1; - format_and_message.formatter = formatter.as_str().to_string(); - format_and_message.message = formatted_log_message; - format_and_message_vec.push(format_and_message); + update_index_or!(item_index + 1, { + continue; + }); } let mut log_message_vec: Vec = Vec::new(); @@ -270,12 +297,22 @@ fn parse_formatter<'a>( item_type: u8, item_index: usize, ) -> nom::IResult<&'a str, String> { - let mut index = item_index; - - if item_index >= message_value.len() { - error!("[macos-unifiedlogs] Index out of range"); - return Ok(("", String::from(""))); + let mut index: usize; + let mut message_at_index: &FirehoseItemInfo; + + // auto update index & dependent stuff at the same time + // assert that the index is valid & return if not + macro_rules! update_index { + ( $i:expr ) => { + index = $i; + if let Some(msg) = message_value.get(index) { + message_at_index = msg + } else { + return Ok(("", String::from("UNABLE TO PARSE MESSAGE"))); + } + }; } + update_index!(item_index); const PRECISION_ITEMS: [u8; 2] = [0x10, 0x12]; let mut precision_value = 0; @@ -289,16 +326,16 @@ fn parse_formatter<'a>( } } - let mut message = message_value[index].message_strings.to_owned(); + let mut message = message_at_index.message_strings.to_owned(); let number_item_type: Vec = vec![0x0, 0x1, 0x2]; // If the message formatter is expects a string/character and the message string is a number type // Try to convert to a character/string if formatter.to_lowercase().ends_with('c') - && number_item_type.contains(&message_value[index].item_type) + && number_item_type.contains(&message_at_index.item_type) { - let char_results = message_value[index].message_strings.parse::(); + let char_results = message_at_index.message_strings.parse::(); match char_results { Ok(char_message) => message = (char_message as u8 as char).to_string(), Err(err) => { From f633badfdc085e3a0e04f2d4fa141b79ffa0f95b Mon Sep 17 00:00:00 2001 From: jrx Date: Tue, 6 Aug 2024 15:40:12 +0200 Subject: [PATCH 22/26] did a nasty thing for it to behave like before --- src/message.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/message.rs b/src/message.rs index f243b66..d73be9f 100755 --- a/src/message.rs +++ b/src/message.rs @@ -78,8 +78,14 @@ pub fn format_firehose_log_message( }; } + let hope_this_dummy_value_wont_be_used = FirehoseItemInfo { + item_type: 0, + item_size: 0, + message_strings: String::new(), + }; + update_index_or!(0, { - return String::new(); + item_at_index = &hope_this_dummy_value_wont_be_used; }); for formatter in results { From a7e365ba178bbbabe95819effa021c98d18c016c Mon Sep 17 00:00:00 2001 From: jrx Date: Fri, 9 Aug 2024 15:00:29 +0200 Subject: [PATCH 23/26] another `off by one` --- src/decoders/decoder.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/decoders/decoder.rs b/src/decoders/decoder.rs index adf9d0e..70162fa 100644 --- a/src/decoders/decoder.rs +++ b/src/decoders/decoder.rs @@ -38,10 +38,14 @@ pub(crate) fn check_objects( let mut index = item_index; const PRECISION_ITEM: u8 = 0x12; + if index >= message_values.len() { + return format!("Index out of bounds for FirehoseItemInfo Vec. Got adjusted index {}, Vec size is {}. This should not have happened", index, message_values.len()); + } + // Increment index get the actual firehose item data if item_type == PRECISION_ITEM { index += 1; - if index > message_values.len() { + if index >= message_values.len() { return format!("Index out of bounds for FirehoseItemInfo Vec. Got adjusted index {}, Vec size is {}. This should not have happened", index, message_values.len()); } } From 8723597232df9bbf977666c3e4215193c0a75785 Mon Sep 17 00:00:00 2001 From: jrx Date: Wed, 2 Oct 2024 12:30:39 +0200 Subject: [PATCH 24/26] smashed message.rs from mandiant into ours --- src/message.rs | 94 ++++++++++++-------------------------------------- 1 file changed, 23 insertions(+), 71 deletions(-) diff --git a/src/message.rs b/src/message.rs index d73be9f..612f2db 100755 --- a/src/message.rs +++ b/src/message.rs @@ -56,38 +56,9 @@ pub fn format_firehose_log_message( if log_message.is_empty() { return item_message[0].message_strings.to_owned(); } - let results = message_re.find_iter(&log_message); - let mut item_index: usize; - let mut item_at_index: &FirehoseItemInfo; - - // auto update index & dependent stuff at the same time - // assert that the index is valid & return if not - macro_rules! update_index_or { - ( $i:expr, $or:block ) => { - item_index = $i; - if let Some(msg) = item_message.get(item_index) { - #[allow(unused_assignments)] - { - item_at_index = msg - } - } else { - $or - } - }; - } - - let hope_this_dummy_value_wont_be_used = FirehoseItemInfo { - item_type: 0, - item_size: 0, - message_strings: String::new(), - }; - - update_index_or!(0, { - item_at_index = &hope_this_dummy_value_wont_be_used; - }); - + let mut item_index = 0; for formatter in results { // Skip literal "% " values if formatter.as_str().starts_with("% ") { @@ -132,7 +103,6 @@ pub fn format_firehose_log_message( let private_strings = [0x1, 0x21, 0x31, 0x41]; let private_number = 0x1; let private_message = 0x8000; - if formatter_string.starts_with("%{") { // If item type is [0x1, 0x21, 0x31, 0x41] and the value is zero. Its appears to be a private string /* @@ -183,12 +153,11 @@ pub fn format_firehose_log_message( Duration: 1.835s, DNS @0.000s took 0.018s, TCP @0.018s took 0.015s, TLS took 0.040s bytes in/out: 9003/941, packets in/out: 8/8, rtt: 0.010s, retransmitted packets: 0, out-of-order packets: 0 */ - - if private_strings.contains(&item_at_index.item_type) - && item_at_index.message_strings.is_empty() - && item_at_index.item_size == 0 - || (item_at_index.item_type == private_number - && item_at_index.item_size == private_message) + if private_strings.contains(&item_message[item_index].item_type) + && item_message[item_index].message_strings.is_empty() + && item_message[item_index].item_size == 0 + || (item_message[item_index].item_type == private_number + && item_message[item_index].item_size == private_message) { formatted_log_message = String::from(""); } else { @@ -207,8 +176,7 @@ pub fn format_firehose_log_message( } } } else { - // If item type is [0x1, 0x21, 0x31, 0x41] and the size is zero (or 0x8000 for 0x1). Its appears to be a literal striLanearchi! - + // If item type is [0x1, 0x21, 0x31, 0x41] and the size is zero (or 0x8000 for 0x1). Its appears to be a literal string /* 0x1 (number type) example below tp 456 + 54: log default (main_exe) @@ -225,12 +193,11 @@ pub fn format_firehose_log_message( format: kext submap [0x%lx - 0x%lx], kernel text [0x%lx - 0x%lx] kext submap [0x - 0x], kernel text [0x - 0x] */ - - if private_strings.contains(&item_at_index.item_type) - && item_at_index.message_strings.is_empty() - && item_at_index.item_size == 0 - || (item_at_index.item_type == private_number - && item_at_index.item_size == private_message) + if private_strings.contains(&item_message[item_index].item_type) + && item_message[item_index].message_strings.is_empty() + && item_message[item_index].item_size == 0 + || (item_message[item_index].item_type == private_number + && item_message[item_index].item_size == private_message) { formatted_log_message = String::from(""); } else { @@ -262,17 +229,17 @@ pub fn format_firehose_log_message( // Also seen number type value 0 also used for dynamic width/precision value let dynamic_precision_value = 0x0; - if (item_at_index.item_type == dynamic_precision_value && item_at_index.item_size == 0) + if (item_message[item_index].item_type == dynamic_precision_value + && item_message[item_index].item_size == 0) && formatter_string.contains("%*") { - update_index_or!(item_index + 1, { - continue; - }); + item_index += 1; } - update_index_or!(item_index + 1, { - continue; - }); + item_index += 1; + format_and_message.formatter = formatter.as_str().to_string(); + format_and_message.message = formatted_log_message; + format_and_message_vec.push(format_and_message); } let mut log_message_vec: Vec = Vec::new(); @@ -303,22 +270,7 @@ fn parse_formatter<'a>( item_type: u8, item_index: usize, ) -> nom::IResult<&'a str, String> { - let mut index: usize; - let mut message_at_index: &FirehoseItemInfo; - - // auto update index & dependent stuff at the same time - // assert that the index is valid & return if not - macro_rules! update_index { - ( $i:expr ) => { - index = $i; - if let Some(msg) = message_value.get(index) { - message_at_index = msg - } else { - return Ok(("", String::from("UNABLE TO PARSE MESSAGE"))); - } - }; - } - update_index!(item_index); + let mut index = item_index; const PRECISION_ITEMS: [u8; 2] = [0x10, 0x12]; let mut precision_value = 0; @@ -332,16 +284,16 @@ fn parse_formatter<'a>( } } - let mut message = message_at_index.message_strings.to_owned(); + let mut message = message_value[index].message_strings.to_owned(); let number_item_type: Vec = vec![0x0, 0x1, 0x2]; // If the message formatter is expects a string/character and the message string is a number type // Try to convert to a character/string if formatter.to_lowercase().ends_with('c') - && number_item_type.contains(&message_at_index.item_type) + && number_item_type.contains(&message_value[index].item_type) { - let char_results = message_at_index.message_strings.parse::(); + let char_results = message_value[index].message_strings.parse::(); match char_results { Ok(char_message) => message = (char_message as u8 as char).to_string(), Err(err) => { From d2685e5faa97b6104e7395e5a8e9a1df8f8b375f Mon Sep 17 00:00:00 2001 From: jrx Date: Thu, 28 Nov 2024 17:15:30 +0100 Subject: [PATCH 25/26] toml deps workspace = true --- Cargo.toml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index f781b1e..0e26ae1 100755 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,23 +10,23 @@ keywords = ["forensics", "macOS", "unifiedlog"] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -nom = "7.1.3" -serde_json = "1.0.133" -serde = { version = "1.0.215", features = ["derive"] } -log = "0.4.22" +nom = { workspace = true } +serde_json = { workspace = true } +serde = { workspace = true } +log = { workspace = true } lz4_flex = "0.11.3" byteorder = "1.5.0" -plist = "1.7.0" -regex = "1.11.1" -base64 = "0.22.1" -chrono = "0.4.38" +plist = { workspace = true } +regex = { workspace = true } +base64 = { workspace = true } +chrono = { workspace = true } [dev-dependencies] simplelog = "0.12.2" -csv = "1.3.1" -chrono = "0.4.38" -criterion = "0.5.1" -anyhow = "1.0.94" +csv = { workspace = true } +chrono = { workspace = true } +criterion = { workspace = true } +anyhow = { workspace = true } [[bench]] name = "high_sierra_benchmark" From 135ab0df17b4d0707519f66397d349f4bdf38f4b Mon Sep 17 00:00:00 2001 From: jrx Date: Mon, 16 Dec 2024 14:07:19 +0100 Subject: [PATCH 26/26] removed changes from upstream --- src/decoders/decoder.rs | 6 +----- src/lib.rs | 26 +++++++++++++------------- src/timesync.rs | 12 ++++++------ src/unified_log.rs | 1 - 4 files changed, 20 insertions(+), 25 deletions(-) diff --git a/src/decoders/decoder.rs b/src/decoders/decoder.rs index 09b7d9b..54c67be 100644 --- a/src/decoders/decoder.rs +++ b/src/decoders/decoder.rs @@ -42,14 +42,10 @@ pub(crate) fn check_objects( return format!("Index out of bounds for FirehoseItemInfo Vec. Got adjusted index {}, Vec size is {}. This should not have happened", index, message_values.len()); } - if index >= message_values.len() { - return format!("Index out of bounds for FirehoseItemInfo Vec. Got adjusted index {}, Vec size is {}. This should not have happened", index, message_values.len()); - } - // Increment index get the actual firehose item data if item_type == PRECISION_ITEM { index += 1; - if index >= message_values.len() { + if index > message_values.len() { return format!("Index out of bounds for FirehoseItemInfo Vec. Got adjusted index {}, Vec size is {}. This should not have happened", index, message_values.len()); } } diff --git a/src/lib.rs b/src/lib.rs index 3dd6d3b..5df61d2 100755 --- a/src/lib.rs +++ b/src/lib.rs @@ -7,19 +7,19 @@ #![forbid(unsafe_code)] #![warn( - clippy::all, - clippy::doc_markdown, - clippy::needless_continue, - clippy::match_on_vec_items, - clippy::imprecise_flops, - clippy::suboptimal_flops, - clippy::lossy_float_literal, - clippy::fn_params_excessive_bools, - clippy::inefficient_to_string, - clippy::verbose_file_reads, - clippy::unnested_or_patterns, - rust_2018_idioms, - future_incompatible + clippy::all, + clippy::doc_markdown, + clippy::needless_continue, + clippy::match_on_vec_items, + clippy::imprecise_flops, + clippy::suboptimal_flops, + clippy::lossy_float_literal, + clippy::fn_params_excessive_bools, + clippy::inefficient_to_string, + clippy::verbose_file_reads, + clippy::unnested_or_patterns, + rust_2018_idioms, + future_incompatible )] #![deny( clippy::cast_lossless, diff --git a/src/timesync.rs b/src/timesync.rs index 73b616e..b827639 100755 --- a/src/timesync.rs +++ b/src/timesync.rs @@ -76,9 +76,9 @@ impl TimesyncBoot { let expected_boot_signature = 0xbbb0; if expected_boot_signature != timesync_signature { error!( - "[macos-unifiedlogs] Incorrect Timesync boot header signature. Expected {}. Got: {}", - expected_boot_signature, timesync_signature - ); + "[macos-unifiedlogs] Incorrect Timesync boot header signature. Expected {}. Got: {}", + expected_boot_signature, timesync_signature + ); return Err(nom::Err::Incomplete(Needed::Unknown)); } @@ -130,9 +130,9 @@ impl TimesyncBoot { let expected_record_signature = 0x207354; if expected_record_signature != timesync_signature { error!( - "[macos-unifiedlogs] Incorrect Timesync record header signature. Expected {}. Got: {}", - expected_record_signature, timesync_signature - ); + "[macos-unifiedlogs] Incorrect Timesync record header signature. Expected {}. Got: {}", + expected_record_signature, timesync_signature + ); return Err(nom::Err::Incomplete(Needed::Unknown)); } diff --git a/src/unified_log.rs b/src/unified_log.rs index e78624c..1b10069 100755 --- a/src/unified_log.rs +++ b/src/unified_log.rs @@ -560,7 +560,6 @@ impl Iterator for LogIterator<'_> { } } }; - let timestamp = TimesyncBoot::get_timestamp( self.timesync_data, &self.unified_log_data.header[0].boot_uuid,