From 1df45f49fc3d0d2a444aa831e940a7b0a54e4ab1 Mon Sep 17 00:00:00 2001 From: Vladimir Vukicevic Date: Sat, 4 May 2024 09:13:08 -0700 Subject: [PATCH 01/10] remove etw-gecko --- etw-gecko/Cargo.toml | 17 - etw-gecko/NOTES.md | 2 - etw-gecko/README.md | 36 - etw-gecko/advanced-tracing.md | 54 - etw-gecko/src/context_switch.rs | 321 ----- etw-gecko/src/jit_category_manager.rs | 176 --- etw-gecko/src/jit_function_add_marker.rs | 41 - etw-gecko/src/lib_mappings.rs | 175 --- etw-gecko/src/main.rs | 1144 ----------------- etw-gecko/src/marker_file.rs | 83 -- etw-gecko/src/process_sample_data.rs | 304 ----- etw-gecko/src/stack_converter.rs | 157 --- .../src/stack_depth_limiting_frame_iter.rs | 138 -- etw-gecko/src/timestamp_converter.rs | 23 - etw-gecko/src/types.rs | 19 - etw-gecko/src/unresolved_samples.rs | 265 ---- 16 files changed, 2955 deletions(-) delete mode 100644 etw-gecko/Cargo.toml delete mode 100644 etw-gecko/NOTES.md delete mode 100644 etw-gecko/README.md delete mode 100644 etw-gecko/advanced-tracing.md delete mode 100644 etw-gecko/src/context_switch.rs delete mode 100644 etw-gecko/src/jit_category_manager.rs delete mode 100644 etw-gecko/src/jit_function_add_marker.rs delete mode 100644 etw-gecko/src/lib_mappings.rs delete mode 100644 etw-gecko/src/main.rs delete mode 100644 etw-gecko/src/marker_file.rs delete mode 100644 etw-gecko/src/process_sample_data.rs delete mode 100644 etw-gecko/src/stack_converter.rs delete mode 100644 etw-gecko/src/stack_depth_limiting_frame_iter.rs delete mode 100644 etw-gecko/src/timestamp_converter.rs delete mode 100644 etw-gecko/src/types.rs delete mode 100644 etw-gecko/src/unresolved_samples.rs diff --git a/etw-gecko/Cargo.toml b/etw-gecko/Cargo.toml deleted file mode 100644 index b792e091..00000000 --- a/etw-gecko/Cargo.toml +++ /dev/null @@ -1,17 +0,0 @@ -[package] -name = "etw-gecko" -version = "0.1.0" -authors = ["Jeff Muizelaar "] -edition = "2021" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] -serde_json = "1.0" -fxprof-processed-profile = { git = "https://github.com/mstange/samply/", rev="c6cc82a7d2580a73fd7fa4831a2bf87d519a5f69" } -uuid = "1" # We really want gecko_profile::debugid::uuid, but debugid doesn't re-export uuid -etw-reader = { path = "../etw-reader" } -pico-args = "0.4.2" -fxhash = "0.2.1" -rangemap = "1.3.0" -bitflags = "2.4.2" diff --git a/etw-gecko/NOTES.md b/etw-gecko/NOTES.md deleted file mode 100644 index 9c918ae1..00000000 --- a/etw-gecko/NOTES.md +++ /dev/null @@ -1,2 +0,0 @@ -- Buffering traces don't have all the information in them to do a single pass over the data because - some of the information that we need about threads etc. only shows up at the end of the trace. \ No newline at end of file diff --git a/etw-gecko/README.md b/etw-gecko/README.md deleted file mode 100644 index 1a1ecf1b..00000000 --- a/etw-gecko/README.md +++ /dev/null @@ -1,36 +0,0 @@ -# etw-gecko - -Converts from ETW logs to json consumable by https://profiler.firefox.com/. - -## Setup - -You need four tools for the full experience: - - 1. `xperf` to record profiling data to an ETL file. - 2. This repo to convert from the ETL file to a `gecko.json`. - 3. [Rust](https://rustup.rs/) - 4. `profiler-symbol-server` to open the profile in the profiler and to provide symbols. - -You most likely already have `xperf`; it's part of the [Windows Performance Toolkit](https://docs.microsoft.com/en-us/windows-hardware/test/wpt/) which is installed with a Windows SDK. - -To install `profiler-symbol-server`, run `cargo install profiler-symbol-server`. - -## Usage - -Open an Administrator command shell with Win+R, "cmd", Ctrl+Shift+Enter. - -Start profiling session by running `xperf -on latency -stackwalk profile+cswitch` in the Adminstrator shell. Then run `xperf -d out.etl` to capture it. - -Then run `cargo run --release out.etl [process-name]` to produce a gecko.json. - -Now set the `_NT_SYMBOL_PATH` environment variable: `set _NT_SYMBOL_PATH=srv*C:\symbols*http://msdl.microsoft.com/download/symbols*https://symbols.mozilla.org*https://chromium-browser-symsrv.commondatastorage.googleapis.com` -(use `$Env:_NT_SYMBOL_PATH = "srv*C:\symbols*http://msdl.microsoft.com/download/symbols*https://symbols.mozilla.org*https://chromium-browser-symsrv.commondatastorage.googleapis.com"` when using powershell) - -Finally run `profiler-symbol-server gecko.json` to open the profile in profiler.firefox.com. - -### Sampling Interval - -The default sampling rate is 0.1221 ms (8192Hz). This can be set to a different value -with something like `xperf -on latency -stackwalk profile -SetProfInt 10000` for a rate -of 1ms. (The units are 100 nanoseconds) - diff --git a/etw-gecko/advanced-tracing.md b/etw-gecko/advanced-tracing.md deleted file mode 100644 index 3fd5420b..00000000 --- a/etw-gecko/advanced-tracing.md +++ /dev/null @@ -1,54 +0,0 @@ -### Circular Buffering -ETW supports recording into an in memory circular buffer. This will avoid -dropping events if the disk can't keep up. This is useful when profiling on low -performance machines under high load. -``` -xperf -on latency -stackwalk profile -Buffering -Buffersize 1024 -MinBuffers 50000 -MaxBuffers 50000 -[do stuff you want to profile] -xperf -flush -f [output file] -xperf -stop -``` - -NOTE: converting circular buffer profiles is not yet supported. - -### Unblocking stacks (Not yet suported) - -``` -xperf -on Latency+DISPATCHER -stackwalk Profile+CSwitch+ReadyThread -``` - - -### Looking up providers/events - -https://github.com/lallousx86/WinTools/tree/master/WEPExplorer is useful browser of this information - -### Tracing with vsync -`xperf -start "NT Kernel Logger" -on latency -stackwalk profile+cswitch -start "usersession" -on Microsoft-Windows-DxgKrnl:1:1` -`xperf -stop "NT Kernel Logger" -stop "usersession" -d out.etl` - -### Tracing with Firefox events -``` -xperf -start "NT Kernel Logger" -on latency -stackwalk profile+cswitch -start "usersession" -on c923f508-96e4-5515-e32c-7539d1b10504 -xperf -stop "NT Kernel Logger" -stop "usersession" -d out.etl -``` - -### Graphics tracing suggestions -``` -xperf -start "NT Kernel Logger" -on latency -stackwalk profile+cswitch -start "usersession" -on c923f508-96e4-5515-e32c-7539d1b10504 Microsoft-Windows-DxgKrnl:1:1 Microsoft-Windows-DirectComposition -xperf -stop "NT Kernel Logger" -stop "usersession" -d out.etl -``` - - -### Stacks on page faults: -e.g. `xperf -on latency+ALL_FAULTS -stackwalk PagefaultDemandZero` -`latency` seems to be needed to get process information. -Add in calls to VirtualAlloc/VirtualFree -`xperf -on latency+ALL_FAULTS+VIRT_ALLOC -stackwalk PagefaultDemandZero` - -### Stacks on syscalls: -`xperf -on syscall -stackwalk SyscallEnter` -- Use syscall branch - -### Jscript -- Start Chrome with `chrome --js-flags="--enable-etw-stack-walking --interpreted-frames-native-stack"` -- `xperf -start "NT Kernel Logger" -on latency -stackwalk profile -start "usersession" -on Microsoft-JScript:0x3` -- `xperf -stop "NT Kernel Logger" -stop "usersession" -d out.etl` diff --git a/etw-gecko/src/context_switch.rs b/etw-gecko/src/context_switch.rs deleted file mode 100644 index a3cccefc..00000000 --- a/etw-gecko/src/context_switch.rs +++ /dev/null @@ -1,321 +0,0 @@ -/// Accumulates thread running times (for "CPU deltas") and simulates off-cpu sampling, -/// with the help of context switch events. -/// -/// In the Firefox Profiler format, a sample's "CPU delta" is the accumulated duration -/// for which the thread was running on the CPU since the previous sample. -/// -/// # Off-CPU sampling -/// -/// The goal of off-cpu sampling is to know what happened on the thread between on-CPU -/// samples, with about the same "accuracy" as for on-CPU samples. -/// -/// There can be lots of context switch events and we don't want to flood the profile -/// with a sample for each context switch. -/// -/// However, we also don't want to enforce a "minimum sleep time" because doing so -/// would skew the weighting of short sleeps. Really, what we're after is something -/// that looks comparable to what a wall-clock profiler would produce. -/// -/// We solve this by accumulating the off-cpu duration of all thread sleeps, regardless -/// of the individual sleep length. Once the accumulated off-cpu duration exceeds a -/// threshold (a multiple of the off-cpu "sampling interval"), we emit a sample. -/// -/// ## Details -/// -/// A thread is either running (= on-cpu) or sleeping (= off-cpu). -/// -/// The running time is the time between switch-in and switch-out. -/// The sleeping time is the time between switch-out and switch-in. -/// -/// After every thread sleep, we accumulate the off-cpu time. -/// Now there are two cases: -/// -/// Does the accumulated time cross an "off-cpu sampling" threshold? -/// If yes, turn it into an off-cpu sampling group and consume a multiple of the interval. -/// If no, don't emit any samples. The next sample's cpu delta will just be smaller. -pub struct ContextSwitchHandler { - off_cpu_sampling_interval: u64, -} - -impl ContextSwitchHandler { - pub fn new(off_cpu_sampling_interval: u64) -> Self { - Self { - off_cpu_sampling_interval, - } - } - - pub fn handle_switch_out(&self, timestamp: u64, thread: &mut ThreadContextSwitchData) { - match &thread.state { - ThreadState::Unknown => { - // This "switch-out" is the first time we've heard of the thread. So it must - // have been running until just now, but we didn't get any samples from it. - - // Just store the new state. - thread.state = ThreadState::Off { - off_switch_timestamp: timestamp, - }; - } - - ThreadState::On { - last_observed_on_timestamp, - } => { - // The thread was running and is now context-switched out. - // Accumulate the running time since we last saw it. This delta will be picked - // up by the next sample we emit. - let on_duration = timestamp - last_observed_on_timestamp; - thread.on_cpu_duration_since_last_sample += on_duration; - - thread.state = ThreadState::Off { - off_switch_timestamp: timestamp, - }; - } - ThreadState::Off { .. } => { - // We are already in the Off state but received another Switch-Out record. - // This is unexpected; Switch-Out records are the only records that can - // get us into the Off state and we do not expect two Switch-Out records - // without an in-between Switch-In record. - // However, in practice this case has been observed due to a duplicated - // Switch-Out record in the perf.data file: the record just appeared twice - // right after itself, same timestamp, same everything. I don't know if - // this duplication indicates a bug in the kernel or in the perf tool or - // maybe is not considered a bug at all. - } - } - } - - pub fn handle_switch_in( - &self, - timestamp: u64, - thread: &mut ThreadContextSwitchData, - ) -> Option { - let off_cpu_sample = match thread.state { - ThreadState::On { - last_observed_on_timestamp, - } => { - // We are already in the On state, most likely due to a Sample record which - // arrived just before the Switch-In record. - // This is quite normal. Thread switching is done by some kernel code which - // executes on the CPU, and this CPU work can get sampled before the CPU gets - // to the code that emits the Switch-In record. - let on_duration = timestamp - last_observed_on_timestamp; - thread.on_cpu_duration_since_last_sample += on_duration; - - None - } - ThreadState::Off { - off_switch_timestamp, - } => { - // The thread was sleeping and is now starting to run again. - // Accumulate the off-cpu time. - let off_duration = timestamp - off_switch_timestamp; - thread.off_cpu_duration_since_last_off_cpu_sample += off_duration; - - // We just added some off-cpu time. If the accumulated off-cpu time exceeds the - // off-cpu sampling interval, we want to consume some of it and turn it into an - // off-cpu sampling group. - self.maybe_consume_off_cpu(timestamp, thread) - } - ThreadState::Unknown => { - // This "switch-in" is the first time we've heard of the thread. - // It must have been sleeping at some stack, but we don't know in what stack, - // so it seems pointless to emit a sample for the sleep time. We also don't - // know what the reason for the "sleep" was: It could have been because the - // thread was blocked (most common) or because it was pre-empted. - - None - } - }; - - thread.state = ThreadState::On { - last_observed_on_timestamp: timestamp, - }; - - off_cpu_sample - } - - pub fn handle_on_cpu_sample( - &self, - timestamp: u64, - thread: &mut ThreadContextSwitchData, - ) -> Option { - let off_cpu_sample = match thread.state { - ThreadState::On { - last_observed_on_timestamp, - } => { - // The last time we heard from this thread, it was already running. - // Accumulate the running time. - let on_duration = timestamp - last_observed_on_timestamp; - thread.on_cpu_duration_since_last_sample += on_duration; - - None - } - ThreadState::Off { - off_switch_timestamp, - } => { - // The last time we heard from this thread, it was being context switched away from. - // We are processing a sample on it so we know it is running again. Treat this sample - // as a switch-in event. - let off_duration = timestamp - off_switch_timestamp; - thread.off_cpu_duration_since_last_off_cpu_sample += off_duration; - - // We just added some off-cpu time. If the accumulated off-cpu time exceeds the - // off-cpu sampling interval, we want to consume some of it and turn it into an - // off-cpu sampling group. - self.maybe_consume_off_cpu(timestamp, thread) - } - ThreadState::Unknown => { - // This sample is the first time we've ever head from a thread. - // We don't know whether it was running or sleeping. - // Do nothing. The first sample will have a CPU delta of 0. - - None - } - }; - - thread.state = ThreadState::On { - last_observed_on_timestamp: timestamp, - }; - - off_cpu_sample - } - - fn maybe_consume_off_cpu( - &self, - timestamp: u64, - thread: &mut ThreadContextSwitchData, - ) -> Option { - // If the accumulated off-cpu time exceeds the off-cpu sampling interval, - // we want to consume some of it and turn it into an off-cpu sampling group. - let interval = self.off_cpu_sampling_interval; - if thread.off_cpu_duration_since_last_off_cpu_sample < interval { - return None; - } - - // Let's turn the accumulated off-cpu time into an off-cpu sample group. - let sample_count = thread.off_cpu_duration_since_last_off_cpu_sample / interval; - debug_assert!(sample_count >= 1); - - let consumed_duration = sample_count * interval; - let remaining_duration = - thread.off_cpu_duration_since_last_off_cpu_sample - consumed_duration; - - let begin_timestamp = - timestamp - (thread.off_cpu_duration_since_last_off_cpu_sample - interval); - let end_timestamp = timestamp - remaining_duration; - debug_assert_eq!( - end_timestamp - begin_timestamp, - (sample_count - 1) * interval - ); - - // Consume the consumed duration and save the leftover duration. - thread.off_cpu_duration_since_last_off_cpu_sample = remaining_duration; - - Some(OffCpuSampleGroup { - begin_timestamp, - end_timestamp, - sample_count, - }) - } - - pub fn consume_cpu_delta(&self, thread: &mut ThreadContextSwitchData) -> u64 { - std::mem::replace(&mut thread.on_cpu_duration_since_last_sample, 0) - } -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct OffCpuSampleGroup { - pub begin_timestamp: u64, - pub end_timestamp: u64, - pub sample_count: u64, -} - -#[derive(Default, Clone, Debug, PartialEq, Eq)] -pub struct ThreadContextSwitchData { - state: ThreadState, - on_cpu_duration_since_last_sample: u64, - off_cpu_duration_since_last_off_cpu_sample: u64, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -enum ThreadState { - Unknown, - Off { off_switch_timestamp: u64 }, - On { last_observed_on_timestamp: u64 }, -} - -impl Default for ThreadState { - fn default() -> Self { - ThreadState::Unknown - } -} - -#[cfg(test)] -mod test { - use super::{ContextSwitchHandler, OffCpuSampleGroup, ThreadContextSwitchData}; - - #[test] - fn it_works() { - // sampling interval: 10 - // - // 0 10 20 30 40 50 60 - // 01234567890123456789012345678901234567890123456789012345678901 - // ===__========__=_____==____===___________________============= - // ^ v v v ^ ^ - // - // Graph legend: - // = Thread is running. - // _ Thread is sleeping. - // ^ On-cpu sample - // v Off-cpu sample - - let mut thread = ThreadContextSwitchData::default(); - let handler = ContextSwitchHandler::new(10); - let s = handler.handle_switch_in(0, &mut thread); - assert_eq!(s, None); - handler.handle_switch_out(3, &mut thread); - let s = handler.handle_switch_in(5, &mut thread); - assert_eq!(s, None); - let s = handler.handle_on_cpu_sample(12, &mut thread); - let delta = handler.consume_cpu_delta(&mut thread); - assert_eq!(s, None); - assert_eq!(delta, 10); - handler.handle_switch_out(13, &mut thread); - let s = handler.handle_switch_in(15, &mut thread); - assert_eq!(s, None); - handler.handle_switch_out(16, &mut thread); - let s = handler.handle_switch_in(21, &mut thread); - assert_eq!(s, None); - handler.handle_switch_out(23, &mut thread); - let s = handler.handle_switch_in(27, &mut thread); - assert_eq!( - s, - Some(OffCpuSampleGroup { - begin_timestamp: 24, - end_timestamp: 24, - sample_count: 1 - }) - ); - let delta = handler.consume_cpu_delta(&mut thread); - assert_eq!(delta, 4); - handler.handle_switch_out(30, &mut thread); - let s = handler.handle_switch_in(48, &mut thread); - assert_eq!( - s, - Some(OffCpuSampleGroup { - begin_timestamp: 37, - end_timestamp: 47, - sample_count: 2 - }) - ); - let delta = handler.consume_cpu_delta(&mut thread); - assert_eq!(delta, 3); - let s = handler.handle_on_cpu_sample(51, &mut thread); - let delta = handler.consume_cpu_delta(&mut thread); - assert_eq!(s, None); - assert_eq!(delta, 3); - let s = handler.handle_on_cpu_sample(61, &mut thread); - let delta = handler.consume_cpu_delta(&mut thread); - assert_eq!(s, None); - assert_eq!(delta, 10); - } -} diff --git a/etw-gecko/src/jit_category_manager.rs b/etw-gecko/src/jit_category_manager.rs deleted file mode 100644 index 07985b8b..00000000 --- a/etw-gecko/src/jit_category_manager.rs +++ /dev/null @@ -1,176 +0,0 @@ -use fxprof_processed_profile::{ - CategoryColor, CategoryHandle, CategoryPairHandle, Profile, StringHandle, -}; - -#[derive(Debug, Clone, Copy)] -pub enum JsFrame { - Regular(JsName), - BaselineInterpreterStub(JsName), - BaselineInterpreter, -} - -#[derive(Debug, Clone, Copy)] -pub enum JsName { - SelfHosted(StringHandle), - NonSelfHosted(StringHandle), -} - -#[derive(Debug, Clone)] -pub struct JitCategoryManager { - categories: Vec, - baseline_interpreter_category: LazilyCreatedCategory, - ion_ic_category: LazilyCreatedCategory, -} - -impl JitCategoryManager { - /// (prefix, name, color, is_js) - const CATEGORIES: &'static [(&'static str, &'static str, CategoryColor, bool)] = &[ - ("JS:~", "Interpreter", CategoryColor::Magenta, true), - ("Script:~", "Interpreter", CategoryColor::Magenta, true), - ("JS:^", "Baseline", CategoryColor::Blue, true), - ("JS:+", "Maglev", CategoryColor::Green, true), - ("JS:*", "Turbofan", CategoryColor::Green, true), - ("Builtin:", "Builtin", CategoryColor::Brown, false), - ("BytecodeHandler:", "Interpreter", CategoryColor::Red, false), - ("Interpreter: ", "Interpreter", CategoryColor::Red, true), - ("Baseline: ", "Baseline", CategoryColor::Blue, true), - ("Ion: ", "Ion", CategoryColor::Green, true), - ("BaselineIC: ", "BaselineIC", CategoryColor::Brown, false), - ("IC: ", "IC", CategoryColor::Brown, false), - ("Trampoline: ", "Trampoline", CategoryColor::DarkGray, false), - ( - "Baseline JIT code for ", - "Baseline", - CategoryColor::Blue, - true, - ), - ("DFG JIT code for ", "DFG", CategoryColor::Green, true), - ("FTL B3 code for ", "FTL", CategoryColor::Green, true), - ("", "JIT", CategoryColor::Purple, false), // Generic fallback category for JIT code - ]; - - pub fn new() -> Self { - Self { - categories: Self::CATEGORIES - .iter() - .map(|(_prefix, name, color, _is_js)| LazilyCreatedCategory::new(name, *color)) - .collect(), - baseline_interpreter_category: LazilyCreatedCategory::new( - "BaselineInterpreter", - CategoryColor::Magenta, - ), - ion_ic_category: LazilyCreatedCategory::new("IonIC", CategoryColor::Brown), - } - } - - /// Get the category and JS function name for a function from JIT code. - /// - /// The category is only created in the profile once a function with that - /// category is encountered. - pub fn classify_jit_symbol( - &mut self, - name: &str, - profile: &mut Profile, - ) -> (CategoryPairHandle, Option) { - if name == "BaselineInterpreter" || name.starts_with("BlinterpOp: ") { - return ( - self.baseline_interpreter_category.get(profile).into(), - Some(JsFrame::BaselineInterpreter), - ); - } - - if let Some(js_func) = name.strip_prefix("BaselineInterpreter: ") { - let js_func = JsFrame::BaselineInterpreterStub(Self::intern_js_name(profile, js_func)); - return ( - self.baseline_interpreter_category.get(profile).into(), - Some(js_func), - ); - } - - if let Some(ion_ic_rest) = name.strip_prefix("IonIC: ") { - let category = self.ion_ic_category.get(profile); - if let Some((_ic_type, js_func)) = ion_ic_rest.split_once(" : ") { - let js_func = JsFrame::Regular(Self::intern_js_name(profile, js_func)); - return (category.into(), Some(js_func)); - } - return (category.into(), None); - } - - for (&(prefix, _category_name, _color, is_js), lazy_category_handle) in - Self::CATEGORIES.iter().zip(self.categories.iter_mut()) - { - if let Some(name_without_prefix) = name.strip_prefix(prefix) { - let category = lazy_category_handle.get(profile); - - let js_name = if is_js { - Some(JsFrame::Regular(Self::intern_js_name( - profile, - name_without_prefix, - ))) - } else { - None - }; - return (category.into(), js_name); - } - } - panic!("the last category has prefix '' so it should always be hit") - } - - fn intern_js_name(profile: &mut Profile, func_name: &str) -> JsName { - let s = profile.intern_string(func_name); - match func_name.contains("(self-hosted:") { - true => JsName::SelfHosted(s), - false => JsName::NonSelfHosted(s), - } - } -} - -#[derive(Debug, Clone)] -struct LazilyCreatedCategory { - name: &'static str, - color: CategoryColor, - handle: Option, -} - -impl LazilyCreatedCategory { - pub fn new(name: &'static str, color: CategoryColor) -> Self { - Self { - name, - color, - handle: None, - } - } - - pub fn get(&mut self, profile: &mut Profile) -> CategoryHandle { - *self - .handle - .get_or_insert_with(|| profile.add_category(self.name, self.color)) - } -} - -#[cfg(test)] -mod test { - use fxprof_processed_profile::{ReferenceTimestamp, SamplingInterval}; - - use super::*; - - #[test] - fn test() { - let mut manager = JitCategoryManager::new(); - let mut profile = Profile::new( - "", - ReferenceTimestamp::from_millis_since_unix_epoch(0.0), - SamplingInterval::from_millis(1), - ); - let (_category, js_name) = manager.classify_jit_symbol( - "IonIC: SetElem : AccessibleButton (main.js:3560:25)", - &mut profile, - ); - match js_name { - Some(JsFrame::Regular(JsName::NonSelfHosted(s))) => { - assert_eq!(profile.get_string(s), "AccessibleButton (main.js:3560:25)") - } - _ => panic!(), - } - } -} diff --git a/etw-gecko/src/jit_function_add_marker.rs b/etw-gecko/src/jit_function_add_marker.rs deleted file mode 100644 index c622c724..00000000 --- a/etw-gecko/src/jit_function_add_marker.rs +++ /dev/null @@ -1,41 +0,0 @@ -use fxprof_processed_profile::{ - MarkerDynamicField, MarkerFieldFormat, MarkerLocation, MarkerSchema, MarkerSchemaField, - MarkerStaticField, ProfilerMarker, -}; -use serde_json::json; - -#[derive(Debug, Clone)] -pub struct JitFunctionAddMarker(pub String); - -impl ProfilerMarker for JitFunctionAddMarker { - const MARKER_TYPE_NAME: &'static str = "JitFunctionAdd"; - - fn json_marker_data(&self) -> serde_json::Value { - json!({ - "type": Self::MARKER_TYPE_NAME, - "functionName": self.0 - }) - } - - fn schema() -> MarkerSchema { - MarkerSchema { - type_name: Self::MARKER_TYPE_NAME, - locations: vec![MarkerLocation::MarkerChart, MarkerLocation::MarkerTable], - chart_label: Some("{marker.data.functionName}"), - tooltip_label: Some("{marker.data.functionName}"), - table_label: Some("{marker.data.functionName}"), - fields: vec![ - MarkerSchemaField::Dynamic(MarkerDynamicField { - key: "functionName", - label: "Function", - format: MarkerFieldFormat::String, - searchable: true, - }), - MarkerSchemaField::Static(MarkerStaticField { - label: "Description", - value: "Emitted when a JIT function is added to the process.", - }), - ], - } - } -} \ No newline at end of file diff --git a/etw-gecko/src/lib_mappings.rs b/etw-gecko/src/lib_mappings.rs deleted file mode 100644 index f163616c..00000000 --- a/etw-gecko/src/lib_mappings.rs +++ /dev/null @@ -1,175 +0,0 @@ -use std::iter::Peekable; - -use fxprof_processed_profile::{CategoryPairHandle, LibMappings, LibraryHandle}; - -use super::jit_category_manager::JsFrame; - -#[derive(Debug, Clone)] -pub struct LibMappingInfo { - pub lib_handle: LibraryHandle, - pub category: Option, - pub js_frame: Option, -} - -impl LibMappingInfo { - pub fn new_lib(lib_handle: LibraryHandle) -> Self { - Self { - lib_handle, - category: None, - js_frame: None, - } - } - - pub fn new_jit_function( - lib_handle: LibraryHandle, - category: CategoryPairHandle, - js_frame: Option, - ) -> Self { - Self { - lib_handle, - category: Some(category), - js_frame, - } - } -} - -pub struct LibMappingsHierarchy { - regular_libs: (LibMappings, LibMappingOpQueueIter), - jitdumps: Vec<(LibMappings, LibMappingOpQueueIter)>, - perf_map: Option>, -} - -impl LibMappingsHierarchy { - pub fn new(regular_lib_mappings_ops: LibMappingOpQueue) -> Self { - Self { - regular_libs: (LibMappings::default(), regular_lib_mappings_ops.into_iter()), - jitdumps: Vec::new(), - perf_map: None, - } - } - - pub fn add_jitdump_lib_mappings_ops(&mut self, lib_mappings_ops: LibMappingOpQueue) { - self.jitdumps - .push((LibMappings::default(), lib_mappings_ops.into_iter())); - } - - pub fn add_perf_map_mappings(&mut self, mappings: LibMappings) { - self.perf_map = Some(mappings); - } - - pub fn process_ops(&mut self, timestamp: u64) { - while let Some(op) = self.regular_libs.1.next_op_if_at_or_before(timestamp) { - op.apply_to(&mut self.regular_libs.0); - } - for (mappings, ops) in &mut self.jitdumps { - while let Some(op) = ops.next_op_if_at_or_before(timestamp) { - op.apply_to(mappings); - } - } - } - - pub fn convert_address(&self, address: u64) -> Option<(u32, &LibMappingInfo)> { - if let Some(x) = self.regular_libs.0.convert_address(address) { - return Some(x); - } - for (mappings, _ops) in &self.jitdumps { - if let Some(x) = mappings.convert_address(address) { - return Some(x); - } - } - if let Some(perf_map) = &self.perf_map { - if let Some(x) = perf_map.convert_address(address) { - return Some(x); - } - } - None - } -} - -#[derive(Debug, Clone, Default)] -pub struct LibMappingOpQueue(Vec<(u64, LibMappingOp)>); - -impl LibMappingOpQueue { - pub fn push(&mut self, timestamp: u64, op: LibMappingOp) { - self.0.push((timestamp, op)); - } - - pub fn into_iter(self) -> LibMappingOpQueueIter { - LibMappingOpQueueIter(self.0.into_iter().peekable()) - } -} - -pub struct LibMappingOpQueueIter(Peekable>); - -impl LibMappingOpQueueIter { - pub fn next_op_if_at_or_before(&mut self, timestamp: u64) -> Option { - if self.0.peek()?.0 > timestamp { - return None; - } - let (_timestamp, op) = self.0.next().unwrap(); - Some(op) - } -} - -#[derive(Debug, Clone)] -pub enum LibMappingOp { - Add(LibMappingAdd), - Move(LibMappingMove), - #[allow(unused)] - Remove(LibMappingRemove), - Clear, -} - -impl LibMappingOp { - pub fn apply_to(self, lib_mappings: &mut LibMappings) { - match self { - LibMappingOp::Add(op) => { - lib_mappings.add_mapping( - op.start_avma, - op.end_avma, - op.relative_address_at_start, - op.info, - ); - } - LibMappingOp::Move(op) => { - if let Some((relative_address_at_start, info)) = - lib_mappings.remove_mapping(op.old_start_avma) - { - lib_mappings.add_mapping( - op.new_start_avma, - op.new_end_avma, - relative_address_at_start, - info, - ); - } - } - LibMappingOp::Remove(op) => { - lib_mappings.remove_mapping(op.start_avma); - } - LibMappingOp::Clear => { - lib_mappings.clear(); - } - } - } -} - -#[derive(Debug, Clone)] -pub struct LibMappingAdd { - pub start_avma: u64, - pub end_avma: u64, - pub relative_address_at_start: u32, - pub info: LibMappingInfo, -} - -#[derive(Debug, Clone)] -pub struct LibMappingMove { - pub old_start_avma: u64, - pub new_start_avma: u64, - pub new_end_avma: u64, -} - -#[allow(unused)] -#[derive(Debug, Clone)] -pub struct LibMappingRemove { - pub start_avma: u64, -} diff --git a/etw-gecko/src/main.rs b/etw-gecko/src/main.rs deleted file mode 100644 index 3bcbeeef..00000000 --- a/etw-gecko/src/main.rs +++ /dev/null @@ -1,1144 +0,0 @@ -use std::{collections::{HashMap, HashSet, hash_map::Entry, VecDeque}, convert::TryInto, fs::File, io::BufWriter, path::Path, time::{Duration, Instant, SystemTime}, sync::Arc}; - -use context_switch::{OffCpuSampleGroup, ThreadContextSwitchData}; -use etw_reader::{GUID, open_trace, parser::{Parser, TryParse, Address}, print_property, schema::SchemaLocator, write_property}; -use lib_mappings::{LibMappingOpQueue, LibMappingOp, LibMappingAdd}; -use serde_json::{Value, json, to_writer}; -use fxprof_processed_profile::{debugid, CategoryColor, CategoryHandle, CategoryPairHandle, CounterHandle, CpuDelta, FrameFlags, FrameInfo, LibraryHandle, LibraryInfo, MarkerDynamicField, MarkerFieldFormat, MarkerLocation, MarkerSchema, MarkerSchemaField, MarkerTiming, ProcessHandle, Profile, ProfilerMarker, ReferenceTimestamp, SamplingInterval, Symbol, SymbolTable, ThreadHandle, Timestamp}; -use debugid::DebugId; -use bitflags::bitflags; - - -mod context_switch; -mod jit_category_manager; -mod jit_function_add_marker; -mod lib_mappings; -mod marker_file; -mod process_sample_data; -mod stack_converter; -mod stack_depth_limiting_frame_iter; -mod timestamp_converter; -mod types; -mod unresolved_samples; - -use jit_category_manager::JitCategoryManager; -use stack_converter::StackConverter; -use lib_mappings::LibMappingInfo; -use types::{StackFrame, StackMode}; -use unresolved_samples::{UnresolvedSamples, UnresolvedStacks}; -use uuid::Uuid; -use process_sample_data::ProcessSampleData; - -use crate::{context_switch::ContextSwitchHandler, jit_function_add_marker::JitFunctionAddMarker, marker_file::get_markers, process_sample_data::UserTimingMarker, timestamp_converter::TimestampConverter}; - -/// An example marker type with some text content. -#[derive(Debug, Clone)] -pub struct TextMarker(pub String); - -impl ProfilerMarker for TextMarker { - const MARKER_TYPE_NAME: &'static str = "Text"; - - fn json_marker_data(&self) -> serde_json::Value { - json!({ - "type": Self::MARKER_TYPE_NAME, - "name": self.0 - }) - } - - fn schema() -> MarkerSchema { - MarkerSchema { - type_name: Self::MARKER_TYPE_NAME, - locations: vec![MarkerLocation::MarkerChart, MarkerLocation::MarkerTable], - chart_label: Some("{marker.data.name}"), - tooltip_label: Some("{marker.data.name}"), - table_label: Some("{marker.name} - {marker.data.name}"), - fields: vec![MarkerSchemaField::Dynamic(MarkerDynamicField { - key: "name", - label: "Name", - format: MarkerFieldFormat::String, - searchable: true, - })], - } - } -} - -fn is_kernel_address(ip: u64, pointer_size: u32) -> bool { - if pointer_size == 4 { - return ip >= 0x80000000; - } - return ip >= 0xFFFF000000000000; // TODO I don't know what the true cutoff is. -} - -fn stack_mode_for_address(address: u64, pointer_size: u32) -> StackMode { - if is_kernel_address(address, pointer_size) { - StackMode::Kernel - } else { - StackMode::User - } -} - -/// An on- or off-cpu-sample for which the user stack is not known yet. -/// Consumed once the user stack arrives. -#[derive(Debug, Clone)] -struct PendingStack { - /// The timestamp of the SampleProf or CSwitch event - timestamp: u64, - /// Starts out as None. Once we encounter the kernel stack (if any), we put it here. - kernel_stack: Option>, - off_cpu_sample_group: Option, - on_cpu_sample_cpu_delta: Option, -} - -struct ThreadState { - // When merging threads `handle` is the global thread handle and we use `merge_name` to store the name - handle: ThreadHandle, - merge_name: Option, - pending_stacks: VecDeque, - context_switch_data: ThreadContextSwitchData, - thread_id: u32 -} - -impl ThreadState { - fn new(handle: ThreadHandle, tid: u32) -> Self { - ThreadState { - handle, - pending_stacks: VecDeque::new(), - context_switch_data: ThreadContextSwitchData::default(), - merge_name: None, - thread_id: tid - } - } -} - - -fn strip_thread_numbers(name: &str) -> &str { - if let Some(hash) = name.find('#') { - let (prefix, suffix) = name.split_at(hash); - if suffix[1..].parse::().is_ok() { - return prefix.trim(); - } - } - return name; -} - -struct MemoryUsage { - counter: CounterHandle, - value: f64 -} - -struct ProcessJitInfo { - lib_handle: LibraryHandle, - jit_mapping_ops: LibMappingOpQueue, - next_relative_address: u32, - symbols: Vec, -} - -struct ProcessState { - process_handle: ProcessHandle, - unresolved_samples: UnresolvedSamples, - regular_lib_mapping_ops: LibMappingOpQueue, - main_thread_handle: Option, - pending_libraries: HashMap, -} - -impl ProcessState { - pub fn new(process_handle: ProcessHandle) -> Self { - Self { - process_handle, - unresolved_samples: UnresolvedSamples::default(), - regular_lib_mapping_ops: LibMappingOpQueue::default(), - main_thread_handle: None, - pending_libraries: HashMap::new(), - } - } -} - -fn main() { - let profile_start_instant = Timestamp::from_nanos_since_reference(0); - let profile_start_system = SystemTime::now(); - - let mut schema_locator = SchemaLocator::new(); - etw_reader::add_custom_schemas(&mut schema_locator); - let mut threads: HashMap = HashMap::new(); - let mut processes: HashMap = HashMap::new(); - let mut kernel_pending_libraries: HashMap = HashMap::new(); - let mut memory_usage: HashMap = HashMap::new(); - - let mut libs: HashMap = HashMap::new(); - let start = Instant::now(); - let mut pargs = pico_args::Arguments::from_env(); - let merge_threads = pargs.contains("--merge-threads"); - let include_idle = pargs.contains("--idle"); - let demand_zero_faults = pargs.contains("--demand-zero-faults"); - let marker_file: Option = pargs.opt_value_from_str("--marker-file").unwrap(); - let marker_prefix: Option = pargs.opt_value_from_str("--filter-by-marker-prefix").unwrap(); - - let trace_file: String = pargs.free_from_str().unwrap(); - - let mut process_targets = HashSet::new(); - let mut process_target_name = None; - if let Ok(process_filter) = pargs.free_from_str::() { - if let Ok(process_id) = process_filter.parse() { - process_targets.insert(process_id); - } else { - println!("targeting {}", process_filter); - process_target_name = Some(process_filter); - } - } else { - println!("No process specified"); - std::process::exit(1); - } - - let command_name = process_target_name.as_deref().unwrap_or("firefox"); - let mut profile = Profile::new(command_name, ReferenceTimestamp::from_system_time(profile_start_system), SamplingInterval::from_nanos(122100)); // 8192Hz - - let user_category: CategoryPairHandle = profile.add_category("User", fxprof_processed_profile::CategoryColor::Yellow).into(); - let kernel_category: CategoryPairHandle = profile.add_category("Kernel", fxprof_processed_profile::CategoryColor::Orange).into(); - - let mut jit_category_manager = JitCategoryManager::new(); - let mut unresolved_stacks = UnresolvedStacks::default(); - let mut context_switch_handler = ContextSwitchHandler::new(122100); - - let mut thread_index = 0; - let mut sample_count = 0; - let mut stack_sample_count = 0; - let mut dropped_sample_count = 0; - let mut timer_resolution: u32 = 0; // Resolution of the hardware timer, in units of 100 nanoseconds. - let mut event_count = 0; - let (global_thread, global_process) = if merge_threads { - let global_process = profile.add_process("All processes", 1, profile_start_instant); - (Some(profile.add_thread(global_process, 1, profile_start_instant, true)), Some(global_process)) - } else { - (None, None) - }; - let mut gpu_thread = None; - let mut jscript_symbols: HashMap = HashMap::new(); - let mut jscript_sources: HashMap = HashMap::new(); - - // Make a dummy TimestampConverter. Once we've parsed the header, this will have correct values. - let mut timestamp_converter = TimestampConverter { - reference_raw: 0, - raw_to_ns_factor: 1, - }; - let mut event_timestamps_are_qpc = false; - - let mut categories = HashMap::::new(); - let result = open_trace(Path::new(&trace_file), |e| { - event_count += 1; - let s = schema_locator.event_schema(e); - if let Ok(s) = s { - match s.name() { - "MSNT_SystemTrace/EventTrace/Header" => { - let mut parser = Parser::create(&s); - timer_resolution = parser.parse("TimerResolution"); - let perf_freq: u64 = parser.parse("PerfFreq"); - let clock_type: u32 = parser.parse("ReservedFlags"); - if clock_type != 1 { - println!("WARNING: QPC not used as clock"); - event_timestamps_are_qpc = false; - } else { - event_timestamps_are_qpc = true; - } - let events_lost: u32 = parser.parse("EventsLost"); - if events_lost != 0 { - println!("WARNING: {} events lost", events_lost); - } - - timestamp_converter = TimestampConverter { - reference_raw: e.EventHeader.TimeStamp as u64, - raw_to_ns_factor: 1000 * 1000 * 1000 / perf_freq, - }; - - for i in 0..s.property_count() { - let property = s.property(i); - print_property(&mut parser, &property, false); - } - } - "MSNT_SystemTrace/PerfInfo/CollectionStart" => { - let mut parser = Parser::create(&s); - let interval_raw: u32 = parser.parse("NewInterval"); - let interval_nanos = interval_raw as u64 * 100; - let interval = SamplingInterval::from_nanos(interval_nanos); - println!("Sample rate {}ms", interval.as_secs_f64() * 1000.); - profile.set_interval(interval); - context_switch_handler = ContextSwitchHandler::new(interval_raw as u64); - } - "MSNT_SystemTrace/Thread/SetName" => { - let mut parser = Parser::create(&s); - - let process_id: u32 = parser.parse("ProcessId"); - if !process_targets.contains(&process_id) { - return; - } - let thread_id: u32 = parser.parse("ThreadId"); - let thread_name: String = parser.parse("ThreadName"); - let thread = match threads.entry(thread_id) { - Entry::Occupied(e) => e.into_mut(), - Entry::Vacant(e) => { - let thread_start_instant = profile_start_instant; - let handle = match global_thread { - Some(global_thread) => global_thread, - None => { - let process = processes[&process_id].process_handle; - profile.add_thread(process, thread_id, thread_start_instant, false) - } - }; - let tb = e.insert( - ThreadState::new(handle, thread_id) - ); - thread_index += 1; - tb - } - }; - if Some(thread.handle) != global_thread { - profile.set_thread_name(thread.handle, &thread_name); - } - thread.merge_name = Some(thread_name); - } - "MSNT_SystemTrace/Thread/Start" | - "MSNT_SystemTrace/Thread/DCStart" => { - let timestamp = e.EventHeader.TimeStamp as u64; - let timestamp = timestamp_converter.convert_raw(timestamp); - let mut parser = Parser::create(&s); - - let thread_id: u32 = parser.parse("TThreadId"); - let process_id: u32 = parser.parse("ProcessId"); - //assert_eq!(process_id,s.process_id()); - //println!("thread_name pid: {} tid: {} name: {:?}", process_id, thread_id, thread_name); - - if !process_targets.contains(&process_id) { - return; - } - - let thread_start_instant = profile_start_instant; - let handle = match global_thread { - Some(global_thread) => global_thread, - None => { - let process = processes.get_mut(&process_id).unwrap(); - - let is_main = process.main_thread_handle.is_none(); - let thread_handle = profile.add_thread(process.process_handle, thread_id, timestamp, is_main); - if is_main { - process.main_thread_handle = Some(thread_handle); - } - thread_handle - } - }; - let thread = ThreadState::new(handle, thread_id); - - let thread = match threads.entry(thread_id) { - Entry::Occupied(e) => { - // Clobber the existing thread. We don't rely on thread end events to remove threads - // because they can be dropped and there can be subsequent events that refer to an ended thread. - // eg. - // MSNT_SystemTrace/Thread/End MSNT_SystemTrace 2-0 14 7369515373 - // ProcessId: InTypeUInt32 = 4532 - // TThreadId: InTypeUInt32 = 17524 - // MSNT_SystemTrace/Thread/ReadyThread MSNT_SystemTrace 50-0 5 7369515411 - // TThreadId: InTypeUInt32 = 1644 - // MSNT_SystemTrace/StackWalk/Stack MSNT_SystemTrace 32-0 35 7369515425 - // EventTimeStamp: InTypeUInt64 = 7369515411 - // StackProcess: InTypeUInt32 = 4532 - // StackThread: InTypeUInt32 = 17524 - // MSNT_SystemTrace/Thread/CSwitch MSNT_SystemTrace 36-0 12 7369515482 - // NewThreadId: InTypeUInt32 = 1644 - // OldThreadId: InTypeUInt32 = 0 - - let existing = e.into_mut(); - *existing = thread; - existing - } - Entry::Vacant(e) => { - e.insert(thread) - } - }; - - let thread_name: Result = parser.try_parse("ThreadName"); - - match thread_name { - Ok(thread_name) if !thread_name.is_empty() => { - if Some(thread.handle) != global_thread { - profile.set_thread_name(thread.handle, &thread_name); - } - thread.merge_name = Some(thread_name) - }, - _ => {} - } - } - "MSNT_SystemTrace/Thread/End" | - "MSNT_SystemTrace/Thread/DCEnd" => { - let timestamp = e.EventHeader.TimeStamp as u64; - let timestamp = timestamp_converter.convert_raw(timestamp); - let mut parser = Parser::create(&s); - - let thread_id: u32 = parser.parse("TThreadId"); - let process_id: u32 = parser.parse("ProcessId"); - - let thread = match threads.entry(thread_id) { - Entry::Occupied(e) => { - profile.set_thread_end_time(e.get().handle, timestamp); - } - Entry::Vacant(e) => { - } - }; - - } - "MSNT_SystemTrace/Process/Start" | - "MSNT_SystemTrace/Process/DCStart" => { - if let Some(process_target_name) = &process_target_name { - let timestamp = e.EventHeader.TimeStamp as u64; - let timestamp = timestamp_converter.convert_raw(timestamp); - let mut parser = Parser::create(&s); - - - let image_file_name: String = parser.parse("ImageFileName"); - println!("process start {}", image_file_name); - - let process_id: u32 = parser.parse("ProcessId"); - if image_file_name.contains(process_target_name) { - process_targets.insert(process_id); - println!("tracing {}", process_id); - let process_handle = match global_process { - Some(global_process) => global_process, - None => profile.add_process(&image_file_name, process_id, timestamp), - }; - - processes.insert(process_id, ProcessState::new(process_handle)); - } - } - } - "MSNT_SystemTrace/StackWalk/Stack" => { - let mut parser = Parser::create(&s); - - let thread_id: u32 = parser.parse("StackThread"); - let process_id: u32 = parser.parse("StackProcess"); - - let timestamp: u64 = parser.parse("EventTimeStamp"); - if !process_targets.contains(&process_id) { - // eprintln!("not watching"); - return; - } - - let thread = match threads.entry(thread_id) { - Entry::Occupied(e) => e.into_mut(), - Entry::Vacant(e) => { - let thread_start_instant = profile_start_instant; - let handle = match global_thread { - Some(global_thread) => global_thread, - None => { - let process = processes[&process_id].process_handle; - profile.add_thread(process, thread_id, thread_start_instant, false) - } - }; - let tb = e.insert( - ThreadState::new(handle, thread_id) - ); - thread_index += 1; - tb - } - }; - // eprint!("{} {} {}", thread_id, e.EventHeader.TimeStamp, timestamp); - - // Iterate over the stack addresses, starting with the instruction pointer - let mut stack: Vec = Vec::with_capacity(parser.buffer.len() / 8); - let mut address_iter = parser.buffer.chunks_exact(8).map(|a| u64::from_ne_bytes(a.try_into().unwrap())); - let Some(first_frame_address) = address_iter.next() else { return }; - let first_frame_stack_mode = stack_mode_for_address(first_frame_address, 8); - stack.push(StackFrame::InstructionPointer(first_frame_address, first_frame_stack_mode)); - for frame_address in address_iter { - let stack_mode = stack_mode_for_address(first_frame_address, 8); - stack.push(StackFrame::ReturnAddress(frame_address, stack_mode)); - } - - if first_frame_stack_mode == StackMode::Kernel { - if let Some(pending_stack ) = thread.pending_stacks.iter_mut().rev().find(|s| s.timestamp == timestamp) { - if let Some(kernel_stack) = pending_stack.kernel_stack.as_mut() { - eprintln!("Multiple kernel stacks for timestamp {timestamp} on thread {thread_id}"); - kernel_stack.extend(&stack); - } else { - pending_stack.kernel_stack = Some(stack); - } - } - return; - } - - // We now know that we have a user stack. User stacks always come last. Consume - // the pending stack with matching timestamp. - - let mut add_sample = |thread: &ThreadState, process: &mut ProcessState, timestamp: u64, cpu_delta: CpuDelta, weight: i32, stack: Vec| { - let profile_timestamp = timestamp_converter.convert_raw(timestamp); - let stack_index = unresolved_stacks.convert(stack.into_iter().rev()); - let extra_label_frame = if let Some(global_thread) = global_thread { - let thread_name = thread.merge_name.as_ref().map(|x| strip_thread_numbers(x).to_owned()).unwrap_or_else(|| format!("thread {}", thread.thread_id)); - Some(FrameInfo { - frame: fxprof_processed_profile::Frame::Label(profile.intern_string(&thread_name)), - category_pair: user_category, - flags: FrameFlags::empty(), - }) - } else { None }; - process.unresolved_samples.add_sample(thread.handle, profile_timestamp, timestamp, stack_index, cpu_delta, weight, extra_label_frame); - }; - - // Use this user stack for all pending stacks from this thread. - while thread.pending_stacks.front().is_some_and(|s| s.timestamp <= timestamp) { - let PendingStack { - timestamp, - kernel_stack, - off_cpu_sample_group, - on_cpu_sample_cpu_delta, - } = thread.pending_stacks.pop_front().unwrap(); - let process = processes.get_mut(&process_id).unwrap(); - - if let Some(off_cpu_sample_group) = off_cpu_sample_group { - let OffCpuSampleGroup { begin_timestamp, end_timestamp, sample_count } = off_cpu_sample_group; - - let cpu_delta_raw = context_switch_handler.consume_cpu_delta(&mut thread.context_switch_data); - let cpu_delta = CpuDelta::from_nanos(cpu_delta_raw as u64 * timestamp_converter.raw_to_ns_factor); - - // Add a sample at the beginning of the paused range. - // This "first sample" will carry any leftover accumulated running time ("cpu delta"). - add_sample(thread, process, begin_timestamp, cpu_delta, 1, stack.clone()); - - if sample_count > 1 { - // Emit a "rest sample" with a CPU delta of zero covering the rest of the paused range. - let weight = i32::try_from(sample_count - 1).unwrap_or(0) * 1; - add_sample(thread, process, end_timestamp, CpuDelta::ZERO, weight, stack.clone()); - } - } - - if let Some(cpu_delta) = on_cpu_sample_cpu_delta { - if let Some(mut combined_stack) = kernel_stack { - combined_stack.extend_from_slice(&stack[..]); - add_sample(thread, process, timestamp, cpu_delta, 1, combined_stack); - } else { - add_sample(thread, process, timestamp, cpu_delta, 1, stack.clone()); - } - stack_sample_count += 1; - } - } - } - "MSNT_SystemTrace/PerfInfo/SampleProf" => { - let mut parser = Parser::create(&s); - - let thread_id: u32 = parser.parse("ThreadId"); - //println!("sample {}", thread_id); - sample_count += 1; - - let thread = match threads.entry(thread_id) { - Entry::Occupied(e) => e.into_mut(), - Entry::Vacant(_) => { - if include_idle { - if let Some(global_thread) = global_thread { - let mut frames = Vec::new(); - let thread_name = match thread_id { - 0 => "Idle", - _ => "Other" - }; - let timestamp = e.EventHeader.TimeStamp as u64; - let timestamp = timestamp_converter.convert_raw(timestamp); - - frames.push(FrameInfo { - frame: fxprof_processed_profile::Frame::Label(profile.intern_string(&thread_name)), - category_pair: user_category, - flags: FrameFlags::empty() - }); - profile.add_sample(global_thread, timestamp, frames.into_iter(), Duration::ZERO.into(), 1); - } - } - dropped_sample_count += 1; - // We don't know what process this will before so just drop it for now - return; - } - }; - - let timestamp = e.EventHeader.TimeStamp as u64; - let off_cpu_sample_group = context_switch_handler.handle_on_cpu_sample(timestamp, &mut thread.context_switch_data); - let delta = context_switch_handler.consume_cpu_delta(&mut thread.context_switch_data); - let cpu_delta = CpuDelta::from_nanos(delta as u64 * timestamp_converter.raw_to_ns_factor); - thread.pending_stacks.push_back(PendingStack { timestamp, kernel_stack: None, off_cpu_sample_group, on_cpu_sample_cpu_delta: Some(cpu_delta) }); - } - "MSNT_SystemTrace/PageFault/DemandZeroFault" => { - if !demand_zero_faults { return } - - let thread_id: u32 = s.thread_id(); - //println!("sample {}", thread_id); - sample_count += 1; - - let thread = match threads.entry(thread_id) { - Entry::Occupied(e) => e.into_mut(), - Entry::Vacant(_) => { - if include_idle { - if let Some(global_thread) = global_thread { - let mut frames = Vec::new(); - let thread_name = match thread_id { - 0 => "Idle", - _ => "Other" - }; - let timestamp = e.EventHeader.TimeStamp as u64; - let timestamp = timestamp_converter.convert_raw(timestamp); - - frames.push(FrameInfo { - frame: fxprof_processed_profile::Frame::Label(profile.intern_string(&thread_name)), - category_pair: user_category, - flags: FrameFlags::empty(), - }); - - profile.add_sample(global_thread, timestamp, frames.into_iter(), Duration::ZERO.into(), 1); - } - } - dropped_sample_count += 1; - // We don't know what process this will before so just drop it for now - return; - } - }; - let timestamp = e.EventHeader.TimeStamp as u64; - thread.pending_stacks.push_back(PendingStack { timestamp, kernel_stack: None, off_cpu_sample_group: None, on_cpu_sample_cpu_delta: Some(CpuDelta::from_millis(1.0)) }); - } - "MSNT_SystemTrace/PageFault/VirtualFree" => { - if !process_targets.contains(&e.EventHeader.ProcessId) { - return; - } - let mut parser = Parser::create(&s); - let timestamp = e.EventHeader.TimeStamp as u64; - let timestamp = timestamp_converter.convert_raw(timestamp); - let thread_id = e.EventHeader.ThreadId; - let counter = match memory_usage.entry(e.EventHeader.ProcessId) { - Entry::Occupied(e) => e.into_mut(), - Entry::Vacant(entry) => { - entry.insert(MemoryUsage { counter: profile.add_counter(processes[&e.EventHeader.ProcessId].process_handle, "VirtualAlloc", "Memory", "Amount of VirtualAlloc allocated memory"), value: 0. }) - } - }; - let thread = match threads.entry(thread_id) { - Entry::Occupied(e) => e.into_mut(), - Entry::Vacant(_) => { - dropped_sample_count += 1; - // We don't know what process this will before so just drop it for now - return; - } - }; - let timing = MarkerTiming::Instant(timestamp); - let mut text = String::new(); - let region_size: u64 = parser.parse("RegionSize"); - counter.value -= region_size as f64; - - //println!("{} VirtualFree({}) = {}", e.EventHeader.ProcessId, region_size, counter.value); - - profile.add_counter_sample(counter.counter, timestamp, -(region_size as f64), 1); - for i in 0..s.property_count() { - let property = s.property(i); - //dbg!(&property); - write_property(&mut text, &mut parser, &property, false); - text += ", " - } - - profile.add_marker(thread.handle, CategoryHandle::OTHER, "VirtualFree", TextMarker(text), timing) - } - "MSNT_SystemTrace/PageFault/VirtualAlloc" => { - if !process_targets.contains(&e.EventHeader.ProcessId) { - return; - } - let mut parser = Parser::create(&s); - let timestamp = e.EventHeader.TimeStamp as u64; - let timestamp = timestamp_converter.convert_raw(timestamp); - let thread_id = e.EventHeader.ThreadId; - let counter = match memory_usage.entry(e.EventHeader.ProcessId) { - Entry::Occupied(e) => e.into_mut(), - Entry::Vacant(entry) => { - entry.insert(MemoryUsage { counter: profile.add_counter(processes[&e.EventHeader.ProcessId].process_handle, "VirtualAlloc", "Memory", "Amount of VirtualAlloc allocated memory"), value: 0. }) - } - }; - let thread = match threads.entry(thread_id) { - Entry::Occupied(e) => e.into_mut(), - Entry::Vacant(_) => { - dropped_sample_count += 1; - // We don't know what process this will before so just drop it for now - return; - } - }; - let timing = MarkerTiming::Instant(timestamp); - let mut text = String::new(); - let region_size: u64 = parser.parse("RegionSize"); - for i in 0..s.property_count() { - let property = s.property(i); - //dbg!(&property); - write_property(&mut text, &mut parser, &property, false); - text += ", " - } - counter.value += region_size as f64; - //println!("{}.{} VirtualAlloc({}) = {}", e.EventHeader.ProcessId, thread_id, region_size, counter.value); - - profile.add_counter_sample(counter.counter, timestamp, region_size as f64, 1); - profile.add_marker(thread.handle, CategoryHandle::OTHER, "VirtualAlloc", TextMarker(text), timing) - } - "KernelTraceControl/ImageID/" => { - - let process_id = s.process_id(); - if !process_targets.contains(&process_id) && process_id != 0 { - return; - } - let mut parser = Parser::create(&s); - - let image_base: u64 = parser.try_parse("ImageBase").unwrap(); - let timestamp = parser.try_parse("TimeDateStamp").unwrap(); - let image_size: u32 = parser.try_parse("ImageSize").unwrap(); - let binary_path: String = parser.try_parse("OriginalFileName").unwrap(); - let path = binary_path; - libs.insert(image_base, (path, image_size, timestamp)); - } - "KernelTraceControl/ImageID/DbgID_RSDS" => { - let mut parser = Parser::create(&s); - - let process_id = s.process_id(); - if !process_targets.contains(&process_id) && process_id != 0 { - return; - } - let image_base: u64 = parser.try_parse("ImageBase").unwrap(); - - let guid: GUID = parser.try_parse("GuidSig").unwrap(); - let age: u32 = parser.try_parse("Age").unwrap(); - let debug_id = DebugId::from_parts(Uuid::from_fields(guid.data1, guid.data2, guid.data3, &guid.data4), age); - let pdb_path: String = parser.try_parse("PdbFileName").unwrap(); - //let pdb_path = Path::new(&pdb_path); - let (ref path, image_size, timestamp) = libs[&image_base]; - let code_id = Some(format!("{timestamp:08X}{image_size:x}")); - let name = Path::new(path).file_name().unwrap().to_str().unwrap().to_owned(); - let debug_name = Path::new(&pdb_path).file_name().unwrap().to_str().unwrap().to_owned(); - let info = LibraryInfo { - name, - debug_name, - path: path.clone(), - code_id, - symbol_table: None, - debug_path: pdb_path, - debug_id, - arch: Some("x86_64".into()) - }; - if process_id == 0 { - kernel_pending_libraries.insert(image_base, info); - } else { - let process = processes.get_mut(&process_id).unwrap(); - process.pending_libraries.insert(image_base, info); - } - - } - "MSNT_SystemTrace/Image/Load" | "MSNT_SystemTrace/Image/DCStart" => { - // KernelTraceControl/ImageID/ and KernelTraceControl/ImageID/DbgID_RSDS are synthesized from MSNT_SystemTrace/Image/Load - // but don't contain the full path of the binary. We go through a bit of a dance to store the information from those events - // in pending_libraries and deal with it here. We assume that the KernelTraceControl events come before the Image/Load event. - - let mut parser = Parser::create(&s); - // the ProcessId field doesn't necessarily match s.process_id(); - let process_id = parser.try_parse("ProcessId").unwrap(); - if !process_targets.contains(&process_id) && process_id != 0 { - return; - } - let image_base: u64 = parser.try_parse("ImageBase").unwrap(); - let image_size: u64 = parser.try_parse("ImageSize").unwrap(); - - let path: String = parser.try_parse("FileName").unwrap(); - // The filename is a NT kernel path (https://chrisdenton.github.io/omnipath/NT.html) which isn't direclty usable from user space. - // perfview goes through a dance to convert it to a regular user space path - // https://github.com/microsoft/perfview/blob/4fb9ec6947cb4e68ac7cb5e80f50ae3757d0ede4/src/TraceEvent/Parsers/KernelTraceEventParser.cs#L3461 - // We'll just concatenate \\?\GLOBALROOT\ - let path = format!("\\\\?\\GLOBALROOT{}", path); - - let info = if process_id == 0 { - kernel_pending_libraries.remove(&image_base) - } else { - let process = processes.get_mut(&process_id).unwrap(); - process.pending_libraries.remove(&image_base) - }; - // If the file doesn't exist on disk we won't have KernelTraceControl/ImageID events - // This happens for the ghost drivers mentioned here: https://devblogs.microsoft.com/oldnewthing/20160913-00/?p=94305 - if let Some(mut info) = info { - info.path = path; - let lib_handle = profile.add_lib(info); - if process_id == 0 { - profile.add_kernel_lib_mapping(lib_handle, image_base, image_base + image_size as u64, 0); - } else { - let process = processes.get_mut(&process_id).unwrap(); - process.regular_lib_mapping_ops.push(e.EventHeader.TimeStamp as u64, LibMappingOp::Add(LibMappingAdd { - start_avma: image_base, - end_avma: image_base + image_size as u64, - relative_address_at_start: 0, - info: LibMappingInfo::new_lib(lib_handle), - })); - } - } - } - "Microsoft-Windows-DxgKrnl/VSyncDPC/Info " => { - let timestamp = e.EventHeader.TimeStamp as u64; - let timestamp = timestamp_converter.convert_raw(timestamp); - - #[derive(Debug, Clone)] - pub struct VSyncMarker; - - impl ProfilerMarker for VSyncMarker { - const MARKER_TYPE_NAME: &'static str = "Vsync"; - - fn json_marker_data(&self) -> Value { - json!({ - "type": Self::MARKER_TYPE_NAME, - "name": "" - }) - } - - fn schema() -> MarkerSchema { - MarkerSchema { - type_name: Self::MARKER_TYPE_NAME, - locations: vec![MarkerLocation::MarkerChart, MarkerLocation::MarkerTable, MarkerLocation::TimelineOverview], - chart_label: Some("{marker.data.name}"), - tooltip_label: None, - table_label: Some("{marker.name} - {marker.data.name}"), - fields: vec![MarkerSchemaField::Dynamic(MarkerDynamicField { - key: "name", - label: "Details", - format: MarkerFieldFormat::String, - searchable: false, - })], - } - } - } - - let gpu_thread = gpu_thread.get_or_insert_with(|| { - let gpu = profile.add_process("GPU", 1, profile_start_instant); - profile.add_thread(gpu, 1, profile_start_instant, false) - }); - profile.add_marker(*gpu_thread, - CategoryHandle::OTHER, - "Vsync", - VSyncMarker{}, - MarkerTiming::Instant(timestamp) - ); - } - "MSNT_SystemTrace/Thread/CSwitch" => { - let mut parser = Parser::create(&s); - let new_thread: u32 = parser.parse("NewThreadId"); - let old_thread: u32 = parser.parse("OldThreadId"); - let timestamp = e.EventHeader.TimeStamp as u64; - // println!("CSwitch {} -> {} @ {} on {}", old_thread, new_thread, e.EventHeader.TimeStamp, unsafe { e.BufferContext.Anonymous.ProcessorIndex }); - if let Some(old_thread) = threads.get_mut(&old_thread) { - context_switch_handler.handle_switch_out(timestamp, &mut old_thread.context_switch_data); - }; - if let Some(new_thread) = threads.get_mut(&new_thread) { - let off_cpu_sample_group = context_switch_handler.handle_switch_in(timestamp, &mut new_thread.context_switch_data); - if let Some(off_cpu_sample_group) = off_cpu_sample_group { - new_thread.pending_stacks.push_back(PendingStack { timestamp, kernel_stack: None, off_cpu_sample_group: Some(off_cpu_sample_group), on_cpu_sample_cpu_delta: None }); - } - }; - - } - "MSNT_SystemTrace/Thread/ReadyThread" => { - // these events can give us the unblocking stack - let mut parser = Parser::create(&s); - let _thread_id: u32 = parser.parse("TThreadId"); - } - "V8.js/MethodLoad/" | - "Microsoft-JScript/MethodRuntime/MethodDCStart" | - "Microsoft-JScript/MethodRuntime/MethodLoad" => { - let mut parser = Parser::create(&s); - let method_name: String = parser.parse("MethodName"); - let method_start_address: Address = parser.parse("MethodStartAddress"); - let method_size: u64 = parser.parse("MethodSize"); - // let source_id: u64 = parser.parse("SourceID"); - let process_id = s.process_id(); - let process = match processes.get_mut(&process_id) { - Some(process) => process, - None => { - // This event is probably from a process which doesn't match our name filter. - // Ignore it. - return; - } - }; - let process_jit_info = jscript_symbols.entry(s.process_id()).or_insert_with(|| { - let lib_handle = profile.add_lib(LibraryInfo { name: format!("JIT-{process_id}"), debug_name: format!("JIT-{process_id}"), path: format!("JIT-{process_id}"), debug_path: format!("JIT-{process_id}"), debug_id: DebugId::nil(), code_id: None, arch: None, symbol_table: None }); - ProcessJitInfo { lib_handle, jit_mapping_ops: LibMappingOpQueue::default(), next_relative_address: 0, symbols: Vec::new() } - }); - let start_address = method_start_address.as_u64(); - let relative_address = process_jit_info.next_relative_address; - process_jit_info.next_relative_address += method_size as u32; - - let timestamp = e.EventHeader.TimeStamp as u64; - let timestamp = timestamp_converter.convert_raw(timestamp); - - if let Some(main_thread) = process.main_thread_handle { - profile.add_marker( - main_thread, - CategoryHandle::OTHER, - "JitFunctionAdd", - JitFunctionAddMarker(method_name.to_owned()), - MarkerTiming::Instant(timestamp), - ); - } - - let (category, js_frame) = jit_category_manager.classify_jit_symbol(&method_name, &mut profile); - let info = LibMappingInfo::new_jit_function(process_jit_info.lib_handle, category, js_frame); - process_jit_info.jit_mapping_ops.push(e.EventHeader.TimeStamp as u64, LibMappingOp::Add(LibMappingAdd { - start_avma: start_address, - end_avma: start_address + method_size, - relative_address_at_start: relative_address, - info - })); - process_jit_info.symbols.push(Symbol { - address: relative_address, - size: Some(method_size as u32), - name: method_name, - }); - } - "V8.js/SourceLoad/" /*| - "Microsoft-JScript/MethodRuntime/MethodDCStart" | - "Microsoft-JScript/MethodRuntime/MethodLoad"*/ => { - let mut parser = Parser::create(&s); - let source_id: u64 = parser.parse("SourceID"); - let url: String = parser.parse("Url"); - //if s.process_id() == 6736 { dbg!(s.process_id(), &method_name, method_start_address, method_size); } - jscript_sources.insert(source_id, url); - //dbg!(s.process_id(), jscript_symbols.keys()); - - } - _ => { - if let Some(marker_name) = s.name().strip_prefix("Mozilla.FirefoxTraceLogger/").and_then(|s| s.strip_suffix("/")) { - let thread_id = e.EventHeader.ThreadId; - let thread = match threads.entry(thread_id) { - Entry::Occupied(e) => e.into_mut(), - Entry::Vacant(_) => { - dropped_sample_count += 1; - // We don't know what process this will before so just drop it for now - return; - } - }; - let mut parser = Parser::create(&s); - let mut text = String::new(); - for i in 0..s.property_count() { - let property = s.property(i); - match property.name.as_str() { - "MarkerName" | "StartTime" | "EndTime" | "Phase" | "InnerWindowId" | "CategoryPair" => { continue; } - _ => {} - } - write_property(&mut text, &mut parser, &property, false); - text += ", " - } - - /// From https://searchfox.org/mozilla-central/rev/0e7394a77cdbe1df5e04a1d4171d6da67b57fa17/mozglue/baseprofiler/public/BaseProfilerMarkersPrerequisites.h#355-360 - const PHASE_INSTANT: u8 = 0; - const PHASE_INTERVAL: u8 = 1; - const PHASE_INTERVAL_START: u8 = 2; - const PHASE_INTERVAL_END: u8 = 3; - - // We ignore e.EventHeader.TimeStamp and instead take the timestamp from the fields. - let start_time_qpc: u64 = parser.try_parse("StartTime").unwrap(); - let end_time_qpc: u64 = parser.try_parse("EndTime").unwrap(); - assert!(event_timestamps_are_qpc, "Inconsistent timestamp formats! ETW traces with Firefox events should be captured with QPC timestamps (-ClockType PerfCounter) so that ETW sample timestamps are compatible with the QPC timestamps in Firefox ETW trace events, so that the markers appear in the right place."); - let (phase, instant_time_qpc): (u8, u64) = match parser.try_parse("Phase") { - Ok(phase) => (phase, start_time_qpc), - Err(_) => { - // Before the landing of https://bugzilla.mozilla.org/show_bug.cgi?id=1882640 , - // Firefox ETW trace events didn't have phase information, so we need to - // guess a phase based on the timestamps. - if start_time_qpc != 0 && end_time_qpc != 0 { - (PHASE_INTERVAL, 0) - } else if start_time_qpc != 0 { - (PHASE_INSTANT, start_time_qpc) - } else { - (PHASE_INSTANT, end_time_qpc) - } - } - }; - let timing = match phase { - PHASE_INSTANT => MarkerTiming::Instant(timestamp_converter.convert_raw(instant_time_qpc)), - PHASE_INTERVAL => MarkerTiming::Interval(timestamp_converter.convert_raw(start_time_qpc), timestamp_converter.convert_raw(end_time_qpc)), - PHASE_INTERVAL_START => MarkerTiming::IntervalStart(timestamp_converter.convert_raw(start_time_qpc)), - PHASE_INTERVAL_END => MarkerTiming::IntervalEnd(timestamp_converter.convert_raw(end_time_qpc)), - _ => panic!("Unexpected marker phase {phase}"), - }; - - if marker_name == "UserTiming" { - let name: String = parser.try_parse("name").unwrap(); - profile.add_marker(thread.handle, CategoryHandle::OTHER, "UserTiming", UserTimingMarker(name), timing); - } else if marker_name == "SimpleMarker" || marker_name == "Text" || marker_name == "tracing" { - let marker_name: String = parser.try_parse("MarkerName").unwrap(); - profile.add_marker(thread.handle, CategoryHandle::OTHER, &marker_name, TextMarker(text.clone()), timing); - } else { - profile.add_marker(thread.handle, CategoryHandle::OTHER, marker_name, TextMarker(text.clone()), timing); - } - } else if let Some(marker_name) = s.name().strip_prefix("Google.Chrome/").and_then(|s| s.strip_suffix("/")) { - // a bitfield of keywords - bitflags! { - #[derive(PartialEq, Eq)] - pub struct KeywordNames: u64 { - const benchmark = 0x1; - const blink = 0x2; - const browser = 0x4; - const cc = 0x8; - const evdev = 0x10; - const gpu = 0x20; - const input = 0x40; - const netlog = 0x80; - const sequence_manager = 0x100; - const toplevel = 0x200; - const v8 = 0x400; - const disabled_by_default_cc_debug = 0x800; - const disabled_by_default_cc_debug_picture = 0x1000; - const disabled_by_default_toplevel_flow = 0x2000; - const startup = 0x4000; - const latency = 0x8000; - const blink_user_timing = 0x10000; - const media = 0x20000; - const loading = 0x40000; - const base = 0x80000; - const devtools_timeline = 0x100000; - const unused_bit_21 = 0x200000; - const unused_bit_22 = 0x400000; - const unused_bit_23 = 0x800000; - const unused_bit_24 = 0x1000000; - const unused_bit_25 = 0x2000000; - const unused_bit_26 = 0x4000000; - const unused_bit_27 = 0x8000000; - const unused_bit_28 = 0x10000000; - const unused_bit_29 = 0x20000000; - const unused_bit_30 = 0x40000000; - const unused_bit_31 = 0x80000000; - const unused_bit_32 = 0x100000000; - const unused_bit_33 = 0x200000000; - const unused_bit_34 = 0x400000000; - const unused_bit_35 = 0x800000000; - const unused_bit_36 = 0x1000000000; - const unused_bit_37 = 0x2000000000; - const unused_bit_38 = 0x4000000000; - const unused_bit_39 = 0x8000000000; - const unused_bit_40 = 0x10000000000; - const unused_bit_41 = 0x20000000000; - const navigation = 0x40000000000; - const ServiceWorker = 0x80000000000; - const edge_webview = 0x100000000000; - const diagnostic_event = 0x200000000000; - const __OTHER_EVENTS = 0x400000000000; - const __DISABLED_OTHER_EVENTS = 0x800000000000; - } - } - - let mut parser = Parser::create(&s); - let thread_id = e.EventHeader.ThreadId; - let phase: String = parser.try_parse("Phase").unwrap(); - - let thread = match threads.entry(thread_id) { - Entry::Occupied(e) => e.into_mut(), - Entry::Vacant(_) => { - dropped_sample_count += 1; - // We don't know what process this will before so just drop it for now - return; - } - }; - let mut text = String::new(); - for i in 0..s.property_count() { - let property = s.property(i); - if property.name == "Timestamp" || property.name == "Phase" || property.name == "Duration" { - continue; - } - //dbg!(&property); - write_property(&mut text, &mut parser, &property, false); - text += ", " - } - - // We ignore e.EventHeader.TimeStamp and instead take the timestamp from the fields. - let timestamp_us: u64 = parser.try_parse("Timestamp").unwrap(); - let timestamp = timestamp_converter.convert_us(timestamp_us); - - let timing = match phase.as_str() { - "Begin" => MarkerTiming::IntervalStart(timestamp), - "End" => MarkerTiming::IntervalEnd(timestamp), - _ => MarkerTiming::Instant(timestamp), - }; - let keyword = KeywordNames::from_bits(e.EventHeader.EventDescriptor.Keyword).unwrap(); - if keyword == KeywordNames::blink_user_timing { - profile.add_marker(thread.handle, CategoryHandle::OTHER, "UserTiming", UserTimingMarker(marker_name.to_owned()), timing); - } else { - profile.add_marker(thread.handle, CategoryHandle::OTHER, marker_name, TextMarker(text.clone()), timing); - } - } else { - let mut parser = Parser::create(&s); - - let timestamp = e.EventHeader.TimeStamp as u64; - let timestamp = timestamp_converter.convert_raw(timestamp); - let thread_id = e.EventHeader.ThreadId; - let thread = match threads.entry(thread_id) { - Entry::Occupied(e) => e.into_mut(), - Entry::Vacant(_) => { - dropped_sample_count += 1; - // We don't know what process this will before so just drop it for now - return; - } - }; - let mut text = String::new(); - for i in 0..s.property_count() { - let property = s.property(i); - //dbg!(&property); - write_property(&mut text, &mut parser, &property, false); - text += ", " - } - - let timing = MarkerTiming::Instant(timestamp); - let category = match categories.entry(s.provider_name()) { - Entry::Occupied(e) => *e.get(), - Entry::Vacant(e) => { - let category = profile.add_category(e.key(), CategoryColor::Transparent); - *e.insert(category) - } - }; - - profile.add_marker(thread.handle, category, s.name().split_once("/").unwrap().1, TextMarker(text), timing) - } - //println!("unhandled {}", s.name()) - } - } - //println!("{}", name); - } - }); - - if !result.is_ok() { - dbg!(&result); - std::process::exit(1); - } - - let (marker_spans, sample_ranges) = match marker_file { - Some(marker_file) => get_markers( - &marker_file, - marker_prefix.as_deref(), - timestamp_converter, - ) - .expect("Could not get markers"), - None => (Vec::new(), None), - }; - - // Push queued samples into the profile. - // We queue them so that we can get symbolicated JIT function names. To get symbolicated JIT function names, - // we have to call profile.add_sample after we call profile.set_lib_symbol_table, and we don't have the - // complete JIT symbol table before we've seen all JIT symbols. - // (This is a rather weak justification. The better justification is that this is consistent with what - // samply does on Linux and macOS, where the queued samples also want to respect JIT function names from - // a /tmp/perf-1234.map file, and this file may not exist until the profiled process finishes.) - let mut stack_frame_scratch_buf = Vec::new(); - for (process_id, process) in processes { - let ProcessState { unresolved_samples, regular_lib_mapping_ops, main_thread_handle, .. } = process; - let jitdump_lib_mapping_op_queues = match jscript_symbols.remove(&process_id) { - Some(jit_info) => { - profile.set_lib_symbol_table(jit_info.lib_handle, Arc::new(SymbolTable::new(jit_info.symbols))); - vec![jit_info.jit_mapping_ops] - }, - None => Vec::new(), - }; - let process_sample_data = ProcessSampleData::new(unresolved_samples, regular_lib_mapping_ops, jitdump_lib_mapping_op_queues, None, main_thread_handle.unwrap_or_else(|| panic!("process no main thread {:?}", process_id))); - process_sample_data.flush_samples_to_profile(&mut profile, user_category, kernel_category, &mut stack_frame_scratch_buf, &mut unresolved_stacks, &[], &marker_spans, sample_ranges.as_ref()) - } - - /*if merge_threads { - profile.add_thread(global_thread); - } else { - for (_, thread) in threads.drain() { profile.add_thread(thread.builder); } - }*/ - - let f = File::create("gecko.json").unwrap(); - to_writer(BufWriter::new(f), &profile).unwrap(); - println!("Took {} seconds", (Instant::now()-start).as_secs_f32()); - println!("{} events, {} samples, {} dropped, {} stack-samples", event_count, sample_count, dropped_sample_count, stack_sample_count); -} diff --git a/etw-gecko/src/marker_file.rs b/etw-gecko/src/marker_file.rs deleted file mode 100644 index 05e7526c..00000000 --- a/etw-gecko/src/marker_file.rs +++ /dev/null @@ -1,83 +0,0 @@ -use fxprof_processed_profile::Timestamp; -use rangemap::RangeSet; - -use std::fs::File; -use std::io::{BufRead, BufReader, Lines}; - -use super::timestamp_converter::TimestampConverter; - -#[derive(Debug, Clone)] -pub struct MarkerSpan { - pub start_time: Timestamp, - pub end_time: Timestamp, - pub name: String, -} - -fn process_marker_span_line( - line: &str, - timestamp_converter: &TimestampConverter, -) -> Option { - let mut split = line.splitn(3, ' '); - let start_time = split.next()?; - let end_time = split.next()?; - let name = split.next()?.to_owned(); - if name.is_empty() { - return None; - } - let start_time = timestamp_converter.convert_raw(start_time.parse::().ok()?); - let end_time = timestamp_converter.convert_raw(end_time.parse::().ok()?); - Some(MarkerSpan { - start_time, - end_time, - name, - }) -} - -pub struct MarkerFile { - lines: Lines>, - timestamp_converter: TimestampConverter, -} - -impl MarkerFile { - pub fn parse(file: File, timestamp_converter: TimestampConverter) -> Self { - Self { - lines: BufReader::new(file).lines(), - timestamp_converter, - } - } -} - -impl Iterator for MarkerFile { - type Item = MarkerSpan; - - fn next(&mut self) -> Option { - let line = self.lines.next()?.ok()?; - process_marker_span_line(&line, &self.timestamp_converter) - } -} - -pub fn get_markers( - marker_file: &str, - marker_name_prefix_for_filtering: Option<&str>, - timestamp_converter: TimestampConverter, -) -> Result<(Vec, Option>), std::io::Error> { - let f = std::fs::File::open(marker_file)?; - let marker_file = MarkerFile::parse(f, timestamp_converter); - let mut marker_spans: Vec = marker_file.collect(); - marker_spans.sort_by_key(|m| m.start_time); - - let sample_ranges = if let Some(prefix) = marker_name_prefix_for_filtering { - let mut sample_ranges = RangeSet::new(); - for marker_span in &marker_spans { - if marker_span.name.starts_with(prefix) && marker_span.end_time > marker_span.start_time - { - sample_ranges.insert(marker_span.start_time..marker_span.end_time); - } - } - Some(sample_ranges) - } else { - None - }; - - Ok((marker_spans, sample_ranges)) -} diff --git a/etw-gecko/src/process_sample_data.rs b/etw-gecko/src/process_sample_data.rs deleted file mode 100644 index 59ce60b0..00000000 --- a/etw-gecko/src/process_sample_data.rs +++ /dev/null @@ -1,304 +0,0 @@ -use fxprof_processed_profile::{ - CategoryHandle, CategoryPairHandle, LibMappings, MarkerDynamicField, MarkerFieldFormat, MarkerLocation, MarkerSchema, MarkerSchemaField, MarkerStaticField, MarkerTiming, Profile, ProfilerMarker, ThreadHandle, Timestamp -}; -use rangemap::RangeSet; -use serde_json::json; - -use super::{ - lib_mappings::{LibMappingInfo, LibMappingOpQueue, LibMappingsHierarchy}, - marker_file::MarkerSpan, - stack_converter::StackConverter, - stack_depth_limiting_frame_iter::StackDepthLimitingFrameIter, - types::StackFrame, - unresolved_samples::{ - OtherEventMarkerData, RssStatMarkerData, SampleData, SampleOrMarker, - UnresolvedSampleOrMarker, UnresolvedSamples, UnresolvedStacks, - }, -}; - -#[derive(Debug, Clone)] -pub enum RssStatMember { - ResidentFileMappingPages, - ResidentAnonymousPages, - AnonymousSwapEntries, - ResidentSharedMemoryPages, -} - -#[derive(Debug, Clone)] -pub struct ProcessSampleData { - unresolved_samples: UnresolvedSamples, - regular_lib_mapping_op_queue: LibMappingOpQueue, - jitdump_lib_mapping_op_queues: Vec, - perf_map_mappings: Option>, - main_thread_handle: ThreadHandle, -} - -impl ProcessSampleData { - pub fn new( - unresolved_samples: UnresolvedSamples, - regular_lib_mapping_op_queue: LibMappingOpQueue, - jitdump_lib_mapping_op_queues: Vec, - perf_map_mappings: Option>, - main_thread_handle: ThreadHandle, - ) -> Self { - Self { - unresolved_samples, - regular_lib_mapping_op_queue, - jitdump_lib_mapping_op_queues, - perf_map_mappings, - main_thread_handle, - } - } - - pub fn is_empty(&self) -> bool { - self.unresolved_samples.is_empty() - } - - #[allow(clippy::too_many_arguments)] - pub fn flush_samples_to_profile( - self, - profile: &mut Profile, - user_category: CategoryPairHandle, - kernel_category: CategoryPairHandle, - stack_frame_scratch_buf: &mut Vec, - stacks: &UnresolvedStacks, - event_names: &[String], - marker_spans: &[MarkerSpan], - sample_range_set: Option<&RangeSet>, - ) { - let ProcessSampleData { - unresolved_samples, - regular_lib_mapping_op_queue, - jitdump_lib_mapping_op_queues, - perf_map_mappings, - main_thread_handle, - } = self; - let mut lib_mappings_hierarchy = LibMappingsHierarchy::new(regular_lib_mapping_op_queue); - for jitdump_lib_mapping_ops in jitdump_lib_mapping_op_queues { - lib_mappings_hierarchy.add_jitdump_lib_mappings_ops(jitdump_lib_mapping_ops); - } - if let Some(perf_map_mappings) = perf_map_mappings { - lib_mappings_hierarchy.add_perf_map_mappings(perf_map_mappings); - } - let stack_converter = StackConverter::new(user_category, kernel_category); - let samples = unresolved_samples.into_inner(); - for sample in samples { - lib_mappings_hierarchy.process_ops(sample.timestamp_mono); - let UnresolvedSampleOrMarker { - thread_handle, - timestamp, - stack, - sample_or_marker, - extra_label_frame, - .. - } = sample; - - if sample_range_set.is_some() - && !sample_range_set.as_ref().unwrap().contains(×tamp) - { - continue; - } - - stack_frame_scratch_buf.clear(); - stacks.convert_back(stack, stack_frame_scratch_buf); - let frames = stack_converter.convert_stack( - stack_frame_scratch_buf, - &lib_mappings_hierarchy, - extra_label_frame, - ); - let frames = StackDepthLimitingFrameIter::new(profile, frames, user_category); - match sample_or_marker { - SampleOrMarker::Sample(SampleData { cpu_delta, weight }) => { - profile.add_sample(thread_handle, timestamp, frames, cpu_delta, weight); - } - SampleOrMarker::RssStatMarker(RssStatMarkerData { - size, - delta, - member, - }) => { - let timing = MarkerTiming::Instant(timestamp); - let name = match member { - RssStatMember::ResidentFileMappingPages => "RSS Stat FILEPAGES", - RssStatMember::ResidentAnonymousPages => "RSS Stat ANONPAGES", - RssStatMember::AnonymousSwapEntries => "RSS Stat SHMEMPAGES", - RssStatMember::ResidentSharedMemoryPages => "RSS Stat SWAPENTS", - }; - profile.add_marker_with_stack( - thread_handle, - CategoryHandle::OTHER, - name, - RssStatMarker(size, delta), - timing, - frames, - ); - } - SampleOrMarker::OtherEventMarker(OtherEventMarkerData { attr_index }) => { - if let Some(name) = event_names.get(attr_index) { - let timing = MarkerTiming::Instant(timestamp); - profile.add_marker_with_stack( - thread_handle, - CategoryHandle::OTHER, - name, - OtherEventMarker, - timing, - frames, - ); - } - } - } - } - - for marker in marker_spans { - profile.add_marker( - main_thread_handle, - CategoryHandle::OTHER, - "MarkerFileMarker", - MarkerFileMarker(marker.name.clone()), - MarkerTiming::Interval(marker.start_time, marker.end_time), - ); - } - } -} - -#[derive(Debug, Clone)] -pub struct RssStatMarker(pub i64, pub i64); - -impl ProfilerMarker for RssStatMarker { - const MARKER_TYPE_NAME: &'static str = "RSS Anon"; - - fn json_marker_data(&self) -> serde_json::Value { - json!({ - "type": Self::MARKER_TYPE_NAME, - "totalBytes": self.0, - "deltaBytes": self.1 - }) - } - - fn schema() -> MarkerSchema { - MarkerSchema { - type_name: Self::MARKER_TYPE_NAME, - locations: vec![MarkerLocation::MarkerChart, MarkerLocation::MarkerTable], - chart_label: Some("{marker.data.totalBytes}"), - tooltip_label: Some("{marker.data.totalBytes}"), - table_label: Some("Total: {marker.data.totalBytes}, delta: {marker.data.deltaBytes}"), - fields: vec![ - MarkerSchemaField::Dynamic(MarkerDynamicField { - key: "totalBytes", - label: "Total bytes", - format: MarkerFieldFormat::Bytes, - searchable: true, - }), - MarkerSchemaField::Dynamic(MarkerDynamicField { - key: "deltaBytes", - label: "Delta", - format: MarkerFieldFormat::Bytes, - searchable: true, - }), - MarkerSchemaField::Static(MarkerStaticField { - label: "Description", - value: "Emitted when the kmem:rss_stat tracepoint is hit.", - }), - ], - } - } -} - -#[derive(Debug, Clone)] -pub struct OtherEventMarker; - -impl ProfilerMarker for OtherEventMarker { - const MARKER_TYPE_NAME: &'static str = "Other event"; - - fn json_marker_data(&self) -> serde_json::Value { - json!({ - "type": Self::MARKER_TYPE_NAME, - }) - } - - fn schema() -> MarkerSchema { - MarkerSchema { - type_name: Self::MARKER_TYPE_NAME, - locations: vec![MarkerLocation::MarkerChart, MarkerLocation::MarkerTable], - chart_label: None, - tooltip_label: None, - table_label: None, - fields: vec![MarkerSchemaField::Static(MarkerStaticField { - label: "Description", - value: - "Emitted for any records in a perf.data file which don't map to a known event.", - })], - } - } -} - -#[derive(Debug, Clone)] -pub struct UserTimingMarker(pub String); - -impl ProfilerMarker for UserTimingMarker { - const MARKER_TYPE_NAME: &'static str = "UserTiming"; - - fn json_marker_data(&self) -> serde_json::Value { - json!({ - "type": Self::MARKER_TYPE_NAME, - "name": self.0, - }) - } - - fn schema() -> MarkerSchema { - MarkerSchema { - type_name: Self::MARKER_TYPE_NAME, - locations: vec![MarkerLocation::MarkerChart, MarkerLocation::MarkerTable], - chart_label: Some("{marker.data.name}"), - tooltip_label: Some("{marker.data.name}"), - table_label: Some("{marker.data.name}"), - fields: vec![ - MarkerSchemaField::Dynamic(MarkerDynamicField { - key: "name", - label: "Name", - format: MarkerFieldFormat::String, - searchable: true, - }), - MarkerSchemaField::Static(MarkerStaticField { - label: "Description", - value: "Emitted for performance.mark and performance.measure.", - }), - ], - } - } -} - -#[derive(Debug, Clone)] -pub struct MarkerFileMarker(pub String); - -impl ProfilerMarker for MarkerFileMarker { - const MARKER_TYPE_NAME: &'static str = "MarkerFileMarker"; - - fn json_marker_data(&self) -> serde_json::Value { - json!({ - "type": Self::MARKER_TYPE_NAME, - "name": self.0, - }) - } - - fn schema() -> MarkerSchema { - MarkerSchema { - type_name: Self::MARKER_TYPE_NAME, - locations: vec![MarkerLocation::MarkerChart, MarkerLocation::MarkerTable], - chart_label: Some("{marker.data.name}"), - tooltip_label: Some("{marker.data.name}"), - table_label: Some("{marker.data.name}"), - fields: vec![ - MarkerSchemaField::Dynamic(MarkerDynamicField { - key: "name", - label: "Name", - format: MarkerFieldFormat::String, - searchable: true, - }), - MarkerSchemaField::Static(MarkerStaticField { - label: "Description", - value: "Emitted for marker spans in a markers text file.", - }), - ], - } - } -} diff --git a/etw-gecko/src/stack_converter.rs b/etw-gecko/src/stack_converter.rs deleted file mode 100644 index 99b9762c..00000000 --- a/etw-gecko/src/stack_converter.rs +++ /dev/null @@ -1,157 +0,0 @@ -use fxprof_processed_profile::{CategoryPairHandle, Frame, FrameFlags, FrameInfo}; - -use super::jit_category_manager::{JsFrame, JsName}; -use super::lib_mappings::LibMappingsHierarchy; -use super::types::{StackFrame, StackMode}; - -#[derive(Debug, Clone, Copy)] -pub struct StackConverter { - user_category: CategoryPairHandle, - kernel_category: CategoryPairHandle, -} - -pub struct ConvertedStackIter<'a> { - inner: std::iter::Rev>, - lib_mappings: &'a LibMappingsHierarchy, - user_category: CategoryPairHandle, - kernel_category: CategoryPairHandle, - pending_frame: Option, - js_name_for_baseline_interpreter: Option, -} - -impl<'a> Iterator for ConvertedStackIter<'a> { - type Item = FrameInfo; - - // Implement this because it's called by StackDepthLimitingFrameIter - fn size_hint(&self) -> (usize, Option) { - // Use the slice length as the size hint. This is a bit of a lie, unfortunately. - // This iterator can yield more elements than self.inner if we add JS frames, - // or fewer elements if the original iterator contains TruncatedStackMarker frames. - // But it's a relatively good approximation. - self.inner.size_hint() - } - - fn next(&mut self) -> Option { - loop { - if let Some(pending_frame) = self.pending_frame.take() { - return Some(pending_frame); - } - let frame = self.inner.next()?; - let (mode, addr, lookup_address, from_ip) = match *frame { - StackFrame::InstructionPointer(addr, mode) => (mode, addr, addr, true), - StackFrame::ReturnAddress(addr, mode) => { - (mode, addr, addr.saturating_sub(1), false) - } - StackFrame::TruncatedStackMarker => continue, - }; - let (location, category, js_frame) = match mode { - StackMode::User => match self.lib_mappings.convert_address(lookup_address) { - Some((relative_lookup_address, info)) => { - let location = if from_ip { - let relative_address = relative_lookup_address; - Frame::RelativeAddressFromInstructionPointer( - info.lib_handle, - relative_address, - ) - } else { - let relative_address = relative_lookup_address + 1; - Frame::RelativeAddressFromReturnAddress( - info.lib_handle, - relative_address, - ) - }; - ( - location, - info.category.unwrap_or(self.user_category), - info.js_frame, - ) - } - None => { - let location = match from_ip { - true => Frame::InstructionPointer(addr), - false => Frame::ReturnAddress(addr), - }; - (location, self.user_category, None) - } - }, - StackMode::Kernel => { - let location = match from_ip { - true => Frame::InstructionPointer(addr), - false => Frame::ReturnAddress(addr), - }; - (location, self.kernel_category, None) - } - }; - let frame_info = FrameInfo { - frame: location, - category_pair: category, - flags: FrameFlags::empty(), - }; - - // Work around an imperfection in Spidermonkey's stack frames. - // We sometimes have missing BaselineInterpreterStubs in the OSR-into-BaselineInterpreter case. - // Usually, a BaselineInterpreter frame is directly preceded by a BaselineInterpreterStub frame. - // However, sometimes you get Regular(x) -> None -> None -> None -> BaselineInterpreter, - // without a BaselineInterpreterStub frame. In that case, the name "x" from the ancestor - // JsFrame::Regular (which is really an InterpreterStub frame for the C++ interpreter) - // should be used for the BaselineInterpreter frame. This will create a stack - // node with the right name, category and JS-only flag, and helps with correct attribution. - // Unfortunately it means that we'll have two prepended JS label frames for the same function - // in that case, but that's still better than accounting those samples to the wrong JS function. - let js_name = match js_frame { - Some(JsFrame::Regular(js_name)) => { - // Remember the name for a potentially upcoming unnamed BaselineInterpreter frame. - self.js_name_for_baseline_interpreter = Some(js_name); - Some(js_name) - } - Some(JsFrame::BaselineInterpreterStub(js_name)) => { - // Discard the name of an ancestor JS function. - self.js_name_for_baseline_interpreter = None; - Some(js_name) - } - Some(JsFrame::BaselineInterpreter) => self.js_name_for_baseline_interpreter.take(), - None => None, - }; - - let frame_info = match js_name { - Some(JsName::NonSelfHosted(js_name)) => { - // Prepend a JS frame. - self.pending_frame = Some(frame_info); - FrameInfo { - frame: Frame::Label(js_name), - category_pair: category, - flags: FrameFlags::IS_JS, - } - } - // Don't treat Spidermonkey "self-hosted" functions as JS (e.g. filter/map/push). - Some(JsName::SelfHosted(_)) | None => frame_info, - }; - return Some(frame_info); - } - } -} - -impl StackConverter { - pub fn new(user_category: CategoryPairHandle, kernel_category: CategoryPairHandle) -> Self { - Self { - user_category, - kernel_category, - } - } - - pub fn convert_stack<'a>( - &self, - stack: &'a [StackFrame], - lib_mappings: &'a LibMappingsHierarchy, - extra_first_frame: Option, - ) -> impl Iterator + 'a { - ConvertedStackIter { - inner: stack.iter().rev(), - lib_mappings, - user_category: self.user_category, - kernel_category: self.kernel_category, - pending_frame: extra_first_frame, - js_name_for_baseline_interpreter: None, - } - } -} diff --git a/etw-gecko/src/stack_depth_limiting_frame_iter.rs b/etw-gecko/src/stack_depth_limiting_frame_iter.rs deleted file mode 100644 index 7d90a7ab..00000000 --- a/etw-gecko/src/stack_depth_limiting_frame_iter.rs +++ /dev/null @@ -1,138 +0,0 @@ -use fxprof_processed_profile::{ - CategoryPairHandle, Frame, FrameFlags, FrameInfo, Profile, StringHandle, -}; - -/// Returns `Some((start_index, count))` if part of the stack should be elided -/// in order to limit the stack length to < 2.5 * N. -/// -/// The stack is partitioned into three pieces: -/// 1. N frames at the beginning which are kept. -/// 2. k * N frames in the middle which are elided and replaced with a placeholder. -/// 3. ~avg N frames at the end which are kept. -/// -/// The third piece is m frames, and k is chosen such that 0.5 * N <= m < 1.5 * N -fn should_elide_frames(full_len: usize) -> Option<(usize, usize)> { - if full_len >= N + N + N / 2 { - let elided_count = (full_len - N - N / 2) / N * N; - Some((N, elided_count)) - } else { - None - } -} - -#[test] -fn test_should_elide_frames() { - assert_eq!(should_elide_frames::<100>(100), None); - assert_eq!(should_elide_frames::<100>(220), None); - assert_eq!(should_elide_frames::<100>(249), None); - assert_eq!(should_elide_frames::<100>(250), Some((100, 100))); - assert_eq!(should_elide_frames::<100>(290), Some((100, 100))); - assert_eq!(should_elide_frames::<100>(349), Some((100, 100))); - assert_eq!(should_elide_frames::<100>(350), Some((100, 200))); - assert_eq!(should_elide_frames::<100>(352), Some((100, 200))); - assert_eq!(should_elide_frames::<100>(449), Some((100, 200))); - assert_eq!(should_elide_frames::<100>(450), Some((100, 300))); -} - -pub struct StackDepthLimitingFrameIter> { - inner: I, - category: CategoryPairHandle, - state: StackDepthLimitingFrameIterState, -} - -enum StackDepthLimitingFrameIterState { - BeforeElidedPiece { - index: usize, - first_elided_frame: usize, - elision_frame_string: StringHandle, - first_frame_after_elision: usize, - }, - AtElidedPiece { - elision_frame_string: StringHandle, - first_frame_after_elision: usize, - }, - NoMoreElision { - index: usize, - }, -} - -impl> StackDepthLimitingFrameIter { - pub fn new(profile: &mut Profile, iter: I, category: CategoryPairHandle) -> Self { - // Check if part of the stack should be elided, to limit the stack depth. - // Without such a limit, profiles with deep recursion may become too big - // to be processed. - // We limit to a depth of 500 frames, eliding chunks of 200 frames in the - // middle, keeping 200 frames at the start and 100 to 300 frames at the end. - let full_len = iter.size_hint().0; - let state = if let Some((first_elided_frame, elided_count)) = - should_elide_frames::<200>(full_len) - { - let first_frame_after_elision = first_elided_frame + elided_count; - let elision_frame_string = - profile.intern_string(&format!("({elided_count} frames elided)")); - StackDepthLimitingFrameIterState::BeforeElidedPiece { - index: 0, - first_elided_frame, - elision_frame_string, - first_frame_after_elision, - } - } else { - StackDepthLimitingFrameIterState::NoMoreElision { index: 0 } - }; - Self { - inner: iter, - category, - state, - } - } -} - -impl> Iterator for StackDepthLimitingFrameIter { - type Item = FrameInfo; - - fn next(&mut self) -> Option { - let frame = match &mut self.state { - StackDepthLimitingFrameIterState::BeforeElidedPiece { - index, - first_elided_frame, - elision_frame_string, - first_frame_after_elision, - } => { - let frame = self.inner.next()?; - *index += 1; - if *index == *first_elided_frame { - while *index < *first_frame_after_elision { - let _frame = self.inner.next()?; - *index += 1; - } - self.state = StackDepthLimitingFrameIterState::AtElidedPiece { - elision_frame_string: *elision_frame_string, - first_frame_after_elision: *first_frame_after_elision, - }; - } - frame - } - StackDepthLimitingFrameIterState::AtElidedPiece { - elision_frame_string, - first_frame_after_elision, - } => { - let frame = Frame::Label(*elision_frame_string); - self.state = StackDepthLimitingFrameIterState::NoMoreElision { - index: *first_frame_after_elision, - }; - return Some(FrameInfo { - frame, - category_pair: self.category, - flags: FrameFlags::empty(), - }); - } - StackDepthLimitingFrameIterState::NoMoreElision { index } => { - let frame = self.inner.next()?; - *index += 1; - frame - } - }; - - Some(frame) - } -} diff --git a/etw-gecko/src/timestamp_converter.rs b/etw-gecko/src/timestamp_converter.rs deleted file mode 100644 index f12b71d3..00000000 --- a/etw-gecko/src/timestamp_converter.rs +++ /dev/null @@ -1,23 +0,0 @@ -use fxprof_processed_profile::Timestamp; - -#[derive(Debug, Clone, Copy)] -pub struct TimestampConverter { - /// A reference timestamp, as a raw timestamp. - pub reference_raw: u64, - /// A "ticks per nanosecond" conversion factor. If raw values are in nanoseconds, this is 1. - pub raw_to_ns_factor: u64, -} - -impl TimestampConverter { - pub fn convert_raw(&self, raw: u64) -> Timestamp { - Timestamp::from_nanos_since_reference( - raw.saturating_sub(self.reference_raw) * self.raw_to_ns_factor, - ) - } - - pub fn convert_us(&self, time_us: u64) -> Timestamp { - Timestamp::from_nanos_since_reference( - (time_us * 1000).saturating_sub(self.reference_raw * self.raw_to_ns_factor), - ) - } -} diff --git a/etw-gecko/src/types.rs b/etw-gecko/src/types.rs deleted file mode 100644 index 82367897..00000000 --- a/etw-gecko/src/types.rs +++ /dev/null @@ -1,19 +0,0 @@ -use fxhash::FxHasher; - -use std::collections::HashMap; -use std::hash::BuildHasherDefault; - -pub type FastHashMap = HashMap>; - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub enum StackMode { - User, - Kernel, -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] -pub enum StackFrame { - InstructionPointer(u64, StackMode), - ReturnAddress(u64, StackMode), - TruncatedStackMarker, -} diff --git a/etw-gecko/src/unresolved_samples.rs b/etw-gecko/src/unresolved_samples.rs deleted file mode 100644 index 57aff930..00000000 --- a/etw-gecko/src/unresolved_samples.rs +++ /dev/null @@ -1,265 +0,0 @@ -use std::collections::hash_map::Entry; - -use fxprof_processed_profile::{CpuDelta, FrameInfo, ThreadHandle, Timestamp}; - -use super::process_sample_data::RssStatMember; -use super::types::{FastHashMap, StackFrame, StackMode}; - -#[derive(Debug, Clone, Default)] -pub struct UnresolvedSamples { - samples_and_markers: Vec, - prev_sample_info_per_thread: FastHashMap, -} - -#[derive(Debug, Clone)] -struct PreviousSampleInfo { - stack: UnresolvedStackHandle, - prev_sample_index_if_zero_cpu: Option, -} - -impl UnresolvedSamples { - pub fn into_inner(self) -> Vec { - self.samples_and_markers - } - - pub fn is_empty(&self) -> bool { - self.samples_and_markers.is_empty() - } - - #[allow(clippy::too_many_arguments)] - pub fn add_sample( - &mut self, - thread_handle: ThreadHandle, - timestamp: Timestamp, - timestamp_mono: u64, - stack: UnresolvedStackHandle, - cpu_delta: CpuDelta, - weight: i32, - extra_label_frame: Option, - ) { - let sample_index = self.samples_and_markers.len(); - self.samples_and_markers.push(UnresolvedSampleOrMarker { - thread_handle, - timestamp, - timestamp_mono, - stack, - extra_label_frame, - sample_or_marker: SampleOrMarker::Sample(SampleData { weight, cpu_delta }), - }); - self.prev_sample_info_per_thread.insert( - thread_handle, - PreviousSampleInfo { - stack, - prev_sample_index_if_zero_cpu: (cpu_delta == CpuDelta::ZERO) - .then_some(sample_index), - }, - ); - } - - #[allow(unused)] - pub fn add_sample_same_stack_zero_cpu( - &mut self, - thread_handle: ThreadHandle, - timestamp: Timestamp, - timestamp_mono: u64, - weight: i32, - extra_label_frame: Option, - ) { - match self.prev_sample_info_per_thread.entry(thread_handle) { - Entry::Occupied(mut entry) => { - let sample_info = entry.get_mut(); - if let Some(sample_index) = sample_info.prev_sample_index_if_zero_cpu { - let sample = &mut self.samples_and_markers[sample_index]; - sample.timestamp = timestamp; - let SampleOrMarker::Sample(ref mut data) = &mut sample.sample_or_marker else { panic!() }; - data.weight += weight; - } else { - let stack = sample_info.stack; - let sample_index = self.samples_and_markers.len(); - self.samples_and_markers.push(UnresolvedSampleOrMarker { - thread_handle, - timestamp, - timestamp_mono, - stack, - extra_label_frame, - sample_or_marker: SampleOrMarker::Sample(SampleData { - weight, - cpu_delta: CpuDelta::ZERO, - }), - }); - sample_info.prev_sample_index_if_zero_cpu = Some(sample_index); - } - } - Entry::Vacant(entry) => { - let stack = UnresolvedStackHandle::EMPTY; - let sample_index = self.samples_and_markers.len(); - self.samples_and_markers.push(UnresolvedSampleOrMarker { - thread_handle, - timestamp, - timestamp_mono, - stack, - extra_label_frame, - sample_or_marker: SampleOrMarker::Sample(SampleData { - weight, - cpu_delta: CpuDelta::ZERO, - }), - }); - entry.insert(PreviousSampleInfo { - stack, - prev_sample_index_if_zero_cpu: Some(sample_index), - }); - } - } - } - - #[allow(clippy::too_many_arguments)] - pub fn add_rss_stat_marker( - &mut self, - thread_handle: ThreadHandle, - timestamp: Timestamp, - timestamp_mono: u64, - stack: UnresolvedStackHandle, - rss_member: RssStatMember, - rss_size: i64, - rss_delta: i64, - ) { - self.samples_and_markers.push(UnresolvedSampleOrMarker { - thread_handle, - timestamp, - timestamp_mono, - stack, - extra_label_frame: None, - sample_or_marker: SampleOrMarker::RssStatMarker(RssStatMarkerData { - member: rss_member, - size: rss_size, - delta: rss_delta, - }), - }); - } - - pub fn add_other_event_marker( - &mut self, - thread_handle: ThreadHandle, - timestamp: Timestamp, - timestamp_mono: u64, - stack: UnresolvedStackHandle, - attr_index: usize, - ) { - self.samples_and_markers.push(UnresolvedSampleOrMarker { - thread_handle, - timestamp, - timestamp_mono, - stack, - extra_label_frame: None, - sample_or_marker: SampleOrMarker::OtherEventMarker(OtherEventMarkerData { attr_index }), - }); - } -} - -#[derive(Debug, Clone)] -pub struct UnresolvedSampleOrMarker { - pub thread_handle: ThreadHandle, - pub timestamp: Timestamp, - pub timestamp_mono: u64, - pub stack: UnresolvedStackHandle, - pub extra_label_frame: Option, - pub sample_or_marker: SampleOrMarker, -} - -#[derive(Debug, Clone)] -pub enum SampleOrMarker { - Sample(SampleData), - RssStatMarker(RssStatMarkerData), - OtherEventMarker(OtherEventMarkerData), -} - -#[derive(Debug, Clone)] -pub struct SampleData { - pub cpu_delta: CpuDelta, - pub weight: i32, -} - -#[derive(Debug, Clone)] -pub struct RssStatMarkerData { - pub member: RssStatMember, - pub size: i64, - pub delta: i64, -} - -#[derive(Debug, Clone)] -pub struct OtherEventMarkerData { - pub attr_index: usize, -} - -#[derive(Debug, Clone)] -pub struct UnresolvedRssStatMarker { - pub thread_handle: ThreadHandle, - pub timestamp: Timestamp, - pub timestamp_mono: u64, - pub stack: UnresolvedStackHandle, -} - -#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)] -pub struct UnresolvedStackHandle(u32); - -impl UnresolvedStackHandle { - /// Represents the empty stack / the root stack node - pub const EMPTY: Self = Self(u32::MAX); -} - -#[derive(Debug, Clone, Default)] -pub struct UnresolvedStacks { - pub stacks: Vec<(UnresolvedStackHandle, StackFrame)>, // (prefix, frame) - pub stack_lookup: FastHashMap<(UnresolvedStackHandle, StackFrame), UnresolvedStackHandle>, // (prefix, frame) -> stack index -} - -impl UnresolvedStacks { - /// Get the `UnresolvedStackHandle` for a stack. The stack must be ordered from - /// caller-most to callee-most ("outside to inside"). - pub fn convert(&mut self, frames: impl Iterator) -> UnresolvedStackHandle { - let mut prefix = UnresolvedStackHandle::EMPTY; - for frame in frames { - let x = (prefix, frame); - let node = *self.stack_lookup.entry(x).or_insert_with(|| { - let new_index = self.stacks.len() as u32; - self.stacks.push(x); - UnresolvedStackHandle(new_index) - }); - prefix = node; - } - prefix - } - - /// Get the `UnresolvedStackHandle` for a stack, skipping any kernel frames. - /// The stack must be ordered from caller-most to callee-most ("outside to inside"). - pub fn convert_no_kernel( - &mut self, - frames: impl Iterator, - ) -> UnresolvedStackHandle { - let mut prefix = UnresolvedStackHandle::EMPTY; - for frame in frames { - match frame { - StackFrame::InstructionPointer(_, StackMode::Kernel) => continue, - StackFrame::ReturnAddress(_, StackMode::Kernel) => continue, - _ => {} - } - let x = (prefix, frame); - let node = *self.stack_lookup.entry(x).or_insert_with(|| { - let new_index = self.stacks.len() as u32; - self.stacks.push(x); - UnresolvedStackHandle(new_index) - }); - prefix = node; - } - prefix - } - - // Appends the stack to `buf`, starting with the callee-most frame. - pub fn convert_back(&self, mut stack_index: UnresolvedStackHandle, buf: &mut Vec) { - while stack_index != UnresolvedStackHandle::EMPTY { - let (prefix, frame) = self.stacks[stack_index.0 as usize]; - buf.push(frame); - stack_index = prefix; - } - } -} From 22c67a14f2483d24200b730a0a206688f618c13d Mon Sep 17 00:00:00 2001 From: Vladimir Vukicevic Date: Sat, 4 May 2024 09:14:25 -0700 Subject: [PATCH 02/10] Fixes from while this was inside samply --- etw-reader/src/custom_schemas.rs | 1077 +++++++++++++++---------- etw-reader/src/etw_types.rs | 594 +++++++------- etw-reader/src/parser.rs | 1267 +++++++++++++++--------------- etw-reader/src/property.rs | 109 +-- etw-reader/src/schema.rs | 931 +++++++++++----------- etw-reader/src/sddl.rs | 128 +-- etw-reader/src/tdh.rs | 269 +++---- etw-reader/src/tdh_types.rs | 442 ++++++----- etw-reader/src/traits.rs | 38 +- etw-reader/src/utils.rs | 183 ++--- 10 files changed, 2672 insertions(+), 2366 deletions(-) diff --git a/etw-reader/src/custom_schemas.rs b/etw-reader/src/custom_schemas.rs index ae1fcf10..0db0d0c8 100644 --- a/etw-reader/src/custom_schemas.rs +++ b/etw-reader/src/custom_schemas.rs @@ -1,430 +1,647 @@ -use windows::core::GUID; - -use crate::{etw_types::DecodingSource, schema::EventSchema, tdh_types::{Property, PropertyDesc, PrimitiveDesc, PropertyFlags, TdhInType, TdhOutType, PropertyLength}}; - -struct PropDesc { - name: &'static str, - in_type: TdhInType, - out_type: TdhOutType, -} - -pub struct ImageID {} - -const ImageID_PROPS: [PropDesc; 5] = [ - PropDesc{ name: "ImageBase", in_type: TdhInType::InTypePointer, out_type: TdhOutType::OutTypeHexInt64}, - PropDesc{ name: "ImageSize", in_type: TdhInType::InTypeUInt32, out_type: TdhOutType::OutTypeUInt32}, - PropDesc{ name: "Unknown", in_type: TdhInType::InTypePointer, out_type: TdhOutType::OutTypeUInt32}, - PropDesc{ name: "TimeDateStamp", in_type: TdhInType::InTypeUInt32, out_type: TdhOutType::OutTypeUInt32}, - PropDesc{ name: "OriginalFileName", in_type: TdhInType::InTypeUnicodeString, out_type: TdhOutType::OutTypeString}, - ]; - -impl EventSchema for ImageID { - fn provider_guid(&self) -> GUID { - GUID::from("b3e675d7-2554-4f18-830b-2762732560de") - } - - fn event_id(&self) -> u16 { - 0 - } - - fn opcode(&self) -> u8 { - 0 - } - - fn event_version(&self) -> u8 { - 2 - } - - fn level(&self) -> u8 { 0 } - - fn decoding_source(&self) -> DecodingSource { - panic!() - } - - fn provider_name(&self) -> String { - "KernelTraceControl".to_owned() - } - - fn task_name(&self) -> String { - "ImageID".to_owned() - } - - fn opcode_name(&self) -> String { - "".to_string() - } - - fn property_count(&self) -> u32 { - ImageID_PROPS.len() as u32 - } - - fn property(&self, index: u32) -> Property { - let prop = &ImageID_PROPS[index as usize]; - Property { name: prop.name.to_owned(), - desc: PropertyDesc::Primitive(PrimitiveDesc{ in_type: prop.in_type, - out_type: prop.out_type,}), - length: PropertyLength::Length(0), - count: 1, - map_info: None, - flags: PropertyFlags::empty()} - } -} - - -pub struct DbgID {} - -const DbgID_PROPS: [PropDesc; 5] = [ - PropDesc{ name: "ImageBase", in_type: TdhInType::InTypeUInt64, out_type: TdhOutType::OutTypeHexInt64}, - PropDesc{ name: "ProcessId", in_type: TdhInType::InTypeUInt32, out_type: TdhOutType::OutTypeUInt32}, - PropDesc{ name: "GuidSig", in_type: TdhInType::InTypeGuid, out_type: TdhOutType::OutTypeGuid}, - PropDesc{ name: "Age", in_type: TdhInType::InTypeUInt32, out_type: TdhOutType::OutTypeUInt32}, - PropDesc{ name: "PdbFileName", in_type: TdhInType::InTypeAnsiString, out_type: TdhOutType::OutTypeString}, - ]; - -impl EventSchema for DbgID { - fn provider_guid(&self) -> GUID { - GUID::from("b3e675d7-2554-4f18-830b-2762732560de") - } - - fn event_id(&self) -> u16 { - 0 - } - - fn opcode(&self) -> u8 { - 36 - } - - fn event_version(&self) -> u8 { - 2 - } - - fn level(&self) -> u8 { 0 } - - fn decoding_source(&self) -> DecodingSource { - panic!() - } - - fn provider_name(&self) -> String { - "KernelTraceControl".to_owned() - } - - fn task_name(&self) -> String { - "ImageID".to_owned() - } - - fn opcode_name(&self) -> String { - "DbgID_RSDS".to_string() - } - - fn property_count(&self) -> u32 { - DbgID_PROPS.len() as u32 - } - - fn property(&self, index: u32) -> Property { - let prop = &DbgID_PROPS[index as usize]; - Property { name: prop.name.to_owned(), - desc: PropertyDesc::Primitive(PrimitiveDesc{ in_type: prop.in_type, - out_type: prop.out_type,}), - count: 1, - length: PropertyLength::Length(0), - map_info: None, - flags: PropertyFlags::empty()} - } -} - - // Info about EventInfo comes from SymbolTraceEventParser.cs in PerfView - // It contains an EVENT_DESCRIPTOR -pub struct EventInfo {} -const EventInfo_PROPS: [PropDesc; 9] = [ - PropDesc{ name: "ProviderGuid", in_type: TdhInType::InTypeGuid, out_type: TdhOutType::OutTypeGuid}, - PropDesc{ name: "EventGuid", in_type: TdhInType::InTypeGuid, out_type: TdhOutType::OutTypeGuid}, - PropDesc{ name: "EventDescriptorId", in_type: TdhInType::InTypeUInt16, out_type: TdhOutType::OutTypeUInt16}, - PropDesc{ name: "EventDescriptor.Version", in_type: TdhInType::InTypeUInt8, out_type: TdhOutType::OutTypeUInt8}, - PropDesc{ name: "EventDescriptor.Channel", in_type: TdhInType::InTypeUInt8, out_type: TdhOutType::OutTypeUInt8}, - PropDesc{ name: "EventDescriptor.Level", in_type: TdhInType::InTypeUInt8, out_type: TdhOutType::OutTypeUInt8}, - PropDesc{ name: "EventDescriptor.Opcode", in_type: TdhInType::InTypeUInt8, out_type: TdhOutType::OutTypeUInt8}, - PropDesc{ name: "EventDescriptor.Task", in_type: TdhInType::InTypeUInt16, out_type: TdhOutType::OutTypeUInt16}, - PropDesc{ name: "EventDescriptor.Keyword", in_type: TdhInType::InTypeUInt64, out_type: TdhOutType::OutTypeHexInt64}, - - ]; -impl EventSchema for EventInfo { - fn provider_guid(&self) -> GUID { - GUID::from("bbccf6c1-6cd1-48c4-80ff-839482e37671") - } - - fn event_id(&self) -> u16 { - 0 - } - - fn opcode(&self) -> u8 { - 32 - } - - fn event_version(&self) -> u8 { - 0 - } - - fn level(&self) -> u8 { 0 } - - fn decoding_source(&self) -> DecodingSource { - panic!() - } - - fn provider_name(&self) -> String { - "KernelTraceControl".to_owned() - } - - fn task_name(&self) -> String { - "MetaData".to_owned() - } - - fn opcode_name(&self) -> String { - "EventInfo".to_string() - } - - fn property_count(&self) -> u32 { - EventInfo_PROPS.len() as u32 - } - - fn is_event_metadata(&self) -> bool { - true - } - - fn property(&self, index: u32) -> Property { - let prop = &EventInfo_PROPS[index as usize]; - Property { name: prop.name.to_owned(), - desc: PropertyDesc::Primitive(PrimitiveDesc{ in_type: prop.in_type, - out_type: prop.out_type,}), - count: 1, - length: PropertyLength::Length(0), - map_info: None, - flags: PropertyFlags::empty()} - } -} -// We could override ThreadStop using the same properties to get a ThreadName there too. -pub struct ThreadStart {} - -const Thread_PROPS: [PropDesc; 15] = [ - PropDesc{ name: "ProcessId", in_type: TdhInType::InTypeUInt32, out_type: TdhOutType::OutTypeHexInt32}, - PropDesc{ name: "TThreadId", in_type: TdhInType::InTypeUInt32, out_type: TdhOutType::OutTypeHexInt32}, - PropDesc{ name: "StackBase", in_type: TdhInType::InTypePointer, out_type: TdhOutType::OutTypeNull}, - PropDesc{ name: "StackLimit", in_type: TdhInType::InTypePointer, out_type: TdhOutType::OutTypeNull}, - PropDesc{ name: "UserStackBase", in_type: TdhInType::InTypePointer, out_type: TdhOutType::OutTypeNull}, - PropDesc{ name: "UserStackLimit", in_type: TdhInType::InTypePointer, out_type: TdhOutType::OutTypeNull}, - PropDesc{ name: "Affinity", in_type: TdhInType::InTypePointer, out_type: TdhOutType::OutTypeNull}, - PropDesc{ name: "Win32StartAddr", in_type: TdhInType::InTypePointer, out_type: TdhOutType::OutTypeNull}, - PropDesc{ name: "TebBase", in_type: TdhInType::InTypePointer, out_type: TdhOutType::OutTypeNull}, - PropDesc{ name: "SubProcessTag", in_type: TdhInType::InTypeUInt32, out_type: TdhOutType::OutTypeHexInt32}, - PropDesc{ name: "BasePriority", in_type: TdhInType::InTypeUInt8, out_type: TdhOutType::OutTypeNull}, - PropDesc{ name: "PagePriority", in_type: TdhInType::InTypeUInt8, out_type: TdhOutType::OutTypeNull}, - PropDesc{ name: "IoPriority", in_type: TdhInType::InTypeUInt8, out_type: TdhOutType::OutTypeNull}, - PropDesc{ name: "ThreadFlags", in_type: TdhInType::InTypeUInt8, out_type: TdhOutType::OutTypeNull}, - PropDesc{ name: "ThreadName", in_type: TdhInType::InTypeUnicodeString, out_type: TdhOutType::OutTypeString}, - ]; - - - -impl EventSchema for ThreadStart { - fn provider_guid(&self) -> GUID { - GUID::from("3D6FA8D1-FE05-11D0-9DDA-00C04FD7BA7C") - } - - fn event_id(&self) -> u16 { - 0 - } - - fn opcode(&self) -> u8 { - 3 - } - - fn event_version(&self) -> u8 { - 3 - } - - fn level(&self) -> u8 { 0 } - - - fn decoding_source(&self) -> DecodingSource { - panic!() - } - - fn provider_name(&self) -> String { - "MSNT_SystemTrace".to_owned() - } - - fn task_name(&self) -> String { - "Thread".to_owned() - } - - fn opcode_name(&self) -> String { - "DCStart".to_string() - } - - fn property_count(&self) -> u32 { - Thread_PROPS.len() as u32 - } - - fn property(&self, index: u32) -> Property { - let prop = &Thread_PROPS[index as usize]; - Property { name: prop.name.to_owned(), - desc: PropertyDesc::Primitive(PrimitiveDesc{ in_type: prop.in_type, - out_type: prop.out_type,}), - length: PropertyLength::Length(0), - count:1, - map_info: None, - flags: PropertyFlags::empty()} - } -} - -// from umdprovider.h -pub struct D3DUmdLogging_MapAllocation {} - -const D3DUmdLogging_PROPS: [PropDesc; 6] = [ - PropDesc{ name: "hD3DAllocation", in_type: TdhInType::InTypeUInt64, out_type: TdhOutType::OutTypeHexInt64}, - PropDesc{ name: "hDxgAllocation", in_type: TdhInType::InTypeUInt64, out_type: TdhOutType::OutTypeHexInt64}, - PropDesc{ name: "Offset", in_type: TdhInType::InTypeUInt64, out_type: TdhOutType::OutTypeUInt64}, - PropDesc{ name: "Size", in_type: TdhInType::InTypeUInt32, out_type: TdhOutType::OutTypeUInt64}, - // XXX: use an enum for these - PropDesc{ name: "Usage", in_type: TdhInType::InTypeUInt32, out_type: TdhOutType::OutTypeUInt32}, - PropDesc{ name: "Semantic", in_type: TdhInType::InTypeUInt32, out_type: TdhOutType::OutTypeUInt32}, - ]; - -impl EventSchema for D3DUmdLogging_MapAllocation { - fn provider_guid(&self) -> GUID { - GUID::from("A688EE40-D8D9-4736-B6F9-6B74935BA3B1") - } - - fn event_id(&self) -> u16 { 1 } - - fn event_version(&self) -> u8 { 0 } - - fn opcode(&self) -> u8 { 1 } - - fn level(&self) -> u8 { 0 } - - fn decoding_source(&self) -> DecodingSource { - panic!() - } - - fn provider_name(&self) -> String { - "D3DUmdLogging".to_owned() - } - - fn task_name(&self) -> String { - "MapAllocation".to_owned() - } - - fn opcode_name(&self) -> String { - "Start".to_string() - } - - fn property_count(&self) -> u32 { - D3DUmdLogging_PROPS.len() as u32 - } - - fn property(&self, index: u32) -> Property { - let prop = &D3DUmdLogging_PROPS[index as usize]; - Property { name: prop.name.to_owned(), - desc: PropertyDesc::Primitive(PrimitiveDesc{ in_type: prop.in_type, - out_type: prop.out_type,}), - count: 1, - length: PropertyLength::Length(0), - map_info: None, - flags: PropertyFlags::empty()} - } -} - -pub struct D3DUmdLogging_RundownAllocation {} - -impl EventSchema for D3DUmdLogging_RundownAllocation { - fn provider_guid(&self) -> GUID { - GUID::from("A688EE40-D8D9-4736-B6F9-6B74935BA3B1") - } - - fn event_id(&self) -> u16 { 2 } - - fn event_version(&self) -> u8 { 0 } - - fn opcode(&self) -> u8 { 3 } - - fn level(&self) -> u8 { 0 } - - fn decoding_source(&self) -> DecodingSource { - panic!() - } - - fn provider_name(&self) -> String { - "D3DUmdLogging".to_owned() - } - - fn task_name(&self) -> String { - "MapAllocation".to_owned() - } - - fn opcode_name(&self) -> String { - "DC Start".to_string() - } - - fn property_count(&self) -> u32 { - D3DUmdLogging_PROPS.len() as u32 - } - - fn property(&self, index: u32) -> Property { - let prop = &D3DUmdLogging_PROPS[index as usize]; - Property { name: prop.name.to_owned(), - desc: PropertyDesc::Primitive(PrimitiveDesc{ in_type: prop.in_type, - out_type: prop.out_type,}), - count: 1, - length: PropertyLength::Length(0), - map_info: None, - flags: PropertyFlags::empty()} - } -} - -pub struct D3DUmdLogging_UnmapAllocation {} - - -impl EventSchema for D3DUmdLogging_UnmapAllocation { - fn provider_guid(&self) -> GUID { - GUID::from("A688EE40-D8D9-4736-B6F9-6B74935BA3B1") - } - - fn event_id(&self) -> u16 { 3 } - - fn event_version(&self) -> u8 { 0 } - - fn opcode(&self) -> u8 { 2 } - - fn level(&self) -> u8 { 0 } - - fn decoding_source(&self) -> DecodingSource { - panic!() - } - - fn provider_name(&self) -> String { - "D3DUmdLogging".to_owned() - } - - fn task_name(&self) -> String { - "MapAllocation".to_owned() - } - - fn opcode_name(&self) -> String { - "End".to_string() - } - - fn property_count(&self) -> u32 { - D3DUmdLogging_PROPS.len() as u32 - } - - fn property(&self, index: u32) -> Property { - let prop = &D3DUmdLogging_PROPS[index as usize]; - Property { name: prop.name.to_owned(), - desc: PropertyDesc::Primitive(PrimitiveDesc{ in_type: prop.in_type, - out_type: prop.out_type,}), - count: 1, - length: PropertyLength::Length(0), - map_info: None, - flags: PropertyFlags::empty()} - } -} - +use windows::core::GUID; + +use super::etw_types::DecodingSource; +use super::schema::EventSchema; +use super::tdh_types::{ + PrimitiveDesc, Property, PropertyDesc, PropertyFlags, PropertyLength, TdhInType, TdhOutType, +}; + +struct PropDesc { + name: &'static str, + in_type: TdhInType, + out_type: TdhOutType, +} + +pub struct ImageID {} + +const ImageID_PROPS: [PropDesc; 5] = [ + PropDesc { + name: "ImageBase", + in_type: TdhInType::InTypePointer, + out_type: TdhOutType::OutTypeHexInt64, + }, + PropDesc { + name: "ImageSize", + in_type: TdhInType::InTypeUInt32, + out_type: TdhOutType::OutTypeUInt32, + }, + PropDesc { + name: "Unknown", + in_type: TdhInType::InTypePointer, + out_type: TdhOutType::OutTypeUInt32, + }, + PropDesc { + name: "TimeDateStamp", + in_type: TdhInType::InTypeUInt32, + out_type: TdhOutType::OutTypeUInt32, + }, + PropDesc { + name: "OriginalFileName", + in_type: TdhInType::InTypeUnicodeString, + out_type: TdhOutType::OutTypeString, + }, +]; + +impl EventSchema for ImageID { + fn provider_guid(&self) -> GUID { + GUID::from("b3e675d7-2554-4f18-830b-2762732560de") + } + + fn event_id(&self) -> u16 { + 0 + } + + fn opcode(&self) -> u8 { + 0 + } + + fn event_version(&self) -> u8 { + 2 + } + + fn level(&self) -> u8 { + 0 + } + + fn decoding_source(&self) -> DecodingSource { + panic!() + } + + fn provider_name(&self) -> String { + "KernelTraceControl".to_owned() + } + + fn task_name(&self) -> String { + "ImageID".to_owned() + } + + fn opcode_name(&self) -> String { + "".to_string() + } + + fn property_count(&self) -> u32 { + ImageID_PROPS.len() as u32 + } + + fn property(&self, index: u32) -> Property { + let prop = &ImageID_PROPS[index as usize]; + Property { + name: prop.name.to_owned(), + desc: PropertyDesc::Primitive(PrimitiveDesc { + in_type: prop.in_type, + out_type: prop.out_type, + }), + length: PropertyLength::Length(0), + count: 1, + map_info: None, + flags: PropertyFlags::empty(), + } + } +} + +pub struct DbgID {} + +const DbgID_PROPS: [PropDesc; 5] = [ + PropDesc { + name: "ImageBase", + in_type: TdhInType::InTypeUInt64, + out_type: TdhOutType::OutTypeHexInt64, + }, + PropDesc { + name: "ProcessId", + in_type: TdhInType::InTypeUInt32, + out_type: TdhOutType::OutTypeUInt32, + }, + PropDesc { + name: "GuidSig", + in_type: TdhInType::InTypeGuid, + out_type: TdhOutType::OutTypeGuid, + }, + PropDesc { + name: "Age", + in_type: TdhInType::InTypeUInt32, + out_type: TdhOutType::OutTypeUInt32, + }, + PropDesc { + name: "PdbFileName", + in_type: TdhInType::InTypeAnsiString, + out_type: TdhOutType::OutTypeString, + }, +]; + +impl EventSchema for DbgID { + fn provider_guid(&self) -> GUID { + GUID::from("b3e675d7-2554-4f18-830b-2762732560de") + } + + fn event_id(&self) -> u16 { + 0 + } + + fn opcode(&self) -> u8 { + 36 + } + + fn event_version(&self) -> u8 { + 2 + } + + fn level(&self) -> u8 { + 0 + } + + fn decoding_source(&self) -> DecodingSource { + panic!() + } + + fn provider_name(&self) -> String { + "KernelTraceControl".to_owned() + } + + fn task_name(&self) -> String { + "ImageID".to_owned() + } + + fn opcode_name(&self) -> String { + "DbgID_RSDS".to_string() + } + + fn property_count(&self) -> u32 { + DbgID_PROPS.len() as u32 + } + + fn property(&self, index: u32) -> Property { + let prop = &DbgID_PROPS[index as usize]; + Property { + name: prop.name.to_owned(), + desc: PropertyDesc::Primitive(PrimitiveDesc { + in_type: prop.in_type, + out_type: prop.out_type, + }), + count: 1, + length: PropertyLength::Length(0), + map_info: None, + flags: PropertyFlags::empty(), + } + } +} + +// Info about EventInfo comes from SymbolTraceEventParser.cs in PerfView +// It contains an EVENT_DESCRIPTOR +pub struct EventInfo {} +const EventInfo_PROPS: [PropDesc; 9] = [ + PropDesc { + name: "ProviderGuid", + in_type: TdhInType::InTypeGuid, + out_type: TdhOutType::OutTypeGuid, + }, + PropDesc { + name: "EventGuid", + in_type: TdhInType::InTypeGuid, + out_type: TdhOutType::OutTypeGuid, + }, + PropDesc { + name: "EventDescriptorId", + in_type: TdhInType::InTypeUInt16, + out_type: TdhOutType::OutTypeUInt16, + }, + PropDesc { + name: "EventDescriptor.Version", + in_type: TdhInType::InTypeUInt8, + out_type: TdhOutType::OutTypeUInt8, + }, + PropDesc { + name: "EventDescriptor.Channel", + in_type: TdhInType::InTypeUInt8, + out_type: TdhOutType::OutTypeUInt8, + }, + PropDesc { + name: "EventDescriptor.Level", + in_type: TdhInType::InTypeUInt8, + out_type: TdhOutType::OutTypeUInt8, + }, + PropDesc { + name: "EventDescriptor.Opcode", + in_type: TdhInType::InTypeUInt8, + out_type: TdhOutType::OutTypeUInt8, + }, + PropDesc { + name: "EventDescriptor.Task", + in_type: TdhInType::InTypeUInt16, + out_type: TdhOutType::OutTypeUInt16, + }, + PropDesc { + name: "EventDescriptor.Keyword", + in_type: TdhInType::InTypeUInt64, + out_type: TdhOutType::OutTypeHexInt64, + }, +]; +impl EventSchema for EventInfo { + fn provider_guid(&self) -> GUID { + GUID::from("bbccf6c1-6cd1-48c4-80ff-839482e37671") + } + + fn event_id(&self) -> u16 { + 0 + } + + fn opcode(&self) -> u8 { + 32 + } + + fn event_version(&self) -> u8 { + 0 + } + + fn level(&self) -> u8 { + 0 + } + + fn decoding_source(&self) -> DecodingSource { + panic!() + } + + fn provider_name(&self) -> String { + "KernelTraceControl".to_owned() + } + + fn task_name(&self) -> String { + "MetaData".to_owned() + } + + fn opcode_name(&self) -> String { + "EventInfo".to_string() + } + + fn property_count(&self) -> u32 { + EventInfo_PROPS.len() as u32 + } + + fn is_event_metadata(&self) -> bool { + true + } + + fn property(&self, index: u32) -> Property { + let prop = &EventInfo_PROPS[index as usize]; + Property { + name: prop.name.to_owned(), + desc: PropertyDesc::Primitive(PrimitiveDesc { + in_type: prop.in_type, + out_type: prop.out_type, + }), + count: 1, + length: PropertyLength::Length(0), + map_info: None, + flags: PropertyFlags::empty(), + } + } +} +// We could override ThreadStop using the same properties to get a ThreadName there too. +pub struct ThreadStart {} + +const Thread_PROPS: [PropDesc; 15] = [ + PropDesc { + name: "ProcessId", + in_type: TdhInType::InTypeUInt32, + out_type: TdhOutType::OutTypeHexInt32, + }, + PropDesc { + name: "TThreadId", + in_type: TdhInType::InTypeUInt32, + out_type: TdhOutType::OutTypeHexInt32, + }, + PropDesc { + name: "StackBase", + in_type: TdhInType::InTypePointer, + out_type: TdhOutType::OutTypeNull, + }, + PropDesc { + name: "StackLimit", + in_type: TdhInType::InTypePointer, + out_type: TdhOutType::OutTypeNull, + }, + PropDesc { + name: "UserStackBase", + in_type: TdhInType::InTypePointer, + out_type: TdhOutType::OutTypeNull, + }, + PropDesc { + name: "UserStackLimit", + in_type: TdhInType::InTypePointer, + out_type: TdhOutType::OutTypeNull, + }, + PropDesc { + name: "Affinity", + in_type: TdhInType::InTypePointer, + out_type: TdhOutType::OutTypeNull, + }, + PropDesc { + name: "Win32StartAddr", + in_type: TdhInType::InTypePointer, + out_type: TdhOutType::OutTypeNull, + }, + PropDesc { + name: "TebBase", + in_type: TdhInType::InTypePointer, + out_type: TdhOutType::OutTypeNull, + }, + PropDesc { + name: "SubProcessTag", + in_type: TdhInType::InTypeUInt32, + out_type: TdhOutType::OutTypeHexInt32, + }, + PropDesc { + name: "BasePriority", + in_type: TdhInType::InTypeUInt8, + out_type: TdhOutType::OutTypeNull, + }, + PropDesc { + name: "PagePriority", + in_type: TdhInType::InTypeUInt8, + out_type: TdhOutType::OutTypeNull, + }, + PropDesc { + name: "IoPriority", + in_type: TdhInType::InTypeUInt8, + out_type: TdhOutType::OutTypeNull, + }, + PropDesc { + name: "ThreadFlags", + in_type: TdhInType::InTypeUInt8, + out_type: TdhOutType::OutTypeNull, + }, + PropDesc { + name: "ThreadName", + in_type: TdhInType::InTypeUnicodeString, + out_type: TdhOutType::OutTypeString, + }, +]; + +impl EventSchema for ThreadStart { + fn provider_guid(&self) -> GUID { + GUID::from("3D6FA8D1-FE05-11D0-9DDA-00C04FD7BA7C") + } + + fn event_id(&self) -> u16 { + 0 + } + + fn opcode(&self) -> u8 { + 3 + } + + fn event_version(&self) -> u8 { + 3 + } + + fn level(&self) -> u8 { + 0 + } + + fn decoding_source(&self) -> DecodingSource { + panic!() + } + + fn provider_name(&self) -> String { + "MSNT_SystemTrace".to_owned() + } + + fn task_name(&self) -> String { + "Thread".to_owned() + } + + fn opcode_name(&self) -> String { + "DCStart".to_string() + } + + fn property_count(&self) -> u32 { + Thread_PROPS.len() as u32 + } + + fn property(&self, index: u32) -> Property { + let prop = &Thread_PROPS[index as usize]; + Property { + name: prop.name.to_owned(), + desc: PropertyDesc::Primitive(PrimitiveDesc { + in_type: prop.in_type, + out_type: prop.out_type, + }), + length: PropertyLength::Length(0), + count: 1, + map_info: None, + flags: PropertyFlags::empty(), + } + } +} + +// from umdprovider.h +pub struct D3DUmdLogging_MapAllocation {} + +const D3DUmdLogging_PROPS: [PropDesc; 6] = [ + PropDesc { + name: "hD3DAllocation", + in_type: TdhInType::InTypeUInt64, + out_type: TdhOutType::OutTypeHexInt64, + }, + PropDesc { + name: "hDxgAllocation", + in_type: TdhInType::InTypeUInt64, + out_type: TdhOutType::OutTypeHexInt64, + }, + PropDesc { + name: "Offset", + in_type: TdhInType::InTypeUInt64, + out_type: TdhOutType::OutTypeUInt64, + }, + PropDesc { + name: "Size", + in_type: TdhInType::InTypeUInt32, + out_type: TdhOutType::OutTypeUInt64, + }, + // XXX: use an enum for these + PropDesc { + name: "Usage", + in_type: TdhInType::InTypeUInt32, + out_type: TdhOutType::OutTypeUInt32, + }, + PropDesc { + name: "Semantic", + in_type: TdhInType::InTypeUInt32, + out_type: TdhOutType::OutTypeUInt32, + }, +]; + +impl EventSchema for D3DUmdLogging_MapAllocation { + fn provider_guid(&self) -> GUID { + GUID::from("A688EE40-D8D9-4736-B6F9-6B74935BA3B1") + } + + fn event_id(&self) -> u16 { + 1 + } + + fn event_version(&self) -> u8 { + 0 + } + + fn opcode(&self) -> u8 { + 1 + } + + fn level(&self) -> u8 { + 0 + } + + fn decoding_source(&self) -> DecodingSource { + panic!() + } + + fn provider_name(&self) -> String { + "D3DUmdLogging".to_owned() + } + + fn task_name(&self) -> String { + "MapAllocation".to_owned() + } + + fn opcode_name(&self) -> String { + "Start".to_string() + } + + fn property_count(&self) -> u32 { + D3DUmdLogging_PROPS.len() as u32 + } + + fn property(&self, index: u32) -> Property { + let prop = &D3DUmdLogging_PROPS[index as usize]; + Property { + name: prop.name.to_owned(), + desc: PropertyDesc::Primitive(PrimitiveDesc { + in_type: prop.in_type, + out_type: prop.out_type, + }), + count: 1, + length: PropertyLength::Length(0), + map_info: None, + flags: PropertyFlags::empty(), + } + } +} + +pub struct D3DUmdLogging_RundownAllocation {} + +impl EventSchema for D3DUmdLogging_RundownAllocation { + fn provider_guid(&self) -> GUID { + GUID::from("A688EE40-D8D9-4736-B6F9-6B74935BA3B1") + } + + fn event_id(&self) -> u16 { + 2 + } + + fn event_version(&self) -> u8 { + 0 + } + + fn opcode(&self) -> u8 { + 3 + } + + fn level(&self) -> u8 { + 0 + } + + fn decoding_source(&self) -> DecodingSource { + panic!() + } + + fn provider_name(&self) -> String { + "D3DUmdLogging".to_owned() + } + + fn task_name(&self) -> String { + "MapAllocation".to_owned() + } + + fn opcode_name(&self) -> String { + "DC Start".to_string() + } + + fn property_count(&self) -> u32 { + D3DUmdLogging_PROPS.len() as u32 + } + + fn property(&self, index: u32) -> Property { + let prop = &D3DUmdLogging_PROPS[index as usize]; + Property { + name: prop.name.to_owned(), + desc: PropertyDesc::Primitive(PrimitiveDesc { + in_type: prop.in_type, + out_type: prop.out_type, + }), + count: 1, + length: PropertyLength::Length(0), + map_info: None, + flags: PropertyFlags::empty(), + } + } +} + +pub struct D3DUmdLogging_UnmapAllocation {} + +impl EventSchema for D3DUmdLogging_UnmapAllocation { + fn provider_guid(&self) -> GUID { + GUID::from("A688EE40-D8D9-4736-B6F9-6B74935BA3B1") + } + + fn event_id(&self) -> u16 { + 3 + } + + fn event_version(&self) -> u8 { + 0 + } + + fn opcode(&self) -> u8 { + 2 + } + + fn level(&self) -> u8 { + 0 + } + + fn decoding_source(&self) -> DecodingSource { + panic!() + } + + fn provider_name(&self) -> String { + "D3DUmdLogging".to_owned() + } + + fn task_name(&self) -> String { + "MapAllocation".to_owned() + } + + fn opcode_name(&self) -> String { + "End".to_string() + } + + fn property_count(&self) -> u32 { + D3DUmdLogging_PROPS.len() as u32 + } + + fn property(&self, index: u32) -> Property { + let prop = &D3DUmdLogging_PROPS[index as usize]; + Property { + name: prop.name.to_owned(), + desc: PropertyDesc::Primitive(PrimitiveDesc { + in_type: prop.in_type, + out_type: prop.out_type, + }), + count: 1, + length: PropertyLength::Length(0), + map_info: None, + flags: PropertyFlags::empty(), + } + } +} diff --git a/etw-reader/src/etw_types.rs b/etw-reader/src/etw_types.rs index 46ace189..729deb96 100644 --- a/etw-reader/src/etw_types.rs +++ b/etw-reader/src/etw_types.rs @@ -1,291 +1,303 @@ -use std::ops::Deref; -use std::rc::Rc; - -use once_cell::unsync::OnceCell; -use windows::Win32::System::Diagnostics::Etw::{self, PropertyStruct}; -use crate::{tdh_types::PropertyMapInfo}; -use crate::schema::EventSchema; -use crate::utils; -use crate::tdh_types::Property; -use windows::core::{GUID, PCWSTR}; - -#[repr(transparent)] -pub struct EventRecord(Etw::EVENT_RECORD); - -impl Deref for EventRecord { - type Target = Etw::EVENT_RECORD; - - fn deref(&self) -> &Self::Target { - &self.0 - } -} - -impl EventRecord { - pub(crate) fn user_buffer(&self) -> &[u8] { - unsafe { - std::slice::from_raw_parts( - self.UserData as *mut _, - self.UserDataLength.into(), - ) - } - } -} - -/// Newtype wrapper over an [EVENT_PROPERTY_INFO] -/// -/// [EVENT_PROPERTY_INFO]: https://microsoft.github.io/windows-docs-rs/doc/bindings/Windows/Win32/Etw/struct.EVENT_PROPERTY_INFO.html -#[repr(C)] -#[derive(Clone, Copy)] -pub struct EventPropertyInfo(Etw::EVENT_PROPERTY_INFO); - -impl std::ops::Deref for EventPropertyInfo { - type Target = Etw::EVENT_PROPERTY_INFO; - - fn deref(&self) -> &self::Etw::EVENT_PROPERTY_INFO { - &self.0 - } -} - -impl std::ops::DerefMut for EventPropertyInfo { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.0 - } -} - -impl From<&[u8]> for EventPropertyInfo { - fn from(val: &[u8]) -> Self { - unsafe { *(val.as_ptr() as *mut EventPropertyInfo) } - } -} - -impl Default for EventPropertyInfo { - fn default() -> Self { - unsafe { std::mem::zeroed::() } - } -} - -// Safe cast (EVENT_HEADER_FLAG_32_BIT_HEADER = 32) -#[doc(hidden)] -pub const EVENT_HEADER_FLAG_32_BIT_HEADER: u16 = Etw::EVENT_HEADER_FLAG_32_BIT_HEADER as u16; - -/// Wrapper over the [DECODING_SOURCE] type -/// -/// [DECODING_SOURCE]: https://microsoft.github.io/windows-docs-rs/doc/bindings/Windows/Win32/Etw/struct.DECODING_SOURCE.html -#[derive(Debug)] -pub enum DecodingSource { - DecodingSourceXMLFile, - DecodingSourceWbem, - DecodingSourceWPP, - DecodingSourceTlg, - DecodingSourceMax, -} - -impl From for DecodingSource { - fn from(val: Etw::DECODING_SOURCE) -> Self { - match val { - Etw::DecodingSourceXMLFile => DecodingSource::DecodingSourceXMLFile, - Etw::DecodingSourceWbem => DecodingSource::DecodingSourceWbem, - Etw::DecodingSourceWPP => DecodingSource::DecodingSourceWPP, - Etw::DecodingSourceTlg => DecodingSource::DecodingSourceTlg, - _ => DecodingSource::DecodingSourceMax, - } - } -} - - -/// Newtype wrapper over an [TRACE_EVENT_INFO] -/// -/// [TRACE_EVENT_INFO]: https://microsoft.github.io/windows-docs-rs/doc/bindings/Windows/Win32/Etw/struct.TRACE_EVENT_INFO.html -#[repr(C)] -#[derive(Clone, Copy)] -pub struct TraceEventInfo(Etw::TRACE_EVENT_INFO); - -impl std::ops::Deref for TraceEventInfo { - type Target = Etw::TRACE_EVENT_INFO; - - fn deref(&self) -> &self::Etw::TRACE_EVENT_INFO { - &self.0 - } -} - - -#[repr(C)] -#[derive(Debug, Clone, Default)] -pub struct TraceEventInfoRaw { - pub(crate) info: Vec, - property_maps: OnceCell>>>>, -} - -impl std::ops::DerefMut for TraceEventInfo { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.0 - } -} - -impl From<&TraceEventInfoRaw> for TraceEventInfo { - fn from(val: &TraceEventInfoRaw) -> Self { - unsafe { *(val.info.as_ptr() as *mut TraceEventInfo) } - } -} - - -impl TraceEventInfoRaw { - pub(crate) fn new(info: Vec) -> Self { - TraceEventInfoRaw { - info, - property_maps: OnceCell::new(), - } - } - pub(crate) fn alloc(len: u32) -> Self { - TraceEventInfoRaw { - info: vec![0; len as usize], - property_maps: OnceCell::new(), - } - } - - pub fn info_as_ptr(&mut self) -> *mut u8 { - self.info.as_mut_ptr() - } - - fn property_map_info(&self, index: u32) -> Option> { - - // let's make sure index is not bigger thant the PropertyCount - assert!(index <= TraceEventInfo::from(self).PropertyCount); - let property_maps = self.property_maps.get_or_init(|| { - vec![OnceCell::new(); TraceEventInfo::from(self).PropertyCount as usize] - }); - let map = property_maps[index as usize].get_or_init(|| { - // We need to subtract the sizeof(EVENT_PROPERTY_INFO) due to how TRACE_EVENT_INFO is declared - // in the bindings, the last field `EventPropertyInfoArray[ANYSIZE_ARRAY]` is declared as - // [EVENT_PROPERTY_INFO; 1] - // https://microsoft.github.io/windows-docs-rs/doc/bindings/Windows/Win32/Etw/struct.TRACE_EVENT_INFO.html#structfield.EventPropertyInfoArray - let curr_prop_offset = index as usize * std::mem::size_of::() - + (std::mem::size_of::() - std::mem::size_of::()); - - let curr_prop = EventPropertyInfo::from(&self.info[curr_prop_offset..]); - if curr_prop.Flags.0 & PropertyStruct.0 == 0 { - // This property is a struct so it has no map info - return None; - } else { - unsafe { - if curr_prop.Anonymous1.nonStructType.MapNameOffset != 0 { - // build an empty event record that we can use to get the map info - let mut event: Etw::EVENT_RECORD = std::mem::zeroed(); - event.EventHeader.ProviderId = self.provider_guid(); - - let mut buffer_size = 0; - let map_name = PCWSTR(self.info[curr_prop.Anonymous1.nonStructType.MapNameOffset as usize..].as_ptr() as *mut u16); - use windows::Win32::Foundation::ERROR_INSUFFICIENT_BUFFER; - // println!("map_name {}", utils::parse_unk_size_null_utf16_string(&self.info[curr_prop.Anonymous1.nonStructType.MapNameOffset as usize..])); - if Etw::TdhGetEventMapInformation(&event, map_name, None, &mut buffer_size) != ERROR_INSUFFICIENT_BUFFER.0 { - panic!("expected this to fail"); - } - - let mut buffer = vec![0; buffer_size as usize]; - if Etw::TdhGetEventMapInformation(&event, map_name, Some(buffer.as_mut_ptr() as *mut _), &mut buffer_size) != 0 { - panic!(); - } - - let map_info: &crate::Etw::EVENT_MAP_INFO = &*(buffer.as_ptr() as *const _); - if map_info.Flag == crate::Etw::EVENTMAP_INFO_FLAG_MANIFEST_VALUEMAP || map_info.Flag == crate::Etw::EVENTMAP_INFO_FLAG_MANIFEST_BITMAP { - let is_bitmap = map_info.Flag == crate::Etw::EVENTMAP_INFO_FLAG_MANIFEST_BITMAP; - let mut map = crate::FastHashMap::default(); - assert!(map_info.Anonymous.MapEntryValueType == crate::Etw::EVENTMAP_ENTRY_VALUETYPE_ULONG); - let entries = std::slice::from_raw_parts(map_info.MapEntryArray.as_ptr(), map_info.EntryCount as usize); - for e in entries { - let value = e.Anonymous.Value; - let name = utils::parse_unk_size_null_utf16_string(&buffer[e.OutputOffset as usize..]); - // println!("{} -> {:?}", value, name); - map.insert(value, name); - } - return Some(Rc::new(PropertyMapInfo { is_bitmap, map })); - } else { - eprint!("unsupported map type {:?}", map_info.Flag); - } - - } - } - } - return None; - }); - map.clone() - } - -} - -impl EventSchema for TraceEventInfoRaw { - fn provider_guid(&self) -> GUID { - TraceEventInfo::from(self).ProviderGuid - } - - fn event_id(&self) -> u16 { - TraceEventInfo::from(self).EventDescriptor.Id - } - - fn opcode(&self) -> u8 { - TraceEventInfo::from(self).EventDescriptor.Opcode - } - - fn event_version(&self) -> u8 { - TraceEventInfo::from(self).EventDescriptor.Version - } - - fn level(&self) -> u8 { - TraceEventInfo::from(self).EventDescriptor.Level - } - - fn decoding_source(&self) -> DecodingSource { - DecodingSource::from(TraceEventInfo::from(self).DecodingSource) - } - - fn provider_name(&self) -> String { - let provider_name_offset = TraceEventInfo::from(self).ProviderNameOffset as usize; - // TODO: Evaluate performance, but this sounds better than creating a whole Vec and getting the string from the offset/2 - utils::parse_unk_size_null_utf16_string(&self.info[provider_name_offset..]) - } - - fn task_name(&self) -> String { - let task_name_offset = TraceEventInfo::from(self).TaskNameOffset as usize; - utils::parse_unk_size_null_utf16_string(&self.info[task_name_offset..]) - } - - fn opcode_name(&self) -> String { - let opcode_name_offset = TraceEventInfo::from(self).OpcodeNameOffset as usize; - if opcode_name_offset == 0 { - return String::from(""); - } - utils::parse_unk_size_null_utf16_string(&self.info[opcode_name_offset..]) - } - - fn property_count(&self) -> u32 { - TraceEventInfo::from(self).TopLevelPropertyCount - } - - fn property(&self, index: u32) -> Property { - // let's make sure index is not bigger thant the PropertyCount - assert!(index <= TraceEventInfo::from(self).PropertyCount); - - // We need to subtract the sizeof(EVENT_PROPERTY_INFO) due to how TRACE_EVENT_INFO is declared - // in the bindings, the last field `EventPropertyInfoArray[ANYSIZE_ARRAY]` is declared as - // [EVENT_PROPERTY_INFO; 1] - // https://microsoft.github.io/windows-docs-rs/doc/bindings/Windows/Win32/Etw/struct.TRACE_EVENT_INFO.html#structfield.EventPropertyInfoArray - let curr_prop_offset = index as usize * std::mem::size_of::() - + (std::mem::size_of::() - std::mem::size_of::()); - - let curr_prop = EventPropertyInfo::from(&self.info[curr_prop_offset..]); - let name = - utils::parse_unk_size_null_utf16_string(&self.info[curr_prop.NameOffset as usize..]); - Property::new(name, &curr_prop, self.property_map_info(index)) - } - - fn event_message(&self) -> Option { - let offset = TraceEventInfo::from(self).EventMessageOffset; - if offset != 0 { - Some(utils::parse_unk_size_null_utf16_string(&self.info[offset as usize..])) - } else { - None - } - } - - -} \ No newline at end of file +use std::ops::Deref; +use std::rc::Rc; + +use once_cell::unsync::OnceCell; +use windows::core::{GUID, PCWSTR}; +use windows::Win32::System::Diagnostics::Etw::{self, PropertyStruct}; + +use super::schema::EventSchema; +use super::tdh_types::{Property, PropertyMapInfo}; +use super::utils; + +#[repr(transparent)] +pub struct EventRecord(Etw::EVENT_RECORD); + +impl Deref for EventRecord { + type Target = Etw::EVENT_RECORD; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl EventRecord { + pub(crate) fn user_buffer(&self) -> &[u8] { + unsafe { std::slice::from_raw_parts(self.UserData as *mut _, self.UserDataLength.into()) } + } +} + +/// Newtype wrapper over an [EVENT_PROPERTY_INFO] +/// +/// [EVENT_PROPERTY_INFO]: https://microsoft.github.io/windows-docs-rs/doc/bindings/Windows/Win32/Etw/struct.EVENT_PROPERTY_INFO.html +#[repr(C)] +#[derive(Clone, Copy)] +pub struct EventPropertyInfo(Etw::EVENT_PROPERTY_INFO); + +impl std::ops::Deref for EventPropertyInfo { + type Target = Etw::EVENT_PROPERTY_INFO; + + fn deref(&self) -> &self::Etw::EVENT_PROPERTY_INFO { + &self.0 + } +} + +impl std::ops::DerefMut for EventPropertyInfo { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +impl From<&[u8]> for EventPropertyInfo { + fn from(val: &[u8]) -> Self { + unsafe { *(val.as_ptr() as *mut EventPropertyInfo) } + } +} + +impl Default for EventPropertyInfo { + fn default() -> Self { + unsafe { std::mem::zeroed::() } + } +} + +// Safe cast (EVENT_HEADER_FLAG_32_BIT_HEADER = 32) +#[doc(hidden)] +pub const EVENT_HEADER_FLAG_32_BIT_HEADER: u16 = Etw::EVENT_HEADER_FLAG_32_BIT_HEADER as u16; + +/// Wrapper over the [DECODING_SOURCE] type +/// +/// [DECODING_SOURCE]: https://microsoft.github.io/windows-docs-rs/doc/bindings/Windows/Win32/Etw/struct.DECODING_SOURCE.html +#[derive(Debug)] +pub enum DecodingSource { + DecodingSourceXMLFile, + DecodingSourceWbem, + DecodingSourceWPP, + DecodingSourceTlg, + DecodingSourceMax, +} + +impl From for DecodingSource { + fn from(val: Etw::DECODING_SOURCE) -> Self { + match val { + Etw::DecodingSourceXMLFile => DecodingSource::DecodingSourceXMLFile, + Etw::DecodingSourceWbem => DecodingSource::DecodingSourceWbem, + Etw::DecodingSourceWPP => DecodingSource::DecodingSourceWPP, + Etw::DecodingSourceTlg => DecodingSource::DecodingSourceTlg, + _ => DecodingSource::DecodingSourceMax, + } + } +} + +/// Newtype wrapper over an [TRACE_EVENT_INFO] +/// +/// [TRACE_EVENT_INFO]: https://microsoft.github.io/windows-docs-rs/doc/bindings/Windows/Win32/Etw/struct.TRACE_EVENT_INFO.html +#[repr(C)] +#[derive(Clone, Copy)] +pub struct TraceEventInfo(Etw::TRACE_EVENT_INFO); + +impl std::ops::Deref for TraceEventInfo { + type Target = Etw::TRACE_EVENT_INFO; + + fn deref(&self) -> &self::Etw::TRACE_EVENT_INFO { + &self.0 + } +} + +#[repr(C)] +#[derive(Debug, Clone, Default)] +pub struct TraceEventInfoRaw { + pub(crate) info: Vec, + property_maps: OnceCell>>>>, +} + +impl std::ops::DerefMut for TraceEventInfo { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +impl From<&TraceEventInfoRaw> for TraceEventInfo { + fn from(val: &TraceEventInfoRaw) -> Self { + unsafe { *(val.info.as_ptr() as *mut TraceEventInfo) } + } +} + +impl TraceEventInfoRaw { + pub(crate) fn new(info: Vec) -> Self { + TraceEventInfoRaw { + info, + property_maps: OnceCell::new(), + } + } + pub(crate) fn alloc(len: u32) -> Self { + TraceEventInfoRaw { + info: vec![0; len as usize], + property_maps: OnceCell::new(), + } + } + + pub fn info_as_ptr(&mut self) -> *mut u8 { + self.info.as_mut_ptr() + } + + fn property_map_info(&self, index: u32) -> Option> { + // let's make sure index is not bigger thant the PropertyCount + assert!(index <= TraceEventInfo::from(self).PropertyCount); + let property_maps = self.property_maps.get_or_init(|| { + vec![OnceCell::new(); TraceEventInfo::from(self).PropertyCount as usize] + }); + let map = property_maps[index as usize].get_or_init(|| { + // We need to subtract the sizeof(EVENT_PROPERTY_INFO) due to how TRACE_EVENT_INFO is declared + // in the bindings, the last field `EventPropertyInfoArray[ANYSIZE_ARRAY]` is declared as + // [EVENT_PROPERTY_INFO; 1] + // https://microsoft.github.io/windows-docs-rs/doc/bindings/Windows/Win32/Etw/struct.TRACE_EVENT_INFO.html#structfield.EventPropertyInfoArray + let curr_prop_offset = index as usize * std::mem::size_of::() + + (std::mem::size_of::() + - std::mem::size_of::()); + + let curr_prop = EventPropertyInfo::from(&self.info[curr_prop_offset..]); + if curr_prop.Flags.0 & PropertyStruct.0 == 0 { + // This property is a struct so it has no map info + return None; + } else { + unsafe { + if curr_prop.Anonymous1.nonStructType.MapNameOffset != 0 { + // build an empty event record that we can use to get the map info + let mut event: Etw::EVENT_RECORD = std::mem::zeroed(); + event.EventHeader.ProviderId = self.provider_guid(); + + let mut buffer_size = 0; + let map_name = PCWSTR( + self.info[curr_prop.Anonymous1.nonStructType.MapNameOffset as usize..] + .as_ptr() as *mut u16, + ); + use windows::Win32::Foundation::ERROR_INSUFFICIENT_BUFFER; + // println!("map_name {}", utils::parse_unk_size_null_utf16_string(&self.info[curr_prop.Anonymous1.nonStructType.MapNameOffset as usize..])); + if Etw::TdhGetEventMapInformation(&event, map_name, None, &mut buffer_size) + != ERROR_INSUFFICIENT_BUFFER.0 + { + panic!("expected this to fail"); + } + + let mut buffer = vec![0; buffer_size as usize]; + if Etw::TdhGetEventMapInformation( + &event, + map_name, + Some(buffer.as_mut_ptr() as *mut _), + &mut buffer_size, + ) != 0 + { + panic!(); + } + + let map_info: &super::Etw::EVENT_MAP_INFO = &*(buffer.as_ptr() as *const _); + if map_info.Flag == super::Etw::EVENTMAP_INFO_FLAG_MANIFEST_VALUEMAP + || map_info.Flag == super::Etw::EVENTMAP_INFO_FLAG_MANIFEST_BITMAP + { + let is_bitmap = + map_info.Flag == super::Etw::EVENTMAP_INFO_FLAG_MANIFEST_BITMAP; + let mut map = super::FastHashMap::default(); + assert!( + map_info.Anonymous.MapEntryValueType + == super::Etw::EVENTMAP_ENTRY_VALUETYPE_ULONG + ); + let entries = std::slice::from_raw_parts( + map_info.MapEntryArray.as_ptr(), + map_info.EntryCount as usize, + ); + for e in entries { + let value = e.Anonymous.Value; + let name = utils::parse_unk_size_null_utf16_string( + &buffer[e.OutputOffset as usize..], + ); + // println!("{} -> {:?}", value, name); + map.insert(value, name); + } + return Some(Rc::new(PropertyMapInfo { is_bitmap, map })); + } else { + eprint!("unsupported map type {:?}", map_info.Flag); + } + } + } + } + return None; + }); + map.clone() + } +} + +impl EventSchema for TraceEventInfoRaw { + fn provider_guid(&self) -> GUID { + TraceEventInfo::from(self).ProviderGuid + } + + fn event_id(&self) -> u16 { + TraceEventInfo::from(self).EventDescriptor.Id + } + + fn opcode(&self) -> u8 { + TraceEventInfo::from(self).EventDescriptor.Opcode + } + + fn event_version(&self) -> u8 { + TraceEventInfo::from(self).EventDescriptor.Version + } + + fn level(&self) -> u8 { + TraceEventInfo::from(self).EventDescriptor.Level + } + + fn decoding_source(&self) -> DecodingSource { + DecodingSource::from(TraceEventInfo::from(self).DecodingSource) + } + + fn provider_name(&self) -> String { + let provider_name_offset = TraceEventInfo::from(self).ProviderNameOffset as usize; + // TODO: Evaluate performance, but this sounds better than creating a whole Vec and getting the string from the offset/2 + utils::parse_unk_size_null_utf16_string(&self.info[provider_name_offset..]) + } + + fn task_name(&self) -> String { + let task_name_offset = TraceEventInfo::from(self).TaskNameOffset as usize; + utils::parse_unk_size_null_utf16_string(&self.info[task_name_offset..]) + } + + fn opcode_name(&self) -> String { + let opcode_name_offset = TraceEventInfo::from(self).OpcodeNameOffset as usize; + if opcode_name_offset == 0 { + return String::from(""); + } + utils::parse_unk_size_null_utf16_string(&self.info[opcode_name_offset..]) + } + + fn property_count(&self) -> u32 { + TraceEventInfo::from(self).TopLevelPropertyCount + } + + fn property(&self, index: u32) -> Property { + // let's make sure index is not bigger thant the PropertyCount + assert!(index <= TraceEventInfo::from(self).PropertyCount); + + // We need to subtract the sizeof(EVENT_PROPERTY_INFO) due to how TRACE_EVENT_INFO is declared + // in the bindings, the last field `EventPropertyInfoArray[ANYSIZE_ARRAY]` is declared as + // [EVENT_PROPERTY_INFO; 1] + // https://microsoft.github.io/windows-docs-rs/doc/bindings/Windows/Win32/Etw/struct.TRACE_EVENT_INFO.html#structfield.EventPropertyInfoArray + let curr_prop_offset = index as usize * std::mem::size_of::() + + (std::mem::size_of::() - std::mem::size_of::()); + + let curr_prop = EventPropertyInfo::from(&self.info[curr_prop_offset..]); + let name = + utils::parse_unk_size_null_utf16_string(&self.info[curr_prop.NameOffset as usize..]); + Property::new(name, &curr_prop, self.property_map_info(index)) + } + + fn event_message(&self) -> Option { + let offset = TraceEventInfo::from(self).EventMessageOffset; + if offset != 0 { + Some(utils::parse_unk_size_null_utf16_string( + &self.info[offset as usize..], + )) + } else { + None + } + } +} diff --git a/etw-reader/src/parser.rs b/etw-reader/src/parser.rs index 3b7fee05..047e46e6 100644 --- a/etw-reader/src/parser.rs +++ b/etw-reader/src/parser.rs @@ -1,620 +1,647 @@ -//! ETW Types Parser -//! -//! This module act as a helper to parse the Buffer from an ETW Event -use crate::etw_types::EVENT_HEADER_FLAG_32_BIT_HEADER; -use crate::sddl; -use crate::tdh; -use crate::tdh_types::PrimitiveDesc; -use crate::tdh_types::PropertyDesc; -use crate::tdh_types::PropertyLength; -use crate::tdh_types::{Property, PropertyFlags, TdhInType, TdhOutType}; -use crate::property::{PropertyInfo, PropertyIter}; -use crate::schema::TypedEvent; -use crate::utils; -use std::borrow::Borrow; -use std::convert::TryInto; -use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; -use windows::core::GUID; - -#[derive(Debug, Clone, Copy)] -pub enum Address { - Address64(u64), - Address32(u32) -} - -impl Address { - pub fn as_u64(&self) -> u64 { - match self { - Address::Address64(a) => *a, - Address::Address32(a) => *a as u64 - } - } -} - -/// Parser module errors -#[derive(Debug)] -pub enum ParserError { - /// An invalid type... - InvalidType, - /// Error parsing - ParseError, - /// Length mismatch when parsing a type - LengthMismatch, - PropertyError(String), - /// An error while transforming an Utf-8 buffer into String - Utf8Error(std::string::FromUtf8Error), - /// An error trying to get an slice as an array - SliceError(std::array::TryFromSliceError), - /// Represents an internal [SddlNativeError] - /// - /// [SddlNativeError]: sddl::SddlNativeError - //SddlNativeError(sddl::SddlNativeError), - /// Represents an internal [TdhNativeError] - /// - /// [TdhNativeError]: tdh::TdhNativeError - TdhNativeError(tdh::TdhNativeError), -} - -impl From for ParserError { - fn from(err: tdh::TdhNativeError) -> Self { - ParserError::TdhNativeError(err) - } -} -/* -impl From for ParserError { - fn from(err: sddl::SddlNativeError) -> Self { - ParserError::SddlNativeError(err) - } -}*/ - -impl From for ParserError { - fn from(err: std::string::FromUtf8Error) -> Self { - ParserError::Utf8Error(err) - } -} - -impl From for ParserError { - fn from(err: std::array::TryFromSliceError) -> Self { - ParserError::SliceError(err) - } -} - -type ParserResult = Result; - -/// Trait to try and parse a type -/// -/// This trait has to be implemented in order to be able to parse a type we want to retrieve from -/// within an Event. On success the parsed value will be returned within a Result, on error an Err -/// should be returned accordingly -/// -/// An implementation for most of the Primitive Types is created by using a Macro, any other needed type -/// requires this trait to be implemented -// TODO: Find a way to use turbofish operator -pub trait TryParse { - /// Implement the `try_parse` function to provide a way to Parse `T` from an ETW event or - /// return an Error in case the type `T` can't be parsed - /// - /// # Arguments - /// * `name` - Name of the property to be found in the Schema - fn try_parse(&mut self, name: &str) -> Result; - fn parse(&mut self, name: &str) -> T { - self.try_parse(name).unwrap_or_else(|e| panic!("{:?} name {} {:?}", e, std::any::type_name::(), name)) - } -} - -/// Represents a Parser -/// -/// This structure holds the necessary data to parse the ETW event and retrieve the data from the -/// event -#[allow(dead_code)] -pub struct Parser<'a> { - event: &'a TypedEvent<'a>, - properties: &'a PropertyIter, - pub buffer: &'a [u8], - last_property: u32, - offset: usize, - // a map from property indx to PropertyInfo - cache: Vec>, -} - -impl<'a> Parser<'a> { - /// Use the `create` function to create an instance of a Parser - /// - /// # Arguments - /// * `schema` - The [Schema] from the ETW Event we want to parse - /// - /// # Example - /// ```rust - /// let my_callback = |record: EventRecord, schema_locator: &mut SchemaLocator| { - /// let schema = schema_locator.event_schema(record)?; - /// let parser = Parse::create(&schema); - /// }; - /// ``` - pub fn create(event: &'a TypedEvent) -> Self { - Parser { - event, - buffer: event.user_buffer(), - properties: event.schema.properties(), - last_property: 0, - offset: 0, - cache: Vec::new(), // We could fill the cache on creation - } - } - /* - #[allow(dead_code)] - fn fill_cache( - schema: &TypedEvent, - properties: &PropertyIter, - ) -> ParserResult> { - let user_buffer_len = schema.user_buffer().len(); - let mut prop_offset = 0; - panic!(); - Ok(properties.properties_iter().iter().try_fold( - HashMap::new(), - |mut cache, x| -> ParserResult> { - let prop_size = tdh::property_size(schema.record(), &x.name)? as usize; - - if user_buffer_len < prop_size { - return Err(ParserError::PropertyError( - "Property length out of buffer bounds".to_owned(), - )); - } - let prop_buffer = schema.user_buffer()[..prop_size] - .iter() - .take(prop_size) - .cloned() - .collect(); - - cache.insert(x.name.clone(), PropertyInfo::create(x.clone(), prop_offset, prop_buffer)); - prop_offset += prop_size; - - Ok(cache) - }, - )?) - }*/ - - // TODO: Find a cleaner way to do this, not very happy with it rn - fn find_property_size(&self, property: &Property) -> ParserResult { - match property.length { - PropertyLength::Index(_) => { - // e.g. Microsoft-Windows-Kernel-Power/SystemTimerResolutionStackRundown uses the AppNameLength property - // as the size of AppName - - // Fallback to Tdh - return Ok(tdh::property_size(self.event.record(), &property.name).unwrap() as usize); - } - PropertyLength::Length(length) => { - // TODO: Study heuristic method used in krabsetw :) - if property.flags.is_empty() && length > 0 && property.count == 1 { - return Ok(length as usize); - } - if property.count == 1 { - if let PropertyDesc::Primitive(desc) = &property.desc { - match desc.in_type { - TdhInType::InTypeBoolean => return Ok(4), - TdhInType::InTypeInt32 | TdhInType::InTypeUInt32 | TdhInType::InTypeHexInt32 => return Ok(4), - TdhInType::InTypeInt64 | TdhInType::InTypeUInt64 | TdhInType::InTypeHexInt64 => return Ok(8), - TdhInType::InTypeInt8 | TdhInType::InTypeUInt8 => return Ok(1), - TdhInType::InTypeInt16 | TdhInType::InTypeUInt16 => return Ok(2), - TdhInType::InTypePointer => return Ok(if (self.event.event_flags() & EVENT_HEADER_FLAG_32_BIT_HEADER) != 0 { - 4 - } else { - 8 - }), - TdhInType::InTypeGuid => return Ok(std::mem::size_of::()), - TdhInType::InTypeUnicodeString => { - return Ok(utils::parse_unk_size_null_unicode_size(&self.buffer)) - } - TdhInType::InTypeAnsiString => { - return Ok(utils::parse_unk_size_null_ansi_size(&self.buffer)); - } - _ => {} - } - } - } - return Ok(tdh::property_size(self.event.record(), &property.name).unwrap() as usize) - } - } - } - - pub fn find_property(&mut self, name: &str) -> ParserResult { - let indx = *self.properties.name_to_indx.get(name).ok_or_else( - || ParserError::PropertyError("Unknown property".to_owned()))?; - if indx < self.cache.len() { - return Ok(indx); - } - - // TODO: Find a way to do this with an iter, try_find looks promising but is not stable yet - // TODO: Clean this a bit, not a big fan of this loop - for i in self.cache.len()..=indx { - let curr_prop = self.properties.property(i).unwrap(); - - let prop_size = self.find_property_size(&curr_prop)?; - - if self.buffer.len() < prop_size { - return Err(ParserError::PropertyError( - format!("Property of {} bytes out of buffer bounds ({})", prop_size, self.buffer.len()), - )); - } - - // We split the buffer, if everything works correctly in the end the buffer will be empty - // and we should have all properties in the cache - let (prop_buffer, remaining) = self.buffer.split_at(prop_size); - self.buffer = remaining; - self.cache.push(PropertyInfo::create(curr_prop, self.offset, prop_buffer)); - self.offset += prop_size; - } - Ok(indx) - } -} - -/* -impl<'a> std::fmt::Debug for Parser<'a> { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let mut s = f.debug_struct("ParsedEvent"); - for i in 0..self.event.property_count() { - let property = self.event.property(i); - let value = match property.in_type() { - TdhInType::InTypeUnicodeString => format!("{}", TryParse::::parse(self, &property.name)), - TdhInType::InTypeAnsiString => format!("{}", TryParse::::parse(self, &property.name)), - TdhInType::InTypeUInt32 => format!("{}", TryParse::::parse(self, &property.name)), - TdhInType::InTypeUInt8 => format!("{}", TryParse::::parse(self, &property.name)), - TdhInType::InTypePointer => format!("{}", TryParse::::parse(self, &property.name)), - TdhInType::InTypeInt64 => format!("{}", TryParse::::parse(self, &property.name)), - TdhInType::InTypeUInt64 => format!("{}", TryParse::::parse(self, &property.name)), - TdhInType::InTypeGuid => format!("{:?}", TryParse::::parse(self, &property.name)), - _ => panic!() - }; - s.field(&property.name, &value); - //dbg!(&property); - } - s.finish() - } -}*/ - -macro_rules! impl_try_parse_primitive { - ($T:ident, $ty:ident) => { - impl TryParse<$T> for Parser<'_> { - fn try_parse(&mut self, name: &str) -> ParserResult<$T> { - use TdhInType::*; - let indx = self.find_property(name)?; - let prop_info = &self.cache[indx]; - let prop_info: &PropertyInfo = prop_info.borrow(); - if let PropertyDesc::Primitive(desc) = &prop_info.property.desc { - if desc.in_type != $ty { - return Err(ParserError::InvalidType) - } - if std::mem::size_of::<$T>() != prop_info.buffer.len() { - return Err(ParserError::LengthMismatch); - } - return Ok($T::from_ne_bytes(prop_info.buffer.try_into()?)) - }; - Err(ParserError::InvalidType) - } - } - }; -} - -impl_try_parse_primitive!(u8, InTypeUInt8); -impl_try_parse_primitive!(i8, InTypeInt8); -impl_try_parse_primitive!(u16, InTypeUInt16); -impl_try_parse_primitive!(i16, InTypeInt16); -impl_try_parse_primitive!(u32, InTypeUInt32); -//impl_try_parse_primitive!(u64, InTypeUInt64); -//impl_try_parse_primitive!(i64, InTypeInt64); - -impl TryParse for Parser<'_> { - fn try_parse(&mut self, name: &str) -> ParserResult { - use TdhInType::*; - let indx = self.find_property(name)?; - let prop_info = &self.cache[indx]; - if let PropertyDesc::Primitive(desc) = &prop_info.property.desc { - if desc.in_type == InTypeUInt64 { - if std::mem::size_of::() != prop_info.buffer.len() { - return Err(ParserError::LengthMismatch); - } - return Ok(u64::from_ne_bytes(prop_info.buffer.try_into()?)); - } - if desc.in_type == InTypePointer || desc.in_type == InTypeSizeT { - if (self.event.event_flags() & EVENT_HEADER_FLAG_32_BIT_HEADER) != 0 { - if std::mem::size_of::() != prop_info.buffer.len() { - return Err(ParserError::LengthMismatch); - } - return Ok(u32::from_ne_bytes(prop_info.buffer.try_into()?) as u64); - } - if std::mem::size_of::() != prop_info.buffer.len() { - return Err(ParserError::LengthMismatch); - } - return Ok(u64::from_ne_bytes(prop_info.buffer.try_into()?)); - } - } - return Err(ParserError::InvalidType) - } -} - -impl TryParse for Parser<'_> { - fn try_parse(&mut self, name: &str) -> ParserResult { - use TdhInType::*; - let indx = self.find_property(name)?; - let prop_info = &self.cache[indx]; - if let PropertyDesc::Primitive(desc) = &prop_info.property.desc { - if desc.in_type == InTypeInt64 || desc.in_type == InTypeHexInt64 { - if std::mem::size_of::() != prop_info.buffer.len() { - return Err(ParserError::LengthMismatch); - } - return Ok(i64::from_ne_bytes(prop_info.buffer.try_into()?)); - } - } - return Err(ParserError::InvalidType) - } -} - -impl TryParse for Parser<'_> { - fn try_parse(&mut self, name: &str) -> ParserResult { - use TdhInType::*; - let indx = self.find_property(name)?; - let prop_info = &self.cache[indx]; - if let PropertyDesc::Primitive(desc) = &prop_info.property.desc { - if desc.in_type == InTypeInt32 || desc.in_type == InTypeHexInt32 { - if std::mem::size_of::() != prop_info.buffer.len() { - return Err(ParserError::LengthMismatch); - } - return Ok(i32::from_ne_bytes(prop_info.buffer.try_into()?)); - } - } - return Err(ParserError::InvalidType) - } -} - -impl TryParse
for Parser<'_> { - fn try_parse(&mut self, name: &str) -> ParserResult
{ - use TdhInType::*; - let indx = self.find_property(name)?; - let prop_info = &self.cache[indx]; - - if let PropertyDesc::Primitive(desc) = &prop_info.property.desc { - if self.event.is_64bit() { - if desc.in_type == InTypeUInt64 || desc.in_type == InTypePointer || desc.in_type == InTypeHexInt64 { - if std::mem::size_of::() != prop_info.buffer.len() { - return Err(ParserError::LengthMismatch); - } - return Ok(Address::Address64(u64::from_ne_bytes(prop_info.buffer.try_into()?))); - } - } else { - if desc.in_type == InTypeUInt32 || desc.in_type == InTypePointer || desc.in_type == InTypeHexInt32 { - if std::mem::size_of::() != prop_info.buffer.len() { - return Err(ParserError::LengthMismatch); - } - return Ok(Address::Address32(u32::from_ne_bytes(prop_info.buffer.try_into()?))); - } - } - } - return Err(ParserError::InvalidType) - } -} - -impl TryParse for Parser<'_> { - fn try_parse(&mut self, name: &str) -> ParserResult { - use TdhInType::*; - let indx = self.find_property(name)?; - let prop_info = &self.cache[indx]; - if let PropertyDesc::Primitive(desc) = &prop_info.property.desc { - if desc.in_type != InTypeBoolean { - return Err(ParserError::InvalidType) - } - if prop_info.buffer.len() != 4 { - return Err(ParserError::LengthMismatch); - } - return match u32::from_ne_bytes(prop_info.buffer.try_into()?) { - 1 => Ok(true), - 0 => Ok(false), - _ => Err(ParserError::InvalidType) - } - }; - Err(ParserError::InvalidType) - } -} - -impl TryParse for Parser<'_> { - fn try_parse(&mut self, name: &str) -> ParserResult { - use TdhInType::*; - let indx = self.find_property(name)?; - let prop_info = &self.cache[indx]; - if let PropertyDesc::Primitive(desc) = &prop_info.property.desc { - if desc.in_type == InTypeFloat { - if std::mem::size_of::() != prop_info.buffer.len() { - return Err(ParserError::LengthMismatch); - } - return Ok(f32::from_ne_bytes(prop_info.buffer.try_into()?)); - } - } - return Err(ParserError::InvalidType); - } -} - -/// The `String` impl of the `TryParse` trait should be used to retrieve the following [TdhInTypes]: -/// -/// * InTypeUnicodeString -/// * InTypeAnsiString -/// * InTypeCountedString -/// * InTypeGuid -/// -/// On success a `String` with the with the data from the `name` property will be returned -/// -/// # Arguments -/// * `name` - Name of the property to be found in the Schema - -/// # Example -/// ```rust -/// let my_callback = |record: EventRecord, schema_locator: &mut SchemaLocator| { -/// let schema = schema_locator.event_schema(record)?; -/// let parser = Parse::create(&schema); -/// let image_name: String = parser.try_parse("ImageName")?; -/// }; -/// ``` -/// -/// [TdhInTypes]: TdhInType -impl TryParse for Parser<'_> { - fn try_parse(&mut self, name: &str) -> ParserResult { - let indx = self.find_property(name)?; - let prop_info = &self.cache[indx]; - - // TODO: Handle errors and type checking better - if let PropertyDesc::Primitive(desc) = &prop_info.property.desc { - let res = match desc.in_type { - TdhInType::InTypeUnicodeString => { - utils::parse_null_utf16_string(prop_info.buffer) - } - TdhInType::InTypeAnsiString => String::from_utf8(prop_info.buffer.to_vec())? - .trim_matches(char::default()) - .to_string(), - TdhInType::InTypeSid => { - panic!() - //sddl::convert_sid_to_string(prop_info.buffer.as_ptr() as isize)? - } - TdhInType::InTypeCountedString => unimplemented!(), - _ => return Err(ParserError::InvalidType), - }; - return Ok(res) - } - Err(ParserError::InvalidType) - } -} - -impl TryParse for Parser<'_> { - fn try_parse(&mut self, name: &str) -> Result { - let indx = self.find_property(name)?; - let prop_info = &self.cache[indx]; - if let PropertyDesc::Primitive(desc) = &prop_info.property.desc { - match desc.in_type { - TdhInType::InTypeUnicodeString => { - let guid_string = utils::parse_utf16_guid(prop_info.buffer); - - if guid_string.len() != 36 { - return Err(ParserError::LengthMismatch); - } - - return Ok(GUID::from(guid_string.as_str())) - } - TdhInType::InTypeGuid => { - return Ok(GUID::from_values(u32::from_ne_bytes((&prop_info.buffer[0..4]).try_into()?), - u16::from_ne_bytes((&prop_info.buffer[4..6]).try_into()?), - u16::from_ne_bytes((&prop_info.buffer[6..8]).try_into()?), - [ - prop_info.buffer[8], - prop_info.buffer[9], - prop_info.buffer[10], - prop_info.buffer[11], - prop_info.buffer[12], - prop_info.buffer[13], - prop_info.buffer[14], - prop_info.buffer[15], - ] - )) - } - _ => return Err(ParserError::InvalidType), - } - }; - Err(ParserError::InvalidType) - } -} - -impl TryParse for Parser<'_> { - fn try_parse(&mut self, name: &str) -> ParserResult { - let indx = self.find_property(name)?; - let prop_info = &self.cache[indx]; - if let PropertyDesc::Primitive(desc) = &prop_info.property.desc { - - if desc.out_type != TdhOutType::OutTypeIpv4 - && desc.out_type != TdhOutType::OutTypeIpv6 - { - return Err(ParserError::InvalidType); - } - - // Hardcoded values for now - let res = match prop_info.property.length { - PropertyLength::Length(16) => { - let tmp: [u8; 16] = prop_info.buffer.try_into()?; - IpAddr::V6(Ipv6Addr::from(tmp)) - } - PropertyLength::Length(4) => { - let tmp: [u8; 4] = prop_info.buffer.try_into()?; - IpAddr::V4(Ipv4Addr::from(tmp)) - } - _ => return Err(ParserError::LengthMismatch), - }; - - return Ok(res) - } - Err(ParserError::InvalidType) - } -} - -#[derive(Clone, Default, Debug)] -pub struct Pointer(usize); - -impl std::ops::Deref for Pointer { - type Target = usize; - - fn deref(&self) -> &Self::Target { - &self.0 - } -} - -impl std::ops::DerefMut for Pointer { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.0 - } -} - -impl std::fmt::LowerHex for Pointer { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let val = self.0; - - std::fmt::LowerHex::fmt(&val, f) // delegate to u32/u64 implementation - } -} - -impl std::fmt::UpperHex for Pointer { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let val = self.0; - - std::fmt::UpperHex::fmt(&val, f) // delegate to u32/u64 implementation - } -} - -impl std::fmt::Display for Pointer { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let val = self.0; - - std::fmt::Display::fmt(&val, f) // delegate to u32/u64 implementation - } -} - -impl TryParse for Parser<'_> { - fn try_parse(&mut self, name: &str) -> ParserResult { - let indx = self.find_property(name)?; - let prop_info = &self.cache[indx]; - - let mut res = Pointer::default(); - if prop_info.buffer.len() == std::mem::size_of::() { - res.0 = TryParse::::try_parse(self, name)? as usize; - } else { - res.0 = TryParse::::try_parse(self, name)? as usize; - } - - Ok(res) - } -} - -impl TryParse> for Parser<'_> { - fn try_parse(&mut self, name: &str) -> Result, ParserError> { - let indx = self.find_property(name)?; - let prop_info = &self.cache[indx]; - - Ok(prop_info.buffer.to_vec()) - } -} - -// TODO: Implement SocketAddress -// TODO: Study if we can use primitive types for HexInt64, HexInt32 and Pointer +//! ETW Types Parser +//! +//! This module act as a helper to parse the Buffer from an ETW Event +use std::borrow::Borrow; +use std::convert::TryInto; +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; + +use windows::core::GUID; + +use super::etw_types::EVENT_HEADER_FLAG_32_BIT_HEADER; +use super::property::{PropertyInfo, PropertyIter}; +use super::schema::TypedEvent; +use super::tdh_types::{ + PrimitiveDesc, Property, PropertyDesc, PropertyFlags, PropertyLength, TdhInType, TdhOutType, +}; +use super::{sddl, tdh, utils}; + +#[derive(Debug, Clone, Copy)] +pub enum Address { + Address64(u64), + Address32(u32), +} + +impl Address { + pub fn as_u64(&self) -> u64 { + match self { + Address::Address64(a) => *a, + Address::Address32(a) => *a as u64, + } + } +} + +/// Parser module errors +#[derive(Debug)] +pub enum ParserError { + /// An invalid type... + InvalidType, + /// Error parsing + ParseError, + /// Length mismatch when parsing a type + LengthMismatch, + PropertyError(String), + /// An error while transforming an Utf-8 buffer into String + Utf8Error(std::string::FromUtf8Error), + /// An error trying to get an slice as an array + SliceError(std::array::TryFromSliceError), + /// Represents an internal [SddlNativeError] + /// + /// [SddlNativeError]: sddl::SddlNativeError + //SddlNativeError(sddl::SddlNativeError), + /// Represents an internal [TdhNativeError] + /// + /// [TdhNativeError]: tdh::TdhNativeError + TdhNativeError(tdh::TdhNativeError), +} + +impl From for ParserError { + fn from(err: tdh::TdhNativeError) -> Self { + ParserError::TdhNativeError(err) + } +} +/* +impl From for ParserError { + fn from(err: sddl::SddlNativeError) -> Self { + ParserError::SddlNativeError(err) + } +}*/ + +impl From for ParserError { + fn from(err: std::string::FromUtf8Error) -> Self { + ParserError::Utf8Error(err) + } +} + +impl From for ParserError { + fn from(err: std::array::TryFromSliceError) -> Self { + ParserError::SliceError(err) + } +} + +type ParserResult = Result; + +/// Trait to try and parse a type +/// +/// This trait has to be implemented in order to be able to parse a type we want to retrieve from +/// within an Event. On success the parsed value will be returned within a Result, on error an Err +/// should be returned accordingly +/// +/// An implementation for most of the Primitive Types is created by using a Macro, any other needed type +/// requires this trait to be implemented +// TODO: Find a way to use turbofish operator +pub trait TryParse { + /// Implement the `try_parse` function to provide a way to Parse `T` from an ETW event or + /// return an Error in case the type `T` can't be parsed + /// + /// # Arguments + /// * `name` - Name of the property to be found in the Schema + fn try_parse(&mut self, name: &str) -> Result; + fn parse(&mut self, name: &str) -> T { + self.try_parse(name) + .unwrap_or_else(|e| panic!("{:?} name {} {:?}", e, std::any::type_name::(), name)) + } +} + +/// Represents a Parser +/// +/// This structure holds the necessary data to parse the ETW event and retrieve the data from the +/// event +#[allow(dead_code)] +pub struct Parser<'a> { + event: &'a TypedEvent<'a>, + properties: &'a PropertyIter, + pub buffer: &'a [u8], + last_property: u32, + offset: usize, + // a map from property indx to PropertyInfo + cache: Vec>, +} + +impl<'a> Parser<'a> { + /// Use the `create` function to create an instance of a Parser + /// + /// # Arguments + /// * `schema` - The [Schema] from the ETW Event we want to parse + /// + /// # Example + /// ```rust + /// let my_callback = |record: EventRecord, schema_locator: &mut SchemaLocator| { + /// let schema = schema_locator.event_schema(record)?; + /// let parser = Parse::create(&schema); + /// }; + /// ``` + pub fn create(event: &'a TypedEvent) -> Self { + Parser { + event, + buffer: event.user_buffer(), + properties: event.schema.properties(), + last_property: 0, + offset: 0, + cache: Vec::new(), // We could fill the cache on creation + } + } + /* + #[allow(dead_code)] + fn fill_cache( + schema: &TypedEvent, + properties: &PropertyIter, + ) -> ParserResult> { + let user_buffer_len = schema.user_buffer().len(); + let mut prop_offset = 0; + panic!(); + Ok(properties.properties_iter().iter().try_fold( + HashMap::new(), + |mut cache, x| -> ParserResult> { + let prop_size = tdh::property_size(schema.record(), &x.name)? as usize; + + if user_buffer_len < prop_size { + return Err(ParserError::PropertyError( + "Property length out of buffer bounds".to_owned(), + )); + } + let prop_buffer = schema.user_buffer()[..prop_size] + .iter() + .take(prop_size) + .cloned() + .collect(); + + cache.insert(x.name.clone(), PropertyInfo::create(x.clone(), prop_offset, prop_buffer)); + prop_offset += prop_size; + + Ok(cache) + }, + )?) + }*/ + + // TODO: Find a cleaner way to do this, not very happy with it rn + fn find_property_size(&self, property: &Property) -> ParserResult { + match property.length { + PropertyLength::Index(_) => { + // e.g. Microsoft-Windows-Kernel-Power/SystemTimerResolutionStackRundown uses the AppNameLength property + // as the size of AppName + + // Fallback to Tdh + return Ok( + tdh::property_size(self.event.record(), &property.name).unwrap() as usize, + ); + } + PropertyLength::Length(length) => { + // TODO: Study heuristic method used in krabsetw :) + if property.flags.is_empty() && length > 0 && property.count == 1 { + return Ok(length as usize); + } + if property.count == 1 { + if let PropertyDesc::Primitive(desc) = &property.desc { + match desc.in_type { + TdhInType::InTypeBoolean => return Ok(4), + TdhInType::InTypeInt32 + | TdhInType::InTypeUInt32 + | TdhInType::InTypeHexInt32 => return Ok(4), + TdhInType::InTypeInt64 + | TdhInType::InTypeUInt64 + | TdhInType::InTypeHexInt64 => return Ok(8), + TdhInType::InTypeInt8 | TdhInType::InTypeUInt8 => return Ok(1), + TdhInType::InTypeInt16 | TdhInType::InTypeUInt16 => return Ok(2), + TdhInType::InTypePointer => { + return Ok( + if (self.event.event_flags() & EVENT_HEADER_FLAG_32_BIT_HEADER) + != 0 + { + 4 + } else { + 8 + }, + ) + } + TdhInType::InTypeGuid => return Ok(std::mem::size_of::()), + TdhInType::InTypeUnicodeString => { + return Ok(utils::parse_unk_size_null_unicode_size(&self.buffer)) + } + TdhInType::InTypeAnsiString => { + return Ok(utils::parse_unk_size_null_ansi_size(&self.buffer)); + } + _ => {} + } + } + } + return Ok( + tdh::property_size(self.event.record(), &property.name).unwrap() as usize, + ); + } + } + } + + pub fn find_property(&mut self, name: &str) -> ParserResult { + let indx = *self + .properties + .name_to_indx + .get(name) + .ok_or_else(|| ParserError::PropertyError("Unknown property".to_owned()))?; + if indx < self.cache.len() { + return Ok(indx); + } + + // TODO: Find a way to do this with an iter, try_find looks promising but is not stable yet + // TODO: Clean this a bit, not a big fan of this loop + for i in self.cache.len()..=indx { + let curr_prop = self.properties.property(i).unwrap(); + + let prop_size = self.find_property_size(&curr_prop)?; + + if self.buffer.len() < prop_size { + return Err(ParserError::PropertyError(format!( + "Property of {} bytes out of buffer bounds ({})", + prop_size, + self.buffer.len() + ))); + } + + // We split the buffer, if everything works correctly in the end the buffer will be empty + // and we should have all properties in the cache + let (prop_buffer, remaining) = self.buffer.split_at(prop_size); + self.buffer = remaining; + self.cache + .push(PropertyInfo::create(curr_prop, self.offset, prop_buffer)); + self.offset += prop_size; + } + Ok(indx) + } +} + +/* +impl<'a> std::fmt::Debug for Parser<'a> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut s = f.debug_struct("ParsedEvent"); + for i in 0..self.event.property_count() { + let property = self.event.property(i); + let value = match property.in_type() { + TdhInType::InTypeUnicodeString => format!("{}", TryParse::::parse(self, &property.name)), + TdhInType::InTypeAnsiString => format!("{}", TryParse::::parse(self, &property.name)), + TdhInType::InTypeUInt32 => format!("{}", TryParse::::parse(self, &property.name)), + TdhInType::InTypeUInt8 => format!("{}", TryParse::::parse(self, &property.name)), + TdhInType::InTypePointer => format!("{}", TryParse::::parse(self, &property.name)), + TdhInType::InTypeInt64 => format!("{}", TryParse::::parse(self, &property.name)), + TdhInType::InTypeUInt64 => format!("{}", TryParse::::parse(self, &property.name)), + TdhInType::InTypeGuid => format!("{:?}", TryParse::::parse(self, &property.name)), + _ => panic!() + }; + s.field(&property.name, &value); + //dbg!(&property); + } + s.finish() + } +}*/ + +macro_rules! impl_try_parse_primitive { + ($T:ident, $ty:ident) => { + impl TryParse<$T> for Parser<'_> { + fn try_parse(&mut self, name: &str) -> ParserResult<$T> { + use TdhInType::*; + let indx = self.find_property(name)?; + let prop_info = &self.cache[indx]; + let prop_info: &PropertyInfo = prop_info.borrow(); + if let PropertyDesc::Primitive(desc) = &prop_info.property.desc { + if desc.in_type != $ty { + return Err(ParserError::InvalidType); + } + if std::mem::size_of::<$T>() != prop_info.buffer.len() { + return Err(ParserError::LengthMismatch); + } + return Ok($T::from_ne_bytes(prop_info.buffer.try_into()?)); + }; + Err(ParserError::InvalidType) + } + } + }; +} + +impl_try_parse_primitive!(u8, InTypeUInt8); +impl_try_parse_primitive!(i8, InTypeInt8); +impl_try_parse_primitive!(u16, InTypeUInt16); +impl_try_parse_primitive!(i16, InTypeInt16); +impl_try_parse_primitive!(u32, InTypeUInt32); +//impl_try_parse_primitive!(u64, InTypeUInt64); +//impl_try_parse_primitive!(i64, InTypeInt64); + +impl TryParse for Parser<'_> { + fn try_parse(&mut self, name: &str) -> ParserResult { + use TdhInType::*; + let indx = self.find_property(name)?; + let prop_info = &self.cache[indx]; + if let PropertyDesc::Primitive(desc) = &prop_info.property.desc { + if desc.in_type == InTypeUInt64 { + if std::mem::size_of::() != prop_info.buffer.len() { + return Err(ParserError::LengthMismatch); + } + return Ok(u64::from_ne_bytes(prop_info.buffer.try_into()?)); + } + if desc.in_type == InTypePointer || desc.in_type == InTypeSizeT { + if (self.event.event_flags() & EVENT_HEADER_FLAG_32_BIT_HEADER) != 0 { + if std::mem::size_of::() != prop_info.buffer.len() { + return Err(ParserError::LengthMismatch); + } + return Ok(u32::from_ne_bytes(prop_info.buffer.try_into()?) as u64); + } + if std::mem::size_of::() != prop_info.buffer.len() { + return Err(ParserError::LengthMismatch); + } + return Ok(u64::from_ne_bytes(prop_info.buffer.try_into()?)); + } + } + return Err(ParserError::InvalidType); + } +} + +impl TryParse for Parser<'_> { + fn try_parse(&mut self, name: &str) -> ParserResult { + use TdhInType::*; + let indx = self.find_property(name)?; + let prop_info = &self.cache[indx]; + if let PropertyDesc::Primitive(desc) = &prop_info.property.desc { + if desc.in_type == InTypeInt64 || desc.in_type == InTypeHexInt64 { + if std::mem::size_of::() != prop_info.buffer.len() { + return Err(ParserError::LengthMismatch); + } + return Ok(i64::from_ne_bytes(prop_info.buffer.try_into()?)); + } + } + return Err(ParserError::InvalidType); + } +} + +impl TryParse for Parser<'_> { + fn try_parse(&mut self, name: &str) -> ParserResult { + use TdhInType::*; + let indx = self.find_property(name)?; + let prop_info = &self.cache[indx]; + if let PropertyDesc::Primitive(desc) = &prop_info.property.desc { + if desc.in_type == InTypeInt32 || desc.in_type == InTypeHexInt32 { + if std::mem::size_of::() != prop_info.buffer.len() { + return Err(ParserError::LengthMismatch); + } + return Ok(i32::from_ne_bytes(prop_info.buffer.try_into()?)); + } + } + return Err(ParserError::InvalidType); + } +} + +impl TryParse
for Parser<'_> { + fn try_parse(&mut self, name: &str) -> ParserResult
{ + use TdhInType::*; + let indx = self.find_property(name)?; + let prop_info = &self.cache[indx]; + + if let PropertyDesc::Primitive(desc) = &prop_info.property.desc { + if self.event.is_64bit() { + if desc.in_type == InTypeUInt64 + || desc.in_type == InTypePointer + || desc.in_type == InTypeHexInt64 + { + if std::mem::size_of::() != prop_info.buffer.len() { + return Err(ParserError::LengthMismatch); + } + return Ok(Address::Address64(u64::from_ne_bytes( + prop_info.buffer.try_into()?, + ))); + } + } else { + if desc.in_type == InTypeUInt32 + || desc.in_type == InTypePointer + || desc.in_type == InTypeHexInt32 + { + if std::mem::size_of::() != prop_info.buffer.len() { + return Err(ParserError::LengthMismatch); + } + return Ok(Address::Address32(u32::from_ne_bytes( + prop_info.buffer.try_into()?, + ))); + } + } + } + return Err(ParserError::InvalidType); + } +} + +impl TryParse for Parser<'_> { + fn try_parse(&mut self, name: &str) -> ParserResult { + use TdhInType::*; + let indx = self.find_property(name)?; + let prop_info = &self.cache[indx]; + if let PropertyDesc::Primitive(desc) = &prop_info.property.desc { + if desc.in_type != InTypeBoolean { + return Err(ParserError::InvalidType); + } + if prop_info.buffer.len() != 4 { + return Err(ParserError::LengthMismatch); + } + return match u32::from_ne_bytes(prop_info.buffer.try_into()?) { + 1 => Ok(true), + 0 => Ok(false), + _ => Err(ParserError::InvalidType), + }; + }; + Err(ParserError::InvalidType) + } +} + +impl TryParse for Parser<'_> { + fn try_parse(&mut self, name: &str) -> ParserResult { + use TdhInType::*; + let indx = self.find_property(name)?; + let prop_info = &self.cache[indx]; + if let PropertyDesc::Primitive(desc) = &prop_info.property.desc { + if desc.in_type == InTypeFloat { + if std::mem::size_of::() != prop_info.buffer.len() { + return Err(ParserError::LengthMismatch); + } + return Ok(f32::from_ne_bytes(prop_info.buffer.try_into()?)); + } + } + return Err(ParserError::InvalidType); + } +} + +/// The `String` impl of the `TryParse` trait should be used to retrieve the following [TdhInTypes]: +/// +/// * InTypeUnicodeString +/// * InTypeAnsiString +/// * InTypeCountedString +/// * InTypeGuid +/// +/// On success a `String` with the with the data from the `name` property will be returned +/// +/// # Arguments +/// * `name` - Name of the property to be found in the Schema + +/// # Example +/// ```rust +/// let my_callback = |record: EventRecord, schema_locator: &mut SchemaLocator| { +/// let schema = schema_locator.event_schema(record)?; +/// let parser = Parse::create(&schema); +/// let image_name: String = parser.try_parse("ImageName")?; +/// }; +/// ``` +/// +/// [TdhInTypes]: TdhInType +impl TryParse for Parser<'_> { + fn try_parse(&mut self, name: &str) -> ParserResult { + let indx = self.find_property(name)?; + let prop_info = &self.cache[indx]; + + // TODO: Handle errors and type checking better + if let PropertyDesc::Primitive(desc) = &prop_info.property.desc { + let res = match desc.in_type { + TdhInType::InTypeUnicodeString => utils::parse_null_utf16_string(prop_info.buffer), + TdhInType::InTypeAnsiString => String::from_utf8(prop_info.buffer.to_vec())? + .trim_matches(char::default()) + .to_string(), + TdhInType::InTypeSid => { + panic!() + //sddl::convert_sid_to_string(prop_info.buffer.as_ptr() as isize)? + } + TdhInType::InTypeCountedString => unimplemented!(), + _ => return Err(ParserError::InvalidType), + }; + return Ok(res); + } + Err(ParserError::InvalidType) + } +} + +impl TryParse for Parser<'_> { + fn try_parse(&mut self, name: &str) -> Result { + let indx = self.find_property(name)?; + let prop_info = &self.cache[indx]; + if let PropertyDesc::Primitive(desc) = &prop_info.property.desc { + match desc.in_type { + TdhInType::InTypeUnicodeString => { + let guid_string = utils::parse_utf16_guid(prop_info.buffer); + + if guid_string.len() != 36 { + return Err(ParserError::LengthMismatch); + } + + return Ok(GUID::from(guid_string.as_str())); + } + TdhInType::InTypeGuid => { + return Ok(GUID::from_values( + u32::from_ne_bytes((&prop_info.buffer[0..4]).try_into()?), + u16::from_ne_bytes((&prop_info.buffer[4..6]).try_into()?), + u16::from_ne_bytes((&prop_info.buffer[6..8]).try_into()?), + [ + prop_info.buffer[8], + prop_info.buffer[9], + prop_info.buffer[10], + prop_info.buffer[11], + prop_info.buffer[12], + prop_info.buffer[13], + prop_info.buffer[14], + prop_info.buffer[15], + ], + )) + } + _ => return Err(ParserError::InvalidType), + } + }; + Err(ParserError::InvalidType) + } +} + +impl TryParse for Parser<'_> { + fn try_parse(&mut self, name: &str) -> ParserResult { + let indx = self.find_property(name)?; + let prop_info = &self.cache[indx]; + if let PropertyDesc::Primitive(desc) = &prop_info.property.desc { + if desc.out_type != TdhOutType::OutTypeIpv4 && desc.out_type != TdhOutType::OutTypeIpv6 + { + return Err(ParserError::InvalidType); + } + + // Hardcoded values for now + let res = match prop_info.property.length { + PropertyLength::Length(16) => { + let tmp: [u8; 16] = prop_info.buffer.try_into()?; + IpAddr::V6(Ipv6Addr::from(tmp)) + } + PropertyLength::Length(4) => { + let tmp: [u8; 4] = prop_info.buffer.try_into()?; + IpAddr::V4(Ipv4Addr::from(tmp)) + } + _ => return Err(ParserError::LengthMismatch), + }; + + return Ok(res); + } + Err(ParserError::InvalidType) + } +} + +#[derive(Clone, Default, Debug)] +pub struct Pointer(usize); + +impl std::ops::Deref for Pointer { + type Target = usize; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl std::ops::DerefMut for Pointer { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +impl std::fmt::LowerHex for Pointer { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let val = self.0; + + std::fmt::LowerHex::fmt(&val, f) // delegate to u32/u64 implementation + } +} + +impl std::fmt::UpperHex for Pointer { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let val = self.0; + + std::fmt::UpperHex::fmt(&val, f) // delegate to u32/u64 implementation + } +} + +impl std::fmt::Display for Pointer { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let val = self.0; + + std::fmt::Display::fmt(&val, f) // delegate to u32/u64 implementation + } +} + +impl TryParse for Parser<'_> { + fn try_parse(&mut self, name: &str) -> ParserResult { + let indx = self.find_property(name)?; + let prop_info = &self.cache[indx]; + + let mut res = Pointer::default(); + if prop_info.buffer.len() == std::mem::size_of::() { + res.0 = TryParse::::try_parse(self, name)? as usize; + } else { + res.0 = TryParse::::try_parse(self, name)? as usize; + } + + Ok(res) + } +} + +impl TryParse> for Parser<'_> { + fn try_parse(&mut self, name: &str) -> Result, ParserError> { + let indx = self.find_property(name)?; + let prop_info = &self.cache[indx]; + + Ok(prop_info.buffer.to_vec()) + } +} + +// TODO: Implement SocketAddress +// TODO: Study if we can use primitive types for HexInt64, HexInt32 and Pointer diff --git a/etw-reader/src/property.rs b/etw-reader/src/property.rs index 0864047c..997f1142 100644 --- a/etw-reader/src/property.rs +++ b/etw-reader/src/property.rs @@ -1,51 +1,58 @@ -//! ETW Event Property information -//! -//! The `property` module expose the basic structures that represent the Properties an Event contains -//! based on it's Schema. This Properties can then be used to parse accordingly their values. -use crate::FastHashMap; -use crate::tdh_types::Property; -use crate::schema::Schema; - -/// Event Property information -#[derive(Clone, Debug)] -pub struct PropertyInfo<'a> { - /// Property attributes - pub property: &'a Property, - pub offset: usize, - /// Buffer with the Property data - pub buffer: &'a [u8], -} - -impl<'a> PropertyInfo<'a> { - pub fn create(property: &'a Property, offset: usize, buffer: &'a [u8]) -> Self { - PropertyInfo { property, offset, buffer } - } -} - -pub(crate) struct PropertyIter { - properties: Vec, - pub (crate) name_to_indx: FastHashMap, -} - -impl PropertyIter { - pub fn new(schema: &Schema) -> Self { - let prop_count = schema.event_schema.property_count(); - let mut properties = Vec::new(); - let mut name_to_indx = FastHashMap::default(); - for i in 0..prop_count { - let prop = schema.event_schema.property(i); - name_to_indx.insert(prop.name.clone(), i as usize); - properties.push(prop); - } - - PropertyIter { properties, name_to_indx } - } - - pub fn property(&self, index: usize) -> Option<&Property> { - self.properties.get(index) - } - - pub fn properties_iter(&self) -> &[Property] { - &self.properties - } -} +//! ETW Event Property information +//! +//! The `property` module expose the basic structures that represent the Properties an Event contains +//! based on it's Schema. This Properties can then be used to parse accordingly their values. +use super::schema::Schema; +use super::tdh_types::Property; +use super::FastHashMap; + +/// Event Property information +#[derive(Clone, Debug)] +pub struct PropertyInfo<'a> { + /// Property attributes + pub property: &'a Property, + pub offset: usize, + /// Buffer with the Property data + pub buffer: &'a [u8], +} + +impl<'a> PropertyInfo<'a> { + pub fn create(property: &'a Property, offset: usize, buffer: &'a [u8]) -> Self { + PropertyInfo { + property, + offset, + buffer, + } + } +} + +pub(crate) struct PropertyIter { + properties: Vec, + pub(crate) name_to_indx: FastHashMap, +} + +impl PropertyIter { + pub fn new(schema: &Schema) -> Self { + let prop_count = schema.event_schema.property_count(); + let mut properties = Vec::new(); + let mut name_to_indx = FastHashMap::default(); + for i in 0..prop_count { + let prop = schema.event_schema.property(i); + name_to_indx.insert(prop.name.clone(), i as usize); + properties.push(prop); + } + + PropertyIter { + properties, + name_to_indx, + } + } + + pub fn property(&self, index: usize) -> Option<&Property> { + self.properties.get(index) + } + + pub fn properties_iter(&self) -> &[Property] { + &self.properties + } +} diff --git a/etw-reader/src/schema.rs b/etw-reader/src/schema.rs index eaedf843..ea689a47 100644 --- a/etw-reader/src/schema.rs +++ b/etw-reader/src/schema.rs @@ -1,452 +1,479 @@ -//! ETW Event Schema locator and handler -//! -//! This module contains the means needed to locate and interact with the Schema of an ETW event -use crate::custom_schemas::EventInfo; -use crate::etw_types::{DecodingSource, EventRecord, TraceEventInfoRaw}; -use crate::property::PropertyIter; -use crate::tdh; -use crate::tdh_types::Property; -use crate::FastHashMap; -use std::any::{Any, TypeId}; -use std::collections::hash_map::Entry; -use std::sync::Arc; -use once_cell::unsync::OnceCell; -use windows::Win32::System::Diagnostics::Etw::{self, EVENT_HEADER_FLAG_64_BIT_HEADER}; -use windows::core::GUID; - -/// Schema module errors -#[derive(Debug)] -pub enum SchemaError { - /// Represents a Parser error - ParseError, - /// Represents an internal [TdhNativeError] - /// - /// [TdhNativeError]: tdh::TdhNativeError - TdhNativeError(tdh::TdhNativeError), -} - -impl From for SchemaError { - fn from(err: tdh::TdhNativeError) -> Self { - SchemaError::TdhNativeError(err) - } -} - -type SchemaResult = Result; - -// TraceEvent::RegisteredTraceEventParser::ExternalTraceEventParserState::TraceEventComparer -// doesn't compare the version or level and does different things depending on the kind of event -// https://github.com/microsoft/perfview/blob/5c9f6059f54db41b4ac5c4fc8f57261779634489/src/TraceEvent/RegisteredTraceEventParser.cs#L1338 -#[derive(Debug, Eq, PartialEq, Hash)] -struct SchemaKey { - provider: GUID, - id: u16, - version: u8, - level: u8, - opcode: u8, -} - -// A map from tracelogging schema metdata to ids -struct TraceLoggingProviderIds { - ids: FastHashMap, u16>, - next_id: u16, -} - -impl TraceLoggingProviderIds { - fn new() -> Self { - // start the ids at 1 because of 0 is typically the value stored - TraceLoggingProviderIds { ids: FastHashMap::default(), next_id: 1} - } -} - -impl SchemaKey { - pub fn new(event: &EventRecord, locator: &mut SchemaLocator) -> Self { - // TraceLogging events all use the same id and are distinguished from each other using the metadata. - // Instead of storing the metadata in the SchemaKey we follow the approach of PerfView and have a side table of synthetic ids keyed on metadata. - // It might be better to store the metadata in the SchemaKey but then we may want to be careful not to allocate a fresh metadata for every event. - let mut id = event.EventHeader.EventDescriptor.Id; - if event.ExtendedDataCount > 0 { - let extended = unsafe { std::slice::from_raw_parts(event.ExtendedData, event.ExtendedDataCount as usize) }; - for e in extended { - if e.ExtType as u32 == Etw::EVENT_HEADER_EXT_TYPE_EVENT_SCHEMA_TL { - let mut provider = locator.tracelogging_providers.entry(event.EventHeader.ProviderId).or_insert(TraceLoggingProviderIds::new()); - let data = unsafe { std::slice::from_raw_parts(e.DataPtr as *const u8, e.DataSize as usize) } ; - if let Some(metadata_id) = provider.ids.get(data) { - // we want to ensure that our synthetic ids don't overlap with any ids used in the events - assert_ne!(id, *metadata_id); - id = *metadata_id; - } else { - provider.ids.insert(data.to_vec(), provider.next_id); - id = provider.next_id; - provider.next_id += 1; - } - } - } - } - SchemaKey { - provider: event.EventHeader.ProviderId, - id, - version: event.EventHeader.EventDescriptor.Version, - level: event.EventHeader.EventDescriptor.Level, - opcode: event.EventHeader.EventDescriptor.Opcode, - } - } -} - -/// Represents a cache of Schemas already located -/// -/// This cache is implemented as a [HashMap] where the key is a combination of the following elements -/// of an [Event Record](https://docs.microsoft.com/en-us/windows/win32/api/evntcons/ns-evntcons-event_record) -/// * EventHeader.ProviderId -/// * EventHeader.EventDescriptor.Id -/// * EventHeader.EventDescriptor.Opcode -/// * EventHeader.EventDescriptor.Version -/// * EventHeader.EventDescriptor.Level -/// -/// Credits: [KrabsETW::schema_locator](https://github.com/microsoft/krabsetw/blob/master/krabs/krabs/schema_locator.hpp) -#[derive(Default)] -pub struct SchemaLocator { - schemas: FastHashMap>, - tracelogging_providers: FastHashMap, -} - - - -pub trait EventSchema { - fn decoding_source(&self) -> DecodingSource; - - fn provider_guid(&self) -> GUID; - fn event_id(&self) -> u16; - fn opcode(&self) -> u8; - fn event_version(&self) -> u8; - fn provider_name(&self) -> String; - fn task_name(&self) -> String; - fn opcode_name(&self) -> String; - fn level(&self) -> u8; - - fn property_count(&self) -> u32; - fn property(&self, index: u32) -> Property; - - fn event_message(&self) -> Option { return None } - fn is_event_metadata(&self) -> bool { return false } -} - - - -impl std::fmt::Debug for SchemaLocator { - fn fmt(&self, _f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - todo!() - } -} - -impl SchemaLocator { - pub fn new() -> Self { - SchemaLocator { - schemas: FastHashMap::default(), - tracelogging_providers: FastHashMap::default(), - } - } - - pub fn add_custom_schema(&mut self, schema: Box) { - let key = SchemaKey { - provider: schema.provider_guid(), - id: schema.event_id(), - opcode: schema.opcode(), - version: schema.event_version(), - level: schema.level() }; - self.schemas.insert(key, Arc::new(Schema::new(schema))); - } - - /// Use the `event_schema` function to retrieve the Schema of an ETW Event - /// - /// # Arguments - /// * `event` - The [EventRecord] that's passed to the callback - /// - /// # Remark - /// This is the first function that should be called within a Provider callback, if everything - /// works as expected this function will return a Result with the [Schema] that represents - /// the ETW event that triggered the callback - /// - /// This function can fail, if it does it will return a [SchemaError] - /// - /// # Example - /// ```rust - /// let my_callback = |record: EventRecord, schema_locator: &mut SchemaLocator| { - /// let schema = schema_locator.event_schema(record)?; - /// }; - /// ``` - pub fn event_schema<'a>(&mut self, event: &'a EventRecord) -> SchemaResult> { - let key = SchemaKey::new(&event, self); - let info = match self.schemas.entry(key) { - Entry::Occupied(entry) => entry.into_mut(), - Entry::Vacant(entry) => { - let info = Box::new(tdh::schema_from_tdh(event.clone())?); - // dbg!(info.provider_guid(), info.provider_name(), info.decoding_source()); - // TODO: Cloning for now, should be a reference at some point... - entry.insert(Arc::new(Schema::new(info))) - } - }.clone(); - - // Some events contain schemas so add them when we find them. - if info.event_schema.is_event_metadata() { - let event_info = TraceEventInfoRaw::new(event.user_buffer().to_owned()); - println!("Adding custom schema for {}/{}/{}/{}", event_info.provider_name(), event_info.event_id(), event_info.task_name(), event_info.opcode_name()); - self.add_custom_schema(Box::new(event_info)); - } - - Ok(TypedEvent::new(event, info)) - } -} - -pub struct Schema { - pub event_schema: Box, - properties: OnceCell, - name: OnceCell, -} - -impl Schema { - fn new(event_schema: Box) -> Self { - Schema { event_schema, properties: OnceCell::new(), name: OnceCell::new() } - } - pub(crate) fn properties(&self) -> &PropertyIter { - self.properties.get_or_init(|| PropertyIter::new(self)) - } - pub (crate) fn name(&self) -> &str { - self.name.get_or_init(|| format!("{}/{}/{}", - self.event_schema.provider_name(), - self.event_schema.task_name(), - self.event_schema.opcode_name())) - } -} - -pub struct TypedEvent<'a> { - record: &'a EventRecord, - pub (crate) schema: Arc, -} - -impl<'a> TypedEvent<'a> { - pub fn new(record: &'a EventRecord, schema: Arc) -> Self { - TypedEvent { record, schema } - } - - pub(crate) fn user_buffer(&self) -> &[u8] { - self.record.user_buffer() - } - - // Horrible getters FTW!! :D - // TODO: Not a big fan of this, think a better way.. - pub(crate) fn record(&self) -> &EventRecord { - self.record - } - - /// Use the `event_id` function to obtain the EventId of the Event Record - /// - /// This getter returns the EventId of the ETW Event that triggered the registered callback - /// - /// # Example - /// ```rust - /// let my_callback = |record: EventRecord, schema_locator: &mut SchemaLocator| { - /// let schema = schema_locator.event_schema(record)?; - /// let event_id = schema.event_id(); - /// }; - /// ``` - pub fn event_id(&self) -> u16 { - self.record.EventHeader.EventDescriptor.Id - } - - /// Use the `opcode` function to obtain the Opcode of the Event Record - /// - /// This getter returns the opcode of the ETW Event that triggered the registered callback - /// - /// # Example - /// ```rust - /// let my_callback = |record: EventRecord, schema_locator: &mut SchemaLocator| { - /// let schema = schema_locator.event_schema(record)?; - /// let event_id = schema.opcode(); - /// }; - /// ``` - pub fn opcode(&self) -> u8 { - self.record.EventHeader.EventDescriptor.Opcode - } - - /// Use the `event_flags` function to obtain the Event Flags of the [EventRecord] - /// - /// This getter returns the Event Flags of the ETW Event that triggered the registered callback - /// - /// # Example - /// ```rust - /// let my_callback = |record: EventRecord, schema_locator: &mut SchemaLocator| { - /// let schema = schema_locator.event_schema(record)?; - /// let event_flags = schema.event_flags(); - /// }; - /// ``` - pub fn event_flags(&self) -> u16 { - self.record.EventHeader.Flags - } - - pub fn is_64bit(&self) -> bool { - (self.record.EventHeader.Flags & EVENT_HEADER_FLAG_64_BIT_HEADER as u16) != 0 - } - - /// Use the `event_version` function to obtain the Version of the [EventRecord] - /// - /// This getter returns the Version of the ETW Event that triggered the registered callback - /// - /// # Example - /// ```rust - /// let my_callback = |record: EventRecord, schema_locator: &mut SchemaLocator| { - /// let schema = schema_locator.event_schema(record)?; - /// let event_version = schema.event_version(); - /// }; - /// ``` - pub fn event_version(&self) -> u8 { - self.record.EventHeader.EventDescriptor.Version - } - - /// Use the `process_id` function to obtain the ProcessId of the [EventRecord] - /// - /// This getter returns the ProcessId of the process that triggered the ETW Event - /// - /// # Example - /// ```rust - /// let my_callback = |record: EventRecord, schema_locator: &mut SchemaLocator| { - /// let schema = schema_locator.event_schema(record)?; - /// let pid = schema.process_id(); - /// }; - /// ``` - pub fn process_id(&self) -> u32 { - self.record.EventHeader.ProcessId - } - - /// Use the `thread_id` function to obtain the ThreadId of the [EventRecord] - /// - /// This getter returns the ThreadId of the thread that triggered the ETW Event - /// - /// # Example - /// ```rust - /// let my_callback = |record: EventRecord, schema_locator: &mut SchemaLocator| { - /// let schema = schema_locator.event_schema(record)?; - /// let tid = schema.thread_id(); - /// }; - /// ``` - pub fn thread_id(&self) -> u32 { - self.record.EventHeader.ThreadId - } - - /// Use the `timestamp` function to obtain the TimeStamp of the [EventRecord] - /// - /// This getter returns the TimeStamp of the ETW Event - /// - /// # Example - /// ```rust - /// let my_callback = |record: EventRecord, schema_locator: &mut SchemaLocator| { - /// let schema = schema_locator.event_schema(record)?; - /// let timestamp = schema.timestamp(); - /// }; - /// ``` - pub fn timestamp(&self) -> i64 { - self.record.EventHeader.TimeStamp - } - - /// Use the `activity_id` function to obtain the ActivityId of the [EventRecord] - /// - /// This getter returns the ActivityId from the ETW Event, this value is used to related Two events - /// - /// # Example - /// ```rust - /// let my_callback = |record: EventRecord, schema_locator: &mut SchemaLocator| { - /// let schema = schema_locator.event_schema(record)?; - /// let activity_id = schema.activity_id(); - /// }; - /// ``` - /// [TraceEventInfo]: crate::native::etw_types::TraceEventInfo - pub fn activity_id(&self) -> GUID { - self.record.EventHeader.ActivityId - } - - /// Use the `decoding_source` function to obtain the [DecodingSource] from the [TraceEventInfo] - /// - /// This getter returns the DecodingSource from the event, this value identifies the source used - /// parse the event data - /// - /// # Example - /// ```rust - /// let my_callback = |record: EventRecord, schema_locator: &mut SchemaLocator| { - /// let schema = schema_locator.event_schema(record)?; - /// let decoding_source = schema.decoding_source(); - /// }; - /// ``` - /// [TraceEventInfo]: crate::native::etw_types::TraceEventInfo - pub fn decoding_source(&self) -> DecodingSource { - self.schema.event_schema.decoding_source() - } - - /// Use the `provider_name` function to obtain the Provider name from the [TraceEventInfo] - /// - /// # Example - /// ```rust - /// let my_callback = |record: EventRecord, schema_locator: &mut SchemaLocator| { - /// let schema = schema_locator.event_schema(record)?; - /// let provider_name = schema.provider_name(); - /// }; - /// ``` - /// [TraceEventInfo]: crate::native::etw_types::TraceEventInfo - pub fn provider_name(&self) -> String { - self.schema.event_schema.provider_name() - } - - /// Use the `task_name` function to obtain the Task name from the [TraceEventInfo] - /// - /// See: [TaskType](https://docs.microsoft.com/en-us/windows/win32/wes/eventmanifestschema-tasktype-complextype) - /// # Example - /// ```rust - /// let my_callback = |record: EventRecord, schema_locator: &mut SchemaLocator| { - /// let schema = schema_locator.event_schema(record)?; - /// let task_name = schema.task_name(); - /// }; - /// ``` - /// [TraceEventInfo]: crate::native::etw_types::TraceEventInfo - pub fn task_name(&self) -> String { - self.schema.event_schema.task_name() - } - - /// Use the `opcode_name` function to obtain the Opcode name from the [TraceEventInfo] - /// - /// See: [OpcodeType](https://docs.microsoft.com/en-us/windows/win32/wes/eventmanifestschema-opcodetype-complextype) - /// # Example - /// ```rust - /// let my_callback = |record: EventRecord, schema_locator: &mut SchemaLocator| { - /// let schema = schema_locator.event_schema(record)?; - /// let opcode_name = schema.opcode_name(); - /// }; - /// ``` - /// [TraceEventInfo]: crate::native::etw_types::TraceEventInfo - pub fn opcode_name(&self) -> String { - self.schema.event_schema.opcode_name() - } - - pub fn property_count(&self) -> u32 { - self.schema.event_schema.property_count() - } - - pub fn property(&self, index: u32) -> Property { - self.schema.event_schema.property(index) - } - - pub fn name(&self) -> &str { - self.schema.name() - } - - pub fn event_message(&self) -> Option { - self.schema.event_schema.event_message() - } -} - -impl<'a> PartialEq for TypedEvent<'a> { - fn eq(&self, other: &Self) -> bool { - self.schema.event_schema.event_id() == other.schema.event_schema.event_id() - && self.schema.event_schema.provider_guid() == other.schema.event_schema.provider_guid() - && self.schema.event_schema.event_version() == other.schema.event_schema.event_version() - } -} - -impl<'a> Eq for TypedEvent<'a> {} +//! ETW Event Schema locator and handler +//! +//! This module contains the means needed to locate and interact with the Schema of an ETW event +use std::any::{Any, TypeId}; +use std::collections::hash_map::Entry; +use std::sync::Arc; + +use once_cell::unsync::OnceCell; +use windows::core::GUID; +use windows::Win32::System::Diagnostics::Etw::{self, EVENT_HEADER_FLAG_64_BIT_HEADER}; + +use super::custom_schemas::EventInfo; +use super::etw_types::{DecodingSource, EventRecord, TraceEventInfoRaw}; +use super::property::PropertyIter; +use super::tdh_types::Property; +use super::{tdh, FastHashMap}; + +/// Schema module errors +#[derive(Debug)] +pub enum SchemaError { + /// Represents a Parser error + ParseError, + /// Represents an internal [TdhNativeError] + /// + /// [TdhNativeError]: tdh::TdhNativeError + TdhNativeError(tdh::TdhNativeError), +} + +impl From for SchemaError { + fn from(err: tdh::TdhNativeError) -> Self { + SchemaError::TdhNativeError(err) + } +} + +type SchemaResult = Result; + +// TraceEvent::RegisteredTraceEventParser::ExternalTraceEventParserState::TraceEventComparer +// doesn't compare the version or level and does different things depending on the kind of event +// https://github.com/microsoft/perfview/blob/5c9f6059f54db41b4ac5c4fc8f57261779634489/src/TraceEvent/RegisteredTraceEventParser.cs#L1338 +#[derive(Debug, Eq, PartialEq, Hash)] +struct SchemaKey { + provider: GUID, + id: u16, + version: u8, + level: u8, + opcode: u8, +} + +// A map from tracelogging schema metdata to ids +struct TraceLoggingProviderIds { + ids: FastHashMap, u16>, + next_id: u16, +} + +impl TraceLoggingProviderIds { + fn new() -> Self { + // start the ids at 1 because of 0 is typically the value stored + TraceLoggingProviderIds { + ids: FastHashMap::default(), + next_id: 1, + } + } +} + +impl SchemaKey { + pub fn new(event: &EventRecord, locator: &mut SchemaLocator) -> Self { + // TraceLogging events all use the same id and are distinguished from each other using the metadata. + // Instead of storing the metadata in the SchemaKey we follow the approach of PerfView and have a side table of synthetic ids keyed on metadata. + // It might be better to store the metadata in the SchemaKey but then we may want to be careful not to allocate a fresh metadata for every event. + let mut id = event.EventHeader.EventDescriptor.Id; + if event.ExtendedDataCount > 0 { + let extended = unsafe { + std::slice::from_raw_parts(event.ExtendedData, event.ExtendedDataCount as usize) + }; + for e in extended { + if e.ExtType as u32 == Etw::EVENT_HEADER_EXT_TYPE_EVENT_SCHEMA_TL { + let mut provider = locator + .tracelogging_providers + .entry(event.EventHeader.ProviderId) + .or_insert(TraceLoggingProviderIds::new()); + let data = unsafe { + std::slice::from_raw_parts(e.DataPtr as *const u8, e.DataSize as usize) + }; + if let Some(metadata_id) = provider.ids.get(data) { + // we want to ensure that our synthetic ids don't overlap with any ids used in the events + assert_ne!(id, *metadata_id); + id = *metadata_id; + } else { + provider.ids.insert(data.to_vec(), provider.next_id); + id = provider.next_id; + provider.next_id += 1; + } + } + } + } + SchemaKey { + provider: event.EventHeader.ProviderId, + id, + version: event.EventHeader.EventDescriptor.Version, + level: event.EventHeader.EventDescriptor.Level, + opcode: event.EventHeader.EventDescriptor.Opcode, + } + } +} + +/// Represents a cache of Schemas already located +/// +/// This cache is implemented as a [HashMap] where the key is a combination of the following elements +/// of an [Event Record](https://docs.microsoft.com/en-us/windows/win32/api/evntcons/ns-evntcons-event_record) +/// * EventHeader.ProviderId +/// * EventHeader.EventDescriptor.Id +/// * EventHeader.EventDescriptor.Opcode +/// * EventHeader.EventDescriptor.Version +/// * EventHeader.EventDescriptor.Level +/// +/// Credits: [KrabsETW::schema_locator](https://github.com/microsoft/krabsetw/blob/master/krabs/krabs/schema_locator.hpp) +#[derive(Default)] +pub struct SchemaLocator { + schemas: FastHashMap>, + tracelogging_providers: FastHashMap, +} + +pub trait EventSchema { + fn decoding_source(&self) -> DecodingSource; + + fn provider_guid(&self) -> GUID; + fn event_id(&self) -> u16; + fn opcode(&self) -> u8; + fn event_version(&self) -> u8; + fn provider_name(&self) -> String; + fn task_name(&self) -> String; + fn opcode_name(&self) -> String; + fn level(&self) -> u8; + + fn property_count(&self) -> u32; + fn property(&self, index: u32) -> Property; + + fn event_message(&self) -> Option { + return None; + } + fn is_event_metadata(&self) -> bool { + return false; + } +} + +impl std::fmt::Debug for SchemaLocator { + fn fmt(&self, _f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + todo!() + } +} + +impl SchemaLocator { + pub fn new() -> Self { + SchemaLocator { + schemas: FastHashMap::default(), + tracelogging_providers: FastHashMap::default(), + } + } + + pub fn add_custom_schema(&mut self, schema: Box) { + let key = SchemaKey { + provider: schema.provider_guid(), + id: schema.event_id(), + opcode: schema.opcode(), + version: schema.event_version(), + level: schema.level(), + }; + self.schemas.insert(key, Arc::new(Schema::new(schema))); + } + + /// Use the `event_schema` function to retrieve the Schema of an ETW Event + /// + /// # Arguments + /// * `event` - The [EventRecord] that's passed to the callback + /// + /// # Remark + /// This is the first function that should be called within a Provider callback, if everything + /// works as expected this function will return a Result with the [Schema] that represents + /// the ETW event that triggered the callback + /// + /// This function can fail, if it does it will return a [SchemaError] + /// + /// # Example + /// ```rust + /// let my_callback = |record: EventRecord, schema_locator: &mut SchemaLocator| { + /// let schema = schema_locator.event_schema(record)?; + /// }; + /// ``` + pub fn event_schema<'a>(&mut self, event: &'a EventRecord) -> SchemaResult> { + let key = SchemaKey::new(&event, self); + let info = match self.schemas.entry(key) { + Entry::Occupied(entry) => entry.into_mut(), + Entry::Vacant(entry) => { + let info = Box::new(tdh::schema_from_tdh(event.clone())?); + // dbg!(info.provider_guid(), info.provider_name(), info.decoding_source()); + // TODO: Cloning for now, should be a reference at some point... + entry.insert(Arc::new(Schema::new(info))) + } + } + .clone(); + + // Some events contain schemas so add them when we find them. + if info.event_schema.is_event_metadata() { + let event_info = TraceEventInfoRaw::new(event.user_buffer().to_owned()); + println!( + "Adding custom schema for {}/{}/{}/{}", + event_info.provider_name(), + event_info.event_id(), + event_info.task_name(), + event_info.opcode_name() + ); + self.add_custom_schema(Box::new(event_info)); + } + + Ok(TypedEvent::new(event, info)) + } +} + +pub struct Schema { + pub event_schema: Box, + properties: OnceCell, + name: OnceCell, +} + +impl Schema { + fn new(event_schema: Box) -> Self { + Schema { + event_schema, + properties: OnceCell::new(), + name: OnceCell::new(), + } + } + pub(crate) fn properties(&self) -> &PropertyIter { + self.properties.get_or_init(|| PropertyIter::new(self)) + } + pub(crate) fn name(&self) -> &str { + self.name.get_or_init(|| { + format!( + "{}/{}/{}", + self.event_schema.provider_name(), + self.event_schema.task_name(), + self.event_schema.opcode_name() + ) + }) + } +} + +pub struct TypedEvent<'a> { + record: &'a EventRecord, + pub(crate) schema: Arc, +} + +impl<'a> TypedEvent<'a> { + pub fn new(record: &'a EventRecord, schema: Arc) -> Self { + TypedEvent { record, schema } + } + + pub(crate) fn user_buffer(&self) -> &[u8] { + self.record.user_buffer() + } + + // Horrible getters FTW!! :D + // TODO: Not a big fan of this, think a better way.. + pub(crate) fn record(&self) -> &EventRecord { + self.record + } + + /// Use the `event_id` function to obtain the EventId of the Event Record + /// + /// This getter returns the EventId of the ETW Event that triggered the registered callback + /// + /// # Example + /// ```rust + /// let my_callback = |record: EventRecord, schema_locator: &mut SchemaLocator| { + /// let schema = schema_locator.event_schema(record)?; + /// let event_id = schema.event_id(); + /// }; + /// ``` + pub fn event_id(&self) -> u16 { + self.record.EventHeader.EventDescriptor.Id + } + + /// Use the `opcode` function to obtain the Opcode of the Event Record + /// + /// This getter returns the opcode of the ETW Event that triggered the registered callback + /// + /// # Example + /// ```rust + /// let my_callback = |record: EventRecord, schema_locator: &mut SchemaLocator| { + /// let schema = schema_locator.event_schema(record)?; + /// let event_id = schema.opcode(); + /// }; + /// ``` + pub fn opcode(&self) -> u8 { + self.record.EventHeader.EventDescriptor.Opcode + } + + /// Use the `event_flags` function to obtain the Event Flags of the [EventRecord] + /// + /// This getter returns the Event Flags of the ETW Event that triggered the registered callback + /// + /// # Example + /// ```rust + /// let my_callback = |record: EventRecord, schema_locator: &mut SchemaLocator| { + /// let schema = schema_locator.event_schema(record)?; + /// let event_flags = schema.event_flags(); + /// }; + /// ``` + pub fn event_flags(&self) -> u16 { + self.record.EventHeader.Flags + } + + pub fn is_64bit(&self) -> bool { + (self.record.EventHeader.Flags & EVENT_HEADER_FLAG_64_BIT_HEADER as u16) != 0 + } + + /// Use the `event_version` function to obtain the Version of the [EventRecord] + /// + /// This getter returns the Version of the ETW Event that triggered the registered callback + /// + /// # Example + /// ```rust + /// let my_callback = |record: EventRecord, schema_locator: &mut SchemaLocator| { + /// let schema = schema_locator.event_schema(record)?; + /// let event_version = schema.event_version(); + /// }; + /// ``` + pub fn event_version(&self) -> u8 { + self.record.EventHeader.EventDescriptor.Version + } + + /// Use the `process_id` function to obtain the ProcessId of the [EventRecord] + /// + /// This getter returns the ProcessId of the process that triggered the ETW Event + /// + /// # Example + /// ```rust + /// let my_callback = |record: EventRecord, schema_locator: &mut SchemaLocator| { + /// let schema = schema_locator.event_schema(record)?; + /// let pid = schema.process_id(); + /// }; + /// ``` + pub fn process_id(&self) -> u32 { + self.record.EventHeader.ProcessId + } + + /// Use the `thread_id` function to obtain the ThreadId of the [EventRecord] + /// + /// This getter returns the ThreadId of the thread that triggered the ETW Event + /// + /// # Example + /// ```rust + /// let my_callback = |record: EventRecord, schema_locator: &mut SchemaLocator| { + /// let schema = schema_locator.event_schema(record)?; + /// let tid = schema.thread_id(); + /// }; + /// ``` + pub fn thread_id(&self) -> u32 { + self.record.EventHeader.ThreadId + } + + /// Use the `timestamp` function to obtain the TimeStamp of the [EventRecord] + /// + /// This getter returns the TimeStamp of the ETW Event + /// + /// # Example + /// ```rust + /// let my_callback = |record: EventRecord, schema_locator: &mut SchemaLocator| { + /// let schema = schema_locator.event_schema(record)?; + /// let timestamp = schema.timestamp(); + /// }; + /// ``` + pub fn timestamp(&self) -> i64 { + self.record.EventHeader.TimeStamp + } + + /// Use the `activity_id` function to obtain the ActivityId of the [EventRecord] + /// + /// This getter returns the ActivityId from the ETW Event, this value is used to related Two events + /// + /// # Example + /// ```rust + /// let my_callback = |record: EventRecord, schema_locator: &mut SchemaLocator| { + /// let schema = schema_locator.event_schema(record)?; + /// let activity_id = schema.activity_id(); + /// }; + /// ``` + /// [TraceEventInfo]: super::native::etw_types::TraceEventInfo + pub fn activity_id(&self) -> GUID { + self.record.EventHeader.ActivityId + } + + /// Use the `decoding_source` function to obtain the [DecodingSource] from the [TraceEventInfo] + /// + /// This getter returns the DecodingSource from the event, this value identifies the source used + /// parse the event data + /// + /// # Example + /// ```rust + /// let my_callback = |record: EventRecord, schema_locator: &mut SchemaLocator| { + /// let schema = schema_locator.event_schema(record)?; + /// let decoding_source = schema.decoding_source(); + /// }; + /// ``` + /// [TraceEventInfo]: super::native::etw_types::TraceEventInfo + pub fn decoding_source(&self) -> DecodingSource { + self.schema.event_schema.decoding_source() + } + + /// Use the `provider_name` function to obtain the Provider name from the [TraceEventInfo] + /// + /// # Example + /// ```rust + /// let my_callback = |record: EventRecord, schema_locator: &mut SchemaLocator| { + /// let schema = schema_locator.event_schema(record)?; + /// let provider_name = schema.provider_name(); + /// }; + /// ``` + /// [TraceEventInfo]: super::native::etw_types::TraceEventInfo + pub fn provider_name(&self) -> String { + self.schema.event_schema.provider_name() + } + + /// Use the `task_name` function to obtain the Task name from the [TraceEventInfo] + /// + /// See: [TaskType](https://docs.microsoft.com/en-us/windows/win32/wes/eventmanifestschema-tasktype-complextype) + /// # Example + /// ```rust + /// let my_callback = |record: EventRecord, schema_locator: &mut SchemaLocator| { + /// let schema = schema_locator.event_schema(record)?; + /// let task_name = schema.task_name(); + /// }; + /// ``` + /// [TraceEventInfo]: super::native::etw_types::TraceEventInfo + pub fn task_name(&self) -> String { + self.schema.event_schema.task_name() + } + + /// Use the `opcode_name` function to obtain the Opcode name from the [TraceEventInfo] + /// + /// See: [OpcodeType](https://docs.microsoft.com/en-us/windows/win32/wes/eventmanifestschema-opcodetype-complextype) + /// # Example + /// ```rust + /// let my_callback = |record: EventRecord, schema_locator: &mut SchemaLocator| { + /// let schema = schema_locator.event_schema(record)?; + /// let opcode_name = schema.opcode_name(); + /// }; + /// ``` + /// [TraceEventInfo]: super::native::etw_types::TraceEventInfo + pub fn opcode_name(&self) -> String { + self.schema.event_schema.opcode_name() + } + + pub fn property_count(&self) -> u32 { + self.schema.event_schema.property_count() + } + + pub fn property(&self, index: u32) -> Property { + self.schema.event_schema.property(index) + } + + pub fn name(&self) -> &str { + self.schema.name() + } + + pub fn event_message(&self) -> Option { + self.schema.event_schema.event_message() + } +} + +impl<'a> PartialEq for TypedEvent<'a> { + fn eq(&self, other: &Self) -> bool { + self.schema.event_schema.event_id() == other.schema.event_schema.event_id() + && self.schema.event_schema.provider_guid() == other.schema.event_schema.provider_guid() + && self.schema.event_schema.event_version() == other.schema.event_schema.event_version() + } +} + +impl<'a> Eq for TypedEvent<'a> {} diff --git a/etw-reader/src/sddl.rs b/etw-reader/src/sddl.rs index 2526139b..cd2a7a2a 100644 --- a/etw-reader/src/sddl.rs +++ b/etw-reader/src/sddl.rs @@ -1,62 +1,66 @@ -use windows::{Win32::{ - Security, - Foundation::{PSID, HLOCAL, LocalFree}, -}, core::PSTR}; -//use crate::traits::*; -use std::str::Utf8Error; - -/// SDDL native error -#[derive(Debug)] -pub enum SddlNativeError { - /// Represents an error parsing the SID into a String - SidParseError(Utf8Error), - /// Represents an standard IO Error - IoError(std::io::Error), -} - -//impl LastOsError for SddlNativeError {} - -impl From for SddlNativeError { - fn from(err: std::io::Error) -> Self { - SddlNativeError::IoError(err) - } -} - -impl From for SddlNativeError { - fn from(err: Utf8Error) -> Self { - SddlNativeError::SidParseError(err) - } -} - -pub(crate) type SddlResult = Result; - -pub fn convert_sid_to_string(sid: *const u8) -> SddlResult { - - let mut tmp = PSTR::null(); - unsafe { - if !Security::Authorization::ConvertSidToStringSidA(PSID(sid as *const _ as *mut _), &mut tmp).is_ok() { - return Err(SddlNativeError::IoError(std::io::Error::last_os_error())); - } - - let sid_string = std::ffi::CStr::from_ptr(tmp.0 as *mut _) - .to_str()? - .to_owned(); - - let _ = LocalFree(HLOCAL(tmp.0 as *mut _)); - - Ok(sid_string) - } -} - -#[cfg(test)] -mod test { - use super::*; - - #[test] - fn test_convert_string_to_sid() { - let sid: Vec = vec![1, 2, 0, 0, 0, 0, 0, 5, 0x20, 0, 0, 0, 0x20, 2, 0, 0]; - if let Ok(string_sid) = convert_sid_to_string(sid.as_ptr()) { - assert_eq!(string_sid, "S-1-5-32-544"); - } - } -} +//use super::traits::*; +use std::str::Utf8Error; + +use windows::core::PSTR; +use windows::Win32::Foundation::{LocalFree, HLOCAL, PSID}; +use windows::Win32::Security; + +/// SDDL native error +#[derive(Debug)] +pub enum SddlNativeError { + /// Represents an error parsing the SID into a String + SidParseError(Utf8Error), + /// Represents an standard IO Error + IoError(std::io::Error), +} + +//impl LastOsError for SddlNativeError {} + +impl From for SddlNativeError { + fn from(err: std::io::Error) -> Self { + SddlNativeError::IoError(err) + } +} + +impl From for SddlNativeError { + fn from(err: Utf8Error) -> Self { + SddlNativeError::SidParseError(err) + } +} + +pub(crate) type SddlResult = Result; + +pub fn convert_sid_to_string(sid: *const u8) -> SddlResult { + let mut tmp = PSTR::null(); + unsafe { + if !Security::Authorization::ConvertSidToStringSidA( + PSID(sid as *const _ as *mut _), + &mut tmp, + ) + .is_ok() + { + return Err(SddlNativeError::IoError(std::io::Error::last_os_error())); + } + + let sid_string = std::ffi::CStr::from_ptr(tmp.0 as *mut _) + .to_str()? + .to_owned(); + + let _ = LocalFree(HLOCAL(tmp.0 as *mut _)); + + Ok(sid_string) + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_convert_string_to_sid() { + let sid: Vec = vec![1, 2, 0, 0, 0, 0, 0, 5, 0x20, 0, 0, 0, 0x20, 2, 0, 0]; + if let Ok(string_sid) = convert_sid_to_string(sid.as_ptr()) { + assert_eq!(string_sid, "S-1-5-32-544"); + } + } +} diff --git a/etw-reader/src/tdh.rs b/etw-reader/src/tdh.rs index 6f782785..f2f6af7b 100644 --- a/etw-reader/src/tdh.rs +++ b/etw-reader/src/tdh.rs @@ -1,132 +1,137 @@ -use std::ffi::OsString; -use std::ops::Deref; -use std::os::windows::ffi::OsStringExt; -use std::ptr; - -use windows::core::HRESULT; -use windows::Win32::Foundation::ERROR_SUCCESS; -use windows::Win32::Foundation::S_OK; -use windows::Win32::System::Diagnostics::Etw; -use windows::Win32::Foundation::ERROR_INSUFFICIENT_BUFFER; -use windows::Win32::System::Diagnostics::Etw::TdhEnumerateProviders; -use windows::Win32::System::Diagnostics::Etw::PROVIDER_ENUMERATION_INFO; -use crate::etw_types::*; - -use crate::traits::*; - -#[derive(Debug)] -pub enum TdhNativeError { - /// Represents an standard IO Error - IoError(std::io::Error), -} - - -impl From for TdhNativeError { - fn from(err: std::io::Error) -> Self { - TdhNativeError::IoError(err) - } -} - -pub(crate) type TdhNativeResult = Result; - -pub fn schema_from_tdh(event: &Etw::EVENT_RECORD) -> TdhNativeResult { - let mut buffer_size = 0; - unsafe { - if Etw::TdhGetEventInformation( - event, - None, - None, - &mut buffer_size, - ) != ERROR_INSUFFICIENT_BUFFER.0 - { - return Err(TdhNativeError::IoError(std::io::Error::last_os_error())); - } - - let mut buffer = TraceEventInfoRaw::alloc(buffer_size); - if Etw::TdhGetEventInformation( - event, - None, - Some(buffer.info_as_ptr() as *mut _), - &mut buffer_size, - ) != 0 - { - return Err(TdhNativeError::IoError(std::io::Error::last_os_error())); - } - - Ok(buffer) - } -} - -pub(crate) fn property_size(event: &EventRecord, name: &str) -> TdhNativeResult { - let mut property_size = 0; - - let mut desc = Etw::PROPERTY_DATA_DESCRIPTOR::default(); - desc.ArrayIndex = u32::MAX; - let utf16_name = name.as_utf16(); - desc.PropertyName = utf16_name.as_ptr() as u64; - - unsafe { - let status = Etw::TdhGetPropertySize( - event.deref(), - None, - &[desc], - &mut property_size, - ); - if status != 0 { - return Err(TdhNativeError::IoError(std::io::Error::from_raw_os_error( - status as i32, - ))); - } - } - - Ok(property_size) -} - - -pub fn list_etw_providers() { - let mut buffer_size: u32 = 0; - let mut status: u32; - - // Query required buffer size - unsafe { - status = TdhEnumerateProviders(None, &mut buffer_size); - } - if status == ERROR_INSUFFICIENT_BUFFER.0 { - let mut provider_info = vec![0u8; buffer_size as usize]; - let mut buffer_size_copied = buffer_size; - - // Retrieve provider information - unsafe { - status = TdhEnumerateProviders(Some(provider_info.as_mut_ptr() as *mut PROVIDER_ENUMERATION_INFO), &mut buffer_size_copied); - } - if status == ERROR_SUCCESS.0 { - let provider_info = unsafe { &*(provider_info.as_ptr() as *const PROVIDER_ENUMERATION_INFO) }; - let provider_info_array = provider_info.TraceProviderInfoArray.as_ptr(); - - for i in 0..provider_info.NumberOfProviders { - // windows-rs defines TraceProviderInfoArray as a fixed size array of 1 so we need to use get_unchecked to get the other things - let provider_name_offset = unsafe { *provider_info_array.offset(i as isize) }.ProviderNameOffset as usize; - let provider_name_ptr = provider_info as *const PROVIDER_ENUMERATION_INFO as usize + provider_name_offset; - // Find the length of the null-terminated string - let mut len = 0; - while unsafe { *(provider_name_ptr as *const u16).add(len) } != 0 { - len += 1; - } - let provider_name = unsafe { OsString::from_wide(std::slice::from_raw_parts(provider_name_ptr as *const u16, len)) - .into_string().unwrap_or_else(|_| "Error converting to string".to_string()) }; - - let provider_guid = &unsafe { *provider_info_array.offset(i as isize) }.ProviderGuid; - let schema_source = unsafe { *provider_info_array.offset(i as isize) }.SchemaSource; - - - println!(" {:?} - {} - {}", provider_guid, provider_name, if schema_source == 0 { "XML manifest" } else { "MOF" }); - } - } else { - println!("TdhEnumerateProviders failed with error code {:?}", status); - } - } else { - println!("TdhEnumerateProviders failed with error code {:?}", status); - } -} - - +use std::ffi::OsString; +use std::ops::Deref; +use std::os::windows::ffi::OsStringExt; +use std::ptr; + +use windows::core::HRESULT; +use windows::Win32::Foundation::{ERROR_INSUFFICIENT_BUFFER, ERROR_SUCCESS, S_OK}; +use windows::Win32::System::Diagnostics::Etw; +use windows::Win32::System::Diagnostics::Etw::{TdhEnumerateProviders, PROVIDER_ENUMERATION_INFO}; + +use super::etw_types::*; +use super::traits::*; + +#[derive(Debug)] +pub enum TdhNativeError { + /// Represents an standard IO Error + IoError(std::io::Error), +} + +impl From for TdhNativeError { + fn from(err: std::io::Error) -> Self { + TdhNativeError::IoError(err) + } +} + +pub(crate) type TdhNativeResult = Result; + +pub fn schema_from_tdh(event: &Etw::EVENT_RECORD) -> TdhNativeResult { + let mut buffer_size = 0; + unsafe { + if Etw::TdhGetEventInformation(event, None, None, &mut buffer_size) + != ERROR_INSUFFICIENT_BUFFER.0 + { + return Err(TdhNativeError::IoError(std::io::Error::last_os_error())); + } + + let mut buffer = TraceEventInfoRaw::alloc(buffer_size); + if Etw::TdhGetEventInformation( + event, + None, + Some(buffer.info_as_ptr() as *mut _), + &mut buffer_size, + ) != 0 + { + return Err(TdhNativeError::IoError(std::io::Error::last_os_error())); + } + + Ok(buffer) + } +} + +pub(crate) fn property_size(event: &EventRecord, name: &str) -> TdhNativeResult { + let mut property_size = 0; + + let mut desc = Etw::PROPERTY_DATA_DESCRIPTOR::default(); + desc.ArrayIndex = u32::MAX; + let utf16_name = name.as_utf16(); + desc.PropertyName = utf16_name.as_ptr() as u64; + + unsafe { + let status = Etw::TdhGetPropertySize(event.deref(), None, &[desc], &mut property_size); + if status != 0 { + return Err(TdhNativeError::IoError(std::io::Error::from_raw_os_error( + status as i32, + ))); + } + } + + Ok(property_size) +} + +pub fn list_etw_providers() { + let mut buffer_size: u32 = 0; + let mut status: u32; + + // Query required buffer size + unsafe { + status = TdhEnumerateProviders(None, &mut buffer_size); + } + if status == ERROR_INSUFFICIENT_BUFFER.0 { + let mut provider_info = vec![0u8; buffer_size as usize]; + let mut buffer_size_copied = buffer_size; + + // Retrieve provider information + unsafe { + status = TdhEnumerateProviders( + Some(provider_info.as_mut_ptr() as *mut PROVIDER_ENUMERATION_INFO), + &mut buffer_size_copied, + ); + } + if status == ERROR_SUCCESS.0 { + let provider_info = + unsafe { &*(provider_info.as_ptr() as *const PROVIDER_ENUMERATION_INFO) }; + let provider_info_array = provider_info.TraceProviderInfoArray.as_ptr(); + + for i in 0..provider_info.NumberOfProviders { + // windows-rs defines TraceProviderInfoArray as a fixed size array of 1 so we need to use get_unchecked to get the other things + let provider_name_offset = + unsafe { *provider_info_array.offset(i as isize) }.ProviderNameOffset as usize; + let provider_name_ptr = provider_info as *const PROVIDER_ENUMERATION_INFO as usize + + provider_name_offset; + // Find the length of the null-terminated string + let mut len = 0; + while unsafe { *(provider_name_ptr as *const u16).add(len) } != 0 { + len += 1; + } + let provider_name = unsafe { + OsString::from_wide(std::slice::from_raw_parts( + provider_name_ptr as *const u16, + len, + )) + .into_string() + .unwrap_or_else(|_| "Error converting to string".to_string()) + }; + + let provider_guid = + &unsafe { *provider_info_array.offset(i as isize) }.ProviderGuid; + let schema_source = unsafe { *provider_info_array.offset(i as isize) }.SchemaSource; + + println!( + " {:?} - {} - {}", + provider_guid, + provider_name, + if schema_source == 0 { + "XML manifest" + } else { + "MOF" + } + ); + } + } else { + println!("TdhEnumerateProviders failed with error code {:?}", status); + } + } else { + println!("TdhEnumerateProviders failed with error code {:?}", status); + } +} diff --git a/etw-reader/src/tdh_types.rs b/etw-reader/src/tdh_types.rs index bf613041..aa78063f 100644 --- a/etw-reader/src/tdh_types.rs +++ b/etw-reader/src/tdh_types.rs @@ -1,218 +1,224 @@ -//! Basic TDH types -//! -//! The `tdh_type` module provides an abstraction over the basic TDH types, this module act as a -//! helper for the parser to determine which IN and OUT type are expected from a property within an -//! event -//! -//! This is a bit extra but is basically a redefinition of the In an Out TDH types following the -//! rust naming convention, it can also come in handy when implementing the [TryParse] trait for a type -//! to determine how to handle a [Property] based on this values -//! -//! [TryParse]: crate::parser::TryParse -//! [Property]: crate::native::tdh_types::Property -use std::rc::Rc; - -use windows::Win32::System::Diagnostics::Etw; - - -use crate::etw_types::EventPropertyInfo; -use num_traits::FromPrimitive; - -#[derive(Debug, Clone, Default)] -pub struct PropertyMapInfo { - pub is_bitmap: bool, - pub map: crate::FastHashMap -} -#[derive(Debug, Clone)] -pub struct PrimitiveDesc { - pub in_type: TdhInType, - pub out_type: TdhOutType, -} - -#[derive(Debug, Clone, Default)] -pub struct StructDesc { - pub start_index: u16, - pub num_members: u16, -} - -#[derive(Debug, Clone)] -pub enum PropertyDesc { - Primitive(PrimitiveDesc), - Struct(StructDesc), -} - -/// Notes if the property length is a concrete length or an index to another property -/// which contains the length. -#[derive(Debug, Clone, Copy, PartialEq)] -pub enum PropertyLength { - Length(u16), - Index(u16), -} - -/// Attributes of a property -#[derive(Debug, Clone)] -pub struct Property { - /// Name of the Property - pub name: String, - /// Represent the [PropertyFlags] - pub flags: PropertyFlags, - pub length: PropertyLength, - pub desc: PropertyDesc, - pub map_info: Option>, - pub count: u16, -} - -#[doc(hidden)] -impl Property { - pub fn new(name: String, property: &EventPropertyInfo, map_info: Option>) -> Self { - let flags = PropertyFlags::from(property.Flags); - let length = if flags.contains(PropertyFlags::PROPERTY_PARAM_LENGTH) { - // The property length is stored in another property, this is the index of that property - PropertyLength::Index(unsafe { property.Anonymous3.lengthPropertyIndex }) - } else { - // The property has no param for its length, it makes sense to access this field of the union - PropertyLength::Length(unsafe { property.Anonymous3.length }) - }; - if property.Flags.0 & Etw::PropertyStruct.0 != 0 { - unsafe { - let start_index = property.Anonymous1.structType.StructStartIndex; - let num_members = property.Anonymous1.structType.NumOfStructMembers; - Property { - name, - flags: PropertyFlags::from(property.Flags), - length, - desc: PropertyDesc::Struct(StructDesc{start_index, num_members}), - map_info, - count: property.Anonymous2.count, - } - } - } else { - unsafe { - let out_type = FromPrimitive::from_u16(property.Anonymous1.nonStructType.OutType) - .unwrap_or(TdhOutType::OutTypeNull); - let in_type = FromPrimitive::from_u16(property.Anonymous1.nonStructType.InType) - .unwrap_or_else(|| panic!("{:?}", property.Anonymous1.nonStructType.InType)); - - Property { - name, - flags: PropertyFlags::from(property.Flags), - length, - desc: PropertyDesc::Primitive(PrimitiveDesc{in_type, out_type}), - map_info, - count: property.Anonymous2.count, - } - } - } - } -} - -/// Represent a TDH_IN_TYPE -#[repr(u16)] -#[derive(Debug, Clone, Copy, FromPrimitive, ToPrimitive, PartialEq)] -pub enum TdhInType { - // Deprecated values are not defined - InTypeNull, - InTypeUnicodeString, - InTypeAnsiString, - InTypeInt8, // Field size is 1 byte - InTypeUInt8, // Field size is 1 byte - InTypeInt16, // Field size is 2 bytes - InTypeUInt16, // Field size is 2 bytes - InTypeInt32, // Field size is 4 bytes - InTypeUInt32, // Field size is 4 bytes - InTypeInt64, // Field size is 8 bytes - InTypeUInt64, // Field size is 8 bytes - InTypeFloat, // Field size is 4 bytes - InTypeDouble, // Field size is 8 bytes - InTypeBoolean, // Field size is 4 bytes - InTypeBinary, // Depends on the OutType - InTypeGuid, - InTypePointer, - InTypeFileTime, // Field size is 8 bytes - InTypeSystemTime, // Field size is 16 bytes - InTypeSid, // Field size determined by the first few bytes of the field - InTypeHexInt32, - InTypeHexInt64, - InTypeCountedString = 300, - InTypeCountedAnsiString, - InTypeReverseCountedString, - InTypeReverseCountedAnsiString, - InTypeNonNullTerminatedString, - InTypeNonNullTerminatedAnsiString, - InTypeUnicodeChar, - InTypeAnsiChar, - InTypeSizeT, - InTypeHexdump, - InTypeWBEMSID, -} - - - -/// Represent a TDH_OUT_TYPE -#[repr(u16)] -#[derive(Debug, Clone, Copy, FromPrimitive, ToPrimitive, PartialEq)] -pub enum TdhOutType { - OutTypeNull, - OutTypeString, - OutTypeDateTime, - OutTypeInt8, // Field size is 1 byte - OutTypeUInt8, // Field size is 1 byte - OutTypeInt16, // Field size is 2 bytes - OutTypeUInt16, // Field size is 2 bytes - OutTypeInt32, // Field size is 4 bytes - OutTypeUInt32, // Field size is 4 bytes - OutTypeInt64, // Field size is 8 bytes - OutTypeUInt64, // Field size is 8 bytes - OutTypeFloat, // Field size is 4 bytes - OutTypeDouble, // Field size is 8 bytes - OutTypeBoolean, // Field size is 4 bytes - OutTypeGuid, - OutTypeHexBinary, - OutTypeHexInt8, - OutTypeHexInt16, - OutTypeHexInt32, - OutTypeHexInt64, - OutTypePid, - OutTypeTid, - OutTypePort, - OutTypeIpv4, - OutTypeIpv6, - OutTypeWin32Error = 30, - OutTypeNtStatus = 31, - OutTypeHResult = 32, - OutTypeJson = 34, - OutTypeUtf8 = 35, - OutTypePkcs7 = 36, - OutTypeCodePointer = 37, - OutTypeDatetimeUtc = 38, -} - -impl Default for TdhOutType { - fn default() -> TdhOutType { - TdhOutType::OutTypeNull - } -} - -bitflags! { - /// Represents the Property flags - /// - /// See: [Property Flags enum](https://docs.microsoft.com/en-us/windows/win32/api/tdh/ne-tdh-property_flags) - #[derive(Default)] - pub struct PropertyFlags: u32 { - const PROPERTY_STRUCT = 0x1; - const PROPERTY_PARAM_LENGTH = 0x2; - const PROPERTY_PARAM_COUNT = 0x4; - const PROPERTY_WBEMXML_FRAGMENT = 0x8; - const PROPERTY_PARAM_FIXED_LENGTH = 0x10; - const PROPERTY_PARAM_FIXED_COUNT = 0x20; - const PROPERTY_HAS_TAGS = 0x40; - const PROPERTY_HAS_CUSTOM_SCHEMA = 0x80; - } -} - -impl From for PropertyFlags { - fn from(flags: Etw::PROPERTY_FLAGS) -> Self { - // Should be a safe cast - PropertyFlags::from_bits_truncate(flags.0 as u32) - } -} +//! Basic TDH types +//! +//! The `tdh_type` module provides an abstraction over the basic TDH types, this module act as a +//! helper for the parser to determine which IN and OUT type are expected from a property within an +//! event +//! +//! This is a bit extra but is basically a redefinition of the In an Out TDH types following the +//! rust naming convention, it can also come in handy when implementing the [TryParse] trait for a type +//! to determine how to handle a [Property] based on this values +//! +//! [TryParse]: super::parser::TryParse +//! [Property]: super::native::tdh_types::Property +use std::rc::Rc; + +use bitflags::bitflags; +use num_derive::{FromPrimitive, ToPrimitive}; +use num_traits::FromPrimitive; +use windows::Win32::System::Diagnostics::Etw; + +use super::etw_types::EventPropertyInfo; + +#[derive(Debug, Clone, Default)] +pub struct PropertyMapInfo { + pub is_bitmap: bool, + pub map: super::FastHashMap, +} +#[derive(Debug, Clone)] +pub struct PrimitiveDesc { + pub in_type: TdhInType, + pub out_type: TdhOutType, +} + +#[derive(Debug, Clone, Default)] +pub struct StructDesc { + pub start_index: u16, + pub num_members: u16, +} + +#[derive(Debug, Clone)] +pub enum PropertyDesc { + Primitive(PrimitiveDesc), + Struct(StructDesc), +} + +/// Notes if the property length is a concrete length or an index to another property +/// which contains the length. +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum PropertyLength { + Length(u16), + Index(u16), +} + +/// Attributes of a property +#[derive(Debug, Clone)] +pub struct Property { + /// Name of the Property + pub name: String, + /// Represent the [PropertyFlags] + pub flags: PropertyFlags, + pub length: PropertyLength, + pub desc: PropertyDesc, + pub map_info: Option>, + pub count: u16, +} + +#[doc(hidden)] +impl Property { + pub fn new( + name: String, + property: &EventPropertyInfo, + map_info: Option>, + ) -> Self { + let flags = PropertyFlags::from(property.Flags); + let length = if flags.contains(PropertyFlags::PROPERTY_PARAM_LENGTH) { + // The property length is stored in another property, this is the index of that property + PropertyLength::Index(unsafe { property.Anonymous3.lengthPropertyIndex }) + } else { + // The property has no param for its length, it makes sense to access this field of the union + PropertyLength::Length(unsafe { property.Anonymous3.length }) + }; + if property.Flags.0 & Etw::PropertyStruct.0 != 0 { + unsafe { + let start_index = property.Anonymous1.structType.StructStartIndex; + let num_members = property.Anonymous1.structType.NumOfStructMembers; + Property { + name, + flags: PropertyFlags::from(property.Flags), + length, + desc: PropertyDesc::Struct(StructDesc { + start_index, + num_members, + }), + map_info, + count: property.Anonymous2.count, + } + } + } else { + unsafe { + let out_type = FromPrimitive::from_u16(property.Anonymous1.nonStructType.OutType) + .unwrap_or(TdhOutType::OutTypeNull); + let in_type = FromPrimitive::from_u16(property.Anonymous1.nonStructType.InType) + .unwrap_or_else(|| panic!("{:?}", property.Anonymous1.nonStructType.InType)); + + Property { + name, + flags: PropertyFlags::from(property.Flags), + length, + desc: PropertyDesc::Primitive(PrimitiveDesc { in_type, out_type }), + map_info, + count: property.Anonymous2.count, + } + } + } + } +} + +/// Represent a TDH_IN_TYPE +#[repr(u16)] +#[derive(Debug, Clone, Copy, FromPrimitive, ToPrimitive, PartialEq)] +pub enum TdhInType { + // Deprecated values are not defined + InTypeNull, + InTypeUnicodeString, + InTypeAnsiString, + InTypeInt8, // Field size is 1 byte + InTypeUInt8, // Field size is 1 byte + InTypeInt16, // Field size is 2 bytes + InTypeUInt16, // Field size is 2 bytes + InTypeInt32, // Field size is 4 bytes + InTypeUInt32, // Field size is 4 bytes + InTypeInt64, // Field size is 8 bytes + InTypeUInt64, // Field size is 8 bytes + InTypeFloat, // Field size is 4 bytes + InTypeDouble, // Field size is 8 bytes + InTypeBoolean, // Field size is 4 bytes + InTypeBinary, // Depends on the OutType + InTypeGuid, + InTypePointer, + InTypeFileTime, // Field size is 8 bytes + InTypeSystemTime, // Field size is 16 bytes + InTypeSid, // Field size determined by the first few bytes of the field + InTypeHexInt32, + InTypeHexInt64, + InTypeCountedString = 300, + InTypeCountedAnsiString, + InTypeReverseCountedString, + InTypeReverseCountedAnsiString, + InTypeNonNullTerminatedString, + InTypeNonNullTerminatedAnsiString, + InTypeUnicodeChar, + InTypeAnsiChar, + InTypeSizeT, + InTypeHexdump, + InTypeWBEMSID, +} + +/// Represent a TDH_OUT_TYPE +#[repr(u16)] +#[derive(Debug, Clone, Copy, FromPrimitive, ToPrimitive, PartialEq)] +pub enum TdhOutType { + OutTypeNull, + OutTypeString, + OutTypeDateTime, + OutTypeInt8, // Field size is 1 byte + OutTypeUInt8, // Field size is 1 byte + OutTypeInt16, // Field size is 2 bytes + OutTypeUInt16, // Field size is 2 bytes + OutTypeInt32, // Field size is 4 bytes + OutTypeUInt32, // Field size is 4 bytes + OutTypeInt64, // Field size is 8 bytes + OutTypeUInt64, // Field size is 8 bytes + OutTypeFloat, // Field size is 4 bytes + OutTypeDouble, // Field size is 8 bytes + OutTypeBoolean, // Field size is 4 bytes + OutTypeGuid, + OutTypeHexBinary, + OutTypeHexInt8, + OutTypeHexInt16, + OutTypeHexInt32, + OutTypeHexInt64, + OutTypePid, + OutTypeTid, + OutTypePort, + OutTypeIpv4, + OutTypeIpv6, + OutTypeWin32Error = 30, + OutTypeNtStatus = 31, + OutTypeHResult = 32, + OutTypeJson = 34, + OutTypeUtf8 = 35, + OutTypePkcs7 = 36, + OutTypeCodePointer = 37, + OutTypeDatetimeUtc = 38, +} + +impl Default for TdhOutType { + fn default() -> TdhOutType { + TdhOutType::OutTypeNull + } +} + +bitflags! { + /// Represents the Property flags + /// + /// See: [Property Flags enum](https://docs.microsoft.com/en-us/windows/win32/api/tdh/ne-tdh-property_flags) + #[derive(Default, Debug, Clone)] + pub struct PropertyFlags: u32 { + const PROPERTY_STRUCT = 0x1; + const PROPERTY_PARAM_LENGTH = 0x2; + const PROPERTY_PARAM_COUNT = 0x4; + const PROPERTY_WBEMXML_FRAGMENT = 0x8; + const PROPERTY_PARAM_FIXED_LENGTH = 0x10; + const PROPERTY_PARAM_FIXED_COUNT = 0x20; + const PROPERTY_HAS_TAGS = 0x40; + const PROPERTY_HAS_CUSTOM_SCHEMA = 0x80; + } +} + +impl From for PropertyFlags { + fn from(flags: Etw::PROPERTY_FLAGS) -> Self { + // Should be a safe cast + PropertyFlags::from_bits_truncate(flags.0 as u32) + } +} diff --git a/etw-reader/src/traits.rs b/etw-reader/src/traits.rs index 300c5269..9a240d6f 100644 --- a/etw-reader/src/traits.rs +++ b/etw-reader/src/traits.rs @@ -1,19 +1,19 @@ -use std::iter; - -pub trait EncodeUtf16 { - fn as_utf16(self: Self) -> Vec; -} - -impl EncodeUtf16 for &str { - fn as_utf16(self: Self) -> Vec { - self.encode_utf16() // Make a UTF-16 iterator - .chain(iter::once(0)) // Append a null - .collect() // Collect the iterator into a vector - } -} - -impl EncodeUtf16 for String { - fn as_utf16(self: Self) -> Vec { - self.as_str().as_utf16() - } -} +use std::iter; + +pub trait EncodeUtf16 { + fn as_utf16(self: Self) -> Vec; +} + +impl EncodeUtf16 for &str { + fn as_utf16(self: Self) -> Vec { + self.encode_utf16() // Make a UTF-16 iterator + .chain(iter::once(0)) // Append a null + .collect() // Collect the iterator into a vector + } +} + +impl EncodeUtf16 for String { + fn as_utf16(self: Self) -> Vec { + self.as_str().as_utf16() + } +} diff --git a/etw-reader/src/utils.rs b/etw-reader/src/utils.rs index 26a48bb2..8d96e8a9 100644 --- a/etw-reader/src/utils.rs +++ b/etw-reader/src/utils.rs @@ -1,91 +1,92 @@ - -fn is_aligned(ptr: *const T) -> bool -where - T: Sized, -{ - ptr as usize & (std::mem::align_of::() - 1) == 0 -} - -pub fn parse_unk_size_null_utf16_string(v: &[u8]) -> String { - // Instead of doing the following unsafe stuff ourselves we could - // use cast_slice from bytemuck. Unfortunately, it won't work if - // the length of the u8 is not a multiple of 2 vs. just truncating. - // Alternatively, using safe_transmute::transmute_many_permisive should work. - - let start: *const u16 = v.as_ptr().cast(); - if !is_aligned(start) { - panic!("Not aligned"); - } - - // safe because we not going past the end of the slice - let end: *const u16 = unsafe { v.as_ptr().offset(v.len() as isize) }.cast(); - - // find the null termination - let mut len = 0; - let mut ptr = start; - while unsafe { *ptr } != 0 && ptr < end { - len += 1; - ptr = unsafe { ptr.offset(1) }; - } - - let slice = unsafe { std::slice::from_raw_parts(start, len) }; - String::from_utf16_lossy(slice) -} - - -pub fn parse_unk_size_null_unicode_size(v: &[u8]) -> usize { - // TODO: Make sure is aligned - v.chunks_exact(2) - .into_iter() - .take_while(|&a| a != &[0, 0]) // Take until null terminator - .map(|a| u16::from_ne_bytes([a[0], a[1]])) - .count() * 2 + 2 -} - -pub fn parse_unk_size_null_unicode_vec(v: &[u8]) -> Vec { - // TODO: Make sure is aligned - v.chunks_exact(2) - .into_iter() - .take_while(|&a| a != &[0, 0]) // Take until null terminator - .map(|a| u16::from_ne_bytes([a[0], a[1]])) - .collect::>() -} - -pub fn parse_unk_size_null_ansi_size(v: &[u8]) -> usize { - v.into_iter() - .take_while(|&&a| a != 0) // Take until null terminator - .count() + 1 -} - -pub fn parse_unk_size_null_ansi_vec(v: &[u8]) -> Vec { - v.into_iter() - .take_while(|&&a| a != 0)// Take until null terminator - .map(|&a| a) - .collect::>() -} - -pub fn parse_null_utf16_string(v: &[u8]) -> String { - String::from_utf16_lossy( - v.chunks_exact(2) - .into_iter() - .map(|a| u16::from_ne_bytes([a[0], a[1]])) - .collect::>() - .as_slice(), - ) - .trim_matches(char::default()) - .to_string() -} - -pub fn parse_utf16_guid(v: &[u8]) -> String { - String::from_utf16_lossy( - v.chunks_exact(2) - .into_iter() - .map(|a| u16::from_ne_bytes([a[0], a[1]])) - .collect::>() - .as_slice(), - ) - .trim_matches(char::default()) - .trim_matches('{') - .trim_matches('}') - .to_string() -} +fn is_aligned(ptr: *const T) -> bool +where + T: Sized, +{ + ptr as usize & (std::mem::align_of::() - 1) == 0 +} + +pub fn parse_unk_size_null_utf16_string(v: &[u8]) -> String { + // Instead of doing the following unsafe stuff ourselves we could + // use cast_slice from bytemuck. Unfortunately, it won't work if + // the length of the u8 is not a multiple of 2 vs. just truncating. + // Alternatively, using safe_transmute::transmute_many_permisive should work. + + let start: *const u16 = v.as_ptr().cast(); + if !is_aligned(start) { + panic!("Not aligned"); + } + + // safe because we not going past the end of the slice + let end: *const u16 = unsafe { v.as_ptr().offset(v.len() as isize) }.cast(); + + // find the null termination + let mut len = 0; + let mut ptr = start; + while unsafe { *ptr } != 0 && ptr < end { + len += 1; + ptr = unsafe { ptr.offset(1) }; + } + + let slice = unsafe { std::slice::from_raw_parts(start, len) }; + String::from_utf16_lossy(slice) +} + +pub fn parse_unk_size_null_unicode_size(v: &[u8]) -> usize { + // TODO: Make sure is aligned + v.chunks_exact(2) + .into_iter() + .take_while(|&a| a != &[0, 0]) // Take until null terminator + .map(|a| u16::from_ne_bytes([a[0], a[1]])) + .count() + * 2 + + 2 +} + +pub fn parse_unk_size_null_unicode_vec(v: &[u8]) -> Vec { + // TODO: Make sure is aligned + v.chunks_exact(2) + .into_iter() + .take_while(|&a| a != &[0, 0]) // Take until null terminator + .map(|a| u16::from_ne_bytes([a[0], a[1]])) + .collect::>() +} + +pub fn parse_unk_size_null_ansi_size(v: &[u8]) -> usize { + v.into_iter() + .take_while(|&&a| a != 0) // Take until null terminator + .count() + + 1 +} + +pub fn parse_unk_size_null_ansi_vec(v: &[u8]) -> Vec { + v.into_iter() + .take_while(|&&a| a != 0) // Take until null terminator + .map(|&a| a) + .collect::>() +} + +pub fn parse_null_utf16_string(v: &[u8]) -> String { + String::from_utf16_lossy( + v.chunks_exact(2) + .into_iter() + .map(|a| u16::from_ne_bytes([a[0], a[1]])) + .collect::>() + .as_slice(), + ) + .trim_matches(char::default()) + .to_string() +} + +pub fn parse_utf16_guid(v: &[u8]) -> String { + String::from_utf16_lossy( + v.chunks_exact(2) + .into_iter() + .map(|a| u16::from_ne_bytes([a[0], a[1]])) + .collect::>() + .as_slice(), + ) + .trim_matches(char::default()) + .trim_matches('{') + .trim_matches('}') + .to_string() +} From 6fa4717e97647671652df7def1eb67c154b66e7b Mon Sep 17 00:00:00 2001 From: Vladimir Vukicevic Date: Sat, 4 May 2024 09:16:51 -0700 Subject: [PATCH 03/10] Fill in some unnamed ops --- etw-reader/src/etw_types.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/etw-reader/src/etw_types.rs b/etw-reader/src/etw_types.rs index 729deb96..8e1cf52c 100644 --- a/etw-reader/src/etw_types.rs +++ b/etw-reader/src/etw_types.rs @@ -264,7 +264,14 @@ impl EventSchema for TraceEventInfoRaw { fn opcode_name(&self) -> String { let opcode_name_offset = TraceEventInfo::from(self).OpcodeNameOffset as usize; if opcode_name_offset == 0 { - return String::from(""); + return String::from(match self.opcode() { + 0 => "Info", + 1 => "Start", + 2 => "Stop", + 3 => "DCStart", + 4 => "DCStop", + _ => "" + }); } utils::parse_unk_size_null_utf16_string(&self.info[opcode_name_offset..]) } From e8123fd0baee3c1962c0e678d77896b2f88b2982 Mon Sep 17 00:00:00 2001 From: Vladimir Vukicevic Date: Sat, 4 May 2024 12:50:45 -0700 Subject: [PATCH 04/10] Remove windows/etw_reader --- samply/src/windows/etw_reader/README.md | 2 - .../src/windows/etw_reader/custom_schemas.rs | 647 ------------------ samply/src/windows/etw_reader/etw_types.rs | 303 -------- samply/src/windows/etw_reader/mod.rs | 615 ----------------- samply/src/windows/etw_reader/parser.rs | 647 ------------------ samply/src/windows/etw_reader/property.rs | 58 -- samply/src/windows/etw_reader/schema.rs | 479 ------------- samply/src/windows/etw_reader/sddl.rs | 66 -- samply/src/windows/etw_reader/tdh.rs | 137 ---- samply/src/windows/etw_reader/tdh_types.rs | 224 ------ samply/src/windows/etw_reader/traits.rs | 19 - samply/src/windows/etw_reader/utils.rs | 92 --- 12 files changed, 3289 deletions(-) delete mode 100644 samply/src/windows/etw_reader/README.md delete mode 100644 samply/src/windows/etw_reader/custom_schemas.rs delete mode 100644 samply/src/windows/etw_reader/etw_types.rs delete mode 100644 samply/src/windows/etw_reader/mod.rs delete mode 100644 samply/src/windows/etw_reader/parser.rs delete mode 100644 samply/src/windows/etw_reader/property.rs delete mode 100644 samply/src/windows/etw_reader/schema.rs delete mode 100644 samply/src/windows/etw_reader/sddl.rs delete mode 100644 samply/src/windows/etw_reader/tdh.rs delete mode 100644 samply/src/windows/etw_reader/tdh_types.rs delete mode 100644 samply/src/windows/etw_reader/traits.rs delete mode 100644 samply/src/windows/etw_reader/utils.rs diff --git a/samply/src/windows/etw_reader/README.md b/samply/src/windows/etw_reader/README.md deleted file mode 100644 index 0f59b56f..00000000 --- a/samply/src/windows/etw_reader/README.md +++ /dev/null @@ -1,2 +0,0 @@ -This is mostly a hacked up version of https://github.com/n4r1b/ferrisetw modified -to support custom schemas and other things needed by etw-gecko diff --git a/samply/src/windows/etw_reader/custom_schemas.rs b/samply/src/windows/etw_reader/custom_schemas.rs deleted file mode 100644 index 0db0d0c8..00000000 --- a/samply/src/windows/etw_reader/custom_schemas.rs +++ /dev/null @@ -1,647 +0,0 @@ -use windows::core::GUID; - -use super::etw_types::DecodingSource; -use super::schema::EventSchema; -use super::tdh_types::{ - PrimitiveDesc, Property, PropertyDesc, PropertyFlags, PropertyLength, TdhInType, TdhOutType, -}; - -struct PropDesc { - name: &'static str, - in_type: TdhInType, - out_type: TdhOutType, -} - -pub struct ImageID {} - -const ImageID_PROPS: [PropDesc; 5] = [ - PropDesc { - name: "ImageBase", - in_type: TdhInType::InTypePointer, - out_type: TdhOutType::OutTypeHexInt64, - }, - PropDesc { - name: "ImageSize", - in_type: TdhInType::InTypeUInt32, - out_type: TdhOutType::OutTypeUInt32, - }, - PropDesc { - name: "Unknown", - in_type: TdhInType::InTypePointer, - out_type: TdhOutType::OutTypeUInt32, - }, - PropDesc { - name: "TimeDateStamp", - in_type: TdhInType::InTypeUInt32, - out_type: TdhOutType::OutTypeUInt32, - }, - PropDesc { - name: "OriginalFileName", - in_type: TdhInType::InTypeUnicodeString, - out_type: TdhOutType::OutTypeString, - }, -]; - -impl EventSchema for ImageID { - fn provider_guid(&self) -> GUID { - GUID::from("b3e675d7-2554-4f18-830b-2762732560de") - } - - fn event_id(&self) -> u16 { - 0 - } - - fn opcode(&self) -> u8 { - 0 - } - - fn event_version(&self) -> u8 { - 2 - } - - fn level(&self) -> u8 { - 0 - } - - fn decoding_source(&self) -> DecodingSource { - panic!() - } - - fn provider_name(&self) -> String { - "KernelTraceControl".to_owned() - } - - fn task_name(&self) -> String { - "ImageID".to_owned() - } - - fn opcode_name(&self) -> String { - "".to_string() - } - - fn property_count(&self) -> u32 { - ImageID_PROPS.len() as u32 - } - - fn property(&self, index: u32) -> Property { - let prop = &ImageID_PROPS[index as usize]; - Property { - name: prop.name.to_owned(), - desc: PropertyDesc::Primitive(PrimitiveDesc { - in_type: prop.in_type, - out_type: prop.out_type, - }), - length: PropertyLength::Length(0), - count: 1, - map_info: None, - flags: PropertyFlags::empty(), - } - } -} - -pub struct DbgID {} - -const DbgID_PROPS: [PropDesc; 5] = [ - PropDesc { - name: "ImageBase", - in_type: TdhInType::InTypeUInt64, - out_type: TdhOutType::OutTypeHexInt64, - }, - PropDesc { - name: "ProcessId", - in_type: TdhInType::InTypeUInt32, - out_type: TdhOutType::OutTypeUInt32, - }, - PropDesc { - name: "GuidSig", - in_type: TdhInType::InTypeGuid, - out_type: TdhOutType::OutTypeGuid, - }, - PropDesc { - name: "Age", - in_type: TdhInType::InTypeUInt32, - out_type: TdhOutType::OutTypeUInt32, - }, - PropDesc { - name: "PdbFileName", - in_type: TdhInType::InTypeAnsiString, - out_type: TdhOutType::OutTypeString, - }, -]; - -impl EventSchema for DbgID { - fn provider_guid(&self) -> GUID { - GUID::from("b3e675d7-2554-4f18-830b-2762732560de") - } - - fn event_id(&self) -> u16 { - 0 - } - - fn opcode(&self) -> u8 { - 36 - } - - fn event_version(&self) -> u8 { - 2 - } - - fn level(&self) -> u8 { - 0 - } - - fn decoding_source(&self) -> DecodingSource { - panic!() - } - - fn provider_name(&self) -> String { - "KernelTraceControl".to_owned() - } - - fn task_name(&self) -> String { - "ImageID".to_owned() - } - - fn opcode_name(&self) -> String { - "DbgID_RSDS".to_string() - } - - fn property_count(&self) -> u32 { - DbgID_PROPS.len() as u32 - } - - fn property(&self, index: u32) -> Property { - let prop = &DbgID_PROPS[index as usize]; - Property { - name: prop.name.to_owned(), - desc: PropertyDesc::Primitive(PrimitiveDesc { - in_type: prop.in_type, - out_type: prop.out_type, - }), - count: 1, - length: PropertyLength::Length(0), - map_info: None, - flags: PropertyFlags::empty(), - } - } -} - -// Info about EventInfo comes from SymbolTraceEventParser.cs in PerfView -// It contains an EVENT_DESCRIPTOR -pub struct EventInfo {} -const EventInfo_PROPS: [PropDesc; 9] = [ - PropDesc { - name: "ProviderGuid", - in_type: TdhInType::InTypeGuid, - out_type: TdhOutType::OutTypeGuid, - }, - PropDesc { - name: "EventGuid", - in_type: TdhInType::InTypeGuid, - out_type: TdhOutType::OutTypeGuid, - }, - PropDesc { - name: "EventDescriptorId", - in_type: TdhInType::InTypeUInt16, - out_type: TdhOutType::OutTypeUInt16, - }, - PropDesc { - name: "EventDescriptor.Version", - in_type: TdhInType::InTypeUInt8, - out_type: TdhOutType::OutTypeUInt8, - }, - PropDesc { - name: "EventDescriptor.Channel", - in_type: TdhInType::InTypeUInt8, - out_type: TdhOutType::OutTypeUInt8, - }, - PropDesc { - name: "EventDescriptor.Level", - in_type: TdhInType::InTypeUInt8, - out_type: TdhOutType::OutTypeUInt8, - }, - PropDesc { - name: "EventDescriptor.Opcode", - in_type: TdhInType::InTypeUInt8, - out_type: TdhOutType::OutTypeUInt8, - }, - PropDesc { - name: "EventDescriptor.Task", - in_type: TdhInType::InTypeUInt16, - out_type: TdhOutType::OutTypeUInt16, - }, - PropDesc { - name: "EventDescriptor.Keyword", - in_type: TdhInType::InTypeUInt64, - out_type: TdhOutType::OutTypeHexInt64, - }, -]; -impl EventSchema for EventInfo { - fn provider_guid(&self) -> GUID { - GUID::from("bbccf6c1-6cd1-48c4-80ff-839482e37671") - } - - fn event_id(&self) -> u16 { - 0 - } - - fn opcode(&self) -> u8 { - 32 - } - - fn event_version(&self) -> u8 { - 0 - } - - fn level(&self) -> u8 { - 0 - } - - fn decoding_source(&self) -> DecodingSource { - panic!() - } - - fn provider_name(&self) -> String { - "KernelTraceControl".to_owned() - } - - fn task_name(&self) -> String { - "MetaData".to_owned() - } - - fn opcode_name(&self) -> String { - "EventInfo".to_string() - } - - fn property_count(&self) -> u32 { - EventInfo_PROPS.len() as u32 - } - - fn is_event_metadata(&self) -> bool { - true - } - - fn property(&self, index: u32) -> Property { - let prop = &EventInfo_PROPS[index as usize]; - Property { - name: prop.name.to_owned(), - desc: PropertyDesc::Primitive(PrimitiveDesc { - in_type: prop.in_type, - out_type: prop.out_type, - }), - count: 1, - length: PropertyLength::Length(0), - map_info: None, - flags: PropertyFlags::empty(), - } - } -} -// We could override ThreadStop using the same properties to get a ThreadName there too. -pub struct ThreadStart {} - -const Thread_PROPS: [PropDesc; 15] = [ - PropDesc { - name: "ProcessId", - in_type: TdhInType::InTypeUInt32, - out_type: TdhOutType::OutTypeHexInt32, - }, - PropDesc { - name: "TThreadId", - in_type: TdhInType::InTypeUInt32, - out_type: TdhOutType::OutTypeHexInt32, - }, - PropDesc { - name: "StackBase", - in_type: TdhInType::InTypePointer, - out_type: TdhOutType::OutTypeNull, - }, - PropDesc { - name: "StackLimit", - in_type: TdhInType::InTypePointer, - out_type: TdhOutType::OutTypeNull, - }, - PropDesc { - name: "UserStackBase", - in_type: TdhInType::InTypePointer, - out_type: TdhOutType::OutTypeNull, - }, - PropDesc { - name: "UserStackLimit", - in_type: TdhInType::InTypePointer, - out_type: TdhOutType::OutTypeNull, - }, - PropDesc { - name: "Affinity", - in_type: TdhInType::InTypePointer, - out_type: TdhOutType::OutTypeNull, - }, - PropDesc { - name: "Win32StartAddr", - in_type: TdhInType::InTypePointer, - out_type: TdhOutType::OutTypeNull, - }, - PropDesc { - name: "TebBase", - in_type: TdhInType::InTypePointer, - out_type: TdhOutType::OutTypeNull, - }, - PropDesc { - name: "SubProcessTag", - in_type: TdhInType::InTypeUInt32, - out_type: TdhOutType::OutTypeHexInt32, - }, - PropDesc { - name: "BasePriority", - in_type: TdhInType::InTypeUInt8, - out_type: TdhOutType::OutTypeNull, - }, - PropDesc { - name: "PagePriority", - in_type: TdhInType::InTypeUInt8, - out_type: TdhOutType::OutTypeNull, - }, - PropDesc { - name: "IoPriority", - in_type: TdhInType::InTypeUInt8, - out_type: TdhOutType::OutTypeNull, - }, - PropDesc { - name: "ThreadFlags", - in_type: TdhInType::InTypeUInt8, - out_type: TdhOutType::OutTypeNull, - }, - PropDesc { - name: "ThreadName", - in_type: TdhInType::InTypeUnicodeString, - out_type: TdhOutType::OutTypeString, - }, -]; - -impl EventSchema for ThreadStart { - fn provider_guid(&self) -> GUID { - GUID::from("3D6FA8D1-FE05-11D0-9DDA-00C04FD7BA7C") - } - - fn event_id(&self) -> u16 { - 0 - } - - fn opcode(&self) -> u8 { - 3 - } - - fn event_version(&self) -> u8 { - 3 - } - - fn level(&self) -> u8 { - 0 - } - - fn decoding_source(&self) -> DecodingSource { - panic!() - } - - fn provider_name(&self) -> String { - "MSNT_SystemTrace".to_owned() - } - - fn task_name(&self) -> String { - "Thread".to_owned() - } - - fn opcode_name(&self) -> String { - "DCStart".to_string() - } - - fn property_count(&self) -> u32 { - Thread_PROPS.len() as u32 - } - - fn property(&self, index: u32) -> Property { - let prop = &Thread_PROPS[index as usize]; - Property { - name: prop.name.to_owned(), - desc: PropertyDesc::Primitive(PrimitiveDesc { - in_type: prop.in_type, - out_type: prop.out_type, - }), - length: PropertyLength::Length(0), - count: 1, - map_info: None, - flags: PropertyFlags::empty(), - } - } -} - -// from umdprovider.h -pub struct D3DUmdLogging_MapAllocation {} - -const D3DUmdLogging_PROPS: [PropDesc; 6] = [ - PropDesc { - name: "hD3DAllocation", - in_type: TdhInType::InTypeUInt64, - out_type: TdhOutType::OutTypeHexInt64, - }, - PropDesc { - name: "hDxgAllocation", - in_type: TdhInType::InTypeUInt64, - out_type: TdhOutType::OutTypeHexInt64, - }, - PropDesc { - name: "Offset", - in_type: TdhInType::InTypeUInt64, - out_type: TdhOutType::OutTypeUInt64, - }, - PropDesc { - name: "Size", - in_type: TdhInType::InTypeUInt32, - out_type: TdhOutType::OutTypeUInt64, - }, - // XXX: use an enum for these - PropDesc { - name: "Usage", - in_type: TdhInType::InTypeUInt32, - out_type: TdhOutType::OutTypeUInt32, - }, - PropDesc { - name: "Semantic", - in_type: TdhInType::InTypeUInt32, - out_type: TdhOutType::OutTypeUInt32, - }, -]; - -impl EventSchema for D3DUmdLogging_MapAllocation { - fn provider_guid(&self) -> GUID { - GUID::from("A688EE40-D8D9-4736-B6F9-6B74935BA3B1") - } - - fn event_id(&self) -> u16 { - 1 - } - - fn event_version(&self) -> u8 { - 0 - } - - fn opcode(&self) -> u8 { - 1 - } - - fn level(&self) -> u8 { - 0 - } - - fn decoding_source(&self) -> DecodingSource { - panic!() - } - - fn provider_name(&self) -> String { - "D3DUmdLogging".to_owned() - } - - fn task_name(&self) -> String { - "MapAllocation".to_owned() - } - - fn opcode_name(&self) -> String { - "Start".to_string() - } - - fn property_count(&self) -> u32 { - D3DUmdLogging_PROPS.len() as u32 - } - - fn property(&self, index: u32) -> Property { - let prop = &D3DUmdLogging_PROPS[index as usize]; - Property { - name: prop.name.to_owned(), - desc: PropertyDesc::Primitive(PrimitiveDesc { - in_type: prop.in_type, - out_type: prop.out_type, - }), - count: 1, - length: PropertyLength::Length(0), - map_info: None, - flags: PropertyFlags::empty(), - } - } -} - -pub struct D3DUmdLogging_RundownAllocation {} - -impl EventSchema for D3DUmdLogging_RundownAllocation { - fn provider_guid(&self) -> GUID { - GUID::from("A688EE40-D8D9-4736-B6F9-6B74935BA3B1") - } - - fn event_id(&self) -> u16 { - 2 - } - - fn event_version(&self) -> u8 { - 0 - } - - fn opcode(&self) -> u8 { - 3 - } - - fn level(&self) -> u8 { - 0 - } - - fn decoding_source(&self) -> DecodingSource { - panic!() - } - - fn provider_name(&self) -> String { - "D3DUmdLogging".to_owned() - } - - fn task_name(&self) -> String { - "MapAllocation".to_owned() - } - - fn opcode_name(&self) -> String { - "DC Start".to_string() - } - - fn property_count(&self) -> u32 { - D3DUmdLogging_PROPS.len() as u32 - } - - fn property(&self, index: u32) -> Property { - let prop = &D3DUmdLogging_PROPS[index as usize]; - Property { - name: prop.name.to_owned(), - desc: PropertyDesc::Primitive(PrimitiveDesc { - in_type: prop.in_type, - out_type: prop.out_type, - }), - count: 1, - length: PropertyLength::Length(0), - map_info: None, - flags: PropertyFlags::empty(), - } - } -} - -pub struct D3DUmdLogging_UnmapAllocation {} - -impl EventSchema for D3DUmdLogging_UnmapAllocation { - fn provider_guid(&self) -> GUID { - GUID::from("A688EE40-D8D9-4736-B6F9-6B74935BA3B1") - } - - fn event_id(&self) -> u16 { - 3 - } - - fn event_version(&self) -> u8 { - 0 - } - - fn opcode(&self) -> u8 { - 2 - } - - fn level(&self) -> u8 { - 0 - } - - fn decoding_source(&self) -> DecodingSource { - panic!() - } - - fn provider_name(&self) -> String { - "D3DUmdLogging".to_owned() - } - - fn task_name(&self) -> String { - "MapAllocation".to_owned() - } - - fn opcode_name(&self) -> String { - "End".to_string() - } - - fn property_count(&self) -> u32 { - D3DUmdLogging_PROPS.len() as u32 - } - - fn property(&self, index: u32) -> Property { - let prop = &D3DUmdLogging_PROPS[index as usize]; - Property { - name: prop.name.to_owned(), - desc: PropertyDesc::Primitive(PrimitiveDesc { - in_type: prop.in_type, - out_type: prop.out_type, - }), - count: 1, - length: PropertyLength::Length(0), - map_info: None, - flags: PropertyFlags::empty(), - } - } -} diff --git a/samply/src/windows/etw_reader/etw_types.rs b/samply/src/windows/etw_reader/etw_types.rs deleted file mode 100644 index 729deb96..00000000 --- a/samply/src/windows/etw_reader/etw_types.rs +++ /dev/null @@ -1,303 +0,0 @@ -use std::ops::Deref; -use std::rc::Rc; - -use once_cell::unsync::OnceCell; -use windows::core::{GUID, PCWSTR}; -use windows::Win32::System::Diagnostics::Etw::{self, PropertyStruct}; - -use super::schema::EventSchema; -use super::tdh_types::{Property, PropertyMapInfo}; -use super::utils; - -#[repr(transparent)] -pub struct EventRecord(Etw::EVENT_RECORD); - -impl Deref for EventRecord { - type Target = Etw::EVENT_RECORD; - - fn deref(&self) -> &Self::Target { - &self.0 - } -} - -impl EventRecord { - pub(crate) fn user_buffer(&self) -> &[u8] { - unsafe { std::slice::from_raw_parts(self.UserData as *mut _, self.UserDataLength.into()) } - } -} - -/// Newtype wrapper over an [EVENT_PROPERTY_INFO] -/// -/// [EVENT_PROPERTY_INFO]: https://microsoft.github.io/windows-docs-rs/doc/bindings/Windows/Win32/Etw/struct.EVENT_PROPERTY_INFO.html -#[repr(C)] -#[derive(Clone, Copy)] -pub struct EventPropertyInfo(Etw::EVENT_PROPERTY_INFO); - -impl std::ops::Deref for EventPropertyInfo { - type Target = Etw::EVENT_PROPERTY_INFO; - - fn deref(&self) -> &self::Etw::EVENT_PROPERTY_INFO { - &self.0 - } -} - -impl std::ops::DerefMut for EventPropertyInfo { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.0 - } -} - -impl From<&[u8]> for EventPropertyInfo { - fn from(val: &[u8]) -> Self { - unsafe { *(val.as_ptr() as *mut EventPropertyInfo) } - } -} - -impl Default for EventPropertyInfo { - fn default() -> Self { - unsafe { std::mem::zeroed::() } - } -} - -// Safe cast (EVENT_HEADER_FLAG_32_BIT_HEADER = 32) -#[doc(hidden)] -pub const EVENT_HEADER_FLAG_32_BIT_HEADER: u16 = Etw::EVENT_HEADER_FLAG_32_BIT_HEADER as u16; - -/// Wrapper over the [DECODING_SOURCE] type -/// -/// [DECODING_SOURCE]: https://microsoft.github.io/windows-docs-rs/doc/bindings/Windows/Win32/Etw/struct.DECODING_SOURCE.html -#[derive(Debug)] -pub enum DecodingSource { - DecodingSourceXMLFile, - DecodingSourceWbem, - DecodingSourceWPP, - DecodingSourceTlg, - DecodingSourceMax, -} - -impl From for DecodingSource { - fn from(val: Etw::DECODING_SOURCE) -> Self { - match val { - Etw::DecodingSourceXMLFile => DecodingSource::DecodingSourceXMLFile, - Etw::DecodingSourceWbem => DecodingSource::DecodingSourceWbem, - Etw::DecodingSourceWPP => DecodingSource::DecodingSourceWPP, - Etw::DecodingSourceTlg => DecodingSource::DecodingSourceTlg, - _ => DecodingSource::DecodingSourceMax, - } - } -} - -/// Newtype wrapper over an [TRACE_EVENT_INFO] -/// -/// [TRACE_EVENT_INFO]: https://microsoft.github.io/windows-docs-rs/doc/bindings/Windows/Win32/Etw/struct.TRACE_EVENT_INFO.html -#[repr(C)] -#[derive(Clone, Copy)] -pub struct TraceEventInfo(Etw::TRACE_EVENT_INFO); - -impl std::ops::Deref for TraceEventInfo { - type Target = Etw::TRACE_EVENT_INFO; - - fn deref(&self) -> &self::Etw::TRACE_EVENT_INFO { - &self.0 - } -} - -#[repr(C)] -#[derive(Debug, Clone, Default)] -pub struct TraceEventInfoRaw { - pub(crate) info: Vec, - property_maps: OnceCell>>>>, -} - -impl std::ops::DerefMut for TraceEventInfo { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.0 - } -} - -impl From<&TraceEventInfoRaw> for TraceEventInfo { - fn from(val: &TraceEventInfoRaw) -> Self { - unsafe { *(val.info.as_ptr() as *mut TraceEventInfo) } - } -} - -impl TraceEventInfoRaw { - pub(crate) fn new(info: Vec) -> Self { - TraceEventInfoRaw { - info, - property_maps: OnceCell::new(), - } - } - pub(crate) fn alloc(len: u32) -> Self { - TraceEventInfoRaw { - info: vec![0; len as usize], - property_maps: OnceCell::new(), - } - } - - pub fn info_as_ptr(&mut self) -> *mut u8 { - self.info.as_mut_ptr() - } - - fn property_map_info(&self, index: u32) -> Option> { - // let's make sure index is not bigger thant the PropertyCount - assert!(index <= TraceEventInfo::from(self).PropertyCount); - let property_maps = self.property_maps.get_or_init(|| { - vec![OnceCell::new(); TraceEventInfo::from(self).PropertyCount as usize] - }); - let map = property_maps[index as usize].get_or_init(|| { - // We need to subtract the sizeof(EVENT_PROPERTY_INFO) due to how TRACE_EVENT_INFO is declared - // in the bindings, the last field `EventPropertyInfoArray[ANYSIZE_ARRAY]` is declared as - // [EVENT_PROPERTY_INFO; 1] - // https://microsoft.github.io/windows-docs-rs/doc/bindings/Windows/Win32/Etw/struct.TRACE_EVENT_INFO.html#structfield.EventPropertyInfoArray - let curr_prop_offset = index as usize * std::mem::size_of::() - + (std::mem::size_of::() - - std::mem::size_of::()); - - let curr_prop = EventPropertyInfo::from(&self.info[curr_prop_offset..]); - if curr_prop.Flags.0 & PropertyStruct.0 == 0 { - // This property is a struct so it has no map info - return None; - } else { - unsafe { - if curr_prop.Anonymous1.nonStructType.MapNameOffset != 0 { - // build an empty event record that we can use to get the map info - let mut event: Etw::EVENT_RECORD = std::mem::zeroed(); - event.EventHeader.ProviderId = self.provider_guid(); - - let mut buffer_size = 0; - let map_name = PCWSTR( - self.info[curr_prop.Anonymous1.nonStructType.MapNameOffset as usize..] - .as_ptr() as *mut u16, - ); - use windows::Win32::Foundation::ERROR_INSUFFICIENT_BUFFER; - // println!("map_name {}", utils::parse_unk_size_null_utf16_string(&self.info[curr_prop.Anonymous1.nonStructType.MapNameOffset as usize..])); - if Etw::TdhGetEventMapInformation(&event, map_name, None, &mut buffer_size) - != ERROR_INSUFFICIENT_BUFFER.0 - { - panic!("expected this to fail"); - } - - let mut buffer = vec![0; buffer_size as usize]; - if Etw::TdhGetEventMapInformation( - &event, - map_name, - Some(buffer.as_mut_ptr() as *mut _), - &mut buffer_size, - ) != 0 - { - panic!(); - } - - let map_info: &super::Etw::EVENT_MAP_INFO = &*(buffer.as_ptr() as *const _); - if map_info.Flag == super::Etw::EVENTMAP_INFO_FLAG_MANIFEST_VALUEMAP - || map_info.Flag == super::Etw::EVENTMAP_INFO_FLAG_MANIFEST_BITMAP - { - let is_bitmap = - map_info.Flag == super::Etw::EVENTMAP_INFO_FLAG_MANIFEST_BITMAP; - let mut map = super::FastHashMap::default(); - assert!( - map_info.Anonymous.MapEntryValueType - == super::Etw::EVENTMAP_ENTRY_VALUETYPE_ULONG - ); - let entries = std::slice::from_raw_parts( - map_info.MapEntryArray.as_ptr(), - map_info.EntryCount as usize, - ); - for e in entries { - let value = e.Anonymous.Value; - let name = utils::parse_unk_size_null_utf16_string( - &buffer[e.OutputOffset as usize..], - ); - // println!("{} -> {:?}", value, name); - map.insert(value, name); - } - return Some(Rc::new(PropertyMapInfo { is_bitmap, map })); - } else { - eprint!("unsupported map type {:?}", map_info.Flag); - } - } - } - } - return None; - }); - map.clone() - } -} - -impl EventSchema for TraceEventInfoRaw { - fn provider_guid(&self) -> GUID { - TraceEventInfo::from(self).ProviderGuid - } - - fn event_id(&self) -> u16 { - TraceEventInfo::from(self).EventDescriptor.Id - } - - fn opcode(&self) -> u8 { - TraceEventInfo::from(self).EventDescriptor.Opcode - } - - fn event_version(&self) -> u8 { - TraceEventInfo::from(self).EventDescriptor.Version - } - - fn level(&self) -> u8 { - TraceEventInfo::from(self).EventDescriptor.Level - } - - fn decoding_source(&self) -> DecodingSource { - DecodingSource::from(TraceEventInfo::from(self).DecodingSource) - } - - fn provider_name(&self) -> String { - let provider_name_offset = TraceEventInfo::from(self).ProviderNameOffset as usize; - // TODO: Evaluate performance, but this sounds better than creating a whole Vec and getting the string from the offset/2 - utils::parse_unk_size_null_utf16_string(&self.info[provider_name_offset..]) - } - - fn task_name(&self) -> String { - let task_name_offset = TraceEventInfo::from(self).TaskNameOffset as usize; - utils::parse_unk_size_null_utf16_string(&self.info[task_name_offset..]) - } - - fn opcode_name(&self) -> String { - let opcode_name_offset = TraceEventInfo::from(self).OpcodeNameOffset as usize; - if opcode_name_offset == 0 { - return String::from(""); - } - utils::parse_unk_size_null_utf16_string(&self.info[opcode_name_offset..]) - } - - fn property_count(&self) -> u32 { - TraceEventInfo::from(self).TopLevelPropertyCount - } - - fn property(&self, index: u32) -> Property { - // let's make sure index is not bigger thant the PropertyCount - assert!(index <= TraceEventInfo::from(self).PropertyCount); - - // We need to subtract the sizeof(EVENT_PROPERTY_INFO) due to how TRACE_EVENT_INFO is declared - // in the bindings, the last field `EventPropertyInfoArray[ANYSIZE_ARRAY]` is declared as - // [EVENT_PROPERTY_INFO; 1] - // https://microsoft.github.io/windows-docs-rs/doc/bindings/Windows/Win32/Etw/struct.TRACE_EVENT_INFO.html#structfield.EventPropertyInfoArray - let curr_prop_offset = index as usize * std::mem::size_of::() - + (std::mem::size_of::() - std::mem::size_of::()); - - let curr_prop = EventPropertyInfo::from(&self.info[curr_prop_offset..]); - let name = - utils::parse_unk_size_null_utf16_string(&self.info[curr_prop.NameOffset as usize..]); - Property::new(name, &curr_prop, self.property_map_info(index)) - } - - fn event_message(&self) -> Option { - let offset = TraceEventInfo::from(self).EventMessageOffset; - if offset != 0 { - Some(utils::parse_unk_size_null_utf16_string( - &self.info[offset as usize..], - )) - } else { - None - } - } -} diff --git a/samply/src/windows/etw_reader/mod.rs b/samply/src/windows/etw_reader/mod.rs deleted file mode 100644 index cd397992..00000000 --- a/samply/src/windows/etw_reader/mod.rs +++ /dev/null @@ -1,615 +0,0 @@ -#![allow(warnings)] - -use std::borrow::Cow; -use std::collections::HashMap; -use std::hash::BuildHasherDefault; -use std::mem; -use std::path::Path; - -use bitflags::bitflags; -use etw_types::EventRecord; -use fxhash::FxHasher; -use memoffset::offset_of; -use parser::{Parser, ParserError, TryParse}; -use schema::SchemaLocator; -use tdh_types::{PrimitiveDesc, Property, PropertyDesc, TdhInType, TdhOutType}; -use traits::EncodeUtf16; -use windows::core::{h, Error, HRESULT, HSTRING, PWSTR}; -use windows::Win32::Foundation::{ - GetLastError, ERROR_INSUFFICIENT_BUFFER, ERROR_MORE_DATA, ERROR_SUCCESS, MAX_PATH, -}; -use windows::Win32::System::Diagnostics::Etw; -use windows::Win32::System::Diagnostics::Etw::{ - EnumerateTraceGuids, EnumerateTraceGuidsEx, TraceGuidQueryInfo, TraceGuidQueryList, - CONTROLTRACE_HANDLE, EVENT_TRACE_FLAG, TRACE_GUID_INFO, TRACE_GUID_PROPERTIES, - TRACE_PROVIDER_INSTANCE_INFO, -}; - -// typedef ULONG64 TRACEHANDLE, *PTRACEHANDLE; -pub(crate) type TraceHandle = u64; -pub const INVALID_TRACE_HANDLE: TraceHandle = u64::MAX; -//, WindowsProgramming}; - -pub mod custom_schemas; -pub mod etw_types; -pub mod parser; -pub mod property; -pub mod schema; -pub mod sddl; -pub mod tdh; -pub mod tdh_types; -pub mod traits; -pub mod utils; -//pub mod trace; -//pub mod provider; - -pub use windows::core::GUID; - -pub type FastHashMap = HashMap>; -#[repr(C)] -#[derive(Clone)] -pub struct EventTraceLogfile(Etw::EVENT_TRACE_LOGFILEW); - -impl Default for EventTraceLogfile { - fn default() -> Self { - unsafe { std::mem::zeroed::() } - } -} - -impl std::ops::Deref for EventTraceLogfile { - type Target = Etw::EVENT_TRACE_LOGFILEW; - - fn deref(&self) -> &self::Etw::EVENT_TRACE_LOGFILEW { - &self.0 - } -} -impl std::ops::DerefMut for EventTraceLogfile { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.0 - } -} - -unsafe extern "system" fn trace_callback_thunk(event_record: *mut Etw::EVENT_RECORD) { - let f: &mut &mut dyn FnMut(&EventRecord) = std::mem::transmute((*event_record).UserContext); - f(std::mem::transmute(event_record)) -} - -pub fn open_trace( - path: &Path, - mut callback: F, -) -> Result<(), std::io::Error> { - let mut log_file = EventTraceLogfile::default(); - - #[cfg(windows)] - let path = HSTRING::from(path.as_os_str()); - #[cfg(not(windows))] - let path: HSTRING = panic!(); - log_file.0.LogFileName = PWSTR(path.as_wide().as_ptr() as *mut _); - log_file.0.Anonymous1.ProcessTraceMode = - Etw::PROCESS_TRACE_MODE_EVENT_RECORD | Etw::PROCESS_TRACE_MODE_RAW_TIMESTAMP; - let mut cb: &mut dyn FnMut(&EventRecord) = &mut callback; - log_file.0.Context = unsafe { std::mem::transmute(&mut cb) }; - log_file.0.Anonymous2.EventRecordCallback = Some(trace_callback_thunk); - - let session_handle = unsafe { Etw::OpenTraceW(&mut *log_file) }; - let result = unsafe { Etw::ProcessTrace(&[session_handle], None, None) }; - result - .ok() - .map_err(|e| std::io::Error::from_raw_os_error(e.code().0)) -} - -/// Complete Trace Properties struct -/// -/// The [EventTraceProperties] struct contains the information about a tracing session, this struct -/// also needs two buffers right after it to hold the log file name and the session name. This struct -/// provides the full definition of the properties plus the the allocation for both names -/// -/// See: [EVENT_TRACE_PROPERTIES](https://docs.microsoft.com/en-us/windows/win32/api/evntrace/ns-evntrace-event_trace_properties) -#[repr(C)] -#[derive(Clone, Copy)] -pub struct TraceInfo { - pub properties: Etw::EVENT_TRACE_PROPERTIES, - // it's not clear that these need to be u16 if using with the Unicode versions of the functions ETW functions - trace_name: [u16; MAX_PATH as usize], - log_file_name: [u16; MAX_PATH as usize], -} - -impl Default for TraceInfo { - fn default() -> Self { - let properties = Etw::EVENT_TRACE_PROPERTIES::default(); - TraceInfo { - properties, - trace_name: [0; 260], - log_file_name: [0; 260], - } - } -} - -impl TraceInfo { - pub(crate) fn fill( - &mut self, - trace_name: &str, - //trace_properties: &TraceProperties, - ) { - self.properties.Wnode.BufferSize = std::mem::size_of::() as u32; - self.properties.Wnode.Guid = GUID::new().unwrap(); - self.properties.Wnode.Flags = Etw::WNODE_FLAG_TRACED_GUID; - self.properties.Wnode.ClientContext = 1; // QPC clock resolution - self.properties.FlushTimer = 1; - - self.properties.LogFileMode = - Etw::EVENT_TRACE_REAL_TIME_MODE | Etw::EVENT_TRACE_NO_PER_PROCESSOR_BUFFERING; - - self.properties.EnableFlags = EVENT_TRACE_FLAG(0); - - //self.properties.LoggerNameOffset = offset_of!(TraceInfo, log_file_name) as u32; - //self.trace_name[..trace_name.len()].copy_from_slice(trace_name.as_bytes()) - - // it doesn't seem like it matters if we fill in trace_name - self.properties.LoggerNameOffset = offset_of!(TraceInfo, trace_name) as u32; - self.properties.LogFileNameOffset = offset_of!(TraceInfo, log_file_name) as u32; - self.trace_name[..trace_name.len() + 1].copy_from_slice(&trace_name.as_utf16()) - } -} - -#[repr(C)] -#[derive(Clone, Copy, Default)] -pub struct EnableTraceParameters(Etw::ENABLE_TRACE_PARAMETERS); - -impl EnableTraceParameters { - pub fn create(guid: GUID, trace_flags: u32) -> Self { - let mut params = EnableTraceParameters::default(); - params.0.ControlFlags = 0; - params.0.Version = Etw::ENABLE_TRACE_PARAMETERS_VERSION_2; - params.0.SourceId = guid; - params.0.EnableProperty = trace_flags; - - // TODO: Add Filters option - params.0.EnableFilterDesc = std::ptr::null_mut(); - params.0.FilterDescCount = 0; - - params - } -} - -impl std::ops::Deref for EnableTraceParameters { - type Target = Etw::ENABLE_TRACE_PARAMETERS; - - fn deref(&self) -> &self::Etw::ENABLE_TRACE_PARAMETERS { - &self.0 - } -} - -impl std::ops::DerefMut for EnableTraceParameters { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.0 - } -} - -impl Provider { - /// Use the `new` function to create a Provider builder - /// - /// This function will create a by-default provider which can be tweaked afterwards - /// - /// # Example - /// ```rust - /// let my_provider = Provider::new(); - /// ``` - pub fn new() -> Self { - Provider { - guid: None, - any: 0, - all: 0, - level: 5, - trace_flags: 0, - flags: 0, - } - } - - pub fn by_guid(mut self, guid: &str) -> Self { - self.guid = Some(GUID::from(guid)); - self - } -} - -pub struct Provider { - /// Option that represents a Provider GUID - pub guid: Option, - /// Provider Any keyword - pub any: u64, - /// Provider All keyword - pub all: u64, - /// Provider level flag - pub level: u8, - /// Provider trace flags - pub trace_flags: u32, - /// Provider kernel flags, only apply to KernelProvider - pub flags: u32, // Only applies to KernelProviders - // perfinfo - - // filters: RwLock>, -} - -pub fn start_trace(mut callback: F) { - let guid_str = "22fb2cd6-0e7b-422b-a0c7-2fad1fd0e716"; - let guid_str = "DB6F6DDB-AC77-4E88-8253-819DF9BBF140"; - let mut video_blt_guid = GUID::from(guid_str); //GUID::from("DB6F6DDB-AC77-4E88-8253-819DF9BBF140"); - - let session_name = h!("aaaaaa"); - - let mut info = TraceInfo::default(); - info.fill(&session_name.to_string()); - - let mut handle = CONTROLTRACE_HANDLE { Value: 0 }; - - unsafe { - let status = Etw::ControlTraceW( - handle, - session_name, - &info.properties as *const _ as *mut _, - Etw::EVENT_TRACE_CONTROL_STOP, - ); - println!("ControlTrace = {:?}", status); - let status = Etw::StartTraceW( - &mut handle, - session_name, - &info.properties as *const _ as *mut _, - ); - println!("StartTrace = {:?} handle {:?}", status, handle); - info.trace_name = [0; 260]; - - let status = Etw::ControlTraceW( - handle, - session_name, - &mut info.properties, - Etw::EVENT_TRACE_CONTROL_QUERY, - ); - println!( - "ControlTrace = {:?} {} {:?} {:?}", - status, info.properties.BufferSize, info.properties.LoggerThreadId, info.trace_name - ); - } - - let prov = Provider::new().by_guid(guid_str); - let parameters = EnableTraceParameters::create(video_blt_guid, prov.trace_flags); - /* - let status = unsafe { Etw::EnableTrace(1, 0xffffffff, Etw::TRACE_LEVEL_VERBOSE, &video_blt_guid, handle)}; - println!("EnableTrace = {}", status); - */ - unsafe { - Etw::EnableTraceEx2( - handle, - &mut video_blt_guid, - 1, // Fixme: EVENT_CONTROL_CODE_ENABLE_PROVIDER - prov.level, - prov.any, - prov.all, - 0, - Some(&*parameters), - ); - } - - let mut trace = EventTraceLogfile::default(); - trace.0.LoggerName = PWSTR(session_name.as_ptr() as *mut _); - trace.0.Anonymous1.ProcessTraceMode = - Etw::PROCESS_TRACE_MODE_REAL_TIME | Etw::PROCESS_TRACE_MODE_EVENT_RECORD; - let mut cb: &mut dyn FnMut(&EventRecord) = &mut callback; - trace.0.Context = unsafe { std::mem::transmute(&mut cb) }; - trace.0.Anonymous2.EventRecordCallback = Some(trace_callback_thunk); - - let session_handle = unsafe { Etw::OpenTraceW(&mut *trace) }; - if session_handle.Value == INVALID_TRACE_HANDLE { - println!( - "{:?} {:?}", - unsafe { GetLastError() }, - windows::core::Error::from_win32() - ); - - panic!("Invalid handle"); - } - println!("OpenTrace {:?}", session_handle); - let status = unsafe { Etw::ProcessTrace(&[session_handle], None, None) }; - println!("status: {:?}", status); -} - -pub fn event_properties_to_string( - s: &schema::TypedEvent, - parser: &mut Parser, - skip_properties: Option<&[&str]>, -) -> String { - let mut text = String::new(); - for i in 0..s.property_count() { - let property = s.property(i); - if let Some(propfilter) = skip_properties { - if propfilter.iter().any(|&s| s == property.name) { - continue; - } - } - - write_property(&mut text, parser, &property, false); - text += ", " - } - - text -} - -pub fn write_property( - output: &mut dyn std::fmt::Write, - parser: &mut Parser, - property: &Property, - write_types: bool, -) { - if write_types { - let type_name = if let PropertyDesc::Primitive(prim) = &property.desc { - format!("{:?}", prim.in_type) - } else { - format!("{:?}", property.desc) - }; - if property.flags.is_empty() { - write!(output, " {}: {} = ", property.name, type_name).unwrap(); - } else { - write!( - output, - " {}({:?}): {} = ", - property.name, property.flags, type_name - ) - .unwrap(); - } - } else { - write!(output, " {}= ", property.name).unwrap(); - } - if let Some(map_info) = &property.map_info { - let value = match property.desc { - PropertyDesc::Primitive(PrimitiveDesc { - in_type: TdhInType::InTypeUInt32, - .. - }) => TryParse::::parse(parser, &property.name), - PropertyDesc::Primitive(PrimitiveDesc { - in_type: TdhInType::InTypeUInt16, - .. - }) => TryParse::::parse(parser, &property.name) as u32, - PropertyDesc::Primitive(PrimitiveDesc { - in_type: TdhInType::InTypeUInt8, - .. - }) => TryParse::::parse(parser, &property.name) as u32, - _ => panic!("{:?}", property.desc), - }; - if map_info.is_bitmap { - let mut remaining_bits_str = String::new(); - let mut matches: Vec<&str> = Vec::new(); - let mut cleared_value = value; - for (k, v) in &map_info.map { - if value & k != 0 { - matches.push(v.trim()); - cleared_value &= !k; - } - } - if cleared_value != 0 { - remaining_bits_str = format!("{:x}", cleared_value); - matches.push(&remaining_bits_str); - //println!("unnamed bits {:x} {:x} {:x?}", value, cleared_value, map_info.map); - } - write!(output, "{}", matches.join(" | ")).unwrap(); - } else { - write!( - output, - "{}", - map_info - .map - .get(&value) - .map(|x| Cow::from(x)) - .unwrap_or_else(|| Cow::from(format!("Unknown: {}", value))) - ) - .unwrap(); - } - } else { - let value = match &property.desc { - PropertyDesc::Primitive(desc) => { - // XXX: we should be using the out_type here instead of in_type - match desc.in_type { - TdhInType::InTypeUnicodeString => { - TryParse::::try_parse(parser, &property.name) - } - TdhInType::InTypeAnsiString => { - TryParse::::try_parse(parser, &property.name) - } - TdhInType::InTypeBoolean => { - TryParse::::try_parse(parser, &property.name).map(|x| x.to_string()) - } - TdhInType::InTypeHexInt32 => { - TryParse::::try_parse(parser, &property.name).map(|x| x.to_string()) - } - TdhInType::InTypeUInt32 => { - TryParse::::try_parse(parser, &property.name).map(|x| x.to_string()) - } - TdhInType::InTypeUInt16 => { - TryParse::::try_parse(parser, &property.name).map(|x| x.to_string()) - } - TdhInType::InTypeUInt8 => { - TryParse::::try_parse(parser, &property.name).map(|x| x.to_string()) - } - TdhInType::InTypeInt8 => { - TryParse::::try_parse(parser, &property.name).map(|x| x.to_string()) - } - TdhInType::InTypeInt64 => { - TryParse::::try_parse(parser, &property.name).map(|x| x.to_string()) - } - TdhInType::InTypeUInt64 => { - let i = TryParse::::try_parse(parser, &property.name); - if desc.out_type == TdhOutType::OutTypeHexInt64 { - i.map(|x| format!("0x{:x}", x)) - } else { - i.map(|x| x.to_string()) - } - } - TdhInType::InTypeHexInt64 => { - let i = TryParse::::try_parse(parser, &property.name); - i.map(|x| format!("0x{:x}", x)) - } - TdhInType::InTypePointer | TdhInType::InTypeSizeT => { - TryParse::::try_parse(parser, &property.name) - .map(|x| format!("0x{:x}", x)) - } - TdhInType::InTypeGuid => TryParse::::try_parse(parser, &property.name) - .map(|x| format!("{:?}", x)), - TdhInType::InTypeInt32 => { - TryParse::::try_parse(parser, &property.name).map(|x| x.to_string()) - } - TdhInType::InTypeFloat => { - TryParse::::try_parse(parser, &property.name).map(|x| x.to_string()) - } - _ => Ok(format!("Unknown {:?} -> {:?}", desc.in_type, desc.out_type)), - } - } - PropertyDesc::Struct(desc) => Ok(format!( - "unhandled struct {} {}", - desc.start_index, desc.num_members - )), - }; - let value = match value { - Ok(value) => value, - Err(ParserError::InvalidType) => format!("invalid type {:?}", property.desc), - Err(ParserError::LengthMismatch) => format!( - "Err(LengthMismatch) type: {:?}, flags: {:?}, buf: {}", - property.desc, - property.flags, - parser.buffer.len() - ), - Err(e) => format!("Err({:?}) type: {:?}", e, property.desc), - }; - write!(output, "{}", value).unwrap(); - } -} - -pub fn print_property(parser: &mut Parser, property: &Property, write_types: bool) { - let mut result = String::new(); - write_property(&mut result, parser, property, write_types); - println!("{}", result); -} - -pub fn add_custom_schemas(locator: &mut SchemaLocator) { - locator.add_custom_schema(Box::new(custom_schemas::ImageID {})); - locator.add_custom_schema(Box::new(custom_schemas::DbgID {})); - locator.add_custom_schema(Box::new(custom_schemas::EventInfo {})); - locator.add_custom_schema(Box::new(custom_schemas::ThreadStart {})); - locator.add_custom_schema(Box::new(custom_schemas::D3DUmdLogging_MapAllocation {})); - locator.add_custom_schema(Box::new(custom_schemas::D3DUmdLogging_RundownAllocation {})); - locator.add_custom_schema(Box::new(custom_schemas::D3DUmdLogging_UnmapAllocation {})); -} - -pub fn enumerate_trace_guids() { - let mut count = 1; - loop { - let mut guids: Vec = - vec![unsafe { std::mem::zeroed() }; count as usize]; - let mut ptrs: Vec<*mut TRACE_GUID_PROPERTIES> = Vec::new(); - for guid in &mut guids { - ptrs.push(guid) - } - - let result = unsafe { EnumerateTraceGuids(&mut ptrs.as_mut_slice(), &mut count) }; - match result.ok() { - Ok(()) => { - for guid in guids[..count as usize].iter() { - println!("{:?}", guid.Guid); - } - break; - } - Err(e) => { - if e.code() != ERROR_MORE_DATA.to_hresult() { - break; - } - } - } - } -} - -pub fn enumerate_trace_guids_ex(print_instances: bool) { - let mut required_size: u32 = 0; - - loop { - let mut guids: Vec = - vec![GUID::zeroed(); required_size as usize / mem::size_of::()]; - - let size = (guids.len() * mem::size_of::()) as u32; - println!("get {}", required_size); - - let result = unsafe { - EnumerateTraceGuidsEx( - TraceGuidQueryList, - None, - 0, - Some(guids.as_mut_ptr() as *mut _), - size, - &mut required_size as *mut _, - ) - }; - match result.ok() { - Ok(()) => { - for guid in guids.iter() { - println!("{:?}", guid); - let info = get_provider_info(guid); - let instance_count = - unsafe { *(info.as_ptr() as *const TRACE_GUID_INFO) }.InstanceCount; - let mut instance_ptr: *const TRACE_PROVIDER_INSTANCE_INFO = unsafe { - (info.as_ptr().add(mem::size_of::()) - as *const TRACE_PROVIDER_INSTANCE_INFO) - }; - - for _ in 0..instance_count { - let instance = unsafe { &*instance_ptr }; - if print_instances { - println!( - "enable_count {}, pid {}, flags {}", - instance.EnableCount, instance.Pid, instance.Flags, - ) - } - instance_ptr = unsafe { - ((instance_ptr as *const TRACE_PROVIDER_INSTANCE_INFO as *const u8) - .add(instance.NextOffset as usize) - as *const TRACE_PROVIDER_INSTANCE_INFO) - }; - } - } - break; - } - Err(e) => { - if e.code() != ERROR_INSUFFICIENT_BUFFER.to_hresult() { - println!("some other error"); - break; - } - } - } - } -} - -pub fn get_provider_info(guid: &GUID) -> Vec { - let mut required_size: u32 = 0; - - loop { - let mut info: Vec = vec![0; required_size as usize]; - - let size = info.len() as u32; - - let result = unsafe { - EnumerateTraceGuidsEx( - TraceGuidQueryInfo, - Some(guid as *const GUID as *const _), - mem::size_of::() as u32, - Some(info.as_mut_ptr() as *mut _), - size, - &mut required_size as *mut _, - ) - }; - match result.ok() { - Ok(()) => { - return info; - } - Err(e) => { - if e.code() != ERROR_INSUFFICIENT_BUFFER.to_hresult() { - panic!("{:?}", e); - } - } - } - } -} diff --git a/samply/src/windows/etw_reader/parser.rs b/samply/src/windows/etw_reader/parser.rs deleted file mode 100644 index 047e46e6..00000000 --- a/samply/src/windows/etw_reader/parser.rs +++ /dev/null @@ -1,647 +0,0 @@ -//! ETW Types Parser -//! -//! This module act as a helper to parse the Buffer from an ETW Event -use std::borrow::Borrow; -use std::convert::TryInto; -use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; - -use windows::core::GUID; - -use super::etw_types::EVENT_HEADER_FLAG_32_BIT_HEADER; -use super::property::{PropertyInfo, PropertyIter}; -use super::schema::TypedEvent; -use super::tdh_types::{ - PrimitiveDesc, Property, PropertyDesc, PropertyFlags, PropertyLength, TdhInType, TdhOutType, -}; -use super::{sddl, tdh, utils}; - -#[derive(Debug, Clone, Copy)] -pub enum Address { - Address64(u64), - Address32(u32), -} - -impl Address { - pub fn as_u64(&self) -> u64 { - match self { - Address::Address64(a) => *a, - Address::Address32(a) => *a as u64, - } - } -} - -/// Parser module errors -#[derive(Debug)] -pub enum ParserError { - /// An invalid type... - InvalidType, - /// Error parsing - ParseError, - /// Length mismatch when parsing a type - LengthMismatch, - PropertyError(String), - /// An error while transforming an Utf-8 buffer into String - Utf8Error(std::string::FromUtf8Error), - /// An error trying to get an slice as an array - SliceError(std::array::TryFromSliceError), - /// Represents an internal [SddlNativeError] - /// - /// [SddlNativeError]: sddl::SddlNativeError - //SddlNativeError(sddl::SddlNativeError), - /// Represents an internal [TdhNativeError] - /// - /// [TdhNativeError]: tdh::TdhNativeError - TdhNativeError(tdh::TdhNativeError), -} - -impl From for ParserError { - fn from(err: tdh::TdhNativeError) -> Self { - ParserError::TdhNativeError(err) - } -} -/* -impl From for ParserError { - fn from(err: sddl::SddlNativeError) -> Self { - ParserError::SddlNativeError(err) - } -}*/ - -impl From for ParserError { - fn from(err: std::string::FromUtf8Error) -> Self { - ParserError::Utf8Error(err) - } -} - -impl From for ParserError { - fn from(err: std::array::TryFromSliceError) -> Self { - ParserError::SliceError(err) - } -} - -type ParserResult = Result; - -/// Trait to try and parse a type -/// -/// This trait has to be implemented in order to be able to parse a type we want to retrieve from -/// within an Event. On success the parsed value will be returned within a Result, on error an Err -/// should be returned accordingly -/// -/// An implementation for most of the Primitive Types is created by using a Macro, any other needed type -/// requires this trait to be implemented -// TODO: Find a way to use turbofish operator -pub trait TryParse { - /// Implement the `try_parse` function to provide a way to Parse `T` from an ETW event or - /// return an Error in case the type `T` can't be parsed - /// - /// # Arguments - /// * `name` - Name of the property to be found in the Schema - fn try_parse(&mut self, name: &str) -> Result; - fn parse(&mut self, name: &str) -> T { - self.try_parse(name) - .unwrap_or_else(|e| panic!("{:?} name {} {:?}", e, std::any::type_name::(), name)) - } -} - -/// Represents a Parser -/// -/// This structure holds the necessary data to parse the ETW event and retrieve the data from the -/// event -#[allow(dead_code)] -pub struct Parser<'a> { - event: &'a TypedEvent<'a>, - properties: &'a PropertyIter, - pub buffer: &'a [u8], - last_property: u32, - offset: usize, - // a map from property indx to PropertyInfo - cache: Vec>, -} - -impl<'a> Parser<'a> { - /// Use the `create` function to create an instance of a Parser - /// - /// # Arguments - /// * `schema` - The [Schema] from the ETW Event we want to parse - /// - /// # Example - /// ```rust - /// let my_callback = |record: EventRecord, schema_locator: &mut SchemaLocator| { - /// let schema = schema_locator.event_schema(record)?; - /// let parser = Parse::create(&schema); - /// }; - /// ``` - pub fn create(event: &'a TypedEvent) -> Self { - Parser { - event, - buffer: event.user_buffer(), - properties: event.schema.properties(), - last_property: 0, - offset: 0, - cache: Vec::new(), // We could fill the cache on creation - } - } - /* - #[allow(dead_code)] - fn fill_cache( - schema: &TypedEvent, - properties: &PropertyIter, - ) -> ParserResult> { - let user_buffer_len = schema.user_buffer().len(); - let mut prop_offset = 0; - panic!(); - Ok(properties.properties_iter().iter().try_fold( - HashMap::new(), - |mut cache, x| -> ParserResult> { - let prop_size = tdh::property_size(schema.record(), &x.name)? as usize; - - if user_buffer_len < prop_size { - return Err(ParserError::PropertyError( - "Property length out of buffer bounds".to_owned(), - )); - } - let prop_buffer = schema.user_buffer()[..prop_size] - .iter() - .take(prop_size) - .cloned() - .collect(); - - cache.insert(x.name.clone(), PropertyInfo::create(x.clone(), prop_offset, prop_buffer)); - prop_offset += prop_size; - - Ok(cache) - }, - )?) - }*/ - - // TODO: Find a cleaner way to do this, not very happy with it rn - fn find_property_size(&self, property: &Property) -> ParserResult { - match property.length { - PropertyLength::Index(_) => { - // e.g. Microsoft-Windows-Kernel-Power/SystemTimerResolutionStackRundown uses the AppNameLength property - // as the size of AppName - - // Fallback to Tdh - return Ok( - tdh::property_size(self.event.record(), &property.name).unwrap() as usize, - ); - } - PropertyLength::Length(length) => { - // TODO: Study heuristic method used in krabsetw :) - if property.flags.is_empty() && length > 0 && property.count == 1 { - return Ok(length as usize); - } - if property.count == 1 { - if let PropertyDesc::Primitive(desc) = &property.desc { - match desc.in_type { - TdhInType::InTypeBoolean => return Ok(4), - TdhInType::InTypeInt32 - | TdhInType::InTypeUInt32 - | TdhInType::InTypeHexInt32 => return Ok(4), - TdhInType::InTypeInt64 - | TdhInType::InTypeUInt64 - | TdhInType::InTypeHexInt64 => return Ok(8), - TdhInType::InTypeInt8 | TdhInType::InTypeUInt8 => return Ok(1), - TdhInType::InTypeInt16 | TdhInType::InTypeUInt16 => return Ok(2), - TdhInType::InTypePointer => { - return Ok( - if (self.event.event_flags() & EVENT_HEADER_FLAG_32_BIT_HEADER) - != 0 - { - 4 - } else { - 8 - }, - ) - } - TdhInType::InTypeGuid => return Ok(std::mem::size_of::()), - TdhInType::InTypeUnicodeString => { - return Ok(utils::parse_unk_size_null_unicode_size(&self.buffer)) - } - TdhInType::InTypeAnsiString => { - return Ok(utils::parse_unk_size_null_ansi_size(&self.buffer)); - } - _ => {} - } - } - } - return Ok( - tdh::property_size(self.event.record(), &property.name).unwrap() as usize, - ); - } - } - } - - pub fn find_property(&mut self, name: &str) -> ParserResult { - let indx = *self - .properties - .name_to_indx - .get(name) - .ok_or_else(|| ParserError::PropertyError("Unknown property".to_owned()))?; - if indx < self.cache.len() { - return Ok(indx); - } - - // TODO: Find a way to do this with an iter, try_find looks promising but is not stable yet - // TODO: Clean this a bit, not a big fan of this loop - for i in self.cache.len()..=indx { - let curr_prop = self.properties.property(i).unwrap(); - - let prop_size = self.find_property_size(&curr_prop)?; - - if self.buffer.len() < prop_size { - return Err(ParserError::PropertyError(format!( - "Property of {} bytes out of buffer bounds ({})", - prop_size, - self.buffer.len() - ))); - } - - // We split the buffer, if everything works correctly in the end the buffer will be empty - // and we should have all properties in the cache - let (prop_buffer, remaining) = self.buffer.split_at(prop_size); - self.buffer = remaining; - self.cache - .push(PropertyInfo::create(curr_prop, self.offset, prop_buffer)); - self.offset += prop_size; - } - Ok(indx) - } -} - -/* -impl<'a> std::fmt::Debug for Parser<'a> { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let mut s = f.debug_struct("ParsedEvent"); - for i in 0..self.event.property_count() { - let property = self.event.property(i); - let value = match property.in_type() { - TdhInType::InTypeUnicodeString => format!("{}", TryParse::::parse(self, &property.name)), - TdhInType::InTypeAnsiString => format!("{}", TryParse::::parse(self, &property.name)), - TdhInType::InTypeUInt32 => format!("{}", TryParse::::parse(self, &property.name)), - TdhInType::InTypeUInt8 => format!("{}", TryParse::::parse(self, &property.name)), - TdhInType::InTypePointer => format!("{}", TryParse::::parse(self, &property.name)), - TdhInType::InTypeInt64 => format!("{}", TryParse::::parse(self, &property.name)), - TdhInType::InTypeUInt64 => format!("{}", TryParse::::parse(self, &property.name)), - TdhInType::InTypeGuid => format!("{:?}", TryParse::::parse(self, &property.name)), - _ => panic!() - }; - s.field(&property.name, &value); - //dbg!(&property); - } - s.finish() - } -}*/ - -macro_rules! impl_try_parse_primitive { - ($T:ident, $ty:ident) => { - impl TryParse<$T> for Parser<'_> { - fn try_parse(&mut self, name: &str) -> ParserResult<$T> { - use TdhInType::*; - let indx = self.find_property(name)?; - let prop_info = &self.cache[indx]; - let prop_info: &PropertyInfo = prop_info.borrow(); - if let PropertyDesc::Primitive(desc) = &prop_info.property.desc { - if desc.in_type != $ty { - return Err(ParserError::InvalidType); - } - if std::mem::size_of::<$T>() != prop_info.buffer.len() { - return Err(ParserError::LengthMismatch); - } - return Ok($T::from_ne_bytes(prop_info.buffer.try_into()?)); - }; - Err(ParserError::InvalidType) - } - } - }; -} - -impl_try_parse_primitive!(u8, InTypeUInt8); -impl_try_parse_primitive!(i8, InTypeInt8); -impl_try_parse_primitive!(u16, InTypeUInt16); -impl_try_parse_primitive!(i16, InTypeInt16); -impl_try_parse_primitive!(u32, InTypeUInt32); -//impl_try_parse_primitive!(u64, InTypeUInt64); -//impl_try_parse_primitive!(i64, InTypeInt64); - -impl TryParse for Parser<'_> { - fn try_parse(&mut self, name: &str) -> ParserResult { - use TdhInType::*; - let indx = self.find_property(name)?; - let prop_info = &self.cache[indx]; - if let PropertyDesc::Primitive(desc) = &prop_info.property.desc { - if desc.in_type == InTypeUInt64 { - if std::mem::size_of::() != prop_info.buffer.len() { - return Err(ParserError::LengthMismatch); - } - return Ok(u64::from_ne_bytes(prop_info.buffer.try_into()?)); - } - if desc.in_type == InTypePointer || desc.in_type == InTypeSizeT { - if (self.event.event_flags() & EVENT_HEADER_FLAG_32_BIT_HEADER) != 0 { - if std::mem::size_of::() != prop_info.buffer.len() { - return Err(ParserError::LengthMismatch); - } - return Ok(u32::from_ne_bytes(prop_info.buffer.try_into()?) as u64); - } - if std::mem::size_of::() != prop_info.buffer.len() { - return Err(ParserError::LengthMismatch); - } - return Ok(u64::from_ne_bytes(prop_info.buffer.try_into()?)); - } - } - return Err(ParserError::InvalidType); - } -} - -impl TryParse for Parser<'_> { - fn try_parse(&mut self, name: &str) -> ParserResult { - use TdhInType::*; - let indx = self.find_property(name)?; - let prop_info = &self.cache[indx]; - if let PropertyDesc::Primitive(desc) = &prop_info.property.desc { - if desc.in_type == InTypeInt64 || desc.in_type == InTypeHexInt64 { - if std::mem::size_of::() != prop_info.buffer.len() { - return Err(ParserError::LengthMismatch); - } - return Ok(i64::from_ne_bytes(prop_info.buffer.try_into()?)); - } - } - return Err(ParserError::InvalidType); - } -} - -impl TryParse for Parser<'_> { - fn try_parse(&mut self, name: &str) -> ParserResult { - use TdhInType::*; - let indx = self.find_property(name)?; - let prop_info = &self.cache[indx]; - if let PropertyDesc::Primitive(desc) = &prop_info.property.desc { - if desc.in_type == InTypeInt32 || desc.in_type == InTypeHexInt32 { - if std::mem::size_of::() != prop_info.buffer.len() { - return Err(ParserError::LengthMismatch); - } - return Ok(i32::from_ne_bytes(prop_info.buffer.try_into()?)); - } - } - return Err(ParserError::InvalidType); - } -} - -impl TryParse
for Parser<'_> { - fn try_parse(&mut self, name: &str) -> ParserResult
{ - use TdhInType::*; - let indx = self.find_property(name)?; - let prop_info = &self.cache[indx]; - - if let PropertyDesc::Primitive(desc) = &prop_info.property.desc { - if self.event.is_64bit() { - if desc.in_type == InTypeUInt64 - || desc.in_type == InTypePointer - || desc.in_type == InTypeHexInt64 - { - if std::mem::size_of::() != prop_info.buffer.len() { - return Err(ParserError::LengthMismatch); - } - return Ok(Address::Address64(u64::from_ne_bytes( - prop_info.buffer.try_into()?, - ))); - } - } else { - if desc.in_type == InTypeUInt32 - || desc.in_type == InTypePointer - || desc.in_type == InTypeHexInt32 - { - if std::mem::size_of::() != prop_info.buffer.len() { - return Err(ParserError::LengthMismatch); - } - return Ok(Address::Address32(u32::from_ne_bytes( - prop_info.buffer.try_into()?, - ))); - } - } - } - return Err(ParserError::InvalidType); - } -} - -impl TryParse for Parser<'_> { - fn try_parse(&mut self, name: &str) -> ParserResult { - use TdhInType::*; - let indx = self.find_property(name)?; - let prop_info = &self.cache[indx]; - if let PropertyDesc::Primitive(desc) = &prop_info.property.desc { - if desc.in_type != InTypeBoolean { - return Err(ParserError::InvalidType); - } - if prop_info.buffer.len() != 4 { - return Err(ParserError::LengthMismatch); - } - return match u32::from_ne_bytes(prop_info.buffer.try_into()?) { - 1 => Ok(true), - 0 => Ok(false), - _ => Err(ParserError::InvalidType), - }; - }; - Err(ParserError::InvalidType) - } -} - -impl TryParse for Parser<'_> { - fn try_parse(&mut self, name: &str) -> ParserResult { - use TdhInType::*; - let indx = self.find_property(name)?; - let prop_info = &self.cache[indx]; - if let PropertyDesc::Primitive(desc) = &prop_info.property.desc { - if desc.in_type == InTypeFloat { - if std::mem::size_of::() != prop_info.buffer.len() { - return Err(ParserError::LengthMismatch); - } - return Ok(f32::from_ne_bytes(prop_info.buffer.try_into()?)); - } - } - return Err(ParserError::InvalidType); - } -} - -/// The `String` impl of the `TryParse` trait should be used to retrieve the following [TdhInTypes]: -/// -/// * InTypeUnicodeString -/// * InTypeAnsiString -/// * InTypeCountedString -/// * InTypeGuid -/// -/// On success a `String` with the with the data from the `name` property will be returned -/// -/// # Arguments -/// * `name` - Name of the property to be found in the Schema - -/// # Example -/// ```rust -/// let my_callback = |record: EventRecord, schema_locator: &mut SchemaLocator| { -/// let schema = schema_locator.event_schema(record)?; -/// let parser = Parse::create(&schema); -/// let image_name: String = parser.try_parse("ImageName")?; -/// }; -/// ``` -/// -/// [TdhInTypes]: TdhInType -impl TryParse for Parser<'_> { - fn try_parse(&mut self, name: &str) -> ParserResult { - let indx = self.find_property(name)?; - let prop_info = &self.cache[indx]; - - // TODO: Handle errors and type checking better - if let PropertyDesc::Primitive(desc) = &prop_info.property.desc { - let res = match desc.in_type { - TdhInType::InTypeUnicodeString => utils::parse_null_utf16_string(prop_info.buffer), - TdhInType::InTypeAnsiString => String::from_utf8(prop_info.buffer.to_vec())? - .trim_matches(char::default()) - .to_string(), - TdhInType::InTypeSid => { - panic!() - //sddl::convert_sid_to_string(prop_info.buffer.as_ptr() as isize)? - } - TdhInType::InTypeCountedString => unimplemented!(), - _ => return Err(ParserError::InvalidType), - }; - return Ok(res); - } - Err(ParserError::InvalidType) - } -} - -impl TryParse for Parser<'_> { - fn try_parse(&mut self, name: &str) -> Result { - let indx = self.find_property(name)?; - let prop_info = &self.cache[indx]; - if let PropertyDesc::Primitive(desc) = &prop_info.property.desc { - match desc.in_type { - TdhInType::InTypeUnicodeString => { - let guid_string = utils::parse_utf16_guid(prop_info.buffer); - - if guid_string.len() != 36 { - return Err(ParserError::LengthMismatch); - } - - return Ok(GUID::from(guid_string.as_str())); - } - TdhInType::InTypeGuid => { - return Ok(GUID::from_values( - u32::from_ne_bytes((&prop_info.buffer[0..4]).try_into()?), - u16::from_ne_bytes((&prop_info.buffer[4..6]).try_into()?), - u16::from_ne_bytes((&prop_info.buffer[6..8]).try_into()?), - [ - prop_info.buffer[8], - prop_info.buffer[9], - prop_info.buffer[10], - prop_info.buffer[11], - prop_info.buffer[12], - prop_info.buffer[13], - prop_info.buffer[14], - prop_info.buffer[15], - ], - )) - } - _ => return Err(ParserError::InvalidType), - } - }; - Err(ParserError::InvalidType) - } -} - -impl TryParse for Parser<'_> { - fn try_parse(&mut self, name: &str) -> ParserResult { - let indx = self.find_property(name)?; - let prop_info = &self.cache[indx]; - if let PropertyDesc::Primitive(desc) = &prop_info.property.desc { - if desc.out_type != TdhOutType::OutTypeIpv4 && desc.out_type != TdhOutType::OutTypeIpv6 - { - return Err(ParserError::InvalidType); - } - - // Hardcoded values for now - let res = match prop_info.property.length { - PropertyLength::Length(16) => { - let tmp: [u8; 16] = prop_info.buffer.try_into()?; - IpAddr::V6(Ipv6Addr::from(tmp)) - } - PropertyLength::Length(4) => { - let tmp: [u8; 4] = prop_info.buffer.try_into()?; - IpAddr::V4(Ipv4Addr::from(tmp)) - } - _ => return Err(ParserError::LengthMismatch), - }; - - return Ok(res); - } - Err(ParserError::InvalidType) - } -} - -#[derive(Clone, Default, Debug)] -pub struct Pointer(usize); - -impl std::ops::Deref for Pointer { - type Target = usize; - - fn deref(&self) -> &Self::Target { - &self.0 - } -} - -impl std::ops::DerefMut for Pointer { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.0 - } -} - -impl std::fmt::LowerHex for Pointer { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let val = self.0; - - std::fmt::LowerHex::fmt(&val, f) // delegate to u32/u64 implementation - } -} - -impl std::fmt::UpperHex for Pointer { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let val = self.0; - - std::fmt::UpperHex::fmt(&val, f) // delegate to u32/u64 implementation - } -} - -impl std::fmt::Display for Pointer { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let val = self.0; - - std::fmt::Display::fmt(&val, f) // delegate to u32/u64 implementation - } -} - -impl TryParse for Parser<'_> { - fn try_parse(&mut self, name: &str) -> ParserResult { - let indx = self.find_property(name)?; - let prop_info = &self.cache[indx]; - - let mut res = Pointer::default(); - if prop_info.buffer.len() == std::mem::size_of::() { - res.0 = TryParse::::try_parse(self, name)? as usize; - } else { - res.0 = TryParse::::try_parse(self, name)? as usize; - } - - Ok(res) - } -} - -impl TryParse> for Parser<'_> { - fn try_parse(&mut self, name: &str) -> Result, ParserError> { - let indx = self.find_property(name)?; - let prop_info = &self.cache[indx]; - - Ok(prop_info.buffer.to_vec()) - } -} - -// TODO: Implement SocketAddress -// TODO: Study if we can use primitive types for HexInt64, HexInt32 and Pointer diff --git a/samply/src/windows/etw_reader/property.rs b/samply/src/windows/etw_reader/property.rs deleted file mode 100644 index 997f1142..00000000 --- a/samply/src/windows/etw_reader/property.rs +++ /dev/null @@ -1,58 +0,0 @@ -//! ETW Event Property information -//! -//! The `property` module expose the basic structures that represent the Properties an Event contains -//! based on it's Schema. This Properties can then be used to parse accordingly their values. -use super::schema::Schema; -use super::tdh_types::Property; -use super::FastHashMap; - -/// Event Property information -#[derive(Clone, Debug)] -pub struct PropertyInfo<'a> { - /// Property attributes - pub property: &'a Property, - pub offset: usize, - /// Buffer with the Property data - pub buffer: &'a [u8], -} - -impl<'a> PropertyInfo<'a> { - pub fn create(property: &'a Property, offset: usize, buffer: &'a [u8]) -> Self { - PropertyInfo { - property, - offset, - buffer, - } - } -} - -pub(crate) struct PropertyIter { - properties: Vec, - pub(crate) name_to_indx: FastHashMap, -} - -impl PropertyIter { - pub fn new(schema: &Schema) -> Self { - let prop_count = schema.event_schema.property_count(); - let mut properties = Vec::new(); - let mut name_to_indx = FastHashMap::default(); - for i in 0..prop_count { - let prop = schema.event_schema.property(i); - name_to_indx.insert(prop.name.clone(), i as usize); - properties.push(prop); - } - - PropertyIter { - properties, - name_to_indx, - } - } - - pub fn property(&self, index: usize) -> Option<&Property> { - self.properties.get(index) - } - - pub fn properties_iter(&self) -> &[Property] { - &self.properties - } -} diff --git a/samply/src/windows/etw_reader/schema.rs b/samply/src/windows/etw_reader/schema.rs deleted file mode 100644 index ea689a47..00000000 --- a/samply/src/windows/etw_reader/schema.rs +++ /dev/null @@ -1,479 +0,0 @@ -//! ETW Event Schema locator and handler -//! -//! This module contains the means needed to locate and interact with the Schema of an ETW event -use std::any::{Any, TypeId}; -use std::collections::hash_map::Entry; -use std::sync::Arc; - -use once_cell::unsync::OnceCell; -use windows::core::GUID; -use windows::Win32::System::Diagnostics::Etw::{self, EVENT_HEADER_FLAG_64_BIT_HEADER}; - -use super::custom_schemas::EventInfo; -use super::etw_types::{DecodingSource, EventRecord, TraceEventInfoRaw}; -use super::property::PropertyIter; -use super::tdh_types::Property; -use super::{tdh, FastHashMap}; - -/// Schema module errors -#[derive(Debug)] -pub enum SchemaError { - /// Represents a Parser error - ParseError, - /// Represents an internal [TdhNativeError] - /// - /// [TdhNativeError]: tdh::TdhNativeError - TdhNativeError(tdh::TdhNativeError), -} - -impl From for SchemaError { - fn from(err: tdh::TdhNativeError) -> Self { - SchemaError::TdhNativeError(err) - } -} - -type SchemaResult = Result; - -// TraceEvent::RegisteredTraceEventParser::ExternalTraceEventParserState::TraceEventComparer -// doesn't compare the version or level and does different things depending on the kind of event -// https://github.com/microsoft/perfview/blob/5c9f6059f54db41b4ac5c4fc8f57261779634489/src/TraceEvent/RegisteredTraceEventParser.cs#L1338 -#[derive(Debug, Eq, PartialEq, Hash)] -struct SchemaKey { - provider: GUID, - id: u16, - version: u8, - level: u8, - opcode: u8, -} - -// A map from tracelogging schema metdata to ids -struct TraceLoggingProviderIds { - ids: FastHashMap, u16>, - next_id: u16, -} - -impl TraceLoggingProviderIds { - fn new() -> Self { - // start the ids at 1 because of 0 is typically the value stored - TraceLoggingProviderIds { - ids: FastHashMap::default(), - next_id: 1, - } - } -} - -impl SchemaKey { - pub fn new(event: &EventRecord, locator: &mut SchemaLocator) -> Self { - // TraceLogging events all use the same id and are distinguished from each other using the metadata. - // Instead of storing the metadata in the SchemaKey we follow the approach of PerfView and have a side table of synthetic ids keyed on metadata. - // It might be better to store the metadata in the SchemaKey but then we may want to be careful not to allocate a fresh metadata for every event. - let mut id = event.EventHeader.EventDescriptor.Id; - if event.ExtendedDataCount > 0 { - let extended = unsafe { - std::slice::from_raw_parts(event.ExtendedData, event.ExtendedDataCount as usize) - }; - for e in extended { - if e.ExtType as u32 == Etw::EVENT_HEADER_EXT_TYPE_EVENT_SCHEMA_TL { - let mut provider = locator - .tracelogging_providers - .entry(event.EventHeader.ProviderId) - .or_insert(TraceLoggingProviderIds::new()); - let data = unsafe { - std::slice::from_raw_parts(e.DataPtr as *const u8, e.DataSize as usize) - }; - if let Some(metadata_id) = provider.ids.get(data) { - // we want to ensure that our synthetic ids don't overlap with any ids used in the events - assert_ne!(id, *metadata_id); - id = *metadata_id; - } else { - provider.ids.insert(data.to_vec(), provider.next_id); - id = provider.next_id; - provider.next_id += 1; - } - } - } - } - SchemaKey { - provider: event.EventHeader.ProviderId, - id, - version: event.EventHeader.EventDescriptor.Version, - level: event.EventHeader.EventDescriptor.Level, - opcode: event.EventHeader.EventDescriptor.Opcode, - } - } -} - -/// Represents a cache of Schemas already located -/// -/// This cache is implemented as a [HashMap] where the key is a combination of the following elements -/// of an [Event Record](https://docs.microsoft.com/en-us/windows/win32/api/evntcons/ns-evntcons-event_record) -/// * EventHeader.ProviderId -/// * EventHeader.EventDescriptor.Id -/// * EventHeader.EventDescriptor.Opcode -/// * EventHeader.EventDescriptor.Version -/// * EventHeader.EventDescriptor.Level -/// -/// Credits: [KrabsETW::schema_locator](https://github.com/microsoft/krabsetw/blob/master/krabs/krabs/schema_locator.hpp) -#[derive(Default)] -pub struct SchemaLocator { - schemas: FastHashMap>, - tracelogging_providers: FastHashMap, -} - -pub trait EventSchema { - fn decoding_source(&self) -> DecodingSource; - - fn provider_guid(&self) -> GUID; - fn event_id(&self) -> u16; - fn opcode(&self) -> u8; - fn event_version(&self) -> u8; - fn provider_name(&self) -> String; - fn task_name(&self) -> String; - fn opcode_name(&self) -> String; - fn level(&self) -> u8; - - fn property_count(&self) -> u32; - fn property(&self, index: u32) -> Property; - - fn event_message(&self) -> Option { - return None; - } - fn is_event_metadata(&self) -> bool { - return false; - } -} - -impl std::fmt::Debug for SchemaLocator { - fn fmt(&self, _f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - todo!() - } -} - -impl SchemaLocator { - pub fn new() -> Self { - SchemaLocator { - schemas: FastHashMap::default(), - tracelogging_providers: FastHashMap::default(), - } - } - - pub fn add_custom_schema(&mut self, schema: Box) { - let key = SchemaKey { - provider: schema.provider_guid(), - id: schema.event_id(), - opcode: schema.opcode(), - version: schema.event_version(), - level: schema.level(), - }; - self.schemas.insert(key, Arc::new(Schema::new(schema))); - } - - /// Use the `event_schema` function to retrieve the Schema of an ETW Event - /// - /// # Arguments - /// * `event` - The [EventRecord] that's passed to the callback - /// - /// # Remark - /// This is the first function that should be called within a Provider callback, if everything - /// works as expected this function will return a Result with the [Schema] that represents - /// the ETW event that triggered the callback - /// - /// This function can fail, if it does it will return a [SchemaError] - /// - /// # Example - /// ```rust - /// let my_callback = |record: EventRecord, schema_locator: &mut SchemaLocator| { - /// let schema = schema_locator.event_schema(record)?; - /// }; - /// ``` - pub fn event_schema<'a>(&mut self, event: &'a EventRecord) -> SchemaResult> { - let key = SchemaKey::new(&event, self); - let info = match self.schemas.entry(key) { - Entry::Occupied(entry) => entry.into_mut(), - Entry::Vacant(entry) => { - let info = Box::new(tdh::schema_from_tdh(event.clone())?); - // dbg!(info.provider_guid(), info.provider_name(), info.decoding_source()); - // TODO: Cloning for now, should be a reference at some point... - entry.insert(Arc::new(Schema::new(info))) - } - } - .clone(); - - // Some events contain schemas so add them when we find them. - if info.event_schema.is_event_metadata() { - let event_info = TraceEventInfoRaw::new(event.user_buffer().to_owned()); - println!( - "Adding custom schema for {}/{}/{}/{}", - event_info.provider_name(), - event_info.event_id(), - event_info.task_name(), - event_info.opcode_name() - ); - self.add_custom_schema(Box::new(event_info)); - } - - Ok(TypedEvent::new(event, info)) - } -} - -pub struct Schema { - pub event_schema: Box, - properties: OnceCell, - name: OnceCell, -} - -impl Schema { - fn new(event_schema: Box) -> Self { - Schema { - event_schema, - properties: OnceCell::new(), - name: OnceCell::new(), - } - } - pub(crate) fn properties(&self) -> &PropertyIter { - self.properties.get_or_init(|| PropertyIter::new(self)) - } - pub(crate) fn name(&self) -> &str { - self.name.get_or_init(|| { - format!( - "{}/{}/{}", - self.event_schema.provider_name(), - self.event_schema.task_name(), - self.event_schema.opcode_name() - ) - }) - } -} - -pub struct TypedEvent<'a> { - record: &'a EventRecord, - pub(crate) schema: Arc, -} - -impl<'a> TypedEvent<'a> { - pub fn new(record: &'a EventRecord, schema: Arc) -> Self { - TypedEvent { record, schema } - } - - pub(crate) fn user_buffer(&self) -> &[u8] { - self.record.user_buffer() - } - - // Horrible getters FTW!! :D - // TODO: Not a big fan of this, think a better way.. - pub(crate) fn record(&self) -> &EventRecord { - self.record - } - - /// Use the `event_id` function to obtain the EventId of the Event Record - /// - /// This getter returns the EventId of the ETW Event that triggered the registered callback - /// - /// # Example - /// ```rust - /// let my_callback = |record: EventRecord, schema_locator: &mut SchemaLocator| { - /// let schema = schema_locator.event_schema(record)?; - /// let event_id = schema.event_id(); - /// }; - /// ``` - pub fn event_id(&self) -> u16 { - self.record.EventHeader.EventDescriptor.Id - } - - /// Use the `opcode` function to obtain the Opcode of the Event Record - /// - /// This getter returns the opcode of the ETW Event that triggered the registered callback - /// - /// # Example - /// ```rust - /// let my_callback = |record: EventRecord, schema_locator: &mut SchemaLocator| { - /// let schema = schema_locator.event_schema(record)?; - /// let event_id = schema.opcode(); - /// }; - /// ``` - pub fn opcode(&self) -> u8 { - self.record.EventHeader.EventDescriptor.Opcode - } - - /// Use the `event_flags` function to obtain the Event Flags of the [EventRecord] - /// - /// This getter returns the Event Flags of the ETW Event that triggered the registered callback - /// - /// # Example - /// ```rust - /// let my_callback = |record: EventRecord, schema_locator: &mut SchemaLocator| { - /// let schema = schema_locator.event_schema(record)?; - /// let event_flags = schema.event_flags(); - /// }; - /// ``` - pub fn event_flags(&self) -> u16 { - self.record.EventHeader.Flags - } - - pub fn is_64bit(&self) -> bool { - (self.record.EventHeader.Flags & EVENT_HEADER_FLAG_64_BIT_HEADER as u16) != 0 - } - - /// Use the `event_version` function to obtain the Version of the [EventRecord] - /// - /// This getter returns the Version of the ETW Event that triggered the registered callback - /// - /// # Example - /// ```rust - /// let my_callback = |record: EventRecord, schema_locator: &mut SchemaLocator| { - /// let schema = schema_locator.event_schema(record)?; - /// let event_version = schema.event_version(); - /// }; - /// ``` - pub fn event_version(&self) -> u8 { - self.record.EventHeader.EventDescriptor.Version - } - - /// Use the `process_id` function to obtain the ProcessId of the [EventRecord] - /// - /// This getter returns the ProcessId of the process that triggered the ETW Event - /// - /// # Example - /// ```rust - /// let my_callback = |record: EventRecord, schema_locator: &mut SchemaLocator| { - /// let schema = schema_locator.event_schema(record)?; - /// let pid = schema.process_id(); - /// }; - /// ``` - pub fn process_id(&self) -> u32 { - self.record.EventHeader.ProcessId - } - - /// Use the `thread_id` function to obtain the ThreadId of the [EventRecord] - /// - /// This getter returns the ThreadId of the thread that triggered the ETW Event - /// - /// # Example - /// ```rust - /// let my_callback = |record: EventRecord, schema_locator: &mut SchemaLocator| { - /// let schema = schema_locator.event_schema(record)?; - /// let tid = schema.thread_id(); - /// }; - /// ``` - pub fn thread_id(&self) -> u32 { - self.record.EventHeader.ThreadId - } - - /// Use the `timestamp` function to obtain the TimeStamp of the [EventRecord] - /// - /// This getter returns the TimeStamp of the ETW Event - /// - /// # Example - /// ```rust - /// let my_callback = |record: EventRecord, schema_locator: &mut SchemaLocator| { - /// let schema = schema_locator.event_schema(record)?; - /// let timestamp = schema.timestamp(); - /// }; - /// ``` - pub fn timestamp(&self) -> i64 { - self.record.EventHeader.TimeStamp - } - - /// Use the `activity_id` function to obtain the ActivityId of the [EventRecord] - /// - /// This getter returns the ActivityId from the ETW Event, this value is used to related Two events - /// - /// # Example - /// ```rust - /// let my_callback = |record: EventRecord, schema_locator: &mut SchemaLocator| { - /// let schema = schema_locator.event_schema(record)?; - /// let activity_id = schema.activity_id(); - /// }; - /// ``` - /// [TraceEventInfo]: super::native::etw_types::TraceEventInfo - pub fn activity_id(&self) -> GUID { - self.record.EventHeader.ActivityId - } - - /// Use the `decoding_source` function to obtain the [DecodingSource] from the [TraceEventInfo] - /// - /// This getter returns the DecodingSource from the event, this value identifies the source used - /// parse the event data - /// - /// # Example - /// ```rust - /// let my_callback = |record: EventRecord, schema_locator: &mut SchemaLocator| { - /// let schema = schema_locator.event_schema(record)?; - /// let decoding_source = schema.decoding_source(); - /// }; - /// ``` - /// [TraceEventInfo]: super::native::etw_types::TraceEventInfo - pub fn decoding_source(&self) -> DecodingSource { - self.schema.event_schema.decoding_source() - } - - /// Use the `provider_name` function to obtain the Provider name from the [TraceEventInfo] - /// - /// # Example - /// ```rust - /// let my_callback = |record: EventRecord, schema_locator: &mut SchemaLocator| { - /// let schema = schema_locator.event_schema(record)?; - /// let provider_name = schema.provider_name(); - /// }; - /// ``` - /// [TraceEventInfo]: super::native::etw_types::TraceEventInfo - pub fn provider_name(&self) -> String { - self.schema.event_schema.provider_name() - } - - /// Use the `task_name` function to obtain the Task name from the [TraceEventInfo] - /// - /// See: [TaskType](https://docs.microsoft.com/en-us/windows/win32/wes/eventmanifestschema-tasktype-complextype) - /// # Example - /// ```rust - /// let my_callback = |record: EventRecord, schema_locator: &mut SchemaLocator| { - /// let schema = schema_locator.event_schema(record)?; - /// let task_name = schema.task_name(); - /// }; - /// ``` - /// [TraceEventInfo]: super::native::etw_types::TraceEventInfo - pub fn task_name(&self) -> String { - self.schema.event_schema.task_name() - } - - /// Use the `opcode_name` function to obtain the Opcode name from the [TraceEventInfo] - /// - /// See: [OpcodeType](https://docs.microsoft.com/en-us/windows/win32/wes/eventmanifestschema-opcodetype-complextype) - /// # Example - /// ```rust - /// let my_callback = |record: EventRecord, schema_locator: &mut SchemaLocator| { - /// let schema = schema_locator.event_schema(record)?; - /// let opcode_name = schema.opcode_name(); - /// }; - /// ``` - /// [TraceEventInfo]: super::native::etw_types::TraceEventInfo - pub fn opcode_name(&self) -> String { - self.schema.event_schema.opcode_name() - } - - pub fn property_count(&self) -> u32 { - self.schema.event_schema.property_count() - } - - pub fn property(&self, index: u32) -> Property { - self.schema.event_schema.property(index) - } - - pub fn name(&self) -> &str { - self.schema.name() - } - - pub fn event_message(&self) -> Option { - self.schema.event_schema.event_message() - } -} - -impl<'a> PartialEq for TypedEvent<'a> { - fn eq(&self, other: &Self) -> bool { - self.schema.event_schema.event_id() == other.schema.event_schema.event_id() - && self.schema.event_schema.provider_guid() == other.schema.event_schema.provider_guid() - && self.schema.event_schema.event_version() == other.schema.event_schema.event_version() - } -} - -impl<'a> Eq for TypedEvent<'a> {} diff --git a/samply/src/windows/etw_reader/sddl.rs b/samply/src/windows/etw_reader/sddl.rs deleted file mode 100644 index cd2a7a2a..00000000 --- a/samply/src/windows/etw_reader/sddl.rs +++ /dev/null @@ -1,66 +0,0 @@ -//use super::traits::*; -use std::str::Utf8Error; - -use windows::core::PSTR; -use windows::Win32::Foundation::{LocalFree, HLOCAL, PSID}; -use windows::Win32::Security; - -/// SDDL native error -#[derive(Debug)] -pub enum SddlNativeError { - /// Represents an error parsing the SID into a String - SidParseError(Utf8Error), - /// Represents an standard IO Error - IoError(std::io::Error), -} - -//impl LastOsError for SddlNativeError {} - -impl From for SddlNativeError { - fn from(err: std::io::Error) -> Self { - SddlNativeError::IoError(err) - } -} - -impl From for SddlNativeError { - fn from(err: Utf8Error) -> Self { - SddlNativeError::SidParseError(err) - } -} - -pub(crate) type SddlResult = Result; - -pub fn convert_sid_to_string(sid: *const u8) -> SddlResult { - let mut tmp = PSTR::null(); - unsafe { - if !Security::Authorization::ConvertSidToStringSidA( - PSID(sid as *const _ as *mut _), - &mut tmp, - ) - .is_ok() - { - return Err(SddlNativeError::IoError(std::io::Error::last_os_error())); - } - - let sid_string = std::ffi::CStr::from_ptr(tmp.0 as *mut _) - .to_str()? - .to_owned(); - - let _ = LocalFree(HLOCAL(tmp.0 as *mut _)); - - Ok(sid_string) - } -} - -#[cfg(test)] -mod test { - use super::*; - - #[test] - fn test_convert_string_to_sid() { - let sid: Vec = vec![1, 2, 0, 0, 0, 0, 0, 5, 0x20, 0, 0, 0, 0x20, 2, 0, 0]; - if let Ok(string_sid) = convert_sid_to_string(sid.as_ptr()) { - assert_eq!(string_sid, "S-1-5-32-544"); - } - } -} diff --git a/samply/src/windows/etw_reader/tdh.rs b/samply/src/windows/etw_reader/tdh.rs deleted file mode 100644 index f2f6af7b..00000000 --- a/samply/src/windows/etw_reader/tdh.rs +++ /dev/null @@ -1,137 +0,0 @@ -use std::ffi::OsString; -use std::ops::Deref; -use std::os::windows::ffi::OsStringExt; -use std::ptr; - -use windows::core::HRESULT; -use windows::Win32::Foundation::{ERROR_INSUFFICIENT_BUFFER, ERROR_SUCCESS, S_OK}; -use windows::Win32::System::Diagnostics::Etw; -use windows::Win32::System::Diagnostics::Etw::{TdhEnumerateProviders, PROVIDER_ENUMERATION_INFO}; - -use super::etw_types::*; -use super::traits::*; - -#[derive(Debug)] -pub enum TdhNativeError { - /// Represents an standard IO Error - IoError(std::io::Error), -} - -impl From for TdhNativeError { - fn from(err: std::io::Error) -> Self { - TdhNativeError::IoError(err) - } -} - -pub(crate) type TdhNativeResult = Result; - -pub fn schema_from_tdh(event: &Etw::EVENT_RECORD) -> TdhNativeResult { - let mut buffer_size = 0; - unsafe { - if Etw::TdhGetEventInformation(event, None, None, &mut buffer_size) - != ERROR_INSUFFICIENT_BUFFER.0 - { - return Err(TdhNativeError::IoError(std::io::Error::last_os_error())); - } - - let mut buffer = TraceEventInfoRaw::alloc(buffer_size); - if Etw::TdhGetEventInformation( - event, - None, - Some(buffer.info_as_ptr() as *mut _), - &mut buffer_size, - ) != 0 - { - return Err(TdhNativeError::IoError(std::io::Error::last_os_error())); - } - - Ok(buffer) - } -} - -pub(crate) fn property_size(event: &EventRecord, name: &str) -> TdhNativeResult { - let mut property_size = 0; - - let mut desc = Etw::PROPERTY_DATA_DESCRIPTOR::default(); - desc.ArrayIndex = u32::MAX; - let utf16_name = name.as_utf16(); - desc.PropertyName = utf16_name.as_ptr() as u64; - - unsafe { - let status = Etw::TdhGetPropertySize(event.deref(), None, &[desc], &mut property_size); - if status != 0 { - return Err(TdhNativeError::IoError(std::io::Error::from_raw_os_error( - status as i32, - ))); - } - } - - Ok(property_size) -} - -pub fn list_etw_providers() { - let mut buffer_size: u32 = 0; - let mut status: u32; - - // Query required buffer size - unsafe { - status = TdhEnumerateProviders(None, &mut buffer_size); - } - if status == ERROR_INSUFFICIENT_BUFFER.0 { - let mut provider_info = vec![0u8; buffer_size as usize]; - let mut buffer_size_copied = buffer_size; - - // Retrieve provider information - unsafe { - status = TdhEnumerateProviders( - Some(provider_info.as_mut_ptr() as *mut PROVIDER_ENUMERATION_INFO), - &mut buffer_size_copied, - ); - } - if status == ERROR_SUCCESS.0 { - let provider_info = - unsafe { &*(provider_info.as_ptr() as *const PROVIDER_ENUMERATION_INFO) }; - let provider_info_array = provider_info.TraceProviderInfoArray.as_ptr(); - - for i in 0..provider_info.NumberOfProviders { - // windows-rs defines TraceProviderInfoArray as a fixed size array of 1 so we need to use get_unchecked to get the other things - let provider_name_offset = - unsafe { *provider_info_array.offset(i as isize) }.ProviderNameOffset as usize; - let provider_name_ptr = provider_info as *const PROVIDER_ENUMERATION_INFO as usize - + provider_name_offset; - // Find the length of the null-terminated string - let mut len = 0; - while unsafe { *(provider_name_ptr as *const u16).add(len) } != 0 { - len += 1; - } - let provider_name = unsafe { - OsString::from_wide(std::slice::from_raw_parts( - provider_name_ptr as *const u16, - len, - )) - .into_string() - .unwrap_or_else(|_| "Error converting to string".to_string()) - }; - - let provider_guid = - &unsafe { *provider_info_array.offset(i as isize) }.ProviderGuid; - let schema_source = unsafe { *provider_info_array.offset(i as isize) }.SchemaSource; - - println!( - " {:?} - {} - {}", - provider_guid, - provider_name, - if schema_source == 0 { - "XML manifest" - } else { - "MOF" - } - ); - } - } else { - println!("TdhEnumerateProviders failed with error code {:?}", status); - } - } else { - println!("TdhEnumerateProviders failed with error code {:?}", status); - } -} diff --git a/samply/src/windows/etw_reader/tdh_types.rs b/samply/src/windows/etw_reader/tdh_types.rs deleted file mode 100644 index aa78063f..00000000 --- a/samply/src/windows/etw_reader/tdh_types.rs +++ /dev/null @@ -1,224 +0,0 @@ -//! Basic TDH types -//! -//! The `tdh_type` module provides an abstraction over the basic TDH types, this module act as a -//! helper for the parser to determine which IN and OUT type are expected from a property within an -//! event -//! -//! This is a bit extra but is basically a redefinition of the In an Out TDH types following the -//! rust naming convention, it can also come in handy when implementing the [TryParse] trait for a type -//! to determine how to handle a [Property] based on this values -//! -//! [TryParse]: super::parser::TryParse -//! [Property]: super::native::tdh_types::Property -use std::rc::Rc; - -use bitflags::bitflags; -use num_derive::{FromPrimitive, ToPrimitive}; -use num_traits::FromPrimitive; -use windows::Win32::System::Diagnostics::Etw; - -use super::etw_types::EventPropertyInfo; - -#[derive(Debug, Clone, Default)] -pub struct PropertyMapInfo { - pub is_bitmap: bool, - pub map: super::FastHashMap, -} -#[derive(Debug, Clone)] -pub struct PrimitiveDesc { - pub in_type: TdhInType, - pub out_type: TdhOutType, -} - -#[derive(Debug, Clone, Default)] -pub struct StructDesc { - pub start_index: u16, - pub num_members: u16, -} - -#[derive(Debug, Clone)] -pub enum PropertyDesc { - Primitive(PrimitiveDesc), - Struct(StructDesc), -} - -/// Notes if the property length is a concrete length or an index to another property -/// which contains the length. -#[derive(Debug, Clone, Copy, PartialEq)] -pub enum PropertyLength { - Length(u16), - Index(u16), -} - -/// Attributes of a property -#[derive(Debug, Clone)] -pub struct Property { - /// Name of the Property - pub name: String, - /// Represent the [PropertyFlags] - pub flags: PropertyFlags, - pub length: PropertyLength, - pub desc: PropertyDesc, - pub map_info: Option>, - pub count: u16, -} - -#[doc(hidden)] -impl Property { - pub fn new( - name: String, - property: &EventPropertyInfo, - map_info: Option>, - ) -> Self { - let flags = PropertyFlags::from(property.Flags); - let length = if flags.contains(PropertyFlags::PROPERTY_PARAM_LENGTH) { - // The property length is stored in another property, this is the index of that property - PropertyLength::Index(unsafe { property.Anonymous3.lengthPropertyIndex }) - } else { - // The property has no param for its length, it makes sense to access this field of the union - PropertyLength::Length(unsafe { property.Anonymous3.length }) - }; - if property.Flags.0 & Etw::PropertyStruct.0 != 0 { - unsafe { - let start_index = property.Anonymous1.structType.StructStartIndex; - let num_members = property.Anonymous1.structType.NumOfStructMembers; - Property { - name, - flags: PropertyFlags::from(property.Flags), - length, - desc: PropertyDesc::Struct(StructDesc { - start_index, - num_members, - }), - map_info, - count: property.Anonymous2.count, - } - } - } else { - unsafe { - let out_type = FromPrimitive::from_u16(property.Anonymous1.nonStructType.OutType) - .unwrap_or(TdhOutType::OutTypeNull); - let in_type = FromPrimitive::from_u16(property.Anonymous1.nonStructType.InType) - .unwrap_or_else(|| panic!("{:?}", property.Anonymous1.nonStructType.InType)); - - Property { - name, - flags: PropertyFlags::from(property.Flags), - length, - desc: PropertyDesc::Primitive(PrimitiveDesc { in_type, out_type }), - map_info, - count: property.Anonymous2.count, - } - } - } - } -} - -/// Represent a TDH_IN_TYPE -#[repr(u16)] -#[derive(Debug, Clone, Copy, FromPrimitive, ToPrimitive, PartialEq)] -pub enum TdhInType { - // Deprecated values are not defined - InTypeNull, - InTypeUnicodeString, - InTypeAnsiString, - InTypeInt8, // Field size is 1 byte - InTypeUInt8, // Field size is 1 byte - InTypeInt16, // Field size is 2 bytes - InTypeUInt16, // Field size is 2 bytes - InTypeInt32, // Field size is 4 bytes - InTypeUInt32, // Field size is 4 bytes - InTypeInt64, // Field size is 8 bytes - InTypeUInt64, // Field size is 8 bytes - InTypeFloat, // Field size is 4 bytes - InTypeDouble, // Field size is 8 bytes - InTypeBoolean, // Field size is 4 bytes - InTypeBinary, // Depends on the OutType - InTypeGuid, - InTypePointer, - InTypeFileTime, // Field size is 8 bytes - InTypeSystemTime, // Field size is 16 bytes - InTypeSid, // Field size determined by the first few bytes of the field - InTypeHexInt32, - InTypeHexInt64, - InTypeCountedString = 300, - InTypeCountedAnsiString, - InTypeReverseCountedString, - InTypeReverseCountedAnsiString, - InTypeNonNullTerminatedString, - InTypeNonNullTerminatedAnsiString, - InTypeUnicodeChar, - InTypeAnsiChar, - InTypeSizeT, - InTypeHexdump, - InTypeWBEMSID, -} - -/// Represent a TDH_OUT_TYPE -#[repr(u16)] -#[derive(Debug, Clone, Copy, FromPrimitive, ToPrimitive, PartialEq)] -pub enum TdhOutType { - OutTypeNull, - OutTypeString, - OutTypeDateTime, - OutTypeInt8, // Field size is 1 byte - OutTypeUInt8, // Field size is 1 byte - OutTypeInt16, // Field size is 2 bytes - OutTypeUInt16, // Field size is 2 bytes - OutTypeInt32, // Field size is 4 bytes - OutTypeUInt32, // Field size is 4 bytes - OutTypeInt64, // Field size is 8 bytes - OutTypeUInt64, // Field size is 8 bytes - OutTypeFloat, // Field size is 4 bytes - OutTypeDouble, // Field size is 8 bytes - OutTypeBoolean, // Field size is 4 bytes - OutTypeGuid, - OutTypeHexBinary, - OutTypeHexInt8, - OutTypeHexInt16, - OutTypeHexInt32, - OutTypeHexInt64, - OutTypePid, - OutTypeTid, - OutTypePort, - OutTypeIpv4, - OutTypeIpv6, - OutTypeWin32Error = 30, - OutTypeNtStatus = 31, - OutTypeHResult = 32, - OutTypeJson = 34, - OutTypeUtf8 = 35, - OutTypePkcs7 = 36, - OutTypeCodePointer = 37, - OutTypeDatetimeUtc = 38, -} - -impl Default for TdhOutType { - fn default() -> TdhOutType { - TdhOutType::OutTypeNull - } -} - -bitflags! { - /// Represents the Property flags - /// - /// See: [Property Flags enum](https://docs.microsoft.com/en-us/windows/win32/api/tdh/ne-tdh-property_flags) - #[derive(Default, Debug, Clone)] - pub struct PropertyFlags: u32 { - const PROPERTY_STRUCT = 0x1; - const PROPERTY_PARAM_LENGTH = 0x2; - const PROPERTY_PARAM_COUNT = 0x4; - const PROPERTY_WBEMXML_FRAGMENT = 0x8; - const PROPERTY_PARAM_FIXED_LENGTH = 0x10; - const PROPERTY_PARAM_FIXED_COUNT = 0x20; - const PROPERTY_HAS_TAGS = 0x40; - const PROPERTY_HAS_CUSTOM_SCHEMA = 0x80; - } -} - -impl From for PropertyFlags { - fn from(flags: Etw::PROPERTY_FLAGS) -> Self { - // Should be a safe cast - PropertyFlags::from_bits_truncate(flags.0 as u32) - } -} diff --git a/samply/src/windows/etw_reader/traits.rs b/samply/src/windows/etw_reader/traits.rs deleted file mode 100644 index 9a240d6f..00000000 --- a/samply/src/windows/etw_reader/traits.rs +++ /dev/null @@ -1,19 +0,0 @@ -use std::iter; - -pub trait EncodeUtf16 { - fn as_utf16(self: Self) -> Vec; -} - -impl EncodeUtf16 for &str { - fn as_utf16(self: Self) -> Vec { - self.encode_utf16() // Make a UTF-16 iterator - .chain(iter::once(0)) // Append a null - .collect() // Collect the iterator into a vector - } -} - -impl EncodeUtf16 for String { - fn as_utf16(self: Self) -> Vec { - self.as_str().as_utf16() - } -} diff --git a/samply/src/windows/etw_reader/utils.rs b/samply/src/windows/etw_reader/utils.rs deleted file mode 100644 index 8d96e8a9..00000000 --- a/samply/src/windows/etw_reader/utils.rs +++ /dev/null @@ -1,92 +0,0 @@ -fn is_aligned(ptr: *const T) -> bool -where - T: Sized, -{ - ptr as usize & (std::mem::align_of::() - 1) == 0 -} - -pub fn parse_unk_size_null_utf16_string(v: &[u8]) -> String { - // Instead of doing the following unsafe stuff ourselves we could - // use cast_slice from bytemuck. Unfortunately, it won't work if - // the length of the u8 is not a multiple of 2 vs. just truncating. - // Alternatively, using safe_transmute::transmute_many_permisive should work. - - let start: *const u16 = v.as_ptr().cast(); - if !is_aligned(start) { - panic!("Not aligned"); - } - - // safe because we not going past the end of the slice - let end: *const u16 = unsafe { v.as_ptr().offset(v.len() as isize) }.cast(); - - // find the null termination - let mut len = 0; - let mut ptr = start; - while unsafe { *ptr } != 0 && ptr < end { - len += 1; - ptr = unsafe { ptr.offset(1) }; - } - - let slice = unsafe { std::slice::from_raw_parts(start, len) }; - String::from_utf16_lossy(slice) -} - -pub fn parse_unk_size_null_unicode_size(v: &[u8]) -> usize { - // TODO: Make sure is aligned - v.chunks_exact(2) - .into_iter() - .take_while(|&a| a != &[0, 0]) // Take until null terminator - .map(|a| u16::from_ne_bytes([a[0], a[1]])) - .count() - * 2 - + 2 -} - -pub fn parse_unk_size_null_unicode_vec(v: &[u8]) -> Vec { - // TODO: Make sure is aligned - v.chunks_exact(2) - .into_iter() - .take_while(|&a| a != &[0, 0]) // Take until null terminator - .map(|a| u16::from_ne_bytes([a[0], a[1]])) - .collect::>() -} - -pub fn parse_unk_size_null_ansi_size(v: &[u8]) -> usize { - v.into_iter() - .take_while(|&&a| a != 0) // Take until null terminator - .count() - + 1 -} - -pub fn parse_unk_size_null_ansi_vec(v: &[u8]) -> Vec { - v.into_iter() - .take_while(|&&a| a != 0) // Take until null terminator - .map(|&a| a) - .collect::>() -} - -pub fn parse_null_utf16_string(v: &[u8]) -> String { - String::from_utf16_lossy( - v.chunks_exact(2) - .into_iter() - .map(|a| u16::from_ne_bytes([a[0], a[1]])) - .collect::>() - .as_slice(), - ) - .trim_matches(char::default()) - .to_string() -} - -pub fn parse_utf16_guid(v: &[u8]) -> String { - String::from_utf16_lossy( - v.chunks_exact(2) - .into_iter() - .map(|a| u16::from_ne_bytes([a[0], a[1]])) - .collect::>() - .as_slice(), - ) - .trim_matches(char::default()) - .trim_matches('{') - .trim_matches('}') - .to_string() -} From 73cc6c14e8b31f28adc60f67b79cc6285e588a84 Mon Sep 17 00:00:00 2001 From: Vladimir Vukicevic Date: Sat, 4 May 2024 13:04:44 -0700 Subject: [PATCH 05/10] cargo fmt --- etw-reader/Cargo.toml | 2 +- etw-reader/examples/dump.rs | 354 ++++++++++--------- etw-reader/examples/event-types.rs | 115 ++++--- etw-reader/examples/list-processes.rs | 58 ++-- etw-reader/examples/list-providers.rs | 12 +- etw-reader/examples/list-threads.rs | 172 +++++----- etw-reader/examples/log-js-stacks.rs | 420 ++++++++++++----------- etw-reader/examples/log-stacks.rs | 466 +++++++++++++------------- etw-reader/examples/record.rs | 78 +++-- etw-reader/src/etw_types.rs | 2 +- etw-reader/src/lib.rs | 294 +++++++++++----- 11 files changed, 1115 insertions(+), 858 deletions(-) diff --git a/etw-reader/Cargo.toml b/etw-reader/Cargo.toml index 85c89efc..6635ca1a 100644 --- a/etw-reader/Cargo.toml +++ b/etw-reader/Cargo.toml @@ -11,7 +11,7 @@ category = "os::windows-apis" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -bitflags = "1.2.1" +bitflags = "2.4.2" num-traits = "0.2" num-derive = "0.3" once_cell = "1.8.0" diff --git a/etw-reader/examples/dump.rs b/etw-reader/examples/dump.rs index fc3dde23..b5ac0390 100644 --- a/etw-reader/examples/dump.rs +++ b/etw-reader/examples/dump.rs @@ -1,181 +1,221 @@ -use etw_reader::{open_trace, parser::{Parser, TryParse}, print_property, schema::SchemaLocator, GUID}; -use windows::Win32::System::{Diagnostics::Etw::{self, EtwProviderTraitDecodeGuid, EtwProviderTraitTypeGroup, ETW_PROVIDER_TRAIT_TYPE}, SystemInformation::PRODUCT_CORE_COUNTRYSPECIFIC}; -use std::{path::Path, collections::HashMap}; - - +use etw_reader::{ + open_trace, + parser::{Parser, TryParse}, + print_property, + schema::SchemaLocator, + GUID, +}; +use std::{collections::HashMap, path::Path}; +use windows::Win32::System::{ + Diagnostics::Etw::{ + self, EtwProviderTraitDecodeGuid, EtwProviderTraitTypeGroup, ETW_PROVIDER_TRAIT_TYPE, + }, + SystemInformation::PRODUCT_CORE_COUNTRYSPECIFIC, +}; fn main() { - let mut schema_locator = SchemaLocator::new(); etw_reader::add_custom_schemas(&mut schema_locator); let pattern = std::env::args().nth(2); let mut processes = HashMap::new(); - open_trace(Path::new(&std::env::args().nth(1).unwrap()), -|e| { - //dbg!(e.EventHeader.TimeStamp); - - - let s = schema_locator.event_schema(e); - if let Ok(s) = s { - - if let "MSNT_SystemTrace/Process/Start" | "MSNT_SystemTrace/Process/DCStart" | "MSNT_SystemTrace/Process/DCEnd" = s.name() { - let mut parser = Parser::create(&s); - - let image_file_name: String = parser.parse("ImageFileName"); - let process_id: u32 = parser.parse("ProcessId"); - processes.insert(process_id, image_file_name); - } - - if let Some(pattern) = &pattern { - if !s.name().contains(pattern) { - return; + open_trace(Path::new(&std::env::args().nth(1).unwrap()), |e| { + //dbg!(e.EventHeader.TimeStamp); + + let s = schema_locator.event_schema(e); + if let Ok(s) = s { + if let "MSNT_SystemTrace/Process/Start" + | "MSNT_SystemTrace/Process/DCStart" + | "MSNT_SystemTrace/Process/DCEnd" = s.name() + { + let mut parser = Parser::create(&s); + + let image_file_name: String = parser.parse("ImageFileName"); + let process_id: u32 = parser.parse("ProcessId"); + processes.insert(process_id, image_file_name); } - } - println!("{:?} {} {} {}-{} {} {}", e.EventHeader.ProviderId, s.name(), s.provider_name(), e.EventHeader.EventDescriptor.Opcode, e.EventHeader.EventDescriptor.Id, s.property_count(), e.EventHeader.TimeStamp); - println!("pid: {} {:?}", s.process_id(), processes.get(&s.process_id())); - if e.EventHeader.ActivityId != GUID::zeroed() { - println!("ActivityId: {:?}", e.EventHeader.ActivityId); - } - if e.ExtendedDataCount > 0 { - let items = unsafe { std::slice::from_raw_parts(e.ExtendedData, e.ExtendedDataCount as usize) }; - for i in items { - match i.ExtType as u32 { - Etw::EVENT_HEADER_EXT_TYPE_EVENT_SCHEMA_TL => { - println!("extended: SCHEMA_TL"); - let data: &[u8] = unsafe { std::slice::from_raw_parts(i.DataPtr as *const u8, i.DataSize as usize) }; - // from TraceLoggingProvider.h - let size = u16::from_ne_bytes(<[u8; 2]>::try_from(&data[0..2]).unwrap()); - println!(" size: {}", size); - let mut extension_size = 1; - while data[2 + extension_size] & 0x80 != 0 { - extension_size += 1; - } - println!(" extension: {:?}", &data[2..2 + extension_size]); - let name_start = 2 + extension_size; - let mut name_end = name_start; - while data[name_end] != 0 { - name_end += 1; - } - let name = String::from_utf8((&data[name_start..name_end]).to_owned()).unwrap(); - println!(" name: {}", name); + if let Some(pattern) = &pattern { + if !s.name().contains(pattern) { + return; + } + } + println!( + "{:?} {} {} {}-{} {} {}", + e.EventHeader.ProviderId, + s.name(), + s.provider_name(), + e.EventHeader.EventDescriptor.Opcode, + e.EventHeader.EventDescriptor.Id, + s.property_count(), + e.EventHeader.TimeStamp + ); + println!( + "pid: {} {:?}", + s.process_id(), + processes.get(&s.process_id()) + ); + if e.EventHeader.ActivityId != GUID::zeroed() { + println!("ActivityId: {:?}", e.EventHeader.ActivityId); + } + if e.ExtendedDataCount > 0 { + let items = unsafe { + std::slice::from_raw_parts(e.ExtendedData, e.ExtendedDataCount as usize) + }; + for i in items { + match i.ExtType as u32 { + Etw::EVENT_HEADER_EXT_TYPE_EVENT_SCHEMA_TL => { + println!("extended: SCHEMA_TL"); + let data: &[u8] = unsafe { + std::slice::from_raw_parts( + i.DataPtr as *const u8, + i.DataSize as usize, + ) + }; + + // from TraceLoggingProvider.h + let size = + u16::from_ne_bytes(<[u8; 2]>::try_from(&data[0..2]).unwrap()); + println!(" size: {}", size); + let mut extension_size = 1; + while data[2 + extension_size] & 0x80 != 0 { + extension_size += 1; + } + println!(" extension: {:?}", &data[2..2 + extension_size]); + let name_start = 2 + extension_size; + let mut name_end = name_start; + while data[name_end] != 0 { + name_end += 1; + } + let name = String::from_utf8((&data[name_start..name_end]).to_owned()) + .unwrap(); + println!(" name: {}", name); - let mut field_start = name_end + 1; + let mut field_start = name_end + 1; - while field_start < data.len() { - let field_name_start = field_start; - let mut field_name_end = field_name_start; - while data[field_name_end] != 0 { - field_name_end += 1; - } - let field_name = String::from_utf8((&data[field_name_start..field_name_end]).to_owned()).unwrap(); - println!(" field_name: {}", field_name); - let mut field_pos = field_name_end + 1; - let field_in_type = data[field_pos]; - dbg!(field_in_type); - field_pos += 1; - if field_in_type & 128 == 128 { - let field_out_type = data[field_pos]; + while field_start < data.len() { + let field_name_start = field_start; + let mut field_name_end = field_name_start; + while data[field_name_end] != 0 { + field_name_end += 1; + } + let field_name = String::from_utf8( + (&data[field_name_start..field_name_end]).to_owned(), + ) + .unwrap(); + println!(" field_name: {}", field_name); + let mut field_pos = field_name_end + 1; + let field_in_type = data[field_pos]; + dbg!(field_in_type); field_pos += 1; - dbg!(field_out_type); - if field_out_type & 128 == 128 { - // field extension - println!(" field extension"); - while data[field_pos] & 0x80 != 0 { + if field_in_type & 128 == 128 { + let field_out_type = data[field_pos]; + field_pos += 1; + dbg!(field_out_type); + if field_out_type & 128 == 128 { + // field extension + println!(" field extension"); + while data[field_pos] & 0x80 != 0 { + field_pos += 1; + } field_pos += 1; } - field_pos += 1; } + let c_count = 32; + let v_count = 64; + let custom = v_count | c_count; + let count_mask = v_count | c_count; + if field_in_type & count_mask == c_count { + // value count + field_pos += 2 + } + if field_in_type & count_mask == custom { + let type_info_size = u16::from_ne_bytes( + <[u8; 2]>::try_from(&data[field_pos..field_pos + 2]) + .unwrap(), + ); + field_pos += 2; + field_pos += type_info_size as usize; + } + field_start = field_pos; } - let c_count = 32; - let v_count = 64; - let custom = v_count | c_count; - let count_mask = v_count | c_count; - if field_in_type & count_mask == c_count { - // value count - field_pos += 2 + } + Etw::EVENT_HEADER_EXT_TYPE_PROV_TRAITS => { + println!("extended: PROV_TRAITS"); + let data: &[u8] = unsafe { + std::slice::from_raw_parts( + i.DataPtr as *const u8, + i.DataSize as usize, + ) + }; + // ProviderMetadata + let size = + u16::from_ne_bytes(<[u8; 2]>::try_from(&data[0..2]).unwrap()); + println!(" size: {}", size); + let name_start = 2; + let mut name_end = name_start; + while data[name_end] != 0 { + name_end += 1; } - if field_in_type & count_mask == custom { - let type_info_size = u16::from_ne_bytes(<[u8; 2]>::try_from(&data[field_pos..field_pos+2]).unwrap()); - field_pos += 2; - field_pos += type_info_size as usize; + let name = String::from_utf8((&data[name_start..name_end]).to_owned()) + .unwrap(); + println!(" name: {}", name); + let mut metadata_start = name_end + 1; + // ProviderMetadataChunk + while metadata_start < data.len() { + let mut metadata_end = metadata_start; + let metadata_size = u16::from_ne_bytes( + <[u8; 2]>::try_from(&data[metadata_start..metadata_start + 2]) + .unwrap(), + ); + let metadata_type = data[metadata_start + 2]; + println!(" metadata_size: {}", metadata_size); + if metadata_type as i32 == EtwProviderTraitTypeGroup.0 { + println!(" EtwProviderTraitTypeGroup"); + // read GUID + let guid = &data[(metadata_start + 3)..]; + let guid = GUID::from_values( + u32::from_ne_bytes((&guid[0..4]).try_into().unwrap()), + u16::from_ne_bytes((&guid[4..6]).try_into().unwrap()), + u16::from_ne_bytes((&guid[6..8]).try_into().unwrap()), + [ + guid[8], guid[9], guid[10], guid[11], guid[12], + guid[13], guid[14], guid[15], + ], + ); + println!(" GUID {:?}", guid); + } else if metadata_type as i32 == EtwProviderTraitDecodeGuid.0 { + println!(" EtwProviderTraitDecodeGuid"); + } else { + println!(" Unexpected {}", metadata_type); + } + metadata_start += metadata_size as usize; } - field_start = field_pos; - } - } - Etw::EVENT_HEADER_EXT_TYPE_PROV_TRAITS => { - println!("extended: PROV_TRAITS"); - let data: &[u8] = unsafe { std::slice::from_raw_parts(i.DataPtr as *const u8, i.DataSize as usize) }; - // ProviderMetadata - let size = u16::from_ne_bytes(<[u8; 2]>::try_from(&data[0..2]).unwrap()); - println!(" size: {}", size); - let name_start = 2; - let mut name_end = name_start; - while data[name_end] != 0 { - name_end += 1; } - let name = String::from_utf8((&data[name_start..name_end]).to_owned()).unwrap(); - println!(" name: {}", name); - let mut metadata_start = name_end + 1; - // ProviderMetadataChunk - while metadata_start < data.len() { - let mut metadata_end = metadata_start; - let metadata_size = u16::from_ne_bytes(<[u8; 2]>::try_from(&data[metadata_start..metadata_start+2]).unwrap()); - let metadata_type = data[metadata_start+2]; - println!(" metadata_size: {}", metadata_size); - if metadata_type as i32 == EtwProviderTraitTypeGroup.0 { - println!(" EtwProviderTraitTypeGroup"); - // read GUID - let guid = &data[(metadata_start + 3)..]; - let guid = GUID::from_values(u32::from_ne_bytes((&guid[0..4]).try_into().unwrap()), - u16::from_ne_bytes((&guid[4..6]).try_into().unwrap()), - u16::from_ne_bytes((&guid[6..8]).try_into().unwrap()), - [ - guid[8], - guid[9], - guid[10], - guid[11], - guid[12], - guid[13], - guid[14], - guid[15], - ] - ); - println!(" GUID {:?}", guid); - } else if metadata_type as i32 == EtwProviderTraitDecodeGuid.0 { - println!(" EtwProviderTraitDecodeGuid"); - } else { - println!(" Unexpected {}", metadata_type); - } - metadata_start += metadata_size as usize; + _ => { + println!("extended: {:?}", i); } - - } - _ => { - println!("extended: {:?}", i); } - } } + let formatted_message = s.event_message(); + if let Some(message) = formatted_message { + println!("message: {}", message); + } + let mut parser = Parser::create(&s); + for i in 0..s.property_count() { + let property = s.property(i); + //dbg!(&property); + print_property(&mut parser, &property, true); + } + } else { + if pattern.is_none() { + println!( + "unknown event {:x?}:{} size: {}", + e.EventHeader.ProviderId, + e.EventHeader.EventDescriptor.Opcode, + e.UserDataLength + ); + } } - let formatted_message = s.event_message(); - if let Some(message) = formatted_message { - println!("message: {}", message); - } - let mut parser = Parser::create(&s); - for i in 0..s.property_count() { - let property = s.property(i); - //dbg!(&property); - print_property(&mut parser, &property, true); - } - } else { - if pattern.is_none() { - println!("unknown event {:x?}:{} size: {}", e.EventHeader.ProviderId, e.EventHeader.EventDescriptor.Opcode, e.UserDataLength); - } - } - - -}); - + }); } - diff --git a/etw-reader/examples/event-types.rs b/etw-reader/examples/event-types.rs index 8df60dda..81b7e736 100644 --- a/etw-reader/examples/event-types.rs +++ b/etw-reader/examples/event-types.rs @@ -1,39 +1,76 @@ -use etw_reader::{GUID, open_trace, schema::{SchemaLocator}}; -use std::{cmp::Reverse, collections::HashMap, path::Path}; - -fn main() { - let mut schema_locator = SchemaLocator::new(); - etw_reader::add_custom_schemas(&mut schema_locator); - let mut event_counts = HashMap::new(); - open_trace(Path::new(&std::env::args().nth(1).unwrap()), |e| { - let s = schema_locator.event_schema(e); - if let Ok(s) = s { - let name = format!("{} {:?}/{}/{}", s.name(), e.EventHeader.ProviderId, e.EventHeader.EventDescriptor.Id, e.EventHeader.EventDescriptor.Opcode); - if let Some(count) = event_counts.get_mut(&name) { - *count += 1; - } else { - event_counts.insert(name, 1); - } - } else { - let provider_name = - match e.EventHeader.ProviderId { - GUID{ data1: 0x9B79EE91, data2: 0xB5FD, data3: 0x41C0, data4: [0xA2, 0x43, 0x42, 0x48, 0xE2, 0x66, 0xE9, 0xD0]} => "SysConfig ", - GUID{ data1: 0xB3E675D7, data2: 0x2554, data3: 0x4F18, data4: [0x83, 0x0B, 0x27, 0x62, 0x73, 0x25, 0x60, 0xDE]} => "KernelTraceControl ", - GUID{ data1: 0xED54DFF8, data2: 0xC409, data3: 0x4CF6, data4: [0xBF, 0x83, 0x05, 0xE1, 0xE6, 0x1A, 0x09, 0xC4]} => "WinSat ", - // see https://docs.microsoft.com/en-us/windows-hardware/drivers/ddi/umdprovider/nf-umdprovider-umdetwregister - GUID{ data1: 0xa688ee40, data2: 0xd8d9, data3: 0x4736, data4: [0xb6, 0xf9, 0x6b, 0x74, 0x93, 0x5b, 0xa3, 0xb1]} => "D3DUmdLogging ", - GUID{ data1: 0x3d6fa8d3, data2: 0xfe05, data3: 0x11d0, data4: [0x9d, 0xda, 0x00, 0xc0, 0x4f, 0xd7, 0xba, 0x7c]} => "PageFault_V2 ", - _ => "" - }; - - let provider = format!("{}{:?}/{}-{}/{}", provider_name, e.EventHeader.ProviderId, e.EventHeader.EventDescriptor.Id, e.EventHeader.EventDescriptor.Version, e.EventHeader.EventDescriptor.Task); - *event_counts.entry(provider).or_insert(0) += 1; - } - }); - let mut event_counts: Vec<_> = event_counts.into_iter().collect(); - // event_counts.sort_by_key(|x| x.0.clone()); //alphabetical - event_counts.sort_by_key(|x| Reverse(x.1)); - for (k, v) in event_counts { - println!("{} {}", k, v); - } -} +use etw_reader::{open_trace, schema::SchemaLocator, GUID}; +use std::{cmp::Reverse, collections::HashMap, path::Path}; + +fn main() { + let mut schema_locator = SchemaLocator::new(); + etw_reader::add_custom_schemas(&mut schema_locator); + let mut event_counts = HashMap::new(); + open_trace(Path::new(&std::env::args().nth(1).unwrap()), |e| { + let s = schema_locator.event_schema(e); + if let Ok(s) = s { + let name = format!( + "{} {:?}/{}/{}", + s.name(), + e.EventHeader.ProviderId, + e.EventHeader.EventDescriptor.Id, + e.EventHeader.EventDescriptor.Opcode + ); + if let Some(count) = event_counts.get_mut(&name) { + *count += 1; + } else { + event_counts.insert(name, 1); + } + } else { + let provider_name = match e.EventHeader.ProviderId { + GUID { + data1: 0x9B79EE91, + data2: 0xB5FD, + data3: 0x41C0, + data4: [0xA2, 0x43, 0x42, 0x48, 0xE2, 0x66, 0xE9, 0xD0], + } => "SysConfig ", + GUID { + data1: 0xB3E675D7, + data2: 0x2554, + data3: 0x4F18, + data4: [0x83, 0x0B, 0x27, 0x62, 0x73, 0x25, 0x60, 0xDE], + } => "KernelTraceControl ", + GUID { + data1: 0xED54DFF8, + data2: 0xC409, + data3: 0x4CF6, + data4: [0xBF, 0x83, 0x05, 0xE1, 0xE6, 0x1A, 0x09, 0xC4], + } => "WinSat ", + // see https://docs.microsoft.com/en-us/windows-hardware/drivers/ddi/umdprovider/nf-umdprovider-umdetwregister + GUID { + data1: 0xa688ee40, + data2: 0xd8d9, + data3: 0x4736, + data4: [0xb6, 0xf9, 0x6b, 0x74, 0x93, 0x5b, 0xa3, 0xb1], + } => "D3DUmdLogging ", + GUID { + data1: 0x3d6fa8d3, + data2: 0xfe05, + data3: 0x11d0, + data4: [0x9d, 0xda, 0x00, 0xc0, 0x4f, 0xd7, 0xba, 0x7c], + } => "PageFault_V2 ", + _ => "", + }; + + let provider = format!( + "{}{:?}/{}-{}/{}", + provider_name, + e.EventHeader.ProviderId, + e.EventHeader.EventDescriptor.Id, + e.EventHeader.EventDescriptor.Version, + e.EventHeader.EventDescriptor.Task + ); + *event_counts.entry(provider).or_insert(0) += 1; + } + }); + let mut event_counts: Vec<_> = event_counts.into_iter().collect(); + // event_counts.sort_by_key(|x| x.0.clone()); //alphabetical + event_counts.sort_by_key(|x| Reverse(x.1)); + for (k, v) in event_counts { + println!("{} {}", k, v); + } +} diff --git a/etw-reader/examples/list-processes.rs b/etw-reader/examples/list-processes.rs index efd4a419..b4867b0e 100644 --- a/etw-reader/examples/list-processes.rs +++ b/etw-reader/examples/list-processes.rs @@ -1,29 +1,29 @@ - - - -use etw_reader::{ - open_trace, - parser::{Parser, TryParse}, - schema::SchemaLocator, -}; -use std::path::Path; - -fn main() { - let mut schema_locator = SchemaLocator::new(); - etw_reader::add_custom_schemas(&mut schema_locator); - open_trace(Path::new(&std::env::args().nth(1).unwrap()), |e| { - let s = schema_locator.event_schema(e); - if let Ok(s) = s { - // DCEnd is used '-Buffering' traces - if let "MSNT_SystemTrace/Process/Start" | "MSNT_SystemTrace/Process/DCStart" | "MSNT_SystemTrace/Process/DCEnd" = s.name() { - let mut parser = Parser::create(&s); - - let image_file_name: String = parser.parse("ImageFileName"); - let process_id: u32 = parser.parse("ProcessId"); - let command_line: String = parser.parse("CommandLine"); - - println!("{} {} {}", image_file_name, process_id, command_line); - } - } - }); -} \ No newline at end of file +use etw_reader::{ + open_trace, + parser::{Parser, TryParse}, + schema::SchemaLocator, +}; +use std::path::Path; + +fn main() { + let mut schema_locator = SchemaLocator::new(); + etw_reader::add_custom_schemas(&mut schema_locator); + open_trace(Path::new(&std::env::args().nth(1).unwrap()), |e| { + let s = schema_locator.event_schema(e); + if let Ok(s) = s { + // DCEnd is used '-Buffering' traces + if let "MSNT_SystemTrace/Process/Start" + | "MSNT_SystemTrace/Process/DCStart" + | "MSNT_SystemTrace/Process/DCEnd" = s.name() + { + let mut parser = Parser::create(&s); + + let image_file_name: String = parser.parse("ImageFileName"); + let process_id: u32 = parser.parse("ProcessId"); + let command_line: String = parser.parse("CommandLine"); + + println!("{} {} {}", image_file_name, process_id, command_line); + } + } + }); +} diff --git a/etw-reader/examples/list-providers.rs b/etw-reader/examples/list-providers.rs index c4bc617e..a5ec4f1f 100644 --- a/etw-reader/examples/list-providers.rs +++ b/etw-reader/examples/list-providers.rs @@ -1,6 +1,6 @@ -use etw_reader::{enumerate_trace_guids_ex, tdh}; - -pub fn main() { - tdh::list_etw_providers(); - enumerate_trace_guids_ex(false); -} \ No newline at end of file +use etw_reader::{enumerate_trace_guids_ex, tdh}; + +pub fn main() { + tdh::list_etw_providers(); + enumerate_trace_guids_ex(false); +} diff --git a/etw-reader/examples/list-threads.rs b/etw-reader/examples/list-threads.rs index 6dd2f02b..57d49da3 100644 --- a/etw-reader/examples/list-threads.rs +++ b/etw-reader/examples/list-threads.rs @@ -1,86 +1,86 @@ - - - -use etw_reader::{ - open_trace, - parser::{Parser, TryParse}, - schema::SchemaLocator, -}; -use std::{path::Path, collections::HashMap}; - -struct Process { - image_file_name: String -} - - - -fn main() { - let mut schema_locator = SchemaLocator::new(); - etw_reader::add_custom_schemas(&mut schema_locator); - let mut processes = HashMap::new(); - open_trace(Path::new(&std::env::args().nth(1).unwrap()), |e| { - let s = schema_locator.event_schema(e); - if let Ok(s) = s { - // DCEnd is used '-Buffering' traces - match s.name() { - "MSNT_SystemTrace/Process/Start" | "MSNT_SystemTrace/Process/DCStart" | "MSNT_SystemTrace/Process/DCEnd" => { - let mut parser = Parser::create(&s); - - let image_file_name: String = parser.parse("ImageFileName"); - let process_id: u32 = parser.parse("ProcessId"); - let command_line: String = parser.parse("CommandLine"); - println!("{} {} {}", image_file_name, process_id, command_line); - - processes.insert(process_id, Process{ image_file_name }); - - } - "MSNT_SystemTrace/Thread/Start" | - "MSNT_SystemTrace/Thread/DCStart" => { - let mut parser = Parser::create(&s); - - let thread_id: u32 = parser.parse("TThreadId"); - let process_id: u32 = parser.parse("ProcessId"); - println!("thread process {}({}) tid {}", processes[&process_id].image_file_name, process_id, thread_id); - //assert_eq!(process_id,s.process_id()); - - let thread_name: Result = parser.try_parse("ThreadName"); - match thread_name { - Ok(thread_name) if !thread_name.is_empty() => { - println!("thread_name pid: {} tid: {} name: {:?}", process_id, thread_id, thread_name); - - }, - _ => {} - } - - } - "MSNT_SystemTrace/Thread/SetName" => { - let mut parser = Parser::create(&s); - - let process_id: u32 = parser.parse("ProcessId"); - let thread_id: u32 = parser.parse("ThreadId"); - let thread_name: String = parser.parse("ThreadName"); - /* - let thread = match threads.entry(thread_id) { - Entry::Occupied(e) => e.into_mut(), - Entry::Vacant(e) => { - let thread_start_instant = profile_start_instant; - let handle = match global_thread { - Some(global_thread) => global_thread, - None => { - let process = processes[&process_id].process_handle; - profile.add_thread(process, thread_id, thread_start_instant, false) - } - }; - let tb = e.insert( - ThreadState::new(handle, thread_id) - ); - thread_index += 1; - tb - } - };*/ - } - _ => {} - } - } - }).expect("failed"); -} \ No newline at end of file +use etw_reader::{ + open_trace, + parser::{Parser, TryParse}, + schema::SchemaLocator, +}; +use std::{collections::HashMap, path::Path}; + +struct Process { + image_file_name: String, +} + +fn main() { + let mut schema_locator = SchemaLocator::new(); + etw_reader::add_custom_schemas(&mut schema_locator); + let mut processes = HashMap::new(); + open_trace(Path::new(&std::env::args().nth(1).unwrap()), |e| { + let s = schema_locator.event_schema(e); + if let Ok(s) = s { + // DCEnd is used '-Buffering' traces + match s.name() { + "MSNT_SystemTrace/Process/Start" + | "MSNT_SystemTrace/Process/DCStart" + | "MSNT_SystemTrace/Process/DCEnd" => { + let mut parser = Parser::create(&s); + + let image_file_name: String = parser.parse("ImageFileName"); + let process_id: u32 = parser.parse("ProcessId"); + let command_line: String = parser.parse("CommandLine"); + println!("{} {} {}", image_file_name, process_id, command_line); + + processes.insert(process_id, Process { image_file_name }); + } + "MSNT_SystemTrace/Thread/Start" | "MSNT_SystemTrace/Thread/DCStart" => { + let mut parser = Parser::create(&s); + + let thread_id: u32 = parser.parse("TThreadId"); + let process_id: u32 = parser.parse("ProcessId"); + println!( + "thread process {}({}) tid {}", + processes[&process_id].image_file_name, process_id, thread_id + ); + //assert_eq!(process_id,s.process_id()); + + let thread_name: Result = parser.try_parse("ThreadName"); + match thread_name { + Ok(thread_name) if !thread_name.is_empty() => { + println!( + "thread_name pid: {} tid: {} name: {:?}", + process_id, thread_id, thread_name + ); + } + _ => {} + } + } + "MSNT_SystemTrace/Thread/SetName" => { + let mut parser = Parser::create(&s); + + let process_id: u32 = parser.parse("ProcessId"); + let thread_id: u32 = parser.parse("ThreadId"); + let thread_name: String = parser.parse("ThreadName"); + /* + let thread = match threads.entry(thread_id) { + Entry::Occupied(e) => e.into_mut(), + Entry::Vacant(e) => { + let thread_start_instant = profile_start_instant; + let handle = match global_thread { + Some(global_thread) => global_thread, + None => { + let process = processes[&process_id].process_handle; + profile.add_thread(process, thread_id, thread_start_instant, false) + } + }; + let tb = e.insert( + ThreadState::new(handle, thread_id) + ); + thread_index += 1; + tb + } + };*/ + } + _ => {} + } + } + }) + .expect("failed"); +} diff --git a/etw-reader/examples/log-js-stacks.rs b/etw-reader/examples/log-js-stacks.rs index 07323bd0..ba63a54c 100644 --- a/etw-reader/examples/log-js-stacks.rs +++ b/etw-reader/examples/log-js-stacks.rs @@ -1,195 +1,225 @@ -use etw_reader::{ - open_trace, - parser::{Parser, TryParse, Address}, - print_property, - schema::SchemaLocator, -}; -use std::{ - cell::Cell, - collections::{hash_map::Entry, HashMap, BTreeMap}, - convert::TryInto, - path::Path, -}; -use windows::Win32::System::Diagnostics::Etw; -use std::collections::Bound::{Included, Unbounded}; - - -/// A single symbol from a [`SymbolTable`]. -#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub struct Symbol { - /// The symbol's address, as a "relative address", i.e. relative to the library's base address. - pub address: u32, - /// The symbol's size, if known. This is often just set based on the address of the next symbol. - pub size: Option, - /// The symbol name. - pub name: String, -} - - -struct Event { - name: String, - timestamp: i64, - stack: Option>, -} - -struct ThreadState { - process_id: u32, - unfinished_kernel_stacks: Vec, -} -impl ThreadState { - fn new(process_id: u32) -> Self { - ThreadState { - process_id, - unfinished_kernel_stacks: Vec::new(), - } - } -} - -fn main() { - let mut schema_locator = SchemaLocator::new(); - etw_reader::add_custom_schemas(&mut schema_locator); - let pattern = std::env::args().nth(2); - let mut processes = HashMap::new(); - let mut events: Vec = Vec::new(); - let mut threads = HashMap::new(); - let mut jscript_symbols: HashMap> = HashMap::new(); - let mut jscript_sources: HashMap = HashMap::new(); - open_trace(Path::new(&std::env::args().nth(1).unwrap()), |e| { - //dbg!(e.EventHeader.TimeStamp); - - let s = schema_locator.event_schema(e); - let mut thread_id = e.EventHeader.ThreadId; - if let Ok(s) = s { - match s.name() { - "MSNT_SystemTrace/StackWalk/Stack" => { - let mut parser = Parser::create(&s); - - let thread_id: u32 = parser.parse("StackThread"); - let process_id: u32 = parser.parse("StackProcess"); - - println!("Sample"); - let thread = match threads.entry(thread_id) { - Entry::Occupied(e) => e.into_mut(), - Entry::Vacant(e) => e.insert(ThreadState::new(process_id)), - }; - let timestamp: u64 = parser.parse("EventTimeStamp"); - - let mut stack: Vec = parser - .buffer - .chunks_exact(8) - .map(|a| u64::from_ne_bytes(a.try_into().unwrap())) - .collect(); - - let mut js_stack = Vec::new(); - let mut child = String::new(); - let mut child_addr = 0; - - stack.iter().for_each(|addr| { - if let Some(syms) = jscript_symbols.get(&process_id) { - if let Some(sym) = syms.range((Unbounded, Included(addr))).last() { - if *addr < *sym.0 + sym.1.0 { - js_stack.push((sym.1.1.clone(), *addr)); - //println!("found match for {} calls {:x}:{}", sym.1.1, child_addr, child); - child = sym.1.1.clone(); - child_addr = *sym.0; - return - } - } - } - //println!("{:x}", addr); - }); - - for i in 0..js_stack.len() { - if js_stack[i].0.contains("runSync") && i > 2 && js_stack[i-2].0.contains("click") { - println!("found match {} {:x}/{} {}", js_stack[i].0, js_stack[i-1].1, js_stack[i-1].0, js_stack[i-2].0); - } - } - - - - - - - } - "MSNT_SystemTrace/PerfInfo/SampleProf" => { - let mut parser = Parser::create(&s); - - thread_id = parser.parse("ThreadId"); - } - "V8.js/MethodLoad/" | - "Microsoft-JScript/MethodRuntime/MethodDCStart" | - "Microsoft-JScript/MethodRuntime/MethodLoad" => { - let mut parser = Parser::create(&s); - let method_name: String = parser.parse("MethodName"); - let method_start_address: Address = parser.parse("MethodStartAddress"); - let method_size: u64 = parser.parse("MethodSize"); - if method_start_address.as_u64() <= 0x7ffde0297cc0 && method_start_address.as_u64() + method_size >= 0x7ffde0297cc0 { - println!("before: {} {:x} {}", method_name, method_start_address.as_u64(), method_size); - } - - let source_id: u64 = parser.parse("SourceID"); - let process_id = s.process_id(); - if method_name.contains("getNearestLContainer") || method_name.contains("277:53") { - println!("load {} {} {:x}", method_name, method_size, method_start_address.as_u64()); - } - let syms = jscript_symbols.entry(s.process_id()).or_insert(BTreeMap::new()); - //let name_and_file = format!("{} {}", method_name, jscript_sources.get(&source_id).map(|x| x.as_ref()).unwrap_or("?")); - let start_address = method_start_address.as_u64(); - let mut overlaps = Vec::new(); - for sym in syms.range_mut((Included(start_address), Included(start_address + method_size))) { - if method_name != sym.1.1 || start_address != *sym.0 || method_size != sym.1.0 { - println!("overlap {} {} {} - {:?}", method_name, start_address, method_size, sym); - overlaps.push(*sym.0); - } else { - println!("overlap same {} {} {} - {:?}", method_name, start_address, method_size, sym); - } - } - for sym in overlaps { - syms.remove(&sym); - } - - syms.insert(start_address, (method_size, method_name)); - //dbg!(s.process_id(), jscript_symbols.keys()); - - } - _ => {} - } - if let "MSNT_SystemTrace/Process/Start" - | "MSNT_SystemTrace/Process/DCStart" - | "MSNT_SystemTrace/Process/DCEnd" = s.name() - { - let mut parser = Parser::create(&s); - - let image_file_name: String = parser.parse("ImageFileName"); - let process_id: u32 = parser.parse("ProcessId"); - processes.insert(process_id, image_file_name); - } - - - } else { - if pattern.is_none() { - /*println!( - "unknown event {:x?}:{}", - e.EventHeader.ProviderId, e.EventHeader.EventDescriptor.Opcode - );*/ - } - } - }) - .unwrap(); - for e in &mut events { - if let Some(stack) = &e.stack { - println!("{} {}", e.timestamp, e.name); - for addr in stack { - println!(" {:x}", addr); - } - } - } - for (tid, state) in threads { - if state.unfinished_kernel_stacks.len() > 0 { - println!("thread `{tid}` of {} has {} unfinished kernel stacks", state.process_id, state.unfinished_kernel_stacks.len()); - for stack in state.unfinished_kernel_stacks { - println!(" {}", events[stack].timestamp); - } - } - } -} +use etw_reader::{ + open_trace, + parser::{Address, Parser, TryParse}, + print_property, + schema::SchemaLocator, +}; +use std::collections::Bound::{Included, Unbounded}; +use std::{ + cell::Cell, + collections::{hash_map::Entry, BTreeMap, HashMap}, + convert::TryInto, + path::Path, +}; +use windows::Win32::System::Diagnostics::Etw; + +/// A single symbol from a [`SymbolTable`]. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct Symbol { + /// The symbol's address, as a "relative address", i.e. relative to the library's base address. + pub address: u32, + /// The symbol's size, if known. This is often just set based on the address of the next symbol. + pub size: Option, + /// The symbol name. + pub name: String, +} + +struct Event { + name: String, + timestamp: i64, + stack: Option>, +} + +struct ThreadState { + process_id: u32, + unfinished_kernel_stacks: Vec, +} +impl ThreadState { + fn new(process_id: u32) -> Self { + ThreadState { + process_id, + unfinished_kernel_stacks: Vec::new(), + } + } +} + +fn main() { + let mut schema_locator = SchemaLocator::new(); + etw_reader::add_custom_schemas(&mut schema_locator); + let pattern = std::env::args().nth(2); + let mut processes = HashMap::new(); + let mut events: Vec = Vec::new(); + let mut threads = HashMap::new(); + let mut jscript_symbols: HashMap> = HashMap::new(); + let mut jscript_sources: HashMap = HashMap::new(); + open_trace(Path::new(&std::env::args().nth(1).unwrap()), |e| { + //dbg!(e.EventHeader.TimeStamp); + + let s = schema_locator.event_schema(e); + let mut thread_id = e.EventHeader.ThreadId; + if let Ok(s) = s { + match s.name() { + "MSNT_SystemTrace/StackWalk/Stack" => { + let mut parser = Parser::create(&s); + + let thread_id: u32 = parser.parse("StackThread"); + let process_id: u32 = parser.parse("StackProcess"); + + println!("Sample"); + let thread = match threads.entry(thread_id) { + Entry::Occupied(e) => e.into_mut(), + Entry::Vacant(e) => e.insert(ThreadState::new(process_id)), + }; + let timestamp: u64 = parser.parse("EventTimeStamp"); + + let mut stack: Vec = parser + .buffer + .chunks_exact(8) + .map(|a| u64::from_ne_bytes(a.try_into().unwrap())) + .collect(); + + let mut js_stack = Vec::new(); + let mut child = String::new(); + let mut child_addr = 0; + + stack.iter().for_each(|addr| { + if let Some(syms) = jscript_symbols.get(&process_id) { + if let Some(sym) = syms.range((Unbounded, Included(addr))).last() { + if *addr < *sym.0 + sym.1 .0 { + js_stack.push((sym.1 .1.clone(), *addr)); + //println!("found match for {} calls {:x}:{}", sym.1.1, child_addr, child); + child = sym.1 .1.clone(); + child_addr = *sym.0; + return; + } + } + } + //println!("{:x}", addr); + }); + + for i in 0..js_stack.len() { + if js_stack[i].0.contains("runSync") + && i > 2 + && js_stack[i - 2].0.contains("click") + { + println!( + "found match {} {:x}/{} {}", + js_stack[i].0, + js_stack[i - 1].1, + js_stack[i - 1].0, + js_stack[i - 2].0 + ); + } + } + } + "MSNT_SystemTrace/PerfInfo/SampleProf" => { + let mut parser = Parser::create(&s); + + thread_id = parser.parse("ThreadId"); + } + "V8.js/MethodLoad/" + | "Microsoft-JScript/MethodRuntime/MethodDCStart" + | "Microsoft-JScript/MethodRuntime/MethodLoad" => { + let mut parser = Parser::create(&s); + let method_name: String = parser.parse("MethodName"); + let method_start_address: Address = parser.parse("MethodStartAddress"); + let method_size: u64 = parser.parse("MethodSize"); + if method_start_address.as_u64() <= 0x7ffde0297cc0 + && method_start_address.as_u64() + method_size >= 0x7ffde0297cc0 + { + println!( + "before: {} {:x} {}", + method_name, + method_start_address.as_u64(), + method_size + ); + } + + let source_id: u64 = parser.parse("SourceID"); + let process_id = s.process_id(); + if method_name.contains("getNearestLContainer") + || method_name.contains("277:53") + { + println!( + "load {} {} {:x}", + method_name, + method_size, + method_start_address.as_u64() + ); + } + let syms = jscript_symbols + .entry(s.process_id()) + .or_insert(BTreeMap::new()); + //let name_and_file = format!("{} {}", method_name, jscript_sources.get(&source_id).map(|x| x.as_ref()).unwrap_or("?")); + let start_address = method_start_address.as_u64(); + let mut overlaps = Vec::new(); + for sym in syms.range_mut(( + Included(start_address), + Included(start_address + method_size), + )) { + if method_name != sym.1 .1 + || start_address != *sym.0 + || method_size != sym.1 .0 + { + println!( + "overlap {} {} {} - {:?}", + method_name, start_address, method_size, sym + ); + overlaps.push(*sym.0); + } else { + println!( + "overlap same {} {} {} - {:?}", + method_name, start_address, method_size, sym + ); + } + } + for sym in overlaps { + syms.remove(&sym); + } + + syms.insert(start_address, (method_size, method_name)); + //dbg!(s.process_id(), jscript_symbols.keys()); + } + _ => {} + } + if let "MSNT_SystemTrace/Process/Start" + | "MSNT_SystemTrace/Process/DCStart" + | "MSNT_SystemTrace/Process/DCEnd" = s.name() + { + let mut parser = Parser::create(&s); + + let image_file_name: String = parser.parse("ImageFileName"); + let process_id: u32 = parser.parse("ProcessId"); + processes.insert(process_id, image_file_name); + } + } else { + if pattern.is_none() { + /*println!( + "unknown event {:x?}:{}", + e.EventHeader.ProviderId, e.EventHeader.EventDescriptor.Opcode + );*/ + } + } + }) + .unwrap(); + for e in &mut events { + if let Some(stack) = &e.stack { + println!("{} {}", e.timestamp, e.name); + for addr in stack { + println!(" {:x}", addr); + } + } + } + for (tid, state) in threads { + if state.unfinished_kernel_stacks.len() > 0 { + println!( + "thread `{tid}` of {} has {} unfinished kernel stacks", + state.process_id, + state.unfinished_kernel_stacks.len() + ); + for stack in state.unfinished_kernel_stacks { + println!(" {}", events[stack].timestamp); + } + } + } +} diff --git a/etw-reader/examples/log-stacks.rs b/etw-reader/examples/log-stacks.rs index 550f4185..69ecc1f5 100644 --- a/etw-reader/examples/log-stacks.rs +++ b/etw-reader/examples/log-stacks.rs @@ -1,224 +1,242 @@ -use etw_reader::{ - open_trace, - parser::{Parser, TryParse}, - print_property, - schema::SchemaLocator, -}; -use std::{ - cell::Cell, - collections::{hash_map::Entry, HashMap}, - convert::TryInto, - path::Path, -}; -use windows::Win32::System::Diagnostics::Etw; - -fn is_kernel_address(ip: u64, pointer_size: u32) -> bool { - if pointer_size == 4 { - return ip >= 0x80000000; - } - return ip >= 0xFFFF000000000000; // TODO I don't know what the true cutoff is. -} - -struct Event { - name: String, - timestamp: i64, - thread_id: u32, - stack: Option>, - cpu: u16, - bad_stack: bool -} - -struct ThreadState { - process_id: u32, - events_with_unfinished_kernel_stacks: Vec, -} -impl ThreadState { - fn new(process_id: u32) -> Self { - ThreadState { - process_id, - events_with_unfinished_kernel_stacks: Vec::new(), - } - } -} - -fn main() { - let mut schema_locator = SchemaLocator::new(); - etw_reader::add_custom_schemas(&mut schema_locator); - let pattern = std::env::args().nth(2); - let mut processes = HashMap::new(); - let mut events: Vec = Vec::new(); - let mut threads = HashMap::new(); - open_trace(Path::new(&std::env::args().nth(1).unwrap()), |e| { - //dbg!(e.EventHeader.TimeStamp); - - let s = schema_locator.event_schema(e); - let mut thread_id = e.EventHeader.ThreadId; - if let Ok(s) = s { - match s.name() { - "MSNT_SystemTrace/StackWalk/Stack" => { - let mut parser = Parser::create(&s); - - let thread_id: u32 = parser.parse("StackThread"); - let process_id: u32 = parser.parse("StackProcess"); - - - let thread = match threads.entry(thread_id) { - Entry::Occupied(e) => e.into_mut(), - Entry::Vacant(e) => e.insert(ThreadState::new(process_id)), - }; - let timestamp: u64 = parser.parse("EventTimeStamp"); - - let mut stack: Vec = parser - .buffer - .chunks_exact(8) - .map(|a| u64::from_ne_bytes(a.try_into().unwrap())) - .collect(); - - let ends_in_kernel = is_kernel_address(*stack.last().unwrap(), 8); - let mut i = events.len() - 1; - let mut found_event: Option = None; - let cpu = unsafe { e.BufferContext.Anonymous.ProcessorIndex }; - while i > 0 { - if events[i].timestamp < timestamp as i64 { - break; - } - // sometimes the thread_id won't match (virtualalloc?) - // we adjust the thread_id of the SampleProf event to get this to work - // otherwise TraceLog will use the cpu index - if events[i].timestamp == timestamp as i64 - && events[i].thread_id == thread_id - { - if let Some(first_event) = found_event { - println!( - "more than one associated event {}/{}:{}@{} {}/{}:{}@{} {}/{}@{}", - first_event, - events[first_event].name, - events[first_event].thread_id, - events[first_event].cpu, - i, - events[i].name, - events[i].thread_id, - events[i].cpu, - timestamp, - thread_id, - cpu, - ); - } - if ends_in_kernel { - match &mut events[i].stack { - Some(existing_stack) => { - // Sometimes the kernel will call back into userspace (KeUserModeCallback) - // this can cause there to be multiple stacks that end in a kernel address. - // - // Microsoft's TraceLog library seems to discard the initial kernel stack replacing - // it with a subsequent one which seems wrong because the initial stack contains - // the address which matches the 'InstructionPointer' field in the SampleProf event. - // - // Instead of discarding, we concatenate the stacks - assert!(thread.events_with_unfinished_kernel_stacks.contains(&i)); - existing_stack.extend_from_slice(&stack[..]) - } - None => { - thread.events_with_unfinished_kernel_stacks.push(i); - events[i].stack = Some(stack.clone()); - } - }; - } else { - for e in &thread.events_with_unfinished_kernel_stacks { - events[*e] - .stack - .as_mut() - .unwrap() - .extend_from_slice(&stack[..]); - } - match &mut events[i].stack { - Some(_) => { - // any existing stacks should only have come from kernel stacks - assert!(thread.events_with_unfinished_kernel_stacks.contains(&i)); - } - None => { - events[i].stack = Some(stack.clone()); - } - }; - if let Some(event_index_with_last_unfinished_stack) = thread.events_with_unfinished_kernel_stacks.last() { - if events[*event_index_with_last_unfinished_stack].timestamp < events[i].timestamp { - // We had an event A with a kernel stack, then an event B without a kernel stack, and this user stack is for B. - // So we must have exited the kernel at some point in between. We would have expected the user stack for A - // to be captured during that exit. But we didn't get one! The user stack for B might be different from the - // (missing) user stack for A. - println!("missing userspace stack? {} < {}", events[*event_index_with_last_unfinished_stack].timestamp, events[i].timestamp); - events[*event_index_with_last_unfinished_stack].bad_stack = true; - } - } - thread.events_with_unfinished_kernel_stacks.clear(); - - } - - found_event = Some(i); - } - i -= 1; - } - - if found_event.is_none() { - println!("no matching event"); - } - } - "MSNT_SystemTrace/PerfInfo/SampleProf" => { - let mut parser = Parser::create(&s); - - thread_id = parser.parse("ThreadId"); - } - "MSNT_SystemTrace/Thread/CSwitch" => { - let mut parser = Parser::create(&s); - - thread_id = parser.parse("NewThreadId"); - } - _ => {} - } - if let "MSNT_SystemTrace/Process/Start" - | "MSNT_SystemTrace/Process/DCStart" - | "MSNT_SystemTrace/Process/DCEnd" = s.name() - { - let mut parser = Parser::create(&s); - - let image_file_name: String = parser.parse("ImageFileName"); - let process_id: u32 = parser.parse("ProcessId"); - processes.insert(process_id, image_file_name); - } - - events.push(Event { - name: s.name().to_owned(), - timestamp: e.EventHeader.TimeStamp, - thread_id, - cpu: unsafe { e.BufferContext.Anonymous.ProcessorIndex }, - stack: None, - bad_stack: false, - }); - } else { - if pattern.is_none() { - /*println!( - "unknown event {:x?}:{}", - e.EventHeader.ProviderId, e.EventHeader.EventDescriptor.Opcode - );*/ - } - } - }) - .unwrap(); - for e in &mut events { - if let Some(stack) = &e.stack { - println!("{} {}", e.timestamp, e.name); - if (e.bad_stack) { println!("bad stack");} - for addr in stack { - println!(" {:x}", addr); - } - } - } - for (tid, state) in threads { - if state.events_with_unfinished_kernel_stacks.len() > 0 { - println!("thread `{tid}` of {} has {} unfinished kernel stacks", state.process_id, state.events_with_unfinished_kernel_stacks.len()); - for stack in state.events_with_unfinished_kernel_stacks { - println!(" {}", events[stack].timestamp); - } - } - } -} +use etw_reader::{ + open_trace, + parser::{Parser, TryParse}, + print_property, + schema::SchemaLocator, +}; +use std::{ + cell::Cell, + collections::{hash_map::Entry, HashMap}, + convert::TryInto, + path::Path, +}; +use windows::Win32::System::Diagnostics::Etw; + +fn is_kernel_address(ip: u64, pointer_size: u32) -> bool { + if pointer_size == 4 { + return ip >= 0x80000000; + } + return ip >= 0xFFFF000000000000; // TODO I don't know what the true cutoff is. +} + +struct Event { + name: String, + timestamp: i64, + thread_id: u32, + stack: Option>, + cpu: u16, + bad_stack: bool, +} + +struct ThreadState { + process_id: u32, + events_with_unfinished_kernel_stacks: Vec, +} +impl ThreadState { + fn new(process_id: u32) -> Self { + ThreadState { + process_id, + events_with_unfinished_kernel_stacks: Vec::new(), + } + } +} + +fn main() { + let mut schema_locator = SchemaLocator::new(); + etw_reader::add_custom_schemas(&mut schema_locator); + let pattern = std::env::args().nth(2); + let mut processes = HashMap::new(); + let mut events: Vec = Vec::new(); + let mut threads = HashMap::new(); + open_trace(Path::new(&std::env::args().nth(1).unwrap()), |e| { + //dbg!(e.EventHeader.TimeStamp); + + let s = schema_locator.event_schema(e); + let mut thread_id = e.EventHeader.ThreadId; + if let Ok(s) = s { + match s.name() { + "MSNT_SystemTrace/StackWalk/Stack" => { + let mut parser = Parser::create(&s); + + let thread_id: u32 = parser.parse("StackThread"); + let process_id: u32 = parser.parse("StackProcess"); + + let thread = match threads.entry(thread_id) { + Entry::Occupied(e) => e.into_mut(), + Entry::Vacant(e) => e.insert(ThreadState::new(process_id)), + }; + let timestamp: u64 = parser.parse("EventTimeStamp"); + + let mut stack: Vec = parser + .buffer + .chunks_exact(8) + .map(|a| u64::from_ne_bytes(a.try_into().unwrap())) + .collect(); + + let ends_in_kernel = is_kernel_address(*stack.last().unwrap(), 8); + let mut i = events.len() - 1; + let mut found_event: Option = None; + let cpu = unsafe { e.BufferContext.Anonymous.ProcessorIndex }; + while i > 0 { + if events[i].timestamp < timestamp as i64 { + break; + } + // sometimes the thread_id won't match (virtualalloc?) + // we adjust the thread_id of the SampleProf event to get this to work + // otherwise TraceLog will use the cpu index + if events[i].timestamp == timestamp as i64 + && events[i].thread_id == thread_id + { + if let Some(first_event) = found_event { + println!( + "more than one associated event {}/{}:{}@{} {}/{}:{}@{} {}/{}@{}", + first_event, + events[first_event].name, + events[first_event].thread_id, + events[first_event].cpu, + i, + events[i].name, + events[i].thread_id, + events[i].cpu, + timestamp, + thread_id, + cpu, + ); + } + if ends_in_kernel { + match &mut events[i].stack { + Some(existing_stack) => { + // Sometimes the kernel will call back into userspace (KeUserModeCallback) + // this can cause there to be multiple stacks that end in a kernel address. + // + // Microsoft's TraceLog library seems to discard the initial kernel stack replacing + // it with a subsequent one which seems wrong because the initial stack contains + // the address which matches the 'InstructionPointer' field in the SampleProf event. + // + // Instead of discarding, we concatenate the stacks + assert!(thread + .events_with_unfinished_kernel_stacks + .contains(&i)); + existing_stack.extend_from_slice(&stack[..]) + } + None => { + thread.events_with_unfinished_kernel_stacks.push(i); + events[i].stack = Some(stack.clone()); + } + }; + } else { + for e in &thread.events_with_unfinished_kernel_stacks { + events[*e] + .stack + .as_mut() + .unwrap() + .extend_from_slice(&stack[..]); + } + match &mut events[i].stack { + Some(_) => { + // any existing stacks should only have come from kernel stacks + assert!(thread + .events_with_unfinished_kernel_stacks + .contains(&i)); + } + None => { + events[i].stack = Some(stack.clone()); + } + }; + if let Some(event_index_with_last_unfinished_stack) = + thread.events_with_unfinished_kernel_stacks.last() + { + if events[*event_index_with_last_unfinished_stack].timestamp + < events[i].timestamp + { + // We had an event A with a kernel stack, then an event B without a kernel stack, and this user stack is for B. + // So we must have exited the kernel at some point in between. We would have expected the user stack for A + // to be captured during that exit. But we didn't get one! The user stack for B might be different from the + // (missing) user stack for A. + println!( + "missing userspace stack? {} < {}", + events[*event_index_with_last_unfinished_stack] + .timestamp, + events[i].timestamp + ); + events[*event_index_with_last_unfinished_stack].bad_stack = + true; + } + } + thread.events_with_unfinished_kernel_stacks.clear(); + } + + found_event = Some(i); + } + i -= 1; + } + + if found_event.is_none() { + println!("no matching event"); + } + } + "MSNT_SystemTrace/PerfInfo/SampleProf" => { + let mut parser = Parser::create(&s); + + thread_id = parser.parse("ThreadId"); + } + "MSNT_SystemTrace/Thread/CSwitch" => { + let mut parser = Parser::create(&s); + + thread_id = parser.parse("NewThreadId"); + } + _ => {} + } + if let "MSNT_SystemTrace/Process/Start" + | "MSNT_SystemTrace/Process/DCStart" + | "MSNT_SystemTrace/Process/DCEnd" = s.name() + { + let mut parser = Parser::create(&s); + + let image_file_name: String = parser.parse("ImageFileName"); + let process_id: u32 = parser.parse("ProcessId"); + processes.insert(process_id, image_file_name); + } + + events.push(Event { + name: s.name().to_owned(), + timestamp: e.EventHeader.TimeStamp, + thread_id, + cpu: unsafe { e.BufferContext.Anonymous.ProcessorIndex }, + stack: None, + bad_stack: false, + }); + } else { + if pattern.is_none() { + /*println!( + "unknown event {:x?}:{}", + e.EventHeader.ProviderId, e.EventHeader.EventDescriptor.Opcode + );*/ + } + } + }) + .unwrap(); + for e in &mut events { + if let Some(stack) = &e.stack { + println!("{} {}", e.timestamp, e.name); + if (e.bad_stack) { + println!("bad stack"); + } + for addr in stack { + println!(" {:x}", addr); + } + } + } + for (tid, state) in threads { + if state.events_with_unfinished_kernel_stacks.len() > 0 { + println!( + "thread `{tid}` of {} has {} unfinished kernel stacks", + state.process_id, + state.events_with_unfinished_kernel_stacks.len() + ); + for stack in state.events_with_unfinished_kernel_stacks { + println!(" {}", events[stack].timestamp); + } + } + } +} diff --git a/etw-reader/examples/record.rs b/etw-reader/examples/record.rs index 3c8f580b..e95bb94a 100644 --- a/etw-reader/examples/record.rs +++ b/etw-reader/examples/record.rs @@ -1,36 +1,42 @@ -use etw_reader::{start_trace, parser::{Parser}, print_property, schema::SchemaLocator}; - - -fn main() { - - let mut schema_locator = SchemaLocator::new(); - etw_reader::add_custom_schemas(&mut schema_locator); - let pattern = std::env::args().nth(2); - start_trace( -|e| { - - - let s = schema_locator.event_schema(e); - if let Ok(s) = s { - if !s.name().contains("VideoProcessorBltParameters") { - return; - } - println!("pid {} time {}", e.EventHeader.ProcessId, e.EventHeader.TimeStamp); - println!("{:?} {} {}-{} {} {}", e.EventHeader.ProviderId, s.name(), e.EventHeader.EventDescriptor.Opcode, e.EventHeader.EventDescriptor.Id, s.property_count(), e.EventHeader.TimeStamp); - - let mut parser = Parser::create(&s); - for i in 0..s.property_count() { - let property = s.property(i); - //dbg!(&property); - print_property(&mut parser, &property, false); - } - } else { - if pattern.is_none() { - println!("unknown event {:x?}:{}", e.EventHeader.ProviderId, e.EventHeader.EventDescriptor.Opcode); - } - } - - -}); - -} \ No newline at end of file +use etw_reader::{parser::Parser, print_property, schema::SchemaLocator, start_trace}; + +fn main() { + let mut schema_locator = SchemaLocator::new(); + etw_reader::add_custom_schemas(&mut schema_locator); + let pattern = std::env::args().nth(2); + start_trace(|e| { + let s = schema_locator.event_schema(e); + if let Ok(s) = s { + if !s.name().contains("VideoProcessorBltParameters") { + return; + } + println!( + "pid {} time {}", + e.EventHeader.ProcessId, e.EventHeader.TimeStamp + ); + println!( + "{:?} {} {}-{} {} {}", + e.EventHeader.ProviderId, + s.name(), + e.EventHeader.EventDescriptor.Opcode, + e.EventHeader.EventDescriptor.Id, + s.property_count(), + e.EventHeader.TimeStamp + ); + + let mut parser = Parser::create(&s); + for i in 0..s.property_count() { + let property = s.property(i); + //dbg!(&property); + print_property(&mut parser, &property, false); + } + } else { + if pattern.is_none() { + println!( + "unknown event {:x?}:{}", + e.EventHeader.ProviderId, e.EventHeader.EventDescriptor.Opcode + ); + } + } + }); +} diff --git a/etw-reader/src/etw_types.rs b/etw-reader/src/etw_types.rs index 8e1cf52c..3951716b 100644 --- a/etw-reader/src/etw_types.rs +++ b/etw-reader/src/etw_types.rs @@ -270,7 +270,7 @@ impl EventSchema for TraceEventInfoRaw { 2 => "Stop", 3 => "DCStart", 4 => "DCStop", - _ => "" + _ => "", }); } utils::parse_unk_size_null_utf16_string(&self.info[opcode_name_offset..]) diff --git a/etw-reader/src/lib.rs b/etw-reader/src/lib.rs index b91442cf..f55ae2d7 100644 --- a/etw-reader/src/lib.rs +++ b/etw-reader/src/lib.rs @@ -6,33 +6,50 @@ extern crate bitflags; #[macro_use] extern crate num_derive; -use windows::{core::{h, Error, HRESULT, HSTRING, PWSTR}, Win32::{Foundation::{GetLastError, ERROR_INSUFFICIENT_BUFFER, ERROR_MORE_DATA, ERROR_SUCCESS, MAX_PATH}, System::Diagnostics::Etw::{EnumerateTraceGuids, EnumerateTraceGuidsEx, TraceGuidQueryInfo, TraceGuidQueryList, CONTROLTRACE_HANDLE, EVENT_TRACE_FLAG, TRACE_GUID_INFO, TRACE_GUID_PROPERTIES, TRACE_PROVIDER_INSTANCE_INFO}}}; -use crate::{parser::{Parser, ParserError, TryParse}, schema::SchemaLocator, tdh_types::{PropertyDesc, PrimitiveDesc, TdhInType}, traits::EncodeUtf16}; +use crate::{ + parser::{Parser, ParserError, TryParse}, + schema::SchemaLocator, + tdh_types::{PrimitiveDesc, PropertyDesc, TdhInType}, + traits::EncodeUtf16, +}; +use windows::{ + core::{h, Error, HRESULT, HSTRING, PWSTR}, + Win32::{ + Foundation::{ + GetLastError, ERROR_INSUFFICIENT_BUFFER, ERROR_MORE_DATA, ERROR_SUCCESS, MAX_PATH, + }, + System::Diagnostics::Etw::{ + EnumerateTraceGuids, EnumerateTraceGuidsEx, TraceGuidQueryInfo, TraceGuidQueryList, + CONTROLTRACE_HANDLE, EVENT_TRACE_FLAG, TRACE_GUID_INFO, TRACE_GUID_PROPERTIES, + TRACE_PROVIDER_INSTANCE_INFO, + }, + }, +}; #[macro_use] extern crate memoffset; use etw_types::EventRecord; -use tdh_types::{Property, TdhOutType}; +use fxhash::FxHasher; use std::{borrow::Cow, collections::HashMap, hash::BuildHasherDefault, mem, path::Path}; +use tdh_types::{Property, TdhOutType}; use windows::Win32::System::Diagnostics::Etw; -use fxhash::FxHasher; // typedef ULONG64 TRACEHANDLE, *PTRACEHANDLE; pub(crate) type TraceHandle = u64; pub const INVALID_TRACE_HANDLE: TraceHandle = u64::MAX; //, WindowsProgramming}; +pub mod custom_schemas; pub mod etw_types; -pub mod tdh; -pub mod tdh_types; -pub mod utils; pub mod parser; pub mod property; pub mod schema; pub mod sddl; +pub mod tdh; +pub mod tdh_types; pub mod traits; -pub mod custom_schemas; +pub mod utils; //pub mod trace; //pub mod provider; @@ -67,7 +84,10 @@ unsafe extern "system" fn trace_callback_thunk(event_record: *mut Etw::EVENT_REC f(std::mem::transmute(event_record)) } -pub fn open_trace(path: &Path, mut callback: F) -> Result<(), std::io::Error> { +pub fn open_trace( + path: &Path, + mut callback: F, +) -> Result<(), std::io::Error> { let mut log_file = EventTraceLogfile::default(); #[cfg(windows)] @@ -75,7 +95,8 @@ pub fn open_trace(path: &Path, mut callback: F) -> Resul #[cfg(not(windows))] let path: HSTRING = panic!(); log_file.0.LogFileName = PWSTR(path.as_wide().as_ptr() as *mut _); - log_file.0.Anonymous1.ProcessTraceMode = Etw::PROCESS_TRACE_MODE_EVENT_RECORD | Etw::PROCESS_TRACE_MODE_RAW_TIMESTAMP; + log_file.0.Anonymous1.ProcessTraceMode = + Etw::PROCESS_TRACE_MODE_EVENT_RECORD | Etw::PROCESS_TRACE_MODE_RAW_TIMESTAMP; let mut cb: &mut dyn FnMut(&EventRecord) = &mut callback; log_file.0.Context = unsafe { std::mem::transmute(&mut cb) }; log_file.0.Anonymous2.EventRecordCallback = Some(trace_callback_thunk); @@ -117,17 +138,15 @@ impl TraceInfo { &mut self, trace_name: &str, //trace_properties: &TraceProperties, - ) - { + ) { self.properties.Wnode.BufferSize = std::mem::size_of::() as u32; self.properties.Wnode.Guid = GUID::new().unwrap(); self.properties.Wnode.Flags = Etw::WNODE_FLAG_TRACED_GUID; self.properties.Wnode.ClientContext = 1; // QPC clock resolution self.properties.FlushTimer = 1; - self.properties.LogFileMode = - Etw::EVENT_TRACE_REAL_TIME_MODE | Etw::EVENT_TRACE_NO_PER_PROCESSOR_BUFFERING; + Etw::EVENT_TRACE_REAL_TIME_MODE | Etw::EVENT_TRACE_NO_PER_PROCESSOR_BUFFERING; self.properties.EnableFlags = EVENT_TRACE_FLAG(0); @@ -141,7 +160,6 @@ impl TraceInfo { } } - #[repr(C)] #[derive(Clone, Copy, Default)] pub struct EnableTraceParameters(Etw::ENABLE_TRACE_PARAMETERS); @@ -202,7 +220,6 @@ impl Provider { } } - pub struct Provider { /// Option that represents a Provider GUID pub guid: Option, @@ -216,15 +233,15 @@ pub struct Provider { pub trace_flags: u32, /// Provider kernel flags, only apply to KernelProvider pub flags: u32, // Only applies to KernelProviders - // perfinfo + // perfinfo - // filters: RwLock>, + // filters: RwLock>, } -pub fn start_trace(mut callback: F) { +pub fn start_trace(mut callback: F) { let guid_str = "22fb2cd6-0e7b-422b-a0c7-2fad1fd0e716"; let guid_str = "DB6F6DDB-AC77-4E88-8253-819DF9BBF140"; - let mut video_blt_guid = GUID::from(guid_str);//GUID::from("DB6F6DDB-AC77-4E88-8253-819DF9BBF140"); + let mut video_blt_guid = GUID::from(guid_str); //GUID::from("DB6F6DDB-AC77-4E88-8253-819DF9BBF140"); let session_name = h!("aaaaaa"); @@ -234,14 +251,31 @@ pub fn start_trace(mut callback: F) { let mut handle = CONTROLTRACE_HANDLE { Value: 0 }; unsafe { - let status = Etw::ControlTraceW(handle, session_name, &info.properties as *const _ as *mut _, Etw::EVENT_TRACE_CONTROL_STOP); + let status = Etw::ControlTraceW( + handle, + session_name, + &info.properties as *const _ as *mut _, + Etw::EVENT_TRACE_CONTROL_STOP, + ); println!("ControlTrace = {:?}", status); - let status = Etw::StartTraceW(&mut handle, session_name, &info.properties as *const _ as *mut _); + let status = Etw::StartTraceW( + &mut handle, + session_name, + &info.properties as *const _ as *mut _, + ); println!("StartTrace = {:?} handle {:?}", status, handle); info.trace_name = [0; 260]; - let status = Etw::ControlTraceW(handle, session_name, &mut info.properties, Etw::EVENT_TRACE_CONTROL_QUERY); - println!("ControlTrace = {:?} {} {:?} {:?}", status, info.properties.BufferSize, info.properties.LoggerThreadId, info.trace_name); + let status = Etw::ControlTraceW( + handle, + session_name, + &mut info.properties, + Etw::EVENT_TRACE_CONTROL_QUERY, + ); + println!( + "ControlTrace = {:?} {} {:?} {:?}", + status, info.properties.BufferSize, info.properties.LoggerThreadId, info.trace_name + ); } let prov = Provider::new().by_guid(guid_str); @@ -250,27 +284,34 @@ pub fn start_trace(mut callback: F) { let status = unsafe { Etw::EnableTrace(1, 0xffffffff, Etw::TRACE_LEVEL_VERBOSE, &video_blt_guid, handle)}; println!("EnableTrace = {}", status); */ - unsafe { Etw::EnableTraceEx2( - handle, - &mut video_blt_guid, - 1, // Fixme: EVENT_CONTROL_CODE_ENABLE_PROVIDER - prov.level, - prov.any, - prov.all, - 0, - Some(&*parameters), - ); } - + unsafe { + Etw::EnableTraceEx2( + handle, + &mut video_blt_guid, + 1, // Fixme: EVENT_CONTROL_CODE_ENABLE_PROVIDER + prov.level, + prov.any, + prov.all, + 0, + Some(&*parameters), + ); + } + let mut trace = EventTraceLogfile::default(); - trace.0.LoggerName = PWSTR(session_name.as_ptr() as *mut _ ); - trace.0.Anonymous1.ProcessTraceMode = Etw::PROCESS_TRACE_MODE_REAL_TIME | Etw::PROCESS_TRACE_MODE_EVENT_RECORD; + trace.0.LoggerName = PWSTR(session_name.as_ptr() as *mut _); + trace.0.Anonymous1.ProcessTraceMode = + Etw::PROCESS_TRACE_MODE_REAL_TIME | Etw::PROCESS_TRACE_MODE_EVENT_RECORD; let mut cb: &mut dyn FnMut(&EventRecord) = &mut callback; trace.0.Context = unsafe { std::mem::transmute(&mut cb) }; trace.0.Anonymous2.EventRecordCallback = Some(trace_callback_thunk); let session_handle = unsafe { Etw::OpenTraceW(&mut *trace) }; if session_handle.Value == INVALID_TRACE_HANDLE { - println!("{:?} {:?}", unsafe { GetLastError() }, windows::core::Error::from_win32()); + println!( + "{:?} {:?}", + unsafe { GetLastError() }, + windows::core::Error::from_win32() + ); panic!("Invalid handle"); } @@ -279,7 +320,12 @@ pub fn start_trace(mut callback: F) { println!("status: {:?}", status); } -pub fn write_property(output: &mut dyn std::fmt::Write, parser: &mut Parser, property: &Property, write_types: bool) { +pub fn write_property( + output: &mut dyn std::fmt::Write, + parser: &mut Parser, + property: &Property, + write_types: bool, +) { if write_types { let type_name = if let PropertyDesc::Primitive(prim) = &property.desc { format!("{:?}", prim.in_type) @@ -289,17 +335,31 @@ pub fn write_property(output: &mut dyn std::fmt::Write, parser: &mut Parser, pro if property.flags.is_empty() { write!(output, " {}: {} = ", property.name, type_name).unwrap(); } else { - write!(output, " {}({:?}): {} = ", property.name, property.flags, type_name).unwrap(); + write!( + output, + " {}({:?}): {} = ", + property.name, property.flags, type_name + ) + .unwrap(); } } else { write!(output, " {}= ", property.name).unwrap(); } if let Some(map_info) = &property.map_info { let value = match property.desc { - PropertyDesc::Primitive(PrimitiveDesc{ in_type: TdhInType::InTypeUInt32, ..}) => TryParse::::parse(parser, &property.name), - PropertyDesc::Primitive(PrimitiveDesc{ in_type: TdhInType::InTypeUInt16, ..}) => TryParse::::parse(parser, &property.name) as u32, - PropertyDesc::Primitive(PrimitiveDesc{ in_type: TdhInType::InTypeUInt8, ..}) => TryParse::::parse(parser, &property.name) as u32, - _ => panic!("{:?}", property.desc) + PropertyDesc::Primitive(PrimitiveDesc { + in_type: TdhInType::InTypeUInt32, + .. + }) => TryParse::::parse(parser, &property.name), + PropertyDesc::Primitive(PrimitiveDesc { + in_type: TdhInType::InTypeUInt16, + .. + }) => TryParse::::parse(parser, &property.name) as u32, + PropertyDesc::Primitive(PrimitiveDesc { + in_type: TdhInType::InTypeUInt8, + .. + }) => TryParse::::parse(parser, &property.name) as u32, + _ => panic!("{:?}", property.desc), }; if map_info.is_bitmap { let mut remaining_bits_str = String::new(); @@ -318,22 +378,49 @@ pub fn write_property(output: &mut dyn std::fmt::Write, parser: &mut Parser, pro } write!(output, "{}", matches.join(" | ")).unwrap(); } else { - write!(output, "{}", map_info.map.get(&value).map(|x| Cow::from(x)).unwrap_or_else(|| Cow::from(format!("Unknown: {}", value)))).unwrap(); + write!( + output, + "{}", + map_info + .map + .get(&value) + .map(|x| Cow::from(x)) + .unwrap_or_else(|| Cow::from(format!("Unknown: {}", value))) + ) + .unwrap(); } } else { let value = match &property.desc { PropertyDesc::Primitive(desc) => { // XXX: we should be using the out_type here instead of in_type match desc.in_type { - TdhInType::InTypeUnicodeString => TryParse::::try_parse(parser, &property.name), - TdhInType::InTypeAnsiString => TryParse::::try_parse(parser, &property.name), - TdhInType::InTypeBoolean => TryParse::::try_parse(parser, &property.name).map(|x| x.to_string()), - TdhInType::InTypeHexInt32 => TryParse::::try_parse(parser, &property.name).map(|x| x.to_string()), - TdhInType::InTypeUInt32 => TryParse::::try_parse(parser, &property.name).map(|x| x.to_string()), - TdhInType::InTypeUInt16 => TryParse::::try_parse(parser, &property.name).map(|x| x.to_string()), - TdhInType::InTypeUInt8 => TryParse::::try_parse(parser, &property.name).map(|x| x.to_string()), - TdhInType::InTypeInt8 => TryParse::::try_parse(parser, &property.name).map(|x| x.to_string()), - TdhInType::InTypeInt64 => TryParse::::try_parse(parser, &property.name).map(|x| x.to_string()), + TdhInType::InTypeUnicodeString => { + TryParse::::try_parse(parser, &property.name) + } + TdhInType::InTypeAnsiString => { + TryParse::::try_parse(parser, &property.name) + } + TdhInType::InTypeBoolean => { + TryParse::::try_parse(parser, &property.name).map(|x| x.to_string()) + } + TdhInType::InTypeHexInt32 => { + TryParse::::try_parse(parser, &property.name).map(|x| x.to_string()) + } + TdhInType::InTypeUInt32 => { + TryParse::::try_parse(parser, &property.name).map(|x| x.to_string()) + } + TdhInType::InTypeUInt16 => { + TryParse::::try_parse(parser, &property.name).map(|x| x.to_string()) + } + TdhInType::InTypeUInt8 => { + TryParse::::try_parse(parser, &property.name).map(|x| x.to_string()) + } + TdhInType::InTypeInt8 => { + TryParse::::try_parse(parser, &property.name).map(|x| x.to_string()) + } + TdhInType::InTypeInt64 => { + TryParse::::try_parse(parser, &property.name).map(|x| x.to_string()) + } TdhInType::InTypeUInt64 => { let i = TryParse::::try_parse(parser, &property.name); if desc.out_type == TdhOutType::OutTypeHexInt64 { @@ -341,29 +428,41 @@ pub fn write_property(output: &mut dyn std::fmt::Write, parser: &mut Parser, pro } else { i.map(|x| x.to_string()) } - }, + } TdhInType::InTypeHexInt64 => { let i = TryParse::::try_parse(parser, &property.name); i.map(|x| format!("0x{:x}", x)) - }, - TdhInType::InTypePointer | TdhInType::InTypeSizeT => TryParse::::try_parse(parser, &property.name).map(|x| format!("0x{:x}", x)), - TdhInType::InTypeGuid => TryParse::::try_parse(parser, &property.name).map(|x| format!("{:?}", x)), + } + TdhInType::InTypePointer | TdhInType::InTypeSizeT => { + TryParse::::try_parse(parser, &property.name) + .map(|x| format!("0x{:x}", x)) + } + TdhInType::InTypeGuid => TryParse::::try_parse(parser, &property.name) + .map(|x| format!("{:?}", x)), TdhInType::InTypeInt32 => { TryParse::::try_parse(parser, &property.name).map(|x| x.to_string()) } TdhInType::InTypeFloat => { TryParse::::try_parse(parser, &property.name).map(|x| x.to_string()) } - _ => Ok(format!("Unknown {:?} -> {:?}", desc.in_type, desc.out_type)) + _ => Ok(format!("Unknown {:?} -> {:?}", desc.in_type, desc.out_type)), } } - PropertyDesc::Struct(desc) => Ok(format!("unhandled struct {} {}", desc.start_index, desc.num_members)), + PropertyDesc::Struct(desc) => Ok(format!( + "unhandled struct {} {}", + desc.start_index, desc.num_members + )), }; let value = match value { Ok(value) => value, Err(ParserError::InvalidType) => format!("invalid type {:?}", property.desc), - Err(ParserError::LengthMismatch) => format!("Err(LengthMismatch) type: {:?}, flags: {:?}, buf: {}", property.desc, property.flags, parser.buffer.len()), - Err(e) => format!("Err({:?}) type: {:?}", e, property.desc) + Err(ParserError::LengthMismatch) => format!( + "Err(LengthMismatch) type: {:?}, flags: {:?}, buf: {}", + property.desc, + property.flags, + parser.buffer.len() + ), + Err(e) => format!("Err({:?}) type: {:?}", e, property.desc), }; write!(output, "{}", value).unwrap(); } @@ -375,21 +474,21 @@ pub fn print_property(parser: &mut Parser, property: &Property, write_types: boo println!("{}", result); } - pub fn add_custom_schemas(locator: &mut SchemaLocator) { - locator.add_custom_schema(Box::new(custom_schemas::ImageID{})); - locator.add_custom_schema(Box::new(custom_schemas::DbgID{})); - locator.add_custom_schema(Box::new(custom_schemas::EventInfo{})); - locator.add_custom_schema(Box::new(custom_schemas::ThreadStart{})); - locator.add_custom_schema(Box::new(custom_schemas::D3DUmdLogging_MapAllocation{})); - locator.add_custom_schema(Box::new(custom_schemas::D3DUmdLogging_RundownAllocation{})); - locator.add_custom_schema(Box::new(custom_schemas::D3DUmdLogging_UnmapAllocation{})); + locator.add_custom_schema(Box::new(custom_schemas::ImageID {})); + locator.add_custom_schema(Box::new(custom_schemas::DbgID {})); + locator.add_custom_schema(Box::new(custom_schemas::EventInfo {})); + locator.add_custom_schema(Box::new(custom_schemas::ThreadStart {})); + locator.add_custom_schema(Box::new(custom_schemas::D3DUmdLogging_MapAllocation {})); + locator.add_custom_schema(Box::new(custom_schemas::D3DUmdLogging_RundownAllocation {})); + locator.add_custom_schema(Box::new(custom_schemas::D3DUmdLogging_UnmapAllocation {})); } pub fn enumerate_trace_guids() { let mut count = 1; loop { - let mut guids: Vec = vec![unsafe { std::mem::zeroed() }; count as usize]; + let mut guids: Vec = + vec![unsafe { std::mem::zeroed() }; count as usize]; let mut ptrs: Vec<*mut TRACE_GUID_PROPERTIES> = Vec::new(); for guid in &mut guids { ptrs.push(guid) @@ -416,28 +515,48 @@ pub fn enumerate_trace_guids_ex(print_instances: bool) { let mut required_size: u32 = 0; loop { - let mut guids: Vec = vec![GUID::zeroed(); required_size as usize/mem::size_of::()]; + let mut guids: Vec = + vec![GUID::zeroed(); required_size as usize / mem::size_of::()]; let size = (guids.len() * mem::size_of::()) as u32; println!("get {}", required_size); - let result = unsafe { EnumerateTraceGuidsEx(TraceGuidQueryList, None, 0, Some(guids.as_mut_ptr() as *mut _), size, &mut required_size as *mut _) }; + let result = unsafe { + EnumerateTraceGuidsEx( + TraceGuidQueryList, + None, + 0, + Some(guids.as_mut_ptr() as *mut _), + size, + &mut required_size as *mut _, + ) + }; match result { Ok(()) => { for guid in guids.iter() { - println!("{:?}", guid); let info = get_provider_info(guid); - let instance_count= unsafe { *(info.as_ptr() as *const TRACE_GUID_INFO) }.InstanceCount; - let mut instance_ptr: *const TRACE_PROVIDER_INSTANCE_INFO = unsafe { (info.as_ptr().add(mem::size_of::()) as *const TRACE_PROVIDER_INSTANCE_INFO) }; + let instance_count = + unsafe { *(info.as_ptr() as *const TRACE_GUID_INFO) }.InstanceCount; + let mut instance_ptr: *const TRACE_PROVIDER_INSTANCE_INFO = unsafe { + (info.as_ptr().add(mem::size_of::()) + as *const TRACE_PROVIDER_INSTANCE_INFO) + }; for _ in 0..instance_count { let instance = unsafe { &*instance_ptr }; - if print_instances { - println!("enable_count {}, pid {}, flags {}", instance.EnableCount, instance.Pid, instance.Flags, ) + if print_instances { + println!( + "enable_count {}, pid {}, flags {}", + instance.EnableCount, instance.Pid, instance.Flags, + ) } - instance_ptr = unsafe {((instance_ptr as *const TRACE_PROVIDER_INSTANCE_INFO as *const u8).add(instance.NextOffset as usize) as *const TRACE_PROVIDER_INSTANCE_INFO)}; - } + instance_ptr = unsafe { + ((instance_ptr as *const TRACE_PROVIDER_INSTANCE_INFO as *const u8) + .add(instance.NextOffset as usize) + as *const TRACE_PROVIDER_INSTANCE_INFO) + }; + } } break; } @@ -451,7 +570,6 @@ pub fn enumerate_trace_guids_ex(print_instances: bool) { } } - pub fn get_provider_info(guid: &GUID) -> Vec { let mut required_size: u32 = 0; @@ -460,10 +578,18 @@ pub fn get_provider_info(guid: &GUID) -> Vec { let size = info.len() as u32; - let result = unsafe { EnumerateTraceGuidsEx(TraceGuidQueryInfo, Some(guid as *const GUID as *const _), mem::size_of::() as u32, Some(info.as_mut_ptr() as *mut _), size, &mut required_size as *mut _) }; + let result = unsafe { + EnumerateTraceGuidsEx( + TraceGuidQueryInfo, + Some(guid as *const GUID as *const _), + mem::size_of::() as u32, + Some(info.as_mut_ptr() as *mut _), + size, + &mut required_size as *mut _, + ) + }; match result { Ok(()) => { - return info; } Err(e) => { @@ -473,4 +599,4 @@ pub fn get_provider_info(guid: &GUID) -> Vec { } } } -} \ No newline at end of file +} From de8ed17f5bbf124226a6b268ed55155bbf92c961 Mon Sep 17 00:00:00 2001 From: Vladimir Vukicevic Date: Sat, 4 May 2024 13:05:57 -0700 Subject: [PATCH 06/10] event_properties_to_string --- etw-reader/src/lib.rs | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/etw-reader/src/lib.rs b/etw-reader/src/lib.rs index f55ae2d7..0517fdb6 100644 --- a/etw-reader/src/lib.rs +++ b/etw-reader/src/lib.rs @@ -320,6 +320,27 @@ pub fn start_trace(mut callback: F) { println!("status: {:?}", status); } +pub fn event_properties_to_string( + s: &schema::TypedEvent, + parser: &mut Parser, + skip_properties: Option<&[&str]>, +) -> String { + let mut text = String::new(); + for i in 0..s.property_count() { + let property = s.property(i); + if let Some(propfilter) = skip_properties { + if propfilter.iter().any(|&s| s == property.name) { + continue; + } + } + + write_property(&mut text, parser, &property, false); + text += ", " + } + + text +} + pub fn write_property( output: &mut dyn std::fmt::Write, parser: &mut Parser, From 5648cbd8d10f6af3ec297a2590f13d6f01e664f1 Mon Sep 17 00:00:00 2001 From: Vladimir Vukicevic Date: Sat, 4 May 2024 13:06:25 -0700 Subject: [PATCH 07/10] samply fix --- samply/src/windows/etw_gecko.rs | 6 +++--- samply/src/windows/mod.rs | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/samply/src/windows/etw_gecko.rs b/samply/src/windows/etw_gecko.rs index a3734e61..035021ed 100644 --- a/samply/src/windows/etw_gecko.rs +++ b/samply/src/windows/etw_gecko.rs @@ -15,9 +15,9 @@ use fxprof_processed_profile::{ use serde_json::{json, Value}; use uuid::Uuid; -use super::etw_reader::parser::{Address, Parser, TryParse}; -use super::etw_reader::schema::SchemaLocator; -use super::etw_reader::{ +use etw_reader::parser::{Address, Parser, TryParse}; +use etw_reader::schema::SchemaLocator; +use etw_reader::{ add_custom_schemas, event_properties_to_string, open_trace, print_property, GUID, }; use super::profile_context::ProfileContext; diff --git a/samply/src/windows/mod.rs b/samply/src/windows/mod.rs index 47396a69..b0bcfd76 100644 --- a/samply/src/windows/mod.rs +++ b/samply/src/windows/mod.rs @@ -1,5 +1,4 @@ mod etw_gecko; -mod etw_reader; pub mod import; mod profile_context; pub mod profiler; From baa5575237b44b6fffa0e0b23cdf59a4dc592d40 Mon Sep 17 00:00:00 2001 From: Vladimir Vukicevic Date: Sat, 4 May 2024 13:06:44 -0700 Subject: [PATCH 08/10] Use local etw-reader --- samply/Cargo.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/samply/Cargo.toml b/samply/Cargo.toml index fc7ee460..b8bd0429 100644 --- a/samply/Cargo.toml +++ b/samply/Cargo.toml @@ -76,6 +76,8 @@ num-traits = "0.2" num-derive = "0.4" runas = "1.2.0" which = "6.0.1" +etw-reader = { path = "../etw-reader" } +# linux-perf-data = "0.10.1" [target.'cfg(windows)'.dependencies.windows] version = "0.56" From 9929ff5ec3a3a1c869516b46b6af7b296aa9551f Mon Sep 17 00:00:00 2001 From: Jeff Muizelaar Date: Sat, 4 May 2024 13:38:40 -0700 Subject: [PATCH 09/10] Match up ID3D11VideoContext_SubmitDecoderBuffers markers --- samply/src/windows/etw_gecko.rs | 32 ++++++++++++++++++++++++++- samply/src/windows/profile_context.rs | 8 +++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/samply/src/windows/etw_gecko.rs b/samply/src/windows/etw_gecko.rs index 035021ed..a6c92cbf 100644 --- a/samply/src/windows/etw_gecko.rs +++ b/samply/src/windows/etw_gecko.rs @@ -31,7 +31,7 @@ use crate::shared::process_sample_data::{ }; use crate::shared::timestamp_converter::TimestampConverter; use crate::shared::types::{StackFrame, StackMode}; -use crate::windows::profile_context::{PendingStack, ProcessJitInfo}; +use crate::windows::profile_context::{PendingStack, PendingMarker, ProcessJitInfo}; pub fn profile_pid_from_etl_file(context: &mut ProfileContext, etl_file: &Path) { let profile_start_instant = Timestamp::from_nanos_since_reference(0); @@ -570,6 +570,36 @@ pub fn profile_pid_from_etl_file(context: &mut ProfileContext, etl_file: &Path) //dbg!(s.process_id(), jscript_symbols.keys()); } + "Microsoft-Windows-Direct3D11/ID3D11VideoContext_SubmitDecoderBuffers/win:Start" => { + let mut parser = Parser::create(&s); + let thread_id = s.thread_id(); + let Some(mut thread) = context.get_thread_mut(thread_id) else { return }; + let text = event_properties_to_string(&s, &mut parser, None); + thread.pending_markers.insert(s.name().to_owned(), PendingMarker { text, start: timestamp }); + } + "Microsoft-Windows-Direct3D11/ID3D11VideoContext_SubmitDecoderBuffers/win:Stop" => { + let mut parser = Parser::create(&s); + let thread_id = s.thread_id(); + let Some(mut thread) = context.get_thread_mut(thread_id) else { return }; + + let mut text = event_properties_to_string(&s, &mut parser, None); + let timing = if let Some(pending) = thread.pending_markers.remove("Microsoft-Windows-Direct3D11/ID3D11VideoContext_SubmitDecoderBuffers/win:Start") { + text = pending.text; + MarkerTiming::Interval(pending.start, timestamp) + } else { + MarkerTiming::IntervalEnd(timestamp) + }; + + let category = match categories.entry(s.provider_name()) { + Entry::Occupied(e) => *e.get(), + Entry::Vacant(e) => { + let category = context.profile.borrow_mut().add_category(e.key(), CategoryColor::Transparent); + *e.insert(category) + } + }; + + context.profile.borrow_mut().add_marker(thread.handle, category, s.name().split_once("/").unwrap().1, SimpleMarker(text), timing); + } marker_name if marker_name.starts_with("Mozilla.FirefoxTraceLogger/") => { let Some(marker_name) = marker_name.strip_prefix("Mozilla.FirefoxTraceLogger/").and_then(|s| s.strip_suffix('/')) else { return }; diff --git a/samply/src/windows/profile_context.rs b/samply/src/windows/profile_context.rs index 7d1254af..493da6b1 100644 --- a/samply/src/windows/profile_context.rs +++ b/samply/src/windows/profile_context.rs @@ -44,6 +44,12 @@ pub struct ProcessJitInfo { pub symbols: Vec, } +#[derive(Debug)] +pub struct PendingMarker { + pub text: String, + pub start: Timestamp, +} + #[derive(Debug)] pub struct ThreadState { // When merging threads `handle` is the global thread handle and we use `merge_name` to store the name @@ -54,6 +60,7 @@ pub struct ThreadState { pub memory_usage: Option, pub thread_id: u32, pub process_id: u32, + pub pending_markers: HashMap, } impl ThreadState { @@ -63,6 +70,7 @@ impl ThreadState { merge_name: None, pending_stacks: VecDeque::new(), context_switch_data: Default::default(), + pending_markers: HashMap::new(), memory_usage: None, thread_id: tid, process_id: pid, From 31770468d88bf0cf76fbcb653254fbd61fd6f7e7 Mon Sep 17 00:00:00 2001 From: Jeff Muizelaar Date: Sat, 4 May 2024 13:39:34 -0700 Subject: [PATCH 10/10] Fix Chrome/Firefox marker regression caused by adding op names for tracelogging --- samply/src/windows/etw_gecko.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/samply/src/windows/etw_gecko.rs b/samply/src/windows/etw_gecko.rs index a6c92cbf..3a308b33 100644 --- a/samply/src/windows/etw_gecko.rs +++ b/samply/src/windows/etw_gecko.rs @@ -601,7 +601,7 @@ pub fn profile_pid_from_etl_file(context: &mut ProfileContext, etl_file: &Path) context.profile.borrow_mut().add_marker(thread.handle, category, s.name().split_once("/").unwrap().1, SimpleMarker(text), timing); } marker_name if marker_name.starts_with("Mozilla.FirefoxTraceLogger/") => { - let Some(marker_name) = marker_name.strip_prefix("Mozilla.FirefoxTraceLogger/").and_then(|s| s.strip_suffix('/')) else { return }; + let Some(marker_name) = marker_name.strip_prefix("Mozilla.FirefoxTraceLogger/").and_then(|s| s.strip_suffix("/Info")) else { return }; let thread_id = e.EventHeader.ThreadId; let Some(thread) = context.get_thread(thread_id) else { return }; @@ -652,7 +652,7 @@ pub fn profile_pid_from_etl_file(context: &mut ProfileContext, etl_file: &Path) } } marker_name if marker_name.starts_with("Google.Chrome/") => { - let Some(marker_name) = marker_name.strip_prefix("Google.Chrome/").and_then(|s| s.strip_suffix('/')) else { return }; + let Some(marker_name) = marker_name.strip_prefix("Google.Chrome/").and_then(|s| s.strip_suffix("/Info")) else { return }; // a bitfield of keywords bitflags! { #[derive(PartialEq, Eq)]